930 	struct uprobe_cpu_buffer *ucb;  in uprobe_buffer_get()  local
934 	ucb = per_cpu_ptr(uprobe_cpu_buffer, cpu);  in uprobe_buffer_get()
940 	mutex_lock(&ucb->mutex);  in uprobe_buffer_get()
942 	return ucb;  in uprobe_buffer_get()
945 static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb)  in uprobe_buffer_put()  argument
947 	if (!ucb)  in uprobe_buffer_put()
949 	mutex_unlock(&ucb->mutex);  in uprobe_buffer_put()
956 	struct uprobe_cpu_buffer *ucb;  in prepare_uprobe_buffer()  local
965 	ucb = uprobe_buffer_get();  in prepare_uprobe_buffer()
966 	ucb->dsize = tu->tp.size + dsize;  in prepare_uprobe_buffer()
968 	if (WARN_ON_ONCE(ucb->dsize > MAX_UCB_BUFFER_SIZE)) {  in prepare_uprobe_buffer()
969 		ucb->dsize = MAX_UCB_BUFFER_SIZE;  in prepare_uprobe_buffer()
973 	store_trace_args(ucb->buf, &tu->tp, regs, NULL, esize, dsize);  in prepare_uprobe_buffer()
975 	*ucbp = ucb;  in prepare_uprobe_buffer()
976 	return ucb;  in prepare_uprobe_buffer()
981 				struct uprobe_cpu_buffer *ucb,  in __uprobe_trace_func()  argument
996 	size = esize + ucb->dsize;  in __uprobe_trace_func()
1010 	memcpy(data, ucb->buf, ucb->dsize);  in __uprobe_trace_func()
1020 	struct uprobe_cpu_buffer *ucb;  in uprobe_trace_func()  local
1025 	ucb = prepare_uprobe_buffer(tu, regs, ucbp);  in uprobe_trace_func()
1029 		__uprobe_trace_func(tu, 0, regs, ucb, link->file);  in uprobe_trace_func()
1040 	struct uprobe_cpu_buffer *ucb;  in uretprobe_trace_func()  local
1042 	ucb = prepare_uprobe_buffer(tu, regs, ucbp);  in uretprobe_trace_func()
1046 		__uprobe_trace_func(tu, func, regs, ucb, link->file);  in uretprobe_trace_func()
1378 	struct uprobe_cpu_buffer *ucb;  in __uprobe_perf_func()  local
1400 	ucb = prepare_uprobe_buffer(tu, regs, ucbp);  in __uprobe_perf_func()
1401 	size = esize + ucb->dsize;  in __uprobe_perf_func()
1424 	memcpy(data, ucb->buf, ucb->dsize);  in __uprobe_perf_func()
1426 	if (size - esize > ucb->dsize)  in __uprobe_perf_func()
1427 		memset(data + ucb->dsize, 0, size - esize - ucb->dsize);  in __uprobe_perf_func()
1516 	struct uprobe_cpu_buffer *ucb = NULL;  in uprobe_dispatcher()  local
1531 		ret |= uprobe_trace_func(tu, regs, &ucb);  in uprobe_dispatcher()
1535 		ret |= uprobe_perf_func(tu, regs, &ucb);  in uprobe_dispatcher()
1537 	uprobe_buffer_put(ucb);  in uprobe_dispatcher()
1546 	struct uprobe_cpu_buffer *ucb = NULL;  in uretprobe_dispatcher()  local
1559 		uretprobe_trace_func(tu, func, regs, &ucb);  in uretprobe_dispatcher()
1563 		uretprobe_perf_func(tu, func, regs, &ucb);  in uretprobe_dispatcher()
1565 	uprobe_buffer_put(ucb);  in uretprobe_dispatcher()