1 // SPDX-License-Identifier: GPL-2.0
3  * uprobes-based tracing events
5  * Copyright (C) IBM Corporation, 2010-2012
10 #include <linux/bpf-cgroup.h>
66 	struct trace_probe		tp;  member
71 	return ev->ops == &trace_uprobe_ops;  in is_trace_uprobe()
80  * for_each_trace_uprobe - iterate over the trace_uprobe list
98 	return addr - (n * sizeof(long));  in adjust_stack_addr()
121  * Uprobes-specific fetch functions
128 	return copy_from_user(dest, vaddr, size) ? -EFAULT : 0;  in probe_mem_read()
138  * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
151 		return -ENOMEM;  in fetch_store_string()
154 		ret = strlcpy(dst, current->comm, maxlen);  in fetch_store_string()
159 			dst[ret - 1] = '\0';  in fetch_store_string()
167 		*(u32 *)dest = make_data_loc(ret, (void *)dst - base);  in fetch_store_string()
169 		*(u32 *)dest = make_data_loc(0, (void *)dst - base);  in fetch_store_string()
180 /* Return the length of string -- including null terminal byte */
188 		len = strlen(current->comm) + 1;  in fetch_store_strlen()
206 	udd = (void *) current->utask->vaddr;  in translate_user_vaddr()
208 	base_addr = udd->bp_addr - udd->tu->offset;  in translate_user_vaddr()
222 	switch (code->op) {  in process_fetch_insn()
224 		val = regs_get_register(regs, code->param);  in process_fetch_insn()
227 		val = get_user_stack_nth(regs, code->param);  in process_fetch_insn()
239 		val = translate_user_vaddr(code->immediate);  in process_fetch_insn()
254 	rwlock_init(&filter->rwlock);  in NOKPROBE_SYMBOL()
255 	filter->nr_systemwide = 0;  in NOKPROBE_SYMBOL()
256 	INIT_LIST_HEAD(&filter->perf_events);  in NOKPROBE_SYMBOL()
261 	return !filter->nr_systemwide && list_empty(&filter->perf_events);  in uprobe_filter_is_empty()
266 	return tu->consumer.ret_handler != NULL;  in is_ret_probe()
273 	return trace_probe_is_enabled(&tu->tp);  in trace_uprobe_is_busy()
285 	len = strlen(tu->filename);  in trace_uprobe_match_command_head()
286 	if (strncmp(tu->filename, argv[0], len) || argv[0][len] != ':')  in trace_uprobe_match_command_head()
289 	if (tu->ref_ctr_offset == 0)  in trace_uprobe_match_command_head()
291 				(int)(sizeof(void *) * 2), tu->offset);  in trace_uprobe_match_command_head()
294 				(int)(sizeof(void *) * 2), tu->offset,  in trace_uprobe_match_command_head()
295 				tu->ref_ctr_offset);  in trace_uprobe_match_command_head()
299 	argc--; argv++;  in trace_uprobe_match_command_head()
301 	return trace_probe_match_command_args(&tu->tp, argc, argv);  in trace_uprobe_match_command_head()
310 		strcmp(trace_probe_name(&tu->tp), event) == 0) &&  in trace_uprobe_match()
311 	   (!system || strcmp(trace_probe_group_name(&tu->tp), system) == 0) &&  in trace_uprobe_match()
318 	struct trace_probe *tp;  in trace_uprobe_primary_from_call()  local
320 	tp = trace_probe_primary_from_call(call);  in trace_uprobe_primary_from_call()
321 	if (WARN_ON_ONCE(!tp))  in trace_uprobe_primary_from_call()
324 	return container_of(tp, struct trace_uprobe, tp);  in trace_uprobe_primary_from_call()
336 	tu = kzalloc(struct_size(tu, tp.args, nargs), GFP_KERNEL);  in alloc_trace_uprobe()
338 		return ERR_PTR(-ENOMEM);  in alloc_trace_uprobe()
340 	ret = trace_probe_init(&tu->tp, event, group, true, nargs);  in alloc_trace_uprobe()
344 	dyn_event_init(&tu->devent, &trace_uprobe_ops);  in alloc_trace_uprobe()
345 	tu->consumer.handler = uprobe_dispatcher;  in alloc_trace_uprobe()
347 		tu->consumer.ret_handler = uretprobe_dispatcher;  in alloc_trace_uprobe()
348 	init_trace_uprobe_filter(tu->tp.event->filter);  in alloc_trace_uprobe()
362 	path_put(&tu->path);  in free_trace_uprobe()
363 	trace_probe_cleanup(&tu->tp);  in free_trace_uprobe()
364 	kfree(tu->filename);  in free_trace_uprobe()
374 		if (strcmp(trace_probe_name(&tu->tp), event) == 0 &&  in find_probe_event()
375 		    strcmp(trace_probe_group_name(&tu->tp), group) == 0)  in find_probe_event()
386 	if (trace_probe_has_sibling(&tu->tp))  in unregister_trace_uprobe()
390 	if (trace_event_dyn_busy(trace_probe_event_call(&tu->tp)))  in unregister_trace_uprobe()
391 		return -EBUSY;  in unregister_trace_uprobe()
398 	dyn_event_remove(&tu->devent);  in unregister_trace_uprobe()
399 	trace_probe_unlink(&tu->tp);  in unregister_trace_uprobe()
407 	struct trace_probe_event *tpe = orig->tp.event;  in trace_uprobe_has_same_uprobe()
408 	struct inode *comp_inode = d_real_inode(comp->path.dentry);  in trace_uprobe_has_same_uprobe()
411 	list_for_each_entry(orig, &tpe->probes, tp.list) {  in trace_uprobe_has_same_uprobe()
412 		if (comp_inode != d_real_inode(orig->path.dentry) ||  in trace_uprobe_has_same_uprobe()
413 		    comp->offset != orig->offset)  in trace_uprobe_has_same_uprobe()
420 		for (i = 0; i < orig->tp.nr_args; i++) {  in trace_uprobe_has_same_uprobe()
421 			if (strcmp(orig->tp.args[i].comm,  in trace_uprobe_has_same_uprobe()
422 				   comp->tp.args[i].comm))  in trace_uprobe_has_same_uprobe()
426 		if (i == orig->tp.nr_args)  in trace_uprobe_has_same_uprobe()
437 	ret = trace_probe_compare_arg_type(&tu->tp, &to->tp);  in append_trace_uprobe()
442 		return -EEXIST;  in append_trace_uprobe()
447 		return -EEXIST;  in append_trace_uprobe()
451 	ret = trace_probe_append(&tu->tp, &to->tp);  in append_trace_uprobe()
453 		dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp));  in append_trace_uprobe()
471 	struct inode *new_inode = d_real_inode(new->path.dentry);  in validate_ref_ctr_offset()
474 		if (new_inode == d_real_inode(tmp->path.dentry) &&  in validate_ref_ctr_offset()
475 		    new->offset == tmp->offset &&  in validate_ref_ctr_offset()
476 		    new->ref_ctr_offset != tmp->ref_ctr_offset) {  in validate_ref_ctr_offset()
478 			return -EINVAL;  in validate_ref_ctr_offset()
497 	old_tu = find_probe_event(trace_probe_name(&tu->tp),  in register_trace_uprobe()
498 				  trace_probe_group_name(&tu->tp));  in register_trace_uprobe()
503 			ret = -EEXIST;  in register_trace_uprobe()
512 		if (ret == -EEXIST) {  in register_trace_uprobe()
520 	dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp));  in register_trace_uprobe()
530  *  - Add uprobe: p|r[:[GRP/][EVENT]] PATH:OFFSET[%return][(REF)] [FETCHARGS]
554 		return -ECANCELED;  in __trace_uprobe_create()
558 		return -ECANCELED;  in __trace_uprobe_create()
559 	if (argc - 2 > MAX_TRACE_ARGS)  in __trace_uprobe_create()
560 		return -E2BIG;  in __trace_uprobe_create()
566 		return -ECANCELED;  in __trace_uprobe_create()
570 		return -ENOMEM;  in __trace_uprobe_create()
576 		return -ECANCELED;  in __trace_uprobe_create()
592 		ret = -EINVAL;  in __trace_uprobe_create()
601 			ret = -EINVAL;  in __trace_uprobe_create()
603 			trace_probe_log_err(rctr_end - filename,  in __trace_uprobe_create()
607 			ret = -EINVAL;  in __trace_uprobe_create()
608 			trace_probe_log_err(rctr_end + 1 - filename,  in __trace_uprobe_create()
617 			trace_probe_log_err(rctr - filename, BAD_REFCNT);  in __trace_uprobe_create()
629 			trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX);  in __trace_uprobe_create()
630 			ret = -EINVAL;  in __trace_uprobe_create()
638 		trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS);  in __trace_uprobe_create()
646 						  event - argv[0]);  in __trace_uprobe_create()
657 			ret = -ENOMEM;  in __trace_uprobe_create()
661 		ptr = strpbrk(tail, ".-_");  in __trace_uprobe_create()
670 	argc -= 2;  in __trace_uprobe_create()
676 		/* This must return -ENOMEM otherwise there is a bug */  in __trace_uprobe_create()
677 		WARN_ON_ONCE(ret != -ENOMEM);  in __trace_uprobe_create()
680 	tu->offset = offset;  in __trace_uprobe_create()
681 	tu->ref_ctr_offset = ref_ctr_offset;  in __trace_uprobe_create()
682 	tu->path = path;  in __trace_uprobe_create()
683 	tu->filename = filename;  in __trace_uprobe_create()
692 		ret = traceprobe_parse_probe_arg(&tu->tp, i, argv[i], &ctx);  in __trace_uprobe_create()
699 	ret = traceprobe_set_print_fmt(&tu->tp, ptype);  in __trace_uprobe_create()
730 	if (raw_command[0] == '-')  in create_or_delete_trace_uprobe()
734 	return ret == -ECANCELED ? -EINVAL : ret;  in create_or_delete_trace_uprobe()
751 	seq_printf(m, "%c:%s/%s %s:0x%0*lx", c, trace_probe_group_name(&tu->tp),  in trace_uprobe_show()
752 			trace_probe_name(&tu->tp), tu->filename,  in trace_uprobe_show()
753 			(int)(sizeof(void *) * 2), tu->offset);  in trace_uprobe_show()
755 	if (tu->ref_ctr_offset)  in trace_uprobe_show()
756 		seq_printf(m, "(0x%lx)", tu->ref_ctr_offset);  in trace_uprobe_show()
758 	for (i = 0; i < tu->tp.nr_args; i++)  in trace_uprobe_show()
759 		seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm);  in trace_uprobe_show()
790 	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {  in probes_open()
825 	seq_printf(m, "  %s %-44s %15lu\n", tu->filename,  in probes_profile_seq_show()
826 			trace_probe_name(&tu->tp), tu->nhit);  in probes_profile_seq_show()
871 		return -ENOMEM;  in uprobe_buffer_init()
880 		per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf = page_address(p);  in uprobe_buffer_init()
881 		mutex_init(&per_cpu_ptr(uprobe_cpu_buffer, cpu)->mutex);  in uprobe_buffer_init()
890 		free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf);  in uprobe_buffer_init()
894 	return -ENOMEM;  in uprobe_buffer_init()
906 			uprobe_buffer_refcnt--;  in uprobe_buffer_enable()
918 	if (--uprobe_buffer_refcnt == 0) {  in uprobe_buffer_disable()
921 							     cpu)->buf);  in uprobe_buffer_disable()
937 	 * Use per-cpu buffers for fastest access, but we might migrate  in uprobe_buffer_get()
940 	mutex_lock(&ucb->mutex);  in uprobe_buffer_get()
949 	mutex_unlock(&ucb->mutex);  in uprobe_buffer_put()
963 	dsize = __get_data_size(&tu->tp, regs, NULL);  in prepare_uprobe_buffer()
966 	ucb->dsize = tu->tp.size + dsize;  in prepare_uprobe_buffer()
968 	if (WARN_ON_ONCE(ucb->dsize > MAX_UCB_BUFFER_SIZE)) {  in prepare_uprobe_buffer()
969 		ucb->dsize = MAX_UCB_BUFFER_SIZE;  in prepare_uprobe_buffer()
970 		dsize = MAX_UCB_BUFFER_SIZE - tu->tp.size;  in prepare_uprobe_buffer()
973 	store_trace_args(ucb->buf, &tu->tp, regs, NULL, esize, dsize);  in prepare_uprobe_buffer()
988 	struct trace_event_call *call = trace_probe_event_call(&tu->tp);  in __uprobe_trace_func()
990 	WARN_ON(call != trace_file->event_call);  in __uprobe_trace_func()
996 	size = esize + ucb->dsize;  in __uprobe_trace_func()
1002 		entry->vaddr[0] = func;  in __uprobe_trace_func()
1003 		entry->vaddr[1] = instruction_pointer(regs);  in __uprobe_trace_func()
1006 		entry->vaddr[0] = instruction_pointer(regs);  in __uprobe_trace_func()
1010 	memcpy(data, ucb->buf, ucb->dsize);  in __uprobe_trace_func()
1019 	struct event_file_link *link;  in uprobe_trace_func()  local
1028 	trace_probe_for_each_link_rcu(link, &tu->tp)  in uprobe_trace_func()
1029 		__uprobe_trace_func(tu, 0, regs, ucb, link->file);  in uprobe_trace_func()
1039 	struct event_file_link *link;  in uretprobe_trace_func()  local
1045 	trace_probe_for_each_link_rcu(link, &tu->tp)  in uretprobe_trace_func()
1046 		__uprobe_trace_func(tu, func, regs, ucb, link->file);  in uretprobe_trace_func()
1055 	struct trace_seq *s = &iter->seq;  in print_uprobe_event()
1059 	entry = (struct uprobe_trace_entry_head *)iter->ent;  in print_uprobe_event()
1066 		trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)",  in print_uprobe_event()
1067 				 trace_probe_name(&tu->tp),  in print_uprobe_event()
1068 				 entry->vaddr[1], entry->vaddr[0]);  in print_uprobe_event()
1072 				 trace_probe_name(&tu->tp),  in print_uprobe_event()
1073 				 entry->vaddr[0]);  in print_uprobe_event()
1077 	if (trace_probe_print_args(s, tu->tp.args, tu->tp.nr_args, data, entry) < 0)  in print_uprobe_event()
1094 	tu->consumer.filter = filter;  in trace_uprobe_enable()
1095 	tu->inode = d_real_inode(tu->path.dentry);  in trace_uprobe_enable()
1097 	if (tu->ref_ctr_offset)  in trace_uprobe_enable()
1098 		ret = uprobe_register_refctr(tu->inode, tu->offset,  in trace_uprobe_enable()
1099 				tu->ref_ctr_offset, &tu->consumer);  in trace_uprobe_enable()
1101 		ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);  in trace_uprobe_enable()
1104 		tu->inode = NULL;  in trace_uprobe_enable()
1109 static void __probe_event_disable(struct trace_probe *tp)  in __probe_event_disable()  argument
1113 	tu = container_of(tp, struct trace_uprobe, tp);  in __probe_event_disable()
1114 	WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));  in __probe_event_disable()
1116 	list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {  in __probe_event_disable()
1117 		if (!tu->inode)  in __probe_event_disable()
1120 		uprobe_unregister(tu->inode, tu->offset, &tu->consumer);  in __probe_event_disable()
1121 		tu->inode = NULL;  in __probe_event_disable()
1128 	struct trace_probe *tp;  in probe_event_enable()  local
1133 	tp = trace_probe_primary_from_call(call);  in probe_event_enable()
1134 	if (WARN_ON_ONCE(!tp))  in probe_event_enable()
1135 		return -ENODEV;  in probe_event_enable()
1136 	enabled = trace_probe_is_enabled(tp);  in probe_event_enable()
1140 		if (trace_probe_test_flag(tp, TP_FLAG_PROFILE))  in probe_event_enable()
1141 			return -EINTR;  in probe_event_enable()
1143 		ret = trace_probe_add_file(tp, file);  in probe_event_enable()
1147 		if (trace_probe_test_flag(tp, TP_FLAG_TRACE))  in probe_event_enable()
1148 			return -EINTR;  in probe_event_enable()
1150 		trace_probe_set_flag(tp, TP_FLAG_PROFILE);  in probe_event_enable()
1153 	tu = container_of(tp, struct trace_uprobe, tp);  in probe_event_enable()
1154 	WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));  in probe_event_enable()
1163 	list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {  in probe_event_enable()
1166 			__probe_event_disable(tp);  in probe_event_enable()
1178 		trace_probe_remove_file(tp, file);  in probe_event_enable()
1180 		trace_probe_clear_flag(tp, TP_FLAG_PROFILE);  in probe_event_enable()
1188 	struct trace_probe *tp;  in probe_event_disable()  local
1190 	tp = trace_probe_primary_from_call(call);  in probe_event_disable()
1191 	if (WARN_ON_ONCE(!tp))  in probe_event_disable()
1194 	if (!trace_probe_is_enabled(tp))  in probe_event_disable()
1198 		if (trace_probe_remove_file(tp, file) < 0)  in probe_event_disable()
1201 		if (trace_probe_is_enabled(tp))  in probe_event_disable()
1204 		trace_probe_clear_flag(tp, TP_FLAG_PROFILE);  in probe_event_disable()
1206 	__probe_event_disable(tp);  in probe_event_disable()
1218 		return -ENODEV;  in uprobe_event_define_fields()
1229 	return traceprobe_define_arg_fields(event_call, size, &tu->tp);  in uprobe_event_define_fields()
1238 	if (filter->nr_systemwide)  in __uprobe_perf_filter()
1241 	list_for_each_entry(event, &filter->perf_events, hw.tp_list) {  in __uprobe_perf_filter()
1242 		if (event->hw.target->mm == mm)  in __uprobe_perf_filter()
1253 	return __uprobe_perf_filter(filter, event->hw.target->mm);  in trace_uprobe_filter_event()
1261 	write_lock(&filter->rwlock);  in trace_uprobe_filter_remove()
1262 	if (event->hw.target) {  in trace_uprobe_filter_remove()
1263 		list_del(&event->hw.tp_list);  in trace_uprobe_filter_remove()
1264 		done = filter->nr_systemwide ||  in trace_uprobe_filter_remove()
1265 			(event->hw.target->flags & PF_EXITING) ||  in trace_uprobe_filter_remove()
1268 		filter->nr_systemwide--;  in trace_uprobe_filter_remove()
1269 		done = filter->nr_systemwide;  in trace_uprobe_filter_remove()
1271 	write_unlock(&filter->rwlock);  in trace_uprobe_filter_remove()
1282 	write_lock(&filter->rwlock);  in trace_uprobe_filter_add()
1283 	if (event->hw.target) {  in trace_uprobe_filter_add()
1285 		 * event->parent != NULL means copy_process(), we can avoid  in trace_uprobe_filter_add()
1286 		 * uprobe_apply(). current->mm must be probed and we can rely  in trace_uprobe_filter_add()
1292 		done = filter->nr_systemwide ||  in trace_uprobe_filter_add()
1293 			event->parent || event->attr.enable_on_exec ||  in trace_uprobe_filter_add()
1295 		list_add(&event->hw.tp_list, &filter->perf_events);  in trace_uprobe_filter_add()
1297 		done = filter->nr_systemwide;  in trace_uprobe_filter_add()
1298 		filter->nr_systemwide++;  in trace_uprobe_filter_add()
1300 	write_unlock(&filter->rwlock);  in trace_uprobe_filter_add()
1308 	struct trace_probe *tp;  in uprobe_perf_close()  local
1312 	tp = trace_probe_primary_from_call(call);  in uprobe_perf_close()
1313 	if (WARN_ON_ONCE(!tp))  in uprobe_perf_close()
1314 		return -ENODEV;  in uprobe_perf_close()
1316 	tu = container_of(tp, struct trace_uprobe, tp);  in uprobe_perf_close()
1317 	if (trace_uprobe_filter_remove(tu->tp.event->filter, event))  in uprobe_perf_close()
1320 	list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {  in uprobe_perf_close()
1321 		ret = uprobe_apply(tu->inode, tu->offset, &tu->consumer, false);  in uprobe_perf_close()
1332 	struct trace_probe *tp;  in uprobe_perf_open()  local
1336 	tp = trace_probe_primary_from_call(call);  in uprobe_perf_open()
1337 	if (WARN_ON_ONCE(!tp))  in uprobe_perf_open()
1338 		return -ENODEV;  in uprobe_perf_open()
1340 	tu = container_of(tp, struct trace_uprobe, tp);  in uprobe_perf_open()
1341 	if (trace_uprobe_filter_add(tu->tp.event->filter, event))  in uprobe_perf_open()
1344 	list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {  in uprobe_perf_open()
1345 		err = uprobe_apply(tu->inode, tu->offset, &tu->consumer, true);  in uprobe_perf_open()
1363 	filter = tu->tp.event->filter;  in uprobe_perf_filter()
1365 	read_lock(&filter->rwlock);  in uprobe_perf_filter()
1367 	read_unlock(&filter->rwlock);  in uprobe_perf_filter()
1376 	struct trace_event_call *call = trace_probe_event_call(&tu->tp);  in __uprobe_perf_func()
1390 		array = rcu_dereference_check(call->prog_array, rcu_read_lock_trace_held());  in __uprobe_perf_func()
1401 	size = esize + ucb->dsize;  in __uprobe_perf_func()
1402 	size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32);  in __uprobe_perf_func()
1407 	head = this_cpu_ptr(call->perf_events);  in __uprobe_perf_func()
1416 		entry->vaddr[0] = func;  in __uprobe_perf_func()
1417 		entry->vaddr[1] = instruction_pointer(regs);  in __uprobe_perf_func()
1420 		entry->vaddr[0] = instruction_pointer(regs);  in __uprobe_perf_func()
1424 	memcpy(data, ucb->buf, ucb->dsize);  in __uprobe_perf_func()
1426 	if (size - esize > ucb->dsize)  in __uprobe_perf_func()
1427 		memset(data + ucb->dsize, 0, size - esize - ucb->dsize);  in __uprobe_perf_func()
1429 	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,  in __uprobe_perf_func()
1439 	if (!uprobe_perf_filter(&tu->consumer, 0, current->mm))  in uprobe_perf_func()
1458 	const char *pevent = trace_event_name(event->tp_event);  in bpf_get_uprobe_info()
1459 	const char *group = event->tp_event->class->system;  in bpf_get_uprobe_info()
1465 		tu = trace_uprobe_primary_from_call(event->tp_event);  in bpf_get_uprobe_info()
1467 		return -EINVAL;  in bpf_get_uprobe_info()
1471 	*filename = tu->filename;  in bpf_get_uprobe_info()
1472 	*probe_offset = tu->offset;  in bpf_get_uprobe_info()
1520 	tu->nhit++;  in uprobe_dispatcher()
1525 	current->utask->vaddr = (unsigned long) &udd;  in uprobe_dispatcher()
1530 	if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE))  in uprobe_dispatcher()
1534 	if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE))  in uprobe_dispatcher()
1553 	current->utask->vaddr = (unsigned long) &udd;  in uretprobe_dispatcher()
1558 	if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE))  in uretprobe_dispatcher()
1562 	if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE))  in uretprobe_dispatcher()
1581 	struct trace_event_call *call = trace_probe_event_call(&tu->tp);  in init_trace_event_call()
1582 	call->event.funcs = &uprobe_funcs;  in init_trace_event_call()
1583 	call->class->fields_array = uprobe_fields_array;  in init_trace_event_call()
1585 	call->flags = TRACE_EVENT_FL_UPROBE | TRACE_EVENT_FL_CAP_ANY;  in init_trace_event_call()
1586 	call->class->reg = trace_uprobe_register;  in init_trace_event_call()
1593 	return trace_probe_register_event_call(&tu->tp);  in register_uprobe_event()
1598 	return trace_probe_unregister_event_call(&tu->tp);  in unregister_uprobe_event()
1617 		return ERR_PTR(-EINVAL);  in create_local_trace_uprobe()
1635 	tu->offset = offs;  in create_local_trace_uprobe()
1636 	tu->path = path;  in create_local_trace_uprobe()
1637 	tu->ref_ctr_offset = ref_ctr_offset;  in create_local_trace_uprobe()
1638 	tu->filename = kstrdup(name, GFP_KERNEL);  in create_local_trace_uprobe()
1639 	if (!tu->filename) {  in create_local_trace_uprobe()
1640 		ret = -ENOMEM;  in create_local_trace_uprobe()
1647 	if (traceprobe_set_print_fmt(&tu->tp, ptype) < 0) {  in create_local_trace_uprobe()
1648 		ret = -ENOMEM;  in create_local_trace_uprobe()
1652 	return trace_probe_event_call(&tu->tp);  in create_local_trace_uprobe()