Lines Matching +full:tp +full:- +full:link
1 // SPDX-License-Identifier: GPL-2.0
3 * Kprobes-based tracing events
10 #include <linux/bpf-cgroup.h>
15 #include <linux/error-injection.h>
63 struct trace_probe tp; member
68 return ev->ops == &trace_kprobe_ops; in is_trace_kprobe()
77 * for_each_trace_kprobe - iterate over the trace_kprobe list
87 return tk->rp.handler != NULL; in trace_kprobe_is_return()
92 return tk->symbol ? tk->symbol : "unknown"; in trace_kprobe_symbol()
97 return tk->rp.kp.offset; in trace_kprobe_offset()
102 return kprobe_gone(&tk->rp.kp); in trace_kprobe_has_gone()
119 if (!tk->symbol) in trace_kprobe_module_exist()
121 p = strchr(tk->symbol, ':'); in trace_kprobe_module_exist()
126 ret = !!find_module(tk->symbol); in trace_kprobe_module_exist()
137 return trace_probe_is_enabled(&tk->tp); in trace_kprobe_is_busy()
148 if (!tk->symbol) in trace_kprobe_match_command_head()
149 snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr); in trace_kprobe_match_command_head()
150 else if (tk->rp.kp.offset) in trace_kprobe_match_command_head()
152 trace_kprobe_symbol(tk), tk->rp.kp.offset); in trace_kprobe_match_command_head()
157 argc--; argv++; in trace_kprobe_match_command_head()
159 return trace_probe_match_command_args(&tk->tp, argc, argv); in trace_kprobe_match_command_head()
168 strcmp(trace_probe_name(&tk->tp), event) == 0) && in trace_kprobe_match()
169 (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0) && in trace_kprobe_match()
179 nhit += *per_cpu_ptr(tk->nhit, cpu); in trace_kprobe_nhit()
186 return !(list_empty(&tk->rp.kp.list) && in trace_kprobe_is_registered()
187 hlist_unhashed(&tk->rp.kp.hlist)); in trace_kprobe_is_registered()
196 if (tk->symbol) { in trace_kprobe_address()
200 addr += tk->rp.kp.offset; in trace_kprobe_address()
202 addr = (unsigned long)tk->rp.kp.addr; in trace_kprobe_address()
210 struct trace_probe *tp; in trace_kprobe_primary_from_call() local
212 tp = trace_probe_primary_from_call(call); in trace_kprobe_primary_from_call()
213 if (WARN_ON_ONCE(!tp)) in trace_kprobe_primary_from_call()
216 return container_of(tp, struct trace_kprobe, tp); in trace_kprobe_primary_from_call()
223 return tk ? (kprobe_on_func_entry(tk->rp.kp.addr, in trace_kprobe_on_func_entry()
224 tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name, in trace_kprobe_on_func_entry()
225 tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0) : false; in trace_kprobe_on_func_entry()
246 trace_probe_cleanup(&tk->tp); in free_trace_kprobe()
247 kfree(tk->symbol); in free_trace_kprobe()
248 free_percpu(tk->nhit); in free_trace_kprobe()
265 int ret = -ENOMEM; in alloc_trace_kprobe()
267 tk = kzalloc(struct_size(tk, tp.args, nargs), GFP_KERNEL); in alloc_trace_kprobe()
271 tk->nhit = alloc_percpu(unsigned long); in alloc_trace_kprobe()
272 if (!tk->nhit) in alloc_trace_kprobe()
276 tk->symbol = kstrdup(symbol, GFP_KERNEL); in alloc_trace_kprobe()
277 if (!tk->symbol) in alloc_trace_kprobe()
279 tk->rp.kp.symbol_name = tk->symbol; in alloc_trace_kprobe()
280 tk->rp.kp.offset = offs; in alloc_trace_kprobe()
282 tk->rp.kp.addr = addr; in alloc_trace_kprobe()
285 tk->rp.handler = kretprobe_dispatcher; in alloc_trace_kprobe()
287 tk->rp.kp.pre_handler = kprobe_dispatcher; in alloc_trace_kprobe()
289 tk->rp.maxactive = maxactive; in alloc_trace_kprobe()
290 INIT_HLIST_NODE(&tk->rp.kp.hlist); in alloc_trace_kprobe()
291 INIT_LIST_HEAD(&tk->rp.kp.list); in alloc_trace_kprobe()
293 ret = trace_probe_init(&tk->tp, event, group, false, nargs); in alloc_trace_kprobe()
297 dyn_event_init(&tk->devent, &trace_kprobe_ops); in alloc_trace_kprobe()
311 if (strcmp(trace_probe_name(&tk->tp), event) == 0 && in find_trace_kprobe()
312 strcmp(trace_probe_group_name(&tk->tp), group) == 0) in find_trace_kprobe()
323 ret = enable_kretprobe(&tk->rp); in __enable_trace_kprobe()
325 ret = enable_kprobe(&tk->rp.kp); in __enable_trace_kprobe()
331 static void __disable_trace_kprobe(struct trace_probe *tp) in __disable_trace_kprobe() argument
335 list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) { in __disable_trace_kprobe()
339 disable_kretprobe(&tk->rp); in __disable_trace_kprobe()
341 disable_kprobe(&tk->rp.kp); in __disable_trace_kprobe()
352 struct trace_probe *tp; in enable_trace_kprobe() local
357 tp = trace_probe_primary_from_call(call); in enable_trace_kprobe()
358 if (WARN_ON_ONCE(!tp)) in enable_trace_kprobe()
359 return -ENODEV; in enable_trace_kprobe()
360 enabled = trace_probe_is_enabled(tp); in enable_trace_kprobe()
364 ret = trace_probe_add_file(tp, file); in enable_trace_kprobe()
368 trace_probe_set_flag(tp, TP_FLAG_PROFILE); in enable_trace_kprobe()
373 list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) { in enable_trace_kprobe()
385 __disable_trace_kprobe(tp); in enable_trace_kprobe()
387 trace_probe_remove_file(tp, file); in enable_trace_kprobe()
389 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); in enable_trace_kprobe()
402 struct trace_probe *tp; in disable_trace_kprobe() local
404 tp = trace_probe_primary_from_call(call); in disable_trace_kprobe()
405 if (WARN_ON_ONCE(!tp)) in disable_trace_kprobe()
406 return -ENODEV; in disable_trace_kprobe()
409 if (!trace_probe_get_file_link(tp, file)) in disable_trace_kprobe()
410 return -ENOENT; in disable_trace_kprobe()
411 if (!trace_probe_has_single_file(tp)) in disable_trace_kprobe()
413 trace_probe_clear_flag(tp, TP_FLAG_TRACE); in disable_trace_kprobe()
415 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); in disable_trace_kprobe()
417 if (!trace_probe_is_enabled(tp)) in disable_trace_kprobe()
418 __disable_trace_kprobe(tp); in disable_trace_kprobe()
428 trace_probe_remove_file(tp, file); in disable_trace_kprobe()
443 addr -= offset; in __within_notrace_func()
449 return !ftrace_location_range(addr, addr + size - 1); in __within_notrace_func()
460 /* Check if the address is on a suffixed-symbol */ in within_notrace_func()
477 /* Internal register function - just handle k*probes and flags */
487 return -EINVAL; in __register_trace_kprobe()
492 return -EINVAL; in __register_trace_kprobe()
495 for (i = 0; i < tk->tp.nr_args; i++) { in __register_trace_kprobe()
496 ret = traceprobe_update_arg(&tk->tp.args[i]); in __register_trace_kprobe()
501 /* Set/clear disabled flag according to tp->flag */ in __register_trace_kprobe()
502 if (trace_probe_is_enabled(&tk->tp)) in __register_trace_kprobe()
503 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED; in __register_trace_kprobe()
505 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED; in __register_trace_kprobe()
508 ret = register_kretprobe(&tk->rp); in __register_trace_kprobe()
510 ret = register_kprobe(&tk->rp.kp); in __register_trace_kprobe()
515 /* Internal unregister function - just handle k*probes and flags */
520 unregister_kretprobe(&tk->rp); in __unregister_trace_kprobe()
522 unregister_kprobe(&tk->rp.kp); in __unregister_trace_kprobe()
524 INIT_HLIST_NODE(&tk->rp.kp.hlist); in __unregister_trace_kprobe()
525 INIT_LIST_HEAD(&tk->rp.kp.list); in __unregister_trace_kprobe()
526 if (tk->rp.kp.symbol_name) in __unregister_trace_kprobe()
527 tk->rp.kp.addr = NULL; in __unregister_trace_kprobe()
535 if (trace_probe_has_sibling(&tk->tp)) in unregister_trace_kprobe()
539 if (trace_probe_is_enabled(&tk->tp)) in unregister_trace_kprobe()
540 return -EBUSY; in unregister_trace_kprobe()
543 if (trace_event_dyn_busy(trace_probe_event_call(&tk->tp))) in unregister_trace_kprobe()
544 return -EBUSY; in unregister_trace_kprobe()
548 return -EBUSY; in unregister_trace_kprobe()
552 dyn_event_remove(&tk->devent); in unregister_trace_kprobe()
553 trace_probe_unlink(&tk->tp); in unregister_trace_kprobe()
561 struct trace_probe_event *tpe = orig->tp.event; in trace_kprobe_has_same_kprobe()
564 list_for_each_entry(orig, &tpe->probes, tp.list) { in trace_kprobe_has_same_kprobe()
574 for (i = 0; i < orig->tp.nr_args; i++) { in trace_kprobe_has_same_kprobe()
575 if (strcmp(orig->tp.args[i].comm, in trace_kprobe_has_same_kprobe()
576 comp->tp.args[i].comm)) in trace_kprobe_has_same_kprobe()
580 if (i == orig->tp.nr_args) in trace_kprobe_has_same_kprobe()
591 ret = trace_probe_compare_arg_type(&tk->tp, &to->tp); in append_trace_kprobe()
596 return -EEXIST; in append_trace_kprobe()
601 return -EEXIST; in append_trace_kprobe()
605 ret = trace_probe_append(&tk->tp, &to->tp); in append_trace_kprobe()
611 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { in append_trace_kprobe()
617 trace_probe_unlink(&tk->tp); in append_trace_kprobe()
619 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp)); in append_trace_kprobe()
632 old_tk = find_trace_kprobe(trace_probe_name(&tk->tp), in register_trace_kprobe()
633 trace_probe_group_name(&tk->tp)); in register_trace_kprobe()
638 ret = -EEXIST; in register_trace_kprobe()
648 if (ret == -EEXIST) { in register_trace_kprobe()
658 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { in register_trace_kprobe()
666 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp)); in register_trace_kprobe()
689 /* Don't need to check busy - this should have gone. */ in trace_kprobe_module_callback()
693 pr_warn("Failed to re-register probe %s on %s: %d\n", in trace_kprobe_module_callback()
694 trace_probe_name(&tk->tp), in trace_kprobe_module_callback()
726 if (strcmp(name, ctx->name) == 0) in count_mod_symbols()
727 ctx->count++; in count_mod_symbols()
750 * - Add kprobe: in __trace_kprobe_create()
752 * - Add kretprobe: in __trace_kprobe_create()
760 * $stackN : fetch Nth of stack (N:0-) in __trace_kprobe_create()
763 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) in __trace_kprobe_create()
766 * +|-offs(ARG) : fetch memory at ARG +|- offs address. in __trace_kprobe_create()
794 return -ECANCELED; in __trace_kprobe_create()
797 return -ECANCELED; in __trace_kprobe_create()
811 len = event - &argv[0][1] - 1; in __trace_kprobe_create()
814 if (len > MAX_EVENT_NAME_LEN - 1) { in __trace_kprobe_create()
840 ret = -ECANCELED; in __trace_kprobe_create()
846 return -ENOMEM; in __trace_kprobe_create()
854 trace_probe_log_err(tmp - symbol, BAD_ADDR_SUFFIX); in __trace_kprobe_create()
871 if (ret == -EINVAL && is_return) { in __trace_kprobe_create()
887 ret = -EADDRNOTAVAIL; in __trace_kprobe_create()
896 ret = -ENOENT; in __trace_kprobe_create()
905 event - argv[0]); in __trace_kprobe_create()
922 argc -= 2; argv += 2; in __trace_kprobe_create()
936 ret = -E2BIG; in __trace_kprobe_create()
945 /* This must return -ENOMEM, else there is a bug */ in __trace_kprobe_create()
946 WARN_ON_ONCE(ret != -ENOMEM); in __trace_kprobe_create()
954 ret = traceprobe_parse_probe_arg(&tk->tp, i, argv[i], &ctx); in __trace_kprobe_create()
956 goto error; /* This can be -ENOMEM */ in __trace_kprobe_create()
959 if (is_return && tk->tp.entry_arg) { in __trace_kprobe_create()
960 tk->rp.entry_handler = trace_kprobe_entry_handler; in __trace_kprobe_create()
961 tk->rp.data_size = traceprobe_get_entry_data_size(&tk->tp); in __trace_kprobe_create()
965 ret = traceprobe_set_print_fmt(&tk->tp, ptype); in __trace_kprobe_create()
972 if (ret == -EILSEQ) in __trace_kprobe_create()
974 else if (ret == -ENOENT) in __trace_kprobe_create()
976 else if (ret != -ENOMEM && ret != -EEXIST) in __trace_kprobe_create()
989 ret = -EINVAL; in __trace_kprobe_create()
1004 if (raw_command[0] == '-') in create_or_delete_trace_kprobe()
1008 return ret == -ECANCELED ? -EINVAL : ret; in create_or_delete_trace_kprobe()
1013 return create_or_delete_trace_kprobe(cmd->seq.buffer); in trace_kprobe_run_command()
1017 * kprobe_event_cmd_init - Initialize a kprobe event command object
1033 * __kprobe_event_gen_cmd_start - Generate a kprobe event command from arg list
1051 * returns -EINVAL if @loc == NULL.
1063 if (cmd->type != DYNEVENT_TYPE_KPROBE) in __kprobe_event_gen_cmd_start()
1064 return -EINVAL; in __kprobe_event_gen_cmd_start()
1067 return -EINVAL; in __kprobe_event_gen_cmd_start()
1092 if (++cmd->n_fields > MAX_TRACE_ARGS) { in __kprobe_event_gen_cmd_start()
1093 ret = -EINVAL; in __kprobe_event_gen_cmd_start()
1109 * __kprobe_event_add_fields - Add probe fields to a kprobe command from arg list
1130 if (cmd->type != DYNEVENT_TYPE_KPROBE) in __kprobe_event_add_fields()
1131 return -EINVAL; in __kprobe_event_add_fields()
1143 if (++cmd->n_fields > MAX_TRACE_ARGS) { in __kprobe_event_add_fields()
1144 ret = -EINVAL; in __kprobe_event_add_fields()
1160 * kprobe_event_delete - Delete a kprobe event
1172 snprintf(buf, MAX_EVENT_NAME_LEN, "-:%s", name); in kprobe_event_delete()
1194 if (trace_kprobe_is_return(tk) && tk->rp.maxactive) in trace_kprobe_show()
1195 seq_printf(m, "%d", tk->rp.maxactive); in trace_kprobe_show()
1196 seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp), in trace_kprobe_show()
1197 trace_probe_name(&tk->tp)); in trace_kprobe_show()
1199 if (!tk->symbol) in trace_kprobe_show()
1200 seq_printf(m, " 0x%p", tk->rp.kp.addr); in trace_kprobe_show()
1201 else if (tk->rp.kp.offset) in trace_kprobe_show()
1203 tk->rp.kp.offset); in trace_kprobe_show()
1207 for (i = 0; i < tk->tp.nr_args; i++) in trace_kprobe_show()
1208 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm); in trace_kprobe_show()
1239 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { in probes_open()
1267 tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed; in trace_kprobe_missed()
1282 seq_printf(m, " %-44s %15lu %15lu\n", in probes_profile_seq_show()
1283 trace_probe_name(&tk->tp), in probes_profile_seq_show()
1327 switch (code->op) { in process_fetch_insn()
1329 val = regs_get_register(regs, code->param); in process_fetch_insn()
1332 val = regs_get_kernel_stack_nth(regs, code->param); in process_fetch_insn()
1342 val = regs_get_kernel_argument(regs, code->param); in process_fetch_insn()
1345 val = *(unsigned long *)((unsigned long)edata + code->offset); in process_fetch_insn()
1368 struct trace_event_call *call = trace_probe_event_call(&tk->tp); in NOKPROBE_SYMBOL()
1372 WARN_ON(call != trace_file->event_call); in NOKPROBE_SYMBOL()
1377 dsize = __get_data_size(&tk->tp, regs, NULL); in NOKPROBE_SYMBOL()
1380 sizeof(*entry) + tk->tp.size + dsize); in NOKPROBE_SYMBOL()
1385 entry->ip = (unsigned long)tk->rp.kp.addr; in NOKPROBE_SYMBOL()
1386 store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize); in NOKPROBE_SYMBOL()
1394 struct event_file_link *link; in kprobe_trace_func() local
1396 trace_probe_for_each_link_rcu(link, &tk->tp) in kprobe_trace_func()
1397 __kprobe_trace_func(tk, regs, link->file); in kprobe_trace_func()
1415 return -ENOENT; in trace_kprobe_entry_handler()
1419 /* store argument values into ri->data as entry data */ in trace_kprobe_entry_handler()
1420 if (tk->tp.entry_arg) in trace_kprobe_entry_handler()
1421 store_trace_entry_data(ri->data, &tk->tp, regs); in trace_kprobe_entry_handler()
1434 struct trace_event_call *call = trace_probe_event_call(&tk->tp); in __kretprobe_trace_func()
1437 WARN_ON(call != trace_file->event_call); in __kretprobe_trace_func()
1442 dsize = __get_data_size(&tk->tp, regs, ri->data); in __kretprobe_trace_func()
1445 sizeof(*entry) + tk->tp.size + dsize); in __kretprobe_trace_func()
1450 entry->func = (unsigned long)tk->rp.kp.addr; in __kretprobe_trace_func()
1451 entry->ret_ip = get_kretprobe_retaddr(ri); in __kretprobe_trace_func()
1452 store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize); in __kretprobe_trace_func()
1461 struct event_file_link *link; in kretprobe_trace_func() local
1463 trace_probe_for_each_link_rcu(link, &tk->tp) in kretprobe_trace_func()
1464 __kretprobe_trace_func(tk, ri, regs, link->file); in kretprobe_trace_func()
1474 struct trace_seq *s = &iter->seq; in print_kprobe_event()
1475 struct trace_probe *tp; in print_kprobe_event() local
1477 field = (struct kprobe_trace_entry_head *)iter->ent; in print_kprobe_event()
1478 tp = trace_probe_primary_from_call( in print_kprobe_event()
1480 if (WARN_ON_ONCE(!tp)) in print_kprobe_event()
1483 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); in print_kprobe_event()
1485 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET)) in print_kprobe_event()
1490 if (trace_probe_print_args(s, tp->args, tp->nr_args, in print_kprobe_event()
1504 struct trace_seq *s = &iter->seq; in print_kretprobe_event()
1505 struct trace_probe *tp; in print_kretprobe_event() local
1507 field = (struct kretprobe_trace_entry_head *)iter->ent; in print_kretprobe_event()
1508 tp = trace_probe_primary_from_call( in print_kretprobe_event()
1510 if (WARN_ON_ONCE(!tp)) in print_kretprobe_event()
1513 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); in print_kretprobe_event()
1515 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET)) in print_kretprobe_event()
1518 trace_seq_puts(s, " <- "); in print_kretprobe_event()
1520 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET)) in print_kretprobe_event()
1525 if (trace_probe_print_args(s, tp->args, tp->nr_args, in print_kretprobe_event()
1540 struct trace_probe *tp; in kprobe_event_define_fields() local
1542 tp = trace_probe_primary_from_call(event_call); in kprobe_event_define_fields()
1543 if (WARN_ON_ONCE(!tp)) in kprobe_event_define_fields()
1544 return -ENOENT; in kprobe_event_define_fields()
1548 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); in kprobe_event_define_fields()
1555 struct trace_probe *tp; in kretprobe_event_define_fields() local
1557 tp = trace_probe_primary_from_call(event_call); in kretprobe_event_define_fields()
1558 if (WARN_ON_ONCE(!tp)) in kretprobe_event_define_fields()
1559 return -ENOENT; in kretprobe_event_define_fields()
1564 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); in kretprobe_event_define_fields()
1573 struct trace_event_call *call = trace_probe_event_call(&tk->tp); in kprobe_perf_func()
1596 head = this_cpu_ptr(call->perf_events); in kprobe_perf_func()
1600 dsize = __get_data_size(&tk->tp, regs, NULL); in kprobe_perf_func()
1601 __size = sizeof(*entry) + tk->tp.size + dsize; in kprobe_perf_func()
1603 size -= sizeof(u32); in kprobe_perf_func()
1609 entry->ip = (unsigned long)tk->rp.kp.addr; in kprobe_perf_func()
1611 store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize); in kprobe_perf_func()
1612 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, in kprobe_perf_func()
1623 struct trace_event_call *call = trace_probe_event_call(&tk->tp); in kretprobe_perf_func()
1632 head = this_cpu_ptr(call->perf_events); in kretprobe_perf_func()
1636 dsize = __get_data_size(&tk->tp, regs, ri->data); in kretprobe_perf_func()
1637 __size = sizeof(*entry) + tk->tp.size + dsize; in kretprobe_perf_func()
1639 size -= sizeof(u32); in kretprobe_perf_func()
1645 entry->func = (unsigned long)tk->rp.kp.addr; in kretprobe_perf_func()
1646 entry->ret_ip = get_kretprobe_retaddr(ri); in kretprobe_perf_func()
1647 store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize); in kretprobe_perf_func()
1648 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, in kretprobe_perf_func()
1658 const char *pevent = trace_event_name(event->tp_event); in bpf_get_kprobe_info()
1659 const char *group = event->tp_event->class->system; in bpf_get_kprobe_info()
1665 tk = trace_kprobe_primary_from_call(event->tp_event); in bpf_get_kprobe_info()
1667 return -EINVAL; in bpf_get_kprobe_info()
1671 *probe_offset = tk->rp.kp.offset; in bpf_get_kprobe_info()
1673 (unsigned long)tk->rp.kp.addr : 0; in bpf_get_kprobe_info()
1674 *symbol = tk->symbol; in bpf_get_kprobe_info()
1718 raw_cpu_inc(*tk->nhit); in kprobe_dispatcher()
1720 if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE)) in kprobe_dispatcher()
1723 if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE)) in kprobe_dispatcher()
1745 raw_cpu_inc(*tk->nhit); in kretprobe_dispatcher()
1747 if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE)) in kretprobe_dispatcher()
1750 if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE)) in kretprobe_dispatcher()
1779 struct trace_event_call *call = trace_probe_event_call(&tk->tp); in init_trace_event_call()
1782 call->event.funcs = &kretprobe_funcs; in init_trace_event_call()
1783 call->class->fields_array = kretprobe_fields_array; in init_trace_event_call()
1785 call->event.funcs = &kprobe_funcs; in init_trace_event_call()
1786 call->class->fields_array = kprobe_fields_array; in init_trace_event_call()
1789 call->flags = TRACE_EVENT_FL_KPROBE; in init_trace_event_call()
1790 call->class->reg = kprobe_register; in init_trace_event_call()
1797 return trace_probe_register_event_call(&tk->tp); in register_kprobe_event()
1802 return trace_probe_unregister_event_call(&tk->tp); in unregister_kprobe_event()
1826 return ERR_PTR(-EADDRNOTAVAIL); in create_local_trace_kprobe()
1832 return ERR_PTR(-ENOENT); in create_local_trace_kprobe()
1856 if (traceprobe_set_print_fmt(&tk->tp, ptype) < 0) { in create_local_trace_kprobe()
1857 ret = -ENOMEM; in create_local_trace_kprobe()
1865 return trace_probe_event_call(&tk->tp); in create_local_trace_kprobe()
1879 if (trace_probe_is_enabled(&tk->tp)) { in destroy_local_trace_kprobe()
1899 list_for_each_entry(file, &tr->events, list) in enable_boot_kprobe_events()
1900 if (file->event_call == trace_probe_event_call(&tk->tp)) in enable_boot_kprobe_events()
1941 return -EINVAL; in init_kprobe_trace_early()
1977 list_for_each_entry(file, &tr->events, list) in find_trace_probe_file()
1978 if (file->event_call == trace_probe_event_call(&tk->tp)) in find_trace_probe_file()
1996 return -ENODEV; in kprobe_trace_self_tests_init()
2022 trace_probe_event_call(&tk->tp), file); in kprobe_trace_self_tests_init()
2043 trace_probe_event_call(&tk->tp), file); in kprobe_trace_self_tests_init()
2055 * are no side-effects and the call is never performed. in kprobe_trace_self_tests_init()
2077 trace_probe_event_call(&tk->tp), file); in kprobe_trace_self_tests_init()
2096 trace_probe_event_call(&tk->tp), file); in kprobe_trace_self_tests_init()
2099 ret = create_or_delete_trace_kprobe("-:testprobe"); in kprobe_trace_self_tests_init()
2105 ret = create_or_delete_trace_kprobe("-:testprobe2"); in kprobe_trace_self_tests_init()