1bcea3f96SSteven Rostedt (VMware) // SPDX-License-Identifier: GPL-2.0
2413d37d1SMasami Hiramatsu /*
377b44d1bSMasami Hiramatsu * Kprobes-based tracing events
4413d37d1SMasami Hiramatsu *
5413d37d1SMasami Hiramatsu * Created by Masami Hiramatsu <mhiramat@redhat.com>
6413d37d1SMasami Hiramatsu *
7413d37d1SMasami Hiramatsu */
872576341SMasami Hiramatsu #define pr_fmt(fmt) "trace_kprobe: " fmt
9413d37d1SMasami Hiramatsu
10aef2fedaSJakub Kicinski #include <linux/bpf-cgroup.h>
1117911ff3SSteven Rostedt (VMware) #include <linux/security.h>
12413d37d1SMasami Hiramatsu #include <linux/module.h>
13413d37d1SMasami Hiramatsu #include <linux/uaccess.h>
14b2d09103SIngo Molnar #include <linux/rculist.h>
15540adea3SMasami Hiramatsu #include <linux/error-injection.h>
16413d37d1SMasami Hiramatsu
17970988e1SMasami Hiramatsu #include <asm/setup.h> /* for COMMAND_LINE_SIZE */
18970988e1SMasami Hiramatsu
196212dd29SMasami Hiramatsu #include "trace_dynevent.h"
20d899926fSFrancis Deslauriers #include "trace_kprobe_selftest.h"
218ab83f56SSrikar Dronamraju #include "trace_probe.h"
2253305928SMasami Hiramatsu #include "trace_probe_tmpl.h"
23f1d3cbfaSSteven Rostedt (Google) #include "trace_probe_kernel.h"
24413d37d1SMasami Hiramatsu
25f52487e9SMasami Hiramatsu #define KPROBE_EVENT_SYSTEM "kprobes"
26696ced4fSAlban Crequy #define KRETPROBE_MAXACTIVE_MAX 4096
27970988e1SMasami Hiramatsu
28970988e1SMasami Hiramatsu /* Kprobe early definition from command line */
29970988e1SMasami Hiramatsu static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata;
30970988e1SMasami Hiramatsu
set_kprobe_boot_events(char * str)31970988e1SMasami Hiramatsu static int __init set_kprobe_boot_events(char *str)
32970988e1SMasami Hiramatsu {
33c7dce4c5SAzeem Shaikh strscpy(kprobe_boot_events_buf, str, COMMAND_LINE_SIZE);
3460efe21eSMasami Hiramatsu disable_tracing_selftest("running kprobe events");
3560efe21eSMasami Hiramatsu
361d02b444SRandy Dunlap return 1;
37970988e1SMasami Hiramatsu }
38970988e1SMasami Hiramatsu __setup("kprobe_event=", set_kprobe_boot_events);
39413d37d1SMasami Hiramatsu
40d262271dSMasami Hiramatsu static int trace_kprobe_create(const char *raw_command);
416212dd29SMasami Hiramatsu static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev);
426212dd29SMasami Hiramatsu static int trace_kprobe_release(struct dyn_event *ev);
436212dd29SMasami Hiramatsu static bool trace_kprobe_is_busy(struct dyn_event *ev);
446212dd29SMasami Hiramatsu static bool trace_kprobe_match(const char *system, const char *event,
4530199137SMasami Hiramatsu int argc, const char **argv, struct dyn_event *ev);
466212dd29SMasami Hiramatsu
476212dd29SMasami Hiramatsu static struct dyn_event_operations trace_kprobe_ops = {
486212dd29SMasami Hiramatsu .create = trace_kprobe_create,
496212dd29SMasami Hiramatsu .show = trace_kprobe_show,
506212dd29SMasami Hiramatsu .is_busy = trace_kprobe_is_busy,
516212dd29SMasami Hiramatsu .free = trace_kprobe_release,
526212dd29SMasami Hiramatsu .match = trace_kprobe_match,
536212dd29SMasami Hiramatsu };
546212dd29SMasami Hiramatsu
55cede666eSValdis Klētnieks /*
5677b44d1bSMasami Hiramatsu * Kprobe event core functions
57413d37d1SMasami Hiramatsu */
58c31ffb3fSNamhyung Kim struct trace_kprobe {
596212dd29SMasami Hiramatsu struct dyn_event devent;
604a846b44SMasami Hiramatsu struct kretprobe rp; /* Use rp.kp for kprobe use */
61a7636d9eSMartin KaFai Lau unsigned long __percpu *nhit;
62413d37d1SMasami Hiramatsu const char *symbol; /* symbol name */
63c31ffb3fSNamhyung Kim struct trace_probe tp;
64413d37d1SMasami Hiramatsu };
65413d37d1SMasami Hiramatsu
is_trace_kprobe(struct dyn_event * ev)666212dd29SMasami Hiramatsu static bool is_trace_kprobe(struct dyn_event *ev)
676212dd29SMasami Hiramatsu {
686212dd29SMasami Hiramatsu return ev->ops == &trace_kprobe_ops;
696212dd29SMasami Hiramatsu }
706212dd29SMasami Hiramatsu
to_trace_kprobe(struct dyn_event * ev)716212dd29SMasami Hiramatsu static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev)
726212dd29SMasami Hiramatsu {
736212dd29SMasami Hiramatsu return container_of(ev, struct trace_kprobe, devent);
746212dd29SMasami Hiramatsu }
756212dd29SMasami Hiramatsu
766212dd29SMasami Hiramatsu /**
776212dd29SMasami Hiramatsu * for_each_trace_kprobe - iterate over the trace_kprobe list
786212dd29SMasami Hiramatsu * @pos: the struct trace_kprobe * for each entry
796212dd29SMasami Hiramatsu * @dpos: the struct dyn_event * to use as a loop cursor
806212dd29SMasami Hiramatsu */
816212dd29SMasami Hiramatsu #define for_each_trace_kprobe(pos, dpos) \
826212dd29SMasami Hiramatsu for_each_dyn_event(dpos) \
836212dd29SMasami Hiramatsu if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos)))
846212dd29SMasami Hiramatsu
trace_kprobe_is_return(struct trace_kprobe * tk)853da0f180SMasami Hiramatsu static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)
86413d37d1SMasami Hiramatsu {
87c31ffb3fSNamhyung Kim return tk->rp.handler != NULL;
88413d37d1SMasami Hiramatsu }
89413d37d1SMasami Hiramatsu
trace_kprobe_symbol(struct trace_kprobe * tk)903da0f180SMasami Hiramatsu static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk)
91413d37d1SMasami Hiramatsu {
92c31ffb3fSNamhyung Kim return tk->symbol ? tk->symbol : "unknown";
93413d37d1SMasami Hiramatsu }
94413d37d1SMasami Hiramatsu
trace_kprobe_offset(struct trace_kprobe * tk)953da0f180SMasami Hiramatsu static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
9661424318SMasami Hiramatsu {
97c31ffb3fSNamhyung Kim return tk->rp.kp.offset;
9861424318SMasami Hiramatsu }
9961424318SMasami Hiramatsu
trace_kprobe_has_gone(struct trace_kprobe * tk)1003da0f180SMasami Hiramatsu static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)
10161424318SMasami Hiramatsu {
10229e8077aSMasami Hiramatsu return kprobe_gone(&tk->rp.kp);
10361424318SMasami Hiramatsu }
10461424318SMasami Hiramatsu
trace_kprobe_within_module(struct trace_kprobe * tk,struct module * mod)1053da0f180SMasami Hiramatsu static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
10661424318SMasami Hiramatsu struct module *mod)
10761424318SMasami Hiramatsu {
108f3d36426SJarkko Sakkinen int len = strlen(module_name(mod));
109c31ffb3fSNamhyung Kim const char *name = trace_kprobe_symbol(tk);
110f3d36426SJarkko Sakkinen
111f3d36426SJarkko Sakkinen return strncmp(module_name(mod), name, len) == 0 && name[len] == ':';
11261424318SMasami Hiramatsu }
11361424318SMasami Hiramatsu
trace_kprobe_module_exist(struct trace_kprobe * tk)11459158ec4SMasami Hiramatsu static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk)
11561424318SMasami Hiramatsu {
11659158ec4SMasami Hiramatsu char *p;
11759158ec4SMasami Hiramatsu bool ret;
11859158ec4SMasami Hiramatsu
11959158ec4SMasami Hiramatsu if (!tk->symbol)
12059158ec4SMasami Hiramatsu return false;
12159158ec4SMasami Hiramatsu p = strchr(tk->symbol, ':');
12259158ec4SMasami Hiramatsu if (!p)
12359158ec4SMasami Hiramatsu return true;
12459158ec4SMasami Hiramatsu *p = '\0';
125a0060505SChristoph Hellwig rcu_read_lock_sched();
12659158ec4SMasami Hiramatsu ret = !!find_module(tk->symbol);
127a0060505SChristoph Hellwig rcu_read_unlock_sched();
12859158ec4SMasami Hiramatsu *p = ':';
12959158ec4SMasami Hiramatsu
13059158ec4SMasami Hiramatsu return ret;
13161424318SMasami Hiramatsu }
13261424318SMasami Hiramatsu
trace_kprobe_is_busy(struct dyn_event * ev)1336212dd29SMasami Hiramatsu static bool trace_kprobe_is_busy(struct dyn_event *ev)
1346212dd29SMasami Hiramatsu {
1356212dd29SMasami Hiramatsu struct trace_kprobe *tk = to_trace_kprobe(ev);
1366212dd29SMasami Hiramatsu
1376212dd29SMasami Hiramatsu return trace_probe_is_enabled(&tk->tp);
1386212dd29SMasami Hiramatsu }
1396212dd29SMasami Hiramatsu
trace_kprobe_match_command_head(struct trace_kprobe * tk,int argc,const char ** argv)140eb5bf813SMasami Hiramatsu static bool trace_kprobe_match_command_head(struct trace_kprobe *tk,
141eb5bf813SMasami Hiramatsu int argc, const char **argv)
142eb5bf813SMasami Hiramatsu {
143eb5bf813SMasami Hiramatsu char buf[MAX_ARGSTR_LEN + 1];
144eb5bf813SMasami Hiramatsu
145eb5bf813SMasami Hiramatsu if (!argc)
146eb5bf813SMasami Hiramatsu return true;
147eb5bf813SMasami Hiramatsu
148eb5bf813SMasami Hiramatsu if (!tk->symbol)
149eb5bf813SMasami Hiramatsu snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr);
150eb5bf813SMasami Hiramatsu else if (tk->rp.kp.offset)
151eb5bf813SMasami Hiramatsu snprintf(buf, sizeof(buf), "%s+%u",
152eb5bf813SMasami Hiramatsu trace_kprobe_symbol(tk), tk->rp.kp.offset);
153eb5bf813SMasami Hiramatsu else
154eb5bf813SMasami Hiramatsu snprintf(buf, sizeof(buf), "%s", trace_kprobe_symbol(tk));
155eb5bf813SMasami Hiramatsu if (strcmp(buf, argv[0]))
156eb5bf813SMasami Hiramatsu return false;
157eb5bf813SMasami Hiramatsu argc--; argv++;
158eb5bf813SMasami Hiramatsu
159eb5bf813SMasami Hiramatsu return trace_probe_match_command_args(&tk->tp, argc, argv);
160eb5bf813SMasami Hiramatsu }
161eb5bf813SMasami Hiramatsu
trace_kprobe_match(const char * system,const char * event,int argc,const char ** argv,struct dyn_event * ev)1626212dd29SMasami Hiramatsu static bool trace_kprobe_match(const char *system, const char *event,
16330199137SMasami Hiramatsu int argc, const char **argv, struct dyn_event *ev)
1646212dd29SMasami Hiramatsu {
1656212dd29SMasami Hiramatsu struct trace_kprobe *tk = to_trace_kprobe(ev);
1666212dd29SMasami Hiramatsu
16795c104c3SLinyu Yuan return (event[0] == '\0' ||
16895c104c3SLinyu Yuan strcmp(trace_probe_name(&tk->tp), event) == 0) &&
169eb5bf813SMasami Hiramatsu (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0) &&
170eb5bf813SMasami Hiramatsu trace_kprobe_match_command_head(tk, argc, argv);
1716212dd29SMasami Hiramatsu }
1726212dd29SMasami Hiramatsu
trace_kprobe_nhit(struct trace_kprobe * tk)173f18f97acSMarcin Nowakowski static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk)
174f18f97acSMarcin Nowakowski {
175f18f97acSMarcin Nowakowski unsigned long nhit = 0;
176f18f97acSMarcin Nowakowski int cpu;
177f18f97acSMarcin Nowakowski
178f18f97acSMarcin Nowakowski for_each_possible_cpu(cpu)
179f18f97acSMarcin Nowakowski nhit += *per_cpu_ptr(tk->nhit, cpu);
180f18f97acSMarcin Nowakowski
181f18f97acSMarcin Nowakowski return nhit;
182f18f97acSMarcin Nowakowski }
183f18f97acSMarcin Nowakowski
trace_kprobe_is_registered(struct trace_kprobe * tk)184715fa2fdSMasami Hiramatsu static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk)
185715fa2fdSMasami Hiramatsu {
186715fa2fdSMasami Hiramatsu return !(list_empty(&tk->rp.kp.list) &&
187715fa2fdSMasami Hiramatsu hlist_unhashed(&tk->rp.kp.hlist));
188715fa2fdSMasami Hiramatsu }
189715fa2fdSMasami Hiramatsu
1906bc6c77cSMasami Hiramatsu /* Return 0 if it fails to find the symbol address */
19145408c4fSMasami Hiramatsu static nokprobe_inline
trace_kprobe_address(struct trace_kprobe * tk)19245408c4fSMasami Hiramatsu unsigned long trace_kprobe_address(struct trace_kprobe *tk)
19345408c4fSMasami Hiramatsu {
19445408c4fSMasami Hiramatsu unsigned long addr;
19545408c4fSMasami Hiramatsu
19645408c4fSMasami Hiramatsu if (tk->symbol) {
19745408c4fSMasami Hiramatsu addr = (unsigned long)
19845408c4fSMasami Hiramatsu kallsyms_lookup_name(trace_kprobe_symbol(tk));
1996bc6c77cSMasami Hiramatsu if (addr)
20045408c4fSMasami Hiramatsu addr += tk->rp.kp.offset;
20145408c4fSMasami Hiramatsu } else {
20245408c4fSMasami Hiramatsu addr = (unsigned long)tk->rp.kp.addr;
20345408c4fSMasami Hiramatsu }
20445408c4fSMasami Hiramatsu return addr;
20545408c4fSMasami Hiramatsu }
20645408c4fSMasami Hiramatsu
20760d53e2cSMasami Hiramatsu static nokprobe_inline struct trace_kprobe *
trace_kprobe_primary_from_call(struct trace_event_call * call)20860d53e2cSMasami Hiramatsu trace_kprobe_primary_from_call(struct trace_event_call *call)
20960d53e2cSMasami Hiramatsu {
21060d53e2cSMasami Hiramatsu struct trace_probe *tp;
21160d53e2cSMasami Hiramatsu
21260d53e2cSMasami Hiramatsu tp = trace_probe_primary_from_call(call);
21360d53e2cSMasami Hiramatsu if (WARN_ON_ONCE(!tp))
21460d53e2cSMasami Hiramatsu return NULL;
21560d53e2cSMasami Hiramatsu
21660d53e2cSMasami Hiramatsu return container_of(tp, struct trace_kprobe, tp);
21760d53e2cSMasami Hiramatsu }
21860d53e2cSMasami Hiramatsu
trace_kprobe_on_func_entry(struct trace_event_call * call)219b4da3340SMasami Hiramatsu bool trace_kprobe_on_func_entry(struct trace_event_call *call)
2209802d865SJosef Bacik {
22160d53e2cSMasami Hiramatsu struct trace_kprobe *tk = trace_kprobe_primary_from_call(call);
222b4da3340SMasami Hiramatsu
22397c753e6SMasami Hiramatsu return tk ? (kprobe_on_func_entry(tk->rp.kp.addr,
224b4da3340SMasami Hiramatsu tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name,
22597c753e6SMasami Hiramatsu tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0) : false;
2269802d865SJosef Bacik }
2279802d865SJosef Bacik
trace_kprobe_error_injectable(struct trace_event_call * call)228b4da3340SMasami Hiramatsu bool trace_kprobe_error_injectable(struct trace_event_call *call)
2299802d865SJosef Bacik {
23060d53e2cSMasami Hiramatsu struct trace_kprobe *tk = trace_kprobe_primary_from_call(call);
2319802d865SJosef Bacik
23260d53e2cSMasami Hiramatsu return tk ? within_error_injection_list(trace_kprobe_address(tk)) :
23360d53e2cSMasami Hiramatsu false;
2349802d865SJosef Bacik }
2359802d865SJosef Bacik
236c31ffb3fSNamhyung Kim static int register_kprobe_event(struct trace_kprobe *tk);
237c31ffb3fSNamhyung Kim static int unregister_kprobe_event(struct trace_kprobe *tk);
238413d37d1SMasami Hiramatsu
23950d78056SMasami Hiramatsu static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
24050d78056SMasami Hiramatsu static int kretprobe_dispatcher(struct kretprobe_instance *ri,
24150d78056SMasami Hiramatsu struct pt_regs *regs);
24250d78056SMasami Hiramatsu
free_trace_kprobe(struct trace_kprobe * tk)243455b2899SMasami Hiramatsu static void free_trace_kprobe(struct trace_kprobe *tk)
244455b2899SMasami Hiramatsu {
245455b2899SMasami Hiramatsu if (tk) {
246455b2899SMasami Hiramatsu trace_probe_cleanup(&tk->tp);
247455b2899SMasami Hiramatsu kfree(tk->symbol);
248455b2899SMasami Hiramatsu free_percpu(tk->nhit);
249455b2899SMasami Hiramatsu kfree(tk);
250455b2899SMasami Hiramatsu }
251455b2899SMasami Hiramatsu }
252455b2899SMasami Hiramatsu
2534a846b44SMasami Hiramatsu /*
2544a846b44SMasami Hiramatsu * Allocate new trace_probe and initialize it (including kprobes).
2554a846b44SMasami Hiramatsu */
alloc_trace_kprobe(const char * group,const char * event,void * addr,const char * symbol,unsigned long offs,int maxactive,int nargs,bool is_return)256c31ffb3fSNamhyung Kim static struct trace_kprobe *alloc_trace_kprobe(const char *group,
257f52487e9SMasami Hiramatsu const char *event,
2584a846b44SMasami Hiramatsu void *addr,
2594a846b44SMasami Hiramatsu const char *symbol,
2604a846b44SMasami Hiramatsu unsigned long offs,
261696ced4fSAlban Crequy int maxactive,
2623a6b7666SSrikar Dronamraju int nargs, bool is_return)
263413d37d1SMasami Hiramatsu {
264c31ffb3fSNamhyung Kim struct trace_kprobe *tk;
2656f3cf440SMasami Hiramatsu int ret = -ENOMEM;
266413d37d1SMasami Hiramatsu
267845cbf3eSSteven Rostedt (VMware) tk = kzalloc(struct_size(tk, tp.args, nargs), GFP_KERNEL);
268c31ffb3fSNamhyung Kim if (!tk)
2696f3cf440SMasami Hiramatsu return ERR_PTR(ret);
270413d37d1SMasami Hiramatsu
271a7636d9eSMartin KaFai Lau tk->nhit = alloc_percpu(unsigned long);
272a7636d9eSMartin KaFai Lau if (!tk->nhit)
273a7636d9eSMartin KaFai Lau goto error;
274a7636d9eSMartin KaFai Lau
275413d37d1SMasami Hiramatsu if (symbol) {
276c31ffb3fSNamhyung Kim tk->symbol = kstrdup(symbol, GFP_KERNEL);
277c31ffb3fSNamhyung Kim if (!tk->symbol)
278413d37d1SMasami Hiramatsu goto error;
279c31ffb3fSNamhyung Kim tk->rp.kp.symbol_name = tk->symbol;
280c31ffb3fSNamhyung Kim tk->rp.kp.offset = offs;
2814a846b44SMasami Hiramatsu } else
282c31ffb3fSNamhyung Kim tk->rp.kp.addr = addr;
2834a846b44SMasami Hiramatsu
2844a846b44SMasami Hiramatsu if (is_return)
285c31ffb3fSNamhyung Kim tk->rp.handler = kretprobe_dispatcher;
2864a846b44SMasami Hiramatsu else
287c31ffb3fSNamhyung Kim tk->rp.kp.pre_handler = kprobe_dispatcher;
2884a846b44SMasami Hiramatsu
289696ced4fSAlban Crequy tk->rp.maxactive = maxactive;
290715fa2fdSMasami Hiramatsu INIT_HLIST_NODE(&tk->rp.kp.hlist);
291715fa2fdSMasami Hiramatsu INIT_LIST_HEAD(&tk->rp.kp.list);
292696ced4fSAlban Crequy
293b61387cbSMasami Hiramatsu ret = trace_probe_init(&tk->tp, event, group, false);
294455b2899SMasami Hiramatsu if (ret < 0)
295f52487e9SMasami Hiramatsu goto error;
296f52487e9SMasami Hiramatsu
2976212dd29SMasami Hiramatsu dyn_event_init(&tk->devent, &trace_kprobe_ops);
298c31ffb3fSNamhyung Kim return tk;
299413d37d1SMasami Hiramatsu error:
300455b2899SMasami Hiramatsu free_trace_kprobe(tk);
3016f3cf440SMasami Hiramatsu return ERR_PTR(ret);
302413d37d1SMasami Hiramatsu }
303413d37d1SMasami Hiramatsu
find_trace_kprobe(const char * event,const char * group)304c31ffb3fSNamhyung Kim static struct trace_kprobe *find_trace_kprobe(const char *event,
305dd004c47SMasami Hiramatsu const char *group)
306413d37d1SMasami Hiramatsu {
3076212dd29SMasami Hiramatsu struct dyn_event *pos;
308c31ffb3fSNamhyung Kim struct trace_kprobe *tk;
309413d37d1SMasami Hiramatsu
3106212dd29SMasami Hiramatsu for_each_trace_kprobe(tk, pos)
311b55ce203SMasami Hiramatsu if (strcmp(trace_probe_name(&tk->tp), event) == 0 &&
312b55ce203SMasami Hiramatsu strcmp(trace_probe_group_name(&tk->tp), group) == 0)
313c31ffb3fSNamhyung Kim return tk;
314413d37d1SMasami Hiramatsu return NULL;
315413d37d1SMasami Hiramatsu }
316413d37d1SMasami Hiramatsu
__enable_trace_kprobe(struct trace_kprobe * tk)31787107a25SSteven Rostedt (VMware) static inline int __enable_trace_kprobe(struct trace_kprobe *tk)
31887107a25SSteven Rostedt (VMware) {
31987107a25SSteven Rostedt (VMware) int ret = 0;
32087107a25SSteven Rostedt (VMware)
321715fa2fdSMasami Hiramatsu if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) {
32287107a25SSteven Rostedt (VMware) if (trace_kprobe_is_return(tk))
32387107a25SSteven Rostedt (VMware) ret = enable_kretprobe(&tk->rp);
32487107a25SSteven Rostedt (VMware) else
32587107a25SSteven Rostedt (VMware) ret = enable_kprobe(&tk->rp.kp);
32687107a25SSteven Rostedt (VMware) }
32787107a25SSteven Rostedt (VMware)
32887107a25SSteven Rostedt (VMware) return ret;
32987107a25SSteven Rostedt (VMware) }
33087107a25SSteven Rostedt (VMware)
__disable_trace_kprobe(struct trace_probe * tp)33160d53e2cSMasami Hiramatsu static void __disable_trace_kprobe(struct trace_probe *tp)
33260d53e2cSMasami Hiramatsu {
33360d53e2cSMasami Hiramatsu struct trace_kprobe *tk;
33460d53e2cSMasami Hiramatsu
335e161c6bfSJiri Olsa list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) {
33660d53e2cSMasami Hiramatsu if (!trace_kprobe_is_registered(tk))
33760d53e2cSMasami Hiramatsu continue;
33860d53e2cSMasami Hiramatsu if (trace_kprobe_is_return(tk))
33960d53e2cSMasami Hiramatsu disable_kretprobe(&tk->rp);
34060d53e2cSMasami Hiramatsu else
34160d53e2cSMasami Hiramatsu disable_kprobe(&tk->rp.kp);
34260d53e2cSMasami Hiramatsu }
34360d53e2cSMasami Hiramatsu }
34460d53e2cSMasami Hiramatsu
3453fe3d619SOleg Nesterov /*
34641a7dd42SMasami Hiramatsu * Enable trace_probe
34741a7dd42SMasami Hiramatsu * if the file is NULL, enable "perf" handler, or enable "trace" handler.
34841a7dd42SMasami Hiramatsu */
enable_trace_kprobe(struct trace_event_call * call,struct trace_event_file * file)34960d53e2cSMasami Hiramatsu static int enable_trace_kprobe(struct trace_event_call *call,
35060d53e2cSMasami Hiramatsu struct trace_event_file *file)
3511538f888SMasami Hiramatsu {
352e161c6bfSJiri Olsa struct trace_probe *tp;
35360d53e2cSMasami Hiramatsu struct trace_kprobe *tk;
35460d53e2cSMasami Hiramatsu bool enabled;
3551538f888SMasami Hiramatsu int ret = 0;
3561538f888SMasami Hiramatsu
35760d53e2cSMasami Hiramatsu tp = trace_probe_primary_from_call(call);
35860d53e2cSMasami Hiramatsu if (WARN_ON_ONCE(!tp))
35960d53e2cSMasami Hiramatsu return -ENODEV;
36060d53e2cSMasami Hiramatsu enabled = trace_probe_is_enabled(tp);
36160d53e2cSMasami Hiramatsu
36260d53e2cSMasami Hiramatsu /* This also changes "enabled" state */
36341a7dd42SMasami Hiramatsu if (file) {
36460d53e2cSMasami Hiramatsu ret = trace_probe_add_file(tp, file);
365b5f935eeSMasami Hiramatsu if (ret)
366b5f935eeSMasami Hiramatsu return ret;
367b5f935eeSMasami Hiramatsu } else
36860d53e2cSMasami Hiramatsu trace_probe_set_flag(tp, TP_FLAG_PROFILE);
36941a7dd42SMasami Hiramatsu
370b5f935eeSMasami Hiramatsu if (enabled)
371b5f935eeSMasami Hiramatsu return 0;
372b04d52e3SOleg Nesterov
373e161c6bfSJiri Olsa list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) {
37460d53e2cSMasami Hiramatsu if (trace_kprobe_has_gone(tk))
37560d53e2cSMasami Hiramatsu continue;
37687107a25SSteven Rostedt (VMware) ret = __enable_trace_kprobe(tk);
37744d00dc7SMasami Hiramatsu if (ret)
37860d53e2cSMasami Hiramatsu break;
37960d53e2cSMasami Hiramatsu enabled = true;
38060d53e2cSMasami Hiramatsu }
38160d53e2cSMasami Hiramatsu
38244d00dc7SMasami Hiramatsu if (ret) {
38344d00dc7SMasami Hiramatsu /* Failed to enable one of them. Roll back all */
38444d00dc7SMasami Hiramatsu if (enabled)
38544d00dc7SMasami Hiramatsu __disable_trace_kprobe(tp);
386b5f935eeSMasami Hiramatsu if (file)
38760d53e2cSMasami Hiramatsu trace_probe_remove_file(tp, file);
388b5f935eeSMasami Hiramatsu else
38960d53e2cSMasami Hiramatsu trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
39057ea2a34SArtem Savkov }
391b5f935eeSMasami Hiramatsu
3921538f888SMasami Hiramatsu return ret;
3931538f888SMasami Hiramatsu }
3941538f888SMasami Hiramatsu
39541a7dd42SMasami Hiramatsu /*
39641a7dd42SMasami Hiramatsu * Disable trace_probe
39741a7dd42SMasami Hiramatsu * if the file is NULL, disable "perf" handler, or disable "trace" handler.
39841a7dd42SMasami Hiramatsu */
disable_trace_kprobe(struct trace_event_call * call,struct trace_event_file * file)39960d53e2cSMasami Hiramatsu static int disable_trace_kprobe(struct trace_event_call *call,
40060d53e2cSMasami Hiramatsu struct trace_event_file *file)
40141a7dd42SMasami Hiramatsu {
40260d53e2cSMasami Hiramatsu struct trace_probe *tp;
40360d53e2cSMasami Hiramatsu
40460d53e2cSMasami Hiramatsu tp = trace_probe_primary_from_call(call);
40560d53e2cSMasami Hiramatsu if (WARN_ON_ONCE(!tp))
40660d53e2cSMasami Hiramatsu return -ENODEV;
40741a7dd42SMasami Hiramatsu
40841a7dd42SMasami Hiramatsu if (file) {
409b5f935eeSMasami Hiramatsu if (!trace_probe_get_file_link(tp, file))
410b5f935eeSMasami Hiramatsu return -ENOENT;
411b5f935eeSMasami Hiramatsu if (!trace_probe_has_single_file(tp))
4123fe3d619SOleg Nesterov goto out;
413747774d6SMasami Hiramatsu trace_probe_clear_flag(tp, TP_FLAG_TRACE);
41441a7dd42SMasami Hiramatsu } else
415747774d6SMasami Hiramatsu trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
41641a7dd42SMasami Hiramatsu
41760d53e2cSMasami Hiramatsu if (!trace_probe_is_enabled(tp))
41860d53e2cSMasami Hiramatsu __disable_trace_kprobe(tp);
419e12f03d7SSong Liu
4203fe3d619SOleg Nesterov out:
421b5f935eeSMasami Hiramatsu if (file)
422a232e270SMasami Hiramatsu /*
423b5f935eeSMasami Hiramatsu * Synchronization is done in below function. For perf event,
424b5f935eeSMasami Hiramatsu * file == NULL and perf_trace_event_unreg() calls
425b5f935eeSMasami Hiramatsu * tracepoint_synchronize_unregister() to ensure synchronize
426b5f935eeSMasami Hiramatsu * event. We don't need to care about it.
427a232e270SMasami Hiramatsu */
428b5f935eeSMasami Hiramatsu trace_probe_remove_file(tp, file);
429a232e270SMasami Hiramatsu
43060d53e2cSMasami Hiramatsu return 0;
4311538f888SMasami Hiramatsu }
4321538f888SMasami Hiramatsu
4337bb83f6fSMasami Hiramatsu #if defined(CONFIG_DYNAMIC_FTRACE) && \
43445408c4fSMasami Hiramatsu !defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE)
__within_notrace_func(unsigned long addr)435c7411a1aSMasami Hiramatsu static bool __within_notrace_func(unsigned long addr)
43645408c4fSMasami Hiramatsu {
437c7411a1aSMasami Hiramatsu unsigned long offset, size;
43845408c4fSMasami Hiramatsu
4396bc6c77cSMasami Hiramatsu if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset))
4406bc6c77cSMasami Hiramatsu return false;
44145408c4fSMasami Hiramatsu
4429161a864SMasami Hiramatsu /* Get the entry address of the target function */
4439161a864SMasami Hiramatsu addr -= offset;
4449161a864SMasami Hiramatsu
4459161a864SMasami Hiramatsu /*
4469161a864SMasami Hiramatsu * Since ftrace_location_range() does inclusive range check, we need
4479161a864SMasami Hiramatsu * to subtract 1 byte from the end address.
4489161a864SMasami Hiramatsu */
4499161a864SMasami Hiramatsu return !ftrace_location_range(addr, addr + size - 1);
45045408c4fSMasami Hiramatsu }
451c7411a1aSMasami Hiramatsu
within_notrace_func(struct trace_kprobe * tk)452c7411a1aSMasami Hiramatsu static bool within_notrace_func(struct trace_kprobe *tk)
453c7411a1aSMasami Hiramatsu {
454dcbd21c9SMasami Hiramatsu unsigned long addr = trace_kprobe_address(tk);
455c7411a1aSMasami Hiramatsu char symname[KSYM_NAME_LEN], *p;
456c7411a1aSMasami Hiramatsu
457c7411a1aSMasami Hiramatsu if (!__within_notrace_func(addr))
458c7411a1aSMasami Hiramatsu return false;
459c7411a1aSMasami Hiramatsu
460c7411a1aSMasami Hiramatsu /* Check if the address is on a suffixed-symbol */
461c7411a1aSMasami Hiramatsu if (!lookup_symbol_name(addr, symname)) {
462c7411a1aSMasami Hiramatsu p = strchr(symname, '.');
463c7411a1aSMasami Hiramatsu if (!p)
464c7411a1aSMasami Hiramatsu return true;
465c7411a1aSMasami Hiramatsu *p = '\0';
466c7411a1aSMasami Hiramatsu addr = (unsigned long)kprobe_lookup_name(symname, 0);
467c7411a1aSMasami Hiramatsu if (addr)
468c7411a1aSMasami Hiramatsu return __within_notrace_func(addr);
469c7411a1aSMasami Hiramatsu }
470c7411a1aSMasami Hiramatsu
471c7411a1aSMasami Hiramatsu return true;
472c7411a1aSMasami Hiramatsu }
47345408c4fSMasami Hiramatsu #else
47445408c4fSMasami Hiramatsu #define within_notrace_func(tk) (false)
47545408c4fSMasami Hiramatsu #endif
47645408c4fSMasami Hiramatsu
47761424318SMasami Hiramatsu /* Internal register function - just handle k*probes and flags */
__register_trace_kprobe(struct trace_kprobe * tk)478c31ffb3fSNamhyung Kim static int __register_trace_kprobe(struct trace_kprobe *tk)
479413d37d1SMasami Hiramatsu {
480a6682814SMasami Hiramatsu int i, ret;
48161424318SMasami Hiramatsu
482a94549ddSDavid Howells ret = security_locked_down(LOCKDOWN_KPROBES);
483a94549ddSDavid Howells if (ret)
484a94549ddSDavid Howells return ret;
485a94549ddSDavid Howells
486715fa2fdSMasami Hiramatsu if (trace_kprobe_is_registered(tk))
48761424318SMasami Hiramatsu return -EINVAL;
48861424318SMasami Hiramatsu
48945408c4fSMasami Hiramatsu if (within_notrace_func(tk)) {
49045408c4fSMasami Hiramatsu pr_warn("Could not probe notrace function %s\n",
49145408c4fSMasami Hiramatsu trace_kprobe_symbol(tk));
49245408c4fSMasami Hiramatsu return -EINVAL;
49345408c4fSMasami Hiramatsu }
49445408c4fSMasami Hiramatsu
495a6682814SMasami Hiramatsu for (i = 0; i < tk->tp.nr_args; i++) {
496a6682814SMasami Hiramatsu ret = traceprobe_update_arg(&tk->tp.args[i]);
497a6682814SMasami Hiramatsu if (ret)
498a6682814SMasami Hiramatsu return ret;
499a6682814SMasami Hiramatsu }
500a6682814SMasami Hiramatsu
50161424318SMasami Hiramatsu /* Set/clear disabled flag according to tp->flag */
502c31ffb3fSNamhyung Kim if (trace_probe_is_enabled(&tk->tp))
503c31ffb3fSNamhyung Kim tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
50461424318SMasami Hiramatsu else
505c31ffb3fSNamhyung Kim tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;
50661424318SMasami Hiramatsu
507c31ffb3fSNamhyung Kim if (trace_kprobe_is_return(tk))
508c31ffb3fSNamhyung Kim ret = register_kretprobe(&tk->rp);
50961424318SMasami Hiramatsu else
510c31ffb3fSNamhyung Kim ret = register_kprobe(&tk->rp.kp);
51161424318SMasami Hiramatsu
51261424318SMasami Hiramatsu return ret;
51361424318SMasami Hiramatsu }
51461424318SMasami Hiramatsu
51561424318SMasami Hiramatsu /* Internal unregister function - just handle k*probes and flags */
__unregister_trace_kprobe(struct trace_kprobe * tk)516c31ffb3fSNamhyung Kim static void __unregister_trace_kprobe(struct trace_kprobe *tk)
51761424318SMasami Hiramatsu {
518715fa2fdSMasami Hiramatsu if (trace_kprobe_is_registered(tk)) {
519c31ffb3fSNamhyung Kim if (trace_kprobe_is_return(tk))
520c31ffb3fSNamhyung Kim unregister_kretprobe(&tk->rp);
521413d37d1SMasami Hiramatsu else
522c31ffb3fSNamhyung Kim unregister_kprobe(&tk->rp.kp);
523715fa2fdSMasami Hiramatsu /* Cleanup kprobe for reuse and mark it unregistered */
524715fa2fdSMasami Hiramatsu INIT_HLIST_NODE(&tk->rp.kp.hlist);
525715fa2fdSMasami Hiramatsu INIT_LIST_HEAD(&tk->rp.kp.list);
526c31ffb3fSNamhyung Kim if (tk->rp.kp.symbol_name)
527c31ffb3fSNamhyung Kim tk->rp.kp.addr = NULL;
52861424318SMasami Hiramatsu }
52961424318SMasami Hiramatsu }
53061424318SMasami Hiramatsu
5316212dd29SMasami Hiramatsu /* Unregister a trace_probe and probe_event */
unregister_trace_kprobe(struct trace_kprobe * tk)532c31ffb3fSNamhyung Kim static int unregister_trace_kprobe(struct trace_kprobe *tk)
53361424318SMasami Hiramatsu {
534ca89bc07SMasami Hiramatsu /* If other probes are on the event, just unregister kprobe */
535ca89bc07SMasami Hiramatsu if (trace_probe_has_sibling(&tk->tp))
536ca89bc07SMasami Hiramatsu goto unreg;
537ca89bc07SMasami Hiramatsu
53802ca1521SMasami Hiramatsu /* Enabled event can not be unregistered */
539c31ffb3fSNamhyung Kim if (trace_probe_is_enabled(&tk->tp))
54002ca1521SMasami Hiramatsu return -EBUSY;
54102ca1521SMasami Hiramatsu
5421d18538eSSteven Rostedt (VMware) /* If there's a reference to the dynamic event */
5431d18538eSSteven Rostedt (VMware) if (trace_event_dyn_busy(trace_probe_event_call(&tk->tp)))
5441d18538eSSteven Rostedt (VMware) return -EBUSY;
5451d18538eSSteven Rostedt (VMware)
54640c32592SSteven Rostedt (Red Hat) /* Will fail if probe is being used by ftrace or perf */
547c31ffb3fSNamhyung Kim if (unregister_kprobe_event(tk))
54840c32592SSteven Rostedt (Red Hat) return -EBUSY;
54940c32592SSteven Rostedt (Red Hat)
550ca89bc07SMasami Hiramatsu unreg:
551c31ffb3fSNamhyung Kim __unregister_trace_kprobe(tk);
5526212dd29SMasami Hiramatsu dyn_event_remove(&tk->devent);
553ca89bc07SMasami Hiramatsu trace_probe_unlink(&tk->tp);
55402ca1521SMasami Hiramatsu
55502ca1521SMasami Hiramatsu return 0;
556413d37d1SMasami Hiramatsu }
557413d37d1SMasami Hiramatsu
trace_kprobe_has_same_kprobe(struct trace_kprobe * orig,struct trace_kprobe * comp)558fe60b0ceSMasami Hiramatsu static bool trace_kprobe_has_same_kprobe(struct trace_kprobe *orig,
559fe60b0ceSMasami Hiramatsu struct trace_kprobe *comp)
560fe60b0ceSMasami Hiramatsu {
561fe60b0ceSMasami Hiramatsu struct trace_probe_event *tpe = orig->tp.event;
562fe60b0ceSMasami Hiramatsu int i;
563fe60b0ceSMasami Hiramatsu
564e161c6bfSJiri Olsa list_for_each_entry(orig, &tpe->probes, tp.list) {
565fe60b0ceSMasami Hiramatsu if (strcmp(trace_kprobe_symbol(orig),
566fe60b0ceSMasami Hiramatsu trace_kprobe_symbol(comp)) ||
567fe60b0ceSMasami Hiramatsu trace_kprobe_offset(orig) != trace_kprobe_offset(comp))
568fe60b0ceSMasami Hiramatsu continue;
569fe60b0ceSMasami Hiramatsu
570fe60b0ceSMasami Hiramatsu /*
571fe60b0ceSMasami Hiramatsu * trace_probe_compare_arg_type() ensured that nr_args and
572fe60b0ceSMasami Hiramatsu * each argument name and type are same. Let's compare comm.
573fe60b0ceSMasami Hiramatsu */
574fe60b0ceSMasami Hiramatsu for (i = 0; i < orig->tp.nr_args; i++) {
575fe60b0ceSMasami Hiramatsu if (strcmp(orig->tp.args[i].comm,
576fe60b0ceSMasami Hiramatsu comp->tp.args[i].comm))
577f8d7ab2bSSrikar Dronamraju break;
578fe60b0ceSMasami Hiramatsu }
579fe60b0ceSMasami Hiramatsu
580f8d7ab2bSSrikar Dronamraju if (i == orig->tp.nr_args)
581fe60b0ceSMasami Hiramatsu return true;
582fe60b0ceSMasami Hiramatsu }
583fe60b0ceSMasami Hiramatsu
584fe60b0ceSMasami Hiramatsu return false;
585fe60b0ceSMasami Hiramatsu }
586fe60b0ceSMasami Hiramatsu
append_trace_kprobe(struct trace_kprobe * tk,struct trace_kprobe * to)587ca89bc07SMasami Hiramatsu static int append_trace_kprobe(struct trace_kprobe *tk, struct trace_kprobe *to)
588ca89bc07SMasami Hiramatsu {
589ca89bc07SMasami Hiramatsu int ret;
590ca89bc07SMasami Hiramatsu
591fe60b0ceSMasami Hiramatsu ret = trace_probe_compare_arg_type(&tk->tp, &to->tp);
592fe60b0ceSMasami Hiramatsu if (ret) {
593fe60b0ceSMasami Hiramatsu /* Note that argument starts index = 2 */
594fe60b0ceSMasami Hiramatsu trace_probe_log_set_index(ret + 1);
595fe60b0ceSMasami Hiramatsu trace_probe_log_err(0, DIFF_ARG_TYPE);
596fe60b0ceSMasami Hiramatsu return -EEXIST;
597fe60b0ceSMasami Hiramatsu }
598fe60b0ceSMasami Hiramatsu if (trace_kprobe_has_same_kprobe(to, tk)) {
599fe60b0ceSMasami Hiramatsu trace_probe_log_set_index(0);
600fe60b0ceSMasami Hiramatsu trace_probe_log_err(0, SAME_PROBE);
601fe60b0ceSMasami Hiramatsu return -EEXIST;
602fe60b0ceSMasami Hiramatsu }
603fe60b0ceSMasami Hiramatsu
604ca89bc07SMasami Hiramatsu /* Append to existing event */
605ca89bc07SMasami Hiramatsu ret = trace_probe_append(&tk->tp, &to->tp);
606ca89bc07SMasami Hiramatsu if (ret)
607ca89bc07SMasami Hiramatsu return ret;
608ca89bc07SMasami Hiramatsu
609ca89bc07SMasami Hiramatsu /* Register k*probe */
610ca89bc07SMasami Hiramatsu ret = __register_trace_kprobe(tk);
611ca89bc07SMasami Hiramatsu if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {
612ca89bc07SMasami Hiramatsu pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
613ca89bc07SMasami Hiramatsu ret = 0;
614ca89bc07SMasami Hiramatsu }
615ca89bc07SMasami Hiramatsu
616ca89bc07SMasami Hiramatsu if (ret)
617ca89bc07SMasami Hiramatsu trace_probe_unlink(&tk->tp);
618ca89bc07SMasami Hiramatsu else
6198b0e6c74SSteven Rostedt (VMware) dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp));
620ca89bc07SMasami Hiramatsu
621ca89bc07SMasami Hiramatsu return ret;
622ca89bc07SMasami Hiramatsu }
623ca89bc07SMasami Hiramatsu
624413d37d1SMasami Hiramatsu /* Register a trace_probe and probe_event */
register_trace_kprobe(struct trace_kprobe * tk)625c31ffb3fSNamhyung Kim static int register_trace_kprobe(struct trace_kprobe *tk)
626413d37d1SMasami Hiramatsu {
627c31ffb3fSNamhyung Kim struct trace_kprobe *old_tk;
628413d37d1SMasami Hiramatsu int ret;
629413d37d1SMasami Hiramatsu
6306212dd29SMasami Hiramatsu mutex_lock(&event_mutex);
631413d37d1SMasami Hiramatsu
632b55ce203SMasami Hiramatsu old_tk = find_trace_kprobe(trace_probe_name(&tk->tp),
633b55ce203SMasami Hiramatsu trace_probe_group_name(&tk->tp));
634c31ffb3fSNamhyung Kim if (old_tk) {
635ca89bc07SMasami Hiramatsu if (trace_kprobe_is_return(tk) != trace_kprobe_is_return(old_tk)) {
636ca89bc07SMasami Hiramatsu trace_probe_log_set_index(0);
637ca89bc07SMasami Hiramatsu trace_probe_log_err(0, DIFF_PROBE_TYPE);
638ca89bc07SMasami Hiramatsu ret = -EEXIST;
639ca89bc07SMasami Hiramatsu } else {
640ca89bc07SMasami Hiramatsu ret = append_trace_kprobe(tk, old_tk);
641ca89bc07SMasami Hiramatsu }
64202ca1521SMasami Hiramatsu goto end;
6432d5e067eSMasami Hiramatsu }
64461424318SMasami Hiramatsu
64561424318SMasami Hiramatsu /* Register new event */
646c31ffb3fSNamhyung Kim ret = register_kprobe_event(tk);
6472d5e067eSMasami Hiramatsu if (ret) {
6488e242060SMasami Hiramatsu if (ret == -EEXIST) {
6498e242060SMasami Hiramatsu trace_probe_log_set_index(0);
6508e242060SMasami Hiramatsu trace_probe_log_err(0, EVENT_EXIST);
6518e242060SMasami Hiramatsu } else
652a395d6a7SJoe Perches pr_warn("Failed to register probe event(%d)\n", ret);
6532d5e067eSMasami Hiramatsu goto end;
6542d5e067eSMasami Hiramatsu }
6552d5e067eSMasami Hiramatsu
65661424318SMasami Hiramatsu /* Register k*probe */
657c31ffb3fSNamhyung Kim ret = __register_trace_kprobe(tk);
65859158ec4SMasami Hiramatsu if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {
65959158ec4SMasami Hiramatsu pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
66059158ec4SMasami Hiramatsu ret = 0;
66159158ec4SMasami Hiramatsu }
66259158ec4SMasami Hiramatsu
66361424318SMasami Hiramatsu if (ret < 0)
664c31ffb3fSNamhyung Kim unregister_kprobe_event(tk);
66561424318SMasami Hiramatsu else
6668b0e6c74SSteven Rostedt (VMware) dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp));
66761424318SMasami Hiramatsu
668413d37d1SMasami Hiramatsu end:
6696212dd29SMasami Hiramatsu mutex_unlock(&event_mutex);
670413d37d1SMasami Hiramatsu return ret;
671413d37d1SMasami Hiramatsu }
672413d37d1SMasami Hiramatsu
67361424318SMasami Hiramatsu /* Module notifier call back, checking event on the module */
trace_kprobe_module_callback(struct notifier_block * nb,unsigned long val,void * data)674c31ffb3fSNamhyung Kim static int trace_kprobe_module_callback(struct notifier_block *nb,
67561424318SMasami Hiramatsu unsigned long val, void *data)
67661424318SMasami Hiramatsu {
67761424318SMasami Hiramatsu struct module *mod = data;
6786212dd29SMasami Hiramatsu struct dyn_event *pos;
679c31ffb3fSNamhyung Kim struct trace_kprobe *tk;
68061424318SMasami Hiramatsu int ret;
68161424318SMasami Hiramatsu
68261424318SMasami Hiramatsu if (val != MODULE_STATE_COMING)
68361424318SMasami Hiramatsu return NOTIFY_DONE;
68461424318SMasami Hiramatsu
68561424318SMasami Hiramatsu /* Update probes on coming module */
6866212dd29SMasami Hiramatsu mutex_lock(&event_mutex);
6876212dd29SMasami Hiramatsu for_each_trace_kprobe(tk, pos) {
688c31ffb3fSNamhyung Kim if (trace_kprobe_within_module(tk, mod)) {
68902ca1521SMasami Hiramatsu /* Don't need to check busy - this should have gone. */
690c31ffb3fSNamhyung Kim __unregister_trace_kprobe(tk);
691c31ffb3fSNamhyung Kim ret = __register_trace_kprobe(tk);
69261424318SMasami Hiramatsu if (ret)
693a395d6a7SJoe Perches pr_warn("Failed to re-register probe %s on %s: %d\n",
694b55ce203SMasami Hiramatsu trace_probe_name(&tk->tp),
695f3d36426SJarkko Sakkinen module_name(mod), ret);
69661424318SMasami Hiramatsu }
69761424318SMasami Hiramatsu }
6986212dd29SMasami Hiramatsu mutex_unlock(&event_mutex);
69961424318SMasami Hiramatsu
70061424318SMasami Hiramatsu return NOTIFY_DONE;
70161424318SMasami Hiramatsu }
70261424318SMasami Hiramatsu
703c31ffb3fSNamhyung Kim static struct notifier_block trace_kprobe_module_nb = {
704c31ffb3fSNamhyung Kim .notifier_call = trace_kprobe_module_callback,
70561424318SMasami Hiramatsu .priority = 1 /* Invoked after kprobe module callback */
70661424318SMasami Hiramatsu };
70761424318SMasami Hiramatsu
count_symbols(void * data,unsigned long unused)708b022f0c7SFrancis Laniel static int count_symbols(void *data, unsigned long unused)
709b022f0c7SFrancis Laniel {
710b022f0c7SFrancis Laniel unsigned int *count = data;
711b022f0c7SFrancis Laniel
712b022f0c7SFrancis Laniel (*count)++;
713b022f0c7SFrancis Laniel
714b022f0c7SFrancis Laniel return 0;
715b022f0c7SFrancis Laniel }
716b022f0c7SFrancis Laniel
717926fe783SAndrii Nakryiko struct sym_count_ctx {
718926fe783SAndrii Nakryiko unsigned int count;
719926fe783SAndrii Nakryiko const char *name;
720926fe783SAndrii Nakryiko };
721926fe783SAndrii Nakryiko
count_mod_symbols(void * data,const char * name,unsigned long unused)722926fe783SAndrii Nakryiko static int count_mod_symbols(void *data, const char *name, unsigned long unused)
723926fe783SAndrii Nakryiko {
724926fe783SAndrii Nakryiko struct sym_count_ctx *ctx = data;
725926fe783SAndrii Nakryiko
726926fe783SAndrii Nakryiko if (strcmp(name, ctx->name) == 0)
727926fe783SAndrii Nakryiko ctx->count++;
728926fe783SAndrii Nakryiko
729926fe783SAndrii Nakryiko return 0;
730926fe783SAndrii Nakryiko }
731926fe783SAndrii Nakryiko
number_of_same_symbols(char * func_name)732b022f0c7SFrancis Laniel static unsigned int number_of_same_symbols(char *func_name)
733b022f0c7SFrancis Laniel {
734926fe783SAndrii Nakryiko struct sym_count_ctx ctx = { .count = 0, .name = func_name };
735b022f0c7SFrancis Laniel
736926fe783SAndrii Nakryiko kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count);
737b022f0c7SFrancis Laniel
738926fe783SAndrii Nakryiko module_kallsyms_on_each_symbol(NULL, count_mod_symbols, &ctx);
739926fe783SAndrii Nakryiko
740926fe783SAndrii Nakryiko return ctx.count;
741b022f0c7SFrancis Laniel }
742b022f0c7SFrancis Laniel
__trace_kprobe_create(int argc,const char * argv[])743d262271dSMasami Hiramatsu static int __trace_kprobe_create(int argc, const char *argv[])
744413d37d1SMasami Hiramatsu {
745413d37d1SMasami Hiramatsu /*
746413d37d1SMasami Hiramatsu * Argument syntax:
747696ced4fSAlban Crequy * - Add kprobe:
74895c104c3SLinyu Yuan * p[:[GRP/][EVENT]] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
749696ced4fSAlban Crequy * - Add kretprobe:
75095c104c3SLinyu Yuan * r[MAXACTIVE][:[GRP/][EVENT]] [MOD:]KSYM[+0] [FETCHARGS]
7514725cd89SMasami Hiramatsu * Or
75295c104c3SLinyu Yuan * p[:[GRP/][EVENT]] [MOD:]KSYM[+0]%return [FETCHARGS]
7534725cd89SMasami Hiramatsu *
754413d37d1SMasami Hiramatsu * Fetch args:
7552e06ff63SMasami Hiramatsu * $retval : fetch return value
7562e06ff63SMasami Hiramatsu * $stack : fetch stack address
7572e06ff63SMasami Hiramatsu * $stackN : fetch Nth of stack (N:0-)
75835abb67dSOmar Sandoval * $comm : fetch current task comm
759413d37d1SMasami Hiramatsu * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
760413d37d1SMasami Hiramatsu * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
761413d37d1SMasami Hiramatsu * %REG : fetch register REG
76293ccae7aSMasami Hiramatsu * Dereferencing memory fetch:
763413d37d1SMasami Hiramatsu * +|-offs(ARG) : fetch memory at ARG +|- offs address.
764eca0d916SMasami Hiramatsu * Alias name of args:
765eca0d916SMasami Hiramatsu * NAME=FETCHARG : set NAME as alias of FETCHARG.
76693ccae7aSMasami Hiramatsu * Type of args:
76793ccae7aSMasami Hiramatsu * FETCHARG:TYPE : use TYPE instead of unsigned long.
768413d37d1SMasami Hiramatsu */
769ab105a4fSMasami Hiramatsu struct trace_kprobe *tk = NULL;
77018b1e870SMasami Hiramatsu (Google) int i, len, new_argc = 0, ret = 0;
7716212dd29SMasami Hiramatsu bool is_return = false;
7726212dd29SMasami Hiramatsu char *symbol = NULL, *tmp = NULL;
77318b1e870SMasami Hiramatsu (Google) const char **new_argv = NULL;
7746212dd29SMasami Hiramatsu const char *event = NULL, *group = KPROBE_EVENT_SYSTEM;
775007517a0SSteven Rostedt (VMware) enum probe_print_type ptype;
776696ced4fSAlban Crequy int maxactive = 0;
777c5d343b6SMasami Hiramatsu long offset = 0;
778413d37d1SMasami Hiramatsu void *addr = NULL;
7794a846b44SMasami Hiramatsu char buf[MAX_EVENT_NAME_LEN];
78095c104c3SLinyu Yuan char gbuf[MAX_EVENT_NAME_LEN];
78118b1e870SMasami Hiramatsu (Google) char abuf[MAX_BTF_ARGS_LEN];
782b576e097SMasami Hiramatsu (Google) struct traceprobe_parse_context ctx = { .flags = TPARG_FL_KERNEL };
783413d37d1SMasami Hiramatsu
7848b05a3a7SAndrea Righi switch (argv[0][0]) {
7858b05a3a7SAndrea Righi case 'r':
7863a6b7666SSrikar Dronamraju is_return = true;
7878b05a3a7SAndrea Righi break;
7888b05a3a7SAndrea Righi case 'p':
7898b05a3a7SAndrea Righi break;
7908b05a3a7SAndrea Righi default:
7918b05a3a7SAndrea Righi return -ECANCELED;
7928b05a3a7SAndrea Righi }
7938b05a3a7SAndrea Righi if (argc < 2)
7946212dd29SMasami Hiramatsu return -ECANCELED;
795413d37d1SMasami Hiramatsu
796ab105a4fSMasami Hiramatsu trace_probe_log_init("trace_kprobe", argc, argv);
797ab105a4fSMasami Hiramatsu
798696ced4fSAlban Crequy event = strchr(&argv[0][1], ':');
7996212dd29SMasami Hiramatsu if (event)
800696ced4fSAlban Crequy event++;
8016212dd29SMasami Hiramatsu
802287c038cSMasami Hiramatsu if (isdigit(argv[0][1])) {
803287c038cSMasami Hiramatsu if (!is_return) {
804334e5519SMasami Hiramatsu (Google) trace_probe_log_err(1, BAD_MAXACT_TYPE);
805ab105a4fSMasami Hiramatsu goto parse_error;
806287c038cSMasami Hiramatsu }
8076212dd29SMasami Hiramatsu if (event)
8086212dd29SMasami Hiramatsu len = event - &argv[0][1] - 1;
8096212dd29SMasami Hiramatsu else
8106212dd29SMasami Hiramatsu len = strlen(&argv[0][1]);
811ab105a4fSMasami Hiramatsu if (len > MAX_EVENT_NAME_LEN - 1) {
812ab105a4fSMasami Hiramatsu trace_probe_log_err(1, BAD_MAXACT);
813ab105a4fSMasami Hiramatsu goto parse_error;
814ab105a4fSMasami Hiramatsu }
8156212dd29SMasami Hiramatsu memcpy(buf, &argv[0][1], len);
8166212dd29SMasami Hiramatsu buf[len] = '\0';
8176212dd29SMasami Hiramatsu ret = kstrtouint(buf, 0, &maxactive);
818287c038cSMasami Hiramatsu if (ret || !maxactive) {
819ab105a4fSMasami Hiramatsu trace_probe_log_err(1, BAD_MAXACT);
820ab105a4fSMasami Hiramatsu goto parse_error;
821696ced4fSAlban Crequy }
822696ced4fSAlban Crequy /* kretprobes instances are iterated over via a list. The
823696ced4fSAlban Crequy * maximum should stay reasonable.
824696ced4fSAlban Crequy */
825696ced4fSAlban Crequy if (maxactive > KRETPROBE_MAXACTIVE_MAX) {
826ab105a4fSMasami Hiramatsu trace_probe_log_err(1, MAXACT_TOO_BIG);
827ab105a4fSMasami Hiramatsu goto parse_error;
828696ced4fSAlban Crequy }
829696ced4fSAlban Crequy }
830696ced4fSAlban Crequy
8319e52b325SSabrina Dubroca /* try to parse an address. if that fails, try to read the
8329e52b325SSabrina Dubroca * input as a symbol. */
8339e52b325SSabrina Dubroca if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) {
834ab105a4fSMasami Hiramatsu trace_probe_log_set_index(1);
8356212dd29SMasami Hiramatsu /* Check whether uprobe event specified */
836ab105a4fSMasami Hiramatsu if (strchr(argv[1], '/') && strchr(argv[1], ':')) {
837ab105a4fSMasami Hiramatsu ret = -ECANCELED;
838ab105a4fSMasami Hiramatsu goto error;
839ab105a4fSMasami Hiramatsu }
840413d37d1SMasami Hiramatsu /* a symbol specified */
8416212dd29SMasami Hiramatsu symbol = kstrdup(argv[1], GFP_KERNEL);
8426212dd29SMasami Hiramatsu if (!symbol)
8436212dd29SMasami Hiramatsu return -ENOMEM;
8444725cd89SMasami Hiramatsu
8454725cd89SMasami Hiramatsu tmp = strchr(symbol, '%');
8464725cd89SMasami Hiramatsu if (tmp) {
8474725cd89SMasami Hiramatsu if (!strcmp(tmp, "%return")) {
8484725cd89SMasami Hiramatsu *tmp = '\0';
8494725cd89SMasami Hiramatsu is_return = true;
8504725cd89SMasami Hiramatsu } else {
8514725cd89SMasami Hiramatsu trace_probe_log_err(tmp - symbol, BAD_ADDR_SUFFIX);
8524725cd89SMasami Hiramatsu goto parse_error;
8534725cd89SMasami Hiramatsu }
8544725cd89SMasami Hiramatsu }
8554725cd89SMasami Hiramatsu
856413d37d1SMasami Hiramatsu /* TODO: support .init module functions */
8578ab83f56SSrikar Dronamraju ret = traceprobe_split_symbol_offset(symbol, &offset);
858c5d343b6SMasami Hiramatsu if (ret || offset < 0 || offset > UINT_MAX) {
859ab105a4fSMasami Hiramatsu trace_probe_log_err(0, BAD_PROBE_ADDR);
860ab105a4fSMasami Hiramatsu goto parse_error;
861e63cc239SMasami Hiramatsu }
8624725cd89SMasami Hiramatsu if (is_return)
863b576e097SMasami Hiramatsu (Google) ctx.flags |= TPARG_FL_RETURN;
86497c753e6SMasami Hiramatsu ret = kprobe_on_func_entry(NULL, symbol, offset);
86530460c21SMasami Hiramatsu (Google) if (ret == 0 && !is_return)
866b576e097SMasami Hiramatsu (Google) ctx.flags |= TPARG_FL_FENTRY;
86797c753e6SMasami Hiramatsu /* Defer the ENOENT case until register kprobe */
86897c753e6SMasami Hiramatsu if (ret == -EINVAL && is_return) {
869ab105a4fSMasami Hiramatsu trace_probe_log_err(0, BAD_RETPROBE);
870ab105a4fSMasami Hiramatsu goto parse_error;
871413d37d1SMasami Hiramatsu }
872e63cc239SMasami Hiramatsu }
873413d37d1SMasami Hiramatsu
874b022f0c7SFrancis Laniel if (symbol && !strchr(symbol, ':')) {
875b022f0c7SFrancis Laniel unsigned int count;
876b022f0c7SFrancis Laniel
877b022f0c7SFrancis Laniel count = number_of_same_symbols(symbol);
878b022f0c7SFrancis Laniel if (count > 1) {
879b022f0c7SFrancis Laniel /*
880b022f0c7SFrancis Laniel * Users should use ADDR to remove the ambiguity of
881b022f0c7SFrancis Laniel * using KSYM only.
882b022f0c7SFrancis Laniel */
883b022f0c7SFrancis Laniel trace_probe_log_err(0, NON_UNIQ_SYMBOL);
884b022f0c7SFrancis Laniel ret = -EADDRNOTAVAIL;
885b022f0c7SFrancis Laniel
886b022f0c7SFrancis Laniel goto error;
887b022f0c7SFrancis Laniel } else if (count == 0) {
888b022f0c7SFrancis Laniel /*
889b022f0c7SFrancis Laniel * We can return ENOENT earlier than when register the
890b022f0c7SFrancis Laniel * kprobe.
891b022f0c7SFrancis Laniel */
892b022f0c7SFrancis Laniel trace_probe_log_err(0, BAD_PROBE_ADDR);
893b022f0c7SFrancis Laniel ret = -ENOENT;
894b022f0c7SFrancis Laniel
895b022f0c7SFrancis Laniel goto error;
896b022f0c7SFrancis Laniel }
897b022f0c7SFrancis Laniel }
898b022f0c7SFrancis Laniel
899ab105a4fSMasami Hiramatsu trace_probe_log_set_index(0);
9006212dd29SMasami Hiramatsu if (event) {
90195c104c3SLinyu Yuan ret = traceprobe_parse_event_name(&event, &group, gbuf,
902ab105a4fSMasami Hiramatsu event - argv[0]);
9036212dd29SMasami Hiramatsu if (ret)
904ab105a4fSMasami Hiramatsu goto parse_error;
90595c104c3SLinyu Yuan }
90695c104c3SLinyu Yuan
90795c104c3SLinyu Yuan if (!event) {
9084263565dSMasami Hiramatsu /* Make a new event name */
9094263565dSMasami Hiramatsu if (symbol)
9106f3cf440SMasami Hiramatsu snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
9114263565dSMasami Hiramatsu is_return ? 'r' : 'p', symbol, offset);
9124263565dSMasami Hiramatsu else
9136f3cf440SMasami Hiramatsu snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p",
9144263565dSMasami Hiramatsu is_return ? 'r' : 'p', addr);
915fca18a47SNaveen N. Rao sanitize_event_name(buf);
9164a846b44SMasami Hiramatsu event = buf;
9174a846b44SMasami Hiramatsu }
9186212dd29SMasami Hiramatsu
91918b1e870SMasami Hiramatsu (Google) argc -= 2; argv += 2;
92018b1e870SMasami Hiramatsu (Google) ctx.funcname = symbol;
92118b1e870SMasami Hiramatsu (Google) new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc,
92218b1e870SMasami Hiramatsu (Google) abuf, MAX_BTF_ARGS_LEN, &ctx);
92318b1e870SMasami Hiramatsu (Google) if (IS_ERR(new_argv)) {
92418b1e870SMasami Hiramatsu (Google) ret = PTR_ERR(new_argv);
92518b1e870SMasami Hiramatsu (Google) new_argv = NULL;
92618b1e870SMasami Hiramatsu (Google) goto out;
92718b1e870SMasami Hiramatsu (Google) }
92818b1e870SMasami Hiramatsu (Google) if (new_argv) {
92918b1e870SMasami Hiramatsu (Google) argc = new_argc;
93018b1e870SMasami Hiramatsu (Google) argv = new_argv;
93118b1e870SMasami Hiramatsu (Google) }
93218b1e870SMasami Hiramatsu (Google)
9336212dd29SMasami Hiramatsu /* setup a probe */
934696ced4fSAlban Crequy tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive,
93518b1e870SMasami Hiramatsu (Google) argc, is_return);
936c31ffb3fSNamhyung Kim if (IS_ERR(tk)) {
9376212dd29SMasami Hiramatsu ret = PTR_ERR(tk);
938ab105a4fSMasami Hiramatsu /* This must return -ENOMEM, else there is a bug */
939a039480eSMasami Hiramatsu WARN_ON_ONCE(ret != -ENOMEM);
940ab105a4fSMasami Hiramatsu goto out; /* We know tk is not allocated */
941e63cc239SMasami Hiramatsu }
942413d37d1SMasami Hiramatsu
943413d37d1SMasami Hiramatsu /* parse arguments */
944a82378d8SMasami Hiramatsu for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
945ab105a4fSMasami Hiramatsu trace_probe_log_set_index(i + 2);
946b576e097SMasami Hiramatsu (Google) ctx.offset = 0;
9471b8b0cd7SMasami Hiramatsu (Google) ret = traceprobe_parse_probe_arg(&tk->tp, i, argv[i], &ctx);
948d00bbea9SMasami Hiramatsu if (ret)
949ab105a4fSMasami Hiramatsu goto error; /* This can be -ENOMEM */
950da34634fSMasami Hiramatsu }
95193ccae7aSMasami Hiramatsu
952007517a0SSteven Rostedt (VMware) ptype = is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
953007517a0SSteven Rostedt (VMware) ret = traceprobe_set_print_fmt(&tk->tp, ptype);
954f730e0f2SMasami Hiramatsu if (ret < 0)
955f730e0f2SMasami Hiramatsu goto error;
956f730e0f2SMasami Hiramatsu
957c31ffb3fSNamhyung Kim ret = register_trace_kprobe(tk);
958ab105a4fSMasami Hiramatsu if (ret) {
959ab105a4fSMasami Hiramatsu trace_probe_log_set_index(1);
960ab105a4fSMasami Hiramatsu if (ret == -EILSEQ)
961ab105a4fSMasami Hiramatsu trace_probe_log_err(0, BAD_INSN_BNDRY);
962ab105a4fSMasami Hiramatsu else if (ret == -ENOENT)
963ab105a4fSMasami Hiramatsu trace_probe_log_err(0, BAD_PROBE_ADDR);
964ca89bc07SMasami Hiramatsu else if (ret != -ENOMEM && ret != -EEXIST)
965ab105a4fSMasami Hiramatsu trace_probe_log_err(0, FAIL_REG_PROBE);
966413d37d1SMasami Hiramatsu goto error;
967ab105a4fSMasami Hiramatsu }
968ab105a4fSMasami Hiramatsu
9696212dd29SMasami Hiramatsu out:
970b1d1e904SMasami Hiramatsu (Google) traceprobe_finish_parse(&ctx);
971ab105a4fSMasami Hiramatsu trace_probe_log_clear();
97218b1e870SMasami Hiramatsu (Google) kfree(new_argv);
9736212dd29SMasami Hiramatsu kfree(symbol);
9746212dd29SMasami Hiramatsu return ret;
975413d37d1SMasami Hiramatsu
976ab105a4fSMasami Hiramatsu parse_error:
977ab105a4fSMasami Hiramatsu ret = -EINVAL;
978413d37d1SMasami Hiramatsu error:
979c31ffb3fSNamhyung Kim free_trace_kprobe(tk);
9806212dd29SMasami Hiramatsu goto out;
981413d37d1SMasami Hiramatsu }
982413d37d1SMasami Hiramatsu
trace_kprobe_create(const char * raw_command)983d262271dSMasami Hiramatsu static int trace_kprobe_create(const char *raw_command)
984d262271dSMasami Hiramatsu {
985d262271dSMasami Hiramatsu return trace_probe_create(raw_command, __trace_kprobe_create);
986d262271dSMasami Hiramatsu }
987d262271dSMasami Hiramatsu
create_or_delete_trace_kprobe(const char * raw_command)988d262271dSMasami Hiramatsu static int create_or_delete_trace_kprobe(const char *raw_command)
989413d37d1SMasami Hiramatsu {
9906212dd29SMasami Hiramatsu int ret;
991413d37d1SMasami Hiramatsu
992d262271dSMasami Hiramatsu if (raw_command[0] == '-')
993d262271dSMasami Hiramatsu return dyn_event_release(raw_command, &trace_kprobe_ops);
9946212dd29SMasami Hiramatsu
995d262271dSMasami Hiramatsu ret = trace_kprobe_create(raw_command);
9966212dd29SMasami Hiramatsu return ret == -ECANCELED ? -EINVAL : ret;
99702ca1521SMasami Hiramatsu }
9986212dd29SMasami Hiramatsu
trace_kprobe_run_command(struct dynevent_cmd * cmd)99929a15481STom Zanussi static int trace_kprobe_run_command(struct dynevent_cmd *cmd)
10002a588dd1STom Zanussi {
1001d262271dSMasami Hiramatsu return create_or_delete_trace_kprobe(cmd->seq.buffer);
10022a588dd1STom Zanussi }
10032a588dd1STom Zanussi
10042a588dd1STom Zanussi /**
10052a588dd1STom Zanussi * kprobe_event_cmd_init - Initialize a kprobe event command object
10062a588dd1STom Zanussi * @cmd: A pointer to the dynevent_cmd struct representing the new event
10072a588dd1STom Zanussi * @buf: A pointer to the buffer used to build the command
10082a588dd1STom Zanussi * @maxlen: The length of the buffer passed in @buf
10092a588dd1STom Zanussi *
10102a588dd1STom Zanussi * Initialize a synthetic event command object. Use this before
10112a588dd1STom Zanussi * calling any of the other kprobe_event functions.
10122a588dd1STom Zanussi */
kprobe_event_cmd_init(struct dynevent_cmd * cmd,char * buf,int maxlen)10132a588dd1STom Zanussi void kprobe_event_cmd_init(struct dynevent_cmd *cmd, char *buf, int maxlen)
10142a588dd1STom Zanussi {
10152a588dd1STom Zanussi dynevent_cmd_init(cmd, buf, maxlen, DYNEVENT_TYPE_KPROBE,
101629a15481STom Zanussi trace_kprobe_run_command);
10172a588dd1STom Zanussi }
10182a588dd1STom Zanussi EXPORT_SYMBOL_GPL(kprobe_event_cmd_init);
10192a588dd1STom Zanussi
10202a588dd1STom Zanussi /**
10212a588dd1STom Zanussi * __kprobe_event_gen_cmd_start - Generate a kprobe event command from arg list
10222a588dd1STom Zanussi * @cmd: A pointer to the dynevent_cmd struct representing the new event
1023*dc43609dSYujie Liu * @kretprobe: Is this a return probe?
10242a588dd1STom Zanussi * @name: The name of the kprobe event
10252a588dd1STom Zanussi * @loc: The location of the kprobe event
1026e0f83183SYujie Liu * @...: Variable number of arg (pairs), one pair for each field
10272a588dd1STom Zanussi *
10282a588dd1STom Zanussi * NOTE: Users normally won't want to call this function directly, but
10292a588dd1STom Zanussi * rather use the kprobe_event_gen_cmd_start() wrapper, which automatically
10302a588dd1STom Zanussi * adds a NULL to the end of the arg list. If this function is used
10312a588dd1STom Zanussi * directly, make sure the last arg in the variable arg list is NULL.
10322a588dd1STom Zanussi *
10332a588dd1STom Zanussi * Generate a kprobe event command to be executed by
10342a588dd1STom Zanussi * kprobe_event_gen_cmd_end(). This function can be used to generate the
10352a588dd1STom Zanussi * complete command or only the first part of it; in the latter case,
10362a588dd1STom Zanussi * kprobe_event_add_fields() can be used to add more fields following this.
10372a588dd1STom Zanussi *
10385b4dcd2dSMasami Hiramatsu * Unlikely the synth_event_gen_cmd_start(), @loc must be specified. This
10395b4dcd2dSMasami Hiramatsu * returns -EINVAL if @loc == NULL.
10405b4dcd2dSMasami Hiramatsu *
10412a588dd1STom Zanussi * Return: 0 if successful, error otherwise.
10422a588dd1STom Zanussi */
__kprobe_event_gen_cmd_start(struct dynevent_cmd * cmd,bool kretprobe,const char * name,const char * loc,...)10432a588dd1STom Zanussi int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd, bool kretprobe,
10442a588dd1STom Zanussi const char *name, const char *loc, ...)
10452a588dd1STom Zanussi {
10462a588dd1STom Zanussi char buf[MAX_EVENT_NAME_LEN];
10472a588dd1STom Zanussi struct dynevent_arg arg;
10482a588dd1STom Zanussi va_list args;
10492a588dd1STom Zanussi int ret;
10502a588dd1STom Zanussi
10512a588dd1STom Zanussi if (cmd->type != DYNEVENT_TYPE_KPROBE)
10522a588dd1STom Zanussi return -EINVAL;
10532a588dd1STom Zanussi
10545b4dcd2dSMasami Hiramatsu if (!loc)
10555b4dcd2dSMasami Hiramatsu return -EINVAL;
10565b4dcd2dSMasami Hiramatsu
10572a588dd1STom Zanussi if (kretprobe)
10582a588dd1STom Zanussi snprintf(buf, MAX_EVENT_NAME_LEN, "r:kprobes/%s", name);
10592a588dd1STom Zanussi else
10602a588dd1STom Zanussi snprintf(buf, MAX_EVENT_NAME_LEN, "p:kprobes/%s", name);
10612a588dd1STom Zanussi
10622a588dd1STom Zanussi ret = dynevent_str_add(cmd, buf);
10632a588dd1STom Zanussi if (ret)
10642a588dd1STom Zanussi return ret;
10652a588dd1STom Zanussi
106674403b6cSTom Zanussi dynevent_arg_init(&arg, 0);
10672a588dd1STom Zanussi arg.str = loc;
106874403b6cSTom Zanussi ret = dynevent_arg_add(cmd, &arg, NULL);
10692a588dd1STom Zanussi if (ret)
10702a588dd1STom Zanussi return ret;
10712a588dd1STom Zanussi
10722a588dd1STom Zanussi va_start(args, loc);
10732a588dd1STom Zanussi for (;;) {
10742a588dd1STom Zanussi const char *field;
10752a588dd1STom Zanussi
10762a588dd1STom Zanussi field = va_arg(args, const char *);
10772a588dd1STom Zanussi if (!field)
10782a588dd1STom Zanussi break;
10792a588dd1STom Zanussi
10802a588dd1STom Zanussi if (++cmd->n_fields > MAX_TRACE_ARGS) {
10812a588dd1STom Zanussi ret = -EINVAL;
10822a588dd1STom Zanussi break;
10832a588dd1STom Zanussi }
10842a588dd1STom Zanussi
10852a588dd1STom Zanussi arg.str = field;
108674403b6cSTom Zanussi ret = dynevent_arg_add(cmd, &arg, NULL);
10872a588dd1STom Zanussi if (ret)
10882a588dd1STom Zanussi break;
10892a588dd1STom Zanussi }
10902a588dd1STom Zanussi va_end(args);
10912a588dd1STom Zanussi
10922a588dd1STom Zanussi return ret;
10932a588dd1STom Zanussi }
10942a588dd1STom Zanussi EXPORT_SYMBOL_GPL(__kprobe_event_gen_cmd_start);
10952a588dd1STom Zanussi
10962a588dd1STom Zanussi /**
10972a588dd1STom Zanussi * __kprobe_event_add_fields - Add probe fields to a kprobe command from arg list
10982a588dd1STom Zanussi * @cmd: A pointer to the dynevent_cmd struct representing the new event
1099e0f83183SYujie Liu * @...: Variable number of arg (pairs), one pair for each field
11002a588dd1STom Zanussi *
11012a588dd1STom Zanussi * NOTE: Users normally won't want to call this function directly, but
11022a588dd1STom Zanussi * rather use the kprobe_event_add_fields() wrapper, which
11032a588dd1STom Zanussi * automatically adds a NULL to the end of the arg list. If this
11042a588dd1STom Zanussi * function is used directly, make sure the last arg in the variable
11052a588dd1STom Zanussi * arg list is NULL.
11062a588dd1STom Zanussi *
11072a588dd1STom Zanussi * Add probe fields to an existing kprobe command using a variable
11082a588dd1STom Zanussi * list of args. Fields are added in the same order they're listed.
11092a588dd1STom Zanussi *
11102a588dd1STom Zanussi * Return: 0 if successful, error otherwise.
11112a588dd1STom Zanussi */
__kprobe_event_add_fields(struct dynevent_cmd * cmd,...)11122a588dd1STom Zanussi int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...)
11132a588dd1STom Zanussi {
11142a588dd1STom Zanussi struct dynevent_arg arg;
11152a588dd1STom Zanussi va_list args;
111610f129cbSGustavo A. R. Silva int ret = 0;
11172a588dd1STom Zanussi
11182a588dd1STom Zanussi if (cmd->type != DYNEVENT_TYPE_KPROBE)
11192a588dd1STom Zanussi return -EINVAL;
11202a588dd1STom Zanussi
112174403b6cSTom Zanussi dynevent_arg_init(&arg, 0);
11222a588dd1STom Zanussi
11232a588dd1STom Zanussi va_start(args, cmd);
11242a588dd1STom Zanussi for (;;) {
11252a588dd1STom Zanussi const char *field;
11262a588dd1STom Zanussi
11272a588dd1STom Zanussi field = va_arg(args, const char *);
11282a588dd1STom Zanussi if (!field)
11292a588dd1STom Zanussi break;
11302a588dd1STom Zanussi
11312a588dd1STom Zanussi if (++cmd->n_fields > MAX_TRACE_ARGS) {
11322a588dd1STom Zanussi ret = -EINVAL;
11332a588dd1STom Zanussi break;
11342a588dd1STom Zanussi }
11352a588dd1STom Zanussi
11362a588dd1STom Zanussi arg.str = field;
113774403b6cSTom Zanussi ret = dynevent_arg_add(cmd, &arg, NULL);
11382a588dd1STom Zanussi if (ret)
11392a588dd1STom Zanussi break;
11402a588dd1STom Zanussi }
11412a588dd1STom Zanussi va_end(args);
11422a588dd1STom Zanussi
11432a588dd1STom Zanussi return ret;
11442a588dd1STom Zanussi }
11452a588dd1STom Zanussi EXPORT_SYMBOL_GPL(__kprobe_event_add_fields);
11462a588dd1STom Zanussi
11472a588dd1STom Zanussi /**
11482a588dd1STom Zanussi * kprobe_event_delete - Delete a kprobe event
11492a588dd1STom Zanussi * @name: The name of the kprobe event to delete
11502a588dd1STom Zanussi *
11512a588dd1STom Zanussi * Delete a kprobe event with the give @name from kernel code rather
11522a588dd1STom Zanussi * than directly from the command line.
11532a588dd1STom Zanussi *
11542a588dd1STom Zanussi * Return: 0 if successful, error otherwise.
11552a588dd1STom Zanussi */
kprobe_event_delete(const char * name)11562a588dd1STom Zanussi int kprobe_event_delete(const char *name)
11572a588dd1STom Zanussi {
11582a588dd1STom Zanussi char buf[MAX_EVENT_NAME_LEN];
11592a588dd1STom Zanussi
11602a588dd1STom Zanussi snprintf(buf, MAX_EVENT_NAME_LEN, "-:%s", name);
11612a588dd1STom Zanussi
1162d262271dSMasami Hiramatsu return create_or_delete_trace_kprobe(buf);
11632a588dd1STom Zanussi }
11642a588dd1STom Zanussi EXPORT_SYMBOL_GPL(kprobe_event_delete);
11652a588dd1STom Zanussi
trace_kprobe_release(struct dyn_event * ev)11666212dd29SMasami Hiramatsu static int trace_kprobe_release(struct dyn_event *ev)
11676212dd29SMasami Hiramatsu {
11686212dd29SMasami Hiramatsu struct trace_kprobe *tk = to_trace_kprobe(ev);
11696212dd29SMasami Hiramatsu int ret = unregister_trace_kprobe(tk);
11706212dd29SMasami Hiramatsu
11716212dd29SMasami Hiramatsu if (!ret)
1172c31ffb3fSNamhyung Kim free_trace_kprobe(tk);
117302ca1521SMasami Hiramatsu return ret;
1174413d37d1SMasami Hiramatsu }
1175413d37d1SMasami Hiramatsu
trace_kprobe_show(struct seq_file * m,struct dyn_event * ev)11766212dd29SMasami Hiramatsu static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev)
1177413d37d1SMasami Hiramatsu {
11786212dd29SMasami Hiramatsu struct trace_kprobe *tk = to_trace_kprobe(ev);
117993ccae7aSMasami Hiramatsu int i;
1180413d37d1SMasami Hiramatsu
1181fa6f0cc7SRasmus Villemoes seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p');
11826a13a0d7SMasami Hiramatsu if (trace_kprobe_is_return(tk) && tk->rp.maxactive)
11836a13a0d7SMasami Hiramatsu seq_printf(m, "%d", tk->rp.maxactive);
1184b55ce203SMasami Hiramatsu seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp),
1185b55ce203SMasami Hiramatsu trace_probe_name(&tk->tp));
1186413d37d1SMasami Hiramatsu
1187c31ffb3fSNamhyung Kim if (!tk->symbol)
1188c31ffb3fSNamhyung Kim seq_printf(m, " 0x%p", tk->rp.kp.addr);
1189c31ffb3fSNamhyung Kim else if (tk->rp.kp.offset)
1190c31ffb3fSNamhyung Kim seq_printf(m, " %s+%u", trace_kprobe_symbol(tk),
1191c31ffb3fSNamhyung Kim tk->rp.kp.offset);
1192413d37d1SMasami Hiramatsu else
1193c31ffb3fSNamhyung Kim seq_printf(m, " %s", trace_kprobe_symbol(tk));
1194413d37d1SMasami Hiramatsu
1195c31ffb3fSNamhyung Kim for (i = 0; i < tk->tp.nr_args; i++)
1196c31ffb3fSNamhyung Kim seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
1197fa6f0cc7SRasmus Villemoes seq_putc(m, '\n');
119893ccae7aSMasami Hiramatsu
1199413d37d1SMasami Hiramatsu return 0;
1200413d37d1SMasami Hiramatsu }
1201413d37d1SMasami Hiramatsu
probes_seq_show(struct seq_file * m,void * v)12026212dd29SMasami Hiramatsu static int probes_seq_show(struct seq_file *m, void *v)
12036212dd29SMasami Hiramatsu {
12046212dd29SMasami Hiramatsu struct dyn_event *ev = v;
12056212dd29SMasami Hiramatsu
12066212dd29SMasami Hiramatsu if (!is_trace_kprobe(ev))
12076212dd29SMasami Hiramatsu return 0;
12086212dd29SMasami Hiramatsu
12096212dd29SMasami Hiramatsu return trace_kprobe_show(m, ev);
12106212dd29SMasami Hiramatsu }
12116212dd29SMasami Hiramatsu
1212413d37d1SMasami Hiramatsu static const struct seq_operations probes_seq_op = {
12136212dd29SMasami Hiramatsu .start = dyn_event_seq_start,
12146212dd29SMasami Hiramatsu .next = dyn_event_seq_next,
12156212dd29SMasami Hiramatsu .stop = dyn_event_seq_stop,
1216413d37d1SMasami Hiramatsu .show = probes_seq_show
1217413d37d1SMasami Hiramatsu };
1218413d37d1SMasami Hiramatsu
probes_open(struct inode * inode,struct file * file)1219413d37d1SMasami Hiramatsu static int probes_open(struct inode *inode, struct file *file)
1220413d37d1SMasami Hiramatsu {
122102ca1521SMasami Hiramatsu int ret;
122202ca1521SMasami Hiramatsu
122317911ff3SSteven Rostedt (VMware) ret = security_locked_down(LOCKDOWN_TRACEFS);
122417911ff3SSteven Rostedt (VMware) if (ret)
122517911ff3SSteven Rostedt (VMware) return ret;
122617911ff3SSteven Rostedt (VMware)
122702ca1521SMasami Hiramatsu if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
12286212dd29SMasami Hiramatsu ret = dyn_events_release_all(&trace_kprobe_ops);
122902ca1521SMasami Hiramatsu if (ret < 0)
123002ca1521SMasami Hiramatsu return ret;
123102ca1521SMasami Hiramatsu }
1232413d37d1SMasami Hiramatsu
1233413d37d1SMasami Hiramatsu return seq_open(file, &probes_seq_op);
1234413d37d1SMasami Hiramatsu }
1235413d37d1SMasami Hiramatsu
probes_write(struct file * file,const char __user * buffer,size_t count,loff_t * ppos)1236413d37d1SMasami Hiramatsu static ssize_t probes_write(struct file *file, const char __user *buffer,
1237413d37d1SMasami Hiramatsu size_t count, loff_t *ppos)
1238413d37d1SMasami Hiramatsu {
12397e465baaSTom Zanussi return trace_parse_run_command(file, buffer, count, ppos,
12406212dd29SMasami Hiramatsu create_or_delete_trace_kprobe);
1241413d37d1SMasami Hiramatsu }
1242413d37d1SMasami Hiramatsu
1243413d37d1SMasami Hiramatsu static const struct file_operations kprobe_events_ops = {
1244413d37d1SMasami Hiramatsu .owner = THIS_MODULE,
1245413d37d1SMasami Hiramatsu .open = probes_open,
1246413d37d1SMasami Hiramatsu .read = seq_read,
1247413d37d1SMasami Hiramatsu .llseek = seq_lseek,
1248413d37d1SMasami Hiramatsu .release = seq_release,
1249413d37d1SMasami Hiramatsu .write = probes_write,
1250413d37d1SMasami Hiramatsu };
1251413d37d1SMasami Hiramatsu
1252cd7e7bd5SMasami Hiramatsu /* Probes profiling interfaces */
probes_profile_seq_show(struct seq_file * m,void * v)1253cd7e7bd5SMasami Hiramatsu static int probes_profile_seq_show(struct seq_file *m, void *v)
1254cd7e7bd5SMasami Hiramatsu {
12556212dd29SMasami Hiramatsu struct dyn_event *ev = v;
12566212dd29SMasami Hiramatsu struct trace_kprobe *tk;
1257dfea08a2SXiangyang Zhang unsigned long nmissed;
1258cd7e7bd5SMasami Hiramatsu
12596212dd29SMasami Hiramatsu if (!is_trace_kprobe(ev))
12606212dd29SMasami Hiramatsu return 0;
12616212dd29SMasami Hiramatsu
12626212dd29SMasami Hiramatsu tk = to_trace_kprobe(ev);
1263dfea08a2SXiangyang Zhang nmissed = trace_kprobe_is_return(tk) ?
1264dfea08a2SXiangyang Zhang tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed;
1265de7b2973SMathieu Desnoyers seq_printf(m, " %-44s %15lu %15lu\n",
1266b55ce203SMasami Hiramatsu trace_probe_name(&tk->tp),
1267f18f97acSMarcin Nowakowski trace_kprobe_nhit(tk),
1268dfea08a2SXiangyang Zhang nmissed);
1269cd7e7bd5SMasami Hiramatsu
1270cd7e7bd5SMasami Hiramatsu return 0;
1271cd7e7bd5SMasami Hiramatsu }
1272cd7e7bd5SMasami Hiramatsu
1273cd7e7bd5SMasami Hiramatsu static const struct seq_operations profile_seq_op = {
12746212dd29SMasami Hiramatsu .start = dyn_event_seq_start,
12756212dd29SMasami Hiramatsu .next = dyn_event_seq_next,
12766212dd29SMasami Hiramatsu .stop = dyn_event_seq_stop,
1277cd7e7bd5SMasami Hiramatsu .show = probes_profile_seq_show
1278cd7e7bd5SMasami Hiramatsu };
1279cd7e7bd5SMasami Hiramatsu
profile_open(struct inode * inode,struct file * file)1280cd7e7bd5SMasami Hiramatsu static int profile_open(struct inode *inode, struct file *file)
1281cd7e7bd5SMasami Hiramatsu {
128217911ff3SSteven Rostedt (VMware) int ret;
128317911ff3SSteven Rostedt (VMware)
128417911ff3SSteven Rostedt (VMware) ret = security_locked_down(LOCKDOWN_TRACEFS);
128517911ff3SSteven Rostedt (VMware) if (ret)
128617911ff3SSteven Rostedt (VMware) return ret;
128717911ff3SSteven Rostedt (VMware)
1288cd7e7bd5SMasami Hiramatsu return seq_open(file, &profile_seq_op);
1289cd7e7bd5SMasami Hiramatsu }
1290cd7e7bd5SMasami Hiramatsu
1291cd7e7bd5SMasami Hiramatsu static const struct file_operations kprobe_profile_ops = {
1292cd7e7bd5SMasami Hiramatsu .owner = THIS_MODULE,
1293cd7e7bd5SMasami Hiramatsu .open = profile_open,
1294cd7e7bd5SMasami Hiramatsu .read = seq_read,
1295cd7e7bd5SMasami Hiramatsu .llseek = seq_lseek,
1296cd7e7bd5SMasami Hiramatsu .release = seq_release,
1297cd7e7bd5SMasami Hiramatsu };
1298cd7e7bd5SMasami Hiramatsu
129953305928SMasami Hiramatsu /* Note that we don't verify it, since the code does not come from user space */
130053305928SMasami Hiramatsu static int
process_fetch_insn(struct fetch_insn * code,void * rec,void * dest,void * base)13018565a45dSSteven Rostedt (VMware) process_fetch_insn(struct fetch_insn *code, void *rec, void *dest,
13029178412dSMasami Hiramatsu void *base)
130353305928SMasami Hiramatsu {
13048565a45dSSteven Rostedt (VMware) struct pt_regs *regs = rec;
130553305928SMasami Hiramatsu unsigned long val;
1306bd78acc8SSong Chen int ret;
130753305928SMasami Hiramatsu
1308a6682814SMasami Hiramatsu retry:
130953305928SMasami Hiramatsu /* 1st stage: get value from context */
131053305928SMasami Hiramatsu switch (code->op) {
131153305928SMasami Hiramatsu case FETCH_OP_REG:
131253305928SMasami Hiramatsu val = regs_get_register(regs, code->param);
131353305928SMasami Hiramatsu break;
131453305928SMasami Hiramatsu case FETCH_OP_STACK:
131553305928SMasami Hiramatsu val = regs_get_kernel_stack_nth(regs, code->param);
131653305928SMasami Hiramatsu break;
131753305928SMasami Hiramatsu case FETCH_OP_STACKP:
131853305928SMasami Hiramatsu val = kernel_stack_pointer(regs);
131953305928SMasami Hiramatsu break;
132053305928SMasami Hiramatsu case FETCH_OP_RETVAL:
132153305928SMasami Hiramatsu val = regs_return_value(regs);
132253305928SMasami Hiramatsu break;
1323a1303af5SMasami Hiramatsu #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
1324a1303af5SMasami Hiramatsu case FETCH_OP_ARG:
1325a1303af5SMasami Hiramatsu val = regs_get_kernel_argument(regs, code->param);
1326a1303af5SMasami Hiramatsu break;
1327a1303af5SMasami Hiramatsu #endif
1328a6682814SMasami Hiramatsu case FETCH_NOP_SYMBOL: /* Ignore a place holder */
1329a6682814SMasami Hiramatsu code++;
1330a6682814SMasami Hiramatsu goto retry;
133153305928SMasami Hiramatsu default:
1332bd78acc8SSong Chen ret = process_common_fetch_insn(code, &val);
1333bd78acc8SSong Chen if (ret < 0)
1334bd78acc8SSong Chen return ret;
133553305928SMasami Hiramatsu }
133653305928SMasami Hiramatsu code++;
133753305928SMasami Hiramatsu
13389b960a38SMasami Hiramatsu return process_fetch_insn_bottom(code, val, dest, base);
133953305928SMasami Hiramatsu }
NOKPROBE_SYMBOL(process_fetch_insn)134053305928SMasami Hiramatsu NOKPROBE_SYMBOL(process_fetch_insn)
134153305928SMasami Hiramatsu
1342413d37d1SMasami Hiramatsu /* Kprobe handler */
13433da0f180SMasami Hiramatsu static nokprobe_inline void
1344c31ffb3fSNamhyung Kim __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
13457f1d2f82SSteven Rostedt (Red Hat) struct trace_event_file *trace_file)
1346413d37d1SMasami Hiramatsu {
134793ccae7aSMasami Hiramatsu struct kprobe_trace_entry_head *entry;
1348e3dc9f89SMasami Hiramatsu struct trace_event_call *call = trace_probe_event_call(&tk->tp);
13498cfcf155SMasami Hiramatsu struct trace_event_buffer fbuffer;
13508cfcf155SMasami Hiramatsu int dsize;
1351413d37d1SMasami Hiramatsu
13527f1d2f82SSteven Rostedt (Red Hat) WARN_ON(call != trace_file->event_call);
135341a7dd42SMasami Hiramatsu
135409a5059aSSteven Rostedt (Red Hat) if (trace_trigger_soft_disabled(trace_file))
1355b8820084SMasami Hiramatsu return;
1356b8820084SMasami Hiramatsu
1357c31ffb3fSNamhyung Kim dsize = __get_data_size(&tk->tp, regs);
1358413d37d1SMasami Hiramatsu
13595e6cd84eSSteven Rostedt (VMware) entry = trace_event_buffer_reserve(&fbuffer, trace_file,
13605e6cd84eSSteven Rostedt (VMware) sizeof(*entry) + tk->tp.size + dsize);
13615e6cd84eSSteven Rostedt (VMware) if (!entry)
13621e12a4a7SXiao Guangrong return;
1363413d37d1SMasami Hiramatsu
13648cfcf155SMasami Hiramatsu fbuffer.regs = regs;
1365c31ffb3fSNamhyung Kim entry->ip = (unsigned long)tk->rp.kp.addr;
13669178412dSMasami Hiramatsu store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1367413d37d1SMasami Hiramatsu
13688cfcf155SMasami Hiramatsu trace_event_buffer_commit(&fbuffer);
1369413d37d1SMasami Hiramatsu }
1370413d37d1SMasami Hiramatsu
13713da0f180SMasami Hiramatsu static void
kprobe_trace_func(struct trace_kprobe * tk,struct pt_regs * regs)1372c31ffb3fSNamhyung Kim kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
137341a7dd42SMasami Hiramatsu {
1374b04d52e3SOleg Nesterov struct event_file_link *link;
137541a7dd42SMasami Hiramatsu
1376b5f935eeSMasami Hiramatsu trace_probe_for_each_link_rcu(link, &tk->tp)
1377c31ffb3fSNamhyung Kim __kprobe_trace_func(tk, regs, link->file);
137841a7dd42SMasami Hiramatsu }
13793da0f180SMasami Hiramatsu NOKPROBE_SYMBOL(kprobe_trace_func);
138041a7dd42SMasami Hiramatsu
1381413d37d1SMasami Hiramatsu /* Kretprobe handler */
13823da0f180SMasami Hiramatsu static nokprobe_inline void
__kretprobe_trace_func(struct trace_kprobe * tk,struct kretprobe_instance * ri,struct pt_regs * regs,struct trace_event_file * trace_file)1383c31ffb3fSNamhyung Kim __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
138441a7dd42SMasami Hiramatsu struct pt_regs *regs,
13857f1d2f82SSteven Rostedt (Red Hat) struct trace_event_file *trace_file)
1386413d37d1SMasami Hiramatsu {
138793ccae7aSMasami Hiramatsu struct kretprobe_trace_entry_head *entry;
13888cfcf155SMasami Hiramatsu struct trace_event_buffer fbuffer;
1389e3dc9f89SMasami Hiramatsu struct trace_event_call *call = trace_probe_event_call(&tk->tp);
13908cfcf155SMasami Hiramatsu int dsize;
1391413d37d1SMasami Hiramatsu
13927f1d2f82SSteven Rostedt (Red Hat) WARN_ON(call != trace_file->event_call);
139341a7dd42SMasami Hiramatsu
139409a5059aSSteven Rostedt (Red Hat) if (trace_trigger_soft_disabled(trace_file))
1395b8820084SMasami Hiramatsu return;
1396b8820084SMasami Hiramatsu
1397c31ffb3fSNamhyung Kim dsize = __get_data_size(&tk->tp, regs);
13985e6cd84eSSteven Rostedt (VMware)
13995e6cd84eSSteven Rostedt (VMware) entry = trace_event_buffer_reserve(&fbuffer, trace_file,
14005e6cd84eSSteven Rostedt (VMware) sizeof(*entry) + tk->tp.size + dsize);
14015e6cd84eSSteven Rostedt (VMware) if (!entry)
14021e12a4a7SXiao Guangrong return;
1403413d37d1SMasami Hiramatsu
14048cfcf155SMasami Hiramatsu fbuffer.regs = regs;
1405c31ffb3fSNamhyung Kim entry->func = (unsigned long)tk->rp.kp.addr;
140673f9b911SMasami Hiramatsu entry->ret_ip = get_kretprobe_retaddr(ri);
14079178412dSMasami Hiramatsu store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1408413d37d1SMasami Hiramatsu
14098cfcf155SMasami Hiramatsu trace_event_buffer_commit(&fbuffer);
1410413d37d1SMasami Hiramatsu }
1411413d37d1SMasami Hiramatsu
14123da0f180SMasami Hiramatsu static void
kretprobe_trace_func(struct trace_kprobe * tk,struct kretprobe_instance * ri,struct pt_regs * regs)1413c31ffb3fSNamhyung Kim kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
141441a7dd42SMasami Hiramatsu struct pt_regs *regs)
141541a7dd42SMasami Hiramatsu {
1416b04d52e3SOleg Nesterov struct event_file_link *link;
141741a7dd42SMasami Hiramatsu
1418b5f935eeSMasami Hiramatsu trace_probe_for_each_link_rcu(link, &tk->tp)
1419c31ffb3fSNamhyung Kim __kretprobe_trace_func(tk, ri, regs, link->file);
142041a7dd42SMasami Hiramatsu }
14213da0f180SMasami Hiramatsu NOKPROBE_SYMBOL(kretprobe_trace_func);
142241a7dd42SMasami Hiramatsu
1423413d37d1SMasami Hiramatsu /* Event entry printers */
1424b62fdd97SMasami Hiramatsu static enum print_line_t
print_kprobe_event(struct trace_iterator * iter,int flags,struct trace_event * event)1425a9a57763SSteven Rostedt print_kprobe_event(struct trace_iterator *iter, int flags,
1426a9a57763SSteven Rostedt struct trace_event *event)
1427413d37d1SMasami Hiramatsu {
142893ccae7aSMasami Hiramatsu struct kprobe_trace_entry_head *field;
1429413d37d1SMasami Hiramatsu struct trace_seq *s = &iter->seq;
1430eca0d916SMasami Hiramatsu struct trace_probe *tp;
1431413d37d1SMasami Hiramatsu
143293ccae7aSMasami Hiramatsu field = (struct kprobe_trace_entry_head *)iter->ent;
143360d53e2cSMasami Hiramatsu tp = trace_probe_primary_from_call(
143460d53e2cSMasami Hiramatsu container_of(event, struct trace_event_call, event));
143560d53e2cSMasami Hiramatsu if (WARN_ON_ONCE(!tp))
143660d53e2cSMasami Hiramatsu goto out;
1437413d37d1SMasami Hiramatsu
1438b55ce203SMasami Hiramatsu trace_seq_printf(s, "%s: (", trace_probe_name(tp));
14396e9f23d1SMasami Hiramatsu
1440413d37d1SMasami Hiramatsu if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
144185224da0SSteven Rostedt (Red Hat) goto out;
1442413d37d1SMasami Hiramatsu
144385224da0SSteven Rostedt (Red Hat) trace_seq_putc(s, ')');
1444413d37d1SMasami Hiramatsu
1445196b6389SSong Chen if (trace_probe_print_args(s, tp->args, tp->nr_args,
144656de7630SMasami Hiramatsu (u8 *)&field[1], field) < 0)
144785224da0SSteven Rostedt (Red Hat) goto out;
1448413d37d1SMasami Hiramatsu
144985224da0SSteven Rostedt (Red Hat) trace_seq_putc(s, '\n');
145085224da0SSteven Rostedt (Red Hat) out:
145185224da0SSteven Rostedt (Red Hat) return trace_handle_return(s);
1452413d37d1SMasami Hiramatsu }
1453413d37d1SMasami Hiramatsu
1454b62fdd97SMasami Hiramatsu static enum print_line_t
print_kretprobe_event(struct trace_iterator * iter,int flags,struct trace_event * event)1455a9a57763SSteven Rostedt print_kretprobe_event(struct trace_iterator *iter, int flags,
1456a9a57763SSteven Rostedt struct trace_event *event)
1457413d37d1SMasami Hiramatsu {
145893ccae7aSMasami Hiramatsu struct kretprobe_trace_entry_head *field;
1459413d37d1SMasami Hiramatsu struct trace_seq *s = &iter->seq;
1460eca0d916SMasami Hiramatsu struct trace_probe *tp;
1461413d37d1SMasami Hiramatsu
146293ccae7aSMasami Hiramatsu field = (struct kretprobe_trace_entry_head *)iter->ent;
146360d53e2cSMasami Hiramatsu tp = trace_probe_primary_from_call(
146460d53e2cSMasami Hiramatsu container_of(event, struct trace_event_call, event));
146560d53e2cSMasami Hiramatsu if (WARN_ON_ONCE(!tp))
146660d53e2cSMasami Hiramatsu goto out;
1467413d37d1SMasami Hiramatsu
1468b55ce203SMasami Hiramatsu trace_seq_printf(s, "%s: (", trace_probe_name(tp));
14696e9f23d1SMasami Hiramatsu
1470413d37d1SMasami Hiramatsu if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
147185224da0SSteven Rostedt (Red Hat) goto out;
1472413d37d1SMasami Hiramatsu
147385224da0SSteven Rostedt (Red Hat) trace_seq_puts(s, " <- ");
1474413d37d1SMasami Hiramatsu
1475413d37d1SMasami Hiramatsu if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
147685224da0SSteven Rostedt (Red Hat) goto out;
1477413d37d1SMasami Hiramatsu
147885224da0SSteven Rostedt (Red Hat) trace_seq_putc(s, ')');
1479413d37d1SMasami Hiramatsu
1480196b6389SSong Chen if (trace_probe_print_args(s, tp->args, tp->nr_args,
148156de7630SMasami Hiramatsu (u8 *)&field[1], field) < 0)
148285224da0SSteven Rostedt (Red Hat) goto out;
1483413d37d1SMasami Hiramatsu
148485224da0SSteven Rostedt (Red Hat) trace_seq_putc(s, '\n');
1485413d37d1SMasami Hiramatsu
148685224da0SSteven Rostedt (Red Hat) out:
148785224da0SSteven Rostedt (Red Hat) return trace_handle_return(s);
1488413d37d1SMasami Hiramatsu }
1489413d37d1SMasami Hiramatsu
1490413d37d1SMasami Hiramatsu
kprobe_event_define_fields(struct trace_event_call * event_call)14912425bcb9SSteven Rostedt (Red Hat) static int kprobe_event_define_fields(struct trace_event_call *event_call)
1492413d37d1SMasami Hiramatsu {
1493eeb07b06SMasami Hiramatsu int ret;
149493ccae7aSMasami Hiramatsu struct kprobe_trace_entry_head field;
149560d53e2cSMasami Hiramatsu struct trace_probe *tp;
149660d53e2cSMasami Hiramatsu
149760d53e2cSMasami Hiramatsu tp = trace_probe_primary_from_call(event_call);
149860d53e2cSMasami Hiramatsu if (WARN_ON_ONCE(!tp))
149960d53e2cSMasami Hiramatsu return -ENOENT;
1500413d37d1SMasami Hiramatsu
1501a703d946SMasami Hiramatsu DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
1502c31ffb3fSNamhyung Kim
150360d53e2cSMasami Hiramatsu return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
1504413d37d1SMasami Hiramatsu }
1505413d37d1SMasami Hiramatsu
kretprobe_event_define_fields(struct trace_event_call * event_call)15062425bcb9SSteven Rostedt (Red Hat) static int kretprobe_event_define_fields(struct trace_event_call *event_call)
1507413d37d1SMasami Hiramatsu {
1508eeb07b06SMasami Hiramatsu int ret;
150993ccae7aSMasami Hiramatsu struct kretprobe_trace_entry_head field;
151060d53e2cSMasami Hiramatsu struct trace_probe *tp;
151160d53e2cSMasami Hiramatsu
151260d53e2cSMasami Hiramatsu tp = trace_probe_primary_from_call(event_call);
151360d53e2cSMasami Hiramatsu if (WARN_ON_ONCE(!tp))
151460d53e2cSMasami Hiramatsu return -ENOENT;
1515413d37d1SMasami Hiramatsu
1516a703d946SMasami Hiramatsu DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
1517a703d946SMasami Hiramatsu DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
1518c31ffb3fSNamhyung Kim
151960d53e2cSMasami Hiramatsu return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
1520413d37d1SMasami Hiramatsu }
1521413d37d1SMasami Hiramatsu
152207b139c8SLi Zefan #ifdef CONFIG_PERF_EVENTS
1523e08d1c65SMasami Hiramatsu
1524e08d1c65SMasami Hiramatsu /* Kprobe profile handler */
15259802d865SJosef Bacik static int
kprobe_perf_func(struct trace_kprobe * tk,struct pt_regs * regs)1526c31ffb3fSNamhyung Kim kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
1527e08d1c65SMasami Hiramatsu {
1528e3dc9f89SMasami Hiramatsu struct trace_event_call *call = trace_probe_event_call(&tk->tp);
152993ccae7aSMasami Hiramatsu struct kprobe_trace_entry_head *entry;
15301c024ecaSPeter Zijlstra struct hlist_head *head;
1531e09c8614SMasami Hiramatsu int size, __size, dsize;
15324ed7c92dSPeter Zijlstra int rctx;
1533e08d1c65SMasami Hiramatsu
15349802d865SJosef Bacik if (bpf_prog_array_valid(call)) {
153566665ad2SMasami Hiramatsu unsigned long orig_ip = instruction_pointer(regs);
15369802d865SJosef Bacik int ret;
15379802d865SJosef Bacik
15389802d865SJosef Bacik ret = trace_call_bpf(call, regs);
15399802d865SJosef Bacik
15409802d865SJosef Bacik /*
15419802d865SJosef Bacik * We need to check and see if we modified the pc of the
1542cce188bdSMasami Hiramatsu * pt_regs, and if so return 1 so that we don't do the
1543cce188bdSMasami Hiramatsu * single stepping.
15449802d865SJosef Bacik */
1545cce188bdSMasami Hiramatsu if (orig_ip != instruction_pointer(regs))
15469802d865SJosef Bacik return 1;
15479802d865SJosef Bacik if (!ret)
15489802d865SJosef Bacik return 0;
15499802d865SJosef Bacik }
15502541517cSAlexei Starovoitov
1551288e984eSOleg Nesterov head = this_cpu_ptr(call->perf_events);
1552288e984eSOleg Nesterov if (hlist_empty(head))
15539802d865SJosef Bacik return 0;
1554288e984eSOleg Nesterov
1555c31ffb3fSNamhyung Kim dsize = __get_data_size(&tk->tp, regs);
1556c31ffb3fSNamhyung Kim __size = sizeof(*entry) + tk->tp.size + dsize;
155774ebb63eSMasami Hiramatsu size = ALIGN(__size + sizeof(u32), sizeof(u64));
155874ebb63eSMasami Hiramatsu size -= sizeof(u32);
1559e08d1c65SMasami Hiramatsu
15601e1dcd93SAlexei Starovoitov entry = perf_trace_buf_alloc(size, NULL, &rctx);
1561430ad5a6SXiao Guangrong if (!entry)
15629802d865SJosef Bacik return 0;
1563ce71b9dfSFrederic Weisbecker
1564c31ffb3fSNamhyung Kim entry->ip = (unsigned long)tk->rp.kp.addr;
1565e09c8614SMasami Hiramatsu memset(&entry[1], 0, dsize);
15669178412dSMasami Hiramatsu store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
15671e1dcd93SAlexei Starovoitov perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
15688fd0fbbeSPeter Zijlstra head, NULL);
15699802d865SJosef Bacik return 0;
1570e08d1c65SMasami Hiramatsu }
15713da0f180SMasami Hiramatsu NOKPROBE_SYMBOL(kprobe_perf_func);
1572e08d1c65SMasami Hiramatsu
1573e08d1c65SMasami Hiramatsu /* Kretprobe profile handler */
15743da0f180SMasami Hiramatsu static void
kretprobe_perf_func(struct trace_kprobe * tk,struct kretprobe_instance * ri,struct pt_regs * regs)1575c31ffb3fSNamhyung Kim kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1576e08d1c65SMasami Hiramatsu struct pt_regs *regs)
1577e08d1c65SMasami Hiramatsu {
1578e3dc9f89SMasami Hiramatsu struct trace_event_call *call = trace_probe_event_call(&tk->tp);
157993ccae7aSMasami Hiramatsu struct kretprobe_trace_entry_head *entry;
15801c024ecaSPeter Zijlstra struct hlist_head *head;
1581e09c8614SMasami Hiramatsu int size, __size, dsize;
15824ed7c92dSPeter Zijlstra int rctx;
1583e08d1c65SMasami Hiramatsu
1584e87c6bc3SYonghong Song if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs))
15852541517cSAlexei Starovoitov return;
15862541517cSAlexei Starovoitov
1587288e984eSOleg Nesterov head = this_cpu_ptr(call->perf_events);
1588288e984eSOleg Nesterov if (hlist_empty(head))
1589288e984eSOleg Nesterov return;
1590288e984eSOleg Nesterov
1591c31ffb3fSNamhyung Kim dsize = __get_data_size(&tk->tp, regs);
1592c31ffb3fSNamhyung Kim __size = sizeof(*entry) + tk->tp.size + dsize;
159374ebb63eSMasami Hiramatsu size = ALIGN(__size + sizeof(u32), sizeof(u64));
159474ebb63eSMasami Hiramatsu size -= sizeof(u32);
1595e08d1c65SMasami Hiramatsu
15961e1dcd93SAlexei Starovoitov entry = perf_trace_buf_alloc(size, NULL, &rctx);
1597430ad5a6SXiao Guangrong if (!entry)
15981e12a4a7SXiao Guangrong return;
1599ce71b9dfSFrederic Weisbecker
1600c31ffb3fSNamhyung Kim entry->func = (unsigned long)tk->rp.kp.addr;
160173f9b911SMasami Hiramatsu entry->ret_ip = get_kretprobe_retaddr(ri);
16029178412dSMasami Hiramatsu store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
16031e1dcd93SAlexei Starovoitov perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
16048fd0fbbeSPeter Zijlstra head, NULL);
1605e08d1c65SMasami Hiramatsu }
16063da0f180SMasami Hiramatsu NOKPROBE_SYMBOL(kretprobe_perf_func);
160741bdc4b4SYonghong Song
bpf_get_kprobe_info(const struct perf_event * event,u32 * fd_type,const char ** symbol,u64 * probe_offset,u64 * probe_addr,bool perf_type_tracepoint)160841bdc4b4SYonghong Song int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type,
160941bdc4b4SYonghong Song const char **symbol, u64 *probe_offset,
161041bdc4b4SYonghong Song u64 *probe_addr, bool perf_type_tracepoint)
161141bdc4b4SYonghong Song {
161241bdc4b4SYonghong Song const char *pevent = trace_event_name(event->tp_event);
161341bdc4b4SYonghong Song const char *group = event->tp_event->class->system;
161441bdc4b4SYonghong Song struct trace_kprobe *tk;
161541bdc4b4SYonghong Song
161641bdc4b4SYonghong Song if (perf_type_tracepoint)
161741bdc4b4SYonghong Song tk = find_trace_kprobe(pevent, group);
161841bdc4b4SYonghong Song else
161922d5bd68SJean-Philippe Brucker tk = trace_kprobe_primary_from_call(event->tp_event);
162041bdc4b4SYonghong Song if (!tk)
162141bdc4b4SYonghong Song return -EINVAL;
162241bdc4b4SYonghong Song
162341bdc4b4SYonghong Song *fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE
162441bdc4b4SYonghong Song : BPF_FD_TYPE_KPROBE;
162541bdc4b4SYonghong Song *probe_offset = tk->rp.kp.offset;
1626cd3910d0SYafang Shao *probe_addr = kallsyms_show_value(current_cred()) ?
1627cd3910d0SYafang Shao (unsigned long)tk->rp.kp.addr : 0;
1628cd3910d0SYafang Shao *symbol = tk->symbol;
162941bdc4b4SYonghong Song return 0;
163041bdc4b4SYonghong Song }
163107b139c8SLi Zefan #endif /* CONFIG_PERF_EVENTS */
163250d78056SMasami Hiramatsu
16333fe3d619SOleg Nesterov /*
16343fe3d619SOleg Nesterov * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
16353fe3d619SOleg Nesterov *
16363fe3d619SOleg Nesterov * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
16373fe3d619SOleg Nesterov * lockless, but we can't race with this __init function.
16383fe3d619SOleg Nesterov */
kprobe_register(struct trace_event_call * event,enum trace_reg type,void * data)16392425bcb9SSteven Rostedt (Red Hat) static int kprobe_register(struct trace_event_call *event,
1640ceec0b6fSJiri Olsa enum trace_reg type, void *data)
16412239291aSSteven Rostedt {
16427f1d2f82SSteven Rostedt (Red Hat) struct trace_event_file *file = data;
16431538f888SMasami Hiramatsu
16442239291aSSteven Rostedt switch (type) {
16452239291aSSteven Rostedt case TRACE_REG_REGISTER:
164660d53e2cSMasami Hiramatsu return enable_trace_kprobe(event, file);
16472239291aSSteven Rostedt case TRACE_REG_UNREGISTER:
164860d53e2cSMasami Hiramatsu return disable_trace_kprobe(event, file);
16492239291aSSteven Rostedt
16502239291aSSteven Rostedt #ifdef CONFIG_PERF_EVENTS
16512239291aSSteven Rostedt case TRACE_REG_PERF_REGISTER:
165260d53e2cSMasami Hiramatsu return enable_trace_kprobe(event, NULL);
16532239291aSSteven Rostedt case TRACE_REG_PERF_UNREGISTER:
165460d53e2cSMasami Hiramatsu return disable_trace_kprobe(event, NULL);
1655ceec0b6fSJiri Olsa case TRACE_REG_PERF_OPEN:
1656ceec0b6fSJiri Olsa case TRACE_REG_PERF_CLOSE:
1657489c75c3SJiri Olsa case TRACE_REG_PERF_ADD:
1658489c75c3SJiri Olsa case TRACE_REG_PERF_DEL:
1659ceec0b6fSJiri Olsa return 0;
16602239291aSSteven Rostedt #endif
16612239291aSSteven Rostedt }
16622239291aSSteven Rostedt return 0;
16632239291aSSteven Rostedt }
166450d78056SMasami Hiramatsu
kprobe_dispatcher(struct kprobe * kp,struct pt_regs * regs)16653da0f180SMasami Hiramatsu static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
166650d78056SMasami Hiramatsu {
1667c31ffb3fSNamhyung Kim struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
16689802d865SJosef Bacik int ret = 0;
166950d78056SMasami Hiramatsu
1670a7636d9eSMartin KaFai Lau raw_cpu_inc(*tk->nhit);
167148182bd2SMasami Hiramatsu
1672747774d6SMasami Hiramatsu if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE))
1673c31ffb3fSNamhyung Kim kprobe_trace_func(tk, regs);
167407b139c8SLi Zefan #ifdef CONFIG_PERF_EVENTS
1675747774d6SMasami Hiramatsu if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE))
16769802d865SJosef Bacik ret = kprobe_perf_func(tk, regs);
167707b139c8SLi Zefan #endif
16789802d865SJosef Bacik return ret;
1679e08d1c65SMasami Hiramatsu }
16803da0f180SMasami Hiramatsu NOKPROBE_SYMBOL(kprobe_dispatcher);
1681e08d1c65SMasami Hiramatsu
16823da0f180SMasami Hiramatsu static int
kretprobe_dispatcher(struct kretprobe_instance * ri,struct pt_regs * regs)16833da0f180SMasami Hiramatsu kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
168450d78056SMasami Hiramatsu {
1685d741bf41SPeter Zijlstra struct kretprobe *rp = get_kretprobe(ri);
1686cc72b720SMasami Hiramatsu (Google) struct trace_kprobe *tk;
168750d78056SMasami Hiramatsu
1688cc72b720SMasami Hiramatsu (Google) /*
1689cc72b720SMasami Hiramatsu (Google) * There is a small chance that get_kretprobe(ri) returns NULL when
1690cc72b720SMasami Hiramatsu (Google) * the kretprobe is unregister on another CPU between kretprobe's
1691cc72b720SMasami Hiramatsu (Google) * trampoline_handler and this function.
1692cc72b720SMasami Hiramatsu (Google) */
1693cc72b720SMasami Hiramatsu (Google) if (unlikely(!rp))
1694cc72b720SMasami Hiramatsu (Google) return 0;
1695cc72b720SMasami Hiramatsu (Google)
1696cc72b720SMasami Hiramatsu (Google) tk = container_of(rp, struct trace_kprobe, rp);
1697a7636d9eSMartin KaFai Lau raw_cpu_inc(*tk->nhit);
169848182bd2SMasami Hiramatsu
1699747774d6SMasami Hiramatsu if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE))
1700c31ffb3fSNamhyung Kim kretprobe_trace_func(tk, ri, regs);
170107b139c8SLi Zefan #ifdef CONFIG_PERF_EVENTS
1702747774d6SMasami Hiramatsu if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE))
1703c31ffb3fSNamhyung Kim kretprobe_perf_func(tk, ri, regs);
170407b139c8SLi Zefan #endif
1705f2cc020dSIngo Molnar return 0; /* We don't tweak kernel, so just return 0 */
170650d78056SMasami Hiramatsu }
17073da0f180SMasami Hiramatsu NOKPROBE_SYMBOL(kretprobe_dispatcher);
1708e08d1c65SMasami Hiramatsu
1709a9a57763SSteven Rostedt static struct trace_event_functions kretprobe_funcs = {
1710a9a57763SSteven Rostedt .trace = print_kretprobe_event
1711a9a57763SSteven Rostedt };
1712a9a57763SSteven Rostedt
1713a9a57763SSteven Rostedt static struct trace_event_functions kprobe_funcs = {
1714a9a57763SSteven Rostedt .trace = print_kprobe_event
1715a9a57763SSteven Rostedt };
1716a9a57763SSteven Rostedt
171704ae87a5SPeter Zijlstra static struct trace_event_fields kretprobe_fields_array[] = {
171804ae87a5SPeter Zijlstra { .type = TRACE_FUNCTION_TYPE,
171904ae87a5SPeter Zijlstra .define_fields = kretprobe_event_define_fields },
172004ae87a5SPeter Zijlstra {}
172104ae87a5SPeter Zijlstra };
172204ae87a5SPeter Zijlstra
172304ae87a5SPeter Zijlstra static struct trace_event_fields kprobe_fields_array[] = {
172404ae87a5SPeter Zijlstra { .type = TRACE_FUNCTION_TYPE,
172504ae87a5SPeter Zijlstra .define_fields = kprobe_event_define_fields },
172604ae87a5SPeter Zijlstra {}
172704ae87a5SPeter Zijlstra };
172804ae87a5SPeter Zijlstra
init_trace_event_call(struct trace_kprobe * tk)1729e3dc9f89SMasami Hiramatsu static inline void init_trace_event_call(struct trace_kprobe *tk)
1730413d37d1SMasami Hiramatsu {
1731e3dc9f89SMasami Hiramatsu struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1732e3dc9f89SMasami Hiramatsu
1733c31ffb3fSNamhyung Kim if (trace_kprobe_is_return(tk)) {
173480decc70SSteven Rostedt call->event.funcs = &kretprobe_funcs;
173504ae87a5SPeter Zijlstra call->class->fields_array = kretprobe_fields_array;
1736413d37d1SMasami Hiramatsu } else {
173780decc70SSteven Rostedt call->event.funcs = &kprobe_funcs;
173804ae87a5SPeter Zijlstra call->class->fields_array = kprobe_fields_array;
1739413d37d1SMasami Hiramatsu }
1740e12f03d7SSong Liu
1741e12f03d7SSong Liu call->flags = TRACE_EVENT_FL_KPROBE;
1742e12f03d7SSong Liu call->class->reg = kprobe_register;
1743e12f03d7SSong Liu }
1744e12f03d7SSong Liu
register_kprobe_event(struct trace_kprobe * tk)1745e12f03d7SSong Liu static int register_kprobe_event(struct trace_kprobe *tk)
1746e12f03d7SSong Liu {
1747e3dc9f89SMasami Hiramatsu init_trace_event_call(tk);
1748e12f03d7SSong Liu
174946e5376dSMasami Hiramatsu return trace_probe_register_event_call(&tk->tp);
1750413d37d1SMasami Hiramatsu }
1751413d37d1SMasami Hiramatsu
unregister_kprobe_event(struct trace_kprobe * tk)1752c31ffb3fSNamhyung Kim static int unregister_kprobe_event(struct trace_kprobe *tk)
1753413d37d1SMasami Hiramatsu {
175446e5376dSMasami Hiramatsu return trace_probe_unregister_event_call(&tk->tp);
1755413d37d1SMasami Hiramatsu }
1756413d37d1SMasami Hiramatsu
1757e12f03d7SSong Liu #ifdef CONFIG_PERF_EVENTS
1758b022f0c7SFrancis Laniel
1759e12f03d7SSong Liu /* create a trace_kprobe, but don't add it to global lists */
1760e12f03d7SSong Liu struct trace_event_call *
create_local_trace_kprobe(char * func,void * addr,unsigned long offs,bool is_return)1761e12f03d7SSong Liu create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
1762e12f03d7SSong Liu bool is_return)
1763e12f03d7SSong Liu {
1764007517a0SSteven Rostedt (VMware) enum probe_print_type ptype;
1765e12f03d7SSong Liu struct trace_kprobe *tk;
1766e12f03d7SSong Liu int ret;
1767e12f03d7SSong Liu char *event;
1768e12f03d7SSong Liu
1769b022f0c7SFrancis Laniel if (func) {
1770b022f0c7SFrancis Laniel unsigned int count;
1771b022f0c7SFrancis Laniel
1772b022f0c7SFrancis Laniel count = number_of_same_symbols(func);
1773b022f0c7SFrancis Laniel if (count > 1)
1774b022f0c7SFrancis Laniel /*
1775b022f0c7SFrancis Laniel * Users should use addr to remove the ambiguity of
1776b022f0c7SFrancis Laniel * using func only.
1777b022f0c7SFrancis Laniel */
1778b022f0c7SFrancis Laniel return ERR_PTR(-EADDRNOTAVAIL);
1779b022f0c7SFrancis Laniel else if (count == 0)
1780b022f0c7SFrancis Laniel /*
1781b022f0c7SFrancis Laniel * We can return ENOENT earlier than when register the
1782b022f0c7SFrancis Laniel * kprobe.
1783b022f0c7SFrancis Laniel */
1784b022f0c7SFrancis Laniel return ERR_PTR(-ENOENT);
1785b022f0c7SFrancis Laniel }
1786b022f0c7SFrancis Laniel
1787e12f03d7SSong Liu /*
17886212dd29SMasami Hiramatsu * local trace_kprobes are not added to dyn_event, so they are never
1789e12f03d7SSong Liu * searched in find_trace_kprobe(). Therefore, there is no concern of
1790e12f03d7SSong Liu * duplicated name here.
1791e12f03d7SSong Liu */
1792e12f03d7SSong Liu event = func ? func : "DUMMY_EVENT";
1793e12f03d7SSong Liu
1794e12f03d7SSong Liu tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func,
1795e12f03d7SSong Liu offs, 0 /* maxactive */, 0 /* nargs */,
1796e12f03d7SSong Liu is_return);
1797e12f03d7SSong Liu
1798e12f03d7SSong Liu if (IS_ERR(tk)) {
1799e12f03d7SSong Liu pr_info("Failed to allocate trace_probe.(%d)\n",
1800e12f03d7SSong Liu (int)PTR_ERR(tk));
1801e12f03d7SSong Liu return ERR_CAST(tk);
1802e12f03d7SSong Liu }
1803e12f03d7SSong Liu
1804e3dc9f89SMasami Hiramatsu init_trace_event_call(tk);
1805e12f03d7SSong Liu
1806007517a0SSteven Rostedt (VMware) ptype = trace_kprobe_is_return(tk) ?
1807007517a0SSteven Rostedt (VMware) PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
1808007517a0SSteven Rostedt (VMware) if (traceprobe_set_print_fmt(&tk->tp, ptype) < 0) {
1809e12f03d7SSong Liu ret = -ENOMEM;
1810e12f03d7SSong Liu goto error;
1811e12f03d7SSong Liu }
1812e12f03d7SSong Liu
1813e12f03d7SSong Liu ret = __register_trace_kprobe(tk);
1814f730e0f2SMasami Hiramatsu if (ret < 0)
1815e12f03d7SSong Liu goto error;
1816e12f03d7SSong Liu
1817e3dc9f89SMasami Hiramatsu return trace_probe_event_call(&tk->tp);
1818e12f03d7SSong Liu error:
1819e12f03d7SSong Liu free_trace_kprobe(tk);
1820e12f03d7SSong Liu return ERR_PTR(ret);
1821e12f03d7SSong Liu }
1822e12f03d7SSong Liu
destroy_local_trace_kprobe(struct trace_event_call * event_call)1823e12f03d7SSong Liu void destroy_local_trace_kprobe(struct trace_event_call *event_call)
1824e12f03d7SSong Liu {
1825e12f03d7SSong Liu struct trace_kprobe *tk;
1826e12f03d7SSong Liu
182760d53e2cSMasami Hiramatsu tk = trace_kprobe_primary_from_call(event_call);
182860d53e2cSMasami Hiramatsu if (unlikely(!tk))
182960d53e2cSMasami Hiramatsu return;
1830e12f03d7SSong Liu
1831e12f03d7SSong Liu if (trace_probe_is_enabled(&tk->tp)) {
1832e12f03d7SSong Liu WARN_ON(1);
1833e12f03d7SSong Liu return;
1834e12f03d7SSong Liu }
1835e12f03d7SSong Liu
1836e12f03d7SSong Liu __unregister_trace_kprobe(tk);
18370fc8c358SJiri Olsa
1838e12f03d7SSong Liu free_trace_kprobe(tk);
1839e12f03d7SSong Liu }
1840e12f03d7SSong Liu #endif /* CONFIG_PERF_EVENTS */
1841e12f03d7SSong Liu
enable_boot_kprobe_events(void)1842970988e1SMasami Hiramatsu static __init void enable_boot_kprobe_events(void)
1843970988e1SMasami Hiramatsu {
1844970988e1SMasami Hiramatsu struct trace_array *tr = top_trace_array();
1845970988e1SMasami Hiramatsu struct trace_event_file *file;
1846970988e1SMasami Hiramatsu struct trace_kprobe *tk;
1847970988e1SMasami Hiramatsu struct dyn_event *pos;
1848970988e1SMasami Hiramatsu
1849970988e1SMasami Hiramatsu mutex_lock(&event_mutex);
1850970988e1SMasami Hiramatsu for_each_trace_kprobe(tk, pos) {
1851970988e1SMasami Hiramatsu list_for_each_entry(file, &tr->events, list)
1852e3dc9f89SMasami Hiramatsu if (file->event_call == trace_probe_event_call(&tk->tp))
1853970988e1SMasami Hiramatsu trace_event_enable_disable(file, 1, 0);
1854970988e1SMasami Hiramatsu }
1855970988e1SMasami Hiramatsu mutex_unlock(&event_mutex);
1856970988e1SMasami Hiramatsu }
1857970988e1SMasami Hiramatsu
setup_boot_kprobe_events(void)1858970988e1SMasami Hiramatsu static __init void setup_boot_kprobe_events(void)
1859970988e1SMasami Hiramatsu {
1860970988e1SMasami Hiramatsu char *p, *cmd = kprobe_boot_events_buf;
1861970988e1SMasami Hiramatsu int ret;
1862970988e1SMasami Hiramatsu
1863970988e1SMasami Hiramatsu strreplace(kprobe_boot_events_buf, ',', ' ');
1864970988e1SMasami Hiramatsu
1865970988e1SMasami Hiramatsu while (cmd && *cmd != '\0') {
1866970988e1SMasami Hiramatsu p = strchr(cmd, ';');
1867970988e1SMasami Hiramatsu if (p)
1868970988e1SMasami Hiramatsu *p++ = '\0';
1869970988e1SMasami Hiramatsu
1870d262271dSMasami Hiramatsu ret = create_or_delete_trace_kprobe(cmd);
1871970988e1SMasami Hiramatsu if (ret)
1872970988e1SMasami Hiramatsu pr_warn("Failed to add event(%d): %s\n", ret, cmd);
1873970988e1SMasami Hiramatsu
1874970988e1SMasami Hiramatsu cmd = p;
1875970988e1SMasami Hiramatsu }
1876970988e1SMasami Hiramatsu
1877970988e1SMasami Hiramatsu enable_boot_kprobe_events();
1878970988e1SMasami Hiramatsu }
1879970988e1SMasami Hiramatsu
1880d8d4c6d0SMasami Hiramatsu /*
1881ba0fbfbbSMasami Hiramatsu * Register dynevent at core_initcall. This allows kernel to setup kprobe
1882ba0fbfbbSMasami Hiramatsu * events in postcore_initcall without tracefs.
1883d8d4c6d0SMasami Hiramatsu */
init_kprobe_trace_early(void)1884d8d4c6d0SMasami Hiramatsu static __init int init_kprobe_trace_early(void)
1885413d37d1SMasami Hiramatsu {
18866212dd29SMasami Hiramatsu int ret;
18876212dd29SMasami Hiramatsu
18886212dd29SMasami Hiramatsu ret = dyn_event_register(&trace_kprobe_ops);
18896212dd29SMasami Hiramatsu if (ret)
18906212dd29SMasami Hiramatsu return ret;
1891413d37d1SMasami Hiramatsu
1892c31ffb3fSNamhyung Kim if (register_module_notifier(&trace_kprobe_module_nb))
189361424318SMasami Hiramatsu return -EINVAL;
189461424318SMasami Hiramatsu
1895d8d4c6d0SMasami Hiramatsu return 0;
1896d8d4c6d0SMasami Hiramatsu }
1897ba0fbfbbSMasami Hiramatsu core_initcall(init_kprobe_trace_early);
1898d8d4c6d0SMasami Hiramatsu
1899d8d4c6d0SMasami Hiramatsu /* Make a tracefs interface for controlling probe points */
init_kprobe_trace(void)1900d8d4c6d0SMasami Hiramatsu static __init int init_kprobe_trace(void)
1901d8d4c6d0SMasami Hiramatsu {
190222c36b18SWei Yang int ret;
1903d8d4c6d0SMasami Hiramatsu
190422c36b18SWei Yang ret = tracing_init_dentry();
190522c36b18SWei Yang if (ret)
1906413d37d1SMasami Hiramatsu return 0;
1907413d37d1SMasami Hiramatsu
1908e4931b82SYuntao Wang /* Event list interface */
1909e4931b82SYuntao Wang trace_create_file("kprobe_events", TRACE_MODE_WRITE,
191021ccc9cdSSteven Rostedt (VMware) NULL, NULL, &kprobe_events_ops);
1911413d37d1SMasami Hiramatsu
1912cd7e7bd5SMasami Hiramatsu /* Profile interface */
1913e4931b82SYuntao Wang trace_create_file("kprobe_profile", TRACE_MODE_READ,
191421ccc9cdSSteven Rostedt (VMware) NULL, NULL, &kprobe_profile_ops);
1915cd7e7bd5SMasami Hiramatsu
1916970988e1SMasami Hiramatsu setup_boot_kprobe_events();
1917970988e1SMasami Hiramatsu
1918413d37d1SMasami Hiramatsu return 0;
1919413d37d1SMasami Hiramatsu }
1920413d37d1SMasami Hiramatsu fs_initcall(init_kprobe_trace);
1921413d37d1SMasami Hiramatsu
1922413d37d1SMasami Hiramatsu
1923413d37d1SMasami Hiramatsu #ifdef CONFIG_FTRACE_STARTUP_TEST
192426a346f2SArnd Bergmann static __init struct trace_event_file *
find_trace_probe_file(struct trace_kprobe * tk,struct trace_array * tr)1925c31ffb3fSNamhyung Kim find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)
192641a7dd42SMasami Hiramatsu {
19277f1d2f82SSteven Rostedt (Red Hat) struct trace_event_file *file;
192841a7dd42SMasami Hiramatsu
192941a7dd42SMasami Hiramatsu list_for_each_entry(file, &tr->events, list)
1930e3dc9f89SMasami Hiramatsu if (file->event_call == trace_probe_event_call(&tk->tp))
193141a7dd42SMasami Hiramatsu return file;
193241a7dd42SMasami Hiramatsu
193341a7dd42SMasami Hiramatsu return NULL;
193441a7dd42SMasami Hiramatsu }
193541a7dd42SMasami Hiramatsu
19363fe3d619SOleg Nesterov /*
1937c31ffb3fSNamhyung Kim * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this
19383fe3d619SOleg Nesterov * stage, we can do this lockless.
19393fe3d619SOleg Nesterov */
kprobe_trace_self_tests_init(void)1940413d37d1SMasami Hiramatsu static __init int kprobe_trace_self_tests_init(void)
1941413d37d1SMasami Hiramatsu {
1942231e36f4SMasami Hiramatsu int ret, warn = 0;
1943413d37d1SMasami Hiramatsu int (*target)(int, int, int, int, int, int);
1944c31ffb3fSNamhyung Kim struct trace_kprobe *tk;
19457f1d2f82SSteven Rostedt (Red Hat) struct trace_event_file *file;
1946413d37d1SMasami Hiramatsu
1947748ec3a2SYoshihiro YUNOMAE if (tracing_is_disabled())
1948748ec3a2SYoshihiro YUNOMAE return -ENODEV;
1949748ec3a2SYoshihiro YUNOMAE
195060efe21eSMasami Hiramatsu if (tracing_selftest_disabled)
1951b6399cc7SSteven Rostedt (VMware) return 0;
1952b6399cc7SSteven Rostedt (VMware)
1953413d37d1SMasami Hiramatsu target = kprobe_trace_selftest_target;
1954413d37d1SMasami Hiramatsu
1955413d37d1SMasami Hiramatsu pr_info("Testing kprobe tracing: ");
1956413d37d1SMasami Hiramatsu
1957d262271dSMasami Hiramatsu ret = create_or_delete_trace_kprobe("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)");
1958231e36f4SMasami Hiramatsu if (WARN_ON_ONCE(ret)) {
195941a7dd42SMasami Hiramatsu pr_warn("error on probing function entry.\n");
1960231e36f4SMasami Hiramatsu warn++;
1961231e36f4SMasami Hiramatsu } else {
1962231e36f4SMasami Hiramatsu /* Enable trace point */
1963c31ffb3fSNamhyung Kim tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1964c31ffb3fSNamhyung Kim if (WARN_ON_ONCE(tk == NULL)) {
196541a7dd42SMasami Hiramatsu pr_warn("error on getting new probe.\n");
196641a7dd42SMasami Hiramatsu warn++;
196741a7dd42SMasami Hiramatsu } else {
1968c31ffb3fSNamhyung Kim file = find_trace_probe_file(tk, top_trace_array());
196941a7dd42SMasami Hiramatsu if (WARN_ON_ONCE(file == NULL)) {
197041a7dd42SMasami Hiramatsu pr_warn("error on getting probe file.\n");
1971231e36f4SMasami Hiramatsu warn++;
1972231e36f4SMasami Hiramatsu } else
197360d53e2cSMasami Hiramatsu enable_trace_kprobe(
197460d53e2cSMasami Hiramatsu trace_probe_event_call(&tk->tp), file);
197541a7dd42SMasami Hiramatsu }
1976231e36f4SMasami Hiramatsu }
1977413d37d1SMasami Hiramatsu
1978d262271dSMasami Hiramatsu ret = create_or_delete_trace_kprobe("r:testprobe2 kprobe_trace_selftest_target $retval");
1979231e36f4SMasami Hiramatsu if (WARN_ON_ONCE(ret)) {
198041a7dd42SMasami Hiramatsu pr_warn("error on probing function return.\n");
1981231e36f4SMasami Hiramatsu warn++;
1982231e36f4SMasami Hiramatsu } else {
1983231e36f4SMasami Hiramatsu /* Enable trace point */
1984c31ffb3fSNamhyung Kim tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1985c31ffb3fSNamhyung Kim if (WARN_ON_ONCE(tk == NULL)) {
198641a7dd42SMasami Hiramatsu pr_warn("error on getting 2nd new probe.\n");
198741a7dd42SMasami Hiramatsu warn++;
198841a7dd42SMasami Hiramatsu } else {
1989c31ffb3fSNamhyung Kim file = find_trace_probe_file(tk, top_trace_array());
199041a7dd42SMasami Hiramatsu if (WARN_ON_ONCE(file == NULL)) {
199141a7dd42SMasami Hiramatsu pr_warn("error on getting probe file.\n");
1992231e36f4SMasami Hiramatsu warn++;
1993231e36f4SMasami Hiramatsu } else
199460d53e2cSMasami Hiramatsu enable_trace_kprobe(
199560d53e2cSMasami Hiramatsu trace_probe_event_call(&tk->tp), file);
199641a7dd42SMasami Hiramatsu }
1997231e36f4SMasami Hiramatsu }
1998231e36f4SMasami Hiramatsu
1999231e36f4SMasami Hiramatsu if (warn)
2000231e36f4SMasami Hiramatsu goto end;
2001413d37d1SMasami Hiramatsu
2002413d37d1SMasami Hiramatsu ret = target(1, 2, 3, 4, 5, 6);
2003413d37d1SMasami Hiramatsu
2004d4d7ccc8SMarcin Nowakowski /*
2005d4d7ccc8SMarcin Nowakowski * Not expecting an error here, the check is only to prevent the
2006d4d7ccc8SMarcin Nowakowski * optimizer from removing the call to target() as otherwise there
2007d4d7ccc8SMarcin Nowakowski * are no side-effects and the call is never performed.
2008d4d7ccc8SMarcin Nowakowski */
2009d4d7ccc8SMarcin Nowakowski if (ret != 21)
2010d4d7ccc8SMarcin Nowakowski warn++;
2011d4d7ccc8SMarcin Nowakowski
201202ca1521SMasami Hiramatsu /* Disable trace points before removing it */
2013c31ffb3fSNamhyung Kim tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
2014c31ffb3fSNamhyung Kim if (WARN_ON_ONCE(tk == NULL)) {
201541a7dd42SMasami Hiramatsu pr_warn("error on getting test probe.\n");
201641a7dd42SMasami Hiramatsu warn++;
201741a7dd42SMasami Hiramatsu } else {
2018d4d7ccc8SMarcin Nowakowski if (trace_kprobe_nhit(tk) != 1) {
2019d4d7ccc8SMarcin Nowakowski pr_warn("incorrect number of testprobe hits\n");
2020d4d7ccc8SMarcin Nowakowski warn++;
2021d4d7ccc8SMarcin Nowakowski }
2022d4d7ccc8SMarcin Nowakowski
2023c31ffb3fSNamhyung Kim file = find_trace_probe_file(tk, top_trace_array());
202441a7dd42SMasami Hiramatsu if (WARN_ON_ONCE(file == NULL)) {
202541a7dd42SMasami Hiramatsu pr_warn("error on getting probe file.\n");
202602ca1521SMasami Hiramatsu warn++;
202702ca1521SMasami Hiramatsu } else
202860d53e2cSMasami Hiramatsu disable_trace_kprobe(
202960d53e2cSMasami Hiramatsu trace_probe_event_call(&tk->tp), file);
203041a7dd42SMasami Hiramatsu }
203102ca1521SMasami Hiramatsu
2032c31ffb3fSNamhyung Kim tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
2033c31ffb3fSNamhyung Kim if (WARN_ON_ONCE(tk == NULL)) {
203441a7dd42SMasami Hiramatsu pr_warn("error on getting 2nd test probe.\n");
203541a7dd42SMasami Hiramatsu warn++;
203641a7dd42SMasami Hiramatsu } else {
2037d4d7ccc8SMarcin Nowakowski if (trace_kprobe_nhit(tk) != 1) {
2038d4d7ccc8SMarcin Nowakowski pr_warn("incorrect number of testprobe2 hits\n");
2039d4d7ccc8SMarcin Nowakowski warn++;
2040d4d7ccc8SMarcin Nowakowski }
2041d4d7ccc8SMarcin Nowakowski
2042c31ffb3fSNamhyung Kim file = find_trace_probe_file(tk, top_trace_array());
204341a7dd42SMasami Hiramatsu if (WARN_ON_ONCE(file == NULL)) {
204441a7dd42SMasami Hiramatsu pr_warn("error on getting probe file.\n");
204502ca1521SMasami Hiramatsu warn++;
204602ca1521SMasami Hiramatsu } else
204760d53e2cSMasami Hiramatsu disable_trace_kprobe(
204860d53e2cSMasami Hiramatsu trace_probe_event_call(&tk->tp), file);
204941a7dd42SMasami Hiramatsu }
205002ca1521SMasami Hiramatsu
2051d262271dSMasami Hiramatsu ret = create_or_delete_trace_kprobe("-:testprobe");
2052231e36f4SMasami Hiramatsu if (WARN_ON_ONCE(ret)) {
205341a7dd42SMasami Hiramatsu pr_warn("error on deleting a probe.\n");
2054231e36f4SMasami Hiramatsu warn++;
2055231e36f4SMasami Hiramatsu }
2056413d37d1SMasami Hiramatsu
2057d262271dSMasami Hiramatsu ret = create_or_delete_trace_kprobe("-:testprobe2");
2058231e36f4SMasami Hiramatsu if (WARN_ON_ONCE(ret)) {
205941a7dd42SMasami Hiramatsu pr_warn("error on deleting a probe.\n");
2060231e36f4SMasami Hiramatsu warn++;
2061231e36f4SMasami Hiramatsu }
2062231e36f4SMasami Hiramatsu
2063231e36f4SMasami Hiramatsu end:
20646212dd29SMasami Hiramatsu ret = dyn_events_release_all(&trace_kprobe_ops);
20656212dd29SMasami Hiramatsu if (WARN_ON_ONCE(ret)) {
20666212dd29SMasami Hiramatsu pr_warn("error on cleaning up probes.\n");
20676212dd29SMasami Hiramatsu warn++;
20686212dd29SMasami Hiramatsu }
206930e7d894SThomas Gleixner /*
207030e7d894SThomas Gleixner * Wait for the optimizer work to finish. Otherwise it might fiddle
207130e7d894SThomas Gleixner * with probes in already freed __init text.
207230e7d894SThomas Gleixner */
207330e7d894SThomas Gleixner wait_for_kprobe_optimizer();
2074231e36f4SMasami Hiramatsu if (warn)
2075231e36f4SMasami Hiramatsu pr_cont("NG: Some tests are failed. Please check them.\n");
2076231e36f4SMasami Hiramatsu else
2077413d37d1SMasami Hiramatsu pr_cont("OK\n");
2078413d37d1SMasami Hiramatsu return 0;
2079413d37d1SMasami Hiramatsu }
2080413d37d1SMasami Hiramatsu
2081413d37d1SMasami Hiramatsu late_initcall(kprobe_trace_self_tests_init);
2082413d37d1SMasami Hiramatsu
2083413d37d1SMasami Hiramatsu #endif
2084