xref: /openbmc/linux/kernel/trace/trace_fprobe.c (revision 0e9a6b8a)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Fprobe-based tracing events
4  * Copyright (C) 2022 Google LLC.
5  */
6 #define pr_fmt(fmt)	"trace_fprobe: " fmt
7 
8 #include <linux/fprobe.h>
9 #include <linux/module.h>
10 #include <linux/rculist.h>
11 #include <linux/security.h>
12 #include <linux/tracepoint.h>
13 #include <linux/uaccess.h>
14 
15 #include "trace_dynevent.h"
16 #include "trace_probe.h"
17 #include "trace_probe_kernel.h"
18 #include "trace_probe_tmpl.h"
19 
20 #define FPROBE_EVENT_SYSTEM "fprobes"
21 #define TRACEPOINT_EVENT_SYSTEM "tracepoints"
22 #define RETHOOK_MAXACTIVE_MAX 4096
23 
24 static int trace_fprobe_create(const char *raw_command);
25 static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev);
26 static int trace_fprobe_release(struct dyn_event *ev);
27 static bool trace_fprobe_is_busy(struct dyn_event *ev);
28 static bool trace_fprobe_match(const char *system, const char *event,
29 			int argc, const char **argv, struct dyn_event *ev);
30 
31 static struct dyn_event_operations trace_fprobe_ops = {
32 	.create = trace_fprobe_create,
33 	.show = trace_fprobe_show,
34 	.is_busy = trace_fprobe_is_busy,
35 	.free = trace_fprobe_release,
36 	.match = trace_fprobe_match,
37 };
38 
39 /*
40  * Fprobe event core functions
41  */
42 struct trace_fprobe {
43 	struct dyn_event	devent;
44 	struct fprobe		fp;
45 	const char		*symbol;
46 	struct tracepoint	*tpoint;
47 	struct module		*mod;
48 	struct trace_probe	tp;
49 };
50 
is_trace_fprobe(struct dyn_event * ev)51 static bool is_trace_fprobe(struct dyn_event *ev)
52 {
53 	return ev->ops == &trace_fprobe_ops;
54 }
55 
to_trace_fprobe(struct dyn_event * ev)56 static struct trace_fprobe *to_trace_fprobe(struct dyn_event *ev)
57 {
58 	return container_of(ev, struct trace_fprobe, devent);
59 }
60 
61 /**
62  * for_each_trace_fprobe - iterate over the trace_fprobe list
63  * @pos:	the struct trace_fprobe * for each entry
64  * @dpos:	the struct dyn_event * to use as a loop cursor
65  */
66 #define for_each_trace_fprobe(pos, dpos)	\
67 	for_each_dyn_event(dpos)		\
68 		if (is_trace_fprobe(dpos) && (pos = to_trace_fprobe(dpos)))
69 
trace_fprobe_is_return(struct trace_fprobe * tf)70 static bool trace_fprobe_is_return(struct trace_fprobe *tf)
71 {
72 	return tf->fp.exit_handler != NULL;
73 }
74 
trace_fprobe_is_tracepoint(struct trace_fprobe * tf)75 static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf)
76 {
77 	return tf->tpoint != NULL;
78 }
79 
trace_fprobe_symbol(struct trace_fprobe * tf)80 static const char *trace_fprobe_symbol(struct trace_fprobe *tf)
81 {
82 	return tf->symbol ? tf->symbol : "unknown";
83 }
84 
trace_fprobe_is_busy(struct dyn_event * ev)85 static bool trace_fprobe_is_busy(struct dyn_event *ev)
86 {
87 	struct trace_fprobe *tf = to_trace_fprobe(ev);
88 
89 	return trace_probe_is_enabled(&tf->tp);
90 }
91 
trace_fprobe_match_command_head(struct trace_fprobe * tf,int argc,const char ** argv)92 static bool trace_fprobe_match_command_head(struct trace_fprobe *tf,
93 					    int argc, const char **argv)
94 {
95 	char buf[MAX_ARGSTR_LEN + 1];
96 
97 	if (!argc)
98 		return true;
99 
100 	snprintf(buf, sizeof(buf), "%s", trace_fprobe_symbol(tf));
101 	if (strcmp(buf, argv[0]))
102 		return false;
103 	argc--; argv++;
104 
105 	return trace_probe_match_command_args(&tf->tp, argc, argv);
106 }
107 
trace_fprobe_match(const char * system,const char * event,int argc,const char ** argv,struct dyn_event * ev)108 static bool trace_fprobe_match(const char *system, const char *event,
109 			int argc, const char **argv, struct dyn_event *ev)
110 {
111 	struct trace_fprobe *tf = to_trace_fprobe(ev);
112 
113 	if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event))
114 		return false;
115 
116 	if (system && strcmp(trace_probe_group_name(&tf->tp), system))
117 		return false;
118 
119 	return trace_fprobe_match_command_head(tf, argc, argv);
120 }
121 
trace_fprobe_is_registered(struct trace_fprobe * tf)122 static bool trace_fprobe_is_registered(struct trace_fprobe *tf)
123 {
124 	return fprobe_is_registered(&tf->fp);
125 }
126 
127 /*
128  * Note that we don't verify the fetch_insn code, since it does not come
129  * from user space.
130  */
131 static int
process_fetch_insn(struct fetch_insn * code,void * rec,void * dest,void * base)132 process_fetch_insn(struct fetch_insn *code, void *rec, void *dest,
133 		   void *base)
134 {
135 	struct pt_regs *regs = rec;
136 	unsigned long val;
137 	int ret;
138 
139 retry:
140 	/* 1st stage: get value from context */
141 	switch (code->op) {
142 	case FETCH_OP_STACK:
143 		val = regs_get_kernel_stack_nth(regs, code->param);
144 		break;
145 	case FETCH_OP_STACKP:
146 		val = kernel_stack_pointer(regs);
147 		break;
148 	case FETCH_OP_RETVAL:
149 		val = regs_return_value(regs);
150 		break;
151 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
152 	case FETCH_OP_ARG:
153 		val = regs_get_kernel_argument(regs, code->param);
154 		break;
155 #endif
156 	case FETCH_NOP_SYMBOL:	/* Ignore a place holder */
157 		code++;
158 		goto retry;
159 	default:
160 		ret = process_common_fetch_insn(code, &val);
161 		if (ret < 0)
162 			return ret;
163 	}
164 	code++;
165 
166 	return process_fetch_insn_bottom(code, val, dest, base);
167 }
NOKPROBE_SYMBOL(process_fetch_insn)168 NOKPROBE_SYMBOL(process_fetch_insn)
169 
170 /* function entry handler */
171 static nokprobe_inline void
172 __fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
173 		    struct pt_regs *regs,
174 		    struct trace_event_file *trace_file)
175 {
176 	struct fentry_trace_entry_head *entry;
177 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);
178 	struct trace_event_buffer fbuffer;
179 	int dsize;
180 
181 	if (WARN_ON_ONCE(call != trace_file->event_call))
182 		return;
183 
184 	if (trace_trigger_soft_disabled(trace_file))
185 		return;
186 
187 	dsize = __get_data_size(&tf->tp, regs);
188 
189 	entry = trace_event_buffer_reserve(&fbuffer, trace_file,
190 					   sizeof(*entry) + tf->tp.size + dsize);
191 	if (!entry)
192 		return;
193 
194 	fbuffer.regs = regs;
195 	entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
196 	entry->ip = entry_ip;
197 	store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize);
198 
199 	trace_event_buffer_commit(&fbuffer);
200 }
201 
202 static void
fentry_trace_func(struct trace_fprobe * tf,unsigned long entry_ip,struct pt_regs * regs)203 fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
204 		  struct pt_regs *regs)
205 {
206 	struct event_file_link *link;
207 
208 	trace_probe_for_each_link_rcu(link, &tf->tp)
209 		__fentry_trace_func(tf, entry_ip, regs, link->file);
210 }
211 NOKPROBE_SYMBOL(fentry_trace_func);
212 
213 /* Kretprobe handler */
214 static nokprobe_inline void
__fexit_trace_func(struct trace_fprobe * tf,unsigned long entry_ip,unsigned long ret_ip,struct pt_regs * regs,struct trace_event_file * trace_file)215 __fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
216 		   unsigned long ret_ip, struct pt_regs *regs,
217 		   struct trace_event_file *trace_file)
218 {
219 	struct fexit_trace_entry_head *entry;
220 	struct trace_event_buffer fbuffer;
221 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);
222 	int dsize;
223 
224 	if (WARN_ON_ONCE(call != trace_file->event_call))
225 		return;
226 
227 	if (trace_trigger_soft_disabled(trace_file))
228 		return;
229 
230 	dsize = __get_data_size(&tf->tp, regs);
231 
232 	entry = trace_event_buffer_reserve(&fbuffer, trace_file,
233 					   sizeof(*entry) + tf->tp.size + dsize);
234 	if (!entry)
235 		return;
236 
237 	fbuffer.regs = regs;
238 	entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
239 	entry->func = entry_ip;
240 	entry->ret_ip = ret_ip;
241 	store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize);
242 
243 	trace_event_buffer_commit(&fbuffer);
244 }
245 
246 static void
fexit_trace_func(struct trace_fprobe * tf,unsigned long entry_ip,unsigned long ret_ip,struct pt_regs * regs)247 fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
248 		 unsigned long ret_ip, struct pt_regs *regs)
249 {
250 	struct event_file_link *link;
251 
252 	trace_probe_for_each_link_rcu(link, &tf->tp)
253 		__fexit_trace_func(tf, entry_ip, ret_ip, regs, link->file);
254 }
255 NOKPROBE_SYMBOL(fexit_trace_func);
256 
257 #ifdef CONFIG_PERF_EVENTS
258 
fentry_perf_func(struct trace_fprobe * tf,unsigned long entry_ip,struct pt_regs * regs)259 static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,
260 			    struct pt_regs *regs)
261 {
262 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);
263 	struct fentry_trace_entry_head *entry;
264 	struct hlist_head *head;
265 	int size, __size, dsize;
266 	int rctx;
267 
268 	head = this_cpu_ptr(call->perf_events);
269 	if (hlist_empty(head))
270 		return 0;
271 
272 	dsize = __get_data_size(&tf->tp, regs);
273 	__size = sizeof(*entry) + tf->tp.size + dsize;
274 	size = ALIGN(__size + sizeof(u32), sizeof(u64));
275 	size -= sizeof(u32);
276 
277 	entry = perf_trace_buf_alloc(size, NULL, &rctx);
278 	if (!entry)
279 		return 0;
280 
281 	entry->ip = entry_ip;
282 	memset(&entry[1], 0, dsize);
283 	store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize);
284 	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
285 			      head, NULL);
286 	return 0;
287 }
288 NOKPROBE_SYMBOL(fentry_perf_func);
289 
290 static void
fexit_perf_func(struct trace_fprobe * tf,unsigned long entry_ip,unsigned long ret_ip,struct pt_regs * regs)291 fexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,
292 		unsigned long ret_ip, struct pt_regs *regs)
293 {
294 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);
295 	struct fexit_trace_entry_head *entry;
296 	struct hlist_head *head;
297 	int size, __size, dsize;
298 	int rctx;
299 
300 	head = this_cpu_ptr(call->perf_events);
301 	if (hlist_empty(head))
302 		return;
303 
304 	dsize = __get_data_size(&tf->tp, regs);
305 	__size = sizeof(*entry) + tf->tp.size + dsize;
306 	size = ALIGN(__size + sizeof(u32), sizeof(u64));
307 	size -= sizeof(u32);
308 
309 	entry = perf_trace_buf_alloc(size, NULL, &rctx);
310 	if (!entry)
311 		return;
312 
313 	entry->func = entry_ip;
314 	entry->ret_ip = ret_ip;
315 	store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize);
316 	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
317 			      head, NULL);
318 }
319 NOKPROBE_SYMBOL(fexit_perf_func);
320 #endif	/* CONFIG_PERF_EVENTS */
321 
fentry_dispatcher(struct fprobe * fp,unsigned long entry_ip,unsigned long ret_ip,struct pt_regs * regs,void * entry_data)322 static int fentry_dispatcher(struct fprobe *fp, unsigned long entry_ip,
323 			     unsigned long ret_ip, struct pt_regs *regs,
324 			     void *entry_data)
325 {
326 	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
327 	int ret = 0;
328 
329 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE))
330 		fentry_trace_func(tf, entry_ip, regs);
331 #ifdef CONFIG_PERF_EVENTS
332 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE))
333 		ret = fentry_perf_func(tf, entry_ip, regs);
334 #endif
335 	return ret;
336 }
337 NOKPROBE_SYMBOL(fentry_dispatcher);
338 
fexit_dispatcher(struct fprobe * fp,unsigned long entry_ip,unsigned long ret_ip,struct pt_regs * regs,void * entry_data)339 static void fexit_dispatcher(struct fprobe *fp, unsigned long entry_ip,
340 			     unsigned long ret_ip, struct pt_regs *regs,
341 			     void *entry_data)
342 {
343 	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
344 
345 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE))
346 		fexit_trace_func(tf, entry_ip, ret_ip, regs);
347 #ifdef CONFIG_PERF_EVENTS
348 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE))
349 		fexit_perf_func(tf, entry_ip, ret_ip, regs);
350 #endif
351 }
352 NOKPROBE_SYMBOL(fexit_dispatcher);
353 
free_trace_fprobe(struct trace_fprobe * tf)354 static void free_trace_fprobe(struct trace_fprobe *tf)
355 {
356 	if (tf) {
357 		trace_probe_cleanup(&tf->tp);
358 		kfree(tf->symbol);
359 		kfree(tf);
360 	}
361 }
362 
363 /*
364  * Allocate new trace_probe and initialize it (including fprobe).
365  */
alloc_trace_fprobe(const char * group,const char * event,const char * symbol,struct tracepoint * tpoint,int maxactive,int nargs,bool is_return)366 static struct trace_fprobe *alloc_trace_fprobe(const char *group,
367 					       const char *event,
368 					       const char *symbol,
369 					       struct tracepoint *tpoint,
370 					       int maxactive,
371 					       int nargs, bool is_return)
372 {
373 	struct trace_fprobe *tf;
374 	int ret = -ENOMEM;
375 
376 	tf = kzalloc(struct_size(tf, tp.args, nargs), GFP_KERNEL);
377 	if (!tf)
378 		return ERR_PTR(ret);
379 
380 	tf->symbol = kstrdup(symbol, GFP_KERNEL);
381 	if (!tf->symbol)
382 		goto error;
383 
384 	if (is_return)
385 		tf->fp.exit_handler = fexit_dispatcher;
386 	else
387 		tf->fp.entry_handler = fentry_dispatcher;
388 
389 	tf->tpoint = tpoint;
390 	tf->fp.nr_maxactive = maxactive;
391 
392 	ret = trace_probe_init(&tf->tp, event, group, false);
393 	if (ret < 0)
394 		goto error;
395 
396 	dyn_event_init(&tf->devent, &trace_fprobe_ops);
397 	return tf;
398 error:
399 	free_trace_fprobe(tf);
400 	return ERR_PTR(ret);
401 }
402 
find_trace_fprobe(const char * event,const char * group)403 static struct trace_fprobe *find_trace_fprobe(const char *event,
404 					      const char *group)
405 {
406 	struct dyn_event *pos;
407 	struct trace_fprobe *tf;
408 
409 	for_each_trace_fprobe(tf, pos)
410 		if (strcmp(trace_probe_name(&tf->tp), event) == 0 &&
411 		    strcmp(trace_probe_group_name(&tf->tp), group) == 0)
412 			return tf;
413 	return NULL;
414 }
415 
__enable_trace_fprobe(struct trace_fprobe * tf)416 static inline int __enable_trace_fprobe(struct trace_fprobe *tf)
417 {
418 	if (trace_fprobe_is_registered(tf))
419 		enable_fprobe(&tf->fp);
420 
421 	return 0;
422 }
423 
__disable_trace_fprobe(struct trace_probe * tp)424 static void __disable_trace_fprobe(struct trace_probe *tp)
425 {
426 	struct trace_fprobe *tf;
427 
428 	list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {
429 		if (!trace_fprobe_is_registered(tf))
430 			continue;
431 		disable_fprobe(&tf->fp);
432 	}
433 }
434 
435 /*
436  * Enable trace_probe
437  * if the file is NULL, enable "perf" handler, or enable "trace" handler.
438  */
enable_trace_fprobe(struct trace_event_call * call,struct trace_event_file * file)439 static int enable_trace_fprobe(struct trace_event_call *call,
440 			       struct trace_event_file *file)
441 {
442 	struct trace_probe *tp;
443 	struct trace_fprobe *tf;
444 	bool enabled;
445 	int ret = 0;
446 
447 	tp = trace_probe_primary_from_call(call);
448 	if (WARN_ON_ONCE(!tp))
449 		return -ENODEV;
450 	enabled = trace_probe_is_enabled(tp);
451 
452 	/* This also changes "enabled" state */
453 	if (file) {
454 		ret = trace_probe_add_file(tp, file);
455 		if (ret)
456 			return ret;
457 	} else
458 		trace_probe_set_flag(tp, TP_FLAG_PROFILE);
459 
460 	if (!enabled) {
461 		list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {
462 			/* TODO: check the fprobe is gone */
463 			__enable_trace_fprobe(tf);
464 		}
465 	}
466 
467 	return 0;
468 }
469 
470 /*
471  * Disable trace_probe
472  * if the file is NULL, disable "perf" handler, or disable "trace" handler.
473  */
disable_trace_fprobe(struct trace_event_call * call,struct trace_event_file * file)474 static int disable_trace_fprobe(struct trace_event_call *call,
475 				struct trace_event_file *file)
476 {
477 	struct trace_probe *tp;
478 
479 	tp = trace_probe_primary_from_call(call);
480 	if (WARN_ON_ONCE(!tp))
481 		return -ENODEV;
482 
483 	if (file) {
484 		if (!trace_probe_get_file_link(tp, file))
485 			return -ENOENT;
486 		if (!trace_probe_has_single_file(tp))
487 			goto out;
488 		trace_probe_clear_flag(tp, TP_FLAG_TRACE);
489 	} else
490 		trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
491 
492 	if (!trace_probe_is_enabled(tp))
493 		__disable_trace_fprobe(tp);
494 
495  out:
496 	if (file)
497 		/*
498 		 * Synchronization is done in below function. For perf event,
499 		 * file == NULL and perf_trace_event_unreg() calls
500 		 * tracepoint_synchronize_unregister() to ensure synchronize
501 		 * event. We don't need to care about it.
502 		 */
503 		trace_probe_remove_file(tp, file);
504 
505 	return 0;
506 }
507 
508 /* Event entry printers */
509 static enum print_line_t
print_fentry_event(struct trace_iterator * iter,int flags,struct trace_event * event)510 print_fentry_event(struct trace_iterator *iter, int flags,
511 		   struct trace_event *event)
512 {
513 	struct fentry_trace_entry_head *field;
514 	struct trace_seq *s = &iter->seq;
515 	struct trace_probe *tp;
516 
517 	field = (struct fentry_trace_entry_head *)iter->ent;
518 	tp = trace_probe_primary_from_call(
519 		container_of(event, struct trace_event_call, event));
520 	if (WARN_ON_ONCE(!tp))
521 		goto out;
522 
523 	trace_seq_printf(s, "%s: (", trace_probe_name(tp));
524 
525 	if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
526 		goto out;
527 
528 	trace_seq_putc(s, ')');
529 
530 	if (trace_probe_print_args(s, tp->args, tp->nr_args,
531 			     (u8 *)&field[1], field) < 0)
532 		goto out;
533 
534 	trace_seq_putc(s, '\n');
535  out:
536 	return trace_handle_return(s);
537 }
538 
539 static enum print_line_t
print_fexit_event(struct trace_iterator * iter,int flags,struct trace_event * event)540 print_fexit_event(struct trace_iterator *iter, int flags,
541 		  struct trace_event *event)
542 {
543 	struct fexit_trace_entry_head *field;
544 	struct trace_seq *s = &iter->seq;
545 	struct trace_probe *tp;
546 
547 	field = (struct fexit_trace_entry_head *)iter->ent;
548 	tp = trace_probe_primary_from_call(
549 		container_of(event, struct trace_event_call, event));
550 	if (WARN_ON_ONCE(!tp))
551 		goto out;
552 
553 	trace_seq_printf(s, "%s: (", trace_probe_name(tp));
554 
555 	if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
556 		goto out;
557 
558 	trace_seq_puts(s, " <- ");
559 
560 	if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
561 		goto out;
562 
563 	trace_seq_putc(s, ')');
564 
565 	if (trace_probe_print_args(s, tp->args, tp->nr_args,
566 			     (u8 *)&field[1], field) < 0)
567 		goto out;
568 
569 	trace_seq_putc(s, '\n');
570 
571  out:
572 	return trace_handle_return(s);
573 }
574 
fentry_event_define_fields(struct trace_event_call * event_call)575 static int fentry_event_define_fields(struct trace_event_call *event_call)
576 {
577 	int ret;
578 	struct fentry_trace_entry_head field;
579 	struct trace_probe *tp;
580 
581 	tp = trace_probe_primary_from_call(event_call);
582 	if (WARN_ON_ONCE(!tp))
583 		return -ENOENT;
584 
585 	DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
586 
587 	return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
588 }
589 
fexit_event_define_fields(struct trace_event_call * event_call)590 static int fexit_event_define_fields(struct trace_event_call *event_call)
591 {
592 	int ret;
593 	struct fexit_trace_entry_head field;
594 	struct trace_probe *tp;
595 
596 	tp = trace_probe_primary_from_call(event_call);
597 	if (WARN_ON_ONCE(!tp))
598 		return -ENOENT;
599 
600 	DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
601 	DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
602 
603 	return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
604 }
605 
606 static struct trace_event_functions fentry_funcs = {
607 	.trace		= print_fentry_event
608 };
609 
610 static struct trace_event_functions fexit_funcs = {
611 	.trace		= print_fexit_event
612 };
613 
614 static struct trace_event_fields fentry_fields_array[] = {
615 	{ .type = TRACE_FUNCTION_TYPE,
616 	  .define_fields = fentry_event_define_fields },
617 	{}
618 };
619 
620 static struct trace_event_fields fexit_fields_array[] = {
621 	{ .type = TRACE_FUNCTION_TYPE,
622 	  .define_fields = fexit_event_define_fields },
623 	{}
624 };
625 
626 static int fprobe_register(struct trace_event_call *event,
627 			   enum trace_reg type, void *data);
628 
init_trace_event_call(struct trace_fprobe * tf)629 static inline void init_trace_event_call(struct trace_fprobe *tf)
630 {
631 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);
632 
633 	if (trace_fprobe_is_return(tf)) {
634 		call->event.funcs = &fexit_funcs;
635 		call->class->fields_array = fexit_fields_array;
636 	} else {
637 		call->event.funcs = &fentry_funcs;
638 		call->class->fields_array = fentry_fields_array;
639 	}
640 
641 	call->flags = TRACE_EVENT_FL_FPROBE;
642 	call->class->reg = fprobe_register;
643 }
644 
register_fprobe_event(struct trace_fprobe * tf)645 static int register_fprobe_event(struct trace_fprobe *tf)
646 {
647 	init_trace_event_call(tf);
648 
649 	return trace_probe_register_event_call(&tf->tp);
650 }
651 
unregister_fprobe_event(struct trace_fprobe * tf)652 static int unregister_fprobe_event(struct trace_fprobe *tf)
653 {
654 	return trace_probe_unregister_event_call(&tf->tp);
655 }
656 
657 /* Internal register function - just handle fprobe and flags */
__register_trace_fprobe(struct trace_fprobe * tf)658 static int __register_trace_fprobe(struct trace_fprobe *tf)
659 {
660 	int i, ret;
661 
662 	/* Should we need new LOCKDOWN flag for fprobe? */
663 	ret = security_locked_down(LOCKDOWN_KPROBES);
664 	if (ret)
665 		return ret;
666 
667 	if (trace_fprobe_is_registered(tf))
668 		return -EINVAL;
669 
670 	for (i = 0; i < tf->tp.nr_args; i++) {
671 		ret = traceprobe_update_arg(&tf->tp.args[i]);
672 		if (ret)
673 			return ret;
674 	}
675 
676 	/* Set/clear disabled flag according to tp->flag */
677 	if (trace_probe_is_enabled(&tf->tp))
678 		tf->fp.flags &= ~FPROBE_FL_DISABLED;
679 	else
680 		tf->fp.flags |= FPROBE_FL_DISABLED;
681 
682 	if (trace_fprobe_is_tracepoint(tf)) {
683 		struct tracepoint *tpoint = tf->tpoint;
684 		unsigned long ip = (unsigned long)tpoint->probestub;
685 		/*
686 		 * Here, we do 2 steps to enable fprobe on a tracepoint.
687 		 * At first, put __probestub_##TP function on the tracepoint
688 		 * and put a fprobe on the stub function.
689 		 */
690 		ret = tracepoint_probe_register_prio_may_exist(tpoint,
691 					tpoint->probestub, NULL, 0);
692 		if (ret < 0)
693 			return ret;
694 		return register_fprobe_ips(&tf->fp, &ip, 1);
695 	}
696 
697 	/* TODO: handle filter, nofilter or symbol list */
698 	return register_fprobe(&tf->fp, tf->symbol, NULL);
699 }
700 
701 /* Internal unregister function - just handle fprobe and flags */
__unregister_trace_fprobe(struct trace_fprobe * tf)702 static void __unregister_trace_fprobe(struct trace_fprobe *tf)
703 {
704 	if (trace_fprobe_is_registered(tf)) {
705 		unregister_fprobe(&tf->fp);
706 		memset(&tf->fp, 0, sizeof(tf->fp));
707 		if (trace_fprobe_is_tracepoint(tf)) {
708 			tracepoint_probe_unregister(tf->tpoint,
709 					tf->tpoint->probestub, NULL);
710 			tf->tpoint = NULL;
711 			tf->mod = NULL;
712 		}
713 	}
714 }
715 
716 /* TODO: make this trace_*probe common function */
717 /* Unregister a trace_probe and probe_event */
unregister_trace_fprobe(struct trace_fprobe * tf)718 static int unregister_trace_fprobe(struct trace_fprobe *tf)
719 {
720 	/* If other probes are on the event, just unregister fprobe */
721 	if (trace_probe_has_sibling(&tf->tp))
722 		goto unreg;
723 
724 	/* Enabled event can not be unregistered */
725 	if (trace_probe_is_enabled(&tf->tp))
726 		return -EBUSY;
727 
728 	/* If there's a reference to the dynamic event */
729 	if (trace_event_dyn_busy(trace_probe_event_call(&tf->tp)))
730 		return -EBUSY;
731 
732 	/* Will fail if probe is being used by ftrace or perf */
733 	if (unregister_fprobe_event(tf))
734 		return -EBUSY;
735 
736 unreg:
737 	__unregister_trace_fprobe(tf);
738 	dyn_event_remove(&tf->devent);
739 	trace_probe_unlink(&tf->tp);
740 
741 	return 0;
742 }
743 
trace_fprobe_has_same_fprobe(struct trace_fprobe * orig,struct trace_fprobe * comp)744 static bool trace_fprobe_has_same_fprobe(struct trace_fprobe *orig,
745 					 struct trace_fprobe *comp)
746 {
747 	struct trace_probe_event *tpe = orig->tp.event;
748 	int i;
749 
750 	list_for_each_entry(orig, &tpe->probes, tp.list) {
751 		if (strcmp(trace_fprobe_symbol(orig),
752 			   trace_fprobe_symbol(comp)))
753 			continue;
754 
755 		/*
756 		 * trace_probe_compare_arg_type() ensured that nr_args and
757 		 * each argument name and type are same. Let's compare comm.
758 		 */
759 		for (i = 0; i < orig->tp.nr_args; i++) {
760 			if (strcmp(orig->tp.args[i].comm,
761 				   comp->tp.args[i].comm))
762 				break;
763 		}
764 
765 		if (i == orig->tp.nr_args)
766 			return true;
767 	}
768 
769 	return false;
770 }
771 
append_trace_fprobe(struct trace_fprobe * tf,struct trace_fprobe * to)772 static int append_trace_fprobe(struct trace_fprobe *tf, struct trace_fprobe *to)
773 {
774 	int ret;
775 
776 	if (trace_fprobe_is_return(tf) != trace_fprobe_is_return(to) ||
777 	    trace_fprobe_is_tracepoint(tf) != trace_fprobe_is_tracepoint(to)) {
778 		trace_probe_log_set_index(0);
779 		trace_probe_log_err(0, DIFF_PROBE_TYPE);
780 		return -EEXIST;
781 	}
782 	ret = trace_probe_compare_arg_type(&tf->tp, &to->tp);
783 	if (ret) {
784 		/* Note that argument starts index = 2 */
785 		trace_probe_log_set_index(ret + 1);
786 		trace_probe_log_err(0, DIFF_ARG_TYPE);
787 		return -EEXIST;
788 	}
789 	if (trace_fprobe_has_same_fprobe(to, tf)) {
790 		trace_probe_log_set_index(0);
791 		trace_probe_log_err(0, SAME_PROBE);
792 		return -EEXIST;
793 	}
794 
795 	/* Append to existing event */
796 	ret = trace_probe_append(&tf->tp, &to->tp);
797 	if (ret)
798 		return ret;
799 
800 	ret = __register_trace_fprobe(tf);
801 	if (ret)
802 		trace_probe_unlink(&tf->tp);
803 	else
804 		dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));
805 
806 	return ret;
807 }
808 
809 /* Register a trace_probe and probe_event */
register_trace_fprobe(struct trace_fprobe * tf)810 static int register_trace_fprobe(struct trace_fprobe *tf)
811 {
812 	struct trace_fprobe *old_tf;
813 	int ret;
814 
815 	mutex_lock(&event_mutex);
816 
817 	old_tf = find_trace_fprobe(trace_probe_name(&tf->tp),
818 				   trace_probe_group_name(&tf->tp));
819 	if (old_tf) {
820 		ret = append_trace_fprobe(tf, old_tf);
821 		goto end;
822 	}
823 
824 	/* Register new event */
825 	ret = register_fprobe_event(tf);
826 	if (ret) {
827 		if (ret == -EEXIST) {
828 			trace_probe_log_set_index(0);
829 			trace_probe_log_err(0, EVENT_EXIST);
830 		} else
831 			pr_warn("Failed to register probe event(%d)\n", ret);
832 		goto end;
833 	}
834 
835 	/* Register fprobe */
836 	ret = __register_trace_fprobe(tf);
837 	if (ret < 0)
838 		unregister_fprobe_event(tf);
839 	else
840 		dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));
841 
842 end:
843 	mutex_unlock(&event_mutex);
844 	return ret;
845 }
846 
847 #ifdef CONFIG_MODULES
__tracepoint_probe_module_cb(struct notifier_block * self,unsigned long val,void * data)848 static int __tracepoint_probe_module_cb(struct notifier_block *self,
849 					unsigned long val, void *data)
850 {
851 	struct tp_module *tp_mod = data;
852 	struct trace_fprobe *tf;
853 	struct dyn_event *pos;
854 
855 	if (val != MODULE_STATE_GOING)
856 		return NOTIFY_DONE;
857 
858 	mutex_lock(&event_mutex);
859 	for_each_trace_fprobe(tf, pos) {
860 		if (tp_mod->mod == tf->mod) {
861 			tracepoint_probe_unregister(tf->tpoint,
862 					tf->tpoint->probestub, NULL);
863 			tf->tpoint = NULL;
864 			tf->mod = NULL;
865 		}
866 	}
867 	mutex_unlock(&event_mutex);
868 
869 	return NOTIFY_DONE;
870 }
871 
872 static struct notifier_block tracepoint_module_nb = {
873 	.notifier_call = __tracepoint_probe_module_cb,
874 };
875 #endif /* CONFIG_MODULES */
876 
877 struct __find_tracepoint_cb_data {
878 	const char *tp_name;
879 	struct tracepoint *tpoint;
880 };
881 
__find_tracepoint_cb(struct tracepoint * tp,void * priv)882 static void __find_tracepoint_cb(struct tracepoint *tp, void *priv)
883 {
884 	struct __find_tracepoint_cb_data *data = priv;
885 
886 	if (!data->tpoint && !strcmp(data->tp_name, tp->name))
887 		data->tpoint = tp;
888 }
889 
find_tracepoint(const char * tp_name)890 static struct tracepoint *find_tracepoint(const char *tp_name)
891 {
892 	struct __find_tracepoint_cb_data data = {
893 		.tp_name = tp_name,
894 	};
895 
896 	for_each_kernel_tracepoint(__find_tracepoint_cb, &data);
897 
898 	return data.tpoint;
899 }
900 
parse_symbol_and_return(int argc,const char * argv[],char ** symbol,bool * is_return,bool is_tracepoint)901 static int parse_symbol_and_return(int argc, const char *argv[],
902 				   char **symbol, bool *is_return,
903 				   bool is_tracepoint)
904 {
905 	char *tmp = strchr(argv[1], '%');
906 	int i;
907 
908 	if (tmp) {
909 		int len = tmp - argv[1];
910 
911 		if (!is_tracepoint && !strcmp(tmp, "%return")) {
912 			*is_return = true;
913 		} else {
914 			trace_probe_log_err(len, BAD_ADDR_SUFFIX);
915 			return -EINVAL;
916 		}
917 		*symbol = kmemdup_nul(argv[1], len, GFP_KERNEL);
918 	} else
919 		*symbol = kstrdup(argv[1], GFP_KERNEL);
920 	if (!*symbol)
921 		return -ENOMEM;
922 
923 	if (*is_return)
924 		return 0;
925 
926 	/* If there is $retval, this should be a return fprobe. */
927 	for (i = 2; i < argc; i++) {
928 		tmp = strstr(argv[i], "$retval");
929 		if (tmp && !isalnum(tmp[7]) && tmp[7] != '_') {
930 			if (is_tracepoint) {
931 				trace_probe_log_set_index(i);
932 				trace_probe_log_err(tmp - argv[i], RETVAL_ON_PROBE);
933 				return -EINVAL;
934 			}
935 			*is_return = true;
936 			break;
937 		}
938 	}
939 	return 0;
940 }
941 
__trace_fprobe_create(int argc,const char * argv[])942 static int __trace_fprobe_create(int argc, const char *argv[])
943 {
944 	/*
945 	 * Argument syntax:
946 	 *  - Add fentry probe:
947 	 *      f[:[GRP/][EVENT]] [MOD:]KSYM [FETCHARGS]
948 	 *  - Add fexit probe:
949 	 *      f[N][:[GRP/][EVENT]] [MOD:]KSYM%return [FETCHARGS]
950 	 *  - Add tracepoint probe:
951 	 *      t[:[GRP/][EVENT]] TRACEPOINT [FETCHARGS]
952 	 *
953 	 * Fetch args:
954 	 *  $retval	: fetch return value
955 	 *  $stack	: fetch stack address
956 	 *  $stackN	: fetch Nth entry of stack (N:0-)
957 	 *  $argN	: fetch Nth argument (N:1-)
958 	 *  $comm       : fetch current task comm
959 	 *  @ADDR	: fetch memory at ADDR (ADDR should be in kernel)
960 	 *  @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
961 	 * Dereferencing memory fetch:
962 	 *  +|-offs(ARG) : fetch memory at ARG +|- offs address.
963 	 * Alias name of args:
964 	 *  NAME=FETCHARG : set NAME as alias of FETCHARG.
965 	 * Type of args:
966 	 *  FETCHARG:TYPE : use TYPE instead of unsigned long.
967 	 */
968 	struct trace_fprobe *tf = NULL;
969 	int i, len, new_argc = 0, ret = 0;
970 	bool is_return = false;
971 	char *symbol = NULL;
972 	const char *event = NULL, *group = FPROBE_EVENT_SYSTEM;
973 	const char **new_argv = NULL;
974 	int maxactive = 0;
975 	char buf[MAX_EVENT_NAME_LEN];
976 	char gbuf[MAX_EVENT_NAME_LEN];
977 	char sbuf[KSYM_NAME_LEN];
978 	char abuf[MAX_BTF_ARGS_LEN];
979 	bool is_tracepoint = false;
980 	struct tracepoint *tpoint = NULL;
981 	struct traceprobe_parse_context ctx = {
982 		.flags = TPARG_FL_KERNEL | TPARG_FL_FPROBE,
983 	};
984 
985 	if ((argv[0][0] != 'f' && argv[0][0] != 't') || argc < 2)
986 		return -ECANCELED;
987 
988 	if (argv[0][0] == 't') {
989 		is_tracepoint = true;
990 		group = TRACEPOINT_EVENT_SYSTEM;
991 	}
992 
993 	trace_probe_log_init("trace_fprobe", argc, argv);
994 
995 	event = strchr(&argv[0][1], ':');
996 	if (event)
997 		event++;
998 
999 	if (isdigit(argv[0][1])) {
1000 		if (event)
1001 			len = event - &argv[0][1] - 1;
1002 		else
1003 			len = strlen(&argv[0][1]);
1004 		if (len > MAX_EVENT_NAME_LEN - 1) {
1005 			trace_probe_log_err(1, BAD_MAXACT);
1006 			goto parse_error;
1007 		}
1008 		memcpy(buf, &argv[0][1], len);
1009 		buf[len] = '\0';
1010 		ret = kstrtouint(buf, 0, &maxactive);
1011 		if (ret || !maxactive) {
1012 			trace_probe_log_err(1, BAD_MAXACT);
1013 			goto parse_error;
1014 		}
1015 		/* fprobe rethook instances are iterated over via a list. The
1016 		 * maximum should stay reasonable.
1017 		 */
1018 		if (maxactive > RETHOOK_MAXACTIVE_MAX) {
1019 			trace_probe_log_err(1, MAXACT_TOO_BIG);
1020 			goto parse_error;
1021 		}
1022 	}
1023 
1024 	trace_probe_log_set_index(1);
1025 
1026 	/* a symbol(or tracepoint) must be specified */
1027 	ret = parse_symbol_and_return(argc, argv, &symbol, &is_return, is_tracepoint);
1028 	if (ret < 0)
1029 		goto parse_error;
1030 
1031 	if (!is_return && maxactive) {
1032 		trace_probe_log_set_index(0);
1033 		trace_probe_log_err(1, BAD_MAXACT_TYPE);
1034 		goto parse_error;
1035 	}
1036 
1037 	trace_probe_log_set_index(0);
1038 	if (event) {
1039 		ret = traceprobe_parse_event_name(&event, &group, gbuf,
1040 						  event - argv[0]);
1041 		if (ret)
1042 			goto parse_error;
1043 	}
1044 
1045 	if (!event) {
1046 		/* Make a new event name */
1047 		if (is_tracepoint)
1048 			snprintf(buf, MAX_EVENT_NAME_LEN, "%s%s",
1049 				 isdigit(*symbol) ? "_" : "", symbol);
1050 		else
1051 			snprintf(buf, MAX_EVENT_NAME_LEN, "%s__%s", symbol,
1052 				 is_return ? "exit" : "entry");
1053 		sanitize_event_name(buf);
1054 		event = buf;
1055 	}
1056 
1057 	if (is_return)
1058 		ctx.flags |= TPARG_FL_RETURN;
1059 	else
1060 		ctx.flags |= TPARG_FL_FENTRY;
1061 
1062 	if (is_tracepoint) {
1063 		ctx.flags |= TPARG_FL_TPOINT;
1064 		tpoint = find_tracepoint(symbol);
1065 		if (!tpoint) {
1066 			trace_probe_log_set_index(1);
1067 			trace_probe_log_err(0, NO_TRACEPOINT);
1068 			goto parse_error;
1069 		}
1070 		ctx.funcname = kallsyms_lookup(
1071 				(unsigned long)tpoint->probestub,
1072 				NULL, NULL, NULL, sbuf);
1073 	} else
1074 		ctx.funcname = symbol;
1075 
1076 	argc -= 2; argv += 2;
1077 	new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc,
1078 					       abuf, MAX_BTF_ARGS_LEN, &ctx);
1079 	if (IS_ERR(new_argv)) {
1080 		ret = PTR_ERR(new_argv);
1081 		new_argv = NULL;
1082 		goto out;
1083 	}
1084 	if (new_argv) {
1085 		argc = new_argc;
1086 		argv = new_argv;
1087 	}
1088 
1089 	/* setup a probe */
1090 	tf = alloc_trace_fprobe(group, event, symbol, tpoint, maxactive,
1091 				argc, is_return);
1092 	if (IS_ERR(tf)) {
1093 		ret = PTR_ERR(tf);
1094 		/* This must return -ENOMEM, else there is a bug */
1095 		WARN_ON_ONCE(ret != -ENOMEM);
1096 		goto out;	/* We know tf is not allocated */
1097 	}
1098 
1099 	if (is_tracepoint)
1100 		tf->mod = __module_text_address(
1101 				(unsigned long)tf->tpoint->probestub);
1102 
1103 	/* parse arguments */
1104 	for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
1105 		trace_probe_log_set_index(i + 2);
1106 		ctx.offset = 0;
1107 		ret = traceprobe_parse_probe_arg(&tf->tp, i, argv[i], &ctx);
1108 		if (ret)
1109 			goto error;	/* This can be -ENOMEM */
1110 	}
1111 
1112 	ret = traceprobe_set_print_fmt(&tf->tp,
1113 			is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL);
1114 	if (ret < 0)
1115 		goto error;
1116 
1117 	ret = register_trace_fprobe(tf);
1118 	if (ret) {
1119 		trace_probe_log_set_index(1);
1120 		if (ret == -EILSEQ)
1121 			trace_probe_log_err(0, BAD_INSN_BNDRY);
1122 		else if (ret == -ENOENT)
1123 			trace_probe_log_err(0, BAD_PROBE_ADDR);
1124 		else if (ret != -ENOMEM && ret != -EEXIST)
1125 			trace_probe_log_err(0, FAIL_REG_PROBE);
1126 		goto error;
1127 	}
1128 
1129 out:
1130 	traceprobe_finish_parse(&ctx);
1131 	trace_probe_log_clear();
1132 	kfree(new_argv);
1133 	kfree(symbol);
1134 	return ret;
1135 
1136 parse_error:
1137 	ret = -EINVAL;
1138 error:
1139 	free_trace_fprobe(tf);
1140 	goto out;
1141 }
1142 
trace_fprobe_create(const char * raw_command)1143 static int trace_fprobe_create(const char *raw_command)
1144 {
1145 	return trace_probe_create(raw_command, __trace_fprobe_create);
1146 }
1147 
trace_fprobe_release(struct dyn_event * ev)1148 static int trace_fprobe_release(struct dyn_event *ev)
1149 {
1150 	struct trace_fprobe *tf = to_trace_fprobe(ev);
1151 	int ret = unregister_trace_fprobe(tf);
1152 
1153 	if (!ret)
1154 		free_trace_fprobe(tf);
1155 	return ret;
1156 }
1157 
trace_fprobe_show(struct seq_file * m,struct dyn_event * ev)1158 static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev)
1159 {
1160 	struct trace_fprobe *tf = to_trace_fprobe(ev);
1161 	int i;
1162 
1163 	if (trace_fprobe_is_tracepoint(tf))
1164 		seq_putc(m, 't');
1165 	else
1166 		seq_putc(m, 'f');
1167 	if (trace_fprobe_is_return(tf) && tf->fp.nr_maxactive)
1168 		seq_printf(m, "%d", tf->fp.nr_maxactive);
1169 	seq_printf(m, ":%s/%s", trace_probe_group_name(&tf->tp),
1170 				trace_probe_name(&tf->tp));
1171 
1172 	seq_printf(m, " %s%s", trace_fprobe_symbol(tf),
1173 			       trace_fprobe_is_return(tf) ? "%return" : "");
1174 
1175 	for (i = 0; i < tf->tp.nr_args; i++)
1176 		seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm);
1177 	seq_putc(m, '\n');
1178 
1179 	return 0;
1180 }
1181 
1182 /*
1183  * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1184  */
fprobe_register(struct trace_event_call * event,enum trace_reg type,void * data)1185 static int fprobe_register(struct trace_event_call *event,
1186 			   enum trace_reg type, void *data)
1187 {
1188 	struct trace_event_file *file = data;
1189 
1190 	switch (type) {
1191 	case TRACE_REG_REGISTER:
1192 		return enable_trace_fprobe(event, file);
1193 	case TRACE_REG_UNREGISTER:
1194 		return disable_trace_fprobe(event, file);
1195 
1196 #ifdef CONFIG_PERF_EVENTS
1197 	case TRACE_REG_PERF_REGISTER:
1198 		return enable_trace_fprobe(event, NULL);
1199 	case TRACE_REG_PERF_UNREGISTER:
1200 		return disable_trace_fprobe(event, NULL);
1201 	case TRACE_REG_PERF_OPEN:
1202 	case TRACE_REG_PERF_CLOSE:
1203 	case TRACE_REG_PERF_ADD:
1204 	case TRACE_REG_PERF_DEL:
1205 		return 0;
1206 #endif
1207 	}
1208 	return 0;
1209 }
1210 
1211 /*
1212  * Register dynevent at core_initcall. This allows kernel to setup fprobe
1213  * events in postcore_initcall without tracefs.
1214  */
init_fprobe_trace_early(void)1215 static __init int init_fprobe_trace_early(void)
1216 {
1217 	int ret;
1218 
1219 	ret = dyn_event_register(&trace_fprobe_ops);
1220 	if (ret)
1221 		return ret;
1222 
1223 #ifdef CONFIG_MODULES
1224 	ret = register_tracepoint_module_notifier(&tracepoint_module_nb);
1225 	if (ret)
1226 		return ret;
1227 #endif
1228 
1229 	return 0;
1230 }
1231 core_initcall(init_fprobe_trace_early);
1232