xref: /openbmc/linux/kernel/trace/trace_eprobe.c (revision 53f9cd5c)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * event probes
4  *
5  * Part of this code was copied from kernel/trace/trace_kprobe.c written by
6  * Masami Hiramatsu <mhiramat@kernel.org>
7  *
8  * Copyright (C) 2021, VMware Inc, Steven Rostedt <rostedt@goodmis.org>
9  * Copyright (C) 2021, VMware Inc, Tzvetomir Stoyanov tz.stoyanov@gmail.com>
10  *
11  */
12 #include <linux/module.h>
13 #include <linux/mutex.h>
14 #include <linux/ftrace.h>
15 
16 #include "trace_dynevent.h"
17 #include "trace_probe.h"
18 #include "trace_probe_tmpl.h"
19 
20 #define EPROBE_EVENT_SYSTEM "eprobes"
21 
22 struct trace_eprobe {
23 	/* tracepoint system */
24 	const char *event_system;
25 
26 	/* tracepoint event */
27 	const char *event_name;
28 
29 	struct trace_event_call *event;
30 
31 	struct dyn_event	devent;
32 	struct trace_probe	tp;
33 };
34 
35 struct eprobe_data {
36 	struct trace_event_file	*file;
37 	struct trace_eprobe	*ep;
38 };
39 
40 static int __trace_eprobe_create(int argc, const char *argv[]);
41 
42 static void trace_event_probe_cleanup(struct trace_eprobe *ep)
43 {
44 	if (!ep)
45 		return;
46 	trace_probe_cleanup(&ep->tp);
47 	kfree(ep->event_name);
48 	kfree(ep->event_system);
49 	if (ep->event)
50 		trace_event_put_ref(ep->event);
51 	kfree(ep);
52 }
53 
54 static struct trace_eprobe *to_trace_eprobe(struct dyn_event *ev)
55 {
56 	return container_of(ev, struct trace_eprobe, devent);
57 }
58 
59 static int eprobe_dyn_event_create(const char *raw_command)
60 {
61 	return trace_probe_create(raw_command, __trace_eprobe_create);
62 }
63 
64 static int eprobe_dyn_event_show(struct seq_file *m, struct dyn_event *ev)
65 {
66 	struct trace_eprobe *ep = to_trace_eprobe(ev);
67 	int i;
68 
69 	seq_printf(m, "e:%s/%s", trace_probe_group_name(&ep->tp),
70 				trace_probe_name(&ep->tp));
71 	seq_printf(m, " %s.%s", ep->event_system, ep->event_name);
72 
73 	for (i = 0; i < ep->tp.nr_args; i++)
74 		seq_printf(m, " %s=%s", ep->tp.args[i].name, ep->tp.args[i].comm);
75 	seq_putc(m, '\n');
76 
77 	return 0;
78 }
79 
80 static int unregister_trace_eprobe(struct trace_eprobe *ep)
81 {
82 	/* If other probes are on the event, just unregister eprobe */
83 	if (trace_probe_has_sibling(&ep->tp))
84 		goto unreg;
85 
86 	/* Enabled event can not be unregistered */
87 	if (trace_probe_is_enabled(&ep->tp))
88 		return -EBUSY;
89 
90 	/* Will fail if probe is being used by ftrace or perf */
91 	if (trace_probe_unregister_event_call(&ep->tp))
92 		return -EBUSY;
93 
94 unreg:
95 	dyn_event_remove(&ep->devent);
96 	trace_probe_unlink(&ep->tp);
97 
98 	return 0;
99 }
100 
101 static int eprobe_dyn_event_release(struct dyn_event *ev)
102 {
103 	struct trace_eprobe *ep = to_trace_eprobe(ev);
104 	int ret = unregister_trace_eprobe(ep);
105 
106 	if (!ret)
107 		trace_event_probe_cleanup(ep);
108 	return ret;
109 }
110 
111 static bool eprobe_dyn_event_is_busy(struct dyn_event *ev)
112 {
113 	struct trace_eprobe *ep = to_trace_eprobe(ev);
114 
115 	return trace_probe_is_enabled(&ep->tp);
116 }
117 
118 static bool eprobe_dyn_event_match(const char *system, const char *event,
119 			int argc, const char **argv, struct dyn_event *ev)
120 {
121 	struct trace_eprobe *ep = to_trace_eprobe(ev);
122 	const char *slash;
123 
124 	/*
125 	 * We match the following:
126 	 *  event only			- match all eprobes with event name
127 	 *  system and event only	- match all system/event probes
128 	 *  system only			- match all system probes
129 	 *
130 	 * The below has the above satisfied with more arguments:
131 	 *
132 	 *  attached system/event	- If the arg has the system and event
133 	 *				  the probe is attached to, match
134 	 *				  probes with the attachment.
135 	 *
136 	 *  If any more args are given, then it requires a full match.
137 	 */
138 
139 	/*
140 	 * If system exists, but this probe is not part of that system
141 	 * do not match.
142 	 */
143 	if (system && strcmp(trace_probe_group_name(&ep->tp), system) != 0)
144 		return false;
145 
146 	/* Must match the event name */
147 	if (event[0] != '\0' && strcmp(trace_probe_name(&ep->tp), event) != 0)
148 		return false;
149 
150 	/* No arguments match all */
151 	if (argc < 1)
152 		return true;
153 
154 	/* First argument is the system/event the probe is attached to */
155 
156 	slash = strchr(argv[0], '/');
157 	if (!slash)
158 		slash = strchr(argv[0], '.');
159 	if (!slash)
160 		return false;
161 
162 	if (strncmp(ep->event_system, argv[0], slash - argv[0]))
163 		return false;
164 	if (strcmp(ep->event_name, slash + 1))
165 		return false;
166 
167 	argc--;
168 	argv++;
169 
170 	/* If there are no other args, then match */
171 	if (argc < 1)
172 		return true;
173 
174 	return trace_probe_match_command_args(&ep->tp, argc, argv);
175 }
176 
177 static struct dyn_event_operations eprobe_dyn_event_ops = {
178 	.create = eprobe_dyn_event_create,
179 	.show = eprobe_dyn_event_show,
180 	.is_busy = eprobe_dyn_event_is_busy,
181 	.free = eprobe_dyn_event_release,
182 	.match = eprobe_dyn_event_match,
183 };
184 
185 static struct trace_eprobe *alloc_event_probe(const char *group,
186 					      const char *this_event,
187 					      struct trace_event_call *event,
188 					      int nargs)
189 {
190 	struct trace_eprobe *ep;
191 	const char *event_name;
192 	const char *sys_name;
193 	int ret = -ENOMEM;
194 
195 	if (!event)
196 		return ERR_PTR(-ENODEV);
197 
198 	sys_name = event->class->system;
199 	event_name = trace_event_name(event);
200 
201 	ep = kzalloc(struct_size(ep, tp.args, nargs), GFP_KERNEL);
202 	if (!ep) {
203 		trace_event_put_ref(event);
204 		goto error;
205 	}
206 	ep->event = event;
207 	ep->event_name = kstrdup(event_name, GFP_KERNEL);
208 	if (!ep->event_name)
209 		goto error;
210 	ep->event_system = kstrdup(sys_name, GFP_KERNEL);
211 	if (!ep->event_system)
212 		goto error;
213 
214 	ret = trace_probe_init(&ep->tp, this_event, group, false);
215 	if (ret < 0)
216 		goto error;
217 
218 	dyn_event_init(&ep->devent, &eprobe_dyn_event_ops);
219 	return ep;
220 error:
221 	trace_event_probe_cleanup(ep);
222 	return ERR_PTR(ret);
223 }
224 
225 static int trace_eprobe_tp_arg_update(struct trace_eprobe *ep, int i)
226 {
227 	struct probe_arg *parg = &ep->tp.args[i];
228 	struct ftrace_event_field *field;
229 	struct list_head *head;
230 	int ret = -ENOENT;
231 
232 	head = trace_get_fields(ep->event);
233 	list_for_each_entry(field, head, link) {
234 		if (!strcmp(parg->code->data, field->name)) {
235 			kfree(parg->code->data);
236 			parg->code->data = field;
237 			return 0;
238 		}
239 	}
240 
241 	/*
242 	 * Argument not found on event. But allow for comm and COMM
243 	 * to be used to get the current->comm.
244 	 */
245 	if (strcmp(parg->code->data, "COMM") == 0 ||
246 	    strcmp(parg->code->data, "comm") == 0) {
247 		parg->code->op = FETCH_OP_COMM;
248 		ret = 0;
249 	}
250 
251 	kfree(parg->code->data);
252 	parg->code->data = NULL;
253 	return ret;
254 }
255 
256 static int eprobe_event_define_fields(struct trace_event_call *event_call)
257 {
258 	struct eprobe_trace_entry_head field;
259 	struct trace_probe *tp;
260 
261 	tp = trace_probe_primary_from_call(event_call);
262 	if (WARN_ON_ONCE(!tp))
263 		return -ENOENT;
264 
265 	return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
266 }
267 
268 static struct trace_event_fields eprobe_fields_array[] = {
269 	{ .type = TRACE_FUNCTION_TYPE,
270 	  .define_fields = eprobe_event_define_fields },
271 	{}
272 };
273 
274 /* Event entry printers */
275 static enum print_line_t
276 print_eprobe_event(struct trace_iterator *iter, int flags,
277 		   struct trace_event *event)
278 {
279 	struct eprobe_trace_entry_head *field;
280 	struct trace_event_call *pevent;
281 	struct trace_event *probed_event;
282 	struct trace_seq *s = &iter->seq;
283 	struct trace_eprobe *ep;
284 	struct trace_probe *tp;
285 	unsigned int type;
286 
287 	field = (struct eprobe_trace_entry_head *)iter->ent;
288 	tp = trace_probe_primary_from_call(
289 		container_of(event, struct trace_event_call, event));
290 	if (WARN_ON_ONCE(!tp))
291 		goto out;
292 
293 	ep = container_of(tp, struct trace_eprobe, tp);
294 	type = ep->event->event.type;
295 
296 	trace_seq_printf(s, "%s: (", trace_probe_name(tp));
297 
298 	probed_event = ftrace_find_event(type);
299 	if (probed_event) {
300 		pevent = container_of(probed_event, struct trace_event_call, event);
301 		trace_seq_printf(s, "%s.%s", pevent->class->system,
302 				 trace_event_name(pevent));
303 	} else {
304 		trace_seq_printf(s, "%u", type);
305 	}
306 
307 	trace_seq_putc(s, ')');
308 
309 	if (print_probe_args(s, tp->args, tp->nr_args,
310 			     (u8 *)&field[1], field) < 0)
311 		goto out;
312 
313 	trace_seq_putc(s, '\n');
314  out:
315 	return trace_handle_return(s);
316 }
317 
318 static unsigned long get_event_field(struct fetch_insn *code, void *rec)
319 {
320 	struct ftrace_event_field *field = code->data;
321 	unsigned long val;
322 	void *addr;
323 
324 	addr = rec + field->offset;
325 
326 	if (is_string_field(field)) {
327 		switch (field->filter_type) {
328 		case FILTER_DYN_STRING:
329 			val = (unsigned long)(rec + (*(unsigned int *)addr & 0xffff));
330 			break;
331 		case FILTER_RDYN_STRING:
332 			val = (unsigned long)(addr + (*(unsigned int *)addr & 0xffff));
333 			break;
334 		case FILTER_STATIC_STRING:
335 			val = (unsigned long)addr;
336 			break;
337 		case FILTER_PTR_STRING:
338 			val = (unsigned long)(*(char *)addr);
339 			break;
340 		default:
341 			WARN_ON_ONCE(1);
342 			return 0;
343 		}
344 		return val;
345 	}
346 
347 	switch (field->size) {
348 	case 1:
349 		if (field->is_signed)
350 			val = *(char *)addr;
351 		else
352 			val = *(unsigned char *)addr;
353 		break;
354 	case 2:
355 		if (field->is_signed)
356 			val = *(short *)addr;
357 		else
358 			val = *(unsigned short *)addr;
359 		break;
360 	case 4:
361 		if (field->is_signed)
362 			val = *(int *)addr;
363 		else
364 			val = *(unsigned int *)addr;
365 		break;
366 	default:
367 		if (field->is_signed)
368 			val = *(long *)addr;
369 		else
370 			val = *(unsigned long *)addr;
371 		break;
372 	}
373 	return val;
374 }
375 
376 static int get_eprobe_size(struct trace_probe *tp, void *rec)
377 {
378 	struct fetch_insn *code;
379 	struct probe_arg *arg;
380 	int i, len, ret = 0;
381 
382 	for (i = 0; i < tp->nr_args; i++) {
383 		arg = tp->args + i;
384 		if (arg->dynamic) {
385 			unsigned long val;
386 
387 			code = arg->code;
388  retry:
389 			switch (code->op) {
390 			case FETCH_OP_TP_ARG:
391 				val = get_event_field(code, rec);
392 				break;
393 			case FETCH_OP_IMM:
394 				val = code->immediate;
395 				break;
396 			case FETCH_OP_COMM:
397 				val = (unsigned long)current->comm;
398 				break;
399 			case FETCH_OP_DATA:
400 				val = (unsigned long)code->data;
401 				break;
402 			case FETCH_NOP_SYMBOL:	/* Ignore a place holder */
403 				code++;
404 				goto retry;
405 			default:
406 				continue;
407 			}
408 			code++;
409 			len = process_fetch_insn_bottom(code, val, NULL, NULL);
410 			if (len > 0)
411 				ret += len;
412 		}
413 	}
414 
415 	return ret;
416 }
417 
418 /* Kprobe specific fetch functions */
419 
420 /* Note that we don't verify it, since the code does not come from user space */
421 static int
422 process_fetch_insn(struct fetch_insn *code, void *rec, void *dest,
423 		   void *base)
424 {
425 	unsigned long val;
426 
427  retry:
428 	switch (code->op) {
429 	case FETCH_OP_TP_ARG:
430 		val = get_event_field(code, rec);
431 		break;
432 	case FETCH_OP_IMM:
433 		val = code->immediate;
434 		break;
435 	case FETCH_OP_COMM:
436 		val = (unsigned long)current->comm;
437 		break;
438 	case FETCH_OP_DATA:
439 		val = (unsigned long)code->data;
440 		break;
441 	case FETCH_NOP_SYMBOL:	/* Ignore a place holder */
442 		code++;
443 		goto retry;
444 	default:
445 		return -EILSEQ;
446 	}
447 	code++;
448 	return process_fetch_insn_bottom(code, val, dest, base);
449 }
450 NOKPROBE_SYMBOL(process_fetch_insn)
451 
452 /* Return the length of string -- including null terminal byte */
453 static nokprobe_inline int
454 fetch_store_strlen_user(unsigned long addr)
455 {
456 	const void __user *uaddr =  (__force const void __user *)addr;
457 
458 	return strnlen_user_nofault(uaddr, MAX_STRING_SIZE);
459 }
460 
461 /* Return the length of string -- including null terminal byte */
462 static nokprobe_inline int
463 fetch_store_strlen(unsigned long addr)
464 {
465 	int ret, len = 0;
466 	u8 c;
467 
468 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
469 	if (addr < TASK_SIZE)
470 		return fetch_store_strlen_user(addr);
471 #endif
472 
473 	do {
474 		ret = copy_from_kernel_nofault(&c, (u8 *)addr + len, 1);
475 		len++;
476 	} while (c && ret == 0 && len < MAX_STRING_SIZE);
477 
478 	return (ret < 0) ? ret : len;
479 }
480 
481 /*
482  * Fetch a null-terminated string from user. Caller MUST set *(u32 *)buf
483  * with max length and relative data location.
484  */
485 static nokprobe_inline int
486 fetch_store_string_user(unsigned long addr, void *dest, void *base)
487 {
488 	const void __user *uaddr =  (__force const void __user *)addr;
489 	int maxlen = get_loc_len(*(u32 *)dest);
490 	void *__dest;
491 	long ret;
492 
493 	if (unlikely(!maxlen))
494 		return -ENOMEM;
495 
496 	__dest = get_loc_data(dest, base);
497 
498 	ret = strncpy_from_user_nofault(__dest, uaddr, maxlen);
499 	if (ret >= 0)
500 		*(u32 *)dest = make_data_loc(ret, __dest - base);
501 
502 	return ret;
503 }
504 
505 /*
506  * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max
507  * length and relative data location.
508  */
509 static nokprobe_inline int
510 fetch_store_string(unsigned long addr, void *dest, void *base)
511 {
512 	int maxlen = get_loc_len(*(u32 *)dest);
513 	void *__dest;
514 	long ret;
515 
516 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
517 	if ((unsigned long)addr < TASK_SIZE)
518 		return fetch_store_string_user(addr, dest, base);
519 #endif
520 
521 	if (unlikely(!maxlen))
522 		return -ENOMEM;
523 
524 	__dest = get_loc_data(dest, base);
525 
526 	/*
527 	 * Try to get string again, since the string can be changed while
528 	 * probing.
529 	 */
530 	ret = strncpy_from_kernel_nofault(__dest, (void *)addr, maxlen);
531 	if (ret >= 0)
532 		*(u32 *)dest = make_data_loc(ret, __dest - base);
533 
534 	return ret;
535 }
536 
537 static nokprobe_inline int
538 probe_mem_read_user(void *dest, void *src, size_t size)
539 {
540 	const void __user *uaddr =  (__force const void __user *)src;
541 
542 	return copy_from_user_nofault(dest, uaddr, size);
543 }
544 
545 static nokprobe_inline int
546 probe_mem_read(void *dest, void *src, size_t size)
547 {
548 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
549 	if ((unsigned long)src < TASK_SIZE)
550 		return probe_mem_read_user(dest, src, size);
551 #endif
552 	return copy_from_kernel_nofault(dest, src, size);
553 }
554 
555 /* eprobe handler */
556 static inline void
557 __eprobe_trace_func(struct eprobe_data *edata, void *rec)
558 {
559 	struct eprobe_trace_entry_head *entry;
560 	struct trace_event_call *call = trace_probe_event_call(&edata->ep->tp);
561 	struct trace_event_buffer fbuffer;
562 	int dsize;
563 
564 	if (WARN_ON_ONCE(call != edata->file->event_call))
565 		return;
566 
567 	if (trace_trigger_soft_disabled(edata->file))
568 		return;
569 
570 	dsize = get_eprobe_size(&edata->ep->tp, rec);
571 
572 	entry = trace_event_buffer_reserve(&fbuffer, edata->file,
573 					   sizeof(*entry) + edata->ep->tp.size + dsize);
574 
575 	if (!entry)
576 		return;
577 
578 	entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
579 	store_trace_args(&entry[1], &edata->ep->tp, rec, sizeof(*entry), dsize);
580 
581 	trace_event_buffer_commit(&fbuffer);
582 }
583 
584 /*
585  * The event probe implementation uses event triggers to get access to
586  * the event it is attached to, but is not an actual trigger. The below
587  * functions are just stubs to fulfill what is needed to use the trigger
588  * infrastructure.
589  */
590 static int eprobe_trigger_init(struct event_trigger_data *data)
591 {
592 	return 0;
593 }
594 
595 static void eprobe_trigger_free(struct event_trigger_data *data)
596 {
597 
598 }
599 
600 static int eprobe_trigger_print(struct seq_file *m,
601 				struct event_trigger_data *data)
602 {
603 	/* Do not print eprobe event triggers */
604 	return 0;
605 }
606 
607 static void eprobe_trigger_func(struct event_trigger_data *data,
608 				struct trace_buffer *buffer, void *rec,
609 				struct ring_buffer_event *rbe)
610 {
611 	struct eprobe_data *edata = data->private_data;
612 
613 	__eprobe_trace_func(edata, rec);
614 }
615 
616 static struct event_trigger_ops eprobe_trigger_ops = {
617 	.trigger		= eprobe_trigger_func,
618 	.print			= eprobe_trigger_print,
619 	.init			= eprobe_trigger_init,
620 	.free			= eprobe_trigger_free,
621 };
622 
623 static int eprobe_trigger_cmd_parse(struct event_command *cmd_ops,
624 				    struct trace_event_file *file,
625 				    char *glob, char *cmd,
626 				    char *param_and_filter)
627 {
628 	return -1;
629 }
630 
631 static int eprobe_trigger_reg_func(char *glob,
632 				   struct event_trigger_data *data,
633 				   struct trace_event_file *file)
634 {
635 	return -1;
636 }
637 
638 static void eprobe_trigger_unreg_func(char *glob,
639 				      struct event_trigger_data *data,
640 				      struct trace_event_file *file)
641 {
642 
643 }
644 
645 static struct event_trigger_ops *eprobe_trigger_get_ops(char *cmd,
646 							char *param)
647 {
648 	return &eprobe_trigger_ops;
649 }
650 
651 static struct event_command event_trigger_cmd = {
652 	.name			= "eprobe",
653 	.trigger_type		= ETT_EVENT_EPROBE,
654 	.flags			= EVENT_CMD_FL_NEEDS_REC,
655 	.parse			= eprobe_trigger_cmd_parse,
656 	.reg			= eprobe_trigger_reg_func,
657 	.unreg			= eprobe_trigger_unreg_func,
658 	.unreg_all		= NULL,
659 	.get_trigger_ops	= eprobe_trigger_get_ops,
660 	.set_filter		= NULL,
661 };
662 
663 static struct event_trigger_data *
664 new_eprobe_trigger(struct trace_eprobe *ep, struct trace_event_file *file)
665 {
666 	struct event_trigger_data *trigger;
667 	struct eprobe_data *edata;
668 
669 	edata = kzalloc(sizeof(*edata), GFP_KERNEL);
670 	trigger = kzalloc(sizeof(*trigger), GFP_KERNEL);
671 	if (!trigger || !edata) {
672 		kfree(edata);
673 		kfree(trigger);
674 		return ERR_PTR(-ENOMEM);
675 	}
676 
677 	trigger->flags = EVENT_TRIGGER_FL_PROBE;
678 	trigger->count = -1;
679 	trigger->ops = &eprobe_trigger_ops;
680 
681 	/*
682 	 * EVENT PROBE triggers are not registered as commands with
683 	 * register_event_command(), as they are not controlled by the user
684 	 * from the trigger file
685 	 */
686 	trigger->cmd_ops = &event_trigger_cmd;
687 
688 	INIT_LIST_HEAD(&trigger->list);
689 	RCU_INIT_POINTER(trigger->filter, NULL);
690 
691 	edata->file = file;
692 	edata->ep = ep;
693 	trigger->private_data = edata;
694 
695 	return trigger;
696 }
697 
698 static int enable_eprobe(struct trace_eprobe *ep,
699 			 struct trace_event_file *eprobe_file)
700 {
701 	struct event_trigger_data *trigger;
702 	struct trace_event_file *file;
703 	struct trace_array *tr = eprobe_file->tr;
704 
705 	file = find_event_file(tr, ep->event_system, ep->event_name);
706 	if (!file)
707 		return -ENOENT;
708 	trigger = new_eprobe_trigger(ep, eprobe_file);
709 	if (IS_ERR(trigger))
710 		return PTR_ERR(trigger);
711 
712 	list_add_tail_rcu(&trigger->list, &file->triggers);
713 
714 	trace_event_trigger_enable_disable(file, 1);
715 	update_cond_flag(file);
716 
717 	return 0;
718 }
719 
720 static struct trace_event_functions eprobe_funcs = {
721 	.trace		= print_eprobe_event
722 };
723 
724 static int disable_eprobe(struct trace_eprobe *ep,
725 			  struct trace_array *tr)
726 {
727 	struct event_trigger_data *trigger = NULL, *iter;
728 	struct trace_event_file *file;
729 	struct eprobe_data *edata;
730 
731 	file = find_event_file(tr, ep->event_system, ep->event_name);
732 	if (!file)
733 		return -ENOENT;
734 
735 	list_for_each_entry(iter, &file->triggers, list) {
736 		if (!(iter->flags & EVENT_TRIGGER_FL_PROBE))
737 			continue;
738 		edata = iter->private_data;
739 		if (edata->ep == ep) {
740 			trigger = iter;
741 			break;
742 		}
743 	}
744 	if (!trigger)
745 		return -ENODEV;
746 
747 	list_del_rcu(&trigger->list);
748 
749 	trace_event_trigger_enable_disable(file, 0);
750 	update_cond_flag(file);
751 
752 	/* Make sure nothing is using the edata or trigger */
753 	tracepoint_synchronize_unregister();
754 
755 	kfree(edata);
756 	kfree(trigger);
757 
758 	return 0;
759 }
760 
761 static int enable_trace_eprobe(struct trace_event_call *call,
762 			       struct trace_event_file *file)
763 {
764 	struct trace_probe *pos, *tp;
765 	struct trace_eprobe *ep;
766 	bool enabled;
767 	int ret = 0;
768 
769 	tp = trace_probe_primary_from_call(call);
770 	if (WARN_ON_ONCE(!tp))
771 		return -ENODEV;
772 	enabled = trace_probe_is_enabled(tp);
773 
774 	/* This also changes "enabled" state */
775 	if (file) {
776 		ret = trace_probe_add_file(tp, file);
777 		if (ret)
778 			return ret;
779 	} else
780 		trace_probe_set_flag(tp, TP_FLAG_PROFILE);
781 
782 	if (enabled)
783 		return 0;
784 
785 	list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
786 		ep = container_of(pos, struct trace_eprobe, tp);
787 		ret = enable_eprobe(ep, file);
788 		if (ret)
789 			break;
790 		enabled = true;
791 	}
792 
793 	if (ret) {
794 		/* Failed to enable one of them. Roll back all */
795 		if (enabled)
796 			disable_eprobe(ep, file->tr);
797 		if (file)
798 			trace_probe_remove_file(tp, file);
799 		else
800 			trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
801 	}
802 
803 	return ret;
804 }
805 
806 static int disable_trace_eprobe(struct trace_event_call *call,
807 				struct trace_event_file *file)
808 {
809 	struct trace_probe *pos, *tp;
810 	struct trace_eprobe *ep;
811 
812 	tp = trace_probe_primary_from_call(call);
813 	if (WARN_ON_ONCE(!tp))
814 		return -ENODEV;
815 
816 	if (file) {
817 		if (!trace_probe_get_file_link(tp, file))
818 			return -ENOENT;
819 		if (!trace_probe_has_single_file(tp))
820 			goto out;
821 		trace_probe_clear_flag(tp, TP_FLAG_TRACE);
822 	} else
823 		trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
824 
825 	if (!trace_probe_is_enabled(tp)) {
826 		list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
827 			ep = container_of(pos, struct trace_eprobe, tp);
828 			disable_eprobe(ep, file->tr);
829 		}
830 	}
831 
832  out:
833 	if (file)
834 		/*
835 		 * Synchronization is done in below function. For perf event,
836 		 * file == NULL and perf_trace_event_unreg() calls
837 		 * tracepoint_synchronize_unregister() to ensure synchronize
838 		 * event. We don't need to care about it.
839 		 */
840 		trace_probe_remove_file(tp, file);
841 
842 	return 0;
843 }
844 
845 static int eprobe_register(struct trace_event_call *event,
846 			   enum trace_reg type, void *data)
847 {
848 	struct trace_event_file *file = data;
849 
850 	switch (type) {
851 	case TRACE_REG_REGISTER:
852 		return enable_trace_eprobe(event, file);
853 	case TRACE_REG_UNREGISTER:
854 		return disable_trace_eprobe(event, file);
855 #ifdef CONFIG_PERF_EVENTS
856 	case TRACE_REG_PERF_REGISTER:
857 	case TRACE_REG_PERF_UNREGISTER:
858 	case TRACE_REG_PERF_OPEN:
859 	case TRACE_REG_PERF_CLOSE:
860 	case TRACE_REG_PERF_ADD:
861 	case TRACE_REG_PERF_DEL:
862 		return 0;
863 #endif
864 	}
865 	return 0;
866 }
867 
868 static inline void init_trace_eprobe_call(struct trace_eprobe *ep)
869 {
870 	struct trace_event_call *call = trace_probe_event_call(&ep->tp);
871 
872 	call->flags = TRACE_EVENT_FL_EPROBE;
873 	call->event.funcs = &eprobe_funcs;
874 	call->class->fields_array = eprobe_fields_array;
875 	call->class->reg = eprobe_register;
876 }
877 
878 static struct trace_event_call *
879 find_and_get_event(const char *system, const char *event_name)
880 {
881 	struct trace_event_call *tp_event;
882 	const char *name;
883 
884 	list_for_each_entry(tp_event, &ftrace_events, list) {
885 		/* Skip other probes and ftrace events */
886 		if (tp_event->flags &
887 		    (TRACE_EVENT_FL_IGNORE_ENABLE |
888 		     TRACE_EVENT_FL_KPROBE |
889 		     TRACE_EVENT_FL_UPROBE |
890 		     TRACE_EVENT_FL_EPROBE))
891 			continue;
892 		if (!tp_event->class->system ||
893 		    strcmp(system, tp_event->class->system))
894 			continue;
895 		name = trace_event_name(tp_event);
896 		if (!name || strcmp(event_name, name))
897 			continue;
898 		if (!trace_event_try_get_ref(tp_event)) {
899 			return NULL;
900 			break;
901 		}
902 		return tp_event;
903 		break;
904 	}
905 	return NULL;
906 }
907 
908 static int trace_eprobe_tp_update_arg(struct trace_eprobe *ep, const char *argv[], int i)
909 {
910 	unsigned int flags = TPARG_FL_KERNEL | TPARG_FL_TPOINT;
911 	int ret;
912 
913 	ret = traceprobe_parse_probe_arg(&ep->tp, i, argv[i], flags);
914 	if (ret)
915 		return ret;
916 
917 	if (ep->tp.args[i].code->op == FETCH_OP_TP_ARG) {
918 		ret = trace_eprobe_tp_arg_update(ep, i);
919 		if (ret)
920 			trace_probe_log_err(0, BAD_ATTACH_ARG);
921 	}
922 
923 	/* Handle symbols "@" */
924 	if (!ret)
925 		ret = traceprobe_update_arg(&ep->tp.args[i]);
926 
927 	return ret;
928 }
929 
930 static int __trace_eprobe_create(int argc, const char *argv[])
931 {
932 	/*
933 	 * Argument syntax:
934 	 *      e[:[GRP/][ENAME]] SYSTEM.EVENT [FETCHARGS]
935 	 * Fetch args:
936 	 *  <name>=$<field>[:TYPE]
937 	 */
938 	const char *event = NULL, *group = EPROBE_EVENT_SYSTEM;
939 	const char *sys_event = NULL, *sys_name = NULL;
940 	struct trace_event_call *event_call;
941 	struct trace_eprobe *ep = NULL;
942 	char buf1[MAX_EVENT_NAME_LEN];
943 	char buf2[MAX_EVENT_NAME_LEN];
944 	char gbuf[MAX_EVENT_NAME_LEN];
945 	int ret = 0;
946 	int i;
947 
948 	if (argc < 2 || argv[0][0] != 'e')
949 		return -ECANCELED;
950 
951 	trace_probe_log_init("event_probe", argc, argv);
952 
953 	event = strchr(&argv[0][1], ':');
954 	if (event) {
955 		event++;
956 		ret = traceprobe_parse_event_name(&event, &group, gbuf,
957 						  event - argv[0]);
958 		if (ret)
959 			goto parse_error;
960 	}
961 
962 	trace_probe_log_set_index(1);
963 	sys_event = argv[1];
964 	ret = traceprobe_parse_event_name(&sys_event, &sys_name, buf2, 0);
965 	if (ret || !sys_event || !sys_name) {
966 		trace_probe_log_err(0, NO_EVENT_INFO);
967 		goto parse_error;
968 	}
969 
970 	if (!event) {
971 		strscpy(buf1, argv[1], MAX_EVENT_NAME_LEN);
972 		sanitize_event_name(buf1);
973 		event = buf1;
974 	}
975 
976 	mutex_lock(&event_mutex);
977 	event_call = find_and_get_event(sys_name, sys_event);
978 	ep = alloc_event_probe(group, event, event_call, argc - 2);
979 	mutex_unlock(&event_mutex);
980 
981 	if (IS_ERR(ep)) {
982 		ret = PTR_ERR(ep);
983 		if (ret == -ENODEV)
984 			trace_probe_log_err(0, BAD_ATTACH_EVENT);
985 		/* This must return -ENOMEM or missing event, else there is a bug */
986 		WARN_ON_ONCE(ret != -ENOMEM && ret != -ENODEV);
987 		ep = NULL;
988 		goto error;
989 	}
990 
991 	argc -= 2; argv += 2;
992 	/* parse arguments */
993 	for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
994 		trace_probe_log_set_index(i + 2);
995 		ret = trace_eprobe_tp_update_arg(ep, argv, i);
996 		if (ret)
997 			goto error;
998 	}
999 	ret = traceprobe_set_print_fmt(&ep->tp, PROBE_PRINT_EVENT);
1000 	if (ret < 0)
1001 		goto error;
1002 	init_trace_eprobe_call(ep);
1003 	mutex_lock(&event_mutex);
1004 	ret = trace_probe_register_event_call(&ep->tp);
1005 	if (ret) {
1006 		if (ret == -EEXIST) {
1007 			trace_probe_log_set_index(0);
1008 			trace_probe_log_err(0, EVENT_EXIST);
1009 		}
1010 		mutex_unlock(&event_mutex);
1011 		goto error;
1012 	}
1013 	ret = dyn_event_add(&ep->devent, &ep->tp.event->call);
1014 	mutex_unlock(&event_mutex);
1015 	return ret;
1016 parse_error:
1017 	ret = -EINVAL;
1018 error:
1019 	trace_event_probe_cleanup(ep);
1020 	return ret;
1021 }
1022 
1023 /*
1024  * Register dynevent at core_initcall. This allows kernel to setup eprobe
1025  * events in postcore_initcall without tracefs.
1026  */
1027 static __init int trace_events_eprobe_init_early(void)
1028 {
1029 	int err = 0;
1030 
1031 	err = dyn_event_register(&eprobe_dyn_event_ops);
1032 	if (err)
1033 		pr_warn("Could not register eprobe_dyn_event_ops\n");
1034 
1035 	return err;
1036 }
1037 core_initcall(trace_events_eprobe_init_early);
1038