xref: /openbmc/linux/kernel/trace/trace_events.c (revision 14aa4f3e)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * event tracer
4  *
5  * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
6  *
7  *  - Added format output of fields of the trace point.
8  *    This was based off of work by Tom Zanussi <tzanussi@gmail.com>.
9  *
10  */
11 
12 #define pr_fmt(fmt) fmt
13 
14 #include <linux/workqueue.h>
15 #include <linux/security.h>
16 #include <linux/spinlock.h>
17 #include <linux/kthread.h>
18 #include <linux/tracefs.h>
19 #include <linux/uaccess.h>
20 #include <linux/module.h>
21 #include <linux/ctype.h>
22 #include <linux/sort.h>
23 #include <linux/slab.h>
24 #include <linux/delay.h>
25 
26 #include <trace/events/sched.h>
27 #include <trace/syscall.h>
28 
29 #include <asm/setup.h>
30 
31 #include "trace_output.h"
32 
33 #undef TRACE_SYSTEM
34 #define TRACE_SYSTEM "TRACE_SYSTEM"
35 
36 DEFINE_MUTEX(event_mutex);
37 
38 LIST_HEAD(ftrace_events);
39 static LIST_HEAD(ftrace_generic_fields);
40 static LIST_HEAD(ftrace_common_fields);
41 static bool eventdir_initialized;
42 
43 static LIST_HEAD(module_strings);
44 
45 struct module_string {
46 	struct list_head	next;
47 	struct module		*module;
48 	char			*str;
49 };
50 
51 #define GFP_TRACE (GFP_KERNEL | __GFP_ZERO)
52 
53 static struct kmem_cache *field_cachep;
54 static struct kmem_cache *file_cachep;
55 
56 static inline int system_refcount(struct event_subsystem *system)
57 {
58 	return system->ref_count;
59 }
60 
61 static int system_refcount_inc(struct event_subsystem *system)
62 {
63 	return system->ref_count++;
64 }
65 
66 static int system_refcount_dec(struct event_subsystem *system)
67 {
68 	return --system->ref_count;
69 }
70 
71 /* Double loops, do not use break, only goto's work */
72 #define do_for_each_event_file(tr, file)			\
73 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
74 		list_for_each_entry(file, &tr->events, list)
75 
76 #define do_for_each_event_file_safe(tr, file)			\
77 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
78 		struct trace_event_file *___n;				\
79 		list_for_each_entry_safe(file, ___n, &tr->events, list)
80 
81 #define while_for_each_event_file()		\
82 	}
83 
84 static struct ftrace_event_field *
85 __find_event_field(struct list_head *head, char *name)
86 {
87 	struct ftrace_event_field *field;
88 
89 	list_for_each_entry(field, head, link) {
90 		if (!strcmp(field->name, name))
91 			return field;
92 	}
93 
94 	return NULL;
95 }
96 
97 struct ftrace_event_field *
98 trace_find_event_field(struct trace_event_call *call, char *name)
99 {
100 	struct ftrace_event_field *field;
101 	struct list_head *head;
102 
103 	head = trace_get_fields(call);
104 	field = __find_event_field(head, name);
105 	if (field)
106 		return field;
107 
108 	field = __find_event_field(&ftrace_generic_fields, name);
109 	if (field)
110 		return field;
111 
112 	return __find_event_field(&ftrace_common_fields, name);
113 }
114 
115 static int __trace_define_field(struct list_head *head, const char *type,
116 				const char *name, int offset, int size,
117 				int is_signed, int filter_type, int len)
118 {
119 	struct ftrace_event_field *field;
120 
121 	field = kmem_cache_alloc(field_cachep, GFP_TRACE);
122 	if (!field)
123 		return -ENOMEM;
124 
125 	field->name = name;
126 	field->type = type;
127 
128 	if (filter_type == FILTER_OTHER)
129 		field->filter_type = filter_assign_type(type);
130 	else
131 		field->filter_type = filter_type;
132 
133 	field->offset = offset;
134 	field->size = size;
135 	field->is_signed = is_signed;
136 	field->len = len;
137 
138 	list_add(&field->link, head);
139 
140 	return 0;
141 }
142 
143 int trace_define_field(struct trace_event_call *call, const char *type,
144 		       const char *name, int offset, int size, int is_signed,
145 		       int filter_type)
146 {
147 	struct list_head *head;
148 
149 	if (WARN_ON(!call->class))
150 		return 0;
151 
152 	head = trace_get_fields(call);
153 	return __trace_define_field(head, type, name, offset, size,
154 				    is_signed, filter_type, 0);
155 }
156 EXPORT_SYMBOL_GPL(trace_define_field);
157 
158 static int trace_define_field_ext(struct trace_event_call *call, const char *type,
159 		       const char *name, int offset, int size, int is_signed,
160 		       int filter_type, int len)
161 {
162 	struct list_head *head;
163 
164 	if (WARN_ON(!call->class))
165 		return 0;
166 
167 	head = trace_get_fields(call);
168 	return __trace_define_field(head, type, name, offset, size,
169 				    is_signed, filter_type, len);
170 }
171 
172 #define __generic_field(type, item, filter_type)			\
173 	ret = __trace_define_field(&ftrace_generic_fields, #type,	\
174 				   #item, 0, 0, is_signed_type(type),	\
175 				   filter_type, 0);			\
176 	if (ret)							\
177 		return ret;
178 
179 #define __common_field(type, item)					\
180 	ret = __trace_define_field(&ftrace_common_fields, #type,	\
181 				   "common_" #item,			\
182 				   offsetof(typeof(ent), item),		\
183 				   sizeof(ent.item),			\
184 				   is_signed_type(type), FILTER_OTHER, 0);	\
185 	if (ret)							\
186 		return ret;
187 
188 static int trace_define_generic_fields(void)
189 {
190 	int ret;
191 
192 	__generic_field(int, CPU, FILTER_CPU);
193 	__generic_field(int, cpu, FILTER_CPU);
194 	__generic_field(int, common_cpu, FILTER_CPU);
195 	__generic_field(char *, COMM, FILTER_COMM);
196 	__generic_field(char *, comm, FILTER_COMM);
197 	__generic_field(char *, stacktrace, FILTER_STACKTRACE);
198 	__generic_field(char *, STACKTRACE, FILTER_STACKTRACE);
199 
200 	return ret;
201 }
202 
203 static int trace_define_common_fields(void)
204 {
205 	int ret;
206 	struct trace_entry ent;
207 
208 	__common_field(unsigned short, type);
209 	__common_field(unsigned char, flags);
210 	/* Holds both preempt_count and migrate_disable */
211 	__common_field(unsigned char, preempt_count);
212 	__common_field(int, pid);
213 
214 	return ret;
215 }
216 
217 static void trace_destroy_fields(struct trace_event_call *call)
218 {
219 	struct ftrace_event_field *field, *next;
220 	struct list_head *head;
221 
222 	head = trace_get_fields(call);
223 	list_for_each_entry_safe(field, next, head, link) {
224 		list_del(&field->link);
225 		kmem_cache_free(field_cachep, field);
226 	}
227 }
228 
229 /*
230  * run-time version of trace_event_get_offsets_<call>() that returns the last
231  * accessible offset of trace fields excluding __dynamic_array bytes
232  */
233 int trace_event_get_offsets(struct trace_event_call *call)
234 {
235 	struct ftrace_event_field *tail;
236 	struct list_head *head;
237 
238 	head = trace_get_fields(call);
239 	/*
240 	 * head->next points to the last field with the largest offset,
241 	 * since it was added last by trace_define_field()
242 	 */
243 	tail = list_first_entry(head, struct ftrace_event_field, link);
244 	return tail->offset + tail->size;
245 }
246 
247 /*
248  * Check if the referenced field is an array and return true,
249  * as arrays are OK to dereference.
250  */
251 static bool test_field(const char *fmt, struct trace_event_call *call)
252 {
253 	struct trace_event_fields *field = call->class->fields_array;
254 	const char *array_descriptor;
255 	const char *p = fmt;
256 	int len;
257 
258 	if (!(len = str_has_prefix(fmt, "REC->")))
259 		return false;
260 	fmt += len;
261 	for (p = fmt; *p; p++) {
262 		if (!isalnum(*p) && *p != '_')
263 			break;
264 	}
265 	len = p - fmt;
266 
267 	for (; field->type; field++) {
268 		if (strncmp(field->name, fmt, len) ||
269 		    field->name[len])
270 			continue;
271 		array_descriptor = strchr(field->type, '[');
272 		/* This is an array and is OK to dereference. */
273 		return array_descriptor != NULL;
274 	}
275 	return false;
276 }
277 
278 /*
279  * Examine the print fmt of the event looking for unsafe dereference
280  * pointers using %p* that could be recorded in the trace event and
281  * much later referenced after the pointer was freed. Dereferencing
282  * pointers are OK, if it is dereferenced into the event itself.
283  */
284 static void test_event_printk(struct trace_event_call *call)
285 {
286 	u64 dereference_flags = 0;
287 	bool first = true;
288 	const char *fmt, *c, *r, *a;
289 	int parens = 0;
290 	char in_quote = 0;
291 	int start_arg = 0;
292 	int arg = 0;
293 	int i;
294 
295 	fmt = call->print_fmt;
296 
297 	if (!fmt)
298 		return;
299 
300 	for (i = 0; fmt[i]; i++) {
301 		switch (fmt[i]) {
302 		case '\\':
303 			i++;
304 			if (!fmt[i])
305 				return;
306 			continue;
307 		case '"':
308 		case '\'':
309 			/*
310 			 * The print fmt starts with a string that
311 			 * is processed first to find %p* usage,
312 			 * then after the first string, the print fmt
313 			 * contains arguments that are used to check
314 			 * if the dereferenced %p* usage is safe.
315 			 */
316 			if (first) {
317 				if (fmt[i] == '\'')
318 					continue;
319 				if (in_quote) {
320 					arg = 0;
321 					first = false;
322 					/*
323 					 * If there was no %p* uses
324 					 * the fmt is OK.
325 					 */
326 					if (!dereference_flags)
327 						return;
328 				}
329 			}
330 			if (in_quote) {
331 				if (in_quote == fmt[i])
332 					in_quote = 0;
333 			} else {
334 				in_quote = fmt[i];
335 			}
336 			continue;
337 		case '%':
338 			if (!first || !in_quote)
339 				continue;
340 			i++;
341 			if (!fmt[i])
342 				return;
343 			switch (fmt[i]) {
344 			case '%':
345 				continue;
346 			case 'p':
347 				/* Find dereferencing fields */
348 				switch (fmt[i + 1]) {
349 				case 'B': case 'R': case 'r':
350 				case 'b': case 'M': case 'm':
351 				case 'I': case 'i': case 'E':
352 				case 'U': case 'V': case 'N':
353 				case 'a': case 'd': case 'D':
354 				case 'g': case 't': case 'C':
355 				case 'O': case 'f':
356 					if (WARN_ONCE(arg == 63,
357 						      "Too many args for event: %s",
358 						      trace_event_name(call)))
359 						return;
360 					dereference_flags |= 1ULL << arg;
361 				}
362 				break;
363 			default:
364 			{
365 				bool star = false;
366 				int j;
367 
368 				/* Increment arg if %*s exists. */
369 				for (j = 0; fmt[i + j]; j++) {
370 					if (isdigit(fmt[i + j]) ||
371 					    fmt[i + j] == '.')
372 						continue;
373 					if (fmt[i + j] == '*') {
374 						star = true;
375 						continue;
376 					}
377 					if ((fmt[i + j] == 's') && star)
378 						arg++;
379 					break;
380 				}
381 				break;
382 			} /* default */
383 
384 			} /* switch */
385 			arg++;
386 			continue;
387 		case '(':
388 			if (in_quote)
389 				continue;
390 			parens++;
391 			continue;
392 		case ')':
393 			if (in_quote)
394 				continue;
395 			parens--;
396 			if (WARN_ONCE(parens < 0,
397 				      "Paren mismatch for event: %s\narg='%s'\n%*s",
398 				      trace_event_name(call),
399 				      fmt + start_arg,
400 				      (i - start_arg) + 5, "^"))
401 				return;
402 			continue;
403 		case ',':
404 			if (in_quote || parens)
405 				continue;
406 			i++;
407 			while (isspace(fmt[i]))
408 				i++;
409 			start_arg = i;
410 			if (!(dereference_flags & (1ULL << arg)))
411 				goto next_arg;
412 
413 			/* Find the REC-> in the argument */
414 			c = strchr(fmt + i, ',');
415 			r = strstr(fmt + i, "REC->");
416 			if (r && (!c || r < c)) {
417 				/*
418 				 * Addresses of events on the buffer,
419 				 * or an array on the buffer is
420 				 * OK to dereference.
421 				 * There's ways to fool this, but
422 				 * this is to catch common mistakes,
423 				 * not malicious code.
424 				 */
425 				a = strchr(fmt + i, '&');
426 				if ((a && (a < r)) || test_field(r, call))
427 					dereference_flags &= ~(1ULL << arg);
428 			} else if ((r = strstr(fmt + i, "__get_dynamic_array(")) &&
429 				   (!c || r < c)) {
430 				dereference_flags &= ~(1ULL << arg);
431 			} else if ((r = strstr(fmt + i, "__get_sockaddr(")) &&
432 				   (!c || r < c)) {
433 				dereference_flags &= ~(1ULL << arg);
434 			}
435 
436 		next_arg:
437 			i--;
438 			arg++;
439 		}
440 	}
441 
442 	/*
443 	 * If you triggered the below warning, the trace event reported
444 	 * uses an unsafe dereference pointer %p*. As the data stored
445 	 * at the trace event time may no longer exist when the trace
446 	 * event is printed, dereferencing to the original source is
447 	 * unsafe. The source of the dereference must be copied into the
448 	 * event itself, and the dereference must access the copy instead.
449 	 */
450 	if (WARN_ON_ONCE(dereference_flags)) {
451 		arg = 1;
452 		while (!(dereference_flags & 1)) {
453 			dereference_flags >>= 1;
454 			arg++;
455 		}
456 		pr_warn("event %s has unsafe dereference of argument %d\n",
457 			trace_event_name(call), arg);
458 		pr_warn("print_fmt: %s\n", fmt);
459 	}
460 }
461 
462 int trace_event_raw_init(struct trace_event_call *call)
463 {
464 	int id;
465 
466 	id = register_trace_event(&call->event);
467 	if (!id)
468 		return -ENODEV;
469 
470 	test_event_printk(call);
471 
472 	return 0;
473 }
474 EXPORT_SYMBOL_GPL(trace_event_raw_init);
475 
476 bool trace_event_ignore_this_pid(struct trace_event_file *trace_file)
477 {
478 	struct trace_array *tr = trace_file->tr;
479 	struct trace_array_cpu *data;
480 	struct trace_pid_list *no_pid_list;
481 	struct trace_pid_list *pid_list;
482 
483 	pid_list = rcu_dereference_raw(tr->filtered_pids);
484 	no_pid_list = rcu_dereference_raw(tr->filtered_no_pids);
485 
486 	if (!pid_list && !no_pid_list)
487 		return false;
488 
489 	data = this_cpu_ptr(tr->array_buffer.data);
490 
491 	return data->ignore_pid;
492 }
493 EXPORT_SYMBOL_GPL(trace_event_ignore_this_pid);
494 
495 void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer,
496 				 struct trace_event_file *trace_file,
497 				 unsigned long len)
498 {
499 	struct trace_event_call *event_call = trace_file->event_call;
500 
501 	if ((trace_file->flags & EVENT_FILE_FL_PID_FILTER) &&
502 	    trace_event_ignore_this_pid(trace_file))
503 		return NULL;
504 
505 	/*
506 	 * If CONFIG_PREEMPTION is enabled, then the tracepoint itself disables
507 	 * preemption (adding one to the preempt_count). Since we are
508 	 * interested in the preempt_count at the time the tracepoint was
509 	 * hit, we need to subtract one to offset the increment.
510 	 */
511 	fbuffer->trace_ctx = tracing_gen_ctx_dec();
512 	fbuffer->trace_file = trace_file;
513 
514 	fbuffer->event =
515 		trace_event_buffer_lock_reserve(&fbuffer->buffer, trace_file,
516 						event_call->event.type, len,
517 						fbuffer->trace_ctx);
518 	if (!fbuffer->event)
519 		return NULL;
520 
521 	fbuffer->regs = NULL;
522 	fbuffer->entry = ring_buffer_event_data(fbuffer->event);
523 	return fbuffer->entry;
524 }
525 EXPORT_SYMBOL_GPL(trace_event_buffer_reserve);
526 
527 int trace_event_reg(struct trace_event_call *call,
528 		    enum trace_reg type, void *data)
529 {
530 	struct trace_event_file *file = data;
531 
532 	WARN_ON(!(call->flags & TRACE_EVENT_FL_TRACEPOINT));
533 	switch (type) {
534 	case TRACE_REG_REGISTER:
535 		return tracepoint_probe_register(call->tp,
536 						 call->class->probe,
537 						 file);
538 	case TRACE_REG_UNREGISTER:
539 		tracepoint_probe_unregister(call->tp,
540 					    call->class->probe,
541 					    file);
542 		return 0;
543 
544 #ifdef CONFIG_PERF_EVENTS
545 	case TRACE_REG_PERF_REGISTER:
546 		return tracepoint_probe_register(call->tp,
547 						 call->class->perf_probe,
548 						 call);
549 	case TRACE_REG_PERF_UNREGISTER:
550 		tracepoint_probe_unregister(call->tp,
551 					    call->class->perf_probe,
552 					    call);
553 		return 0;
554 	case TRACE_REG_PERF_OPEN:
555 	case TRACE_REG_PERF_CLOSE:
556 	case TRACE_REG_PERF_ADD:
557 	case TRACE_REG_PERF_DEL:
558 		return 0;
559 #endif
560 	}
561 	return 0;
562 }
563 EXPORT_SYMBOL_GPL(trace_event_reg);
564 
565 void trace_event_enable_cmd_record(bool enable)
566 {
567 	struct trace_event_file *file;
568 	struct trace_array *tr;
569 
570 	lockdep_assert_held(&event_mutex);
571 
572 	do_for_each_event_file(tr, file) {
573 
574 		if (!(file->flags & EVENT_FILE_FL_ENABLED))
575 			continue;
576 
577 		if (enable) {
578 			tracing_start_cmdline_record();
579 			set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
580 		} else {
581 			tracing_stop_cmdline_record();
582 			clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
583 		}
584 	} while_for_each_event_file();
585 }
586 
587 void trace_event_enable_tgid_record(bool enable)
588 {
589 	struct trace_event_file *file;
590 	struct trace_array *tr;
591 
592 	lockdep_assert_held(&event_mutex);
593 
594 	do_for_each_event_file(tr, file) {
595 		if (!(file->flags & EVENT_FILE_FL_ENABLED))
596 			continue;
597 
598 		if (enable) {
599 			tracing_start_tgid_record();
600 			set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
601 		} else {
602 			tracing_stop_tgid_record();
603 			clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT,
604 				  &file->flags);
605 		}
606 	} while_for_each_event_file();
607 }
608 
609 static int __ftrace_event_enable_disable(struct trace_event_file *file,
610 					 int enable, int soft_disable)
611 {
612 	struct trace_event_call *call = file->event_call;
613 	struct trace_array *tr = file->tr;
614 	int ret = 0;
615 	int disable;
616 
617 	switch (enable) {
618 	case 0:
619 		/*
620 		 * When soft_disable is set and enable is cleared, the sm_ref
621 		 * reference counter is decremented. If it reaches 0, we want
622 		 * to clear the SOFT_DISABLED flag but leave the event in the
623 		 * state that it was. That is, if the event was enabled and
624 		 * SOFT_DISABLED isn't set, then do nothing. But if SOFT_DISABLED
625 		 * is set we do not want the event to be enabled before we
626 		 * clear the bit.
627 		 *
628 		 * When soft_disable is not set but the SOFT_MODE flag is,
629 		 * we do nothing. Do not disable the tracepoint, otherwise
630 		 * "soft enable"s (clearing the SOFT_DISABLED bit) wont work.
631 		 */
632 		if (soft_disable) {
633 			if (atomic_dec_return(&file->sm_ref) > 0)
634 				break;
635 			disable = file->flags & EVENT_FILE_FL_SOFT_DISABLED;
636 			clear_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
637 			/* Disable use of trace_buffered_event */
638 			trace_buffered_event_disable();
639 		} else
640 			disable = !(file->flags & EVENT_FILE_FL_SOFT_MODE);
641 
642 		if (disable && (file->flags & EVENT_FILE_FL_ENABLED)) {
643 			clear_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
644 			if (file->flags & EVENT_FILE_FL_RECORDED_CMD) {
645 				tracing_stop_cmdline_record();
646 				clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
647 			}
648 
649 			if (file->flags & EVENT_FILE_FL_RECORDED_TGID) {
650 				tracing_stop_tgid_record();
651 				clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
652 			}
653 
654 			call->class->reg(call, TRACE_REG_UNREGISTER, file);
655 		}
656 		/* If in SOFT_MODE, just set the SOFT_DISABLE_BIT, else clear it */
657 		if (file->flags & EVENT_FILE_FL_SOFT_MODE)
658 			set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
659 		else
660 			clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
661 		break;
662 	case 1:
663 		/*
664 		 * When soft_disable is set and enable is set, we want to
665 		 * register the tracepoint for the event, but leave the event
666 		 * as is. That means, if the event was already enabled, we do
667 		 * nothing (but set SOFT_MODE). If the event is disabled, we
668 		 * set SOFT_DISABLED before enabling the event tracepoint, so
669 		 * it still seems to be disabled.
670 		 */
671 		if (!soft_disable)
672 			clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
673 		else {
674 			if (atomic_inc_return(&file->sm_ref) > 1)
675 				break;
676 			set_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
677 			/* Enable use of trace_buffered_event */
678 			trace_buffered_event_enable();
679 		}
680 
681 		if (!(file->flags & EVENT_FILE_FL_ENABLED)) {
682 			bool cmd = false, tgid = false;
683 
684 			/* Keep the event disabled, when going to SOFT_MODE. */
685 			if (soft_disable)
686 				set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
687 
688 			if (tr->trace_flags & TRACE_ITER_RECORD_CMD) {
689 				cmd = true;
690 				tracing_start_cmdline_record();
691 				set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
692 			}
693 
694 			if (tr->trace_flags & TRACE_ITER_RECORD_TGID) {
695 				tgid = true;
696 				tracing_start_tgid_record();
697 				set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
698 			}
699 
700 			ret = call->class->reg(call, TRACE_REG_REGISTER, file);
701 			if (ret) {
702 				if (cmd)
703 					tracing_stop_cmdline_record();
704 				if (tgid)
705 					tracing_stop_tgid_record();
706 				pr_info("event trace: Could not enable event "
707 					"%s\n", trace_event_name(call));
708 				break;
709 			}
710 			set_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
711 
712 			/* WAS_ENABLED gets set but never cleared. */
713 			set_bit(EVENT_FILE_FL_WAS_ENABLED_BIT, &file->flags);
714 		}
715 		break;
716 	}
717 
718 	return ret;
719 }
720 
721 int trace_event_enable_disable(struct trace_event_file *file,
722 			       int enable, int soft_disable)
723 {
724 	return __ftrace_event_enable_disable(file, enable, soft_disable);
725 }
726 
727 static int ftrace_event_enable_disable(struct trace_event_file *file,
728 				       int enable)
729 {
730 	return __ftrace_event_enable_disable(file, enable, 0);
731 }
732 
733 static void ftrace_clear_events(struct trace_array *tr)
734 {
735 	struct trace_event_file *file;
736 
737 	mutex_lock(&event_mutex);
738 	list_for_each_entry(file, &tr->events, list) {
739 		ftrace_event_enable_disable(file, 0);
740 	}
741 	mutex_unlock(&event_mutex);
742 }
743 
744 static void
745 event_filter_pid_sched_process_exit(void *data, struct task_struct *task)
746 {
747 	struct trace_pid_list *pid_list;
748 	struct trace_array *tr = data;
749 
750 	pid_list = rcu_dereference_raw(tr->filtered_pids);
751 	trace_filter_add_remove_task(pid_list, NULL, task);
752 
753 	pid_list = rcu_dereference_raw(tr->filtered_no_pids);
754 	trace_filter_add_remove_task(pid_list, NULL, task);
755 }
756 
757 static void
758 event_filter_pid_sched_process_fork(void *data,
759 				    struct task_struct *self,
760 				    struct task_struct *task)
761 {
762 	struct trace_pid_list *pid_list;
763 	struct trace_array *tr = data;
764 
765 	pid_list = rcu_dereference_sched(tr->filtered_pids);
766 	trace_filter_add_remove_task(pid_list, self, task);
767 
768 	pid_list = rcu_dereference_sched(tr->filtered_no_pids);
769 	trace_filter_add_remove_task(pid_list, self, task);
770 }
771 
772 void trace_event_follow_fork(struct trace_array *tr, bool enable)
773 {
774 	if (enable) {
775 		register_trace_prio_sched_process_fork(event_filter_pid_sched_process_fork,
776 						       tr, INT_MIN);
777 		register_trace_prio_sched_process_free(event_filter_pid_sched_process_exit,
778 						       tr, INT_MAX);
779 	} else {
780 		unregister_trace_sched_process_fork(event_filter_pid_sched_process_fork,
781 						    tr);
782 		unregister_trace_sched_process_free(event_filter_pid_sched_process_exit,
783 						    tr);
784 	}
785 }
786 
787 static void
788 event_filter_pid_sched_switch_probe_pre(void *data, bool preempt,
789 					struct task_struct *prev,
790 					struct task_struct *next,
791 					unsigned int prev_state)
792 {
793 	struct trace_array *tr = data;
794 	struct trace_pid_list *no_pid_list;
795 	struct trace_pid_list *pid_list;
796 	bool ret;
797 
798 	pid_list = rcu_dereference_sched(tr->filtered_pids);
799 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
800 
801 	/*
802 	 * Sched switch is funny, as we only want to ignore it
803 	 * in the notrace case if both prev and next should be ignored.
804 	 */
805 	ret = trace_ignore_this_task(NULL, no_pid_list, prev) &&
806 		trace_ignore_this_task(NULL, no_pid_list, next);
807 
808 	this_cpu_write(tr->array_buffer.data->ignore_pid, ret ||
809 		       (trace_ignore_this_task(pid_list, NULL, prev) &&
810 			trace_ignore_this_task(pid_list, NULL, next)));
811 }
812 
813 static void
814 event_filter_pid_sched_switch_probe_post(void *data, bool preempt,
815 					 struct task_struct *prev,
816 					 struct task_struct *next,
817 					 unsigned int prev_state)
818 {
819 	struct trace_array *tr = data;
820 	struct trace_pid_list *no_pid_list;
821 	struct trace_pid_list *pid_list;
822 
823 	pid_list = rcu_dereference_sched(tr->filtered_pids);
824 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
825 
826 	this_cpu_write(tr->array_buffer.data->ignore_pid,
827 		       trace_ignore_this_task(pid_list, no_pid_list, next));
828 }
829 
830 static void
831 event_filter_pid_sched_wakeup_probe_pre(void *data, struct task_struct *task)
832 {
833 	struct trace_array *tr = data;
834 	struct trace_pid_list *no_pid_list;
835 	struct trace_pid_list *pid_list;
836 
837 	/* Nothing to do if we are already tracing */
838 	if (!this_cpu_read(tr->array_buffer.data->ignore_pid))
839 		return;
840 
841 	pid_list = rcu_dereference_sched(tr->filtered_pids);
842 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
843 
844 	this_cpu_write(tr->array_buffer.data->ignore_pid,
845 		       trace_ignore_this_task(pid_list, no_pid_list, task));
846 }
847 
848 static void
849 event_filter_pid_sched_wakeup_probe_post(void *data, struct task_struct *task)
850 {
851 	struct trace_array *tr = data;
852 	struct trace_pid_list *no_pid_list;
853 	struct trace_pid_list *pid_list;
854 
855 	/* Nothing to do if we are not tracing */
856 	if (this_cpu_read(tr->array_buffer.data->ignore_pid))
857 		return;
858 
859 	pid_list = rcu_dereference_sched(tr->filtered_pids);
860 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
861 
862 	/* Set tracing if current is enabled */
863 	this_cpu_write(tr->array_buffer.data->ignore_pid,
864 		       trace_ignore_this_task(pid_list, no_pid_list, current));
865 }
866 
867 static void unregister_pid_events(struct trace_array *tr)
868 {
869 	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_pre, tr);
870 	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_post, tr);
871 
872 	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre, tr);
873 	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_post, tr);
874 
875 	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre, tr);
876 	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post, tr);
877 
878 	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_pre, tr);
879 	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_post, tr);
880 }
881 
882 static void __ftrace_clear_event_pids(struct trace_array *tr, int type)
883 {
884 	struct trace_pid_list *pid_list;
885 	struct trace_pid_list *no_pid_list;
886 	struct trace_event_file *file;
887 	int cpu;
888 
889 	pid_list = rcu_dereference_protected(tr->filtered_pids,
890 					     lockdep_is_held(&event_mutex));
891 	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
892 					     lockdep_is_held(&event_mutex));
893 
894 	/* Make sure there's something to do */
895 	if (!pid_type_enabled(type, pid_list, no_pid_list))
896 		return;
897 
898 	if (!still_need_pid_events(type, pid_list, no_pid_list)) {
899 		unregister_pid_events(tr);
900 
901 		list_for_each_entry(file, &tr->events, list) {
902 			clear_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
903 		}
904 
905 		for_each_possible_cpu(cpu)
906 			per_cpu_ptr(tr->array_buffer.data, cpu)->ignore_pid = false;
907 	}
908 
909 	if (type & TRACE_PIDS)
910 		rcu_assign_pointer(tr->filtered_pids, NULL);
911 
912 	if (type & TRACE_NO_PIDS)
913 		rcu_assign_pointer(tr->filtered_no_pids, NULL);
914 
915 	/* Wait till all users are no longer using pid filtering */
916 	tracepoint_synchronize_unregister();
917 
918 	if ((type & TRACE_PIDS) && pid_list)
919 		trace_pid_list_free(pid_list);
920 
921 	if ((type & TRACE_NO_PIDS) && no_pid_list)
922 		trace_pid_list_free(no_pid_list);
923 }
924 
925 static void ftrace_clear_event_pids(struct trace_array *tr, int type)
926 {
927 	mutex_lock(&event_mutex);
928 	__ftrace_clear_event_pids(tr, type);
929 	mutex_unlock(&event_mutex);
930 }
931 
932 static void __put_system(struct event_subsystem *system)
933 {
934 	struct event_filter *filter = system->filter;
935 
936 	WARN_ON_ONCE(system_refcount(system) == 0);
937 	if (system_refcount_dec(system))
938 		return;
939 
940 	list_del(&system->list);
941 
942 	if (filter) {
943 		kfree(filter->filter_string);
944 		kfree(filter);
945 	}
946 	kfree_const(system->name);
947 	kfree(system);
948 }
949 
950 static void __get_system(struct event_subsystem *system)
951 {
952 	WARN_ON_ONCE(system_refcount(system) == 0);
953 	system_refcount_inc(system);
954 }
955 
956 static void __get_system_dir(struct trace_subsystem_dir *dir)
957 {
958 	WARN_ON_ONCE(dir->ref_count == 0);
959 	dir->ref_count++;
960 	__get_system(dir->subsystem);
961 }
962 
963 static void __put_system_dir(struct trace_subsystem_dir *dir)
964 {
965 	WARN_ON_ONCE(dir->ref_count == 0);
966 	/* If the subsystem is about to be freed, the dir must be too */
967 	WARN_ON_ONCE(system_refcount(dir->subsystem) == 1 && dir->ref_count != 1);
968 
969 	__put_system(dir->subsystem);
970 	if (!--dir->ref_count)
971 		kfree(dir);
972 }
973 
974 static void put_system(struct trace_subsystem_dir *dir)
975 {
976 	mutex_lock(&event_mutex);
977 	__put_system_dir(dir);
978 	mutex_unlock(&event_mutex);
979 }
980 
981 static void remove_subsystem(struct trace_subsystem_dir *dir)
982 {
983 	if (!dir)
984 		return;
985 
986 	if (!--dir->nr_events) {
987 		eventfs_remove_dir(dir->ei);
988 		list_del(&dir->list);
989 		__put_system_dir(dir);
990 	}
991 }
992 
993 void event_file_get(struct trace_event_file *file)
994 {
995 	atomic_inc(&file->ref);
996 }
997 
998 void event_file_put(struct trace_event_file *file)
999 {
1000 	if (WARN_ON_ONCE(!atomic_read(&file->ref))) {
1001 		if (file->flags & EVENT_FILE_FL_FREED)
1002 			kmem_cache_free(file_cachep, file);
1003 		return;
1004 	}
1005 
1006 	if (atomic_dec_and_test(&file->ref)) {
1007 		/* Count should only go to zero when it is freed */
1008 		if (WARN_ON_ONCE(!(file->flags & EVENT_FILE_FL_FREED)))
1009 			return;
1010 		kmem_cache_free(file_cachep, file);
1011 	}
1012 }
1013 
1014 static void remove_event_file_dir(struct trace_event_file *file)
1015 {
1016 	eventfs_remove_dir(file->ei);
1017 	list_del(&file->list);
1018 	remove_subsystem(file->system);
1019 	free_event_filter(file->filter);
1020 	file->flags |= EVENT_FILE_FL_FREED;
1021 	event_file_put(file);
1022 }
1023 
1024 /*
1025  * __ftrace_set_clr_event(NULL, NULL, NULL, set) will set/unset all events.
1026  */
1027 static int
1028 __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,
1029 			      const char *sub, const char *event, int set)
1030 {
1031 	struct trace_event_file *file;
1032 	struct trace_event_call *call;
1033 	const char *name;
1034 	int ret = -EINVAL;
1035 	int eret = 0;
1036 
1037 	list_for_each_entry(file, &tr->events, list) {
1038 
1039 		call = file->event_call;
1040 		name = trace_event_name(call);
1041 
1042 		if (!name || !call->class || !call->class->reg)
1043 			continue;
1044 
1045 		if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
1046 			continue;
1047 
1048 		if (match &&
1049 		    strcmp(match, name) != 0 &&
1050 		    strcmp(match, call->class->system) != 0)
1051 			continue;
1052 
1053 		if (sub && strcmp(sub, call->class->system) != 0)
1054 			continue;
1055 
1056 		if (event && strcmp(event, name) != 0)
1057 			continue;
1058 
1059 		ret = ftrace_event_enable_disable(file, set);
1060 
1061 		/*
1062 		 * Save the first error and return that. Some events
1063 		 * may still have been enabled, but let the user
1064 		 * know that something went wrong.
1065 		 */
1066 		if (ret && !eret)
1067 			eret = ret;
1068 
1069 		ret = eret;
1070 	}
1071 
1072 	return ret;
1073 }
1074 
1075 static int __ftrace_set_clr_event(struct trace_array *tr, const char *match,
1076 				  const char *sub, const char *event, int set)
1077 {
1078 	int ret;
1079 
1080 	mutex_lock(&event_mutex);
1081 	ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set);
1082 	mutex_unlock(&event_mutex);
1083 
1084 	return ret;
1085 }
1086 
1087 int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set)
1088 {
1089 	char *event = NULL, *sub = NULL, *match;
1090 	int ret;
1091 
1092 	if (!tr)
1093 		return -ENOENT;
1094 	/*
1095 	 * The buf format can be <subsystem>:<event-name>
1096 	 *  *:<event-name> means any event by that name.
1097 	 *  :<event-name> is the same.
1098 	 *
1099 	 *  <subsystem>:* means all events in that subsystem
1100 	 *  <subsystem>: means the same.
1101 	 *
1102 	 *  <name> (no ':') means all events in a subsystem with
1103 	 *  the name <name> or any event that matches <name>
1104 	 */
1105 
1106 	match = strsep(&buf, ":");
1107 	if (buf) {
1108 		sub = match;
1109 		event = buf;
1110 		match = NULL;
1111 
1112 		if (!strlen(sub) || strcmp(sub, "*") == 0)
1113 			sub = NULL;
1114 		if (!strlen(event) || strcmp(event, "*") == 0)
1115 			event = NULL;
1116 	}
1117 
1118 	ret = __ftrace_set_clr_event(tr, match, sub, event, set);
1119 
1120 	/* Put back the colon to allow this to be called again */
1121 	if (buf)
1122 		*(buf - 1) = ':';
1123 
1124 	return ret;
1125 }
1126 
1127 /**
1128  * trace_set_clr_event - enable or disable an event
1129  * @system: system name to match (NULL for any system)
1130  * @event: event name to match (NULL for all events, within system)
1131  * @set: 1 to enable, 0 to disable
1132  *
1133  * This is a way for other parts of the kernel to enable or disable
1134  * event recording.
1135  *
1136  * Returns 0 on success, -EINVAL if the parameters do not match any
1137  * registered events.
1138  */
1139 int trace_set_clr_event(const char *system, const char *event, int set)
1140 {
1141 	struct trace_array *tr = top_trace_array();
1142 
1143 	if (!tr)
1144 		return -ENODEV;
1145 
1146 	return __ftrace_set_clr_event(tr, NULL, system, event, set);
1147 }
1148 EXPORT_SYMBOL_GPL(trace_set_clr_event);
1149 
1150 /**
1151  * trace_array_set_clr_event - enable or disable an event for a trace array.
1152  * @tr: concerned trace array.
1153  * @system: system name to match (NULL for any system)
1154  * @event: event name to match (NULL for all events, within system)
1155  * @enable: true to enable, false to disable
1156  *
1157  * This is a way for other parts of the kernel to enable or disable
1158  * event recording.
1159  *
1160  * Returns 0 on success, -EINVAL if the parameters do not match any
1161  * registered events.
1162  */
1163 int trace_array_set_clr_event(struct trace_array *tr, const char *system,
1164 		const char *event, bool enable)
1165 {
1166 	int set;
1167 
1168 	if (!tr)
1169 		return -ENOENT;
1170 
1171 	set = (enable == true) ? 1 : 0;
1172 	return __ftrace_set_clr_event(tr, NULL, system, event, set);
1173 }
1174 EXPORT_SYMBOL_GPL(trace_array_set_clr_event);
1175 
1176 /* 128 should be much more than enough */
1177 #define EVENT_BUF_SIZE		127
1178 
1179 static ssize_t
1180 ftrace_event_write(struct file *file, const char __user *ubuf,
1181 		   size_t cnt, loff_t *ppos)
1182 {
1183 	struct trace_parser parser;
1184 	struct seq_file *m = file->private_data;
1185 	struct trace_array *tr = m->private;
1186 	ssize_t read, ret;
1187 
1188 	if (!cnt)
1189 		return 0;
1190 
1191 	ret = tracing_update_buffers();
1192 	if (ret < 0)
1193 		return ret;
1194 
1195 	if (trace_parser_get_init(&parser, EVENT_BUF_SIZE + 1))
1196 		return -ENOMEM;
1197 
1198 	read = trace_get_user(&parser, ubuf, cnt, ppos);
1199 
1200 	if (read >= 0 && trace_parser_loaded((&parser))) {
1201 		int set = 1;
1202 
1203 		if (*parser.buffer == '!')
1204 			set = 0;
1205 
1206 		ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);
1207 		if (ret)
1208 			goto out_put;
1209 	}
1210 
1211 	ret = read;
1212 
1213  out_put:
1214 	trace_parser_put(&parser);
1215 
1216 	return ret;
1217 }
1218 
1219 static void *
1220 t_next(struct seq_file *m, void *v, loff_t *pos)
1221 {
1222 	struct trace_event_file *file = v;
1223 	struct trace_event_call *call;
1224 	struct trace_array *tr = m->private;
1225 
1226 	(*pos)++;
1227 
1228 	list_for_each_entry_continue(file, &tr->events, list) {
1229 		call = file->event_call;
1230 		/*
1231 		 * The ftrace subsystem is for showing formats only.
1232 		 * They can not be enabled or disabled via the event files.
1233 		 */
1234 		if (call->class && call->class->reg &&
1235 		    !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
1236 			return file;
1237 	}
1238 
1239 	return NULL;
1240 }
1241 
1242 static void *t_start(struct seq_file *m, loff_t *pos)
1243 {
1244 	struct trace_event_file *file;
1245 	struct trace_array *tr = m->private;
1246 	loff_t l;
1247 
1248 	mutex_lock(&event_mutex);
1249 
1250 	file = list_entry(&tr->events, struct trace_event_file, list);
1251 	for (l = 0; l <= *pos; ) {
1252 		file = t_next(m, file, &l);
1253 		if (!file)
1254 			break;
1255 	}
1256 	return file;
1257 }
1258 
1259 static void *
1260 s_next(struct seq_file *m, void *v, loff_t *pos)
1261 {
1262 	struct trace_event_file *file = v;
1263 	struct trace_array *tr = m->private;
1264 
1265 	(*pos)++;
1266 
1267 	list_for_each_entry_continue(file, &tr->events, list) {
1268 		if (file->flags & EVENT_FILE_FL_ENABLED)
1269 			return file;
1270 	}
1271 
1272 	return NULL;
1273 }
1274 
1275 static void *s_start(struct seq_file *m, loff_t *pos)
1276 {
1277 	struct trace_event_file *file;
1278 	struct trace_array *tr = m->private;
1279 	loff_t l;
1280 
1281 	mutex_lock(&event_mutex);
1282 
1283 	file = list_entry(&tr->events, struct trace_event_file, list);
1284 	for (l = 0; l <= *pos; ) {
1285 		file = s_next(m, file, &l);
1286 		if (!file)
1287 			break;
1288 	}
1289 	return file;
1290 }
1291 
1292 static int t_show(struct seq_file *m, void *v)
1293 {
1294 	struct trace_event_file *file = v;
1295 	struct trace_event_call *call = file->event_call;
1296 
1297 	if (strcmp(call->class->system, TRACE_SYSTEM) != 0)
1298 		seq_printf(m, "%s:", call->class->system);
1299 	seq_printf(m, "%s\n", trace_event_name(call));
1300 
1301 	return 0;
1302 }
1303 
1304 static void t_stop(struct seq_file *m, void *p)
1305 {
1306 	mutex_unlock(&event_mutex);
1307 }
1308 
1309 static void *
1310 __next(struct seq_file *m, void *v, loff_t *pos, int type)
1311 {
1312 	struct trace_array *tr = m->private;
1313 	struct trace_pid_list *pid_list;
1314 
1315 	if (type == TRACE_PIDS)
1316 		pid_list = rcu_dereference_sched(tr->filtered_pids);
1317 	else
1318 		pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1319 
1320 	return trace_pid_next(pid_list, v, pos);
1321 }
1322 
1323 static void *
1324 p_next(struct seq_file *m, void *v, loff_t *pos)
1325 {
1326 	return __next(m, v, pos, TRACE_PIDS);
1327 }
1328 
1329 static void *
1330 np_next(struct seq_file *m, void *v, loff_t *pos)
1331 {
1332 	return __next(m, v, pos, TRACE_NO_PIDS);
1333 }
1334 
1335 static void *__start(struct seq_file *m, loff_t *pos, int type)
1336 	__acquires(RCU)
1337 {
1338 	struct trace_pid_list *pid_list;
1339 	struct trace_array *tr = m->private;
1340 
1341 	/*
1342 	 * Grab the mutex, to keep calls to p_next() having the same
1343 	 * tr->filtered_pids as p_start() has.
1344 	 * If we just passed the tr->filtered_pids around, then RCU would
1345 	 * have been enough, but doing that makes things more complex.
1346 	 */
1347 	mutex_lock(&event_mutex);
1348 	rcu_read_lock_sched();
1349 
1350 	if (type == TRACE_PIDS)
1351 		pid_list = rcu_dereference_sched(tr->filtered_pids);
1352 	else
1353 		pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1354 
1355 	if (!pid_list)
1356 		return NULL;
1357 
1358 	return trace_pid_start(pid_list, pos);
1359 }
1360 
1361 static void *p_start(struct seq_file *m, loff_t *pos)
1362 	__acquires(RCU)
1363 {
1364 	return __start(m, pos, TRACE_PIDS);
1365 }
1366 
1367 static void *np_start(struct seq_file *m, loff_t *pos)
1368 	__acquires(RCU)
1369 {
1370 	return __start(m, pos, TRACE_NO_PIDS);
1371 }
1372 
1373 static void p_stop(struct seq_file *m, void *p)
1374 	__releases(RCU)
1375 {
1376 	rcu_read_unlock_sched();
1377 	mutex_unlock(&event_mutex);
1378 }
1379 
1380 static ssize_t
1381 event_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1382 		  loff_t *ppos)
1383 {
1384 	struct trace_event_file *file;
1385 	unsigned long flags;
1386 	char buf[4] = "0";
1387 
1388 	mutex_lock(&event_mutex);
1389 	file = event_file_data(filp);
1390 	if (likely(file))
1391 		flags = file->flags;
1392 	mutex_unlock(&event_mutex);
1393 
1394 	if (!file || flags & EVENT_FILE_FL_FREED)
1395 		return -ENODEV;
1396 
1397 	if (flags & EVENT_FILE_FL_ENABLED &&
1398 	    !(flags & EVENT_FILE_FL_SOFT_DISABLED))
1399 		strcpy(buf, "1");
1400 
1401 	if (flags & EVENT_FILE_FL_SOFT_DISABLED ||
1402 	    flags & EVENT_FILE_FL_SOFT_MODE)
1403 		strcat(buf, "*");
1404 
1405 	strcat(buf, "\n");
1406 
1407 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, strlen(buf));
1408 }
1409 
1410 static ssize_t
1411 event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1412 		   loff_t *ppos)
1413 {
1414 	struct trace_event_file *file;
1415 	unsigned long val;
1416 	int ret;
1417 
1418 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1419 	if (ret)
1420 		return ret;
1421 
1422 	ret = tracing_update_buffers();
1423 	if (ret < 0)
1424 		return ret;
1425 
1426 	switch (val) {
1427 	case 0:
1428 	case 1:
1429 		ret = -ENODEV;
1430 		mutex_lock(&event_mutex);
1431 		file = event_file_data(filp);
1432 		if (likely(file && !(file->flags & EVENT_FILE_FL_FREED)))
1433 			ret = ftrace_event_enable_disable(file, val);
1434 		mutex_unlock(&event_mutex);
1435 		break;
1436 
1437 	default:
1438 		return -EINVAL;
1439 	}
1440 
1441 	*ppos += cnt;
1442 
1443 	return ret ? ret : cnt;
1444 }
1445 
1446 static ssize_t
1447 system_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1448 		   loff_t *ppos)
1449 {
1450 	const char set_to_char[4] = { '?', '0', '1', 'X' };
1451 	struct trace_subsystem_dir *dir = filp->private_data;
1452 	struct event_subsystem *system = dir->subsystem;
1453 	struct trace_event_call *call;
1454 	struct trace_event_file *file;
1455 	struct trace_array *tr = dir->tr;
1456 	char buf[2];
1457 	int set = 0;
1458 	int ret;
1459 
1460 	mutex_lock(&event_mutex);
1461 	list_for_each_entry(file, &tr->events, list) {
1462 		call = file->event_call;
1463 		if ((call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
1464 		    !trace_event_name(call) || !call->class || !call->class->reg)
1465 			continue;
1466 
1467 		if (system && strcmp(call->class->system, system->name) != 0)
1468 			continue;
1469 
1470 		/*
1471 		 * We need to find out if all the events are set
1472 		 * or if all events or cleared, or if we have
1473 		 * a mixture.
1474 		 */
1475 		set |= (1 << !!(file->flags & EVENT_FILE_FL_ENABLED));
1476 
1477 		/*
1478 		 * If we have a mixture, no need to look further.
1479 		 */
1480 		if (set == 3)
1481 			break;
1482 	}
1483 	mutex_unlock(&event_mutex);
1484 
1485 	buf[0] = set_to_char[set];
1486 	buf[1] = '\n';
1487 
1488 	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
1489 
1490 	return ret;
1491 }
1492 
1493 static ssize_t
1494 system_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1495 		    loff_t *ppos)
1496 {
1497 	struct trace_subsystem_dir *dir = filp->private_data;
1498 	struct event_subsystem *system = dir->subsystem;
1499 	const char *name = NULL;
1500 	unsigned long val;
1501 	ssize_t ret;
1502 
1503 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1504 	if (ret)
1505 		return ret;
1506 
1507 	ret = tracing_update_buffers();
1508 	if (ret < 0)
1509 		return ret;
1510 
1511 	if (val != 0 && val != 1)
1512 		return -EINVAL;
1513 
1514 	/*
1515 	 * Opening of "enable" adds a ref count to system,
1516 	 * so the name is safe to use.
1517 	 */
1518 	if (system)
1519 		name = system->name;
1520 
1521 	ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val);
1522 	if (ret)
1523 		goto out;
1524 
1525 	ret = cnt;
1526 
1527 out:
1528 	*ppos += cnt;
1529 
1530 	return ret;
1531 }
1532 
1533 enum {
1534 	FORMAT_HEADER		= 1,
1535 	FORMAT_FIELD_SEPERATOR	= 2,
1536 	FORMAT_PRINTFMT		= 3,
1537 };
1538 
1539 static void *f_next(struct seq_file *m, void *v, loff_t *pos)
1540 {
1541 	struct trace_event_call *call = event_file_data(m->private);
1542 	struct list_head *common_head = &ftrace_common_fields;
1543 	struct list_head *head = trace_get_fields(call);
1544 	struct list_head *node = v;
1545 
1546 	(*pos)++;
1547 
1548 	switch ((unsigned long)v) {
1549 	case FORMAT_HEADER:
1550 		node = common_head;
1551 		break;
1552 
1553 	case FORMAT_FIELD_SEPERATOR:
1554 		node = head;
1555 		break;
1556 
1557 	case FORMAT_PRINTFMT:
1558 		/* all done */
1559 		return NULL;
1560 	}
1561 
1562 	node = node->prev;
1563 	if (node == common_head)
1564 		return (void *)FORMAT_FIELD_SEPERATOR;
1565 	else if (node == head)
1566 		return (void *)FORMAT_PRINTFMT;
1567 	else
1568 		return node;
1569 }
1570 
1571 static int f_show(struct seq_file *m, void *v)
1572 {
1573 	struct trace_event_call *call = event_file_data(m->private);
1574 	struct ftrace_event_field *field;
1575 	const char *array_descriptor;
1576 
1577 	switch ((unsigned long)v) {
1578 	case FORMAT_HEADER:
1579 		seq_printf(m, "name: %s\n", trace_event_name(call));
1580 		seq_printf(m, "ID: %d\n", call->event.type);
1581 		seq_puts(m, "format:\n");
1582 		return 0;
1583 
1584 	case FORMAT_FIELD_SEPERATOR:
1585 		seq_putc(m, '\n');
1586 		return 0;
1587 
1588 	case FORMAT_PRINTFMT:
1589 		seq_printf(m, "\nprint fmt: %s\n",
1590 			   call->print_fmt);
1591 		return 0;
1592 	}
1593 
1594 	field = list_entry(v, struct ftrace_event_field, link);
1595 	/*
1596 	 * Smartly shows the array type(except dynamic array).
1597 	 * Normal:
1598 	 *	field:TYPE VAR
1599 	 * If TYPE := TYPE[LEN], it is shown:
1600 	 *	field:TYPE VAR[LEN]
1601 	 */
1602 	array_descriptor = strchr(field->type, '[');
1603 
1604 	if (str_has_prefix(field->type, "__data_loc"))
1605 		array_descriptor = NULL;
1606 
1607 	if (!array_descriptor)
1608 		seq_printf(m, "\tfield:%s %s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1609 			   field->type, field->name, field->offset,
1610 			   field->size, !!field->is_signed);
1611 	else if (field->len)
1612 		seq_printf(m, "\tfield:%.*s %s[%d];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1613 			   (int)(array_descriptor - field->type),
1614 			   field->type, field->name,
1615 			   field->len, field->offset,
1616 			   field->size, !!field->is_signed);
1617 	else
1618 		seq_printf(m, "\tfield:%.*s %s[];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1619 				(int)(array_descriptor - field->type),
1620 				field->type, field->name,
1621 				field->offset, field->size, !!field->is_signed);
1622 
1623 	return 0;
1624 }
1625 
1626 static void *f_start(struct seq_file *m, loff_t *pos)
1627 {
1628 	void *p = (void *)FORMAT_HEADER;
1629 	loff_t l = 0;
1630 
1631 	/* ->stop() is called even if ->start() fails */
1632 	mutex_lock(&event_mutex);
1633 	if (!event_file_data(m->private))
1634 		return ERR_PTR(-ENODEV);
1635 
1636 	while (l < *pos && p)
1637 		p = f_next(m, p, &l);
1638 
1639 	return p;
1640 }
1641 
1642 static void f_stop(struct seq_file *m, void *p)
1643 {
1644 	mutex_unlock(&event_mutex);
1645 }
1646 
1647 static const struct seq_operations trace_format_seq_ops = {
1648 	.start		= f_start,
1649 	.next		= f_next,
1650 	.stop		= f_stop,
1651 	.show		= f_show,
1652 };
1653 
1654 static int trace_format_open(struct inode *inode, struct file *file)
1655 {
1656 	struct seq_file *m;
1657 	int ret;
1658 
1659 	/* Do we want to hide event format files on tracefs lockdown? */
1660 
1661 	ret = seq_open(file, &trace_format_seq_ops);
1662 	if (ret < 0)
1663 		return ret;
1664 
1665 	m = file->private_data;
1666 	m->private = file;
1667 
1668 	return 0;
1669 }
1670 
1671 #ifdef CONFIG_PERF_EVENTS
1672 static ssize_t
1673 event_id_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1674 {
1675 	int id = (long)event_file_data(filp);
1676 	char buf[32];
1677 	int len;
1678 
1679 	if (unlikely(!id))
1680 		return -ENODEV;
1681 
1682 	len = sprintf(buf, "%d\n", id);
1683 
1684 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
1685 }
1686 #endif
1687 
1688 static ssize_t
1689 event_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
1690 		  loff_t *ppos)
1691 {
1692 	struct trace_event_file *file;
1693 	struct trace_seq *s;
1694 	int r = -ENODEV;
1695 
1696 	if (*ppos)
1697 		return 0;
1698 
1699 	s = kmalloc(sizeof(*s), GFP_KERNEL);
1700 
1701 	if (!s)
1702 		return -ENOMEM;
1703 
1704 	trace_seq_init(s);
1705 
1706 	mutex_lock(&event_mutex);
1707 	file = event_file_data(filp);
1708 	if (file && !(file->flags & EVENT_FILE_FL_FREED))
1709 		print_event_filter(file, s);
1710 	mutex_unlock(&event_mutex);
1711 
1712 	if (file)
1713 		r = simple_read_from_buffer(ubuf, cnt, ppos,
1714 					    s->buffer, trace_seq_used(s));
1715 
1716 	kfree(s);
1717 
1718 	return r;
1719 }
1720 
1721 static ssize_t
1722 event_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
1723 		   loff_t *ppos)
1724 {
1725 	struct trace_event_file *file;
1726 	char *buf;
1727 	int err = -ENODEV;
1728 
1729 	if (cnt >= PAGE_SIZE)
1730 		return -EINVAL;
1731 
1732 	buf = memdup_user_nul(ubuf, cnt);
1733 	if (IS_ERR(buf))
1734 		return PTR_ERR(buf);
1735 
1736 	mutex_lock(&event_mutex);
1737 	file = event_file_data(filp);
1738 	if (file)
1739 		err = apply_event_filter(file, buf);
1740 	mutex_unlock(&event_mutex);
1741 
1742 	kfree(buf);
1743 	if (err < 0)
1744 		return err;
1745 
1746 	*ppos += cnt;
1747 
1748 	return cnt;
1749 }
1750 
1751 static LIST_HEAD(event_subsystems);
1752 
1753 static int subsystem_open(struct inode *inode, struct file *filp)
1754 {
1755 	struct trace_subsystem_dir *dir = NULL, *iter_dir;
1756 	struct trace_array *tr = NULL, *iter_tr;
1757 	struct event_subsystem *system = NULL;
1758 	int ret;
1759 
1760 	if (tracing_is_disabled())
1761 		return -ENODEV;
1762 
1763 	/* Make sure the system still exists */
1764 	mutex_lock(&event_mutex);
1765 	mutex_lock(&trace_types_lock);
1766 	list_for_each_entry(iter_tr, &ftrace_trace_arrays, list) {
1767 		list_for_each_entry(iter_dir, &iter_tr->systems, list) {
1768 			if (iter_dir == inode->i_private) {
1769 				/* Don't open systems with no events */
1770 				tr = iter_tr;
1771 				dir = iter_dir;
1772 				if (dir->nr_events) {
1773 					__get_system_dir(dir);
1774 					system = dir->subsystem;
1775 				}
1776 				goto exit_loop;
1777 			}
1778 		}
1779 	}
1780  exit_loop:
1781 	mutex_unlock(&trace_types_lock);
1782 	mutex_unlock(&event_mutex);
1783 
1784 	if (!system)
1785 		return -ENODEV;
1786 
1787 	/* Still need to increment the ref count of the system */
1788 	if (trace_array_get(tr) < 0) {
1789 		put_system(dir);
1790 		return -ENODEV;
1791 	}
1792 
1793 	ret = tracing_open_generic(inode, filp);
1794 	if (ret < 0) {
1795 		trace_array_put(tr);
1796 		put_system(dir);
1797 	}
1798 
1799 	return ret;
1800 }
1801 
1802 static int system_tr_open(struct inode *inode, struct file *filp)
1803 {
1804 	struct trace_subsystem_dir *dir;
1805 	struct trace_array *tr = inode->i_private;
1806 	int ret;
1807 
1808 	/* Make a temporary dir that has no system but points to tr */
1809 	dir = kzalloc(sizeof(*dir), GFP_KERNEL);
1810 	if (!dir)
1811 		return -ENOMEM;
1812 
1813 	ret = tracing_open_generic_tr(inode, filp);
1814 	if (ret < 0) {
1815 		kfree(dir);
1816 		return ret;
1817 	}
1818 	dir->tr = tr;
1819 	filp->private_data = dir;
1820 
1821 	return 0;
1822 }
1823 
1824 static int subsystem_release(struct inode *inode, struct file *file)
1825 {
1826 	struct trace_subsystem_dir *dir = file->private_data;
1827 
1828 	trace_array_put(dir->tr);
1829 
1830 	/*
1831 	 * If dir->subsystem is NULL, then this is a temporary
1832 	 * descriptor that was made for a trace_array to enable
1833 	 * all subsystems.
1834 	 */
1835 	if (dir->subsystem)
1836 		put_system(dir);
1837 	else
1838 		kfree(dir);
1839 
1840 	return 0;
1841 }
1842 
1843 static ssize_t
1844 subsystem_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
1845 		      loff_t *ppos)
1846 {
1847 	struct trace_subsystem_dir *dir = filp->private_data;
1848 	struct event_subsystem *system = dir->subsystem;
1849 	struct trace_seq *s;
1850 	int r;
1851 
1852 	if (*ppos)
1853 		return 0;
1854 
1855 	s = kmalloc(sizeof(*s), GFP_KERNEL);
1856 	if (!s)
1857 		return -ENOMEM;
1858 
1859 	trace_seq_init(s);
1860 
1861 	print_subsystem_event_filter(system, s);
1862 	r = simple_read_from_buffer(ubuf, cnt, ppos,
1863 				    s->buffer, trace_seq_used(s));
1864 
1865 	kfree(s);
1866 
1867 	return r;
1868 }
1869 
1870 static ssize_t
1871 subsystem_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
1872 		       loff_t *ppos)
1873 {
1874 	struct trace_subsystem_dir *dir = filp->private_data;
1875 	char *buf;
1876 	int err;
1877 
1878 	if (cnt >= PAGE_SIZE)
1879 		return -EINVAL;
1880 
1881 	buf = memdup_user_nul(ubuf, cnt);
1882 	if (IS_ERR(buf))
1883 		return PTR_ERR(buf);
1884 
1885 	err = apply_subsystem_event_filter(dir, buf);
1886 	kfree(buf);
1887 	if (err < 0)
1888 		return err;
1889 
1890 	*ppos += cnt;
1891 
1892 	return cnt;
1893 }
1894 
1895 static ssize_t
1896 show_header(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1897 {
1898 	int (*func)(struct trace_seq *s) = filp->private_data;
1899 	struct trace_seq *s;
1900 	int r;
1901 
1902 	if (*ppos)
1903 		return 0;
1904 
1905 	s = kmalloc(sizeof(*s), GFP_KERNEL);
1906 	if (!s)
1907 		return -ENOMEM;
1908 
1909 	trace_seq_init(s);
1910 
1911 	func(s);
1912 	r = simple_read_from_buffer(ubuf, cnt, ppos,
1913 				    s->buffer, trace_seq_used(s));
1914 
1915 	kfree(s);
1916 
1917 	return r;
1918 }
1919 
1920 static void ignore_task_cpu(void *data)
1921 {
1922 	struct trace_array *tr = data;
1923 	struct trace_pid_list *pid_list;
1924 	struct trace_pid_list *no_pid_list;
1925 
1926 	/*
1927 	 * This function is called by on_each_cpu() while the
1928 	 * event_mutex is held.
1929 	 */
1930 	pid_list = rcu_dereference_protected(tr->filtered_pids,
1931 					     mutex_is_locked(&event_mutex));
1932 	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
1933 					     mutex_is_locked(&event_mutex));
1934 
1935 	this_cpu_write(tr->array_buffer.data->ignore_pid,
1936 		       trace_ignore_this_task(pid_list, no_pid_list, current));
1937 }
1938 
1939 static void register_pid_events(struct trace_array *tr)
1940 {
1941 	/*
1942 	 * Register a probe that is called before all other probes
1943 	 * to set ignore_pid if next or prev do not match.
1944 	 * Register a probe this is called after all other probes
1945 	 * to only keep ignore_pid set if next pid matches.
1946 	 */
1947 	register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_pre,
1948 					 tr, INT_MAX);
1949 	register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_post,
1950 					 tr, 0);
1951 
1952 	register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre,
1953 					 tr, INT_MAX);
1954 	register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_post,
1955 					 tr, 0);
1956 
1957 	register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre,
1958 					     tr, INT_MAX);
1959 	register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post,
1960 					     tr, 0);
1961 
1962 	register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_pre,
1963 					 tr, INT_MAX);
1964 	register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_post,
1965 					 tr, 0);
1966 }
1967 
1968 static ssize_t
1969 event_pid_write(struct file *filp, const char __user *ubuf,
1970 		size_t cnt, loff_t *ppos, int type)
1971 {
1972 	struct seq_file *m = filp->private_data;
1973 	struct trace_array *tr = m->private;
1974 	struct trace_pid_list *filtered_pids = NULL;
1975 	struct trace_pid_list *other_pids = NULL;
1976 	struct trace_pid_list *pid_list;
1977 	struct trace_event_file *file;
1978 	ssize_t ret;
1979 
1980 	if (!cnt)
1981 		return 0;
1982 
1983 	ret = tracing_update_buffers();
1984 	if (ret < 0)
1985 		return ret;
1986 
1987 	mutex_lock(&event_mutex);
1988 
1989 	if (type == TRACE_PIDS) {
1990 		filtered_pids = rcu_dereference_protected(tr->filtered_pids,
1991 							  lockdep_is_held(&event_mutex));
1992 		other_pids = rcu_dereference_protected(tr->filtered_no_pids,
1993 							  lockdep_is_held(&event_mutex));
1994 	} else {
1995 		filtered_pids = rcu_dereference_protected(tr->filtered_no_pids,
1996 							  lockdep_is_held(&event_mutex));
1997 		other_pids = rcu_dereference_protected(tr->filtered_pids,
1998 							  lockdep_is_held(&event_mutex));
1999 	}
2000 
2001 	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
2002 	if (ret < 0)
2003 		goto out;
2004 
2005 	if (type == TRACE_PIDS)
2006 		rcu_assign_pointer(tr->filtered_pids, pid_list);
2007 	else
2008 		rcu_assign_pointer(tr->filtered_no_pids, pid_list);
2009 
2010 	list_for_each_entry(file, &tr->events, list) {
2011 		set_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
2012 	}
2013 
2014 	if (filtered_pids) {
2015 		tracepoint_synchronize_unregister();
2016 		trace_pid_list_free(filtered_pids);
2017 	} else if (pid_list && !other_pids) {
2018 		register_pid_events(tr);
2019 	}
2020 
2021 	/*
2022 	 * Ignoring of pids is done at task switch. But we have to
2023 	 * check for those tasks that are currently running.
2024 	 * Always do this in case a pid was appended or removed.
2025 	 */
2026 	on_each_cpu(ignore_task_cpu, tr, 1);
2027 
2028  out:
2029 	mutex_unlock(&event_mutex);
2030 
2031 	if (ret > 0)
2032 		*ppos += ret;
2033 
2034 	return ret;
2035 }
2036 
2037 static ssize_t
2038 ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
2039 		       size_t cnt, loff_t *ppos)
2040 {
2041 	return event_pid_write(filp, ubuf, cnt, ppos, TRACE_PIDS);
2042 }
2043 
2044 static ssize_t
2045 ftrace_event_npid_write(struct file *filp, const char __user *ubuf,
2046 			size_t cnt, loff_t *ppos)
2047 {
2048 	return event_pid_write(filp, ubuf, cnt, ppos, TRACE_NO_PIDS);
2049 }
2050 
2051 static int ftrace_event_avail_open(struct inode *inode, struct file *file);
2052 static int ftrace_event_set_open(struct inode *inode, struct file *file);
2053 static int ftrace_event_set_pid_open(struct inode *inode, struct file *file);
2054 static int ftrace_event_set_npid_open(struct inode *inode, struct file *file);
2055 static int ftrace_event_release(struct inode *inode, struct file *file);
2056 
2057 static const struct seq_operations show_event_seq_ops = {
2058 	.start = t_start,
2059 	.next = t_next,
2060 	.show = t_show,
2061 	.stop = t_stop,
2062 };
2063 
2064 static const struct seq_operations show_set_event_seq_ops = {
2065 	.start = s_start,
2066 	.next = s_next,
2067 	.show = t_show,
2068 	.stop = t_stop,
2069 };
2070 
2071 static const struct seq_operations show_set_pid_seq_ops = {
2072 	.start = p_start,
2073 	.next = p_next,
2074 	.show = trace_pid_show,
2075 	.stop = p_stop,
2076 };
2077 
2078 static const struct seq_operations show_set_no_pid_seq_ops = {
2079 	.start = np_start,
2080 	.next = np_next,
2081 	.show = trace_pid_show,
2082 	.stop = p_stop,
2083 };
2084 
2085 static const struct file_operations ftrace_avail_fops = {
2086 	.open = ftrace_event_avail_open,
2087 	.read = seq_read,
2088 	.llseek = seq_lseek,
2089 	.release = seq_release,
2090 };
2091 
2092 static const struct file_operations ftrace_set_event_fops = {
2093 	.open = ftrace_event_set_open,
2094 	.read = seq_read,
2095 	.write = ftrace_event_write,
2096 	.llseek = seq_lseek,
2097 	.release = ftrace_event_release,
2098 };
2099 
2100 static const struct file_operations ftrace_set_event_pid_fops = {
2101 	.open = ftrace_event_set_pid_open,
2102 	.read = seq_read,
2103 	.write = ftrace_event_pid_write,
2104 	.llseek = seq_lseek,
2105 	.release = ftrace_event_release,
2106 };
2107 
2108 static const struct file_operations ftrace_set_event_notrace_pid_fops = {
2109 	.open = ftrace_event_set_npid_open,
2110 	.read = seq_read,
2111 	.write = ftrace_event_npid_write,
2112 	.llseek = seq_lseek,
2113 	.release = ftrace_event_release,
2114 };
2115 
2116 static const struct file_operations ftrace_enable_fops = {
2117 	.open = tracing_open_file_tr,
2118 	.read = event_enable_read,
2119 	.write = event_enable_write,
2120 	.release = tracing_release_file_tr,
2121 	.llseek = default_llseek,
2122 };
2123 
2124 static const struct file_operations ftrace_event_format_fops = {
2125 	.open = trace_format_open,
2126 	.read = seq_read,
2127 	.llseek = seq_lseek,
2128 	.release = seq_release,
2129 };
2130 
2131 #ifdef CONFIG_PERF_EVENTS
2132 static const struct file_operations ftrace_event_id_fops = {
2133 	.read = event_id_read,
2134 	.llseek = default_llseek,
2135 };
2136 #endif
2137 
2138 static const struct file_operations ftrace_event_filter_fops = {
2139 	.open = tracing_open_file_tr,
2140 	.read = event_filter_read,
2141 	.write = event_filter_write,
2142 	.release = tracing_release_file_tr,
2143 	.llseek = default_llseek,
2144 };
2145 
2146 static const struct file_operations ftrace_subsystem_filter_fops = {
2147 	.open = subsystem_open,
2148 	.read = subsystem_filter_read,
2149 	.write = subsystem_filter_write,
2150 	.llseek = default_llseek,
2151 	.release = subsystem_release,
2152 };
2153 
2154 static const struct file_operations ftrace_system_enable_fops = {
2155 	.open = subsystem_open,
2156 	.read = system_enable_read,
2157 	.write = system_enable_write,
2158 	.llseek = default_llseek,
2159 	.release = subsystem_release,
2160 };
2161 
2162 static const struct file_operations ftrace_tr_enable_fops = {
2163 	.open = system_tr_open,
2164 	.read = system_enable_read,
2165 	.write = system_enable_write,
2166 	.llseek = default_llseek,
2167 	.release = subsystem_release,
2168 };
2169 
2170 static const struct file_operations ftrace_show_header_fops = {
2171 	.open = tracing_open_generic,
2172 	.read = show_header,
2173 	.llseek = default_llseek,
2174 };
2175 
2176 static int
2177 ftrace_event_open(struct inode *inode, struct file *file,
2178 		  const struct seq_operations *seq_ops)
2179 {
2180 	struct seq_file *m;
2181 	int ret;
2182 
2183 	ret = security_locked_down(LOCKDOWN_TRACEFS);
2184 	if (ret)
2185 		return ret;
2186 
2187 	ret = seq_open(file, seq_ops);
2188 	if (ret < 0)
2189 		return ret;
2190 	m = file->private_data;
2191 	/* copy tr over to seq ops */
2192 	m->private = inode->i_private;
2193 
2194 	return ret;
2195 }
2196 
2197 static int ftrace_event_release(struct inode *inode, struct file *file)
2198 {
2199 	struct trace_array *tr = inode->i_private;
2200 
2201 	trace_array_put(tr);
2202 
2203 	return seq_release(inode, file);
2204 }
2205 
2206 static int
2207 ftrace_event_avail_open(struct inode *inode, struct file *file)
2208 {
2209 	const struct seq_operations *seq_ops = &show_event_seq_ops;
2210 
2211 	/* Checks for tracefs lockdown */
2212 	return ftrace_event_open(inode, file, seq_ops);
2213 }
2214 
2215 static int
2216 ftrace_event_set_open(struct inode *inode, struct file *file)
2217 {
2218 	const struct seq_operations *seq_ops = &show_set_event_seq_ops;
2219 	struct trace_array *tr = inode->i_private;
2220 	int ret;
2221 
2222 	ret = tracing_check_open_get_tr(tr);
2223 	if (ret)
2224 		return ret;
2225 
2226 	if ((file->f_mode & FMODE_WRITE) &&
2227 	    (file->f_flags & O_TRUNC))
2228 		ftrace_clear_events(tr);
2229 
2230 	ret = ftrace_event_open(inode, file, seq_ops);
2231 	if (ret < 0)
2232 		trace_array_put(tr);
2233 	return ret;
2234 }
2235 
2236 static int
2237 ftrace_event_set_pid_open(struct inode *inode, struct file *file)
2238 {
2239 	const struct seq_operations *seq_ops = &show_set_pid_seq_ops;
2240 	struct trace_array *tr = inode->i_private;
2241 	int ret;
2242 
2243 	ret = tracing_check_open_get_tr(tr);
2244 	if (ret)
2245 		return ret;
2246 
2247 	if ((file->f_mode & FMODE_WRITE) &&
2248 	    (file->f_flags & O_TRUNC))
2249 		ftrace_clear_event_pids(tr, TRACE_PIDS);
2250 
2251 	ret = ftrace_event_open(inode, file, seq_ops);
2252 	if (ret < 0)
2253 		trace_array_put(tr);
2254 	return ret;
2255 }
2256 
2257 static int
2258 ftrace_event_set_npid_open(struct inode *inode, struct file *file)
2259 {
2260 	const struct seq_operations *seq_ops = &show_set_no_pid_seq_ops;
2261 	struct trace_array *tr = inode->i_private;
2262 	int ret;
2263 
2264 	ret = tracing_check_open_get_tr(tr);
2265 	if (ret)
2266 		return ret;
2267 
2268 	if ((file->f_mode & FMODE_WRITE) &&
2269 	    (file->f_flags & O_TRUNC))
2270 		ftrace_clear_event_pids(tr, TRACE_NO_PIDS);
2271 
2272 	ret = ftrace_event_open(inode, file, seq_ops);
2273 	if (ret < 0)
2274 		trace_array_put(tr);
2275 	return ret;
2276 }
2277 
2278 static struct event_subsystem *
2279 create_new_subsystem(const char *name)
2280 {
2281 	struct event_subsystem *system;
2282 
2283 	/* need to create new entry */
2284 	system = kmalloc(sizeof(*system), GFP_KERNEL);
2285 	if (!system)
2286 		return NULL;
2287 
2288 	system->ref_count = 1;
2289 
2290 	/* Only allocate if dynamic (kprobes and modules) */
2291 	system->name = kstrdup_const(name, GFP_KERNEL);
2292 	if (!system->name)
2293 		goto out_free;
2294 
2295 	system->filter = kzalloc(sizeof(struct event_filter), GFP_KERNEL);
2296 	if (!system->filter)
2297 		goto out_free;
2298 
2299 	list_add(&system->list, &event_subsystems);
2300 
2301 	return system;
2302 
2303  out_free:
2304 	kfree_const(system->name);
2305 	kfree(system);
2306 	return NULL;
2307 }
2308 
2309 static int system_callback(const char *name, umode_t *mode, void **data,
2310 		    const struct file_operations **fops)
2311 {
2312 	if (strcmp(name, "filter") == 0)
2313 		*fops = &ftrace_subsystem_filter_fops;
2314 
2315 	else if (strcmp(name, "enable") == 0)
2316 		*fops = &ftrace_system_enable_fops;
2317 
2318 	else
2319 		return 0;
2320 
2321 	*mode = TRACE_MODE_WRITE;
2322 	return 1;
2323 }
2324 
2325 static struct eventfs_inode *
2326 event_subsystem_dir(struct trace_array *tr, const char *name,
2327 		    struct trace_event_file *file, struct eventfs_inode *parent)
2328 {
2329 	struct event_subsystem *system, *iter;
2330 	struct trace_subsystem_dir *dir;
2331 	struct eventfs_inode *ei;
2332 	int nr_entries;
2333 	static struct eventfs_entry system_entries[] = {
2334 		{
2335 			.name		= "filter",
2336 			.callback	= system_callback,
2337 		},
2338 		{
2339 			.name		= "enable",
2340 			.callback	= system_callback,
2341 		}
2342 	};
2343 
2344 	/* First see if we did not already create this dir */
2345 	list_for_each_entry(dir, &tr->systems, list) {
2346 		system = dir->subsystem;
2347 		if (strcmp(system->name, name) == 0) {
2348 			dir->nr_events++;
2349 			file->system = dir;
2350 			return dir->ei;
2351 		}
2352 	}
2353 
2354 	/* Now see if the system itself exists. */
2355 	system = NULL;
2356 	list_for_each_entry(iter, &event_subsystems, list) {
2357 		if (strcmp(iter->name, name) == 0) {
2358 			system = iter;
2359 			break;
2360 		}
2361 	}
2362 
2363 	dir = kmalloc(sizeof(*dir), GFP_KERNEL);
2364 	if (!dir)
2365 		goto out_fail;
2366 
2367 	if (!system) {
2368 		system = create_new_subsystem(name);
2369 		if (!system)
2370 			goto out_free;
2371 	} else
2372 		__get_system(system);
2373 
2374 	/* ftrace only has directories no files */
2375 	if (strcmp(name, "ftrace") == 0)
2376 		nr_entries = 0;
2377 	else
2378 		nr_entries = ARRAY_SIZE(system_entries);
2379 
2380 	ei = eventfs_create_dir(name, parent, system_entries, nr_entries, dir);
2381 	if (IS_ERR(ei)) {
2382 		pr_warn("Failed to create system directory %s\n", name);
2383 		__put_system(system);
2384 		goto out_free;
2385 	}
2386 
2387 	dir->ei = ei;
2388 	dir->tr = tr;
2389 	dir->ref_count = 1;
2390 	dir->nr_events = 1;
2391 	dir->subsystem = system;
2392 	file->system = dir;
2393 
2394 	list_add(&dir->list, &tr->systems);
2395 
2396 	return dir->ei;
2397 
2398  out_free:
2399 	kfree(dir);
2400  out_fail:
2401 	/* Only print this message if failed on memory allocation */
2402 	if (!dir || !system)
2403 		pr_warn("No memory to create event subsystem %s\n", name);
2404 	return NULL;
2405 }
2406 
2407 static int
2408 event_define_fields(struct trace_event_call *call)
2409 {
2410 	struct list_head *head;
2411 	int ret = 0;
2412 
2413 	/*
2414 	 * Other events may have the same class. Only update
2415 	 * the fields if they are not already defined.
2416 	 */
2417 	head = trace_get_fields(call);
2418 	if (list_empty(head)) {
2419 		struct trace_event_fields *field = call->class->fields_array;
2420 		unsigned int offset = sizeof(struct trace_entry);
2421 
2422 		for (; field->type; field++) {
2423 			if (field->type == TRACE_FUNCTION_TYPE) {
2424 				field->define_fields(call);
2425 				break;
2426 			}
2427 
2428 			offset = ALIGN(offset, field->align);
2429 			ret = trace_define_field_ext(call, field->type, field->name,
2430 						 offset, field->size,
2431 						 field->is_signed, field->filter_type,
2432 						 field->len);
2433 			if (WARN_ON_ONCE(ret)) {
2434 				pr_err("error code is %d\n", ret);
2435 				break;
2436 			}
2437 
2438 			offset += field->size;
2439 		}
2440 	}
2441 
2442 	return ret;
2443 }
2444 
2445 static int event_callback(const char *name, umode_t *mode, void **data,
2446 			  const struct file_operations **fops)
2447 {
2448 	struct trace_event_file *file = *data;
2449 	struct trace_event_call *call = file->event_call;
2450 
2451 	if (strcmp(name, "format") == 0) {
2452 		*mode = TRACE_MODE_READ;
2453 		*fops = &ftrace_event_format_fops;
2454 		*data = call;
2455 		return 1;
2456 	}
2457 
2458 	/*
2459 	 * Only event directories that can be enabled should have
2460 	 * triggers or filters, with the exception of the "print"
2461 	 * event that can have a "trigger" file.
2462 	 */
2463 	if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) {
2464 		if (call->class->reg && strcmp(name, "enable") == 0) {
2465 			*mode = TRACE_MODE_WRITE;
2466 			*fops = &ftrace_enable_fops;
2467 			return 1;
2468 		}
2469 
2470 		if (strcmp(name, "filter") == 0) {
2471 			*mode = TRACE_MODE_WRITE;
2472 			*fops = &ftrace_event_filter_fops;
2473 			return 1;
2474 		}
2475 	}
2476 
2477 	if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
2478 	    strcmp(trace_event_name(call), "print") == 0) {
2479 		if (strcmp(name, "trigger") == 0) {
2480 			*mode = TRACE_MODE_WRITE;
2481 			*fops = &event_trigger_fops;
2482 			return 1;
2483 		}
2484 	}
2485 
2486 #ifdef CONFIG_PERF_EVENTS
2487 	if (call->event.type && call->class->reg &&
2488 	    strcmp(name, "id") == 0) {
2489 		*mode = TRACE_MODE_READ;
2490 		*data = (void *)(long)call->event.type;
2491 		*fops = &ftrace_event_id_fops;
2492 		return 1;
2493 	}
2494 #endif
2495 
2496 #ifdef CONFIG_HIST_TRIGGERS
2497 	if (strcmp(name, "hist") == 0) {
2498 		*mode = TRACE_MODE_READ;
2499 		*fops = &event_hist_fops;
2500 		return 1;
2501 	}
2502 #endif
2503 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
2504 	if (strcmp(name, "hist_debug") == 0) {
2505 		*mode = TRACE_MODE_READ;
2506 		*fops = &event_hist_debug_fops;
2507 		return 1;
2508 	}
2509 #endif
2510 #ifdef CONFIG_TRACE_EVENT_INJECT
2511 	if (call->event.type && call->class->reg &&
2512 	    strcmp(name, "inject") == 0) {
2513 		*mode = 0200;
2514 		*fops = &event_inject_fops;
2515 		return 1;
2516 	}
2517 #endif
2518 	return 0;
2519 }
2520 
2521 /* The file is incremented on creation and freeing the enable file decrements it */
2522 static void event_release(const char *name, void *data)
2523 {
2524 	struct trace_event_file *file = data;
2525 
2526 	event_file_put(file);
2527 }
2528 
2529 static int
2530 event_create_dir(struct eventfs_inode *parent, struct trace_event_file *file)
2531 {
2532 	struct trace_event_call *call = file->event_call;
2533 	struct trace_array *tr = file->tr;
2534 	struct eventfs_inode *e_events;
2535 	struct eventfs_inode *ei;
2536 	const char *name;
2537 	int nr_entries;
2538 	int ret;
2539 	static struct eventfs_entry event_entries[] = {
2540 		{
2541 			.name		= "enable",
2542 			.callback	= event_callback,
2543 			.release	= event_release,
2544 		},
2545 		{
2546 			.name		= "filter",
2547 			.callback	= event_callback,
2548 		},
2549 		{
2550 			.name		= "trigger",
2551 			.callback	= event_callback,
2552 		},
2553 		{
2554 			.name		= "format",
2555 			.callback	= event_callback,
2556 		},
2557 #ifdef CONFIG_PERF_EVENTS
2558 		{
2559 			.name		= "id",
2560 			.callback	= event_callback,
2561 		},
2562 #endif
2563 #ifdef CONFIG_HIST_TRIGGERS
2564 		{
2565 			.name		= "hist",
2566 			.callback	= event_callback,
2567 		},
2568 #endif
2569 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
2570 		{
2571 			.name		= "hist_debug",
2572 			.callback	= event_callback,
2573 		},
2574 #endif
2575 #ifdef CONFIG_TRACE_EVENT_INJECT
2576 		{
2577 			.name		= "inject",
2578 			.callback	= event_callback,
2579 		},
2580 #endif
2581 	};
2582 
2583 	/*
2584 	 * If the trace point header did not define TRACE_SYSTEM
2585 	 * then the system would be called "TRACE_SYSTEM". This should
2586 	 * never happen.
2587 	 */
2588 	if (WARN_ON_ONCE(strcmp(call->class->system, TRACE_SYSTEM) == 0))
2589 		return -ENODEV;
2590 
2591 	e_events = event_subsystem_dir(tr, call->class->system, file, parent);
2592 	if (!e_events)
2593 		return -ENOMEM;
2594 
2595 	nr_entries = ARRAY_SIZE(event_entries);
2596 
2597 	name = trace_event_name(call);
2598 	ei = eventfs_create_dir(name, e_events, event_entries, nr_entries, file);
2599 	if (IS_ERR(ei)) {
2600 		pr_warn("Could not create tracefs '%s' directory\n", name);
2601 		return -1;
2602 	}
2603 
2604 	file->ei = ei;
2605 
2606 	ret = event_define_fields(call);
2607 	if (ret < 0) {
2608 		pr_warn("Could not initialize trace point events/%s\n", name);
2609 		return ret;
2610 	}
2611 
2612 	/* Gets decremented on freeing of the "enable" file */
2613 	event_file_get(file);
2614 
2615 	return 0;
2616 }
2617 
2618 static void remove_event_from_tracers(struct trace_event_call *call)
2619 {
2620 	struct trace_event_file *file;
2621 	struct trace_array *tr;
2622 
2623 	do_for_each_event_file_safe(tr, file) {
2624 		if (file->event_call != call)
2625 			continue;
2626 
2627 		remove_event_file_dir(file);
2628 		/*
2629 		 * The do_for_each_event_file_safe() is
2630 		 * a double loop. After finding the call for this
2631 		 * trace_array, we use break to jump to the next
2632 		 * trace_array.
2633 		 */
2634 		break;
2635 	} while_for_each_event_file();
2636 }
2637 
2638 static void event_remove(struct trace_event_call *call)
2639 {
2640 	struct trace_array *tr;
2641 	struct trace_event_file *file;
2642 
2643 	do_for_each_event_file(tr, file) {
2644 		if (file->event_call != call)
2645 			continue;
2646 
2647 		if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
2648 			tr->clear_trace = true;
2649 
2650 		ftrace_event_enable_disable(file, 0);
2651 		/*
2652 		 * The do_for_each_event_file() is
2653 		 * a double loop. After finding the call for this
2654 		 * trace_array, we use break to jump to the next
2655 		 * trace_array.
2656 		 */
2657 		break;
2658 	} while_for_each_event_file();
2659 
2660 	if (call->event.funcs)
2661 		__unregister_trace_event(&call->event);
2662 	remove_event_from_tracers(call);
2663 	list_del(&call->list);
2664 }
2665 
2666 static int event_init(struct trace_event_call *call)
2667 {
2668 	int ret = 0;
2669 	const char *name;
2670 
2671 	name = trace_event_name(call);
2672 	if (WARN_ON(!name))
2673 		return -EINVAL;
2674 
2675 	if (call->class->raw_init) {
2676 		ret = call->class->raw_init(call);
2677 		if (ret < 0 && ret != -ENOSYS)
2678 			pr_warn("Could not initialize trace events/%s\n", name);
2679 	}
2680 
2681 	return ret;
2682 }
2683 
2684 static int
2685 __register_event(struct trace_event_call *call, struct module *mod)
2686 {
2687 	int ret;
2688 
2689 	ret = event_init(call);
2690 	if (ret < 0)
2691 		return ret;
2692 
2693 	list_add(&call->list, &ftrace_events);
2694 	if (call->flags & TRACE_EVENT_FL_DYNAMIC)
2695 		atomic_set(&call->refcnt, 0);
2696 	else
2697 		call->module = mod;
2698 
2699 	return 0;
2700 }
2701 
2702 static char *eval_replace(char *ptr, struct trace_eval_map *map, int len)
2703 {
2704 	int rlen;
2705 	int elen;
2706 
2707 	/* Find the length of the eval value as a string */
2708 	elen = snprintf(ptr, 0, "%ld", map->eval_value);
2709 	/* Make sure there's enough room to replace the string with the value */
2710 	if (len < elen)
2711 		return NULL;
2712 
2713 	snprintf(ptr, elen + 1, "%ld", map->eval_value);
2714 
2715 	/* Get the rest of the string of ptr */
2716 	rlen = strlen(ptr + len);
2717 	memmove(ptr + elen, ptr + len, rlen);
2718 	/* Make sure we end the new string */
2719 	ptr[elen + rlen] = 0;
2720 
2721 	return ptr + elen;
2722 }
2723 
2724 static void update_event_printk(struct trace_event_call *call,
2725 				struct trace_eval_map *map)
2726 {
2727 	char *ptr;
2728 	int quote = 0;
2729 	int len = strlen(map->eval_string);
2730 
2731 	for (ptr = call->print_fmt; *ptr; ptr++) {
2732 		if (*ptr == '\\') {
2733 			ptr++;
2734 			/* paranoid */
2735 			if (!*ptr)
2736 				break;
2737 			continue;
2738 		}
2739 		if (*ptr == '"') {
2740 			quote ^= 1;
2741 			continue;
2742 		}
2743 		if (quote)
2744 			continue;
2745 		if (isdigit(*ptr)) {
2746 			/* skip numbers */
2747 			do {
2748 				ptr++;
2749 				/* Check for alpha chars like ULL */
2750 			} while (isalnum(*ptr));
2751 			if (!*ptr)
2752 				break;
2753 			/*
2754 			 * A number must have some kind of delimiter after
2755 			 * it, and we can ignore that too.
2756 			 */
2757 			continue;
2758 		}
2759 		if (isalpha(*ptr) || *ptr == '_') {
2760 			if (strncmp(map->eval_string, ptr, len) == 0 &&
2761 			    !isalnum(ptr[len]) && ptr[len] != '_') {
2762 				ptr = eval_replace(ptr, map, len);
2763 				/* enum/sizeof string smaller than value */
2764 				if (WARN_ON_ONCE(!ptr))
2765 					return;
2766 				/*
2767 				 * No need to decrement here, as eval_replace()
2768 				 * returns the pointer to the character passed
2769 				 * the eval, and two evals can not be placed
2770 				 * back to back without something in between.
2771 				 * We can skip that something in between.
2772 				 */
2773 				continue;
2774 			}
2775 		skip_more:
2776 			do {
2777 				ptr++;
2778 			} while (isalnum(*ptr) || *ptr == '_');
2779 			if (!*ptr)
2780 				break;
2781 			/*
2782 			 * If what comes after this variable is a '.' or
2783 			 * '->' then we can continue to ignore that string.
2784 			 */
2785 			if (*ptr == '.' || (ptr[0] == '-' && ptr[1] == '>')) {
2786 				ptr += *ptr == '.' ? 1 : 2;
2787 				if (!*ptr)
2788 					break;
2789 				goto skip_more;
2790 			}
2791 			/*
2792 			 * Once again, we can skip the delimiter that came
2793 			 * after the string.
2794 			 */
2795 			continue;
2796 		}
2797 	}
2798 }
2799 
2800 static void add_str_to_module(struct module *module, char *str)
2801 {
2802 	struct module_string *modstr;
2803 
2804 	modstr = kmalloc(sizeof(*modstr), GFP_KERNEL);
2805 
2806 	/*
2807 	 * If we failed to allocate memory here, then we'll just
2808 	 * let the str memory leak when the module is removed.
2809 	 * If this fails to allocate, there's worse problems than
2810 	 * a leaked string on module removal.
2811 	 */
2812 	if (WARN_ON_ONCE(!modstr))
2813 		return;
2814 
2815 	modstr->module = module;
2816 	modstr->str = str;
2817 
2818 	list_add(&modstr->next, &module_strings);
2819 }
2820 
2821 static void update_event_fields(struct trace_event_call *call,
2822 				struct trace_eval_map *map)
2823 {
2824 	struct ftrace_event_field *field;
2825 	struct list_head *head;
2826 	char *ptr;
2827 	char *str;
2828 	int len = strlen(map->eval_string);
2829 
2830 	/* Dynamic events should never have field maps */
2831 	if (WARN_ON_ONCE(call->flags & TRACE_EVENT_FL_DYNAMIC))
2832 		return;
2833 
2834 	head = trace_get_fields(call);
2835 	list_for_each_entry(field, head, link) {
2836 		ptr = strchr(field->type, '[');
2837 		if (!ptr)
2838 			continue;
2839 		ptr++;
2840 
2841 		if (!isalpha(*ptr) && *ptr != '_')
2842 			continue;
2843 
2844 		if (strncmp(map->eval_string, ptr, len) != 0)
2845 			continue;
2846 
2847 		str = kstrdup(field->type, GFP_KERNEL);
2848 		if (WARN_ON_ONCE(!str))
2849 			return;
2850 		ptr = str + (ptr - field->type);
2851 		ptr = eval_replace(ptr, map, len);
2852 		/* enum/sizeof string smaller than value */
2853 		if (WARN_ON_ONCE(!ptr)) {
2854 			kfree(str);
2855 			continue;
2856 		}
2857 
2858 		/*
2859 		 * If the event is part of a module, then we need to free the string
2860 		 * when the module is removed. Otherwise, it will stay allocated
2861 		 * until a reboot.
2862 		 */
2863 		if (call->module)
2864 			add_str_to_module(call->module, str);
2865 
2866 		field->type = str;
2867 	}
2868 }
2869 
2870 void trace_event_eval_update(struct trace_eval_map **map, int len)
2871 {
2872 	struct trace_event_call *call, *p;
2873 	const char *last_system = NULL;
2874 	bool first = false;
2875 	int last_i;
2876 	int i;
2877 
2878 	down_write(&trace_event_sem);
2879 	list_for_each_entry_safe(call, p, &ftrace_events, list) {
2880 		/* events are usually grouped together with systems */
2881 		if (!last_system || call->class->system != last_system) {
2882 			first = true;
2883 			last_i = 0;
2884 			last_system = call->class->system;
2885 		}
2886 
2887 		/*
2888 		 * Since calls are grouped by systems, the likelihood that the
2889 		 * next call in the iteration belongs to the same system as the
2890 		 * previous call is high. As an optimization, we skip searching
2891 		 * for a map[] that matches the call's system if the last call
2892 		 * was from the same system. That's what last_i is for. If the
2893 		 * call has the same system as the previous call, then last_i
2894 		 * will be the index of the first map[] that has a matching
2895 		 * system.
2896 		 */
2897 		for (i = last_i; i < len; i++) {
2898 			if (call->class->system == map[i]->system) {
2899 				/* Save the first system if need be */
2900 				if (first) {
2901 					last_i = i;
2902 					first = false;
2903 				}
2904 				update_event_printk(call, map[i]);
2905 				update_event_fields(call, map[i]);
2906 			}
2907 		}
2908 		cond_resched();
2909 	}
2910 	up_write(&trace_event_sem);
2911 }
2912 
2913 static struct trace_event_file *
2914 trace_create_new_event(struct trace_event_call *call,
2915 		       struct trace_array *tr)
2916 {
2917 	struct trace_pid_list *no_pid_list;
2918 	struct trace_pid_list *pid_list;
2919 	struct trace_event_file *file;
2920 	unsigned int first;
2921 
2922 	file = kmem_cache_alloc(file_cachep, GFP_TRACE);
2923 	if (!file)
2924 		return NULL;
2925 
2926 	pid_list = rcu_dereference_protected(tr->filtered_pids,
2927 					     lockdep_is_held(&event_mutex));
2928 	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
2929 					     lockdep_is_held(&event_mutex));
2930 
2931 	if (!trace_pid_list_first(pid_list, &first) ||
2932 	    !trace_pid_list_first(no_pid_list, &first))
2933 		file->flags |= EVENT_FILE_FL_PID_FILTER;
2934 
2935 	file->event_call = call;
2936 	file->tr = tr;
2937 	atomic_set(&file->sm_ref, 0);
2938 	atomic_set(&file->tm_ref, 0);
2939 	INIT_LIST_HEAD(&file->triggers);
2940 	list_add(&file->list, &tr->events);
2941 	event_file_get(file);
2942 
2943 	return file;
2944 }
2945 
2946 #define MAX_BOOT_TRIGGERS 32
2947 
2948 static struct boot_triggers {
2949 	const char		*event;
2950 	char			*trigger;
2951 } bootup_triggers[MAX_BOOT_TRIGGERS];
2952 
2953 static char bootup_trigger_buf[COMMAND_LINE_SIZE];
2954 static int nr_boot_triggers;
2955 
2956 static __init int setup_trace_triggers(char *str)
2957 {
2958 	char *trigger;
2959 	char *buf;
2960 	int i;
2961 
2962 	strscpy(bootup_trigger_buf, str, COMMAND_LINE_SIZE);
2963 	ring_buffer_expanded = true;
2964 	disable_tracing_selftest("running event triggers");
2965 
2966 	buf = bootup_trigger_buf;
2967 	for (i = 0; i < MAX_BOOT_TRIGGERS; i++) {
2968 		trigger = strsep(&buf, ",");
2969 		if (!trigger)
2970 			break;
2971 		bootup_triggers[i].event = strsep(&trigger, ".");
2972 		bootup_triggers[i].trigger = trigger;
2973 		if (!bootup_triggers[i].trigger)
2974 			break;
2975 	}
2976 
2977 	nr_boot_triggers = i;
2978 	return 1;
2979 }
2980 __setup("trace_trigger=", setup_trace_triggers);
2981 
2982 /* Add an event to a trace directory */
2983 static int
2984 __trace_add_new_event(struct trace_event_call *call, struct trace_array *tr)
2985 {
2986 	struct trace_event_file *file;
2987 
2988 	file = trace_create_new_event(call, tr);
2989 	if (!file)
2990 		return -ENOMEM;
2991 
2992 	if (eventdir_initialized)
2993 		return event_create_dir(tr->event_dir, file);
2994 	else
2995 		return event_define_fields(call);
2996 }
2997 
2998 static void trace_early_triggers(struct trace_event_file *file, const char *name)
2999 {
3000 	int ret;
3001 	int i;
3002 
3003 	for (i = 0; i < nr_boot_triggers; i++) {
3004 		if (strcmp(name, bootup_triggers[i].event))
3005 			continue;
3006 		mutex_lock(&event_mutex);
3007 		ret = trigger_process_regex(file, bootup_triggers[i].trigger);
3008 		mutex_unlock(&event_mutex);
3009 		if (ret)
3010 			pr_err("Failed to register trigger '%s' on event %s\n",
3011 			       bootup_triggers[i].trigger,
3012 			       bootup_triggers[i].event);
3013 	}
3014 }
3015 
3016 /*
3017  * Just create a descriptor for early init. A descriptor is required
3018  * for enabling events at boot. We want to enable events before
3019  * the filesystem is initialized.
3020  */
3021 static int
3022 __trace_early_add_new_event(struct trace_event_call *call,
3023 			    struct trace_array *tr)
3024 {
3025 	struct trace_event_file *file;
3026 	int ret;
3027 
3028 	file = trace_create_new_event(call, tr);
3029 	if (!file)
3030 		return -ENOMEM;
3031 
3032 	ret = event_define_fields(call);
3033 	if (ret)
3034 		return ret;
3035 
3036 	trace_early_triggers(file, trace_event_name(call));
3037 
3038 	return 0;
3039 }
3040 
3041 struct ftrace_module_file_ops;
3042 static void __add_event_to_tracers(struct trace_event_call *call);
3043 
3044 /* Add an additional event_call dynamically */
3045 int trace_add_event_call(struct trace_event_call *call)
3046 {
3047 	int ret;
3048 	lockdep_assert_held(&event_mutex);
3049 
3050 	mutex_lock(&trace_types_lock);
3051 
3052 	ret = __register_event(call, NULL);
3053 	if (ret >= 0)
3054 		__add_event_to_tracers(call);
3055 
3056 	mutex_unlock(&trace_types_lock);
3057 	return ret;
3058 }
3059 EXPORT_SYMBOL_GPL(trace_add_event_call);
3060 
3061 /*
3062  * Must be called under locking of trace_types_lock, event_mutex and
3063  * trace_event_sem.
3064  */
3065 static void __trace_remove_event_call(struct trace_event_call *call)
3066 {
3067 	event_remove(call);
3068 	trace_destroy_fields(call);
3069 	free_event_filter(call->filter);
3070 	call->filter = NULL;
3071 }
3072 
3073 static int probe_remove_event_call(struct trace_event_call *call)
3074 {
3075 	struct trace_array *tr;
3076 	struct trace_event_file *file;
3077 
3078 #ifdef CONFIG_PERF_EVENTS
3079 	if (call->perf_refcount)
3080 		return -EBUSY;
3081 #endif
3082 	do_for_each_event_file(tr, file) {
3083 		if (file->event_call != call)
3084 			continue;
3085 		/*
3086 		 * We can't rely on ftrace_event_enable_disable(enable => 0)
3087 		 * we are going to do, EVENT_FILE_FL_SOFT_MODE can suppress
3088 		 * TRACE_REG_UNREGISTER.
3089 		 */
3090 		if (file->flags & EVENT_FILE_FL_ENABLED)
3091 			goto busy;
3092 
3093 		if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
3094 			tr->clear_trace = true;
3095 		/*
3096 		 * The do_for_each_event_file_safe() is
3097 		 * a double loop. After finding the call for this
3098 		 * trace_array, we use break to jump to the next
3099 		 * trace_array.
3100 		 */
3101 		break;
3102 	} while_for_each_event_file();
3103 
3104 	__trace_remove_event_call(call);
3105 
3106 	return 0;
3107  busy:
3108 	/* No need to clear the trace now */
3109 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
3110 		tr->clear_trace = false;
3111 	}
3112 	return -EBUSY;
3113 }
3114 
3115 /* Remove an event_call */
3116 int trace_remove_event_call(struct trace_event_call *call)
3117 {
3118 	int ret;
3119 
3120 	lockdep_assert_held(&event_mutex);
3121 
3122 	mutex_lock(&trace_types_lock);
3123 	down_write(&trace_event_sem);
3124 	ret = probe_remove_event_call(call);
3125 	up_write(&trace_event_sem);
3126 	mutex_unlock(&trace_types_lock);
3127 
3128 	return ret;
3129 }
3130 EXPORT_SYMBOL_GPL(trace_remove_event_call);
3131 
3132 #define for_each_event(event, start, end)			\
3133 	for (event = start;					\
3134 	     (unsigned long)event < (unsigned long)end;		\
3135 	     event++)
3136 
3137 #ifdef CONFIG_MODULES
3138 
3139 static void trace_module_add_events(struct module *mod)
3140 {
3141 	struct trace_event_call **call, **start, **end;
3142 
3143 	if (!mod->num_trace_events)
3144 		return;
3145 
3146 	/* Don't add infrastructure for mods without tracepoints */
3147 	if (trace_module_has_bad_taint(mod)) {
3148 		pr_err("%s: module has bad taint, not creating trace events\n",
3149 		       mod->name);
3150 		return;
3151 	}
3152 
3153 	start = mod->trace_events;
3154 	end = mod->trace_events + mod->num_trace_events;
3155 
3156 	for_each_event(call, start, end) {
3157 		__register_event(*call, mod);
3158 		__add_event_to_tracers(*call);
3159 	}
3160 }
3161 
3162 static void trace_module_remove_events(struct module *mod)
3163 {
3164 	struct trace_event_call *call, *p;
3165 	struct module_string *modstr, *m;
3166 
3167 	down_write(&trace_event_sem);
3168 	list_for_each_entry_safe(call, p, &ftrace_events, list) {
3169 		if ((call->flags & TRACE_EVENT_FL_DYNAMIC) || !call->module)
3170 			continue;
3171 		if (call->module == mod)
3172 			__trace_remove_event_call(call);
3173 	}
3174 	/* Check for any strings allocade for this module */
3175 	list_for_each_entry_safe(modstr, m, &module_strings, next) {
3176 		if (modstr->module != mod)
3177 			continue;
3178 		list_del(&modstr->next);
3179 		kfree(modstr->str);
3180 		kfree(modstr);
3181 	}
3182 	up_write(&trace_event_sem);
3183 
3184 	/*
3185 	 * It is safest to reset the ring buffer if the module being unloaded
3186 	 * registered any events that were used. The only worry is if
3187 	 * a new module gets loaded, and takes on the same id as the events
3188 	 * of this module. When printing out the buffer, traced events left
3189 	 * over from this module may be passed to the new module events and
3190 	 * unexpected results may occur.
3191 	 */
3192 	tracing_reset_all_online_cpus_unlocked();
3193 }
3194 
3195 static int trace_module_notify(struct notifier_block *self,
3196 			       unsigned long val, void *data)
3197 {
3198 	struct module *mod = data;
3199 
3200 	mutex_lock(&event_mutex);
3201 	mutex_lock(&trace_types_lock);
3202 	switch (val) {
3203 	case MODULE_STATE_COMING:
3204 		trace_module_add_events(mod);
3205 		break;
3206 	case MODULE_STATE_GOING:
3207 		trace_module_remove_events(mod);
3208 		break;
3209 	}
3210 	mutex_unlock(&trace_types_lock);
3211 	mutex_unlock(&event_mutex);
3212 
3213 	return NOTIFY_OK;
3214 }
3215 
3216 static struct notifier_block trace_module_nb = {
3217 	.notifier_call = trace_module_notify,
3218 	.priority = 1, /* higher than trace.c module notify */
3219 };
3220 #endif /* CONFIG_MODULES */
3221 
3222 /* Create a new event directory structure for a trace directory. */
3223 static void
3224 __trace_add_event_dirs(struct trace_array *tr)
3225 {
3226 	struct trace_event_call *call;
3227 	int ret;
3228 
3229 	list_for_each_entry(call, &ftrace_events, list) {
3230 		ret = __trace_add_new_event(call, tr);
3231 		if (ret < 0)
3232 			pr_warn("Could not create directory for event %s\n",
3233 				trace_event_name(call));
3234 	}
3235 }
3236 
3237 /* Returns any file that matches the system and event */
3238 struct trace_event_file *
3239 __find_event_file(struct trace_array *tr, const char *system, const char *event)
3240 {
3241 	struct trace_event_file *file;
3242 	struct trace_event_call *call;
3243 	const char *name;
3244 
3245 	list_for_each_entry(file, &tr->events, list) {
3246 
3247 		call = file->event_call;
3248 		name = trace_event_name(call);
3249 
3250 		if (!name || !call->class)
3251 			continue;
3252 
3253 		if (strcmp(event, name) == 0 &&
3254 		    strcmp(system, call->class->system) == 0)
3255 			return file;
3256 	}
3257 	return NULL;
3258 }
3259 
3260 /* Returns valid trace event files that match system and event */
3261 struct trace_event_file *
3262 find_event_file(struct trace_array *tr, const char *system, const char *event)
3263 {
3264 	struct trace_event_file *file;
3265 
3266 	file = __find_event_file(tr, system, event);
3267 	if (!file || !file->event_call->class->reg ||
3268 	    file->event_call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
3269 		return NULL;
3270 
3271 	return file;
3272 }
3273 
3274 /**
3275  * trace_get_event_file - Find and return a trace event file
3276  * @instance: The name of the trace instance containing the event
3277  * @system: The name of the system containing the event
3278  * @event: The name of the event
3279  *
3280  * Return a trace event file given the trace instance name, trace
3281  * system, and trace event name.  If the instance name is NULL, it
3282  * refers to the top-level trace array.
3283  *
3284  * This function will look it up and return it if found, after calling
3285  * trace_array_get() to prevent the instance from going away, and
3286  * increment the event's module refcount to prevent it from being
3287  * removed.
3288  *
3289  * To release the file, call trace_put_event_file(), which will call
3290  * trace_array_put() and decrement the event's module refcount.
3291  *
3292  * Return: The trace event on success, ERR_PTR otherwise.
3293  */
3294 struct trace_event_file *trace_get_event_file(const char *instance,
3295 					      const char *system,
3296 					      const char *event)
3297 {
3298 	struct trace_array *tr = top_trace_array();
3299 	struct trace_event_file *file = NULL;
3300 	int ret = -EINVAL;
3301 
3302 	if (instance) {
3303 		tr = trace_array_find_get(instance);
3304 		if (!tr)
3305 			return ERR_PTR(-ENOENT);
3306 	} else {
3307 		ret = trace_array_get(tr);
3308 		if (ret)
3309 			return ERR_PTR(ret);
3310 	}
3311 
3312 	mutex_lock(&event_mutex);
3313 
3314 	file = find_event_file(tr, system, event);
3315 	if (!file) {
3316 		trace_array_put(tr);
3317 		ret = -EINVAL;
3318 		goto out;
3319 	}
3320 
3321 	/* Don't let event modules unload while in use */
3322 	ret = trace_event_try_get_ref(file->event_call);
3323 	if (!ret) {
3324 		trace_array_put(tr);
3325 		ret = -EBUSY;
3326 		goto out;
3327 	}
3328 
3329 	ret = 0;
3330  out:
3331 	mutex_unlock(&event_mutex);
3332 
3333 	if (ret)
3334 		file = ERR_PTR(ret);
3335 
3336 	return file;
3337 }
3338 EXPORT_SYMBOL_GPL(trace_get_event_file);
3339 
3340 /**
3341  * trace_put_event_file - Release a file from trace_get_event_file()
3342  * @file: The trace event file
3343  *
3344  * If a file was retrieved using trace_get_event_file(), this should
3345  * be called when it's no longer needed.  It will cancel the previous
3346  * trace_array_get() called by that function, and decrement the
3347  * event's module refcount.
3348  */
3349 void trace_put_event_file(struct trace_event_file *file)
3350 {
3351 	mutex_lock(&event_mutex);
3352 	trace_event_put_ref(file->event_call);
3353 	mutex_unlock(&event_mutex);
3354 
3355 	trace_array_put(file->tr);
3356 }
3357 EXPORT_SYMBOL_GPL(trace_put_event_file);
3358 
3359 #ifdef CONFIG_DYNAMIC_FTRACE
3360 
3361 /* Avoid typos */
3362 #define ENABLE_EVENT_STR	"enable_event"
3363 #define DISABLE_EVENT_STR	"disable_event"
3364 
3365 struct event_probe_data {
3366 	struct trace_event_file	*file;
3367 	unsigned long			count;
3368 	int				ref;
3369 	bool				enable;
3370 };
3371 
3372 static void update_event_probe(struct event_probe_data *data)
3373 {
3374 	if (data->enable)
3375 		clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
3376 	else
3377 		set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
3378 }
3379 
3380 static void
3381 event_enable_probe(unsigned long ip, unsigned long parent_ip,
3382 		   struct trace_array *tr, struct ftrace_probe_ops *ops,
3383 		   void *data)
3384 {
3385 	struct ftrace_func_mapper *mapper = data;
3386 	struct event_probe_data *edata;
3387 	void **pdata;
3388 
3389 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
3390 	if (!pdata || !*pdata)
3391 		return;
3392 
3393 	edata = *pdata;
3394 	update_event_probe(edata);
3395 }
3396 
3397 static void
3398 event_enable_count_probe(unsigned long ip, unsigned long parent_ip,
3399 			 struct trace_array *tr, struct ftrace_probe_ops *ops,
3400 			 void *data)
3401 {
3402 	struct ftrace_func_mapper *mapper = data;
3403 	struct event_probe_data *edata;
3404 	void **pdata;
3405 
3406 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
3407 	if (!pdata || !*pdata)
3408 		return;
3409 
3410 	edata = *pdata;
3411 
3412 	if (!edata->count)
3413 		return;
3414 
3415 	/* Skip if the event is in a state we want to switch to */
3416 	if (edata->enable == !(edata->file->flags & EVENT_FILE_FL_SOFT_DISABLED))
3417 		return;
3418 
3419 	if (edata->count != -1)
3420 		(edata->count)--;
3421 
3422 	update_event_probe(edata);
3423 }
3424 
3425 static int
3426 event_enable_print(struct seq_file *m, unsigned long ip,
3427 		   struct ftrace_probe_ops *ops, void *data)
3428 {
3429 	struct ftrace_func_mapper *mapper = data;
3430 	struct event_probe_data *edata;
3431 	void **pdata;
3432 
3433 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
3434 
3435 	if (WARN_ON_ONCE(!pdata || !*pdata))
3436 		return 0;
3437 
3438 	edata = *pdata;
3439 
3440 	seq_printf(m, "%ps:", (void *)ip);
3441 
3442 	seq_printf(m, "%s:%s:%s",
3443 		   edata->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
3444 		   edata->file->event_call->class->system,
3445 		   trace_event_name(edata->file->event_call));
3446 
3447 	if (edata->count == -1)
3448 		seq_puts(m, ":unlimited\n");
3449 	else
3450 		seq_printf(m, ":count=%ld\n", edata->count);
3451 
3452 	return 0;
3453 }
3454 
3455 static int
3456 event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
3457 		  unsigned long ip, void *init_data, void **data)
3458 {
3459 	struct ftrace_func_mapper *mapper = *data;
3460 	struct event_probe_data *edata = init_data;
3461 	int ret;
3462 
3463 	if (!mapper) {
3464 		mapper = allocate_ftrace_func_mapper();
3465 		if (!mapper)
3466 			return -ENODEV;
3467 		*data = mapper;
3468 	}
3469 
3470 	ret = ftrace_func_mapper_add_ip(mapper, ip, edata);
3471 	if (ret < 0)
3472 		return ret;
3473 
3474 	edata->ref++;
3475 
3476 	return 0;
3477 }
3478 
3479 static int free_probe_data(void *data)
3480 {
3481 	struct event_probe_data *edata = data;
3482 
3483 	edata->ref--;
3484 	if (!edata->ref) {
3485 		/* Remove the SOFT_MODE flag */
3486 		__ftrace_event_enable_disable(edata->file, 0, 1);
3487 		trace_event_put_ref(edata->file->event_call);
3488 		kfree(edata);
3489 	}
3490 	return 0;
3491 }
3492 
3493 static void
3494 event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
3495 		  unsigned long ip, void *data)
3496 {
3497 	struct ftrace_func_mapper *mapper = data;
3498 	struct event_probe_data *edata;
3499 
3500 	if (!ip) {
3501 		if (!mapper)
3502 			return;
3503 		free_ftrace_func_mapper(mapper, free_probe_data);
3504 		return;
3505 	}
3506 
3507 	edata = ftrace_func_mapper_remove_ip(mapper, ip);
3508 
3509 	if (WARN_ON_ONCE(!edata))
3510 		return;
3511 
3512 	if (WARN_ON_ONCE(edata->ref <= 0))
3513 		return;
3514 
3515 	free_probe_data(edata);
3516 }
3517 
3518 static struct ftrace_probe_ops event_enable_probe_ops = {
3519 	.func			= event_enable_probe,
3520 	.print			= event_enable_print,
3521 	.init			= event_enable_init,
3522 	.free			= event_enable_free,
3523 };
3524 
3525 static struct ftrace_probe_ops event_enable_count_probe_ops = {
3526 	.func			= event_enable_count_probe,
3527 	.print			= event_enable_print,
3528 	.init			= event_enable_init,
3529 	.free			= event_enable_free,
3530 };
3531 
3532 static struct ftrace_probe_ops event_disable_probe_ops = {
3533 	.func			= event_enable_probe,
3534 	.print			= event_enable_print,
3535 	.init			= event_enable_init,
3536 	.free			= event_enable_free,
3537 };
3538 
3539 static struct ftrace_probe_ops event_disable_count_probe_ops = {
3540 	.func			= event_enable_count_probe,
3541 	.print			= event_enable_print,
3542 	.init			= event_enable_init,
3543 	.free			= event_enable_free,
3544 };
3545 
3546 static int
3547 event_enable_func(struct trace_array *tr, struct ftrace_hash *hash,
3548 		  char *glob, char *cmd, char *param, int enabled)
3549 {
3550 	struct trace_event_file *file;
3551 	struct ftrace_probe_ops *ops;
3552 	struct event_probe_data *data;
3553 	const char *system;
3554 	const char *event;
3555 	char *number;
3556 	bool enable;
3557 	int ret;
3558 
3559 	if (!tr)
3560 		return -ENODEV;
3561 
3562 	/* hash funcs only work with set_ftrace_filter */
3563 	if (!enabled || !param)
3564 		return -EINVAL;
3565 
3566 	system = strsep(&param, ":");
3567 	if (!param)
3568 		return -EINVAL;
3569 
3570 	event = strsep(&param, ":");
3571 
3572 	mutex_lock(&event_mutex);
3573 
3574 	ret = -EINVAL;
3575 	file = find_event_file(tr, system, event);
3576 	if (!file)
3577 		goto out;
3578 
3579 	enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
3580 
3581 	if (enable)
3582 		ops = param ? &event_enable_count_probe_ops : &event_enable_probe_ops;
3583 	else
3584 		ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops;
3585 
3586 	if (glob[0] == '!') {
3587 		ret = unregister_ftrace_function_probe_func(glob+1, tr, ops);
3588 		goto out;
3589 	}
3590 
3591 	ret = -ENOMEM;
3592 
3593 	data = kzalloc(sizeof(*data), GFP_KERNEL);
3594 	if (!data)
3595 		goto out;
3596 
3597 	data->enable = enable;
3598 	data->count = -1;
3599 	data->file = file;
3600 
3601 	if (!param)
3602 		goto out_reg;
3603 
3604 	number = strsep(&param, ":");
3605 
3606 	ret = -EINVAL;
3607 	if (!strlen(number))
3608 		goto out_free;
3609 
3610 	/*
3611 	 * We use the callback data field (which is a pointer)
3612 	 * as our counter.
3613 	 */
3614 	ret = kstrtoul(number, 0, &data->count);
3615 	if (ret)
3616 		goto out_free;
3617 
3618  out_reg:
3619 	/* Don't let event modules unload while probe registered */
3620 	ret = trace_event_try_get_ref(file->event_call);
3621 	if (!ret) {
3622 		ret = -EBUSY;
3623 		goto out_free;
3624 	}
3625 
3626 	ret = __ftrace_event_enable_disable(file, 1, 1);
3627 	if (ret < 0)
3628 		goto out_put;
3629 
3630 	ret = register_ftrace_function_probe(glob, tr, ops, data);
3631 	/*
3632 	 * The above returns on success the # of functions enabled,
3633 	 * but if it didn't find any functions it returns zero.
3634 	 * Consider no functions a failure too.
3635 	 */
3636 	if (!ret) {
3637 		ret = -ENOENT;
3638 		goto out_disable;
3639 	} else if (ret < 0)
3640 		goto out_disable;
3641 	/* Just return zero, not the number of enabled functions */
3642 	ret = 0;
3643  out:
3644 	mutex_unlock(&event_mutex);
3645 	return ret;
3646 
3647  out_disable:
3648 	__ftrace_event_enable_disable(file, 0, 1);
3649  out_put:
3650 	trace_event_put_ref(file->event_call);
3651  out_free:
3652 	kfree(data);
3653 	goto out;
3654 }
3655 
3656 static struct ftrace_func_command event_enable_cmd = {
3657 	.name			= ENABLE_EVENT_STR,
3658 	.func			= event_enable_func,
3659 };
3660 
3661 static struct ftrace_func_command event_disable_cmd = {
3662 	.name			= DISABLE_EVENT_STR,
3663 	.func			= event_enable_func,
3664 };
3665 
3666 static __init int register_event_cmds(void)
3667 {
3668 	int ret;
3669 
3670 	ret = register_ftrace_command(&event_enable_cmd);
3671 	if (WARN_ON(ret < 0))
3672 		return ret;
3673 	ret = register_ftrace_command(&event_disable_cmd);
3674 	if (WARN_ON(ret < 0))
3675 		unregister_ftrace_command(&event_enable_cmd);
3676 	return ret;
3677 }
3678 #else
3679 static inline int register_event_cmds(void) { return 0; }
3680 #endif /* CONFIG_DYNAMIC_FTRACE */
3681 
3682 /*
3683  * The top level array and trace arrays created by boot-time tracing
3684  * have already had its trace_event_file descriptors created in order
3685  * to allow for early events to be recorded.
3686  * This function is called after the tracefs has been initialized,
3687  * and we now have to create the files associated to the events.
3688  */
3689 static void __trace_early_add_event_dirs(struct trace_array *tr)
3690 {
3691 	struct trace_event_file *file;
3692 	int ret;
3693 
3694 
3695 	list_for_each_entry(file, &tr->events, list) {
3696 		ret = event_create_dir(tr->event_dir, file);
3697 		if (ret < 0)
3698 			pr_warn("Could not create directory for event %s\n",
3699 				trace_event_name(file->event_call));
3700 	}
3701 }
3702 
3703 /*
3704  * For early boot up, the top trace array and the trace arrays created
3705  * by boot-time tracing require to have a list of events that can be
3706  * enabled. This must be done before the filesystem is set up in order
3707  * to allow events to be traced early.
3708  */
3709 void __trace_early_add_events(struct trace_array *tr)
3710 {
3711 	struct trace_event_call *call;
3712 	int ret;
3713 
3714 	list_for_each_entry(call, &ftrace_events, list) {
3715 		/* Early boot up should not have any modules loaded */
3716 		if (!(call->flags & TRACE_EVENT_FL_DYNAMIC) &&
3717 		    WARN_ON_ONCE(call->module))
3718 			continue;
3719 
3720 		ret = __trace_early_add_new_event(call, tr);
3721 		if (ret < 0)
3722 			pr_warn("Could not create early event %s\n",
3723 				trace_event_name(call));
3724 	}
3725 }
3726 
3727 /* Remove the event directory structure for a trace directory. */
3728 static void
3729 __trace_remove_event_dirs(struct trace_array *tr)
3730 {
3731 	struct trace_event_file *file, *next;
3732 
3733 	list_for_each_entry_safe(file, next, &tr->events, list)
3734 		remove_event_file_dir(file);
3735 }
3736 
3737 static void __add_event_to_tracers(struct trace_event_call *call)
3738 {
3739 	struct trace_array *tr;
3740 
3741 	list_for_each_entry(tr, &ftrace_trace_arrays, list)
3742 		__trace_add_new_event(call, tr);
3743 }
3744 
3745 extern struct trace_event_call *__start_ftrace_events[];
3746 extern struct trace_event_call *__stop_ftrace_events[];
3747 
3748 static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata;
3749 
3750 static __init int setup_trace_event(char *str)
3751 {
3752 	strscpy(bootup_event_buf, str, COMMAND_LINE_SIZE);
3753 	ring_buffer_expanded = true;
3754 	disable_tracing_selftest("running event tracing");
3755 
3756 	return 1;
3757 }
3758 __setup("trace_event=", setup_trace_event);
3759 
3760 static int events_callback(const char *name, umode_t *mode, void **data,
3761 			   const struct file_operations **fops)
3762 {
3763 	if (strcmp(name, "enable") == 0) {
3764 		*mode = TRACE_MODE_WRITE;
3765 		*fops = &ftrace_tr_enable_fops;
3766 		return 1;
3767 	}
3768 
3769 	if (strcmp(name, "header_page") == 0)
3770 		*data = ring_buffer_print_page_header;
3771 
3772 	else if (strcmp(name, "header_event") == 0)
3773 		*data = ring_buffer_print_entry_header;
3774 
3775 	else
3776 		return 0;
3777 
3778 	*mode = TRACE_MODE_READ;
3779 	*fops = &ftrace_show_header_fops;
3780 	return 1;
3781 }
3782 
3783 /* Expects to have event_mutex held when called */
3784 static int
3785 create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
3786 {
3787 	struct eventfs_inode *e_events;
3788 	struct dentry *entry;
3789 	int nr_entries;
3790 	static struct eventfs_entry events_entries[] = {
3791 		{
3792 			.name		= "enable",
3793 			.callback	= events_callback,
3794 		},
3795 		{
3796 			.name		= "header_page",
3797 			.callback	= events_callback,
3798 		},
3799 		{
3800 			.name		= "header_event",
3801 			.callback	= events_callback,
3802 		},
3803 	};
3804 
3805 	entry = trace_create_file("set_event", TRACE_MODE_WRITE, parent,
3806 				  tr, &ftrace_set_event_fops);
3807 	if (!entry)
3808 		return -ENOMEM;
3809 
3810 	nr_entries = ARRAY_SIZE(events_entries);
3811 
3812 	e_events = eventfs_create_events_dir("events", parent, events_entries,
3813 					     nr_entries, tr);
3814 	if (IS_ERR(e_events)) {
3815 		pr_warn("Could not create tracefs 'events' directory\n");
3816 		return -ENOMEM;
3817 	}
3818 
3819 	/* There are not as crucial, just warn if they are not created */
3820 
3821 	trace_create_file("set_event_pid", TRACE_MODE_WRITE, parent,
3822 			  tr, &ftrace_set_event_pid_fops);
3823 
3824 	trace_create_file("set_event_notrace_pid",
3825 			  TRACE_MODE_WRITE, parent, tr,
3826 			  &ftrace_set_event_notrace_pid_fops);
3827 
3828 	tr->event_dir = e_events;
3829 
3830 	return 0;
3831 }
3832 
3833 /**
3834  * event_trace_add_tracer - add a instance of a trace_array to events
3835  * @parent: The parent dentry to place the files/directories for events in
3836  * @tr: The trace array associated with these events
3837  *
3838  * When a new instance is created, it needs to set up its events
3839  * directory, as well as other files associated with events. It also
3840  * creates the event hierarchy in the @parent/events directory.
3841  *
3842  * Returns 0 on success.
3843  *
3844  * Must be called with event_mutex held.
3845  */
3846 int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr)
3847 {
3848 	int ret;
3849 
3850 	lockdep_assert_held(&event_mutex);
3851 
3852 	ret = create_event_toplevel_files(parent, tr);
3853 	if (ret)
3854 		goto out;
3855 
3856 	down_write(&trace_event_sem);
3857 	/* If tr already has the event list, it is initialized in early boot. */
3858 	if (unlikely(!list_empty(&tr->events)))
3859 		__trace_early_add_event_dirs(tr);
3860 	else
3861 		__trace_add_event_dirs(tr);
3862 	up_write(&trace_event_sem);
3863 
3864  out:
3865 	return ret;
3866 }
3867 
3868 /*
3869  * The top trace array already had its file descriptors created.
3870  * Now the files themselves need to be created.
3871  */
3872 static __init int
3873 early_event_add_tracer(struct dentry *parent, struct trace_array *tr)
3874 {
3875 	int ret;
3876 
3877 	mutex_lock(&event_mutex);
3878 
3879 	ret = create_event_toplevel_files(parent, tr);
3880 	if (ret)
3881 		goto out_unlock;
3882 
3883 	down_write(&trace_event_sem);
3884 	__trace_early_add_event_dirs(tr);
3885 	up_write(&trace_event_sem);
3886 
3887  out_unlock:
3888 	mutex_unlock(&event_mutex);
3889 
3890 	return ret;
3891 }
3892 
3893 /* Must be called with event_mutex held */
3894 int event_trace_del_tracer(struct trace_array *tr)
3895 {
3896 	lockdep_assert_held(&event_mutex);
3897 
3898 	/* Disable any event triggers and associated soft-disabled events */
3899 	clear_event_triggers(tr);
3900 
3901 	/* Clear the pid list */
3902 	__ftrace_clear_event_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);
3903 
3904 	/* Disable any running events */
3905 	__ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0);
3906 
3907 	/* Make sure no more events are being executed */
3908 	tracepoint_synchronize_unregister();
3909 
3910 	down_write(&trace_event_sem);
3911 	__trace_remove_event_dirs(tr);
3912 	eventfs_remove_events_dir(tr->event_dir);
3913 	up_write(&trace_event_sem);
3914 
3915 	tr->event_dir = NULL;
3916 
3917 	return 0;
3918 }
3919 
3920 static __init int event_trace_memsetup(void)
3921 {
3922 	field_cachep = KMEM_CACHE(ftrace_event_field, SLAB_PANIC);
3923 	file_cachep = KMEM_CACHE(trace_event_file, SLAB_PANIC);
3924 	return 0;
3925 }
3926 
3927 __init void
3928 early_enable_events(struct trace_array *tr, char *buf, bool disable_first)
3929 {
3930 	char *token;
3931 	int ret;
3932 
3933 	while (true) {
3934 		token = strsep(&buf, ",");
3935 
3936 		if (!token)
3937 			break;
3938 
3939 		if (*token) {
3940 			/* Restarting syscalls requires that we stop them first */
3941 			if (disable_first)
3942 				ftrace_set_clr_event(tr, token, 0);
3943 
3944 			ret = ftrace_set_clr_event(tr, token, 1);
3945 			if (ret)
3946 				pr_warn("Failed to enable trace event: %s\n", token);
3947 		}
3948 
3949 		/* Put back the comma to allow this to be called again */
3950 		if (buf)
3951 			*(buf - 1) = ',';
3952 	}
3953 }
3954 
3955 static __init int event_trace_enable(void)
3956 {
3957 	struct trace_array *tr = top_trace_array();
3958 	struct trace_event_call **iter, *call;
3959 	int ret;
3960 
3961 	if (!tr)
3962 		return -ENODEV;
3963 
3964 	for_each_event(iter, __start_ftrace_events, __stop_ftrace_events) {
3965 
3966 		call = *iter;
3967 		ret = event_init(call);
3968 		if (!ret)
3969 			list_add(&call->list, &ftrace_events);
3970 	}
3971 
3972 	register_trigger_cmds();
3973 
3974 	/*
3975 	 * We need the top trace array to have a working set of trace
3976 	 * points at early init, before the debug files and directories
3977 	 * are created. Create the file entries now, and attach them
3978 	 * to the actual file dentries later.
3979 	 */
3980 	__trace_early_add_events(tr);
3981 
3982 	early_enable_events(tr, bootup_event_buf, false);
3983 
3984 	trace_printk_start_comm();
3985 
3986 	register_event_cmds();
3987 
3988 
3989 	return 0;
3990 }
3991 
3992 /*
3993  * event_trace_enable() is called from trace_event_init() first to
3994  * initialize events and perhaps start any events that are on the
3995  * command line. Unfortunately, there are some events that will not
3996  * start this early, like the system call tracepoints that need
3997  * to set the %SYSCALL_WORK_SYSCALL_TRACEPOINT flag of pid 1. But
3998  * event_trace_enable() is called before pid 1 starts, and this flag
3999  * is never set, making the syscall tracepoint never get reached, but
4000  * the event is enabled regardless (and not doing anything).
4001  */
4002 static __init int event_trace_enable_again(void)
4003 {
4004 	struct trace_array *tr;
4005 
4006 	tr = top_trace_array();
4007 	if (!tr)
4008 		return -ENODEV;
4009 
4010 	early_enable_events(tr, bootup_event_buf, true);
4011 
4012 	return 0;
4013 }
4014 
4015 early_initcall(event_trace_enable_again);
4016 
4017 /* Init fields which doesn't related to the tracefs */
4018 static __init int event_trace_init_fields(void)
4019 {
4020 	if (trace_define_generic_fields())
4021 		pr_warn("tracing: Failed to allocated generic fields");
4022 
4023 	if (trace_define_common_fields())
4024 		pr_warn("tracing: Failed to allocate common fields");
4025 
4026 	return 0;
4027 }
4028 
4029 __init int event_trace_init(void)
4030 {
4031 	struct trace_array *tr;
4032 	int ret;
4033 
4034 	tr = top_trace_array();
4035 	if (!tr)
4036 		return -ENODEV;
4037 
4038 	trace_create_file("available_events", TRACE_MODE_READ,
4039 			  NULL, tr, &ftrace_avail_fops);
4040 
4041 	ret = early_event_add_tracer(NULL, tr);
4042 	if (ret)
4043 		return ret;
4044 
4045 #ifdef CONFIG_MODULES
4046 	ret = register_module_notifier(&trace_module_nb);
4047 	if (ret)
4048 		pr_warn("Failed to register trace events module notifier\n");
4049 #endif
4050 
4051 	eventdir_initialized = true;
4052 
4053 	return 0;
4054 }
4055 
4056 void __init trace_event_init(void)
4057 {
4058 	event_trace_memsetup();
4059 	init_ftrace_syscalls();
4060 	event_trace_enable();
4061 	event_trace_init_fields();
4062 }
4063 
4064 #ifdef CONFIG_EVENT_TRACE_STARTUP_TEST
4065 
4066 static DEFINE_SPINLOCK(test_spinlock);
4067 static DEFINE_SPINLOCK(test_spinlock_irq);
4068 static DEFINE_MUTEX(test_mutex);
4069 
4070 static __init void test_work(struct work_struct *dummy)
4071 {
4072 	spin_lock(&test_spinlock);
4073 	spin_lock_irq(&test_spinlock_irq);
4074 	udelay(1);
4075 	spin_unlock_irq(&test_spinlock_irq);
4076 	spin_unlock(&test_spinlock);
4077 
4078 	mutex_lock(&test_mutex);
4079 	msleep(1);
4080 	mutex_unlock(&test_mutex);
4081 }
4082 
4083 static __init int event_test_thread(void *unused)
4084 {
4085 	void *test_malloc;
4086 
4087 	test_malloc = kmalloc(1234, GFP_KERNEL);
4088 	if (!test_malloc)
4089 		pr_info("failed to kmalloc\n");
4090 
4091 	schedule_on_each_cpu(test_work);
4092 
4093 	kfree(test_malloc);
4094 
4095 	set_current_state(TASK_INTERRUPTIBLE);
4096 	while (!kthread_should_stop()) {
4097 		schedule();
4098 		set_current_state(TASK_INTERRUPTIBLE);
4099 	}
4100 	__set_current_state(TASK_RUNNING);
4101 
4102 	return 0;
4103 }
4104 
4105 /*
4106  * Do various things that may trigger events.
4107  */
4108 static __init void event_test_stuff(void)
4109 {
4110 	struct task_struct *test_thread;
4111 
4112 	test_thread = kthread_run(event_test_thread, NULL, "test-events");
4113 	msleep(1);
4114 	kthread_stop(test_thread);
4115 }
4116 
4117 /*
4118  * For every trace event defined, we will test each trace point separately,
4119  * and then by groups, and finally all trace points.
4120  */
4121 static __init void event_trace_self_tests(void)
4122 {
4123 	struct trace_subsystem_dir *dir;
4124 	struct trace_event_file *file;
4125 	struct trace_event_call *call;
4126 	struct event_subsystem *system;
4127 	struct trace_array *tr;
4128 	int ret;
4129 
4130 	tr = top_trace_array();
4131 	if (!tr)
4132 		return;
4133 
4134 	pr_info("Running tests on trace events:\n");
4135 
4136 	list_for_each_entry(file, &tr->events, list) {
4137 
4138 		call = file->event_call;
4139 
4140 		/* Only test those that have a probe */
4141 		if (!call->class || !call->class->probe)
4142 			continue;
4143 
4144 /*
4145  * Testing syscall events here is pretty useless, but
4146  * we still do it if configured. But this is time consuming.
4147  * What we really need is a user thread to perform the
4148  * syscalls as we test.
4149  */
4150 #ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS
4151 		if (call->class->system &&
4152 		    strcmp(call->class->system, "syscalls") == 0)
4153 			continue;
4154 #endif
4155 
4156 		pr_info("Testing event %s: ", trace_event_name(call));
4157 
4158 		/*
4159 		 * If an event is already enabled, someone is using
4160 		 * it and the self test should not be on.
4161 		 */
4162 		if (file->flags & EVENT_FILE_FL_ENABLED) {
4163 			pr_warn("Enabled event during self test!\n");
4164 			WARN_ON_ONCE(1);
4165 			continue;
4166 		}
4167 
4168 		ftrace_event_enable_disable(file, 1);
4169 		event_test_stuff();
4170 		ftrace_event_enable_disable(file, 0);
4171 
4172 		pr_cont("OK\n");
4173 	}
4174 
4175 	/* Now test at the sub system level */
4176 
4177 	pr_info("Running tests on trace event systems:\n");
4178 
4179 	list_for_each_entry(dir, &tr->systems, list) {
4180 
4181 		system = dir->subsystem;
4182 
4183 		/* the ftrace system is special, skip it */
4184 		if (strcmp(system->name, "ftrace") == 0)
4185 			continue;
4186 
4187 		pr_info("Testing event system %s: ", system->name);
4188 
4189 		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1);
4190 		if (WARN_ON_ONCE(ret)) {
4191 			pr_warn("error enabling system %s\n",
4192 				system->name);
4193 			continue;
4194 		}
4195 
4196 		event_test_stuff();
4197 
4198 		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0);
4199 		if (WARN_ON_ONCE(ret)) {
4200 			pr_warn("error disabling system %s\n",
4201 				system->name);
4202 			continue;
4203 		}
4204 
4205 		pr_cont("OK\n");
4206 	}
4207 
4208 	/* Test with all events enabled */
4209 
4210 	pr_info("Running tests on all trace events:\n");
4211 	pr_info("Testing all events: ");
4212 
4213 	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1);
4214 	if (WARN_ON_ONCE(ret)) {
4215 		pr_warn("error enabling all events\n");
4216 		return;
4217 	}
4218 
4219 	event_test_stuff();
4220 
4221 	/* reset sysname */
4222 	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0);
4223 	if (WARN_ON_ONCE(ret)) {
4224 		pr_warn("error disabling all events\n");
4225 		return;
4226 	}
4227 
4228 	pr_cont("OK\n");
4229 }
4230 
4231 #ifdef CONFIG_FUNCTION_TRACER
4232 
4233 static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
4234 
4235 static struct trace_event_file event_trace_file __initdata;
4236 
4237 static void __init
4238 function_test_events_call(unsigned long ip, unsigned long parent_ip,
4239 			  struct ftrace_ops *op, struct ftrace_regs *regs)
4240 {
4241 	struct trace_buffer *buffer;
4242 	struct ring_buffer_event *event;
4243 	struct ftrace_entry *entry;
4244 	unsigned int trace_ctx;
4245 	long disabled;
4246 	int cpu;
4247 
4248 	trace_ctx = tracing_gen_ctx();
4249 	preempt_disable_notrace();
4250 	cpu = raw_smp_processor_id();
4251 	disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
4252 
4253 	if (disabled != 1)
4254 		goto out;
4255 
4256 	event = trace_event_buffer_lock_reserve(&buffer, &event_trace_file,
4257 						TRACE_FN, sizeof(*entry),
4258 						trace_ctx);
4259 	if (!event)
4260 		goto out;
4261 	entry	= ring_buffer_event_data(event);
4262 	entry->ip			= ip;
4263 	entry->parent_ip		= parent_ip;
4264 
4265 	event_trigger_unlock_commit(&event_trace_file, buffer, event,
4266 				    entry, trace_ctx);
4267  out:
4268 	atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
4269 	preempt_enable_notrace();
4270 }
4271 
4272 static struct ftrace_ops trace_ops __initdata  =
4273 {
4274 	.func = function_test_events_call,
4275 };
4276 
4277 static __init void event_trace_self_test_with_function(void)
4278 {
4279 	int ret;
4280 
4281 	event_trace_file.tr = top_trace_array();
4282 	if (WARN_ON(!event_trace_file.tr))
4283 		return;
4284 
4285 	ret = register_ftrace_function(&trace_ops);
4286 	if (WARN_ON(ret < 0)) {
4287 		pr_info("Failed to enable function tracer for event tests\n");
4288 		return;
4289 	}
4290 	pr_info("Running tests again, along with the function tracer\n");
4291 	event_trace_self_tests();
4292 	unregister_ftrace_function(&trace_ops);
4293 }
4294 #else
4295 static __init void event_trace_self_test_with_function(void)
4296 {
4297 }
4298 #endif
4299 
4300 static __init int event_trace_self_tests_init(void)
4301 {
4302 	if (!tracing_selftest_disabled) {
4303 		event_trace_self_tests();
4304 		event_trace_self_test_with_function();
4305 	}
4306 
4307 	return 0;
4308 }
4309 
4310 late_initcall(event_trace_self_tests_init);
4311 
4312 #endif
4313