xref: /openbmc/linux/kernel/trace/trace_events.c (revision 151f4e2b)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * event tracer
4  *
5  * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
6  *
7  *  - Added format output of fields of the trace point.
8  *    This was based off of work by Tom Zanussi <tzanussi@gmail.com>.
9  *
10  */
11 
12 #define pr_fmt(fmt) fmt
13 
14 #include <linux/workqueue.h>
15 #include <linux/spinlock.h>
16 #include <linux/kthread.h>
17 #include <linux/tracefs.h>
18 #include <linux/uaccess.h>
19 #include <linux/module.h>
20 #include <linux/ctype.h>
21 #include <linux/sort.h>
22 #include <linux/slab.h>
23 #include <linux/delay.h>
24 
25 #include <trace/events/sched.h>
26 
27 #include <asm/setup.h>
28 
29 #include "trace_output.h"
30 
31 #undef TRACE_SYSTEM
32 #define TRACE_SYSTEM "TRACE_SYSTEM"
33 
34 DEFINE_MUTEX(event_mutex);
35 
36 LIST_HEAD(ftrace_events);
37 static LIST_HEAD(ftrace_generic_fields);
38 static LIST_HEAD(ftrace_common_fields);
39 
40 #define GFP_TRACE (GFP_KERNEL | __GFP_ZERO)
41 
42 static struct kmem_cache *field_cachep;
43 static struct kmem_cache *file_cachep;
44 
45 static inline int system_refcount(struct event_subsystem *system)
46 {
47 	return system->ref_count;
48 }
49 
50 static int system_refcount_inc(struct event_subsystem *system)
51 {
52 	return system->ref_count++;
53 }
54 
55 static int system_refcount_dec(struct event_subsystem *system)
56 {
57 	return --system->ref_count;
58 }
59 
60 /* Double loops, do not use break, only goto's work */
61 #define do_for_each_event_file(tr, file)			\
62 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
63 		list_for_each_entry(file, &tr->events, list)
64 
65 #define do_for_each_event_file_safe(tr, file)			\
66 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
67 		struct trace_event_file *___n;				\
68 		list_for_each_entry_safe(file, ___n, &tr->events, list)
69 
70 #define while_for_each_event_file()		\
71 	}
72 
73 static struct list_head *
74 trace_get_fields(struct trace_event_call *event_call)
75 {
76 	if (!event_call->class->get_fields)
77 		return &event_call->class->fields;
78 	return event_call->class->get_fields(event_call);
79 }
80 
81 static struct ftrace_event_field *
82 __find_event_field(struct list_head *head, char *name)
83 {
84 	struct ftrace_event_field *field;
85 
86 	list_for_each_entry(field, head, link) {
87 		if (!strcmp(field->name, name))
88 			return field;
89 	}
90 
91 	return NULL;
92 }
93 
94 struct ftrace_event_field *
95 trace_find_event_field(struct trace_event_call *call, char *name)
96 {
97 	struct ftrace_event_field *field;
98 	struct list_head *head;
99 
100 	head = trace_get_fields(call);
101 	field = __find_event_field(head, name);
102 	if (field)
103 		return field;
104 
105 	field = __find_event_field(&ftrace_generic_fields, name);
106 	if (field)
107 		return field;
108 
109 	return __find_event_field(&ftrace_common_fields, name);
110 }
111 
112 static int __trace_define_field(struct list_head *head, const char *type,
113 				const char *name, int offset, int size,
114 				int is_signed, int filter_type)
115 {
116 	struct ftrace_event_field *field;
117 
118 	field = kmem_cache_alloc(field_cachep, GFP_TRACE);
119 	if (!field)
120 		return -ENOMEM;
121 
122 	field->name = name;
123 	field->type = type;
124 
125 	if (filter_type == FILTER_OTHER)
126 		field->filter_type = filter_assign_type(type);
127 	else
128 		field->filter_type = filter_type;
129 
130 	field->offset = offset;
131 	field->size = size;
132 	field->is_signed = is_signed;
133 
134 	list_add(&field->link, head);
135 
136 	return 0;
137 }
138 
139 int trace_define_field(struct trace_event_call *call, const char *type,
140 		       const char *name, int offset, int size, int is_signed,
141 		       int filter_type)
142 {
143 	struct list_head *head;
144 
145 	if (WARN_ON(!call->class))
146 		return 0;
147 
148 	head = trace_get_fields(call);
149 	return __trace_define_field(head, type, name, offset, size,
150 				    is_signed, filter_type);
151 }
152 EXPORT_SYMBOL_GPL(trace_define_field);
153 
154 #define __generic_field(type, item, filter_type)			\
155 	ret = __trace_define_field(&ftrace_generic_fields, #type,	\
156 				   #item, 0, 0, is_signed_type(type),	\
157 				   filter_type);			\
158 	if (ret)							\
159 		return ret;
160 
161 #define __common_field(type, item)					\
162 	ret = __trace_define_field(&ftrace_common_fields, #type,	\
163 				   "common_" #item,			\
164 				   offsetof(typeof(ent), item),		\
165 				   sizeof(ent.item),			\
166 				   is_signed_type(type), FILTER_OTHER);	\
167 	if (ret)							\
168 		return ret;
169 
170 static int trace_define_generic_fields(void)
171 {
172 	int ret;
173 
174 	__generic_field(int, CPU, FILTER_CPU);
175 	__generic_field(int, cpu, FILTER_CPU);
176 	__generic_field(char *, COMM, FILTER_COMM);
177 	__generic_field(char *, comm, FILTER_COMM);
178 
179 	return ret;
180 }
181 
182 static int trace_define_common_fields(void)
183 {
184 	int ret;
185 	struct trace_entry ent;
186 
187 	__common_field(unsigned short, type);
188 	__common_field(unsigned char, flags);
189 	__common_field(unsigned char, preempt_count);
190 	__common_field(int, pid);
191 
192 	return ret;
193 }
194 
195 static void trace_destroy_fields(struct trace_event_call *call)
196 {
197 	struct ftrace_event_field *field, *next;
198 	struct list_head *head;
199 
200 	head = trace_get_fields(call);
201 	list_for_each_entry_safe(field, next, head, link) {
202 		list_del(&field->link);
203 		kmem_cache_free(field_cachep, field);
204 	}
205 }
206 
207 /*
208  * run-time version of trace_event_get_offsets_<call>() that returns the last
209  * accessible offset of trace fields excluding __dynamic_array bytes
210  */
211 int trace_event_get_offsets(struct trace_event_call *call)
212 {
213 	struct ftrace_event_field *tail;
214 	struct list_head *head;
215 
216 	head = trace_get_fields(call);
217 	/*
218 	 * head->next points to the last field with the largest offset,
219 	 * since it was added last by trace_define_field()
220 	 */
221 	tail = list_first_entry(head, struct ftrace_event_field, link);
222 	return tail->offset + tail->size;
223 }
224 
225 int trace_event_raw_init(struct trace_event_call *call)
226 {
227 	int id;
228 
229 	id = register_trace_event(&call->event);
230 	if (!id)
231 		return -ENODEV;
232 
233 	return 0;
234 }
235 EXPORT_SYMBOL_GPL(trace_event_raw_init);
236 
237 bool trace_event_ignore_this_pid(struct trace_event_file *trace_file)
238 {
239 	struct trace_array *tr = trace_file->tr;
240 	struct trace_array_cpu *data;
241 	struct trace_pid_list *pid_list;
242 
243 	pid_list = rcu_dereference_raw(tr->filtered_pids);
244 	if (!pid_list)
245 		return false;
246 
247 	data = this_cpu_ptr(tr->trace_buffer.data);
248 
249 	return data->ignore_pid;
250 }
251 EXPORT_SYMBOL_GPL(trace_event_ignore_this_pid);
252 
253 void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer,
254 				 struct trace_event_file *trace_file,
255 				 unsigned long len)
256 {
257 	struct trace_event_call *event_call = trace_file->event_call;
258 
259 	if ((trace_file->flags & EVENT_FILE_FL_PID_FILTER) &&
260 	    trace_event_ignore_this_pid(trace_file))
261 		return NULL;
262 
263 	local_save_flags(fbuffer->flags);
264 	fbuffer->pc = preempt_count();
265 	/*
266 	 * If CONFIG_PREEMPT is enabled, then the tracepoint itself disables
267 	 * preemption (adding one to the preempt_count). Since we are
268 	 * interested in the preempt_count at the time the tracepoint was
269 	 * hit, we need to subtract one to offset the increment.
270 	 */
271 	if (IS_ENABLED(CONFIG_PREEMPT))
272 		fbuffer->pc--;
273 	fbuffer->trace_file = trace_file;
274 
275 	fbuffer->event =
276 		trace_event_buffer_lock_reserve(&fbuffer->buffer, trace_file,
277 						event_call->event.type, len,
278 						fbuffer->flags, fbuffer->pc);
279 	if (!fbuffer->event)
280 		return NULL;
281 
282 	fbuffer->entry = ring_buffer_event_data(fbuffer->event);
283 	return fbuffer->entry;
284 }
285 EXPORT_SYMBOL_GPL(trace_event_buffer_reserve);
286 
287 int trace_event_reg(struct trace_event_call *call,
288 		    enum trace_reg type, void *data)
289 {
290 	struct trace_event_file *file = data;
291 
292 	WARN_ON(!(call->flags & TRACE_EVENT_FL_TRACEPOINT));
293 	switch (type) {
294 	case TRACE_REG_REGISTER:
295 		return tracepoint_probe_register(call->tp,
296 						 call->class->probe,
297 						 file);
298 	case TRACE_REG_UNREGISTER:
299 		tracepoint_probe_unregister(call->tp,
300 					    call->class->probe,
301 					    file);
302 		return 0;
303 
304 #ifdef CONFIG_PERF_EVENTS
305 	case TRACE_REG_PERF_REGISTER:
306 		return tracepoint_probe_register(call->tp,
307 						 call->class->perf_probe,
308 						 call);
309 	case TRACE_REG_PERF_UNREGISTER:
310 		tracepoint_probe_unregister(call->tp,
311 					    call->class->perf_probe,
312 					    call);
313 		return 0;
314 	case TRACE_REG_PERF_OPEN:
315 	case TRACE_REG_PERF_CLOSE:
316 	case TRACE_REG_PERF_ADD:
317 	case TRACE_REG_PERF_DEL:
318 		return 0;
319 #endif
320 	}
321 	return 0;
322 }
323 EXPORT_SYMBOL_GPL(trace_event_reg);
324 
325 void trace_event_enable_cmd_record(bool enable)
326 {
327 	struct trace_event_file *file;
328 	struct trace_array *tr;
329 
330 	mutex_lock(&event_mutex);
331 	do_for_each_event_file(tr, file) {
332 
333 		if (!(file->flags & EVENT_FILE_FL_ENABLED))
334 			continue;
335 
336 		if (enable) {
337 			tracing_start_cmdline_record();
338 			set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
339 		} else {
340 			tracing_stop_cmdline_record();
341 			clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
342 		}
343 	} while_for_each_event_file();
344 	mutex_unlock(&event_mutex);
345 }
346 
347 void trace_event_enable_tgid_record(bool enable)
348 {
349 	struct trace_event_file *file;
350 	struct trace_array *tr;
351 
352 	mutex_lock(&event_mutex);
353 	do_for_each_event_file(tr, file) {
354 		if (!(file->flags & EVENT_FILE_FL_ENABLED))
355 			continue;
356 
357 		if (enable) {
358 			tracing_start_tgid_record();
359 			set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
360 		} else {
361 			tracing_stop_tgid_record();
362 			clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT,
363 				  &file->flags);
364 		}
365 	} while_for_each_event_file();
366 	mutex_unlock(&event_mutex);
367 }
368 
369 static int __ftrace_event_enable_disable(struct trace_event_file *file,
370 					 int enable, int soft_disable)
371 {
372 	struct trace_event_call *call = file->event_call;
373 	struct trace_array *tr = file->tr;
374 	unsigned long file_flags = file->flags;
375 	int ret = 0;
376 	int disable;
377 
378 	switch (enable) {
379 	case 0:
380 		/*
381 		 * When soft_disable is set and enable is cleared, the sm_ref
382 		 * reference counter is decremented. If it reaches 0, we want
383 		 * to clear the SOFT_DISABLED flag but leave the event in the
384 		 * state that it was. That is, if the event was enabled and
385 		 * SOFT_DISABLED isn't set, then do nothing. But if SOFT_DISABLED
386 		 * is set we do not want the event to be enabled before we
387 		 * clear the bit.
388 		 *
389 		 * When soft_disable is not set but the SOFT_MODE flag is,
390 		 * we do nothing. Do not disable the tracepoint, otherwise
391 		 * "soft enable"s (clearing the SOFT_DISABLED bit) wont work.
392 		 */
393 		if (soft_disable) {
394 			if (atomic_dec_return(&file->sm_ref) > 0)
395 				break;
396 			disable = file->flags & EVENT_FILE_FL_SOFT_DISABLED;
397 			clear_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
398 		} else
399 			disable = !(file->flags & EVENT_FILE_FL_SOFT_MODE);
400 
401 		if (disable && (file->flags & EVENT_FILE_FL_ENABLED)) {
402 			clear_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
403 			if (file->flags & EVENT_FILE_FL_RECORDED_CMD) {
404 				tracing_stop_cmdline_record();
405 				clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
406 			}
407 
408 			if (file->flags & EVENT_FILE_FL_RECORDED_TGID) {
409 				tracing_stop_tgid_record();
410 				clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
411 			}
412 
413 			call->class->reg(call, TRACE_REG_UNREGISTER, file);
414 		}
415 		/* If in SOFT_MODE, just set the SOFT_DISABLE_BIT, else clear it */
416 		if (file->flags & EVENT_FILE_FL_SOFT_MODE)
417 			set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
418 		else
419 			clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
420 		break;
421 	case 1:
422 		/*
423 		 * When soft_disable is set and enable is set, we want to
424 		 * register the tracepoint for the event, but leave the event
425 		 * as is. That means, if the event was already enabled, we do
426 		 * nothing (but set SOFT_MODE). If the event is disabled, we
427 		 * set SOFT_DISABLED before enabling the event tracepoint, so
428 		 * it still seems to be disabled.
429 		 */
430 		if (!soft_disable)
431 			clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
432 		else {
433 			if (atomic_inc_return(&file->sm_ref) > 1)
434 				break;
435 			set_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
436 		}
437 
438 		if (!(file->flags & EVENT_FILE_FL_ENABLED)) {
439 			bool cmd = false, tgid = false;
440 
441 			/* Keep the event disabled, when going to SOFT_MODE. */
442 			if (soft_disable)
443 				set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
444 
445 			if (tr->trace_flags & TRACE_ITER_RECORD_CMD) {
446 				cmd = true;
447 				tracing_start_cmdline_record();
448 				set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
449 			}
450 
451 			if (tr->trace_flags & TRACE_ITER_RECORD_TGID) {
452 				tgid = true;
453 				tracing_start_tgid_record();
454 				set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
455 			}
456 
457 			ret = call->class->reg(call, TRACE_REG_REGISTER, file);
458 			if (ret) {
459 				if (cmd)
460 					tracing_stop_cmdline_record();
461 				if (tgid)
462 					tracing_stop_tgid_record();
463 				pr_info("event trace: Could not enable event "
464 					"%s\n", trace_event_name(call));
465 				break;
466 			}
467 			set_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
468 
469 			/* WAS_ENABLED gets set but never cleared. */
470 			set_bit(EVENT_FILE_FL_WAS_ENABLED_BIT, &file->flags);
471 		}
472 		break;
473 	}
474 
475 	/* Enable or disable use of trace_buffered_event */
476 	if ((file_flags & EVENT_FILE_FL_SOFT_DISABLED) !=
477 	    (file->flags & EVENT_FILE_FL_SOFT_DISABLED)) {
478 		if (file->flags & EVENT_FILE_FL_SOFT_DISABLED)
479 			trace_buffered_event_enable();
480 		else
481 			trace_buffered_event_disable();
482 	}
483 
484 	return ret;
485 }
486 
487 int trace_event_enable_disable(struct trace_event_file *file,
488 			       int enable, int soft_disable)
489 {
490 	return __ftrace_event_enable_disable(file, enable, soft_disable);
491 }
492 
493 static int ftrace_event_enable_disable(struct trace_event_file *file,
494 				       int enable)
495 {
496 	return __ftrace_event_enable_disable(file, enable, 0);
497 }
498 
499 static void ftrace_clear_events(struct trace_array *tr)
500 {
501 	struct trace_event_file *file;
502 
503 	mutex_lock(&event_mutex);
504 	list_for_each_entry(file, &tr->events, list) {
505 		ftrace_event_enable_disable(file, 0);
506 	}
507 	mutex_unlock(&event_mutex);
508 }
509 
510 static void
511 event_filter_pid_sched_process_exit(void *data, struct task_struct *task)
512 {
513 	struct trace_pid_list *pid_list;
514 	struct trace_array *tr = data;
515 
516 	pid_list = rcu_dereference_raw(tr->filtered_pids);
517 	trace_filter_add_remove_task(pid_list, NULL, task);
518 }
519 
520 static void
521 event_filter_pid_sched_process_fork(void *data,
522 				    struct task_struct *self,
523 				    struct task_struct *task)
524 {
525 	struct trace_pid_list *pid_list;
526 	struct trace_array *tr = data;
527 
528 	pid_list = rcu_dereference_sched(tr->filtered_pids);
529 	trace_filter_add_remove_task(pid_list, self, task);
530 }
531 
532 void trace_event_follow_fork(struct trace_array *tr, bool enable)
533 {
534 	if (enable) {
535 		register_trace_prio_sched_process_fork(event_filter_pid_sched_process_fork,
536 						       tr, INT_MIN);
537 		register_trace_prio_sched_process_exit(event_filter_pid_sched_process_exit,
538 						       tr, INT_MAX);
539 	} else {
540 		unregister_trace_sched_process_fork(event_filter_pid_sched_process_fork,
541 						    tr);
542 		unregister_trace_sched_process_exit(event_filter_pid_sched_process_exit,
543 						    tr);
544 	}
545 }
546 
547 static void
548 event_filter_pid_sched_switch_probe_pre(void *data, bool preempt,
549 		    struct task_struct *prev, struct task_struct *next)
550 {
551 	struct trace_array *tr = data;
552 	struct trace_pid_list *pid_list;
553 
554 	pid_list = rcu_dereference_sched(tr->filtered_pids);
555 
556 	this_cpu_write(tr->trace_buffer.data->ignore_pid,
557 		       trace_ignore_this_task(pid_list, prev) &&
558 		       trace_ignore_this_task(pid_list, next));
559 }
560 
561 static void
562 event_filter_pid_sched_switch_probe_post(void *data, bool preempt,
563 		    struct task_struct *prev, struct task_struct *next)
564 {
565 	struct trace_array *tr = data;
566 	struct trace_pid_list *pid_list;
567 
568 	pid_list = rcu_dereference_sched(tr->filtered_pids);
569 
570 	this_cpu_write(tr->trace_buffer.data->ignore_pid,
571 		       trace_ignore_this_task(pid_list, next));
572 }
573 
574 static void
575 event_filter_pid_sched_wakeup_probe_pre(void *data, struct task_struct *task)
576 {
577 	struct trace_array *tr = data;
578 	struct trace_pid_list *pid_list;
579 
580 	/* Nothing to do if we are already tracing */
581 	if (!this_cpu_read(tr->trace_buffer.data->ignore_pid))
582 		return;
583 
584 	pid_list = rcu_dereference_sched(tr->filtered_pids);
585 
586 	this_cpu_write(tr->trace_buffer.data->ignore_pid,
587 		       trace_ignore_this_task(pid_list, task));
588 }
589 
590 static void
591 event_filter_pid_sched_wakeup_probe_post(void *data, struct task_struct *task)
592 {
593 	struct trace_array *tr = data;
594 	struct trace_pid_list *pid_list;
595 
596 	/* Nothing to do if we are not tracing */
597 	if (this_cpu_read(tr->trace_buffer.data->ignore_pid))
598 		return;
599 
600 	pid_list = rcu_dereference_sched(tr->filtered_pids);
601 
602 	/* Set tracing if current is enabled */
603 	this_cpu_write(tr->trace_buffer.data->ignore_pid,
604 		       trace_ignore_this_task(pid_list, current));
605 }
606 
607 static void __ftrace_clear_event_pids(struct trace_array *tr)
608 {
609 	struct trace_pid_list *pid_list;
610 	struct trace_event_file *file;
611 	int cpu;
612 
613 	pid_list = rcu_dereference_protected(tr->filtered_pids,
614 					     lockdep_is_held(&event_mutex));
615 	if (!pid_list)
616 		return;
617 
618 	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_pre, tr);
619 	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_post, tr);
620 
621 	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre, tr);
622 	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_post, tr);
623 
624 	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre, tr);
625 	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post, tr);
626 
627 	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_pre, tr);
628 	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_post, tr);
629 
630 	list_for_each_entry(file, &tr->events, list) {
631 		clear_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
632 	}
633 
634 	for_each_possible_cpu(cpu)
635 		per_cpu_ptr(tr->trace_buffer.data, cpu)->ignore_pid = false;
636 
637 	rcu_assign_pointer(tr->filtered_pids, NULL);
638 
639 	/* Wait till all users are no longer using pid filtering */
640 	tracepoint_synchronize_unregister();
641 
642 	trace_free_pid_list(pid_list);
643 }
644 
645 static void ftrace_clear_event_pids(struct trace_array *tr)
646 {
647 	mutex_lock(&event_mutex);
648 	__ftrace_clear_event_pids(tr);
649 	mutex_unlock(&event_mutex);
650 }
651 
652 static void __put_system(struct event_subsystem *system)
653 {
654 	struct event_filter *filter = system->filter;
655 
656 	WARN_ON_ONCE(system_refcount(system) == 0);
657 	if (system_refcount_dec(system))
658 		return;
659 
660 	list_del(&system->list);
661 
662 	if (filter) {
663 		kfree(filter->filter_string);
664 		kfree(filter);
665 	}
666 	kfree_const(system->name);
667 	kfree(system);
668 }
669 
670 static void __get_system(struct event_subsystem *system)
671 {
672 	WARN_ON_ONCE(system_refcount(system) == 0);
673 	system_refcount_inc(system);
674 }
675 
676 static void __get_system_dir(struct trace_subsystem_dir *dir)
677 {
678 	WARN_ON_ONCE(dir->ref_count == 0);
679 	dir->ref_count++;
680 	__get_system(dir->subsystem);
681 }
682 
683 static void __put_system_dir(struct trace_subsystem_dir *dir)
684 {
685 	WARN_ON_ONCE(dir->ref_count == 0);
686 	/* If the subsystem is about to be freed, the dir must be too */
687 	WARN_ON_ONCE(system_refcount(dir->subsystem) == 1 && dir->ref_count != 1);
688 
689 	__put_system(dir->subsystem);
690 	if (!--dir->ref_count)
691 		kfree(dir);
692 }
693 
694 static void put_system(struct trace_subsystem_dir *dir)
695 {
696 	mutex_lock(&event_mutex);
697 	__put_system_dir(dir);
698 	mutex_unlock(&event_mutex);
699 }
700 
701 static void remove_subsystem(struct trace_subsystem_dir *dir)
702 {
703 	if (!dir)
704 		return;
705 
706 	if (!--dir->nr_events) {
707 		tracefs_remove_recursive(dir->entry);
708 		list_del(&dir->list);
709 		__put_system_dir(dir);
710 	}
711 }
712 
713 static void remove_event_file_dir(struct trace_event_file *file)
714 {
715 	struct dentry *dir = file->dir;
716 	struct dentry *child;
717 
718 	if (dir) {
719 		spin_lock(&dir->d_lock);	/* probably unneeded */
720 		list_for_each_entry(child, &dir->d_subdirs, d_child) {
721 			if (d_really_is_positive(child))	/* probably unneeded */
722 				d_inode(child)->i_private = NULL;
723 		}
724 		spin_unlock(&dir->d_lock);
725 
726 		tracefs_remove_recursive(dir);
727 	}
728 
729 	list_del(&file->list);
730 	remove_subsystem(file->system);
731 	free_event_filter(file->filter);
732 	kmem_cache_free(file_cachep, file);
733 }
734 
735 /*
736  * __ftrace_set_clr_event(NULL, NULL, NULL, set) will set/unset all events.
737  */
738 static int
739 __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,
740 			      const char *sub, const char *event, int set)
741 {
742 	struct trace_event_file *file;
743 	struct trace_event_call *call;
744 	const char *name;
745 	int ret = -EINVAL;
746 	int eret = 0;
747 
748 	list_for_each_entry(file, &tr->events, list) {
749 
750 		call = file->event_call;
751 		name = trace_event_name(call);
752 
753 		if (!name || !call->class || !call->class->reg)
754 			continue;
755 
756 		if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
757 			continue;
758 
759 		if (match &&
760 		    strcmp(match, name) != 0 &&
761 		    strcmp(match, call->class->system) != 0)
762 			continue;
763 
764 		if (sub && strcmp(sub, call->class->system) != 0)
765 			continue;
766 
767 		if (event && strcmp(event, name) != 0)
768 			continue;
769 
770 		ret = ftrace_event_enable_disable(file, set);
771 
772 		/*
773 		 * Save the first error and return that. Some events
774 		 * may still have been enabled, but let the user
775 		 * know that something went wrong.
776 		 */
777 		if (ret && !eret)
778 			eret = ret;
779 
780 		ret = eret;
781 	}
782 
783 	return ret;
784 }
785 
786 static int __ftrace_set_clr_event(struct trace_array *tr, const char *match,
787 				  const char *sub, const char *event, int set)
788 {
789 	int ret;
790 
791 	mutex_lock(&event_mutex);
792 	ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set);
793 	mutex_unlock(&event_mutex);
794 
795 	return ret;
796 }
797 
798 static int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set)
799 {
800 	char *event = NULL, *sub = NULL, *match;
801 	int ret;
802 
803 	/*
804 	 * The buf format can be <subsystem>:<event-name>
805 	 *  *:<event-name> means any event by that name.
806 	 *  :<event-name> is the same.
807 	 *
808 	 *  <subsystem>:* means all events in that subsystem
809 	 *  <subsystem>: means the same.
810 	 *
811 	 *  <name> (no ':') means all events in a subsystem with
812 	 *  the name <name> or any event that matches <name>
813 	 */
814 
815 	match = strsep(&buf, ":");
816 	if (buf) {
817 		sub = match;
818 		event = buf;
819 		match = NULL;
820 
821 		if (!strlen(sub) || strcmp(sub, "*") == 0)
822 			sub = NULL;
823 		if (!strlen(event) || strcmp(event, "*") == 0)
824 			event = NULL;
825 	}
826 
827 	ret = __ftrace_set_clr_event(tr, match, sub, event, set);
828 
829 	/* Put back the colon to allow this to be called again */
830 	if (buf)
831 		*(buf - 1) = ':';
832 
833 	return ret;
834 }
835 EXPORT_SYMBOL_GPL(ftrace_set_clr_event);
836 
837 /**
838  * trace_set_clr_event - enable or disable an event
839  * @system: system name to match (NULL for any system)
840  * @event: event name to match (NULL for all events, within system)
841  * @set: 1 to enable, 0 to disable
842  *
843  * This is a way for other parts of the kernel to enable or disable
844  * event recording.
845  *
846  * Returns 0 on success, -EINVAL if the parameters do not match any
847  * registered events.
848  */
849 int trace_set_clr_event(const char *system, const char *event, int set)
850 {
851 	struct trace_array *tr = top_trace_array();
852 
853 	if (!tr)
854 		return -ENODEV;
855 
856 	return __ftrace_set_clr_event(tr, NULL, system, event, set);
857 }
858 EXPORT_SYMBOL_GPL(trace_set_clr_event);
859 
860 /* 128 should be much more than enough */
861 #define EVENT_BUF_SIZE		127
862 
863 static ssize_t
864 ftrace_event_write(struct file *file, const char __user *ubuf,
865 		   size_t cnt, loff_t *ppos)
866 {
867 	struct trace_parser parser;
868 	struct seq_file *m = file->private_data;
869 	struct trace_array *tr = m->private;
870 	ssize_t read, ret;
871 
872 	if (!cnt)
873 		return 0;
874 
875 	ret = tracing_update_buffers();
876 	if (ret < 0)
877 		return ret;
878 
879 	if (trace_parser_get_init(&parser, EVENT_BUF_SIZE + 1))
880 		return -ENOMEM;
881 
882 	read = trace_get_user(&parser, ubuf, cnt, ppos);
883 
884 	if (read >= 0 && trace_parser_loaded((&parser))) {
885 		int set = 1;
886 
887 		if (*parser.buffer == '!')
888 			set = 0;
889 
890 		ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);
891 		if (ret)
892 			goto out_put;
893 	}
894 
895 	ret = read;
896 
897  out_put:
898 	trace_parser_put(&parser);
899 
900 	return ret;
901 }
902 
903 static void *
904 t_next(struct seq_file *m, void *v, loff_t *pos)
905 {
906 	struct trace_event_file *file = v;
907 	struct trace_event_call *call;
908 	struct trace_array *tr = m->private;
909 
910 	(*pos)++;
911 
912 	list_for_each_entry_continue(file, &tr->events, list) {
913 		call = file->event_call;
914 		/*
915 		 * The ftrace subsystem is for showing formats only.
916 		 * They can not be enabled or disabled via the event files.
917 		 */
918 		if (call->class && call->class->reg &&
919 		    !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
920 			return file;
921 	}
922 
923 	return NULL;
924 }
925 
926 static void *t_start(struct seq_file *m, loff_t *pos)
927 {
928 	struct trace_event_file *file;
929 	struct trace_array *tr = m->private;
930 	loff_t l;
931 
932 	mutex_lock(&event_mutex);
933 
934 	file = list_entry(&tr->events, struct trace_event_file, list);
935 	for (l = 0; l <= *pos; ) {
936 		file = t_next(m, file, &l);
937 		if (!file)
938 			break;
939 	}
940 	return file;
941 }
942 
943 static void *
944 s_next(struct seq_file *m, void *v, loff_t *pos)
945 {
946 	struct trace_event_file *file = v;
947 	struct trace_array *tr = m->private;
948 
949 	(*pos)++;
950 
951 	list_for_each_entry_continue(file, &tr->events, list) {
952 		if (file->flags & EVENT_FILE_FL_ENABLED)
953 			return file;
954 	}
955 
956 	return NULL;
957 }
958 
959 static void *s_start(struct seq_file *m, loff_t *pos)
960 {
961 	struct trace_event_file *file;
962 	struct trace_array *tr = m->private;
963 	loff_t l;
964 
965 	mutex_lock(&event_mutex);
966 
967 	file = list_entry(&tr->events, struct trace_event_file, list);
968 	for (l = 0; l <= *pos; ) {
969 		file = s_next(m, file, &l);
970 		if (!file)
971 			break;
972 	}
973 	return file;
974 }
975 
976 static int t_show(struct seq_file *m, void *v)
977 {
978 	struct trace_event_file *file = v;
979 	struct trace_event_call *call = file->event_call;
980 
981 	if (strcmp(call->class->system, TRACE_SYSTEM) != 0)
982 		seq_printf(m, "%s:", call->class->system);
983 	seq_printf(m, "%s\n", trace_event_name(call));
984 
985 	return 0;
986 }
987 
988 static void t_stop(struct seq_file *m, void *p)
989 {
990 	mutex_unlock(&event_mutex);
991 }
992 
993 static void *
994 p_next(struct seq_file *m, void *v, loff_t *pos)
995 {
996 	struct trace_array *tr = m->private;
997 	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->filtered_pids);
998 
999 	return trace_pid_next(pid_list, v, pos);
1000 }
1001 
1002 static void *p_start(struct seq_file *m, loff_t *pos)
1003 	__acquires(RCU)
1004 {
1005 	struct trace_pid_list *pid_list;
1006 	struct trace_array *tr = m->private;
1007 
1008 	/*
1009 	 * Grab the mutex, to keep calls to p_next() having the same
1010 	 * tr->filtered_pids as p_start() has.
1011 	 * If we just passed the tr->filtered_pids around, then RCU would
1012 	 * have been enough, but doing that makes things more complex.
1013 	 */
1014 	mutex_lock(&event_mutex);
1015 	rcu_read_lock_sched();
1016 
1017 	pid_list = rcu_dereference_sched(tr->filtered_pids);
1018 
1019 	if (!pid_list)
1020 		return NULL;
1021 
1022 	return trace_pid_start(pid_list, pos);
1023 }
1024 
1025 static void p_stop(struct seq_file *m, void *p)
1026 	__releases(RCU)
1027 {
1028 	rcu_read_unlock_sched();
1029 	mutex_unlock(&event_mutex);
1030 }
1031 
1032 static ssize_t
1033 event_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1034 		  loff_t *ppos)
1035 {
1036 	struct trace_event_file *file;
1037 	unsigned long flags;
1038 	char buf[4] = "0";
1039 
1040 	mutex_lock(&event_mutex);
1041 	file = event_file_data(filp);
1042 	if (likely(file))
1043 		flags = file->flags;
1044 	mutex_unlock(&event_mutex);
1045 
1046 	if (!file)
1047 		return -ENODEV;
1048 
1049 	if (flags & EVENT_FILE_FL_ENABLED &&
1050 	    !(flags & EVENT_FILE_FL_SOFT_DISABLED))
1051 		strcpy(buf, "1");
1052 
1053 	if (flags & EVENT_FILE_FL_SOFT_DISABLED ||
1054 	    flags & EVENT_FILE_FL_SOFT_MODE)
1055 		strcat(buf, "*");
1056 
1057 	strcat(buf, "\n");
1058 
1059 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, strlen(buf));
1060 }
1061 
1062 static ssize_t
1063 event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1064 		   loff_t *ppos)
1065 {
1066 	struct trace_event_file *file;
1067 	unsigned long val;
1068 	int ret;
1069 
1070 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1071 	if (ret)
1072 		return ret;
1073 
1074 	ret = tracing_update_buffers();
1075 	if (ret < 0)
1076 		return ret;
1077 
1078 	switch (val) {
1079 	case 0:
1080 	case 1:
1081 		ret = -ENODEV;
1082 		mutex_lock(&event_mutex);
1083 		file = event_file_data(filp);
1084 		if (likely(file))
1085 			ret = ftrace_event_enable_disable(file, val);
1086 		mutex_unlock(&event_mutex);
1087 		break;
1088 
1089 	default:
1090 		return -EINVAL;
1091 	}
1092 
1093 	*ppos += cnt;
1094 
1095 	return ret ? ret : cnt;
1096 }
1097 
1098 static ssize_t
1099 system_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1100 		   loff_t *ppos)
1101 {
1102 	const char set_to_char[4] = { '?', '0', '1', 'X' };
1103 	struct trace_subsystem_dir *dir = filp->private_data;
1104 	struct event_subsystem *system = dir->subsystem;
1105 	struct trace_event_call *call;
1106 	struct trace_event_file *file;
1107 	struct trace_array *tr = dir->tr;
1108 	char buf[2];
1109 	int set = 0;
1110 	int ret;
1111 
1112 	mutex_lock(&event_mutex);
1113 	list_for_each_entry(file, &tr->events, list) {
1114 		call = file->event_call;
1115 		if (!trace_event_name(call) || !call->class || !call->class->reg)
1116 			continue;
1117 
1118 		if (system && strcmp(call->class->system, system->name) != 0)
1119 			continue;
1120 
1121 		/*
1122 		 * We need to find out if all the events are set
1123 		 * or if all events or cleared, or if we have
1124 		 * a mixture.
1125 		 */
1126 		set |= (1 << !!(file->flags & EVENT_FILE_FL_ENABLED));
1127 
1128 		/*
1129 		 * If we have a mixture, no need to look further.
1130 		 */
1131 		if (set == 3)
1132 			break;
1133 	}
1134 	mutex_unlock(&event_mutex);
1135 
1136 	buf[0] = set_to_char[set];
1137 	buf[1] = '\n';
1138 
1139 	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
1140 
1141 	return ret;
1142 }
1143 
1144 static ssize_t
1145 system_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1146 		    loff_t *ppos)
1147 {
1148 	struct trace_subsystem_dir *dir = filp->private_data;
1149 	struct event_subsystem *system = dir->subsystem;
1150 	const char *name = NULL;
1151 	unsigned long val;
1152 	ssize_t ret;
1153 
1154 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1155 	if (ret)
1156 		return ret;
1157 
1158 	ret = tracing_update_buffers();
1159 	if (ret < 0)
1160 		return ret;
1161 
1162 	if (val != 0 && val != 1)
1163 		return -EINVAL;
1164 
1165 	/*
1166 	 * Opening of "enable" adds a ref count to system,
1167 	 * so the name is safe to use.
1168 	 */
1169 	if (system)
1170 		name = system->name;
1171 
1172 	ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val);
1173 	if (ret)
1174 		goto out;
1175 
1176 	ret = cnt;
1177 
1178 out:
1179 	*ppos += cnt;
1180 
1181 	return ret;
1182 }
1183 
1184 enum {
1185 	FORMAT_HEADER		= 1,
1186 	FORMAT_FIELD_SEPERATOR	= 2,
1187 	FORMAT_PRINTFMT		= 3,
1188 };
1189 
1190 static void *f_next(struct seq_file *m, void *v, loff_t *pos)
1191 {
1192 	struct trace_event_call *call = event_file_data(m->private);
1193 	struct list_head *common_head = &ftrace_common_fields;
1194 	struct list_head *head = trace_get_fields(call);
1195 	struct list_head *node = v;
1196 
1197 	(*pos)++;
1198 
1199 	switch ((unsigned long)v) {
1200 	case FORMAT_HEADER:
1201 		node = common_head;
1202 		break;
1203 
1204 	case FORMAT_FIELD_SEPERATOR:
1205 		node = head;
1206 		break;
1207 
1208 	case FORMAT_PRINTFMT:
1209 		/* all done */
1210 		return NULL;
1211 	}
1212 
1213 	node = node->prev;
1214 	if (node == common_head)
1215 		return (void *)FORMAT_FIELD_SEPERATOR;
1216 	else if (node == head)
1217 		return (void *)FORMAT_PRINTFMT;
1218 	else
1219 		return node;
1220 }
1221 
1222 static int f_show(struct seq_file *m, void *v)
1223 {
1224 	struct trace_event_call *call = event_file_data(m->private);
1225 	struct ftrace_event_field *field;
1226 	const char *array_descriptor;
1227 
1228 	switch ((unsigned long)v) {
1229 	case FORMAT_HEADER:
1230 		seq_printf(m, "name: %s\n", trace_event_name(call));
1231 		seq_printf(m, "ID: %d\n", call->event.type);
1232 		seq_puts(m, "format:\n");
1233 		return 0;
1234 
1235 	case FORMAT_FIELD_SEPERATOR:
1236 		seq_putc(m, '\n');
1237 		return 0;
1238 
1239 	case FORMAT_PRINTFMT:
1240 		seq_printf(m, "\nprint fmt: %s\n",
1241 			   call->print_fmt);
1242 		return 0;
1243 	}
1244 
1245 	field = list_entry(v, struct ftrace_event_field, link);
1246 	/*
1247 	 * Smartly shows the array type(except dynamic array).
1248 	 * Normal:
1249 	 *	field:TYPE VAR
1250 	 * If TYPE := TYPE[LEN], it is shown:
1251 	 *	field:TYPE VAR[LEN]
1252 	 */
1253 	array_descriptor = strchr(field->type, '[');
1254 
1255 	if (str_has_prefix(field->type, "__data_loc"))
1256 		array_descriptor = NULL;
1257 
1258 	if (!array_descriptor)
1259 		seq_printf(m, "\tfield:%s %s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1260 			   field->type, field->name, field->offset,
1261 			   field->size, !!field->is_signed);
1262 	else
1263 		seq_printf(m, "\tfield:%.*s %s%s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1264 			   (int)(array_descriptor - field->type),
1265 			   field->type, field->name,
1266 			   array_descriptor, field->offset,
1267 			   field->size, !!field->is_signed);
1268 
1269 	return 0;
1270 }
1271 
1272 static void *f_start(struct seq_file *m, loff_t *pos)
1273 {
1274 	void *p = (void *)FORMAT_HEADER;
1275 	loff_t l = 0;
1276 
1277 	/* ->stop() is called even if ->start() fails */
1278 	mutex_lock(&event_mutex);
1279 	if (!event_file_data(m->private))
1280 		return ERR_PTR(-ENODEV);
1281 
1282 	while (l < *pos && p)
1283 		p = f_next(m, p, &l);
1284 
1285 	return p;
1286 }
1287 
1288 static void f_stop(struct seq_file *m, void *p)
1289 {
1290 	mutex_unlock(&event_mutex);
1291 }
1292 
1293 static const struct seq_operations trace_format_seq_ops = {
1294 	.start		= f_start,
1295 	.next		= f_next,
1296 	.stop		= f_stop,
1297 	.show		= f_show,
1298 };
1299 
1300 static int trace_format_open(struct inode *inode, struct file *file)
1301 {
1302 	struct seq_file *m;
1303 	int ret;
1304 
1305 	ret = seq_open(file, &trace_format_seq_ops);
1306 	if (ret < 0)
1307 		return ret;
1308 
1309 	m = file->private_data;
1310 	m->private = file;
1311 
1312 	return 0;
1313 }
1314 
1315 static ssize_t
1316 event_id_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1317 {
1318 	int id = (long)event_file_data(filp);
1319 	char buf[32];
1320 	int len;
1321 
1322 	if (unlikely(!id))
1323 		return -ENODEV;
1324 
1325 	len = sprintf(buf, "%d\n", id);
1326 
1327 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
1328 }
1329 
1330 static ssize_t
1331 event_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
1332 		  loff_t *ppos)
1333 {
1334 	struct trace_event_file *file;
1335 	struct trace_seq *s;
1336 	int r = -ENODEV;
1337 
1338 	if (*ppos)
1339 		return 0;
1340 
1341 	s = kmalloc(sizeof(*s), GFP_KERNEL);
1342 
1343 	if (!s)
1344 		return -ENOMEM;
1345 
1346 	trace_seq_init(s);
1347 
1348 	mutex_lock(&event_mutex);
1349 	file = event_file_data(filp);
1350 	if (file)
1351 		print_event_filter(file, s);
1352 	mutex_unlock(&event_mutex);
1353 
1354 	if (file)
1355 		r = simple_read_from_buffer(ubuf, cnt, ppos,
1356 					    s->buffer, trace_seq_used(s));
1357 
1358 	kfree(s);
1359 
1360 	return r;
1361 }
1362 
1363 static ssize_t
1364 event_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
1365 		   loff_t *ppos)
1366 {
1367 	struct trace_event_file *file;
1368 	char *buf;
1369 	int err = -ENODEV;
1370 
1371 	if (cnt >= PAGE_SIZE)
1372 		return -EINVAL;
1373 
1374 	buf = memdup_user_nul(ubuf, cnt);
1375 	if (IS_ERR(buf))
1376 		return PTR_ERR(buf);
1377 
1378 	mutex_lock(&event_mutex);
1379 	file = event_file_data(filp);
1380 	if (file)
1381 		err = apply_event_filter(file, buf);
1382 	mutex_unlock(&event_mutex);
1383 
1384 	kfree(buf);
1385 	if (err < 0)
1386 		return err;
1387 
1388 	*ppos += cnt;
1389 
1390 	return cnt;
1391 }
1392 
1393 static LIST_HEAD(event_subsystems);
1394 
1395 static int subsystem_open(struct inode *inode, struct file *filp)
1396 {
1397 	struct event_subsystem *system = NULL;
1398 	struct trace_subsystem_dir *dir = NULL; /* Initialize for gcc */
1399 	struct trace_array *tr;
1400 	int ret;
1401 
1402 	if (tracing_is_disabled())
1403 		return -ENODEV;
1404 
1405 	/* Make sure the system still exists */
1406 	mutex_lock(&event_mutex);
1407 	mutex_lock(&trace_types_lock);
1408 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1409 		list_for_each_entry(dir, &tr->systems, list) {
1410 			if (dir == inode->i_private) {
1411 				/* Don't open systems with no events */
1412 				if (dir->nr_events) {
1413 					__get_system_dir(dir);
1414 					system = dir->subsystem;
1415 				}
1416 				goto exit_loop;
1417 			}
1418 		}
1419 	}
1420  exit_loop:
1421 	mutex_unlock(&trace_types_lock);
1422 	mutex_unlock(&event_mutex);
1423 
1424 	if (!system)
1425 		return -ENODEV;
1426 
1427 	/* Some versions of gcc think dir can be uninitialized here */
1428 	WARN_ON(!dir);
1429 
1430 	/* Still need to increment the ref count of the system */
1431 	if (trace_array_get(tr) < 0) {
1432 		put_system(dir);
1433 		return -ENODEV;
1434 	}
1435 
1436 	ret = tracing_open_generic(inode, filp);
1437 	if (ret < 0) {
1438 		trace_array_put(tr);
1439 		put_system(dir);
1440 	}
1441 
1442 	return ret;
1443 }
1444 
1445 static int system_tr_open(struct inode *inode, struct file *filp)
1446 {
1447 	struct trace_subsystem_dir *dir;
1448 	struct trace_array *tr = inode->i_private;
1449 	int ret;
1450 
1451 	if (tracing_is_disabled())
1452 		return -ENODEV;
1453 
1454 	if (trace_array_get(tr) < 0)
1455 		return -ENODEV;
1456 
1457 	/* Make a temporary dir that has no system but points to tr */
1458 	dir = kzalloc(sizeof(*dir), GFP_KERNEL);
1459 	if (!dir) {
1460 		trace_array_put(tr);
1461 		return -ENOMEM;
1462 	}
1463 
1464 	dir->tr = tr;
1465 
1466 	ret = tracing_open_generic(inode, filp);
1467 	if (ret < 0) {
1468 		trace_array_put(tr);
1469 		kfree(dir);
1470 		return ret;
1471 	}
1472 
1473 	filp->private_data = dir;
1474 
1475 	return 0;
1476 }
1477 
1478 static int subsystem_release(struct inode *inode, struct file *file)
1479 {
1480 	struct trace_subsystem_dir *dir = file->private_data;
1481 
1482 	trace_array_put(dir->tr);
1483 
1484 	/*
1485 	 * If dir->subsystem is NULL, then this is a temporary
1486 	 * descriptor that was made for a trace_array to enable
1487 	 * all subsystems.
1488 	 */
1489 	if (dir->subsystem)
1490 		put_system(dir);
1491 	else
1492 		kfree(dir);
1493 
1494 	return 0;
1495 }
1496 
1497 static ssize_t
1498 subsystem_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
1499 		      loff_t *ppos)
1500 {
1501 	struct trace_subsystem_dir *dir = filp->private_data;
1502 	struct event_subsystem *system = dir->subsystem;
1503 	struct trace_seq *s;
1504 	int r;
1505 
1506 	if (*ppos)
1507 		return 0;
1508 
1509 	s = kmalloc(sizeof(*s), GFP_KERNEL);
1510 	if (!s)
1511 		return -ENOMEM;
1512 
1513 	trace_seq_init(s);
1514 
1515 	print_subsystem_event_filter(system, s);
1516 	r = simple_read_from_buffer(ubuf, cnt, ppos,
1517 				    s->buffer, trace_seq_used(s));
1518 
1519 	kfree(s);
1520 
1521 	return r;
1522 }
1523 
1524 static ssize_t
1525 subsystem_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
1526 		       loff_t *ppos)
1527 {
1528 	struct trace_subsystem_dir *dir = filp->private_data;
1529 	char *buf;
1530 	int err;
1531 
1532 	if (cnt >= PAGE_SIZE)
1533 		return -EINVAL;
1534 
1535 	buf = memdup_user_nul(ubuf, cnt);
1536 	if (IS_ERR(buf))
1537 		return PTR_ERR(buf);
1538 
1539 	err = apply_subsystem_event_filter(dir, buf);
1540 	kfree(buf);
1541 	if (err < 0)
1542 		return err;
1543 
1544 	*ppos += cnt;
1545 
1546 	return cnt;
1547 }
1548 
1549 static ssize_t
1550 show_header(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1551 {
1552 	int (*func)(struct trace_seq *s) = filp->private_data;
1553 	struct trace_seq *s;
1554 	int r;
1555 
1556 	if (*ppos)
1557 		return 0;
1558 
1559 	s = kmalloc(sizeof(*s), GFP_KERNEL);
1560 	if (!s)
1561 		return -ENOMEM;
1562 
1563 	trace_seq_init(s);
1564 
1565 	func(s);
1566 	r = simple_read_from_buffer(ubuf, cnt, ppos,
1567 				    s->buffer, trace_seq_used(s));
1568 
1569 	kfree(s);
1570 
1571 	return r;
1572 }
1573 
1574 static void ignore_task_cpu(void *data)
1575 {
1576 	struct trace_array *tr = data;
1577 	struct trace_pid_list *pid_list;
1578 
1579 	/*
1580 	 * This function is called by on_each_cpu() while the
1581 	 * event_mutex is held.
1582 	 */
1583 	pid_list = rcu_dereference_protected(tr->filtered_pids,
1584 					     mutex_is_locked(&event_mutex));
1585 
1586 	this_cpu_write(tr->trace_buffer.data->ignore_pid,
1587 		       trace_ignore_this_task(pid_list, current));
1588 }
1589 
1590 static ssize_t
1591 ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
1592 		       size_t cnt, loff_t *ppos)
1593 {
1594 	struct seq_file *m = filp->private_data;
1595 	struct trace_array *tr = m->private;
1596 	struct trace_pid_list *filtered_pids = NULL;
1597 	struct trace_pid_list *pid_list;
1598 	struct trace_event_file *file;
1599 	ssize_t ret;
1600 
1601 	if (!cnt)
1602 		return 0;
1603 
1604 	ret = tracing_update_buffers();
1605 	if (ret < 0)
1606 		return ret;
1607 
1608 	mutex_lock(&event_mutex);
1609 
1610 	filtered_pids = rcu_dereference_protected(tr->filtered_pids,
1611 					     lockdep_is_held(&event_mutex));
1612 
1613 	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
1614 	if (ret < 0)
1615 		goto out;
1616 
1617 	rcu_assign_pointer(tr->filtered_pids, pid_list);
1618 
1619 	list_for_each_entry(file, &tr->events, list) {
1620 		set_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
1621 	}
1622 
1623 	if (filtered_pids) {
1624 		tracepoint_synchronize_unregister();
1625 		trace_free_pid_list(filtered_pids);
1626 	} else if (pid_list) {
1627 		/*
1628 		 * Register a probe that is called before all other probes
1629 		 * to set ignore_pid if next or prev do not match.
1630 		 * Register a probe this is called after all other probes
1631 		 * to only keep ignore_pid set if next pid matches.
1632 		 */
1633 		register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_pre,
1634 						 tr, INT_MAX);
1635 		register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_post,
1636 						 tr, 0);
1637 
1638 		register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre,
1639 						 tr, INT_MAX);
1640 		register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_post,
1641 						 tr, 0);
1642 
1643 		register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre,
1644 						     tr, INT_MAX);
1645 		register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post,
1646 						     tr, 0);
1647 
1648 		register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_pre,
1649 						 tr, INT_MAX);
1650 		register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_post,
1651 						 tr, 0);
1652 	}
1653 
1654 	/*
1655 	 * Ignoring of pids is done at task switch. But we have to
1656 	 * check for those tasks that are currently running.
1657 	 * Always do this in case a pid was appended or removed.
1658 	 */
1659 	on_each_cpu(ignore_task_cpu, tr, 1);
1660 
1661  out:
1662 	mutex_unlock(&event_mutex);
1663 
1664 	if (ret > 0)
1665 		*ppos += ret;
1666 
1667 	return ret;
1668 }
1669 
1670 static int ftrace_event_avail_open(struct inode *inode, struct file *file);
1671 static int ftrace_event_set_open(struct inode *inode, struct file *file);
1672 static int ftrace_event_set_pid_open(struct inode *inode, struct file *file);
1673 static int ftrace_event_release(struct inode *inode, struct file *file);
1674 
1675 static const struct seq_operations show_event_seq_ops = {
1676 	.start = t_start,
1677 	.next = t_next,
1678 	.show = t_show,
1679 	.stop = t_stop,
1680 };
1681 
1682 static const struct seq_operations show_set_event_seq_ops = {
1683 	.start = s_start,
1684 	.next = s_next,
1685 	.show = t_show,
1686 	.stop = t_stop,
1687 };
1688 
1689 static const struct seq_operations show_set_pid_seq_ops = {
1690 	.start = p_start,
1691 	.next = p_next,
1692 	.show = trace_pid_show,
1693 	.stop = p_stop,
1694 };
1695 
1696 static const struct file_operations ftrace_avail_fops = {
1697 	.open = ftrace_event_avail_open,
1698 	.read = seq_read,
1699 	.llseek = seq_lseek,
1700 	.release = seq_release,
1701 };
1702 
1703 static const struct file_operations ftrace_set_event_fops = {
1704 	.open = ftrace_event_set_open,
1705 	.read = seq_read,
1706 	.write = ftrace_event_write,
1707 	.llseek = seq_lseek,
1708 	.release = ftrace_event_release,
1709 };
1710 
1711 static const struct file_operations ftrace_set_event_pid_fops = {
1712 	.open = ftrace_event_set_pid_open,
1713 	.read = seq_read,
1714 	.write = ftrace_event_pid_write,
1715 	.llseek = seq_lseek,
1716 	.release = ftrace_event_release,
1717 };
1718 
1719 static const struct file_operations ftrace_enable_fops = {
1720 	.open = tracing_open_generic,
1721 	.read = event_enable_read,
1722 	.write = event_enable_write,
1723 	.llseek = default_llseek,
1724 };
1725 
1726 static const struct file_operations ftrace_event_format_fops = {
1727 	.open = trace_format_open,
1728 	.read = seq_read,
1729 	.llseek = seq_lseek,
1730 	.release = seq_release,
1731 };
1732 
1733 static const struct file_operations ftrace_event_id_fops = {
1734 	.read = event_id_read,
1735 	.llseek = default_llseek,
1736 };
1737 
1738 static const struct file_operations ftrace_event_filter_fops = {
1739 	.open = tracing_open_generic,
1740 	.read = event_filter_read,
1741 	.write = event_filter_write,
1742 	.llseek = default_llseek,
1743 };
1744 
1745 static const struct file_operations ftrace_subsystem_filter_fops = {
1746 	.open = subsystem_open,
1747 	.read = subsystem_filter_read,
1748 	.write = subsystem_filter_write,
1749 	.llseek = default_llseek,
1750 	.release = subsystem_release,
1751 };
1752 
1753 static const struct file_operations ftrace_system_enable_fops = {
1754 	.open = subsystem_open,
1755 	.read = system_enable_read,
1756 	.write = system_enable_write,
1757 	.llseek = default_llseek,
1758 	.release = subsystem_release,
1759 };
1760 
1761 static const struct file_operations ftrace_tr_enable_fops = {
1762 	.open = system_tr_open,
1763 	.read = system_enable_read,
1764 	.write = system_enable_write,
1765 	.llseek = default_llseek,
1766 	.release = subsystem_release,
1767 };
1768 
1769 static const struct file_operations ftrace_show_header_fops = {
1770 	.open = tracing_open_generic,
1771 	.read = show_header,
1772 	.llseek = default_llseek,
1773 };
1774 
1775 static int
1776 ftrace_event_open(struct inode *inode, struct file *file,
1777 		  const struct seq_operations *seq_ops)
1778 {
1779 	struct seq_file *m;
1780 	int ret;
1781 
1782 	ret = seq_open(file, seq_ops);
1783 	if (ret < 0)
1784 		return ret;
1785 	m = file->private_data;
1786 	/* copy tr over to seq ops */
1787 	m->private = inode->i_private;
1788 
1789 	return ret;
1790 }
1791 
1792 static int ftrace_event_release(struct inode *inode, struct file *file)
1793 {
1794 	struct trace_array *tr = inode->i_private;
1795 
1796 	trace_array_put(tr);
1797 
1798 	return seq_release(inode, file);
1799 }
1800 
1801 static int
1802 ftrace_event_avail_open(struct inode *inode, struct file *file)
1803 {
1804 	const struct seq_operations *seq_ops = &show_event_seq_ops;
1805 
1806 	return ftrace_event_open(inode, file, seq_ops);
1807 }
1808 
1809 static int
1810 ftrace_event_set_open(struct inode *inode, struct file *file)
1811 {
1812 	const struct seq_operations *seq_ops = &show_set_event_seq_ops;
1813 	struct trace_array *tr = inode->i_private;
1814 	int ret;
1815 
1816 	if (trace_array_get(tr) < 0)
1817 		return -ENODEV;
1818 
1819 	if ((file->f_mode & FMODE_WRITE) &&
1820 	    (file->f_flags & O_TRUNC))
1821 		ftrace_clear_events(tr);
1822 
1823 	ret = ftrace_event_open(inode, file, seq_ops);
1824 	if (ret < 0)
1825 		trace_array_put(tr);
1826 	return ret;
1827 }
1828 
1829 static int
1830 ftrace_event_set_pid_open(struct inode *inode, struct file *file)
1831 {
1832 	const struct seq_operations *seq_ops = &show_set_pid_seq_ops;
1833 	struct trace_array *tr = inode->i_private;
1834 	int ret;
1835 
1836 	if (trace_array_get(tr) < 0)
1837 		return -ENODEV;
1838 
1839 	if ((file->f_mode & FMODE_WRITE) &&
1840 	    (file->f_flags & O_TRUNC))
1841 		ftrace_clear_event_pids(tr);
1842 
1843 	ret = ftrace_event_open(inode, file, seq_ops);
1844 	if (ret < 0)
1845 		trace_array_put(tr);
1846 	return ret;
1847 }
1848 
1849 static struct event_subsystem *
1850 create_new_subsystem(const char *name)
1851 {
1852 	struct event_subsystem *system;
1853 
1854 	/* need to create new entry */
1855 	system = kmalloc(sizeof(*system), GFP_KERNEL);
1856 	if (!system)
1857 		return NULL;
1858 
1859 	system->ref_count = 1;
1860 
1861 	/* Only allocate if dynamic (kprobes and modules) */
1862 	system->name = kstrdup_const(name, GFP_KERNEL);
1863 	if (!system->name)
1864 		goto out_free;
1865 
1866 	system->filter = NULL;
1867 
1868 	system->filter = kzalloc(sizeof(struct event_filter), GFP_KERNEL);
1869 	if (!system->filter)
1870 		goto out_free;
1871 
1872 	list_add(&system->list, &event_subsystems);
1873 
1874 	return system;
1875 
1876  out_free:
1877 	kfree_const(system->name);
1878 	kfree(system);
1879 	return NULL;
1880 }
1881 
1882 static struct dentry *
1883 event_subsystem_dir(struct trace_array *tr, const char *name,
1884 		    struct trace_event_file *file, struct dentry *parent)
1885 {
1886 	struct trace_subsystem_dir *dir;
1887 	struct event_subsystem *system;
1888 	struct dentry *entry;
1889 
1890 	/* First see if we did not already create this dir */
1891 	list_for_each_entry(dir, &tr->systems, list) {
1892 		system = dir->subsystem;
1893 		if (strcmp(system->name, name) == 0) {
1894 			dir->nr_events++;
1895 			file->system = dir;
1896 			return dir->entry;
1897 		}
1898 	}
1899 
1900 	/* Now see if the system itself exists. */
1901 	list_for_each_entry(system, &event_subsystems, list) {
1902 		if (strcmp(system->name, name) == 0)
1903 			break;
1904 	}
1905 	/* Reset system variable when not found */
1906 	if (&system->list == &event_subsystems)
1907 		system = NULL;
1908 
1909 	dir = kmalloc(sizeof(*dir), GFP_KERNEL);
1910 	if (!dir)
1911 		goto out_fail;
1912 
1913 	if (!system) {
1914 		system = create_new_subsystem(name);
1915 		if (!system)
1916 			goto out_free;
1917 	} else
1918 		__get_system(system);
1919 
1920 	dir->entry = tracefs_create_dir(name, parent);
1921 	if (!dir->entry) {
1922 		pr_warn("Failed to create system directory %s\n", name);
1923 		__put_system(system);
1924 		goto out_free;
1925 	}
1926 
1927 	dir->tr = tr;
1928 	dir->ref_count = 1;
1929 	dir->nr_events = 1;
1930 	dir->subsystem = system;
1931 	file->system = dir;
1932 
1933 	entry = tracefs_create_file("filter", 0644, dir->entry, dir,
1934 				    &ftrace_subsystem_filter_fops);
1935 	if (!entry) {
1936 		kfree(system->filter);
1937 		system->filter = NULL;
1938 		pr_warn("Could not create tracefs '%s/filter' entry\n", name);
1939 	}
1940 
1941 	trace_create_file("enable", 0644, dir->entry, dir,
1942 			  &ftrace_system_enable_fops);
1943 
1944 	list_add(&dir->list, &tr->systems);
1945 
1946 	return dir->entry;
1947 
1948  out_free:
1949 	kfree(dir);
1950  out_fail:
1951 	/* Only print this message if failed on memory allocation */
1952 	if (!dir || !system)
1953 		pr_warn("No memory to create event subsystem %s\n", name);
1954 	return NULL;
1955 }
1956 
1957 static int
1958 event_create_dir(struct dentry *parent, struct trace_event_file *file)
1959 {
1960 	struct trace_event_call *call = file->event_call;
1961 	struct trace_array *tr = file->tr;
1962 	struct list_head *head;
1963 	struct dentry *d_events;
1964 	const char *name;
1965 	int ret;
1966 
1967 	/*
1968 	 * If the trace point header did not define TRACE_SYSTEM
1969 	 * then the system would be called "TRACE_SYSTEM".
1970 	 */
1971 	if (strcmp(call->class->system, TRACE_SYSTEM) != 0) {
1972 		d_events = event_subsystem_dir(tr, call->class->system, file, parent);
1973 		if (!d_events)
1974 			return -ENOMEM;
1975 	} else
1976 		d_events = parent;
1977 
1978 	name = trace_event_name(call);
1979 	file->dir = tracefs_create_dir(name, d_events);
1980 	if (!file->dir) {
1981 		pr_warn("Could not create tracefs '%s' directory\n", name);
1982 		return -1;
1983 	}
1984 
1985 	if (call->class->reg && !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
1986 		trace_create_file("enable", 0644, file->dir, file,
1987 				  &ftrace_enable_fops);
1988 
1989 #ifdef CONFIG_PERF_EVENTS
1990 	if (call->event.type && call->class->reg)
1991 		trace_create_file("id", 0444, file->dir,
1992 				  (void *)(long)call->event.type,
1993 				  &ftrace_event_id_fops);
1994 #endif
1995 
1996 	/*
1997 	 * Other events may have the same class. Only update
1998 	 * the fields if they are not already defined.
1999 	 */
2000 	head = trace_get_fields(call);
2001 	if (list_empty(head)) {
2002 		ret = call->class->define_fields(call);
2003 		if (ret < 0) {
2004 			pr_warn("Could not initialize trace point events/%s\n",
2005 				name);
2006 			return -1;
2007 		}
2008 	}
2009 
2010 	/*
2011 	 * Only event directories that can be enabled should have
2012 	 * triggers or filters.
2013 	 */
2014 	if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) {
2015 		trace_create_file("filter", 0644, file->dir, file,
2016 				  &ftrace_event_filter_fops);
2017 
2018 		trace_create_file("trigger", 0644, file->dir, file,
2019 				  &event_trigger_fops);
2020 	}
2021 
2022 #ifdef CONFIG_HIST_TRIGGERS
2023 	trace_create_file("hist", 0444, file->dir, file,
2024 			  &event_hist_fops);
2025 #endif
2026 	trace_create_file("format", 0444, file->dir, call,
2027 			  &ftrace_event_format_fops);
2028 
2029 	return 0;
2030 }
2031 
2032 static void remove_event_from_tracers(struct trace_event_call *call)
2033 {
2034 	struct trace_event_file *file;
2035 	struct trace_array *tr;
2036 
2037 	do_for_each_event_file_safe(tr, file) {
2038 		if (file->event_call != call)
2039 			continue;
2040 
2041 		remove_event_file_dir(file);
2042 		/*
2043 		 * The do_for_each_event_file_safe() is
2044 		 * a double loop. After finding the call for this
2045 		 * trace_array, we use break to jump to the next
2046 		 * trace_array.
2047 		 */
2048 		break;
2049 	} while_for_each_event_file();
2050 }
2051 
2052 static void event_remove(struct trace_event_call *call)
2053 {
2054 	struct trace_array *tr;
2055 	struct trace_event_file *file;
2056 
2057 	do_for_each_event_file(tr, file) {
2058 		if (file->event_call != call)
2059 			continue;
2060 
2061 		if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
2062 			tr->clear_trace = true;
2063 
2064 		ftrace_event_enable_disable(file, 0);
2065 		/*
2066 		 * The do_for_each_event_file() is
2067 		 * a double loop. After finding the call for this
2068 		 * trace_array, we use break to jump to the next
2069 		 * trace_array.
2070 		 */
2071 		break;
2072 	} while_for_each_event_file();
2073 
2074 	if (call->event.funcs)
2075 		__unregister_trace_event(&call->event);
2076 	remove_event_from_tracers(call);
2077 	list_del(&call->list);
2078 }
2079 
2080 static int event_init(struct trace_event_call *call)
2081 {
2082 	int ret = 0;
2083 	const char *name;
2084 
2085 	name = trace_event_name(call);
2086 	if (WARN_ON(!name))
2087 		return -EINVAL;
2088 
2089 	if (call->class->raw_init) {
2090 		ret = call->class->raw_init(call);
2091 		if (ret < 0 && ret != -ENOSYS)
2092 			pr_warn("Could not initialize trace events/%s\n", name);
2093 	}
2094 
2095 	return ret;
2096 }
2097 
2098 static int
2099 __register_event(struct trace_event_call *call, struct module *mod)
2100 {
2101 	int ret;
2102 
2103 	ret = event_init(call);
2104 	if (ret < 0)
2105 		return ret;
2106 
2107 	list_add(&call->list, &ftrace_events);
2108 	call->mod = mod;
2109 
2110 	return 0;
2111 }
2112 
2113 static char *eval_replace(char *ptr, struct trace_eval_map *map, int len)
2114 {
2115 	int rlen;
2116 	int elen;
2117 
2118 	/* Find the length of the eval value as a string */
2119 	elen = snprintf(ptr, 0, "%ld", map->eval_value);
2120 	/* Make sure there's enough room to replace the string with the value */
2121 	if (len < elen)
2122 		return NULL;
2123 
2124 	snprintf(ptr, elen + 1, "%ld", map->eval_value);
2125 
2126 	/* Get the rest of the string of ptr */
2127 	rlen = strlen(ptr + len);
2128 	memmove(ptr + elen, ptr + len, rlen);
2129 	/* Make sure we end the new string */
2130 	ptr[elen + rlen] = 0;
2131 
2132 	return ptr + elen;
2133 }
2134 
2135 static void update_event_printk(struct trace_event_call *call,
2136 				struct trace_eval_map *map)
2137 {
2138 	char *ptr;
2139 	int quote = 0;
2140 	int len = strlen(map->eval_string);
2141 
2142 	for (ptr = call->print_fmt; *ptr; ptr++) {
2143 		if (*ptr == '\\') {
2144 			ptr++;
2145 			/* paranoid */
2146 			if (!*ptr)
2147 				break;
2148 			continue;
2149 		}
2150 		if (*ptr == '"') {
2151 			quote ^= 1;
2152 			continue;
2153 		}
2154 		if (quote)
2155 			continue;
2156 		if (isdigit(*ptr)) {
2157 			/* skip numbers */
2158 			do {
2159 				ptr++;
2160 				/* Check for alpha chars like ULL */
2161 			} while (isalnum(*ptr));
2162 			if (!*ptr)
2163 				break;
2164 			/*
2165 			 * A number must have some kind of delimiter after
2166 			 * it, and we can ignore that too.
2167 			 */
2168 			continue;
2169 		}
2170 		if (isalpha(*ptr) || *ptr == '_') {
2171 			if (strncmp(map->eval_string, ptr, len) == 0 &&
2172 			    !isalnum(ptr[len]) && ptr[len] != '_') {
2173 				ptr = eval_replace(ptr, map, len);
2174 				/* enum/sizeof string smaller than value */
2175 				if (WARN_ON_ONCE(!ptr))
2176 					return;
2177 				/*
2178 				 * No need to decrement here, as eval_replace()
2179 				 * returns the pointer to the character passed
2180 				 * the eval, and two evals can not be placed
2181 				 * back to back without something in between.
2182 				 * We can skip that something in between.
2183 				 */
2184 				continue;
2185 			}
2186 		skip_more:
2187 			do {
2188 				ptr++;
2189 			} while (isalnum(*ptr) || *ptr == '_');
2190 			if (!*ptr)
2191 				break;
2192 			/*
2193 			 * If what comes after this variable is a '.' or
2194 			 * '->' then we can continue to ignore that string.
2195 			 */
2196 			if (*ptr == '.' || (ptr[0] == '-' && ptr[1] == '>')) {
2197 				ptr += *ptr == '.' ? 1 : 2;
2198 				if (!*ptr)
2199 					break;
2200 				goto skip_more;
2201 			}
2202 			/*
2203 			 * Once again, we can skip the delimiter that came
2204 			 * after the string.
2205 			 */
2206 			continue;
2207 		}
2208 	}
2209 }
2210 
2211 void trace_event_eval_update(struct trace_eval_map **map, int len)
2212 {
2213 	struct trace_event_call *call, *p;
2214 	const char *last_system = NULL;
2215 	bool first = false;
2216 	int last_i;
2217 	int i;
2218 
2219 	down_write(&trace_event_sem);
2220 	list_for_each_entry_safe(call, p, &ftrace_events, list) {
2221 		/* events are usually grouped together with systems */
2222 		if (!last_system || call->class->system != last_system) {
2223 			first = true;
2224 			last_i = 0;
2225 			last_system = call->class->system;
2226 		}
2227 
2228 		/*
2229 		 * Since calls are grouped by systems, the likelyhood that the
2230 		 * next call in the iteration belongs to the same system as the
2231 		 * previous call is high. As an optimization, we skip seaching
2232 		 * for a map[] that matches the call's system if the last call
2233 		 * was from the same system. That's what last_i is for. If the
2234 		 * call has the same system as the previous call, then last_i
2235 		 * will be the index of the first map[] that has a matching
2236 		 * system.
2237 		 */
2238 		for (i = last_i; i < len; i++) {
2239 			if (call->class->system == map[i]->system) {
2240 				/* Save the first system if need be */
2241 				if (first) {
2242 					last_i = i;
2243 					first = false;
2244 				}
2245 				update_event_printk(call, map[i]);
2246 			}
2247 		}
2248 	}
2249 	up_write(&trace_event_sem);
2250 }
2251 
2252 static struct trace_event_file *
2253 trace_create_new_event(struct trace_event_call *call,
2254 		       struct trace_array *tr)
2255 {
2256 	struct trace_event_file *file;
2257 
2258 	file = kmem_cache_alloc(file_cachep, GFP_TRACE);
2259 	if (!file)
2260 		return NULL;
2261 
2262 	file->event_call = call;
2263 	file->tr = tr;
2264 	atomic_set(&file->sm_ref, 0);
2265 	atomic_set(&file->tm_ref, 0);
2266 	INIT_LIST_HEAD(&file->triggers);
2267 	list_add(&file->list, &tr->events);
2268 
2269 	return file;
2270 }
2271 
2272 /* Add an event to a trace directory */
2273 static int
2274 __trace_add_new_event(struct trace_event_call *call, struct trace_array *tr)
2275 {
2276 	struct trace_event_file *file;
2277 
2278 	file = trace_create_new_event(call, tr);
2279 	if (!file)
2280 		return -ENOMEM;
2281 
2282 	return event_create_dir(tr->event_dir, file);
2283 }
2284 
2285 /*
2286  * Just create a decriptor for early init. A descriptor is required
2287  * for enabling events at boot. We want to enable events before
2288  * the filesystem is initialized.
2289  */
2290 static __init int
2291 __trace_early_add_new_event(struct trace_event_call *call,
2292 			    struct trace_array *tr)
2293 {
2294 	struct trace_event_file *file;
2295 
2296 	file = trace_create_new_event(call, tr);
2297 	if (!file)
2298 		return -ENOMEM;
2299 
2300 	return 0;
2301 }
2302 
2303 struct ftrace_module_file_ops;
2304 static void __add_event_to_tracers(struct trace_event_call *call);
2305 
2306 /* Add an additional event_call dynamically */
2307 int trace_add_event_call(struct trace_event_call *call)
2308 {
2309 	int ret;
2310 	lockdep_assert_held(&event_mutex);
2311 
2312 	mutex_lock(&trace_types_lock);
2313 
2314 	ret = __register_event(call, NULL);
2315 	if (ret >= 0)
2316 		__add_event_to_tracers(call);
2317 
2318 	mutex_unlock(&trace_types_lock);
2319 	return ret;
2320 }
2321 
2322 /*
2323  * Must be called under locking of trace_types_lock, event_mutex and
2324  * trace_event_sem.
2325  */
2326 static void __trace_remove_event_call(struct trace_event_call *call)
2327 {
2328 	event_remove(call);
2329 	trace_destroy_fields(call);
2330 	free_event_filter(call->filter);
2331 	call->filter = NULL;
2332 }
2333 
2334 static int probe_remove_event_call(struct trace_event_call *call)
2335 {
2336 	struct trace_array *tr;
2337 	struct trace_event_file *file;
2338 
2339 #ifdef CONFIG_PERF_EVENTS
2340 	if (call->perf_refcount)
2341 		return -EBUSY;
2342 #endif
2343 	do_for_each_event_file(tr, file) {
2344 		if (file->event_call != call)
2345 			continue;
2346 		/*
2347 		 * We can't rely on ftrace_event_enable_disable(enable => 0)
2348 		 * we are going to do, EVENT_FILE_FL_SOFT_MODE can suppress
2349 		 * TRACE_REG_UNREGISTER.
2350 		 */
2351 		if (file->flags & EVENT_FILE_FL_ENABLED)
2352 			return -EBUSY;
2353 		/*
2354 		 * The do_for_each_event_file_safe() is
2355 		 * a double loop. After finding the call for this
2356 		 * trace_array, we use break to jump to the next
2357 		 * trace_array.
2358 		 */
2359 		break;
2360 	} while_for_each_event_file();
2361 
2362 	__trace_remove_event_call(call);
2363 
2364 	return 0;
2365 }
2366 
2367 /* Remove an event_call */
2368 int trace_remove_event_call(struct trace_event_call *call)
2369 {
2370 	int ret;
2371 
2372 	lockdep_assert_held(&event_mutex);
2373 
2374 	mutex_lock(&trace_types_lock);
2375 	down_write(&trace_event_sem);
2376 	ret = probe_remove_event_call(call);
2377 	up_write(&trace_event_sem);
2378 	mutex_unlock(&trace_types_lock);
2379 
2380 	return ret;
2381 }
2382 
2383 #define for_each_event(event, start, end)			\
2384 	for (event = start;					\
2385 	     (unsigned long)event < (unsigned long)end;		\
2386 	     event++)
2387 
2388 #ifdef CONFIG_MODULES
2389 
2390 static void trace_module_add_events(struct module *mod)
2391 {
2392 	struct trace_event_call **call, **start, **end;
2393 
2394 	if (!mod->num_trace_events)
2395 		return;
2396 
2397 	/* Don't add infrastructure for mods without tracepoints */
2398 	if (trace_module_has_bad_taint(mod)) {
2399 		pr_err("%s: module has bad taint, not creating trace events\n",
2400 		       mod->name);
2401 		return;
2402 	}
2403 
2404 	start = mod->trace_events;
2405 	end = mod->trace_events + mod->num_trace_events;
2406 
2407 	for_each_event(call, start, end) {
2408 		__register_event(*call, mod);
2409 		__add_event_to_tracers(*call);
2410 	}
2411 }
2412 
2413 static void trace_module_remove_events(struct module *mod)
2414 {
2415 	struct trace_event_call *call, *p;
2416 
2417 	down_write(&trace_event_sem);
2418 	list_for_each_entry_safe(call, p, &ftrace_events, list) {
2419 		if (call->mod == mod)
2420 			__trace_remove_event_call(call);
2421 	}
2422 	up_write(&trace_event_sem);
2423 
2424 	/*
2425 	 * It is safest to reset the ring buffer if the module being unloaded
2426 	 * registered any events that were used. The only worry is if
2427 	 * a new module gets loaded, and takes on the same id as the events
2428 	 * of this module. When printing out the buffer, traced events left
2429 	 * over from this module may be passed to the new module events and
2430 	 * unexpected results may occur.
2431 	 */
2432 	tracing_reset_all_online_cpus();
2433 }
2434 
2435 static int trace_module_notify(struct notifier_block *self,
2436 			       unsigned long val, void *data)
2437 {
2438 	struct module *mod = data;
2439 
2440 	mutex_lock(&event_mutex);
2441 	mutex_lock(&trace_types_lock);
2442 	switch (val) {
2443 	case MODULE_STATE_COMING:
2444 		trace_module_add_events(mod);
2445 		break;
2446 	case MODULE_STATE_GOING:
2447 		trace_module_remove_events(mod);
2448 		break;
2449 	}
2450 	mutex_unlock(&trace_types_lock);
2451 	mutex_unlock(&event_mutex);
2452 
2453 	return 0;
2454 }
2455 
2456 static struct notifier_block trace_module_nb = {
2457 	.notifier_call = trace_module_notify,
2458 	.priority = 1, /* higher than trace.c module notify */
2459 };
2460 #endif /* CONFIG_MODULES */
2461 
2462 /* Create a new event directory structure for a trace directory. */
2463 static void
2464 __trace_add_event_dirs(struct trace_array *tr)
2465 {
2466 	struct trace_event_call *call;
2467 	int ret;
2468 
2469 	list_for_each_entry(call, &ftrace_events, list) {
2470 		ret = __trace_add_new_event(call, tr);
2471 		if (ret < 0)
2472 			pr_warn("Could not create directory for event %s\n",
2473 				trace_event_name(call));
2474 	}
2475 }
2476 
2477 /* Returns any file that matches the system and event */
2478 struct trace_event_file *
2479 __find_event_file(struct trace_array *tr, const char *system, const char *event)
2480 {
2481 	struct trace_event_file *file;
2482 	struct trace_event_call *call;
2483 	const char *name;
2484 
2485 	list_for_each_entry(file, &tr->events, list) {
2486 
2487 		call = file->event_call;
2488 		name = trace_event_name(call);
2489 
2490 		if (!name || !call->class)
2491 			continue;
2492 
2493 		if (strcmp(event, name) == 0 &&
2494 		    strcmp(system, call->class->system) == 0)
2495 			return file;
2496 	}
2497 	return NULL;
2498 }
2499 
2500 /* Returns valid trace event files that match system and event */
2501 struct trace_event_file *
2502 find_event_file(struct trace_array *tr, const char *system, const char *event)
2503 {
2504 	struct trace_event_file *file;
2505 
2506 	file = __find_event_file(tr, system, event);
2507 	if (!file || !file->event_call->class->reg ||
2508 	    file->event_call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
2509 		return NULL;
2510 
2511 	return file;
2512 }
2513 
2514 #ifdef CONFIG_DYNAMIC_FTRACE
2515 
2516 /* Avoid typos */
2517 #define ENABLE_EVENT_STR	"enable_event"
2518 #define DISABLE_EVENT_STR	"disable_event"
2519 
2520 struct event_probe_data {
2521 	struct trace_event_file	*file;
2522 	unsigned long			count;
2523 	int				ref;
2524 	bool				enable;
2525 };
2526 
2527 static void update_event_probe(struct event_probe_data *data)
2528 {
2529 	if (data->enable)
2530 		clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
2531 	else
2532 		set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
2533 }
2534 
2535 static void
2536 event_enable_probe(unsigned long ip, unsigned long parent_ip,
2537 		   struct trace_array *tr, struct ftrace_probe_ops *ops,
2538 		   void *data)
2539 {
2540 	struct ftrace_func_mapper *mapper = data;
2541 	struct event_probe_data *edata;
2542 	void **pdata;
2543 
2544 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
2545 	if (!pdata || !*pdata)
2546 		return;
2547 
2548 	edata = *pdata;
2549 	update_event_probe(edata);
2550 }
2551 
2552 static void
2553 event_enable_count_probe(unsigned long ip, unsigned long parent_ip,
2554 			 struct trace_array *tr, struct ftrace_probe_ops *ops,
2555 			 void *data)
2556 {
2557 	struct ftrace_func_mapper *mapper = data;
2558 	struct event_probe_data *edata;
2559 	void **pdata;
2560 
2561 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
2562 	if (!pdata || !*pdata)
2563 		return;
2564 
2565 	edata = *pdata;
2566 
2567 	if (!edata->count)
2568 		return;
2569 
2570 	/* Skip if the event is in a state we want to switch to */
2571 	if (edata->enable == !(edata->file->flags & EVENT_FILE_FL_SOFT_DISABLED))
2572 		return;
2573 
2574 	if (edata->count != -1)
2575 		(edata->count)--;
2576 
2577 	update_event_probe(edata);
2578 }
2579 
2580 static int
2581 event_enable_print(struct seq_file *m, unsigned long ip,
2582 		   struct ftrace_probe_ops *ops, void *data)
2583 {
2584 	struct ftrace_func_mapper *mapper = data;
2585 	struct event_probe_data *edata;
2586 	void **pdata;
2587 
2588 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
2589 
2590 	if (WARN_ON_ONCE(!pdata || !*pdata))
2591 		return 0;
2592 
2593 	edata = *pdata;
2594 
2595 	seq_printf(m, "%ps:", (void *)ip);
2596 
2597 	seq_printf(m, "%s:%s:%s",
2598 		   edata->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
2599 		   edata->file->event_call->class->system,
2600 		   trace_event_name(edata->file->event_call));
2601 
2602 	if (edata->count == -1)
2603 		seq_puts(m, ":unlimited\n");
2604 	else
2605 		seq_printf(m, ":count=%ld\n", edata->count);
2606 
2607 	return 0;
2608 }
2609 
2610 static int
2611 event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
2612 		  unsigned long ip, void *init_data, void **data)
2613 {
2614 	struct ftrace_func_mapper *mapper = *data;
2615 	struct event_probe_data *edata = init_data;
2616 	int ret;
2617 
2618 	if (!mapper) {
2619 		mapper = allocate_ftrace_func_mapper();
2620 		if (!mapper)
2621 			return -ENODEV;
2622 		*data = mapper;
2623 	}
2624 
2625 	ret = ftrace_func_mapper_add_ip(mapper, ip, edata);
2626 	if (ret < 0)
2627 		return ret;
2628 
2629 	edata->ref++;
2630 
2631 	return 0;
2632 }
2633 
2634 static int free_probe_data(void *data)
2635 {
2636 	struct event_probe_data *edata = data;
2637 
2638 	edata->ref--;
2639 	if (!edata->ref) {
2640 		/* Remove the SOFT_MODE flag */
2641 		__ftrace_event_enable_disable(edata->file, 0, 1);
2642 		module_put(edata->file->event_call->mod);
2643 		kfree(edata);
2644 	}
2645 	return 0;
2646 }
2647 
2648 static void
2649 event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
2650 		  unsigned long ip, void *data)
2651 {
2652 	struct ftrace_func_mapper *mapper = data;
2653 	struct event_probe_data *edata;
2654 
2655 	if (!ip) {
2656 		if (!mapper)
2657 			return;
2658 		free_ftrace_func_mapper(mapper, free_probe_data);
2659 		return;
2660 	}
2661 
2662 	edata = ftrace_func_mapper_remove_ip(mapper, ip);
2663 
2664 	if (WARN_ON_ONCE(!edata))
2665 		return;
2666 
2667 	if (WARN_ON_ONCE(edata->ref <= 0))
2668 		return;
2669 
2670 	free_probe_data(edata);
2671 }
2672 
2673 static struct ftrace_probe_ops event_enable_probe_ops = {
2674 	.func			= event_enable_probe,
2675 	.print			= event_enable_print,
2676 	.init			= event_enable_init,
2677 	.free			= event_enable_free,
2678 };
2679 
2680 static struct ftrace_probe_ops event_enable_count_probe_ops = {
2681 	.func			= event_enable_count_probe,
2682 	.print			= event_enable_print,
2683 	.init			= event_enable_init,
2684 	.free			= event_enable_free,
2685 };
2686 
2687 static struct ftrace_probe_ops event_disable_probe_ops = {
2688 	.func			= event_enable_probe,
2689 	.print			= event_enable_print,
2690 	.init			= event_enable_init,
2691 	.free			= event_enable_free,
2692 };
2693 
2694 static struct ftrace_probe_ops event_disable_count_probe_ops = {
2695 	.func			= event_enable_count_probe,
2696 	.print			= event_enable_print,
2697 	.init			= event_enable_init,
2698 	.free			= event_enable_free,
2699 };
2700 
2701 static int
2702 event_enable_func(struct trace_array *tr, struct ftrace_hash *hash,
2703 		  char *glob, char *cmd, char *param, int enabled)
2704 {
2705 	struct trace_event_file *file;
2706 	struct ftrace_probe_ops *ops;
2707 	struct event_probe_data *data;
2708 	const char *system;
2709 	const char *event;
2710 	char *number;
2711 	bool enable;
2712 	int ret;
2713 
2714 	if (!tr)
2715 		return -ENODEV;
2716 
2717 	/* hash funcs only work with set_ftrace_filter */
2718 	if (!enabled || !param)
2719 		return -EINVAL;
2720 
2721 	system = strsep(&param, ":");
2722 	if (!param)
2723 		return -EINVAL;
2724 
2725 	event = strsep(&param, ":");
2726 
2727 	mutex_lock(&event_mutex);
2728 
2729 	ret = -EINVAL;
2730 	file = find_event_file(tr, system, event);
2731 	if (!file)
2732 		goto out;
2733 
2734 	enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
2735 
2736 	if (enable)
2737 		ops = param ? &event_enable_count_probe_ops : &event_enable_probe_ops;
2738 	else
2739 		ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops;
2740 
2741 	if (glob[0] == '!') {
2742 		ret = unregister_ftrace_function_probe_func(glob+1, tr, ops);
2743 		goto out;
2744 	}
2745 
2746 	ret = -ENOMEM;
2747 
2748 	data = kzalloc(sizeof(*data), GFP_KERNEL);
2749 	if (!data)
2750 		goto out;
2751 
2752 	data->enable = enable;
2753 	data->count = -1;
2754 	data->file = file;
2755 
2756 	if (!param)
2757 		goto out_reg;
2758 
2759 	number = strsep(&param, ":");
2760 
2761 	ret = -EINVAL;
2762 	if (!strlen(number))
2763 		goto out_free;
2764 
2765 	/*
2766 	 * We use the callback data field (which is a pointer)
2767 	 * as our counter.
2768 	 */
2769 	ret = kstrtoul(number, 0, &data->count);
2770 	if (ret)
2771 		goto out_free;
2772 
2773  out_reg:
2774 	/* Don't let event modules unload while probe registered */
2775 	ret = try_module_get(file->event_call->mod);
2776 	if (!ret) {
2777 		ret = -EBUSY;
2778 		goto out_free;
2779 	}
2780 
2781 	ret = __ftrace_event_enable_disable(file, 1, 1);
2782 	if (ret < 0)
2783 		goto out_put;
2784 
2785 	ret = register_ftrace_function_probe(glob, tr, ops, data);
2786 	/*
2787 	 * The above returns on success the # of functions enabled,
2788 	 * but if it didn't find any functions it returns zero.
2789 	 * Consider no functions a failure too.
2790 	 */
2791 	if (!ret) {
2792 		ret = -ENOENT;
2793 		goto out_disable;
2794 	} else if (ret < 0)
2795 		goto out_disable;
2796 	/* Just return zero, not the number of enabled functions */
2797 	ret = 0;
2798  out:
2799 	mutex_unlock(&event_mutex);
2800 	return ret;
2801 
2802  out_disable:
2803 	__ftrace_event_enable_disable(file, 0, 1);
2804  out_put:
2805 	module_put(file->event_call->mod);
2806  out_free:
2807 	kfree(data);
2808 	goto out;
2809 }
2810 
2811 static struct ftrace_func_command event_enable_cmd = {
2812 	.name			= ENABLE_EVENT_STR,
2813 	.func			= event_enable_func,
2814 };
2815 
2816 static struct ftrace_func_command event_disable_cmd = {
2817 	.name			= DISABLE_EVENT_STR,
2818 	.func			= event_enable_func,
2819 };
2820 
2821 static __init int register_event_cmds(void)
2822 {
2823 	int ret;
2824 
2825 	ret = register_ftrace_command(&event_enable_cmd);
2826 	if (WARN_ON(ret < 0))
2827 		return ret;
2828 	ret = register_ftrace_command(&event_disable_cmd);
2829 	if (WARN_ON(ret < 0))
2830 		unregister_ftrace_command(&event_enable_cmd);
2831 	return ret;
2832 }
2833 #else
2834 static inline int register_event_cmds(void) { return 0; }
2835 #endif /* CONFIG_DYNAMIC_FTRACE */
2836 
2837 /*
2838  * The top level array has already had its trace_event_file
2839  * descriptors created in order to allow for early events to
2840  * be recorded. This function is called after the tracefs has been
2841  * initialized, and we now have to create the files associated
2842  * to the events.
2843  */
2844 static __init void
2845 __trace_early_add_event_dirs(struct trace_array *tr)
2846 {
2847 	struct trace_event_file *file;
2848 	int ret;
2849 
2850 
2851 	list_for_each_entry(file, &tr->events, list) {
2852 		ret = event_create_dir(tr->event_dir, file);
2853 		if (ret < 0)
2854 			pr_warn("Could not create directory for event %s\n",
2855 				trace_event_name(file->event_call));
2856 	}
2857 }
2858 
2859 /*
2860  * For early boot up, the top trace array requires to have
2861  * a list of events that can be enabled. This must be done before
2862  * the filesystem is set up in order to allow events to be traced
2863  * early.
2864  */
2865 static __init void
2866 __trace_early_add_events(struct trace_array *tr)
2867 {
2868 	struct trace_event_call *call;
2869 	int ret;
2870 
2871 	list_for_each_entry(call, &ftrace_events, list) {
2872 		/* Early boot up should not have any modules loaded */
2873 		if (WARN_ON_ONCE(call->mod))
2874 			continue;
2875 
2876 		ret = __trace_early_add_new_event(call, tr);
2877 		if (ret < 0)
2878 			pr_warn("Could not create early event %s\n",
2879 				trace_event_name(call));
2880 	}
2881 }
2882 
2883 /* Remove the event directory structure for a trace directory. */
2884 static void
2885 __trace_remove_event_dirs(struct trace_array *tr)
2886 {
2887 	struct trace_event_file *file, *next;
2888 
2889 	list_for_each_entry_safe(file, next, &tr->events, list)
2890 		remove_event_file_dir(file);
2891 }
2892 
2893 static void __add_event_to_tracers(struct trace_event_call *call)
2894 {
2895 	struct trace_array *tr;
2896 
2897 	list_for_each_entry(tr, &ftrace_trace_arrays, list)
2898 		__trace_add_new_event(call, tr);
2899 }
2900 
2901 extern struct trace_event_call *__start_ftrace_events[];
2902 extern struct trace_event_call *__stop_ftrace_events[];
2903 
2904 static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata;
2905 
2906 static __init int setup_trace_event(char *str)
2907 {
2908 	strlcpy(bootup_event_buf, str, COMMAND_LINE_SIZE);
2909 	ring_buffer_expanded = true;
2910 	tracing_selftest_disabled = true;
2911 
2912 	return 1;
2913 }
2914 __setup("trace_event=", setup_trace_event);
2915 
2916 /* Expects to have event_mutex held when called */
2917 static int
2918 create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
2919 {
2920 	struct dentry *d_events;
2921 	struct dentry *entry;
2922 
2923 	entry = tracefs_create_file("set_event", 0644, parent,
2924 				    tr, &ftrace_set_event_fops);
2925 	if (!entry) {
2926 		pr_warn("Could not create tracefs 'set_event' entry\n");
2927 		return -ENOMEM;
2928 	}
2929 
2930 	d_events = tracefs_create_dir("events", parent);
2931 	if (!d_events) {
2932 		pr_warn("Could not create tracefs 'events' directory\n");
2933 		return -ENOMEM;
2934 	}
2935 
2936 	entry = trace_create_file("enable", 0644, d_events,
2937 				  tr, &ftrace_tr_enable_fops);
2938 	if (!entry) {
2939 		pr_warn("Could not create tracefs 'enable' entry\n");
2940 		return -ENOMEM;
2941 	}
2942 
2943 	/* There are not as crucial, just warn if they are not created */
2944 
2945 	entry = tracefs_create_file("set_event_pid", 0644, parent,
2946 				    tr, &ftrace_set_event_pid_fops);
2947 	if (!entry)
2948 		pr_warn("Could not create tracefs 'set_event_pid' entry\n");
2949 
2950 	/* ring buffer internal formats */
2951 	entry = trace_create_file("header_page", 0444, d_events,
2952 				  ring_buffer_print_page_header,
2953 				  &ftrace_show_header_fops);
2954 	if (!entry)
2955 		pr_warn("Could not create tracefs 'header_page' entry\n");
2956 
2957 	entry = trace_create_file("header_event", 0444, d_events,
2958 				  ring_buffer_print_entry_header,
2959 				  &ftrace_show_header_fops);
2960 	if (!entry)
2961 		pr_warn("Could not create tracefs 'header_event' entry\n");
2962 
2963 	tr->event_dir = d_events;
2964 
2965 	return 0;
2966 }
2967 
2968 /**
2969  * event_trace_add_tracer - add a instance of a trace_array to events
2970  * @parent: The parent dentry to place the files/directories for events in
2971  * @tr: The trace array associated with these events
2972  *
2973  * When a new instance is created, it needs to set up its events
2974  * directory, as well as other files associated with events. It also
2975  * creates the event hierachry in the @parent/events directory.
2976  *
2977  * Returns 0 on success.
2978  *
2979  * Must be called with event_mutex held.
2980  */
2981 int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr)
2982 {
2983 	int ret;
2984 
2985 	lockdep_assert_held(&event_mutex);
2986 
2987 	ret = create_event_toplevel_files(parent, tr);
2988 	if (ret)
2989 		goto out;
2990 
2991 	down_write(&trace_event_sem);
2992 	__trace_add_event_dirs(tr);
2993 	up_write(&trace_event_sem);
2994 
2995  out:
2996 	return ret;
2997 }
2998 
2999 /*
3000  * The top trace array already had its file descriptors created.
3001  * Now the files themselves need to be created.
3002  */
3003 static __init int
3004 early_event_add_tracer(struct dentry *parent, struct trace_array *tr)
3005 {
3006 	int ret;
3007 
3008 	mutex_lock(&event_mutex);
3009 
3010 	ret = create_event_toplevel_files(parent, tr);
3011 	if (ret)
3012 		goto out_unlock;
3013 
3014 	down_write(&trace_event_sem);
3015 	__trace_early_add_event_dirs(tr);
3016 	up_write(&trace_event_sem);
3017 
3018  out_unlock:
3019 	mutex_unlock(&event_mutex);
3020 
3021 	return ret;
3022 }
3023 
3024 /* Must be called with event_mutex held */
3025 int event_trace_del_tracer(struct trace_array *tr)
3026 {
3027 	lockdep_assert_held(&event_mutex);
3028 
3029 	/* Disable any event triggers and associated soft-disabled events */
3030 	clear_event_triggers(tr);
3031 
3032 	/* Clear the pid list */
3033 	__ftrace_clear_event_pids(tr);
3034 
3035 	/* Disable any running events */
3036 	__ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0);
3037 
3038 	/* Make sure no more events are being executed */
3039 	tracepoint_synchronize_unregister();
3040 
3041 	down_write(&trace_event_sem);
3042 	__trace_remove_event_dirs(tr);
3043 	tracefs_remove_recursive(tr->event_dir);
3044 	up_write(&trace_event_sem);
3045 
3046 	tr->event_dir = NULL;
3047 
3048 	return 0;
3049 }
3050 
3051 static __init int event_trace_memsetup(void)
3052 {
3053 	field_cachep = KMEM_CACHE(ftrace_event_field, SLAB_PANIC);
3054 	file_cachep = KMEM_CACHE(trace_event_file, SLAB_PANIC);
3055 	return 0;
3056 }
3057 
3058 static __init void
3059 early_enable_events(struct trace_array *tr, bool disable_first)
3060 {
3061 	char *buf = bootup_event_buf;
3062 	char *token;
3063 	int ret;
3064 
3065 	while (true) {
3066 		token = strsep(&buf, ",");
3067 
3068 		if (!token)
3069 			break;
3070 
3071 		if (*token) {
3072 			/* Restarting syscalls requires that we stop them first */
3073 			if (disable_first)
3074 				ftrace_set_clr_event(tr, token, 0);
3075 
3076 			ret = ftrace_set_clr_event(tr, token, 1);
3077 			if (ret)
3078 				pr_warn("Failed to enable trace event: %s\n", token);
3079 		}
3080 
3081 		/* Put back the comma to allow this to be called again */
3082 		if (buf)
3083 			*(buf - 1) = ',';
3084 	}
3085 }
3086 
3087 static __init int event_trace_enable(void)
3088 {
3089 	struct trace_array *tr = top_trace_array();
3090 	struct trace_event_call **iter, *call;
3091 	int ret;
3092 
3093 	if (!tr)
3094 		return -ENODEV;
3095 
3096 	for_each_event(iter, __start_ftrace_events, __stop_ftrace_events) {
3097 
3098 		call = *iter;
3099 		ret = event_init(call);
3100 		if (!ret)
3101 			list_add(&call->list, &ftrace_events);
3102 	}
3103 
3104 	/*
3105 	 * We need the top trace array to have a working set of trace
3106 	 * points at early init, before the debug files and directories
3107 	 * are created. Create the file entries now, and attach them
3108 	 * to the actual file dentries later.
3109 	 */
3110 	__trace_early_add_events(tr);
3111 
3112 	early_enable_events(tr, false);
3113 
3114 	trace_printk_start_comm();
3115 
3116 	register_event_cmds();
3117 
3118 	register_trigger_cmds();
3119 
3120 	return 0;
3121 }
3122 
3123 /*
3124  * event_trace_enable() is called from trace_event_init() first to
3125  * initialize events and perhaps start any events that are on the
3126  * command line. Unfortunately, there are some events that will not
3127  * start this early, like the system call tracepoints that need
3128  * to set the TIF_SYSCALL_TRACEPOINT flag of pid 1. But event_trace_enable()
3129  * is called before pid 1 starts, and this flag is never set, making
3130  * the syscall tracepoint never get reached, but the event is enabled
3131  * regardless (and not doing anything).
3132  */
3133 static __init int event_trace_enable_again(void)
3134 {
3135 	struct trace_array *tr;
3136 
3137 	tr = top_trace_array();
3138 	if (!tr)
3139 		return -ENODEV;
3140 
3141 	early_enable_events(tr, true);
3142 
3143 	return 0;
3144 }
3145 
3146 early_initcall(event_trace_enable_again);
3147 
3148 __init int event_trace_init(void)
3149 {
3150 	struct trace_array *tr;
3151 	struct dentry *d_tracer;
3152 	struct dentry *entry;
3153 	int ret;
3154 
3155 	tr = top_trace_array();
3156 	if (!tr)
3157 		return -ENODEV;
3158 
3159 	d_tracer = tracing_init_dentry();
3160 	if (IS_ERR(d_tracer))
3161 		return 0;
3162 
3163 	entry = tracefs_create_file("available_events", 0444, d_tracer,
3164 				    tr, &ftrace_avail_fops);
3165 	if (!entry)
3166 		pr_warn("Could not create tracefs 'available_events' entry\n");
3167 
3168 	if (trace_define_generic_fields())
3169 		pr_warn("tracing: Failed to allocated generic fields");
3170 
3171 	if (trace_define_common_fields())
3172 		pr_warn("tracing: Failed to allocate common fields");
3173 
3174 	ret = early_event_add_tracer(d_tracer, tr);
3175 	if (ret)
3176 		return ret;
3177 
3178 #ifdef CONFIG_MODULES
3179 	ret = register_module_notifier(&trace_module_nb);
3180 	if (ret)
3181 		pr_warn("Failed to register trace events module notifier\n");
3182 #endif
3183 	return 0;
3184 }
3185 
3186 void __init trace_event_init(void)
3187 {
3188 	event_trace_memsetup();
3189 	init_ftrace_syscalls();
3190 	event_trace_enable();
3191 }
3192 
3193 #ifdef CONFIG_FTRACE_STARTUP_TEST
3194 
3195 static DEFINE_SPINLOCK(test_spinlock);
3196 static DEFINE_SPINLOCK(test_spinlock_irq);
3197 static DEFINE_MUTEX(test_mutex);
3198 
3199 static __init void test_work(struct work_struct *dummy)
3200 {
3201 	spin_lock(&test_spinlock);
3202 	spin_lock_irq(&test_spinlock_irq);
3203 	udelay(1);
3204 	spin_unlock_irq(&test_spinlock_irq);
3205 	spin_unlock(&test_spinlock);
3206 
3207 	mutex_lock(&test_mutex);
3208 	msleep(1);
3209 	mutex_unlock(&test_mutex);
3210 }
3211 
3212 static __init int event_test_thread(void *unused)
3213 {
3214 	void *test_malloc;
3215 
3216 	test_malloc = kmalloc(1234, GFP_KERNEL);
3217 	if (!test_malloc)
3218 		pr_info("failed to kmalloc\n");
3219 
3220 	schedule_on_each_cpu(test_work);
3221 
3222 	kfree(test_malloc);
3223 
3224 	set_current_state(TASK_INTERRUPTIBLE);
3225 	while (!kthread_should_stop()) {
3226 		schedule();
3227 		set_current_state(TASK_INTERRUPTIBLE);
3228 	}
3229 	__set_current_state(TASK_RUNNING);
3230 
3231 	return 0;
3232 }
3233 
3234 /*
3235  * Do various things that may trigger events.
3236  */
3237 static __init void event_test_stuff(void)
3238 {
3239 	struct task_struct *test_thread;
3240 
3241 	test_thread = kthread_run(event_test_thread, NULL, "test-events");
3242 	msleep(1);
3243 	kthread_stop(test_thread);
3244 }
3245 
3246 /*
3247  * For every trace event defined, we will test each trace point separately,
3248  * and then by groups, and finally all trace points.
3249  */
3250 static __init void event_trace_self_tests(void)
3251 {
3252 	struct trace_subsystem_dir *dir;
3253 	struct trace_event_file *file;
3254 	struct trace_event_call *call;
3255 	struct event_subsystem *system;
3256 	struct trace_array *tr;
3257 	int ret;
3258 
3259 	tr = top_trace_array();
3260 	if (!tr)
3261 		return;
3262 
3263 	pr_info("Running tests on trace events:\n");
3264 
3265 	list_for_each_entry(file, &tr->events, list) {
3266 
3267 		call = file->event_call;
3268 
3269 		/* Only test those that have a probe */
3270 		if (!call->class || !call->class->probe)
3271 			continue;
3272 
3273 /*
3274  * Testing syscall events here is pretty useless, but
3275  * we still do it if configured. But this is time consuming.
3276  * What we really need is a user thread to perform the
3277  * syscalls as we test.
3278  */
3279 #ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS
3280 		if (call->class->system &&
3281 		    strcmp(call->class->system, "syscalls") == 0)
3282 			continue;
3283 #endif
3284 
3285 		pr_info("Testing event %s: ", trace_event_name(call));
3286 
3287 		/*
3288 		 * If an event is already enabled, someone is using
3289 		 * it and the self test should not be on.
3290 		 */
3291 		if (file->flags & EVENT_FILE_FL_ENABLED) {
3292 			pr_warn("Enabled event during self test!\n");
3293 			WARN_ON_ONCE(1);
3294 			continue;
3295 		}
3296 
3297 		ftrace_event_enable_disable(file, 1);
3298 		event_test_stuff();
3299 		ftrace_event_enable_disable(file, 0);
3300 
3301 		pr_cont("OK\n");
3302 	}
3303 
3304 	/* Now test at the sub system level */
3305 
3306 	pr_info("Running tests on trace event systems:\n");
3307 
3308 	list_for_each_entry(dir, &tr->systems, list) {
3309 
3310 		system = dir->subsystem;
3311 
3312 		/* the ftrace system is special, skip it */
3313 		if (strcmp(system->name, "ftrace") == 0)
3314 			continue;
3315 
3316 		pr_info("Testing event system %s: ", system->name);
3317 
3318 		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1);
3319 		if (WARN_ON_ONCE(ret)) {
3320 			pr_warn("error enabling system %s\n",
3321 				system->name);
3322 			continue;
3323 		}
3324 
3325 		event_test_stuff();
3326 
3327 		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0);
3328 		if (WARN_ON_ONCE(ret)) {
3329 			pr_warn("error disabling system %s\n",
3330 				system->name);
3331 			continue;
3332 		}
3333 
3334 		pr_cont("OK\n");
3335 	}
3336 
3337 	/* Test with all events enabled */
3338 
3339 	pr_info("Running tests on all trace events:\n");
3340 	pr_info("Testing all events: ");
3341 
3342 	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1);
3343 	if (WARN_ON_ONCE(ret)) {
3344 		pr_warn("error enabling all events\n");
3345 		return;
3346 	}
3347 
3348 	event_test_stuff();
3349 
3350 	/* reset sysname */
3351 	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0);
3352 	if (WARN_ON_ONCE(ret)) {
3353 		pr_warn("error disabling all events\n");
3354 		return;
3355 	}
3356 
3357 	pr_cont("OK\n");
3358 }
3359 
3360 #ifdef CONFIG_FUNCTION_TRACER
3361 
3362 static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
3363 
3364 static struct trace_event_file event_trace_file __initdata;
3365 
3366 static void __init
3367 function_test_events_call(unsigned long ip, unsigned long parent_ip,
3368 			  struct ftrace_ops *op, struct pt_regs *pt_regs)
3369 {
3370 	struct ring_buffer_event *event;
3371 	struct ring_buffer *buffer;
3372 	struct ftrace_entry *entry;
3373 	unsigned long flags;
3374 	long disabled;
3375 	int cpu;
3376 	int pc;
3377 
3378 	pc = preempt_count();
3379 	preempt_disable_notrace();
3380 	cpu = raw_smp_processor_id();
3381 	disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
3382 
3383 	if (disabled != 1)
3384 		goto out;
3385 
3386 	local_save_flags(flags);
3387 
3388 	event = trace_event_buffer_lock_reserve(&buffer, &event_trace_file,
3389 						TRACE_FN, sizeof(*entry),
3390 						flags, pc);
3391 	if (!event)
3392 		goto out;
3393 	entry	= ring_buffer_event_data(event);
3394 	entry->ip			= ip;
3395 	entry->parent_ip		= parent_ip;
3396 
3397 	event_trigger_unlock_commit(&event_trace_file, buffer, event,
3398 				    entry, flags, pc);
3399  out:
3400 	atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
3401 	preempt_enable_notrace();
3402 }
3403 
3404 static struct ftrace_ops trace_ops __initdata  =
3405 {
3406 	.func = function_test_events_call,
3407 	.flags = FTRACE_OPS_FL_RECURSION_SAFE,
3408 };
3409 
3410 static __init void event_trace_self_test_with_function(void)
3411 {
3412 	int ret;
3413 
3414 	event_trace_file.tr = top_trace_array();
3415 	if (WARN_ON(!event_trace_file.tr))
3416 		return;
3417 
3418 	ret = register_ftrace_function(&trace_ops);
3419 	if (WARN_ON(ret < 0)) {
3420 		pr_info("Failed to enable function tracer for event tests\n");
3421 		return;
3422 	}
3423 	pr_info("Running tests again, along with the function tracer\n");
3424 	event_trace_self_tests();
3425 	unregister_ftrace_function(&trace_ops);
3426 }
3427 #else
3428 static __init void event_trace_self_test_with_function(void)
3429 {
3430 }
3431 #endif
3432 
3433 static __init int event_trace_self_tests_init(void)
3434 {
3435 	if (!tracing_selftest_disabled) {
3436 		event_trace_self_tests();
3437 		event_trace_self_test_with_function();
3438 	}
3439 
3440 	return 0;
3441 }
3442 
3443 late_initcall(event_trace_self_tests_init);
3444 
3445 #endif
3446