1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * ring buffer based function tracer
4  *
5  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7  *
8  * Based on code from the latency_tracer, that is:
9  *
10  *  Copyright (C) 2004-2006 Ingo Molnar
11  *  Copyright (C) 2004 Nadia Yvette Chambers
12  */
13 #include <linux/ring_buffer.h>
14 #include <linux/debugfs.h>
15 #include <linux/uaccess.h>
16 #include <linux/ftrace.h>
17 #include <linux/slab.h>
18 #include <linux/fs.h>
19 
20 #include "trace.h"
21 
22 static void tracing_start_function_trace(struct trace_array *tr);
23 static void tracing_stop_function_trace(struct trace_array *tr);
24 static void
25 function_trace_call(unsigned long ip, unsigned long parent_ip,
26 		    struct ftrace_ops *op, struct pt_regs *pt_regs);
27 static void
28 function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
29 			  struct ftrace_ops *op, struct pt_regs *pt_regs);
30 static struct tracer_flags func_flags;
31 
32 /* Our option */
33 enum {
34 	TRACE_FUNC_OPT_STACK	= 0x1,
35 };
36 
37 int ftrace_allocate_ftrace_ops(struct trace_array *tr)
38 {
39 	struct ftrace_ops *ops;
40 
41 	/* The top level array uses the "global_ops" */
42 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
43 		return 0;
44 
45 	ops = kzalloc(sizeof(*ops), GFP_KERNEL);
46 	if (!ops)
47 		return -ENOMEM;
48 
49 	/* Currently only the non stack version is supported */
50 	ops->func = function_trace_call;
51 	ops->flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_PID;
52 
53 	tr->ops = ops;
54 	ops->private = tr;
55 
56 	return 0;
57 }
58 
59 void ftrace_free_ftrace_ops(struct trace_array *tr)
60 {
61 	kfree(tr->ops);
62 	tr->ops = NULL;
63 }
64 
65 int ftrace_create_function_files(struct trace_array *tr,
66 				 struct dentry *parent)
67 {
68 	/*
69 	 * The top level array uses the "global_ops", and the files are
70 	 * created on boot up.
71 	 */
72 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
73 		return 0;
74 
75 	if (!tr->ops)
76 		return -EINVAL;
77 
78 	ftrace_create_filter_files(tr->ops, parent);
79 
80 	return 0;
81 }
82 
83 void ftrace_destroy_function_files(struct trace_array *tr)
84 {
85 	ftrace_destroy_filter_files(tr->ops);
86 	ftrace_free_ftrace_ops(tr);
87 }
88 
89 static int function_trace_init(struct trace_array *tr)
90 {
91 	ftrace_func_t func;
92 
93 	/*
94 	 * Instance trace_arrays get their ops allocated
95 	 * at instance creation. Unless it failed
96 	 * the allocation.
97 	 */
98 	if (!tr->ops)
99 		return -ENOMEM;
100 
101 	/* Currently only the global instance can do stack tracing */
102 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL &&
103 	    func_flags.val & TRACE_FUNC_OPT_STACK)
104 		func = function_stack_trace_call;
105 	else
106 		func = function_trace_call;
107 
108 	ftrace_init_array_ops(tr, func);
109 
110 	tr->array_buffer.cpu = get_cpu();
111 	put_cpu();
112 
113 	tracing_start_cmdline_record();
114 	tracing_start_function_trace(tr);
115 	return 0;
116 }
117 
118 static void function_trace_reset(struct trace_array *tr)
119 {
120 	tracing_stop_function_trace(tr);
121 	tracing_stop_cmdline_record();
122 	ftrace_reset_array_ops(tr);
123 }
124 
125 static void function_trace_start(struct trace_array *tr)
126 {
127 	tracing_reset_online_cpus(&tr->array_buffer);
128 }
129 
130 static void
131 function_trace_call(unsigned long ip, unsigned long parent_ip,
132 		    struct ftrace_ops *op, struct pt_regs *pt_regs)
133 {
134 	struct trace_array *tr = op->private;
135 	struct trace_array_cpu *data;
136 	unsigned long flags;
137 	int bit;
138 	int cpu;
139 	int pc;
140 
141 	if (unlikely(!tr->function_enabled))
142 		return;
143 
144 	pc = preempt_count();
145 	preempt_disable_notrace();
146 
147 	bit = trace_test_and_set_recursion(TRACE_FTRACE_START, TRACE_FTRACE_MAX);
148 	if (bit < 0)
149 		goto out;
150 
151 	cpu = smp_processor_id();
152 	data = per_cpu_ptr(tr->array_buffer.data, cpu);
153 	if (!atomic_read(&data->disabled)) {
154 		local_save_flags(flags);
155 		trace_function(tr, ip, parent_ip, flags, pc);
156 	}
157 	trace_clear_recursion(bit);
158 
159  out:
160 	preempt_enable_notrace();
161 }
162 
163 #ifdef CONFIG_UNWINDER_ORC
164 /*
165  * Skip 2:
166  *
167  *   function_stack_trace_call()
168  *   ftrace_call()
169  */
170 #define STACK_SKIP 2
171 #else
172 /*
173  * Skip 3:
174  *   __trace_stack()
175  *   function_stack_trace_call()
176  *   ftrace_call()
177  */
178 #define STACK_SKIP 3
179 #endif
180 
181 static void
182 function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
183 			  struct ftrace_ops *op, struct pt_regs *pt_regs)
184 {
185 	struct trace_array *tr = op->private;
186 	struct trace_array_cpu *data;
187 	unsigned long flags;
188 	long disabled;
189 	int cpu;
190 	int pc;
191 
192 	if (unlikely(!tr->function_enabled))
193 		return;
194 
195 	/*
196 	 * Need to use raw, since this must be called before the
197 	 * recursive protection is performed.
198 	 */
199 	local_irq_save(flags);
200 	cpu = raw_smp_processor_id();
201 	data = per_cpu_ptr(tr->array_buffer.data, cpu);
202 	disabled = atomic_inc_return(&data->disabled);
203 
204 	if (likely(disabled == 1)) {
205 		pc = preempt_count();
206 		trace_function(tr, ip, parent_ip, flags, pc);
207 		__trace_stack(tr, flags, STACK_SKIP, pc);
208 	}
209 
210 	atomic_dec(&data->disabled);
211 	local_irq_restore(flags);
212 }
213 
214 static struct tracer_opt func_opts[] = {
215 #ifdef CONFIG_STACKTRACE
216 	{ TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) },
217 #endif
218 	{ } /* Always set a last empty entry */
219 };
220 
221 static struct tracer_flags func_flags = {
222 	.val = 0, /* By default: all flags disabled */
223 	.opts = func_opts
224 };
225 
226 static void tracing_start_function_trace(struct trace_array *tr)
227 {
228 	tr->function_enabled = 0;
229 	register_ftrace_function(tr->ops);
230 	tr->function_enabled = 1;
231 }
232 
233 static void tracing_stop_function_trace(struct trace_array *tr)
234 {
235 	tr->function_enabled = 0;
236 	unregister_ftrace_function(tr->ops);
237 }
238 
239 static struct tracer function_trace;
240 
241 static int
242 func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
243 {
244 	switch (bit) {
245 	case TRACE_FUNC_OPT_STACK:
246 		/* do nothing if already set */
247 		if (!!set == !!(func_flags.val & TRACE_FUNC_OPT_STACK))
248 			break;
249 
250 		/* We can change this flag when not running. */
251 		if (tr->current_trace != &function_trace)
252 			break;
253 
254 		unregister_ftrace_function(tr->ops);
255 
256 		if (set) {
257 			tr->ops->func = function_stack_trace_call;
258 			register_ftrace_function(tr->ops);
259 		} else {
260 			tr->ops->func = function_trace_call;
261 			register_ftrace_function(tr->ops);
262 		}
263 
264 		break;
265 	default:
266 		return -EINVAL;
267 	}
268 
269 	return 0;
270 }
271 
272 static struct tracer function_trace __tracer_data =
273 {
274 	.name		= "function",
275 	.init		= function_trace_init,
276 	.reset		= function_trace_reset,
277 	.start		= function_trace_start,
278 	.flags		= &func_flags,
279 	.set_flag	= func_set_flag,
280 	.allow_instances = true,
281 #ifdef CONFIG_FTRACE_SELFTEST
282 	.selftest	= trace_selftest_startup_function,
283 #endif
284 };
285 
286 #ifdef CONFIG_DYNAMIC_FTRACE
287 static void update_traceon_count(struct ftrace_probe_ops *ops,
288 				 unsigned long ip,
289 				 struct trace_array *tr, bool on,
290 				 void *data)
291 {
292 	struct ftrace_func_mapper *mapper = data;
293 	long *count;
294 	long old_count;
295 
296 	/*
297 	 * Tracing gets disabled (or enabled) once per count.
298 	 * This function can be called at the same time on multiple CPUs.
299 	 * It is fine if both disable (or enable) tracing, as disabling
300 	 * (or enabling) the second time doesn't do anything as the
301 	 * state of the tracer is already disabled (or enabled).
302 	 * What needs to be synchronized in this case is that the count
303 	 * only gets decremented once, even if the tracer is disabled
304 	 * (or enabled) twice, as the second one is really a nop.
305 	 *
306 	 * The memory barriers guarantee that we only decrement the
307 	 * counter once. First the count is read to a local variable
308 	 * and a read barrier is used to make sure that it is loaded
309 	 * before checking if the tracer is in the state we want.
310 	 * If the tracer is not in the state we want, then the count
311 	 * is guaranteed to be the old count.
312 	 *
313 	 * Next the tracer is set to the state we want (disabled or enabled)
314 	 * then a write memory barrier is used to make sure that
315 	 * the new state is visible before changing the counter by
316 	 * one minus the old counter. This guarantees that another CPU
317 	 * executing this code will see the new state before seeing
318 	 * the new counter value, and would not do anything if the new
319 	 * counter is seen.
320 	 *
321 	 * Note, there is no synchronization between this and a user
322 	 * setting the tracing_on file. But we currently don't care
323 	 * about that.
324 	 */
325 	count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
326 	old_count = *count;
327 
328 	if (old_count <= 0)
329 		return;
330 
331 	/* Make sure we see count before checking tracing state */
332 	smp_rmb();
333 
334 	if (on == !!tracer_tracing_is_on(tr))
335 		return;
336 
337 	if (on)
338 		tracer_tracing_on(tr);
339 	else
340 		tracer_tracing_off(tr);
341 
342 	/* Make sure tracing state is visible before updating count */
343 	smp_wmb();
344 
345 	*count = old_count - 1;
346 }
347 
348 static void
349 ftrace_traceon_count(unsigned long ip, unsigned long parent_ip,
350 		     struct trace_array *tr, struct ftrace_probe_ops *ops,
351 		     void *data)
352 {
353 	update_traceon_count(ops, ip, tr, 1, data);
354 }
355 
356 static void
357 ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip,
358 		      struct trace_array *tr, struct ftrace_probe_ops *ops,
359 		      void *data)
360 {
361 	update_traceon_count(ops, ip, tr, 0, data);
362 }
363 
364 static void
365 ftrace_traceon(unsigned long ip, unsigned long parent_ip,
366 	       struct trace_array *tr, struct ftrace_probe_ops *ops,
367 	       void *data)
368 {
369 	if (tracer_tracing_is_on(tr))
370 		return;
371 
372 	tracer_tracing_on(tr);
373 }
374 
375 static void
376 ftrace_traceoff(unsigned long ip, unsigned long parent_ip,
377 		struct trace_array *tr, struct ftrace_probe_ops *ops,
378 		void *data)
379 {
380 	if (!tracer_tracing_is_on(tr))
381 		return;
382 
383 	tracer_tracing_off(tr);
384 }
385 
386 #ifdef CONFIG_UNWINDER_ORC
387 /*
388  * Skip 3:
389  *
390  *   function_trace_probe_call()
391  *   ftrace_ops_assist_func()
392  *   ftrace_call()
393  */
394 #define FTRACE_STACK_SKIP 3
395 #else
396 /*
397  * Skip 5:
398  *
399  *   __trace_stack()
400  *   ftrace_stacktrace()
401  *   function_trace_probe_call()
402  *   ftrace_ops_assist_func()
403  *   ftrace_call()
404  */
405 #define FTRACE_STACK_SKIP 5
406 #endif
407 
408 static __always_inline void trace_stack(struct trace_array *tr)
409 {
410 	unsigned long flags;
411 	int pc;
412 
413 	local_save_flags(flags);
414 	pc = preempt_count();
415 
416 	__trace_stack(tr, flags, FTRACE_STACK_SKIP, pc);
417 }
418 
419 static void
420 ftrace_stacktrace(unsigned long ip, unsigned long parent_ip,
421 		  struct trace_array *tr, struct ftrace_probe_ops *ops,
422 		  void *data)
423 {
424 	trace_stack(tr);
425 }
426 
427 static void
428 ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip,
429 			struct trace_array *tr, struct ftrace_probe_ops *ops,
430 			void *data)
431 {
432 	struct ftrace_func_mapper *mapper = data;
433 	long *count;
434 	long old_count;
435 	long new_count;
436 
437 	if (!tracing_is_on())
438 		return;
439 
440 	/* unlimited? */
441 	if (!mapper) {
442 		trace_stack(tr);
443 		return;
444 	}
445 
446 	count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
447 
448 	/*
449 	 * Stack traces should only execute the number of times the
450 	 * user specified in the counter.
451 	 */
452 	do {
453 		old_count = *count;
454 
455 		if (!old_count)
456 			return;
457 
458 		new_count = old_count - 1;
459 		new_count = cmpxchg(count, old_count, new_count);
460 		if (new_count == old_count)
461 			trace_stack(tr);
462 
463 		if (!tracing_is_on())
464 			return;
465 
466 	} while (new_count != old_count);
467 }
468 
469 static int update_count(struct ftrace_probe_ops *ops, unsigned long ip,
470 			void *data)
471 {
472 	struct ftrace_func_mapper *mapper = data;
473 	long *count = NULL;
474 
475 	if (mapper)
476 		count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
477 
478 	if (count) {
479 		if (*count <= 0)
480 			return 0;
481 		(*count)--;
482 	}
483 
484 	return 1;
485 }
486 
487 static void
488 ftrace_dump_probe(unsigned long ip, unsigned long parent_ip,
489 		  struct trace_array *tr, struct ftrace_probe_ops *ops,
490 		  void *data)
491 {
492 	if (update_count(ops, ip, data))
493 		ftrace_dump(DUMP_ALL);
494 }
495 
496 /* Only dump the current CPU buffer. */
497 static void
498 ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip,
499 		     struct trace_array *tr, struct ftrace_probe_ops *ops,
500 		     void *data)
501 {
502 	if (update_count(ops, ip, data))
503 		ftrace_dump(DUMP_ORIG);
504 }
505 
506 static int
507 ftrace_probe_print(const char *name, struct seq_file *m,
508 		   unsigned long ip, struct ftrace_probe_ops *ops,
509 		   void *data)
510 {
511 	struct ftrace_func_mapper *mapper = data;
512 	long *count = NULL;
513 
514 	seq_printf(m, "%ps:%s", (void *)ip, name);
515 
516 	if (mapper)
517 		count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
518 
519 	if (count)
520 		seq_printf(m, ":count=%ld\n", *count);
521 	else
522 		seq_puts(m, ":unlimited\n");
523 
524 	return 0;
525 }
526 
527 static int
528 ftrace_traceon_print(struct seq_file *m, unsigned long ip,
529 		     struct ftrace_probe_ops *ops,
530 		     void *data)
531 {
532 	return ftrace_probe_print("traceon", m, ip, ops, data);
533 }
534 
535 static int
536 ftrace_traceoff_print(struct seq_file *m, unsigned long ip,
537 			 struct ftrace_probe_ops *ops, void *data)
538 {
539 	return ftrace_probe_print("traceoff", m, ip, ops, data);
540 }
541 
542 static int
543 ftrace_stacktrace_print(struct seq_file *m, unsigned long ip,
544 			struct ftrace_probe_ops *ops, void *data)
545 {
546 	return ftrace_probe_print("stacktrace", m, ip, ops, data);
547 }
548 
549 static int
550 ftrace_dump_print(struct seq_file *m, unsigned long ip,
551 			struct ftrace_probe_ops *ops, void *data)
552 {
553 	return ftrace_probe_print("dump", m, ip, ops, data);
554 }
555 
556 static int
557 ftrace_cpudump_print(struct seq_file *m, unsigned long ip,
558 			struct ftrace_probe_ops *ops, void *data)
559 {
560 	return ftrace_probe_print("cpudump", m, ip, ops, data);
561 }
562 
563 
564 static int
565 ftrace_count_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
566 		  unsigned long ip, void *init_data, void **data)
567 {
568 	struct ftrace_func_mapper *mapper = *data;
569 
570 	if (!mapper) {
571 		mapper = allocate_ftrace_func_mapper();
572 		if (!mapper)
573 			return -ENOMEM;
574 		*data = mapper;
575 	}
576 
577 	return ftrace_func_mapper_add_ip(mapper, ip, init_data);
578 }
579 
580 static void
581 ftrace_count_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
582 		  unsigned long ip, void *data)
583 {
584 	struct ftrace_func_mapper *mapper = data;
585 
586 	if (!ip) {
587 		free_ftrace_func_mapper(mapper, NULL);
588 		return;
589 	}
590 
591 	ftrace_func_mapper_remove_ip(mapper, ip);
592 }
593 
594 static struct ftrace_probe_ops traceon_count_probe_ops = {
595 	.func			= ftrace_traceon_count,
596 	.print			= ftrace_traceon_print,
597 	.init			= ftrace_count_init,
598 	.free			= ftrace_count_free,
599 };
600 
601 static struct ftrace_probe_ops traceoff_count_probe_ops = {
602 	.func			= ftrace_traceoff_count,
603 	.print			= ftrace_traceoff_print,
604 	.init			= ftrace_count_init,
605 	.free			= ftrace_count_free,
606 };
607 
608 static struct ftrace_probe_ops stacktrace_count_probe_ops = {
609 	.func			= ftrace_stacktrace_count,
610 	.print			= ftrace_stacktrace_print,
611 	.init			= ftrace_count_init,
612 	.free			= ftrace_count_free,
613 };
614 
615 static struct ftrace_probe_ops dump_probe_ops = {
616 	.func			= ftrace_dump_probe,
617 	.print			= ftrace_dump_print,
618 	.init			= ftrace_count_init,
619 	.free			= ftrace_count_free,
620 };
621 
622 static struct ftrace_probe_ops cpudump_probe_ops = {
623 	.func			= ftrace_cpudump_probe,
624 	.print			= ftrace_cpudump_print,
625 };
626 
627 static struct ftrace_probe_ops traceon_probe_ops = {
628 	.func			= ftrace_traceon,
629 	.print			= ftrace_traceon_print,
630 };
631 
632 static struct ftrace_probe_ops traceoff_probe_ops = {
633 	.func			= ftrace_traceoff,
634 	.print			= ftrace_traceoff_print,
635 };
636 
637 static struct ftrace_probe_ops stacktrace_probe_ops = {
638 	.func			= ftrace_stacktrace,
639 	.print			= ftrace_stacktrace_print,
640 };
641 
642 static int
643 ftrace_trace_probe_callback(struct trace_array *tr,
644 			    struct ftrace_probe_ops *ops,
645 			    struct ftrace_hash *hash, char *glob,
646 			    char *cmd, char *param, int enable)
647 {
648 	void *count = (void *)-1;
649 	char *number;
650 	int ret;
651 
652 	/* hash funcs only work with set_ftrace_filter */
653 	if (!enable)
654 		return -EINVAL;
655 
656 	if (glob[0] == '!')
657 		return unregister_ftrace_function_probe_func(glob+1, tr, ops);
658 
659 	if (!param)
660 		goto out_reg;
661 
662 	number = strsep(&param, ":");
663 
664 	if (!strlen(number))
665 		goto out_reg;
666 
667 	/*
668 	 * We use the callback data field (which is a pointer)
669 	 * as our counter.
670 	 */
671 	ret = kstrtoul(number, 0, (unsigned long *)&count);
672 	if (ret)
673 		return ret;
674 
675  out_reg:
676 	ret = register_ftrace_function_probe(glob, tr, ops, count);
677 
678 	return ret < 0 ? ret : 0;
679 }
680 
681 static int
682 ftrace_trace_onoff_callback(struct trace_array *tr, struct ftrace_hash *hash,
683 			    char *glob, char *cmd, char *param, int enable)
684 {
685 	struct ftrace_probe_ops *ops;
686 
687 	if (!tr)
688 		return -ENODEV;
689 
690 	/* we register both traceon and traceoff to this callback */
691 	if (strcmp(cmd, "traceon") == 0)
692 		ops = param ? &traceon_count_probe_ops : &traceon_probe_ops;
693 	else
694 		ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops;
695 
696 	return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
697 					   param, enable);
698 }
699 
700 static int
701 ftrace_stacktrace_callback(struct trace_array *tr, struct ftrace_hash *hash,
702 			   char *glob, char *cmd, char *param, int enable)
703 {
704 	struct ftrace_probe_ops *ops;
705 
706 	if (!tr)
707 		return -ENODEV;
708 
709 	ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops;
710 
711 	return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
712 					   param, enable);
713 }
714 
715 static int
716 ftrace_dump_callback(struct trace_array *tr, struct ftrace_hash *hash,
717 			   char *glob, char *cmd, char *param, int enable)
718 {
719 	struct ftrace_probe_ops *ops;
720 
721 	if (!tr)
722 		return -ENODEV;
723 
724 	ops = &dump_probe_ops;
725 
726 	/* Only dump once. */
727 	return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
728 					   "1", enable);
729 }
730 
731 static int
732 ftrace_cpudump_callback(struct trace_array *tr, struct ftrace_hash *hash,
733 			   char *glob, char *cmd, char *param, int enable)
734 {
735 	struct ftrace_probe_ops *ops;
736 
737 	if (!tr)
738 		return -ENODEV;
739 
740 	ops = &cpudump_probe_ops;
741 
742 	/* Only dump once. */
743 	return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
744 					   "1", enable);
745 }
746 
747 static struct ftrace_func_command ftrace_traceon_cmd = {
748 	.name			= "traceon",
749 	.func			= ftrace_trace_onoff_callback,
750 };
751 
752 static struct ftrace_func_command ftrace_traceoff_cmd = {
753 	.name			= "traceoff",
754 	.func			= ftrace_trace_onoff_callback,
755 };
756 
757 static struct ftrace_func_command ftrace_stacktrace_cmd = {
758 	.name			= "stacktrace",
759 	.func			= ftrace_stacktrace_callback,
760 };
761 
762 static struct ftrace_func_command ftrace_dump_cmd = {
763 	.name			= "dump",
764 	.func			= ftrace_dump_callback,
765 };
766 
767 static struct ftrace_func_command ftrace_cpudump_cmd = {
768 	.name			= "cpudump",
769 	.func			= ftrace_cpudump_callback,
770 };
771 
772 static int __init init_func_cmd_traceon(void)
773 {
774 	int ret;
775 
776 	ret = register_ftrace_command(&ftrace_traceoff_cmd);
777 	if (ret)
778 		return ret;
779 
780 	ret = register_ftrace_command(&ftrace_traceon_cmd);
781 	if (ret)
782 		goto out_free_traceoff;
783 
784 	ret = register_ftrace_command(&ftrace_stacktrace_cmd);
785 	if (ret)
786 		goto out_free_traceon;
787 
788 	ret = register_ftrace_command(&ftrace_dump_cmd);
789 	if (ret)
790 		goto out_free_stacktrace;
791 
792 	ret = register_ftrace_command(&ftrace_cpudump_cmd);
793 	if (ret)
794 		goto out_free_dump;
795 
796 	return 0;
797 
798  out_free_dump:
799 	unregister_ftrace_command(&ftrace_dump_cmd);
800  out_free_stacktrace:
801 	unregister_ftrace_command(&ftrace_stacktrace_cmd);
802  out_free_traceon:
803 	unregister_ftrace_command(&ftrace_traceon_cmd);
804  out_free_traceoff:
805 	unregister_ftrace_command(&ftrace_traceoff_cmd);
806 
807 	return ret;
808 }
809 #else
810 static inline int init_func_cmd_traceon(void)
811 {
812 	return 0;
813 }
814 #endif /* CONFIG_DYNAMIC_FTRACE */
815 
816 __init int init_function_trace(void)
817 {
818 	init_func_cmd_traceon();
819 	return register_tracer(&function_trace);
820 }
821