xref: /openbmc/linux/kernel/trace/ftrace.c (revision 9d56dd3b083a3bec56e9da35ce07baca81030b03)
1 /*
2  * Infrastructure for profiling code inserted by 'gcc -pg'.
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally ported from the -rt patch by:
8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code in the latency_tracer, that is:
11  *
12  *  Copyright (C) 2004-2006 Ingo Molnar
13  *  Copyright (C) 2004 William Lee Irwin III
14  */
15 
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/suspend.h>
21 #include <linux/debugfs.h>
22 #include <linux/hardirq.h>
23 #include <linux/kthread.h>
24 #include <linux/uaccess.h>
25 #include <linux/kprobes.h>
26 #include <linux/ftrace.h>
27 #include <linux/sysctl.h>
28 #include <linux/ctype.h>
29 #include <linux/list.h>
30 #include <linux/hash.h>
31 
32 #include <trace/events/sched.h>
33 
34 #include <asm/ftrace.h>
35 #include <asm/setup.h>
36 
37 #include "trace_output.h"
38 #include "trace_stat.h"
39 
40 #define FTRACE_WARN_ON(cond)			\
41 	do {					\
42 		if (WARN_ON(cond))		\
43 			ftrace_kill();		\
44 	} while (0)
45 
46 #define FTRACE_WARN_ON_ONCE(cond)		\
47 	do {					\
48 		if (WARN_ON_ONCE(cond))		\
49 			ftrace_kill();		\
50 	} while (0)
51 
52 /* hash bits for specific function selection */
53 #define FTRACE_HASH_BITS 7
54 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
55 
56 /* ftrace_enabled is a method to turn ftrace on or off */
57 int ftrace_enabled __read_mostly;
58 static int last_ftrace_enabled;
59 
60 /* Quick disabling of function tracer. */
61 int function_trace_stop;
62 
63 /* List for set_ftrace_pid's pids. */
64 LIST_HEAD(ftrace_pids);
65 struct ftrace_pid {
66 	struct list_head list;
67 	struct pid *pid;
68 };
69 
70 /*
71  * ftrace_disabled is set when an anomaly is discovered.
72  * ftrace_disabled is much stronger than ftrace_enabled.
73  */
74 static int ftrace_disabled __read_mostly;
75 
76 static DEFINE_MUTEX(ftrace_lock);
77 
78 static struct ftrace_ops ftrace_list_end __read_mostly =
79 {
80 	.func		= ftrace_stub,
81 };
82 
83 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
84 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
85 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
86 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
87 
88 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
89 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
90 #endif
91 
92 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
93 {
94 	struct ftrace_ops *op = ftrace_list;
95 
96 	/* in case someone actually ports this to alpha! */
97 	read_barrier_depends();
98 
99 	while (op != &ftrace_list_end) {
100 		/* silly alpha */
101 		read_barrier_depends();
102 		op->func(ip, parent_ip);
103 		op = op->next;
104 	};
105 }
106 
107 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
108 {
109 	if (!test_tsk_trace_trace(current))
110 		return;
111 
112 	ftrace_pid_function(ip, parent_ip);
113 }
114 
115 static void set_ftrace_pid_function(ftrace_func_t func)
116 {
117 	/* do not set ftrace_pid_function to itself! */
118 	if (func != ftrace_pid_func)
119 		ftrace_pid_function = func;
120 }
121 
122 /**
123  * clear_ftrace_function - reset the ftrace function
124  *
125  * This NULLs the ftrace function and in essence stops
126  * tracing.  There may be lag
127  */
128 void clear_ftrace_function(void)
129 {
130 	ftrace_trace_function = ftrace_stub;
131 	__ftrace_trace_function = ftrace_stub;
132 	ftrace_pid_function = ftrace_stub;
133 }
134 
135 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
136 /*
137  * For those archs that do not test ftrace_trace_stop in their
138  * mcount call site, we need to do it from C.
139  */
140 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
141 {
142 	if (function_trace_stop)
143 		return;
144 
145 	__ftrace_trace_function(ip, parent_ip);
146 }
147 #endif
148 
149 static int __register_ftrace_function(struct ftrace_ops *ops)
150 {
151 	ops->next = ftrace_list;
152 	/*
153 	 * We are entering ops into the ftrace_list but another
154 	 * CPU might be walking that list. We need to make sure
155 	 * the ops->next pointer is valid before another CPU sees
156 	 * the ops pointer included into the ftrace_list.
157 	 */
158 	smp_wmb();
159 	ftrace_list = ops;
160 
161 	if (ftrace_enabled) {
162 		ftrace_func_t func;
163 
164 		if (ops->next == &ftrace_list_end)
165 			func = ops->func;
166 		else
167 			func = ftrace_list_func;
168 
169 		if (!list_empty(&ftrace_pids)) {
170 			set_ftrace_pid_function(func);
171 			func = ftrace_pid_func;
172 		}
173 
174 		/*
175 		 * For one func, simply call it directly.
176 		 * For more than one func, call the chain.
177 		 */
178 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
179 		ftrace_trace_function = func;
180 #else
181 		__ftrace_trace_function = func;
182 		ftrace_trace_function = ftrace_test_stop_func;
183 #endif
184 	}
185 
186 	return 0;
187 }
188 
189 static int __unregister_ftrace_function(struct ftrace_ops *ops)
190 {
191 	struct ftrace_ops **p;
192 
193 	/*
194 	 * If we are removing the last function, then simply point
195 	 * to the ftrace_stub.
196 	 */
197 	if (ftrace_list == ops && ops->next == &ftrace_list_end) {
198 		ftrace_trace_function = ftrace_stub;
199 		ftrace_list = &ftrace_list_end;
200 		return 0;
201 	}
202 
203 	for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
204 		if (*p == ops)
205 			break;
206 
207 	if (*p != ops)
208 		return -1;
209 
210 	*p = (*p)->next;
211 
212 	if (ftrace_enabled) {
213 		/* If we only have one func left, then call that directly */
214 		if (ftrace_list->next == &ftrace_list_end) {
215 			ftrace_func_t func = ftrace_list->func;
216 
217 			if (!list_empty(&ftrace_pids)) {
218 				set_ftrace_pid_function(func);
219 				func = ftrace_pid_func;
220 			}
221 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
222 			ftrace_trace_function = func;
223 #else
224 			__ftrace_trace_function = func;
225 #endif
226 		}
227 	}
228 
229 	return 0;
230 }
231 
232 static void ftrace_update_pid_func(void)
233 {
234 	ftrace_func_t func;
235 
236 	if (ftrace_trace_function == ftrace_stub)
237 		return;
238 
239 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
240 	func = ftrace_trace_function;
241 #else
242 	func = __ftrace_trace_function;
243 #endif
244 
245 	if (!list_empty(&ftrace_pids)) {
246 		set_ftrace_pid_function(func);
247 		func = ftrace_pid_func;
248 	} else {
249 		if (func == ftrace_pid_func)
250 			func = ftrace_pid_function;
251 	}
252 
253 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
254 	ftrace_trace_function = func;
255 #else
256 	__ftrace_trace_function = func;
257 #endif
258 }
259 
260 #ifdef CONFIG_FUNCTION_PROFILER
261 struct ftrace_profile {
262 	struct hlist_node		node;
263 	unsigned long			ip;
264 	unsigned long			counter;
265 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
266 	unsigned long long		time;
267 #endif
268 };
269 
270 struct ftrace_profile_page {
271 	struct ftrace_profile_page	*next;
272 	unsigned long			index;
273 	struct ftrace_profile		records[];
274 };
275 
276 struct ftrace_profile_stat {
277 	atomic_t			disabled;
278 	struct hlist_head		*hash;
279 	struct ftrace_profile_page	*pages;
280 	struct ftrace_profile_page	*start;
281 	struct tracer_stat		stat;
282 };
283 
284 #define PROFILE_RECORDS_SIZE						\
285 	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
286 
287 #define PROFILES_PER_PAGE					\
288 	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
289 
290 static int ftrace_profile_bits __read_mostly;
291 static int ftrace_profile_enabled __read_mostly;
292 
293 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
294 static DEFINE_MUTEX(ftrace_profile_lock);
295 
296 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
297 
298 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
299 
300 static void *
301 function_stat_next(void *v, int idx)
302 {
303 	struct ftrace_profile *rec = v;
304 	struct ftrace_profile_page *pg;
305 
306 	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
307 
308  again:
309 	if (idx != 0)
310 		rec++;
311 
312 	if ((void *)rec >= (void *)&pg->records[pg->index]) {
313 		pg = pg->next;
314 		if (!pg)
315 			return NULL;
316 		rec = &pg->records[0];
317 		if (!rec->counter)
318 			goto again;
319 	}
320 
321 	return rec;
322 }
323 
324 static void *function_stat_start(struct tracer_stat *trace)
325 {
326 	struct ftrace_profile_stat *stat =
327 		container_of(trace, struct ftrace_profile_stat, stat);
328 
329 	if (!stat || !stat->start)
330 		return NULL;
331 
332 	return function_stat_next(&stat->start->records[0], 0);
333 }
334 
335 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
336 /* function graph compares on total time */
337 static int function_stat_cmp(void *p1, void *p2)
338 {
339 	struct ftrace_profile *a = p1;
340 	struct ftrace_profile *b = p2;
341 
342 	if (a->time < b->time)
343 		return -1;
344 	if (a->time > b->time)
345 		return 1;
346 	else
347 		return 0;
348 }
349 #else
350 /* not function graph compares against hits */
351 static int function_stat_cmp(void *p1, void *p2)
352 {
353 	struct ftrace_profile *a = p1;
354 	struct ftrace_profile *b = p2;
355 
356 	if (a->counter < b->counter)
357 		return -1;
358 	if (a->counter > b->counter)
359 		return 1;
360 	else
361 		return 0;
362 }
363 #endif
364 
365 static int function_stat_headers(struct seq_file *m)
366 {
367 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
368 	seq_printf(m, "  Function                               "
369 		   "Hit    Time            Avg\n"
370 		      "  --------                               "
371 		   "---    ----            ---\n");
372 #else
373 	seq_printf(m, "  Function                               Hit\n"
374 		      "  --------                               ---\n");
375 #endif
376 	return 0;
377 }
378 
379 static int function_stat_show(struct seq_file *m, void *v)
380 {
381 	struct ftrace_profile *rec = v;
382 	char str[KSYM_SYMBOL_LEN];
383 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
384 	static DEFINE_MUTEX(mutex);
385 	static struct trace_seq s;
386 	unsigned long long avg;
387 #endif
388 
389 	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
390 	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
391 
392 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
393 	seq_printf(m, "    ");
394 	avg = rec->time;
395 	do_div(avg, rec->counter);
396 
397 	mutex_lock(&mutex);
398 	trace_seq_init(&s);
399 	trace_print_graph_duration(rec->time, &s);
400 	trace_seq_puts(&s, "    ");
401 	trace_print_graph_duration(avg, &s);
402 	trace_print_seq(m, &s);
403 	mutex_unlock(&mutex);
404 #endif
405 	seq_putc(m, '\n');
406 
407 	return 0;
408 }
409 
410 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
411 {
412 	struct ftrace_profile_page *pg;
413 
414 	pg = stat->pages = stat->start;
415 
416 	while (pg) {
417 		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
418 		pg->index = 0;
419 		pg = pg->next;
420 	}
421 
422 	memset(stat->hash, 0,
423 	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
424 }
425 
426 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
427 {
428 	struct ftrace_profile_page *pg;
429 	int functions;
430 	int pages;
431 	int i;
432 
433 	/* If we already allocated, do nothing */
434 	if (stat->pages)
435 		return 0;
436 
437 	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
438 	if (!stat->pages)
439 		return -ENOMEM;
440 
441 #ifdef CONFIG_DYNAMIC_FTRACE
442 	functions = ftrace_update_tot_cnt;
443 #else
444 	/*
445 	 * We do not know the number of functions that exist because
446 	 * dynamic tracing is what counts them. With past experience
447 	 * we have around 20K functions. That should be more than enough.
448 	 * It is highly unlikely we will execute every function in
449 	 * the kernel.
450 	 */
451 	functions = 20000;
452 #endif
453 
454 	pg = stat->start = stat->pages;
455 
456 	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
457 
458 	for (i = 0; i < pages; i++) {
459 		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
460 		if (!pg->next)
461 			goto out_free;
462 		pg = pg->next;
463 	}
464 
465 	return 0;
466 
467  out_free:
468 	pg = stat->start;
469 	while (pg) {
470 		unsigned long tmp = (unsigned long)pg;
471 
472 		pg = pg->next;
473 		free_page(tmp);
474 	}
475 
476 	free_page((unsigned long)stat->pages);
477 	stat->pages = NULL;
478 	stat->start = NULL;
479 
480 	return -ENOMEM;
481 }
482 
483 static int ftrace_profile_init_cpu(int cpu)
484 {
485 	struct ftrace_profile_stat *stat;
486 	int size;
487 
488 	stat = &per_cpu(ftrace_profile_stats, cpu);
489 
490 	if (stat->hash) {
491 		/* If the profile is already created, simply reset it */
492 		ftrace_profile_reset(stat);
493 		return 0;
494 	}
495 
496 	/*
497 	 * We are profiling all functions, but usually only a few thousand
498 	 * functions are hit. We'll make a hash of 1024 items.
499 	 */
500 	size = FTRACE_PROFILE_HASH_SIZE;
501 
502 	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
503 
504 	if (!stat->hash)
505 		return -ENOMEM;
506 
507 	if (!ftrace_profile_bits) {
508 		size--;
509 
510 		for (; size; size >>= 1)
511 			ftrace_profile_bits++;
512 	}
513 
514 	/* Preallocate the function profiling pages */
515 	if (ftrace_profile_pages_init(stat) < 0) {
516 		kfree(stat->hash);
517 		stat->hash = NULL;
518 		return -ENOMEM;
519 	}
520 
521 	return 0;
522 }
523 
524 static int ftrace_profile_init(void)
525 {
526 	int cpu;
527 	int ret = 0;
528 
529 	for_each_online_cpu(cpu) {
530 		ret = ftrace_profile_init_cpu(cpu);
531 		if (ret)
532 			break;
533 	}
534 
535 	return ret;
536 }
537 
538 /* interrupts must be disabled */
539 static struct ftrace_profile *
540 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
541 {
542 	struct ftrace_profile *rec;
543 	struct hlist_head *hhd;
544 	struct hlist_node *n;
545 	unsigned long key;
546 
547 	key = hash_long(ip, ftrace_profile_bits);
548 	hhd = &stat->hash[key];
549 
550 	if (hlist_empty(hhd))
551 		return NULL;
552 
553 	hlist_for_each_entry_rcu(rec, n, hhd, node) {
554 		if (rec->ip == ip)
555 			return rec;
556 	}
557 
558 	return NULL;
559 }
560 
561 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
562 			       struct ftrace_profile *rec)
563 {
564 	unsigned long key;
565 
566 	key = hash_long(rec->ip, ftrace_profile_bits);
567 	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
568 }
569 
570 /*
571  * The memory is already allocated, this simply finds a new record to use.
572  */
573 static struct ftrace_profile *
574 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
575 {
576 	struct ftrace_profile *rec = NULL;
577 
578 	/* prevent recursion (from NMIs) */
579 	if (atomic_inc_return(&stat->disabled) != 1)
580 		goto out;
581 
582 	/*
583 	 * Try to find the function again since an NMI
584 	 * could have added it
585 	 */
586 	rec = ftrace_find_profiled_func(stat, ip);
587 	if (rec)
588 		goto out;
589 
590 	if (stat->pages->index == PROFILES_PER_PAGE) {
591 		if (!stat->pages->next)
592 			goto out;
593 		stat->pages = stat->pages->next;
594 	}
595 
596 	rec = &stat->pages->records[stat->pages->index++];
597 	rec->ip = ip;
598 	ftrace_add_profile(stat, rec);
599 
600  out:
601 	atomic_dec(&stat->disabled);
602 
603 	return rec;
604 }
605 
606 static void
607 function_profile_call(unsigned long ip, unsigned long parent_ip)
608 {
609 	struct ftrace_profile_stat *stat;
610 	struct ftrace_profile *rec;
611 	unsigned long flags;
612 
613 	if (!ftrace_profile_enabled)
614 		return;
615 
616 	local_irq_save(flags);
617 
618 	stat = &__get_cpu_var(ftrace_profile_stats);
619 	if (!stat->hash || !ftrace_profile_enabled)
620 		goto out;
621 
622 	rec = ftrace_find_profiled_func(stat, ip);
623 	if (!rec) {
624 		rec = ftrace_profile_alloc(stat, ip);
625 		if (!rec)
626 			goto out;
627 	}
628 
629 	rec->counter++;
630  out:
631 	local_irq_restore(flags);
632 }
633 
634 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
635 static int profile_graph_entry(struct ftrace_graph_ent *trace)
636 {
637 	function_profile_call(trace->func, 0);
638 	return 1;
639 }
640 
641 static void profile_graph_return(struct ftrace_graph_ret *trace)
642 {
643 	struct ftrace_profile_stat *stat;
644 	unsigned long long calltime;
645 	struct ftrace_profile *rec;
646 	unsigned long flags;
647 
648 	local_irq_save(flags);
649 	stat = &__get_cpu_var(ftrace_profile_stats);
650 	if (!stat->hash || !ftrace_profile_enabled)
651 		goto out;
652 
653 	calltime = trace->rettime - trace->calltime;
654 
655 	if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
656 		int index;
657 
658 		index = trace->depth;
659 
660 		/* Append this call time to the parent time to subtract */
661 		if (index)
662 			current->ret_stack[index - 1].subtime += calltime;
663 
664 		if (current->ret_stack[index].subtime < calltime)
665 			calltime -= current->ret_stack[index].subtime;
666 		else
667 			calltime = 0;
668 	}
669 
670 	rec = ftrace_find_profiled_func(stat, trace->func);
671 	if (rec)
672 		rec->time += calltime;
673 
674  out:
675 	local_irq_restore(flags);
676 }
677 
678 static int register_ftrace_profiler(void)
679 {
680 	return register_ftrace_graph(&profile_graph_return,
681 				     &profile_graph_entry);
682 }
683 
684 static void unregister_ftrace_profiler(void)
685 {
686 	unregister_ftrace_graph();
687 }
688 #else
689 static struct ftrace_ops ftrace_profile_ops __read_mostly =
690 {
691 	.func		= function_profile_call,
692 };
693 
694 static int register_ftrace_profiler(void)
695 {
696 	return register_ftrace_function(&ftrace_profile_ops);
697 }
698 
699 static void unregister_ftrace_profiler(void)
700 {
701 	unregister_ftrace_function(&ftrace_profile_ops);
702 }
703 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
704 
705 static ssize_t
706 ftrace_profile_write(struct file *filp, const char __user *ubuf,
707 		     size_t cnt, loff_t *ppos)
708 {
709 	unsigned long val;
710 	char buf[64];		/* big enough to hold a number */
711 	int ret;
712 
713 	if (cnt >= sizeof(buf))
714 		return -EINVAL;
715 
716 	if (copy_from_user(&buf, ubuf, cnt))
717 		return -EFAULT;
718 
719 	buf[cnt] = 0;
720 
721 	ret = strict_strtoul(buf, 10, &val);
722 	if (ret < 0)
723 		return ret;
724 
725 	val = !!val;
726 
727 	mutex_lock(&ftrace_profile_lock);
728 	if (ftrace_profile_enabled ^ val) {
729 		if (val) {
730 			ret = ftrace_profile_init();
731 			if (ret < 0) {
732 				cnt = ret;
733 				goto out;
734 			}
735 
736 			ret = register_ftrace_profiler();
737 			if (ret < 0) {
738 				cnt = ret;
739 				goto out;
740 			}
741 			ftrace_profile_enabled = 1;
742 		} else {
743 			ftrace_profile_enabled = 0;
744 			/*
745 			 * unregister_ftrace_profiler calls stop_machine
746 			 * so this acts like an synchronize_sched.
747 			 */
748 			unregister_ftrace_profiler();
749 		}
750 	}
751  out:
752 	mutex_unlock(&ftrace_profile_lock);
753 
754 	*ppos += cnt;
755 
756 	return cnt;
757 }
758 
759 static ssize_t
760 ftrace_profile_read(struct file *filp, char __user *ubuf,
761 		     size_t cnt, loff_t *ppos)
762 {
763 	char buf[64];		/* big enough to hold a number */
764 	int r;
765 
766 	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
767 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
768 }
769 
770 static const struct file_operations ftrace_profile_fops = {
771 	.open		= tracing_open_generic,
772 	.read		= ftrace_profile_read,
773 	.write		= ftrace_profile_write,
774 };
775 
776 /* used to initialize the real stat files */
777 static struct tracer_stat function_stats __initdata = {
778 	.name		= "functions",
779 	.stat_start	= function_stat_start,
780 	.stat_next	= function_stat_next,
781 	.stat_cmp	= function_stat_cmp,
782 	.stat_headers	= function_stat_headers,
783 	.stat_show	= function_stat_show
784 };
785 
786 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
787 {
788 	struct ftrace_profile_stat *stat;
789 	struct dentry *entry;
790 	char *name;
791 	int ret;
792 	int cpu;
793 
794 	for_each_possible_cpu(cpu) {
795 		stat = &per_cpu(ftrace_profile_stats, cpu);
796 
797 		/* allocate enough for function name + cpu number */
798 		name = kmalloc(32, GFP_KERNEL);
799 		if (!name) {
800 			/*
801 			 * The files created are permanent, if something happens
802 			 * we still do not free memory.
803 			 */
804 			WARN(1,
805 			     "Could not allocate stat file for cpu %d\n",
806 			     cpu);
807 			return;
808 		}
809 		stat->stat = function_stats;
810 		snprintf(name, 32, "function%d", cpu);
811 		stat->stat.name = name;
812 		ret = register_stat_tracer(&stat->stat);
813 		if (ret) {
814 			WARN(1,
815 			     "Could not register function stat for cpu %d\n",
816 			     cpu);
817 			kfree(name);
818 			return;
819 		}
820 	}
821 
822 	entry = debugfs_create_file("function_profile_enabled", 0644,
823 				    d_tracer, NULL, &ftrace_profile_fops);
824 	if (!entry)
825 		pr_warning("Could not create debugfs "
826 			   "'function_profile_enabled' entry\n");
827 }
828 
829 #else /* CONFIG_FUNCTION_PROFILER */
830 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
831 {
832 }
833 #endif /* CONFIG_FUNCTION_PROFILER */
834 
835 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
836 
837 #ifdef CONFIG_DYNAMIC_FTRACE
838 
839 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
840 # error Dynamic ftrace depends on MCOUNT_RECORD
841 #endif
842 
843 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
844 
845 struct ftrace_func_probe {
846 	struct hlist_node	node;
847 	struct ftrace_probe_ops	*ops;
848 	unsigned long		flags;
849 	unsigned long		ip;
850 	void			*data;
851 	struct rcu_head		rcu;
852 };
853 
854 enum {
855 	FTRACE_ENABLE_CALLS		= (1 << 0),
856 	FTRACE_DISABLE_CALLS		= (1 << 1),
857 	FTRACE_UPDATE_TRACE_FUNC	= (1 << 2),
858 	FTRACE_ENABLE_MCOUNT		= (1 << 3),
859 	FTRACE_DISABLE_MCOUNT		= (1 << 4),
860 	FTRACE_START_FUNC_RET		= (1 << 5),
861 	FTRACE_STOP_FUNC_RET		= (1 << 6),
862 };
863 
864 static int ftrace_filtered;
865 
866 static struct dyn_ftrace *ftrace_new_addrs;
867 
868 static DEFINE_MUTEX(ftrace_regex_lock);
869 
870 struct ftrace_page {
871 	struct ftrace_page	*next;
872 	int			index;
873 	struct dyn_ftrace	records[];
874 };
875 
876 #define ENTRIES_PER_PAGE \
877   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
878 
879 /* estimate from running different kernels */
880 #define NR_TO_INIT		10000
881 
882 static struct ftrace_page	*ftrace_pages_start;
883 static struct ftrace_page	*ftrace_pages;
884 
885 static struct dyn_ftrace *ftrace_free_records;
886 
887 /*
888  * This is a double for. Do not use 'break' to break out of the loop,
889  * you must use a goto.
890  */
891 #define do_for_each_ftrace_rec(pg, rec)					\
892 	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
893 		int _____i;						\
894 		for (_____i = 0; _____i < pg->index; _____i++) {	\
895 			rec = &pg->records[_____i];
896 
897 #define while_for_each_ftrace_rec()		\
898 		}				\
899 	}
900 
901 #ifdef CONFIG_KPROBES
902 
903 static int frozen_record_count;
904 
905 static inline void freeze_record(struct dyn_ftrace *rec)
906 {
907 	if (!(rec->flags & FTRACE_FL_FROZEN)) {
908 		rec->flags |= FTRACE_FL_FROZEN;
909 		frozen_record_count++;
910 	}
911 }
912 
913 static inline void unfreeze_record(struct dyn_ftrace *rec)
914 {
915 	if (rec->flags & FTRACE_FL_FROZEN) {
916 		rec->flags &= ~FTRACE_FL_FROZEN;
917 		frozen_record_count--;
918 	}
919 }
920 
921 static inline int record_frozen(struct dyn_ftrace *rec)
922 {
923 	return rec->flags & FTRACE_FL_FROZEN;
924 }
925 #else
926 # define freeze_record(rec)			({ 0; })
927 # define unfreeze_record(rec)			({ 0; })
928 # define record_frozen(rec)			({ 0; })
929 #endif /* CONFIG_KPROBES */
930 
931 static void ftrace_free_rec(struct dyn_ftrace *rec)
932 {
933 	rec->freelist = ftrace_free_records;
934 	ftrace_free_records = rec;
935 	rec->flags |= FTRACE_FL_FREE;
936 }
937 
938 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
939 {
940 	struct dyn_ftrace *rec;
941 
942 	/* First check for freed records */
943 	if (ftrace_free_records) {
944 		rec = ftrace_free_records;
945 
946 		if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
947 			FTRACE_WARN_ON_ONCE(1);
948 			ftrace_free_records = NULL;
949 			return NULL;
950 		}
951 
952 		ftrace_free_records = rec->freelist;
953 		memset(rec, 0, sizeof(*rec));
954 		return rec;
955 	}
956 
957 	if (ftrace_pages->index == ENTRIES_PER_PAGE) {
958 		if (!ftrace_pages->next) {
959 			/* allocate another page */
960 			ftrace_pages->next =
961 				(void *)get_zeroed_page(GFP_KERNEL);
962 			if (!ftrace_pages->next)
963 				return NULL;
964 		}
965 		ftrace_pages = ftrace_pages->next;
966 	}
967 
968 	return &ftrace_pages->records[ftrace_pages->index++];
969 }
970 
971 static struct dyn_ftrace *
972 ftrace_record_ip(unsigned long ip)
973 {
974 	struct dyn_ftrace *rec;
975 
976 	if (ftrace_disabled)
977 		return NULL;
978 
979 	rec = ftrace_alloc_dyn_node(ip);
980 	if (!rec)
981 		return NULL;
982 
983 	rec->ip = ip;
984 	rec->newlist = ftrace_new_addrs;
985 	ftrace_new_addrs = rec;
986 
987 	return rec;
988 }
989 
990 static void print_ip_ins(const char *fmt, unsigned char *p)
991 {
992 	int i;
993 
994 	printk(KERN_CONT "%s", fmt);
995 
996 	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
997 		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
998 }
999 
1000 static void ftrace_bug(int failed, unsigned long ip)
1001 {
1002 	switch (failed) {
1003 	case -EFAULT:
1004 		FTRACE_WARN_ON_ONCE(1);
1005 		pr_info("ftrace faulted on modifying ");
1006 		print_ip_sym(ip);
1007 		break;
1008 	case -EINVAL:
1009 		FTRACE_WARN_ON_ONCE(1);
1010 		pr_info("ftrace failed to modify ");
1011 		print_ip_sym(ip);
1012 		print_ip_ins(" actual: ", (unsigned char *)ip);
1013 		printk(KERN_CONT "\n");
1014 		break;
1015 	case -EPERM:
1016 		FTRACE_WARN_ON_ONCE(1);
1017 		pr_info("ftrace faulted on writing ");
1018 		print_ip_sym(ip);
1019 		break;
1020 	default:
1021 		FTRACE_WARN_ON_ONCE(1);
1022 		pr_info("ftrace faulted on unknown error ");
1023 		print_ip_sym(ip);
1024 	}
1025 }
1026 
1027 
1028 static int
1029 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1030 {
1031 	unsigned long ftrace_addr;
1032 	unsigned long flag = 0UL;
1033 
1034 	ftrace_addr = (unsigned long)FTRACE_ADDR;
1035 
1036 	/*
1037 	 * If this record is not to be traced or we want to disable it,
1038 	 * then disable it.
1039 	 *
1040 	 * If we want to enable it and filtering is off, then enable it.
1041 	 *
1042 	 * If we want to enable it and filtering is on, enable it only if
1043 	 * it's filtered
1044 	 */
1045 	if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) {
1046 		if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER))
1047 			flag = FTRACE_FL_ENABLED;
1048 	}
1049 
1050 	/* If the state of this record hasn't changed, then do nothing */
1051 	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1052 		return 0;
1053 
1054 	if (flag) {
1055 		rec->flags |= FTRACE_FL_ENABLED;
1056 		return ftrace_make_call(rec, ftrace_addr);
1057 	}
1058 
1059 	rec->flags &= ~FTRACE_FL_ENABLED;
1060 	return ftrace_make_nop(NULL, rec, ftrace_addr);
1061 }
1062 
1063 static void ftrace_replace_code(int enable)
1064 {
1065 	struct dyn_ftrace *rec;
1066 	struct ftrace_page *pg;
1067 	int failed;
1068 
1069 	do_for_each_ftrace_rec(pg, rec) {
1070 		/*
1071 		 * Skip over free records, records that have
1072 		 * failed and not converted.
1073 		 */
1074 		if (rec->flags & FTRACE_FL_FREE ||
1075 		    rec->flags & FTRACE_FL_FAILED ||
1076 		    !(rec->flags & FTRACE_FL_CONVERTED))
1077 			continue;
1078 
1079 		/* ignore updates to this record's mcount site */
1080 		if (get_kprobe((void *)rec->ip)) {
1081 			freeze_record(rec);
1082 			continue;
1083 		} else {
1084 			unfreeze_record(rec);
1085 		}
1086 
1087 		failed = __ftrace_replace_code(rec, enable);
1088 		if (failed) {
1089 			rec->flags |= FTRACE_FL_FAILED;
1090 			ftrace_bug(failed, rec->ip);
1091 			/* Stop processing */
1092 			return;
1093 		}
1094 	} while_for_each_ftrace_rec();
1095 }
1096 
1097 static int
1098 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1099 {
1100 	unsigned long ip;
1101 	int ret;
1102 
1103 	ip = rec->ip;
1104 
1105 	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1106 	if (ret) {
1107 		ftrace_bug(ret, ip);
1108 		rec->flags |= FTRACE_FL_FAILED;
1109 		return 0;
1110 	}
1111 	return 1;
1112 }
1113 
1114 /*
1115  * archs can override this function if they must do something
1116  * before the modifying code is performed.
1117  */
1118 int __weak ftrace_arch_code_modify_prepare(void)
1119 {
1120 	return 0;
1121 }
1122 
1123 /*
1124  * archs can override this function if they must do something
1125  * after the modifying code is performed.
1126  */
1127 int __weak ftrace_arch_code_modify_post_process(void)
1128 {
1129 	return 0;
1130 }
1131 
1132 static int __ftrace_modify_code(void *data)
1133 {
1134 	int *command = data;
1135 
1136 	if (*command & FTRACE_ENABLE_CALLS)
1137 		ftrace_replace_code(1);
1138 	else if (*command & FTRACE_DISABLE_CALLS)
1139 		ftrace_replace_code(0);
1140 
1141 	if (*command & FTRACE_UPDATE_TRACE_FUNC)
1142 		ftrace_update_ftrace_func(ftrace_trace_function);
1143 
1144 	if (*command & FTRACE_START_FUNC_RET)
1145 		ftrace_enable_ftrace_graph_caller();
1146 	else if (*command & FTRACE_STOP_FUNC_RET)
1147 		ftrace_disable_ftrace_graph_caller();
1148 
1149 	return 0;
1150 }
1151 
1152 static void ftrace_run_update_code(int command)
1153 {
1154 	int ret;
1155 
1156 	ret = ftrace_arch_code_modify_prepare();
1157 	FTRACE_WARN_ON(ret);
1158 	if (ret)
1159 		return;
1160 
1161 	stop_machine(__ftrace_modify_code, &command, NULL);
1162 
1163 	ret = ftrace_arch_code_modify_post_process();
1164 	FTRACE_WARN_ON(ret);
1165 }
1166 
1167 static ftrace_func_t saved_ftrace_func;
1168 static int ftrace_start_up;
1169 
1170 static void ftrace_startup_enable(int command)
1171 {
1172 	if (saved_ftrace_func != ftrace_trace_function) {
1173 		saved_ftrace_func = ftrace_trace_function;
1174 		command |= FTRACE_UPDATE_TRACE_FUNC;
1175 	}
1176 
1177 	if (!command || !ftrace_enabled)
1178 		return;
1179 
1180 	ftrace_run_update_code(command);
1181 }
1182 
1183 static void ftrace_startup(int command)
1184 {
1185 	if (unlikely(ftrace_disabled))
1186 		return;
1187 
1188 	ftrace_start_up++;
1189 	command |= FTRACE_ENABLE_CALLS;
1190 
1191 	ftrace_startup_enable(command);
1192 }
1193 
1194 static void ftrace_shutdown(int command)
1195 {
1196 	if (unlikely(ftrace_disabled))
1197 		return;
1198 
1199 	ftrace_start_up--;
1200 	/*
1201 	 * Just warn in case of unbalance, no need to kill ftrace, it's not
1202 	 * critical but the ftrace_call callers may be never nopped again after
1203 	 * further ftrace uses.
1204 	 */
1205 	WARN_ON_ONCE(ftrace_start_up < 0);
1206 
1207 	if (!ftrace_start_up)
1208 		command |= FTRACE_DISABLE_CALLS;
1209 
1210 	if (saved_ftrace_func != ftrace_trace_function) {
1211 		saved_ftrace_func = ftrace_trace_function;
1212 		command |= FTRACE_UPDATE_TRACE_FUNC;
1213 	}
1214 
1215 	if (!command || !ftrace_enabled)
1216 		return;
1217 
1218 	ftrace_run_update_code(command);
1219 }
1220 
1221 static void ftrace_startup_sysctl(void)
1222 {
1223 	int command = FTRACE_ENABLE_MCOUNT;
1224 
1225 	if (unlikely(ftrace_disabled))
1226 		return;
1227 
1228 	/* Force update next time */
1229 	saved_ftrace_func = NULL;
1230 	/* ftrace_start_up is true if we want ftrace running */
1231 	if (ftrace_start_up)
1232 		command |= FTRACE_ENABLE_CALLS;
1233 
1234 	ftrace_run_update_code(command);
1235 }
1236 
1237 static void ftrace_shutdown_sysctl(void)
1238 {
1239 	int command = FTRACE_DISABLE_MCOUNT;
1240 
1241 	if (unlikely(ftrace_disabled))
1242 		return;
1243 
1244 	/* ftrace_start_up is true if ftrace is running */
1245 	if (ftrace_start_up)
1246 		command |= FTRACE_DISABLE_CALLS;
1247 
1248 	ftrace_run_update_code(command);
1249 }
1250 
1251 static cycle_t		ftrace_update_time;
1252 static unsigned long	ftrace_update_cnt;
1253 unsigned long		ftrace_update_tot_cnt;
1254 
1255 static int ftrace_update_code(struct module *mod)
1256 {
1257 	struct dyn_ftrace *p;
1258 	cycle_t start, stop;
1259 
1260 	start = ftrace_now(raw_smp_processor_id());
1261 	ftrace_update_cnt = 0;
1262 
1263 	while (ftrace_new_addrs) {
1264 
1265 		/* If something went wrong, bail without enabling anything */
1266 		if (unlikely(ftrace_disabled))
1267 			return -1;
1268 
1269 		p = ftrace_new_addrs;
1270 		ftrace_new_addrs = p->newlist;
1271 		p->flags = 0L;
1272 
1273 		/*
1274 		 * Do the initial record convertion from mcount jump
1275 		 * to the NOP instructions.
1276 		 */
1277 		if (!ftrace_code_disable(mod, p)) {
1278 			ftrace_free_rec(p);
1279 			continue;
1280 		}
1281 
1282 		p->flags |= FTRACE_FL_CONVERTED;
1283 		ftrace_update_cnt++;
1284 
1285 		/*
1286 		 * If the tracing is enabled, go ahead and enable the record.
1287 		 *
1288 		 * The reason not to enable the record immediatelly is the
1289 		 * inherent check of ftrace_make_nop/ftrace_make_call for
1290 		 * correct previous instructions.  Making first the NOP
1291 		 * conversion puts the module to the correct state, thus
1292 		 * passing the ftrace_make_call check.
1293 		 */
1294 		if (ftrace_start_up) {
1295 			int failed = __ftrace_replace_code(p, 1);
1296 			if (failed) {
1297 				ftrace_bug(failed, p->ip);
1298 				ftrace_free_rec(p);
1299 			}
1300 		}
1301 	}
1302 
1303 	stop = ftrace_now(raw_smp_processor_id());
1304 	ftrace_update_time = stop - start;
1305 	ftrace_update_tot_cnt += ftrace_update_cnt;
1306 
1307 	return 0;
1308 }
1309 
1310 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1311 {
1312 	struct ftrace_page *pg;
1313 	int cnt;
1314 	int i;
1315 
1316 	/* allocate a few pages */
1317 	ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
1318 	if (!ftrace_pages_start)
1319 		return -1;
1320 
1321 	/*
1322 	 * Allocate a few more pages.
1323 	 *
1324 	 * TODO: have some parser search vmlinux before
1325 	 *   final linking to find all calls to ftrace.
1326 	 *   Then we can:
1327 	 *    a) know how many pages to allocate.
1328 	 *     and/or
1329 	 *    b) set up the table then.
1330 	 *
1331 	 *  The dynamic code is still necessary for
1332 	 *  modules.
1333 	 */
1334 
1335 	pg = ftrace_pages = ftrace_pages_start;
1336 
1337 	cnt = num_to_init / ENTRIES_PER_PAGE;
1338 	pr_info("ftrace: allocating %ld entries in %d pages\n",
1339 		num_to_init, cnt + 1);
1340 
1341 	for (i = 0; i < cnt; i++) {
1342 		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
1343 
1344 		/* If we fail, we'll try later anyway */
1345 		if (!pg->next)
1346 			break;
1347 
1348 		pg = pg->next;
1349 	}
1350 
1351 	return 0;
1352 }
1353 
1354 enum {
1355 	FTRACE_ITER_FILTER	= (1 << 0),
1356 	FTRACE_ITER_NOTRACE	= (1 << 1),
1357 	FTRACE_ITER_FAILURES	= (1 << 2),
1358 	FTRACE_ITER_PRINTALL	= (1 << 3),
1359 	FTRACE_ITER_HASH	= (1 << 4),
1360 };
1361 
1362 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1363 
1364 struct ftrace_iterator {
1365 	struct ftrace_page	*pg;
1366 	int			hidx;
1367 	int			idx;
1368 	unsigned		flags;
1369 	struct trace_parser	parser;
1370 };
1371 
1372 static void *
1373 t_hash_next(struct seq_file *m, void *v, loff_t *pos)
1374 {
1375 	struct ftrace_iterator *iter = m->private;
1376 	struct hlist_node *hnd = v;
1377 	struct hlist_head *hhd;
1378 
1379 	WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
1380 
1381 	(*pos)++;
1382 
1383  retry:
1384 	if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
1385 		return NULL;
1386 
1387 	hhd = &ftrace_func_hash[iter->hidx];
1388 
1389 	if (hlist_empty(hhd)) {
1390 		iter->hidx++;
1391 		hnd = NULL;
1392 		goto retry;
1393 	}
1394 
1395 	if (!hnd)
1396 		hnd = hhd->first;
1397 	else {
1398 		hnd = hnd->next;
1399 		if (!hnd) {
1400 			iter->hidx++;
1401 			goto retry;
1402 		}
1403 	}
1404 
1405 	return hnd;
1406 }
1407 
1408 static void *t_hash_start(struct seq_file *m, loff_t *pos)
1409 {
1410 	struct ftrace_iterator *iter = m->private;
1411 	void *p = NULL;
1412 	loff_t l;
1413 
1414 	if (!(iter->flags & FTRACE_ITER_HASH))
1415 		*pos = 0;
1416 
1417 	iter->flags |= FTRACE_ITER_HASH;
1418 
1419 	iter->hidx = 0;
1420 	for (l = 0; l <= *pos; ) {
1421 		p = t_hash_next(m, p, &l);
1422 		if (!p)
1423 			break;
1424 	}
1425 	return p;
1426 }
1427 
1428 static int t_hash_show(struct seq_file *m, void *v)
1429 {
1430 	struct ftrace_func_probe *rec;
1431 	struct hlist_node *hnd = v;
1432 
1433 	rec = hlist_entry(hnd, struct ftrace_func_probe, node);
1434 
1435 	if (rec->ops->print)
1436 		return rec->ops->print(m, rec->ip, rec->ops, rec->data);
1437 
1438 	seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
1439 
1440 	if (rec->data)
1441 		seq_printf(m, ":%p", rec->data);
1442 	seq_putc(m, '\n');
1443 
1444 	return 0;
1445 }
1446 
1447 static void *
1448 t_next(struct seq_file *m, void *v, loff_t *pos)
1449 {
1450 	struct ftrace_iterator *iter = m->private;
1451 	struct dyn_ftrace *rec = NULL;
1452 
1453 	if (iter->flags & FTRACE_ITER_HASH)
1454 		return t_hash_next(m, v, pos);
1455 
1456 	(*pos)++;
1457 
1458 	if (iter->flags & FTRACE_ITER_PRINTALL)
1459 		return NULL;
1460 
1461  retry:
1462 	if (iter->idx >= iter->pg->index) {
1463 		if (iter->pg->next) {
1464 			iter->pg = iter->pg->next;
1465 			iter->idx = 0;
1466 			goto retry;
1467 		}
1468 	} else {
1469 		rec = &iter->pg->records[iter->idx++];
1470 		if ((rec->flags & FTRACE_FL_FREE) ||
1471 
1472 		    (!(iter->flags & FTRACE_ITER_FAILURES) &&
1473 		     (rec->flags & FTRACE_FL_FAILED)) ||
1474 
1475 		    ((iter->flags & FTRACE_ITER_FAILURES) &&
1476 		     !(rec->flags & FTRACE_FL_FAILED)) ||
1477 
1478 		    ((iter->flags & FTRACE_ITER_FILTER) &&
1479 		     !(rec->flags & FTRACE_FL_FILTER)) ||
1480 
1481 		    ((iter->flags & FTRACE_ITER_NOTRACE) &&
1482 		     !(rec->flags & FTRACE_FL_NOTRACE))) {
1483 			rec = NULL;
1484 			goto retry;
1485 		}
1486 	}
1487 
1488 	return rec;
1489 }
1490 
1491 static void *t_start(struct seq_file *m, loff_t *pos)
1492 {
1493 	struct ftrace_iterator *iter = m->private;
1494 	void *p = NULL;
1495 	loff_t l;
1496 
1497 	mutex_lock(&ftrace_lock);
1498 	/*
1499 	 * For set_ftrace_filter reading, if we have the filter
1500 	 * off, we can short cut and just print out that all
1501 	 * functions are enabled.
1502 	 */
1503 	if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
1504 		if (*pos > 0)
1505 			return t_hash_start(m, pos);
1506 		iter->flags |= FTRACE_ITER_PRINTALL;
1507 		return iter;
1508 	}
1509 
1510 	if (iter->flags & FTRACE_ITER_HASH)
1511 		return t_hash_start(m, pos);
1512 
1513 	iter->pg = ftrace_pages_start;
1514 	iter->idx = 0;
1515 	for (l = 0; l <= *pos; ) {
1516 		p = t_next(m, p, &l);
1517 		if (!p)
1518 			break;
1519 	}
1520 
1521 	if (!p && iter->flags & FTRACE_ITER_FILTER)
1522 		return t_hash_start(m, pos);
1523 
1524 	return p;
1525 }
1526 
1527 static void t_stop(struct seq_file *m, void *p)
1528 {
1529 	mutex_unlock(&ftrace_lock);
1530 }
1531 
1532 static int t_show(struct seq_file *m, void *v)
1533 {
1534 	struct ftrace_iterator *iter = m->private;
1535 	struct dyn_ftrace *rec = v;
1536 
1537 	if (iter->flags & FTRACE_ITER_HASH)
1538 		return t_hash_show(m, v);
1539 
1540 	if (iter->flags & FTRACE_ITER_PRINTALL) {
1541 		seq_printf(m, "#### all functions enabled ####\n");
1542 		return 0;
1543 	}
1544 
1545 	if (!rec)
1546 		return 0;
1547 
1548 	seq_printf(m, "%ps\n", (void *)rec->ip);
1549 
1550 	return 0;
1551 }
1552 
1553 static const struct seq_operations show_ftrace_seq_ops = {
1554 	.start = t_start,
1555 	.next = t_next,
1556 	.stop = t_stop,
1557 	.show = t_show,
1558 };
1559 
1560 static int
1561 ftrace_avail_open(struct inode *inode, struct file *file)
1562 {
1563 	struct ftrace_iterator *iter;
1564 	int ret;
1565 
1566 	if (unlikely(ftrace_disabled))
1567 		return -ENODEV;
1568 
1569 	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1570 	if (!iter)
1571 		return -ENOMEM;
1572 
1573 	iter->pg = ftrace_pages_start;
1574 
1575 	ret = seq_open(file, &show_ftrace_seq_ops);
1576 	if (!ret) {
1577 		struct seq_file *m = file->private_data;
1578 
1579 		m->private = iter;
1580 	} else {
1581 		kfree(iter);
1582 	}
1583 
1584 	return ret;
1585 }
1586 
1587 static int
1588 ftrace_failures_open(struct inode *inode, struct file *file)
1589 {
1590 	int ret;
1591 	struct seq_file *m;
1592 	struct ftrace_iterator *iter;
1593 
1594 	ret = ftrace_avail_open(inode, file);
1595 	if (!ret) {
1596 		m = (struct seq_file *)file->private_data;
1597 		iter = (struct ftrace_iterator *)m->private;
1598 		iter->flags = FTRACE_ITER_FAILURES;
1599 	}
1600 
1601 	return ret;
1602 }
1603 
1604 
1605 static void ftrace_filter_reset(int enable)
1606 {
1607 	struct ftrace_page *pg;
1608 	struct dyn_ftrace *rec;
1609 	unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1610 
1611 	mutex_lock(&ftrace_lock);
1612 	if (enable)
1613 		ftrace_filtered = 0;
1614 	do_for_each_ftrace_rec(pg, rec) {
1615 		if (rec->flags & FTRACE_FL_FAILED)
1616 			continue;
1617 		rec->flags &= ~type;
1618 	} while_for_each_ftrace_rec();
1619 	mutex_unlock(&ftrace_lock);
1620 }
1621 
1622 static int
1623 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1624 {
1625 	struct ftrace_iterator *iter;
1626 	int ret = 0;
1627 
1628 	if (unlikely(ftrace_disabled))
1629 		return -ENODEV;
1630 
1631 	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1632 	if (!iter)
1633 		return -ENOMEM;
1634 
1635 	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
1636 		kfree(iter);
1637 		return -ENOMEM;
1638 	}
1639 
1640 	mutex_lock(&ftrace_regex_lock);
1641 	if ((file->f_mode & FMODE_WRITE) &&
1642 	    (file->f_flags & O_TRUNC))
1643 		ftrace_filter_reset(enable);
1644 
1645 	if (file->f_mode & FMODE_READ) {
1646 		iter->pg = ftrace_pages_start;
1647 		iter->flags = enable ? FTRACE_ITER_FILTER :
1648 			FTRACE_ITER_NOTRACE;
1649 
1650 		ret = seq_open(file, &show_ftrace_seq_ops);
1651 		if (!ret) {
1652 			struct seq_file *m = file->private_data;
1653 			m->private = iter;
1654 		} else {
1655 			trace_parser_put(&iter->parser);
1656 			kfree(iter);
1657 		}
1658 	} else
1659 		file->private_data = iter;
1660 	mutex_unlock(&ftrace_regex_lock);
1661 
1662 	return ret;
1663 }
1664 
1665 static int
1666 ftrace_filter_open(struct inode *inode, struct file *file)
1667 {
1668 	return ftrace_regex_open(inode, file, 1);
1669 }
1670 
1671 static int
1672 ftrace_notrace_open(struct inode *inode, struct file *file)
1673 {
1674 	return ftrace_regex_open(inode, file, 0);
1675 }
1676 
1677 static loff_t
1678 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1679 {
1680 	loff_t ret;
1681 
1682 	if (file->f_mode & FMODE_READ)
1683 		ret = seq_lseek(file, offset, origin);
1684 	else
1685 		file->f_pos = ret = 1;
1686 
1687 	return ret;
1688 }
1689 
1690 static int ftrace_match(char *str, char *regex, int len, int type)
1691 {
1692 	int matched = 0;
1693 	int slen;
1694 
1695 	switch (type) {
1696 	case MATCH_FULL:
1697 		if (strcmp(str, regex) == 0)
1698 			matched = 1;
1699 		break;
1700 	case MATCH_FRONT_ONLY:
1701 		if (strncmp(str, regex, len) == 0)
1702 			matched = 1;
1703 		break;
1704 	case MATCH_MIDDLE_ONLY:
1705 		if (strstr(str, regex))
1706 			matched = 1;
1707 		break;
1708 	case MATCH_END_ONLY:
1709 		slen = strlen(str);
1710 		if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
1711 			matched = 1;
1712 		break;
1713 	}
1714 
1715 	return matched;
1716 }
1717 
1718 static int
1719 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1720 {
1721 	char str[KSYM_SYMBOL_LEN];
1722 
1723 	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1724 	return ftrace_match(str, regex, len, type);
1725 }
1726 
1727 static int ftrace_match_records(char *buff, int len, int enable)
1728 {
1729 	unsigned int search_len;
1730 	struct ftrace_page *pg;
1731 	struct dyn_ftrace *rec;
1732 	unsigned long flag;
1733 	char *search;
1734 	int type;
1735 	int not;
1736 	int found = 0;
1737 
1738 	flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1739 	type = filter_parse_regex(buff, len, &search, &not);
1740 
1741 	search_len = strlen(search);
1742 
1743 	mutex_lock(&ftrace_lock);
1744 	do_for_each_ftrace_rec(pg, rec) {
1745 
1746 		if (rec->flags & FTRACE_FL_FAILED)
1747 			continue;
1748 
1749 		if (ftrace_match_record(rec, search, search_len, type)) {
1750 			if (not)
1751 				rec->flags &= ~flag;
1752 			else
1753 				rec->flags |= flag;
1754 			found = 1;
1755 		}
1756 		/*
1757 		 * Only enable filtering if we have a function that
1758 		 * is filtered on.
1759 		 */
1760 		if (enable && (rec->flags & FTRACE_FL_FILTER))
1761 			ftrace_filtered = 1;
1762 	} while_for_each_ftrace_rec();
1763 	mutex_unlock(&ftrace_lock);
1764 
1765 	return found;
1766 }
1767 
1768 static int
1769 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1770 			   char *regex, int len, int type)
1771 {
1772 	char str[KSYM_SYMBOL_LEN];
1773 	char *modname;
1774 
1775 	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1776 
1777 	if (!modname || strcmp(modname, mod))
1778 		return 0;
1779 
1780 	/* blank search means to match all funcs in the mod */
1781 	if (len)
1782 		return ftrace_match(str, regex, len, type);
1783 	else
1784 		return 1;
1785 }
1786 
1787 static int ftrace_match_module_records(char *buff, char *mod, int enable)
1788 {
1789 	unsigned search_len = 0;
1790 	struct ftrace_page *pg;
1791 	struct dyn_ftrace *rec;
1792 	int type = MATCH_FULL;
1793 	char *search = buff;
1794 	unsigned long flag;
1795 	int not = 0;
1796 	int found = 0;
1797 
1798 	flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1799 
1800 	/* blank or '*' mean the same */
1801 	if (strcmp(buff, "*") == 0)
1802 		buff[0] = 0;
1803 
1804 	/* handle the case of 'dont filter this module' */
1805 	if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1806 		buff[0] = 0;
1807 		not = 1;
1808 	}
1809 
1810 	if (strlen(buff)) {
1811 		type = filter_parse_regex(buff, strlen(buff), &search, &not);
1812 		search_len = strlen(search);
1813 	}
1814 
1815 	mutex_lock(&ftrace_lock);
1816 	do_for_each_ftrace_rec(pg, rec) {
1817 
1818 		if (rec->flags & FTRACE_FL_FAILED)
1819 			continue;
1820 
1821 		if (ftrace_match_module_record(rec, mod,
1822 					       search, search_len, type)) {
1823 			if (not)
1824 				rec->flags &= ~flag;
1825 			else
1826 				rec->flags |= flag;
1827 			found = 1;
1828 		}
1829 		if (enable && (rec->flags & FTRACE_FL_FILTER))
1830 			ftrace_filtered = 1;
1831 
1832 	} while_for_each_ftrace_rec();
1833 	mutex_unlock(&ftrace_lock);
1834 
1835 	return found;
1836 }
1837 
1838 /*
1839  * We register the module command as a template to show others how
1840  * to register the a command as well.
1841  */
1842 
1843 static int
1844 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1845 {
1846 	char *mod;
1847 
1848 	/*
1849 	 * cmd == 'mod' because we only registered this func
1850 	 * for the 'mod' ftrace_func_command.
1851 	 * But if you register one func with multiple commands,
1852 	 * you can tell which command was used by the cmd
1853 	 * parameter.
1854 	 */
1855 
1856 	/* we must have a module name */
1857 	if (!param)
1858 		return -EINVAL;
1859 
1860 	mod = strsep(&param, ":");
1861 	if (!strlen(mod))
1862 		return -EINVAL;
1863 
1864 	if (ftrace_match_module_records(func, mod, enable))
1865 		return 0;
1866 	return -EINVAL;
1867 }
1868 
1869 static struct ftrace_func_command ftrace_mod_cmd = {
1870 	.name			= "mod",
1871 	.func			= ftrace_mod_callback,
1872 };
1873 
1874 static int __init ftrace_mod_cmd_init(void)
1875 {
1876 	return register_ftrace_command(&ftrace_mod_cmd);
1877 }
1878 device_initcall(ftrace_mod_cmd_init);
1879 
1880 static void
1881 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1882 {
1883 	struct ftrace_func_probe *entry;
1884 	struct hlist_head *hhd;
1885 	struct hlist_node *n;
1886 	unsigned long key;
1887 	int resched;
1888 
1889 	key = hash_long(ip, FTRACE_HASH_BITS);
1890 
1891 	hhd = &ftrace_func_hash[key];
1892 
1893 	if (hlist_empty(hhd))
1894 		return;
1895 
1896 	/*
1897 	 * Disable preemption for these calls to prevent a RCU grace
1898 	 * period. This syncs the hash iteration and freeing of items
1899 	 * on the hash. rcu_read_lock is too dangerous here.
1900 	 */
1901 	resched = ftrace_preempt_disable();
1902 	hlist_for_each_entry_rcu(entry, n, hhd, node) {
1903 		if (entry->ip == ip)
1904 			entry->ops->func(ip, parent_ip, &entry->data);
1905 	}
1906 	ftrace_preempt_enable(resched);
1907 }
1908 
1909 static struct ftrace_ops trace_probe_ops __read_mostly =
1910 {
1911 	.func		= function_trace_probe_call,
1912 };
1913 
1914 static int ftrace_probe_registered;
1915 
1916 static void __enable_ftrace_function_probe(void)
1917 {
1918 	int i;
1919 
1920 	if (ftrace_probe_registered)
1921 		return;
1922 
1923 	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1924 		struct hlist_head *hhd = &ftrace_func_hash[i];
1925 		if (hhd->first)
1926 			break;
1927 	}
1928 	/* Nothing registered? */
1929 	if (i == FTRACE_FUNC_HASHSIZE)
1930 		return;
1931 
1932 	__register_ftrace_function(&trace_probe_ops);
1933 	ftrace_startup(0);
1934 	ftrace_probe_registered = 1;
1935 }
1936 
1937 static void __disable_ftrace_function_probe(void)
1938 {
1939 	int i;
1940 
1941 	if (!ftrace_probe_registered)
1942 		return;
1943 
1944 	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1945 		struct hlist_head *hhd = &ftrace_func_hash[i];
1946 		if (hhd->first)
1947 			return;
1948 	}
1949 
1950 	/* no more funcs left */
1951 	__unregister_ftrace_function(&trace_probe_ops);
1952 	ftrace_shutdown(0);
1953 	ftrace_probe_registered = 0;
1954 }
1955 
1956 
1957 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
1958 {
1959 	struct ftrace_func_probe *entry =
1960 		container_of(rhp, struct ftrace_func_probe, rcu);
1961 
1962 	if (entry->ops->free)
1963 		entry->ops->free(&entry->data);
1964 	kfree(entry);
1965 }
1966 
1967 
1968 int
1969 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1970 			      void *data)
1971 {
1972 	struct ftrace_func_probe *entry;
1973 	struct ftrace_page *pg;
1974 	struct dyn_ftrace *rec;
1975 	int type, len, not;
1976 	unsigned long key;
1977 	int count = 0;
1978 	char *search;
1979 
1980 	type = filter_parse_regex(glob, strlen(glob), &search, &not);
1981 	len = strlen(search);
1982 
1983 	/* we do not support '!' for function probes */
1984 	if (WARN_ON(not))
1985 		return -EINVAL;
1986 
1987 	mutex_lock(&ftrace_lock);
1988 	do_for_each_ftrace_rec(pg, rec) {
1989 
1990 		if (rec->flags & FTRACE_FL_FAILED)
1991 			continue;
1992 
1993 		if (!ftrace_match_record(rec, search, len, type))
1994 			continue;
1995 
1996 		entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1997 		if (!entry) {
1998 			/* If we did not process any, then return error */
1999 			if (!count)
2000 				count = -ENOMEM;
2001 			goto out_unlock;
2002 		}
2003 
2004 		count++;
2005 
2006 		entry->data = data;
2007 
2008 		/*
2009 		 * The caller might want to do something special
2010 		 * for each function we find. We call the callback
2011 		 * to give the caller an opportunity to do so.
2012 		 */
2013 		if (ops->callback) {
2014 			if (ops->callback(rec->ip, &entry->data) < 0) {
2015 				/* caller does not like this func */
2016 				kfree(entry);
2017 				continue;
2018 			}
2019 		}
2020 
2021 		entry->ops = ops;
2022 		entry->ip = rec->ip;
2023 
2024 		key = hash_long(entry->ip, FTRACE_HASH_BITS);
2025 		hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2026 
2027 	} while_for_each_ftrace_rec();
2028 	__enable_ftrace_function_probe();
2029 
2030  out_unlock:
2031 	mutex_unlock(&ftrace_lock);
2032 
2033 	return count;
2034 }
2035 
2036 enum {
2037 	PROBE_TEST_FUNC		= 1,
2038 	PROBE_TEST_DATA		= 2
2039 };
2040 
2041 static void
2042 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2043 				  void *data, int flags)
2044 {
2045 	struct ftrace_func_probe *entry;
2046 	struct hlist_node *n, *tmp;
2047 	char str[KSYM_SYMBOL_LEN];
2048 	int type = MATCH_FULL;
2049 	int i, len = 0;
2050 	char *search;
2051 
2052 	if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
2053 		glob = NULL;
2054 	else if (glob) {
2055 		int not;
2056 
2057 		type = filter_parse_regex(glob, strlen(glob), &search, &not);
2058 		len = strlen(search);
2059 
2060 		/* we do not support '!' for function probes */
2061 		if (WARN_ON(not))
2062 			return;
2063 	}
2064 
2065 	mutex_lock(&ftrace_lock);
2066 	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2067 		struct hlist_head *hhd = &ftrace_func_hash[i];
2068 
2069 		hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2070 
2071 			/* break up if statements for readability */
2072 			if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2073 				continue;
2074 
2075 			if ((flags & PROBE_TEST_DATA) && entry->data != data)
2076 				continue;
2077 
2078 			/* do this last, since it is the most expensive */
2079 			if (glob) {
2080 				kallsyms_lookup(entry->ip, NULL, NULL,
2081 						NULL, str);
2082 				if (!ftrace_match(str, glob, len, type))
2083 					continue;
2084 			}
2085 
2086 			hlist_del(&entry->node);
2087 			call_rcu(&entry->rcu, ftrace_free_entry_rcu);
2088 		}
2089 	}
2090 	__disable_ftrace_function_probe();
2091 	mutex_unlock(&ftrace_lock);
2092 }
2093 
2094 void
2095 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2096 				void *data)
2097 {
2098 	__unregister_ftrace_function_probe(glob, ops, data,
2099 					  PROBE_TEST_FUNC | PROBE_TEST_DATA);
2100 }
2101 
2102 void
2103 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2104 {
2105 	__unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2106 }
2107 
2108 void unregister_ftrace_function_probe_all(char *glob)
2109 {
2110 	__unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2111 }
2112 
2113 static LIST_HEAD(ftrace_commands);
2114 static DEFINE_MUTEX(ftrace_cmd_mutex);
2115 
2116 int register_ftrace_command(struct ftrace_func_command *cmd)
2117 {
2118 	struct ftrace_func_command *p;
2119 	int ret = 0;
2120 
2121 	mutex_lock(&ftrace_cmd_mutex);
2122 	list_for_each_entry(p, &ftrace_commands, list) {
2123 		if (strcmp(cmd->name, p->name) == 0) {
2124 			ret = -EBUSY;
2125 			goto out_unlock;
2126 		}
2127 	}
2128 	list_add(&cmd->list, &ftrace_commands);
2129  out_unlock:
2130 	mutex_unlock(&ftrace_cmd_mutex);
2131 
2132 	return ret;
2133 }
2134 
2135 int unregister_ftrace_command(struct ftrace_func_command *cmd)
2136 {
2137 	struct ftrace_func_command *p, *n;
2138 	int ret = -ENODEV;
2139 
2140 	mutex_lock(&ftrace_cmd_mutex);
2141 	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
2142 		if (strcmp(cmd->name, p->name) == 0) {
2143 			ret = 0;
2144 			list_del_init(&p->list);
2145 			goto out_unlock;
2146 		}
2147 	}
2148  out_unlock:
2149 	mutex_unlock(&ftrace_cmd_mutex);
2150 
2151 	return ret;
2152 }
2153 
2154 static int ftrace_process_regex(char *buff, int len, int enable)
2155 {
2156 	char *func, *command, *next = buff;
2157 	struct ftrace_func_command *p;
2158 	int ret = -EINVAL;
2159 
2160 	func = strsep(&next, ":");
2161 
2162 	if (!next) {
2163 		if (ftrace_match_records(func, len, enable))
2164 			return 0;
2165 		return ret;
2166 	}
2167 
2168 	/* command found */
2169 
2170 	command = strsep(&next, ":");
2171 
2172 	mutex_lock(&ftrace_cmd_mutex);
2173 	list_for_each_entry(p, &ftrace_commands, list) {
2174 		if (strcmp(p->name, command) == 0) {
2175 			ret = p->func(func, command, next, enable);
2176 			goto out_unlock;
2177 		}
2178 	}
2179  out_unlock:
2180 	mutex_unlock(&ftrace_cmd_mutex);
2181 
2182 	return ret;
2183 }
2184 
2185 static ssize_t
2186 ftrace_regex_write(struct file *file, const char __user *ubuf,
2187 		   size_t cnt, loff_t *ppos, int enable)
2188 {
2189 	struct ftrace_iterator *iter;
2190 	struct trace_parser *parser;
2191 	ssize_t ret, read;
2192 
2193 	if (!cnt)
2194 		return 0;
2195 
2196 	mutex_lock(&ftrace_regex_lock);
2197 
2198 	if (file->f_mode & FMODE_READ) {
2199 		struct seq_file *m = file->private_data;
2200 		iter = m->private;
2201 	} else
2202 		iter = file->private_data;
2203 
2204 	parser = &iter->parser;
2205 	read = trace_get_user(parser, ubuf, cnt, ppos);
2206 
2207 	if (read >= 0 && trace_parser_loaded(parser) &&
2208 	    !trace_parser_cont(parser)) {
2209 		ret = ftrace_process_regex(parser->buffer,
2210 					   parser->idx, enable);
2211 		trace_parser_clear(parser);
2212 		if (ret)
2213 			goto out_unlock;
2214 	}
2215 
2216 	ret = read;
2217 out_unlock:
2218 	mutex_unlock(&ftrace_regex_lock);
2219 
2220 	return ret;
2221 }
2222 
2223 static ssize_t
2224 ftrace_filter_write(struct file *file, const char __user *ubuf,
2225 		    size_t cnt, loff_t *ppos)
2226 {
2227 	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
2228 }
2229 
2230 static ssize_t
2231 ftrace_notrace_write(struct file *file, const char __user *ubuf,
2232 		     size_t cnt, loff_t *ppos)
2233 {
2234 	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2235 }
2236 
2237 static void
2238 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
2239 {
2240 	if (unlikely(ftrace_disabled))
2241 		return;
2242 
2243 	mutex_lock(&ftrace_regex_lock);
2244 	if (reset)
2245 		ftrace_filter_reset(enable);
2246 	if (buf)
2247 		ftrace_match_records(buf, len, enable);
2248 	mutex_unlock(&ftrace_regex_lock);
2249 }
2250 
2251 /**
2252  * ftrace_set_filter - set a function to filter on in ftrace
2253  * @buf - the string that holds the function filter text.
2254  * @len - the length of the string.
2255  * @reset - non zero to reset all filters before applying this filter.
2256  *
2257  * Filters denote which functions should be enabled when tracing is enabled.
2258  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2259  */
2260 void ftrace_set_filter(unsigned char *buf, int len, int reset)
2261 {
2262 	ftrace_set_regex(buf, len, reset, 1);
2263 }
2264 
2265 /**
2266  * ftrace_set_notrace - set a function to not trace in ftrace
2267  * @buf - the string that holds the function notrace text.
2268  * @len - the length of the string.
2269  * @reset - non zero to reset all filters before applying this filter.
2270  *
2271  * Notrace Filters denote which functions should not be enabled when tracing
2272  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2273  * for tracing.
2274  */
2275 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
2276 {
2277 	ftrace_set_regex(buf, len, reset, 0);
2278 }
2279 
2280 /*
2281  * command line interface to allow users to set filters on boot up.
2282  */
2283 #define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
2284 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
2285 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
2286 
2287 static int __init set_ftrace_notrace(char *str)
2288 {
2289 	strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
2290 	return 1;
2291 }
2292 __setup("ftrace_notrace=", set_ftrace_notrace);
2293 
2294 static int __init set_ftrace_filter(char *str)
2295 {
2296 	strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
2297 	return 1;
2298 }
2299 __setup("ftrace_filter=", set_ftrace_filter);
2300 
2301 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2302 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
2303 static int __init set_graph_function(char *str)
2304 {
2305 	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
2306 	return 1;
2307 }
2308 __setup("ftrace_graph_filter=", set_graph_function);
2309 
2310 static void __init set_ftrace_early_graph(char *buf)
2311 {
2312 	int ret;
2313 	char *func;
2314 
2315 	while (buf) {
2316 		func = strsep(&buf, ",");
2317 		/* we allow only one expression at a time */
2318 		ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2319 				      func);
2320 		if (ret)
2321 			printk(KERN_DEBUG "ftrace: function %s not "
2322 					  "traceable\n", func);
2323 	}
2324 }
2325 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2326 
2327 static void __init set_ftrace_early_filter(char *buf, int enable)
2328 {
2329 	char *func;
2330 
2331 	while (buf) {
2332 		func = strsep(&buf, ",");
2333 		ftrace_set_regex(func, strlen(func), 0, enable);
2334 	}
2335 }
2336 
2337 static void __init set_ftrace_early_filters(void)
2338 {
2339 	if (ftrace_filter_buf[0])
2340 		set_ftrace_early_filter(ftrace_filter_buf, 1);
2341 	if (ftrace_notrace_buf[0])
2342 		set_ftrace_early_filter(ftrace_notrace_buf, 0);
2343 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2344 	if (ftrace_graph_buf[0])
2345 		set_ftrace_early_graph(ftrace_graph_buf);
2346 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2347 }
2348 
2349 static int
2350 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
2351 {
2352 	struct seq_file *m = (struct seq_file *)file->private_data;
2353 	struct ftrace_iterator *iter;
2354 	struct trace_parser *parser;
2355 
2356 	mutex_lock(&ftrace_regex_lock);
2357 	if (file->f_mode & FMODE_READ) {
2358 		iter = m->private;
2359 
2360 		seq_release(inode, file);
2361 	} else
2362 		iter = file->private_data;
2363 
2364 	parser = &iter->parser;
2365 	if (trace_parser_loaded(parser)) {
2366 		parser->buffer[parser->idx] = 0;
2367 		ftrace_match_records(parser->buffer, parser->idx, enable);
2368 	}
2369 
2370 	mutex_lock(&ftrace_lock);
2371 	if (ftrace_start_up && ftrace_enabled)
2372 		ftrace_run_update_code(FTRACE_ENABLE_CALLS);
2373 	mutex_unlock(&ftrace_lock);
2374 
2375 	trace_parser_put(parser);
2376 	kfree(iter);
2377 
2378 	mutex_unlock(&ftrace_regex_lock);
2379 	return 0;
2380 }
2381 
2382 static int
2383 ftrace_filter_release(struct inode *inode, struct file *file)
2384 {
2385 	return ftrace_regex_release(inode, file, 1);
2386 }
2387 
2388 static int
2389 ftrace_notrace_release(struct inode *inode, struct file *file)
2390 {
2391 	return ftrace_regex_release(inode, file, 0);
2392 }
2393 
2394 static const struct file_operations ftrace_avail_fops = {
2395 	.open = ftrace_avail_open,
2396 	.read = seq_read,
2397 	.llseek = seq_lseek,
2398 	.release = seq_release_private,
2399 };
2400 
2401 static const struct file_operations ftrace_failures_fops = {
2402 	.open = ftrace_failures_open,
2403 	.read = seq_read,
2404 	.llseek = seq_lseek,
2405 	.release = seq_release_private,
2406 };
2407 
2408 static const struct file_operations ftrace_filter_fops = {
2409 	.open = ftrace_filter_open,
2410 	.read = seq_read,
2411 	.write = ftrace_filter_write,
2412 	.llseek = ftrace_regex_lseek,
2413 	.release = ftrace_filter_release,
2414 };
2415 
2416 static const struct file_operations ftrace_notrace_fops = {
2417 	.open = ftrace_notrace_open,
2418 	.read = seq_read,
2419 	.write = ftrace_notrace_write,
2420 	.llseek = ftrace_regex_lseek,
2421 	.release = ftrace_notrace_release,
2422 };
2423 
2424 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2425 
2426 static DEFINE_MUTEX(graph_lock);
2427 
2428 int ftrace_graph_count;
2429 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
2430 
2431 static void *
2432 __g_next(struct seq_file *m, loff_t *pos)
2433 {
2434 	if (*pos >= ftrace_graph_count)
2435 		return NULL;
2436 	return &ftrace_graph_funcs[*pos];
2437 }
2438 
2439 static void *
2440 g_next(struct seq_file *m, void *v, loff_t *pos)
2441 {
2442 	(*pos)++;
2443 	return __g_next(m, pos);
2444 }
2445 
2446 static void *g_start(struct seq_file *m, loff_t *pos)
2447 {
2448 	mutex_lock(&graph_lock);
2449 
2450 	/* Nothing, tell g_show to print all functions are enabled */
2451 	if (!ftrace_graph_count && !*pos)
2452 		return (void *)1;
2453 
2454 	return __g_next(m, pos);
2455 }
2456 
2457 static void g_stop(struct seq_file *m, void *p)
2458 {
2459 	mutex_unlock(&graph_lock);
2460 }
2461 
2462 static int g_show(struct seq_file *m, void *v)
2463 {
2464 	unsigned long *ptr = v;
2465 
2466 	if (!ptr)
2467 		return 0;
2468 
2469 	if (ptr == (unsigned long *)1) {
2470 		seq_printf(m, "#### all functions enabled ####\n");
2471 		return 0;
2472 	}
2473 
2474 	seq_printf(m, "%ps\n", (void *)*ptr);
2475 
2476 	return 0;
2477 }
2478 
2479 static const struct seq_operations ftrace_graph_seq_ops = {
2480 	.start = g_start,
2481 	.next = g_next,
2482 	.stop = g_stop,
2483 	.show = g_show,
2484 };
2485 
2486 static int
2487 ftrace_graph_open(struct inode *inode, struct file *file)
2488 {
2489 	int ret = 0;
2490 
2491 	if (unlikely(ftrace_disabled))
2492 		return -ENODEV;
2493 
2494 	mutex_lock(&graph_lock);
2495 	if ((file->f_mode & FMODE_WRITE) &&
2496 	    (file->f_flags & O_TRUNC)) {
2497 		ftrace_graph_count = 0;
2498 		memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
2499 	}
2500 	mutex_unlock(&graph_lock);
2501 
2502 	if (file->f_mode & FMODE_READ)
2503 		ret = seq_open(file, &ftrace_graph_seq_ops);
2504 
2505 	return ret;
2506 }
2507 
2508 static int
2509 ftrace_graph_release(struct inode *inode, struct file *file)
2510 {
2511 	if (file->f_mode & FMODE_READ)
2512 		seq_release(inode, file);
2513 	return 0;
2514 }
2515 
2516 static int
2517 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2518 {
2519 	struct dyn_ftrace *rec;
2520 	struct ftrace_page *pg;
2521 	int search_len;
2522 	int found = 0;
2523 	int type, not;
2524 	char *search;
2525 	bool exists;
2526 	int i;
2527 
2528 	if (ftrace_disabled)
2529 		return -ENODEV;
2530 
2531 	/* decode regex */
2532 	type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
2533 	if (not)
2534 		return -EINVAL;
2535 
2536 	search_len = strlen(search);
2537 
2538 	mutex_lock(&ftrace_lock);
2539 	do_for_each_ftrace_rec(pg, rec) {
2540 
2541 		if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2542 			break;
2543 
2544 		if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2545 			continue;
2546 
2547 		if (ftrace_match_record(rec, search, search_len, type)) {
2548 			/* ensure it is not already in the array */
2549 			exists = false;
2550 			for (i = 0; i < *idx; i++)
2551 				if (array[i] == rec->ip) {
2552 					exists = true;
2553 					break;
2554 				}
2555 			if (!exists)
2556 				array[(*idx)++] = rec->ip;
2557 			found = 1;
2558 		}
2559 	} while_for_each_ftrace_rec();
2560 
2561 	mutex_unlock(&ftrace_lock);
2562 
2563 	return found ? 0 : -EINVAL;
2564 }
2565 
2566 static ssize_t
2567 ftrace_graph_write(struct file *file, const char __user *ubuf,
2568 		   size_t cnt, loff_t *ppos)
2569 {
2570 	struct trace_parser parser;
2571 	ssize_t read, ret;
2572 
2573 	if (!cnt || cnt < 0)
2574 		return 0;
2575 
2576 	mutex_lock(&graph_lock);
2577 
2578 	if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
2579 		ret = -EBUSY;
2580 		goto out_unlock;
2581 	}
2582 
2583 	if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
2584 		ret = -ENOMEM;
2585 		goto out_unlock;
2586 	}
2587 
2588 	read = trace_get_user(&parser, ubuf, cnt, ppos);
2589 
2590 	if (read >= 0 && trace_parser_loaded((&parser))) {
2591 		parser.buffer[parser.idx] = 0;
2592 
2593 		/* we allow only one expression at a time */
2594 		ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2595 					parser.buffer);
2596 		if (ret)
2597 			goto out_free;
2598 	}
2599 
2600 	ret = read;
2601 
2602 out_free:
2603 	trace_parser_put(&parser);
2604 out_unlock:
2605 	mutex_unlock(&graph_lock);
2606 
2607 	return ret;
2608 }
2609 
2610 static const struct file_operations ftrace_graph_fops = {
2611 	.open		= ftrace_graph_open,
2612 	.read		= seq_read,
2613 	.write		= ftrace_graph_write,
2614 	.release	= ftrace_graph_release,
2615 };
2616 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2617 
2618 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2619 {
2620 
2621 	trace_create_file("available_filter_functions", 0444,
2622 			d_tracer, NULL, &ftrace_avail_fops);
2623 
2624 	trace_create_file("failures", 0444,
2625 			d_tracer, NULL, &ftrace_failures_fops);
2626 
2627 	trace_create_file("set_ftrace_filter", 0644, d_tracer,
2628 			NULL, &ftrace_filter_fops);
2629 
2630 	trace_create_file("set_ftrace_notrace", 0644, d_tracer,
2631 				    NULL, &ftrace_notrace_fops);
2632 
2633 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2634 	trace_create_file("set_graph_function", 0444, d_tracer,
2635 				    NULL,
2636 				    &ftrace_graph_fops);
2637 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2638 
2639 	return 0;
2640 }
2641 
2642 static int ftrace_process_locs(struct module *mod,
2643 			       unsigned long *start,
2644 			       unsigned long *end)
2645 {
2646 	unsigned long *p;
2647 	unsigned long addr;
2648 	unsigned long flags;
2649 
2650 	mutex_lock(&ftrace_lock);
2651 	p = start;
2652 	while (p < end) {
2653 		addr = ftrace_call_adjust(*p++);
2654 		/*
2655 		 * Some architecture linkers will pad between
2656 		 * the different mcount_loc sections of different
2657 		 * object files to satisfy alignments.
2658 		 * Skip any NULL pointers.
2659 		 */
2660 		if (!addr)
2661 			continue;
2662 		ftrace_record_ip(addr);
2663 	}
2664 
2665 	/* disable interrupts to prevent kstop machine */
2666 	local_irq_save(flags);
2667 	ftrace_update_code(mod);
2668 	local_irq_restore(flags);
2669 	mutex_unlock(&ftrace_lock);
2670 
2671 	return 0;
2672 }
2673 
2674 #ifdef CONFIG_MODULES
2675 void ftrace_release_mod(struct module *mod)
2676 {
2677 	struct dyn_ftrace *rec;
2678 	struct ftrace_page *pg;
2679 
2680 	if (ftrace_disabled)
2681 		return;
2682 
2683 	mutex_lock(&ftrace_lock);
2684 	do_for_each_ftrace_rec(pg, rec) {
2685 		if (within_module_core(rec->ip, mod)) {
2686 			/*
2687 			 * rec->ip is changed in ftrace_free_rec()
2688 			 * It should not between s and e if record was freed.
2689 			 */
2690 			FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
2691 			ftrace_free_rec(rec);
2692 		}
2693 	} while_for_each_ftrace_rec();
2694 	mutex_unlock(&ftrace_lock);
2695 }
2696 
2697 static void ftrace_init_module(struct module *mod,
2698 			       unsigned long *start, unsigned long *end)
2699 {
2700 	if (ftrace_disabled || start == end)
2701 		return;
2702 	ftrace_process_locs(mod, start, end);
2703 }
2704 
2705 static int ftrace_module_notify(struct notifier_block *self,
2706 				unsigned long val, void *data)
2707 {
2708 	struct module *mod = data;
2709 
2710 	switch (val) {
2711 	case MODULE_STATE_COMING:
2712 		ftrace_init_module(mod, mod->ftrace_callsites,
2713 				   mod->ftrace_callsites +
2714 				   mod->num_ftrace_callsites);
2715 		break;
2716 	case MODULE_STATE_GOING:
2717 		ftrace_release_mod(mod);
2718 		break;
2719 	}
2720 
2721 	return 0;
2722 }
2723 #else
2724 static int ftrace_module_notify(struct notifier_block *self,
2725 				unsigned long val, void *data)
2726 {
2727 	return 0;
2728 }
2729 #endif /* CONFIG_MODULES */
2730 
2731 struct notifier_block ftrace_module_nb = {
2732 	.notifier_call = ftrace_module_notify,
2733 	.priority = 0,
2734 };
2735 
2736 extern unsigned long __start_mcount_loc[];
2737 extern unsigned long __stop_mcount_loc[];
2738 
2739 void __init ftrace_init(void)
2740 {
2741 	unsigned long count, addr, flags;
2742 	int ret;
2743 
2744 	/* Keep the ftrace pointer to the stub */
2745 	addr = (unsigned long)ftrace_stub;
2746 
2747 	local_irq_save(flags);
2748 	ftrace_dyn_arch_init(&addr);
2749 	local_irq_restore(flags);
2750 
2751 	/* ftrace_dyn_arch_init places the return code in addr */
2752 	if (addr)
2753 		goto failed;
2754 
2755 	count = __stop_mcount_loc - __start_mcount_loc;
2756 
2757 	ret = ftrace_dyn_table_alloc(count);
2758 	if (ret)
2759 		goto failed;
2760 
2761 	last_ftrace_enabled = ftrace_enabled = 1;
2762 
2763 	ret = ftrace_process_locs(NULL,
2764 				  __start_mcount_loc,
2765 				  __stop_mcount_loc);
2766 
2767 	ret = register_module_notifier(&ftrace_module_nb);
2768 	if (ret)
2769 		pr_warning("Failed to register trace ftrace module notifier\n");
2770 
2771 	set_ftrace_early_filters();
2772 
2773 	return;
2774  failed:
2775 	ftrace_disabled = 1;
2776 }
2777 
2778 #else
2779 
2780 static int __init ftrace_nodyn_init(void)
2781 {
2782 	ftrace_enabled = 1;
2783 	return 0;
2784 }
2785 device_initcall(ftrace_nodyn_init);
2786 
2787 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2788 static inline void ftrace_startup_enable(int command) { }
2789 /* Keep as macros so we do not need to define the commands */
2790 # define ftrace_startup(command)	do { } while (0)
2791 # define ftrace_shutdown(command)	do { } while (0)
2792 # define ftrace_startup_sysctl()	do { } while (0)
2793 # define ftrace_shutdown_sysctl()	do { } while (0)
2794 #endif /* CONFIG_DYNAMIC_FTRACE */
2795 
2796 static void clear_ftrace_swapper(void)
2797 {
2798 	struct task_struct *p;
2799 	int cpu;
2800 
2801 	get_online_cpus();
2802 	for_each_online_cpu(cpu) {
2803 		p = idle_task(cpu);
2804 		clear_tsk_trace_trace(p);
2805 	}
2806 	put_online_cpus();
2807 }
2808 
2809 static void set_ftrace_swapper(void)
2810 {
2811 	struct task_struct *p;
2812 	int cpu;
2813 
2814 	get_online_cpus();
2815 	for_each_online_cpu(cpu) {
2816 		p = idle_task(cpu);
2817 		set_tsk_trace_trace(p);
2818 	}
2819 	put_online_cpus();
2820 }
2821 
2822 static void clear_ftrace_pid(struct pid *pid)
2823 {
2824 	struct task_struct *p;
2825 
2826 	rcu_read_lock();
2827 	do_each_pid_task(pid, PIDTYPE_PID, p) {
2828 		clear_tsk_trace_trace(p);
2829 	} while_each_pid_task(pid, PIDTYPE_PID, p);
2830 	rcu_read_unlock();
2831 
2832 	put_pid(pid);
2833 }
2834 
2835 static void set_ftrace_pid(struct pid *pid)
2836 {
2837 	struct task_struct *p;
2838 
2839 	rcu_read_lock();
2840 	do_each_pid_task(pid, PIDTYPE_PID, p) {
2841 		set_tsk_trace_trace(p);
2842 	} while_each_pid_task(pid, PIDTYPE_PID, p);
2843 	rcu_read_unlock();
2844 }
2845 
2846 static void clear_ftrace_pid_task(struct pid *pid)
2847 {
2848 	if (pid == ftrace_swapper_pid)
2849 		clear_ftrace_swapper();
2850 	else
2851 		clear_ftrace_pid(pid);
2852 }
2853 
2854 static void set_ftrace_pid_task(struct pid *pid)
2855 {
2856 	if (pid == ftrace_swapper_pid)
2857 		set_ftrace_swapper();
2858 	else
2859 		set_ftrace_pid(pid);
2860 }
2861 
2862 static int ftrace_pid_add(int p)
2863 {
2864 	struct pid *pid;
2865 	struct ftrace_pid *fpid;
2866 	int ret = -EINVAL;
2867 
2868 	mutex_lock(&ftrace_lock);
2869 
2870 	if (!p)
2871 		pid = ftrace_swapper_pid;
2872 	else
2873 		pid = find_get_pid(p);
2874 
2875 	if (!pid)
2876 		goto out;
2877 
2878 	ret = 0;
2879 
2880 	list_for_each_entry(fpid, &ftrace_pids, list)
2881 		if (fpid->pid == pid)
2882 			goto out_put;
2883 
2884 	ret = -ENOMEM;
2885 
2886 	fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
2887 	if (!fpid)
2888 		goto out_put;
2889 
2890 	list_add(&fpid->list, &ftrace_pids);
2891 	fpid->pid = pid;
2892 
2893 	set_ftrace_pid_task(pid);
2894 
2895 	ftrace_update_pid_func();
2896 	ftrace_startup_enable(0);
2897 
2898 	mutex_unlock(&ftrace_lock);
2899 	return 0;
2900 
2901 out_put:
2902 	if (pid != ftrace_swapper_pid)
2903 		put_pid(pid);
2904 
2905 out:
2906 	mutex_unlock(&ftrace_lock);
2907 	return ret;
2908 }
2909 
2910 static void ftrace_pid_reset(void)
2911 {
2912 	struct ftrace_pid *fpid, *safe;
2913 
2914 	mutex_lock(&ftrace_lock);
2915 	list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
2916 		struct pid *pid = fpid->pid;
2917 
2918 		clear_ftrace_pid_task(pid);
2919 
2920 		list_del(&fpid->list);
2921 		kfree(fpid);
2922 	}
2923 
2924 	ftrace_update_pid_func();
2925 	ftrace_startup_enable(0);
2926 
2927 	mutex_unlock(&ftrace_lock);
2928 }
2929 
2930 static void *fpid_start(struct seq_file *m, loff_t *pos)
2931 {
2932 	mutex_lock(&ftrace_lock);
2933 
2934 	if (list_empty(&ftrace_pids) && (!*pos))
2935 		return (void *) 1;
2936 
2937 	return seq_list_start(&ftrace_pids, *pos);
2938 }
2939 
2940 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
2941 {
2942 	if (v == (void *)1)
2943 		return NULL;
2944 
2945 	return seq_list_next(v, &ftrace_pids, pos);
2946 }
2947 
2948 static void fpid_stop(struct seq_file *m, void *p)
2949 {
2950 	mutex_unlock(&ftrace_lock);
2951 }
2952 
2953 static int fpid_show(struct seq_file *m, void *v)
2954 {
2955 	const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
2956 
2957 	if (v == (void *)1) {
2958 		seq_printf(m, "no pid\n");
2959 		return 0;
2960 	}
2961 
2962 	if (fpid->pid == ftrace_swapper_pid)
2963 		seq_printf(m, "swapper tasks\n");
2964 	else
2965 		seq_printf(m, "%u\n", pid_vnr(fpid->pid));
2966 
2967 	return 0;
2968 }
2969 
2970 static const struct seq_operations ftrace_pid_sops = {
2971 	.start = fpid_start,
2972 	.next = fpid_next,
2973 	.stop = fpid_stop,
2974 	.show = fpid_show,
2975 };
2976 
2977 static int
2978 ftrace_pid_open(struct inode *inode, struct file *file)
2979 {
2980 	int ret = 0;
2981 
2982 	if ((file->f_mode & FMODE_WRITE) &&
2983 	    (file->f_flags & O_TRUNC))
2984 		ftrace_pid_reset();
2985 
2986 	if (file->f_mode & FMODE_READ)
2987 		ret = seq_open(file, &ftrace_pid_sops);
2988 
2989 	return ret;
2990 }
2991 
2992 static ssize_t
2993 ftrace_pid_write(struct file *filp, const char __user *ubuf,
2994 		   size_t cnt, loff_t *ppos)
2995 {
2996 	char buf[64], *tmp;
2997 	long val;
2998 	int ret;
2999 
3000 	if (cnt >= sizeof(buf))
3001 		return -EINVAL;
3002 
3003 	if (copy_from_user(&buf, ubuf, cnt))
3004 		return -EFAULT;
3005 
3006 	buf[cnt] = 0;
3007 
3008 	/*
3009 	 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
3010 	 * to clean the filter quietly.
3011 	 */
3012 	tmp = strstrip(buf);
3013 	if (strlen(tmp) == 0)
3014 		return 1;
3015 
3016 	ret = strict_strtol(tmp, 10, &val);
3017 	if (ret < 0)
3018 		return ret;
3019 
3020 	ret = ftrace_pid_add(val);
3021 
3022 	return ret ? ret : cnt;
3023 }
3024 
3025 static int
3026 ftrace_pid_release(struct inode *inode, struct file *file)
3027 {
3028 	if (file->f_mode & FMODE_READ)
3029 		seq_release(inode, file);
3030 
3031 	return 0;
3032 }
3033 
3034 static const struct file_operations ftrace_pid_fops = {
3035 	.open		= ftrace_pid_open,
3036 	.write		= ftrace_pid_write,
3037 	.read		= seq_read,
3038 	.llseek		= seq_lseek,
3039 	.release	= ftrace_pid_release,
3040 };
3041 
3042 static __init int ftrace_init_debugfs(void)
3043 {
3044 	struct dentry *d_tracer;
3045 
3046 	d_tracer = tracing_init_dentry();
3047 	if (!d_tracer)
3048 		return 0;
3049 
3050 	ftrace_init_dyn_debugfs(d_tracer);
3051 
3052 	trace_create_file("set_ftrace_pid", 0644, d_tracer,
3053 			    NULL, &ftrace_pid_fops);
3054 
3055 	ftrace_profile_debugfs(d_tracer);
3056 
3057 	return 0;
3058 }
3059 fs_initcall(ftrace_init_debugfs);
3060 
3061 /**
3062  * ftrace_kill - kill ftrace
3063  *
3064  * This function should be used by panic code. It stops ftrace
3065  * but in a not so nice way. If you need to simply kill ftrace
3066  * from a non-atomic section, use ftrace_kill.
3067  */
3068 void ftrace_kill(void)
3069 {
3070 	ftrace_disabled = 1;
3071 	ftrace_enabled = 0;
3072 	clear_ftrace_function();
3073 }
3074 
3075 /**
3076  * register_ftrace_function - register a function for profiling
3077  * @ops - ops structure that holds the function for profiling.
3078  *
3079  * Register a function to be called by all functions in the
3080  * kernel.
3081  *
3082  * Note: @ops->func and all the functions it calls must be labeled
3083  *       with "notrace", otherwise it will go into a
3084  *       recursive loop.
3085  */
3086 int register_ftrace_function(struct ftrace_ops *ops)
3087 {
3088 	int ret;
3089 
3090 	if (unlikely(ftrace_disabled))
3091 		return -1;
3092 
3093 	mutex_lock(&ftrace_lock);
3094 
3095 	ret = __register_ftrace_function(ops);
3096 	ftrace_startup(0);
3097 
3098 	mutex_unlock(&ftrace_lock);
3099 	return ret;
3100 }
3101 
3102 /**
3103  * unregister_ftrace_function - unregister a function for profiling.
3104  * @ops - ops structure that holds the function to unregister
3105  *
3106  * Unregister a function that was added to be called by ftrace profiling.
3107  */
3108 int unregister_ftrace_function(struct ftrace_ops *ops)
3109 {
3110 	int ret;
3111 
3112 	mutex_lock(&ftrace_lock);
3113 	ret = __unregister_ftrace_function(ops);
3114 	ftrace_shutdown(0);
3115 	mutex_unlock(&ftrace_lock);
3116 
3117 	return ret;
3118 }
3119 
3120 int
3121 ftrace_enable_sysctl(struct ctl_table *table, int write,
3122 		     void __user *buffer, size_t *lenp,
3123 		     loff_t *ppos)
3124 {
3125 	int ret;
3126 
3127 	if (unlikely(ftrace_disabled))
3128 		return -ENODEV;
3129 
3130 	mutex_lock(&ftrace_lock);
3131 
3132 	ret  = proc_dointvec(table, write, buffer, lenp, ppos);
3133 
3134 	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
3135 		goto out;
3136 
3137 	last_ftrace_enabled = !!ftrace_enabled;
3138 
3139 	if (ftrace_enabled) {
3140 
3141 		ftrace_startup_sysctl();
3142 
3143 		/* we are starting ftrace again */
3144 		if (ftrace_list != &ftrace_list_end) {
3145 			if (ftrace_list->next == &ftrace_list_end)
3146 				ftrace_trace_function = ftrace_list->func;
3147 			else
3148 				ftrace_trace_function = ftrace_list_func;
3149 		}
3150 
3151 	} else {
3152 		/* stopping ftrace calls (just send to ftrace_stub) */
3153 		ftrace_trace_function = ftrace_stub;
3154 
3155 		ftrace_shutdown_sysctl();
3156 	}
3157 
3158  out:
3159 	mutex_unlock(&ftrace_lock);
3160 	return ret;
3161 }
3162 
3163 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3164 
3165 static int ftrace_graph_active;
3166 static struct notifier_block ftrace_suspend_notifier;
3167 
3168 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
3169 {
3170 	return 0;
3171 }
3172 
3173 /* The callbacks that hook a function */
3174 trace_func_graph_ret_t ftrace_graph_return =
3175 			(trace_func_graph_ret_t)ftrace_stub;
3176 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
3177 
3178 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
3179 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
3180 {
3181 	int i;
3182 	int ret = 0;
3183 	unsigned long flags;
3184 	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
3185 	struct task_struct *g, *t;
3186 
3187 	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
3188 		ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
3189 					* sizeof(struct ftrace_ret_stack),
3190 					GFP_KERNEL);
3191 		if (!ret_stack_list[i]) {
3192 			start = 0;
3193 			end = i;
3194 			ret = -ENOMEM;
3195 			goto free;
3196 		}
3197 	}
3198 
3199 	read_lock_irqsave(&tasklist_lock, flags);
3200 	do_each_thread(g, t) {
3201 		if (start == end) {
3202 			ret = -EAGAIN;
3203 			goto unlock;
3204 		}
3205 
3206 		if (t->ret_stack == NULL) {
3207 			atomic_set(&t->tracing_graph_pause, 0);
3208 			atomic_set(&t->trace_overrun, 0);
3209 			t->curr_ret_stack = -1;
3210 			/* Make sure the tasks see the -1 first: */
3211 			smp_wmb();
3212 			t->ret_stack = ret_stack_list[start++];
3213 		}
3214 	} while_each_thread(g, t);
3215 
3216 unlock:
3217 	read_unlock_irqrestore(&tasklist_lock, flags);
3218 free:
3219 	for (i = start; i < end; i++)
3220 		kfree(ret_stack_list[i]);
3221 	return ret;
3222 }
3223 
3224 static void
3225 ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev,
3226 				struct task_struct *next)
3227 {
3228 	unsigned long long timestamp;
3229 	int index;
3230 
3231 	/*
3232 	 * Does the user want to count the time a function was asleep.
3233 	 * If so, do not update the time stamps.
3234 	 */
3235 	if (trace_flags & TRACE_ITER_SLEEP_TIME)
3236 		return;
3237 
3238 	timestamp = trace_clock_local();
3239 
3240 	prev->ftrace_timestamp = timestamp;
3241 
3242 	/* only process tasks that we timestamped */
3243 	if (!next->ftrace_timestamp)
3244 		return;
3245 
3246 	/*
3247 	 * Update all the counters in next to make up for the
3248 	 * time next was sleeping.
3249 	 */
3250 	timestamp -= next->ftrace_timestamp;
3251 
3252 	for (index = next->curr_ret_stack; index >= 0; index--)
3253 		next->ret_stack[index].calltime += timestamp;
3254 }
3255 
3256 /* Allocate a return stack for each task */
3257 static int start_graph_tracing(void)
3258 {
3259 	struct ftrace_ret_stack **ret_stack_list;
3260 	int ret, cpu;
3261 
3262 	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
3263 				sizeof(struct ftrace_ret_stack *),
3264 				GFP_KERNEL);
3265 
3266 	if (!ret_stack_list)
3267 		return -ENOMEM;
3268 
3269 	/* The cpu_boot init_task->ret_stack will never be freed */
3270 	for_each_online_cpu(cpu) {
3271 		if (!idle_task(cpu)->ret_stack)
3272 			ftrace_graph_init_task(idle_task(cpu));
3273 	}
3274 
3275 	do {
3276 		ret = alloc_retstack_tasklist(ret_stack_list);
3277 	} while (ret == -EAGAIN);
3278 
3279 	if (!ret) {
3280 		ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch);
3281 		if (ret)
3282 			pr_info("ftrace_graph: Couldn't activate tracepoint"
3283 				" probe to kernel_sched_switch\n");
3284 	}
3285 
3286 	kfree(ret_stack_list);
3287 	return ret;
3288 }
3289 
3290 /*
3291  * Hibernation protection.
3292  * The state of the current task is too much unstable during
3293  * suspend/restore to disk. We want to protect against that.
3294  */
3295 static int
3296 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
3297 							void *unused)
3298 {
3299 	switch (state) {
3300 	case PM_HIBERNATION_PREPARE:
3301 		pause_graph_tracing();
3302 		break;
3303 
3304 	case PM_POST_HIBERNATION:
3305 		unpause_graph_tracing();
3306 		break;
3307 	}
3308 	return NOTIFY_DONE;
3309 }
3310 
3311 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
3312 			trace_func_graph_ent_t entryfunc)
3313 {
3314 	int ret = 0;
3315 
3316 	mutex_lock(&ftrace_lock);
3317 
3318 	/* we currently allow only one tracer registered at a time */
3319 	if (ftrace_graph_active) {
3320 		ret = -EBUSY;
3321 		goto out;
3322 	}
3323 
3324 	ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
3325 	register_pm_notifier(&ftrace_suspend_notifier);
3326 
3327 	ftrace_graph_active++;
3328 	ret = start_graph_tracing();
3329 	if (ret) {
3330 		ftrace_graph_active--;
3331 		goto out;
3332 	}
3333 
3334 	ftrace_graph_return = retfunc;
3335 	ftrace_graph_entry = entryfunc;
3336 
3337 	ftrace_startup(FTRACE_START_FUNC_RET);
3338 
3339 out:
3340 	mutex_unlock(&ftrace_lock);
3341 	return ret;
3342 }
3343 
3344 void unregister_ftrace_graph(void)
3345 {
3346 	mutex_lock(&ftrace_lock);
3347 
3348 	if (unlikely(!ftrace_graph_active))
3349 		goto out;
3350 
3351 	ftrace_graph_active--;
3352 	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch);
3353 	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
3354 	ftrace_graph_entry = ftrace_graph_entry_stub;
3355 	ftrace_shutdown(FTRACE_STOP_FUNC_RET);
3356 	unregister_pm_notifier(&ftrace_suspend_notifier);
3357 
3358  out:
3359 	mutex_unlock(&ftrace_lock);
3360 }
3361 
3362 /* Allocate a return stack for newly created task */
3363 void ftrace_graph_init_task(struct task_struct *t)
3364 {
3365 	/* Make sure we do not use the parent ret_stack */
3366 	t->ret_stack = NULL;
3367 
3368 	if (ftrace_graph_active) {
3369 		struct ftrace_ret_stack *ret_stack;
3370 
3371 		ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
3372 				* sizeof(struct ftrace_ret_stack),
3373 				GFP_KERNEL);
3374 		if (!ret_stack)
3375 			return;
3376 		t->curr_ret_stack = -1;
3377 		atomic_set(&t->tracing_graph_pause, 0);
3378 		atomic_set(&t->trace_overrun, 0);
3379 		t->ftrace_timestamp = 0;
3380 		/* make curr_ret_stack visable before we add the ret_stack */
3381 		smp_wmb();
3382 		t->ret_stack = ret_stack;
3383 	}
3384 }
3385 
3386 void ftrace_graph_exit_task(struct task_struct *t)
3387 {
3388 	struct ftrace_ret_stack	*ret_stack = t->ret_stack;
3389 
3390 	t->ret_stack = NULL;
3391 	/* NULL must become visible to IRQs before we free it: */
3392 	barrier();
3393 
3394 	kfree(ret_stack);
3395 }
3396 
3397 void ftrace_graph_stop(void)
3398 {
3399 	ftrace_stop();
3400 }
3401 #endif
3402