1 /* 2 * Infrastructure for profiling code inserted by 'gcc -pg'. 3 * 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> 6 * 7 * Originally ported from the -rt patch by: 8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> 9 * 10 * Based on code in the latency_tracer, that is: 11 * 12 * Copyright (C) 2004-2006 Ingo Molnar 13 * Copyright (C) 2004 William Lee Irwin III 14 */ 15 16 #include <linux/stop_machine.h> 17 #include <linux/clocksource.h> 18 #include <linux/kallsyms.h> 19 #include <linux/seq_file.h> 20 #include <linux/suspend.h> 21 #include <linux/debugfs.h> 22 #include <linux/hardirq.h> 23 #include <linux/kthread.h> 24 #include <linux/uaccess.h> 25 #include <linux/kprobes.h> 26 #include <linux/ftrace.h> 27 #include <linux/sysctl.h> 28 #include <linux/ctype.h> 29 #include <linux/list.h> 30 #include <linux/hash.h> 31 32 #include <trace/events/sched.h> 33 34 #include <asm/ftrace.h> 35 #include <asm/setup.h> 36 37 #include "trace_output.h" 38 #include "trace_stat.h" 39 40 #define FTRACE_WARN_ON(cond) \ 41 do { \ 42 if (WARN_ON(cond)) \ 43 ftrace_kill(); \ 44 } while (0) 45 46 #define FTRACE_WARN_ON_ONCE(cond) \ 47 do { \ 48 if (WARN_ON_ONCE(cond)) \ 49 ftrace_kill(); \ 50 } while (0) 51 52 /* hash bits for specific function selection */ 53 #define FTRACE_HASH_BITS 7 54 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS) 55 56 /* ftrace_enabled is a method to turn ftrace on or off */ 57 int ftrace_enabled __read_mostly; 58 static int last_ftrace_enabled; 59 60 /* Quick disabling of function tracer. */ 61 int function_trace_stop; 62 63 /* List for set_ftrace_pid's pids. */ 64 LIST_HEAD(ftrace_pids); 65 struct ftrace_pid { 66 struct list_head list; 67 struct pid *pid; 68 }; 69 70 /* 71 * ftrace_disabled is set when an anomaly is discovered. 72 * ftrace_disabled is much stronger than ftrace_enabled. 73 */ 74 static int ftrace_disabled __read_mostly; 75 76 static DEFINE_MUTEX(ftrace_lock); 77 78 static struct ftrace_ops ftrace_list_end __read_mostly = 79 { 80 .func = ftrace_stub, 81 }; 82 83 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end; 84 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; 85 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; 86 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; 87 88 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 89 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer); 90 #endif 91 92 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) 93 { 94 struct ftrace_ops *op = ftrace_list; 95 96 /* in case someone actually ports this to alpha! */ 97 read_barrier_depends(); 98 99 while (op != &ftrace_list_end) { 100 /* silly alpha */ 101 read_barrier_depends(); 102 op->func(ip, parent_ip); 103 op = op->next; 104 }; 105 } 106 107 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip) 108 { 109 if (!test_tsk_trace_trace(current)) 110 return; 111 112 ftrace_pid_function(ip, parent_ip); 113 } 114 115 static void set_ftrace_pid_function(ftrace_func_t func) 116 { 117 /* do not set ftrace_pid_function to itself! */ 118 if (func != ftrace_pid_func) 119 ftrace_pid_function = func; 120 } 121 122 /** 123 * clear_ftrace_function - reset the ftrace function 124 * 125 * This NULLs the ftrace function and in essence stops 126 * tracing. There may be lag 127 */ 128 void clear_ftrace_function(void) 129 { 130 ftrace_trace_function = ftrace_stub; 131 __ftrace_trace_function = ftrace_stub; 132 ftrace_pid_function = ftrace_stub; 133 } 134 135 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 136 /* 137 * For those archs that do not test ftrace_trace_stop in their 138 * mcount call site, we need to do it from C. 139 */ 140 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip) 141 { 142 if (function_trace_stop) 143 return; 144 145 __ftrace_trace_function(ip, parent_ip); 146 } 147 #endif 148 149 static int __register_ftrace_function(struct ftrace_ops *ops) 150 { 151 ops->next = ftrace_list; 152 /* 153 * We are entering ops into the ftrace_list but another 154 * CPU might be walking that list. We need to make sure 155 * the ops->next pointer is valid before another CPU sees 156 * the ops pointer included into the ftrace_list. 157 */ 158 smp_wmb(); 159 ftrace_list = ops; 160 161 if (ftrace_enabled) { 162 ftrace_func_t func; 163 164 if (ops->next == &ftrace_list_end) 165 func = ops->func; 166 else 167 func = ftrace_list_func; 168 169 if (!list_empty(&ftrace_pids)) { 170 set_ftrace_pid_function(func); 171 func = ftrace_pid_func; 172 } 173 174 /* 175 * For one func, simply call it directly. 176 * For more than one func, call the chain. 177 */ 178 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 179 ftrace_trace_function = func; 180 #else 181 __ftrace_trace_function = func; 182 ftrace_trace_function = ftrace_test_stop_func; 183 #endif 184 } 185 186 return 0; 187 } 188 189 static int __unregister_ftrace_function(struct ftrace_ops *ops) 190 { 191 struct ftrace_ops **p; 192 193 /* 194 * If we are removing the last function, then simply point 195 * to the ftrace_stub. 196 */ 197 if (ftrace_list == ops && ops->next == &ftrace_list_end) { 198 ftrace_trace_function = ftrace_stub; 199 ftrace_list = &ftrace_list_end; 200 return 0; 201 } 202 203 for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next) 204 if (*p == ops) 205 break; 206 207 if (*p != ops) 208 return -1; 209 210 *p = (*p)->next; 211 212 if (ftrace_enabled) { 213 /* If we only have one func left, then call that directly */ 214 if (ftrace_list->next == &ftrace_list_end) { 215 ftrace_func_t func = ftrace_list->func; 216 217 if (!list_empty(&ftrace_pids)) { 218 set_ftrace_pid_function(func); 219 func = ftrace_pid_func; 220 } 221 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 222 ftrace_trace_function = func; 223 #else 224 __ftrace_trace_function = func; 225 #endif 226 } 227 } 228 229 return 0; 230 } 231 232 static void ftrace_update_pid_func(void) 233 { 234 ftrace_func_t func; 235 236 if (ftrace_trace_function == ftrace_stub) 237 return; 238 239 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 240 func = ftrace_trace_function; 241 #else 242 func = __ftrace_trace_function; 243 #endif 244 245 if (!list_empty(&ftrace_pids)) { 246 set_ftrace_pid_function(func); 247 func = ftrace_pid_func; 248 } else { 249 if (func == ftrace_pid_func) 250 func = ftrace_pid_function; 251 } 252 253 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 254 ftrace_trace_function = func; 255 #else 256 __ftrace_trace_function = func; 257 #endif 258 } 259 260 #ifdef CONFIG_FUNCTION_PROFILER 261 struct ftrace_profile { 262 struct hlist_node node; 263 unsigned long ip; 264 unsigned long counter; 265 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 266 unsigned long long time; 267 #endif 268 }; 269 270 struct ftrace_profile_page { 271 struct ftrace_profile_page *next; 272 unsigned long index; 273 struct ftrace_profile records[]; 274 }; 275 276 struct ftrace_profile_stat { 277 atomic_t disabled; 278 struct hlist_head *hash; 279 struct ftrace_profile_page *pages; 280 struct ftrace_profile_page *start; 281 struct tracer_stat stat; 282 }; 283 284 #define PROFILE_RECORDS_SIZE \ 285 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records)) 286 287 #define PROFILES_PER_PAGE \ 288 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile)) 289 290 static int ftrace_profile_bits __read_mostly; 291 static int ftrace_profile_enabled __read_mostly; 292 293 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */ 294 static DEFINE_MUTEX(ftrace_profile_lock); 295 296 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats); 297 298 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */ 299 300 static void * 301 function_stat_next(void *v, int idx) 302 { 303 struct ftrace_profile *rec = v; 304 struct ftrace_profile_page *pg; 305 306 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); 307 308 again: 309 if (idx != 0) 310 rec++; 311 312 if ((void *)rec >= (void *)&pg->records[pg->index]) { 313 pg = pg->next; 314 if (!pg) 315 return NULL; 316 rec = &pg->records[0]; 317 if (!rec->counter) 318 goto again; 319 } 320 321 return rec; 322 } 323 324 static void *function_stat_start(struct tracer_stat *trace) 325 { 326 struct ftrace_profile_stat *stat = 327 container_of(trace, struct ftrace_profile_stat, stat); 328 329 if (!stat || !stat->start) 330 return NULL; 331 332 return function_stat_next(&stat->start->records[0], 0); 333 } 334 335 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 336 /* function graph compares on total time */ 337 static int function_stat_cmp(void *p1, void *p2) 338 { 339 struct ftrace_profile *a = p1; 340 struct ftrace_profile *b = p2; 341 342 if (a->time < b->time) 343 return -1; 344 if (a->time > b->time) 345 return 1; 346 else 347 return 0; 348 } 349 #else 350 /* not function graph compares against hits */ 351 static int function_stat_cmp(void *p1, void *p2) 352 { 353 struct ftrace_profile *a = p1; 354 struct ftrace_profile *b = p2; 355 356 if (a->counter < b->counter) 357 return -1; 358 if (a->counter > b->counter) 359 return 1; 360 else 361 return 0; 362 } 363 #endif 364 365 static int function_stat_headers(struct seq_file *m) 366 { 367 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 368 seq_printf(m, " Function " 369 "Hit Time Avg\n" 370 " -------- " 371 "--- ---- ---\n"); 372 #else 373 seq_printf(m, " Function Hit\n" 374 " -------- ---\n"); 375 #endif 376 return 0; 377 } 378 379 static int function_stat_show(struct seq_file *m, void *v) 380 { 381 struct ftrace_profile *rec = v; 382 char str[KSYM_SYMBOL_LEN]; 383 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 384 static DEFINE_MUTEX(mutex); 385 static struct trace_seq s; 386 unsigned long long avg; 387 #endif 388 389 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 390 seq_printf(m, " %-30.30s %10lu", str, rec->counter); 391 392 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 393 seq_printf(m, " "); 394 avg = rec->time; 395 do_div(avg, rec->counter); 396 397 mutex_lock(&mutex); 398 trace_seq_init(&s); 399 trace_print_graph_duration(rec->time, &s); 400 trace_seq_puts(&s, " "); 401 trace_print_graph_duration(avg, &s); 402 trace_print_seq(m, &s); 403 mutex_unlock(&mutex); 404 #endif 405 seq_putc(m, '\n'); 406 407 return 0; 408 } 409 410 static void ftrace_profile_reset(struct ftrace_profile_stat *stat) 411 { 412 struct ftrace_profile_page *pg; 413 414 pg = stat->pages = stat->start; 415 416 while (pg) { 417 memset(pg->records, 0, PROFILE_RECORDS_SIZE); 418 pg->index = 0; 419 pg = pg->next; 420 } 421 422 memset(stat->hash, 0, 423 FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head)); 424 } 425 426 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat) 427 { 428 struct ftrace_profile_page *pg; 429 int functions; 430 int pages; 431 int i; 432 433 /* If we already allocated, do nothing */ 434 if (stat->pages) 435 return 0; 436 437 stat->pages = (void *)get_zeroed_page(GFP_KERNEL); 438 if (!stat->pages) 439 return -ENOMEM; 440 441 #ifdef CONFIG_DYNAMIC_FTRACE 442 functions = ftrace_update_tot_cnt; 443 #else 444 /* 445 * We do not know the number of functions that exist because 446 * dynamic tracing is what counts them. With past experience 447 * we have around 20K functions. That should be more than enough. 448 * It is highly unlikely we will execute every function in 449 * the kernel. 450 */ 451 functions = 20000; 452 #endif 453 454 pg = stat->start = stat->pages; 455 456 pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE); 457 458 for (i = 0; i < pages; i++) { 459 pg->next = (void *)get_zeroed_page(GFP_KERNEL); 460 if (!pg->next) 461 goto out_free; 462 pg = pg->next; 463 } 464 465 return 0; 466 467 out_free: 468 pg = stat->start; 469 while (pg) { 470 unsigned long tmp = (unsigned long)pg; 471 472 pg = pg->next; 473 free_page(tmp); 474 } 475 476 free_page((unsigned long)stat->pages); 477 stat->pages = NULL; 478 stat->start = NULL; 479 480 return -ENOMEM; 481 } 482 483 static int ftrace_profile_init_cpu(int cpu) 484 { 485 struct ftrace_profile_stat *stat; 486 int size; 487 488 stat = &per_cpu(ftrace_profile_stats, cpu); 489 490 if (stat->hash) { 491 /* If the profile is already created, simply reset it */ 492 ftrace_profile_reset(stat); 493 return 0; 494 } 495 496 /* 497 * We are profiling all functions, but usually only a few thousand 498 * functions are hit. We'll make a hash of 1024 items. 499 */ 500 size = FTRACE_PROFILE_HASH_SIZE; 501 502 stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL); 503 504 if (!stat->hash) 505 return -ENOMEM; 506 507 if (!ftrace_profile_bits) { 508 size--; 509 510 for (; size; size >>= 1) 511 ftrace_profile_bits++; 512 } 513 514 /* Preallocate the function profiling pages */ 515 if (ftrace_profile_pages_init(stat) < 0) { 516 kfree(stat->hash); 517 stat->hash = NULL; 518 return -ENOMEM; 519 } 520 521 return 0; 522 } 523 524 static int ftrace_profile_init(void) 525 { 526 int cpu; 527 int ret = 0; 528 529 for_each_online_cpu(cpu) { 530 ret = ftrace_profile_init_cpu(cpu); 531 if (ret) 532 break; 533 } 534 535 return ret; 536 } 537 538 /* interrupts must be disabled */ 539 static struct ftrace_profile * 540 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip) 541 { 542 struct ftrace_profile *rec; 543 struct hlist_head *hhd; 544 struct hlist_node *n; 545 unsigned long key; 546 547 key = hash_long(ip, ftrace_profile_bits); 548 hhd = &stat->hash[key]; 549 550 if (hlist_empty(hhd)) 551 return NULL; 552 553 hlist_for_each_entry_rcu(rec, n, hhd, node) { 554 if (rec->ip == ip) 555 return rec; 556 } 557 558 return NULL; 559 } 560 561 static void ftrace_add_profile(struct ftrace_profile_stat *stat, 562 struct ftrace_profile *rec) 563 { 564 unsigned long key; 565 566 key = hash_long(rec->ip, ftrace_profile_bits); 567 hlist_add_head_rcu(&rec->node, &stat->hash[key]); 568 } 569 570 /* 571 * The memory is already allocated, this simply finds a new record to use. 572 */ 573 static struct ftrace_profile * 574 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip) 575 { 576 struct ftrace_profile *rec = NULL; 577 578 /* prevent recursion (from NMIs) */ 579 if (atomic_inc_return(&stat->disabled) != 1) 580 goto out; 581 582 /* 583 * Try to find the function again since an NMI 584 * could have added it 585 */ 586 rec = ftrace_find_profiled_func(stat, ip); 587 if (rec) 588 goto out; 589 590 if (stat->pages->index == PROFILES_PER_PAGE) { 591 if (!stat->pages->next) 592 goto out; 593 stat->pages = stat->pages->next; 594 } 595 596 rec = &stat->pages->records[stat->pages->index++]; 597 rec->ip = ip; 598 ftrace_add_profile(stat, rec); 599 600 out: 601 atomic_dec(&stat->disabled); 602 603 return rec; 604 } 605 606 static void 607 function_profile_call(unsigned long ip, unsigned long parent_ip) 608 { 609 struct ftrace_profile_stat *stat; 610 struct ftrace_profile *rec; 611 unsigned long flags; 612 613 if (!ftrace_profile_enabled) 614 return; 615 616 local_irq_save(flags); 617 618 stat = &__get_cpu_var(ftrace_profile_stats); 619 if (!stat->hash || !ftrace_profile_enabled) 620 goto out; 621 622 rec = ftrace_find_profiled_func(stat, ip); 623 if (!rec) { 624 rec = ftrace_profile_alloc(stat, ip); 625 if (!rec) 626 goto out; 627 } 628 629 rec->counter++; 630 out: 631 local_irq_restore(flags); 632 } 633 634 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 635 static int profile_graph_entry(struct ftrace_graph_ent *trace) 636 { 637 function_profile_call(trace->func, 0); 638 return 1; 639 } 640 641 static void profile_graph_return(struct ftrace_graph_ret *trace) 642 { 643 struct ftrace_profile_stat *stat; 644 unsigned long long calltime; 645 struct ftrace_profile *rec; 646 unsigned long flags; 647 648 local_irq_save(flags); 649 stat = &__get_cpu_var(ftrace_profile_stats); 650 if (!stat->hash || !ftrace_profile_enabled) 651 goto out; 652 653 calltime = trace->rettime - trace->calltime; 654 655 if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) { 656 int index; 657 658 index = trace->depth; 659 660 /* Append this call time to the parent time to subtract */ 661 if (index) 662 current->ret_stack[index - 1].subtime += calltime; 663 664 if (current->ret_stack[index].subtime < calltime) 665 calltime -= current->ret_stack[index].subtime; 666 else 667 calltime = 0; 668 } 669 670 rec = ftrace_find_profiled_func(stat, trace->func); 671 if (rec) 672 rec->time += calltime; 673 674 out: 675 local_irq_restore(flags); 676 } 677 678 static int register_ftrace_profiler(void) 679 { 680 return register_ftrace_graph(&profile_graph_return, 681 &profile_graph_entry); 682 } 683 684 static void unregister_ftrace_profiler(void) 685 { 686 unregister_ftrace_graph(); 687 } 688 #else 689 static struct ftrace_ops ftrace_profile_ops __read_mostly = 690 { 691 .func = function_profile_call, 692 }; 693 694 static int register_ftrace_profiler(void) 695 { 696 return register_ftrace_function(&ftrace_profile_ops); 697 } 698 699 static void unregister_ftrace_profiler(void) 700 { 701 unregister_ftrace_function(&ftrace_profile_ops); 702 } 703 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 704 705 static ssize_t 706 ftrace_profile_write(struct file *filp, const char __user *ubuf, 707 size_t cnt, loff_t *ppos) 708 { 709 unsigned long val; 710 char buf[64]; /* big enough to hold a number */ 711 int ret; 712 713 if (cnt >= sizeof(buf)) 714 return -EINVAL; 715 716 if (copy_from_user(&buf, ubuf, cnt)) 717 return -EFAULT; 718 719 buf[cnt] = 0; 720 721 ret = strict_strtoul(buf, 10, &val); 722 if (ret < 0) 723 return ret; 724 725 val = !!val; 726 727 mutex_lock(&ftrace_profile_lock); 728 if (ftrace_profile_enabled ^ val) { 729 if (val) { 730 ret = ftrace_profile_init(); 731 if (ret < 0) { 732 cnt = ret; 733 goto out; 734 } 735 736 ret = register_ftrace_profiler(); 737 if (ret < 0) { 738 cnt = ret; 739 goto out; 740 } 741 ftrace_profile_enabled = 1; 742 } else { 743 ftrace_profile_enabled = 0; 744 /* 745 * unregister_ftrace_profiler calls stop_machine 746 * so this acts like an synchronize_sched. 747 */ 748 unregister_ftrace_profiler(); 749 } 750 } 751 out: 752 mutex_unlock(&ftrace_profile_lock); 753 754 *ppos += cnt; 755 756 return cnt; 757 } 758 759 static ssize_t 760 ftrace_profile_read(struct file *filp, char __user *ubuf, 761 size_t cnt, loff_t *ppos) 762 { 763 char buf[64]; /* big enough to hold a number */ 764 int r; 765 766 r = sprintf(buf, "%u\n", ftrace_profile_enabled); 767 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 768 } 769 770 static const struct file_operations ftrace_profile_fops = { 771 .open = tracing_open_generic, 772 .read = ftrace_profile_read, 773 .write = ftrace_profile_write, 774 }; 775 776 /* used to initialize the real stat files */ 777 static struct tracer_stat function_stats __initdata = { 778 .name = "functions", 779 .stat_start = function_stat_start, 780 .stat_next = function_stat_next, 781 .stat_cmp = function_stat_cmp, 782 .stat_headers = function_stat_headers, 783 .stat_show = function_stat_show 784 }; 785 786 static __init void ftrace_profile_debugfs(struct dentry *d_tracer) 787 { 788 struct ftrace_profile_stat *stat; 789 struct dentry *entry; 790 char *name; 791 int ret; 792 int cpu; 793 794 for_each_possible_cpu(cpu) { 795 stat = &per_cpu(ftrace_profile_stats, cpu); 796 797 /* allocate enough for function name + cpu number */ 798 name = kmalloc(32, GFP_KERNEL); 799 if (!name) { 800 /* 801 * The files created are permanent, if something happens 802 * we still do not free memory. 803 */ 804 WARN(1, 805 "Could not allocate stat file for cpu %d\n", 806 cpu); 807 return; 808 } 809 stat->stat = function_stats; 810 snprintf(name, 32, "function%d", cpu); 811 stat->stat.name = name; 812 ret = register_stat_tracer(&stat->stat); 813 if (ret) { 814 WARN(1, 815 "Could not register function stat for cpu %d\n", 816 cpu); 817 kfree(name); 818 return; 819 } 820 } 821 822 entry = debugfs_create_file("function_profile_enabled", 0644, 823 d_tracer, NULL, &ftrace_profile_fops); 824 if (!entry) 825 pr_warning("Could not create debugfs " 826 "'function_profile_enabled' entry\n"); 827 } 828 829 #else /* CONFIG_FUNCTION_PROFILER */ 830 static __init void ftrace_profile_debugfs(struct dentry *d_tracer) 831 { 832 } 833 #endif /* CONFIG_FUNCTION_PROFILER */ 834 835 static struct pid * const ftrace_swapper_pid = &init_struct_pid; 836 837 #ifdef CONFIG_DYNAMIC_FTRACE 838 839 #ifndef CONFIG_FTRACE_MCOUNT_RECORD 840 # error Dynamic ftrace depends on MCOUNT_RECORD 841 #endif 842 843 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly; 844 845 struct ftrace_func_probe { 846 struct hlist_node node; 847 struct ftrace_probe_ops *ops; 848 unsigned long flags; 849 unsigned long ip; 850 void *data; 851 struct rcu_head rcu; 852 }; 853 854 enum { 855 FTRACE_ENABLE_CALLS = (1 << 0), 856 FTRACE_DISABLE_CALLS = (1 << 1), 857 FTRACE_UPDATE_TRACE_FUNC = (1 << 2), 858 FTRACE_ENABLE_MCOUNT = (1 << 3), 859 FTRACE_DISABLE_MCOUNT = (1 << 4), 860 FTRACE_START_FUNC_RET = (1 << 5), 861 FTRACE_STOP_FUNC_RET = (1 << 6), 862 }; 863 864 static int ftrace_filtered; 865 866 static struct dyn_ftrace *ftrace_new_addrs; 867 868 static DEFINE_MUTEX(ftrace_regex_lock); 869 870 struct ftrace_page { 871 struct ftrace_page *next; 872 int index; 873 struct dyn_ftrace records[]; 874 }; 875 876 #define ENTRIES_PER_PAGE \ 877 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace)) 878 879 /* estimate from running different kernels */ 880 #define NR_TO_INIT 10000 881 882 static struct ftrace_page *ftrace_pages_start; 883 static struct ftrace_page *ftrace_pages; 884 885 static struct dyn_ftrace *ftrace_free_records; 886 887 /* 888 * This is a double for. Do not use 'break' to break out of the loop, 889 * you must use a goto. 890 */ 891 #define do_for_each_ftrace_rec(pg, rec) \ 892 for (pg = ftrace_pages_start; pg; pg = pg->next) { \ 893 int _____i; \ 894 for (_____i = 0; _____i < pg->index; _____i++) { \ 895 rec = &pg->records[_____i]; 896 897 #define while_for_each_ftrace_rec() \ 898 } \ 899 } 900 901 #ifdef CONFIG_KPROBES 902 903 static int frozen_record_count; 904 905 static inline void freeze_record(struct dyn_ftrace *rec) 906 { 907 if (!(rec->flags & FTRACE_FL_FROZEN)) { 908 rec->flags |= FTRACE_FL_FROZEN; 909 frozen_record_count++; 910 } 911 } 912 913 static inline void unfreeze_record(struct dyn_ftrace *rec) 914 { 915 if (rec->flags & FTRACE_FL_FROZEN) { 916 rec->flags &= ~FTRACE_FL_FROZEN; 917 frozen_record_count--; 918 } 919 } 920 921 static inline int record_frozen(struct dyn_ftrace *rec) 922 { 923 return rec->flags & FTRACE_FL_FROZEN; 924 } 925 #else 926 # define freeze_record(rec) ({ 0; }) 927 # define unfreeze_record(rec) ({ 0; }) 928 # define record_frozen(rec) ({ 0; }) 929 #endif /* CONFIG_KPROBES */ 930 931 static void ftrace_free_rec(struct dyn_ftrace *rec) 932 { 933 rec->freelist = ftrace_free_records; 934 ftrace_free_records = rec; 935 rec->flags |= FTRACE_FL_FREE; 936 } 937 938 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip) 939 { 940 struct dyn_ftrace *rec; 941 942 /* First check for freed records */ 943 if (ftrace_free_records) { 944 rec = ftrace_free_records; 945 946 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) { 947 FTRACE_WARN_ON_ONCE(1); 948 ftrace_free_records = NULL; 949 return NULL; 950 } 951 952 ftrace_free_records = rec->freelist; 953 memset(rec, 0, sizeof(*rec)); 954 return rec; 955 } 956 957 if (ftrace_pages->index == ENTRIES_PER_PAGE) { 958 if (!ftrace_pages->next) { 959 /* allocate another page */ 960 ftrace_pages->next = 961 (void *)get_zeroed_page(GFP_KERNEL); 962 if (!ftrace_pages->next) 963 return NULL; 964 } 965 ftrace_pages = ftrace_pages->next; 966 } 967 968 return &ftrace_pages->records[ftrace_pages->index++]; 969 } 970 971 static struct dyn_ftrace * 972 ftrace_record_ip(unsigned long ip) 973 { 974 struct dyn_ftrace *rec; 975 976 if (ftrace_disabled) 977 return NULL; 978 979 rec = ftrace_alloc_dyn_node(ip); 980 if (!rec) 981 return NULL; 982 983 rec->ip = ip; 984 rec->newlist = ftrace_new_addrs; 985 ftrace_new_addrs = rec; 986 987 return rec; 988 } 989 990 static void print_ip_ins(const char *fmt, unsigned char *p) 991 { 992 int i; 993 994 printk(KERN_CONT "%s", fmt); 995 996 for (i = 0; i < MCOUNT_INSN_SIZE; i++) 997 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]); 998 } 999 1000 static void ftrace_bug(int failed, unsigned long ip) 1001 { 1002 switch (failed) { 1003 case -EFAULT: 1004 FTRACE_WARN_ON_ONCE(1); 1005 pr_info("ftrace faulted on modifying "); 1006 print_ip_sym(ip); 1007 break; 1008 case -EINVAL: 1009 FTRACE_WARN_ON_ONCE(1); 1010 pr_info("ftrace failed to modify "); 1011 print_ip_sym(ip); 1012 print_ip_ins(" actual: ", (unsigned char *)ip); 1013 printk(KERN_CONT "\n"); 1014 break; 1015 case -EPERM: 1016 FTRACE_WARN_ON_ONCE(1); 1017 pr_info("ftrace faulted on writing "); 1018 print_ip_sym(ip); 1019 break; 1020 default: 1021 FTRACE_WARN_ON_ONCE(1); 1022 pr_info("ftrace faulted on unknown error "); 1023 print_ip_sym(ip); 1024 } 1025 } 1026 1027 1028 static int 1029 __ftrace_replace_code(struct dyn_ftrace *rec, int enable) 1030 { 1031 unsigned long ftrace_addr; 1032 unsigned long flag = 0UL; 1033 1034 ftrace_addr = (unsigned long)FTRACE_ADDR; 1035 1036 /* 1037 * If this record is not to be traced or we want to disable it, 1038 * then disable it. 1039 * 1040 * If we want to enable it and filtering is off, then enable it. 1041 * 1042 * If we want to enable it and filtering is on, enable it only if 1043 * it's filtered 1044 */ 1045 if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) { 1046 if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER)) 1047 flag = FTRACE_FL_ENABLED; 1048 } 1049 1050 /* If the state of this record hasn't changed, then do nothing */ 1051 if ((rec->flags & FTRACE_FL_ENABLED) == flag) 1052 return 0; 1053 1054 if (flag) { 1055 rec->flags |= FTRACE_FL_ENABLED; 1056 return ftrace_make_call(rec, ftrace_addr); 1057 } 1058 1059 rec->flags &= ~FTRACE_FL_ENABLED; 1060 return ftrace_make_nop(NULL, rec, ftrace_addr); 1061 } 1062 1063 static void ftrace_replace_code(int enable) 1064 { 1065 struct dyn_ftrace *rec; 1066 struct ftrace_page *pg; 1067 int failed; 1068 1069 do_for_each_ftrace_rec(pg, rec) { 1070 /* 1071 * Skip over free records, records that have 1072 * failed and not converted. 1073 */ 1074 if (rec->flags & FTRACE_FL_FREE || 1075 rec->flags & FTRACE_FL_FAILED || 1076 !(rec->flags & FTRACE_FL_CONVERTED)) 1077 continue; 1078 1079 /* ignore updates to this record's mcount site */ 1080 if (get_kprobe((void *)rec->ip)) { 1081 freeze_record(rec); 1082 continue; 1083 } else { 1084 unfreeze_record(rec); 1085 } 1086 1087 failed = __ftrace_replace_code(rec, enable); 1088 if (failed) { 1089 rec->flags |= FTRACE_FL_FAILED; 1090 ftrace_bug(failed, rec->ip); 1091 /* Stop processing */ 1092 return; 1093 } 1094 } while_for_each_ftrace_rec(); 1095 } 1096 1097 static int 1098 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec) 1099 { 1100 unsigned long ip; 1101 int ret; 1102 1103 ip = rec->ip; 1104 1105 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR); 1106 if (ret) { 1107 ftrace_bug(ret, ip); 1108 rec->flags |= FTRACE_FL_FAILED; 1109 return 0; 1110 } 1111 return 1; 1112 } 1113 1114 /* 1115 * archs can override this function if they must do something 1116 * before the modifying code is performed. 1117 */ 1118 int __weak ftrace_arch_code_modify_prepare(void) 1119 { 1120 return 0; 1121 } 1122 1123 /* 1124 * archs can override this function if they must do something 1125 * after the modifying code is performed. 1126 */ 1127 int __weak ftrace_arch_code_modify_post_process(void) 1128 { 1129 return 0; 1130 } 1131 1132 static int __ftrace_modify_code(void *data) 1133 { 1134 int *command = data; 1135 1136 if (*command & FTRACE_ENABLE_CALLS) 1137 ftrace_replace_code(1); 1138 else if (*command & FTRACE_DISABLE_CALLS) 1139 ftrace_replace_code(0); 1140 1141 if (*command & FTRACE_UPDATE_TRACE_FUNC) 1142 ftrace_update_ftrace_func(ftrace_trace_function); 1143 1144 if (*command & FTRACE_START_FUNC_RET) 1145 ftrace_enable_ftrace_graph_caller(); 1146 else if (*command & FTRACE_STOP_FUNC_RET) 1147 ftrace_disable_ftrace_graph_caller(); 1148 1149 return 0; 1150 } 1151 1152 static void ftrace_run_update_code(int command) 1153 { 1154 int ret; 1155 1156 ret = ftrace_arch_code_modify_prepare(); 1157 FTRACE_WARN_ON(ret); 1158 if (ret) 1159 return; 1160 1161 stop_machine(__ftrace_modify_code, &command, NULL); 1162 1163 ret = ftrace_arch_code_modify_post_process(); 1164 FTRACE_WARN_ON(ret); 1165 } 1166 1167 static ftrace_func_t saved_ftrace_func; 1168 static int ftrace_start_up; 1169 1170 static void ftrace_startup_enable(int command) 1171 { 1172 if (saved_ftrace_func != ftrace_trace_function) { 1173 saved_ftrace_func = ftrace_trace_function; 1174 command |= FTRACE_UPDATE_TRACE_FUNC; 1175 } 1176 1177 if (!command || !ftrace_enabled) 1178 return; 1179 1180 ftrace_run_update_code(command); 1181 } 1182 1183 static void ftrace_startup(int command) 1184 { 1185 if (unlikely(ftrace_disabled)) 1186 return; 1187 1188 ftrace_start_up++; 1189 command |= FTRACE_ENABLE_CALLS; 1190 1191 ftrace_startup_enable(command); 1192 } 1193 1194 static void ftrace_shutdown(int command) 1195 { 1196 if (unlikely(ftrace_disabled)) 1197 return; 1198 1199 ftrace_start_up--; 1200 /* 1201 * Just warn in case of unbalance, no need to kill ftrace, it's not 1202 * critical but the ftrace_call callers may be never nopped again after 1203 * further ftrace uses. 1204 */ 1205 WARN_ON_ONCE(ftrace_start_up < 0); 1206 1207 if (!ftrace_start_up) 1208 command |= FTRACE_DISABLE_CALLS; 1209 1210 if (saved_ftrace_func != ftrace_trace_function) { 1211 saved_ftrace_func = ftrace_trace_function; 1212 command |= FTRACE_UPDATE_TRACE_FUNC; 1213 } 1214 1215 if (!command || !ftrace_enabled) 1216 return; 1217 1218 ftrace_run_update_code(command); 1219 } 1220 1221 static void ftrace_startup_sysctl(void) 1222 { 1223 int command = FTRACE_ENABLE_MCOUNT; 1224 1225 if (unlikely(ftrace_disabled)) 1226 return; 1227 1228 /* Force update next time */ 1229 saved_ftrace_func = NULL; 1230 /* ftrace_start_up is true if we want ftrace running */ 1231 if (ftrace_start_up) 1232 command |= FTRACE_ENABLE_CALLS; 1233 1234 ftrace_run_update_code(command); 1235 } 1236 1237 static void ftrace_shutdown_sysctl(void) 1238 { 1239 int command = FTRACE_DISABLE_MCOUNT; 1240 1241 if (unlikely(ftrace_disabled)) 1242 return; 1243 1244 /* ftrace_start_up is true if ftrace is running */ 1245 if (ftrace_start_up) 1246 command |= FTRACE_DISABLE_CALLS; 1247 1248 ftrace_run_update_code(command); 1249 } 1250 1251 static cycle_t ftrace_update_time; 1252 static unsigned long ftrace_update_cnt; 1253 unsigned long ftrace_update_tot_cnt; 1254 1255 static int ftrace_update_code(struct module *mod) 1256 { 1257 struct dyn_ftrace *p; 1258 cycle_t start, stop; 1259 1260 start = ftrace_now(raw_smp_processor_id()); 1261 ftrace_update_cnt = 0; 1262 1263 while (ftrace_new_addrs) { 1264 1265 /* If something went wrong, bail without enabling anything */ 1266 if (unlikely(ftrace_disabled)) 1267 return -1; 1268 1269 p = ftrace_new_addrs; 1270 ftrace_new_addrs = p->newlist; 1271 p->flags = 0L; 1272 1273 /* 1274 * Do the initial record convertion from mcount jump 1275 * to the NOP instructions. 1276 */ 1277 if (!ftrace_code_disable(mod, p)) { 1278 ftrace_free_rec(p); 1279 continue; 1280 } 1281 1282 p->flags |= FTRACE_FL_CONVERTED; 1283 ftrace_update_cnt++; 1284 1285 /* 1286 * If the tracing is enabled, go ahead and enable the record. 1287 * 1288 * The reason not to enable the record immediatelly is the 1289 * inherent check of ftrace_make_nop/ftrace_make_call for 1290 * correct previous instructions. Making first the NOP 1291 * conversion puts the module to the correct state, thus 1292 * passing the ftrace_make_call check. 1293 */ 1294 if (ftrace_start_up) { 1295 int failed = __ftrace_replace_code(p, 1); 1296 if (failed) { 1297 ftrace_bug(failed, p->ip); 1298 ftrace_free_rec(p); 1299 } 1300 } 1301 } 1302 1303 stop = ftrace_now(raw_smp_processor_id()); 1304 ftrace_update_time = stop - start; 1305 ftrace_update_tot_cnt += ftrace_update_cnt; 1306 1307 return 0; 1308 } 1309 1310 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init) 1311 { 1312 struct ftrace_page *pg; 1313 int cnt; 1314 int i; 1315 1316 /* allocate a few pages */ 1317 ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL); 1318 if (!ftrace_pages_start) 1319 return -1; 1320 1321 /* 1322 * Allocate a few more pages. 1323 * 1324 * TODO: have some parser search vmlinux before 1325 * final linking to find all calls to ftrace. 1326 * Then we can: 1327 * a) know how many pages to allocate. 1328 * and/or 1329 * b) set up the table then. 1330 * 1331 * The dynamic code is still necessary for 1332 * modules. 1333 */ 1334 1335 pg = ftrace_pages = ftrace_pages_start; 1336 1337 cnt = num_to_init / ENTRIES_PER_PAGE; 1338 pr_info("ftrace: allocating %ld entries in %d pages\n", 1339 num_to_init, cnt + 1); 1340 1341 for (i = 0; i < cnt; i++) { 1342 pg->next = (void *)get_zeroed_page(GFP_KERNEL); 1343 1344 /* If we fail, we'll try later anyway */ 1345 if (!pg->next) 1346 break; 1347 1348 pg = pg->next; 1349 } 1350 1351 return 0; 1352 } 1353 1354 enum { 1355 FTRACE_ITER_FILTER = (1 << 0), 1356 FTRACE_ITER_NOTRACE = (1 << 1), 1357 FTRACE_ITER_FAILURES = (1 << 2), 1358 FTRACE_ITER_PRINTALL = (1 << 3), 1359 FTRACE_ITER_HASH = (1 << 4), 1360 }; 1361 1362 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ 1363 1364 struct ftrace_iterator { 1365 struct ftrace_page *pg; 1366 int hidx; 1367 int idx; 1368 unsigned flags; 1369 struct trace_parser parser; 1370 }; 1371 1372 static void * 1373 t_hash_next(struct seq_file *m, void *v, loff_t *pos) 1374 { 1375 struct ftrace_iterator *iter = m->private; 1376 struct hlist_node *hnd = v; 1377 struct hlist_head *hhd; 1378 1379 WARN_ON(!(iter->flags & FTRACE_ITER_HASH)); 1380 1381 (*pos)++; 1382 1383 retry: 1384 if (iter->hidx >= FTRACE_FUNC_HASHSIZE) 1385 return NULL; 1386 1387 hhd = &ftrace_func_hash[iter->hidx]; 1388 1389 if (hlist_empty(hhd)) { 1390 iter->hidx++; 1391 hnd = NULL; 1392 goto retry; 1393 } 1394 1395 if (!hnd) 1396 hnd = hhd->first; 1397 else { 1398 hnd = hnd->next; 1399 if (!hnd) { 1400 iter->hidx++; 1401 goto retry; 1402 } 1403 } 1404 1405 return hnd; 1406 } 1407 1408 static void *t_hash_start(struct seq_file *m, loff_t *pos) 1409 { 1410 struct ftrace_iterator *iter = m->private; 1411 void *p = NULL; 1412 loff_t l; 1413 1414 if (!(iter->flags & FTRACE_ITER_HASH)) 1415 *pos = 0; 1416 1417 iter->flags |= FTRACE_ITER_HASH; 1418 1419 iter->hidx = 0; 1420 for (l = 0; l <= *pos; ) { 1421 p = t_hash_next(m, p, &l); 1422 if (!p) 1423 break; 1424 } 1425 return p; 1426 } 1427 1428 static int t_hash_show(struct seq_file *m, void *v) 1429 { 1430 struct ftrace_func_probe *rec; 1431 struct hlist_node *hnd = v; 1432 1433 rec = hlist_entry(hnd, struct ftrace_func_probe, node); 1434 1435 if (rec->ops->print) 1436 return rec->ops->print(m, rec->ip, rec->ops, rec->data); 1437 1438 seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func); 1439 1440 if (rec->data) 1441 seq_printf(m, ":%p", rec->data); 1442 seq_putc(m, '\n'); 1443 1444 return 0; 1445 } 1446 1447 static void * 1448 t_next(struct seq_file *m, void *v, loff_t *pos) 1449 { 1450 struct ftrace_iterator *iter = m->private; 1451 struct dyn_ftrace *rec = NULL; 1452 1453 if (iter->flags & FTRACE_ITER_HASH) 1454 return t_hash_next(m, v, pos); 1455 1456 (*pos)++; 1457 1458 if (iter->flags & FTRACE_ITER_PRINTALL) 1459 return NULL; 1460 1461 retry: 1462 if (iter->idx >= iter->pg->index) { 1463 if (iter->pg->next) { 1464 iter->pg = iter->pg->next; 1465 iter->idx = 0; 1466 goto retry; 1467 } 1468 } else { 1469 rec = &iter->pg->records[iter->idx++]; 1470 if ((rec->flags & FTRACE_FL_FREE) || 1471 1472 (!(iter->flags & FTRACE_ITER_FAILURES) && 1473 (rec->flags & FTRACE_FL_FAILED)) || 1474 1475 ((iter->flags & FTRACE_ITER_FAILURES) && 1476 !(rec->flags & FTRACE_FL_FAILED)) || 1477 1478 ((iter->flags & FTRACE_ITER_FILTER) && 1479 !(rec->flags & FTRACE_FL_FILTER)) || 1480 1481 ((iter->flags & FTRACE_ITER_NOTRACE) && 1482 !(rec->flags & FTRACE_FL_NOTRACE))) { 1483 rec = NULL; 1484 goto retry; 1485 } 1486 } 1487 1488 return rec; 1489 } 1490 1491 static void *t_start(struct seq_file *m, loff_t *pos) 1492 { 1493 struct ftrace_iterator *iter = m->private; 1494 void *p = NULL; 1495 loff_t l; 1496 1497 mutex_lock(&ftrace_lock); 1498 /* 1499 * For set_ftrace_filter reading, if we have the filter 1500 * off, we can short cut and just print out that all 1501 * functions are enabled. 1502 */ 1503 if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) { 1504 if (*pos > 0) 1505 return t_hash_start(m, pos); 1506 iter->flags |= FTRACE_ITER_PRINTALL; 1507 return iter; 1508 } 1509 1510 if (iter->flags & FTRACE_ITER_HASH) 1511 return t_hash_start(m, pos); 1512 1513 iter->pg = ftrace_pages_start; 1514 iter->idx = 0; 1515 for (l = 0; l <= *pos; ) { 1516 p = t_next(m, p, &l); 1517 if (!p) 1518 break; 1519 } 1520 1521 if (!p && iter->flags & FTRACE_ITER_FILTER) 1522 return t_hash_start(m, pos); 1523 1524 return p; 1525 } 1526 1527 static void t_stop(struct seq_file *m, void *p) 1528 { 1529 mutex_unlock(&ftrace_lock); 1530 } 1531 1532 static int t_show(struct seq_file *m, void *v) 1533 { 1534 struct ftrace_iterator *iter = m->private; 1535 struct dyn_ftrace *rec = v; 1536 1537 if (iter->flags & FTRACE_ITER_HASH) 1538 return t_hash_show(m, v); 1539 1540 if (iter->flags & FTRACE_ITER_PRINTALL) { 1541 seq_printf(m, "#### all functions enabled ####\n"); 1542 return 0; 1543 } 1544 1545 if (!rec) 1546 return 0; 1547 1548 seq_printf(m, "%ps\n", (void *)rec->ip); 1549 1550 return 0; 1551 } 1552 1553 static const struct seq_operations show_ftrace_seq_ops = { 1554 .start = t_start, 1555 .next = t_next, 1556 .stop = t_stop, 1557 .show = t_show, 1558 }; 1559 1560 static int 1561 ftrace_avail_open(struct inode *inode, struct file *file) 1562 { 1563 struct ftrace_iterator *iter; 1564 int ret; 1565 1566 if (unlikely(ftrace_disabled)) 1567 return -ENODEV; 1568 1569 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 1570 if (!iter) 1571 return -ENOMEM; 1572 1573 iter->pg = ftrace_pages_start; 1574 1575 ret = seq_open(file, &show_ftrace_seq_ops); 1576 if (!ret) { 1577 struct seq_file *m = file->private_data; 1578 1579 m->private = iter; 1580 } else { 1581 kfree(iter); 1582 } 1583 1584 return ret; 1585 } 1586 1587 static int 1588 ftrace_failures_open(struct inode *inode, struct file *file) 1589 { 1590 int ret; 1591 struct seq_file *m; 1592 struct ftrace_iterator *iter; 1593 1594 ret = ftrace_avail_open(inode, file); 1595 if (!ret) { 1596 m = (struct seq_file *)file->private_data; 1597 iter = (struct ftrace_iterator *)m->private; 1598 iter->flags = FTRACE_ITER_FAILURES; 1599 } 1600 1601 return ret; 1602 } 1603 1604 1605 static void ftrace_filter_reset(int enable) 1606 { 1607 struct ftrace_page *pg; 1608 struct dyn_ftrace *rec; 1609 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 1610 1611 mutex_lock(&ftrace_lock); 1612 if (enable) 1613 ftrace_filtered = 0; 1614 do_for_each_ftrace_rec(pg, rec) { 1615 if (rec->flags & FTRACE_FL_FAILED) 1616 continue; 1617 rec->flags &= ~type; 1618 } while_for_each_ftrace_rec(); 1619 mutex_unlock(&ftrace_lock); 1620 } 1621 1622 static int 1623 ftrace_regex_open(struct inode *inode, struct file *file, int enable) 1624 { 1625 struct ftrace_iterator *iter; 1626 int ret = 0; 1627 1628 if (unlikely(ftrace_disabled)) 1629 return -ENODEV; 1630 1631 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 1632 if (!iter) 1633 return -ENOMEM; 1634 1635 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) { 1636 kfree(iter); 1637 return -ENOMEM; 1638 } 1639 1640 mutex_lock(&ftrace_regex_lock); 1641 if ((file->f_mode & FMODE_WRITE) && 1642 (file->f_flags & O_TRUNC)) 1643 ftrace_filter_reset(enable); 1644 1645 if (file->f_mode & FMODE_READ) { 1646 iter->pg = ftrace_pages_start; 1647 iter->flags = enable ? FTRACE_ITER_FILTER : 1648 FTRACE_ITER_NOTRACE; 1649 1650 ret = seq_open(file, &show_ftrace_seq_ops); 1651 if (!ret) { 1652 struct seq_file *m = file->private_data; 1653 m->private = iter; 1654 } else { 1655 trace_parser_put(&iter->parser); 1656 kfree(iter); 1657 } 1658 } else 1659 file->private_data = iter; 1660 mutex_unlock(&ftrace_regex_lock); 1661 1662 return ret; 1663 } 1664 1665 static int 1666 ftrace_filter_open(struct inode *inode, struct file *file) 1667 { 1668 return ftrace_regex_open(inode, file, 1); 1669 } 1670 1671 static int 1672 ftrace_notrace_open(struct inode *inode, struct file *file) 1673 { 1674 return ftrace_regex_open(inode, file, 0); 1675 } 1676 1677 static loff_t 1678 ftrace_regex_lseek(struct file *file, loff_t offset, int origin) 1679 { 1680 loff_t ret; 1681 1682 if (file->f_mode & FMODE_READ) 1683 ret = seq_lseek(file, offset, origin); 1684 else 1685 file->f_pos = ret = 1; 1686 1687 return ret; 1688 } 1689 1690 static int ftrace_match(char *str, char *regex, int len, int type) 1691 { 1692 int matched = 0; 1693 int slen; 1694 1695 switch (type) { 1696 case MATCH_FULL: 1697 if (strcmp(str, regex) == 0) 1698 matched = 1; 1699 break; 1700 case MATCH_FRONT_ONLY: 1701 if (strncmp(str, regex, len) == 0) 1702 matched = 1; 1703 break; 1704 case MATCH_MIDDLE_ONLY: 1705 if (strstr(str, regex)) 1706 matched = 1; 1707 break; 1708 case MATCH_END_ONLY: 1709 slen = strlen(str); 1710 if (slen >= len && memcmp(str + slen - len, regex, len) == 0) 1711 matched = 1; 1712 break; 1713 } 1714 1715 return matched; 1716 } 1717 1718 static int 1719 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type) 1720 { 1721 char str[KSYM_SYMBOL_LEN]; 1722 1723 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 1724 return ftrace_match(str, regex, len, type); 1725 } 1726 1727 static int ftrace_match_records(char *buff, int len, int enable) 1728 { 1729 unsigned int search_len; 1730 struct ftrace_page *pg; 1731 struct dyn_ftrace *rec; 1732 unsigned long flag; 1733 char *search; 1734 int type; 1735 int not; 1736 int found = 0; 1737 1738 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 1739 type = filter_parse_regex(buff, len, &search, ¬); 1740 1741 search_len = strlen(search); 1742 1743 mutex_lock(&ftrace_lock); 1744 do_for_each_ftrace_rec(pg, rec) { 1745 1746 if (rec->flags & FTRACE_FL_FAILED) 1747 continue; 1748 1749 if (ftrace_match_record(rec, search, search_len, type)) { 1750 if (not) 1751 rec->flags &= ~flag; 1752 else 1753 rec->flags |= flag; 1754 found = 1; 1755 } 1756 /* 1757 * Only enable filtering if we have a function that 1758 * is filtered on. 1759 */ 1760 if (enable && (rec->flags & FTRACE_FL_FILTER)) 1761 ftrace_filtered = 1; 1762 } while_for_each_ftrace_rec(); 1763 mutex_unlock(&ftrace_lock); 1764 1765 return found; 1766 } 1767 1768 static int 1769 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod, 1770 char *regex, int len, int type) 1771 { 1772 char str[KSYM_SYMBOL_LEN]; 1773 char *modname; 1774 1775 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); 1776 1777 if (!modname || strcmp(modname, mod)) 1778 return 0; 1779 1780 /* blank search means to match all funcs in the mod */ 1781 if (len) 1782 return ftrace_match(str, regex, len, type); 1783 else 1784 return 1; 1785 } 1786 1787 static int ftrace_match_module_records(char *buff, char *mod, int enable) 1788 { 1789 unsigned search_len = 0; 1790 struct ftrace_page *pg; 1791 struct dyn_ftrace *rec; 1792 int type = MATCH_FULL; 1793 char *search = buff; 1794 unsigned long flag; 1795 int not = 0; 1796 int found = 0; 1797 1798 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 1799 1800 /* blank or '*' mean the same */ 1801 if (strcmp(buff, "*") == 0) 1802 buff[0] = 0; 1803 1804 /* handle the case of 'dont filter this module' */ 1805 if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) { 1806 buff[0] = 0; 1807 not = 1; 1808 } 1809 1810 if (strlen(buff)) { 1811 type = filter_parse_regex(buff, strlen(buff), &search, ¬); 1812 search_len = strlen(search); 1813 } 1814 1815 mutex_lock(&ftrace_lock); 1816 do_for_each_ftrace_rec(pg, rec) { 1817 1818 if (rec->flags & FTRACE_FL_FAILED) 1819 continue; 1820 1821 if (ftrace_match_module_record(rec, mod, 1822 search, search_len, type)) { 1823 if (not) 1824 rec->flags &= ~flag; 1825 else 1826 rec->flags |= flag; 1827 found = 1; 1828 } 1829 if (enable && (rec->flags & FTRACE_FL_FILTER)) 1830 ftrace_filtered = 1; 1831 1832 } while_for_each_ftrace_rec(); 1833 mutex_unlock(&ftrace_lock); 1834 1835 return found; 1836 } 1837 1838 /* 1839 * We register the module command as a template to show others how 1840 * to register the a command as well. 1841 */ 1842 1843 static int 1844 ftrace_mod_callback(char *func, char *cmd, char *param, int enable) 1845 { 1846 char *mod; 1847 1848 /* 1849 * cmd == 'mod' because we only registered this func 1850 * for the 'mod' ftrace_func_command. 1851 * But if you register one func with multiple commands, 1852 * you can tell which command was used by the cmd 1853 * parameter. 1854 */ 1855 1856 /* we must have a module name */ 1857 if (!param) 1858 return -EINVAL; 1859 1860 mod = strsep(¶m, ":"); 1861 if (!strlen(mod)) 1862 return -EINVAL; 1863 1864 if (ftrace_match_module_records(func, mod, enable)) 1865 return 0; 1866 return -EINVAL; 1867 } 1868 1869 static struct ftrace_func_command ftrace_mod_cmd = { 1870 .name = "mod", 1871 .func = ftrace_mod_callback, 1872 }; 1873 1874 static int __init ftrace_mod_cmd_init(void) 1875 { 1876 return register_ftrace_command(&ftrace_mod_cmd); 1877 } 1878 device_initcall(ftrace_mod_cmd_init); 1879 1880 static void 1881 function_trace_probe_call(unsigned long ip, unsigned long parent_ip) 1882 { 1883 struct ftrace_func_probe *entry; 1884 struct hlist_head *hhd; 1885 struct hlist_node *n; 1886 unsigned long key; 1887 int resched; 1888 1889 key = hash_long(ip, FTRACE_HASH_BITS); 1890 1891 hhd = &ftrace_func_hash[key]; 1892 1893 if (hlist_empty(hhd)) 1894 return; 1895 1896 /* 1897 * Disable preemption for these calls to prevent a RCU grace 1898 * period. This syncs the hash iteration and freeing of items 1899 * on the hash. rcu_read_lock is too dangerous here. 1900 */ 1901 resched = ftrace_preempt_disable(); 1902 hlist_for_each_entry_rcu(entry, n, hhd, node) { 1903 if (entry->ip == ip) 1904 entry->ops->func(ip, parent_ip, &entry->data); 1905 } 1906 ftrace_preempt_enable(resched); 1907 } 1908 1909 static struct ftrace_ops trace_probe_ops __read_mostly = 1910 { 1911 .func = function_trace_probe_call, 1912 }; 1913 1914 static int ftrace_probe_registered; 1915 1916 static void __enable_ftrace_function_probe(void) 1917 { 1918 int i; 1919 1920 if (ftrace_probe_registered) 1921 return; 1922 1923 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 1924 struct hlist_head *hhd = &ftrace_func_hash[i]; 1925 if (hhd->first) 1926 break; 1927 } 1928 /* Nothing registered? */ 1929 if (i == FTRACE_FUNC_HASHSIZE) 1930 return; 1931 1932 __register_ftrace_function(&trace_probe_ops); 1933 ftrace_startup(0); 1934 ftrace_probe_registered = 1; 1935 } 1936 1937 static void __disable_ftrace_function_probe(void) 1938 { 1939 int i; 1940 1941 if (!ftrace_probe_registered) 1942 return; 1943 1944 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 1945 struct hlist_head *hhd = &ftrace_func_hash[i]; 1946 if (hhd->first) 1947 return; 1948 } 1949 1950 /* no more funcs left */ 1951 __unregister_ftrace_function(&trace_probe_ops); 1952 ftrace_shutdown(0); 1953 ftrace_probe_registered = 0; 1954 } 1955 1956 1957 static void ftrace_free_entry_rcu(struct rcu_head *rhp) 1958 { 1959 struct ftrace_func_probe *entry = 1960 container_of(rhp, struct ftrace_func_probe, rcu); 1961 1962 if (entry->ops->free) 1963 entry->ops->free(&entry->data); 1964 kfree(entry); 1965 } 1966 1967 1968 int 1969 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 1970 void *data) 1971 { 1972 struct ftrace_func_probe *entry; 1973 struct ftrace_page *pg; 1974 struct dyn_ftrace *rec; 1975 int type, len, not; 1976 unsigned long key; 1977 int count = 0; 1978 char *search; 1979 1980 type = filter_parse_regex(glob, strlen(glob), &search, ¬); 1981 len = strlen(search); 1982 1983 /* we do not support '!' for function probes */ 1984 if (WARN_ON(not)) 1985 return -EINVAL; 1986 1987 mutex_lock(&ftrace_lock); 1988 do_for_each_ftrace_rec(pg, rec) { 1989 1990 if (rec->flags & FTRACE_FL_FAILED) 1991 continue; 1992 1993 if (!ftrace_match_record(rec, search, len, type)) 1994 continue; 1995 1996 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 1997 if (!entry) { 1998 /* If we did not process any, then return error */ 1999 if (!count) 2000 count = -ENOMEM; 2001 goto out_unlock; 2002 } 2003 2004 count++; 2005 2006 entry->data = data; 2007 2008 /* 2009 * The caller might want to do something special 2010 * for each function we find. We call the callback 2011 * to give the caller an opportunity to do so. 2012 */ 2013 if (ops->callback) { 2014 if (ops->callback(rec->ip, &entry->data) < 0) { 2015 /* caller does not like this func */ 2016 kfree(entry); 2017 continue; 2018 } 2019 } 2020 2021 entry->ops = ops; 2022 entry->ip = rec->ip; 2023 2024 key = hash_long(entry->ip, FTRACE_HASH_BITS); 2025 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]); 2026 2027 } while_for_each_ftrace_rec(); 2028 __enable_ftrace_function_probe(); 2029 2030 out_unlock: 2031 mutex_unlock(&ftrace_lock); 2032 2033 return count; 2034 } 2035 2036 enum { 2037 PROBE_TEST_FUNC = 1, 2038 PROBE_TEST_DATA = 2 2039 }; 2040 2041 static void 2042 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 2043 void *data, int flags) 2044 { 2045 struct ftrace_func_probe *entry; 2046 struct hlist_node *n, *tmp; 2047 char str[KSYM_SYMBOL_LEN]; 2048 int type = MATCH_FULL; 2049 int i, len = 0; 2050 char *search; 2051 2052 if (glob && (strcmp(glob, "*") == 0 || !strlen(glob))) 2053 glob = NULL; 2054 else if (glob) { 2055 int not; 2056 2057 type = filter_parse_regex(glob, strlen(glob), &search, ¬); 2058 len = strlen(search); 2059 2060 /* we do not support '!' for function probes */ 2061 if (WARN_ON(not)) 2062 return; 2063 } 2064 2065 mutex_lock(&ftrace_lock); 2066 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 2067 struct hlist_head *hhd = &ftrace_func_hash[i]; 2068 2069 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) { 2070 2071 /* break up if statements for readability */ 2072 if ((flags & PROBE_TEST_FUNC) && entry->ops != ops) 2073 continue; 2074 2075 if ((flags & PROBE_TEST_DATA) && entry->data != data) 2076 continue; 2077 2078 /* do this last, since it is the most expensive */ 2079 if (glob) { 2080 kallsyms_lookup(entry->ip, NULL, NULL, 2081 NULL, str); 2082 if (!ftrace_match(str, glob, len, type)) 2083 continue; 2084 } 2085 2086 hlist_del(&entry->node); 2087 call_rcu(&entry->rcu, ftrace_free_entry_rcu); 2088 } 2089 } 2090 __disable_ftrace_function_probe(); 2091 mutex_unlock(&ftrace_lock); 2092 } 2093 2094 void 2095 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 2096 void *data) 2097 { 2098 __unregister_ftrace_function_probe(glob, ops, data, 2099 PROBE_TEST_FUNC | PROBE_TEST_DATA); 2100 } 2101 2102 void 2103 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) 2104 { 2105 __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC); 2106 } 2107 2108 void unregister_ftrace_function_probe_all(char *glob) 2109 { 2110 __unregister_ftrace_function_probe(glob, NULL, NULL, 0); 2111 } 2112 2113 static LIST_HEAD(ftrace_commands); 2114 static DEFINE_MUTEX(ftrace_cmd_mutex); 2115 2116 int register_ftrace_command(struct ftrace_func_command *cmd) 2117 { 2118 struct ftrace_func_command *p; 2119 int ret = 0; 2120 2121 mutex_lock(&ftrace_cmd_mutex); 2122 list_for_each_entry(p, &ftrace_commands, list) { 2123 if (strcmp(cmd->name, p->name) == 0) { 2124 ret = -EBUSY; 2125 goto out_unlock; 2126 } 2127 } 2128 list_add(&cmd->list, &ftrace_commands); 2129 out_unlock: 2130 mutex_unlock(&ftrace_cmd_mutex); 2131 2132 return ret; 2133 } 2134 2135 int unregister_ftrace_command(struct ftrace_func_command *cmd) 2136 { 2137 struct ftrace_func_command *p, *n; 2138 int ret = -ENODEV; 2139 2140 mutex_lock(&ftrace_cmd_mutex); 2141 list_for_each_entry_safe(p, n, &ftrace_commands, list) { 2142 if (strcmp(cmd->name, p->name) == 0) { 2143 ret = 0; 2144 list_del_init(&p->list); 2145 goto out_unlock; 2146 } 2147 } 2148 out_unlock: 2149 mutex_unlock(&ftrace_cmd_mutex); 2150 2151 return ret; 2152 } 2153 2154 static int ftrace_process_regex(char *buff, int len, int enable) 2155 { 2156 char *func, *command, *next = buff; 2157 struct ftrace_func_command *p; 2158 int ret = -EINVAL; 2159 2160 func = strsep(&next, ":"); 2161 2162 if (!next) { 2163 if (ftrace_match_records(func, len, enable)) 2164 return 0; 2165 return ret; 2166 } 2167 2168 /* command found */ 2169 2170 command = strsep(&next, ":"); 2171 2172 mutex_lock(&ftrace_cmd_mutex); 2173 list_for_each_entry(p, &ftrace_commands, list) { 2174 if (strcmp(p->name, command) == 0) { 2175 ret = p->func(func, command, next, enable); 2176 goto out_unlock; 2177 } 2178 } 2179 out_unlock: 2180 mutex_unlock(&ftrace_cmd_mutex); 2181 2182 return ret; 2183 } 2184 2185 static ssize_t 2186 ftrace_regex_write(struct file *file, const char __user *ubuf, 2187 size_t cnt, loff_t *ppos, int enable) 2188 { 2189 struct ftrace_iterator *iter; 2190 struct trace_parser *parser; 2191 ssize_t ret, read; 2192 2193 if (!cnt) 2194 return 0; 2195 2196 mutex_lock(&ftrace_regex_lock); 2197 2198 if (file->f_mode & FMODE_READ) { 2199 struct seq_file *m = file->private_data; 2200 iter = m->private; 2201 } else 2202 iter = file->private_data; 2203 2204 parser = &iter->parser; 2205 read = trace_get_user(parser, ubuf, cnt, ppos); 2206 2207 if (read >= 0 && trace_parser_loaded(parser) && 2208 !trace_parser_cont(parser)) { 2209 ret = ftrace_process_regex(parser->buffer, 2210 parser->idx, enable); 2211 trace_parser_clear(parser); 2212 if (ret) 2213 goto out_unlock; 2214 } 2215 2216 ret = read; 2217 out_unlock: 2218 mutex_unlock(&ftrace_regex_lock); 2219 2220 return ret; 2221 } 2222 2223 static ssize_t 2224 ftrace_filter_write(struct file *file, const char __user *ubuf, 2225 size_t cnt, loff_t *ppos) 2226 { 2227 return ftrace_regex_write(file, ubuf, cnt, ppos, 1); 2228 } 2229 2230 static ssize_t 2231 ftrace_notrace_write(struct file *file, const char __user *ubuf, 2232 size_t cnt, loff_t *ppos) 2233 { 2234 return ftrace_regex_write(file, ubuf, cnt, ppos, 0); 2235 } 2236 2237 static void 2238 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable) 2239 { 2240 if (unlikely(ftrace_disabled)) 2241 return; 2242 2243 mutex_lock(&ftrace_regex_lock); 2244 if (reset) 2245 ftrace_filter_reset(enable); 2246 if (buf) 2247 ftrace_match_records(buf, len, enable); 2248 mutex_unlock(&ftrace_regex_lock); 2249 } 2250 2251 /** 2252 * ftrace_set_filter - set a function to filter on in ftrace 2253 * @buf - the string that holds the function filter text. 2254 * @len - the length of the string. 2255 * @reset - non zero to reset all filters before applying this filter. 2256 * 2257 * Filters denote which functions should be enabled when tracing is enabled. 2258 * If @buf is NULL and reset is set, all functions will be enabled for tracing. 2259 */ 2260 void ftrace_set_filter(unsigned char *buf, int len, int reset) 2261 { 2262 ftrace_set_regex(buf, len, reset, 1); 2263 } 2264 2265 /** 2266 * ftrace_set_notrace - set a function to not trace in ftrace 2267 * @buf - the string that holds the function notrace text. 2268 * @len - the length of the string. 2269 * @reset - non zero to reset all filters before applying this filter. 2270 * 2271 * Notrace Filters denote which functions should not be enabled when tracing 2272 * is enabled. If @buf is NULL and reset is set, all functions will be enabled 2273 * for tracing. 2274 */ 2275 void ftrace_set_notrace(unsigned char *buf, int len, int reset) 2276 { 2277 ftrace_set_regex(buf, len, reset, 0); 2278 } 2279 2280 /* 2281 * command line interface to allow users to set filters on boot up. 2282 */ 2283 #define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE 2284 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata; 2285 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata; 2286 2287 static int __init set_ftrace_notrace(char *str) 2288 { 2289 strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE); 2290 return 1; 2291 } 2292 __setup("ftrace_notrace=", set_ftrace_notrace); 2293 2294 static int __init set_ftrace_filter(char *str) 2295 { 2296 strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE); 2297 return 1; 2298 } 2299 __setup("ftrace_filter=", set_ftrace_filter); 2300 2301 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 2302 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; 2303 static int __init set_graph_function(char *str) 2304 { 2305 strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE); 2306 return 1; 2307 } 2308 __setup("ftrace_graph_filter=", set_graph_function); 2309 2310 static void __init set_ftrace_early_graph(char *buf) 2311 { 2312 int ret; 2313 char *func; 2314 2315 while (buf) { 2316 func = strsep(&buf, ","); 2317 /* we allow only one expression at a time */ 2318 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count, 2319 func); 2320 if (ret) 2321 printk(KERN_DEBUG "ftrace: function %s not " 2322 "traceable\n", func); 2323 } 2324 } 2325 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 2326 2327 static void __init set_ftrace_early_filter(char *buf, int enable) 2328 { 2329 char *func; 2330 2331 while (buf) { 2332 func = strsep(&buf, ","); 2333 ftrace_set_regex(func, strlen(func), 0, enable); 2334 } 2335 } 2336 2337 static void __init set_ftrace_early_filters(void) 2338 { 2339 if (ftrace_filter_buf[0]) 2340 set_ftrace_early_filter(ftrace_filter_buf, 1); 2341 if (ftrace_notrace_buf[0]) 2342 set_ftrace_early_filter(ftrace_notrace_buf, 0); 2343 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 2344 if (ftrace_graph_buf[0]) 2345 set_ftrace_early_graph(ftrace_graph_buf); 2346 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 2347 } 2348 2349 static int 2350 ftrace_regex_release(struct inode *inode, struct file *file, int enable) 2351 { 2352 struct seq_file *m = (struct seq_file *)file->private_data; 2353 struct ftrace_iterator *iter; 2354 struct trace_parser *parser; 2355 2356 mutex_lock(&ftrace_regex_lock); 2357 if (file->f_mode & FMODE_READ) { 2358 iter = m->private; 2359 2360 seq_release(inode, file); 2361 } else 2362 iter = file->private_data; 2363 2364 parser = &iter->parser; 2365 if (trace_parser_loaded(parser)) { 2366 parser->buffer[parser->idx] = 0; 2367 ftrace_match_records(parser->buffer, parser->idx, enable); 2368 } 2369 2370 mutex_lock(&ftrace_lock); 2371 if (ftrace_start_up && ftrace_enabled) 2372 ftrace_run_update_code(FTRACE_ENABLE_CALLS); 2373 mutex_unlock(&ftrace_lock); 2374 2375 trace_parser_put(parser); 2376 kfree(iter); 2377 2378 mutex_unlock(&ftrace_regex_lock); 2379 return 0; 2380 } 2381 2382 static int 2383 ftrace_filter_release(struct inode *inode, struct file *file) 2384 { 2385 return ftrace_regex_release(inode, file, 1); 2386 } 2387 2388 static int 2389 ftrace_notrace_release(struct inode *inode, struct file *file) 2390 { 2391 return ftrace_regex_release(inode, file, 0); 2392 } 2393 2394 static const struct file_operations ftrace_avail_fops = { 2395 .open = ftrace_avail_open, 2396 .read = seq_read, 2397 .llseek = seq_lseek, 2398 .release = seq_release_private, 2399 }; 2400 2401 static const struct file_operations ftrace_failures_fops = { 2402 .open = ftrace_failures_open, 2403 .read = seq_read, 2404 .llseek = seq_lseek, 2405 .release = seq_release_private, 2406 }; 2407 2408 static const struct file_operations ftrace_filter_fops = { 2409 .open = ftrace_filter_open, 2410 .read = seq_read, 2411 .write = ftrace_filter_write, 2412 .llseek = ftrace_regex_lseek, 2413 .release = ftrace_filter_release, 2414 }; 2415 2416 static const struct file_operations ftrace_notrace_fops = { 2417 .open = ftrace_notrace_open, 2418 .read = seq_read, 2419 .write = ftrace_notrace_write, 2420 .llseek = ftrace_regex_lseek, 2421 .release = ftrace_notrace_release, 2422 }; 2423 2424 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 2425 2426 static DEFINE_MUTEX(graph_lock); 2427 2428 int ftrace_graph_count; 2429 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly; 2430 2431 static void * 2432 __g_next(struct seq_file *m, loff_t *pos) 2433 { 2434 if (*pos >= ftrace_graph_count) 2435 return NULL; 2436 return &ftrace_graph_funcs[*pos]; 2437 } 2438 2439 static void * 2440 g_next(struct seq_file *m, void *v, loff_t *pos) 2441 { 2442 (*pos)++; 2443 return __g_next(m, pos); 2444 } 2445 2446 static void *g_start(struct seq_file *m, loff_t *pos) 2447 { 2448 mutex_lock(&graph_lock); 2449 2450 /* Nothing, tell g_show to print all functions are enabled */ 2451 if (!ftrace_graph_count && !*pos) 2452 return (void *)1; 2453 2454 return __g_next(m, pos); 2455 } 2456 2457 static void g_stop(struct seq_file *m, void *p) 2458 { 2459 mutex_unlock(&graph_lock); 2460 } 2461 2462 static int g_show(struct seq_file *m, void *v) 2463 { 2464 unsigned long *ptr = v; 2465 2466 if (!ptr) 2467 return 0; 2468 2469 if (ptr == (unsigned long *)1) { 2470 seq_printf(m, "#### all functions enabled ####\n"); 2471 return 0; 2472 } 2473 2474 seq_printf(m, "%ps\n", (void *)*ptr); 2475 2476 return 0; 2477 } 2478 2479 static const struct seq_operations ftrace_graph_seq_ops = { 2480 .start = g_start, 2481 .next = g_next, 2482 .stop = g_stop, 2483 .show = g_show, 2484 }; 2485 2486 static int 2487 ftrace_graph_open(struct inode *inode, struct file *file) 2488 { 2489 int ret = 0; 2490 2491 if (unlikely(ftrace_disabled)) 2492 return -ENODEV; 2493 2494 mutex_lock(&graph_lock); 2495 if ((file->f_mode & FMODE_WRITE) && 2496 (file->f_flags & O_TRUNC)) { 2497 ftrace_graph_count = 0; 2498 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs)); 2499 } 2500 mutex_unlock(&graph_lock); 2501 2502 if (file->f_mode & FMODE_READ) 2503 ret = seq_open(file, &ftrace_graph_seq_ops); 2504 2505 return ret; 2506 } 2507 2508 static int 2509 ftrace_graph_release(struct inode *inode, struct file *file) 2510 { 2511 if (file->f_mode & FMODE_READ) 2512 seq_release(inode, file); 2513 return 0; 2514 } 2515 2516 static int 2517 ftrace_set_func(unsigned long *array, int *idx, char *buffer) 2518 { 2519 struct dyn_ftrace *rec; 2520 struct ftrace_page *pg; 2521 int search_len; 2522 int found = 0; 2523 int type, not; 2524 char *search; 2525 bool exists; 2526 int i; 2527 2528 if (ftrace_disabled) 2529 return -ENODEV; 2530 2531 /* decode regex */ 2532 type = filter_parse_regex(buffer, strlen(buffer), &search, ¬); 2533 if (not) 2534 return -EINVAL; 2535 2536 search_len = strlen(search); 2537 2538 mutex_lock(&ftrace_lock); 2539 do_for_each_ftrace_rec(pg, rec) { 2540 2541 if (*idx >= FTRACE_GRAPH_MAX_FUNCS) 2542 break; 2543 2544 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE)) 2545 continue; 2546 2547 if (ftrace_match_record(rec, search, search_len, type)) { 2548 /* ensure it is not already in the array */ 2549 exists = false; 2550 for (i = 0; i < *idx; i++) 2551 if (array[i] == rec->ip) { 2552 exists = true; 2553 break; 2554 } 2555 if (!exists) 2556 array[(*idx)++] = rec->ip; 2557 found = 1; 2558 } 2559 } while_for_each_ftrace_rec(); 2560 2561 mutex_unlock(&ftrace_lock); 2562 2563 return found ? 0 : -EINVAL; 2564 } 2565 2566 static ssize_t 2567 ftrace_graph_write(struct file *file, const char __user *ubuf, 2568 size_t cnt, loff_t *ppos) 2569 { 2570 struct trace_parser parser; 2571 ssize_t read, ret; 2572 2573 if (!cnt || cnt < 0) 2574 return 0; 2575 2576 mutex_lock(&graph_lock); 2577 2578 if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) { 2579 ret = -EBUSY; 2580 goto out_unlock; 2581 } 2582 2583 if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) { 2584 ret = -ENOMEM; 2585 goto out_unlock; 2586 } 2587 2588 read = trace_get_user(&parser, ubuf, cnt, ppos); 2589 2590 if (read >= 0 && trace_parser_loaded((&parser))) { 2591 parser.buffer[parser.idx] = 0; 2592 2593 /* we allow only one expression at a time */ 2594 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count, 2595 parser.buffer); 2596 if (ret) 2597 goto out_free; 2598 } 2599 2600 ret = read; 2601 2602 out_free: 2603 trace_parser_put(&parser); 2604 out_unlock: 2605 mutex_unlock(&graph_lock); 2606 2607 return ret; 2608 } 2609 2610 static const struct file_operations ftrace_graph_fops = { 2611 .open = ftrace_graph_open, 2612 .read = seq_read, 2613 .write = ftrace_graph_write, 2614 .release = ftrace_graph_release, 2615 }; 2616 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 2617 2618 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer) 2619 { 2620 2621 trace_create_file("available_filter_functions", 0444, 2622 d_tracer, NULL, &ftrace_avail_fops); 2623 2624 trace_create_file("failures", 0444, 2625 d_tracer, NULL, &ftrace_failures_fops); 2626 2627 trace_create_file("set_ftrace_filter", 0644, d_tracer, 2628 NULL, &ftrace_filter_fops); 2629 2630 trace_create_file("set_ftrace_notrace", 0644, d_tracer, 2631 NULL, &ftrace_notrace_fops); 2632 2633 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 2634 trace_create_file("set_graph_function", 0444, d_tracer, 2635 NULL, 2636 &ftrace_graph_fops); 2637 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 2638 2639 return 0; 2640 } 2641 2642 static int ftrace_process_locs(struct module *mod, 2643 unsigned long *start, 2644 unsigned long *end) 2645 { 2646 unsigned long *p; 2647 unsigned long addr; 2648 unsigned long flags; 2649 2650 mutex_lock(&ftrace_lock); 2651 p = start; 2652 while (p < end) { 2653 addr = ftrace_call_adjust(*p++); 2654 /* 2655 * Some architecture linkers will pad between 2656 * the different mcount_loc sections of different 2657 * object files to satisfy alignments. 2658 * Skip any NULL pointers. 2659 */ 2660 if (!addr) 2661 continue; 2662 ftrace_record_ip(addr); 2663 } 2664 2665 /* disable interrupts to prevent kstop machine */ 2666 local_irq_save(flags); 2667 ftrace_update_code(mod); 2668 local_irq_restore(flags); 2669 mutex_unlock(&ftrace_lock); 2670 2671 return 0; 2672 } 2673 2674 #ifdef CONFIG_MODULES 2675 void ftrace_release_mod(struct module *mod) 2676 { 2677 struct dyn_ftrace *rec; 2678 struct ftrace_page *pg; 2679 2680 if (ftrace_disabled) 2681 return; 2682 2683 mutex_lock(&ftrace_lock); 2684 do_for_each_ftrace_rec(pg, rec) { 2685 if (within_module_core(rec->ip, mod)) { 2686 /* 2687 * rec->ip is changed in ftrace_free_rec() 2688 * It should not between s and e if record was freed. 2689 */ 2690 FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE); 2691 ftrace_free_rec(rec); 2692 } 2693 } while_for_each_ftrace_rec(); 2694 mutex_unlock(&ftrace_lock); 2695 } 2696 2697 static void ftrace_init_module(struct module *mod, 2698 unsigned long *start, unsigned long *end) 2699 { 2700 if (ftrace_disabled || start == end) 2701 return; 2702 ftrace_process_locs(mod, start, end); 2703 } 2704 2705 static int ftrace_module_notify(struct notifier_block *self, 2706 unsigned long val, void *data) 2707 { 2708 struct module *mod = data; 2709 2710 switch (val) { 2711 case MODULE_STATE_COMING: 2712 ftrace_init_module(mod, mod->ftrace_callsites, 2713 mod->ftrace_callsites + 2714 mod->num_ftrace_callsites); 2715 break; 2716 case MODULE_STATE_GOING: 2717 ftrace_release_mod(mod); 2718 break; 2719 } 2720 2721 return 0; 2722 } 2723 #else 2724 static int ftrace_module_notify(struct notifier_block *self, 2725 unsigned long val, void *data) 2726 { 2727 return 0; 2728 } 2729 #endif /* CONFIG_MODULES */ 2730 2731 struct notifier_block ftrace_module_nb = { 2732 .notifier_call = ftrace_module_notify, 2733 .priority = 0, 2734 }; 2735 2736 extern unsigned long __start_mcount_loc[]; 2737 extern unsigned long __stop_mcount_loc[]; 2738 2739 void __init ftrace_init(void) 2740 { 2741 unsigned long count, addr, flags; 2742 int ret; 2743 2744 /* Keep the ftrace pointer to the stub */ 2745 addr = (unsigned long)ftrace_stub; 2746 2747 local_irq_save(flags); 2748 ftrace_dyn_arch_init(&addr); 2749 local_irq_restore(flags); 2750 2751 /* ftrace_dyn_arch_init places the return code in addr */ 2752 if (addr) 2753 goto failed; 2754 2755 count = __stop_mcount_loc - __start_mcount_loc; 2756 2757 ret = ftrace_dyn_table_alloc(count); 2758 if (ret) 2759 goto failed; 2760 2761 last_ftrace_enabled = ftrace_enabled = 1; 2762 2763 ret = ftrace_process_locs(NULL, 2764 __start_mcount_loc, 2765 __stop_mcount_loc); 2766 2767 ret = register_module_notifier(&ftrace_module_nb); 2768 if (ret) 2769 pr_warning("Failed to register trace ftrace module notifier\n"); 2770 2771 set_ftrace_early_filters(); 2772 2773 return; 2774 failed: 2775 ftrace_disabled = 1; 2776 } 2777 2778 #else 2779 2780 static int __init ftrace_nodyn_init(void) 2781 { 2782 ftrace_enabled = 1; 2783 return 0; 2784 } 2785 device_initcall(ftrace_nodyn_init); 2786 2787 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; } 2788 static inline void ftrace_startup_enable(int command) { } 2789 /* Keep as macros so we do not need to define the commands */ 2790 # define ftrace_startup(command) do { } while (0) 2791 # define ftrace_shutdown(command) do { } while (0) 2792 # define ftrace_startup_sysctl() do { } while (0) 2793 # define ftrace_shutdown_sysctl() do { } while (0) 2794 #endif /* CONFIG_DYNAMIC_FTRACE */ 2795 2796 static void clear_ftrace_swapper(void) 2797 { 2798 struct task_struct *p; 2799 int cpu; 2800 2801 get_online_cpus(); 2802 for_each_online_cpu(cpu) { 2803 p = idle_task(cpu); 2804 clear_tsk_trace_trace(p); 2805 } 2806 put_online_cpus(); 2807 } 2808 2809 static void set_ftrace_swapper(void) 2810 { 2811 struct task_struct *p; 2812 int cpu; 2813 2814 get_online_cpus(); 2815 for_each_online_cpu(cpu) { 2816 p = idle_task(cpu); 2817 set_tsk_trace_trace(p); 2818 } 2819 put_online_cpus(); 2820 } 2821 2822 static void clear_ftrace_pid(struct pid *pid) 2823 { 2824 struct task_struct *p; 2825 2826 rcu_read_lock(); 2827 do_each_pid_task(pid, PIDTYPE_PID, p) { 2828 clear_tsk_trace_trace(p); 2829 } while_each_pid_task(pid, PIDTYPE_PID, p); 2830 rcu_read_unlock(); 2831 2832 put_pid(pid); 2833 } 2834 2835 static void set_ftrace_pid(struct pid *pid) 2836 { 2837 struct task_struct *p; 2838 2839 rcu_read_lock(); 2840 do_each_pid_task(pid, PIDTYPE_PID, p) { 2841 set_tsk_trace_trace(p); 2842 } while_each_pid_task(pid, PIDTYPE_PID, p); 2843 rcu_read_unlock(); 2844 } 2845 2846 static void clear_ftrace_pid_task(struct pid *pid) 2847 { 2848 if (pid == ftrace_swapper_pid) 2849 clear_ftrace_swapper(); 2850 else 2851 clear_ftrace_pid(pid); 2852 } 2853 2854 static void set_ftrace_pid_task(struct pid *pid) 2855 { 2856 if (pid == ftrace_swapper_pid) 2857 set_ftrace_swapper(); 2858 else 2859 set_ftrace_pid(pid); 2860 } 2861 2862 static int ftrace_pid_add(int p) 2863 { 2864 struct pid *pid; 2865 struct ftrace_pid *fpid; 2866 int ret = -EINVAL; 2867 2868 mutex_lock(&ftrace_lock); 2869 2870 if (!p) 2871 pid = ftrace_swapper_pid; 2872 else 2873 pid = find_get_pid(p); 2874 2875 if (!pid) 2876 goto out; 2877 2878 ret = 0; 2879 2880 list_for_each_entry(fpid, &ftrace_pids, list) 2881 if (fpid->pid == pid) 2882 goto out_put; 2883 2884 ret = -ENOMEM; 2885 2886 fpid = kmalloc(sizeof(*fpid), GFP_KERNEL); 2887 if (!fpid) 2888 goto out_put; 2889 2890 list_add(&fpid->list, &ftrace_pids); 2891 fpid->pid = pid; 2892 2893 set_ftrace_pid_task(pid); 2894 2895 ftrace_update_pid_func(); 2896 ftrace_startup_enable(0); 2897 2898 mutex_unlock(&ftrace_lock); 2899 return 0; 2900 2901 out_put: 2902 if (pid != ftrace_swapper_pid) 2903 put_pid(pid); 2904 2905 out: 2906 mutex_unlock(&ftrace_lock); 2907 return ret; 2908 } 2909 2910 static void ftrace_pid_reset(void) 2911 { 2912 struct ftrace_pid *fpid, *safe; 2913 2914 mutex_lock(&ftrace_lock); 2915 list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) { 2916 struct pid *pid = fpid->pid; 2917 2918 clear_ftrace_pid_task(pid); 2919 2920 list_del(&fpid->list); 2921 kfree(fpid); 2922 } 2923 2924 ftrace_update_pid_func(); 2925 ftrace_startup_enable(0); 2926 2927 mutex_unlock(&ftrace_lock); 2928 } 2929 2930 static void *fpid_start(struct seq_file *m, loff_t *pos) 2931 { 2932 mutex_lock(&ftrace_lock); 2933 2934 if (list_empty(&ftrace_pids) && (!*pos)) 2935 return (void *) 1; 2936 2937 return seq_list_start(&ftrace_pids, *pos); 2938 } 2939 2940 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos) 2941 { 2942 if (v == (void *)1) 2943 return NULL; 2944 2945 return seq_list_next(v, &ftrace_pids, pos); 2946 } 2947 2948 static void fpid_stop(struct seq_file *m, void *p) 2949 { 2950 mutex_unlock(&ftrace_lock); 2951 } 2952 2953 static int fpid_show(struct seq_file *m, void *v) 2954 { 2955 const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list); 2956 2957 if (v == (void *)1) { 2958 seq_printf(m, "no pid\n"); 2959 return 0; 2960 } 2961 2962 if (fpid->pid == ftrace_swapper_pid) 2963 seq_printf(m, "swapper tasks\n"); 2964 else 2965 seq_printf(m, "%u\n", pid_vnr(fpid->pid)); 2966 2967 return 0; 2968 } 2969 2970 static const struct seq_operations ftrace_pid_sops = { 2971 .start = fpid_start, 2972 .next = fpid_next, 2973 .stop = fpid_stop, 2974 .show = fpid_show, 2975 }; 2976 2977 static int 2978 ftrace_pid_open(struct inode *inode, struct file *file) 2979 { 2980 int ret = 0; 2981 2982 if ((file->f_mode & FMODE_WRITE) && 2983 (file->f_flags & O_TRUNC)) 2984 ftrace_pid_reset(); 2985 2986 if (file->f_mode & FMODE_READ) 2987 ret = seq_open(file, &ftrace_pid_sops); 2988 2989 return ret; 2990 } 2991 2992 static ssize_t 2993 ftrace_pid_write(struct file *filp, const char __user *ubuf, 2994 size_t cnt, loff_t *ppos) 2995 { 2996 char buf[64], *tmp; 2997 long val; 2998 int ret; 2999 3000 if (cnt >= sizeof(buf)) 3001 return -EINVAL; 3002 3003 if (copy_from_user(&buf, ubuf, cnt)) 3004 return -EFAULT; 3005 3006 buf[cnt] = 0; 3007 3008 /* 3009 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid" 3010 * to clean the filter quietly. 3011 */ 3012 tmp = strstrip(buf); 3013 if (strlen(tmp) == 0) 3014 return 1; 3015 3016 ret = strict_strtol(tmp, 10, &val); 3017 if (ret < 0) 3018 return ret; 3019 3020 ret = ftrace_pid_add(val); 3021 3022 return ret ? ret : cnt; 3023 } 3024 3025 static int 3026 ftrace_pid_release(struct inode *inode, struct file *file) 3027 { 3028 if (file->f_mode & FMODE_READ) 3029 seq_release(inode, file); 3030 3031 return 0; 3032 } 3033 3034 static const struct file_operations ftrace_pid_fops = { 3035 .open = ftrace_pid_open, 3036 .write = ftrace_pid_write, 3037 .read = seq_read, 3038 .llseek = seq_lseek, 3039 .release = ftrace_pid_release, 3040 }; 3041 3042 static __init int ftrace_init_debugfs(void) 3043 { 3044 struct dentry *d_tracer; 3045 3046 d_tracer = tracing_init_dentry(); 3047 if (!d_tracer) 3048 return 0; 3049 3050 ftrace_init_dyn_debugfs(d_tracer); 3051 3052 trace_create_file("set_ftrace_pid", 0644, d_tracer, 3053 NULL, &ftrace_pid_fops); 3054 3055 ftrace_profile_debugfs(d_tracer); 3056 3057 return 0; 3058 } 3059 fs_initcall(ftrace_init_debugfs); 3060 3061 /** 3062 * ftrace_kill - kill ftrace 3063 * 3064 * This function should be used by panic code. It stops ftrace 3065 * but in a not so nice way. If you need to simply kill ftrace 3066 * from a non-atomic section, use ftrace_kill. 3067 */ 3068 void ftrace_kill(void) 3069 { 3070 ftrace_disabled = 1; 3071 ftrace_enabled = 0; 3072 clear_ftrace_function(); 3073 } 3074 3075 /** 3076 * register_ftrace_function - register a function for profiling 3077 * @ops - ops structure that holds the function for profiling. 3078 * 3079 * Register a function to be called by all functions in the 3080 * kernel. 3081 * 3082 * Note: @ops->func and all the functions it calls must be labeled 3083 * with "notrace", otherwise it will go into a 3084 * recursive loop. 3085 */ 3086 int register_ftrace_function(struct ftrace_ops *ops) 3087 { 3088 int ret; 3089 3090 if (unlikely(ftrace_disabled)) 3091 return -1; 3092 3093 mutex_lock(&ftrace_lock); 3094 3095 ret = __register_ftrace_function(ops); 3096 ftrace_startup(0); 3097 3098 mutex_unlock(&ftrace_lock); 3099 return ret; 3100 } 3101 3102 /** 3103 * unregister_ftrace_function - unregister a function for profiling. 3104 * @ops - ops structure that holds the function to unregister 3105 * 3106 * Unregister a function that was added to be called by ftrace profiling. 3107 */ 3108 int unregister_ftrace_function(struct ftrace_ops *ops) 3109 { 3110 int ret; 3111 3112 mutex_lock(&ftrace_lock); 3113 ret = __unregister_ftrace_function(ops); 3114 ftrace_shutdown(0); 3115 mutex_unlock(&ftrace_lock); 3116 3117 return ret; 3118 } 3119 3120 int 3121 ftrace_enable_sysctl(struct ctl_table *table, int write, 3122 void __user *buffer, size_t *lenp, 3123 loff_t *ppos) 3124 { 3125 int ret; 3126 3127 if (unlikely(ftrace_disabled)) 3128 return -ENODEV; 3129 3130 mutex_lock(&ftrace_lock); 3131 3132 ret = proc_dointvec(table, write, buffer, lenp, ppos); 3133 3134 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled)) 3135 goto out; 3136 3137 last_ftrace_enabled = !!ftrace_enabled; 3138 3139 if (ftrace_enabled) { 3140 3141 ftrace_startup_sysctl(); 3142 3143 /* we are starting ftrace again */ 3144 if (ftrace_list != &ftrace_list_end) { 3145 if (ftrace_list->next == &ftrace_list_end) 3146 ftrace_trace_function = ftrace_list->func; 3147 else 3148 ftrace_trace_function = ftrace_list_func; 3149 } 3150 3151 } else { 3152 /* stopping ftrace calls (just send to ftrace_stub) */ 3153 ftrace_trace_function = ftrace_stub; 3154 3155 ftrace_shutdown_sysctl(); 3156 } 3157 3158 out: 3159 mutex_unlock(&ftrace_lock); 3160 return ret; 3161 } 3162 3163 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 3164 3165 static int ftrace_graph_active; 3166 static struct notifier_block ftrace_suspend_notifier; 3167 3168 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) 3169 { 3170 return 0; 3171 } 3172 3173 /* The callbacks that hook a function */ 3174 trace_func_graph_ret_t ftrace_graph_return = 3175 (trace_func_graph_ret_t)ftrace_stub; 3176 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub; 3177 3178 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */ 3179 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list) 3180 { 3181 int i; 3182 int ret = 0; 3183 unsigned long flags; 3184 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE; 3185 struct task_struct *g, *t; 3186 3187 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) { 3188 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH 3189 * sizeof(struct ftrace_ret_stack), 3190 GFP_KERNEL); 3191 if (!ret_stack_list[i]) { 3192 start = 0; 3193 end = i; 3194 ret = -ENOMEM; 3195 goto free; 3196 } 3197 } 3198 3199 read_lock_irqsave(&tasklist_lock, flags); 3200 do_each_thread(g, t) { 3201 if (start == end) { 3202 ret = -EAGAIN; 3203 goto unlock; 3204 } 3205 3206 if (t->ret_stack == NULL) { 3207 atomic_set(&t->tracing_graph_pause, 0); 3208 atomic_set(&t->trace_overrun, 0); 3209 t->curr_ret_stack = -1; 3210 /* Make sure the tasks see the -1 first: */ 3211 smp_wmb(); 3212 t->ret_stack = ret_stack_list[start++]; 3213 } 3214 } while_each_thread(g, t); 3215 3216 unlock: 3217 read_unlock_irqrestore(&tasklist_lock, flags); 3218 free: 3219 for (i = start; i < end; i++) 3220 kfree(ret_stack_list[i]); 3221 return ret; 3222 } 3223 3224 static void 3225 ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev, 3226 struct task_struct *next) 3227 { 3228 unsigned long long timestamp; 3229 int index; 3230 3231 /* 3232 * Does the user want to count the time a function was asleep. 3233 * If so, do not update the time stamps. 3234 */ 3235 if (trace_flags & TRACE_ITER_SLEEP_TIME) 3236 return; 3237 3238 timestamp = trace_clock_local(); 3239 3240 prev->ftrace_timestamp = timestamp; 3241 3242 /* only process tasks that we timestamped */ 3243 if (!next->ftrace_timestamp) 3244 return; 3245 3246 /* 3247 * Update all the counters in next to make up for the 3248 * time next was sleeping. 3249 */ 3250 timestamp -= next->ftrace_timestamp; 3251 3252 for (index = next->curr_ret_stack; index >= 0; index--) 3253 next->ret_stack[index].calltime += timestamp; 3254 } 3255 3256 /* Allocate a return stack for each task */ 3257 static int start_graph_tracing(void) 3258 { 3259 struct ftrace_ret_stack **ret_stack_list; 3260 int ret, cpu; 3261 3262 ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE * 3263 sizeof(struct ftrace_ret_stack *), 3264 GFP_KERNEL); 3265 3266 if (!ret_stack_list) 3267 return -ENOMEM; 3268 3269 /* The cpu_boot init_task->ret_stack will never be freed */ 3270 for_each_online_cpu(cpu) { 3271 if (!idle_task(cpu)->ret_stack) 3272 ftrace_graph_init_task(idle_task(cpu)); 3273 } 3274 3275 do { 3276 ret = alloc_retstack_tasklist(ret_stack_list); 3277 } while (ret == -EAGAIN); 3278 3279 if (!ret) { 3280 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch); 3281 if (ret) 3282 pr_info("ftrace_graph: Couldn't activate tracepoint" 3283 " probe to kernel_sched_switch\n"); 3284 } 3285 3286 kfree(ret_stack_list); 3287 return ret; 3288 } 3289 3290 /* 3291 * Hibernation protection. 3292 * The state of the current task is too much unstable during 3293 * suspend/restore to disk. We want to protect against that. 3294 */ 3295 static int 3296 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state, 3297 void *unused) 3298 { 3299 switch (state) { 3300 case PM_HIBERNATION_PREPARE: 3301 pause_graph_tracing(); 3302 break; 3303 3304 case PM_POST_HIBERNATION: 3305 unpause_graph_tracing(); 3306 break; 3307 } 3308 return NOTIFY_DONE; 3309 } 3310 3311 int register_ftrace_graph(trace_func_graph_ret_t retfunc, 3312 trace_func_graph_ent_t entryfunc) 3313 { 3314 int ret = 0; 3315 3316 mutex_lock(&ftrace_lock); 3317 3318 /* we currently allow only one tracer registered at a time */ 3319 if (ftrace_graph_active) { 3320 ret = -EBUSY; 3321 goto out; 3322 } 3323 3324 ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call; 3325 register_pm_notifier(&ftrace_suspend_notifier); 3326 3327 ftrace_graph_active++; 3328 ret = start_graph_tracing(); 3329 if (ret) { 3330 ftrace_graph_active--; 3331 goto out; 3332 } 3333 3334 ftrace_graph_return = retfunc; 3335 ftrace_graph_entry = entryfunc; 3336 3337 ftrace_startup(FTRACE_START_FUNC_RET); 3338 3339 out: 3340 mutex_unlock(&ftrace_lock); 3341 return ret; 3342 } 3343 3344 void unregister_ftrace_graph(void) 3345 { 3346 mutex_lock(&ftrace_lock); 3347 3348 if (unlikely(!ftrace_graph_active)) 3349 goto out; 3350 3351 ftrace_graph_active--; 3352 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch); 3353 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; 3354 ftrace_graph_entry = ftrace_graph_entry_stub; 3355 ftrace_shutdown(FTRACE_STOP_FUNC_RET); 3356 unregister_pm_notifier(&ftrace_suspend_notifier); 3357 3358 out: 3359 mutex_unlock(&ftrace_lock); 3360 } 3361 3362 /* Allocate a return stack for newly created task */ 3363 void ftrace_graph_init_task(struct task_struct *t) 3364 { 3365 /* Make sure we do not use the parent ret_stack */ 3366 t->ret_stack = NULL; 3367 3368 if (ftrace_graph_active) { 3369 struct ftrace_ret_stack *ret_stack; 3370 3371 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH 3372 * sizeof(struct ftrace_ret_stack), 3373 GFP_KERNEL); 3374 if (!ret_stack) 3375 return; 3376 t->curr_ret_stack = -1; 3377 atomic_set(&t->tracing_graph_pause, 0); 3378 atomic_set(&t->trace_overrun, 0); 3379 t->ftrace_timestamp = 0; 3380 /* make curr_ret_stack visable before we add the ret_stack */ 3381 smp_wmb(); 3382 t->ret_stack = ret_stack; 3383 } 3384 } 3385 3386 void ftrace_graph_exit_task(struct task_struct *t) 3387 { 3388 struct ftrace_ret_stack *ret_stack = t->ret_stack; 3389 3390 t->ret_stack = NULL; 3391 /* NULL must become visible to IRQs before we free it: */ 3392 barrier(); 3393 3394 kfree(ret_stack); 3395 } 3396 3397 void ftrace_graph_stop(void) 3398 { 3399 ftrace_stop(); 3400 } 3401 #endif 3402