1 /* 2 * Infrastructure for profiling code inserted by 'gcc -pg'. 3 * 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> 6 * 7 * Originally ported from the -rt patch by: 8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> 9 * 10 * Based on code in the latency_tracer, that is: 11 * 12 * Copyright (C) 2004-2006 Ingo Molnar 13 * Copyright (C) 2004 Nadia Yvette Chambers 14 */ 15 16 #include <linux/stop_machine.h> 17 #include <linux/clocksource.h> 18 #include <linux/kallsyms.h> 19 #include <linux/seq_file.h> 20 #include <linux/suspend.h> 21 #include <linux/tracefs.h> 22 #include <linux/hardirq.h> 23 #include <linux/kthread.h> 24 #include <linux/uaccess.h> 25 #include <linux/bsearch.h> 26 #include <linux/module.h> 27 #include <linux/ftrace.h> 28 #include <linux/sysctl.h> 29 #include <linux/slab.h> 30 #include <linux/ctype.h> 31 #include <linux/sort.h> 32 #include <linux/list.h> 33 #include <linux/hash.h> 34 #include <linux/rcupdate.h> 35 36 #include <trace/events/sched.h> 37 38 #include <asm/setup.h> 39 40 #include "trace_output.h" 41 #include "trace_stat.h" 42 43 #define FTRACE_WARN_ON(cond) \ 44 ({ \ 45 int ___r = cond; \ 46 if (WARN_ON(___r)) \ 47 ftrace_kill(); \ 48 ___r; \ 49 }) 50 51 #define FTRACE_WARN_ON_ONCE(cond) \ 52 ({ \ 53 int ___r = cond; \ 54 if (WARN_ON_ONCE(___r)) \ 55 ftrace_kill(); \ 56 ___r; \ 57 }) 58 59 /* hash bits for specific function selection */ 60 #define FTRACE_HASH_BITS 7 61 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS) 62 #define FTRACE_HASH_DEFAULT_BITS 10 63 #define FTRACE_HASH_MAX_BITS 12 64 65 #ifdef CONFIG_DYNAMIC_FTRACE 66 #define INIT_OPS_HASH(opsname) \ 67 .func_hash = &opsname.local_hash, \ 68 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), 69 #define ASSIGN_OPS_HASH(opsname, val) \ 70 .func_hash = val, \ 71 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), 72 #else 73 #define INIT_OPS_HASH(opsname) 74 #define ASSIGN_OPS_HASH(opsname, val) 75 #endif 76 77 static struct ftrace_ops ftrace_list_end __read_mostly = { 78 .func = ftrace_stub, 79 .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB, 80 INIT_OPS_HASH(ftrace_list_end) 81 }; 82 83 /* ftrace_enabled is a method to turn ftrace on or off */ 84 int ftrace_enabled __read_mostly; 85 static int last_ftrace_enabled; 86 87 /* Current function tracing op */ 88 struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end; 89 /* What to set function_trace_op to */ 90 static struct ftrace_ops *set_function_trace_op; 91 92 /* List for set_ftrace_pid's pids. */ 93 LIST_HEAD(ftrace_pids); 94 struct ftrace_pid { 95 struct list_head list; 96 struct pid *pid; 97 }; 98 99 static bool ftrace_pids_enabled(void) 100 { 101 return !list_empty(&ftrace_pids); 102 } 103 104 static void ftrace_update_trampoline(struct ftrace_ops *ops); 105 106 /* 107 * ftrace_disabled is set when an anomaly is discovered. 108 * ftrace_disabled is much stronger than ftrace_enabled. 109 */ 110 static int ftrace_disabled __read_mostly; 111 112 static DEFINE_MUTEX(ftrace_lock); 113 114 static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end; 115 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; 116 static struct ftrace_ops global_ops; 117 118 #if ARCH_SUPPORTS_FTRACE_OPS 119 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 120 struct ftrace_ops *op, struct pt_regs *regs); 121 #else 122 /* See comment below, where ftrace_ops_list_func is defined */ 123 static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip); 124 #define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops) 125 #endif 126 127 /* 128 * Traverse the ftrace_global_list, invoking all entries. The reason that we 129 * can use rcu_dereference_raw_notrace() is that elements removed from this list 130 * are simply leaked, so there is no need to interact with a grace-period 131 * mechanism. The rcu_dereference_raw_notrace() calls are needed to handle 132 * concurrent insertions into the ftrace_global_list. 133 * 134 * Silly Alpha and silly pointer-speculation compiler optimizations! 135 */ 136 #define do_for_each_ftrace_op(op, list) \ 137 op = rcu_dereference_raw_notrace(list); \ 138 do 139 140 /* 141 * Optimized for just a single item in the list (as that is the normal case). 142 */ 143 #define while_for_each_ftrace_op(op) \ 144 while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \ 145 unlikely((op) != &ftrace_list_end)) 146 147 static inline void ftrace_ops_init(struct ftrace_ops *ops) 148 { 149 #ifdef CONFIG_DYNAMIC_FTRACE 150 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { 151 mutex_init(&ops->local_hash.regex_lock); 152 ops->func_hash = &ops->local_hash; 153 ops->flags |= FTRACE_OPS_FL_INITIALIZED; 154 } 155 #endif 156 } 157 158 /** 159 * ftrace_nr_registered_ops - return number of ops registered 160 * 161 * Returns the number of ftrace_ops registered and tracing functions 162 */ 163 int ftrace_nr_registered_ops(void) 164 { 165 struct ftrace_ops *ops; 166 int cnt = 0; 167 168 mutex_lock(&ftrace_lock); 169 170 for (ops = ftrace_ops_list; 171 ops != &ftrace_list_end; ops = ops->next) 172 cnt++; 173 174 mutex_unlock(&ftrace_lock); 175 176 return cnt; 177 } 178 179 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip, 180 struct ftrace_ops *op, struct pt_regs *regs) 181 { 182 if (!test_tsk_trace_trace(current)) 183 return; 184 185 op->saved_func(ip, parent_ip, op, regs); 186 } 187 188 /** 189 * clear_ftrace_function - reset the ftrace function 190 * 191 * This NULLs the ftrace function and in essence stops 192 * tracing. There may be lag 193 */ 194 void clear_ftrace_function(void) 195 { 196 ftrace_trace_function = ftrace_stub; 197 } 198 199 static void per_cpu_ops_disable_all(struct ftrace_ops *ops) 200 { 201 int cpu; 202 203 for_each_possible_cpu(cpu) 204 *per_cpu_ptr(ops->disabled, cpu) = 1; 205 } 206 207 static int per_cpu_ops_alloc(struct ftrace_ops *ops) 208 { 209 int __percpu *disabled; 210 211 if (WARN_ON_ONCE(!(ops->flags & FTRACE_OPS_FL_PER_CPU))) 212 return -EINVAL; 213 214 disabled = alloc_percpu(int); 215 if (!disabled) 216 return -ENOMEM; 217 218 ops->disabled = disabled; 219 per_cpu_ops_disable_all(ops); 220 return 0; 221 } 222 223 static void ftrace_sync(struct work_struct *work) 224 { 225 /* 226 * This function is just a stub to implement a hard force 227 * of synchronize_sched(). This requires synchronizing 228 * tasks even in userspace and idle. 229 * 230 * Yes, function tracing is rude. 231 */ 232 } 233 234 static void ftrace_sync_ipi(void *data) 235 { 236 /* Probably not needed, but do it anyway */ 237 smp_rmb(); 238 } 239 240 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 241 static void update_function_graph_func(void); 242 243 /* Both enabled by default (can be cleared by function_graph tracer flags */ 244 static bool fgraph_sleep_time = true; 245 static bool fgraph_graph_time = true; 246 247 #else 248 static inline void update_function_graph_func(void) { } 249 #endif 250 251 252 static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops) 253 { 254 /* 255 * If this is a dynamic, RCU, or per CPU ops, or we force list func, 256 * then it needs to call the list anyway. 257 */ 258 if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU | 259 FTRACE_OPS_FL_RCU) || FTRACE_FORCE_LIST_FUNC) 260 return ftrace_ops_list_func; 261 262 return ftrace_ops_get_func(ops); 263 } 264 265 static void update_ftrace_function(void) 266 { 267 ftrace_func_t func; 268 269 /* 270 * Prepare the ftrace_ops that the arch callback will use. 271 * If there's only one ftrace_ops registered, the ftrace_ops_list 272 * will point to the ops we want. 273 */ 274 set_function_trace_op = ftrace_ops_list; 275 276 /* If there's no ftrace_ops registered, just call the stub function */ 277 if (ftrace_ops_list == &ftrace_list_end) { 278 func = ftrace_stub; 279 280 /* 281 * If we are at the end of the list and this ops is 282 * recursion safe and not dynamic and the arch supports passing ops, 283 * then have the mcount trampoline call the function directly. 284 */ 285 } else if (ftrace_ops_list->next == &ftrace_list_end) { 286 func = ftrace_ops_get_list_func(ftrace_ops_list); 287 288 } else { 289 /* Just use the default ftrace_ops */ 290 set_function_trace_op = &ftrace_list_end; 291 func = ftrace_ops_list_func; 292 } 293 294 update_function_graph_func(); 295 296 /* If there's no change, then do nothing more here */ 297 if (ftrace_trace_function == func) 298 return; 299 300 /* 301 * If we are using the list function, it doesn't care 302 * about the function_trace_ops. 303 */ 304 if (func == ftrace_ops_list_func) { 305 ftrace_trace_function = func; 306 /* 307 * Don't even bother setting function_trace_ops, 308 * it would be racy to do so anyway. 309 */ 310 return; 311 } 312 313 #ifndef CONFIG_DYNAMIC_FTRACE 314 /* 315 * For static tracing, we need to be a bit more careful. 316 * The function change takes affect immediately. Thus, 317 * we need to coorditate the setting of the function_trace_ops 318 * with the setting of the ftrace_trace_function. 319 * 320 * Set the function to the list ops, which will call the 321 * function we want, albeit indirectly, but it handles the 322 * ftrace_ops and doesn't depend on function_trace_op. 323 */ 324 ftrace_trace_function = ftrace_ops_list_func; 325 /* 326 * Make sure all CPUs see this. Yes this is slow, but static 327 * tracing is slow and nasty to have enabled. 328 */ 329 schedule_on_each_cpu(ftrace_sync); 330 /* Now all cpus are using the list ops. */ 331 function_trace_op = set_function_trace_op; 332 /* Make sure the function_trace_op is visible on all CPUs */ 333 smp_wmb(); 334 /* Nasty way to force a rmb on all cpus */ 335 smp_call_function(ftrace_sync_ipi, NULL, 1); 336 /* OK, we are all set to update the ftrace_trace_function now! */ 337 #endif /* !CONFIG_DYNAMIC_FTRACE */ 338 339 ftrace_trace_function = func; 340 } 341 342 int using_ftrace_ops_list_func(void) 343 { 344 return ftrace_trace_function == ftrace_ops_list_func; 345 } 346 347 static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops) 348 { 349 ops->next = *list; 350 /* 351 * We are entering ops into the list but another 352 * CPU might be walking that list. We need to make sure 353 * the ops->next pointer is valid before another CPU sees 354 * the ops pointer included into the list. 355 */ 356 rcu_assign_pointer(*list, ops); 357 } 358 359 static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops) 360 { 361 struct ftrace_ops **p; 362 363 /* 364 * If we are removing the last function, then simply point 365 * to the ftrace_stub. 366 */ 367 if (*list == ops && ops->next == &ftrace_list_end) { 368 *list = &ftrace_list_end; 369 return 0; 370 } 371 372 for (p = list; *p != &ftrace_list_end; p = &(*p)->next) 373 if (*p == ops) 374 break; 375 376 if (*p != ops) 377 return -1; 378 379 *p = (*p)->next; 380 return 0; 381 } 382 383 static void ftrace_update_trampoline(struct ftrace_ops *ops); 384 385 static int __register_ftrace_function(struct ftrace_ops *ops) 386 { 387 if (ops->flags & FTRACE_OPS_FL_DELETED) 388 return -EINVAL; 389 390 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) 391 return -EBUSY; 392 393 #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS 394 /* 395 * If the ftrace_ops specifies SAVE_REGS, then it only can be used 396 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set. 397 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant. 398 */ 399 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS && 400 !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)) 401 return -EINVAL; 402 403 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED) 404 ops->flags |= FTRACE_OPS_FL_SAVE_REGS; 405 #endif 406 407 if (!core_kernel_data((unsigned long)ops)) 408 ops->flags |= FTRACE_OPS_FL_DYNAMIC; 409 410 if (ops->flags & FTRACE_OPS_FL_PER_CPU) { 411 if (per_cpu_ops_alloc(ops)) 412 return -ENOMEM; 413 } 414 415 add_ftrace_ops(&ftrace_ops_list, ops); 416 417 /* Always save the function, and reset at unregistering */ 418 ops->saved_func = ops->func; 419 420 if (ops->flags & FTRACE_OPS_FL_PID && ftrace_pids_enabled()) 421 ops->func = ftrace_pid_func; 422 423 ftrace_update_trampoline(ops); 424 425 if (ftrace_enabled) 426 update_ftrace_function(); 427 428 return 0; 429 } 430 431 static int __unregister_ftrace_function(struct ftrace_ops *ops) 432 { 433 int ret; 434 435 if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED))) 436 return -EBUSY; 437 438 ret = remove_ftrace_ops(&ftrace_ops_list, ops); 439 440 if (ret < 0) 441 return ret; 442 443 if (ftrace_enabled) 444 update_ftrace_function(); 445 446 ops->func = ops->saved_func; 447 448 return 0; 449 } 450 451 static void ftrace_update_pid_func(void) 452 { 453 bool enabled = ftrace_pids_enabled(); 454 struct ftrace_ops *op; 455 456 /* Only do something if we are tracing something */ 457 if (ftrace_trace_function == ftrace_stub) 458 return; 459 460 do_for_each_ftrace_op(op, ftrace_ops_list) { 461 if (op->flags & FTRACE_OPS_FL_PID) { 462 op->func = enabled ? ftrace_pid_func : 463 op->saved_func; 464 ftrace_update_trampoline(op); 465 } 466 } while_for_each_ftrace_op(op); 467 468 update_ftrace_function(); 469 } 470 471 #ifdef CONFIG_FUNCTION_PROFILER 472 struct ftrace_profile { 473 struct hlist_node node; 474 unsigned long ip; 475 unsigned long counter; 476 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 477 unsigned long long time; 478 unsigned long long time_squared; 479 #endif 480 }; 481 482 struct ftrace_profile_page { 483 struct ftrace_profile_page *next; 484 unsigned long index; 485 struct ftrace_profile records[]; 486 }; 487 488 struct ftrace_profile_stat { 489 atomic_t disabled; 490 struct hlist_head *hash; 491 struct ftrace_profile_page *pages; 492 struct ftrace_profile_page *start; 493 struct tracer_stat stat; 494 }; 495 496 #define PROFILE_RECORDS_SIZE \ 497 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records)) 498 499 #define PROFILES_PER_PAGE \ 500 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile)) 501 502 static int ftrace_profile_enabled __read_mostly; 503 504 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */ 505 static DEFINE_MUTEX(ftrace_profile_lock); 506 507 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats); 508 509 #define FTRACE_PROFILE_HASH_BITS 10 510 #define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS) 511 512 static void * 513 function_stat_next(void *v, int idx) 514 { 515 struct ftrace_profile *rec = v; 516 struct ftrace_profile_page *pg; 517 518 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); 519 520 again: 521 if (idx != 0) 522 rec++; 523 524 if ((void *)rec >= (void *)&pg->records[pg->index]) { 525 pg = pg->next; 526 if (!pg) 527 return NULL; 528 rec = &pg->records[0]; 529 if (!rec->counter) 530 goto again; 531 } 532 533 return rec; 534 } 535 536 static void *function_stat_start(struct tracer_stat *trace) 537 { 538 struct ftrace_profile_stat *stat = 539 container_of(trace, struct ftrace_profile_stat, stat); 540 541 if (!stat || !stat->start) 542 return NULL; 543 544 return function_stat_next(&stat->start->records[0], 0); 545 } 546 547 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 548 /* function graph compares on total time */ 549 static int function_stat_cmp(void *p1, void *p2) 550 { 551 struct ftrace_profile *a = p1; 552 struct ftrace_profile *b = p2; 553 554 if (a->time < b->time) 555 return -1; 556 if (a->time > b->time) 557 return 1; 558 else 559 return 0; 560 } 561 #else 562 /* not function graph compares against hits */ 563 static int function_stat_cmp(void *p1, void *p2) 564 { 565 struct ftrace_profile *a = p1; 566 struct ftrace_profile *b = p2; 567 568 if (a->counter < b->counter) 569 return -1; 570 if (a->counter > b->counter) 571 return 1; 572 else 573 return 0; 574 } 575 #endif 576 577 static int function_stat_headers(struct seq_file *m) 578 { 579 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 580 seq_puts(m, " Function " 581 "Hit Time Avg s^2\n" 582 " -------- " 583 "--- ---- --- ---\n"); 584 #else 585 seq_puts(m, " Function Hit\n" 586 " -------- ---\n"); 587 #endif 588 return 0; 589 } 590 591 static int function_stat_show(struct seq_file *m, void *v) 592 { 593 struct ftrace_profile *rec = v; 594 char str[KSYM_SYMBOL_LEN]; 595 int ret = 0; 596 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 597 static struct trace_seq s; 598 unsigned long long avg; 599 unsigned long long stddev; 600 #endif 601 mutex_lock(&ftrace_profile_lock); 602 603 /* we raced with function_profile_reset() */ 604 if (unlikely(rec->counter == 0)) { 605 ret = -EBUSY; 606 goto out; 607 } 608 609 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 610 avg = rec->time; 611 do_div(avg, rec->counter); 612 if (tracing_thresh && (avg < tracing_thresh)) 613 goto out; 614 #endif 615 616 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 617 seq_printf(m, " %-30.30s %10lu", str, rec->counter); 618 619 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 620 seq_puts(m, " "); 621 622 /* Sample standard deviation (s^2) */ 623 if (rec->counter <= 1) 624 stddev = 0; 625 else { 626 /* 627 * Apply Welford's method: 628 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2) 629 */ 630 stddev = rec->counter * rec->time_squared - 631 rec->time * rec->time; 632 633 /* 634 * Divide only 1000 for ns^2 -> us^2 conversion. 635 * trace_print_graph_duration will divide 1000 again. 636 */ 637 do_div(stddev, rec->counter * (rec->counter - 1) * 1000); 638 } 639 640 trace_seq_init(&s); 641 trace_print_graph_duration(rec->time, &s); 642 trace_seq_puts(&s, " "); 643 trace_print_graph_duration(avg, &s); 644 trace_seq_puts(&s, " "); 645 trace_print_graph_duration(stddev, &s); 646 trace_print_seq(m, &s); 647 #endif 648 seq_putc(m, '\n'); 649 out: 650 mutex_unlock(&ftrace_profile_lock); 651 652 return ret; 653 } 654 655 static void ftrace_profile_reset(struct ftrace_profile_stat *stat) 656 { 657 struct ftrace_profile_page *pg; 658 659 pg = stat->pages = stat->start; 660 661 while (pg) { 662 memset(pg->records, 0, PROFILE_RECORDS_SIZE); 663 pg->index = 0; 664 pg = pg->next; 665 } 666 667 memset(stat->hash, 0, 668 FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head)); 669 } 670 671 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat) 672 { 673 struct ftrace_profile_page *pg; 674 int functions; 675 int pages; 676 int i; 677 678 /* If we already allocated, do nothing */ 679 if (stat->pages) 680 return 0; 681 682 stat->pages = (void *)get_zeroed_page(GFP_KERNEL); 683 if (!stat->pages) 684 return -ENOMEM; 685 686 #ifdef CONFIG_DYNAMIC_FTRACE 687 functions = ftrace_update_tot_cnt; 688 #else 689 /* 690 * We do not know the number of functions that exist because 691 * dynamic tracing is what counts them. With past experience 692 * we have around 20K functions. That should be more than enough. 693 * It is highly unlikely we will execute every function in 694 * the kernel. 695 */ 696 functions = 20000; 697 #endif 698 699 pg = stat->start = stat->pages; 700 701 pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE); 702 703 for (i = 1; i < pages; i++) { 704 pg->next = (void *)get_zeroed_page(GFP_KERNEL); 705 if (!pg->next) 706 goto out_free; 707 pg = pg->next; 708 } 709 710 return 0; 711 712 out_free: 713 pg = stat->start; 714 while (pg) { 715 unsigned long tmp = (unsigned long)pg; 716 717 pg = pg->next; 718 free_page(tmp); 719 } 720 721 stat->pages = NULL; 722 stat->start = NULL; 723 724 return -ENOMEM; 725 } 726 727 static int ftrace_profile_init_cpu(int cpu) 728 { 729 struct ftrace_profile_stat *stat; 730 int size; 731 732 stat = &per_cpu(ftrace_profile_stats, cpu); 733 734 if (stat->hash) { 735 /* If the profile is already created, simply reset it */ 736 ftrace_profile_reset(stat); 737 return 0; 738 } 739 740 /* 741 * We are profiling all functions, but usually only a few thousand 742 * functions are hit. We'll make a hash of 1024 items. 743 */ 744 size = FTRACE_PROFILE_HASH_SIZE; 745 746 stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL); 747 748 if (!stat->hash) 749 return -ENOMEM; 750 751 /* Preallocate the function profiling pages */ 752 if (ftrace_profile_pages_init(stat) < 0) { 753 kfree(stat->hash); 754 stat->hash = NULL; 755 return -ENOMEM; 756 } 757 758 return 0; 759 } 760 761 static int ftrace_profile_init(void) 762 { 763 int cpu; 764 int ret = 0; 765 766 for_each_possible_cpu(cpu) { 767 ret = ftrace_profile_init_cpu(cpu); 768 if (ret) 769 break; 770 } 771 772 return ret; 773 } 774 775 /* interrupts must be disabled */ 776 static struct ftrace_profile * 777 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip) 778 { 779 struct ftrace_profile *rec; 780 struct hlist_head *hhd; 781 unsigned long key; 782 783 key = hash_long(ip, FTRACE_PROFILE_HASH_BITS); 784 hhd = &stat->hash[key]; 785 786 if (hlist_empty(hhd)) 787 return NULL; 788 789 hlist_for_each_entry_rcu_notrace(rec, hhd, node) { 790 if (rec->ip == ip) 791 return rec; 792 } 793 794 return NULL; 795 } 796 797 static void ftrace_add_profile(struct ftrace_profile_stat *stat, 798 struct ftrace_profile *rec) 799 { 800 unsigned long key; 801 802 key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS); 803 hlist_add_head_rcu(&rec->node, &stat->hash[key]); 804 } 805 806 /* 807 * The memory is already allocated, this simply finds a new record to use. 808 */ 809 static struct ftrace_profile * 810 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip) 811 { 812 struct ftrace_profile *rec = NULL; 813 814 /* prevent recursion (from NMIs) */ 815 if (atomic_inc_return(&stat->disabled) != 1) 816 goto out; 817 818 /* 819 * Try to find the function again since an NMI 820 * could have added it 821 */ 822 rec = ftrace_find_profiled_func(stat, ip); 823 if (rec) 824 goto out; 825 826 if (stat->pages->index == PROFILES_PER_PAGE) { 827 if (!stat->pages->next) 828 goto out; 829 stat->pages = stat->pages->next; 830 } 831 832 rec = &stat->pages->records[stat->pages->index++]; 833 rec->ip = ip; 834 ftrace_add_profile(stat, rec); 835 836 out: 837 atomic_dec(&stat->disabled); 838 839 return rec; 840 } 841 842 static void 843 function_profile_call(unsigned long ip, unsigned long parent_ip, 844 struct ftrace_ops *ops, struct pt_regs *regs) 845 { 846 struct ftrace_profile_stat *stat; 847 struct ftrace_profile *rec; 848 unsigned long flags; 849 850 if (!ftrace_profile_enabled) 851 return; 852 853 local_irq_save(flags); 854 855 stat = this_cpu_ptr(&ftrace_profile_stats); 856 if (!stat->hash || !ftrace_profile_enabled) 857 goto out; 858 859 rec = ftrace_find_profiled_func(stat, ip); 860 if (!rec) { 861 rec = ftrace_profile_alloc(stat, ip); 862 if (!rec) 863 goto out; 864 } 865 866 rec->counter++; 867 out: 868 local_irq_restore(flags); 869 } 870 871 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 872 static int profile_graph_entry(struct ftrace_graph_ent *trace) 873 { 874 function_profile_call(trace->func, 0, NULL, NULL); 875 return 1; 876 } 877 878 static void profile_graph_return(struct ftrace_graph_ret *trace) 879 { 880 struct ftrace_profile_stat *stat; 881 unsigned long long calltime; 882 struct ftrace_profile *rec; 883 unsigned long flags; 884 885 local_irq_save(flags); 886 stat = this_cpu_ptr(&ftrace_profile_stats); 887 if (!stat->hash || !ftrace_profile_enabled) 888 goto out; 889 890 /* If the calltime was zero'd ignore it */ 891 if (!trace->calltime) 892 goto out; 893 894 calltime = trace->rettime - trace->calltime; 895 896 if (!fgraph_graph_time) { 897 int index; 898 899 index = trace->depth; 900 901 /* Append this call time to the parent time to subtract */ 902 if (index) 903 current->ret_stack[index - 1].subtime += calltime; 904 905 if (current->ret_stack[index].subtime < calltime) 906 calltime -= current->ret_stack[index].subtime; 907 else 908 calltime = 0; 909 } 910 911 rec = ftrace_find_profiled_func(stat, trace->func); 912 if (rec) { 913 rec->time += calltime; 914 rec->time_squared += calltime * calltime; 915 } 916 917 out: 918 local_irq_restore(flags); 919 } 920 921 static int register_ftrace_profiler(void) 922 { 923 return register_ftrace_graph(&profile_graph_return, 924 &profile_graph_entry); 925 } 926 927 static void unregister_ftrace_profiler(void) 928 { 929 unregister_ftrace_graph(); 930 } 931 #else 932 static struct ftrace_ops ftrace_profile_ops __read_mostly = { 933 .func = function_profile_call, 934 .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED, 935 INIT_OPS_HASH(ftrace_profile_ops) 936 }; 937 938 static int register_ftrace_profiler(void) 939 { 940 return register_ftrace_function(&ftrace_profile_ops); 941 } 942 943 static void unregister_ftrace_profiler(void) 944 { 945 unregister_ftrace_function(&ftrace_profile_ops); 946 } 947 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 948 949 static ssize_t 950 ftrace_profile_write(struct file *filp, const char __user *ubuf, 951 size_t cnt, loff_t *ppos) 952 { 953 unsigned long val; 954 int ret; 955 956 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 957 if (ret) 958 return ret; 959 960 val = !!val; 961 962 mutex_lock(&ftrace_profile_lock); 963 if (ftrace_profile_enabled ^ val) { 964 if (val) { 965 ret = ftrace_profile_init(); 966 if (ret < 0) { 967 cnt = ret; 968 goto out; 969 } 970 971 ret = register_ftrace_profiler(); 972 if (ret < 0) { 973 cnt = ret; 974 goto out; 975 } 976 ftrace_profile_enabled = 1; 977 } else { 978 ftrace_profile_enabled = 0; 979 /* 980 * unregister_ftrace_profiler calls stop_machine 981 * so this acts like an synchronize_sched. 982 */ 983 unregister_ftrace_profiler(); 984 } 985 } 986 out: 987 mutex_unlock(&ftrace_profile_lock); 988 989 *ppos += cnt; 990 991 return cnt; 992 } 993 994 static ssize_t 995 ftrace_profile_read(struct file *filp, char __user *ubuf, 996 size_t cnt, loff_t *ppos) 997 { 998 char buf[64]; /* big enough to hold a number */ 999 int r; 1000 1001 r = sprintf(buf, "%u\n", ftrace_profile_enabled); 1002 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 1003 } 1004 1005 static const struct file_operations ftrace_profile_fops = { 1006 .open = tracing_open_generic, 1007 .read = ftrace_profile_read, 1008 .write = ftrace_profile_write, 1009 .llseek = default_llseek, 1010 }; 1011 1012 /* used to initialize the real stat files */ 1013 static struct tracer_stat function_stats __initdata = { 1014 .name = "functions", 1015 .stat_start = function_stat_start, 1016 .stat_next = function_stat_next, 1017 .stat_cmp = function_stat_cmp, 1018 .stat_headers = function_stat_headers, 1019 .stat_show = function_stat_show 1020 }; 1021 1022 static __init void ftrace_profile_tracefs(struct dentry *d_tracer) 1023 { 1024 struct ftrace_profile_stat *stat; 1025 struct dentry *entry; 1026 char *name; 1027 int ret; 1028 int cpu; 1029 1030 for_each_possible_cpu(cpu) { 1031 stat = &per_cpu(ftrace_profile_stats, cpu); 1032 1033 name = kasprintf(GFP_KERNEL, "function%d", cpu); 1034 if (!name) { 1035 /* 1036 * The files created are permanent, if something happens 1037 * we still do not free memory. 1038 */ 1039 WARN(1, 1040 "Could not allocate stat file for cpu %d\n", 1041 cpu); 1042 return; 1043 } 1044 stat->stat = function_stats; 1045 stat->stat.name = name; 1046 ret = register_stat_tracer(&stat->stat); 1047 if (ret) { 1048 WARN(1, 1049 "Could not register function stat for cpu %d\n", 1050 cpu); 1051 kfree(name); 1052 return; 1053 } 1054 } 1055 1056 entry = tracefs_create_file("function_profile_enabled", 0644, 1057 d_tracer, NULL, &ftrace_profile_fops); 1058 if (!entry) 1059 pr_warn("Could not create tracefs 'function_profile_enabled' entry\n"); 1060 } 1061 1062 #else /* CONFIG_FUNCTION_PROFILER */ 1063 static __init void ftrace_profile_tracefs(struct dentry *d_tracer) 1064 { 1065 } 1066 #endif /* CONFIG_FUNCTION_PROFILER */ 1067 1068 static struct pid * const ftrace_swapper_pid = &init_struct_pid; 1069 1070 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 1071 static int ftrace_graph_active; 1072 #else 1073 # define ftrace_graph_active 0 1074 #endif 1075 1076 #ifdef CONFIG_DYNAMIC_FTRACE 1077 1078 static struct ftrace_ops *removed_ops; 1079 1080 /* 1081 * Set when doing a global update, like enabling all recs or disabling them. 1082 * It is not set when just updating a single ftrace_ops. 1083 */ 1084 static bool update_all_ops; 1085 1086 #ifndef CONFIG_FTRACE_MCOUNT_RECORD 1087 # error Dynamic ftrace depends on MCOUNT_RECORD 1088 #endif 1089 1090 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly; 1091 1092 struct ftrace_func_probe { 1093 struct hlist_node node; 1094 struct ftrace_probe_ops *ops; 1095 unsigned long flags; 1096 unsigned long ip; 1097 void *data; 1098 struct list_head free_list; 1099 }; 1100 1101 struct ftrace_func_entry { 1102 struct hlist_node hlist; 1103 unsigned long ip; 1104 }; 1105 1106 struct ftrace_hash { 1107 unsigned long size_bits; 1108 struct hlist_head *buckets; 1109 unsigned long count; 1110 struct rcu_head rcu; 1111 }; 1112 1113 /* 1114 * We make these constant because no one should touch them, 1115 * but they are used as the default "empty hash", to avoid allocating 1116 * it all the time. These are in a read only section such that if 1117 * anyone does try to modify it, it will cause an exception. 1118 */ 1119 static const struct hlist_head empty_buckets[1]; 1120 static const struct ftrace_hash empty_hash = { 1121 .buckets = (struct hlist_head *)empty_buckets, 1122 }; 1123 #define EMPTY_HASH ((struct ftrace_hash *)&empty_hash) 1124 1125 static struct ftrace_ops global_ops = { 1126 .func = ftrace_stub, 1127 .local_hash.notrace_hash = EMPTY_HASH, 1128 .local_hash.filter_hash = EMPTY_HASH, 1129 INIT_OPS_HASH(global_ops) 1130 .flags = FTRACE_OPS_FL_RECURSION_SAFE | 1131 FTRACE_OPS_FL_INITIALIZED | 1132 FTRACE_OPS_FL_PID, 1133 }; 1134 1135 /* 1136 * This is used by __kernel_text_address() to return true if the 1137 * address is on a dynamically allocated trampoline that would 1138 * not return true for either core_kernel_text() or 1139 * is_module_text_address(). 1140 */ 1141 bool is_ftrace_trampoline(unsigned long addr) 1142 { 1143 struct ftrace_ops *op; 1144 bool ret = false; 1145 1146 /* 1147 * Some of the ops may be dynamically allocated, 1148 * they are freed after a synchronize_sched(). 1149 */ 1150 preempt_disable_notrace(); 1151 1152 do_for_each_ftrace_op(op, ftrace_ops_list) { 1153 /* 1154 * This is to check for dynamically allocated trampolines. 1155 * Trampolines that are in kernel text will have 1156 * core_kernel_text() return true. 1157 */ 1158 if (op->trampoline && op->trampoline_size) 1159 if (addr >= op->trampoline && 1160 addr < op->trampoline + op->trampoline_size) { 1161 ret = true; 1162 goto out; 1163 } 1164 } while_for_each_ftrace_op(op); 1165 1166 out: 1167 preempt_enable_notrace(); 1168 1169 return ret; 1170 } 1171 1172 struct ftrace_page { 1173 struct ftrace_page *next; 1174 struct dyn_ftrace *records; 1175 int index; 1176 int size; 1177 }; 1178 1179 #define ENTRY_SIZE sizeof(struct dyn_ftrace) 1180 #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE) 1181 1182 /* estimate from running different kernels */ 1183 #define NR_TO_INIT 10000 1184 1185 static struct ftrace_page *ftrace_pages_start; 1186 static struct ftrace_page *ftrace_pages; 1187 1188 static bool __always_inline ftrace_hash_empty(struct ftrace_hash *hash) 1189 { 1190 return !hash || !hash->count; 1191 } 1192 1193 static struct ftrace_func_entry * 1194 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) 1195 { 1196 unsigned long key; 1197 struct ftrace_func_entry *entry; 1198 struct hlist_head *hhd; 1199 1200 if (ftrace_hash_empty(hash)) 1201 return NULL; 1202 1203 if (hash->size_bits > 0) 1204 key = hash_long(ip, hash->size_bits); 1205 else 1206 key = 0; 1207 1208 hhd = &hash->buckets[key]; 1209 1210 hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) { 1211 if (entry->ip == ip) 1212 return entry; 1213 } 1214 return NULL; 1215 } 1216 1217 static void __add_hash_entry(struct ftrace_hash *hash, 1218 struct ftrace_func_entry *entry) 1219 { 1220 struct hlist_head *hhd; 1221 unsigned long key; 1222 1223 if (hash->size_bits) 1224 key = hash_long(entry->ip, hash->size_bits); 1225 else 1226 key = 0; 1227 1228 hhd = &hash->buckets[key]; 1229 hlist_add_head(&entry->hlist, hhd); 1230 hash->count++; 1231 } 1232 1233 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip) 1234 { 1235 struct ftrace_func_entry *entry; 1236 1237 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 1238 if (!entry) 1239 return -ENOMEM; 1240 1241 entry->ip = ip; 1242 __add_hash_entry(hash, entry); 1243 1244 return 0; 1245 } 1246 1247 static void 1248 free_hash_entry(struct ftrace_hash *hash, 1249 struct ftrace_func_entry *entry) 1250 { 1251 hlist_del(&entry->hlist); 1252 kfree(entry); 1253 hash->count--; 1254 } 1255 1256 static void 1257 remove_hash_entry(struct ftrace_hash *hash, 1258 struct ftrace_func_entry *entry) 1259 { 1260 hlist_del(&entry->hlist); 1261 hash->count--; 1262 } 1263 1264 static void ftrace_hash_clear(struct ftrace_hash *hash) 1265 { 1266 struct hlist_head *hhd; 1267 struct hlist_node *tn; 1268 struct ftrace_func_entry *entry; 1269 int size = 1 << hash->size_bits; 1270 int i; 1271 1272 if (!hash->count) 1273 return; 1274 1275 for (i = 0; i < size; i++) { 1276 hhd = &hash->buckets[i]; 1277 hlist_for_each_entry_safe(entry, tn, hhd, hlist) 1278 free_hash_entry(hash, entry); 1279 } 1280 FTRACE_WARN_ON(hash->count); 1281 } 1282 1283 static void free_ftrace_hash(struct ftrace_hash *hash) 1284 { 1285 if (!hash || hash == EMPTY_HASH) 1286 return; 1287 ftrace_hash_clear(hash); 1288 kfree(hash->buckets); 1289 kfree(hash); 1290 } 1291 1292 static void __free_ftrace_hash_rcu(struct rcu_head *rcu) 1293 { 1294 struct ftrace_hash *hash; 1295 1296 hash = container_of(rcu, struct ftrace_hash, rcu); 1297 free_ftrace_hash(hash); 1298 } 1299 1300 static void free_ftrace_hash_rcu(struct ftrace_hash *hash) 1301 { 1302 if (!hash || hash == EMPTY_HASH) 1303 return; 1304 call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu); 1305 } 1306 1307 void ftrace_free_filter(struct ftrace_ops *ops) 1308 { 1309 ftrace_ops_init(ops); 1310 free_ftrace_hash(ops->func_hash->filter_hash); 1311 free_ftrace_hash(ops->func_hash->notrace_hash); 1312 } 1313 1314 static struct ftrace_hash *alloc_ftrace_hash(int size_bits) 1315 { 1316 struct ftrace_hash *hash; 1317 int size; 1318 1319 hash = kzalloc(sizeof(*hash), GFP_KERNEL); 1320 if (!hash) 1321 return NULL; 1322 1323 size = 1 << size_bits; 1324 hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL); 1325 1326 if (!hash->buckets) { 1327 kfree(hash); 1328 return NULL; 1329 } 1330 1331 hash->size_bits = size_bits; 1332 1333 return hash; 1334 } 1335 1336 static struct ftrace_hash * 1337 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash) 1338 { 1339 struct ftrace_func_entry *entry; 1340 struct ftrace_hash *new_hash; 1341 int size; 1342 int ret; 1343 int i; 1344 1345 new_hash = alloc_ftrace_hash(size_bits); 1346 if (!new_hash) 1347 return NULL; 1348 1349 /* Empty hash? */ 1350 if (ftrace_hash_empty(hash)) 1351 return new_hash; 1352 1353 size = 1 << hash->size_bits; 1354 for (i = 0; i < size; i++) { 1355 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 1356 ret = add_hash_entry(new_hash, entry->ip); 1357 if (ret < 0) 1358 goto free_hash; 1359 } 1360 } 1361 1362 FTRACE_WARN_ON(new_hash->count != hash->count); 1363 1364 return new_hash; 1365 1366 free_hash: 1367 free_ftrace_hash(new_hash); 1368 return NULL; 1369 } 1370 1371 static void 1372 ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash); 1373 static void 1374 ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash); 1375 1376 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, 1377 struct ftrace_hash *new_hash); 1378 1379 static int 1380 ftrace_hash_move(struct ftrace_ops *ops, int enable, 1381 struct ftrace_hash **dst, struct ftrace_hash *src) 1382 { 1383 struct ftrace_func_entry *entry; 1384 struct hlist_node *tn; 1385 struct hlist_head *hhd; 1386 struct ftrace_hash *new_hash; 1387 int size = src->count; 1388 int bits = 0; 1389 int ret; 1390 int i; 1391 1392 /* Reject setting notrace hash on IPMODIFY ftrace_ops */ 1393 if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable) 1394 return -EINVAL; 1395 1396 /* 1397 * If the new source is empty, just free dst and assign it 1398 * the empty_hash. 1399 */ 1400 if (!src->count) { 1401 new_hash = EMPTY_HASH; 1402 goto update; 1403 } 1404 1405 /* 1406 * Make the hash size about 1/2 the # found 1407 */ 1408 for (size /= 2; size; size >>= 1) 1409 bits++; 1410 1411 /* Don't allocate too much */ 1412 if (bits > FTRACE_HASH_MAX_BITS) 1413 bits = FTRACE_HASH_MAX_BITS; 1414 1415 new_hash = alloc_ftrace_hash(bits); 1416 if (!new_hash) 1417 return -ENOMEM; 1418 1419 size = 1 << src->size_bits; 1420 for (i = 0; i < size; i++) { 1421 hhd = &src->buckets[i]; 1422 hlist_for_each_entry_safe(entry, tn, hhd, hlist) { 1423 remove_hash_entry(src, entry); 1424 __add_hash_entry(new_hash, entry); 1425 } 1426 } 1427 1428 update: 1429 /* Make sure this can be applied if it is IPMODIFY ftrace_ops */ 1430 if (enable) { 1431 /* IPMODIFY should be updated only when filter_hash updating */ 1432 ret = ftrace_hash_ipmodify_update(ops, new_hash); 1433 if (ret < 0) { 1434 free_ftrace_hash(new_hash); 1435 return ret; 1436 } 1437 } 1438 1439 /* 1440 * Remove the current set, update the hash and add 1441 * them back. 1442 */ 1443 ftrace_hash_rec_disable_modify(ops, enable); 1444 1445 rcu_assign_pointer(*dst, new_hash); 1446 1447 ftrace_hash_rec_enable_modify(ops, enable); 1448 1449 return 0; 1450 } 1451 1452 static bool hash_contains_ip(unsigned long ip, 1453 struct ftrace_ops_hash *hash) 1454 { 1455 /* 1456 * The function record is a match if it exists in the filter 1457 * hash and not in the notrace hash. Note, an emty hash is 1458 * considered a match for the filter hash, but an empty 1459 * notrace hash is considered not in the notrace hash. 1460 */ 1461 return (ftrace_hash_empty(hash->filter_hash) || 1462 ftrace_lookup_ip(hash->filter_hash, ip)) && 1463 (ftrace_hash_empty(hash->notrace_hash) || 1464 !ftrace_lookup_ip(hash->notrace_hash, ip)); 1465 } 1466 1467 /* 1468 * Test the hashes for this ops to see if we want to call 1469 * the ops->func or not. 1470 * 1471 * It's a match if the ip is in the ops->filter_hash or 1472 * the filter_hash does not exist or is empty, 1473 * AND 1474 * the ip is not in the ops->notrace_hash. 1475 * 1476 * This needs to be called with preemption disabled as 1477 * the hashes are freed with call_rcu_sched(). 1478 */ 1479 static int 1480 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) 1481 { 1482 struct ftrace_ops_hash hash; 1483 int ret; 1484 1485 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 1486 /* 1487 * There's a small race when adding ops that the ftrace handler 1488 * that wants regs, may be called without them. We can not 1489 * allow that handler to be called if regs is NULL. 1490 */ 1491 if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS)) 1492 return 0; 1493 #endif 1494 1495 hash.filter_hash = rcu_dereference_raw_notrace(ops->func_hash->filter_hash); 1496 hash.notrace_hash = rcu_dereference_raw_notrace(ops->func_hash->notrace_hash); 1497 1498 if (hash_contains_ip(ip, &hash)) 1499 ret = 1; 1500 else 1501 ret = 0; 1502 1503 return ret; 1504 } 1505 1506 /* 1507 * This is a double for. Do not use 'break' to break out of the loop, 1508 * you must use a goto. 1509 */ 1510 #define do_for_each_ftrace_rec(pg, rec) \ 1511 for (pg = ftrace_pages_start; pg; pg = pg->next) { \ 1512 int _____i; \ 1513 for (_____i = 0; _____i < pg->index; _____i++) { \ 1514 rec = &pg->records[_____i]; 1515 1516 #define while_for_each_ftrace_rec() \ 1517 } \ 1518 } 1519 1520 1521 static int ftrace_cmp_recs(const void *a, const void *b) 1522 { 1523 const struct dyn_ftrace *key = a; 1524 const struct dyn_ftrace *rec = b; 1525 1526 if (key->flags < rec->ip) 1527 return -1; 1528 if (key->ip >= rec->ip + MCOUNT_INSN_SIZE) 1529 return 1; 1530 return 0; 1531 } 1532 1533 static unsigned long ftrace_location_range(unsigned long start, unsigned long end) 1534 { 1535 struct ftrace_page *pg; 1536 struct dyn_ftrace *rec; 1537 struct dyn_ftrace key; 1538 1539 key.ip = start; 1540 key.flags = end; /* overload flags, as it is unsigned long */ 1541 1542 for (pg = ftrace_pages_start; pg; pg = pg->next) { 1543 if (end < pg->records[0].ip || 1544 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) 1545 continue; 1546 rec = bsearch(&key, pg->records, pg->index, 1547 sizeof(struct dyn_ftrace), 1548 ftrace_cmp_recs); 1549 if (rec) 1550 return rec->ip; 1551 } 1552 1553 return 0; 1554 } 1555 1556 /** 1557 * ftrace_location - return true if the ip giving is a traced location 1558 * @ip: the instruction pointer to check 1559 * 1560 * Returns rec->ip if @ip given is a pointer to a ftrace location. 1561 * That is, the instruction that is either a NOP or call to 1562 * the function tracer. It checks the ftrace internal tables to 1563 * determine if the address belongs or not. 1564 */ 1565 unsigned long ftrace_location(unsigned long ip) 1566 { 1567 return ftrace_location_range(ip, ip); 1568 } 1569 1570 /** 1571 * ftrace_text_reserved - return true if range contains an ftrace location 1572 * @start: start of range to search 1573 * @end: end of range to search (inclusive). @end points to the last byte to check. 1574 * 1575 * Returns 1 if @start and @end contains a ftrace location. 1576 * That is, the instruction that is either a NOP or call to 1577 * the function tracer. It checks the ftrace internal tables to 1578 * determine if the address belongs or not. 1579 */ 1580 int ftrace_text_reserved(const void *start, const void *end) 1581 { 1582 unsigned long ret; 1583 1584 ret = ftrace_location_range((unsigned long)start, 1585 (unsigned long)end); 1586 1587 return (int)!!ret; 1588 } 1589 1590 /* Test if ops registered to this rec needs regs */ 1591 static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec) 1592 { 1593 struct ftrace_ops *ops; 1594 bool keep_regs = false; 1595 1596 for (ops = ftrace_ops_list; 1597 ops != &ftrace_list_end; ops = ops->next) { 1598 /* pass rec in as regs to have non-NULL val */ 1599 if (ftrace_ops_test(ops, rec->ip, rec)) { 1600 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) { 1601 keep_regs = true; 1602 break; 1603 } 1604 } 1605 } 1606 1607 return keep_regs; 1608 } 1609 1610 static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, 1611 int filter_hash, 1612 bool inc) 1613 { 1614 struct ftrace_hash *hash; 1615 struct ftrace_hash *other_hash; 1616 struct ftrace_page *pg; 1617 struct dyn_ftrace *rec; 1618 bool update = false; 1619 int count = 0; 1620 int all = 0; 1621 1622 /* Only update if the ops has been registered */ 1623 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 1624 return false; 1625 1626 /* 1627 * In the filter_hash case: 1628 * If the count is zero, we update all records. 1629 * Otherwise we just update the items in the hash. 1630 * 1631 * In the notrace_hash case: 1632 * We enable the update in the hash. 1633 * As disabling notrace means enabling the tracing, 1634 * and enabling notrace means disabling, the inc variable 1635 * gets inversed. 1636 */ 1637 if (filter_hash) { 1638 hash = ops->func_hash->filter_hash; 1639 other_hash = ops->func_hash->notrace_hash; 1640 if (ftrace_hash_empty(hash)) 1641 all = 1; 1642 } else { 1643 inc = !inc; 1644 hash = ops->func_hash->notrace_hash; 1645 other_hash = ops->func_hash->filter_hash; 1646 /* 1647 * If the notrace hash has no items, 1648 * then there's nothing to do. 1649 */ 1650 if (ftrace_hash_empty(hash)) 1651 return false; 1652 } 1653 1654 do_for_each_ftrace_rec(pg, rec) { 1655 int in_other_hash = 0; 1656 int in_hash = 0; 1657 int match = 0; 1658 1659 if (rec->flags & FTRACE_FL_DISABLED) 1660 continue; 1661 1662 if (all) { 1663 /* 1664 * Only the filter_hash affects all records. 1665 * Update if the record is not in the notrace hash. 1666 */ 1667 if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip)) 1668 match = 1; 1669 } else { 1670 in_hash = !!ftrace_lookup_ip(hash, rec->ip); 1671 in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip); 1672 1673 /* 1674 * If filter_hash is set, we want to match all functions 1675 * that are in the hash but not in the other hash. 1676 * 1677 * If filter_hash is not set, then we are decrementing. 1678 * That means we match anything that is in the hash 1679 * and also in the other_hash. That is, we need to turn 1680 * off functions in the other hash because they are disabled 1681 * by this hash. 1682 */ 1683 if (filter_hash && in_hash && !in_other_hash) 1684 match = 1; 1685 else if (!filter_hash && in_hash && 1686 (in_other_hash || ftrace_hash_empty(other_hash))) 1687 match = 1; 1688 } 1689 if (!match) 1690 continue; 1691 1692 if (inc) { 1693 rec->flags++; 1694 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX)) 1695 return false; 1696 1697 /* 1698 * If there's only a single callback registered to a 1699 * function, and the ops has a trampoline registered 1700 * for it, then we can call it directly. 1701 */ 1702 if (ftrace_rec_count(rec) == 1 && ops->trampoline) 1703 rec->flags |= FTRACE_FL_TRAMP; 1704 else 1705 /* 1706 * If we are adding another function callback 1707 * to this function, and the previous had a 1708 * custom trampoline in use, then we need to go 1709 * back to the default trampoline. 1710 */ 1711 rec->flags &= ~FTRACE_FL_TRAMP; 1712 1713 /* 1714 * If any ops wants regs saved for this function 1715 * then all ops will get saved regs. 1716 */ 1717 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) 1718 rec->flags |= FTRACE_FL_REGS; 1719 } else { 1720 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0)) 1721 return false; 1722 rec->flags--; 1723 1724 /* 1725 * If the rec had REGS enabled and the ops that is 1726 * being removed had REGS set, then see if there is 1727 * still any ops for this record that wants regs. 1728 * If not, we can stop recording them. 1729 */ 1730 if (ftrace_rec_count(rec) > 0 && 1731 rec->flags & FTRACE_FL_REGS && 1732 ops->flags & FTRACE_OPS_FL_SAVE_REGS) { 1733 if (!test_rec_ops_needs_regs(rec)) 1734 rec->flags &= ~FTRACE_FL_REGS; 1735 } 1736 1737 /* 1738 * If the rec had TRAMP enabled, then it needs to 1739 * be cleared. As TRAMP can only be enabled iff 1740 * there is only a single ops attached to it. 1741 * In otherwords, always disable it on decrementing. 1742 * In the future, we may set it if rec count is 1743 * decremented to one, and the ops that is left 1744 * has a trampoline. 1745 */ 1746 rec->flags &= ~FTRACE_FL_TRAMP; 1747 1748 /* 1749 * flags will be cleared in ftrace_check_record() 1750 * if rec count is zero. 1751 */ 1752 } 1753 count++; 1754 1755 /* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */ 1756 update |= ftrace_test_record(rec, 1) != FTRACE_UPDATE_IGNORE; 1757 1758 /* Shortcut, if we handled all records, we are done. */ 1759 if (!all && count == hash->count) 1760 return update; 1761 } while_for_each_ftrace_rec(); 1762 1763 return update; 1764 } 1765 1766 static bool ftrace_hash_rec_disable(struct ftrace_ops *ops, 1767 int filter_hash) 1768 { 1769 return __ftrace_hash_rec_update(ops, filter_hash, 0); 1770 } 1771 1772 static bool ftrace_hash_rec_enable(struct ftrace_ops *ops, 1773 int filter_hash) 1774 { 1775 return __ftrace_hash_rec_update(ops, filter_hash, 1); 1776 } 1777 1778 static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, 1779 int filter_hash, int inc) 1780 { 1781 struct ftrace_ops *op; 1782 1783 __ftrace_hash_rec_update(ops, filter_hash, inc); 1784 1785 if (ops->func_hash != &global_ops.local_hash) 1786 return; 1787 1788 /* 1789 * If the ops shares the global_ops hash, then we need to update 1790 * all ops that are enabled and use this hash. 1791 */ 1792 do_for_each_ftrace_op(op, ftrace_ops_list) { 1793 /* Already done */ 1794 if (op == ops) 1795 continue; 1796 if (op->func_hash == &global_ops.local_hash) 1797 __ftrace_hash_rec_update(op, filter_hash, inc); 1798 } while_for_each_ftrace_op(op); 1799 } 1800 1801 static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, 1802 int filter_hash) 1803 { 1804 ftrace_hash_rec_update_modify(ops, filter_hash, 0); 1805 } 1806 1807 static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, 1808 int filter_hash) 1809 { 1810 ftrace_hash_rec_update_modify(ops, filter_hash, 1); 1811 } 1812 1813 /* 1814 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK 1815 * or no-needed to update, -EBUSY if it detects a conflict of the flag 1816 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs. 1817 * Note that old_hash and new_hash has below meanings 1818 * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected) 1819 * - If the hash is EMPTY_HASH, it hits nothing 1820 * - Anything else hits the recs which match the hash entries. 1821 */ 1822 static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops, 1823 struct ftrace_hash *old_hash, 1824 struct ftrace_hash *new_hash) 1825 { 1826 struct ftrace_page *pg; 1827 struct dyn_ftrace *rec, *end = NULL; 1828 int in_old, in_new; 1829 1830 /* Only update if the ops has been registered */ 1831 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 1832 return 0; 1833 1834 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) 1835 return 0; 1836 1837 /* 1838 * Since the IPMODIFY is a very address sensitive action, we do not 1839 * allow ftrace_ops to set all functions to new hash. 1840 */ 1841 if (!new_hash || !old_hash) 1842 return -EINVAL; 1843 1844 /* Update rec->flags */ 1845 do_for_each_ftrace_rec(pg, rec) { 1846 /* We need to update only differences of filter_hash */ 1847 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); 1848 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); 1849 if (in_old == in_new) 1850 continue; 1851 1852 if (in_new) { 1853 /* New entries must ensure no others are using it */ 1854 if (rec->flags & FTRACE_FL_IPMODIFY) 1855 goto rollback; 1856 rec->flags |= FTRACE_FL_IPMODIFY; 1857 } else /* Removed entry */ 1858 rec->flags &= ~FTRACE_FL_IPMODIFY; 1859 } while_for_each_ftrace_rec(); 1860 1861 return 0; 1862 1863 rollback: 1864 end = rec; 1865 1866 /* Roll back what we did above */ 1867 do_for_each_ftrace_rec(pg, rec) { 1868 if (rec == end) 1869 goto err_out; 1870 1871 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); 1872 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); 1873 if (in_old == in_new) 1874 continue; 1875 1876 if (in_new) 1877 rec->flags &= ~FTRACE_FL_IPMODIFY; 1878 else 1879 rec->flags |= FTRACE_FL_IPMODIFY; 1880 } while_for_each_ftrace_rec(); 1881 1882 err_out: 1883 return -EBUSY; 1884 } 1885 1886 static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops) 1887 { 1888 struct ftrace_hash *hash = ops->func_hash->filter_hash; 1889 1890 if (ftrace_hash_empty(hash)) 1891 hash = NULL; 1892 1893 return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash); 1894 } 1895 1896 /* Disabling always succeeds */ 1897 static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops) 1898 { 1899 struct ftrace_hash *hash = ops->func_hash->filter_hash; 1900 1901 if (ftrace_hash_empty(hash)) 1902 hash = NULL; 1903 1904 __ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH); 1905 } 1906 1907 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, 1908 struct ftrace_hash *new_hash) 1909 { 1910 struct ftrace_hash *old_hash = ops->func_hash->filter_hash; 1911 1912 if (ftrace_hash_empty(old_hash)) 1913 old_hash = NULL; 1914 1915 if (ftrace_hash_empty(new_hash)) 1916 new_hash = NULL; 1917 1918 return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash); 1919 } 1920 1921 static void print_ip_ins(const char *fmt, const unsigned char *p) 1922 { 1923 int i; 1924 1925 printk(KERN_CONT "%s", fmt); 1926 1927 for (i = 0; i < MCOUNT_INSN_SIZE; i++) 1928 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]); 1929 } 1930 1931 static struct ftrace_ops * 1932 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec); 1933 static struct ftrace_ops * 1934 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops); 1935 1936 enum ftrace_bug_type ftrace_bug_type; 1937 const void *ftrace_expected; 1938 1939 static void print_bug_type(void) 1940 { 1941 switch (ftrace_bug_type) { 1942 case FTRACE_BUG_UNKNOWN: 1943 break; 1944 case FTRACE_BUG_INIT: 1945 pr_info("Initializing ftrace call sites\n"); 1946 break; 1947 case FTRACE_BUG_NOP: 1948 pr_info("Setting ftrace call site to NOP\n"); 1949 break; 1950 case FTRACE_BUG_CALL: 1951 pr_info("Setting ftrace call site to call ftrace function\n"); 1952 break; 1953 case FTRACE_BUG_UPDATE: 1954 pr_info("Updating ftrace call site to call a different ftrace function\n"); 1955 break; 1956 } 1957 } 1958 1959 /** 1960 * ftrace_bug - report and shutdown function tracer 1961 * @failed: The failed type (EFAULT, EINVAL, EPERM) 1962 * @rec: The record that failed 1963 * 1964 * The arch code that enables or disables the function tracing 1965 * can call ftrace_bug() when it has detected a problem in 1966 * modifying the code. @failed should be one of either: 1967 * EFAULT - if the problem happens on reading the @ip address 1968 * EINVAL - if what is read at @ip is not what was expected 1969 * EPERM - if the problem happens on writting to the @ip address 1970 */ 1971 void ftrace_bug(int failed, struct dyn_ftrace *rec) 1972 { 1973 unsigned long ip = rec ? rec->ip : 0; 1974 1975 switch (failed) { 1976 case -EFAULT: 1977 FTRACE_WARN_ON_ONCE(1); 1978 pr_info("ftrace faulted on modifying "); 1979 print_ip_sym(ip); 1980 break; 1981 case -EINVAL: 1982 FTRACE_WARN_ON_ONCE(1); 1983 pr_info("ftrace failed to modify "); 1984 print_ip_sym(ip); 1985 print_ip_ins(" actual: ", (unsigned char *)ip); 1986 pr_cont("\n"); 1987 if (ftrace_expected) { 1988 print_ip_ins(" expected: ", ftrace_expected); 1989 pr_cont("\n"); 1990 } 1991 break; 1992 case -EPERM: 1993 FTRACE_WARN_ON_ONCE(1); 1994 pr_info("ftrace faulted on writing "); 1995 print_ip_sym(ip); 1996 break; 1997 default: 1998 FTRACE_WARN_ON_ONCE(1); 1999 pr_info("ftrace faulted on unknown error "); 2000 print_ip_sym(ip); 2001 } 2002 print_bug_type(); 2003 if (rec) { 2004 struct ftrace_ops *ops = NULL; 2005 2006 pr_info("ftrace record flags: %lx\n", rec->flags); 2007 pr_cont(" (%ld)%s", ftrace_rec_count(rec), 2008 rec->flags & FTRACE_FL_REGS ? " R" : " "); 2009 if (rec->flags & FTRACE_FL_TRAMP_EN) { 2010 ops = ftrace_find_tramp_ops_any(rec); 2011 if (ops) { 2012 do { 2013 pr_cont("\ttramp: %pS (%pS)", 2014 (void *)ops->trampoline, 2015 (void *)ops->func); 2016 ops = ftrace_find_tramp_ops_next(rec, ops); 2017 } while (ops); 2018 } else 2019 pr_cont("\ttramp: ERROR!"); 2020 2021 } 2022 ip = ftrace_get_addr_curr(rec); 2023 pr_cont("\n expected tramp: %lx\n", ip); 2024 } 2025 } 2026 2027 static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update) 2028 { 2029 unsigned long flag = 0UL; 2030 2031 ftrace_bug_type = FTRACE_BUG_UNKNOWN; 2032 2033 if (rec->flags & FTRACE_FL_DISABLED) 2034 return FTRACE_UPDATE_IGNORE; 2035 2036 /* 2037 * If we are updating calls: 2038 * 2039 * If the record has a ref count, then we need to enable it 2040 * because someone is using it. 2041 * 2042 * Otherwise we make sure its disabled. 2043 * 2044 * If we are disabling calls, then disable all records that 2045 * are enabled. 2046 */ 2047 if (enable && ftrace_rec_count(rec)) 2048 flag = FTRACE_FL_ENABLED; 2049 2050 /* 2051 * If enabling and the REGS flag does not match the REGS_EN, or 2052 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore 2053 * this record. Set flags to fail the compare against ENABLED. 2054 */ 2055 if (flag) { 2056 if (!(rec->flags & FTRACE_FL_REGS) != 2057 !(rec->flags & FTRACE_FL_REGS_EN)) 2058 flag |= FTRACE_FL_REGS; 2059 2060 if (!(rec->flags & FTRACE_FL_TRAMP) != 2061 !(rec->flags & FTRACE_FL_TRAMP_EN)) 2062 flag |= FTRACE_FL_TRAMP; 2063 } 2064 2065 /* If the state of this record hasn't changed, then do nothing */ 2066 if ((rec->flags & FTRACE_FL_ENABLED) == flag) 2067 return FTRACE_UPDATE_IGNORE; 2068 2069 if (flag) { 2070 /* Save off if rec is being enabled (for return value) */ 2071 flag ^= rec->flags & FTRACE_FL_ENABLED; 2072 2073 if (update) { 2074 rec->flags |= FTRACE_FL_ENABLED; 2075 if (flag & FTRACE_FL_REGS) { 2076 if (rec->flags & FTRACE_FL_REGS) 2077 rec->flags |= FTRACE_FL_REGS_EN; 2078 else 2079 rec->flags &= ~FTRACE_FL_REGS_EN; 2080 } 2081 if (flag & FTRACE_FL_TRAMP) { 2082 if (rec->flags & FTRACE_FL_TRAMP) 2083 rec->flags |= FTRACE_FL_TRAMP_EN; 2084 else 2085 rec->flags &= ~FTRACE_FL_TRAMP_EN; 2086 } 2087 } 2088 2089 /* 2090 * If this record is being updated from a nop, then 2091 * return UPDATE_MAKE_CALL. 2092 * Otherwise, 2093 * return UPDATE_MODIFY_CALL to tell the caller to convert 2094 * from the save regs, to a non-save regs function or 2095 * vice versa, or from a trampoline call. 2096 */ 2097 if (flag & FTRACE_FL_ENABLED) { 2098 ftrace_bug_type = FTRACE_BUG_CALL; 2099 return FTRACE_UPDATE_MAKE_CALL; 2100 } 2101 2102 ftrace_bug_type = FTRACE_BUG_UPDATE; 2103 return FTRACE_UPDATE_MODIFY_CALL; 2104 } 2105 2106 if (update) { 2107 /* If there's no more users, clear all flags */ 2108 if (!ftrace_rec_count(rec)) 2109 rec->flags = 0; 2110 else 2111 /* 2112 * Just disable the record, but keep the ops TRAMP 2113 * and REGS states. The _EN flags must be disabled though. 2114 */ 2115 rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN | 2116 FTRACE_FL_REGS_EN); 2117 } 2118 2119 ftrace_bug_type = FTRACE_BUG_NOP; 2120 return FTRACE_UPDATE_MAKE_NOP; 2121 } 2122 2123 /** 2124 * ftrace_update_record, set a record that now is tracing or not 2125 * @rec: the record to update 2126 * @enable: set to 1 if the record is tracing, zero to force disable 2127 * 2128 * The records that represent all functions that can be traced need 2129 * to be updated when tracing has been enabled. 2130 */ 2131 int ftrace_update_record(struct dyn_ftrace *rec, int enable) 2132 { 2133 return ftrace_check_record(rec, enable, 1); 2134 } 2135 2136 /** 2137 * ftrace_test_record, check if the record has been enabled or not 2138 * @rec: the record to test 2139 * @enable: set to 1 to check if enabled, 0 if it is disabled 2140 * 2141 * The arch code may need to test if a record is already set to 2142 * tracing to determine how to modify the function code that it 2143 * represents. 2144 */ 2145 int ftrace_test_record(struct dyn_ftrace *rec, int enable) 2146 { 2147 return ftrace_check_record(rec, enable, 0); 2148 } 2149 2150 static struct ftrace_ops * 2151 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec) 2152 { 2153 struct ftrace_ops *op; 2154 unsigned long ip = rec->ip; 2155 2156 do_for_each_ftrace_op(op, ftrace_ops_list) { 2157 2158 if (!op->trampoline) 2159 continue; 2160 2161 if (hash_contains_ip(ip, op->func_hash)) 2162 return op; 2163 } while_for_each_ftrace_op(op); 2164 2165 return NULL; 2166 } 2167 2168 static struct ftrace_ops * 2169 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, 2170 struct ftrace_ops *op) 2171 { 2172 unsigned long ip = rec->ip; 2173 2174 while_for_each_ftrace_op(op) { 2175 2176 if (!op->trampoline) 2177 continue; 2178 2179 if (hash_contains_ip(ip, op->func_hash)) 2180 return op; 2181 } 2182 2183 return NULL; 2184 } 2185 2186 static struct ftrace_ops * 2187 ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec) 2188 { 2189 struct ftrace_ops *op; 2190 unsigned long ip = rec->ip; 2191 2192 /* 2193 * Need to check removed ops first. 2194 * If they are being removed, and this rec has a tramp, 2195 * and this rec is in the ops list, then it would be the 2196 * one with the tramp. 2197 */ 2198 if (removed_ops) { 2199 if (hash_contains_ip(ip, &removed_ops->old_hash)) 2200 return removed_ops; 2201 } 2202 2203 /* 2204 * Need to find the current trampoline for a rec. 2205 * Now, a trampoline is only attached to a rec if there 2206 * was a single 'ops' attached to it. But this can be called 2207 * when we are adding another op to the rec or removing the 2208 * current one. Thus, if the op is being added, we can 2209 * ignore it because it hasn't attached itself to the rec 2210 * yet. 2211 * 2212 * If an ops is being modified (hooking to different functions) 2213 * then we don't care about the new functions that are being 2214 * added, just the old ones (that are probably being removed). 2215 * 2216 * If we are adding an ops to a function that already is using 2217 * a trampoline, it needs to be removed (trampolines are only 2218 * for single ops connected), then an ops that is not being 2219 * modified also needs to be checked. 2220 */ 2221 do_for_each_ftrace_op(op, ftrace_ops_list) { 2222 2223 if (!op->trampoline) 2224 continue; 2225 2226 /* 2227 * If the ops is being added, it hasn't gotten to 2228 * the point to be removed from this tree yet. 2229 */ 2230 if (op->flags & FTRACE_OPS_FL_ADDING) 2231 continue; 2232 2233 2234 /* 2235 * If the ops is being modified and is in the old 2236 * hash, then it is probably being removed from this 2237 * function. 2238 */ 2239 if ((op->flags & FTRACE_OPS_FL_MODIFYING) && 2240 hash_contains_ip(ip, &op->old_hash)) 2241 return op; 2242 /* 2243 * If the ops is not being added or modified, and it's 2244 * in its normal filter hash, then this must be the one 2245 * we want! 2246 */ 2247 if (!(op->flags & FTRACE_OPS_FL_MODIFYING) && 2248 hash_contains_ip(ip, op->func_hash)) 2249 return op; 2250 2251 } while_for_each_ftrace_op(op); 2252 2253 return NULL; 2254 } 2255 2256 static struct ftrace_ops * 2257 ftrace_find_tramp_ops_new(struct dyn_ftrace *rec) 2258 { 2259 struct ftrace_ops *op; 2260 unsigned long ip = rec->ip; 2261 2262 do_for_each_ftrace_op(op, ftrace_ops_list) { 2263 /* pass rec in as regs to have non-NULL val */ 2264 if (hash_contains_ip(ip, op->func_hash)) 2265 return op; 2266 } while_for_each_ftrace_op(op); 2267 2268 return NULL; 2269 } 2270 2271 /** 2272 * ftrace_get_addr_new - Get the call address to set to 2273 * @rec: The ftrace record descriptor 2274 * 2275 * If the record has the FTRACE_FL_REGS set, that means that it 2276 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS 2277 * is not not set, then it wants to convert to the normal callback. 2278 * 2279 * Returns the address of the trampoline to set to 2280 */ 2281 unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec) 2282 { 2283 struct ftrace_ops *ops; 2284 2285 /* Trampolines take precedence over regs */ 2286 if (rec->flags & FTRACE_FL_TRAMP) { 2287 ops = ftrace_find_tramp_ops_new(rec); 2288 if (FTRACE_WARN_ON(!ops || !ops->trampoline)) { 2289 pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n", 2290 (void *)rec->ip, (void *)rec->ip, rec->flags); 2291 /* Ftrace is shutting down, return anything */ 2292 return (unsigned long)FTRACE_ADDR; 2293 } 2294 return ops->trampoline; 2295 } 2296 2297 if (rec->flags & FTRACE_FL_REGS) 2298 return (unsigned long)FTRACE_REGS_ADDR; 2299 else 2300 return (unsigned long)FTRACE_ADDR; 2301 } 2302 2303 /** 2304 * ftrace_get_addr_curr - Get the call address that is already there 2305 * @rec: The ftrace record descriptor 2306 * 2307 * The FTRACE_FL_REGS_EN is set when the record already points to 2308 * a function that saves all the regs. Basically the '_EN' version 2309 * represents the current state of the function. 2310 * 2311 * Returns the address of the trampoline that is currently being called 2312 */ 2313 unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec) 2314 { 2315 struct ftrace_ops *ops; 2316 2317 /* Trampolines take precedence over regs */ 2318 if (rec->flags & FTRACE_FL_TRAMP_EN) { 2319 ops = ftrace_find_tramp_ops_curr(rec); 2320 if (FTRACE_WARN_ON(!ops)) { 2321 pr_warn("Bad trampoline accounting at: %p (%pS)\n", 2322 (void *)rec->ip, (void *)rec->ip); 2323 /* Ftrace is shutting down, return anything */ 2324 return (unsigned long)FTRACE_ADDR; 2325 } 2326 return ops->trampoline; 2327 } 2328 2329 if (rec->flags & FTRACE_FL_REGS_EN) 2330 return (unsigned long)FTRACE_REGS_ADDR; 2331 else 2332 return (unsigned long)FTRACE_ADDR; 2333 } 2334 2335 static int 2336 __ftrace_replace_code(struct dyn_ftrace *rec, int enable) 2337 { 2338 unsigned long ftrace_old_addr; 2339 unsigned long ftrace_addr; 2340 int ret; 2341 2342 ftrace_addr = ftrace_get_addr_new(rec); 2343 2344 /* This needs to be done before we call ftrace_update_record */ 2345 ftrace_old_addr = ftrace_get_addr_curr(rec); 2346 2347 ret = ftrace_update_record(rec, enable); 2348 2349 ftrace_bug_type = FTRACE_BUG_UNKNOWN; 2350 2351 switch (ret) { 2352 case FTRACE_UPDATE_IGNORE: 2353 return 0; 2354 2355 case FTRACE_UPDATE_MAKE_CALL: 2356 ftrace_bug_type = FTRACE_BUG_CALL; 2357 return ftrace_make_call(rec, ftrace_addr); 2358 2359 case FTRACE_UPDATE_MAKE_NOP: 2360 ftrace_bug_type = FTRACE_BUG_NOP; 2361 return ftrace_make_nop(NULL, rec, ftrace_old_addr); 2362 2363 case FTRACE_UPDATE_MODIFY_CALL: 2364 ftrace_bug_type = FTRACE_BUG_UPDATE; 2365 return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr); 2366 } 2367 2368 return -1; /* unknow ftrace bug */ 2369 } 2370 2371 void __weak ftrace_replace_code(int enable) 2372 { 2373 struct dyn_ftrace *rec; 2374 struct ftrace_page *pg; 2375 int failed; 2376 2377 if (unlikely(ftrace_disabled)) 2378 return; 2379 2380 do_for_each_ftrace_rec(pg, rec) { 2381 failed = __ftrace_replace_code(rec, enable); 2382 if (failed) { 2383 ftrace_bug(failed, rec); 2384 /* Stop processing */ 2385 return; 2386 } 2387 } while_for_each_ftrace_rec(); 2388 } 2389 2390 struct ftrace_rec_iter { 2391 struct ftrace_page *pg; 2392 int index; 2393 }; 2394 2395 /** 2396 * ftrace_rec_iter_start, start up iterating over traced functions 2397 * 2398 * Returns an iterator handle that is used to iterate over all 2399 * the records that represent address locations where functions 2400 * are traced. 2401 * 2402 * May return NULL if no records are available. 2403 */ 2404 struct ftrace_rec_iter *ftrace_rec_iter_start(void) 2405 { 2406 /* 2407 * We only use a single iterator. 2408 * Protected by the ftrace_lock mutex. 2409 */ 2410 static struct ftrace_rec_iter ftrace_rec_iter; 2411 struct ftrace_rec_iter *iter = &ftrace_rec_iter; 2412 2413 iter->pg = ftrace_pages_start; 2414 iter->index = 0; 2415 2416 /* Could have empty pages */ 2417 while (iter->pg && !iter->pg->index) 2418 iter->pg = iter->pg->next; 2419 2420 if (!iter->pg) 2421 return NULL; 2422 2423 return iter; 2424 } 2425 2426 /** 2427 * ftrace_rec_iter_next, get the next record to process. 2428 * @iter: The handle to the iterator. 2429 * 2430 * Returns the next iterator after the given iterator @iter. 2431 */ 2432 struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter) 2433 { 2434 iter->index++; 2435 2436 if (iter->index >= iter->pg->index) { 2437 iter->pg = iter->pg->next; 2438 iter->index = 0; 2439 2440 /* Could have empty pages */ 2441 while (iter->pg && !iter->pg->index) 2442 iter->pg = iter->pg->next; 2443 } 2444 2445 if (!iter->pg) 2446 return NULL; 2447 2448 return iter; 2449 } 2450 2451 /** 2452 * ftrace_rec_iter_record, get the record at the iterator location 2453 * @iter: The current iterator location 2454 * 2455 * Returns the record that the current @iter is at. 2456 */ 2457 struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter) 2458 { 2459 return &iter->pg->records[iter->index]; 2460 } 2461 2462 static int 2463 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec) 2464 { 2465 int ret; 2466 2467 if (unlikely(ftrace_disabled)) 2468 return 0; 2469 2470 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR); 2471 if (ret) { 2472 ftrace_bug_type = FTRACE_BUG_INIT; 2473 ftrace_bug(ret, rec); 2474 return 0; 2475 } 2476 return 1; 2477 } 2478 2479 /* 2480 * archs can override this function if they must do something 2481 * before the modifying code is performed. 2482 */ 2483 int __weak ftrace_arch_code_modify_prepare(void) 2484 { 2485 return 0; 2486 } 2487 2488 /* 2489 * archs can override this function if they must do something 2490 * after the modifying code is performed. 2491 */ 2492 int __weak ftrace_arch_code_modify_post_process(void) 2493 { 2494 return 0; 2495 } 2496 2497 void ftrace_modify_all_code(int command) 2498 { 2499 int update = command & FTRACE_UPDATE_TRACE_FUNC; 2500 int err = 0; 2501 2502 /* 2503 * If the ftrace_caller calls a ftrace_ops func directly, 2504 * we need to make sure that it only traces functions it 2505 * expects to trace. When doing the switch of functions, 2506 * we need to update to the ftrace_ops_list_func first 2507 * before the transition between old and new calls are set, 2508 * as the ftrace_ops_list_func will check the ops hashes 2509 * to make sure the ops are having the right functions 2510 * traced. 2511 */ 2512 if (update) { 2513 err = ftrace_update_ftrace_func(ftrace_ops_list_func); 2514 if (FTRACE_WARN_ON(err)) 2515 return; 2516 } 2517 2518 if (command & FTRACE_UPDATE_CALLS) 2519 ftrace_replace_code(1); 2520 else if (command & FTRACE_DISABLE_CALLS) 2521 ftrace_replace_code(0); 2522 2523 if (update && ftrace_trace_function != ftrace_ops_list_func) { 2524 function_trace_op = set_function_trace_op; 2525 smp_wmb(); 2526 /* If irqs are disabled, we are in stop machine */ 2527 if (!irqs_disabled()) 2528 smp_call_function(ftrace_sync_ipi, NULL, 1); 2529 err = ftrace_update_ftrace_func(ftrace_trace_function); 2530 if (FTRACE_WARN_ON(err)) 2531 return; 2532 } 2533 2534 if (command & FTRACE_START_FUNC_RET) 2535 err = ftrace_enable_ftrace_graph_caller(); 2536 else if (command & FTRACE_STOP_FUNC_RET) 2537 err = ftrace_disable_ftrace_graph_caller(); 2538 FTRACE_WARN_ON(err); 2539 } 2540 2541 static int __ftrace_modify_code(void *data) 2542 { 2543 int *command = data; 2544 2545 ftrace_modify_all_code(*command); 2546 2547 return 0; 2548 } 2549 2550 /** 2551 * ftrace_run_stop_machine, go back to the stop machine method 2552 * @command: The command to tell ftrace what to do 2553 * 2554 * If an arch needs to fall back to the stop machine method, the 2555 * it can call this function. 2556 */ 2557 void ftrace_run_stop_machine(int command) 2558 { 2559 stop_machine(__ftrace_modify_code, &command, NULL); 2560 } 2561 2562 /** 2563 * arch_ftrace_update_code, modify the code to trace or not trace 2564 * @command: The command that needs to be done 2565 * 2566 * Archs can override this function if it does not need to 2567 * run stop_machine() to modify code. 2568 */ 2569 void __weak arch_ftrace_update_code(int command) 2570 { 2571 ftrace_run_stop_machine(command); 2572 } 2573 2574 static void ftrace_run_update_code(int command) 2575 { 2576 int ret; 2577 2578 ret = ftrace_arch_code_modify_prepare(); 2579 FTRACE_WARN_ON(ret); 2580 if (ret) 2581 return; 2582 2583 /* 2584 * By default we use stop_machine() to modify the code. 2585 * But archs can do what ever they want as long as it 2586 * is safe. The stop_machine() is the safest, but also 2587 * produces the most overhead. 2588 */ 2589 arch_ftrace_update_code(command); 2590 2591 ret = ftrace_arch_code_modify_post_process(); 2592 FTRACE_WARN_ON(ret); 2593 } 2594 2595 static void ftrace_run_modify_code(struct ftrace_ops *ops, int command, 2596 struct ftrace_ops_hash *old_hash) 2597 { 2598 ops->flags |= FTRACE_OPS_FL_MODIFYING; 2599 ops->old_hash.filter_hash = old_hash->filter_hash; 2600 ops->old_hash.notrace_hash = old_hash->notrace_hash; 2601 ftrace_run_update_code(command); 2602 ops->old_hash.filter_hash = NULL; 2603 ops->old_hash.notrace_hash = NULL; 2604 ops->flags &= ~FTRACE_OPS_FL_MODIFYING; 2605 } 2606 2607 static ftrace_func_t saved_ftrace_func; 2608 static int ftrace_start_up; 2609 2610 void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops) 2611 { 2612 } 2613 2614 static void per_cpu_ops_free(struct ftrace_ops *ops) 2615 { 2616 free_percpu(ops->disabled); 2617 } 2618 2619 static void ftrace_startup_enable(int command) 2620 { 2621 if (saved_ftrace_func != ftrace_trace_function) { 2622 saved_ftrace_func = ftrace_trace_function; 2623 command |= FTRACE_UPDATE_TRACE_FUNC; 2624 } 2625 2626 if (!command || !ftrace_enabled) 2627 return; 2628 2629 ftrace_run_update_code(command); 2630 } 2631 2632 static void ftrace_startup_all(int command) 2633 { 2634 update_all_ops = true; 2635 ftrace_startup_enable(command); 2636 update_all_ops = false; 2637 } 2638 2639 static int ftrace_startup(struct ftrace_ops *ops, int command) 2640 { 2641 int ret; 2642 2643 if (unlikely(ftrace_disabled)) 2644 return -ENODEV; 2645 2646 ret = __register_ftrace_function(ops); 2647 if (ret) 2648 return ret; 2649 2650 ftrace_start_up++; 2651 2652 /* 2653 * Note that ftrace probes uses this to start up 2654 * and modify functions it will probe. But we still 2655 * set the ADDING flag for modification, as probes 2656 * do not have trampolines. If they add them in the 2657 * future, then the probes will need to distinguish 2658 * between adding and updating probes. 2659 */ 2660 ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING; 2661 2662 ret = ftrace_hash_ipmodify_enable(ops); 2663 if (ret < 0) { 2664 /* Rollback registration process */ 2665 __unregister_ftrace_function(ops); 2666 ftrace_start_up--; 2667 ops->flags &= ~FTRACE_OPS_FL_ENABLED; 2668 return ret; 2669 } 2670 2671 if (ftrace_hash_rec_enable(ops, 1)) 2672 command |= FTRACE_UPDATE_CALLS; 2673 2674 ftrace_startup_enable(command); 2675 2676 ops->flags &= ~FTRACE_OPS_FL_ADDING; 2677 2678 return 0; 2679 } 2680 2681 static int ftrace_shutdown(struct ftrace_ops *ops, int command) 2682 { 2683 int ret; 2684 2685 if (unlikely(ftrace_disabled)) 2686 return -ENODEV; 2687 2688 ret = __unregister_ftrace_function(ops); 2689 if (ret) 2690 return ret; 2691 2692 ftrace_start_up--; 2693 /* 2694 * Just warn in case of unbalance, no need to kill ftrace, it's not 2695 * critical but the ftrace_call callers may be never nopped again after 2696 * further ftrace uses. 2697 */ 2698 WARN_ON_ONCE(ftrace_start_up < 0); 2699 2700 /* Disabling ipmodify never fails */ 2701 ftrace_hash_ipmodify_disable(ops); 2702 2703 if (ftrace_hash_rec_disable(ops, 1)) 2704 command |= FTRACE_UPDATE_CALLS; 2705 2706 ops->flags &= ~FTRACE_OPS_FL_ENABLED; 2707 2708 if (saved_ftrace_func != ftrace_trace_function) { 2709 saved_ftrace_func = ftrace_trace_function; 2710 command |= FTRACE_UPDATE_TRACE_FUNC; 2711 } 2712 2713 if (!command || !ftrace_enabled) { 2714 /* 2715 * If these are per_cpu ops, they still need their 2716 * per_cpu field freed. Since, function tracing is 2717 * not currently active, we can just free them 2718 * without synchronizing all CPUs. 2719 */ 2720 if (ops->flags & FTRACE_OPS_FL_PER_CPU) 2721 per_cpu_ops_free(ops); 2722 return 0; 2723 } 2724 2725 /* 2726 * If the ops uses a trampoline, then it needs to be 2727 * tested first on update. 2728 */ 2729 ops->flags |= FTRACE_OPS_FL_REMOVING; 2730 removed_ops = ops; 2731 2732 /* The trampoline logic checks the old hashes */ 2733 ops->old_hash.filter_hash = ops->func_hash->filter_hash; 2734 ops->old_hash.notrace_hash = ops->func_hash->notrace_hash; 2735 2736 ftrace_run_update_code(command); 2737 2738 /* 2739 * If there's no more ops registered with ftrace, run a 2740 * sanity check to make sure all rec flags are cleared. 2741 */ 2742 if (ftrace_ops_list == &ftrace_list_end) { 2743 struct ftrace_page *pg; 2744 struct dyn_ftrace *rec; 2745 2746 do_for_each_ftrace_rec(pg, rec) { 2747 if (FTRACE_WARN_ON_ONCE(rec->flags)) 2748 pr_warn(" %pS flags:%lx\n", 2749 (void *)rec->ip, rec->flags); 2750 } while_for_each_ftrace_rec(); 2751 } 2752 2753 ops->old_hash.filter_hash = NULL; 2754 ops->old_hash.notrace_hash = NULL; 2755 2756 removed_ops = NULL; 2757 ops->flags &= ~FTRACE_OPS_FL_REMOVING; 2758 2759 /* 2760 * Dynamic ops may be freed, we must make sure that all 2761 * callers are done before leaving this function. 2762 * The same goes for freeing the per_cpu data of the per_cpu 2763 * ops. 2764 * 2765 * Again, normal synchronize_sched() is not good enough. 2766 * We need to do a hard force of sched synchronization. 2767 * This is because we use preempt_disable() to do RCU, but 2768 * the function tracers can be called where RCU is not watching 2769 * (like before user_exit()). We can not rely on the RCU 2770 * infrastructure to do the synchronization, thus we must do it 2771 * ourselves. 2772 */ 2773 if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU)) { 2774 schedule_on_each_cpu(ftrace_sync); 2775 2776 arch_ftrace_trampoline_free(ops); 2777 2778 if (ops->flags & FTRACE_OPS_FL_PER_CPU) 2779 per_cpu_ops_free(ops); 2780 } 2781 2782 return 0; 2783 } 2784 2785 static void ftrace_startup_sysctl(void) 2786 { 2787 int command; 2788 2789 if (unlikely(ftrace_disabled)) 2790 return; 2791 2792 /* Force update next time */ 2793 saved_ftrace_func = NULL; 2794 /* ftrace_start_up is true if we want ftrace running */ 2795 if (ftrace_start_up) { 2796 command = FTRACE_UPDATE_CALLS; 2797 if (ftrace_graph_active) 2798 command |= FTRACE_START_FUNC_RET; 2799 ftrace_startup_enable(command); 2800 } 2801 } 2802 2803 static void ftrace_shutdown_sysctl(void) 2804 { 2805 int command; 2806 2807 if (unlikely(ftrace_disabled)) 2808 return; 2809 2810 /* ftrace_start_up is true if ftrace is running */ 2811 if (ftrace_start_up) { 2812 command = FTRACE_DISABLE_CALLS; 2813 if (ftrace_graph_active) 2814 command |= FTRACE_STOP_FUNC_RET; 2815 ftrace_run_update_code(command); 2816 } 2817 } 2818 2819 static cycle_t ftrace_update_time; 2820 unsigned long ftrace_update_tot_cnt; 2821 2822 static inline int ops_traces_mod(struct ftrace_ops *ops) 2823 { 2824 /* 2825 * Filter_hash being empty will default to trace module. 2826 * But notrace hash requires a test of individual module functions. 2827 */ 2828 return ftrace_hash_empty(ops->func_hash->filter_hash) && 2829 ftrace_hash_empty(ops->func_hash->notrace_hash); 2830 } 2831 2832 /* 2833 * Check if the current ops references the record. 2834 * 2835 * If the ops traces all functions, then it was already accounted for. 2836 * If the ops does not trace the current record function, skip it. 2837 * If the ops ignores the function via notrace filter, skip it. 2838 */ 2839 static inline bool 2840 ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec) 2841 { 2842 /* If ops isn't enabled, ignore it */ 2843 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 2844 return 0; 2845 2846 /* If ops traces all then it includes this function */ 2847 if (ops_traces_mod(ops)) 2848 return 1; 2849 2850 /* The function must be in the filter */ 2851 if (!ftrace_hash_empty(ops->func_hash->filter_hash) && 2852 !ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip)) 2853 return 0; 2854 2855 /* If in notrace hash, we ignore it too */ 2856 if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) 2857 return 0; 2858 2859 return 1; 2860 } 2861 2862 static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs) 2863 { 2864 struct ftrace_page *pg; 2865 struct dyn_ftrace *p; 2866 cycle_t start, stop; 2867 unsigned long update_cnt = 0; 2868 unsigned long rec_flags = 0; 2869 int i; 2870 2871 start = ftrace_now(raw_smp_processor_id()); 2872 2873 /* 2874 * When a module is loaded, this function is called to convert 2875 * the calls to mcount in its text to nops, and also to create 2876 * an entry in the ftrace data. Now, if ftrace is activated 2877 * after this call, but before the module sets its text to 2878 * read-only, the modification of enabling ftrace can fail if 2879 * the read-only is done while ftrace is converting the calls. 2880 * To prevent this, the module's records are set as disabled 2881 * and will be enabled after the call to set the module's text 2882 * to read-only. 2883 */ 2884 if (mod) 2885 rec_flags |= FTRACE_FL_DISABLED; 2886 2887 for (pg = new_pgs; pg; pg = pg->next) { 2888 2889 for (i = 0; i < pg->index; i++) { 2890 2891 /* If something went wrong, bail without enabling anything */ 2892 if (unlikely(ftrace_disabled)) 2893 return -1; 2894 2895 p = &pg->records[i]; 2896 p->flags = rec_flags; 2897 2898 /* 2899 * Do the initial record conversion from mcount jump 2900 * to the NOP instructions. 2901 */ 2902 if (!ftrace_code_disable(mod, p)) 2903 break; 2904 2905 update_cnt++; 2906 } 2907 } 2908 2909 stop = ftrace_now(raw_smp_processor_id()); 2910 ftrace_update_time = stop - start; 2911 ftrace_update_tot_cnt += update_cnt; 2912 2913 return 0; 2914 } 2915 2916 static int ftrace_allocate_records(struct ftrace_page *pg, int count) 2917 { 2918 int order; 2919 int cnt; 2920 2921 if (WARN_ON(!count)) 2922 return -EINVAL; 2923 2924 order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE)); 2925 2926 /* 2927 * We want to fill as much as possible. No more than a page 2928 * may be empty. 2929 */ 2930 while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE) 2931 order--; 2932 2933 again: 2934 pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); 2935 2936 if (!pg->records) { 2937 /* if we can't allocate this size, try something smaller */ 2938 if (!order) 2939 return -ENOMEM; 2940 order >>= 1; 2941 goto again; 2942 } 2943 2944 cnt = (PAGE_SIZE << order) / ENTRY_SIZE; 2945 pg->size = cnt; 2946 2947 if (cnt > count) 2948 cnt = count; 2949 2950 return cnt; 2951 } 2952 2953 static struct ftrace_page * 2954 ftrace_allocate_pages(unsigned long num_to_init) 2955 { 2956 struct ftrace_page *start_pg; 2957 struct ftrace_page *pg; 2958 int order; 2959 int cnt; 2960 2961 if (!num_to_init) 2962 return 0; 2963 2964 start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL); 2965 if (!pg) 2966 return NULL; 2967 2968 /* 2969 * Try to allocate as much as possible in one continues 2970 * location that fills in all of the space. We want to 2971 * waste as little space as possible. 2972 */ 2973 for (;;) { 2974 cnt = ftrace_allocate_records(pg, num_to_init); 2975 if (cnt < 0) 2976 goto free_pages; 2977 2978 num_to_init -= cnt; 2979 if (!num_to_init) 2980 break; 2981 2982 pg->next = kzalloc(sizeof(*pg), GFP_KERNEL); 2983 if (!pg->next) 2984 goto free_pages; 2985 2986 pg = pg->next; 2987 } 2988 2989 return start_pg; 2990 2991 free_pages: 2992 pg = start_pg; 2993 while (pg) { 2994 order = get_count_order(pg->size / ENTRIES_PER_PAGE); 2995 free_pages((unsigned long)pg->records, order); 2996 start_pg = pg->next; 2997 kfree(pg); 2998 pg = start_pg; 2999 } 3000 pr_info("ftrace: FAILED to allocate memory for functions\n"); 3001 return NULL; 3002 } 3003 3004 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ 3005 3006 struct ftrace_iterator { 3007 loff_t pos; 3008 loff_t func_pos; 3009 struct ftrace_page *pg; 3010 struct dyn_ftrace *func; 3011 struct ftrace_func_probe *probe; 3012 struct trace_parser parser; 3013 struct ftrace_hash *hash; 3014 struct ftrace_ops *ops; 3015 int hidx; 3016 int idx; 3017 unsigned flags; 3018 }; 3019 3020 static void * 3021 t_hash_next(struct seq_file *m, loff_t *pos) 3022 { 3023 struct ftrace_iterator *iter = m->private; 3024 struct hlist_node *hnd = NULL; 3025 struct hlist_head *hhd; 3026 3027 (*pos)++; 3028 iter->pos = *pos; 3029 3030 if (iter->probe) 3031 hnd = &iter->probe->node; 3032 retry: 3033 if (iter->hidx >= FTRACE_FUNC_HASHSIZE) 3034 return NULL; 3035 3036 hhd = &ftrace_func_hash[iter->hidx]; 3037 3038 if (hlist_empty(hhd)) { 3039 iter->hidx++; 3040 hnd = NULL; 3041 goto retry; 3042 } 3043 3044 if (!hnd) 3045 hnd = hhd->first; 3046 else { 3047 hnd = hnd->next; 3048 if (!hnd) { 3049 iter->hidx++; 3050 goto retry; 3051 } 3052 } 3053 3054 if (WARN_ON_ONCE(!hnd)) 3055 return NULL; 3056 3057 iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node); 3058 3059 return iter; 3060 } 3061 3062 static void *t_hash_start(struct seq_file *m, loff_t *pos) 3063 { 3064 struct ftrace_iterator *iter = m->private; 3065 void *p = NULL; 3066 loff_t l; 3067 3068 if (!(iter->flags & FTRACE_ITER_DO_HASH)) 3069 return NULL; 3070 3071 if (iter->func_pos > *pos) 3072 return NULL; 3073 3074 iter->hidx = 0; 3075 for (l = 0; l <= (*pos - iter->func_pos); ) { 3076 p = t_hash_next(m, &l); 3077 if (!p) 3078 break; 3079 } 3080 if (!p) 3081 return NULL; 3082 3083 /* Only set this if we have an item */ 3084 iter->flags |= FTRACE_ITER_HASH; 3085 3086 return iter; 3087 } 3088 3089 static int 3090 t_hash_show(struct seq_file *m, struct ftrace_iterator *iter) 3091 { 3092 struct ftrace_func_probe *rec; 3093 3094 rec = iter->probe; 3095 if (WARN_ON_ONCE(!rec)) 3096 return -EIO; 3097 3098 if (rec->ops->print) 3099 return rec->ops->print(m, rec->ip, rec->ops, rec->data); 3100 3101 seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func); 3102 3103 if (rec->data) 3104 seq_printf(m, ":%p", rec->data); 3105 seq_putc(m, '\n'); 3106 3107 return 0; 3108 } 3109 3110 static void * 3111 t_next(struct seq_file *m, void *v, loff_t *pos) 3112 { 3113 struct ftrace_iterator *iter = m->private; 3114 struct ftrace_ops *ops = iter->ops; 3115 struct dyn_ftrace *rec = NULL; 3116 3117 if (unlikely(ftrace_disabled)) 3118 return NULL; 3119 3120 if (iter->flags & FTRACE_ITER_HASH) 3121 return t_hash_next(m, pos); 3122 3123 (*pos)++; 3124 iter->pos = iter->func_pos = *pos; 3125 3126 if (iter->flags & FTRACE_ITER_PRINTALL) 3127 return t_hash_start(m, pos); 3128 3129 retry: 3130 if (iter->idx >= iter->pg->index) { 3131 if (iter->pg->next) { 3132 iter->pg = iter->pg->next; 3133 iter->idx = 0; 3134 goto retry; 3135 } 3136 } else { 3137 rec = &iter->pg->records[iter->idx++]; 3138 if (((iter->flags & FTRACE_ITER_FILTER) && 3139 !(ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))) || 3140 3141 ((iter->flags & FTRACE_ITER_NOTRACE) && 3142 !ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) || 3143 3144 ((iter->flags & FTRACE_ITER_ENABLED) && 3145 !(rec->flags & FTRACE_FL_ENABLED))) { 3146 3147 rec = NULL; 3148 goto retry; 3149 } 3150 } 3151 3152 if (!rec) 3153 return t_hash_start(m, pos); 3154 3155 iter->func = rec; 3156 3157 return iter; 3158 } 3159 3160 static void reset_iter_read(struct ftrace_iterator *iter) 3161 { 3162 iter->pos = 0; 3163 iter->func_pos = 0; 3164 iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH); 3165 } 3166 3167 static void *t_start(struct seq_file *m, loff_t *pos) 3168 { 3169 struct ftrace_iterator *iter = m->private; 3170 struct ftrace_ops *ops = iter->ops; 3171 void *p = NULL; 3172 loff_t l; 3173 3174 mutex_lock(&ftrace_lock); 3175 3176 if (unlikely(ftrace_disabled)) 3177 return NULL; 3178 3179 /* 3180 * If an lseek was done, then reset and start from beginning. 3181 */ 3182 if (*pos < iter->pos) 3183 reset_iter_read(iter); 3184 3185 /* 3186 * For set_ftrace_filter reading, if we have the filter 3187 * off, we can short cut and just print out that all 3188 * functions are enabled. 3189 */ 3190 if ((iter->flags & FTRACE_ITER_FILTER && 3191 ftrace_hash_empty(ops->func_hash->filter_hash)) || 3192 (iter->flags & FTRACE_ITER_NOTRACE && 3193 ftrace_hash_empty(ops->func_hash->notrace_hash))) { 3194 if (*pos > 0) 3195 return t_hash_start(m, pos); 3196 iter->flags |= FTRACE_ITER_PRINTALL; 3197 /* reset in case of seek/pread */ 3198 iter->flags &= ~FTRACE_ITER_HASH; 3199 return iter; 3200 } 3201 3202 if (iter->flags & FTRACE_ITER_HASH) 3203 return t_hash_start(m, pos); 3204 3205 /* 3206 * Unfortunately, we need to restart at ftrace_pages_start 3207 * every time we let go of the ftrace_mutex. This is because 3208 * those pointers can change without the lock. 3209 */ 3210 iter->pg = ftrace_pages_start; 3211 iter->idx = 0; 3212 for (l = 0; l <= *pos; ) { 3213 p = t_next(m, p, &l); 3214 if (!p) 3215 break; 3216 } 3217 3218 if (!p) 3219 return t_hash_start(m, pos); 3220 3221 return iter; 3222 } 3223 3224 static void t_stop(struct seq_file *m, void *p) 3225 { 3226 mutex_unlock(&ftrace_lock); 3227 } 3228 3229 void * __weak 3230 arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec) 3231 { 3232 return NULL; 3233 } 3234 3235 static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops, 3236 struct dyn_ftrace *rec) 3237 { 3238 void *ptr; 3239 3240 ptr = arch_ftrace_trampoline_func(ops, rec); 3241 if (ptr) 3242 seq_printf(m, " ->%pS", ptr); 3243 } 3244 3245 static int t_show(struct seq_file *m, void *v) 3246 { 3247 struct ftrace_iterator *iter = m->private; 3248 struct dyn_ftrace *rec; 3249 3250 if (iter->flags & FTRACE_ITER_HASH) 3251 return t_hash_show(m, iter); 3252 3253 if (iter->flags & FTRACE_ITER_PRINTALL) { 3254 if (iter->flags & FTRACE_ITER_NOTRACE) 3255 seq_puts(m, "#### no functions disabled ####\n"); 3256 else 3257 seq_puts(m, "#### all functions enabled ####\n"); 3258 return 0; 3259 } 3260 3261 rec = iter->func; 3262 3263 if (!rec) 3264 return 0; 3265 3266 seq_printf(m, "%ps", (void *)rec->ip); 3267 if (iter->flags & FTRACE_ITER_ENABLED) { 3268 struct ftrace_ops *ops; 3269 3270 seq_printf(m, " (%ld)%s%s", 3271 ftrace_rec_count(rec), 3272 rec->flags & FTRACE_FL_REGS ? " R" : " ", 3273 rec->flags & FTRACE_FL_IPMODIFY ? " I" : " "); 3274 if (rec->flags & FTRACE_FL_TRAMP_EN) { 3275 ops = ftrace_find_tramp_ops_any(rec); 3276 if (ops) { 3277 do { 3278 seq_printf(m, "\ttramp: %pS (%pS)", 3279 (void *)ops->trampoline, 3280 (void *)ops->func); 3281 add_trampoline_func(m, ops, rec); 3282 ops = ftrace_find_tramp_ops_next(rec, ops); 3283 } while (ops); 3284 } else 3285 seq_puts(m, "\ttramp: ERROR!"); 3286 } else { 3287 add_trampoline_func(m, NULL, rec); 3288 } 3289 } 3290 3291 seq_putc(m, '\n'); 3292 3293 return 0; 3294 } 3295 3296 static const struct seq_operations show_ftrace_seq_ops = { 3297 .start = t_start, 3298 .next = t_next, 3299 .stop = t_stop, 3300 .show = t_show, 3301 }; 3302 3303 static int 3304 ftrace_avail_open(struct inode *inode, struct file *file) 3305 { 3306 struct ftrace_iterator *iter; 3307 3308 if (unlikely(ftrace_disabled)) 3309 return -ENODEV; 3310 3311 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); 3312 if (iter) { 3313 iter->pg = ftrace_pages_start; 3314 iter->ops = &global_ops; 3315 } 3316 3317 return iter ? 0 : -ENOMEM; 3318 } 3319 3320 static int 3321 ftrace_enabled_open(struct inode *inode, struct file *file) 3322 { 3323 struct ftrace_iterator *iter; 3324 3325 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); 3326 if (iter) { 3327 iter->pg = ftrace_pages_start; 3328 iter->flags = FTRACE_ITER_ENABLED; 3329 iter->ops = &global_ops; 3330 } 3331 3332 return iter ? 0 : -ENOMEM; 3333 } 3334 3335 /** 3336 * ftrace_regex_open - initialize function tracer filter files 3337 * @ops: The ftrace_ops that hold the hash filters 3338 * @flag: The type of filter to process 3339 * @inode: The inode, usually passed in to your open routine 3340 * @file: The file, usually passed in to your open routine 3341 * 3342 * ftrace_regex_open() initializes the filter files for the 3343 * @ops. Depending on @flag it may process the filter hash or 3344 * the notrace hash of @ops. With this called from the open 3345 * routine, you can use ftrace_filter_write() for the write 3346 * routine if @flag has FTRACE_ITER_FILTER set, or 3347 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set. 3348 * tracing_lseek() should be used as the lseek routine, and 3349 * release must call ftrace_regex_release(). 3350 */ 3351 int 3352 ftrace_regex_open(struct ftrace_ops *ops, int flag, 3353 struct inode *inode, struct file *file) 3354 { 3355 struct ftrace_iterator *iter; 3356 struct ftrace_hash *hash; 3357 int ret = 0; 3358 3359 ftrace_ops_init(ops); 3360 3361 if (unlikely(ftrace_disabled)) 3362 return -ENODEV; 3363 3364 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 3365 if (!iter) 3366 return -ENOMEM; 3367 3368 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) { 3369 kfree(iter); 3370 return -ENOMEM; 3371 } 3372 3373 iter->ops = ops; 3374 iter->flags = flag; 3375 3376 mutex_lock(&ops->func_hash->regex_lock); 3377 3378 if (flag & FTRACE_ITER_NOTRACE) 3379 hash = ops->func_hash->notrace_hash; 3380 else 3381 hash = ops->func_hash->filter_hash; 3382 3383 if (file->f_mode & FMODE_WRITE) { 3384 const int size_bits = FTRACE_HASH_DEFAULT_BITS; 3385 3386 if (file->f_flags & O_TRUNC) 3387 iter->hash = alloc_ftrace_hash(size_bits); 3388 else 3389 iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash); 3390 3391 if (!iter->hash) { 3392 trace_parser_put(&iter->parser); 3393 kfree(iter); 3394 ret = -ENOMEM; 3395 goto out_unlock; 3396 } 3397 } 3398 3399 if (file->f_mode & FMODE_READ) { 3400 iter->pg = ftrace_pages_start; 3401 3402 ret = seq_open(file, &show_ftrace_seq_ops); 3403 if (!ret) { 3404 struct seq_file *m = file->private_data; 3405 m->private = iter; 3406 } else { 3407 /* Failed */ 3408 free_ftrace_hash(iter->hash); 3409 trace_parser_put(&iter->parser); 3410 kfree(iter); 3411 } 3412 } else 3413 file->private_data = iter; 3414 3415 out_unlock: 3416 mutex_unlock(&ops->func_hash->regex_lock); 3417 3418 return ret; 3419 } 3420 3421 static int 3422 ftrace_filter_open(struct inode *inode, struct file *file) 3423 { 3424 struct ftrace_ops *ops = inode->i_private; 3425 3426 return ftrace_regex_open(ops, 3427 FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH, 3428 inode, file); 3429 } 3430 3431 static int 3432 ftrace_notrace_open(struct inode *inode, struct file *file) 3433 { 3434 struct ftrace_ops *ops = inode->i_private; 3435 3436 return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE, 3437 inode, file); 3438 } 3439 3440 /* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */ 3441 struct ftrace_glob { 3442 char *search; 3443 unsigned len; 3444 int type; 3445 }; 3446 3447 static int ftrace_match(char *str, struct ftrace_glob *g) 3448 { 3449 int matched = 0; 3450 int slen; 3451 3452 switch (g->type) { 3453 case MATCH_FULL: 3454 if (strcmp(str, g->search) == 0) 3455 matched = 1; 3456 break; 3457 case MATCH_FRONT_ONLY: 3458 if (strncmp(str, g->search, g->len) == 0) 3459 matched = 1; 3460 break; 3461 case MATCH_MIDDLE_ONLY: 3462 if (strstr(str, g->search)) 3463 matched = 1; 3464 break; 3465 case MATCH_END_ONLY: 3466 slen = strlen(str); 3467 if (slen >= g->len && 3468 memcmp(str + slen - g->len, g->search, g->len) == 0) 3469 matched = 1; 3470 break; 3471 } 3472 3473 return matched; 3474 } 3475 3476 static int 3477 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter) 3478 { 3479 struct ftrace_func_entry *entry; 3480 int ret = 0; 3481 3482 entry = ftrace_lookup_ip(hash, rec->ip); 3483 if (clear_filter) { 3484 /* Do nothing if it doesn't exist */ 3485 if (!entry) 3486 return 0; 3487 3488 free_hash_entry(hash, entry); 3489 } else { 3490 /* Do nothing if it exists */ 3491 if (entry) 3492 return 0; 3493 3494 ret = add_hash_entry(hash, rec->ip); 3495 } 3496 return ret; 3497 } 3498 3499 static int 3500 ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g, 3501 struct ftrace_glob *mod_g, int exclude_mod) 3502 { 3503 char str[KSYM_SYMBOL_LEN]; 3504 char *modname; 3505 3506 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); 3507 3508 if (mod_g) { 3509 int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0; 3510 3511 /* blank module name to match all modules */ 3512 if (!mod_g->len) { 3513 /* blank module globbing: modname xor exclude_mod */ 3514 if ((!exclude_mod) != (!modname)) 3515 goto func_match; 3516 return 0; 3517 } 3518 3519 /* not matching the module */ 3520 if (!modname || !mod_matches) { 3521 if (exclude_mod) 3522 goto func_match; 3523 else 3524 return 0; 3525 } 3526 3527 if (mod_matches && exclude_mod) 3528 return 0; 3529 3530 func_match: 3531 /* blank search means to match all funcs in the mod */ 3532 if (!func_g->len) 3533 return 1; 3534 } 3535 3536 return ftrace_match(str, func_g); 3537 } 3538 3539 static int 3540 match_records(struct ftrace_hash *hash, char *func, int len, char *mod) 3541 { 3542 struct ftrace_page *pg; 3543 struct dyn_ftrace *rec; 3544 struct ftrace_glob func_g = { .type = MATCH_FULL }; 3545 struct ftrace_glob mod_g = { .type = MATCH_FULL }; 3546 struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL; 3547 int exclude_mod = 0; 3548 int found = 0; 3549 int ret; 3550 int clear_filter; 3551 3552 if (func) { 3553 func_g.type = filter_parse_regex(func, len, &func_g.search, 3554 &clear_filter); 3555 func_g.len = strlen(func_g.search); 3556 } 3557 3558 if (mod) { 3559 mod_g.type = filter_parse_regex(mod, strlen(mod), 3560 &mod_g.search, &exclude_mod); 3561 mod_g.len = strlen(mod_g.search); 3562 } 3563 3564 mutex_lock(&ftrace_lock); 3565 3566 if (unlikely(ftrace_disabled)) 3567 goto out_unlock; 3568 3569 do_for_each_ftrace_rec(pg, rec) { 3570 if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) { 3571 ret = enter_record(hash, rec, clear_filter); 3572 if (ret < 0) { 3573 found = ret; 3574 goto out_unlock; 3575 } 3576 found = 1; 3577 } 3578 } while_for_each_ftrace_rec(); 3579 out_unlock: 3580 mutex_unlock(&ftrace_lock); 3581 3582 return found; 3583 } 3584 3585 static int 3586 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len) 3587 { 3588 return match_records(hash, buff, len, NULL); 3589 } 3590 3591 3592 /* 3593 * We register the module command as a template to show others how 3594 * to register the a command as well. 3595 */ 3596 3597 static int 3598 ftrace_mod_callback(struct ftrace_hash *hash, 3599 char *func, char *cmd, char *module, int enable) 3600 { 3601 int ret; 3602 3603 /* 3604 * cmd == 'mod' because we only registered this func 3605 * for the 'mod' ftrace_func_command. 3606 * But if you register one func with multiple commands, 3607 * you can tell which command was used by the cmd 3608 * parameter. 3609 */ 3610 ret = match_records(hash, func, strlen(func), module); 3611 if (!ret) 3612 return -EINVAL; 3613 if (ret < 0) 3614 return ret; 3615 return 0; 3616 } 3617 3618 static struct ftrace_func_command ftrace_mod_cmd = { 3619 .name = "mod", 3620 .func = ftrace_mod_callback, 3621 }; 3622 3623 static int __init ftrace_mod_cmd_init(void) 3624 { 3625 return register_ftrace_command(&ftrace_mod_cmd); 3626 } 3627 core_initcall(ftrace_mod_cmd_init); 3628 3629 static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip, 3630 struct ftrace_ops *op, struct pt_regs *pt_regs) 3631 { 3632 struct ftrace_func_probe *entry; 3633 struct hlist_head *hhd; 3634 unsigned long key; 3635 3636 key = hash_long(ip, FTRACE_HASH_BITS); 3637 3638 hhd = &ftrace_func_hash[key]; 3639 3640 if (hlist_empty(hhd)) 3641 return; 3642 3643 /* 3644 * Disable preemption for these calls to prevent a RCU grace 3645 * period. This syncs the hash iteration and freeing of items 3646 * on the hash. rcu_read_lock is too dangerous here. 3647 */ 3648 preempt_disable_notrace(); 3649 hlist_for_each_entry_rcu_notrace(entry, hhd, node) { 3650 if (entry->ip == ip) 3651 entry->ops->func(ip, parent_ip, &entry->data); 3652 } 3653 preempt_enable_notrace(); 3654 } 3655 3656 static struct ftrace_ops trace_probe_ops __read_mostly = 3657 { 3658 .func = function_trace_probe_call, 3659 .flags = FTRACE_OPS_FL_INITIALIZED, 3660 INIT_OPS_HASH(trace_probe_ops) 3661 }; 3662 3663 static int ftrace_probe_registered; 3664 3665 static void __enable_ftrace_function_probe(struct ftrace_ops_hash *old_hash) 3666 { 3667 int ret; 3668 int i; 3669 3670 if (ftrace_probe_registered) { 3671 /* still need to update the function call sites */ 3672 if (ftrace_enabled) 3673 ftrace_run_modify_code(&trace_probe_ops, FTRACE_UPDATE_CALLS, 3674 old_hash); 3675 return; 3676 } 3677 3678 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 3679 struct hlist_head *hhd = &ftrace_func_hash[i]; 3680 if (hhd->first) 3681 break; 3682 } 3683 /* Nothing registered? */ 3684 if (i == FTRACE_FUNC_HASHSIZE) 3685 return; 3686 3687 ret = ftrace_startup(&trace_probe_ops, 0); 3688 3689 ftrace_probe_registered = 1; 3690 } 3691 3692 static void __disable_ftrace_function_probe(void) 3693 { 3694 int i; 3695 3696 if (!ftrace_probe_registered) 3697 return; 3698 3699 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 3700 struct hlist_head *hhd = &ftrace_func_hash[i]; 3701 if (hhd->first) 3702 return; 3703 } 3704 3705 /* no more funcs left */ 3706 ftrace_shutdown(&trace_probe_ops, 0); 3707 3708 ftrace_probe_registered = 0; 3709 } 3710 3711 3712 static void ftrace_free_entry(struct ftrace_func_probe *entry) 3713 { 3714 if (entry->ops->free) 3715 entry->ops->free(entry->ops, entry->ip, &entry->data); 3716 kfree(entry); 3717 } 3718 3719 int 3720 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 3721 void *data) 3722 { 3723 struct ftrace_ops_hash old_hash_ops; 3724 struct ftrace_func_probe *entry; 3725 struct ftrace_glob func_g; 3726 struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash; 3727 struct ftrace_hash *old_hash = *orig_hash; 3728 struct ftrace_hash *hash; 3729 struct ftrace_page *pg; 3730 struct dyn_ftrace *rec; 3731 int not; 3732 unsigned long key; 3733 int count = 0; 3734 int ret; 3735 3736 func_g.type = filter_parse_regex(glob, strlen(glob), 3737 &func_g.search, ¬); 3738 func_g.len = strlen(func_g.search); 3739 3740 /* we do not support '!' for function probes */ 3741 if (WARN_ON(not)) 3742 return -EINVAL; 3743 3744 mutex_lock(&trace_probe_ops.func_hash->regex_lock); 3745 3746 old_hash_ops.filter_hash = old_hash; 3747 /* Probes only have filters */ 3748 old_hash_ops.notrace_hash = NULL; 3749 3750 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash); 3751 if (!hash) { 3752 count = -ENOMEM; 3753 goto out; 3754 } 3755 3756 if (unlikely(ftrace_disabled)) { 3757 count = -ENODEV; 3758 goto out; 3759 } 3760 3761 mutex_lock(&ftrace_lock); 3762 3763 do_for_each_ftrace_rec(pg, rec) { 3764 3765 if (!ftrace_match_record(rec, &func_g, NULL, 0)) 3766 continue; 3767 3768 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 3769 if (!entry) { 3770 /* If we did not process any, then return error */ 3771 if (!count) 3772 count = -ENOMEM; 3773 goto out_unlock; 3774 } 3775 3776 count++; 3777 3778 entry->data = data; 3779 3780 /* 3781 * The caller might want to do something special 3782 * for each function we find. We call the callback 3783 * to give the caller an opportunity to do so. 3784 */ 3785 if (ops->init) { 3786 if (ops->init(ops, rec->ip, &entry->data) < 0) { 3787 /* caller does not like this func */ 3788 kfree(entry); 3789 continue; 3790 } 3791 } 3792 3793 ret = enter_record(hash, rec, 0); 3794 if (ret < 0) { 3795 kfree(entry); 3796 count = ret; 3797 goto out_unlock; 3798 } 3799 3800 entry->ops = ops; 3801 entry->ip = rec->ip; 3802 3803 key = hash_long(entry->ip, FTRACE_HASH_BITS); 3804 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]); 3805 3806 } while_for_each_ftrace_rec(); 3807 3808 ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash); 3809 3810 __enable_ftrace_function_probe(&old_hash_ops); 3811 3812 if (!ret) 3813 free_ftrace_hash_rcu(old_hash); 3814 else 3815 count = ret; 3816 3817 out_unlock: 3818 mutex_unlock(&ftrace_lock); 3819 out: 3820 mutex_unlock(&trace_probe_ops.func_hash->regex_lock); 3821 free_ftrace_hash(hash); 3822 3823 return count; 3824 } 3825 3826 enum { 3827 PROBE_TEST_FUNC = 1, 3828 PROBE_TEST_DATA = 2 3829 }; 3830 3831 static void 3832 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 3833 void *data, int flags) 3834 { 3835 struct ftrace_func_entry *rec_entry; 3836 struct ftrace_func_probe *entry; 3837 struct ftrace_func_probe *p; 3838 struct ftrace_glob func_g; 3839 struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash; 3840 struct ftrace_hash *old_hash = *orig_hash; 3841 struct list_head free_list; 3842 struct ftrace_hash *hash; 3843 struct hlist_node *tmp; 3844 char str[KSYM_SYMBOL_LEN]; 3845 int i, ret; 3846 3847 if (glob && (strcmp(glob, "*") == 0 || !strlen(glob))) 3848 func_g.search = NULL; 3849 else if (glob) { 3850 int not; 3851 3852 func_g.type = filter_parse_regex(glob, strlen(glob), 3853 &func_g.search, ¬); 3854 func_g.len = strlen(func_g.search); 3855 func_g.search = glob; 3856 3857 /* we do not support '!' for function probes */ 3858 if (WARN_ON(not)) 3859 return; 3860 } 3861 3862 mutex_lock(&trace_probe_ops.func_hash->regex_lock); 3863 3864 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash); 3865 if (!hash) 3866 /* Hmm, should report this somehow */ 3867 goto out_unlock; 3868 3869 INIT_LIST_HEAD(&free_list); 3870 3871 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 3872 struct hlist_head *hhd = &ftrace_func_hash[i]; 3873 3874 hlist_for_each_entry_safe(entry, tmp, hhd, node) { 3875 3876 /* break up if statements for readability */ 3877 if ((flags & PROBE_TEST_FUNC) && entry->ops != ops) 3878 continue; 3879 3880 if ((flags & PROBE_TEST_DATA) && entry->data != data) 3881 continue; 3882 3883 /* do this last, since it is the most expensive */ 3884 if (func_g.search) { 3885 kallsyms_lookup(entry->ip, NULL, NULL, 3886 NULL, str); 3887 if (!ftrace_match(str, &func_g)) 3888 continue; 3889 } 3890 3891 rec_entry = ftrace_lookup_ip(hash, entry->ip); 3892 /* It is possible more than one entry had this ip */ 3893 if (rec_entry) 3894 free_hash_entry(hash, rec_entry); 3895 3896 hlist_del_rcu(&entry->node); 3897 list_add(&entry->free_list, &free_list); 3898 } 3899 } 3900 mutex_lock(&ftrace_lock); 3901 __disable_ftrace_function_probe(); 3902 /* 3903 * Remove after the disable is called. Otherwise, if the last 3904 * probe is removed, a null hash means *all enabled*. 3905 */ 3906 ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash); 3907 synchronize_sched(); 3908 if (!ret) 3909 free_ftrace_hash_rcu(old_hash); 3910 3911 list_for_each_entry_safe(entry, p, &free_list, free_list) { 3912 list_del(&entry->free_list); 3913 ftrace_free_entry(entry); 3914 } 3915 mutex_unlock(&ftrace_lock); 3916 3917 out_unlock: 3918 mutex_unlock(&trace_probe_ops.func_hash->regex_lock); 3919 free_ftrace_hash(hash); 3920 } 3921 3922 void 3923 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 3924 void *data) 3925 { 3926 __unregister_ftrace_function_probe(glob, ops, data, 3927 PROBE_TEST_FUNC | PROBE_TEST_DATA); 3928 } 3929 3930 void 3931 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) 3932 { 3933 __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC); 3934 } 3935 3936 void unregister_ftrace_function_probe_all(char *glob) 3937 { 3938 __unregister_ftrace_function_probe(glob, NULL, NULL, 0); 3939 } 3940 3941 static LIST_HEAD(ftrace_commands); 3942 static DEFINE_MUTEX(ftrace_cmd_mutex); 3943 3944 /* 3945 * Currently we only register ftrace commands from __init, so mark this 3946 * __init too. 3947 */ 3948 __init int register_ftrace_command(struct ftrace_func_command *cmd) 3949 { 3950 struct ftrace_func_command *p; 3951 int ret = 0; 3952 3953 mutex_lock(&ftrace_cmd_mutex); 3954 list_for_each_entry(p, &ftrace_commands, list) { 3955 if (strcmp(cmd->name, p->name) == 0) { 3956 ret = -EBUSY; 3957 goto out_unlock; 3958 } 3959 } 3960 list_add(&cmd->list, &ftrace_commands); 3961 out_unlock: 3962 mutex_unlock(&ftrace_cmd_mutex); 3963 3964 return ret; 3965 } 3966 3967 /* 3968 * Currently we only unregister ftrace commands from __init, so mark 3969 * this __init too. 3970 */ 3971 __init int unregister_ftrace_command(struct ftrace_func_command *cmd) 3972 { 3973 struct ftrace_func_command *p, *n; 3974 int ret = -ENODEV; 3975 3976 mutex_lock(&ftrace_cmd_mutex); 3977 list_for_each_entry_safe(p, n, &ftrace_commands, list) { 3978 if (strcmp(cmd->name, p->name) == 0) { 3979 ret = 0; 3980 list_del_init(&p->list); 3981 goto out_unlock; 3982 } 3983 } 3984 out_unlock: 3985 mutex_unlock(&ftrace_cmd_mutex); 3986 3987 return ret; 3988 } 3989 3990 static int ftrace_process_regex(struct ftrace_hash *hash, 3991 char *buff, int len, int enable) 3992 { 3993 char *func, *command, *next = buff; 3994 struct ftrace_func_command *p; 3995 int ret = -EINVAL; 3996 3997 func = strsep(&next, ":"); 3998 3999 if (!next) { 4000 ret = ftrace_match_records(hash, func, len); 4001 if (!ret) 4002 ret = -EINVAL; 4003 if (ret < 0) 4004 return ret; 4005 return 0; 4006 } 4007 4008 /* command found */ 4009 4010 command = strsep(&next, ":"); 4011 4012 mutex_lock(&ftrace_cmd_mutex); 4013 list_for_each_entry(p, &ftrace_commands, list) { 4014 if (strcmp(p->name, command) == 0) { 4015 ret = p->func(hash, func, command, next, enable); 4016 goto out_unlock; 4017 } 4018 } 4019 out_unlock: 4020 mutex_unlock(&ftrace_cmd_mutex); 4021 4022 return ret; 4023 } 4024 4025 static ssize_t 4026 ftrace_regex_write(struct file *file, const char __user *ubuf, 4027 size_t cnt, loff_t *ppos, int enable) 4028 { 4029 struct ftrace_iterator *iter; 4030 struct trace_parser *parser; 4031 ssize_t ret, read; 4032 4033 if (!cnt) 4034 return 0; 4035 4036 if (file->f_mode & FMODE_READ) { 4037 struct seq_file *m = file->private_data; 4038 iter = m->private; 4039 } else 4040 iter = file->private_data; 4041 4042 if (unlikely(ftrace_disabled)) 4043 return -ENODEV; 4044 4045 /* iter->hash is a local copy, so we don't need regex_lock */ 4046 4047 parser = &iter->parser; 4048 read = trace_get_user(parser, ubuf, cnt, ppos); 4049 4050 if (read >= 0 && trace_parser_loaded(parser) && 4051 !trace_parser_cont(parser)) { 4052 ret = ftrace_process_regex(iter->hash, parser->buffer, 4053 parser->idx, enable); 4054 trace_parser_clear(parser); 4055 if (ret < 0) 4056 goto out; 4057 } 4058 4059 ret = read; 4060 out: 4061 return ret; 4062 } 4063 4064 ssize_t 4065 ftrace_filter_write(struct file *file, const char __user *ubuf, 4066 size_t cnt, loff_t *ppos) 4067 { 4068 return ftrace_regex_write(file, ubuf, cnt, ppos, 1); 4069 } 4070 4071 ssize_t 4072 ftrace_notrace_write(struct file *file, const char __user *ubuf, 4073 size_t cnt, loff_t *ppos) 4074 { 4075 return ftrace_regex_write(file, ubuf, cnt, ppos, 0); 4076 } 4077 4078 static int 4079 ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove) 4080 { 4081 struct ftrace_func_entry *entry; 4082 4083 if (!ftrace_location(ip)) 4084 return -EINVAL; 4085 4086 if (remove) { 4087 entry = ftrace_lookup_ip(hash, ip); 4088 if (!entry) 4089 return -ENOENT; 4090 free_hash_entry(hash, entry); 4091 return 0; 4092 } 4093 4094 return add_hash_entry(hash, ip); 4095 } 4096 4097 static void ftrace_ops_update_code(struct ftrace_ops *ops, 4098 struct ftrace_ops_hash *old_hash) 4099 { 4100 struct ftrace_ops *op; 4101 4102 if (!ftrace_enabled) 4103 return; 4104 4105 if (ops->flags & FTRACE_OPS_FL_ENABLED) { 4106 ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash); 4107 return; 4108 } 4109 4110 /* 4111 * If this is the shared global_ops filter, then we need to 4112 * check if there is another ops that shares it, is enabled. 4113 * If so, we still need to run the modify code. 4114 */ 4115 if (ops->func_hash != &global_ops.local_hash) 4116 return; 4117 4118 do_for_each_ftrace_op(op, ftrace_ops_list) { 4119 if (op->func_hash == &global_ops.local_hash && 4120 op->flags & FTRACE_OPS_FL_ENABLED) { 4121 ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash); 4122 /* Only need to do this once */ 4123 return; 4124 } 4125 } while_for_each_ftrace_op(op); 4126 } 4127 4128 static int 4129 ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len, 4130 unsigned long ip, int remove, int reset, int enable) 4131 { 4132 struct ftrace_hash **orig_hash; 4133 struct ftrace_ops_hash old_hash_ops; 4134 struct ftrace_hash *old_hash; 4135 struct ftrace_hash *hash; 4136 int ret; 4137 4138 if (unlikely(ftrace_disabled)) 4139 return -ENODEV; 4140 4141 mutex_lock(&ops->func_hash->regex_lock); 4142 4143 if (enable) 4144 orig_hash = &ops->func_hash->filter_hash; 4145 else 4146 orig_hash = &ops->func_hash->notrace_hash; 4147 4148 if (reset) 4149 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); 4150 else 4151 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash); 4152 4153 if (!hash) { 4154 ret = -ENOMEM; 4155 goto out_regex_unlock; 4156 } 4157 4158 if (buf && !ftrace_match_records(hash, buf, len)) { 4159 ret = -EINVAL; 4160 goto out_regex_unlock; 4161 } 4162 if (ip) { 4163 ret = ftrace_match_addr(hash, ip, remove); 4164 if (ret < 0) 4165 goto out_regex_unlock; 4166 } 4167 4168 mutex_lock(&ftrace_lock); 4169 old_hash = *orig_hash; 4170 old_hash_ops.filter_hash = ops->func_hash->filter_hash; 4171 old_hash_ops.notrace_hash = ops->func_hash->notrace_hash; 4172 ret = ftrace_hash_move(ops, enable, orig_hash, hash); 4173 if (!ret) { 4174 ftrace_ops_update_code(ops, &old_hash_ops); 4175 free_ftrace_hash_rcu(old_hash); 4176 } 4177 mutex_unlock(&ftrace_lock); 4178 4179 out_regex_unlock: 4180 mutex_unlock(&ops->func_hash->regex_lock); 4181 4182 free_ftrace_hash(hash); 4183 return ret; 4184 } 4185 4186 static int 4187 ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove, 4188 int reset, int enable) 4189 { 4190 return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable); 4191 } 4192 4193 /** 4194 * ftrace_set_filter_ip - set a function to filter on in ftrace by address 4195 * @ops - the ops to set the filter with 4196 * @ip - the address to add to or remove from the filter. 4197 * @remove - non zero to remove the ip from the filter 4198 * @reset - non zero to reset all filters before applying this filter. 4199 * 4200 * Filters denote which functions should be enabled when tracing is enabled 4201 * If @ip is NULL, it failes to update filter. 4202 */ 4203 int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip, 4204 int remove, int reset) 4205 { 4206 ftrace_ops_init(ops); 4207 return ftrace_set_addr(ops, ip, remove, reset, 1); 4208 } 4209 EXPORT_SYMBOL_GPL(ftrace_set_filter_ip); 4210 4211 static int 4212 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len, 4213 int reset, int enable) 4214 { 4215 return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable); 4216 } 4217 4218 /** 4219 * ftrace_set_filter - set a function to filter on in ftrace 4220 * @ops - the ops to set the filter with 4221 * @buf - the string that holds the function filter text. 4222 * @len - the length of the string. 4223 * @reset - non zero to reset all filters before applying this filter. 4224 * 4225 * Filters denote which functions should be enabled when tracing is enabled. 4226 * If @buf is NULL and reset is set, all functions will be enabled for tracing. 4227 */ 4228 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf, 4229 int len, int reset) 4230 { 4231 ftrace_ops_init(ops); 4232 return ftrace_set_regex(ops, buf, len, reset, 1); 4233 } 4234 EXPORT_SYMBOL_GPL(ftrace_set_filter); 4235 4236 /** 4237 * ftrace_set_notrace - set a function to not trace in ftrace 4238 * @ops - the ops to set the notrace filter with 4239 * @buf - the string that holds the function notrace text. 4240 * @len - the length of the string. 4241 * @reset - non zero to reset all filters before applying this filter. 4242 * 4243 * Notrace Filters denote which functions should not be enabled when tracing 4244 * is enabled. If @buf is NULL and reset is set, all functions will be enabled 4245 * for tracing. 4246 */ 4247 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf, 4248 int len, int reset) 4249 { 4250 ftrace_ops_init(ops); 4251 return ftrace_set_regex(ops, buf, len, reset, 0); 4252 } 4253 EXPORT_SYMBOL_GPL(ftrace_set_notrace); 4254 /** 4255 * ftrace_set_global_filter - set a function to filter on with global tracers 4256 * @buf - the string that holds the function filter text. 4257 * @len - the length of the string. 4258 * @reset - non zero to reset all filters before applying this filter. 4259 * 4260 * Filters denote which functions should be enabled when tracing is enabled. 4261 * If @buf is NULL and reset is set, all functions will be enabled for tracing. 4262 */ 4263 void ftrace_set_global_filter(unsigned char *buf, int len, int reset) 4264 { 4265 ftrace_set_regex(&global_ops, buf, len, reset, 1); 4266 } 4267 EXPORT_SYMBOL_GPL(ftrace_set_global_filter); 4268 4269 /** 4270 * ftrace_set_global_notrace - set a function to not trace with global tracers 4271 * @buf - the string that holds the function notrace text. 4272 * @len - the length of the string. 4273 * @reset - non zero to reset all filters before applying this filter. 4274 * 4275 * Notrace Filters denote which functions should not be enabled when tracing 4276 * is enabled. If @buf is NULL and reset is set, all functions will be enabled 4277 * for tracing. 4278 */ 4279 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset) 4280 { 4281 ftrace_set_regex(&global_ops, buf, len, reset, 0); 4282 } 4283 EXPORT_SYMBOL_GPL(ftrace_set_global_notrace); 4284 4285 /* 4286 * command line interface to allow users to set filters on boot up. 4287 */ 4288 #define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE 4289 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata; 4290 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata; 4291 4292 /* Used by function selftest to not test if filter is set */ 4293 bool ftrace_filter_param __initdata; 4294 4295 static int __init set_ftrace_notrace(char *str) 4296 { 4297 ftrace_filter_param = true; 4298 strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE); 4299 return 1; 4300 } 4301 __setup("ftrace_notrace=", set_ftrace_notrace); 4302 4303 static int __init set_ftrace_filter(char *str) 4304 { 4305 ftrace_filter_param = true; 4306 strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE); 4307 return 1; 4308 } 4309 __setup("ftrace_filter=", set_ftrace_filter); 4310 4311 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 4312 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; 4313 static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata; 4314 static int ftrace_set_func(unsigned long *array, int *idx, int size, char *buffer); 4315 4316 static unsigned long save_global_trampoline; 4317 static unsigned long save_global_flags; 4318 4319 static int __init set_graph_function(char *str) 4320 { 4321 strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE); 4322 return 1; 4323 } 4324 __setup("ftrace_graph_filter=", set_graph_function); 4325 4326 static int __init set_graph_notrace_function(char *str) 4327 { 4328 strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE); 4329 return 1; 4330 } 4331 __setup("ftrace_graph_notrace=", set_graph_notrace_function); 4332 4333 static void __init set_ftrace_early_graph(char *buf, int enable) 4334 { 4335 int ret; 4336 char *func; 4337 unsigned long *table = ftrace_graph_funcs; 4338 int *count = &ftrace_graph_count; 4339 4340 if (!enable) { 4341 table = ftrace_graph_notrace_funcs; 4342 count = &ftrace_graph_notrace_count; 4343 } 4344 4345 while (buf) { 4346 func = strsep(&buf, ","); 4347 /* we allow only one expression at a time */ 4348 ret = ftrace_set_func(table, count, FTRACE_GRAPH_MAX_FUNCS, func); 4349 if (ret) 4350 printk(KERN_DEBUG "ftrace: function %s not " 4351 "traceable\n", func); 4352 } 4353 } 4354 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 4355 4356 void __init 4357 ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable) 4358 { 4359 char *func; 4360 4361 ftrace_ops_init(ops); 4362 4363 while (buf) { 4364 func = strsep(&buf, ","); 4365 ftrace_set_regex(ops, func, strlen(func), 0, enable); 4366 } 4367 } 4368 4369 static void __init set_ftrace_early_filters(void) 4370 { 4371 if (ftrace_filter_buf[0]) 4372 ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1); 4373 if (ftrace_notrace_buf[0]) 4374 ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0); 4375 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 4376 if (ftrace_graph_buf[0]) 4377 set_ftrace_early_graph(ftrace_graph_buf, 1); 4378 if (ftrace_graph_notrace_buf[0]) 4379 set_ftrace_early_graph(ftrace_graph_notrace_buf, 0); 4380 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 4381 } 4382 4383 int ftrace_regex_release(struct inode *inode, struct file *file) 4384 { 4385 struct seq_file *m = (struct seq_file *)file->private_data; 4386 struct ftrace_ops_hash old_hash_ops; 4387 struct ftrace_iterator *iter; 4388 struct ftrace_hash **orig_hash; 4389 struct ftrace_hash *old_hash; 4390 struct trace_parser *parser; 4391 int filter_hash; 4392 int ret; 4393 4394 if (file->f_mode & FMODE_READ) { 4395 iter = m->private; 4396 seq_release(inode, file); 4397 } else 4398 iter = file->private_data; 4399 4400 parser = &iter->parser; 4401 if (trace_parser_loaded(parser)) { 4402 parser->buffer[parser->idx] = 0; 4403 ftrace_match_records(iter->hash, parser->buffer, parser->idx); 4404 } 4405 4406 trace_parser_put(parser); 4407 4408 mutex_lock(&iter->ops->func_hash->regex_lock); 4409 4410 if (file->f_mode & FMODE_WRITE) { 4411 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER); 4412 4413 if (filter_hash) 4414 orig_hash = &iter->ops->func_hash->filter_hash; 4415 else 4416 orig_hash = &iter->ops->func_hash->notrace_hash; 4417 4418 mutex_lock(&ftrace_lock); 4419 old_hash = *orig_hash; 4420 old_hash_ops.filter_hash = iter->ops->func_hash->filter_hash; 4421 old_hash_ops.notrace_hash = iter->ops->func_hash->notrace_hash; 4422 ret = ftrace_hash_move(iter->ops, filter_hash, 4423 orig_hash, iter->hash); 4424 if (!ret) { 4425 ftrace_ops_update_code(iter->ops, &old_hash_ops); 4426 free_ftrace_hash_rcu(old_hash); 4427 } 4428 mutex_unlock(&ftrace_lock); 4429 } 4430 4431 mutex_unlock(&iter->ops->func_hash->regex_lock); 4432 free_ftrace_hash(iter->hash); 4433 kfree(iter); 4434 4435 return 0; 4436 } 4437 4438 static const struct file_operations ftrace_avail_fops = { 4439 .open = ftrace_avail_open, 4440 .read = seq_read, 4441 .llseek = seq_lseek, 4442 .release = seq_release_private, 4443 }; 4444 4445 static const struct file_operations ftrace_enabled_fops = { 4446 .open = ftrace_enabled_open, 4447 .read = seq_read, 4448 .llseek = seq_lseek, 4449 .release = seq_release_private, 4450 }; 4451 4452 static const struct file_operations ftrace_filter_fops = { 4453 .open = ftrace_filter_open, 4454 .read = seq_read, 4455 .write = ftrace_filter_write, 4456 .llseek = tracing_lseek, 4457 .release = ftrace_regex_release, 4458 }; 4459 4460 static const struct file_operations ftrace_notrace_fops = { 4461 .open = ftrace_notrace_open, 4462 .read = seq_read, 4463 .write = ftrace_notrace_write, 4464 .llseek = tracing_lseek, 4465 .release = ftrace_regex_release, 4466 }; 4467 4468 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 4469 4470 static DEFINE_MUTEX(graph_lock); 4471 4472 int ftrace_graph_count; 4473 int ftrace_graph_notrace_count; 4474 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly; 4475 unsigned long ftrace_graph_notrace_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly; 4476 4477 struct ftrace_graph_data { 4478 unsigned long *table; 4479 size_t size; 4480 int *count; 4481 const struct seq_operations *seq_ops; 4482 }; 4483 4484 static void * 4485 __g_next(struct seq_file *m, loff_t *pos) 4486 { 4487 struct ftrace_graph_data *fgd = m->private; 4488 4489 if (*pos >= *fgd->count) 4490 return NULL; 4491 return &fgd->table[*pos]; 4492 } 4493 4494 static void * 4495 g_next(struct seq_file *m, void *v, loff_t *pos) 4496 { 4497 (*pos)++; 4498 return __g_next(m, pos); 4499 } 4500 4501 static void *g_start(struct seq_file *m, loff_t *pos) 4502 { 4503 struct ftrace_graph_data *fgd = m->private; 4504 4505 mutex_lock(&graph_lock); 4506 4507 /* Nothing, tell g_show to print all functions are enabled */ 4508 if (!*fgd->count && !*pos) 4509 return (void *)1; 4510 4511 return __g_next(m, pos); 4512 } 4513 4514 static void g_stop(struct seq_file *m, void *p) 4515 { 4516 mutex_unlock(&graph_lock); 4517 } 4518 4519 static int g_show(struct seq_file *m, void *v) 4520 { 4521 unsigned long *ptr = v; 4522 4523 if (!ptr) 4524 return 0; 4525 4526 if (ptr == (unsigned long *)1) { 4527 struct ftrace_graph_data *fgd = m->private; 4528 4529 if (fgd->table == ftrace_graph_funcs) 4530 seq_puts(m, "#### all functions enabled ####\n"); 4531 else 4532 seq_puts(m, "#### no functions disabled ####\n"); 4533 return 0; 4534 } 4535 4536 seq_printf(m, "%ps\n", (void *)*ptr); 4537 4538 return 0; 4539 } 4540 4541 static const struct seq_operations ftrace_graph_seq_ops = { 4542 .start = g_start, 4543 .next = g_next, 4544 .stop = g_stop, 4545 .show = g_show, 4546 }; 4547 4548 static int 4549 __ftrace_graph_open(struct inode *inode, struct file *file, 4550 struct ftrace_graph_data *fgd) 4551 { 4552 int ret = 0; 4553 4554 mutex_lock(&graph_lock); 4555 if ((file->f_mode & FMODE_WRITE) && 4556 (file->f_flags & O_TRUNC)) { 4557 *fgd->count = 0; 4558 memset(fgd->table, 0, fgd->size * sizeof(*fgd->table)); 4559 } 4560 mutex_unlock(&graph_lock); 4561 4562 if (file->f_mode & FMODE_READ) { 4563 ret = seq_open(file, fgd->seq_ops); 4564 if (!ret) { 4565 struct seq_file *m = file->private_data; 4566 m->private = fgd; 4567 } 4568 } else 4569 file->private_data = fgd; 4570 4571 return ret; 4572 } 4573 4574 static int 4575 ftrace_graph_open(struct inode *inode, struct file *file) 4576 { 4577 struct ftrace_graph_data *fgd; 4578 4579 if (unlikely(ftrace_disabled)) 4580 return -ENODEV; 4581 4582 fgd = kmalloc(sizeof(*fgd), GFP_KERNEL); 4583 if (fgd == NULL) 4584 return -ENOMEM; 4585 4586 fgd->table = ftrace_graph_funcs; 4587 fgd->size = FTRACE_GRAPH_MAX_FUNCS; 4588 fgd->count = &ftrace_graph_count; 4589 fgd->seq_ops = &ftrace_graph_seq_ops; 4590 4591 return __ftrace_graph_open(inode, file, fgd); 4592 } 4593 4594 static int 4595 ftrace_graph_notrace_open(struct inode *inode, struct file *file) 4596 { 4597 struct ftrace_graph_data *fgd; 4598 4599 if (unlikely(ftrace_disabled)) 4600 return -ENODEV; 4601 4602 fgd = kmalloc(sizeof(*fgd), GFP_KERNEL); 4603 if (fgd == NULL) 4604 return -ENOMEM; 4605 4606 fgd->table = ftrace_graph_notrace_funcs; 4607 fgd->size = FTRACE_GRAPH_MAX_FUNCS; 4608 fgd->count = &ftrace_graph_notrace_count; 4609 fgd->seq_ops = &ftrace_graph_seq_ops; 4610 4611 return __ftrace_graph_open(inode, file, fgd); 4612 } 4613 4614 static int 4615 ftrace_graph_release(struct inode *inode, struct file *file) 4616 { 4617 if (file->f_mode & FMODE_READ) { 4618 struct seq_file *m = file->private_data; 4619 4620 kfree(m->private); 4621 seq_release(inode, file); 4622 } else { 4623 kfree(file->private_data); 4624 } 4625 4626 return 0; 4627 } 4628 4629 static int 4630 ftrace_set_func(unsigned long *array, int *idx, int size, char *buffer) 4631 { 4632 struct ftrace_glob func_g; 4633 struct dyn_ftrace *rec; 4634 struct ftrace_page *pg; 4635 int fail = 1; 4636 int not; 4637 bool exists; 4638 int i; 4639 4640 /* decode regex */ 4641 func_g.type = filter_parse_regex(buffer, strlen(buffer), 4642 &func_g.search, ¬); 4643 if (!not && *idx >= size) 4644 return -EBUSY; 4645 4646 func_g.len = strlen(func_g.search); 4647 4648 mutex_lock(&ftrace_lock); 4649 4650 if (unlikely(ftrace_disabled)) { 4651 mutex_unlock(&ftrace_lock); 4652 return -ENODEV; 4653 } 4654 4655 do_for_each_ftrace_rec(pg, rec) { 4656 4657 if (ftrace_match_record(rec, &func_g, NULL, 0)) { 4658 /* if it is in the array */ 4659 exists = false; 4660 for (i = 0; i < *idx; i++) { 4661 if (array[i] == rec->ip) { 4662 exists = true; 4663 break; 4664 } 4665 } 4666 4667 if (!not) { 4668 fail = 0; 4669 if (!exists) { 4670 array[(*idx)++] = rec->ip; 4671 if (*idx >= size) 4672 goto out; 4673 } 4674 } else { 4675 if (exists) { 4676 array[i] = array[--(*idx)]; 4677 array[*idx] = 0; 4678 fail = 0; 4679 } 4680 } 4681 } 4682 } while_for_each_ftrace_rec(); 4683 out: 4684 mutex_unlock(&ftrace_lock); 4685 4686 if (fail) 4687 return -EINVAL; 4688 4689 return 0; 4690 } 4691 4692 static ssize_t 4693 ftrace_graph_write(struct file *file, const char __user *ubuf, 4694 size_t cnt, loff_t *ppos) 4695 { 4696 struct trace_parser parser; 4697 ssize_t read, ret = 0; 4698 struct ftrace_graph_data *fgd = file->private_data; 4699 4700 if (!cnt) 4701 return 0; 4702 4703 if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) 4704 return -ENOMEM; 4705 4706 read = trace_get_user(&parser, ubuf, cnt, ppos); 4707 4708 if (read >= 0 && trace_parser_loaded((&parser))) { 4709 parser.buffer[parser.idx] = 0; 4710 4711 mutex_lock(&graph_lock); 4712 4713 /* we allow only one expression at a time */ 4714 ret = ftrace_set_func(fgd->table, fgd->count, fgd->size, 4715 parser.buffer); 4716 4717 mutex_unlock(&graph_lock); 4718 } 4719 4720 if (!ret) 4721 ret = read; 4722 4723 trace_parser_put(&parser); 4724 4725 return ret; 4726 } 4727 4728 static const struct file_operations ftrace_graph_fops = { 4729 .open = ftrace_graph_open, 4730 .read = seq_read, 4731 .write = ftrace_graph_write, 4732 .llseek = tracing_lseek, 4733 .release = ftrace_graph_release, 4734 }; 4735 4736 static const struct file_operations ftrace_graph_notrace_fops = { 4737 .open = ftrace_graph_notrace_open, 4738 .read = seq_read, 4739 .write = ftrace_graph_write, 4740 .llseek = tracing_lseek, 4741 .release = ftrace_graph_release, 4742 }; 4743 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 4744 4745 void ftrace_create_filter_files(struct ftrace_ops *ops, 4746 struct dentry *parent) 4747 { 4748 4749 trace_create_file("set_ftrace_filter", 0644, parent, 4750 ops, &ftrace_filter_fops); 4751 4752 trace_create_file("set_ftrace_notrace", 0644, parent, 4753 ops, &ftrace_notrace_fops); 4754 } 4755 4756 /* 4757 * The name "destroy_filter_files" is really a misnomer. Although 4758 * in the future, it may actualy delete the files, but this is 4759 * really intended to make sure the ops passed in are disabled 4760 * and that when this function returns, the caller is free to 4761 * free the ops. 4762 * 4763 * The "destroy" name is only to match the "create" name that this 4764 * should be paired with. 4765 */ 4766 void ftrace_destroy_filter_files(struct ftrace_ops *ops) 4767 { 4768 mutex_lock(&ftrace_lock); 4769 if (ops->flags & FTRACE_OPS_FL_ENABLED) 4770 ftrace_shutdown(ops, 0); 4771 ops->flags |= FTRACE_OPS_FL_DELETED; 4772 mutex_unlock(&ftrace_lock); 4773 } 4774 4775 static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer) 4776 { 4777 4778 trace_create_file("available_filter_functions", 0444, 4779 d_tracer, NULL, &ftrace_avail_fops); 4780 4781 trace_create_file("enabled_functions", 0444, 4782 d_tracer, NULL, &ftrace_enabled_fops); 4783 4784 ftrace_create_filter_files(&global_ops, d_tracer); 4785 4786 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 4787 trace_create_file("set_graph_function", 0444, d_tracer, 4788 NULL, 4789 &ftrace_graph_fops); 4790 trace_create_file("set_graph_notrace", 0444, d_tracer, 4791 NULL, 4792 &ftrace_graph_notrace_fops); 4793 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 4794 4795 return 0; 4796 } 4797 4798 static int ftrace_cmp_ips(const void *a, const void *b) 4799 { 4800 const unsigned long *ipa = a; 4801 const unsigned long *ipb = b; 4802 4803 if (*ipa > *ipb) 4804 return 1; 4805 if (*ipa < *ipb) 4806 return -1; 4807 return 0; 4808 } 4809 4810 static int ftrace_process_locs(struct module *mod, 4811 unsigned long *start, 4812 unsigned long *end) 4813 { 4814 struct ftrace_page *start_pg; 4815 struct ftrace_page *pg; 4816 struct dyn_ftrace *rec; 4817 unsigned long count; 4818 unsigned long *p; 4819 unsigned long addr; 4820 unsigned long flags = 0; /* Shut up gcc */ 4821 int ret = -ENOMEM; 4822 4823 count = end - start; 4824 4825 if (!count) 4826 return 0; 4827 4828 sort(start, count, sizeof(*start), 4829 ftrace_cmp_ips, NULL); 4830 4831 start_pg = ftrace_allocate_pages(count); 4832 if (!start_pg) 4833 return -ENOMEM; 4834 4835 mutex_lock(&ftrace_lock); 4836 4837 /* 4838 * Core and each module needs their own pages, as 4839 * modules will free them when they are removed. 4840 * Force a new page to be allocated for modules. 4841 */ 4842 if (!mod) { 4843 WARN_ON(ftrace_pages || ftrace_pages_start); 4844 /* First initialization */ 4845 ftrace_pages = ftrace_pages_start = start_pg; 4846 } else { 4847 if (!ftrace_pages) 4848 goto out; 4849 4850 if (WARN_ON(ftrace_pages->next)) { 4851 /* Hmm, we have free pages? */ 4852 while (ftrace_pages->next) 4853 ftrace_pages = ftrace_pages->next; 4854 } 4855 4856 ftrace_pages->next = start_pg; 4857 } 4858 4859 p = start; 4860 pg = start_pg; 4861 while (p < end) { 4862 addr = ftrace_call_adjust(*p++); 4863 /* 4864 * Some architecture linkers will pad between 4865 * the different mcount_loc sections of different 4866 * object files to satisfy alignments. 4867 * Skip any NULL pointers. 4868 */ 4869 if (!addr) 4870 continue; 4871 4872 if (pg->index == pg->size) { 4873 /* We should have allocated enough */ 4874 if (WARN_ON(!pg->next)) 4875 break; 4876 pg = pg->next; 4877 } 4878 4879 rec = &pg->records[pg->index++]; 4880 rec->ip = addr; 4881 } 4882 4883 /* We should have used all pages */ 4884 WARN_ON(pg->next); 4885 4886 /* Assign the last page to ftrace_pages */ 4887 ftrace_pages = pg; 4888 4889 /* 4890 * We only need to disable interrupts on start up 4891 * because we are modifying code that an interrupt 4892 * may execute, and the modification is not atomic. 4893 * But for modules, nothing runs the code we modify 4894 * until we are finished with it, and there's no 4895 * reason to cause large interrupt latencies while we do it. 4896 */ 4897 if (!mod) 4898 local_irq_save(flags); 4899 ftrace_update_code(mod, start_pg); 4900 if (!mod) 4901 local_irq_restore(flags); 4902 ret = 0; 4903 out: 4904 mutex_unlock(&ftrace_lock); 4905 4906 return ret; 4907 } 4908 4909 #ifdef CONFIG_MODULES 4910 4911 #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next) 4912 4913 static int referenced_filters(struct dyn_ftrace *rec) 4914 { 4915 struct ftrace_ops *ops; 4916 int cnt = 0; 4917 4918 for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) { 4919 if (ops_references_rec(ops, rec)) 4920 cnt++; 4921 } 4922 4923 return cnt; 4924 } 4925 4926 void ftrace_release_mod(struct module *mod) 4927 { 4928 struct dyn_ftrace *rec; 4929 struct ftrace_page **last_pg; 4930 struct ftrace_page *pg; 4931 int order; 4932 4933 mutex_lock(&ftrace_lock); 4934 4935 if (ftrace_disabled) 4936 goto out_unlock; 4937 4938 /* 4939 * Each module has its own ftrace_pages, remove 4940 * them from the list. 4941 */ 4942 last_pg = &ftrace_pages_start; 4943 for (pg = ftrace_pages_start; pg; pg = *last_pg) { 4944 rec = &pg->records[0]; 4945 if (within_module_core(rec->ip, mod)) { 4946 /* 4947 * As core pages are first, the first 4948 * page should never be a module page. 4949 */ 4950 if (WARN_ON(pg == ftrace_pages_start)) 4951 goto out_unlock; 4952 4953 /* Check if we are deleting the last page */ 4954 if (pg == ftrace_pages) 4955 ftrace_pages = next_to_ftrace_page(last_pg); 4956 4957 *last_pg = pg->next; 4958 order = get_count_order(pg->size / ENTRIES_PER_PAGE); 4959 free_pages((unsigned long)pg->records, order); 4960 kfree(pg); 4961 } else 4962 last_pg = &pg->next; 4963 } 4964 out_unlock: 4965 mutex_unlock(&ftrace_lock); 4966 } 4967 4968 void ftrace_module_enable(struct module *mod) 4969 { 4970 struct dyn_ftrace *rec; 4971 struct ftrace_page *pg; 4972 4973 mutex_lock(&ftrace_lock); 4974 4975 if (ftrace_disabled) 4976 goto out_unlock; 4977 4978 /* 4979 * If the tracing is enabled, go ahead and enable the record. 4980 * 4981 * The reason not to enable the record immediatelly is the 4982 * inherent check of ftrace_make_nop/ftrace_make_call for 4983 * correct previous instructions. Making first the NOP 4984 * conversion puts the module to the correct state, thus 4985 * passing the ftrace_make_call check. 4986 * 4987 * We also delay this to after the module code already set the 4988 * text to read-only, as we now need to set it back to read-write 4989 * so that we can modify the text. 4990 */ 4991 if (ftrace_start_up) 4992 ftrace_arch_code_modify_prepare(); 4993 4994 do_for_each_ftrace_rec(pg, rec) { 4995 int cnt; 4996 /* 4997 * do_for_each_ftrace_rec() is a double loop. 4998 * module text shares the pg. If a record is 4999 * not part of this module, then skip this pg, 5000 * which the "break" will do. 5001 */ 5002 if (!within_module_core(rec->ip, mod)) 5003 break; 5004 5005 cnt = 0; 5006 5007 /* 5008 * When adding a module, we need to check if tracers are 5009 * currently enabled and if they are, and can trace this record, 5010 * we need to enable the module functions as well as update the 5011 * reference counts for those function records. 5012 */ 5013 if (ftrace_start_up) 5014 cnt += referenced_filters(rec); 5015 5016 /* This clears FTRACE_FL_DISABLED */ 5017 rec->flags = cnt; 5018 5019 if (ftrace_start_up && cnt) { 5020 int failed = __ftrace_replace_code(rec, 1); 5021 if (failed) { 5022 ftrace_bug(failed, rec); 5023 goto out_loop; 5024 } 5025 } 5026 5027 } while_for_each_ftrace_rec(); 5028 5029 out_loop: 5030 if (ftrace_start_up) 5031 ftrace_arch_code_modify_post_process(); 5032 5033 out_unlock: 5034 mutex_unlock(&ftrace_lock); 5035 } 5036 5037 void ftrace_module_init(struct module *mod) 5038 { 5039 if (ftrace_disabled || !mod->num_ftrace_callsites) 5040 return; 5041 5042 ftrace_process_locs(mod, mod->ftrace_callsites, 5043 mod->ftrace_callsites + mod->num_ftrace_callsites); 5044 } 5045 #endif /* CONFIG_MODULES */ 5046 5047 void __init ftrace_init(void) 5048 { 5049 extern unsigned long __start_mcount_loc[]; 5050 extern unsigned long __stop_mcount_loc[]; 5051 unsigned long count, flags; 5052 int ret; 5053 5054 local_irq_save(flags); 5055 ret = ftrace_dyn_arch_init(); 5056 local_irq_restore(flags); 5057 if (ret) 5058 goto failed; 5059 5060 count = __stop_mcount_loc - __start_mcount_loc; 5061 if (!count) { 5062 pr_info("ftrace: No functions to be traced?\n"); 5063 goto failed; 5064 } 5065 5066 pr_info("ftrace: allocating %ld entries in %ld pages\n", 5067 count, count / ENTRIES_PER_PAGE + 1); 5068 5069 last_ftrace_enabled = ftrace_enabled = 1; 5070 5071 ret = ftrace_process_locs(NULL, 5072 __start_mcount_loc, 5073 __stop_mcount_loc); 5074 5075 set_ftrace_early_filters(); 5076 5077 return; 5078 failed: 5079 ftrace_disabled = 1; 5080 } 5081 5082 /* Do nothing if arch does not support this */ 5083 void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops) 5084 { 5085 } 5086 5087 static void ftrace_update_trampoline(struct ftrace_ops *ops) 5088 { 5089 5090 /* 5091 * Currently there's no safe way to free a trampoline when the kernel 5092 * is configured with PREEMPT. That is because a task could be preempted 5093 * when it jumped to the trampoline, it may be preempted for a long time 5094 * depending on the system load, and currently there's no way to know 5095 * when it will be off the trampoline. If the trampoline is freed 5096 * too early, when the task runs again, it will be executing on freed 5097 * memory and crash. 5098 */ 5099 #ifdef CONFIG_PREEMPT 5100 /* Currently, only non dynamic ops can have a trampoline */ 5101 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) 5102 return; 5103 #endif 5104 5105 arch_ftrace_update_trampoline(ops); 5106 } 5107 5108 #else 5109 5110 static struct ftrace_ops global_ops = { 5111 .func = ftrace_stub, 5112 .flags = FTRACE_OPS_FL_RECURSION_SAFE | 5113 FTRACE_OPS_FL_INITIALIZED | 5114 FTRACE_OPS_FL_PID, 5115 }; 5116 5117 static int __init ftrace_nodyn_init(void) 5118 { 5119 ftrace_enabled = 1; 5120 return 0; 5121 } 5122 core_initcall(ftrace_nodyn_init); 5123 5124 static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; } 5125 static inline void ftrace_startup_enable(int command) { } 5126 static inline void ftrace_startup_all(int command) { } 5127 /* Keep as macros so we do not need to define the commands */ 5128 # define ftrace_startup(ops, command) \ 5129 ({ \ 5130 int ___ret = __register_ftrace_function(ops); \ 5131 if (!___ret) \ 5132 (ops)->flags |= FTRACE_OPS_FL_ENABLED; \ 5133 ___ret; \ 5134 }) 5135 # define ftrace_shutdown(ops, command) \ 5136 ({ \ 5137 int ___ret = __unregister_ftrace_function(ops); \ 5138 if (!___ret) \ 5139 (ops)->flags &= ~FTRACE_OPS_FL_ENABLED; \ 5140 ___ret; \ 5141 }) 5142 5143 # define ftrace_startup_sysctl() do { } while (0) 5144 # define ftrace_shutdown_sysctl() do { } while (0) 5145 5146 static inline int 5147 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) 5148 { 5149 return 1; 5150 } 5151 5152 static void ftrace_update_trampoline(struct ftrace_ops *ops) 5153 { 5154 } 5155 5156 #endif /* CONFIG_DYNAMIC_FTRACE */ 5157 5158 __init void ftrace_init_global_array_ops(struct trace_array *tr) 5159 { 5160 tr->ops = &global_ops; 5161 tr->ops->private = tr; 5162 } 5163 5164 void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func) 5165 { 5166 /* If we filter on pids, update to use the pid function */ 5167 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) { 5168 if (WARN_ON(tr->ops->func != ftrace_stub)) 5169 printk("ftrace ops had %pS for function\n", 5170 tr->ops->func); 5171 } 5172 tr->ops->func = func; 5173 tr->ops->private = tr; 5174 } 5175 5176 void ftrace_reset_array_ops(struct trace_array *tr) 5177 { 5178 tr->ops->func = ftrace_stub; 5179 } 5180 5181 static inline void 5182 __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 5183 struct ftrace_ops *ignored, struct pt_regs *regs) 5184 { 5185 struct ftrace_ops *op; 5186 int bit; 5187 5188 bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX); 5189 if (bit < 0) 5190 return; 5191 5192 /* 5193 * Some of the ops may be dynamically allocated, 5194 * they must be freed after a synchronize_sched(). 5195 */ 5196 preempt_disable_notrace(); 5197 5198 do_for_each_ftrace_op(op, ftrace_ops_list) { 5199 /* 5200 * Check the following for each ops before calling their func: 5201 * if RCU flag is set, then rcu_is_watching() must be true 5202 * if PER_CPU is set, then ftrace_function_local_disable() 5203 * must be false 5204 * Otherwise test if the ip matches the ops filter 5205 * 5206 * If any of the above fails then the op->func() is not executed. 5207 */ 5208 if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) && 5209 (!(op->flags & FTRACE_OPS_FL_PER_CPU) || 5210 !ftrace_function_local_disabled(op)) && 5211 ftrace_ops_test(op, ip, regs)) { 5212 5213 if (FTRACE_WARN_ON(!op->func)) { 5214 pr_warn("op=%p %pS\n", op, op); 5215 goto out; 5216 } 5217 op->func(ip, parent_ip, op, regs); 5218 } 5219 } while_for_each_ftrace_op(op); 5220 out: 5221 preempt_enable_notrace(); 5222 trace_clear_recursion(bit); 5223 } 5224 5225 /* 5226 * Some archs only support passing ip and parent_ip. Even though 5227 * the list function ignores the op parameter, we do not want any 5228 * C side effects, where a function is called without the caller 5229 * sending a third parameter. 5230 * Archs are to support both the regs and ftrace_ops at the same time. 5231 * If they support ftrace_ops, it is assumed they support regs. 5232 * If call backs want to use regs, they must either check for regs 5233 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS. 5234 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved. 5235 * An architecture can pass partial regs with ftrace_ops and still 5236 * set the ARCH_SUPPORTS_FTRACE_OPS. 5237 */ 5238 #if ARCH_SUPPORTS_FTRACE_OPS 5239 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 5240 struct ftrace_ops *op, struct pt_regs *regs) 5241 { 5242 __ftrace_ops_list_func(ip, parent_ip, NULL, regs); 5243 } 5244 #else 5245 static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip) 5246 { 5247 __ftrace_ops_list_func(ip, parent_ip, NULL, NULL); 5248 } 5249 #endif 5250 5251 /* 5252 * If there's only one function registered but it does not support 5253 * recursion, needs RCU protection and/or requires per cpu handling, then 5254 * this function will be called by the mcount trampoline. 5255 */ 5256 static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip, 5257 struct ftrace_ops *op, struct pt_regs *regs) 5258 { 5259 int bit; 5260 5261 if ((op->flags & FTRACE_OPS_FL_RCU) && !rcu_is_watching()) 5262 return; 5263 5264 bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX); 5265 if (bit < 0) 5266 return; 5267 5268 preempt_disable_notrace(); 5269 5270 if (!(op->flags & FTRACE_OPS_FL_PER_CPU) || 5271 !ftrace_function_local_disabled(op)) { 5272 op->func(ip, parent_ip, op, regs); 5273 } 5274 5275 preempt_enable_notrace(); 5276 trace_clear_recursion(bit); 5277 } 5278 5279 /** 5280 * ftrace_ops_get_func - get the function a trampoline should call 5281 * @ops: the ops to get the function for 5282 * 5283 * Normally the mcount trampoline will call the ops->func, but there 5284 * are times that it should not. For example, if the ops does not 5285 * have its own recursion protection, then it should call the 5286 * ftrace_ops_recurs_func() instead. 5287 * 5288 * Returns the function that the trampoline should call for @ops. 5289 */ 5290 ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops) 5291 { 5292 /* 5293 * If the function does not handle recursion, needs to be RCU safe, 5294 * or does per cpu logic, then we need to call the assist handler. 5295 */ 5296 if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE) || 5297 ops->flags & (FTRACE_OPS_FL_RCU | FTRACE_OPS_FL_PER_CPU)) 5298 return ftrace_ops_assist_func; 5299 5300 return ops->func; 5301 } 5302 5303 static void clear_ftrace_swapper(void) 5304 { 5305 struct task_struct *p; 5306 int cpu; 5307 5308 get_online_cpus(); 5309 for_each_online_cpu(cpu) { 5310 p = idle_task(cpu); 5311 clear_tsk_trace_trace(p); 5312 } 5313 put_online_cpus(); 5314 } 5315 5316 static void set_ftrace_swapper(void) 5317 { 5318 struct task_struct *p; 5319 int cpu; 5320 5321 get_online_cpus(); 5322 for_each_online_cpu(cpu) { 5323 p = idle_task(cpu); 5324 set_tsk_trace_trace(p); 5325 } 5326 put_online_cpus(); 5327 } 5328 5329 static void clear_ftrace_pid(struct pid *pid) 5330 { 5331 struct task_struct *p; 5332 5333 rcu_read_lock(); 5334 do_each_pid_task(pid, PIDTYPE_PID, p) { 5335 clear_tsk_trace_trace(p); 5336 } while_each_pid_task(pid, PIDTYPE_PID, p); 5337 rcu_read_unlock(); 5338 5339 put_pid(pid); 5340 } 5341 5342 static void set_ftrace_pid(struct pid *pid) 5343 { 5344 struct task_struct *p; 5345 5346 rcu_read_lock(); 5347 do_each_pid_task(pid, PIDTYPE_PID, p) { 5348 set_tsk_trace_trace(p); 5349 } while_each_pid_task(pid, PIDTYPE_PID, p); 5350 rcu_read_unlock(); 5351 } 5352 5353 static void clear_ftrace_pid_task(struct pid *pid) 5354 { 5355 if (pid == ftrace_swapper_pid) 5356 clear_ftrace_swapper(); 5357 else 5358 clear_ftrace_pid(pid); 5359 } 5360 5361 static void set_ftrace_pid_task(struct pid *pid) 5362 { 5363 if (pid == ftrace_swapper_pid) 5364 set_ftrace_swapper(); 5365 else 5366 set_ftrace_pid(pid); 5367 } 5368 5369 static int ftrace_pid_add(int p) 5370 { 5371 struct pid *pid; 5372 struct ftrace_pid *fpid; 5373 int ret = -EINVAL; 5374 5375 mutex_lock(&ftrace_lock); 5376 5377 if (!p) 5378 pid = ftrace_swapper_pid; 5379 else 5380 pid = find_get_pid(p); 5381 5382 if (!pid) 5383 goto out; 5384 5385 ret = 0; 5386 5387 list_for_each_entry(fpid, &ftrace_pids, list) 5388 if (fpid->pid == pid) 5389 goto out_put; 5390 5391 ret = -ENOMEM; 5392 5393 fpid = kmalloc(sizeof(*fpid), GFP_KERNEL); 5394 if (!fpid) 5395 goto out_put; 5396 5397 list_add(&fpid->list, &ftrace_pids); 5398 fpid->pid = pid; 5399 5400 set_ftrace_pid_task(pid); 5401 5402 ftrace_update_pid_func(); 5403 5404 ftrace_startup_all(0); 5405 5406 mutex_unlock(&ftrace_lock); 5407 return 0; 5408 5409 out_put: 5410 if (pid != ftrace_swapper_pid) 5411 put_pid(pid); 5412 5413 out: 5414 mutex_unlock(&ftrace_lock); 5415 return ret; 5416 } 5417 5418 static void ftrace_pid_reset(void) 5419 { 5420 struct ftrace_pid *fpid, *safe; 5421 5422 mutex_lock(&ftrace_lock); 5423 list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) { 5424 struct pid *pid = fpid->pid; 5425 5426 clear_ftrace_pid_task(pid); 5427 5428 list_del(&fpid->list); 5429 kfree(fpid); 5430 } 5431 5432 ftrace_update_pid_func(); 5433 ftrace_startup_all(0); 5434 5435 mutex_unlock(&ftrace_lock); 5436 } 5437 5438 static void *fpid_start(struct seq_file *m, loff_t *pos) 5439 { 5440 mutex_lock(&ftrace_lock); 5441 5442 if (!ftrace_pids_enabled() && (!*pos)) 5443 return (void *) 1; 5444 5445 return seq_list_start(&ftrace_pids, *pos); 5446 } 5447 5448 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos) 5449 { 5450 if (v == (void *)1) 5451 return NULL; 5452 5453 return seq_list_next(v, &ftrace_pids, pos); 5454 } 5455 5456 static void fpid_stop(struct seq_file *m, void *p) 5457 { 5458 mutex_unlock(&ftrace_lock); 5459 } 5460 5461 static int fpid_show(struct seq_file *m, void *v) 5462 { 5463 const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list); 5464 5465 if (v == (void *)1) { 5466 seq_puts(m, "no pid\n"); 5467 return 0; 5468 } 5469 5470 if (fpid->pid == ftrace_swapper_pid) 5471 seq_puts(m, "swapper tasks\n"); 5472 else 5473 seq_printf(m, "%u\n", pid_vnr(fpid->pid)); 5474 5475 return 0; 5476 } 5477 5478 static const struct seq_operations ftrace_pid_sops = { 5479 .start = fpid_start, 5480 .next = fpid_next, 5481 .stop = fpid_stop, 5482 .show = fpid_show, 5483 }; 5484 5485 static int 5486 ftrace_pid_open(struct inode *inode, struct file *file) 5487 { 5488 int ret = 0; 5489 5490 if ((file->f_mode & FMODE_WRITE) && 5491 (file->f_flags & O_TRUNC)) 5492 ftrace_pid_reset(); 5493 5494 if (file->f_mode & FMODE_READ) 5495 ret = seq_open(file, &ftrace_pid_sops); 5496 5497 return ret; 5498 } 5499 5500 static ssize_t 5501 ftrace_pid_write(struct file *filp, const char __user *ubuf, 5502 size_t cnt, loff_t *ppos) 5503 { 5504 char buf[64], *tmp; 5505 long val; 5506 int ret; 5507 5508 if (cnt >= sizeof(buf)) 5509 return -EINVAL; 5510 5511 if (copy_from_user(&buf, ubuf, cnt)) 5512 return -EFAULT; 5513 5514 buf[cnt] = 0; 5515 5516 /* 5517 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid" 5518 * to clean the filter quietly. 5519 */ 5520 tmp = strstrip(buf); 5521 if (strlen(tmp) == 0) 5522 return 1; 5523 5524 ret = kstrtol(tmp, 10, &val); 5525 if (ret < 0) 5526 return ret; 5527 5528 ret = ftrace_pid_add(val); 5529 5530 return ret ? ret : cnt; 5531 } 5532 5533 static int 5534 ftrace_pid_release(struct inode *inode, struct file *file) 5535 { 5536 if (file->f_mode & FMODE_READ) 5537 seq_release(inode, file); 5538 5539 return 0; 5540 } 5541 5542 static const struct file_operations ftrace_pid_fops = { 5543 .open = ftrace_pid_open, 5544 .write = ftrace_pid_write, 5545 .read = seq_read, 5546 .llseek = tracing_lseek, 5547 .release = ftrace_pid_release, 5548 }; 5549 5550 static __init int ftrace_init_tracefs(void) 5551 { 5552 struct dentry *d_tracer; 5553 5554 d_tracer = tracing_init_dentry(); 5555 if (IS_ERR(d_tracer)) 5556 return 0; 5557 5558 ftrace_init_dyn_tracefs(d_tracer); 5559 5560 trace_create_file("set_ftrace_pid", 0644, d_tracer, 5561 NULL, &ftrace_pid_fops); 5562 5563 ftrace_profile_tracefs(d_tracer); 5564 5565 return 0; 5566 } 5567 fs_initcall(ftrace_init_tracefs); 5568 5569 /** 5570 * ftrace_kill - kill ftrace 5571 * 5572 * This function should be used by panic code. It stops ftrace 5573 * but in a not so nice way. If you need to simply kill ftrace 5574 * from a non-atomic section, use ftrace_kill. 5575 */ 5576 void ftrace_kill(void) 5577 { 5578 ftrace_disabled = 1; 5579 ftrace_enabled = 0; 5580 clear_ftrace_function(); 5581 } 5582 5583 /** 5584 * Test if ftrace is dead or not. 5585 */ 5586 int ftrace_is_dead(void) 5587 { 5588 return ftrace_disabled; 5589 } 5590 5591 /** 5592 * register_ftrace_function - register a function for profiling 5593 * @ops - ops structure that holds the function for profiling. 5594 * 5595 * Register a function to be called by all functions in the 5596 * kernel. 5597 * 5598 * Note: @ops->func and all the functions it calls must be labeled 5599 * with "notrace", otherwise it will go into a 5600 * recursive loop. 5601 */ 5602 int register_ftrace_function(struct ftrace_ops *ops) 5603 { 5604 int ret = -1; 5605 5606 ftrace_ops_init(ops); 5607 5608 mutex_lock(&ftrace_lock); 5609 5610 ret = ftrace_startup(ops, 0); 5611 5612 mutex_unlock(&ftrace_lock); 5613 5614 return ret; 5615 } 5616 EXPORT_SYMBOL_GPL(register_ftrace_function); 5617 5618 /** 5619 * unregister_ftrace_function - unregister a function for profiling. 5620 * @ops - ops structure that holds the function to unregister 5621 * 5622 * Unregister a function that was added to be called by ftrace profiling. 5623 */ 5624 int unregister_ftrace_function(struct ftrace_ops *ops) 5625 { 5626 int ret; 5627 5628 mutex_lock(&ftrace_lock); 5629 ret = ftrace_shutdown(ops, 0); 5630 mutex_unlock(&ftrace_lock); 5631 5632 return ret; 5633 } 5634 EXPORT_SYMBOL_GPL(unregister_ftrace_function); 5635 5636 int 5637 ftrace_enable_sysctl(struct ctl_table *table, int write, 5638 void __user *buffer, size_t *lenp, 5639 loff_t *ppos) 5640 { 5641 int ret = -ENODEV; 5642 5643 mutex_lock(&ftrace_lock); 5644 5645 if (unlikely(ftrace_disabled)) 5646 goto out; 5647 5648 ret = proc_dointvec(table, write, buffer, lenp, ppos); 5649 5650 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled)) 5651 goto out; 5652 5653 last_ftrace_enabled = !!ftrace_enabled; 5654 5655 if (ftrace_enabled) { 5656 5657 /* we are starting ftrace again */ 5658 if (ftrace_ops_list != &ftrace_list_end) 5659 update_ftrace_function(); 5660 5661 ftrace_startup_sysctl(); 5662 5663 } else { 5664 /* stopping ftrace calls (just send to ftrace_stub) */ 5665 ftrace_trace_function = ftrace_stub; 5666 5667 ftrace_shutdown_sysctl(); 5668 } 5669 5670 out: 5671 mutex_unlock(&ftrace_lock); 5672 return ret; 5673 } 5674 5675 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 5676 5677 static struct ftrace_ops graph_ops = { 5678 .func = ftrace_stub, 5679 .flags = FTRACE_OPS_FL_RECURSION_SAFE | 5680 FTRACE_OPS_FL_INITIALIZED | 5681 FTRACE_OPS_FL_PID | 5682 FTRACE_OPS_FL_STUB, 5683 #ifdef FTRACE_GRAPH_TRAMP_ADDR 5684 .trampoline = FTRACE_GRAPH_TRAMP_ADDR, 5685 /* trampoline_size is only needed for dynamically allocated tramps */ 5686 #endif 5687 ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash) 5688 }; 5689 5690 void ftrace_graph_sleep_time_control(bool enable) 5691 { 5692 fgraph_sleep_time = enable; 5693 } 5694 5695 void ftrace_graph_graph_time_control(bool enable) 5696 { 5697 fgraph_graph_time = enable; 5698 } 5699 5700 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) 5701 { 5702 return 0; 5703 } 5704 5705 /* The callbacks that hook a function */ 5706 trace_func_graph_ret_t ftrace_graph_return = 5707 (trace_func_graph_ret_t)ftrace_stub; 5708 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub; 5709 static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub; 5710 5711 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */ 5712 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list) 5713 { 5714 int i; 5715 int ret = 0; 5716 unsigned long flags; 5717 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE; 5718 struct task_struct *g, *t; 5719 5720 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) { 5721 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH 5722 * sizeof(struct ftrace_ret_stack), 5723 GFP_KERNEL); 5724 if (!ret_stack_list[i]) { 5725 start = 0; 5726 end = i; 5727 ret = -ENOMEM; 5728 goto free; 5729 } 5730 } 5731 5732 read_lock_irqsave(&tasklist_lock, flags); 5733 do_each_thread(g, t) { 5734 if (start == end) { 5735 ret = -EAGAIN; 5736 goto unlock; 5737 } 5738 5739 if (t->ret_stack == NULL) { 5740 atomic_set(&t->tracing_graph_pause, 0); 5741 atomic_set(&t->trace_overrun, 0); 5742 t->curr_ret_stack = -1; 5743 /* Make sure the tasks see the -1 first: */ 5744 smp_wmb(); 5745 t->ret_stack = ret_stack_list[start++]; 5746 } 5747 } while_each_thread(g, t); 5748 5749 unlock: 5750 read_unlock_irqrestore(&tasklist_lock, flags); 5751 free: 5752 for (i = start; i < end; i++) 5753 kfree(ret_stack_list[i]); 5754 return ret; 5755 } 5756 5757 static void 5758 ftrace_graph_probe_sched_switch(void *ignore, bool preempt, 5759 struct task_struct *prev, struct task_struct *next) 5760 { 5761 unsigned long long timestamp; 5762 int index; 5763 5764 /* 5765 * Does the user want to count the time a function was asleep. 5766 * If so, do not update the time stamps. 5767 */ 5768 if (fgraph_sleep_time) 5769 return; 5770 5771 timestamp = trace_clock_local(); 5772 5773 prev->ftrace_timestamp = timestamp; 5774 5775 /* only process tasks that we timestamped */ 5776 if (!next->ftrace_timestamp) 5777 return; 5778 5779 /* 5780 * Update all the counters in next to make up for the 5781 * time next was sleeping. 5782 */ 5783 timestamp -= next->ftrace_timestamp; 5784 5785 for (index = next->curr_ret_stack; index >= 0; index--) 5786 next->ret_stack[index].calltime += timestamp; 5787 } 5788 5789 /* Allocate a return stack for each task */ 5790 static int start_graph_tracing(void) 5791 { 5792 struct ftrace_ret_stack **ret_stack_list; 5793 int ret, cpu; 5794 5795 ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE * 5796 sizeof(struct ftrace_ret_stack *), 5797 GFP_KERNEL); 5798 5799 if (!ret_stack_list) 5800 return -ENOMEM; 5801 5802 /* The cpu_boot init_task->ret_stack will never be freed */ 5803 for_each_online_cpu(cpu) { 5804 if (!idle_task(cpu)->ret_stack) 5805 ftrace_graph_init_idle_task(idle_task(cpu), cpu); 5806 } 5807 5808 do { 5809 ret = alloc_retstack_tasklist(ret_stack_list); 5810 } while (ret == -EAGAIN); 5811 5812 if (!ret) { 5813 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); 5814 if (ret) 5815 pr_info("ftrace_graph: Couldn't activate tracepoint" 5816 " probe to kernel_sched_switch\n"); 5817 } 5818 5819 kfree(ret_stack_list); 5820 return ret; 5821 } 5822 5823 /* 5824 * Hibernation protection. 5825 * The state of the current task is too much unstable during 5826 * suspend/restore to disk. We want to protect against that. 5827 */ 5828 static int 5829 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state, 5830 void *unused) 5831 { 5832 switch (state) { 5833 case PM_HIBERNATION_PREPARE: 5834 pause_graph_tracing(); 5835 break; 5836 5837 case PM_POST_HIBERNATION: 5838 unpause_graph_tracing(); 5839 break; 5840 } 5841 return NOTIFY_DONE; 5842 } 5843 5844 static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace) 5845 { 5846 if (!ftrace_ops_test(&global_ops, trace->func, NULL)) 5847 return 0; 5848 return __ftrace_graph_entry(trace); 5849 } 5850 5851 /* 5852 * The function graph tracer should only trace the functions defined 5853 * by set_ftrace_filter and set_ftrace_notrace. If another function 5854 * tracer ops is registered, the graph tracer requires testing the 5855 * function against the global ops, and not just trace any function 5856 * that any ftrace_ops registered. 5857 */ 5858 static void update_function_graph_func(void) 5859 { 5860 struct ftrace_ops *op; 5861 bool do_test = false; 5862 5863 /* 5864 * The graph and global ops share the same set of functions 5865 * to test. If any other ops is on the list, then 5866 * the graph tracing needs to test if its the function 5867 * it should call. 5868 */ 5869 do_for_each_ftrace_op(op, ftrace_ops_list) { 5870 if (op != &global_ops && op != &graph_ops && 5871 op != &ftrace_list_end) { 5872 do_test = true; 5873 /* in double loop, break out with goto */ 5874 goto out; 5875 } 5876 } while_for_each_ftrace_op(op); 5877 out: 5878 if (do_test) 5879 ftrace_graph_entry = ftrace_graph_entry_test; 5880 else 5881 ftrace_graph_entry = __ftrace_graph_entry; 5882 } 5883 5884 static struct notifier_block ftrace_suspend_notifier = { 5885 .notifier_call = ftrace_suspend_notifier_call, 5886 }; 5887 5888 int register_ftrace_graph(trace_func_graph_ret_t retfunc, 5889 trace_func_graph_ent_t entryfunc) 5890 { 5891 int ret = 0; 5892 5893 mutex_lock(&ftrace_lock); 5894 5895 /* we currently allow only one tracer registered at a time */ 5896 if (ftrace_graph_active) { 5897 ret = -EBUSY; 5898 goto out; 5899 } 5900 5901 register_pm_notifier(&ftrace_suspend_notifier); 5902 5903 ftrace_graph_active++; 5904 ret = start_graph_tracing(); 5905 if (ret) { 5906 ftrace_graph_active--; 5907 goto out; 5908 } 5909 5910 ftrace_graph_return = retfunc; 5911 5912 /* 5913 * Update the indirect function to the entryfunc, and the 5914 * function that gets called to the entry_test first. Then 5915 * call the update fgraph entry function to determine if 5916 * the entryfunc should be called directly or not. 5917 */ 5918 __ftrace_graph_entry = entryfunc; 5919 ftrace_graph_entry = ftrace_graph_entry_test; 5920 update_function_graph_func(); 5921 5922 ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET); 5923 out: 5924 mutex_unlock(&ftrace_lock); 5925 return ret; 5926 } 5927 5928 void unregister_ftrace_graph(void) 5929 { 5930 mutex_lock(&ftrace_lock); 5931 5932 if (unlikely(!ftrace_graph_active)) 5933 goto out; 5934 5935 ftrace_graph_active--; 5936 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; 5937 ftrace_graph_entry = ftrace_graph_entry_stub; 5938 __ftrace_graph_entry = ftrace_graph_entry_stub; 5939 ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET); 5940 unregister_pm_notifier(&ftrace_suspend_notifier); 5941 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); 5942 5943 #ifdef CONFIG_DYNAMIC_FTRACE 5944 /* 5945 * Function graph does not allocate the trampoline, but 5946 * other global_ops do. We need to reset the ALLOC_TRAMP flag 5947 * if one was used. 5948 */ 5949 global_ops.trampoline = save_global_trampoline; 5950 if (save_global_flags & FTRACE_OPS_FL_ALLOC_TRAMP) 5951 global_ops.flags |= FTRACE_OPS_FL_ALLOC_TRAMP; 5952 #endif 5953 5954 out: 5955 mutex_unlock(&ftrace_lock); 5956 } 5957 5958 static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack); 5959 5960 static void 5961 graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack) 5962 { 5963 atomic_set(&t->tracing_graph_pause, 0); 5964 atomic_set(&t->trace_overrun, 0); 5965 t->ftrace_timestamp = 0; 5966 /* make curr_ret_stack visible before we add the ret_stack */ 5967 smp_wmb(); 5968 t->ret_stack = ret_stack; 5969 } 5970 5971 /* 5972 * Allocate a return stack for the idle task. May be the first 5973 * time through, or it may be done by CPU hotplug online. 5974 */ 5975 void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) 5976 { 5977 t->curr_ret_stack = -1; 5978 /* 5979 * The idle task has no parent, it either has its own 5980 * stack or no stack at all. 5981 */ 5982 if (t->ret_stack) 5983 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu)); 5984 5985 if (ftrace_graph_active) { 5986 struct ftrace_ret_stack *ret_stack; 5987 5988 ret_stack = per_cpu(idle_ret_stack, cpu); 5989 if (!ret_stack) { 5990 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH 5991 * sizeof(struct ftrace_ret_stack), 5992 GFP_KERNEL); 5993 if (!ret_stack) 5994 return; 5995 per_cpu(idle_ret_stack, cpu) = ret_stack; 5996 } 5997 graph_init_task(t, ret_stack); 5998 } 5999 } 6000 6001 /* Allocate a return stack for newly created task */ 6002 void ftrace_graph_init_task(struct task_struct *t) 6003 { 6004 /* Make sure we do not use the parent ret_stack */ 6005 t->ret_stack = NULL; 6006 t->curr_ret_stack = -1; 6007 6008 if (ftrace_graph_active) { 6009 struct ftrace_ret_stack *ret_stack; 6010 6011 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH 6012 * sizeof(struct ftrace_ret_stack), 6013 GFP_KERNEL); 6014 if (!ret_stack) 6015 return; 6016 graph_init_task(t, ret_stack); 6017 } 6018 } 6019 6020 void ftrace_graph_exit_task(struct task_struct *t) 6021 { 6022 struct ftrace_ret_stack *ret_stack = t->ret_stack; 6023 6024 t->ret_stack = NULL; 6025 /* NULL must become visible to IRQs before we free it: */ 6026 barrier(); 6027 6028 kfree(ret_stack); 6029 } 6030 #endif 6031