1 /* 2 * Infrastructure for profiling code inserted by 'gcc -pg'. 3 * 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> 6 * 7 * Originally ported from the -rt patch by: 8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> 9 * 10 * Based on code in the latency_tracer, that is: 11 * 12 * Copyright (C) 2004-2006 Ingo Molnar 13 * Copyright (C) 2004 Nadia Yvette Chambers 14 */ 15 16 #include <linux/stop_machine.h> 17 #include <linux/clocksource.h> 18 #include <linux/kallsyms.h> 19 #include <linux/seq_file.h> 20 #include <linux/suspend.h> 21 #include <linux/tracefs.h> 22 #include <linux/hardirq.h> 23 #include <linux/kthread.h> 24 #include <linux/uaccess.h> 25 #include <linux/bsearch.h> 26 #include <linux/module.h> 27 #include <linux/ftrace.h> 28 #include <linux/sysctl.h> 29 #include <linux/slab.h> 30 #include <linux/ctype.h> 31 #include <linux/sort.h> 32 #include <linux/list.h> 33 #include <linux/hash.h> 34 #include <linux/rcupdate.h> 35 36 #include <trace/events/sched.h> 37 38 #include <asm/setup.h> 39 40 #include "trace_output.h" 41 #include "trace_stat.h" 42 43 #define FTRACE_WARN_ON(cond) \ 44 ({ \ 45 int ___r = cond; \ 46 if (WARN_ON(___r)) \ 47 ftrace_kill(); \ 48 ___r; \ 49 }) 50 51 #define FTRACE_WARN_ON_ONCE(cond) \ 52 ({ \ 53 int ___r = cond; \ 54 if (WARN_ON_ONCE(___r)) \ 55 ftrace_kill(); \ 56 ___r; \ 57 }) 58 59 /* hash bits for specific function selection */ 60 #define FTRACE_HASH_BITS 7 61 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS) 62 #define FTRACE_HASH_DEFAULT_BITS 10 63 #define FTRACE_HASH_MAX_BITS 12 64 65 #ifdef CONFIG_DYNAMIC_FTRACE 66 #define INIT_OPS_HASH(opsname) \ 67 .func_hash = &opsname.local_hash, \ 68 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), 69 #define ASSIGN_OPS_HASH(opsname, val) \ 70 .func_hash = val, \ 71 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), 72 #else 73 #define INIT_OPS_HASH(opsname) 74 #define ASSIGN_OPS_HASH(opsname, val) 75 #endif 76 77 static struct ftrace_ops ftrace_list_end __read_mostly = { 78 .func = ftrace_stub, 79 .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB, 80 INIT_OPS_HASH(ftrace_list_end) 81 }; 82 83 /* ftrace_enabled is a method to turn ftrace on or off */ 84 int ftrace_enabled __read_mostly; 85 static int last_ftrace_enabled; 86 87 /* Current function tracing op */ 88 struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end; 89 /* What to set function_trace_op to */ 90 static struct ftrace_ops *set_function_trace_op; 91 92 static bool ftrace_pids_enabled(struct ftrace_ops *ops) 93 { 94 struct trace_array *tr; 95 96 if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private) 97 return false; 98 99 tr = ops->private; 100 101 return tr->function_pids != NULL; 102 } 103 104 static void ftrace_update_trampoline(struct ftrace_ops *ops); 105 106 /* 107 * ftrace_disabled is set when an anomaly is discovered. 108 * ftrace_disabled is much stronger than ftrace_enabled. 109 */ 110 static int ftrace_disabled __read_mostly; 111 112 static DEFINE_MUTEX(ftrace_lock); 113 114 static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end; 115 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; 116 static struct ftrace_ops global_ops; 117 118 #if ARCH_SUPPORTS_FTRACE_OPS 119 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 120 struct ftrace_ops *op, struct pt_regs *regs); 121 #else 122 /* See comment below, where ftrace_ops_list_func is defined */ 123 static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip); 124 #define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops) 125 #endif 126 127 /* 128 * Traverse the ftrace_global_list, invoking all entries. The reason that we 129 * can use rcu_dereference_raw_notrace() is that elements removed from this list 130 * are simply leaked, so there is no need to interact with a grace-period 131 * mechanism. The rcu_dereference_raw_notrace() calls are needed to handle 132 * concurrent insertions into the ftrace_global_list. 133 * 134 * Silly Alpha and silly pointer-speculation compiler optimizations! 135 */ 136 #define do_for_each_ftrace_op(op, list) \ 137 op = rcu_dereference_raw_notrace(list); \ 138 do 139 140 /* 141 * Optimized for just a single item in the list (as that is the normal case). 142 */ 143 #define while_for_each_ftrace_op(op) \ 144 while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \ 145 unlikely((op) != &ftrace_list_end)) 146 147 static inline void ftrace_ops_init(struct ftrace_ops *ops) 148 { 149 #ifdef CONFIG_DYNAMIC_FTRACE 150 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { 151 mutex_init(&ops->local_hash.regex_lock); 152 ops->func_hash = &ops->local_hash; 153 ops->flags |= FTRACE_OPS_FL_INITIALIZED; 154 } 155 #endif 156 } 157 158 /** 159 * ftrace_nr_registered_ops - return number of ops registered 160 * 161 * Returns the number of ftrace_ops registered and tracing functions 162 */ 163 int ftrace_nr_registered_ops(void) 164 { 165 struct ftrace_ops *ops; 166 int cnt = 0; 167 168 mutex_lock(&ftrace_lock); 169 170 for (ops = ftrace_ops_list; 171 ops != &ftrace_list_end; ops = ops->next) 172 cnt++; 173 174 mutex_unlock(&ftrace_lock); 175 176 return cnt; 177 } 178 179 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip, 180 struct ftrace_ops *op, struct pt_regs *regs) 181 { 182 struct trace_array *tr = op->private; 183 184 if (tr && this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid)) 185 return; 186 187 op->saved_func(ip, parent_ip, op, regs); 188 } 189 190 /** 191 * clear_ftrace_function - reset the ftrace function 192 * 193 * This NULLs the ftrace function and in essence stops 194 * tracing. There may be lag 195 */ 196 void clear_ftrace_function(void) 197 { 198 ftrace_trace_function = ftrace_stub; 199 } 200 201 static void per_cpu_ops_disable_all(struct ftrace_ops *ops) 202 { 203 int cpu; 204 205 for_each_possible_cpu(cpu) 206 *per_cpu_ptr(ops->disabled, cpu) = 1; 207 } 208 209 static int per_cpu_ops_alloc(struct ftrace_ops *ops) 210 { 211 int __percpu *disabled; 212 213 if (WARN_ON_ONCE(!(ops->flags & FTRACE_OPS_FL_PER_CPU))) 214 return -EINVAL; 215 216 disabled = alloc_percpu(int); 217 if (!disabled) 218 return -ENOMEM; 219 220 ops->disabled = disabled; 221 per_cpu_ops_disable_all(ops); 222 return 0; 223 } 224 225 static void ftrace_sync(struct work_struct *work) 226 { 227 /* 228 * This function is just a stub to implement a hard force 229 * of synchronize_sched(). This requires synchronizing 230 * tasks even in userspace and idle. 231 * 232 * Yes, function tracing is rude. 233 */ 234 } 235 236 static void ftrace_sync_ipi(void *data) 237 { 238 /* Probably not needed, but do it anyway */ 239 smp_rmb(); 240 } 241 242 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 243 static void update_function_graph_func(void); 244 245 /* Both enabled by default (can be cleared by function_graph tracer flags */ 246 static bool fgraph_sleep_time = true; 247 static bool fgraph_graph_time = true; 248 249 #else 250 static inline void update_function_graph_func(void) { } 251 #endif 252 253 254 static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops) 255 { 256 /* 257 * If this is a dynamic, RCU, or per CPU ops, or we force list func, 258 * then it needs to call the list anyway. 259 */ 260 if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU | 261 FTRACE_OPS_FL_RCU) || FTRACE_FORCE_LIST_FUNC) 262 return ftrace_ops_list_func; 263 264 return ftrace_ops_get_func(ops); 265 } 266 267 static void update_ftrace_function(void) 268 { 269 ftrace_func_t func; 270 271 /* 272 * Prepare the ftrace_ops that the arch callback will use. 273 * If there's only one ftrace_ops registered, the ftrace_ops_list 274 * will point to the ops we want. 275 */ 276 set_function_trace_op = ftrace_ops_list; 277 278 /* If there's no ftrace_ops registered, just call the stub function */ 279 if (ftrace_ops_list == &ftrace_list_end) { 280 func = ftrace_stub; 281 282 /* 283 * If we are at the end of the list and this ops is 284 * recursion safe and not dynamic and the arch supports passing ops, 285 * then have the mcount trampoline call the function directly. 286 */ 287 } else if (ftrace_ops_list->next == &ftrace_list_end) { 288 func = ftrace_ops_get_list_func(ftrace_ops_list); 289 290 } else { 291 /* Just use the default ftrace_ops */ 292 set_function_trace_op = &ftrace_list_end; 293 func = ftrace_ops_list_func; 294 } 295 296 update_function_graph_func(); 297 298 /* If there's no change, then do nothing more here */ 299 if (ftrace_trace_function == func) 300 return; 301 302 /* 303 * If we are using the list function, it doesn't care 304 * about the function_trace_ops. 305 */ 306 if (func == ftrace_ops_list_func) { 307 ftrace_trace_function = func; 308 /* 309 * Don't even bother setting function_trace_ops, 310 * it would be racy to do so anyway. 311 */ 312 return; 313 } 314 315 #ifndef CONFIG_DYNAMIC_FTRACE 316 /* 317 * For static tracing, we need to be a bit more careful. 318 * The function change takes affect immediately. Thus, 319 * we need to coorditate the setting of the function_trace_ops 320 * with the setting of the ftrace_trace_function. 321 * 322 * Set the function to the list ops, which will call the 323 * function we want, albeit indirectly, but it handles the 324 * ftrace_ops and doesn't depend on function_trace_op. 325 */ 326 ftrace_trace_function = ftrace_ops_list_func; 327 /* 328 * Make sure all CPUs see this. Yes this is slow, but static 329 * tracing is slow and nasty to have enabled. 330 */ 331 schedule_on_each_cpu(ftrace_sync); 332 /* Now all cpus are using the list ops. */ 333 function_trace_op = set_function_trace_op; 334 /* Make sure the function_trace_op is visible on all CPUs */ 335 smp_wmb(); 336 /* Nasty way to force a rmb on all cpus */ 337 smp_call_function(ftrace_sync_ipi, NULL, 1); 338 /* OK, we are all set to update the ftrace_trace_function now! */ 339 #endif /* !CONFIG_DYNAMIC_FTRACE */ 340 341 ftrace_trace_function = func; 342 } 343 344 int using_ftrace_ops_list_func(void) 345 { 346 return ftrace_trace_function == ftrace_ops_list_func; 347 } 348 349 static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops) 350 { 351 ops->next = *list; 352 /* 353 * We are entering ops into the list but another 354 * CPU might be walking that list. We need to make sure 355 * the ops->next pointer is valid before another CPU sees 356 * the ops pointer included into the list. 357 */ 358 rcu_assign_pointer(*list, ops); 359 } 360 361 static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops) 362 { 363 struct ftrace_ops **p; 364 365 /* 366 * If we are removing the last function, then simply point 367 * to the ftrace_stub. 368 */ 369 if (*list == ops && ops->next == &ftrace_list_end) { 370 *list = &ftrace_list_end; 371 return 0; 372 } 373 374 for (p = list; *p != &ftrace_list_end; p = &(*p)->next) 375 if (*p == ops) 376 break; 377 378 if (*p != ops) 379 return -1; 380 381 *p = (*p)->next; 382 return 0; 383 } 384 385 static void ftrace_update_trampoline(struct ftrace_ops *ops); 386 387 static int __register_ftrace_function(struct ftrace_ops *ops) 388 { 389 if (ops->flags & FTRACE_OPS_FL_DELETED) 390 return -EINVAL; 391 392 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) 393 return -EBUSY; 394 395 #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS 396 /* 397 * If the ftrace_ops specifies SAVE_REGS, then it only can be used 398 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set. 399 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant. 400 */ 401 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS && 402 !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)) 403 return -EINVAL; 404 405 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED) 406 ops->flags |= FTRACE_OPS_FL_SAVE_REGS; 407 #endif 408 409 if (!core_kernel_data((unsigned long)ops)) 410 ops->flags |= FTRACE_OPS_FL_DYNAMIC; 411 412 if (ops->flags & FTRACE_OPS_FL_PER_CPU) { 413 if (per_cpu_ops_alloc(ops)) 414 return -ENOMEM; 415 } 416 417 add_ftrace_ops(&ftrace_ops_list, ops); 418 419 /* Always save the function, and reset at unregistering */ 420 ops->saved_func = ops->func; 421 422 if (ftrace_pids_enabled(ops)) 423 ops->func = ftrace_pid_func; 424 425 ftrace_update_trampoline(ops); 426 427 if (ftrace_enabled) 428 update_ftrace_function(); 429 430 return 0; 431 } 432 433 static int __unregister_ftrace_function(struct ftrace_ops *ops) 434 { 435 int ret; 436 437 if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED))) 438 return -EBUSY; 439 440 ret = remove_ftrace_ops(&ftrace_ops_list, ops); 441 442 if (ret < 0) 443 return ret; 444 445 if (ftrace_enabled) 446 update_ftrace_function(); 447 448 ops->func = ops->saved_func; 449 450 return 0; 451 } 452 453 static void ftrace_update_pid_func(void) 454 { 455 struct ftrace_ops *op; 456 457 /* Only do something if we are tracing something */ 458 if (ftrace_trace_function == ftrace_stub) 459 return; 460 461 do_for_each_ftrace_op(op, ftrace_ops_list) { 462 if (op->flags & FTRACE_OPS_FL_PID) { 463 op->func = ftrace_pids_enabled(op) ? 464 ftrace_pid_func : op->saved_func; 465 ftrace_update_trampoline(op); 466 } 467 } while_for_each_ftrace_op(op); 468 469 update_ftrace_function(); 470 } 471 472 #ifdef CONFIG_FUNCTION_PROFILER 473 struct ftrace_profile { 474 struct hlist_node node; 475 unsigned long ip; 476 unsigned long counter; 477 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 478 unsigned long long time; 479 unsigned long long time_squared; 480 #endif 481 }; 482 483 struct ftrace_profile_page { 484 struct ftrace_profile_page *next; 485 unsigned long index; 486 struct ftrace_profile records[]; 487 }; 488 489 struct ftrace_profile_stat { 490 atomic_t disabled; 491 struct hlist_head *hash; 492 struct ftrace_profile_page *pages; 493 struct ftrace_profile_page *start; 494 struct tracer_stat stat; 495 }; 496 497 #define PROFILE_RECORDS_SIZE \ 498 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records)) 499 500 #define PROFILES_PER_PAGE \ 501 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile)) 502 503 static int ftrace_profile_enabled __read_mostly; 504 505 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */ 506 static DEFINE_MUTEX(ftrace_profile_lock); 507 508 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats); 509 510 #define FTRACE_PROFILE_HASH_BITS 10 511 #define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS) 512 513 static void * 514 function_stat_next(void *v, int idx) 515 { 516 struct ftrace_profile *rec = v; 517 struct ftrace_profile_page *pg; 518 519 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); 520 521 again: 522 if (idx != 0) 523 rec++; 524 525 if ((void *)rec >= (void *)&pg->records[pg->index]) { 526 pg = pg->next; 527 if (!pg) 528 return NULL; 529 rec = &pg->records[0]; 530 if (!rec->counter) 531 goto again; 532 } 533 534 return rec; 535 } 536 537 static void *function_stat_start(struct tracer_stat *trace) 538 { 539 struct ftrace_profile_stat *stat = 540 container_of(trace, struct ftrace_profile_stat, stat); 541 542 if (!stat || !stat->start) 543 return NULL; 544 545 return function_stat_next(&stat->start->records[0], 0); 546 } 547 548 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 549 /* function graph compares on total time */ 550 static int function_stat_cmp(void *p1, void *p2) 551 { 552 struct ftrace_profile *a = p1; 553 struct ftrace_profile *b = p2; 554 555 if (a->time < b->time) 556 return -1; 557 if (a->time > b->time) 558 return 1; 559 else 560 return 0; 561 } 562 #else 563 /* not function graph compares against hits */ 564 static int function_stat_cmp(void *p1, void *p2) 565 { 566 struct ftrace_profile *a = p1; 567 struct ftrace_profile *b = p2; 568 569 if (a->counter < b->counter) 570 return -1; 571 if (a->counter > b->counter) 572 return 1; 573 else 574 return 0; 575 } 576 #endif 577 578 static int function_stat_headers(struct seq_file *m) 579 { 580 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 581 seq_puts(m, " Function " 582 "Hit Time Avg s^2\n" 583 " -------- " 584 "--- ---- --- ---\n"); 585 #else 586 seq_puts(m, " Function Hit\n" 587 " -------- ---\n"); 588 #endif 589 return 0; 590 } 591 592 static int function_stat_show(struct seq_file *m, void *v) 593 { 594 struct ftrace_profile *rec = v; 595 char str[KSYM_SYMBOL_LEN]; 596 int ret = 0; 597 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 598 static struct trace_seq s; 599 unsigned long long avg; 600 unsigned long long stddev; 601 #endif 602 mutex_lock(&ftrace_profile_lock); 603 604 /* we raced with function_profile_reset() */ 605 if (unlikely(rec->counter == 0)) { 606 ret = -EBUSY; 607 goto out; 608 } 609 610 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 611 avg = rec->time; 612 do_div(avg, rec->counter); 613 if (tracing_thresh && (avg < tracing_thresh)) 614 goto out; 615 #endif 616 617 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 618 seq_printf(m, " %-30.30s %10lu", str, rec->counter); 619 620 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 621 seq_puts(m, " "); 622 623 /* Sample standard deviation (s^2) */ 624 if (rec->counter <= 1) 625 stddev = 0; 626 else { 627 /* 628 * Apply Welford's method: 629 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2) 630 */ 631 stddev = rec->counter * rec->time_squared - 632 rec->time * rec->time; 633 634 /* 635 * Divide only 1000 for ns^2 -> us^2 conversion. 636 * trace_print_graph_duration will divide 1000 again. 637 */ 638 do_div(stddev, rec->counter * (rec->counter - 1) * 1000); 639 } 640 641 trace_seq_init(&s); 642 trace_print_graph_duration(rec->time, &s); 643 trace_seq_puts(&s, " "); 644 trace_print_graph_duration(avg, &s); 645 trace_seq_puts(&s, " "); 646 trace_print_graph_duration(stddev, &s); 647 trace_print_seq(m, &s); 648 #endif 649 seq_putc(m, '\n'); 650 out: 651 mutex_unlock(&ftrace_profile_lock); 652 653 return ret; 654 } 655 656 static void ftrace_profile_reset(struct ftrace_profile_stat *stat) 657 { 658 struct ftrace_profile_page *pg; 659 660 pg = stat->pages = stat->start; 661 662 while (pg) { 663 memset(pg->records, 0, PROFILE_RECORDS_SIZE); 664 pg->index = 0; 665 pg = pg->next; 666 } 667 668 memset(stat->hash, 0, 669 FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head)); 670 } 671 672 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat) 673 { 674 struct ftrace_profile_page *pg; 675 int functions; 676 int pages; 677 int i; 678 679 /* If we already allocated, do nothing */ 680 if (stat->pages) 681 return 0; 682 683 stat->pages = (void *)get_zeroed_page(GFP_KERNEL); 684 if (!stat->pages) 685 return -ENOMEM; 686 687 #ifdef CONFIG_DYNAMIC_FTRACE 688 functions = ftrace_update_tot_cnt; 689 #else 690 /* 691 * We do not know the number of functions that exist because 692 * dynamic tracing is what counts them. With past experience 693 * we have around 20K functions. That should be more than enough. 694 * It is highly unlikely we will execute every function in 695 * the kernel. 696 */ 697 functions = 20000; 698 #endif 699 700 pg = stat->start = stat->pages; 701 702 pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE); 703 704 for (i = 1; i < pages; i++) { 705 pg->next = (void *)get_zeroed_page(GFP_KERNEL); 706 if (!pg->next) 707 goto out_free; 708 pg = pg->next; 709 } 710 711 return 0; 712 713 out_free: 714 pg = stat->start; 715 while (pg) { 716 unsigned long tmp = (unsigned long)pg; 717 718 pg = pg->next; 719 free_page(tmp); 720 } 721 722 stat->pages = NULL; 723 stat->start = NULL; 724 725 return -ENOMEM; 726 } 727 728 static int ftrace_profile_init_cpu(int cpu) 729 { 730 struct ftrace_profile_stat *stat; 731 int size; 732 733 stat = &per_cpu(ftrace_profile_stats, cpu); 734 735 if (stat->hash) { 736 /* If the profile is already created, simply reset it */ 737 ftrace_profile_reset(stat); 738 return 0; 739 } 740 741 /* 742 * We are profiling all functions, but usually only a few thousand 743 * functions are hit. We'll make a hash of 1024 items. 744 */ 745 size = FTRACE_PROFILE_HASH_SIZE; 746 747 stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL); 748 749 if (!stat->hash) 750 return -ENOMEM; 751 752 /* Preallocate the function profiling pages */ 753 if (ftrace_profile_pages_init(stat) < 0) { 754 kfree(stat->hash); 755 stat->hash = NULL; 756 return -ENOMEM; 757 } 758 759 return 0; 760 } 761 762 static int ftrace_profile_init(void) 763 { 764 int cpu; 765 int ret = 0; 766 767 for_each_possible_cpu(cpu) { 768 ret = ftrace_profile_init_cpu(cpu); 769 if (ret) 770 break; 771 } 772 773 return ret; 774 } 775 776 /* interrupts must be disabled */ 777 static struct ftrace_profile * 778 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip) 779 { 780 struct ftrace_profile *rec; 781 struct hlist_head *hhd; 782 unsigned long key; 783 784 key = hash_long(ip, FTRACE_PROFILE_HASH_BITS); 785 hhd = &stat->hash[key]; 786 787 if (hlist_empty(hhd)) 788 return NULL; 789 790 hlist_for_each_entry_rcu_notrace(rec, hhd, node) { 791 if (rec->ip == ip) 792 return rec; 793 } 794 795 return NULL; 796 } 797 798 static void ftrace_add_profile(struct ftrace_profile_stat *stat, 799 struct ftrace_profile *rec) 800 { 801 unsigned long key; 802 803 key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS); 804 hlist_add_head_rcu(&rec->node, &stat->hash[key]); 805 } 806 807 /* 808 * The memory is already allocated, this simply finds a new record to use. 809 */ 810 static struct ftrace_profile * 811 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip) 812 { 813 struct ftrace_profile *rec = NULL; 814 815 /* prevent recursion (from NMIs) */ 816 if (atomic_inc_return(&stat->disabled) != 1) 817 goto out; 818 819 /* 820 * Try to find the function again since an NMI 821 * could have added it 822 */ 823 rec = ftrace_find_profiled_func(stat, ip); 824 if (rec) 825 goto out; 826 827 if (stat->pages->index == PROFILES_PER_PAGE) { 828 if (!stat->pages->next) 829 goto out; 830 stat->pages = stat->pages->next; 831 } 832 833 rec = &stat->pages->records[stat->pages->index++]; 834 rec->ip = ip; 835 ftrace_add_profile(stat, rec); 836 837 out: 838 atomic_dec(&stat->disabled); 839 840 return rec; 841 } 842 843 static void 844 function_profile_call(unsigned long ip, unsigned long parent_ip, 845 struct ftrace_ops *ops, struct pt_regs *regs) 846 { 847 struct ftrace_profile_stat *stat; 848 struct ftrace_profile *rec; 849 unsigned long flags; 850 851 if (!ftrace_profile_enabled) 852 return; 853 854 local_irq_save(flags); 855 856 stat = this_cpu_ptr(&ftrace_profile_stats); 857 if (!stat->hash || !ftrace_profile_enabled) 858 goto out; 859 860 rec = ftrace_find_profiled_func(stat, ip); 861 if (!rec) { 862 rec = ftrace_profile_alloc(stat, ip); 863 if (!rec) 864 goto out; 865 } 866 867 rec->counter++; 868 out: 869 local_irq_restore(flags); 870 } 871 872 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 873 static int profile_graph_entry(struct ftrace_graph_ent *trace) 874 { 875 function_profile_call(trace->func, 0, NULL, NULL); 876 return 1; 877 } 878 879 static void profile_graph_return(struct ftrace_graph_ret *trace) 880 { 881 struct ftrace_profile_stat *stat; 882 unsigned long long calltime; 883 struct ftrace_profile *rec; 884 unsigned long flags; 885 886 local_irq_save(flags); 887 stat = this_cpu_ptr(&ftrace_profile_stats); 888 if (!stat->hash || !ftrace_profile_enabled) 889 goto out; 890 891 /* If the calltime was zero'd ignore it */ 892 if (!trace->calltime) 893 goto out; 894 895 calltime = trace->rettime - trace->calltime; 896 897 if (!fgraph_graph_time) { 898 int index; 899 900 index = trace->depth; 901 902 /* Append this call time to the parent time to subtract */ 903 if (index) 904 current->ret_stack[index - 1].subtime += calltime; 905 906 if (current->ret_stack[index].subtime < calltime) 907 calltime -= current->ret_stack[index].subtime; 908 else 909 calltime = 0; 910 } 911 912 rec = ftrace_find_profiled_func(stat, trace->func); 913 if (rec) { 914 rec->time += calltime; 915 rec->time_squared += calltime * calltime; 916 } 917 918 out: 919 local_irq_restore(flags); 920 } 921 922 static int register_ftrace_profiler(void) 923 { 924 return register_ftrace_graph(&profile_graph_return, 925 &profile_graph_entry); 926 } 927 928 static void unregister_ftrace_profiler(void) 929 { 930 unregister_ftrace_graph(); 931 } 932 #else 933 static struct ftrace_ops ftrace_profile_ops __read_mostly = { 934 .func = function_profile_call, 935 .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED, 936 INIT_OPS_HASH(ftrace_profile_ops) 937 }; 938 939 static int register_ftrace_profiler(void) 940 { 941 return register_ftrace_function(&ftrace_profile_ops); 942 } 943 944 static void unregister_ftrace_profiler(void) 945 { 946 unregister_ftrace_function(&ftrace_profile_ops); 947 } 948 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 949 950 static ssize_t 951 ftrace_profile_write(struct file *filp, const char __user *ubuf, 952 size_t cnt, loff_t *ppos) 953 { 954 unsigned long val; 955 int ret; 956 957 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 958 if (ret) 959 return ret; 960 961 val = !!val; 962 963 mutex_lock(&ftrace_profile_lock); 964 if (ftrace_profile_enabled ^ val) { 965 if (val) { 966 ret = ftrace_profile_init(); 967 if (ret < 0) { 968 cnt = ret; 969 goto out; 970 } 971 972 ret = register_ftrace_profiler(); 973 if (ret < 0) { 974 cnt = ret; 975 goto out; 976 } 977 ftrace_profile_enabled = 1; 978 } else { 979 ftrace_profile_enabled = 0; 980 /* 981 * unregister_ftrace_profiler calls stop_machine 982 * so this acts like an synchronize_sched. 983 */ 984 unregister_ftrace_profiler(); 985 } 986 } 987 out: 988 mutex_unlock(&ftrace_profile_lock); 989 990 *ppos += cnt; 991 992 return cnt; 993 } 994 995 static ssize_t 996 ftrace_profile_read(struct file *filp, char __user *ubuf, 997 size_t cnt, loff_t *ppos) 998 { 999 char buf[64]; /* big enough to hold a number */ 1000 int r; 1001 1002 r = sprintf(buf, "%u\n", ftrace_profile_enabled); 1003 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 1004 } 1005 1006 static const struct file_operations ftrace_profile_fops = { 1007 .open = tracing_open_generic, 1008 .read = ftrace_profile_read, 1009 .write = ftrace_profile_write, 1010 .llseek = default_llseek, 1011 }; 1012 1013 /* used to initialize the real stat files */ 1014 static struct tracer_stat function_stats __initdata = { 1015 .name = "functions", 1016 .stat_start = function_stat_start, 1017 .stat_next = function_stat_next, 1018 .stat_cmp = function_stat_cmp, 1019 .stat_headers = function_stat_headers, 1020 .stat_show = function_stat_show 1021 }; 1022 1023 static __init void ftrace_profile_tracefs(struct dentry *d_tracer) 1024 { 1025 struct ftrace_profile_stat *stat; 1026 struct dentry *entry; 1027 char *name; 1028 int ret; 1029 int cpu; 1030 1031 for_each_possible_cpu(cpu) { 1032 stat = &per_cpu(ftrace_profile_stats, cpu); 1033 1034 name = kasprintf(GFP_KERNEL, "function%d", cpu); 1035 if (!name) { 1036 /* 1037 * The files created are permanent, if something happens 1038 * we still do not free memory. 1039 */ 1040 WARN(1, 1041 "Could not allocate stat file for cpu %d\n", 1042 cpu); 1043 return; 1044 } 1045 stat->stat = function_stats; 1046 stat->stat.name = name; 1047 ret = register_stat_tracer(&stat->stat); 1048 if (ret) { 1049 WARN(1, 1050 "Could not register function stat for cpu %d\n", 1051 cpu); 1052 kfree(name); 1053 return; 1054 } 1055 } 1056 1057 entry = tracefs_create_file("function_profile_enabled", 0644, 1058 d_tracer, NULL, &ftrace_profile_fops); 1059 if (!entry) 1060 pr_warn("Could not create tracefs 'function_profile_enabled' entry\n"); 1061 } 1062 1063 #else /* CONFIG_FUNCTION_PROFILER */ 1064 static __init void ftrace_profile_tracefs(struct dentry *d_tracer) 1065 { 1066 } 1067 #endif /* CONFIG_FUNCTION_PROFILER */ 1068 1069 static struct pid * const ftrace_swapper_pid = &init_struct_pid; 1070 1071 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 1072 static int ftrace_graph_active; 1073 #else 1074 # define ftrace_graph_active 0 1075 #endif 1076 1077 #ifdef CONFIG_DYNAMIC_FTRACE 1078 1079 static struct ftrace_ops *removed_ops; 1080 1081 /* 1082 * Set when doing a global update, like enabling all recs or disabling them. 1083 * It is not set when just updating a single ftrace_ops. 1084 */ 1085 static bool update_all_ops; 1086 1087 #ifndef CONFIG_FTRACE_MCOUNT_RECORD 1088 # error Dynamic ftrace depends on MCOUNT_RECORD 1089 #endif 1090 1091 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly; 1092 1093 struct ftrace_func_probe { 1094 struct hlist_node node; 1095 struct ftrace_probe_ops *ops; 1096 unsigned long flags; 1097 unsigned long ip; 1098 void *data; 1099 struct list_head free_list; 1100 }; 1101 1102 struct ftrace_func_entry { 1103 struct hlist_node hlist; 1104 unsigned long ip; 1105 }; 1106 1107 struct ftrace_hash { 1108 unsigned long size_bits; 1109 struct hlist_head *buckets; 1110 unsigned long count; 1111 struct rcu_head rcu; 1112 }; 1113 1114 /* 1115 * We make these constant because no one should touch them, 1116 * but they are used as the default "empty hash", to avoid allocating 1117 * it all the time. These are in a read only section such that if 1118 * anyone does try to modify it, it will cause an exception. 1119 */ 1120 static const struct hlist_head empty_buckets[1]; 1121 static const struct ftrace_hash empty_hash = { 1122 .buckets = (struct hlist_head *)empty_buckets, 1123 }; 1124 #define EMPTY_HASH ((struct ftrace_hash *)&empty_hash) 1125 1126 static struct ftrace_ops global_ops = { 1127 .func = ftrace_stub, 1128 .local_hash.notrace_hash = EMPTY_HASH, 1129 .local_hash.filter_hash = EMPTY_HASH, 1130 INIT_OPS_HASH(global_ops) 1131 .flags = FTRACE_OPS_FL_RECURSION_SAFE | 1132 FTRACE_OPS_FL_INITIALIZED | 1133 FTRACE_OPS_FL_PID, 1134 }; 1135 1136 /* 1137 * This is used by __kernel_text_address() to return true if the 1138 * address is on a dynamically allocated trampoline that would 1139 * not return true for either core_kernel_text() or 1140 * is_module_text_address(). 1141 */ 1142 bool is_ftrace_trampoline(unsigned long addr) 1143 { 1144 struct ftrace_ops *op; 1145 bool ret = false; 1146 1147 /* 1148 * Some of the ops may be dynamically allocated, 1149 * they are freed after a synchronize_sched(). 1150 */ 1151 preempt_disable_notrace(); 1152 1153 do_for_each_ftrace_op(op, ftrace_ops_list) { 1154 /* 1155 * This is to check for dynamically allocated trampolines. 1156 * Trampolines that are in kernel text will have 1157 * core_kernel_text() return true. 1158 */ 1159 if (op->trampoline && op->trampoline_size) 1160 if (addr >= op->trampoline && 1161 addr < op->trampoline + op->trampoline_size) { 1162 ret = true; 1163 goto out; 1164 } 1165 } while_for_each_ftrace_op(op); 1166 1167 out: 1168 preempt_enable_notrace(); 1169 1170 return ret; 1171 } 1172 1173 struct ftrace_page { 1174 struct ftrace_page *next; 1175 struct dyn_ftrace *records; 1176 int index; 1177 int size; 1178 }; 1179 1180 #define ENTRY_SIZE sizeof(struct dyn_ftrace) 1181 #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE) 1182 1183 /* estimate from running different kernels */ 1184 #define NR_TO_INIT 10000 1185 1186 static struct ftrace_page *ftrace_pages_start; 1187 static struct ftrace_page *ftrace_pages; 1188 1189 static bool __always_inline ftrace_hash_empty(struct ftrace_hash *hash) 1190 { 1191 return !hash || !hash->count; 1192 } 1193 1194 static struct ftrace_func_entry * 1195 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) 1196 { 1197 unsigned long key; 1198 struct ftrace_func_entry *entry; 1199 struct hlist_head *hhd; 1200 1201 if (ftrace_hash_empty(hash)) 1202 return NULL; 1203 1204 if (hash->size_bits > 0) 1205 key = hash_long(ip, hash->size_bits); 1206 else 1207 key = 0; 1208 1209 hhd = &hash->buckets[key]; 1210 1211 hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) { 1212 if (entry->ip == ip) 1213 return entry; 1214 } 1215 return NULL; 1216 } 1217 1218 static void __add_hash_entry(struct ftrace_hash *hash, 1219 struct ftrace_func_entry *entry) 1220 { 1221 struct hlist_head *hhd; 1222 unsigned long key; 1223 1224 if (hash->size_bits) 1225 key = hash_long(entry->ip, hash->size_bits); 1226 else 1227 key = 0; 1228 1229 hhd = &hash->buckets[key]; 1230 hlist_add_head(&entry->hlist, hhd); 1231 hash->count++; 1232 } 1233 1234 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip) 1235 { 1236 struct ftrace_func_entry *entry; 1237 1238 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 1239 if (!entry) 1240 return -ENOMEM; 1241 1242 entry->ip = ip; 1243 __add_hash_entry(hash, entry); 1244 1245 return 0; 1246 } 1247 1248 static void 1249 free_hash_entry(struct ftrace_hash *hash, 1250 struct ftrace_func_entry *entry) 1251 { 1252 hlist_del(&entry->hlist); 1253 kfree(entry); 1254 hash->count--; 1255 } 1256 1257 static void 1258 remove_hash_entry(struct ftrace_hash *hash, 1259 struct ftrace_func_entry *entry) 1260 { 1261 hlist_del(&entry->hlist); 1262 hash->count--; 1263 } 1264 1265 static void ftrace_hash_clear(struct ftrace_hash *hash) 1266 { 1267 struct hlist_head *hhd; 1268 struct hlist_node *tn; 1269 struct ftrace_func_entry *entry; 1270 int size = 1 << hash->size_bits; 1271 int i; 1272 1273 if (!hash->count) 1274 return; 1275 1276 for (i = 0; i < size; i++) { 1277 hhd = &hash->buckets[i]; 1278 hlist_for_each_entry_safe(entry, tn, hhd, hlist) 1279 free_hash_entry(hash, entry); 1280 } 1281 FTRACE_WARN_ON(hash->count); 1282 } 1283 1284 static void free_ftrace_hash(struct ftrace_hash *hash) 1285 { 1286 if (!hash || hash == EMPTY_HASH) 1287 return; 1288 ftrace_hash_clear(hash); 1289 kfree(hash->buckets); 1290 kfree(hash); 1291 } 1292 1293 static void __free_ftrace_hash_rcu(struct rcu_head *rcu) 1294 { 1295 struct ftrace_hash *hash; 1296 1297 hash = container_of(rcu, struct ftrace_hash, rcu); 1298 free_ftrace_hash(hash); 1299 } 1300 1301 static void free_ftrace_hash_rcu(struct ftrace_hash *hash) 1302 { 1303 if (!hash || hash == EMPTY_HASH) 1304 return; 1305 call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu); 1306 } 1307 1308 void ftrace_free_filter(struct ftrace_ops *ops) 1309 { 1310 ftrace_ops_init(ops); 1311 free_ftrace_hash(ops->func_hash->filter_hash); 1312 free_ftrace_hash(ops->func_hash->notrace_hash); 1313 } 1314 1315 static struct ftrace_hash *alloc_ftrace_hash(int size_bits) 1316 { 1317 struct ftrace_hash *hash; 1318 int size; 1319 1320 hash = kzalloc(sizeof(*hash), GFP_KERNEL); 1321 if (!hash) 1322 return NULL; 1323 1324 size = 1 << size_bits; 1325 hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL); 1326 1327 if (!hash->buckets) { 1328 kfree(hash); 1329 return NULL; 1330 } 1331 1332 hash->size_bits = size_bits; 1333 1334 return hash; 1335 } 1336 1337 static struct ftrace_hash * 1338 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash) 1339 { 1340 struct ftrace_func_entry *entry; 1341 struct ftrace_hash *new_hash; 1342 int size; 1343 int ret; 1344 int i; 1345 1346 new_hash = alloc_ftrace_hash(size_bits); 1347 if (!new_hash) 1348 return NULL; 1349 1350 /* Empty hash? */ 1351 if (ftrace_hash_empty(hash)) 1352 return new_hash; 1353 1354 size = 1 << hash->size_bits; 1355 for (i = 0; i < size; i++) { 1356 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 1357 ret = add_hash_entry(new_hash, entry->ip); 1358 if (ret < 0) 1359 goto free_hash; 1360 } 1361 } 1362 1363 FTRACE_WARN_ON(new_hash->count != hash->count); 1364 1365 return new_hash; 1366 1367 free_hash: 1368 free_ftrace_hash(new_hash); 1369 return NULL; 1370 } 1371 1372 static void 1373 ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash); 1374 static void 1375 ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash); 1376 1377 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, 1378 struct ftrace_hash *new_hash); 1379 1380 static int 1381 ftrace_hash_move(struct ftrace_ops *ops, int enable, 1382 struct ftrace_hash **dst, struct ftrace_hash *src) 1383 { 1384 struct ftrace_func_entry *entry; 1385 struct hlist_node *tn; 1386 struct hlist_head *hhd; 1387 struct ftrace_hash *new_hash; 1388 int size = src->count; 1389 int bits = 0; 1390 int ret; 1391 int i; 1392 1393 /* Reject setting notrace hash on IPMODIFY ftrace_ops */ 1394 if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable) 1395 return -EINVAL; 1396 1397 /* 1398 * If the new source is empty, just free dst and assign it 1399 * the empty_hash. 1400 */ 1401 if (!src->count) { 1402 new_hash = EMPTY_HASH; 1403 goto update; 1404 } 1405 1406 /* 1407 * Make the hash size about 1/2 the # found 1408 */ 1409 for (size /= 2; size; size >>= 1) 1410 bits++; 1411 1412 /* Don't allocate too much */ 1413 if (bits > FTRACE_HASH_MAX_BITS) 1414 bits = FTRACE_HASH_MAX_BITS; 1415 1416 new_hash = alloc_ftrace_hash(bits); 1417 if (!new_hash) 1418 return -ENOMEM; 1419 1420 size = 1 << src->size_bits; 1421 for (i = 0; i < size; i++) { 1422 hhd = &src->buckets[i]; 1423 hlist_for_each_entry_safe(entry, tn, hhd, hlist) { 1424 remove_hash_entry(src, entry); 1425 __add_hash_entry(new_hash, entry); 1426 } 1427 } 1428 1429 update: 1430 /* Make sure this can be applied if it is IPMODIFY ftrace_ops */ 1431 if (enable) { 1432 /* IPMODIFY should be updated only when filter_hash updating */ 1433 ret = ftrace_hash_ipmodify_update(ops, new_hash); 1434 if (ret < 0) { 1435 free_ftrace_hash(new_hash); 1436 return ret; 1437 } 1438 } 1439 1440 /* 1441 * Remove the current set, update the hash and add 1442 * them back. 1443 */ 1444 ftrace_hash_rec_disable_modify(ops, enable); 1445 1446 rcu_assign_pointer(*dst, new_hash); 1447 1448 ftrace_hash_rec_enable_modify(ops, enable); 1449 1450 return 0; 1451 } 1452 1453 static bool hash_contains_ip(unsigned long ip, 1454 struct ftrace_ops_hash *hash) 1455 { 1456 /* 1457 * The function record is a match if it exists in the filter 1458 * hash and not in the notrace hash. Note, an emty hash is 1459 * considered a match for the filter hash, but an empty 1460 * notrace hash is considered not in the notrace hash. 1461 */ 1462 return (ftrace_hash_empty(hash->filter_hash) || 1463 ftrace_lookup_ip(hash->filter_hash, ip)) && 1464 (ftrace_hash_empty(hash->notrace_hash) || 1465 !ftrace_lookup_ip(hash->notrace_hash, ip)); 1466 } 1467 1468 /* 1469 * Test the hashes for this ops to see if we want to call 1470 * the ops->func or not. 1471 * 1472 * It's a match if the ip is in the ops->filter_hash or 1473 * the filter_hash does not exist or is empty, 1474 * AND 1475 * the ip is not in the ops->notrace_hash. 1476 * 1477 * This needs to be called with preemption disabled as 1478 * the hashes are freed with call_rcu_sched(). 1479 */ 1480 static int 1481 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) 1482 { 1483 struct ftrace_ops_hash hash; 1484 int ret; 1485 1486 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 1487 /* 1488 * There's a small race when adding ops that the ftrace handler 1489 * that wants regs, may be called without them. We can not 1490 * allow that handler to be called if regs is NULL. 1491 */ 1492 if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS)) 1493 return 0; 1494 #endif 1495 1496 hash.filter_hash = rcu_dereference_raw_notrace(ops->func_hash->filter_hash); 1497 hash.notrace_hash = rcu_dereference_raw_notrace(ops->func_hash->notrace_hash); 1498 1499 if (hash_contains_ip(ip, &hash)) 1500 ret = 1; 1501 else 1502 ret = 0; 1503 1504 return ret; 1505 } 1506 1507 /* 1508 * This is a double for. Do not use 'break' to break out of the loop, 1509 * you must use a goto. 1510 */ 1511 #define do_for_each_ftrace_rec(pg, rec) \ 1512 for (pg = ftrace_pages_start; pg; pg = pg->next) { \ 1513 int _____i; \ 1514 for (_____i = 0; _____i < pg->index; _____i++) { \ 1515 rec = &pg->records[_____i]; 1516 1517 #define while_for_each_ftrace_rec() \ 1518 } \ 1519 } 1520 1521 1522 static int ftrace_cmp_recs(const void *a, const void *b) 1523 { 1524 const struct dyn_ftrace *key = a; 1525 const struct dyn_ftrace *rec = b; 1526 1527 if (key->flags < rec->ip) 1528 return -1; 1529 if (key->ip >= rec->ip + MCOUNT_INSN_SIZE) 1530 return 1; 1531 return 0; 1532 } 1533 1534 /** 1535 * ftrace_location_range - return the first address of a traced location 1536 * if it touches the given ip range 1537 * @start: start of range to search. 1538 * @end: end of range to search (inclusive). @end points to the last byte 1539 * to check. 1540 * 1541 * Returns rec->ip if the related ftrace location is a least partly within 1542 * the given address range. That is, the first address of the instruction 1543 * that is either a NOP or call to the function tracer. It checks the ftrace 1544 * internal tables to determine if the address belongs or not. 1545 */ 1546 unsigned long ftrace_location_range(unsigned long start, unsigned long end) 1547 { 1548 struct ftrace_page *pg; 1549 struct dyn_ftrace *rec; 1550 struct dyn_ftrace key; 1551 1552 key.ip = start; 1553 key.flags = end; /* overload flags, as it is unsigned long */ 1554 1555 for (pg = ftrace_pages_start; pg; pg = pg->next) { 1556 if (end < pg->records[0].ip || 1557 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) 1558 continue; 1559 rec = bsearch(&key, pg->records, pg->index, 1560 sizeof(struct dyn_ftrace), 1561 ftrace_cmp_recs); 1562 if (rec) 1563 return rec->ip; 1564 } 1565 1566 return 0; 1567 } 1568 1569 /** 1570 * ftrace_location - return true if the ip giving is a traced location 1571 * @ip: the instruction pointer to check 1572 * 1573 * Returns rec->ip if @ip given is a pointer to a ftrace location. 1574 * That is, the instruction that is either a NOP or call to 1575 * the function tracer. It checks the ftrace internal tables to 1576 * determine if the address belongs or not. 1577 */ 1578 unsigned long ftrace_location(unsigned long ip) 1579 { 1580 return ftrace_location_range(ip, ip); 1581 } 1582 1583 /** 1584 * ftrace_text_reserved - return true if range contains an ftrace location 1585 * @start: start of range to search 1586 * @end: end of range to search (inclusive). @end points to the last byte to check. 1587 * 1588 * Returns 1 if @start and @end contains a ftrace location. 1589 * That is, the instruction that is either a NOP or call to 1590 * the function tracer. It checks the ftrace internal tables to 1591 * determine if the address belongs or not. 1592 */ 1593 int ftrace_text_reserved(const void *start, const void *end) 1594 { 1595 unsigned long ret; 1596 1597 ret = ftrace_location_range((unsigned long)start, 1598 (unsigned long)end); 1599 1600 return (int)!!ret; 1601 } 1602 1603 /* Test if ops registered to this rec needs regs */ 1604 static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec) 1605 { 1606 struct ftrace_ops *ops; 1607 bool keep_regs = false; 1608 1609 for (ops = ftrace_ops_list; 1610 ops != &ftrace_list_end; ops = ops->next) { 1611 /* pass rec in as regs to have non-NULL val */ 1612 if (ftrace_ops_test(ops, rec->ip, rec)) { 1613 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) { 1614 keep_regs = true; 1615 break; 1616 } 1617 } 1618 } 1619 1620 return keep_regs; 1621 } 1622 1623 static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, 1624 int filter_hash, 1625 bool inc) 1626 { 1627 struct ftrace_hash *hash; 1628 struct ftrace_hash *other_hash; 1629 struct ftrace_page *pg; 1630 struct dyn_ftrace *rec; 1631 bool update = false; 1632 int count = 0; 1633 int all = 0; 1634 1635 /* Only update if the ops has been registered */ 1636 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 1637 return false; 1638 1639 /* 1640 * In the filter_hash case: 1641 * If the count is zero, we update all records. 1642 * Otherwise we just update the items in the hash. 1643 * 1644 * In the notrace_hash case: 1645 * We enable the update in the hash. 1646 * As disabling notrace means enabling the tracing, 1647 * and enabling notrace means disabling, the inc variable 1648 * gets inversed. 1649 */ 1650 if (filter_hash) { 1651 hash = ops->func_hash->filter_hash; 1652 other_hash = ops->func_hash->notrace_hash; 1653 if (ftrace_hash_empty(hash)) 1654 all = 1; 1655 } else { 1656 inc = !inc; 1657 hash = ops->func_hash->notrace_hash; 1658 other_hash = ops->func_hash->filter_hash; 1659 /* 1660 * If the notrace hash has no items, 1661 * then there's nothing to do. 1662 */ 1663 if (ftrace_hash_empty(hash)) 1664 return false; 1665 } 1666 1667 do_for_each_ftrace_rec(pg, rec) { 1668 int in_other_hash = 0; 1669 int in_hash = 0; 1670 int match = 0; 1671 1672 if (rec->flags & FTRACE_FL_DISABLED) 1673 continue; 1674 1675 if (all) { 1676 /* 1677 * Only the filter_hash affects all records. 1678 * Update if the record is not in the notrace hash. 1679 */ 1680 if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip)) 1681 match = 1; 1682 } else { 1683 in_hash = !!ftrace_lookup_ip(hash, rec->ip); 1684 in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip); 1685 1686 /* 1687 * If filter_hash is set, we want to match all functions 1688 * that are in the hash but not in the other hash. 1689 * 1690 * If filter_hash is not set, then we are decrementing. 1691 * That means we match anything that is in the hash 1692 * and also in the other_hash. That is, we need to turn 1693 * off functions in the other hash because they are disabled 1694 * by this hash. 1695 */ 1696 if (filter_hash && in_hash && !in_other_hash) 1697 match = 1; 1698 else if (!filter_hash && in_hash && 1699 (in_other_hash || ftrace_hash_empty(other_hash))) 1700 match = 1; 1701 } 1702 if (!match) 1703 continue; 1704 1705 if (inc) { 1706 rec->flags++; 1707 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX)) 1708 return false; 1709 1710 /* 1711 * If there's only a single callback registered to a 1712 * function, and the ops has a trampoline registered 1713 * for it, then we can call it directly. 1714 */ 1715 if (ftrace_rec_count(rec) == 1 && ops->trampoline) 1716 rec->flags |= FTRACE_FL_TRAMP; 1717 else 1718 /* 1719 * If we are adding another function callback 1720 * to this function, and the previous had a 1721 * custom trampoline in use, then we need to go 1722 * back to the default trampoline. 1723 */ 1724 rec->flags &= ~FTRACE_FL_TRAMP; 1725 1726 /* 1727 * If any ops wants regs saved for this function 1728 * then all ops will get saved regs. 1729 */ 1730 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) 1731 rec->flags |= FTRACE_FL_REGS; 1732 } else { 1733 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0)) 1734 return false; 1735 rec->flags--; 1736 1737 /* 1738 * If the rec had REGS enabled and the ops that is 1739 * being removed had REGS set, then see if there is 1740 * still any ops for this record that wants regs. 1741 * If not, we can stop recording them. 1742 */ 1743 if (ftrace_rec_count(rec) > 0 && 1744 rec->flags & FTRACE_FL_REGS && 1745 ops->flags & FTRACE_OPS_FL_SAVE_REGS) { 1746 if (!test_rec_ops_needs_regs(rec)) 1747 rec->flags &= ~FTRACE_FL_REGS; 1748 } 1749 1750 /* 1751 * If the rec had TRAMP enabled, then it needs to 1752 * be cleared. As TRAMP can only be enabled iff 1753 * there is only a single ops attached to it. 1754 * In otherwords, always disable it on decrementing. 1755 * In the future, we may set it if rec count is 1756 * decremented to one, and the ops that is left 1757 * has a trampoline. 1758 */ 1759 rec->flags &= ~FTRACE_FL_TRAMP; 1760 1761 /* 1762 * flags will be cleared in ftrace_check_record() 1763 * if rec count is zero. 1764 */ 1765 } 1766 count++; 1767 1768 /* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */ 1769 update |= ftrace_test_record(rec, 1) != FTRACE_UPDATE_IGNORE; 1770 1771 /* Shortcut, if we handled all records, we are done. */ 1772 if (!all && count == hash->count) 1773 return update; 1774 } while_for_each_ftrace_rec(); 1775 1776 return update; 1777 } 1778 1779 static bool ftrace_hash_rec_disable(struct ftrace_ops *ops, 1780 int filter_hash) 1781 { 1782 return __ftrace_hash_rec_update(ops, filter_hash, 0); 1783 } 1784 1785 static bool ftrace_hash_rec_enable(struct ftrace_ops *ops, 1786 int filter_hash) 1787 { 1788 return __ftrace_hash_rec_update(ops, filter_hash, 1); 1789 } 1790 1791 static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, 1792 int filter_hash, int inc) 1793 { 1794 struct ftrace_ops *op; 1795 1796 __ftrace_hash_rec_update(ops, filter_hash, inc); 1797 1798 if (ops->func_hash != &global_ops.local_hash) 1799 return; 1800 1801 /* 1802 * If the ops shares the global_ops hash, then we need to update 1803 * all ops that are enabled and use this hash. 1804 */ 1805 do_for_each_ftrace_op(op, ftrace_ops_list) { 1806 /* Already done */ 1807 if (op == ops) 1808 continue; 1809 if (op->func_hash == &global_ops.local_hash) 1810 __ftrace_hash_rec_update(op, filter_hash, inc); 1811 } while_for_each_ftrace_op(op); 1812 } 1813 1814 static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, 1815 int filter_hash) 1816 { 1817 ftrace_hash_rec_update_modify(ops, filter_hash, 0); 1818 } 1819 1820 static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, 1821 int filter_hash) 1822 { 1823 ftrace_hash_rec_update_modify(ops, filter_hash, 1); 1824 } 1825 1826 /* 1827 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK 1828 * or no-needed to update, -EBUSY if it detects a conflict of the flag 1829 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs. 1830 * Note that old_hash and new_hash has below meanings 1831 * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected) 1832 * - If the hash is EMPTY_HASH, it hits nothing 1833 * - Anything else hits the recs which match the hash entries. 1834 */ 1835 static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops, 1836 struct ftrace_hash *old_hash, 1837 struct ftrace_hash *new_hash) 1838 { 1839 struct ftrace_page *pg; 1840 struct dyn_ftrace *rec, *end = NULL; 1841 int in_old, in_new; 1842 1843 /* Only update if the ops has been registered */ 1844 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 1845 return 0; 1846 1847 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) 1848 return 0; 1849 1850 /* 1851 * Since the IPMODIFY is a very address sensitive action, we do not 1852 * allow ftrace_ops to set all functions to new hash. 1853 */ 1854 if (!new_hash || !old_hash) 1855 return -EINVAL; 1856 1857 /* Update rec->flags */ 1858 do_for_each_ftrace_rec(pg, rec) { 1859 /* We need to update only differences of filter_hash */ 1860 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); 1861 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); 1862 if (in_old == in_new) 1863 continue; 1864 1865 if (in_new) { 1866 /* New entries must ensure no others are using it */ 1867 if (rec->flags & FTRACE_FL_IPMODIFY) 1868 goto rollback; 1869 rec->flags |= FTRACE_FL_IPMODIFY; 1870 } else /* Removed entry */ 1871 rec->flags &= ~FTRACE_FL_IPMODIFY; 1872 } while_for_each_ftrace_rec(); 1873 1874 return 0; 1875 1876 rollback: 1877 end = rec; 1878 1879 /* Roll back what we did above */ 1880 do_for_each_ftrace_rec(pg, rec) { 1881 if (rec == end) 1882 goto err_out; 1883 1884 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); 1885 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); 1886 if (in_old == in_new) 1887 continue; 1888 1889 if (in_new) 1890 rec->flags &= ~FTRACE_FL_IPMODIFY; 1891 else 1892 rec->flags |= FTRACE_FL_IPMODIFY; 1893 } while_for_each_ftrace_rec(); 1894 1895 err_out: 1896 return -EBUSY; 1897 } 1898 1899 static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops) 1900 { 1901 struct ftrace_hash *hash = ops->func_hash->filter_hash; 1902 1903 if (ftrace_hash_empty(hash)) 1904 hash = NULL; 1905 1906 return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash); 1907 } 1908 1909 /* Disabling always succeeds */ 1910 static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops) 1911 { 1912 struct ftrace_hash *hash = ops->func_hash->filter_hash; 1913 1914 if (ftrace_hash_empty(hash)) 1915 hash = NULL; 1916 1917 __ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH); 1918 } 1919 1920 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, 1921 struct ftrace_hash *new_hash) 1922 { 1923 struct ftrace_hash *old_hash = ops->func_hash->filter_hash; 1924 1925 if (ftrace_hash_empty(old_hash)) 1926 old_hash = NULL; 1927 1928 if (ftrace_hash_empty(new_hash)) 1929 new_hash = NULL; 1930 1931 return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash); 1932 } 1933 1934 static void print_ip_ins(const char *fmt, const unsigned char *p) 1935 { 1936 int i; 1937 1938 printk(KERN_CONT "%s", fmt); 1939 1940 for (i = 0; i < MCOUNT_INSN_SIZE; i++) 1941 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]); 1942 } 1943 1944 static struct ftrace_ops * 1945 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec); 1946 static struct ftrace_ops * 1947 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops); 1948 1949 enum ftrace_bug_type ftrace_bug_type; 1950 const void *ftrace_expected; 1951 1952 static void print_bug_type(void) 1953 { 1954 switch (ftrace_bug_type) { 1955 case FTRACE_BUG_UNKNOWN: 1956 break; 1957 case FTRACE_BUG_INIT: 1958 pr_info("Initializing ftrace call sites\n"); 1959 break; 1960 case FTRACE_BUG_NOP: 1961 pr_info("Setting ftrace call site to NOP\n"); 1962 break; 1963 case FTRACE_BUG_CALL: 1964 pr_info("Setting ftrace call site to call ftrace function\n"); 1965 break; 1966 case FTRACE_BUG_UPDATE: 1967 pr_info("Updating ftrace call site to call a different ftrace function\n"); 1968 break; 1969 } 1970 } 1971 1972 /** 1973 * ftrace_bug - report and shutdown function tracer 1974 * @failed: The failed type (EFAULT, EINVAL, EPERM) 1975 * @rec: The record that failed 1976 * 1977 * The arch code that enables or disables the function tracing 1978 * can call ftrace_bug() when it has detected a problem in 1979 * modifying the code. @failed should be one of either: 1980 * EFAULT - if the problem happens on reading the @ip address 1981 * EINVAL - if what is read at @ip is not what was expected 1982 * EPERM - if the problem happens on writting to the @ip address 1983 */ 1984 void ftrace_bug(int failed, struct dyn_ftrace *rec) 1985 { 1986 unsigned long ip = rec ? rec->ip : 0; 1987 1988 switch (failed) { 1989 case -EFAULT: 1990 FTRACE_WARN_ON_ONCE(1); 1991 pr_info("ftrace faulted on modifying "); 1992 print_ip_sym(ip); 1993 break; 1994 case -EINVAL: 1995 FTRACE_WARN_ON_ONCE(1); 1996 pr_info("ftrace failed to modify "); 1997 print_ip_sym(ip); 1998 print_ip_ins(" actual: ", (unsigned char *)ip); 1999 pr_cont("\n"); 2000 if (ftrace_expected) { 2001 print_ip_ins(" expected: ", ftrace_expected); 2002 pr_cont("\n"); 2003 } 2004 break; 2005 case -EPERM: 2006 FTRACE_WARN_ON_ONCE(1); 2007 pr_info("ftrace faulted on writing "); 2008 print_ip_sym(ip); 2009 break; 2010 default: 2011 FTRACE_WARN_ON_ONCE(1); 2012 pr_info("ftrace faulted on unknown error "); 2013 print_ip_sym(ip); 2014 } 2015 print_bug_type(); 2016 if (rec) { 2017 struct ftrace_ops *ops = NULL; 2018 2019 pr_info("ftrace record flags: %lx\n", rec->flags); 2020 pr_cont(" (%ld)%s", ftrace_rec_count(rec), 2021 rec->flags & FTRACE_FL_REGS ? " R" : " "); 2022 if (rec->flags & FTRACE_FL_TRAMP_EN) { 2023 ops = ftrace_find_tramp_ops_any(rec); 2024 if (ops) { 2025 do { 2026 pr_cont("\ttramp: %pS (%pS)", 2027 (void *)ops->trampoline, 2028 (void *)ops->func); 2029 ops = ftrace_find_tramp_ops_next(rec, ops); 2030 } while (ops); 2031 } else 2032 pr_cont("\ttramp: ERROR!"); 2033 2034 } 2035 ip = ftrace_get_addr_curr(rec); 2036 pr_cont("\n expected tramp: %lx\n", ip); 2037 } 2038 } 2039 2040 static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update) 2041 { 2042 unsigned long flag = 0UL; 2043 2044 ftrace_bug_type = FTRACE_BUG_UNKNOWN; 2045 2046 if (rec->flags & FTRACE_FL_DISABLED) 2047 return FTRACE_UPDATE_IGNORE; 2048 2049 /* 2050 * If we are updating calls: 2051 * 2052 * If the record has a ref count, then we need to enable it 2053 * because someone is using it. 2054 * 2055 * Otherwise we make sure its disabled. 2056 * 2057 * If we are disabling calls, then disable all records that 2058 * are enabled. 2059 */ 2060 if (enable && ftrace_rec_count(rec)) 2061 flag = FTRACE_FL_ENABLED; 2062 2063 /* 2064 * If enabling and the REGS flag does not match the REGS_EN, or 2065 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore 2066 * this record. Set flags to fail the compare against ENABLED. 2067 */ 2068 if (flag) { 2069 if (!(rec->flags & FTRACE_FL_REGS) != 2070 !(rec->flags & FTRACE_FL_REGS_EN)) 2071 flag |= FTRACE_FL_REGS; 2072 2073 if (!(rec->flags & FTRACE_FL_TRAMP) != 2074 !(rec->flags & FTRACE_FL_TRAMP_EN)) 2075 flag |= FTRACE_FL_TRAMP; 2076 } 2077 2078 /* If the state of this record hasn't changed, then do nothing */ 2079 if ((rec->flags & FTRACE_FL_ENABLED) == flag) 2080 return FTRACE_UPDATE_IGNORE; 2081 2082 if (flag) { 2083 /* Save off if rec is being enabled (for return value) */ 2084 flag ^= rec->flags & FTRACE_FL_ENABLED; 2085 2086 if (update) { 2087 rec->flags |= FTRACE_FL_ENABLED; 2088 if (flag & FTRACE_FL_REGS) { 2089 if (rec->flags & FTRACE_FL_REGS) 2090 rec->flags |= FTRACE_FL_REGS_EN; 2091 else 2092 rec->flags &= ~FTRACE_FL_REGS_EN; 2093 } 2094 if (flag & FTRACE_FL_TRAMP) { 2095 if (rec->flags & FTRACE_FL_TRAMP) 2096 rec->flags |= FTRACE_FL_TRAMP_EN; 2097 else 2098 rec->flags &= ~FTRACE_FL_TRAMP_EN; 2099 } 2100 } 2101 2102 /* 2103 * If this record is being updated from a nop, then 2104 * return UPDATE_MAKE_CALL. 2105 * Otherwise, 2106 * return UPDATE_MODIFY_CALL to tell the caller to convert 2107 * from the save regs, to a non-save regs function or 2108 * vice versa, or from a trampoline call. 2109 */ 2110 if (flag & FTRACE_FL_ENABLED) { 2111 ftrace_bug_type = FTRACE_BUG_CALL; 2112 return FTRACE_UPDATE_MAKE_CALL; 2113 } 2114 2115 ftrace_bug_type = FTRACE_BUG_UPDATE; 2116 return FTRACE_UPDATE_MODIFY_CALL; 2117 } 2118 2119 if (update) { 2120 /* If there's no more users, clear all flags */ 2121 if (!ftrace_rec_count(rec)) 2122 rec->flags = 0; 2123 else 2124 /* 2125 * Just disable the record, but keep the ops TRAMP 2126 * and REGS states. The _EN flags must be disabled though. 2127 */ 2128 rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN | 2129 FTRACE_FL_REGS_EN); 2130 } 2131 2132 ftrace_bug_type = FTRACE_BUG_NOP; 2133 return FTRACE_UPDATE_MAKE_NOP; 2134 } 2135 2136 /** 2137 * ftrace_update_record, set a record that now is tracing or not 2138 * @rec: the record to update 2139 * @enable: set to 1 if the record is tracing, zero to force disable 2140 * 2141 * The records that represent all functions that can be traced need 2142 * to be updated when tracing has been enabled. 2143 */ 2144 int ftrace_update_record(struct dyn_ftrace *rec, int enable) 2145 { 2146 return ftrace_check_record(rec, enable, 1); 2147 } 2148 2149 /** 2150 * ftrace_test_record, check if the record has been enabled or not 2151 * @rec: the record to test 2152 * @enable: set to 1 to check if enabled, 0 if it is disabled 2153 * 2154 * The arch code may need to test if a record is already set to 2155 * tracing to determine how to modify the function code that it 2156 * represents. 2157 */ 2158 int ftrace_test_record(struct dyn_ftrace *rec, int enable) 2159 { 2160 return ftrace_check_record(rec, enable, 0); 2161 } 2162 2163 static struct ftrace_ops * 2164 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec) 2165 { 2166 struct ftrace_ops *op; 2167 unsigned long ip = rec->ip; 2168 2169 do_for_each_ftrace_op(op, ftrace_ops_list) { 2170 2171 if (!op->trampoline) 2172 continue; 2173 2174 if (hash_contains_ip(ip, op->func_hash)) 2175 return op; 2176 } while_for_each_ftrace_op(op); 2177 2178 return NULL; 2179 } 2180 2181 static struct ftrace_ops * 2182 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, 2183 struct ftrace_ops *op) 2184 { 2185 unsigned long ip = rec->ip; 2186 2187 while_for_each_ftrace_op(op) { 2188 2189 if (!op->trampoline) 2190 continue; 2191 2192 if (hash_contains_ip(ip, op->func_hash)) 2193 return op; 2194 } 2195 2196 return NULL; 2197 } 2198 2199 static struct ftrace_ops * 2200 ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec) 2201 { 2202 struct ftrace_ops *op; 2203 unsigned long ip = rec->ip; 2204 2205 /* 2206 * Need to check removed ops first. 2207 * If they are being removed, and this rec has a tramp, 2208 * and this rec is in the ops list, then it would be the 2209 * one with the tramp. 2210 */ 2211 if (removed_ops) { 2212 if (hash_contains_ip(ip, &removed_ops->old_hash)) 2213 return removed_ops; 2214 } 2215 2216 /* 2217 * Need to find the current trampoline for a rec. 2218 * Now, a trampoline is only attached to a rec if there 2219 * was a single 'ops' attached to it. But this can be called 2220 * when we are adding another op to the rec or removing the 2221 * current one. Thus, if the op is being added, we can 2222 * ignore it because it hasn't attached itself to the rec 2223 * yet. 2224 * 2225 * If an ops is being modified (hooking to different functions) 2226 * then we don't care about the new functions that are being 2227 * added, just the old ones (that are probably being removed). 2228 * 2229 * If we are adding an ops to a function that already is using 2230 * a trampoline, it needs to be removed (trampolines are only 2231 * for single ops connected), then an ops that is not being 2232 * modified also needs to be checked. 2233 */ 2234 do_for_each_ftrace_op(op, ftrace_ops_list) { 2235 2236 if (!op->trampoline) 2237 continue; 2238 2239 /* 2240 * If the ops is being added, it hasn't gotten to 2241 * the point to be removed from this tree yet. 2242 */ 2243 if (op->flags & FTRACE_OPS_FL_ADDING) 2244 continue; 2245 2246 2247 /* 2248 * If the ops is being modified and is in the old 2249 * hash, then it is probably being removed from this 2250 * function. 2251 */ 2252 if ((op->flags & FTRACE_OPS_FL_MODIFYING) && 2253 hash_contains_ip(ip, &op->old_hash)) 2254 return op; 2255 /* 2256 * If the ops is not being added or modified, and it's 2257 * in its normal filter hash, then this must be the one 2258 * we want! 2259 */ 2260 if (!(op->flags & FTRACE_OPS_FL_MODIFYING) && 2261 hash_contains_ip(ip, op->func_hash)) 2262 return op; 2263 2264 } while_for_each_ftrace_op(op); 2265 2266 return NULL; 2267 } 2268 2269 static struct ftrace_ops * 2270 ftrace_find_tramp_ops_new(struct dyn_ftrace *rec) 2271 { 2272 struct ftrace_ops *op; 2273 unsigned long ip = rec->ip; 2274 2275 do_for_each_ftrace_op(op, ftrace_ops_list) { 2276 /* pass rec in as regs to have non-NULL val */ 2277 if (hash_contains_ip(ip, op->func_hash)) 2278 return op; 2279 } while_for_each_ftrace_op(op); 2280 2281 return NULL; 2282 } 2283 2284 /** 2285 * ftrace_get_addr_new - Get the call address to set to 2286 * @rec: The ftrace record descriptor 2287 * 2288 * If the record has the FTRACE_FL_REGS set, that means that it 2289 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS 2290 * is not not set, then it wants to convert to the normal callback. 2291 * 2292 * Returns the address of the trampoline to set to 2293 */ 2294 unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec) 2295 { 2296 struct ftrace_ops *ops; 2297 2298 /* Trampolines take precedence over regs */ 2299 if (rec->flags & FTRACE_FL_TRAMP) { 2300 ops = ftrace_find_tramp_ops_new(rec); 2301 if (FTRACE_WARN_ON(!ops || !ops->trampoline)) { 2302 pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n", 2303 (void *)rec->ip, (void *)rec->ip, rec->flags); 2304 /* Ftrace is shutting down, return anything */ 2305 return (unsigned long)FTRACE_ADDR; 2306 } 2307 return ops->trampoline; 2308 } 2309 2310 if (rec->flags & FTRACE_FL_REGS) 2311 return (unsigned long)FTRACE_REGS_ADDR; 2312 else 2313 return (unsigned long)FTRACE_ADDR; 2314 } 2315 2316 /** 2317 * ftrace_get_addr_curr - Get the call address that is already there 2318 * @rec: The ftrace record descriptor 2319 * 2320 * The FTRACE_FL_REGS_EN is set when the record already points to 2321 * a function that saves all the regs. Basically the '_EN' version 2322 * represents the current state of the function. 2323 * 2324 * Returns the address of the trampoline that is currently being called 2325 */ 2326 unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec) 2327 { 2328 struct ftrace_ops *ops; 2329 2330 /* Trampolines take precedence over regs */ 2331 if (rec->flags & FTRACE_FL_TRAMP_EN) { 2332 ops = ftrace_find_tramp_ops_curr(rec); 2333 if (FTRACE_WARN_ON(!ops)) { 2334 pr_warn("Bad trampoline accounting at: %p (%pS)\n", 2335 (void *)rec->ip, (void *)rec->ip); 2336 /* Ftrace is shutting down, return anything */ 2337 return (unsigned long)FTRACE_ADDR; 2338 } 2339 return ops->trampoline; 2340 } 2341 2342 if (rec->flags & FTRACE_FL_REGS_EN) 2343 return (unsigned long)FTRACE_REGS_ADDR; 2344 else 2345 return (unsigned long)FTRACE_ADDR; 2346 } 2347 2348 static int 2349 __ftrace_replace_code(struct dyn_ftrace *rec, int enable) 2350 { 2351 unsigned long ftrace_old_addr; 2352 unsigned long ftrace_addr; 2353 int ret; 2354 2355 ftrace_addr = ftrace_get_addr_new(rec); 2356 2357 /* This needs to be done before we call ftrace_update_record */ 2358 ftrace_old_addr = ftrace_get_addr_curr(rec); 2359 2360 ret = ftrace_update_record(rec, enable); 2361 2362 ftrace_bug_type = FTRACE_BUG_UNKNOWN; 2363 2364 switch (ret) { 2365 case FTRACE_UPDATE_IGNORE: 2366 return 0; 2367 2368 case FTRACE_UPDATE_MAKE_CALL: 2369 ftrace_bug_type = FTRACE_BUG_CALL; 2370 return ftrace_make_call(rec, ftrace_addr); 2371 2372 case FTRACE_UPDATE_MAKE_NOP: 2373 ftrace_bug_type = FTRACE_BUG_NOP; 2374 return ftrace_make_nop(NULL, rec, ftrace_old_addr); 2375 2376 case FTRACE_UPDATE_MODIFY_CALL: 2377 ftrace_bug_type = FTRACE_BUG_UPDATE; 2378 return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr); 2379 } 2380 2381 return -1; /* unknow ftrace bug */ 2382 } 2383 2384 void __weak ftrace_replace_code(int enable) 2385 { 2386 struct dyn_ftrace *rec; 2387 struct ftrace_page *pg; 2388 int failed; 2389 2390 if (unlikely(ftrace_disabled)) 2391 return; 2392 2393 do_for_each_ftrace_rec(pg, rec) { 2394 failed = __ftrace_replace_code(rec, enable); 2395 if (failed) { 2396 ftrace_bug(failed, rec); 2397 /* Stop processing */ 2398 return; 2399 } 2400 } while_for_each_ftrace_rec(); 2401 } 2402 2403 struct ftrace_rec_iter { 2404 struct ftrace_page *pg; 2405 int index; 2406 }; 2407 2408 /** 2409 * ftrace_rec_iter_start, start up iterating over traced functions 2410 * 2411 * Returns an iterator handle that is used to iterate over all 2412 * the records that represent address locations where functions 2413 * are traced. 2414 * 2415 * May return NULL if no records are available. 2416 */ 2417 struct ftrace_rec_iter *ftrace_rec_iter_start(void) 2418 { 2419 /* 2420 * We only use a single iterator. 2421 * Protected by the ftrace_lock mutex. 2422 */ 2423 static struct ftrace_rec_iter ftrace_rec_iter; 2424 struct ftrace_rec_iter *iter = &ftrace_rec_iter; 2425 2426 iter->pg = ftrace_pages_start; 2427 iter->index = 0; 2428 2429 /* Could have empty pages */ 2430 while (iter->pg && !iter->pg->index) 2431 iter->pg = iter->pg->next; 2432 2433 if (!iter->pg) 2434 return NULL; 2435 2436 return iter; 2437 } 2438 2439 /** 2440 * ftrace_rec_iter_next, get the next record to process. 2441 * @iter: The handle to the iterator. 2442 * 2443 * Returns the next iterator after the given iterator @iter. 2444 */ 2445 struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter) 2446 { 2447 iter->index++; 2448 2449 if (iter->index >= iter->pg->index) { 2450 iter->pg = iter->pg->next; 2451 iter->index = 0; 2452 2453 /* Could have empty pages */ 2454 while (iter->pg && !iter->pg->index) 2455 iter->pg = iter->pg->next; 2456 } 2457 2458 if (!iter->pg) 2459 return NULL; 2460 2461 return iter; 2462 } 2463 2464 /** 2465 * ftrace_rec_iter_record, get the record at the iterator location 2466 * @iter: The current iterator location 2467 * 2468 * Returns the record that the current @iter is at. 2469 */ 2470 struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter) 2471 { 2472 return &iter->pg->records[iter->index]; 2473 } 2474 2475 static int 2476 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec) 2477 { 2478 int ret; 2479 2480 if (unlikely(ftrace_disabled)) 2481 return 0; 2482 2483 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR); 2484 if (ret) { 2485 ftrace_bug_type = FTRACE_BUG_INIT; 2486 ftrace_bug(ret, rec); 2487 return 0; 2488 } 2489 return 1; 2490 } 2491 2492 /* 2493 * archs can override this function if they must do something 2494 * before the modifying code is performed. 2495 */ 2496 int __weak ftrace_arch_code_modify_prepare(void) 2497 { 2498 return 0; 2499 } 2500 2501 /* 2502 * archs can override this function if they must do something 2503 * after the modifying code is performed. 2504 */ 2505 int __weak ftrace_arch_code_modify_post_process(void) 2506 { 2507 return 0; 2508 } 2509 2510 void ftrace_modify_all_code(int command) 2511 { 2512 int update = command & FTRACE_UPDATE_TRACE_FUNC; 2513 int err = 0; 2514 2515 /* 2516 * If the ftrace_caller calls a ftrace_ops func directly, 2517 * we need to make sure that it only traces functions it 2518 * expects to trace. When doing the switch of functions, 2519 * we need to update to the ftrace_ops_list_func first 2520 * before the transition between old and new calls are set, 2521 * as the ftrace_ops_list_func will check the ops hashes 2522 * to make sure the ops are having the right functions 2523 * traced. 2524 */ 2525 if (update) { 2526 err = ftrace_update_ftrace_func(ftrace_ops_list_func); 2527 if (FTRACE_WARN_ON(err)) 2528 return; 2529 } 2530 2531 if (command & FTRACE_UPDATE_CALLS) 2532 ftrace_replace_code(1); 2533 else if (command & FTRACE_DISABLE_CALLS) 2534 ftrace_replace_code(0); 2535 2536 if (update && ftrace_trace_function != ftrace_ops_list_func) { 2537 function_trace_op = set_function_trace_op; 2538 smp_wmb(); 2539 /* If irqs are disabled, we are in stop machine */ 2540 if (!irqs_disabled()) 2541 smp_call_function(ftrace_sync_ipi, NULL, 1); 2542 err = ftrace_update_ftrace_func(ftrace_trace_function); 2543 if (FTRACE_WARN_ON(err)) 2544 return; 2545 } 2546 2547 if (command & FTRACE_START_FUNC_RET) 2548 err = ftrace_enable_ftrace_graph_caller(); 2549 else if (command & FTRACE_STOP_FUNC_RET) 2550 err = ftrace_disable_ftrace_graph_caller(); 2551 FTRACE_WARN_ON(err); 2552 } 2553 2554 static int __ftrace_modify_code(void *data) 2555 { 2556 int *command = data; 2557 2558 ftrace_modify_all_code(*command); 2559 2560 return 0; 2561 } 2562 2563 /** 2564 * ftrace_run_stop_machine, go back to the stop machine method 2565 * @command: The command to tell ftrace what to do 2566 * 2567 * If an arch needs to fall back to the stop machine method, the 2568 * it can call this function. 2569 */ 2570 void ftrace_run_stop_machine(int command) 2571 { 2572 stop_machine(__ftrace_modify_code, &command, NULL); 2573 } 2574 2575 /** 2576 * arch_ftrace_update_code, modify the code to trace or not trace 2577 * @command: The command that needs to be done 2578 * 2579 * Archs can override this function if it does not need to 2580 * run stop_machine() to modify code. 2581 */ 2582 void __weak arch_ftrace_update_code(int command) 2583 { 2584 ftrace_run_stop_machine(command); 2585 } 2586 2587 static void ftrace_run_update_code(int command) 2588 { 2589 int ret; 2590 2591 ret = ftrace_arch_code_modify_prepare(); 2592 FTRACE_WARN_ON(ret); 2593 if (ret) 2594 return; 2595 2596 /* 2597 * By default we use stop_machine() to modify the code. 2598 * But archs can do what ever they want as long as it 2599 * is safe. The stop_machine() is the safest, but also 2600 * produces the most overhead. 2601 */ 2602 arch_ftrace_update_code(command); 2603 2604 ret = ftrace_arch_code_modify_post_process(); 2605 FTRACE_WARN_ON(ret); 2606 } 2607 2608 static void ftrace_run_modify_code(struct ftrace_ops *ops, int command, 2609 struct ftrace_ops_hash *old_hash) 2610 { 2611 ops->flags |= FTRACE_OPS_FL_MODIFYING; 2612 ops->old_hash.filter_hash = old_hash->filter_hash; 2613 ops->old_hash.notrace_hash = old_hash->notrace_hash; 2614 ftrace_run_update_code(command); 2615 ops->old_hash.filter_hash = NULL; 2616 ops->old_hash.notrace_hash = NULL; 2617 ops->flags &= ~FTRACE_OPS_FL_MODIFYING; 2618 } 2619 2620 static ftrace_func_t saved_ftrace_func; 2621 static int ftrace_start_up; 2622 2623 void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops) 2624 { 2625 } 2626 2627 static void per_cpu_ops_free(struct ftrace_ops *ops) 2628 { 2629 free_percpu(ops->disabled); 2630 } 2631 2632 static void ftrace_startup_enable(int command) 2633 { 2634 if (saved_ftrace_func != ftrace_trace_function) { 2635 saved_ftrace_func = ftrace_trace_function; 2636 command |= FTRACE_UPDATE_TRACE_FUNC; 2637 } 2638 2639 if (!command || !ftrace_enabled) 2640 return; 2641 2642 ftrace_run_update_code(command); 2643 } 2644 2645 static void ftrace_startup_all(int command) 2646 { 2647 update_all_ops = true; 2648 ftrace_startup_enable(command); 2649 update_all_ops = false; 2650 } 2651 2652 static int ftrace_startup(struct ftrace_ops *ops, int command) 2653 { 2654 int ret; 2655 2656 if (unlikely(ftrace_disabled)) 2657 return -ENODEV; 2658 2659 ret = __register_ftrace_function(ops); 2660 if (ret) 2661 return ret; 2662 2663 ftrace_start_up++; 2664 2665 /* 2666 * Note that ftrace probes uses this to start up 2667 * and modify functions it will probe. But we still 2668 * set the ADDING flag for modification, as probes 2669 * do not have trampolines. If they add them in the 2670 * future, then the probes will need to distinguish 2671 * between adding and updating probes. 2672 */ 2673 ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING; 2674 2675 ret = ftrace_hash_ipmodify_enable(ops); 2676 if (ret < 0) { 2677 /* Rollback registration process */ 2678 __unregister_ftrace_function(ops); 2679 ftrace_start_up--; 2680 ops->flags &= ~FTRACE_OPS_FL_ENABLED; 2681 return ret; 2682 } 2683 2684 if (ftrace_hash_rec_enable(ops, 1)) 2685 command |= FTRACE_UPDATE_CALLS; 2686 2687 ftrace_startup_enable(command); 2688 2689 ops->flags &= ~FTRACE_OPS_FL_ADDING; 2690 2691 return 0; 2692 } 2693 2694 static int ftrace_shutdown(struct ftrace_ops *ops, int command) 2695 { 2696 int ret; 2697 2698 if (unlikely(ftrace_disabled)) 2699 return -ENODEV; 2700 2701 ret = __unregister_ftrace_function(ops); 2702 if (ret) 2703 return ret; 2704 2705 ftrace_start_up--; 2706 /* 2707 * Just warn in case of unbalance, no need to kill ftrace, it's not 2708 * critical but the ftrace_call callers may be never nopped again after 2709 * further ftrace uses. 2710 */ 2711 WARN_ON_ONCE(ftrace_start_up < 0); 2712 2713 /* Disabling ipmodify never fails */ 2714 ftrace_hash_ipmodify_disable(ops); 2715 2716 if (ftrace_hash_rec_disable(ops, 1)) 2717 command |= FTRACE_UPDATE_CALLS; 2718 2719 ops->flags &= ~FTRACE_OPS_FL_ENABLED; 2720 2721 if (saved_ftrace_func != ftrace_trace_function) { 2722 saved_ftrace_func = ftrace_trace_function; 2723 command |= FTRACE_UPDATE_TRACE_FUNC; 2724 } 2725 2726 if (!command || !ftrace_enabled) { 2727 /* 2728 * If these are per_cpu ops, they still need their 2729 * per_cpu field freed. Since, function tracing is 2730 * not currently active, we can just free them 2731 * without synchronizing all CPUs. 2732 */ 2733 if (ops->flags & FTRACE_OPS_FL_PER_CPU) 2734 per_cpu_ops_free(ops); 2735 return 0; 2736 } 2737 2738 /* 2739 * If the ops uses a trampoline, then it needs to be 2740 * tested first on update. 2741 */ 2742 ops->flags |= FTRACE_OPS_FL_REMOVING; 2743 removed_ops = ops; 2744 2745 /* The trampoline logic checks the old hashes */ 2746 ops->old_hash.filter_hash = ops->func_hash->filter_hash; 2747 ops->old_hash.notrace_hash = ops->func_hash->notrace_hash; 2748 2749 ftrace_run_update_code(command); 2750 2751 /* 2752 * If there's no more ops registered with ftrace, run a 2753 * sanity check to make sure all rec flags are cleared. 2754 */ 2755 if (ftrace_ops_list == &ftrace_list_end) { 2756 struct ftrace_page *pg; 2757 struct dyn_ftrace *rec; 2758 2759 do_for_each_ftrace_rec(pg, rec) { 2760 if (FTRACE_WARN_ON_ONCE(rec->flags)) 2761 pr_warn(" %pS flags:%lx\n", 2762 (void *)rec->ip, rec->flags); 2763 } while_for_each_ftrace_rec(); 2764 } 2765 2766 ops->old_hash.filter_hash = NULL; 2767 ops->old_hash.notrace_hash = NULL; 2768 2769 removed_ops = NULL; 2770 ops->flags &= ~FTRACE_OPS_FL_REMOVING; 2771 2772 /* 2773 * Dynamic ops may be freed, we must make sure that all 2774 * callers are done before leaving this function. 2775 * The same goes for freeing the per_cpu data of the per_cpu 2776 * ops. 2777 * 2778 * Again, normal synchronize_sched() is not good enough. 2779 * We need to do a hard force of sched synchronization. 2780 * This is because we use preempt_disable() to do RCU, but 2781 * the function tracers can be called where RCU is not watching 2782 * (like before user_exit()). We can not rely on the RCU 2783 * infrastructure to do the synchronization, thus we must do it 2784 * ourselves. 2785 */ 2786 if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU)) { 2787 schedule_on_each_cpu(ftrace_sync); 2788 2789 arch_ftrace_trampoline_free(ops); 2790 2791 if (ops->flags & FTRACE_OPS_FL_PER_CPU) 2792 per_cpu_ops_free(ops); 2793 } 2794 2795 return 0; 2796 } 2797 2798 static void ftrace_startup_sysctl(void) 2799 { 2800 int command; 2801 2802 if (unlikely(ftrace_disabled)) 2803 return; 2804 2805 /* Force update next time */ 2806 saved_ftrace_func = NULL; 2807 /* ftrace_start_up is true if we want ftrace running */ 2808 if (ftrace_start_up) { 2809 command = FTRACE_UPDATE_CALLS; 2810 if (ftrace_graph_active) 2811 command |= FTRACE_START_FUNC_RET; 2812 ftrace_startup_enable(command); 2813 } 2814 } 2815 2816 static void ftrace_shutdown_sysctl(void) 2817 { 2818 int command; 2819 2820 if (unlikely(ftrace_disabled)) 2821 return; 2822 2823 /* ftrace_start_up is true if ftrace is running */ 2824 if (ftrace_start_up) { 2825 command = FTRACE_DISABLE_CALLS; 2826 if (ftrace_graph_active) 2827 command |= FTRACE_STOP_FUNC_RET; 2828 ftrace_run_update_code(command); 2829 } 2830 } 2831 2832 static cycle_t ftrace_update_time; 2833 unsigned long ftrace_update_tot_cnt; 2834 2835 static inline int ops_traces_mod(struct ftrace_ops *ops) 2836 { 2837 /* 2838 * Filter_hash being empty will default to trace module. 2839 * But notrace hash requires a test of individual module functions. 2840 */ 2841 return ftrace_hash_empty(ops->func_hash->filter_hash) && 2842 ftrace_hash_empty(ops->func_hash->notrace_hash); 2843 } 2844 2845 /* 2846 * Check if the current ops references the record. 2847 * 2848 * If the ops traces all functions, then it was already accounted for. 2849 * If the ops does not trace the current record function, skip it. 2850 * If the ops ignores the function via notrace filter, skip it. 2851 */ 2852 static inline bool 2853 ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec) 2854 { 2855 /* If ops isn't enabled, ignore it */ 2856 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 2857 return 0; 2858 2859 /* If ops traces all then it includes this function */ 2860 if (ops_traces_mod(ops)) 2861 return 1; 2862 2863 /* The function must be in the filter */ 2864 if (!ftrace_hash_empty(ops->func_hash->filter_hash) && 2865 !ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip)) 2866 return 0; 2867 2868 /* If in notrace hash, we ignore it too */ 2869 if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) 2870 return 0; 2871 2872 return 1; 2873 } 2874 2875 static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs) 2876 { 2877 struct ftrace_page *pg; 2878 struct dyn_ftrace *p; 2879 cycle_t start, stop; 2880 unsigned long update_cnt = 0; 2881 unsigned long rec_flags = 0; 2882 int i; 2883 2884 start = ftrace_now(raw_smp_processor_id()); 2885 2886 /* 2887 * When a module is loaded, this function is called to convert 2888 * the calls to mcount in its text to nops, and also to create 2889 * an entry in the ftrace data. Now, if ftrace is activated 2890 * after this call, but before the module sets its text to 2891 * read-only, the modification of enabling ftrace can fail if 2892 * the read-only is done while ftrace is converting the calls. 2893 * To prevent this, the module's records are set as disabled 2894 * and will be enabled after the call to set the module's text 2895 * to read-only. 2896 */ 2897 if (mod) 2898 rec_flags |= FTRACE_FL_DISABLED; 2899 2900 for (pg = new_pgs; pg; pg = pg->next) { 2901 2902 for (i = 0; i < pg->index; i++) { 2903 2904 /* If something went wrong, bail without enabling anything */ 2905 if (unlikely(ftrace_disabled)) 2906 return -1; 2907 2908 p = &pg->records[i]; 2909 p->flags = rec_flags; 2910 2911 /* 2912 * Do the initial record conversion from mcount jump 2913 * to the NOP instructions. 2914 */ 2915 if (!ftrace_code_disable(mod, p)) 2916 break; 2917 2918 update_cnt++; 2919 } 2920 } 2921 2922 stop = ftrace_now(raw_smp_processor_id()); 2923 ftrace_update_time = stop - start; 2924 ftrace_update_tot_cnt += update_cnt; 2925 2926 return 0; 2927 } 2928 2929 static int ftrace_allocate_records(struct ftrace_page *pg, int count) 2930 { 2931 int order; 2932 int cnt; 2933 2934 if (WARN_ON(!count)) 2935 return -EINVAL; 2936 2937 order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE)); 2938 2939 /* 2940 * We want to fill as much as possible. No more than a page 2941 * may be empty. 2942 */ 2943 while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE) 2944 order--; 2945 2946 again: 2947 pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); 2948 2949 if (!pg->records) { 2950 /* if we can't allocate this size, try something smaller */ 2951 if (!order) 2952 return -ENOMEM; 2953 order >>= 1; 2954 goto again; 2955 } 2956 2957 cnt = (PAGE_SIZE << order) / ENTRY_SIZE; 2958 pg->size = cnt; 2959 2960 if (cnt > count) 2961 cnt = count; 2962 2963 return cnt; 2964 } 2965 2966 static struct ftrace_page * 2967 ftrace_allocate_pages(unsigned long num_to_init) 2968 { 2969 struct ftrace_page *start_pg; 2970 struct ftrace_page *pg; 2971 int order; 2972 int cnt; 2973 2974 if (!num_to_init) 2975 return 0; 2976 2977 start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL); 2978 if (!pg) 2979 return NULL; 2980 2981 /* 2982 * Try to allocate as much as possible in one continues 2983 * location that fills in all of the space. We want to 2984 * waste as little space as possible. 2985 */ 2986 for (;;) { 2987 cnt = ftrace_allocate_records(pg, num_to_init); 2988 if (cnt < 0) 2989 goto free_pages; 2990 2991 num_to_init -= cnt; 2992 if (!num_to_init) 2993 break; 2994 2995 pg->next = kzalloc(sizeof(*pg), GFP_KERNEL); 2996 if (!pg->next) 2997 goto free_pages; 2998 2999 pg = pg->next; 3000 } 3001 3002 return start_pg; 3003 3004 free_pages: 3005 pg = start_pg; 3006 while (pg) { 3007 order = get_count_order(pg->size / ENTRIES_PER_PAGE); 3008 free_pages((unsigned long)pg->records, order); 3009 start_pg = pg->next; 3010 kfree(pg); 3011 pg = start_pg; 3012 } 3013 pr_info("ftrace: FAILED to allocate memory for functions\n"); 3014 return NULL; 3015 } 3016 3017 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ 3018 3019 struct ftrace_iterator { 3020 loff_t pos; 3021 loff_t func_pos; 3022 struct ftrace_page *pg; 3023 struct dyn_ftrace *func; 3024 struct ftrace_func_probe *probe; 3025 struct trace_parser parser; 3026 struct ftrace_hash *hash; 3027 struct ftrace_ops *ops; 3028 int hidx; 3029 int idx; 3030 unsigned flags; 3031 }; 3032 3033 static void * 3034 t_hash_next(struct seq_file *m, loff_t *pos) 3035 { 3036 struct ftrace_iterator *iter = m->private; 3037 struct hlist_node *hnd = NULL; 3038 struct hlist_head *hhd; 3039 3040 (*pos)++; 3041 iter->pos = *pos; 3042 3043 if (iter->probe) 3044 hnd = &iter->probe->node; 3045 retry: 3046 if (iter->hidx >= FTRACE_FUNC_HASHSIZE) 3047 return NULL; 3048 3049 hhd = &ftrace_func_hash[iter->hidx]; 3050 3051 if (hlist_empty(hhd)) { 3052 iter->hidx++; 3053 hnd = NULL; 3054 goto retry; 3055 } 3056 3057 if (!hnd) 3058 hnd = hhd->first; 3059 else { 3060 hnd = hnd->next; 3061 if (!hnd) { 3062 iter->hidx++; 3063 goto retry; 3064 } 3065 } 3066 3067 if (WARN_ON_ONCE(!hnd)) 3068 return NULL; 3069 3070 iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node); 3071 3072 return iter; 3073 } 3074 3075 static void *t_hash_start(struct seq_file *m, loff_t *pos) 3076 { 3077 struct ftrace_iterator *iter = m->private; 3078 void *p = NULL; 3079 loff_t l; 3080 3081 if (!(iter->flags & FTRACE_ITER_DO_HASH)) 3082 return NULL; 3083 3084 if (iter->func_pos > *pos) 3085 return NULL; 3086 3087 iter->hidx = 0; 3088 for (l = 0; l <= (*pos - iter->func_pos); ) { 3089 p = t_hash_next(m, &l); 3090 if (!p) 3091 break; 3092 } 3093 if (!p) 3094 return NULL; 3095 3096 /* Only set this if we have an item */ 3097 iter->flags |= FTRACE_ITER_HASH; 3098 3099 return iter; 3100 } 3101 3102 static int 3103 t_hash_show(struct seq_file *m, struct ftrace_iterator *iter) 3104 { 3105 struct ftrace_func_probe *rec; 3106 3107 rec = iter->probe; 3108 if (WARN_ON_ONCE(!rec)) 3109 return -EIO; 3110 3111 if (rec->ops->print) 3112 return rec->ops->print(m, rec->ip, rec->ops, rec->data); 3113 3114 seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func); 3115 3116 if (rec->data) 3117 seq_printf(m, ":%p", rec->data); 3118 seq_putc(m, '\n'); 3119 3120 return 0; 3121 } 3122 3123 static void * 3124 t_next(struct seq_file *m, void *v, loff_t *pos) 3125 { 3126 struct ftrace_iterator *iter = m->private; 3127 struct ftrace_ops *ops = iter->ops; 3128 struct dyn_ftrace *rec = NULL; 3129 3130 if (unlikely(ftrace_disabled)) 3131 return NULL; 3132 3133 if (iter->flags & FTRACE_ITER_HASH) 3134 return t_hash_next(m, pos); 3135 3136 (*pos)++; 3137 iter->pos = iter->func_pos = *pos; 3138 3139 if (iter->flags & FTRACE_ITER_PRINTALL) 3140 return t_hash_start(m, pos); 3141 3142 retry: 3143 if (iter->idx >= iter->pg->index) { 3144 if (iter->pg->next) { 3145 iter->pg = iter->pg->next; 3146 iter->idx = 0; 3147 goto retry; 3148 } 3149 } else { 3150 rec = &iter->pg->records[iter->idx++]; 3151 if (((iter->flags & FTRACE_ITER_FILTER) && 3152 !(ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))) || 3153 3154 ((iter->flags & FTRACE_ITER_NOTRACE) && 3155 !ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) || 3156 3157 ((iter->flags & FTRACE_ITER_ENABLED) && 3158 !(rec->flags & FTRACE_FL_ENABLED))) { 3159 3160 rec = NULL; 3161 goto retry; 3162 } 3163 } 3164 3165 if (!rec) 3166 return t_hash_start(m, pos); 3167 3168 iter->func = rec; 3169 3170 return iter; 3171 } 3172 3173 static void reset_iter_read(struct ftrace_iterator *iter) 3174 { 3175 iter->pos = 0; 3176 iter->func_pos = 0; 3177 iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH); 3178 } 3179 3180 static void *t_start(struct seq_file *m, loff_t *pos) 3181 { 3182 struct ftrace_iterator *iter = m->private; 3183 struct ftrace_ops *ops = iter->ops; 3184 void *p = NULL; 3185 loff_t l; 3186 3187 mutex_lock(&ftrace_lock); 3188 3189 if (unlikely(ftrace_disabled)) 3190 return NULL; 3191 3192 /* 3193 * If an lseek was done, then reset and start from beginning. 3194 */ 3195 if (*pos < iter->pos) 3196 reset_iter_read(iter); 3197 3198 /* 3199 * For set_ftrace_filter reading, if we have the filter 3200 * off, we can short cut and just print out that all 3201 * functions are enabled. 3202 */ 3203 if ((iter->flags & FTRACE_ITER_FILTER && 3204 ftrace_hash_empty(ops->func_hash->filter_hash)) || 3205 (iter->flags & FTRACE_ITER_NOTRACE && 3206 ftrace_hash_empty(ops->func_hash->notrace_hash))) { 3207 if (*pos > 0) 3208 return t_hash_start(m, pos); 3209 iter->flags |= FTRACE_ITER_PRINTALL; 3210 /* reset in case of seek/pread */ 3211 iter->flags &= ~FTRACE_ITER_HASH; 3212 return iter; 3213 } 3214 3215 if (iter->flags & FTRACE_ITER_HASH) 3216 return t_hash_start(m, pos); 3217 3218 /* 3219 * Unfortunately, we need to restart at ftrace_pages_start 3220 * every time we let go of the ftrace_mutex. This is because 3221 * those pointers can change without the lock. 3222 */ 3223 iter->pg = ftrace_pages_start; 3224 iter->idx = 0; 3225 for (l = 0; l <= *pos; ) { 3226 p = t_next(m, p, &l); 3227 if (!p) 3228 break; 3229 } 3230 3231 if (!p) 3232 return t_hash_start(m, pos); 3233 3234 return iter; 3235 } 3236 3237 static void t_stop(struct seq_file *m, void *p) 3238 { 3239 mutex_unlock(&ftrace_lock); 3240 } 3241 3242 void * __weak 3243 arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec) 3244 { 3245 return NULL; 3246 } 3247 3248 static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops, 3249 struct dyn_ftrace *rec) 3250 { 3251 void *ptr; 3252 3253 ptr = arch_ftrace_trampoline_func(ops, rec); 3254 if (ptr) 3255 seq_printf(m, " ->%pS", ptr); 3256 } 3257 3258 static int t_show(struct seq_file *m, void *v) 3259 { 3260 struct ftrace_iterator *iter = m->private; 3261 struct dyn_ftrace *rec; 3262 3263 if (iter->flags & FTRACE_ITER_HASH) 3264 return t_hash_show(m, iter); 3265 3266 if (iter->flags & FTRACE_ITER_PRINTALL) { 3267 if (iter->flags & FTRACE_ITER_NOTRACE) 3268 seq_puts(m, "#### no functions disabled ####\n"); 3269 else 3270 seq_puts(m, "#### all functions enabled ####\n"); 3271 return 0; 3272 } 3273 3274 rec = iter->func; 3275 3276 if (!rec) 3277 return 0; 3278 3279 seq_printf(m, "%ps", (void *)rec->ip); 3280 if (iter->flags & FTRACE_ITER_ENABLED) { 3281 struct ftrace_ops *ops; 3282 3283 seq_printf(m, " (%ld)%s%s", 3284 ftrace_rec_count(rec), 3285 rec->flags & FTRACE_FL_REGS ? " R" : " ", 3286 rec->flags & FTRACE_FL_IPMODIFY ? " I" : " "); 3287 if (rec->flags & FTRACE_FL_TRAMP_EN) { 3288 ops = ftrace_find_tramp_ops_any(rec); 3289 if (ops) { 3290 do { 3291 seq_printf(m, "\ttramp: %pS (%pS)", 3292 (void *)ops->trampoline, 3293 (void *)ops->func); 3294 add_trampoline_func(m, ops, rec); 3295 ops = ftrace_find_tramp_ops_next(rec, ops); 3296 } while (ops); 3297 } else 3298 seq_puts(m, "\ttramp: ERROR!"); 3299 } else { 3300 add_trampoline_func(m, NULL, rec); 3301 } 3302 } 3303 3304 seq_putc(m, '\n'); 3305 3306 return 0; 3307 } 3308 3309 static const struct seq_operations show_ftrace_seq_ops = { 3310 .start = t_start, 3311 .next = t_next, 3312 .stop = t_stop, 3313 .show = t_show, 3314 }; 3315 3316 static int 3317 ftrace_avail_open(struct inode *inode, struct file *file) 3318 { 3319 struct ftrace_iterator *iter; 3320 3321 if (unlikely(ftrace_disabled)) 3322 return -ENODEV; 3323 3324 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); 3325 if (iter) { 3326 iter->pg = ftrace_pages_start; 3327 iter->ops = &global_ops; 3328 } 3329 3330 return iter ? 0 : -ENOMEM; 3331 } 3332 3333 static int 3334 ftrace_enabled_open(struct inode *inode, struct file *file) 3335 { 3336 struct ftrace_iterator *iter; 3337 3338 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); 3339 if (iter) { 3340 iter->pg = ftrace_pages_start; 3341 iter->flags = FTRACE_ITER_ENABLED; 3342 iter->ops = &global_ops; 3343 } 3344 3345 return iter ? 0 : -ENOMEM; 3346 } 3347 3348 /** 3349 * ftrace_regex_open - initialize function tracer filter files 3350 * @ops: The ftrace_ops that hold the hash filters 3351 * @flag: The type of filter to process 3352 * @inode: The inode, usually passed in to your open routine 3353 * @file: The file, usually passed in to your open routine 3354 * 3355 * ftrace_regex_open() initializes the filter files for the 3356 * @ops. Depending on @flag it may process the filter hash or 3357 * the notrace hash of @ops. With this called from the open 3358 * routine, you can use ftrace_filter_write() for the write 3359 * routine if @flag has FTRACE_ITER_FILTER set, or 3360 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set. 3361 * tracing_lseek() should be used as the lseek routine, and 3362 * release must call ftrace_regex_release(). 3363 */ 3364 int 3365 ftrace_regex_open(struct ftrace_ops *ops, int flag, 3366 struct inode *inode, struct file *file) 3367 { 3368 struct ftrace_iterator *iter; 3369 struct ftrace_hash *hash; 3370 int ret = 0; 3371 3372 ftrace_ops_init(ops); 3373 3374 if (unlikely(ftrace_disabled)) 3375 return -ENODEV; 3376 3377 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 3378 if (!iter) 3379 return -ENOMEM; 3380 3381 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) { 3382 kfree(iter); 3383 return -ENOMEM; 3384 } 3385 3386 iter->ops = ops; 3387 iter->flags = flag; 3388 3389 mutex_lock(&ops->func_hash->regex_lock); 3390 3391 if (flag & FTRACE_ITER_NOTRACE) 3392 hash = ops->func_hash->notrace_hash; 3393 else 3394 hash = ops->func_hash->filter_hash; 3395 3396 if (file->f_mode & FMODE_WRITE) { 3397 const int size_bits = FTRACE_HASH_DEFAULT_BITS; 3398 3399 if (file->f_flags & O_TRUNC) 3400 iter->hash = alloc_ftrace_hash(size_bits); 3401 else 3402 iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash); 3403 3404 if (!iter->hash) { 3405 trace_parser_put(&iter->parser); 3406 kfree(iter); 3407 ret = -ENOMEM; 3408 goto out_unlock; 3409 } 3410 } 3411 3412 if (file->f_mode & FMODE_READ) { 3413 iter->pg = ftrace_pages_start; 3414 3415 ret = seq_open(file, &show_ftrace_seq_ops); 3416 if (!ret) { 3417 struct seq_file *m = file->private_data; 3418 m->private = iter; 3419 } else { 3420 /* Failed */ 3421 free_ftrace_hash(iter->hash); 3422 trace_parser_put(&iter->parser); 3423 kfree(iter); 3424 } 3425 } else 3426 file->private_data = iter; 3427 3428 out_unlock: 3429 mutex_unlock(&ops->func_hash->regex_lock); 3430 3431 return ret; 3432 } 3433 3434 static int 3435 ftrace_filter_open(struct inode *inode, struct file *file) 3436 { 3437 struct ftrace_ops *ops = inode->i_private; 3438 3439 return ftrace_regex_open(ops, 3440 FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH, 3441 inode, file); 3442 } 3443 3444 static int 3445 ftrace_notrace_open(struct inode *inode, struct file *file) 3446 { 3447 struct ftrace_ops *ops = inode->i_private; 3448 3449 return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE, 3450 inode, file); 3451 } 3452 3453 /* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */ 3454 struct ftrace_glob { 3455 char *search; 3456 unsigned len; 3457 int type; 3458 }; 3459 3460 /* 3461 * If symbols in an architecture don't correspond exactly to the user-visible 3462 * name of what they represent, it is possible to define this function to 3463 * perform the necessary adjustments. 3464 */ 3465 char * __weak arch_ftrace_match_adjust(char *str, const char *search) 3466 { 3467 return str; 3468 } 3469 3470 static int ftrace_match(char *str, struct ftrace_glob *g) 3471 { 3472 int matched = 0; 3473 int slen; 3474 3475 str = arch_ftrace_match_adjust(str, g->search); 3476 3477 switch (g->type) { 3478 case MATCH_FULL: 3479 if (strcmp(str, g->search) == 0) 3480 matched = 1; 3481 break; 3482 case MATCH_FRONT_ONLY: 3483 if (strncmp(str, g->search, g->len) == 0) 3484 matched = 1; 3485 break; 3486 case MATCH_MIDDLE_ONLY: 3487 if (strstr(str, g->search)) 3488 matched = 1; 3489 break; 3490 case MATCH_END_ONLY: 3491 slen = strlen(str); 3492 if (slen >= g->len && 3493 memcmp(str + slen - g->len, g->search, g->len) == 0) 3494 matched = 1; 3495 break; 3496 } 3497 3498 return matched; 3499 } 3500 3501 static int 3502 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter) 3503 { 3504 struct ftrace_func_entry *entry; 3505 int ret = 0; 3506 3507 entry = ftrace_lookup_ip(hash, rec->ip); 3508 if (clear_filter) { 3509 /* Do nothing if it doesn't exist */ 3510 if (!entry) 3511 return 0; 3512 3513 free_hash_entry(hash, entry); 3514 } else { 3515 /* Do nothing if it exists */ 3516 if (entry) 3517 return 0; 3518 3519 ret = add_hash_entry(hash, rec->ip); 3520 } 3521 return ret; 3522 } 3523 3524 static int 3525 ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g, 3526 struct ftrace_glob *mod_g, int exclude_mod) 3527 { 3528 char str[KSYM_SYMBOL_LEN]; 3529 char *modname; 3530 3531 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); 3532 3533 if (mod_g) { 3534 int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0; 3535 3536 /* blank module name to match all modules */ 3537 if (!mod_g->len) { 3538 /* blank module globbing: modname xor exclude_mod */ 3539 if ((!exclude_mod) != (!modname)) 3540 goto func_match; 3541 return 0; 3542 } 3543 3544 /* not matching the module */ 3545 if (!modname || !mod_matches) { 3546 if (exclude_mod) 3547 goto func_match; 3548 else 3549 return 0; 3550 } 3551 3552 if (mod_matches && exclude_mod) 3553 return 0; 3554 3555 func_match: 3556 /* blank search means to match all funcs in the mod */ 3557 if (!func_g->len) 3558 return 1; 3559 } 3560 3561 return ftrace_match(str, func_g); 3562 } 3563 3564 static int 3565 match_records(struct ftrace_hash *hash, char *func, int len, char *mod) 3566 { 3567 struct ftrace_page *pg; 3568 struct dyn_ftrace *rec; 3569 struct ftrace_glob func_g = { .type = MATCH_FULL }; 3570 struct ftrace_glob mod_g = { .type = MATCH_FULL }; 3571 struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL; 3572 int exclude_mod = 0; 3573 int found = 0; 3574 int ret; 3575 int clear_filter; 3576 3577 if (func) { 3578 func_g.type = filter_parse_regex(func, len, &func_g.search, 3579 &clear_filter); 3580 func_g.len = strlen(func_g.search); 3581 } 3582 3583 if (mod) { 3584 mod_g.type = filter_parse_regex(mod, strlen(mod), 3585 &mod_g.search, &exclude_mod); 3586 mod_g.len = strlen(mod_g.search); 3587 } 3588 3589 mutex_lock(&ftrace_lock); 3590 3591 if (unlikely(ftrace_disabled)) 3592 goto out_unlock; 3593 3594 do_for_each_ftrace_rec(pg, rec) { 3595 if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) { 3596 ret = enter_record(hash, rec, clear_filter); 3597 if (ret < 0) { 3598 found = ret; 3599 goto out_unlock; 3600 } 3601 found = 1; 3602 } 3603 } while_for_each_ftrace_rec(); 3604 out_unlock: 3605 mutex_unlock(&ftrace_lock); 3606 3607 return found; 3608 } 3609 3610 static int 3611 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len) 3612 { 3613 return match_records(hash, buff, len, NULL); 3614 } 3615 3616 3617 /* 3618 * We register the module command as a template to show others how 3619 * to register the a command as well. 3620 */ 3621 3622 static int 3623 ftrace_mod_callback(struct ftrace_hash *hash, 3624 char *func, char *cmd, char *module, int enable) 3625 { 3626 int ret; 3627 3628 /* 3629 * cmd == 'mod' because we only registered this func 3630 * for the 'mod' ftrace_func_command. 3631 * But if you register one func with multiple commands, 3632 * you can tell which command was used by the cmd 3633 * parameter. 3634 */ 3635 ret = match_records(hash, func, strlen(func), module); 3636 if (!ret) 3637 return -EINVAL; 3638 if (ret < 0) 3639 return ret; 3640 return 0; 3641 } 3642 3643 static struct ftrace_func_command ftrace_mod_cmd = { 3644 .name = "mod", 3645 .func = ftrace_mod_callback, 3646 }; 3647 3648 static int __init ftrace_mod_cmd_init(void) 3649 { 3650 return register_ftrace_command(&ftrace_mod_cmd); 3651 } 3652 core_initcall(ftrace_mod_cmd_init); 3653 3654 static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip, 3655 struct ftrace_ops *op, struct pt_regs *pt_regs) 3656 { 3657 struct ftrace_func_probe *entry; 3658 struct hlist_head *hhd; 3659 unsigned long key; 3660 3661 key = hash_long(ip, FTRACE_HASH_BITS); 3662 3663 hhd = &ftrace_func_hash[key]; 3664 3665 if (hlist_empty(hhd)) 3666 return; 3667 3668 /* 3669 * Disable preemption for these calls to prevent a RCU grace 3670 * period. This syncs the hash iteration and freeing of items 3671 * on the hash. rcu_read_lock is too dangerous here. 3672 */ 3673 preempt_disable_notrace(); 3674 hlist_for_each_entry_rcu_notrace(entry, hhd, node) { 3675 if (entry->ip == ip) 3676 entry->ops->func(ip, parent_ip, &entry->data); 3677 } 3678 preempt_enable_notrace(); 3679 } 3680 3681 static struct ftrace_ops trace_probe_ops __read_mostly = 3682 { 3683 .func = function_trace_probe_call, 3684 .flags = FTRACE_OPS_FL_INITIALIZED, 3685 INIT_OPS_HASH(trace_probe_ops) 3686 }; 3687 3688 static int ftrace_probe_registered; 3689 3690 static void __enable_ftrace_function_probe(struct ftrace_ops_hash *old_hash) 3691 { 3692 int ret; 3693 int i; 3694 3695 if (ftrace_probe_registered) { 3696 /* still need to update the function call sites */ 3697 if (ftrace_enabled) 3698 ftrace_run_modify_code(&trace_probe_ops, FTRACE_UPDATE_CALLS, 3699 old_hash); 3700 return; 3701 } 3702 3703 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 3704 struct hlist_head *hhd = &ftrace_func_hash[i]; 3705 if (hhd->first) 3706 break; 3707 } 3708 /* Nothing registered? */ 3709 if (i == FTRACE_FUNC_HASHSIZE) 3710 return; 3711 3712 ret = ftrace_startup(&trace_probe_ops, 0); 3713 3714 ftrace_probe_registered = 1; 3715 } 3716 3717 static void __disable_ftrace_function_probe(void) 3718 { 3719 int i; 3720 3721 if (!ftrace_probe_registered) 3722 return; 3723 3724 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 3725 struct hlist_head *hhd = &ftrace_func_hash[i]; 3726 if (hhd->first) 3727 return; 3728 } 3729 3730 /* no more funcs left */ 3731 ftrace_shutdown(&trace_probe_ops, 0); 3732 3733 ftrace_probe_registered = 0; 3734 } 3735 3736 3737 static void ftrace_free_entry(struct ftrace_func_probe *entry) 3738 { 3739 if (entry->ops->free) 3740 entry->ops->free(entry->ops, entry->ip, &entry->data); 3741 kfree(entry); 3742 } 3743 3744 int 3745 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 3746 void *data) 3747 { 3748 struct ftrace_ops_hash old_hash_ops; 3749 struct ftrace_func_probe *entry; 3750 struct ftrace_glob func_g; 3751 struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash; 3752 struct ftrace_hash *old_hash = *orig_hash; 3753 struct ftrace_hash *hash; 3754 struct ftrace_page *pg; 3755 struct dyn_ftrace *rec; 3756 int not; 3757 unsigned long key; 3758 int count = 0; 3759 int ret; 3760 3761 func_g.type = filter_parse_regex(glob, strlen(glob), 3762 &func_g.search, ¬); 3763 func_g.len = strlen(func_g.search); 3764 3765 /* we do not support '!' for function probes */ 3766 if (WARN_ON(not)) 3767 return -EINVAL; 3768 3769 mutex_lock(&trace_probe_ops.func_hash->regex_lock); 3770 3771 old_hash_ops.filter_hash = old_hash; 3772 /* Probes only have filters */ 3773 old_hash_ops.notrace_hash = NULL; 3774 3775 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash); 3776 if (!hash) { 3777 count = -ENOMEM; 3778 goto out; 3779 } 3780 3781 if (unlikely(ftrace_disabled)) { 3782 count = -ENODEV; 3783 goto out; 3784 } 3785 3786 mutex_lock(&ftrace_lock); 3787 3788 do_for_each_ftrace_rec(pg, rec) { 3789 3790 if (!ftrace_match_record(rec, &func_g, NULL, 0)) 3791 continue; 3792 3793 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 3794 if (!entry) { 3795 /* If we did not process any, then return error */ 3796 if (!count) 3797 count = -ENOMEM; 3798 goto out_unlock; 3799 } 3800 3801 count++; 3802 3803 entry->data = data; 3804 3805 /* 3806 * The caller might want to do something special 3807 * for each function we find. We call the callback 3808 * to give the caller an opportunity to do so. 3809 */ 3810 if (ops->init) { 3811 if (ops->init(ops, rec->ip, &entry->data) < 0) { 3812 /* caller does not like this func */ 3813 kfree(entry); 3814 continue; 3815 } 3816 } 3817 3818 ret = enter_record(hash, rec, 0); 3819 if (ret < 0) { 3820 kfree(entry); 3821 count = ret; 3822 goto out_unlock; 3823 } 3824 3825 entry->ops = ops; 3826 entry->ip = rec->ip; 3827 3828 key = hash_long(entry->ip, FTRACE_HASH_BITS); 3829 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]); 3830 3831 } while_for_each_ftrace_rec(); 3832 3833 ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash); 3834 3835 __enable_ftrace_function_probe(&old_hash_ops); 3836 3837 if (!ret) 3838 free_ftrace_hash_rcu(old_hash); 3839 else 3840 count = ret; 3841 3842 out_unlock: 3843 mutex_unlock(&ftrace_lock); 3844 out: 3845 mutex_unlock(&trace_probe_ops.func_hash->regex_lock); 3846 free_ftrace_hash(hash); 3847 3848 return count; 3849 } 3850 3851 enum { 3852 PROBE_TEST_FUNC = 1, 3853 PROBE_TEST_DATA = 2 3854 }; 3855 3856 static void 3857 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 3858 void *data, int flags) 3859 { 3860 struct ftrace_func_entry *rec_entry; 3861 struct ftrace_func_probe *entry; 3862 struct ftrace_func_probe *p; 3863 struct ftrace_glob func_g; 3864 struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash; 3865 struct ftrace_hash *old_hash = *orig_hash; 3866 struct list_head free_list; 3867 struct ftrace_hash *hash; 3868 struct hlist_node *tmp; 3869 char str[KSYM_SYMBOL_LEN]; 3870 int i, ret; 3871 3872 if (glob && (strcmp(glob, "*") == 0 || !strlen(glob))) 3873 func_g.search = NULL; 3874 else if (glob) { 3875 int not; 3876 3877 func_g.type = filter_parse_regex(glob, strlen(glob), 3878 &func_g.search, ¬); 3879 func_g.len = strlen(func_g.search); 3880 func_g.search = glob; 3881 3882 /* we do not support '!' for function probes */ 3883 if (WARN_ON(not)) 3884 return; 3885 } 3886 3887 mutex_lock(&trace_probe_ops.func_hash->regex_lock); 3888 3889 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash); 3890 if (!hash) 3891 /* Hmm, should report this somehow */ 3892 goto out_unlock; 3893 3894 INIT_LIST_HEAD(&free_list); 3895 3896 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 3897 struct hlist_head *hhd = &ftrace_func_hash[i]; 3898 3899 hlist_for_each_entry_safe(entry, tmp, hhd, node) { 3900 3901 /* break up if statements for readability */ 3902 if ((flags & PROBE_TEST_FUNC) && entry->ops != ops) 3903 continue; 3904 3905 if ((flags & PROBE_TEST_DATA) && entry->data != data) 3906 continue; 3907 3908 /* do this last, since it is the most expensive */ 3909 if (func_g.search) { 3910 kallsyms_lookup(entry->ip, NULL, NULL, 3911 NULL, str); 3912 if (!ftrace_match(str, &func_g)) 3913 continue; 3914 } 3915 3916 rec_entry = ftrace_lookup_ip(hash, entry->ip); 3917 /* It is possible more than one entry had this ip */ 3918 if (rec_entry) 3919 free_hash_entry(hash, rec_entry); 3920 3921 hlist_del_rcu(&entry->node); 3922 list_add(&entry->free_list, &free_list); 3923 } 3924 } 3925 mutex_lock(&ftrace_lock); 3926 __disable_ftrace_function_probe(); 3927 /* 3928 * Remove after the disable is called. Otherwise, if the last 3929 * probe is removed, a null hash means *all enabled*. 3930 */ 3931 ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash); 3932 synchronize_sched(); 3933 if (!ret) 3934 free_ftrace_hash_rcu(old_hash); 3935 3936 list_for_each_entry_safe(entry, p, &free_list, free_list) { 3937 list_del(&entry->free_list); 3938 ftrace_free_entry(entry); 3939 } 3940 mutex_unlock(&ftrace_lock); 3941 3942 out_unlock: 3943 mutex_unlock(&trace_probe_ops.func_hash->regex_lock); 3944 free_ftrace_hash(hash); 3945 } 3946 3947 void 3948 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 3949 void *data) 3950 { 3951 __unregister_ftrace_function_probe(glob, ops, data, 3952 PROBE_TEST_FUNC | PROBE_TEST_DATA); 3953 } 3954 3955 void 3956 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) 3957 { 3958 __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC); 3959 } 3960 3961 void unregister_ftrace_function_probe_all(char *glob) 3962 { 3963 __unregister_ftrace_function_probe(glob, NULL, NULL, 0); 3964 } 3965 3966 static LIST_HEAD(ftrace_commands); 3967 static DEFINE_MUTEX(ftrace_cmd_mutex); 3968 3969 /* 3970 * Currently we only register ftrace commands from __init, so mark this 3971 * __init too. 3972 */ 3973 __init int register_ftrace_command(struct ftrace_func_command *cmd) 3974 { 3975 struct ftrace_func_command *p; 3976 int ret = 0; 3977 3978 mutex_lock(&ftrace_cmd_mutex); 3979 list_for_each_entry(p, &ftrace_commands, list) { 3980 if (strcmp(cmd->name, p->name) == 0) { 3981 ret = -EBUSY; 3982 goto out_unlock; 3983 } 3984 } 3985 list_add(&cmd->list, &ftrace_commands); 3986 out_unlock: 3987 mutex_unlock(&ftrace_cmd_mutex); 3988 3989 return ret; 3990 } 3991 3992 /* 3993 * Currently we only unregister ftrace commands from __init, so mark 3994 * this __init too. 3995 */ 3996 __init int unregister_ftrace_command(struct ftrace_func_command *cmd) 3997 { 3998 struct ftrace_func_command *p, *n; 3999 int ret = -ENODEV; 4000 4001 mutex_lock(&ftrace_cmd_mutex); 4002 list_for_each_entry_safe(p, n, &ftrace_commands, list) { 4003 if (strcmp(cmd->name, p->name) == 0) { 4004 ret = 0; 4005 list_del_init(&p->list); 4006 goto out_unlock; 4007 } 4008 } 4009 out_unlock: 4010 mutex_unlock(&ftrace_cmd_mutex); 4011 4012 return ret; 4013 } 4014 4015 static int ftrace_process_regex(struct ftrace_hash *hash, 4016 char *buff, int len, int enable) 4017 { 4018 char *func, *command, *next = buff; 4019 struct ftrace_func_command *p; 4020 int ret = -EINVAL; 4021 4022 func = strsep(&next, ":"); 4023 4024 if (!next) { 4025 ret = ftrace_match_records(hash, func, len); 4026 if (!ret) 4027 ret = -EINVAL; 4028 if (ret < 0) 4029 return ret; 4030 return 0; 4031 } 4032 4033 /* command found */ 4034 4035 command = strsep(&next, ":"); 4036 4037 mutex_lock(&ftrace_cmd_mutex); 4038 list_for_each_entry(p, &ftrace_commands, list) { 4039 if (strcmp(p->name, command) == 0) { 4040 ret = p->func(hash, func, command, next, enable); 4041 goto out_unlock; 4042 } 4043 } 4044 out_unlock: 4045 mutex_unlock(&ftrace_cmd_mutex); 4046 4047 return ret; 4048 } 4049 4050 static ssize_t 4051 ftrace_regex_write(struct file *file, const char __user *ubuf, 4052 size_t cnt, loff_t *ppos, int enable) 4053 { 4054 struct ftrace_iterator *iter; 4055 struct trace_parser *parser; 4056 ssize_t ret, read; 4057 4058 if (!cnt) 4059 return 0; 4060 4061 if (file->f_mode & FMODE_READ) { 4062 struct seq_file *m = file->private_data; 4063 iter = m->private; 4064 } else 4065 iter = file->private_data; 4066 4067 if (unlikely(ftrace_disabled)) 4068 return -ENODEV; 4069 4070 /* iter->hash is a local copy, so we don't need regex_lock */ 4071 4072 parser = &iter->parser; 4073 read = trace_get_user(parser, ubuf, cnt, ppos); 4074 4075 if (read >= 0 && trace_parser_loaded(parser) && 4076 !trace_parser_cont(parser)) { 4077 ret = ftrace_process_regex(iter->hash, parser->buffer, 4078 parser->idx, enable); 4079 trace_parser_clear(parser); 4080 if (ret < 0) 4081 goto out; 4082 } 4083 4084 ret = read; 4085 out: 4086 return ret; 4087 } 4088 4089 ssize_t 4090 ftrace_filter_write(struct file *file, const char __user *ubuf, 4091 size_t cnt, loff_t *ppos) 4092 { 4093 return ftrace_regex_write(file, ubuf, cnt, ppos, 1); 4094 } 4095 4096 ssize_t 4097 ftrace_notrace_write(struct file *file, const char __user *ubuf, 4098 size_t cnt, loff_t *ppos) 4099 { 4100 return ftrace_regex_write(file, ubuf, cnt, ppos, 0); 4101 } 4102 4103 static int 4104 ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove) 4105 { 4106 struct ftrace_func_entry *entry; 4107 4108 if (!ftrace_location(ip)) 4109 return -EINVAL; 4110 4111 if (remove) { 4112 entry = ftrace_lookup_ip(hash, ip); 4113 if (!entry) 4114 return -ENOENT; 4115 free_hash_entry(hash, entry); 4116 return 0; 4117 } 4118 4119 return add_hash_entry(hash, ip); 4120 } 4121 4122 static void ftrace_ops_update_code(struct ftrace_ops *ops, 4123 struct ftrace_ops_hash *old_hash) 4124 { 4125 struct ftrace_ops *op; 4126 4127 if (!ftrace_enabled) 4128 return; 4129 4130 if (ops->flags & FTRACE_OPS_FL_ENABLED) { 4131 ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash); 4132 return; 4133 } 4134 4135 /* 4136 * If this is the shared global_ops filter, then we need to 4137 * check if there is another ops that shares it, is enabled. 4138 * If so, we still need to run the modify code. 4139 */ 4140 if (ops->func_hash != &global_ops.local_hash) 4141 return; 4142 4143 do_for_each_ftrace_op(op, ftrace_ops_list) { 4144 if (op->func_hash == &global_ops.local_hash && 4145 op->flags & FTRACE_OPS_FL_ENABLED) { 4146 ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash); 4147 /* Only need to do this once */ 4148 return; 4149 } 4150 } while_for_each_ftrace_op(op); 4151 } 4152 4153 static int 4154 ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len, 4155 unsigned long ip, int remove, int reset, int enable) 4156 { 4157 struct ftrace_hash **orig_hash; 4158 struct ftrace_ops_hash old_hash_ops; 4159 struct ftrace_hash *old_hash; 4160 struct ftrace_hash *hash; 4161 int ret; 4162 4163 if (unlikely(ftrace_disabled)) 4164 return -ENODEV; 4165 4166 mutex_lock(&ops->func_hash->regex_lock); 4167 4168 if (enable) 4169 orig_hash = &ops->func_hash->filter_hash; 4170 else 4171 orig_hash = &ops->func_hash->notrace_hash; 4172 4173 if (reset) 4174 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); 4175 else 4176 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash); 4177 4178 if (!hash) { 4179 ret = -ENOMEM; 4180 goto out_regex_unlock; 4181 } 4182 4183 if (buf && !ftrace_match_records(hash, buf, len)) { 4184 ret = -EINVAL; 4185 goto out_regex_unlock; 4186 } 4187 if (ip) { 4188 ret = ftrace_match_addr(hash, ip, remove); 4189 if (ret < 0) 4190 goto out_regex_unlock; 4191 } 4192 4193 mutex_lock(&ftrace_lock); 4194 old_hash = *orig_hash; 4195 old_hash_ops.filter_hash = ops->func_hash->filter_hash; 4196 old_hash_ops.notrace_hash = ops->func_hash->notrace_hash; 4197 ret = ftrace_hash_move(ops, enable, orig_hash, hash); 4198 if (!ret) { 4199 ftrace_ops_update_code(ops, &old_hash_ops); 4200 free_ftrace_hash_rcu(old_hash); 4201 } 4202 mutex_unlock(&ftrace_lock); 4203 4204 out_regex_unlock: 4205 mutex_unlock(&ops->func_hash->regex_lock); 4206 4207 free_ftrace_hash(hash); 4208 return ret; 4209 } 4210 4211 static int 4212 ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove, 4213 int reset, int enable) 4214 { 4215 return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable); 4216 } 4217 4218 /** 4219 * ftrace_set_filter_ip - set a function to filter on in ftrace by address 4220 * @ops - the ops to set the filter with 4221 * @ip - the address to add to or remove from the filter. 4222 * @remove - non zero to remove the ip from the filter 4223 * @reset - non zero to reset all filters before applying this filter. 4224 * 4225 * Filters denote which functions should be enabled when tracing is enabled 4226 * If @ip is NULL, it failes to update filter. 4227 */ 4228 int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip, 4229 int remove, int reset) 4230 { 4231 ftrace_ops_init(ops); 4232 return ftrace_set_addr(ops, ip, remove, reset, 1); 4233 } 4234 EXPORT_SYMBOL_GPL(ftrace_set_filter_ip); 4235 4236 static int 4237 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len, 4238 int reset, int enable) 4239 { 4240 return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable); 4241 } 4242 4243 /** 4244 * ftrace_set_filter - set a function to filter on in ftrace 4245 * @ops - the ops to set the filter with 4246 * @buf - the string that holds the function filter text. 4247 * @len - the length of the string. 4248 * @reset - non zero to reset all filters before applying this filter. 4249 * 4250 * Filters denote which functions should be enabled when tracing is enabled. 4251 * If @buf is NULL and reset is set, all functions will be enabled for tracing. 4252 */ 4253 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf, 4254 int len, int reset) 4255 { 4256 ftrace_ops_init(ops); 4257 return ftrace_set_regex(ops, buf, len, reset, 1); 4258 } 4259 EXPORT_SYMBOL_GPL(ftrace_set_filter); 4260 4261 /** 4262 * ftrace_set_notrace - set a function to not trace in ftrace 4263 * @ops - the ops to set the notrace filter with 4264 * @buf - the string that holds the function notrace text. 4265 * @len - the length of the string. 4266 * @reset - non zero to reset all filters before applying this filter. 4267 * 4268 * Notrace Filters denote which functions should not be enabled when tracing 4269 * is enabled. If @buf is NULL and reset is set, all functions will be enabled 4270 * for tracing. 4271 */ 4272 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf, 4273 int len, int reset) 4274 { 4275 ftrace_ops_init(ops); 4276 return ftrace_set_regex(ops, buf, len, reset, 0); 4277 } 4278 EXPORT_SYMBOL_GPL(ftrace_set_notrace); 4279 /** 4280 * ftrace_set_global_filter - set a function to filter on with global tracers 4281 * @buf - the string that holds the function filter text. 4282 * @len - the length of the string. 4283 * @reset - non zero to reset all filters before applying this filter. 4284 * 4285 * Filters denote which functions should be enabled when tracing is enabled. 4286 * If @buf is NULL and reset is set, all functions will be enabled for tracing. 4287 */ 4288 void ftrace_set_global_filter(unsigned char *buf, int len, int reset) 4289 { 4290 ftrace_set_regex(&global_ops, buf, len, reset, 1); 4291 } 4292 EXPORT_SYMBOL_GPL(ftrace_set_global_filter); 4293 4294 /** 4295 * ftrace_set_global_notrace - set a function to not trace with global tracers 4296 * @buf - the string that holds the function notrace text. 4297 * @len - the length of the string. 4298 * @reset - non zero to reset all filters before applying this filter. 4299 * 4300 * Notrace Filters denote which functions should not be enabled when tracing 4301 * is enabled. If @buf is NULL and reset is set, all functions will be enabled 4302 * for tracing. 4303 */ 4304 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset) 4305 { 4306 ftrace_set_regex(&global_ops, buf, len, reset, 0); 4307 } 4308 EXPORT_SYMBOL_GPL(ftrace_set_global_notrace); 4309 4310 /* 4311 * command line interface to allow users to set filters on boot up. 4312 */ 4313 #define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE 4314 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata; 4315 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata; 4316 4317 /* Used by function selftest to not test if filter is set */ 4318 bool ftrace_filter_param __initdata; 4319 4320 static int __init set_ftrace_notrace(char *str) 4321 { 4322 ftrace_filter_param = true; 4323 strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE); 4324 return 1; 4325 } 4326 __setup("ftrace_notrace=", set_ftrace_notrace); 4327 4328 static int __init set_ftrace_filter(char *str) 4329 { 4330 ftrace_filter_param = true; 4331 strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE); 4332 return 1; 4333 } 4334 __setup("ftrace_filter=", set_ftrace_filter); 4335 4336 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 4337 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; 4338 static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata; 4339 static int ftrace_set_func(unsigned long *array, int *idx, int size, char *buffer); 4340 4341 static unsigned long save_global_trampoline; 4342 static unsigned long save_global_flags; 4343 4344 static int __init set_graph_function(char *str) 4345 { 4346 strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE); 4347 return 1; 4348 } 4349 __setup("ftrace_graph_filter=", set_graph_function); 4350 4351 static int __init set_graph_notrace_function(char *str) 4352 { 4353 strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE); 4354 return 1; 4355 } 4356 __setup("ftrace_graph_notrace=", set_graph_notrace_function); 4357 4358 static void __init set_ftrace_early_graph(char *buf, int enable) 4359 { 4360 int ret; 4361 char *func; 4362 unsigned long *table = ftrace_graph_funcs; 4363 int *count = &ftrace_graph_count; 4364 4365 if (!enable) { 4366 table = ftrace_graph_notrace_funcs; 4367 count = &ftrace_graph_notrace_count; 4368 } 4369 4370 while (buf) { 4371 func = strsep(&buf, ","); 4372 /* we allow only one expression at a time */ 4373 ret = ftrace_set_func(table, count, FTRACE_GRAPH_MAX_FUNCS, func); 4374 if (ret) 4375 printk(KERN_DEBUG "ftrace: function %s not " 4376 "traceable\n", func); 4377 } 4378 } 4379 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 4380 4381 void __init 4382 ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable) 4383 { 4384 char *func; 4385 4386 ftrace_ops_init(ops); 4387 4388 while (buf) { 4389 func = strsep(&buf, ","); 4390 ftrace_set_regex(ops, func, strlen(func), 0, enable); 4391 } 4392 } 4393 4394 static void __init set_ftrace_early_filters(void) 4395 { 4396 if (ftrace_filter_buf[0]) 4397 ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1); 4398 if (ftrace_notrace_buf[0]) 4399 ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0); 4400 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 4401 if (ftrace_graph_buf[0]) 4402 set_ftrace_early_graph(ftrace_graph_buf, 1); 4403 if (ftrace_graph_notrace_buf[0]) 4404 set_ftrace_early_graph(ftrace_graph_notrace_buf, 0); 4405 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 4406 } 4407 4408 int ftrace_regex_release(struct inode *inode, struct file *file) 4409 { 4410 struct seq_file *m = (struct seq_file *)file->private_data; 4411 struct ftrace_ops_hash old_hash_ops; 4412 struct ftrace_iterator *iter; 4413 struct ftrace_hash **orig_hash; 4414 struct ftrace_hash *old_hash; 4415 struct trace_parser *parser; 4416 int filter_hash; 4417 int ret; 4418 4419 if (file->f_mode & FMODE_READ) { 4420 iter = m->private; 4421 seq_release(inode, file); 4422 } else 4423 iter = file->private_data; 4424 4425 parser = &iter->parser; 4426 if (trace_parser_loaded(parser)) { 4427 parser->buffer[parser->idx] = 0; 4428 ftrace_match_records(iter->hash, parser->buffer, parser->idx); 4429 } 4430 4431 trace_parser_put(parser); 4432 4433 mutex_lock(&iter->ops->func_hash->regex_lock); 4434 4435 if (file->f_mode & FMODE_WRITE) { 4436 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER); 4437 4438 if (filter_hash) 4439 orig_hash = &iter->ops->func_hash->filter_hash; 4440 else 4441 orig_hash = &iter->ops->func_hash->notrace_hash; 4442 4443 mutex_lock(&ftrace_lock); 4444 old_hash = *orig_hash; 4445 old_hash_ops.filter_hash = iter->ops->func_hash->filter_hash; 4446 old_hash_ops.notrace_hash = iter->ops->func_hash->notrace_hash; 4447 ret = ftrace_hash_move(iter->ops, filter_hash, 4448 orig_hash, iter->hash); 4449 if (!ret) { 4450 ftrace_ops_update_code(iter->ops, &old_hash_ops); 4451 free_ftrace_hash_rcu(old_hash); 4452 } 4453 mutex_unlock(&ftrace_lock); 4454 } 4455 4456 mutex_unlock(&iter->ops->func_hash->regex_lock); 4457 free_ftrace_hash(iter->hash); 4458 kfree(iter); 4459 4460 return 0; 4461 } 4462 4463 static const struct file_operations ftrace_avail_fops = { 4464 .open = ftrace_avail_open, 4465 .read = seq_read, 4466 .llseek = seq_lseek, 4467 .release = seq_release_private, 4468 }; 4469 4470 static const struct file_operations ftrace_enabled_fops = { 4471 .open = ftrace_enabled_open, 4472 .read = seq_read, 4473 .llseek = seq_lseek, 4474 .release = seq_release_private, 4475 }; 4476 4477 static const struct file_operations ftrace_filter_fops = { 4478 .open = ftrace_filter_open, 4479 .read = seq_read, 4480 .write = ftrace_filter_write, 4481 .llseek = tracing_lseek, 4482 .release = ftrace_regex_release, 4483 }; 4484 4485 static const struct file_operations ftrace_notrace_fops = { 4486 .open = ftrace_notrace_open, 4487 .read = seq_read, 4488 .write = ftrace_notrace_write, 4489 .llseek = tracing_lseek, 4490 .release = ftrace_regex_release, 4491 }; 4492 4493 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 4494 4495 static DEFINE_MUTEX(graph_lock); 4496 4497 int ftrace_graph_count; 4498 int ftrace_graph_notrace_count; 4499 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly; 4500 unsigned long ftrace_graph_notrace_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly; 4501 4502 struct ftrace_graph_data { 4503 unsigned long *table; 4504 size_t size; 4505 int *count; 4506 const struct seq_operations *seq_ops; 4507 }; 4508 4509 static void * 4510 __g_next(struct seq_file *m, loff_t *pos) 4511 { 4512 struct ftrace_graph_data *fgd = m->private; 4513 4514 if (*pos >= *fgd->count) 4515 return NULL; 4516 return &fgd->table[*pos]; 4517 } 4518 4519 static void * 4520 g_next(struct seq_file *m, void *v, loff_t *pos) 4521 { 4522 (*pos)++; 4523 return __g_next(m, pos); 4524 } 4525 4526 static void *g_start(struct seq_file *m, loff_t *pos) 4527 { 4528 struct ftrace_graph_data *fgd = m->private; 4529 4530 mutex_lock(&graph_lock); 4531 4532 /* Nothing, tell g_show to print all functions are enabled */ 4533 if (!*fgd->count && !*pos) 4534 return (void *)1; 4535 4536 return __g_next(m, pos); 4537 } 4538 4539 static void g_stop(struct seq_file *m, void *p) 4540 { 4541 mutex_unlock(&graph_lock); 4542 } 4543 4544 static int g_show(struct seq_file *m, void *v) 4545 { 4546 unsigned long *ptr = v; 4547 4548 if (!ptr) 4549 return 0; 4550 4551 if (ptr == (unsigned long *)1) { 4552 struct ftrace_graph_data *fgd = m->private; 4553 4554 if (fgd->table == ftrace_graph_funcs) 4555 seq_puts(m, "#### all functions enabled ####\n"); 4556 else 4557 seq_puts(m, "#### no functions disabled ####\n"); 4558 return 0; 4559 } 4560 4561 seq_printf(m, "%ps\n", (void *)*ptr); 4562 4563 return 0; 4564 } 4565 4566 static const struct seq_operations ftrace_graph_seq_ops = { 4567 .start = g_start, 4568 .next = g_next, 4569 .stop = g_stop, 4570 .show = g_show, 4571 }; 4572 4573 static int 4574 __ftrace_graph_open(struct inode *inode, struct file *file, 4575 struct ftrace_graph_data *fgd) 4576 { 4577 int ret = 0; 4578 4579 mutex_lock(&graph_lock); 4580 if ((file->f_mode & FMODE_WRITE) && 4581 (file->f_flags & O_TRUNC)) { 4582 *fgd->count = 0; 4583 memset(fgd->table, 0, fgd->size * sizeof(*fgd->table)); 4584 } 4585 mutex_unlock(&graph_lock); 4586 4587 if (file->f_mode & FMODE_READ) { 4588 ret = seq_open(file, fgd->seq_ops); 4589 if (!ret) { 4590 struct seq_file *m = file->private_data; 4591 m->private = fgd; 4592 } 4593 } else 4594 file->private_data = fgd; 4595 4596 return ret; 4597 } 4598 4599 static int 4600 ftrace_graph_open(struct inode *inode, struct file *file) 4601 { 4602 struct ftrace_graph_data *fgd; 4603 4604 if (unlikely(ftrace_disabled)) 4605 return -ENODEV; 4606 4607 fgd = kmalloc(sizeof(*fgd), GFP_KERNEL); 4608 if (fgd == NULL) 4609 return -ENOMEM; 4610 4611 fgd->table = ftrace_graph_funcs; 4612 fgd->size = FTRACE_GRAPH_MAX_FUNCS; 4613 fgd->count = &ftrace_graph_count; 4614 fgd->seq_ops = &ftrace_graph_seq_ops; 4615 4616 return __ftrace_graph_open(inode, file, fgd); 4617 } 4618 4619 static int 4620 ftrace_graph_notrace_open(struct inode *inode, struct file *file) 4621 { 4622 struct ftrace_graph_data *fgd; 4623 4624 if (unlikely(ftrace_disabled)) 4625 return -ENODEV; 4626 4627 fgd = kmalloc(sizeof(*fgd), GFP_KERNEL); 4628 if (fgd == NULL) 4629 return -ENOMEM; 4630 4631 fgd->table = ftrace_graph_notrace_funcs; 4632 fgd->size = FTRACE_GRAPH_MAX_FUNCS; 4633 fgd->count = &ftrace_graph_notrace_count; 4634 fgd->seq_ops = &ftrace_graph_seq_ops; 4635 4636 return __ftrace_graph_open(inode, file, fgd); 4637 } 4638 4639 static int 4640 ftrace_graph_release(struct inode *inode, struct file *file) 4641 { 4642 if (file->f_mode & FMODE_READ) { 4643 struct seq_file *m = file->private_data; 4644 4645 kfree(m->private); 4646 seq_release(inode, file); 4647 } else { 4648 kfree(file->private_data); 4649 } 4650 4651 return 0; 4652 } 4653 4654 static int 4655 ftrace_set_func(unsigned long *array, int *idx, int size, char *buffer) 4656 { 4657 struct ftrace_glob func_g; 4658 struct dyn_ftrace *rec; 4659 struct ftrace_page *pg; 4660 int fail = 1; 4661 int not; 4662 bool exists; 4663 int i; 4664 4665 /* decode regex */ 4666 func_g.type = filter_parse_regex(buffer, strlen(buffer), 4667 &func_g.search, ¬); 4668 if (!not && *idx >= size) 4669 return -EBUSY; 4670 4671 func_g.len = strlen(func_g.search); 4672 4673 mutex_lock(&ftrace_lock); 4674 4675 if (unlikely(ftrace_disabled)) { 4676 mutex_unlock(&ftrace_lock); 4677 return -ENODEV; 4678 } 4679 4680 do_for_each_ftrace_rec(pg, rec) { 4681 4682 if (ftrace_match_record(rec, &func_g, NULL, 0)) { 4683 /* if it is in the array */ 4684 exists = false; 4685 for (i = 0; i < *idx; i++) { 4686 if (array[i] == rec->ip) { 4687 exists = true; 4688 break; 4689 } 4690 } 4691 4692 if (!not) { 4693 fail = 0; 4694 if (!exists) { 4695 array[(*idx)++] = rec->ip; 4696 if (*idx >= size) 4697 goto out; 4698 } 4699 } else { 4700 if (exists) { 4701 array[i] = array[--(*idx)]; 4702 array[*idx] = 0; 4703 fail = 0; 4704 } 4705 } 4706 } 4707 } while_for_each_ftrace_rec(); 4708 out: 4709 mutex_unlock(&ftrace_lock); 4710 4711 if (fail) 4712 return -EINVAL; 4713 4714 return 0; 4715 } 4716 4717 static ssize_t 4718 ftrace_graph_write(struct file *file, const char __user *ubuf, 4719 size_t cnt, loff_t *ppos) 4720 { 4721 struct trace_parser parser; 4722 ssize_t read, ret = 0; 4723 struct ftrace_graph_data *fgd = file->private_data; 4724 4725 if (!cnt) 4726 return 0; 4727 4728 if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) 4729 return -ENOMEM; 4730 4731 read = trace_get_user(&parser, ubuf, cnt, ppos); 4732 4733 if (read >= 0 && trace_parser_loaded((&parser))) { 4734 parser.buffer[parser.idx] = 0; 4735 4736 mutex_lock(&graph_lock); 4737 4738 /* we allow only one expression at a time */ 4739 ret = ftrace_set_func(fgd->table, fgd->count, fgd->size, 4740 parser.buffer); 4741 4742 mutex_unlock(&graph_lock); 4743 } 4744 4745 if (!ret) 4746 ret = read; 4747 4748 trace_parser_put(&parser); 4749 4750 return ret; 4751 } 4752 4753 static const struct file_operations ftrace_graph_fops = { 4754 .open = ftrace_graph_open, 4755 .read = seq_read, 4756 .write = ftrace_graph_write, 4757 .llseek = tracing_lseek, 4758 .release = ftrace_graph_release, 4759 }; 4760 4761 static const struct file_operations ftrace_graph_notrace_fops = { 4762 .open = ftrace_graph_notrace_open, 4763 .read = seq_read, 4764 .write = ftrace_graph_write, 4765 .llseek = tracing_lseek, 4766 .release = ftrace_graph_release, 4767 }; 4768 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 4769 4770 void ftrace_create_filter_files(struct ftrace_ops *ops, 4771 struct dentry *parent) 4772 { 4773 4774 trace_create_file("set_ftrace_filter", 0644, parent, 4775 ops, &ftrace_filter_fops); 4776 4777 trace_create_file("set_ftrace_notrace", 0644, parent, 4778 ops, &ftrace_notrace_fops); 4779 } 4780 4781 /* 4782 * The name "destroy_filter_files" is really a misnomer. Although 4783 * in the future, it may actualy delete the files, but this is 4784 * really intended to make sure the ops passed in are disabled 4785 * and that when this function returns, the caller is free to 4786 * free the ops. 4787 * 4788 * The "destroy" name is only to match the "create" name that this 4789 * should be paired with. 4790 */ 4791 void ftrace_destroy_filter_files(struct ftrace_ops *ops) 4792 { 4793 mutex_lock(&ftrace_lock); 4794 if (ops->flags & FTRACE_OPS_FL_ENABLED) 4795 ftrace_shutdown(ops, 0); 4796 ops->flags |= FTRACE_OPS_FL_DELETED; 4797 mutex_unlock(&ftrace_lock); 4798 } 4799 4800 static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer) 4801 { 4802 4803 trace_create_file("available_filter_functions", 0444, 4804 d_tracer, NULL, &ftrace_avail_fops); 4805 4806 trace_create_file("enabled_functions", 0444, 4807 d_tracer, NULL, &ftrace_enabled_fops); 4808 4809 ftrace_create_filter_files(&global_ops, d_tracer); 4810 4811 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 4812 trace_create_file("set_graph_function", 0444, d_tracer, 4813 NULL, 4814 &ftrace_graph_fops); 4815 trace_create_file("set_graph_notrace", 0444, d_tracer, 4816 NULL, 4817 &ftrace_graph_notrace_fops); 4818 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 4819 4820 return 0; 4821 } 4822 4823 static int ftrace_cmp_ips(const void *a, const void *b) 4824 { 4825 const unsigned long *ipa = a; 4826 const unsigned long *ipb = b; 4827 4828 if (*ipa > *ipb) 4829 return 1; 4830 if (*ipa < *ipb) 4831 return -1; 4832 return 0; 4833 } 4834 4835 static int ftrace_process_locs(struct module *mod, 4836 unsigned long *start, 4837 unsigned long *end) 4838 { 4839 struct ftrace_page *start_pg; 4840 struct ftrace_page *pg; 4841 struct dyn_ftrace *rec; 4842 unsigned long count; 4843 unsigned long *p; 4844 unsigned long addr; 4845 unsigned long flags = 0; /* Shut up gcc */ 4846 int ret = -ENOMEM; 4847 4848 count = end - start; 4849 4850 if (!count) 4851 return 0; 4852 4853 sort(start, count, sizeof(*start), 4854 ftrace_cmp_ips, NULL); 4855 4856 start_pg = ftrace_allocate_pages(count); 4857 if (!start_pg) 4858 return -ENOMEM; 4859 4860 mutex_lock(&ftrace_lock); 4861 4862 /* 4863 * Core and each module needs their own pages, as 4864 * modules will free them when they are removed. 4865 * Force a new page to be allocated for modules. 4866 */ 4867 if (!mod) { 4868 WARN_ON(ftrace_pages || ftrace_pages_start); 4869 /* First initialization */ 4870 ftrace_pages = ftrace_pages_start = start_pg; 4871 } else { 4872 if (!ftrace_pages) 4873 goto out; 4874 4875 if (WARN_ON(ftrace_pages->next)) { 4876 /* Hmm, we have free pages? */ 4877 while (ftrace_pages->next) 4878 ftrace_pages = ftrace_pages->next; 4879 } 4880 4881 ftrace_pages->next = start_pg; 4882 } 4883 4884 p = start; 4885 pg = start_pg; 4886 while (p < end) { 4887 addr = ftrace_call_adjust(*p++); 4888 /* 4889 * Some architecture linkers will pad between 4890 * the different mcount_loc sections of different 4891 * object files to satisfy alignments. 4892 * Skip any NULL pointers. 4893 */ 4894 if (!addr) 4895 continue; 4896 4897 if (pg->index == pg->size) { 4898 /* We should have allocated enough */ 4899 if (WARN_ON(!pg->next)) 4900 break; 4901 pg = pg->next; 4902 } 4903 4904 rec = &pg->records[pg->index++]; 4905 rec->ip = addr; 4906 } 4907 4908 /* We should have used all pages */ 4909 WARN_ON(pg->next); 4910 4911 /* Assign the last page to ftrace_pages */ 4912 ftrace_pages = pg; 4913 4914 /* 4915 * We only need to disable interrupts on start up 4916 * because we are modifying code that an interrupt 4917 * may execute, and the modification is not atomic. 4918 * But for modules, nothing runs the code we modify 4919 * until we are finished with it, and there's no 4920 * reason to cause large interrupt latencies while we do it. 4921 */ 4922 if (!mod) 4923 local_irq_save(flags); 4924 ftrace_update_code(mod, start_pg); 4925 if (!mod) 4926 local_irq_restore(flags); 4927 ret = 0; 4928 out: 4929 mutex_unlock(&ftrace_lock); 4930 4931 return ret; 4932 } 4933 4934 #ifdef CONFIG_MODULES 4935 4936 #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next) 4937 4938 static int referenced_filters(struct dyn_ftrace *rec) 4939 { 4940 struct ftrace_ops *ops; 4941 int cnt = 0; 4942 4943 for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) { 4944 if (ops_references_rec(ops, rec)) 4945 cnt++; 4946 } 4947 4948 return cnt; 4949 } 4950 4951 void ftrace_release_mod(struct module *mod) 4952 { 4953 struct dyn_ftrace *rec; 4954 struct ftrace_page **last_pg; 4955 struct ftrace_page *pg; 4956 int order; 4957 4958 mutex_lock(&ftrace_lock); 4959 4960 if (ftrace_disabled) 4961 goto out_unlock; 4962 4963 /* 4964 * Each module has its own ftrace_pages, remove 4965 * them from the list. 4966 */ 4967 last_pg = &ftrace_pages_start; 4968 for (pg = ftrace_pages_start; pg; pg = *last_pg) { 4969 rec = &pg->records[0]; 4970 if (within_module_core(rec->ip, mod)) { 4971 /* 4972 * As core pages are first, the first 4973 * page should never be a module page. 4974 */ 4975 if (WARN_ON(pg == ftrace_pages_start)) 4976 goto out_unlock; 4977 4978 /* Check if we are deleting the last page */ 4979 if (pg == ftrace_pages) 4980 ftrace_pages = next_to_ftrace_page(last_pg); 4981 4982 *last_pg = pg->next; 4983 order = get_count_order(pg->size / ENTRIES_PER_PAGE); 4984 free_pages((unsigned long)pg->records, order); 4985 kfree(pg); 4986 } else 4987 last_pg = &pg->next; 4988 } 4989 out_unlock: 4990 mutex_unlock(&ftrace_lock); 4991 } 4992 4993 void ftrace_module_enable(struct module *mod) 4994 { 4995 struct dyn_ftrace *rec; 4996 struct ftrace_page *pg; 4997 4998 mutex_lock(&ftrace_lock); 4999 5000 if (ftrace_disabled) 5001 goto out_unlock; 5002 5003 /* 5004 * If the tracing is enabled, go ahead and enable the record. 5005 * 5006 * The reason not to enable the record immediatelly is the 5007 * inherent check of ftrace_make_nop/ftrace_make_call for 5008 * correct previous instructions. Making first the NOP 5009 * conversion puts the module to the correct state, thus 5010 * passing the ftrace_make_call check. 5011 * 5012 * We also delay this to after the module code already set the 5013 * text to read-only, as we now need to set it back to read-write 5014 * so that we can modify the text. 5015 */ 5016 if (ftrace_start_up) 5017 ftrace_arch_code_modify_prepare(); 5018 5019 do_for_each_ftrace_rec(pg, rec) { 5020 int cnt; 5021 /* 5022 * do_for_each_ftrace_rec() is a double loop. 5023 * module text shares the pg. If a record is 5024 * not part of this module, then skip this pg, 5025 * which the "break" will do. 5026 */ 5027 if (!within_module_core(rec->ip, mod)) 5028 break; 5029 5030 cnt = 0; 5031 5032 /* 5033 * When adding a module, we need to check if tracers are 5034 * currently enabled and if they are, and can trace this record, 5035 * we need to enable the module functions as well as update the 5036 * reference counts for those function records. 5037 */ 5038 if (ftrace_start_up) 5039 cnt += referenced_filters(rec); 5040 5041 /* This clears FTRACE_FL_DISABLED */ 5042 rec->flags = cnt; 5043 5044 if (ftrace_start_up && cnt) { 5045 int failed = __ftrace_replace_code(rec, 1); 5046 if (failed) { 5047 ftrace_bug(failed, rec); 5048 goto out_loop; 5049 } 5050 } 5051 5052 } while_for_each_ftrace_rec(); 5053 5054 out_loop: 5055 if (ftrace_start_up) 5056 ftrace_arch_code_modify_post_process(); 5057 5058 out_unlock: 5059 mutex_unlock(&ftrace_lock); 5060 } 5061 5062 void ftrace_module_init(struct module *mod) 5063 { 5064 if (ftrace_disabled || !mod->num_ftrace_callsites) 5065 return; 5066 5067 ftrace_process_locs(mod, mod->ftrace_callsites, 5068 mod->ftrace_callsites + mod->num_ftrace_callsites); 5069 } 5070 #endif /* CONFIG_MODULES */ 5071 5072 void __init ftrace_init(void) 5073 { 5074 extern unsigned long __start_mcount_loc[]; 5075 extern unsigned long __stop_mcount_loc[]; 5076 unsigned long count, flags; 5077 int ret; 5078 5079 local_irq_save(flags); 5080 ret = ftrace_dyn_arch_init(); 5081 local_irq_restore(flags); 5082 if (ret) 5083 goto failed; 5084 5085 count = __stop_mcount_loc - __start_mcount_loc; 5086 if (!count) { 5087 pr_info("ftrace: No functions to be traced?\n"); 5088 goto failed; 5089 } 5090 5091 pr_info("ftrace: allocating %ld entries in %ld pages\n", 5092 count, count / ENTRIES_PER_PAGE + 1); 5093 5094 last_ftrace_enabled = ftrace_enabled = 1; 5095 5096 ret = ftrace_process_locs(NULL, 5097 __start_mcount_loc, 5098 __stop_mcount_loc); 5099 5100 set_ftrace_early_filters(); 5101 5102 return; 5103 failed: 5104 ftrace_disabled = 1; 5105 } 5106 5107 /* Do nothing if arch does not support this */ 5108 void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops) 5109 { 5110 } 5111 5112 static void ftrace_update_trampoline(struct ftrace_ops *ops) 5113 { 5114 5115 /* 5116 * Currently there's no safe way to free a trampoline when the kernel 5117 * is configured with PREEMPT. That is because a task could be preempted 5118 * when it jumped to the trampoline, it may be preempted for a long time 5119 * depending on the system load, and currently there's no way to know 5120 * when it will be off the trampoline. If the trampoline is freed 5121 * too early, when the task runs again, it will be executing on freed 5122 * memory and crash. 5123 */ 5124 #ifdef CONFIG_PREEMPT 5125 /* Currently, only non dynamic ops can have a trampoline */ 5126 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) 5127 return; 5128 #endif 5129 5130 arch_ftrace_update_trampoline(ops); 5131 } 5132 5133 #else 5134 5135 static struct ftrace_ops global_ops = { 5136 .func = ftrace_stub, 5137 .flags = FTRACE_OPS_FL_RECURSION_SAFE | 5138 FTRACE_OPS_FL_INITIALIZED | 5139 FTRACE_OPS_FL_PID, 5140 }; 5141 5142 static int __init ftrace_nodyn_init(void) 5143 { 5144 ftrace_enabled = 1; 5145 return 0; 5146 } 5147 core_initcall(ftrace_nodyn_init); 5148 5149 static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; } 5150 static inline void ftrace_startup_enable(int command) { } 5151 static inline void ftrace_startup_all(int command) { } 5152 /* Keep as macros so we do not need to define the commands */ 5153 # define ftrace_startup(ops, command) \ 5154 ({ \ 5155 int ___ret = __register_ftrace_function(ops); \ 5156 if (!___ret) \ 5157 (ops)->flags |= FTRACE_OPS_FL_ENABLED; \ 5158 ___ret; \ 5159 }) 5160 # define ftrace_shutdown(ops, command) \ 5161 ({ \ 5162 int ___ret = __unregister_ftrace_function(ops); \ 5163 if (!___ret) \ 5164 (ops)->flags &= ~FTRACE_OPS_FL_ENABLED; \ 5165 ___ret; \ 5166 }) 5167 5168 # define ftrace_startup_sysctl() do { } while (0) 5169 # define ftrace_shutdown_sysctl() do { } while (0) 5170 5171 static inline int 5172 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) 5173 { 5174 return 1; 5175 } 5176 5177 static void ftrace_update_trampoline(struct ftrace_ops *ops) 5178 { 5179 } 5180 5181 #endif /* CONFIG_DYNAMIC_FTRACE */ 5182 5183 __init void ftrace_init_global_array_ops(struct trace_array *tr) 5184 { 5185 tr->ops = &global_ops; 5186 tr->ops->private = tr; 5187 } 5188 5189 void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func) 5190 { 5191 /* If we filter on pids, update to use the pid function */ 5192 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) { 5193 if (WARN_ON(tr->ops->func != ftrace_stub)) 5194 printk("ftrace ops had %pS for function\n", 5195 tr->ops->func); 5196 } 5197 tr->ops->func = func; 5198 tr->ops->private = tr; 5199 } 5200 5201 void ftrace_reset_array_ops(struct trace_array *tr) 5202 { 5203 tr->ops->func = ftrace_stub; 5204 } 5205 5206 static inline void 5207 __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 5208 struct ftrace_ops *ignored, struct pt_regs *regs) 5209 { 5210 struct ftrace_ops *op; 5211 int bit; 5212 5213 bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX); 5214 if (bit < 0) 5215 return; 5216 5217 /* 5218 * Some of the ops may be dynamically allocated, 5219 * they must be freed after a synchronize_sched(). 5220 */ 5221 preempt_disable_notrace(); 5222 5223 do_for_each_ftrace_op(op, ftrace_ops_list) { 5224 /* 5225 * Check the following for each ops before calling their func: 5226 * if RCU flag is set, then rcu_is_watching() must be true 5227 * if PER_CPU is set, then ftrace_function_local_disable() 5228 * must be false 5229 * Otherwise test if the ip matches the ops filter 5230 * 5231 * If any of the above fails then the op->func() is not executed. 5232 */ 5233 if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) && 5234 (!(op->flags & FTRACE_OPS_FL_PER_CPU) || 5235 !ftrace_function_local_disabled(op)) && 5236 ftrace_ops_test(op, ip, regs)) { 5237 5238 if (FTRACE_WARN_ON(!op->func)) { 5239 pr_warn("op=%p %pS\n", op, op); 5240 goto out; 5241 } 5242 op->func(ip, parent_ip, op, regs); 5243 } 5244 } while_for_each_ftrace_op(op); 5245 out: 5246 preempt_enable_notrace(); 5247 trace_clear_recursion(bit); 5248 } 5249 5250 /* 5251 * Some archs only support passing ip and parent_ip. Even though 5252 * the list function ignores the op parameter, we do not want any 5253 * C side effects, where a function is called without the caller 5254 * sending a third parameter. 5255 * Archs are to support both the regs and ftrace_ops at the same time. 5256 * If they support ftrace_ops, it is assumed they support regs. 5257 * If call backs want to use regs, they must either check for regs 5258 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS. 5259 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved. 5260 * An architecture can pass partial regs with ftrace_ops and still 5261 * set the ARCH_SUPPORTS_FTRACE_OPS. 5262 */ 5263 #if ARCH_SUPPORTS_FTRACE_OPS 5264 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 5265 struct ftrace_ops *op, struct pt_regs *regs) 5266 { 5267 __ftrace_ops_list_func(ip, parent_ip, NULL, regs); 5268 } 5269 #else 5270 static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip) 5271 { 5272 __ftrace_ops_list_func(ip, parent_ip, NULL, NULL); 5273 } 5274 #endif 5275 5276 /* 5277 * If there's only one function registered but it does not support 5278 * recursion, needs RCU protection and/or requires per cpu handling, then 5279 * this function will be called by the mcount trampoline. 5280 */ 5281 static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip, 5282 struct ftrace_ops *op, struct pt_regs *regs) 5283 { 5284 int bit; 5285 5286 if ((op->flags & FTRACE_OPS_FL_RCU) && !rcu_is_watching()) 5287 return; 5288 5289 bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX); 5290 if (bit < 0) 5291 return; 5292 5293 preempt_disable_notrace(); 5294 5295 if (!(op->flags & FTRACE_OPS_FL_PER_CPU) || 5296 !ftrace_function_local_disabled(op)) { 5297 op->func(ip, parent_ip, op, regs); 5298 } 5299 5300 preempt_enable_notrace(); 5301 trace_clear_recursion(bit); 5302 } 5303 5304 /** 5305 * ftrace_ops_get_func - get the function a trampoline should call 5306 * @ops: the ops to get the function for 5307 * 5308 * Normally the mcount trampoline will call the ops->func, but there 5309 * are times that it should not. For example, if the ops does not 5310 * have its own recursion protection, then it should call the 5311 * ftrace_ops_recurs_func() instead. 5312 * 5313 * Returns the function that the trampoline should call for @ops. 5314 */ 5315 ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops) 5316 { 5317 /* 5318 * If the function does not handle recursion, needs to be RCU safe, 5319 * or does per cpu logic, then we need to call the assist handler. 5320 */ 5321 if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE) || 5322 ops->flags & (FTRACE_OPS_FL_RCU | FTRACE_OPS_FL_PER_CPU)) 5323 return ftrace_ops_assist_func; 5324 5325 return ops->func; 5326 } 5327 5328 static void 5329 ftrace_filter_pid_sched_switch_probe(void *data, bool preempt, 5330 struct task_struct *prev, struct task_struct *next) 5331 { 5332 struct trace_array *tr = data; 5333 struct trace_pid_list *pid_list; 5334 5335 pid_list = rcu_dereference_sched(tr->function_pids); 5336 5337 this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid, 5338 trace_ignore_this_task(pid_list, next)); 5339 } 5340 5341 static void clear_ftrace_pids(struct trace_array *tr) 5342 { 5343 struct trace_pid_list *pid_list; 5344 int cpu; 5345 5346 pid_list = rcu_dereference_protected(tr->function_pids, 5347 lockdep_is_held(&ftrace_lock)); 5348 if (!pid_list) 5349 return; 5350 5351 unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr); 5352 5353 for_each_possible_cpu(cpu) 5354 per_cpu_ptr(tr->trace_buffer.data, cpu)->ftrace_ignore_pid = false; 5355 5356 rcu_assign_pointer(tr->function_pids, NULL); 5357 5358 /* Wait till all users are no longer using pid filtering */ 5359 synchronize_sched(); 5360 5361 trace_free_pid_list(pid_list); 5362 } 5363 5364 static void ftrace_pid_reset(struct trace_array *tr) 5365 { 5366 mutex_lock(&ftrace_lock); 5367 clear_ftrace_pids(tr); 5368 5369 ftrace_update_pid_func(); 5370 ftrace_startup_all(0); 5371 5372 mutex_unlock(&ftrace_lock); 5373 } 5374 5375 /* Greater than any max PID */ 5376 #define FTRACE_NO_PIDS (void *)(PID_MAX_LIMIT + 1) 5377 5378 static void *fpid_start(struct seq_file *m, loff_t *pos) 5379 __acquires(RCU) 5380 { 5381 struct trace_pid_list *pid_list; 5382 struct trace_array *tr = m->private; 5383 5384 mutex_lock(&ftrace_lock); 5385 rcu_read_lock_sched(); 5386 5387 pid_list = rcu_dereference_sched(tr->function_pids); 5388 5389 if (!pid_list) 5390 return !(*pos) ? FTRACE_NO_PIDS : NULL; 5391 5392 return trace_pid_start(pid_list, pos); 5393 } 5394 5395 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos) 5396 { 5397 struct trace_array *tr = m->private; 5398 struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids); 5399 5400 if (v == FTRACE_NO_PIDS) 5401 return NULL; 5402 5403 return trace_pid_next(pid_list, v, pos); 5404 } 5405 5406 static void fpid_stop(struct seq_file *m, void *p) 5407 __releases(RCU) 5408 { 5409 rcu_read_unlock_sched(); 5410 mutex_unlock(&ftrace_lock); 5411 } 5412 5413 static int fpid_show(struct seq_file *m, void *v) 5414 { 5415 if (v == FTRACE_NO_PIDS) { 5416 seq_puts(m, "no pid\n"); 5417 return 0; 5418 } 5419 5420 return trace_pid_show(m, v); 5421 } 5422 5423 static const struct seq_operations ftrace_pid_sops = { 5424 .start = fpid_start, 5425 .next = fpid_next, 5426 .stop = fpid_stop, 5427 .show = fpid_show, 5428 }; 5429 5430 static int 5431 ftrace_pid_open(struct inode *inode, struct file *file) 5432 { 5433 struct trace_array *tr = inode->i_private; 5434 struct seq_file *m; 5435 int ret = 0; 5436 5437 if (trace_array_get(tr) < 0) 5438 return -ENODEV; 5439 5440 if ((file->f_mode & FMODE_WRITE) && 5441 (file->f_flags & O_TRUNC)) 5442 ftrace_pid_reset(tr); 5443 5444 ret = seq_open(file, &ftrace_pid_sops); 5445 if (ret < 0) { 5446 trace_array_put(tr); 5447 } else { 5448 m = file->private_data; 5449 /* copy tr over to seq ops */ 5450 m->private = tr; 5451 } 5452 5453 return ret; 5454 } 5455 5456 static void ignore_task_cpu(void *data) 5457 { 5458 struct trace_array *tr = data; 5459 struct trace_pid_list *pid_list; 5460 5461 /* 5462 * This function is called by on_each_cpu() while the 5463 * event_mutex is held. 5464 */ 5465 pid_list = rcu_dereference_protected(tr->function_pids, 5466 mutex_is_locked(&ftrace_lock)); 5467 5468 this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid, 5469 trace_ignore_this_task(pid_list, current)); 5470 } 5471 5472 static ssize_t 5473 ftrace_pid_write(struct file *filp, const char __user *ubuf, 5474 size_t cnt, loff_t *ppos) 5475 { 5476 struct seq_file *m = filp->private_data; 5477 struct trace_array *tr = m->private; 5478 struct trace_pid_list *filtered_pids = NULL; 5479 struct trace_pid_list *pid_list; 5480 ssize_t ret; 5481 5482 if (!cnt) 5483 return 0; 5484 5485 mutex_lock(&ftrace_lock); 5486 5487 filtered_pids = rcu_dereference_protected(tr->function_pids, 5488 lockdep_is_held(&ftrace_lock)); 5489 5490 ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt); 5491 if (ret < 0) 5492 goto out; 5493 5494 rcu_assign_pointer(tr->function_pids, pid_list); 5495 5496 if (filtered_pids) { 5497 synchronize_sched(); 5498 trace_free_pid_list(filtered_pids); 5499 } else if (pid_list) { 5500 /* Register a probe to set whether to ignore the tracing of a task */ 5501 register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr); 5502 } 5503 5504 /* 5505 * Ignoring of pids is done at task switch. But we have to 5506 * check for those tasks that are currently running. 5507 * Always do this in case a pid was appended or removed. 5508 */ 5509 on_each_cpu(ignore_task_cpu, tr, 1); 5510 5511 ftrace_update_pid_func(); 5512 ftrace_startup_all(0); 5513 out: 5514 mutex_unlock(&ftrace_lock); 5515 5516 if (ret > 0) 5517 *ppos += ret; 5518 5519 return ret; 5520 } 5521 5522 static int 5523 ftrace_pid_release(struct inode *inode, struct file *file) 5524 { 5525 struct trace_array *tr = inode->i_private; 5526 5527 trace_array_put(tr); 5528 5529 return seq_release(inode, file); 5530 } 5531 5532 static const struct file_operations ftrace_pid_fops = { 5533 .open = ftrace_pid_open, 5534 .write = ftrace_pid_write, 5535 .read = seq_read, 5536 .llseek = tracing_lseek, 5537 .release = ftrace_pid_release, 5538 }; 5539 5540 void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer) 5541 { 5542 trace_create_file("set_ftrace_pid", 0644, d_tracer, 5543 tr, &ftrace_pid_fops); 5544 } 5545 5546 void __init ftrace_init_tracefs_toplevel(struct trace_array *tr, 5547 struct dentry *d_tracer) 5548 { 5549 /* Only the top level directory has the dyn_tracefs and profile */ 5550 WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL)); 5551 5552 ftrace_init_dyn_tracefs(d_tracer); 5553 ftrace_profile_tracefs(d_tracer); 5554 } 5555 5556 /** 5557 * ftrace_kill - kill ftrace 5558 * 5559 * This function should be used by panic code. It stops ftrace 5560 * but in a not so nice way. If you need to simply kill ftrace 5561 * from a non-atomic section, use ftrace_kill. 5562 */ 5563 void ftrace_kill(void) 5564 { 5565 ftrace_disabled = 1; 5566 ftrace_enabled = 0; 5567 clear_ftrace_function(); 5568 } 5569 5570 /** 5571 * Test if ftrace is dead or not. 5572 */ 5573 int ftrace_is_dead(void) 5574 { 5575 return ftrace_disabled; 5576 } 5577 5578 /** 5579 * register_ftrace_function - register a function for profiling 5580 * @ops - ops structure that holds the function for profiling. 5581 * 5582 * Register a function to be called by all functions in the 5583 * kernel. 5584 * 5585 * Note: @ops->func and all the functions it calls must be labeled 5586 * with "notrace", otherwise it will go into a 5587 * recursive loop. 5588 */ 5589 int register_ftrace_function(struct ftrace_ops *ops) 5590 { 5591 int ret = -1; 5592 5593 ftrace_ops_init(ops); 5594 5595 mutex_lock(&ftrace_lock); 5596 5597 ret = ftrace_startup(ops, 0); 5598 5599 mutex_unlock(&ftrace_lock); 5600 5601 return ret; 5602 } 5603 EXPORT_SYMBOL_GPL(register_ftrace_function); 5604 5605 /** 5606 * unregister_ftrace_function - unregister a function for profiling. 5607 * @ops - ops structure that holds the function to unregister 5608 * 5609 * Unregister a function that was added to be called by ftrace profiling. 5610 */ 5611 int unregister_ftrace_function(struct ftrace_ops *ops) 5612 { 5613 int ret; 5614 5615 mutex_lock(&ftrace_lock); 5616 ret = ftrace_shutdown(ops, 0); 5617 mutex_unlock(&ftrace_lock); 5618 5619 return ret; 5620 } 5621 EXPORT_SYMBOL_GPL(unregister_ftrace_function); 5622 5623 int 5624 ftrace_enable_sysctl(struct ctl_table *table, int write, 5625 void __user *buffer, size_t *lenp, 5626 loff_t *ppos) 5627 { 5628 int ret = -ENODEV; 5629 5630 mutex_lock(&ftrace_lock); 5631 5632 if (unlikely(ftrace_disabled)) 5633 goto out; 5634 5635 ret = proc_dointvec(table, write, buffer, lenp, ppos); 5636 5637 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled)) 5638 goto out; 5639 5640 last_ftrace_enabled = !!ftrace_enabled; 5641 5642 if (ftrace_enabled) { 5643 5644 /* we are starting ftrace again */ 5645 if (ftrace_ops_list != &ftrace_list_end) 5646 update_ftrace_function(); 5647 5648 ftrace_startup_sysctl(); 5649 5650 } else { 5651 /* stopping ftrace calls (just send to ftrace_stub) */ 5652 ftrace_trace_function = ftrace_stub; 5653 5654 ftrace_shutdown_sysctl(); 5655 } 5656 5657 out: 5658 mutex_unlock(&ftrace_lock); 5659 return ret; 5660 } 5661 5662 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 5663 5664 static struct ftrace_ops graph_ops = { 5665 .func = ftrace_stub, 5666 .flags = FTRACE_OPS_FL_RECURSION_SAFE | 5667 FTRACE_OPS_FL_INITIALIZED | 5668 FTRACE_OPS_FL_PID | 5669 FTRACE_OPS_FL_STUB, 5670 #ifdef FTRACE_GRAPH_TRAMP_ADDR 5671 .trampoline = FTRACE_GRAPH_TRAMP_ADDR, 5672 /* trampoline_size is only needed for dynamically allocated tramps */ 5673 #endif 5674 ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash) 5675 }; 5676 5677 void ftrace_graph_sleep_time_control(bool enable) 5678 { 5679 fgraph_sleep_time = enable; 5680 } 5681 5682 void ftrace_graph_graph_time_control(bool enable) 5683 { 5684 fgraph_graph_time = enable; 5685 } 5686 5687 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) 5688 { 5689 return 0; 5690 } 5691 5692 /* The callbacks that hook a function */ 5693 trace_func_graph_ret_t ftrace_graph_return = 5694 (trace_func_graph_ret_t)ftrace_stub; 5695 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub; 5696 static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub; 5697 5698 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */ 5699 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list) 5700 { 5701 int i; 5702 int ret = 0; 5703 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE; 5704 struct task_struct *g, *t; 5705 5706 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) { 5707 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH 5708 * sizeof(struct ftrace_ret_stack), 5709 GFP_KERNEL); 5710 if (!ret_stack_list[i]) { 5711 start = 0; 5712 end = i; 5713 ret = -ENOMEM; 5714 goto free; 5715 } 5716 } 5717 5718 read_lock(&tasklist_lock); 5719 do_each_thread(g, t) { 5720 if (start == end) { 5721 ret = -EAGAIN; 5722 goto unlock; 5723 } 5724 5725 if (t->ret_stack == NULL) { 5726 atomic_set(&t->tracing_graph_pause, 0); 5727 atomic_set(&t->trace_overrun, 0); 5728 t->curr_ret_stack = -1; 5729 /* Make sure the tasks see the -1 first: */ 5730 smp_wmb(); 5731 t->ret_stack = ret_stack_list[start++]; 5732 } 5733 } while_each_thread(g, t); 5734 5735 unlock: 5736 read_unlock(&tasklist_lock); 5737 free: 5738 for (i = start; i < end; i++) 5739 kfree(ret_stack_list[i]); 5740 return ret; 5741 } 5742 5743 static void 5744 ftrace_graph_probe_sched_switch(void *ignore, bool preempt, 5745 struct task_struct *prev, struct task_struct *next) 5746 { 5747 unsigned long long timestamp; 5748 int index; 5749 5750 /* 5751 * Does the user want to count the time a function was asleep. 5752 * If so, do not update the time stamps. 5753 */ 5754 if (fgraph_sleep_time) 5755 return; 5756 5757 timestamp = trace_clock_local(); 5758 5759 prev->ftrace_timestamp = timestamp; 5760 5761 /* only process tasks that we timestamped */ 5762 if (!next->ftrace_timestamp) 5763 return; 5764 5765 /* 5766 * Update all the counters in next to make up for the 5767 * time next was sleeping. 5768 */ 5769 timestamp -= next->ftrace_timestamp; 5770 5771 for (index = next->curr_ret_stack; index >= 0; index--) 5772 next->ret_stack[index].calltime += timestamp; 5773 } 5774 5775 /* Allocate a return stack for each task */ 5776 static int start_graph_tracing(void) 5777 { 5778 struct ftrace_ret_stack **ret_stack_list; 5779 int ret, cpu; 5780 5781 ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE * 5782 sizeof(struct ftrace_ret_stack *), 5783 GFP_KERNEL); 5784 5785 if (!ret_stack_list) 5786 return -ENOMEM; 5787 5788 /* The cpu_boot init_task->ret_stack will never be freed */ 5789 for_each_online_cpu(cpu) { 5790 if (!idle_task(cpu)->ret_stack) 5791 ftrace_graph_init_idle_task(idle_task(cpu), cpu); 5792 } 5793 5794 do { 5795 ret = alloc_retstack_tasklist(ret_stack_list); 5796 } while (ret == -EAGAIN); 5797 5798 if (!ret) { 5799 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); 5800 if (ret) 5801 pr_info("ftrace_graph: Couldn't activate tracepoint" 5802 " probe to kernel_sched_switch\n"); 5803 } 5804 5805 kfree(ret_stack_list); 5806 return ret; 5807 } 5808 5809 /* 5810 * Hibernation protection. 5811 * The state of the current task is too much unstable during 5812 * suspend/restore to disk. We want to protect against that. 5813 */ 5814 static int 5815 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state, 5816 void *unused) 5817 { 5818 switch (state) { 5819 case PM_HIBERNATION_PREPARE: 5820 pause_graph_tracing(); 5821 break; 5822 5823 case PM_POST_HIBERNATION: 5824 unpause_graph_tracing(); 5825 break; 5826 } 5827 return NOTIFY_DONE; 5828 } 5829 5830 static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace) 5831 { 5832 if (!ftrace_ops_test(&global_ops, trace->func, NULL)) 5833 return 0; 5834 return __ftrace_graph_entry(trace); 5835 } 5836 5837 /* 5838 * The function graph tracer should only trace the functions defined 5839 * by set_ftrace_filter and set_ftrace_notrace. If another function 5840 * tracer ops is registered, the graph tracer requires testing the 5841 * function against the global ops, and not just trace any function 5842 * that any ftrace_ops registered. 5843 */ 5844 static void update_function_graph_func(void) 5845 { 5846 struct ftrace_ops *op; 5847 bool do_test = false; 5848 5849 /* 5850 * The graph and global ops share the same set of functions 5851 * to test. If any other ops is on the list, then 5852 * the graph tracing needs to test if its the function 5853 * it should call. 5854 */ 5855 do_for_each_ftrace_op(op, ftrace_ops_list) { 5856 if (op != &global_ops && op != &graph_ops && 5857 op != &ftrace_list_end) { 5858 do_test = true; 5859 /* in double loop, break out with goto */ 5860 goto out; 5861 } 5862 } while_for_each_ftrace_op(op); 5863 out: 5864 if (do_test) 5865 ftrace_graph_entry = ftrace_graph_entry_test; 5866 else 5867 ftrace_graph_entry = __ftrace_graph_entry; 5868 } 5869 5870 static struct notifier_block ftrace_suspend_notifier = { 5871 .notifier_call = ftrace_suspend_notifier_call, 5872 }; 5873 5874 int register_ftrace_graph(trace_func_graph_ret_t retfunc, 5875 trace_func_graph_ent_t entryfunc) 5876 { 5877 int ret = 0; 5878 5879 mutex_lock(&ftrace_lock); 5880 5881 /* we currently allow only one tracer registered at a time */ 5882 if (ftrace_graph_active) { 5883 ret = -EBUSY; 5884 goto out; 5885 } 5886 5887 register_pm_notifier(&ftrace_suspend_notifier); 5888 5889 ftrace_graph_active++; 5890 ret = start_graph_tracing(); 5891 if (ret) { 5892 ftrace_graph_active--; 5893 goto out; 5894 } 5895 5896 ftrace_graph_return = retfunc; 5897 5898 /* 5899 * Update the indirect function to the entryfunc, and the 5900 * function that gets called to the entry_test first. Then 5901 * call the update fgraph entry function to determine if 5902 * the entryfunc should be called directly or not. 5903 */ 5904 __ftrace_graph_entry = entryfunc; 5905 ftrace_graph_entry = ftrace_graph_entry_test; 5906 update_function_graph_func(); 5907 5908 ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET); 5909 out: 5910 mutex_unlock(&ftrace_lock); 5911 return ret; 5912 } 5913 5914 void unregister_ftrace_graph(void) 5915 { 5916 mutex_lock(&ftrace_lock); 5917 5918 if (unlikely(!ftrace_graph_active)) 5919 goto out; 5920 5921 ftrace_graph_active--; 5922 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; 5923 ftrace_graph_entry = ftrace_graph_entry_stub; 5924 __ftrace_graph_entry = ftrace_graph_entry_stub; 5925 ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET); 5926 unregister_pm_notifier(&ftrace_suspend_notifier); 5927 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); 5928 5929 #ifdef CONFIG_DYNAMIC_FTRACE 5930 /* 5931 * Function graph does not allocate the trampoline, but 5932 * other global_ops do. We need to reset the ALLOC_TRAMP flag 5933 * if one was used. 5934 */ 5935 global_ops.trampoline = save_global_trampoline; 5936 if (save_global_flags & FTRACE_OPS_FL_ALLOC_TRAMP) 5937 global_ops.flags |= FTRACE_OPS_FL_ALLOC_TRAMP; 5938 #endif 5939 5940 out: 5941 mutex_unlock(&ftrace_lock); 5942 } 5943 5944 static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack); 5945 5946 static void 5947 graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack) 5948 { 5949 atomic_set(&t->tracing_graph_pause, 0); 5950 atomic_set(&t->trace_overrun, 0); 5951 t->ftrace_timestamp = 0; 5952 /* make curr_ret_stack visible before we add the ret_stack */ 5953 smp_wmb(); 5954 t->ret_stack = ret_stack; 5955 } 5956 5957 /* 5958 * Allocate a return stack for the idle task. May be the first 5959 * time through, or it may be done by CPU hotplug online. 5960 */ 5961 void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) 5962 { 5963 t->curr_ret_stack = -1; 5964 /* 5965 * The idle task has no parent, it either has its own 5966 * stack or no stack at all. 5967 */ 5968 if (t->ret_stack) 5969 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu)); 5970 5971 if (ftrace_graph_active) { 5972 struct ftrace_ret_stack *ret_stack; 5973 5974 ret_stack = per_cpu(idle_ret_stack, cpu); 5975 if (!ret_stack) { 5976 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH 5977 * sizeof(struct ftrace_ret_stack), 5978 GFP_KERNEL); 5979 if (!ret_stack) 5980 return; 5981 per_cpu(idle_ret_stack, cpu) = ret_stack; 5982 } 5983 graph_init_task(t, ret_stack); 5984 } 5985 } 5986 5987 /* Allocate a return stack for newly created task */ 5988 void ftrace_graph_init_task(struct task_struct *t) 5989 { 5990 /* Make sure we do not use the parent ret_stack */ 5991 t->ret_stack = NULL; 5992 t->curr_ret_stack = -1; 5993 5994 if (ftrace_graph_active) { 5995 struct ftrace_ret_stack *ret_stack; 5996 5997 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH 5998 * sizeof(struct ftrace_ret_stack), 5999 GFP_KERNEL); 6000 if (!ret_stack) 6001 return; 6002 graph_init_task(t, ret_stack); 6003 } 6004 } 6005 6006 void ftrace_graph_exit_task(struct task_struct *t) 6007 { 6008 struct ftrace_ret_stack *ret_stack = t->ret_stack; 6009 6010 t->ret_stack = NULL; 6011 /* NULL must become visible to IRQs before we free it: */ 6012 barrier(); 6013 6014 kfree(ret_stack); 6015 } 6016 #endif 6017