1 /* 2 * Infrastructure for profiling code inserted by 'gcc -pg'. 3 * 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> 6 * 7 * Originally ported from the -rt patch by: 8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> 9 * 10 * Based on code in the latency_tracer, that is: 11 * 12 * Copyright (C) 2004-2006 Ingo Molnar 13 * Copyright (C) 2004 Nadia Yvette Chambers 14 */ 15 16 #include <linux/stop_machine.h> 17 #include <linux/clocksource.h> 18 #include <linux/sched/task.h> 19 #include <linux/kallsyms.h> 20 #include <linux/seq_file.h> 21 #include <linux/suspend.h> 22 #include <linux/tracefs.h> 23 #include <linux/hardirq.h> 24 #include <linux/kthread.h> 25 #include <linux/uaccess.h> 26 #include <linux/bsearch.h> 27 #include <linux/module.h> 28 #include <linux/ftrace.h> 29 #include <linux/sysctl.h> 30 #include <linux/slab.h> 31 #include <linux/ctype.h> 32 #include <linux/sort.h> 33 #include <linux/list.h> 34 #include <linux/hash.h> 35 #include <linux/rcupdate.h> 36 37 #include <trace/events/sched.h> 38 39 #include <asm/sections.h> 40 #include <asm/setup.h> 41 42 #include "trace_output.h" 43 #include "trace_stat.h" 44 45 #define FTRACE_WARN_ON(cond) \ 46 ({ \ 47 int ___r = cond; \ 48 if (WARN_ON(___r)) \ 49 ftrace_kill(); \ 50 ___r; \ 51 }) 52 53 #define FTRACE_WARN_ON_ONCE(cond) \ 54 ({ \ 55 int ___r = cond; \ 56 if (WARN_ON_ONCE(___r)) \ 57 ftrace_kill(); \ 58 ___r; \ 59 }) 60 61 /* hash bits for specific function selection */ 62 #define FTRACE_HASH_BITS 7 63 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS) 64 #define FTRACE_HASH_DEFAULT_BITS 10 65 #define FTRACE_HASH_MAX_BITS 12 66 67 #ifdef CONFIG_DYNAMIC_FTRACE 68 #define INIT_OPS_HASH(opsname) \ 69 .func_hash = &opsname.local_hash, \ 70 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), 71 #define ASSIGN_OPS_HASH(opsname, val) \ 72 .func_hash = val, \ 73 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), 74 #else 75 #define INIT_OPS_HASH(opsname) 76 #define ASSIGN_OPS_HASH(opsname, val) 77 #endif 78 79 static struct ftrace_ops ftrace_list_end __read_mostly = { 80 .func = ftrace_stub, 81 .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB, 82 INIT_OPS_HASH(ftrace_list_end) 83 }; 84 85 /* ftrace_enabled is a method to turn ftrace on or off */ 86 int ftrace_enabled __read_mostly; 87 static int last_ftrace_enabled; 88 89 /* Current function tracing op */ 90 struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end; 91 /* What to set function_trace_op to */ 92 static struct ftrace_ops *set_function_trace_op; 93 94 static bool ftrace_pids_enabled(struct ftrace_ops *ops) 95 { 96 struct trace_array *tr; 97 98 if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private) 99 return false; 100 101 tr = ops->private; 102 103 return tr->function_pids != NULL; 104 } 105 106 static void ftrace_update_trampoline(struct ftrace_ops *ops); 107 108 /* 109 * ftrace_disabled is set when an anomaly is discovered. 110 * ftrace_disabled is much stronger than ftrace_enabled. 111 */ 112 static int ftrace_disabled __read_mostly; 113 114 static DEFINE_MUTEX(ftrace_lock); 115 116 static struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end; 117 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; 118 static struct ftrace_ops global_ops; 119 120 #if ARCH_SUPPORTS_FTRACE_OPS 121 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 122 struct ftrace_ops *op, struct pt_regs *regs); 123 #else 124 /* See comment below, where ftrace_ops_list_func is defined */ 125 static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip); 126 #define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops) 127 #endif 128 129 /* 130 * Traverse the ftrace_global_list, invoking all entries. The reason that we 131 * can use rcu_dereference_raw_notrace() is that elements removed from this list 132 * are simply leaked, so there is no need to interact with a grace-period 133 * mechanism. The rcu_dereference_raw_notrace() calls are needed to handle 134 * concurrent insertions into the ftrace_global_list. 135 * 136 * Silly Alpha and silly pointer-speculation compiler optimizations! 137 */ 138 #define do_for_each_ftrace_op(op, list) \ 139 op = rcu_dereference_raw_notrace(list); \ 140 do 141 142 /* 143 * Optimized for just a single item in the list (as that is the normal case). 144 */ 145 #define while_for_each_ftrace_op(op) \ 146 while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \ 147 unlikely((op) != &ftrace_list_end)) 148 149 static inline void ftrace_ops_init(struct ftrace_ops *ops) 150 { 151 #ifdef CONFIG_DYNAMIC_FTRACE 152 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { 153 mutex_init(&ops->local_hash.regex_lock); 154 ops->func_hash = &ops->local_hash; 155 ops->flags |= FTRACE_OPS_FL_INITIALIZED; 156 } 157 #endif 158 } 159 160 /** 161 * ftrace_nr_registered_ops - return number of ops registered 162 * 163 * Returns the number of ftrace_ops registered and tracing functions 164 */ 165 int ftrace_nr_registered_ops(void) 166 { 167 struct ftrace_ops *ops; 168 int cnt = 0; 169 170 mutex_lock(&ftrace_lock); 171 172 for (ops = rcu_dereference_protected(ftrace_ops_list, 173 lockdep_is_held(&ftrace_lock)); 174 ops != &ftrace_list_end; 175 ops = rcu_dereference_protected(ops->next, 176 lockdep_is_held(&ftrace_lock))) 177 cnt++; 178 179 mutex_unlock(&ftrace_lock); 180 181 return cnt; 182 } 183 184 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip, 185 struct ftrace_ops *op, struct pt_regs *regs) 186 { 187 struct trace_array *tr = op->private; 188 189 if (tr && this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid)) 190 return; 191 192 op->saved_func(ip, parent_ip, op, regs); 193 } 194 195 /** 196 * clear_ftrace_function - reset the ftrace function 197 * 198 * This NULLs the ftrace function and in essence stops 199 * tracing. There may be lag 200 */ 201 void clear_ftrace_function(void) 202 { 203 ftrace_trace_function = ftrace_stub; 204 } 205 206 static void per_cpu_ops_disable_all(struct ftrace_ops *ops) 207 { 208 int cpu; 209 210 for_each_possible_cpu(cpu) 211 *per_cpu_ptr(ops->disabled, cpu) = 1; 212 } 213 214 static int per_cpu_ops_alloc(struct ftrace_ops *ops) 215 { 216 int __percpu *disabled; 217 218 if (WARN_ON_ONCE(!(ops->flags & FTRACE_OPS_FL_PER_CPU))) 219 return -EINVAL; 220 221 disabled = alloc_percpu(int); 222 if (!disabled) 223 return -ENOMEM; 224 225 ops->disabled = disabled; 226 per_cpu_ops_disable_all(ops); 227 return 0; 228 } 229 230 static void ftrace_sync(struct work_struct *work) 231 { 232 /* 233 * This function is just a stub to implement a hard force 234 * of synchronize_sched(). This requires synchronizing 235 * tasks even in userspace and idle. 236 * 237 * Yes, function tracing is rude. 238 */ 239 } 240 241 static void ftrace_sync_ipi(void *data) 242 { 243 /* Probably not needed, but do it anyway */ 244 smp_rmb(); 245 } 246 247 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 248 static void update_function_graph_func(void); 249 250 /* Both enabled by default (can be cleared by function_graph tracer flags */ 251 static bool fgraph_sleep_time = true; 252 static bool fgraph_graph_time = true; 253 254 #else 255 static inline void update_function_graph_func(void) { } 256 #endif 257 258 259 static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops) 260 { 261 /* 262 * If this is a dynamic, RCU, or per CPU ops, or we force list func, 263 * then it needs to call the list anyway. 264 */ 265 if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU | 266 FTRACE_OPS_FL_RCU) || FTRACE_FORCE_LIST_FUNC) 267 return ftrace_ops_list_func; 268 269 return ftrace_ops_get_func(ops); 270 } 271 272 static void update_ftrace_function(void) 273 { 274 ftrace_func_t func; 275 276 /* 277 * Prepare the ftrace_ops that the arch callback will use. 278 * If there's only one ftrace_ops registered, the ftrace_ops_list 279 * will point to the ops we want. 280 */ 281 set_function_trace_op = rcu_dereference_protected(ftrace_ops_list, 282 lockdep_is_held(&ftrace_lock)); 283 284 /* If there's no ftrace_ops registered, just call the stub function */ 285 if (set_function_trace_op == &ftrace_list_end) { 286 func = ftrace_stub; 287 288 /* 289 * If we are at the end of the list and this ops is 290 * recursion safe and not dynamic and the arch supports passing ops, 291 * then have the mcount trampoline call the function directly. 292 */ 293 } else if (rcu_dereference_protected(ftrace_ops_list->next, 294 lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) { 295 func = ftrace_ops_get_list_func(ftrace_ops_list); 296 297 } else { 298 /* Just use the default ftrace_ops */ 299 set_function_trace_op = &ftrace_list_end; 300 func = ftrace_ops_list_func; 301 } 302 303 update_function_graph_func(); 304 305 /* If there's no change, then do nothing more here */ 306 if (ftrace_trace_function == func) 307 return; 308 309 /* 310 * If we are using the list function, it doesn't care 311 * about the function_trace_ops. 312 */ 313 if (func == ftrace_ops_list_func) { 314 ftrace_trace_function = func; 315 /* 316 * Don't even bother setting function_trace_ops, 317 * it would be racy to do so anyway. 318 */ 319 return; 320 } 321 322 #ifndef CONFIG_DYNAMIC_FTRACE 323 /* 324 * For static tracing, we need to be a bit more careful. 325 * The function change takes affect immediately. Thus, 326 * we need to coorditate the setting of the function_trace_ops 327 * with the setting of the ftrace_trace_function. 328 * 329 * Set the function to the list ops, which will call the 330 * function we want, albeit indirectly, but it handles the 331 * ftrace_ops and doesn't depend on function_trace_op. 332 */ 333 ftrace_trace_function = ftrace_ops_list_func; 334 /* 335 * Make sure all CPUs see this. Yes this is slow, but static 336 * tracing is slow and nasty to have enabled. 337 */ 338 schedule_on_each_cpu(ftrace_sync); 339 /* Now all cpus are using the list ops. */ 340 function_trace_op = set_function_trace_op; 341 /* Make sure the function_trace_op is visible on all CPUs */ 342 smp_wmb(); 343 /* Nasty way to force a rmb on all cpus */ 344 smp_call_function(ftrace_sync_ipi, NULL, 1); 345 /* OK, we are all set to update the ftrace_trace_function now! */ 346 #endif /* !CONFIG_DYNAMIC_FTRACE */ 347 348 ftrace_trace_function = func; 349 } 350 351 int using_ftrace_ops_list_func(void) 352 { 353 return ftrace_trace_function == ftrace_ops_list_func; 354 } 355 356 static void add_ftrace_ops(struct ftrace_ops __rcu **list, 357 struct ftrace_ops *ops) 358 { 359 rcu_assign_pointer(ops->next, *list); 360 361 /* 362 * We are entering ops into the list but another 363 * CPU might be walking that list. We need to make sure 364 * the ops->next pointer is valid before another CPU sees 365 * the ops pointer included into the list. 366 */ 367 rcu_assign_pointer(*list, ops); 368 } 369 370 static int remove_ftrace_ops(struct ftrace_ops __rcu **list, 371 struct ftrace_ops *ops) 372 { 373 struct ftrace_ops **p; 374 375 /* 376 * If we are removing the last function, then simply point 377 * to the ftrace_stub. 378 */ 379 if (rcu_dereference_protected(*list, 380 lockdep_is_held(&ftrace_lock)) == ops && 381 rcu_dereference_protected(ops->next, 382 lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) { 383 *list = &ftrace_list_end; 384 return 0; 385 } 386 387 for (p = list; *p != &ftrace_list_end; p = &(*p)->next) 388 if (*p == ops) 389 break; 390 391 if (*p != ops) 392 return -1; 393 394 *p = (*p)->next; 395 return 0; 396 } 397 398 static void ftrace_update_trampoline(struct ftrace_ops *ops); 399 400 static int __register_ftrace_function(struct ftrace_ops *ops) 401 { 402 if (ops->flags & FTRACE_OPS_FL_DELETED) 403 return -EINVAL; 404 405 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) 406 return -EBUSY; 407 408 #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS 409 /* 410 * If the ftrace_ops specifies SAVE_REGS, then it only can be used 411 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set. 412 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant. 413 */ 414 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS && 415 !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)) 416 return -EINVAL; 417 418 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED) 419 ops->flags |= FTRACE_OPS_FL_SAVE_REGS; 420 #endif 421 422 if (!core_kernel_data((unsigned long)ops)) 423 ops->flags |= FTRACE_OPS_FL_DYNAMIC; 424 425 if (ops->flags & FTRACE_OPS_FL_PER_CPU) { 426 if (per_cpu_ops_alloc(ops)) 427 return -ENOMEM; 428 } 429 430 add_ftrace_ops(&ftrace_ops_list, ops); 431 432 /* Always save the function, and reset at unregistering */ 433 ops->saved_func = ops->func; 434 435 if (ftrace_pids_enabled(ops)) 436 ops->func = ftrace_pid_func; 437 438 ftrace_update_trampoline(ops); 439 440 if (ftrace_enabled) 441 update_ftrace_function(); 442 443 return 0; 444 } 445 446 static int __unregister_ftrace_function(struct ftrace_ops *ops) 447 { 448 int ret; 449 450 if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED))) 451 return -EBUSY; 452 453 ret = remove_ftrace_ops(&ftrace_ops_list, ops); 454 455 if (ret < 0) 456 return ret; 457 458 if (ftrace_enabled) 459 update_ftrace_function(); 460 461 ops->func = ops->saved_func; 462 463 return 0; 464 } 465 466 static void ftrace_update_pid_func(void) 467 { 468 struct ftrace_ops *op; 469 470 /* Only do something if we are tracing something */ 471 if (ftrace_trace_function == ftrace_stub) 472 return; 473 474 do_for_each_ftrace_op(op, ftrace_ops_list) { 475 if (op->flags & FTRACE_OPS_FL_PID) { 476 op->func = ftrace_pids_enabled(op) ? 477 ftrace_pid_func : op->saved_func; 478 ftrace_update_trampoline(op); 479 } 480 } while_for_each_ftrace_op(op); 481 482 update_ftrace_function(); 483 } 484 485 #ifdef CONFIG_FUNCTION_PROFILER 486 struct ftrace_profile { 487 struct hlist_node node; 488 unsigned long ip; 489 unsigned long counter; 490 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 491 unsigned long long time; 492 unsigned long long time_squared; 493 #endif 494 }; 495 496 struct ftrace_profile_page { 497 struct ftrace_profile_page *next; 498 unsigned long index; 499 struct ftrace_profile records[]; 500 }; 501 502 struct ftrace_profile_stat { 503 atomic_t disabled; 504 struct hlist_head *hash; 505 struct ftrace_profile_page *pages; 506 struct ftrace_profile_page *start; 507 struct tracer_stat stat; 508 }; 509 510 #define PROFILE_RECORDS_SIZE \ 511 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records)) 512 513 #define PROFILES_PER_PAGE \ 514 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile)) 515 516 static int ftrace_profile_enabled __read_mostly; 517 518 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */ 519 static DEFINE_MUTEX(ftrace_profile_lock); 520 521 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats); 522 523 #define FTRACE_PROFILE_HASH_BITS 10 524 #define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS) 525 526 static void * 527 function_stat_next(void *v, int idx) 528 { 529 struct ftrace_profile *rec = v; 530 struct ftrace_profile_page *pg; 531 532 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); 533 534 again: 535 if (idx != 0) 536 rec++; 537 538 if ((void *)rec >= (void *)&pg->records[pg->index]) { 539 pg = pg->next; 540 if (!pg) 541 return NULL; 542 rec = &pg->records[0]; 543 if (!rec->counter) 544 goto again; 545 } 546 547 return rec; 548 } 549 550 static void *function_stat_start(struct tracer_stat *trace) 551 { 552 struct ftrace_profile_stat *stat = 553 container_of(trace, struct ftrace_profile_stat, stat); 554 555 if (!stat || !stat->start) 556 return NULL; 557 558 return function_stat_next(&stat->start->records[0], 0); 559 } 560 561 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 562 /* function graph compares on total time */ 563 static int function_stat_cmp(void *p1, void *p2) 564 { 565 struct ftrace_profile *a = p1; 566 struct ftrace_profile *b = p2; 567 568 if (a->time < b->time) 569 return -1; 570 if (a->time > b->time) 571 return 1; 572 else 573 return 0; 574 } 575 #else 576 /* not function graph compares against hits */ 577 static int function_stat_cmp(void *p1, void *p2) 578 { 579 struct ftrace_profile *a = p1; 580 struct ftrace_profile *b = p2; 581 582 if (a->counter < b->counter) 583 return -1; 584 if (a->counter > b->counter) 585 return 1; 586 else 587 return 0; 588 } 589 #endif 590 591 static int function_stat_headers(struct seq_file *m) 592 { 593 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 594 seq_puts(m, " Function " 595 "Hit Time Avg s^2\n" 596 " -------- " 597 "--- ---- --- ---\n"); 598 #else 599 seq_puts(m, " Function Hit\n" 600 " -------- ---\n"); 601 #endif 602 return 0; 603 } 604 605 static int function_stat_show(struct seq_file *m, void *v) 606 { 607 struct ftrace_profile *rec = v; 608 char str[KSYM_SYMBOL_LEN]; 609 int ret = 0; 610 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 611 static struct trace_seq s; 612 unsigned long long avg; 613 unsigned long long stddev; 614 #endif 615 mutex_lock(&ftrace_profile_lock); 616 617 /* we raced with function_profile_reset() */ 618 if (unlikely(rec->counter == 0)) { 619 ret = -EBUSY; 620 goto out; 621 } 622 623 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 624 avg = rec->time; 625 do_div(avg, rec->counter); 626 if (tracing_thresh && (avg < tracing_thresh)) 627 goto out; 628 #endif 629 630 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 631 seq_printf(m, " %-30.30s %10lu", str, rec->counter); 632 633 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 634 seq_puts(m, " "); 635 636 /* Sample standard deviation (s^2) */ 637 if (rec->counter <= 1) 638 stddev = 0; 639 else { 640 /* 641 * Apply Welford's method: 642 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2) 643 */ 644 stddev = rec->counter * rec->time_squared - 645 rec->time * rec->time; 646 647 /* 648 * Divide only 1000 for ns^2 -> us^2 conversion. 649 * trace_print_graph_duration will divide 1000 again. 650 */ 651 do_div(stddev, rec->counter * (rec->counter - 1) * 1000); 652 } 653 654 trace_seq_init(&s); 655 trace_print_graph_duration(rec->time, &s); 656 trace_seq_puts(&s, " "); 657 trace_print_graph_duration(avg, &s); 658 trace_seq_puts(&s, " "); 659 trace_print_graph_duration(stddev, &s); 660 trace_print_seq(m, &s); 661 #endif 662 seq_putc(m, '\n'); 663 out: 664 mutex_unlock(&ftrace_profile_lock); 665 666 return ret; 667 } 668 669 static void ftrace_profile_reset(struct ftrace_profile_stat *stat) 670 { 671 struct ftrace_profile_page *pg; 672 673 pg = stat->pages = stat->start; 674 675 while (pg) { 676 memset(pg->records, 0, PROFILE_RECORDS_SIZE); 677 pg->index = 0; 678 pg = pg->next; 679 } 680 681 memset(stat->hash, 0, 682 FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head)); 683 } 684 685 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat) 686 { 687 struct ftrace_profile_page *pg; 688 int functions; 689 int pages; 690 int i; 691 692 /* If we already allocated, do nothing */ 693 if (stat->pages) 694 return 0; 695 696 stat->pages = (void *)get_zeroed_page(GFP_KERNEL); 697 if (!stat->pages) 698 return -ENOMEM; 699 700 #ifdef CONFIG_DYNAMIC_FTRACE 701 functions = ftrace_update_tot_cnt; 702 #else 703 /* 704 * We do not know the number of functions that exist because 705 * dynamic tracing is what counts them. With past experience 706 * we have around 20K functions. That should be more than enough. 707 * It is highly unlikely we will execute every function in 708 * the kernel. 709 */ 710 functions = 20000; 711 #endif 712 713 pg = stat->start = stat->pages; 714 715 pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE); 716 717 for (i = 1; i < pages; i++) { 718 pg->next = (void *)get_zeroed_page(GFP_KERNEL); 719 if (!pg->next) 720 goto out_free; 721 pg = pg->next; 722 } 723 724 return 0; 725 726 out_free: 727 pg = stat->start; 728 while (pg) { 729 unsigned long tmp = (unsigned long)pg; 730 731 pg = pg->next; 732 free_page(tmp); 733 } 734 735 stat->pages = NULL; 736 stat->start = NULL; 737 738 return -ENOMEM; 739 } 740 741 static int ftrace_profile_init_cpu(int cpu) 742 { 743 struct ftrace_profile_stat *stat; 744 int size; 745 746 stat = &per_cpu(ftrace_profile_stats, cpu); 747 748 if (stat->hash) { 749 /* If the profile is already created, simply reset it */ 750 ftrace_profile_reset(stat); 751 return 0; 752 } 753 754 /* 755 * We are profiling all functions, but usually only a few thousand 756 * functions are hit. We'll make a hash of 1024 items. 757 */ 758 size = FTRACE_PROFILE_HASH_SIZE; 759 760 stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL); 761 762 if (!stat->hash) 763 return -ENOMEM; 764 765 /* Preallocate the function profiling pages */ 766 if (ftrace_profile_pages_init(stat) < 0) { 767 kfree(stat->hash); 768 stat->hash = NULL; 769 return -ENOMEM; 770 } 771 772 return 0; 773 } 774 775 static int ftrace_profile_init(void) 776 { 777 int cpu; 778 int ret = 0; 779 780 for_each_possible_cpu(cpu) { 781 ret = ftrace_profile_init_cpu(cpu); 782 if (ret) 783 break; 784 } 785 786 return ret; 787 } 788 789 /* interrupts must be disabled */ 790 static struct ftrace_profile * 791 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip) 792 { 793 struct ftrace_profile *rec; 794 struct hlist_head *hhd; 795 unsigned long key; 796 797 key = hash_long(ip, FTRACE_PROFILE_HASH_BITS); 798 hhd = &stat->hash[key]; 799 800 if (hlist_empty(hhd)) 801 return NULL; 802 803 hlist_for_each_entry_rcu_notrace(rec, hhd, node) { 804 if (rec->ip == ip) 805 return rec; 806 } 807 808 return NULL; 809 } 810 811 static void ftrace_add_profile(struct ftrace_profile_stat *stat, 812 struct ftrace_profile *rec) 813 { 814 unsigned long key; 815 816 key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS); 817 hlist_add_head_rcu(&rec->node, &stat->hash[key]); 818 } 819 820 /* 821 * The memory is already allocated, this simply finds a new record to use. 822 */ 823 static struct ftrace_profile * 824 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip) 825 { 826 struct ftrace_profile *rec = NULL; 827 828 /* prevent recursion (from NMIs) */ 829 if (atomic_inc_return(&stat->disabled) != 1) 830 goto out; 831 832 /* 833 * Try to find the function again since an NMI 834 * could have added it 835 */ 836 rec = ftrace_find_profiled_func(stat, ip); 837 if (rec) 838 goto out; 839 840 if (stat->pages->index == PROFILES_PER_PAGE) { 841 if (!stat->pages->next) 842 goto out; 843 stat->pages = stat->pages->next; 844 } 845 846 rec = &stat->pages->records[stat->pages->index++]; 847 rec->ip = ip; 848 ftrace_add_profile(stat, rec); 849 850 out: 851 atomic_dec(&stat->disabled); 852 853 return rec; 854 } 855 856 static void 857 function_profile_call(unsigned long ip, unsigned long parent_ip, 858 struct ftrace_ops *ops, struct pt_regs *regs) 859 { 860 struct ftrace_profile_stat *stat; 861 struct ftrace_profile *rec; 862 unsigned long flags; 863 864 if (!ftrace_profile_enabled) 865 return; 866 867 local_irq_save(flags); 868 869 stat = this_cpu_ptr(&ftrace_profile_stats); 870 if (!stat->hash || !ftrace_profile_enabled) 871 goto out; 872 873 rec = ftrace_find_profiled_func(stat, ip); 874 if (!rec) { 875 rec = ftrace_profile_alloc(stat, ip); 876 if (!rec) 877 goto out; 878 } 879 880 rec->counter++; 881 out: 882 local_irq_restore(flags); 883 } 884 885 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 886 static int profile_graph_entry(struct ftrace_graph_ent *trace) 887 { 888 int index = trace->depth; 889 890 function_profile_call(trace->func, 0, NULL, NULL); 891 892 if (index >= 0 && index < FTRACE_RETFUNC_DEPTH) 893 current->ret_stack[index].subtime = 0; 894 895 return 1; 896 } 897 898 static void profile_graph_return(struct ftrace_graph_ret *trace) 899 { 900 struct ftrace_profile_stat *stat; 901 unsigned long long calltime; 902 struct ftrace_profile *rec; 903 unsigned long flags; 904 905 local_irq_save(flags); 906 stat = this_cpu_ptr(&ftrace_profile_stats); 907 if (!stat->hash || !ftrace_profile_enabled) 908 goto out; 909 910 /* If the calltime was zero'd ignore it */ 911 if (!trace->calltime) 912 goto out; 913 914 calltime = trace->rettime - trace->calltime; 915 916 if (!fgraph_graph_time) { 917 int index; 918 919 index = trace->depth; 920 921 /* Append this call time to the parent time to subtract */ 922 if (index) 923 current->ret_stack[index - 1].subtime += calltime; 924 925 if (current->ret_stack[index].subtime < calltime) 926 calltime -= current->ret_stack[index].subtime; 927 else 928 calltime = 0; 929 } 930 931 rec = ftrace_find_profiled_func(stat, trace->func); 932 if (rec) { 933 rec->time += calltime; 934 rec->time_squared += calltime * calltime; 935 } 936 937 out: 938 local_irq_restore(flags); 939 } 940 941 static int register_ftrace_profiler(void) 942 { 943 return register_ftrace_graph(&profile_graph_return, 944 &profile_graph_entry); 945 } 946 947 static void unregister_ftrace_profiler(void) 948 { 949 unregister_ftrace_graph(); 950 } 951 #else 952 static struct ftrace_ops ftrace_profile_ops __read_mostly = { 953 .func = function_profile_call, 954 .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED, 955 INIT_OPS_HASH(ftrace_profile_ops) 956 }; 957 958 static int register_ftrace_profiler(void) 959 { 960 return register_ftrace_function(&ftrace_profile_ops); 961 } 962 963 static void unregister_ftrace_profiler(void) 964 { 965 unregister_ftrace_function(&ftrace_profile_ops); 966 } 967 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 968 969 static ssize_t 970 ftrace_profile_write(struct file *filp, const char __user *ubuf, 971 size_t cnt, loff_t *ppos) 972 { 973 unsigned long val; 974 int ret; 975 976 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 977 if (ret) 978 return ret; 979 980 val = !!val; 981 982 mutex_lock(&ftrace_profile_lock); 983 if (ftrace_profile_enabled ^ val) { 984 if (val) { 985 ret = ftrace_profile_init(); 986 if (ret < 0) { 987 cnt = ret; 988 goto out; 989 } 990 991 ret = register_ftrace_profiler(); 992 if (ret < 0) { 993 cnt = ret; 994 goto out; 995 } 996 ftrace_profile_enabled = 1; 997 } else { 998 ftrace_profile_enabled = 0; 999 /* 1000 * unregister_ftrace_profiler calls stop_machine 1001 * so this acts like an synchronize_sched. 1002 */ 1003 unregister_ftrace_profiler(); 1004 } 1005 } 1006 out: 1007 mutex_unlock(&ftrace_profile_lock); 1008 1009 *ppos += cnt; 1010 1011 return cnt; 1012 } 1013 1014 static ssize_t 1015 ftrace_profile_read(struct file *filp, char __user *ubuf, 1016 size_t cnt, loff_t *ppos) 1017 { 1018 char buf[64]; /* big enough to hold a number */ 1019 int r; 1020 1021 r = sprintf(buf, "%u\n", ftrace_profile_enabled); 1022 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 1023 } 1024 1025 static const struct file_operations ftrace_profile_fops = { 1026 .open = tracing_open_generic, 1027 .read = ftrace_profile_read, 1028 .write = ftrace_profile_write, 1029 .llseek = default_llseek, 1030 }; 1031 1032 /* used to initialize the real stat files */ 1033 static struct tracer_stat function_stats __initdata = { 1034 .name = "functions", 1035 .stat_start = function_stat_start, 1036 .stat_next = function_stat_next, 1037 .stat_cmp = function_stat_cmp, 1038 .stat_headers = function_stat_headers, 1039 .stat_show = function_stat_show 1040 }; 1041 1042 static __init void ftrace_profile_tracefs(struct dentry *d_tracer) 1043 { 1044 struct ftrace_profile_stat *stat; 1045 struct dentry *entry; 1046 char *name; 1047 int ret; 1048 int cpu; 1049 1050 for_each_possible_cpu(cpu) { 1051 stat = &per_cpu(ftrace_profile_stats, cpu); 1052 1053 name = kasprintf(GFP_KERNEL, "function%d", cpu); 1054 if (!name) { 1055 /* 1056 * The files created are permanent, if something happens 1057 * we still do not free memory. 1058 */ 1059 WARN(1, 1060 "Could not allocate stat file for cpu %d\n", 1061 cpu); 1062 return; 1063 } 1064 stat->stat = function_stats; 1065 stat->stat.name = name; 1066 ret = register_stat_tracer(&stat->stat); 1067 if (ret) { 1068 WARN(1, 1069 "Could not register function stat for cpu %d\n", 1070 cpu); 1071 kfree(name); 1072 return; 1073 } 1074 } 1075 1076 entry = tracefs_create_file("function_profile_enabled", 0644, 1077 d_tracer, NULL, &ftrace_profile_fops); 1078 if (!entry) 1079 pr_warn("Could not create tracefs 'function_profile_enabled' entry\n"); 1080 } 1081 1082 #else /* CONFIG_FUNCTION_PROFILER */ 1083 static __init void ftrace_profile_tracefs(struct dentry *d_tracer) 1084 { 1085 } 1086 #endif /* CONFIG_FUNCTION_PROFILER */ 1087 1088 static struct pid * const ftrace_swapper_pid = &init_struct_pid; 1089 1090 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 1091 static int ftrace_graph_active; 1092 #else 1093 # define ftrace_graph_active 0 1094 #endif 1095 1096 #ifdef CONFIG_DYNAMIC_FTRACE 1097 1098 static struct ftrace_ops *removed_ops; 1099 1100 /* 1101 * Set when doing a global update, like enabling all recs or disabling them. 1102 * It is not set when just updating a single ftrace_ops. 1103 */ 1104 static bool update_all_ops; 1105 1106 #ifndef CONFIG_FTRACE_MCOUNT_RECORD 1107 # error Dynamic ftrace depends on MCOUNT_RECORD 1108 #endif 1109 1110 struct ftrace_func_entry { 1111 struct hlist_node hlist; 1112 unsigned long ip; 1113 }; 1114 1115 struct ftrace_func_probe { 1116 struct ftrace_probe_ops *probe_ops; 1117 struct ftrace_ops ops; 1118 struct trace_array *tr; 1119 struct list_head list; 1120 void *data; 1121 int ref; 1122 }; 1123 1124 /* 1125 * We make these constant because no one should touch them, 1126 * but they are used as the default "empty hash", to avoid allocating 1127 * it all the time. These are in a read only section such that if 1128 * anyone does try to modify it, it will cause an exception. 1129 */ 1130 static const struct hlist_head empty_buckets[1]; 1131 static const struct ftrace_hash empty_hash = { 1132 .buckets = (struct hlist_head *)empty_buckets, 1133 }; 1134 #define EMPTY_HASH ((struct ftrace_hash *)&empty_hash) 1135 1136 static struct ftrace_ops global_ops = { 1137 .func = ftrace_stub, 1138 .local_hash.notrace_hash = EMPTY_HASH, 1139 .local_hash.filter_hash = EMPTY_HASH, 1140 INIT_OPS_HASH(global_ops) 1141 .flags = FTRACE_OPS_FL_RECURSION_SAFE | 1142 FTRACE_OPS_FL_INITIALIZED | 1143 FTRACE_OPS_FL_PID, 1144 }; 1145 1146 /* 1147 * This is used by __kernel_text_address() to return true if the 1148 * address is on a dynamically allocated trampoline that would 1149 * not return true for either core_kernel_text() or 1150 * is_module_text_address(). 1151 */ 1152 bool is_ftrace_trampoline(unsigned long addr) 1153 { 1154 struct ftrace_ops *op; 1155 bool ret = false; 1156 1157 /* 1158 * Some of the ops may be dynamically allocated, 1159 * they are freed after a synchronize_sched(). 1160 */ 1161 preempt_disable_notrace(); 1162 1163 do_for_each_ftrace_op(op, ftrace_ops_list) { 1164 /* 1165 * This is to check for dynamically allocated trampolines. 1166 * Trampolines that are in kernel text will have 1167 * core_kernel_text() return true. 1168 */ 1169 if (op->trampoline && op->trampoline_size) 1170 if (addr >= op->trampoline && 1171 addr < op->trampoline + op->trampoline_size) { 1172 ret = true; 1173 goto out; 1174 } 1175 } while_for_each_ftrace_op(op); 1176 1177 out: 1178 preempt_enable_notrace(); 1179 1180 return ret; 1181 } 1182 1183 struct ftrace_page { 1184 struct ftrace_page *next; 1185 struct dyn_ftrace *records; 1186 int index; 1187 int size; 1188 }; 1189 1190 #define ENTRY_SIZE sizeof(struct dyn_ftrace) 1191 #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE) 1192 1193 /* estimate from running different kernels */ 1194 #define NR_TO_INIT 10000 1195 1196 static struct ftrace_page *ftrace_pages_start; 1197 static struct ftrace_page *ftrace_pages; 1198 1199 static __always_inline unsigned long 1200 ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip) 1201 { 1202 if (hash->size_bits > 0) 1203 return hash_long(ip, hash->size_bits); 1204 1205 return 0; 1206 } 1207 1208 /* Only use this function if ftrace_hash_empty() has already been tested */ 1209 static __always_inline struct ftrace_func_entry * 1210 __ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) 1211 { 1212 unsigned long key; 1213 struct ftrace_func_entry *entry; 1214 struct hlist_head *hhd; 1215 1216 key = ftrace_hash_key(hash, ip); 1217 hhd = &hash->buckets[key]; 1218 1219 hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) { 1220 if (entry->ip == ip) 1221 return entry; 1222 } 1223 return NULL; 1224 } 1225 1226 /** 1227 * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash 1228 * @hash: The hash to look at 1229 * @ip: The instruction pointer to test 1230 * 1231 * Search a given @hash to see if a given instruction pointer (@ip) 1232 * exists in it. 1233 * 1234 * Returns the entry that holds the @ip if found. NULL otherwise. 1235 */ 1236 struct ftrace_func_entry * 1237 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) 1238 { 1239 if (ftrace_hash_empty(hash)) 1240 return NULL; 1241 1242 return __ftrace_lookup_ip(hash, ip); 1243 } 1244 1245 static void __add_hash_entry(struct ftrace_hash *hash, 1246 struct ftrace_func_entry *entry) 1247 { 1248 struct hlist_head *hhd; 1249 unsigned long key; 1250 1251 key = ftrace_hash_key(hash, entry->ip); 1252 hhd = &hash->buckets[key]; 1253 hlist_add_head(&entry->hlist, hhd); 1254 hash->count++; 1255 } 1256 1257 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip) 1258 { 1259 struct ftrace_func_entry *entry; 1260 1261 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 1262 if (!entry) 1263 return -ENOMEM; 1264 1265 entry->ip = ip; 1266 __add_hash_entry(hash, entry); 1267 1268 return 0; 1269 } 1270 1271 static void 1272 free_hash_entry(struct ftrace_hash *hash, 1273 struct ftrace_func_entry *entry) 1274 { 1275 hlist_del(&entry->hlist); 1276 kfree(entry); 1277 hash->count--; 1278 } 1279 1280 static void 1281 remove_hash_entry(struct ftrace_hash *hash, 1282 struct ftrace_func_entry *entry) 1283 { 1284 hlist_del_rcu(&entry->hlist); 1285 hash->count--; 1286 } 1287 1288 static void ftrace_hash_clear(struct ftrace_hash *hash) 1289 { 1290 struct hlist_head *hhd; 1291 struct hlist_node *tn; 1292 struct ftrace_func_entry *entry; 1293 int size = 1 << hash->size_bits; 1294 int i; 1295 1296 if (!hash->count) 1297 return; 1298 1299 for (i = 0; i < size; i++) { 1300 hhd = &hash->buckets[i]; 1301 hlist_for_each_entry_safe(entry, tn, hhd, hlist) 1302 free_hash_entry(hash, entry); 1303 } 1304 FTRACE_WARN_ON(hash->count); 1305 } 1306 1307 static void free_ftrace_mod(struct ftrace_mod_load *ftrace_mod) 1308 { 1309 list_del(&ftrace_mod->list); 1310 kfree(ftrace_mod->module); 1311 kfree(ftrace_mod->func); 1312 kfree(ftrace_mod); 1313 } 1314 1315 static void clear_ftrace_mod_list(struct list_head *head) 1316 { 1317 struct ftrace_mod_load *p, *n; 1318 1319 /* stack tracer isn't supported yet */ 1320 if (!head) 1321 return; 1322 1323 mutex_lock(&ftrace_lock); 1324 list_for_each_entry_safe(p, n, head, list) 1325 free_ftrace_mod(p); 1326 mutex_unlock(&ftrace_lock); 1327 } 1328 1329 static void free_ftrace_hash(struct ftrace_hash *hash) 1330 { 1331 if (!hash || hash == EMPTY_HASH) 1332 return; 1333 ftrace_hash_clear(hash); 1334 kfree(hash->buckets); 1335 kfree(hash); 1336 } 1337 1338 static void __free_ftrace_hash_rcu(struct rcu_head *rcu) 1339 { 1340 struct ftrace_hash *hash; 1341 1342 hash = container_of(rcu, struct ftrace_hash, rcu); 1343 free_ftrace_hash(hash); 1344 } 1345 1346 static void free_ftrace_hash_rcu(struct ftrace_hash *hash) 1347 { 1348 if (!hash || hash == EMPTY_HASH) 1349 return; 1350 call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu); 1351 } 1352 1353 void ftrace_free_filter(struct ftrace_ops *ops) 1354 { 1355 ftrace_ops_init(ops); 1356 free_ftrace_hash(ops->func_hash->filter_hash); 1357 free_ftrace_hash(ops->func_hash->notrace_hash); 1358 } 1359 1360 static struct ftrace_hash *alloc_ftrace_hash(int size_bits) 1361 { 1362 struct ftrace_hash *hash; 1363 int size; 1364 1365 hash = kzalloc(sizeof(*hash), GFP_KERNEL); 1366 if (!hash) 1367 return NULL; 1368 1369 size = 1 << size_bits; 1370 hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL); 1371 1372 if (!hash->buckets) { 1373 kfree(hash); 1374 return NULL; 1375 } 1376 1377 hash->size_bits = size_bits; 1378 1379 return hash; 1380 } 1381 1382 1383 static int ftrace_add_mod(struct trace_array *tr, 1384 const char *func, const char *module, 1385 int enable) 1386 { 1387 struct ftrace_mod_load *ftrace_mod; 1388 struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace; 1389 1390 ftrace_mod = kzalloc(sizeof(*ftrace_mod), GFP_KERNEL); 1391 if (!ftrace_mod) 1392 return -ENOMEM; 1393 1394 ftrace_mod->func = kstrdup(func, GFP_KERNEL); 1395 ftrace_mod->module = kstrdup(module, GFP_KERNEL); 1396 ftrace_mod->enable = enable; 1397 1398 if (!ftrace_mod->func || !ftrace_mod->module) 1399 goto out_free; 1400 1401 list_add(&ftrace_mod->list, mod_head); 1402 1403 return 0; 1404 1405 out_free: 1406 free_ftrace_mod(ftrace_mod); 1407 1408 return -ENOMEM; 1409 } 1410 1411 static struct ftrace_hash * 1412 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash) 1413 { 1414 struct ftrace_func_entry *entry; 1415 struct ftrace_hash *new_hash; 1416 int size; 1417 int ret; 1418 int i; 1419 1420 new_hash = alloc_ftrace_hash(size_bits); 1421 if (!new_hash) 1422 return NULL; 1423 1424 if (hash) 1425 new_hash->flags = hash->flags; 1426 1427 /* Empty hash? */ 1428 if (ftrace_hash_empty(hash)) 1429 return new_hash; 1430 1431 size = 1 << hash->size_bits; 1432 for (i = 0; i < size; i++) { 1433 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 1434 ret = add_hash_entry(new_hash, entry->ip); 1435 if (ret < 0) 1436 goto free_hash; 1437 } 1438 } 1439 1440 FTRACE_WARN_ON(new_hash->count != hash->count); 1441 1442 return new_hash; 1443 1444 free_hash: 1445 free_ftrace_hash(new_hash); 1446 return NULL; 1447 } 1448 1449 static void 1450 ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash); 1451 static void 1452 ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash); 1453 1454 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, 1455 struct ftrace_hash *new_hash); 1456 1457 static struct ftrace_hash * 1458 __ftrace_hash_move(struct ftrace_hash *src) 1459 { 1460 struct ftrace_func_entry *entry; 1461 struct hlist_node *tn; 1462 struct hlist_head *hhd; 1463 struct ftrace_hash *new_hash; 1464 int size = src->count; 1465 int bits = 0; 1466 int i; 1467 1468 /* 1469 * If the new source is empty, just return the empty_hash. 1470 */ 1471 if (ftrace_hash_empty(src)) 1472 return EMPTY_HASH; 1473 1474 /* 1475 * Make the hash size about 1/2 the # found 1476 */ 1477 for (size /= 2; size; size >>= 1) 1478 bits++; 1479 1480 /* Don't allocate too much */ 1481 if (bits > FTRACE_HASH_MAX_BITS) 1482 bits = FTRACE_HASH_MAX_BITS; 1483 1484 new_hash = alloc_ftrace_hash(bits); 1485 if (!new_hash) 1486 return NULL; 1487 1488 new_hash->flags = src->flags; 1489 1490 size = 1 << src->size_bits; 1491 for (i = 0; i < size; i++) { 1492 hhd = &src->buckets[i]; 1493 hlist_for_each_entry_safe(entry, tn, hhd, hlist) { 1494 remove_hash_entry(src, entry); 1495 __add_hash_entry(new_hash, entry); 1496 } 1497 } 1498 1499 return new_hash; 1500 } 1501 1502 static int 1503 ftrace_hash_move(struct ftrace_ops *ops, int enable, 1504 struct ftrace_hash **dst, struct ftrace_hash *src) 1505 { 1506 struct ftrace_hash *new_hash; 1507 int ret; 1508 1509 /* Reject setting notrace hash on IPMODIFY ftrace_ops */ 1510 if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable) 1511 return -EINVAL; 1512 1513 new_hash = __ftrace_hash_move(src); 1514 if (!new_hash) 1515 return -ENOMEM; 1516 1517 /* Make sure this can be applied if it is IPMODIFY ftrace_ops */ 1518 if (enable) { 1519 /* IPMODIFY should be updated only when filter_hash updating */ 1520 ret = ftrace_hash_ipmodify_update(ops, new_hash); 1521 if (ret < 0) { 1522 free_ftrace_hash(new_hash); 1523 return ret; 1524 } 1525 } 1526 1527 /* 1528 * Remove the current set, update the hash and add 1529 * them back. 1530 */ 1531 ftrace_hash_rec_disable_modify(ops, enable); 1532 1533 rcu_assign_pointer(*dst, new_hash); 1534 1535 ftrace_hash_rec_enable_modify(ops, enable); 1536 1537 return 0; 1538 } 1539 1540 static bool hash_contains_ip(unsigned long ip, 1541 struct ftrace_ops_hash *hash) 1542 { 1543 /* 1544 * The function record is a match if it exists in the filter 1545 * hash and not in the notrace hash. Note, an emty hash is 1546 * considered a match for the filter hash, but an empty 1547 * notrace hash is considered not in the notrace hash. 1548 */ 1549 return (ftrace_hash_empty(hash->filter_hash) || 1550 __ftrace_lookup_ip(hash->filter_hash, ip)) && 1551 (ftrace_hash_empty(hash->notrace_hash) || 1552 !__ftrace_lookup_ip(hash->notrace_hash, ip)); 1553 } 1554 1555 /* 1556 * Test the hashes for this ops to see if we want to call 1557 * the ops->func or not. 1558 * 1559 * It's a match if the ip is in the ops->filter_hash or 1560 * the filter_hash does not exist or is empty, 1561 * AND 1562 * the ip is not in the ops->notrace_hash. 1563 * 1564 * This needs to be called with preemption disabled as 1565 * the hashes are freed with call_rcu_sched(). 1566 */ 1567 static int 1568 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) 1569 { 1570 struct ftrace_ops_hash hash; 1571 int ret; 1572 1573 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 1574 /* 1575 * There's a small race when adding ops that the ftrace handler 1576 * that wants regs, may be called without them. We can not 1577 * allow that handler to be called if regs is NULL. 1578 */ 1579 if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS)) 1580 return 0; 1581 #endif 1582 1583 rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash); 1584 rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash); 1585 1586 if (hash_contains_ip(ip, &hash)) 1587 ret = 1; 1588 else 1589 ret = 0; 1590 1591 return ret; 1592 } 1593 1594 /* 1595 * This is a double for. Do not use 'break' to break out of the loop, 1596 * you must use a goto. 1597 */ 1598 #define do_for_each_ftrace_rec(pg, rec) \ 1599 for (pg = ftrace_pages_start; pg; pg = pg->next) { \ 1600 int _____i; \ 1601 for (_____i = 0; _____i < pg->index; _____i++) { \ 1602 rec = &pg->records[_____i]; 1603 1604 #define while_for_each_ftrace_rec() \ 1605 } \ 1606 } 1607 1608 1609 static int ftrace_cmp_recs(const void *a, const void *b) 1610 { 1611 const struct dyn_ftrace *key = a; 1612 const struct dyn_ftrace *rec = b; 1613 1614 if (key->flags < rec->ip) 1615 return -1; 1616 if (key->ip >= rec->ip + MCOUNT_INSN_SIZE) 1617 return 1; 1618 return 0; 1619 } 1620 1621 /** 1622 * ftrace_location_range - return the first address of a traced location 1623 * if it touches the given ip range 1624 * @start: start of range to search. 1625 * @end: end of range to search (inclusive). @end points to the last byte 1626 * to check. 1627 * 1628 * Returns rec->ip if the related ftrace location is a least partly within 1629 * the given address range. That is, the first address of the instruction 1630 * that is either a NOP or call to the function tracer. It checks the ftrace 1631 * internal tables to determine if the address belongs or not. 1632 */ 1633 unsigned long ftrace_location_range(unsigned long start, unsigned long end) 1634 { 1635 struct ftrace_page *pg; 1636 struct dyn_ftrace *rec; 1637 struct dyn_ftrace key; 1638 1639 key.ip = start; 1640 key.flags = end; /* overload flags, as it is unsigned long */ 1641 1642 for (pg = ftrace_pages_start; pg; pg = pg->next) { 1643 if (end < pg->records[0].ip || 1644 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) 1645 continue; 1646 rec = bsearch(&key, pg->records, pg->index, 1647 sizeof(struct dyn_ftrace), 1648 ftrace_cmp_recs); 1649 if (rec) 1650 return rec->ip; 1651 } 1652 1653 return 0; 1654 } 1655 1656 /** 1657 * ftrace_location - return true if the ip giving is a traced location 1658 * @ip: the instruction pointer to check 1659 * 1660 * Returns rec->ip if @ip given is a pointer to a ftrace location. 1661 * That is, the instruction that is either a NOP or call to 1662 * the function tracer. It checks the ftrace internal tables to 1663 * determine if the address belongs or not. 1664 */ 1665 unsigned long ftrace_location(unsigned long ip) 1666 { 1667 return ftrace_location_range(ip, ip); 1668 } 1669 1670 /** 1671 * ftrace_text_reserved - return true if range contains an ftrace location 1672 * @start: start of range to search 1673 * @end: end of range to search (inclusive). @end points to the last byte to check. 1674 * 1675 * Returns 1 if @start and @end contains a ftrace location. 1676 * That is, the instruction that is either a NOP or call to 1677 * the function tracer. It checks the ftrace internal tables to 1678 * determine if the address belongs or not. 1679 */ 1680 int ftrace_text_reserved(const void *start, const void *end) 1681 { 1682 unsigned long ret; 1683 1684 ret = ftrace_location_range((unsigned long)start, 1685 (unsigned long)end); 1686 1687 return (int)!!ret; 1688 } 1689 1690 /* Test if ops registered to this rec needs regs */ 1691 static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec) 1692 { 1693 struct ftrace_ops *ops; 1694 bool keep_regs = false; 1695 1696 for (ops = ftrace_ops_list; 1697 ops != &ftrace_list_end; ops = ops->next) { 1698 /* pass rec in as regs to have non-NULL val */ 1699 if (ftrace_ops_test(ops, rec->ip, rec)) { 1700 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) { 1701 keep_regs = true; 1702 break; 1703 } 1704 } 1705 } 1706 1707 return keep_regs; 1708 } 1709 1710 static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, 1711 int filter_hash, 1712 bool inc) 1713 { 1714 struct ftrace_hash *hash; 1715 struct ftrace_hash *other_hash; 1716 struct ftrace_page *pg; 1717 struct dyn_ftrace *rec; 1718 bool update = false; 1719 int count = 0; 1720 int all = false; 1721 1722 /* Only update if the ops has been registered */ 1723 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 1724 return false; 1725 1726 /* 1727 * In the filter_hash case: 1728 * If the count is zero, we update all records. 1729 * Otherwise we just update the items in the hash. 1730 * 1731 * In the notrace_hash case: 1732 * We enable the update in the hash. 1733 * As disabling notrace means enabling the tracing, 1734 * and enabling notrace means disabling, the inc variable 1735 * gets inversed. 1736 */ 1737 if (filter_hash) { 1738 hash = ops->func_hash->filter_hash; 1739 other_hash = ops->func_hash->notrace_hash; 1740 if (ftrace_hash_empty(hash)) 1741 all = true; 1742 } else { 1743 inc = !inc; 1744 hash = ops->func_hash->notrace_hash; 1745 other_hash = ops->func_hash->filter_hash; 1746 /* 1747 * If the notrace hash has no items, 1748 * then there's nothing to do. 1749 */ 1750 if (ftrace_hash_empty(hash)) 1751 return false; 1752 } 1753 1754 do_for_each_ftrace_rec(pg, rec) { 1755 int in_other_hash = 0; 1756 int in_hash = 0; 1757 int match = 0; 1758 1759 if (rec->flags & FTRACE_FL_DISABLED) 1760 continue; 1761 1762 if (all) { 1763 /* 1764 * Only the filter_hash affects all records. 1765 * Update if the record is not in the notrace hash. 1766 */ 1767 if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip)) 1768 match = 1; 1769 } else { 1770 in_hash = !!ftrace_lookup_ip(hash, rec->ip); 1771 in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip); 1772 1773 /* 1774 * If filter_hash is set, we want to match all functions 1775 * that are in the hash but not in the other hash. 1776 * 1777 * If filter_hash is not set, then we are decrementing. 1778 * That means we match anything that is in the hash 1779 * and also in the other_hash. That is, we need to turn 1780 * off functions in the other hash because they are disabled 1781 * by this hash. 1782 */ 1783 if (filter_hash && in_hash && !in_other_hash) 1784 match = 1; 1785 else if (!filter_hash && in_hash && 1786 (in_other_hash || ftrace_hash_empty(other_hash))) 1787 match = 1; 1788 } 1789 if (!match) 1790 continue; 1791 1792 if (inc) { 1793 rec->flags++; 1794 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX)) 1795 return false; 1796 1797 /* 1798 * If there's only a single callback registered to a 1799 * function, and the ops has a trampoline registered 1800 * for it, then we can call it directly. 1801 */ 1802 if (ftrace_rec_count(rec) == 1 && ops->trampoline) 1803 rec->flags |= FTRACE_FL_TRAMP; 1804 else 1805 /* 1806 * If we are adding another function callback 1807 * to this function, and the previous had a 1808 * custom trampoline in use, then we need to go 1809 * back to the default trampoline. 1810 */ 1811 rec->flags &= ~FTRACE_FL_TRAMP; 1812 1813 /* 1814 * If any ops wants regs saved for this function 1815 * then all ops will get saved regs. 1816 */ 1817 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) 1818 rec->flags |= FTRACE_FL_REGS; 1819 } else { 1820 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0)) 1821 return false; 1822 rec->flags--; 1823 1824 /* 1825 * If the rec had REGS enabled and the ops that is 1826 * being removed had REGS set, then see if there is 1827 * still any ops for this record that wants regs. 1828 * If not, we can stop recording them. 1829 */ 1830 if (ftrace_rec_count(rec) > 0 && 1831 rec->flags & FTRACE_FL_REGS && 1832 ops->flags & FTRACE_OPS_FL_SAVE_REGS) { 1833 if (!test_rec_ops_needs_regs(rec)) 1834 rec->flags &= ~FTRACE_FL_REGS; 1835 } 1836 1837 /* 1838 * If the rec had TRAMP enabled, then it needs to 1839 * be cleared. As TRAMP can only be enabled iff 1840 * there is only a single ops attached to it. 1841 * In otherwords, always disable it on decrementing. 1842 * In the future, we may set it if rec count is 1843 * decremented to one, and the ops that is left 1844 * has a trampoline. 1845 */ 1846 rec->flags &= ~FTRACE_FL_TRAMP; 1847 1848 /* 1849 * flags will be cleared in ftrace_check_record() 1850 * if rec count is zero. 1851 */ 1852 } 1853 count++; 1854 1855 /* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */ 1856 update |= ftrace_test_record(rec, 1) != FTRACE_UPDATE_IGNORE; 1857 1858 /* Shortcut, if we handled all records, we are done. */ 1859 if (!all && count == hash->count) 1860 return update; 1861 } while_for_each_ftrace_rec(); 1862 1863 return update; 1864 } 1865 1866 static bool ftrace_hash_rec_disable(struct ftrace_ops *ops, 1867 int filter_hash) 1868 { 1869 return __ftrace_hash_rec_update(ops, filter_hash, 0); 1870 } 1871 1872 static bool ftrace_hash_rec_enable(struct ftrace_ops *ops, 1873 int filter_hash) 1874 { 1875 return __ftrace_hash_rec_update(ops, filter_hash, 1); 1876 } 1877 1878 static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, 1879 int filter_hash, int inc) 1880 { 1881 struct ftrace_ops *op; 1882 1883 __ftrace_hash_rec_update(ops, filter_hash, inc); 1884 1885 if (ops->func_hash != &global_ops.local_hash) 1886 return; 1887 1888 /* 1889 * If the ops shares the global_ops hash, then we need to update 1890 * all ops that are enabled and use this hash. 1891 */ 1892 do_for_each_ftrace_op(op, ftrace_ops_list) { 1893 /* Already done */ 1894 if (op == ops) 1895 continue; 1896 if (op->func_hash == &global_ops.local_hash) 1897 __ftrace_hash_rec_update(op, filter_hash, inc); 1898 } while_for_each_ftrace_op(op); 1899 } 1900 1901 static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, 1902 int filter_hash) 1903 { 1904 ftrace_hash_rec_update_modify(ops, filter_hash, 0); 1905 } 1906 1907 static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, 1908 int filter_hash) 1909 { 1910 ftrace_hash_rec_update_modify(ops, filter_hash, 1); 1911 } 1912 1913 /* 1914 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK 1915 * or no-needed to update, -EBUSY if it detects a conflict of the flag 1916 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs. 1917 * Note that old_hash and new_hash has below meanings 1918 * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected) 1919 * - If the hash is EMPTY_HASH, it hits nothing 1920 * - Anything else hits the recs which match the hash entries. 1921 */ 1922 static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops, 1923 struct ftrace_hash *old_hash, 1924 struct ftrace_hash *new_hash) 1925 { 1926 struct ftrace_page *pg; 1927 struct dyn_ftrace *rec, *end = NULL; 1928 int in_old, in_new; 1929 1930 /* Only update if the ops has been registered */ 1931 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 1932 return 0; 1933 1934 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) 1935 return 0; 1936 1937 /* 1938 * Since the IPMODIFY is a very address sensitive action, we do not 1939 * allow ftrace_ops to set all functions to new hash. 1940 */ 1941 if (!new_hash || !old_hash) 1942 return -EINVAL; 1943 1944 /* Update rec->flags */ 1945 do_for_each_ftrace_rec(pg, rec) { 1946 1947 if (rec->flags & FTRACE_FL_DISABLED) 1948 continue; 1949 1950 /* We need to update only differences of filter_hash */ 1951 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); 1952 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); 1953 if (in_old == in_new) 1954 continue; 1955 1956 if (in_new) { 1957 /* New entries must ensure no others are using it */ 1958 if (rec->flags & FTRACE_FL_IPMODIFY) 1959 goto rollback; 1960 rec->flags |= FTRACE_FL_IPMODIFY; 1961 } else /* Removed entry */ 1962 rec->flags &= ~FTRACE_FL_IPMODIFY; 1963 } while_for_each_ftrace_rec(); 1964 1965 return 0; 1966 1967 rollback: 1968 end = rec; 1969 1970 /* Roll back what we did above */ 1971 do_for_each_ftrace_rec(pg, rec) { 1972 1973 if (rec->flags & FTRACE_FL_DISABLED) 1974 continue; 1975 1976 if (rec == end) 1977 goto err_out; 1978 1979 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); 1980 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); 1981 if (in_old == in_new) 1982 continue; 1983 1984 if (in_new) 1985 rec->flags &= ~FTRACE_FL_IPMODIFY; 1986 else 1987 rec->flags |= FTRACE_FL_IPMODIFY; 1988 } while_for_each_ftrace_rec(); 1989 1990 err_out: 1991 return -EBUSY; 1992 } 1993 1994 static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops) 1995 { 1996 struct ftrace_hash *hash = ops->func_hash->filter_hash; 1997 1998 if (ftrace_hash_empty(hash)) 1999 hash = NULL; 2000 2001 return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash); 2002 } 2003 2004 /* Disabling always succeeds */ 2005 static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops) 2006 { 2007 struct ftrace_hash *hash = ops->func_hash->filter_hash; 2008 2009 if (ftrace_hash_empty(hash)) 2010 hash = NULL; 2011 2012 __ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH); 2013 } 2014 2015 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, 2016 struct ftrace_hash *new_hash) 2017 { 2018 struct ftrace_hash *old_hash = ops->func_hash->filter_hash; 2019 2020 if (ftrace_hash_empty(old_hash)) 2021 old_hash = NULL; 2022 2023 if (ftrace_hash_empty(new_hash)) 2024 new_hash = NULL; 2025 2026 return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash); 2027 } 2028 2029 static void print_ip_ins(const char *fmt, const unsigned char *p) 2030 { 2031 int i; 2032 2033 printk(KERN_CONT "%s", fmt); 2034 2035 for (i = 0; i < MCOUNT_INSN_SIZE; i++) 2036 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]); 2037 } 2038 2039 static struct ftrace_ops * 2040 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec); 2041 static struct ftrace_ops * 2042 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops); 2043 2044 enum ftrace_bug_type ftrace_bug_type; 2045 const void *ftrace_expected; 2046 2047 static void print_bug_type(void) 2048 { 2049 switch (ftrace_bug_type) { 2050 case FTRACE_BUG_UNKNOWN: 2051 break; 2052 case FTRACE_BUG_INIT: 2053 pr_info("Initializing ftrace call sites\n"); 2054 break; 2055 case FTRACE_BUG_NOP: 2056 pr_info("Setting ftrace call site to NOP\n"); 2057 break; 2058 case FTRACE_BUG_CALL: 2059 pr_info("Setting ftrace call site to call ftrace function\n"); 2060 break; 2061 case FTRACE_BUG_UPDATE: 2062 pr_info("Updating ftrace call site to call a different ftrace function\n"); 2063 break; 2064 } 2065 } 2066 2067 /** 2068 * ftrace_bug - report and shutdown function tracer 2069 * @failed: The failed type (EFAULT, EINVAL, EPERM) 2070 * @rec: The record that failed 2071 * 2072 * The arch code that enables or disables the function tracing 2073 * can call ftrace_bug() when it has detected a problem in 2074 * modifying the code. @failed should be one of either: 2075 * EFAULT - if the problem happens on reading the @ip address 2076 * EINVAL - if what is read at @ip is not what was expected 2077 * EPERM - if the problem happens on writting to the @ip address 2078 */ 2079 void ftrace_bug(int failed, struct dyn_ftrace *rec) 2080 { 2081 unsigned long ip = rec ? rec->ip : 0; 2082 2083 switch (failed) { 2084 case -EFAULT: 2085 FTRACE_WARN_ON_ONCE(1); 2086 pr_info("ftrace faulted on modifying "); 2087 print_ip_sym(ip); 2088 break; 2089 case -EINVAL: 2090 FTRACE_WARN_ON_ONCE(1); 2091 pr_info("ftrace failed to modify "); 2092 print_ip_sym(ip); 2093 print_ip_ins(" actual: ", (unsigned char *)ip); 2094 pr_cont("\n"); 2095 if (ftrace_expected) { 2096 print_ip_ins(" expected: ", ftrace_expected); 2097 pr_cont("\n"); 2098 } 2099 break; 2100 case -EPERM: 2101 FTRACE_WARN_ON_ONCE(1); 2102 pr_info("ftrace faulted on writing "); 2103 print_ip_sym(ip); 2104 break; 2105 default: 2106 FTRACE_WARN_ON_ONCE(1); 2107 pr_info("ftrace faulted on unknown error "); 2108 print_ip_sym(ip); 2109 } 2110 print_bug_type(); 2111 if (rec) { 2112 struct ftrace_ops *ops = NULL; 2113 2114 pr_info("ftrace record flags: %lx\n", rec->flags); 2115 pr_cont(" (%ld)%s", ftrace_rec_count(rec), 2116 rec->flags & FTRACE_FL_REGS ? " R" : " "); 2117 if (rec->flags & FTRACE_FL_TRAMP_EN) { 2118 ops = ftrace_find_tramp_ops_any(rec); 2119 if (ops) { 2120 do { 2121 pr_cont("\ttramp: %pS (%pS)", 2122 (void *)ops->trampoline, 2123 (void *)ops->func); 2124 ops = ftrace_find_tramp_ops_next(rec, ops); 2125 } while (ops); 2126 } else 2127 pr_cont("\ttramp: ERROR!"); 2128 2129 } 2130 ip = ftrace_get_addr_curr(rec); 2131 pr_cont("\n expected tramp: %lx\n", ip); 2132 } 2133 } 2134 2135 static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update) 2136 { 2137 unsigned long flag = 0UL; 2138 2139 ftrace_bug_type = FTRACE_BUG_UNKNOWN; 2140 2141 if (rec->flags & FTRACE_FL_DISABLED) 2142 return FTRACE_UPDATE_IGNORE; 2143 2144 /* 2145 * If we are updating calls: 2146 * 2147 * If the record has a ref count, then we need to enable it 2148 * because someone is using it. 2149 * 2150 * Otherwise we make sure its disabled. 2151 * 2152 * If we are disabling calls, then disable all records that 2153 * are enabled. 2154 */ 2155 if (enable && ftrace_rec_count(rec)) 2156 flag = FTRACE_FL_ENABLED; 2157 2158 /* 2159 * If enabling and the REGS flag does not match the REGS_EN, or 2160 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore 2161 * this record. Set flags to fail the compare against ENABLED. 2162 */ 2163 if (flag) { 2164 if (!(rec->flags & FTRACE_FL_REGS) != 2165 !(rec->flags & FTRACE_FL_REGS_EN)) 2166 flag |= FTRACE_FL_REGS; 2167 2168 if (!(rec->flags & FTRACE_FL_TRAMP) != 2169 !(rec->flags & FTRACE_FL_TRAMP_EN)) 2170 flag |= FTRACE_FL_TRAMP; 2171 } 2172 2173 /* If the state of this record hasn't changed, then do nothing */ 2174 if ((rec->flags & FTRACE_FL_ENABLED) == flag) 2175 return FTRACE_UPDATE_IGNORE; 2176 2177 if (flag) { 2178 /* Save off if rec is being enabled (for return value) */ 2179 flag ^= rec->flags & FTRACE_FL_ENABLED; 2180 2181 if (update) { 2182 rec->flags |= FTRACE_FL_ENABLED; 2183 if (flag & FTRACE_FL_REGS) { 2184 if (rec->flags & FTRACE_FL_REGS) 2185 rec->flags |= FTRACE_FL_REGS_EN; 2186 else 2187 rec->flags &= ~FTRACE_FL_REGS_EN; 2188 } 2189 if (flag & FTRACE_FL_TRAMP) { 2190 if (rec->flags & FTRACE_FL_TRAMP) 2191 rec->flags |= FTRACE_FL_TRAMP_EN; 2192 else 2193 rec->flags &= ~FTRACE_FL_TRAMP_EN; 2194 } 2195 } 2196 2197 /* 2198 * If this record is being updated from a nop, then 2199 * return UPDATE_MAKE_CALL. 2200 * Otherwise, 2201 * return UPDATE_MODIFY_CALL to tell the caller to convert 2202 * from the save regs, to a non-save regs function or 2203 * vice versa, or from a trampoline call. 2204 */ 2205 if (flag & FTRACE_FL_ENABLED) { 2206 ftrace_bug_type = FTRACE_BUG_CALL; 2207 return FTRACE_UPDATE_MAKE_CALL; 2208 } 2209 2210 ftrace_bug_type = FTRACE_BUG_UPDATE; 2211 return FTRACE_UPDATE_MODIFY_CALL; 2212 } 2213 2214 if (update) { 2215 /* If there's no more users, clear all flags */ 2216 if (!ftrace_rec_count(rec)) 2217 rec->flags = 0; 2218 else 2219 /* 2220 * Just disable the record, but keep the ops TRAMP 2221 * and REGS states. The _EN flags must be disabled though. 2222 */ 2223 rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN | 2224 FTRACE_FL_REGS_EN); 2225 } 2226 2227 ftrace_bug_type = FTRACE_BUG_NOP; 2228 return FTRACE_UPDATE_MAKE_NOP; 2229 } 2230 2231 /** 2232 * ftrace_update_record, set a record that now is tracing or not 2233 * @rec: the record to update 2234 * @enable: set to 1 if the record is tracing, zero to force disable 2235 * 2236 * The records that represent all functions that can be traced need 2237 * to be updated when tracing has been enabled. 2238 */ 2239 int ftrace_update_record(struct dyn_ftrace *rec, int enable) 2240 { 2241 return ftrace_check_record(rec, enable, 1); 2242 } 2243 2244 /** 2245 * ftrace_test_record, check if the record has been enabled or not 2246 * @rec: the record to test 2247 * @enable: set to 1 to check if enabled, 0 if it is disabled 2248 * 2249 * The arch code may need to test if a record is already set to 2250 * tracing to determine how to modify the function code that it 2251 * represents. 2252 */ 2253 int ftrace_test_record(struct dyn_ftrace *rec, int enable) 2254 { 2255 return ftrace_check_record(rec, enable, 0); 2256 } 2257 2258 static struct ftrace_ops * 2259 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec) 2260 { 2261 struct ftrace_ops *op; 2262 unsigned long ip = rec->ip; 2263 2264 do_for_each_ftrace_op(op, ftrace_ops_list) { 2265 2266 if (!op->trampoline) 2267 continue; 2268 2269 if (hash_contains_ip(ip, op->func_hash)) 2270 return op; 2271 } while_for_each_ftrace_op(op); 2272 2273 return NULL; 2274 } 2275 2276 static struct ftrace_ops * 2277 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, 2278 struct ftrace_ops *op) 2279 { 2280 unsigned long ip = rec->ip; 2281 2282 while_for_each_ftrace_op(op) { 2283 2284 if (!op->trampoline) 2285 continue; 2286 2287 if (hash_contains_ip(ip, op->func_hash)) 2288 return op; 2289 } 2290 2291 return NULL; 2292 } 2293 2294 static struct ftrace_ops * 2295 ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec) 2296 { 2297 struct ftrace_ops *op; 2298 unsigned long ip = rec->ip; 2299 2300 /* 2301 * Need to check removed ops first. 2302 * If they are being removed, and this rec has a tramp, 2303 * and this rec is in the ops list, then it would be the 2304 * one with the tramp. 2305 */ 2306 if (removed_ops) { 2307 if (hash_contains_ip(ip, &removed_ops->old_hash)) 2308 return removed_ops; 2309 } 2310 2311 /* 2312 * Need to find the current trampoline for a rec. 2313 * Now, a trampoline is only attached to a rec if there 2314 * was a single 'ops' attached to it. But this can be called 2315 * when we are adding another op to the rec or removing the 2316 * current one. Thus, if the op is being added, we can 2317 * ignore it because it hasn't attached itself to the rec 2318 * yet. 2319 * 2320 * If an ops is being modified (hooking to different functions) 2321 * then we don't care about the new functions that are being 2322 * added, just the old ones (that are probably being removed). 2323 * 2324 * If we are adding an ops to a function that already is using 2325 * a trampoline, it needs to be removed (trampolines are only 2326 * for single ops connected), then an ops that is not being 2327 * modified also needs to be checked. 2328 */ 2329 do_for_each_ftrace_op(op, ftrace_ops_list) { 2330 2331 if (!op->trampoline) 2332 continue; 2333 2334 /* 2335 * If the ops is being added, it hasn't gotten to 2336 * the point to be removed from this tree yet. 2337 */ 2338 if (op->flags & FTRACE_OPS_FL_ADDING) 2339 continue; 2340 2341 2342 /* 2343 * If the ops is being modified and is in the old 2344 * hash, then it is probably being removed from this 2345 * function. 2346 */ 2347 if ((op->flags & FTRACE_OPS_FL_MODIFYING) && 2348 hash_contains_ip(ip, &op->old_hash)) 2349 return op; 2350 /* 2351 * If the ops is not being added or modified, and it's 2352 * in its normal filter hash, then this must be the one 2353 * we want! 2354 */ 2355 if (!(op->flags & FTRACE_OPS_FL_MODIFYING) && 2356 hash_contains_ip(ip, op->func_hash)) 2357 return op; 2358 2359 } while_for_each_ftrace_op(op); 2360 2361 return NULL; 2362 } 2363 2364 static struct ftrace_ops * 2365 ftrace_find_tramp_ops_new(struct dyn_ftrace *rec) 2366 { 2367 struct ftrace_ops *op; 2368 unsigned long ip = rec->ip; 2369 2370 do_for_each_ftrace_op(op, ftrace_ops_list) { 2371 /* pass rec in as regs to have non-NULL val */ 2372 if (hash_contains_ip(ip, op->func_hash)) 2373 return op; 2374 } while_for_each_ftrace_op(op); 2375 2376 return NULL; 2377 } 2378 2379 /** 2380 * ftrace_get_addr_new - Get the call address to set to 2381 * @rec: The ftrace record descriptor 2382 * 2383 * If the record has the FTRACE_FL_REGS set, that means that it 2384 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS 2385 * is not not set, then it wants to convert to the normal callback. 2386 * 2387 * Returns the address of the trampoline to set to 2388 */ 2389 unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec) 2390 { 2391 struct ftrace_ops *ops; 2392 2393 /* Trampolines take precedence over regs */ 2394 if (rec->flags & FTRACE_FL_TRAMP) { 2395 ops = ftrace_find_tramp_ops_new(rec); 2396 if (FTRACE_WARN_ON(!ops || !ops->trampoline)) { 2397 pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n", 2398 (void *)rec->ip, (void *)rec->ip, rec->flags); 2399 /* Ftrace is shutting down, return anything */ 2400 return (unsigned long)FTRACE_ADDR; 2401 } 2402 return ops->trampoline; 2403 } 2404 2405 if (rec->flags & FTRACE_FL_REGS) 2406 return (unsigned long)FTRACE_REGS_ADDR; 2407 else 2408 return (unsigned long)FTRACE_ADDR; 2409 } 2410 2411 /** 2412 * ftrace_get_addr_curr - Get the call address that is already there 2413 * @rec: The ftrace record descriptor 2414 * 2415 * The FTRACE_FL_REGS_EN is set when the record already points to 2416 * a function that saves all the regs. Basically the '_EN' version 2417 * represents the current state of the function. 2418 * 2419 * Returns the address of the trampoline that is currently being called 2420 */ 2421 unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec) 2422 { 2423 struct ftrace_ops *ops; 2424 2425 /* Trampolines take precedence over regs */ 2426 if (rec->flags & FTRACE_FL_TRAMP_EN) { 2427 ops = ftrace_find_tramp_ops_curr(rec); 2428 if (FTRACE_WARN_ON(!ops)) { 2429 pr_warn("Bad trampoline accounting at: %p (%pS)\n", 2430 (void *)rec->ip, (void *)rec->ip); 2431 /* Ftrace is shutting down, return anything */ 2432 return (unsigned long)FTRACE_ADDR; 2433 } 2434 return ops->trampoline; 2435 } 2436 2437 if (rec->flags & FTRACE_FL_REGS_EN) 2438 return (unsigned long)FTRACE_REGS_ADDR; 2439 else 2440 return (unsigned long)FTRACE_ADDR; 2441 } 2442 2443 static int 2444 __ftrace_replace_code(struct dyn_ftrace *rec, int enable) 2445 { 2446 unsigned long ftrace_old_addr; 2447 unsigned long ftrace_addr; 2448 int ret; 2449 2450 ftrace_addr = ftrace_get_addr_new(rec); 2451 2452 /* This needs to be done before we call ftrace_update_record */ 2453 ftrace_old_addr = ftrace_get_addr_curr(rec); 2454 2455 ret = ftrace_update_record(rec, enable); 2456 2457 ftrace_bug_type = FTRACE_BUG_UNKNOWN; 2458 2459 switch (ret) { 2460 case FTRACE_UPDATE_IGNORE: 2461 return 0; 2462 2463 case FTRACE_UPDATE_MAKE_CALL: 2464 ftrace_bug_type = FTRACE_BUG_CALL; 2465 return ftrace_make_call(rec, ftrace_addr); 2466 2467 case FTRACE_UPDATE_MAKE_NOP: 2468 ftrace_bug_type = FTRACE_BUG_NOP; 2469 return ftrace_make_nop(NULL, rec, ftrace_old_addr); 2470 2471 case FTRACE_UPDATE_MODIFY_CALL: 2472 ftrace_bug_type = FTRACE_BUG_UPDATE; 2473 return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr); 2474 } 2475 2476 return -1; /* unknow ftrace bug */ 2477 } 2478 2479 void __weak ftrace_replace_code(int enable) 2480 { 2481 struct dyn_ftrace *rec; 2482 struct ftrace_page *pg; 2483 int failed; 2484 2485 if (unlikely(ftrace_disabled)) 2486 return; 2487 2488 do_for_each_ftrace_rec(pg, rec) { 2489 2490 if (rec->flags & FTRACE_FL_DISABLED) 2491 continue; 2492 2493 failed = __ftrace_replace_code(rec, enable); 2494 if (failed) { 2495 ftrace_bug(failed, rec); 2496 /* Stop processing */ 2497 return; 2498 } 2499 } while_for_each_ftrace_rec(); 2500 } 2501 2502 struct ftrace_rec_iter { 2503 struct ftrace_page *pg; 2504 int index; 2505 }; 2506 2507 /** 2508 * ftrace_rec_iter_start, start up iterating over traced functions 2509 * 2510 * Returns an iterator handle that is used to iterate over all 2511 * the records that represent address locations where functions 2512 * are traced. 2513 * 2514 * May return NULL if no records are available. 2515 */ 2516 struct ftrace_rec_iter *ftrace_rec_iter_start(void) 2517 { 2518 /* 2519 * We only use a single iterator. 2520 * Protected by the ftrace_lock mutex. 2521 */ 2522 static struct ftrace_rec_iter ftrace_rec_iter; 2523 struct ftrace_rec_iter *iter = &ftrace_rec_iter; 2524 2525 iter->pg = ftrace_pages_start; 2526 iter->index = 0; 2527 2528 /* Could have empty pages */ 2529 while (iter->pg && !iter->pg->index) 2530 iter->pg = iter->pg->next; 2531 2532 if (!iter->pg) 2533 return NULL; 2534 2535 return iter; 2536 } 2537 2538 /** 2539 * ftrace_rec_iter_next, get the next record to process. 2540 * @iter: The handle to the iterator. 2541 * 2542 * Returns the next iterator after the given iterator @iter. 2543 */ 2544 struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter) 2545 { 2546 iter->index++; 2547 2548 if (iter->index >= iter->pg->index) { 2549 iter->pg = iter->pg->next; 2550 iter->index = 0; 2551 2552 /* Could have empty pages */ 2553 while (iter->pg && !iter->pg->index) 2554 iter->pg = iter->pg->next; 2555 } 2556 2557 if (!iter->pg) 2558 return NULL; 2559 2560 return iter; 2561 } 2562 2563 /** 2564 * ftrace_rec_iter_record, get the record at the iterator location 2565 * @iter: The current iterator location 2566 * 2567 * Returns the record that the current @iter is at. 2568 */ 2569 struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter) 2570 { 2571 return &iter->pg->records[iter->index]; 2572 } 2573 2574 static int 2575 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec) 2576 { 2577 int ret; 2578 2579 if (unlikely(ftrace_disabled)) 2580 return 0; 2581 2582 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR); 2583 if (ret) { 2584 ftrace_bug_type = FTRACE_BUG_INIT; 2585 ftrace_bug(ret, rec); 2586 return 0; 2587 } 2588 return 1; 2589 } 2590 2591 /* 2592 * archs can override this function if they must do something 2593 * before the modifying code is performed. 2594 */ 2595 int __weak ftrace_arch_code_modify_prepare(void) 2596 { 2597 return 0; 2598 } 2599 2600 /* 2601 * archs can override this function if they must do something 2602 * after the modifying code is performed. 2603 */ 2604 int __weak ftrace_arch_code_modify_post_process(void) 2605 { 2606 return 0; 2607 } 2608 2609 void ftrace_modify_all_code(int command) 2610 { 2611 int update = command & FTRACE_UPDATE_TRACE_FUNC; 2612 int err = 0; 2613 2614 /* 2615 * If the ftrace_caller calls a ftrace_ops func directly, 2616 * we need to make sure that it only traces functions it 2617 * expects to trace. When doing the switch of functions, 2618 * we need to update to the ftrace_ops_list_func first 2619 * before the transition between old and new calls are set, 2620 * as the ftrace_ops_list_func will check the ops hashes 2621 * to make sure the ops are having the right functions 2622 * traced. 2623 */ 2624 if (update) { 2625 err = ftrace_update_ftrace_func(ftrace_ops_list_func); 2626 if (FTRACE_WARN_ON(err)) 2627 return; 2628 } 2629 2630 if (command & FTRACE_UPDATE_CALLS) 2631 ftrace_replace_code(1); 2632 else if (command & FTRACE_DISABLE_CALLS) 2633 ftrace_replace_code(0); 2634 2635 if (update && ftrace_trace_function != ftrace_ops_list_func) { 2636 function_trace_op = set_function_trace_op; 2637 smp_wmb(); 2638 /* If irqs are disabled, we are in stop machine */ 2639 if (!irqs_disabled()) 2640 smp_call_function(ftrace_sync_ipi, NULL, 1); 2641 err = ftrace_update_ftrace_func(ftrace_trace_function); 2642 if (FTRACE_WARN_ON(err)) 2643 return; 2644 } 2645 2646 if (command & FTRACE_START_FUNC_RET) 2647 err = ftrace_enable_ftrace_graph_caller(); 2648 else if (command & FTRACE_STOP_FUNC_RET) 2649 err = ftrace_disable_ftrace_graph_caller(); 2650 FTRACE_WARN_ON(err); 2651 } 2652 2653 static int __ftrace_modify_code(void *data) 2654 { 2655 int *command = data; 2656 2657 ftrace_modify_all_code(*command); 2658 2659 return 0; 2660 } 2661 2662 /** 2663 * ftrace_run_stop_machine, go back to the stop machine method 2664 * @command: The command to tell ftrace what to do 2665 * 2666 * If an arch needs to fall back to the stop machine method, the 2667 * it can call this function. 2668 */ 2669 void ftrace_run_stop_machine(int command) 2670 { 2671 stop_machine(__ftrace_modify_code, &command, NULL); 2672 } 2673 2674 /** 2675 * arch_ftrace_update_code, modify the code to trace or not trace 2676 * @command: The command that needs to be done 2677 * 2678 * Archs can override this function if it does not need to 2679 * run stop_machine() to modify code. 2680 */ 2681 void __weak arch_ftrace_update_code(int command) 2682 { 2683 ftrace_run_stop_machine(command); 2684 } 2685 2686 static void ftrace_run_update_code(int command) 2687 { 2688 int ret; 2689 2690 ret = ftrace_arch_code_modify_prepare(); 2691 FTRACE_WARN_ON(ret); 2692 if (ret) 2693 return; 2694 2695 /* 2696 * By default we use stop_machine() to modify the code. 2697 * But archs can do what ever they want as long as it 2698 * is safe. The stop_machine() is the safest, but also 2699 * produces the most overhead. 2700 */ 2701 arch_ftrace_update_code(command); 2702 2703 ret = ftrace_arch_code_modify_post_process(); 2704 FTRACE_WARN_ON(ret); 2705 } 2706 2707 static void ftrace_run_modify_code(struct ftrace_ops *ops, int command, 2708 struct ftrace_ops_hash *old_hash) 2709 { 2710 ops->flags |= FTRACE_OPS_FL_MODIFYING; 2711 ops->old_hash.filter_hash = old_hash->filter_hash; 2712 ops->old_hash.notrace_hash = old_hash->notrace_hash; 2713 ftrace_run_update_code(command); 2714 ops->old_hash.filter_hash = NULL; 2715 ops->old_hash.notrace_hash = NULL; 2716 ops->flags &= ~FTRACE_OPS_FL_MODIFYING; 2717 } 2718 2719 static ftrace_func_t saved_ftrace_func; 2720 static int ftrace_start_up; 2721 2722 void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops) 2723 { 2724 } 2725 2726 static void per_cpu_ops_free(struct ftrace_ops *ops) 2727 { 2728 free_percpu(ops->disabled); 2729 } 2730 2731 static void ftrace_startup_enable(int command) 2732 { 2733 if (saved_ftrace_func != ftrace_trace_function) { 2734 saved_ftrace_func = ftrace_trace_function; 2735 command |= FTRACE_UPDATE_TRACE_FUNC; 2736 } 2737 2738 if (!command || !ftrace_enabled) 2739 return; 2740 2741 ftrace_run_update_code(command); 2742 } 2743 2744 static void ftrace_startup_all(int command) 2745 { 2746 update_all_ops = true; 2747 ftrace_startup_enable(command); 2748 update_all_ops = false; 2749 } 2750 2751 static int ftrace_startup(struct ftrace_ops *ops, int command) 2752 { 2753 int ret; 2754 2755 if (unlikely(ftrace_disabled)) 2756 return -ENODEV; 2757 2758 ret = __register_ftrace_function(ops); 2759 if (ret) 2760 return ret; 2761 2762 ftrace_start_up++; 2763 2764 /* 2765 * Note that ftrace probes uses this to start up 2766 * and modify functions it will probe. But we still 2767 * set the ADDING flag for modification, as probes 2768 * do not have trampolines. If they add them in the 2769 * future, then the probes will need to distinguish 2770 * between adding and updating probes. 2771 */ 2772 ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING; 2773 2774 ret = ftrace_hash_ipmodify_enable(ops); 2775 if (ret < 0) { 2776 /* Rollback registration process */ 2777 __unregister_ftrace_function(ops); 2778 ftrace_start_up--; 2779 ops->flags &= ~FTRACE_OPS_FL_ENABLED; 2780 return ret; 2781 } 2782 2783 if (ftrace_hash_rec_enable(ops, 1)) 2784 command |= FTRACE_UPDATE_CALLS; 2785 2786 ftrace_startup_enable(command); 2787 2788 ops->flags &= ~FTRACE_OPS_FL_ADDING; 2789 2790 return 0; 2791 } 2792 2793 static int ftrace_shutdown(struct ftrace_ops *ops, int command) 2794 { 2795 int ret; 2796 2797 if (unlikely(ftrace_disabled)) 2798 return -ENODEV; 2799 2800 ret = __unregister_ftrace_function(ops); 2801 if (ret) 2802 return ret; 2803 2804 ftrace_start_up--; 2805 /* 2806 * Just warn in case of unbalance, no need to kill ftrace, it's not 2807 * critical but the ftrace_call callers may be never nopped again after 2808 * further ftrace uses. 2809 */ 2810 WARN_ON_ONCE(ftrace_start_up < 0); 2811 2812 /* Disabling ipmodify never fails */ 2813 ftrace_hash_ipmodify_disable(ops); 2814 2815 if (ftrace_hash_rec_disable(ops, 1)) 2816 command |= FTRACE_UPDATE_CALLS; 2817 2818 ops->flags &= ~FTRACE_OPS_FL_ENABLED; 2819 2820 if (saved_ftrace_func != ftrace_trace_function) { 2821 saved_ftrace_func = ftrace_trace_function; 2822 command |= FTRACE_UPDATE_TRACE_FUNC; 2823 } 2824 2825 if (!command || !ftrace_enabled) { 2826 /* 2827 * If these are per_cpu ops, they still need their 2828 * per_cpu field freed. Since, function tracing is 2829 * not currently active, we can just free them 2830 * without synchronizing all CPUs. 2831 */ 2832 if (ops->flags & FTRACE_OPS_FL_PER_CPU) 2833 per_cpu_ops_free(ops); 2834 return 0; 2835 } 2836 2837 /* 2838 * If the ops uses a trampoline, then it needs to be 2839 * tested first on update. 2840 */ 2841 ops->flags |= FTRACE_OPS_FL_REMOVING; 2842 removed_ops = ops; 2843 2844 /* The trampoline logic checks the old hashes */ 2845 ops->old_hash.filter_hash = ops->func_hash->filter_hash; 2846 ops->old_hash.notrace_hash = ops->func_hash->notrace_hash; 2847 2848 ftrace_run_update_code(command); 2849 2850 /* 2851 * If there's no more ops registered with ftrace, run a 2852 * sanity check to make sure all rec flags are cleared. 2853 */ 2854 if (rcu_dereference_protected(ftrace_ops_list, 2855 lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) { 2856 struct ftrace_page *pg; 2857 struct dyn_ftrace *rec; 2858 2859 do_for_each_ftrace_rec(pg, rec) { 2860 if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_FL_DISABLED)) 2861 pr_warn(" %pS flags:%lx\n", 2862 (void *)rec->ip, rec->flags); 2863 } while_for_each_ftrace_rec(); 2864 } 2865 2866 ops->old_hash.filter_hash = NULL; 2867 ops->old_hash.notrace_hash = NULL; 2868 2869 removed_ops = NULL; 2870 ops->flags &= ~FTRACE_OPS_FL_REMOVING; 2871 2872 /* 2873 * Dynamic ops may be freed, we must make sure that all 2874 * callers are done before leaving this function. 2875 * The same goes for freeing the per_cpu data of the per_cpu 2876 * ops. 2877 */ 2878 if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU)) { 2879 /* 2880 * We need to do a hard force of sched synchronization. 2881 * This is because we use preempt_disable() to do RCU, but 2882 * the function tracers can be called where RCU is not watching 2883 * (like before user_exit()). We can not rely on the RCU 2884 * infrastructure to do the synchronization, thus we must do it 2885 * ourselves. 2886 */ 2887 schedule_on_each_cpu(ftrace_sync); 2888 2889 /* 2890 * When the kernel is preeptive, tasks can be preempted 2891 * while on a ftrace trampoline. Just scheduling a task on 2892 * a CPU is not good enough to flush them. Calling 2893 * synchornize_rcu_tasks() will wait for those tasks to 2894 * execute and either schedule voluntarily or enter user space. 2895 */ 2896 if (IS_ENABLED(CONFIG_PREEMPT)) 2897 synchronize_rcu_tasks(); 2898 2899 arch_ftrace_trampoline_free(ops); 2900 2901 if (ops->flags & FTRACE_OPS_FL_PER_CPU) 2902 per_cpu_ops_free(ops); 2903 } 2904 2905 return 0; 2906 } 2907 2908 static void ftrace_startup_sysctl(void) 2909 { 2910 int command; 2911 2912 if (unlikely(ftrace_disabled)) 2913 return; 2914 2915 /* Force update next time */ 2916 saved_ftrace_func = NULL; 2917 /* ftrace_start_up is true if we want ftrace running */ 2918 if (ftrace_start_up) { 2919 command = FTRACE_UPDATE_CALLS; 2920 if (ftrace_graph_active) 2921 command |= FTRACE_START_FUNC_RET; 2922 ftrace_startup_enable(command); 2923 } 2924 } 2925 2926 static void ftrace_shutdown_sysctl(void) 2927 { 2928 int command; 2929 2930 if (unlikely(ftrace_disabled)) 2931 return; 2932 2933 /* ftrace_start_up is true if ftrace is running */ 2934 if (ftrace_start_up) { 2935 command = FTRACE_DISABLE_CALLS; 2936 if (ftrace_graph_active) 2937 command |= FTRACE_STOP_FUNC_RET; 2938 ftrace_run_update_code(command); 2939 } 2940 } 2941 2942 static u64 ftrace_update_time; 2943 unsigned long ftrace_update_tot_cnt; 2944 2945 static inline int ops_traces_mod(struct ftrace_ops *ops) 2946 { 2947 /* 2948 * Filter_hash being empty will default to trace module. 2949 * But notrace hash requires a test of individual module functions. 2950 */ 2951 return ftrace_hash_empty(ops->func_hash->filter_hash) && 2952 ftrace_hash_empty(ops->func_hash->notrace_hash); 2953 } 2954 2955 /* 2956 * Check if the current ops references the record. 2957 * 2958 * If the ops traces all functions, then it was already accounted for. 2959 * If the ops does not trace the current record function, skip it. 2960 * If the ops ignores the function via notrace filter, skip it. 2961 */ 2962 static inline bool 2963 ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec) 2964 { 2965 /* If ops isn't enabled, ignore it */ 2966 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 2967 return 0; 2968 2969 /* If ops traces all then it includes this function */ 2970 if (ops_traces_mod(ops)) 2971 return 1; 2972 2973 /* The function must be in the filter */ 2974 if (!ftrace_hash_empty(ops->func_hash->filter_hash) && 2975 !__ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip)) 2976 return 0; 2977 2978 /* If in notrace hash, we ignore it too */ 2979 if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) 2980 return 0; 2981 2982 return 1; 2983 } 2984 2985 static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs) 2986 { 2987 struct ftrace_page *pg; 2988 struct dyn_ftrace *p; 2989 u64 start, stop; 2990 unsigned long update_cnt = 0; 2991 unsigned long rec_flags = 0; 2992 int i; 2993 2994 start = ftrace_now(raw_smp_processor_id()); 2995 2996 /* 2997 * When a module is loaded, this function is called to convert 2998 * the calls to mcount in its text to nops, and also to create 2999 * an entry in the ftrace data. Now, if ftrace is activated 3000 * after this call, but before the module sets its text to 3001 * read-only, the modification of enabling ftrace can fail if 3002 * the read-only is done while ftrace is converting the calls. 3003 * To prevent this, the module's records are set as disabled 3004 * and will be enabled after the call to set the module's text 3005 * to read-only. 3006 */ 3007 if (mod) 3008 rec_flags |= FTRACE_FL_DISABLED; 3009 3010 for (pg = new_pgs; pg; pg = pg->next) { 3011 3012 for (i = 0; i < pg->index; i++) { 3013 3014 /* If something went wrong, bail without enabling anything */ 3015 if (unlikely(ftrace_disabled)) 3016 return -1; 3017 3018 p = &pg->records[i]; 3019 p->flags = rec_flags; 3020 3021 /* 3022 * Do the initial record conversion from mcount jump 3023 * to the NOP instructions. 3024 */ 3025 if (!ftrace_code_disable(mod, p)) 3026 break; 3027 3028 update_cnt++; 3029 } 3030 } 3031 3032 stop = ftrace_now(raw_smp_processor_id()); 3033 ftrace_update_time = stop - start; 3034 ftrace_update_tot_cnt += update_cnt; 3035 3036 return 0; 3037 } 3038 3039 static int ftrace_allocate_records(struct ftrace_page *pg, int count) 3040 { 3041 int order; 3042 int cnt; 3043 3044 if (WARN_ON(!count)) 3045 return -EINVAL; 3046 3047 order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE)); 3048 3049 /* 3050 * We want to fill as much as possible. No more than a page 3051 * may be empty. 3052 */ 3053 while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE) 3054 order--; 3055 3056 again: 3057 pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); 3058 3059 if (!pg->records) { 3060 /* if we can't allocate this size, try something smaller */ 3061 if (!order) 3062 return -ENOMEM; 3063 order >>= 1; 3064 goto again; 3065 } 3066 3067 cnt = (PAGE_SIZE << order) / ENTRY_SIZE; 3068 pg->size = cnt; 3069 3070 if (cnt > count) 3071 cnt = count; 3072 3073 return cnt; 3074 } 3075 3076 static struct ftrace_page * 3077 ftrace_allocate_pages(unsigned long num_to_init) 3078 { 3079 struct ftrace_page *start_pg; 3080 struct ftrace_page *pg; 3081 int order; 3082 int cnt; 3083 3084 if (!num_to_init) 3085 return 0; 3086 3087 start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL); 3088 if (!pg) 3089 return NULL; 3090 3091 /* 3092 * Try to allocate as much as possible in one continues 3093 * location that fills in all of the space. We want to 3094 * waste as little space as possible. 3095 */ 3096 for (;;) { 3097 cnt = ftrace_allocate_records(pg, num_to_init); 3098 if (cnt < 0) 3099 goto free_pages; 3100 3101 num_to_init -= cnt; 3102 if (!num_to_init) 3103 break; 3104 3105 pg->next = kzalloc(sizeof(*pg), GFP_KERNEL); 3106 if (!pg->next) 3107 goto free_pages; 3108 3109 pg = pg->next; 3110 } 3111 3112 return start_pg; 3113 3114 free_pages: 3115 pg = start_pg; 3116 while (pg) { 3117 order = get_count_order(pg->size / ENTRIES_PER_PAGE); 3118 free_pages((unsigned long)pg->records, order); 3119 start_pg = pg->next; 3120 kfree(pg); 3121 pg = start_pg; 3122 } 3123 pr_info("ftrace: FAILED to allocate memory for functions\n"); 3124 return NULL; 3125 } 3126 3127 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ 3128 3129 struct ftrace_iterator { 3130 loff_t pos; 3131 loff_t func_pos; 3132 loff_t mod_pos; 3133 struct ftrace_page *pg; 3134 struct dyn_ftrace *func; 3135 struct ftrace_func_probe *probe; 3136 struct ftrace_func_entry *probe_entry; 3137 struct trace_parser parser; 3138 struct ftrace_hash *hash; 3139 struct ftrace_ops *ops; 3140 struct trace_array *tr; 3141 struct list_head *mod_list; 3142 int pidx; 3143 int idx; 3144 unsigned flags; 3145 }; 3146 3147 static void * 3148 t_probe_next(struct seq_file *m, loff_t *pos) 3149 { 3150 struct ftrace_iterator *iter = m->private; 3151 struct trace_array *tr = iter->ops->private; 3152 struct list_head *func_probes; 3153 struct ftrace_hash *hash; 3154 struct list_head *next; 3155 struct hlist_node *hnd = NULL; 3156 struct hlist_head *hhd; 3157 int size; 3158 3159 (*pos)++; 3160 iter->pos = *pos; 3161 3162 if (!tr) 3163 return NULL; 3164 3165 func_probes = &tr->func_probes; 3166 if (list_empty(func_probes)) 3167 return NULL; 3168 3169 if (!iter->probe) { 3170 next = func_probes->next; 3171 iter->probe = list_entry(next, struct ftrace_func_probe, list); 3172 } 3173 3174 if (iter->probe_entry) 3175 hnd = &iter->probe_entry->hlist; 3176 3177 hash = iter->probe->ops.func_hash->filter_hash; 3178 size = 1 << hash->size_bits; 3179 3180 retry: 3181 if (iter->pidx >= size) { 3182 if (iter->probe->list.next == func_probes) 3183 return NULL; 3184 next = iter->probe->list.next; 3185 iter->probe = list_entry(next, struct ftrace_func_probe, list); 3186 hash = iter->probe->ops.func_hash->filter_hash; 3187 size = 1 << hash->size_bits; 3188 iter->pidx = 0; 3189 } 3190 3191 hhd = &hash->buckets[iter->pidx]; 3192 3193 if (hlist_empty(hhd)) { 3194 iter->pidx++; 3195 hnd = NULL; 3196 goto retry; 3197 } 3198 3199 if (!hnd) 3200 hnd = hhd->first; 3201 else { 3202 hnd = hnd->next; 3203 if (!hnd) { 3204 iter->pidx++; 3205 goto retry; 3206 } 3207 } 3208 3209 if (WARN_ON_ONCE(!hnd)) 3210 return NULL; 3211 3212 iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist); 3213 3214 return iter; 3215 } 3216 3217 static void *t_probe_start(struct seq_file *m, loff_t *pos) 3218 { 3219 struct ftrace_iterator *iter = m->private; 3220 void *p = NULL; 3221 loff_t l; 3222 3223 if (!(iter->flags & FTRACE_ITER_DO_PROBES)) 3224 return NULL; 3225 3226 if (iter->mod_pos > *pos) 3227 return NULL; 3228 3229 iter->probe = NULL; 3230 iter->probe_entry = NULL; 3231 iter->pidx = 0; 3232 for (l = 0; l <= (*pos - iter->mod_pos); ) { 3233 p = t_probe_next(m, &l); 3234 if (!p) 3235 break; 3236 } 3237 if (!p) 3238 return NULL; 3239 3240 /* Only set this if we have an item */ 3241 iter->flags |= FTRACE_ITER_PROBE; 3242 3243 return iter; 3244 } 3245 3246 static int 3247 t_probe_show(struct seq_file *m, struct ftrace_iterator *iter) 3248 { 3249 struct ftrace_func_entry *probe_entry; 3250 struct ftrace_probe_ops *probe_ops; 3251 struct ftrace_func_probe *probe; 3252 3253 probe = iter->probe; 3254 probe_entry = iter->probe_entry; 3255 3256 if (WARN_ON_ONCE(!probe || !probe_entry)) 3257 return -EIO; 3258 3259 probe_ops = probe->probe_ops; 3260 3261 if (probe_ops->print) 3262 return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data); 3263 3264 seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip, 3265 (void *)probe_ops->func); 3266 3267 return 0; 3268 } 3269 3270 static void * 3271 t_mod_next(struct seq_file *m, loff_t *pos) 3272 { 3273 struct ftrace_iterator *iter = m->private; 3274 struct trace_array *tr = iter->tr; 3275 3276 (*pos)++; 3277 iter->pos = *pos; 3278 3279 iter->mod_list = iter->mod_list->next; 3280 3281 if (iter->mod_list == &tr->mod_trace || 3282 iter->mod_list == &tr->mod_notrace) { 3283 iter->flags &= ~FTRACE_ITER_MOD; 3284 return NULL; 3285 } 3286 3287 iter->mod_pos = *pos; 3288 3289 return iter; 3290 } 3291 3292 static void *t_mod_start(struct seq_file *m, loff_t *pos) 3293 { 3294 struct ftrace_iterator *iter = m->private; 3295 void *p = NULL; 3296 loff_t l; 3297 3298 if (iter->func_pos > *pos) 3299 return NULL; 3300 3301 iter->mod_pos = iter->func_pos; 3302 3303 /* probes are only available if tr is set */ 3304 if (!iter->tr) 3305 return NULL; 3306 3307 for (l = 0; l <= (*pos - iter->func_pos); ) { 3308 p = t_mod_next(m, &l); 3309 if (!p) 3310 break; 3311 } 3312 if (!p) { 3313 iter->flags &= ~FTRACE_ITER_MOD; 3314 return t_probe_start(m, pos); 3315 } 3316 3317 /* Only set this if we have an item */ 3318 iter->flags |= FTRACE_ITER_MOD; 3319 3320 return iter; 3321 } 3322 3323 static int 3324 t_mod_show(struct seq_file *m, struct ftrace_iterator *iter) 3325 { 3326 struct ftrace_mod_load *ftrace_mod; 3327 struct trace_array *tr = iter->tr; 3328 3329 if (WARN_ON_ONCE(!iter->mod_list) || 3330 iter->mod_list == &tr->mod_trace || 3331 iter->mod_list == &tr->mod_notrace) 3332 return -EIO; 3333 3334 ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list); 3335 3336 if (ftrace_mod->func) 3337 seq_printf(m, "%s", ftrace_mod->func); 3338 else 3339 seq_putc(m, '*'); 3340 3341 seq_printf(m, ":mod:%s\n", ftrace_mod->module); 3342 3343 return 0; 3344 } 3345 3346 static void * 3347 t_func_next(struct seq_file *m, loff_t *pos) 3348 { 3349 struct ftrace_iterator *iter = m->private; 3350 struct dyn_ftrace *rec = NULL; 3351 3352 (*pos)++; 3353 3354 retry: 3355 if (iter->idx >= iter->pg->index) { 3356 if (iter->pg->next) { 3357 iter->pg = iter->pg->next; 3358 iter->idx = 0; 3359 goto retry; 3360 } 3361 } else { 3362 rec = &iter->pg->records[iter->idx++]; 3363 if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) && 3364 !ftrace_lookup_ip(iter->hash, rec->ip)) || 3365 3366 ((iter->flags & FTRACE_ITER_ENABLED) && 3367 !(rec->flags & FTRACE_FL_ENABLED))) { 3368 3369 rec = NULL; 3370 goto retry; 3371 } 3372 } 3373 3374 if (!rec) 3375 return NULL; 3376 3377 iter->pos = iter->func_pos = *pos; 3378 iter->func = rec; 3379 3380 return iter; 3381 } 3382 3383 static void * 3384 t_next(struct seq_file *m, void *v, loff_t *pos) 3385 { 3386 struct ftrace_iterator *iter = m->private; 3387 loff_t l = *pos; /* t_probe_start() must use original pos */ 3388 void *ret; 3389 3390 if (unlikely(ftrace_disabled)) 3391 return NULL; 3392 3393 if (iter->flags & FTRACE_ITER_PROBE) 3394 return t_probe_next(m, pos); 3395 3396 if (iter->flags & FTRACE_ITER_MOD) 3397 return t_mod_next(m, pos); 3398 3399 if (iter->flags & FTRACE_ITER_PRINTALL) { 3400 /* next must increment pos, and t_probe_start does not */ 3401 (*pos)++; 3402 return t_mod_start(m, &l); 3403 } 3404 3405 ret = t_func_next(m, pos); 3406 3407 if (!ret) 3408 return t_mod_start(m, &l); 3409 3410 return ret; 3411 } 3412 3413 static void reset_iter_read(struct ftrace_iterator *iter) 3414 { 3415 iter->pos = 0; 3416 iter->func_pos = 0; 3417 iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD); 3418 } 3419 3420 static void *t_start(struct seq_file *m, loff_t *pos) 3421 { 3422 struct ftrace_iterator *iter = m->private; 3423 void *p = NULL; 3424 loff_t l; 3425 3426 mutex_lock(&ftrace_lock); 3427 3428 if (unlikely(ftrace_disabled)) 3429 return NULL; 3430 3431 /* 3432 * If an lseek was done, then reset and start from beginning. 3433 */ 3434 if (*pos < iter->pos) 3435 reset_iter_read(iter); 3436 3437 /* 3438 * For set_ftrace_filter reading, if we have the filter 3439 * off, we can short cut and just print out that all 3440 * functions are enabled. 3441 */ 3442 if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) && 3443 ftrace_hash_empty(iter->hash)) { 3444 iter->func_pos = 1; /* Account for the message */ 3445 if (*pos > 0) 3446 return t_mod_start(m, pos); 3447 iter->flags |= FTRACE_ITER_PRINTALL; 3448 /* reset in case of seek/pread */ 3449 iter->flags &= ~FTRACE_ITER_PROBE; 3450 return iter; 3451 } 3452 3453 if (iter->flags & FTRACE_ITER_MOD) 3454 return t_mod_start(m, pos); 3455 3456 /* 3457 * Unfortunately, we need to restart at ftrace_pages_start 3458 * every time we let go of the ftrace_mutex. This is because 3459 * those pointers can change without the lock. 3460 */ 3461 iter->pg = ftrace_pages_start; 3462 iter->idx = 0; 3463 for (l = 0; l <= *pos; ) { 3464 p = t_func_next(m, &l); 3465 if (!p) 3466 break; 3467 } 3468 3469 if (!p) 3470 return t_mod_start(m, pos); 3471 3472 return iter; 3473 } 3474 3475 static void t_stop(struct seq_file *m, void *p) 3476 { 3477 mutex_unlock(&ftrace_lock); 3478 } 3479 3480 void * __weak 3481 arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec) 3482 { 3483 return NULL; 3484 } 3485 3486 static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops, 3487 struct dyn_ftrace *rec) 3488 { 3489 void *ptr; 3490 3491 ptr = arch_ftrace_trampoline_func(ops, rec); 3492 if (ptr) 3493 seq_printf(m, " ->%pS", ptr); 3494 } 3495 3496 static int t_show(struct seq_file *m, void *v) 3497 { 3498 struct ftrace_iterator *iter = m->private; 3499 struct dyn_ftrace *rec; 3500 3501 if (iter->flags & FTRACE_ITER_PROBE) 3502 return t_probe_show(m, iter); 3503 3504 if (iter->flags & FTRACE_ITER_MOD) 3505 return t_mod_show(m, iter); 3506 3507 if (iter->flags & FTRACE_ITER_PRINTALL) { 3508 if (iter->flags & FTRACE_ITER_NOTRACE) 3509 seq_puts(m, "#### no functions disabled ####\n"); 3510 else 3511 seq_puts(m, "#### all functions enabled ####\n"); 3512 return 0; 3513 } 3514 3515 rec = iter->func; 3516 3517 if (!rec) 3518 return 0; 3519 3520 seq_printf(m, "%ps", (void *)rec->ip); 3521 if (iter->flags & FTRACE_ITER_ENABLED) { 3522 struct ftrace_ops *ops; 3523 3524 seq_printf(m, " (%ld)%s%s", 3525 ftrace_rec_count(rec), 3526 rec->flags & FTRACE_FL_REGS ? " R" : " ", 3527 rec->flags & FTRACE_FL_IPMODIFY ? " I" : " "); 3528 if (rec->flags & FTRACE_FL_TRAMP_EN) { 3529 ops = ftrace_find_tramp_ops_any(rec); 3530 if (ops) { 3531 do { 3532 seq_printf(m, "\ttramp: %pS (%pS)", 3533 (void *)ops->trampoline, 3534 (void *)ops->func); 3535 add_trampoline_func(m, ops, rec); 3536 ops = ftrace_find_tramp_ops_next(rec, ops); 3537 } while (ops); 3538 } else 3539 seq_puts(m, "\ttramp: ERROR!"); 3540 } else { 3541 add_trampoline_func(m, NULL, rec); 3542 } 3543 } 3544 3545 seq_putc(m, '\n'); 3546 3547 return 0; 3548 } 3549 3550 static const struct seq_operations show_ftrace_seq_ops = { 3551 .start = t_start, 3552 .next = t_next, 3553 .stop = t_stop, 3554 .show = t_show, 3555 }; 3556 3557 static int 3558 ftrace_avail_open(struct inode *inode, struct file *file) 3559 { 3560 struct ftrace_iterator *iter; 3561 3562 if (unlikely(ftrace_disabled)) 3563 return -ENODEV; 3564 3565 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); 3566 if (!iter) 3567 return -ENOMEM; 3568 3569 iter->pg = ftrace_pages_start; 3570 iter->ops = &global_ops; 3571 3572 return 0; 3573 } 3574 3575 static int 3576 ftrace_enabled_open(struct inode *inode, struct file *file) 3577 { 3578 struct ftrace_iterator *iter; 3579 3580 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); 3581 if (!iter) 3582 return -ENOMEM; 3583 3584 iter->pg = ftrace_pages_start; 3585 iter->flags = FTRACE_ITER_ENABLED; 3586 iter->ops = &global_ops; 3587 3588 return 0; 3589 } 3590 3591 /** 3592 * ftrace_regex_open - initialize function tracer filter files 3593 * @ops: The ftrace_ops that hold the hash filters 3594 * @flag: The type of filter to process 3595 * @inode: The inode, usually passed in to your open routine 3596 * @file: The file, usually passed in to your open routine 3597 * 3598 * ftrace_regex_open() initializes the filter files for the 3599 * @ops. Depending on @flag it may process the filter hash or 3600 * the notrace hash of @ops. With this called from the open 3601 * routine, you can use ftrace_filter_write() for the write 3602 * routine if @flag has FTRACE_ITER_FILTER set, or 3603 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set. 3604 * tracing_lseek() should be used as the lseek routine, and 3605 * release must call ftrace_regex_release(). 3606 */ 3607 int 3608 ftrace_regex_open(struct ftrace_ops *ops, int flag, 3609 struct inode *inode, struct file *file) 3610 { 3611 struct ftrace_iterator *iter; 3612 struct ftrace_hash *hash; 3613 struct list_head *mod_head; 3614 struct trace_array *tr = ops->private; 3615 int ret = 0; 3616 3617 ftrace_ops_init(ops); 3618 3619 if (unlikely(ftrace_disabled)) 3620 return -ENODEV; 3621 3622 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 3623 if (!iter) 3624 return -ENOMEM; 3625 3626 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) { 3627 kfree(iter); 3628 return -ENOMEM; 3629 } 3630 3631 iter->ops = ops; 3632 iter->flags = flag; 3633 iter->tr = tr; 3634 3635 mutex_lock(&ops->func_hash->regex_lock); 3636 3637 if (flag & FTRACE_ITER_NOTRACE) { 3638 hash = ops->func_hash->notrace_hash; 3639 mod_head = tr ? &tr->mod_notrace : NULL; 3640 } else { 3641 hash = ops->func_hash->filter_hash; 3642 mod_head = tr ? &tr->mod_trace : NULL; 3643 } 3644 3645 iter->mod_list = mod_head; 3646 3647 if (file->f_mode & FMODE_WRITE) { 3648 const int size_bits = FTRACE_HASH_DEFAULT_BITS; 3649 3650 if (file->f_flags & O_TRUNC) { 3651 iter->hash = alloc_ftrace_hash(size_bits); 3652 clear_ftrace_mod_list(mod_head); 3653 } else { 3654 iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash); 3655 } 3656 3657 if (!iter->hash) { 3658 trace_parser_put(&iter->parser); 3659 kfree(iter); 3660 ret = -ENOMEM; 3661 goto out_unlock; 3662 } 3663 } else 3664 iter->hash = hash; 3665 3666 if (file->f_mode & FMODE_READ) { 3667 iter->pg = ftrace_pages_start; 3668 3669 ret = seq_open(file, &show_ftrace_seq_ops); 3670 if (!ret) { 3671 struct seq_file *m = file->private_data; 3672 m->private = iter; 3673 } else { 3674 /* Failed */ 3675 free_ftrace_hash(iter->hash); 3676 trace_parser_put(&iter->parser); 3677 kfree(iter); 3678 } 3679 } else 3680 file->private_data = iter; 3681 3682 out_unlock: 3683 mutex_unlock(&ops->func_hash->regex_lock); 3684 3685 return ret; 3686 } 3687 3688 static int 3689 ftrace_filter_open(struct inode *inode, struct file *file) 3690 { 3691 struct ftrace_ops *ops = inode->i_private; 3692 3693 return ftrace_regex_open(ops, 3694 FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES, 3695 inode, file); 3696 } 3697 3698 static int 3699 ftrace_notrace_open(struct inode *inode, struct file *file) 3700 { 3701 struct ftrace_ops *ops = inode->i_private; 3702 3703 return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE, 3704 inode, file); 3705 } 3706 3707 /* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */ 3708 struct ftrace_glob { 3709 char *search; 3710 unsigned len; 3711 int type; 3712 }; 3713 3714 /* 3715 * If symbols in an architecture don't correspond exactly to the user-visible 3716 * name of what they represent, it is possible to define this function to 3717 * perform the necessary adjustments. 3718 */ 3719 char * __weak arch_ftrace_match_adjust(char *str, const char *search) 3720 { 3721 return str; 3722 } 3723 3724 static int ftrace_match(char *str, struct ftrace_glob *g) 3725 { 3726 int matched = 0; 3727 int slen; 3728 3729 str = arch_ftrace_match_adjust(str, g->search); 3730 3731 switch (g->type) { 3732 case MATCH_FULL: 3733 if (strcmp(str, g->search) == 0) 3734 matched = 1; 3735 break; 3736 case MATCH_FRONT_ONLY: 3737 if (strncmp(str, g->search, g->len) == 0) 3738 matched = 1; 3739 break; 3740 case MATCH_MIDDLE_ONLY: 3741 if (strstr(str, g->search)) 3742 matched = 1; 3743 break; 3744 case MATCH_END_ONLY: 3745 slen = strlen(str); 3746 if (slen >= g->len && 3747 memcmp(str + slen - g->len, g->search, g->len) == 0) 3748 matched = 1; 3749 break; 3750 case MATCH_GLOB: 3751 if (glob_match(g->search, str)) 3752 matched = 1; 3753 break; 3754 } 3755 3756 return matched; 3757 } 3758 3759 static int 3760 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter) 3761 { 3762 struct ftrace_func_entry *entry; 3763 int ret = 0; 3764 3765 entry = ftrace_lookup_ip(hash, rec->ip); 3766 if (clear_filter) { 3767 /* Do nothing if it doesn't exist */ 3768 if (!entry) 3769 return 0; 3770 3771 free_hash_entry(hash, entry); 3772 } else { 3773 /* Do nothing if it exists */ 3774 if (entry) 3775 return 0; 3776 3777 ret = add_hash_entry(hash, rec->ip); 3778 } 3779 return ret; 3780 } 3781 3782 static int 3783 ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g, 3784 struct ftrace_glob *mod_g, int exclude_mod) 3785 { 3786 char str[KSYM_SYMBOL_LEN]; 3787 char *modname; 3788 3789 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); 3790 3791 if (mod_g) { 3792 int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0; 3793 3794 /* blank module name to match all modules */ 3795 if (!mod_g->len) { 3796 /* blank module globbing: modname xor exclude_mod */ 3797 if (!exclude_mod != !modname) 3798 goto func_match; 3799 return 0; 3800 } 3801 3802 /* 3803 * exclude_mod is set to trace everything but the given 3804 * module. If it is set and the module matches, then 3805 * return 0. If it is not set, and the module doesn't match 3806 * also return 0. Otherwise, check the function to see if 3807 * that matches. 3808 */ 3809 if (!mod_matches == !exclude_mod) 3810 return 0; 3811 func_match: 3812 /* blank search means to match all funcs in the mod */ 3813 if (!func_g->len) 3814 return 1; 3815 } 3816 3817 return ftrace_match(str, func_g); 3818 } 3819 3820 static int 3821 match_records(struct ftrace_hash *hash, char *func, int len, char *mod) 3822 { 3823 struct ftrace_page *pg; 3824 struct dyn_ftrace *rec; 3825 struct ftrace_glob func_g = { .type = MATCH_FULL }; 3826 struct ftrace_glob mod_g = { .type = MATCH_FULL }; 3827 struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL; 3828 int exclude_mod = 0; 3829 int found = 0; 3830 int ret; 3831 int clear_filter = 0; 3832 3833 if (func) { 3834 func_g.type = filter_parse_regex(func, len, &func_g.search, 3835 &clear_filter); 3836 func_g.len = strlen(func_g.search); 3837 } 3838 3839 if (mod) { 3840 mod_g.type = filter_parse_regex(mod, strlen(mod), 3841 &mod_g.search, &exclude_mod); 3842 mod_g.len = strlen(mod_g.search); 3843 } 3844 3845 mutex_lock(&ftrace_lock); 3846 3847 if (unlikely(ftrace_disabled)) 3848 goto out_unlock; 3849 3850 do_for_each_ftrace_rec(pg, rec) { 3851 3852 if (rec->flags & FTRACE_FL_DISABLED) 3853 continue; 3854 3855 if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) { 3856 ret = enter_record(hash, rec, clear_filter); 3857 if (ret < 0) { 3858 found = ret; 3859 goto out_unlock; 3860 } 3861 found = 1; 3862 } 3863 } while_for_each_ftrace_rec(); 3864 out_unlock: 3865 mutex_unlock(&ftrace_lock); 3866 3867 return found; 3868 } 3869 3870 static int 3871 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len) 3872 { 3873 return match_records(hash, buff, len, NULL); 3874 } 3875 3876 static void ftrace_ops_update_code(struct ftrace_ops *ops, 3877 struct ftrace_ops_hash *old_hash) 3878 { 3879 struct ftrace_ops *op; 3880 3881 if (!ftrace_enabled) 3882 return; 3883 3884 if (ops->flags & FTRACE_OPS_FL_ENABLED) { 3885 ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash); 3886 return; 3887 } 3888 3889 /* 3890 * If this is the shared global_ops filter, then we need to 3891 * check if there is another ops that shares it, is enabled. 3892 * If so, we still need to run the modify code. 3893 */ 3894 if (ops->func_hash != &global_ops.local_hash) 3895 return; 3896 3897 do_for_each_ftrace_op(op, ftrace_ops_list) { 3898 if (op->func_hash == &global_ops.local_hash && 3899 op->flags & FTRACE_OPS_FL_ENABLED) { 3900 ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash); 3901 /* Only need to do this once */ 3902 return; 3903 } 3904 } while_for_each_ftrace_op(op); 3905 } 3906 3907 static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops, 3908 struct ftrace_hash **orig_hash, 3909 struct ftrace_hash *hash, 3910 int enable) 3911 { 3912 struct ftrace_ops_hash old_hash_ops; 3913 struct ftrace_hash *old_hash; 3914 int ret; 3915 3916 old_hash = *orig_hash; 3917 old_hash_ops.filter_hash = ops->func_hash->filter_hash; 3918 old_hash_ops.notrace_hash = ops->func_hash->notrace_hash; 3919 ret = ftrace_hash_move(ops, enable, orig_hash, hash); 3920 if (!ret) { 3921 ftrace_ops_update_code(ops, &old_hash_ops); 3922 free_ftrace_hash_rcu(old_hash); 3923 } 3924 return ret; 3925 } 3926 3927 static bool module_exists(const char *module) 3928 { 3929 /* All modules have the symbol __this_module */ 3930 const char this_mod[] = "__this_module"; 3931 const int modname_size = MAX_PARAM_PREFIX_LEN + sizeof(this_mod) + 1; 3932 char modname[modname_size + 1]; 3933 unsigned long val; 3934 int n; 3935 3936 n = snprintf(modname, modname_size + 1, "%s:%s", module, this_mod); 3937 3938 if (n > modname_size) 3939 return false; 3940 3941 val = module_kallsyms_lookup_name(modname); 3942 return val != 0; 3943 } 3944 3945 static int cache_mod(struct trace_array *tr, 3946 const char *func, char *module, int enable) 3947 { 3948 struct ftrace_mod_load *ftrace_mod, *n; 3949 struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace; 3950 int ret; 3951 3952 mutex_lock(&ftrace_lock); 3953 3954 /* We do not cache inverse filters */ 3955 if (func[0] == '!') { 3956 func++; 3957 ret = -EINVAL; 3958 3959 /* Look to remove this hash */ 3960 list_for_each_entry_safe(ftrace_mod, n, head, list) { 3961 if (strcmp(ftrace_mod->module, module) != 0) 3962 continue; 3963 3964 /* no func matches all */ 3965 if (strcmp(func, "*") == 0 || 3966 (ftrace_mod->func && 3967 strcmp(ftrace_mod->func, func) == 0)) { 3968 ret = 0; 3969 free_ftrace_mod(ftrace_mod); 3970 continue; 3971 } 3972 } 3973 goto out; 3974 } 3975 3976 ret = -EINVAL; 3977 /* We only care about modules that have not been loaded yet */ 3978 if (module_exists(module)) 3979 goto out; 3980 3981 /* Save this string off, and execute it when the module is loaded */ 3982 ret = ftrace_add_mod(tr, func, module, enable); 3983 out: 3984 mutex_unlock(&ftrace_lock); 3985 3986 return ret; 3987 } 3988 3989 static int 3990 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len, 3991 int reset, int enable); 3992 3993 #ifdef CONFIG_MODULES 3994 static void process_mod_list(struct list_head *head, struct ftrace_ops *ops, 3995 char *mod, bool enable) 3996 { 3997 struct ftrace_mod_load *ftrace_mod, *n; 3998 struct ftrace_hash **orig_hash, *new_hash; 3999 LIST_HEAD(process_mods); 4000 char *func; 4001 int ret; 4002 4003 mutex_lock(&ops->func_hash->regex_lock); 4004 4005 if (enable) 4006 orig_hash = &ops->func_hash->filter_hash; 4007 else 4008 orig_hash = &ops->func_hash->notrace_hash; 4009 4010 new_hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, 4011 *orig_hash); 4012 if (!new_hash) 4013 goto out; /* warn? */ 4014 4015 mutex_lock(&ftrace_lock); 4016 4017 list_for_each_entry_safe(ftrace_mod, n, head, list) { 4018 4019 if (strcmp(ftrace_mod->module, mod) != 0) 4020 continue; 4021 4022 if (ftrace_mod->func) 4023 func = kstrdup(ftrace_mod->func, GFP_KERNEL); 4024 else 4025 func = kstrdup("*", GFP_KERNEL); 4026 4027 if (!func) /* warn? */ 4028 continue; 4029 4030 list_del(&ftrace_mod->list); 4031 list_add(&ftrace_mod->list, &process_mods); 4032 4033 /* Use the newly allocated func, as it may be "*" */ 4034 kfree(ftrace_mod->func); 4035 ftrace_mod->func = func; 4036 } 4037 4038 mutex_unlock(&ftrace_lock); 4039 4040 list_for_each_entry_safe(ftrace_mod, n, &process_mods, list) { 4041 4042 func = ftrace_mod->func; 4043 4044 /* Grabs ftrace_lock, which is why we have this extra step */ 4045 match_records(new_hash, func, strlen(func), mod); 4046 free_ftrace_mod(ftrace_mod); 4047 } 4048 4049 if (enable && list_empty(head)) 4050 new_hash->flags &= ~FTRACE_HASH_FL_MOD; 4051 4052 mutex_lock(&ftrace_lock); 4053 4054 ret = ftrace_hash_move_and_update_ops(ops, orig_hash, 4055 new_hash, enable); 4056 mutex_unlock(&ftrace_lock); 4057 4058 out: 4059 mutex_unlock(&ops->func_hash->regex_lock); 4060 4061 free_ftrace_hash(new_hash); 4062 } 4063 4064 static void process_cached_mods(const char *mod_name) 4065 { 4066 struct trace_array *tr; 4067 char *mod; 4068 4069 mod = kstrdup(mod_name, GFP_KERNEL); 4070 if (!mod) 4071 return; 4072 4073 mutex_lock(&trace_types_lock); 4074 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 4075 if (!list_empty(&tr->mod_trace)) 4076 process_mod_list(&tr->mod_trace, tr->ops, mod, true); 4077 if (!list_empty(&tr->mod_notrace)) 4078 process_mod_list(&tr->mod_notrace, tr->ops, mod, false); 4079 } 4080 mutex_unlock(&trace_types_lock); 4081 4082 kfree(mod); 4083 } 4084 #endif 4085 4086 /* 4087 * We register the module command as a template to show others how 4088 * to register the a command as well. 4089 */ 4090 4091 static int 4092 ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash, 4093 char *func_orig, char *cmd, char *module, int enable) 4094 { 4095 char *func; 4096 int ret; 4097 4098 /* match_records() modifies func, and we need the original */ 4099 func = kstrdup(func_orig, GFP_KERNEL); 4100 if (!func) 4101 return -ENOMEM; 4102 4103 /* 4104 * cmd == 'mod' because we only registered this func 4105 * for the 'mod' ftrace_func_command. 4106 * But if you register one func with multiple commands, 4107 * you can tell which command was used by the cmd 4108 * parameter. 4109 */ 4110 ret = match_records(hash, func, strlen(func), module); 4111 kfree(func); 4112 4113 if (!ret) 4114 return cache_mod(tr, func_orig, module, enable); 4115 if (ret < 0) 4116 return ret; 4117 return 0; 4118 } 4119 4120 static struct ftrace_func_command ftrace_mod_cmd = { 4121 .name = "mod", 4122 .func = ftrace_mod_callback, 4123 }; 4124 4125 static int __init ftrace_mod_cmd_init(void) 4126 { 4127 return register_ftrace_command(&ftrace_mod_cmd); 4128 } 4129 core_initcall(ftrace_mod_cmd_init); 4130 4131 static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip, 4132 struct ftrace_ops *op, struct pt_regs *pt_regs) 4133 { 4134 struct ftrace_probe_ops *probe_ops; 4135 struct ftrace_func_probe *probe; 4136 4137 probe = container_of(op, struct ftrace_func_probe, ops); 4138 probe_ops = probe->probe_ops; 4139 4140 /* 4141 * Disable preemption for these calls to prevent a RCU grace 4142 * period. This syncs the hash iteration and freeing of items 4143 * on the hash. rcu_read_lock is too dangerous here. 4144 */ 4145 preempt_disable_notrace(); 4146 probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data); 4147 preempt_enable_notrace(); 4148 } 4149 4150 struct ftrace_func_map { 4151 struct ftrace_func_entry entry; 4152 void *data; 4153 }; 4154 4155 struct ftrace_func_mapper { 4156 struct ftrace_hash hash; 4157 }; 4158 4159 /** 4160 * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper 4161 * 4162 * Returns a ftrace_func_mapper descriptor that can be used to map ips to data. 4163 */ 4164 struct ftrace_func_mapper *allocate_ftrace_func_mapper(void) 4165 { 4166 struct ftrace_hash *hash; 4167 4168 /* 4169 * The mapper is simply a ftrace_hash, but since the entries 4170 * in the hash are not ftrace_func_entry type, we define it 4171 * as a separate structure. 4172 */ 4173 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); 4174 return (struct ftrace_func_mapper *)hash; 4175 } 4176 4177 /** 4178 * ftrace_func_mapper_find_ip - Find some data mapped to an ip 4179 * @mapper: The mapper that has the ip maps 4180 * @ip: the instruction pointer to find the data for 4181 * 4182 * Returns the data mapped to @ip if found otherwise NULL. The return 4183 * is actually the address of the mapper data pointer. The address is 4184 * returned for use cases where the data is no bigger than a long, and 4185 * the user can use the data pointer as its data instead of having to 4186 * allocate more memory for the reference. 4187 */ 4188 void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper, 4189 unsigned long ip) 4190 { 4191 struct ftrace_func_entry *entry; 4192 struct ftrace_func_map *map; 4193 4194 entry = ftrace_lookup_ip(&mapper->hash, ip); 4195 if (!entry) 4196 return NULL; 4197 4198 map = (struct ftrace_func_map *)entry; 4199 return &map->data; 4200 } 4201 4202 /** 4203 * ftrace_func_mapper_add_ip - Map some data to an ip 4204 * @mapper: The mapper that has the ip maps 4205 * @ip: The instruction pointer address to map @data to 4206 * @data: The data to map to @ip 4207 * 4208 * Returns 0 on succes otherwise an error. 4209 */ 4210 int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper, 4211 unsigned long ip, void *data) 4212 { 4213 struct ftrace_func_entry *entry; 4214 struct ftrace_func_map *map; 4215 4216 entry = ftrace_lookup_ip(&mapper->hash, ip); 4217 if (entry) 4218 return -EBUSY; 4219 4220 map = kmalloc(sizeof(*map), GFP_KERNEL); 4221 if (!map) 4222 return -ENOMEM; 4223 4224 map->entry.ip = ip; 4225 map->data = data; 4226 4227 __add_hash_entry(&mapper->hash, &map->entry); 4228 4229 return 0; 4230 } 4231 4232 /** 4233 * ftrace_func_mapper_remove_ip - Remove an ip from the mapping 4234 * @mapper: The mapper that has the ip maps 4235 * @ip: The instruction pointer address to remove the data from 4236 * 4237 * Returns the data if it is found, otherwise NULL. 4238 * Note, if the data pointer is used as the data itself, (see 4239 * ftrace_func_mapper_find_ip(), then the return value may be meaningless, 4240 * if the data pointer was set to zero. 4241 */ 4242 void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper, 4243 unsigned long ip) 4244 { 4245 struct ftrace_func_entry *entry; 4246 struct ftrace_func_map *map; 4247 void *data; 4248 4249 entry = ftrace_lookup_ip(&mapper->hash, ip); 4250 if (!entry) 4251 return NULL; 4252 4253 map = (struct ftrace_func_map *)entry; 4254 data = map->data; 4255 4256 remove_hash_entry(&mapper->hash, entry); 4257 kfree(entry); 4258 4259 return data; 4260 } 4261 4262 /** 4263 * free_ftrace_func_mapper - free a mapping of ips and data 4264 * @mapper: The mapper that has the ip maps 4265 * @free_func: A function to be called on each data item. 4266 * 4267 * This is used to free the function mapper. The @free_func is optional 4268 * and can be used if the data needs to be freed as well. 4269 */ 4270 void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper, 4271 ftrace_mapper_func free_func) 4272 { 4273 struct ftrace_func_entry *entry; 4274 struct ftrace_func_map *map; 4275 struct hlist_head *hhd; 4276 int size = 1 << mapper->hash.size_bits; 4277 int i; 4278 4279 if (free_func && mapper->hash.count) { 4280 for (i = 0; i < size; i++) { 4281 hhd = &mapper->hash.buckets[i]; 4282 hlist_for_each_entry(entry, hhd, hlist) { 4283 map = (struct ftrace_func_map *)entry; 4284 free_func(map); 4285 } 4286 } 4287 } 4288 free_ftrace_hash(&mapper->hash); 4289 } 4290 4291 static void release_probe(struct ftrace_func_probe *probe) 4292 { 4293 struct ftrace_probe_ops *probe_ops; 4294 4295 mutex_lock(&ftrace_lock); 4296 4297 WARN_ON(probe->ref <= 0); 4298 4299 /* Subtract the ref that was used to protect this instance */ 4300 probe->ref--; 4301 4302 if (!probe->ref) { 4303 probe_ops = probe->probe_ops; 4304 /* 4305 * Sending zero as ip tells probe_ops to free 4306 * the probe->data itself 4307 */ 4308 if (probe_ops->free) 4309 probe_ops->free(probe_ops, probe->tr, 0, probe->data); 4310 list_del(&probe->list); 4311 kfree(probe); 4312 } 4313 mutex_unlock(&ftrace_lock); 4314 } 4315 4316 static void acquire_probe_locked(struct ftrace_func_probe *probe) 4317 { 4318 /* 4319 * Add one ref to keep it from being freed when releasing the 4320 * ftrace_lock mutex. 4321 */ 4322 probe->ref++; 4323 } 4324 4325 int 4326 register_ftrace_function_probe(char *glob, struct trace_array *tr, 4327 struct ftrace_probe_ops *probe_ops, 4328 void *data) 4329 { 4330 struct ftrace_func_entry *entry; 4331 struct ftrace_func_probe *probe; 4332 struct ftrace_hash **orig_hash; 4333 struct ftrace_hash *old_hash; 4334 struct ftrace_hash *hash; 4335 int count = 0; 4336 int size; 4337 int ret; 4338 int i; 4339 4340 if (WARN_ON(!tr)) 4341 return -EINVAL; 4342 4343 /* We do not support '!' for function probes */ 4344 if (WARN_ON(glob[0] == '!')) 4345 return -EINVAL; 4346 4347 4348 mutex_lock(&ftrace_lock); 4349 /* Check if the probe_ops is already registered */ 4350 list_for_each_entry(probe, &tr->func_probes, list) { 4351 if (probe->probe_ops == probe_ops) 4352 break; 4353 } 4354 if (&probe->list == &tr->func_probes) { 4355 probe = kzalloc(sizeof(*probe), GFP_KERNEL); 4356 if (!probe) { 4357 mutex_unlock(&ftrace_lock); 4358 return -ENOMEM; 4359 } 4360 probe->probe_ops = probe_ops; 4361 probe->ops.func = function_trace_probe_call; 4362 probe->tr = tr; 4363 ftrace_ops_init(&probe->ops); 4364 list_add(&probe->list, &tr->func_probes); 4365 } 4366 4367 acquire_probe_locked(probe); 4368 4369 mutex_unlock(&ftrace_lock); 4370 4371 mutex_lock(&probe->ops.func_hash->regex_lock); 4372 4373 orig_hash = &probe->ops.func_hash->filter_hash; 4374 old_hash = *orig_hash; 4375 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash); 4376 4377 ret = ftrace_match_records(hash, glob, strlen(glob)); 4378 4379 /* Nothing found? */ 4380 if (!ret) 4381 ret = -EINVAL; 4382 4383 if (ret < 0) 4384 goto out; 4385 4386 size = 1 << hash->size_bits; 4387 for (i = 0; i < size; i++) { 4388 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 4389 if (ftrace_lookup_ip(old_hash, entry->ip)) 4390 continue; 4391 /* 4392 * The caller might want to do something special 4393 * for each function we find. We call the callback 4394 * to give the caller an opportunity to do so. 4395 */ 4396 if (probe_ops->init) { 4397 ret = probe_ops->init(probe_ops, tr, 4398 entry->ip, data, 4399 &probe->data); 4400 if (ret < 0) { 4401 if (probe_ops->free && count) 4402 probe_ops->free(probe_ops, tr, 4403 0, probe->data); 4404 probe->data = NULL; 4405 goto out; 4406 } 4407 } 4408 count++; 4409 } 4410 } 4411 4412 mutex_lock(&ftrace_lock); 4413 4414 if (!count) { 4415 /* Nothing was added? */ 4416 ret = -EINVAL; 4417 goto out_unlock; 4418 } 4419 4420 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, 4421 hash, 1); 4422 if (ret < 0) 4423 goto err_unlock; 4424 4425 /* One ref for each new function traced */ 4426 probe->ref += count; 4427 4428 if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED)) 4429 ret = ftrace_startup(&probe->ops, 0); 4430 4431 out_unlock: 4432 mutex_unlock(&ftrace_lock); 4433 4434 if (!ret) 4435 ret = count; 4436 out: 4437 mutex_unlock(&probe->ops.func_hash->regex_lock); 4438 free_ftrace_hash(hash); 4439 4440 release_probe(probe); 4441 4442 return ret; 4443 4444 err_unlock: 4445 if (!probe_ops->free || !count) 4446 goto out_unlock; 4447 4448 /* Failed to do the move, need to call the free functions */ 4449 for (i = 0; i < size; i++) { 4450 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 4451 if (ftrace_lookup_ip(old_hash, entry->ip)) 4452 continue; 4453 probe_ops->free(probe_ops, tr, entry->ip, probe->data); 4454 } 4455 } 4456 goto out_unlock; 4457 } 4458 4459 int 4460 unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr, 4461 struct ftrace_probe_ops *probe_ops) 4462 { 4463 struct ftrace_ops_hash old_hash_ops; 4464 struct ftrace_func_entry *entry; 4465 struct ftrace_func_probe *probe; 4466 struct ftrace_glob func_g; 4467 struct ftrace_hash **orig_hash; 4468 struct ftrace_hash *old_hash; 4469 struct ftrace_hash *hash = NULL; 4470 struct hlist_node *tmp; 4471 struct hlist_head hhd; 4472 char str[KSYM_SYMBOL_LEN]; 4473 int count = 0; 4474 int i, ret = -ENODEV; 4475 int size; 4476 4477 if (!glob || !strlen(glob) || !strcmp(glob, "*")) 4478 func_g.search = NULL; 4479 else { 4480 int not; 4481 4482 func_g.type = filter_parse_regex(glob, strlen(glob), 4483 &func_g.search, ¬); 4484 func_g.len = strlen(func_g.search); 4485 func_g.search = glob; 4486 4487 /* we do not support '!' for function probes */ 4488 if (WARN_ON(not)) 4489 return -EINVAL; 4490 } 4491 4492 mutex_lock(&ftrace_lock); 4493 /* Check if the probe_ops is already registered */ 4494 list_for_each_entry(probe, &tr->func_probes, list) { 4495 if (probe->probe_ops == probe_ops) 4496 break; 4497 } 4498 if (&probe->list == &tr->func_probes) 4499 goto err_unlock_ftrace; 4500 4501 ret = -EINVAL; 4502 if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED)) 4503 goto err_unlock_ftrace; 4504 4505 acquire_probe_locked(probe); 4506 4507 mutex_unlock(&ftrace_lock); 4508 4509 mutex_lock(&probe->ops.func_hash->regex_lock); 4510 4511 orig_hash = &probe->ops.func_hash->filter_hash; 4512 old_hash = *orig_hash; 4513 4514 if (ftrace_hash_empty(old_hash)) 4515 goto out_unlock; 4516 4517 old_hash_ops.filter_hash = old_hash; 4518 /* Probes only have filters */ 4519 old_hash_ops.notrace_hash = NULL; 4520 4521 ret = -ENOMEM; 4522 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash); 4523 if (!hash) 4524 goto out_unlock; 4525 4526 INIT_HLIST_HEAD(&hhd); 4527 4528 size = 1 << hash->size_bits; 4529 for (i = 0; i < size; i++) { 4530 hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) { 4531 4532 if (func_g.search) { 4533 kallsyms_lookup(entry->ip, NULL, NULL, 4534 NULL, str); 4535 if (!ftrace_match(str, &func_g)) 4536 continue; 4537 } 4538 count++; 4539 remove_hash_entry(hash, entry); 4540 hlist_add_head(&entry->hlist, &hhd); 4541 } 4542 } 4543 4544 /* Nothing found? */ 4545 if (!count) { 4546 ret = -EINVAL; 4547 goto out_unlock; 4548 } 4549 4550 mutex_lock(&ftrace_lock); 4551 4552 WARN_ON(probe->ref < count); 4553 4554 probe->ref -= count; 4555 4556 if (ftrace_hash_empty(hash)) 4557 ftrace_shutdown(&probe->ops, 0); 4558 4559 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, 4560 hash, 1); 4561 4562 /* still need to update the function call sites */ 4563 if (ftrace_enabled && !ftrace_hash_empty(hash)) 4564 ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS, 4565 &old_hash_ops); 4566 synchronize_sched(); 4567 4568 hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) { 4569 hlist_del(&entry->hlist); 4570 if (probe_ops->free) 4571 probe_ops->free(probe_ops, tr, entry->ip, probe->data); 4572 kfree(entry); 4573 } 4574 mutex_unlock(&ftrace_lock); 4575 4576 out_unlock: 4577 mutex_unlock(&probe->ops.func_hash->regex_lock); 4578 free_ftrace_hash(hash); 4579 4580 release_probe(probe); 4581 4582 return ret; 4583 4584 err_unlock_ftrace: 4585 mutex_unlock(&ftrace_lock); 4586 return ret; 4587 } 4588 4589 void clear_ftrace_function_probes(struct trace_array *tr) 4590 { 4591 struct ftrace_func_probe *probe, *n; 4592 4593 list_for_each_entry_safe(probe, n, &tr->func_probes, list) 4594 unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops); 4595 } 4596 4597 static LIST_HEAD(ftrace_commands); 4598 static DEFINE_MUTEX(ftrace_cmd_mutex); 4599 4600 /* 4601 * Currently we only register ftrace commands from __init, so mark this 4602 * __init too. 4603 */ 4604 __init int register_ftrace_command(struct ftrace_func_command *cmd) 4605 { 4606 struct ftrace_func_command *p; 4607 int ret = 0; 4608 4609 mutex_lock(&ftrace_cmd_mutex); 4610 list_for_each_entry(p, &ftrace_commands, list) { 4611 if (strcmp(cmd->name, p->name) == 0) { 4612 ret = -EBUSY; 4613 goto out_unlock; 4614 } 4615 } 4616 list_add(&cmd->list, &ftrace_commands); 4617 out_unlock: 4618 mutex_unlock(&ftrace_cmd_mutex); 4619 4620 return ret; 4621 } 4622 4623 /* 4624 * Currently we only unregister ftrace commands from __init, so mark 4625 * this __init too. 4626 */ 4627 __init int unregister_ftrace_command(struct ftrace_func_command *cmd) 4628 { 4629 struct ftrace_func_command *p, *n; 4630 int ret = -ENODEV; 4631 4632 mutex_lock(&ftrace_cmd_mutex); 4633 list_for_each_entry_safe(p, n, &ftrace_commands, list) { 4634 if (strcmp(cmd->name, p->name) == 0) { 4635 ret = 0; 4636 list_del_init(&p->list); 4637 goto out_unlock; 4638 } 4639 } 4640 out_unlock: 4641 mutex_unlock(&ftrace_cmd_mutex); 4642 4643 return ret; 4644 } 4645 4646 static int ftrace_process_regex(struct ftrace_iterator *iter, 4647 char *buff, int len, int enable) 4648 { 4649 struct ftrace_hash *hash = iter->hash; 4650 struct trace_array *tr = iter->ops->private; 4651 char *func, *command, *next = buff; 4652 struct ftrace_func_command *p; 4653 int ret = -EINVAL; 4654 4655 func = strsep(&next, ":"); 4656 4657 if (!next) { 4658 ret = ftrace_match_records(hash, func, len); 4659 if (!ret) 4660 ret = -EINVAL; 4661 if (ret < 0) 4662 return ret; 4663 return 0; 4664 } 4665 4666 /* command found */ 4667 4668 command = strsep(&next, ":"); 4669 4670 mutex_lock(&ftrace_cmd_mutex); 4671 list_for_each_entry(p, &ftrace_commands, list) { 4672 if (strcmp(p->name, command) == 0) { 4673 ret = p->func(tr, hash, func, command, next, enable); 4674 goto out_unlock; 4675 } 4676 } 4677 out_unlock: 4678 mutex_unlock(&ftrace_cmd_mutex); 4679 4680 return ret; 4681 } 4682 4683 static ssize_t 4684 ftrace_regex_write(struct file *file, const char __user *ubuf, 4685 size_t cnt, loff_t *ppos, int enable) 4686 { 4687 struct ftrace_iterator *iter; 4688 struct trace_parser *parser; 4689 ssize_t ret, read; 4690 4691 if (!cnt) 4692 return 0; 4693 4694 if (file->f_mode & FMODE_READ) { 4695 struct seq_file *m = file->private_data; 4696 iter = m->private; 4697 } else 4698 iter = file->private_data; 4699 4700 if (unlikely(ftrace_disabled)) 4701 return -ENODEV; 4702 4703 /* iter->hash is a local copy, so we don't need regex_lock */ 4704 4705 parser = &iter->parser; 4706 read = trace_get_user(parser, ubuf, cnt, ppos); 4707 4708 if (read >= 0 && trace_parser_loaded(parser) && 4709 !trace_parser_cont(parser)) { 4710 ret = ftrace_process_regex(iter, parser->buffer, 4711 parser->idx, enable); 4712 trace_parser_clear(parser); 4713 if (ret < 0) 4714 goto out; 4715 } 4716 4717 ret = read; 4718 out: 4719 return ret; 4720 } 4721 4722 ssize_t 4723 ftrace_filter_write(struct file *file, const char __user *ubuf, 4724 size_t cnt, loff_t *ppos) 4725 { 4726 return ftrace_regex_write(file, ubuf, cnt, ppos, 1); 4727 } 4728 4729 ssize_t 4730 ftrace_notrace_write(struct file *file, const char __user *ubuf, 4731 size_t cnt, loff_t *ppos) 4732 { 4733 return ftrace_regex_write(file, ubuf, cnt, ppos, 0); 4734 } 4735 4736 static int 4737 ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove) 4738 { 4739 struct ftrace_func_entry *entry; 4740 4741 if (!ftrace_location(ip)) 4742 return -EINVAL; 4743 4744 if (remove) { 4745 entry = ftrace_lookup_ip(hash, ip); 4746 if (!entry) 4747 return -ENOENT; 4748 free_hash_entry(hash, entry); 4749 return 0; 4750 } 4751 4752 return add_hash_entry(hash, ip); 4753 } 4754 4755 static int 4756 ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len, 4757 unsigned long ip, int remove, int reset, int enable) 4758 { 4759 struct ftrace_hash **orig_hash; 4760 struct ftrace_hash *hash; 4761 int ret; 4762 4763 if (unlikely(ftrace_disabled)) 4764 return -ENODEV; 4765 4766 mutex_lock(&ops->func_hash->regex_lock); 4767 4768 if (enable) 4769 orig_hash = &ops->func_hash->filter_hash; 4770 else 4771 orig_hash = &ops->func_hash->notrace_hash; 4772 4773 if (reset) 4774 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); 4775 else 4776 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash); 4777 4778 if (!hash) { 4779 ret = -ENOMEM; 4780 goto out_regex_unlock; 4781 } 4782 4783 if (buf && !ftrace_match_records(hash, buf, len)) { 4784 ret = -EINVAL; 4785 goto out_regex_unlock; 4786 } 4787 if (ip) { 4788 ret = ftrace_match_addr(hash, ip, remove); 4789 if (ret < 0) 4790 goto out_regex_unlock; 4791 } 4792 4793 mutex_lock(&ftrace_lock); 4794 ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable); 4795 mutex_unlock(&ftrace_lock); 4796 4797 out_regex_unlock: 4798 mutex_unlock(&ops->func_hash->regex_lock); 4799 4800 free_ftrace_hash(hash); 4801 return ret; 4802 } 4803 4804 static int 4805 ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove, 4806 int reset, int enable) 4807 { 4808 return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable); 4809 } 4810 4811 /** 4812 * ftrace_set_filter_ip - set a function to filter on in ftrace by address 4813 * @ops - the ops to set the filter with 4814 * @ip - the address to add to or remove from the filter. 4815 * @remove - non zero to remove the ip from the filter 4816 * @reset - non zero to reset all filters before applying this filter. 4817 * 4818 * Filters denote which functions should be enabled when tracing is enabled 4819 * If @ip is NULL, it failes to update filter. 4820 */ 4821 int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip, 4822 int remove, int reset) 4823 { 4824 ftrace_ops_init(ops); 4825 return ftrace_set_addr(ops, ip, remove, reset, 1); 4826 } 4827 EXPORT_SYMBOL_GPL(ftrace_set_filter_ip); 4828 4829 /** 4830 * ftrace_ops_set_global_filter - setup ops to use global filters 4831 * @ops - the ops which will use the global filters 4832 * 4833 * ftrace users who need global function trace filtering should call this. 4834 * It can set the global filter only if ops were not initialized before. 4835 */ 4836 void ftrace_ops_set_global_filter(struct ftrace_ops *ops) 4837 { 4838 if (ops->flags & FTRACE_OPS_FL_INITIALIZED) 4839 return; 4840 4841 ftrace_ops_init(ops); 4842 ops->func_hash = &global_ops.local_hash; 4843 } 4844 EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter); 4845 4846 static int 4847 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len, 4848 int reset, int enable) 4849 { 4850 return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable); 4851 } 4852 4853 /** 4854 * ftrace_set_filter - set a function to filter on in ftrace 4855 * @ops - the ops to set the filter with 4856 * @buf - the string that holds the function filter text. 4857 * @len - the length of the string. 4858 * @reset - non zero to reset all filters before applying this filter. 4859 * 4860 * Filters denote which functions should be enabled when tracing is enabled. 4861 * If @buf is NULL and reset is set, all functions will be enabled for tracing. 4862 */ 4863 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf, 4864 int len, int reset) 4865 { 4866 ftrace_ops_init(ops); 4867 return ftrace_set_regex(ops, buf, len, reset, 1); 4868 } 4869 EXPORT_SYMBOL_GPL(ftrace_set_filter); 4870 4871 /** 4872 * ftrace_set_notrace - set a function to not trace in ftrace 4873 * @ops - the ops to set the notrace filter with 4874 * @buf - the string that holds the function notrace text. 4875 * @len - the length of the string. 4876 * @reset - non zero to reset all filters before applying this filter. 4877 * 4878 * Notrace Filters denote which functions should not be enabled when tracing 4879 * is enabled. If @buf is NULL and reset is set, all functions will be enabled 4880 * for tracing. 4881 */ 4882 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf, 4883 int len, int reset) 4884 { 4885 ftrace_ops_init(ops); 4886 return ftrace_set_regex(ops, buf, len, reset, 0); 4887 } 4888 EXPORT_SYMBOL_GPL(ftrace_set_notrace); 4889 /** 4890 * ftrace_set_global_filter - set a function to filter on with global tracers 4891 * @buf - the string that holds the function filter text. 4892 * @len - the length of the string. 4893 * @reset - non zero to reset all filters before applying this filter. 4894 * 4895 * Filters denote which functions should be enabled when tracing is enabled. 4896 * If @buf is NULL and reset is set, all functions will be enabled for tracing. 4897 */ 4898 void ftrace_set_global_filter(unsigned char *buf, int len, int reset) 4899 { 4900 ftrace_set_regex(&global_ops, buf, len, reset, 1); 4901 } 4902 EXPORT_SYMBOL_GPL(ftrace_set_global_filter); 4903 4904 /** 4905 * ftrace_set_global_notrace - set a function to not trace with global tracers 4906 * @buf - the string that holds the function notrace text. 4907 * @len - the length of the string. 4908 * @reset - non zero to reset all filters before applying this filter. 4909 * 4910 * Notrace Filters denote which functions should not be enabled when tracing 4911 * is enabled. If @buf is NULL and reset is set, all functions will be enabled 4912 * for tracing. 4913 */ 4914 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset) 4915 { 4916 ftrace_set_regex(&global_ops, buf, len, reset, 0); 4917 } 4918 EXPORT_SYMBOL_GPL(ftrace_set_global_notrace); 4919 4920 /* 4921 * command line interface to allow users to set filters on boot up. 4922 */ 4923 #define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE 4924 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata; 4925 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata; 4926 4927 /* Used by function selftest to not test if filter is set */ 4928 bool ftrace_filter_param __initdata; 4929 4930 static int __init set_ftrace_notrace(char *str) 4931 { 4932 ftrace_filter_param = true; 4933 strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE); 4934 return 1; 4935 } 4936 __setup("ftrace_notrace=", set_ftrace_notrace); 4937 4938 static int __init set_ftrace_filter(char *str) 4939 { 4940 ftrace_filter_param = true; 4941 strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE); 4942 return 1; 4943 } 4944 __setup("ftrace_filter=", set_ftrace_filter); 4945 4946 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 4947 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; 4948 static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata; 4949 static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer); 4950 4951 static unsigned long save_global_trampoline; 4952 static unsigned long save_global_flags; 4953 4954 static int __init set_graph_function(char *str) 4955 { 4956 strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE); 4957 return 1; 4958 } 4959 __setup("ftrace_graph_filter=", set_graph_function); 4960 4961 static int __init set_graph_notrace_function(char *str) 4962 { 4963 strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE); 4964 return 1; 4965 } 4966 __setup("ftrace_graph_notrace=", set_graph_notrace_function); 4967 4968 static int __init set_graph_max_depth_function(char *str) 4969 { 4970 if (!str) 4971 return 0; 4972 fgraph_max_depth = simple_strtoul(str, NULL, 0); 4973 return 1; 4974 } 4975 __setup("ftrace_graph_max_depth=", set_graph_max_depth_function); 4976 4977 static void __init set_ftrace_early_graph(char *buf, int enable) 4978 { 4979 int ret; 4980 char *func; 4981 struct ftrace_hash *hash; 4982 4983 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); 4984 if (WARN_ON(!hash)) 4985 return; 4986 4987 while (buf) { 4988 func = strsep(&buf, ","); 4989 /* we allow only one expression at a time */ 4990 ret = ftrace_graph_set_hash(hash, func); 4991 if (ret) 4992 printk(KERN_DEBUG "ftrace: function %s not " 4993 "traceable\n", func); 4994 } 4995 4996 if (enable) 4997 ftrace_graph_hash = hash; 4998 else 4999 ftrace_graph_notrace_hash = hash; 5000 } 5001 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 5002 5003 void __init 5004 ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable) 5005 { 5006 char *func; 5007 5008 ftrace_ops_init(ops); 5009 5010 while (buf) { 5011 func = strsep(&buf, ","); 5012 ftrace_set_regex(ops, func, strlen(func), 0, enable); 5013 } 5014 } 5015 5016 static void __init set_ftrace_early_filters(void) 5017 { 5018 if (ftrace_filter_buf[0]) 5019 ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1); 5020 if (ftrace_notrace_buf[0]) 5021 ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0); 5022 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 5023 if (ftrace_graph_buf[0]) 5024 set_ftrace_early_graph(ftrace_graph_buf, 1); 5025 if (ftrace_graph_notrace_buf[0]) 5026 set_ftrace_early_graph(ftrace_graph_notrace_buf, 0); 5027 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 5028 } 5029 5030 int ftrace_regex_release(struct inode *inode, struct file *file) 5031 { 5032 struct seq_file *m = (struct seq_file *)file->private_data; 5033 struct ftrace_iterator *iter; 5034 struct ftrace_hash **orig_hash; 5035 struct trace_parser *parser; 5036 int filter_hash; 5037 int ret; 5038 5039 if (file->f_mode & FMODE_READ) { 5040 iter = m->private; 5041 seq_release(inode, file); 5042 } else 5043 iter = file->private_data; 5044 5045 parser = &iter->parser; 5046 if (trace_parser_loaded(parser)) { 5047 parser->buffer[parser->idx] = 0; 5048 ftrace_match_records(iter->hash, parser->buffer, parser->idx); 5049 } 5050 5051 trace_parser_put(parser); 5052 5053 mutex_lock(&iter->ops->func_hash->regex_lock); 5054 5055 if (file->f_mode & FMODE_WRITE) { 5056 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER); 5057 5058 if (filter_hash) { 5059 orig_hash = &iter->ops->func_hash->filter_hash; 5060 if (iter->tr && !list_empty(&iter->tr->mod_trace)) 5061 iter->hash->flags |= FTRACE_HASH_FL_MOD; 5062 } else 5063 orig_hash = &iter->ops->func_hash->notrace_hash; 5064 5065 mutex_lock(&ftrace_lock); 5066 ret = ftrace_hash_move_and_update_ops(iter->ops, orig_hash, 5067 iter->hash, filter_hash); 5068 mutex_unlock(&ftrace_lock); 5069 } else { 5070 /* For read only, the hash is the ops hash */ 5071 iter->hash = NULL; 5072 } 5073 5074 mutex_unlock(&iter->ops->func_hash->regex_lock); 5075 free_ftrace_hash(iter->hash); 5076 kfree(iter); 5077 5078 return 0; 5079 } 5080 5081 static const struct file_operations ftrace_avail_fops = { 5082 .open = ftrace_avail_open, 5083 .read = seq_read, 5084 .llseek = seq_lseek, 5085 .release = seq_release_private, 5086 }; 5087 5088 static const struct file_operations ftrace_enabled_fops = { 5089 .open = ftrace_enabled_open, 5090 .read = seq_read, 5091 .llseek = seq_lseek, 5092 .release = seq_release_private, 5093 }; 5094 5095 static const struct file_operations ftrace_filter_fops = { 5096 .open = ftrace_filter_open, 5097 .read = seq_read, 5098 .write = ftrace_filter_write, 5099 .llseek = tracing_lseek, 5100 .release = ftrace_regex_release, 5101 }; 5102 5103 static const struct file_operations ftrace_notrace_fops = { 5104 .open = ftrace_notrace_open, 5105 .read = seq_read, 5106 .write = ftrace_notrace_write, 5107 .llseek = tracing_lseek, 5108 .release = ftrace_regex_release, 5109 }; 5110 5111 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 5112 5113 static DEFINE_MUTEX(graph_lock); 5114 5115 struct ftrace_hash *ftrace_graph_hash = EMPTY_HASH; 5116 struct ftrace_hash *ftrace_graph_notrace_hash = EMPTY_HASH; 5117 5118 enum graph_filter_type { 5119 GRAPH_FILTER_NOTRACE = 0, 5120 GRAPH_FILTER_FUNCTION, 5121 }; 5122 5123 #define FTRACE_GRAPH_EMPTY ((void *)1) 5124 5125 struct ftrace_graph_data { 5126 struct ftrace_hash *hash; 5127 struct ftrace_func_entry *entry; 5128 int idx; /* for hash table iteration */ 5129 enum graph_filter_type type; 5130 struct ftrace_hash *new_hash; 5131 const struct seq_operations *seq_ops; 5132 struct trace_parser parser; 5133 }; 5134 5135 static void * 5136 __g_next(struct seq_file *m, loff_t *pos) 5137 { 5138 struct ftrace_graph_data *fgd = m->private; 5139 struct ftrace_func_entry *entry = fgd->entry; 5140 struct hlist_head *head; 5141 int i, idx = fgd->idx; 5142 5143 if (*pos >= fgd->hash->count) 5144 return NULL; 5145 5146 if (entry) { 5147 hlist_for_each_entry_continue(entry, hlist) { 5148 fgd->entry = entry; 5149 return entry; 5150 } 5151 5152 idx++; 5153 } 5154 5155 for (i = idx; i < 1 << fgd->hash->size_bits; i++) { 5156 head = &fgd->hash->buckets[i]; 5157 hlist_for_each_entry(entry, head, hlist) { 5158 fgd->entry = entry; 5159 fgd->idx = i; 5160 return entry; 5161 } 5162 } 5163 return NULL; 5164 } 5165 5166 static void * 5167 g_next(struct seq_file *m, void *v, loff_t *pos) 5168 { 5169 (*pos)++; 5170 return __g_next(m, pos); 5171 } 5172 5173 static void *g_start(struct seq_file *m, loff_t *pos) 5174 { 5175 struct ftrace_graph_data *fgd = m->private; 5176 5177 mutex_lock(&graph_lock); 5178 5179 if (fgd->type == GRAPH_FILTER_FUNCTION) 5180 fgd->hash = rcu_dereference_protected(ftrace_graph_hash, 5181 lockdep_is_held(&graph_lock)); 5182 else 5183 fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash, 5184 lockdep_is_held(&graph_lock)); 5185 5186 /* Nothing, tell g_show to print all functions are enabled */ 5187 if (ftrace_hash_empty(fgd->hash) && !*pos) 5188 return FTRACE_GRAPH_EMPTY; 5189 5190 fgd->idx = 0; 5191 fgd->entry = NULL; 5192 return __g_next(m, pos); 5193 } 5194 5195 static void g_stop(struct seq_file *m, void *p) 5196 { 5197 mutex_unlock(&graph_lock); 5198 } 5199 5200 static int g_show(struct seq_file *m, void *v) 5201 { 5202 struct ftrace_func_entry *entry = v; 5203 5204 if (!entry) 5205 return 0; 5206 5207 if (entry == FTRACE_GRAPH_EMPTY) { 5208 struct ftrace_graph_data *fgd = m->private; 5209 5210 if (fgd->type == GRAPH_FILTER_FUNCTION) 5211 seq_puts(m, "#### all functions enabled ####\n"); 5212 else 5213 seq_puts(m, "#### no functions disabled ####\n"); 5214 return 0; 5215 } 5216 5217 seq_printf(m, "%ps\n", (void *)entry->ip); 5218 5219 return 0; 5220 } 5221 5222 static const struct seq_operations ftrace_graph_seq_ops = { 5223 .start = g_start, 5224 .next = g_next, 5225 .stop = g_stop, 5226 .show = g_show, 5227 }; 5228 5229 static int 5230 __ftrace_graph_open(struct inode *inode, struct file *file, 5231 struct ftrace_graph_data *fgd) 5232 { 5233 int ret = 0; 5234 struct ftrace_hash *new_hash = NULL; 5235 5236 if (file->f_mode & FMODE_WRITE) { 5237 const int size_bits = FTRACE_HASH_DEFAULT_BITS; 5238 5239 if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX)) 5240 return -ENOMEM; 5241 5242 if (file->f_flags & O_TRUNC) 5243 new_hash = alloc_ftrace_hash(size_bits); 5244 else 5245 new_hash = alloc_and_copy_ftrace_hash(size_bits, 5246 fgd->hash); 5247 if (!new_hash) { 5248 ret = -ENOMEM; 5249 goto out; 5250 } 5251 } 5252 5253 if (file->f_mode & FMODE_READ) { 5254 ret = seq_open(file, &ftrace_graph_seq_ops); 5255 if (!ret) { 5256 struct seq_file *m = file->private_data; 5257 m->private = fgd; 5258 } else { 5259 /* Failed */ 5260 free_ftrace_hash(new_hash); 5261 new_hash = NULL; 5262 } 5263 } else 5264 file->private_data = fgd; 5265 5266 out: 5267 if (ret < 0 && file->f_mode & FMODE_WRITE) 5268 trace_parser_put(&fgd->parser); 5269 5270 fgd->new_hash = new_hash; 5271 5272 /* 5273 * All uses of fgd->hash must be taken with the graph_lock 5274 * held. The graph_lock is going to be released, so force 5275 * fgd->hash to be reinitialized when it is taken again. 5276 */ 5277 fgd->hash = NULL; 5278 5279 return ret; 5280 } 5281 5282 static int 5283 ftrace_graph_open(struct inode *inode, struct file *file) 5284 { 5285 struct ftrace_graph_data *fgd; 5286 int ret; 5287 5288 if (unlikely(ftrace_disabled)) 5289 return -ENODEV; 5290 5291 fgd = kmalloc(sizeof(*fgd), GFP_KERNEL); 5292 if (fgd == NULL) 5293 return -ENOMEM; 5294 5295 mutex_lock(&graph_lock); 5296 5297 fgd->hash = rcu_dereference_protected(ftrace_graph_hash, 5298 lockdep_is_held(&graph_lock)); 5299 fgd->type = GRAPH_FILTER_FUNCTION; 5300 fgd->seq_ops = &ftrace_graph_seq_ops; 5301 5302 ret = __ftrace_graph_open(inode, file, fgd); 5303 if (ret < 0) 5304 kfree(fgd); 5305 5306 mutex_unlock(&graph_lock); 5307 return ret; 5308 } 5309 5310 static int 5311 ftrace_graph_notrace_open(struct inode *inode, struct file *file) 5312 { 5313 struct ftrace_graph_data *fgd; 5314 int ret; 5315 5316 if (unlikely(ftrace_disabled)) 5317 return -ENODEV; 5318 5319 fgd = kmalloc(sizeof(*fgd), GFP_KERNEL); 5320 if (fgd == NULL) 5321 return -ENOMEM; 5322 5323 mutex_lock(&graph_lock); 5324 5325 fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash, 5326 lockdep_is_held(&graph_lock)); 5327 fgd->type = GRAPH_FILTER_NOTRACE; 5328 fgd->seq_ops = &ftrace_graph_seq_ops; 5329 5330 ret = __ftrace_graph_open(inode, file, fgd); 5331 if (ret < 0) 5332 kfree(fgd); 5333 5334 mutex_unlock(&graph_lock); 5335 return ret; 5336 } 5337 5338 static int 5339 ftrace_graph_release(struct inode *inode, struct file *file) 5340 { 5341 struct ftrace_graph_data *fgd; 5342 struct ftrace_hash *old_hash, *new_hash; 5343 struct trace_parser *parser; 5344 int ret = 0; 5345 5346 if (file->f_mode & FMODE_READ) { 5347 struct seq_file *m = file->private_data; 5348 5349 fgd = m->private; 5350 seq_release(inode, file); 5351 } else { 5352 fgd = file->private_data; 5353 } 5354 5355 5356 if (file->f_mode & FMODE_WRITE) { 5357 5358 parser = &fgd->parser; 5359 5360 if (trace_parser_loaded((parser))) { 5361 parser->buffer[parser->idx] = 0; 5362 ret = ftrace_graph_set_hash(fgd->new_hash, 5363 parser->buffer); 5364 } 5365 5366 trace_parser_put(parser); 5367 5368 new_hash = __ftrace_hash_move(fgd->new_hash); 5369 if (!new_hash) { 5370 ret = -ENOMEM; 5371 goto out; 5372 } 5373 5374 mutex_lock(&graph_lock); 5375 5376 if (fgd->type == GRAPH_FILTER_FUNCTION) { 5377 old_hash = rcu_dereference_protected(ftrace_graph_hash, 5378 lockdep_is_held(&graph_lock)); 5379 rcu_assign_pointer(ftrace_graph_hash, new_hash); 5380 } else { 5381 old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash, 5382 lockdep_is_held(&graph_lock)); 5383 rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash); 5384 } 5385 5386 mutex_unlock(&graph_lock); 5387 5388 /* Wait till all users are no longer using the old hash */ 5389 synchronize_sched(); 5390 5391 free_ftrace_hash(old_hash); 5392 } 5393 5394 out: 5395 free_ftrace_hash(fgd->new_hash); 5396 kfree(fgd); 5397 5398 return ret; 5399 } 5400 5401 static int 5402 ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer) 5403 { 5404 struct ftrace_glob func_g; 5405 struct dyn_ftrace *rec; 5406 struct ftrace_page *pg; 5407 struct ftrace_func_entry *entry; 5408 int fail = 1; 5409 int not; 5410 5411 /* decode regex */ 5412 func_g.type = filter_parse_regex(buffer, strlen(buffer), 5413 &func_g.search, ¬); 5414 5415 func_g.len = strlen(func_g.search); 5416 5417 mutex_lock(&ftrace_lock); 5418 5419 if (unlikely(ftrace_disabled)) { 5420 mutex_unlock(&ftrace_lock); 5421 return -ENODEV; 5422 } 5423 5424 do_for_each_ftrace_rec(pg, rec) { 5425 5426 if (rec->flags & FTRACE_FL_DISABLED) 5427 continue; 5428 5429 if (ftrace_match_record(rec, &func_g, NULL, 0)) { 5430 entry = ftrace_lookup_ip(hash, rec->ip); 5431 5432 if (!not) { 5433 fail = 0; 5434 5435 if (entry) 5436 continue; 5437 if (add_hash_entry(hash, rec->ip) < 0) 5438 goto out; 5439 } else { 5440 if (entry) { 5441 free_hash_entry(hash, entry); 5442 fail = 0; 5443 } 5444 } 5445 } 5446 } while_for_each_ftrace_rec(); 5447 out: 5448 mutex_unlock(&ftrace_lock); 5449 5450 if (fail) 5451 return -EINVAL; 5452 5453 return 0; 5454 } 5455 5456 static ssize_t 5457 ftrace_graph_write(struct file *file, const char __user *ubuf, 5458 size_t cnt, loff_t *ppos) 5459 { 5460 ssize_t read, ret = 0; 5461 struct ftrace_graph_data *fgd = file->private_data; 5462 struct trace_parser *parser; 5463 5464 if (!cnt) 5465 return 0; 5466 5467 /* Read mode uses seq functions */ 5468 if (file->f_mode & FMODE_READ) { 5469 struct seq_file *m = file->private_data; 5470 fgd = m->private; 5471 } 5472 5473 parser = &fgd->parser; 5474 5475 read = trace_get_user(parser, ubuf, cnt, ppos); 5476 5477 if (read >= 0 && trace_parser_loaded(parser) && 5478 !trace_parser_cont(parser)) { 5479 5480 ret = ftrace_graph_set_hash(fgd->new_hash, 5481 parser->buffer); 5482 trace_parser_clear(parser); 5483 } 5484 5485 if (!ret) 5486 ret = read; 5487 5488 return ret; 5489 } 5490 5491 static const struct file_operations ftrace_graph_fops = { 5492 .open = ftrace_graph_open, 5493 .read = seq_read, 5494 .write = ftrace_graph_write, 5495 .llseek = tracing_lseek, 5496 .release = ftrace_graph_release, 5497 }; 5498 5499 static const struct file_operations ftrace_graph_notrace_fops = { 5500 .open = ftrace_graph_notrace_open, 5501 .read = seq_read, 5502 .write = ftrace_graph_write, 5503 .llseek = tracing_lseek, 5504 .release = ftrace_graph_release, 5505 }; 5506 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 5507 5508 void ftrace_create_filter_files(struct ftrace_ops *ops, 5509 struct dentry *parent) 5510 { 5511 5512 trace_create_file("set_ftrace_filter", 0644, parent, 5513 ops, &ftrace_filter_fops); 5514 5515 trace_create_file("set_ftrace_notrace", 0644, parent, 5516 ops, &ftrace_notrace_fops); 5517 } 5518 5519 /* 5520 * The name "destroy_filter_files" is really a misnomer. Although 5521 * in the future, it may actualy delete the files, but this is 5522 * really intended to make sure the ops passed in are disabled 5523 * and that when this function returns, the caller is free to 5524 * free the ops. 5525 * 5526 * The "destroy" name is only to match the "create" name that this 5527 * should be paired with. 5528 */ 5529 void ftrace_destroy_filter_files(struct ftrace_ops *ops) 5530 { 5531 mutex_lock(&ftrace_lock); 5532 if (ops->flags & FTRACE_OPS_FL_ENABLED) 5533 ftrace_shutdown(ops, 0); 5534 ops->flags |= FTRACE_OPS_FL_DELETED; 5535 mutex_unlock(&ftrace_lock); 5536 } 5537 5538 static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer) 5539 { 5540 5541 trace_create_file("available_filter_functions", 0444, 5542 d_tracer, NULL, &ftrace_avail_fops); 5543 5544 trace_create_file("enabled_functions", 0444, 5545 d_tracer, NULL, &ftrace_enabled_fops); 5546 5547 ftrace_create_filter_files(&global_ops, d_tracer); 5548 5549 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 5550 trace_create_file("set_graph_function", 0444, d_tracer, 5551 NULL, 5552 &ftrace_graph_fops); 5553 trace_create_file("set_graph_notrace", 0444, d_tracer, 5554 NULL, 5555 &ftrace_graph_notrace_fops); 5556 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 5557 5558 return 0; 5559 } 5560 5561 static int ftrace_cmp_ips(const void *a, const void *b) 5562 { 5563 const unsigned long *ipa = a; 5564 const unsigned long *ipb = b; 5565 5566 if (*ipa > *ipb) 5567 return 1; 5568 if (*ipa < *ipb) 5569 return -1; 5570 return 0; 5571 } 5572 5573 static int ftrace_process_locs(struct module *mod, 5574 unsigned long *start, 5575 unsigned long *end) 5576 { 5577 struct ftrace_page *start_pg; 5578 struct ftrace_page *pg; 5579 struct dyn_ftrace *rec; 5580 unsigned long count; 5581 unsigned long *p; 5582 unsigned long addr; 5583 unsigned long flags = 0; /* Shut up gcc */ 5584 int ret = -ENOMEM; 5585 5586 count = end - start; 5587 5588 if (!count) 5589 return 0; 5590 5591 sort(start, count, sizeof(*start), 5592 ftrace_cmp_ips, NULL); 5593 5594 start_pg = ftrace_allocate_pages(count); 5595 if (!start_pg) 5596 return -ENOMEM; 5597 5598 mutex_lock(&ftrace_lock); 5599 5600 /* 5601 * Core and each module needs their own pages, as 5602 * modules will free them when they are removed. 5603 * Force a new page to be allocated for modules. 5604 */ 5605 if (!mod) { 5606 WARN_ON(ftrace_pages || ftrace_pages_start); 5607 /* First initialization */ 5608 ftrace_pages = ftrace_pages_start = start_pg; 5609 } else { 5610 if (!ftrace_pages) 5611 goto out; 5612 5613 if (WARN_ON(ftrace_pages->next)) { 5614 /* Hmm, we have free pages? */ 5615 while (ftrace_pages->next) 5616 ftrace_pages = ftrace_pages->next; 5617 } 5618 5619 ftrace_pages->next = start_pg; 5620 } 5621 5622 p = start; 5623 pg = start_pg; 5624 while (p < end) { 5625 addr = ftrace_call_adjust(*p++); 5626 /* 5627 * Some architecture linkers will pad between 5628 * the different mcount_loc sections of different 5629 * object files to satisfy alignments. 5630 * Skip any NULL pointers. 5631 */ 5632 if (!addr) 5633 continue; 5634 5635 if (pg->index == pg->size) { 5636 /* We should have allocated enough */ 5637 if (WARN_ON(!pg->next)) 5638 break; 5639 pg = pg->next; 5640 } 5641 5642 rec = &pg->records[pg->index++]; 5643 rec->ip = addr; 5644 } 5645 5646 /* We should have used all pages */ 5647 WARN_ON(pg->next); 5648 5649 /* Assign the last page to ftrace_pages */ 5650 ftrace_pages = pg; 5651 5652 /* 5653 * We only need to disable interrupts on start up 5654 * because we are modifying code that an interrupt 5655 * may execute, and the modification is not atomic. 5656 * But for modules, nothing runs the code we modify 5657 * until we are finished with it, and there's no 5658 * reason to cause large interrupt latencies while we do it. 5659 */ 5660 if (!mod) 5661 local_irq_save(flags); 5662 ftrace_update_code(mod, start_pg); 5663 if (!mod) 5664 local_irq_restore(flags); 5665 ret = 0; 5666 out: 5667 mutex_unlock(&ftrace_lock); 5668 5669 return ret; 5670 } 5671 5672 #ifdef CONFIG_MODULES 5673 5674 #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next) 5675 5676 static int referenced_filters(struct dyn_ftrace *rec) 5677 { 5678 struct ftrace_ops *ops; 5679 int cnt = 0; 5680 5681 for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) { 5682 if (ops_references_rec(ops, rec)) 5683 cnt++; 5684 } 5685 5686 return cnt; 5687 } 5688 5689 void ftrace_release_mod(struct module *mod) 5690 { 5691 struct dyn_ftrace *rec; 5692 struct ftrace_page **last_pg; 5693 struct ftrace_page *pg; 5694 int order; 5695 5696 mutex_lock(&ftrace_lock); 5697 5698 if (ftrace_disabled) 5699 goto out_unlock; 5700 5701 /* 5702 * Each module has its own ftrace_pages, remove 5703 * them from the list. 5704 */ 5705 last_pg = &ftrace_pages_start; 5706 for (pg = ftrace_pages_start; pg; pg = *last_pg) { 5707 rec = &pg->records[0]; 5708 if (within_module_core(rec->ip, mod)) { 5709 /* 5710 * As core pages are first, the first 5711 * page should never be a module page. 5712 */ 5713 if (WARN_ON(pg == ftrace_pages_start)) 5714 goto out_unlock; 5715 5716 /* Check if we are deleting the last page */ 5717 if (pg == ftrace_pages) 5718 ftrace_pages = next_to_ftrace_page(last_pg); 5719 5720 ftrace_update_tot_cnt -= pg->index; 5721 *last_pg = pg->next; 5722 order = get_count_order(pg->size / ENTRIES_PER_PAGE); 5723 free_pages((unsigned long)pg->records, order); 5724 kfree(pg); 5725 } else 5726 last_pg = &pg->next; 5727 } 5728 out_unlock: 5729 mutex_unlock(&ftrace_lock); 5730 } 5731 5732 void ftrace_module_enable(struct module *mod) 5733 { 5734 struct dyn_ftrace *rec; 5735 struct ftrace_page *pg; 5736 5737 mutex_lock(&ftrace_lock); 5738 5739 if (ftrace_disabled) 5740 goto out_unlock; 5741 5742 /* 5743 * If the tracing is enabled, go ahead and enable the record. 5744 * 5745 * The reason not to enable the record immediatelly is the 5746 * inherent check of ftrace_make_nop/ftrace_make_call for 5747 * correct previous instructions. Making first the NOP 5748 * conversion puts the module to the correct state, thus 5749 * passing the ftrace_make_call check. 5750 * 5751 * We also delay this to after the module code already set the 5752 * text to read-only, as we now need to set it back to read-write 5753 * so that we can modify the text. 5754 */ 5755 if (ftrace_start_up) 5756 ftrace_arch_code_modify_prepare(); 5757 5758 do_for_each_ftrace_rec(pg, rec) { 5759 int cnt; 5760 /* 5761 * do_for_each_ftrace_rec() is a double loop. 5762 * module text shares the pg. If a record is 5763 * not part of this module, then skip this pg, 5764 * which the "break" will do. 5765 */ 5766 if (!within_module_core(rec->ip, mod)) 5767 break; 5768 5769 cnt = 0; 5770 5771 /* 5772 * When adding a module, we need to check if tracers are 5773 * currently enabled and if they are, and can trace this record, 5774 * we need to enable the module functions as well as update the 5775 * reference counts for those function records. 5776 */ 5777 if (ftrace_start_up) 5778 cnt += referenced_filters(rec); 5779 5780 /* This clears FTRACE_FL_DISABLED */ 5781 rec->flags = cnt; 5782 5783 if (ftrace_start_up && cnt) { 5784 int failed = __ftrace_replace_code(rec, 1); 5785 if (failed) { 5786 ftrace_bug(failed, rec); 5787 goto out_loop; 5788 } 5789 } 5790 5791 } while_for_each_ftrace_rec(); 5792 5793 out_loop: 5794 if (ftrace_start_up) 5795 ftrace_arch_code_modify_post_process(); 5796 5797 out_unlock: 5798 mutex_unlock(&ftrace_lock); 5799 5800 process_cached_mods(mod->name); 5801 } 5802 5803 void ftrace_module_init(struct module *mod) 5804 { 5805 if (ftrace_disabled || !mod->num_ftrace_callsites) 5806 return; 5807 5808 ftrace_process_locs(mod, mod->ftrace_callsites, 5809 mod->ftrace_callsites + mod->num_ftrace_callsites); 5810 } 5811 #endif /* CONFIG_MODULES */ 5812 5813 void __init ftrace_free_init_mem(void) 5814 { 5815 unsigned long start = (unsigned long)(&__init_begin); 5816 unsigned long end = (unsigned long)(&__init_end); 5817 struct ftrace_page **last_pg = &ftrace_pages_start; 5818 struct ftrace_page *pg; 5819 struct dyn_ftrace *rec; 5820 struct dyn_ftrace key; 5821 int order; 5822 5823 key.ip = start; 5824 key.flags = end; /* overload flags, as it is unsigned long */ 5825 5826 mutex_lock(&ftrace_lock); 5827 5828 for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) { 5829 if (end < pg->records[0].ip || 5830 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) 5831 continue; 5832 again: 5833 rec = bsearch(&key, pg->records, pg->index, 5834 sizeof(struct dyn_ftrace), 5835 ftrace_cmp_recs); 5836 if (!rec) 5837 continue; 5838 pg->index--; 5839 ftrace_update_tot_cnt--; 5840 if (!pg->index) { 5841 *last_pg = pg->next; 5842 order = get_count_order(pg->size / ENTRIES_PER_PAGE); 5843 free_pages((unsigned long)pg->records, order); 5844 kfree(pg); 5845 pg = container_of(last_pg, struct ftrace_page, next); 5846 if (!(*last_pg)) 5847 ftrace_pages = pg; 5848 continue; 5849 } 5850 memmove(rec, rec + 1, 5851 (pg->index - (rec - pg->records)) * sizeof(*rec)); 5852 /* More than one function may be in this block */ 5853 goto again; 5854 } 5855 mutex_unlock(&ftrace_lock); 5856 } 5857 5858 void __init ftrace_init(void) 5859 { 5860 extern unsigned long __start_mcount_loc[]; 5861 extern unsigned long __stop_mcount_loc[]; 5862 unsigned long count, flags; 5863 int ret; 5864 5865 local_irq_save(flags); 5866 ret = ftrace_dyn_arch_init(); 5867 local_irq_restore(flags); 5868 if (ret) 5869 goto failed; 5870 5871 count = __stop_mcount_loc - __start_mcount_loc; 5872 if (!count) { 5873 pr_info("ftrace: No functions to be traced?\n"); 5874 goto failed; 5875 } 5876 5877 pr_info("ftrace: allocating %ld entries in %ld pages\n", 5878 count, count / ENTRIES_PER_PAGE + 1); 5879 5880 last_ftrace_enabled = ftrace_enabled = 1; 5881 5882 ret = ftrace_process_locs(NULL, 5883 __start_mcount_loc, 5884 __stop_mcount_loc); 5885 5886 set_ftrace_early_filters(); 5887 5888 return; 5889 failed: 5890 ftrace_disabled = 1; 5891 } 5892 5893 /* Do nothing if arch does not support this */ 5894 void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops) 5895 { 5896 } 5897 5898 static void ftrace_update_trampoline(struct ftrace_ops *ops) 5899 { 5900 arch_ftrace_update_trampoline(ops); 5901 } 5902 5903 void ftrace_init_trace_array(struct trace_array *tr) 5904 { 5905 INIT_LIST_HEAD(&tr->func_probes); 5906 INIT_LIST_HEAD(&tr->mod_trace); 5907 INIT_LIST_HEAD(&tr->mod_notrace); 5908 } 5909 #else 5910 5911 static struct ftrace_ops global_ops = { 5912 .func = ftrace_stub, 5913 .flags = FTRACE_OPS_FL_RECURSION_SAFE | 5914 FTRACE_OPS_FL_INITIALIZED | 5915 FTRACE_OPS_FL_PID, 5916 }; 5917 5918 static int __init ftrace_nodyn_init(void) 5919 { 5920 ftrace_enabled = 1; 5921 return 0; 5922 } 5923 core_initcall(ftrace_nodyn_init); 5924 5925 static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; } 5926 static inline void ftrace_startup_enable(int command) { } 5927 static inline void ftrace_startup_all(int command) { } 5928 /* Keep as macros so we do not need to define the commands */ 5929 # define ftrace_startup(ops, command) \ 5930 ({ \ 5931 int ___ret = __register_ftrace_function(ops); \ 5932 if (!___ret) \ 5933 (ops)->flags |= FTRACE_OPS_FL_ENABLED; \ 5934 ___ret; \ 5935 }) 5936 # define ftrace_shutdown(ops, command) \ 5937 ({ \ 5938 int ___ret = __unregister_ftrace_function(ops); \ 5939 if (!___ret) \ 5940 (ops)->flags &= ~FTRACE_OPS_FL_ENABLED; \ 5941 ___ret; \ 5942 }) 5943 5944 # define ftrace_startup_sysctl() do { } while (0) 5945 # define ftrace_shutdown_sysctl() do { } while (0) 5946 5947 static inline int 5948 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) 5949 { 5950 return 1; 5951 } 5952 5953 static void ftrace_update_trampoline(struct ftrace_ops *ops) 5954 { 5955 } 5956 5957 #endif /* CONFIG_DYNAMIC_FTRACE */ 5958 5959 __init void ftrace_init_global_array_ops(struct trace_array *tr) 5960 { 5961 tr->ops = &global_ops; 5962 tr->ops->private = tr; 5963 ftrace_init_trace_array(tr); 5964 } 5965 5966 void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func) 5967 { 5968 /* If we filter on pids, update to use the pid function */ 5969 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) { 5970 if (WARN_ON(tr->ops->func != ftrace_stub)) 5971 printk("ftrace ops had %pS for function\n", 5972 tr->ops->func); 5973 } 5974 tr->ops->func = func; 5975 tr->ops->private = tr; 5976 } 5977 5978 void ftrace_reset_array_ops(struct trace_array *tr) 5979 { 5980 tr->ops->func = ftrace_stub; 5981 } 5982 5983 static inline void 5984 __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 5985 struct ftrace_ops *ignored, struct pt_regs *regs) 5986 { 5987 struct ftrace_ops *op; 5988 int bit; 5989 5990 bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX); 5991 if (bit < 0) 5992 return; 5993 5994 /* 5995 * Some of the ops may be dynamically allocated, 5996 * they must be freed after a synchronize_sched(). 5997 */ 5998 preempt_disable_notrace(); 5999 6000 do_for_each_ftrace_op(op, ftrace_ops_list) { 6001 /* 6002 * Check the following for each ops before calling their func: 6003 * if RCU flag is set, then rcu_is_watching() must be true 6004 * if PER_CPU is set, then ftrace_function_local_disable() 6005 * must be false 6006 * Otherwise test if the ip matches the ops filter 6007 * 6008 * If any of the above fails then the op->func() is not executed. 6009 */ 6010 if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) && 6011 (!(op->flags & FTRACE_OPS_FL_PER_CPU) || 6012 !ftrace_function_local_disabled(op)) && 6013 ftrace_ops_test(op, ip, regs)) { 6014 6015 if (FTRACE_WARN_ON(!op->func)) { 6016 pr_warn("op=%p %pS\n", op, op); 6017 goto out; 6018 } 6019 op->func(ip, parent_ip, op, regs); 6020 } 6021 } while_for_each_ftrace_op(op); 6022 out: 6023 preempt_enable_notrace(); 6024 trace_clear_recursion(bit); 6025 } 6026 6027 /* 6028 * Some archs only support passing ip and parent_ip. Even though 6029 * the list function ignores the op parameter, we do not want any 6030 * C side effects, where a function is called without the caller 6031 * sending a third parameter. 6032 * Archs are to support both the regs and ftrace_ops at the same time. 6033 * If they support ftrace_ops, it is assumed they support regs. 6034 * If call backs want to use regs, they must either check for regs 6035 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS. 6036 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved. 6037 * An architecture can pass partial regs with ftrace_ops and still 6038 * set the ARCH_SUPPORTS_FTRACE_OPS. 6039 */ 6040 #if ARCH_SUPPORTS_FTRACE_OPS 6041 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 6042 struct ftrace_ops *op, struct pt_regs *regs) 6043 { 6044 __ftrace_ops_list_func(ip, parent_ip, NULL, regs); 6045 } 6046 #else 6047 static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip) 6048 { 6049 __ftrace_ops_list_func(ip, parent_ip, NULL, NULL); 6050 } 6051 #endif 6052 6053 /* 6054 * If there's only one function registered but it does not support 6055 * recursion, needs RCU protection and/or requires per cpu handling, then 6056 * this function will be called by the mcount trampoline. 6057 */ 6058 static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip, 6059 struct ftrace_ops *op, struct pt_regs *regs) 6060 { 6061 int bit; 6062 6063 if ((op->flags & FTRACE_OPS_FL_RCU) && !rcu_is_watching()) 6064 return; 6065 6066 bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX); 6067 if (bit < 0) 6068 return; 6069 6070 preempt_disable_notrace(); 6071 6072 if (!(op->flags & FTRACE_OPS_FL_PER_CPU) || 6073 !ftrace_function_local_disabled(op)) { 6074 op->func(ip, parent_ip, op, regs); 6075 } 6076 6077 preempt_enable_notrace(); 6078 trace_clear_recursion(bit); 6079 } 6080 6081 /** 6082 * ftrace_ops_get_func - get the function a trampoline should call 6083 * @ops: the ops to get the function for 6084 * 6085 * Normally the mcount trampoline will call the ops->func, but there 6086 * are times that it should not. For example, if the ops does not 6087 * have its own recursion protection, then it should call the 6088 * ftrace_ops_assist_func() instead. 6089 * 6090 * Returns the function that the trampoline should call for @ops. 6091 */ 6092 ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops) 6093 { 6094 /* 6095 * If the function does not handle recursion, needs to be RCU safe, 6096 * or does per cpu logic, then we need to call the assist handler. 6097 */ 6098 if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE) || 6099 ops->flags & (FTRACE_OPS_FL_RCU | FTRACE_OPS_FL_PER_CPU)) 6100 return ftrace_ops_assist_func; 6101 6102 return ops->func; 6103 } 6104 6105 static void 6106 ftrace_filter_pid_sched_switch_probe(void *data, bool preempt, 6107 struct task_struct *prev, struct task_struct *next) 6108 { 6109 struct trace_array *tr = data; 6110 struct trace_pid_list *pid_list; 6111 6112 pid_list = rcu_dereference_sched(tr->function_pids); 6113 6114 this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid, 6115 trace_ignore_this_task(pid_list, next)); 6116 } 6117 6118 static void 6119 ftrace_pid_follow_sched_process_fork(void *data, 6120 struct task_struct *self, 6121 struct task_struct *task) 6122 { 6123 struct trace_pid_list *pid_list; 6124 struct trace_array *tr = data; 6125 6126 pid_list = rcu_dereference_sched(tr->function_pids); 6127 trace_filter_add_remove_task(pid_list, self, task); 6128 } 6129 6130 static void 6131 ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task) 6132 { 6133 struct trace_pid_list *pid_list; 6134 struct trace_array *tr = data; 6135 6136 pid_list = rcu_dereference_sched(tr->function_pids); 6137 trace_filter_add_remove_task(pid_list, NULL, task); 6138 } 6139 6140 void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) 6141 { 6142 if (enable) { 6143 register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork, 6144 tr); 6145 register_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit, 6146 tr); 6147 } else { 6148 unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork, 6149 tr); 6150 unregister_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit, 6151 tr); 6152 } 6153 } 6154 6155 static void clear_ftrace_pids(struct trace_array *tr) 6156 { 6157 struct trace_pid_list *pid_list; 6158 int cpu; 6159 6160 pid_list = rcu_dereference_protected(tr->function_pids, 6161 lockdep_is_held(&ftrace_lock)); 6162 if (!pid_list) 6163 return; 6164 6165 unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr); 6166 6167 for_each_possible_cpu(cpu) 6168 per_cpu_ptr(tr->trace_buffer.data, cpu)->ftrace_ignore_pid = false; 6169 6170 rcu_assign_pointer(tr->function_pids, NULL); 6171 6172 /* Wait till all users are no longer using pid filtering */ 6173 synchronize_sched(); 6174 6175 trace_free_pid_list(pid_list); 6176 } 6177 6178 void ftrace_clear_pids(struct trace_array *tr) 6179 { 6180 mutex_lock(&ftrace_lock); 6181 6182 clear_ftrace_pids(tr); 6183 6184 mutex_unlock(&ftrace_lock); 6185 } 6186 6187 static void ftrace_pid_reset(struct trace_array *tr) 6188 { 6189 mutex_lock(&ftrace_lock); 6190 clear_ftrace_pids(tr); 6191 6192 ftrace_update_pid_func(); 6193 ftrace_startup_all(0); 6194 6195 mutex_unlock(&ftrace_lock); 6196 } 6197 6198 /* Greater than any max PID */ 6199 #define FTRACE_NO_PIDS (void *)(PID_MAX_LIMIT + 1) 6200 6201 static void *fpid_start(struct seq_file *m, loff_t *pos) 6202 __acquires(RCU) 6203 { 6204 struct trace_pid_list *pid_list; 6205 struct trace_array *tr = m->private; 6206 6207 mutex_lock(&ftrace_lock); 6208 rcu_read_lock_sched(); 6209 6210 pid_list = rcu_dereference_sched(tr->function_pids); 6211 6212 if (!pid_list) 6213 return !(*pos) ? FTRACE_NO_PIDS : NULL; 6214 6215 return trace_pid_start(pid_list, pos); 6216 } 6217 6218 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos) 6219 { 6220 struct trace_array *tr = m->private; 6221 struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids); 6222 6223 if (v == FTRACE_NO_PIDS) 6224 return NULL; 6225 6226 return trace_pid_next(pid_list, v, pos); 6227 } 6228 6229 static void fpid_stop(struct seq_file *m, void *p) 6230 __releases(RCU) 6231 { 6232 rcu_read_unlock_sched(); 6233 mutex_unlock(&ftrace_lock); 6234 } 6235 6236 static int fpid_show(struct seq_file *m, void *v) 6237 { 6238 if (v == FTRACE_NO_PIDS) { 6239 seq_puts(m, "no pid\n"); 6240 return 0; 6241 } 6242 6243 return trace_pid_show(m, v); 6244 } 6245 6246 static const struct seq_operations ftrace_pid_sops = { 6247 .start = fpid_start, 6248 .next = fpid_next, 6249 .stop = fpid_stop, 6250 .show = fpid_show, 6251 }; 6252 6253 static int 6254 ftrace_pid_open(struct inode *inode, struct file *file) 6255 { 6256 struct trace_array *tr = inode->i_private; 6257 struct seq_file *m; 6258 int ret = 0; 6259 6260 if (trace_array_get(tr) < 0) 6261 return -ENODEV; 6262 6263 if ((file->f_mode & FMODE_WRITE) && 6264 (file->f_flags & O_TRUNC)) 6265 ftrace_pid_reset(tr); 6266 6267 ret = seq_open(file, &ftrace_pid_sops); 6268 if (ret < 0) { 6269 trace_array_put(tr); 6270 } else { 6271 m = file->private_data; 6272 /* copy tr over to seq ops */ 6273 m->private = tr; 6274 } 6275 6276 return ret; 6277 } 6278 6279 static void ignore_task_cpu(void *data) 6280 { 6281 struct trace_array *tr = data; 6282 struct trace_pid_list *pid_list; 6283 6284 /* 6285 * This function is called by on_each_cpu() while the 6286 * event_mutex is held. 6287 */ 6288 pid_list = rcu_dereference_protected(tr->function_pids, 6289 mutex_is_locked(&ftrace_lock)); 6290 6291 this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid, 6292 trace_ignore_this_task(pid_list, current)); 6293 } 6294 6295 static ssize_t 6296 ftrace_pid_write(struct file *filp, const char __user *ubuf, 6297 size_t cnt, loff_t *ppos) 6298 { 6299 struct seq_file *m = filp->private_data; 6300 struct trace_array *tr = m->private; 6301 struct trace_pid_list *filtered_pids = NULL; 6302 struct trace_pid_list *pid_list; 6303 ssize_t ret; 6304 6305 if (!cnt) 6306 return 0; 6307 6308 mutex_lock(&ftrace_lock); 6309 6310 filtered_pids = rcu_dereference_protected(tr->function_pids, 6311 lockdep_is_held(&ftrace_lock)); 6312 6313 ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt); 6314 if (ret < 0) 6315 goto out; 6316 6317 rcu_assign_pointer(tr->function_pids, pid_list); 6318 6319 if (filtered_pids) { 6320 synchronize_sched(); 6321 trace_free_pid_list(filtered_pids); 6322 } else if (pid_list) { 6323 /* Register a probe to set whether to ignore the tracing of a task */ 6324 register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr); 6325 } 6326 6327 /* 6328 * Ignoring of pids is done at task switch. But we have to 6329 * check for those tasks that are currently running. 6330 * Always do this in case a pid was appended or removed. 6331 */ 6332 on_each_cpu(ignore_task_cpu, tr, 1); 6333 6334 ftrace_update_pid_func(); 6335 ftrace_startup_all(0); 6336 out: 6337 mutex_unlock(&ftrace_lock); 6338 6339 if (ret > 0) 6340 *ppos += ret; 6341 6342 return ret; 6343 } 6344 6345 static int 6346 ftrace_pid_release(struct inode *inode, struct file *file) 6347 { 6348 struct trace_array *tr = inode->i_private; 6349 6350 trace_array_put(tr); 6351 6352 return seq_release(inode, file); 6353 } 6354 6355 static const struct file_operations ftrace_pid_fops = { 6356 .open = ftrace_pid_open, 6357 .write = ftrace_pid_write, 6358 .read = seq_read, 6359 .llseek = tracing_lseek, 6360 .release = ftrace_pid_release, 6361 }; 6362 6363 void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer) 6364 { 6365 trace_create_file("set_ftrace_pid", 0644, d_tracer, 6366 tr, &ftrace_pid_fops); 6367 } 6368 6369 void __init ftrace_init_tracefs_toplevel(struct trace_array *tr, 6370 struct dentry *d_tracer) 6371 { 6372 /* Only the top level directory has the dyn_tracefs and profile */ 6373 WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL)); 6374 6375 ftrace_init_dyn_tracefs(d_tracer); 6376 ftrace_profile_tracefs(d_tracer); 6377 } 6378 6379 /** 6380 * ftrace_kill - kill ftrace 6381 * 6382 * This function should be used by panic code. It stops ftrace 6383 * but in a not so nice way. If you need to simply kill ftrace 6384 * from a non-atomic section, use ftrace_kill. 6385 */ 6386 void ftrace_kill(void) 6387 { 6388 ftrace_disabled = 1; 6389 ftrace_enabled = 0; 6390 clear_ftrace_function(); 6391 } 6392 6393 /** 6394 * Test if ftrace is dead or not. 6395 */ 6396 int ftrace_is_dead(void) 6397 { 6398 return ftrace_disabled; 6399 } 6400 6401 /** 6402 * register_ftrace_function - register a function for profiling 6403 * @ops - ops structure that holds the function for profiling. 6404 * 6405 * Register a function to be called by all functions in the 6406 * kernel. 6407 * 6408 * Note: @ops->func and all the functions it calls must be labeled 6409 * with "notrace", otherwise it will go into a 6410 * recursive loop. 6411 */ 6412 int register_ftrace_function(struct ftrace_ops *ops) 6413 { 6414 int ret = -1; 6415 6416 ftrace_ops_init(ops); 6417 6418 mutex_lock(&ftrace_lock); 6419 6420 ret = ftrace_startup(ops, 0); 6421 6422 mutex_unlock(&ftrace_lock); 6423 6424 return ret; 6425 } 6426 EXPORT_SYMBOL_GPL(register_ftrace_function); 6427 6428 /** 6429 * unregister_ftrace_function - unregister a function for profiling. 6430 * @ops - ops structure that holds the function to unregister 6431 * 6432 * Unregister a function that was added to be called by ftrace profiling. 6433 */ 6434 int unregister_ftrace_function(struct ftrace_ops *ops) 6435 { 6436 int ret; 6437 6438 mutex_lock(&ftrace_lock); 6439 ret = ftrace_shutdown(ops, 0); 6440 mutex_unlock(&ftrace_lock); 6441 6442 return ret; 6443 } 6444 EXPORT_SYMBOL_GPL(unregister_ftrace_function); 6445 6446 int 6447 ftrace_enable_sysctl(struct ctl_table *table, int write, 6448 void __user *buffer, size_t *lenp, 6449 loff_t *ppos) 6450 { 6451 int ret = -ENODEV; 6452 6453 mutex_lock(&ftrace_lock); 6454 6455 if (unlikely(ftrace_disabled)) 6456 goto out; 6457 6458 ret = proc_dointvec(table, write, buffer, lenp, ppos); 6459 6460 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled)) 6461 goto out; 6462 6463 last_ftrace_enabled = !!ftrace_enabled; 6464 6465 if (ftrace_enabled) { 6466 6467 /* we are starting ftrace again */ 6468 if (rcu_dereference_protected(ftrace_ops_list, 6469 lockdep_is_held(&ftrace_lock)) != &ftrace_list_end) 6470 update_ftrace_function(); 6471 6472 ftrace_startup_sysctl(); 6473 6474 } else { 6475 /* stopping ftrace calls (just send to ftrace_stub) */ 6476 ftrace_trace_function = ftrace_stub; 6477 6478 ftrace_shutdown_sysctl(); 6479 } 6480 6481 out: 6482 mutex_unlock(&ftrace_lock); 6483 return ret; 6484 } 6485 6486 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 6487 6488 static struct ftrace_ops graph_ops = { 6489 .func = ftrace_stub, 6490 .flags = FTRACE_OPS_FL_RECURSION_SAFE | 6491 FTRACE_OPS_FL_INITIALIZED | 6492 FTRACE_OPS_FL_PID | 6493 FTRACE_OPS_FL_STUB, 6494 #ifdef FTRACE_GRAPH_TRAMP_ADDR 6495 .trampoline = FTRACE_GRAPH_TRAMP_ADDR, 6496 /* trampoline_size is only needed for dynamically allocated tramps */ 6497 #endif 6498 ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash) 6499 }; 6500 6501 void ftrace_graph_sleep_time_control(bool enable) 6502 { 6503 fgraph_sleep_time = enable; 6504 } 6505 6506 void ftrace_graph_graph_time_control(bool enable) 6507 { 6508 fgraph_graph_time = enable; 6509 } 6510 6511 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) 6512 { 6513 return 0; 6514 } 6515 6516 /* The callbacks that hook a function */ 6517 trace_func_graph_ret_t ftrace_graph_return = 6518 (trace_func_graph_ret_t)ftrace_stub; 6519 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub; 6520 static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub; 6521 6522 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */ 6523 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list) 6524 { 6525 int i; 6526 int ret = 0; 6527 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE; 6528 struct task_struct *g, *t; 6529 6530 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) { 6531 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH 6532 * sizeof(struct ftrace_ret_stack), 6533 GFP_KERNEL); 6534 if (!ret_stack_list[i]) { 6535 start = 0; 6536 end = i; 6537 ret = -ENOMEM; 6538 goto free; 6539 } 6540 } 6541 6542 read_lock(&tasklist_lock); 6543 do_each_thread(g, t) { 6544 if (start == end) { 6545 ret = -EAGAIN; 6546 goto unlock; 6547 } 6548 6549 if (t->ret_stack == NULL) { 6550 atomic_set(&t->tracing_graph_pause, 0); 6551 atomic_set(&t->trace_overrun, 0); 6552 t->curr_ret_stack = -1; 6553 /* Make sure the tasks see the -1 first: */ 6554 smp_wmb(); 6555 t->ret_stack = ret_stack_list[start++]; 6556 } 6557 } while_each_thread(g, t); 6558 6559 unlock: 6560 read_unlock(&tasklist_lock); 6561 free: 6562 for (i = start; i < end; i++) 6563 kfree(ret_stack_list[i]); 6564 return ret; 6565 } 6566 6567 static void 6568 ftrace_graph_probe_sched_switch(void *ignore, bool preempt, 6569 struct task_struct *prev, struct task_struct *next) 6570 { 6571 unsigned long long timestamp; 6572 int index; 6573 6574 /* 6575 * Does the user want to count the time a function was asleep. 6576 * If so, do not update the time stamps. 6577 */ 6578 if (fgraph_sleep_time) 6579 return; 6580 6581 timestamp = trace_clock_local(); 6582 6583 prev->ftrace_timestamp = timestamp; 6584 6585 /* only process tasks that we timestamped */ 6586 if (!next->ftrace_timestamp) 6587 return; 6588 6589 /* 6590 * Update all the counters in next to make up for the 6591 * time next was sleeping. 6592 */ 6593 timestamp -= next->ftrace_timestamp; 6594 6595 for (index = next->curr_ret_stack; index >= 0; index--) 6596 next->ret_stack[index].calltime += timestamp; 6597 } 6598 6599 /* Allocate a return stack for each task */ 6600 static int start_graph_tracing(void) 6601 { 6602 struct ftrace_ret_stack **ret_stack_list; 6603 int ret, cpu; 6604 6605 ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE * 6606 sizeof(struct ftrace_ret_stack *), 6607 GFP_KERNEL); 6608 6609 if (!ret_stack_list) 6610 return -ENOMEM; 6611 6612 /* The cpu_boot init_task->ret_stack will never be freed */ 6613 for_each_online_cpu(cpu) { 6614 if (!idle_task(cpu)->ret_stack) 6615 ftrace_graph_init_idle_task(idle_task(cpu), cpu); 6616 } 6617 6618 do { 6619 ret = alloc_retstack_tasklist(ret_stack_list); 6620 } while (ret == -EAGAIN); 6621 6622 if (!ret) { 6623 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); 6624 if (ret) 6625 pr_info("ftrace_graph: Couldn't activate tracepoint" 6626 " probe to kernel_sched_switch\n"); 6627 } 6628 6629 kfree(ret_stack_list); 6630 return ret; 6631 } 6632 6633 /* 6634 * Hibernation protection. 6635 * The state of the current task is too much unstable during 6636 * suspend/restore to disk. We want to protect against that. 6637 */ 6638 static int 6639 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state, 6640 void *unused) 6641 { 6642 switch (state) { 6643 case PM_HIBERNATION_PREPARE: 6644 pause_graph_tracing(); 6645 break; 6646 6647 case PM_POST_HIBERNATION: 6648 unpause_graph_tracing(); 6649 break; 6650 } 6651 return NOTIFY_DONE; 6652 } 6653 6654 static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace) 6655 { 6656 if (!ftrace_ops_test(&global_ops, trace->func, NULL)) 6657 return 0; 6658 return __ftrace_graph_entry(trace); 6659 } 6660 6661 /* 6662 * The function graph tracer should only trace the functions defined 6663 * by set_ftrace_filter and set_ftrace_notrace. If another function 6664 * tracer ops is registered, the graph tracer requires testing the 6665 * function against the global ops, and not just trace any function 6666 * that any ftrace_ops registered. 6667 */ 6668 static void update_function_graph_func(void) 6669 { 6670 struct ftrace_ops *op; 6671 bool do_test = false; 6672 6673 /* 6674 * The graph and global ops share the same set of functions 6675 * to test. If any other ops is on the list, then 6676 * the graph tracing needs to test if its the function 6677 * it should call. 6678 */ 6679 do_for_each_ftrace_op(op, ftrace_ops_list) { 6680 if (op != &global_ops && op != &graph_ops && 6681 op != &ftrace_list_end) { 6682 do_test = true; 6683 /* in double loop, break out with goto */ 6684 goto out; 6685 } 6686 } while_for_each_ftrace_op(op); 6687 out: 6688 if (do_test) 6689 ftrace_graph_entry = ftrace_graph_entry_test; 6690 else 6691 ftrace_graph_entry = __ftrace_graph_entry; 6692 } 6693 6694 static struct notifier_block ftrace_suspend_notifier = { 6695 .notifier_call = ftrace_suspend_notifier_call, 6696 }; 6697 6698 int register_ftrace_graph(trace_func_graph_ret_t retfunc, 6699 trace_func_graph_ent_t entryfunc) 6700 { 6701 int ret = 0; 6702 6703 mutex_lock(&ftrace_lock); 6704 6705 /* we currently allow only one tracer registered at a time */ 6706 if (ftrace_graph_active) { 6707 ret = -EBUSY; 6708 goto out; 6709 } 6710 6711 register_pm_notifier(&ftrace_suspend_notifier); 6712 6713 ftrace_graph_active++; 6714 ret = start_graph_tracing(); 6715 if (ret) { 6716 ftrace_graph_active--; 6717 goto out; 6718 } 6719 6720 ftrace_graph_return = retfunc; 6721 6722 /* 6723 * Update the indirect function to the entryfunc, and the 6724 * function that gets called to the entry_test first. Then 6725 * call the update fgraph entry function to determine if 6726 * the entryfunc should be called directly or not. 6727 */ 6728 __ftrace_graph_entry = entryfunc; 6729 ftrace_graph_entry = ftrace_graph_entry_test; 6730 update_function_graph_func(); 6731 6732 ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET); 6733 out: 6734 mutex_unlock(&ftrace_lock); 6735 return ret; 6736 } 6737 6738 void unregister_ftrace_graph(void) 6739 { 6740 mutex_lock(&ftrace_lock); 6741 6742 if (unlikely(!ftrace_graph_active)) 6743 goto out; 6744 6745 ftrace_graph_active--; 6746 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; 6747 ftrace_graph_entry = ftrace_graph_entry_stub; 6748 __ftrace_graph_entry = ftrace_graph_entry_stub; 6749 ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET); 6750 unregister_pm_notifier(&ftrace_suspend_notifier); 6751 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); 6752 6753 #ifdef CONFIG_DYNAMIC_FTRACE 6754 /* 6755 * Function graph does not allocate the trampoline, but 6756 * other global_ops do. We need to reset the ALLOC_TRAMP flag 6757 * if one was used. 6758 */ 6759 global_ops.trampoline = save_global_trampoline; 6760 if (save_global_flags & FTRACE_OPS_FL_ALLOC_TRAMP) 6761 global_ops.flags |= FTRACE_OPS_FL_ALLOC_TRAMP; 6762 #endif 6763 6764 out: 6765 mutex_unlock(&ftrace_lock); 6766 } 6767 6768 static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack); 6769 6770 static void 6771 graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack) 6772 { 6773 atomic_set(&t->tracing_graph_pause, 0); 6774 atomic_set(&t->trace_overrun, 0); 6775 t->ftrace_timestamp = 0; 6776 /* make curr_ret_stack visible before we add the ret_stack */ 6777 smp_wmb(); 6778 t->ret_stack = ret_stack; 6779 } 6780 6781 /* 6782 * Allocate a return stack for the idle task. May be the first 6783 * time through, or it may be done by CPU hotplug online. 6784 */ 6785 void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) 6786 { 6787 t->curr_ret_stack = -1; 6788 /* 6789 * The idle task has no parent, it either has its own 6790 * stack or no stack at all. 6791 */ 6792 if (t->ret_stack) 6793 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu)); 6794 6795 if (ftrace_graph_active) { 6796 struct ftrace_ret_stack *ret_stack; 6797 6798 ret_stack = per_cpu(idle_ret_stack, cpu); 6799 if (!ret_stack) { 6800 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH 6801 * sizeof(struct ftrace_ret_stack), 6802 GFP_KERNEL); 6803 if (!ret_stack) 6804 return; 6805 per_cpu(idle_ret_stack, cpu) = ret_stack; 6806 } 6807 graph_init_task(t, ret_stack); 6808 } 6809 } 6810 6811 /* Allocate a return stack for newly created task */ 6812 void ftrace_graph_init_task(struct task_struct *t) 6813 { 6814 /* Make sure we do not use the parent ret_stack */ 6815 t->ret_stack = NULL; 6816 t->curr_ret_stack = -1; 6817 6818 if (ftrace_graph_active) { 6819 struct ftrace_ret_stack *ret_stack; 6820 6821 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH 6822 * sizeof(struct ftrace_ret_stack), 6823 GFP_KERNEL); 6824 if (!ret_stack) 6825 return; 6826 graph_init_task(t, ret_stack); 6827 } 6828 } 6829 6830 void ftrace_graph_exit_task(struct task_struct *t) 6831 { 6832 struct ftrace_ret_stack *ret_stack = t->ret_stack; 6833 6834 t->ret_stack = NULL; 6835 /* NULL must become visible to IRQs before we free it: */ 6836 barrier(); 6837 6838 kfree(ret_stack); 6839 } 6840 #endif 6841