1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Infrastructure for profiling code inserted by 'gcc -pg'. 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> 7 * 8 * Originally ported from the -rt patch by: 9 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> 10 * 11 * Based on code in the latency_tracer, that is: 12 * 13 * Copyright (C) 2004-2006 Ingo Molnar 14 * Copyright (C) 2004 Nadia Yvette Chambers 15 */ 16 17 #include <linux/stop_machine.h> 18 #include <linux/clocksource.h> 19 #include <linux/sched/task.h> 20 #include <linux/kallsyms.h> 21 #include <linux/security.h> 22 #include <linux/seq_file.h> 23 #include <linux/tracefs.h> 24 #include <linux/hardirq.h> 25 #include <linux/kthread.h> 26 #include <linux/uaccess.h> 27 #include <linux/bsearch.h> 28 #include <linux/module.h> 29 #include <linux/ftrace.h> 30 #include <linux/sysctl.h> 31 #include <linux/slab.h> 32 #include <linux/ctype.h> 33 #include <linux/sort.h> 34 #include <linux/list.h> 35 #include <linux/hash.h> 36 #include <linux/rcupdate.h> 37 #include <linux/kprobes.h> 38 39 #include <trace/events/sched.h> 40 41 #include <asm/sections.h> 42 #include <asm/setup.h> 43 44 #include "ftrace_internal.h" 45 #include "trace_output.h" 46 #include "trace_stat.h" 47 48 #define FTRACE_WARN_ON(cond) \ 49 ({ \ 50 int ___r = cond; \ 51 if (WARN_ON(___r)) \ 52 ftrace_kill(); \ 53 ___r; \ 54 }) 55 56 #define FTRACE_WARN_ON_ONCE(cond) \ 57 ({ \ 58 int ___r = cond; \ 59 if (WARN_ON_ONCE(___r)) \ 60 ftrace_kill(); \ 61 ___r; \ 62 }) 63 64 /* hash bits for specific function selection */ 65 #define FTRACE_HASH_DEFAULT_BITS 10 66 #define FTRACE_HASH_MAX_BITS 12 67 68 #ifdef CONFIG_DYNAMIC_FTRACE 69 #define INIT_OPS_HASH(opsname) \ 70 .func_hash = &opsname.local_hash, \ 71 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), 72 #else 73 #define INIT_OPS_HASH(opsname) 74 #endif 75 76 enum { 77 FTRACE_MODIFY_ENABLE_FL = (1 << 0), 78 FTRACE_MODIFY_MAY_SLEEP_FL = (1 << 1), 79 }; 80 81 struct ftrace_ops ftrace_list_end __read_mostly = { 82 .func = ftrace_stub, 83 .flags = FTRACE_OPS_FL_STUB, 84 INIT_OPS_HASH(ftrace_list_end) 85 }; 86 87 /* ftrace_enabled is a method to turn ftrace on or off */ 88 int ftrace_enabled __read_mostly; 89 static int last_ftrace_enabled; 90 91 /* Current function tracing op */ 92 struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end; 93 /* What to set function_trace_op to */ 94 static struct ftrace_ops *set_function_trace_op; 95 96 static bool ftrace_pids_enabled(struct ftrace_ops *ops) 97 { 98 struct trace_array *tr; 99 100 if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private) 101 return false; 102 103 tr = ops->private; 104 105 return tr->function_pids != NULL || tr->function_no_pids != NULL; 106 } 107 108 static void ftrace_update_trampoline(struct ftrace_ops *ops); 109 110 /* 111 * ftrace_disabled is set when an anomaly is discovered. 112 * ftrace_disabled is much stronger than ftrace_enabled. 113 */ 114 static int ftrace_disabled __read_mostly; 115 116 DEFINE_MUTEX(ftrace_lock); 117 118 struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end; 119 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; 120 struct ftrace_ops global_ops; 121 122 #if ARCH_SUPPORTS_FTRACE_OPS 123 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 124 struct ftrace_ops *op, struct ftrace_regs *fregs); 125 #else 126 /* See comment below, where ftrace_ops_list_func is defined */ 127 static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip); 128 #define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops) 129 #endif 130 131 static inline void ftrace_ops_init(struct ftrace_ops *ops) 132 { 133 #ifdef CONFIG_DYNAMIC_FTRACE 134 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { 135 mutex_init(&ops->local_hash.regex_lock); 136 ops->func_hash = &ops->local_hash; 137 ops->flags |= FTRACE_OPS_FL_INITIALIZED; 138 } 139 #endif 140 } 141 142 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip, 143 struct ftrace_ops *op, struct ftrace_regs *fregs) 144 { 145 struct trace_array *tr = op->private; 146 int pid; 147 148 if (tr) { 149 pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid); 150 if (pid == FTRACE_PID_IGNORE) 151 return; 152 if (pid != FTRACE_PID_TRACE && 153 pid != current->pid) 154 return; 155 } 156 157 op->saved_func(ip, parent_ip, op, fregs); 158 } 159 160 static void ftrace_sync_ipi(void *data) 161 { 162 /* Probably not needed, but do it anyway */ 163 smp_rmb(); 164 } 165 166 static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops) 167 { 168 /* 169 * If this is a dynamic, RCU, or per CPU ops, or we force list func, 170 * then it needs to call the list anyway. 171 */ 172 if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) || 173 FTRACE_FORCE_LIST_FUNC) 174 return ftrace_ops_list_func; 175 176 return ftrace_ops_get_func(ops); 177 } 178 179 static void update_ftrace_function(void) 180 { 181 ftrace_func_t func; 182 183 /* 184 * Prepare the ftrace_ops that the arch callback will use. 185 * If there's only one ftrace_ops registered, the ftrace_ops_list 186 * will point to the ops we want. 187 */ 188 set_function_trace_op = rcu_dereference_protected(ftrace_ops_list, 189 lockdep_is_held(&ftrace_lock)); 190 191 /* If there's no ftrace_ops registered, just call the stub function */ 192 if (set_function_trace_op == &ftrace_list_end) { 193 func = ftrace_stub; 194 195 /* 196 * If we are at the end of the list and this ops is 197 * recursion safe and not dynamic and the arch supports passing ops, 198 * then have the mcount trampoline call the function directly. 199 */ 200 } else if (rcu_dereference_protected(ftrace_ops_list->next, 201 lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) { 202 func = ftrace_ops_get_list_func(ftrace_ops_list); 203 204 } else { 205 /* Just use the default ftrace_ops */ 206 set_function_trace_op = &ftrace_list_end; 207 func = ftrace_ops_list_func; 208 } 209 210 update_function_graph_func(); 211 212 /* If there's no change, then do nothing more here */ 213 if (ftrace_trace_function == func) 214 return; 215 216 /* 217 * If we are using the list function, it doesn't care 218 * about the function_trace_ops. 219 */ 220 if (func == ftrace_ops_list_func) { 221 ftrace_trace_function = func; 222 /* 223 * Don't even bother setting function_trace_ops, 224 * it would be racy to do so anyway. 225 */ 226 return; 227 } 228 229 #ifndef CONFIG_DYNAMIC_FTRACE 230 /* 231 * For static tracing, we need to be a bit more careful. 232 * The function change takes affect immediately. Thus, 233 * we need to coordinate the setting of the function_trace_ops 234 * with the setting of the ftrace_trace_function. 235 * 236 * Set the function to the list ops, which will call the 237 * function we want, albeit indirectly, but it handles the 238 * ftrace_ops and doesn't depend on function_trace_op. 239 */ 240 ftrace_trace_function = ftrace_ops_list_func; 241 /* 242 * Make sure all CPUs see this. Yes this is slow, but static 243 * tracing is slow and nasty to have enabled. 244 */ 245 synchronize_rcu_tasks_rude(); 246 /* Now all cpus are using the list ops. */ 247 function_trace_op = set_function_trace_op; 248 /* Make sure the function_trace_op is visible on all CPUs */ 249 smp_wmb(); 250 /* Nasty way to force a rmb on all cpus */ 251 smp_call_function(ftrace_sync_ipi, NULL, 1); 252 /* OK, we are all set to update the ftrace_trace_function now! */ 253 #endif /* !CONFIG_DYNAMIC_FTRACE */ 254 255 ftrace_trace_function = func; 256 } 257 258 static void add_ftrace_ops(struct ftrace_ops __rcu **list, 259 struct ftrace_ops *ops) 260 { 261 rcu_assign_pointer(ops->next, *list); 262 263 /* 264 * We are entering ops into the list but another 265 * CPU might be walking that list. We need to make sure 266 * the ops->next pointer is valid before another CPU sees 267 * the ops pointer included into the list. 268 */ 269 rcu_assign_pointer(*list, ops); 270 } 271 272 static int remove_ftrace_ops(struct ftrace_ops __rcu **list, 273 struct ftrace_ops *ops) 274 { 275 struct ftrace_ops **p; 276 277 /* 278 * If we are removing the last function, then simply point 279 * to the ftrace_stub. 280 */ 281 if (rcu_dereference_protected(*list, 282 lockdep_is_held(&ftrace_lock)) == ops && 283 rcu_dereference_protected(ops->next, 284 lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) { 285 *list = &ftrace_list_end; 286 return 0; 287 } 288 289 for (p = list; *p != &ftrace_list_end; p = &(*p)->next) 290 if (*p == ops) 291 break; 292 293 if (*p != ops) 294 return -1; 295 296 *p = (*p)->next; 297 return 0; 298 } 299 300 static void ftrace_update_trampoline(struct ftrace_ops *ops); 301 302 int __register_ftrace_function(struct ftrace_ops *ops) 303 { 304 if (ops->flags & FTRACE_OPS_FL_DELETED) 305 return -EINVAL; 306 307 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) 308 return -EBUSY; 309 310 #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS 311 /* 312 * If the ftrace_ops specifies SAVE_REGS, then it only can be used 313 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set. 314 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant. 315 */ 316 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS && 317 !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)) 318 return -EINVAL; 319 320 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED) 321 ops->flags |= FTRACE_OPS_FL_SAVE_REGS; 322 #endif 323 if (!ftrace_enabled && (ops->flags & FTRACE_OPS_FL_PERMANENT)) 324 return -EBUSY; 325 326 if (!core_kernel_data((unsigned long)ops)) 327 ops->flags |= FTRACE_OPS_FL_DYNAMIC; 328 329 add_ftrace_ops(&ftrace_ops_list, ops); 330 331 /* Always save the function, and reset at unregistering */ 332 ops->saved_func = ops->func; 333 334 if (ftrace_pids_enabled(ops)) 335 ops->func = ftrace_pid_func; 336 337 ftrace_update_trampoline(ops); 338 339 if (ftrace_enabled) 340 update_ftrace_function(); 341 342 return 0; 343 } 344 345 int __unregister_ftrace_function(struct ftrace_ops *ops) 346 { 347 int ret; 348 349 if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED))) 350 return -EBUSY; 351 352 ret = remove_ftrace_ops(&ftrace_ops_list, ops); 353 354 if (ret < 0) 355 return ret; 356 357 if (ftrace_enabled) 358 update_ftrace_function(); 359 360 ops->func = ops->saved_func; 361 362 return 0; 363 } 364 365 static void ftrace_update_pid_func(void) 366 { 367 struct ftrace_ops *op; 368 369 /* Only do something if we are tracing something */ 370 if (ftrace_trace_function == ftrace_stub) 371 return; 372 373 do_for_each_ftrace_op(op, ftrace_ops_list) { 374 if (op->flags & FTRACE_OPS_FL_PID) { 375 op->func = ftrace_pids_enabled(op) ? 376 ftrace_pid_func : op->saved_func; 377 ftrace_update_trampoline(op); 378 } 379 } while_for_each_ftrace_op(op); 380 381 update_ftrace_function(); 382 } 383 384 #ifdef CONFIG_FUNCTION_PROFILER 385 struct ftrace_profile { 386 struct hlist_node node; 387 unsigned long ip; 388 unsigned long counter; 389 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 390 unsigned long long time; 391 unsigned long long time_squared; 392 #endif 393 }; 394 395 struct ftrace_profile_page { 396 struct ftrace_profile_page *next; 397 unsigned long index; 398 struct ftrace_profile records[]; 399 }; 400 401 struct ftrace_profile_stat { 402 atomic_t disabled; 403 struct hlist_head *hash; 404 struct ftrace_profile_page *pages; 405 struct ftrace_profile_page *start; 406 struct tracer_stat stat; 407 }; 408 409 #define PROFILE_RECORDS_SIZE \ 410 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records)) 411 412 #define PROFILES_PER_PAGE \ 413 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile)) 414 415 static int ftrace_profile_enabled __read_mostly; 416 417 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */ 418 static DEFINE_MUTEX(ftrace_profile_lock); 419 420 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats); 421 422 #define FTRACE_PROFILE_HASH_BITS 10 423 #define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS) 424 425 static void * 426 function_stat_next(void *v, int idx) 427 { 428 struct ftrace_profile *rec = v; 429 struct ftrace_profile_page *pg; 430 431 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); 432 433 again: 434 if (idx != 0) 435 rec++; 436 437 if ((void *)rec >= (void *)&pg->records[pg->index]) { 438 pg = pg->next; 439 if (!pg) 440 return NULL; 441 rec = &pg->records[0]; 442 if (!rec->counter) 443 goto again; 444 } 445 446 return rec; 447 } 448 449 static void *function_stat_start(struct tracer_stat *trace) 450 { 451 struct ftrace_profile_stat *stat = 452 container_of(trace, struct ftrace_profile_stat, stat); 453 454 if (!stat || !stat->start) 455 return NULL; 456 457 return function_stat_next(&stat->start->records[0], 0); 458 } 459 460 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 461 /* function graph compares on total time */ 462 static int function_stat_cmp(const void *p1, const void *p2) 463 { 464 const struct ftrace_profile *a = p1; 465 const struct ftrace_profile *b = p2; 466 467 if (a->time < b->time) 468 return -1; 469 if (a->time > b->time) 470 return 1; 471 else 472 return 0; 473 } 474 #else 475 /* not function graph compares against hits */ 476 static int function_stat_cmp(const void *p1, const void *p2) 477 { 478 const struct ftrace_profile *a = p1; 479 const struct ftrace_profile *b = p2; 480 481 if (a->counter < b->counter) 482 return -1; 483 if (a->counter > b->counter) 484 return 1; 485 else 486 return 0; 487 } 488 #endif 489 490 static int function_stat_headers(struct seq_file *m) 491 { 492 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 493 seq_puts(m, " Function " 494 "Hit Time Avg s^2\n" 495 " -------- " 496 "--- ---- --- ---\n"); 497 #else 498 seq_puts(m, " Function Hit\n" 499 " -------- ---\n"); 500 #endif 501 return 0; 502 } 503 504 static int function_stat_show(struct seq_file *m, void *v) 505 { 506 struct ftrace_profile *rec = v; 507 char str[KSYM_SYMBOL_LEN]; 508 int ret = 0; 509 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 510 static struct trace_seq s; 511 unsigned long long avg; 512 unsigned long long stddev; 513 #endif 514 mutex_lock(&ftrace_profile_lock); 515 516 /* we raced with function_profile_reset() */ 517 if (unlikely(rec->counter == 0)) { 518 ret = -EBUSY; 519 goto out; 520 } 521 522 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 523 avg = div64_ul(rec->time, rec->counter); 524 if (tracing_thresh && (avg < tracing_thresh)) 525 goto out; 526 #endif 527 528 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 529 seq_printf(m, " %-30.30s %10lu", str, rec->counter); 530 531 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 532 seq_puts(m, " "); 533 534 /* Sample standard deviation (s^2) */ 535 if (rec->counter <= 1) 536 stddev = 0; 537 else { 538 /* 539 * Apply Welford's method: 540 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2) 541 */ 542 stddev = rec->counter * rec->time_squared - 543 rec->time * rec->time; 544 545 /* 546 * Divide only 1000 for ns^2 -> us^2 conversion. 547 * trace_print_graph_duration will divide 1000 again. 548 */ 549 stddev = div64_ul(stddev, 550 rec->counter * (rec->counter - 1) * 1000); 551 } 552 553 trace_seq_init(&s); 554 trace_print_graph_duration(rec->time, &s); 555 trace_seq_puts(&s, " "); 556 trace_print_graph_duration(avg, &s); 557 trace_seq_puts(&s, " "); 558 trace_print_graph_duration(stddev, &s); 559 trace_print_seq(m, &s); 560 #endif 561 seq_putc(m, '\n'); 562 out: 563 mutex_unlock(&ftrace_profile_lock); 564 565 return ret; 566 } 567 568 static void ftrace_profile_reset(struct ftrace_profile_stat *stat) 569 { 570 struct ftrace_profile_page *pg; 571 572 pg = stat->pages = stat->start; 573 574 while (pg) { 575 memset(pg->records, 0, PROFILE_RECORDS_SIZE); 576 pg->index = 0; 577 pg = pg->next; 578 } 579 580 memset(stat->hash, 0, 581 FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head)); 582 } 583 584 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat) 585 { 586 struct ftrace_profile_page *pg; 587 int functions; 588 int pages; 589 int i; 590 591 /* If we already allocated, do nothing */ 592 if (stat->pages) 593 return 0; 594 595 stat->pages = (void *)get_zeroed_page(GFP_KERNEL); 596 if (!stat->pages) 597 return -ENOMEM; 598 599 #ifdef CONFIG_DYNAMIC_FTRACE 600 functions = ftrace_update_tot_cnt; 601 #else 602 /* 603 * We do not know the number of functions that exist because 604 * dynamic tracing is what counts them. With past experience 605 * we have around 20K functions. That should be more than enough. 606 * It is highly unlikely we will execute every function in 607 * the kernel. 608 */ 609 functions = 20000; 610 #endif 611 612 pg = stat->start = stat->pages; 613 614 pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE); 615 616 for (i = 1; i < pages; i++) { 617 pg->next = (void *)get_zeroed_page(GFP_KERNEL); 618 if (!pg->next) 619 goto out_free; 620 pg = pg->next; 621 } 622 623 return 0; 624 625 out_free: 626 pg = stat->start; 627 while (pg) { 628 unsigned long tmp = (unsigned long)pg; 629 630 pg = pg->next; 631 free_page(tmp); 632 } 633 634 stat->pages = NULL; 635 stat->start = NULL; 636 637 return -ENOMEM; 638 } 639 640 static int ftrace_profile_init_cpu(int cpu) 641 { 642 struct ftrace_profile_stat *stat; 643 int size; 644 645 stat = &per_cpu(ftrace_profile_stats, cpu); 646 647 if (stat->hash) { 648 /* If the profile is already created, simply reset it */ 649 ftrace_profile_reset(stat); 650 return 0; 651 } 652 653 /* 654 * We are profiling all functions, but usually only a few thousand 655 * functions are hit. We'll make a hash of 1024 items. 656 */ 657 size = FTRACE_PROFILE_HASH_SIZE; 658 659 stat->hash = kcalloc(size, sizeof(struct hlist_head), GFP_KERNEL); 660 661 if (!stat->hash) 662 return -ENOMEM; 663 664 /* Preallocate the function profiling pages */ 665 if (ftrace_profile_pages_init(stat) < 0) { 666 kfree(stat->hash); 667 stat->hash = NULL; 668 return -ENOMEM; 669 } 670 671 return 0; 672 } 673 674 static int ftrace_profile_init(void) 675 { 676 int cpu; 677 int ret = 0; 678 679 for_each_possible_cpu(cpu) { 680 ret = ftrace_profile_init_cpu(cpu); 681 if (ret) 682 break; 683 } 684 685 return ret; 686 } 687 688 /* interrupts must be disabled */ 689 static struct ftrace_profile * 690 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip) 691 { 692 struct ftrace_profile *rec; 693 struct hlist_head *hhd; 694 unsigned long key; 695 696 key = hash_long(ip, FTRACE_PROFILE_HASH_BITS); 697 hhd = &stat->hash[key]; 698 699 if (hlist_empty(hhd)) 700 return NULL; 701 702 hlist_for_each_entry_rcu_notrace(rec, hhd, node) { 703 if (rec->ip == ip) 704 return rec; 705 } 706 707 return NULL; 708 } 709 710 static void ftrace_add_profile(struct ftrace_profile_stat *stat, 711 struct ftrace_profile *rec) 712 { 713 unsigned long key; 714 715 key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS); 716 hlist_add_head_rcu(&rec->node, &stat->hash[key]); 717 } 718 719 /* 720 * The memory is already allocated, this simply finds a new record to use. 721 */ 722 static struct ftrace_profile * 723 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip) 724 { 725 struct ftrace_profile *rec = NULL; 726 727 /* prevent recursion (from NMIs) */ 728 if (atomic_inc_return(&stat->disabled) != 1) 729 goto out; 730 731 /* 732 * Try to find the function again since an NMI 733 * could have added it 734 */ 735 rec = ftrace_find_profiled_func(stat, ip); 736 if (rec) 737 goto out; 738 739 if (stat->pages->index == PROFILES_PER_PAGE) { 740 if (!stat->pages->next) 741 goto out; 742 stat->pages = stat->pages->next; 743 } 744 745 rec = &stat->pages->records[stat->pages->index++]; 746 rec->ip = ip; 747 ftrace_add_profile(stat, rec); 748 749 out: 750 atomic_dec(&stat->disabled); 751 752 return rec; 753 } 754 755 static void 756 function_profile_call(unsigned long ip, unsigned long parent_ip, 757 struct ftrace_ops *ops, struct ftrace_regs *fregs) 758 { 759 struct ftrace_profile_stat *stat; 760 struct ftrace_profile *rec; 761 unsigned long flags; 762 763 if (!ftrace_profile_enabled) 764 return; 765 766 local_irq_save(flags); 767 768 stat = this_cpu_ptr(&ftrace_profile_stats); 769 if (!stat->hash || !ftrace_profile_enabled) 770 goto out; 771 772 rec = ftrace_find_profiled_func(stat, ip); 773 if (!rec) { 774 rec = ftrace_profile_alloc(stat, ip); 775 if (!rec) 776 goto out; 777 } 778 779 rec->counter++; 780 out: 781 local_irq_restore(flags); 782 } 783 784 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 785 static bool fgraph_graph_time = true; 786 787 void ftrace_graph_graph_time_control(bool enable) 788 { 789 fgraph_graph_time = enable; 790 } 791 792 static int profile_graph_entry(struct ftrace_graph_ent *trace) 793 { 794 struct ftrace_ret_stack *ret_stack; 795 796 function_profile_call(trace->func, 0, NULL, NULL); 797 798 /* If function graph is shutting down, ret_stack can be NULL */ 799 if (!current->ret_stack) 800 return 0; 801 802 ret_stack = ftrace_graph_get_ret_stack(current, 0); 803 if (ret_stack) 804 ret_stack->subtime = 0; 805 806 return 1; 807 } 808 809 static void profile_graph_return(struct ftrace_graph_ret *trace) 810 { 811 struct ftrace_ret_stack *ret_stack; 812 struct ftrace_profile_stat *stat; 813 unsigned long long calltime; 814 struct ftrace_profile *rec; 815 unsigned long flags; 816 817 local_irq_save(flags); 818 stat = this_cpu_ptr(&ftrace_profile_stats); 819 if (!stat->hash || !ftrace_profile_enabled) 820 goto out; 821 822 /* If the calltime was zero'd ignore it */ 823 if (!trace->calltime) 824 goto out; 825 826 calltime = trace->rettime - trace->calltime; 827 828 if (!fgraph_graph_time) { 829 830 /* Append this call time to the parent time to subtract */ 831 ret_stack = ftrace_graph_get_ret_stack(current, 1); 832 if (ret_stack) 833 ret_stack->subtime += calltime; 834 835 ret_stack = ftrace_graph_get_ret_stack(current, 0); 836 if (ret_stack && ret_stack->subtime < calltime) 837 calltime -= ret_stack->subtime; 838 else 839 calltime = 0; 840 } 841 842 rec = ftrace_find_profiled_func(stat, trace->func); 843 if (rec) { 844 rec->time += calltime; 845 rec->time_squared += calltime * calltime; 846 } 847 848 out: 849 local_irq_restore(flags); 850 } 851 852 static struct fgraph_ops fprofiler_ops = { 853 .entryfunc = &profile_graph_entry, 854 .retfunc = &profile_graph_return, 855 }; 856 857 static int register_ftrace_profiler(void) 858 { 859 return register_ftrace_graph(&fprofiler_ops); 860 } 861 862 static void unregister_ftrace_profiler(void) 863 { 864 unregister_ftrace_graph(&fprofiler_ops); 865 } 866 #else 867 static struct ftrace_ops ftrace_profile_ops __read_mostly = { 868 .func = function_profile_call, 869 .flags = FTRACE_OPS_FL_INITIALIZED, 870 INIT_OPS_HASH(ftrace_profile_ops) 871 }; 872 873 static int register_ftrace_profiler(void) 874 { 875 return register_ftrace_function(&ftrace_profile_ops); 876 } 877 878 static void unregister_ftrace_profiler(void) 879 { 880 unregister_ftrace_function(&ftrace_profile_ops); 881 } 882 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 883 884 static ssize_t 885 ftrace_profile_write(struct file *filp, const char __user *ubuf, 886 size_t cnt, loff_t *ppos) 887 { 888 unsigned long val; 889 int ret; 890 891 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 892 if (ret) 893 return ret; 894 895 val = !!val; 896 897 mutex_lock(&ftrace_profile_lock); 898 if (ftrace_profile_enabled ^ val) { 899 if (val) { 900 ret = ftrace_profile_init(); 901 if (ret < 0) { 902 cnt = ret; 903 goto out; 904 } 905 906 ret = register_ftrace_profiler(); 907 if (ret < 0) { 908 cnt = ret; 909 goto out; 910 } 911 ftrace_profile_enabled = 1; 912 } else { 913 ftrace_profile_enabled = 0; 914 /* 915 * unregister_ftrace_profiler calls stop_machine 916 * so this acts like an synchronize_rcu. 917 */ 918 unregister_ftrace_profiler(); 919 } 920 } 921 out: 922 mutex_unlock(&ftrace_profile_lock); 923 924 *ppos += cnt; 925 926 return cnt; 927 } 928 929 static ssize_t 930 ftrace_profile_read(struct file *filp, char __user *ubuf, 931 size_t cnt, loff_t *ppos) 932 { 933 char buf[64]; /* big enough to hold a number */ 934 int r; 935 936 r = sprintf(buf, "%u\n", ftrace_profile_enabled); 937 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 938 } 939 940 static const struct file_operations ftrace_profile_fops = { 941 .open = tracing_open_generic, 942 .read = ftrace_profile_read, 943 .write = ftrace_profile_write, 944 .llseek = default_llseek, 945 }; 946 947 /* used to initialize the real stat files */ 948 static struct tracer_stat function_stats __initdata = { 949 .name = "functions", 950 .stat_start = function_stat_start, 951 .stat_next = function_stat_next, 952 .stat_cmp = function_stat_cmp, 953 .stat_headers = function_stat_headers, 954 .stat_show = function_stat_show 955 }; 956 957 static __init void ftrace_profile_tracefs(struct dentry *d_tracer) 958 { 959 struct ftrace_profile_stat *stat; 960 struct dentry *entry; 961 char *name; 962 int ret; 963 int cpu; 964 965 for_each_possible_cpu(cpu) { 966 stat = &per_cpu(ftrace_profile_stats, cpu); 967 968 name = kasprintf(GFP_KERNEL, "function%d", cpu); 969 if (!name) { 970 /* 971 * The files created are permanent, if something happens 972 * we still do not free memory. 973 */ 974 WARN(1, 975 "Could not allocate stat file for cpu %d\n", 976 cpu); 977 return; 978 } 979 stat->stat = function_stats; 980 stat->stat.name = name; 981 ret = register_stat_tracer(&stat->stat); 982 if (ret) { 983 WARN(1, 984 "Could not register function stat for cpu %d\n", 985 cpu); 986 kfree(name); 987 return; 988 } 989 } 990 991 entry = tracefs_create_file("function_profile_enabled", 0644, 992 d_tracer, NULL, &ftrace_profile_fops); 993 if (!entry) 994 pr_warn("Could not create tracefs 'function_profile_enabled' entry\n"); 995 } 996 997 #else /* CONFIG_FUNCTION_PROFILER */ 998 static __init void ftrace_profile_tracefs(struct dentry *d_tracer) 999 { 1000 } 1001 #endif /* CONFIG_FUNCTION_PROFILER */ 1002 1003 #ifdef CONFIG_DYNAMIC_FTRACE 1004 1005 static struct ftrace_ops *removed_ops; 1006 1007 /* 1008 * Set when doing a global update, like enabling all recs or disabling them. 1009 * It is not set when just updating a single ftrace_ops. 1010 */ 1011 static bool update_all_ops; 1012 1013 #ifndef CONFIG_FTRACE_MCOUNT_RECORD 1014 # error Dynamic ftrace depends on MCOUNT_RECORD 1015 #endif 1016 1017 struct ftrace_func_probe { 1018 struct ftrace_probe_ops *probe_ops; 1019 struct ftrace_ops ops; 1020 struct trace_array *tr; 1021 struct list_head list; 1022 void *data; 1023 int ref; 1024 }; 1025 1026 /* 1027 * We make these constant because no one should touch them, 1028 * but they are used as the default "empty hash", to avoid allocating 1029 * it all the time. These are in a read only section such that if 1030 * anyone does try to modify it, it will cause an exception. 1031 */ 1032 static const struct hlist_head empty_buckets[1]; 1033 static const struct ftrace_hash empty_hash = { 1034 .buckets = (struct hlist_head *)empty_buckets, 1035 }; 1036 #define EMPTY_HASH ((struct ftrace_hash *)&empty_hash) 1037 1038 struct ftrace_ops global_ops = { 1039 .func = ftrace_stub, 1040 .local_hash.notrace_hash = EMPTY_HASH, 1041 .local_hash.filter_hash = EMPTY_HASH, 1042 INIT_OPS_HASH(global_ops) 1043 .flags = FTRACE_OPS_FL_INITIALIZED | 1044 FTRACE_OPS_FL_PID, 1045 }; 1046 1047 /* 1048 * Used by the stack undwinder to know about dynamic ftrace trampolines. 1049 */ 1050 struct ftrace_ops *ftrace_ops_trampoline(unsigned long addr) 1051 { 1052 struct ftrace_ops *op = NULL; 1053 1054 /* 1055 * Some of the ops may be dynamically allocated, 1056 * they are freed after a synchronize_rcu(). 1057 */ 1058 preempt_disable_notrace(); 1059 1060 do_for_each_ftrace_op(op, ftrace_ops_list) { 1061 /* 1062 * This is to check for dynamically allocated trampolines. 1063 * Trampolines that are in kernel text will have 1064 * core_kernel_text() return true. 1065 */ 1066 if (op->trampoline && op->trampoline_size) 1067 if (addr >= op->trampoline && 1068 addr < op->trampoline + op->trampoline_size) { 1069 preempt_enable_notrace(); 1070 return op; 1071 } 1072 } while_for_each_ftrace_op(op); 1073 preempt_enable_notrace(); 1074 1075 return NULL; 1076 } 1077 1078 /* 1079 * This is used by __kernel_text_address() to return true if the 1080 * address is on a dynamically allocated trampoline that would 1081 * not return true for either core_kernel_text() or 1082 * is_module_text_address(). 1083 */ 1084 bool is_ftrace_trampoline(unsigned long addr) 1085 { 1086 return ftrace_ops_trampoline(addr) != NULL; 1087 } 1088 1089 struct ftrace_page { 1090 struct ftrace_page *next; 1091 struct dyn_ftrace *records; 1092 int index; 1093 int size; 1094 }; 1095 1096 #define ENTRY_SIZE sizeof(struct dyn_ftrace) 1097 #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE) 1098 1099 static struct ftrace_page *ftrace_pages_start; 1100 static struct ftrace_page *ftrace_pages; 1101 1102 static __always_inline unsigned long 1103 ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip) 1104 { 1105 if (hash->size_bits > 0) 1106 return hash_long(ip, hash->size_bits); 1107 1108 return 0; 1109 } 1110 1111 /* Only use this function if ftrace_hash_empty() has already been tested */ 1112 static __always_inline struct ftrace_func_entry * 1113 __ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) 1114 { 1115 unsigned long key; 1116 struct ftrace_func_entry *entry; 1117 struct hlist_head *hhd; 1118 1119 key = ftrace_hash_key(hash, ip); 1120 hhd = &hash->buckets[key]; 1121 1122 hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) { 1123 if (entry->ip == ip) 1124 return entry; 1125 } 1126 return NULL; 1127 } 1128 1129 /** 1130 * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash 1131 * @hash: The hash to look at 1132 * @ip: The instruction pointer to test 1133 * 1134 * Search a given @hash to see if a given instruction pointer (@ip) 1135 * exists in it. 1136 * 1137 * Returns the entry that holds the @ip if found. NULL otherwise. 1138 */ 1139 struct ftrace_func_entry * 1140 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) 1141 { 1142 if (ftrace_hash_empty(hash)) 1143 return NULL; 1144 1145 return __ftrace_lookup_ip(hash, ip); 1146 } 1147 1148 static void __add_hash_entry(struct ftrace_hash *hash, 1149 struct ftrace_func_entry *entry) 1150 { 1151 struct hlist_head *hhd; 1152 unsigned long key; 1153 1154 key = ftrace_hash_key(hash, entry->ip); 1155 hhd = &hash->buckets[key]; 1156 hlist_add_head(&entry->hlist, hhd); 1157 hash->count++; 1158 } 1159 1160 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip) 1161 { 1162 struct ftrace_func_entry *entry; 1163 1164 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 1165 if (!entry) 1166 return -ENOMEM; 1167 1168 entry->ip = ip; 1169 __add_hash_entry(hash, entry); 1170 1171 return 0; 1172 } 1173 1174 static void 1175 free_hash_entry(struct ftrace_hash *hash, 1176 struct ftrace_func_entry *entry) 1177 { 1178 hlist_del(&entry->hlist); 1179 kfree(entry); 1180 hash->count--; 1181 } 1182 1183 static void 1184 remove_hash_entry(struct ftrace_hash *hash, 1185 struct ftrace_func_entry *entry) 1186 { 1187 hlist_del_rcu(&entry->hlist); 1188 hash->count--; 1189 } 1190 1191 static void ftrace_hash_clear(struct ftrace_hash *hash) 1192 { 1193 struct hlist_head *hhd; 1194 struct hlist_node *tn; 1195 struct ftrace_func_entry *entry; 1196 int size = 1 << hash->size_bits; 1197 int i; 1198 1199 if (!hash->count) 1200 return; 1201 1202 for (i = 0; i < size; i++) { 1203 hhd = &hash->buckets[i]; 1204 hlist_for_each_entry_safe(entry, tn, hhd, hlist) 1205 free_hash_entry(hash, entry); 1206 } 1207 FTRACE_WARN_ON(hash->count); 1208 } 1209 1210 static void free_ftrace_mod(struct ftrace_mod_load *ftrace_mod) 1211 { 1212 list_del(&ftrace_mod->list); 1213 kfree(ftrace_mod->module); 1214 kfree(ftrace_mod->func); 1215 kfree(ftrace_mod); 1216 } 1217 1218 static void clear_ftrace_mod_list(struct list_head *head) 1219 { 1220 struct ftrace_mod_load *p, *n; 1221 1222 /* stack tracer isn't supported yet */ 1223 if (!head) 1224 return; 1225 1226 mutex_lock(&ftrace_lock); 1227 list_for_each_entry_safe(p, n, head, list) 1228 free_ftrace_mod(p); 1229 mutex_unlock(&ftrace_lock); 1230 } 1231 1232 static void free_ftrace_hash(struct ftrace_hash *hash) 1233 { 1234 if (!hash || hash == EMPTY_HASH) 1235 return; 1236 ftrace_hash_clear(hash); 1237 kfree(hash->buckets); 1238 kfree(hash); 1239 } 1240 1241 static void __free_ftrace_hash_rcu(struct rcu_head *rcu) 1242 { 1243 struct ftrace_hash *hash; 1244 1245 hash = container_of(rcu, struct ftrace_hash, rcu); 1246 free_ftrace_hash(hash); 1247 } 1248 1249 static void free_ftrace_hash_rcu(struct ftrace_hash *hash) 1250 { 1251 if (!hash || hash == EMPTY_HASH) 1252 return; 1253 call_rcu(&hash->rcu, __free_ftrace_hash_rcu); 1254 } 1255 1256 void ftrace_free_filter(struct ftrace_ops *ops) 1257 { 1258 ftrace_ops_init(ops); 1259 free_ftrace_hash(ops->func_hash->filter_hash); 1260 free_ftrace_hash(ops->func_hash->notrace_hash); 1261 } 1262 1263 static struct ftrace_hash *alloc_ftrace_hash(int size_bits) 1264 { 1265 struct ftrace_hash *hash; 1266 int size; 1267 1268 hash = kzalloc(sizeof(*hash), GFP_KERNEL); 1269 if (!hash) 1270 return NULL; 1271 1272 size = 1 << size_bits; 1273 hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL); 1274 1275 if (!hash->buckets) { 1276 kfree(hash); 1277 return NULL; 1278 } 1279 1280 hash->size_bits = size_bits; 1281 1282 return hash; 1283 } 1284 1285 1286 static int ftrace_add_mod(struct trace_array *tr, 1287 const char *func, const char *module, 1288 int enable) 1289 { 1290 struct ftrace_mod_load *ftrace_mod; 1291 struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace; 1292 1293 ftrace_mod = kzalloc(sizeof(*ftrace_mod), GFP_KERNEL); 1294 if (!ftrace_mod) 1295 return -ENOMEM; 1296 1297 ftrace_mod->func = kstrdup(func, GFP_KERNEL); 1298 ftrace_mod->module = kstrdup(module, GFP_KERNEL); 1299 ftrace_mod->enable = enable; 1300 1301 if (!ftrace_mod->func || !ftrace_mod->module) 1302 goto out_free; 1303 1304 list_add(&ftrace_mod->list, mod_head); 1305 1306 return 0; 1307 1308 out_free: 1309 free_ftrace_mod(ftrace_mod); 1310 1311 return -ENOMEM; 1312 } 1313 1314 static struct ftrace_hash * 1315 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash) 1316 { 1317 struct ftrace_func_entry *entry; 1318 struct ftrace_hash *new_hash; 1319 int size; 1320 int ret; 1321 int i; 1322 1323 new_hash = alloc_ftrace_hash(size_bits); 1324 if (!new_hash) 1325 return NULL; 1326 1327 if (hash) 1328 new_hash->flags = hash->flags; 1329 1330 /* Empty hash? */ 1331 if (ftrace_hash_empty(hash)) 1332 return new_hash; 1333 1334 size = 1 << hash->size_bits; 1335 for (i = 0; i < size; i++) { 1336 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 1337 ret = add_hash_entry(new_hash, entry->ip); 1338 if (ret < 0) 1339 goto free_hash; 1340 } 1341 } 1342 1343 FTRACE_WARN_ON(new_hash->count != hash->count); 1344 1345 return new_hash; 1346 1347 free_hash: 1348 free_ftrace_hash(new_hash); 1349 return NULL; 1350 } 1351 1352 static void 1353 ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash); 1354 static void 1355 ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash); 1356 1357 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, 1358 struct ftrace_hash *new_hash); 1359 1360 static struct ftrace_hash *dup_hash(struct ftrace_hash *src, int size) 1361 { 1362 struct ftrace_func_entry *entry; 1363 struct ftrace_hash *new_hash; 1364 struct hlist_head *hhd; 1365 struct hlist_node *tn; 1366 int bits = 0; 1367 int i; 1368 1369 /* 1370 * Use around half the size (max bit of it), but 1371 * a minimum of 2 is fine (as size of 0 or 1 both give 1 for bits). 1372 */ 1373 bits = fls(size / 2); 1374 1375 /* Don't allocate too much */ 1376 if (bits > FTRACE_HASH_MAX_BITS) 1377 bits = FTRACE_HASH_MAX_BITS; 1378 1379 new_hash = alloc_ftrace_hash(bits); 1380 if (!new_hash) 1381 return NULL; 1382 1383 new_hash->flags = src->flags; 1384 1385 size = 1 << src->size_bits; 1386 for (i = 0; i < size; i++) { 1387 hhd = &src->buckets[i]; 1388 hlist_for_each_entry_safe(entry, tn, hhd, hlist) { 1389 remove_hash_entry(src, entry); 1390 __add_hash_entry(new_hash, entry); 1391 } 1392 } 1393 return new_hash; 1394 } 1395 1396 static struct ftrace_hash * 1397 __ftrace_hash_move(struct ftrace_hash *src) 1398 { 1399 int size = src->count; 1400 1401 /* 1402 * If the new source is empty, just return the empty_hash. 1403 */ 1404 if (ftrace_hash_empty(src)) 1405 return EMPTY_HASH; 1406 1407 return dup_hash(src, size); 1408 } 1409 1410 static int 1411 ftrace_hash_move(struct ftrace_ops *ops, int enable, 1412 struct ftrace_hash **dst, struct ftrace_hash *src) 1413 { 1414 struct ftrace_hash *new_hash; 1415 int ret; 1416 1417 /* Reject setting notrace hash on IPMODIFY ftrace_ops */ 1418 if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable) 1419 return -EINVAL; 1420 1421 new_hash = __ftrace_hash_move(src); 1422 if (!new_hash) 1423 return -ENOMEM; 1424 1425 /* Make sure this can be applied if it is IPMODIFY ftrace_ops */ 1426 if (enable) { 1427 /* IPMODIFY should be updated only when filter_hash updating */ 1428 ret = ftrace_hash_ipmodify_update(ops, new_hash); 1429 if (ret < 0) { 1430 free_ftrace_hash(new_hash); 1431 return ret; 1432 } 1433 } 1434 1435 /* 1436 * Remove the current set, update the hash and add 1437 * them back. 1438 */ 1439 ftrace_hash_rec_disable_modify(ops, enable); 1440 1441 rcu_assign_pointer(*dst, new_hash); 1442 1443 ftrace_hash_rec_enable_modify(ops, enable); 1444 1445 return 0; 1446 } 1447 1448 static bool hash_contains_ip(unsigned long ip, 1449 struct ftrace_ops_hash *hash) 1450 { 1451 /* 1452 * The function record is a match if it exists in the filter 1453 * hash and not in the notrace hash. Note, an empty hash is 1454 * considered a match for the filter hash, but an empty 1455 * notrace hash is considered not in the notrace hash. 1456 */ 1457 return (ftrace_hash_empty(hash->filter_hash) || 1458 __ftrace_lookup_ip(hash->filter_hash, ip)) && 1459 (ftrace_hash_empty(hash->notrace_hash) || 1460 !__ftrace_lookup_ip(hash->notrace_hash, ip)); 1461 } 1462 1463 /* 1464 * Test the hashes for this ops to see if we want to call 1465 * the ops->func or not. 1466 * 1467 * It's a match if the ip is in the ops->filter_hash or 1468 * the filter_hash does not exist or is empty, 1469 * AND 1470 * the ip is not in the ops->notrace_hash. 1471 * 1472 * This needs to be called with preemption disabled as 1473 * the hashes are freed with call_rcu(). 1474 */ 1475 int 1476 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) 1477 { 1478 struct ftrace_ops_hash hash; 1479 int ret; 1480 1481 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 1482 /* 1483 * There's a small race when adding ops that the ftrace handler 1484 * that wants regs, may be called without them. We can not 1485 * allow that handler to be called if regs is NULL. 1486 */ 1487 if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS)) 1488 return 0; 1489 #endif 1490 1491 rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash); 1492 rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash); 1493 1494 if (hash_contains_ip(ip, &hash)) 1495 ret = 1; 1496 else 1497 ret = 0; 1498 1499 return ret; 1500 } 1501 1502 /* 1503 * This is a double for. Do not use 'break' to break out of the loop, 1504 * you must use a goto. 1505 */ 1506 #define do_for_each_ftrace_rec(pg, rec) \ 1507 for (pg = ftrace_pages_start; pg; pg = pg->next) { \ 1508 int _____i; \ 1509 for (_____i = 0; _____i < pg->index; _____i++) { \ 1510 rec = &pg->records[_____i]; 1511 1512 #define while_for_each_ftrace_rec() \ 1513 } \ 1514 } 1515 1516 1517 static int ftrace_cmp_recs(const void *a, const void *b) 1518 { 1519 const struct dyn_ftrace *key = a; 1520 const struct dyn_ftrace *rec = b; 1521 1522 if (key->flags < rec->ip) 1523 return -1; 1524 if (key->ip >= rec->ip + MCOUNT_INSN_SIZE) 1525 return 1; 1526 return 0; 1527 } 1528 1529 static struct dyn_ftrace *lookup_rec(unsigned long start, unsigned long end) 1530 { 1531 struct ftrace_page *pg; 1532 struct dyn_ftrace *rec = NULL; 1533 struct dyn_ftrace key; 1534 1535 key.ip = start; 1536 key.flags = end; /* overload flags, as it is unsigned long */ 1537 1538 for (pg = ftrace_pages_start; pg; pg = pg->next) { 1539 if (end < pg->records[0].ip || 1540 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) 1541 continue; 1542 rec = bsearch(&key, pg->records, pg->index, 1543 sizeof(struct dyn_ftrace), 1544 ftrace_cmp_recs); 1545 if (rec) 1546 break; 1547 } 1548 return rec; 1549 } 1550 1551 /** 1552 * ftrace_location_range - return the first address of a traced location 1553 * if it touches the given ip range 1554 * @start: start of range to search. 1555 * @end: end of range to search (inclusive). @end points to the last byte 1556 * to check. 1557 * 1558 * Returns rec->ip if the related ftrace location is a least partly within 1559 * the given address range. That is, the first address of the instruction 1560 * that is either a NOP or call to the function tracer. It checks the ftrace 1561 * internal tables to determine if the address belongs or not. 1562 */ 1563 unsigned long ftrace_location_range(unsigned long start, unsigned long end) 1564 { 1565 struct dyn_ftrace *rec; 1566 1567 rec = lookup_rec(start, end); 1568 if (rec) 1569 return rec->ip; 1570 1571 return 0; 1572 } 1573 1574 /** 1575 * ftrace_location - return true if the ip giving is a traced location 1576 * @ip: the instruction pointer to check 1577 * 1578 * Returns rec->ip if @ip given is a pointer to a ftrace location. 1579 * That is, the instruction that is either a NOP or call to 1580 * the function tracer. It checks the ftrace internal tables to 1581 * determine if the address belongs or not. 1582 */ 1583 unsigned long ftrace_location(unsigned long ip) 1584 { 1585 return ftrace_location_range(ip, ip); 1586 } 1587 1588 /** 1589 * ftrace_text_reserved - return true if range contains an ftrace location 1590 * @start: start of range to search 1591 * @end: end of range to search (inclusive). @end points to the last byte to check. 1592 * 1593 * Returns 1 if @start and @end contains a ftrace location. 1594 * That is, the instruction that is either a NOP or call to 1595 * the function tracer. It checks the ftrace internal tables to 1596 * determine if the address belongs or not. 1597 */ 1598 int ftrace_text_reserved(const void *start, const void *end) 1599 { 1600 unsigned long ret; 1601 1602 ret = ftrace_location_range((unsigned long)start, 1603 (unsigned long)end); 1604 1605 return (int)!!ret; 1606 } 1607 1608 /* Test if ops registered to this rec needs regs */ 1609 static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec) 1610 { 1611 struct ftrace_ops *ops; 1612 bool keep_regs = false; 1613 1614 for (ops = ftrace_ops_list; 1615 ops != &ftrace_list_end; ops = ops->next) { 1616 /* pass rec in as regs to have non-NULL val */ 1617 if (ftrace_ops_test(ops, rec->ip, rec)) { 1618 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) { 1619 keep_regs = true; 1620 break; 1621 } 1622 } 1623 } 1624 1625 return keep_regs; 1626 } 1627 1628 static struct ftrace_ops * 1629 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec); 1630 static struct ftrace_ops * 1631 ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude); 1632 static struct ftrace_ops * 1633 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops); 1634 1635 static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, 1636 int filter_hash, 1637 bool inc) 1638 { 1639 struct ftrace_hash *hash; 1640 struct ftrace_hash *other_hash; 1641 struct ftrace_page *pg; 1642 struct dyn_ftrace *rec; 1643 bool update = false; 1644 int count = 0; 1645 int all = false; 1646 1647 /* Only update if the ops has been registered */ 1648 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 1649 return false; 1650 1651 /* 1652 * In the filter_hash case: 1653 * If the count is zero, we update all records. 1654 * Otherwise we just update the items in the hash. 1655 * 1656 * In the notrace_hash case: 1657 * We enable the update in the hash. 1658 * As disabling notrace means enabling the tracing, 1659 * and enabling notrace means disabling, the inc variable 1660 * gets inversed. 1661 */ 1662 if (filter_hash) { 1663 hash = ops->func_hash->filter_hash; 1664 other_hash = ops->func_hash->notrace_hash; 1665 if (ftrace_hash_empty(hash)) 1666 all = true; 1667 } else { 1668 inc = !inc; 1669 hash = ops->func_hash->notrace_hash; 1670 other_hash = ops->func_hash->filter_hash; 1671 /* 1672 * If the notrace hash has no items, 1673 * then there's nothing to do. 1674 */ 1675 if (ftrace_hash_empty(hash)) 1676 return false; 1677 } 1678 1679 do_for_each_ftrace_rec(pg, rec) { 1680 int in_other_hash = 0; 1681 int in_hash = 0; 1682 int match = 0; 1683 1684 if (rec->flags & FTRACE_FL_DISABLED) 1685 continue; 1686 1687 if (all) { 1688 /* 1689 * Only the filter_hash affects all records. 1690 * Update if the record is not in the notrace hash. 1691 */ 1692 if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip)) 1693 match = 1; 1694 } else { 1695 in_hash = !!ftrace_lookup_ip(hash, rec->ip); 1696 in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip); 1697 1698 /* 1699 * If filter_hash is set, we want to match all functions 1700 * that are in the hash but not in the other hash. 1701 * 1702 * If filter_hash is not set, then we are decrementing. 1703 * That means we match anything that is in the hash 1704 * and also in the other_hash. That is, we need to turn 1705 * off functions in the other hash because they are disabled 1706 * by this hash. 1707 */ 1708 if (filter_hash && in_hash && !in_other_hash) 1709 match = 1; 1710 else if (!filter_hash && in_hash && 1711 (in_other_hash || ftrace_hash_empty(other_hash))) 1712 match = 1; 1713 } 1714 if (!match) 1715 continue; 1716 1717 if (inc) { 1718 rec->flags++; 1719 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX)) 1720 return false; 1721 1722 if (ops->flags & FTRACE_OPS_FL_DIRECT) 1723 rec->flags |= FTRACE_FL_DIRECT; 1724 1725 /* 1726 * If there's only a single callback registered to a 1727 * function, and the ops has a trampoline registered 1728 * for it, then we can call it directly. 1729 */ 1730 if (ftrace_rec_count(rec) == 1 && ops->trampoline) 1731 rec->flags |= FTRACE_FL_TRAMP; 1732 else 1733 /* 1734 * If we are adding another function callback 1735 * to this function, and the previous had a 1736 * custom trampoline in use, then we need to go 1737 * back to the default trampoline. 1738 */ 1739 rec->flags &= ~FTRACE_FL_TRAMP; 1740 1741 /* 1742 * If any ops wants regs saved for this function 1743 * then all ops will get saved regs. 1744 */ 1745 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) 1746 rec->flags |= FTRACE_FL_REGS; 1747 } else { 1748 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0)) 1749 return false; 1750 rec->flags--; 1751 1752 /* 1753 * Only the internal direct_ops should have the 1754 * DIRECT flag set. Thus, if it is removing a 1755 * function, then that function should no longer 1756 * be direct. 1757 */ 1758 if (ops->flags & FTRACE_OPS_FL_DIRECT) 1759 rec->flags &= ~FTRACE_FL_DIRECT; 1760 1761 /* 1762 * If the rec had REGS enabled and the ops that is 1763 * being removed had REGS set, then see if there is 1764 * still any ops for this record that wants regs. 1765 * If not, we can stop recording them. 1766 */ 1767 if (ftrace_rec_count(rec) > 0 && 1768 rec->flags & FTRACE_FL_REGS && 1769 ops->flags & FTRACE_OPS_FL_SAVE_REGS) { 1770 if (!test_rec_ops_needs_regs(rec)) 1771 rec->flags &= ~FTRACE_FL_REGS; 1772 } 1773 1774 /* 1775 * The TRAMP needs to be set only if rec count 1776 * is decremented to one, and the ops that is 1777 * left has a trampoline. As TRAMP can only be 1778 * enabled if there is only a single ops attached 1779 * to it. 1780 */ 1781 if (ftrace_rec_count(rec) == 1 && 1782 ftrace_find_tramp_ops_any_other(rec, ops)) 1783 rec->flags |= FTRACE_FL_TRAMP; 1784 else 1785 rec->flags &= ~FTRACE_FL_TRAMP; 1786 1787 /* 1788 * flags will be cleared in ftrace_check_record() 1789 * if rec count is zero. 1790 */ 1791 } 1792 count++; 1793 1794 /* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */ 1795 update |= ftrace_test_record(rec, true) != FTRACE_UPDATE_IGNORE; 1796 1797 /* Shortcut, if we handled all records, we are done. */ 1798 if (!all && count == hash->count) 1799 return update; 1800 } while_for_each_ftrace_rec(); 1801 1802 return update; 1803 } 1804 1805 static bool ftrace_hash_rec_disable(struct ftrace_ops *ops, 1806 int filter_hash) 1807 { 1808 return __ftrace_hash_rec_update(ops, filter_hash, 0); 1809 } 1810 1811 static bool ftrace_hash_rec_enable(struct ftrace_ops *ops, 1812 int filter_hash) 1813 { 1814 return __ftrace_hash_rec_update(ops, filter_hash, 1); 1815 } 1816 1817 static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, 1818 int filter_hash, int inc) 1819 { 1820 struct ftrace_ops *op; 1821 1822 __ftrace_hash_rec_update(ops, filter_hash, inc); 1823 1824 if (ops->func_hash != &global_ops.local_hash) 1825 return; 1826 1827 /* 1828 * If the ops shares the global_ops hash, then we need to update 1829 * all ops that are enabled and use this hash. 1830 */ 1831 do_for_each_ftrace_op(op, ftrace_ops_list) { 1832 /* Already done */ 1833 if (op == ops) 1834 continue; 1835 if (op->func_hash == &global_ops.local_hash) 1836 __ftrace_hash_rec_update(op, filter_hash, inc); 1837 } while_for_each_ftrace_op(op); 1838 } 1839 1840 static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, 1841 int filter_hash) 1842 { 1843 ftrace_hash_rec_update_modify(ops, filter_hash, 0); 1844 } 1845 1846 static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, 1847 int filter_hash) 1848 { 1849 ftrace_hash_rec_update_modify(ops, filter_hash, 1); 1850 } 1851 1852 /* 1853 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK 1854 * or no-needed to update, -EBUSY if it detects a conflict of the flag 1855 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs. 1856 * Note that old_hash and new_hash has below meanings 1857 * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected) 1858 * - If the hash is EMPTY_HASH, it hits nothing 1859 * - Anything else hits the recs which match the hash entries. 1860 */ 1861 static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops, 1862 struct ftrace_hash *old_hash, 1863 struct ftrace_hash *new_hash) 1864 { 1865 struct ftrace_page *pg; 1866 struct dyn_ftrace *rec, *end = NULL; 1867 int in_old, in_new; 1868 1869 /* Only update if the ops has been registered */ 1870 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 1871 return 0; 1872 1873 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) 1874 return 0; 1875 1876 /* 1877 * Since the IPMODIFY is a very address sensitive action, we do not 1878 * allow ftrace_ops to set all functions to new hash. 1879 */ 1880 if (!new_hash || !old_hash) 1881 return -EINVAL; 1882 1883 /* Update rec->flags */ 1884 do_for_each_ftrace_rec(pg, rec) { 1885 1886 if (rec->flags & FTRACE_FL_DISABLED) 1887 continue; 1888 1889 /* We need to update only differences of filter_hash */ 1890 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); 1891 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); 1892 if (in_old == in_new) 1893 continue; 1894 1895 if (in_new) { 1896 /* New entries must ensure no others are using it */ 1897 if (rec->flags & FTRACE_FL_IPMODIFY) 1898 goto rollback; 1899 rec->flags |= FTRACE_FL_IPMODIFY; 1900 } else /* Removed entry */ 1901 rec->flags &= ~FTRACE_FL_IPMODIFY; 1902 } while_for_each_ftrace_rec(); 1903 1904 return 0; 1905 1906 rollback: 1907 end = rec; 1908 1909 /* Roll back what we did above */ 1910 do_for_each_ftrace_rec(pg, rec) { 1911 1912 if (rec->flags & FTRACE_FL_DISABLED) 1913 continue; 1914 1915 if (rec == end) 1916 goto err_out; 1917 1918 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); 1919 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); 1920 if (in_old == in_new) 1921 continue; 1922 1923 if (in_new) 1924 rec->flags &= ~FTRACE_FL_IPMODIFY; 1925 else 1926 rec->flags |= FTRACE_FL_IPMODIFY; 1927 } while_for_each_ftrace_rec(); 1928 1929 err_out: 1930 return -EBUSY; 1931 } 1932 1933 static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops) 1934 { 1935 struct ftrace_hash *hash = ops->func_hash->filter_hash; 1936 1937 if (ftrace_hash_empty(hash)) 1938 hash = NULL; 1939 1940 return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash); 1941 } 1942 1943 /* Disabling always succeeds */ 1944 static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops) 1945 { 1946 struct ftrace_hash *hash = ops->func_hash->filter_hash; 1947 1948 if (ftrace_hash_empty(hash)) 1949 hash = NULL; 1950 1951 __ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH); 1952 } 1953 1954 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, 1955 struct ftrace_hash *new_hash) 1956 { 1957 struct ftrace_hash *old_hash = ops->func_hash->filter_hash; 1958 1959 if (ftrace_hash_empty(old_hash)) 1960 old_hash = NULL; 1961 1962 if (ftrace_hash_empty(new_hash)) 1963 new_hash = NULL; 1964 1965 return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash); 1966 } 1967 1968 static void print_ip_ins(const char *fmt, const unsigned char *p) 1969 { 1970 int i; 1971 1972 printk(KERN_CONT "%s", fmt); 1973 1974 for (i = 0; i < MCOUNT_INSN_SIZE; i++) 1975 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]); 1976 } 1977 1978 enum ftrace_bug_type ftrace_bug_type; 1979 const void *ftrace_expected; 1980 1981 static void print_bug_type(void) 1982 { 1983 switch (ftrace_bug_type) { 1984 case FTRACE_BUG_UNKNOWN: 1985 break; 1986 case FTRACE_BUG_INIT: 1987 pr_info("Initializing ftrace call sites\n"); 1988 break; 1989 case FTRACE_BUG_NOP: 1990 pr_info("Setting ftrace call site to NOP\n"); 1991 break; 1992 case FTRACE_BUG_CALL: 1993 pr_info("Setting ftrace call site to call ftrace function\n"); 1994 break; 1995 case FTRACE_BUG_UPDATE: 1996 pr_info("Updating ftrace call site to call a different ftrace function\n"); 1997 break; 1998 } 1999 } 2000 2001 /** 2002 * ftrace_bug - report and shutdown function tracer 2003 * @failed: The failed type (EFAULT, EINVAL, EPERM) 2004 * @rec: The record that failed 2005 * 2006 * The arch code that enables or disables the function tracing 2007 * can call ftrace_bug() when it has detected a problem in 2008 * modifying the code. @failed should be one of either: 2009 * EFAULT - if the problem happens on reading the @ip address 2010 * EINVAL - if what is read at @ip is not what was expected 2011 * EPERM - if the problem happens on writing to the @ip address 2012 */ 2013 void ftrace_bug(int failed, struct dyn_ftrace *rec) 2014 { 2015 unsigned long ip = rec ? rec->ip : 0; 2016 2017 pr_info("------------[ ftrace bug ]------------\n"); 2018 2019 switch (failed) { 2020 case -EFAULT: 2021 pr_info("ftrace faulted on modifying "); 2022 print_ip_sym(KERN_INFO, ip); 2023 break; 2024 case -EINVAL: 2025 pr_info("ftrace failed to modify "); 2026 print_ip_sym(KERN_INFO, ip); 2027 print_ip_ins(" actual: ", (unsigned char *)ip); 2028 pr_cont("\n"); 2029 if (ftrace_expected) { 2030 print_ip_ins(" expected: ", ftrace_expected); 2031 pr_cont("\n"); 2032 } 2033 break; 2034 case -EPERM: 2035 pr_info("ftrace faulted on writing "); 2036 print_ip_sym(KERN_INFO, ip); 2037 break; 2038 default: 2039 pr_info("ftrace faulted on unknown error "); 2040 print_ip_sym(KERN_INFO, ip); 2041 } 2042 print_bug_type(); 2043 if (rec) { 2044 struct ftrace_ops *ops = NULL; 2045 2046 pr_info("ftrace record flags: %lx\n", rec->flags); 2047 pr_cont(" (%ld)%s", ftrace_rec_count(rec), 2048 rec->flags & FTRACE_FL_REGS ? " R" : " "); 2049 if (rec->flags & FTRACE_FL_TRAMP_EN) { 2050 ops = ftrace_find_tramp_ops_any(rec); 2051 if (ops) { 2052 do { 2053 pr_cont("\ttramp: %pS (%pS)", 2054 (void *)ops->trampoline, 2055 (void *)ops->func); 2056 ops = ftrace_find_tramp_ops_next(rec, ops); 2057 } while (ops); 2058 } else 2059 pr_cont("\ttramp: ERROR!"); 2060 2061 } 2062 ip = ftrace_get_addr_curr(rec); 2063 pr_cont("\n expected tramp: %lx\n", ip); 2064 } 2065 2066 FTRACE_WARN_ON_ONCE(1); 2067 } 2068 2069 static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update) 2070 { 2071 unsigned long flag = 0UL; 2072 2073 ftrace_bug_type = FTRACE_BUG_UNKNOWN; 2074 2075 if (rec->flags & FTRACE_FL_DISABLED) 2076 return FTRACE_UPDATE_IGNORE; 2077 2078 /* 2079 * If we are updating calls: 2080 * 2081 * If the record has a ref count, then we need to enable it 2082 * because someone is using it. 2083 * 2084 * Otherwise we make sure its disabled. 2085 * 2086 * If we are disabling calls, then disable all records that 2087 * are enabled. 2088 */ 2089 if (enable && ftrace_rec_count(rec)) 2090 flag = FTRACE_FL_ENABLED; 2091 2092 /* 2093 * If enabling and the REGS flag does not match the REGS_EN, or 2094 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore 2095 * this record. Set flags to fail the compare against ENABLED. 2096 * Same for direct calls. 2097 */ 2098 if (flag) { 2099 if (!(rec->flags & FTRACE_FL_REGS) != 2100 !(rec->flags & FTRACE_FL_REGS_EN)) 2101 flag |= FTRACE_FL_REGS; 2102 2103 if (!(rec->flags & FTRACE_FL_TRAMP) != 2104 !(rec->flags & FTRACE_FL_TRAMP_EN)) 2105 flag |= FTRACE_FL_TRAMP; 2106 2107 /* 2108 * Direct calls are special, as count matters. 2109 * We must test the record for direct, if the 2110 * DIRECT and DIRECT_EN do not match, but only 2111 * if the count is 1. That's because, if the 2112 * count is something other than one, we do not 2113 * want the direct enabled (it will be done via the 2114 * direct helper). But if DIRECT_EN is set, and 2115 * the count is not one, we need to clear it. 2116 */ 2117 if (ftrace_rec_count(rec) == 1) { 2118 if (!(rec->flags & FTRACE_FL_DIRECT) != 2119 !(rec->flags & FTRACE_FL_DIRECT_EN)) 2120 flag |= FTRACE_FL_DIRECT; 2121 } else if (rec->flags & FTRACE_FL_DIRECT_EN) { 2122 flag |= FTRACE_FL_DIRECT; 2123 } 2124 } 2125 2126 /* If the state of this record hasn't changed, then do nothing */ 2127 if ((rec->flags & FTRACE_FL_ENABLED) == flag) 2128 return FTRACE_UPDATE_IGNORE; 2129 2130 if (flag) { 2131 /* Save off if rec is being enabled (for return value) */ 2132 flag ^= rec->flags & FTRACE_FL_ENABLED; 2133 2134 if (update) { 2135 rec->flags |= FTRACE_FL_ENABLED; 2136 if (flag & FTRACE_FL_REGS) { 2137 if (rec->flags & FTRACE_FL_REGS) 2138 rec->flags |= FTRACE_FL_REGS_EN; 2139 else 2140 rec->flags &= ~FTRACE_FL_REGS_EN; 2141 } 2142 if (flag & FTRACE_FL_TRAMP) { 2143 if (rec->flags & FTRACE_FL_TRAMP) 2144 rec->flags |= FTRACE_FL_TRAMP_EN; 2145 else 2146 rec->flags &= ~FTRACE_FL_TRAMP_EN; 2147 } 2148 2149 if (flag & FTRACE_FL_DIRECT) { 2150 /* 2151 * If there's only one user (direct_ops helper) 2152 * then we can call the direct function 2153 * directly (no ftrace trampoline). 2154 */ 2155 if (ftrace_rec_count(rec) == 1) { 2156 if (rec->flags & FTRACE_FL_DIRECT) 2157 rec->flags |= FTRACE_FL_DIRECT_EN; 2158 else 2159 rec->flags &= ~FTRACE_FL_DIRECT_EN; 2160 } else { 2161 /* 2162 * Can only call directly if there's 2163 * only one callback to the function. 2164 */ 2165 rec->flags &= ~FTRACE_FL_DIRECT_EN; 2166 } 2167 } 2168 } 2169 2170 /* 2171 * If this record is being updated from a nop, then 2172 * return UPDATE_MAKE_CALL. 2173 * Otherwise, 2174 * return UPDATE_MODIFY_CALL to tell the caller to convert 2175 * from the save regs, to a non-save regs function or 2176 * vice versa, or from a trampoline call. 2177 */ 2178 if (flag & FTRACE_FL_ENABLED) { 2179 ftrace_bug_type = FTRACE_BUG_CALL; 2180 return FTRACE_UPDATE_MAKE_CALL; 2181 } 2182 2183 ftrace_bug_type = FTRACE_BUG_UPDATE; 2184 return FTRACE_UPDATE_MODIFY_CALL; 2185 } 2186 2187 if (update) { 2188 /* If there's no more users, clear all flags */ 2189 if (!ftrace_rec_count(rec)) 2190 rec->flags = 0; 2191 else 2192 /* 2193 * Just disable the record, but keep the ops TRAMP 2194 * and REGS states. The _EN flags must be disabled though. 2195 */ 2196 rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN | 2197 FTRACE_FL_REGS_EN | FTRACE_FL_DIRECT_EN); 2198 } 2199 2200 ftrace_bug_type = FTRACE_BUG_NOP; 2201 return FTRACE_UPDATE_MAKE_NOP; 2202 } 2203 2204 /** 2205 * ftrace_update_record, set a record that now is tracing or not 2206 * @rec: the record to update 2207 * @enable: set to true if the record is tracing, false to force disable 2208 * 2209 * The records that represent all functions that can be traced need 2210 * to be updated when tracing has been enabled. 2211 */ 2212 int ftrace_update_record(struct dyn_ftrace *rec, bool enable) 2213 { 2214 return ftrace_check_record(rec, enable, true); 2215 } 2216 2217 /** 2218 * ftrace_test_record, check if the record has been enabled or not 2219 * @rec: the record to test 2220 * @enable: set to true to check if enabled, false if it is disabled 2221 * 2222 * The arch code may need to test if a record is already set to 2223 * tracing to determine how to modify the function code that it 2224 * represents. 2225 */ 2226 int ftrace_test_record(struct dyn_ftrace *rec, bool enable) 2227 { 2228 return ftrace_check_record(rec, enable, false); 2229 } 2230 2231 static struct ftrace_ops * 2232 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec) 2233 { 2234 struct ftrace_ops *op; 2235 unsigned long ip = rec->ip; 2236 2237 do_for_each_ftrace_op(op, ftrace_ops_list) { 2238 2239 if (!op->trampoline) 2240 continue; 2241 2242 if (hash_contains_ip(ip, op->func_hash)) 2243 return op; 2244 } while_for_each_ftrace_op(op); 2245 2246 return NULL; 2247 } 2248 2249 static struct ftrace_ops * 2250 ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude) 2251 { 2252 struct ftrace_ops *op; 2253 unsigned long ip = rec->ip; 2254 2255 do_for_each_ftrace_op(op, ftrace_ops_list) { 2256 2257 if (op == op_exclude || !op->trampoline) 2258 continue; 2259 2260 if (hash_contains_ip(ip, op->func_hash)) 2261 return op; 2262 } while_for_each_ftrace_op(op); 2263 2264 return NULL; 2265 } 2266 2267 static struct ftrace_ops * 2268 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, 2269 struct ftrace_ops *op) 2270 { 2271 unsigned long ip = rec->ip; 2272 2273 while_for_each_ftrace_op(op) { 2274 2275 if (!op->trampoline) 2276 continue; 2277 2278 if (hash_contains_ip(ip, op->func_hash)) 2279 return op; 2280 } 2281 2282 return NULL; 2283 } 2284 2285 static struct ftrace_ops * 2286 ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec) 2287 { 2288 struct ftrace_ops *op; 2289 unsigned long ip = rec->ip; 2290 2291 /* 2292 * Need to check removed ops first. 2293 * If they are being removed, and this rec has a tramp, 2294 * and this rec is in the ops list, then it would be the 2295 * one with the tramp. 2296 */ 2297 if (removed_ops) { 2298 if (hash_contains_ip(ip, &removed_ops->old_hash)) 2299 return removed_ops; 2300 } 2301 2302 /* 2303 * Need to find the current trampoline for a rec. 2304 * Now, a trampoline is only attached to a rec if there 2305 * was a single 'ops' attached to it. But this can be called 2306 * when we are adding another op to the rec or removing the 2307 * current one. Thus, if the op is being added, we can 2308 * ignore it because it hasn't attached itself to the rec 2309 * yet. 2310 * 2311 * If an ops is being modified (hooking to different functions) 2312 * then we don't care about the new functions that are being 2313 * added, just the old ones (that are probably being removed). 2314 * 2315 * If we are adding an ops to a function that already is using 2316 * a trampoline, it needs to be removed (trampolines are only 2317 * for single ops connected), then an ops that is not being 2318 * modified also needs to be checked. 2319 */ 2320 do_for_each_ftrace_op(op, ftrace_ops_list) { 2321 2322 if (!op->trampoline) 2323 continue; 2324 2325 /* 2326 * If the ops is being added, it hasn't gotten to 2327 * the point to be removed from this tree yet. 2328 */ 2329 if (op->flags & FTRACE_OPS_FL_ADDING) 2330 continue; 2331 2332 2333 /* 2334 * If the ops is being modified and is in the old 2335 * hash, then it is probably being removed from this 2336 * function. 2337 */ 2338 if ((op->flags & FTRACE_OPS_FL_MODIFYING) && 2339 hash_contains_ip(ip, &op->old_hash)) 2340 return op; 2341 /* 2342 * If the ops is not being added or modified, and it's 2343 * in its normal filter hash, then this must be the one 2344 * we want! 2345 */ 2346 if (!(op->flags & FTRACE_OPS_FL_MODIFYING) && 2347 hash_contains_ip(ip, op->func_hash)) 2348 return op; 2349 2350 } while_for_each_ftrace_op(op); 2351 2352 return NULL; 2353 } 2354 2355 static struct ftrace_ops * 2356 ftrace_find_tramp_ops_new(struct dyn_ftrace *rec) 2357 { 2358 struct ftrace_ops *op; 2359 unsigned long ip = rec->ip; 2360 2361 do_for_each_ftrace_op(op, ftrace_ops_list) { 2362 /* pass rec in as regs to have non-NULL val */ 2363 if (hash_contains_ip(ip, op->func_hash)) 2364 return op; 2365 } while_for_each_ftrace_op(op); 2366 2367 return NULL; 2368 } 2369 2370 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS 2371 /* Protected by rcu_tasks for reading, and direct_mutex for writing */ 2372 static struct ftrace_hash *direct_functions = EMPTY_HASH; 2373 static DEFINE_MUTEX(direct_mutex); 2374 int ftrace_direct_func_count; 2375 2376 /* 2377 * Search the direct_functions hash to see if the given instruction pointer 2378 * has a direct caller attached to it. 2379 */ 2380 unsigned long ftrace_find_rec_direct(unsigned long ip) 2381 { 2382 struct ftrace_func_entry *entry; 2383 2384 entry = __ftrace_lookup_ip(direct_functions, ip); 2385 if (!entry) 2386 return 0; 2387 2388 return entry->direct; 2389 } 2390 2391 static void call_direct_funcs(unsigned long ip, unsigned long pip, 2392 struct ftrace_ops *ops, struct ftrace_regs *fregs) 2393 { 2394 struct pt_regs *regs = ftrace_get_regs(fregs); 2395 unsigned long addr; 2396 2397 addr = ftrace_find_rec_direct(ip); 2398 if (!addr) 2399 return; 2400 2401 arch_ftrace_set_direct_caller(regs, addr); 2402 } 2403 2404 struct ftrace_ops direct_ops = { 2405 .func = call_direct_funcs, 2406 .flags = FTRACE_OPS_FL_IPMODIFY 2407 | FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS 2408 | FTRACE_OPS_FL_PERMANENT, 2409 /* 2410 * By declaring the main trampoline as this trampoline 2411 * it will never have one allocated for it. Allocated 2412 * trampolines should not call direct functions. 2413 * The direct_ops should only be called by the builtin 2414 * ftrace_regs_caller trampoline. 2415 */ 2416 .trampoline = FTRACE_REGS_ADDR, 2417 }; 2418 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ 2419 2420 /** 2421 * ftrace_get_addr_new - Get the call address to set to 2422 * @rec: The ftrace record descriptor 2423 * 2424 * If the record has the FTRACE_FL_REGS set, that means that it 2425 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS 2426 * is not set, then it wants to convert to the normal callback. 2427 * 2428 * Returns the address of the trampoline to set to 2429 */ 2430 unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec) 2431 { 2432 struct ftrace_ops *ops; 2433 unsigned long addr; 2434 2435 if ((rec->flags & FTRACE_FL_DIRECT) && 2436 (ftrace_rec_count(rec) == 1)) { 2437 addr = ftrace_find_rec_direct(rec->ip); 2438 if (addr) 2439 return addr; 2440 WARN_ON_ONCE(1); 2441 } 2442 2443 /* Trampolines take precedence over regs */ 2444 if (rec->flags & FTRACE_FL_TRAMP) { 2445 ops = ftrace_find_tramp_ops_new(rec); 2446 if (FTRACE_WARN_ON(!ops || !ops->trampoline)) { 2447 pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n", 2448 (void *)rec->ip, (void *)rec->ip, rec->flags); 2449 /* Ftrace is shutting down, return anything */ 2450 return (unsigned long)FTRACE_ADDR; 2451 } 2452 return ops->trampoline; 2453 } 2454 2455 if (rec->flags & FTRACE_FL_REGS) 2456 return (unsigned long)FTRACE_REGS_ADDR; 2457 else 2458 return (unsigned long)FTRACE_ADDR; 2459 } 2460 2461 /** 2462 * ftrace_get_addr_curr - Get the call address that is already there 2463 * @rec: The ftrace record descriptor 2464 * 2465 * The FTRACE_FL_REGS_EN is set when the record already points to 2466 * a function that saves all the regs. Basically the '_EN' version 2467 * represents the current state of the function. 2468 * 2469 * Returns the address of the trampoline that is currently being called 2470 */ 2471 unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec) 2472 { 2473 struct ftrace_ops *ops; 2474 unsigned long addr; 2475 2476 /* Direct calls take precedence over trampolines */ 2477 if (rec->flags & FTRACE_FL_DIRECT_EN) { 2478 addr = ftrace_find_rec_direct(rec->ip); 2479 if (addr) 2480 return addr; 2481 WARN_ON_ONCE(1); 2482 } 2483 2484 /* Trampolines take precedence over regs */ 2485 if (rec->flags & FTRACE_FL_TRAMP_EN) { 2486 ops = ftrace_find_tramp_ops_curr(rec); 2487 if (FTRACE_WARN_ON(!ops)) { 2488 pr_warn("Bad trampoline accounting at: %p (%pS)\n", 2489 (void *)rec->ip, (void *)rec->ip); 2490 /* Ftrace is shutting down, return anything */ 2491 return (unsigned long)FTRACE_ADDR; 2492 } 2493 return ops->trampoline; 2494 } 2495 2496 if (rec->flags & FTRACE_FL_REGS_EN) 2497 return (unsigned long)FTRACE_REGS_ADDR; 2498 else 2499 return (unsigned long)FTRACE_ADDR; 2500 } 2501 2502 static int 2503 __ftrace_replace_code(struct dyn_ftrace *rec, bool enable) 2504 { 2505 unsigned long ftrace_old_addr; 2506 unsigned long ftrace_addr; 2507 int ret; 2508 2509 ftrace_addr = ftrace_get_addr_new(rec); 2510 2511 /* This needs to be done before we call ftrace_update_record */ 2512 ftrace_old_addr = ftrace_get_addr_curr(rec); 2513 2514 ret = ftrace_update_record(rec, enable); 2515 2516 ftrace_bug_type = FTRACE_BUG_UNKNOWN; 2517 2518 switch (ret) { 2519 case FTRACE_UPDATE_IGNORE: 2520 return 0; 2521 2522 case FTRACE_UPDATE_MAKE_CALL: 2523 ftrace_bug_type = FTRACE_BUG_CALL; 2524 return ftrace_make_call(rec, ftrace_addr); 2525 2526 case FTRACE_UPDATE_MAKE_NOP: 2527 ftrace_bug_type = FTRACE_BUG_NOP; 2528 return ftrace_make_nop(NULL, rec, ftrace_old_addr); 2529 2530 case FTRACE_UPDATE_MODIFY_CALL: 2531 ftrace_bug_type = FTRACE_BUG_UPDATE; 2532 return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr); 2533 } 2534 2535 return -1; /* unknown ftrace bug */ 2536 } 2537 2538 void __weak ftrace_replace_code(int mod_flags) 2539 { 2540 struct dyn_ftrace *rec; 2541 struct ftrace_page *pg; 2542 bool enable = mod_flags & FTRACE_MODIFY_ENABLE_FL; 2543 int schedulable = mod_flags & FTRACE_MODIFY_MAY_SLEEP_FL; 2544 int failed; 2545 2546 if (unlikely(ftrace_disabled)) 2547 return; 2548 2549 do_for_each_ftrace_rec(pg, rec) { 2550 2551 if (rec->flags & FTRACE_FL_DISABLED) 2552 continue; 2553 2554 failed = __ftrace_replace_code(rec, enable); 2555 if (failed) { 2556 ftrace_bug(failed, rec); 2557 /* Stop processing */ 2558 return; 2559 } 2560 if (schedulable) 2561 cond_resched(); 2562 } while_for_each_ftrace_rec(); 2563 } 2564 2565 struct ftrace_rec_iter { 2566 struct ftrace_page *pg; 2567 int index; 2568 }; 2569 2570 /** 2571 * ftrace_rec_iter_start, start up iterating over traced functions 2572 * 2573 * Returns an iterator handle that is used to iterate over all 2574 * the records that represent address locations where functions 2575 * are traced. 2576 * 2577 * May return NULL if no records are available. 2578 */ 2579 struct ftrace_rec_iter *ftrace_rec_iter_start(void) 2580 { 2581 /* 2582 * We only use a single iterator. 2583 * Protected by the ftrace_lock mutex. 2584 */ 2585 static struct ftrace_rec_iter ftrace_rec_iter; 2586 struct ftrace_rec_iter *iter = &ftrace_rec_iter; 2587 2588 iter->pg = ftrace_pages_start; 2589 iter->index = 0; 2590 2591 /* Could have empty pages */ 2592 while (iter->pg && !iter->pg->index) 2593 iter->pg = iter->pg->next; 2594 2595 if (!iter->pg) 2596 return NULL; 2597 2598 return iter; 2599 } 2600 2601 /** 2602 * ftrace_rec_iter_next, get the next record to process. 2603 * @iter: The handle to the iterator. 2604 * 2605 * Returns the next iterator after the given iterator @iter. 2606 */ 2607 struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter) 2608 { 2609 iter->index++; 2610 2611 if (iter->index >= iter->pg->index) { 2612 iter->pg = iter->pg->next; 2613 iter->index = 0; 2614 2615 /* Could have empty pages */ 2616 while (iter->pg && !iter->pg->index) 2617 iter->pg = iter->pg->next; 2618 } 2619 2620 if (!iter->pg) 2621 return NULL; 2622 2623 return iter; 2624 } 2625 2626 /** 2627 * ftrace_rec_iter_record, get the record at the iterator location 2628 * @iter: The current iterator location 2629 * 2630 * Returns the record that the current @iter is at. 2631 */ 2632 struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter) 2633 { 2634 return &iter->pg->records[iter->index]; 2635 } 2636 2637 static int 2638 ftrace_nop_initialize(struct module *mod, struct dyn_ftrace *rec) 2639 { 2640 int ret; 2641 2642 if (unlikely(ftrace_disabled)) 2643 return 0; 2644 2645 ret = ftrace_init_nop(mod, rec); 2646 if (ret) { 2647 ftrace_bug_type = FTRACE_BUG_INIT; 2648 ftrace_bug(ret, rec); 2649 return 0; 2650 } 2651 return 1; 2652 } 2653 2654 /* 2655 * archs can override this function if they must do something 2656 * before the modifying code is performed. 2657 */ 2658 int __weak ftrace_arch_code_modify_prepare(void) 2659 { 2660 return 0; 2661 } 2662 2663 /* 2664 * archs can override this function if they must do something 2665 * after the modifying code is performed. 2666 */ 2667 int __weak ftrace_arch_code_modify_post_process(void) 2668 { 2669 return 0; 2670 } 2671 2672 void ftrace_modify_all_code(int command) 2673 { 2674 int update = command & FTRACE_UPDATE_TRACE_FUNC; 2675 int mod_flags = 0; 2676 int err = 0; 2677 2678 if (command & FTRACE_MAY_SLEEP) 2679 mod_flags = FTRACE_MODIFY_MAY_SLEEP_FL; 2680 2681 /* 2682 * If the ftrace_caller calls a ftrace_ops func directly, 2683 * we need to make sure that it only traces functions it 2684 * expects to trace. When doing the switch of functions, 2685 * we need to update to the ftrace_ops_list_func first 2686 * before the transition between old and new calls are set, 2687 * as the ftrace_ops_list_func will check the ops hashes 2688 * to make sure the ops are having the right functions 2689 * traced. 2690 */ 2691 if (update) { 2692 err = ftrace_update_ftrace_func(ftrace_ops_list_func); 2693 if (FTRACE_WARN_ON(err)) 2694 return; 2695 } 2696 2697 if (command & FTRACE_UPDATE_CALLS) 2698 ftrace_replace_code(mod_flags | FTRACE_MODIFY_ENABLE_FL); 2699 else if (command & FTRACE_DISABLE_CALLS) 2700 ftrace_replace_code(mod_flags); 2701 2702 if (update && ftrace_trace_function != ftrace_ops_list_func) { 2703 function_trace_op = set_function_trace_op; 2704 smp_wmb(); 2705 /* If irqs are disabled, we are in stop machine */ 2706 if (!irqs_disabled()) 2707 smp_call_function(ftrace_sync_ipi, NULL, 1); 2708 err = ftrace_update_ftrace_func(ftrace_trace_function); 2709 if (FTRACE_WARN_ON(err)) 2710 return; 2711 } 2712 2713 if (command & FTRACE_START_FUNC_RET) 2714 err = ftrace_enable_ftrace_graph_caller(); 2715 else if (command & FTRACE_STOP_FUNC_RET) 2716 err = ftrace_disable_ftrace_graph_caller(); 2717 FTRACE_WARN_ON(err); 2718 } 2719 2720 static int __ftrace_modify_code(void *data) 2721 { 2722 int *command = data; 2723 2724 ftrace_modify_all_code(*command); 2725 2726 return 0; 2727 } 2728 2729 /** 2730 * ftrace_run_stop_machine, go back to the stop machine method 2731 * @command: The command to tell ftrace what to do 2732 * 2733 * If an arch needs to fall back to the stop machine method, the 2734 * it can call this function. 2735 */ 2736 void ftrace_run_stop_machine(int command) 2737 { 2738 stop_machine(__ftrace_modify_code, &command, NULL); 2739 } 2740 2741 /** 2742 * arch_ftrace_update_code, modify the code to trace or not trace 2743 * @command: The command that needs to be done 2744 * 2745 * Archs can override this function if it does not need to 2746 * run stop_machine() to modify code. 2747 */ 2748 void __weak arch_ftrace_update_code(int command) 2749 { 2750 ftrace_run_stop_machine(command); 2751 } 2752 2753 static void ftrace_run_update_code(int command) 2754 { 2755 int ret; 2756 2757 ret = ftrace_arch_code_modify_prepare(); 2758 FTRACE_WARN_ON(ret); 2759 if (ret) 2760 return; 2761 2762 /* 2763 * By default we use stop_machine() to modify the code. 2764 * But archs can do what ever they want as long as it 2765 * is safe. The stop_machine() is the safest, but also 2766 * produces the most overhead. 2767 */ 2768 arch_ftrace_update_code(command); 2769 2770 ret = ftrace_arch_code_modify_post_process(); 2771 FTRACE_WARN_ON(ret); 2772 } 2773 2774 static void ftrace_run_modify_code(struct ftrace_ops *ops, int command, 2775 struct ftrace_ops_hash *old_hash) 2776 { 2777 ops->flags |= FTRACE_OPS_FL_MODIFYING; 2778 ops->old_hash.filter_hash = old_hash->filter_hash; 2779 ops->old_hash.notrace_hash = old_hash->notrace_hash; 2780 ftrace_run_update_code(command); 2781 ops->old_hash.filter_hash = NULL; 2782 ops->old_hash.notrace_hash = NULL; 2783 ops->flags &= ~FTRACE_OPS_FL_MODIFYING; 2784 } 2785 2786 static ftrace_func_t saved_ftrace_func; 2787 static int ftrace_start_up; 2788 2789 void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops) 2790 { 2791 } 2792 2793 /* List of trace_ops that have allocated trampolines */ 2794 static LIST_HEAD(ftrace_ops_trampoline_list); 2795 2796 static void ftrace_add_trampoline_to_kallsyms(struct ftrace_ops *ops) 2797 { 2798 lockdep_assert_held(&ftrace_lock); 2799 list_add_rcu(&ops->list, &ftrace_ops_trampoline_list); 2800 } 2801 2802 static void ftrace_remove_trampoline_from_kallsyms(struct ftrace_ops *ops) 2803 { 2804 lockdep_assert_held(&ftrace_lock); 2805 list_del_rcu(&ops->list); 2806 synchronize_rcu(); 2807 } 2808 2809 /* 2810 * "__builtin__ftrace" is used as a module name in /proc/kallsyms for symbols 2811 * for pages allocated for ftrace purposes, even though "__builtin__ftrace" is 2812 * not a module. 2813 */ 2814 #define FTRACE_TRAMPOLINE_MOD "__builtin__ftrace" 2815 #define FTRACE_TRAMPOLINE_SYM "ftrace_trampoline" 2816 2817 static void ftrace_trampoline_free(struct ftrace_ops *ops) 2818 { 2819 if (ops && (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP) && 2820 ops->trampoline) { 2821 /* 2822 * Record the text poke event before the ksymbol unregister 2823 * event. 2824 */ 2825 perf_event_text_poke((void *)ops->trampoline, 2826 (void *)ops->trampoline, 2827 ops->trampoline_size, NULL, 0); 2828 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, 2829 ops->trampoline, ops->trampoline_size, 2830 true, FTRACE_TRAMPOLINE_SYM); 2831 /* Remove from kallsyms after the perf events */ 2832 ftrace_remove_trampoline_from_kallsyms(ops); 2833 } 2834 2835 arch_ftrace_trampoline_free(ops); 2836 } 2837 2838 static void ftrace_startup_enable(int command) 2839 { 2840 if (saved_ftrace_func != ftrace_trace_function) { 2841 saved_ftrace_func = ftrace_trace_function; 2842 command |= FTRACE_UPDATE_TRACE_FUNC; 2843 } 2844 2845 if (!command || !ftrace_enabled) 2846 return; 2847 2848 ftrace_run_update_code(command); 2849 } 2850 2851 static void ftrace_startup_all(int command) 2852 { 2853 update_all_ops = true; 2854 ftrace_startup_enable(command); 2855 update_all_ops = false; 2856 } 2857 2858 int ftrace_startup(struct ftrace_ops *ops, int command) 2859 { 2860 int ret; 2861 2862 if (unlikely(ftrace_disabled)) 2863 return -ENODEV; 2864 2865 ret = __register_ftrace_function(ops); 2866 if (ret) 2867 return ret; 2868 2869 ftrace_start_up++; 2870 2871 /* 2872 * Note that ftrace probes uses this to start up 2873 * and modify functions it will probe. But we still 2874 * set the ADDING flag for modification, as probes 2875 * do not have trampolines. If they add them in the 2876 * future, then the probes will need to distinguish 2877 * between adding and updating probes. 2878 */ 2879 ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING; 2880 2881 ret = ftrace_hash_ipmodify_enable(ops); 2882 if (ret < 0) { 2883 /* Rollback registration process */ 2884 __unregister_ftrace_function(ops); 2885 ftrace_start_up--; 2886 ops->flags &= ~FTRACE_OPS_FL_ENABLED; 2887 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) 2888 ftrace_trampoline_free(ops); 2889 return ret; 2890 } 2891 2892 if (ftrace_hash_rec_enable(ops, 1)) 2893 command |= FTRACE_UPDATE_CALLS; 2894 2895 ftrace_startup_enable(command); 2896 2897 ops->flags &= ~FTRACE_OPS_FL_ADDING; 2898 2899 return 0; 2900 } 2901 2902 int ftrace_shutdown(struct ftrace_ops *ops, int command) 2903 { 2904 int ret; 2905 2906 if (unlikely(ftrace_disabled)) 2907 return -ENODEV; 2908 2909 ret = __unregister_ftrace_function(ops); 2910 if (ret) 2911 return ret; 2912 2913 ftrace_start_up--; 2914 /* 2915 * Just warn in case of unbalance, no need to kill ftrace, it's not 2916 * critical but the ftrace_call callers may be never nopped again after 2917 * further ftrace uses. 2918 */ 2919 WARN_ON_ONCE(ftrace_start_up < 0); 2920 2921 /* Disabling ipmodify never fails */ 2922 ftrace_hash_ipmodify_disable(ops); 2923 2924 if (ftrace_hash_rec_disable(ops, 1)) 2925 command |= FTRACE_UPDATE_CALLS; 2926 2927 ops->flags &= ~FTRACE_OPS_FL_ENABLED; 2928 2929 if (saved_ftrace_func != ftrace_trace_function) { 2930 saved_ftrace_func = ftrace_trace_function; 2931 command |= FTRACE_UPDATE_TRACE_FUNC; 2932 } 2933 2934 if (!command || !ftrace_enabled) { 2935 /* 2936 * If these are dynamic or per_cpu ops, they still 2937 * need their data freed. Since, function tracing is 2938 * not currently active, we can just free them 2939 * without synchronizing all CPUs. 2940 */ 2941 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) 2942 goto free_ops; 2943 2944 return 0; 2945 } 2946 2947 /* 2948 * If the ops uses a trampoline, then it needs to be 2949 * tested first on update. 2950 */ 2951 ops->flags |= FTRACE_OPS_FL_REMOVING; 2952 removed_ops = ops; 2953 2954 /* The trampoline logic checks the old hashes */ 2955 ops->old_hash.filter_hash = ops->func_hash->filter_hash; 2956 ops->old_hash.notrace_hash = ops->func_hash->notrace_hash; 2957 2958 ftrace_run_update_code(command); 2959 2960 /* 2961 * If there's no more ops registered with ftrace, run a 2962 * sanity check to make sure all rec flags are cleared. 2963 */ 2964 if (rcu_dereference_protected(ftrace_ops_list, 2965 lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) { 2966 struct ftrace_page *pg; 2967 struct dyn_ftrace *rec; 2968 2969 do_for_each_ftrace_rec(pg, rec) { 2970 if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_FL_DISABLED)) 2971 pr_warn(" %pS flags:%lx\n", 2972 (void *)rec->ip, rec->flags); 2973 } while_for_each_ftrace_rec(); 2974 } 2975 2976 ops->old_hash.filter_hash = NULL; 2977 ops->old_hash.notrace_hash = NULL; 2978 2979 removed_ops = NULL; 2980 ops->flags &= ~FTRACE_OPS_FL_REMOVING; 2981 2982 /* 2983 * Dynamic ops may be freed, we must make sure that all 2984 * callers are done before leaving this function. 2985 * The same goes for freeing the per_cpu data of the per_cpu 2986 * ops. 2987 */ 2988 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) { 2989 /* 2990 * We need to do a hard force of sched synchronization. 2991 * This is because we use preempt_disable() to do RCU, but 2992 * the function tracers can be called where RCU is not watching 2993 * (like before user_exit()). We can not rely on the RCU 2994 * infrastructure to do the synchronization, thus we must do it 2995 * ourselves. 2996 */ 2997 synchronize_rcu_tasks_rude(); 2998 2999 /* 3000 * When the kernel is preemptive, tasks can be preempted 3001 * while on a ftrace trampoline. Just scheduling a task on 3002 * a CPU is not good enough to flush them. Calling 3003 * synchornize_rcu_tasks() will wait for those tasks to 3004 * execute and either schedule voluntarily or enter user space. 3005 */ 3006 if (IS_ENABLED(CONFIG_PREEMPTION)) 3007 synchronize_rcu_tasks(); 3008 3009 free_ops: 3010 ftrace_trampoline_free(ops); 3011 } 3012 3013 return 0; 3014 } 3015 3016 static void ftrace_startup_sysctl(void) 3017 { 3018 int command; 3019 3020 if (unlikely(ftrace_disabled)) 3021 return; 3022 3023 /* Force update next time */ 3024 saved_ftrace_func = NULL; 3025 /* ftrace_start_up is true if we want ftrace running */ 3026 if (ftrace_start_up) { 3027 command = FTRACE_UPDATE_CALLS; 3028 if (ftrace_graph_active) 3029 command |= FTRACE_START_FUNC_RET; 3030 ftrace_startup_enable(command); 3031 } 3032 } 3033 3034 static void ftrace_shutdown_sysctl(void) 3035 { 3036 int command; 3037 3038 if (unlikely(ftrace_disabled)) 3039 return; 3040 3041 /* ftrace_start_up is true if ftrace is running */ 3042 if (ftrace_start_up) { 3043 command = FTRACE_DISABLE_CALLS; 3044 if (ftrace_graph_active) 3045 command |= FTRACE_STOP_FUNC_RET; 3046 ftrace_run_update_code(command); 3047 } 3048 } 3049 3050 static u64 ftrace_update_time; 3051 unsigned long ftrace_update_tot_cnt; 3052 unsigned long ftrace_number_of_pages; 3053 unsigned long ftrace_number_of_groups; 3054 3055 static inline int ops_traces_mod(struct ftrace_ops *ops) 3056 { 3057 /* 3058 * Filter_hash being empty will default to trace module. 3059 * But notrace hash requires a test of individual module functions. 3060 */ 3061 return ftrace_hash_empty(ops->func_hash->filter_hash) && 3062 ftrace_hash_empty(ops->func_hash->notrace_hash); 3063 } 3064 3065 /* 3066 * Check if the current ops references the record. 3067 * 3068 * If the ops traces all functions, then it was already accounted for. 3069 * If the ops does not trace the current record function, skip it. 3070 * If the ops ignores the function via notrace filter, skip it. 3071 */ 3072 static inline bool 3073 ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec) 3074 { 3075 /* If ops isn't enabled, ignore it */ 3076 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 3077 return false; 3078 3079 /* If ops traces all then it includes this function */ 3080 if (ops_traces_mod(ops)) 3081 return true; 3082 3083 /* The function must be in the filter */ 3084 if (!ftrace_hash_empty(ops->func_hash->filter_hash) && 3085 !__ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip)) 3086 return false; 3087 3088 /* If in notrace hash, we ignore it too */ 3089 if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) 3090 return false; 3091 3092 return true; 3093 } 3094 3095 static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs) 3096 { 3097 struct ftrace_page *pg; 3098 struct dyn_ftrace *p; 3099 u64 start, stop; 3100 unsigned long update_cnt = 0; 3101 unsigned long rec_flags = 0; 3102 int i; 3103 3104 start = ftrace_now(raw_smp_processor_id()); 3105 3106 /* 3107 * When a module is loaded, this function is called to convert 3108 * the calls to mcount in its text to nops, and also to create 3109 * an entry in the ftrace data. Now, if ftrace is activated 3110 * after this call, but before the module sets its text to 3111 * read-only, the modification of enabling ftrace can fail if 3112 * the read-only is done while ftrace is converting the calls. 3113 * To prevent this, the module's records are set as disabled 3114 * and will be enabled after the call to set the module's text 3115 * to read-only. 3116 */ 3117 if (mod) 3118 rec_flags |= FTRACE_FL_DISABLED; 3119 3120 for (pg = new_pgs; pg; pg = pg->next) { 3121 3122 for (i = 0; i < pg->index; i++) { 3123 3124 /* If something went wrong, bail without enabling anything */ 3125 if (unlikely(ftrace_disabled)) 3126 return -1; 3127 3128 p = &pg->records[i]; 3129 p->flags = rec_flags; 3130 3131 /* 3132 * Do the initial record conversion from mcount jump 3133 * to the NOP instructions. 3134 */ 3135 if (!__is_defined(CC_USING_NOP_MCOUNT) && 3136 !ftrace_nop_initialize(mod, p)) 3137 break; 3138 3139 update_cnt++; 3140 } 3141 } 3142 3143 stop = ftrace_now(raw_smp_processor_id()); 3144 ftrace_update_time = stop - start; 3145 ftrace_update_tot_cnt += update_cnt; 3146 3147 return 0; 3148 } 3149 3150 static int ftrace_allocate_records(struct ftrace_page *pg, int count) 3151 { 3152 int order; 3153 int pages; 3154 int cnt; 3155 3156 if (WARN_ON(!count)) 3157 return -EINVAL; 3158 3159 pages = DIV_ROUND_UP(count, ENTRIES_PER_PAGE); 3160 order = get_count_order(pages); 3161 3162 /* 3163 * We want to fill as much as possible. No more than a page 3164 * may be empty. 3165 */ 3166 if (!is_power_of_2(pages)) 3167 order--; 3168 3169 again: 3170 pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); 3171 3172 if (!pg->records) { 3173 /* if we can't allocate this size, try something smaller */ 3174 if (!order) 3175 return -ENOMEM; 3176 order >>= 1; 3177 goto again; 3178 } 3179 3180 ftrace_number_of_pages += 1 << order; 3181 ftrace_number_of_groups++; 3182 3183 cnt = (PAGE_SIZE << order) / ENTRY_SIZE; 3184 pg->size = cnt; 3185 3186 if (cnt > count) 3187 cnt = count; 3188 3189 return cnt; 3190 } 3191 3192 static struct ftrace_page * 3193 ftrace_allocate_pages(unsigned long num_to_init) 3194 { 3195 struct ftrace_page *start_pg; 3196 struct ftrace_page *pg; 3197 int order; 3198 int cnt; 3199 3200 if (!num_to_init) 3201 return NULL; 3202 3203 start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL); 3204 if (!pg) 3205 return NULL; 3206 3207 /* 3208 * Try to allocate as much as possible in one continues 3209 * location that fills in all of the space. We want to 3210 * waste as little space as possible. 3211 */ 3212 for (;;) { 3213 cnt = ftrace_allocate_records(pg, num_to_init); 3214 if (cnt < 0) 3215 goto free_pages; 3216 3217 num_to_init -= cnt; 3218 if (!num_to_init) 3219 break; 3220 3221 pg->next = kzalloc(sizeof(*pg), GFP_KERNEL); 3222 if (!pg->next) 3223 goto free_pages; 3224 3225 pg = pg->next; 3226 } 3227 3228 return start_pg; 3229 3230 free_pages: 3231 pg = start_pg; 3232 while (pg) { 3233 order = get_count_order(pg->size / ENTRIES_PER_PAGE); 3234 if (order >= 0) 3235 free_pages((unsigned long)pg->records, order); 3236 start_pg = pg->next; 3237 kfree(pg); 3238 pg = start_pg; 3239 ftrace_number_of_pages -= 1 << order; 3240 ftrace_number_of_groups--; 3241 } 3242 pr_info("ftrace: FAILED to allocate memory for functions\n"); 3243 return NULL; 3244 } 3245 3246 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ 3247 3248 struct ftrace_iterator { 3249 loff_t pos; 3250 loff_t func_pos; 3251 loff_t mod_pos; 3252 struct ftrace_page *pg; 3253 struct dyn_ftrace *func; 3254 struct ftrace_func_probe *probe; 3255 struct ftrace_func_entry *probe_entry; 3256 struct trace_parser parser; 3257 struct ftrace_hash *hash; 3258 struct ftrace_ops *ops; 3259 struct trace_array *tr; 3260 struct list_head *mod_list; 3261 int pidx; 3262 int idx; 3263 unsigned flags; 3264 }; 3265 3266 static void * 3267 t_probe_next(struct seq_file *m, loff_t *pos) 3268 { 3269 struct ftrace_iterator *iter = m->private; 3270 struct trace_array *tr = iter->ops->private; 3271 struct list_head *func_probes; 3272 struct ftrace_hash *hash; 3273 struct list_head *next; 3274 struct hlist_node *hnd = NULL; 3275 struct hlist_head *hhd; 3276 int size; 3277 3278 (*pos)++; 3279 iter->pos = *pos; 3280 3281 if (!tr) 3282 return NULL; 3283 3284 func_probes = &tr->func_probes; 3285 if (list_empty(func_probes)) 3286 return NULL; 3287 3288 if (!iter->probe) { 3289 next = func_probes->next; 3290 iter->probe = list_entry(next, struct ftrace_func_probe, list); 3291 } 3292 3293 if (iter->probe_entry) 3294 hnd = &iter->probe_entry->hlist; 3295 3296 hash = iter->probe->ops.func_hash->filter_hash; 3297 3298 /* 3299 * A probe being registered may temporarily have an empty hash 3300 * and it's at the end of the func_probes list. 3301 */ 3302 if (!hash || hash == EMPTY_HASH) 3303 return NULL; 3304 3305 size = 1 << hash->size_bits; 3306 3307 retry: 3308 if (iter->pidx >= size) { 3309 if (iter->probe->list.next == func_probes) 3310 return NULL; 3311 next = iter->probe->list.next; 3312 iter->probe = list_entry(next, struct ftrace_func_probe, list); 3313 hash = iter->probe->ops.func_hash->filter_hash; 3314 size = 1 << hash->size_bits; 3315 iter->pidx = 0; 3316 } 3317 3318 hhd = &hash->buckets[iter->pidx]; 3319 3320 if (hlist_empty(hhd)) { 3321 iter->pidx++; 3322 hnd = NULL; 3323 goto retry; 3324 } 3325 3326 if (!hnd) 3327 hnd = hhd->first; 3328 else { 3329 hnd = hnd->next; 3330 if (!hnd) { 3331 iter->pidx++; 3332 goto retry; 3333 } 3334 } 3335 3336 if (WARN_ON_ONCE(!hnd)) 3337 return NULL; 3338 3339 iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist); 3340 3341 return iter; 3342 } 3343 3344 static void *t_probe_start(struct seq_file *m, loff_t *pos) 3345 { 3346 struct ftrace_iterator *iter = m->private; 3347 void *p = NULL; 3348 loff_t l; 3349 3350 if (!(iter->flags & FTRACE_ITER_DO_PROBES)) 3351 return NULL; 3352 3353 if (iter->mod_pos > *pos) 3354 return NULL; 3355 3356 iter->probe = NULL; 3357 iter->probe_entry = NULL; 3358 iter->pidx = 0; 3359 for (l = 0; l <= (*pos - iter->mod_pos); ) { 3360 p = t_probe_next(m, &l); 3361 if (!p) 3362 break; 3363 } 3364 if (!p) 3365 return NULL; 3366 3367 /* Only set this if we have an item */ 3368 iter->flags |= FTRACE_ITER_PROBE; 3369 3370 return iter; 3371 } 3372 3373 static int 3374 t_probe_show(struct seq_file *m, struct ftrace_iterator *iter) 3375 { 3376 struct ftrace_func_entry *probe_entry; 3377 struct ftrace_probe_ops *probe_ops; 3378 struct ftrace_func_probe *probe; 3379 3380 probe = iter->probe; 3381 probe_entry = iter->probe_entry; 3382 3383 if (WARN_ON_ONCE(!probe || !probe_entry)) 3384 return -EIO; 3385 3386 probe_ops = probe->probe_ops; 3387 3388 if (probe_ops->print) 3389 return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data); 3390 3391 seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip, 3392 (void *)probe_ops->func); 3393 3394 return 0; 3395 } 3396 3397 static void * 3398 t_mod_next(struct seq_file *m, loff_t *pos) 3399 { 3400 struct ftrace_iterator *iter = m->private; 3401 struct trace_array *tr = iter->tr; 3402 3403 (*pos)++; 3404 iter->pos = *pos; 3405 3406 iter->mod_list = iter->mod_list->next; 3407 3408 if (iter->mod_list == &tr->mod_trace || 3409 iter->mod_list == &tr->mod_notrace) { 3410 iter->flags &= ~FTRACE_ITER_MOD; 3411 return NULL; 3412 } 3413 3414 iter->mod_pos = *pos; 3415 3416 return iter; 3417 } 3418 3419 static void *t_mod_start(struct seq_file *m, loff_t *pos) 3420 { 3421 struct ftrace_iterator *iter = m->private; 3422 void *p = NULL; 3423 loff_t l; 3424 3425 if (iter->func_pos > *pos) 3426 return NULL; 3427 3428 iter->mod_pos = iter->func_pos; 3429 3430 /* probes are only available if tr is set */ 3431 if (!iter->tr) 3432 return NULL; 3433 3434 for (l = 0; l <= (*pos - iter->func_pos); ) { 3435 p = t_mod_next(m, &l); 3436 if (!p) 3437 break; 3438 } 3439 if (!p) { 3440 iter->flags &= ~FTRACE_ITER_MOD; 3441 return t_probe_start(m, pos); 3442 } 3443 3444 /* Only set this if we have an item */ 3445 iter->flags |= FTRACE_ITER_MOD; 3446 3447 return iter; 3448 } 3449 3450 static int 3451 t_mod_show(struct seq_file *m, struct ftrace_iterator *iter) 3452 { 3453 struct ftrace_mod_load *ftrace_mod; 3454 struct trace_array *tr = iter->tr; 3455 3456 if (WARN_ON_ONCE(!iter->mod_list) || 3457 iter->mod_list == &tr->mod_trace || 3458 iter->mod_list == &tr->mod_notrace) 3459 return -EIO; 3460 3461 ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list); 3462 3463 if (ftrace_mod->func) 3464 seq_printf(m, "%s", ftrace_mod->func); 3465 else 3466 seq_putc(m, '*'); 3467 3468 seq_printf(m, ":mod:%s\n", ftrace_mod->module); 3469 3470 return 0; 3471 } 3472 3473 static void * 3474 t_func_next(struct seq_file *m, loff_t *pos) 3475 { 3476 struct ftrace_iterator *iter = m->private; 3477 struct dyn_ftrace *rec = NULL; 3478 3479 (*pos)++; 3480 3481 retry: 3482 if (iter->idx >= iter->pg->index) { 3483 if (iter->pg->next) { 3484 iter->pg = iter->pg->next; 3485 iter->idx = 0; 3486 goto retry; 3487 } 3488 } else { 3489 rec = &iter->pg->records[iter->idx++]; 3490 if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) && 3491 !ftrace_lookup_ip(iter->hash, rec->ip)) || 3492 3493 ((iter->flags & FTRACE_ITER_ENABLED) && 3494 !(rec->flags & FTRACE_FL_ENABLED))) { 3495 3496 rec = NULL; 3497 goto retry; 3498 } 3499 } 3500 3501 if (!rec) 3502 return NULL; 3503 3504 iter->pos = iter->func_pos = *pos; 3505 iter->func = rec; 3506 3507 return iter; 3508 } 3509 3510 static void * 3511 t_next(struct seq_file *m, void *v, loff_t *pos) 3512 { 3513 struct ftrace_iterator *iter = m->private; 3514 loff_t l = *pos; /* t_probe_start() must use original pos */ 3515 void *ret; 3516 3517 if (unlikely(ftrace_disabled)) 3518 return NULL; 3519 3520 if (iter->flags & FTRACE_ITER_PROBE) 3521 return t_probe_next(m, pos); 3522 3523 if (iter->flags & FTRACE_ITER_MOD) 3524 return t_mod_next(m, pos); 3525 3526 if (iter->flags & FTRACE_ITER_PRINTALL) { 3527 /* next must increment pos, and t_probe_start does not */ 3528 (*pos)++; 3529 return t_mod_start(m, &l); 3530 } 3531 3532 ret = t_func_next(m, pos); 3533 3534 if (!ret) 3535 return t_mod_start(m, &l); 3536 3537 return ret; 3538 } 3539 3540 static void reset_iter_read(struct ftrace_iterator *iter) 3541 { 3542 iter->pos = 0; 3543 iter->func_pos = 0; 3544 iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD); 3545 } 3546 3547 static void *t_start(struct seq_file *m, loff_t *pos) 3548 { 3549 struct ftrace_iterator *iter = m->private; 3550 void *p = NULL; 3551 loff_t l; 3552 3553 mutex_lock(&ftrace_lock); 3554 3555 if (unlikely(ftrace_disabled)) 3556 return NULL; 3557 3558 /* 3559 * If an lseek was done, then reset and start from beginning. 3560 */ 3561 if (*pos < iter->pos) 3562 reset_iter_read(iter); 3563 3564 /* 3565 * For set_ftrace_filter reading, if we have the filter 3566 * off, we can short cut and just print out that all 3567 * functions are enabled. 3568 */ 3569 if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) && 3570 ftrace_hash_empty(iter->hash)) { 3571 iter->func_pos = 1; /* Account for the message */ 3572 if (*pos > 0) 3573 return t_mod_start(m, pos); 3574 iter->flags |= FTRACE_ITER_PRINTALL; 3575 /* reset in case of seek/pread */ 3576 iter->flags &= ~FTRACE_ITER_PROBE; 3577 return iter; 3578 } 3579 3580 if (iter->flags & FTRACE_ITER_MOD) 3581 return t_mod_start(m, pos); 3582 3583 /* 3584 * Unfortunately, we need to restart at ftrace_pages_start 3585 * every time we let go of the ftrace_mutex. This is because 3586 * those pointers can change without the lock. 3587 */ 3588 iter->pg = ftrace_pages_start; 3589 iter->idx = 0; 3590 for (l = 0; l <= *pos; ) { 3591 p = t_func_next(m, &l); 3592 if (!p) 3593 break; 3594 } 3595 3596 if (!p) 3597 return t_mod_start(m, pos); 3598 3599 return iter; 3600 } 3601 3602 static void t_stop(struct seq_file *m, void *p) 3603 { 3604 mutex_unlock(&ftrace_lock); 3605 } 3606 3607 void * __weak 3608 arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec) 3609 { 3610 return NULL; 3611 } 3612 3613 static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops, 3614 struct dyn_ftrace *rec) 3615 { 3616 void *ptr; 3617 3618 ptr = arch_ftrace_trampoline_func(ops, rec); 3619 if (ptr) 3620 seq_printf(m, " ->%pS", ptr); 3621 } 3622 3623 static int t_show(struct seq_file *m, void *v) 3624 { 3625 struct ftrace_iterator *iter = m->private; 3626 struct dyn_ftrace *rec; 3627 3628 if (iter->flags & FTRACE_ITER_PROBE) 3629 return t_probe_show(m, iter); 3630 3631 if (iter->flags & FTRACE_ITER_MOD) 3632 return t_mod_show(m, iter); 3633 3634 if (iter->flags & FTRACE_ITER_PRINTALL) { 3635 if (iter->flags & FTRACE_ITER_NOTRACE) 3636 seq_puts(m, "#### no functions disabled ####\n"); 3637 else 3638 seq_puts(m, "#### all functions enabled ####\n"); 3639 return 0; 3640 } 3641 3642 rec = iter->func; 3643 3644 if (!rec) 3645 return 0; 3646 3647 seq_printf(m, "%ps", (void *)rec->ip); 3648 if (iter->flags & FTRACE_ITER_ENABLED) { 3649 struct ftrace_ops *ops; 3650 3651 seq_printf(m, " (%ld)%s%s%s", 3652 ftrace_rec_count(rec), 3653 rec->flags & FTRACE_FL_REGS ? " R" : " ", 3654 rec->flags & FTRACE_FL_IPMODIFY ? " I" : " ", 3655 rec->flags & FTRACE_FL_DIRECT ? " D" : " "); 3656 if (rec->flags & FTRACE_FL_TRAMP_EN) { 3657 ops = ftrace_find_tramp_ops_any(rec); 3658 if (ops) { 3659 do { 3660 seq_printf(m, "\ttramp: %pS (%pS)", 3661 (void *)ops->trampoline, 3662 (void *)ops->func); 3663 add_trampoline_func(m, ops, rec); 3664 ops = ftrace_find_tramp_ops_next(rec, ops); 3665 } while (ops); 3666 } else 3667 seq_puts(m, "\ttramp: ERROR!"); 3668 } else { 3669 add_trampoline_func(m, NULL, rec); 3670 } 3671 if (rec->flags & FTRACE_FL_DIRECT) { 3672 unsigned long direct; 3673 3674 direct = ftrace_find_rec_direct(rec->ip); 3675 if (direct) 3676 seq_printf(m, "\n\tdirect-->%pS", (void *)direct); 3677 } 3678 } 3679 3680 seq_putc(m, '\n'); 3681 3682 return 0; 3683 } 3684 3685 static const struct seq_operations show_ftrace_seq_ops = { 3686 .start = t_start, 3687 .next = t_next, 3688 .stop = t_stop, 3689 .show = t_show, 3690 }; 3691 3692 static int 3693 ftrace_avail_open(struct inode *inode, struct file *file) 3694 { 3695 struct ftrace_iterator *iter; 3696 int ret; 3697 3698 ret = security_locked_down(LOCKDOWN_TRACEFS); 3699 if (ret) 3700 return ret; 3701 3702 if (unlikely(ftrace_disabled)) 3703 return -ENODEV; 3704 3705 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); 3706 if (!iter) 3707 return -ENOMEM; 3708 3709 iter->pg = ftrace_pages_start; 3710 iter->ops = &global_ops; 3711 3712 return 0; 3713 } 3714 3715 static int 3716 ftrace_enabled_open(struct inode *inode, struct file *file) 3717 { 3718 struct ftrace_iterator *iter; 3719 3720 /* 3721 * This shows us what functions are currently being 3722 * traced and by what. Not sure if we want lockdown 3723 * to hide such critical information for an admin. 3724 * Although, perhaps it can show information we don't 3725 * want people to see, but if something is tracing 3726 * something, we probably want to know about it. 3727 */ 3728 3729 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); 3730 if (!iter) 3731 return -ENOMEM; 3732 3733 iter->pg = ftrace_pages_start; 3734 iter->flags = FTRACE_ITER_ENABLED; 3735 iter->ops = &global_ops; 3736 3737 return 0; 3738 } 3739 3740 /** 3741 * ftrace_regex_open - initialize function tracer filter files 3742 * @ops: The ftrace_ops that hold the hash filters 3743 * @flag: The type of filter to process 3744 * @inode: The inode, usually passed in to your open routine 3745 * @file: The file, usually passed in to your open routine 3746 * 3747 * ftrace_regex_open() initializes the filter files for the 3748 * @ops. Depending on @flag it may process the filter hash or 3749 * the notrace hash of @ops. With this called from the open 3750 * routine, you can use ftrace_filter_write() for the write 3751 * routine if @flag has FTRACE_ITER_FILTER set, or 3752 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set. 3753 * tracing_lseek() should be used as the lseek routine, and 3754 * release must call ftrace_regex_release(). 3755 */ 3756 int 3757 ftrace_regex_open(struct ftrace_ops *ops, int flag, 3758 struct inode *inode, struct file *file) 3759 { 3760 struct ftrace_iterator *iter; 3761 struct ftrace_hash *hash; 3762 struct list_head *mod_head; 3763 struct trace_array *tr = ops->private; 3764 int ret = -ENOMEM; 3765 3766 ftrace_ops_init(ops); 3767 3768 if (unlikely(ftrace_disabled)) 3769 return -ENODEV; 3770 3771 if (tracing_check_open_get_tr(tr)) 3772 return -ENODEV; 3773 3774 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 3775 if (!iter) 3776 goto out; 3777 3778 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) 3779 goto out; 3780 3781 iter->ops = ops; 3782 iter->flags = flag; 3783 iter->tr = tr; 3784 3785 mutex_lock(&ops->func_hash->regex_lock); 3786 3787 if (flag & FTRACE_ITER_NOTRACE) { 3788 hash = ops->func_hash->notrace_hash; 3789 mod_head = tr ? &tr->mod_notrace : NULL; 3790 } else { 3791 hash = ops->func_hash->filter_hash; 3792 mod_head = tr ? &tr->mod_trace : NULL; 3793 } 3794 3795 iter->mod_list = mod_head; 3796 3797 if (file->f_mode & FMODE_WRITE) { 3798 const int size_bits = FTRACE_HASH_DEFAULT_BITS; 3799 3800 if (file->f_flags & O_TRUNC) { 3801 iter->hash = alloc_ftrace_hash(size_bits); 3802 clear_ftrace_mod_list(mod_head); 3803 } else { 3804 iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash); 3805 } 3806 3807 if (!iter->hash) { 3808 trace_parser_put(&iter->parser); 3809 goto out_unlock; 3810 } 3811 } else 3812 iter->hash = hash; 3813 3814 ret = 0; 3815 3816 if (file->f_mode & FMODE_READ) { 3817 iter->pg = ftrace_pages_start; 3818 3819 ret = seq_open(file, &show_ftrace_seq_ops); 3820 if (!ret) { 3821 struct seq_file *m = file->private_data; 3822 m->private = iter; 3823 } else { 3824 /* Failed */ 3825 free_ftrace_hash(iter->hash); 3826 trace_parser_put(&iter->parser); 3827 } 3828 } else 3829 file->private_data = iter; 3830 3831 out_unlock: 3832 mutex_unlock(&ops->func_hash->regex_lock); 3833 3834 out: 3835 if (ret) { 3836 kfree(iter); 3837 if (tr) 3838 trace_array_put(tr); 3839 } 3840 3841 return ret; 3842 } 3843 3844 static int 3845 ftrace_filter_open(struct inode *inode, struct file *file) 3846 { 3847 struct ftrace_ops *ops = inode->i_private; 3848 3849 /* Checks for tracefs lockdown */ 3850 return ftrace_regex_open(ops, 3851 FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES, 3852 inode, file); 3853 } 3854 3855 static int 3856 ftrace_notrace_open(struct inode *inode, struct file *file) 3857 { 3858 struct ftrace_ops *ops = inode->i_private; 3859 3860 /* Checks for tracefs lockdown */ 3861 return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE, 3862 inode, file); 3863 } 3864 3865 /* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */ 3866 struct ftrace_glob { 3867 char *search; 3868 unsigned len; 3869 int type; 3870 }; 3871 3872 /* 3873 * If symbols in an architecture don't correspond exactly to the user-visible 3874 * name of what they represent, it is possible to define this function to 3875 * perform the necessary adjustments. 3876 */ 3877 char * __weak arch_ftrace_match_adjust(char *str, const char *search) 3878 { 3879 return str; 3880 } 3881 3882 static int ftrace_match(char *str, struct ftrace_glob *g) 3883 { 3884 int matched = 0; 3885 int slen; 3886 3887 str = arch_ftrace_match_adjust(str, g->search); 3888 3889 switch (g->type) { 3890 case MATCH_FULL: 3891 if (strcmp(str, g->search) == 0) 3892 matched = 1; 3893 break; 3894 case MATCH_FRONT_ONLY: 3895 if (strncmp(str, g->search, g->len) == 0) 3896 matched = 1; 3897 break; 3898 case MATCH_MIDDLE_ONLY: 3899 if (strstr(str, g->search)) 3900 matched = 1; 3901 break; 3902 case MATCH_END_ONLY: 3903 slen = strlen(str); 3904 if (slen >= g->len && 3905 memcmp(str + slen - g->len, g->search, g->len) == 0) 3906 matched = 1; 3907 break; 3908 case MATCH_GLOB: 3909 if (glob_match(g->search, str)) 3910 matched = 1; 3911 break; 3912 } 3913 3914 return matched; 3915 } 3916 3917 static int 3918 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter) 3919 { 3920 struct ftrace_func_entry *entry; 3921 int ret = 0; 3922 3923 entry = ftrace_lookup_ip(hash, rec->ip); 3924 if (clear_filter) { 3925 /* Do nothing if it doesn't exist */ 3926 if (!entry) 3927 return 0; 3928 3929 free_hash_entry(hash, entry); 3930 } else { 3931 /* Do nothing if it exists */ 3932 if (entry) 3933 return 0; 3934 3935 ret = add_hash_entry(hash, rec->ip); 3936 } 3937 return ret; 3938 } 3939 3940 static int 3941 add_rec_by_index(struct ftrace_hash *hash, struct ftrace_glob *func_g, 3942 int clear_filter) 3943 { 3944 long index = simple_strtoul(func_g->search, NULL, 0); 3945 struct ftrace_page *pg; 3946 struct dyn_ftrace *rec; 3947 3948 /* The index starts at 1 */ 3949 if (--index < 0) 3950 return 0; 3951 3952 do_for_each_ftrace_rec(pg, rec) { 3953 if (pg->index <= index) { 3954 index -= pg->index; 3955 /* this is a double loop, break goes to the next page */ 3956 break; 3957 } 3958 rec = &pg->records[index]; 3959 enter_record(hash, rec, clear_filter); 3960 return 1; 3961 } while_for_each_ftrace_rec(); 3962 return 0; 3963 } 3964 3965 static int 3966 ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g, 3967 struct ftrace_glob *mod_g, int exclude_mod) 3968 { 3969 char str[KSYM_SYMBOL_LEN]; 3970 char *modname; 3971 3972 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); 3973 3974 if (mod_g) { 3975 int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0; 3976 3977 /* blank module name to match all modules */ 3978 if (!mod_g->len) { 3979 /* blank module globbing: modname xor exclude_mod */ 3980 if (!exclude_mod != !modname) 3981 goto func_match; 3982 return 0; 3983 } 3984 3985 /* 3986 * exclude_mod is set to trace everything but the given 3987 * module. If it is set and the module matches, then 3988 * return 0. If it is not set, and the module doesn't match 3989 * also return 0. Otherwise, check the function to see if 3990 * that matches. 3991 */ 3992 if (!mod_matches == !exclude_mod) 3993 return 0; 3994 func_match: 3995 /* blank search means to match all funcs in the mod */ 3996 if (!func_g->len) 3997 return 1; 3998 } 3999 4000 return ftrace_match(str, func_g); 4001 } 4002 4003 static int 4004 match_records(struct ftrace_hash *hash, char *func, int len, char *mod) 4005 { 4006 struct ftrace_page *pg; 4007 struct dyn_ftrace *rec; 4008 struct ftrace_glob func_g = { .type = MATCH_FULL }; 4009 struct ftrace_glob mod_g = { .type = MATCH_FULL }; 4010 struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL; 4011 int exclude_mod = 0; 4012 int found = 0; 4013 int ret; 4014 int clear_filter = 0; 4015 4016 if (func) { 4017 func_g.type = filter_parse_regex(func, len, &func_g.search, 4018 &clear_filter); 4019 func_g.len = strlen(func_g.search); 4020 } 4021 4022 if (mod) { 4023 mod_g.type = filter_parse_regex(mod, strlen(mod), 4024 &mod_g.search, &exclude_mod); 4025 mod_g.len = strlen(mod_g.search); 4026 } 4027 4028 mutex_lock(&ftrace_lock); 4029 4030 if (unlikely(ftrace_disabled)) 4031 goto out_unlock; 4032 4033 if (func_g.type == MATCH_INDEX) { 4034 found = add_rec_by_index(hash, &func_g, clear_filter); 4035 goto out_unlock; 4036 } 4037 4038 do_for_each_ftrace_rec(pg, rec) { 4039 4040 if (rec->flags & FTRACE_FL_DISABLED) 4041 continue; 4042 4043 if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) { 4044 ret = enter_record(hash, rec, clear_filter); 4045 if (ret < 0) { 4046 found = ret; 4047 goto out_unlock; 4048 } 4049 found = 1; 4050 } 4051 } while_for_each_ftrace_rec(); 4052 out_unlock: 4053 mutex_unlock(&ftrace_lock); 4054 4055 return found; 4056 } 4057 4058 static int 4059 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len) 4060 { 4061 return match_records(hash, buff, len, NULL); 4062 } 4063 4064 static void ftrace_ops_update_code(struct ftrace_ops *ops, 4065 struct ftrace_ops_hash *old_hash) 4066 { 4067 struct ftrace_ops *op; 4068 4069 if (!ftrace_enabled) 4070 return; 4071 4072 if (ops->flags & FTRACE_OPS_FL_ENABLED) { 4073 ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash); 4074 return; 4075 } 4076 4077 /* 4078 * If this is the shared global_ops filter, then we need to 4079 * check if there is another ops that shares it, is enabled. 4080 * If so, we still need to run the modify code. 4081 */ 4082 if (ops->func_hash != &global_ops.local_hash) 4083 return; 4084 4085 do_for_each_ftrace_op(op, ftrace_ops_list) { 4086 if (op->func_hash == &global_ops.local_hash && 4087 op->flags & FTRACE_OPS_FL_ENABLED) { 4088 ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash); 4089 /* Only need to do this once */ 4090 return; 4091 } 4092 } while_for_each_ftrace_op(op); 4093 } 4094 4095 static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops, 4096 struct ftrace_hash **orig_hash, 4097 struct ftrace_hash *hash, 4098 int enable) 4099 { 4100 struct ftrace_ops_hash old_hash_ops; 4101 struct ftrace_hash *old_hash; 4102 int ret; 4103 4104 old_hash = *orig_hash; 4105 old_hash_ops.filter_hash = ops->func_hash->filter_hash; 4106 old_hash_ops.notrace_hash = ops->func_hash->notrace_hash; 4107 ret = ftrace_hash_move(ops, enable, orig_hash, hash); 4108 if (!ret) { 4109 ftrace_ops_update_code(ops, &old_hash_ops); 4110 free_ftrace_hash_rcu(old_hash); 4111 } 4112 return ret; 4113 } 4114 4115 static bool module_exists(const char *module) 4116 { 4117 /* All modules have the symbol __this_module */ 4118 static const char this_mod[] = "__this_module"; 4119 char modname[MAX_PARAM_PREFIX_LEN + sizeof(this_mod) + 2]; 4120 unsigned long val; 4121 int n; 4122 4123 n = snprintf(modname, sizeof(modname), "%s:%s", module, this_mod); 4124 4125 if (n > sizeof(modname) - 1) 4126 return false; 4127 4128 val = module_kallsyms_lookup_name(modname); 4129 return val != 0; 4130 } 4131 4132 static int cache_mod(struct trace_array *tr, 4133 const char *func, char *module, int enable) 4134 { 4135 struct ftrace_mod_load *ftrace_mod, *n; 4136 struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace; 4137 int ret; 4138 4139 mutex_lock(&ftrace_lock); 4140 4141 /* We do not cache inverse filters */ 4142 if (func[0] == '!') { 4143 func++; 4144 ret = -EINVAL; 4145 4146 /* Look to remove this hash */ 4147 list_for_each_entry_safe(ftrace_mod, n, head, list) { 4148 if (strcmp(ftrace_mod->module, module) != 0) 4149 continue; 4150 4151 /* no func matches all */ 4152 if (strcmp(func, "*") == 0 || 4153 (ftrace_mod->func && 4154 strcmp(ftrace_mod->func, func) == 0)) { 4155 ret = 0; 4156 free_ftrace_mod(ftrace_mod); 4157 continue; 4158 } 4159 } 4160 goto out; 4161 } 4162 4163 ret = -EINVAL; 4164 /* We only care about modules that have not been loaded yet */ 4165 if (module_exists(module)) 4166 goto out; 4167 4168 /* Save this string off, and execute it when the module is loaded */ 4169 ret = ftrace_add_mod(tr, func, module, enable); 4170 out: 4171 mutex_unlock(&ftrace_lock); 4172 4173 return ret; 4174 } 4175 4176 static int 4177 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len, 4178 int reset, int enable); 4179 4180 #ifdef CONFIG_MODULES 4181 static void process_mod_list(struct list_head *head, struct ftrace_ops *ops, 4182 char *mod, bool enable) 4183 { 4184 struct ftrace_mod_load *ftrace_mod, *n; 4185 struct ftrace_hash **orig_hash, *new_hash; 4186 LIST_HEAD(process_mods); 4187 char *func; 4188 4189 mutex_lock(&ops->func_hash->regex_lock); 4190 4191 if (enable) 4192 orig_hash = &ops->func_hash->filter_hash; 4193 else 4194 orig_hash = &ops->func_hash->notrace_hash; 4195 4196 new_hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, 4197 *orig_hash); 4198 if (!new_hash) 4199 goto out; /* warn? */ 4200 4201 mutex_lock(&ftrace_lock); 4202 4203 list_for_each_entry_safe(ftrace_mod, n, head, list) { 4204 4205 if (strcmp(ftrace_mod->module, mod) != 0) 4206 continue; 4207 4208 if (ftrace_mod->func) 4209 func = kstrdup(ftrace_mod->func, GFP_KERNEL); 4210 else 4211 func = kstrdup("*", GFP_KERNEL); 4212 4213 if (!func) /* warn? */ 4214 continue; 4215 4216 list_del(&ftrace_mod->list); 4217 list_add(&ftrace_mod->list, &process_mods); 4218 4219 /* Use the newly allocated func, as it may be "*" */ 4220 kfree(ftrace_mod->func); 4221 ftrace_mod->func = func; 4222 } 4223 4224 mutex_unlock(&ftrace_lock); 4225 4226 list_for_each_entry_safe(ftrace_mod, n, &process_mods, list) { 4227 4228 func = ftrace_mod->func; 4229 4230 /* Grabs ftrace_lock, which is why we have this extra step */ 4231 match_records(new_hash, func, strlen(func), mod); 4232 free_ftrace_mod(ftrace_mod); 4233 } 4234 4235 if (enable && list_empty(head)) 4236 new_hash->flags &= ~FTRACE_HASH_FL_MOD; 4237 4238 mutex_lock(&ftrace_lock); 4239 4240 ftrace_hash_move_and_update_ops(ops, orig_hash, 4241 new_hash, enable); 4242 mutex_unlock(&ftrace_lock); 4243 4244 out: 4245 mutex_unlock(&ops->func_hash->regex_lock); 4246 4247 free_ftrace_hash(new_hash); 4248 } 4249 4250 static void process_cached_mods(const char *mod_name) 4251 { 4252 struct trace_array *tr; 4253 char *mod; 4254 4255 mod = kstrdup(mod_name, GFP_KERNEL); 4256 if (!mod) 4257 return; 4258 4259 mutex_lock(&trace_types_lock); 4260 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 4261 if (!list_empty(&tr->mod_trace)) 4262 process_mod_list(&tr->mod_trace, tr->ops, mod, true); 4263 if (!list_empty(&tr->mod_notrace)) 4264 process_mod_list(&tr->mod_notrace, tr->ops, mod, false); 4265 } 4266 mutex_unlock(&trace_types_lock); 4267 4268 kfree(mod); 4269 } 4270 #endif 4271 4272 /* 4273 * We register the module command as a template to show others how 4274 * to register the a command as well. 4275 */ 4276 4277 static int 4278 ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash, 4279 char *func_orig, char *cmd, char *module, int enable) 4280 { 4281 char *func; 4282 int ret; 4283 4284 /* match_records() modifies func, and we need the original */ 4285 func = kstrdup(func_orig, GFP_KERNEL); 4286 if (!func) 4287 return -ENOMEM; 4288 4289 /* 4290 * cmd == 'mod' because we only registered this func 4291 * for the 'mod' ftrace_func_command. 4292 * But if you register one func with multiple commands, 4293 * you can tell which command was used by the cmd 4294 * parameter. 4295 */ 4296 ret = match_records(hash, func, strlen(func), module); 4297 kfree(func); 4298 4299 if (!ret) 4300 return cache_mod(tr, func_orig, module, enable); 4301 if (ret < 0) 4302 return ret; 4303 return 0; 4304 } 4305 4306 static struct ftrace_func_command ftrace_mod_cmd = { 4307 .name = "mod", 4308 .func = ftrace_mod_callback, 4309 }; 4310 4311 static int __init ftrace_mod_cmd_init(void) 4312 { 4313 return register_ftrace_command(&ftrace_mod_cmd); 4314 } 4315 core_initcall(ftrace_mod_cmd_init); 4316 4317 static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip, 4318 struct ftrace_ops *op, struct ftrace_regs *fregs) 4319 { 4320 struct ftrace_probe_ops *probe_ops; 4321 struct ftrace_func_probe *probe; 4322 4323 probe = container_of(op, struct ftrace_func_probe, ops); 4324 probe_ops = probe->probe_ops; 4325 4326 /* 4327 * Disable preemption for these calls to prevent a RCU grace 4328 * period. This syncs the hash iteration and freeing of items 4329 * on the hash. rcu_read_lock is too dangerous here. 4330 */ 4331 preempt_disable_notrace(); 4332 probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data); 4333 preempt_enable_notrace(); 4334 } 4335 4336 struct ftrace_func_map { 4337 struct ftrace_func_entry entry; 4338 void *data; 4339 }; 4340 4341 struct ftrace_func_mapper { 4342 struct ftrace_hash hash; 4343 }; 4344 4345 /** 4346 * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper 4347 * 4348 * Returns a ftrace_func_mapper descriptor that can be used to map ips to data. 4349 */ 4350 struct ftrace_func_mapper *allocate_ftrace_func_mapper(void) 4351 { 4352 struct ftrace_hash *hash; 4353 4354 /* 4355 * The mapper is simply a ftrace_hash, but since the entries 4356 * in the hash are not ftrace_func_entry type, we define it 4357 * as a separate structure. 4358 */ 4359 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); 4360 return (struct ftrace_func_mapper *)hash; 4361 } 4362 4363 /** 4364 * ftrace_func_mapper_find_ip - Find some data mapped to an ip 4365 * @mapper: The mapper that has the ip maps 4366 * @ip: the instruction pointer to find the data for 4367 * 4368 * Returns the data mapped to @ip if found otherwise NULL. The return 4369 * is actually the address of the mapper data pointer. The address is 4370 * returned for use cases where the data is no bigger than a long, and 4371 * the user can use the data pointer as its data instead of having to 4372 * allocate more memory for the reference. 4373 */ 4374 void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper, 4375 unsigned long ip) 4376 { 4377 struct ftrace_func_entry *entry; 4378 struct ftrace_func_map *map; 4379 4380 entry = ftrace_lookup_ip(&mapper->hash, ip); 4381 if (!entry) 4382 return NULL; 4383 4384 map = (struct ftrace_func_map *)entry; 4385 return &map->data; 4386 } 4387 4388 /** 4389 * ftrace_func_mapper_add_ip - Map some data to an ip 4390 * @mapper: The mapper that has the ip maps 4391 * @ip: The instruction pointer address to map @data to 4392 * @data: The data to map to @ip 4393 * 4394 * Returns 0 on success otherwise an error. 4395 */ 4396 int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper, 4397 unsigned long ip, void *data) 4398 { 4399 struct ftrace_func_entry *entry; 4400 struct ftrace_func_map *map; 4401 4402 entry = ftrace_lookup_ip(&mapper->hash, ip); 4403 if (entry) 4404 return -EBUSY; 4405 4406 map = kmalloc(sizeof(*map), GFP_KERNEL); 4407 if (!map) 4408 return -ENOMEM; 4409 4410 map->entry.ip = ip; 4411 map->data = data; 4412 4413 __add_hash_entry(&mapper->hash, &map->entry); 4414 4415 return 0; 4416 } 4417 4418 /** 4419 * ftrace_func_mapper_remove_ip - Remove an ip from the mapping 4420 * @mapper: The mapper that has the ip maps 4421 * @ip: The instruction pointer address to remove the data from 4422 * 4423 * Returns the data if it is found, otherwise NULL. 4424 * Note, if the data pointer is used as the data itself, (see 4425 * ftrace_func_mapper_find_ip(), then the return value may be meaningless, 4426 * if the data pointer was set to zero. 4427 */ 4428 void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper, 4429 unsigned long ip) 4430 { 4431 struct ftrace_func_entry *entry; 4432 struct ftrace_func_map *map; 4433 void *data; 4434 4435 entry = ftrace_lookup_ip(&mapper->hash, ip); 4436 if (!entry) 4437 return NULL; 4438 4439 map = (struct ftrace_func_map *)entry; 4440 data = map->data; 4441 4442 remove_hash_entry(&mapper->hash, entry); 4443 kfree(entry); 4444 4445 return data; 4446 } 4447 4448 /** 4449 * free_ftrace_func_mapper - free a mapping of ips and data 4450 * @mapper: The mapper that has the ip maps 4451 * @free_func: A function to be called on each data item. 4452 * 4453 * This is used to free the function mapper. The @free_func is optional 4454 * and can be used if the data needs to be freed as well. 4455 */ 4456 void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper, 4457 ftrace_mapper_func free_func) 4458 { 4459 struct ftrace_func_entry *entry; 4460 struct ftrace_func_map *map; 4461 struct hlist_head *hhd; 4462 int size, i; 4463 4464 if (!mapper) 4465 return; 4466 4467 if (free_func && mapper->hash.count) { 4468 size = 1 << mapper->hash.size_bits; 4469 for (i = 0; i < size; i++) { 4470 hhd = &mapper->hash.buckets[i]; 4471 hlist_for_each_entry(entry, hhd, hlist) { 4472 map = (struct ftrace_func_map *)entry; 4473 free_func(map); 4474 } 4475 } 4476 } 4477 free_ftrace_hash(&mapper->hash); 4478 } 4479 4480 static void release_probe(struct ftrace_func_probe *probe) 4481 { 4482 struct ftrace_probe_ops *probe_ops; 4483 4484 mutex_lock(&ftrace_lock); 4485 4486 WARN_ON(probe->ref <= 0); 4487 4488 /* Subtract the ref that was used to protect this instance */ 4489 probe->ref--; 4490 4491 if (!probe->ref) { 4492 probe_ops = probe->probe_ops; 4493 /* 4494 * Sending zero as ip tells probe_ops to free 4495 * the probe->data itself 4496 */ 4497 if (probe_ops->free) 4498 probe_ops->free(probe_ops, probe->tr, 0, probe->data); 4499 list_del(&probe->list); 4500 kfree(probe); 4501 } 4502 mutex_unlock(&ftrace_lock); 4503 } 4504 4505 static void acquire_probe_locked(struct ftrace_func_probe *probe) 4506 { 4507 /* 4508 * Add one ref to keep it from being freed when releasing the 4509 * ftrace_lock mutex. 4510 */ 4511 probe->ref++; 4512 } 4513 4514 int 4515 register_ftrace_function_probe(char *glob, struct trace_array *tr, 4516 struct ftrace_probe_ops *probe_ops, 4517 void *data) 4518 { 4519 struct ftrace_func_entry *entry; 4520 struct ftrace_func_probe *probe; 4521 struct ftrace_hash **orig_hash; 4522 struct ftrace_hash *old_hash; 4523 struct ftrace_hash *hash; 4524 int count = 0; 4525 int size; 4526 int ret; 4527 int i; 4528 4529 if (WARN_ON(!tr)) 4530 return -EINVAL; 4531 4532 /* We do not support '!' for function probes */ 4533 if (WARN_ON(glob[0] == '!')) 4534 return -EINVAL; 4535 4536 4537 mutex_lock(&ftrace_lock); 4538 /* Check if the probe_ops is already registered */ 4539 list_for_each_entry(probe, &tr->func_probes, list) { 4540 if (probe->probe_ops == probe_ops) 4541 break; 4542 } 4543 if (&probe->list == &tr->func_probes) { 4544 probe = kzalloc(sizeof(*probe), GFP_KERNEL); 4545 if (!probe) { 4546 mutex_unlock(&ftrace_lock); 4547 return -ENOMEM; 4548 } 4549 probe->probe_ops = probe_ops; 4550 probe->ops.func = function_trace_probe_call; 4551 probe->tr = tr; 4552 ftrace_ops_init(&probe->ops); 4553 list_add(&probe->list, &tr->func_probes); 4554 } 4555 4556 acquire_probe_locked(probe); 4557 4558 mutex_unlock(&ftrace_lock); 4559 4560 /* 4561 * Note, there's a small window here that the func_hash->filter_hash 4562 * may be NULL or empty. Need to be careful when reading the loop. 4563 */ 4564 mutex_lock(&probe->ops.func_hash->regex_lock); 4565 4566 orig_hash = &probe->ops.func_hash->filter_hash; 4567 old_hash = *orig_hash; 4568 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash); 4569 4570 if (!hash) { 4571 ret = -ENOMEM; 4572 goto out; 4573 } 4574 4575 ret = ftrace_match_records(hash, glob, strlen(glob)); 4576 4577 /* Nothing found? */ 4578 if (!ret) 4579 ret = -EINVAL; 4580 4581 if (ret < 0) 4582 goto out; 4583 4584 size = 1 << hash->size_bits; 4585 for (i = 0; i < size; i++) { 4586 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 4587 if (ftrace_lookup_ip(old_hash, entry->ip)) 4588 continue; 4589 /* 4590 * The caller might want to do something special 4591 * for each function we find. We call the callback 4592 * to give the caller an opportunity to do so. 4593 */ 4594 if (probe_ops->init) { 4595 ret = probe_ops->init(probe_ops, tr, 4596 entry->ip, data, 4597 &probe->data); 4598 if (ret < 0) { 4599 if (probe_ops->free && count) 4600 probe_ops->free(probe_ops, tr, 4601 0, probe->data); 4602 probe->data = NULL; 4603 goto out; 4604 } 4605 } 4606 count++; 4607 } 4608 } 4609 4610 mutex_lock(&ftrace_lock); 4611 4612 if (!count) { 4613 /* Nothing was added? */ 4614 ret = -EINVAL; 4615 goto out_unlock; 4616 } 4617 4618 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, 4619 hash, 1); 4620 if (ret < 0) 4621 goto err_unlock; 4622 4623 /* One ref for each new function traced */ 4624 probe->ref += count; 4625 4626 if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED)) 4627 ret = ftrace_startup(&probe->ops, 0); 4628 4629 out_unlock: 4630 mutex_unlock(&ftrace_lock); 4631 4632 if (!ret) 4633 ret = count; 4634 out: 4635 mutex_unlock(&probe->ops.func_hash->regex_lock); 4636 free_ftrace_hash(hash); 4637 4638 release_probe(probe); 4639 4640 return ret; 4641 4642 err_unlock: 4643 if (!probe_ops->free || !count) 4644 goto out_unlock; 4645 4646 /* Failed to do the move, need to call the free functions */ 4647 for (i = 0; i < size; i++) { 4648 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 4649 if (ftrace_lookup_ip(old_hash, entry->ip)) 4650 continue; 4651 probe_ops->free(probe_ops, tr, entry->ip, probe->data); 4652 } 4653 } 4654 goto out_unlock; 4655 } 4656 4657 int 4658 unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr, 4659 struct ftrace_probe_ops *probe_ops) 4660 { 4661 struct ftrace_ops_hash old_hash_ops; 4662 struct ftrace_func_entry *entry; 4663 struct ftrace_func_probe *probe; 4664 struct ftrace_glob func_g; 4665 struct ftrace_hash **orig_hash; 4666 struct ftrace_hash *old_hash; 4667 struct ftrace_hash *hash = NULL; 4668 struct hlist_node *tmp; 4669 struct hlist_head hhd; 4670 char str[KSYM_SYMBOL_LEN]; 4671 int count = 0; 4672 int i, ret = -ENODEV; 4673 int size; 4674 4675 if (!glob || !strlen(glob) || !strcmp(glob, "*")) 4676 func_g.search = NULL; 4677 else { 4678 int not; 4679 4680 func_g.type = filter_parse_regex(glob, strlen(glob), 4681 &func_g.search, ¬); 4682 func_g.len = strlen(func_g.search); 4683 4684 /* we do not support '!' for function probes */ 4685 if (WARN_ON(not)) 4686 return -EINVAL; 4687 } 4688 4689 mutex_lock(&ftrace_lock); 4690 /* Check if the probe_ops is already registered */ 4691 list_for_each_entry(probe, &tr->func_probes, list) { 4692 if (probe->probe_ops == probe_ops) 4693 break; 4694 } 4695 if (&probe->list == &tr->func_probes) 4696 goto err_unlock_ftrace; 4697 4698 ret = -EINVAL; 4699 if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED)) 4700 goto err_unlock_ftrace; 4701 4702 acquire_probe_locked(probe); 4703 4704 mutex_unlock(&ftrace_lock); 4705 4706 mutex_lock(&probe->ops.func_hash->regex_lock); 4707 4708 orig_hash = &probe->ops.func_hash->filter_hash; 4709 old_hash = *orig_hash; 4710 4711 if (ftrace_hash_empty(old_hash)) 4712 goto out_unlock; 4713 4714 old_hash_ops.filter_hash = old_hash; 4715 /* Probes only have filters */ 4716 old_hash_ops.notrace_hash = NULL; 4717 4718 ret = -ENOMEM; 4719 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash); 4720 if (!hash) 4721 goto out_unlock; 4722 4723 INIT_HLIST_HEAD(&hhd); 4724 4725 size = 1 << hash->size_bits; 4726 for (i = 0; i < size; i++) { 4727 hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) { 4728 4729 if (func_g.search) { 4730 kallsyms_lookup(entry->ip, NULL, NULL, 4731 NULL, str); 4732 if (!ftrace_match(str, &func_g)) 4733 continue; 4734 } 4735 count++; 4736 remove_hash_entry(hash, entry); 4737 hlist_add_head(&entry->hlist, &hhd); 4738 } 4739 } 4740 4741 /* Nothing found? */ 4742 if (!count) { 4743 ret = -EINVAL; 4744 goto out_unlock; 4745 } 4746 4747 mutex_lock(&ftrace_lock); 4748 4749 WARN_ON(probe->ref < count); 4750 4751 probe->ref -= count; 4752 4753 if (ftrace_hash_empty(hash)) 4754 ftrace_shutdown(&probe->ops, 0); 4755 4756 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, 4757 hash, 1); 4758 4759 /* still need to update the function call sites */ 4760 if (ftrace_enabled && !ftrace_hash_empty(hash)) 4761 ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS, 4762 &old_hash_ops); 4763 synchronize_rcu(); 4764 4765 hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) { 4766 hlist_del(&entry->hlist); 4767 if (probe_ops->free) 4768 probe_ops->free(probe_ops, tr, entry->ip, probe->data); 4769 kfree(entry); 4770 } 4771 mutex_unlock(&ftrace_lock); 4772 4773 out_unlock: 4774 mutex_unlock(&probe->ops.func_hash->regex_lock); 4775 free_ftrace_hash(hash); 4776 4777 release_probe(probe); 4778 4779 return ret; 4780 4781 err_unlock_ftrace: 4782 mutex_unlock(&ftrace_lock); 4783 return ret; 4784 } 4785 4786 void clear_ftrace_function_probes(struct trace_array *tr) 4787 { 4788 struct ftrace_func_probe *probe, *n; 4789 4790 list_for_each_entry_safe(probe, n, &tr->func_probes, list) 4791 unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops); 4792 } 4793 4794 static LIST_HEAD(ftrace_commands); 4795 static DEFINE_MUTEX(ftrace_cmd_mutex); 4796 4797 /* 4798 * Currently we only register ftrace commands from __init, so mark this 4799 * __init too. 4800 */ 4801 __init int register_ftrace_command(struct ftrace_func_command *cmd) 4802 { 4803 struct ftrace_func_command *p; 4804 int ret = 0; 4805 4806 mutex_lock(&ftrace_cmd_mutex); 4807 list_for_each_entry(p, &ftrace_commands, list) { 4808 if (strcmp(cmd->name, p->name) == 0) { 4809 ret = -EBUSY; 4810 goto out_unlock; 4811 } 4812 } 4813 list_add(&cmd->list, &ftrace_commands); 4814 out_unlock: 4815 mutex_unlock(&ftrace_cmd_mutex); 4816 4817 return ret; 4818 } 4819 4820 /* 4821 * Currently we only unregister ftrace commands from __init, so mark 4822 * this __init too. 4823 */ 4824 __init int unregister_ftrace_command(struct ftrace_func_command *cmd) 4825 { 4826 struct ftrace_func_command *p, *n; 4827 int ret = -ENODEV; 4828 4829 mutex_lock(&ftrace_cmd_mutex); 4830 list_for_each_entry_safe(p, n, &ftrace_commands, list) { 4831 if (strcmp(cmd->name, p->name) == 0) { 4832 ret = 0; 4833 list_del_init(&p->list); 4834 goto out_unlock; 4835 } 4836 } 4837 out_unlock: 4838 mutex_unlock(&ftrace_cmd_mutex); 4839 4840 return ret; 4841 } 4842 4843 static int ftrace_process_regex(struct ftrace_iterator *iter, 4844 char *buff, int len, int enable) 4845 { 4846 struct ftrace_hash *hash = iter->hash; 4847 struct trace_array *tr = iter->ops->private; 4848 char *func, *command, *next = buff; 4849 struct ftrace_func_command *p; 4850 int ret = -EINVAL; 4851 4852 func = strsep(&next, ":"); 4853 4854 if (!next) { 4855 ret = ftrace_match_records(hash, func, len); 4856 if (!ret) 4857 ret = -EINVAL; 4858 if (ret < 0) 4859 return ret; 4860 return 0; 4861 } 4862 4863 /* command found */ 4864 4865 command = strsep(&next, ":"); 4866 4867 mutex_lock(&ftrace_cmd_mutex); 4868 list_for_each_entry(p, &ftrace_commands, list) { 4869 if (strcmp(p->name, command) == 0) { 4870 ret = p->func(tr, hash, func, command, next, enable); 4871 goto out_unlock; 4872 } 4873 } 4874 out_unlock: 4875 mutex_unlock(&ftrace_cmd_mutex); 4876 4877 return ret; 4878 } 4879 4880 static ssize_t 4881 ftrace_regex_write(struct file *file, const char __user *ubuf, 4882 size_t cnt, loff_t *ppos, int enable) 4883 { 4884 struct ftrace_iterator *iter; 4885 struct trace_parser *parser; 4886 ssize_t ret, read; 4887 4888 if (!cnt) 4889 return 0; 4890 4891 if (file->f_mode & FMODE_READ) { 4892 struct seq_file *m = file->private_data; 4893 iter = m->private; 4894 } else 4895 iter = file->private_data; 4896 4897 if (unlikely(ftrace_disabled)) 4898 return -ENODEV; 4899 4900 /* iter->hash is a local copy, so we don't need regex_lock */ 4901 4902 parser = &iter->parser; 4903 read = trace_get_user(parser, ubuf, cnt, ppos); 4904 4905 if (read >= 0 && trace_parser_loaded(parser) && 4906 !trace_parser_cont(parser)) { 4907 ret = ftrace_process_regex(iter, parser->buffer, 4908 parser->idx, enable); 4909 trace_parser_clear(parser); 4910 if (ret < 0) 4911 goto out; 4912 } 4913 4914 ret = read; 4915 out: 4916 return ret; 4917 } 4918 4919 ssize_t 4920 ftrace_filter_write(struct file *file, const char __user *ubuf, 4921 size_t cnt, loff_t *ppos) 4922 { 4923 return ftrace_regex_write(file, ubuf, cnt, ppos, 1); 4924 } 4925 4926 ssize_t 4927 ftrace_notrace_write(struct file *file, const char __user *ubuf, 4928 size_t cnt, loff_t *ppos) 4929 { 4930 return ftrace_regex_write(file, ubuf, cnt, ppos, 0); 4931 } 4932 4933 static int 4934 ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove) 4935 { 4936 struct ftrace_func_entry *entry; 4937 4938 if (!ftrace_location(ip)) 4939 return -EINVAL; 4940 4941 if (remove) { 4942 entry = ftrace_lookup_ip(hash, ip); 4943 if (!entry) 4944 return -ENOENT; 4945 free_hash_entry(hash, entry); 4946 return 0; 4947 } 4948 4949 return add_hash_entry(hash, ip); 4950 } 4951 4952 static int 4953 ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len, 4954 unsigned long ip, int remove, int reset, int enable) 4955 { 4956 struct ftrace_hash **orig_hash; 4957 struct ftrace_hash *hash; 4958 int ret; 4959 4960 if (unlikely(ftrace_disabled)) 4961 return -ENODEV; 4962 4963 mutex_lock(&ops->func_hash->regex_lock); 4964 4965 if (enable) 4966 orig_hash = &ops->func_hash->filter_hash; 4967 else 4968 orig_hash = &ops->func_hash->notrace_hash; 4969 4970 if (reset) 4971 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); 4972 else 4973 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash); 4974 4975 if (!hash) { 4976 ret = -ENOMEM; 4977 goto out_regex_unlock; 4978 } 4979 4980 if (buf && !ftrace_match_records(hash, buf, len)) { 4981 ret = -EINVAL; 4982 goto out_regex_unlock; 4983 } 4984 if (ip) { 4985 ret = ftrace_match_addr(hash, ip, remove); 4986 if (ret < 0) 4987 goto out_regex_unlock; 4988 } 4989 4990 mutex_lock(&ftrace_lock); 4991 ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable); 4992 mutex_unlock(&ftrace_lock); 4993 4994 out_regex_unlock: 4995 mutex_unlock(&ops->func_hash->regex_lock); 4996 4997 free_ftrace_hash(hash); 4998 return ret; 4999 } 5000 5001 static int 5002 ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove, 5003 int reset, int enable) 5004 { 5005 return ftrace_set_hash(ops, NULL, 0, ip, remove, reset, enable); 5006 } 5007 5008 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS 5009 5010 struct ftrace_direct_func { 5011 struct list_head next; 5012 unsigned long addr; 5013 int count; 5014 }; 5015 5016 static LIST_HEAD(ftrace_direct_funcs); 5017 5018 /** 5019 * ftrace_find_direct_func - test an address if it is a registered direct caller 5020 * @addr: The address of a registered direct caller 5021 * 5022 * This searches to see if a ftrace direct caller has been registered 5023 * at a specific address, and if so, it returns a descriptor for it. 5024 * 5025 * This can be used by architecture code to see if an address is 5026 * a direct caller (trampoline) attached to a fentry/mcount location. 5027 * This is useful for the function_graph tracer, as it may need to 5028 * do adjustments if it traced a location that also has a direct 5029 * trampoline attached to it. 5030 */ 5031 struct ftrace_direct_func *ftrace_find_direct_func(unsigned long addr) 5032 { 5033 struct ftrace_direct_func *entry; 5034 bool found = false; 5035 5036 /* May be called by fgraph trampoline (protected by rcu tasks) */ 5037 list_for_each_entry_rcu(entry, &ftrace_direct_funcs, next) { 5038 if (entry->addr == addr) { 5039 found = true; 5040 break; 5041 } 5042 } 5043 if (found) 5044 return entry; 5045 5046 return NULL; 5047 } 5048 5049 static struct ftrace_direct_func *ftrace_alloc_direct_func(unsigned long addr) 5050 { 5051 struct ftrace_direct_func *direct; 5052 5053 direct = kmalloc(sizeof(*direct), GFP_KERNEL); 5054 if (!direct) 5055 return NULL; 5056 direct->addr = addr; 5057 direct->count = 0; 5058 list_add_rcu(&direct->next, &ftrace_direct_funcs); 5059 ftrace_direct_func_count++; 5060 return direct; 5061 } 5062 5063 /** 5064 * register_ftrace_direct - Call a custom trampoline directly 5065 * @ip: The address of the nop at the beginning of a function 5066 * @addr: The address of the trampoline to call at @ip 5067 * 5068 * This is used to connect a direct call from the nop location (@ip) 5069 * at the start of ftrace traced functions. The location that it calls 5070 * (@addr) must be able to handle a direct call, and save the parameters 5071 * of the function being traced, and restore them (or inject new ones 5072 * if needed), before returning. 5073 * 5074 * Returns: 5075 * 0 on success 5076 * -EBUSY - Another direct function is already attached (there can be only one) 5077 * -ENODEV - @ip does not point to a ftrace nop location (or not supported) 5078 * -ENOMEM - There was an allocation failure. 5079 */ 5080 int register_ftrace_direct(unsigned long ip, unsigned long addr) 5081 { 5082 struct ftrace_direct_func *direct; 5083 struct ftrace_func_entry *entry; 5084 struct ftrace_hash *free_hash = NULL; 5085 struct dyn_ftrace *rec; 5086 int ret = -EBUSY; 5087 5088 mutex_lock(&direct_mutex); 5089 5090 /* See if there's a direct function at @ip already */ 5091 if (ftrace_find_rec_direct(ip)) 5092 goto out_unlock; 5093 5094 ret = -ENODEV; 5095 rec = lookup_rec(ip, ip); 5096 if (!rec) 5097 goto out_unlock; 5098 5099 /* 5100 * Check if the rec says it has a direct call but we didn't 5101 * find one earlier? 5102 */ 5103 if (WARN_ON(rec->flags & FTRACE_FL_DIRECT)) 5104 goto out_unlock; 5105 5106 /* Make sure the ip points to the exact record */ 5107 if (ip != rec->ip) { 5108 ip = rec->ip; 5109 /* Need to check this ip for a direct. */ 5110 if (ftrace_find_rec_direct(ip)) 5111 goto out_unlock; 5112 } 5113 5114 ret = -ENOMEM; 5115 if (ftrace_hash_empty(direct_functions) || 5116 direct_functions->count > 2 * (1 << direct_functions->size_bits)) { 5117 struct ftrace_hash *new_hash; 5118 int size = ftrace_hash_empty(direct_functions) ? 0 : 5119 direct_functions->count + 1; 5120 5121 if (size < 32) 5122 size = 32; 5123 5124 new_hash = dup_hash(direct_functions, size); 5125 if (!new_hash) 5126 goto out_unlock; 5127 5128 free_hash = direct_functions; 5129 direct_functions = new_hash; 5130 } 5131 5132 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 5133 if (!entry) 5134 goto out_unlock; 5135 5136 direct = ftrace_find_direct_func(addr); 5137 if (!direct) { 5138 direct = ftrace_alloc_direct_func(addr); 5139 if (!direct) { 5140 kfree(entry); 5141 goto out_unlock; 5142 } 5143 } 5144 5145 entry->ip = ip; 5146 entry->direct = addr; 5147 __add_hash_entry(direct_functions, entry); 5148 5149 ret = ftrace_set_filter_ip(&direct_ops, ip, 0, 0); 5150 if (ret) 5151 remove_hash_entry(direct_functions, entry); 5152 5153 if (!ret && !(direct_ops.flags & FTRACE_OPS_FL_ENABLED)) { 5154 ret = register_ftrace_function(&direct_ops); 5155 if (ret) 5156 ftrace_set_filter_ip(&direct_ops, ip, 1, 0); 5157 } 5158 5159 if (ret) { 5160 kfree(entry); 5161 if (!direct->count) { 5162 list_del_rcu(&direct->next); 5163 synchronize_rcu_tasks(); 5164 kfree(direct); 5165 if (free_hash) 5166 free_ftrace_hash(free_hash); 5167 free_hash = NULL; 5168 ftrace_direct_func_count--; 5169 } 5170 } else { 5171 direct->count++; 5172 } 5173 out_unlock: 5174 mutex_unlock(&direct_mutex); 5175 5176 if (free_hash) { 5177 synchronize_rcu_tasks(); 5178 free_ftrace_hash(free_hash); 5179 } 5180 5181 return ret; 5182 } 5183 EXPORT_SYMBOL_GPL(register_ftrace_direct); 5184 5185 static struct ftrace_func_entry *find_direct_entry(unsigned long *ip, 5186 struct dyn_ftrace **recp) 5187 { 5188 struct ftrace_func_entry *entry; 5189 struct dyn_ftrace *rec; 5190 5191 rec = lookup_rec(*ip, *ip); 5192 if (!rec) 5193 return NULL; 5194 5195 entry = __ftrace_lookup_ip(direct_functions, rec->ip); 5196 if (!entry) { 5197 WARN_ON(rec->flags & FTRACE_FL_DIRECT); 5198 return NULL; 5199 } 5200 5201 WARN_ON(!(rec->flags & FTRACE_FL_DIRECT)); 5202 5203 /* Passed in ip just needs to be on the call site */ 5204 *ip = rec->ip; 5205 5206 if (recp) 5207 *recp = rec; 5208 5209 return entry; 5210 } 5211 5212 int unregister_ftrace_direct(unsigned long ip, unsigned long addr) 5213 { 5214 struct ftrace_direct_func *direct; 5215 struct ftrace_func_entry *entry; 5216 int ret = -ENODEV; 5217 5218 mutex_lock(&direct_mutex); 5219 5220 entry = find_direct_entry(&ip, NULL); 5221 if (!entry) 5222 goto out_unlock; 5223 5224 if (direct_functions->count == 1) 5225 unregister_ftrace_function(&direct_ops); 5226 5227 ret = ftrace_set_filter_ip(&direct_ops, ip, 1, 0); 5228 5229 WARN_ON(ret); 5230 5231 remove_hash_entry(direct_functions, entry); 5232 5233 direct = ftrace_find_direct_func(addr); 5234 if (!WARN_ON(!direct)) { 5235 /* This is the good path (see the ! before WARN) */ 5236 direct->count--; 5237 WARN_ON(direct->count < 0); 5238 if (!direct->count) { 5239 list_del_rcu(&direct->next); 5240 synchronize_rcu_tasks(); 5241 kfree(direct); 5242 kfree(entry); 5243 ftrace_direct_func_count--; 5244 } 5245 } 5246 out_unlock: 5247 mutex_unlock(&direct_mutex); 5248 5249 return ret; 5250 } 5251 EXPORT_SYMBOL_GPL(unregister_ftrace_direct); 5252 5253 static struct ftrace_ops stub_ops = { 5254 .func = ftrace_stub, 5255 }; 5256 5257 /** 5258 * ftrace_modify_direct_caller - modify ftrace nop directly 5259 * @entry: The ftrace hash entry of the direct helper for @rec 5260 * @rec: The record representing the function site to patch 5261 * @old_addr: The location that the site at @rec->ip currently calls 5262 * @new_addr: The location that the site at @rec->ip should call 5263 * 5264 * An architecture may overwrite this function to optimize the 5265 * changing of the direct callback on an ftrace nop location. 5266 * This is called with the ftrace_lock mutex held, and no other 5267 * ftrace callbacks are on the associated record (@rec). Thus, 5268 * it is safe to modify the ftrace record, where it should be 5269 * currently calling @old_addr directly, to call @new_addr. 5270 * 5271 * Safety checks should be made to make sure that the code at 5272 * @rec->ip is currently calling @old_addr. And this must 5273 * also update entry->direct to @new_addr. 5274 */ 5275 int __weak ftrace_modify_direct_caller(struct ftrace_func_entry *entry, 5276 struct dyn_ftrace *rec, 5277 unsigned long old_addr, 5278 unsigned long new_addr) 5279 { 5280 unsigned long ip = rec->ip; 5281 int ret; 5282 5283 /* 5284 * The ftrace_lock was used to determine if the record 5285 * had more than one registered user to it. If it did, 5286 * we needed to prevent that from changing to do the quick 5287 * switch. But if it did not (only a direct caller was attached) 5288 * then this function is called. But this function can deal 5289 * with attached callers to the rec that we care about, and 5290 * since this function uses standard ftrace calls that take 5291 * the ftrace_lock mutex, we need to release it. 5292 */ 5293 mutex_unlock(&ftrace_lock); 5294 5295 /* 5296 * By setting a stub function at the same address, we force 5297 * the code to call the iterator and the direct_ops helper. 5298 * This means that @ip does not call the direct call, and 5299 * we can simply modify it. 5300 */ 5301 ret = ftrace_set_filter_ip(&stub_ops, ip, 0, 0); 5302 if (ret) 5303 goto out_lock; 5304 5305 ret = register_ftrace_function(&stub_ops); 5306 if (ret) { 5307 ftrace_set_filter_ip(&stub_ops, ip, 1, 0); 5308 goto out_lock; 5309 } 5310 5311 entry->direct = new_addr; 5312 5313 /* 5314 * By removing the stub, we put back the direct call, calling 5315 * the @new_addr. 5316 */ 5317 unregister_ftrace_function(&stub_ops); 5318 ftrace_set_filter_ip(&stub_ops, ip, 1, 0); 5319 5320 out_lock: 5321 mutex_lock(&ftrace_lock); 5322 5323 return ret; 5324 } 5325 5326 /** 5327 * modify_ftrace_direct - Modify an existing direct call to call something else 5328 * @ip: The instruction pointer to modify 5329 * @old_addr: The address that the current @ip calls directly 5330 * @new_addr: The address that the @ip should call 5331 * 5332 * This modifies a ftrace direct caller at an instruction pointer without 5333 * having to disable it first. The direct call will switch over to the 5334 * @new_addr without missing anything. 5335 * 5336 * Returns: zero on success. Non zero on error, which includes: 5337 * -ENODEV : the @ip given has no direct caller attached 5338 * -EINVAL : the @old_addr does not match the current direct caller 5339 */ 5340 int modify_ftrace_direct(unsigned long ip, 5341 unsigned long old_addr, unsigned long new_addr) 5342 { 5343 struct ftrace_direct_func *direct, *new_direct = NULL; 5344 struct ftrace_func_entry *entry; 5345 struct dyn_ftrace *rec; 5346 int ret = -ENODEV; 5347 5348 mutex_lock(&direct_mutex); 5349 5350 mutex_lock(&ftrace_lock); 5351 entry = find_direct_entry(&ip, &rec); 5352 if (!entry) 5353 goto out_unlock; 5354 5355 ret = -EINVAL; 5356 if (entry->direct != old_addr) 5357 goto out_unlock; 5358 5359 direct = ftrace_find_direct_func(old_addr); 5360 if (WARN_ON(!direct)) 5361 goto out_unlock; 5362 if (direct->count > 1) { 5363 ret = -ENOMEM; 5364 new_direct = ftrace_alloc_direct_func(new_addr); 5365 if (!new_direct) 5366 goto out_unlock; 5367 direct->count--; 5368 new_direct->count++; 5369 } else { 5370 direct->addr = new_addr; 5371 } 5372 5373 /* 5374 * If there's no other ftrace callback on the rec->ip location, 5375 * then it can be changed directly by the architecture. 5376 * If there is another caller, then we just need to change the 5377 * direct caller helper to point to @new_addr. 5378 */ 5379 if (ftrace_rec_count(rec) == 1) { 5380 ret = ftrace_modify_direct_caller(entry, rec, old_addr, new_addr); 5381 } else { 5382 entry->direct = new_addr; 5383 ret = 0; 5384 } 5385 5386 if (unlikely(ret && new_direct)) { 5387 direct->count++; 5388 list_del_rcu(&new_direct->next); 5389 synchronize_rcu_tasks(); 5390 kfree(new_direct); 5391 ftrace_direct_func_count--; 5392 } 5393 5394 out_unlock: 5395 mutex_unlock(&ftrace_lock); 5396 mutex_unlock(&direct_mutex); 5397 return ret; 5398 } 5399 EXPORT_SYMBOL_GPL(modify_ftrace_direct); 5400 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ 5401 5402 /** 5403 * ftrace_set_filter_ip - set a function to filter on in ftrace by address 5404 * @ops - the ops to set the filter with 5405 * @ip - the address to add to or remove from the filter. 5406 * @remove - non zero to remove the ip from the filter 5407 * @reset - non zero to reset all filters before applying this filter. 5408 * 5409 * Filters denote which functions should be enabled when tracing is enabled 5410 * If @ip is NULL, it failes to update filter. 5411 */ 5412 int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip, 5413 int remove, int reset) 5414 { 5415 ftrace_ops_init(ops); 5416 return ftrace_set_addr(ops, ip, remove, reset, 1); 5417 } 5418 EXPORT_SYMBOL_GPL(ftrace_set_filter_ip); 5419 5420 /** 5421 * ftrace_ops_set_global_filter - setup ops to use global filters 5422 * @ops - the ops which will use the global filters 5423 * 5424 * ftrace users who need global function trace filtering should call this. 5425 * It can set the global filter only if ops were not initialized before. 5426 */ 5427 void ftrace_ops_set_global_filter(struct ftrace_ops *ops) 5428 { 5429 if (ops->flags & FTRACE_OPS_FL_INITIALIZED) 5430 return; 5431 5432 ftrace_ops_init(ops); 5433 ops->func_hash = &global_ops.local_hash; 5434 } 5435 EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter); 5436 5437 static int 5438 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len, 5439 int reset, int enable) 5440 { 5441 return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable); 5442 } 5443 5444 /** 5445 * ftrace_set_filter - set a function to filter on in ftrace 5446 * @ops - the ops to set the filter with 5447 * @buf - the string that holds the function filter text. 5448 * @len - the length of the string. 5449 * @reset - non zero to reset all filters before applying this filter. 5450 * 5451 * Filters denote which functions should be enabled when tracing is enabled. 5452 * If @buf is NULL and reset is set, all functions will be enabled for tracing. 5453 */ 5454 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf, 5455 int len, int reset) 5456 { 5457 ftrace_ops_init(ops); 5458 return ftrace_set_regex(ops, buf, len, reset, 1); 5459 } 5460 EXPORT_SYMBOL_GPL(ftrace_set_filter); 5461 5462 /** 5463 * ftrace_set_notrace - set a function to not trace in ftrace 5464 * @ops - the ops to set the notrace filter with 5465 * @buf - the string that holds the function notrace text. 5466 * @len - the length of the string. 5467 * @reset - non zero to reset all filters before applying this filter. 5468 * 5469 * Notrace Filters denote which functions should not be enabled when tracing 5470 * is enabled. If @buf is NULL and reset is set, all functions will be enabled 5471 * for tracing. 5472 */ 5473 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf, 5474 int len, int reset) 5475 { 5476 ftrace_ops_init(ops); 5477 return ftrace_set_regex(ops, buf, len, reset, 0); 5478 } 5479 EXPORT_SYMBOL_GPL(ftrace_set_notrace); 5480 /** 5481 * ftrace_set_global_filter - set a function to filter on with global tracers 5482 * @buf - the string that holds the function filter text. 5483 * @len - the length of the string. 5484 * @reset - non zero to reset all filters before applying this filter. 5485 * 5486 * Filters denote which functions should be enabled when tracing is enabled. 5487 * If @buf is NULL and reset is set, all functions will be enabled for tracing. 5488 */ 5489 void ftrace_set_global_filter(unsigned char *buf, int len, int reset) 5490 { 5491 ftrace_set_regex(&global_ops, buf, len, reset, 1); 5492 } 5493 EXPORT_SYMBOL_GPL(ftrace_set_global_filter); 5494 5495 /** 5496 * ftrace_set_global_notrace - set a function to not trace with global tracers 5497 * @buf - the string that holds the function notrace text. 5498 * @len - the length of the string. 5499 * @reset - non zero to reset all filters before applying this filter. 5500 * 5501 * Notrace Filters denote which functions should not be enabled when tracing 5502 * is enabled. If @buf is NULL and reset is set, all functions will be enabled 5503 * for tracing. 5504 */ 5505 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset) 5506 { 5507 ftrace_set_regex(&global_ops, buf, len, reset, 0); 5508 } 5509 EXPORT_SYMBOL_GPL(ftrace_set_global_notrace); 5510 5511 /* 5512 * command line interface to allow users to set filters on boot up. 5513 */ 5514 #define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE 5515 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata; 5516 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata; 5517 5518 /* Used by function selftest to not test if filter is set */ 5519 bool ftrace_filter_param __initdata; 5520 5521 static int __init set_ftrace_notrace(char *str) 5522 { 5523 ftrace_filter_param = true; 5524 strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE); 5525 return 1; 5526 } 5527 __setup("ftrace_notrace=", set_ftrace_notrace); 5528 5529 static int __init set_ftrace_filter(char *str) 5530 { 5531 ftrace_filter_param = true; 5532 strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE); 5533 return 1; 5534 } 5535 __setup("ftrace_filter=", set_ftrace_filter); 5536 5537 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 5538 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; 5539 static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata; 5540 static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer); 5541 5542 static int __init set_graph_function(char *str) 5543 { 5544 strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE); 5545 return 1; 5546 } 5547 __setup("ftrace_graph_filter=", set_graph_function); 5548 5549 static int __init set_graph_notrace_function(char *str) 5550 { 5551 strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE); 5552 return 1; 5553 } 5554 __setup("ftrace_graph_notrace=", set_graph_notrace_function); 5555 5556 static int __init set_graph_max_depth_function(char *str) 5557 { 5558 if (!str) 5559 return 0; 5560 fgraph_max_depth = simple_strtoul(str, NULL, 0); 5561 return 1; 5562 } 5563 __setup("ftrace_graph_max_depth=", set_graph_max_depth_function); 5564 5565 static void __init set_ftrace_early_graph(char *buf, int enable) 5566 { 5567 int ret; 5568 char *func; 5569 struct ftrace_hash *hash; 5570 5571 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); 5572 if (MEM_FAIL(!hash, "Failed to allocate hash\n")) 5573 return; 5574 5575 while (buf) { 5576 func = strsep(&buf, ","); 5577 /* we allow only one expression at a time */ 5578 ret = ftrace_graph_set_hash(hash, func); 5579 if (ret) 5580 printk(KERN_DEBUG "ftrace: function %s not " 5581 "traceable\n", func); 5582 } 5583 5584 if (enable) 5585 ftrace_graph_hash = hash; 5586 else 5587 ftrace_graph_notrace_hash = hash; 5588 } 5589 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 5590 5591 void __init 5592 ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable) 5593 { 5594 char *func; 5595 5596 ftrace_ops_init(ops); 5597 5598 while (buf) { 5599 func = strsep(&buf, ","); 5600 ftrace_set_regex(ops, func, strlen(func), 0, enable); 5601 } 5602 } 5603 5604 static void __init set_ftrace_early_filters(void) 5605 { 5606 if (ftrace_filter_buf[0]) 5607 ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1); 5608 if (ftrace_notrace_buf[0]) 5609 ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0); 5610 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 5611 if (ftrace_graph_buf[0]) 5612 set_ftrace_early_graph(ftrace_graph_buf, 1); 5613 if (ftrace_graph_notrace_buf[0]) 5614 set_ftrace_early_graph(ftrace_graph_notrace_buf, 0); 5615 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 5616 } 5617 5618 int ftrace_regex_release(struct inode *inode, struct file *file) 5619 { 5620 struct seq_file *m = (struct seq_file *)file->private_data; 5621 struct ftrace_iterator *iter; 5622 struct ftrace_hash **orig_hash; 5623 struct trace_parser *parser; 5624 int filter_hash; 5625 5626 if (file->f_mode & FMODE_READ) { 5627 iter = m->private; 5628 seq_release(inode, file); 5629 } else 5630 iter = file->private_data; 5631 5632 parser = &iter->parser; 5633 if (trace_parser_loaded(parser)) { 5634 ftrace_match_records(iter->hash, parser->buffer, parser->idx); 5635 } 5636 5637 trace_parser_put(parser); 5638 5639 mutex_lock(&iter->ops->func_hash->regex_lock); 5640 5641 if (file->f_mode & FMODE_WRITE) { 5642 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER); 5643 5644 if (filter_hash) { 5645 orig_hash = &iter->ops->func_hash->filter_hash; 5646 if (iter->tr && !list_empty(&iter->tr->mod_trace)) 5647 iter->hash->flags |= FTRACE_HASH_FL_MOD; 5648 } else 5649 orig_hash = &iter->ops->func_hash->notrace_hash; 5650 5651 mutex_lock(&ftrace_lock); 5652 ftrace_hash_move_and_update_ops(iter->ops, orig_hash, 5653 iter->hash, filter_hash); 5654 mutex_unlock(&ftrace_lock); 5655 } else { 5656 /* For read only, the hash is the ops hash */ 5657 iter->hash = NULL; 5658 } 5659 5660 mutex_unlock(&iter->ops->func_hash->regex_lock); 5661 free_ftrace_hash(iter->hash); 5662 if (iter->tr) 5663 trace_array_put(iter->tr); 5664 kfree(iter); 5665 5666 return 0; 5667 } 5668 5669 static const struct file_operations ftrace_avail_fops = { 5670 .open = ftrace_avail_open, 5671 .read = seq_read, 5672 .llseek = seq_lseek, 5673 .release = seq_release_private, 5674 }; 5675 5676 static const struct file_operations ftrace_enabled_fops = { 5677 .open = ftrace_enabled_open, 5678 .read = seq_read, 5679 .llseek = seq_lseek, 5680 .release = seq_release_private, 5681 }; 5682 5683 static const struct file_operations ftrace_filter_fops = { 5684 .open = ftrace_filter_open, 5685 .read = seq_read, 5686 .write = ftrace_filter_write, 5687 .llseek = tracing_lseek, 5688 .release = ftrace_regex_release, 5689 }; 5690 5691 static const struct file_operations ftrace_notrace_fops = { 5692 .open = ftrace_notrace_open, 5693 .read = seq_read, 5694 .write = ftrace_notrace_write, 5695 .llseek = tracing_lseek, 5696 .release = ftrace_regex_release, 5697 }; 5698 5699 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 5700 5701 static DEFINE_MUTEX(graph_lock); 5702 5703 struct ftrace_hash __rcu *ftrace_graph_hash = EMPTY_HASH; 5704 struct ftrace_hash __rcu *ftrace_graph_notrace_hash = EMPTY_HASH; 5705 5706 enum graph_filter_type { 5707 GRAPH_FILTER_NOTRACE = 0, 5708 GRAPH_FILTER_FUNCTION, 5709 }; 5710 5711 #define FTRACE_GRAPH_EMPTY ((void *)1) 5712 5713 struct ftrace_graph_data { 5714 struct ftrace_hash *hash; 5715 struct ftrace_func_entry *entry; 5716 int idx; /* for hash table iteration */ 5717 enum graph_filter_type type; 5718 struct ftrace_hash *new_hash; 5719 const struct seq_operations *seq_ops; 5720 struct trace_parser parser; 5721 }; 5722 5723 static void * 5724 __g_next(struct seq_file *m, loff_t *pos) 5725 { 5726 struct ftrace_graph_data *fgd = m->private; 5727 struct ftrace_func_entry *entry = fgd->entry; 5728 struct hlist_head *head; 5729 int i, idx = fgd->idx; 5730 5731 if (*pos >= fgd->hash->count) 5732 return NULL; 5733 5734 if (entry) { 5735 hlist_for_each_entry_continue(entry, hlist) { 5736 fgd->entry = entry; 5737 return entry; 5738 } 5739 5740 idx++; 5741 } 5742 5743 for (i = idx; i < 1 << fgd->hash->size_bits; i++) { 5744 head = &fgd->hash->buckets[i]; 5745 hlist_for_each_entry(entry, head, hlist) { 5746 fgd->entry = entry; 5747 fgd->idx = i; 5748 return entry; 5749 } 5750 } 5751 return NULL; 5752 } 5753 5754 static void * 5755 g_next(struct seq_file *m, void *v, loff_t *pos) 5756 { 5757 (*pos)++; 5758 return __g_next(m, pos); 5759 } 5760 5761 static void *g_start(struct seq_file *m, loff_t *pos) 5762 { 5763 struct ftrace_graph_data *fgd = m->private; 5764 5765 mutex_lock(&graph_lock); 5766 5767 if (fgd->type == GRAPH_FILTER_FUNCTION) 5768 fgd->hash = rcu_dereference_protected(ftrace_graph_hash, 5769 lockdep_is_held(&graph_lock)); 5770 else 5771 fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash, 5772 lockdep_is_held(&graph_lock)); 5773 5774 /* Nothing, tell g_show to print all functions are enabled */ 5775 if (ftrace_hash_empty(fgd->hash) && !*pos) 5776 return FTRACE_GRAPH_EMPTY; 5777 5778 fgd->idx = 0; 5779 fgd->entry = NULL; 5780 return __g_next(m, pos); 5781 } 5782 5783 static void g_stop(struct seq_file *m, void *p) 5784 { 5785 mutex_unlock(&graph_lock); 5786 } 5787 5788 static int g_show(struct seq_file *m, void *v) 5789 { 5790 struct ftrace_func_entry *entry = v; 5791 5792 if (!entry) 5793 return 0; 5794 5795 if (entry == FTRACE_GRAPH_EMPTY) { 5796 struct ftrace_graph_data *fgd = m->private; 5797 5798 if (fgd->type == GRAPH_FILTER_FUNCTION) 5799 seq_puts(m, "#### all functions enabled ####\n"); 5800 else 5801 seq_puts(m, "#### no functions disabled ####\n"); 5802 return 0; 5803 } 5804 5805 seq_printf(m, "%ps\n", (void *)entry->ip); 5806 5807 return 0; 5808 } 5809 5810 static const struct seq_operations ftrace_graph_seq_ops = { 5811 .start = g_start, 5812 .next = g_next, 5813 .stop = g_stop, 5814 .show = g_show, 5815 }; 5816 5817 static int 5818 __ftrace_graph_open(struct inode *inode, struct file *file, 5819 struct ftrace_graph_data *fgd) 5820 { 5821 int ret; 5822 struct ftrace_hash *new_hash = NULL; 5823 5824 ret = security_locked_down(LOCKDOWN_TRACEFS); 5825 if (ret) 5826 return ret; 5827 5828 if (file->f_mode & FMODE_WRITE) { 5829 const int size_bits = FTRACE_HASH_DEFAULT_BITS; 5830 5831 if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX)) 5832 return -ENOMEM; 5833 5834 if (file->f_flags & O_TRUNC) 5835 new_hash = alloc_ftrace_hash(size_bits); 5836 else 5837 new_hash = alloc_and_copy_ftrace_hash(size_bits, 5838 fgd->hash); 5839 if (!new_hash) { 5840 ret = -ENOMEM; 5841 goto out; 5842 } 5843 } 5844 5845 if (file->f_mode & FMODE_READ) { 5846 ret = seq_open(file, &ftrace_graph_seq_ops); 5847 if (!ret) { 5848 struct seq_file *m = file->private_data; 5849 m->private = fgd; 5850 } else { 5851 /* Failed */ 5852 free_ftrace_hash(new_hash); 5853 new_hash = NULL; 5854 } 5855 } else 5856 file->private_data = fgd; 5857 5858 out: 5859 if (ret < 0 && file->f_mode & FMODE_WRITE) 5860 trace_parser_put(&fgd->parser); 5861 5862 fgd->new_hash = new_hash; 5863 5864 /* 5865 * All uses of fgd->hash must be taken with the graph_lock 5866 * held. The graph_lock is going to be released, so force 5867 * fgd->hash to be reinitialized when it is taken again. 5868 */ 5869 fgd->hash = NULL; 5870 5871 return ret; 5872 } 5873 5874 static int 5875 ftrace_graph_open(struct inode *inode, struct file *file) 5876 { 5877 struct ftrace_graph_data *fgd; 5878 int ret; 5879 5880 if (unlikely(ftrace_disabled)) 5881 return -ENODEV; 5882 5883 fgd = kmalloc(sizeof(*fgd), GFP_KERNEL); 5884 if (fgd == NULL) 5885 return -ENOMEM; 5886 5887 mutex_lock(&graph_lock); 5888 5889 fgd->hash = rcu_dereference_protected(ftrace_graph_hash, 5890 lockdep_is_held(&graph_lock)); 5891 fgd->type = GRAPH_FILTER_FUNCTION; 5892 fgd->seq_ops = &ftrace_graph_seq_ops; 5893 5894 ret = __ftrace_graph_open(inode, file, fgd); 5895 if (ret < 0) 5896 kfree(fgd); 5897 5898 mutex_unlock(&graph_lock); 5899 return ret; 5900 } 5901 5902 static int 5903 ftrace_graph_notrace_open(struct inode *inode, struct file *file) 5904 { 5905 struct ftrace_graph_data *fgd; 5906 int ret; 5907 5908 if (unlikely(ftrace_disabled)) 5909 return -ENODEV; 5910 5911 fgd = kmalloc(sizeof(*fgd), GFP_KERNEL); 5912 if (fgd == NULL) 5913 return -ENOMEM; 5914 5915 mutex_lock(&graph_lock); 5916 5917 fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash, 5918 lockdep_is_held(&graph_lock)); 5919 fgd->type = GRAPH_FILTER_NOTRACE; 5920 fgd->seq_ops = &ftrace_graph_seq_ops; 5921 5922 ret = __ftrace_graph_open(inode, file, fgd); 5923 if (ret < 0) 5924 kfree(fgd); 5925 5926 mutex_unlock(&graph_lock); 5927 return ret; 5928 } 5929 5930 static int 5931 ftrace_graph_release(struct inode *inode, struct file *file) 5932 { 5933 struct ftrace_graph_data *fgd; 5934 struct ftrace_hash *old_hash, *new_hash; 5935 struct trace_parser *parser; 5936 int ret = 0; 5937 5938 if (file->f_mode & FMODE_READ) { 5939 struct seq_file *m = file->private_data; 5940 5941 fgd = m->private; 5942 seq_release(inode, file); 5943 } else { 5944 fgd = file->private_data; 5945 } 5946 5947 5948 if (file->f_mode & FMODE_WRITE) { 5949 5950 parser = &fgd->parser; 5951 5952 if (trace_parser_loaded((parser))) { 5953 ret = ftrace_graph_set_hash(fgd->new_hash, 5954 parser->buffer); 5955 } 5956 5957 trace_parser_put(parser); 5958 5959 new_hash = __ftrace_hash_move(fgd->new_hash); 5960 if (!new_hash) { 5961 ret = -ENOMEM; 5962 goto out; 5963 } 5964 5965 mutex_lock(&graph_lock); 5966 5967 if (fgd->type == GRAPH_FILTER_FUNCTION) { 5968 old_hash = rcu_dereference_protected(ftrace_graph_hash, 5969 lockdep_is_held(&graph_lock)); 5970 rcu_assign_pointer(ftrace_graph_hash, new_hash); 5971 } else { 5972 old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash, 5973 lockdep_is_held(&graph_lock)); 5974 rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash); 5975 } 5976 5977 mutex_unlock(&graph_lock); 5978 5979 /* 5980 * We need to do a hard force of sched synchronization. 5981 * This is because we use preempt_disable() to do RCU, but 5982 * the function tracers can be called where RCU is not watching 5983 * (like before user_exit()). We can not rely on the RCU 5984 * infrastructure to do the synchronization, thus we must do it 5985 * ourselves. 5986 */ 5987 synchronize_rcu_tasks_rude(); 5988 5989 free_ftrace_hash(old_hash); 5990 } 5991 5992 out: 5993 free_ftrace_hash(fgd->new_hash); 5994 kfree(fgd); 5995 5996 return ret; 5997 } 5998 5999 static int 6000 ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer) 6001 { 6002 struct ftrace_glob func_g; 6003 struct dyn_ftrace *rec; 6004 struct ftrace_page *pg; 6005 struct ftrace_func_entry *entry; 6006 int fail = 1; 6007 int not; 6008 6009 /* decode regex */ 6010 func_g.type = filter_parse_regex(buffer, strlen(buffer), 6011 &func_g.search, ¬); 6012 6013 func_g.len = strlen(func_g.search); 6014 6015 mutex_lock(&ftrace_lock); 6016 6017 if (unlikely(ftrace_disabled)) { 6018 mutex_unlock(&ftrace_lock); 6019 return -ENODEV; 6020 } 6021 6022 do_for_each_ftrace_rec(pg, rec) { 6023 6024 if (rec->flags & FTRACE_FL_DISABLED) 6025 continue; 6026 6027 if (ftrace_match_record(rec, &func_g, NULL, 0)) { 6028 entry = ftrace_lookup_ip(hash, rec->ip); 6029 6030 if (!not) { 6031 fail = 0; 6032 6033 if (entry) 6034 continue; 6035 if (add_hash_entry(hash, rec->ip) < 0) 6036 goto out; 6037 } else { 6038 if (entry) { 6039 free_hash_entry(hash, entry); 6040 fail = 0; 6041 } 6042 } 6043 } 6044 } while_for_each_ftrace_rec(); 6045 out: 6046 mutex_unlock(&ftrace_lock); 6047 6048 if (fail) 6049 return -EINVAL; 6050 6051 return 0; 6052 } 6053 6054 static ssize_t 6055 ftrace_graph_write(struct file *file, const char __user *ubuf, 6056 size_t cnt, loff_t *ppos) 6057 { 6058 ssize_t read, ret = 0; 6059 struct ftrace_graph_data *fgd = file->private_data; 6060 struct trace_parser *parser; 6061 6062 if (!cnt) 6063 return 0; 6064 6065 /* Read mode uses seq functions */ 6066 if (file->f_mode & FMODE_READ) { 6067 struct seq_file *m = file->private_data; 6068 fgd = m->private; 6069 } 6070 6071 parser = &fgd->parser; 6072 6073 read = trace_get_user(parser, ubuf, cnt, ppos); 6074 6075 if (read >= 0 && trace_parser_loaded(parser) && 6076 !trace_parser_cont(parser)) { 6077 6078 ret = ftrace_graph_set_hash(fgd->new_hash, 6079 parser->buffer); 6080 trace_parser_clear(parser); 6081 } 6082 6083 if (!ret) 6084 ret = read; 6085 6086 return ret; 6087 } 6088 6089 static const struct file_operations ftrace_graph_fops = { 6090 .open = ftrace_graph_open, 6091 .read = seq_read, 6092 .write = ftrace_graph_write, 6093 .llseek = tracing_lseek, 6094 .release = ftrace_graph_release, 6095 }; 6096 6097 static const struct file_operations ftrace_graph_notrace_fops = { 6098 .open = ftrace_graph_notrace_open, 6099 .read = seq_read, 6100 .write = ftrace_graph_write, 6101 .llseek = tracing_lseek, 6102 .release = ftrace_graph_release, 6103 }; 6104 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 6105 6106 void ftrace_create_filter_files(struct ftrace_ops *ops, 6107 struct dentry *parent) 6108 { 6109 6110 trace_create_file("set_ftrace_filter", 0644, parent, 6111 ops, &ftrace_filter_fops); 6112 6113 trace_create_file("set_ftrace_notrace", 0644, parent, 6114 ops, &ftrace_notrace_fops); 6115 } 6116 6117 /* 6118 * The name "destroy_filter_files" is really a misnomer. Although 6119 * in the future, it may actually delete the files, but this is 6120 * really intended to make sure the ops passed in are disabled 6121 * and that when this function returns, the caller is free to 6122 * free the ops. 6123 * 6124 * The "destroy" name is only to match the "create" name that this 6125 * should be paired with. 6126 */ 6127 void ftrace_destroy_filter_files(struct ftrace_ops *ops) 6128 { 6129 mutex_lock(&ftrace_lock); 6130 if (ops->flags & FTRACE_OPS_FL_ENABLED) 6131 ftrace_shutdown(ops, 0); 6132 ops->flags |= FTRACE_OPS_FL_DELETED; 6133 ftrace_free_filter(ops); 6134 mutex_unlock(&ftrace_lock); 6135 } 6136 6137 static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer) 6138 { 6139 6140 trace_create_file("available_filter_functions", 0444, 6141 d_tracer, NULL, &ftrace_avail_fops); 6142 6143 trace_create_file("enabled_functions", 0444, 6144 d_tracer, NULL, &ftrace_enabled_fops); 6145 6146 ftrace_create_filter_files(&global_ops, d_tracer); 6147 6148 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 6149 trace_create_file("set_graph_function", 0644, d_tracer, 6150 NULL, 6151 &ftrace_graph_fops); 6152 trace_create_file("set_graph_notrace", 0644, d_tracer, 6153 NULL, 6154 &ftrace_graph_notrace_fops); 6155 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 6156 6157 return 0; 6158 } 6159 6160 static int ftrace_cmp_ips(const void *a, const void *b) 6161 { 6162 const unsigned long *ipa = a; 6163 const unsigned long *ipb = b; 6164 6165 if (*ipa > *ipb) 6166 return 1; 6167 if (*ipa < *ipb) 6168 return -1; 6169 return 0; 6170 } 6171 6172 static int ftrace_process_locs(struct module *mod, 6173 unsigned long *start, 6174 unsigned long *end) 6175 { 6176 struct ftrace_page *start_pg; 6177 struct ftrace_page *pg; 6178 struct dyn_ftrace *rec; 6179 unsigned long count; 6180 unsigned long *p; 6181 unsigned long addr; 6182 unsigned long flags = 0; /* Shut up gcc */ 6183 int ret = -ENOMEM; 6184 6185 count = end - start; 6186 6187 if (!count) 6188 return 0; 6189 6190 sort(start, count, sizeof(*start), 6191 ftrace_cmp_ips, NULL); 6192 6193 start_pg = ftrace_allocate_pages(count); 6194 if (!start_pg) 6195 return -ENOMEM; 6196 6197 mutex_lock(&ftrace_lock); 6198 6199 /* 6200 * Core and each module needs their own pages, as 6201 * modules will free them when they are removed. 6202 * Force a new page to be allocated for modules. 6203 */ 6204 if (!mod) { 6205 WARN_ON(ftrace_pages || ftrace_pages_start); 6206 /* First initialization */ 6207 ftrace_pages = ftrace_pages_start = start_pg; 6208 } else { 6209 if (!ftrace_pages) 6210 goto out; 6211 6212 if (WARN_ON(ftrace_pages->next)) { 6213 /* Hmm, we have free pages? */ 6214 while (ftrace_pages->next) 6215 ftrace_pages = ftrace_pages->next; 6216 } 6217 6218 ftrace_pages->next = start_pg; 6219 } 6220 6221 p = start; 6222 pg = start_pg; 6223 while (p < end) { 6224 addr = ftrace_call_adjust(*p++); 6225 /* 6226 * Some architecture linkers will pad between 6227 * the different mcount_loc sections of different 6228 * object files to satisfy alignments. 6229 * Skip any NULL pointers. 6230 */ 6231 if (!addr) 6232 continue; 6233 6234 if (pg->index == pg->size) { 6235 /* We should have allocated enough */ 6236 if (WARN_ON(!pg->next)) 6237 break; 6238 pg = pg->next; 6239 } 6240 6241 rec = &pg->records[pg->index++]; 6242 rec->ip = addr; 6243 } 6244 6245 /* We should have used all pages */ 6246 WARN_ON(pg->next); 6247 6248 /* Assign the last page to ftrace_pages */ 6249 ftrace_pages = pg; 6250 6251 /* 6252 * We only need to disable interrupts on start up 6253 * because we are modifying code that an interrupt 6254 * may execute, and the modification is not atomic. 6255 * But for modules, nothing runs the code we modify 6256 * until we are finished with it, and there's no 6257 * reason to cause large interrupt latencies while we do it. 6258 */ 6259 if (!mod) 6260 local_irq_save(flags); 6261 ftrace_update_code(mod, start_pg); 6262 if (!mod) 6263 local_irq_restore(flags); 6264 ret = 0; 6265 out: 6266 mutex_unlock(&ftrace_lock); 6267 6268 return ret; 6269 } 6270 6271 struct ftrace_mod_func { 6272 struct list_head list; 6273 char *name; 6274 unsigned long ip; 6275 unsigned int size; 6276 }; 6277 6278 struct ftrace_mod_map { 6279 struct rcu_head rcu; 6280 struct list_head list; 6281 struct module *mod; 6282 unsigned long start_addr; 6283 unsigned long end_addr; 6284 struct list_head funcs; 6285 unsigned int num_funcs; 6286 }; 6287 6288 static int ftrace_get_trampoline_kallsym(unsigned int symnum, 6289 unsigned long *value, char *type, 6290 char *name, char *module_name, 6291 int *exported) 6292 { 6293 struct ftrace_ops *op; 6294 6295 list_for_each_entry_rcu(op, &ftrace_ops_trampoline_list, list) { 6296 if (!op->trampoline || symnum--) 6297 continue; 6298 *value = op->trampoline; 6299 *type = 't'; 6300 strlcpy(name, FTRACE_TRAMPOLINE_SYM, KSYM_NAME_LEN); 6301 strlcpy(module_name, FTRACE_TRAMPOLINE_MOD, MODULE_NAME_LEN); 6302 *exported = 0; 6303 return 0; 6304 } 6305 6306 return -ERANGE; 6307 } 6308 6309 #ifdef CONFIG_MODULES 6310 6311 #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next) 6312 6313 static LIST_HEAD(ftrace_mod_maps); 6314 6315 static int referenced_filters(struct dyn_ftrace *rec) 6316 { 6317 struct ftrace_ops *ops; 6318 int cnt = 0; 6319 6320 for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) { 6321 if (ops_references_rec(ops, rec)) { 6322 if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_DIRECT)) 6323 continue; 6324 if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_IPMODIFY)) 6325 continue; 6326 cnt++; 6327 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) 6328 rec->flags |= FTRACE_FL_REGS; 6329 if (cnt == 1 && ops->trampoline) 6330 rec->flags |= FTRACE_FL_TRAMP; 6331 else 6332 rec->flags &= ~FTRACE_FL_TRAMP; 6333 } 6334 } 6335 6336 return cnt; 6337 } 6338 6339 static void 6340 clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash) 6341 { 6342 struct ftrace_func_entry *entry; 6343 struct dyn_ftrace *rec; 6344 int i; 6345 6346 if (ftrace_hash_empty(hash)) 6347 return; 6348 6349 for (i = 0; i < pg->index; i++) { 6350 rec = &pg->records[i]; 6351 entry = __ftrace_lookup_ip(hash, rec->ip); 6352 /* 6353 * Do not allow this rec to match again. 6354 * Yeah, it may waste some memory, but will be removed 6355 * if/when the hash is modified again. 6356 */ 6357 if (entry) 6358 entry->ip = 0; 6359 } 6360 } 6361 6362 /* Clear any records from hashs */ 6363 static void clear_mod_from_hashes(struct ftrace_page *pg) 6364 { 6365 struct trace_array *tr; 6366 6367 mutex_lock(&trace_types_lock); 6368 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 6369 if (!tr->ops || !tr->ops->func_hash) 6370 continue; 6371 mutex_lock(&tr->ops->func_hash->regex_lock); 6372 clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash); 6373 clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash); 6374 mutex_unlock(&tr->ops->func_hash->regex_lock); 6375 } 6376 mutex_unlock(&trace_types_lock); 6377 } 6378 6379 static void ftrace_free_mod_map(struct rcu_head *rcu) 6380 { 6381 struct ftrace_mod_map *mod_map = container_of(rcu, struct ftrace_mod_map, rcu); 6382 struct ftrace_mod_func *mod_func; 6383 struct ftrace_mod_func *n; 6384 6385 /* All the contents of mod_map are now not visible to readers */ 6386 list_for_each_entry_safe(mod_func, n, &mod_map->funcs, list) { 6387 kfree(mod_func->name); 6388 list_del(&mod_func->list); 6389 kfree(mod_func); 6390 } 6391 6392 kfree(mod_map); 6393 } 6394 6395 void ftrace_release_mod(struct module *mod) 6396 { 6397 struct ftrace_mod_map *mod_map; 6398 struct ftrace_mod_map *n; 6399 struct dyn_ftrace *rec; 6400 struct ftrace_page **last_pg; 6401 struct ftrace_page *tmp_page = NULL; 6402 struct ftrace_page *pg; 6403 int order; 6404 6405 mutex_lock(&ftrace_lock); 6406 6407 if (ftrace_disabled) 6408 goto out_unlock; 6409 6410 list_for_each_entry_safe(mod_map, n, &ftrace_mod_maps, list) { 6411 if (mod_map->mod == mod) { 6412 list_del_rcu(&mod_map->list); 6413 call_rcu(&mod_map->rcu, ftrace_free_mod_map); 6414 break; 6415 } 6416 } 6417 6418 /* 6419 * Each module has its own ftrace_pages, remove 6420 * them from the list. 6421 */ 6422 last_pg = &ftrace_pages_start; 6423 for (pg = ftrace_pages_start; pg; pg = *last_pg) { 6424 rec = &pg->records[0]; 6425 if (within_module_core(rec->ip, mod) || 6426 within_module_init(rec->ip, mod)) { 6427 /* 6428 * As core pages are first, the first 6429 * page should never be a module page. 6430 */ 6431 if (WARN_ON(pg == ftrace_pages_start)) 6432 goto out_unlock; 6433 6434 /* Check if we are deleting the last page */ 6435 if (pg == ftrace_pages) 6436 ftrace_pages = next_to_ftrace_page(last_pg); 6437 6438 ftrace_update_tot_cnt -= pg->index; 6439 *last_pg = pg->next; 6440 6441 pg->next = tmp_page; 6442 tmp_page = pg; 6443 } else 6444 last_pg = &pg->next; 6445 } 6446 out_unlock: 6447 mutex_unlock(&ftrace_lock); 6448 6449 for (pg = tmp_page; pg; pg = tmp_page) { 6450 6451 /* Needs to be called outside of ftrace_lock */ 6452 clear_mod_from_hashes(pg); 6453 6454 order = get_count_order(pg->size / ENTRIES_PER_PAGE); 6455 if (order >= 0) 6456 free_pages((unsigned long)pg->records, order); 6457 tmp_page = pg->next; 6458 kfree(pg); 6459 ftrace_number_of_pages -= 1 << order; 6460 ftrace_number_of_groups--; 6461 } 6462 } 6463 6464 void ftrace_module_enable(struct module *mod) 6465 { 6466 struct dyn_ftrace *rec; 6467 struct ftrace_page *pg; 6468 6469 mutex_lock(&ftrace_lock); 6470 6471 if (ftrace_disabled) 6472 goto out_unlock; 6473 6474 /* 6475 * If the tracing is enabled, go ahead and enable the record. 6476 * 6477 * The reason not to enable the record immediately is the 6478 * inherent check of ftrace_make_nop/ftrace_make_call for 6479 * correct previous instructions. Making first the NOP 6480 * conversion puts the module to the correct state, thus 6481 * passing the ftrace_make_call check. 6482 * 6483 * We also delay this to after the module code already set the 6484 * text to read-only, as we now need to set it back to read-write 6485 * so that we can modify the text. 6486 */ 6487 if (ftrace_start_up) 6488 ftrace_arch_code_modify_prepare(); 6489 6490 do_for_each_ftrace_rec(pg, rec) { 6491 int cnt; 6492 /* 6493 * do_for_each_ftrace_rec() is a double loop. 6494 * module text shares the pg. If a record is 6495 * not part of this module, then skip this pg, 6496 * which the "break" will do. 6497 */ 6498 if (!within_module_core(rec->ip, mod) && 6499 !within_module_init(rec->ip, mod)) 6500 break; 6501 6502 cnt = 0; 6503 6504 /* 6505 * When adding a module, we need to check if tracers are 6506 * currently enabled and if they are, and can trace this record, 6507 * we need to enable the module functions as well as update the 6508 * reference counts for those function records. 6509 */ 6510 if (ftrace_start_up) 6511 cnt += referenced_filters(rec); 6512 6513 rec->flags &= ~FTRACE_FL_DISABLED; 6514 rec->flags += cnt; 6515 6516 if (ftrace_start_up && cnt) { 6517 int failed = __ftrace_replace_code(rec, 1); 6518 if (failed) { 6519 ftrace_bug(failed, rec); 6520 goto out_loop; 6521 } 6522 } 6523 6524 } while_for_each_ftrace_rec(); 6525 6526 out_loop: 6527 if (ftrace_start_up) 6528 ftrace_arch_code_modify_post_process(); 6529 6530 out_unlock: 6531 mutex_unlock(&ftrace_lock); 6532 6533 process_cached_mods(mod->name); 6534 } 6535 6536 void ftrace_module_init(struct module *mod) 6537 { 6538 if (ftrace_disabled || !mod->num_ftrace_callsites) 6539 return; 6540 6541 ftrace_process_locs(mod, mod->ftrace_callsites, 6542 mod->ftrace_callsites + mod->num_ftrace_callsites); 6543 } 6544 6545 static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map, 6546 struct dyn_ftrace *rec) 6547 { 6548 struct ftrace_mod_func *mod_func; 6549 unsigned long symsize; 6550 unsigned long offset; 6551 char str[KSYM_SYMBOL_LEN]; 6552 char *modname; 6553 const char *ret; 6554 6555 ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str); 6556 if (!ret) 6557 return; 6558 6559 mod_func = kmalloc(sizeof(*mod_func), GFP_KERNEL); 6560 if (!mod_func) 6561 return; 6562 6563 mod_func->name = kstrdup(str, GFP_KERNEL); 6564 if (!mod_func->name) { 6565 kfree(mod_func); 6566 return; 6567 } 6568 6569 mod_func->ip = rec->ip - offset; 6570 mod_func->size = symsize; 6571 6572 mod_map->num_funcs++; 6573 6574 list_add_rcu(&mod_func->list, &mod_map->funcs); 6575 } 6576 6577 static struct ftrace_mod_map * 6578 allocate_ftrace_mod_map(struct module *mod, 6579 unsigned long start, unsigned long end) 6580 { 6581 struct ftrace_mod_map *mod_map; 6582 6583 mod_map = kmalloc(sizeof(*mod_map), GFP_KERNEL); 6584 if (!mod_map) 6585 return NULL; 6586 6587 mod_map->mod = mod; 6588 mod_map->start_addr = start; 6589 mod_map->end_addr = end; 6590 mod_map->num_funcs = 0; 6591 6592 INIT_LIST_HEAD_RCU(&mod_map->funcs); 6593 6594 list_add_rcu(&mod_map->list, &ftrace_mod_maps); 6595 6596 return mod_map; 6597 } 6598 6599 static const char * 6600 ftrace_func_address_lookup(struct ftrace_mod_map *mod_map, 6601 unsigned long addr, unsigned long *size, 6602 unsigned long *off, char *sym) 6603 { 6604 struct ftrace_mod_func *found_func = NULL; 6605 struct ftrace_mod_func *mod_func; 6606 6607 list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) { 6608 if (addr >= mod_func->ip && 6609 addr < mod_func->ip + mod_func->size) { 6610 found_func = mod_func; 6611 break; 6612 } 6613 } 6614 6615 if (found_func) { 6616 if (size) 6617 *size = found_func->size; 6618 if (off) 6619 *off = addr - found_func->ip; 6620 if (sym) 6621 strlcpy(sym, found_func->name, KSYM_NAME_LEN); 6622 6623 return found_func->name; 6624 } 6625 6626 return NULL; 6627 } 6628 6629 const char * 6630 ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, 6631 unsigned long *off, char **modname, char *sym) 6632 { 6633 struct ftrace_mod_map *mod_map; 6634 const char *ret = NULL; 6635 6636 /* mod_map is freed via call_rcu() */ 6637 preempt_disable(); 6638 list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) { 6639 ret = ftrace_func_address_lookup(mod_map, addr, size, off, sym); 6640 if (ret) { 6641 if (modname) 6642 *modname = mod_map->mod->name; 6643 break; 6644 } 6645 } 6646 preempt_enable(); 6647 6648 return ret; 6649 } 6650 6651 int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value, 6652 char *type, char *name, 6653 char *module_name, int *exported) 6654 { 6655 struct ftrace_mod_map *mod_map; 6656 struct ftrace_mod_func *mod_func; 6657 int ret; 6658 6659 preempt_disable(); 6660 list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) { 6661 6662 if (symnum >= mod_map->num_funcs) { 6663 symnum -= mod_map->num_funcs; 6664 continue; 6665 } 6666 6667 list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) { 6668 if (symnum > 1) { 6669 symnum--; 6670 continue; 6671 } 6672 6673 *value = mod_func->ip; 6674 *type = 'T'; 6675 strlcpy(name, mod_func->name, KSYM_NAME_LEN); 6676 strlcpy(module_name, mod_map->mod->name, MODULE_NAME_LEN); 6677 *exported = 1; 6678 preempt_enable(); 6679 return 0; 6680 } 6681 WARN_ON(1); 6682 break; 6683 } 6684 ret = ftrace_get_trampoline_kallsym(symnum, value, type, name, 6685 module_name, exported); 6686 preempt_enable(); 6687 return ret; 6688 } 6689 6690 #else 6691 static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map, 6692 struct dyn_ftrace *rec) { } 6693 static inline struct ftrace_mod_map * 6694 allocate_ftrace_mod_map(struct module *mod, 6695 unsigned long start, unsigned long end) 6696 { 6697 return NULL; 6698 } 6699 int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value, 6700 char *type, char *name, char *module_name, 6701 int *exported) 6702 { 6703 int ret; 6704 6705 preempt_disable(); 6706 ret = ftrace_get_trampoline_kallsym(symnum, value, type, name, 6707 module_name, exported); 6708 preempt_enable(); 6709 return ret; 6710 } 6711 #endif /* CONFIG_MODULES */ 6712 6713 struct ftrace_init_func { 6714 struct list_head list; 6715 unsigned long ip; 6716 }; 6717 6718 /* Clear any init ips from hashes */ 6719 static void 6720 clear_func_from_hash(struct ftrace_init_func *func, struct ftrace_hash *hash) 6721 { 6722 struct ftrace_func_entry *entry; 6723 6724 entry = ftrace_lookup_ip(hash, func->ip); 6725 /* 6726 * Do not allow this rec to match again. 6727 * Yeah, it may waste some memory, but will be removed 6728 * if/when the hash is modified again. 6729 */ 6730 if (entry) 6731 entry->ip = 0; 6732 } 6733 6734 static void 6735 clear_func_from_hashes(struct ftrace_init_func *func) 6736 { 6737 struct trace_array *tr; 6738 6739 mutex_lock(&trace_types_lock); 6740 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 6741 if (!tr->ops || !tr->ops->func_hash) 6742 continue; 6743 mutex_lock(&tr->ops->func_hash->regex_lock); 6744 clear_func_from_hash(func, tr->ops->func_hash->filter_hash); 6745 clear_func_from_hash(func, tr->ops->func_hash->notrace_hash); 6746 mutex_unlock(&tr->ops->func_hash->regex_lock); 6747 } 6748 mutex_unlock(&trace_types_lock); 6749 } 6750 6751 static void add_to_clear_hash_list(struct list_head *clear_list, 6752 struct dyn_ftrace *rec) 6753 { 6754 struct ftrace_init_func *func; 6755 6756 func = kmalloc(sizeof(*func), GFP_KERNEL); 6757 if (!func) { 6758 MEM_FAIL(1, "alloc failure, ftrace filter could be stale\n"); 6759 return; 6760 } 6761 6762 func->ip = rec->ip; 6763 list_add(&func->list, clear_list); 6764 } 6765 6766 void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr) 6767 { 6768 unsigned long start = (unsigned long)(start_ptr); 6769 unsigned long end = (unsigned long)(end_ptr); 6770 struct ftrace_page **last_pg = &ftrace_pages_start; 6771 struct ftrace_page *pg; 6772 struct dyn_ftrace *rec; 6773 struct dyn_ftrace key; 6774 struct ftrace_mod_map *mod_map = NULL; 6775 struct ftrace_init_func *func, *func_next; 6776 struct list_head clear_hash; 6777 int order; 6778 6779 INIT_LIST_HEAD(&clear_hash); 6780 6781 key.ip = start; 6782 key.flags = end; /* overload flags, as it is unsigned long */ 6783 6784 mutex_lock(&ftrace_lock); 6785 6786 /* 6787 * If we are freeing module init memory, then check if 6788 * any tracer is active. If so, we need to save a mapping of 6789 * the module functions being freed with the address. 6790 */ 6791 if (mod && ftrace_ops_list != &ftrace_list_end) 6792 mod_map = allocate_ftrace_mod_map(mod, start, end); 6793 6794 for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) { 6795 if (end < pg->records[0].ip || 6796 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) 6797 continue; 6798 again: 6799 rec = bsearch(&key, pg->records, pg->index, 6800 sizeof(struct dyn_ftrace), 6801 ftrace_cmp_recs); 6802 if (!rec) 6803 continue; 6804 6805 /* rec will be cleared from hashes after ftrace_lock unlock */ 6806 add_to_clear_hash_list(&clear_hash, rec); 6807 6808 if (mod_map) 6809 save_ftrace_mod_rec(mod_map, rec); 6810 6811 pg->index--; 6812 ftrace_update_tot_cnt--; 6813 if (!pg->index) { 6814 *last_pg = pg->next; 6815 order = get_count_order(pg->size / ENTRIES_PER_PAGE); 6816 if (order >= 0) 6817 free_pages((unsigned long)pg->records, order); 6818 ftrace_number_of_pages -= 1 << order; 6819 ftrace_number_of_groups--; 6820 kfree(pg); 6821 pg = container_of(last_pg, struct ftrace_page, next); 6822 if (!(*last_pg)) 6823 ftrace_pages = pg; 6824 continue; 6825 } 6826 memmove(rec, rec + 1, 6827 (pg->index - (rec - pg->records)) * sizeof(*rec)); 6828 /* More than one function may be in this block */ 6829 goto again; 6830 } 6831 mutex_unlock(&ftrace_lock); 6832 6833 list_for_each_entry_safe(func, func_next, &clear_hash, list) { 6834 clear_func_from_hashes(func); 6835 kfree(func); 6836 } 6837 } 6838 6839 void __init ftrace_free_init_mem(void) 6840 { 6841 void *start = (void *)(&__init_begin); 6842 void *end = (void *)(&__init_end); 6843 6844 ftrace_free_mem(NULL, start, end); 6845 } 6846 6847 void __init ftrace_init(void) 6848 { 6849 extern unsigned long __start_mcount_loc[]; 6850 extern unsigned long __stop_mcount_loc[]; 6851 unsigned long count, flags; 6852 int ret; 6853 6854 local_irq_save(flags); 6855 ret = ftrace_dyn_arch_init(); 6856 local_irq_restore(flags); 6857 if (ret) 6858 goto failed; 6859 6860 count = __stop_mcount_loc - __start_mcount_loc; 6861 if (!count) { 6862 pr_info("ftrace: No functions to be traced?\n"); 6863 goto failed; 6864 } 6865 6866 pr_info("ftrace: allocating %ld entries in %ld pages\n", 6867 count, count / ENTRIES_PER_PAGE + 1); 6868 6869 last_ftrace_enabled = ftrace_enabled = 1; 6870 6871 ret = ftrace_process_locs(NULL, 6872 __start_mcount_loc, 6873 __stop_mcount_loc); 6874 6875 pr_info("ftrace: allocated %ld pages with %ld groups\n", 6876 ftrace_number_of_pages, ftrace_number_of_groups); 6877 6878 set_ftrace_early_filters(); 6879 6880 return; 6881 failed: 6882 ftrace_disabled = 1; 6883 } 6884 6885 /* Do nothing if arch does not support this */ 6886 void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops) 6887 { 6888 } 6889 6890 static void ftrace_update_trampoline(struct ftrace_ops *ops) 6891 { 6892 unsigned long trampoline = ops->trampoline; 6893 6894 arch_ftrace_update_trampoline(ops); 6895 if (ops->trampoline && ops->trampoline != trampoline && 6896 (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP)) { 6897 /* Add to kallsyms before the perf events */ 6898 ftrace_add_trampoline_to_kallsyms(ops); 6899 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, 6900 ops->trampoline, ops->trampoline_size, false, 6901 FTRACE_TRAMPOLINE_SYM); 6902 /* 6903 * Record the perf text poke event after the ksymbol register 6904 * event. 6905 */ 6906 perf_event_text_poke((void *)ops->trampoline, NULL, 0, 6907 (void *)ops->trampoline, 6908 ops->trampoline_size); 6909 } 6910 } 6911 6912 void ftrace_init_trace_array(struct trace_array *tr) 6913 { 6914 INIT_LIST_HEAD(&tr->func_probes); 6915 INIT_LIST_HEAD(&tr->mod_trace); 6916 INIT_LIST_HEAD(&tr->mod_notrace); 6917 } 6918 #else 6919 6920 struct ftrace_ops global_ops = { 6921 .func = ftrace_stub, 6922 .flags = FTRACE_OPS_FL_INITIALIZED | 6923 FTRACE_OPS_FL_PID, 6924 }; 6925 6926 static int __init ftrace_nodyn_init(void) 6927 { 6928 ftrace_enabled = 1; 6929 return 0; 6930 } 6931 core_initcall(ftrace_nodyn_init); 6932 6933 static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; } 6934 static inline void ftrace_startup_enable(int command) { } 6935 static inline void ftrace_startup_all(int command) { } 6936 6937 # define ftrace_startup_sysctl() do { } while (0) 6938 # define ftrace_shutdown_sysctl() do { } while (0) 6939 6940 static void ftrace_update_trampoline(struct ftrace_ops *ops) 6941 { 6942 } 6943 6944 #endif /* CONFIG_DYNAMIC_FTRACE */ 6945 6946 __init void ftrace_init_global_array_ops(struct trace_array *tr) 6947 { 6948 tr->ops = &global_ops; 6949 tr->ops->private = tr; 6950 ftrace_init_trace_array(tr); 6951 } 6952 6953 void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func) 6954 { 6955 /* If we filter on pids, update to use the pid function */ 6956 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) { 6957 if (WARN_ON(tr->ops->func != ftrace_stub)) 6958 printk("ftrace ops had %pS for function\n", 6959 tr->ops->func); 6960 } 6961 tr->ops->func = func; 6962 tr->ops->private = tr; 6963 } 6964 6965 void ftrace_reset_array_ops(struct trace_array *tr) 6966 { 6967 tr->ops->func = ftrace_stub; 6968 } 6969 6970 static nokprobe_inline void 6971 __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 6972 struct ftrace_ops *ignored, struct ftrace_regs *fregs) 6973 { 6974 struct pt_regs *regs = ftrace_get_regs(fregs); 6975 struct ftrace_ops *op; 6976 int bit; 6977 6978 bit = trace_test_and_set_recursion(ip, parent_ip, TRACE_LIST_START, TRACE_LIST_MAX); 6979 if (bit < 0) 6980 return; 6981 6982 /* 6983 * Some of the ops may be dynamically allocated, 6984 * they must be freed after a synchronize_rcu(). 6985 */ 6986 preempt_disable_notrace(); 6987 6988 do_for_each_ftrace_op(op, ftrace_ops_list) { 6989 /* Stub functions don't need to be called nor tested */ 6990 if (op->flags & FTRACE_OPS_FL_STUB) 6991 continue; 6992 /* 6993 * Check the following for each ops before calling their func: 6994 * if RCU flag is set, then rcu_is_watching() must be true 6995 * if PER_CPU is set, then ftrace_function_local_disable() 6996 * must be false 6997 * Otherwise test if the ip matches the ops filter 6998 * 6999 * If any of the above fails then the op->func() is not executed. 7000 */ 7001 if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) && 7002 ftrace_ops_test(op, ip, regs)) { 7003 if (FTRACE_WARN_ON(!op->func)) { 7004 pr_warn("op=%p %pS\n", op, op); 7005 goto out; 7006 } 7007 op->func(ip, parent_ip, op, fregs); 7008 } 7009 } while_for_each_ftrace_op(op); 7010 out: 7011 preempt_enable_notrace(); 7012 trace_clear_recursion(bit); 7013 } 7014 7015 /* 7016 * Some archs only support passing ip and parent_ip. Even though 7017 * the list function ignores the op parameter, we do not want any 7018 * C side effects, where a function is called without the caller 7019 * sending a third parameter. 7020 * Archs are to support both the regs and ftrace_ops at the same time. 7021 * If they support ftrace_ops, it is assumed they support regs. 7022 * If call backs want to use regs, they must either check for regs 7023 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS. 7024 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved. 7025 * An architecture can pass partial regs with ftrace_ops and still 7026 * set the ARCH_SUPPORTS_FTRACE_OPS. 7027 */ 7028 #if ARCH_SUPPORTS_FTRACE_OPS 7029 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 7030 struct ftrace_ops *op, struct ftrace_regs *fregs) 7031 { 7032 __ftrace_ops_list_func(ip, parent_ip, NULL, fregs); 7033 } 7034 NOKPROBE_SYMBOL(ftrace_ops_list_func); 7035 #else 7036 static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip) 7037 { 7038 __ftrace_ops_list_func(ip, parent_ip, NULL, NULL); 7039 } 7040 NOKPROBE_SYMBOL(ftrace_ops_no_ops); 7041 #endif 7042 7043 /* 7044 * If there's only one function registered but it does not support 7045 * recursion, needs RCU protection and/or requires per cpu handling, then 7046 * this function will be called by the mcount trampoline. 7047 */ 7048 static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip, 7049 struct ftrace_ops *op, struct ftrace_regs *fregs) 7050 { 7051 int bit; 7052 7053 bit = trace_test_and_set_recursion(ip, parent_ip, TRACE_LIST_START, TRACE_LIST_MAX); 7054 if (bit < 0) 7055 return; 7056 7057 preempt_disable_notrace(); 7058 7059 if (!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) 7060 op->func(ip, parent_ip, op, fregs); 7061 7062 preempt_enable_notrace(); 7063 trace_clear_recursion(bit); 7064 } 7065 NOKPROBE_SYMBOL(ftrace_ops_assist_func); 7066 7067 /** 7068 * ftrace_ops_get_func - get the function a trampoline should call 7069 * @ops: the ops to get the function for 7070 * 7071 * Normally the mcount trampoline will call the ops->func, but there 7072 * are times that it should not. For example, if the ops does not 7073 * have its own recursion protection, then it should call the 7074 * ftrace_ops_assist_func() instead. 7075 * 7076 * Returns the function that the trampoline should call for @ops. 7077 */ 7078 ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops) 7079 { 7080 /* 7081 * If the function does not handle recursion or needs to be RCU safe, 7082 * then we need to call the assist handler. 7083 */ 7084 if (ops->flags & (FTRACE_OPS_FL_RECURSION | 7085 FTRACE_OPS_FL_RCU)) 7086 return ftrace_ops_assist_func; 7087 7088 return ops->func; 7089 } 7090 7091 static void 7092 ftrace_filter_pid_sched_switch_probe(void *data, bool preempt, 7093 struct task_struct *prev, struct task_struct *next) 7094 { 7095 struct trace_array *tr = data; 7096 struct trace_pid_list *pid_list; 7097 struct trace_pid_list *no_pid_list; 7098 7099 pid_list = rcu_dereference_sched(tr->function_pids); 7100 no_pid_list = rcu_dereference_sched(tr->function_no_pids); 7101 7102 if (trace_ignore_this_task(pid_list, no_pid_list, next)) 7103 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, 7104 FTRACE_PID_IGNORE); 7105 else 7106 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, 7107 next->pid); 7108 } 7109 7110 static void 7111 ftrace_pid_follow_sched_process_fork(void *data, 7112 struct task_struct *self, 7113 struct task_struct *task) 7114 { 7115 struct trace_pid_list *pid_list; 7116 struct trace_array *tr = data; 7117 7118 pid_list = rcu_dereference_sched(tr->function_pids); 7119 trace_filter_add_remove_task(pid_list, self, task); 7120 7121 pid_list = rcu_dereference_sched(tr->function_no_pids); 7122 trace_filter_add_remove_task(pid_list, self, task); 7123 } 7124 7125 static void 7126 ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task) 7127 { 7128 struct trace_pid_list *pid_list; 7129 struct trace_array *tr = data; 7130 7131 pid_list = rcu_dereference_sched(tr->function_pids); 7132 trace_filter_add_remove_task(pid_list, NULL, task); 7133 7134 pid_list = rcu_dereference_sched(tr->function_no_pids); 7135 trace_filter_add_remove_task(pid_list, NULL, task); 7136 } 7137 7138 void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) 7139 { 7140 if (enable) { 7141 register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork, 7142 tr); 7143 register_trace_sched_process_free(ftrace_pid_follow_sched_process_exit, 7144 tr); 7145 } else { 7146 unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork, 7147 tr); 7148 unregister_trace_sched_process_free(ftrace_pid_follow_sched_process_exit, 7149 tr); 7150 } 7151 } 7152 7153 static void clear_ftrace_pids(struct trace_array *tr, int type) 7154 { 7155 struct trace_pid_list *pid_list; 7156 struct trace_pid_list *no_pid_list; 7157 int cpu; 7158 7159 pid_list = rcu_dereference_protected(tr->function_pids, 7160 lockdep_is_held(&ftrace_lock)); 7161 no_pid_list = rcu_dereference_protected(tr->function_no_pids, 7162 lockdep_is_held(&ftrace_lock)); 7163 7164 /* Make sure there's something to do */ 7165 if (!pid_type_enabled(type, pid_list, no_pid_list)) 7166 return; 7167 7168 /* See if the pids still need to be checked after this */ 7169 if (!still_need_pid_events(type, pid_list, no_pid_list)) { 7170 unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr); 7171 for_each_possible_cpu(cpu) 7172 per_cpu_ptr(tr->array_buffer.data, cpu)->ftrace_ignore_pid = FTRACE_PID_TRACE; 7173 } 7174 7175 if (type & TRACE_PIDS) 7176 rcu_assign_pointer(tr->function_pids, NULL); 7177 7178 if (type & TRACE_NO_PIDS) 7179 rcu_assign_pointer(tr->function_no_pids, NULL); 7180 7181 /* Wait till all users are no longer using pid filtering */ 7182 synchronize_rcu(); 7183 7184 if ((type & TRACE_PIDS) && pid_list) 7185 trace_free_pid_list(pid_list); 7186 7187 if ((type & TRACE_NO_PIDS) && no_pid_list) 7188 trace_free_pid_list(no_pid_list); 7189 } 7190 7191 void ftrace_clear_pids(struct trace_array *tr) 7192 { 7193 mutex_lock(&ftrace_lock); 7194 7195 clear_ftrace_pids(tr, TRACE_PIDS | TRACE_NO_PIDS); 7196 7197 mutex_unlock(&ftrace_lock); 7198 } 7199 7200 static void ftrace_pid_reset(struct trace_array *tr, int type) 7201 { 7202 mutex_lock(&ftrace_lock); 7203 clear_ftrace_pids(tr, type); 7204 7205 ftrace_update_pid_func(); 7206 ftrace_startup_all(0); 7207 7208 mutex_unlock(&ftrace_lock); 7209 } 7210 7211 /* Greater than any max PID */ 7212 #define FTRACE_NO_PIDS (void *)(PID_MAX_LIMIT + 1) 7213 7214 static void *fpid_start(struct seq_file *m, loff_t *pos) 7215 __acquires(RCU) 7216 { 7217 struct trace_pid_list *pid_list; 7218 struct trace_array *tr = m->private; 7219 7220 mutex_lock(&ftrace_lock); 7221 rcu_read_lock_sched(); 7222 7223 pid_list = rcu_dereference_sched(tr->function_pids); 7224 7225 if (!pid_list) 7226 return !(*pos) ? FTRACE_NO_PIDS : NULL; 7227 7228 return trace_pid_start(pid_list, pos); 7229 } 7230 7231 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos) 7232 { 7233 struct trace_array *tr = m->private; 7234 struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids); 7235 7236 if (v == FTRACE_NO_PIDS) { 7237 (*pos)++; 7238 return NULL; 7239 } 7240 return trace_pid_next(pid_list, v, pos); 7241 } 7242 7243 static void fpid_stop(struct seq_file *m, void *p) 7244 __releases(RCU) 7245 { 7246 rcu_read_unlock_sched(); 7247 mutex_unlock(&ftrace_lock); 7248 } 7249 7250 static int fpid_show(struct seq_file *m, void *v) 7251 { 7252 if (v == FTRACE_NO_PIDS) { 7253 seq_puts(m, "no pid\n"); 7254 return 0; 7255 } 7256 7257 return trace_pid_show(m, v); 7258 } 7259 7260 static const struct seq_operations ftrace_pid_sops = { 7261 .start = fpid_start, 7262 .next = fpid_next, 7263 .stop = fpid_stop, 7264 .show = fpid_show, 7265 }; 7266 7267 static void *fnpid_start(struct seq_file *m, loff_t *pos) 7268 __acquires(RCU) 7269 { 7270 struct trace_pid_list *pid_list; 7271 struct trace_array *tr = m->private; 7272 7273 mutex_lock(&ftrace_lock); 7274 rcu_read_lock_sched(); 7275 7276 pid_list = rcu_dereference_sched(tr->function_no_pids); 7277 7278 if (!pid_list) 7279 return !(*pos) ? FTRACE_NO_PIDS : NULL; 7280 7281 return trace_pid_start(pid_list, pos); 7282 } 7283 7284 static void *fnpid_next(struct seq_file *m, void *v, loff_t *pos) 7285 { 7286 struct trace_array *tr = m->private; 7287 struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_no_pids); 7288 7289 if (v == FTRACE_NO_PIDS) { 7290 (*pos)++; 7291 return NULL; 7292 } 7293 return trace_pid_next(pid_list, v, pos); 7294 } 7295 7296 static const struct seq_operations ftrace_no_pid_sops = { 7297 .start = fnpid_start, 7298 .next = fnpid_next, 7299 .stop = fpid_stop, 7300 .show = fpid_show, 7301 }; 7302 7303 static int pid_open(struct inode *inode, struct file *file, int type) 7304 { 7305 const struct seq_operations *seq_ops; 7306 struct trace_array *tr = inode->i_private; 7307 struct seq_file *m; 7308 int ret = 0; 7309 7310 ret = tracing_check_open_get_tr(tr); 7311 if (ret) 7312 return ret; 7313 7314 if ((file->f_mode & FMODE_WRITE) && 7315 (file->f_flags & O_TRUNC)) 7316 ftrace_pid_reset(tr, type); 7317 7318 switch (type) { 7319 case TRACE_PIDS: 7320 seq_ops = &ftrace_pid_sops; 7321 break; 7322 case TRACE_NO_PIDS: 7323 seq_ops = &ftrace_no_pid_sops; 7324 break; 7325 default: 7326 trace_array_put(tr); 7327 WARN_ON_ONCE(1); 7328 return -EINVAL; 7329 } 7330 7331 ret = seq_open(file, seq_ops); 7332 if (ret < 0) { 7333 trace_array_put(tr); 7334 } else { 7335 m = file->private_data; 7336 /* copy tr over to seq ops */ 7337 m->private = tr; 7338 } 7339 7340 return ret; 7341 } 7342 7343 static int 7344 ftrace_pid_open(struct inode *inode, struct file *file) 7345 { 7346 return pid_open(inode, file, TRACE_PIDS); 7347 } 7348 7349 static int 7350 ftrace_no_pid_open(struct inode *inode, struct file *file) 7351 { 7352 return pid_open(inode, file, TRACE_NO_PIDS); 7353 } 7354 7355 static void ignore_task_cpu(void *data) 7356 { 7357 struct trace_array *tr = data; 7358 struct trace_pid_list *pid_list; 7359 struct trace_pid_list *no_pid_list; 7360 7361 /* 7362 * This function is called by on_each_cpu() while the 7363 * event_mutex is held. 7364 */ 7365 pid_list = rcu_dereference_protected(tr->function_pids, 7366 mutex_is_locked(&ftrace_lock)); 7367 no_pid_list = rcu_dereference_protected(tr->function_no_pids, 7368 mutex_is_locked(&ftrace_lock)); 7369 7370 if (trace_ignore_this_task(pid_list, no_pid_list, current)) 7371 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, 7372 FTRACE_PID_IGNORE); 7373 else 7374 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, 7375 current->pid); 7376 } 7377 7378 static ssize_t 7379 pid_write(struct file *filp, const char __user *ubuf, 7380 size_t cnt, loff_t *ppos, int type) 7381 { 7382 struct seq_file *m = filp->private_data; 7383 struct trace_array *tr = m->private; 7384 struct trace_pid_list *filtered_pids; 7385 struct trace_pid_list *other_pids; 7386 struct trace_pid_list *pid_list; 7387 ssize_t ret; 7388 7389 if (!cnt) 7390 return 0; 7391 7392 mutex_lock(&ftrace_lock); 7393 7394 switch (type) { 7395 case TRACE_PIDS: 7396 filtered_pids = rcu_dereference_protected(tr->function_pids, 7397 lockdep_is_held(&ftrace_lock)); 7398 other_pids = rcu_dereference_protected(tr->function_no_pids, 7399 lockdep_is_held(&ftrace_lock)); 7400 break; 7401 case TRACE_NO_PIDS: 7402 filtered_pids = rcu_dereference_protected(tr->function_no_pids, 7403 lockdep_is_held(&ftrace_lock)); 7404 other_pids = rcu_dereference_protected(tr->function_pids, 7405 lockdep_is_held(&ftrace_lock)); 7406 break; 7407 default: 7408 ret = -EINVAL; 7409 WARN_ON_ONCE(1); 7410 goto out; 7411 } 7412 7413 ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt); 7414 if (ret < 0) 7415 goto out; 7416 7417 switch (type) { 7418 case TRACE_PIDS: 7419 rcu_assign_pointer(tr->function_pids, pid_list); 7420 break; 7421 case TRACE_NO_PIDS: 7422 rcu_assign_pointer(tr->function_no_pids, pid_list); 7423 break; 7424 } 7425 7426 7427 if (filtered_pids) { 7428 synchronize_rcu(); 7429 trace_free_pid_list(filtered_pids); 7430 } else if (pid_list && !other_pids) { 7431 /* Register a probe to set whether to ignore the tracing of a task */ 7432 register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr); 7433 } 7434 7435 /* 7436 * Ignoring of pids is done at task switch. But we have to 7437 * check for those tasks that are currently running. 7438 * Always do this in case a pid was appended or removed. 7439 */ 7440 on_each_cpu(ignore_task_cpu, tr, 1); 7441 7442 ftrace_update_pid_func(); 7443 ftrace_startup_all(0); 7444 out: 7445 mutex_unlock(&ftrace_lock); 7446 7447 if (ret > 0) 7448 *ppos += ret; 7449 7450 return ret; 7451 } 7452 7453 static ssize_t 7454 ftrace_pid_write(struct file *filp, const char __user *ubuf, 7455 size_t cnt, loff_t *ppos) 7456 { 7457 return pid_write(filp, ubuf, cnt, ppos, TRACE_PIDS); 7458 } 7459 7460 static ssize_t 7461 ftrace_no_pid_write(struct file *filp, const char __user *ubuf, 7462 size_t cnt, loff_t *ppos) 7463 { 7464 return pid_write(filp, ubuf, cnt, ppos, TRACE_NO_PIDS); 7465 } 7466 7467 static int 7468 ftrace_pid_release(struct inode *inode, struct file *file) 7469 { 7470 struct trace_array *tr = inode->i_private; 7471 7472 trace_array_put(tr); 7473 7474 return seq_release(inode, file); 7475 } 7476 7477 static const struct file_operations ftrace_pid_fops = { 7478 .open = ftrace_pid_open, 7479 .write = ftrace_pid_write, 7480 .read = seq_read, 7481 .llseek = tracing_lseek, 7482 .release = ftrace_pid_release, 7483 }; 7484 7485 static const struct file_operations ftrace_no_pid_fops = { 7486 .open = ftrace_no_pid_open, 7487 .write = ftrace_no_pid_write, 7488 .read = seq_read, 7489 .llseek = tracing_lseek, 7490 .release = ftrace_pid_release, 7491 }; 7492 7493 void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer) 7494 { 7495 trace_create_file("set_ftrace_pid", 0644, d_tracer, 7496 tr, &ftrace_pid_fops); 7497 trace_create_file("set_ftrace_notrace_pid", 0644, d_tracer, 7498 tr, &ftrace_no_pid_fops); 7499 } 7500 7501 void __init ftrace_init_tracefs_toplevel(struct trace_array *tr, 7502 struct dentry *d_tracer) 7503 { 7504 /* Only the top level directory has the dyn_tracefs and profile */ 7505 WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL)); 7506 7507 ftrace_init_dyn_tracefs(d_tracer); 7508 ftrace_profile_tracefs(d_tracer); 7509 } 7510 7511 /** 7512 * ftrace_kill - kill ftrace 7513 * 7514 * This function should be used by panic code. It stops ftrace 7515 * but in a not so nice way. If you need to simply kill ftrace 7516 * from a non-atomic section, use ftrace_kill. 7517 */ 7518 void ftrace_kill(void) 7519 { 7520 ftrace_disabled = 1; 7521 ftrace_enabled = 0; 7522 ftrace_trace_function = ftrace_stub; 7523 } 7524 7525 /** 7526 * Test if ftrace is dead or not. 7527 */ 7528 int ftrace_is_dead(void) 7529 { 7530 return ftrace_disabled; 7531 } 7532 7533 /** 7534 * register_ftrace_function - register a function for profiling 7535 * @ops - ops structure that holds the function for profiling. 7536 * 7537 * Register a function to be called by all functions in the 7538 * kernel. 7539 * 7540 * Note: @ops->func and all the functions it calls must be labeled 7541 * with "notrace", otherwise it will go into a 7542 * recursive loop. 7543 */ 7544 int register_ftrace_function(struct ftrace_ops *ops) 7545 { 7546 int ret = -1; 7547 7548 ftrace_ops_init(ops); 7549 7550 mutex_lock(&ftrace_lock); 7551 7552 ret = ftrace_startup(ops, 0); 7553 7554 mutex_unlock(&ftrace_lock); 7555 7556 return ret; 7557 } 7558 EXPORT_SYMBOL_GPL(register_ftrace_function); 7559 7560 /** 7561 * unregister_ftrace_function - unregister a function for profiling. 7562 * @ops - ops structure that holds the function to unregister 7563 * 7564 * Unregister a function that was added to be called by ftrace profiling. 7565 */ 7566 int unregister_ftrace_function(struct ftrace_ops *ops) 7567 { 7568 int ret; 7569 7570 mutex_lock(&ftrace_lock); 7571 ret = ftrace_shutdown(ops, 0); 7572 mutex_unlock(&ftrace_lock); 7573 7574 return ret; 7575 } 7576 EXPORT_SYMBOL_GPL(unregister_ftrace_function); 7577 7578 static bool is_permanent_ops_registered(void) 7579 { 7580 struct ftrace_ops *op; 7581 7582 do_for_each_ftrace_op(op, ftrace_ops_list) { 7583 if (op->flags & FTRACE_OPS_FL_PERMANENT) 7584 return true; 7585 } while_for_each_ftrace_op(op); 7586 7587 return false; 7588 } 7589 7590 int 7591 ftrace_enable_sysctl(struct ctl_table *table, int write, 7592 void *buffer, size_t *lenp, loff_t *ppos) 7593 { 7594 int ret = -ENODEV; 7595 7596 mutex_lock(&ftrace_lock); 7597 7598 if (unlikely(ftrace_disabled)) 7599 goto out; 7600 7601 ret = proc_dointvec(table, write, buffer, lenp, ppos); 7602 7603 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled)) 7604 goto out; 7605 7606 if (ftrace_enabled) { 7607 7608 /* we are starting ftrace again */ 7609 if (rcu_dereference_protected(ftrace_ops_list, 7610 lockdep_is_held(&ftrace_lock)) != &ftrace_list_end) 7611 update_ftrace_function(); 7612 7613 ftrace_startup_sysctl(); 7614 7615 } else { 7616 if (is_permanent_ops_registered()) { 7617 ftrace_enabled = true; 7618 ret = -EBUSY; 7619 goto out; 7620 } 7621 7622 /* stopping ftrace calls (just send to ftrace_stub) */ 7623 ftrace_trace_function = ftrace_stub; 7624 7625 ftrace_shutdown_sysctl(); 7626 } 7627 7628 last_ftrace_enabled = !!ftrace_enabled; 7629 out: 7630 mutex_unlock(&ftrace_lock); 7631 return ret; 7632 } 7633