1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * ring buffer based function tracer 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 7 * 8 * Based on code from the latency_tracer, that is: 9 * 10 * Copyright (C) 2004-2006 Ingo Molnar 11 * Copyright (C) 2004 Nadia Yvette Chambers 12 */ 13 #include <linux/ring_buffer.h> 14 #include <linux/debugfs.h> 15 #include <linux/uaccess.h> 16 #include <linux/ftrace.h> 17 #include <linux/slab.h> 18 #include <linux/fs.h> 19 20 #include "trace.h" 21 22 static void tracing_start_function_trace(struct trace_array *tr); 23 static void tracing_stop_function_trace(struct trace_array *tr); 24 static void 25 function_trace_call(unsigned long ip, unsigned long parent_ip, 26 struct ftrace_ops *op, struct pt_regs *pt_regs); 27 static void 28 function_stack_trace_call(unsigned long ip, unsigned long parent_ip, 29 struct ftrace_ops *op, struct pt_regs *pt_regs); 30 static struct tracer_flags func_flags; 31 32 /* Our option */ 33 enum { 34 TRACE_FUNC_OPT_STACK = 0x1, 35 }; 36 37 static int allocate_ftrace_ops(struct trace_array *tr) 38 { 39 struct ftrace_ops *ops; 40 41 ops = kzalloc(sizeof(*ops), GFP_KERNEL); 42 if (!ops) 43 return -ENOMEM; 44 45 /* Currently only the non stack version is supported */ 46 ops->func = function_trace_call; 47 ops->flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_PID; 48 49 tr->ops = ops; 50 ops->private = tr; 51 return 0; 52 } 53 54 55 int ftrace_create_function_files(struct trace_array *tr, 56 struct dentry *parent) 57 { 58 int ret; 59 60 /* 61 * The top level array uses the "global_ops", and the files are 62 * created on boot up. 63 */ 64 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 65 return 0; 66 67 ret = allocate_ftrace_ops(tr); 68 if (ret) 69 return ret; 70 71 ftrace_create_filter_files(tr->ops, parent); 72 73 return 0; 74 } 75 76 void ftrace_destroy_function_files(struct trace_array *tr) 77 { 78 ftrace_destroy_filter_files(tr->ops); 79 kfree(tr->ops); 80 tr->ops = NULL; 81 } 82 83 static int function_trace_init(struct trace_array *tr) 84 { 85 ftrace_func_t func; 86 87 /* 88 * Instance trace_arrays get their ops allocated 89 * at instance creation. Unless it failed 90 * the allocation. 91 */ 92 if (!tr->ops) 93 return -ENOMEM; 94 95 /* Currently only the global instance can do stack tracing */ 96 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && 97 func_flags.val & TRACE_FUNC_OPT_STACK) 98 func = function_stack_trace_call; 99 else 100 func = function_trace_call; 101 102 ftrace_init_array_ops(tr, func); 103 104 tr->array_buffer.cpu = get_cpu(); 105 put_cpu(); 106 107 tracing_start_cmdline_record(); 108 tracing_start_function_trace(tr); 109 return 0; 110 } 111 112 static void function_trace_reset(struct trace_array *tr) 113 { 114 tracing_stop_function_trace(tr); 115 tracing_stop_cmdline_record(); 116 ftrace_reset_array_ops(tr); 117 } 118 119 static void function_trace_start(struct trace_array *tr) 120 { 121 tracing_reset_online_cpus(&tr->array_buffer); 122 } 123 124 static void 125 function_trace_call(unsigned long ip, unsigned long parent_ip, 126 struct ftrace_ops *op, struct pt_regs *pt_regs) 127 { 128 struct trace_array *tr = op->private; 129 struct trace_array_cpu *data; 130 unsigned long flags; 131 int bit; 132 int cpu; 133 int pc; 134 135 if (unlikely(!tr->function_enabled)) 136 return; 137 138 pc = preempt_count(); 139 preempt_disable_notrace(); 140 141 bit = trace_test_and_set_recursion(TRACE_FTRACE_START, TRACE_FTRACE_MAX); 142 if (bit < 0) 143 goto out; 144 145 cpu = smp_processor_id(); 146 data = per_cpu_ptr(tr->array_buffer.data, cpu); 147 if (!atomic_read(&data->disabled)) { 148 local_save_flags(flags); 149 trace_function(tr, ip, parent_ip, flags, pc); 150 } 151 trace_clear_recursion(bit); 152 153 out: 154 preempt_enable_notrace(); 155 } 156 157 #ifdef CONFIG_UNWINDER_ORC 158 /* 159 * Skip 2: 160 * 161 * function_stack_trace_call() 162 * ftrace_call() 163 */ 164 #define STACK_SKIP 2 165 #else 166 /* 167 * Skip 3: 168 * __trace_stack() 169 * function_stack_trace_call() 170 * ftrace_call() 171 */ 172 #define STACK_SKIP 3 173 #endif 174 175 static void 176 function_stack_trace_call(unsigned long ip, unsigned long parent_ip, 177 struct ftrace_ops *op, struct pt_regs *pt_regs) 178 { 179 struct trace_array *tr = op->private; 180 struct trace_array_cpu *data; 181 unsigned long flags; 182 long disabled; 183 int cpu; 184 int pc; 185 186 if (unlikely(!tr->function_enabled)) 187 return; 188 189 /* 190 * Need to use raw, since this must be called before the 191 * recursive protection is performed. 192 */ 193 local_irq_save(flags); 194 cpu = raw_smp_processor_id(); 195 data = per_cpu_ptr(tr->array_buffer.data, cpu); 196 disabled = atomic_inc_return(&data->disabled); 197 198 if (likely(disabled == 1)) { 199 pc = preempt_count(); 200 trace_function(tr, ip, parent_ip, flags, pc); 201 __trace_stack(tr, flags, STACK_SKIP, pc); 202 } 203 204 atomic_dec(&data->disabled); 205 local_irq_restore(flags); 206 } 207 208 static struct tracer_opt func_opts[] = { 209 #ifdef CONFIG_STACKTRACE 210 { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) }, 211 #endif 212 { } /* Always set a last empty entry */ 213 }; 214 215 static struct tracer_flags func_flags = { 216 .val = 0, /* By default: all flags disabled */ 217 .opts = func_opts 218 }; 219 220 static void tracing_start_function_trace(struct trace_array *tr) 221 { 222 tr->function_enabled = 0; 223 register_ftrace_function(tr->ops); 224 tr->function_enabled = 1; 225 } 226 227 static void tracing_stop_function_trace(struct trace_array *tr) 228 { 229 tr->function_enabled = 0; 230 unregister_ftrace_function(tr->ops); 231 } 232 233 static struct tracer function_trace; 234 235 static int 236 func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 237 { 238 switch (bit) { 239 case TRACE_FUNC_OPT_STACK: 240 /* do nothing if already set */ 241 if (!!set == !!(func_flags.val & TRACE_FUNC_OPT_STACK)) 242 break; 243 244 /* We can change this flag when not running. */ 245 if (tr->current_trace != &function_trace) 246 break; 247 248 unregister_ftrace_function(tr->ops); 249 250 if (set) { 251 tr->ops->func = function_stack_trace_call; 252 register_ftrace_function(tr->ops); 253 } else { 254 tr->ops->func = function_trace_call; 255 register_ftrace_function(tr->ops); 256 } 257 258 break; 259 default: 260 return -EINVAL; 261 } 262 263 return 0; 264 } 265 266 static struct tracer function_trace __tracer_data = 267 { 268 .name = "function", 269 .init = function_trace_init, 270 .reset = function_trace_reset, 271 .start = function_trace_start, 272 .flags = &func_flags, 273 .set_flag = func_set_flag, 274 .allow_instances = true, 275 #ifdef CONFIG_FTRACE_SELFTEST 276 .selftest = trace_selftest_startup_function, 277 #endif 278 }; 279 280 #ifdef CONFIG_DYNAMIC_FTRACE 281 static void update_traceon_count(struct ftrace_probe_ops *ops, 282 unsigned long ip, 283 struct trace_array *tr, bool on, 284 void *data) 285 { 286 struct ftrace_func_mapper *mapper = data; 287 long *count; 288 long old_count; 289 290 /* 291 * Tracing gets disabled (or enabled) once per count. 292 * This function can be called at the same time on multiple CPUs. 293 * It is fine if both disable (or enable) tracing, as disabling 294 * (or enabling) the second time doesn't do anything as the 295 * state of the tracer is already disabled (or enabled). 296 * What needs to be synchronized in this case is that the count 297 * only gets decremented once, even if the tracer is disabled 298 * (or enabled) twice, as the second one is really a nop. 299 * 300 * The memory barriers guarantee that we only decrement the 301 * counter once. First the count is read to a local variable 302 * and a read barrier is used to make sure that it is loaded 303 * before checking if the tracer is in the state we want. 304 * If the tracer is not in the state we want, then the count 305 * is guaranteed to be the old count. 306 * 307 * Next the tracer is set to the state we want (disabled or enabled) 308 * then a write memory barrier is used to make sure that 309 * the new state is visible before changing the counter by 310 * one minus the old counter. This guarantees that another CPU 311 * executing this code will see the new state before seeing 312 * the new counter value, and would not do anything if the new 313 * counter is seen. 314 * 315 * Note, there is no synchronization between this and a user 316 * setting the tracing_on file. But we currently don't care 317 * about that. 318 */ 319 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 320 old_count = *count; 321 322 if (old_count <= 0) 323 return; 324 325 /* Make sure we see count before checking tracing state */ 326 smp_rmb(); 327 328 if (on == !!tracer_tracing_is_on(tr)) 329 return; 330 331 if (on) 332 tracer_tracing_on(tr); 333 else 334 tracer_tracing_off(tr); 335 336 /* Make sure tracing state is visible before updating count */ 337 smp_wmb(); 338 339 *count = old_count - 1; 340 } 341 342 static void 343 ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, 344 struct trace_array *tr, struct ftrace_probe_ops *ops, 345 void *data) 346 { 347 update_traceon_count(ops, ip, tr, 1, data); 348 } 349 350 static void 351 ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, 352 struct trace_array *tr, struct ftrace_probe_ops *ops, 353 void *data) 354 { 355 update_traceon_count(ops, ip, tr, 0, data); 356 } 357 358 static void 359 ftrace_traceon(unsigned long ip, unsigned long parent_ip, 360 struct trace_array *tr, struct ftrace_probe_ops *ops, 361 void *data) 362 { 363 if (tracer_tracing_is_on(tr)) 364 return; 365 366 tracer_tracing_on(tr); 367 } 368 369 static void 370 ftrace_traceoff(unsigned long ip, unsigned long parent_ip, 371 struct trace_array *tr, struct ftrace_probe_ops *ops, 372 void *data) 373 { 374 if (!tracer_tracing_is_on(tr)) 375 return; 376 377 tracer_tracing_off(tr); 378 } 379 380 #ifdef CONFIG_UNWINDER_ORC 381 /* 382 * Skip 3: 383 * 384 * function_trace_probe_call() 385 * ftrace_ops_assist_func() 386 * ftrace_call() 387 */ 388 #define FTRACE_STACK_SKIP 3 389 #else 390 /* 391 * Skip 5: 392 * 393 * __trace_stack() 394 * ftrace_stacktrace() 395 * function_trace_probe_call() 396 * ftrace_ops_assist_func() 397 * ftrace_call() 398 */ 399 #define FTRACE_STACK_SKIP 5 400 #endif 401 402 static __always_inline void trace_stack(struct trace_array *tr) 403 { 404 unsigned long flags; 405 int pc; 406 407 local_save_flags(flags); 408 pc = preempt_count(); 409 410 __trace_stack(tr, flags, FTRACE_STACK_SKIP, pc); 411 } 412 413 static void 414 ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, 415 struct trace_array *tr, struct ftrace_probe_ops *ops, 416 void *data) 417 { 418 trace_stack(tr); 419 } 420 421 static void 422 ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, 423 struct trace_array *tr, struct ftrace_probe_ops *ops, 424 void *data) 425 { 426 struct ftrace_func_mapper *mapper = data; 427 long *count; 428 long old_count; 429 long new_count; 430 431 if (!tracing_is_on()) 432 return; 433 434 /* unlimited? */ 435 if (!mapper) { 436 trace_stack(tr); 437 return; 438 } 439 440 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 441 442 /* 443 * Stack traces should only execute the number of times the 444 * user specified in the counter. 445 */ 446 do { 447 old_count = *count; 448 449 if (!old_count) 450 return; 451 452 new_count = old_count - 1; 453 new_count = cmpxchg(count, old_count, new_count); 454 if (new_count == old_count) 455 trace_stack(tr); 456 457 if (!tracing_is_on()) 458 return; 459 460 } while (new_count != old_count); 461 } 462 463 static int update_count(struct ftrace_probe_ops *ops, unsigned long ip, 464 void *data) 465 { 466 struct ftrace_func_mapper *mapper = data; 467 long *count = NULL; 468 469 if (mapper) 470 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 471 472 if (count) { 473 if (*count <= 0) 474 return 0; 475 (*count)--; 476 } 477 478 return 1; 479 } 480 481 static void 482 ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, 483 struct trace_array *tr, struct ftrace_probe_ops *ops, 484 void *data) 485 { 486 if (update_count(ops, ip, data)) 487 ftrace_dump(DUMP_ALL); 488 } 489 490 /* Only dump the current CPU buffer. */ 491 static void 492 ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, 493 struct trace_array *tr, struct ftrace_probe_ops *ops, 494 void *data) 495 { 496 if (update_count(ops, ip, data)) 497 ftrace_dump(DUMP_ORIG); 498 } 499 500 static int 501 ftrace_probe_print(const char *name, struct seq_file *m, 502 unsigned long ip, struct ftrace_probe_ops *ops, 503 void *data) 504 { 505 struct ftrace_func_mapper *mapper = data; 506 long *count = NULL; 507 508 seq_printf(m, "%ps:%s", (void *)ip, name); 509 510 if (mapper) 511 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 512 513 if (count) 514 seq_printf(m, ":count=%ld\n", *count); 515 else 516 seq_puts(m, ":unlimited\n"); 517 518 return 0; 519 } 520 521 static int 522 ftrace_traceon_print(struct seq_file *m, unsigned long ip, 523 struct ftrace_probe_ops *ops, 524 void *data) 525 { 526 return ftrace_probe_print("traceon", m, ip, ops, data); 527 } 528 529 static int 530 ftrace_traceoff_print(struct seq_file *m, unsigned long ip, 531 struct ftrace_probe_ops *ops, void *data) 532 { 533 return ftrace_probe_print("traceoff", m, ip, ops, data); 534 } 535 536 static int 537 ftrace_stacktrace_print(struct seq_file *m, unsigned long ip, 538 struct ftrace_probe_ops *ops, void *data) 539 { 540 return ftrace_probe_print("stacktrace", m, ip, ops, data); 541 } 542 543 static int 544 ftrace_dump_print(struct seq_file *m, unsigned long ip, 545 struct ftrace_probe_ops *ops, void *data) 546 { 547 return ftrace_probe_print("dump", m, ip, ops, data); 548 } 549 550 static int 551 ftrace_cpudump_print(struct seq_file *m, unsigned long ip, 552 struct ftrace_probe_ops *ops, void *data) 553 { 554 return ftrace_probe_print("cpudump", m, ip, ops, data); 555 } 556 557 558 static int 559 ftrace_count_init(struct ftrace_probe_ops *ops, struct trace_array *tr, 560 unsigned long ip, void *init_data, void **data) 561 { 562 struct ftrace_func_mapper *mapper = *data; 563 564 if (!mapper) { 565 mapper = allocate_ftrace_func_mapper(); 566 if (!mapper) 567 return -ENOMEM; 568 *data = mapper; 569 } 570 571 return ftrace_func_mapper_add_ip(mapper, ip, init_data); 572 } 573 574 static void 575 ftrace_count_free(struct ftrace_probe_ops *ops, struct trace_array *tr, 576 unsigned long ip, void *data) 577 { 578 struct ftrace_func_mapper *mapper = data; 579 580 if (!ip) { 581 free_ftrace_func_mapper(mapper, NULL); 582 return; 583 } 584 585 ftrace_func_mapper_remove_ip(mapper, ip); 586 } 587 588 static struct ftrace_probe_ops traceon_count_probe_ops = { 589 .func = ftrace_traceon_count, 590 .print = ftrace_traceon_print, 591 .init = ftrace_count_init, 592 .free = ftrace_count_free, 593 }; 594 595 static struct ftrace_probe_ops traceoff_count_probe_ops = { 596 .func = ftrace_traceoff_count, 597 .print = ftrace_traceoff_print, 598 .init = ftrace_count_init, 599 .free = ftrace_count_free, 600 }; 601 602 static struct ftrace_probe_ops stacktrace_count_probe_ops = { 603 .func = ftrace_stacktrace_count, 604 .print = ftrace_stacktrace_print, 605 .init = ftrace_count_init, 606 .free = ftrace_count_free, 607 }; 608 609 static struct ftrace_probe_ops dump_probe_ops = { 610 .func = ftrace_dump_probe, 611 .print = ftrace_dump_print, 612 .init = ftrace_count_init, 613 .free = ftrace_count_free, 614 }; 615 616 static struct ftrace_probe_ops cpudump_probe_ops = { 617 .func = ftrace_cpudump_probe, 618 .print = ftrace_cpudump_print, 619 }; 620 621 static struct ftrace_probe_ops traceon_probe_ops = { 622 .func = ftrace_traceon, 623 .print = ftrace_traceon_print, 624 }; 625 626 static struct ftrace_probe_ops traceoff_probe_ops = { 627 .func = ftrace_traceoff, 628 .print = ftrace_traceoff_print, 629 }; 630 631 static struct ftrace_probe_ops stacktrace_probe_ops = { 632 .func = ftrace_stacktrace, 633 .print = ftrace_stacktrace_print, 634 }; 635 636 static int 637 ftrace_trace_probe_callback(struct trace_array *tr, 638 struct ftrace_probe_ops *ops, 639 struct ftrace_hash *hash, char *glob, 640 char *cmd, char *param, int enable) 641 { 642 void *count = (void *)-1; 643 char *number; 644 int ret; 645 646 /* hash funcs only work with set_ftrace_filter */ 647 if (!enable) 648 return -EINVAL; 649 650 if (glob[0] == '!') 651 return unregister_ftrace_function_probe_func(glob+1, tr, ops); 652 653 if (!param) 654 goto out_reg; 655 656 number = strsep(¶m, ":"); 657 658 if (!strlen(number)) 659 goto out_reg; 660 661 /* 662 * We use the callback data field (which is a pointer) 663 * as our counter. 664 */ 665 ret = kstrtoul(number, 0, (unsigned long *)&count); 666 if (ret) 667 return ret; 668 669 out_reg: 670 ret = register_ftrace_function_probe(glob, tr, ops, count); 671 672 return ret < 0 ? ret : 0; 673 } 674 675 static int 676 ftrace_trace_onoff_callback(struct trace_array *tr, struct ftrace_hash *hash, 677 char *glob, char *cmd, char *param, int enable) 678 { 679 struct ftrace_probe_ops *ops; 680 681 if (!tr) 682 return -ENODEV; 683 684 /* we register both traceon and traceoff to this callback */ 685 if (strcmp(cmd, "traceon") == 0) 686 ops = param ? &traceon_count_probe_ops : &traceon_probe_ops; 687 else 688 ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops; 689 690 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 691 param, enable); 692 } 693 694 static int 695 ftrace_stacktrace_callback(struct trace_array *tr, struct ftrace_hash *hash, 696 char *glob, char *cmd, char *param, int enable) 697 { 698 struct ftrace_probe_ops *ops; 699 700 if (!tr) 701 return -ENODEV; 702 703 ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops; 704 705 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 706 param, enable); 707 } 708 709 static int 710 ftrace_dump_callback(struct trace_array *tr, struct ftrace_hash *hash, 711 char *glob, char *cmd, char *param, int enable) 712 { 713 struct ftrace_probe_ops *ops; 714 715 if (!tr) 716 return -ENODEV; 717 718 ops = &dump_probe_ops; 719 720 /* Only dump once. */ 721 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 722 "1", enable); 723 } 724 725 static int 726 ftrace_cpudump_callback(struct trace_array *tr, struct ftrace_hash *hash, 727 char *glob, char *cmd, char *param, int enable) 728 { 729 struct ftrace_probe_ops *ops; 730 731 if (!tr) 732 return -ENODEV; 733 734 ops = &cpudump_probe_ops; 735 736 /* Only dump once. */ 737 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 738 "1", enable); 739 } 740 741 static struct ftrace_func_command ftrace_traceon_cmd = { 742 .name = "traceon", 743 .func = ftrace_trace_onoff_callback, 744 }; 745 746 static struct ftrace_func_command ftrace_traceoff_cmd = { 747 .name = "traceoff", 748 .func = ftrace_trace_onoff_callback, 749 }; 750 751 static struct ftrace_func_command ftrace_stacktrace_cmd = { 752 .name = "stacktrace", 753 .func = ftrace_stacktrace_callback, 754 }; 755 756 static struct ftrace_func_command ftrace_dump_cmd = { 757 .name = "dump", 758 .func = ftrace_dump_callback, 759 }; 760 761 static struct ftrace_func_command ftrace_cpudump_cmd = { 762 .name = "cpudump", 763 .func = ftrace_cpudump_callback, 764 }; 765 766 static int __init init_func_cmd_traceon(void) 767 { 768 int ret; 769 770 ret = register_ftrace_command(&ftrace_traceoff_cmd); 771 if (ret) 772 return ret; 773 774 ret = register_ftrace_command(&ftrace_traceon_cmd); 775 if (ret) 776 goto out_free_traceoff; 777 778 ret = register_ftrace_command(&ftrace_stacktrace_cmd); 779 if (ret) 780 goto out_free_traceon; 781 782 ret = register_ftrace_command(&ftrace_dump_cmd); 783 if (ret) 784 goto out_free_stacktrace; 785 786 ret = register_ftrace_command(&ftrace_cpudump_cmd); 787 if (ret) 788 goto out_free_dump; 789 790 return 0; 791 792 out_free_dump: 793 unregister_ftrace_command(&ftrace_dump_cmd); 794 out_free_stacktrace: 795 unregister_ftrace_command(&ftrace_stacktrace_cmd); 796 out_free_traceon: 797 unregister_ftrace_command(&ftrace_traceon_cmd); 798 out_free_traceoff: 799 unregister_ftrace_command(&ftrace_traceoff_cmd); 800 801 return ret; 802 } 803 #else 804 static inline int init_func_cmd_traceon(void) 805 { 806 return 0; 807 } 808 #endif /* CONFIG_DYNAMIC_FTRACE */ 809 810 __init int init_function_trace(void) 811 { 812 init_func_cmd_traceon(); 813 return register_tracer(&function_trace); 814 } 815