1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * ring buffer based function tracer 4 * 5 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com> 6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 7 * 8 * Originally taken from the RT patch by: 9 * Arnaldo Carvalho de Melo <acme@redhat.com> 10 * 11 * Based on code from the latency_tracer, that is: 12 * Copyright (C) 2004-2006 Ingo Molnar 13 * Copyright (C) 2004 Nadia Yvette Chambers 14 */ 15 #include <linux/ring_buffer.h> 16 #include <generated/utsrelease.h> 17 #include <linux/stacktrace.h> 18 #include <linux/writeback.h> 19 #include <linux/kallsyms.h> 20 #include <linux/security.h> 21 #include <linux/seq_file.h> 22 #include <linux/notifier.h> 23 #include <linux/irqflags.h> 24 #include <linux/debugfs.h> 25 #include <linux/tracefs.h> 26 #include <linux/pagemap.h> 27 #include <linux/hardirq.h> 28 #include <linux/linkage.h> 29 #include <linux/uaccess.h> 30 #include <linux/vmalloc.h> 31 #include <linux/ftrace.h> 32 #include <linux/module.h> 33 #include <linux/percpu.h> 34 #include <linux/splice.h> 35 #include <linux/kdebug.h> 36 #include <linux/string.h> 37 #include <linux/mount.h> 38 #include <linux/rwsem.h> 39 #include <linux/slab.h> 40 #include <linux/ctype.h> 41 #include <linux/init.h> 42 #include <linux/poll.h> 43 #include <linux/nmi.h> 44 #include <linux/fs.h> 45 #include <linux/trace.h> 46 #include <linux/sched/clock.h> 47 #include <linux/sched/rt.h> 48 #include <linux/fsnotify.h> 49 #include <linux/irq_work.h> 50 #include <linux/workqueue.h> 51 52 #include "trace.h" 53 #include "trace_output.h" 54 55 /* 56 * On boot up, the ring buffer is set to the minimum size, so that 57 * we do not waste memory on systems that are not using tracing. 58 */ 59 bool ring_buffer_expanded; 60 61 /* 62 * We need to change this state when a selftest is running. 63 * A selftest will lurk into the ring-buffer to count the 64 * entries inserted during the selftest although some concurrent 65 * insertions into the ring-buffer such as trace_printk could occurred 66 * at the same time, giving false positive or negative results. 67 */ 68 static bool __read_mostly tracing_selftest_running; 69 70 /* 71 * If a tracer is running, we do not want to run SELFTEST. 72 */ 73 bool __read_mostly tracing_selftest_disabled; 74 75 /* Pipe tracepoints to printk */ 76 struct trace_iterator *tracepoint_print_iter; 77 int tracepoint_printk; 78 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key); 79 80 /* For tracers that don't implement custom flags */ 81 static struct tracer_opt dummy_tracer_opt[] = { 82 { } 83 }; 84 85 static int 86 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 87 { 88 return 0; 89 } 90 91 /* 92 * To prevent the comm cache from being overwritten when no 93 * tracing is active, only save the comm when a trace event 94 * occurred. 95 */ 96 static DEFINE_PER_CPU(bool, trace_taskinfo_save); 97 98 /* 99 * Kill all tracing for good (never come back). 100 * It is initialized to 1 but will turn to zero if the initialization 101 * of the tracer is successful. But that is the only place that sets 102 * this back to zero. 103 */ 104 static int tracing_disabled = 1; 105 106 cpumask_var_t __read_mostly tracing_buffer_mask; 107 108 /* 109 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops 110 * 111 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops 112 * is set, then ftrace_dump is called. This will output the contents 113 * of the ftrace buffers to the console. This is very useful for 114 * capturing traces that lead to crashes and outputing it to a 115 * serial console. 116 * 117 * It is default off, but you can enable it with either specifying 118 * "ftrace_dump_on_oops" in the kernel command line, or setting 119 * /proc/sys/kernel/ftrace_dump_on_oops 120 * Set 1 if you want to dump buffers of all CPUs 121 * Set 2 if you want to dump the buffer of the CPU that triggered oops 122 */ 123 124 enum ftrace_dump_mode ftrace_dump_on_oops; 125 126 /* When set, tracing will stop when a WARN*() is hit */ 127 int __disable_trace_on_warning; 128 129 #ifdef CONFIG_TRACE_EVAL_MAP_FILE 130 /* Map of enums to their values, for "eval_map" file */ 131 struct trace_eval_map_head { 132 struct module *mod; 133 unsigned long length; 134 }; 135 136 union trace_eval_map_item; 137 138 struct trace_eval_map_tail { 139 /* 140 * "end" is first and points to NULL as it must be different 141 * than "mod" or "eval_string" 142 */ 143 union trace_eval_map_item *next; 144 const char *end; /* points to NULL */ 145 }; 146 147 static DEFINE_MUTEX(trace_eval_mutex); 148 149 /* 150 * The trace_eval_maps are saved in an array with two extra elements, 151 * one at the beginning, and one at the end. The beginning item contains 152 * the count of the saved maps (head.length), and the module they 153 * belong to if not built in (head.mod). The ending item contains a 154 * pointer to the next array of saved eval_map items. 155 */ 156 union trace_eval_map_item { 157 struct trace_eval_map map; 158 struct trace_eval_map_head head; 159 struct trace_eval_map_tail tail; 160 }; 161 162 static union trace_eval_map_item *trace_eval_maps; 163 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */ 164 165 int tracing_set_tracer(struct trace_array *tr, const char *buf); 166 static void ftrace_trace_userstack(struct trace_buffer *buffer, 167 unsigned long flags, int pc); 168 169 #define MAX_TRACER_SIZE 100 170 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; 171 static char *default_bootup_tracer; 172 173 static bool allocate_snapshot; 174 175 static int __init set_cmdline_ftrace(char *str) 176 { 177 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE); 178 default_bootup_tracer = bootup_tracer_buf; 179 /* We are using ftrace early, expand it */ 180 ring_buffer_expanded = true; 181 return 1; 182 } 183 __setup("ftrace=", set_cmdline_ftrace); 184 185 static int __init set_ftrace_dump_on_oops(char *str) 186 { 187 if (*str++ != '=' || !*str) { 188 ftrace_dump_on_oops = DUMP_ALL; 189 return 1; 190 } 191 192 if (!strcmp("orig_cpu", str)) { 193 ftrace_dump_on_oops = DUMP_ORIG; 194 return 1; 195 } 196 197 return 0; 198 } 199 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops); 200 201 static int __init stop_trace_on_warning(char *str) 202 { 203 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0)) 204 __disable_trace_on_warning = 1; 205 return 1; 206 } 207 __setup("traceoff_on_warning", stop_trace_on_warning); 208 209 static int __init boot_alloc_snapshot(char *str) 210 { 211 allocate_snapshot = true; 212 /* We also need the main ring buffer expanded */ 213 ring_buffer_expanded = true; 214 return 1; 215 } 216 __setup("alloc_snapshot", boot_alloc_snapshot); 217 218 219 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata; 220 221 static int __init set_trace_boot_options(char *str) 222 { 223 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE); 224 return 0; 225 } 226 __setup("trace_options=", set_trace_boot_options); 227 228 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata; 229 static char *trace_boot_clock __initdata; 230 231 static int __init set_trace_boot_clock(char *str) 232 { 233 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE); 234 trace_boot_clock = trace_boot_clock_buf; 235 return 0; 236 } 237 __setup("trace_clock=", set_trace_boot_clock); 238 239 static int __init set_tracepoint_printk(char *str) 240 { 241 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0)) 242 tracepoint_printk = 1; 243 return 1; 244 } 245 __setup("tp_printk", set_tracepoint_printk); 246 247 unsigned long long ns2usecs(u64 nsec) 248 { 249 nsec += 500; 250 do_div(nsec, 1000); 251 return nsec; 252 } 253 254 /* trace_flags holds trace_options default values */ 255 #define TRACE_DEFAULT_FLAGS \ 256 (FUNCTION_DEFAULT_FLAGS | \ 257 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \ 258 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \ 259 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \ 260 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS) 261 262 /* trace_options that are only supported by global_trace */ 263 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \ 264 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD) 265 266 /* trace_flags that are default zero for instances */ 267 #define ZEROED_TRACE_FLAGS \ 268 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK) 269 270 /* 271 * The global_trace is the descriptor that holds the top-level tracing 272 * buffers for the live tracing. 273 */ 274 static struct trace_array global_trace = { 275 .trace_flags = TRACE_DEFAULT_FLAGS, 276 }; 277 278 LIST_HEAD(ftrace_trace_arrays); 279 280 int trace_array_get(struct trace_array *this_tr) 281 { 282 struct trace_array *tr; 283 int ret = -ENODEV; 284 285 mutex_lock(&trace_types_lock); 286 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 287 if (tr == this_tr) { 288 tr->ref++; 289 ret = 0; 290 break; 291 } 292 } 293 mutex_unlock(&trace_types_lock); 294 295 return ret; 296 } 297 298 static void __trace_array_put(struct trace_array *this_tr) 299 { 300 WARN_ON(!this_tr->ref); 301 this_tr->ref--; 302 } 303 304 /** 305 * trace_array_put - Decrement the reference counter for this trace array. 306 * 307 * NOTE: Use this when we no longer need the trace array returned by 308 * trace_array_get_by_name(). This ensures the trace array can be later 309 * destroyed. 310 * 311 */ 312 void trace_array_put(struct trace_array *this_tr) 313 { 314 if (!this_tr) 315 return; 316 317 mutex_lock(&trace_types_lock); 318 __trace_array_put(this_tr); 319 mutex_unlock(&trace_types_lock); 320 } 321 EXPORT_SYMBOL_GPL(trace_array_put); 322 323 int tracing_check_open_get_tr(struct trace_array *tr) 324 { 325 int ret; 326 327 ret = security_locked_down(LOCKDOWN_TRACEFS); 328 if (ret) 329 return ret; 330 331 if (tracing_disabled) 332 return -ENODEV; 333 334 if (tr && trace_array_get(tr) < 0) 335 return -ENODEV; 336 337 return 0; 338 } 339 340 int call_filter_check_discard(struct trace_event_call *call, void *rec, 341 struct trace_buffer *buffer, 342 struct ring_buffer_event *event) 343 { 344 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) && 345 !filter_match_preds(call->filter, rec)) { 346 __trace_event_discard_commit(buffer, event); 347 return 1; 348 } 349 350 return 0; 351 } 352 353 void trace_free_pid_list(struct trace_pid_list *pid_list) 354 { 355 vfree(pid_list->pids); 356 kfree(pid_list); 357 } 358 359 /** 360 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list 361 * @filtered_pids: The list of pids to check 362 * @search_pid: The PID to find in @filtered_pids 363 * 364 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis. 365 */ 366 bool 367 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid) 368 { 369 /* 370 * If pid_max changed after filtered_pids was created, we 371 * by default ignore all pids greater than the previous pid_max. 372 */ 373 if (search_pid >= filtered_pids->pid_max) 374 return false; 375 376 return test_bit(search_pid, filtered_pids->pids); 377 } 378 379 /** 380 * trace_ignore_this_task - should a task be ignored for tracing 381 * @filtered_pids: The list of pids to check 382 * @task: The task that should be ignored if not filtered 383 * 384 * Checks if @task should be traced or not from @filtered_pids. 385 * Returns true if @task should *NOT* be traced. 386 * Returns false if @task should be traced. 387 */ 388 bool 389 trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task) 390 { 391 /* 392 * Return false, because if filtered_pids does not exist, 393 * all pids are good to trace. 394 */ 395 if (!filtered_pids) 396 return false; 397 398 return !trace_find_filtered_pid(filtered_pids, task->pid); 399 } 400 401 /** 402 * trace_filter_add_remove_task - Add or remove a task from a pid_list 403 * @pid_list: The list to modify 404 * @self: The current task for fork or NULL for exit 405 * @task: The task to add or remove 406 * 407 * If adding a task, if @self is defined, the task is only added if @self 408 * is also included in @pid_list. This happens on fork and tasks should 409 * only be added when the parent is listed. If @self is NULL, then the 410 * @task pid will be removed from the list, which would happen on exit 411 * of a task. 412 */ 413 void trace_filter_add_remove_task(struct trace_pid_list *pid_list, 414 struct task_struct *self, 415 struct task_struct *task) 416 { 417 if (!pid_list) 418 return; 419 420 /* For forks, we only add if the forking task is listed */ 421 if (self) { 422 if (!trace_find_filtered_pid(pid_list, self->pid)) 423 return; 424 } 425 426 /* Sorry, but we don't support pid_max changing after setting */ 427 if (task->pid >= pid_list->pid_max) 428 return; 429 430 /* "self" is set for forks, and NULL for exits */ 431 if (self) 432 set_bit(task->pid, pid_list->pids); 433 else 434 clear_bit(task->pid, pid_list->pids); 435 } 436 437 /** 438 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list 439 * @pid_list: The pid list to show 440 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed) 441 * @pos: The position of the file 442 * 443 * This is used by the seq_file "next" operation to iterate the pids 444 * listed in a trace_pid_list structure. 445 * 446 * Returns the pid+1 as we want to display pid of zero, but NULL would 447 * stop the iteration. 448 */ 449 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos) 450 { 451 unsigned long pid = (unsigned long)v; 452 453 (*pos)++; 454 455 /* pid already is +1 of the actual prevous bit */ 456 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid); 457 458 /* Return pid + 1 to allow zero to be represented */ 459 if (pid < pid_list->pid_max) 460 return (void *)(pid + 1); 461 462 return NULL; 463 } 464 465 /** 466 * trace_pid_start - Used for seq_file to start reading pid lists 467 * @pid_list: The pid list to show 468 * @pos: The position of the file 469 * 470 * This is used by seq_file "start" operation to start the iteration 471 * of listing pids. 472 * 473 * Returns the pid+1 as we want to display pid of zero, but NULL would 474 * stop the iteration. 475 */ 476 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos) 477 { 478 unsigned long pid; 479 loff_t l = 0; 480 481 pid = find_first_bit(pid_list->pids, pid_list->pid_max); 482 if (pid >= pid_list->pid_max) 483 return NULL; 484 485 /* Return pid + 1 so that zero can be the exit value */ 486 for (pid++; pid && l < *pos; 487 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l)) 488 ; 489 return (void *)pid; 490 } 491 492 /** 493 * trace_pid_show - show the current pid in seq_file processing 494 * @m: The seq_file structure to write into 495 * @v: A void pointer of the pid (+1) value to display 496 * 497 * Can be directly used by seq_file operations to display the current 498 * pid value. 499 */ 500 int trace_pid_show(struct seq_file *m, void *v) 501 { 502 unsigned long pid = (unsigned long)v - 1; 503 504 seq_printf(m, "%lu\n", pid); 505 return 0; 506 } 507 508 /* 128 should be much more than enough */ 509 #define PID_BUF_SIZE 127 510 511 int trace_pid_write(struct trace_pid_list *filtered_pids, 512 struct trace_pid_list **new_pid_list, 513 const char __user *ubuf, size_t cnt) 514 { 515 struct trace_pid_list *pid_list; 516 struct trace_parser parser; 517 unsigned long val; 518 int nr_pids = 0; 519 ssize_t read = 0; 520 ssize_t ret = 0; 521 loff_t pos; 522 pid_t pid; 523 524 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1)) 525 return -ENOMEM; 526 527 /* 528 * Always recreate a new array. The write is an all or nothing 529 * operation. Always create a new array when adding new pids by 530 * the user. If the operation fails, then the current list is 531 * not modified. 532 */ 533 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL); 534 if (!pid_list) { 535 trace_parser_put(&parser); 536 return -ENOMEM; 537 } 538 539 pid_list->pid_max = READ_ONCE(pid_max); 540 541 /* Only truncating will shrink pid_max */ 542 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max) 543 pid_list->pid_max = filtered_pids->pid_max; 544 545 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3); 546 if (!pid_list->pids) { 547 trace_parser_put(&parser); 548 kfree(pid_list); 549 return -ENOMEM; 550 } 551 552 if (filtered_pids) { 553 /* copy the current bits to the new max */ 554 for_each_set_bit(pid, filtered_pids->pids, 555 filtered_pids->pid_max) { 556 set_bit(pid, pid_list->pids); 557 nr_pids++; 558 } 559 } 560 561 while (cnt > 0) { 562 563 pos = 0; 564 565 ret = trace_get_user(&parser, ubuf, cnt, &pos); 566 if (ret < 0 || !trace_parser_loaded(&parser)) 567 break; 568 569 read += ret; 570 ubuf += ret; 571 cnt -= ret; 572 573 ret = -EINVAL; 574 if (kstrtoul(parser.buffer, 0, &val)) 575 break; 576 if (val >= pid_list->pid_max) 577 break; 578 579 pid = (pid_t)val; 580 581 set_bit(pid, pid_list->pids); 582 nr_pids++; 583 584 trace_parser_clear(&parser); 585 ret = 0; 586 } 587 trace_parser_put(&parser); 588 589 if (ret < 0) { 590 trace_free_pid_list(pid_list); 591 return ret; 592 } 593 594 if (!nr_pids) { 595 /* Cleared the list of pids */ 596 trace_free_pid_list(pid_list); 597 read = ret; 598 pid_list = NULL; 599 } 600 601 *new_pid_list = pid_list; 602 603 return read; 604 } 605 606 static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu) 607 { 608 u64 ts; 609 610 /* Early boot up does not have a buffer yet */ 611 if (!buf->buffer) 612 return trace_clock_local(); 613 614 ts = ring_buffer_time_stamp(buf->buffer, cpu); 615 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts); 616 617 return ts; 618 } 619 620 u64 ftrace_now(int cpu) 621 { 622 return buffer_ftrace_now(&global_trace.array_buffer, cpu); 623 } 624 625 /** 626 * tracing_is_enabled - Show if global_trace has been disabled 627 * 628 * Shows if the global trace has been enabled or not. It uses the 629 * mirror flag "buffer_disabled" to be used in fast paths such as for 630 * the irqsoff tracer. But it may be inaccurate due to races. If you 631 * need to know the accurate state, use tracing_is_on() which is a little 632 * slower, but accurate. 633 */ 634 int tracing_is_enabled(void) 635 { 636 /* 637 * For quick access (irqsoff uses this in fast path), just 638 * return the mirror variable of the state of the ring buffer. 639 * It's a little racy, but we don't really care. 640 */ 641 smp_rmb(); 642 return !global_trace.buffer_disabled; 643 } 644 645 /* 646 * trace_buf_size is the size in bytes that is allocated 647 * for a buffer. Note, the number of bytes is always rounded 648 * to page size. 649 * 650 * This number is purposely set to a low number of 16384. 651 * If the dump on oops happens, it will be much appreciated 652 * to not have to wait for all that output. Anyway this can be 653 * boot time and run time configurable. 654 */ 655 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */ 656 657 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT; 658 659 /* trace_types holds a link list of available tracers. */ 660 static struct tracer *trace_types __read_mostly; 661 662 /* 663 * trace_types_lock is used to protect the trace_types list. 664 */ 665 DEFINE_MUTEX(trace_types_lock); 666 667 /* 668 * serialize the access of the ring buffer 669 * 670 * ring buffer serializes readers, but it is low level protection. 671 * The validity of the events (which returns by ring_buffer_peek() ..etc) 672 * are not protected by ring buffer. 673 * 674 * The content of events may become garbage if we allow other process consumes 675 * these events concurrently: 676 * A) the page of the consumed events may become a normal page 677 * (not reader page) in ring buffer, and this page will be rewrited 678 * by events producer. 679 * B) The page of the consumed events may become a page for splice_read, 680 * and this page will be returned to system. 681 * 682 * These primitives allow multi process access to different cpu ring buffer 683 * concurrently. 684 * 685 * These primitives don't distinguish read-only and read-consume access. 686 * Multi read-only access are also serialized. 687 */ 688 689 #ifdef CONFIG_SMP 690 static DECLARE_RWSEM(all_cpu_access_lock); 691 static DEFINE_PER_CPU(struct mutex, cpu_access_lock); 692 693 static inline void trace_access_lock(int cpu) 694 { 695 if (cpu == RING_BUFFER_ALL_CPUS) { 696 /* gain it for accessing the whole ring buffer. */ 697 down_write(&all_cpu_access_lock); 698 } else { 699 /* gain it for accessing a cpu ring buffer. */ 700 701 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */ 702 down_read(&all_cpu_access_lock); 703 704 /* Secondly block other access to this @cpu ring buffer. */ 705 mutex_lock(&per_cpu(cpu_access_lock, cpu)); 706 } 707 } 708 709 static inline void trace_access_unlock(int cpu) 710 { 711 if (cpu == RING_BUFFER_ALL_CPUS) { 712 up_write(&all_cpu_access_lock); 713 } else { 714 mutex_unlock(&per_cpu(cpu_access_lock, cpu)); 715 up_read(&all_cpu_access_lock); 716 } 717 } 718 719 static inline void trace_access_lock_init(void) 720 { 721 int cpu; 722 723 for_each_possible_cpu(cpu) 724 mutex_init(&per_cpu(cpu_access_lock, cpu)); 725 } 726 727 #else 728 729 static DEFINE_MUTEX(access_lock); 730 731 static inline void trace_access_lock(int cpu) 732 { 733 (void)cpu; 734 mutex_lock(&access_lock); 735 } 736 737 static inline void trace_access_unlock(int cpu) 738 { 739 (void)cpu; 740 mutex_unlock(&access_lock); 741 } 742 743 static inline void trace_access_lock_init(void) 744 { 745 } 746 747 #endif 748 749 #ifdef CONFIG_STACKTRACE 750 static void __ftrace_trace_stack(struct trace_buffer *buffer, 751 unsigned long flags, 752 int skip, int pc, struct pt_regs *regs); 753 static inline void ftrace_trace_stack(struct trace_array *tr, 754 struct trace_buffer *buffer, 755 unsigned long flags, 756 int skip, int pc, struct pt_regs *regs); 757 758 #else 759 static inline void __ftrace_trace_stack(struct trace_buffer *buffer, 760 unsigned long flags, 761 int skip, int pc, struct pt_regs *regs) 762 { 763 } 764 static inline void ftrace_trace_stack(struct trace_array *tr, 765 struct trace_buffer *buffer, 766 unsigned long flags, 767 int skip, int pc, struct pt_regs *regs) 768 { 769 } 770 771 #endif 772 773 static __always_inline void 774 trace_event_setup(struct ring_buffer_event *event, 775 int type, unsigned long flags, int pc) 776 { 777 struct trace_entry *ent = ring_buffer_event_data(event); 778 779 tracing_generic_entry_update(ent, type, flags, pc); 780 } 781 782 static __always_inline struct ring_buffer_event * 783 __trace_buffer_lock_reserve(struct trace_buffer *buffer, 784 int type, 785 unsigned long len, 786 unsigned long flags, int pc) 787 { 788 struct ring_buffer_event *event; 789 790 event = ring_buffer_lock_reserve(buffer, len); 791 if (event != NULL) 792 trace_event_setup(event, type, flags, pc); 793 794 return event; 795 } 796 797 void tracer_tracing_on(struct trace_array *tr) 798 { 799 if (tr->array_buffer.buffer) 800 ring_buffer_record_on(tr->array_buffer.buffer); 801 /* 802 * This flag is looked at when buffers haven't been allocated 803 * yet, or by some tracers (like irqsoff), that just want to 804 * know if the ring buffer has been disabled, but it can handle 805 * races of where it gets disabled but we still do a record. 806 * As the check is in the fast path of the tracers, it is more 807 * important to be fast than accurate. 808 */ 809 tr->buffer_disabled = 0; 810 /* Make the flag seen by readers */ 811 smp_wmb(); 812 } 813 814 /** 815 * tracing_on - enable tracing buffers 816 * 817 * This function enables tracing buffers that may have been 818 * disabled with tracing_off. 819 */ 820 void tracing_on(void) 821 { 822 tracer_tracing_on(&global_trace); 823 } 824 EXPORT_SYMBOL_GPL(tracing_on); 825 826 827 static __always_inline void 828 __buffer_unlock_commit(struct trace_buffer *buffer, struct ring_buffer_event *event) 829 { 830 __this_cpu_write(trace_taskinfo_save, true); 831 832 /* If this is the temp buffer, we need to commit fully */ 833 if (this_cpu_read(trace_buffered_event) == event) { 834 /* Length is in event->array[0] */ 835 ring_buffer_write(buffer, event->array[0], &event->array[1]); 836 /* Release the temp buffer */ 837 this_cpu_dec(trace_buffered_event_cnt); 838 } else 839 ring_buffer_unlock_commit(buffer, event); 840 } 841 842 /** 843 * __trace_puts - write a constant string into the trace buffer. 844 * @ip: The address of the caller 845 * @str: The constant string to write 846 * @size: The size of the string. 847 */ 848 int __trace_puts(unsigned long ip, const char *str, int size) 849 { 850 struct ring_buffer_event *event; 851 struct trace_buffer *buffer; 852 struct print_entry *entry; 853 unsigned long irq_flags; 854 int alloc; 855 int pc; 856 857 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 858 return 0; 859 860 pc = preempt_count(); 861 862 if (unlikely(tracing_selftest_running || tracing_disabled)) 863 return 0; 864 865 alloc = sizeof(*entry) + size + 2; /* possible \n added */ 866 867 local_save_flags(irq_flags); 868 buffer = global_trace.array_buffer.buffer; 869 ring_buffer_nest_start(buffer); 870 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 871 irq_flags, pc); 872 if (!event) { 873 size = 0; 874 goto out; 875 } 876 877 entry = ring_buffer_event_data(event); 878 entry->ip = ip; 879 880 memcpy(&entry->buf, str, size); 881 882 /* Add a newline if necessary */ 883 if (entry->buf[size - 1] != '\n') { 884 entry->buf[size] = '\n'; 885 entry->buf[size + 1] = '\0'; 886 } else 887 entry->buf[size] = '\0'; 888 889 __buffer_unlock_commit(buffer, event); 890 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL); 891 out: 892 ring_buffer_nest_end(buffer); 893 return size; 894 } 895 EXPORT_SYMBOL_GPL(__trace_puts); 896 897 /** 898 * __trace_bputs - write the pointer to a constant string into trace buffer 899 * @ip: The address of the caller 900 * @str: The constant string to write to the buffer to 901 */ 902 int __trace_bputs(unsigned long ip, const char *str) 903 { 904 struct ring_buffer_event *event; 905 struct trace_buffer *buffer; 906 struct bputs_entry *entry; 907 unsigned long irq_flags; 908 int size = sizeof(struct bputs_entry); 909 int ret = 0; 910 int pc; 911 912 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 913 return 0; 914 915 pc = preempt_count(); 916 917 if (unlikely(tracing_selftest_running || tracing_disabled)) 918 return 0; 919 920 local_save_flags(irq_flags); 921 buffer = global_trace.array_buffer.buffer; 922 923 ring_buffer_nest_start(buffer); 924 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size, 925 irq_flags, pc); 926 if (!event) 927 goto out; 928 929 entry = ring_buffer_event_data(event); 930 entry->ip = ip; 931 entry->str = str; 932 933 __buffer_unlock_commit(buffer, event); 934 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL); 935 936 ret = 1; 937 out: 938 ring_buffer_nest_end(buffer); 939 return ret; 940 } 941 EXPORT_SYMBOL_GPL(__trace_bputs); 942 943 #ifdef CONFIG_TRACER_SNAPSHOT 944 void tracing_snapshot_instance_cond(struct trace_array *tr, void *cond_data) 945 { 946 struct tracer *tracer = tr->current_trace; 947 unsigned long flags; 948 949 if (in_nmi()) { 950 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n"); 951 internal_trace_puts("*** snapshot is being ignored ***\n"); 952 return; 953 } 954 955 if (!tr->allocated_snapshot) { 956 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n"); 957 internal_trace_puts("*** stopping trace here! ***\n"); 958 tracing_off(); 959 return; 960 } 961 962 /* Note, snapshot can not be used when the tracer uses it */ 963 if (tracer->use_max_tr) { 964 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n"); 965 internal_trace_puts("*** Can not use snapshot (sorry) ***\n"); 966 return; 967 } 968 969 local_irq_save(flags); 970 update_max_tr(tr, current, smp_processor_id(), cond_data); 971 local_irq_restore(flags); 972 } 973 974 void tracing_snapshot_instance(struct trace_array *tr) 975 { 976 tracing_snapshot_instance_cond(tr, NULL); 977 } 978 979 /** 980 * tracing_snapshot - take a snapshot of the current buffer. 981 * 982 * This causes a swap between the snapshot buffer and the current live 983 * tracing buffer. You can use this to take snapshots of the live 984 * trace when some condition is triggered, but continue to trace. 985 * 986 * Note, make sure to allocate the snapshot with either 987 * a tracing_snapshot_alloc(), or by doing it manually 988 * with: echo 1 > /sys/kernel/debug/tracing/snapshot 989 * 990 * If the snapshot buffer is not allocated, it will stop tracing. 991 * Basically making a permanent snapshot. 992 */ 993 void tracing_snapshot(void) 994 { 995 struct trace_array *tr = &global_trace; 996 997 tracing_snapshot_instance(tr); 998 } 999 EXPORT_SYMBOL_GPL(tracing_snapshot); 1000 1001 /** 1002 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer. 1003 * @tr: The tracing instance to snapshot 1004 * @cond_data: The data to be tested conditionally, and possibly saved 1005 * 1006 * This is the same as tracing_snapshot() except that the snapshot is 1007 * conditional - the snapshot will only happen if the 1008 * cond_snapshot.update() implementation receiving the cond_data 1009 * returns true, which means that the trace array's cond_snapshot 1010 * update() operation used the cond_data to determine whether the 1011 * snapshot should be taken, and if it was, presumably saved it along 1012 * with the snapshot. 1013 */ 1014 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data) 1015 { 1016 tracing_snapshot_instance_cond(tr, cond_data); 1017 } 1018 EXPORT_SYMBOL_GPL(tracing_snapshot_cond); 1019 1020 /** 1021 * tracing_snapshot_cond_data - get the user data associated with a snapshot 1022 * @tr: The tracing instance 1023 * 1024 * When the user enables a conditional snapshot using 1025 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved 1026 * with the snapshot. This accessor is used to retrieve it. 1027 * 1028 * Should not be called from cond_snapshot.update(), since it takes 1029 * the tr->max_lock lock, which the code calling 1030 * cond_snapshot.update() has already done. 1031 * 1032 * Returns the cond_data associated with the trace array's snapshot. 1033 */ 1034 void *tracing_cond_snapshot_data(struct trace_array *tr) 1035 { 1036 void *cond_data = NULL; 1037 1038 arch_spin_lock(&tr->max_lock); 1039 1040 if (tr->cond_snapshot) 1041 cond_data = tr->cond_snapshot->cond_data; 1042 1043 arch_spin_unlock(&tr->max_lock); 1044 1045 return cond_data; 1046 } 1047 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data); 1048 1049 static int resize_buffer_duplicate_size(struct array_buffer *trace_buf, 1050 struct array_buffer *size_buf, int cpu_id); 1051 static void set_buffer_entries(struct array_buffer *buf, unsigned long val); 1052 1053 int tracing_alloc_snapshot_instance(struct trace_array *tr) 1054 { 1055 int ret; 1056 1057 if (!tr->allocated_snapshot) { 1058 1059 /* allocate spare buffer */ 1060 ret = resize_buffer_duplicate_size(&tr->max_buffer, 1061 &tr->array_buffer, RING_BUFFER_ALL_CPUS); 1062 if (ret < 0) 1063 return ret; 1064 1065 tr->allocated_snapshot = true; 1066 } 1067 1068 return 0; 1069 } 1070 1071 static void free_snapshot(struct trace_array *tr) 1072 { 1073 /* 1074 * We don't free the ring buffer. instead, resize it because 1075 * The max_tr ring buffer has some state (e.g. ring->clock) and 1076 * we want preserve it. 1077 */ 1078 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS); 1079 set_buffer_entries(&tr->max_buffer, 1); 1080 tracing_reset_online_cpus(&tr->max_buffer); 1081 tr->allocated_snapshot = false; 1082 } 1083 1084 /** 1085 * tracing_alloc_snapshot - allocate snapshot buffer. 1086 * 1087 * This only allocates the snapshot buffer if it isn't already 1088 * allocated - it doesn't also take a snapshot. 1089 * 1090 * This is meant to be used in cases where the snapshot buffer needs 1091 * to be set up for events that can't sleep but need to be able to 1092 * trigger a snapshot. 1093 */ 1094 int tracing_alloc_snapshot(void) 1095 { 1096 struct trace_array *tr = &global_trace; 1097 int ret; 1098 1099 ret = tracing_alloc_snapshot_instance(tr); 1100 WARN_ON(ret < 0); 1101 1102 return ret; 1103 } 1104 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); 1105 1106 /** 1107 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer. 1108 * 1109 * This is similar to tracing_snapshot(), but it will allocate the 1110 * snapshot buffer if it isn't already allocated. Use this only 1111 * where it is safe to sleep, as the allocation may sleep. 1112 * 1113 * This causes a swap between the snapshot buffer and the current live 1114 * tracing buffer. You can use this to take snapshots of the live 1115 * trace when some condition is triggered, but continue to trace. 1116 */ 1117 void tracing_snapshot_alloc(void) 1118 { 1119 int ret; 1120 1121 ret = tracing_alloc_snapshot(); 1122 if (ret < 0) 1123 return; 1124 1125 tracing_snapshot(); 1126 } 1127 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); 1128 1129 /** 1130 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance 1131 * @tr: The tracing instance 1132 * @cond_data: User data to associate with the snapshot 1133 * @update: Implementation of the cond_snapshot update function 1134 * 1135 * Check whether the conditional snapshot for the given instance has 1136 * already been enabled, or if the current tracer is already using a 1137 * snapshot; if so, return -EBUSY, else create a cond_snapshot and 1138 * save the cond_data and update function inside. 1139 * 1140 * Returns 0 if successful, error otherwise. 1141 */ 1142 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, 1143 cond_update_fn_t update) 1144 { 1145 struct cond_snapshot *cond_snapshot; 1146 int ret = 0; 1147 1148 cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL); 1149 if (!cond_snapshot) 1150 return -ENOMEM; 1151 1152 cond_snapshot->cond_data = cond_data; 1153 cond_snapshot->update = update; 1154 1155 mutex_lock(&trace_types_lock); 1156 1157 ret = tracing_alloc_snapshot_instance(tr); 1158 if (ret) 1159 goto fail_unlock; 1160 1161 if (tr->current_trace->use_max_tr) { 1162 ret = -EBUSY; 1163 goto fail_unlock; 1164 } 1165 1166 /* 1167 * The cond_snapshot can only change to NULL without the 1168 * trace_types_lock. We don't care if we race with it going 1169 * to NULL, but we want to make sure that it's not set to 1170 * something other than NULL when we get here, which we can 1171 * do safely with only holding the trace_types_lock and not 1172 * having to take the max_lock. 1173 */ 1174 if (tr->cond_snapshot) { 1175 ret = -EBUSY; 1176 goto fail_unlock; 1177 } 1178 1179 arch_spin_lock(&tr->max_lock); 1180 tr->cond_snapshot = cond_snapshot; 1181 arch_spin_unlock(&tr->max_lock); 1182 1183 mutex_unlock(&trace_types_lock); 1184 1185 return ret; 1186 1187 fail_unlock: 1188 mutex_unlock(&trace_types_lock); 1189 kfree(cond_snapshot); 1190 return ret; 1191 } 1192 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable); 1193 1194 /** 1195 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance 1196 * @tr: The tracing instance 1197 * 1198 * Check whether the conditional snapshot for the given instance is 1199 * enabled; if so, free the cond_snapshot associated with it, 1200 * otherwise return -EINVAL. 1201 * 1202 * Returns 0 if successful, error otherwise. 1203 */ 1204 int tracing_snapshot_cond_disable(struct trace_array *tr) 1205 { 1206 int ret = 0; 1207 1208 arch_spin_lock(&tr->max_lock); 1209 1210 if (!tr->cond_snapshot) 1211 ret = -EINVAL; 1212 else { 1213 kfree(tr->cond_snapshot); 1214 tr->cond_snapshot = NULL; 1215 } 1216 1217 arch_spin_unlock(&tr->max_lock); 1218 1219 return ret; 1220 } 1221 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable); 1222 #else 1223 void tracing_snapshot(void) 1224 { 1225 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used"); 1226 } 1227 EXPORT_SYMBOL_GPL(tracing_snapshot); 1228 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data) 1229 { 1230 WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used"); 1231 } 1232 EXPORT_SYMBOL_GPL(tracing_snapshot_cond); 1233 int tracing_alloc_snapshot(void) 1234 { 1235 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used"); 1236 return -ENODEV; 1237 } 1238 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); 1239 void tracing_snapshot_alloc(void) 1240 { 1241 /* Give warning */ 1242 tracing_snapshot(); 1243 } 1244 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); 1245 void *tracing_cond_snapshot_data(struct trace_array *tr) 1246 { 1247 return NULL; 1248 } 1249 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data); 1250 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update) 1251 { 1252 return -ENODEV; 1253 } 1254 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable); 1255 int tracing_snapshot_cond_disable(struct trace_array *tr) 1256 { 1257 return false; 1258 } 1259 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable); 1260 #endif /* CONFIG_TRACER_SNAPSHOT */ 1261 1262 void tracer_tracing_off(struct trace_array *tr) 1263 { 1264 if (tr->array_buffer.buffer) 1265 ring_buffer_record_off(tr->array_buffer.buffer); 1266 /* 1267 * This flag is looked at when buffers haven't been allocated 1268 * yet, or by some tracers (like irqsoff), that just want to 1269 * know if the ring buffer has been disabled, but it can handle 1270 * races of where it gets disabled but we still do a record. 1271 * As the check is in the fast path of the tracers, it is more 1272 * important to be fast than accurate. 1273 */ 1274 tr->buffer_disabled = 1; 1275 /* Make the flag seen by readers */ 1276 smp_wmb(); 1277 } 1278 1279 /** 1280 * tracing_off - turn off tracing buffers 1281 * 1282 * This function stops the tracing buffers from recording data. 1283 * It does not disable any overhead the tracers themselves may 1284 * be causing. This function simply causes all recording to 1285 * the ring buffers to fail. 1286 */ 1287 void tracing_off(void) 1288 { 1289 tracer_tracing_off(&global_trace); 1290 } 1291 EXPORT_SYMBOL_GPL(tracing_off); 1292 1293 void disable_trace_on_warning(void) 1294 { 1295 if (__disable_trace_on_warning) 1296 tracing_off(); 1297 } 1298 1299 /** 1300 * tracer_tracing_is_on - show real state of ring buffer enabled 1301 * @tr : the trace array to know if ring buffer is enabled 1302 * 1303 * Shows real state of the ring buffer if it is enabled or not. 1304 */ 1305 bool tracer_tracing_is_on(struct trace_array *tr) 1306 { 1307 if (tr->array_buffer.buffer) 1308 return ring_buffer_record_is_on(tr->array_buffer.buffer); 1309 return !tr->buffer_disabled; 1310 } 1311 1312 /** 1313 * tracing_is_on - show state of ring buffers enabled 1314 */ 1315 int tracing_is_on(void) 1316 { 1317 return tracer_tracing_is_on(&global_trace); 1318 } 1319 EXPORT_SYMBOL_GPL(tracing_is_on); 1320 1321 static int __init set_buf_size(char *str) 1322 { 1323 unsigned long buf_size; 1324 1325 if (!str) 1326 return 0; 1327 buf_size = memparse(str, &str); 1328 /* nr_entries can not be zero */ 1329 if (buf_size == 0) 1330 return 0; 1331 trace_buf_size = buf_size; 1332 return 1; 1333 } 1334 __setup("trace_buf_size=", set_buf_size); 1335 1336 static int __init set_tracing_thresh(char *str) 1337 { 1338 unsigned long threshold; 1339 int ret; 1340 1341 if (!str) 1342 return 0; 1343 ret = kstrtoul(str, 0, &threshold); 1344 if (ret < 0) 1345 return 0; 1346 tracing_thresh = threshold * 1000; 1347 return 1; 1348 } 1349 __setup("tracing_thresh=", set_tracing_thresh); 1350 1351 unsigned long nsecs_to_usecs(unsigned long nsecs) 1352 { 1353 return nsecs / 1000; 1354 } 1355 1356 /* 1357 * TRACE_FLAGS is defined as a tuple matching bit masks with strings. 1358 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that 1359 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list 1360 * of strings in the order that the evals (enum) were defined. 1361 */ 1362 #undef C 1363 #define C(a, b) b 1364 1365 /* These must match the bit postions in trace_iterator_flags */ 1366 static const char *trace_options[] = { 1367 TRACE_FLAGS 1368 NULL 1369 }; 1370 1371 static struct { 1372 u64 (*func)(void); 1373 const char *name; 1374 int in_ns; /* is this clock in nanoseconds? */ 1375 } trace_clocks[] = { 1376 { trace_clock_local, "local", 1 }, 1377 { trace_clock_global, "global", 1 }, 1378 { trace_clock_counter, "counter", 0 }, 1379 { trace_clock_jiffies, "uptime", 0 }, 1380 { trace_clock, "perf", 1 }, 1381 { ktime_get_mono_fast_ns, "mono", 1 }, 1382 { ktime_get_raw_fast_ns, "mono_raw", 1 }, 1383 { ktime_get_boot_fast_ns, "boot", 1 }, 1384 ARCH_TRACE_CLOCKS 1385 }; 1386 1387 bool trace_clock_in_ns(struct trace_array *tr) 1388 { 1389 if (trace_clocks[tr->clock_id].in_ns) 1390 return true; 1391 1392 return false; 1393 } 1394 1395 /* 1396 * trace_parser_get_init - gets the buffer for trace parser 1397 */ 1398 int trace_parser_get_init(struct trace_parser *parser, int size) 1399 { 1400 memset(parser, 0, sizeof(*parser)); 1401 1402 parser->buffer = kmalloc(size, GFP_KERNEL); 1403 if (!parser->buffer) 1404 return 1; 1405 1406 parser->size = size; 1407 return 0; 1408 } 1409 1410 /* 1411 * trace_parser_put - frees the buffer for trace parser 1412 */ 1413 void trace_parser_put(struct trace_parser *parser) 1414 { 1415 kfree(parser->buffer); 1416 parser->buffer = NULL; 1417 } 1418 1419 /* 1420 * trace_get_user - reads the user input string separated by space 1421 * (matched by isspace(ch)) 1422 * 1423 * For each string found the 'struct trace_parser' is updated, 1424 * and the function returns. 1425 * 1426 * Returns number of bytes read. 1427 * 1428 * See kernel/trace/trace.h for 'struct trace_parser' details. 1429 */ 1430 int trace_get_user(struct trace_parser *parser, const char __user *ubuf, 1431 size_t cnt, loff_t *ppos) 1432 { 1433 char ch; 1434 size_t read = 0; 1435 ssize_t ret; 1436 1437 if (!*ppos) 1438 trace_parser_clear(parser); 1439 1440 ret = get_user(ch, ubuf++); 1441 if (ret) 1442 goto out; 1443 1444 read++; 1445 cnt--; 1446 1447 /* 1448 * The parser is not finished with the last write, 1449 * continue reading the user input without skipping spaces. 1450 */ 1451 if (!parser->cont) { 1452 /* skip white space */ 1453 while (cnt && isspace(ch)) { 1454 ret = get_user(ch, ubuf++); 1455 if (ret) 1456 goto out; 1457 read++; 1458 cnt--; 1459 } 1460 1461 parser->idx = 0; 1462 1463 /* only spaces were written */ 1464 if (isspace(ch) || !ch) { 1465 *ppos += read; 1466 ret = read; 1467 goto out; 1468 } 1469 } 1470 1471 /* read the non-space input */ 1472 while (cnt && !isspace(ch) && ch) { 1473 if (parser->idx < parser->size - 1) 1474 parser->buffer[parser->idx++] = ch; 1475 else { 1476 ret = -EINVAL; 1477 goto out; 1478 } 1479 ret = get_user(ch, ubuf++); 1480 if (ret) 1481 goto out; 1482 read++; 1483 cnt--; 1484 } 1485 1486 /* We either got finished input or we have to wait for another call. */ 1487 if (isspace(ch) || !ch) { 1488 parser->buffer[parser->idx] = 0; 1489 parser->cont = false; 1490 } else if (parser->idx < parser->size - 1) { 1491 parser->cont = true; 1492 parser->buffer[parser->idx++] = ch; 1493 /* Make sure the parsed string always terminates with '\0'. */ 1494 parser->buffer[parser->idx] = 0; 1495 } else { 1496 ret = -EINVAL; 1497 goto out; 1498 } 1499 1500 *ppos += read; 1501 ret = read; 1502 1503 out: 1504 return ret; 1505 } 1506 1507 /* TODO add a seq_buf_to_buffer() */ 1508 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) 1509 { 1510 int len; 1511 1512 if (trace_seq_used(s) <= s->seq.readpos) 1513 return -EBUSY; 1514 1515 len = trace_seq_used(s) - s->seq.readpos; 1516 if (cnt > len) 1517 cnt = len; 1518 memcpy(buf, s->buffer + s->seq.readpos, cnt); 1519 1520 s->seq.readpos += cnt; 1521 return cnt; 1522 } 1523 1524 unsigned long __read_mostly tracing_thresh; 1525 static const struct file_operations tracing_max_lat_fops; 1526 1527 #if (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \ 1528 defined(CONFIG_FSNOTIFY) 1529 1530 static struct workqueue_struct *fsnotify_wq; 1531 1532 static void latency_fsnotify_workfn(struct work_struct *work) 1533 { 1534 struct trace_array *tr = container_of(work, struct trace_array, 1535 fsnotify_work); 1536 fsnotify(tr->d_max_latency->d_inode, FS_MODIFY, 1537 tr->d_max_latency->d_inode, FSNOTIFY_EVENT_INODE, NULL, 0); 1538 } 1539 1540 static void latency_fsnotify_workfn_irq(struct irq_work *iwork) 1541 { 1542 struct trace_array *tr = container_of(iwork, struct trace_array, 1543 fsnotify_irqwork); 1544 queue_work(fsnotify_wq, &tr->fsnotify_work); 1545 } 1546 1547 static void trace_create_maxlat_file(struct trace_array *tr, 1548 struct dentry *d_tracer) 1549 { 1550 INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn); 1551 init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq); 1552 tr->d_max_latency = trace_create_file("tracing_max_latency", 0644, 1553 d_tracer, &tr->max_latency, 1554 &tracing_max_lat_fops); 1555 } 1556 1557 __init static int latency_fsnotify_init(void) 1558 { 1559 fsnotify_wq = alloc_workqueue("tr_max_lat_wq", 1560 WQ_UNBOUND | WQ_HIGHPRI, 0); 1561 if (!fsnotify_wq) { 1562 pr_err("Unable to allocate tr_max_lat_wq\n"); 1563 return -ENOMEM; 1564 } 1565 return 0; 1566 } 1567 1568 late_initcall_sync(latency_fsnotify_init); 1569 1570 void latency_fsnotify(struct trace_array *tr) 1571 { 1572 if (!fsnotify_wq) 1573 return; 1574 /* 1575 * We cannot call queue_work(&tr->fsnotify_work) from here because it's 1576 * possible that we are called from __schedule() or do_idle(), which 1577 * could cause a deadlock. 1578 */ 1579 irq_work_queue(&tr->fsnotify_irqwork); 1580 } 1581 1582 /* 1583 * (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \ 1584 * defined(CONFIG_FSNOTIFY) 1585 */ 1586 #else 1587 1588 #define trace_create_maxlat_file(tr, d_tracer) \ 1589 trace_create_file("tracing_max_latency", 0644, d_tracer, \ 1590 &tr->max_latency, &tracing_max_lat_fops) 1591 1592 #endif 1593 1594 #ifdef CONFIG_TRACER_MAX_TRACE 1595 /* 1596 * Copy the new maximum trace into the separate maximum-trace 1597 * structure. (this way the maximum trace is permanently saved, 1598 * for later retrieval via /sys/kernel/tracing/tracing_max_latency) 1599 */ 1600 static void 1601 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 1602 { 1603 struct array_buffer *trace_buf = &tr->array_buffer; 1604 struct array_buffer *max_buf = &tr->max_buffer; 1605 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu); 1606 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu); 1607 1608 max_buf->cpu = cpu; 1609 max_buf->time_start = data->preempt_timestamp; 1610 1611 max_data->saved_latency = tr->max_latency; 1612 max_data->critical_start = data->critical_start; 1613 max_data->critical_end = data->critical_end; 1614 1615 strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN); 1616 max_data->pid = tsk->pid; 1617 /* 1618 * If tsk == current, then use current_uid(), as that does not use 1619 * RCU. The irq tracer can be called out of RCU scope. 1620 */ 1621 if (tsk == current) 1622 max_data->uid = current_uid(); 1623 else 1624 max_data->uid = task_uid(tsk); 1625 1626 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; 1627 max_data->policy = tsk->policy; 1628 max_data->rt_priority = tsk->rt_priority; 1629 1630 /* record this tasks comm */ 1631 tracing_record_cmdline(tsk); 1632 latency_fsnotify(tr); 1633 } 1634 1635 /** 1636 * update_max_tr - snapshot all trace buffers from global_trace to max_tr 1637 * @tr: tracer 1638 * @tsk: the task with the latency 1639 * @cpu: The cpu that initiated the trace. 1640 * @cond_data: User data associated with a conditional snapshot 1641 * 1642 * Flip the buffers between the @tr and the max_tr and record information 1643 * about which task was the cause of this latency. 1644 */ 1645 void 1646 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu, 1647 void *cond_data) 1648 { 1649 if (tr->stop_count) 1650 return; 1651 1652 WARN_ON_ONCE(!irqs_disabled()); 1653 1654 if (!tr->allocated_snapshot) { 1655 /* Only the nop tracer should hit this when disabling */ 1656 WARN_ON_ONCE(tr->current_trace != &nop_trace); 1657 return; 1658 } 1659 1660 arch_spin_lock(&tr->max_lock); 1661 1662 /* Inherit the recordable setting from array_buffer */ 1663 if (ring_buffer_record_is_set_on(tr->array_buffer.buffer)) 1664 ring_buffer_record_on(tr->max_buffer.buffer); 1665 else 1666 ring_buffer_record_off(tr->max_buffer.buffer); 1667 1668 #ifdef CONFIG_TRACER_SNAPSHOT 1669 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data)) 1670 goto out_unlock; 1671 #endif 1672 swap(tr->array_buffer.buffer, tr->max_buffer.buffer); 1673 1674 __update_max_tr(tr, tsk, cpu); 1675 1676 out_unlock: 1677 arch_spin_unlock(&tr->max_lock); 1678 } 1679 1680 /** 1681 * update_max_tr_single - only copy one trace over, and reset the rest 1682 * @tr: tracer 1683 * @tsk: task with the latency 1684 * @cpu: the cpu of the buffer to copy. 1685 * 1686 * Flip the trace of a single CPU buffer between the @tr and the max_tr. 1687 */ 1688 void 1689 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) 1690 { 1691 int ret; 1692 1693 if (tr->stop_count) 1694 return; 1695 1696 WARN_ON_ONCE(!irqs_disabled()); 1697 if (!tr->allocated_snapshot) { 1698 /* Only the nop tracer should hit this when disabling */ 1699 WARN_ON_ONCE(tr->current_trace != &nop_trace); 1700 return; 1701 } 1702 1703 arch_spin_lock(&tr->max_lock); 1704 1705 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu); 1706 1707 if (ret == -EBUSY) { 1708 /* 1709 * We failed to swap the buffer due to a commit taking 1710 * place on this CPU. We fail to record, but we reset 1711 * the max trace buffer (no one writes directly to it) 1712 * and flag that it failed. 1713 */ 1714 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_, 1715 "Failed to swap buffers due to commit in progress\n"); 1716 } 1717 1718 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); 1719 1720 __update_max_tr(tr, tsk, cpu); 1721 arch_spin_unlock(&tr->max_lock); 1722 } 1723 #endif /* CONFIG_TRACER_MAX_TRACE */ 1724 1725 static int wait_on_pipe(struct trace_iterator *iter, int full) 1726 { 1727 /* Iterators are static, they should be filled or empty */ 1728 if (trace_buffer_iter(iter, iter->cpu_file)) 1729 return 0; 1730 1731 return ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file, 1732 full); 1733 } 1734 1735 #ifdef CONFIG_FTRACE_STARTUP_TEST 1736 static bool selftests_can_run; 1737 1738 struct trace_selftests { 1739 struct list_head list; 1740 struct tracer *type; 1741 }; 1742 1743 static LIST_HEAD(postponed_selftests); 1744 1745 static int save_selftest(struct tracer *type) 1746 { 1747 struct trace_selftests *selftest; 1748 1749 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL); 1750 if (!selftest) 1751 return -ENOMEM; 1752 1753 selftest->type = type; 1754 list_add(&selftest->list, &postponed_selftests); 1755 return 0; 1756 } 1757 1758 static int run_tracer_selftest(struct tracer *type) 1759 { 1760 struct trace_array *tr = &global_trace; 1761 struct tracer *saved_tracer = tr->current_trace; 1762 int ret; 1763 1764 if (!type->selftest || tracing_selftest_disabled) 1765 return 0; 1766 1767 /* 1768 * If a tracer registers early in boot up (before scheduling is 1769 * initialized and such), then do not run its selftests yet. 1770 * Instead, run it a little later in the boot process. 1771 */ 1772 if (!selftests_can_run) 1773 return save_selftest(type); 1774 1775 /* 1776 * Run a selftest on this tracer. 1777 * Here we reset the trace buffer, and set the current 1778 * tracer to be this tracer. The tracer can then run some 1779 * internal tracing to verify that everything is in order. 1780 * If we fail, we do not register this tracer. 1781 */ 1782 tracing_reset_online_cpus(&tr->array_buffer); 1783 1784 tr->current_trace = type; 1785 1786 #ifdef CONFIG_TRACER_MAX_TRACE 1787 if (type->use_max_tr) { 1788 /* If we expanded the buffers, make sure the max is expanded too */ 1789 if (ring_buffer_expanded) 1790 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size, 1791 RING_BUFFER_ALL_CPUS); 1792 tr->allocated_snapshot = true; 1793 } 1794 #endif 1795 1796 /* the test is responsible for initializing and enabling */ 1797 pr_info("Testing tracer %s: ", type->name); 1798 ret = type->selftest(type, tr); 1799 /* the test is responsible for resetting too */ 1800 tr->current_trace = saved_tracer; 1801 if (ret) { 1802 printk(KERN_CONT "FAILED!\n"); 1803 /* Add the warning after printing 'FAILED' */ 1804 WARN_ON(1); 1805 return -1; 1806 } 1807 /* Only reset on passing, to avoid touching corrupted buffers */ 1808 tracing_reset_online_cpus(&tr->array_buffer); 1809 1810 #ifdef CONFIG_TRACER_MAX_TRACE 1811 if (type->use_max_tr) { 1812 tr->allocated_snapshot = false; 1813 1814 /* Shrink the max buffer again */ 1815 if (ring_buffer_expanded) 1816 ring_buffer_resize(tr->max_buffer.buffer, 1, 1817 RING_BUFFER_ALL_CPUS); 1818 } 1819 #endif 1820 1821 printk(KERN_CONT "PASSED\n"); 1822 return 0; 1823 } 1824 1825 static __init int init_trace_selftests(void) 1826 { 1827 struct trace_selftests *p, *n; 1828 struct tracer *t, **last; 1829 int ret; 1830 1831 selftests_can_run = true; 1832 1833 mutex_lock(&trace_types_lock); 1834 1835 if (list_empty(&postponed_selftests)) 1836 goto out; 1837 1838 pr_info("Running postponed tracer tests:\n"); 1839 1840 tracing_selftest_running = true; 1841 list_for_each_entry_safe(p, n, &postponed_selftests, list) { 1842 /* This loop can take minutes when sanitizers are enabled, so 1843 * lets make sure we allow RCU processing. 1844 */ 1845 cond_resched(); 1846 ret = run_tracer_selftest(p->type); 1847 /* If the test fails, then warn and remove from available_tracers */ 1848 if (ret < 0) { 1849 WARN(1, "tracer: %s failed selftest, disabling\n", 1850 p->type->name); 1851 last = &trace_types; 1852 for (t = trace_types; t; t = t->next) { 1853 if (t == p->type) { 1854 *last = t->next; 1855 break; 1856 } 1857 last = &t->next; 1858 } 1859 } 1860 list_del(&p->list); 1861 kfree(p); 1862 } 1863 tracing_selftest_running = false; 1864 1865 out: 1866 mutex_unlock(&trace_types_lock); 1867 1868 return 0; 1869 } 1870 core_initcall(init_trace_selftests); 1871 #else 1872 static inline int run_tracer_selftest(struct tracer *type) 1873 { 1874 return 0; 1875 } 1876 #endif /* CONFIG_FTRACE_STARTUP_TEST */ 1877 1878 static void add_tracer_options(struct trace_array *tr, struct tracer *t); 1879 1880 static void __init apply_trace_boot_options(void); 1881 1882 /** 1883 * register_tracer - register a tracer with the ftrace system. 1884 * @type: the plugin for the tracer 1885 * 1886 * Register a new plugin tracer. 1887 */ 1888 int __init register_tracer(struct tracer *type) 1889 { 1890 struct tracer *t; 1891 int ret = 0; 1892 1893 if (!type->name) { 1894 pr_info("Tracer must have a name\n"); 1895 return -1; 1896 } 1897 1898 if (strlen(type->name) >= MAX_TRACER_SIZE) { 1899 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE); 1900 return -1; 1901 } 1902 1903 if (security_locked_down(LOCKDOWN_TRACEFS)) { 1904 pr_warn("Can not register tracer %s due to lockdown\n", 1905 type->name); 1906 return -EPERM; 1907 } 1908 1909 mutex_lock(&trace_types_lock); 1910 1911 tracing_selftest_running = true; 1912 1913 for (t = trace_types; t; t = t->next) { 1914 if (strcmp(type->name, t->name) == 0) { 1915 /* already found */ 1916 pr_info("Tracer %s already registered\n", 1917 type->name); 1918 ret = -1; 1919 goto out; 1920 } 1921 } 1922 1923 if (!type->set_flag) 1924 type->set_flag = &dummy_set_flag; 1925 if (!type->flags) { 1926 /*allocate a dummy tracer_flags*/ 1927 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL); 1928 if (!type->flags) { 1929 ret = -ENOMEM; 1930 goto out; 1931 } 1932 type->flags->val = 0; 1933 type->flags->opts = dummy_tracer_opt; 1934 } else 1935 if (!type->flags->opts) 1936 type->flags->opts = dummy_tracer_opt; 1937 1938 /* store the tracer for __set_tracer_option */ 1939 type->flags->trace = type; 1940 1941 ret = run_tracer_selftest(type); 1942 if (ret < 0) 1943 goto out; 1944 1945 type->next = trace_types; 1946 trace_types = type; 1947 add_tracer_options(&global_trace, type); 1948 1949 out: 1950 tracing_selftest_running = false; 1951 mutex_unlock(&trace_types_lock); 1952 1953 if (ret || !default_bootup_tracer) 1954 goto out_unlock; 1955 1956 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE)) 1957 goto out_unlock; 1958 1959 printk(KERN_INFO "Starting tracer '%s'\n", type->name); 1960 /* Do we want this tracer to start on bootup? */ 1961 tracing_set_tracer(&global_trace, type->name); 1962 default_bootup_tracer = NULL; 1963 1964 apply_trace_boot_options(); 1965 1966 /* disable other selftests, since this will break it. */ 1967 tracing_selftest_disabled = true; 1968 #ifdef CONFIG_FTRACE_STARTUP_TEST 1969 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n", 1970 type->name); 1971 #endif 1972 1973 out_unlock: 1974 return ret; 1975 } 1976 1977 static void tracing_reset_cpu(struct array_buffer *buf, int cpu) 1978 { 1979 struct trace_buffer *buffer = buf->buffer; 1980 1981 if (!buffer) 1982 return; 1983 1984 ring_buffer_record_disable(buffer); 1985 1986 /* Make sure all commits have finished */ 1987 synchronize_rcu(); 1988 ring_buffer_reset_cpu(buffer, cpu); 1989 1990 ring_buffer_record_enable(buffer); 1991 } 1992 1993 void tracing_reset_online_cpus(struct array_buffer *buf) 1994 { 1995 struct trace_buffer *buffer = buf->buffer; 1996 int cpu; 1997 1998 if (!buffer) 1999 return; 2000 2001 ring_buffer_record_disable(buffer); 2002 2003 /* Make sure all commits have finished */ 2004 synchronize_rcu(); 2005 2006 buf->time_start = buffer_ftrace_now(buf, buf->cpu); 2007 2008 for_each_online_cpu(cpu) 2009 ring_buffer_reset_cpu(buffer, cpu); 2010 2011 ring_buffer_record_enable(buffer); 2012 } 2013 2014 /* Must have trace_types_lock held */ 2015 void tracing_reset_all_online_cpus(void) 2016 { 2017 struct trace_array *tr; 2018 2019 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 2020 if (!tr->clear_trace) 2021 continue; 2022 tr->clear_trace = false; 2023 tracing_reset_online_cpus(&tr->array_buffer); 2024 #ifdef CONFIG_TRACER_MAX_TRACE 2025 tracing_reset_online_cpus(&tr->max_buffer); 2026 #endif 2027 } 2028 } 2029 2030 static int *tgid_map; 2031 2032 #define SAVED_CMDLINES_DEFAULT 128 2033 #define NO_CMDLINE_MAP UINT_MAX 2034 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED; 2035 struct saved_cmdlines_buffer { 2036 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; 2037 unsigned *map_cmdline_to_pid; 2038 unsigned cmdline_num; 2039 int cmdline_idx; 2040 char *saved_cmdlines; 2041 }; 2042 static struct saved_cmdlines_buffer *savedcmd; 2043 2044 /* temporary disable recording */ 2045 static atomic_t trace_record_taskinfo_disabled __read_mostly; 2046 2047 static inline char *get_saved_cmdlines(int idx) 2048 { 2049 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN]; 2050 } 2051 2052 static inline void set_cmdline(int idx, const char *cmdline) 2053 { 2054 strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN); 2055 } 2056 2057 static int allocate_cmdlines_buffer(unsigned int val, 2058 struct saved_cmdlines_buffer *s) 2059 { 2060 s->map_cmdline_to_pid = kmalloc_array(val, 2061 sizeof(*s->map_cmdline_to_pid), 2062 GFP_KERNEL); 2063 if (!s->map_cmdline_to_pid) 2064 return -ENOMEM; 2065 2066 s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL); 2067 if (!s->saved_cmdlines) { 2068 kfree(s->map_cmdline_to_pid); 2069 return -ENOMEM; 2070 } 2071 2072 s->cmdline_idx = 0; 2073 s->cmdline_num = val; 2074 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP, 2075 sizeof(s->map_pid_to_cmdline)); 2076 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP, 2077 val * sizeof(*s->map_cmdline_to_pid)); 2078 2079 return 0; 2080 } 2081 2082 static int trace_create_savedcmd(void) 2083 { 2084 int ret; 2085 2086 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL); 2087 if (!savedcmd) 2088 return -ENOMEM; 2089 2090 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd); 2091 if (ret < 0) { 2092 kfree(savedcmd); 2093 savedcmd = NULL; 2094 return -ENOMEM; 2095 } 2096 2097 return 0; 2098 } 2099 2100 int is_tracing_stopped(void) 2101 { 2102 return global_trace.stop_count; 2103 } 2104 2105 /** 2106 * tracing_start - quick start of the tracer 2107 * 2108 * If tracing is enabled but was stopped by tracing_stop, 2109 * this will start the tracer back up. 2110 */ 2111 void tracing_start(void) 2112 { 2113 struct trace_buffer *buffer; 2114 unsigned long flags; 2115 2116 if (tracing_disabled) 2117 return; 2118 2119 raw_spin_lock_irqsave(&global_trace.start_lock, flags); 2120 if (--global_trace.stop_count) { 2121 if (global_trace.stop_count < 0) { 2122 /* Someone screwed up their debugging */ 2123 WARN_ON_ONCE(1); 2124 global_trace.stop_count = 0; 2125 } 2126 goto out; 2127 } 2128 2129 /* Prevent the buffers from switching */ 2130 arch_spin_lock(&global_trace.max_lock); 2131 2132 buffer = global_trace.array_buffer.buffer; 2133 if (buffer) 2134 ring_buffer_record_enable(buffer); 2135 2136 #ifdef CONFIG_TRACER_MAX_TRACE 2137 buffer = global_trace.max_buffer.buffer; 2138 if (buffer) 2139 ring_buffer_record_enable(buffer); 2140 #endif 2141 2142 arch_spin_unlock(&global_trace.max_lock); 2143 2144 out: 2145 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags); 2146 } 2147 2148 static void tracing_start_tr(struct trace_array *tr) 2149 { 2150 struct trace_buffer *buffer; 2151 unsigned long flags; 2152 2153 if (tracing_disabled) 2154 return; 2155 2156 /* If global, we need to also start the max tracer */ 2157 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 2158 return tracing_start(); 2159 2160 raw_spin_lock_irqsave(&tr->start_lock, flags); 2161 2162 if (--tr->stop_count) { 2163 if (tr->stop_count < 0) { 2164 /* Someone screwed up their debugging */ 2165 WARN_ON_ONCE(1); 2166 tr->stop_count = 0; 2167 } 2168 goto out; 2169 } 2170 2171 buffer = tr->array_buffer.buffer; 2172 if (buffer) 2173 ring_buffer_record_enable(buffer); 2174 2175 out: 2176 raw_spin_unlock_irqrestore(&tr->start_lock, flags); 2177 } 2178 2179 /** 2180 * tracing_stop - quick stop of the tracer 2181 * 2182 * Light weight way to stop tracing. Use in conjunction with 2183 * tracing_start. 2184 */ 2185 void tracing_stop(void) 2186 { 2187 struct trace_buffer *buffer; 2188 unsigned long flags; 2189 2190 raw_spin_lock_irqsave(&global_trace.start_lock, flags); 2191 if (global_trace.stop_count++) 2192 goto out; 2193 2194 /* Prevent the buffers from switching */ 2195 arch_spin_lock(&global_trace.max_lock); 2196 2197 buffer = global_trace.array_buffer.buffer; 2198 if (buffer) 2199 ring_buffer_record_disable(buffer); 2200 2201 #ifdef CONFIG_TRACER_MAX_TRACE 2202 buffer = global_trace.max_buffer.buffer; 2203 if (buffer) 2204 ring_buffer_record_disable(buffer); 2205 #endif 2206 2207 arch_spin_unlock(&global_trace.max_lock); 2208 2209 out: 2210 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags); 2211 } 2212 2213 static void tracing_stop_tr(struct trace_array *tr) 2214 { 2215 struct trace_buffer *buffer; 2216 unsigned long flags; 2217 2218 /* If global, we need to also stop the max tracer */ 2219 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 2220 return tracing_stop(); 2221 2222 raw_spin_lock_irqsave(&tr->start_lock, flags); 2223 if (tr->stop_count++) 2224 goto out; 2225 2226 buffer = tr->array_buffer.buffer; 2227 if (buffer) 2228 ring_buffer_record_disable(buffer); 2229 2230 out: 2231 raw_spin_unlock_irqrestore(&tr->start_lock, flags); 2232 } 2233 2234 static int trace_save_cmdline(struct task_struct *tsk) 2235 { 2236 unsigned pid, idx; 2237 2238 /* treat recording of idle task as a success */ 2239 if (!tsk->pid) 2240 return 1; 2241 2242 if (unlikely(tsk->pid > PID_MAX_DEFAULT)) 2243 return 0; 2244 2245 /* 2246 * It's not the end of the world if we don't get 2247 * the lock, but we also don't want to spin 2248 * nor do we want to disable interrupts, 2249 * so if we miss here, then better luck next time. 2250 */ 2251 if (!arch_spin_trylock(&trace_cmdline_lock)) 2252 return 0; 2253 2254 idx = savedcmd->map_pid_to_cmdline[tsk->pid]; 2255 if (idx == NO_CMDLINE_MAP) { 2256 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num; 2257 2258 /* 2259 * Check whether the cmdline buffer at idx has a pid 2260 * mapped. We are going to overwrite that entry so we 2261 * need to clear the map_pid_to_cmdline. Otherwise we 2262 * would read the new comm for the old pid. 2263 */ 2264 pid = savedcmd->map_cmdline_to_pid[idx]; 2265 if (pid != NO_CMDLINE_MAP) 2266 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP; 2267 2268 savedcmd->map_cmdline_to_pid[idx] = tsk->pid; 2269 savedcmd->map_pid_to_cmdline[tsk->pid] = idx; 2270 2271 savedcmd->cmdline_idx = idx; 2272 } 2273 2274 set_cmdline(idx, tsk->comm); 2275 2276 arch_spin_unlock(&trace_cmdline_lock); 2277 2278 return 1; 2279 } 2280 2281 static void __trace_find_cmdline(int pid, char comm[]) 2282 { 2283 unsigned map; 2284 2285 if (!pid) { 2286 strcpy(comm, "<idle>"); 2287 return; 2288 } 2289 2290 if (WARN_ON_ONCE(pid < 0)) { 2291 strcpy(comm, "<XXX>"); 2292 return; 2293 } 2294 2295 if (pid > PID_MAX_DEFAULT) { 2296 strcpy(comm, "<...>"); 2297 return; 2298 } 2299 2300 map = savedcmd->map_pid_to_cmdline[pid]; 2301 if (map != NO_CMDLINE_MAP) 2302 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN); 2303 else 2304 strcpy(comm, "<...>"); 2305 } 2306 2307 void trace_find_cmdline(int pid, char comm[]) 2308 { 2309 preempt_disable(); 2310 arch_spin_lock(&trace_cmdline_lock); 2311 2312 __trace_find_cmdline(pid, comm); 2313 2314 arch_spin_unlock(&trace_cmdline_lock); 2315 preempt_enable(); 2316 } 2317 2318 int trace_find_tgid(int pid) 2319 { 2320 if (unlikely(!tgid_map || !pid || pid > PID_MAX_DEFAULT)) 2321 return 0; 2322 2323 return tgid_map[pid]; 2324 } 2325 2326 static int trace_save_tgid(struct task_struct *tsk) 2327 { 2328 /* treat recording of idle task as a success */ 2329 if (!tsk->pid) 2330 return 1; 2331 2332 if (unlikely(!tgid_map || tsk->pid > PID_MAX_DEFAULT)) 2333 return 0; 2334 2335 tgid_map[tsk->pid] = tsk->tgid; 2336 return 1; 2337 } 2338 2339 static bool tracing_record_taskinfo_skip(int flags) 2340 { 2341 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID)))) 2342 return true; 2343 if (atomic_read(&trace_record_taskinfo_disabled) || !tracing_is_on()) 2344 return true; 2345 if (!__this_cpu_read(trace_taskinfo_save)) 2346 return true; 2347 return false; 2348 } 2349 2350 /** 2351 * tracing_record_taskinfo - record the task info of a task 2352 * 2353 * @task: task to record 2354 * @flags: TRACE_RECORD_CMDLINE for recording comm 2355 * TRACE_RECORD_TGID for recording tgid 2356 */ 2357 void tracing_record_taskinfo(struct task_struct *task, int flags) 2358 { 2359 bool done; 2360 2361 if (tracing_record_taskinfo_skip(flags)) 2362 return; 2363 2364 /* 2365 * Record as much task information as possible. If some fail, continue 2366 * to try to record the others. 2367 */ 2368 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task); 2369 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task); 2370 2371 /* If recording any information failed, retry again soon. */ 2372 if (!done) 2373 return; 2374 2375 __this_cpu_write(trace_taskinfo_save, false); 2376 } 2377 2378 /** 2379 * tracing_record_taskinfo_sched_switch - record task info for sched_switch 2380 * 2381 * @prev: previous task during sched_switch 2382 * @next: next task during sched_switch 2383 * @flags: TRACE_RECORD_CMDLINE for recording comm 2384 * TRACE_RECORD_TGID for recording tgid 2385 */ 2386 void tracing_record_taskinfo_sched_switch(struct task_struct *prev, 2387 struct task_struct *next, int flags) 2388 { 2389 bool done; 2390 2391 if (tracing_record_taskinfo_skip(flags)) 2392 return; 2393 2394 /* 2395 * Record as much task information as possible. If some fail, continue 2396 * to try to record the others. 2397 */ 2398 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev); 2399 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next); 2400 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev); 2401 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next); 2402 2403 /* If recording any information failed, retry again soon. */ 2404 if (!done) 2405 return; 2406 2407 __this_cpu_write(trace_taskinfo_save, false); 2408 } 2409 2410 /* Helpers to record a specific task information */ 2411 void tracing_record_cmdline(struct task_struct *task) 2412 { 2413 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE); 2414 } 2415 2416 void tracing_record_tgid(struct task_struct *task) 2417 { 2418 tracing_record_taskinfo(task, TRACE_RECORD_TGID); 2419 } 2420 2421 /* 2422 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq 2423 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function 2424 * simplifies those functions and keeps them in sync. 2425 */ 2426 enum print_line_t trace_handle_return(struct trace_seq *s) 2427 { 2428 return trace_seq_has_overflowed(s) ? 2429 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED; 2430 } 2431 EXPORT_SYMBOL_GPL(trace_handle_return); 2432 2433 void 2434 tracing_generic_entry_update(struct trace_entry *entry, unsigned short type, 2435 unsigned long flags, int pc) 2436 { 2437 struct task_struct *tsk = current; 2438 2439 entry->preempt_count = pc & 0xff; 2440 entry->pid = (tsk) ? tsk->pid : 0; 2441 entry->type = type; 2442 entry->flags = 2443 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT 2444 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) | 2445 #else 2446 TRACE_FLAG_IRQS_NOSUPPORT | 2447 #endif 2448 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) | 2449 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) | 2450 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) | 2451 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) | 2452 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0); 2453 } 2454 EXPORT_SYMBOL_GPL(tracing_generic_entry_update); 2455 2456 struct ring_buffer_event * 2457 trace_buffer_lock_reserve(struct trace_buffer *buffer, 2458 int type, 2459 unsigned long len, 2460 unsigned long flags, int pc) 2461 { 2462 return __trace_buffer_lock_reserve(buffer, type, len, flags, pc); 2463 } 2464 2465 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event); 2466 DEFINE_PER_CPU(int, trace_buffered_event_cnt); 2467 static int trace_buffered_event_ref; 2468 2469 /** 2470 * trace_buffered_event_enable - enable buffering events 2471 * 2472 * When events are being filtered, it is quicker to use a temporary 2473 * buffer to write the event data into if there's a likely chance 2474 * that it will not be committed. The discard of the ring buffer 2475 * is not as fast as committing, and is much slower than copying 2476 * a commit. 2477 * 2478 * When an event is to be filtered, allocate per cpu buffers to 2479 * write the event data into, and if the event is filtered and discarded 2480 * it is simply dropped, otherwise, the entire data is to be committed 2481 * in one shot. 2482 */ 2483 void trace_buffered_event_enable(void) 2484 { 2485 struct ring_buffer_event *event; 2486 struct page *page; 2487 int cpu; 2488 2489 WARN_ON_ONCE(!mutex_is_locked(&event_mutex)); 2490 2491 if (trace_buffered_event_ref++) 2492 return; 2493 2494 for_each_tracing_cpu(cpu) { 2495 page = alloc_pages_node(cpu_to_node(cpu), 2496 GFP_KERNEL | __GFP_NORETRY, 0); 2497 if (!page) 2498 goto failed; 2499 2500 event = page_address(page); 2501 memset(event, 0, sizeof(*event)); 2502 2503 per_cpu(trace_buffered_event, cpu) = event; 2504 2505 preempt_disable(); 2506 if (cpu == smp_processor_id() && 2507 this_cpu_read(trace_buffered_event) != 2508 per_cpu(trace_buffered_event, cpu)) 2509 WARN_ON_ONCE(1); 2510 preempt_enable(); 2511 } 2512 2513 return; 2514 failed: 2515 trace_buffered_event_disable(); 2516 } 2517 2518 static void enable_trace_buffered_event(void *data) 2519 { 2520 /* Probably not needed, but do it anyway */ 2521 smp_rmb(); 2522 this_cpu_dec(trace_buffered_event_cnt); 2523 } 2524 2525 static void disable_trace_buffered_event(void *data) 2526 { 2527 this_cpu_inc(trace_buffered_event_cnt); 2528 } 2529 2530 /** 2531 * trace_buffered_event_disable - disable buffering events 2532 * 2533 * When a filter is removed, it is faster to not use the buffered 2534 * events, and to commit directly into the ring buffer. Free up 2535 * the temp buffers when there are no more users. This requires 2536 * special synchronization with current events. 2537 */ 2538 void trace_buffered_event_disable(void) 2539 { 2540 int cpu; 2541 2542 WARN_ON_ONCE(!mutex_is_locked(&event_mutex)); 2543 2544 if (WARN_ON_ONCE(!trace_buffered_event_ref)) 2545 return; 2546 2547 if (--trace_buffered_event_ref) 2548 return; 2549 2550 preempt_disable(); 2551 /* For each CPU, set the buffer as used. */ 2552 smp_call_function_many(tracing_buffer_mask, 2553 disable_trace_buffered_event, NULL, 1); 2554 preempt_enable(); 2555 2556 /* Wait for all current users to finish */ 2557 synchronize_rcu(); 2558 2559 for_each_tracing_cpu(cpu) { 2560 free_page((unsigned long)per_cpu(trace_buffered_event, cpu)); 2561 per_cpu(trace_buffered_event, cpu) = NULL; 2562 } 2563 /* 2564 * Make sure trace_buffered_event is NULL before clearing 2565 * trace_buffered_event_cnt. 2566 */ 2567 smp_wmb(); 2568 2569 preempt_disable(); 2570 /* Do the work on each cpu */ 2571 smp_call_function_many(tracing_buffer_mask, 2572 enable_trace_buffered_event, NULL, 1); 2573 preempt_enable(); 2574 } 2575 2576 static struct trace_buffer *temp_buffer; 2577 2578 struct ring_buffer_event * 2579 trace_event_buffer_lock_reserve(struct trace_buffer **current_rb, 2580 struct trace_event_file *trace_file, 2581 int type, unsigned long len, 2582 unsigned long flags, int pc) 2583 { 2584 struct ring_buffer_event *entry; 2585 int val; 2586 2587 *current_rb = trace_file->tr->array_buffer.buffer; 2588 2589 if (!ring_buffer_time_stamp_abs(*current_rb) && (trace_file->flags & 2590 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) && 2591 (entry = this_cpu_read(trace_buffered_event))) { 2592 /* Try to use the per cpu buffer first */ 2593 val = this_cpu_inc_return(trace_buffered_event_cnt); 2594 if (val == 1) { 2595 trace_event_setup(entry, type, flags, pc); 2596 entry->array[0] = len; 2597 return entry; 2598 } 2599 this_cpu_dec(trace_buffered_event_cnt); 2600 } 2601 2602 entry = __trace_buffer_lock_reserve(*current_rb, 2603 type, len, flags, pc); 2604 /* 2605 * If tracing is off, but we have triggers enabled 2606 * we still need to look at the event data. Use the temp_buffer 2607 * to store the trace event for the tigger to use. It's recusive 2608 * safe and will not be recorded anywhere. 2609 */ 2610 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) { 2611 *current_rb = temp_buffer; 2612 entry = __trace_buffer_lock_reserve(*current_rb, 2613 type, len, flags, pc); 2614 } 2615 return entry; 2616 } 2617 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve); 2618 2619 static DEFINE_SPINLOCK(tracepoint_iter_lock); 2620 static DEFINE_MUTEX(tracepoint_printk_mutex); 2621 2622 static void output_printk(struct trace_event_buffer *fbuffer) 2623 { 2624 struct trace_event_call *event_call; 2625 struct trace_event_file *file; 2626 struct trace_event *event; 2627 unsigned long flags; 2628 struct trace_iterator *iter = tracepoint_print_iter; 2629 2630 /* We should never get here if iter is NULL */ 2631 if (WARN_ON_ONCE(!iter)) 2632 return; 2633 2634 event_call = fbuffer->trace_file->event_call; 2635 if (!event_call || !event_call->event.funcs || 2636 !event_call->event.funcs->trace) 2637 return; 2638 2639 file = fbuffer->trace_file; 2640 if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) || 2641 (unlikely(file->flags & EVENT_FILE_FL_FILTERED) && 2642 !filter_match_preds(file->filter, fbuffer->entry))) 2643 return; 2644 2645 event = &fbuffer->trace_file->event_call->event; 2646 2647 spin_lock_irqsave(&tracepoint_iter_lock, flags); 2648 trace_seq_init(&iter->seq); 2649 iter->ent = fbuffer->entry; 2650 event_call->event.funcs->trace(iter, 0, event); 2651 trace_seq_putc(&iter->seq, 0); 2652 printk("%s", iter->seq.buffer); 2653 2654 spin_unlock_irqrestore(&tracepoint_iter_lock, flags); 2655 } 2656 2657 int tracepoint_printk_sysctl(struct ctl_table *table, int write, 2658 void __user *buffer, size_t *lenp, 2659 loff_t *ppos) 2660 { 2661 int save_tracepoint_printk; 2662 int ret; 2663 2664 mutex_lock(&tracepoint_printk_mutex); 2665 save_tracepoint_printk = tracepoint_printk; 2666 2667 ret = proc_dointvec(table, write, buffer, lenp, ppos); 2668 2669 /* 2670 * This will force exiting early, as tracepoint_printk 2671 * is always zero when tracepoint_printk_iter is not allocated 2672 */ 2673 if (!tracepoint_print_iter) 2674 tracepoint_printk = 0; 2675 2676 if (save_tracepoint_printk == tracepoint_printk) 2677 goto out; 2678 2679 if (tracepoint_printk) 2680 static_key_enable(&tracepoint_printk_key.key); 2681 else 2682 static_key_disable(&tracepoint_printk_key.key); 2683 2684 out: 2685 mutex_unlock(&tracepoint_printk_mutex); 2686 2687 return ret; 2688 } 2689 2690 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer) 2691 { 2692 if (static_key_false(&tracepoint_printk_key.key)) 2693 output_printk(fbuffer); 2694 2695 event_trigger_unlock_commit_regs(fbuffer->trace_file, fbuffer->buffer, 2696 fbuffer->event, fbuffer->entry, 2697 fbuffer->flags, fbuffer->pc, fbuffer->regs); 2698 } 2699 EXPORT_SYMBOL_GPL(trace_event_buffer_commit); 2700 2701 /* 2702 * Skip 3: 2703 * 2704 * trace_buffer_unlock_commit_regs() 2705 * trace_event_buffer_commit() 2706 * trace_event_raw_event_xxx() 2707 */ 2708 # define STACK_SKIP 3 2709 2710 void trace_buffer_unlock_commit_regs(struct trace_array *tr, 2711 struct trace_buffer *buffer, 2712 struct ring_buffer_event *event, 2713 unsigned long flags, int pc, 2714 struct pt_regs *regs) 2715 { 2716 __buffer_unlock_commit(buffer, event); 2717 2718 /* 2719 * If regs is not set, then skip the necessary functions. 2720 * Note, we can still get here via blktrace, wakeup tracer 2721 * and mmiotrace, but that's ok if they lose a function or 2722 * two. They are not that meaningful. 2723 */ 2724 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : STACK_SKIP, pc, regs); 2725 ftrace_trace_userstack(buffer, flags, pc); 2726 } 2727 2728 /* 2729 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack. 2730 */ 2731 void 2732 trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer, 2733 struct ring_buffer_event *event) 2734 { 2735 __buffer_unlock_commit(buffer, event); 2736 } 2737 2738 static void 2739 trace_process_export(struct trace_export *export, 2740 struct ring_buffer_event *event) 2741 { 2742 struct trace_entry *entry; 2743 unsigned int size = 0; 2744 2745 entry = ring_buffer_event_data(event); 2746 size = ring_buffer_event_length(event); 2747 export->write(export, entry, size); 2748 } 2749 2750 static DEFINE_MUTEX(ftrace_export_lock); 2751 2752 static struct trace_export __rcu *ftrace_exports_list __read_mostly; 2753 2754 static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled); 2755 2756 static inline void ftrace_exports_enable(void) 2757 { 2758 static_branch_enable(&ftrace_exports_enabled); 2759 } 2760 2761 static inline void ftrace_exports_disable(void) 2762 { 2763 static_branch_disable(&ftrace_exports_enabled); 2764 } 2765 2766 static void ftrace_exports(struct ring_buffer_event *event) 2767 { 2768 struct trace_export *export; 2769 2770 preempt_disable_notrace(); 2771 2772 export = rcu_dereference_raw_check(ftrace_exports_list); 2773 while (export) { 2774 trace_process_export(export, event); 2775 export = rcu_dereference_raw_check(export->next); 2776 } 2777 2778 preempt_enable_notrace(); 2779 } 2780 2781 static inline void 2782 add_trace_export(struct trace_export **list, struct trace_export *export) 2783 { 2784 rcu_assign_pointer(export->next, *list); 2785 /* 2786 * We are entering export into the list but another 2787 * CPU might be walking that list. We need to make sure 2788 * the export->next pointer is valid before another CPU sees 2789 * the export pointer included into the list. 2790 */ 2791 rcu_assign_pointer(*list, export); 2792 } 2793 2794 static inline int 2795 rm_trace_export(struct trace_export **list, struct trace_export *export) 2796 { 2797 struct trace_export **p; 2798 2799 for (p = list; *p != NULL; p = &(*p)->next) 2800 if (*p == export) 2801 break; 2802 2803 if (*p != export) 2804 return -1; 2805 2806 rcu_assign_pointer(*p, (*p)->next); 2807 2808 return 0; 2809 } 2810 2811 static inline void 2812 add_ftrace_export(struct trace_export **list, struct trace_export *export) 2813 { 2814 if (*list == NULL) 2815 ftrace_exports_enable(); 2816 2817 add_trace_export(list, export); 2818 } 2819 2820 static inline int 2821 rm_ftrace_export(struct trace_export **list, struct trace_export *export) 2822 { 2823 int ret; 2824 2825 ret = rm_trace_export(list, export); 2826 if (*list == NULL) 2827 ftrace_exports_disable(); 2828 2829 return ret; 2830 } 2831 2832 int register_ftrace_export(struct trace_export *export) 2833 { 2834 if (WARN_ON_ONCE(!export->write)) 2835 return -1; 2836 2837 mutex_lock(&ftrace_export_lock); 2838 2839 add_ftrace_export(&ftrace_exports_list, export); 2840 2841 mutex_unlock(&ftrace_export_lock); 2842 2843 return 0; 2844 } 2845 EXPORT_SYMBOL_GPL(register_ftrace_export); 2846 2847 int unregister_ftrace_export(struct trace_export *export) 2848 { 2849 int ret; 2850 2851 mutex_lock(&ftrace_export_lock); 2852 2853 ret = rm_ftrace_export(&ftrace_exports_list, export); 2854 2855 mutex_unlock(&ftrace_export_lock); 2856 2857 return ret; 2858 } 2859 EXPORT_SYMBOL_GPL(unregister_ftrace_export); 2860 2861 void 2862 trace_function(struct trace_array *tr, 2863 unsigned long ip, unsigned long parent_ip, unsigned long flags, 2864 int pc) 2865 { 2866 struct trace_event_call *call = &event_function; 2867 struct trace_buffer *buffer = tr->array_buffer.buffer; 2868 struct ring_buffer_event *event; 2869 struct ftrace_entry *entry; 2870 2871 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), 2872 flags, pc); 2873 if (!event) 2874 return; 2875 entry = ring_buffer_event_data(event); 2876 entry->ip = ip; 2877 entry->parent_ip = parent_ip; 2878 2879 if (!call_filter_check_discard(call, entry, buffer, event)) { 2880 if (static_branch_unlikely(&ftrace_exports_enabled)) 2881 ftrace_exports(event); 2882 __buffer_unlock_commit(buffer, event); 2883 } 2884 } 2885 2886 #ifdef CONFIG_STACKTRACE 2887 2888 /* Allow 4 levels of nesting: normal, softirq, irq, NMI */ 2889 #define FTRACE_KSTACK_NESTING 4 2890 2891 #define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING) 2892 2893 struct ftrace_stack { 2894 unsigned long calls[FTRACE_KSTACK_ENTRIES]; 2895 }; 2896 2897 2898 struct ftrace_stacks { 2899 struct ftrace_stack stacks[FTRACE_KSTACK_NESTING]; 2900 }; 2901 2902 static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks); 2903 static DEFINE_PER_CPU(int, ftrace_stack_reserve); 2904 2905 static void __ftrace_trace_stack(struct trace_buffer *buffer, 2906 unsigned long flags, 2907 int skip, int pc, struct pt_regs *regs) 2908 { 2909 struct trace_event_call *call = &event_kernel_stack; 2910 struct ring_buffer_event *event; 2911 unsigned int size, nr_entries; 2912 struct ftrace_stack *fstack; 2913 struct stack_entry *entry; 2914 int stackidx; 2915 2916 /* 2917 * Add one, for this function and the call to save_stack_trace() 2918 * If regs is set, then these functions will not be in the way. 2919 */ 2920 #ifndef CONFIG_UNWINDER_ORC 2921 if (!regs) 2922 skip++; 2923 #endif 2924 2925 /* 2926 * Since events can happen in NMIs there's no safe way to 2927 * use the per cpu ftrace_stacks. We reserve it and if an interrupt 2928 * or NMI comes in, it will just have to use the default 2929 * FTRACE_STACK_SIZE. 2930 */ 2931 preempt_disable_notrace(); 2932 2933 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1; 2934 2935 /* This should never happen. If it does, yell once and skip */ 2936 if (WARN_ON_ONCE(stackidx > FTRACE_KSTACK_NESTING)) 2937 goto out; 2938 2939 /* 2940 * The above __this_cpu_inc_return() is 'atomic' cpu local. An 2941 * interrupt will either see the value pre increment or post 2942 * increment. If the interrupt happens pre increment it will have 2943 * restored the counter when it returns. We just need a barrier to 2944 * keep gcc from moving things around. 2945 */ 2946 barrier(); 2947 2948 fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx; 2949 size = ARRAY_SIZE(fstack->calls); 2950 2951 if (regs) { 2952 nr_entries = stack_trace_save_regs(regs, fstack->calls, 2953 size, skip); 2954 } else { 2955 nr_entries = stack_trace_save(fstack->calls, size, skip); 2956 } 2957 2958 size = nr_entries * sizeof(unsigned long); 2959 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK, 2960 sizeof(*entry) + size, flags, pc); 2961 if (!event) 2962 goto out; 2963 entry = ring_buffer_event_data(event); 2964 2965 memcpy(&entry->caller, fstack->calls, size); 2966 entry->size = nr_entries; 2967 2968 if (!call_filter_check_discard(call, entry, buffer, event)) 2969 __buffer_unlock_commit(buffer, event); 2970 2971 out: 2972 /* Again, don't let gcc optimize things here */ 2973 barrier(); 2974 __this_cpu_dec(ftrace_stack_reserve); 2975 preempt_enable_notrace(); 2976 2977 } 2978 2979 static inline void ftrace_trace_stack(struct trace_array *tr, 2980 struct trace_buffer *buffer, 2981 unsigned long flags, 2982 int skip, int pc, struct pt_regs *regs) 2983 { 2984 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE)) 2985 return; 2986 2987 __ftrace_trace_stack(buffer, flags, skip, pc, regs); 2988 } 2989 2990 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, 2991 int pc) 2992 { 2993 struct trace_buffer *buffer = tr->array_buffer.buffer; 2994 2995 if (rcu_is_watching()) { 2996 __ftrace_trace_stack(buffer, flags, skip, pc, NULL); 2997 return; 2998 } 2999 3000 /* 3001 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(), 3002 * but if the above rcu_is_watching() failed, then the NMI 3003 * triggered someplace critical, and rcu_irq_enter() should 3004 * not be called from NMI. 3005 */ 3006 if (unlikely(in_nmi())) 3007 return; 3008 3009 rcu_irq_enter_irqson(); 3010 __ftrace_trace_stack(buffer, flags, skip, pc, NULL); 3011 rcu_irq_exit_irqson(); 3012 } 3013 3014 /** 3015 * trace_dump_stack - record a stack back trace in the trace buffer 3016 * @skip: Number of functions to skip (helper handlers) 3017 */ 3018 void trace_dump_stack(int skip) 3019 { 3020 unsigned long flags; 3021 3022 if (tracing_disabled || tracing_selftest_running) 3023 return; 3024 3025 local_save_flags(flags); 3026 3027 #ifndef CONFIG_UNWINDER_ORC 3028 /* Skip 1 to skip this function. */ 3029 skip++; 3030 #endif 3031 __ftrace_trace_stack(global_trace.array_buffer.buffer, 3032 flags, skip, preempt_count(), NULL); 3033 } 3034 EXPORT_SYMBOL_GPL(trace_dump_stack); 3035 3036 #ifdef CONFIG_USER_STACKTRACE_SUPPORT 3037 static DEFINE_PER_CPU(int, user_stack_count); 3038 3039 static void 3040 ftrace_trace_userstack(struct trace_buffer *buffer, unsigned long flags, int pc) 3041 { 3042 struct trace_event_call *call = &event_user_stack; 3043 struct ring_buffer_event *event; 3044 struct userstack_entry *entry; 3045 3046 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE)) 3047 return; 3048 3049 /* 3050 * NMIs can not handle page faults, even with fix ups. 3051 * The save user stack can (and often does) fault. 3052 */ 3053 if (unlikely(in_nmi())) 3054 return; 3055 3056 /* 3057 * prevent recursion, since the user stack tracing may 3058 * trigger other kernel events. 3059 */ 3060 preempt_disable(); 3061 if (__this_cpu_read(user_stack_count)) 3062 goto out; 3063 3064 __this_cpu_inc(user_stack_count); 3065 3066 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, 3067 sizeof(*entry), flags, pc); 3068 if (!event) 3069 goto out_drop_count; 3070 entry = ring_buffer_event_data(event); 3071 3072 entry->tgid = current->tgid; 3073 memset(&entry->caller, 0, sizeof(entry->caller)); 3074 3075 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES); 3076 if (!call_filter_check_discard(call, entry, buffer, event)) 3077 __buffer_unlock_commit(buffer, event); 3078 3079 out_drop_count: 3080 __this_cpu_dec(user_stack_count); 3081 out: 3082 preempt_enable(); 3083 } 3084 #else /* CONFIG_USER_STACKTRACE_SUPPORT */ 3085 static void ftrace_trace_userstack(struct trace_buffer *buffer, 3086 unsigned long flags, int pc) 3087 { 3088 } 3089 #endif /* !CONFIG_USER_STACKTRACE_SUPPORT */ 3090 3091 #endif /* CONFIG_STACKTRACE */ 3092 3093 /* created for use with alloc_percpu */ 3094 struct trace_buffer_struct { 3095 int nesting; 3096 char buffer[4][TRACE_BUF_SIZE]; 3097 }; 3098 3099 static struct trace_buffer_struct *trace_percpu_buffer; 3100 3101 /* 3102 * Thise allows for lockless recording. If we're nested too deeply, then 3103 * this returns NULL. 3104 */ 3105 static char *get_trace_buf(void) 3106 { 3107 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer); 3108 3109 if (!buffer || buffer->nesting >= 4) 3110 return NULL; 3111 3112 buffer->nesting++; 3113 3114 /* Interrupts must see nesting incremented before we use the buffer */ 3115 barrier(); 3116 return &buffer->buffer[buffer->nesting][0]; 3117 } 3118 3119 static void put_trace_buf(void) 3120 { 3121 /* Don't let the decrement of nesting leak before this */ 3122 barrier(); 3123 this_cpu_dec(trace_percpu_buffer->nesting); 3124 } 3125 3126 static int alloc_percpu_trace_buffer(void) 3127 { 3128 struct trace_buffer_struct *buffers; 3129 3130 buffers = alloc_percpu(struct trace_buffer_struct); 3131 if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer")) 3132 return -ENOMEM; 3133 3134 trace_percpu_buffer = buffers; 3135 return 0; 3136 } 3137 3138 static int buffers_allocated; 3139 3140 void trace_printk_init_buffers(void) 3141 { 3142 if (buffers_allocated) 3143 return; 3144 3145 if (alloc_percpu_trace_buffer()) 3146 return; 3147 3148 /* trace_printk() is for debug use only. Don't use it in production. */ 3149 3150 pr_warn("\n"); 3151 pr_warn("**********************************************************\n"); 3152 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 3153 pr_warn("** **\n"); 3154 pr_warn("** trace_printk() being used. Allocating extra memory. **\n"); 3155 pr_warn("** **\n"); 3156 pr_warn("** This means that this is a DEBUG kernel and it is **\n"); 3157 pr_warn("** unsafe for production use. **\n"); 3158 pr_warn("** **\n"); 3159 pr_warn("** If you see this message and you are not debugging **\n"); 3160 pr_warn("** the kernel, report this immediately to your vendor! **\n"); 3161 pr_warn("** **\n"); 3162 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 3163 pr_warn("**********************************************************\n"); 3164 3165 /* Expand the buffers to set size */ 3166 tracing_update_buffers(); 3167 3168 buffers_allocated = 1; 3169 3170 /* 3171 * trace_printk_init_buffers() can be called by modules. 3172 * If that happens, then we need to start cmdline recording 3173 * directly here. If the global_trace.buffer is already 3174 * allocated here, then this was called by module code. 3175 */ 3176 if (global_trace.array_buffer.buffer) 3177 tracing_start_cmdline_record(); 3178 } 3179 EXPORT_SYMBOL_GPL(trace_printk_init_buffers); 3180 3181 void trace_printk_start_comm(void) 3182 { 3183 /* Start tracing comms if trace printk is set */ 3184 if (!buffers_allocated) 3185 return; 3186 tracing_start_cmdline_record(); 3187 } 3188 3189 static void trace_printk_start_stop_comm(int enabled) 3190 { 3191 if (!buffers_allocated) 3192 return; 3193 3194 if (enabled) 3195 tracing_start_cmdline_record(); 3196 else 3197 tracing_stop_cmdline_record(); 3198 } 3199 3200 /** 3201 * trace_vbprintk - write binary msg to tracing buffer 3202 * @ip: The address of the caller 3203 * @fmt: The string format to write to the buffer 3204 * @args: Arguments for @fmt 3205 */ 3206 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) 3207 { 3208 struct trace_event_call *call = &event_bprint; 3209 struct ring_buffer_event *event; 3210 struct trace_buffer *buffer; 3211 struct trace_array *tr = &global_trace; 3212 struct bprint_entry *entry; 3213 unsigned long flags; 3214 char *tbuffer; 3215 int len = 0, size, pc; 3216 3217 if (unlikely(tracing_selftest_running || tracing_disabled)) 3218 return 0; 3219 3220 /* Don't pollute graph traces with trace_vprintk internals */ 3221 pause_graph_tracing(); 3222 3223 pc = preempt_count(); 3224 preempt_disable_notrace(); 3225 3226 tbuffer = get_trace_buf(); 3227 if (!tbuffer) { 3228 len = 0; 3229 goto out_nobuffer; 3230 } 3231 3232 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args); 3233 3234 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0) 3235 goto out_put; 3236 3237 local_save_flags(flags); 3238 size = sizeof(*entry) + sizeof(u32) * len; 3239 buffer = tr->array_buffer.buffer; 3240 ring_buffer_nest_start(buffer); 3241 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, 3242 flags, pc); 3243 if (!event) 3244 goto out; 3245 entry = ring_buffer_event_data(event); 3246 entry->ip = ip; 3247 entry->fmt = fmt; 3248 3249 memcpy(entry->buf, tbuffer, sizeof(u32) * len); 3250 if (!call_filter_check_discard(call, entry, buffer, event)) { 3251 __buffer_unlock_commit(buffer, event); 3252 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL); 3253 } 3254 3255 out: 3256 ring_buffer_nest_end(buffer); 3257 out_put: 3258 put_trace_buf(); 3259 3260 out_nobuffer: 3261 preempt_enable_notrace(); 3262 unpause_graph_tracing(); 3263 3264 return len; 3265 } 3266 EXPORT_SYMBOL_GPL(trace_vbprintk); 3267 3268 __printf(3, 0) 3269 static int 3270 __trace_array_vprintk(struct trace_buffer *buffer, 3271 unsigned long ip, const char *fmt, va_list args) 3272 { 3273 struct trace_event_call *call = &event_print; 3274 struct ring_buffer_event *event; 3275 int len = 0, size, pc; 3276 struct print_entry *entry; 3277 unsigned long flags; 3278 char *tbuffer; 3279 3280 if (tracing_disabled || tracing_selftest_running) 3281 return 0; 3282 3283 /* Don't pollute graph traces with trace_vprintk internals */ 3284 pause_graph_tracing(); 3285 3286 pc = preempt_count(); 3287 preempt_disable_notrace(); 3288 3289 3290 tbuffer = get_trace_buf(); 3291 if (!tbuffer) { 3292 len = 0; 3293 goto out_nobuffer; 3294 } 3295 3296 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args); 3297 3298 local_save_flags(flags); 3299 size = sizeof(*entry) + len + 1; 3300 ring_buffer_nest_start(buffer); 3301 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 3302 flags, pc); 3303 if (!event) 3304 goto out; 3305 entry = ring_buffer_event_data(event); 3306 entry->ip = ip; 3307 3308 memcpy(&entry->buf, tbuffer, len + 1); 3309 if (!call_filter_check_discard(call, entry, buffer, event)) { 3310 __buffer_unlock_commit(buffer, event); 3311 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL); 3312 } 3313 3314 out: 3315 ring_buffer_nest_end(buffer); 3316 put_trace_buf(); 3317 3318 out_nobuffer: 3319 preempt_enable_notrace(); 3320 unpause_graph_tracing(); 3321 3322 return len; 3323 } 3324 3325 __printf(3, 0) 3326 int trace_array_vprintk(struct trace_array *tr, 3327 unsigned long ip, const char *fmt, va_list args) 3328 { 3329 return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args); 3330 } 3331 3332 __printf(3, 0) 3333 int trace_array_printk(struct trace_array *tr, 3334 unsigned long ip, const char *fmt, ...) 3335 { 3336 int ret; 3337 va_list ap; 3338 3339 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 3340 return 0; 3341 3342 if (!tr) 3343 return -ENOENT; 3344 3345 va_start(ap, fmt); 3346 ret = trace_array_vprintk(tr, ip, fmt, ap); 3347 va_end(ap); 3348 return ret; 3349 } 3350 EXPORT_SYMBOL_GPL(trace_array_printk); 3351 3352 __printf(3, 4) 3353 int trace_array_printk_buf(struct trace_buffer *buffer, 3354 unsigned long ip, const char *fmt, ...) 3355 { 3356 int ret; 3357 va_list ap; 3358 3359 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 3360 return 0; 3361 3362 va_start(ap, fmt); 3363 ret = __trace_array_vprintk(buffer, ip, fmt, ap); 3364 va_end(ap); 3365 return ret; 3366 } 3367 3368 __printf(2, 0) 3369 int trace_vprintk(unsigned long ip, const char *fmt, va_list args) 3370 { 3371 return trace_array_vprintk(&global_trace, ip, fmt, args); 3372 } 3373 EXPORT_SYMBOL_GPL(trace_vprintk); 3374 3375 static void trace_iterator_increment(struct trace_iterator *iter) 3376 { 3377 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu); 3378 3379 iter->idx++; 3380 if (buf_iter) 3381 ring_buffer_read(buf_iter, NULL); 3382 } 3383 3384 static struct trace_entry * 3385 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts, 3386 unsigned long *lost_events) 3387 { 3388 struct ring_buffer_event *event; 3389 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu); 3390 3391 if (buf_iter) 3392 event = ring_buffer_iter_peek(buf_iter, ts); 3393 else 3394 event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts, 3395 lost_events); 3396 3397 if (event) { 3398 iter->ent_size = ring_buffer_event_length(event); 3399 return ring_buffer_event_data(event); 3400 } 3401 iter->ent_size = 0; 3402 return NULL; 3403 } 3404 3405 static struct trace_entry * 3406 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, 3407 unsigned long *missing_events, u64 *ent_ts) 3408 { 3409 struct trace_buffer *buffer = iter->array_buffer->buffer; 3410 struct trace_entry *ent, *next = NULL; 3411 unsigned long lost_events = 0, next_lost = 0; 3412 int cpu_file = iter->cpu_file; 3413 u64 next_ts = 0, ts; 3414 int next_cpu = -1; 3415 int next_size = 0; 3416 int cpu; 3417 3418 /* 3419 * If we are in a per_cpu trace file, don't bother by iterating over 3420 * all cpu and peek directly. 3421 */ 3422 if (cpu_file > RING_BUFFER_ALL_CPUS) { 3423 if (ring_buffer_empty_cpu(buffer, cpu_file)) 3424 return NULL; 3425 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events); 3426 if (ent_cpu) 3427 *ent_cpu = cpu_file; 3428 3429 return ent; 3430 } 3431 3432 for_each_tracing_cpu(cpu) { 3433 3434 if (ring_buffer_empty_cpu(buffer, cpu)) 3435 continue; 3436 3437 ent = peek_next_entry(iter, cpu, &ts, &lost_events); 3438 3439 /* 3440 * Pick the entry with the smallest timestamp: 3441 */ 3442 if (ent && (!next || ts < next_ts)) { 3443 next = ent; 3444 next_cpu = cpu; 3445 next_ts = ts; 3446 next_lost = lost_events; 3447 next_size = iter->ent_size; 3448 } 3449 } 3450 3451 iter->ent_size = next_size; 3452 3453 if (ent_cpu) 3454 *ent_cpu = next_cpu; 3455 3456 if (ent_ts) 3457 *ent_ts = next_ts; 3458 3459 if (missing_events) 3460 *missing_events = next_lost; 3461 3462 return next; 3463 } 3464 3465 /* Find the next real entry, without updating the iterator itself */ 3466 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter, 3467 int *ent_cpu, u64 *ent_ts) 3468 { 3469 return __find_next_entry(iter, ent_cpu, NULL, ent_ts); 3470 } 3471 3472 /* Find the next real entry, and increment the iterator to the next entry */ 3473 void *trace_find_next_entry_inc(struct trace_iterator *iter) 3474 { 3475 iter->ent = __find_next_entry(iter, &iter->cpu, 3476 &iter->lost_events, &iter->ts); 3477 3478 if (iter->ent) 3479 trace_iterator_increment(iter); 3480 3481 return iter->ent ? iter : NULL; 3482 } 3483 3484 static void trace_consume(struct trace_iterator *iter) 3485 { 3486 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts, 3487 &iter->lost_events); 3488 } 3489 3490 static void *s_next(struct seq_file *m, void *v, loff_t *pos) 3491 { 3492 struct trace_iterator *iter = m->private; 3493 int i = (int)*pos; 3494 void *ent; 3495 3496 WARN_ON_ONCE(iter->leftover); 3497 3498 (*pos)++; 3499 3500 /* can't go backwards */ 3501 if (iter->idx > i) 3502 return NULL; 3503 3504 if (iter->idx < 0) 3505 ent = trace_find_next_entry_inc(iter); 3506 else 3507 ent = iter; 3508 3509 while (ent && iter->idx < i) 3510 ent = trace_find_next_entry_inc(iter); 3511 3512 iter->pos = *pos; 3513 3514 return ent; 3515 } 3516 3517 void tracing_iter_reset(struct trace_iterator *iter, int cpu) 3518 { 3519 struct ring_buffer_event *event; 3520 struct ring_buffer_iter *buf_iter; 3521 unsigned long entries = 0; 3522 u64 ts; 3523 3524 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0; 3525 3526 buf_iter = trace_buffer_iter(iter, cpu); 3527 if (!buf_iter) 3528 return; 3529 3530 ring_buffer_iter_reset(buf_iter); 3531 3532 /* 3533 * We could have the case with the max latency tracers 3534 * that a reset never took place on a cpu. This is evident 3535 * by the timestamp being before the start of the buffer. 3536 */ 3537 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) { 3538 if (ts >= iter->array_buffer->time_start) 3539 break; 3540 entries++; 3541 ring_buffer_read(buf_iter, NULL); 3542 } 3543 3544 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries; 3545 } 3546 3547 /* 3548 * The current tracer is copied to avoid a global locking 3549 * all around. 3550 */ 3551 static void *s_start(struct seq_file *m, loff_t *pos) 3552 { 3553 struct trace_iterator *iter = m->private; 3554 struct trace_array *tr = iter->tr; 3555 int cpu_file = iter->cpu_file; 3556 void *p = NULL; 3557 loff_t l = 0; 3558 int cpu; 3559 3560 /* 3561 * copy the tracer to avoid using a global lock all around. 3562 * iter->trace is a copy of current_trace, the pointer to the 3563 * name may be used instead of a strcmp(), as iter->trace->name 3564 * will point to the same string as current_trace->name. 3565 */ 3566 mutex_lock(&trace_types_lock); 3567 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name)) 3568 *iter->trace = *tr->current_trace; 3569 mutex_unlock(&trace_types_lock); 3570 3571 #ifdef CONFIG_TRACER_MAX_TRACE 3572 if (iter->snapshot && iter->trace->use_max_tr) 3573 return ERR_PTR(-EBUSY); 3574 #endif 3575 3576 if (!iter->snapshot) 3577 atomic_inc(&trace_record_taskinfo_disabled); 3578 3579 if (*pos != iter->pos) { 3580 iter->ent = NULL; 3581 iter->cpu = 0; 3582 iter->idx = -1; 3583 3584 if (cpu_file == RING_BUFFER_ALL_CPUS) { 3585 for_each_tracing_cpu(cpu) 3586 tracing_iter_reset(iter, cpu); 3587 } else 3588 tracing_iter_reset(iter, cpu_file); 3589 3590 iter->leftover = 0; 3591 for (p = iter; p && l < *pos; p = s_next(m, p, &l)) 3592 ; 3593 3594 } else { 3595 /* 3596 * If we overflowed the seq_file before, then we want 3597 * to just reuse the trace_seq buffer again. 3598 */ 3599 if (iter->leftover) 3600 p = iter; 3601 else { 3602 l = *pos - 1; 3603 p = s_next(m, p, &l); 3604 } 3605 } 3606 3607 trace_event_read_lock(); 3608 trace_access_lock(cpu_file); 3609 return p; 3610 } 3611 3612 static void s_stop(struct seq_file *m, void *p) 3613 { 3614 struct trace_iterator *iter = m->private; 3615 3616 #ifdef CONFIG_TRACER_MAX_TRACE 3617 if (iter->snapshot && iter->trace->use_max_tr) 3618 return; 3619 #endif 3620 3621 if (!iter->snapshot) 3622 atomic_dec(&trace_record_taskinfo_disabled); 3623 3624 trace_access_unlock(iter->cpu_file); 3625 trace_event_read_unlock(); 3626 } 3627 3628 static void 3629 get_total_entries_cpu(struct array_buffer *buf, unsigned long *total, 3630 unsigned long *entries, int cpu) 3631 { 3632 unsigned long count; 3633 3634 count = ring_buffer_entries_cpu(buf->buffer, cpu); 3635 /* 3636 * If this buffer has skipped entries, then we hold all 3637 * entries for the trace and we need to ignore the 3638 * ones before the time stamp. 3639 */ 3640 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) { 3641 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries; 3642 /* total is the same as the entries */ 3643 *total = count; 3644 } else 3645 *total = count + 3646 ring_buffer_overrun_cpu(buf->buffer, cpu); 3647 *entries = count; 3648 } 3649 3650 static void 3651 get_total_entries(struct array_buffer *buf, 3652 unsigned long *total, unsigned long *entries) 3653 { 3654 unsigned long t, e; 3655 int cpu; 3656 3657 *total = 0; 3658 *entries = 0; 3659 3660 for_each_tracing_cpu(cpu) { 3661 get_total_entries_cpu(buf, &t, &e, cpu); 3662 *total += t; 3663 *entries += e; 3664 } 3665 } 3666 3667 unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu) 3668 { 3669 unsigned long total, entries; 3670 3671 if (!tr) 3672 tr = &global_trace; 3673 3674 get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu); 3675 3676 return entries; 3677 } 3678 3679 unsigned long trace_total_entries(struct trace_array *tr) 3680 { 3681 unsigned long total, entries; 3682 3683 if (!tr) 3684 tr = &global_trace; 3685 3686 get_total_entries(&tr->array_buffer, &total, &entries); 3687 3688 return entries; 3689 } 3690 3691 static void print_lat_help_header(struct seq_file *m) 3692 { 3693 seq_puts(m, "# _------=> CPU# \n" 3694 "# / _-----=> irqs-off \n" 3695 "# | / _----=> need-resched \n" 3696 "# || / _---=> hardirq/softirq \n" 3697 "# ||| / _--=> preempt-depth \n" 3698 "# |||| / delay \n" 3699 "# cmd pid ||||| time | caller \n" 3700 "# \\ / ||||| \\ | / \n"); 3701 } 3702 3703 static void print_event_info(struct array_buffer *buf, struct seq_file *m) 3704 { 3705 unsigned long total; 3706 unsigned long entries; 3707 3708 get_total_entries(buf, &total, &entries); 3709 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n", 3710 entries, total, num_online_cpus()); 3711 seq_puts(m, "#\n"); 3712 } 3713 3714 static void print_func_help_header(struct array_buffer *buf, struct seq_file *m, 3715 unsigned int flags) 3716 { 3717 bool tgid = flags & TRACE_ITER_RECORD_TGID; 3718 3719 print_event_info(buf, m); 3720 3721 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? "TGID " : ""); 3722 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : ""); 3723 } 3724 3725 static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file *m, 3726 unsigned int flags) 3727 { 3728 bool tgid = flags & TRACE_ITER_RECORD_TGID; 3729 const char *space = " "; 3730 int prec = tgid ? 10 : 2; 3731 3732 print_event_info(buf, m); 3733 3734 seq_printf(m, "# %.*s _-----=> irqs-off\n", prec, space); 3735 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space); 3736 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space); 3737 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space); 3738 seq_printf(m, "# %.*s||| / delay\n", prec, space); 3739 seq_printf(m, "# TASK-PID %.*sCPU# |||| TIMESTAMP FUNCTION\n", prec, " TGID "); 3740 seq_printf(m, "# | | %.*s | |||| | |\n", prec, " | "); 3741 } 3742 3743 void 3744 print_trace_header(struct seq_file *m, struct trace_iterator *iter) 3745 { 3746 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK); 3747 struct array_buffer *buf = iter->array_buffer; 3748 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu); 3749 struct tracer *type = iter->trace; 3750 unsigned long entries; 3751 unsigned long total; 3752 const char *name = "preemption"; 3753 3754 name = type->name; 3755 3756 get_total_entries(buf, &total, &entries); 3757 3758 seq_printf(m, "# %s latency trace v1.1.5 on %s\n", 3759 name, UTS_RELEASE); 3760 seq_puts(m, "# -----------------------------------" 3761 "---------------------------------\n"); 3762 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |" 3763 " (M:%s VP:%d, KP:%d, SP:%d HP:%d", 3764 nsecs_to_usecs(data->saved_latency), 3765 entries, 3766 total, 3767 buf->cpu, 3768 #if defined(CONFIG_PREEMPT_NONE) 3769 "server", 3770 #elif defined(CONFIG_PREEMPT_VOLUNTARY) 3771 "desktop", 3772 #elif defined(CONFIG_PREEMPT) 3773 "preempt", 3774 #elif defined(CONFIG_PREEMPT_RT) 3775 "preempt_rt", 3776 #else 3777 "unknown", 3778 #endif 3779 /* These are reserved for later use */ 3780 0, 0, 0, 0); 3781 #ifdef CONFIG_SMP 3782 seq_printf(m, " #P:%d)\n", num_online_cpus()); 3783 #else 3784 seq_puts(m, ")\n"); 3785 #endif 3786 seq_puts(m, "# -----------------\n"); 3787 seq_printf(m, "# | task: %.16s-%d " 3788 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n", 3789 data->comm, data->pid, 3790 from_kuid_munged(seq_user_ns(m), data->uid), data->nice, 3791 data->policy, data->rt_priority); 3792 seq_puts(m, "# -----------------\n"); 3793 3794 if (data->critical_start) { 3795 seq_puts(m, "# => started at: "); 3796 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags); 3797 trace_print_seq(m, &iter->seq); 3798 seq_puts(m, "\n# => ended at: "); 3799 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); 3800 trace_print_seq(m, &iter->seq); 3801 seq_puts(m, "\n#\n"); 3802 } 3803 3804 seq_puts(m, "#\n"); 3805 } 3806 3807 static void test_cpu_buff_start(struct trace_iterator *iter) 3808 { 3809 struct trace_seq *s = &iter->seq; 3810 struct trace_array *tr = iter->tr; 3811 3812 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE)) 3813 return; 3814 3815 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE)) 3816 return; 3817 3818 if (cpumask_available(iter->started) && 3819 cpumask_test_cpu(iter->cpu, iter->started)) 3820 return; 3821 3822 if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries) 3823 return; 3824 3825 if (cpumask_available(iter->started)) 3826 cpumask_set_cpu(iter->cpu, iter->started); 3827 3828 /* Don't print started cpu buffer for the first entry of the trace */ 3829 if (iter->idx > 1) 3830 trace_seq_printf(s, "##### CPU %u buffer started ####\n", 3831 iter->cpu); 3832 } 3833 3834 static enum print_line_t print_trace_fmt(struct trace_iterator *iter) 3835 { 3836 struct trace_array *tr = iter->tr; 3837 struct trace_seq *s = &iter->seq; 3838 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK); 3839 struct trace_entry *entry; 3840 struct trace_event *event; 3841 3842 entry = iter->ent; 3843 3844 test_cpu_buff_start(iter); 3845 3846 event = ftrace_find_event(entry->type); 3847 3848 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 3849 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 3850 trace_print_lat_context(iter); 3851 else 3852 trace_print_context(iter); 3853 } 3854 3855 if (trace_seq_has_overflowed(s)) 3856 return TRACE_TYPE_PARTIAL_LINE; 3857 3858 if (event) 3859 return event->funcs->trace(iter, sym_flags, event); 3860 3861 trace_seq_printf(s, "Unknown type %d\n", entry->type); 3862 3863 return trace_handle_return(s); 3864 } 3865 3866 static enum print_line_t print_raw_fmt(struct trace_iterator *iter) 3867 { 3868 struct trace_array *tr = iter->tr; 3869 struct trace_seq *s = &iter->seq; 3870 struct trace_entry *entry; 3871 struct trace_event *event; 3872 3873 entry = iter->ent; 3874 3875 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) 3876 trace_seq_printf(s, "%d %d %llu ", 3877 entry->pid, iter->cpu, iter->ts); 3878 3879 if (trace_seq_has_overflowed(s)) 3880 return TRACE_TYPE_PARTIAL_LINE; 3881 3882 event = ftrace_find_event(entry->type); 3883 if (event) 3884 return event->funcs->raw(iter, 0, event); 3885 3886 trace_seq_printf(s, "%d ?\n", entry->type); 3887 3888 return trace_handle_return(s); 3889 } 3890 3891 static enum print_line_t print_hex_fmt(struct trace_iterator *iter) 3892 { 3893 struct trace_array *tr = iter->tr; 3894 struct trace_seq *s = &iter->seq; 3895 unsigned char newline = '\n'; 3896 struct trace_entry *entry; 3897 struct trace_event *event; 3898 3899 entry = iter->ent; 3900 3901 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 3902 SEQ_PUT_HEX_FIELD(s, entry->pid); 3903 SEQ_PUT_HEX_FIELD(s, iter->cpu); 3904 SEQ_PUT_HEX_FIELD(s, iter->ts); 3905 if (trace_seq_has_overflowed(s)) 3906 return TRACE_TYPE_PARTIAL_LINE; 3907 } 3908 3909 event = ftrace_find_event(entry->type); 3910 if (event) { 3911 enum print_line_t ret = event->funcs->hex(iter, 0, event); 3912 if (ret != TRACE_TYPE_HANDLED) 3913 return ret; 3914 } 3915 3916 SEQ_PUT_FIELD(s, newline); 3917 3918 return trace_handle_return(s); 3919 } 3920 3921 static enum print_line_t print_bin_fmt(struct trace_iterator *iter) 3922 { 3923 struct trace_array *tr = iter->tr; 3924 struct trace_seq *s = &iter->seq; 3925 struct trace_entry *entry; 3926 struct trace_event *event; 3927 3928 entry = iter->ent; 3929 3930 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 3931 SEQ_PUT_FIELD(s, entry->pid); 3932 SEQ_PUT_FIELD(s, iter->cpu); 3933 SEQ_PUT_FIELD(s, iter->ts); 3934 if (trace_seq_has_overflowed(s)) 3935 return TRACE_TYPE_PARTIAL_LINE; 3936 } 3937 3938 event = ftrace_find_event(entry->type); 3939 return event ? event->funcs->binary(iter, 0, event) : 3940 TRACE_TYPE_HANDLED; 3941 } 3942 3943 int trace_empty(struct trace_iterator *iter) 3944 { 3945 struct ring_buffer_iter *buf_iter; 3946 int cpu; 3947 3948 /* If we are looking at one CPU buffer, only check that one */ 3949 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 3950 cpu = iter->cpu_file; 3951 buf_iter = trace_buffer_iter(iter, cpu); 3952 if (buf_iter) { 3953 if (!ring_buffer_iter_empty(buf_iter)) 3954 return 0; 3955 } else { 3956 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu)) 3957 return 0; 3958 } 3959 return 1; 3960 } 3961 3962 for_each_tracing_cpu(cpu) { 3963 buf_iter = trace_buffer_iter(iter, cpu); 3964 if (buf_iter) { 3965 if (!ring_buffer_iter_empty(buf_iter)) 3966 return 0; 3967 } else { 3968 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu)) 3969 return 0; 3970 } 3971 } 3972 3973 return 1; 3974 } 3975 3976 /* Called with trace_event_read_lock() held. */ 3977 enum print_line_t print_trace_line(struct trace_iterator *iter) 3978 { 3979 struct trace_array *tr = iter->tr; 3980 unsigned long trace_flags = tr->trace_flags; 3981 enum print_line_t ret; 3982 3983 if (iter->lost_events) { 3984 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", 3985 iter->cpu, iter->lost_events); 3986 if (trace_seq_has_overflowed(&iter->seq)) 3987 return TRACE_TYPE_PARTIAL_LINE; 3988 } 3989 3990 if (iter->trace && iter->trace->print_line) { 3991 ret = iter->trace->print_line(iter); 3992 if (ret != TRACE_TYPE_UNHANDLED) 3993 return ret; 3994 } 3995 3996 if (iter->ent->type == TRACE_BPUTS && 3997 trace_flags & TRACE_ITER_PRINTK && 3998 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 3999 return trace_print_bputs_msg_only(iter); 4000 4001 if (iter->ent->type == TRACE_BPRINT && 4002 trace_flags & TRACE_ITER_PRINTK && 4003 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 4004 return trace_print_bprintk_msg_only(iter); 4005 4006 if (iter->ent->type == TRACE_PRINT && 4007 trace_flags & TRACE_ITER_PRINTK && 4008 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 4009 return trace_print_printk_msg_only(iter); 4010 4011 if (trace_flags & TRACE_ITER_BIN) 4012 return print_bin_fmt(iter); 4013 4014 if (trace_flags & TRACE_ITER_HEX) 4015 return print_hex_fmt(iter); 4016 4017 if (trace_flags & TRACE_ITER_RAW) 4018 return print_raw_fmt(iter); 4019 4020 return print_trace_fmt(iter); 4021 } 4022 4023 void trace_latency_header(struct seq_file *m) 4024 { 4025 struct trace_iterator *iter = m->private; 4026 struct trace_array *tr = iter->tr; 4027 4028 /* print nothing if the buffers are empty */ 4029 if (trace_empty(iter)) 4030 return; 4031 4032 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 4033 print_trace_header(m, iter); 4034 4035 if (!(tr->trace_flags & TRACE_ITER_VERBOSE)) 4036 print_lat_help_header(m); 4037 } 4038 4039 void trace_default_header(struct seq_file *m) 4040 { 4041 struct trace_iterator *iter = m->private; 4042 struct trace_array *tr = iter->tr; 4043 unsigned long trace_flags = tr->trace_flags; 4044 4045 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO)) 4046 return; 4047 4048 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { 4049 /* print nothing if the buffers are empty */ 4050 if (trace_empty(iter)) 4051 return; 4052 print_trace_header(m, iter); 4053 if (!(trace_flags & TRACE_ITER_VERBOSE)) 4054 print_lat_help_header(m); 4055 } else { 4056 if (!(trace_flags & TRACE_ITER_VERBOSE)) { 4057 if (trace_flags & TRACE_ITER_IRQ_INFO) 4058 print_func_help_header_irq(iter->array_buffer, 4059 m, trace_flags); 4060 else 4061 print_func_help_header(iter->array_buffer, m, 4062 trace_flags); 4063 } 4064 } 4065 } 4066 4067 static void test_ftrace_alive(struct seq_file *m) 4068 { 4069 if (!ftrace_is_dead()) 4070 return; 4071 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n" 4072 "# MAY BE MISSING FUNCTION EVENTS\n"); 4073 } 4074 4075 #ifdef CONFIG_TRACER_MAX_TRACE 4076 static void show_snapshot_main_help(struct seq_file *m) 4077 { 4078 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n" 4079 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" 4080 "# Takes a snapshot of the main buffer.\n" 4081 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n" 4082 "# (Doesn't have to be '2' works with any number that\n" 4083 "# is not a '0' or '1')\n"); 4084 } 4085 4086 static void show_snapshot_percpu_help(struct seq_file *m) 4087 { 4088 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n"); 4089 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP 4090 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" 4091 "# Takes a snapshot of the main buffer for this cpu.\n"); 4092 #else 4093 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n" 4094 "# Must use main snapshot file to allocate.\n"); 4095 #endif 4096 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n" 4097 "# (Doesn't have to be '2' works with any number that\n" 4098 "# is not a '0' or '1')\n"); 4099 } 4100 4101 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) 4102 { 4103 if (iter->tr->allocated_snapshot) 4104 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n"); 4105 else 4106 seq_puts(m, "#\n# * Snapshot is freed *\n#\n"); 4107 4108 seq_puts(m, "# Snapshot commands:\n"); 4109 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 4110 show_snapshot_main_help(m); 4111 else 4112 show_snapshot_percpu_help(m); 4113 } 4114 #else 4115 /* Should never be called */ 4116 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { } 4117 #endif 4118 4119 static int s_show(struct seq_file *m, void *v) 4120 { 4121 struct trace_iterator *iter = v; 4122 int ret; 4123 4124 if (iter->ent == NULL) { 4125 if (iter->tr) { 4126 seq_printf(m, "# tracer: %s\n", iter->trace->name); 4127 seq_puts(m, "#\n"); 4128 test_ftrace_alive(m); 4129 } 4130 if (iter->snapshot && trace_empty(iter)) 4131 print_snapshot_help(m, iter); 4132 else if (iter->trace && iter->trace->print_header) 4133 iter->trace->print_header(m); 4134 else 4135 trace_default_header(m); 4136 4137 } else if (iter->leftover) { 4138 /* 4139 * If we filled the seq_file buffer earlier, we 4140 * want to just show it now. 4141 */ 4142 ret = trace_print_seq(m, &iter->seq); 4143 4144 /* ret should this time be zero, but you never know */ 4145 iter->leftover = ret; 4146 4147 } else { 4148 print_trace_line(iter); 4149 ret = trace_print_seq(m, &iter->seq); 4150 /* 4151 * If we overflow the seq_file buffer, then it will 4152 * ask us for this data again at start up. 4153 * Use that instead. 4154 * ret is 0 if seq_file write succeeded. 4155 * -1 otherwise. 4156 */ 4157 iter->leftover = ret; 4158 } 4159 4160 return 0; 4161 } 4162 4163 /* 4164 * Should be used after trace_array_get(), trace_types_lock 4165 * ensures that i_cdev was already initialized. 4166 */ 4167 static inline int tracing_get_cpu(struct inode *inode) 4168 { 4169 if (inode->i_cdev) /* See trace_create_cpu_file() */ 4170 return (long)inode->i_cdev - 1; 4171 return RING_BUFFER_ALL_CPUS; 4172 } 4173 4174 static const struct seq_operations tracer_seq_ops = { 4175 .start = s_start, 4176 .next = s_next, 4177 .stop = s_stop, 4178 .show = s_show, 4179 }; 4180 4181 static struct trace_iterator * 4182 __tracing_open(struct inode *inode, struct file *file, bool snapshot) 4183 { 4184 struct trace_array *tr = inode->i_private; 4185 struct trace_iterator *iter; 4186 int cpu; 4187 4188 if (tracing_disabled) 4189 return ERR_PTR(-ENODEV); 4190 4191 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter)); 4192 if (!iter) 4193 return ERR_PTR(-ENOMEM); 4194 4195 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter), 4196 GFP_KERNEL); 4197 if (!iter->buffer_iter) 4198 goto release; 4199 4200 /* 4201 * We make a copy of the current tracer to avoid concurrent 4202 * changes on it while we are reading. 4203 */ 4204 mutex_lock(&trace_types_lock); 4205 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL); 4206 if (!iter->trace) 4207 goto fail; 4208 4209 *iter->trace = *tr->current_trace; 4210 4211 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL)) 4212 goto fail; 4213 4214 iter->tr = tr; 4215 4216 #ifdef CONFIG_TRACER_MAX_TRACE 4217 /* Currently only the top directory has a snapshot */ 4218 if (tr->current_trace->print_max || snapshot) 4219 iter->array_buffer = &tr->max_buffer; 4220 else 4221 #endif 4222 iter->array_buffer = &tr->array_buffer; 4223 iter->snapshot = snapshot; 4224 iter->pos = -1; 4225 iter->cpu_file = tracing_get_cpu(inode); 4226 mutex_init(&iter->mutex); 4227 4228 /* Notify the tracer early; before we stop tracing. */ 4229 if (iter->trace->open) 4230 iter->trace->open(iter); 4231 4232 /* Annotate start of buffers if we had overruns */ 4233 if (ring_buffer_overruns(iter->array_buffer->buffer)) 4234 iter->iter_flags |= TRACE_FILE_ANNOTATE; 4235 4236 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 4237 if (trace_clocks[tr->clock_id].in_ns) 4238 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 4239 4240 /* stop the trace while dumping if we are not opening "snapshot" */ 4241 if (!iter->snapshot) 4242 tracing_stop_tr(tr); 4243 4244 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { 4245 for_each_tracing_cpu(cpu) { 4246 iter->buffer_iter[cpu] = 4247 ring_buffer_read_prepare(iter->array_buffer->buffer, 4248 cpu, GFP_KERNEL); 4249 } 4250 ring_buffer_read_prepare_sync(); 4251 for_each_tracing_cpu(cpu) { 4252 ring_buffer_read_start(iter->buffer_iter[cpu]); 4253 tracing_iter_reset(iter, cpu); 4254 } 4255 } else { 4256 cpu = iter->cpu_file; 4257 iter->buffer_iter[cpu] = 4258 ring_buffer_read_prepare(iter->array_buffer->buffer, 4259 cpu, GFP_KERNEL); 4260 ring_buffer_read_prepare_sync(); 4261 ring_buffer_read_start(iter->buffer_iter[cpu]); 4262 tracing_iter_reset(iter, cpu); 4263 } 4264 4265 mutex_unlock(&trace_types_lock); 4266 4267 return iter; 4268 4269 fail: 4270 mutex_unlock(&trace_types_lock); 4271 kfree(iter->trace); 4272 kfree(iter->buffer_iter); 4273 release: 4274 seq_release_private(inode, file); 4275 return ERR_PTR(-ENOMEM); 4276 } 4277 4278 int tracing_open_generic(struct inode *inode, struct file *filp) 4279 { 4280 int ret; 4281 4282 ret = tracing_check_open_get_tr(NULL); 4283 if (ret) 4284 return ret; 4285 4286 filp->private_data = inode->i_private; 4287 return 0; 4288 } 4289 4290 bool tracing_is_disabled(void) 4291 { 4292 return (tracing_disabled) ? true: false; 4293 } 4294 4295 /* 4296 * Open and update trace_array ref count. 4297 * Must have the current trace_array passed to it. 4298 */ 4299 int tracing_open_generic_tr(struct inode *inode, struct file *filp) 4300 { 4301 struct trace_array *tr = inode->i_private; 4302 int ret; 4303 4304 ret = tracing_check_open_get_tr(tr); 4305 if (ret) 4306 return ret; 4307 4308 filp->private_data = inode->i_private; 4309 4310 return 0; 4311 } 4312 4313 static int tracing_release(struct inode *inode, struct file *file) 4314 { 4315 struct trace_array *tr = inode->i_private; 4316 struct seq_file *m = file->private_data; 4317 struct trace_iterator *iter; 4318 int cpu; 4319 4320 if (!(file->f_mode & FMODE_READ)) { 4321 trace_array_put(tr); 4322 return 0; 4323 } 4324 4325 /* Writes do not use seq_file */ 4326 iter = m->private; 4327 mutex_lock(&trace_types_lock); 4328 4329 for_each_tracing_cpu(cpu) { 4330 if (iter->buffer_iter[cpu]) 4331 ring_buffer_read_finish(iter->buffer_iter[cpu]); 4332 } 4333 4334 if (iter->trace && iter->trace->close) 4335 iter->trace->close(iter); 4336 4337 if (!iter->snapshot) 4338 /* reenable tracing if it was previously enabled */ 4339 tracing_start_tr(tr); 4340 4341 __trace_array_put(tr); 4342 4343 mutex_unlock(&trace_types_lock); 4344 4345 mutex_destroy(&iter->mutex); 4346 free_cpumask_var(iter->started); 4347 kfree(iter->trace); 4348 kfree(iter->buffer_iter); 4349 seq_release_private(inode, file); 4350 4351 return 0; 4352 } 4353 4354 static int tracing_release_generic_tr(struct inode *inode, struct file *file) 4355 { 4356 struct trace_array *tr = inode->i_private; 4357 4358 trace_array_put(tr); 4359 return 0; 4360 } 4361 4362 static int tracing_single_release_tr(struct inode *inode, struct file *file) 4363 { 4364 struct trace_array *tr = inode->i_private; 4365 4366 trace_array_put(tr); 4367 4368 return single_release(inode, file); 4369 } 4370 4371 static int tracing_open(struct inode *inode, struct file *file) 4372 { 4373 struct trace_array *tr = inode->i_private; 4374 struct trace_iterator *iter; 4375 int ret; 4376 4377 ret = tracing_check_open_get_tr(tr); 4378 if (ret) 4379 return ret; 4380 4381 /* If this file was open for write, then erase contents */ 4382 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 4383 int cpu = tracing_get_cpu(inode); 4384 struct array_buffer *trace_buf = &tr->array_buffer; 4385 4386 #ifdef CONFIG_TRACER_MAX_TRACE 4387 if (tr->current_trace->print_max) 4388 trace_buf = &tr->max_buffer; 4389 #endif 4390 4391 if (cpu == RING_BUFFER_ALL_CPUS) 4392 tracing_reset_online_cpus(trace_buf); 4393 else 4394 tracing_reset_cpu(trace_buf, cpu); 4395 } 4396 4397 if (file->f_mode & FMODE_READ) { 4398 iter = __tracing_open(inode, file, false); 4399 if (IS_ERR(iter)) 4400 ret = PTR_ERR(iter); 4401 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 4402 iter->iter_flags |= TRACE_FILE_LAT_FMT; 4403 } 4404 4405 if (ret < 0) 4406 trace_array_put(tr); 4407 4408 return ret; 4409 } 4410 4411 /* 4412 * Some tracers are not suitable for instance buffers. 4413 * A tracer is always available for the global array (toplevel) 4414 * or if it explicitly states that it is. 4415 */ 4416 static bool 4417 trace_ok_for_array(struct tracer *t, struct trace_array *tr) 4418 { 4419 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances; 4420 } 4421 4422 /* Find the next tracer that this trace array may use */ 4423 static struct tracer * 4424 get_tracer_for_array(struct trace_array *tr, struct tracer *t) 4425 { 4426 while (t && !trace_ok_for_array(t, tr)) 4427 t = t->next; 4428 4429 return t; 4430 } 4431 4432 static void * 4433 t_next(struct seq_file *m, void *v, loff_t *pos) 4434 { 4435 struct trace_array *tr = m->private; 4436 struct tracer *t = v; 4437 4438 (*pos)++; 4439 4440 if (t) 4441 t = get_tracer_for_array(tr, t->next); 4442 4443 return t; 4444 } 4445 4446 static void *t_start(struct seq_file *m, loff_t *pos) 4447 { 4448 struct trace_array *tr = m->private; 4449 struct tracer *t; 4450 loff_t l = 0; 4451 4452 mutex_lock(&trace_types_lock); 4453 4454 t = get_tracer_for_array(tr, trace_types); 4455 for (; t && l < *pos; t = t_next(m, t, &l)) 4456 ; 4457 4458 return t; 4459 } 4460 4461 static void t_stop(struct seq_file *m, void *p) 4462 { 4463 mutex_unlock(&trace_types_lock); 4464 } 4465 4466 static int t_show(struct seq_file *m, void *v) 4467 { 4468 struct tracer *t = v; 4469 4470 if (!t) 4471 return 0; 4472 4473 seq_puts(m, t->name); 4474 if (t->next) 4475 seq_putc(m, ' '); 4476 else 4477 seq_putc(m, '\n'); 4478 4479 return 0; 4480 } 4481 4482 static const struct seq_operations show_traces_seq_ops = { 4483 .start = t_start, 4484 .next = t_next, 4485 .stop = t_stop, 4486 .show = t_show, 4487 }; 4488 4489 static int show_traces_open(struct inode *inode, struct file *file) 4490 { 4491 struct trace_array *tr = inode->i_private; 4492 struct seq_file *m; 4493 int ret; 4494 4495 ret = tracing_check_open_get_tr(tr); 4496 if (ret) 4497 return ret; 4498 4499 ret = seq_open(file, &show_traces_seq_ops); 4500 if (ret) { 4501 trace_array_put(tr); 4502 return ret; 4503 } 4504 4505 m = file->private_data; 4506 m->private = tr; 4507 4508 return 0; 4509 } 4510 4511 static int show_traces_release(struct inode *inode, struct file *file) 4512 { 4513 struct trace_array *tr = inode->i_private; 4514 4515 trace_array_put(tr); 4516 return seq_release(inode, file); 4517 } 4518 4519 static ssize_t 4520 tracing_write_stub(struct file *filp, const char __user *ubuf, 4521 size_t count, loff_t *ppos) 4522 { 4523 return count; 4524 } 4525 4526 loff_t tracing_lseek(struct file *file, loff_t offset, int whence) 4527 { 4528 int ret; 4529 4530 if (file->f_mode & FMODE_READ) 4531 ret = seq_lseek(file, offset, whence); 4532 else 4533 file->f_pos = ret = 0; 4534 4535 return ret; 4536 } 4537 4538 static const struct file_operations tracing_fops = { 4539 .open = tracing_open, 4540 .read = seq_read, 4541 .write = tracing_write_stub, 4542 .llseek = tracing_lseek, 4543 .release = tracing_release, 4544 }; 4545 4546 static const struct file_operations show_traces_fops = { 4547 .open = show_traces_open, 4548 .read = seq_read, 4549 .llseek = seq_lseek, 4550 .release = show_traces_release, 4551 }; 4552 4553 static ssize_t 4554 tracing_cpumask_read(struct file *filp, char __user *ubuf, 4555 size_t count, loff_t *ppos) 4556 { 4557 struct trace_array *tr = file_inode(filp)->i_private; 4558 char *mask_str; 4559 int len; 4560 4561 len = snprintf(NULL, 0, "%*pb\n", 4562 cpumask_pr_args(tr->tracing_cpumask)) + 1; 4563 mask_str = kmalloc(len, GFP_KERNEL); 4564 if (!mask_str) 4565 return -ENOMEM; 4566 4567 len = snprintf(mask_str, len, "%*pb\n", 4568 cpumask_pr_args(tr->tracing_cpumask)); 4569 if (len >= count) { 4570 count = -EINVAL; 4571 goto out_err; 4572 } 4573 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len); 4574 4575 out_err: 4576 kfree(mask_str); 4577 4578 return count; 4579 } 4580 4581 int tracing_set_cpumask(struct trace_array *tr, 4582 cpumask_var_t tracing_cpumask_new) 4583 { 4584 int cpu; 4585 4586 if (!tr) 4587 return -EINVAL; 4588 4589 local_irq_disable(); 4590 arch_spin_lock(&tr->max_lock); 4591 for_each_tracing_cpu(cpu) { 4592 /* 4593 * Increase/decrease the disabled counter if we are 4594 * about to flip a bit in the cpumask: 4595 */ 4596 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) && 4597 !cpumask_test_cpu(cpu, tracing_cpumask_new)) { 4598 atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled); 4599 ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu); 4600 } 4601 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) && 4602 cpumask_test_cpu(cpu, tracing_cpumask_new)) { 4603 atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled); 4604 ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu); 4605 } 4606 } 4607 arch_spin_unlock(&tr->max_lock); 4608 local_irq_enable(); 4609 4610 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new); 4611 4612 return 0; 4613 } 4614 4615 static ssize_t 4616 tracing_cpumask_write(struct file *filp, const char __user *ubuf, 4617 size_t count, loff_t *ppos) 4618 { 4619 struct trace_array *tr = file_inode(filp)->i_private; 4620 cpumask_var_t tracing_cpumask_new; 4621 int err; 4622 4623 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL)) 4624 return -ENOMEM; 4625 4626 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new); 4627 if (err) 4628 goto err_free; 4629 4630 err = tracing_set_cpumask(tr, tracing_cpumask_new); 4631 if (err) 4632 goto err_free; 4633 4634 free_cpumask_var(tracing_cpumask_new); 4635 4636 return count; 4637 4638 err_free: 4639 free_cpumask_var(tracing_cpumask_new); 4640 4641 return err; 4642 } 4643 4644 static const struct file_operations tracing_cpumask_fops = { 4645 .open = tracing_open_generic_tr, 4646 .read = tracing_cpumask_read, 4647 .write = tracing_cpumask_write, 4648 .release = tracing_release_generic_tr, 4649 .llseek = generic_file_llseek, 4650 }; 4651 4652 static int tracing_trace_options_show(struct seq_file *m, void *v) 4653 { 4654 struct tracer_opt *trace_opts; 4655 struct trace_array *tr = m->private; 4656 u32 tracer_flags; 4657 int i; 4658 4659 mutex_lock(&trace_types_lock); 4660 tracer_flags = tr->current_trace->flags->val; 4661 trace_opts = tr->current_trace->flags->opts; 4662 4663 for (i = 0; trace_options[i]; i++) { 4664 if (tr->trace_flags & (1 << i)) 4665 seq_printf(m, "%s\n", trace_options[i]); 4666 else 4667 seq_printf(m, "no%s\n", trace_options[i]); 4668 } 4669 4670 for (i = 0; trace_opts[i].name; i++) { 4671 if (tracer_flags & trace_opts[i].bit) 4672 seq_printf(m, "%s\n", trace_opts[i].name); 4673 else 4674 seq_printf(m, "no%s\n", trace_opts[i].name); 4675 } 4676 mutex_unlock(&trace_types_lock); 4677 4678 return 0; 4679 } 4680 4681 static int __set_tracer_option(struct trace_array *tr, 4682 struct tracer_flags *tracer_flags, 4683 struct tracer_opt *opts, int neg) 4684 { 4685 struct tracer *trace = tracer_flags->trace; 4686 int ret; 4687 4688 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg); 4689 if (ret) 4690 return ret; 4691 4692 if (neg) 4693 tracer_flags->val &= ~opts->bit; 4694 else 4695 tracer_flags->val |= opts->bit; 4696 return 0; 4697 } 4698 4699 /* Try to assign a tracer specific option */ 4700 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg) 4701 { 4702 struct tracer *trace = tr->current_trace; 4703 struct tracer_flags *tracer_flags = trace->flags; 4704 struct tracer_opt *opts = NULL; 4705 int i; 4706 4707 for (i = 0; tracer_flags->opts[i].name; i++) { 4708 opts = &tracer_flags->opts[i]; 4709 4710 if (strcmp(cmp, opts->name) == 0) 4711 return __set_tracer_option(tr, trace->flags, opts, neg); 4712 } 4713 4714 return -EINVAL; 4715 } 4716 4717 /* Some tracers require overwrite to stay enabled */ 4718 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set) 4719 { 4720 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set) 4721 return -1; 4722 4723 return 0; 4724 } 4725 4726 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled) 4727 { 4728 if ((mask == TRACE_ITER_RECORD_TGID) || 4729 (mask == TRACE_ITER_RECORD_CMD)) 4730 lockdep_assert_held(&event_mutex); 4731 4732 /* do nothing if flag is already set */ 4733 if (!!(tr->trace_flags & mask) == !!enabled) 4734 return 0; 4735 4736 /* Give the tracer a chance to approve the change */ 4737 if (tr->current_trace->flag_changed) 4738 if (tr->current_trace->flag_changed(tr, mask, !!enabled)) 4739 return -EINVAL; 4740 4741 if (enabled) 4742 tr->trace_flags |= mask; 4743 else 4744 tr->trace_flags &= ~mask; 4745 4746 if (mask == TRACE_ITER_RECORD_CMD) 4747 trace_event_enable_cmd_record(enabled); 4748 4749 if (mask == TRACE_ITER_RECORD_TGID) { 4750 if (!tgid_map) 4751 tgid_map = kvcalloc(PID_MAX_DEFAULT + 1, 4752 sizeof(*tgid_map), 4753 GFP_KERNEL); 4754 if (!tgid_map) { 4755 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID; 4756 return -ENOMEM; 4757 } 4758 4759 trace_event_enable_tgid_record(enabled); 4760 } 4761 4762 if (mask == TRACE_ITER_EVENT_FORK) 4763 trace_event_follow_fork(tr, enabled); 4764 4765 if (mask == TRACE_ITER_FUNC_FORK) 4766 ftrace_pid_follow_fork(tr, enabled); 4767 4768 if (mask == TRACE_ITER_OVERWRITE) { 4769 ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled); 4770 #ifdef CONFIG_TRACER_MAX_TRACE 4771 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled); 4772 #endif 4773 } 4774 4775 if (mask == TRACE_ITER_PRINTK) { 4776 trace_printk_start_stop_comm(enabled); 4777 trace_printk_control(enabled); 4778 } 4779 4780 return 0; 4781 } 4782 4783 int trace_set_options(struct trace_array *tr, char *option) 4784 { 4785 char *cmp; 4786 int neg = 0; 4787 int ret; 4788 size_t orig_len = strlen(option); 4789 int len; 4790 4791 cmp = strstrip(option); 4792 4793 len = str_has_prefix(cmp, "no"); 4794 if (len) 4795 neg = 1; 4796 4797 cmp += len; 4798 4799 mutex_lock(&event_mutex); 4800 mutex_lock(&trace_types_lock); 4801 4802 ret = match_string(trace_options, -1, cmp); 4803 /* If no option could be set, test the specific tracer options */ 4804 if (ret < 0) 4805 ret = set_tracer_option(tr, cmp, neg); 4806 else 4807 ret = set_tracer_flag(tr, 1 << ret, !neg); 4808 4809 mutex_unlock(&trace_types_lock); 4810 mutex_unlock(&event_mutex); 4811 4812 /* 4813 * If the first trailing whitespace is replaced with '\0' by strstrip, 4814 * turn it back into a space. 4815 */ 4816 if (orig_len > strlen(option)) 4817 option[strlen(option)] = ' '; 4818 4819 return ret; 4820 } 4821 4822 static void __init apply_trace_boot_options(void) 4823 { 4824 char *buf = trace_boot_options_buf; 4825 char *option; 4826 4827 while (true) { 4828 option = strsep(&buf, ","); 4829 4830 if (!option) 4831 break; 4832 4833 if (*option) 4834 trace_set_options(&global_trace, option); 4835 4836 /* Put back the comma to allow this to be called again */ 4837 if (buf) 4838 *(buf - 1) = ','; 4839 } 4840 } 4841 4842 static ssize_t 4843 tracing_trace_options_write(struct file *filp, const char __user *ubuf, 4844 size_t cnt, loff_t *ppos) 4845 { 4846 struct seq_file *m = filp->private_data; 4847 struct trace_array *tr = m->private; 4848 char buf[64]; 4849 int ret; 4850 4851 if (cnt >= sizeof(buf)) 4852 return -EINVAL; 4853 4854 if (copy_from_user(buf, ubuf, cnt)) 4855 return -EFAULT; 4856 4857 buf[cnt] = 0; 4858 4859 ret = trace_set_options(tr, buf); 4860 if (ret < 0) 4861 return ret; 4862 4863 *ppos += cnt; 4864 4865 return cnt; 4866 } 4867 4868 static int tracing_trace_options_open(struct inode *inode, struct file *file) 4869 { 4870 struct trace_array *tr = inode->i_private; 4871 int ret; 4872 4873 ret = tracing_check_open_get_tr(tr); 4874 if (ret) 4875 return ret; 4876 4877 ret = single_open(file, tracing_trace_options_show, inode->i_private); 4878 if (ret < 0) 4879 trace_array_put(tr); 4880 4881 return ret; 4882 } 4883 4884 static const struct file_operations tracing_iter_fops = { 4885 .open = tracing_trace_options_open, 4886 .read = seq_read, 4887 .llseek = seq_lseek, 4888 .release = tracing_single_release_tr, 4889 .write = tracing_trace_options_write, 4890 }; 4891 4892 static const char readme_msg[] = 4893 "tracing mini-HOWTO:\n\n" 4894 "# echo 0 > tracing_on : quick way to disable tracing\n" 4895 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n" 4896 " Important files:\n" 4897 " trace\t\t\t- The static contents of the buffer\n" 4898 "\t\t\t To clear the buffer write into this file: echo > trace\n" 4899 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n" 4900 " current_tracer\t- function and latency tracers\n" 4901 " available_tracers\t- list of configured tracers for current_tracer\n" 4902 " error_log\t- error log for failed commands (that support it)\n" 4903 " buffer_size_kb\t- view and modify size of per cpu buffer\n" 4904 " buffer_total_size_kb - view total size of all cpu buffers\n\n" 4905 " trace_clock\t\t-change the clock used to order events\n" 4906 " local: Per cpu clock but may not be synced across CPUs\n" 4907 " global: Synced across CPUs but slows tracing down.\n" 4908 " counter: Not a clock, but just an increment\n" 4909 " uptime: Jiffy counter from time of boot\n" 4910 " perf: Same clock that perf events use\n" 4911 #ifdef CONFIG_X86_64 4912 " x86-tsc: TSC cycle counter\n" 4913 #endif 4914 "\n timestamp_mode\t-view the mode used to timestamp events\n" 4915 " delta: Delta difference against a buffer-wide timestamp\n" 4916 " absolute: Absolute (standalone) timestamp\n" 4917 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n" 4918 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n" 4919 " tracing_cpumask\t- Limit which CPUs to trace\n" 4920 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n" 4921 "\t\t\t Remove sub-buffer with rmdir\n" 4922 " trace_options\t\t- Set format or modify how tracing happens\n" 4923 "\t\t\t Disable an option by prefixing 'no' to the\n" 4924 "\t\t\t option name\n" 4925 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n" 4926 #ifdef CONFIG_DYNAMIC_FTRACE 4927 "\n available_filter_functions - list of functions that can be filtered on\n" 4928 " set_ftrace_filter\t- echo function name in here to only trace these\n" 4929 "\t\t\t functions\n" 4930 "\t accepts: func_full_name or glob-matching-pattern\n" 4931 "\t modules: Can select a group via module\n" 4932 "\t Format: :mod:<module-name>\n" 4933 "\t example: echo :mod:ext3 > set_ftrace_filter\n" 4934 "\t triggers: a command to perform when function is hit\n" 4935 "\t Format: <function>:<trigger>[:count]\n" 4936 "\t trigger: traceon, traceoff\n" 4937 "\t\t enable_event:<system>:<event>\n" 4938 "\t\t disable_event:<system>:<event>\n" 4939 #ifdef CONFIG_STACKTRACE 4940 "\t\t stacktrace\n" 4941 #endif 4942 #ifdef CONFIG_TRACER_SNAPSHOT 4943 "\t\t snapshot\n" 4944 #endif 4945 "\t\t dump\n" 4946 "\t\t cpudump\n" 4947 "\t example: echo do_fault:traceoff > set_ftrace_filter\n" 4948 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n" 4949 "\t The first one will disable tracing every time do_fault is hit\n" 4950 "\t The second will disable tracing at most 3 times when do_trap is hit\n" 4951 "\t The first time do trap is hit and it disables tracing, the\n" 4952 "\t counter will decrement to 2. If tracing is already disabled,\n" 4953 "\t the counter will not decrement. It only decrements when the\n" 4954 "\t trigger did work\n" 4955 "\t To remove trigger without count:\n" 4956 "\t echo '!<function>:<trigger> > set_ftrace_filter\n" 4957 "\t To remove trigger with a count:\n" 4958 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n" 4959 " set_ftrace_notrace\t- echo function name in here to never trace.\n" 4960 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n" 4961 "\t modules: Can select a group via module command :mod:\n" 4962 "\t Does not accept triggers\n" 4963 #endif /* CONFIG_DYNAMIC_FTRACE */ 4964 #ifdef CONFIG_FUNCTION_TRACER 4965 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n" 4966 "\t\t (function)\n" 4967 #endif 4968 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 4969 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n" 4970 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n" 4971 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n" 4972 #endif 4973 #ifdef CONFIG_TRACER_SNAPSHOT 4974 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n" 4975 "\t\t\t snapshot buffer. Read the contents for more\n" 4976 "\t\t\t information\n" 4977 #endif 4978 #ifdef CONFIG_STACK_TRACER 4979 " stack_trace\t\t- Shows the max stack trace when active\n" 4980 " stack_max_size\t- Shows current max stack size that was traced\n" 4981 "\t\t\t Write into this file to reset the max size (trigger a\n" 4982 "\t\t\t new trace)\n" 4983 #ifdef CONFIG_DYNAMIC_FTRACE 4984 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n" 4985 "\t\t\t traces\n" 4986 #endif 4987 #endif /* CONFIG_STACK_TRACER */ 4988 #ifdef CONFIG_DYNAMIC_EVENTS 4989 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n" 4990 "\t\t\t Write into this file to define/undefine new trace events.\n" 4991 #endif 4992 #ifdef CONFIG_KPROBE_EVENTS 4993 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n" 4994 "\t\t\t Write into this file to define/undefine new trace events.\n" 4995 #endif 4996 #ifdef CONFIG_UPROBE_EVENTS 4997 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n" 4998 "\t\t\t Write into this file to define/undefine new trace events.\n" 4999 #endif 5000 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS) 5001 "\t accepts: event-definitions (one definition per line)\n" 5002 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n" 5003 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n" 5004 #ifdef CONFIG_HIST_TRIGGERS 5005 "\t s:[synthetic/]<event> <field> [<field>]\n" 5006 #endif 5007 "\t -:[<group>/]<event>\n" 5008 #ifdef CONFIG_KPROBE_EVENTS 5009 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n" 5010 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n" 5011 #endif 5012 #ifdef CONFIG_UPROBE_EVENTS 5013 " place (uprobe): <path>:<offset>[(ref_ctr_offset)]\n" 5014 #endif 5015 "\t args: <name>=fetcharg[:type]\n" 5016 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n" 5017 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 5018 "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n" 5019 #else 5020 "\t $stack<index>, $stack, $retval, $comm,\n" 5021 #endif 5022 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n" 5023 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n" 5024 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n" 5025 "\t <type>\\[<array-size>\\]\n" 5026 #ifdef CONFIG_HIST_TRIGGERS 5027 "\t field: <stype> <name>;\n" 5028 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n" 5029 "\t [unsigned] char/int/long\n" 5030 #endif 5031 #endif 5032 " events/\t\t- Directory containing all trace event subsystems:\n" 5033 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n" 5034 " events/<system>/\t- Directory containing all trace events for <system>:\n" 5035 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n" 5036 "\t\t\t events\n" 5037 " filter\t\t- If set, only events passing filter are traced\n" 5038 " events/<system>/<event>/\t- Directory containing control files for\n" 5039 "\t\t\t <event>:\n" 5040 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n" 5041 " filter\t\t- If set, only events passing filter are traced\n" 5042 " trigger\t\t- If set, a command to perform when event is hit\n" 5043 "\t Format: <trigger>[:count][if <filter>]\n" 5044 "\t trigger: traceon, traceoff\n" 5045 "\t enable_event:<system>:<event>\n" 5046 "\t disable_event:<system>:<event>\n" 5047 #ifdef CONFIG_HIST_TRIGGERS 5048 "\t enable_hist:<system>:<event>\n" 5049 "\t disable_hist:<system>:<event>\n" 5050 #endif 5051 #ifdef CONFIG_STACKTRACE 5052 "\t\t stacktrace\n" 5053 #endif 5054 #ifdef CONFIG_TRACER_SNAPSHOT 5055 "\t\t snapshot\n" 5056 #endif 5057 #ifdef CONFIG_HIST_TRIGGERS 5058 "\t\t hist (see below)\n" 5059 #endif 5060 "\t example: echo traceoff > events/block/block_unplug/trigger\n" 5061 "\t echo traceoff:3 > events/block/block_unplug/trigger\n" 5062 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n" 5063 "\t events/block/block_unplug/trigger\n" 5064 "\t The first disables tracing every time block_unplug is hit.\n" 5065 "\t The second disables tracing the first 3 times block_unplug is hit.\n" 5066 "\t The third enables the kmalloc event the first 3 times block_unplug\n" 5067 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n" 5068 "\t Like function triggers, the counter is only decremented if it\n" 5069 "\t enabled or disabled tracing.\n" 5070 "\t To remove a trigger without a count:\n" 5071 "\t echo '!<trigger> > <system>/<event>/trigger\n" 5072 "\t To remove a trigger with a count:\n" 5073 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n" 5074 "\t Filters can be ignored when removing a trigger.\n" 5075 #ifdef CONFIG_HIST_TRIGGERS 5076 " hist trigger\t- If set, event hits are aggregated into a hash table\n" 5077 "\t Format: hist:keys=<field1[,field2,...]>\n" 5078 "\t [:values=<field1[,field2,...]>]\n" 5079 "\t [:sort=<field1[,field2,...]>]\n" 5080 "\t [:size=#entries]\n" 5081 "\t [:pause][:continue][:clear]\n" 5082 "\t [:name=histname1]\n" 5083 "\t [:<handler>.<action>]\n" 5084 "\t [if <filter>]\n\n" 5085 "\t When a matching event is hit, an entry is added to a hash\n" 5086 "\t table using the key(s) and value(s) named, and the value of a\n" 5087 "\t sum called 'hitcount' is incremented. Keys and values\n" 5088 "\t correspond to fields in the event's format description. Keys\n" 5089 "\t can be any field, or the special string 'stacktrace'.\n" 5090 "\t Compound keys consisting of up to two fields can be specified\n" 5091 "\t by the 'keys' keyword. Values must correspond to numeric\n" 5092 "\t fields. Sort keys consisting of up to two fields can be\n" 5093 "\t specified using the 'sort' keyword. The sort direction can\n" 5094 "\t be modified by appending '.descending' or '.ascending' to a\n" 5095 "\t sort field. The 'size' parameter can be used to specify more\n" 5096 "\t or fewer than the default 2048 entries for the hashtable size.\n" 5097 "\t If a hist trigger is given a name using the 'name' parameter,\n" 5098 "\t its histogram data will be shared with other triggers of the\n" 5099 "\t same name, and trigger hits will update this common data.\n\n" 5100 "\t Reading the 'hist' file for the event will dump the hash\n" 5101 "\t table in its entirety to stdout. If there are multiple hist\n" 5102 "\t triggers attached to an event, there will be a table for each\n" 5103 "\t trigger in the output. The table displayed for a named\n" 5104 "\t trigger will be the same as any other instance having the\n" 5105 "\t same name. The default format used to display a given field\n" 5106 "\t can be modified by appending any of the following modifiers\n" 5107 "\t to the field name, as applicable:\n\n" 5108 "\t .hex display a number as a hex value\n" 5109 "\t .sym display an address as a symbol\n" 5110 "\t .sym-offset display an address as a symbol and offset\n" 5111 "\t .execname display a common_pid as a program name\n" 5112 "\t .syscall display a syscall id as a syscall name\n" 5113 "\t .log2 display log2 value rather than raw number\n" 5114 "\t .usecs display a common_timestamp in microseconds\n\n" 5115 "\t The 'pause' parameter can be used to pause an existing hist\n" 5116 "\t trigger or to start a hist trigger but not log any events\n" 5117 "\t until told to do so. 'continue' can be used to start or\n" 5118 "\t restart a paused hist trigger.\n\n" 5119 "\t The 'clear' parameter will clear the contents of a running\n" 5120 "\t hist trigger and leave its current paused/active state\n" 5121 "\t unchanged.\n\n" 5122 "\t The enable_hist and disable_hist triggers can be used to\n" 5123 "\t have one event conditionally start and stop another event's\n" 5124 "\t already-attached hist trigger. The syntax is analogous to\n" 5125 "\t the enable_event and disable_event triggers.\n\n" 5126 "\t Hist trigger handlers and actions are executed whenever a\n" 5127 "\t a histogram entry is added or updated. They take the form:\n\n" 5128 "\t <handler>.<action>\n\n" 5129 "\t The available handlers are:\n\n" 5130 "\t onmatch(matching.event) - invoke on addition or update\n" 5131 "\t onmax(var) - invoke if var exceeds current max\n" 5132 "\t onchange(var) - invoke action if var changes\n\n" 5133 "\t The available actions are:\n\n" 5134 "\t trace(<synthetic_event>,param list) - generate synthetic event\n" 5135 "\t save(field,...) - save current event fields\n" 5136 #ifdef CONFIG_TRACER_SNAPSHOT 5137 "\t snapshot() - snapshot the trace buffer\n" 5138 #endif 5139 #endif 5140 ; 5141 5142 static ssize_t 5143 tracing_readme_read(struct file *filp, char __user *ubuf, 5144 size_t cnt, loff_t *ppos) 5145 { 5146 return simple_read_from_buffer(ubuf, cnt, ppos, 5147 readme_msg, strlen(readme_msg)); 5148 } 5149 5150 static const struct file_operations tracing_readme_fops = { 5151 .open = tracing_open_generic, 5152 .read = tracing_readme_read, 5153 .llseek = generic_file_llseek, 5154 }; 5155 5156 static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos) 5157 { 5158 int *ptr = v; 5159 5160 if (*pos || m->count) 5161 ptr++; 5162 5163 (*pos)++; 5164 5165 for (; ptr <= &tgid_map[PID_MAX_DEFAULT]; ptr++) { 5166 if (trace_find_tgid(*ptr)) 5167 return ptr; 5168 } 5169 5170 return NULL; 5171 } 5172 5173 static void *saved_tgids_start(struct seq_file *m, loff_t *pos) 5174 { 5175 void *v; 5176 loff_t l = 0; 5177 5178 if (!tgid_map) 5179 return NULL; 5180 5181 v = &tgid_map[0]; 5182 while (l <= *pos) { 5183 v = saved_tgids_next(m, v, &l); 5184 if (!v) 5185 return NULL; 5186 } 5187 5188 return v; 5189 } 5190 5191 static void saved_tgids_stop(struct seq_file *m, void *v) 5192 { 5193 } 5194 5195 static int saved_tgids_show(struct seq_file *m, void *v) 5196 { 5197 int pid = (int *)v - tgid_map; 5198 5199 seq_printf(m, "%d %d\n", pid, trace_find_tgid(pid)); 5200 return 0; 5201 } 5202 5203 static const struct seq_operations tracing_saved_tgids_seq_ops = { 5204 .start = saved_tgids_start, 5205 .stop = saved_tgids_stop, 5206 .next = saved_tgids_next, 5207 .show = saved_tgids_show, 5208 }; 5209 5210 static int tracing_saved_tgids_open(struct inode *inode, struct file *filp) 5211 { 5212 int ret; 5213 5214 ret = tracing_check_open_get_tr(NULL); 5215 if (ret) 5216 return ret; 5217 5218 return seq_open(filp, &tracing_saved_tgids_seq_ops); 5219 } 5220 5221 5222 static const struct file_operations tracing_saved_tgids_fops = { 5223 .open = tracing_saved_tgids_open, 5224 .read = seq_read, 5225 .llseek = seq_lseek, 5226 .release = seq_release, 5227 }; 5228 5229 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos) 5230 { 5231 unsigned int *ptr = v; 5232 5233 if (*pos || m->count) 5234 ptr++; 5235 5236 (*pos)++; 5237 5238 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num]; 5239 ptr++) { 5240 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP) 5241 continue; 5242 5243 return ptr; 5244 } 5245 5246 return NULL; 5247 } 5248 5249 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos) 5250 { 5251 void *v; 5252 loff_t l = 0; 5253 5254 preempt_disable(); 5255 arch_spin_lock(&trace_cmdline_lock); 5256 5257 v = &savedcmd->map_cmdline_to_pid[0]; 5258 while (l <= *pos) { 5259 v = saved_cmdlines_next(m, v, &l); 5260 if (!v) 5261 return NULL; 5262 } 5263 5264 return v; 5265 } 5266 5267 static void saved_cmdlines_stop(struct seq_file *m, void *v) 5268 { 5269 arch_spin_unlock(&trace_cmdline_lock); 5270 preempt_enable(); 5271 } 5272 5273 static int saved_cmdlines_show(struct seq_file *m, void *v) 5274 { 5275 char buf[TASK_COMM_LEN]; 5276 unsigned int *pid = v; 5277 5278 __trace_find_cmdline(*pid, buf); 5279 seq_printf(m, "%d %s\n", *pid, buf); 5280 return 0; 5281 } 5282 5283 static const struct seq_operations tracing_saved_cmdlines_seq_ops = { 5284 .start = saved_cmdlines_start, 5285 .next = saved_cmdlines_next, 5286 .stop = saved_cmdlines_stop, 5287 .show = saved_cmdlines_show, 5288 }; 5289 5290 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp) 5291 { 5292 int ret; 5293 5294 ret = tracing_check_open_get_tr(NULL); 5295 if (ret) 5296 return ret; 5297 5298 return seq_open(filp, &tracing_saved_cmdlines_seq_ops); 5299 } 5300 5301 static const struct file_operations tracing_saved_cmdlines_fops = { 5302 .open = tracing_saved_cmdlines_open, 5303 .read = seq_read, 5304 .llseek = seq_lseek, 5305 .release = seq_release, 5306 }; 5307 5308 static ssize_t 5309 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf, 5310 size_t cnt, loff_t *ppos) 5311 { 5312 char buf[64]; 5313 int r; 5314 5315 arch_spin_lock(&trace_cmdline_lock); 5316 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num); 5317 arch_spin_unlock(&trace_cmdline_lock); 5318 5319 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5320 } 5321 5322 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s) 5323 { 5324 kfree(s->saved_cmdlines); 5325 kfree(s->map_cmdline_to_pid); 5326 kfree(s); 5327 } 5328 5329 static int tracing_resize_saved_cmdlines(unsigned int val) 5330 { 5331 struct saved_cmdlines_buffer *s, *savedcmd_temp; 5332 5333 s = kmalloc(sizeof(*s), GFP_KERNEL); 5334 if (!s) 5335 return -ENOMEM; 5336 5337 if (allocate_cmdlines_buffer(val, s) < 0) { 5338 kfree(s); 5339 return -ENOMEM; 5340 } 5341 5342 arch_spin_lock(&trace_cmdline_lock); 5343 savedcmd_temp = savedcmd; 5344 savedcmd = s; 5345 arch_spin_unlock(&trace_cmdline_lock); 5346 free_saved_cmdlines_buffer(savedcmd_temp); 5347 5348 return 0; 5349 } 5350 5351 static ssize_t 5352 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf, 5353 size_t cnt, loff_t *ppos) 5354 { 5355 unsigned long val; 5356 int ret; 5357 5358 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 5359 if (ret) 5360 return ret; 5361 5362 /* must have at least 1 entry or less than PID_MAX_DEFAULT */ 5363 if (!val || val > PID_MAX_DEFAULT) 5364 return -EINVAL; 5365 5366 ret = tracing_resize_saved_cmdlines((unsigned int)val); 5367 if (ret < 0) 5368 return ret; 5369 5370 *ppos += cnt; 5371 5372 return cnt; 5373 } 5374 5375 static const struct file_operations tracing_saved_cmdlines_size_fops = { 5376 .open = tracing_open_generic, 5377 .read = tracing_saved_cmdlines_size_read, 5378 .write = tracing_saved_cmdlines_size_write, 5379 }; 5380 5381 #ifdef CONFIG_TRACE_EVAL_MAP_FILE 5382 static union trace_eval_map_item * 5383 update_eval_map(union trace_eval_map_item *ptr) 5384 { 5385 if (!ptr->map.eval_string) { 5386 if (ptr->tail.next) { 5387 ptr = ptr->tail.next; 5388 /* Set ptr to the next real item (skip head) */ 5389 ptr++; 5390 } else 5391 return NULL; 5392 } 5393 return ptr; 5394 } 5395 5396 static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos) 5397 { 5398 union trace_eval_map_item *ptr = v; 5399 5400 /* 5401 * Paranoid! If ptr points to end, we don't want to increment past it. 5402 * This really should never happen. 5403 */ 5404 (*pos)++; 5405 ptr = update_eval_map(ptr); 5406 if (WARN_ON_ONCE(!ptr)) 5407 return NULL; 5408 5409 ptr++; 5410 ptr = update_eval_map(ptr); 5411 5412 return ptr; 5413 } 5414 5415 static void *eval_map_start(struct seq_file *m, loff_t *pos) 5416 { 5417 union trace_eval_map_item *v; 5418 loff_t l = 0; 5419 5420 mutex_lock(&trace_eval_mutex); 5421 5422 v = trace_eval_maps; 5423 if (v) 5424 v++; 5425 5426 while (v && l < *pos) { 5427 v = eval_map_next(m, v, &l); 5428 } 5429 5430 return v; 5431 } 5432 5433 static void eval_map_stop(struct seq_file *m, void *v) 5434 { 5435 mutex_unlock(&trace_eval_mutex); 5436 } 5437 5438 static int eval_map_show(struct seq_file *m, void *v) 5439 { 5440 union trace_eval_map_item *ptr = v; 5441 5442 seq_printf(m, "%s %ld (%s)\n", 5443 ptr->map.eval_string, ptr->map.eval_value, 5444 ptr->map.system); 5445 5446 return 0; 5447 } 5448 5449 static const struct seq_operations tracing_eval_map_seq_ops = { 5450 .start = eval_map_start, 5451 .next = eval_map_next, 5452 .stop = eval_map_stop, 5453 .show = eval_map_show, 5454 }; 5455 5456 static int tracing_eval_map_open(struct inode *inode, struct file *filp) 5457 { 5458 int ret; 5459 5460 ret = tracing_check_open_get_tr(NULL); 5461 if (ret) 5462 return ret; 5463 5464 return seq_open(filp, &tracing_eval_map_seq_ops); 5465 } 5466 5467 static const struct file_operations tracing_eval_map_fops = { 5468 .open = tracing_eval_map_open, 5469 .read = seq_read, 5470 .llseek = seq_lseek, 5471 .release = seq_release, 5472 }; 5473 5474 static inline union trace_eval_map_item * 5475 trace_eval_jmp_to_tail(union trace_eval_map_item *ptr) 5476 { 5477 /* Return tail of array given the head */ 5478 return ptr + ptr->head.length + 1; 5479 } 5480 5481 static void 5482 trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start, 5483 int len) 5484 { 5485 struct trace_eval_map **stop; 5486 struct trace_eval_map **map; 5487 union trace_eval_map_item *map_array; 5488 union trace_eval_map_item *ptr; 5489 5490 stop = start + len; 5491 5492 /* 5493 * The trace_eval_maps contains the map plus a head and tail item, 5494 * where the head holds the module and length of array, and the 5495 * tail holds a pointer to the next list. 5496 */ 5497 map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL); 5498 if (!map_array) { 5499 pr_warn("Unable to allocate trace eval mapping\n"); 5500 return; 5501 } 5502 5503 mutex_lock(&trace_eval_mutex); 5504 5505 if (!trace_eval_maps) 5506 trace_eval_maps = map_array; 5507 else { 5508 ptr = trace_eval_maps; 5509 for (;;) { 5510 ptr = trace_eval_jmp_to_tail(ptr); 5511 if (!ptr->tail.next) 5512 break; 5513 ptr = ptr->tail.next; 5514 5515 } 5516 ptr->tail.next = map_array; 5517 } 5518 map_array->head.mod = mod; 5519 map_array->head.length = len; 5520 map_array++; 5521 5522 for (map = start; (unsigned long)map < (unsigned long)stop; map++) { 5523 map_array->map = **map; 5524 map_array++; 5525 } 5526 memset(map_array, 0, sizeof(*map_array)); 5527 5528 mutex_unlock(&trace_eval_mutex); 5529 } 5530 5531 static void trace_create_eval_file(struct dentry *d_tracer) 5532 { 5533 trace_create_file("eval_map", 0444, d_tracer, 5534 NULL, &tracing_eval_map_fops); 5535 } 5536 5537 #else /* CONFIG_TRACE_EVAL_MAP_FILE */ 5538 static inline void trace_create_eval_file(struct dentry *d_tracer) { } 5539 static inline void trace_insert_eval_map_file(struct module *mod, 5540 struct trace_eval_map **start, int len) { } 5541 #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */ 5542 5543 static void trace_insert_eval_map(struct module *mod, 5544 struct trace_eval_map **start, int len) 5545 { 5546 struct trace_eval_map **map; 5547 5548 if (len <= 0) 5549 return; 5550 5551 map = start; 5552 5553 trace_event_eval_update(map, len); 5554 5555 trace_insert_eval_map_file(mod, start, len); 5556 } 5557 5558 static ssize_t 5559 tracing_set_trace_read(struct file *filp, char __user *ubuf, 5560 size_t cnt, loff_t *ppos) 5561 { 5562 struct trace_array *tr = filp->private_data; 5563 char buf[MAX_TRACER_SIZE+2]; 5564 int r; 5565 5566 mutex_lock(&trace_types_lock); 5567 r = sprintf(buf, "%s\n", tr->current_trace->name); 5568 mutex_unlock(&trace_types_lock); 5569 5570 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5571 } 5572 5573 int tracer_init(struct tracer *t, struct trace_array *tr) 5574 { 5575 tracing_reset_online_cpus(&tr->array_buffer); 5576 return t->init(tr); 5577 } 5578 5579 static void set_buffer_entries(struct array_buffer *buf, unsigned long val) 5580 { 5581 int cpu; 5582 5583 for_each_tracing_cpu(cpu) 5584 per_cpu_ptr(buf->data, cpu)->entries = val; 5585 } 5586 5587 #ifdef CONFIG_TRACER_MAX_TRACE 5588 /* resize @tr's buffer to the size of @size_tr's entries */ 5589 static int resize_buffer_duplicate_size(struct array_buffer *trace_buf, 5590 struct array_buffer *size_buf, int cpu_id) 5591 { 5592 int cpu, ret = 0; 5593 5594 if (cpu_id == RING_BUFFER_ALL_CPUS) { 5595 for_each_tracing_cpu(cpu) { 5596 ret = ring_buffer_resize(trace_buf->buffer, 5597 per_cpu_ptr(size_buf->data, cpu)->entries, cpu); 5598 if (ret < 0) 5599 break; 5600 per_cpu_ptr(trace_buf->data, cpu)->entries = 5601 per_cpu_ptr(size_buf->data, cpu)->entries; 5602 } 5603 } else { 5604 ret = ring_buffer_resize(trace_buf->buffer, 5605 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id); 5606 if (ret == 0) 5607 per_cpu_ptr(trace_buf->data, cpu_id)->entries = 5608 per_cpu_ptr(size_buf->data, cpu_id)->entries; 5609 } 5610 5611 return ret; 5612 } 5613 #endif /* CONFIG_TRACER_MAX_TRACE */ 5614 5615 static int __tracing_resize_ring_buffer(struct trace_array *tr, 5616 unsigned long size, int cpu) 5617 { 5618 int ret; 5619 5620 /* 5621 * If kernel or user changes the size of the ring buffer 5622 * we use the size that was given, and we can forget about 5623 * expanding it later. 5624 */ 5625 ring_buffer_expanded = true; 5626 5627 /* May be called before buffers are initialized */ 5628 if (!tr->array_buffer.buffer) 5629 return 0; 5630 5631 ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu); 5632 if (ret < 0) 5633 return ret; 5634 5635 #ifdef CONFIG_TRACER_MAX_TRACE 5636 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) || 5637 !tr->current_trace->use_max_tr) 5638 goto out; 5639 5640 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu); 5641 if (ret < 0) { 5642 int r = resize_buffer_duplicate_size(&tr->array_buffer, 5643 &tr->array_buffer, cpu); 5644 if (r < 0) { 5645 /* 5646 * AARGH! We are left with different 5647 * size max buffer!!!! 5648 * The max buffer is our "snapshot" buffer. 5649 * When a tracer needs a snapshot (one of the 5650 * latency tracers), it swaps the max buffer 5651 * with the saved snap shot. We succeeded to 5652 * update the size of the main buffer, but failed to 5653 * update the size of the max buffer. But when we tried 5654 * to reset the main buffer to the original size, we 5655 * failed there too. This is very unlikely to 5656 * happen, but if it does, warn and kill all 5657 * tracing. 5658 */ 5659 WARN_ON(1); 5660 tracing_disabled = 1; 5661 } 5662 return ret; 5663 } 5664 5665 if (cpu == RING_BUFFER_ALL_CPUS) 5666 set_buffer_entries(&tr->max_buffer, size); 5667 else 5668 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size; 5669 5670 out: 5671 #endif /* CONFIG_TRACER_MAX_TRACE */ 5672 5673 if (cpu == RING_BUFFER_ALL_CPUS) 5674 set_buffer_entries(&tr->array_buffer, size); 5675 else 5676 per_cpu_ptr(tr->array_buffer.data, cpu)->entries = size; 5677 5678 return ret; 5679 } 5680 5681 ssize_t tracing_resize_ring_buffer(struct trace_array *tr, 5682 unsigned long size, int cpu_id) 5683 { 5684 int ret = size; 5685 5686 mutex_lock(&trace_types_lock); 5687 5688 if (cpu_id != RING_BUFFER_ALL_CPUS) { 5689 /* make sure, this cpu is enabled in the mask */ 5690 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) { 5691 ret = -EINVAL; 5692 goto out; 5693 } 5694 } 5695 5696 ret = __tracing_resize_ring_buffer(tr, size, cpu_id); 5697 if (ret < 0) 5698 ret = -ENOMEM; 5699 5700 out: 5701 mutex_unlock(&trace_types_lock); 5702 5703 return ret; 5704 } 5705 5706 5707 /** 5708 * tracing_update_buffers - used by tracing facility to expand ring buffers 5709 * 5710 * To save on memory when the tracing is never used on a system with it 5711 * configured in. The ring buffers are set to a minimum size. But once 5712 * a user starts to use the tracing facility, then they need to grow 5713 * to their default size. 5714 * 5715 * This function is to be called when a tracer is about to be used. 5716 */ 5717 int tracing_update_buffers(void) 5718 { 5719 int ret = 0; 5720 5721 mutex_lock(&trace_types_lock); 5722 if (!ring_buffer_expanded) 5723 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size, 5724 RING_BUFFER_ALL_CPUS); 5725 mutex_unlock(&trace_types_lock); 5726 5727 return ret; 5728 } 5729 5730 struct trace_option_dentry; 5731 5732 static void 5733 create_trace_option_files(struct trace_array *tr, struct tracer *tracer); 5734 5735 /* 5736 * Used to clear out the tracer before deletion of an instance. 5737 * Must have trace_types_lock held. 5738 */ 5739 static void tracing_set_nop(struct trace_array *tr) 5740 { 5741 if (tr->current_trace == &nop_trace) 5742 return; 5743 5744 tr->current_trace->enabled--; 5745 5746 if (tr->current_trace->reset) 5747 tr->current_trace->reset(tr); 5748 5749 tr->current_trace = &nop_trace; 5750 } 5751 5752 static void add_tracer_options(struct trace_array *tr, struct tracer *t) 5753 { 5754 /* Only enable if the directory has been created already. */ 5755 if (!tr->dir) 5756 return; 5757 5758 create_trace_option_files(tr, t); 5759 } 5760 5761 int tracing_set_tracer(struct trace_array *tr, const char *buf) 5762 { 5763 struct tracer *t; 5764 #ifdef CONFIG_TRACER_MAX_TRACE 5765 bool had_max_tr; 5766 #endif 5767 int ret = 0; 5768 5769 mutex_lock(&trace_types_lock); 5770 5771 if (!ring_buffer_expanded) { 5772 ret = __tracing_resize_ring_buffer(tr, trace_buf_size, 5773 RING_BUFFER_ALL_CPUS); 5774 if (ret < 0) 5775 goto out; 5776 ret = 0; 5777 } 5778 5779 for (t = trace_types; t; t = t->next) { 5780 if (strcmp(t->name, buf) == 0) 5781 break; 5782 } 5783 if (!t) { 5784 ret = -EINVAL; 5785 goto out; 5786 } 5787 if (t == tr->current_trace) 5788 goto out; 5789 5790 #ifdef CONFIG_TRACER_SNAPSHOT 5791 if (t->use_max_tr) { 5792 arch_spin_lock(&tr->max_lock); 5793 if (tr->cond_snapshot) 5794 ret = -EBUSY; 5795 arch_spin_unlock(&tr->max_lock); 5796 if (ret) 5797 goto out; 5798 } 5799 #endif 5800 /* Some tracers won't work on kernel command line */ 5801 if (system_state < SYSTEM_RUNNING && t->noboot) { 5802 pr_warn("Tracer '%s' is not allowed on command line, ignored\n", 5803 t->name); 5804 goto out; 5805 } 5806 5807 /* Some tracers are only allowed for the top level buffer */ 5808 if (!trace_ok_for_array(t, tr)) { 5809 ret = -EINVAL; 5810 goto out; 5811 } 5812 5813 /* If trace pipe files are being read, we can't change the tracer */ 5814 if (tr->current_trace->ref) { 5815 ret = -EBUSY; 5816 goto out; 5817 } 5818 5819 trace_branch_disable(); 5820 5821 tr->current_trace->enabled--; 5822 5823 if (tr->current_trace->reset) 5824 tr->current_trace->reset(tr); 5825 5826 /* Current trace needs to be nop_trace before synchronize_rcu */ 5827 tr->current_trace = &nop_trace; 5828 5829 #ifdef CONFIG_TRACER_MAX_TRACE 5830 had_max_tr = tr->allocated_snapshot; 5831 5832 if (had_max_tr && !t->use_max_tr) { 5833 /* 5834 * We need to make sure that the update_max_tr sees that 5835 * current_trace changed to nop_trace to keep it from 5836 * swapping the buffers after we resize it. 5837 * The update_max_tr is called from interrupts disabled 5838 * so a synchronized_sched() is sufficient. 5839 */ 5840 synchronize_rcu(); 5841 free_snapshot(tr); 5842 } 5843 #endif 5844 5845 #ifdef CONFIG_TRACER_MAX_TRACE 5846 if (t->use_max_tr && !had_max_tr) { 5847 ret = tracing_alloc_snapshot_instance(tr); 5848 if (ret < 0) 5849 goto out; 5850 } 5851 #endif 5852 5853 if (t->init) { 5854 ret = tracer_init(t, tr); 5855 if (ret) 5856 goto out; 5857 } 5858 5859 tr->current_trace = t; 5860 tr->current_trace->enabled++; 5861 trace_branch_enable(tr); 5862 out: 5863 mutex_unlock(&trace_types_lock); 5864 5865 return ret; 5866 } 5867 5868 static ssize_t 5869 tracing_set_trace_write(struct file *filp, const char __user *ubuf, 5870 size_t cnt, loff_t *ppos) 5871 { 5872 struct trace_array *tr = filp->private_data; 5873 char buf[MAX_TRACER_SIZE+1]; 5874 int i; 5875 size_t ret; 5876 int err; 5877 5878 ret = cnt; 5879 5880 if (cnt > MAX_TRACER_SIZE) 5881 cnt = MAX_TRACER_SIZE; 5882 5883 if (copy_from_user(buf, ubuf, cnt)) 5884 return -EFAULT; 5885 5886 buf[cnt] = 0; 5887 5888 /* strip ending whitespace. */ 5889 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--) 5890 buf[i] = 0; 5891 5892 err = tracing_set_tracer(tr, buf); 5893 if (err) 5894 return err; 5895 5896 *ppos += ret; 5897 5898 return ret; 5899 } 5900 5901 static ssize_t 5902 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf, 5903 size_t cnt, loff_t *ppos) 5904 { 5905 char buf[64]; 5906 int r; 5907 5908 r = snprintf(buf, sizeof(buf), "%ld\n", 5909 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr)); 5910 if (r > sizeof(buf)) 5911 r = sizeof(buf); 5912 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5913 } 5914 5915 static ssize_t 5916 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf, 5917 size_t cnt, loff_t *ppos) 5918 { 5919 unsigned long val; 5920 int ret; 5921 5922 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 5923 if (ret) 5924 return ret; 5925 5926 *ptr = val * 1000; 5927 5928 return cnt; 5929 } 5930 5931 static ssize_t 5932 tracing_thresh_read(struct file *filp, char __user *ubuf, 5933 size_t cnt, loff_t *ppos) 5934 { 5935 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos); 5936 } 5937 5938 static ssize_t 5939 tracing_thresh_write(struct file *filp, const char __user *ubuf, 5940 size_t cnt, loff_t *ppos) 5941 { 5942 struct trace_array *tr = filp->private_data; 5943 int ret; 5944 5945 mutex_lock(&trace_types_lock); 5946 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos); 5947 if (ret < 0) 5948 goto out; 5949 5950 if (tr->current_trace->update_thresh) { 5951 ret = tr->current_trace->update_thresh(tr); 5952 if (ret < 0) 5953 goto out; 5954 } 5955 5956 ret = cnt; 5957 out: 5958 mutex_unlock(&trace_types_lock); 5959 5960 return ret; 5961 } 5962 5963 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) 5964 5965 static ssize_t 5966 tracing_max_lat_read(struct file *filp, char __user *ubuf, 5967 size_t cnt, loff_t *ppos) 5968 { 5969 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos); 5970 } 5971 5972 static ssize_t 5973 tracing_max_lat_write(struct file *filp, const char __user *ubuf, 5974 size_t cnt, loff_t *ppos) 5975 { 5976 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos); 5977 } 5978 5979 #endif 5980 5981 static int tracing_open_pipe(struct inode *inode, struct file *filp) 5982 { 5983 struct trace_array *tr = inode->i_private; 5984 struct trace_iterator *iter; 5985 int ret; 5986 5987 ret = tracing_check_open_get_tr(tr); 5988 if (ret) 5989 return ret; 5990 5991 mutex_lock(&trace_types_lock); 5992 5993 /* create a buffer to store the information to pass to userspace */ 5994 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 5995 if (!iter) { 5996 ret = -ENOMEM; 5997 __trace_array_put(tr); 5998 goto out; 5999 } 6000 6001 trace_seq_init(&iter->seq); 6002 iter->trace = tr->current_trace; 6003 6004 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) { 6005 ret = -ENOMEM; 6006 goto fail; 6007 } 6008 6009 /* trace pipe does not show start of buffer */ 6010 cpumask_setall(iter->started); 6011 6012 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 6013 iter->iter_flags |= TRACE_FILE_LAT_FMT; 6014 6015 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 6016 if (trace_clocks[tr->clock_id].in_ns) 6017 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 6018 6019 iter->tr = tr; 6020 iter->array_buffer = &tr->array_buffer; 6021 iter->cpu_file = tracing_get_cpu(inode); 6022 mutex_init(&iter->mutex); 6023 filp->private_data = iter; 6024 6025 if (iter->trace->pipe_open) 6026 iter->trace->pipe_open(iter); 6027 6028 nonseekable_open(inode, filp); 6029 6030 tr->current_trace->ref++; 6031 out: 6032 mutex_unlock(&trace_types_lock); 6033 return ret; 6034 6035 fail: 6036 kfree(iter); 6037 __trace_array_put(tr); 6038 mutex_unlock(&trace_types_lock); 6039 return ret; 6040 } 6041 6042 static int tracing_release_pipe(struct inode *inode, struct file *file) 6043 { 6044 struct trace_iterator *iter = file->private_data; 6045 struct trace_array *tr = inode->i_private; 6046 6047 mutex_lock(&trace_types_lock); 6048 6049 tr->current_trace->ref--; 6050 6051 if (iter->trace->pipe_close) 6052 iter->trace->pipe_close(iter); 6053 6054 mutex_unlock(&trace_types_lock); 6055 6056 free_cpumask_var(iter->started); 6057 mutex_destroy(&iter->mutex); 6058 kfree(iter); 6059 6060 trace_array_put(tr); 6061 6062 return 0; 6063 } 6064 6065 static __poll_t 6066 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table) 6067 { 6068 struct trace_array *tr = iter->tr; 6069 6070 /* Iterators are static, they should be filled or empty */ 6071 if (trace_buffer_iter(iter, iter->cpu_file)) 6072 return EPOLLIN | EPOLLRDNORM; 6073 6074 if (tr->trace_flags & TRACE_ITER_BLOCK) 6075 /* 6076 * Always select as readable when in blocking mode 6077 */ 6078 return EPOLLIN | EPOLLRDNORM; 6079 else 6080 return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file, 6081 filp, poll_table); 6082 } 6083 6084 static __poll_t 6085 tracing_poll_pipe(struct file *filp, poll_table *poll_table) 6086 { 6087 struct trace_iterator *iter = filp->private_data; 6088 6089 return trace_poll(iter, filp, poll_table); 6090 } 6091 6092 /* Must be called with iter->mutex held. */ 6093 static int tracing_wait_pipe(struct file *filp) 6094 { 6095 struct trace_iterator *iter = filp->private_data; 6096 int ret; 6097 6098 while (trace_empty(iter)) { 6099 6100 if ((filp->f_flags & O_NONBLOCK)) { 6101 return -EAGAIN; 6102 } 6103 6104 /* 6105 * We block until we read something and tracing is disabled. 6106 * We still block if tracing is disabled, but we have never 6107 * read anything. This allows a user to cat this file, and 6108 * then enable tracing. But after we have read something, 6109 * we give an EOF when tracing is again disabled. 6110 * 6111 * iter->pos will be 0 if we haven't read anything. 6112 */ 6113 if (!tracer_tracing_is_on(iter->tr) && iter->pos) 6114 break; 6115 6116 mutex_unlock(&iter->mutex); 6117 6118 ret = wait_on_pipe(iter, 0); 6119 6120 mutex_lock(&iter->mutex); 6121 6122 if (ret) 6123 return ret; 6124 } 6125 6126 return 1; 6127 } 6128 6129 /* 6130 * Consumer reader. 6131 */ 6132 static ssize_t 6133 tracing_read_pipe(struct file *filp, char __user *ubuf, 6134 size_t cnt, loff_t *ppos) 6135 { 6136 struct trace_iterator *iter = filp->private_data; 6137 ssize_t sret; 6138 6139 /* 6140 * Avoid more than one consumer on a single file descriptor 6141 * This is just a matter of traces coherency, the ring buffer itself 6142 * is protected. 6143 */ 6144 mutex_lock(&iter->mutex); 6145 6146 /* return any leftover data */ 6147 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 6148 if (sret != -EBUSY) 6149 goto out; 6150 6151 trace_seq_init(&iter->seq); 6152 6153 if (iter->trace->read) { 6154 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); 6155 if (sret) 6156 goto out; 6157 } 6158 6159 waitagain: 6160 sret = tracing_wait_pipe(filp); 6161 if (sret <= 0) 6162 goto out; 6163 6164 /* stop when tracing is finished */ 6165 if (trace_empty(iter)) { 6166 sret = 0; 6167 goto out; 6168 } 6169 6170 if (cnt >= PAGE_SIZE) 6171 cnt = PAGE_SIZE - 1; 6172 6173 /* reset all but tr, trace, and overruns */ 6174 memset(&iter->seq, 0, 6175 sizeof(struct trace_iterator) - 6176 offsetof(struct trace_iterator, seq)); 6177 cpumask_clear(iter->started); 6178 trace_seq_init(&iter->seq); 6179 iter->pos = -1; 6180 6181 trace_event_read_lock(); 6182 trace_access_lock(iter->cpu_file); 6183 while (trace_find_next_entry_inc(iter) != NULL) { 6184 enum print_line_t ret; 6185 int save_len = iter->seq.seq.len; 6186 6187 ret = print_trace_line(iter); 6188 if (ret == TRACE_TYPE_PARTIAL_LINE) { 6189 /* don't print partial lines */ 6190 iter->seq.seq.len = save_len; 6191 break; 6192 } 6193 if (ret != TRACE_TYPE_NO_CONSUME) 6194 trace_consume(iter); 6195 6196 if (trace_seq_used(&iter->seq) >= cnt) 6197 break; 6198 6199 /* 6200 * Setting the full flag means we reached the trace_seq buffer 6201 * size and we should leave by partial output condition above. 6202 * One of the trace_seq_* functions is not used properly. 6203 */ 6204 WARN_ONCE(iter->seq.full, "full flag set for trace type %d", 6205 iter->ent->type); 6206 } 6207 trace_access_unlock(iter->cpu_file); 6208 trace_event_read_unlock(); 6209 6210 /* Now copy what we have to the user */ 6211 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 6212 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq)) 6213 trace_seq_init(&iter->seq); 6214 6215 /* 6216 * If there was nothing to send to user, in spite of consuming trace 6217 * entries, go back to wait for more entries. 6218 */ 6219 if (sret == -EBUSY) 6220 goto waitagain; 6221 6222 out: 6223 mutex_unlock(&iter->mutex); 6224 6225 return sret; 6226 } 6227 6228 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd, 6229 unsigned int idx) 6230 { 6231 __free_page(spd->pages[idx]); 6232 } 6233 6234 static const struct pipe_buf_operations tracing_pipe_buf_ops = { 6235 .confirm = generic_pipe_buf_confirm, 6236 .release = generic_pipe_buf_release, 6237 .steal = generic_pipe_buf_steal, 6238 .get = generic_pipe_buf_get, 6239 }; 6240 6241 static size_t 6242 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter) 6243 { 6244 size_t count; 6245 int save_len; 6246 int ret; 6247 6248 /* Seq buffer is page-sized, exactly what we need. */ 6249 for (;;) { 6250 save_len = iter->seq.seq.len; 6251 ret = print_trace_line(iter); 6252 6253 if (trace_seq_has_overflowed(&iter->seq)) { 6254 iter->seq.seq.len = save_len; 6255 break; 6256 } 6257 6258 /* 6259 * This should not be hit, because it should only 6260 * be set if the iter->seq overflowed. But check it 6261 * anyway to be safe. 6262 */ 6263 if (ret == TRACE_TYPE_PARTIAL_LINE) { 6264 iter->seq.seq.len = save_len; 6265 break; 6266 } 6267 6268 count = trace_seq_used(&iter->seq) - save_len; 6269 if (rem < count) { 6270 rem = 0; 6271 iter->seq.seq.len = save_len; 6272 break; 6273 } 6274 6275 if (ret != TRACE_TYPE_NO_CONSUME) 6276 trace_consume(iter); 6277 rem -= count; 6278 if (!trace_find_next_entry_inc(iter)) { 6279 rem = 0; 6280 iter->ent = NULL; 6281 break; 6282 } 6283 } 6284 6285 return rem; 6286 } 6287 6288 static ssize_t tracing_splice_read_pipe(struct file *filp, 6289 loff_t *ppos, 6290 struct pipe_inode_info *pipe, 6291 size_t len, 6292 unsigned int flags) 6293 { 6294 struct page *pages_def[PIPE_DEF_BUFFERS]; 6295 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 6296 struct trace_iterator *iter = filp->private_data; 6297 struct splice_pipe_desc spd = { 6298 .pages = pages_def, 6299 .partial = partial_def, 6300 .nr_pages = 0, /* This gets updated below. */ 6301 .nr_pages_max = PIPE_DEF_BUFFERS, 6302 .ops = &tracing_pipe_buf_ops, 6303 .spd_release = tracing_spd_release_pipe, 6304 }; 6305 ssize_t ret; 6306 size_t rem; 6307 unsigned int i; 6308 6309 if (splice_grow_spd(pipe, &spd)) 6310 return -ENOMEM; 6311 6312 mutex_lock(&iter->mutex); 6313 6314 if (iter->trace->splice_read) { 6315 ret = iter->trace->splice_read(iter, filp, 6316 ppos, pipe, len, flags); 6317 if (ret) 6318 goto out_err; 6319 } 6320 6321 ret = tracing_wait_pipe(filp); 6322 if (ret <= 0) 6323 goto out_err; 6324 6325 if (!iter->ent && !trace_find_next_entry_inc(iter)) { 6326 ret = -EFAULT; 6327 goto out_err; 6328 } 6329 6330 trace_event_read_lock(); 6331 trace_access_lock(iter->cpu_file); 6332 6333 /* Fill as many pages as possible. */ 6334 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) { 6335 spd.pages[i] = alloc_page(GFP_KERNEL); 6336 if (!spd.pages[i]) 6337 break; 6338 6339 rem = tracing_fill_pipe_page(rem, iter); 6340 6341 /* Copy the data into the page, so we can start over. */ 6342 ret = trace_seq_to_buffer(&iter->seq, 6343 page_address(spd.pages[i]), 6344 trace_seq_used(&iter->seq)); 6345 if (ret < 0) { 6346 __free_page(spd.pages[i]); 6347 break; 6348 } 6349 spd.partial[i].offset = 0; 6350 spd.partial[i].len = trace_seq_used(&iter->seq); 6351 6352 trace_seq_init(&iter->seq); 6353 } 6354 6355 trace_access_unlock(iter->cpu_file); 6356 trace_event_read_unlock(); 6357 mutex_unlock(&iter->mutex); 6358 6359 spd.nr_pages = i; 6360 6361 if (i) 6362 ret = splice_to_pipe(pipe, &spd); 6363 else 6364 ret = 0; 6365 out: 6366 splice_shrink_spd(&spd); 6367 return ret; 6368 6369 out_err: 6370 mutex_unlock(&iter->mutex); 6371 goto out; 6372 } 6373 6374 static ssize_t 6375 tracing_entries_read(struct file *filp, char __user *ubuf, 6376 size_t cnt, loff_t *ppos) 6377 { 6378 struct inode *inode = file_inode(filp); 6379 struct trace_array *tr = inode->i_private; 6380 int cpu = tracing_get_cpu(inode); 6381 char buf[64]; 6382 int r = 0; 6383 ssize_t ret; 6384 6385 mutex_lock(&trace_types_lock); 6386 6387 if (cpu == RING_BUFFER_ALL_CPUS) { 6388 int cpu, buf_size_same; 6389 unsigned long size; 6390 6391 size = 0; 6392 buf_size_same = 1; 6393 /* check if all cpu sizes are same */ 6394 for_each_tracing_cpu(cpu) { 6395 /* fill in the size from first enabled cpu */ 6396 if (size == 0) 6397 size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries; 6398 if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) { 6399 buf_size_same = 0; 6400 break; 6401 } 6402 } 6403 6404 if (buf_size_same) { 6405 if (!ring_buffer_expanded) 6406 r = sprintf(buf, "%lu (expanded: %lu)\n", 6407 size >> 10, 6408 trace_buf_size >> 10); 6409 else 6410 r = sprintf(buf, "%lu\n", size >> 10); 6411 } else 6412 r = sprintf(buf, "X\n"); 6413 } else 6414 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10); 6415 6416 mutex_unlock(&trace_types_lock); 6417 6418 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 6419 return ret; 6420 } 6421 6422 static ssize_t 6423 tracing_entries_write(struct file *filp, const char __user *ubuf, 6424 size_t cnt, loff_t *ppos) 6425 { 6426 struct inode *inode = file_inode(filp); 6427 struct trace_array *tr = inode->i_private; 6428 unsigned long val; 6429 int ret; 6430 6431 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 6432 if (ret) 6433 return ret; 6434 6435 /* must have at least 1 entry */ 6436 if (!val) 6437 return -EINVAL; 6438 6439 /* value is in KB */ 6440 val <<= 10; 6441 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode)); 6442 if (ret < 0) 6443 return ret; 6444 6445 *ppos += cnt; 6446 6447 return cnt; 6448 } 6449 6450 static ssize_t 6451 tracing_total_entries_read(struct file *filp, char __user *ubuf, 6452 size_t cnt, loff_t *ppos) 6453 { 6454 struct trace_array *tr = filp->private_data; 6455 char buf[64]; 6456 int r, cpu; 6457 unsigned long size = 0, expanded_size = 0; 6458 6459 mutex_lock(&trace_types_lock); 6460 for_each_tracing_cpu(cpu) { 6461 size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10; 6462 if (!ring_buffer_expanded) 6463 expanded_size += trace_buf_size >> 10; 6464 } 6465 if (ring_buffer_expanded) 6466 r = sprintf(buf, "%lu\n", size); 6467 else 6468 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size); 6469 mutex_unlock(&trace_types_lock); 6470 6471 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 6472 } 6473 6474 static ssize_t 6475 tracing_free_buffer_write(struct file *filp, const char __user *ubuf, 6476 size_t cnt, loff_t *ppos) 6477 { 6478 /* 6479 * There is no need to read what the user has written, this function 6480 * is just to make sure that there is no error when "echo" is used 6481 */ 6482 6483 *ppos += cnt; 6484 6485 return cnt; 6486 } 6487 6488 static int 6489 tracing_free_buffer_release(struct inode *inode, struct file *filp) 6490 { 6491 struct trace_array *tr = inode->i_private; 6492 6493 /* disable tracing ? */ 6494 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE) 6495 tracer_tracing_off(tr); 6496 /* resize the ring buffer to 0 */ 6497 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS); 6498 6499 trace_array_put(tr); 6500 6501 return 0; 6502 } 6503 6504 static ssize_t 6505 tracing_mark_write(struct file *filp, const char __user *ubuf, 6506 size_t cnt, loff_t *fpos) 6507 { 6508 struct trace_array *tr = filp->private_data; 6509 struct ring_buffer_event *event; 6510 enum event_trigger_type tt = ETT_NONE; 6511 struct trace_buffer *buffer; 6512 struct print_entry *entry; 6513 unsigned long irq_flags; 6514 ssize_t written; 6515 int size; 6516 int len; 6517 6518 /* Used in tracing_mark_raw_write() as well */ 6519 #define FAULTED_STR "<faulted>" 6520 #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */ 6521 6522 if (tracing_disabled) 6523 return -EINVAL; 6524 6525 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) 6526 return -EINVAL; 6527 6528 if (cnt > TRACE_BUF_SIZE) 6529 cnt = TRACE_BUF_SIZE; 6530 6531 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); 6532 6533 local_save_flags(irq_flags); 6534 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */ 6535 6536 /* If less than "<faulted>", then make sure we can still add that */ 6537 if (cnt < FAULTED_SIZE) 6538 size += FAULTED_SIZE - cnt; 6539 6540 buffer = tr->array_buffer.buffer; 6541 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 6542 irq_flags, preempt_count()); 6543 if (unlikely(!event)) 6544 /* Ring buffer disabled, return as if not open for write */ 6545 return -EBADF; 6546 6547 entry = ring_buffer_event_data(event); 6548 entry->ip = _THIS_IP_; 6549 6550 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt); 6551 if (len) { 6552 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE); 6553 cnt = FAULTED_SIZE; 6554 written = -EFAULT; 6555 } else 6556 written = cnt; 6557 len = cnt; 6558 6559 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) { 6560 /* do not add \n before testing triggers, but add \0 */ 6561 entry->buf[cnt] = '\0'; 6562 tt = event_triggers_call(tr->trace_marker_file, entry, event); 6563 } 6564 6565 if (entry->buf[cnt - 1] != '\n') { 6566 entry->buf[cnt] = '\n'; 6567 entry->buf[cnt + 1] = '\0'; 6568 } else 6569 entry->buf[cnt] = '\0'; 6570 6571 __buffer_unlock_commit(buffer, event); 6572 6573 if (tt) 6574 event_triggers_post_call(tr->trace_marker_file, tt); 6575 6576 if (written > 0) 6577 *fpos += written; 6578 6579 return written; 6580 } 6581 6582 /* Limit it for now to 3K (including tag) */ 6583 #define RAW_DATA_MAX_SIZE (1024*3) 6584 6585 static ssize_t 6586 tracing_mark_raw_write(struct file *filp, const char __user *ubuf, 6587 size_t cnt, loff_t *fpos) 6588 { 6589 struct trace_array *tr = filp->private_data; 6590 struct ring_buffer_event *event; 6591 struct trace_buffer *buffer; 6592 struct raw_data_entry *entry; 6593 unsigned long irq_flags; 6594 ssize_t written; 6595 int size; 6596 int len; 6597 6598 #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int)) 6599 6600 if (tracing_disabled) 6601 return -EINVAL; 6602 6603 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) 6604 return -EINVAL; 6605 6606 /* The marker must at least have a tag id */ 6607 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE) 6608 return -EINVAL; 6609 6610 if (cnt > TRACE_BUF_SIZE) 6611 cnt = TRACE_BUF_SIZE; 6612 6613 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); 6614 6615 local_save_flags(irq_flags); 6616 size = sizeof(*entry) + cnt; 6617 if (cnt < FAULT_SIZE_ID) 6618 size += FAULT_SIZE_ID - cnt; 6619 6620 buffer = tr->array_buffer.buffer; 6621 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size, 6622 irq_flags, preempt_count()); 6623 if (!event) 6624 /* Ring buffer disabled, return as if not open for write */ 6625 return -EBADF; 6626 6627 entry = ring_buffer_event_data(event); 6628 6629 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt); 6630 if (len) { 6631 entry->id = -1; 6632 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE); 6633 written = -EFAULT; 6634 } else 6635 written = cnt; 6636 6637 __buffer_unlock_commit(buffer, event); 6638 6639 if (written > 0) 6640 *fpos += written; 6641 6642 return written; 6643 } 6644 6645 static int tracing_clock_show(struct seq_file *m, void *v) 6646 { 6647 struct trace_array *tr = m->private; 6648 int i; 6649 6650 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) 6651 seq_printf(m, 6652 "%s%s%s%s", i ? " " : "", 6653 i == tr->clock_id ? "[" : "", trace_clocks[i].name, 6654 i == tr->clock_id ? "]" : ""); 6655 seq_putc(m, '\n'); 6656 6657 return 0; 6658 } 6659 6660 int tracing_set_clock(struct trace_array *tr, const char *clockstr) 6661 { 6662 int i; 6663 6664 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) { 6665 if (strcmp(trace_clocks[i].name, clockstr) == 0) 6666 break; 6667 } 6668 if (i == ARRAY_SIZE(trace_clocks)) 6669 return -EINVAL; 6670 6671 mutex_lock(&trace_types_lock); 6672 6673 tr->clock_id = i; 6674 6675 ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func); 6676 6677 /* 6678 * New clock may not be consistent with the previous clock. 6679 * Reset the buffer so that it doesn't have incomparable timestamps. 6680 */ 6681 tracing_reset_online_cpus(&tr->array_buffer); 6682 6683 #ifdef CONFIG_TRACER_MAX_TRACE 6684 if (tr->max_buffer.buffer) 6685 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func); 6686 tracing_reset_online_cpus(&tr->max_buffer); 6687 #endif 6688 6689 mutex_unlock(&trace_types_lock); 6690 6691 return 0; 6692 } 6693 6694 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf, 6695 size_t cnt, loff_t *fpos) 6696 { 6697 struct seq_file *m = filp->private_data; 6698 struct trace_array *tr = m->private; 6699 char buf[64]; 6700 const char *clockstr; 6701 int ret; 6702 6703 if (cnt >= sizeof(buf)) 6704 return -EINVAL; 6705 6706 if (copy_from_user(buf, ubuf, cnt)) 6707 return -EFAULT; 6708 6709 buf[cnt] = 0; 6710 6711 clockstr = strstrip(buf); 6712 6713 ret = tracing_set_clock(tr, clockstr); 6714 if (ret) 6715 return ret; 6716 6717 *fpos += cnt; 6718 6719 return cnt; 6720 } 6721 6722 static int tracing_clock_open(struct inode *inode, struct file *file) 6723 { 6724 struct trace_array *tr = inode->i_private; 6725 int ret; 6726 6727 ret = tracing_check_open_get_tr(tr); 6728 if (ret) 6729 return ret; 6730 6731 ret = single_open(file, tracing_clock_show, inode->i_private); 6732 if (ret < 0) 6733 trace_array_put(tr); 6734 6735 return ret; 6736 } 6737 6738 static int tracing_time_stamp_mode_show(struct seq_file *m, void *v) 6739 { 6740 struct trace_array *tr = m->private; 6741 6742 mutex_lock(&trace_types_lock); 6743 6744 if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer)) 6745 seq_puts(m, "delta [absolute]\n"); 6746 else 6747 seq_puts(m, "[delta] absolute\n"); 6748 6749 mutex_unlock(&trace_types_lock); 6750 6751 return 0; 6752 } 6753 6754 static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file) 6755 { 6756 struct trace_array *tr = inode->i_private; 6757 int ret; 6758 6759 ret = tracing_check_open_get_tr(tr); 6760 if (ret) 6761 return ret; 6762 6763 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private); 6764 if (ret < 0) 6765 trace_array_put(tr); 6766 6767 return ret; 6768 } 6769 6770 int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs) 6771 { 6772 int ret = 0; 6773 6774 mutex_lock(&trace_types_lock); 6775 6776 if (abs && tr->time_stamp_abs_ref++) 6777 goto out; 6778 6779 if (!abs) { 6780 if (WARN_ON_ONCE(!tr->time_stamp_abs_ref)) { 6781 ret = -EINVAL; 6782 goto out; 6783 } 6784 6785 if (--tr->time_stamp_abs_ref) 6786 goto out; 6787 } 6788 6789 ring_buffer_set_time_stamp_abs(tr->array_buffer.buffer, abs); 6790 6791 #ifdef CONFIG_TRACER_MAX_TRACE 6792 if (tr->max_buffer.buffer) 6793 ring_buffer_set_time_stamp_abs(tr->max_buffer.buffer, abs); 6794 #endif 6795 out: 6796 mutex_unlock(&trace_types_lock); 6797 6798 return ret; 6799 } 6800 6801 struct ftrace_buffer_info { 6802 struct trace_iterator iter; 6803 void *spare; 6804 unsigned int spare_cpu; 6805 unsigned int read; 6806 }; 6807 6808 #ifdef CONFIG_TRACER_SNAPSHOT 6809 static int tracing_snapshot_open(struct inode *inode, struct file *file) 6810 { 6811 struct trace_array *tr = inode->i_private; 6812 struct trace_iterator *iter; 6813 struct seq_file *m; 6814 int ret; 6815 6816 ret = tracing_check_open_get_tr(tr); 6817 if (ret) 6818 return ret; 6819 6820 if (file->f_mode & FMODE_READ) { 6821 iter = __tracing_open(inode, file, true); 6822 if (IS_ERR(iter)) 6823 ret = PTR_ERR(iter); 6824 } else { 6825 /* Writes still need the seq_file to hold the private data */ 6826 ret = -ENOMEM; 6827 m = kzalloc(sizeof(*m), GFP_KERNEL); 6828 if (!m) 6829 goto out; 6830 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 6831 if (!iter) { 6832 kfree(m); 6833 goto out; 6834 } 6835 ret = 0; 6836 6837 iter->tr = tr; 6838 iter->array_buffer = &tr->max_buffer; 6839 iter->cpu_file = tracing_get_cpu(inode); 6840 m->private = iter; 6841 file->private_data = m; 6842 } 6843 out: 6844 if (ret < 0) 6845 trace_array_put(tr); 6846 6847 return ret; 6848 } 6849 6850 static ssize_t 6851 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt, 6852 loff_t *ppos) 6853 { 6854 struct seq_file *m = filp->private_data; 6855 struct trace_iterator *iter = m->private; 6856 struct trace_array *tr = iter->tr; 6857 unsigned long val; 6858 int ret; 6859 6860 ret = tracing_update_buffers(); 6861 if (ret < 0) 6862 return ret; 6863 6864 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 6865 if (ret) 6866 return ret; 6867 6868 mutex_lock(&trace_types_lock); 6869 6870 if (tr->current_trace->use_max_tr) { 6871 ret = -EBUSY; 6872 goto out; 6873 } 6874 6875 arch_spin_lock(&tr->max_lock); 6876 if (tr->cond_snapshot) 6877 ret = -EBUSY; 6878 arch_spin_unlock(&tr->max_lock); 6879 if (ret) 6880 goto out; 6881 6882 switch (val) { 6883 case 0: 6884 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 6885 ret = -EINVAL; 6886 break; 6887 } 6888 if (tr->allocated_snapshot) 6889 free_snapshot(tr); 6890 break; 6891 case 1: 6892 /* Only allow per-cpu swap if the ring buffer supports it */ 6893 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP 6894 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 6895 ret = -EINVAL; 6896 break; 6897 } 6898 #endif 6899 if (tr->allocated_snapshot) 6900 ret = resize_buffer_duplicate_size(&tr->max_buffer, 6901 &tr->array_buffer, iter->cpu_file); 6902 else 6903 ret = tracing_alloc_snapshot_instance(tr); 6904 if (ret < 0) 6905 break; 6906 local_irq_disable(); 6907 /* Now, we're going to swap */ 6908 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 6909 update_max_tr(tr, current, smp_processor_id(), NULL); 6910 else 6911 update_max_tr_single(tr, current, iter->cpu_file); 6912 local_irq_enable(); 6913 break; 6914 default: 6915 if (tr->allocated_snapshot) { 6916 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 6917 tracing_reset_online_cpus(&tr->max_buffer); 6918 else 6919 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file); 6920 } 6921 break; 6922 } 6923 6924 if (ret >= 0) { 6925 *ppos += cnt; 6926 ret = cnt; 6927 } 6928 out: 6929 mutex_unlock(&trace_types_lock); 6930 return ret; 6931 } 6932 6933 static int tracing_snapshot_release(struct inode *inode, struct file *file) 6934 { 6935 struct seq_file *m = file->private_data; 6936 int ret; 6937 6938 ret = tracing_release(inode, file); 6939 6940 if (file->f_mode & FMODE_READ) 6941 return ret; 6942 6943 /* If write only, the seq_file is just a stub */ 6944 if (m) 6945 kfree(m->private); 6946 kfree(m); 6947 6948 return 0; 6949 } 6950 6951 static int tracing_buffers_open(struct inode *inode, struct file *filp); 6952 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, 6953 size_t count, loff_t *ppos); 6954 static int tracing_buffers_release(struct inode *inode, struct file *file); 6955 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos, 6956 struct pipe_inode_info *pipe, size_t len, unsigned int flags); 6957 6958 static int snapshot_raw_open(struct inode *inode, struct file *filp) 6959 { 6960 struct ftrace_buffer_info *info; 6961 int ret; 6962 6963 /* The following checks for tracefs lockdown */ 6964 ret = tracing_buffers_open(inode, filp); 6965 if (ret < 0) 6966 return ret; 6967 6968 info = filp->private_data; 6969 6970 if (info->iter.trace->use_max_tr) { 6971 tracing_buffers_release(inode, filp); 6972 return -EBUSY; 6973 } 6974 6975 info->iter.snapshot = true; 6976 info->iter.array_buffer = &info->iter.tr->max_buffer; 6977 6978 return ret; 6979 } 6980 6981 #endif /* CONFIG_TRACER_SNAPSHOT */ 6982 6983 6984 static const struct file_operations tracing_thresh_fops = { 6985 .open = tracing_open_generic, 6986 .read = tracing_thresh_read, 6987 .write = tracing_thresh_write, 6988 .llseek = generic_file_llseek, 6989 }; 6990 6991 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) 6992 static const struct file_operations tracing_max_lat_fops = { 6993 .open = tracing_open_generic, 6994 .read = tracing_max_lat_read, 6995 .write = tracing_max_lat_write, 6996 .llseek = generic_file_llseek, 6997 }; 6998 #endif 6999 7000 static const struct file_operations set_tracer_fops = { 7001 .open = tracing_open_generic, 7002 .read = tracing_set_trace_read, 7003 .write = tracing_set_trace_write, 7004 .llseek = generic_file_llseek, 7005 }; 7006 7007 static const struct file_operations tracing_pipe_fops = { 7008 .open = tracing_open_pipe, 7009 .poll = tracing_poll_pipe, 7010 .read = tracing_read_pipe, 7011 .splice_read = tracing_splice_read_pipe, 7012 .release = tracing_release_pipe, 7013 .llseek = no_llseek, 7014 }; 7015 7016 static const struct file_operations tracing_entries_fops = { 7017 .open = tracing_open_generic_tr, 7018 .read = tracing_entries_read, 7019 .write = tracing_entries_write, 7020 .llseek = generic_file_llseek, 7021 .release = tracing_release_generic_tr, 7022 }; 7023 7024 static const struct file_operations tracing_total_entries_fops = { 7025 .open = tracing_open_generic_tr, 7026 .read = tracing_total_entries_read, 7027 .llseek = generic_file_llseek, 7028 .release = tracing_release_generic_tr, 7029 }; 7030 7031 static const struct file_operations tracing_free_buffer_fops = { 7032 .open = tracing_open_generic_tr, 7033 .write = tracing_free_buffer_write, 7034 .release = tracing_free_buffer_release, 7035 }; 7036 7037 static const struct file_operations tracing_mark_fops = { 7038 .open = tracing_open_generic_tr, 7039 .write = tracing_mark_write, 7040 .llseek = generic_file_llseek, 7041 .release = tracing_release_generic_tr, 7042 }; 7043 7044 static const struct file_operations tracing_mark_raw_fops = { 7045 .open = tracing_open_generic_tr, 7046 .write = tracing_mark_raw_write, 7047 .llseek = generic_file_llseek, 7048 .release = tracing_release_generic_tr, 7049 }; 7050 7051 static const struct file_operations trace_clock_fops = { 7052 .open = tracing_clock_open, 7053 .read = seq_read, 7054 .llseek = seq_lseek, 7055 .release = tracing_single_release_tr, 7056 .write = tracing_clock_write, 7057 }; 7058 7059 static const struct file_operations trace_time_stamp_mode_fops = { 7060 .open = tracing_time_stamp_mode_open, 7061 .read = seq_read, 7062 .llseek = seq_lseek, 7063 .release = tracing_single_release_tr, 7064 }; 7065 7066 #ifdef CONFIG_TRACER_SNAPSHOT 7067 static const struct file_operations snapshot_fops = { 7068 .open = tracing_snapshot_open, 7069 .read = seq_read, 7070 .write = tracing_snapshot_write, 7071 .llseek = tracing_lseek, 7072 .release = tracing_snapshot_release, 7073 }; 7074 7075 static const struct file_operations snapshot_raw_fops = { 7076 .open = snapshot_raw_open, 7077 .read = tracing_buffers_read, 7078 .release = tracing_buffers_release, 7079 .splice_read = tracing_buffers_splice_read, 7080 .llseek = no_llseek, 7081 }; 7082 7083 #endif /* CONFIG_TRACER_SNAPSHOT */ 7084 7085 #define TRACING_LOG_ERRS_MAX 8 7086 #define TRACING_LOG_LOC_MAX 128 7087 7088 #define CMD_PREFIX " Command: " 7089 7090 struct err_info { 7091 const char **errs; /* ptr to loc-specific array of err strings */ 7092 u8 type; /* index into errs -> specific err string */ 7093 u8 pos; /* MAX_FILTER_STR_VAL = 256 */ 7094 u64 ts; 7095 }; 7096 7097 struct tracing_log_err { 7098 struct list_head list; 7099 struct err_info info; 7100 char loc[TRACING_LOG_LOC_MAX]; /* err location */ 7101 char cmd[MAX_FILTER_STR_VAL]; /* what caused err */ 7102 }; 7103 7104 static DEFINE_MUTEX(tracing_err_log_lock); 7105 7106 static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr) 7107 { 7108 struct tracing_log_err *err; 7109 7110 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) { 7111 err = kzalloc(sizeof(*err), GFP_KERNEL); 7112 if (!err) 7113 err = ERR_PTR(-ENOMEM); 7114 tr->n_err_log_entries++; 7115 7116 return err; 7117 } 7118 7119 err = list_first_entry(&tr->err_log, struct tracing_log_err, list); 7120 list_del(&err->list); 7121 7122 return err; 7123 } 7124 7125 /** 7126 * err_pos - find the position of a string within a command for error careting 7127 * @cmd: The tracing command that caused the error 7128 * @str: The string to position the caret at within @cmd 7129 * 7130 * Finds the position of the first occurence of @str within @cmd. The 7131 * return value can be passed to tracing_log_err() for caret placement 7132 * within @cmd. 7133 * 7134 * Returns the index within @cmd of the first occurence of @str or 0 7135 * if @str was not found. 7136 */ 7137 unsigned int err_pos(char *cmd, const char *str) 7138 { 7139 char *found; 7140 7141 if (WARN_ON(!strlen(cmd))) 7142 return 0; 7143 7144 found = strstr(cmd, str); 7145 if (found) 7146 return found - cmd; 7147 7148 return 0; 7149 } 7150 7151 /** 7152 * tracing_log_err - write an error to the tracing error log 7153 * @tr: The associated trace array for the error (NULL for top level array) 7154 * @loc: A string describing where the error occurred 7155 * @cmd: The tracing command that caused the error 7156 * @errs: The array of loc-specific static error strings 7157 * @type: The index into errs[], which produces the specific static err string 7158 * @pos: The position the caret should be placed in the cmd 7159 * 7160 * Writes an error into tracing/error_log of the form: 7161 * 7162 * <loc>: error: <text> 7163 * Command: <cmd> 7164 * ^ 7165 * 7166 * tracing/error_log is a small log file containing the last 7167 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated 7168 * unless there has been a tracing error, and the error log can be 7169 * cleared and have its memory freed by writing the empty string in 7170 * truncation mode to it i.e. echo > tracing/error_log. 7171 * 7172 * NOTE: the @errs array along with the @type param are used to 7173 * produce a static error string - this string is not copied and saved 7174 * when the error is logged - only a pointer to it is saved. See 7175 * existing callers for examples of how static strings are typically 7176 * defined for use with tracing_log_err(). 7177 */ 7178 void tracing_log_err(struct trace_array *tr, 7179 const char *loc, const char *cmd, 7180 const char **errs, u8 type, u8 pos) 7181 { 7182 struct tracing_log_err *err; 7183 7184 if (!tr) 7185 tr = &global_trace; 7186 7187 mutex_lock(&tracing_err_log_lock); 7188 err = get_tracing_log_err(tr); 7189 if (PTR_ERR(err) == -ENOMEM) { 7190 mutex_unlock(&tracing_err_log_lock); 7191 return; 7192 } 7193 7194 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc); 7195 snprintf(err->cmd, MAX_FILTER_STR_VAL,"\n" CMD_PREFIX "%s\n", cmd); 7196 7197 err->info.errs = errs; 7198 err->info.type = type; 7199 err->info.pos = pos; 7200 err->info.ts = local_clock(); 7201 7202 list_add_tail(&err->list, &tr->err_log); 7203 mutex_unlock(&tracing_err_log_lock); 7204 } 7205 7206 static void clear_tracing_err_log(struct trace_array *tr) 7207 { 7208 struct tracing_log_err *err, *next; 7209 7210 mutex_lock(&tracing_err_log_lock); 7211 list_for_each_entry_safe(err, next, &tr->err_log, list) { 7212 list_del(&err->list); 7213 kfree(err); 7214 } 7215 7216 tr->n_err_log_entries = 0; 7217 mutex_unlock(&tracing_err_log_lock); 7218 } 7219 7220 static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos) 7221 { 7222 struct trace_array *tr = m->private; 7223 7224 mutex_lock(&tracing_err_log_lock); 7225 7226 return seq_list_start(&tr->err_log, *pos); 7227 } 7228 7229 static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos) 7230 { 7231 struct trace_array *tr = m->private; 7232 7233 return seq_list_next(v, &tr->err_log, pos); 7234 } 7235 7236 static void tracing_err_log_seq_stop(struct seq_file *m, void *v) 7237 { 7238 mutex_unlock(&tracing_err_log_lock); 7239 } 7240 7241 static void tracing_err_log_show_pos(struct seq_file *m, u8 pos) 7242 { 7243 u8 i; 7244 7245 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++) 7246 seq_putc(m, ' '); 7247 for (i = 0; i < pos; i++) 7248 seq_putc(m, ' '); 7249 seq_puts(m, "^\n"); 7250 } 7251 7252 static int tracing_err_log_seq_show(struct seq_file *m, void *v) 7253 { 7254 struct tracing_log_err *err = v; 7255 7256 if (err) { 7257 const char *err_text = err->info.errs[err->info.type]; 7258 u64 sec = err->info.ts; 7259 u32 nsec; 7260 7261 nsec = do_div(sec, NSEC_PER_SEC); 7262 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000, 7263 err->loc, err_text); 7264 seq_printf(m, "%s", err->cmd); 7265 tracing_err_log_show_pos(m, err->info.pos); 7266 } 7267 7268 return 0; 7269 } 7270 7271 static const struct seq_operations tracing_err_log_seq_ops = { 7272 .start = tracing_err_log_seq_start, 7273 .next = tracing_err_log_seq_next, 7274 .stop = tracing_err_log_seq_stop, 7275 .show = tracing_err_log_seq_show 7276 }; 7277 7278 static int tracing_err_log_open(struct inode *inode, struct file *file) 7279 { 7280 struct trace_array *tr = inode->i_private; 7281 int ret = 0; 7282 7283 ret = tracing_check_open_get_tr(tr); 7284 if (ret) 7285 return ret; 7286 7287 /* If this file was opened for write, then erase contents */ 7288 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) 7289 clear_tracing_err_log(tr); 7290 7291 if (file->f_mode & FMODE_READ) { 7292 ret = seq_open(file, &tracing_err_log_seq_ops); 7293 if (!ret) { 7294 struct seq_file *m = file->private_data; 7295 m->private = tr; 7296 } else { 7297 trace_array_put(tr); 7298 } 7299 } 7300 return ret; 7301 } 7302 7303 static ssize_t tracing_err_log_write(struct file *file, 7304 const char __user *buffer, 7305 size_t count, loff_t *ppos) 7306 { 7307 return count; 7308 } 7309 7310 static int tracing_err_log_release(struct inode *inode, struct file *file) 7311 { 7312 struct trace_array *tr = inode->i_private; 7313 7314 trace_array_put(tr); 7315 7316 if (file->f_mode & FMODE_READ) 7317 seq_release(inode, file); 7318 7319 return 0; 7320 } 7321 7322 static const struct file_operations tracing_err_log_fops = { 7323 .open = tracing_err_log_open, 7324 .write = tracing_err_log_write, 7325 .read = seq_read, 7326 .llseek = seq_lseek, 7327 .release = tracing_err_log_release, 7328 }; 7329 7330 static int tracing_buffers_open(struct inode *inode, struct file *filp) 7331 { 7332 struct trace_array *tr = inode->i_private; 7333 struct ftrace_buffer_info *info; 7334 int ret; 7335 7336 ret = tracing_check_open_get_tr(tr); 7337 if (ret) 7338 return ret; 7339 7340 info = kzalloc(sizeof(*info), GFP_KERNEL); 7341 if (!info) { 7342 trace_array_put(tr); 7343 return -ENOMEM; 7344 } 7345 7346 mutex_lock(&trace_types_lock); 7347 7348 info->iter.tr = tr; 7349 info->iter.cpu_file = tracing_get_cpu(inode); 7350 info->iter.trace = tr->current_trace; 7351 info->iter.array_buffer = &tr->array_buffer; 7352 info->spare = NULL; 7353 /* Force reading ring buffer for first read */ 7354 info->read = (unsigned int)-1; 7355 7356 filp->private_data = info; 7357 7358 tr->current_trace->ref++; 7359 7360 mutex_unlock(&trace_types_lock); 7361 7362 ret = nonseekable_open(inode, filp); 7363 if (ret < 0) 7364 trace_array_put(tr); 7365 7366 return ret; 7367 } 7368 7369 static __poll_t 7370 tracing_buffers_poll(struct file *filp, poll_table *poll_table) 7371 { 7372 struct ftrace_buffer_info *info = filp->private_data; 7373 struct trace_iterator *iter = &info->iter; 7374 7375 return trace_poll(iter, filp, poll_table); 7376 } 7377 7378 static ssize_t 7379 tracing_buffers_read(struct file *filp, char __user *ubuf, 7380 size_t count, loff_t *ppos) 7381 { 7382 struct ftrace_buffer_info *info = filp->private_data; 7383 struct trace_iterator *iter = &info->iter; 7384 ssize_t ret = 0; 7385 ssize_t size; 7386 7387 if (!count) 7388 return 0; 7389 7390 #ifdef CONFIG_TRACER_MAX_TRACE 7391 if (iter->snapshot && iter->tr->current_trace->use_max_tr) 7392 return -EBUSY; 7393 #endif 7394 7395 if (!info->spare) { 7396 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer, 7397 iter->cpu_file); 7398 if (IS_ERR(info->spare)) { 7399 ret = PTR_ERR(info->spare); 7400 info->spare = NULL; 7401 } else { 7402 info->spare_cpu = iter->cpu_file; 7403 } 7404 } 7405 if (!info->spare) 7406 return ret; 7407 7408 /* Do we have previous read data to read? */ 7409 if (info->read < PAGE_SIZE) 7410 goto read; 7411 7412 again: 7413 trace_access_lock(iter->cpu_file); 7414 ret = ring_buffer_read_page(iter->array_buffer->buffer, 7415 &info->spare, 7416 count, 7417 iter->cpu_file, 0); 7418 trace_access_unlock(iter->cpu_file); 7419 7420 if (ret < 0) { 7421 if (trace_empty(iter)) { 7422 if ((filp->f_flags & O_NONBLOCK)) 7423 return -EAGAIN; 7424 7425 ret = wait_on_pipe(iter, 0); 7426 if (ret) 7427 return ret; 7428 7429 goto again; 7430 } 7431 return 0; 7432 } 7433 7434 info->read = 0; 7435 read: 7436 size = PAGE_SIZE - info->read; 7437 if (size > count) 7438 size = count; 7439 7440 ret = copy_to_user(ubuf, info->spare + info->read, size); 7441 if (ret == size) 7442 return -EFAULT; 7443 7444 size -= ret; 7445 7446 *ppos += size; 7447 info->read += size; 7448 7449 return size; 7450 } 7451 7452 static int tracing_buffers_release(struct inode *inode, struct file *file) 7453 { 7454 struct ftrace_buffer_info *info = file->private_data; 7455 struct trace_iterator *iter = &info->iter; 7456 7457 mutex_lock(&trace_types_lock); 7458 7459 iter->tr->current_trace->ref--; 7460 7461 __trace_array_put(iter->tr); 7462 7463 if (info->spare) 7464 ring_buffer_free_read_page(iter->array_buffer->buffer, 7465 info->spare_cpu, info->spare); 7466 kfree(info); 7467 7468 mutex_unlock(&trace_types_lock); 7469 7470 return 0; 7471 } 7472 7473 struct buffer_ref { 7474 struct trace_buffer *buffer; 7475 void *page; 7476 int cpu; 7477 refcount_t refcount; 7478 }; 7479 7480 static void buffer_ref_release(struct buffer_ref *ref) 7481 { 7482 if (!refcount_dec_and_test(&ref->refcount)) 7483 return; 7484 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page); 7485 kfree(ref); 7486 } 7487 7488 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe, 7489 struct pipe_buffer *buf) 7490 { 7491 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 7492 7493 buffer_ref_release(ref); 7494 buf->private = 0; 7495 } 7496 7497 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe, 7498 struct pipe_buffer *buf) 7499 { 7500 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 7501 7502 if (refcount_read(&ref->refcount) > INT_MAX/2) 7503 return false; 7504 7505 refcount_inc(&ref->refcount); 7506 return true; 7507 } 7508 7509 /* Pipe buffer operations for a buffer. */ 7510 static const struct pipe_buf_operations buffer_pipe_buf_ops = { 7511 .confirm = generic_pipe_buf_confirm, 7512 .release = buffer_pipe_buf_release, 7513 .steal = generic_pipe_buf_nosteal, 7514 .get = buffer_pipe_buf_get, 7515 }; 7516 7517 /* 7518 * Callback from splice_to_pipe(), if we need to release some pages 7519 * at the end of the spd in case we error'ed out in filling the pipe. 7520 */ 7521 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i) 7522 { 7523 struct buffer_ref *ref = 7524 (struct buffer_ref *)spd->partial[i].private; 7525 7526 buffer_ref_release(ref); 7527 spd->partial[i].private = 0; 7528 } 7529 7530 static ssize_t 7531 tracing_buffers_splice_read(struct file *file, loff_t *ppos, 7532 struct pipe_inode_info *pipe, size_t len, 7533 unsigned int flags) 7534 { 7535 struct ftrace_buffer_info *info = file->private_data; 7536 struct trace_iterator *iter = &info->iter; 7537 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 7538 struct page *pages_def[PIPE_DEF_BUFFERS]; 7539 struct splice_pipe_desc spd = { 7540 .pages = pages_def, 7541 .partial = partial_def, 7542 .nr_pages_max = PIPE_DEF_BUFFERS, 7543 .ops = &buffer_pipe_buf_ops, 7544 .spd_release = buffer_spd_release, 7545 }; 7546 struct buffer_ref *ref; 7547 int entries, i; 7548 ssize_t ret = 0; 7549 7550 #ifdef CONFIG_TRACER_MAX_TRACE 7551 if (iter->snapshot && iter->tr->current_trace->use_max_tr) 7552 return -EBUSY; 7553 #endif 7554 7555 if (*ppos & (PAGE_SIZE - 1)) 7556 return -EINVAL; 7557 7558 if (len & (PAGE_SIZE - 1)) { 7559 if (len < PAGE_SIZE) 7560 return -EINVAL; 7561 len &= PAGE_MASK; 7562 } 7563 7564 if (splice_grow_spd(pipe, &spd)) 7565 return -ENOMEM; 7566 7567 again: 7568 trace_access_lock(iter->cpu_file); 7569 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file); 7570 7571 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) { 7572 struct page *page; 7573 int r; 7574 7575 ref = kzalloc(sizeof(*ref), GFP_KERNEL); 7576 if (!ref) { 7577 ret = -ENOMEM; 7578 break; 7579 } 7580 7581 refcount_set(&ref->refcount, 1); 7582 ref->buffer = iter->array_buffer->buffer; 7583 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file); 7584 if (IS_ERR(ref->page)) { 7585 ret = PTR_ERR(ref->page); 7586 ref->page = NULL; 7587 kfree(ref); 7588 break; 7589 } 7590 ref->cpu = iter->cpu_file; 7591 7592 r = ring_buffer_read_page(ref->buffer, &ref->page, 7593 len, iter->cpu_file, 1); 7594 if (r < 0) { 7595 ring_buffer_free_read_page(ref->buffer, ref->cpu, 7596 ref->page); 7597 kfree(ref); 7598 break; 7599 } 7600 7601 page = virt_to_page(ref->page); 7602 7603 spd.pages[i] = page; 7604 spd.partial[i].len = PAGE_SIZE; 7605 spd.partial[i].offset = 0; 7606 spd.partial[i].private = (unsigned long)ref; 7607 spd.nr_pages++; 7608 *ppos += PAGE_SIZE; 7609 7610 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file); 7611 } 7612 7613 trace_access_unlock(iter->cpu_file); 7614 spd.nr_pages = i; 7615 7616 /* did we read anything? */ 7617 if (!spd.nr_pages) { 7618 if (ret) 7619 goto out; 7620 7621 ret = -EAGAIN; 7622 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) 7623 goto out; 7624 7625 ret = wait_on_pipe(iter, iter->tr->buffer_percent); 7626 if (ret) 7627 goto out; 7628 7629 goto again; 7630 } 7631 7632 ret = splice_to_pipe(pipe, &spd); 7633 out: 7634 splice_shrink_spd(&spd); 7635 7636 return ret; 7637 } 7638 7639 static const struct file_operations tracing_buffers_fops = { 7640 .open = tracing_buffers_open, 7641 .read = tracing_buffers_read, 7642 .poll = tracing_buffers_poll, 7643 .release = tracing_buffers_release, 7644 .splice_read = tracing_buffers_splice_read, 7645 .llseek = no_llseek, 7646 }; 7647 7648 static ssize_t 7649 tracing_stats_read(struct file *filp, char __user *ubuf, 7650 size_t count, loff_t *ppos) 7651 { 7652 struct inode *inode = file_inode(filp); 7653 struct trace_array *tr = inode->i_private; 7654 struct array_buffer *trace_buf = &tr->array_buffer; 7655 int cpu = tracing_get_cpu(inode); 7656 struct trace_seq *s; 7657 unsigned long cnt; 7658 unsigned long long t; 7659 unsigned long usec_rem; 7660 7661 s = kmalloc(sizeof(*s), GFP_KERNEL); 7662 if (!s) 7663 return -ENOMEM; 7664 7665 trace_seq_init(s); 7666 7667 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu); 7668 trace_seq_printf(s, "entries: %ld\n", cnt); 7669 7670 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu); 7671 trace_seq_printf(s, "overrun: %ld\n", cnt); 7672 7673 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu); 7674 trace_seq_printf(s, "commit overrun: %ld\n", cnt); 7675 7676 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu); 7677 trace_seq_printf(s, "bytes: %ld\n", cnt); 7678 7679 if (trace_clocks[tr->clock_id].in_ns) { 7680 /* local or global for trace_clock */ 7681 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); 7682 usec_rem = do_div(t, USEC_PER_SEC); 7683 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", 7684 t, usec_rem); 7685 7686 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu)); 7687 usec_rem = do_div(t, USEC_PER_SEC); 7688 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem); 7689 } else { 7690 /* counter or tsc mode for trace_clock */ 7691 trace_seq_printf(s, "oldest event ts: %llu\n", 7692 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); 7693 7694 trace_seq_printf(s, "now ts: %llu\n", 7695 ring_buffer_time_stamp(trace_buf->buffer, cpu)); 7696 } 7697 7698 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu); 7699 trace_seq_printf(s, "dropped events: %ld\n", cnt); 7700 7701 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu); 7702 trace_seq_printf(s, "read events: %ld\n", cnt); 7703 7704 count = simple_read_from_buffer(ubuf, count, ppos, 7705 s->buffer, trace_seq_used(s)); 7706 7707 kfree(s); 7708 7709 return count; 7710 } 7711 7712 static const struct file_operations tracing_stats_fops = { 7713 .open = tracing_open_generic_tr, 7714 .read = tracing_stats_read, 7715 .llseek = generic_file_llseek, 7716 .release = tracing_release_generic_tr, 7717 }; 7718 7719 #ifdef CONFIG_DYNAMIC_FTRACE 7720 7721 static ssize_t 7722 tracing_read_dyn_info(struct file *filp, char __user *ubuf, 7723 size_t cnt, loff_t *ppos) 7724 { 7725 ssize_t ret; 7726 char *buf; 7727 int r; 7728 7729 /* 256 should be plenty to hold the amount needed */ 7730 buf = kmalloc(256, GFP_KERNEL); 7731 if (!buf) 7732 return -ENOMEM; 7733 7734 r = scnprintf(buf, 256, "%ld pages:%ld groups: %ld\n", 7735 ftrace_update_tot_cnt, 7736 ftrace_number_of_pages, 7737 ftrace_number_of_groups); 7738 7739 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 7740 kfree(buf); 7741 return ret; 7742 } 7743 7744 static const struct file_operations tracing_dyn_info_fops = { 7745 .open = tracing_open_generic, 7746 .read = tracing_read_dyn_info, 7747 .llseek = generic_file_llseek, 7748 }; 7749 #endif /* CONFIG_DYNAMIC_FTRACE */ 7750 7751 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) 7752 static void 7753 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, 7754 struct trace_array *tr, struct ftrace_probe_ops *ops, 7755 void *data) 7756 { 7757 tracing_snapshot_instance(tr); 7758 } 7759 7760 static void 7761 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, 7762 struct trace_array *tr, struct ftrace_probe_ops *ops, 7763 void *data) 7764 { 7765 struct ftrace_func_mapper *mapper = data; 7766 long *count = NULL; 7767 7768 if (mapper) 7769 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 7770 7771 if (count) { 7772 7773 if (*count <= 0) 7774 return; 7775 7776 (*count)--; 7777 } 7778 7779 tracing_snapshot_instance(tr); 7780 } 7781 7782 static int 7783 ftrace_snapshot_print(struct seq_file *m, unsigned long ip, 7784 struct ftrace_probe_ops *ops, void *data) 7785 { 7786 struct ftrace_func_mapper *mapper = data; 7787 long *count = NULL; 7788 7789 seq_printf(m, "%ps:", (void *)ip); 7790 7791 seq_puts(m, "snapshot"); 7792 7793 if (mapper) 7794 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 7795 7796 if (count) 7797 seq_printf(m, ":count=%ld\n", *count); 7798 else 7799 seq_puts(m, ":unlimited\n"); 7800 7801 return 0; 7802 } 7803 7804 static int 7805 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr, 7806 unsigned long ip, void *init_data, void **data) 7807 { 7808 struct ftrace_func_mapper *mapper = *data; 7809 7810 if (!mapper) { 7811 mapper = allocate_ftrace_func_mapper(); 7812 if (!mapper) 7813 return -ENOMEM; 7814 *data = mapper; 7815 } 7816 7817 return ftrace_func_mapper_add_ip(mapper, ip, init_data); 7818 } 7819 7820 static void 7821 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr, 7822 unsigned long ip, void *data) 7823 { 7824 struct ftrace_func_mapper *mapper = data; 7825 7826 if (!ip) { 7827 if (!mapper) 7828 return; 7829 free_ftrace_func_mapper(mapper, NULL); 7830 return; 7831 } 7832 7833 ftrace_func_mapper_remove_ip(mapper, ip); 7834 } 7835 7836 static struct ftrace_probe_ops snapshot_probe_ops = { 7837 .func = ftrace_snapshot, 7838 .print = ftrace_snapshot_print, 7839 }; 7840 7841 static struct ftrace_probe_ops snapshot_count_probe_ops = { 7842 .func = ftrace_count_snapshot, 7843 .print = ftrace_snapshot_print, 7844 .init = ftrace_snapshot_init, 7845 .free = ftrace_snapshot_free, 7846 }; 7847 7848 static int 7849 ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash, 7850 char *glob, char *cmd, char *param, int enable) 7851 { 7852 struct ftrace_probe_ops *ops; 7853 void *count = (void *)-1; 7854 char *number; 7855 int ret; 7856 7857 if (!tr) 7858 return -ENODEV; 7859 7860 /* hash funcs only work with set_ftrace_filter */ 7861 if (!enable) 7862 return -EINVAL; 7863 7864 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops; 7865 7866 if (glob[0] == '!') 7867 return unregister_ftrace_function_probe_func(glob+1, tr, ops); 7868 7869 if (!param) 7870 goto out_reg; 7871 7872 number = strsep(¶m, ":"); 7873 7874 if (!strlen(number)) 7875 goto out_reg; 7876 7877 /* 7878 * We use the callback data field (which is a pointer) 7879 * as our counter. 7880 */ 7881 ret = kstrtoul(number, 0, (unsigned long *)&count); 7882 if (ret) 7883 return ret; 7884 7885 out_reg: 7886 ret = tracing_alloc_snapshot_instance(tr); 7887 if (ret < 0) 7888 goto out; 7889 7890 ret = register_ftrace_function_probe(glob, tr, ops, count); 7891 7892 out: 7893 return ret < 0 ? ret : 0; 7894 } 7895 7896 static struct ftrace_func_command ftrace_snapshot_cmd = { 7897 .name = "snapshot", 7898 .func = ftrace_trace_snapshot_callback, 7899 }; 7900 7901 static __init int register_snapshot_cmd(void) 7902 { 7903 return register_ftrace_command(&ftrace_snapshot_cmd); 7904 } 7905 #else 7906 static inline __init int register_snapshot_cmd(void) { return 0; } 7907 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */ 7908 7909 static struct dentry *tracing_get_dentry(struct trace_array *tr) 7910 { 7911 if (WARN_ON(!tr->dir)) 7912 return ERR_PTR(-ENODEV); 7913 7914 /* Top directory uses NULL as the parent */ 7915 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 7916 return NULL; 7917 7918 /* All sub buffers have a descriptor */ 7919 return tr->dir; 7920 } 7921 7922 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu) 7923 { 7924 struct dentry *d_tracer; 7925 7926 if (tr->percpu_dir) 7927 return tr->percpu_dir; 7928 7929 d_tracer = tracing_get_dentry(tr); 7930 if (IS_ERR(d_tracer)) 7931 return NULL; 7932 7933 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer); 7934 7935 MEM_FAIL(!tr->percpu_dir, 7936 "Could not create tracefs directory 'per_cpu/%d'\n", cpu); 7937 7938 return tr->percpu_dir; 7939 } 7940 7941 static struct dentry * 7942 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent, 7943 void *data, long cpu, const struct file_operations *fops) 7944 { 7945 struct dentry *ret = trace_create_file(name, mode, parent, data, fops); 7946 7947 if (ret) /* See tracing_get_cpu() */ 7948 d_inode(ret)->i_cdev = (void *)(cpu + 1); 7949 return ret; 7950 } 7951 7952 static void 7953 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu) 7954 { 7955 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu); 7956 struct dentry *d_cpu; 7957 char cpu_dir[30]; /* 30 characters should be more than enough */ 7958 7959 if (!d_percpu) 7960 return; 7961 7962 snprintf(cpu_dir, 30, "cpu%ld", cpu); 7963 d_cpu = tracefs_create_dir(cpu_dir, d_percpu); 7964 if (!d_cpu) { 7965 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir); 7966 return; 7967 } 7968 7969 /* per cpu trace_pipe */ 7970 trace_create_cpu_file("trace_pipe", 0444, d_cpu, 7971 tr, cpu, &tracing_pipe_fops); 7972 7973 /* per cpu trace */ 7974 trace_create_cpu_file("trace", 0644, d_cpu, 7975 tr, cpu, &tracing_fops); 7976 7977 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu, 7978 tr, cpu, &tracing_buffers_fops); 7979 7980 trace_create_cpu_file("stats", 0444, d_cpu, 7981 tr, cpu, &tracing_stats_fops); 7982 7983 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu, 7984 tr, cpu, &tracing_entries_fops); 7985 7986 #ifdef CONFIG_TRACER_SNAPSHOT 7987 trace_create_cpu_file("snapshot", 0644, d_cpu, 7988 tr, cpu, &snapshot_fops); 7989 7990 trace_create_cpu_file("snapshot_raw", 0444, d_cpu, 7991 tr, cpu, &snapshot_raw_fops); 7992 #endif 7993 } 7994 7995 #ifdef CONFIG_FTRACE_SELFTEST 7996 /* Let selftest have access to static functions in this file */ 7997 #include "trace_selftest.c" 7998 #endif 7999 8000 static ssize_t 8001 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt, 8002 loff_t *ppos) 8003 { 8004 struct trace_option_dentry *topt = filp->private_data; 8005 char *buf; 8006 8007 if (topt->flags->val & topt->opt->bit) 8008 buf = "1\n"; 8009 else 8010 buf = "0\n"; 8011 8012 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 8013 } 8014 8015 static ssize_t 8016 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt, 8017 loff_t *ppos) 8018 { 8019 struct trace_option_dentry *topt = filp->private_data; 8020 unsigned long val; 8021 int ret; 8022 8023 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 8024 if (ret) 8025 return ret; 8026 8027 if (val != 0 && val != 1) 8028 return -EINVAL; 8029 8030 if (!!(topt->flags->val & topt->opt->bit) != val) { 8031 mutex_lock(&trace_types_lock); 8032 ret = __set_tracer_option(topt->tr, topt->flags, 8033 topt->opt, !val); 8034 mutex_unlock(&trace_types_lock); 8035 if (ret) 8036 return ret; 8037 } 8038 8039 *ppos += cnt; 8040 8041 return cnt; 8042 } 8043 8044 8045 static const struct file_operations trace_options_fops = { 8046 .open = tracing_open_generic, 8047 .read = trace_options_read, 8048 .write = trace_options_write, 8049 .llseek = generic_file_llseek, 8050 }; 8051 8052 /* 8053 * In order to pass in both the trace_array descriptor as well as the index 8054 * to the flag that the trace option file represents, the trace_array 8055 * has a character array of trace_flags_index[], which holds the index 8056 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc. 8057 * The address of this character array is passed to the flag option file 8058 * read/write callbacks. 8059 * 8060 * In order to extract both the index and the trace_array descriptor, 8061 * get_tr_index() uses the following algorithm. 8062 * 8063 * idx = *ptr; 8064 * 8065 * As the pointer itself contains the address of the index (remember 8066 * index[1] == 1). 8067 * 8068 * Then to get the trace_array descriptor, by subtracting that index 8069 * from the ptr, we get to the start of the index itself. 8070 * 8071 * ptr - idx == &index[0] 8072 * 8073 * Then a simple container_of() from that pointer gets us to the 8074 * trace_array descriptor. 8075 */ 8076 static void get_tr_index(void *data, struct trace_array **ptr, 8077 unsigned int *pindex) 8078 { 8079 *pindex = *(unsigned char *)data; 8080 8081 *ptr = container_of(data - *pindex, struct trace_array, 8082 trace_flags_index); 8083 } 8084 8085 static ssize_t 8086 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt, 8087 loff_t *ppos) 8088 { 8089 void *tr_index = filp->private_data; 8090 struct trace_array *tr; 8091 unsigned int index; 8092 char *buf; 8093 8094 get_tr_index(tr_index, &tr, &index); 8095 8096 if (tr->trace_flags & (1 << index)) 8097 buf = "1\n"; 8098 else 8099 buf = "0\n"; 8100 8101 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 8102 } 8103 8104 static ssize_t 8105 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt, 8106 loff_t *ppos) 8107 { 8108 void *tr_index = filp->private_data; 8109 struct trace_array *tr; 8110 unsigned int index; 8111 unsigned long val; 8112 int ret; 8113 8114 get_tr_index(tr_index, &tr, &index); 8115 8116 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 8117 if (ret) 8118 return ret; 8119 8120 if (val != 0 && val != 1) 8121 return -EINVAL; 8122 8123 mutex_lock(&event_mutex); 8124 mutex_lock(&trace_types_lock); 8125 ret = set_tracer_flag(tr, 1 << index, val); 8126 mutex_unlock(&trace_types_lock); 8127 mutex_unlock(&event_mutex); 8128 8129 if (ret < 0) 8130 return ret; 8131 8132 *ppos += cnt; 8133 8134 return cnt; 8135 } 8136 8137 static const struct file_operations trace_options_core_fops = { 8138 .open = tracing_open_generic, 8139 .read = trace_options_core_read, 8140 .write = trace_options_core_write, 8141 .llseek = generic_file_llseek, 8142 }; 8143 8144 struct dentry *trace_create_file(const char *name, 8145 umode_t mode, 8146 struct dentry *parent, 8147 void *data, 8148 const struct file_operations *fops) 8149 { 8150 struct dentry *ret; 8151 8152 ret = tracefs_create_file(name, mode, parent, data, fops); 8153 if (!ret) 8154 pr_warn("Could not create tracefs '%s' entry\n", name); 8155 8156 return ret; 8157 } 8158 8159 8160 static struct dentry *trace_options_init_dentry(struct trace_array *tr) 8161 { 8162 struct dentry *d_tracer; 8163 8164 if (tr->options) 8165 return tr->options; 8166 8167 d_tracer = tracing_get_dentry(tr); 8168 if (IS_ERR(d_tracer)) 8169 return NULL; 8170 8171 tr->options = tracefs_create_dir("options", d_tracer); 8172 if (!tr->options) { 8173 pr_warn("Could not create tracefs directory 'options'\n"); 8174 return NULL; 8175 } 8176 8177 return tr->options; 8178 } 8179 8180 static void 8181 create_trace_option_file(struct trace_array *tr, 8182 struct trace_option_dentry *topt, 8183 struct tracer_flags *flags, 8184 struct tracer_opt *opt) 8185 { 8186 struct dentry *t_options; 8187 8188 t_options = trace_options_init_dentry(tr); 8189 if (!t_options) 8190 return; 8191 8192 topt->flags = flags; 8193 topt->opt = opt; 8194 topt->tr = tr; 8195 8196 topt->entry = trace_create_file(opt->name, 0644, t_options, topt, 8197 &trace_options_fops); 8198 8199 } 8200 8201 static void 8202 create_trace_option_files(struct trace_array *tr, struct tracer *tracer) 8203 { 8204 struct trace_option_dentry *topts; 8205 struct trace_options *tr_topts; 8206 struct tracer_flags *flags; 8207 struct tracer_opt *opts; 8208 int cnt; 8209 int i; 8210 8211 if (!tracer) 8212 return; 8213 8214 flags = tracer->flags; 8215 8216 if (!flags || !flags->opts) 8217 return; 8218 8219 /* 8220 * If this is an instance, only create flags for tracers 8221 * the instance may have. 8222 */ 8223 if (!trace_ok_for_array(tracer, tr)) 8224 return; 8225 8226 for (i = 0; i < tr->nr_topts; i++) { 8227 /* Make sure there's no duplicate flags. */ 8228 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags)) 8229 return; 8230 } 8231 8232 opts = flags->opts; 8233 8234 for (cnt = 0; opts[cnt].name; cnt++) 8235 ; 8236 8237 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL); 8238 if (!topts) 8239 return; 8240 8241 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1), 8242 GFP_KERNEL); 8243 if (!tr_topts) { 8244 kfree(topts); 8245 return; 8246 } 8247 8248 tr->topts = tr_topts; 8249 tr->topts[tr->nr_topts].tracer = tracer; 8250 tr->topts[tr->nr_topts].topts = topts; 8251 tr->nr_topts++; 8252 8253 for (cnt = 0; opts[cnt].name; cnt++) { 8254 create_trace_option_file(tr, &topts[cnt], flags, 8255 &opts[cnt]); 8256 MEM_FAIL(topts[cnt].entry == NULL, 8257 "Failed to create trace option: %s", 8258 opts[cnt].name); 8259 } 8260 } 8261 8262 static struct dentry * 8263 create_trace_option_core_file(struct trace_array *tr, 8264 const char *option, long index) 8265 { 8266 struct dentry *t_options; 8267 8268 t_options = trace_options_init_dentry(tr); 8269 if (!t_options) 8270 return NULL; 8271 8272 return trace_create_file(option, 0644, t_options, 8273 (void *)&tr->trace_flags_index[index], 8274 &trace_options_core_fops); 8275 } 8276 8277 static void create_trace_options_dir(struct trace_array *tr) 8278 { 8279 struct dentry *t_options; 8280 bool top_level = tr == &global_trace; 8281 int i; 8282 8283 t_options = trace_options_init_dentry(tr); 8284 if (!t_options) 8285 return; 8286 8287 for (i = 0; trace_options[i]; i++) { 8288 if (top_level || 8289 !((1 << i) & TOP_LEVEL_TRACE_FLAGS)) 8290 create_trace_option_core_file(tr, trace_options[i], i); 8291 } 8292 } 8293 8294 static ssize_t 8295 rb_simple_read(struct file *filp, char __user *ubuf, 8296 size_t cnt, loff_t *ppos) 8297 { 8298 struct trace_array *tr = filp->private_data; 8299 char buf[64]; 8300 int r; 8301 8302 r = tracer_tracing_is_on(tr); 8303 r = sprintf(buf, "%d\n", r); 8304 8305 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 8306 } 8307 8308 static ssize_t 8309 rb_simple_write(struct file *filp, const char __user *ubuf, 8310 size_t cnt, loff_t *ppos) 8311 { 8312 struct trace_array *tr = filp->private_data; 8313 struct trace_buffer *buffer = tr->array_buffer.buffer; 8314 unsigned long val; 8315 int ret; 8316 8317 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 8318 if (ret) 8319 return ret; 8320 8321 if (buffer) { 8322 mutex_lock(&trace_types_lock); 8323 if (!!val == tracer_tracing_is_on(tr)) { 8324 val = 0; /* do nothing */ 8325 } else if (val) { 8326 tracer_tracing_on(tr); 8327 if (tr->current_trace->start) 8328 tr->current_trace->start(tr); 8329 } else { 8330 tracer_tracing_off(tr); 8331 if (tr->current_trace->stop) 8332 tr->current_trace->stop(tr); 8333 } 8334 mutex_unlock(&trace_types_lock); 8335 } 8336 8337 (*ppos)++; 8338 8339 return cnt; 8340 } 8341 8342 static const struct file_operations rb_simple_fops = { 8343 .open = tracing_open_generic_tr, 8344 .read = rb_simple_read, 8345 .write = rb_simple_write, 8346 .release = tracing_release_generic_tr, 8347 .llseek = default_llseek, 8348 }; 8349 8350 static ssize_t 8351 buffer_percent_read(struct file *filp, char __user *ubuf, 8352 size_t cnt, loff_t *ppos) 8353 { 8354 struct trace_array *tr = filp->private_data; 8355 char buf[64]; 8356 int r; 8357 8358 r = tr->buffer_percent; 8359 r = sprintf(buf, "%d\n", r); 8360 8361 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 8362 } 8363 8364 static ssize_t 8365 buffer_percent_write(struct file *filp, const char __user *ubuf, 8366 size_t cnt, loff_t *ppos) 8367 { 8368 struct trace_array *tr = filp->private_data; 8369 unsigned long val; 8370 int ret; 8371 8372 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 8373 if (ret) 8374 return ret; 8375 8376 if (val > 100) 8377 return -EINVAL; 8378 8379 if (!val) 8380 val = 1; 8381 8382 tr->buffer_percent = val; 8383 8384 (*ppos)++; 8385 8386 return cnt; 8387 } 8388 8389 static const struct file_operations buffer_percent_fops = { 8390 .open = tracing_open_generic_tr, 8391 .read = buffer_percent_read, 8392 .write = buffer_percent_write, 8393 .release = tracing_release_generic_tr, 8394 .llseek = default_llseek, 8395 }; 8396 8397 static struct dentry *trace_instance_dir; 8398 8399 static void 8400 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer); 8401 8402 static int 8403 allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size) 8404 { 8405 enum ring_buffer_flags rb_flags; 8406 8407 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0; 8408 8409 buf->tr = tr; 8410 8411 buf->buffer = ring_buffer_alloc(size, rb_flags); 8412 if (!buf->buffer) 8413 return -ENOMEM; 8414 8415 buf->data = alloc_percpu(struct trace_array_cpu); 8416 if (!buf->data) { 8417 ring_buffer_free(buf->buffer); 8418 buf->buffer = NULL; 8419 return -ENOMEM; 8420 } 8421 8422 /* Allocate the first page for all buffers */ 8423 set_buffer_entries(&tr->array_buffer, 8424 ring_buffer_size(tr->array_buffer.buffer, 0)); 8425 8426 return 0; 8427 } 8428 8429 static int allocate_trace_buffers(struct trace_array *tr, int size) 8430 { 8431 int ret; 8432 8433 ret = allocate_trace_buffer(tr, &tr->array_buffer, size); 8434 if (ret) 8435 return ret; 8436 8437 #ifdef CONFIG_TRACER_MAX_TRACE 8438 ret = allocate_trace_buffer(tr, &tr->max_buffer, 8439 allocate_snapshot ? size : 1); 8440 if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) { 8441 ring_buffer_free(tr->array_buffer.buffer); 8442 tr->array_buffer.buffer = NULL; 8443 free_percpu(tr->array_buffer.data); 8444 tr->array_buffer.data = NULL; 8445 return -ENOMEM; 8446 } 8447 tr->allocated_snapshot = allocate_snapshot; 8448 8449 /* 8450 * Only the top level trace array gets its snapshot allocated 8451 * from the kernel command line. 8452 */ 8453 allocate_snapshot = false; 8454 #endif 8455 return 0; 8456 } 8457 8458 static void free_trace_buffer(struct array_buffer *buf) 8459 { 8460 if (buf->buffer) { 8461 ring_buffer_free(buf->buffer); 8462 buf->buffer = NULL; 8463 free_percpu(buf->data); 8464 buf->data = NULL; 8465 } 8466 } 8467 8468 static void free_trace_buffers(struct trace_array *tr) 8469 { 8470 if (!tr) 8471 return; 8472 8473 free_trace_buffer(&tr->array_buffer); 8474 8475 #ifdef CONFIG_TRACER_MAX_TRACE 8476 free_trace_buffer(&tr->max_buffer); 8477 #endif 8478 } 8479 8480 static void init_trace_flags_index(struct trace_array *tr) 8481 { 8482 int i; 8483 8484 /* Used by the trace options files */ 8485 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) 8486 tr->trace_flags_index[i] = i; 8487 } 8488 8489 static void __update_tracer_options(struct trace_array *tr) 8490 { 8491 struct tracer *t; 8492 8493 for (t = trace_types; t; t = t->next) 8494 add_tracer_options(tr, t); 8495 } 8496 8497 static void update_tracer_options(struct trace_array *tr) 8498 { 8499 mutex_lock(&trace_types_lock); 8500 __update_tracer_options(tr); 8501 mutex_unlock(&trace_types_lock); 8502 } 8503 8504 /* Must have trace_types_lock held */ 8505 struct trace_array *trace_array_find(const char *instance) 8506 { 8507 struct trace_array *tr, *found = NULL; 8508 8509 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 8510 if (tr->name && strcmp(tr->name, instance) == 0) { 8511 found = tr; 8512 break; 8513 } 8514 } 8515 8516 return found; 8517 } 8518 8519 struct trace_array *trace_array_find_get(const char *instance) 8520 { 8521 struct trace_array *tr; 8522 8523 mutex_lock(&trace_types_lock); 8524 tr = trace_array_find(instance); 8525 if (tr) 8526 tr->ref++; 8527 mutex_unlock(&trace_types_lock); 8528 8529 return tr; 8530 } 8531 8532 static struct trace_array *trace_array_create(const char *name) 8533 { 8534 struct trace_array *tr; 8535 int ret; 8536 8537 ret = -ENOMEM; 8538 tr = kzalloc(sizeof(*tr), GFP_KERNEL); 8539 if (!tr) 8540 return ERR_PTR(ret); 8541 8542 tr->name = kstrdup(name, GFP_KERNEL); 8543 if (!tr->name) 8544 goto out_free_tr; 8545 8546 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL)) 8547 goto out_free_tr; 8548 8549 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS; 8550 8551 cpumask_copy(tr->tracing_cpumask, cpu_all_mask); 8552 8553 raw_spin_lock_init(&tr->start_lock); 8554 8555 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 8556 8557 tr->current_trace = &nop_trace; 8558 8559 INIT_LIST_HEAD(&tr->systems); 8560 INIT_LIST_HEAD(&tr->events); 8561 INIT_LIST_HEAD(&tr->hist_vars); 8562 INIT_LIST_HEAD(&tr->err_log); 8563 8564 if (allocate_trace_buffers(tr, trace_buf_size) < 0) 8565 goto out_free_tr; 8566 8567 tr->dir = tracefs_create_dir(name, trace_instance_dir); 8568 if (!tr->dir) 8569 goto out_free_tr; 8570 8571 ret = event_trace_add_tracer(tr->dir, tr); 8572 if (ret) { 8573 tracefs_remove(tr->dir); 8574 goto out_free_tr; 8575 } 8576 8577 ftrace_init_trace_array(tr); 8578 8579 init_tracer_tracefs(tr, tr->dir); 8580 init_trace_flags_index(tr); 8581 __update_tracer_options(tr); 8582 8583 list_add(&tr->list, &ftrace_trace_arrays); 8584 8585 tr->ref++; 8586 8587 8588 return tr; 8589 8590 out_free_tr: 8591 free_trace_buffers(tr); 8592 free_cpumask_var(tr->tracing_cpumask); 8593 kfree(tr->name); 8594 kfree(tr); 8595 8596 return ERR_PTR(ret); 8597 } 8598 8599 static int instance_mkdir(const char *name) 8600 { 8601 struct trace_array *tr; 8602 int ret; 8603 8604 mutex_lock(&event_mutex); 8605 mutex_lock(&trace_types_lock); 8606 8607 ret = -EEXIST; 8608 if (trace_array_find(name)) 8609 goto out_unlock; 8610 8611 tr = trace_array_create(name); 8612 8613 ret = PTR_ERR_OR_ZERO(tr); 8614 8615 out_unlock: 8616 mutex_unlock(&trace_types_lock); 8617 mutex_unlock(&event_mutex); 8618 return ret; 8619 } 8620 8621 /** 8622 * trace_array_get_by_name - Create/Lookup a trace array, given its name. 8623 * @name: The name of the trace array to be looked up/created. 8624 * 8625 * Returns pointer to trace array with given name. 8626 * NULL, if it cannot be created. 8627 * 8628 * NOTE: This function increments the reference counter associated with the 8629 * trace array returned. This makes sure it cannot be freed while in use. 8630 * Use trace_array_put() once the trace array is no longer needed. 8631 * If the trace_array is to be freed, trace_array_destroy() needs to 8632 * be called after the trace_array_put(), or simply let user space delete 8633 * it from the tracefs instances directory. But until the 8634 * trace_array_put() is called, user space can not delete it. 8635 * 8636 */ 8637 struct trace_array *trace_array_get_by_name(const char *name) 8638 { 8639 struct trace_array *tr; 8640 8641 mutex_lock(&event_mutex); 8642 mutex_lock(&trace_types_lock); 8643 8644 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 8645 if (tr->name && strcmp(tr->name, name) == 0) 8646 goto out_unlock; 8647 } 8648 8649 tr = trace_array_create(name); 8650 8651 if (IS_ERR(tr)) 8652 tr = NULL; 8653 out_unlock: 8654 if (tr) 8655 tr->ref++; 8656 8657 mutex_unlock(&trace_types_lock); 8658 mutex_unlock(&event_mutex); 8659 return tr; 8660 } 8661 EXPORT_SYMBOL_GPL(trace_array_get_by_name); 8662 8663 static int __remove_instance(struct trace_array *tr) 8664 { 8665 int i; 8666 8667 /* Reference counter for a newly created trace array = 1. */ 8668 if (tr->ref > 1 || (tr->current_trace && tr->current_trace->ref)) 8669 return -EBUSY; 8670 8671 list_del(&tr->list); 8672 8673 /* Disable all the flags that were enabled coming in */ 8674 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) { 8675 if ((1 << i) & ZEROED_TRACE_FLAGS) 8676 set_tracer_flag(tr, 1 << i, 0); 8677 } 8678 8679 tracing_set_nop(tr); 8680 clear_ftrace_function_probes(tr); 8681 event_trace_del_tracer(tr); 8682 ftrace_clear_pids(tr); 8683 ftrace_destroy_function_files(tr); 8684 tracefs_remove(tr->dir); 8685 free_trace_buffers(tr); 8686 8687 for (i = 0; i < tr->nr_topts; i++) { 8688 kfree(tr->topts[i].topts); 8689 } 8690 kfree(tr->topts); 8691 8692 free_cpumask_var(tr->tracing_cpumask); 8693 kfree(tr->name); 8694 kfree(tr); 8695 tr = NULL; 8696 8697 return 0; 8698 } 8699 8700 int trace_array_destroy(struct trace_array *this_tr) 8701 { 8702 struct trace_array *tr; 8703 int ret; 8704 8705 if (!this_tr) 8706 return -EINVAL; 8707 8708 mutex_lock(&event_mutex); 8709 mutex_lock(&trace_types_lock); 8710 8711 ret = -ENODEV; 8712 8713 /* Making sure trace array exists before destroying it. */ 8714 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 8715 if (tr == this_tr) { 8716 ret = __remove_instance(tr); 8717 break; 8718 } 8719 } 8720 8721 mutex_unlock(&trace_types_lock); 8722 mutex_unlock(&event_mutex); 8723 8724 return ret; 8725 } 8726 EXPORT_SYMBOL_GPL(trace_array_destroy); 8727 8728 static int instance_rmdir(const char *name) 8729 { 8730 struct trace_array *tr; 8731 int ret; 8732 8733 mutex_lock(&event_mutex); 8734 mutex_lock(&trace_types_lock); 8735 8736 ret = -ENODEV; 8737 tr = trace_array_find(name); 8738 if (tr) 8739 ret = __remove_instance(tr); 8740 8741 mutex_unlock(&trace_types_lock); 8742 mutex_unlock(&event_mutex); 8743 8744 return ret; 8745 } 8746 8747 static __init void create_trace_instances(struct dentry *d_tracer) 8748 { 8749 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer, 8750 instance_mkdir, 8751 instance_rmdir); 8752 if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n")) 8753 return; 8754 } 8755 8756 static void 8757 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) 8758 { 8759 struct trace_event_file *file; 8760 int cpu; 8761 8762 trace_create_file("available_tracers", 0444, d_tracer, 8763 tr, &show_traces_fops); 8764 8765 trace_create_file("current_tracer", 0644, d_tracer, 8766 tr, &set_tracer_fops); 8767 8768 trace_create_file("tracing_cpumask", 0644, d_tracer, 8769 tr, &tracing_cpumask_fops); 8770 8771 trace_create_file("trace_options", 0644, d_tracer, 8772 tr, &tracing_iter_fops); 8773 8774 trace_create_file("trace", 0644, d_tracer, 8775 tr, &tracing_fops); 8776 8777 trace_create_file("trace_pipe", 0444, d_tracer, 8778 tr, &tracing_pipe_fops); 8779 8780 trace_create_file("buffer_size_kb", 0644, d_tracer, 8781 tr, &tracing_entries_fops); 8782 8783 trace_create_file("buffer_total_size_kb", 0444, d_tracer, 8784 tr, &tracing_total_entries_fops); 8785 8786 trace_create_file("free_buffer", 0200, d_tracer, 8787 tr, &tracing_free_buffer_fops); 8788 8789 trace_create_file("trace_marker", 0220, d_tracer, 8790 tr, &tracing_mark_fops); 8791 8792 file = __find_event_file(tr, "ftrace", "print"); 8793 if (file && file->dir) 8794 trace_create_file("trigger", 0644, file->dir, file, 8795 &event_trigger_fops); 8796 tr->trace_marker_file = file; 8797 8798 trace_create_file("trace_marker_raw", 0220, d_tracer, 8799 tr, &tracing_mark_raw_fops); 8800 8801 trace_create_file("trace_clock", 0644, d_tracer, tr, 8802 &trace_clock_fops); 8803 8804 trace_create_file("tracing_on", 0644, d_tracer, 8805 tr, &rb_simple_fops); 8806 8807 trace_create_file("timestamp_mode", 0444, d_tracer, tr, 8808 &trace_time_stamp_mode_fops); 8809 8810 tr->buffer_percent = 50; 8811 8812 trace_create_file("buffer_percent", 0444, d_tracer, 8813 tr, &buffer_percent_fops); 8814 8815 create_trace_options_dir(tr); 8816 8817 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) 8818 trace_create_maxlat_file(tr, d_tracer); 8819 #endif 8820 8821 if (ftrace_create_function_files(tr, d_tracer)) 8822 MEM_FAIL(1, "Could not allocate function filter files"); 8823 8824 #ifdef CONFIG_TRACER_SNAPSHOT 8825 trace_create_file("snapshot", 0644, d_tracer, 8826 tr, &snapshot_fops); 8827 #endif 8828 8829 trace_create_file("error_log", 0644, d_tracer, 8830 tr, &tracing_err_log_fops); 8831 8832 for_each_tracing_cpu(cpu) 8833 tracing_init_tracefs_percpu(tr, cpu); 8834 8835 ftrace_init_tracefs(tr, d_tracer); 8836 } 8837 8838 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore) 8839 { 8840 struct vfsmount *mnt; 8841 struct file_system_type *type; 8842 8843 /* 8844 * To maintain backward compatibility for tools that mount 8845 * debugfs to get to the tracing facility, tracefs is automatically 8846 * mounted to the debugfs/tracing directory. 8847 */ 8848 type = get_fs_type("tracefs"); 8849 if (!type) 8850 return NULL; 8851 mnt = vfs_submount(mntpt, type, "tracefs", NULL); 8852 put_filesystem(type); 8853 if (IS_ERR(mnt)) 8854 return NULL; 8855 mntget(mnt); 8856 8857 return mnt; 8858 } 8859 8860 /** 8861 * tracing_init_dentry - initialize top level trace array 8862 * 8863 * This is called when creating files or directories in the tracing 8864 * directory. It is called via fs_initcall() by any of the boot up code 8865 * and expects to return the dentry of the top level tracing directory. 8866 */ 8867 struct dentry *tracing_init_dentry(void) 8868 { 8869 struct trace_array *tr = &global_trace; 8870 8871 if (security_locked_down(LOCKDOWN_TRACEFS)) { 8872 pr_warn("Tracing disabled due to lockdown\n"); 8873 return ERR_PTR(-EPERM); 8874 } 8875 8876 /* The top level trace array uses NULL as parent */ 8877 if (tr->dir) 8878 return NULL; 8879 8880 if (WARN_ON(!tracefs_initialized()) || 8881 (IS_ENABLED(CONFIG_DEBUG_FS) && 8882 WARN_ON(!debugfs_initialized()))) 8883 return ERR_PTR(-ENODEV); 8884 8885 /* 8886 * As there may still be users that expect the tracing 8887 * files to exist in debugfs/tracing, we must automount 8888 * the tracefs file system there, so older tools still 8889 * work with the newer kerenl. 8890 */ 8891 tr->dir = debugfs_create_automount("tracing", NULL, 8892 trace_automount, NULL); 8893 8894 return NULL; 8895 } 8896 8897 extern struct trace_eval_map *__start_ftrace_eval_maps[]; 8898 extern struct trace_eval_map *__stop_ftrace_eval_maps[]; 8899 8900 static void __init trace_eval_init(void) 8901 { 8902 int len; 8903 8904 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps; 8905 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len); 8906 } 8907 8908 #ifdef CONFIG_MODULES 8909 static void trace_module_add_evals(struct module *mod) 8910 { 8911 if (!mod->num_trace_evals) 8912 return; 8913 8914 /* 8915 * Modules with bad taint do not have events created, do 8916 * not bother with enums either. 8917 */ 8918 if (trace_module_has_bad_taint(mod)) 8919 return; 8920 8921 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals); 8922 } 8923 8924 #ifdef CONFIG_TRACE_EVAL_MAP_FILE 8925 static void trace_module_remove_evals(struct module *mod) 8926 { 8927 union trace_eval_map_item *map; 8928 union trace_eval_map_item **last = &trace_eval_maps; 8929 8930 if (!mod->num_trace_evals) 8931 return; 8932 8933 mutex_lock(&trace_eval_mutex); 8934 8935 map = trace_eval_maps; 8936 8937 while (map) { 8938 if (map->head.mod == mod) 8939 break; 8940 map = trace_eval_jmp_to_tail(map); 8941 last = &map->tail.next; 8942 map = map->tail.next; 8943 } 8944 if (!map) 8945 goto out; 8946 8947 *last = trace_eval_jmp_to_tail(map)->tail.next; 8948 kfree(map); 8949 out: 8950 mutex_unlock(&trace_eval_mutex); 8951 } 8952 #else 8953 static inline void trace_module_remove_evals(struct module *mod) { } 8954 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */ 8955 8956 static int trace_module_notify(struct notifier_block *self, 8957 unsigned long val, void *data) 8958 { 8959 struct module *mod = data; 8960 8961 switch (val) { 8962 case MODULE_STATE_COMING: 8963 trace_module_add_evals(mod); 8964 break; 8965 case MODULE_STATE_GOING: 8966 trace_module_remove_evals(mod); 8967 break; 8968 } 8969 8970 return 0; 8971 } 8972 8973 static struct notifier_block trace_module_nb = { 8974 .notifier_call = trace_module_notify, 8975 .priority = 0, 8976 }; 8977 #endif /* CONFIG_MODULES */ 8978 8979 static __init int tracer_init_tracefs(void) 8980 { 8981 struct dentry *d_tracer; 8982 8983 trace_access_lock_init(); 8984 8985 d_tracer = tracing_init_dentry(); 8986 if (IS_ERR(d_tracer)) 8987 return 0; 8988 8989 event_trace_init(); 8990 8991 init_tracer_tracefs(&global_trace, d_tracer); 8992 ftrace_init_tracefs_toplevel(&global_trace, d_tracer); 8993 8994 trace_create_file("tracing_thresh", 0644, d_tracer, 8995 &global_trace, &tracing_thresh_fops); 8996 8997 trace_create_file("README", 0444, d_tracer, 8998 NULL, &tracing_readme_fops); 8999 9000 trace_create_file("saved_cmdlines", 0444, d_tracer, 9001 NULL, &tracing_saved_cmdlines_fops); 9002 9003 trace_create_file("saved_cmdlines_size", 0644, d_tracer, 9004 NULL, &tracing_saved_cmdlines_size_fops); 9005 9006 trace_create_file("saved_tgids", 0444, d_tracer, 9007 NULL, &tracing_saved_tgids_fops); 9008 9009 trace_eval_init(); 9010 9011 trace_create_eval_file(d_tracer); 9012 9013 #ifdef CONFIG_MODULES 9014 register_module_notifier(&trace_module_nb); 9015 #endif 9016 9017 #ifdef CONFIG_DYNAMIC_FTRACE 9018 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer, 9019 NULL, &tracing_dyn_info_fops); 9020 #endif 9021 9022 create_trace_instances(d_tracer); 9023 9024 update_tracer_options(&global_trace); 9025 9026 return 0; 9027 } 9028 9029 static int trace_panic_handler(struct notifier_block *this, 9030 unsigned long event, void *unused) 9031 { 9032 if (ftrace_dump_on_oops) 9033 ftrace_dump(ftrace_dump_on_oops); 9034 return NOTIFY_OK; 9035 } 9036 9037 static struct notifier_block trace_panic_notifier = { 9038 .notifier_call = trace_panic_handler, 9039 .next = NULL, 9040 .priority = 150 /* priority: INT_MAX >= x >= 0 */ 9041 }; 9042 9043 static int trace_die_handler(struct notifier_block *self, 9044 unsigned long val, 9045 void *data) 9046 { 9047 switch (val) { 9048 case DIE_OOPS: 9049 if (ftrace_dump_on_oops) 9050 ftrace_dump(ftrace_dump_on_oops); 9051 break; 9052 default: 9053 break; 9054 } 9055 return NOTIFY_OK; 9056 } 9057 9058 static struct notifier_block trace_die_notifier = { 9059 .notifier_call = trace_die_handler, 9060 .priority = 200 9061 }; 9062 9063 /* 9064 * printk is set to max of 1024, we really don't need it that big. 9065 * Nothing should be printing 1000 characters anyway. 9066 */ 9067 #define TRACE_MAX_PRINT 1000 9068 9069 /* 9070 * Define here KERN_TRACE so that we have one place to modify 9071 * it if we decide to change what log level the ftrace dump 9072 * should be at. 9073 */ 9074 #define KERN_TRACE KERN_EMERG 9075 9076 void 9077 trace_printk_seq(struct trace_seq *s) 9078 { 9079 /* Probably should print a warning here. */ 9080 if (s->seq.len >= TRACE_MAX_PRINT) 9081 s->seq.len = TRACE_MAX_PRINT; 9082 9083 /* 9084 * More paranoid code. Although the buffer size is set to 9085 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just 9086 * an extra layer of protection. 9087 */ 9088 if (WARN_ON_ONCE(s->seq.len >= s->seq.size)) 9089 s->seq.len = s->seq.size - 1; 9090 9091 /* should be zero ended, but we are paranoid. */ 9092 s->buffer[s->seq.len] = 0; 9093 9094 printk(KERN_TRACE "%s", s->buffer); 9095 9096 trace_seq_init(s); 9097 } 9098 9099 void trace_init_global_iter(struct trace_iterator *iter) 9100 { 9101 iter->tr = &global_trace; 9102 iter->trace = iter->tr->current_trace; 9103 iter->cpu_file = RING_BUFFER_ALL_CPUS; 9104 iter->array_buffer = &global_trace.array_buffer; 9105 9106 if (iter->trace && iter->trace->open) 9107 iter->trace->open(iter); 9108 9109 /* Annotate start of buffers if we had overruns */ 9110 if (ring_buffer_overruns(iter->array_buffer->buffer)) 9111 iter->iter_flags |= TRACE_FILE_ANNOTATE; 9112 9113 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 9114 if (trace_clocks[iter->tr->clock_id].in_ns) 9115 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 9116 } 9117 9118 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) 9119 { 9120 /* use static because iter can be a bit big for the stack */ 9121 static struct trace_iterator iter; 9122 static atomic_t dump_running; 9123 struct trace_array *tr = &global_trace; 9124 unsigned int old_userobj; 9125 unsigned long flags; 9126 int cnt = 0, cpu; 9127 9128 /* Only allow one dump user at a time. */ 9129 if (atomic_inc_return(&dump_running) != 1) { 9130 atomic_dec(&dump_running); 9131 return; 9132 } 9133 9134 /* 9135 * Always turn off tracing when we dump. 9136 * We don't need to show trace output of what happens 9137 * between multiple crashes. 9138 * 9139 * If the user does a sysrq-z, then they can re-enable 9140 * tracing with echo 1 > tracing_on. 9141 */ 9142 tracing_off(); 9143 9144 local_irq_save(flags); 9145 printk_nmi_direct_enter(); 9146 9147 /* Simulate the iterator */ 9148 trace_init_global_iter(&iter); 9149 9150 for_each_tracing_cpu(cpu) { 9151 atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); 9152 } 9153 9154 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ; 9155 9156 /* don't look at user memory in panic mode */ 9157 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ; 9158 9159 switch (oops_dump_mode) { 9160 case DUMP_ALL: 9161 iter.cpu_file = RING_BUFFER_ALL_CPUS; 9162 break; 9163 case DUMP_ORIG: 9164 iter.cpu_file = raw_smp_processor_id(); 9165 break; 9166 case DUMP_NONE: 9167 goto out_enable; 9168 default: 9169 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n"); 9170 iter.cpu_file = RING_BUFFER_ALL_CPUS; 9171 } 9172 9173 printk(KERN_TRACE "Dumping ftrace buffer:\n"); 9174 9175 /* Did function tracer already get disabled? */ 9176 if (ftrace_is_dead()) { 9177 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n"); 9178 printk("# MAY BE MISSING FUNCTION EVENTS\n"); 9179 } 9180 9181 /* 9182 * We need to stop all tracing on all CPUS to read the 9183 * the next buffer. This is a bit expensive, but is 9184 * not done often. We fill all what we can read, 9185 * and then release the locks again. 9186 */ 9187 9188 while (!trace_empty(&iter)) { 9189 9190 if (!cnt) 9191 printk(KERN_TRACE "---------------------------------\n"); 9192 9193 cnt++; 9194 9195 trace_iterator_reset(&iter); 9196 iter.iter_flags |= TRACE_FILE_LAT_FMT; 9197 9198 if (trace_find_next_entry_inc(&iter) != NULL) { 9199 int ret; 9200 9201 ret = print_trace_line(&iter); 9202 if (ret != TRACE_TYPE_NO_CONSUME) 9203 trace_consume(&iter); 9204 } 9205 touch_nmi_watchdog(); 9206 9207 trace_printk_seq(&iter.seq); 9208 } 9209 9210 if (!cnt) 9211 printk(KERN_TRACE " (ftrace buffer empty)\n"); 9212 else 9213 printk(KERN_TRACE "---------------------------------\n"); 9214 9215 out_enable: 9216 tr->trace_flags |= old_userobj; 9217 9218 for_each_tracing_cpu(cpu) { 9219 atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); 9220 } 9221 atomic_dec(&dump_running); 9222 printk_nmi_direct_exit(); 9223 local_irq_restore(flags); 9224 } 9225 EXPORT_SYMBOL_GPL(ftrace_dump); 9226 9227 int trace_run_command(const char *buf, int (*createfn)(int, char **)) 9228 { 9229 char **argv; 9230 int argc, ret; 9231 9232 argc = 0; 9233 ret = 0; 9234 argv = argv_split(GFP_KERNEL, buf, &argc); 9235 if (!argv) 9236 return -ENOMEM; 9237 9238 if (argc) 9239 ret = createfn(argc, argv); 9240 9241 argv_free(argv); 9242 9243 return ret; 9244 } 9245 9246 #define WRITE_BUFSIZE 4096 9247 9248 ssize_t trace_parse_run_command(struct file *file, const char __user *buffer, 9249 size_t count, loff_t *ppos, 9250 int (*createfn)(int, char **)) 9251 { 9252 char *kbuf, *buf, *tmp; 9253 int ret = 0; 9254 size_t done = 0; 9255 size_t size; 9256 9257 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL); 9258 if (!kbuf) 9259 return -ENOMEM; 9260 9261 while (done < count) { 9262 size = count - done; 9263 9264 if (size >= WRITE_BUFSIZE) 9265 size = WRITE_BUFSIZE - 1; 9266 9267 if (copy_from_user(kbuf, buffer + done, size)) { 9268 ret = -EFAULT; 9269 goto out; 9270 } 9271 kbuf[size] = '\0'; 9272 buf = kbuf; 9273 do { 9274 tmp = strchr(buf, '\n'); 9275 if (tmp) { 9276 *tmp = '\0'; 9277 size = tmp - buf + 1; 9278 } else { 9279 size = strlen(buf); 9280 if (done + size < count) { 9281 if (buf != kbuf) 9282 break; 9283 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */ 9284 pr_warn("Line length is too long: Should be less than %d\n", 9285 WRITE_BUFSIZE - 2); 9286 ret = -EINVAL; 9287 goto out; 9288 } 9289 } 9290 done += size; 9291 9292 /* Remove comments */ 9293 tmp = strchr(buf, '#'); 9294 9295 if (tmp) 9296 *tmp = '\0'; 9297 9298 ret = trace_run_command(buf, createfn); 9299 if (ret) 9300 goto out; 9301 buf += size; 9302 9303 } while (done < count); 9304 } 9305 ret = done; 9306 9307 out: 9308 kfree(kbuf); 9309 9310 return ret; 9311 } 9312 9313 __init static int tracer_alloc_buffers(void) 9314 { 9315 int ring_buf_size; 9316 int ret = -ENOMEM; 9317 9318 9319 if (security_locked_down(LOCKDOWN_TRACEFS)) { 9320 pr_warn("Tracing disabled due to lockdown\n"); 9321 return -EPERM; 9322 } 9323 9324 /* 9325 * Make sure we don't accidently add more trace options 9326 * than we have bits for. 9327 */ 9328 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE); 9329 9330 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL)) 9331 goto out; 9332 9333 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL)) 9334 goto out_free_buffer_mask; 9335 9336 /* Only allocate trace_printk buffers if a trace_printk exists */ 9337 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt) 9338 /* Must be called before global_trace.buffer is allocated */ 9339 trace_printk_init_buffers(); 9340 9341 /* To save memory, keep the ring buffer size to its minimum */ 9342 if (ring_buffer_expanded) 9343 ring_buf_size = trace_buf_size; 9344 else 9345 ring_buf_size = 1; 9346 9347 cpumask_copy(tracing_buffer_mask, cpu_possible_mask); 9348 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask); 9349 9350 raw_spin_lock_init(&global_trace.start_lock); 9351 9352 /* 9353 * The prepare callbacks allocates some memory for the ring buffer. We 9354 * don't free the buffer if the if the CPU goes down. If we were to free 9355 * the buffer, then the user would lose any trace that was in the 9356 * buffer. The memory will be removed once the "instance" is removed. 9357 */ 9358 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE, 9359 "trace/RB:preapre", trace_rb_cpu_prepare, 9360 NULL); 9361 if (ret < 0) 9362 goto out_free_cpumask; 9363 /* Used for event triggers */ 9364 ret = -ENOMEM; 9365 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE); 9366 if (!temp_buffer) 9367 goto out_rm_hp_state; 9368 9369 if (trace_create_savedcmd() < 0) 9370 goto out_free_temp_buffer; 9371 9372 /* TODO: make the number of buffers hot pluggable with CPUS */ 9373 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) { 9374 MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n"); 9375 goto out_free_savedcmd; 9376 } 9377 9378 if (global_trace.buffer_disabled) 9379 tracing_off(); 9380 9381 if (trace_boot_clock) { 9382 ret = tracing_set_clock(&global_trace, trace_boot_clock); 9383 if (ret < 0) 9384 pr_warn("Trace clock %s not defined, going back to default\n", 9385 trace_boot_clock); 9386 } 9387 9388 /* 9389 * register_tracer() might reference current_trace, so it 9390 * needs to be set before we register anything. This is 9391 * just a bootstrap of current_trace anyway. 9392 */ 9393 global_trace.current_trace = &nop_trace; 9394 9395 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 9396 9397 ftrace_init_global_array_ops(&global_trace); 9398 9399 init_trace_flags_index(&global_trace); 9400 9401 register_tracer(&nop_trace); 9402 9403 /* Function tracing may start here (via kernel command line) */ 9404 init_function_trace(); 9405 9406 /* All seems OK, enable tracing */ 9407 tracing_disabled = 0; 9408 9409 atomic_notifier_chain_register(&panic_notifier_list, 9410 &trace_panic_notifier); 9411 9412 register_die_notifier(&trace_die_notifier); 9413 9414 global_trace.flags = TRACE_ARRAY_FL_GLOBAL; 9415 9416 INIT_LIST_HEAD(&global_trace.systems); 9417 INIT_LIST_HEAD(&global_trace.events); 9418 INIT_LIST_HEAD(&global_trace.hist_vars); 9419 INIT_LIST_HEAD(&global_trace.err_log); 9420 list_add(&global_trace.list, &ftrace_trace_arrays); 9421 9422 apply_trace_boot_options(); 9423 9424 register_snapshot_cmd(); 9425 9426 return 0; 9427 9428 out_free_savedcmd: 9429 free_saved_cmdlines_buffer(savedcmd); 9430 out_free_temp_buffer: 9431 ring_buffer_free(temp_buffer); 9432 out_rm_hp_state: 9433 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE); 9434 out_free_cpumask: 9435 free_cpumask_var(global_trace.tracing_cpumask); 9436 out_free_buffer_mask: 9437 free_cpumask_var(tracing_buffer_mask); 9438 out: 9439 return ret; 9440 } 9441 9442 void __init early_trace_init(void) 9443 { 9444 if (tracepoint_printk) { 9445 tracepoint_print_iter = 9446 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL); 9447 if (MEM_FAIL(!tracepoint_print_iter, 9448 "Failed to allocate trace iterator\n")) 9449 tracepoint_printk = 0; 9450 else 9451 static_key_enable(&tracepoint_printk_key.key); 9452 } 9453 tracer_alloc_buffers(); 9454 } 9455 9456 void __init trace_init(void) 9457 { 9458 trace_event_init(); 9459 } 9460 9461 __init static int clear_boot_tracer(void) 9462 { 9463 /* 9464 * The default tracer at boot buffer is an init section. 9465 * This function is called in lateinit. If we did not 9466 * find the boot tracer, then clear it out, to prevent 9467 * later registration from accessing the buffer that is 9468 * about to be freed. 9469 */ 9470 if (!default_bootup_tracer) 9471 return 0; 9472 9473 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n", 9474 default_bootup_tracer); 9475 default_bootup_tracer = NULL; 9476 9477 return 0; 9478 } 9479 9480 fs_initcall(tracer_init_tracefs); 9481 late_initcall_sync(clear_boot_tracer); 9482 9483 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 9484 __init static int tracing_set_default_clock(void) 9485 { 9486 /* sched_clock_stable() is determined in late_initcall */ 9487 if (!trace_boot_clock && !sched_clock_stable()) { 9488 if (security_locked_down(LOCKDOWN_TRACEFS)) { 9489 pr_warn("Can not set tracing clock due to lockdown\n"); 9490 return -EPERM; 9491 } 9492 9493 printk(KERN_WARNING 9494 "Unstable clock detected, switching default tracing clock to \"global\"\n" 9495 "If you want to keep using the local clock, then add:\n" 9496 " \"trace_clock=local\"\n" 9497 "on the kernel command line\n"); 9498 tracing_set_clock(&global_trace, "global"); 9499 } 9500 9501 return 0; 9502 } 9503 late_initcall_sync(tracing_set_default_clock); 9504 #endif 9505