1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * ring buffer based function tracer 4 * 5 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com> 6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 7 * 8 * Originally taken from the RT patch by: 9 * Arnaldo Carvalho de Melo <acme@redhat.com> 10 * 11 * Based on code from the latency_tracer, that is: 12 * Copyright (C) 2004-2006 Ingo Molnar 13 * Copyright (C) 2004 Nadia Yvette Chambers 14 */ 15 #include <linux/ring_buffer.h> 16 #include <generated/utsrelease.h> 17 #include <linux/stacktrace.h> 18 #include <linux/writeback.h> 19 #include <linux/kallsyms.h> 20 #include <linux/seq_file.h> 21 #include <linux/notifier.h> 22 #include <linux/irqflags.h> 23 #include <linux/debugfs.h> 24 #include <linux/tracefs.h> 25 #include <linux/pagemap.h> 26 #include <linux/hardirq.h> 27 #include <linux/linkage.h> 28 #include <linux/uaccess.h> 29 #include <linux/vmalloc.h> 30 #include <linux/ftrace.h> 31 #include <linux/module.h> 32 #include <linux/percpu.h> 33 #include <linux/splice.h> 34 #include <linux/kdebug.h> 35 #include <linux/string.h> 36 #include <linux/mount.h> 37 #include <linux/rwsem.h> 38 #include <linux/slab.h> 39 #include <linux/ctype.h> 40 #include <linux/init.h> 41 #include <linux/poll.h> 42 #include <linux/nmi.h> 43 #include <linux/fs.h> 44 #include <linux/trace.h> 45 #include <linux/sched/clock.h> 46 #include <linux/sched/rt.h> 47 48 #include "trace.h" 49 #include "trace_output.h" 50 51 /* 52 * On boot up, the ring buffer is set to the minimum size, so that 53 * we do not waste memory on systems that are not using tracing. 54 */ 55 bool ring_buffer_expanded; 56 57 /* 58 * We need to change this state when a selftest is running. 59 * A selftest will lurk into the ring-buffer to count the 60 * entries inserted during the selftest although some concurrent 61 * insertions into the ring-buffer such as trace_printk could occurred 62 * at the same time, giving false positive or negative results. 63 */ 64 static bool __read_mostly tracing_selftest_running; 65 66 /* 67 * If a tracer is running, we do not want to run SELFTEST. 68 */ 69 bool __read_mostly tracing_selftest_disabled; 70 71 /* Pipe tracepoints to printk */ 72 struct trace_iterator *tracepoint_print_iter; 73 int tracepoint_printk; 74 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key); 75 76 /* For tracers that don't implement custom flags */ 77 static struct tracer_opt dummy_tracer_opt[] = { 78 { } 79 }; 80 81 static int 82 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 83 { 84 return 0; 85 } 86 87 /* 88 * To prevent the comm cache from being overwritten when no 89 * tracing is active, only save the comm when a trace event 90 * occurred. 91 */ 92 static DEFINE_PER_CPU(bool, trace_taskinfo_save); 93 94 /* 95 * Kill all tracing for good (never come back). 96 * It is initialized to 1 but will turn to zero if the initialization 97 * of the tracer is successful. But that is the only place that sets 98 * this back to zero. 99 */ 100 static int tracing_disabled = 1; 101 102 cpumask_var_t __read_mostly tracing_buffer_mask; 103 104 /* 105 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops 106 * 107 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops 108 * is set, then ftrace_dump is called. This will output the contents 109 * of the ftrace buffers to the console. This is very useful for 110 * capturing traces that lead to crashes and outputing it to a 111 * serial console. 112 * 113 * It is default off, but you can enable it with either specifying 114 * "ftrace_dump_on_oops" in the kernel command line, or setting 115 * /proc/sys/kernel/ftrace_dump_on_oops 116 * Set 1 if you want to dump buffers of all CPUs 117 * Set 2 if you want to dump the buffer of the CPU that triggered oops 118 */ 119 120 enum ftrace_dump_mode ftrace_dump_on_oops; 121 122 /* When set, tracing will stop when a WARN*() is hit */ 123 int __disable_trace_on_warning; 124 125 #ifdef CONFIG_TRACE_EVAL_MAP_FILE 126 /* Map of enums to their values, for "eval_map" file */ 127 struct trace_eval_map_head { 128 struct module *mod; 129 unsigned long length; 130 }; 131 132 union trace_eval_map_item; 133 134 struct trace_eval_map_tail { 135 /* 136 * "end" is first and points to NULL as it must be different 137 * than "mod" or "eval_string" 138 */ 139 union trace_eval_map_item *next; 140 const char *end; /* points to NULL */ 141 }; 142 143 static DEFINE_MUTEX(trace_eval_mutex); 144 145 /* 146 * The trace_eval_maps are saved in an array with two extra elements, 147 * one at the beginning, and one at the end. The beginning item contains 148 * the count of the saved maps (head.length), and the module they 149 * belong to if not built in (head.mod). The ending item contains a 150 * pointer to the next array of saved eval_map items. 151 */ 152 union trace_eval_map_item { 153 struct trace_eval_map map; 154 struct trace_eval_map_head head; 155 struct trace_eval_map_tail tail; 156 }; 157 158 static union trace_eval_map_item *trace_eval_maps; 159 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */ 160 161 static int tracing_set_tracer(struct trace_array *tr, const char *buf); 162 static void ftrace_trace_userstack(struct ring_buffer *buffer, 163 unsigned long flags, int pc); 164 165 #define MAX_TRACER_SIZE 100 166 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; 167 static char *default_bootup_tracer; 168 169 static bool allocate_snapshot; 170 171 static int __init set_cmdline_ftrace(char *str) 172 { 173 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE); 174 default_bootup_tracer = bootup_tracer_buf; 175 /* We are using ftrace early, expand it */ 176 ring_buffer_expanded = true; 177 return 1; 178 } 179 __setup("ftrace=", set_cmdline_ftrace); 180 181 static int __init set_ftrace_dump_on_oops(char *str) 182 { 183 if (*str++ != '=' || !*str) { 184 ftrace_dump_on_oops = DUMP_ALL; 185 return 1; 186 } 187 188 if (!strcmp("orig_cpu", str)) { 189 ftrace_dump_on_oops = DUMP_ORIG; 190 return 1; 191 } 192 193 return 0; 194 } 195 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops); 196 197 static int __init stop_trace_on_warning(char *str) 198 { 199 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0)) 200 __disable_trace_on_warning = 1; 201 return 1; 202 } 203 __setup("traceoff_on_warning", stop_trace_on_warning); 204 205 static int __init boot_alloc_snapshot(char *str) 206 { 207 allocate_snapshot = true; 208 /* We also need the main ring buffer expanded */ 209 ring_buffer_expanded = true; 210 return 1; 211 } 212 __setup("alloc_snapshot", boot_alloc_snapshot); 213 214 215 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata; 216 217 static int __init set_trace_boot_options(char *str) 218 { 219 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE); 220 return 0; 221 } 222 __setup("trace_options=", set_trace_boot_options); 223 224 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata; 225 static char *trace_boot_clock __initdata; 226 227 static int __init set_trace_boot_clock(char *str) 228 { 229 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE); 230 trace_boot_clock = trace_boot_clock_buf; 231 return 0; 232 } 233 __setup("trace_clock=", set_trace_boot_clock); 234 235 static int __init set_tracepoint_printk(char *str) 236 { 237 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0)) 238 tracepoint_printk = 1; 239 return 1; 240 } 241 __setup("tp_printk", set_tracepoint_printk); 242 243 unsigned long long ns2usecs(u64 nsec) 244 { 245 nsec += 500; 246 do_div(nsec, 1000); 247 return nsec; 248 } 249 250 /* trace_flags holds trace_options default values */ 251 #define TRACE_DEFAULT_FLAGS \ 252 (FUNCTION_DEFAULT_FLAGS | \ 253 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \ 254 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \ 255 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \ 256 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS) 257 258 /* trace_options that are only supported by global_trace */ 259 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \ 260 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD) 261 262 /* trace_flags that are default zero for instances */ 263 #define ZEROED_TRACE_FLAGS \ 264 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK) 265 266 /* 267 * The global_trace is the descriptor that holds the top-level tracing 268 * buffers for the live tracing. 269 */ 270 static struct trace_array global_trace = { 271 .trace_flags = TRACE_DEFAULT_FLAGS, 272 }; 273 274 LIST_HEAD(ftrace_trace_arrays); 275 276 int trace_array_get(struct trace_array *this_tr) 277 { 278 struct trace_array *tr; 279 int ret = -ENODEV; 280 281 mutex_lock(&trace_types_lock); 282 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 283 if (tr == this_tr) { 284 tr->ref++; 285 ret = 0; 286 break; 287 } 288 } 289 mutex_unlock(&trace_types_lock); 290 291 return ret; 292 } 293 294 static void __trace_array_put(struct trace_array *this_tr) 295 { 296 WARN_ON(!this_tr->ref); 297 this_tr->ref--; 298 } 299 300 void trace_array_put(struct trace_array *this_tr) 301 { 302 mutex_lock(&trace_types_lock); 303 __trace_array_put(this_tr); 304 mutex_unlock(&trace_types_lock); 305 } 306 307 int call_filter_check_discard(struct trace_event_call *call, void *rec, 308 struct ring_buffer *buffer, 309 struct ring_buffer_event *event) 310 { 311 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) && 312 !filter_match_preds(call->filter, rec)) { 313 __trace_event_discard_commit(buffer, event); 314 return 1; 315 } 316 317 return 0; 318 } 319 320 void trace_free_pid_list(struct trace_pid_list *pid_list) 321 { 322 vfree(pid_list->pids); 323 kfree(pid_list); 324 } 325 326 /** 327 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list 328 * @filtered_pids: The list of pids to check 329 * @search_pid: The PID to find in @filtered_pids 330 * 331 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis. 332 */ 333 bool 334 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid) 335 { 336 /* 337 * If pid_max changed after filtered_pids was created, we 338 * by default ignore all pids greater than the previous pid_max. 339 */ 340 if (search_pid >= filtered_pids->pid_max) 341 return false; 342 343 return test_bit(search_pid, filtered_pids->pids); 344 } 345 346 /** 347 * trace_ignore_this_task - should a task be ignored for tracing 348 * @filtered_pids: The list of pids to check 349 * @task: The task that should be ignored if not filtered 350 * 351 * Checks if @task should be traced or not from @filtered_pids. 352 * Returns true if @task should *NOT* be traced. 353 * Returns false if @task should be traced. 354 */ 355 bool 356 trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task) 357 { 358 /* 359 * Return false, because if filtered_pids does not exist, 360 * all pids are good to trace. 361 */ 362 if (!filtered_pids) 363 return false; 364 365 return !trace_find_filtered_pid(filtered_pids, task->pid); 366 } 367 368 /** 369 * trace_filter_add_remove_task - Add or remove a task from a pid_list 370 * @pid_list: The list to modify 371 * @self: The current task for fork or NULL for exit 372 * @task: The task to add or remove 373 * 374 * If adding a task, if @self is defined, the task is only added if @self 375 * is also included in @pid_list. This happens on fork and tasks should 376 * only be added when the parent is listed. If @self is NULL, then the 377 * @task pid will be removed from the list, which would happen on exit 378 * of a task. 379 */ 380 void trace_filter_add_remove_task(struct trace_pid_list *pid_list, 381 struct task_struct *self, 382 struct task_struct *task) 383 { 384 if (!pid_list) 385 return; 386 387 /* For forks, we only add if the forking task is listed */ 388 if (self) { 389 if (!trace_find_filtered_pid(pid_list, self->pid)) 390 return; 391 } 392 393 /* Sorry, but we don't support pid_max changing after setting */ 394 if (task->pid >= pid_list->pid_max) 395 return; 396 397 /* "self" is set for forks, and NULL for exits */ 398 if (self) 399 set_bit(task->pid, pid_list->pids); 400 else 401 clear_bit(task->pid, pid_list->pids); 402 } 403 404 /** 405 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list 406 * @pid_list: The pid list to show 407 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed) 408 * @pos: The position of the file 409 * 410 * This is used by the seq_file "next" operation to iterate the pids 411 * listed in a trace_pid_list structure. 412 * 413 * Returns the pid+1 as we want to display pid of zero, but NULL would 414 * stop the iteration. 415 */ 416 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos) 417 { 418 unsigned long pid = (unsigned long)v; 419 420 (*pos)++; 421 422 /* pid already is +1 of the actual prevous bit */ 423 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid); 424 425 /* Return pid + 1 to allow zero to be represented */ 426 if (pid < pid_list->pid_max) 427 return (void *)(pid + 1); 428 429 return NULL; 430 } 431 432 /** 433 * trace_pid_start - Used for seq_file to start reading pid lists 434 * @pid_list: The pid list to show 435 * @pos: The position of the file 436 * 437 * This is used by seq_file "start" operation to start the iteration 438 * of listing pids. 439 * 440 * Returns the pid+1 as we want to display pid of zero, but NULL would 441 * stop the iteration. 442 */ 443 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos) 444 { 445 unsigned long pid; 446 loff_t l = 0; 447 448 pid = find_first_bit(pid_list->pids, pid_list->pid_max); 449 if (pid >= pid_list->pid_max) 450 return NULL; 451 452 /* Return pid + 1 so that zero can be the exit value */ 453 for (pid++; pid && l < *pos; 454 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l)) 455 ; 456 return (void *)pid; 457 } 458 459 /** 460 * trace_pid_show - show the current pid in seq_file processing 461 * @m: The seq_file structure to write into 462 * @v: A void pointer of the pid (+1) value to display 463 * 464 * Can be directly used by seq_file operations to display the current 465 * pid value. 466 */ 467 int trace_pid_show(struct seq_file *m, void *v) 468 { 469 unsigned long pid = (unsigned long)v - 1; 470 471 seq_printf(m, "%lu\n", pid); 472 return 0; 473 } 474 475 /* 128 should be much more than enough */ 476 #define PID_BUF_SIZE 127 477 478 int trace_pid_write(struct trace_pid_list *filtered_pids, 479 struct trace_pid_list **new_pid_list, 480 const char __user *ubuf, size_t cnt) 481 { 482 struct trace_pid_list *pid_list; 483 struct trace_parser parser; 484 unsigned long val; 485 int nr_pids = 0; 486 ssize_t read = 0; 487 ssize_t ret = 0; 488 loff_t pos; 489 pid_t pid; 490 491 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1)) 492 return -ENOMEM; 493 494 /* 495 * Always recreate a new array. The write is an all or nothing 496 * operation. Always create a new array when adding new pids by 497 * the user. If the operation fails, then the current list is 498 * not modified. 499 */ 500 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL); 501 if (!pid_list) { 502 trace_parser_put(&parser); 503 return -ENOMEM; 504 } 505 506 pid_list->pid_max = READ_ONCE(pid_max); 507 508 /* Only truncating will shrink pid_max */ 509 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max) 510 pid_list->pid_max = filtered_pids->pid_max; 511 512 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3); 513 if (!pid_list->pids) { 514 trace_parser_put(&parser); 515 kfree(pid_list); 516 return -ENOMEM; 517 } 518 519 if (filtered_pids) { 520 /* copy the current bits to the new max */ 521 for_each_set_bit(pid, filtered_pids->pids, 522 filtered_pids->pid_max) { 523 set_bit(pid, pid_list->pids); 524 nr_pids++; 525 } 526 } 527 528 while (cnt > 0) { 529 530 pos = 0; 531 532 ret = trace_get_user(&parser, ubuf, cnt, &pos); 533 if (ret < 0 || !trace_parser_loaded(&parser)) 534 break; 535 536 read += ret; 537 ubuf += ret; 538 cnt -= ret; 539 540 ret = -EINVAL; 541 if (kstrtoul(parser.buffer, 0, &val)) 542 break; 543 if (val >= pid_list->pid_max) 544 break; 545 546 pid = (pid_t)val; 547 548 set_bit(pid, pid_list->pids); 549 nr_pids++; 550 551 trace_parser_clear(&parser); 552 ret = 0; 553 } 554 trace_parser_put(&parser); 555 556 if (ret < 0) { 557 trace_free_pid_list(pid_list); 558 return ret; 559 } 560 561 if (!nr_pids) { 562 /* Cleared the list of pids */ 563 trace_free_pid_list(pid_list); 564 read = ret; 565 pid_list = NULL; 566 } 567 568 *new_pid_list = pid_list; 569 570 return read; 571 } 572 573 static u64 buffer_ftrace_now(struct trace_buffer *buf, int cpu) 574 { 575 u64 ts; 576 577 /* Early boot up does not have a buffer yet */ 578 if (!buf->buffer) 579 return trace_clock_local(); 580 581 ts = ring_buffer_time_stamp(buf->buffer, cpu); 582 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts); 583 584 return ts; 585 } 586 587 u64 ftrace_now(int cpu) 588 { 589 return buffer_ftrace_now(&global_trace.trace_buffer, cpu); 590 } 591 592 /** 593 * tracing_is_enabled - Show if global_trace has been disabled 594 * 595 * Shows if the global trace has been enabled or not. It uses the 596 * mirror flag "buffer_disabled" to be used in fast paths such as for 597 * the irqsoff tracer. But it may be inaccurate due to races. If you 598 * need to know the accurate state, use tracing_is_on() which is a little 599 * slower, but accurate. 600 */ 601 int tracing_is_enabled(void) 602 { 603 /* 604 * For quick access (irqsoff uses this in fast path), just 605 * return the mirror variable of the state of the ring buffer. 606 * It's a little racy, but we don't really care. 607 */ 608 smp_rmb(); 609 return !global_trace.buffer_disabled; 610 } 611 612 /* 613 * trace_buf_size is the size in bytes that is allocated 614 * for a buffer. Note, the number of bytes is always rounded 615 * to page size. 616 * 617 * This number is purposely set to a low number of 16384. 618 * If the dump on oops happens, it will be much appreciated 619 * to not have to wait for all that output. Anyway this can be 620 * boot time and run time configurable. 621 */ 622 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */ 623 624 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT; 625 626 /* trace_types holds a link list of available tracers. */ 627 static struct tracer *trace_types __read_mostly; 628 629 /* 630 * trace_types_lock is used to protect the trace_types list. 631 */ 632 DEFINE_MUTEX(trace_types_lock); 633 634 /* 635 * serialize the access of the ring buffer 636 * 637 * ring buffer serializes readers, but it is low level protection. 638 * The validity of the events (which returns by ring_buffer_peek() ..etc) 639 * are not protected by ring buffer. 640 * 641 * The content of events may become garbage if we allow other process consumes 642 * these events concurrently: 643 * A) the page of the consumed events may become a normal page 644 * (not reader page) in ring buffer, and this page will be rewrited 645 * by events producer. 646 * B) The page of the consumed events may become a page for splice_read, 647 * and this page will be returned to system. 648 * 649 * These primitives allow multi process access to different cpu ring buffer 650 * concurrently. 651 * 652 * These primitives don't distinguish read-only and read-consume access. 653 * Multi read-only access are also serialized. 654 */ 655 656 #ifdef CONFIG_SMP 657 static DECLARE_RWSEM(all_cpu_access_lock); 658 static DEFINE_PER_CPU(struct mutex, cpu_access_lock); 659 660 static inline void trace_access_lock(int cpu) 661 { 662 if (cpu == RING_BUFFER_ALL_CPUS) { 663 /* gain it for accessing the whole ring buffer. */ 664 down_write(&all_cpu_access_lock); 665 } else { 666 /* gain it for accessing a cpu ring buffer. */ 667 668 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */ 669 down_read(&all_cpu_access_lock); 670 671 /* Secondly block other access to this @cpu ring buffer. */ 672 mutex_lock(&per_cpu(cpu_access_lock, cpu)); 673 } 674 } 675 676 static inline void trace_access_unlock(int cpu) 677 { 678 if (cpu == RING_BUFFER_ALL_CPUS) { 679 up_write(&all_cpu_access_lock); 680 } else { 681 mutex_unlock(&per_cpu(cpu_access_lock, cpu)); 682 up_read(&all_cpu_access_lock); 683 } 684 } 685 686 static inline void trace_access_lock_init(void) 687 { 688 int cpu; 689 690 for_each_possible_cpu(cpu) 691 mutex_init(&per_cpu(cpu_access_lock, cpu)); 692 } 693 694 #else 695 696 static DEFINE_MUTEX(access_lock); 697 698 static inline void trace_access_lock(int cpu) 699 { 700 (void)cpu; 701 mutex_lock(&access_lock); 702 } 703 704 static inline void trace_access_unlock(int cpu) 705 { 706 (void)cpu; 707 mutex_unlock(&access_lock); 708 } 709 710 static inline void trace_access_lock_init(void) 711 { 712 } 713 714 #endif 715 716 #ifdef CONFIG_STACKTRACE 717 static void __ftrace_trace_stack(struct ring_buffer *buffer, 718 unsigned long flags, 719 int skip, int pc, struct pt_regs *regs); 720 static inline void ftrace_trace_stack(struct trace_array *tr, 721 struct ring_buffer *buffer, 722 unsigned long flags, 723 int skip, int pc, struct pt_regs *regs); 724 725 #else 726 static inline void __ftrace_trace_stack(struct ring_buffer *buffer, 727 unsigned long flags, 728 int skip, int pc, struct pt_regs *regs) 729 { 730 } 731 static inline void ftrace_trace_stack(struct trace_array *tr, 732 struct ring_buffer *buffer, 733 unsigned long flags, 734 int skip, int pc, struct pt_regs *regs) 735 { 736 } 737 738 #endif 739 740 static __always_inline void 741 trace_event_setup(struct ring_buffer_event *event, 742 int type, unsigned long flags, int pc) 743 { 744 struct trace_entry *ent = ring_buffer_event_data(event); 745 746 tracing_generic_entry_update(ent, type, flags, pc); 747 } 748 749 static __always_inline struct ring_buffer_event * 750 __trace_buffer_lock_reserve(struct ring_buffer *buffer, 751 int type, 752 unsigned long len, 753 unsigned long flags, int pc) 754 { 755 struct ring_buffer_event *event; 756 757 event = ring_buffer_lock_reserve(buffer, len); 758 if (event != NULL) 759 trace_event_setup(event, type, flags, pc); 760 761 return event; 762 } 763 764 void tracer_tracing_on(struct trace_array *tr) 765 { 766 if (tr->trace_buffer.buffer) 767 ring_buffer_record_on(tr->trace_buffer.buffer); 768 /* 769 * This flag is looked at when buffers haven't been allocated 770 * yet, or by some tracers (like irqsoff), that just want to 771 * know if the ring buffer has been disabled, but it can handle 772 * races of where it gets disabled but we still do a record. 773 * As the check is in the fast path of the tracers, it is more 774 * important to be fast than accurate. 775 */ 776 tr->buffer_disabled = 0; 777 /* Make the flag seen by readers */ 778 smp_wmb(); 779 } 780 781 /** 782 * tracing_on - enable tracing buffers 783 * 784 * This function enables tracing buffers that may have been 785 * disabled with tracing_off. 786 */ 787 void tracing_on(void) 788 { 789 tracer_tracing_on(&global_trace); 790 } 791 EXPORT_SYMBOL_GPL(tracing_on); 792 793 794 static __always_inline void 795 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event) 796 { 797 __this_cpu_write(trace_taskinfo_save, true); 798 799 /* If this is the temp buffer, we need to commit fully */ 800 if (this_cpu_read(trace_buffered_event) == event) { 801 /* Length is in event->array[0] */ 802 ring_buffer_write(buffer, event->array[0], &event->array[1]); 803 /* Release the temp buffer */ 804 this_cpu_dec(trace_buffered_event_cnt); 805 } else 806 ring_buffer_unlock_commit(buffer, event); 807 } 808 809 /** 810 * __trace_puts - write a constant string into the trace buffer. 811 * @ip: The address of the caller 812 * @str: The constant string to write 813 * @size: The size of the string. 814 */ 815 int __trace_puts(unsigned long ip, const char *str, int size) 816 { 817 struct ring_buffer_event *event; 818 struct ring_buffer *buffer; 819 struct print_entry *entry; 820 unsigned long irq_flags; 821 int alloc; 822 int pc; 823 824 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 825 return 0; 826 827 pc = preempt_count(); 828 829 if (unlikely(tracing_selftest_running || tracing_disabled)) 830 return 0; 831 832 alloc = sizeof(*entry) + size + 2; /* possible \n added */ 833 834 local_save_flags(irq_flags); 835 buffer = global_trace.trace_buffer.buffer; 836 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 837 irq_flags, pc); 838 if (!event) 839 return 0; 840 841 entry = ring_buffer_event_data(event); 842 entry->ip = ip; 843 844 memcpy(&entry->buf, str, size); 845 846 /* Add a newline if necessary */ 847 if (entry->buf[size - 1] != '\n') { 848 entry->buf[size] = '\n'; 849 entry->buf[size + 1] = '\0'; 850 } else 851 entry->buf[size] = '\0'; 852 853 __buffer_unlock_commit(buffer, event); 854 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL); 855 856 return size; 857 } 858 EXPORT_SYMBOL_GPL(__trace_puts); 859 860 /** 861 * __trace_bputs - write the pointer to a constant string into trace buffer 862 * @ip: The address of the caller 863 * @str: The constant string to write to the buffer to 864 */ 865 int __trace_bputs(unsigned long ip, const char *str) 866 { 867 struct ring_buffer_event *event; 868 struct ring_buffer *buffer; 869 struct bputs_entry *entry; 870 unsigned long irq_flags; 871 int size = sizeof(struct bputs_entry); 872 int pc; 873 874 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 875 return 0; 876 877 pc = preempt_count(); 878 879 if (unlikely(tracing_selftest_running || tracing_disabled)) 880 return 0; 881 882 local_save_flags(irq_flags); 883 buffer = global_trace.trace_buffer.buffer; 884 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size, 885 irq_flags, pc); 886 if (!event) 887 return 0; 888 889 entry = ring_buffer_event_data(event); 890 entry->ip = ip; 891 entry->str = str; 892 893 __buffer_unlock_commit(buffer, event); 894 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL); 895 896 return 1; 897 } 898 EXPORT_SYMBOL_GPL(__trace_bputs); 899 900 #ifdef CONFIG_TRACER_SNAPSHOT 901 void tracing_snapshot_instance_cond(struct trace_array *tr, void *cond_data) 902 { 903 struct tracer *tracer = tr->current_trace; 904 unsigned long flags; 905 906 if (in_nmi()) { 907 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n"); 908 internal_trace_puts("*** snapshot is being ignored ***\n"); 909 return; 910 } 911 912 if (!tr->allocated_snapshot) { 913 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n"); 914 internal_trace_puts("*** stopping trace here! ***\n"); 915 tracing_off(); 916 return; 917 } 918 919 /* Note, snapshot can not be used when the tracer uses it */ 920 if (tracer->use_max_tr) { 921 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n"); 922 internal_trace_puts("*** Can not use snapshot (sorry) ***\n"); 923 return; 924 } 925 926 local_irq_save(flags); 927 update_max_tr(tr, current, smp_processor_id(), cond_data); 928 local_irq_restore(flags); 929 } 930 931 void tracing_snapshot_instance(struct trace_array *tr) 932 { 933 tracing_snapshot_instance_cond(tr, NULL); 934 } 935 936 /** 937 * tracing_snapshot - take a snapshot of the current buffer. 938 * 939 * This causes a swap between the snapshot buffer and the current live 940 * tracing buffer. You can use this to take snapshots of the live 941 * trace when some condition is triggered, but continue to trace. 942 * 943 * Note, make sure to allocate the snapshot with either 944 * a tracing_snapshot_alloc(), or by doing it manually 945 * with: echo 1 > /sys/kernel/debug/tracing/snapshot 946 * 947 * If the snapshot buffer is not allocated, it will stop tracing. 948 * Basically making a permanent snapshot. 949 */ 950 void tracing_snapshot(void) 951 { 952 struct trace_array *tr = &global_trace; 953 954 tracing_snapshot_instance(tr); 955 } 956 EXPORT_SYMBOL_GPL(tracing_snapshot); 957 958 /** 959 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer. 960 * @tr: The tracing instance to snapshot 961 * @cond_data: The data to be tested conditionally, and possibly saved 962 * 963 * This is the same as tracing_snapshot() except that the snapshot is 964 * conditional - the snapshot will only happen if the 965 * cond_snapshot.update() implementation receiving the cond_data 966 * returns true, which means that the trace array's cond_snapshot 967 * update() operation used the cond_data to determine whether the 968 * snapshot should be taken, and if it was, presumably saved it along 969 * with the snapshot. 970 */ 971 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data) 972 { 973 tracing_snapshot_instance_cond(tr, cond_data); 974 } 975 EXPORT_SYMBOL_GPL(tracing_snapshot_cond); 976 977 /** 978 * tracing_snapshot_cond_data - get the user data associated with a snapshot 979 * @tr: The tracing instance 980 * 981 * When the user enables a conditional snapshot using 982 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved 983 * with the snapshot. This accessor is used to retrieve it. 984 * 985 * Should not be called from cond_snapshot.update(), since it takes 986 * the tr->max_lock lock, which the code calling 987 * cond_snapshot.update() has already done. 988 * 989 * Returns the cond_data associated with the trace array's snapshot. 990 */ 991 void *tracing_cond_snapshot_data(struct trace_array *tr) 992 { 993 void *cond_data = NULL; 994 995 arch_spin_lock(&tr->max_lock); 996 997 if (tr->cond_snapshot) 998 cond_data = tr->cond_snapshot->cond_data; 999 1000 arch_spin_unlock(&tr->max_lock); 1001 1002 return cond_data; 1003 } 1004 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data); 1005 1006 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf, 1007 struct trace_buffer *size_buf, int cpu_id); 1008 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val); 1009 1010 int tracing_alloc_snapshot_instance(struct trace_array *tr) 1011 { 1012 int ret; 1013 1014 if (!tr->allocated_snapshot) { 1015 1016 /* allocate spare buffer */ 1017 ret = resize_buffer_duplicate_size(&tr->max_buffer, 1018 &tr->trace_buffer, RING_BUFFER_ALL_CPUS); 1019 if (ret < 0) 1020 return ret; 1021 1022 tr->allocated_snapshot = true; 1023 } 1024 1025 return 0; 1026 } 1027 1028 static void free_snapshot(struct trace_array *tr) 1029 { 1030 /* 1031 * We don't free the ring buffer. instead, resize it because 1032 * The max_tr ring buffer has some state (e.g. ring->clock) and 1033 * we want preserve it. 1034 */ 1035 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS); 1036 set_buffer_entries(&tr->max_buffer, 1); 1037 tracing_reset_online_cpus(&tr->max_buffer); 1038 tr->allocated_snapshot = false; 1039 } 1040 1041 /** 1042 * tracing_alloc_snapshot - allocate snapshot buffer. 1043 * 1044 * This only allocates the snapshot buffer if it isn't already 1045 * allocated - it doesn't also take a snapshot. 1046 * 1047 * This is meant to be used in cases where the snapshot buffer needs 1048 * to be set up for events that can't sleep but need to be able to 1049 * trigger a snapshot. 1050 */ 1051 int tracing_alloc_snapshot(void) 1052 { 1053 struct trace_array *tr = &global_trace; 1054 int ret; 1055 1056 ret = tracing_alloc_snapshot_instance(tr); 1057 WARN_ON(ret < 0); 1058 1059 return ret; 1060 } 1061 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); 1062 1063 /** 1064 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer. 1065 * 1066 * This is similar to tracing_snapshot(), but it will allocate the 1067 * snapshot buffer if it isn't already allocated. Use this only 1068 * where it is safe to sleep, as the allocation may sleep. 1069 * 1070 * This causes a swap between the snapshot buffer and the current live 1071 * tracing buffer. You can use this to take snapshots of the live 1072 * trace when some condition is triggered, but continue to trace. 1073 */ 1074 void tracing_snapshot_alloc(void) 1075 { 1076 int ret; 1077 1078 ret = tracing_alloc_snapshot(); 1079 if (ret < 0) 1080 return; 1081 1082 tracing_snapshot(); 1083 } 1084 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); 1085 1086 /** 1087 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance 1088 * @tr: The tracing instance 1089 * @cond_data: User data to associate with the snapshot 1090 * @update: Implementation of the cond_snapshot update function 1091 * 1092 * Check whether the conditional snapshot for the given instance has 1093 * already been enabled, or if the current tracer is already using a 1094 * snapshot; if so, return -EBUSY, else create a cond_snapshot and 1095 * save the cond_data and update function inside. 1096 * 1097 * Returns 0 if successful, error otherwise. 1098 */ 1099 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, 1100 cond_update_fn_t update) 1101 { 1102 struct cond_snapshot *cond_snapshot; 1103 int ret = 0; 1104 1105 cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL); 1106 if (!cond_snapshot) 1107 return -ENOMEM; 1108 1109 cond_snapshot->cond_data = cond_data; 1110 cond_snapshot->update = update; 1111 1112 mutex_lock(&trace_types_lock); 1113 1114 ret = tracing_alloc_snapshot_instance(tr); 1115 if (ret) 1116 goto fail_unlock; 1117 1118 if (tr->current_trace->use_max_tr) { 1119 ret = -EBUSY; 1120 goto fail_unlock; 1121 } 1122 1123 /* 1124 * The cond_snapshot can only change to NULL without the 1125 * trace_types_lock. We don't care if we race with it going 1126 * to NULL, but we want to make sure that it's not set to 1127 * something other than NULL when we get here, which we can 1128 * do safely with only holding the trace_types_lock and not 1129 * having to take the max_lock. 1130 */ 1131 if (tr->cond_snapshot) { 1132 ret = -EBUSY; 1133 goto fail_unlock; 1134 } 1135 1136 arch_spin_lock(&tr->max_lock); 1137 tr->cond_snapshot = cond_snapshot; 1138 arch_spin_unlock(&tr->max_lock); 1139 1140 mutex_unlock(&trace_types_lock); 1141 1142 return ret; 1143 1144 fail_unlock: 1145 mutex_unlock(&trace_types_lock); 1146 kfree(cond_snapshot); 1147 return ret; 1148 } 1149 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable); 1150 1151 /** 1152 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance 1153 * @tr: The tracing instance 1154 * 1155 * Check whether the conditional snapshot for the given instance is 1156 * enabled; if so, free the cond_snapshot associated with it, 1157 * otherwise return -EINVAL. 1158 * 1159 * Returns 0 if successful, error otherwise. 1160 */ 1161 int tracing_snapshot_cond_disable(struct trace_array *tr) 1162 { 1163 int ret = 0; 1164 1165 arch_spin_lock(&tr->max_lock); 1166 1167 if (!tr->cond_snapshot) 1168 ret = -EINVAL; 1169 else { 1170 kfree(tr->cond_snapshot); 1171 tr->cond_snapshot = NULL; 1172 } 1173 1174 arch_spin_unlock(&tr->max_lock); 1175 1176 return ret; 1177 } 1178 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable); 1179 #else 1180 void tracing_snapshot(void) 1181 { 1182 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used"); 1183 } 1184 EXPORT_SYMBOL_GPL(tracing_snapshot); 1185 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data) 1186 { 1187 WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used"); 1188 } 1189 EXPORT_SYMBOL_GPL(tracing_snapshot_cond); 1190 int tracing_alloc_snapshot(void) 1191 { 1192 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used"); 1193 return -ENODEV; 1194 } 1195 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); 1196 void tracing_snapshot_alloc(void) 1197 { 1198 /* Give warning */ 1199 tracing_snapshot(); 1200 } 1201 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); 1202 void *tracing_cond_snapshot_data(struct trace_array *tr) 1203 { 1204 return NULL; 1205 } 1206 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data); 1207 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update) 1208 { 1209 return -ENODEV; 1210 } 1211 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable); 1212 int tracing_snapshot_cond_disable(struct trace_array *tr) 1213 { 1214 return false; 1215 } 1216 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable); 1217 #endif /* CONFIG_TRACER_SNAPSHOT */ 1218 1219 void tracer_tracing_off(struct trace_array *tr) 1220 { 1221 if (tr->trace_buffer.buffer) 1222 ring_buffer_record_off(tr->trace_buffer.buffer); 1223 /* 1224 * This flag is looked at when buffers haven't been allocated 1225 * yet, or by some tracers (like irqsoff), that just want to 1226 * know if the ring buffer has been disabled, but it can handle 1227 * races of where it gets disabled but we still do a record. 1228 * As the check is in the fast path of the tracers, it is more 1229 * important to be fast than accurate. 1230 */ 1231 tr->buffer_disabled = 1; 1232 /* Make the flag seen by readers */ 1233 smp_wmb(); 1234 } 1235 1236 /** 1237 * tracing_off - turn off tracing buffers 1238 * 1239 * This function stops the tracing buffers from recording data. 1240 * It does not disable any overhead the tracers themselves may 1241 * be causing. This function simply causes all recording to 1242 * the ring buffers to fail. 1243 */ 1244 void tracing_off(void) 1245 { 1246 tracer_tracing_off(&global_trace); 1247 } 1248 EXPORT_SYMBOL_GPL(tracing_off); 1249 1250 void disable_trace_on_warning(void) 1251 { 1252 if (__disable_trace_on_warning) 1253 tracing_off(); 1254 } 1255 1256 /** 1257 * tracer_tracing_is_on - show real state of ring buffer enabled 1258 * @tr : the trace array to know if ring buffer is enabled 1259 * 1260 * Shows real state of the ring buffer if it is enabled or not. 1261 */ 1262 bool tracer_tracing_is_on(struct trace_array *tr) 1263 { 1264 if (tr->trace_buffer.buffer) 1265 return ring_buffer_record_is_on(tr->trace_buffer.buffer); 1266 return !tr->buffer_disabled; 1267 } 1268 1269 /** 1270 * tracing_is_on - show state of ring buffers enabled 1271 */ 1272 int tracing_is_on(void) 1273 { 1274 return tracer_tracing_is_on(&global_trace); 1275 } 1276 EXPORT_SYMBOL_GPL(tracing_is_on); 1277 1278 static int __init set_buf_size(char *str) 1279 { 1280 unsigned long buf_size; 1281 1282 if (!str) 1283 return 0; 1284 buf_size = memparse(str, &str); 1285 /* nr_entries can not be zero */ 1286 if (buf_size == 0) 1287 return 0; 1288 trace_buf_size = buf_size; 1289 return 1; 1290 } 1291 __setup("trace_buf_size=", set_buf_size); 1292 1293 static int __init set_tracing_thresh(char *str) 1294 { 1295 unsigned long threshold; 1296 int ret; 1297 1298 if (!str) 1299 return 0; 1300 ret = kstrtoul(str, 0, &threshold); 1301 if (ret < 0) 1302 return 0; 1303 tracing_thresh = threshold * 1000; 1304 return 1; 1305 } 1306 __setup("tracing_thresh=", set_tracing_thresh); 1307 1308 unsigned long nsecs_to_usecs(unsigned long nsecs) 1309 { 1310 return nsecs / 1000; 1311 } 1312 1313 /* 1314 * TRACE_FLAGS is defined as a tuple matching bit masks with strings. 1315 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that 1316 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list 1317 * of strings in the order that the evals (enum) were defined. 1318 */ 1319 #undef C 1320 #define C(a, b) b 1321 1322 /* These must match the bit postions in trace_iterator_flags */ 1323 static const char *trace_options[] = { 1324 TRACE_FLAGS 1325 NULL 1326 }; 1327 1328 static struct { 1329 u64 (*func)(void); 1330 const char *name; 1331 int in_ns; /* is this clock in nanoseconds? */ 1332 } trace_clocks[] = { 1333 { trace_clock_local, "local", 1 }, 1334 { trace_clock_global, "global", 1 }, 1335 { trace_clock_counter, "counter", 0 }, 1336 { trace_clock_jiffies, "uptime", 0 }, 1337 { trace_clock, "perf", 1 }, 1338 { ktime_get_mono_fast_ns, "mono", 1 }, 1339 { ktime_get_raw_fast_ns, "mono_raw", 1 }, 1340 { ktime_get_boot_fast_ns, "boot", 1 }, 1341 ARCH_TRACE_CLOCKS 1342 }; 1343 1344 bool trace_clock_in_ns(struct trace_array *tr) 1345 { 1346 if (trace_clocks[tr->clock_id].in_ns) 1347 return true; 1348 1349 return false; 1350 } 1351 1352 /* 1353 * trace_parser_get_init - gets the buffer for trace parser 1354 */ 1355 int trace_parser_get_init(struct trace_parser *parser, int size) 1356 { 1357 memset(parser, 0, sizeof(*parser)); 1358 1359 parser->buffer = kmalloc(size, GFP_KERNEL); 1360 if (!parser->buffer) 1361 return 1; 1362 1363 parser->size = size; 1364 return 0; 1365 } 1366 1367 /* 1368 * trace_parser_put - frees the buffer for trace parser 1369 */ 1370 void trace_parser_put(struct trace_parser *parser) 1371 { 1372 kfree(parser->buffer); 1373 parser->buffer = NULL; 1374 } 1375 1376 /* 1377 * trace_get_user - reads the user input string separated by space 1378 * (matched by isspace(ch)) 1379 * 1380 * For each string found the 'struct trace_parser' is updated, 1381 * and the function returns. 1382 * 1383 * Returns number of bytes read. 1384 * 1385 * See kernel/trace/trace.h for 'struct trace_parser' details. 1386 */ 1387 int trace_get_user(struct trace_parser *parser, const char __user *ubuf, 1388 size_t cnt, loff_t *ppos) 1389 { 1390 char ch; 1391 size_t read = 0; 1392 ssize_t ret; 1393 1394 if (!*ppos) 1395 trace_parser_clear(parser); 1396 1397 ret = get_user(ch, ubuf++); 1398 if (ret) 1399 goto out; 1400 1401 read++; 1402 cnt--; 1403 1404 /* 1405 * The parser is not finished with the last write, 1406 * continue reading the user input without skipping spaces. 1407 */ 1408 if (!parser->cont) { 1409 /* skip white space */ 1410 while (cnt && isspace(ch)) { 1411 ret = get_user(ch, ubuf++); 1412 if (ret) 1413 goto out; 1414 read++; 1415 cnt--; 1416 } 1417 1418 parser->idx = 0; 1419 1420 /* only spaces were written */ 1421 if (isspace(ch) || !ch) { 1422 *ppos += read; 1423 ret = read; 1424 goto out; 1425 } 1426 } 1427 1428 /* read the non-space input */ 1429 while (cnt && !isspace(ch) && ch) { 1430 if (parser->idx < parser->size - 1) 1431 parser->buffer[parser->idx++] = ch; 1432 else { 1433 ret = -EINVAL; 1434 goto out; 1435 } 1436 ret = get_user(ch, ubuf++); 1437 if (ret) 1438 goto out; 1439 read++; 1440 cnt--; 1441 } 1442 1443 /* We either got finished input or we have to wait for another call. */ 1444 if (isspace(ch) || !ch) { 1445 parser->buffer[parser->idx] = 0; 1446 parser->cont = false; 1447 } else if (parser->idx < parser->size - 1) { 1448 parser->cont = true; 1449 parser->buffer[parser->idx++] = ch; 1450 /* Make sure the parsed string always terminates with '\0'. */ 1451 parser->buffer[parser->idx] = 0; 1452 } else { 1453 ret = -EINVAL; 1454 goto out; 1455 } 1456 1457 *ppos += read; 1458 ret = read; 1459 1460 out: 1461 return ret; 1462 } 1463 1464 /* TODO add a seq_buf_to_buffer() */ 1465 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) 1466 { 1467 int len; 1468 1469 if (trace_seq_used(s) <= s->seq.readpos) 1470 return -EBUSY; 1471 1472 len = trace_seq_used(s) - s->seq.readpos; 1473 if (cnt > len) 1474 cnt = len; 1475 memcpy(buf, s->buffer + s->seq.readpos, cnt); 1476 1477 s->seq.readpos += cnt; 1478 return cnt; 1479 } 1480 1481 unsigned long __read_mostly tracing_thresh; 1482 1483 #ifdef CONFIG_TRACER_MAX_TRACE 1484 /* 1485 * Copy the new maximum trace into the separate maximum-trace 1486 * structure. (this way the maximum trace is permanently saved, 1487 * for later retrieval via /sys/kernel/tracing/tracing_max_latency) 1488 */ 1489 static void 1490 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 1491 { 1492 struct trace_buffer *trace_buf = &tr->trace_buffer; 1493 struct trace_buffer *max_buf = &tr->max_buffer; 1494 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu); 1495 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu); 1496 1497 max_buf->cpu = cpu; 1498 max_buf->time_start = data->preempt_timestamp; 1499 1500 max_data->saved_latency = tr->max_latency; 1501 max_data->critical_start = data->critical_start; 1502 max_data->critical_end = data->critical_end; 1503 1504 strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN); 1505 max_data->pid = tsk->pid; 1506 /* 1507 * If tsk == current, then use current_uid(), as that does not use 1508 * RCU. The irq tracer can be called out of RCU scope. 1509 */ 1510 if (tsk == current) 1511 max_data->uid = current_uid(); 1512 else 1513 max_data->uid = task_uid(tsk); 1514 1515 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; 1516 max_data->policy = tsk->policy; 1517 max_data->rt_priority = tsk->rt_priority; 1518 1519 /* record this tasks comm */ 1520 tracing_record_cmdline(tsk); 1521 } 1522 1523 /** 1524 * update_max_tr - snapshot all trace buffers from global_trace to max_tr 1525 * @tr: tracer 1526 * @tsk: the task with the latency 1527 * @cpu: The cpu that initiated the trace. 1528 * @cond_data: User data associated with a conditional snapshot 1529 * 1530 * Flip the buffers between the @tr and the max_tr and record information 1531 * about which task was the cause of this latency. 1532 */ 1533 void 1534 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu, 1535 void *cond_data) 1536 { 1537 if (tr->stop_count) 1538 return; 1539 1540 WARN_ON_ONCE(!irqs_disabled()); 1541 1542 if (!tr->allocated_snapshot) { 1543 /* Only the nop tracer should hit this when disabling */ 1544 WARN_ON_ONCE(tr->current_trace != &nop_trace); 1545 return; 1546 } 1547 1548 arch_spin_lock(&tr->max_lock); 1549 1550 /* Inherit the recordable setting from trace_buffer */ 1551 if (ring_buffer_record_is_set_on(tr->trace_buffer.buffer)) 1552 ring_buffer_record_on(tr->max_buffer.buffer); 1553 else 1554 ring_buffer_record_off(tr->max_buffer.buffer); 1555 1556 #ifdef CONFIG_TRACER_SNAPSHOT 1557 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data)) 1558 goto out_unlock; 1559 #endif 1560 swap(tr->trace_buffer.buffer, tr->max_buffer.buffer); 1561 1562 __update_max_tr(tr, tsk, cpu); 1563 1564 out_unlock: 1565 arch_spin_unlock(&tr->max_lock); 1566 } 1567 1568 /** 1569 * update_max_tr_single - only copy one trace over, and reset the rest 1570 * @tr: tracer 1571 * @tsk: task with the latency 1572 * @cpu: the cpu of the buffer to copy. 1573 * 1574 * Flip the trace of a single CPU buffer between the @tr and the max_tr. 1575 */ 1576 void 1577 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) 1578 { 1579 int ret; 1580 1581 if (tr->stop_count) 1582 return; 1583 1584 WARN_ON_ONCE(!irqs_disabled()); 1585 if (!tr->allocated_snapshot) { 1586 /* Only the nop tracer should hit this when disabling */ 1587 WARN_ON_ONCE(tr->current_trace != &nop_trace); 1588 return; 1589 } 1590 1591 arch_spin_lock(&tr->max_lock); 1592 1593 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu); 1594 1595 if (ret == -EBUSY) { 1596 /* 1597 * We failed to swap the buffer due to a commit taking 1598 * place on this CPU. We fail to record, but we reset 1599 * the max trace buffer (no one writes directly to it) 1600 * and flag that it failed. 1601 */ 1602 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_, 1603 "Failed to swap buffers due to commit in progress\n"); 1604 } 1605 1606 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); 1607 1608 __update_max_tr(tr, tsk, cpu); 1609 arch_spin_unlock(&tr->max_lock); 1610 } 1611 #endif /* CONFIG_TRACER_MAX_TRACE */ 1612 1613 static int wait_on_pipe(struct trace_iterator *iter, int full) 1614 { 1615 /* Iterators are static, they should be filled or empty */ 1616 if (trace_buffer_iter(iter, iter->cpu_file)) 1617 return 0; 1618 1619 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file, 1620 full); 1621 } 1622 1623 #ifdef CONFIG_FTRACE_STARTUP_TEST 1624 static bool selftests_can_run; 1625 1626 struct trace_selftests { 1627 struct list_head list; 1628 struct tracer *type; 1629 }; 1630 1631 static LIST_HEAD(postponed_selftests); 1632 1633 static int save_selftest(struct tracer *type) 1634 { 1635 struct trace_selftests *selftest; 1636 1637 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL); 1638 if (!selftest) 1639 return -ENOMEM; 1640 1641 selftest->type = type; 1642 list_add(&selftest->list, &postponed_selftests); 1643 return 0; 1644 } 1645 1646 static int run_tracer_selftest(struct tracer *type) 1647 { 1648 struct trace_array *tr = &global_trace; 1649 struct tracer *saved_tracer = tr->current_trace; 1650 int ret; 1651 1652 if (!type->selftest || tracing_selftest_disabled) 1653 return 0; 1654 1655 /* 1656 * If a tracer registers early in boot up (before scheduling is 1657 * initialized and such), then do not run its selftests yet. 1658 * Instead, run it a little later in the boot process. 1659 */ 1660 if (!selftests_can_run) 1661 return save_selftest(type); 1662 1663 /* 1664 * Run a selftest on this tracer. 1665 * Here we reset the trace buffer, and set the current 1666 * tracer to be this tracer. The tracer can then run some 1667 * internal tracing to verify that everything is in order. 1668 * If we fail, we do not register this tracer. 1669 */ 1670 tracing_reset_online_cpus(&tr->trace_buffer); 1671 1672 tr->current_trace = type; 1673 1674 #ifdef CONFIG_TRACER_MAX_TRACE 1675 if (type->use_max_tr) { 1676 /* If we expanded the buffers, make sure the max is expanded too */ 1677 if (ring_buffer_expanded) 1678 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size, 1679 RING_BUFFER_ALL_CPUS); 1680 tr->allocated_snapshot = true; 1681 } 1682 #endif 1683 1684 /* the test is responsible for initializing and enabling */ 1685 pr_info("Testing tracer %s: ", type->name); 1686 ret = type->selftest(type, tr); 1687 /* the test is responsible for resetting too */ 1688 tr->current_trace = saved_tracer; 1689 if (ret) { 1690 printk(KERN_CONT "FAILED!\n"); 1691 /* Add the warning after printing 'FAILED' */ 1692 WARN_ON(1); 1693 return -1; 1694 } 1695 /* Only reset on passing, to avoid touching corrupted buffers */ 1696 tracing_reset_online_cpus(&tr->trace_buffer); 1697 1698 #ifdef CONFIG_TRACER_MAX_TRACE 1699 if (type->use_max_tr) { 1700 tr->allocated_snapshot = false; 1701 1702 /* Shrink the max buffer again */ 1703 if (ring_buffer_expanded) 1704 ring_buffer_resize(tr->max_buffer.buffer, 1, 1705 RING_BUFFER_ALL_CPUS); 1706 } 1707 #endif 1708 1709 printk(KERN_CONT "PASSED\n"); 1710 return 0; 1711 } 1712 1713 static __init int init_trace_selftests(void) 1714 { 1715 struct trace_selftests *p, *n; 1716 struct tracer *t, **last; 1717 int ret; 1718 1719 selftests_can_run = true; 1720 1721 mutex_lock(&trace_types_lock); 1722 1723 if (list_empty(&postponed_selftests)) 1724 goto out; 1725 1726 pr_info("Running postponed tracer tests:\n"); 1727 1728 list_for_each_entry_safe(p, n, &postponed_selftests, list) { 1729 /* This loop can take minutes when sanitizers are enabled, so 1730 * lets make sure we allow RCU processing. 1731 */ 1732 cond_resched(); 1733 ret = run_tracer_selftest(p->type); 1734 /* If the test fails, then warn and remove from available_tracers */ 1735 if (ret < 0) { 1736 WARN(1, "tracer: %s failed selftest, disabling\n", 1737 p->type->name); 1738 last = &trace_types; 1739 for (t = trace_types; t; t = t->next) { 1740 if (t == p->type) { 1741 *last = t->next; 1742 break; 1743 } 1744 last = &t->next; 1745 } 1746 } 1747 list_del(&p->list); 1748 kfree(p); 1749 } 1750 1751 out: 1752 mutex_unlock(&trace_types_lock); 1753 1754 return 0; 1755 } 1756 core_initcall(init_trace_selftests); 1757 #else 1758 static inline int run_tracer_selftest(struct tracer *type) 1759 { 1760 return 0; 1761 } 1762 #endif /* CONFIG_FTRACE_STARTUP_TEST */ 1763 1764 static void add_tracer_options(struct trace_array *tr, struct tracer *t); 1765 1766 static void __init apply_trace_boot_options(void); 1767 1768 /** 1769 * register_tracer - register a tracer with the ftrace system. 1770 * @type: the plugin for the tracer 1771 * 1772 * Register a new plugin tracer. 1773 */ 1774 int __init register_tracer(struct tracer *type) 1775 { 1776 struct tracer *t; 1777 int ret = 0; 1778 1779 if (!type->name) { 1780 pr_info("Tracer must have a name\n"); 1781 return -1; 1782 } 1783 1784 if (strlen(type->name) >= MAX_TRACER_SIZE) { 1785 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE); 1786 return -1; 1787 } 1788 1789 mutex_lock(&trace_types_lock); 1790 1791 tracing_selftest_running = true; 1792 1793 for (t = trace_types; t; t = t->next) { 1794 if (strcmp(type->name, t->name) == 0) { 1795 /* already found */ 1796 pr_info("Tracer %s already registered\n", 1797 type->name); 1798 ret = -1; 1799 goto out; 1800 } 1801 } 1802 1803 if (!type->set_flag) 1804 type->set_flag = &dummy_set_flag; 1805 if (!type->flags) { 1806 /*allocate a dummy tracer_flags*/ 1807 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL); 1808 if (!type->flags) { 1809 ret = -ENOMEM; 1810 goto out; 1811 } 1812 type->flags->val = 0; 1813 type->flags->opts = dummy_tracer_opt; 1814 } else 1815 if (!type->flags->opts) 1816 type->flags->opts = dummy_tracer_opt; 1817 1818 /* store the tracer for __set_tracer_option */ 1819 type->flags->trace = type; 1820 1821 ret = run_tracer_selftest(type); 1822 if (ret < 0) 1823 goto out; 1824 1825 type->next = trace_types; 1826 trace_types = type; 1827 add_tracer_options(&global_trace, type); 1828 1829 out: 1830 tracing_selftest_running = false; 1831 mutex_unlock(&trace_types_lock); 1832 1833 if (ret || !default_bootup_tracer) 1834 goto out_unlock; 1835 1836 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE)) 1837 goto out_unlock; 1838 1839 printk(KERN_INFO "Starting tracer '%s'\n", type->name); 1840 /* Do we want this tracer to start on bootup? */ 1841 tracing_set_tracer(&global_trace, type->name); 1842 default_bootup_tracer = NULL; 1843 1844 apply_trace_boot_options(); 1845 1846 /* disable other selftests, since this will break it. */ 1847 tracing_selftest_disabled = true; 1848 #ifdef CONFIG_FTRACE_STARTUP_TEST 1849 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n", 1850 type->name); 1851 #endif 1852 1853 out_unlock: 1854 return ret; 1855 } 1856 1857 void tracing_reset(struct trace_buffer *buf, int cpu) 1858 { 1859 struct ring_buffer *buffer = buf->buffer; 1860 1861 if (!buffer) 1862 return; 1863 1864 ring_buffer_record_disable(buffer); 1865 1866 /* Make sure all commits have finished */ 1867 synchronize_rcu(); 1868 ring_buffer_reset_cpu(buffer, cpu); 1869 1870 ring_buffer_record_enable(buffer); 1871 } 1872 1873 void tracing_reset_online_cpus(struct trace_buffer *buf) 1874 { 1875 struct ring_buffer *buffer = buf->buffer; 1876 int cpu; 1877 1878 if (!buffer) 1879 return; 1880 1881 ring_buffer_record_disable(buffer); 1882 1883 /* Make sure all commits have finished */ 1884 synchronize_rcu(); 1885 1886 buf->time_start = buffer_ftrace_now(buf, buf->cpu); 1887 1888 for_each_online_cpu(cpu) 1889 ring_buffer_reset_cpu(buffer, cpu); 1890 1891 ring_buffer_record_enable(buffer); 1892 } 1893 1894 /* Must have trace_types_lock held */ 1895 void tracing_reset_all_online_cpus(void) 1896 { 1897 struct trace_array *tr; 1898 1899 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 1900 if (!tr->clear_trace) 1901 continue; 1902 tr->clear_trace = false; 1903 tracing_reset_online_cpus(&tr->trace_buffer); 1904 #ifdef CONFIG_TRACER_MAX_TRACE 1905 tracing_reset_online_cpus(&tr->max_buffer); 1906 #endif 1907 } 1908 } 1909 1910 static int *tgid_map; 1911 1912 #define SAVED_CMDLINES_DEFAULT 128 1913 #define NO_CMDLINE_MAP UINT_MAX 1914 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED; 1915 struct saved_cmdlines_buffer { 1916 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; 1917 unsigned *map_cmdline_to_pid; 1918 unsigned cmdline_num; 1919 int cmdline_idx; 1920 char *saved_cmdlines; 1921 }; 1922 static struct saved_cmdlines_buffer *savedcmd; 1923 1924 /* temporary disable recording */ 1925 static atomic_t trace_record_taskinfo_disabled __read_mostly; 1926 1927 static inline char *get_saved_cmdlines(int idx) 1928 { 1929 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN]; 1930 } 1931 1932 static inline void set_cmdline(int idx, const char *cmdline) 1933 { 1934 strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN); 1935 } 1936 1937 static int allocate_cmdlines_buffer(unsigned int val, 1938 struct saved_cmdlines_buffer *s) 1939 { 1940 s->map_cmdline_to_pid = kmalloc_array(val, 1941 sizeof(*s->map_cmdline_to_pid), 1942 GFP_KERNEL); 1943 if (!s->map_cmdline_to_pid) 1944 return -ENOMEM; 1945 1946 s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL); 1947 if (!s->saved_cmdlines) { 1948 kfree(s->map_cmdline_to_pid); 1949 return -ENOMEM; 1950 } 1951 1952 s->cmdline_idx = 0; 1953 s->cmdline_num = val; 1954 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP, 1955 sizeof(s->map_pid_to_cmdline)); 1956 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP, 1957 val * sizeof(*s->map_cmdline_to_pid)); 1958 1959 return 0; 1960 } 1961 1962 static int trace_create_savedcmd(void) 1963 { 1964 int ret; 1965 1966 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL); 1967 if (!savedcmd) 1968 return -ENOMEM; 1969 1970 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd); 1971 if (ret < 0) { 1972 kfree(savedcmd); 1973 savedcmd = NULL; 1974 return -ENOMEM; 1975 } 1976 1977 return 0; 1978 } 1979 1980 int is_tracing_stopped(void) 1981 { 1982 return global_trace.stop_count; 1983 } 1984 1985 /** 1986 * tracing_start - quick start of the tracer 1987 * 1988 * If tracing is enabled but was stopped by tracing_stop, 1989 * this will start the tracer back up. 1990 */ 1991 void tracing_start(void) 1992 { 1993 struct ring_buffer *buffer; 1994 unsigned long flags; 1995 1996 if (tracing_disabled) 1997 return; 1998 1999 raw_spin_lock_irqsave(&global_trace.start_lock, flags); 2000 if (--global_trace.stop_count) { 2001 if (global_trace.stop_count < 0) { 2002 /* Someone screwed up their debugging */ 2003 WARN_ON_ONCE(1); 2004 global_trace.stop_count = 0; 2005 } 2006 goto out; 2007 } 2008 2009 /* Prevent the buffers from switching */ 2010 arch_spin_lock(&global_trace.max_lock); 2011 2012 buffer = global_trace.trace_buffer.buffer; 2013 if (buffer) 2014 ring_buffer_record_enable(buffer); 2015 2016 #ifdef CONFIG_TRACER_MAX_TRACE 2017 buffer = global_trace.max_buffer.buffer; 2018 if (buffer) 2019 ring_buffer_record_enable(buffer); 2020 #endif 2021 2022 arch_spin_unlock(&global_trace.max_lock); 2023 2024 out: 2025 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags); 2026 } 2027 2028 static void tracing_start_tr(struct trace_array *tr) 2029 { 2030 struct ring_buffer *buffer; 2031 unsigned long flags; 2032 2033 if (tracing_disabled) 2034 return; 2035 2036 /* If global, we need to also start the max tracer */ 2037 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 2038 return tracing_start(); 2039 2040 raw_spin_lock_irqsave(&tr->start_lock, flags); 2041 2042 if (--tr->stop_count) { 2043 if (tr->stop_count < 0) { 2044 /* Someone screwed up their debugging */ 2045 WARN_ON_ONCE(1); 2046 tr->stop_count = 0; 2047 } 2048 goto out; 2049 } 2050 2051 buffer = tr->trace_buffer.buffer; 2052 if (buffer) 2053 ring_buffer_record_enable(buffer); 2054 2055 out: 2056 raw_spin_unlock_irqrestore(&tr->start_lock, flags); 2057 } 2058 2059 /** 2060 * tracing_stop - quick stop of the tracer 2061 * 2062 * Light weight way to stop tracing. Use in conjunction with 2063 * tracing_start. 2064 */ 2065 void tracing_stop(void) 2066 { 2067 struct ring_buffer *buffer; 2068 unsigned long flags; 2069 2070 raw_spin_lock_irqsave(&global_trace.start_lock, flags); 2071 if (global_trace.stop_count++) 2072 goto out; 2073 2074 /* Prevent the buffers from switching */ 2075 arch_spin_lock(&global_trace.max_lock); 2076 2077 buffer = global_trace.trace_buffer.buffer; 2078 if (buffer) 2079 ring_buffer_record_disable(buffer); 2080 2081 #ifdef CONFIG_TRACER_MAX_TRACE 2082 buffer = global_trace.max_buffer.buffer; 2083 if (buffer) 2084 ring_buffer_record_disable(buffer); 2085 #endif 2086 2087 arch_spin_unlock(&global_trace.max_lock); 2088 2089 out: 2090 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags); 2091 } 2092 2093 static void tracing_stop_tr(struct trace_array *tr) 2094 { 2095 struct ring_buffer *buffer; 2096 unsigned long flags; 2097 2098 /* If global, we need to also stop the max tracer */ 2099 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 2100 return tracing_stop(); 2101 2102 raw_spin_lock_irqsave(&tr->start_lock, flags); 2103 if (tr->stop_count++) 2104 goto out; 2105 2106 buffer = tr->trace_buffer.buffer; 2107 if (buffer) 2108 ring_buffer_record_disable(buffer); 2109 2110 out: 2111 raw_spin_unlock_irqrestore(&tr->start_lock, flags); 2112 } 2113 2114 static int trace_save_cmdline(struct task_struct *tsk) 2115 { 2116 unsigned pid, idx; 2117 2118 /* treat recording of idle task as a success */ 2119 if (!tsk->pid) 2120 return 1; 2121 2122 if (unlikely(tsk->pid > PID_MAX_DEFAULT)) 2123 return 0; 2124 2125 /* 2126 * It's not the end of the world if we don't get 2127 * the lock, but we also don't want to spin 2128 * nor do we want to disable interrupts, 2129 * so if we miss here, then better luck next time. 2130 */ 2131 if (!arch_spin_trylock(&trace_cmdline_lock)) 2132 return 0; 2133 2134 idx = savedcmd->map_pid_to_cmdline[tsk->pid]; 2135 if (idx == NO_CMDLINE_MAP) { 2136 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num; 2137 2138 /* 2139 * Check whether the cmdline buffer at idx has a pid 2140 * mapped. We are going to overwrite that entry so we 2141 * need to clear the map_pid_to_cmdline. Otherwise we 2142 * would read the new comm for the old pid. 2143 */ 2144 pid = savedcmd->map_cmdline_to_pid[idx]; 2145 if (pid != NO_CMDLINE_MAP) 2146 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP; 2147 2148 savedcmd->map_cmdline_to_pid[idx] = tsk->pid; 2149 savedcmd->map_pid_to_cmdline[tsk->pid] = idx; 2150 2151 savedcmd->cmdline_idx = idx; 2152 } 2153 2154 set_cmdline(idx, tsk->comm); 2155 2156 arch_spin_unlock(&trace_cmdline_lock); 2157 2158 return 1; 2159 } 2160 2161 static void __trace_find_cmdline(int pid, char comm[]) 2162 { 2163 unsigned map; 2164 2165 if (!pid) { 2166 strcpy(comm, "<idle>"); 2167 return; 2168 } 2169 2170 if (WARN_ON_ONCE(pid < 0)) { 2171 strcpy(comm, "<XXX>"); 2172 return; 2173 } 2174 2175 if (pid > PID_MAX_DEFAULT) { 2176 strcpy(comm, "<...>"); 2177 return; 2178 } 2179 2180 map = savedcmd->map_pid_to_cmdline[pid]; 2181 if (map != NO_CMDLINE_MAP) 2182 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN); 2183 else 2184 strcpy(comm, "<...>"); 2185 } 2186 2187 void trace_find_cmdline(int pid, char comm[]) 2188 { 2189 preempt_disable(); 2190 arch_spin_lock(&trace_cmdline_lock); 2191 2192 __trace_find_cmdline(pid, comm); 2193 2194 arch_spin_unlock(&trace_cmdline_lock); 2195 preempt_enable(); 2196 } 2197 2198 int trace_find_tgid(int pid) 2199 { 2200 if (unlikely(!tgid_map || !pid || pid > PID_MAX_DEFAULT)) 2201 return 0; 2202 2203 return tgid_map[pid]; 2204 } 2205 2206 static int trace_save_tgid(struct task_struct *tsk) 2207 { 2208 /* treat recording of idle task as a success */ 2209 if (!tsk->pid) 2210 return 1; 2211 2212 if (unlikely(!tgid_map || tsk->pid > PID_MAX_DEFAULT)) 2213 return 0; 2214 2215 tgid_map[tsk->pid] = tsk->tgid; 2216 return 1; 2217 } 2218 2219 static bool tracing_record_taskinfo_skip(int flags) 2220 { 2221 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID)))) 2222 return true; 2223 if (atomic_read(&trace_record_taskinfo_disabled) || !tracing_is_on()) 2224 return true; 2225 if (!__this_cpu_read(trace_taskinfo_save)) 2226 return true; 2227 return false; 2228 } 2229 2230 /** 2231 * tracing_record_taskinfo - record the task info of a task 2232 * 2233 * @task: task to record 2234 * @flags: TRACE_RECORD_CMDLINE for recording comm 2235 * TRACE_RECORD_TGID for recording tgid 2236 */ 2237 void tracing_record_taskinfo(struct task_struct *task, int flags) 2238 { 2239 bool done; 2240 2241 if (tracing_record_taskinfo_skip(flags)) 2242 return; 2243 2244 /* 2245 * Record as much task information as possible. If some fail, continue 2246 * to try to record the others. 2247 */ 2248 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task); 2249 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task); 2250 2251 /* If recording any information failed, retry again soon. */ 2252 if (!done) 2253 return; 2254 2255 __this_cpu_write(trace_taskinfo_save, false); 2256 } 2257 2258 /** 2259 * tracing_record_taskinfo_sched_switch - record task info for sched_switch 2260 * 2261 * @prev: previous task during sched_switch 2262 * @next: next task during sched_switch 2263 * @flags: TRACE_RECORD_CMDLINE for recording comm 2264 * TRACE_RECORD_TGID for recording tgid 2265 */ 2266 void tracing_record_taskinfo_sched_switch(struct task_struct *prev, 2267 struct task_struct *next, int flags) 2268 { 2269 bool done; 2270 2271 if (tracing_record_taskinfo_skip(flags)) 2272 return; 2273 2274 /* 2275 * Record as much task information as possible. If some fail, continue 2276 * to try to record the others. 2277 */ 2278 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev); 2279 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next); 2280 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev); 2281 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next); 2282 2283 /* If recording any information failed, retry again soon. */ 2284 if (!done) 2285 return; 2286 2287 __this_cpu_write(trace_taskinfo_save, false); 2288 } 2289 2290 /* Helpers to record a specific task information */ 2291 void tracing_record_cmdline(struct task_struct *task) 2292 { 2293 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE); 2294 } 2295 2296 void tracing_record_tgid(struct task_struct *task) 2297 { 2298 tracing_record_taskinfo(task, TRACE_RECORD_TGID); 2299 } 2300 2301 /* 2302 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq 2303 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function 2304 * simplifies those functions and keeps them in sync. 2305 */ 2306 enum print_line_t trace_handle_return(struct trace_seq *s) 2307 { 2308 return trace_seq_has_overflowed(s) ? 2309 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED; 2310 } 2311 EXPORT_SYMBOL_GPL(trace_handle_return); 2312 2313 void 2314 tracing_generic_entry_update(struct trace_entry *entry, unsigned short type, 2315 unsigned long flags, int pc) 2316 { 2317 struct task_struct *tsk = current; 2318 2319 entry->preempt_count = pc & 0xff; 2320 entry->pid = (tsk) ? tsk->pid : 0; 2321 entry->type = type; 2322 entry->flags = 2323 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT 2324 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) | 2325 #else 2326 TRACE_FLAG_IRQS_NOSUPPORT | 2327 #endif 2328 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) | 2329 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) | 2330 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) | 2331 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) | 2332 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0); 2333 } 2334 EXPORT_SYMBOL_GPL(tracing_generic_entry_update); 2335 2336 struct ring_buffer_event * 2337 trace_buffer_lock_reserve(struct ring_buffer *buffer, 2338 int type, 2339 unsigned long len, 2340 unsigned long flags, int pc) 2341 { 2342 return __trace_buffer_lock_reserve(buffer, type, len, flags, pc); 2343 } 2344 2345 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event); 2346 DEFINE_PER_CPU(int, trace_buffered_event_cnt); 2347 static int trace_buffered_event_ref; 2348 2349 /** 2350 * trace_buffered_event_enable - enable buffering events 2351 * 2352 * When events are being filtered, it is quicker to use a temporary 2353 * buffer to write the event data into if there's a likely chance 2354 * that it will not be committed. The discard of the ring buffer 2355 * is not as fast as committing, and is much slower than copying 2356 * a commit. 2357 * 2358 * When an event is to be filtered, allocate per cpu buffers to 2359 * write the event data into, and if the event is filtered and discarded 2360 * it is simply dropped, otherwise, the entire data is to be committed 2361 * in one shot. 2362 */ 2363 void trace_buffered_event_enable(void) 2364 { 2365 struct ring_buffer_event *event; 2366 struct page *page; 2367 int cpu; 2368 2369 WARN_ON_ONCE(!mutex_is_locked(&event_mutex)); 2370 2371 if (trace_buffered_event_ref++) 2372 return; 2373 2374 for_each_tracing_cpu(cpu) { 2375 page = alloc_pages_node(cpu_to_node(cpu), 2376 GFP_KERNEL | __GFP_NORETRY, 0); 2377 if (!page) 2378 goto failed; 2379 2380 event = page_address(page); 2381 memset(event, 0, sizeof(*event)); 2382 2383 per_cpu(trace_buffered_event, cpu) = event; 2384 2385 preempt_disable(); 2386 if (cpu == smp_processor_id() && 2387 this_cpu_read(trace_buffered_event) != 2388 per_cpu(trace_buffered_event, cpu)) 2389 WARN_ON_ONCE(1); 2390 preempt_enable(); 2391 } 2392 2393 return; 2394 failed: 2395 trace_buffered_event_disable(); 2396 } 2397 2398 static void enable_trace_buffered_event(void *data) 2399 { 2400 /* Probably not needed, but do it anyway */ 2401 smp_rmb(); 2402 this_cpu_dec(trace_buffered_event_cnt); 2403 } 2404 2405 static void disable_trace_buffered_event(void *data) 2406 { 2407 this_cpu_inc(trace_buffered_event_cnt); 2408 } 2409 2410 /** 2411 * trace_buffered_event_disable - disable buffering events 2412 * 2413 * When a filter is removed, it is faster to not use the buffered 2414 * events, and to commit directly into the ring buffer. Free up 2415 * the temp buffers when there are no more users. This requires 2416 * special synchronization with current events. 2417 */ 2418 void trace_buffered_event_disable(void) 2419 { 2420 int cpu; 2421 2422 WARN_ON_ONCE(!mutex_is_locked(&event_mutex)); 2423 2424 if (WARN_ON_ONCE(!trace_buffered_event_ref)) 2425 return; 2426 2427 if (--trace_buffered_event_ref) 2428 return; 2429 2430 preempt_disable(); 2431 /* For each CPU, set the buffer as used. */ 2432 smp_call_function_many(tracing_buffer_mask, 2433 disable_trace_buffered_event, NULL, 1); 2434 preempt_enable(); 2435 2436 /* Wait for all current users to finish */ 2437 synchronize_rcu(); 2438 2439 for_each_tracing_cpu(cpu) { 2440 free_page((unsigned long)per_cpu(trace_buffered_event, cpu)); 2441 per_cpu(trace_buffered_event, cpu) = NULL; 2442 } 2443 /* 2444 * Make sure trace_buffered_event is NULL before clearing 2445 * trace_buffered_event_cnt. 2446 */ 2447 smp_wmb(); 2448 2449 preempt_disable(); 2450 /* Do the work on each cpu */ 2451 smp_call_function_many(tracing_buffer_mask, 2452 enable_trace_buffered_event, NULL, 1); 2453 preempt_enable(); 2454 } 2455 2456 static struct ring_buffer *temp_buffer; 2457 2458 struct ring_buffer_event * 2459 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb, 2460 struct trace_event_file *trace_file, 2461 int type, unsigned long len, 2462 unsigned long flags, int pc) 2463 { 2464 struct ring_buffer_event *entry; 2465 int val; 2466 2467 *current_rb = trace_file->tr->trace_buffer.buffer; 2468 2469 if (!ring_buffer_time_stamp_abs(*current_rb) && (trace_file->flags & 2470 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) && 2471 (entry = this_cpu_read(trace_buffered_event))) { 2472 /* Try to use the per cpu buffer first */ 2473 val = this_cpu_inc_return(trace_buffered_event_cnt); 2474 if (val == 1) { 2475 trace_event_setup(entry, type, flags, pc); 2476 entry->array[0] = len; 2477 return entry; 2478 } 2479 this_cpu_dec(trace_buffered_event_cnt); 2480 } 2481 2482 entry = __trace_buffer_lock_reserve(*current_rb, 2483 type, len, flags, pc); 2484 /* 2485 * If tracing is off, but we have triggers enabled 2486 * we still need to look at the event data. Use the temp_buffer 2487 * to store the trace event for the tigger to use. It's recusive 2488 * safe and will not be recorded anywhere. 2489 */ 2490 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) { 2491 *current_rb = temp_buffer; 2492 entry = __trace_buffer_lock_reserve(*current_rb, 2493 type, len, flags, pc); 2494 } 2495 return entry; 2496 } 2497 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve); 2498 2499 static DEFINE_SPINLOCK(tracepoint_iter_lock); 2500 static DEFINE_MUTEX(tracepoint_printk_mutex); 2501 2502 static void output_printk(struct trace_event_buffer *fbuffer) 2503 { 2504 struct trace_event_call *event_call; 2505 struct trace_event *event; 2506 unsigned long flags; 2507 struct trace_iterator *iter = tracepoint_print_iter; 2508 2509 /* We should never get here if iter is NULL */ 2510 if (WARN_ON_ONCE(!iter)) 2511 return; 2512 2513 event_call = fbuffer->trace_file->event_call; 2514 if (!event_call || !event_call->event.funcs || 2515 !event_call->event.funcs->trace) 2516 return; 2517 2518 event = &fbuffer->trace_file->event_call->event; 2519 2520 spin_lock_irqsave(&tracepoint_iter_lock, flags); 2521 trace_seq_init(&iter->seq); 2522 iter->ent = fbuffer->entry; 2523 event_call->event.funcs->trace(iter, 0, event); 2524 trace_seq_putc(&iter->seq, 0); 2525 printk("%s", iter->seq.buffer); 2526 2527 spin_unlock_irqrestore(&tracepoint_iter_lock, flags); 2528 } 2529 2530 int tracepoint_printk_sysctl(struct ctl_table *table, int write, 2531 void __user *buffer, size_t *lenp, 2532 loff_t *ppos) 2533 { 2534 int save_tracepoint_printk; 2535 int ret; 2536 2537 mutex_lock(&tracepoint_printk_mutex); 2538 save_tracepoint_printk = tracepoint_printk; 2539 2540 ret = proc_dointvec(table, write, buffer, lenp, ppos); 2541 2542 /* 2543 * This will force exiting early, as tracepoint_printk 2544 * is always zero when tracepoint_printk_iter is not allocated 2545 */ 2546 if (!tracepoint_print_iter) 2547 tracepoint_printk = 0; 2548 2549 if (save_tracepoint_printk == tracepoint_printk) 2550 goto out; 2551 2552 if (tracepoint_printk) 2553 static_key_enable(&tracepoint_printk_key.key); 2554 else 2555 static_key_disable(&tracepoint_printk_key.key); 2556 2557 out: 2558 mutex_unlock(&tracepoint_printk_mutex); 2559 2560 return ret; 2561 } 2562 2563 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer) 2564 { 2565 if (static_key_false(&tracepoint_printk_key.key)) 2566 output_printk(fbuffer); 2567 2568 event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer, 2569 fbuffer->event, fbuffer->entry, 2570 fbuffer->flags, fbuffer->pc); 2571 } 2572 EXPORT_SYMBOL_GPL(trace_event_buffer_commit); 2573 2574 /* 2575 * Skip 3: 2576 * 2577 * trace_buffer_unlock_commit_regs() 2578 * trace_event_buffer_commit() 2579 * trace_event_raw_event_xxx() 2580 */ 2581 # define STACK_SKIP 3 2582 2583 void trace_buffer_unlock_commit_regs(struct trace_array *tr, 2584 struct ring_buffer *buffer, 2585 struct ring_buffer_event *event, 2586 unsigned long flags, int pc, 2587 struct pt_regs *regs) 2588 { 2589 __buffer_unlock_commit(buffer, event); 2590 2591 /* 2592 * If regs is not set, then skip the necessary functions. 2593 * Note, we can still get here via blktrace, wakeup tracer 2594 * and mmiotrace, but that's ok if they lose a function or 2595 * two. They are not that meaningful. 2596 */ 2597 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : STACK_SKIP, pc, regs); 2598 ftrace_trace_userstack(buffer, flags, pc); 2599 } 2600 2601 /* 2602 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack. 2603 */ 2604 void 2605 trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer, 2606 struct ring_buffer_event *event) 2607 { 2608 __buffer_unlock_commit(buffer, event); 2609 } 2610 2611 static void 2612 trace_process_export(struct trace_export *export, 2613 struct ring_buffer_event *event) 2614 { 2615 struct trace_entry *entry; 2616 unsigned int size = 0; 2617 2618 entry = ring_buffer_event_data(event); 2619 size = ring_buffer_event_length(event); 2620 export->write(export, entry, size); 2621 } 2622 2623 static DEFINE_MUTEX(ftrace_export_lock); 2624 2625 static struct trace_export __rcu *ftrace_exports_list __read_mostly; 2626 2627 static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled); 2628 2629 static inline void ftrace_exports_enable(void) 2630 { 2631 static_branch_enable(&ftrace_exports_enabled); 2632 } 2633 2634 static inline void ftrace_exports_disable(void) 2635 { 2636 static_branch_disable(&ftrace_exports_enabled); 2637 } 2638 2639 static void ftrace_exports(struct ring_buffer_event *event) 2640 { 2641 struct trace_export *export; 2642 2643 preempt_disable_notrace(); 2644 2645 export = rcu_dereference_raw_notrace(ftrace_exports_list); 2646 while (export) { 2647 trace_process_export(export, event); 2648 export = rcu_dereference_raw_notrace(export->next); 2649 } 2650 2651 preempt_enable_notrace(); 2652 } 2653 2654 static inline void 2655 add_trace_export(struct trace_export **list, struct trace_export *export) 2656 { 2657 rcu_assign_pointer(export->next, *list); 2658 /* 2659 * We are entering export into the list but another 2660 * CPU might be walking that list. We need to make sure 2661 * the export->next pointer is valid before another CPU sees 2662 * the export pointer included into the list. 2663 */ 2664 rcu_assign_pointer(*list, export); 2665 } 2666 2667 static inline int 2668 rm_trace_export(struct trace_export **list, struct trace_export *export) 2669 { 2670 struct trace_export **p; 2671 2672 for (p = list; *p != NULL; p = &(*p)->next) 2673 if (*p == export) 2674 break; 2675 2676 if (*p != export) 2677 return -1; 2678 2679 rcu_assign_pointer(*p, (*p)->next); 2680 2681 return 0; 2682 } 2683 2684 static inline void 2685 add_ftrace_export(struct trace_export **list, struct trace_export *export) 2686 { 2687 if (*list == NULL) 2688 ftrace_exports_enable(); 2689 2690 add_trace_export(list, export); 2691 } 2692 2693 static inline int 2694 rm_ftrace_export(struct trace_export **list, struct trace_export *export) 2695 { 2696 int ret; 2697 2698 ret = rm_trace_export(list, export); 2699 if (*list == NULL) 2700 ftrace_exports_disable(); 2701 2702 return ret; 2703 } 2704 2705 int register_ftrace_export(struct trace_export *export) 2706 { 2707 if (WARN_ON_ONCE(!export->write)) 2708 return -1; 2709 2710 mutex_lock(&ftrace_export_lock); 2711 2712 add_ftrace_export(&ftrace_exports_list, export); 2713 2714 mutex_unlock(&ftrace_export_lock); 2715 2716 return 0; 2717 } 2718 EXPORT_SYMBOL_GPL(register_ftrace_export); 2719 2720 int unregister_ftrace_export(struct trace_export *export) 2721 { 2722 int ret; 2723 2724 mutex_lock(&ftrace_export_lock); 2725 2726 ret = rm_ftrace_export(&ftrace_exports_list, export); 2727 2728 mutex_unlock(&ftrace_export_lock); 2729 2730 return ret; 2731 } 2732 EXPORT_SYMBOL_GPL(unregister_ftrace_export); 2733 2734 void 2735 trace_function(struct trace_array *tr, 2736 unsigned long ip, unsigned long parent_ip, unsigned long flags, 2737 int pc) 2738 { 2739 struct trace_event_call *call = &event_function; 2740 struct ring_buffer *buffer = tr->trace_buffer.buffer; 2741 struct ring_buffer_event *event; 2742 struct ftrace_entry *entry; 2743 2744 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), 2745 flags, pc); 2746 if (!event) 2747 return; 2748 entry = ring_buffer_event_data(event); 2749 entry->ip = ip; 2750 entry->parent_ip = parent_ip; 2751 2752 if (!call_filter_check_discard(call, entry, buffer, event)) { 2753 if (static_branch_unlikely(&ftrace_exports_enabled)) 2754 ftrace_exports(event); 2755 __buffer_unlock_commit(buffer, event); 2756 } 2757 } 2758 2759 #ifdef CONFIG_STACKTRACE 2760 2761 /* Allow 4 levels of nesting: normal, softirq, irq, NMI */ 2762 #define FTRACE_KSTACK_NESTING 4 2763 2764 #define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING) 2765 2766 struct ftrace_stack { 2767 unsigned long calls[FTRACE_KSTACK_ENTRIES]; 2768 }; 2769 2770 2771 struct ftrace_stacks { 2772 struct ftrace_stack stacks[FTRACE_KSTACK_NESTING]; 2773 }; 2774 2775 static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks); 2776 static DEFINE_PER_CPU(int, ftrace_stack_reserve); 2777 2778 static void __ftrace_trace_stack(struct ring_buffer *buffer, 2779 unsigned long flags, 2780 int skip, int pc, struct pt_regs *regs) 2781 { 2782 struct trace_event_call *call = &event_kernel_stack; 2783 struct ring_buffer_event *event; 2784 unsigned int size, nr_entries; 2785 struct ftrace_stack *fstack; 2786 struct stack_entry *entry; 2787 int stackidx; 2788 2789 /* 2790 * Add one, for this function and the call to save_stack_trace() 2791 * If regs is set, then these functions will not be in the way. 2792 */ 2793 #ifndef CONFIG_UNWINDER_ORC 2794 if (!regs) 2795 skip++; 2796 #endif 2797 2798 /* 2799 * Since events can happen in NMIs there's no safe way to 2800 * use the per cpu ftrace_stacks. We reserve it and if an interrupt 2801 * or NMI comes in, it will just have to use the default 2802 * FTRACE_STACK_SIZE. 2803 */ 2804 preempt_disable_notrace(); 2805 2806 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1; 2807 2808 /* This should never happen. If it does, yell once and skip */ 2809 if (WARN_ON_ONCE(stackidx > FTRACE_KSTACK_NESTING)) 2810 goto out; 2811 2812 /* 2813 * The above __this_cpu_inc_return() is 'atomic' cpu local. An 2814 * interrupt will either see the value pre increment or post 2815 * increment. If the interrupt happens pre increment it will have 2816 * restored the counter when it returns. We just need a barrier to 2817 * keep gcc from moving things around. 2818 */ 2819 barrier(); 2820 2821 fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx; 2822 size = ARRAY_SIZE(fstack->calls); 2823 2824 if (regs) { 2825 nr_entries = stack_trace_save_regs(regs, fstack->calls, 2826 size, skip); 2827 } else { 2828 nr_entries = stack_trace_save(fstack->calls, size, skip); 2829 } 2830 2831 size = nr_entries * sizeof(unsigned long); 2832 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK, 2833 sizeof(*entry) + size, flags, pc); 2834 if (!event) 2835 goto out; 2836 entry = ring_buffer_event_data(event); 2837 2838 memcpy(&entry->caller, fstack->calls, size); 2839 entry->size = nr_entries; 2840 2841 if (!call_filter_check_discard(call, entry, buffer, event)) 2842 __buffer_unlock_commit(buffer, event); 2843 2844 out: 2845 /* Again, don't let gcc optimize things here */ 2846 barrier(); 2847 __this_cpu_dec(ftrace_stack_reserve); 2848 preempt_enable_notrace(); 2849 2850 } 2851 2852 static inline void ftrace_trace_stack(struct trace_array *tr, 2853 struct ring_buffer *buffer, 2854 unsigned long flags, 2855 int skip, int pc, struct pt_regs *regs) 2856 { 2857 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE)) 2858 return; 2859 2860 __ftrace_trace_stack(buffer, flags, skip, pc, regs); 2861 } 2862 2863 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, 2864 int pc) 2865 { 2866 struct ring_buffer *buffer = tr->trace_buffer.buffer; 2867 2868 if (rcu_is_watching()) { 2869 __ftrace_trace_stack(buffer, flags, skip, pc, NULL); 2870 return; 2871 } 2872 2873 /* 2874 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(), 2875 * but if the above rcu_is_watching() failed, then the NMI 2876 * triggered someplace critical, and rcu_irq_enter() should 2877 * not be called from NMI. 2878 */ 2879 if (unlikely(in_nmi())) 2880 return; 2881 2882 rcu_irq_enter_irqson(); 2883 __ftrace_trace_stack(buffer, flags, skip, pc, NULL); 2884 rcu_irq_exit_irqson(); 2885 } 2886 2887 /** 2888 * trace_dump_stack - record a stack back trace in the trace buffer 2889 * @skip: Number of functions to skip (helper handlers) 2890 */ 2891 void trace_dump_stack(int skip) 2892 { 2893 unsigned long flags; 2894 2895 if (tracing_disabled || tracing_selftest_running) 2896 return; 2897 2898 local_save_flags(flags); 2899 2900 #ifndef CONFIG_UNWINDER_ORC 2901 /* Skip 1 to skip this function. */ 2902 skip++; 2903 #endif 2904 __ftrace_trace_stack(global_trace.trace_buffer.buffer, 2905 flags, skip, preempt_count(), NULL); 2906 } 2907 EXPORT_SYMBOL_GPL(trace_dump_stack); 2908 2909 #ifdef CONFIG_USER_STACKTRACE_SUPPORT 2910 static DEFINE_PER_CPU(int, user_stack_count); 2911 2912 static void 2913 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc) 2914 { 2915 struct trace_event_call *call = &event_user_stack; 2916 struct ring_buffer_event *event; 2917 struct userstack_entry *entry; 2918 2919 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE)) 2920 return; 2921 2922 /* 2923 * NMIs can not handle page faults, even with fix ups. 2924 * The save user stack can (and often does) fault. 2925 */ 2926 if (unlikely(in_nmi())) 2927 return; 2928 2929 /* 2930 * prevent recursion, since the user stack tracing may 2931 * trigger other kernel events. 2932 */ 2933 preempt_disable(); 2934 if (__this_cpu_read(user_stack_count)) 2935 goto out; 2936 2937 __this_cpu_inc(user_stack_count); 2938 2939 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, 2940 sizeof(*entry), flags, pc); 2941 if (!event) 2942 goto out_drop_count; 2943 entry = ring_buffer_event_data(event); 2944 2945 entry->tgid = current->tgid; 2946 memset(&entry->caller, 0, sizeof(entry->caller)); 2947 2948 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES); 2949 if (!call_filter_check_discard(call, entry, buffer, event)) 2950 __buffer_unlock_commit(buffer, event); 2951 2952 out_drop_count: 2953 __this_cpu_dec(user_stack_count); 2954 out: 2955 preempt_enable(); 2956 } 2957 #else /* CONFIG_USER_STACKTRACE_SUPPORT */ 2958 static void ftrace_trace_userstack(struct ring_buffer *buffer, 2959 unsigned long flags, int pc) 2960 { 2961 } 2962 #endif /* !CONFIG_USER_STACKTRACE_SUPPORT */ 2963 2964 #endif /* CONFIG_STACKTRACE */ 2965 2966 /* created for use with alloc_percpu */ 2967 struct trace_buffer_struct { 2968 int nesting; 2969 char buffer[4][TRACE_BUF_SIZE]; 2970 }; 2971 2972 static struct trace_buffer_struct *trace_percpu_buffer; 2973 2974 /* 2975 * Thise allows for lockless recording. If we're nested too deeply, then 2976 * this returns NULL. 2977 */ 2978 static char *get_trace_buf(void) 2979 { 2980 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer); 2981 2982 if (!buffer || buffer->nesting >= 4) 2983 return NULL; 2984 2985 buffer->nesting++; 2986 2987 /* Interrupts must see nesting incremented before we use the buffer */ 2988 barrier(); 2989 return &buffer->buffer[buffer->nesting][0]; 2990 } 2991 2992 static void put_trace_buf(void) 2993 { 2994 /* Don't let the decrement of nesting leak before this */ 2995 barrier(); 2996 this_cpu_dec(trace_percpu_buffer->nesting); 2997 } 2998 2999 static int alloc_percpu_trace_buffer(void) 3000 { 3001 struct trace_buffer_struct *buffers; 3002 3003 buffers = alloc_percpu(struct trace_buffer_struct); 3004 if (WARN(!buffers, "Could not allocate percpu trace_printk buffer")) 3005 return -ENOMEM; 3006 3007 trace_percpu_buffer = buffers; 3008 return 0; 3009 } 3010 3011 static int buffers_allocated; 3012 3013 void trace_printk_init_buffers(void) 3014 { 3015 if (buffers_allocated) 3016 return; 3017 3018 if (alloc_percpu_trace_buffer()) 3019 return; 3020 3021 /* trace_printk() is for debug use only. Don't use it in production. */ 3022 3023 pr_warn("\n"); 3024 pr_warn("**********************************************************\n"); 3025 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 3026 pr_warn("** **\n"); 3027 pr_warn("** trace_printk() being used. Allocating extra memory. **\n"); 3028 pr_warn("** **\n"); 3029 pr_warn("** This means that this is a DEBUG kernel and it is **\n"); 3030 pr_warn("** unsafe for production use. **\n"); 3031 pr_warn("** **\n"); 3032 pr_warn("** If you see this message and you are not debugging **\n"); 3033 pr_warn("** the kernel, report this immediately to your vendor! **\n"); 3034 pr_warn("** **\n"); 3035 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 3036 pr_warn("**********************************************************\n"); 3037 3038 /* Expand the buffers to set size */ 3039 tracing_update_buffers(); 3040 3041 buffers_allocated = 1; 3042 3043 /* 3044 * trace_printk_init_buffers() can be called by modules. 3045 * If that happens, then we need to start cmdline recording 3046 * directly here. If the global_trace.buffer is already 3047 * allocated here, then this was called by module code. 3048 */ 3049 if (global_trace.trace_buffer.buffer) 3050 tracing_start_cmdline_record(); 3051 } 3052 EXPORT_SYMBOL_GPL(trace_printk_init_buffers); 3053 3054 void trace_printk_start_comm(void) 3055 { 3056 /* Start tracing comms if trace printk is set */ 3057 if (!buffers_allocated) 3058 return; 3059 tracing_start_cmdline_record(); 3060 } 3061 3062 static void trace_printk_start_stop_comm(int enabled) 3063 { 3064 if (!buffers_allocated) 3065 return; 3066 3067 if (enabled) 3068 tracing_start_cmdline_record(); 3069 else 3070 tracing_stop_cmdline_record(); 3071 } 3072 3073 /** 3074 * trace_vbprintk - write binary msg to tracing buffer 3075 * @ip: The address of the caller 3076 * @fmt: The string format to write to the buffer 3077 * @args: Arguments for @fmt 3078 */ 3079 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) 3080 { 3081 struct trace_event_call *call = &event_bprint; 3082 struct ring_buffer_event *event; 3083 struct ring_buffer *buffer; 3084 struct trace_array *tr = &global_trace; 3085 struct bprint_entry *entry; 3086 unsigned long flags; 3087 char *tbuffer; 3088 int len = 0, size, pc; 3089 3090 if (unlikely(tracing_selftest_running || tracing_disabled)) 3091 return 0; 3092 3093 /* Don't pollute graph traces with trace_vprintk internals */ 3094 pause_graph_tracing(); 3095 3096 pc = preempt_count(); 3097 preempt_disable_notrace(); 3098 3099 tbuffer = get_trace_buf(); 3100 if (!tbuffer) { 3101 len = 0; 3102 goto out_nobuffer; 3103 } 3104 3105 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args); 3106 3107 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0) 3108 goto out; 3109 3110 local_save_flags(flags); 3111 size = sizeof(*entry) + sizeof(u32) * len; 3112 buffer = tr->trace_buffer.buffer; 3113 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, 3114 flags, pc); 3115 if (!event) 3116 goto out; 3117 entry = ring_buffer_event_data(event); 3118 entry->ip = ip; 3119 entry->fmt = fmt; 3120 3121 memcpy(entry->buf, tbuffer, sizeof(u32) * len); 3122 if (!call_filter_check_discard(call, entry, buffer, event)) { 3123 __buffer_unlock_commit(buffer, event); 3124 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL); 3125 } 3126 3127 out: 3128 put_trace_buf(); 3129 3130 out_nobuffer: 3131 preempt_enable_notrace(); 3132 unpause_graph_tracing(); 3133 3134 return len; 3135 } 3136 EXPORT_SYMBOL_GPL(trace_vbprintk); 3137 3138 __printf(3, 0) 3139 static int 3140 __trace_array_vprintk(struct ring_buffer *buffer, 3141 unsigned long ip, const char *fmt, va_list args) 3142 { 3143 struct trace_event_call *call = &event_print; 3144 struct ring_buffer_event *event; 3145 int len = 0, size, pc; 3146 struct print_entry *entry; 3147 unsigned long flags; 3148 char *tbuffer; 3149 3150 if (tracing_disabled || tracing_selftest_running) 3151 return 0; 3152 3153 /* Don't pollute graph traces with trace_vprintk internals */ 3154 pause_graph_tracing(); 3155 3156 pc = preempt_count(); 3157 preempt_disable_notrace(); 3158 3159 3160 tbuffer = get_trace_buf(); 3161 if (!tbuffer) { 3162 len = 0; 3163 goto out_nobuffer; 3164 } 3165 3166 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args); 3167 3168 local_save_flags(flags); 3169 size = sizeof(*entry) + len + 1; 3170 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 3171 flags, pc); 3172 if (!event) 3173 goto out; 3174 entry = ring_buffer_event_data(event); 3175 entry->ip = ip; 3176 3177 memcpy(&entry->buf, tbuffer, len + 1); 3178 if (!call_filter_check_discard(call, entry, buffer, event)) { 3179 __buffer_unlock_commit(buffer, event); 3180 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL); 3181 } 3182 3183 out: 3184 put_trace_buf(); 3185 3186 out_nobuffer: 3187 preempt_enable_notrace(); 3188 unpause_graph_tracing(); 3189 3190 return len; 3191 } 3192 3193 __printf(3, 0) 3194 int trace_array_vprintk(struct trace_array *tr, 3195 unsigned long ip, const char *fmt, va_list args) 3196 { 3197 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args); 3198 } 3199 3200 __printf(3, 0) 3201 int trace_array_printk(struct trace_array *tr, 3202 unsigned long ip, const char *fmt, ...) 3203 { 3204 int ret; 3205 va_list ap; 3206 3207 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 3208 return 0; 3209 3210 va_start(ap, fmt); 3211 ret = trace_array_vprintk(tr, ip, fmt, ap); 3212 va_end(ap); 3213 return ret; 3214 } 3215 EXPORT_SYMBOL_GPL(trace_array_printk); 3216 3217 __printf(3, 4) 3218 int trace_array_printk_buf(struct ring_buffer *buffer, 3219 unsigned long ip, const char *fmt, ...) 3220 { 3221 int ret; 3222 va_list ap; 3223 3224 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 3225 return 0; 3226 3227 va_start(ap, fmt); 3228 ret = __trace_array_vprintk(buffer, ip, fmt, ap); 3229 va_end(ap); 3230 return ret; 3231 } 3232 3233 __printf(2, 0) 3234 int trace_vprintk(unsigned long ip, const char *fmt, va_list args) 3235 { 3236 return trace_array_vprintk(&global_trace, ip, fmt, args); 3237 } 3238 EXPORT_SYMBOL_GPL(trace_vprintk); 3239 3240 static void trace_iterator_increment(struct trace_iterator *iter) 3241 { 3242 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu); 3243 3244 iter->idx++; 3245 if (buf_iter) 3246 ring_buffer_read(buf_iter, NULL); 3247 } 3248 3249 static struct trace_entry * 3250 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts, 3251 unsigned long *lost_events) 3252 { 3253 struct ring_buffer_event *event; 3254 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu); 3255 3256 if (buf_iter) 3257 event = ring_buffer_iter_peek(buf_iter, ts); 3258 else 3259 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts, 3260 lost_events); 3261 3262 if (event) { 3263 iter->ent_size = ring_buffer_event_length(event); 3264 return ring_buffer_event_data(event); 3265 } 3266 iter->ent_size = 0; 3267 return NULL; 3268 } 3269 3270 static struct trace_entry * 3271 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, 3272 unsigned long *missing_events, u64 *ent_ts) 3273 { 3274 struct ring_buffer *buffer = iter->trace_buffer->buffer; 3275 struct trace_entry *ent, *next = NULL; 3276 unsigned long lost_events = 0, next_lost = 0; 3277 int cpu_file = iter->cpu_file; 3278 u64 next_ts = 0, ts; 3279 int next_cpu = -1; 3280 int next_size = 0; 3281 int cpu; 3282 3283 /* 3284 * If we are in a per_cpu trace file, don't bother by iterating over 3285 * all cpu and peek directly. 3286 */ 3287 if (cpu_file > RING_BUFFER_ALL_CPUS) { 3288 if (ring_buffer_empty_cpu(buffer, cpu_file)) 3289 return NULL; 3290 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events); 3291 if (ent_cpu) 3292 *ent_cpu = cpu_file; 3293 3294 return ent; 3295 } 3296 3297 for_each_tracing_cpu(cpu) { 3298 3299 if (ring_buffer_empty_cpu(buffer, cpu)) 3300 continue; 3301 3302 ent = peek_next_entry(iter, cpu, &ts, &lost_events); 3303 3304 /* 3305 * Pick the entry with the smallest timestamp: 3306 */ 3307 if (ent && (!next || ts < next_ts)) { 3308 next = ent; 3309 next_cpu = cpu; 3310 next_ts = ts; 3311 next_lost = lost_events; 3312 next_size = iter->ent_size; 3313 } 3314 } 3315 3316 iter->ent_size = next_size; 3317 3318 if (ent_cpu) 3319 *ent_cpu = next_cpu; 3320 3321 if (ent_ts) 3322 *ent_ts = next_ts; 3323 3324 if (missing_events) 3325 *missing_events = next_lost; 3326 3327 return next; 3328 } 3329 3330 /* Find the next real entry, without updating the iterator itself */ 3331 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter, 3332 int *ent_cpu, u64 *ent_ts) 3333 { 3334 return __find_next_entry(iter, ent_cpu, NULL, ent_ts); 3335 } 3336 3337 /* Find the next real entry, and increment the iterator to the next entry */ 3338 void *trace_find_next_entry_inc(struct trace_iterator *iter) 3339 { 3340 iter->ent = __find_next_entry(iter, &iter->cpu, 3341 &iter->lost_events, &iter->ts); 3342 3343 if (iter->ent) 3344 trace_iterator_increment(iter); 3345 3346 return iter->ent ? iter : NULL; 3347 } 3348 3349 static void trace_consume(struct trace_iterator *iter) 3350 { 3351 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts, 3352 &iter->lost_events); 3353 } 3354 3355 static void *s_next(struct seq_file *m, void *v, loff_t *pos) 3356 { 3357 struct trace_iterator *iter = m->private; 3358 int i = (int)*pos; 3359 void *ent; 3360 3361 WARN_ON_ONCE(iter->leftover); 3362 3363 (*pos)++; 3364 3365 /* can't go backwards */ 3366 if (iter->idx > i) 3367 return NULL; 3368 3369 if (iter->idx < 0) 3370 ent = trace_find_next_entry_inc(iter); 3371 else 3372 ent = iter; 3373 3374 while (ent && iter->idx < i) 3375 ent = trace_find_next_entry_inc(iter); 3376 3377 iter->pos = *pos; 3378 3379 return ent; 3380 } 3381 3382 void tracing_iter_reset(struct trace_iterator *iter, int cpu) 3383 { 3384 struct ring_buffer_event *event; 3385 struct ring_buffer_iter *buf_iter; 3386 unsigned long entries = 0; 3387 u64 ts; 3388 3389 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0; 3390 3391 buf_iter = trace_buffer_iter(iter, cpu); 3392 if (!buf_iter) 3393 return; 3394 3395 ring_buffer_iter_reset(buf_iter); 3396 3397 /* 3398 * We could have the case with the max latency tracers 3399 * that a reset never took place on a cpu. This is evident 3400 * by the timestamp being before the start of the buffer. 3401 */ 3402 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) { 3403 if (ts >= iter->trace_buffer->time_start) 3404 break; 3405 entries++; 3406 ring_buffer_read(buf_iter, NULL); 3407 } 3408 3409 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries; 3410 } 3411 3412 /* 3413 * The current tracer is copied to avoid a global locking 3414 * all around. 3415 */ 3416 static void *s_start(struct seq_file *m, loff_t *pos) 3417 { 3418 struct trace_iterator *iter = m->private; 3419 struct trace_array *tr = iter->tr; 3420 int cpu_file = iter->cpu_file; 3421 void *p = NULL; 3422 loff_t l = 0; 3423 int cpu; 3424 3425 /* 3426 * copy the tracer to avoid using a global lock all around. 3427 * iter->trace is a copy of current_trace, the pointer to the 3428 * name may be used instead of a strcmp(), as iter->trace->name 3429 * will point to the same string as current_trace->name. 3430 */ 3431 mutex_lock(&trace_types_lock); 3432 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name)) 3433 *iter->trace = *tr->current_trace; 3434 mutex_unlock(&trace_types_lock); 3435 3436 #ifdef CONFIG_TRACER_MAX_TRACE 3437 if (iter->snapshot && iter->trace->use_max_tr) 3438 return ERR_PTR(-EBUSY); 3439 #endif 3440 3441 if (!iter->snapshot) 3442 atomic_inc(&trace_record_taskinfo_disabled); 3443 3444 if (*pos != iter->pos) { 3445 iter->ent = NULL; 3446 iter->cpu = 0; 3447 iter->idx = -1; 3448 3449 if (cpu_file == RING_BUFFER_ALL_CPUS) { 3450 for_each_tracing_cpu(cpu) 3451 tracing_iter_reset(iter, cpu); 3452 } else 3453 tracing_iter_reset(iter, cpu_file); 3454 3455 iter->leftover = 0; 3456 for (p = iter; p && l < *pos; p = s_next(m, p, &l)) 3457 ; 3458 3459 } else { 3460 /* 3461 * If we overflowed the seq_file before, then we want 3462 * to just reuse the trace_seq buffer again. 3463 */ 3464 if (iter->leftover) 3465 p = iter; 3466 else { 3467 l = *pos - 1; 3468 p = s_next(m, p, &l); 3469 } 3470 } 3471 3472 trace_event_read_lock(); 3473 trace_access_lock(cpu_file); 3474 return p; 3475 } 3476 3477 static void s_stop(struct seq_file *m, void *p) 3478 { 3479 struct trace_iterator *iter = m->private; 3480 3481 #ifdef CONFIG_TRACER_MAX_TRACE 3482 if (iter->snapshot && iter->trace->use_max_tr) 3483 return; 3484 #endif 3485 3486 if (!iter->snapshot) 3487 atomic_dec(&trace_record_taskinfo_disabled); 3488 3489 trace_access_unlock(iter->cpu_file); 3490 trace_event_read_unlock(); 3491 } 3492 3493 static void 3494 get_total_entries_cpu(struct trace_buffer *buf, unsigned long *total, 3495 unsigned long *entries, int cpu) 3496 { 3497 unsigned long count; 3498 3499 count = ring_buffer_entries_cpu(buf->buffer, cpu); 3500 /* 3501 * If this buffer has skipped entries, then we hold all 3502 * entries for the trace and we need to ignore the 3503 * ones before the time stamp. 3504 */ 3505 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) { 3506 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries; 3507 /* total is the same as the entries */ 3508 *total = count; 3509 } else 3510 *total = count + 3511 ring_buffer_overrun_cpu(buf->buffer, cpu); 3512 *entries = count; 3513 } 3514 3515 static void 3516 get_total_entries(struct trace_buffer *buf, 3517 unsigned long *total, unsigned long *entries) 3518 { 3519 unsigned long t, e; 3520 int cpu; 3521 3522 *total = 0; 3523 *entries = 0; 3524 3525 for_each_tracing_cpu(cpu) { 3526 get_total_entries_cpu(buf, &t, &e, cpu); 3527 *total += t; 3528 *entries += e; 3529 } 3530 } 3531 3532 unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu) 3533 { 3534 unsigned long total, entries; 3535 3536 if (!tr) 3537 tr = &global_trace; 3538 3539 get_total_entries_cpu(&tr->trace_buffer, &total, &entries, cpu); 3540 3541 return entries; 3542 } 3543 3544 unsigned long trace_total_entries(struct trace_array *tr) 3545 { 3546 unsigned long total, entries; 3547 3548 if (!tr) 3549 tr = &global_trace; 3550 3551 get_total_entries(&tr->trace_buffer, &total, &entries); 3552 3553 return entries; 3554 } 3555 3556 static void print_lat_help_header(struct seq_file *m) 3557 { 3558 seq_puts(m, "# _------=> CPU# \n" 3559 "# / _-----=> irqs-off \n" 3560 "# | / _----=> need-resched \n" 3561 "# || / _---=> hardirq/softirq \n" 3562 "# ||| / _--=> preempt-depth \n" 3563 "# |||| / delay \n" 3564 "# cmd pid ||||| time | caller \n" 3565 "# \\ / ||||| \\ | / \n"); 3566 } 3567 3568 static void print_event_info(struct trace_buffer *buf, struct seq_file *m) 3569 { 3570 unsigned long total; 3571 unsigned long entries; 3572 3573 get_total_entries(buf, &total, &entries); 3574 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n", 3575 entries, total, num_online_cpus()); 3576 seq_puts(m, "#\n"); 3577 } 3578 3579 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m, 3580 unsigned int flags) 3581 { 3582 bool tgid = flags & TRACE_ITER_RECORD_TGID; 3583 3584 print_event_info(buf, m); 3585 3586 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? "TGID " : ""); 3587 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : ""); 3588 } 3589 3590 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m, 3591 unsigned int flags) 3592 { 3593 bool tgid = flags & TRACE_ITER_RECORD_TGID; 3594 const char *space = " "; 3595 int prec = tgid ? 10 : 2; 3596 3597 print_event_info(buf, m); 3598 3599 seq_printf(m, "# %.*s _-----=> irqs-off\n", prec, space); 3600 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space); 3601 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space); 3602 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space); 3603 seq_printf(m, "# %.*s||| / delay\n", prec, space); 3604 seq_printf(m, "# TASK-PID %.*sCPU# |||| TIMESTAMP FUNCTION\n", prec, " TGID "); 3605 seq_printf(m, "# | | %.*s | |||| | |\n", prec, " | "); 3606 } 3607 3608 void 3609 print_trace_header(struct seq_file *m, struct trace_iterator *iter) 3610 { 3611 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK); 3612 struct trace_buffer *buf = iter->trace_buffer; 3613 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu); 3614 struct tracer *type = iter->trace; 3615 unsigned long entries; 3616 unsigned long total; 3617 const char *name = "preemption"; 3618 3619 name = type->name; 3620 3621 get_total_entries(buf, &total, &entries); 3622 3623 seq_printf(m, "# %s latency trace v1.1.5 on %s\n", 3624 name, UTS_RELEASE); 3625 seq_puts(m, "# -----------------------------------" 3626 "---------------------------------\n"); 3627 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |" 3628 " (M:%s VP:%d, KP:%d, SP:%d HP:%d", 3629 nsecs_to_usecs(data->saved_latency), 3630 entries, 3631 total, 3632 buf->cpu, 3633 #if defined(CONFIG_PREEMPT_NONE) 3634 "server", 3635 #elif defined(CONFIG_PREEMPT_VOLUNTARY) 3636 "desktop", 3637 #elif defined(CONFIG_PREEMPT) 3638 "preempt", 3639 #else 3640 "unknown", 3641 #endif 3642 /* These are reserved for later use */ 3643 0, 0, 0, 0); 3644 #ifdef CONFIG_SMP 3645 seq_printf(m, " #P:%d)\n", num_online_cpus()); 3646 #else 3647 seq_puts(m, ")\n"); 3648 #endif 3649 seq_puts(m, "# -----------------\n"); 3650 seq_printf(m, "# | task: %.16s-%d " 3651 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n", 3652 data->comm, data->pid, 3653 from_kuid_munged(seq_user_ns(m), data->uid), data->nice, 3654 data->policy, data->rt_priority); 3655 seq_puts(m, "# -----------------\n"); 3656 3657 if (data->critical_start) { 3658 seq_puts(m, "# => started at: "); 3659 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags); 3660 trace_print_seq(m, &iter->seq); 3661 seq_puts(m, "\n# => ended at: "); 3662 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); 3663 trace_print_seq(m, &iter->seq); 3664 seq_puts(m, "\n#\n"); 3665 } 3666 3667 seq_puts(m, "#\n"); 3668 } 3669 3670 static void test_cpu_buff_start(struct trace_iterator *iter) 3671 { 3672 struct trace_seq *s = &iter->seq; 3673 struct trace_array *tr = iter->tr; 3674 3675 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE)) 3676 return; 3677 3678 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE)) 3679 return; 3680 3681 if (cpumask_available(iter->started) && 3682 cpumask_test_cpu(iter->cpu, iter->started)) 3683 return; 3684 3685 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries) 3686 return; 3687 3688 if (cpumask_available(iter->started)) 3689 cpumask_set_cpu(iter->cpu, iter->started); 3690 3691 /* Don't print started cpu buffer for the first entry of the trace */ 3692 if (iter->idx > 1) 3693 trace_seq_printf(s, "##### CPU %u buffer started ####\n", 3694 iter->cpu); 3695 } 3696 3697 static enum print_line_t print_trace_fmt(struct trace_iterator *iter) 3698 { 3699 struct trace_array *tr = iter->tr; 3700 struct trace_seq *s = &iter->seq; 3701 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK); 3702 struct trace_entry *entry; 3703 struct trace_event *event; 3704 3705 entry = iter->ent; 3706 3707 test_cpu_buff_start(iter); 3708 3709 event = ftrace_find_event(entry->type); 3710 3711 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 3712 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 3713 trace_print_lat_context(iter); 3714 else 3715 trace_print_context(iter); 3716 } 3717 3718 if (trace_seq_has_overflowed(s)) 3719 return TRACE_TYPE_PARTIAL_LINE; 3720 3721 if (event) 3722 return event->funcs->trace(iter, sym_flags, event); 3723 3724 trace_seq_printf(s, "Unknown type %d\n", entry->type); 3725 3726 return trace_handle_return(s); 3727 } 3728 3729 static enum print_line_t print_raw_fmt(struct trace_iterator *iter) 3730 { 3731 struct trace_array *tr = iter->tr; 3732 struct trace_seq *s = &iter->seq; 3733 struct trace_entry *entry; 3734 struct trace_event *event; 3735 3736 entry = iter->ent; 3737 3738 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) 3739 trace_seq_printf(s, "%d %d %llu ", 3740 entry->pid, iter->cpu, iter->ts); 3741 3742 if (trace_seq_has_overflowed(s)) 3743 return TRACE_TYPE_PARTIAL_LINE; 3744 3745 event = ftrace_find_event(entry->type); 3746 if (event) 3747 return event->funcs->raw(iter, 0, event); 3748 3749 trace_seq_printf(s, "%d ?\n", entry->type); 3750 3751 return trace_handle_return(s); 3752 } 3753 3754 static enum print_line_t print_hex_fmt(struct trace_iterator *iter) 3755 { 3756 struct trace_array *tr = iter->tr; 3757 struct trace_seq *s = &iter->seq; 3758 unsigned char newline = '\n'; 3759 struct trace_entry *entry; 3760 struct trace_event *event; 3761 3762 entry = iter->ent; 3763 3764 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 3765 SEQ_PUT_HEX_FIELD(s, entry->pid); 3766 SEQ_PUT_HEX_FIELD(s, iter->cpu); 3767 SEQ_PUT_HEX_FIELD(s, iter->ts); 3768 if (trace_seq_has_overflowed(s)) 3769 return TRACE_TYPE_PARTIAL_LINE; 3770 } 3771 3772 event = ftrace_find_event(entry->type); 3773 if (event) { 3774 enum print_line_t ret = event->funcs->hex(iter, 0, event); 3775 if (ret != TRACE_TYPE_HANDLED) 3776 return ret; 3777 } 3778 3779 SEQ_PUT_FIELD(s, newline); 3780 3781 return trace_handle_return(s); 3782 } 3783 3784 static enum print_line_t print_bin_fmt(struct trace_iterator *iter) 3785 { 3786 struct trace_array *tr = iter->tr; 3787 struct trace_seq *s = &iter->seq; 3788 struct trace_entry *entry; 3789 struct trace_event *event; 3790 3791 entry = iter->ent; 3792 3793 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 3794 SEQ_PUT_FIELD(s, entry->pid); 3795 SEQ_PUT_FIELD(s, iter->cpu); 3796 SEQ_PUT_FIELD(s, iter->ts); 3797 if (trace_seq_has_overflowed(s)) 3798 return TRACE_TYPE_PARTIAL_LINE; 3799 } 3800 3801 event = ftrace_find_event(entry->type); 3802 return event ? event->funcs->binary(iter, 0, event) : 3803 TRACE_TYPE_HANDLED; 3804 } 3805 3806 int trace_empty(struct trace_iterator *iter) 3807 { 3808 struct ring_buffer_iter *buf_iter; 3809 int cpu; 3810 3811 /* If we are looking at one CPU buffer, only check that one */ 3812 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 3813 cpu = iter->cpu_file; 3814 buf_iter = trace_buffer_iter(iter, cpu); 3815 if (buf_iter) { 3816 if (!ring_buffer_iter_empty(buf_iter)) 3817 return 0; 3818 } else { 3819 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu)) 3820 return 0; 3821 } 3822 return 1; 3823 } 3824 3825 for_each_tracing_cpu(cpu) { 3826 buf_iter = trace_buffer_iter(iter, cpu); 3827 if (buf_iter) { 3828 if (!ring_buffer_iter_empty(buf_iter)) 3829 return 0; 3830 } else { 3831 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu)) 3832 return 0; 3833 } 3834 } 3835 3836 return 1; 3837 } 3838 3839 /* Called with trace_event_read_lock() held. */ 3840 enum print_line_t print_trace_line(struct trace_iterator *iter) 3841 { 3842 struct trace_array *tr = iter->tr; 3843 unsigned long trace_flags = tr->trace_flags; 3844 enum print_line_t ret; 3845 3846 if (iter->lost_events) { 3847 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", 3848 iter->cpu, iter->lost_events); 3849 if (trace_seq_has_overflowed(&iter->seq)) 3850 return TRACE_TYPE_PARTIAL_LINE; 3851 } 3852 3853 if (iter->trace && iter->trace->print_line) { 3854 ret = iter->trace->print_line(iter); 3855 if (ret != TRACE_TYPE_UNHANDLED) 3856 return ret; 3857 } 3858 3859 if (iter->ent->type == TRACE_BPUTS && 3860 trace_flags & TRACE_ITER_PRINTK && 3861 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 3862 return trace_print_bputs_msg_only(iter); 3863 3864 if (iter->ent->type == TRACE_BPRINT && 3865 trace_flags & TRACE_ITER_PRINTK && 3866 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 3867 return trace_print_bprintk_msg_only(iter); 3868 3869 if (iter->ent->type == TRACE_PRINT && 3870 trace_flags & TRACE_ITER_PRINTK && 3871 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 3872 return trace_print_printk_msg_only(iter); 3873 3874 if (trace_flags & TRACE_ITER_BIN) 3875 return print_bin_fmt(iter); 3876 3877 if (trace_flags & TRACE_ITER_HEX) 3878 return print_hex_fmt(iter); 3879 3880 if (trace_flags & TRACE_ITER_RAW) 3881 return print_raw_fmt(iter); 3882 3883 return print_trace_fmt(iter); 3884 } 3885 3886 void trace_latency_header(struct seq_file *m) 3887 { 3888 struct trace_iterator *iter = m->private; 3889 struct trace_array *tr = iter->tr; 3890 3891 /* print nothing if the buffers are empty */ 3892 if (trace_empty(iter)) 3893 return; 3894 3895 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 3896 print_trace_header(m, iter); 3897 3898 if (!(tr->trace_flags & TRACE_ITER_VERBOSE)) 3899 print_lat_help_header(m); 3900 } 3901 3902 void trace_default_header(struct seq_file *m) 3903 { 3904 struct trace_iterator *iter = m->private; 3905 struct trace_array *tr = iter->tr; 3906 unsigned long trace_flags = tr->trace_flags; 3907 3908 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO)) 3909 return; 3910 3911 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { 3912 /* print nothing if the buffers are empty */ 3913 if (trace_empty(iter)) 3914 return; 3915 print_trace_header(m, iter); 3916 if (!(trace_flags & TRACE_ITER_VERBOSE)) 3917 print_lat_help_header(m); 3918 } else { 3919 if (!(trace_flags & TRACE_ITER_VERBOSE)) { 3920 if (trace_flags & TRACE_ITER_IRQ_INFO) 3921 print_func_help_header_irq(iter->trace_buffer, 3922 m, trace_flags); 3923 else 3924 print_func_help_header(iter->trace_buffer, m, 3925 trace_flags); 3926 } 3927 } 3928 } 3929 3930 static void test_ftrace_alive(struct seq_file *m) 3931 { 3932 if (!ftrace_is_dead()) 3933 return; 3934 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n" 3935 "# MAY BE MISSING FUNCTION EVENTS\n"); 3936 } 3937 3938 #ifdef CONFIG_TRACER_MAX_TRACE 3939 static void show_snapshot_main_help(struct seq_file *m) 3940 { 3941 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n" 3942 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" 3943 "# Takes a snapshot of the main buffer.\n" 3944 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n" 3945 "# (Doesn't have to be '2' works with any number that\n" 3946 "# is not a '0' or '1')\n"); 3947 } 3948 3949 static void show_snapshot_percpu_help(struct seq_file *m) 3950 { 3951 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n"); 3952 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP 3953 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" 3954 "# Takes a snapshot of the main buffer for this cpu.\n"); 3955 #else 3956 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n" 3957 "# Must use main snapshot file to allocate.\n"); 3958 #endif 3959 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n" 3960 "# (Doesn't have to be '2' works with any number that\n" 3961 "# is not a '0' or '1')\n"); 3962 } 3963 3964 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) 3965 { 3966 if (iter->tr->allocated_snapshot) 3967 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n"); 3968 else 3969 seq_puts(m, "#\n# * Snapshot is freed *\n#\n"); 3970 3971 seq_puts(m, "# Snapshot commands:\n"); 3972 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 3973 show_snapshot_main_help(m); 3974 else 3975 show_snapshot_percpu_help(m); 3976 } 3977 #else 3978 /* Should never be called */ 3979 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { } 3980 #endif 3981 3982 static int s_show(struct seq_file *m, void *v) 3983 { 3984 struct trace_iterator *iter = v; 3985 int ret; 3986 3987 if (iter->ent == NULL) { 3988 if (iter->tr) { 3989 seq_printf(m, "# tracer: %s\n", iter->trace->name); 3990 seq_puts(m, "#\n"); 3991 test_ftrace_alive(m); 3992 } 3993 if (iter->snapshot && trace_empty(iter)) 3994 print_snapshot_help(m, iter); 3995 else if (iter->trace && iter->trace->print_header) 3996 iter->trace->print_header(m); 3997 else 3998 trace_default_header(m); 3999 4000 } else if (iter->leftover) { 4001 /* 4002 * If we filled the seq_file buffer earlier, we 4003 * want to just show it now. 4004 */ 4005 ret = trace_print_seq(m, &iter->seq); 4006 4007 /* ret should this time be zero, but you never know */ 4008 iter->leftover = ret; 4009 4010 } else { 4011 print_trace_line(iter); 4012 ret = trace_print_seq(m, &iter->seq); 4013 /* 4014 * If we overflow the seq_file buffer, then it will 4015 * ask us for this data again at start up. 4016 * Use that instead. 4017 * ret is 0 if seq_file write succeeded. 4018 * -1 otherwise. 4019 */ 4020 iter->leftover = ret; 4021 } 4022 4023 return 0; 4024 } 4025 4026 /* 4027 * Should be used after trace_array_get(), trace_types_lock 4028 * ensures that i_cdev was already initialized. 4029 */ 4030 static inline int tracing_get_cpu(struct inode *inode) 4031 { 4032 if (inode->i_cdev) /* See trace_create_cpu_file() */ 4033 return (long)inode->i_cdev - 1; 4034 return RING_BUFFER_ALL_CPUS; 4035 } 4036 4037 static const struct seq_operations tracer_seq_ops = { 4038 .start = s_start, 4039 .next = s_next, 4040 .stop = s_stop, 4041 .show = s_show, 4042 }; 4043 4044 static struct trace_iterator * 4045 __tracing_open(struct inode *inode, struct file *file, bool snapshot) 4046 { 4047 struct trace_array *tr = inode->i_private; 4048 struct trace_iterator *iter; 4049 int cpu; 4050 4051 if (tracing_disabled) 4052 return ERR_PTR(-ENODEV); 4053 4054 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter)); 4055 if (!iter) 4056 return ERR_PTR(-ENOMEM); 4057 4058 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter), 4059 GFP_KERNEL); 4060 if (!iter->buffer_iter) 4061 goto release; 4062 4063 /* 4064 * We make a copy of the current tracer to avoid concurrent 4065 * changes on it while we are reading. 4066 */ 4067 mutex_lock(&trace_types_lock); 4068 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL); 4069 if (!iter->trace) 4070 goto fail; 4071 4072 *iter->trace = *tr->current_trace; 4073 4074 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL)) 4075 goto fail; 4076 4077 iter->tr = tr; 4078 4079 #ifdef CONFIG_TRACER_MAX_TRACE 4080 /* Currently only the top directory has a snapshot */ 4081 if (tr->current_trace->print_max || snapshot) 4082 iter->trace_buffer = &tr->max_buffer; 4083 else 4084 #endif 4085 iter->trace_buffer = &tr->trace_buffer; 4086 iter->snapshot = snapshot; 4087 iter->pos = -1; 4088 iter->cpu_file = tracing_get_cpu(inode); 4089 mutex_init(&iter->mutex); 4090 4091 /* Notify the tracer early; before we stop tracing. */ 4092 if (iter->trace && iter->trace->open) 4093 iter->trace->open(iter); 4094 4095 /* Annotate start of buffers if we had overruns */ 4096 if (ring_buffer_overruns(iter->trace_buffer->buffer)) 4097 iter->iter_flags |= TRACE_FILE_ANNOTATE; 4098 4099 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 4100 if (trace_clocks[tr->clock_id].in_ns) 4101 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 4102 4103 /* stop the trace while dumping if we are not opening "snapshot" */ 4104 if (!iter->snapshot) 4105 tracing_stop_tr(tr); 4106 4107 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { 4108 for_each_tracing_cpu(cpu) { 4109 iter->buffer_iter[cpu] = 4110 ring_buffer_read_prepare(iter->trace_buffer->buffer, 4111 cpu, GFP_KERNEL); 4112 } 4113 ring_buffer_read_prepare_sync(); 4114 for_each_tracing_cpu(cpu) { 4115 ring_buffer_read_start(iter->buffer_iter[cpu]); 4116 tracing_iter_reset(iter, cpu); 4117 } 4118 } else { 4119 cpu = iter->cpu_file; 4120 iter->buffer_iter[cpu] = 4121 ring_buffer_read_prepare(iter->trace_buffer->buffer, 4122 cpu, GFP_KERNEL); 4123 ring_buffer_read_prepare_sync(); 4124 ring_buffer_read_start(iter->buffer_iter[cpu]); 4125 tracing_iter_reset(iter, cpu); 4126 } 4127 4128 mutex_unlock(&trace_types_lock); 4129 4130 return iter; 4131 4132 fail: 4133 mutex_unlock(&trace_types_lock); 4134 kfree(iter->trace); 4135 kfree(iter->buffer_iter); 4136 release: 4137 seq_release_private(inode, file); 4138 return ERR_PTR(-ENOMEM); 4139 } 4140 4141 int tracing_open_generic(struct inode *inode, struct file *filp) 4142 { 4143 if (tracing_disabled) 4144 return -ENODEV; 4145 4146 filp->private_data = inode->i_private; 4147 return 0; 4148 } 4149 4150 bool tracing_is_disabled(void) 4151 { 4152 return (tracing_disabled) ? true: false; 4153 } 4154 4155 /* 4156 * Open and update trace_array ref count. 4157 * Must have the current trace_array passed to it. 4158 */ 4159 static int tracing_open_generic_tr(struct inode *inode, struct file *filp) 4160 { 4161 struct trace_array *tr = inode->i_private; 4162 4163 if (tracing_disabled) 4164 return -ENODEV; 4165 4166 if (trace_array_get(tr) < 0) 4167 return -ENODEV; 4168 4169 filp->private_data = inode->i_private; 4170 4171 return 0; 4172 } 4173 4174 static int tracing_release(struct inode *inode, struct file *file) 4175 { 4176 struct trace_array *tr = inode->i_private; 4177 struct seq_file *m = file->private_data; 4178 struct trace_iterator *iter; 4179 int cpu; 4180 4181 if (!(file->f_mode & FMODE_READ)) { 4182 trace_array_put(tr); 4183 return 0; 4184 } 4185 4186 /* Writes do not use seq_file */ 4187 iter = m->private; 4188 mutex_lock(&trace_types_lock); 4189 4190 for_each_tracing_cpu(cpu) { 4191 if (iter->buffer_iter[cpu]) 4192 ring_buffer_read_finish(iter->buffer_iter[cpu]); 4193 } 4194 4195 if (iter->trace && iter->trace->close) 4196 iter->trace->close(iter); 4197 4198 if (!iter->snapshot) 4199 /* reenable tracing if it was previously enabled */ 4200 tracing_start_tr(tr); 4201 4202 __trace_array_put(tr); 4203 4204 mutex_unlock(&trace_types_lock); 4205 4206 mutex_destroy(&iter->mutex); 4207 free_cpumask_var(iter->started); 4208 kfree(iter->trace); 4209 kfree(iter->buffer_iter); 4210 seq_release_private(inode, file); 4211 4212 return 0; 4213 } 4214 4215 static int tracing_release_generic_tr(struct inode *inode, struct file *file) 4216 { 4217 struct trace_array *tr = inode->i_private; 4218 4219 trace_array_put(tr); 4220 return 0; 4221 } 4222 4223 static int tracing_single_release_tr(struct inode *inode, struct file *file) 4224 { 4225 struct trace_array *tr = inode->i_private; 4226 4227 trace_array_put(tr); 4228 4229 return single_release(inode, file); 4230 } 4231 4232 static int tracing_open(struct inode *inode, struct file *file) 4233 { 4234 struct trace_array *tr = inode->i_private; 4235 struct trace_iterator *iter; 4236 int ret = 0; 4237 4238 if (trace_array_get(tr) < 0) 4239 return -ENODEV; 4240 4241 /* If this file was open for write, then erase contents */ 4242 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 4243 int cpu = tracing_get_cpu(inode); 4244 struct trace_buffer *trace_buf = &tr->trace_buffer; 4245 4246 #ifdef CONFIG_TRACER_MAX_TRACE 4247 if (tr->current_trace->print_max) 4248 trace_buf = &tr->max_buffer; 4249 #endif 4250 4251 if (cpu == RING_BUFFER_ALL_CPUS) 4252 tracing_reset_online_cpus(trace_buf); 4253 else 4254 tracing_reset(trace_buf, cpu); 4255 } 4256 4257 if (file->f_mode & FMODE_READ) { 4258 iter = __tracing_open(inode, file, false); 4259 if (IS_ERR(iter)) 4260 ret = PTR_ERR(iter); 4261 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 4262 iter->iter_flags |= TRACE_FILE_LAT_FMT; 4263 } 4264 4265 if (ret < 0) 4266 trace_array_put(tr); 4267 4268 return ret; 4269 } 4270 4271 /* 4272 * Some tracers are not suitable for instance buffers. 4273 * A tracer is always available for the global array (toplevel) 4274 * or if it explicitly states that it is. 4275 */ 4276 static bool 4277 trace_ok_for_array(struct tracer *t, struct trace_array *tr) 4278 { 4279 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances; 4280 } 4281 4282 /* Find the next tracer that this trace array may use */ 4283 static struct tracer * 4284 get_tracer_for_array(struct trace_array *tr, struct tracer *t) 4285 { 4286 while (t && !trace_ok_for_array(t, tr)) 4287 t = t->next; 4288 4289 return t; 4290 } 4291 4292 static void * 4293 t_next(struct seq_file *m, void *v, loff_t *pos) 4294 { 4295 struct trace_array *tr = m->private; 4296 struct tracer *t = v; 4297 4298 (*pos)++; 4299 4300 if (t) 4301 t = get_tracer_for_array(tr, t->next); 4302 4303 return t; 4304 } 4305 4306 static void *t_start(struct seq_file *m, loff_t *pos) 4307 { 4308 struct trace_array *tr = m->private; 4309 struct tracer *t; 4310 loff_t l = 0; 4311 4312 mutex_lock(&trace_types_lock); 4313 4314 t = get_tracer_for_array(tr, trace_types); 4315 for (; t && l < *pos; t = t_next(m, t, &l)) 4316 ; 4317 4318 return t; 4319 } 4320 4321 static void t_stop(struct seq_file *m, void *p) 4322 { 4323 mutex_unlock(&trace_types_lock); 4324 } 4325 4326 static int t_show(struct seq_file *m, void *v) 4327 { 4328 struct tracer *t = v; 4329 4330 if (!t) 4331 return 0; 4332 4333 seq_puts(m, t->name); 4334 if (t->next) 4335 seq_putc(m, ' '); 4336 else 4337 seq_putc(m, '\n'); 4338 4339 return 0; 4340 } 4341 4342 static const struct seq_operations show_traces_seq_ops = { 4343 .start = t_start, 4344 .next = t_next, 4345 .stop = t_stop, 4346 .show = t_show, 4347 }; 4348 4349 static int show_traces_open(struct inode *inode, struct file *file) 4350 { 4351 struct trace_array *tr = inode->i_private; 4352 struct seq_file *m; 4353 int ret; 4354 4355 if (tracing_disabled) 4356 return -ENODEV; 4357 4358 ret = seq_open(file, &show_traces_seq_ops); 4359 if (ret) 4360 return ret; 4361 4362 m = file->private_data; 4363 m->private = tr; 4364 4365 return 0; 4366 } 4367 4368 static ssize_t 4369 tracing_write_stub(struct file *filp, const char __user *ubuf, 4370 size_t count, loff_t *ppos) 4371 { 4372 return count; 4373 } 4374 4375 loff_t tracing_lseek(struct file *file, loff_t offset, int whence) 4376 { 4377 int ret; 4378 4379 if (file->f_mode & FMODE_READ) 4380 ret = seq_lseek(file, offset, whence); 4381 else 4382 file->f_pos = ret = 0; 4383 4384 return ret; 4385 } 4386 4387 static const struct file_operations tracing_fops = { 4388 .open = tracing_open, 4389 .read = seq_read, 4390 .write = tracing_write_stub, 4391 .llseek = tracing_lseek, 4392 .release = tracing_release, 4393 }; 4394 4395 static const struct file_operations show_traces_fops = { 4396 .open = show_traces_open, 4397 .read = seq_read, 4398 .release = seq_release, 4399 .llseek = seq_lseek, 4400 }; 4401 4402 static ssize_t 4403 tracing_cpumask_read(struct file *filp, char __user *ubuf, 4404 size_t count, loff_t *ppos) 4405 { 4406 struct trace_array *tr = file_inode(filp)->i_private; 4407 char *mask_str; 4408 int len; 4409 4410 len = snprintf(NULL, 0, "%*pb\n", 4411 cpumask_pr_args(tr->tracing_cpumask)) + 1; 4412 mask_str = kmalloc(len, GFP_KERNEL); 4413 if (!mask_str) 4414 return -ENOMEM; 4415 4416 len = snprintf(mask_str, len, "%*pb\n", 4417 cpumask_pr_args(tr->tracing_cpumask)); 4418 if (len >= count) { 4419 count = -EINVAL; 4420 goto out_err; 4421 } 4422 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len); 4423 4424 out_err: 4425 kfree(mask_str); 4426 4427 return count; 4428 } 4429 4430 static ssize_t 4431 tracing_cpumask_write(struct file *filp, const char __user *ubuf, 4432 size_t count, loff_t *ppos) 4433 { 4434 struct trace_array *tr = file_inode(filp)->i_private; 4435 cpumask_var_t tracing_cpumask_new; 4436 int err, cpu; 4437 4438 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL)) 4439 return -ENOMEM; 4440 4441 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new); 4442 if (err) 4443 goto err_unlock; 4444 4445 local_irq_disable(); 4446 arch_spin_lock(&tr->max_lock); 4447 for_each_tracing_cpu(cpu) { 4448 /* 4449 * Increase/decrease the disabled counter if we are 4450 * about to flip a bit in the cpumask: 4451 */ 4452 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) && 4453 !cpumask_test_cpu(cpu, tracing_cpumask_new)) { 4454 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled); 4455 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu); 4456 } 4457 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) && 4458 cpumask_test_cpu(cpu, tracing_cpumask_new)) { 4459 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled); 4460 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu); 4461 } 4462 } 4463 arch_spin_unlock(&tr->max_lock); 4464 local_irq_enable(); 4465 4466 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new); 4467 free_cpumask_var(tracing_cpumask_new); 4468 4469 return count; 4470 4471 err_unlock: 4472 free_cpumask_var(tracing_cpumask_new); 4473 4474 return err; 4475 } 4476 4477 static const struct file_operations tracing_cpumask_fops = { 4478 .open = tracing_open_generic_tr, 4479 .read = tracing_cpumask_read, 4480 .write = tracing_cpumask_write, 4481 .release = tracing_release_generic_tr, 4482 .llseek = generic_file_llseek, 4483 }; 4484 4485 static int tracing_trace_options_show(struct seq_file *m, void *v) 4486 { 4487 struct tracer_opt *trace_opts; 4488 struct trace_array *tr = m->private; 4489 u32 tracer_flags; 4490 int i; 4491 4492 mutex_lock(&trace_types_lock); 4493 tracer_flags = tr->current_trace->flags->val; 4494 trace_opts = tr->current_trace->flags->opts; 4495 4496 for (i = 0; trace_options[i]; i++) { 4497 if (tr->trace_flags & (1 << i)) 4498 seq_printf(m, "%s\n", trace_options[i]); 4499 else 4500 seq_printf(m, "no%s\n", trace_options[i]); 4501 } 4502 4503 for (i = 0; trace_opts[i].name; i++) { 4504 if (tracer_flags & trace_opts[i].bit) 4505 seq_printf(m, "%s\n", trace_opts[i].name); 4506 else 4507 seq_printf(m, "no%s\n", trace_opts[i].name); 4508 } 4509 mutex_unlock(&trace_types_lock); 4510 4511 return 0; 4512 } 4513 4514 static int __set_tracer_option(struct trace_array *tr, 4515 struct tracer_flags *tracer_flags, 4516 struct tracer_opt *opts, int neg) 4517 { 4518 struct tracer *trace = tracer_flags->trace; 4519 int ret; 4520 4521 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg); 4522 if (ret) 4523 return ret; 4524 4525 if (neg) 4526 tracer_flags->val &= ~opts->bit; 4527 else 4528 tracer_flags->val |= opts->bit; 4529 return 0; 4530 } 4531 4532 /* Try to assign a tracer specific option */ 4533 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg) 4534 { 4535 struct tracer *trace = tr->current_trace; 4536 struct tracer_flags *tracer_flags = trace->flags; 4537 struct tracer_opt *opts = NULL; 4538 int i; 4539 4540 for (i = 0; tracer_flags->opts[i].name; i++) { 4541 opts = &tracer_flags->opts[i]; 4542 4543 if (strcmp(cmp, opts->name) == 0) 4544 return __set_tracer_option(tr, trace->flags, opts, neg); 4545 } 4546 4547 return -EINVAL; 4548 } 4549 4550 /* Some tracers require overwrite to stay enabled */ 4551 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set) 4552 { 4553 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set) 4554 return -1; 4555 4556 return 0; 4557 } 4558 4559 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled) 4560 { 4561 /* do nothing if flag is already set */ 4562 if (!!(tr->trace_flags & mask) == !!enabled) 4563 return 0; 4564 4565 /* Give the tracer a chance to approve the change */ 4566 if (tr->current_trace->flag_changed) 4567 if (tr->current_trace->flag_changed(tr, mask, !!enabled)) 4568 return -EINVAL; 4569 4570 if (enabled) 4571 tr->trace_flags |= mask; 4572 else 4573 tr->trace_flags &= ~mask; 4574 4575 if (mask == TRACE_ITER_RECORD_CMD) 4576 trace_event_enable_cmd_record(enabled); 4577 4578 if (mask == TRACE_ITER_RECORD_TGID) { 4579 if (!tgid_map) 4580 tgid_map = kcalloc(PID_MAX_DEFAULT + 1, 4581 sizeof(*tgid_map), 4582 GFP_KERNEL); 4583 if (!tgid_map) { 4584 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID; 4585 return -ENOMEM; 4586 } 4587 4588 trace_event_enable_tgid_record(enabled); 4589 } 4590 4591 if (mask == TRACE_ITER_EVENT_FORK) 4592 trace_event_follow_fork(tr, enabled); 4593 4594 if (mask == TRACE_ITER_FUNC_FORK) 4595 ftrace_pid_follow_fork(tr, enabled); 4596 4597 if (mask == TRACE_ITER_OVERWRITE) { 4598 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled); 4599 #ifdef CONFIG_TRACER_MAX_TRACE 4600 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled); 4601 #endif 4602 } 4603 4604 if (mask == TRACE_ITER_PRINTK) { 4605 trace_printk_start_stop_comm(enabled); 4606 trace_printk_control(enabled); 4607 } 4608 4609 return 0; 4610 } 4611 4612 static int trace_set_options(struct trace_array *tr, char *option) 4613 { 4614 char *cmp; 4615 int neg = 0; 4616 int ret; 4617 size_t orig_len = strlen(option); 4618 int len; 4619 4620 cmp = strstrip(option); 4621 4622 len = str_has_prefix(cmp, "no"); 4623 if (len) 4624 neg = 1; 4625 4626 cmp += len; 4627 4628 mutex_lock(&trace_types_lock); 4629 4630 ret = match_string(trace_options, -1, cmp); 4631 /* If no option could be set, test the specific tracer options */ 4632 if (ret < 0) 4633 ret = set_tracer_option(tr, cmp, neg); 4634 else 4635 ret = set_tracer_flag(tr, 1 << ret, !neg); 4636 4637 mutex_unlock(&trace_types_lock); 4638 4639 /* 4640 * If the first trailing whitespace is replaced with '\0' by strstrip, 4641 * turn it back into a space. 4642 */ 4643 if (orig_len > strlen(option)) 4644 option[strlen(option)] = ' '; 4645 4646 return ret; 4647 } 4648 4649 static void __init apply_trace_boot_options(void) 4650 { 4651 char *buf = trace_boot_options_buf; 4652 char *option; 4653 4654 while (true) { 4655 option = strsep(&buf, ","); 4656 4657 if (!option) 4658 break; 4659 4660 if (*option) 4661 trace_set_options(&global_trace, option); 4662 4663 /* Put back the comma to allow this to be called again */ 4664 if (buf) 4665 *(buf - 1) = ','; 4666 } 4667 } 4668 4669 static ssize_t 4670 tracing_trace_options_write(struct file *filp, const char __user *ubuf, 4671 size_t cnt, loff_t *ppos) 4672 { 4673 struct seq_file *m = filp->private_data; 4674 struct trace_array *tr = m->private; 4675 char buf[64]; 4676 int ret; 4677 4678 if (cnt >= sizeof(buf)) 4679 return -EINVAL; 4680 4681 if (copy_from_user(buf, ubuf, cnt)) 4682 return -EFAULT; 4683 4684 buf[cnt] = 0; 4685 4686 ret = trace_set_options(tr, buf); 4687 if (ret < 0) 4688 return ret; 4689 4690 *ppos += cnt; 4691 4692 return cnt; 4693 } 4694 4695 static int tracing_trace_options_open(struct inode *inode, struct file *file) 4696 { 4697 struct trace_array *tr = inode->i_private; 4698 int ret; 4699 4700 if (tracing_disabled) 4701 return -ENODEV; 4702 4703 if (trace_array_get(tr) < 0) 4704 return -ENODEV; 4705 4706 ret = single_open(file, tracing_trace_options_show, inode->i_private); 4707 if (ret < 0) 4708 trace_array_put(tr); 4709 4710 return ret; 4711 } 4712 4713 static const struct file_operations tracing_iter_fops = { 4714 .open = tracing_trace_options_open, 4715 .read = seq_read, 4716 .llseek = seq_lseek, 4717 .release = tracing_single_release_tr, 4718 .write = tracing_trace_options_write, 4719 }; 4720 4721 static const char readme_msg[] = 4722 "tracing mini-HOWTO:\n\n" 4723 "# echo 0 > tracing_on : quick way to disable tracing\n" 4724 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n" 4725 " Important files:\n" 4726 " trace\t\t\t- The static contents of the buffer\n" 4727 "\t\t\t To clear the buffer write into this file: echo > trace\n" 4728 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n" 4729 " current_tracer\t- function and latency tracers\n" 4730 " available_tracers\t- list of configured tracers for current_tracer\n" 4731 " error_log\t- error log for failed commands (that support it)\n" 4732 " buffer_size_kb\t- view and modify size of per cpu buffer\n" 4733 " buffer_total_size_kb - view total size of all cpu buffers\n\n" 4734 " trace_clock\t\t-change the clock used to order events\n" 4735 " local: Per cpu clock but may not be synced across CPUs\n" 4736 " global: Synced across CPUs but slows tracing down.\n" 4737 " counter: Not a clock, but just an increment\n" 4738 " uptime: Jiffy counter from time of boot\n" 4739 " perf: Same clock that perf events use\n" 4740 #ifdef CONFIG_X86_64 4741 " x86-tsc: TSC cycle counter\n" 4742 #endif 4743 "\n timestamp_mode\t-view the mode used to timestamp events\n" 4744 " delta: Delta difference against a buffer-wide timestamp\n" 4745 " absolute: Absolute (standalone) timestamp\n" 4746 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n" 4747 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n" 4748 " tracing_cpumask\t- Limit which CPUs to trace\n" 4749 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n" 4750 "\t\t\t Remove sub-buffer with rmdir\n" 4751 " trace_options\t\t- Set format or modify how tracing happens\n" 4752 "\t\t\t Disable an option by prefixing 'no' to the\n" 4753 "\t\t\t option name\n" 4754 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n" 4755 #ifdef CONFIG_DYNAMIC_FTRACE 4756 "\n available_filter_functions - list of functions that can be filtered on\n" 4757 " set_ftrace_filter\t- echo function name in here to only trace these\n" 4758 "\t\t\t functions\n" 4759 "\t accepts: func_full_name or glob-matching-pattern\n" 4760 "\t modules: Can select a group via module\n" 4761 "\t Format: :mod:<module-name>\n" 4762 "\t example: echo :mod:ext3 > set_ftrace_filter\n" 4763 "\t triggers: a command to perform when function is hit\n" 4764 "\t Format: <function>:<trigger>[:count]\n" 4765 "\t trigger: traceon, traceoff\n" 4766 "\t\t enable_event:<system>:<event>\n" 4767 "\t\t disable_event:<system>:<event>\n" 4768 #ifdef CONFIG_STACKTRACE 4769 "\t\t stacktrace\n" 4770 #endif 4771 #ifdef CONFIG_TRACER_SNAPSHOT 4772 "\t\t snapshot\n" 4773 #endif 4774 "\t\t dump\n" 4775 "\t\t cpudump\n" 4776 "\t example: echo do_fault:traceoff > set_ftrace_filter\n" 4777 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n" 4778 "\t The first one will disable tracing every time do_fault is hit\n" 4779 "\t The second will disable tracing at most 3 times when do_trap is hit\n" 4780 "\t The first time do trap is hit and it disables tracing, the\n" 4781 "\t counter will decrement to 2. If tracing is already disabled,\n" 4782 "\t the counter will not decrement. It only decrements when the\n" 4783 "\t trigger did work\n" 4784 "\t To remove trigger without count:\n" 4785 "\t echo '!<function>:<trigger> > set_ftrace_filter\n" 4786 "\t To remove trigger with a count:\n" 4787 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n" 4788 " set_ftrace_notrace\t- echo function name in here to never trace.\n" 4789 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n" 4790 "\t modules: Can select a group via module command :mod:\n" 4791 "\t Does not accept triggers\n" 4792 #endif /* CONFIG_DYNAMIC_FTRACE */ 4793 #ifdef CONFIG_FUNCTION_TRACER 4794 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n" 4795 "\t\t (function)\n" 4796 #endif 4797 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 4798 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n" 4799 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n" 4800 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n" 4801 #endif 4802 #ifdef CONFIG_TRACER_SNAPSHOT 4803 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n" 4804 "\t\t\t snapshot buffer. Read the contents for more\n" 4805 "\t\t\t information\n" 4806 #endif 4807 #ifdef CONFIG_STACK_TRACER 4808 " stack_trace\t\t- Shows the max stack trace when active\n" 4809 " stack_max_size\t- Shows current max stack size that was traced\n" 4810 "\t\t\t Write into this file to reset the max size (trigger a\n" 4811 "\t\t\t new trace)\n" 4812 #ifdef CONFIG_DYNAMIC_FTRACE 4813 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n" 4814 "\t\t\t traces\n" 4815 #endif 4816 #endif /* CONFIG_STACK_TRACER */ 4817 #ifdef CONFIG_DYNAMIC_EVENTS 4818 " dynamic_events\t\t- Add/remove/show the generic dynamic events\n" 4819 "\t\t\t Write into this file to define/undefine new trace events.\n" 4820 #endif 4821 #ifdef CONFIG_KPROBE_EVENTS 4822 " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n" 4823 "\t\t\t Write into this file to define/undefine new trace events.\n" 4824 #endif 4825 #ifdef CONFIG_UPROBE_EVENTS 4826 " uprobe_events\t\t- Add/remove/show the userspace dynamic events\n" 4827 "\t\t\t Write into this file to define/undefine new trace events.\n" 4828 #endif 4829 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS) 4830 "\t accepts: event-definitions (one definition per line)\n" 4831 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n" 4832 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n" 4833 #ifdef CONFIG_HIST_TRIGGERS 4834 "\t s:[synthetic/]<event> <field> [<field>]\n" 4835 #endif 4836 "\t -:[<group>/]<event>\n" 4837 #ifdef CONFIG_KPROBE_EVENTS 4838 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n" 4839 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n" 4840 #endif 4841 #ifdef CONFIG_UPROBE_EVENTS 4842 " place (uprobe): <path>:<offset>[(ref_ctr_offset)]\n" 4843 #endif 4844 "\t args: <name>=fetcharg[:type]\n" 4845 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n" 4846 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 4847 "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n" 4848 #else 4849 "\t $stack<index>, $stack, $retval, $comm,\n" 4850 #endif 4851 "\t +|-[u]<offset>(<fetcharg>)\n" 4852 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n" 4853 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n" 4854 "\t <type>\\[<array-size>\\]\n" 4855 #ifdef CONFIG_HIST_TRIGGERS 4856 "\t field: <stype> <name>;\n" 4857 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n" 4858 "\t [unsigned] char/int/long\n" 4859 #endif 4860 #endif 4861 " events/\t\t- Directory containing all trace event subsystems:\n" 4862 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n" 4863 " events/<system>/\t- Directory containing all trace events for <system>:\n" 4864 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n" 4865 "\t\t\t events\n" 4866 " filter\t\t- If set, only events passing filter are traced\n" 4867 " events/<system>/<event>/\t- Directory containing control files for\n" 4868 "\t\t\t <event>:\n" 4869 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n" 4870 " filter\t\t- If set, only events passing filter are traced\n" 4871 " trigger\t\t- If set, a command to perform when event is hit\n" 4872 "\t Format: <trigger>[:count][if <filter>]\n" 4873 "\t trigger: traceon, traceoff\n" 4874 "\t enable_event:<system>:<event>\n" 4875 "\t disable_event:<system>:<event>\n" 4876 #ifdef CONFIG_HIST_TRIGGERS 4877 "\t enable_hist:<system>:<event>\n" 4878 "\t disable_hist:<system>:<event>\n" 4879 #endif 4880 #ifdef CONFIG_STACKTRACE 4881 "\t\t stacktrace\n" 4882 #endif 4883 #ifdef CONFIG_TRACER_SNAPSHOT 4884 "\t\t snapshot\n" 4885 #endif 4886 #ifdef CONFIG_HIST_TRIGGERS 4887 "\t\t hist (see below)\n" 4888 #endif 4889 "\t example: echo traceoff > events/block/block_unplug/trigger\n" 4890 "\t echo traceoff:3 > events/block/block_unplug/trigger\n" 4891 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n" 4892 "\t events/block/block_unplug/trigger\n" 4893 "\t The first disables tracing every time block_unplug is hit.\n" 4894 "\t The second disables tracing the first 3 times block_unplug is hit.\n" 4895 "\t The third enables the kmalloc event the first 3 times block_unplug\n" 4896 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n" 4897 "\t Like function triggers, the counter is only decremented if it\n" 4898 "\t enabled or disabled tracing.\n" 4899 "\t To remove a trigger without a count:\n" 4900 "\t echo '!<trigger> > <system>/<event>/trigger\n" 4901 "\t To remove a trigger with a count:\n" 4902 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n" 4903 "\t Filters can be ignored when removing a trigger.\n" 4904 #ifdef CONFIG_HIST_TRIGGERS 4905 " hist trigger\t- If set, event hits are aggregated into a hash table\n" 4906 "\t Format: hist:keys=<field1[,field2,...]>\n" 4907 "\t [:values=<field1[,field2,...]>]\n" 4908 "\t [:sort=<field1[,field2,...]>]\n" 4909 "\t [:size=#entries]\n" 4910 "\t [:pause][:continue][:clear]\n" 4911 "\t [:name=histname1]\n" 4912 "\t [:<handler>.<action>]\n" 4913 "\t [if <filter>]\n\n" 4914 "\t When a matching event is hit, an entry is added to a hash\n" 4915 "\t table using the key(s) and value(s) named, and the value of a\n" 4916 "\t sum called 'hitcount' is incremented. Keys and values\n" 4917 "\t correspond to fields in the event's format description. Keys\n" 4918 "\t can be any field, or the special string 'stacktrace'.\n" 4919 "\t Compound keys consisting of up to two fields can be specified\n" 4920 "\t by the 'keys' keyword. Values must correspond to numeric\n" 4921 "\t fields. Sort keys consisting of up to two fields can be\n" 4922 "\t specified using the 'sort' keyword. The sort direction can\n" 4923 "\t be modified by appending '.descending' or '.ascending' to a\n" 4924 "\t sort field. The 'size' parameter can be used to specify more\n" 4925 "\t or fewer than the default 2048 entries for the hashtable size.\n" 4926 "\t If a hist trigger is given a name using the 'name' parameter,\n" 4927 "\t its histogram data will be shared with other triggers of the\n" 4928 "\t same name, and trigger hits will update this common data.\n\n" 4929 "\t Reading the 'hist' file for the event will dump the hash\n" 4930 "\t table in its entirety to stdout. If there are multiple hist\n" 4931 "\t triggers attached to an event, there will be a table for each\n" 4932 "\t trigger in the output. The table displayed for a named\n" 4933 "\t trigger will be the same as any other instance having the\n" 4934 "\t same name. The default format used to display a given field\n" 4935 "\t can be modified by appending any of the following modifiers\n" 4936 "\t to the field name, as applicable:\n\n" 4937 "\t .hex display a number as a hex value\n" 4938 "\t .sym display an address as a symbol\n" 4939 "\t .sym-offset display an address as a symbol and offset\n" 4940 "\t .execname display a common_pid as a program name\n" 4941 "\t .syscall display a syscall id as a syscall name\n" 4942 "\t .log2 display log2 value rather than raw number\n" 4943 "\t .usecs display a common_timestamp in microseconds\n\n" 4944 "\t The 'pause' parameter can be used to pause an existing hist\n" 4945 "\t trigger or to start a hist trigger but not log any events\n" 4946 "\t until told to do so. 'continue' can be used to start or\n" 4947 "\t restart a paused hist trigger.\n\n" 4948 "\t The 'clear' parameter will clear the contents of a running\n" 4949 "\t hist trigger and leave its current paused/active state\n" 4950 "\t unchanged.\n\n" 4951 "\t The enable_hist and disable_hist triggers can be used to\n" 4952 "\t have one event conditionally start and stop another event's\n" 4953 "\t already-attached hist trigger. The syntax is analogous to\n" 4954 "\t the enable_event and disable_event triggers.\n\n" 4955 "\t Hist trigger handlers and actions are executed whenever a\n" 4956 "\t a histogram entry is added or updated. They take the form:\n\n" 4957 "\t <handler>.<action>\n\n" 4958 "\t The available handlers are:\n\n" 4959 "\t onmatch(matching.event) - invoke on addition or update\n" 4960 "\t onmax(var) - invoke if var exceeds current max\n" 4961 "\t onchange(var) - invoke action if var changes\n\n" 4962 "\t The available actions are:\n\n" 4963 "\t trace(<synthetic_event>,param list) - generate synthetic event\n" 4964 "\t save(field,...) - save current event fields\n" 4965 #ifdef CONFIG_TRACER_SNAPSHOT 4966 "\t snapshot() - snapshot the trace buffer\n" 4967 #endif 4968 #endif 4969 ; 4970 4971 static ssize_t 4972 tracing_readme_read(struct file *filp, char __user *ubuf, 4973 size_t cnt, loff_t *ppos) 4974 { 4975 return simple_read_from_buffer(ubuf, cnt, ppos, 4976 readme_msg, strlen(readme_msg)); 4977 } 4978 4979 static const struct file_operations tracing_readme_fops = { 4980 .open = tracing_open_generic, 4981 .read = tracing_readme_read, 4982 .llseek = generic_file_llseek, 4983 }; 4984 4985 static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos) 4986 { 4987 int *ptr = v; 4988 4989 if (*pos || m->count) 4990 ptr++; 4991 4992 (*pos)++; 4993 4994 for (; ptr <= &tgid_map[PID_MAX_DEFAULT]; ptr++) { 4995 if (trace_find_tgid(*ptr)) 4996 return ptr; 4997 } 4998 4999 return NULL; 5000 } 5001 5002 static void *saved_tgids_start(struct seq_file *m, loff_t *pos) 5003 { 5004 void *v; 5005 loff_t l = 0; 5006 5007 if (!tgid_map) 5008 return NULL; 5009 5010 v = &tgid_map[0]; 5011 while (l <= *pos) { 5012 v = saved_tgids_next(m, v, &l); 5013 if (!v) 5014 return NULL; 5015 } 5016 5017 return v; 5018 } 5019 5020 static void saved_tgids_stop(struct seq_file *m, void *v) 5021 { 5022 } 5023 5024 static int saved_tgids_show(struct seq_file *m, void *v) 5025 { 5026 int pid = (int *)v - tgid_map; 5027 5028 seq_printf(m, "%d %d\n", pid, trace_find_tgid(pid)); 5029 return 0; 5030 } 5031 5032 static const struct seq_operations tracing_saved_tgids_seq_ops = { 5033 .start = saved_tgids_start, 5034 .stop = saved_tgids_stop, 5035 .next = saved_tgids_next, 5036 .show = saved_tgids_show, 5037 }; 5038 5039 static int tracing_saved_tgids_open(struct inode *inode, struct file *filp) 5040 { 5041 if (tracing_disabled) 5042 return -ENODEV; 5043 5044 return seq_open(filp, &tracing_saved_tgids_seq_ops); 5045 } 5046 5047 5048 static const struct file_operations tracing_saved_tgids_fops = { 5049 .open = tracing_saved_tgids_open, 5050 .read = seq_read, 5051 .llseek = seq_lseek, 5052 .release = seq_release, 5053 }; 5054 5055 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos) 5056 { 5057 unsigned int *ptr = v; 5058 5059 if (*pos || m->count) 5060 ptr++; 5061 5062 (*pos)++; 5063 5064 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num]; 5065 ptr++) { 5066 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP) 5067 continue; 5068 5069 return ptr; 5070 } 5071 5072 return NULL; 5073 } 5074 5075 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos) 5076 { 5077 void *v; 5078 loff_t l = 0; 5079 5080 preempt_disable(); 5081 arch_spin_lock(&trace_cmdline_lock); 5082 5083 v = &savedcmd->map_cmdline_to_pid[0]; 5084 while (l <= *pos) { 5085 v = saved_cmdlines_next(m, v, &l); 5086 if (!v) 5087 return NULL; 5088 } 5089 5090 return v; 5091 } 5092 5093 static void saved_cmdlines_stop(struct seq_file *m, void *v) 5094 { 5095 arch_spin_unlock(&trace_cmdline_lock); 5096 preempt_enable(); 5097 } 5098 5099 static int saved_cmdlines_show(struct seq_file *m, void *v) 5100 { 5101 char buf[TASK_COMM_LEN]; 5102 unsigned int *pid = v; 5103 5104 __trace_find_cmdline(*pid, buf); 5105 seq_printf(m, "%d %s\n", *pid, buf); 5106 return 0; 5107 } 5108 5109 static const struct seq_operations tracing_saved_cmdlines_seq_ops = { 5110 .start = saved_cmdlines_start, 5111 .next = saved_cmdlines_next, 5112 .stop = saved_cmdlines_stop, 5113 .show = saved_cmdlines_show, 5114 }; 5115 5116 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp) 5117 { 5118 if (tracing_disabled) 5119 return -ENODEV; 5120 5121 return seq_open(filp, &tracing_saved_cmdlines_seq_ops); 5122 } 5123 5124 static const struct file_operations tracing_saved_cmdlines_fops = { 5125 .open = tracing_saved_cmdlines_open, 5126 .read = seq_read, 5127 .llseek = seq_lseek, 5128 .release = seq_release, 5129 }; 5130 5131 static ssize_t 5132 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf, 5133 size_t cnt, loff_t *ppos) 5134 { 5135 char buf[64]; 5136 int r; 5137 5138 arch_spin_lock(&trace_cmdline_lock); 5139 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num); 5140 arch_spin_unlock(&trace_cmdline_lock); 5141 5142 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5143 } 5144 5145 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s) 5146 { 5147 kfree(s->saved_cmdlines); 5148 kfree(s->map_cmdline_to_pid); 5149 kfree(s); 5150 } 5151 5152 static int tracing_resize_saved_cmdlines(unsigned int val) 5153 { 5154 struct saved_cmdlines_buffer *s, *savedcmd_temp; 5155 5156 s = kmalloc(sizeof(*s), GFP_KERNEL); 5157 if (!s) 5158 return -ENOMEM; 5159 5160 if (allocate_cmdlines_buffer(val, s) < 0) { 5161 kfree(s); 5162 return -ENOMEM; 5163 } 5164 5165 arch_spin_lock(&trace_cmdline_lock); 5166 savedcmd_temp = savedcmd; 5167 savedcmd = s; 5168 arch_spin_unlock(&trace_cmdline_lock); 5169 free_saved_cmdlines_buffer(savedcmd_temp); 5170 5171 return 0; 5172 } 5173 5174 static ssize_t 5175 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf, 5176 size_t cnt, loff_t *ppos) 5177 { 5178 unsigned long val; 5179 int ret; 5180 5181 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 5182 if (ret) 5183 return ret; 5184 5185 /* must have at least 1 entry or less than PID_MAX_DEFAULT */ 5186 if (!val || val > PID_MAX_DEFAULT) 5187 return -EINVAL; 5188 5189 ret = tracing_resize_saved_cmdlines((unsigned int)val); 5190 if (ret < 0) 5191 return ret; 5192 5193 *ppos += cnt; 5194 5195 return cnt; 5196 } 5197 5198 static const struct file_operations tracing_saved_cmdlines_size_fops = { 5199 .open = tracing_open_generic, 5200 .read = tracing_saved_cmdlines_size_read, 5201 .write = tracing_saved_cmdlines_size_write, 5202 }; 5203 5204 #ifdef CONFIG_TRACE_EVAL_MAP_FILE 5205 static union trace_eval_map_item * 5206 update_eval_map(union trace_eval_map_item *ptr) 5207 { 5208 if (!ptr->map.eval_string) { 5209 if (ptr->tail.next) { 5210 ptr = ptr->tail.next; 5211 /* Set ptr to the next real item (skip head) */ 5212 ptr++; 5213 } else 5214 return NULL; 5215 } 5216 return ptr; 5217 } 5218 5219 static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos) 5220 { 5221 union trace_eval_map_item *ptr = v; 5222 5223 /* 5224 * Paranoid! If ptr points to end, we don't want to increment past it. 5225 * This really should never happen. 5226 */ 5227 ptr = update_eval_map(ptr); 5228 if (WARN_ON_ONCE(!ptr)) 5229 return NULL; 5230 5231 ptr++; 5232 5233 (*pos)++; 5234 5235 ptr = update_eval_map(ptr); 5236 5237 return ptr; 5238 } 5239 5240 static void *eval_map_start(struct seq_file *m, loff_t *pos) 5241 { 5242 union trace_eval_map_item *v; 5243 loff_t l = 0; 5244 5245 mutex_lock(&trace_eval_mutex); 5246 5247 v = trace_eval_maps; 5248 if (v) 5249 v++; 5250 5251 while (v && l < *pos) { 5252 v = eval_map_next(m, v, &l); 5253 } 5254 5255 return v; 5256 } 5257 5258 static void eval_map_stop(struct seq_file *m, void *v) 5259 { 5260 mutex_unlock(&trace_eval_mutex); 5261 } 5262 5263 static int eval_map_show(struct seq_file *m, void *v) 5264 { 5265 union trace_eval_map_item *ptr = v; 5266 5267 seq_printf(m, "%s %ld (%s)\n", 5268 ptr->map.eval_string, ptr->map.eval_value, 5269 ptr->map.system); 5270 5271 return 0; 5272 } 5273 5274 static const struct seq_operations tracing_eval_map_seq_ops = { 5275 .start = eval_map_start, 5276 .next = eval_map_next, 5277 .stop = eval_map_stop, 5278 .show = eval_map_show, 5279 }; 5280 5281 static int tracing_eval_map_open(struct inode *inode, struct file *filp) 5282 { 5283 if (tracing_disabled) 5284 return -ENODEV; 5285 5286 return seq_open(filp, &tracing_eval_map_seq_ops); 5287 } 5288 5289 static const struct file_operations tracing_eval_map_fops = { 5290 .open = tracing_eval_map_open, 5291 .read = seq_read, 5292 .llseek = seq_lseek, 5293 .release = seq_release, 5294 }; 5295 5296 static inline union trace_eval_map_item * 5297 trace_eval_jmp_to_tail(union trace_eval_map_item *ptr) 5298 { 5299 /* Return tail of array given the head */ 5300 return ptr + ptr->head.length + 1; 5301 } 5302 5303 static void 5304 trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start, 5305 int len) 5306 { 5307 struct trace_eval_map **stop; 5308 struct trace_eval_map **map; 5309 union trace_eval_map_item *map_array; 5310 union trace_eval_map_item *ptr; 5311 5312 stop = start + len; 5313 5314 /* 5315 * The trace_eval_maps contains the map plus a head and tail item, 5316 * where the head holds the module and length of array, and the 5317 * tail holds a pointer to the next list. 5318 */ 5319 map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL); 5320 if (!map_array) { 5321 pr_warn("Unable to allocate trace eval mapping\n"); 5322 return; 5323 } 5324 5325 mutex_lock(&trace_eval_mutex); 5326 5327 if (!trace_eval_maps) 5328 trace_eval_maps = map_array; 5329 else { 5330 ptr = trace_eval_maps; 5331 for (;;) { 5332 ptr = trace_eval_jmp_to_tail(ptr); 5333 if (!ptr->tail.next) 5334 break; 5335 ptr = ptr->tail.next; 5336 5337 } 5338 ptr->tail.next = map_array; 5339 } 5340 map_array->head.mod = mod; 5341 map_array->head.length = len; 5342 map_array++; 5343 5344 for (map = start; (unsigned long)map < (unsigned long)stop; map++) { 5345 map_array->map = **map; 5346 map_array++; 5347 } 5348 memset(map_array, 0, sizeof(*map_array)); 5349 5350 mutex_unlock(&trace_eval_mutex); 5351 } 5352 5353 static void trace_create_eval_file(struct dentry *d_tracer) 5354 { 5355 trace_create_file("eval_map", 0444, d_tracer, 5356 NULL, &tracing_eval_map_fops); 5357 } 5358 5359 #else /* CONFIG_TRACE_EVAL_MAP_FILE */ 5360 static inline void trace_create_eval_file(struct dentry *d_tracer) { } 5361 static inline void trace_insert_eval_map_file(struct module *mod, 5362 struct trace_eval_map **start, int len) { } 5363 #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */ 5364 5365 static void trace_insert_eval_map(struct module *mod, 5366 struct trace_eval_map **start, int len) 5367 { 5368 struct trace_eval_map **map; 5369 5370 if (len <= 0) 5371 return; 5372 5373 map = start; 5374 5375 trace_event_eval_update(map, len); 5376 5377 trace_insert_eval_map_file(mod, start, len); 5378 } 5379 5380 static ssize_t 5381 tracing_set_trace_read(struct file *filp, char __user *ubuf, 5382 size_t cnt, loff_t *ppos) 5383 { 5384 struct trace_array *tr = filp->private_data; 5385 char buf[MAX_TRACER_SIZE+2]; 5386 int r; 5387 5388 mutex_lock(&trace_types_lock); 5389 r = sprintf(buf, "%s\n", tr->current_trace->name); 5390 mutex_unlock(&trace_types_lock); 5391 5392 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5393 } 5394 5395 int tracer_init(struct tracer *t, struct trace_array *tr) 5396 { 5397 tracing_reset_online_cpus(&tr->trace_buffer); 5398 return t->init(tr); 5399 } 5400 5401 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val) 5402 { 5403 int cpu; 5404 5405 for_each_tracing_cpu(cpu) 5406 per_cpu_ptr(buf->data, cpu)->entries = val; 5407 } 5408 5409 #ifdef CONFIG_TRACER_MAX_TRACE 5410 /* resize @tr's buffer to the size of @size_tr's entries */ 5411 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf, 5412 struct trace_buffer *size_buf, int cpu_id) 5413 { 5414 int cpu, ret = 0; 5415 5416 if (cpu_id == RING_BUFFER_ALL_CPUS) { 5417 for_each_tracing_cpu(cpu) { 5418 ret = ring_buffer_resize(trace_buf->buffer, 5419 per_cpu_ptr(size_buf->data, cpu)->entries, cpu); 5420 if (ret < 0) 5421 break; 5422 per_cpu_ptr(trace_buf->data, cpu)->entries = 5423 per_cpu_ptr(size_buf->data, cpu)->entries; 5424 } 5425 } else { 5426 ret = ring_buffer_resize(trace_buf->buffer, 5427 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id); 5428 if (ret == 0) 5429 per_cpu_ptr(trace_buf->data, cpu_id)->entries = 5430 per_cpu_ptr(size_buf->data, cpu_id)->entries; 5431 } 5432 5433 return ret; 5434 } 5435 #endif /* CONFIG_TRACER_MAX_TRACE */ 5436 5437 static int __tracing_resize_ring_buffer(struct trace_array *tr, 5438 unsigned long size, int cpu) 5439 { 5440 int ret; 5441 5442 /* 5443 * If kernel or user changes the size of the ring buffer 5444 * we use the size that was given, and we can forget about 5445 * expanding it later. 5446 */ 5447 ring_buffer_expanded = true; 5448 5449 /* May be called before buffers are initialized */ 5450 if (!tr->trace_buffer.buffer) 5451 return 0; 5452 5453 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu); 5454 if (ret < 0) 5455 return ret; 5456 5457 #ifdef CONFIG_TRACER_MAX_TRACE 5458 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) || 5459 !tr->current_trace->use_max_tr) 5460 goto out; 5461 5462 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu); 5463 if (ret < 0) { 5464 int r = resize_buffer_duplicate_size(&tr->trace_buffer, 5465 &tr->trace_buffer, cpu); 5466 if (r < 0) { 5467 /* 5468 * AARGH! We are left with different 5469 * size max buffer!!!! 5470 * The max buffer is our "snapshot" buffer. 5471 * When a tracer needs a snapshot (one of the 5472 * latency tracers), it swaps the max buffer 5473 * with the saved snap shot. We succeeded to 5474 * update the size of the main buffer, but failed to 5475 * update the size of the max buffer. But when we tried 5476 * to reset the main buffer to the original size, we 5477 * failed there too. This is very unlikely to 5478 * happen, but if it does, warn and kill all 5479 * tracing. 5480 */ 5481 WARN_ON(1); 5482 tracing_disabled = 1; 5483 } 5484 return ret; 5485 } 5486 5487 if (cpu == RING_BUFFER_ALL_CPUS) 5488 set_buffer_entries(&tr->max_buffer, size); 5489 else 5490 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size; 5491 5492 out: 5493 #endif /* CONFIG_TRACER_MAX_TRACE */ 5494 5495 if (cpu == RING_BUFFER_ALL_CPUS) 5496 set_buffer_entries(&tr->trace_buffer, size); 5497 else 5498 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size; 5499 5500 return ret; 5501 } 5502 5503 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr, 5504 unsigned long size, int cpu_id) 5505 { 5506 int ret = size; 5507 5508 mutex_lock(&trace_types_lock); 5509 5510 if (cpu_id != RING_BUFFER_ALL_CPUS) { 5511 /* make sure, this cpu is enabled in the mask */ 5512 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) { 5513 ret = -EINVAL; 5514 goto out; 5515 } 5516 } 5517 5518 ret = __tracing_resize_ring_buffer(tr, size, cpu_id); 5519 if (ret < 0) 5520 ret = -ENOMEM; 5521 5522 out: 5523 mutex_unlock(&trace_types_lock); 5524 5525 return ret; 5526 } 5527 5528 5529 /** 5530 * tracing_update_buffers - used by tracing facility to expand ring buffers 5531 * 5532 * To save on memory when the tracing is never used on a system with it 5533 * configured in. The ring buffers are set to a minimum size. But once 5534 * a user starts to use the tracing facility, then they need to grow 5535 * to their default size. 5536 * 5537 * This function is to be called when a tracer is about to be used. 5538 */ 5539 int tracing_update_buffers(void) 5540 { 5541 int ret = 0; 5542 5543 mutex_lock(&trace_types_lock); 5544 if (!ring_buffer_expanded) 5545 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size, 5546 RING_BUFFER_ALL_CPUS); 5547 mutex_unlock(&trace_types_lock); 5548 5549 return ret; 5550 } 5551 5552 struct trace_option_dentry; 5553 5554 static void 5555 create_trace_option_files(struct trace_array *tr, struct tracer *tracer); 5556 5557 /* 5558 * Used to clear out the tracer before deletion of an instance. 5559 * Must have trace_types_lock held. 5560 */ 5561 static void tracing_set_nop(struct trace_array *tr) 5562 { 5563 if (tr->current_trace == &nop_trace) 5564 return; 5565 5566 tr->current_trace->enabled--; 5567 5568 if (tr->current_trace->reset) 5569 tr->current_trace->reset(tr); 5570 5571 tr->current_trace = &nop_trace; 5572 } 5573 5574 static void add_tracer_options(struct trace_array *tr, struct tracer *t) 5575 { 5576 /* Only enable if the directory has been created already. */ 5577 if (!tr->dir) 5578 return; 5579 5580 create_trace_option_files(tr, t); 5581 } 5582 5583 static int tracing_set_tracer(struct trace_array *tr, const char *buf) 5584 { 5585 struct tracer *t; 5586 #ifdef CONFIG_TRACER_MAX_TRACE 5587 bool had_max_tr; 5588 #endif 5589 int ret = 0; 5590 5591 mutex_lock(&trace_types_lock); 5592 5593 if (!ring_buffer_expanded) { 5594 ret = __tracing_resize_ring_buffer(tr, trace_buf_size, 5595 RING_BUFFER_ALL_CPUS); 5596 if (ret < 0) 5597 goto out; 5598 ret = 0; 5599 } 5600 5601 for (t = trace_types; t; t = t->next) { 5602 if (strcmp(t->name, buf) == 0) 5603 break; 5604 } 5605 if (!t) { 5606 ret = -EINVAL; 5607 goto out; 5608 } 5609 if (t == tr->current_trace) 5610 goto out; 5611 5612 #ifdef CONFIG_TRACER_SNAPSHOT 5613 if (t->use_max_tr) { 5614 arch_spin_lock(&tr->max_lock); 5615 if (tr->cond_snapshot) 5616 ret = -EBUSY; 5617 arch_spin_unlock(&tr->max_lock); 5618 if (ret) 5619 goto out; 5620 } 5621 #endif 5622 /* Some tracers won't work on kernel command line */ 5623 if (system_state < SYSTEM_RUNNING && t->noboot) { 5624 pr_warn("Tracer '%s' is not allowed on command line, ignored\n", 5625 t->name); 5626 goto out; 5627 } 5628 5629 /* Some tracers are only allowed for the top level buffer */ 5630 if (!trace_ok_for_array(t, tr)) { 5631 ret = -EINVAL; 5632 goto out; 5633 } 5634 5635 /* If trace pipe files are being read, we can't change the tracer */ 5636 if (tr->current_trace->ref) { 5637 ret = -EBUSY; 5638 goto out; 5639 } 5640 5641 trace_branch_disable(); 5642 5643 tr->current_trace->enabled--; 5644 5645 if (tr->current_trace->reset) 5646 tr->current_trace->reset(tr); 5647 5648 /* Current trace needs to be nop_trace before synchronize_rcu */ 5649 tr->current_trace = &nop_trace; 5650 5651 #ifdef CONFIG_TRACER_MAX_TRACE 5652 had_max_tr = tr->allocated_snapshot; 5653 5654 if (had_max_tr && !t->use_max_tr) { 5655 /* 5656 * We need to make sure that the update_max_tr sees that 5657 * current_trace changed to nop_trace to keep it from 5658 * swapping the buffers after we resize it. 5659 * The update_max_tr is called from interrupts disabled 5660 * so a synchronized_sched() is sufficient. 5661 */ 5662 synchronize_rcu(); 5663 free_snapshot(tr); 5664 } 5665 #endif 5666 5667 #ifdef CONFIG_TRACER_MAX_TRACE 5668 if (t->use_max_tr && !had_max_tr) { 5669 ret = tracing_alloc_snapshot_instance(tr); 5670 if (ret < 0) 5671 goto out; 5672 } 5673 #endif 5674 5675 if (t->init) { 5676 ret = tracer_init(t, tr); 5677 if (ret) 5678 goto out; 5679 } 5680 5681 tr->current_trace = t; 5682 tr->current_trace->enabled++; 5683 trace_branch_enable(tr); 5684 out: 5685 mutex_unlock(&trace_types_lock); 5686 5687 return ret; 5688 } 5689 5690 static ssize_t 5691 tracing_set_trace_write(struct file *filp, const char __user *ubuf, 5692 size_t cnt, loff_t *ppos) 5693 { 5694 struct trace_array *tr = filp->private_data; 5695 char buf[MAX_TRACER_SIZE+1]; 5696 int i; 5697 size_t ret; 5698 int err; 5699 5700 ret = cnt; 5701 5702 if (cnt > MAX_TRACER_SIZE) 5703 cnt = MAX_TRACER_SIZE; 5704 5705 if (copy_from_user(buf, ubuf, cnt)) 5706 return -EFAULT; 5707 5708 buf[cnt] = 0; 5709 5710 /* strip ending whitespace. */ 5711 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--) 5712 buf[i] = 0; 5713 5714 err = tracing_set_tracer(tr, buf); 5715 if (err) 5716 return err; 5717 5718 *ppos += ret; 5719 5720 return ret; 5721 } 5722 5723 static ssize_t 5724 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf, 5725 size_t cnt, loff_t *ppos) 5726 { 5727 char buf[64]; 5728 int r; 5729 5730 r = snprintf(buf, sizeof(buf), "%ld\n", 5731 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr)); 5732 if (r > sizeof(buf)) 5733 r = sizeof(buf); 5734 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5735 } 5736 5737 static ssize_t 5738 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf, 5739 size_t cnt, loff_t *ppos) 5740 { 5741 unsigned long val; 5742 int ret; 5743 5744 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 5745 if (ret) 5746 return ret; 5747 5748 *ptr = val * 1000; 5749 5750 return cnt; 5751 } 5752 5753 static ssize_t 5754 tracing_thresh_read(struct file *filp, char __user *ubuf, 5755 size_t cnt, loff_t *ppos) 5756 { 5757 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos); 5758 } 5759 5760 static ssize_t 5761 tracing_thresh_write(struct file *filp, const char __user *ubuf, 5762 size_t cnt, loff_t *ppos) 5763 { 5764 struct trace_array *tr = filp->private_data; 5765 int ret; 5766 5767 mutex_lock(&trace_types_lock); 5768 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos); 5769 if (ret < 0) 5770 goto out; 5771 5772 if (tr->current_trace->update_thresh) { 5773 ret = tr->current_trace->update_thresh(tr); 5774 if (ret < 0) 5775 goto out; 5776 } 5777 5778 ret = cnt; 5779 out: 5780 mutex_unlock(&trace_types_lock); 5781 5782 return ret; 5783 } 5784 5785 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) 5786 5787 static ssize_t 5788 tracing_max_lat_read(struct file *filp, char __user *ubuf, 5789 size_t cnt, loff_t *ppos) 5790 { 5791 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos); 5792 } 5793 5794 static ssize_t 5795 tracing_max_lat_write(struct file *filp, const char __user *ubuf, 5796 size_t cnt, loff_t *ppos) 5797 { 5798 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos); 5799 } 5800 5801 #endif 5802 5803 static int tracing_open_pipe(struct inode *inode, struct file *filp) 5804 { 5805 struct trace_array *tr = inode->i_private; 5806 struct trace_iterator *iter; 5807 int ret = 0; 5808 5809 if (tracing_disabled) 5810 return -ENODEV; 5811 5812 if (trace_array_get(tr) < 0) 5813 return -ENODEV; 5814 5815 mutex_lock(&trace_types_lock); 5816 5817 /* create a buffer to store the information to pass to userspace */ 5818 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 5819 if (!iter) { 5820 ret = -ENOMEM; 5821 __trace_array_put(tr); 5822 goto out; 5823 } 5824 5825 trace_seq_init(&iter->seq); 5826 iter->trace = tr->current_trace; 5827 5828 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) { 5829 ret = -ENOMEM; 5830 goto fail; 5831 } 5832 5833 /* trace pipe does not show start of buffer */ 5834 cpumask_setall(iter->started); 5835 5836 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 5837 iter->iter_flags |= TRACE_FILE_LAT_FMT; 5838 5839 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 5840 if (trace_clocks[tr->clock_id].in_ns) 5841 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 5842 5843 iter->tr = tr; 5844 iter->trace_buffer = &tr->trace_buffer; 5845 iter->cpu_file = tracing_get_cpu(inode); 5846 mutex_init(&iter->mutex); 5847 filp->private_data = iter; 5848 5849 if (iter->trace->pipe_open) 5850 iter->trace->pipe_open(iter); 5851 5852 nonseekable_open(inode, filp); 5853 5854 tr->current_trace->ref++; 5855 out: 5856 mutex_unlock(&trace_types_lock); 5857 return ret; 5858 5859 fail: 5860 kfree(iter); 5861 __trace_array_put(tr); 5862 mutex_unlock(&trace_types_lock); 5863 return ret; 5864 } 5865 5866 static int tracing_release_pipe(struct inode *inode, struct file *file) 5867 { 5868 struct trace_iterator *iter = file->private_data; 5869 struct trace_array *tr = inode->i_private; 5870 5871 mutex_lock(&trace_types_lock); 5872 5873 tr->current_trace->ref--; 5874 5875 if (iter->trace->pipe_close) 5876 iter->trace->pipe_close(iter); 5877 5878 mutex_unlock(&trace_types_lock); 5879 5880 free_cpumask_var(iter->started); 5881 mutex_destroy(&iter->mutex); 5882 kfree(iter); 5883 5884 trace_array_put(tr); 5885 5886 return 0; 5887 } 5888 5889 static __poll_t 5890 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table) 5891 { 5892 struct trace_array *tr = iter->tr; 5893 5894 /* Iterators are static, they should be filled or empty */ 5895 if (trace_buffer_iter(iter, iter->cpu_file)) 5896 return EPOLLIN | EPOLLRDNORM; 5897 5898 if (tr->trace_flags & TRACE_ITER_BLOCK) 5899 /* 5900 * Always select as readable when in blocking mode 5901 */ 5902 return EPOLLIN | EPOLLRDNORM; 5903 else 5904 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file, 5905 filp, poll_table); 5906 } 5907 5908 static __poll_t 5909 tracing_poll_pipe(struct file *filp, poll_table *poll_table) 5910 { 5911 struct trace_iterator *iter = filp->private_data; 5912 5913 return trace_poll(iter, filp, poll_table); 5914 } 5915 5916 /* Must be called with iter->mutex held. */ 5917 static int tracing_wait_pipe(struct file *filp) 5918 { 5919 struct trace_iterator *iter = filp->private_data; 5920 int ret; 5921 5922 while (trace_empty(iter)) { 5923 5924 if ((filp->f_flags & O_NONBLOCK)) { 5925 return -EAGAIN; 5926 } 5927 5928 /* 5929 * We block until we read something and tracing is disabled. 5930 * We still block if tracing is disabled, but we have never 5931 * read anything. This allows a user to cat this file, and 5932 * then enable tracing. But after we have read something, 5933 * we give an EOF when tracing is again disabled. 5934 * 5935 * iter->pos will be 0 if we haven't read anything. 5936 */ 5937 if (!tracer_tracing_is_on(iter->tr) && iter->pos) 5938 break; 5939 5940 mutex_unlock(&iter->mutex); 5941 5942 ret = wait_on_pipe(iter, 0); 5943 5944 mutex_lock(&iter->mutex); 5945 5946 if (ret) 5947 return ret; 5948 } 5949 5950 return 1; 5951 } 5952 5953 /* 5954 * Consumer reader. 5955 */ 5956 static ssize_t 5957 tracing_read_pipe(struct file *filp, char __user *ubuf, 5958 size_t cnt, loff_t *ppos) 5959 { 5960 struct trace_iterator *iter = filp->private_data; 5961 ssize_t sret; 5962 5963 /* 5964 * Avoid more than one consumer on a single file descriptor 5965 * This is just a matter of traces coherency, the ring buffer itself 5966 * is protected. 5967 */ 5968 mutex_lock(&iter->mutex); 5969 5970 /* return any leftover data */ 5971 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 5972 if (sret != -EBUSY) 5973 goto out; 5974 5975 trace_seq_init(&iter->seq); 5976 5977 if (iter->trace->read) { 5978 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); 5979 if (sret) 5980 goto out; 5981 } 5982 5983 waitagain: 5984 sret = tracing_wait_pipe(filp); 5985 if (sret <= 0) 5986 goto out; 5987 5988 /* stop when tracing is finished */ 5989 if (trace_empty(iter)) { 5990 sret = 0; 5991 goto out; 5992 } 5993 5994 if (cnt >= PAGE_SIZE) 5995 cnt = PAGE_SIZE - 1; 5996 5997 /* reset all but tr, trace, and overruns */ 5998 memset(&iter->seq, 0, 5999 sizeof(struct trace_iterator) - 6000 offsetof(struct trace_iterator, seq)); 6001 cpumask_clear(iter->started); 6002 iter->pos = -1; 6003 6004 trace_event_read_lock(); 6005 trace_access_lock(iter->cpu_file); 6006 while (trace_find_next_entry_inc(iter) != NULL) { 6007 enum print_line_t ret; 6008 int save_len = iter->seq.seq.len; 6009 6010 ret = print_trace_line(iter); 6011 if (ret == TRACE_TYPE_PARTIAL_LINE) { 6012 /* don't print partial lines */ 6013 iter->seq.seq.len = save_len; 6014 break; 6015 } 6016 if (ret != TRACE_TYPE_NO_CONSUME) 6017 trace_consume(iter); 6018 6019 if (trace_seq_used(&iter->seq) >= cnt) 6020 break; 6021 6022 /* 6023 * Setting the full flag means we reached the trace_seq buffer 6024 * size and we should leave by partial output condition above. 6025 * One of the trace_seq_* functions is not used properly. 6026 */ 6027 WARN_ONCE(iter->seq.full, "full flag set for trace type %d", 6028 iter->ent->type); 6029 } 6030 trace_access_unlock(iter->cpu_file); 6031 trace_event_read_unlock(); 6032 6033 /* Now copy what we have to the user */ 6034 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 6035 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq)) 6036 trace_seq_init(&iter->seq); 6037 6038 /* 6039 * If there was nothing to send to user, in spite of consuming trace 6040 * entries, go back to wait for more entries. 6041 */ 6042 if (sret == -EBUSY) 6043 goto waitagain; 6044 6045 out: 6046 mutex_unlock(&iter->mutex); 6047 6048 return sret; 6049 } 6050 6051 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd, 6052 unsigned int idx) 6053 { 6054 __free_page(spd->pages[idx]); 6055 } 6056 6057 static const struct pipe_buf_operations tracing_pipe_buf_ops = { 6058 .confirm = generic_pipe_buf_confirm, 6059 .release = generic_pipe_buf_release, 6060 .steal = generic_pipe_buf_steal, 6061 .get = generic_pipe_buf_get, 6062 }; 6063 6064 static size_t 6065 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter) 6066 { 6067 size_t count; 6068 int save_len; 6069 int ret; 6070 6071 /* Seq buffer is page-sized, exactly what we need. */ 6072 for (;;) { 6073 save_len = iter->seq.seq.len; 6074 ret = print_trace_line(iter); 6075 6076 if (trace_seq_has_overflowed(&iter->seq)) { 6077 iter->seq.seq.len = save_len; 6078 break; 6079 } 6080 6081 /* 6082 * This should not be hit, because it should only 6083 * be set if the iter->seq overflowed. But check it 6084 * anyway to be safe. 6085 */ 6086 if (ret == TRACE_TYPE_PARTIAL_LINE) { 6087 iter->seq.seq.len = save_len; 6088 break; 6089 } 6090 6091 count = trace_seq_used(&iter->seq) - save_len; 6092 if (rem < count) { 6093 rem = 0; 6094 iter->seq.seq.len = save_len; 6095 break; 6096 } 6097 6098 if (ret != TRACE_TYPE_NO_CONSUME) 6099 trace_consume(iter); 6100 rem -= count; 6101 if (!trace_find_next_entry_inc(iter)) { 6102 rem = 0; 6103 iter->ent = NULL; 6104 break; 6105 } 6106 } 6107 6108 return rem; 6109 } 6110 6111 static ssize_t tracing_splice_read_pipe(struct file *filp, 6112 loff_t *ppos, 6113 struct pipe_inode_info *pipe, 6114 size_t len, 6115 unsigned int flags) 6116 { 6117 struct page *pages_def[PIPE_DEF_BUFFERS]; 6118 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 6119 struct trace_iterator *iter = filp->private_data; 6120 struct splice_pipe_desc spd = { 6121 .pages = pages_def, 6122 .partial = partial_def, 6123 .nr_pages = 0, /* This gets updated below. */ 6124 .nr_pages_max = PIPE_DEF_BUFFERS, 6125 .ops = &tracing_pipe_buf_ops, 6126 .spd_release = tracing_spd_release_pipe, 6127 }; 6128 ssize_t ret; 6129 size_t rem; 6130 unsigned int i; 6131 6132 if (splice_grow_spd(pipe, &spd)) 6133 return -ENOMEM; 6134 6135 mutex_lock(&iter->mutex); 6136 6137 if (iter->trace->splice_read) { 6138 ret = iter->trace->splice_read(iter, filp, 6139 ppos, pipe, len, flags); 6140 if (ret) 6141 goto out_err; 6142 } 6143 6144 ret = tracing_wait_pipe(filp); 6145 if (ret <= 0) 6146 goto out_err; 6147 6148 if (!iter->ent && !trace_find_next_entry_inc(iter)) { 6149 ret = -EFAULT; 6150 goto out_err; 6151 } 6152 6153 trace_event_read_lock(); 6154 trace_access_lock(iter->cpu_file); 6155 6156 /* Fill as many pages as possible. */ 6157 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) { 6158 spd.pages[i] = alloc_page(GFP_KERNEL); 6159 if (!spd.pages[i]) 6160 break; 6161 6162 rem = tracing_fill_pipe_page(rem, iter); 6163 6164 /* Copy the data into the page, so we can start over. */ 6165 ret = trace_seq_to_buffer(&iter->seq, 6166 page_address(spd.pages[i]), 6167 trace_seq_used(&iter->seq)); 6168 if (ret < 0) { 6169 __free_page(spd.pages[i]); 6170 break; 6171 } 6172 spd.partial[i].offset = 0; 6173 spd.partial[i].len = trace_seq_used(&iter->seq); 6174 6175 trace_seq_init(&iter->seq); 6176 } 6177 6178 trace_access_unlock(iter->cpu_file); 6179 trace_event_read_unlock(); 6180 mutex_unlock(&iter->mutex); 6181 6182 spd.nr_pages = i; 6183 6184 if (i) 6185 ret = splice_to_pipe(pipe, &spd); 6186 else 6187 ret = 0; 6188 out: 6189 splice_shrink_spd(&spd); 6190 return ret; 6191 6192 out_err: 6193 mutex_unlock(&iter->mutex); 6194 goto out; 6195 } 6196 6197 static ssize_t 6198 tracing_entries_read(struct file *filp, char __user *ubuf, 6199 size_t cnt, loff_t *ppos) 6200 { 6201 struct inode *inode = file_inode(filp); 6202 struct trace_array *tr = inode->i_private; 6203 int cpu = tracing_get_cpu(inode); 6204 char buf[64]; 6205 int r = 0; 6206 ssize_t ret; 6207 6208 mutex_lock(&trace_types_lock); 6209 6210 if (cpu == RING_BUFFER_ALL_CPUS) { 6211 int cpu, buf_size_same; 6212 unsigned long size; 6213 6214 size = 0; 6215 buf_size_same = 1; 6216 /* check if all cpu sizes are same */ 6217 for_each_tracing_cpu(cpu) { 6218 /* fill in the size from first enabled cpu */ 6219 if (size == 0) 6220 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries; 6221 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) { 6222 buf_size_same = 0; 6223 break; 6224 } 6225 } 6226 6227 if (buf_size_same) { 6228 if (!ring_buffer_expanded) 6229 r = sprintf(buf, "%lu (expanded: %lu)\n", 6230 size >> 10, 6231 trace_buf_size >> 10); 6232 else 6233 r = sprintf(buf, "%lu\n", size >> 10); 6234 } else 6235 r = sprintf(buf, "X\n"); 6236 } else 6237 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10); 6238 6239 mutex_unlock(&trace_types_lock); 6240 6241 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 6242 return ret; 6243 } 6244 6245 static ssize_t 6246 tracing_entries_write(struct file *filp, const char __user *ubuf, 6247 size_t cnt, loff_t *ppos) 6248 { 6249 struct inode *inode = file_inode(filp); 6250 struct trace_array *tr = inode->i_private; 6251 unsigned long val; 6252 int ret; 6253 6254 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 6255 if (ret) 6256 return ret; 6257 6258 /* must have at least 1 entry */ 6259 if (!val) 6260 return -EINVAL; 6261 6262 /* value is in KB */ 6263 val <<= 10; 6264 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode)); 6265 if (ret < 0) 6266 return ret; 6267 6268 *ppos += cnt; 6269 6270 return cnt; 6271 } 6272 6273 static ssize_t 6274 tracing_total_entries_read(struct file *filp, char __user *ubuf, 6275 size_t cnt, loff_t *ppos) 6276 { 6277 struct trace_array *tr = filp->private_data; 6278 char buf[64]; 6279 int r, cpu; 6280 unsigned long size = 0, expanded_size = 0; 6281 6282 mutex_lock(&trace_types_lock); 6283 for_each_tracing_cpu(cpu) { 6284 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10; 6285 if (!ring_buffer_expanded) 6286 expanded_size += trace_buf_size >> 10; 6287 } 6288 if (ring_buffer_expanded) 6289 r = sprintf(buf, "%lu\n", size); 6290 else 6291 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size); 6292 mutex_unlock(&trace_types_lock); 6293 6294 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 6295 } 6296 6297 static ssize_t 6298 tracing_free_buffer_write(struct file *filp, const char __user *ubuf, 6299 size_t cnt, loff_t *ppos) 6300 { 6301 /* 6302 * There is no need to read what the user has written, this function 6303 * is just to make sure that there is no error when "echo" is used 6304 */ 6305 6306 *ppos += cnt; 6307 6308 return cnt; 6309 } 6310 6311 static int 6312 tracing_free_buffer_release(struct inode *inode, struct file *filp) 6313 { 6314 struct trace_array *tr = inode->i_private; 6315 6316 /* disable tracing ? */ 6317 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE) 6318 tracer_tracing_off(tr); 6319 /* resize the ring buffer to 0 */ 6320 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS); 6321 6322 trace_array_put(tr); 6323 6324 return 0; 6325 } 6326 6327 static ssize_t 6328 tracing_mark_write(struct file *filp, const char __user *ubuf, 6329 size_t cnt, loff_t *fpos) 6330 { 6331 struct trace_array *tr = filp->private_data; 6332 struct ring_buffer_event *event; 6333 enum event_trigger_type tt = ETT_NONE; 6334 struct ring_buffer *buffer; 6335 struct print_entry *entry; 6336 unsigned long irq_flags; 6337 ssize_t written; 6338 int size; 6339 int len; 6340 6341 /* Used in tracing_mark_raw_write() as well */ 6342 #define FAULTED_STR "<faulted>" 6343 #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */ 6344 6345 if (tracing_disabled) 6346 return -EINVAL; 6347 6348 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) 6349 return -EINVAL; 6350 6351 if (cnt > TRACE_BUF_SIZE) 6352 cnt = TRACE_BUF_SIZE; 6353 6354 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); 6355 6356 local_save_flags(irq_flags); 6357 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */ 6358 6359 /* If less than "<faulted>", then make sure we can still add that */ 6360 if (cnt < FAULTED_SIZE) 6361 size += FAULTED_SIZE - cnt; 6362 6363 buffer = tr->trace_buffer.buffer; 6364 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 6365 irq_flags, preempt_count()); 6366 if (unlikely(!event)) 6367 /* Ring buffer disabled, return as if not open for write */ 6368 return -EBADF; 6369 6370 entry = ring_buffer_event_data(event); 6371 entry->ip = _THIS_IP_; 6372 6373 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt); 6374 if (len) { 6375 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE); 6376 cnt = FAULTED_SIZE; 6377 written = -EFAULT; 6378 } else 6379 written = cnt; 6380 len = cnt; 6381 6382 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) { 6383 /* do not add \n before testing triggers, but add \0 */ 6384 entry->buf[cnt] = '\0'; 6385 tt = event_triggers_call(tr->trace_marker_file, entry, event); 6386 } 6387 6388 if (entry->buf[cnt - 1] != '\n') { 6389 entry->buf[cnt] = '\n'; 6390 entry->buf[cnt + 1] = '\0'; 6391 } else 6392 entry->buf[cnt] = '\0'; 6393 6394 __buffer_unlock_commit(buffer, event); 6395 6396 if (tt) 6397 event_triggers_post_call(tr->trace_marker_file, tt); 6398 6399 if (written > 0) 6400 *fpos += written; 6401 6402 return written; 6403 } 6404 6405 /* Limit it for now to 3K (including tag) */ 6406 #define RAW_DATA_MAX_SIZE (1024*3) 6407 6408 static ssize_t 6409 tracing_mark_raw_write(struct file *filp, const char __user *ubuf, 6410 size_t cnt, loff_t *fpos) 6411 { 6412 struct trace_array *tr = filp->private_data; 6413 struct ring_buffer_event *event; 6414 struct ring_buffer *buffer; 6415 struct raw_data_entry *entry; 6416 unsigned long irq_flags; 6417 ssize_t written; 6418 int size; 6419 int len; 6420 6421 #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int)) 6422 6423 if (tracing_disabled) 6424 return -EINVAL; 6425 6426 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) 6427 return -EINVAL; 6428 6429 /* The marker must at least have a tag id */ 6430 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE) 6431 return -EINVAL; 6432 6433 if (cnt > TRACE_BUF_SIZE) 6434 cnt = TRACE_BUF_SIZE; 6435 6436 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); 6437 6438 local_save_flags(irq_flags); 6439 size = sizeof(*entry) + cnt; 6440 if (cnt < FAULT_SIZE_ID) 6441 size += FAULT_SIZE_ID - cnt; 6442 6443 buffer = tr->trace_buffer.buffer; 6444 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size, 6445 irq_flags, preempt_count()); 6446 if (!event) 6447 /* Ring buffer disabled, return as if not open for write */ 6448 return -EBADF; 6449 6450 entry = ring_buffer_event_data(event); 6451 6452 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt); 6453 if (len) { 6454 entry->id = -1; 6455 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE); 6456 written = -EFAULT; 6457 } else 6458 written = cnt; 6459 6460 __buffer_unlock_commit(buffer, event); 6461 6462 if (written > 0) 6463 *fpos += written; 6464 6465 return written; 6466 } 6467 6468 static int tracing_clock_show(struct seq_file *m, void *v) 6469 { 6470 struct trace_array *tr = m->private; 6471 int i; 6472 6473 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) 6474 seq_printf(m, 6475 "%s%s%s%s", i ? " " : "", 6476 i == tr->clock_id ? "[" : "", trace_clocks[i].name, 6477 i == tr->clock_id ? "]" : ""); 6478 seq_putc(m, '\n'); 6479 6480 return 0; 6481 } 6482 6483 int tracing_set_clock(struct trace_array *tr, const char *clockstr) 6484 { 6485 int i; 6486 6487 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) { 6488 if (strcmp(trace_clocks[i].name, clockstr) == 0) 6489 break; 6490 } 6491 if (i == ARRAY_SIZE(trace_clocks)) 6492 return -EINVAL; 6493 6494 mutex_lock(&trace_types_lock); 6495 6496 tr->clock_id = i; 6497 6498 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func); 6499 6500 /* 6501 * New clock may not be consistent with the previous clock. 6502 * Reset the buffer so that it doesn't have incomparable timestamps. 6503 */ 6504 tracing_reset_online_cpus(&tr->trace_buffer); 6505 6506 #ifdef CONFIG_TRACER_MAX_TRACE 6507 if (tr->max_buffer.buffer) 6508 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func); 6509 tracing_reset_online_cpus(&tr->max_buffer); 6510 #endif 6511 6512 mutex_unlock(&trace_types_lock); 6513 6514 return 0; 6515 } 6516 6517 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf, 6518 size_t cnt, loff_t *fpos) 6519 { 6520 struct seq_file *m = filp->private_data; 6521 struct trace_array *tr = m->private; 6522 char buf[64]; 6523 const char *clockstr; 6524 int ret; 6525 6526 if (cnt >= sizeof(buf)) 6527 return -EINVAL; 6528 6529 if (copy_from_user(buf, ubuf, cnt)) 6530 return -EFAULT; 6531 6532 buf[cnt] = 0; 6533 6534 clockstr = strstrip(buf); 6535 6536 ret = tracing_set_clock(tr, clockstr); 6537 if (ret) 6538 return ret; 6539 6540 *fpos += cnt; 6541 6542 return cnt; 6543 } 6544 6545 static int tracing_clock_open(struct inode *inode, struct file *file) 6546 { 6547 struct trace_array *tr = inode->i_private; 6548 int ret; 6549 6550 if (tracing_disabled) 6551 return -ENODEV; 6552 6553 if (trace_array_get(tr)) 6554 return -ENODEV; 6555 6556 ret = single_open(file, tracing_clock_show, inode->i_private); 6557 if (ret < 0) 6558 trace_array_put(tr); 6559 6560 return ret; 6561 } 6562 6563 static int tracing_time_stamp_mode_show(struct seq_file *m, void *v) 6564 { 6565 struct trace_array *tr = m->private; 6566 6567 mutex_lock(&trace_types_lock); 6568 6569 if (ring_buffer_time_stamp_abs(tr->trace_buffer.buffer)) 6570 seq_puts(m, "delta [absolute]\n"); 6571 else 6572 seq_puts(m, "[delta] absolute\n"); 6573 6574 mutex_unlock(&trace_types_lock); 6575 6576 return 0; 6577 } 6578 6579 static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file) 6580 { 6581 struct trace_array *tr = inode->i_private; 6582 int ret; 6583 6584 if (tracing_disabled) 6585 return -ENODEV; 6586 6587 if (trace_array_get(tr)) 6588 return -ENODEV; 6589 6590 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private); 6591 if (ret < 0) 6592 trace_array_put(tr); 6593 6594 return ret; 6595 } 6596 6597 int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs) 6598 { 6599 int ret = 0; 6600 6601 mutex_lock(&trace_types_lock); 6602 6603 if (abs && tr->time_stamp_abs_ref++) 6604 goto out; 6605 6606 if (!abs) { 6607 if (WARN_ON_ONCE(!tr->time_stamp_abs_ref)) { 6608 ret = -EINVAL; 6609 goto out; 6610 } 6611 6612 if (--tr->time_stamp_abs_ref) 6613 goto out; 6614 } 6615 6616 ring_buffer_set_time_stamp_abs(tr->trace_buffer.buffer, abs); 6617 6618 #ifdef CONFIG_TRACER_MAX_TRACE 6619 if (tr->max_buffer.buffer) 6620 ring_buffer_set_time_stamp_abs(tr->max_buffer.buffer, abs); 6621 #endif 6622 out: 6623 mutex_unlock(&trace_types_lock); 6624 6625 return ret; 6626 } 6627 6628 struct ftrace_buffer_info { 6629 struct trace_iterator iter; 6630 void *spare; 6631 unsigned int spare_cpu; 6632 unsigned int read; 6633 }; 6634 6635 #ifdef CONFIG_TRACER_SNAPSHOT 6636 static int tracing_snapshot_open(struct inode *inode, struct file *file) 6637 { 6638 struct trace_array *tr = inode->i_private; 6639 struct trace_iterator *iter; 6640 struct seq_file *m; 6641 int ret = 0; 6642 6643 if (trace_array_get(tr) < 0) 6644 return -ENODEV; 6645 6646 if (file->f_mode & FMODE_READ) { 6647 iter = __tracing_open(inode, file, true); 6648 if (IS_ERR(iter)) 6649 ret = PTR_ERR(iter); 6650 } else { 6651 /* Writes still need the seq_file to hold the private data */ 6652 ret = -ENOMEM; 6653 m = kzalloc(sizeof(*m), GFP_KERNEL); 6654 if (!m) 6655 goto out; 6656 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 6657 if (!iter) { 6658 kfree(m); 6659 goto out; 6660 } 6661 ret = 0; 6662 6663 iter->tr = tr; 6664 iter->trace_buffer = &tr->max_buffer; 6665 iter->cpu_file = tracing_get_cpu(inode); 6666 m->private = iter; 6667 file->private_data = m; 6668 } 6669 out: 6670 if (ret < 0) 6671 trace_array_put(tr); 6672 6673 return ret; 6674 } 6675 6676 static ssize_t 6677 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt, 6678 loff_t *ppos) 6679 { 6680 struct seq_file *m = filp->private_data; 6681 struct trace_iterator *iter = m->private; 6682 struct trace_array *tr = iter->tr; 6683 unsigned long val; 6684 int ret; 6685 6686 ret = tracing_update_buffers(); 6687 if (ret < 0) 6688 return ret; 6689 6690 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 6691 if (ret) 6692 return ret; 6693 6694 mutex_lock(&trace_types_lock); 6695 6696 if (tr->current_trace->use_max_tr) { 6697 ret = -EBUSY; 6698 goto out; 6699 } 6700 6701 arch_spin_lock(&tr->max_lock); 6702 if (tr->cond_snapshot) 6703 ret = -EBUSY; 6704 arch_spin_unlock(&tr->max_lock); 6705 if (ret) 6706 goto out; 6707 6708 switch (val) { 6709 case 0: 6710 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 6711 ret = -EINVAL; 6712 break; 6713 } 6714 if (tr->allocated_snapshot) 6715 free_snapshot(tr); 6716 break; 6717 case 1: 6718 /* Only allow per-cpu swap if the ring buffer supports it */ 6719 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP 6720 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 6721 ret = -EINVAL; 6722 break; 6723 } 6724 #endif 6725 if (tr->allocated_snapshot) 6726 ret = resize_buffer_duplicate_size(&tr->max_buffer, 6727 &tr->trace_buffer, iter->cpu_file); 6728 else 6729 ret = tracing_alloc_snapshot_instance(tr); 6730 if (ret < 0) 6731 break; 6732 local_irq_disable(); 6733 /* Now, we're going to swap */ 6734 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 6735 update_max_tr(tr, current, smp_processor_id(), NULL); 6736 else 6737 update_max_tr_single(tr, current, iter->cpu_file); 6738 local_irq_enable(); 6739 break; 6740 default: 6741 if (tr->allocated_snapshot) { 6742 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 6743 tracing_reset_online_cpus(&tr->max_buffer); 6744 else 6745 tracing_reset(&tr->max_buffer, iter->cpu_file); 6746 } 6747 break; 6748 } 6749 6750 if (ret >= 0) { 6751 *ppos += cnt; 6752 ret = cnt; 6753 } 6754 out: 6755 mutex_unlock(&trace_types_lock); 6756 return ret; 6757 } 6758 6759 static int tracing_snapshot_release(struct inode *inode, struct file *file) 6760 { 6761 struct seq_file *m = file->private_data; 6762 int ret; 6763 6764 ret = tracing_release(inode, file); 6765 6766 if (file->f_mode & FMODE_READ) 6767 return ret; 6768 6769 /* If write only, the seq_file is just a stub */ 6770 if (m) 6771 kfree(m->private); 6772 kfree(m); 6773 6774 return 0; 6775 } 6776 6777 static int tracing_buffers_open(struct inode *inode, struct file *filp); 6778 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, 6779 size_t count, loff_t *ppos); 6780 static int tracing_buffers_release(struct inode *inode, struct file *file); 6781 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos, 6782 struct pipe_inode_info *pipe, size_t len, unsigned int flags); 6783 6784 static int snapshot_raw_open(struct inode *inode, struct file *filp) 6785 { 6786 struct ftrace_buffer_info *info; 6787 int ret; 6788 6789 ret = tracing_buffers_open(inode, filp); 6790 if (ret < 0) 6791 return ret; 6792 6793 info = filp->private_data; 6794 6795 if (info->iter.trace->use_max_tr) { 6796 tracing_buffers_release(inode, filp); 6797 return -EBUSY; 6798 } 6799 6800 info->iter.snapshot = true; 6801 info->iter.trace_buffer = &info->iter.tr->max_buffer; 6802 6803 return ret; 6804 } 6805 6806 #endif /* CONFIG_TRACER_SNAPSHOT */ 6807 6808 6809 static const struct file_operations tracing_thresh_fops = { 6810 .open = tracing_open_generic, 6811 .read = tracing_thresh_read, 6812 .write = tracing_thresh_write, 6813 .llseek = generic_file_llseek, 6814 }; 6815 6816 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) 6817 static const struct file_operations tracing_max_lat_fops = { 6818 .open = tracing_open_generic, 6819 .read = tracing_max_lat_read, 6820 .write = tracing_max_lat_write, 6821 .llseek = generic_file_llseek, 6822 }; 6823 #endif 6824 6825 static const struct file_operations set_tracer_fops = { 6826 .open = tracing_open_generic, 6827 .read = tracing_set_trace_read, 6828 .write = tracing_set_trace_write, 6829 .llseek = generic_file_llseek, 6830 }; 6831 6832 static const struct file_operations tracing_pipe_fops = { 6833 .open = tracing_open_pipe, 6834 .poll = tracing_poll_pipe, 6835 .read = tracing_read_pipe, 6836 .splice_read = tracing_splice_read_pipe, 6837 .release = tracing_release_pipe, 6838 .llseek = no_llseek, 6839 }; 6840 6841 static const struct file_operations tracing_entries_fops = { 6842 .open = tracing_open_generic_tr, 6843 .read = tracing_entries_read, 6844 .write = tracing_entries_write, 6845 .llseek = generic_file_llseek, 6846 .release = tracing_release_generic_tr, 6847 }; 6848 6849 static const struct file_operations tracing_total_entries_fops = { 6850 .open = tracing_open_generic_tr, 6851 .read = tracing_total_entries_read, 6852 .llseek = generic_file_llseek, 6853 .release = tracing_release_generic_tr, 6854 }; 6855 6856 static const struct file_operations tracing_free_buffer_fops = { 6857 .open = tracing_open_generic_tr, 6858 .write = tracing_free_buffer_write, 6859 .release = tracing_free_buffer_release, 6860 }; 6861 6862 static const struct file_operations tracing_mark_fops = { 6863 .open = tracing_open_generic_tr, 6864 .write = tracing_mark_write, 6865 .llseek = generic_file_llseek, 6866 .release = tracing_release_generic_tr, 6867 }; 6868 6869 static const struct file_operations tracing_mark_raw_fops = { 6870 .open = tracing_open_generic_tr, 6871 .write = tracing_mark_raw_write, 6872 .llseek = generic_file_llseek, 6873 .release = tracing_release_generic_tr, 6874 }; 6875 6876 static const struct file_operations trace_clock_fops = { 6877 .open = tracing_clock_open, 6878 .read = seq_read, 6879 .llseek = seq_lseek, 6880 .release = tracing_single_release_tr, 6881 .write = tracing_clock_write, 6882 }; 6883 6884 static const struct file_operations trace_time_stamp_mode_fops = { 6885 .open = tracing_time_stamp_mode_open, 6886 .read = seq_read, 6887 .llseek = seq_lseek, 6888 .release = tracing_single_release_tr, 6889 }; 6890 6891 #ifdef CONFIG_TRACER_SNAPSHOT 6892 static const struct file_operations snapshot_fops = { 6893 .open = tracing_snapshot_open, 6894 .read = seq_read, 6895 .write = tracing_snapshot_write, 6896 .llseek = tracing_lseek, 6897 .release = tracing_snapshot_release, 6898 }; 6899 6900 static const struct file_operations snapshot_raw_fops = { 6901 .open = snapshot_raw_open, 6902 .read = tracing_buffers_read, 6903 .release = tracing_buffers_release, 6904 .splice_read = tracing_buffers_splice_read, 6905 .llseek = no_llseek, 6906 }; 6907 6908 #endif /* CONFIG_TRACER_SNAPSHOT */ 6909 6910 #define TRACING_LOG_ERRS_MAX 8 6911 #define TRACING_LOG_LOC_MAX 128 6912 6913 #define CMD_PREFIX " Command: " 6914 6915 struct err_info { 6916 const char **errs; /* ptr to loc-specific array of err strings */ 6917 u8 type; /* index into errs -> specific err string */ 6918 u8 pos; /* MAX_FILTER_STR_VAL = 256 */ 6919 u64 ts; 6920 }; 6921 6922 struct tracing_log_err { 6923 struct list_head list; 6924 struct err_info info; 6925 char loc[TRACING_LOG_LOC_MAX]; /* err location */ 6926 char cmd[MAX_FILTER_STR_VAL]; /* what caused err */ 6927 }; 6928 6929 static DEFINE_MUTEX(tracing_err_log_lock); 6930 6931 static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr) 6932 { 6933 struct tracing_log_err *err; 6934 6935 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) { 6936 err = kzalloc(sizeof(*err), GFP_KERNEL); 6937 if (!err) 6938 err = ERR_PTR(-ENOMEM); 6939 tr->n_err_log_entries++; 6940 6941 return err; 6942 } 6943 6944 err = list_first_entry(&tr->err_log, struct tracing_log_err, list); 6945 list_del(&err->list); 6946 6947 return err; 6948 } 6949 6950 /** 6951 * err_pos - find the position of a string within a command for error careting 6952 * @cmd: The tracing command that caused the error 6953 * @str: The string to position the caret at within @cmd 6954 * 6955 * Finds the position of the first occurence of @str within @cmd. The 6956 * return value can be passed to tracing_log_err() for caret placement 6957 * within @cmd. 6958 * 6959 * Returns the index within @cmd of the first occurence of @str or 0 6960 * if @str was not found. 6961 */ 6962 unsigned int err_pos(char *cmd, const char *str) 6963 { 6964 char *found; 6965 6966 if (WARN_ON(!strlen(cmd))) 6967 return 0; 6968 6969 found = strstr(cmd, str); 6970 if (found) 6971 return found - cmd; 6972 6973 return 0; 6974 } 6975 6976 /** 6977 * tracing_log_err - write an error to the tracing error log 6978 * @tr: The associated trace array for the error (NULL for top level array) 6979 * @loc: A string describing where the error occurred 6980 * @cmd: The tracing command that caused the error 6981 * @errs: The array of loc-specific static error strings 6982 * @type: The index into errs[], which produces the specific static err string 6983 * @pos: The position the caret should be placed in the cmd 6984 * 6985 * Writes an error into tracing/error_log of the form: 6986 * 6987 * <loc>: error: <text> 6988 * Command: <cmd> 6989 * ^ 6990 * 6991 * tracing/error_log is a small log file containing the last 6992 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated 6993 * unless there has been a tracing error, and the error log can be 6994 * cleared and have its memory freed by writing the empty string in 6995 * truncation mode to it i.e. echo > tracing/error_log. 6996 * 6997 * NOTE: the @errs array along with the @type param are used to 6998 * produce a static error string - this string is not copied and saved 6999 * when the error is logged - only a pointer to it is saved. See 7000 * existing callers for examples of how static strings are typically 7001 * defined for use with tracing_log_err(). 7002 */ 7003 void tracing_log_err(struct trace_array *tr, 7004 const char *loc, const char *cmd, 7005 const char **errs, u8 type, u8 pos) 7006 { 7007 struct tracing_log_err *err; 7008 7009 if (!tr) 7010 tr = &global_trace; 7011 7012 mutex_lock(&tracing_err_log_lock); 7013 err = get_tracing_log_err(tr); 7014 if (PTR_ERR(err) == -ENOMEM) { 7015 mutex_unlock(&tracing_err_log_lock); 7016 return; 7017 } 7018 7019 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc); 7020 snprintf(err->cmd, MAX_FILTER_STR_VAL,"\n" CMD_PREFIX "%s\n", cmd); 7021 7022 err->info.errs = errs; 7023 err->info.type = type; 7024 err->info.pos = pos; 7025 err->info.ts = local_clock(); 7026 7027 list_add_tail(&err->list, &tr->err_log); 7028 mutex_unlock(&tracing_err_log_lock); 7029 } 7030 7031 static void clear_tracing_err_log(struct trace_array *tr) 7032 { 7033 struct tracing_log_err *err, *next; 7034 7035 mutex_lock(&tracing_err_log_lock); 7036 list_for_each_entry_safe(err, next, &tr->err_log, list) { 7037 list_del(&err->list); 7038 kfree(err); 7039 } 7040 7041 tr->n_err_log_entries = 0; 7042 mutex_unlock(&tracing_err_log_lock); 7043 } 7044 7045 static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos) 7046 { 7047 struct trace_array *tr = m->private; 7048 7049 mutex_lock(&tracing_err_log_lock); 7050 7051 return seq_list_start(&tr->err_log, *pos); 7052 } 7053 7054 static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos) 7055 { 7056 struct trace_array *tr = m->private; 7057 7058 return seq_list_next(v, &tr->err_log, pos); 7059 } 7060 7061 static void tracing_err_log_seq_stop(struct seq_file *m, void *v) 7062 { 7063 mutex_unlock(&tracing_err_log_lock); 7064 } 7065 7066 static void tracing_err_log_show_pos(struct seq_file *m, u8 pos) 7067 { 7068 u8 i; 7069 7070 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++) 7071 seq_putc(m, ' '); 7072 for (i = 0; i < pos; i++) 7073 seq_putc(m, ' '); 7074 seq_puts(m, "^\n"); 7075 } 7076 7077 static int tracing_err_log_seq_show(struct seq_file *m, void *v) 7078 { 7079 struct tracing_log_err *err = v; 7080 7081 if (err) { 7082 const char *err_text = err->info.errs[err->info.type]; 7083 u64 sec = err->info.ts; 7084 u32 nsec; 7085 7086 nsec = do_div(sec, NSEC_PER_SEC); 7087 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000, 7088 err->loc, err_text); 7089 seq_printf(m, "%s", err->cmd); 7090 tracing_err_log_show_pos(m, err->info.pos); 7091 } 7092 7093 return 0; 7094 } 7095 7096 static const struct seq_operations tracing_err_log_seq_ops = { 7097 .start = tracing_err_log_seq_start, 7098 .next = tracing_err_log_seq_next, 7099 .stop = tracing_err_log_seq_stop, 7100 .show = tracing_err_log_seq_show 7101 }; 7102 7103 static int tracing_err_log_open(struct inode *inode, struct file *file) 7104 { 7105 struct trace_array *tr = inode->i_private; 7106 int ret = 0; 7107 7108 if (trace_array_get(tr) < 0) 7109 return -ENODEV; 7110 7111 /* If this file was opened for write, then erase contents */ 7112 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) 7113 clear_tracing_err_log(tr); 7114 7115 if (file->f_mode & FMODE_READ) { 7116 ret = seq_open(file, &tracing_err_log_seq_ops); 7117 if (!ret) { 7118 struct seq_file *m = file->private_data; 7119 m->private = tr; 7120 } else { 7121 trace_array_put(tr); 7122 } 7123 } 7124 return ret; 7125 } 7126 7127 static ssize_t tracing_err_log_write(struct file *file, 7128 const char __user *buffer, 7129 size_t count, loff_t *ppos) 7130 { 7131 return count; 7132 } 7133 7134 static int tracing_err_log_release(struct inode *inode, struct file *file) 7135 { 7136 struct trace_array *tr = inode->i_private; 7137 7138 trace_array_put(tr); 7139 7140 if (file->f_mode & FMODE_READ) 7141 seq_release(inode, file); 7142 7143 return 0; 7144 } 7145 7146 static const struct file_operations tracing_err_log_fops = { 7147 .open = tracing_err_log_open, 7148 .write = tracing_err_log_write, 7149 .read = seq_read, 7150 .llseek = seq_lseek, 7151 .release = tracing_err_log_release, 7152 }; 7153 7154 static int tracing_buffers_open(struct inode *inode, struct file *filp) 7155 { 7156 struct trace_array *tr = inode->i_private; 7157 struct ftrace_buffer_info *info; 7158 int ret; 7159 7160 if (tracing_disabled) 7161 return -ENODEV; 7162 7163 if (trace_array_get(tr) < 0) 7164 return -ENODEV; 7165 7166 info = kzalloc(sizeof(*info), GFP_KERNEL); 7167 if (!info) { 7168 trace_array_put(tr); 7169 return -ENOMEM; 7170 } 7171 7172 mutex_lock(&trace_types_lock); 7173 7174 info->iter.tr = tr; 7175 info->iter.cpu_file = tracing_get_cpu(inode); 7176 info->iter.trace = tr->current_trace; 7177 info->iter.trace_buffer = &tr->trace_buffer; 7178 info->spare = NULL; 7179 /* Force reading ring buffer for first read */ 7180 info->read = (unsigned int)-1; 7181 7182 filp->private_data = info; 7183 7184 tr->current_trace->ref++; 7185 7186 mutex_unlock(&trace_types_lock); 7187 7188 ret = nonseekable_open(inode, filp); 7189 if (ret < 0) 7190 trace_array_put(tr); 7191 7192 return ret; 7193 } 7194 7195 static __poll_t 7196 tracing_buffers_poll(struct file *filp, poll_table *poll_table) 7197 { 7198 struct ftrace_buffer_info *info = filp->private_data; 7199 struct trace_iterator *iter = &info->iter; 7200 7201 return trace_poll(iter, filp, poll_table); 7202 } 7203 7204 static ssize_t 7205 tracing_buffers_read(struct file *filp, char __user *ubuf, 7206 size_t count, loff_t *ppos) 7207 { 7208 struct ftrace_buffer_info *info = filp->private_data; 7209 struct trace_iterator *iter = &info->iter; 7210 ssize_t ret = 0; 7211 ssize_t size; 7212 7213 if (!count) 7214 return 0; 7215 7216 #ifdef CONFIG_TRACER_MAX_TRACE 7217 if (iter->snapshot && iter->tr->current_trace->use_max_tr) 7218 return -EBUSY; 7219 #endif 7220 7221 if (!info->spare) { 7222 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer, 7223 iter->cpu_file); 7224 if (IS_ERR(info->spare)) { 7225 ret = PTR_ERR(info->spare); 7226 info->spare = NULL; 7227 } else { 7228 info->spare_cpu = iter->cpu_file; 7229 } 7230 } 7231 if (!info->spare) 7232 return ret; 7233 7234 /* Do we have previous read data to read? */ 7235 if (info->read < PAGE_SIZE) 7236 goto read; 7237 7238 again: 7239 trace_access_lock(iter->cpu_file); 7240 ret = ring_buffer_read_page(iter->trace_buffer->buffer, 7241 &info->spare, 7242 count, 7243 iter->cpu_file, 0); 7244 trace_access_unlock(iter->cpu_file); 7245 7246 if (ret < 0) { 7247 if (trace_empty(iter)) { 7248 if ((filp->f_flags & O_NONBLOCK)) 7249 return -EAGAIN; 7250 7251 ret = wait_on_pipe(iter, 0); 7252 if (ret) 7253 return ret; 7254 7255 goto again; 7256 } 7257 return 0; 7258 } 7259 7260 info->read = 0; 7261 read: 7262 size = PAGE_SIZE - info->read; 7263 if (size > count) 7264 size = count; 7265 7266 ret = copy_to_user(ubuf, info->spare + info->read, size); 7267 if (ret == size) 7268 return -EFAULT; 7269 7270 size -= ret; 7271 7272 *ppos += size; 7273 info->read += size; 7274 7275 return size; 7276 } 7277 7278 static int tracing_buffers_release(struct inode *inode, struct file *file) 7279 { 7280 struct ftrace_buffer_info *info = file->private_data; 7281 struct trace_iterator *iter = &info->iter; 7282 7283 mutex_lock(&trace_types_lock); 7284 7285 iter->tr->current_trace->ref--; 7286 7287 __trace_array_put(iter->tr); 7288 7289 if (info->spare) 7290 ring_buffer_free_read_page(iter->trace_buffer->buffer, 7291 info->spare_cpu, info->spare); 7292 kfree(info); 7293 7294 mutex_unlock(&trace_types_lock); 7295 7296 return 0; 7297 } 7298 7299 struct buffer_ref { 7300 struct ring_buffer *buffer; 7301 void *page; 7302 int cpu; 7303 refcount_t refcount; 7304 }; 7305 7306 static void buffer_ref_release(struct buffer_ref *ref) 7307 { 7308 if (!refcount_dec_and_test(&ref->refcount)) 7309 return; 7310 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page); 7311 kfree(ref); 7312 } 7313 7314 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe, 7315 struct pipe_buffer *buf) 7316 { 7317 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 7318 7319 buffer_ref_release(ref); 7320 buf->private = 0; 7321 } 7322 7323 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe, 7324 struct pipe_buffer *buf) 7325 { 7326 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 7327 7328 if (refcount_read(&ref->refcount) > INT_MAX/2) 7329 return false; 7330 7331 refcount_inc(&ref->refcount); 7332 return true; 7333 } 7334 7335 /* Pipe buffer operations for a buffer. */ 7336 static const struct pipe_buf_operations buffer_pipe_buf_ops = { 7337 .confirm = generic_pipe_buf_confirm, 7338 .release = buffer_pipe_buf_release, 7339 .steal = generic_pipe_buf_nosteal, 7340 .get = buffer_pipe_buf_get, 7341 }; 7342 7343 /* 7344 * Callback from splice_to_pipe(), if we need to release some pages 7345 * at the end of the spd in case we error'ed out in filling the pipe. 7346 */ 7347 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i) 7348 { 7349 struct buffer_ref *ref = 7350 (struct buffer_ref *)spd->partial[i].private; 7351 7352 buffer_ref_release(ref); 7353 spd->partial[i].private = 0; 7354 } 7355 7356 static ssize_t 7357 tracing_buffers_splice_read(struct file *file, loff_t *ppos, 7358 struct pipe_inode_info *pipe, size_t len, 7359 unsigned int flags) 7360 { 7361 struct ftrace_buffer_info *info = file->private_data; 7362 struct trace_iterator *iter = &info->iter; 7363 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 7364 struct page *pages_def[PIPE_DEF_BUFFERS]; 7365 struct splice_pipe_desc spd = { 7366 .pages = pages_def, 7367 .partial = partial_def, 7368 .nr_pages_max = PIPE_DEF_BUFFERS, 7369 .ops = &buffer_pipe_buf_ops, 7370 .spd_release = buffer_spd_release, 7371 }; 7372 struct buffer_ref *ref; 7373 int entries, i; 7374 ssize_t ret = 0; 7375 7376 #ifdef CONFIG_TRACER_MAX_TRACE 7377 if (iter->snapshot && iter->tr->current_trace->use_max_tr) 7378 return -EBUSY; 7379 #endif 7380 7381 if (*ppos & (PAGE_SIZE - 1)) 7382 return -EINVAL; 7383 7384 if (len & (PAGE_SIZE - 1)) { 7385 if (len < PAGE_SIZE) 7386 return -EINVAL; 7387 len &= PAGE_MASK; 7388 } 7389 7390 if (splice_grow_spd(pipe, &spd)) 7391 return -ENOMEM; 7392 7393 again: 7394 trace_access_lock(iter->cpu_file); 7395 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file); 7396 7397 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) { 7398 struct page *page; 7399 int r; 7400 7401 ref = kzalloc(sizeof(*ref), GFP_KERNEL); 7402 if (!ref) { 7403 ret = -ENOMEM; 7404 break; 7405 } 7406 7407 refcount_set(&ref->refcount, 1); 7408 ref->buffer = iter->trace_buffer->buffer; 7409 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file); 7410 if (IS_ERR(ref->page)) { 7411 ret = PTR_ERR(ref->page); 7412 ref->page = NULL; 7413 kfree(ref); 7414 break; 7415 } 7416 ref->cpu = iter->cpu_file; 7417 7418 r = ring_buffer_read_page(ref->buffer, &ref->page, 7419 len, iter->cpu_file, 1); 7420 if (r < 0) { 7421 ring_buffer_free_read_page(ref->buffer, ref->cpu, 7422 ref->page); 7423 kfree(ref); 7424 break; 7425 } 7426 7427 page = virt_to_page(ref->page); 7428 7429 spd.pages[i] = page; 7430 spd.partial[i].len = PAGE_SIZE; 7431 spd.partial[i].offset = 0; 7432 spd.partial[i].private = (unsigned long)ref; 7433 spd.nr_pages++; 7434 *ppos += PAGE_SIZE; 7435 7436 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file); 7437 } 7438 7439 trace_access_unlock(iter->cpu_file); 7440 spd.nr_pages = i; 7441 7442 /* did we read anything? */ 7443 if (!spd.nr_pages) { 7444 if (ret) 7445 goto out; 7446 7447 ret = -EAGAIN; 7448 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) 7449 goto out; 7450 7451 ret = wait_on_pipe(iter, iter->tr->buffer_percent); 7452 if (ret) 7453 goto out; 7454 7455 goto again; 7456 } 7457 7458 ret = splice_to_pipe(pipe, &spd); 7459 out: 7460 splice_shrink_spd(&spd); 7461 7462 return ret; 7463 } 7464 7465 static const struct file_operations tracing_buffers_fops = { 7466 .open = tracing_buffers_open, 7467 .read = tracing_buffers_read, 7468 .poll = tracing_buffers_poll, 7469 .release = tracing_buffers_release, 7470 .splice_read = tracing_buffers_splice_read, 7471 .llseek = no_llseek, 7472 }; 7473 7474 static ssize_t 7475 tracing_stats_read(struct file *filp, char __user *ubuf, 7476 size_t count, loff_t *ppos) 7477 { 7478 struct inode *inode = file_inode(filp); 7479 struct trace_array *tr = inode->i_private; 7480 struct trace_buffer *trace_buf = &tr->trace_buffer; 7481 int cpu = tracing_get_cpu(inode); 7482 struct trace_seq *s; 7483 unsigned long cnt; 7484 unsigned long long t; 7485 unsigned long usec_rem; 7486 7487 s = kmalloc(sizeof(*s), GFP_KERNEL); 7488 if (!s) 7489 return -ENOMEM; 7490 7491 trace_seq_init(s); 7492 7493 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu); 7494 trace_seq_printf(s, "entries: %ld\n", cnt); 7495 7496 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu); 7497 trace_seq_printf(s, "overrun: %ld\n", cnt); 7498 7499 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu); 7500 trace_seq_printf(s, "commit overrun: %ld\n", cnt); 7501 7502 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu); 7503 trace_seq_printf(s, "bytes: %ld\n", cnt); 7504 7505 if (trace_clocks[tr->clock_id].in_ns) { 7506 /* local or global for trace_clock */ 7507 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); 7508 usec_rem = do_div(t, USEC_PER_SEC); 7509 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", 7510 t, usec_rem); 7511 7512 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu)); 7513 usec_rem = do_div(t, USEC_PER_SEC); 7514 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem); 7515 } else { 7516 /* counter or tsc mode for trace_clock */ 7517 trace_seq_printf(s, "oldest event ts: %llu\n", 7518 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); 7519 7520 trace_seq_printf(s, "now ts: %llu\n", 7521 ring_buffer_time_stamp(trace_buf->buffer, cpu)); 7522 } 7523 7524 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu); 7525 trace_seq_printf(s, "dropped events: %ld\n", cnt); 7526 7527 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu); 7528 trace_seq_printf(s, "read events: %ld\n", cnt); 7529 7530 count = simple_read_from_buffer(ubuf, count, ppos, 7531 s->buffer, trace_seq_used(s)); 7532 7533 kfree(s); 7534 7535 return count; 7536 } 7537 7538 static const struct file_operations tracing_stats_fops = { 7539 .open = tracing_open_generic_tr, 7540 .read = tracing_stats_read, 7541 .llseek = generic_file_llseek, 7542 .release = tracing_release_generic_tr, 7543 }; 7544 7545 #ifdef CONFIG_DYNAMIC_FTRACE 7546 7547 static ssize_t 7548 tracing_read_dyn_info(struct file *filp, char __user *ubuf, 7549 size_t cnt, loff_t *ppos) 7550 { 7551 unsigned long *p = filp->private_data; 7552 char buf[64]; /* Not too big for a shallow stack */ 7553 int r; 7554 7555 r = scnprintf(buf, 63, "%ld", *p); 7556 buf[r++] = '\n'; 7557 7558 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 7559 } 7560 7561 static const struct file_operations tracing_dyn_info_fops = { 7562 .open = tracing_open_generic, 7563 .read = tracing_read_dyn_info, 7564 .llseek = generic_file_llseek, 7565 }; 7566 #endif /* CONFIG_DYNAMIC_FTRACE */ 7567 7568 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) 7569 static void 7570 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, 7571 struct trace_array *tr, struct ftrace_probe_ops *ops, 7572 void *data) 7573 { 7574 tracing_snapshot_instance(tr); 7575 } 7576 7577 static void 7578 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, 7579 struct trace_array *tr, struct ftrace_probe_ops *ops, 7580 void *data) 7581 { 7582 struct ftrace_func_mapper *mapper = data; 7583 long *count = NULL; 7584 7585 if (mapper) 7586 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 7587 7588 if (count) { 7589 7590 if (*count <= 0) 7591 return; 7592 7593 (*count)--; 7594 } 7595 7596 tracing_snapshot_instance(tr); 7597 } 7598 7599 static int 7600 ftrace_snapshot_print(struct seq_file *m, unsigned long ip, 7601 struct ftrace_probe_ops *ops, void *data) 7602 { 7603 struct ftrace_func_mapper *mapper = data; 7604 long *count = NULL; 7605 7606 seq_printf(m, "%ps:", (void *)ip); 7607 7608 seq_puts(m, "snapshot"); 7609 7610 if (mapper) 7611 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 7612 7613 if (count) 7614 seq_printf(m, ":count=%ld\n", *count); 7615 else 7616 seq_puts(m, ":unlimited\n"); 7617 7618 return 0; 7619 } 7620 7621 static int 7622 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr, 7623 unsigned long ip, void *init_data, void **data) 7624 { 7625 struct ftrace_func_mapper *mapper = *data; 7626 7627 if (!mapper) { 7628 mapper = allocate_ftrace_func_mapper(); 7629 if (!mapper) 7630 return -ENOMEM; 7631 *data = mapper; 7632 } 7633 7634 return ftrace_func_mapper_add_ip(mapper, ip, init_data); 7635 } 7636 7637 static void 7638 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr, 7639 unsigned long ip, void *data) 7640 { 7641 struct ftrace_func_mapper *mapper = data; 7642 7643 if (!ip) { 7644 if (!mapper) 7645 return; 7646 free_ftrace_func_mapper(mapper, NULL); 7647 return; 7648 } 7649 7650 ftrace_func_mapper_remove_ip(mapper, ip); 7651 } 7652 7653 static struct ftrace_probe_ops snapshot_probe_ops = { 7654 .func = ftrace_snapshot, 7655 .print = ftrace_snapshot_print, 7656 }; 7657 7658 static struct ftrace_probe_ops snapshot_count_probe_ops = { 7659 .func = ftrace_count_snapshot, 7660 .print = ftrace_snapshot_print, 7661 .init = ftrace_snapshot_init, 7662 .free = ftrace_snapshot_free, 7663 }; 7664 7665 static int 7666 ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash, 7667 char *glob, char *cmd, char *param, int enable) 7668 { 7669 struct ftrace_probe_ops *ops; 7670 void *count = (void *)-1; 7671 char *number; 7672 int ret; 7673 7674 if (!tr) 7675 return -ENODEV; 7676 7677 /* hash funcs only work with set_ftrace_filter */ 7678 if (!enable) 7679 return -EINVAL; 7680 7681 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops; 7682 7683 if (glob[0] == '!') 7684 return unregister_ftrace_function_probe_func(glob+1, tr, ops); 7685 7686 if (!param) 7687 goto out_reg; 7688 7689 number = strsep(¶m, ":"); 7690 7691 if (!strlen(number)) 7692 goto out_reg; 7693 7694 /* 7695 * We use the callback data field (which is a pointer) 7696 * as our counter. 7697 */ 7698 ret = kstrtoul(number, 0, (unsigned long *)&count); 7699 if (ret) 7700 return ret; 7701 7702 out_reg: 7703 ret = tracing_alloc_snapshot_instance(tr); 7704 if (ret < 0) 7705 goto out; 7706 7707 ret = register_ftrace_function_probe(glob, tr, ops, count); 7708 7709 out: 7710 return ret < 0 ? ret : 0; 7711 } 7712 7713 static struct ftrace_func_command ftrace_snapshot_cmd = { 7714 .name = "snapshot", 7715 .func = ftrace_trace_snapshot_callback, 7716 }; 7717 7718 static __init int register_snapshot_cmd(void) 7719 { 7720 return register_ftrace_command(&ftrace_snapshot_cmd); 7721 } 7722 #else 7723 static inline __init int register_snapshot_cmd(void) { return 0; } 7724 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */ 7725 7726 static struct dentry *tracing_get_dentry(struct trace_array *tr) 7727 { 7728 if (WARN_ON(!tr->dir)) 7729 return ERR_PTR(-ENODEV); 7730 7731 /* Top directory uses NULL as the parent */ 7732 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 7733 return NULL; 7734 7735 /* All sub buffers have a descriptor */ 7736 return tr->dir; 7737 } 7738 7739 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu) 7740 { 7741 struct dentry *d_tracer; 7742 7743 if (tr->percpu_dir) 7744 return tr->percpu_dir; 7745 7746 d_tracer = tracing_get_dentry(tr); 7747 if (IS_ERR(d_tracer)) 7748 return NULL; 7749 7750 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer); 7751 7752 WARN_ONCE(!tr->percpu_dir, 7753 "Could not create tracefs directory 'per_cpu/%d'\n", cpu); 7754 7755 return tr->percpu_dir; 7756 } 7757 7758 static struct dentry * 7759 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent, 7760 void *data, long cpu, const struct file_operations *fops) 7761 { 7762 struct dentry *ret = trace_create_file(name, mode, parent, data, fops); 7763 7764 if (ret) /* See tracing_get_cpu() */ 7765 d_inode(ret)->i_cdev = (void *)(cpu + 1); 7766 return ret; 7767 } 7768 7769 static void 7770 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu) 7771 { 7772 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu); 7773 struct dentry *d_cpu; 7774 char cpu_dir[30]; /* 30 characters should be more than enough */ 7775 7776 if (!d_percpu) 7777 return; 7778 7779 snprintf(cpu_dir, 30, "cpu%ld", cpu); 7780 d_cpu = tracefs_create_dir(cpu_dir, d_percpu); 7781 if (!d_cpu) { 7782 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir); 7783 return; 7784 } 7785 7786 /* per cpu trace_pipe */ 7787 trace_create_cpu_file("trace_pipe", 0444, d_cpu, 7788 tr, cpu, &tracing_pipe_fops); 7789 7790 /* per cpu trace */ 7791 trace_create_cpu_file("trace", 0644, d_cpu, 7792 tr, cpu, &tracing_fops); 7793 7794 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu, 7795 tr, cpu, &tracing_buffers_fops); 7796 7797 trace_create_cpu_file("stats", 0444, d_cpu, 7798 tr, cpu, &tracing_stats_fops); 7799 7800 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu, 7801 tr, cpu, &tracing_entries_fops); 7802 7803 #ifdef CONFIG_TRACER_SNAPSHOT 7804 trace_create_cpu_file("snapshot", 0644, d_cpu, 7805 tr, cpu, &snapshot_fops); 7806 7807 trace_create_cpu_file("snapshot_raw", 0444, d_cpu, 7808 tr, cpu, &snapshot_raw_fops); 7809 #endif 7810 } 7811 7812 #ifdef CONFIG_FTRACE_SELFTEST 7813 /* Let selftest have access to static functions in this file */ 7814 #include "trace_selftest.c" 7815 #endif 7816 7817 static ssize_t 7818 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt, 7819 loff_t *ppos) 7820 { 7821 struct trace_option_dentry *topt = filp->private_data; 7822 char *buf; 7823 7824 if (topt->flags->val & topt->opt->bit) 7825 buf = "1\n"; 7826 else 7827 buf = "0\n"; 7828 7829 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 7830 } 7831 7832 static ssize_t 7833 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt, 7834 loff_t *ppos) 7835 { 7836 struct trace_option_dentry *topt = filp->private_data; 7837 unsigned long val; 7838 int ret; 7839 7840 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 7841 if (ret) 7842 return ret; 7843 7844 if (val != 0 && val != 1) 7845 return -EINVAL; 7846 7847 if (!!(topt->flags->val & topt->opt->bit) != val) { 7848 mutex_lock(&trace_types_lock); 7849 ret = __set_tracer_option(topt->tr, topt->flags, 7850 topt->opt, !val); 7851 mutex_unlock(&trace_types_lock); 7852 if (ret) 7853 return ret; 7854 } 7855 7856 *ppos += cnt; 7857 7858 return cnt; 7859 } 7860 7861 7862 static const struct file_operations trace_options_fops = { 7863 .open = tracing_open_generic, 7864 .read = trace_options_read, 7865 .write = trace_options_write, 7866 .llseek = generic_file_llseek, 7867 }; 7868 7869 /* 7870 * In order to pass in both the trace_array descriptor as well as the index 7871 * to the flag that the trace option file represents, the trace_array 7872 * has a character array of trace_flags_index[], which holds the index 7873 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc. 7874 * The address of this character array is passed to the flag option file 7875 * read/write callbacks. 7876 * 7877 * In order to extract both the index and the trace_array descriptor, 7878 * get_tr_index() uses the following algorithm. 7879 * 7880 * idx = *ptr; 7881 * 7882 * As the pointer itself contains the address of the index (remember 7883 * index[1] == 1). 7884 * 7885 * Then to get the trace_array descriptor, by subtracting that index 7886 * from the ptr, we get to the start of the index itself. 7887 * 7888 * ptr - idx == &index[0] 7889 * 7890 * Then a simple container_of() from that pointer gets us to the 7891 * trace_array descriptor. 7892 */ 7893 static void get_tr_index(void *data, struct trace_array **ptr, 7894 unsigned int *pindex) 7895 { 7896 *pindex = *(unsigned char *)data; 7897 7898 *ptr = container_of(data - *pindex, struct trace_array, 7899 trace_flags_index); 7900 } 7901 7902 static ssize_t 7903 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt, 7904 loff_t *ppos) 7905 { 7906 void *tr_index = filp->private_data; 7907 struct trace_array *tr; 7908 unsigned int index; 7909 char *buf; 7910 7911 get_tr_index(tr_index, &tr, &index); 7912 7913 if (tr->trace_flags & (1 << index)) 7914 buf = "1\n"; 7915 else 7916 buf = "0\n"; 7917 7918 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 7919 } 7920 7921 static ssize_t 7922 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt, 7923 loff_t *ppos) 7924 { 7925 void *tr_index = filp->private_data; 7926 struct trace_array *tr; 7927 unsigned int index; 7928 unsigned long val; 7929 int ret; 7930 7931 get_tr_index(tr_index, &tr, &index); 7932 7933 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 7934 if (ret) 7935 return ret; 7936 7937 if (val != 0 && val != 1) 7938 return -EINVAL; 7939 7940 mutex_lock(&trace_types_lock); 7941 ret = set_tracer_flag(tr, 1 << index, val); 7942 mutex_unlock(&trace_types_lock); 7943 7944 if (ret < 0) 7945 return ret; 7946 7947 *ppos += cnt; 7948 7949 return cnt; 7950 } 7951 7952 static const struct file_operations trace_options_core_fops = { 7953 .open = tracing_open_generic, 7954 .read = trace_options_core_read, 7955 .write = trace_options_core_write, 7956 .llseek = generic_file_llseek, 7957 }; 7958 7959 struct dentry *trace_create_file(const char *name, 7960 umode_t mode, 7961 struct dentry *parent, 7962 void *data, 7963 const struct file_operations *fops) 7964 { 7965 struct dentry *ret; 7966 7967 ret = tracefs_create_file(name, mode, parent, data, fops); 7968 if (!ret) 7969 pr_warn("Could not create tracefs '%s' entry\n", name); 7970 7971 return ret; 7972 } 7973 7974 7975 static struct dentry *trace_options_init_dentry(struct trace_array *tr) 7976 { 7977 struct dentry *d_tracer; 7978 7979 if (tr->options) 7980 return tr->options; 7981 7982 d_tracer = tracing_get_dentry(tr); 7983 if (IS_ERR(d_tracer)) 7984 return NULL; 7985 7986 tr->options = tracefs_create_dir("options", d_tracer); 7987 if (!tr->options) { 7988 pr_warn("Could not create tracefs directory 'options'\n"); 7989 return NULL; 7990 } 7991 7992 return tr->options; 7993 } 7994 7995 static void 7996 create_trace_option_file(struct trace_array *tr, 7997 struct trace_option_dentry *topt, 7998 struct tracer_flags *flags, 7999 struct tracer_opt *opt) 8000 { 8001 struct dentry *t_options; 8002 8003 t_options = trace_options_init_dentry(tr); 8004 if (!t_options) 8005 return; 8006 8007 topt->flags = flags; 8008 topt->opt = opt; 8009 topt->tr = tr; 8010 8011 topt->entry = trace_create_file(opt->name, 0644, t_options, topt, 8012 &trace_options_fops); 8013 8014 } 8015 8016 static void 8017 create_trace_option_files(struct trace_array *tr, struct tracer *tracer) 8018 { 8019 struct trace_option_dentry *topts; 8020 struct trace_options *tr_topts; 8021 struct tracer_flags *flags; 8022 struct tracer_opt *opts; 8023 int cnt; 8024 int i; 8025 8026 if (!tracer) 8027 return; 8028 8029 flags = tracer->flags; 8030 8031 if (!flags || !flags->opts) 8032 return; 8033 8034 /* 8035 * If this is an instance, only create flags for tracers 8036 * the instance may have. 8037 */ 8038 if (!trace_ok_for_array(tracer, tr)) 8039 return; 8040 8041 for (i = 0; i < tr->nr_topts; i++) { 8042 /* Make sure there's no duplicate flags. */ 8043 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags)) 8044 return; 8045 } 8046 8047 opts = flags->opts; 8048 8049 for (cnt = 0; opts[cnt].name; cnt++) 8050 ; 8051 8052 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL); 8053 if (!topts) 8054 return; 8055 8056 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1), 8057 GFP_KERNEL); 8058 if (!tr_topts) { 8059 kfree(topts); 8060 return; 8061 } 8062 8063 tr->topts = tr_topts; 8064 tr->topts[tr->nr_topts].tracer = tracer; 8065 tr->topts[tr->nr_topts].topts = topts; 8066 tr->nr_topts++; 8067 8068 for (cnt = 0; opts[cnt].name; cnt++) { 8069 create_trace_option_file(tr, &topts[cnt], flags, 8070 &opts[cnt]); 8071 WARN_ONCE(topts[cnt].entry == NULL, 8072 "Failed to create trace option: %s", 8073 opts[cnt].name); 8074 } 8075 } 8076 8077 static struct dentry * 8078 create_trace_option_core_file(struct trace_array *tr, 8079 const char *option, long index) 8080 { 8081 struct dentry *t_options; 8082 8083 t_options = trace_options_init_dentry(tr); 8084 if (!t_options) 8085 return NULL; 8086 8087 return trace_create_file(option, 0644, t_options, 8088 (void *)&tr->trace_flags_index[index], 8089 &trace_options_core_fops); 8090 } 8091 8092 static void create_trace_options_dir(struct trace_array *tr) 8093 { 8094 struct dentry *t_options; 8095 bool top_level = tr == &global_trace; 8096 int i; 8097 8098 t_options = trace_options_init_dentry(tr); 8099 if (!t_options) 8100 return; 8101 8102 for (i = 0; trace_options[i]; i++) { 8103 if (top_level || 8104 !((1 << i) & TOP_LEVEL_TRACE_FLAGS)) 8105 create_trace_option_core_file(tr, trace_options[i], i); 8106 } 8107 } 8108 8109 static ssize_t 8110 rb_simple_read(struct file *filp, char __user *ubuf, 8111 size_t cnt, loff_t *ppos) 8112 { 8113 struct trace_array *tr = filp->private_data; 8114 char buf[64]; 8115 int r; 8116 8117 r = tracer_tracing_is_on(tr); 8118 r = sprintf(buf, "%d\n", r); 8119 8120 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 8121 } 8122 8123 static ssize_t 8124 rb_simple_write(struct file *filp, const char __user *ubuf, 8125 size_t cnt, loff_t *ppos) 8126 { 8127 struct trace_array *tr = filp->private_data; 8128 struct ring_buffer *buffer = tr->trace_buffer.buffer; 8129 unsigned long val; 8130 int ret; 8131 8132 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 8133 if (ret) 8134 return ret; 8135 8136 if (buffer) { 8137 mutex_lock(&trace_types_lock); 8138 if (!!val == tracer_tracing_is_on(tr)) { 8139 val = 0; /* do nothing */ 8140 } else if (val) { 8141 tracer_tracing_on(tr); 8142 if (tr->current_trace->start) 8143 tr->current_trace->start(tr); 8144 } else { 8145 tracer_tracing_off(tr); 8146 if (tr->current_trace->stop) 8147 tr->current_trace->stop(tr); 8148 } 8149 mutex_unlock(&trace_types_lock); 8150 } 8151 8152 (*ppos)++; 8153 8154 return cnt; 8155 } 8156 8157 static const struct file_operations rb_simple_fops = { 8158 .open = tracing_open_generic_tr, 8159 .read = rb_simple_read, 8160 .write = rb_simple_write, 8161 .release = tracing_release_generic_tr, 8162 .llseek = default_llseek, 8163 }; 8164 8165 static ssize_t 8166 buffer_percent_read(struct file *filp, char __user *ubuf, 8167 size_t cnt, loff_t *ppos) 8168 { 8169 struct trace_array *tr = filp->private_data; 8170 char buf[64]; 8171 int r; 8172 8173 r = tr->buffer_percent; 8174 r = sprintf(buf, "%d\n", r); 8175 8176 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 8177 } 8178 8179 static ssize_t 8180 buffer_percent_write(struct file *filp, const char __user *ubuf, 8181 size_t cnt, loff_t *ppos) 8182 { 8183 struct trace_array *tr = filp->private_data; 8184 unsigned long val; 8185 int ret; 8186 8187 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 8188 if (ret) 8189 return ret; 8190 8191 if (val > 100) 8192 return -EINVAL; 8193 8194 if (!val) 8195 val = 1; 8196 8197 tr->buffer_percent = val; 8198 8199 (*ppos)++; 8200 8201 return cnt; 8202 } 8203 8204 static const struct file_operations buffer_percent_fops = { 8205 .open = tracing_open_generic_tr, 8206 .read = buffer_percent_read, 8207 .write = buffer_percent_write, 8208 .release = tracing_release_generic_tr, 8209 .llseek = default_llseek, 8210 }; 8211 8212 static struct dentry *trace_instance_dir; 8213 8214 static void 8215 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer); 8216 8217 static int 8218 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size) 8219 { 8220 enum ring_buffer_flags rb_flags; 8221 8222 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0; 8223 8224 buf->tr = tr; 8225 8226 buf->buffer = ring_buffer_alloc(size, rb_flags); 8227 if (!buf->buffer) 8228 return -ENOMEM; 8229 8230 buf->data = alloc_percpu(struct trace_array_cpu); 8231 if (!buf->data) { 8232 ring_buffer_free(buf->buffer); 8233 buf->buffer = NULL; 8234 return -ENOMEM; 8235 } 8236 8237 /* Allocate the first page for all buffers */ 8238 set_buffer_entries(&tr->trace_buffer, 8239 ring_buffer_size(tr->trace_buffer.buffer, 0)); 8240 8241 return 0; 8242 } 8243 8244 static int allocate_trace_buffers(struct trace_array *tr, int size) 8245 { 8246 int ret; 8247 8248 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size); 8249 if (ret) 8250 return ret; 8251 8252 #ifdef CONFIG_TRACER_MAX_TRACE 8253 ret = allocate_trace_buffer(tr, &tr->max_buffer, 8254 allocate_snapshot ? size : 1); 8255 if (WARN_ON(ret)) { 8256 ring_buffer_free(tr->trace_buffer.buffer); 8257 tr->trace_buffer.buffer = NULL; 8258 free_percpu(tr->trace_buffer.data); 8259 tr->trace_buffer.data = NULL; 8260 return -ENOMEM; 8261 } 8262 tr->allocated_snapshot = allocate_snapshot; 8263 8264 /* 8265 * Only the top level trace array gets its snapshot allocated 8266 * from the kernel command line. 8267 */ 8268 allocate_snapshot = false; 8269 #endif 8270 return 0; 8271 } 8272 8273 static void free_trace_buffer(struct trace_buffer *buf) 8274 { 8275 if (buf->buffer) { 8276 ring_buffer_free(buf->buffer); 8277 buf->buffer = NULL; 8278 free_percpu(buf->data); 8279 buf->data = NULL; 8280 } 8281 } 8282 8283 static void free_trace_buffers(struct trace_array *tr) 8284 { 8285 if (!tr) 8286 return; 8287 8288 free_trace_buffer(&tr->trace_buffer); 8289 8290 #ifdef CONFIG_TRACER_MAX_TRACE 8291 free_trace_buffer(&tr->max_buffer); 8292 #endif 8293 } 8294 8295 static void init_trace_flags_index(struct trace_array *tr) 8296 { 8297 int i; 8298 8299 /* Used by the trace options files */ 8300 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) 8301 tr->trace_flags_index[i] = i; 8302 } 8303 8304 static void __update_tracer_options(struct trace_array *tr) 8305 { 8306 struct tracer *t; 8307 8308 for (t = trace_types; t; t = t->next) 8309 add_tracer_options(tr, t); 8310 } 8311 8312 static void update_tracer_options(struct trace_array *tr) 8313 { 8314 mutex_lock(&trace_types_lock); 8315 __update_tracer_options(tr); 8316 mutex_unlock(&trace_types_lock); 8317 } 8318 8319 struct trace_array *trace_array_create(const char *name) 8320 { 8321 struct trace_array *tr; 8322 int ret; 8323 8324 mutex_lock(&event_mutex); 8325 mutex_lock(&trace_types_lock); 8326 8327 ret = -EEXIST; 8328 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 8329 if (tr->name && strcmp(tr->name, name) == 0) 8330 goto out_unlock; 8331 } 8332 8333 ret = -ENOMEM; 8334 tr = kzalloc(sizeof(*tr), GFP_KERNEL); 8335 if (!tr) 8336 goto out_unlock; 8337 8338 tr->name = kstrdup(name, GFP_KERNEL); 8339 if (!tr->name) 8340 goto out_free_tr; 8341 8342 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL)) 8343 goto out_free_tr; 8344 8345 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS; 8346 8347 cpumask_copy(tr->tracing_cpumask, cpu_all_mask); 8348 8349 raw_spin_lock_init(&tr->start_lock); 8350 8351 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 8352 8353 tr->current_trace = &nop_trace; 8354 8355 INIT_LIST_HEAD(&tr->systems); 8356 INIT_LIST_HEAD(&tr->events); 8357 INIT_LIST_HEAD(&tr->hist_vars); 8358 INIT_LIST_HEAD(&tr->err_log); 8359 8360 if (allocate_trace_buffers(tr, trace_buf_size) < 0) 8361 goto out_free_tr; 8362 8363 tr->dir = tracefs_create_dir(name, trace_instance_dir); 8364 if (!tr->dir) 8365 goto out_free_tr; 8366 8367 ret = event_trace_add_tracer(tr->dir, tr); 8368 if (ret) { 8369 tracefs_remove_recursive(tr->dir); 8370 goto out_free_tr; 8371 } 8372 8373 ftrace_init_trace_array(tr); 8374 8375 init_tracer_tracefs(tr, tr->dir); 8376 init_trace_flags_index(tr); 8377 __update_tracer_options(tr); 8378 8379 list_add(&tr->list, &ftrace_trace_arrays); 8380 8381 mutex_unlock(&trace_types_lock); 8382 mutex_unlock(&event_mutex); 8383 8384 return tr; 8385 8386 out_free_tr: 8387 free_trace_buffers(tr); 8388 free_cpumask_var(tr->tracing_cpumask); 8389 kfree(tr->name); 8390 kfree(tr); 8391 8392 out_unlock: 8393 mutex_unlock(&trace_types_lock); 8394 mutex_unlock(&event_mutex); 8395 8396 return ERR_PTR(ret); 8397 } 8398 EXPORT_SYMBOL_GPL(trace_array_create); 8399 8400 static int instance_mkdir(const char *name) 8401 { 8402 return PTR_ERR_OR_ZERO(trace_array_create(name)); 8403 } 8404 8405 static int __remove_instance(struct trace_array *tr) 8406 { 8407 int i; 8408 8409 if (tr->ref || (tr->current_trace && tr->current_trace->ref)) 8410 return -EBUSY; 8411 8412 list_del(&tr->list); 8413 8414 /* Disable all the flags that were enabled coming in */ 8415 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) { 8416 if ((1 << i) & ZEROED_TRACE_FLAGS) 8417 set_tracer_flag(tr, 1 << i, 0); 8418 } 8419 8420 tracing_set_nop(tr); 8421 clear_ftrace_function_probes(tr); 8422 event_trace_del_tracer(tr); 8423 ftrace_clear_pids(tr); 8424 ftrace_destroy_function_files(tr); 8425 tracefs_remove_recursive(tr->dir); 8426 free_trace_buffers(tr); 8427 8428 for (i = 0; i < tr->nr_topts; i++) { 8429 kfree(tr->topts[i].topts); 8430 } 8431 kfree(tr->topts); 8432 8433 free_cpumask_var(tr->tracing_cpumask); 8434 kfree(tr->name); 8435 kfree(tr); 8436 tr = NULL; 8437 8438 return 0; 8439 } 8440 8441 int trace_array_destroy(struct trace_array *tr) 8442 { 8443 int ret; 8444 8445 if (!tr) 8446 return -EINVAL; 8447 8448 mutex_lock(&event_mutex); 8449 mutex_lock(&trace_types_lock); 8450 8451 ret = __remove_instance(tr); 8452 8453 mutex_unlock(&trace_types_lock); 8454 mutex_unlock(&event_mutex); 8455 8456 return ret; 8457 } 8458 EXPORT_SYMBOL_GPL(trace_array_destroy); 8459 8460 static int instance_rmdir(const char *name) 8461 { 8462 struct trace_array *tr; 8463 int ret; 8464 8465 mutex_lock(&event_mutex); 8466 mutex_lock(&trace_types_lock); 8467 8468 ret = -ENODEV; 8469 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 8470 if (tr->name && strcmp(tr->name, name) == 0) { 8471 ret = __remove_instance(tr); 8472 break; 8473 } 8474 } 8475 8476 mutex_unlock(&trace_types_lock); 8477 mutex_unlock(&event_mutex); 8478 8479 return ret; 8480 } 8481 8482 static __init void create_trace_instances(struct dentry *d_tracer) 8483 { 8484 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer, 8485 instance_mkdir, 8486 instance_rmdir); 8487 if (WARN_ON(!trace_instance_dir)) 8488 return; 8489 } 8490 8491 static void 8492 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) 8493 { 8494 struct trace_event_file *file; 8495 int cpu; 8496 8497 trace_create_file("available_tracers", 0444, d_tracer, 8498 tr, &show_traces_fops); 8499 8500 trace_create_file("current_tracer", 0644, d_tracer, 8501 tr, &set_tracer_fops); 8502 8503 trace_create_file("tracing_cpumask", 0644, d_tracer, 8504 tr, &tracing_cpumask_fops); 8505 8506 trace_create_file("trace_options", 0644, d_tracer, 8507 tr, &tracing_iter_fops); 8508 8509 trace_create_file("trace", 0644, d_tracer, 8510 tr, &tracing_fops); 8511 8512 trace_create_file("trace_pipe", 0444, d_tracer, 8513 tr, &tracing_pipe_fops); 8514 8515 trace_create_file("buffer_size_kb", 0644, d_tracer, 8516 tr, &tracing_entries_fops); 8517 8518 trace_create_file("buffer_total_size_kb", 0444, d_tracer, 8519 tr, &tracing_total_entries_fops); 8520 8521 trace_create_file("free_buffer", 0200, d_tracer, 8522 tr, &tracing_free_buffer_fops); 8523 8524 trace_create_file("trace_marker", 0220, d_tracer, 8525 tr, &tracing_mark_fops); 8526 8527 file = __find_event_file(tr, "ftrace", "print"); 8528 if (file && file->dir) 8529 trace_create_file("trigger", 0644, file->dir, file, 8530 &event_trigger_fops); 8531 tr->trace_marker_file = file; 8532 8533 trace_create_file("trace_marker_raw", 0220, d_tracer, 8534 tr, &tracing_mark_raw_fops); 8535 8536 trace_create_file("trace_clock", 0644, d_tracer, tr, 8537 &trace_clock_fops); 8538 8539 trace_create_file("tracing_on", 0644, d_tracer, 8540 tr, &rb_simple_fops); 8541 8542 trace_create_file("timestamp_mode", 0444, d_tracer, tr, 8543 &trace_time_stamp_mode_fops); 8544 8545 tr->buffer_percent = 50; 8546 8547 trace_create_file("buffer_percent", 0444, d_tracer, 8548 tr, &buffer_percent_fops); 8549 8550 create_trace_options_dir(tr); 8551 8552 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) 8553 trace_create_file("tracing_max_latency", 0644, d_tracer, 8554 &tr->max_latency, &tracing_max_lat_fops); 8555 #endif 8556 8557 if (ftrace_create_function_files(tr, d_tracer)) 8558 WARN(1, "Could not allocate function filter files"); 8559 8560 #ifdef CONFIG_TRACER_SNAPSHOT 8561 trace_create_file("snapshot", 0644, d_tracer, 8562 tr, &snapshot_fops); 8563 #endif 8564 8565 trace_create_file("error_log", 0644, d_tracer, 8566 tr, &tracing_err_log_fops); 8567 8568 for_each_tracing_cpu(cpu) 8569 tracing_init_tracefs_percpu(tr, cpu); 8570 8571 ftrace_init_tracefs(tr, d_tracer); 8572 } 8573 8574 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore) 8575 { 8576 struct vfsmount *mnt; 8577 struct file_system_type *type; 8578 8579 /* 8580 * To maintain backward compatibility for tools that mount 8581 * debugfs to get to the tracing facility, tracefs is automatically 8582 * mounted to the debugfs/tracing directory. 8583 */ 8584 type = get_fs_type("tracefs"); 8585 if (!type) 8586 return NULL; 8587 mnt = vfs_submount(mntpt, type, "tracefs", NULL); 8588 put_filesystem(type); 8589 if (IS_ERR(mnt)) 8590 return NULL; 8591 mntget(mnt); 8592 8593 return mnt; 8594 } 8595 8596 /** 8597 * tracing_init_dentry - initialize top level trace array 8598 * 8599 * This is called when creating files or directories in the tracing 8600 * directory. It is called via fs_initcall() by any of the boot up code 8601 * and expects to return the dentry of the top level tracing directory. 8602 */ 8603 struct dentry *tracing_init_dentry(void) 8604 { 8605 struct trace_array *tr = &global_trace; 8606 8607 /* The top level trace array uses NULL as parent */ 8608 if (tr->dir) 8609 return NULL; 8610 8611 if (WARN_ON(!tracefs_initialized()) || 8612 (IS_ENABLED(CONFIG_DEBUG_FS) && 8613 WARN_ON(!debugfs_initialized()))) 8614 return ERR_PTR(-ENODEV); 8615 8616 /* 8617 * As there may still be users that expect the tracing 8618 * files to exist in debugfs/tracing, we must automount 8619 * the tracefs file system there, so older tools still 8620 * work with the newer kerenl. 8621 */ 8622 tr->dir = debugfs_create_automount("tracing", NULL, 8623 trace_automount, NULL); 8624 8625 return NULL; 8626 } 8627 8628 extern struct trace_eval_map *__start_ftrace_eval_maps[]; 8629 extern struct trace_eval_map *__stop_ftrace_eval_maps[]; 8630 8631 static void __init trace_eval_init(void) 8632 { 8633 int len; 8634 8635 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps; 8636 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len); 8637 } 8638 8639 #ifdef CONFIG_MODULES 8640 static void trace_module_add_evals(struct module *mod) 8641 { 8642 if (!mod->num_trace_evals) 8643 return; 8644 8645 /* 8646 * Modules with bad taint do not have events created, do 8647 * not bother with enums either. 8648 */ 8649 if (trace_module_has_bad_taint(mod)) 8650 return; 8651 8652 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals); 8653 } 8654 8655 #ifdef CONFIG_TRACE_EVAL_MAP_FILE 8656 static void trace_module_remove_evals(struct module *mod) 8657 { 8658 union trace_eval_map_item *map; 8659 union trace_eval_map_item **last = &trace_eval_maps; 8660 8661 if (!mod->num_trace_evals) 8662 return; 8663 8664 mutex_lock(&trace_eval_mutex); 8665 8666 map = trace_eval_maps; 8667 8668 while (map) { 8669 if (map->head.mod == mod) 8670 break; 8671 map = trace_eval_jmp_to_tail(map); 8672 last = &map->tail.next; 8673 map = map->tail.next; 8674 } 8675 if (!map) 8676 goto out; 8677 8678 *last = trace_eval_jmp_to_tail(map)->tail.next; 8679 kfree(map); 8680 out: 8681 mutex_unlock(&trace_eval_mutex); 8682 } 8683 #else 8684 static inline void trace_module_remove_evals(struct module *mod) { } 8685 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */ 8686 8687 static int trace_module_notify(struct notifier_block *self, 8688 unsigned long val, void *data) 8689 { 8690 struct module *mod = data; 8691 8692 switch (val) { 8693 case MODULE_STATE_COMING: 8694 trace_module_add_evals(mod); 8695 break; 8696 case MODULE_STATE_GOING: 8697 trace_module_remove_evals(mod); 8698 break; 8699 } 8700 8701 return 0; 8702 } 8703 8704 static struct notifier_block trace_module_nb = { 8705 .notifier_call = trace_module_notify, 8706 .priority = 0, 8707 }; 8708 #endif /* CONFIG_MODULES */ 8709 8710 static __init int tracer_init_tracefs(void) 8711 { 8712 struct dentry *d_tracer; 8713 8714 trace_access_lock_init(); 8715 8716 d_tracer = tracing_init_dentry(); 8717 if (IS_ERR(d_tracer)) 8718 return 0; 8719 8720 event_trace_init(); 8721 8722 init_tracer_tracefs(&global_trace, d_tracer); 8723 ftrace_init_tracefs_toplevel(&global_trace, d_tracer); 8724 8725 trace_create_file("tracing_thresh", 0644, d_tracer, 8726 &global_trace, &tracing_thresh_fops); 8727 8728 trace_create_file("README", 0444, d_tracer, 8729 NULL, &tracing_readme_fops); 8730 8731 trace_create_file("saved_cmdlines", 0444, d_tracer, 8732 NULL, &tracing_saved_cmdlines_fops); 8733 8734 trace_create_file("saved_cmdlines_size", 0644, d_tracer, 8735 NULL, &tracing_saved_cmdlines_size_fops); 8736 8737 trace_create_file("saved_tgids", 0444, d_tracer, 8738 NULL, &tracing_saved_tgids_fops); 8739 8740 trace_eval_init(); 8741 8742 trace_create_eval_file(d_tracer); 8743 8744 #ifdef CONFIG_MODULES 8745 register_module_notifier(&trace_module_nb); 8746 #endif 8747 8748 #ifdef CONFIG_DYNAMIC_FTRACE 8749 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer, 8750 &ftrace_update_tot_cnt, &tracing_dyn_info_fops); 8751 #endif 8752 8753 create_trace_instances(d_tracer); 8754 8755 update_tracer_options(&global_trace); 8756 8757 return 0; 8758 } 8759 8760 static int trace_panic_handler(struct notifier_block *this, 8761 unsigned long event, void *unused) 8762 { 8763 if (ftrace_dump_on_oops) 8764 ftrace_dump(ftrace_dump_on_oops); 8765 return NOTIFY_OK; 8766 } 8767 8768 static struct notifier_block trace_panic_notifier = { 8769 .notifier_call = trace_panic_handler, 8770 .next = NULL, 8771 .priority = 150 /* priority: INT_MAX >= x >= 0 */ 8772 }; 8773 8774 static int trace_die_handler(struct notifier_block *self, 8775 unsigned long val, 8776 void *data) 8777 { 8778 switch (val) { 8779 case DIE_OOPS: 8780 if (ftrace_dump_on_oops) 8781 ftrace_dump(ftrace_dump_on_oops); 8782 break; 8783 default: 8784 break; 8785 } 8786 return NOTIFY_OK; 8787 } 8788 8789 static struct notifier_block trace_die_notifier = { 8790 .notifier_call = trace_die_handler, 8791 .priority = 200 8792 }; 8793 8794 /* 8795 * printk is set to max of 1024, we really don't need it that big. 8796 * Nothing should be printing 1000 characters anyway. 8797 */ 8798 #define TRACE_MAX_PRINT 1000 8799 8800 /* 8801 * Define here KERN_TRACE so that we have one place to modify 8802 * it if we decide to change what log level the ftrace dump 8803 * should be at. 8804 */ 8805 #define KERN_TRACE KERN_EMERG 8806 8807 void 8808 trace_printk_seq(struct trace_seq *s) 8809 { 8810 /* Probably should print a warning here. */ 8811 if (s->seq.len >= TRACE_MAX_PRINT) 8812 s->seq.len = TRACE_MAX_PRINT; 8813 8814 /* 8815 * More paranoid code. Although the buffer size is set to 8816 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just 8817 * an extra layer of protection. 8818 */ 8819 if (WARN_ON_ONCE(s->seq.len >= s->seq.size)) 8820 s->seq.len = s->seq.size - 1; 8821 8822 /* should be zero ended, but we are paranoid. */ 8823 s->buffer[s->seq.len] = 0; 8824 8825 printk(KERN_TRACE "%s", s->buffer); 8826 8827 trace_seq_init(s); 8828 } 8829 8830 void trace_init_global_iter(struct trace_iterator *iter) 8831 { 8832 iter->tr = &global_trace; 8833 iter->trace = iter->tr->current_trace; 8834 iter->cpu_file = RING_BUFFER_ALL_CPUS; 8835 iter->trace_buffer = &global_trace.trace_buffer; 8836 8837 if (iter->trace && iter->trace->open) 8838 iter->trace->open(iter); 8839 8840 /* Annotate start of buffers if we had overruns */ 8841 if (ring_buffer_overruns(iter->trace_buffer->buffer)) 8842 iter->iter_flags |= TRACE_FILE_ANNOTATE; 8843 8844 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 8845 if (trace_clocks[iter->tr->clock_id].in_ns) 8846 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 8847 } 8848 8849 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) 8850 { 8851 /* use static because iter can be a bit big for the stack */ 8852 static struct trace_iterator iter; 8853 static atomic_t dump_running; 8854 struct trace_array *tr = &global_trace; 8855 unsigned int old_userobj; 8856 unsigned long flags; 8857 int cnt = 0, cpu; 8858 8859 /* Only allow one dump user at a time. */ 8860 if (atomic_inc_return(&dump_running) != 1) { 8861 atomic_dec(&dump_running); 8862 return; 8863 } 8864 8865 /* 8866 * Always turn off tracing when we dump. 8867 * We don't need to show trace output of what happens 8868 * between multiple crashes. 8869 * 8870 * If the user does a sysrq-z, then they can re-enable 8871 * tracing with echo 1 > tracing_on. 8872 */ 8873 tracing_off(); 8874 8875 local_irq_save(flags); 8876 printk_nmi_direct_enter(); 8877 8878 /* Simulate the iterator */ 8879 trace_init_global_iter(&iter); 8880 8881 for_each_tracing_cpu(cpu) { 8882 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled); 8883 } 8884 8885 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ; 8886 8887 /* don't look at user memory in panic mode */ 8888 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ; 8889 8890 switch (oops_dump_mode) { 8891 case DUMP_ALL: 8892 iter.cpu_file = RING_BUFFER_ALL_CPUS; 8893 break; 8894 case DUMP_ORIG: 8895 iter.cpu_file = raw_smp_processor_id(); 8896 break; 8897 case DUMP_NONE: 8898 goto out_enable; 8899 default: 8900 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n"); 8901 iter.cpu_file = RING_BUFFER_ALL_CPUS; 8902 } 8903 8904 printk(KERN_TRACE "Dumping ftrace buffer:\n"); 8905 8906 /* Did function tracer already get disabled? */ 8907 if (ftrace_is_dead()) { 8908 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n"); 8909 printk("# MAY BE MISSING FUNCTION EVENTS\n"); 8910 } 8911 8912 /* 8913 * We need to stop all tracing on all CPUS to read the 8914 * the next buffer. This is a bit expensive, but is 8915 * not done often. We fill all what we can read, 8916 * and then release the locks again. 8917 */ 8918 8919 while (!trace_empty(&iter)) { 8920 8921 if (!cnt) 8922 printk(KERN_TRACE "---------------------------------\n"); 8923 8924 cnt++; 8925 8926 trace_iterator_reset(&iter); 8927 iter.iter_flags |= TRACE_FILE_LAT_FMT; 8928 8929 if (trace_find_next_entry_inc(&iter) != NULL) { 8930 int ret; 8931 8932 ret = print_trace_line(&iter); 8933 if (ret != TRACE_TYPE_NO_CONSUME) 8934 trace_consume(&iter); 8935 } 8936 touch_nmi_watchdog(); 8937 8938 trace_printk_seq(&iter.seq); 8939 } 8940 8941 if (!cnt) 8942 printk(KERN_TRACE " (ftrace buffer empty)\n"); 8943 else 8944 printk(KERN_TRACE "---------------------------------\n"); 8945 8946 out_enable: 8947 tr->trace_flags |= old_userobj; 8948 8949 for_each_tracing_cpu(cpu) { 8950 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled); 8951 } 8952 atomic_dec(&dump_running); 8953 printk_nmi_direct_exit(); 8954 local_irq_restore(flags); 8955 } 8956 EXPORT_SYMBOL_GPL(ftrace_dump); 8957 8958 int trace_run_command(const char *buf, int (*createfn)(int, char **)) 8959 { 8960 char **argv; 8961 int argc, ret; 8962 8963 argc = 0; 8964 ret = 0; 8965 argv = argv_split(GFP_KERNEL, buf, &argc); 8966 if (!argv) 8967 return -ENOMEM; 8968 8969 if (argc) 8970 ret = createfn(argc, argv); 8971 8972 argv_free(argv); 8973 8974 return ret; 8975 } 8976 8977 #define WRITE_BUFSIZE 4096 8978 8979 ssize_t trace_parse_run_command(struct file *file, const char __user *buffer, 8980 size_t count, loff_t *ppos, 8981 int (*createfn)(int, char **)) 8982 { 8983 char *kbuf, *buf, *tmp; 8984 int ret = 0; 8985 size_t done = 0; 8986 size_t size; 8987 8988 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL); 8989 if (!kbuf) 8990 return -ENOMEM; 8991 8992 while (done < count) { 8993 size = count - done; 8994 8995 if (size >= WRITE_BUFSIZE) 8996 size = WRITE_BUFSIZE - 1; 8997 8998 if (copy_from_user(kbuf, buffer + done, size)) { 8999 ret = -EFAULT; 9000 goto out; 9001 } 9002 kbuf[size] = '\0'; 9003 buf = kbuf; 9004 do { 9005 tmp = strchr(buf, '\n'); 9006 if (tmp) { 9007 *tmp = '\0'; 9008 size = tmp - buf + 1; 9009 } else { 9010 size = strlen(buf); 9011 if (done + size < count) { 9012 if (buf != kbuf) 9013 break; 9014 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */ 9015 pr_warn("Line length is too long: Should be less than %d\n", 9016 WRITE_BUFSIZE - 2); 9017 ret = -EINVAL; 9018 goto out; 9019 } 9020 } 9021 done += size; 9022 9023 /* Remove comments */ 9024 tmp = strchr(buf, '#'); 9025 9026 if (tmp) 9027 *tmp = '\0'; 9028 9029 ret = trace_run_command(buf, createfn); 9030 if (ret) 9031 goto out; 9032 buf += size; 9033 9034 } while (done < count); 9035 } 9036 ret = done; 9037 9038 out: 9039 kfree(kbuf); 9040 9041 return ret; 9042 } 9043 9044 __init static int tracer_alloc_buffers(void) 9045 { 9046 int ring_buf_size; 9047 int ret = -ENOMEM; 9048 9049 /* 9050 * Make sure we don't accidently add more trace options 9051 * than we have bits for. 9052 */ 9053 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE); 9054 9055 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL)) 9056 goto out; 9057 9058 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL)) 9059 goto out_free_buffer_mask; 9060 9061 /* Only allocate trace_printk buffers if a trace_printk exists */ 9062 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt) 9063 /* Must be called before global_trace.buffer is allocated */ 9064 trace_printk_init_buffers(); 9065 9066 /* To save memory, keep the ring buffer size to its minimum */ 9067 if (ring_buffer_expanded) 9068 ring_buf_size = trace_buf_size; 9069 else 9070 ring_buf_size = 1; 9071 9072 cpumask_copy(tracing_buffer_mask, cpu_possible_mask); 9073 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask); 9074 9075 raw_spin_lock_init(&global_trace.start_lock); 9076 9077 /* 9078 * The prepare callbacks allocates some memory for the ring buffer. We 9079 * don't free the buffer if the if the CPU goes down. If we were to free 9080 * the buffer, then the user would lose any trace that was in the 9081 * buffer. The memory will be removed once the "instance" is removed. 9082 */ 9083 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE, 9084 "trace/RB:preapre", trace_rb_cpu_prepare, 9085 NULL); 9086 if (ret < 0) 9087 goto out_free_cpumask; 9088 /* Used for event triggers */ 9089 ret = -ENOMEM; 9090 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE); 9091 if (!temp_buffer) 9092 goto out_rm_hp_state; 9093 9094 if (trace_create_savedcmd() < 0) 9095 goto out_free_temp_buffer; 9096 9097 /* TODO: make the number of buffers hot pluggable with CPUS */ 9098 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) { 9099 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n"); 9100 WARN_ON(1); 9101 goto out_free_savedcmd; 9102 } 9103 9104 if (global_trace.buffer_disabled) 9105 tracing_off(); 9106 9107 if (trace_boot_clock) { 9108 ret = tracing_set_clock(&global_trace, trace_boot_clock); 9109 if (ret < 0) 9110 pr_warn("Trace clock %s not defined, going back to default\n", 9111 trace_boot_clock); 9112 } 9113 9114 /* 9115 * register_tracer() might reference current_trace, so it 9116 * needs to be set before we register anything. This is 9117 * just a bootstrap of current_trace anyway. 9118 */ 9119 global_trace.current_trace = &nop_trace; 9120 9121 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 9122 9123 ftrace_init_global_array_ops(&global_trace); 9124 9125 init_trace_flags_index(&global_trace); 9126 9127 register_tracer(&nop_trace); 9128 9129 /* Function tracing may start here (via kernel command line) */ 9130 init_function_trace(); 9131 9132 /* All seems OK, enable tracing */ 9133 tracing_disabled = 0; 9134 9135 atomic_notifier_chain_register(&panic_notifier_list, 9136 &trace_panic_notifier); 9137 9138 register_die_notifier(&trace_die_notifier); 9139 9140 global_trace.flags = TRACE_ARRAY_FL_GLOBAL; 9141 9142 INIT_LIST_HEAD(&global_trace.systems); 9143 INIT_LIST_HEAD(&global_trace.events); 9144 INIT_LIST_HEAD(&global_trace.hist_vars); 9145 INIT_LIST_HEAD(&global_trace.err_log); 9146 list_add(&global_trace.list, &ftrace_trace_arrays); 9147 9148 apply_trace_boot_options(); 9149 9150 register_snapshot_cmd(); 9151 9152 return 0; 9153 9154 out_free_savedcmd: 9155 free_saved_cmdlines_buffer(savedcmd); 9156 out_free_temp_buffer: 9157 ring_buffer_free(temp_buffer); 9158 out_rm_hp_state: 9159 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE); 9160 out_free_cpumask: 9161 free_cpumask_var(global_trace.tracing_cpumask); 9162 out_free_buffer_mask: 9163 free_cpumask_var(tracing_buffer_mask); 9164 out: 9165 return ret; 9166 } 9167 9168 void __init early_trace_init(void) 9169 { 9170 if (tracepoint_printk) { 9171 tracepoint_print_iter = 9172 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL); 9173 if (WARN_ON(!tracepoint_print_iter)) 9174 tracepoint_printk = 0; 9175 else 9176 static_key_enable(&tracepoint_printk_key.key); 9177 } 9178 tracer_alloc_buffers(); 9179 } 9180 9181 void __init trace_init(void) 9182 { 9183 trace_event_init(); 9184 } 9185 9186 __init static int clear_boot_tracer(void) 9187 { 9188 /* 9189 * The default tracer at boot buffer is an init section. 9190 * This function is called in lateinit. If we did not 9191 * find the boot tracer, then clear it out, to prevent 9192 * later registration from accessing the buffer that is 9193 * about to be freed. 9194 */ 9195 if (!default_bootup_tracer) 9196 return 0; 9197 9198 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n", 9199 default_bootup_tracer); 9200 default_bootup_tracer = NULL; 9201 9202 return 0; 9203 } 9204 9205 fs_initcall(tracer_init_tracefs); 9206 late_initcall_sync(clear_boot_tracer); 9207 9208 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 9209 __init static int tracing_set_default_clock(void) 9210 { 9211 /* sched_clock_stable() is determined in late_initcall */ 9212 if (!trace_boot_clock && !sched_clock_stable()) { 9213 printk(KERN_WARNING 9214 "Unstable clock detected, switching default tracing clock to \"global\"\n" 9215 "If you want to keep using the local clock, then add:\n" 9216 " \"trace_clock=local\"\n" 9217 "on the kernel command line\n"); 9218 tracing_set_clock(&global_trace, "global"); 9219 } 9220 9221 return 0; 9222 } 9223 late_initcall_sync(tracing_set_default_clock); 9224 #endif 9225