1 /* 2 * ring buffer based function tracer 3 * 4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com> 5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 6 * 7 * Originally taken from the RT patch by: 8 * Arnaldo Carvalho de Melo <acme@redhat.com> 9 * 10 * Based on code from the latency_tracer, that is: 11 * Copyright (C) 2004-2006 Ingo Molnar 12 * Copyright (C) 2004 Nadia Yvette Chambers 13 */ 14 #include <linux/ring_buffer.h> 15 #include <generated/utsrelease.h> 16 #include <linux/stacktrace.h> 17 #include <linux/writeback.h> 18 #include <linux/kallsyms.h> 19 #include <linux/seq_file.h> 20 #include <linux/notifier.h> 21 #include <linux/irqflags.h> 22 #include <linux/debugfs.h> 23 #include <linux/tracefs.h> 24 #include <linux/pagemap.h> 25 #include <linux/hardirq.h> 26 #include <linux/linkage.h> 27 #include <linux/uaccess.h> 28 #include <linux/vmalloc.h> 29 #include <linux/ftrace.h> 30 #include <linux/module.h> 31 #include <linux/percpu.h> 32 #include <linux/splice.h> 33 #include <linux/kdebug.h> 34 #include <linux/string.h> 35 #include <linux/mount.h> 36 #include <linux/rwsem.h> 37 #include <linux/slab.h> 38 #include <linux/ctype.h> 39 #include <linux/init.h> 40 #include <linux/poll.h> 41 #include <linux/nmi.h> 42 #include <linux/fs.h> 43 #include <linux/trace.h> 44 #include <linux/sched/rt.h> 45 46 #include "trace.h" 47 #include "trace_output.h" 48 49 /* 50 * On boot up, the ring buffer is set to the minimum size, so that 51 * we do not waste memory on systems that are not using tracing. 52 */ 53 bool ring_buffer_expanded; 54 55 /* 56 * We need to change this state when a selftest is running. 57 * A selftest will lurk into the ring-buffer to count the 58 * entries inserted during the selftest although some concurrent 59 * insertions into the ring-buffer such as trace_printk could occurred 60 * at the same time, giving false positive or negative results. 61 */ 62 static bool __read_mostly tracing_selftest_running; 63 64 /* 65 * If a tracer is running, we do not want to run SELFTEST. 66 */ 67 bool __read_mostly tracing_selftest_disabled; 68 69 /* Pipe tracepoints to printk */ 70 struct trace_iterator *tracepoint_print_iter; 71 int tracepoint_printk; 72 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key); 73 74 /* For tracers that don't implement custom flags */ 75 static struct tracer_opt dummy_tracer_opt[] = { 76 { } 77 }; 78 79 static int 80 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 81 { 82 return 0; 83 } 84 85 /* 86 * To prevent the comm cache from being overwritten when no 87 * tracing is active, only save the comm when a trace event 88 * occurred. 89 */ 90 static DEFINE_PER_CPU(bool, trace_taskinfo_save); 91 92 /* 93 * Kill all tracing for good (never come back). 94 * It is initialized to 1 but will turn to zero if the initialization 95 * of the tracer is successful. But that is the only place that sets 96 * this back to zero. 97 */ 98 static int tracing_disabled = 1; 99 100 cpumask_var_t __read_mostly tracing_buffer_mask; 101 102 /* 103 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops 104 * 105 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops 106 * is set, then ftrace_dump is called. This will output the contents 107 * of the ftrace buffers to the console. This is very useful for 108 * capturing traces that lead to crashes and outputing it to a 109 * serial console. 110 * 111 * It is default off, but you can enable it with either specifying 112 * "ftrace_dump_on_oops" in the kernel command line, or setting 113 * /proc/sys/kernel/ftrace_dump_on_oops 114 * Set 1 if you want to dump buffers of all CPUs 115 * Set 2 if you want to dump the buffer of the CPU that triggered oops 116 */ 117 118 enum ftrace_dump_mode ftrace_dump_on_oops; 119 120 /* When set, tracing will stop when a WARN*() is hit */ 121 int __disable_trace_on_warning; 122 123 #ifdef CONFIG_TRACE_EVAL_MAP_FILE 124 /* Map of enums to their values, for "eval_map" file */ 125 struct trace_eval_map_head { 126 struct module *mod; 127 unsigned long length; 128 }; 129 130 union trace_eval_map_item; 131 132 struct trace_eval_map_tail { 133 /* 134 * "end" is first and points to NULL as it must be different 135 * than "mod" or "eval_string" 136 */ 137 union trace_eval_map_item *next; 138 const char *end; /* points to NULL */ 139 }; 140 141 static DEFINE_MUTEX(trace_eval_mutex); 142 143 /* 144 * The trace_eval_maps are saved in an array with two extra elements, 145 * one at the beginning, and one at the end. The beginning item contains 146 * the count of the saved maps (head.length), and the module they 147 * belong to if not built in (head.mod). The ending item contains a 148 * pointer to the next array of saved eval_map items. 149 */ 150 union trace_eval_map_item { 151 struct trace_eval_map map; 152 struct trace_eval_map_head head; 153 struct trace_eval_map_tail tail; 154 }; 155 156 static union trace_eval_map_item *trace_eval_maps; 157 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */ 158 159 static int tracing_set_tracer(struct trace_array *tr, const char *buf); 160 161 #define MAX_TRACER_SIZE 100 162 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; 163 static char *default_bootup_tracer; 164 165 static bool allocate_snapshot; 166 167 static int __init set_cmdline_ftrace(char *str) 168 { 169 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE); 170 default_bootup_tracer = bootup_tracer_buf; 171 /* We are using ftrace early, expand it */ 172 ring_buffer_expanded = true; 173 return 1; 174 } 175 __setup("ftrace=", set_cmdline_ftrace); 176 177 static int __init set_ftrace_dump_on_oops(char *str) 178 { 179 if (*str++ != '=' || !*str) { 180 ftrace_dump_on_oops = DUMP_ALL; 181 return 1; 182 } 183 184 if (!strcmp("orig_cpu", str)) { 185 ftrace_dump_on_oops = DUMP_ORIG; 186 return 1; 187 } 188 189 return 0; 190 } 191 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops); 192 193 static int __init stop_trace_on_warning(char *str) 194 { 195 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0)) 196 __disable_trace_on_warning = 1; 197 return 1; 198 } 199 __setup("traceoff_on_warning", stop_trace_on_warning); 200 201 static int __init boot_alloc_snapshot(char *str) 202 { 203 allocate_snapshot = true; 204 /* We also need the main ring buffer expanded */ 205 ring_buffer_expanded = true; 206 return 1; 207 } 208 __setup("alloc_snapshot", boot_alloc_snapshot); 209 210 211 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata; 212 213 static int __init set_trace_boot_options(char *str) 214 { 215 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE); 216 return 0; 217 } 218 __setup("trace_options=", set_trace_boot_options); 219 220 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata; 221 static char *trace_boot_clock __initdata; 222 223 static int __init set_trace_boot_clock(char *str) 224 { 225 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE); 226 trace_boot_clock = trace_boot_clock_buf; 227 return 0; 228 } 229 __setup("trace_clock=", set_trace_boot_clock); 230 231 static int __init set_tracepoint_printk(char *str) 232 { 233 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0)) 234 tracepoint_printk = 1; 235 return 1; 236 } 237 __setup("tp_printk", set_tracepoint_printk); 238 239 unsigned long long ns2usecs(u64 nsec) 240 { 241 nsec += 500; 242 do_div(nsec, 1000); 243 return nsec; 244 } 245 246 /* trace_flags holds trace_options default values */ 247 #define TRACE_DEFAULT_FLAGS \ 248 (FUNCTION_DEFAULT_FLAGS | \ 249 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \ 250 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \ 251 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \ 252 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS) 253 254 /* trace_options that are only supported by global_trace */ 255 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \ 256 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD) 257 258 /* trace_flags that are default zero for instances */ 259 #define ZEROED_TRACE_FLAGS \ 260 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK) 261 262 /* 263 * The global_trace is the descriptor that holds the top-level tracing 264 * buffers for the live tracing. 265 */ 266 static struct trace_array global_trace = { 267 .trace_flags = TRACE_DEFAULT_FLAGS, 268 }; 269 270 LIST_HEAD(ftrace_trace_arrays); 271 272 int trace_array_get(struct trace_array *this_tr) 273 { 274 struct trace_array *tr; 275 int ret = -ENODEV; 276 277 mutex_lock(&trace_types_lock); 278 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 279 if (tr == this_tr) { 280 tr->ref++; 281 ret = 0; 282 break; 283 } 284 } 285 mutex_unlock(&trace_types_lock); 286 287 return ret; 288 } 289 290 static void __trace_array_put(struct trace_array *this_tr) 291 { 292 WARN_ON(!this_tr->ref); 293 this_tr->ref--; 294 } 295 296 void trace_array_put(struct trace_array *this_tr) 297 { 298 mutex_lock(&trace_types_lock); 299 __trace_array_put(this_tr); 300 mutex_unlock(&trace_types_lock); 301 } 302 303 int call_filter_check_discard(struct trace_event_call *call, void *rec, 304 struct ring_buffer *buffer, 305 struct ring_buffer_event *event) 306 { 307 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) && 308 !filter_match_preds(call->filter, rec)) { 309 __trace_event_discard_commit(buffer, event); 310 return 1; 311 } 312 313 return 0; 314 } 315 316 void trace_free_pid_list(struct trace_pid_list *pid_list) 317 { 318 vfree(pid_list->pids); 319 kfree(pid_list); 320 } 321 322 /** 323 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list 324 * @filtered_pids: The list of pids to check 325 * @search_pid: The PID to find in @filtered_pids 326 * 327 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis. 328 */ 329 bool 330 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid) 331 { 332 /* 333 * If pid_max changed after filtered_pids was created, we 334 * by default ignore all pids greater than the previous pid_max. 335 */ 336 if (search_pid >= filtered_pids->pid_max) 337 return false; 338 339 return test_bit(search_pid, filtered_pids->pids); 340 } 341 342 /** 343 * trace_ignore_this_task - should a task be ignored for tracing 344 * @filtered_pids: The list of pids to check 345 * @task: The task that should be ignored if not filtered 346 * 347 * Checks if @task should be traced or not from @filtered_pids. 348 * Returns true if @task should *NOT* be traced. 349 * Returns false if @task should be traced. 350 */ 351 bool 352 trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task) 353 { 354 /* 355 * Return false, because if filtered_pids does not exist, 356 * all pids are good to trace. 357 */ 358 if (!filtered_pids) 359 return false; 360 361 return !trace_find_filtered_pid(filtered_pids, task->pid); 362 } 363 364 /** 365 * trace_pid_filter_add_remove - Add or remove a task from a pid_list 366 * @pid_list: The list to modify 367 * @self: The current task for fork or NULL for exit 368 * @task: The task to add or remove 369 * 370 * If adding a task, if @self is defined, the task is only added if @self 371 * is also included in @pid_list. This happens on fork and tasks should 372 * only be added when the parent is listed. If @self is NULL, then the 373 * @task pid will be removed from the list, which would happen on exit 374 * of a task. 375 */ 376 void trace_filter_add_remove_task(struct trace_pid_list *pid_list, 377 struct task_struct *self, 378 struct task_struct *task) 379 { 380 if (!pid_list) 381 return; 382 383 /* For forks, we only add if the forking task is listed */ 384 if (self) { 385 if (!trace_find_filtered_pid(pid_list, self->pid)) 386 return; 387 } 388 389 /* Sorry, but we don't support pid_max changing after setting */ 390 if (task->pid >= pid_list->pid_max) 391 return; 392 393 /* "self" is set for forks, and NULL for exits */ 394 if (self) 395 set_bit(task->pid, pid_list->pids); 396 else 397 clear_bit(task->pid, pid_list->pids); 398 } 399 400 /** 401 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list 402 * @pid_list: The pid list to show 403 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed) 404 * @pos: The position of the file 405 * 406 * This is used by the seq_file "next" operation to iterate the pids 407 * listed in a trace_pid_list structure. 408 * 409 * Returns the pid+1 as we want to display pid of zero, but NULL would 410 * stop the iteration. 411 */ 412 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos) 413 { 414 unsigned long pid = (unsigned long)v; 415 416 (*pos)++; 417 418 /* pid already is +1 of the actual prevous bit */ 419 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid); 420 421 /* Return pid + 1 to allow zero to be represented */ 422 if (pid < pid_list->pid_max) 423 return (void *)(pid + 1); 424 425 return NULL; 426 } 427 428 /** 429 * trace_pid_start - Used for seq_file to start reading pid lists 430 * @pid_list: The pid list to show 431 * @pos: The position of the file 432 * 433 * This is used by seq_file "start" operation to start the iteration 434 * of listing pids. 435 * 436 * Returns the pid+1 as we want to display pid of zero, but NULL would 437 * stop the iteration. 438 */ 439 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos) 440 { 441 unsigned long pid; 442 loff_t l = 0; 443 444 pid = find_first_bit(pid_list->pids, pid_list->pid_max); 445 if (pid >= pid_list->pid_max) 446 return NULL; 447 448 /* Return pid + 1 so that zero can be the exit value */ 449 for (pid++; pid && l < *pos; 450 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l)) 451 ; 452 return (void *)pid; 453 } 454 455 /** 456 * trace_pid_show - show the current pid in seq_file processing 457 * @m: The seq_file structure to write into 458 * @v: A void pointer of the pid (+1) value to display 459 * 460 * Can be directly used by seq_file operations to display the current 461 * pid value. 462 */ 463 int trace_pid_show(struct seq_file *m, void *v) 464 { 465 unsigned long pid = (unsigned long)v - 1; 466 467 seq_printf(m, "%lu\n", pid); 468 return 0; 469 } 470 471 /* 128 should be much more than enough */ 472 #define PID_BUF_SIZE 127 473 474 int trace_pid_write(struct trace_pid_list *filtered_pids, 475 struct trace_pid_list **new_pid_list, 476 const char __user *ubuf, size_t cnt) 477 { 478 struct trace_pid_list *pid_list; 479 struct trace_parser parser; 480 unsigned long val; 481 int nr_pids = 0; 482 ssize_t read = 0; 483 ssize_t ret = 0; 484 loff_t pos; 485 pid_t pid; 486 487 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1)) 488 return -ENOMEM; 489 490 /* 491 * Always recreate a new array. The write is an all or nothing 492 * operation. Always create a new array when adding new pids by 493 * the user. If the operation fails, then the current list is 494 * not modified. 495 */ 496 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL); 497 if (!pid_list) 498 return -ENOMEM; 499 500 pid_list->pid_max = READ_ONCE(pid_max); 501 502 /* Only truncating will shrink pid_max */ 503 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max) 504 pid_list->pid_max = filtered_pids->pid_max; 505 506 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3); 507 if (!pid_list->pids) { 508 kfree(pid_list); 509 return -ENOMEM; 510 } 511 512 if (filtered_pids) { 513 /* copy the current bits to the new max */ 514 for_each_set_bit(pid, filtered_pids->pids, 515 filtered_pids->pid_max) { 516 set_bit(pid, pid_list->pids); 517 nr_pids++; 518 } 519 } 520 521 while (cnt > 0) { 522 523 pos = 0; 524 525 ret = trace_get_user(&parser, ubuf, cnt, &pos); 526 if (ret < 0 || !trace_parser_loaded(&parser)) 527 break; 528 529 read += ret; 530 ubuf += ret; 531 cnt -= ret; 532 533 parser.buffer[parser.idx] = 0; 534 535 ret = -EINVAL; 536 if (kstrtoul(parser.buffer, 0, &val)) 537 break; 538 if (val >= pid_list->pid_max) 539 break; 540 541 pid = (pid_t)val; 542 543 set_bit(pid, pid_list->pids); 544 nr_pids++; 545 546 trace_parser_clear(&parser); 547 ret = 0; 548 } 549 trace_parser_put(&parser); 550 551 if (ret < 0) { 552 trace_free_pid_list(pid_list); 553 return ret; 554 } 555 556 if (!nr_pids) { 557 /* Cleared the list of pids */ 558 trace_free_pid_list(pid_list); 559 read = ret; 560 pid_list = NULL; 561 } 562 563 *new_pid_list = pid_list; 564 565 return read; 566 } 567 568 static u64 buffer_ftrace_now(struct trace_buffer *buf, int cpu) 569 { 570 u64 ts; 571 572 /* Early boot up does not have a buffer yet */ 573 if (!buf->buffer) 574 return trace_clock_local(); 575 576 ts = ring_buffer_time_stamp(buf->buffer, cpu); 577 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts); 578 579 return ts; 580 } 581 582 u64 ftrace_now(int cpu) 583 { 584 return buffer_ftrace_now(&global_trace.trace_buffer, cpu); 585 } 586 587 /** 588 * tracing_is_enabled - Show if global_trace has been disabled 589 * 590 * Shows if the global trace has been enabled or not. It uses the 591 * mirror flag "buffer_disabled" to be used in fast paths such as for 592 * the irqsoff tracer. But it may be inaccurate due to races. If you 593 * need to know the accurate state, use tracing_is_on() which is a little 594 * slower, but accurate. 595 */ 596 int tracing_is_enabled(void) 597 { 598 /* 599 * For quick access (irqsoff uses this in fast path), just 600 * return the mirror variable of the state of the ring buffer. 601 * It's a little racy, but we don't really care. 602 */ 603 smp_rmb(); 604 return !global_trace.buffer_disabled; 605 } 606 607 /* 608 * trace_buf_size is the size in bytes that is allocated 609 * for a buffer. Note, the number of bytes is always rounded 610 * to page size. 611 * 612 * This number is purposely set to a low number of 16384. 613 * If the dump on oops happens, it will be much appreciated 614 * to not have to wait for all that output. Anyway this can be 615 * boot time and run time configurable. 616 */ 617 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */ 618 619 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT; 620 621 /* trace_types holds a link list of available tracers. */ 622 static struct tracer *trace_types __read_mostly; 623 624 /* 625 * trace_types_lock is used to protect the trace_types list. 626 */ 627 DEFINE_MUTEX(trace_types_lock); 628 629 /* 630 * serialize the access of the ring buffer 631 * 632 * ring buffer serializes readers, but it is low level protection. 633 * The validity of the events (which returns by ring_buffer_peek() ..etc) 634 * are not protected by ring buffer. 635 * 636 * The content of events may become garbage if we allow other process consumes 637 * these events concurrently: 638 * A) the page of the consumed events may become a normal page 639 * (not reader page) in ring buffer, and this page will be rewrited 640 * by events producer. 641 * B) The page of the consumed events may become a page for splice_read, 642 * and this page will be returned to system. 643 * 644 * These primitives allow multi process access to different cpu ring buffer 645 * concurrently. 646 * 647 * These primitives don't distinguish read-only and read-consume access. 648 * Multi read-only access are also serialized. 649 */ 650 651 #ifdef CONFIG_SMP 652 static DECLARE_RWSEM(all_cpu_access_lock); 653 static DEFINE_PER_CPU(struct mutex, cpu_access_lock); 654 655 static inline void trace_access_lock(int cpu) 656 { 657 if (cpu == RING_BUFFER_ALL_CPUS) { 658 /* gain it for accessing the whole ring buffer. */ 659 down_write(&all_cpu_access_lock); 660 } else { 661 /* gain it for accessing a cpu ring buffer. */ 662 663 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */ 664 down_read(&all_cpu_access_lock); 665 666 /* Secondly block other access to this @cpu ring buffer. */ 667 mutex_lock(&per_cpu(cpu_access_lock, cpu)); 668 } 669 } 670 671 static inline void trace_access_unlock(int cpu) 672 { 673 if (cpu == RING_BUFFER_ALL_CPUS) { 674 up_write(&all_cpu_access_lock); 675 } else { 676 mutex_unlock(&per_cpu(cpu_access_lock, cpu)); 677 up_read(&all_cpu_access_lock); 678 } 679 } 680 681 static inline void trace_access_lock_init(void) 682 { 683 int cpu; 684 685 for_each_possible_cpu(cpu) 686 mutex_init(&per_cpu(cpu_access_lock, cpu)); 687 } 688 689 #else 690 691 static DEFINE_MUTEX(access_lock); 692 693 static inline void trace_access_lock(int cpu) 694 { 695 (void)cpu; 696 mutex_lock(&access_lock); 697 } 698 699 static inline void trace_access_unlock(int cpu) 700 { 701 (void)cpu; 702 mutex_unlock(&access_lock); 703 } 704 705 static inline void trace_access_lock_init(void) 706 { 707 } 708 709 #endif 710 711 #ifdef CONFIG_STACKTRACE 712 static void __ftrace_trace_stack(struct ring_buffer *buffer, 713 unsigned long flags, 714 int skip, int pc, struct pt_regs *regs); 715 static inline void ftrace_trace_stack(struct trace_array *tr, 716 struct ring_buffer *buffer, 717 unsigned long flags, 718 int skip, int pc, struct pt_regs *regs); 719 720 #else 721 static inline void __ftrace_trace_stack(struct ring_buffer *buffer, 722 unsigned long flags, 723 int skip, int pc, struct pt_regs *regs) 724 { 725 } 726 static inline void ftrace_trace_stack(struct trace_array *tr, 727 struct ring_buffer *buffer, 728 unsigned long flags, 729 int skip, int pc, struct pt_regs *regs) 730 { 731 } 732 733 #endif 734 735 static __always_inline void 736 trace_event_setup(struct ring_buffer_event *event, 737 int type, unsigned long flags, int pc) 738 { 739 struct trace_entry *ent = ring_buffer_event_data(event); 740 741 tracing_generic_entry_update(ent, flags, pc); 742 ent->type = type; 743 } 744 745 static __always_inline struct ring_buffer_event * 746 __trace_buffer_lock_reserve(struct ring_buffer *buffer, 747 int type, 748 unsigned long len, 749 unsigned long flags, int pc) 750 { 751 struct ring_buffer_event *event; 752 753 event = ring_buffer_lock_reserve(buffer, len); 754 if (event != NULL) 755 trace_event_setup(event, type, flags, pc); 756 757 return event; 758 } 759 760 void tracer_tracing_on(struct trace_array *tr) 761 { 762 if (tr->trace_buffer.buffer) 763 ring_buffer_record_on(tr->trace_buffer.buffer); 764 /* 765 * This flag is looked at when buffers haven't been allocated 766 * yet, or by some tracers (like irqsoff), that just want to 767 * know if the ring buffer has been disabled, but it can handle 768 * races of where it gets disabled but we still do a record. 769 * As the check is in the fast path of the tracers, it is more 770 * important to be fast than accurate. 771 */ 772 tr->buffer_disabled = 0; 773 /* Make the flag seen by readers */ 774 smp_wmb(); 775 } 776 777 /** 778 * tracing_on - enable tracing buffers 779 * 780 * This function enables tracing buffers that may have been 781 * disabled with tracing_off. 782 */ 783 void tracing_on(void) 784 { 785 tracer_tracing_on(&global_trace); 786 } 787 EXPORT_SYMBOL_GPL(tracing_on); 788 789 790 static __always_inline void 791 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event) 792 { 793 __this_cpu_write(trace_taskinfo_save, true); 794 795 /* If this is the temp buffer, we need to commit fully */ 796 if (this_cpu_read(trace_buffered_event) == event) { 797 /* Length is in event->array[0] */ 798 ring_buffer_write(buffer, event->array[0], &event->array[1]); 799 /* Release the temp buffer */ 800 this_cpu_dec(trace_buffered_event_cnt); 801 } else 802 ring_buffer_unlock_commit(buffer, event); 803 } 804 805 /** 806 * __trace_puts - write a constant string into the trace buffer. 807 * @ip: The address of the caller 808 * @str: The constant string to write 809 * @size: The size of the string. 810 */ 811 int __trace_puts(unsigned long ip, const char *str, int size) 812 { 813 struct ring_buffer_event *event; 814 struct ring_buffer *buffer; 815 struct print_entry *entry; 816 unsigned long irq_flags; 817 int alloc; 818 int pc; 819 820 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 821 return 0; 822 823 pc = preempt_count(); 824 825 if (unlikely(tracing_selftest_running || tracing_disabled)) 826 return 0; 827 828 alloc = sizeof(*entry) + size + 2; /* possible \n added */ 829 830 local_save_flags(irq_flags); 831 buffer = global_trace.trace_buffer.buffer; 832 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 833 irq_flags, pc); 834 if (!event) 835 return 0; 836 837 entry = ring_buffer_event_data(event); 838 entry->ip = ip; 839 840 memcpy(&entry->buf, str, size); 841 842 /* Add a newline if necessary */ 843 if (entry->buf[size - 1] != '\n') { 844 entry->buf[size] = '\n'; 845 entry->buf[size + 1] = '\0'; 846 } else 847 entry->buf[size] = '\0'; 848 849 __buffer_unlock_commit(buffer, event); 850 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL); 851 852 return size; 853 } 854 EXPORT_SYMBOL_GPL(__trace_puts); 855 856 /** 857 * __trace_bputs - write the pointer to a constant string into trace buffer 858 * @ip: The address of the caller 859 * @str: The constant string to write to the buffer to 860 */ 861 int __trace_bputs(unsigned long ip, const char *str) 862 { 863 struct ring_buffer_event *event; 864 struct ring_buffer *buffer; 865 struct bputs_entry *entry; 866 unsigned long irq_flags; 867 int size = sizeof(struct bputs_entry); 868 int pc; 869 870 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 871 return 0; 872 873 pc = preempt_count(); 874 875 if (unlikely(tracing_selftest_running || tracing_disabled)) 876 return 0; 877 878 local_save_flags(irq_flags); 879 buffer = global_trace.trace_buffer.buffer; 880 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size, 881 irq_flags, pc); 882 if (!event) 883 return 0; 884 885 entry = ring_buffer_event_data(event); 886 entry->ip = ip; 887 entry->str = str; 888 889 __buffer_unlock_commit(buffer, event); 890 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL); 891 892 return 1; 893 } 894 EXPORT_SYMBOL_GPL(__trace_bputs); 895 896 #ifdef CONFIG_TRACER_SNAPSHOT 897 static void tracing_snapshot_instance(struct trace_array *tr) 898 { 899 struct tracer *tracer = tr->current_trace; 900 unsigned long flags; 901 902 if (in_nmi()) { 903 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n"); 904 internal_trace_puts("*** snapshot is being ignored ***\n"); 905 return; 906 } 907 908 if (!tr->allocated_snapshot) { 909 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n"); 910 internal_trace_puts("*** stopping trace here! ***\n"); 911 tracing_off(); 912 return; 913 } 914 915 /* Note, snapshot can not be used when the tracer uses it */ 916 if (tracer->use_max_tr) { 917 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n"); 918 internal_trace_puts("*** Can not use snapshot (sorry) ***\n"); 919 return; 920 } 921 922 local_irq_save(flags); 923 update_max_tr(tr, current, smp_processor_id()); 924 local_irq_restore(flags); 925 } 926 927 /** 928 * trace_snapshot - take a snapshot of the current buffer. 929 * 930 * This causes a swap between the snapshot buffer and the current live 931 * tracing buffer. You can use this to take snapshots of the live 932 * trace when some condition is triggered, but continue to trace. 933 * 934 * Note, make sure to allocate the snapshot with either 935 * a tracing_snapshot_alloc(), or by doing it manually 936 * with: echo 1 > /sys/kernel/debug/tracing/snapshot 937 * 938 * If the snapshot buffer is not allocated, it will stop tracing. 939 * Basically making a permanent snapshot. 940 */ 941 void tracing_snapshot(void) 942 { 943 struct trace_array *tr = &global_trace; 944 945 tracing_snapshot_instance(tr); 946 } 947 EXPORT_SYMBOL_GPL(tracing_snapshot); 948 949 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf, 950 struct trace_buffer *size_buf, int cpu_id); 951 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val); 952 953 static int alloc_snapshot(struct trace_array *tr) 954 { 955 int ret; 956 957 if (!tr->allocated_snapshot) { 958 959 /* allocate spare buffer */ 960 ret = resize_buffer_duplicate_size(&tr->max_buffer, 961 &tr->trace_buffer, RING_BUFFER_ALL_CPUS); 962 if (ret < 0) 963 return ret; 964 965 tr->allocated_snapshot = true; 966 } 967 968 return 0; 969 } 970 971 static void free_snapshot(struct trace_array *tr) 972 { 973 /* 974 * We don't free the ring buffer. instead, resize it because 975 * The max_tr ring buffer has some state (e.g. ring->clock) and 976 * we want preserve it. 977 */ 978 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS); 979 set_buffer_entries(&tr->max_buffer, 1); 980 tracing_reset_online_cpus(&tr->max_buffer); 981 tr->allocated_snapshot = false; 982 } 983 984 /** 985 * tracing_alloc_snapshot - allocate snapshot buffer. 986 * 987 * This only allocates the snapshot buffer if it isn't already 988 * allocated - it doesn't also take a snapshot. 989 * 990 * This is meant to be used in cases where the snapshot buffer needs 991 * to be set up for events that can't sleep but need to be able to 992 * trigger a snapshot. 993 */ 994 int tracing_alloc_snapshot(void) 995 { 996 struct trace_array *tr = &global_trace; 997 int ret; 998 999 ret = alloc_snapshot(tr); 1000 WARN_ON(ret < 0); 1001 1002 return ret; 1003 } 1004 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); 1005 1006 /** 1007 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer. 1008 * 1009 * This is similar to trace_snapshot(), but it will allocate the 1010 * snapshot buffer if it isn't already allocated. Use this only 1011 * where it is safe to sleep, as the allocation may sleep. 1012 * 1013 * This causes a swap between the snapshot buffer and the current live 1014 * tracing buffer. You can use this to take snapshots of the live 1015 * trace when some condition is triggered, but continue to trace. 1016 */ 1017 void tracing_snapshot_alloc(void) 1018 { 1019 int ret; 1020 1021 ret = tracing_alloc_snapshot(); 1022 if (ret < 0) 1023 return; 1024 1025 tracing_snapshot(); 1026 } 1027 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); 1028 #else 1029 void tracing_snapshot(void) 1030 { 1031 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used"); 1032 } 1033 EXPORT_SYMBOL_GPL(tracing_snapshot); 1034 int tracing_alloc_snapshot(void) 1035 { 1036 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used"); 1037 return -ENODEV; 1038 } 1039 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); 1040 void tracing_snapshot_alloc(void) 1041 { 1042 /* Give warning */ 1043 tracing_snapshot(); 1044 } 1045 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); 1046 #endif /* CONFIG_TRACER_SNAPSHOT */ 1047 1048 void tracer_tracing_off(struct trace_array *tr) 1049 { 1050 if (tr->trace_buffer.buffer) 1051 ring_buffer_record_off(tr->trace_buffer.buffer); 1052 /* 1053 * This flag is looked at when buffers haven't been allocated 1054 * yet, or by some tracers (like irqsoff), that just want to 1055 * know if the ring buffer has been disabled, but it can handle 1056 * races of where it gets disabled but we still do a record. 1057 * As the check is in the fast path of the tracers, it is more 1058 * important to be fast than accurate. 1059 */ 1060 tr->buffer_disabled = 1; 1061 /* Make the flag seen by readers */ 1062 smp_wmb(); 1063 } 1064 1065 /** 1066 * tracing_off - turn off tracing buffers 1067 * 1068 * This function stops the tracing buffers from recording data. 1069 * It does not disable any overhead the tracers themselves may 1070 * be causing. This function simply causes all recording to 1071 * the ring buffers to fail. 1072 */ 1073 void tracing_off(void) 1074 { 1075 tracer_tracing_off(&global_trace); 1076 } 1077 EXPORT_SYMBOL_GPL(tracing_off); 1078 1079 void disable_trace_on_warning(void) 1080 { 1081 if (__disable_trace_on_warning) 1082 tracing_off(); 1083 } 1084 1085 /** 1086 * tracer_tracing_is_on - show real state of ring buffer enabled 1087 * @tr : the trace array to know if ring buffer is enabled 1088 * 1089 * Shows real state of the ring buffer if it is enabled or not. 1090 */ 1091 int tracer_tracing_is_on(struct trace_array *tr) 1092 { 1093 if (tr->trace_buffer.buffer) 1094 return ring_buffer_record_is_on(tr->trace_buffer.buffer); 1095 return !tr->buffer_disabled; 1096 } 1097 1098 /** 1099 * tracing_is_on - show state of ring buffers enabled 1100 */ 1101 int tracing_is_on(void) 1102 { 1103 return tracer_tracing_is_on(&global_trace); 1104 } 1105 EXPORT_SYMBOL_GPL(tracing_is_on); 1106 1107 static int __init set_buf_size(char *str) 1108 { 1109 unsigned long buf_size; 1110 1111 if (!str) 1112 return 0; 1113 buf_size = memparse(str, &str); 1114 /* nr_entries can not be zero */ 1115 if (buf_size == 0) 1116 return 0; 1117 trace_buf_size = buf_size; 1118 return 1; 1119 } 1120 __setup("trace_buf_size=", set_buf_size); 1121 1122 static int __init set_tracing_thresh(char *str) 1123 { 1124 unsigned long threshold; 1125 int ret; 1126 1127 if (!str) 1128 return 0; 1129 ret = kstrtoul(str, 0, &threshold); 1130 if (ret < 0) 1131 return 0; 1132 tracing_thresh = threshold * 1000; 1133 return 1; 1134 } 1135 __setup("tracing_thresh=", set_tracing_thresh); 1136 1137 unsigned long nsecs_to_usecs(unsigned long nsecs) 1138 { 1139 return nsecs / 1000; 1140 } 1141 1142 /* 1143 * TRACE_FLAGS is defined as a tuple matching bit masks with strings. 1144 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that 1145 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list 1146 * of strings in the order that the evals (enum) were defined. 1147 */ 1148 #undef C 1149 #define C(a, b) b 1150 1151 /* These must match the bit postions in trace_iterator_flags */ 1152 static const char *trace_options[] = { 1153 TRACE_FLAGS 1154 NULL 1155 }; 1156 1157 static struct { 1158 u64 (*func)(void); 1159 const char *name; 1160 int in_ns; /* is this clock in nanoseconds? */ 1161 } trace_clocks[] = { 1162 { trace_clock_local, "local", 1 }, 1163 { trace_clock_global, "global", 1 }, 1164 { trace_clock_counter, "counter", 0 }, 1165 { trace_clock_jiffies, "uptime", 0 }, 1166 { trace_clock, "perf", 1 }, 1167 { ktime_get_mono_fast_ns, "mono", 1 }, 1168 { ktime_get_raw_fast_ns, "mono_raw", 1 }, 1169 { ktime_get_boot_fast_ns, "boot", 1 }, 1170 ARCH_TRACE_CLOCKS 1171 }; 1172 1173 /* 1174 * trace_parser_get_init - gets the buffer for trace parser 1175 */ 1176 int trace_parser_get_init(struct trace_parser *parser, int size) 1177 { 1178 memset(parser, 0, sizeof(*parser)); 1179 1180 parser->buffer = kmalloc(size, GFP_KERNEL); 1181 if (!parser->buffer) 1182 return 1; 1183 1184 parser->size = size; 1185 return 0; 1186 } 1187 1188 /* 1189 * trace_parser_put - frees the buffer for trace parser 1190 */ 1191 void trace_parser_put(struct trace_parser *parser) 1192 { 1193 kfree(parser->buffer); 1194 parser->buffer = NULL; 1195 } 1196 1197 /* 1198 * trace_get_user - reads the user input string separated by space 1199 * (matched by isspace(ch)) 1200 * 1201 * For each string found the 'struct trace_parser' is updated, 1202 * and the function returns. 1203 * 1204 * Returns number of bytes read. 1205 * 1206 * See kernel/trace/trace.h for 'struct trace_parser' details. 1207 */ 1208 int trace_get_user(struct trace_parser *parser, const char __user *ubuf, 1209 size_t cnt, loff_t *ppos) 1210 { 1211 char ch; 1212 size_t read = 0; 1213 ssize_t ret; 1214 1215 if (!*ppos) 1216 trace_parser_clear(parser); 1217 1218 ret = get_user(ch, ubuf++); 1219 if (ret) 1220 goto out; 1221 1222 read++; 1223 cnt--; 1224 1225 /* 1226 * The parser is not finished with the last write, 1227 * continue reading the user input without skipping spaces. 1228 */ 1229 if (!parser->cont) { 1230 /* skip white space */ 1231 while (cnt && isspace(ch)) { 1232 ret = get_user(ch, ubuf++); 1233 if (ret) 1234 goto out; 1235 read++; 1236 cnt--; 1237 } 1238 1239 /* only spaces were written */ 1240 if (isspace(ch)) { 1241 *ppos += read; 1242 ret = read; 1243 goto out; 1244 } 1245 1246 parser->idx = 0; 1247 } 1248 1249 /* read the non-space input */ 1250 while (cnt && !isspace(ch)) { 1251 if (parser->idx < parser->size - 1) 1252 parser->buffer[parser->idx++] = ch; 1253 else { 1254 ret = -EINVAL; 1255 goto out; 1256 } 1257 ret = get_user(ch, ubuf++); 1258 if (ret) 1259 goto out; 1260 read++; 1261 cnt--; 1262 } 1263 1264 /* We either got finished input or we have to wait for another call. */ 1265 if (isspace(ch)) { 1266 parser->buffer[parser->idx] = 0; 1267 parser->cont = false; 1268 } else if (parser->idx < parser->size - 1) { 1269 parser->cont = true; 1270 parser->buffer[parser->idx++] = ch; 1271 } else { 1272 ret = -EINVAL; 1273 goto out; 1274 } 1275 1276 *ppos += read; 1277 ret = read; 1278 1279 out: 1280 return ret; 1281 } 1282 1283 /* TODO add a seq_buf_to_buffer() */ 1284 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) 1285 { 1286 int len; 1287 1288 if (trace_seq_used(s) <= s->seq.readpos) 1289 return -EBUSY; 1290 1291 len = trace_seq_used(s) - s->seq.readpos; 1292 if (cnt > len) 1293 cnt = len; 1294 memcpy(buf, s->buffer + s->seq.readpos, cnt); 1295 1296 s->seq.readpos += cnt; 1297 return cnt; 1298 } 1299 1300 unsigned long __read_mostly tracing_thresh; 1301 1302 #ifdef CONFIG_TRACER_MAX_TRACE 1303 /* 1304 * Copy the new maximum trace into the separate maximum-trace 1305 * structure. (this way the maximum trace is permanently saved, 1306 * for later retrieval via /sys/kernel/debug/tracing/latency_trace) 1307 */ 1308 static void 1309 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 1310 { 1311 struct trace_buffer *trace_buf = &tr->trace_buffer; 1312 struct trace_buffer *max_buf = &tr->max_buffer; 1313 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu); 1314 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu); 1315 1316 max_buf->cpu = cpu; 1317 max_buf->time_start = data->preempt_timestamp; 1318 1319 max_data->saved_latency = tr->max_latency; 1320 max_data->critical_start = data->critical_start; 1321 max_data->critical_end = data->critical_end; 1322 1323 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN); 1324 max_data->pid = tsk->pid; 1325 /* 1326 * If tsk == current, then use current_uid(), as that does not use 1327 * RCU. The irq tracer can be called out of RCU scope. 1328 */ 1329 if (tsk == current) 1330 max_data->uid = current_uid(); 1331 else 1332 max_data->uid = task_uid(tsk); 1333 1334 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; 1335 max_data->policy = tsk->policy; 1336 max_data->rt_priority = tsk->rt_priority; 1337 1338 /* record this tasks comm */ 1339 tracing_record_cmdline(tsk); 1340 } 1341 1342 /** 1343 * update_max_tr - snapshot all trace buffers from global_trace to max_tr 1344 * @tr: tracer 1345 * @tsk: the task with the latency 1346 * @cpu: The cpu that initiated the trace. 1347 * 1348 * Flip the buffers between the @tr and the max_tr and record information 1349 * about which task was the cause of this latency. 1350 */ 1351 void 1352 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 1353 { 1354 struct ring_buffer *buf; 1355 1356 if (tr->stop_count) 1357 return; 1358 1359 WARN_ON_ONCE(!irqs_disabled()); 1360 1361 if (!tr->allocated_snapshot) { 1362 /* Only the nop tracer should hit this when disabling */ 1363 WARN_ON_ONCE(tr->current_trace != &nop_trace); 1364 return; 1365 } 1366 1367 arch_spin_lock(&tr->max_lock); 1368 1369 buf = tr->trace_buffer.buffer; 1370 tr->trace_buffer.buffer = tr->max_buffer.buffer; 1371 tr->max_buffer.buffer = buf; 1372 1373 __update_max_tr(tr, tsk, cpu); 1374 arch_spin_unlock(&tr->max_lock); 1375 } 1376 1377 /** 1378 * update_max_tr_single - only copy one trace over, and reset the rest 1379 * @tr - tracer 1380 * @tsk - task with the latency 1381 * @cpu - the cpu of the buffer to copy. 1382 * 1383 * Flip the trace of a single CPU buffer between the @tr and the max_tr. 1384 */ 1385 void 1386 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) 1387 { 1388 int ret; 1389 1390 if (tr->stop_count) 1391 return; 1392 1393 WARN_ON_ONCE(!irqs_disabled()); 1394 if (!tr->allocated_snapshot) { 1395 /* Only the nop tracer should hit this when disabling */ 1396 WARN_ON_ONCE(tr->current_trace != &nop_trace); 1397 return; 1398 } 1399 1400 arch_spin_lock(&tr->max_lock); 1401 1402 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu); 1403 1404 if (ret == -EBUSY) { 1405 /* 1406 * We failed to swap the buffer due to a commit taking 1407 * place on this CPU. We fail to record, but we reset 1408 * the max trace buffer (no one writes directly to it) 1409 * and flag that it failed. 1410 */ 1411 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_, 1412 "Failed to swap buffers due to commit in progress\n"); 1413 } 1414 1415 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); 1416 1417 __update_max_tr(tr, tsk, cpu); 1418 arch_spin_unlock(&tr->max_lock); 1419 } 1420 #endif /* CONFIG_TRACER_MAX_TRACE */ 1421 1422 static int wait_on_pipe(struct trace_iterator *iter, bool full) 1423 { 1424 /* Iterators are static, they should be filled or empty */ 1425 if (trace_buffer_iter(iter, iter->cpu_file)) 1426 return 0; 1427 1428 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file, 1429 full); 1430 } 1431 1432 #ifdef CONFIG_FTRACE_STARTUP_TEST 1433 static bool selftests_can_run; 1434 1435 struct trace_selftests { 1436 struct list_head list; 1437 struct tracer *type; 1438 }; 1439 1440 static LIST_HEAD(postponed_selftests); 1441 1442 static int save_selftest(struct tracer *type) 1443 { 1444 struct trace_selftests *selftest; 1445 1446 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL); 1447 if (!selftest) 1448 return -ENOMEM; 1449 1450 selftest->type = type; 1451 list_add(&selftest->list, &postponed_selftests); 1452 return 0; 1453 } 1454 1455 static int run_tracer_selftest(struct tracer *type) 1456 { 1457 struct trace_array *tr = &global_trace; 1458 struct tracer *saved_tracer = tr->current_trace; 1459 int ret; 1460 1461 if (!type->selftest || tracing_selftest_disabled) 1462 return 0; 1463 1464 /* 1465 * If a tracer registers early in boot up (before scheduling is 1466 * initialized and such), then do not run its selftests yet. 1467 * Instead, run it a little later in the boot process. 1468 */ 1469 if (!selftests_can_run) 1470 return save_selftest(type); 1471 1472 /* 1473 * Run a selftest on this tracer. 1474 * Here we reset the trace buffer, and set the current 1475 * tracer to be this tracer. The tracer can then run some 1476 * internal tracing to verify that everything is in order. 1477 * If we fail, we do not register this tracer. 1478 */ 1479 tracing_reset_online_cpus(&tr->trace_buffer); 1480 1481 tr->current_trace = type; 1482 1483 #ifdef CONFIG_TRACER_MAX_TRACE 1484 if (type->use_max_tr) { 1485 /* If we expanded the buffers, make sure the max is expanded too */ 1486 if (ring_buffer_expanded) 1487 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size, 1488 RING_BUFFER_ALL_CPUS); 1489 tr->allocated_snapshot = true; 1490 } 1491 #endif 1492 1493 /* the test is responsible for initializing and enabling */ 1494 pr_info("Testing tracer %s: ", type->name); 1495 ret = type->selftest(type, tr); 1496 /* the test is responsible for resetting too */ 1497 tr->current_trace = saved_tracer; 1498 if (ret) { 1499 printk(KERN_CONT "FAILED!\n"); 1500 /* Add the warning after printing 'FAILED' */ 1501 WARN_ON(1); 1502 return -1; 1503 } 1504 /* Only reset on passing, to avoid touching corrupted buffers */ 1505 tracing_reset_online_cpus(&tr->trace_buffer); 1506 1507 #ifdef CONFIG_TRACER_MAX_TRACE 1508 if (type->use_max_tr) { 1509 tr->allocated_snapshot = false; 1510 1511 /* Shrink the max buffer again */ 1512 if (ring_buffer_expanded) 1513 ring_buffer_resize(tr->max_buffer.buffer, 1, 1514 RING_BUFFER_ALL_CPUS); 1515 } 1516 #endif 1517 1518 printk(KERN_CONT "PASSED\n"); 1519 return 0; 1520 } 1521 1522 static __init int init_trace_selftests(void) 1523 { 1524 struct trace_selftests *p, *n; 1525 struct tracer *t, **last; 1526 int ret; 1527 1528 selftests_can_run = true; 1529 1530 mutex_lock(&trace_types_lock); 1531 1532 if (list_empty(&postponed_selftests)) 1533 goto out; 1534 1535 pr_info("Running postponed tracer tests:\n"); 1536 1537 list_for_each_entry_safe(p, n, &postponed_selftests, list) { 1538 ret = run_tracer_selftest(p->type); 1539 /* If the test fails, then warn and remove from available_tracers */ 1540 if (ret < 0) { 1541 WARN(1, "tracer: %s failed selftest, disabling\n", 1542 p->type->name); 1543 last = &trace_types; 1544 for (t = trace_types; t; t = t->next) { 1545 if (t == p->type) { 1546 *last = t->next; 1547 break; 1548 } 1549 last = &t->next; 1550 } 1551 } 1552 list_del(&p->list); 1553 kfree(p); 1554 } 1555 1556 out: 1557 mutex_unlock(&trace_types_lock); 1558 1559 return 0; 1560 } 1561 core_initcall(init_trace_selftests); 1562 #else 1563 static inline int run_tracer_selftest(struct tracer *type) 1564 { 1565 return 0; 1566 } 1567 #endif /* CONFIG_FTRACE_STARTUP_TEST */ 1568 1569 static void add_tracer_options(struct trace_array *tr, struct tracer *t); 1570 1571 static void __init apply_trace_boot_options(void); 1572 1573 /** 1574 * register_tracer - register a tracer with the ftrace system. 1575 * @type - the plugin for the tracer 1576 * 1577 * Register a new plugin tracer. 1578 */ 1579 int __init register_tracer(struct tracer *type) 1580 { 1581 struct tracer *t; 1582 int ret = 0; 1583 1584 if (!type->name) { 1585 pr_info("Tracer must have a name\n"); 1586 return -1; 1587 } 1588 1589 if (strlen(type->name) >= MAX_TRACER_SIZE) { 1590 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE); 1591 return -1; 1592 } 1593 1594 mutex_lock(&trace_types_lock); 1595 1596 tracing_selftest_running = true; 1597 1598 for (t = trace_types; t; t = t->next) { 1599 if (strcmp(type->name, t->name) == 0) { 1600 /* already found */ 1601 pr_info("Tracer %s already registered\n", 1602 type->name); 1603 ret = -1; 1604 goto out; 1605 } 1606 } 1607 1608 if (!type->set_flag) 1609 type->set_flag = &dummy_set_flag; 1610 if (!type->flags) { 1611 /*allocate a dummy tracer_flags*/ 1612 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL); 1613 if (!type->flags) { 1614 ret = -ENOMEM; 1615 goto out; 1616 } 1617 type->flags->val = 0; 1618 type->flags->opts = dummy_tracer_opt; 1619 } else 1620 if (!type->flags->opts) 1621 type->flags->opts = dummy_tracer_opt; 1622 1623 /* store the tracer for __set_tracer_option */ 1624 type->flags->trace = type; 1625 1626 ret = run_tracer_selftest(type); 1627 if (ret < 0) 1628 goto out; 1629 1630 type->next = trace_types; 1631 trace_types = type; 1632 add_tracer_options(&global_trace, type); 1633 1634 out: 1635 tracing_selftest_running = false; 1636 mutex_unlock(&trace_types_lock); 1637 1638 if (ret || !default_bootup_tracer) 1639 goto out_unlock; 1640 1641 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE)) 1642 goto out_unlock; 1643 1644 printk(KERN_INFO "Starting tracer '%s'\n", type->name); 1645 /* Do we want this tracer to start on bootup? */ 1646 tracing_set_tracer(&global_trace, type->name); 1647 default_bootup_tracer = NULL; 1648 1649 apply_trace_boot_options(); 1650 1651 /* disable other selftests, since this will break it. */ 1652 tracing_selftest_disabled = true; 1653 #ifdef CONFIG_FTRACE_STARTUP_TEST 1654 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n", 1655 type->name); 1656 #endif 1657 1658 out_unlock: 1659 return ret; 1660 } 1661 1662 void tracing_reset(struct trace_buffer *buf, int cpu) 1663 { 1664 struct ring_buffer *buffer = buf->buffer; 1665 1666 if (!buffer) 1667 return; 1668 1669 ring_buffer_record_disable(buffer); 1670 1671 /* Make sure all commits have finished */ 1672 synchronize_sched(); 1673 ring_buffer_reset_cpu(buffer, cpu); 1674 1675 ring_buffer_record_enable(buffer); 1676 } 1677 1678 void tracing_reset_online_cpus(struct trace_buffer *buf) 1679 { 1680 struct ring_buffer *buffer = buf->buffer; 1681 int cpu; 1682 1683 if (!buffer) 1684 return; 1685 1686 ring_buffer_record_disable(buffer); 1687 1688 /* Make sure all commits have finished */ 1689 synchronize_sched(); 1690 1691 buf->time_start = buffer_ftrace_now(buf, buf->cpu); 1692 1693 for_each_online_cpu(cpu) 1694 ring_buffer_reset_cpu(buffer, cpu); 1695 1696 ring_buffer_record_enable(buffer); 1697 } 1698 1699 /* Must have trace_types_lock held */ 1700 void tracing_reset_all_online_cpus(void) 1701 { 1702 struct trace_array *tr; 1703 1704 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 1705 if (!tr->clear_trace) 1706 continue; 1707 tr->clear_trace = false; 1708 tracing_reset_online_cpus(&tr->trace_buffer); 1709 #ifdef CONFIG_TRACER_MAX_TRACE 1710 tracing_reset_online_cpus(&tr->max_buffer); 1711 #endif 1712 } 1713 } 1714 1715 static int *tgid_map; 1716 1717 #define SAVED_CMDLINES_DEFAULT 128 1718 #define NO_CMDLINE_MAP UINT_MAX 1719 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED; 1720 struct saved_cmdlines_buffer { 1721 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; 1722 unsigned *map_cmdline_to_pid; 1723 unsigned cmdline_num; 1724 int cmdline_idx; 1725 char *saved_cmdlines; 1726 }; 1727 static struct saved_cmdlines_buffer *savedcmd; 1728 1729 /* temporary disable recording */ 1730 static atomic_t trace_record_taskinfo_disabled __read_mostly; 1731 1732 static inline char *get_saved_cmdlines(int idx) 1733 { 1734 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN]; 1735 } 1736 1737 static inline void set_cmdline(int idx, const char *cmdline) 1738 { 1739 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN); 1740 } 1741 1742 static int allocate_cmdlines_buffer(unsigned int val, 1743 struct saved_cmdlines_buffer *s) 1744 { 1745 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid), 1746 GFP_KERNEL); 1747 if (!s->map_cmdline_to_pid) 1748 return -ENOMEM; 1749 1750 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL); 1751 if (!s->saved_cmdlines) { 1752 kfree(s->map_cmdline_to_pid); 1753 return -ENOMEM; 1754 } 1755 1756 s->cmdline_idx = 0; 1757 s->cmdline_num = val; 1758 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP, 1759 sizeof(s->map_pid_to_cmdline)); 1760 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP, 1761 val * sizeof(*s->map_cmdline_to_pid)); 1762 1763 return 0; 1764 } 1765 1766 static int trace_create_savedcmd(void) 1767 { 1768 int ret; 1769 1770 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL); 1771 if (!savedcmd) 1772 return -ENOMEM; 1773 1774 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd); 1775 if (ret < 0) { 1776 kfree(savedcmd); 1777 savedcmd = NULL; 1778 return -ENOMEM; 1779 } 1780 1781 return 0; 1782 } 1783 1784 int is_tracing_stopped(void) 1785 { 1786 return global_trace.stop_count; 1787 } 1788 1789 /** 1790 * tracing_start - quick start of the tracer 1791 * 1792 * If tracing is enabled but was stopped by tracing_stop, 1793 * this will start the tracer back up. 1794 */ 1795 void tracing_start(void) 1796 { 1797 struct ring_buffer *buffer; 1798 unsigned long flags; 1799 1800 if (tracing_disabled) 1801 return; 1802 1803 raw_spin_lock_irqsave(&global_trace.start_lock, flags); 1804 if (--global_trace.stop_count) { 1805 if (global_trace.stop_count < 0) { 1806 /* Someone screwed up their debugging */ 1807 WARN_ON_ONCE(1); 1808 global_trace.stop_count = 0; 1809 } 1810 goto out; 1811 } 1812 1813 /* Prevent the buffers from switching */ 1814 arch_spin_lock(&global_trace.max_lock); 1815 1816 buffer = global_trace.trace_buffer.buffer; 1817 if (buffer) 1818 ring_buffer_record_enable(buffer); 1819 1820 #ifdef CONFIG_TRACER_MAX_TRACE 1821 buffer = global_trace.max_buffer.buffer; 1822 if (buffer) 1823 ring_buffer_record_enable(buffer); 1824 #endif 1825 1826 arch_spin_unlock(&global_trace.max_lock); 1827 1828 out: 1829 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags); 1830 } 1831 1832 static void tracing_start_tr(struct trace_array *tr) 1833 { 1834 struct ring_buffer *buffer; 1835 unsigned long flags; 1836 1837 if (tracing_disabled) 1838 return; 1839 1840 /* If global, we need to also start the max tracer */ 1841 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 1842 return tracing_start(); 1843 1844 raw_spin_lock_irqsave(&tr->start_lock, flags); 1845 1846 if (--tr->stop_count) { 1847 if (tr->stop_count < 0) { 1848 /* Someone screwed up their debugging */ 1849 WARN_ON_ONCE(1); 1850 tr->stop_count = 0; 1851 } 1852 goto out; 1853 } 1854 1855 buffer = tr->trace_buffer.buffer; 1856 if (buffer) 1857 ring_buffer_record_enable(buffer); 1858 1859 out: 1860 raw_spin_unlock_irqrestore(&tr->start_lock, flags); 1861 } 1862 1863 /** 1864 * tracing_stop - quick stop of the tracer 1865 * 1866 * Light weight way to stop tracing. Use in conjunction with 1867 * tracing_start. 1868 */ 1869 void tracing_stop(void) 1870 { 1871 struct ring_buffer *buffer; 1872 unsigned long flags; 1873 1874 raw_spin_lock_irqsave(&global_trace.start_lock, flags); 1875 if (global_trace.stop_count++) 1876 goto out; 1877 1878 /* Prevent the buffers from switching */ 1879 arch_spin_lock(&global_trace.max_lock); 1880 1881 buffer = global_trace.trace_buffer.buffer; 1882 if (buffer) 1883 ring_buffer_record_disable(buffer); 1884 1885 #ifdef CONFIG_TRACER_MAX_TRACE 1886 buffer = global_trace.max_buffer.buffer; 1887 if (buffer) 1888 ring_buffer_record_disable(buffer); 1889 #endif 1890 1891 arch_spin_unlock(&global_trace.max_lock); 1892 1893 out: 1894 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags); 1895 } 1896 1897 static void tracing_stop_tr(struct trace_array *tr) 1898 { 1899 struct ring_buffer *buffer; 1900 unsigned long flags; 1901 1902 /* If global, we need to also stop the max tracer */ 1903 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 1904 return tracing_stop(); 1905 1906 raw_spin_lock_irqsave(&tr->start_lock, flags); 1907 if (tr->stop_count++) 1908 goto out; 1909 1910 buffer = tr->trace_buffer.buffer; 1911 if (buffer) 1912 ring_buffer_record_disable(buffer); 1913 1914 out: 1915 raw_spin_unlock_irqrestore(&tr->start_lock, flags); 1916 } 1917 1918 static int trace_save_cmdline(struct task_struct *tsk) 1919 { 1920 unsigned pid, idx; 1921 1922 /* treat recording of idle task as a success */ 1923 if (!tsk->pid) 1924 return 1; 1925 1926 if (unlikely(tsk->pid > PID_MAX_DEFAULT)) 1927 return 0; 1928 1929 /* 1930 * It's not the end of the world if we don't get 1931 * the lock, but we also don't want to spin 1932 * nor do we want to disable interrupts, 1933 * so if we miss here, then better luck next time. 1934 */ 1935 if (!arch_spin_trylock(&trace_cmdline_lock)) 1936 return 0; 1937 1938 idx = savedcmd->map_pid_to_cmdline[tsk->pid]; 1939 if (idx == NO_CMDLINE_MAP) { 1940 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num; 1941 1942 /* 1943 * Check whether the cmdline buffer at idx has a pid 1944 * mapped. We are going to overwrite that entry so we 1945 * need to clear the map_pid_to_cmdline. Otherwise we 1946 * would read the new comm for the old pid. 1947 */ 1948 pid = savedcmd->map_cmdline_to_pid[idx]; 1949 if (pid != NO_CMDLINE_MAP) 1950 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP; 1951 1952 savedcmd->map_cmdline_to_pid[idx] = tsk->pid; 1953 savedcmd->map_pid_to_cmdline[tsk->pid] = idx; 1954 1955 savedcmd->cmdline_idx = idx; 1956 } 1957 1958 set_cmdline(idx, tsk->comm); 1959 1960 arch_spin_unlock(&trace_cmdline_lock); 1961 1962 return 1; 1963 } 1964 1965 static void __trace_find_cmdline(int pid, char comm[]) 1966 { 1967 unsigned map; 1968 1969 if (!pid) { 1970 strcpy(comm, "<idle>"); 1971 return; 1972 } 1973 1974 if (WARN_ON_ONCE(pid < 0)) { 1975 strcpy(comm, "<XXX>"); 1976 return; 1977 } 1978 1979 if (pid > PID_MAX_DEFAULT) { 1980 strcpy(comm, "<...>"); 1981 return; 1982 } 1983 1984 map = savedcmd->map_pid_to_cmdline[pid]; 1985 if (map != NO_CMDLINE_MAP) 1986 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN); 1987 else 1988 strcpy(comm, "<...>"); 1989 } 1990 1991 void trace_find_cmdline(int pid, char comm[]) 1992 { 1993 preempt_disable(); 1994 arch_spin_lock(&trace_cmdline_lock); 1995 1996 __trace_find_cmdline(pid, comm); 1997 1998 arch_spin_unlock(&trace_cmdline_lock); 1999 preempt_enable(); 2000 } 2001 2002 int trace_find_tgid(int pid) 2003 { 2004 if (unlikely(!tgid_map || !pid || pid > PID_MAX_DEFAULT)) 2005 return 0; 2006 2007 return tgid_map[pid]; 2008 } 2009 2010 static int trace_save_tgid(struct task_struct *tsk) 2011 { 2012 /* treat recording of idle task as a success */ 2013 if (!tsk->pid) 2014 return 1; 2015 2016 if (unlikely(!tgid_map || tsk->pid > PID_MAX_DEFAULT)) 2017 return 0; 2018 2019 tgid_map[tsk->pid] = tsk->tgid; 2020 return 1; 2021 } 2022 2023 static bool tracing_record_taskinfo_skip(int flags) 2024 { 2025 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID)))) 2026 return true; 2027 if (atomic_read(&trace_record_taskinfo_disabled) || !tracing_is_on()) 2028 return true; 2029 if (!__this_cpu_read(trace_taskinfo_save)) 2030 return true; 2031 return false; 2032 } 2033 2034 /** 2035 * tracing_record_taskinfo - record the task info of a task 2036 * 2037 * @task - task to record 2038 * @flags - TRACE_RECORD_CMDLINE for recording comm 2039 * - TRACE_RECORD_TGID for recording tgid 2040 */ 2041 void tracing_record_taskinfo(struct task_struct *task, int flags) 2042 { 2043 bool done; 2044 2045 if (tracing_record_taskinfo_skip(flags)) 2046 return; 2047 2048 /* 2049 * Record as much task information as possible. If some fail, continue 2050 * to try to record the others. 2051 */ 2052 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task); 2053 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task); 2054 2055 /* If recording any information failed, retry again soon. */ 2056 if (!done) 2057 return; 2058 2059 __this_cpu_write(trace_taskinfo_save, false); 2060 } 2061 2062 /** 2063 * tracing_record_taskinfo_sched_switch - record task info for sched_switch 2064 * 2065 * @prev - previous task during sched_switch 2066 * @next - next task during sched_switch 2067 * @flags - TRACE_RECORD_CMDLINE for recording comm 2068 * TRACE_RECORD_TGID for recording tgid 2069 */ 2070 void tracing_record_taskinfo_sched_switch(struct task_struct *prev, 2071 struct task_struct *next, int flags) 2072 { 2073 bool done; 2074 2075 if (tracing_record_taskinfo_skip(flags)) 2076 return; 2077 2078 /* 2079 * Record as much task information as possible. If some fail, continue 2080 * to try to record the others. 2081 */ 2082 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev); 2083 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next); 2084 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev); 2085 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next); 2086 2087 /* If recording any information failed, retry again soon. */ 2088 if (!done) 2089 return; 2090 2091 __this_cpu_write(trace_taskinfo_save, false); 2092 } 2093 2094 /* Helpers to record a specific task information */ 2095 void tracing_record_cmdline(struct task_struct *task) 2096 { 2097 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE); 2098 } 2099 2100 void tracing_record_tgid(struct task_struct *task) 2101 { 2102 tracing_record_taskinfo(task, TRACE_RECORD_TGID); 2103 } 2104 2105 /* 2106 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq 2107 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function 2108 * simplifies those functions and keeps them in sync. 2109 */ 2110 enum print_line_t trace_handle_return(struct trace_seq *s) 2111 { 2112 return trace_seq_has_overflowed(s) ? 2113 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED; 2114 } 2115 EXPORT_SYMBOL_GPL(trace_handle_return); 2116 2117 void 2118 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags, 2119 int pc) 2120 { 2121 struct task_struct *tsk = current; 2122 2123 entry->preempt_count = pc & 0xff; 2124 entry->pid = (tsk) ? tsk->pid : 0; 2125 entry->flags = 2126 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT 2127 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) | 2128 #else 2129 TRACE_FLAG_IRQS_NOSUPPORT | 2130 #endif 2131 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) | 2132 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) | 2133 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) | 2134 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) | 2135 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0); 2136 } 2137 EXPORT_SYMBOL_GPL(tracing_generic_entry_update); 2138 2139 struct ring_buffer_event * 2140 trace_buffer_lock_reserve(struct ring_buffer *buffer, 2141 int type, 2142 unsigned long len, 2143 unsigned long flags, int pc) 2144 { 2145 return __trace_buffer_lock_reserve(buffer, type, len, flags, pc); 2146 } 2147 2148 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event); 2149 DEFINE_PER_CPU(int, trace_buffered_event_cnt); 2150 static int trace_buffered_event_ref; 2151 2152 /** 2153 * trace_buffered_event_enable - enable buffering events 2154 * 2155 * When events are being filtered, it is quicker to use a temporary 2156 * buffer to write the event data into if there's a likely chance 2157 * that it will not be committed. The discard of the ring buffer 2158 * is not as fast as committing, and is much slower than copying 2159 * a commit. 2160 * 2161 * When an event is to be filtered, allocate per cpu buffers to 2162 * write the event data into, and if the event is filtered and discarded 2163 * it is simply dropped, otherwise, the entire data is to be committed 2164 * in one shot. 2165 */ 2166 void trace_buffered_event_enable(void) 2167 { 2168 struct ring_buffer_event *event; 2169 struct page *page; 2170 int cpu; 2171 2172 WARN_ON_ONCE(!mutex_is_locked(&event_mutex)); 2173 2174 if (trace_buffered_event_ref++) 2175 return; 2176 2177 for_each_tracing_cpu(cpu) { 2178 page = alloc_pages_node(cpu_to_node(cpu), 2179 GFP_KERNEL | __GFP_NORETRY, 0); 2180 if (!page) 2181 goto failed; 2182 2183 event = page_address(page); 2184 memset(event, 0, sizeof(*event)); 2185 2186 per_cpu(trace_buffered_event, cpu) = event; 2187 2188 preempt_disable(); 2189 if (cpu == smp_processor_id() && 2190 this_cpu_read(trace_buffered_event) != 2191 per_cpu(trace_buffered_event, cpu)) 2192 WARN_ON_ONCE(1); 2193 preempt_enable(); 2194 } 2195 2196 return; 2197 failed: 2198 trace_buffered_event_disable(); 2199 } 2200 2201 static void enable_trace_buffered_event(void *data) 2202 { 2203 /* Probably not needed, but do it anyway */ 2204 smp_rmb(); 2205 this_cpu_dec(trace_buffered_event_cnt); 2206 } 2207 2208 static void disable_trace_buffered_event(void *data) 2209 { 2210 this_cpu_inc(trace_buffered_event_cnt); 2211 } 2212 2213 /** 2214 * trace_buffered_event_disable - disable buffering events 2215 * 2216 * When a filter is removed, it is faster to not use the buffered 2217 * events, and to commit directly into the ring buffer. Free up 2218 * the temp buffers when there are no more users. This requires 2219 * special synchronization with current events. 2220 */ 2221 void trace_buffered_event_disable(void) 2222 { 2223 int cpu; 2224 2225 WARN_ON_ONCE(!mutex_is_locked(&event_mutex)); 2226 2227 if (WARN_ON_ONCE(!trace_buffered_event_ref)) 2228 return; 2229 2230 if (--trace_buffered_event_ref) 2231 return; 2232 2233 preempt_disable(); 2234 /* For each CPU, set the buffer as used. */ 2235 smp_call_function_many(tracing_buffer_mask, 2236 disable_trace_buffered_event, NULL, 1); 2237 preempt_enable(); 2238 2239 /* Wait for all current users to finish */ 2240 synchronize_sched(); 2241 2242 for_each_tracing_cpu(cpu) { 2243 free_page((unsigned long)per_cpu(trace_buffered_event, cpu)); 2244 per_cpu(trace_buffered_event, cpu) = NULL; 2245 } 2246 /* 2247 * Make sure trace_buffered_event is NULL before clearing 2248 * trace_buffered_event_cnt. 2249 */ 2250 smp_wmb(); 2251 2252 preempt_disable(); 2253 /* Do the work on each cpu */ 2254 smp_call_function_many(tracing_buffer_mask, 2255 enable_trace_buffered_event, NULL, 1); 2256 preempt_enable(); 2257 } 2258 2259 static struct ring_buffer *temp_buffer; 2260 2261 struct ring_buffer_event * 2262 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb, 2263 struct trace_event_file *trace_file, 2264 int type, unsigned long len, 2265 unsigned long flags, int pc) 2266 { 2267 struct ring_buffer_event *entry; 2268 int val; 2269 2270 *current_rb = trace_file->tr->trace_buffer.buffer; 2271 2272 if ((trace_file->flags & 2273 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) && 2274 (entry = this_cpu_read(trace_buffered_event))) { 2275 /* Try to use the per cpu buffer first */ 2276 val = this_cpu_inc_return(trace_buffered_event_cnt); 2277 if (val == 1) { 2278 trace_event_setup(entry, type, flags, pc); 2279 entry->array[0] = len; 2280 return entry; 2281 } 2282 this_cpu_dec(trace_buffered_event_cnt); 2283 } 2284 2285 entry = __trace_buffer_lock_reserve(*current_rb, 2286 type, len, flags, pc); 2287 /* 2288 * If tracing is off, but we have triggers enabled 2289 * we still need to look at the event data. Use the temp_buffer 2290 * to store the trace event for the tigger to use. It's recusive 2291 * safe and will not be recorded anywhere. 2292 */ 2293 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) { 2294 *current_rb = temp_buffer; 2295 entry = __trace_buffer_lock_reserve(*current_rb, 2296 type, len, flags, pc); 2297 } 2298 return entry; 2299 } 2300 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve); 2301 2302 static DEFINE_SPINLOCK(tracepoint_iter_lock); 2303 static DEFINE_MUTEX(tracepoint_printk_mutex); 2304 2305 static void output_printk(struct trace_event_buffer *fbuffer) 2306 { 2307 struct trace_event_call *event_call; 2308 struct trace_event *event; 2309 unsigned long flags; 2310 struct trace_iterator *iter = tracepoint_print_iter; 2311 2312 /* We should never get here if iter is NULL */ 2313 if (WARN_ON_ONCE(!iter)) 2314 return; 2315 2316 event_call = fbuffer->trace_file->event_call; 2317 if (!event_call || !event_call->event.funcs || 2318 !event_call->event.funcs->trace) 2319 return; 2320 2321 event = &fbuffer->trace_file->event_call->event; 2322 2323 spin_lock_irqsave(&tracepoint_iter_lock, flags); 2324 trace_seq_init(&iter->seq); 2325 iter->ent = fbuffer->entry; 2326 event_call->event.funcs->trace(iter, 0, event); 2327 trace_seq_putc(&iter->seq, 0); 2328 printk("%s", iter->seq.buffer); 2329 2330 spin_unlock_irqrestore(&tracepoint_iter_lock, flags); 2331 } 2332 2333 int tracepoint_printk_sysctl(struct ctl_table *table, int write, 2334 void __user *buffer, size_t *lenp, 2335 loff_t *ppos) 2336 { 2337 int save_tracepoint_printk; 2338 int ret; 2339 2340 mutex_lock(&tracepoint_printk_mutex); 2341 save_tracepoint_printk = tracepoint_printk; 2342 2343 ret = proc_dointvec(table, write, buffer, lenp, ppos); 2344 2345 /* 2346 * This will force exiting early, as tracepoint_printk 2347 * is always zero when tracepoint_printk_iter is not allocated 2348 */ 2349 if (!tracepoint_print_iter) 2350 tracepoint_printk = 0; 2351 2352 if (save_tracepoint_printk == tracepoint_printk) 2353 goto out; 2354 2355 if (tracepoint_printk) 2356 static_key_enable(&tracepoint_printk_key.key); 2357 else 2358 static_key_disable(&tracepoint_printk_key.key); 2359 2360 out: 2361 mutex_unlock(&tracepoint_printk_mutex); 2362 2363 return ret; 2364 } 2365 2366 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer) 2367 { 2368 if (static_key_false(&tracepoint_printk_key.key)) 2369 output_printk(fbuffer); 2370 2371 event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer, 2372 fbuffer->event, fbuffer->entry, 2373 fbuffer->flags, fbuffer->pc); 2374 } 2375 EXPORT_SYMBOL_GPL(trace_event_buffer_commit); 2376 2377 void trace_buffer_unlock_commit_regs(struct trace_array *tr, 2378 struct ring_buffer *buffer, 2379 struct ring_buffer_event *event, 2380 unsigned long flags, int pc, 2381 struct pt_regs *regs) 2382 { 2383 __buffer_unlock_commit(buffer, event); 2384 2385 /* 2386 * If regs is not set, then skip the following callers: 2387 * trace_buffer_unlock_commit_regs 2388 * event_trigger_unlock_commit 2389 * trace_event_buffer_commit 2390 * trace_event_raw_event_sched_switch 2391 * Note, we can still get here via blktrace, wakeup tracer 2392 * and mmiotrace, but that's ok if they lose a function or 2393 * two. They are that meaningful. 2394 */ 2395 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : 4, pc, regs); 2396 ftrace_trace_userstack(buffer, flags, pc); 2397 } 2398 2399 /* 2400 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack. 2401 */ 2402 void 2403 trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer, 2404 struct ring_buffer_event *event) 2405 { 2406 __buffer_unlock_commit(buffer, event); 2407 } 2408 2409 static void 2410 trace_process_export(struct trace_export *export, 2411 struct ring_buffer_event *event) 2412 { 2413 struct trace_entry *entry; 2414 unsigned int size = 0; 2415 2416 entry = ring_buffer_event_data(event); 2417 size = ring_buffer_event_length(event); 2418 export->write(entry, size); 2419 } 2420 2421 static DEFINE_MUTEX(ftrace_export_lock); 2422 2423 static struct trace_export __rcu *ftrace_exports_list __read_mostly; 2424 2425 static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled); 2426 2427 static inline void ftrace_exports_enable(void) 2428 { 2429 static_branch_enable(&ftrace_exports_enabled); 2430 } 2431 2432 static inline void ftrace_exports_disable(void) 2433 { 2434 static_branch_disable(&ftrace_exports_enabled); 2435 } 2436 2437 void ftrace_exports(struct ring_buffer_event *event) 2438 { 2439 struct trace_export *export; 2440 2441 preempt_disable_notrace(); 2442 2443 export = rcu_dereference_raw_notrace(ftrace_exports_list); 2444 while (export) { 2445 trace_process_export(export, event); 2446 export = rcu_dereference_raw_notrace(export->next); 2447 } 2448 2449 preempt_enable_notrace(); 2450 } 2451 2452 static inline void 2453 add_trace_export(struct trace_export **list, struct trace_export *export) 2454 { 2455 rcu_assign_pointer(export->next, *list); 2456 /* 2457 * We are entering export into the list but another 2458 * CPU might be walking that list. We need to make sure 2459 * the export->next pointer is valid before another CPU sees 2460 * the export pointer included into the list. 2461 */ 2462 rcu_assign_pointer(*list, export); 2463 } 2464 2465 static inline int 2466 rm_trace_export(struct trace_export **list, struct trace_export *export) 2467 { 2468 struct trace_export **p; 2469 2470 for (p = list; *p != NULL; p = &(*p)->next) 2471 if (*p == export) 2472 break; 2473 2474 if (*p != export) 2475 return -1; 2476 2477 rcu_assign_pointer(*p, (*p)->next); 2478 2479 return 0; 2480 } 2481 2482 static inline void 2483 add_ftrace_export(struct trace_export **list, struct trace_export *export) 2484 { 2485 if (*list == NULL) 2486 ftrace_exports_enable(); 2487 2488 add_trace_export(list, export); 2489 } 2490 2491 static inline int 2492 rm_ftrace_export(struct trace_export **list, struct trace_export *export) 2493 { 2494 int ret; 2495 2496 ret = rm_trace_export(list, export); 2497 if (*list == NULL) 2498 ftrace_exports_disable(); 2499 2500 return ret; 2501 } 2502 2503 int register_ftrace_export(struct trace_export *export) 2504 { 2505 if (WARN_ON_ONCE(!export->write)) 2506 return -1; 2507 2508 mutex_lock(&ftrace_export_lock); 2509 2510 add_ftrace_export(&ftrace_exports_list, export); 2511 2512 mutex_unlock(&ftrace_export_lock); 2513 2514 return 0; 2515 } 2516 EXPORT_SYMBOL_GPL(register_ftrace_export); 2517 2518 int unregister_ftrace_export(struct trace_export *export) 2519 { 2520 int ret; 2521 2522 mutex_lock(&ftrace_export_lock); 2523 2524 ret = rm_ftrace_export(&ftrace_exports_list, export); 2525 2526 mutex_unlock(&ftrace_export_lock); 2527 2528 return ret; 2529 } 2530 EXPORT_SYMBOL_GPL(unregister_ftrace_export); 2531 2532 void 2533 trace_function(struct trace_array *tr, 2534 unsigned long ip, unsigned long parent_ip, unsigned long flags, 2535 int pc) 2536 { 2537 struct trace_event_call *call = &event_function; 2538 struct ring_buffer *buffer = tr->trace_buffer.buffer; 2539 struct ring_buffer_event *event; 2540 struct ftrace_entry *entry; 2541 2542 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), 2543 flags, pc); 2544 if (!event) 2545 return; 2546 entry = ring_buffer_event_data(event); 2547 entry->ip = ip; 2548 entry->parent_ip = parent_ip; 2549 2550 if (!call_filter_check_discard(call, entry, buffer, event)) { 2551 if (static_branch_unlikely(&ftrace_exports_enabled)) 2552 ftrace_exports(event); 2553 __buffer_unlock_commit(buffer, event); 2554 } 2555 } 2556 2557 #ifdef CONFIG_STACKTRACE 2558 2559 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long)) 2560 struct ftrace_stack { 2561 unsigned long calls[FTRACE_STACK_MAX_ENTRIES]; 2562 }; 2563 2564 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack); 2565 static DEFINE_PER_CPU(int, ftrace_stack_reserve); 2566 2567 static void __ftrace_trace_stack(struct ring_buffer *buffer, 2568 unsigned long flags, 2569 int skip, int pc, struct pt_regs *regs) 2570 { 2571 struct trace_event_call *call = &event_kernel_stack; 2572 struct ring_buffer_event *event; 2573 struct stack_entry *entry; 2574 struct stack_trace trace; 2575 int use_stack; 2576 int size = FTRACE_STACK_ENTRIES; 2577 2578 trace.nr_entries = 0; 2579 trace.skip = skip; 2580 2581 /* 2582 * Add two, for this function and the call to save_stack_trace() 2583 * If regs is set, then these functions will not be in the way. 2584 */ 2585 if (!regs) 2586 trace.skip += 2; 2587 2588 /* 2589 * Since events can happen in NMIs there's no safe way to 2590 * use the per cpu ftrace_stacks. We reserve it and if an interrupt 2591 * or NMI comes in, it will just have to use the default 2592 * FTRACE_STACK_SIZE. 2593 */ 2594 preempt_disable_notrace(); 2595 2596 use_stack = __this_cpu_inc_return(ftrace_stack_reserve); 2597 /* 2598 * We don't need any atomic variables, just a barrier. 2599 * If an interrupt comes in, we don't care, because it would 2600 * have exited and put the counter back to what we want. 2601 * We just need a barrier to keep gcc from moving things 2602 * around. 2603 */ 2604 barrier(); 2605 if (use_stack == 1) { 2606 trace.entries = this_cpu_ptr(ftrace_stack.calls); 2607 trace.max_entries = FTRACE_STACK_MAX_ENTRIES; 2608 2609 if (regs) 2610 save_stack_trace_regs(regs, &trace); 2611 else 2612 save_stack_trace(&trace); 2613 2614 if (trace.nr_entries > size) 2615 size = trace.nr_entries; 2616 } else 2617 /* From now on, use_stack is a boolean */ 2618 use_stack = 0; 2619 2620 size *= sizeof(unsigned long); 2621 2622 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK, 2623 sizeof(*entry) + size, flags, pc); 2624 if (!event) 2625 goto out; 2626 entry = ring_buffer_event_data(event); 2627 2628 memset(&entry->caller, 0, size); 2629 2630 if (use_stack) 2631 memcpy(&entry->caller, trace.entries, 2632 trace.nr_entries * sizeof(unsigned long)); 2633 else { 2634 trace.max_entries = FTRACE_STACK_ENTRIES; 2635 trace.entries = entry->caller; 2636 if (regs) 2637 save_stack_trace_regs(regs, &trace); 2638 else 2639 save_stack_trace(&trace); 2640 } 2641 2642 entry->size = trace.nr_entries; 2643 2644 if (!call_filter_check_discard(call, entry, buffer, event)) 2645 __buffer_unlock_commit(buffer, event); 2646 2647 out: 2648 /* Again, don't let gcc optimize things here */ 2649 barrier(); 2650 __this_cpu_dec(ftrace_stack_reserve); 2651 preempt_enable_notrace(); 2652 2653 } 2654 2655 static inline void ftrace_trace_stack(struct trace_array *tr, 2656 struct ring_buffer *buffer, 2657 unsigned long flags, 2658 int skip, int pc, struct pt_regs *regs) 2659 { 2660 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE)) 2661 return; 2662 2663 __ftrace_trace_stack(buffer, flags, skip, pc, regs); 2664 } 2665 2666 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, 2667 int pc) 2668 { 2669 struct ring_buffer *buffer = tr->trace_buffer.buffer; 2670 2671 if (rcu_is_watching()) { 2672 __ftrace_trace_stack(buffer, flags, skip, pc, NULL); 2673 return; 2674 } 2675 2676 /* 2677 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(), 2678 * but if the above rcu_is_watching() failed, then the NMI 2679 * triggered someplace critical, and rcu_irq_enter() should 2680 * not be called from NMI. 2681 */ 2682 if (unlikely(in_nmi())) 2683 return; 2684 2685 /* 2686 * It is possible that a function is being traced in a 2687 * location that RCU is not watching. A call to 2688 * rcu_irq_enter() will make sure that it is, but there's 2689 * a few internal rcu functions that could be traced 2690 * where that wont work either. In those cases, we just 2691 * do nothing. 2692 */ 2693 if (unlikely(rcu_irq_enter_disabled())) 2694 return; 2695 2696 rcu_irq_enter_irqson(); 2697 __ftrace_trace_stack(buffer, flags, skip, pc, NULL); 2698 rcu_irq_exit_irqson(); 2699 } 2700 2701 /** 2702 * trace_dump_stack - record a stack back trace in the trace buffer 2703 * @skip: Number of functions to skip (helper handlers) 2704 */ 2705 void trace_dump_stack(int skip) 2706 { 2707 unsigned long flags; 2708 2709 if (tracing_disabled || tracing_selftest_running) 2710 return; 2711 2712 local_save_flags(flags); 2713 2714 /* 2715 * Skip 3 more, seems to get us at the caller of 2716 * this function. 2717 */ 2718 skip += 3; 2719 __ftrace_trace_stack(global_trace.trace_buffer.buffer, 2720 flags, skip, preempt_count(), NULL); 2721 } 2722 2723 static DEFINE_PER_CPU(int, user_stack_count); 2724 2725 void 2726 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc) 2727 { 2728 struct trace_event_call *call = &event_user_stack; 2729 struct ring_buffer_event *event; 2730 struct userstack_entry *entry; 2731 struct stack_trace trace; 2732 2733 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE)) 2734 return; 2735 2736 /* 2737 * NMIs can not handle page faults, even with fix ups. 2738 * The save user stack can (and often does) fault. 2739 */ 2740 if (unlikely(in_nmi())) 2741 return; 2742 2743 /* 2744 * prevent recursion, since the user stack tracing may 2745 * trigger other kernel events. 2746 */ 2747 preempt_disable(); 2748 if (__this_cpu_read(user_stack_count)) 2749 goto out; 2750 2751 __this_cpu_inc(user_stack_count); 2752 2753 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, 2754 sizeof(*entry), flags, pc); 2755 if (!event) 2756 goto out_drop_count; 2757 entry = ring_buffer_event_data(event); 2758 2759 entry->tgid = current->tgid; 2760 memset(&entry->caller, 0, sizeof(entry->caller)); 2761 2762 trace.nr_entries = 0; 2763 trace.max_entries = FTRACE_STACK_ENTRIES; 2764 trace.skip = 0; 2765 trace.entries = entry->caller; 2766 2767 save_stack_trace_user(&trace); 2768 if (!call_filter_check_discard(call, entry, buffer, event)) 2769 __buffer_unlock_commit(buffer, event); 2770 2771 out_drop_count: 2772 __this_cpu_dec(user_stack_count); 2773 out: 2774 preempt_enable(); 2775 } 2776 2777 #ifdef UNUSED 2778 static void __trace_userstack(struct trace_array *tr, unsigned long flags) 2779 { 2780 ftrace_trace_userstack(tr, flags, preempt_count()); 2781 } 2782 #endif /* UNUSED */ 2783 2784 #endif /* CONFIG_STACKTRACE */ 2785 2786 /* created for use with alloc_percpu */ 2787 struct trace_buffer_struct { 2788 int nesting; 2789 char buffer[4][TRACE_BUF_SIZE]; 2790 }; 2791 2792 static struct trace_buffer_struct *trace_percpu_buffer; 2793 2794 /* 2795 * Thise allows for lockless recording. If we're nested too deeply, then 2796 * this returns NULL. 2797 */ 2798 static char *get_trace_buf(void) 2799 { 2800 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer); 2801 2802 if (!buffer || buffer->nesting >= 4) 2803 return NULL; 2804 2805 buffer->nesting++; 2806 2807 /* Interrupts must see nesting incremented before we use the buffer */ 2808 barrier(); 2809 return &buffer->buffer[buffer->nesting][0]; 2810 } 2811 2812 static void put_trace_buf(void) 2813 { 2814 /* Don't let the decrement of nesting leak before this */ 2815 barrier(); 2816 this_cpu_dec(trace_percpu_buffer->nesting); 2817 } 2818 2819 static int alloc_percpu_trace_buffer(void) 2820 { 2821 struct trace_buffer_struct *buffers; 2822 2823 buffers = alloc_percpu(struct trace_buffer_struct); 2824 if (WARN(!buffers, "Could not allocate percpu trace_printk buffer")) 2825 return -ENOMEM; 2826 2827 trace_percpu_buffer = buffers; 2828 return 0; 2829 } 2830 2831 static int buffers_allocated; 2832 2833 void trace_printk_init_buffers(void) 2834 { 2835 if (buffers_allocated) 2836 return; 2837 2838 if (alloc_percpu_trace_buffer()) 2839 return; 2840 2841 /* trace_printk() is for debug use only. Don't use it in production. */ 2842 2843 pr_warn("\n"); 2844 pr_warn("**********************************************************\n"); 2845 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 2846 pr_warn("** **\n"); 2847 pr_warn("** trace_printk() being used. Allocating extra memory. **\n"); 2848 pr_warn("** **\n"); 2849 pr_warn("** This means that this is a DEBUG kernel and it is **\n"); 2850 pr_warn("** unsafe for production use. **\n"); 2851 pr_warn("** **\n"); 2852 pr_warn("** If you see this message and you are not debugging **\n"); 2853 pr_warn("** the kernel, report this immediately to your vendor! **\n"); 2854 pr_warn("** **\n"); 2855 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 2856 pr_warn("**********************************************************\n"); 2857 2858 /* Expand the buffers to set size */ 2859 tracing_update_buffers(); 2860 2861 buffers_allocated = 1; 2862 2863 /* 2864 * trace_printk_init_buffers() can be called by modules. 2865 * If that happens, then we need to start cmdline recording 2866 * directly here. If the global_trace.buffer is already 2867 * allocated here, then this was called by module code. 2868 */ 2869 if (global_trace.trace_buffer.buffer) 2870 tracing_start_cmdline_record(); 2871 } 2872 2873 void trace_printk_start_comm(void) 2874 { 2875 /* Start tracing comms if trace printk is set */ 2876 if (!buffers_allocated) 2877 return; 2878 tracing_start_cmdline_record(); 2879 } 2880 2881 static void trace_printk_start_stop_comm(int enabled) 2882 { 2883 if (!buffers_allocated) 2884 return; 2885 2886 if (enabled) 2887 tracing_start_cmdline_record(); 2888 else 2889 tracing_stop_cmdline_record(); 2890 } 2891 2892 /** 2893 * trace_vbprintk - write binary msg to tracing buffer 2894 * 2895 */ 2896 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) 2897 { 2898 struct trace_event_call *call = &event_bprint; 2899 struct ring_buffer_event *event; 2900 struct ring_buffer *buffer; 2901 struct trace_array *tr = &global_trace; 2902 struct bprint_entry *entry; 2903 unsigned long flags; 2904 char *tbuffer; 2905 int len = 0, size, pc; 2906 2907 if (unlikely(tracing_selftest_running || tracing_disabled)) 2908 return 0; 2909 2910 /* Don't pollute graph traces with trace_vprintk internals */ 2911 pause_graph_tracing(); 2912 2913 pc = preempt_count(); 2914 preempt_disable_notrace(); 2915 2916 tbuffer = get_trace_buf(); 2917 if (!tbuffer) { 2918 len = 0; 2919 goto out_nobuffer; 2920 } 2921 2922 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args); 2923 2924 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0) 2925 goto out; 2926 2927 local_save_flags(flags); 2928 size = sizeof(*entry) + sizeof(u32) * len; 2929 buffer = tr->trace_buffer.buffer; 2930 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, 2931 flags, pc); 2932 if (!event) 2933 goto out; 2934 entry = ring_buffer_event_data(event); 2935 entry->ip = ip; 2936 entry->fmt = fmt; 2937 2938 memcpy(entry->buf, tbuffer, sizeof(u32) * len); 2939 if (!call_filter_check_discard(call, entry, buffer, event)) { 2940 __buffer_unlock_commit(buffer, event); 2941 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL); 2942 } 2943 2944 out: 2945 put_trace_buf(); 2946 2947 out_nobuffer: 2948 preempt_enable_notrace(); 2949 unpause_graph_tracing(); 2950 2951 return len; 2952 } 2953 EXPORT_SYMBOL_GPL(trace_vbprintk); 2954 2955 static int 2956 __trace_array_vprintk(struct ring_buffer *buffer, 2957 unsigned long ip, const char *fmt, va_list args) 2958 { 2959 struct trace_event_call *call = &event_print; 2960 struct ring_buffer_event *event; 2961 int len = 0, size, pc; 2962 struct print_entry *entry; 2963 unsigned long flags; 2964 char *tbuffer; 2965 2966 if (tracing_disabled || tracing_selftest_running) 2967 return 0; 2968 2969 /* Don't pollute graph traces with trace_vprintk internals */ 2970 pause_graph_tracing(); 2971 2972 pc = preempt_count(); 2973 preempt_disable_notrace(); 2974 2975 2976 tbuffer = get_trace_buf(); 2977 if (!tbuffer) { 2978 len = 0; 2979 goto out_nobuffer; 2980 } 2981 2982 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args); 2983 2984 local_save_flags(flags); 2985 size = sizeof(*entry) + len + 1; 2986 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 2987 flags, pc); 2988 if (!event) 2989 goto out; 2990 entry = ring_buffer_event_data(event); 2991 entry->ip = ip; 2992 2993 memcpy(&entry->buf, tbuffer, len + 1); 2994 if (!call_filter_check_discard(call, entry, buffer, event)) { 2995 __buffer_unlock_commit(buffer, event); 2996 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL); 2997 } 2998 2999 out: 3000 put_trace_buf(); 3001 3002 out_nobuffer: 3003 preempt_enable_notrace(); 3004 unpause_graph_tracing(); 3005 3006 return len; 3007 } 3008 3009 int trace_array_vprintk(struct trace_array *tr, 3010 unsigned long ip, const char *fmt, va_list args) 3011 { 3012 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args); 3013 } 3014 3015 int trace_array_printk(struct trace_array *tr, 3016 unsigned long ip, const char *fmt, ...) 3017 { 3018 int ret; 3019 va_list ap; 3020 3021 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 3022 return 0; 3023 3024 va_start(ap, fmt); 3025 ret = trace_array_vprintk(tr, ip, fmt, ap); 3026 va_end(ap); 3027 return ret; 3028 } 3029 3030 int trace_array_printk_buf(struct ring_buffer *buffer, 3031 unsigned long ip, const char *fmt, ...) 3032 { 3033 int ret; 3034 va_list ap; 3035 3036 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 3037 return 0; 3038 3039 va_start(ap, fmt); 3040 ret = __trace_array_vprintk(buffer, ip, fmt, ap); 3041 va_end(ap); 3042 return ret; 3043 } 3044 3045 int trace_vprintk(unsigned long ip, const char *fmt, va_list args) 3046 { 3047 return trace_array_vprintk(&global_trace, ip, fmt, args); 3048 } 3049 EXPORT_SYMBOL_GPL(trace_vprintk); 3050 3051 static void trace_iterator_increment(struct trace_iterator *iter) 3052 { 3053 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu); 3054 3055 iter->idx++; 3056 if (buf_iter) 3057 ring_buffer_read(buf_iter, NULL); 3058 } 3059 3060 static struct trace_entry * 3061 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts, 3062 unsigned long *lost_events) 3063 { 3064 struct ring_buffer_event *event; 3065 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu); 3066 3067 if (buf_iter) 3068 event = ring_buffer_iter_peek(buf_iter, ts); 3069 else 3070 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts, 3071 lost_events); 3072 3073 if (event) { 3074 iter->ent_size = ring_buffer_event_length(event); 3075 return ring_buffer_event_data(event); 3076 } 3077 iter->ent_size = 0; 3078 return NULL; 3079 } 3080 3081 static struct trace_entry * 3082 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, 3083 unsigned long *missing_events, u64 *ent_ts) 3084 { 3085 struct ring_buffer *buffer = iter->trace_buffer->buffer; 3086 struct trace_entry *ent, *next = NULL; 3087 unsigned long lost_events = 0, next_lost = 0; 3088 int cpu_file = iter->cpu_file; 3089 u64 next_ts = 0, ts; 3090 int next_cpu = -1; 3091 int next_size = 0; 3092 int cpu; 3093 3094 /* 3095 * If we are in a per_cpu trace file, don't bother by iterating over 3096 * all cpu and peek directly. 3097 */ 3098 if (cpu_file > RING_BUFFER_ALL_CPUS) { 3099 if (ring_buffer_empty_cpu(buffer, cpu_file)) 3100 return NULL; 3101 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events); 3102 if (ent_cpu) 3103 *ent_cpu = cpu_file; 3104 3105 return ent; 3106 } 3107 3108 for_each_tracing_cpu(cpu) { 3109 3110 if (ring_buffer_empty_cpu(buffer, cpu)) 3111 continue; 3112 3113 ent = peek_next_entry(iter, cpu, &ts, &lost_events); 3114 3115 /* 3116 * Pick the entry with the smallest timestamp: 3117 */ 3118 if (ent && (!next || ts < next_ts)) { 3119 next = ent; 3120 next_cpu = cpu; 3121 next_ts = ts; 3122 next_lost = lost_events; 3123 next_size = iter->ent_size; 3124 } 3125 } 3126 3127 iter->ent_size = next_size; 3128 3129 if (ent_cpu) 3130 *ent_cpu = next_cpu; 3131 3132 if (ent_ts) 3133 *ent_ts = next_ts; 3134 3135 if (missing_events) 3136 *missing_events = next_lost; 3137 3138 return next; 3139 } 3140 3141 /* Find the next real entry, without updating the iterator itself */ 3142 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter, 3143 int *ent_cpu, u64 *ent_ts) 3144 { 3145 return __find_next_entry(iter, ent_cpu, NULL, ent_ts); 3146 } 3147 3148 /* Find the next real entry, and increment the iterator to the next entry */ 3149 void *trace_find_next_entry_inc(struct trace_iterator *iter) 3150 { 3151 iter->ent = __find_next_entry(iter, &iter->cpu, 3152 &iter->lost_events, &iter->ts); 3153 3154 if (iter->ent) 3155 trace_iterator_increment(iter); 3156 3157 return iter->ent ? iter : NULL; 3158 } 3159 3160 static void trace_consume(struct trace_iterator *iter) 3161 { 3162 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts, 3163 &iter->lost_events); 3164 } 3165 3166 static void *s_next(struct seq_file *m, void *v, loff_t *pos) 3167 { 3168 struct trace_iterator *iter = m->private; 3169 int i = (int)*pos; 3170 void *ent; 3171 3172 WARN_ON_ONCE(iter->leftover); 3173 3174 (*pos)++; 3175 3176 /* can't go backwards */ 3177 if (iter->idx > i) 3178 return NULL; 3179 3180 if (iter->idx < 0) 3181 ent = trace_find_next_entry_inc(iter); 3182 else 3183 ent = iter; 3184 3185 while (ent && iter->idx < i) 3186 ent = trace_find_next_entry_inc(iter); 3187 3188 iter->pos = *pos; 3189 3190 return ent; 3191 } 3192 3193 void tracing_iter_reset(struct trace_iterator *iter, int cpu) 3194 { 3195 struct ring_buffer_event *event; 3196 struct ring_buffer_iter *buf_iter; 3197 unsigned long entries = 0; 3198 u64 ts; 3199 3200 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0; 3201 3202 buf_iter = trace_buffer_iter(iter, cpu); 3203 if (!buf_iter) 3204 return; 3205 3206 ring_buffer_iter_reset(buf_iter); 3207 3208 /* 3209 * We could have the case with the max latency tracers 3210 * that a reset never took place on a cpu. This is evident 3211 * by the timestamp being before the start of the buffer. 3212 */ 3213 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) { 3214 if (ts >= iter->trace_buffer->time_start) 3215 break; 3216 entries++; 3217 ring_buffer_read(buf_iter, NULL); 3218 } 3219 3220 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries; 3221 } 3222 3223 /* 3224 * The current tracer is copied to avoid a global locking 3225 * all around. 3226 */ 3227 static void *s_start(struct seq_file *m, loff_t *pos) 3228 { 3229 struct trace_iterator *iter = m->private; 3230 struct trace_array *tr = iter->tr; 3231 int cpu_file = iter->cpu_file; 3232 void *p = NULL; 3233 loff_t l = 0; 3234 int cpu; 3235 3236 /* 3237 * copy the tracer to avoid using a global lock all around. 3238 * iter->trace is a copy of current_trace, the pointer to the 3239 * name may be used instead of a strcmp(), as iter->trace->name 3240 * will point to the same string as current_trace->name. 3241 */ 3242 mutex_lock(&trace_types_lock); 3243 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name)) 3244 *iter->trace = *tr->current_trace; 3245 mutex_unlock(&trace_types_lock); 3246 3247 #ifdef CONFIG_TRACER_MAX_TRACE 3248 if (iter->snapshot && iter->trace->use_max_tr) 3249 return ERR_PTR(-EBUSY); 3250 #endif 3251 3252 if (!iter->snapshot) 3253 atomic_inc(&trace_record_taskinfo_disabled); 3254 3255 if (*pos != iter->pos) { 3256 iter->ent = NULL; 3257 iter->cpu = 0; 3258 iter->idx = -1; 3259 3260 if (cpu_file == RING_BUFFER_ALL_CPUS) { 3261 for_each_tracing_cpu(cpu) 3262 tracing_iter_reset(iter, cpu); 3263 } else 3264 tracing_iter_reset(iter, cpu_file); 3265 3266 iter->leftover = 0; 3267 for (p = iter; p && l < *pos; p = s_next(m, p, &l)) 3268 ; 3269 3270 } else { 3271 /* 3272 * If we overflowed the seq_file before, then we want 3273 * to just reuse the trace_seq buffer again. 3274 */ 3275 if (iter->leftover) 3276 p = iter; 3277 else { 3278 l = *pos - 1; 3279 p = s_next(m, p, &l); 3280 } 3281 } 3282 3283 trace_event_read_lock(); 3284 trace_access_lock(cpu_file); 3285 return p; 3286 } 3287 3288 static void s_stop(struct seq_file *m, void *p) 3289 { 3290 struct trace_iterator *iter = m->private; 3291 3292 #ifdef CONFIG_TRACER_MAX_TRACE 3293 if (iter->snapshot && iter->trace->use_max_tr) 3294 return; 3295 #endif 3296 3297 if (!iter->snapshot) 3298 atomic_dec(&trace_record_taskinfo_disabled); 3299 3300 trace_access_unlock(iter->cpu_file); 3301 trace_event_read_unlock(); 3302 } 3303 3304 static void 3305 get_total_entries(struct trace_buffer *buf, 3306 unsigned long *total, unsigned long *entries) 3307 { 3308 unsigned long count; 3309 int cpu; 3310 3311 *total = 0; 3312 *entries = 0; 3313 3314 for_each_tracing_cpu(cpu) { 3315 count = ring_buffer_entries_cpu(buf->buffer, cpu); 3316 /* 3317 * If this buffer has skipped entries, then we hold all 3318 * entries for the trace and we need to ignore the 3319 * ones before the time stamp. 3320 */ 3321 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) { 3322 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries; 3323 /* total is the same as the entries */ 3324 *total += count; 3325 } else 3326 *total += count + 3327 ring_buffer_overrun_cpu(buf->buffer, cpu); 3328 *entries += count; 3329 } 3330 } 3331 3332 static void print_lat_help_header(struct seq_file *m) 3333 { 3334 seq_puts(m, "# _------=> CPU# \n" 3335 "# / _-----=> irqs-off \n" 3336 "# | / _----=> need-resched \n" 3337 "# || / _---=> hardirq/softirq \n" 3338 "# ||| / _--=> preempt-depth \n" 3339 "# |||| / delay \n" 3340 "# cmd pid ||||| time | caller \n" 3341 "# \\ / ||||| \\ | / \n"); 3342 } 3343 3344 static void print_event_info(struct trace_buffer *buf, struct seq_file *m) 3345 { 3346 unsigned long total; 3347 unsigned long entries; 3348 3349 get_total_entries(buf, &total, &entries); 3350 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n", 3351 entries, total, num_online_cpus()); 3352 seq_puts(m, "#\n"); 3353 } 3354 3355 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m, 3356 unsigned int flags) 3357 { 3358 bool tgid = flags & TRACE_ITER_RECORD_TGID; 3359 3360 print_event_info(buf, m); 3361 3362 seq_printf(m, "# TASK-PID CPU# %s TIMESTAMP FUNCTION\n", tgid ? "TGID " : ""); 3363 seq_printf(m, "# | | | %s | |\n", tgid ? " | " : ""); 3364 } 3365 3366 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m, 3367 unsigned int flags) 3368 { 3369 bool tgid = flags & TRACE_ITER_RECORD_TGID; 3370 const char tgid_space[] = " "; 3371 const char space[] = " "; 3372 3373 seq_printf(m, "# %s _-----=> irqs-off\n", 3374 tgid ? tgid_space : space); 3375 seq_printf(m, "# %s / _----=> need-resched\n", 3376 tgid ? tgid_space : space); 3377 seq_printf(m, "# %s| / _---=> hardirq/softirq\n", 3378 tgid ? tgid_space : space); 3379 seq_printf(m, "# %s|| / _--=> preempt-depth\n", 3380 tgid ? tgid_space : space); 3381 seq_printf(m, "# %s||| / delay\n", 3382 tgid ? tgid_space : space); 3383 seq_printf(m, "# TASK-PID CPU#%s|||| TIMESTAMP FUNCTION\n", 3384 tgid ? " TGID " : space); 3385 seq_printf(m, "# | | | %s|||| | |\n", 3386 tgid ? " | " : space); 3387 } 3388 3389 void 3390 print_trace_header(struct seq_file *m, struct trace_iterator *iter) 3391 { 3392 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK); 3393 struct trace_buffer *buf = iter->trace_buffer; 3394 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu); 3395 struct tracer *type = iter->trace; 3396 unsigned long entries; 3397 unsigned long total; 3398 const char *name = "preemption"; 3399 3400 name = type->name; 3401 3402 get_total_entries(buf, &total, &entries); 3403 3404 seq_printf(m, "# %s latency trace v1.1.5 on %s\n", 3405 name, UTS_RELEASE); 3406 seq_puts(m, "# -----------------------------------" 3407 "---------------------------------\n"); 3408 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |" 3409 " (M:%s VP:%d, KP:%d, SP:%d HP:%d", 3410 nsecs_to_usecs(data->saved_latency), 3411 entries, 3412 total, 3413 buf->cpu, 3414 #if defined(CONFIG_PREEMPT_NONE) 3415 "server", 3416 #elif defined(CONFIG_PREEMPT_VOLUNTARY) 3417 "desktop", 3418 #elif defined(CONFIG_PREEMPT) 3419 "preempt", 3420 #else 3421 "unknown", 3422 #endif 3423 /* These are reserved for later use */ 3424 0, 0, 0, 0); 3425 #ifdef CONFIG_SMP 3426 seq_printf(m, " #P:%d)\n", num_online_cpus()); 3427 #else 3428 seq_puts(m, ")\n"); 3429 #endif 3430 seq_puts(m, "# -----------------\n"); 3431 seq_printf(m, "# | task: %.16s-%d " 3432 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n", 3433 data->comm, data->pid, 3434 from_kuid_munged(seq_user_ns(m), data->uid), data->nice, 3435 data->policy, data->rt_priority); 3436 seq_puts(m, "# -----------------\n"); 3437 3438 if (data->critical_start) { 3439 seq_puts(m, "# => started at: "); 3440 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags); 3441 trace_print_seq(m, &iter->seq); 3442 seq_puts(m, "\n# => ended at: "); 3443 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); 3444 trace_print_seq(m, &iter->seq); 3445 seq_puts(m, "\n#\n"); 3446 } 3447 3448 seq_puts(m, "#\n"); 3449 } 3450 3451 static void test_cpu_buff_start(struct trace_iterator *iter) 3452 { 3453 struct trace_seq *s = &iter->seq; 3454 struct trace_array *tr = iter->tr; 3455 3456 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE)) 3457 return; 3458 3459 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE)) 3460 return; 3461 3462 if (cpumask_available(iter->started) && 3463 cpumask_test_cpu(iter->cpu, iter->started)) 3464 return; 3465 3466 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries) 3467 return; 3468 3469 if (cpumask_available(iter->started)) 3470 cpumask_set_cpu(iter->cpu, iter->started); 3471 3472 /* Don't print started cpu buffer for the first entry of the trace */ 3473 if (iter->idx > 1) 3474 trace_seq_printf(s, "##### CPU %u buffer started ####\n", 3475 iter->cpu); 3476 } 3477 3478 static enum print_line_t print_trace_fmt(struct trace_iterator *iter) 3479 { 3480 struct trace_array *tr = iter->tr; 3481 struct trace_seq *s = &iter->seq; 3482 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK); 3483 struct trace_entry *entry; 3484 struct trace_event *event; 3485 3486 entry = iter->ent; 3487 3488 test_cpu_buff_start(iter); 3489 3490 event = ftrace_find_event(entry->type); 3491 3492 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 3493 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 3494 trace_print_lat_context(iter); 3495 else 3496 trace_print_context(iter); 3497 } 3498 3499 if (trace_seq_has_overflowed(s)) 3500 return TRACE_TYPE_PARTIAL_LINE; 3501 3502 if (event) 3503 return event->funcs->trace(iter, sym_flags, event); 3504 3505 trace_seq_printf(s, "Unknown type %d\n", entry->type); 3506 3507 return trace_handle_return(s); 3508 } 3509 3510 static enum print_line_t print_raw_fmt(struct trace_iterator *iter) 3511 { 3512 struct trace_array *tr = iter->tr; 3513 struct trace_seq *s = &iter->seq; 3514 struct trace_entry *entry; 3515 struct trace_event *event; 3516 3517 entry = iter->ent; 3518 3519 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) 3520 trace_seq_printf(s, "%d %d %llu ", 3521 entry->pid, iter->cpu, iter->ts); 3522 3523 if (trace_seq_has_overflowed(s)) 3524 return TRACE_TYPE_PARTIAL_LINE; 3525 3526 event = ftrace_find_event(entry->type); 3527 if (event) 3528 return event->funcs->raw(iter, 0, event); 3529 3530 trace_seq_printf(s, "%d ?\n", entry->type); 3531 3532 return trace_handle_return(s); 3533 } 3534 3535 static enum print_line_t print_hex_fmt(struct trace_iterator *iter) 3536 { 3537 struct trace_array *tr = iter->tr; 3538 struct trace_seq *s = &iter->seq; 3539 unsigned char newline = '\n'; 3540 struct trace_entry *entry; 3541 struct trace_event *event; 3542 3543 entry = iter->ent; 3544 3545 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 3546 SEQ_PUT_HEX_FIELD(s, entry->pid); 3547 SEQ_PUT_HEX_FIELD(s, iter->cpu); 3548 SEQ_PUT_HEX_FIELD(s, iter->ts); 3549 if (trace_seq_has_overflowed(s)) 3550 return TRACE_TYPE_PARTIAL_LINE; 3551 } 3552 3553 event = ftrace_find_event(entry->type); 3554 if (event) { 3555 enum print_line_t ret = event->funcs->hex(iter, 0, event); 3556 if (ret != TRACE_TYPE_HANDLED) 3557 return ret; 3558 } 3559 3560 SEQ_PUT_FIELD(s, newline); 3561 3562 return trace_handle_return(s); 3563 } 3564 3565 static enum print_line_t print_bin_fmt(struct trace_iterator *iter) 3566 { 3567 struct trace_array *tr = iter->tr; 3568 struct trace_seq *s = &iter->seq; 3569 struct trace_entry *entry; 3570 struct trace_event *event; 3571 3572 entry = iter->ent; 3573 3574 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 3575 SEQ_PUT_FIELD(s, entry->pid); 3576 SEQ_PUT_FIELD(s, iter->cpu); 3577 SEQ_PUT_FIELD(s, iter->ts); 3578 if (trace_seq_has_overflowed(s)) 3579 return TRACE_TYPE_PARTIAL_LINE; 3580 } 3581 3582 event = ftrace_find_event(entry->type); 3583 return event ? event->funcs->binary(iter, 0, event) : 3584 TRACE_TYPE_HANDLED; 3585 } 3586 3587 int trace_empty(struct trace_iterator *iter) 3588 { 3589 struct ring_buffer_iter *buf_iter; 3590 int cpu; 3591 3592 /* If we are looking at one CPU buffer, only check that one */ 3593 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 3594 cpu = iter->cpu_file; 3595 buf_iter = trace_buffer_iter(iter, cpu); 3596 if (buf_iter) { 3597 if (!ring_buffer_iter_empty(buf_iter)) 3598 return 0; 3599 } else { 3600 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu)) 3601 return 0; 3602 } 3603 return 1; 3604 } 3605 3606 for_each_tracing_cpu(cpu) { 3607 buf_iter = trace_buffer_iter(iter, cpu); 3608 if (buf_iter) { 3609 if (!ring_buffer_iter_empty(buf_iter)) 3610 return 0; 3611 } else { 3612 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu)) 3613 return 0; 3614 } 3615 } 3616 3617 return 1; 3618 } 3619 3620 /* Called with trace_event_read_lock() held. */ 3621 enum print_line_t print_trace_line(struct trace_iterator *iter) 3622 { 3623 struct trace_array *tr = iter->tr; 3624 unsigned long trace_flags = tr->trace_flags; 3625 enum print_line_t ret; 3626 3627 if (iter->lost_events) { 3628 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", 3629 iter->cpu, iter->lost_events); 3630 if (trace_seq_has_overflowed(&iter->seq)) 3631 return TRACE_TYPE_PARTIAL_LINE; 3632 } 3633 3634 if (iter->trace && iter->trace->print_line) { 3635 ret = iter->trace->print_line(iter); 3636 if (ret != TRACE_TYPE_UNHANDLED) 3637 return ret; 3638 } 3639 3640 if (iter->ent->type == TRACE_BPUTS && 3641 trace_flags & TRACE_ITER_PRINTK && 3642 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 3643 return trace_print_bputs_msg_only(iter); 3644 3645 if (iter->ent->type == TRACE_BPRINT && 3646 trace_flags & TRACE_ITER_PRINTK && 3647 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 3648 return trace_print_bprintk_msg_only(iter); 3649 3650 if (iter->ent->type == TRACE_PRINT && 3651 trace_flags & TRACE_ITER_PRINTK && 3652 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 3653 return trace_print_printk_msg_only(iter); 3654 3655 if (trace_flags & TRACE_ITER_BIN) 3656 return print_bin_fmt(iter); 3657 3658 if (trace_flags & TRACE_ITER_HEX) 3659 return print_hex_fmt(iter); 3660 3661 if (trace_flags & TRACE_ITER_RAW) 3662 return print_raw_fmt(iter); 3663 3664 return print_trace_fmt(iter); 3665 } 3666 3667 void trace_latency_header(struct seq_file *m) 3668 { 3669 struct trace_iterator *iter = m->private; 3670 struct trace_array *tr = iter->tr; 3671 3672 /* print nothing if the buffers are empty */ 3673 if (trace_empty(iter)) 3674 return; 3675 3676 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 3677 print_trace_header(m, iter); 3678 3679 if (!(tr->trace_flags & TRACE_ITER_VERBOSE)) 3680 print_lat_help_header(m); 3681 } 3682 3683 void trace_default_header(struct seq_file *m) 3684 { 3685 struct trace_iterator *iter = m->private; 3686 struct trace_array *tr = iter->tr; 3687 unsigned long trace_flags = tr->trace_flags; 3688 3689 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO)) 3690 return; 3691 3692 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { 3693 /* print nothing if the buffers are empty */ 3694 if (trace_empty(iter)) 3695 return; 3696 print_trace_header(m, iter); 3697 if (!(trace_flags & TRACE_ITER_VERBOSE)) 3698 print_lat_help_header(m); 3699 } else { 3700 if (!(trace_flags & TRACE_ITER_VERBOSE)) { 3701 if (trace_flags & TRACE_ITER_IRQ_INFO) 3702 print_func_help_header_irq(iter->trace_buffer, 3703 m, trace_flags); 3704 else 3705 print_func_help_header(iter->trace_buffer, m, 3706 trace_flags); 3707 } 3708 } 3709 } 3710 3711 static void test_ftrace_alive(struct seq_file *m) 3712 { 3713 if (!ftrace_is_dead()) 3714 return; 3715 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n" 3716 "# MAY BE MISSING FUNCTION EVENTS\n"); 3717 } 3718 3719 #ifdef CONFIG_TRACER_MAX_TRACE 3720 static void show_snapshot_main_help(struct seq_file *m) 3721 { 3722 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n" 3723 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" 3724 "# Takes a snapshot of the main buffer.\n" 3725 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n" 3726 "# (Doesn't have to be '2' works with any number that\n" 3727 "# is not a '0' or '1')\n"); 3728 } 3729 3730 static void show_snapshot_percpu_help(struct seq_file *m) 3731 { 3732 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n"); 3733 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP 3734 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" 3735 "# Takes a snapshot of the main buffer for this cpu.\n"); 3736 #else 3737 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n" 3738 "# Must use main snapshot file to allocate.\n"); 3739 #endif 3740 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n" 3741 "# (Doesn't have to be '2' works with any number that\n" 3742 "# is not a '0' or '1')\n"); 3743 } 3744 3745 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) 3746 { 3747 if (iter->tr->allocated_snapshot) 3748 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n"); 3749 else 3750 seq_puts(m, "#\n# * Snapshot is freed *\n#\n"); 3751 3752 seq_puts(m, "# Snapshot commands:\n"); 3753 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 3754 show_snapshot_main_help(m); 3755 else 3756 show_snapshot_percpu_help(m); 3757 } 3758 #else 3759 /* Should never be called */ 3760 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { } 3761 #endif 3762 3763 static int s_show(struct seq_file *m, void *v) 3764 { 3765 struct trace_iterator *iter = v; 3766 int ret; 3767 3768 if (iter->ent == NULL) { 3769 if (iter->tr) { 3770 seq_printf(m, "# tracer: %s\n", iter->trace->name); 3771 seq_puts(m, "#\n"); 3772 test_ftrace_alive(m); 3773 } 3774 if (iter->snapshot && trace_empty(iter)) 3775 print_snapshot_help(m, iter); 3776 else if (iter->trace && iter->trace->print_header) 3777 iter->trace->print_header(m); 3778 else 3779 trace_default_header(m); 3780 3781 } else if (iter->leftover) { 3782 /* 3783 * If we filled the seq_file buffer earlier, we 3784 * want to just show it now. 3785 */ 3786 ret = trace_print_seq(m, &iter->seq); 3787 3788 /* ret should this time be zero, but you never know */ 3789 iter->leftover = ret; 3790 3791 } else { 3792 print_trace_line(iter); 3793 ret = trace_print_seq(m, &iter->seq); 3794 /* 3795 * If we overflow the seq_file buffer, then it will 3796 * ask us for this data again at start up. 3797 * Use that instead. 3798 * ret is 0 if seq_file write succeeded. 3799 * -1 otherwise. 3800 */ 3801 iter->leftover = ret; 3802 } 3803 3804 return 0; 3805 } 3806 3807 /* 3808 * Should be used after trace_array_get(), trace_types_lock 3809 * ensures that i_cdev was already initialized. 3810 */ 3811 static inline int tracing_get_cpu(struct inode *inode) 3812 { 3813 if (inode->i_cdev) /* See trace_create_cpu_file() */ 3814 return (long)inode->i_cdev - 1; 3815 return RING_BUFFER_ALL_CPUS; 3816 } 3817 3818 static const struct seq_operations tracer_seq_ops = { 3819 .start = s_start, 3820 .next = s_next, 3821 .stop = s_stop, 3822 .show = s_show, 3823 }; 3824 3825 static struct trace_iterator * 3826 __tracing_open(struct inode *inode, struct file *file, bool snapshot) 3827 { 3828 struct trace_array *tr = inode->i_private; 3829 struct trace_iterator *iter; 3830 int cpu; 3831 3832 if (tracing_disabled) 3833 return ERR_PTR(-ENODEV); 3834 3835 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter)); 3836 if (!iter) 3837 return ERR_PTR(-ENOMEM); 3838 3839 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter), 3840 GFP_KERNEL); 3841 if (!iter->buffer_iter) 3842 goto release; 3843 3844 /* 3845 * We make a copy of the current tracer to avoid concurrent 3846 * changes on it while we are reading. 3847 */ 3848 mutex_lock(&trace_types_lock); 3849 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL); 3850 if (!iter->trace) 3851 goto fail; 3852 3853 *iter->trace = *tr->current_trace; 3854 3855 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL)) 3856 goto fail; 3857 3858 iter->tr = tr; 3859 3860 #ifdef CONFIG_TRACER_MAX_TRACE 3861 /* Currently only the top directory has a snapshot */ 3862 if (tr->current_trace->print_max || snapshot) 3863 iter->trace_buffer = &tr->max_buffer; 3864 else 3865 #endif 3866 iter->trace_buffer = &tr->trace_buffer; 3867 iter->snapshot = snapshot; 3868 iter->pos = -1; 3869 iter->cpu_file = tracing_get_cpu(inode); 3870 mutex_init(&iter->mutex); 3871 3872 /* Notify the tracer early; before we stop tracing. */ 3873 if (iter->trace && iter->trace->open) 3874 iter->trace->open(iter); 3875 3876 /* Annotate start of buffers if we had overruns */ 3877 if (ring_buffer_overruns(iter->trace_buffer->buffer)) 3878 iter->iter_flags |= TRACE_FILE_ANNOTATE; 3879 3880 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 3881 if (trace_clocks[tr->clock_id].in_ns) 3882 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 3883 3884 /* stop the trace while dumping if we are not opening "snapshot" */ 3885 if (!iter->snapshot) 3886 tracing_stop_tr(tr); 3887 3888 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { 3889 for_each_tracing_cpu(cpu) { 3890 iter->buffer_iter[cpu] = 3891 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu); 3892 } 3893 ring_buffer_read_prepare_sync(); 3894 for_each_tracing_cpu(cpu) { 3895 ring_buffer_read_start(iter->buffer_iter[cpu]); 3896 tracing_iter_reset(iter, cpu); 3897 } 3898 } else { 3899 cpu = iter->cpu_file; 3900 iter->buffer_iter[cpu] = 3901 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu); 3902 ring_buffer_read_prepare_sync(); 3903 ring_buffer_read_start(iter->buffer_iter[cpu]); 3904 tracing_iter_reset(iter, cpu); 3905 } 3906 3907 mutex_unlock(&trace_types_lock); 3908 3909 return iter; 3910 3911 fail: 3912 mutex_unlock(&trace_types_lock); 3913 kfree(iter->trace); 3914 kfree(iter->buffer_iter); 3915 release: 3916 seq_release_private(inode, file); 3917 return ERR_PTR(-ENOMEM); 3918 } 3919 3920 int tracing_open_generic(struct inode *inode, struct file *filp) 3921 { 3922 if (tracing_disabled) 3923 return -ENODEV; 3924 3925 filp->private_data = inode->i_private; 3926 return 0; 3927 } 3928 3929 bool tracing_is_disabled(void) 3930 { 3931 return (tracing_disabled) ? true: false; 3932 } 3933 3934 /* 3935 * Open and update trace_array ref count. 3936 * Must have the current trace_array passed to it. 3937 */ 3938 static int tracing_open_generic_tr(struct inode *inode, struct file *filp) 3939 { 3940 struct trace_array *tr = inode->i_private; 3941 3942 if (tracing_disabled) 3943 return -ENODEV; 3944 3945 if (trace_array_get(tr) < 0) 3946 return -ENODEV; 3947 3948 filp->private_data = inode->i_private; 3949 3950 return 0; 3951 } 3952 3953 static int tracing_release(struct inode *inode, struct file *file) 3954 { 3955 struct trace_array *tr = inode->i_private; 3956 struct seq_file *m = file->private_data; 3957 struct trace_iterator *iter; 3958 int cpu; 3959 3960 if (!(file->f_mode & FMODE_READ)) { 3961 trace_array_put(tr); 3962 return 0; 3963 } 3964 3965 /* Writes do not use seq_file */ 3966 iter = m->private; 3967 mutex_lock(&trace_types_lock); 3968 3969 for_each_tracing_cpu(cpu) { 3970 if (iter->buffer_iter[cpu]) 3971 ring_buffer_read_finish(iter->buffer_iter[cpu]); 3972 } 3973 3974 if (iter->trace && iter->trace->close) 3975 iter->trace->close(iter); 3976 3977 if (!iter->snapshot) 3978 /* reenable tracing if it was previously enabled */ 3979 tracing_start_tr(tr); 3980 3981 __trace_array_put(tr); 3982 3983 mutex_unlock(&trace_types_lock); 3984 3985 mutex_destroy(&iter->mutex); 3986 free_cpumask_var(iter->started); 3987 kfree(iter->trace); 3988 kfree(iter->buffer_iter); 3989 seq_release_private(inode, file); 3990 3991 return 0; 3992 } 3993 3994 static int tracing_release_generic_tr(struct inode *inode, struct file *file) 3995 { 3996 struct trace_array *tr = inode->i_private; 3997 3998 trace_array_put(tr); 3999 return 0; 4000 } 4001 4002 static int tracing_single_release_tr(struct inode *inode, struct file *file) 4003 { 4004 struct trace_array *tr = inode->i_private; 4005 4006 trace_array_put(tr); 4007 4008 return single_release(inode, file); 4009 } 4010 4011 static int tracing_open(struct inode *inode, struct file *file) 4012 { 4013 struct trace_array *tr = inode->i_private; 4014 struct trace_iterator *iter; 4015 int ret = 0; 4016 4017 if (trace_array_get(tr) < 0) 4018 return -ENODEV; 4019 4020 /* If this file was open for write, then erase contents */ 4021 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 4022 int cpu = tracing_get_cpu(inode); 4023 struct trace_buffer *trace_buf = &tr->trace_buffer; 4024 4025 #ifdef CONFIG_TRACER_MAX_TRACE 4026 if (tr->current_trace->print_max) 4027 trace_buf = &tr->max_buffer; 4028 #endif 4029 4030 if (cpu == RING_BUFFER_ALL_CPUS) 4031 tracing_reset_online_cpus(trace_buf); 4032 else 4033 tracing_reset(trace_buf, cpu); 4034 } 4035 4036 if (file->f_mode & FMODE_READ) { 4037 iter = __tracing_open(inode, file, false); 4038 if (IS_ERR(iter)) 4039 ret = PTR_ERR(iter); 4040 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 4041 iter->iter_flags |= TRACE_FILE_LAT_FMT; 4042 } 4043 4044 if (ret < 0) 4045 trace_array_put(tr); 4046 4047 return ret; 4048 } 4049 4050 /* 4051 * Some tracers are not suitable for instance buffers. 4052 * A tracer is always available for the global array (toplevel) 4053 * or if it explicitly states that it is. 4054 */ 4055 static bool 4056 trace_ok_for_array(struct tracer *t, struct trace_array *tr) 4057 { 4058 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances; 4059 } 4060 4061 /* Find the next tracer that this trace array may use */ 4062 static struct tracer * 4063 get_tracer_for_array(struct trace_array *tr, struct tracer *t) 4064 { 4065 while (t && !trace_ok_for_array(t, tr)) 4066 t = t->next; 4067 4068 return t; 4069 } 4070 4071 static void * 4072 t_next(struct seq_file *m, void *v, loff_t *pos) 4073 { 4074 struct trace_array *tr = m->private; 4075 struct tracer *t = v; 4076 4077 (*pos)++; 4078 4079 if (t) 4080 t = get_tracer_for_array(tr, t->next); 4081 4082 return t; 4083 } 4084 4085 static void *t_start(struct seq_file *m, loff_t *pos) 4086 { 4087 struct trace_array *tr = m->private; 4088 struct tracer *t; 4089 loff_t l = 0; 4090 4091 mutex_lock(&trace_types_lock); 4092 4093 t = get_tracer_for_array(tr, trace_types); 4094 for (; t && l < *pos; t = t_next(m, t, &l)) 4095 ; 4096 4097 return t; 4098 } 4099 4100 static void t_stop(struct seq_file *m, void *p) 4101 { 4102 mutex_unlock(&trace_types_lock); 4103 } 4104 4105 static int t_show(struct seq_file *m, void *v) 4106 { 4107 struct tracer *t = v; 4108 4109 if (!t) 4110 return 0; 4111 4112 seq_puts(m, t->name); 4113 if (t->next) 4114 seq_putc(m, ' '); 4115 else 4116 seq_putc(m, '\n'); 4117 4118 return 0; 4119 } 4120 4121 static const struct seq_operations show_traces_seq_ops = { 4122 .start = t_start, 4123 .next = t_next, 4124 .stop = t_stop, 4125 .show = t_show, 4126 }; 4127 4128 static int show_traces_open(struct inode *inode, struct file *file) 4129 { 4130 struct trace_array *tr = inode->i_private; 4131 struct seq_file *m; 4132 int ret; 4133 4134 if (tracing_disabled) 4135 return -ENODEV; 4136 4137 ret = seq_open(file, &show_traces_seq_ops); 4138 if (ret) 4139 return ret; 4140 4141 m = file->private_data; 4142 m->private = tr; 4143 4144 return 0; 4145 } 4146 4147 static ssize_t 4148 tracing_write_stub(struct file *filp, const char __user *ubuf, 4149 size_t count, loff_t *ppos) 4150 { 4151 return count; 4152 } 4153 4154 loff_t tracing_lseek(struct file *file, loff_t offset, int whence) 4155 { 4156 int ret; 4157 4158 if (file->f_mode & FMODE_READ) 4159 ret = seq_lseek(file, offset, whence); 4160 else 4161 file->f_pos = ret = 0; 4162 4163 return ret; 4164 } 4165 4166 static const struct file_operations tracing_fops = { 4167 .open = tracing_open, 4168 .read = seq_read, 4169 .write = tracing_write_stub, 4170 .llseek = tracing_lseek, 4171 .release = tracing_release, 4172 }; 4173 4174 static const struct file_operations show_traces_fops = { 4175 .open = show_traces_open, 4176 .read = seq_read, 4177 .release = seq_release, 4178 .llseek = seq_lseek, 4179 }; 4180 4181 /* 4182 * The tracer itself will not take this lock, but still we want 4183 * to provide a consistent cpumask to user-space: 4184 */ 4185 static DEFINE_MUTEX(tracing_cpumask_update_lock); 4186 4187 /* 4188 * Temporary storage for the character representation of the 4189 * CPU bitmask (and one more byte for the newline): 4190 */ 4191 static char mask_str[NR_CPUS + 1]; 4192 4193 static ssize_t 4194 tracing_cpumask_read(struct file *filp, char __user *ubuf, 4195 size_t count, loff_t *ppos) 4196 { 4197 struct trace_array *tr = file_inode(filp)->i_private; 4198 int len; 4199 4200 mutex_lock(&tracing_cpumask_update_lock); 4201 4202 len = snprintf(mask_str, count, "%*pb\n", 4203 cpumask_pr_args(tr->tracing_cpumask)); 4204 if (len >= count) { 4205 count = -EINVAL; 4206 goto out_err; 4207 } 4208 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1); 4209 4210 out_err: 4211 mutex_unlock(&tracing_cpumask_update_lock); 4212 4213 return count; 4214 } 4215 4216 static ssize_t 4217 tracing_cpumask_write(struct file *filp, const char __user *ubuf, 4218 size_t count, loff_t *ppos) 4219 { 4220 struct trace_array *tr = file_inode(filp)->i_private; 4221 cpumask_var_t tracing_cpumask_new; 4222 int err, cpu; 4223 4224 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL)) 4225 return -ENOMEM; 4226 4227 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new); 4228 if (err) 4229 goto err_unlock; 4230 4231 mutex_lock(&tracing_cpumask_update_lock); 4232 4233 local_irq_disable(); 4234 arch_spin_lock(&tr->max_lock); 4235 for_each_tracing_cpu(cpu) { 4236 /* 4237 * Increase/decrease the disabled counter if we are 4238 * about to flip a bit in the cpumask: 4239 */ 4240 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) && 4241 !cpumask_test_cpu(cpu, tracing_cpumask_new)) { 4242 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled); 4243 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu); 4244 } 4245 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) && 4246 cpumask_test_cpu(cpu, tracing_cpumask_new)) { 4247 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled); 4248 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu); 4249 } 4250 } 4251 arch_spin_unlock(&tr->max_lock); 4252 local_irq_enable(); 4253 4254 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new); 4255 4256 mutex_unlock(&tracing_cpumask_update_lock); 4257 free_cpumask_var(tracing_cpumask_new); 4258 4259 return count; 4260 4261 err_unlock: 4262 free_cpumask_var(tracing_cpumask_new); 4263 4264 return err; 4265 } 4266 4267 static const struct file_operations tracing_cpumask_fops = { 4268 .open = tracing_open_generic_tr, 4269 .read = tracing_cpumask_read, 4270 .write = tracing_cpumask_write, 4271 .release = tracing_release_generic_tr, 4272 .llseek = generic_file_llseek, 4273 }; 4274 4275 static int tracing_trace_options_show(struct seq_file *m, void *v) 4276 { 4277 struct tracer_opt *trace_opts; 4278 struct trace_array *tr = m->private; 4279 u32 tracer_flags; 4280 int i; 4281 4282 mutex_lock(&trace_types_lock); 4283 tracer_flags = tr->current_trace->flags->val; 4284 trace_opts = tr->current_trace->flags->opts; 4285 4286 for (i = 0; trace_options[i]; i++) { 4287 if (tr->trace_flags & (1 << i)) 4288 seq_printf(m, "%s\n", trace_options[i]); 4289 else 4290 seq_printf(m, "no%s\n", trace_options[i]); 4291 } 4292 4293 for (i = 0; trace_opts[i].name; i++) { 4294 if (tracer_flags & trace_opts[i].bit) 4295 seq_printf(m, "%s\n", trace_opts[i].name); 4296 else 4297 seq_printf(m, "no%s\n", trace_opts[i].name); 4298 } 4299 mutex_unlock(&trace_types_lock); 4300 4301 return 0; 4302 } 4303 4304 static int __set_tracer_option(struct trace_array *tr, 4305 struct tracer_flags *tracer_flags, 4306 struct tracer_opt *opts, int neg) 4307 { 4308 struct tracer *trace = tracer_flags->trace; 4309 int ret; 4310 4311 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg); 4312 if (ret) 4313 return ret; 4314 4315 if (neg) 4316 tracer_flags->val &= ~opts->bit; 4317 else 4318 tracer_flags->val |= opts->bit; 4319 return 0; 4320 } 4321 4322 /* Try to assign a tracer specific option */ 4323 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg) 4324 { 4325 struct tracer *trace = tr->current_trace; 4326 struct tracer_flags *tracer_flags = trace->flags; 4327 struct tracer_opt *opts = NULL; 4328 int i; 4329 4330 for (i = 0; tracer_flags->opts[i].name; i++) { 4331 opts = &tracer_flags->opts[i]; 4332 4333 if (strcmp(cmp, opts->name) == 0) 4334 return __set_tracer_option(tr, trace->flags, opts, neg); 4335 } 4336 4337 return -EINVAL; 4338 } 4339 4340 /* Some tracers require overwrite to stay enabled */ 4341 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set) 4342 { 4343 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set) 4344 return -1; 4345 4346 return 0; 4347 } 4348 4349 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled) 4350 { 4351 /* do nothing if flag is already set */ 4352 if (!!(tr->trace_flags & mask) == !!enabled) 4353 return 0; 4354 4355 /* Give the tracer a chance to approve the change */ 4356 if (tr->current_trace->flag_changed) 4357 if (tr->current_trace->flag_changed(tr, mask, !!enabled)) 4358 return -EINVAL; 4359 4360 if (enabled) 4361 tr->trace_flags |= mask; 4362 else 4363 tr->trace_flags &= ~mask; 4364 4365 if (mask == TRACE_ITER_RECORD_CMD) 4366 trace_event_enable_cmd_record(enabled); 4367 4368 if (mask == TRACE_ITER_RECORD_TGID) { 4369 if (!tgid_map) 4370 tgid_map = kzalloc((PID_MAX_DEFAULT + 1) * sizeof(*tgid_map), 4371 GFP_KERNEL); 4372 if (!tgid_map) { 4373 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID; 4374 return -ENOMEM; 4375 } 4376 4377 trace_event_enable_tgid_record(enabled); 4378 } 4379 4380 if (mask == TRACE_ITER_EVENT_FORK) 4381 trace_event_follow_fork(tr, enabled); 4382 4383 if (mask == TRACE_ITER_FUNC_FORK) 4384 ftrace_pid_follow_fork(tr, enabled); 4385 4386 if (mask == TRACE_ITER_OVERWRITE) { 4387 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled); 4388 #ifdef CONFIG_TRACER_MAX_TRACE 4389 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled); 4390 #endif 4391 } 4392 4393 if (mask == TRACE_ITER_PRINTK) { 4394 trace_printk_start_stop_comm(enabled); 4395 trace_printk_control(enabled); 4396 } 4397 4398 return 0; 4399 } 4400 4401 static int trace_set_options(struct trace_array *tr, char *option) 4402 { 4403 char *cmp; 4404 int neg = 0; 4405 int ret = -ENODEV; 4406 int i; 4407 size_t orig_len = strlen(option); 4408 4409 cmp = strstrip(option); 4410 4411 if (strncmp(cmp, "no", 2) == 0) { 4412 neg = 1; 4413 cmp += 2; 4414 } 4415 4416 mutex_lock(&trace_types_lock); 4417 4418 for (i = 0; trace_options[i]; i++) { 4419 if (strcmp(cmp, trace_options[i]) == 0) { 4420 ret = set_tracer_flag(tr, 1 << i, !neg); 4421 break; 4422 } 4423 } 4424 4425 /* If no option could be set, test the specific tracer options */ 4426 if (!trace_options[i]) 4427 ret = set_tracer_option(tr, cmp, neg); 4428 4429 mutex_unlock(&trace_types_lock); 4430 4431 /* 4432 * If the first trailing whitespace is replaced with '\0' by strstrip, 4433 * turn it back into a space. 4434 */ 4435 if (orig_len > strlen(option)) 4436 option[strlen(option)] = ' '; 4437 4438 return ret; 4439 } 4440 4441 static void __init apply_trace_boot_options(void) 4442 { 4443 char *buf = trace_boot_options_buf; 4444 char *option; 4445 4446 while (true) { 4447 option = strsep(&buf, ","); 4448 4449 if (!option) 4450 break; 4451 4452 if (*option) 4453 trace_set_options(&global_trace, option); 4454 4455 /* Put back the comma to allow this to be called again */ 4456 if (buf) 4457 *(buf - 1) = ','; 4458 } 4459 } 4460 4461 static ssize_t 4462 tracing_trace_options_write(struct file *filp, const char __user *ubuf, 4463 size_t cnt, loff_t *ppos) 4464 { 4465 struct seq_file *m = filp->private_data; 4466 struct trace_array *tr = m->private; 4467 char buf[64]; 4468 int ret; 4469 4470 if (cnt >= sizeof(buf)) 4471 return -EINVAL; 4472 4473 if (copy_from_user(buf, ubuf, cnt)) 4474 return -EFAULT; 4475 4476 buf[cnt] = 0; 4477 4478 ret = trace_set_options(tr, buf); 4479 if (ret < 0) 4480 return ret; 4481 4482 *ppos += cnt; 4483 4484 return cnt; 4485 } 4486 4487 static int tracing_trace_options_open(struct inode *inode, struct file *file) 4488 { 4489 struct trace_array *tr = inode->i_private; 4490 int ret; 4491 4492 if (tracing_disabled) 4493 return -ENODEV; 4494 4495 if (trace_array_get(tr) < 0) 4496 return -ENODEV; 4497 4498 ret = single_open(file, tracing_trace_options_show, inode->i_private); 4499 if (ret < 0) 4500 trace_array_put(tr); 4501 4502 return ret; 4503 } 4504 4505 static const struct file_operations tracing_iter_fops = { 4506 .open = tracing_trace_options_open, 4507 .read = seq_read, 4508 .llseek = seq_lseek, 4509 .release = tracing_single_release_tr, 4510 .write = tracing_trace_options_write, 4511 }; 4512 4513 static const char readme_msg[] = 4514 "tracing mini-HOWTO:\n\n" 4515 "# echo 0 > tracing_on : quick way to disable tracing\n" 4516 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n" 4517 " Important files:\n" 4518 " trace\t\t\t- The static contents of the buffer\n" 4519 "\t\t\t To clear the buffer write into this file: echo > trace\n" 4520 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n" 4521 " current_tracer\t- function and latency tracers\n" 4522 " available_tracers\t- list of configured tracers for current_tracer\n" 4523 " buffer_size_kb\t- view and modify size of per cpu buffer\n" 4524 " buffer_total_size_kb - view total size of all cpu buffers\n\n" 4525 " trace_clock\t\t-change the clock used to order events\n" 4526 " local: Per cpu clock but may not be synced across CPUs\n" 4527 " global: Synced across CPUs but slows tracing down.\n" 4528 " counter: Not a clock, but just an increment\n" 4529 " uptime: Jiffy counter from time of boot\n" 4530 " perf: Same clock that perf events use\n" 4531 #ifdef CONFIG_X86_64 4532 " x86-tsc: TSC cycle counter\n" 4533 #endif 4534 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n" 4535 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n" 4536 " tracing_cpumask\t- Limit which CPUs to trace\n" 4537 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n" 4538 "\t\t\t Remove sub-buffer with rmdir\n" 4539 " trace_options\t\t- Set format or modify how tracing happens\n" 4540 "\t\t\t Disable an option by adding a suffix 'no' to the\n" 4541 "\t\t\t option name\n" 4542 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n" 4543 #ifdef CONFIG_DYNAMIC_FTRACE 4544 "\n available_filter_functions - list of functions that can be filtered on\n" 4545 " set_ftrace_filter\t- echo function name in here to only trace these\n" 4546 "\t\t\t functions\n" 4547 "\t accepts: func_full_name or glob-matching-pattern\n" 4548 "\t modules: Can select a group via module\n" 4549 "\t Format: :mod:<module-name>\n" 4550 "\t example: echo :mod:ext3 > set_ftrace_filter\n" 4551 "\t triggers: a command to perform when function is hit\n" 4552 "\t Format: <function>:<trigger>[:count]\n" 4553 "\t trigger: traceon, traceoff\n" 4554 "\t\t enable_event:<system>:<event>\n" 4555 "\t\t disable_event:<system>:<event>\n" 4556 #ifdef CONFIG_STACKTRACE 4557 "\t\t stacktrace\n" 4558 #endif 4559 #ifdef CONFIG_TRACER_SNAPSHOT 4560 "\t\t snapshot\n" 4561 #endif 4562 "\t\t dump\n" 4563 "\t\t cpudump\n" 4564 "\t example: echo do_fault:traceoff > set_ftrace_filter\n" 4565 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n" 4566 "\t The first one will disable tracing every time do_fault is hit\n" 4567 "\t The second will disable tracing at most 3 times when do_trap is hit\n" 4568 "\t The first time do trap is hit and it disables tracing, the\n" 4569 "\t counter will decrement to 2. If tracing is already disabled,\n" 4570 "\t the counter will not decrement. It only decrements when the\n" 4571 "\t trigger did work\n" 4572 "\t To remove trigger without count:\n" 4573 "\t echo '!<function>:<trigger> > set_ftrace_filter\n" 4574 "\t To remove trigger with a count:\n" 4575 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n" 4576 " set_ftrace_notrace\t- echo function name in here to never trace.\n" 4577 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n" 4578 "\t modules: Can select a group via module command :mod:\n" 4579 "\t Does not accept triggers\n" 4580 #endif /* CONFIG_DYNAMIC_FTRACE */ 4581 #ifdef CONFIG_FUNCTION_TRACER 4582 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n" 4583 "\t\t (function)\n" 4584 #endif 4585 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 4586 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n" 4587 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n" 4588 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n" 4589 #endif 4590 #ifdef CONFIG_TRACER_SNAPSHOT 4591 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n" 4592 "\t\t\t snapshot buffer. Read the contents for more\n" 4593 "\t\t\t information\n" 4594 #endif 4595 #ifdef CONFIG_STACK_TRACER 4596 " stack_trace\t\t- Shows the max stack trace when active\n" 4597 " stack_max_size\t- Shows current max stack size that was traced\n" 4598 "\t\t\t Write into this file to reset the max size (trigger a\n" 4599 "\t\t\t new trace)\n" 4600 #ifdef CONFIG_DYNAMIC_FTRACE 4601 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n" 4602 "\t\t\t traces\n" 4603 #endif 4604 #endif /* CONFIG_STACK_TRACER */ 4605 #ifdef CONFIG_KPROBE_EVENTS 4606 " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n" 4607 "\t\t\t Write into this file to define/undefine new trace events.\n" 4608 #endif 4609 #ifdef CONFIG_UPROBE_EVENTS 4610 " uprobe_events\t\t- Add/remove/show the userspace dynamic events\n" 4611 "\t\t\t Write into this file to define/undefine new trace events.\n" 4612 #endif 4613 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS) 4614 "\t accepts: event-definitions (one definition per line)\n" 4615 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n" 4616 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n" 4617 "\t -:[<group>/]<event>\n" 4618 #ifdef CONFIG_KPROBE_EVENTS 4619 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n" 4620 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n" 4621 #endif 4622 #ifdef CONFIG_UPROBE_EVENTS 4623 "\t place: <path>:<offset>\n" 4624 #endif 4625 "\t args: <name>=fetcharg[:type]\n" 4626 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n" 4627 "\t $stack<index>, $stack, $retval, $comm\n" 4628 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string,\n" 4629 "\t b<bit-width>@<bit-offset>/<container-size>\n" 4630 #endif 4631 " events/\t\t- Directory containing all trace event subsystems:\n" 4632 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n" 4633 " events/<system>/\t- Directory containing all trace events for <system>:\n" 4634 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n" 4635 "\t\t\t events\n" 4636 " filter\t\t- If set, only events passing filter are traced\n" 4637 " events/<system>/<event>/\t- Directory containing control files for\n" 4638 "\t\t\t <event>:\n" 4639 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n" 4640 " filter\t\t- If set, only events passing filter are traced\n" 4641 " trigger\t\t- If set, a command to perform when event is hit\n" 4642 "\t Format: <trigger>[:count][if <filter>]\n" 4643 "\t trigger: traceon, traceoff\n" 4644 "\t enable_event:<system>:<event>\n" 4645 "\t disable_event:<system>:<event>\n" 4646 #ifdef CONFIG_HIST_TRIGGERS 4647 "\t enable_hist:<system>:<event>\n" 4648 "\t disable_hist:<system>:<event>\n" 4649 #endif 4650 #ifdef CONFIG_STACKTRACE 4651 "\t\t stacktrace\n" 4652 #endif 4653 #ifdef CONFIG_TRACER_SNAPSHOT 4654 "\t\t snapshot\n" 4655 #endif 4656 #ifdef CONFIG_HIST_TRIGGERS 4657 "\t\t hist (see below)\n" 4658 #endif 4659 "\t example: echo traceoff > events/block/block_unplug/trigger\n" 4660 "\t echo traceoff:3 > events/block/block_unplug/trigger\n" 4661 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n" 4662 "\t events/block/block_unplug/trigger\n" 4663 "\t The first disables tracing every time block_unplug is hit.\n" 4664 "\t The second disables tracing the first 3 times block_unplug is hit.\n" 4665 "\t The third enables the kmalloc event the first 3 times block_unplug\n" 4666 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n" 4667 "\t Like function triggers, the counter is only decremented if it\n" 4668 "\t enabled or disabled tracing.\n" 4669 "\t To remove a trigger without a count:\n" 4670 "\t echo '!<trigger> > <system>/<event>/trigger\n" 4671 "\t To remove a trigger with a count:\n" 4672 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n" 4673 "\t Filters can be ignored when removing a trigger.\n" 4674 #ifdef CONFIG_HIST_TRIGGERS 4675 " hist trigger\t- If set, event hits are aggregated into a hash table\n" 4676 "\t Format: hist:keys=<field1[,field2,...]>\n" 4677 "\t [:values=<field1[,field2,...]>]\n" 4678 "\t [:sort=<field1[,field2,...]>]\n" 4679 "\t [:size=#entries]\n" 4680 "\t [:pause][:continue][:clear]\n" 4681 "\t [:name=histname1]\n" 4682 "\t [if <filter>]\n\n" 4683 "\t When a matching event is hit, an entry is added to a hash\n" 4684 "\t table using the key(s) and value(s) named, and the value of a\n" 4685 "\t sum called 'hitcount' is incremented. Keys and values\n" 4686 "\t correspond to fields in the event's format description. Keys\n" 4687 "\t can be any field, or the special string 'stacktrace'.\n" 4688 "\t Compound keys consisting of up to two fields can be specified\n" 4689 "\t by the 'keys' keyword. Values must correspond to numeric\n" 4690 "\t fields. Sort keys consisting of up to two fields can be\n" 4691 "\t specified using the 'sort' keyword. The sort direction can\n" 4692 "\t be modified by appending '.descending' or '.ascending' to a\n" 4693 "\t sort field. The 'size' parameter can be used to specify more\n" 4694 "\t or fewer than the default 2048 entries for the hashtable size.\n" 4695 "\t If a hist trigger is given a name using the 'name' parameter,\n" 4696 "\t its histogram data will be shared with other triggers of the\n" 4697 "\t same name, and trigger hits will update this common data.\n\n" 4698 "\t Reading the 'hist' file for the event will dump the hash\n" 4699 "\t table in its entirety to stdout. If there are multiple hist\n" 4700 "\t triggers attached to an event, there will be a table for each\n" 4701 "\t trigger in the output. The table displayed for a named\n" 4702 "\t trigger will be the same as any other instance having the\n" 4703 "\t same name. The default format used to display a given field\n" 4704 "\t can be modified by appending any of the following modifiers\n" 4705 "\t to the field name, as applicable:\n\n" 4706 "\t .hex display a number as a hex value\n" 4707 "\t .sym display an address as a symbol\n" 4708 "\t .sym-offset display an address as a symbol and offset\n" 4709 "\t .execname display a common_pid as a program name\n" 4710 "\t .syscall display a syscall id as a syscall name\n\n" 4711 "\t .log2 display log2 value rather than raw number\n\n" 4712 "\t The 'pause' parameter can be used to pause an existing hist\n" 4713 "\t trigger or to start a hist trigger but not log any events\n" 4714 "\t until told to do so. 'continue' can be used to start or\n" 4715 "\t restart a paused hist trigger.\n\n" 4716 "\t The 'clear' parameter will clear the contents of a running\n" 4717 "\t hist trigger and leave its current paused/active state\n" 4718 "\t unchanged.\n\n" 4719 "\t The enable_hist and disable_hist triggers can be used to\n" 4720 "\t have one event conditionally start and stop another event's\n" 4721 "\t already-attached hist trigger. The syntax is analagous to\n" 4722 "\t the enable_event and disable_event triggers.\n" 4723 #endif 4724 ; 4725 4726 static ssize_t 4727 tracing_readme_read(struct file *filp, char __user *ubuf, 4728 size_t cnt, loff_t *ppos) 4729 { 4730 return simple_read_from_buffer(ubuf, cnt, ppos, 4731 readme_msg, strlen(readme_msg)); 4732 } 4733 4734 static const struct file_operations tracing_readme_fops = { 4735 .open = tracing_open_generic, 4736 .read = tracing_readme_read, 4737 .llseek = generic_file_llseek, 4738 }; 4739 4740 static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos) 4741 { 4742 int *ptr = v; 4743 4744 if (*pos || m->count) 4745 ptr++; 4746 4747 (*pos)++; 4748 4749 for (; ptr <= &tgid_map[PID_MAX_DEFAULT]; ptr++) { 4750 if (trace_find_tgid(*ptr)) 4751 return ptr; 4752 } 4753 4754 return NULL; 4755 } 4756 4757 static void *saved_tgids_start(struct seq_file *m, loff_t *pos) 4758 { 4759 void *v; 4760 loff_t l = 0; 4761 4762 if (!tgid_map) 4763 return NULL; 4764 4765 v = &tgid_map[0]; 4766 while (l <= *pos) { 4767 v = saved_tgids_next(m, v, &l); 4768 if (!v) 4769 return NULL; 4770 } 4771 4772 return v; 4773 } 4774 4775 static void saved_tgids_stop(struct seq_file *m, void *v) 4776 { 4777 } 4778 4779 static int saved_tgids_show(struct seq_file *m, void *v) 4780 { 4781 int pid = (int *)v - tgid_map; 4782 4783 seq_printf(m, "%d %d\n", pid, trace_find_tgid(pid)); 4784 return 0; 4785 } 4786 4787 static const struct seq_operations tracing_saved_tgids_seq_ops = { 4788 .start = saved_tgids_start, 4789 .stop = saved_tgids_stop, 4790 .next = saved_tgids_next, 4791 .show = saved_tgids_show, 4792 }; 4793 4794 static int tracing_saved_tgids_open(struct inode *inode, struct file *filp) 4795 { 4796 if (tracing_disabled) 4797 return -ENODEV; 4798 4799 return seq_open(filp, &tracing_saved_tgids_seq_ops); 4800 } 4801 4802 4803 static const struct file_operations tracing_saved_tgids_fops = { 4804 .open = tracing_saved_tgids_open, 4805 .read = seq_read, 4806 .llseek = seq_lseek, 4807 .release = seq_release, 4808 }; 4809 4810 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos) 4811 { 4812 unsigned int *ptr = v; 4813 4814 if (*pos || m->count) 4815 ptr++; 4816 4817 (*pos)++; 4818 4819 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num]; 4820 ptr++) { 4821 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP) 4822 continue; 4823 4824 return ptr; 4825 } 4826 4827 return NULL; 4828 } 4829 4830 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos) 4831 { 4832 void *v; 4833 loff_t l = 0; 4834 4835 preempt_disable(); 4836 arch_spin_lock(&trace_cmdline_lock); 4837 4838 v = &savedcmd->map_cmdline_to_pid[0]; 4839 while (l <= *pos) { 4840 v = saved_cmdlines_next(m, v, &l); 4841 if (!v) 4842 return NULL; 4843 } 4844 4845 return v; 4846 } 4847 4848 static void saved_cmdlines_stop(struct seq_file *m, void *v) 4849 { 4850 arch_spin_unlock(&trace_cmdline_lock); 4851 preempt_enable(); 4852 } 4853 4854 static int saved_cmdlines_show(struct seq_file *m, void *v) 4855 { 4856 char buf[TASK_COMM_LEN]; 4857 unsigned int *pid = v; 4858 4859 __trace_find_cmdline(*pid, buf); 4860 seq_printf(m, "%d %s\n", *pid, buf); 4861 return 0; 4862 } 4863 4864 static const struct seq_operations tracing_saved_cmdlines_seq_ops = { 4865 .start = saved_cmdlines_start, 4866 .next = saved_cmdlines_next, 4867 .stop = saved_cmdlines_stop, 4868 .show = saved_cmdlines_show, 4869 }; 4870 4871 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp) 4872 { 4873 if (tracing_disabled) 4874 return -ENODEV; 4875 4876 return seq_open(filp, &tracing_saved_cmdlines_seq_ops); 4877 } 4878 4879 static const struct file_operations tracing_saved_cmdlines_fops = { 4880 .open = tracing_saved_cmdlines_open, 4881 .read = seq_read, 4882 .llseek = seq_lseek, 4883 .release = seq_release, 4884 }; 4885 4886 static ssize_t 4887 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf, 4888 size_t cnt, loff_t *ppos) 4889 { 4890 char buf[64]; 4891 int r; 4892 4893 arch_spin_lock(&trace_cmdline_lock); 4894 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num); 4895 arch_spin_unlock(&trace_cmdline_lock); 4896 4897 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 4898 } 4899 4900 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s) 4901 { 4902 kfree(s->saved_cmdlines); 4903 kfree(s->map_cmdline_to_pid); 4904 kfree(s); 4905 } 4906 4907 static int tracing_resize_saved_cmdlines(unsigned int val) 4908 { 4909 struct saved_cmdlines_buffer *s, *savedcmd_temp; 4910 4911 s = kmalloc(sizeof(*s), GFP_KERNEL); 4912 if (!s) 4913 return -ENOMEM; 4914 4915 if (allocate_cmdlines_buffer(val, s) < 0) { 4916 kfree(s); 4917 return -ENOMEM; 4918 } 4919 4920 arch_spin_lock(&trace_cmdline_lock); 4921 savedcmd_temp = savedcmd; 4922 savedcmd = s; 4923 arch_spin_unlock(&trace_cmdline_lock); 4924 free_saved_cmdlines_buffer(savedcmd_temp); 4925 4926 return 0; 4927 } 4928 4929 static ssize_t 4930 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf, 4931 size_t cnt, loff_t *ppos) 4932 { 4933 unsigned long val; 4934 int ret; 4935 4936 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 4937 if (ret) 4938 return ret; 4939 4940 /* must have at least 1 entry or less than PID_MAX_DEFAULT */ 4941 if (!val || val > PID_MAX_DEFAULT) 4942 return -EINVAL; 4943 4944 ret = tracing_resize_saved_cmdlines((unsigned int)val); 4945 if (ret < 0) 4946 return ret; 4947 4948 *ppos += cnt; 4949 4950 return cnt; 4951 } 4952 4953 static const struct file_operations tracing_saved_cmdlines_size_fops = { 4954 .open = tracing_open_generic, 4955 .read = tracing_saved_cmdlines_size_read, 4956 .write = tracing_saved_cmdlines_size_write, 4957 }; 4958 4959 #ifdef CONFIG_TRACE_EVAL_MAP_FILE 4960 static union trace_eval_map_item * 4961 update_eval_map(union trace_eval_map_item *ptr) 4962 { 4963 if (!ptr->map.eval_string) { 4964 if (ptr->tail.next) { 4965 ptr = ptr->tail.next; 4966 /* Set ptr to the next real item (skip head) */ 4967 ptr++; 4968 } else 4969 return NULL; 4970 } 4971 return ptr; 4972 } 4973 4974 static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos) 4975 { 4976 union trace_eval_map_item *ptr = v; 4977 4978 /* 4979 * Paranoid! If ptr points to end, we don't want to increment past it. 4980 * This really should never happen. 4981 */ 4982 ptr = update_eval_map(ptr); 4983 if (WARN_ON_ONCE(!ptr)) 4984 return NULL; 4985 4986 ptr++; 4987 4988 (*pos)++; 4989 4990 ptr = update_eval_map(ptr); 4991 4992 return ptr; 4993 } 4994 4995 static void *eval_map_start(struct seq_file *m, loff_t *pos) 4996 { 4997 union trace_eval_map_item *v; 4998 loff_t l = 0; 4999 5000 mutex_lock(&trace_eval_mutex); 5001 5002 v = trace_eval_maps; 5003 if (v) 5004 v++; 5005 5006 while (v && l < *pos) { 5007 v = eval_map_next(m, v, &l); 5008 } 5009 5010 return v; 5011 } 5012 5013 static void eval_map_stop(struct seq_file *m, void *v) 5014 { 5015 mutex_unlock(&trace_eval_mutex); 5016 } 5017 5018 static int eval_map_show(struct seq_file *m, void *v) 5019 { 5020 union trace_eval_map_item *ptr = v; 5021 5022 seq_printf(m, "%s %ld (%s)\n", 5023 ptr->map.eval_string, ptr->map.eval_value, 5024 ptr->map.system); 5025 5026 return 0; 5027 } 5028 5029 static const struct seq_operations tracing_eval_map_seq_ops = { 5030 .start = eval_map_start, 5031 .next = eval_map_next, 5032 .stop = eval_map_stop, 5033 .show = eval_map_show, 5034 }; 5035 5036 static int tracing_eval_map_open(struct inode *inode, struct file *filp) 5037 { 5038 if (tracing_disabled) 5039 return -ENODEV; 5040 5041 return seq_open(filp, &tracing_eval_map_seq_ops); 5042 } 5043 5044 static const struct file_operations tracing_eval_map_fops = { 5045 .open = tracing_eval_map_open, 5046 .read = seq_read, 5047 .llseek = seq_lseek, 5048 .release = seq_release, 5049 }; 5050 5051 static inline union trace_eval_map_item * 5052 trace_eval_jmp_to_tail(union trace_eval_map_item *ptr) 5053 { 5054 /* Return tail of array given the head */ 5055 return ptr + ptr->head.length + 1; 5056 } 5057 5058 static void 5059 trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start, 5060 int len) 5061 { 5062 struct trace_eval_map **stop; 5063 struct trace_eval_map **map; 5064 union trace_eval_map_item *map_array; 5065 union trace_eval_map_item *ptr; 5066 5067 stop = start + len; 5068 5069 /* 5070 * The trace_eval_maps contains the map plus a head and tail item, 5071 * where the head holds the module and length of array, and the 5072 * tail holds a pointer to the next list. 5073 */ 5074 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL); 5075 if (!map_array) { 5076 pr_warn("Unable to allocate trace eval mapping\n"); 5077 return; 5078 } 5079 5080 mutex_lock(&trace_eval_mutex); 5081 5082 if (!trace_eval_maps) 5083 trace_eval_maps = map_array; 5084 else { 5085 ptr = trace_eval_maps; 5086 for (;;) { 5087 ptr = trace_eval_jmp_to_tail(ptr); 5088 if (!ptr->tail.next) 5089 break; 5090 ptr = ptr->tail.next; 5091 5092 } 5093 ptr->tail.next = map_array; 5094 } 5095 map_array->head.mod = mod; 5096 map_array->head.length = len; 5097 map_array++; 5098 5099 for (map = start; (unsigned long)map < (unsigned long)stop; map++) { 5100 map_array->map = **map; 5101 map_array++; 5102 } 5103 memset(map_array, 0, sizeof(*map_array)); 5104 5105 mutex_unlock(&trace_eval_mutex); 5106 } 5107 5108 static void trace_create_eval_file(struct dentry *d_tracer) 5109 { 5110 trace_create_file("eval_map", 0444, d_tracer, 5111 NULL, &tracing_eval_map_fops); 5112 } 5113 5114 #else /* CONFIG_TRACE_EVAL_MAP_FILE */ 5115 static inline void trace_create_eval_file(struct dentry *d_tracer) { } 5116 static inline void trace_insert_eval_map_file(struct module *mod, 5117 struct trace_eval_map **start, int len) { } 5118 #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */ 5119 5120 static void trace_insert_eval_map(struct module *mod, 5121 struct trace_eval_map **start, int len) 5122 { 5123 struct trace_eval_map **map; 5124 5125 if (len <= 0) 5126 return; 5127 5128 map = start; 5129 5130 trace_event_eval_update(map, len); 5131 5132 trace_insert_eval_map_file(mod, start, len); 5133 } 5134 5135 static ssize_t 5136 tracing_set_trace_read(struct file *filp, char __user *ubuf, 5137 size_t cnt, loff_t *ppos) 5138 { 5139 struct trace_array *tr = filp->private_data; 5140 char buf[MAX_TRACER_SIZE+2]; 5141 int r; 5142 5143 mutex_lock(&trace_types_lock); 5144 r = sprintf(buf, "%s\n", tr->current_trace->name); 5145 mutex_unlock(&trace_types_lock); 5146 5147 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5148 } 5149 5150 int tracer_init(struct tracer *t, struct trace_array *tr) 5151 { 5152 tracing_reset_online_cpus(&tr->trace_buffer); 5153 return t->init(tr); 5154 } 5155 5156 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val) 5157 { 5158 int cpu; 5159 5160 for_each_tracing_cpu(cpu) 5161 per_cpu_ptr(buf->data, cpu)->entries = val; 5162 } 5163 5164 #ifdef CONFIG_TRACER_MAX_TRACE 5165 /* resize @tr's buffer to the size of @size_tr's entries */ 5166 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf, 5167 struct trace_buffer *size_buf, int cpu_id) 5168 { 5169 int cpu, ret = 0; 5170 5171 if (cpu_id == RING_BUFFER_ALL_CPUS) { 5172 for_each_tracing_cpu(cpu) { 5173 ret = ring_buffer_resize(trace_buf->buffer, 5174 per_cpu_ptr(size_buf->data, cpu)->entries, cpu); 5175 if (ret < 0) 5176 break; 5177 per_cpu_ptr(trace_buf->data, cpu)->entries = 5178 per_cpu_ptr(size_buf->data, cpu)->entries; 5179 } 5180 } else { 5181 ret = ring_buffer_resize(trace_buf->buffer, 5182 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id); 5183 if (ret == 0) 5184 per_cpu_ptr(trace_buf->data, cpu_id)->entries = 5185 per_cpu_ptr(size_buf->data, cpu_id)->entries; 5186 } 5187 5188 return ret; 5189 } 5190 #endif /* CONFIG_TRACER_MAX_TRACE */ 5191 5192 static int __tracing_resize_ring_buffer(struct trace_array *tr, 5193 unsigned long size, int cpu) 5194 { 5195 int ret; 5196 5197 /* 5198 * If kernel or user changes the size of the ring buffer 5199 * we use the size that was given, and we can forget about 5200 * expanding it later. 5201 */ 5202 ring_buffer_expanded = true; 5203 5204 /* May be called before buffers are initialized */ 5205 if (!tr->trace_buffer.buffer) 5206 return 0; 5207 5208 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu); 5209 if (ret < 0) 5210 return ret; 5211 5212 #ifdef CONFIG_TRACER_MAX_TRACE 5213 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) || 5214 !tr->current_trace->use_max_tr) 5215 goto out; 5216 5217 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu); 5218 if (ret < 0) { 5219 int r = resize_buffer_duplicate_size(&tr->trace_buffer, 5220 &tr->trace_buffer, cpu); 5221 if (r < 0) { 5222 /* 5223 * AARGH! We are left with different 5224 * size max buffer!!!! 5225 * The max buffer is our "snapshot" buffer. 5226 * When a tracer needs a snapshot (one of the 5227 * latency tracers), it swaps the max buffer 5228 * with the saved snap shot. We succeeded to 5229 * update the size of the main buffer, but failed to 5230 * update the size of the max buffer. But when we tried 5231 * to reset the main buffer to the original size, we 5232 * failed there too. This is very unlikely to 5233 * happen, but if it does, warn and kill all 5234 * tracing. 5235 */ 5236 WARN_ON(1); 5237 tracing_disabled = 1; 5238 } 5239 return ret; 5240 } 5241 5242 if (cpu == RING_BUFFER_ALL_CPUS) 5243 set_buffer_entries(&tr->max_buffer, size); 5244 else 5245 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size; 5246 5247 out: 5248 #endif /* CONFIG_TRACER_MAX_TRACE */ 5249 5250 if (cpu == RING_BUFFER_ALL_CPUS) 5251 set_buffer_entries(&tr->trace_buffer, size); 5252 else 5253 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size; 5254 5255 return ret; 5256 } 5257 5258 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr, 5259 unsigned long size, int cpu_id) 5260 { 5261 int ret = size; 5262 5263 mutex_lock(&trace_types_lock); 5264 5265 if (cpu_id != RING_BUFFER_ALL_CPUS) { 5266 /* make sure, this cpu is enabled in the mask */ 5267 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) { 5268 ret = -EINVAL; 5269 goto out; 5270 } 5271 } 5272 5273 ret = __tracing_resize_ring_buffer(tr, size, cpu_id); 5274 if (ret < 0) 5275 ret = -ENOMEM; 5276 5277 out: 5278 mutex_unlock(&trace_types_lock); 5279 5280 return ret; 5281 } 5282 5283 5284 /** 5285 * tracing_update_buffers - used by tracing facility to expand ring buffers 5286 * 5287 * To save on memory when the tracing is never used on a system with it 5288 * configured in. The ring buffers are set to a minimum size. But once 5289 * a user starts to use the tracing facility, then they need to grow 5290 * to their default size. 5291 * 5292 * This function is to be called when a tracer is about to be used. 5293 */ 5294 int tracing_update_buffers(void) 5295 { 5296 int ret = 0; 5297 5298 mutex_lock(&trace_types_lock); 5299 if (!ring_buffer_expanded) 5300 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size, 5301 RING_BUFFER_ALL_CPUS); 5302 mutex_unlock(&trace_types_lock); 5303 5304 return ret; 5305 } 5306 5307 struct trace_option_dentry; 5308 5309 static void 5310 create_trace_option_files(struct trace_array *tr, struct tracer *tracer); 5311 5312 /* 5313 * Used to clear out the tracer before deletion of an instance. 5314 * Must have trace_types_lock held. 5315 */ 5316 static void tracing_set_nop(struct trace_array *tr) 5317 { 5318 if (tr->current_trace == &nop_trace) 5319 return; 5320 5321 tr->current_trace->enabled--; 5322 5323 if (tr->current_trace->reset) 5324 tr->current_trace->reset(tr); 5325 5326 tr->current_trace = &nop_trace; 5327 } 5328 5329 static void add_tracer_options(struct trace_array *tr, struct tracer *t) 5330 { 5331 /* Only enable if the directory has been created already. */ 5332 if (!tr->dir) 5333 return; 5334 5335 create_trace_option_files(tr, t); 5336 } 5337 5338 static int tracing_set_tracer(struct trace_array *tr, const char *buf) 5339 { 5340 struct tracer *t; 5341 #ifdef CONFIG_TRACER_MAX_TRACE 5342 bool had_max_tr; 5343 #endif 5344 int ret = 0; 5345 5346 mutex_lock(&trace_types_lock); 5347 5348 if (!ring_buffer_expanded) { 5349 ret = __tracing_resize_ring_buffer(tr, trace_buf_size, 5350 RING_BUFFER_ALL_CPUS); 5351 if (ret < 0) 5352 goto out; 5353 ret = 0; 5354 } 5355 5356 for (t = trace_types; t; t = t->next) { 5357 if (strcmp(t->name, buf) == 0) 5358 break; 5359 } 5360 if (!t) { 5361 ret = -EINVAL; 5362 goto out; 5363 } 5364 if (t == tr->current_trace) 5365 goto out; 5366 5367 /* Some tracers won't work on kernel command line */ 5368 if (system_state < SYSTEM_RUNNING && t->noboot) { 5369 pr_warn("Tracer '%s' is not allowed on command line, ignored\n", 5370 t->name); 5371 goto out; 5372 } 5373 5374 /* Some tracers are only allowed for the top level buffer */ 5375 if (!trace_ok_for_array(t, tr)) { 5376 ret = -EINVAL; 5377 goto out; 5378 } 5379 5380 /* If trace pipe files are being read, we can't change the tracer */ 5381 if (tr->current_trace->ref) { 5382 ret = -EBUSY; 5383 goto out; 5384 } 5385 5386 trace_branch_disable(); 5387 5388 tr->current_trace->enabled--; 5389 5390 if (tr->current_trace->reset) 5391 tr->current_trace->reset(tr); 5392 5393 /* Current trace needs to be nop_trace before synchronize_sched */ 5394 tr->current_trace = &nop_trace; 5395 5396 #ifdef CONFIG_TRACER_MAX_TRACE 5397 had_max_tr = tr->allocated_snapshot; 5398 5399 if (had_max_tr && !t->use_max_tr) { 5400 /* 5401 * We need to make sure that the update_max_tr sees that 5402 * current_trace changed to nop_trace to keep it from 5403 * swapping the buffers after we resize it. 5404 * The update_max_tr is called from interrupts disabled 5405 * so a synchronized_sched() is sufficient. 5406 */ 5407 synchronize_sched(); 5408 free_snapshot(tr); 5409 } 5410 #endif 5411 5412 #ifdef CONFIG_TRACER_MAX_TRACE 5413 if (t->use_max_tr && !had_max_tr) { 5414 ret = alloc_snapshot(tr); 5415 if (ret < 0) 5416 goto out; 5417 } 5418 #endif 5419 5420 if (t->init) { 5421 ret = tracer_init(t, tr); 5422 if (ret) 5423 goto out; 5424 } 5425 5426 tr->current_trace = t; 5427 tr->current_trace->enabled++; 5428 trace_branch_enable(tr); 5429 out: 5430 mutex_unlock(&trace_types_lock); 5431 5432 return ret; 5433 } 5434 5435 static ssize_t 5436 tracing_set_trace_write(struct file *filp, const char __user *ubuf, 5437 size_t cnt, loff_t *ppos) 5438 { 5439 struct trace_array *tr = filp->private_data; 5440 char buf[MAX_TRACER_SIZE+1]; 5441 int i; 5442 size_t ret; 5443 int err; 5444 5445 ret = cnt; 5446 5447 if (cnt > MAX_TRACER_SIZE) 5448 cnt = MAX_TRACER_SIZE; 5449 5450 if (copy_from_user(buf, ubuf, cnt)) 5451 return -EFAULT; 5452 5453 buf[cnt] = 0; 5454 5455 /* strip ending whitespace. */ 5456 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--) 5457 buf[i] = 0; 5458 5459 err = tracing_set_tracer(tr, buf); 5460 if (err) 5461 return err; 5462 5463 *ppos += ret; 5464 5465 return ret; 5466 } 5467 5468 static ssize_t 5469 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf, 5470 size_t cnt, loff_t *ppos) 5471 { 5472 char buf[64]; 5473 int r; 5474 5475 r = snprintf(buf, sizeof(buf), "%ld\n", 5476 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr)); 5477 if (r > sizeof(buf)) 5478 r = sizeof(buf); 5479 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5480 } 5481 5482 static ssize_t 5483 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf, 5484 size_t cnt, loff_t *ppos) 5485 { 5486 unsigned long val; 5487 int ret; 5488 5489 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 5490 if (ret) 5491 return ret; 5492 5493 *ptr = val * 1000; 5494 5495 return cnt; 5496 } 5497 5498 static ssize_t 5499 tracing_thresh_read(struct file *filp, char __user *ubuf, 5500 size_t cnt, loff_t *ppos) 5501 { 5502 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos); 5503 } 5504 5505 static ssize_t 5506 tracing_thresh_write(struct file *filp, const char __user *ubuf, 5507 size_t cnt, loff_t *ppos) 5508 { 5509 struct trace_array *tr = filp->private_data; 5510 int ret; 5511 5512 mutex_lock(&trace_types_lock); 5513 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos); 5514 if (ret < 0) 5515 goto out; 5516 5517 if (tr->current_trace->update_thresh) { 5518 ret = tr->current_trace->update_thresh(tr); 5519 if (ret < 0) 5520 goto out; 5521 } 5522 5523 ret = cnt; 5524 out: 5525 mutex_unlock(&trace_types_lock); 5526 5527 return ret; 5528 } 5529 5530 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) 5531 5532 static ssize_t 5533 tracing_max_lat_read(struct file *filp, char __user *ubuf, 5534 size_t cnt, loff_t *ppos) 5535 { 5536 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos); 5537 } 5538 5539 static ssize_t 5540 tracing_max_lat_write(struct file *filp, const char __user *ubuf, 5541 size_t cnt, loff_t *ppos) 5542 { 5543 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos); 5544 } 5545 5546 #endif 5547 5548 static int tracing_open_pipe(struct inode *inode, struct file *filp) 5549 { 5550 struct trace_array *tr = inode->i_private; 5551 struct trace_iterator *iter; 5552 int ret = 0; 5553 5554 if (tracing_disabled) 5555 return -ENODEV; 5556 5557 if (trace_array_get(tr) < 0) 5558 return -ENODEV; 5559 5560 mutex_lock(&trace_types_lock); 5561 5562 /* create a buffer to store the information to pass to userspace */ 5563 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 5564 if (!iter) { 5565 ret = -ENOMEM; 5566 __trace_array_put(tr); 5567 goto out; 5568 } 5569 5570 trace_seq_init(&iter->seq); 5571 iter->trace = tr->current_trace; 5572 5573 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) { 5574 ret = -ENOMEM; 5575 goto fail; 5576 } 5577 5578 /* trace pipe does not show start of buffer */ 5579 cpumask_setall(iter->started); 5580 5581 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 5582 iter->iter_flags |= TRACE_FILE_LAT_FMT; 5583 5584 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 5585 if (trace_clocks[tr->clock_id].in_ns) 5586 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 5587 5588 iter->tr = tr; 5589 iter->trace_buffer = &tr->trace_buffer; 5590 iter->cpu_file = tracing_get_cpu(inode); 5591 mutex_init(&iter->mutex); 5592 filp->private_data = iter; 5593 5594 if (iter->trace->pipe_open) 5595 iter->trace->pipe_open(iter); 5596 5597 nonseekable_open(inode, filp); 5598 5599 tr->current_trace->ref++; 5600 out: 5601 mutex_unlock(&trace_types_lock); 5602 return ret; 5603 5604 fail: 5605 kfree(iter->trace); 5606 kfree(iter); 5607 __trace_array_put(tr); 5608 mutex_unlock(&trace_types_lock); 5609 return ret; 5610 } 5611 5612 static int tracing_release_pipe(struct inode *inode, struct file *file) 5613 { 5614 struct trace_iterator *iter = file->private_data; 5615 struct trace_array *tr = inode->i_private; 5616 5617 mutex_lock(&trace_types_lock); 5618 5619 tr->current_trace->ref--; 5620 5621 if (iter->trace->pipe_close) 5622 iter->trace->pipe_close(iter); 5623 5624 mutex_unlock(&trace_types_lock); 5625 5626 free_cpumask_var(iter->started); 5627 mutex_destroy(&iter->mutex); 5628 kfree(iter); 5629 5630 trace_array_put(tr); 5631 5632 return 0; 5633 } 5634 5635 static unsigned int 5636 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table) 5637 { 5638 struct trace_array *tr = iter->tr; 5639 5640 /* Iterators are static, they should be filled or empty */ 5641 if (trace_buffer_iter(iter, iter->cpu_file)) 5642 return POLLIN | POLLRDNORM; 5643 5644 if (tr->trace_flags & TRACE_ITER_BLOCK) 5645 /* 5646 * Always select as readable when in blocking mode 5647 */ 5648 return POLLIN | POLLRDNORM; 5649 else 5650 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file, 5651 filp, poll_table); 5652 } 5653 5654 static unsigned int 5655 tracing_poll_pipe(struct file *filp, poll_table *poll_table) 5656 { 5657 struct trace_iterator *iter = filp->private_data; 5658 5659 return trace_poll(iter, filp, poll_table); 5660 } 5661 5662 /* Must be called with iter->mutex held. */ 5663 static int tracing_wait_pipe(struct file *filp) 5664 { 5665 struct trace_iterator *iter = filp->private_data; 5666 int ret; 5667 5668 while (trace_empty(iter)) { 5669 5670 if ((filp->f_flags & O_NONBLOCK)) { 5671 return -EAGAIN; 5672 } 5673 5674 /* 5675 * We block until we read something and tracing is disabled. 5676 * We still block if tracing is disabled, but we have never 5677 * read anything. This allows a user to cat this file, and 5678 * then enable tracing. But after we have read something, 5679 * we give an EOF when tracing is again disabled. 5680 * 5681 * iter->pos will be 0 if we haven't read anything. 5682 */ 5683 if (!tracer_tracing_is_on(iter->tr) && iter->pos) 5684 break; 5685 5686 mutex_unlock(&iter->mutex); 5687 5688 ret = wait_on_pipe(iter, false); 5689 5690 mutex_lock(&iter->mutex); 5691 5692 if (ret) 5693 return ret; 5694 } 5695 5696 return 1; 5697 } 5698 5699 /* 5700 * Consumer reader. 5701 */ 5702 static ssize_t 5703 tracing_read_pipe(struct file *filp, char __user *ubuf, 5704 size_t cnt, loff_t *ppos) 5705 { 5706 struct trace_iterator *iter = filp->private_data; 5707 ssize_t sret; 5708 5709 /* 5710 * Avoid more than one consumer on a single file descriptor 5711 * This is just a matter of traces coherency, the ring buffer itself 5712 * is protected. 5713 */ 5714 mutex_lock(&iter->mutex); 5715 5716 /* return any leftover data */ 5717 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 5718 if (sret != -EBUSY) 5719 goto out; 5720 5721 trace_seq_init(&iter->seq); 5722 5723 if (iter->trace->read) { 5724 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); 5725 if (sret) 5726 goto out; 5727 } 5728 5729 waitagain: 5730 sret = tracing_wait_pipe(filp); 5731 if (sret <= 0) 5732 goto out; 5733 5734 /* stop when tracing is finished */ 5735 if (trace_empty(iter)) { 5736 sret = 0; 5737 goto out; 5738 } 5739 5740 if (cnt >= PAGE_SIZE) 5741 cnt = PAGE_SIZE - 1; 5742 5743 /* reset all but tr, trace, and overruns */ 5744 memset(&iter->seq, 0, 5745 sizeof(struct trace_iterator) - 5746 offsetof(struct trace_iterator, seq)); 5747 cpumask_clear(iter->started); 5748 iter->pos = -1; 5749 5750 trace_event_read_lock(); 5751 trace_access_lock(iter->cpu_file); 5752 while (trace_find_next_entry_inc(iter) != NULL) { 5753 enum print_line_t ret; 5754 int save_len = iter->seq.seq.len; 5755 5756 ret = print_trace_line(iter); 5757 if (ret == TRACE_TYPE_PARTIAL_LINE) { 5758 /* don't print partial lines */ 5759 iter->seq.seq.len = save_len; 5760 break; 5761 } 5762 if (ret != TRACE_TYPE_NO_CONSUME) 5763 trace_consume(iter); 5764 5765 if (trace_seq_used(&iter->seq) >= cnt) 5766 break; 5767 5768 /* 5769 * Setting the full flag means we reached the trace_seq buffer 5770 * size and we should leave by partial output condition above. 5771 * One of the trace_seq_* functions is not used properly. 5772 */ 5773 WARN_ONCE(iter->seq.full, "full flag set for trace type %d", 5774 iter->ent->type); 5775 } 5776 trace_access_unlock(iter->cpu_file); 5777 trace_event_read_unlock(); 5778 5779 /* Now copy what we have to the user */ 5780 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 5781 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq)) 5782 trace_seq_init(&iter->seq); 5783 5784 /* 5785 * If there was nothing to send to user, in spite of consuming trace 5786 * entries, go back to wait for more entries. 5787 */ 5788 if (sret == -EBUSY) 5789 goto waitagain; 5790 5791 out: 5792 mutex_unlock(&iter->mutex); 5793 5794 return sret; 5795 } 5796 5797 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd, 5798 unsigned int idx) 5799 { 5800 __free_page(spd->pages[idx]); 5801 } 5802 5803 static const struct pipe_buf_operations tracing_pipe_buf_ops = { 5804 .can_merge = 0, 5805 .confirm = generic_pipe_buf_confirm, 5806 .release = generic_pipe_buf_release, 5807 .steal = generic_pipe_buf_steal, 5808 .get = generic_pipe_buf_get, 5809 }; 5810 5811 static size_t 5812 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter) 5813 { 5814 size_t count; 5815 int save_len; 5816 int ret; 5817 5818 /* Seq buffer is page-sized, exactly what we need. */ 5819 for (;;) { 5820 save_len = iter->seq.seq.len; 5821 ret = print_trace_line(iter); 5822 5823 if (trace_seq_has_overflowed(&iter->seq)) { 5824 iter->seq.seq.len = save_len; 5825 break; 5826 } 5827 5828 /* 5829 * This should not be hit, because it should only 5830 * be set if the iter->seq overflowed. But check it 5831 * anyway to be safe. 5832 */ 5833 if (ret == TRACE_TYPE_PARTIAL_LINE) { 5834 iter->seq.seq.len = save_len; 5835 break; 5836 } 5837 5838 count = trace_seq_used(&iter->seq) - save_len; 5839 if (rem < count) { 5840 rem = 0; 5841 iter->seq.seq.len = save_len; 5842 break; 5843 } 5844 5845 if (ret != TRACE_TYPE_NO_CONSUME) 5846 trace_consume(iter); 5847 rem -= count; 5848 if (!trace_find_next_entry_inc(iter)) { 5849 rem = 0; 5850 iter->ent = NULL; 5851 break; 5852 } 5853 } 5854 5855 return rem; 5856 } 5857 5858 static ssize_t tracing_splice_read_pipe(struct file *filp, 5859 loff_t *ppos, 5860 struct pipe_inode_info *pipe, 5861 size_t len, 5862 unsigned int flags) 5863 { 5864 struct page *pages_def[PIPE_DEF_BUFFERS]; 5865 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 5866 struct trace_iterator *iter = filp->private_data; 5867 struct splice_pipe_desc spd = { 5868 .pages = pages_def, 5869 .partial = partial_def, 5870 .nr_pages = 0, /* This gets updated below. */ 5871 .nr_pages_max = PIPE_DEF_BUFFERS, 5872 .ops = &tracing_pipe_buf_ops, 5873 .spd_release = tracing_spd_release_pipe, 5874 }; 5875 ssize_t ret; 5876 size_t rem; 5877 unsigned int i; 5878 5879 if (splice_grow_spd(pipe, &spd)) 5880 return -ENOMEM; 5881 5882 mutex_lock(&iter->mutex); 5883 5884 if (iter->trace->splice_read) { 5885 ret = iter->trace->splice_read(iter, filp, 5886 ppos, pipe, len, flags); 5887 if (ret) 5888 goto out_err; 5889 } 5890 5891 ret = tracing_wait_pipe(filp); 5892 if (ret <= 0) 5893 goto out_err; 5894 5895 if (!iter->ent && !trace_find_next_entry_inc(iter)) { 5896 ret = -EFAULT; 5897 goto out_err; 5898 } 5899 5900 trace_event_read_lock(); 5901 trace_access_lock(iter->cpu_file); 5902 5903 /* Fill as many pages as possible. */ 5904 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) { 5905 spd.pages[i] = alloc_page(GFP_KERNEL); 5906 if (!spd.pages[i]) 5907 break; 5908 5909 rem = tracing_fill_pipe_page(rem, iter); 5910 5911 /* Copy the data into the page, so we can start over. */ 5912 ret = trace_seq_to_buffer(&iter->seq, 5913 page_address(spd.pages[i]), 5914 trace_seq_used(&iter->seq)); 5915 if (ret < 0) { 5916 __free_page(spd.pages[i]); 5917 break; 5918 } 5919 spd.partial[i].offset = 0; 5920 spd.partial[i].len = trace_seq_used(&iter->seq); 5921 5922 trace_seq_init(&iter->seq); 5923 } 5924 5925 trace_access_unlock(iter->cpu_file); 5926 trace_event_read_unlock(); 5927 mutex_unlock(&iter->mutex); 5928 5929 spd.nr_pages = i; 5930 5931 if (i) 5932 ret = splice_to_pipe(pipe, &spd); 5933 else 5934 ret = 0; 5935 out: 5936 splice_shrink_spd(&spd); 5937 return ret; 5938 5939 out_err: 5940 mutex_unlock(&iter->mutex); 5941 goto out; 5942 } 5943 5944 static ssize_t 5945 tracing_entries_read(struct file *filp, char __user *ubuf, 5946 size_t cnt, loff_t *ppos) 5947 { 5948 struct inode *inode = file_inode(filp); 5949 struct trace_array *tr = inode->i_private; 5950 int cpu = tracing_get_cpu(inode); 5951 char buf[64]; 5952 int r = 0; 5953 ssize_t ret; 5954 5955 mutex_lock(&trace_types_lock); 5956 5957 if (cpu == RING_BUFFER_ALL_CPUS) { 5958 int cpu, buf_size_same; 5959 unsigned long size; 5960 5961 size = 0; 5962 buf_size_same = 1; 5963 /* check if all cpu sizes are same */ 5964 for_each_tracing_cpu(cpu) { 5965 /* fill in the size from first enabled cpu */ 5966 if (size == 0) 5967 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries; 5968 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) { 5969 buf_size_same = 0; 5970 break; 5971 } 5972 } 5973 5974 if (buf_size_same) { 5975 if (!ring_buffer_expanded) 5976 r = sprintf(buf, "%lu (expanded: %lu)\n", 5977 size >> 10, 5978 trace_buf_size >> 10); 5979 else 5980 r = sprintf(buf, "%lu\n", size >> 10); 5981 } else 5982 r = sprintf(buf, "X\n"); 5983 } else 5984 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10); 5985 5986 mutex_unlock(&trace_types_lock); 5987 5988 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5989 return ret; 5990 } 5991 5992 static ssize_t 5993 tracing_entries_write(struct file *filp, const char __user *ubuf, 5994 size_t cnt, loff_t *ppos) 5995 { 5996 struct inode *inode = file_inode(filp); 5997 struct trace_array *tr = inode->i_private; 5998 unsigned long val; 5999 int ret; 6000 6001 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 6002 if (ret) 6003 return ret; 6004 6005 /* must have at least 1 entry */ 6006 if (!val) 6007 return -EINVAL; 6008 6009 /* value is in KB */ 6010 val <<= 10; 6011 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode)); 6012 if (ret < 0) 6013 return ret; 6014 6015 *ppos += cnt; 6016 6017 return cnt; 6018 } 6019 6020 static ssize_t 6021 tracing_total_entries_read(struct file *filp, char __user *ubuf, 6022 size_t cnt, loff_t *ppos) 6023 { 6024 struct trace_array *tr = filp->private_data; 6025 char buf[64]; 6026 int r, cpu; 6027 unsigned long size = 0, expanded_size = 0; 6028 6029 mutex_lock(&trace_types_lock); 6030 for_each_tracing_cpu(cpu) { 6031 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10; 6032 if (!ring_buffer_expanded) 6033 expanded_size += trace_buf_size >> 10; 6034 } 6035 if (ring_buffer_expanded) 6036 r = sprintf(buf, "%lu\n", size); 6037 else 6038 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size); 6039 mutex_unlock(&trace_types_lock); 6040 6041 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 6042 } 6043 6044 static ssize_t 6045 tracing_free_buffer_write(struct file *filp, const char __user *ubuf, 6046 size_t cnt, loff_t *ppos) 6047 { 6048 /* 6049 * There is no need to read what the user has written, this function 6050 * is just to make sure that there is no error when "echo" is used 6051 */ 6052 6053 *ppos += cnt; 6054 6055 return cnt; 6056 } 6057 6058 static int 6059 tracing_free_buffer_release(struct inode *inode, struct file *filp) 6060 { 6061 struct trace_array *tr = inode->i_private; 6062 6063 /* disable tracing ? */ 6064 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE) 6065 tracer_tracing_off(tr); 6066 /* resize the ring buffer to 0 */ 6067 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS); 6068 6069 trace_array_put(tr); 6070 6071 return 0; 6072 } 6073 6074 static ssize_t 6075 tracing_mark_write(struct file *filp, const char __user *ubuf, 6076 size_t cnt, loff_t *fpos) 6077 { 6078 struct trace_array *tr = filp->private_data; 6079 struct ring_buffer_event *event; 6080 struct ring_buffer *buffer; 6081 struct print_entry *entry; 6082 unsigned long irq_flags; 6083 const char faulted[] = "<faulted>"; 6084 ssize_t written; 6085 int size; 6086 int len; 6087 6088 /* Used in tracing_mark_raw_write() as well */ 6089 #define FAULTED_SIZE (sizeof(faulted) - 1) /* '\0' is already accounted for */ 6090 6091 if (tracing_disabled) 6092 return -EINVAL; 6093 6094 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) 6095 return -EINVAL; 6096 6097 if (cnt > TRACE_BUF_SIZE) 6098 cnt = TRACE_BUF_SIZE; 6099 6100 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); 6101 6102 local_save_flags(irq_flags); 6103 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */ 6104 6105 /* If less than "<faulted>", then make sure we can still add that */ 6106 if (cnt < FAULTED_SIZE) 6107 size += FAULTED_SIZE - cnt; 6108 6109 buffer = tr->trace_buffer.buffer; 6110 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 6111 irq_flags, preempt_count()); 6112 if (unlikely(!event)) 6113 /* Ring buffer disabled, return as if not open for write */ 6114 return -EBADF; 6115 6116 entry = ring_buffer_event_data(event); 6117 entry->ip = _THIS_IP_; 6118 6119 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt); 6120 if (len) { 6121 memcpy(&entry->buf, faulted, FAULTED_SIZE); 6122 cnt = FAULTED_SIZE; 6123 written = -EFAULT; 6124 } else 6125 written = cnt; 6126 len = cnt; 6127 6128 if (entry->buf[cnt - 1] != '\n') { 6129 entry->buf[cnt] = '\n'; 6130 entry->buf[cnt + 1] = '\0'; 6131 } else 6132 entry->buf[cnt] = '\0'; 6133 6134 __buffer_unlock_commit(buffer, event); 6135 6136 if (written > 0) 6137 *fpos += written; 6138 6139 return written; 6140 } 6141 6142 /* Limit it for now to 3K (including tag) */ 6143 #define RAW_DATA_MAX_SIZE (1024*3) 6144 6145 static ssize_t 6146 tracing_mark_raw_write(struct file *filp, const char __user *ubuf, 6147 size_t cnt, loff_t *fpos) 6148 { 6149 struct trace_array *tr = filp->private_data; 6150 struct ring_buffer_event *event; 6151 struct ring_buffer *buffer; 6152 struct raw_data_entry *entry; 6153 const char faulted[] = "<faulted>"; 6154 unsigned long irq_flags; 6155 ssize_t written; 6156 int size; 6157 int len; 6158 6159 #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int)) 6160 6161 if (tracing_disabled) 6162 return -EINVAL; 6163 6164 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) 6165 return -EINVAL; 6166 6167 /* The marker must at least have a tag id */ 6168 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE) 6169 return -EINVAL; 6170 6171 if (cnt > TRACE_BUF_SIZE) 6172 cnt = TRACE_BUF_SIZE; 6173 6174 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); 6175 6176 local_save_flags(irq_flags); 6177 size = sizeof(*entry) + cnt; 6178 if (cnt < FAULT_SIZE_ID) 6179 size += FAULT_SIZE_ID - cnt; 6180 6181 buffer = tr->trace_buffer.buffer; 6182 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size, 6183 irq_flags, preempt_count()); 6184 if (!event) 6185 /* Ring buffer disabled, return as if not open for write */ 6186 return -EBADF; 6187 6188 entry = ring_buffer_event_data(event); 6189 6190 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt); 6191 if (len) { 6192 entry->id = -1; 6193 memcpy(&entry->buf, faulted, FAULTED_SIZE); 6194 written = -EFAULT; 6195 } else 6196 written = cnt; 6197 6198 __buffer_unlock_commit(buffer, event); 6199 6200 if (written > 0) 6201 *fpos += written; 6202 6203 return written; 6204 } 6205 6206 static int tracing_clock_show(struct seq_file *m, void *v) 6207 { 6208 struct trace_array *tr = m->private; 6209 int i; 6210 6211 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) 6212 seq_printf(m, 6213 "%s%s%s%s", i ? " " : "", 6214 i == tr->clock_id ? "[" : "", trace_clocks[i].name, 6215 i == tr->clock_id ? "]" : ""); 6216 seq_putc(m, '\n'); 6217 6218 return 0; 6219 } 6220 6221 static int tracing_set_clock(struct trace_array *tr, const char *clockstr) 6222 { 6223 int i; 6224 6225 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) { 6226 if (strcmp(trace_clocks[i].name, clockstr) == 0) 6227 break; 6228 } 6229 if (i == ARRAY_SIZE(trace_clocks)) 6230 return -EINVAL; 6231 6232 mutex_lock(&trace_types_lock); 6233 6234 tr->clock_id = i; 6235 6236 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func); 6237 6238 /* 6239 * New clock may not be consistent with the previous clock. 6240 * Reset the buffer so that it doesn't have incomparable timestamps. 6241 */ 6242 tracing_reset_online_cpus(&tr->trace_buffer); 6243 6244 #ifdef CONFIG_TRACER_MAX_TRACE 6245 if (tr->max_buffer.buffer) 6246 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func); 6247 tracing_reset_online_cpus(&tr->max_buffer); 6248 #endif 6249 6250 mutex_unlock(&trace_types_lock); 6251 6252 return 0; 6253 } 6254 6255 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf, 6256 size_t cnt, loff_t *fpos) 6257 { 6258 struct seq_file *m = filp->private_data; 6259 struct trace_array *tr = m->private; 6260 char buf[64]; 6261 const char *clockstr; 6262 int ret; 6263 6264 if (cnt >= sizeof(buf)) 6265 return -EINVAL; 6266 6267 if (copy_from_user(buf, ubuf, cnt)) 6268 return -EFAULT; 6269 6270 buf[cnt] = 0; 6271 6272 clockstr = strstrip(buf); 6273 6274 ret = tracing_set_clock(tr, clockstr); 6275 if (ret) 6276 return ret; 6277 6278 *fpos += cnt; 6279 6280 return cnt; 6281 } 6282 6283 static int tracing_clock_open(struct inode *inode, struct file *file) 6284 { 6285 struct trace_array *tr = inode->i_private; 6286 int ret; 6287 6288 if (tracing_disabled) 6289 return -ENODEV; 6290 6291 if (trace_array_get(tr)) 6292 return -ENODEV; 6293 6294 ret = single_open(file, tracing_clock_show, inode->i_private); 6295 if (ret < 0) 6296 trace_array_put(tr); 6297 6298 return ret; 6299 } 6300 6301 struct ftrace_buffer_info { 6302 struct trace_iterator iter; 6303 void *spare; 6304 unsigned int spare_cpu; 6305 unsigned int read; 6306 }; 6307 6308 #ifdef CONFIG_TRACER_SNAPSHOT 6309 static int tracing_snapshot_open(struct inode *inode, struct file *file) 6310 { 6311 struct trace_array *tr = inode->i_private; 6312 struct trace_iterator *iter; 6313 struct seq_file *m; 6314 int ret = 0; 6315 6316 if (trace_array_get(tr) < 0) 6317 return -ENODEV; 6318 6319 if (file->f_mode & FMODE_READ) { 6320 iter = __tracing_open(inode, file, true); 6321 if (IS_ERR(iter)) 6322 ret = PTR_ERR(iter); 6323 } else { 6324 /* Writes still need the seq_file to hold the private data */ 6325 ret = -ENOMEM; 6326 m = kzalloc(sizeof(*m), GFP_KERNEL); 6327 if (!m) 6328 goto out; 6329 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 6330 if (!iter) { 6331 kfree(m); 6332 goto out; 6333 } 6334 ret = 0; 6335 6336 iter->tr = tr; 6337 iter->trace_buffer = &tr->max_buffer; 6338 iter->cpu_file = tracing_get_cpu(inode); 6339 m->private = iter; 6340 file->private_data = m; 6341 } 6342 out: 6343 if (ret < 0) 6344 trace_array_put(tr); 6345 6346 return ret; 6347 } 6348 6349 static ssize_t 6350 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt, 6351 loff_t *ppos) 6352 { 6353 struct seq_file *m = filp->private_data; 6354 struct trace_iterator *iter = m->private; 6355 struct trace_array *tr = iter->tr; 6356 unsigned long val; 6357 int ret; 6358 6359 ret = tracing_update_buffers(); 6360 if (ret < 0) 6361 return ret; 6362 6363 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 6364 if (ret) 6365 return ret; 6366 6367 mutex_lock(&trace_types_lock); 6368 6369 if (tr->current_trace->use_max_tr) { 6370 ret = -EBUSY; 6371 goto out; 6372 } 6373 6374 switch (val) { 6375 case 0: 6376 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 6377 ret = -EINVAL; 6378 break; 6379 } 6380 if (tr->allocated_snapshot) 6381 free_snapshot(tr); 6382 break; 6383 case 1: 6384 /* Only allow per-cpu swap if the ring buffer supports it */ 6385 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP 6386 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 6387 ret = -EINVAL; 6388 break; 6389 } 6390 #endif 6391 if (!tr->allocated_snapshot) { 6392 ret = alloc_snapshot(tr); 6393 if (ret < 0) 6394 break; 6395 } 6396 local_irq_disable(); 6397 /* Now, we're going to swap */ 6398 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 6399 update_max_tr(tr, current, smp_processor_id()); 6400 else 6401 update_max_tr_single(tr, current, iter->cpu_file); 6402 local_irq_enable(); 6403 break; 6404 default: 6405 if (tr->allocated_snapshot) { 6406 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 6407 tracing_reset_online_cpus(&tr->max_buffer); 6408 else 6409 tracing_reset(&tr->max_buffer, iter->cpu_file); 6410 } 6411 break; 6412 } 6413 6414 if (ret >= 0) { 6415 *ppos += cnt; 6416 ret = cnt; 6417 } 6418 out: 6419 mutex_unlock(&trace_types_lock); 6420 return ret; 6421 } 6422 6423 static int tracing_snapshot_release(struct inode *inode, struct file *file) 6424 { 6425 struct seq_file *m = file->private_data; 6426 int ret; 6427 6428 ret = tracing_release(inode, file); 6429 6430 if (file->f_mode & FMODE_READ) 6431 return ret; 6432 6433 /* If write only, the seq_file is just a stub */ 6434 if (m) 6435 kfree(m->private); 6436 kfree(m); 6437 6438 return 0; 6439 } 6440 6441 static int tracing_buffers_open(struct inode *inode, struct file *filp); 6442 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, 6443 size_t count, loff_t *ppos); 6444 static int tracing_buffers_release(struct inode *inode, struct file *file); 6445 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos, 6446 struct pipe_inode_info *pipe, size_t len, unsigned int flags); 6447 6448 static int snapshot_raw_open(struct inode *inode, struct file *filp) 6449 { 6450 struct ftrace_buffer_info *info; 6451 int ret; 6452 6453 ret = tracing_buffers_open(inode, filp); 6454 if (ret < 0) 6455 return ret; 6456 6457 info = filp->private_data; 6458 6459 if (info->iter.trace->use_max_tr) { 6460 tracing_buffers_release(inode, filp); 6461 return -EBUSY; 6462 } 6463 6464 info->iter.snapshot = true; 6465 info->iter.trace_buffer = &info->iter.tr->max_buffer; 6466 6467 return ret; 6468 } 6469 6470 #endif /* CONFIG_TRACER_SNAPSHOT */ 6471 6472 6473 static const struct file_operations tracing_thresh_fops = { 6474 .open = tracing_open_generic, 6475 .read = tracing_thresh_read, 6476 .write = tracing_thresh_write, 6477 .llseek = generic_file_llseek, 6478 }; 6479 6480 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) 6481 static const struct file_operations tracing_max_lat_fops = { 6482 .open = tracing_open_generic, 6483 .read = tracing_max_lat_read, 6484 .write = tracing_max_lat_write, 6485 .llseek = generic_file_llseek, 6486 }; 6487 #endif 6488 6489 static const struct file_operations set_tracer_fops = { 6490 .open = tracing_open_generic, 6491 .read = tracing_set_trace_read, 6492 .write = tracing_set_trace_write, 6493 .llseek = generic_file_llseek, 6494 }; 6495 6496 static const struct file_operations tracing_pipe_fops = { 6497 .open = tracing_open_pipe, 6498 .poll = tracing_poll_pipe, 6499 .read = tracing_read_pipe, 6500 .splice_read = tracing_splice_read_pipe, 6501 .release = tracing_release_pipe, 6502 .llseek = no_llseek, 6503 }; 6504 6505 static const struct file_operations tracing_entries_fops = { 6506 .open = tracing_open_generic_tr, 6507 .read = tracing_entries_read, 6508 .write = tracing_entries_write, 6509 .llseek = generic_file_llseek, 6510 .release = tracing_release_generic_tr, 6511 }; 6512 6513 static const struct file_operations tracing_total_entries_fops = { 6514 .open = tracing_open_generic_tr, 6515 .read = tracing_total_entries_read, 6516 .llseek = generic_file_llseek, 6517 .release = tracing_release_generic_tr, 6518 }; 6519 6520 static const struct file_operations tracing_free_buffer_fops = { 6521 .open = tracing_open_generic_tr, 6522 .write = tracing_free_buffer_write, 6523 .release = tracing_free_buffer_release, 6524 }; 6525 6526 static const struct file_operations tracing_mark_fops = { 6527 .open = tracing_open_generic_tr, 6528 .write = tracing_mark_write, 6529 .llseek = generic_file_llseek, 6530 .release = tracing_release_generic_tr, 6531 }; 6532 6533 static const struct file_operations tracing_mark_raw_fops = { 6534 .open = tracing_open_generic_tr, 6535 .write = tracing_mark_raw_write, 6536 .llseek = generic_file_llseek, 6537 .release = tracing_release_generic_tr, 6538 }; 6539 6540 static const struct file_operations trace_clock_fops = { 6541 .open = tracing_clock_open, 6542 .read = seq_read, 6543 .llseek = seq_lseek, 6544 .release = tracing_single_release_tr, 6545 .write = tracing_clock_write, 6546 }; 6547 6548 #ifdef CONFIG_TRACER_SNAPSHOT 6549 static const struct file_operations snapshot_fops = { 6550 .open = tracing_snapshot_open, 6551 .read = seq_read, 6552 .write = tracing_snapshot_write, 6553 .llseek = tracing_lseek, 6554 .release = tracing_snapshot_release, 6555 }; 6556 6557 static const struct file_operations snapshot_raw_fops = { 6558 .open = snapshot_raw_open, 6559 .read = tracing_buffers_read, 6560 .release = tracing_buffers_release, 6561 .splice_read = tracing_buffers_splice_read, 6562 .llseek = no_llseek, 6563 }; 6564 6565 #endif /* CONFIG_TRACER_SNAPSHOT */ 6566 6567 static int tracing_buffers_open(struct inode *inode, struct file *filp) 6568 { 6569 struct trace_array *tr = inode->i_private; 6570 struct ftrace_buffer_info *info; 6571 int ret; 6572 6573 if (tracing_disabled) 6574 return -ENODEV; 6575 6576 if (trace_array_get(tr) < 0) 6577 return -ENODEV; 6578 6579 info = kzalloc(sizeof(*info), GFP_KERNEL); 6580 if (!info) { 6581 trace_array_put(tr); 6582 return -ENOMEM; 6583 } 6584 6585 mutex_lock(&trace_types_lock); 6586 6587 info->iter.tr = tr; 6588 info->iter.cpu_file = tracing_get_cpu(inode); 6589 info->iter.trace = tr->current_trace; 6590 info->iter.trace_buffer = &tr->trace_buffer; 6591 info->spare = NULL; 6592 /* Force reading ring buffer for first read */ 6593 info->read = (unsigned int)-1; 6594 6595 filp->private_data = info; 6596 6597 tr->current_trace->ref++; 6598 6599 mutex_unlock(&trace_types_lock); 6600 6601 ret = nonseekable_open(inode, filp); 6602 if (ret < 0) 6603 trace_array_put(tr); 6604 6605 return ret; 6606 } 6607 6608 static unsigned int 6609 tracing_buffers_poll(struct file *filp, poll_table *poll_table) 6610 { 6611 struct ftrace_buffer_info *info = filp->private_data; 6612 struct trace_iterator *iter = &info->iter; 6613 6614 return trace_poll(iter, filp, poll_table); 6615 } 6616 6617 static ssize_t 6618 tracing_buffers_read(struct file *filp, char __user *ubuf, 6619 size_t count, loff_t *ppos) 6620 { 6621 struct ftrace_buffer_info *info = filp->private_data; 6622 struct trace_iterator *iter = &info->iter; 6623 ssize_t ret = 0; 6624 ssize_t size; 6625 6626 if (!count) 6627 return 0; 6628 6629 #ifdef CONFIG_TRACER_MAX_TRACE 6630 if (iter->snapshot && iter->tr->current_trace->use_max_tr) 6631 return -EBUSY; 6632 #endif 6633 6634 if (!info->spare) { 6635 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer, 6636 iter->cpu_file); 6637 if (IS_ERR(info->spare)) { 6638 ret = PTR_ERR(info->spare); 6639 info->spare = NULL; 6640 } else { 6641 info->spare_cpu = iter->cpu_file; 6642 } 6643 } 6644 if (!info->spare) 6645 return ret; 6646 6647 /* Do we have previous read data to read? */ 6648 if (info->read < PAGE_SIZE) 6649 goto read; 6650 6651 again: 6652 trace_access_lock(iter->cpu_file); 6653 ret = ring_buffer_read_page(iter->trace_buffer->buffer, 6654 &info->spare, 6655 count, 6656 iter->cpu_file, 0); 6657 trace_access_unlock(iter->cpu_file); 6658 6659 if (ret < 0) { 6660 if (trace_empty(iter)) { 6661 if ((filp->f_flags & O_NONBLOCK)) 6662 return -EAGAIN; 6663 6664 ret = wait_on_pipe(iter, false); 6665 if (ret) 6666 return ret; 6667 6668 goto again; 6669 } 6670 return 0; 6671 } 6672 6673 info->read = 0; 6674 read: 6675 size = PAGE_SIZE - info->read; 6676 if (size > count) 6677 size = count; 6678 6679 ret = copy_to_user(ubuf, info->spare + info->read, size); 6680 if (ret == size) 6681 return -EFAULT; 6682 6683 size -= ret; 6684 6685 *ppos += size; 6686 info->read += size; 6687 6688 return size; 6689 } 6690 6691 static int tracing_buffers_release(struct inode *inode, struct file *file) 6692 { 6693 struct ftrace_buffer_info *info = file->private_data; 6694 struct trace_iterator *iter = &info->iter; 6695 6696 mutex_lock(&trace_types_lock); 6697 6698 iter->tr->current_trace->ref--; 6699 6700 __trace_array_put(iter->tr); 6701 6702 if (info->spare) 6703 ring_buffer_free_read_page(iter->trace_buffer->buffer, 6704 info->spare_cpu, info->spare); 6705 kfree(info); 6706 6707 mutex_unlock(&trace_types_lock); 6708 6709 return 0; 6710 } 6711 6712 struct buffer_ref { 6713 struct ring_buffer *buffer; 6714 void *page; 6715 int cpu; 6716 int ref; 6717 }; 6718 6719 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe, 6720 struct pipe_buffer *buf) 6721 { 6722 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 6723 6724 if (--ref->ref) 6725 return; 6726 6727 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page); 6728 kfree(ref); 6729 buf->private = 0; 6730 } 6731 6732 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe, 6733 struct pipe_buffer *buf) 6734 { 6735 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 6736 6737 ref->ref++; 6738 } 6739 6740 /* Pipe buffer operations for a buffer. */ 6741 static const struct pipe_buf_operations buffer_pipe_buf_ops = { 6742 .can_merge = 0, 6743 .confirm = generic_pipe_buf_confirm, 6744 .release = buffer_pipe_buf_release, 6745 .steal = generic_pipe_buf_steal, 6746 .get = buffer_pipe_buf_get, 6747 }; 6748 6749 /* 6750 * Callback from splice_to_pipe(), if we need to release some pages 6751 * at the end of the spd in case we error'ed out in filling the pipe. 6752 */ 6753 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i) 6754 { 6755 struct buffer_ref *ref = 6756 (struct buffer_ref *)spd->partial[i].private; 6757 6758 if (--ref->ref) 6759 return; 6760 6761 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page); 6762 kfree(ref); 6763 spd->partial[i].private = 0; 6764 } 6765 6766 static ssize_t 6767 tracing_buffers_splice_read(struct file *file, loff_t *ppos, 6768 struct pipe_inode_info *pipe, size_t len, 6769 unsigned int flags) 6770 { 6771 struct ftrace_buffer_info *info = file->private_data; 6772 struct trace_iterator *iter = &info->iter; 6773 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 6774 struct page *pages_def[PIPE_DEF_BUFFERS]; 6775 struct splice_pipe_desc spd = { 6776 .pages = pages_def, 6777 .partial = partial_def, 6778 .nr_pages_max = PIPE_DEF_BUFFERS, 6779 .ops = &buffer_pipe_buf_ops, 6780 .spd_release = buffer_spd_release, 6781 }; 6782 struct buffer_ref *ref; 6783 int entries, size, i; 6784 ssize_t ret = 0; 6785 6786 #ifdef CONFIG_TRACER_MAX_TRACE 6787 if (iter->snapshot && iter->tr->current_trace->use_max_tr) 6788 return -EBUSY; 6789 #endif 6790 6791 if (*ppos & (PAGE_SIZE - 1)) 6792 return -EINVAL; 6793 6794 if (len & (PAGE_SIZE - 1)) { 6795 if (len < PAGE_SIZE) 6796 return -EINVAL; 6797 len &= PAGE_MASK; 6798 } 6799 6800 if (splice_grow_spd(pipe, &spd)) 6801 return -ENOMEM; 6802 6803 again: 6804 trace_access_lock(iter->cpu_file); 6805 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file); 6806 6807 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) { 6808 struct page *page; 6809 int r; 6810 6811 ref = kzalloc(sizeof(*ref), GFP_KERNEL); 6812 if (!ref) { 6813 ret = -ENOMEM; 6814 break; 6815 } 6816 6817 ref->ref = 1; 6818 ref->buffer = iter->trace_buffer->buffer; 6819 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file); 6820 if (IS_ERR(ref->page)) { 6821 ret = PTR_ERR(ref->page); 6822 ref->page = NULL; 6823 kfree(ref); 6824 break; 6825 } 6826 ref->cpu = iter->cpu_file; 6827 6828 r = ring_buffer_read_page(ref->buffer, &ref->page, 6829 len, iter->cpu_file, 1); 6830 if (r < 0) { 6831 ring_buffer_free_read_page(ref->buffer, ref->cpu, 6832 ref->page); 6833 kfree(ref); 6834 break; 6835 } 6836 6837 /* 6838 * zero out any left over data, this is going to 6839 * user land. 6840 */ 6841 size = ring_buffer_page_len(ref->page); 6842 if (size < PAGE_SIZE) 6843 memset(ref->page + size, 0, PAGE_SIZE - size); 6844 6845 page = virt_to_page(ref->page); 6846 6847 spd.pages[i] = page; 6848 spd.partial[i].len = PAGE_SIZE; 6849 spd.partial[i].offset = 0; 6850 spd.partial[i].private = (unsigned long)ref; 6851 spd.nr_pages++; 6852 *ppos += PAGE_SIZE; 6853 6854 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file); 6855 } 6856 6857 trace_access_unlock(iter->cpu_file); 6858 spd.nr_pages = i; 6859 6860 /* did we read anything? */ 6861 if (!spd.nr_pages) { 6862 if (ret) 6863 goto out; 6864 6865 ret = -EAGAIN; 6866 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) 6867 goto out; 6868 6869 ret = wait_on_pipe(iter, true); 6870 if (ret) 6871 goto out; 6872 6873 goto again; 6874 } 6875 6876 ret = splice_to_pipe(pipe, &spd); 6877 out: 6878 splice_shrink_spd(&spd); 6879 6880 return ret; 6881 } 6882 6883 static const struct file_operations tracing_buffers_fops = { 6884 .open = tracing_buffers_open, 6885 .read = tracing_buffers_read, 6886 .poll = tracing_buffers_poll, 6887 .release = tracing_buffers_release, 6888 .splice_read = tracing_buffers_splice_read, 6889 .llseek = no_llseek, 6890 }; 6891 6892 static ssize_t 6893 tracing_stats_read(struct file *filp, char __user *ubuf, 6894 size_t count, loff_t *ppos) 6895 { 6896 struct inode *inode = file_inode(filp); 6897 struct trace_array *tr = inode->i_private; 6898 struct trace_buffer *trace_buf = &tr->trace_buffer; 6899 int cpu = tracing_get_cpu(inode); 6900 struct trace_seq *s; 6901 unsigned long cnt; 6902 unsigned long long t; 6903 unsigned long usec_rem; 6904 6905 s = kmalloc(sizeof(*s), GFP_KERNEL); 6906 if (!s) 6907 return -ENOMEM; 6908 6909 trace_seq_init(s); 6910 6911 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu); 6912 trace_seq_printf(s, "entries: %ld\n", cnt); 6913 6914 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu); 6915 trace_seq_printf(s, "overrun: %ld\n", cnt); 6916 6917 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu); 6918 trace_seq_printf(s, "commit overrun: %ld\n", cnt); 6919 6920 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu); 6921 trace_seq_printf(s, "bytes: %ld\n", cnt); 6922 6923 if (trace_clocks[tr->clock_id].in_ns) { 6924 /* local or global for trace_clock */ 6925 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); 6926 usec_rem = do_div(t, USEC_PER_SEC); 6927 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", 6928 t, usec_rem); 6929 6930 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu)); 6931 usec_rem = do_div(t, USEC_PER_SEC); 6932 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem); 6933 } else { 6934 /* counter or tsc mode for trace_clock */ 6935 trace_seq_printf(s, "oldest event ts: %llu\n", 6936 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); 6937 6938 trace_seq_printf(s, "now ts: %llu\n", 6939 ring_buffer_time_stamp(trace_buf->buffer, cpu)); 6940 } 6941 6942 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu); 6943 trace_seq_printf(s, "dropped events: %ld\n", cnt); 6944 6945 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu); 6946 trace_seq_printf(s, "read events: %ld\n", cnt); 6947 6948 count = simple_read_from_buffer(ubuf, count, ppos, 6949 s->buffer, trace_seq_used(s)); 6950 6951 kfree(s); 6952 6953 return count; 6954 } 6955 6956 static const struct file_operations tracing_stats_fops = { 6957 .open = tracing_open_generic_tr, 6958 .read = tracing_stats_read, 6959 .llseek = generic_file_llseek, 6960 .release = tracing_release_generic_tr, 6961 }; 6962 6963 #ifdef CONFIG_DYNAMIC_FTRACE 6964 6965 static ssize_t 6966 tracing_read_dyn_info(struct file *filp, char __user *ubuf, 6967 size_t cnt, loff_t *ppos) 6968 { 6969 unsigned long *p = filp->private_data; 6970 char buf[64]; /* Not too big for a shallow stack */ 6971 int r; 6972 6973 r = scnprintf(buf, 63, "%ld", *p); 6974 buf[r++] = '\n'; 6975 6976 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 6977 } 6978 6979 static const struct file_operations tracing_dyn_info_fops = { 6980 .open = tracing_open_generic, 6981 .read = tracing_read_dyn_info, 6982 .llseek = generic_file_llseek, 6983 }; 6984 #endif /* CONFIG_DYNAMIC_FTRACE */ 6985 6986 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) 6987 static void 6988 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, 6989 struct trace_array *tr, struct ftrace_probe_ops *ops, 6990 void *data) 6991 { 6992 tracing_snapshot_instance(tr); 6993 } 6994 6995 static void 6996 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, 6997 struct trace_array *tr, struct ftrace_probe_ops *ops, 6998 void *data) 6999 { 7000 struct ftrace_func_mapper *mapper = data; 7001 long *count = NULL; 7002 7003 if (mapper) 7004 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 7005 7006 if (count) { 7007 7008 if (*count <= 0) 7009 return; 7010 7011 (*count)--; 7012 } 7013 7014 tracing_snapshot_instance(tr); 7015 } 7016 7017 static int 7018 ftrace_snapshot_print(struct seq_file *m, unsigned long ip, 7019 struct ftrace_probe_ops *ops, void *data) 7020 { 7021 struct ftrace_func_mapper *mapper = data; 7022 long *count = NULL; 7023 7024 seq_printf(m, "%ps:", (void *)ip); 7025 7026 seq_puts(m, "snapshot"); 7027 7028 if (mapper) 7029 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 7030 7031 if (count) 7032 seq_printf(m, ":count=%ld\n", *count); 7033 else 7034 seq_puts(m, ":unlimited\n"); 7035 7036 return 0; 7037 } 7038 7039 static int 7040 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr, 7041 unsigned long ip, void *init_data, void **data) 7042 { 7043 struct ftrace_func_mapper *mapper = *data; 7044 7045 if (!mapper) { 7046 mapper = allocate_ftrace_func_mapper(); 7047 if (!mapper) 7048 return -ENOMEM; 7049 *data = mapper; 7050 } 7051 7052 return ftrace_func_mapper_add_ip(mapper, ip, init_data); 7053 } 7054 7055 static void 7056 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr, 7057 unsigned long ip, void *data) 7058 { 7059 struct ftrace_func_mapper *mapper = data; 7060 7061 if (!ip) { 7062 if (!mapper) 7063 return; 7064 free_ftrace_func_mapper(mapper, NULL); 7065 return; 7066 } 7067 7068 ftrace_func_mapper_remove_ip(mapper, ip); 7069 } 7070 7071 static struct ftrace_probe_ops snapshot_probe_ops = { 7072 .func = ftrace_snapshot, 7073 .print = ftrace_snapshot_print, 7074 }; 7075 7076 static struct ftrace_probe_ops snapshot_count_probe_ops = { 7077 .func = ftrace_count_snapshot, 7078 .print = ftrace_snapshot_print, 7079 .init = ftrace_snapshot_init, 7080 .free = ftrace_snapshot_free, 7081 }; 7082 7083 static int 7084 ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash, 7085 char *glob, char *cmd, char *param, int enable) 7086 { 7087 struct ftrace_probe_ops *ops; 7088 void *count = (void *)-1; 7089 char *number; 7090 int ret; 7091 7092 if (!tr) 7093 return -ENODEV; 7094 7095 /* hash funcs only work with set_ftrace_filter */ 7096 if (!enable) 7097 return -EINVAL; 7098 7099 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops; 7100 7101 if (glob[0] == '!') 7102 return unregister_ftrace_function_probe_func(glob+1, tr, ops); 7103 7104 if (!param) 7105 goto out_reg; 7106 7107 number = strsep(¶m, ":"); 7108 7109 if (!strlen(number)) 7110 goto out_reg; 7111 7112 /* 7113 * We use the callback data field (which is a pointer) 7114 * as our counter. 7115 */ 7116 ret = kstrtoul(number, 0, (unsigned long *)&count); 7117 if (ret) 7118 return ret; 7119 7120 out_reg: 7121 ret = alloc_snapshot(tr); 7122 if (ret < 0) 7123 goto out; 7124 7125 ret = register_ftrace_function_probe(glob, tr, ops, count); 7126 7127 out: 7128 return ret < 0 ? ret : 0; 7129 } 7130 7131 static struct ftrace_func_command ftrace_snapshot_cmd = { 7132 .name = "snapshot", 7133 .func = ftrace_trace_snapshot_callback, 7134 }; 7135 7136 static __init int register_snapshot_cmd(void) 7137 { 7138 return register_ftrace_command(&ftrace_snapshot_cmd); 7139 } 7140 #else 7141 static inline __init int register_snapshot_cmd(void) { return 0; } 7142 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */ 7143 7144 static struct dentry *tracing_get_dentry(struct trace_array *tr) 7145 { 7146 if (WARN_ON(!tr->dir)) 7147 return ERR_PTR(-ENODEV); 7148 7149 /* Top directory uses NULL as the parent */ 7150 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 7151 return NULL; 7152 7153 /* All sub buffers have a descriptor */ 7154 return tr->dir; 7155 } 7156 7157 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu) 7158 { 7159 struct dentry *d_tracer; 7160 7161 if (tr->percpu_dir) 7162 return tr->percpu_dir; 7163 7164 d_tracer = tracing_get_dentry(tr); 7165 if (IS_ERR(d_tracer)) 7166 return NULL; 7167 7168 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer); 7169 7170 WARN_ONCE(!tr->percpu_dir, 7171 "Could not create tracefs directory 'per_cpu/%d'\n", cpu); 7172 7173 return tr->percpu_dir; 7174 } 7175 7176 static struct dentry * 7177 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent, 7178 void *data, long cpu, const struct file_operations *fops) 7179 { 7180 struct dentry *ret = trace_create_file(name, mode, parent, data, fops); 7181 7182 if (ret) /* See tracing_get_cpu() */ 7183 d_inode(ret)->i_cdev = (void *)(cpu + 1); 7184 return ret; 7185 } 7186 7187 static void 7188 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu) 7189 { 7190 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu); 7191 struct dentry *d_cpu; 7192 char cpu_dir[30]; /* 30 characters should be more than enough */ 7193 7194 if (!d_percpu) 7195 return; 7196 7197 snprintf(cpu_dir, 30, "cpu%ld", cpu); 7198 d_cpu = tracefs_create_dir(cpu_dir, d_percpu); 7199 if (!d_cpu) { 7200 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir); 7201 return; 7202 } 7203 7204 /* per cpu trace_pipe */ 7205 trace_create_cpu_file("trace_pipe", 0444, d_cpu, 7206 tr, cpu, &tracing_pipe_fops); 7207 7208 /* per cpu trace */ 7209 trace_create_cpu_file("trace", 0644, d_cpu, 7210 tr, cpu, &tracing_fops); 7211 7212 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu, 7213 tr, cpu, &tracing_buffers_fops); 7214 7215 trace_create_cpu_file("stats", 0444, d_cpu, 7216 tr, cpu, &tracing_stats_fops); 7217 7218 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu, 7219 tr, cpu, &tracing_entries_fops); 7220 7221 #ifdef CONFIG_TRACER_SNAPSHOT 7222 trace_create_cpu_file("snapshot", 0644, d_cpu, 7223 tr, cpu, &snapshot_fops); 7224 7225 trace_create_cpu_file("snapshot_raw", 0444, d_cpu, 7226 tr, cpu, &snapshot_raw_fops); 7227 #endif 7228 } 7229 7230 #ifdef CONFIG_FTRACE_SELFTEST 7231 /* Let selftest have access to static functions in this file */ 7232 #include "trace_selftest.c" 7233 #endif 7234 7235 static ssize_t 7236 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt, 7237 loff_t *ppos) 7238 { 7239 struct trace_option_dentry *topt = filp->private_data; 7240 char *buf; 7241 7242 if (topt->flags->val & topt->opt->bit) 7243 buf = "1\n"; 7244 else 7245 buf = "0\n"; 7246 7247 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 7248 } 7249 7250 static ssize_t 7251 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt, 7252 loff_t *ppos) 7253 { 7254 struct trace_option_dentry *topt = filp->private_data; 7255 unsigned long val; 7256 int ret; 7257 7258 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 7259 if (ret) 7260 return ret; 7261 7262 if (val != 0 && val != 1) 7263 return -EINVAL; 7264 7265 if (!!(topt->flags->val & topt->opt->bit) != val) { 7266 mutex_lock(&trace_types_lock); 7267 ret = __set_tracer_option(topt->tr, topt->flags, 7268 topt->opt, !val); 7269 mutex_unlock(&trace_types_lock); 7270 if (ret) 7271 return ret; 7272 } 7273 7274 *ppos += cnt; 7275 7276 return cnt; 7277 } 7278 7279 7280 static const struct file_operations trace_options_fops = { 7281 .open = tracing_open_generic, 7282 .read = trace_options_read, 7283 .write = trace_options_write, 7284 .llseek = generic_file_llseek, 7285 }; 7286 7287 /* 7288 * In order to pass in both the trace_array descriptor as well as the index 7289 * to the flag that the trace option file represents, the trace_array 7290 * has a character array of trace_flags_index[], which holds the index 7291 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc. 7292 * The address of this character array is passed to the flag option file 7293 * read/write callbacks. 7294 * 7295 * In order to extract both the index and the trace_array descriptor, 7296 * get_tr_index() uses the following algorithm. 7297 * 7298 * idx = *ptr; 7299 * 7300 * As the pointer itself contains the address of the index (remember 7301 * index[1] == 1). 7302 * 7303 * Then to get the trace_array descriptor, by subtracting that index 7304 * from the ptr, we get to the start of the index itself. 7305 * 7306 * ptr - idx == &index[0] 7307 * 7308 * Then a simple container_of() from that pointer gets us to the 7309 * trace_array descriptor. 7310 */ 7311 static void get_tr_index(void *data, struct trace_array **ptr, 7312 unsigned int *pindex) 7313 { 7314 *pindex = *(unsigned char *)data; 7315 7316 *ptr = container_of(data - *pindex, struct trace_array, 7317 trace_flags_index); 7318 } 7319 7320 static ssize_t 7321 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt, 7322 loff_t *ppos) 7323 { 7324 void *tr_index = filp->private_data; 7325 struct trace_array *tr; 7326 unsigned int index; 7327 char *buf; 7328 7329 get_tr_index(tr_index, &tr, &index); 7330 7331 if (tr->trace_flags & (1 << index)) 7332 buf = "1\n"; 7333 else 7334 buf = "0\n"; 7335 7336 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 7337 } 7338 7339 static ssize_t 7340 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt, 7341 loff_t *ppos) 7342 { 7343 void *tr_index = filp->private_data; 7344 struct trace_array *tr; 7345 unsigned int index; 7346 unsigned long val; 7347 int ret; 7348 7349 get_tr_index(tr_index, &tr, &index); 7350 7351 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 7352 if (ret) 7353 return ret; 7354 7355 if (val != 0 && val != 1) 7356 return -EINVAL; 7357 7358 mutex_lock(&trace_types_lock); 7359 ret = set_tracer_flag(tr, 1 << index, val); 7360 mutex_unlock(&trace_types_lock); 7361 7362 if (ret < 0) 7363 return ret; 7364 7365 *ppos += cnt; 7366 7367 return cnt; 7368 } 7369 7370 static const struct file_operations trace_options_core_fops = { 7371 .open = tracing_open_generic, 7372 .read = trace_options_core_read, 7373 .write = trace_options_core_write, 7374 .llseek = generic_file_llseek, 7375 }; 7376 7377 struct dentry *trace_create_file(const char *name, 7378 umode_t mode, 7379 struct dentry *parent, 7380 void *data, 7381 const struct file_operations *fops) 7382 { 7383 struct dentry *ret; 7384 7385 ret = tracefs_create_file(name, mode, parent, data, fops); 7386 if (!ret) 7387 pr_warn("Could not create tracefs '%s' entry\n", name); 7388 7389 return ret; 7390 } 7391 7392 7393 static struct dentry *trace_options_init_dentry(struct trace_array *tr) 7394 { 7395 struct dentry *d_tracer; 7396 7397 if (tr->options) 7398 return tr->options; 7399 7400 d_tracer = tracing_get_dentry(tr); 7401 if (IS_ERR(d_tracer)) 7402 return NULL; 7403 7404 tr->options = tracefs_create_dir("options", d_tracer); 7405 if (!tr->options) { 7406 pr_warn("Could not create tracefs directory 'options'\n"); 7407 return NULL; 7408 } 7409 7410 return tr->options; 7411 } 7412 7413 static void 7414 create_trace_option_file(struct trace_array *tr, 7415 struct trace_option_dentry *topt, 7416 struct tracer_flags *flags, 7417 struct tracer_opt *opt) 7418 { 7419 struct dentry *t_options; 7420 7421 t_options = trace_options_init_dentry(tr); 7422 if (!t_options) 7423 return; 7424 7425 topt->flags = flags; 7426 topt->opt = opt; 7427 topt->tr = tr; 7428 7429 topt->entry = trace_create_file(opt->name, 0644, t_options, topt, 7430 &trace_options_fops); 7431 7432 } 7433 7434 static void 7435 create_trace_option_files(struct trace_array *tr, struct tracer *tracer) 7436 { 7437 struct trace_option_dentry *topts; 7438 struct trace_options *tr_topts; 7439 struct tracer_flags *flags; 7440 struct tracer_opt *opts; 7441 int cnt; 7442 int i; 7443 7444 if (!tracer) 7445 return; 7446 7447 flags = tracer->flags; 7448 7449 if (!flags || !flags->opts) 7450 return; 7451 7452 /* 7453 * If this is an instance, only create flags for tracers 7454 * the instance may have. 7455 */ 7456 if (!trace_ok_for_array(tracer, tr)) 7457 return; 7458 7459 for (i = 0; i < tr->nr_topts; i++) { 7460 /* Make sure there's no duplicate flags. */ 7461 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags)) 7462 return; 7463 } 7464 7465 opts = flags->opts; 7466 7467 for (cnt = 0; opts[cnt].name; cnt++) 7468 ; 7469 7470 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL); 7471 if (!topts) 7472 return; 7473 7474 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1), 7475 GFP_KERNEL); 7476 if (!tr_topts) { 7477 kfree(topts); 7478 return; 7479 } 7480 7481 tr->topts = tr_topts; 7482 tr->topts[tr->nr_topts].tracer = tracer; 7483 tr->topts[tr->nr_topts].topts = topts; 7484 tr->nr_topts++; 7485 7486 for (cnt = 0; opts[cnt].name; cnt++) { 7487 create_trace_option_file(tr, &topts[cnt], flags, 7488 &opts[cnt]); 7489 WARN_ONCE(topts[cnt].entry == NULL, 7490 "Failed to create trace option: %s", 7491 opts[cnt].name); 7492 } 7493 } 7494 7495 static struct dentry * 7496 create_trace_option_core_file(struct trace_array *tr, 7497 const char *option, long index) 7498 { 7499 struct dentry *t_options; 7500 7501 t_options = trace_options_init_dentry(tr); 7502 if (!t_options) 7503 return NULL; 7504 7505 return trace_create_file(option, 0644, t_options, 7506 (void *)&tr->trace_flags_index[index], 7507 &trace_options_core_fops); 7508 } 7509 7510 static void create_trace_options_dir(struct trace_array *tr) 7511 { 7512 struct dentry *t_options; 7513 bool top_level = tr == &global_trace; 7514 int i; 7515 7516 t_options = trace_options_init_dentry(tr); 7517 if (!t_options) 7518 return; 7519 7520 for (i = 0; trace_options[i]; i++) { 7521 if (top_level || 7522 !((1 << i) & TOP_LEVEL_TRACE_FLAGS)) 7523 create_trace_option_core_file(tr, trace_options[i], i); 7524 } 7525 } 7526 7527 static ssize_t 7528 rb_simple_read(struct file *filp, char __user *ubuf, 7529 size_t cnt, loff_t *ppos) 7530 { 7531 struct trace_array *tr = filp->private_data; 7532 char buf[64]; 7533 int r; 7534 7535 r = tracer_tracing_is_on(tr); 7536 r = sprintf(buf, "%d\n", r); 7537 7538 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 7539 } 7540 7541 static ssize_t 7542 rb_simple_write(struct file *filp, const char __user *ubuf, 7543 size_t cnt, loff_t *ppos) 7544 { 7545 struct trace_array *tr = filp->private_data; 7546 struct ring_buffer *buffer = tr->trace_buffer.buffer; 7547 unsigned long val; 7548 int ret; 7549 7550 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 7551 if (ret) 7552 return ret; 7553 7554 if (buffer) { 7555 mutex_lock(&trace_types_lock); 7556 if (val) { 7557 tracer_tracing_on(tr); 7558 if (tr->current_trace->start) 7559 tr->current_trace->start(tr); 7560 } else { 7561 tracer_tracing_off(tr); 7562 if (tr->current_trace->stop) 7563 tr->current_trace->stop(tr); 7564 } 7565 mutex_unlock(&trace_types_lock); 7566 } 7567 7568 (*ppos)++; 7569 7570 return cnt; 7571 } 7572 7573 static const struct file_operations rb_simple_fops = { 7574 .open = tracing_open_generic_tr, 7575 .read = rb_simple_read, 7576 .write = rb_simple_write, 7577 .release = tracing_release_generic_tr, 7578 .llseek = default_llseek, 7579 }; 7580 7581 struct dentry *trace_instance_dir; 7582 7583 static void 7584 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer); 7585 7586 static int 7587 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size) 7588 { 7589 enum ring_buffer_flags rb_flags; 7590 7591 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0; 7592 7593 buf->tr = tr; 7594 7595 buf->buffer = ring_buffer_alloc(size, rb_flags); 7596 if (!buf->buffer) 7597 return -ENOMEM; 7598 7599 buf->data = alloc_percpu(struct trace_array_cpu); 7600 if (!buf->data) { 7601 ring_buffer_free(buf->buffer); 7602 return -ENOMEM; 7603 } 7604 7605 /* Allocate the first page for all buffers */ 7606 set_buffer_entries(&tr->trace_buffer, 7607 ring_buffer_size(tr->trace_buffer.buffer, 0)); 7608 7609 return 0; 7610 } 7611 7612 static int allocate_trace_buffers(struct trace_array *tr, int size) 7613 { 7614 int ret; 7615 7616 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size); 7617 if (ret) 7618 return ret; 7619 7620 #ifdef CONFIG_TRACER_MAX_TRACE 7621 ret = allocate_trace_buffer(tr, &tr->max_buffer, 7622 allocate_snapshot ? size : 1); 7623 if (WARN_ON(ret)) { 7624 ring_buffer_free(tr->trace_buffer.buffer); 7625 free_percpu(tr->trace_buffer.data); 7626 return -ENOMEM; 7627 } 7628 tr->allocated_snapshot = allocate_snapshot; 7629 7630 /* 7631 * Only the top level trace array gets its snapshot allocated 7632 * from the kernel command line. 7633 */ 7634 allocate_snapshot = false; 7635 #endif 7636 return 0; 7637 } 7638 7639 static void free_trace_buffer(struct trace_buffer *buf) 7640 { 7641 if (buf->buffer) { 7642 ring_buffer_free(buf->buffer); 7643 buf->buffer = NULL; 7644 free_percpu(buf->data); 7645 buf->data = NULL; 7646 } 7647 } 7648 7649 static void free_trace_buffers(struct trace_array *tr) 7650 { 7651 if (!tr) 7652 return; 7653 7654 free_trace_buffer(&tr->trace_buffer); 7655 7656 #ifdef CONFIG_TRACER_MAX_TRACE 7657 free_trace_buffer(&tr->max_buffer); 7658 #endif 7659 } 7660 7661 static void init_trace_flags_index(struct trace_array *tr) 7662 { 7663 int i; 7664 7665 /* Used by the trace options files */ 7666 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) 7667 tr->trace_flags_index[i] = i; 7668 } 7669 7670 static void __update_tracer_options(struct trace_array *tr) 7671 { 7672 struct tracer *t; 7673 7674 for (t = trace_types; t; t = t->next) 7675 add_tracer_options(tr, t); 7676 } 7677 7678 static void update_tracer_options(struct trace_array *tr) 7679 { 7680 mutex_lock(&trace_types_lock); 7681 __update_tracer_options(tr); 7682 mutex_unlock(&trace_types_lock); 7683 } 7684 7685 static int instance_mkdir(const char *name) 7686 { 7687 struct trace_array *tr; 7688 int ret; 7689 7690 mutex_lock(&trace_types_lock); 7691 7692 ret = -EEXIST; 7693 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 7694 if (tr->name && strcmp(tr->name, name) == 0) 7695 goto out_unlock; 7696 } 7697 7698 ret = -ENOMEM; 7699 tr = kzalloc(sizeof(*tr), GFP_KERNEL); 7700 if (!tr) 7701 goto out_unlock; 7702 7703 tr->name = kstrdup(name, GFP_KERNEL); 7704 if (!tr->name) 7705 goto out_free_tr; 7706 7707 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL)) 7708 goto out_free_tr; 7709 7710 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS; 7711 7712 cpumask_copy(tr->tracing_cpumask, cpu_all_mask); 7713 7714 raw_spin_lock_init(&tr->start_lock); 7715 7716 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 7717 7718 tr->current_trace = &nop_trace; 7719 7720 INIT_LIST_HEAD(&tr->systems); 7721 INIT_LIST_HEAD(&tr->events); 7722 7723 if (allocate_trace_buffers(tr, trace_buf_size) < 0) 7724 goto out_free_tr; 7725 7726 tr->dir = tracefs_create_dir(name, trace_instance_dir); 7727 if (!tr->dir) 7728 goto out_free_tr; 7729 7730 ret = event_trace_add_tracer(tr->dir, tr); 7731 if (ret) { 7732 tracefs_remove_recursive(tr->dir); 7733 goto out_free_tr; 7734 } 7735 7736 ftrace_init_trace_array(tr); 7737 7738 init_tracer_tracefs(tr, tr->dir); 7739 init_trace_flags_index(tr); 7740 __update_tracer_options(tr); 7741 7742 list_add(&tr->list, &ftrace_trace_arrays); 7743 7744 mutex_unlock(&trace_types_lock); 7745 7746 return 0; 7747 7748 out_free_tr: 7749 free_trace_buffers(tr); 7750 free_cpumask_var(tr->tracing_cpumask); 7751 kfree(tr->name); 7752 kfree(tr); 7753 7754 out_unlock: 7755 mutex_unlock(&trace_types_lock); 7756 7757 return ret; 7758 7759 } 7760 7761 static int instance_rmdir(const char *name) 7762 { 7763 struct trace_array *tr; 7764 int found = 0; 7765 int ret; 7766 int i; 7767 7768 mutex_lock(&trace_types_lock); 7769 7770 ret = -ENODEV; 7771 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 7772 if (tr->name && strcmp(tr->name, name) == 0) { 7773 found = 1; 7774 break; 7775 } 7776 } 7777 if (!found) 7778 goto out_unlock; 7779 7780 ret = -EBUSY; 7781 if (tr->ref || (tr->current_trace && tr->current_trace->ref)) 7782 goto out_unlock; 7783 7784 list_del(&tr->list); 7785 7786 /* Disable all the flags that were enabled coming in */ 7787 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) { 7788 if ((1 << i) & ZEROED_TRACE_FLAGS) 7789 set_tracer_flag(tr, 1 << i, 0); 7790 } 7791 7792 tracing_set_nop(tr); 7793 clear_ftrace_function_probes(tr); 7794 event_trace_del_tracer(tr); 7795 ftrace_clear_pids(tr); 7796 ftrace_destroy_function_files(tr); 7797 tracefs_remove_recursive(tr->dir); 7798 free_trace_buffers(tr); 7799 7800 for (i = 0; i < tr->nr_topts; i++) { 7801 kfree(tr->topts[i].topts); 7802 } 7803 kfree(tr->topts); 7804 7805 free_cpumask_var(tr->tracing_cpumask); 7806 kfree(tr->name); 7807 kfree(tr); 7808 7809 ret = 0; 7810 7811 out_unlock: 7812 mutex_unlock(&trace_types_lock); 7813 7814 return ret; 7815 } 7816 7817 static __init void create_trace_instances(struct dentry *d_tracer) 7818 { 7819 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer, 7820 instance_mkdir, 7821 instance_rmdir); 7822 if (WARN_ON(!trace_instance_dir)) 7823 return; 7824 } 7825 7826 static void 7827 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) 7828 { 7829 int cpu; 7830 7831 trace_create_file("available_tracers", 0444, d_tracer, 7832 tr, &show_traces_fops); 7833 7834 trace_create_file("current_tracer", 0644, d_tracer, 7835 tr, &set_tracer_fops); 7836 7837 trace_create_file("tracing_cpumask", 0644, d_tracer, 7838 tr, &tracing_cpumask_fops); 7839 7840 trace_create_file("trace_options", 0644, d_tracer, 7841 tr, &tracing_iter_fops); 7842 7843 trace_create_file("trace", 0644, d_tracer, 7844 tr, &tracing_fops); 7845 7846 trace_create_file("trace_pipe", 0444, d_tracer, 7847 tr, &tracing_pipe_fops); 7848 7849 trace_create_file("buffer_size_kb", 0644, d_tracer, 7850 tr, &tracing_entries_fops); 7851 7852 trace_create_file("buffer_total_size_kb", 0444, d_tracer, 7853 tr, &tracing_total_entries_fops); 7854 7855 trace_create_file("free_buffer", 0200, d_tracer, 7856 tr, &tracing_free_buffer_fops); 7857 7858 trace_create_file("trace_marker", 0220, d_tracer, 7859 tr, &tracing_mark_fops); 7860 7861 trace_create_file("trace_marker_raw", 0220, d_tracer, 7862 tr, &tracing_mark_raw_fops); 7863 7864 trace_create_file("trace_clock", 0644, d_tracer, tr, 7865 &trace_clock_fops); 7866 7867 trace_create_file("tracing_on", 0644, d_tracer, 7868 tr, &rb_simple_fops); 7869 7870 create_trace_options_dir(tr); 7871 7872 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) 7873 trace_create_file("tracing_max_latency", 0644, d_tracer, 7874 &tr->max_latency, &tracing_max_lat_fops); 7875 #endif 7876 7877 if (ftrace_create_function_files(tr, d_tracer)) 7878 WARN(1, "Could not allocate function filter files"); 7879 7880 #ifdef CONFIG_TRACER_SNAPSHOT 7881 trace_create_file("snapshot", 0644, d_tracer, 7882 tr, &snapshot_fops); 7883 #endif 7884 7885 for_each_tracing_cpu(cpu) 7886 tracing_init_tracefs_percpu(tr, cpu); 7887 7888 ftrace_init_tracefs(tr, d_tracer); 7889 } 7890 7891 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore) 7892 { 7893 struct vfsmount *mnt; 7894 struct file_system_type *type; 7895 7896 /* 7897 * To maintain backward compatibility for tools that mount 7898 * debugfs to get to the tracing facility, tracefs is automatically 7899 * mounted to the debugfs/tracing directory. 7900 */ 7901 type = get_fs_type("tracefs"); 7902 if (!type) 7903 return NULL; 7904 mnt = vfs_submount(mntpt, type, "tracefs", NULL); 7905 put_filesystem(type); 7906 if (IS_ERR(mnt)) 7907 return NULL; 7908 mntget(mnt); 7909 7910 return mnt; 7911 } 7912 7913 /** 7914 * tracing_init_dentry - initialize top level trace array 7915 * 7916 * This is called when creating files or directories in the tracing 7917 * directory. It is called via fs_initcall() by any of the boot up code 7918 * and expects to return the dentry of the top level tracing directory. 7919 */ 7920 struct dentry *tracing_init_dentry(void) 7921 { 7922 struct trace_array *tr = &global_trace; 7923 7924 /* The top level trace array uses NULL as parent */ 7925 if (tr->dir) 7926 return NULL; 7927 7928 if (WARN_ON(!tracefs_initialized()) || 7929 (IS_ENABLED(CONFIG_DEBUG_FS) && 7930 WARN_ON(!debugfs_initialized()))) 7931 return ERR_PTR(-ENODEV); 7932 7933 /* 7934 * As there may still be users that expect the tracing 7935 * files to exist in debugfs/tracing, we must automount 7936 * the tracefs file system there, so older tools still 7937 * work with the newer kerenl. 7938 */ 7939 tr->dir = debugfs_create_automount("tracing", NULL, 7940 trace_automount, NULL); 7941 if (!tr->dir) { 7942 pr_warn_once("Could not create debugfs directory 'tracing'\n"); 7943 return ERR_PTR(-ENOMEM); 7944 } 7945 7946 return NULL; 7947 } 7948 7949 extern struct trace_eval_map *__start_ftrace_eval_maps[]; 7950 extern struct trace_eval_map *__stop_ftrace_eval_maps[]; 7951 7952 static void __init trace_eval_init(void) 7953 { 7954 int len; 7955 7956 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps; 7957 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len); 7958 } 7959 7960 #ifdef CONFIG_MODULES 7961 static void trace_module_add_evals(struct module *mod) 7962 { 7963 if (!mod->num_trace_evals) 7964 return; 7965 7966 /* 7967 * Modules with bad taint do not have events created, do 7968 * not bother with enums either. 7969 */ 7970 if (trace_module_has_bad_taint(mod)) 7971 return; 7972 7973 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals); 7974 } 7975 7976 #ifdef CONFIG_TRACE_EVAL_MAP_FILE 7977 static void trace_module_remove_evals(struct module *mod) 7978 { 7979 union trace_eval_map_item *map; 7980 union trace_eval_map_item **last = &trace_eval_maps; 7981 7982 if (!mod->num_trace_evals) 7983 return; 7984 7985 mutex_lock(&trace_eval_mutex); 7986 7987 map = trace_eval_maps; 7988 7989 while (map) { 7990 if (map->head.mod == mod) 7991 break; 7992 map = trace_eval_jmp_to_tail(map); 7993 last = &map->tail.next; 7994 map = map->tail.next; 7995 } 7996 if (!map) 7997 goto out; 7998 7999 *last = trace_eval_jmp_to_tail(map)->tail.next; 8000 kfree(map); 8001 out: 8002 mutex_unlock(&trace_eval_mutex); 8003 } 8004 #else 8005 static inline void trace_module_remove_evals(struct module *mod) { } 8006 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */ 8007 8008 static int trace_module_notify(struct notifier_block *self, 8009 unsigned long val, void *data) 8010 { 8011 struct module *mod = data; 8012 8013 switch (val) { 8014 case MODULE_STATE_COMING: 8015 trace_module_add_evals(mod); 8016 break; 8017 case MODULE_STATE_GOING: 8018 trace_module_remove_evals(mod); 8019 break; 8020 } 8021 8022 return 0; 8023 } 8024 8025 static struct notifier_block trace_module_nb = { 8026 .notifier_call = trace_module_notify, 8027 .priority = 0, 8028 }; 8029 #endif /* CONFIG_MODULES */ 8030 8031 static __init int tracer_init_tracefs(void) 8032 { 8033 struct dentry *d_tracer; 8034 8035 trace_access_lock_init(); 8036 8037 d_tracer = tracing_init_dentry(); 8038 if (IS_ERR(d_tracer)) 8039 return 0; 8040 8041 init_tracer_tracefs(&global_trace, d_tracer); 8042 ftrace_init_tracefs_toplevel(&global_trace, d_tracer); 8043 8044 trace_create_file("tracing_thresh", 0644, d_tracer, 8045 &global_trace, &tracing_thresh_fops); 8046 8047 trace_create_file("README", 0444, d_tracer, 8048 NULL, &tracing_readme_fops); 8049 8050 trace_create_file("saved_cmdlines", 0444, d_tracer, 8051 NULL, &tracing_saved_cmdlines_fops); 8052 8053 trace_create_file("saved_cmdlines_size", 0644, d_tracer, 8054 NULL, &tracing_saved_cmdlines_size_fops); 8055 8056 trace_create_file("saved_tgids", 0444, d_tracer, 8057 NULL, &tracing_saved_tgids_fops); 8058 8059 trace_eval_init(); 8060 8061 trace_create_eval_file(d_tracer); 8062 8063 #ifdef CONFIG_MODULES 8064 register_module_notifier(&trace_module_nb); 8065 #endif 8066 8067 #ifdef CONFIG_DYNAMIC_FTRACE 8068 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer, 8069 &ftrace_update_tot_cnt, &tracing_dyn_info_fops); 8070 #endif 8071 8072 create_trace_instances(d_tracer); 8073 8074 update_tracer_options(&global_trace); 8075 8076 return 0; 8077 } 8078 8079 static int trace_panic_handler(struct notifier_block *this, 8080 unsigned long event, void *unused) 8081 { 8082 if (ftrace_dump_on_oops) 8083 ftrace_dump(ftrace_dump_on_oops); 8084 return NOTIFY_OK; 8085 } 8086 8087 static struct notifier_block trace_panic_notifier = { 8088 .notifier_call = trace_panic_handler, 8089 .next = NULL, 8090 .priority = 150 /* priority: INT_MAX >= x >= 0 */ 8091 }; 8092 8093 static int trace_die_handler(struct notifier_block *self, 8094 unsigned long val, 8095 void *data) 8096 { 8097 switch (val) { 8098 case DIE_OOPS: 8099 if (ftrace_dump_on_oops) 8100 ftrace_dump(ftrace_dump_on_oops); 8101 break; 8102 default: 8103 break; 8104 } 8105 return NOTIFY_OK; 8106 } 8107 8108 static struct notifier_block trace_die_notifier = { 8109 .notifier_call = trace_die_handler, 8110 .priority = 200 8111 }; 8112 8113 /* 8114 * printk is set to max of 1024, we really don't need it that big. 8115 * Nothing should be printing 1000 characters anyway. 8116 */ 8117 #define TRACE_MAX_PRINT 1000 8118 8119 /* 8120 * Define here KERN_TRACE so that we have one place to modify 8121 * it if we decide to change what log level the ftrace dump 8122 * should be at. 8123 */ 8124 #define KERN_TRACE KERN_EMERG 8125 8126 void 8127 trace_printk_seq(struct trace_seq *s) 8128 { 8129 /* Probably should print a warning here. */ 8130 if (s->seq.len >= TRACE_MAX_PRINT) 8131 s->seq.len = TRACE_MAX_PRINT; 8132 8133 /* 8134 * More paranoid code. Although the buffer size is set to 8135 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just 8136 * an extra layer of protection. 8137 */ 8138 if (WARN_ON_ONCE(s->seq.len >= s->seq.size)) 8139 s->seq.len = s->seq.size - 1; 8140 8141 /* should be zero ended, but we are paranoid. */ 8142 s->buffer[s->seq.len] = 0; 8143 8144 printk(KERN_TRACE "%s", s->buffer); 8145 8146 trace_seq_init(s); 8147 } 8148 8149 void trace_init_global_iter(struct trace_iterator *iter) 8150 { 8151 iter->tr = &global_trace; 8152 iter->trace = iter->tr->current_trace; 8153 iter->cpu_file = RING_BUFFER_ALL_CPUS; 8154 iter->trace_buffer = &global_trace.trace_buffer; 8155 8156 if (iter->trace && iter->trace->open) 8157 iter->trace->open(iter); 8158 8159 /* Annotate start of buffers if we had overruns */ 8160 if (ring_buffer_overruns(iter->trace_buffer->buffer)) 8161 iter->iter_flags |= TRACE_FILE_ANNOTATE; 8162 8163 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 8164 if (trace_clocks[iter->tr->clock_id].in_ns) 8165 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 8166 } 8167 8168 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) 8169 { 8170 /* use static because iter can be a bit big for the stack */ 8171 static struct trace_iterator iter; 8172 static atomic_t dump_running; 8173 struct trace_array *tr = &global_trace; 8174 unsigned int old_userobj; 8175 unsigned long flags; 8176 int cnt = 0, cpu; 8177 8178 /* Only allow one dump user at a time. */ 8179 if (atomic_inc_return(&dump_running) != 1) { 8180 atomic_dec(&dump_running); 8181 return; 8182 } 8183 8184 /* 8185 * Always turn off tracing when we dump. 8186 * We don't need to show trace output of what happens 8187 * between multiple crashes. 8188 * 8189 * If the user does a sysrq-z, then they can re-enable 8190 * tracing with echo 1 > tracing_on. 8191 */ 8192 tracing_off(); 8193 8194 local_irq_save(flags); 8195 8196 /* Simulate the iterator */ 8197 trace_init_global_iter(&iter); 8198 8199 for_each_tracing_cpu(cpu) { 8200 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled); 8201 } 8202 8203 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ; 8204 8205 /* don't look at user memory in panic mode */ 8206 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ; 8207 8208 switch (oops_dump_mode) { 8209 case DUMP_ALL: 8210 iter.cpu_file = RING_BUFFER_ALL_CPUS; 8211 break; 8212 case DUMP_ORIG: 8213 iter.cpu_file = raw_smp_processor_id(); 8214 break; 8215 case DUMP_NONE: 8216 goto out_enable; 8217 default: 8218 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n"); 8219 iter.cpu_file = RING_BUFFER_ALL_CPUS; 8220 } 8221 8222 printk(KERN_TRACE "Dumping ftrace buffer:\n"); 8223 8224 /* Did function tracer already get disabled? */ 8225 if (ftrace_is_dead()) { 8226 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n"); 8227 printk("# MAY BE MISSING FUNCTION EVENTS\n"); 8228 } 8229 8230 /* 8231 * We need to stop all tracing on all CPUS to read the 8232 * the next buffer. This is a bit expensive, but is 8233 * not done often. We fill all what we can read, 8234 * and then release the locks again. 8235 */ 8236 8237 while (!trace_empty(&iter)) { 8238 8239 if (!cnt) 8240 printk(KERN_TRACE "---------------------------------\n"); 8241 8242 cnt++; 8243 8244 /* reset all but tr, trace, and overruns */ 8245 memset(&iter.seq, 0, 8246 sizeof(struct trace_iterator) - 8247 offsetof(struct trace_iterator, seq)); 8248 iter.iter_flags |= TRACE_FILE_LAT_FMT; 8249 iter.pos = -1; 8250 8251 if (trace_find_next_entry_inc(&iter) != NULL) { 8252 int ret; 8253 8254 ret = print_trace_line(&iter); 8255 if (ret != TRACE_TYPE_NO_CONSUME) 8256 trace_consume(&iter); 8257 } 8258 touch_nmi_watchdog(); 8259 8260 trace_printk_seq(&iter.seq); 8261 } 8262 8263 if (!cnt) 8264 printk(KERN_TRACE " (ftrace buffer empty)\n"); 8265 else 8266 printk(KERN_TRACE "---------------------------------\n"); 8267 8268 out_enable: 8269 tr->trace_flags |= old_userobj; 8270 8271 for_each_tracing_cpu(cpu) { 8272 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled); 8273 } 8274 atomic_dec(&dump_running); 8275 local_irq_restore(flags); 8276 } 8277 EXPORT_SYMBOL_GPL(ftrace_dump); 8278 8279 __init static int tracer_alloc_buffers(void) 8280 { 8281 int ring_buf_size; 8282 int ret = -ENOMEM; 8283 8284 /* 8285 * Make sure we don't accidently add more trace options 8286 * than we have bits for. 8287 */ 8288 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE); 8289 8290 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL)) 8291 goto out; 8292 8293 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL)) 8294 goto out_free_buffer_mask; 8295 8296 /* Only allocate trace_printk buffers if a trace_printk exists */ 8297 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt) 8298 /* Must be called before global_trace.buffer is allocated */ 8299 trace_printk_init_buffers(); 8300 8301 /* To save memory, keep the ring buffer size to its minimum */ 8302 if (ring_buffer_expanded) 8303 ring_buf_size = trace_buf_size; 8304 else 8305 ring_buf_size = 1; 8306 8307 cpumask_copy(tracing_buffer_mask, cpu_possible_mask); 8308 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask); 8309 8310 raw_spin_lock_init(&global_trace.start_lock); 8311 8312 /* 8313 * The prepare callbacks allocates some memory for the ring buffer. We 8314 * don't free the buffer if the if the CPU goes down. If we were to free 8315 * the buffer, then the user would lose any trace that was in the 8316 * buffer. The memory will be removed once the "instance" is removed. 8317 */ 8318 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE, 8319 "trace/RB:preapre", trace_rb_cpu_prepare, 8320 NULL); 8321 if (ret < 0) 8322 goto out_free_cpumask; 8323 /* Used for event triggers */ 8324 ret = -ENOMEM; 8325 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE); 8326 if (!temp_buffer) 8327 goto out_rm_hp_state; 8328 8329 if (trace_create_savedcmd() < 0) 8330 goto out_free_temp_buffer; 8331 8332 /* TODO: make the number of buffers hot pluggable with CPUS */ 8333 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) { 8334 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n"); 8335 WARN_ON(1); 8336 goto out_free_savedcmd; 8337 } 8338 8339 if (global_trace.buffer_disabled) 8340 tracing_off(); 8341 8342 if (trace_boot_clock) { 8343 ret = tracing_set_clock(&global_trace, trace_boot_clock); 8344 if (ret < 0) 8345 pr_warn("Trace clock %s not defined, going back to default\n", 8346 trace_boot_clock); 8347 } 8348 8349 /* 8350 * register_tracer() might reference current_trace, so it 8351 * needs to be set before we register anything. This is 8352 * just a bootstrap of current_trace anyway. 8353 */ 8354 global_trace.current_trace = &nop_trace; 8355 8356 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 8357 8358 ftrace_init_global_array_ops(&global_trace); 8359 8360 init_trace_flags_index(&global_trace); 8361 8362 register_tracer(&nop_trace); 8363 8364 /* Function tracing may start here (via kernel command line) */ 8365 init_function_trace(); 8366 8367 /* All seems OK, enable tracing */ 8368 tracing_disabled = 0; 8369 8370 atomic_notifier_chain_register(&panic_notifier_list, 8371 &trace_panic_notifier); 8372 8373 register_die_notifier(&trace_die_notifier); 8374 8375 global_trace.flags = TRACE_ARRAY_FL_GLOBAL; 8376 8377 INIT_LIST_HEAD(&global_trace.systems); 8378 INIT_LIST_HEAD(&global_trace.events); 8379 list_add(&global_trace.list, &ftrace_trace_arrays); 8380 8381 apply_trace_boot_options(); 8382 8383 register_snapshot_cmd(); 8384 8385 return 0; 8386 8387 out_free_savedcmd: 8388 free_saved_cmdlines_buffer(savedcmd); 8389 out_free_temp_buffer: 8390 ring_buffer_free(temp_buffer); 8391 out_rm_hp_state: 8392 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE); 8393 out_free_cpumask: 8394 free_cpumask_var(global_trace.tracing_cpumask); 8395 out_free_buffer_mask: 8396 free_cpumask_var(tracing_buffer_mask); 8397 out: 8398 return ret; 8399 } 8400 8401 void __init early_trace_init(void) 8402 { 8403 if (tracepoint_printk) { 8404 tracepoint_print_iter = 8405 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL); 8406 if (WARN_ON(!tracepoint_print_iter)) 8407 tracepoint_printk = 0; 8408 else 8409 static_key_enable(&tracepoint_printk_key.key); 8410 } 8411 tracer_alloc_buffers(); 8412 } 8413 8414 void __init trace_init(void) 8415 { 8416 trace_event_init(); 8417 } 8418 8419 __init static int clear_boot_tracer(void) 8420 { 8421 /* 8422 * The default tracer at boot buffer is an init section. 8423 * This function is called in lateinit. If we did not 8424 * find the boot tracer, then clear it out, to prevent 8425 * later registration from accessing the buffer that is 8426 * about to be freed. 8427 */ 8428 if (!default_bootup_tracer) 8429 return 0; 8430 8431 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n", 8432 default_bootup_tracer); 8433 default_bootup_tracer = NULL; 8434 8435 return 0; 8436 } 8437 8438 fs_initcall(tracer_init_tracefs); 8439 late_initcall_sync(clear_boot_tracer); 8440