1 /* 2 * ring buffer based function tracer 3 * 4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com> 5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 6 * 7 * Originally taken from the RT patch by: 8 * Arnaldo Carvalho de Melo <acme@redhat.com> 9 * 10 * Based on code from the latency_tracer, that is: 11 * Copyright (C) 2004-2006 Ingo Molnar 12 * Copyright (C) 2004 Nadia Yvette Chambers 13 */ 14 #include <linux/ring_buffer.h> 15 #include <generated/utsrelease.h> 16 #include <linux/stacktrace.h> 17 #include <linux/writeback.h> 18 #include <linux/kallsyms.h> 19 #include <linux/seq_file.h> 20 #include <linux/notifier.h> 21 #include <linux/irqflags.h> 22 #include <linux/debugfs.h> 23 #include <linux/tracefs.h> 24 #include <linux/pagemap.h> 25 #include <linux/hardirq.h> 26 #include <linux/linkage.h> 27 #include <linux/uaccess.h> 28 #include <linux/kprobes.h> 29 #include <linux/ftrace.h> 30 #include <linux/module.h> 31 #include <linux/percpu.h> 32 #include <linux/splice.h> 33 #include <linux/kdebug.h> 34 #include <linux/string.h> 35 #include <linux/mount.h> 36 #include <linux/rwsem.h> 37 #include <linux/slab.h> 38 #include <linux/ctype.h> 39 #include <linux/init.h> 40 #include <linux/poll.h> 41 #include <linux/nmi.h> 42 #include <linux/fs.h> 43 #include <linux/sched/rt.h> 44 45 #include "trace.h" 46 #include "trace_output.h" 47 48 /* 49 * On boot up, the ring buffer is set to the minimum size, so that 50 * we do not waste memory on systems that are not using tracing. 51 */ 52 bool ring_buffer_expanded; 53 54 /* 55 * We need to change this state when a selftest is running. 56 * A selftest will lurk into the ring-buffer to count the 57 * entries inserted during the selftest although some concurrent 58 * insertions into the ring-buffer such as trace_printk could occurred 59 * at the same time, giving false positive or negative results. 60 */ 61 static bool __read_mostly tracing_selftest_running; 62 63 /* 64 * If a tracer is running, we do not want to run SELFTEST. 65 */ 66 bool __read_mostly tracing_selftest_disabled; 67 68 /* Pipe tracepoints to printk */ 69 struct trace_iterator *tracepoint_print_iter; 70 int tracepoint_printk; 71 72 /* For tracers that don't implement custom flags */ 73 static struct tracer_opt dummy_tracer_opt[] = { 74 { } 75 }; 76 77 static struct tracer_flags dummy_tracer_flags = { 78 .val = 0, 79 .opts = dummy_tracer_opt 80 }; 81 82 static int 83 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 84 { 85 return 0; 86 } 87 88 /* 89 * To prevent the comm cache from being overwritten when no 90 * tracing is active, only save the comm when a trace event 91 * occurred. 92 */ 93 static DEFINE_PER_CPU(bool, trace_cmdline_save); 94 95 /* 96 * Kill all tracing for good (never come back). 97 * It is initialized to 1 but will turn to zero if the initialization 98 * of the tracer is successful. But that is the only place that sets 99 * this back to zero. 100 */ 101 static int tracing_disabled = 1; 102 103 cpumask_var_t __read_mostly tracing_buffer_mask; 104 105 /* 106 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops 107 * 108 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops 109 * is set, then ftrace_dump is called. This will output the contents 110 * of the ftrace buffers to the console. This is very useful for 111 * capturing traces that lead to crashes and outputing it to a 112 * serial console. 113 * 114 * It is default off, but you can enable it with either specifying 115 * "ftrace_dump_on_oops" in the kernel command line, or setting 116 * /proc/sys/kernel/ftrace_dump_on_oops 117 * Set 1 if you want to dump buffers of all CPUs 118 * Set 2 if you want to dump the buffer of the CPU that triggered oops 119 */ 120 121 enum ftrace_dump_mode ftrace_dump_on_oops; 122 123 /* When set, tracing will stop when a WARN*() is hit */ 124 int __disable_trace_on_warning; 125 126 #ifdef CONFIG_TRACE_ENUM_MAP_FILE 127 /* Map of enums to their values, for "enum_map" file */ 128 struct trace_enum_map_head { 129 struct module *mod; 130 unsigned long length; 131 }; 132 133 union trace_enum_map_item; 134 135 struct trace_enum_map_tail { 136 /* 137 * "end" is first and points to NULL as it must be different 138 * than "mod" or "enum_string" 139 */ 140 union trace_enum_map_item *next; 141 const char *end; /* points to NULL */ 142 }; 143 144 static DEFINE_MUTEX(trace_enum_mutex); 145 146 /* 147 * The trace_enum_maps are saved in an array with two extra elements, 148 * one at the beginning, and one at the end. The beginning item contains 149 * the count of the saved maps (head.length), and the module they 150 * belong to if not built in (head.mod). The ending item contains a 151 * pointer to the next array of saved enum_map items. 152 */ 153 union trace_enum_map_item { 154 struct trace_enum_map map; 155 struct trace_enum_map_head head; 156 struct trace_enum_map_tail tail; 157 }; 158 159 static union trace_enum_map_item *trace_enum_maps; 160 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */ 161 162 static int tracing_set_tracer(struct trace_array *tr, const char *buf); 163 164 #define MAX_TRACER_SIZE 100 165 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; 166 static char *default_bootup_tracer; 167 168 static bool allocate_snapshot; 169 170 static int __init set_cmdline_ftrace(char *str) 171 { 172 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE); 173 default_bootup_tracer = bootup_tracer_buf; 174 /* We are using ftrace early, expand it */ 175 ring_buffer_expanded = true; 176 return 1; 177 } 178 __setup("ftrace=", set_cmdline_ftrace); 179 180 static int __init set_ftrace_dump_on_oops(char *str) 181 { 182 if (*str++ != '=' || !*str) { 183 ftrace_dump_on_oops = DUMP_ALL; 184 return 1; 185 } 186 187 if (!strcmp("orig_cpu", str)) { 188 ftrace_dump_on_oops = DUMP_ORIG; 189 return 1; 190 } 191 192 return 0; 193 } 194 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops); 195 196 static int __init stop_trace_on_warning(char *str) 197 { 198 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0)) 199 __disable_trace_on_warning = 1; 200 return 1; 201 } 202 __setup("traceoff_on_warning", stop_trace_on_warning); 203 204 static int __init boot_alloc_snapshot(char *str) 205 { 206 allocate_snapshot = true; 207 /* We also need the main ring buffer expanded */ 208 ring_buffer_expanded = true; 209 return 1; 210 } 211 __setup("alloc_snapshot", boot_alloc_snapshot); 212 213 214 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata; 215 216 static int __init set_trace_boot_options(char *str) 217 { 218 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE); 219 return 0; 220 } 221 __setup("trace_options=", set_trace_boot_options); 222 223 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata; 224 static char *trace_boot_clock __initdata; 225 226 static int __init set_trace_boot_clock(char *str) 227 { 228 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE); 229 trace_boot_clock = trace_boot_clock_buf; 230 return 0; 231 } 232 __setup("trace_clock=", set_trace_boot_clock); 233 234 static int __init set_tracepoint_printk(char *str) 235 { 236 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0)) 237 tracepoint_printk = 1; 238 return 1; 239 } 240 __setup("tp_printk", set_tracepoint_printk); 241 242 unsigned long long ns2usecs(cycle_t nsec) 243 { 244 nsec += 500; 245 do_div(nsec, 1000); 246 return nsec; 247 } 248 249 /* trace_flags holds trace_options default values */ 250 #define TRACE_DEFAULT_FLAGS \ 251 (FUNCTION_DEFAULT_FLAGS | \ 252 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \ 253 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \ 254 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \ 255 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS) 256 257 /* trace_options that are only supported by global_trace */ 258 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \ 259 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD) 260 261 262 /* 263 * The global_trace is the descriptor that holds the tracing 264 * buffers for the live tracing. For each CPU, it contains 265 * a link list of pages that will store trace entries. The 266 * page descriptor of the pages in the memory is used to hold 267 * the link list by linking the lru item in the page descriptor 268 * to each of the pages in the buffer per CPU. 269 * 270 * For each active CPU there is a data field that holds the 271 * pages for the buffer for that CPU. Each CPU has the same number 272 * of pages allocated for its buffer. 273 */ 274 static struct trace_array global_trace = { 275 .trace_flags = TRACE_DEFAULT_FLAGS, 276 }; 277 278 LIST_HEAD(ftrace_trace_arrays); 279 280 int trace_array_get(struct trace_array *this_tr) 281 { 282 struct trace_array *tr; 283 int ret = -ENODEV; 284 285 mutex_lock(&trace_types_lock); 286 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 287 if (tr == this_tr) { 288 tr->ref++; 289 ret = 0; 290 break; 291 } 292 } 293 mutex_unlock(&trace_types_lock); 294 295 return ret; 296 } 297 298 static void __trace_array_put(struct trace_array *this_tr) 299 { 300 WARN_ON(!this_tr->ref); 301 this_tr->ref--; 302 } 303 304 void trace_array_put(struct trace_array *this_tr) 305 { 306 mutex_lock(&trace_types_lock); 307 __trace_array_put(this_tr); 308 mutex_unlock(&trace_types_lock); 309 } 310 311 int filter_check_discard(struct trace_event_file *file, void *rec, 312 struct ring_buffer *buffer, 313 struct ring_buffer_event *event) 314 { 315 if (unlikely(file->flags & EVENT_FILE_FL_FILTERED) && 316 !filter_match_preds(file->filter, rec)) { 317 ring_buffer_discard_commit(buffer, event); 318 return 1; 319 } 320 321 return 0; 322 } 323 EXPORT_SYMBOL_GPL(filter_check_discard); 324 325 int call_filter_check_discard(struct trace_event_call *call, void *rec, 326 struct ring_buffer *buffer, 327 struct ring_buffer_event *event) 328 { 329 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) && 330 !filter_match_preds(call->filter, rec)) { 331 ring_buffer_discard_commit(buffer, event); 332 return 1; 333 } 334 335 return 0; 336 } 337 EXPORT_SYMBOL_GPL(call_filter_check_discard); 338 339 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu) 340 { 341 u64 ts; 342 343 /* Early boot up does not have a buffer yet */ 344 if (!buf->buffer) 345 return trace_clock_local(); 346 347 ts = ring_buffer_time_stamp(buf->buffer, cpu); 348 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts); 349 350 return ts; 351 } 352 353 cycle_t ftrace_now(int cpu) 354 { 355 return buffer_ftrace_now(&global_trace.trace_buffer, cpu); 356 } 357 358 /** 359 * tracing_is_enabled - Show if global_trace has been disabled 360 * 361 * Shows if the global trace has been enabled or not. It uses the 362 * mirror flag "buffer_disabled" to be used in fast paths such as for 363 * the irqsoff tracer. But it may be inaccurate due to races. If you 364 * need to know the accurate state, use tracing_is_on() which is a little 365 * slower, but accurate. 366 */ 367 int tracing_is_enabled(void) 368 { 369 /* 370 * For quick access (irqsoff uses this in fast path), just 371 * return the mirror variable of the state of the ring buffer. 372 * It's a little racy, but we don't really care. 373 */ 374 smp_rmb(); 375 return !global_trace.buffer_disabled; 376 } 377 378 /* 379 * trace_buf_size is the size in bytes that is allocated 380 * for a buffer. Note, the number of bytes is always rounded 381 * to page size. 382 * 383 * This number is purposely set to a low number of 16384. 384 * If the dump on oops happens, it will be much appreciated 385 * to not have to wait for all that output. Anyway this can be 386 * boot time and run time configurable. 387 */ 388 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */ 389 390 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT; 391 392 /* trace_types holds a link list of available tracers. */ 393 static struct tracer *trace_types __read_mostly; 394 395 /* 396 * trace_types_lock is used to protect the trace_types list. 397 */ 398 DEFINE_MUTEX(trace_types_lock); 399 400 /* 401 * serialize the access of the ring buffer 402 * 403 * ring buffer serializes readers, but it is low level protection. 404 * The validity of the events (which returns by ring_buffer_peek() ..etc) 405 * are not protected by ring buffer. 406 * 407 * The content of events may become garbage if we allow other process consumes 408 * these events concurrently: 409 * A) the page of the consumed events may become a normal page 410 * (not reader page) in ring buffer, and this page will be rewrited 411 * by events producer. 412 * B) The page of the consumed events may become a page for splice_read, 413 * and this page will be returned to system. 414 * 415 * These primitives allow multi process access to different cpu ring buffer 416 * concurrently. 417 * 418 * These primitives don't distinguish read-only and read-consume access. 419 * Multi read-only access are also serialized. 420 */ 421 422 #ifdef CONFIG_SMP 423 static DECLARE_RWSEM(all_cpu_access_lock); 424 static DEFINE_PER_CPU(struct mutex, cpu_access_lock); 425 426 static inline void trace_access_lock(int cpu) 427 { 428 if (cpu == RING_BUFFER_ALL_CPUS) { 429 /* gain it for accessing the whole ring buffer. */ 430 down_write(&all_cpu_access_lock); 431 } else { 432 /* gain it for accessing a cpu ring buffer. */ 433 434 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */ 435 down_read(&all_cpu_access_lock); 436 437 /* Secondly block other access to this @cpu ring buffer. */ 438 mutex_lock(&per_cpu(cpu_access_lock, cpu)); 439 } 440 } 441 442 static inline void trace_access_unlock(int cpu) 443 { 444 if (cpu == RING_BUFFER_ALL_CPUS) { 445 up_write(&all_cpu_access_lock); 446 } else { 447 mutex_unlock(&per_cpu(cpu_access_lock, cpu)); 448 up_read(&all_cpu_access_lock); 449 } 450 } 451 452 static inline void trace_access_lock_init(void) 453 { 454 int cpu; 455 456 for_each_possible_cpu(cpu) 457 mutex_init(&per_cpu(cpu_access_lock, cpu)); 458 } 459 460 #else 461 462 static DEFINE_MUTEX(access_lock); 463 464 static inline void trace_access_lock(int cpu) 465 { 466 (void)cpu; 467 mutex_lock(&access_lock); 468 } 469 470 static inline void trace_access_unlock(int cpu) 471 { 472 (void)cpu; 473 mutex_unlock(&access_lock); 474 } 475 476 static inline void trace_access_lock_init(void) 477 { 478 } 479 480 #endif 481 482 #ifdef CONFIG_STACKTRACE 483 static void __ftrace_trace_stack(struct ring_buffer *buffer, 484 unsigned long flags, 485 int skip, int pc, struct pt_regs *regs); 486 static inline void ftrace_trace_stack(struct trace_array *tr, 487 struct ring_buffer *buffer, 488 unsigned long flags, 489 int skip, int pc, struct pt_regs *regs); 490 491 #else 492 static inline void __ftrace_trace_stack(struct ring_buffer *buffer, 493 unsigned long flags, 494 int skip, int pc, struct pt_regs *regs) 495 { 496 } 497 static inline void ftrace_trace_stack(struct trace_array *tr, 498 struct ring_buffer *buffer, 499 unsigned long flags, 500 int skip, int pc, struct pt_regs *regs) 501 { 502 } 503 504 #endif 505 506 static void tracer_tracing_on(struct trace_array *tr) 507 { 508 if (tr->trace_buffer.buffer) 509 ring_buffer_record_on(tr->trace_buffer.buffer); 510 /* 511 * This flag is looked at when buffers haven't been allocated 512 * yet, or by some tracers (like irqsoff), that just want to 513 * know if the ring buffer has been disabled, but it can handle 514 * races of where it gets disabled but we still do a record. 515 * As the check is in the fast path of the tracers, it is more 516 * important to be fast than accurate. 517 */ 518 tr->buffer_disabled = 0; 519 /* Make the flag seen by readers */ 520 smp_wmb(); 521 } 522 523 /** 524 * tracing_on - enable tracing buffers 525 * 526 * This function enables tracing buffers that may have been 527 * disabled with tracing_off. 528 */ 529 void tracing_on(void) 530 { 531 tracer_tracing_on(&global_trace); 532 } 533 EXPORT_SYMBOL_GPL(tracing_on); 534 535 /** 536 * __trace_puts - write a constant string into the trace buffer. 537 * @ip: The address of the caller 538 * @str: The constant string to write 539 * @size: The size of the string. 540 */ 541 int __trace_puts(unsigned long ip, const char *str, int size) 542 { 543 struct ring_buffer_event *event; 544 struct ring_buffer *buffer; 545 struct print_entry *entry; 546 unsigned long irq_flags; 547 int alloc; 548 int pc; 549 550 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 551 return 0; 552 553 pc = preempt_count(); 554 555 if (unlikely(tracing_selftest_running || tracing_disabled)) 556 return 0; 557 558 alloc = sizeof(*entry) + size + 2; /* possible \n added */ 559 560 local_save_flags(irq_flags); 561 buffer = global_trace.trace_buffer.buffer; 562 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 563 irq_flags, pc); 564 if (!event) 565 return 0; 566 567 entry = ring_buffer_event_data(event); 568 entry->ip = ip; 569 570 memcpy(&entry->buf, str, size); 571 572 /* Add a newline if necessary */ 573 if (entry->buf[size - 1] != '\n') { 574 entry->buf[size] = '\n'; 575 entry->buf[size + 1] = '\0'; 576 } else 577 entry->buf[size] = '\0'; 578 579 __buffer_unlock_commit(buffer, event); 580 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL); 581 582 return size; 583 } 584 EXPORT_SYMBOL_GPL(__trace_puts); 585 586 /** 587 * __trace_bputs - write the pointer to a constant string into trace buffer 588 * @ip: The address of the caller 589 * @str: The constant string to write to the buffer to 590 */ 591 int __trace_bputs(unsigned long ip, const char *str) 592 { 593 struct ring_buffer_event *event; 594 struct ring_buffer *buffer; 595 struct bputs_entry *entry; 596 unsigned long irq_flags; 597 int size = sizeof(struct bputs_entry); 598 int pc; 599 600 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 601 return 0; 602 603 pc = preempt_count(); 604 605 if (unlikely(tracing_selftest_running || tracing_disabled)) 606 return 0; 607 608 local_save_flags(irq_flags); 609 buffer = global_trace.trace_buffer.buffer; 610 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size, 611 irq_flags, pc); 612 if (!event) 613 return 0; 614 615 entry = ring_buffer_event_data(event); 616 entry->ip = ip; 617 entry->str = str; 618 619 __buffer_unlock_commit(buffer, event); 620 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL); 621 622 return 1; 623 } 624 EXPORT_SYMBOL_GPL(__trace_bputs); 625 626 #ifdef CONFIG_TRACER_SNAPSHOT 627 /** 628 * trace_snapshot - take a snapshot of the current buffer. 629 * 630 * This causes a swap between the snapshot buffer and the current live 631 * tracing buffer. You can use this to take snapshots of the live 632 * trace when some condition is triggered, but continue to trace. 633 * 634 * Note, make sure to allocate the snapshot with either 635 * a tracing_snapshot_alloc(), or by doing it manually 636 * with: echo 1 > /sys/kernel/debug/tracing/snapshot 637 * 638 * If the snapshot buffer is not allocated, it will stop tracing. 639 * Basically making a permanent snapshot. 640 */ 641 void tracing_snapshot(void) 642 { 643 struct trace_array *tr = &global_trace; 644 struct tracer *tracer = tr->current_trace; 645 unsigned long flags; 646 647 if (in_nmi()) { 648 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n"); 649 internal_trace_puts("*** snapshot is being ignored ***\n"); 650 return; 651 } 652 653 if (!tr->allocated_snapshot) { 654 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n"); 655 internal_trace_puts("*** stopping trace here! ***\n"); 656 tracing_off(); 657 return; 658 } 659 660 /* Note, snapshot can not be used when the tracer uses it */ 661 if (tracer->use_max_tr) { 662 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n"); 663 internal_trace_puts("*** Can not use snapshot (sorry) ***\n"); 664 return; 665 } 666 667 local_irq_save(flags); 668 update_max_tr(tr, current, smp_processor_id()); 669 local_irq_restore(flags); 670 } 671 EXPORT_SYMBOL_GPL(tracing_snapshot); 672 673 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf, 674 struct trace_buffer *size_buf, int cpu_id); 675 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val); 676 677 static int alloc_snapshot(struct trace_array *tr) 678 { 679 int ret; 680 681 if (!tr->allocated_snapshot) { 682 683 /* allocate spare buffer */ 684 ret = resize_buffer_duplicate_size(&tr->max_buffer, 685 &tr->trace_buffer, RING_BUFFER_ALL_CPUS); 686 if (ret < 0) 687 return ret; 688 689 tr->allocated_snapshot = true; 690 } 691 692 return 0; 693 } 694 695 static void free_snapshot(struct trace_array *tr) 696 { 697 /* 698 * We don't free the ring buffer. instead, resize it because 699 * The max_tr ring buffer has some state (e.g. ring->clock) and 700 * we want preserve it. 701 */ 702 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS); 703 set_buffer_entries(&tr->max_buffer, 1); 704 tracing_reset_online_cpus(&tr->max_buffer); 705 tr->allocated_snapshot = false; 706 } 707 708 /** 709 * tracing_alloc_snapshot - allocate snapshot buffer. 710 * 711 * This only allocates the snapshot buffer if it isn't already 712 * allocated - it doesn't also take a snapshot. 713 * 714 * This is meant to be used in cases where the snapshot buffer needs 715 * to be set up for events that can't sleep but need to be able to 716 * trigger a snapshot. 717 */ 718 int tracing_alloc_snapshot(void) 719 { 720 struct trace_array *tr = &global_trace; 721 int ret; 722 723 ret = alloc_snapshot(tr); 724 WARN_ON(ret < 0); 725 726 return ret; 727 } 728 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); 729 730 /** 731 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer. 732 * 733 * This is similar to trace_snapshot(), but it will allocate the 734 * snapshot buffer if it isn't already allocated. Use this only 735 * where it is safe to sleep, as the allocation may sleep. 736 * 737 * This causes a swap between the snapshot buffer and the current live 738 * tracing buffer. You can use this to take snapshots of the live 739 * trace when some condition is triggered, but continue to trace. 740 */ 741 void tracing_snapshot_alloc(void) 742 { 743 int ret; 744 745 ret = tracing_alloc_snapshot(); 746 if (ret < 0) 747 return; 748 749 tracing_snapshot(); 750 } 751 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); 752 #else 753 void tracing_snapshot(void) 754 { 755 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used"); 756 } 757 EXPORT_SYMBOL_GPL(tracing_snapshot); 758 int tracing_alloc_snapshot(void) 759 { 760 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used"); 761 return -ENODEV; 762 } 763 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); 764 void tracing_snapshot_alloc(void) 765 { 766 /* Give warning */ 767 tracing_snapshot(); 768 } 769 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); 770 #endif /* CONFIG_TRACER_SNAPSHOT */ 771 772 static void tracer_tracing_off(struct trace_array *tr) 773 { 774 if (tr->trace_buffer.buffer) 775 ring_buffer_record_off(tr->trace_buffer.buffer); 776 /* 777 * This flag is looked at when buffers haven't been allocated 778 * yet, or by some tracers (like irqsoff), that just want to 779 * know if the ring buffer has been disabled, but it can handle 780 * races of where it gets disabled but we still do a record. 781 * As the check is in the fast path of the tracers, it is more 782 * important to be fast than accurate. 783 */ 784 tr->buffer_disabled = 1; 785 /* Make the flag seen by readers */ 786 smp_wmb(); 787 } 788 789 /** 790 * tracing_off - turn off tracing buffers 791 * 792 * This function stops the tracing buffers from recording data. 793 * It does not disable any overhead the tracers themselves may 794 * be causing. This function simply causes all recording to 795 * the ring buffers to fail. 796 */ 797 void tracing_off(void) 798 { 799 tracer_tracing_off(&global_trace); 800 } 801 EXPORT_SYMBOL_GPL(tracing_off); 802 803 void disable_trace_on_warning(void) 804 { 805 if (__disable_trace_on_warning) 806 tracing_off(); 807 } 808 809 /** 810 * tracer_tracing_is_on - show real state of ring buffer enabled 811 * @tr : the trace array to know if ring buffer is enabled 812 * 813 * Shows real state of the ring buffer if it is enabled or not. 814 */ 815 static int tracer_tracing_is_on(struct trace_array *tr) 816 { 817 if (tr->trace_buffer.buffer) 818 return ring_buffer_record_is_on(tr->trace_buffer.buffer); 819 return !tr->buffer_disabled; 820 } 821 822 /** 823 * tracing_is_on - show state of ring buffers enabled 824 */ 825 int tracing_is_on(void) 826 { 827 return tracer_tracing_is_on(&global_trace); 828 } 829 EXPORT_SYMBOL_GPL(tracing_is_on); 830 831 static int __init set_buf_size(char *str) 832 { 833 unsigned long buf_size; 834 835 if (!str) 836 return 0; 837 buf_size = memparse(str, &str); 838 /* nr_entries can not be zero */ 839 if (buf_size == 0) 840 return 0; 841 trace_buf_size = buf_size; 842 return 1; 843 } 844 __setup("trace_buf_size=", set_buf_size); 845 846 static int __init set_tracing_thresh(char *str) 847 { 848 unsigned long threshold; 849 int ret; 850 851 if (!str) 852 return 0; 853 ret = kstrtoul(str, 0, &threshold); 854 if (ret < 0) 855 return 0; 856 tracing_thresh = threshold * 1000; 857 return 1; 858 } 859 __setup("tracing_thresh=", set_tracing_thresh); 860 861 unsigned long nsecs_to_usecs(unsigned long nsecs) 862 { 863 return nsecs / 1000; 864 } 865 866 /* 867 * TRACE_FLAGS is defined as a tuple matching bit masks with strings. 868 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that 869 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list 870 * of strings in the order that the enums were defined. 871 */ 872 #undef C 873 #define C(a, b) b 874 875 /* These must match the bit postions in trace_iterator_flags */ 876 static const char *trace_options[] = { 877 TRACE_FLAGS 878 NULL 879 }; 880 881 static struct { 882 u64 (*func)(void); 883 const char *name; 884 int in_ns; /* is this clock in nanoseconds? */ 885 } trace_clocks[] = { 886 { trace_clock_local, "local", 1 }, 887 { trace_clock_global, "global", 1 }, 888 { trace_clock_counter, "counter", 0 }, 889 { trace_clock_jiffies, "uptime", 0 }, 890 { trace_clock, "perf", 1 }, 891 { ktime_get_mono_fast_ns, "mono", 1 }, 892 { ktime_get_raw_fast_ns, "mono_raw", 1 }, 893 ARCH_TRACE_CLOCKS 894 }; 895 896 /* 897 * trace_parser_get_init - gets the buffer for trace parser 898 */ 899 int trace_parser_get_init(struct trace_parser *parser, int size) 900 { 901 memset(parser, 0, sizeof(*parser)); 902 903 parser->buffer = kmalloc(size, GFP_KERNEL); 904 if (!parser->buffer) 905 return 1; 906 907 parser->size = size; 908 return 0; 909 } 910 911 /* 912 * trace_parser_put - frees the buffer for trace parser 913 */ 914 void trace_parser_put(struct trace_parser *parser) 915 { 916 kfree(parser->buffer); 917 } 918 919 /* 920 * trace_get_user - reads the user input string separated by space 921 * (matched by isspace(ch)) 922 * 923 * For each string found the 'struct trace_parser' is updated, 924 * and the function returns. 925 * 926 * Returns number of bytes read. 927 * 928 * See kernel/trace/trace.h for 'struct trace_parser' details. 929 */ 930 int trace_get_user(struct trace_parser *parser, const char __user *ubuf, 931 size_t cnt, loff_t *ppos) 932 { 933 char ch; 934 size_t read = 0; 935 ssize_t ret; 936 937 if (!*ppos) 938 trace_parser_clear(parser); 939 940 ret = get_user(ch, ubuf++); 941 if (ret) 942 goto out; 943 944 read++; 945 cnt--; 946 947 /* 948 * The parser is not finished with the last write, 949 * continue reading the user input without skipping spaces. 950 */ 951 if (!parser->cont) { 952 /* skip white space */ 953 while (cnt && isspace(ch)) { 954 ret = get_user(ch, ubuf++); 955 if (ret) 956 goto out; 957 read++; 958 cnt--; 959 } 960 961 /* only spaces were written */ 962 if (isspace(ch)) { 963 *ppos += read; 964 ret = read; 965 goto out; 966 } 967 968 parser->idx = 0; 969 } 970 971 /* read the non-space input */ 972 while (cnt && !isspace(ch)) { 973 if (parser->idx < parser->size - 1) 974 parser->buffer[parser->idx++] = ch; 975 else { 976 ret = -EINVAL; 977 goto out; 978 } 979 ret = get_user(ch, ubuf++); 980 if (ret) 981 goto out; 982 read++; 983 cnt--; 984 } 985 986 /* We either got finished input or we have to wait for another call. */ 987 if (isspace(ch)) { 988 parser->buffer[parser->idx] = 0; 989 parser->cont = false; 990 } else if (parser->idx < parser->size - 1) { 991 parser->cont = true; 992 parser->buffer[parser->idx++] = ch; 993 } else { 994 ret = -EINVAL; 995 goto out; 996 } 997 998 *ppos += read; 999 ret = read; 1000 1001 out: 1002 return ret; 1003 } 1004 1005 /* TODO add a seq_buf_to_buffer() */ 1006 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) 1007 { 1008 int len; 1009 1010 if (trace_seq_used(s) <= s->seq.readpos) 1011 return -EBUSY; 1012 1013 len = trace_seq_used(s) - s->seq.readpos; 1014 if (cnt > len) 1015 cnt = len; 1016 memcpy(buf, s->buffer + s->seq.readpos, cnt); 1017 1018 s->seq.readpos += cnt; 1019 return cnt; 1020 } 1021 1022 unsigned long __read_mostly tracing_thresh; 1023 1024 #ifdef CONFIG_TRACER_MAX_TRACE 1025 /* 1026 * Copy the new maximum trace into the separate maximum-trace 1027 * structure. (this way the maximum trace is permanently saved, 1028 * for later retrieval via /sys/kernel/debug/tracing/latency_trace) 1029 */ 1030 static void 1031 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 1032 { 1033 struct trace_buffer *trace_buf = &tr->trace_buffer; 1034 struct trace_buffer *max_buf = &tr->max_buffer; 1035 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu); 1036 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu); 1037 1038 max_buf->cpu = cpu; 1039 max_buf->time_start = data->preempt_timestamp; 1040 1041 max_data->saved_latency = tr->max_latency; 1042 max_data->critical_start = data->critical_start; 1043 max_data->critical_end = data->critical_end; 1044 1045 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN); 1046 max_data->pid = tsk->pid; 1047 /* 1048 * If tsk == current, then use current_uid(), as that does not use 1049 * RCU. The irq tracer can be called out of RCU scope. 1050 */ 1051 if (tsk == current) 1052 max_data->uid = current_uid(); 1053 else 1054 max_data->uid = task_uid(tsk); 1055 1056 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; 1057 max_data->policy = tsk->policy; 1058 max_data->rt_priority = tsk->rt_priority; 1059 1060 /* record this tasks comm */ 1061 tracing_record_cmdline(tsk); 1062 } 1063 1064 /** 1065 * update_max_tr - snapshot all trace buffers from global_trace to max_tr 1066 * @tr: tracer 1067 * @tsk: the task with the latency 1068 * @cpu: The cpu that initiated the trace. 1069 * 1070 * Flip the buffers between the @tr and the max_tr and record information 1071 * about which task was the cause of this latency. 1072 */ 1073 void 1074 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 1075 { 1076 struct ring_buffer *buf; 1077 1078 if (tr->stop_count) 1079 return; 1080 1081 WARN_ON_ONCE(!irqs_disabled()); 1082 1083 if (!tr->allocated_snapshot) { 1084 /* Only the nop tracer should hit this when disabling */ 1085 WARN_ON_ONCE(tr->current_trace != &nop_trace); 1086 return; 1087 } 1088 1089 arch_spin_lock(&tr->max_lock); 1090 1091 buf = tr->trace_buffer.buffer; 1092 tr->trace_buffer.buffer = tr->max_buffer.buffer; 1093 tr->max_buffer.buffer = buf; 1094 1095 __update_max_tr(tr, tsk, cpu); 1096 arch_spin_unlock(&tr->max_lock); 1097 } 1098 1099 /** 1100 * update_max_tr_single - only copy one trace over, and reset the rest 1101 * @tr - tracer 1102 * @tsk - task with the latency 1103 * @cpu - the cpu of the buffer to copy. 1104 * 1105 * Flip the trace of a single CPU buffer between the @tr and the max_tr. 1106 */ 1107 void 1108 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) 1109 { 1110 int ret; 1111 1112 if (tr->stop_count) 1113 return; 1114 1115 WARN_ON_ONCE(!irqs_disabled()); 1116 if (!tr->allocated_snapshot) { 1117 /* Only the nop tracer should hit this when disabling */ 1118 WARN_ON_ONCE(tr->current_trace != &nop_trace); 1119 return; 1120 } 1121 1122 arch_spin_lock(&tr->max_lock); 1123 1124 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu); 1125 1126 if (ret == -EBUSY) { 1127 /* 1128 * We failed to swap the buffer due to a commit taking 1129 * place on this CPU. We fail to record, but we reset 1130 * the max trace buffer (no one writes directly to it) 1131 * and flag that it failed. 1132 */ 1133 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_, 1134 "Failed to swap buffers due to commit in progress\n"); 1135 } 1136 1137 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); 1138 1139 __update_max_tr(tr, tsk, cpu); 1140 arch_spin_unlock(&tr->max_lock); 1141 } 1142 #endif /* CONFIG_TRACER_MAX_TRACE */ 1143 1144 static int wait_on_pipe(struct trace_iterator *iter, bool full) 1145 { 1146 /* Iterators are static, they should be filled or empty */ 1147 if (trace_buffer_iter(iter, iter->cpu_file)) 1148 return 0; 1149 1150 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file, 1151 full); 1152 } 1153 1154 #ifdef CONFIG_FTRACE_STARTUP_TEST 1155 static int run_tracer_selftest(struct tracer *type) 1156 { 1157 struct trace_array *tr = &global_trace; 1158 struct tracer *saved_tracer = tr->current_trace; 1159 int ret; 1160 1161 if (!type->selftest || tracing_selftest_disabled) 1162 return 0; 1163 1164 /* 1165 * Run a selftest on this tracer. 1166 * Here we reset the trace buffer, and set the current 1167 * tracer to be this tracer. The tracer can then run some 1168 * internal tracing to verify that everything is in order. 1169 * If we fail, we do not register this tracer. 1170 */ 1171 tracing_reset_online_cpus(&tr->trace_buffer); 1172 1173 tr->current_trace = type; 1174 1175 #ifdef CONFIG_TRACER_MAX_TRACE 1176 if (type->use_max_tr) { 1177 /* If we expanded the buffers, make sure the max is expanded too */ 1178 if (ring_buffer_expanded) 1179 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size, 1180 RING_BUFFER_ALL_CPUS); 1181 tr->allocated_snapshot = true; 1182 } 1183 #endif 1184 1185 /* the test is responsible for initializing and enabling */ 1186 pr_info("Testing tracer %s: ", type->name); 1187 ret = type->selftest(type, tr); 1188 /* the test is responsible for resetting too */ 1189 tr->current_trace = saved_tracer; 1190 if (ret) { 1191 printk(KERN_CONT "FAILED!\n"); 1192 /* Add the warning after printing 'FAILED' */ 1193 WARN_ON(1); 1194 return -1; 1195 } 1196 /* Only reset on passing, to avoid touching corrupted buffers */ 1197 tracing_reset_online_cpus(&tr->trace_buffer); 1198 1199 #ifdef CONFIG_TRACER_MAX_TRACE 1200 if (type->use_max_tr) { 1201 tr->allocated_snapshot = false; 1202 1203 /* Shrink the max buffer again */ 1204 if (ring_buffer_expanded) 1205 ring_buffer_resize(tr->max_buffer.buffer, 1, 1206 RING_BUFFER_ALL_CPUS); 1207 } 1208 #endif 1209 1210 printk(KERN_CONT "PASSED\n"); 1211 return 0; 1212 } 1213 #else 1214 static inline int run_tracer_selftest(struct tracer *type) 1215 { 1216 return 0; 1217 } 1218 #endif /* CONFIG_FTRACE_STARTUP_TEST */ 1219 1220 static void add_tracer_options(struct trace_array *tr, struct tracer *t); 1221 1222 static void __init apply_trace_boot_options(void); 1223 1224 /** 1225 * register_tracer - register a tracer with the ftrace system. 1226 * @type - the plugin for the tracer 1227 * 1228 * Register a new plugin tracer. 1229 */ 1230 int __init register_tracer(struct tracer *type) 1231 { 1232 struct tracer *t; 1233 int ret = 0; 1234 1235 if (!type->name) { 1236 pr_info("Tracer must have a name\n"); 1237 return -1; 1238 } 1239 1240 if (strlen(type->name) >= MAX_TRACER_SIZE) { 1241 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE); 1242 return -1; 1243 } 1244 1245 mutex_lock(&trace_types_lock); 1246 1247 tracing_selftest_running = true; 1248 1249 for (t = trace_types; t; t = t->next) { 1250 if (strcmp(type->name, t->name) == 0) { 1251 /* already found */ 1252 pr_info("Tracer %s already registered\n", 1253 type->name); 1254 ret = -1; 1255 goto out; 1256 } 1257 } 1258 1259 if (!type->set_flag) 1260 type->set_flag = &dummy_set_flag; 1261 if (!type->flags) 1262 type->flags = &dummy_tracer_flags; 1263 else 1264 if (!type->flags->opts) 1265 type->flags->opts = dummy_tracer_opt; 1266 1267 ret = run_tracer_selftest(type); 1268 if (ret < 0) 1269 goto out; 1270 1271 type->next = trace_types; 1272 trace_types = type; 1273 add_tracer_options(&global_trace, type); 1274 1275 out: 1276 tracing_selftest_running = false; 1277 mutex_unlock(&trace_types_lock); 1278 1279 if (ret || !default_bootup_tracer) 1280 goto out_unlock; 1281 1282 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE)) 1283 goto out_unlock; 1284 1285 printk(KERN_INFO "Starting tracer '%s'\n", type->name); 1286 /* Do we want this tracer to start on bootup? */ 1287 tracing_set_tracer(&global_trace, type->name); 1288 default_bootup_tracer = NULL; 1289 1290 apply_trace_boot_options(); 1291 1292 /* disable other selftests, since this will break it. */ 1293 tracing_selftest_disabled = true; 1294 #ifdef CONFIG_FTRACE_STARTUP_TEST 1295 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n", 1296 type->name); 1297 #endif 1298 1299 out_unlock: 1300 return ret; 1301 } 1302 1303 void tracing_reset(struct trace_buffer *buf, int cpu) 1304 { 1305 struct ring_buffer *buffer = buf->buffer; 1306 1307 if (!buffer) 1308 return; 1309 1310 ring_buffer_record_disable(buffer); 1311 1312 /* Make sure all commits have finished */ 1313 synchronize_sched(); 1314 ring_buffer_reset_cpu(buffer, cpu); 1315 1316 ring_buffer_record_enable(buffer); 1317 } 1318 1319 void tracing_reset_online_cpus(struct trace_buffer *buf) 1320 { 1321 struct ring_buffer *buffer = buf->buffer; 1322 int cpu; 1323 1324 if (!buffer) 1325 return; 1326 1327 ring_buffer_record_disable(buffer); 1328 1329 /* Make sure all commits have finished */ 1330 synchronize_sched(); 1331 1332 buf->time_start = buffer_ftrace_now(buf, buf->cpu); 1333 1334 for_each_online_cpu(cpu) 1335 ring_buffer_reset_cpu(buffer, cpu); 1336 1337 ring_buffer_record_enable(buffer); 1338 } 1339 1340 /* Must have trace_types_lock held */ 1341 void tracing_reset_all_online_cpus(void) 1342 { 1343 struct trace_array *tr; 1344 1345 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 1346 tracing_reset_online_cpus(&tr->trace_buffer); 1347 #ifdef CONFIG_TRACER_MAX_TRACE 1348 tracing_reset_online_cpus(&tr->max_buffer); 1349 #endif 1350 } 1351 } 1352 1353 #define SAVED_CMDLINES_DEFAULT 128 1354 #define NO_CMDLINE_MAP UINT_MAX 1355 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED; 1356 struct saved_cmdlines_buffer { 1357 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; 1358 unsigned *map_cmdline_to_pid; 1359 unsigned cmdline_num; 1360 int cmdline_idx; 1361 char *saved_cmdlines; 1362 }; 1363 static struct saved_cmdlines_buffer *savedcmd; 1364 1365 /* temporary disable recording */ 1366 static atomic_t trace_record_cmdline_disabled __read_mostly; 1367 1368 static inline char *get_saved_cmdlines(int idx) 1369 { 1370 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN]; 1371 } 1372 1373 static inline void set_cmdline(int idx, const char *cmdline) 1374 { 1375 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN); 1376 } 1377 1378 static int allocate_cmdlines_buffer(unsigned int val, 1379 struct saved_cmdlines_buffer *s) 1380 { 1381 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid), 1382 GFP_KERNEL); 1383 if (!s->map_cmdline_to_pid) 1384 return -ENOMEM; 1385 1386 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL); 1387 if (!s->saved_cmdlines) { 1388 kfree(s->map_cmdline_to_pid); 1389 return -ENOMEM; 1390 } 1391 1392 s->cmdline_idx = 0; 1393 s->cmdline_num = val; 1394 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP, 1395 sizeof(s->map_pid_to_cmdline)); 1396 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP, 1397 val * sizeof(*s->map_cmdline_to_pid)); 1398 1399 return 0; 1400 } 1401 1402 static int trace_create_savedcmd(void) 1403 { 1404 int ret; 1405 1406 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL); 1407 if (!savedcmd) 1408 return -ENOMEM; 1409 1410 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd); 1411 if (ret < 0) { 1412 kfree(savedcmd); 1413 savedcmd = NULL; 1414 return -ENOMEM; 1415 } 1416 1417 return 0; 1418 } 1419 1420 int is_tracing_stopped(void) 1421 { 1422 return global_trace.stop_count; 1423 } 1424 1425 /** 1426 * tracing_start - quick start of the tracer 1427 * 1428 * If tracing is enabled but was stopped by tracing_stop, 1429 * this will start the tracer back up. 1430 */ 1431 void tracing_start(void) 1432 { 1433 struct ring_buffer *buffer; 1434 unsigned long flags; 1435 1436 if (tracing_disabled) 1437 return; 1438 1439 raw_spin_lock_irqsave(&global_trace.start_lock, flags); 1440 if (--global_trace.stop_count) { 1441 if (global_trace.stop_count < 0) { 1442 /* Someone screwed up their debugging */ 1443 WARN_ON_ONCE(1); 1444 global_trace.stop_count = 0; 1445 } 1446 goto out; 1447 } 1448 1449 /* Prevent the buffers from switching */ 1450 arch_spin_lock(&global_trace.max_lock); 1451 1452 buffer = global_trace.trace_buffer.buffer; 1453 if (buffer) 1454 ring_buffer_record_enable(buffer); 1455 1456 #ifdef CONFIG_TRACER_MAX_TRACE 1457 buffer = global_trace.max_buffer.buffer; 1458 if (buffer) 1459 ring_buffer_record_enable(buffer); 1460 #endif 1461 1462 arch_spin_unlock(&global_trace.max_lock); 1463 1464 out: 1465 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags); 1466 } 1467 1468 static void tracing_start_tr(struct trace_array *tr) 1469 { 1470 struct ring_buffer *buffer; 1471 unsigned long flags; 1472 1473 if (tracing_disabled) 1474 return; 1475 1476 /* If global, we need to also start the max tracer */ 1477 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 1478 return tracing_start(); 1479 1480 raw_spin_lock_irqsave(&tr->start_lock, flags); 1481 1482 if (--tr->stop_count) { 1483 if (tr->stop_count < 0) { 1484 /* Someone screwed up their debugging */ 1485 WARN_ON_ONCE(1); 1486 tr->stop_count = 0; 1487 } 1488 goto out; 1489 } 1490 1491 buffer = tr->trace_buffer.buffer; 1492 if (buffer) 1493 ring_buffer_record_enable(buffer); 1494 1495 out: 1496 raw_spin_unlock_irqrestore(&tr->start_lock, flags); 1497 } 1498 1499 /** 1500 * tracing_stop - quick stop of the tracer 1501 * 1502 * Light weight way to stop tracing. Use in conjunction with 1503 * tracing_start. 1504 */ 1505 void tracing_stop(void) 1506 { 1507 struct ring_buffer *buffer; 1508 unsigned long flags; 1509 1510 raw_spin_lock_irqsave(&global_trace.start_lock, flags); 1511 if (global_trace.stop_count++) 1512 goto out; 1513 1514 /* Prevent the buffers from switching */ 1515 arch_spin_lock(&global_trace.max_lock); 1516 1517 buffer = global_trace.trace_buffer.buffer; 1518 if (buffer) 1519 ring_buffer_record_disable(buffer); 1520 1521 #ifdef CONFIG_TRACER_MAX_TRACE 1522 buffer = global_trace.max_buffer.buffer; 1523 if (buffer) 1524 ring_buffer_record_disable(buffer); 1525 #endif 1526 1527 arch_spin_unlock(&global_trace.max_lock); 1528 1529 out: 1530 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags); 1531 } 1532 1533 static void tracing_stop_tr(struct trace_array *tr) 1534 { 1535 struct ring_buffer *buffer; 1536 unsigned long flags; 1537 1538 /* If global, we need to also stop the max tracer */ 1539 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 1540 return tracing_stop(); 1541 1542 raw_spin_lock_irqsave(&tr->start_lock, flags); 1543 if (tr->stop_count++) 1544 goto out; 1545 1546 buffer = tr->trace_buffer.buffer; 1547 if (buffer) 1548 ring_buffer_record_disable(buffer); 1549 1550 out: 1551 raw_spin_unlock_irqrestore(&tr->start_lock, flags); 1552 } 1553 1554 void trace_stop_cmdline_recording(void); 1555 1556 static int trace_save_cmdline(struct task_struct *tsk) 1557 { 1558 unsigned pid, idx; 1559 1560 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT)) 1561 return 0; 1562 1563 /* 1564 * It's not the end of the world if we don't get 1565 * the lock, but we also don't want to spin 1566 * nor do we want to disable interrupts, 1567 * so if we miss here, then better luck next time. 1568 */ 1569 if (!arch_spin_trylock(&trace_cmdline_lock)) 1570 return 0; 1571 1572 idx = savedcmd->map_pid_to_cmdline[tsk->pid]; 1573 if (idx == NO_CMDLINE_MAP) { 1574 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num; 1575 1576 /* 1577 * Check whether the cmdline buffer at idx has a pid 1578 * mapped. We are going to overwrite that entry so we 1579 * need to clear the map_pid_to_cmdline. Otherwise we 1580 * would read the new comm for the old pid. 1581 */ 1582 pid = savedcmd->map_cmdline_to_pid[idx]; 1583 if (pid != NO_CMDLINE_MAP) 1584 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP; 1585 1586 savedcmd->map_cmdline_to_pid[idx] = tsk->pid; 1587 savedcmd->map_pid_to_cmdline[tsk->pid] = idx; 1588 1589 savedcmd->cmdline_idx = idx; 1590 } 1591 1592 set_cmdline(idx, tsk->comm); 1593 1594 arch_spin_unlock(&trace_cmdline_lock); 1595 1596 return 1; 1597 } 1598 1599 static void __trace_find_cmdline(int pid, char comm[]) 1600 { 1601 unsigned map; 1602 1603 if (!pid) { 1604 strcpy(comm, "<idle>"); 1605 return; 1606 } 1607 1608 if (WARN_ON_ONCE(pid < 0)) { 1609 strcpy(comm, "<XXX>"); 1610 return; 1611 } 1612 1613 if (pid > PID_MAX_DEFAULT) { 1614 strcpy(comm, "<...>"); 1615 return; 1616 } 1617 1618 map = savedcmd->map_pid_to_cmdline[pid]; 1619 if (map != NO_CMDLINE_MAP) 1620 strcpy(comm, get_saved_cmdlines(map)); 1621 else 1622 strcpy(comm, "<...>"); 1623 } 1624 1625 void trace_find_cmdline(int pid, char comm[]) 1626 { 1627 preempt_disable(); 1628 arch_spin_lock(&trace_cmdline_lock); 1629 1630 __trace_find_cmdline(pid, comm); 1631 1632 arch_spin_unlock(&trace_cmdline_lock); 1633 preempt_enable(); 1634 } 1635 1636 void tracing_record_cmdline(struct task_struct *tsk) 1637 { 1638 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on()) 1639 return; 1640 1641 if (!__this_cpu_read(trace_cmdline_save)) 1642 return; 1643 1644 if (trace_save_cmdline(tsk)) 1645 __this_cpu_write(trace_cmdline_save, false); 1646 } 1647 1648 void 1649 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags, 1650 int pc) 1651 { 1652 struct task_struct *tsk = current; 1653 1654 entry->preempt_count = pc & 0xff; 1655 entry->pid = (tsk) ? tsk->pid : 0; 1656 entry->flags = 1657 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT 1658 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) | 1659 #else 1660 TRACE_FLAG_IRQS_NOSUPPORT | 1661 #endif 1662 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) | 1663 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) | 1664 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) | 1665 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0); 1666 } 1667 EXPORT_SYMBOL_GPL(tracing_generic_entry_update); 1668 1669 struct ring_buffer_event * 1670 trace_buffer_lock_reserve(struct ring_buffer *buffer, 1671 int type, 1672 unsigned long len, 1673 unsigned long flags, int pc) 1674 { 1675 struct ring_buffer_event *event; 1676 1677 event = ring_buffer_lock_reserve(buffer, len); 1678 if (event != NULL) { 1679 struct trace_entry *ent = ring_buffer_event_data(event); 1680 1681 tracing_generic_entry_update(ent, flags, pc); 1682 ent->type = type; 1683 } 1684 1685 return event; 1686 } 1687 1688 void 1689 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event) 1690 { 1691 __this_cpu_write(trace_cmdline_save, true); 1692 ring_buffer_unlock_commit(buffer, event); 1693 } 1694 1695 void trace_buffer_unlock_commit(struct trace_array *tr, 1696 struct ring_buffer *buffer, 1697 struct ring_buffer_event *event, 1698 unsigned long flags, int pc) 1699 { 1700 __buffer_unlock_commit(buffer, event); 1701 1702 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL); 1703 ftrace_trace_userstack(buffer, flags, pc); 1704 } 1705 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit); 1706 1707 static struct ring_buffer *temp_buffer; 1708 1709 struct ring_buffer_event * 1710 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb, 1711 struct trace_event_file *trace_file, 1712 int type, unsigned long len, 1713 unsigned long flags, int pc) 1714 { 1715 struct ring_buffer_event *entry; 1716 1717 *current_rb = trace_file->tr->trace_buffer.buffer; 1718 entry = trace_buffer_lock_reserve(*current_rb, 1719 type, len, flags, pc); 1720 /* 1721 * If tracing is off, but we have triggers enabled 1722 * we still need to look at the event data. Use the temp_buffer 1723 * to store the trace event for the tigger to use. It's recusive 1724 * safe and will not be recorded anywhere. 1725 */ 1726 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) { 1727 *current_rb = temp_buffer; 1728 entry = trace_buffer_lock_reserve(*current_rb, 1729 type, len, flags, pc); 1730 } 1731 return entry; 1732 } 1733 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve); 1734 1735 struct ring_buffer_event * 1736 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb, 1737 int type, unsigned long len, 1738 unsigned long flags, int pc) 1739 { 1740 *current_rb = global_trace.trace_buffer.buffer; 1741 return trace_buffer_lock_reserve(*current_rb, 1742 type, len, flags, pc); 1743 } 1744 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve); 1745 1746 void trace_buffer_unlock_commit_regs(struct trace_array *tr, 1747 struct ring_buffer *buffer, 1748 struct ring_buffer_event *event, 1749 unsigned long flags, int pc, 1750 struct pt_regs *regs) 1751 { 1752 __buffer_unlock_commit(buffer, event); 1753 1754 ftrace_trace_stack(tr, buffer, flags, 6, pc, regs); 1755 ftrace_trace_userstack(buffer, flags, pc); 1756 } 1757 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs); 1758 1759 void trace_current_buffer_discard_commit(struct ring_buffer *buffer, 1760 struct ring_buffer_event *event) 1761 { 1762 ring_buffer_discard_commit(buffer, event); 1763 } 1764 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit); 1765 1766 void 1767 trace_function(struct trace_array *tr, 1768 unsigned long ip, unsigned long parent_ip, unsigned long flags, 1769 int pc) 1770 { 1771 struct trace_event_call *call = &event_function; 1772 struct ring_buffer *buffer = tr->trace_buffer.buffer; 1773 struct ring_buffer_event *event; 1774 struct ftrace_entry *entry; 1775 1776 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), 1777 flags, pc); 1778 if (!event) 1779 return; 1780 entry = ring_buffer_event_data(event); 1781 entry->ip = ip; 1782 entry->parent_ip = parent_ip; 1783 1784 if (!call_filter_check_discard(call, entry, buffer, event)) 1785 __buffer_unlock_commit(buffer, event); 1786 } 1787 1788 #ifdef CONFIG_STACKTRACE 1789 1790 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long)) 1791 struct ftrace_stack { 1792 unsigned long calls[FTRACE_STACK_MAX_ENTRIES]; 1793 }; 1794 1795 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack); 1796 static DEFINE_PER_CPU(int, ftrace_stack_reserve); 1797 1798 static void __ftrace_trace_stack(struct ring_buffer *buffer, 1799 unsigned long flags, 1800 int skip, int pc, struct pt_regs *regs) 1801 { 1802 struct trace_event_call *call = &event_kernel_stack; 1803 struct ring_buffer_event *event; 1804 struct stack_entry *entry; 1805 struct stack_trace trace; 1806 int use_stack; 1807 int size = FTRACE_STACK_ENTRIES; 1808 1809 trace.nr_entries = 0; 1810 trace.skip = skip; 1811 1812 /* 1813 * Since events can happen in NMIs there's no safe way to 1814 * use the per cpu ftrace_stacks. We reserve it and if an interrupt 1815 * or NMI comes in, it will just have to use the default 1816 * FTRACE_STACK_SIZE. 1817 */ 1818 preempt_disable_notrace(); 1819 1820 use_stack = __this_cpu_inc_return(ftrace_stack_reserve); 1821 /* 1822 * We don't need any atomic variables, just a barrier. 1823 * If an interrupt comes in, we don't care, because it would 1824 * have exited and put the counter back to what we want. 1825 * We just need a barrier to keep gcc from moving things 1826 * around. 1827 */ 1828 barrier(); 1829 if (use_stack == 1) { 1830 trace.entries = this_cpu_ptr(ftrace_stack.calls); 1831 trace.max_entries = FTRACE_STACK_MAX_ENTRIES; 1832 1833 if (regs) 1834 save_stack_trace_regs(regs, &trace); 1835 else 1836 save_stack_trace(&trace); 1837 1838 if (trace.nr_entries > size) 1839 size = trace.nr_entries; 1840 } else 1841 /* From now on, use_stack is a boolean */ 1842 use_stack = 0; 1843 1844 size *= sizeof(unsigned long); 1845 1846 event = trace_buffer_lock_reserve(buffer, TRACE_STACK, 1847 sizeof(*entry) + size, flags, pc); 1848 if (!event) 1849 goto out; 1850 entry = ring_buffer_event_data(event); 1851 1852 memset(&entry->caller, 0, size); 1853 1854 if (use_stack) 1855 memcpy(&entry->caller, trace.entries, 1856 trace.nr_entries * sizeof(unsigned long)); 1857 else { 1858 trace.max_entries = FTRACE_STACK_ENTRIES; 1859 trace.entries = entry->caller; 1860 if (regs) 1861 save_stack_trace_regs(regs, &trace); 1862 else 1863 save_stack_trace(&trace); 1864 } 1865 1866 entry->size = trace.nr_entries; 1867 1868 if (!call_filter_check_discard(call, entry, buffer, event)) 1869 __buffer_unlock_commit(buffer, event); 1870 1871 out: 1872 /* Again, don't let gcc optimize things here */ 1873 barrier(); 1874 __this_cpu_dec(ftrace_stack_reserve); 1875 preempt_enable_notrace(); 1876 1877 } 1878 1879 static inline void ftrace_trace_stack(struct trace_array *tr, 1880 struct ring_buffer *buffer, 1881 unsigned long flags, 1882 int skip, int pc, struct pt_regs *regs) 1883 { 1884 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE)) 1885 return; 1886 1887 __ftrace_trace_stack(buffer, flags, skip, pc, regs); 1888 } 1889 1890 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, 1891 int pc) 1892 { 1893 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL); 1894 } 1895 1896 /** 1897 * trace_dump_stack - record a stack back trace in the trace buffer 1898 * @skip: Number of functions to skip (helper handlers) 1899 */ 1900 void trace_dump_stack(int skip) 1901 { 1902 unsigned long flags; 1903 1904 if (tracing_disabled || tracing_selftest_running) 1905 return; 1906 1907 local_save_flags(flags); 1908 1909 /* 1910 * Skip 3 more, seems to get us at the caller of 1911 * this function. 1912 */ 1913 skip += 3; 1914 __ftrace_trace_stack(global_trace.trace_buffer.buffer, 1915 flags, skip, preempt_count(), NULL); 1916 } 1917 1918 static DEFINE_PER_CPU(int, user_stack_count); 1919 1920 void 1921 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc) 1922 { 1923 struct trace_event_call *call = &event_user_stack; 1924 struct ring_buffer_event *event; 1925 struct userstack_entry *entry; 1926 struct stack_trace trace; 1927 1928 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE)) 1929 return; 1930 1931 /* 1932 * NMIs can not handle page faults, even with fix ups. 1933 * The save user stack can (and often does) fault. 1934 */ 1935 if (unlikely(in_nmi())) 1936 return; 1937 1938 /* 1939 * prevent recursion, since the user stack tracing may 1940 * trigger other kernel events. 1941 */ 1942 preempt_disable(); 1943 if (__this_cpu_read(user_stack_count)) 1944 goto out; 1945 1946 __this_cpu_inc(user_stack_count); 1947 1948 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, 1949 sizeof(*entry), flags, pc); 1950 if (!event) 1951 goto out_drop_count; 1952 entry = ring_buffer_event_data(event); 1953 1954 entry->tgid = current->tgid; 1955 memset(&entry->caller, 0, sizeof(entry->caller)); 1956 1957 trace.nr_entries = 0; 1958 trace.max_entries = FTRACE_STACK_ENTRIES; 1959 trace.skip = 0; 1960 trace.entries = entry->caller; 1961 1962 save_stack_trace_user(&trace); 1963 if (!call_filter_check_discard(call, entry, buffer, event)) 1964 __buffer_unlock_commit(buffer, event); 1965 1966 out_drop_count: 1967 __this_cpu_dec(user_stack_count); 1968 out: 1969 preempt_enable(); 1970 } 1971 1972 #ifdef UNUSED 1973 static void __trace_userstack(struct trace_array *tr, unsigned long flags) 1974 { 1975 ftrace_trace_userstack(tr, flags, preempt_count()); 1976 } 1977 #endif /* UNUSED */ 1978 1979 #endif /* CONFIG_STACKTRACE */ 1980 1981 /* created for use with alloc_percpu */ 1982 struct trace_buffer_struct { 1983 char buffer[TRACE_BUF_SIZE]; 1984 }; 1985 1986 static struct trace_buffer_struct *trace_percpu_buffer; 1987 static struct trace_buffer_struct *trace_percpu_sirq_buffer; 1988 static struct trace_buffer_struct *trace_percpu_irq_buffer; 1989 static struct trace_buffer_struct *trace_percpu_nmi_buffer; 1990 1991 /* 1992 * The buffer used is dependent on the context. There is a per cpu 1993 * buffer for normal context, softirq contex, hard irq context and 1994 * for NMI context. Thise allows for lockless recording. 1995 * 1996 * Note, if the buffers failed to be allocated, then this returns NULL 1997 */ 1998 static char *get_trace_buf(void) 1999 { 2000 struct trace_buffer_struct *percpu_buffer; 2001 2002 /* 2003 * If we have allocated per cpu buffers, then we do not 2004 * need to do any locking. 2005 */ 2006 if (in_nmi()) 2007 percpu_buffer = trace_percpu_nmi_buffer; 2008 else if (in_irq()) 2009 percpu_buffer = trace_percpu_irq_buffer; 2010 else if (in_softirq()) 2011 percpu_buffer = trace_percpu_sirq_buffer; 2012 else 2013 percpu_buffer = trace_percpu_buffer; 2014 2015 if (!percpu_buffer) 2016 return NULL; 2017 2018 return this_cpu_ptr(&percpu_buffer->buffer[0]); 2019 } 2020 2021 static int alloc_percpu_trace_buffer(void) 2022 { 2023 struct trace_buffer_struct *buffers; 2024 struct trace_buffer_struct *sirq_buffers; 2025 struct trace_buffer_struct *irq_buffers; 2026 struct trace_buffer_struct *nmi_buffers; 2027 2028 buffers = alloc_percpu(struct trace_buffer_struct); 2029 if (!buffers) 2030 goto err_warn; 2031 2032 sirq_buffers = alloc_percpu(struct trace_buffer_struct); 2033 if (!sirq_buffers) 2034 goto err_sirq; 2035 2036 irq_buffers = alloc_percpu(struct trace_buffer_struct); 2037 if (!irq_buffers) 2038 goto err_irq; 2039 2040 nmi_buffers = alloc_percpu(struct trace_buffer_struct); 2041 if (!nmi_buffers) 2042 goto err_nmi; 2043 2044 trace_percpu_buffer = buffers; 2045 trace_percpu_sirq_buffer = sirq_buffers; 2046 trace_percpu_irq_buffer = irq_buffers; 2047 trace_percpu_nmi_buffer = nmi_buffers; 2048 2049 return 0; 2050 2051 err_nmi: 2052 free_percpu(irq_buffers); 2053 err_irq: 2054 free_percpu(sirq_buffers); 2055 err_sirq: 2056 free_percpu(buffers); 2057 err_warn: 2058 WARN(1, "Could not allocate percpu trace_printk buffer"); 2059 return -ENOMEM; 2060 } 2061 2062 static int buffers_allocated; 2063 2064 void trace_printk_init_buffers(void) 2065 { 2066 if (buffers_allocated) 2067 return; 2068 2069 if (alloc_percpu_trace_buffer()) 2070 return; 2071 2072 /* trace_printk() is for debug use only. Don't use it in production. */ 2073 2074 pr_warning("\n"); 2075 pr_warning("**********************************************************\n"); 2076 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 2077 pr_warning("** **\n"); 2078 pr_warning("** trace_printk() being used. Allocating extra memory. **\n"); 2079 pr_warning("** **\n"); 2080 pr_warning("** This means that this is a DEBUG kernel and it is **\n"); 2081 pr_warning("** unsafe for production use. **\n"); 2082 pr_warning("** **\n"); 2083 pr_warning("** If you see this message and you are not debugging **\n"); 2084 pr_warning("** the kernel, report this immediately to your vendor! **\n"); 2085 pr_warning("** **\n"); 2086 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 2087 pr_warning("**********************************************************\n"); 2088 2089 /* Expand the buffers to set size */ 2090 tracing_update_buffers(); 2091 2092 buffers_allocated = 1; 2093 2094 /* 2095 * trace_printk_init_buffers() can be called by modules. 2096 * If that happens, then we need to start cmdline recording 2097 * directly here. If the global_trace.buffer is already 2098 * allocated here, then this was called by module code. 2099 */ 2100 if (global_trace.trace_buffer.buffer) 2101 tracing_start_cmdline_record(); 2102 } 2103 2104 void trace_printk_start_comm(void) 2105 { 2106 /* Start tracing comms if trace printk is set */ 2107 if (!buffers_allocated) 2108 return; 2109 tracing_start_cmdline_record(); 2110 } 2111 2112 static void trace_printk_start_stop_comm(int enabled) 2113 { 2114 if (!buffers_allocated) 2115 return; 2116 2117 if (enabled) 2118 tracing_start_cmdline_record(); 2119 else 2120 tracing_stop_cmdline_record(); 2121 } 2122 2123 /** 2124 * trace_vbprintk - write binary msg to tracing buffer 2125 * 2126 */ 2127 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) 2128 { 2129 struct trace_event_call *call = &event_bprint; 2130 struct ring_buffer_event *event; 2131 struct ring_buffer *buffer; 2132 struct trace_array *tr = &global_trace; 2133 struct bprint_entry *entry; 2134 unsigned long flags; 2135 char *tbuffer; 2136 int len = 0, size, pc; 2137 2138 if (unlikely(tracing_selftest_running || tracing_disabled)) 2139 return 0; 2140 2141 /* Don't pollute graph traces with trace_vprintk internals */ 2142 pause_graph_tracing(); 2143 2144 pc = preempt_count(); 2145 preempt_disable_notrace(); 2146 2147 tbuffer = get_trace_buf(); 2148 if (!tbuffer) { 2149 len = 0; 2150 goto out; 2151 } 2152 2153 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args); 2154 2155 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0) 2156 goto out; 2157 2158 local_save_flags(flags); 2159 size = sizeof(*entry) + sizeof(u32) * len; 2160 buffer = tr->trace_buffer.buffer; 2161 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, 2162 flags, pc); 2163 if (!event) 2164 goto out; 2165 entry = ring_buffer_event_data(event); 2166 entry->ip = ip; 2167 entry->fmt = fmt; 2168 2169 memcpy(entry->buf, tbuffer, sizeof(u32) * len); 2170 if (!call_filter_check_discard(call, entry, buffer, event)) { 2171 __buffer_unlock_commit(buffer, event); 2172 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL); 2173 } 2174 2175 out: 2176 preempt_enable_notrace(); 2177 unpause_graph_tracing(); 2178 2179 return len; 2180 } 2181 EXPORT_SYMBOL_GPL(trace_vbprintk); 2182 2183 static int 2184 __trace_array_vprintk(struct ring_buffer *buffer, 2185 unsigned long ip, const char *fmt, va_list args) 2186 { 2187 struct trace_event_call *call = &event_print; 2188 struct ring_buffer_event *event; 2189 int len = 0, size, pc; 2190 struct print_entry *entry; 2191 unsigned long flags; 2192 char *tbuffer; 2193 2194 if (tracing_disabled || tracing_selftest_running) 2195 return 0; 2196 2197 /* Don't pollute graph traces with trace_vprintk internals */ 2198 pause_graph_tracing(); 2199 2200 pc = preempt_count(); 2201 preempt_disable_notrace(); 2202 2203 2204 tbuffer = get_trace_buf(); 2205 if (!tbuffer) { 2206 len = 0; 2207 goto out; 2208 } 2209 2210 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args); 2211 2212 local_save_flags(flags); 2213 size = sizeof(*entry) + len + 1; 2214 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 2215 flags, pc); 2216 if (!event) 2217 goto out; 2218 entry = ring_buffer_event_data(event); 2219 entry->ip = ip; 2220 2221 memcpy(&entry->buf, tbuffer, len + 1); 2222 if (!call_filter_check_discard(call, entry, buffer, event)) { 2223 __buffer_unlock_commit(buffer, event); 2224 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL); 2225 } 2226 out: 2227 preempt_enable_notrace(); 2228 unpause_graph_tracing(); 2229 2230 return len; 2231 } 2232 2233 int trace_array_vprintk(struct trace_array *tr, 2234 unsigned long ip, const char *fmt, va_list args) 2235 { 2236 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args); 2237 } 2238 2239 int trace_array_printk(struct trace_array *tr, 2240 unsigned long ip, const char *fmt, ...) 2241 { 2242 int ret; 2243 va_list ap; 2244 2245 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 2246 return 0; 2247 2248 va_start(ap, fmt); 2249 ret = trace_array_vprintk(tr, ip, fmt, ap); 2250 va_end(ap); 2251 return ret; 2252 } 2253 2254 int trace_array_printk_buf(struct ring_buffer *buffer, 2255 unsigned long ip, const char *fmt, ...) 2256 { 2257 int ret; 2258 va_list ap; 2259 2260 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 2261 return 0; 2262 2263 va_start(ap, fmt); 2264 ret = __trace_array_vprintk(buffer, ip, fmt, ap); 2265 va_end(ap); 2266 return ret; 2267 } 2268 2269 int trace_vprintk(unsigned long ip, const char *fmt, va_list args) 2270 { 2271 return trace_array_vprintk(&global_trace, ip, fmt, args); 2272 } 2273 EXPORT_SYMBOL_GPL(trace_vprintk); 2274 2275 static void trace_iterator_increment(struct trace_iterator *iter) 2276 { 2277 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu); 2278 2279 iter->idx++; 2280 if (buf_iter) 2281 ring_buffer_read(buf_iter, NULL); 2282 } 2283 2284 static struct trace_entry * 2285 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts, 2286 unsigned long *lost_events) 2287 { 2288 struct ring_buffer_event *event; 2289 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu); 2290 2291 if (buf_iter) 2292 event = ring_buffer_iter_peek(buf_iter, ts); 2293 else 2294 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts, 2295 lost_events); 2296 2297 if (event) { 2298 iter->ent_size = ring_buffer_event_length(event); 2299 return ring_buffer_event_data(event); 2300 } 2301 iter->ent_size = 0; 2302 return NULL; 2303 } 2304 2305 static struct trace_entry * 2306 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, 2307 unsigned long *missing_events, u64 *ent_ts) 2308 { 2309 struct ring_buffer *buffer = iter->trace_buffer->buffer; 2310 struct trace_entry *ent, *next = NULL; 2311 unsigned long lost_events = 0, next_lost = 0; 2312 int cpu_file = iter->cpu_file; 2313 u64 next_ts = 0, ts; 2314 int next_cpu = -1; 2315 int next_size = 0; 2316 int cpu; 2317 2318 /* 2319 * If we are in a per_cpu trace file, don't bother by iterating over 2320 * all cpu and peek directly. 2321 */ 2322 if (cpu_file > RING_BUFFER_ALL_CPUS) { 2323 if (ring_buffer_empty_cpu(buffer, cpu_file)) 2324 return NULL; 2325 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events); 2326 if (ent_cpu) 2327 *ent_cpu = cpu_file; 2328 2329 return ent; 2330 } 2331 2332 for_each_tracing_cpu(cpu) { 2333 2334 if (ring_buffer_empty_cpu(buffer, cpu)) 2335 continue; 2336 2337 ent = peek_next_entry(iter, cpu, &ts, &lost_events); 2338 2339 /* 2340 * Pick the entry with the smallest timestamp: 2341 */ 2342 if (ent && (!next || ts < next_ts)) { 2343 next = ent; 2344 next_cpu = cpu; 2345 next_ts = ts; 2346 next_lost = lost_events; 2347 next_size = iter->ent_size; 2348 } 2349 } 2350 2351 iter->ent_size = next_size; 2352 2353 if (ent_cpu) 2354 *ent_cpu = next_cpu; 2355 2356 if (ent_ts) 2357 *ent_ts = next_ts; 2358 2359 if (missing_events) 2360 *missing_events = next_lost; 2361 2362 return next; 2363 } 2364 2365 /* Find the next real entry, without updating the iterator itself */ 2366 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter, 2367 int *ent_cpu, u64 *ent_ts) 2368 { 2369 return __find_next_entry(iter, ent_cpu, NULL, ent_ts); 2370 } 2371 2372 /* Find the next real entry, and increment the iterator to the next entry */ 2373 void *trace_find_next_entry_inc(struct trace_iterator *iter) 2374 { 2375 iter->ent = __find_next_entry(iter, &iter->cpu, 2376 &iter->lost_events, &iter->ts); 2377 2378 if (iter->ent) 2379 trace_iterator_increment(iter); 2380 2381 return iter->ent ? iter : NULL; 2382 } 2383 2384 static void trace_consume(struct trace_iterator *iter) 2385 { 2386 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts, 2387 &iter->lost_events); 2388 } 2389 2390 static void *s_next(struct seq_file *m, void *v, loff_t *pos) 2391 { 2392 struct trace_iterator *iter = m->private; 2393 int i = (int)*pos; 2394 void *ent; 2395 2396 WARN_ON_ONCE(iter->leftover); 2397 2398 (*pos)++; 2399 2400 /* can't go backwards */ 2401 if (iter->idx > i) 2402 return NULL; 2403 2404 if (iter->idx < 0) 2405 ent = trace_find_next_entry_inc(iter); 2406 else 2407 ent = iter; 2408 2409 while (ent && iter->idx < i) 2410 ent = trace_find_next_entry_inc(iter); 2411 2412 iter->pos = *pos; 2413 2414 return ent; 2415 } 2416 2417 void tracing_iter_reset(struct trace_iterator *iter, int cpu) 2418 { 2419 struct ring_buffer_event *event; 2420 struct ring_buffer_iter *buf_iter; 2421 unsigned long entries = 0; 2422 u64 ts; 2423 2424 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0; 2425 2426 buf_iter = trace_buffer_iter(iter, cpu); 2427 if (!buf_iter) 2428 return; 2429 2430 ring_buffer_iter_reset(buf_iter); 2431 2432 /* 2433 * We could have the case with the max latency tracers 2434 * that a reset never took place on a cpu. This is evident 2435 * by the timestamp being before the start of the buffer. 2436 */ 2437 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) { 2438 if (ts >= iter->trace_buffer->time_start) 2439 break; 2440 entries++; 2441 ring_buffer_read(buf_iter, NULL); 2442 } 2443 2444 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries; 2445 } 2446 2447 /* 2448 * The current tracer is copied to avoid a global locking 2449 * all around. 2450 */ 2451 static void *s_start(struct seq_file *m, loff_t *pos) 2452 { 2453 struct trace_iterator *iter = m->private; 2454 struct trace_array *tr = iter->tr; 2455 int cpu_file = iter->cpu_file; 2456 void *p = NULL; 2457 loff_t l = 0; 2458 int cpu; 2459 2460 /* 2461 * copy the tracer to avoid using a global lock all around. 2462 * iter->trace is a copy of current_trace, the pointer to the 2463 * name may be used instead of a strcmp(), as iter->trace->name 2464 * will point to the same string as current_trace->name. 2465 */ 2466 mutex_lock(&trace_types_lock); 2467 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name)) 2468 *iter->trace = *tr->current_trace; 2469 mutex_unlock(&trace_types_lock); 2470 2471 #ifdef CONFIG_TRACER_MAX_TRACE 2472 if (iter->snapshot && iter->trace->use_max_tr) 2473 return ERR_PTR(-EBUSY); 2474 #endif 2475 2476 if (!iter->snapshot) 2477 atomic_inc(&trace_record_cmdline_disabled); 2478 2479 if (*pos != iter->pos) { 2480 iter->ent = NULL; 2481 iter->cpu = 0; 2482 iter->idx = -1; 2483 2484 if (cpu_file == RING_BUFFER_ALL_CPUS) { 2485 for_each_tracing_cpu(cpu) 2486 tracing_iter_reset(iter, cpu); 2487 } else 2488 tracing_iter_reset(iter, cpu_file); 2489 2490 iter->leftover = 0; 2491 for (p = iter; p && l < *pos; p = s_next(m, p, &l)) 2492 ; 2493 2494 } else { 2495 /* 2496 * If we overflowed the seq_file before, then we want 2497 * to just reuse the trace_seq buffer again. 2498 */ 2499 if (iter->leftover) 2500 p = iter; 2501 else { 2502 l = *pos - 1; 2503 p = s_next(m, p, &l); 2504 } 2505 } 2506 2507 trace_event_read_lock(); 2508 trace_access_lock(cpu_file); 2509 return p; 2510 } 2511 2512 static void s_stop(struct seq_file *m, void *p) 2513 { 2514 struct trace_iterator *iter = m->private; 2515 2516 #ifdef CONFIG_TRACER_MAX_TRACE 2517 if (iter->snapshot && iter->trace->use_max_tr) 2518 return; 2519 #endif 2520 2521 if (!iter->snapshot) 2522 atomic_dec(&trace_record_cmdline_disabled); 2523 2524 trace_access_unlock(iter->cpu_file); 2525 trace_event_read_unlock(); 2526 } 2527 2528 static void 2529 get_total_entries(struct trace_buffer *buf, 2530 unsigned long *total, unsigned long *entries) 2531 { 2532 unsigned long count; 2533 int cpu; 2534 2535 *total = 0; 2536 *entries = 0; 2537 2538 for_each_tracing_cpu(cpu) { 2539 count = ring_buffer_entries_cpu(buf->buffer, cpu); 2540 /* 2541 * If this buffer has skipped entries, then we hold all 2542 * entries for the trace and we need to ignore the 2543 * ones before the time stamp. 2544 */ 2545 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) { 2546 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries; 2547 /* total is the same as the entries */ 2548 *total += count; 2549 } else 2550 *total += count + 2551 ring_buffer_overrun_cpu(buf->buffer, cpu); 2552 *entries += count; 2553 } 2554 } 2555 2556 static void print_lat_help_header(struct seq_file *m) 2557 { 2558 seq_puts(m, "# _------=> CPU# \n" 2559 "# / _-----=> irqs-off \n" 2560 "# | / _----=> need-resched \n" 2561 "# || / _---=> hardirq/softirq \n" 2562 "# ||| / _--=> preempt-depth \n" 2563 "# |||| / delay \n" 2564 "# cmd pid ||||| time | caller \n" 2565 "# \\ / ||||| \\ | / \n"); 2566 } 2567 2568 static void print_event_info(struct trace_buffer *buf, struct seq_file *m) 2569 { 2570 unsigned long total; 2571 unsigned long entries; 2572 2573 get_total_entries(buf, &total, &entries); 2574 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n", 2575 entries, total, num_online_cpus()); 2576 seq_puts(m, "#\n"); 2577 } 2578 2579 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m) 2580 { 2581 print_event_info(buf, m); 2582 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n" 2583 "# | | | | |\n"); 2584 } 2585 2586 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m) 2587 { 2588 print_event_info(buf, m); 2589 seq_puts(m, "# _-----=> irqs-off\n" 2590 "# / _----=> need-resched\n" 2591 "# | / _---=> hardirq/softirq\n" 2592 "# || / _--=> preempt-depth\n" 2593 "# ||| / delay\n" 2594 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n" 2595 "# | | | |||| | |\n"); 2596 } 2597 2598 void 2599 print_trace_header(struct seq_file *m, struct trace_iterator *iter) 2600 { 2601 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK); 2602 struct trace_buffer *buf = iter->trace_buffer; 2603 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu); 2604 struct tracer *type = iter->trace; 2605 unsigned long entries; 2606 unsigned long total; 2607 const char *name = "preemption"; 2608 2609 name = type->name; 2610 2611 get_total_entries(buf, &total, &entries); 2612 2613 seq_printf(m, "# %s latency trace v1.1.5 on %s\n", 2614 name, UTS_RELEASE); 2615 seq_puts(m, "# -----------------------------------" 2616 "---------------------------------\n"); 2617 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |" 2618 " (M:%s VP:%d, KP:%d, SP:%d HP:%d", 2619 nsecs_to_usecs(data->saved_latency), 2620 entries, 2621 total, 2622 buf->cpu, 2623 #if defined(CONFIG_PREEMPT_NONE) 2624 "server", 2625 #elif defined(CONFIG_PREEMPT_VOLUNTARY) 2626 "desktop", 2627 #elif defined(CONFIG_PREEMPT) 2628 "preempt", 2629 #else 2630 "unknown", 2631 #endif 2632 /* These are reserved for later use */ 2633 0, 0, 0, 0); 2634 #ifdef CONFIG_SMP 2635 seq_printf(m, " #P:%d)\n", num_online_cpus()); 2636 #else 2637 seq_puts(m, ")\n"); 2638 #endif 2639 seq_puts(m, "# -----------------\n"); 2640 seq_printf(m, "# | task: %.16s-%d " 2641 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n", 2642 data->comm, data->pid, 2643 from_kuid_munged(seq_user_ns(m), data->uid), data->nice, 2644 data->policy, data->rt_priority); 2645 seq_puts(m, "# -----------------\n"); 2646 2647 if (data->critical_start) { 2648 seq_puts(m, "# => started at: "); 2649 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags); 2650 trace_print_seq(m, &iter->seq); 2651 seq_puts(m, "\n# => ended at: "); 2652 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); 2653 trace_print_seq(m, &iter->seq); 2654 seq_puts(m, "\n#\n"); 2655 } 2656 2657 seq_puts(m, "#\n"); 2658 } 2659 2660 static void test_cpu_buff_start(struct trace_iterator *iter) 2661 { 2662 struct trace_seq *s = &iter->seq; 2663 struct trace_array *tr = iter->tr; 2664 2665 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE)) 2666 return; 2667 2668 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE)) 2669 return; 2670 2671 if (iter->started && cpumask_test_cpu(iter->cpu, iter->started)) 2672 return; 2673 2674 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries) 2675 return; 2676 2677 if (iter->started) 2678 cpumask_set_cpu(iter->cpu, iter->started); 2679 2680 /* Don't print started cpu buffer for the first entry of the trace */ 2681 if (iter->idx > 1) 2682 trace_seq_printf(s, "##### CPU %u buffer started ####\n", 2683 iter->cpu); 2684 } 2685 2686 static enum print_line_t print_trace_fmt(struct trace_iterator *iter) 2687 { 2688 struct trace_array *tr = iter->tr; 2689 struct trace_seq *s = &iter->seq; 2690 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK); 2691 struct trace_entry *entry; 2692 struct trace_event *event; 2693 2694 entry = iter->ent; 2695 2696 test_cpu_buff_start(iter); 2697 2698 event = ftrace_find_event(entry->type); 2699 2700 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 2701 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 2702 trace_print_lat_context(iter); 2703 else 2704 trace_print_context(iter); 2705 } 2706 2707 if (trace_seq_has_overflowed(s)) 2708 return TRACE_TYPE_PARTIAL_LINE; 2709 2710 if (event) 2711 return event->funcs->trace(iter, sym_flags, event); 2712 2713 trace_seq_printf(s, "Unknown type %d\n", entry->type); 2714 2715 return trace_handle_return(s); 2716 } 2717 2718 static enum print_line_t print_raw_fmt(struct trace_iterator *iter) 2719 { 2720 struct trace_array *tr = iter->tr; 2721 struct trace_seq *s = &iter->seq; 2722 struct trace_entry *entry; 2723 struct trace_event *event; 2724 2725 entry = iter->ent; 2726 2727 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) 2728 trace_seq_printf(s, "%d %d %llu ", 2729 entry->pid, iter->cpu, iter->ts); 2730 2731 if (trace_seq_has_overflowed(s)) 2732 return TRACE_TYPE_PARTIAL_LINE; 2733 2734 event = ftrace_find_event(entry->type); 2735 if (event) 2736 return event->funcs->raw(iter, 0, event); 2737 2738 trace_seq_printf(s, "%d ?\n", entry->type); 2739 2740 return trace_handle_return(s); 2741 } 2742 2743 static enum print_line_t print_hex_fmt(struct trace_iterator *iter) 2744 { 2745 struct trace_array *tr = iter->tr; 2746 struct trace_seq *s = &iter->seq; 2747 unsigned char newline = '\n'; 2748 struct trace_entry *entry; 2749 struct trace_event *event; 2750 2751 entry = iter->ent; 2752 2753 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 2754 SEQ_PUT_HEX_FIELD(s, entry->pid); 2755 SEQ_PUT_HEX_FIELD(s, iter->cpu); 2756 SEQ_PUT_HEX_FIELD(s, iter->ts); 2757 if (trace_seq_has_overflowed(s)) 2758 return TRACE_TYPE_PARTIAL_LINE; 2759 } 2760 2761 event = ftrace_find_event(entry->type); 2762 if (event) { 2763 enum print_line_t ret = event->funcs->hex(iter, 0, event); 2764 if (ret != TRACE_TYPE_HANDLED) 2765 return ret; 2766 } 2767 2768 SEQ_PUT_FIELD(s, newline); 2769 2770 return trace_handle_return(s); 2771 } 2772 2773 static enum print_line_t print_bin_fmt(struct trace_iterator *iter) 2774 { 2775 struct trace_array *tr = iter->tr; 2776 struct trace_seq *s = &iter->seq; 2777 struct trace_entry *entry; 2778 struct trace_event *event; 2779 2780 entry = iter->ent; 2781 2782 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 2783 SEQ_PUT_FIELD(s, entry->pid); 2784 SEQ_PUT_FIELD(s, iter->cpu); 2785 SEQ_PUT_FIELD(s, iter->ts); 2786 if (trace_seq_has_overflowed(s)) 2787 return TRACE_TYPE_PARTIAL_LINE; 2788 } 2789 2790 event = ftrace_find_event(entry->type); 2791 return event ? event->funcs->binary(iter, 0, event) : 2792 TRACE_TYPE_HANDLED; 2793 } 2794 2795 int trace_empty(struct trace_iterator *iter) 2796 { 2797 struct ring_buffer_iter *buf_iter; 2798 int cpu; 2799 2800 /* If we are looking at one CPU buffer, only check that one */ 2801 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 2802 cpu = iter->cpu_file; 2803 buf_iter = trace_buffer_iter(iter, cpu); 2804 if (buf_iter) { 2805 if (!ring_buffer_iter_empty(buf_iter)) 2806 return 0; 2807 } else { 2808 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu)) 2809 return 0; 2810 } 2811 return 1; 2812 } 2813 2814 for_each_tracing_cpu(cpu) { 2815 buf_iter = trace_buffer_iter(iter, cpu); 2816 if (buf_iter) { 2817 if (!ring_buffer_iter_empty(buf_iter)) 2818 return 0; 2819 } else { 2820 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu)) 2821 return 0; 2822 } 2823 } 2824 2825 return 1; 2826 } 2827 2828 /* Called with trace_event_read_lock() held. */ 2829 enum print_line_t print_trace_line(struct trace_iterator *iter) 2830 { 2831 struct trace_array *tr = iter->tr; 2832 unsigned long trace_flags = tr->trace_flags; 2833 enum print_line_t ret; 2834 2835 if (iter->lost_events) { 2836 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", 2837 iter->cpu, iter->lost_events); 2838 if (trace_seq_has_overflowed(&iter->seq)) 2839 return TRACE_TYPE_PARTIAL_LINE; 2840 } 2841 2842 if (iter->trace && iter->trace->print_line) { 2843 ret = iter->trace->print_line(iter); 2844 if (ret != TRACE_TYPE_UNHANDLED) 2845 return ret; 2846 } 2847 2848 if (iter->ent->type == TRACE_BPUTS && 2849 trace_flags & TRACE_ITER_PRINTK && 2850 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 2851 return trace_print_bputs_msg_only(iter); 2852 2853 if (iter->ent->type == TRACE_BPRINT && 2854 trace_flags & TRACE_ITER_PRINTK && 2855 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 2856 return trace_print_bprintk_msg_only(iter); 2857 2858 if (iter->ent->type == TRACE_PRINT && 2859 trace_flags & TRACE_ITER_PRINTK && 2860 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 2861 return trace_print_printk_msg_only(iter); 2862 2863 if (trace_flags & TRACE_ITER_BIN) 2864 return print_bin_fmt(iter); 2865 2866 if (trace_flags & TRACE_ITER_HEX) 2867 return print_hex_fmt(iter); 2868 2869 if (trace_flags & TRACE_ITER_RAW) 2870 return print_raw_fmt(iter); 2871 2872 return print_trace_fmt(iter); 2873 } 2874 2875 void trace_latency_header(struct seq_file *m) 2876 { 2877 struct trace_iterator *iter = m->private; 2878 struct trace_array *tr = iter->tr; 2879 2880 /* print nothing if the buffers are empty */ 2881 if (trace_empty(iter)) 2882 return; 2883 2884 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 2885 print_trace_header(m, iter); 2886 2887 if (!(tr->trace_flags & TRACE_ITER_VERBOSE)) 2888 print_lat_help_header(m); 2889 } 2890 2891 void trace_default_header(struct seq_file *m) 2892 { 2893 struct trace_iterator *iter = m->private; 2894 struct trace_array *tr = iter->tr; 2895 unsigned long trace_flags = tr->trace_flags; 2896 2897 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO)) 2898 return; 2899 2900 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { 2901 /* print nothing if the buffers are empty */ 2902 if (trace_empty(iter)) 2903 return; 2904 print_trace_header(m, iter); 2905 if (!(trace_flags & TRACE_ITER_VERBOSE)) 2906 print_lat_help_header(m); 2907 } else { 2908 if (!(trace_flags & TRACE_ITER_VERBOSE)) { 2909 if (trace_flags & TRACE_ITER_IRQ_INFO) 2910 print_func_help_header_irq(iter->trace_buffer, m); 2911 else 2912 print_func_help_header(iter->trace_buffer, m); 2913 } 2914 } 2915 } 2916 2917 static void test_ftrace_alive(struct seq_file *m) 2918 { 2919 if (!ftrace_is_dead()) 2920 return; 2921 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n" 2922 "# MAY BE MISSING FUNCTION EVENTS\n"); 2923 } 2924 2925 #ifdef CONFIG_TRACER_MAX_TRACE 2926 static void show_snapshot_main_help(struct seq_file *m) 2927 { 2928 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n" 2929 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" 2930 "# Takes a snapshot of the main buffer.\n" 2931 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n" 2932 "# (Doesn't have to be '2' works with any number that\n" 2933 "# is not a '0' or '1')\n"); 2934 } 2935 2936 static void show_snapshot_percpu_help(struct seq_file *m) 2937 { 2938 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n"); 2939 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP 2940 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" 2941 "# Takes a snapshot of the main buffer for this cpu.\n"); 2942 #else 2943 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n" 2944 "# Must use main snapshot file to allocate.\n"); 2945 #endif 2946 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n" 2947 "# (Doesn't have to be '2' works with any number that\n" 2948 "# is not a '0' or '1')\n"); 2949 } 2950 2951 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) 2952 { 2953 if (iter->tr->allocated_snapshot) 2954 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n"); 2955 else 2956 seq_puts(m, "#\n# * Snapshot is freed *\n#\n"); 2957 2958 seq_puts(m, "# Snapshot commands:\n"); 2959 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 2960 show_snapshot_main_help(m); 2961 else 2962 show_snapshot_percpu_help(m); 2963 } 2964 #else 2965 /* Should never be called */ 2966 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { } 2967 #endif 2968 2969 static int s_show(struct seq_file *m, void *v) 2970 { 2971 struct trace_iterator *iter = v; 2972 int ret; 2973 2974 if (iter->ent == NULL) { 2975 if (iter->tr) { 2976 seq_printf(m, "# tracer: %s\n", iter->trace->name); 2977 seq_puts(m, "#\n"); 2978 test_ftrace_alive(m); 2979 } 2980 if (iter->snapshot && trace_empty(iter)) 2981 print_snapshot_help(m, iter); 2982 else if (iter->trace && iter->trace->print_header) 2983 iter->trace->print_header(m); 2984 else 2985 trace_default_header(m); 2986 2987 } else if (iter->leftover) { 2988 /* 2989 * If we filled the seq_file buffer earlier, we 2990 * want to just show it now. 2991 */ 2992 ret = trace_print_seq(m, &iter->seq); 2993 2994 /* ret should this time be zero, but you never know */ 2995 iter->leftover = ret; 2996 2997 } else { 2998 print_trace_line(iter); 2999 ret = trace_print_seq(m, &iter->seq); 3000 /* 3001 * If we overflow the seq_file buffer, then it will 3002 * ask us for this data again at start up. 3003 * Use that instead. 3004 * ret is 0 if seq_file write succeeded. 3005 * -1 otherwise. 3006 */ 3007 iter->leftover = ret; 3008 } 3009 3010 return 0; 3011 } 3012 3013 /* 3014 * Should be used after trace_array_get(), trace_types_lock 3015 * ensures that i_cdev was already initialized. 3016 */ 3017 static inline int tracing_get_cpu(struct inode *inode) 3018 { 3019 if (inode->i_cdev) /* See trace_create_cpu_file() */ 3020 return (long)inode->i_cdev - 1; 3021 return RING_BUFFER_ALL_CPUS; 3022 } 3023 3024 static const struct seq_operations tracer_seq_ops = { 3025 .start = s_start, 3026 .next = s_next, 3027 .stop = s_stop, 3028 .show = s_show, 3029 }; 3030 3031 static struct trace_iterator * 3032 __tracing_open(struct inode *inode, struct file *file, bool snapshot) 3033 { 3034 struct trace_array *tr = inode->i_private; 3035 struct trace_iterator *iter; 3036 int cpu; 3037 3038 if (tracing_disabled) 3039 return ERR_PTR(-ENODEV); 3040 3041 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter)); 3042 if (!iter) 3043 return ERR_PTR(-ENOMEM); 3044 3045 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter), 3046 GFP_KERNEL); 3047 if (!iter->buffer_iter) 3048 goto release; 3049 3050 /* 3051 * We make a copy of the current tracer to avoid concurrent 3052 * changes on it while we are reading. 3053 */ 3054 mutex_lock(&trace_types_lock); 3055 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL); 3056 if (!iter->trace) 3057 goto fail; 3058 3059 *iter->trace = *tr->current_trace; 3060 3061 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL)) 3062 goto fail; 3063 3064 iter->tr = tr; 3065 3066 #ifdef CONFIG_TRACER_MAX_TRACE 3067 /* Currently only the top directory has a snapshot */ 3068 if (tr->current_trace->print_max || snapshot) 3069 iter->trace_buffer = &tr->max_buffer; 3070 else 3071 #endif 3072 iter->trace_buffer = &tr->trace_buffer; 3073 iter->snapshot = snapshot; 3074 iter->pos = -1; 3075 iter->cpu_file = tracing_get_cpu(inode); 3076 mutex_init(&iter->mutex); 3077 3078 /* Notify the tracer early; before we stop tracing. */ 3079 if (iter->trace && iter->trace->open) 3080 iter->trace->open(iter); 3081 3082 /* Annotate start of buffers if we had overruns */ 3083 if (ring_buffer_overruns(iter->trace_buffer->buffer)) 3084 iter->iter_flags |= TRACE_FILE_ANNOTATE; 3085 3086 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 3087 if (trace_clocks[tr->clock_id].in_ns) 3088 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 3089 3090 /* stop the trace while dumping if we are not opening "snapshot" */ 3091 if (!iter->snapshot) 3092 tracing_stop_tr(tr); 3093 3094 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { 3095 for_each_tracing_cpu(cpu) { 3096 iter->buffer_iter[cpu] = 3097 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu); 3098 } 3099 ring_buffer_read_prepare_sync(); 3100 for_each_tracing_cpu(cpu) { 3101 ring_buffer_read_start(iter->buffer_iter[cpu]); 3102 tracing_iter_reset(iter, cpu); 3103 } 3104 } else { 3105 cpu = iter->cpu_file; 3106 iter->buffer_iter[cpu] = 3107 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu); 3108 ring_buffer_read_prepare_sync(); 3109 ring_buffer_read_start(iter->buffer_iter[cpu]); 3110 tracing_iter_reset(iter, cpu); 3111 } 3112 3113 mutex_unlock(&trace_types_lock); 3114 3115 return iter; 3116 3117 fail: 3118 mutex_unlock(&trace_types_lock); 3119 kfree(iter->trace); 3120 kfree(iter->buffer_iter); 3121 release: 3122 seq_release_private(inode, file); 3123 return ERR_PTR(-ENOMEM); 3124 } 3125 3126 int tracing_open_generic(struct inode *inode, struct file *filp) 3127 { 3128 if (tracing_disabled) 3129 return -ENODEV; 3130 3131 filp->private_data = inode->i_private; 3132 return 0; 3133 } 3134 3135 bool tracing_is_disabled(void) 3136 { 3137 return (tracing_disabled) ? true: false; 3138 } 3139 3140 /* 3141 * Open and update trace_array ref count. 3142 * Must have the current trace_array passed to it. 3143 */ 3144 static int tracing_open_generic_tr(struct inode *inode, struct file *filp) 3145 { 3146 struct trace_array *tr = inode->i_private; 3147 3148 if (tracing_disabled) 3149 return -ENODEV; 3150 3151 if (trace_array_get(tr) < 0) 3152 return -ENODEV; 3153 3154 filp->private_data = inode->i_private; 3155 3156 return 0; 3157 } 3158 3159 static int tracing_release(struct inode *inode, struct file *file) 3160 { 3161 struct trace_array *tr = inode->i_private; 3162 struct seq_file *m = file->private_data; 3163 struct trace_iterator *iter; 3164 int cpu; 3165 3166 if (!(file->f_mode & FMODE_READ)) { 3167 trace_array_put(tr); 3168 return 0; 3169 } 3170 3171 /* Writes do not use seq_file */ 3172 iter = m->private; 3173 mutex_lock(&trace_types_lock); 3174 3175 for_each_tracing_cpu(cpu) { 3176 if (iter->buffer_iter[cpu]) 3177 ring_buffer_read_finish(iter->buffer_iter[cpu]); 3178 } 3179 3180 if (iter->trace && iter->trace->close) 3181 iter->trace->close(iter); 3182 3183 if (!iter->snapshot) 3184 /* reenable tracing if it was previously enabled */ 3185 tracing_start_tr(tr); 3186 3187 __trace_array_put(tr); 3188 3189 mutex_unlock(&trace_types_lock); 3190 3191 mutex_destroy(&iter->mutex); 3192 free_cpumask_var(iter->started); 3193 kfree(iter->trace); 3194 kfree(iter->buffer_iter); 3195 seq_release_private(inode, file); 3196 3197 return 0; 3198 } 3199 3200 static int tracing_release_generic_tr(struct inode *inode, struct file *file) 3201 { 3202 struct trace_array *tr = inode->i_private; 3203 3204 trace_array_put(tr); 3205 return 0; 3206 } 3207 3208 static int tracing_single_release_tr(struct inode *inode, struct file *file) 3209 { 3210 struct trace_array *tr = inode->i_private; 3211 3212 trace_array_put(tr); 3213 3214 return single_release(inode, file); 3215 } 3216 3217 static int tracing_open(struct inode *inode, struct file *file) 3218 { 3219 struct trace_array *tr = inode->i_private; 3220 struct trace_iterator *iter; 3221 int ret = 0; 3222 3223 if (trace_array_get(tr) < 0) 3224 return -ENODEV; 3225 3226 /* If this file was open for write, then erase contents */ 3227 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 3228 int cpu = tracing_get_cpu(inode); 3229 3230 if (cpu == RING_BUFFER_ALL_CPUS) 3231 tracing_reset_online_cpus(&tr->trace_buffer); 3232 else 3233 tracing_reset(&tr->trace_buffer, cpu); 3234 } 3235 3236 if (file->f_mode & FMODE_READ) { 3237 iter = __tracing_open(inode, file, false); 3238 if (IS_ERR(iter)) 3239 ret = PTR_ERR(iter); 3240 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 3241 iter->iter_flags |= TRACE_FILE_LAT_FMT; 3242 } 3243 3244 if (ret < 0) 3245 trace_array_put(tr); 3246 3247 return ret; 3248 } 3249 3250 /* 3251 * Some tracers are not suitable for instance buffers. 3252 * A tracer is always available for the global array (toplevel) 3253 * or if it explicitly states that it is. 3254 */ 3255 static bool 3256 trace_ok_for_array(struct tracer *t, struct trace_array *tr) 3257 { 3258 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances; 3259 } 3260 3261 /* Find the next tracer that this trace array may use */ 3262 static struct tracer * 3263 get_tracer_for_array(struct trace_array *tr, struct tracer *t) 3264 { 3265 while (t && !trace_ok_for_array(t, tr)) 3266 t = t->next; 3267 3268 return t; 3269 } 3270 3271 static void * 3272 t_next(struct seq_file *m, void *v, loff_t *pos) 3273 { 3274 struct trace_array *tr = m->private; 3275 struct tracer *t = v; 3276 3277 (*pos)++; 3278 3279 if (t) 3280 t = get_tracer_for_array(tr, t->next); 3281 3282 return t; 3283 } 3284 3285 static void *t_start(struct seq_file *m, loff_t *pos) 3286 { 3287 struct trace_array *tr = m->private; 3288 struct tracer *t; 3289 loff_t l = 0; 3290 3291 mutex_lock(&trace_types_lock); 3292 3293 t = get_tracer_for_array(tr, trace_types); 3294 for (; t && l < *pos; t = t_next(m, t, &l)) 3295 ; 3296 3297 return t; 3298 } 3299 3300 static void t_stop(struct seq_file *m, void *p) 3301 { 3302 mutex_unlock(&trace_types_lock); 3303 } 3304 3305 static int t_show(struct seq_file *m, void *v) 3306 { 3307 struct tracer *t = v; 3308 3309 if (!t) 3310 return 0; 3311 3312 seq_puts(m, t->name); 3313 if (t->next) 3314 seq_putc(m, ' '); 3315 else 3316 seq_putc(m, '\n'); 3317 3318 return 0; 3319 } 3320 3321 static const struct seq_operations show_traces_seq_ops = { 3322 .start = t_start, 3323 .next = t_next, 3324 .stop = t_stop, 3325 .show = t_show, 3326 }; 3327 3328 static int show_traces_open(struct inode *inode, struct file *file) 3329 { 3330 struct trace_array *tr = inode->i_private; 3331 struct seq_file *m; 3332 int ret; 3333 3334 if (tracing_disabled) 3335 return -ENODEV; 3336 3337 ret = seq_open(file, &show_traces_seq_ops); 3338 if (ret) 3339 return ret; 3340 3341 m = file->private_data; 3342 m->private = tr; 3343 3344 return 0; 3345 } 3346 3347 static ssize_t 3348 tracing_write_stub(struct file *filp, const char __user *ubuf, 3349 size_t count, loff_t *ppos) 3350 { 3351 return count; 3352 } 3353 3354 loff_t tracing_lseek(struct file *file, loff_t offset, int whence) 3355 { 3356 int ret; 3357 3358 if (file->f_mode & FMODE_READ) 3359 ret = seq_lseek(file, offset, whence); 3360 else 3361 file->f_pos = ret = 0; 3362 3363 return ret; 3364 } 3365 3366 static const struct file_operations tracing_fops = { 3367 .open = tracing_open, 3368 .read = seq_read, 3369 .write = tracing_write_stub, 3370 .llseek = tracing_lseek, 3371 .release = tracing_release, 3372 }; 3373 3374 static const struct file_operations show_traces_fops = { 3375 .open = show_traces_open, 3376 .read = seq_read, 3377 .release = seq_release, 3378 .llseek = seq_lseek, 3379 }; 3380 3381 /* 3382 * The tracer itself will not take this lock, but still we want 3383 * to provide a consistent cpumask to user-space: 3384 */ 3385 static DEFINE_MUTEX(tracing_cpumask_update_lock); 3386 3387 /* 3388 * Temporary storage for the character representation of the 3389 * CPU bitmask (and one more byte for the newline): 3390 */ 3391 static char mask_str[NR_CPUS + 1]; 3392 3393 static ssize_t 3394 tracing_cpumask_read(struct file *filp, char __user *ubuf, 3395 size_t count, loff_t *ppos) 3396 { 3397 struct trace_array *tr = file_inode(filp)->i_private; 3398 int len; 3399 3400 mutex_lock(&tracing_cpumask_update_lock); 3401 3402 len = snprintf(mask_str, count, "%*pb\n", 3403 cpumask_pr_args(tr->tracing_cpumask)); 3404 if (len >= count) { 3405 count = -EINVAL; 3406 goto out_err; 3407 } 3408 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1); 3409 3410 out_err: 3411 mutex_unlock(&tracing_cpumask_update_lock); 3412 3413 return count; 3414 } 3415 3416 static ssize_t 3417 tracing_cpumask_write(struct file *filp, const char __user *ubuf, 3418 size_t count, loff_t *ppos) 3419 { 3420 struct trace_array *tr = file_inode(filp)->i_private; 3421 cpumask_var_t tracing_cpumask_new; 3422 int err, cpu; 3423 3424 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL)) 3425 return -ENOMEM; 3426 3427 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new); 3428 if (err) 3429 goto err_unlock; 3430 3431 mutex_lock(&tracing_cpumask_update_lock); 3432 3433 local_irq_disable(); 3434 arch_spin_lock(&tr->max_lock); 3435 for_each_tracing_cpu(cpu) { 3436 /* 3437 * Increase/decrease the disabled counter if we are 3438 * about to flip a bit in the cpumask: 3439 */ 3440 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) && 3441 !cpumask_test_cpu(cpu, tracing_cpumask_new)) { 3442 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled); 3443 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu); 3444 } 3445 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) && 3446 cpumask_test_cpu(cpu, tracing_cpumask_new)) { 3447 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled); 3448 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu); 3449 } 3450 } 3451 arch_spin_unlock(&tr->max_lock); 3452 local_irq_enable(); 3453 3454 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new); 3455 3456 mutex_unlock(&tracing_cpumask_update_lock); 3457 free_cpumask_var(tracing_cpumask_new); 3458 3459 return count; 3460 3461 err_unlock: 3462 free_cpumask_var(tracing_cpumask_new); 3463 3464 return err; 3465 } 3466 3467 static const struct file_operations tracing_cpumask_fops = { 3468 .open = tracing_open_generic_tr, 3469 .read = tracing_cpumask_read, 3470 .write = tracing_cpumask_write, 3471 .release = tracing_release_generic_tr, 3472 .llseek = generic_file_llseek, 3473 }; 3474 3475 static int tracing_trace_options_show(struct seq_file *m, void *v) 3476 { 3477 struct tracer_opt *trace_opts; 3478 struct trace_array *tr = m->private; 3479 u32 tracer_flags; 3480 int i; 3481 3482 mutex_lock(&trace_types_lock); 3483 tracer_flags = tr->current_trace->flags->val; 3484 trace_opts = tr->current_trace->flags->opts; 3485 3486 for (i = 0; trace_options[i]; i++) { 3487 if (tr->trace_flags & (1 << i)) 3488 seq_printf(m, "%s\n", trace_options[i]); 3489 else 3490 seq_printf(m, "no%s\n", trace_options[i]); 3491 } 3492 3493 for (i = 0; trace_opts[i].name; i++) { 3494 if (tracer_flags & trace_opts[i].bit) 3495 seq_printf(m, "%s\n", trace_opts[i].name); 3496 else 3497 seq_printf(m, "no%s\n", trace_opts[i].name); 3498 } 3499 mutex_unlock(&trace_types_lock); 3500 3501 return 0; 3502 } 3503 3504 static int __set_tracer_option(struct trace_array *tr, 3505 struct tracer_flags *tracer_flags, 3506 struct tracer_opt *opts, int neg) 3507 { 3508 struct tracer *trace = tr->current_trace; 3509 int ret; 3510 3511 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg); 3512 if (ret) 3513 return ret; 3514 3515 if (neg) 3516 tracer_flags->val &= ~opts->bit; 3517 else 3518 tracer_flags->val |= opts->bit; 3519 return 0; 3520 } 3521 3522 /* Try to assign a tracer specific option */ 3523 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg) 3524 { 3525 struct tracer *trace = tr->current_trace; 3526 struct tracer_flags *tracer_flags = trace->flags; 3527 struct tracer_opt *opts = NULL; 3528 int i; 3529 3530 for (i = 0; tracer_flags->opts[i].name; i++) { 3531 opts = &tracer_flags->opts[i]; 3532 3533 if (strcmp(cmp, opts->name) == 0) 3534 return __set_tracer_option(tr, trace->flags, opts, neg); 3535 } 3536 3537 return -EINVAL; 3538 } 3539 3540 /* Some tracers require overwrite to stay enabled */ 3541 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set) 3542 { 3543 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set) 3544 return -1; 3545 3546 return 0; 3547 } 3548 3549 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled) 3550 { 3551 /* do nothing if flag is already set */ 3552 if (!!(tr->trace_flags & mask) == !!enabled) 3553 return 0; 3554 3555 /* Give the tracer a chance to approve the change */ 3556 if (tr->current_trace->flag_changed) 3557 if (tr->current_trace->flag_changed(tr, mask, !!enabled)) 3558 return -EINVAL; 3559 3560 if (enabled) 3561 tr->trace_flags |= mask; 3562 else 3563 tr->trace_flags &= ~mask; 3564 3565 if (mask == TRACE_ITER_RECORD_CMD) 3566 trace_event_enable_cmd_record(enabled); 3567 3568 if (mask == TRACE_ITER_OVERWRITE) { 3569 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled); 3570 #ifdef CONFIG_TRACER_MAX_TRACE 3571 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled); 3572 #endif 3573 } 3574 3575 if (mask == TRACE_ITER_PRINTK) { 3576 trace_printk_start_stop_comm(enabled); 3577 trace_printk_control(enabled); 3578 } 3579 3580 return 0; 3581 } 3582 3583 static int trace_set_options(struct trace_array *tr, char *option) 3584 { 3585 char *cmp; 3586 int neg = 0; 3587 int ret = -ENODEV; 3588 int i; 3589 size_t orig_len = strlen(option); 3590 3591 cmp = strstrip(option); 3592 3593 if (strncmp(cmp, "no", 2) == 0) { 3594 neg = 1; 3595 cmp += 2; 3596 } 3597 3598 mutex_lock(&trace_types_lock); 3599 3600 for (i = 0; trace_options[i]; i++) { 3601 if (strcmp(cmp, trace_options[i]) == 0) { 3602 ret = set_tracer_flag(tr, 1 << i, !neg); 3603 break; 3604 } 3605 } 3606 3607 /* If no option could be set, test the specific tracer options */ 3608 if (!trace_options[i]) 3609 ret = set_tracer_option(tr, cmp, neg); 3610 3611 mutex_unlock(&trace_types_lock); 3612 3613 /* 3614 * If the first trailing whitespace is replaced with '\0' by strstrip, 3615 * turn it back into a space. 3616 */ 3617 if (orig_len > strlen(option)) 3618 option[strlen(option)] = ' '; 3619 3620 return ret; 3621 } 3622 3623 static void __init apply_trace_boot_options(void) 3624 { 3625 char *buf = trace_boot_options_buf; 3626 char *option; 3627 3628 while (true) { 3629 option = strsep(&buf, ","); 3630 3631 if (!option) 3632 break; 3633 3634 if (*option) 3635 trace_set_options(&global_trace, option); 3636 3637 /* Put back the comma to allow this to be called again */ 3638 if (buf) 3639 *(buf - 1) = ','; 3640 } 3641 } 3642 3643 static ssize_t 3644 tracing_trace_options_write(struct file *filp, const char __user *ubuf, 3645 size_t cnt, loff_t *ppos) 3646 { 3647 struct seq_file *m = filp->private_data; 3648 struct trace_array *tr = m->private; 3649 char buf[64]; 3650 int ret; 3651 3652 if (cnt >= sizeof(buf)) 3653 return -EINVAL; 3654 3655 if (copy_from_user(&buf, ubuf, cnt)) 3656 return -EFAULT; 3657 3658 buf[cnt] = 0; 3659 3660 ret = trace_set_options(tr, buf); 3661 if (ret < 0) 3662 return ret; 3663 3664 *ppos += cnt; 3665 3666 return cnt; 3667 } 3668 3669 static int tracing_trace_options_open(struct inode *inode, struct file *file) 3670 { 3671 struct trace_array *tr = inode->i_private; 3672 int ret; 3673 3674 if (tracing_disabled) 3675 return -ENODEV; 3676 3677 if (trace_array_get(tr) < 0) 3678 return -ENODEV; 3679 3680 ret = single_open(file, tracing_trace_options_show, inode->i_private); 3681 if (ret < 0) 3682 trace_array_put(tr); 3683 3684 return ret; 3685 } 3686 3687 static const struct file_operations tracing_iter_fops = { 3688 .open = tracing_trace_options_open, 3689 .read = seq_read, 3690 .llseek = seq_lseek, 3691 .release = tracing_single_release_tr, 3692 .write = tracing_trace_options_write, 3693 }; 3694 3695 static const char readme_msg[] = 3696 "tracing mini-HOWTO:\n\n" 3697 "# echo 0 > tracing_on : quick way to disable tracing\n" 3698 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n" 3699 " Important files:\n" 3700 " trace\t\t\t- The static contents of the buffer\n" 3701 "\t\t\t To clear the buffer write into this file: echo > trace\n" 3702 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n" 3703 " current_tracer\t- function and latency tracers\n" 3704 " available_tracers\t- list of configured tracers for current_tracer\n" 3705 " buffer_size_kb\t- view and modify size of per cpu buffer\n" 3706 " buffer_total_size_kb - view total size of all cpu buffers\n\n" 3707 " trace_clock\t\t-change the clock used to order events\n" 3708 " local: Per cpu clock but may not be synced across CPUs\n" 3709 " global: Synced across CPUs but slows tracing down.\n" 3710 " counter: Not a clock, but just an increment\n" 3711 " uptime: Jiffy counter from time of boot\n" 3712 " perf: Same clock that perf events use\n" 3713 #ifdef CONFIG_X86_64 3714 " x86-tsc: TSC cycle counter\n" 3715 #endif 3716 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n" 3717 " tracing_cpumask\t- Limit which CPUs to trace\n" 3718 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n" 3719 "\t\t\t Remove sub-buffer with rmdir\n" 3720 " trace_options\t\t- Set format or modify how tracing happens\n" 3721 "\t\t\t Disable an option by adding a suffix 'no' to the\n" 3722 "\t\t\t option name\n" 3723 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n" 3724 #ifdef CONFIG_DYNAMIC_FTRACE 3725 "\n available_filter_functions - list of functions that can be filtered on\n" 3726 " set_ftrace_filter\t- echo function name in here to only trace these\n" 3727 "\t\t\t functions\n" 3728 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n" 3729 "\t modules: Can select a group via module\n" 3730 "\t Format: :mod:<module-name>\n" 3731 "\t example: echo :mod:ext3 > set_ftrace_filter\n" 3732 "\t triggers: a command to perform when function is hit\n" 3733 "\t Format: <function>:<trigger>[:count]\n" 3734 "\t trigger: traceon, traceoff\n" 3735 "\t\t enable_event:<system>:<event>\n" 3736 "\t\t disable_event:<system>:<event>\n" 3737 #ifdef CONFIG_STACKTRACE 3738 "\t\t stacktrace\n" 3739 #endif 3740 #ifdef CONFIG_TRACER_SNAPSHOT 3741 "\t\t snapshot\n" 3742 #endif 3743 "\t\t dump\n" 3744 "\t\t cpudump\n" 3745 "\t example: echo do_fault:traceoff > set_ftrace_filter\n" 3746 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n" 3747 "\t The first one will disable tracing every time do_fault is hit\n" 3748 "\t The second will disable tracing at most 3 times when do_trap is hit\n" 3749 "\t The first time do trap is hit and it disables tracing, the\n" 3750 "\t counter will decrement to 2. If tracing is already disabled,\n" 3751 "\t the counter will not decrement. It only decrements when the\n" 3752 "\t trigger did work\n" 3753 "\t To remove trigger without count:\n" 3754 "\t echo '!<function>:<trigger> > set_ftrace_filter\n" 3755 "\t To remove trigger with a count:\n" 3756 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n" 3757 " set_ftrace_notrace\t- echo function name in here to never trace.\n" 3758 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n" 3759 "\t modules: Can select a group via module command :mod:\n" 3760 "\t Does not accept triggers\n" 3761 #endif /* CONFIG_DYNAMIC_FTRACE */ 3762 #ifdef CONFIG_FUNCTION_TRACER 3763 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n" 3764 "\t\t (function)\n" 3765 #endif 3766 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 3767 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n" 3768 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n" 3769 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n" 3770 #endif 3771 #ifdef CONFIG_TRACER_SNAPSHOT 3772 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n" 3773 "\t\t\t snapshot buffer. Read the contents for more\n" 3774 "\t\t\t information\n" 3775 #endif 3776 #ifdef CONFIG_STACK_TRACER 3777 " stack_trace\t\t- Shows the max stack trace when active\n" 3778 " stack_max_size\t- Shows current max stack size that was traced\n" 3779 "\t\t\t Write into this file to reset the max size (trigger a\n" 3780 "\t\t\t new trace)\n" 3781 #ifdef CONFIG_DYNAMIC_FTRACE 3782 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n" 3783 "\t\t\t traces\n" 3784 #endif 3785 #endif /* CONFIG_STACK_TRACER */ 3786 " events/\t\t- Directory containing all trace event subsystems:\n" 3787 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n" 3788 " events/<system>/\t- Directory containing all trace events for <system>:\n" 3789 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n" 3790 "\t\t\t events\n" 3791 " filter\t\t- If set, only events passing filter are traced\n" 3792 " events/<system>/<event>/\t- Directory containing control files for\n" 3793 "\t\t\t <event>:\n" 3794 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n" 3795 " filter\t\t- If set, only events passing filter are traced\n" 3796 " trigger\t\t- If set, a command to perform when event is hit\n" 3797 "\t Format: <trigger>[:count][if <filter>]\n" 3798 "\t trigger: traceon, traceoff\n" 3799 "\t enable_event:<system>:<event>\n" 3800 "\t disable_event:<system>:<event>\n" 3801 #ifdef CONFIG_STACKTRACE 3802 "\t\t stacktrace\n" 3803 #endif 3804 #ifdef CONFIG_TRACER_SNAPSHOT 3805 "\t\t snapshot\n" 3806 #endif 3807 "\t example: echo traceoff > events/block/block_unplug/trigger\n" 3808 "\t echo traceoff:3 > events/block/block_unplug/trigger\n" 3809 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n" 3810 "\t events/block/block_unplug/trigger\n" 3811 "\t The first disables tracing every time block_unplug is hit.\n" 3812 "\t The second disables tracing the first 3 times block_unplug is hit.\n" 3813 "\t The third enables the kmalloc event the first 3 times block_unplug\n" 3814 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n" 3815 "\t Like function triggers, the counter is only decremented if it\n" 3816 "\t enabled or disabled tracing.\n" 3817 "\t To remove a trigger without a count:\n" 3818 "\t echo '!<trigger> > <system>/<event>/trigger\n" 3819 "\t To remove a trigger with a count:\n" 3820 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n" 3821 "\t Filters can be ignored when removing a trigger.\n" 3822 ; 3823 3824 static ssize_t 3825 tracing_readme_read(struct file *filp, char __user *ubuf, 3826 size_t cnt, loff_t *ppos) 3827 { 3828 return simple_read_from_buffer(ubuf, cnt, ppos, 3829 readme_msg, strlen(readme_msg)); 3830 } 3831 3832 static const struct file_operations tracing_readme_fops = { 3833 .open = tracing_open_generic, 3834 .read = tracing_readme_read, 3835 .llseek = generic_file_llseek, 3836 }; 3837 3838 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos) 3839 { 3840 unsigned int *ptr = v; 3841 3842 if (*pos || m->count) 3843 ptr++; 3844 3845 (*pos)++; 3846 3847 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num]; 3848 ptr++) { 3849 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP) 3850 continue; 3851 3852 return ptr; 3853 } 3854 3855 return NULL; 3856 } 3857 3858 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos) 3859 { 3860 void *v; 3861 loff_t l = 0; 3862 3863 preempt_disable(); 3864 arch_spin_lock(&trace_cmdline_lock); 3865 3866 v = &savedcmd->map_cmdline_to_pid[0]; 3867 while (l <= *pos) { 3868 v = saved_cmdlines_next(m, v, &l); 3869 if (!v) 3870 return NULL; 3871 } 3872 3873 return v; 3874 } 3875 3876 static void saved_cmdlines_stop(struct seq_file *m, void *v) 3877 { 3878 arch_spin_unlock(&trace_cmdline_lock); 3879 preempt_enable(); 3880 } 3881 3882 static int saved_cmdlines_show(struct seq_file *m, void *v) 3883 { 3884 char buf[TASK_COMM_LEN]; 3885 unsigned int *pid = v; 3886 3887 __trace_find_cmdline(*pid, buf); 3888 seq_printf(m, "%d %s\n", *pid, buf); 3889 return 0; 3890 } 3891 3892 static const struct seq_operations tracing_saved_cmdlines_seq_ops = { 3893 .start = saved_cmdlines_start, 3894 .next = saved_cmdlines_next, 3895 .stop = saved_cmdlines_stop, 3896 .show = saved_cmdlines_show, 3897 }; 3898 3899 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp) 3900 { 3901 if (tracing_disabled) 3902 return -ENODEV; 3903 3904 return seq_open(filp, &tracing_saved_cmdlines_seq_ops); 3905 } 3906 3907 static const struct file_operations tracing_saved_cmdlines_fops = { 3908 .open = tracing_saved_cmdlines_open, 3909 .read = seq_read, 3910 .llseek = seq_lseek, 3911 .release = seq_release, 3912 }; 3913 3914 static ssize_t 3915 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf, 3916 size_t cnt, loff_t *ppos) 3917 { 3918 char buf[64]; 3919 int r; 3920 3921 arch_spin_lock(&trace_cmdline_lock); 3922 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num); 3923 arch_spin_unlock(&trace_cmdline_lock); 3924 3925 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 3926 } 3927 3928 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s) 3929 { 3930 kfree(s->saved_cmdlines); 3931 kfree(s->map_cmdline_to_pid); 3932 kfree(s); 3933 } 3934 3935 static int tracing_resize_saved_cmdlines(unsigned int val) 3936 { 3937 struct saved_cmdlines_buffer *s, *savedcmd_temp; 3938 3939 s = kmalloc(sizeof(*s), GFP_KERNEL); 3940 if (!s) 3941 return -ENOMEM; 3942 3943 if (allocate_cmdlines_buffer(val, s) < 0) { 3944 kfree(s); 3945 return -ENOMEM; 3946 } 3947 3948 arch_spin_lock(&trace_cmdline_lock); 3949 savedcmd_temp = savedcmd; 3950 savedcmd = s; 3951 arch_spin_unlock(&trace_cmdline_lock); 3952 free_saved_cmdlines_buffer(savedcmd_temp); 3953 3954 return 0; 3955 } 3956 3957 static ssize_t 3958 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf, 3959 size_t cnt, loff_t *ppos) 3960 { 3961 unsigned long val; 3962 int ret; 3963 3964 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 3965 if (ret) 3966 return ret; 3967 3968 /* must have at least 1 entry or less than PID_MAX_DEFAULT */ 3969 if (!val || val > PID_MAX_DEFAULT) 3970 return -EINVAL; 3971 3972 ret = tracing_resize_saved_cmdlines((unsigned int)val); 3973 if (ret < 0) 3974 return ret; 3975 3976 *ppos += cnt; 3977 3978 return cnt; 3979 } 3980 3981 static const struct file_operations tracing_saved_cmdlines_size_fops = { 3982 .open = tracing_open_generic, 3983 .read = tracing_saved_cmdlines_size_read, 3984 .write = tracing_saved_cmdlines_size_write, 3985 }; 3986 3987 #ifdef CONFIG_TRACE_ENUM_MAP_FILE 3988 static union trace_enum_map_item * 3989 update_enum_map(union trace_enum_map_item *ptr) 3990 { 3991 if (!ptr->map.enum_string) { 3992 if (ptr->tail.next) { 3993 ptr = ptr->tail.next; 3994 /* Set ptr to the next real item (skip head) */ 3995 ptr++; 3996 } else 3997 return NULL; 3998 } 3999 return ptr; 4000 } 4001 4002 static void *enum_map_next(struct seq_file *m, void *v, loff_t *pos) 4003 { 4004 union trace_enum_map_item *ptr = v; 4005 4006 /* 4007 * Paranoid! If ptr points to end, we don't want to increment past it. 4008 * This really should never happen. 4009 */ 4010 ptr = update_enum_map(ptr); 4011 if (WARN_ON_ONCE(!ptr)) 4012 return NULL; 4013 4014 ptr++; 4015 4016 (*pos)++; 4017 4018 ptr = update_enum_map(ptr); 4019 4020 return ptr; 4021 } 4022 4023 static void *enum_map_start(struct seq_file *m, loff_t *pos) 4024 { 4025 union trace_enum_map_item *v; 4026 loff_t l = 0; 4027 4028 mutex_lock(&trace_enum_mutex); 4029 4030 v = trace_enum_maps; 4031 if (v) 4032 v++; 4033 4034 while (v && l < *pos) { 4035 v = enum_map_next(m, v, &l); 4036 } 4037 4038 return v; 4039 } 4040 4041 static void enum_map_stop(struct seq_file *m, void *v) 4042 { 4043 mutex_unlock(&trace_enum_mutex); 4044 } 4045 4046 static int enum_map_show(struct seq_file *m, void *v) 4047 { 4048 union trace_enum_map_item *ptr = v; 4049 4050 seq_printf(m, "%s %ld (%s)\n", 4051 ptr->map.enum_string, ptr->map.enum_value, 4052 ptr->map.system); 4053 4054 return 0; 4055 } 4056 4057 static const struct seq_operations tracing_enum_map_seq_ops = { 4058 .start = enum_map_start, 4059 .next = enum_map_next, 4060 .stop = enum_map_stop, 4061 .show = enum_map_show, 4062 }; 4063 4064 static int tracing_enum_map_open(struct inode *inode, struct file *filp) 4065 { 4066 if (tracing_disabled) 4067 return -ENODEV; 4068 4069 return seq_open(filp, &tracing_enum_map_seq_ops); 4070 } 4071 4072 static const struct file_operations tracing_enum_map_fops = { 4073 .open = tracing_enum_map_open, 4074 .read = seq_read, 4075 .llseek = seq_lseek, 4076 .release = seq_release, 4077 }; 4078 4079 static inline union trace_enum_map_item * 4080 trace_enum_jmp_to_tail(union trace_enum_map_item *ptr) 4081 { 4082 /* Return tail of array given the head */ 4083 return ptr + ptr->head.length + 1; 4084 } 4085 4086 static void 4087 trace_insert_enum_map_file(struct module *mod, struct trace_enum_map **start, 4088 int len) 4089 { 4090 struct trace_enum_map **stop; 4091 struct trace_enum_map **map; 4092 union trace_enum_map_item *map_array; 4093 union trace_enum_map_item *ptr; 4094 4095 stop = start + len; 4096 4097 /* 4098 * The trace_enum_maps contains the map plus a head and tail item, 4099 * where the head holds the module and length of array, and the 4100 * tail holds a pointer to the next list. 4101 */ 4102 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL); 4103 if (!map_array) { 4104 pr_warning("Unable to allocate trace enum mapping\n"); 4105 return; 4106 } 4107 4108 mutex_lock(&trace_enum_mutex); 4109 4110 if (!trace_enum_maps) 4111 trace_enum_maps = map_array; 4112 else { 4113 ptr = trace_enum_maps; 4114 for (;;) { 4115 ptr = trace_enum_jmp_to_tail(ptr); 4116 if (!ptr->tail.next) 4117 break; 4118 ptr = ptr->tail.next; 4119 4120 } 4121 ptr->tail.next = map_array; 4122 } 4123 map_array->head.mod = mod; 4124 map_array->head.length = len; 4125 map_array++; 4126 4127 for (map = start; (unsigned long)map < (unsigned long)stop; map++) { 4128 map_array->map = **map; 4129 map_array++; 4130 } 4131 memset(map_array, 0, sizeof(*map_array)); 4132 4133 mutex_unlock(&trace_enum_mutex); 4134 } 4135 4136 static void trace_create_enum_file(struct dentry *d_tracer) 4137 { 4138 trace_create_file("enum_map", 0444, d_tracer, 4139 NULL, &tracing_enum_map_fops); 4140 } 4141 4142 #else /* CONFIG_TRACE_ENUM_MAP_FILE */ 4143 static inline void trace_create_enum_file(struct dentry *d_tracer) { } 4144 static inline void trace_insert_enum_map_file(struct module *mod, 4145 struct trace_enum_map **start, int len) { } 4146 #endif /* !CONFIG_TRACE_ENUM_MAP_FILE */ 4147 4148 static void trace_insert_enum_map(struct module *mod, 4149 struct trace_enum_map **start, int len) 4150 { 4151 struct trace_enum_map **map; 4152 4153 if (len <= 0) 4154 return; 4155 4156 map = start; 4157 4158 trace_event_enum_update(map, len); 4159 4160 trace_insert_enum_map_file(mod, start, len); 4161 } 4162 4163 static ssize_t 4164 tracing_set_trace_read(struct file *filp, char __user *ubuf, 4165 size_t cnt, loff_t *ppos) 4166 { 4167 struct trace_array *tr = filp->private_data; 4168 char buf[MAX_TRACER_SIZE+2]; 4169 int r; 4170 4171 mutex_lock(&trace_types_lock); 4172 r = sprintf(buf, "%s\n", tr->current_trace->name); 4173 mutex_unlock(&trace_types_lock); 4174 4175 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 4176 } 4177 4178 int tracer_init(struct tracer *t, struct trace_array *tr) 4179 { 4180 tracing_reset_online_cpus(&tr->trace_buffer); 4181 return t->init(tr); 4182 } 4183 4184 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val) 4185 { 4186 int cpu; 4187 4188 for_each_tracing_cpu(cpu) 4189 per_cpu_ptr(buf->data, cpu)->entries = val; 4190 } 4191 4192 #ifdef CONFIG_TRACER_MAX_TRACE 4193 /* resize @tr's buffer to the size of @size_tr's entries */ 4194 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf, 4195 struct trace_buffer *size_buf, int cpu_id) 4196 { 4197 int cpu, ret = 0; 4198 4199 if (cpu_id == RING_BUFFER_ALL_CPUS) { 4200 for_each_tracing_cpu(cpu) { 4201 ret = ring_buffer_resize(trace_buf->buffer, 4202 per_cpu_ptr(size_buf->data, cpu)->entries, cpu); 4203 if (ret < 0) 4204 break; 4205 per_cpu_ptr(trace_buf->data, cpu)->entries = 4206 per_cpu_ptr(size_buf->data, cpu)->entries; 4207 } 4208 } else { 4209 ret = ring_buffer_resize(trace_buf->buffer, 4210 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id); 4211 if (ret == 0) 4212 per_cpu_ptr(trace_buf->data, cpu_id)->entries = 4213 per_cpu_ptr(size_buf->data, cpu_id)->entries; 4214 } 4215 4216 return ret; 4217 } 4218 #endif /* CONFIG_TRACER_MAX_TRACE */ 4219 4220 static int __tracing_resize_ring_buffer(struct trace_array *tr, 4221 unsigned long size, int cpu) 4222 { 4223 int ret; 4224 4225 /* 4226 * If kernel or user changes the size of the ring buffer 4227 * we use the size that was given, and we can forget about 4228 * expanding it later. 4229 */ 4230 ring_buffer_expanded = true; 4231 4232 /* May be called before buffers are initialized */ 4233 if (!tr->trace_buffer.buffer) 4234 return 0; 4235 4236 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu); 4237 if (ret < 0) 4238 return ret; 4239 4240 #ifdef CONFIG_TRACER_MAX_TRACE 4241 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) || 4242 !tr->current_trace->use_max_tr) 4243 goto out; 4244 4245 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu); 4246 if (ret < 0) { 4247 int r = resize_buffer_duplicate_size(&tr->trace_buffer, 4248 &tr->trace_buffer, cpu); 4249 if (r < 0) { 4250 /* 4251 * AARGH! We are left with different 4252 * size max buffer!!!! 4253 * The max buffer is our "snapshot" buffer. 4254 * When a tracer needs a snapshot (one of the 4255 * latency tracers), it swaps the max buffer 4256 * with the saved snap shot. We succeeded to 4257 * update the size of the main buffer, but failed to 4258 * update the size of the max buffer. But when we tried 4259 * to reset the main buffer to the original size, we 4260 * failed there too. This is very unlikely to 4261 * happen, but if it does, warn and kill all 4262 * tracing. 4263 */ 4264 WARN_ON(1); 4265 tracing_disabled = 1; 4266 } 4267 return ret; 4268 } 4269 4270 if (cpu == RING_BUFFER_ALL_CPUS) 4271 set_buffer_entries(&tr->max_buffer, size); 4272 else 4273 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size; 4274 4275 out: 4276 #endif /* CONFIG_TRACER_MAX_TRACE */ 4277 4278 if (cpu == RING_BUFFER_ALL_CPUS) 4279 set_buffer_entries(&tr->trace_buffer, size); 4280 else 4281 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size; 4282 4283 return ret; 4284 } 4285 4286 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr, 4287 unsigned long size, int cpu_id) 4288 { 4289 int ret = size; 4290 4291 mutex_lock(&trace_types_lock); 4292 4293 if (cpu_id != RING_BUFFER_ALL_CPUS) { 4294 /* make sure, this cpu is enabled in the mask */ 4295 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) { 4296 ret = -EINVAL; 4297 goto out; 4298 } 4299 } 4300 4301 ret = __tracing_resize_ring_buffer(tr, size, cpu_id); 4302 if (ret < 0) 4303 ret = -ENOMEM; 4304 4305 out: 4306 mutex_unlock(&trace_types_lock); 4307 4308 return ret; 4309 } 4310 4311 4312 /** 4313 * tracing_update_buffers - used by tracing facility to expand ring buffers 4314 * 4315 * To save on memory when the tracing is never used on a system with it 4316 * configured in. The ring buffers are set to a minimum size. But once 4317 * a user starts to use the tracing facility, then they need to grow 4318 * to their default size. 4319 * 4320 * This function is to be called when a tracer is about to be used. 4321 */ 4322 int tracing_update_buffers(void) 4323 { 4324 int ret = 0; 4325 4326 mutex_lock(&trace_types_lock); 4327 if (!ring_buffer_expanded) 4328 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size, 4329 RING_BUFFER_ALL_CPUS); 4330 mutex_unlock(&trace_types_lock); 4331 4332 return ret; 4333 } 4334 4335 struct trace_option_dentry; 4336 4337 static void 4338 create_trace_option_files(struct trace_array *tr, struct tracer *tracer); 4339 4340 /* 4341 * Used to clear out the tracer before deletion of an instance. 4342 * Must have trace_types_lock held. 4343 */ 4344 static void tracing_set_nop(struct trace_array *tr) 4345 { 4346 if (tr->current_trace == &nop_trace) 4347 return; 4348 4349 tr->current_trace->enabled--; 4350 4351 if (tr->current_trace->reset) 4352 tr->current_trace->reset(tr); 4353 4354 tr->current_trace = &nop_trace; 4355 } 4356 4357 static void add_tracer_options(struct trace_array *tr, struct tracer *t) 4358 { 4359 /* Only enable if the directory has been created already. */ 4360 if (!tr->dir) 4361 return; 4362 4363 create_trace_option_files(tr, t); 4364 } 4365 4366 static int tracing_set_tracer(struct trace_array *tr, const char *buf) 4367 { 4368 struct tracer *t; 4369 #ifdef CONFIG_TRACER_MAX_TRACE 4370 bool had_max_tr; 4371 #endif 4372 int ret = 0; 4373 4374 mutex_lock(&trace_types_lock); 4375 4376 if (!ring_buffer_expanded) { 4377 ret = __tracing_resize_ring_buffer(tr, trace_buf_size, 4378 RING_BUFFER_ALL_CPUS); 4379 if (ret < 0) 4380 goto out; 4381 ret = 0; 4382 } 4383 4384 for (t = trace_types; t; t = t->next) { 4385 if (strcmp(t->name, buf) == 0) 4386 break; 4387 } 4388 if (!t) { 4389 ret = -EINVAL; 4390 goto out; 4391 } 4392 if (t == tr->current_trace) 4393 goto out; 4394 4395 /* Some tracers are only allowed for the top level buffer */ 4396 if (!trace_ok_for_array(t, tr)) { 4397 ret = -EINVAL; 4398 goto out; 4399 } 4400 4401 /* If trace pipe files are being read, we can't change the tracer */ 4402 if (tr->current_trace->ref) { 4403 ret = -EBUSY; 4404 goto out; 4405 } 4406 4407 trace_branch_disable(); 4408 4409 tr->current_trace->enabled--; 4410 4411 if (tr->current_trace->reset) 4412 tr->current_trace->reset(tr); 4413 4414 /* Current trace needs to be nop_trace before synchronize_sched */ 4415 tr->current_trace = &nop_trace; 4416 4417 #ifdef CONFIG_TRACER_MAX_TRACE 4418 had_max_tr = tr->allocated_snapshot; 4419 4420 if (had_max_tr && !t->use_max_tr) { 4421 /* 4422 * We need to make sure that the update_max_tr sees that 4423 * current_trace changed to nop_trace to keep it from 4424 * swapping the buffers after we resize it. 4425 * The update_max_tr is called from interrupts disabled 4426 * so a synchronized_sched() is sufficient. 4427 */ 4428 synchronize_sched(); 4429 free_snapshot(tr); 4430 } 4431 #endif 4432 4433 #ifdef CONFIG_TRACER_MAX_TRACE 4434 if (t->use_max_tr && !had_max_tr) { 4435 ret = alloc_snapshot(tr); 4436 if (ret < 0) 4437 goto out; 4438 } 4439 #endif 4440 4441 if (t->init) { 4442 ret = tracer_init(t, tr); 4443 if (ret) 4444 goto out; 4445 } 4446 4447 tr->current_trace = t; 4448 tr->current_trace->enabled++; 4449 trace_branch_enable(tr); 4450 out: 4451 mutex_unlock(&trace_types_lock); 4452 4453 return ret; 4454 } 4455 4456 static ssize_t 4457 tracing_set_trace_write(struct file *filp, const char __user *ubuf, 4458 size_t cnt, loff_t *ppos) 4459 { 4460 struct trace_array *tr = filp->private_data; 4461 char buf[MAX_TRACER_SIZE+1]; 4462 int i; 4463 size_t ret; 4464 int err; 4465 4466 ret = cnt; 4467 4468 if (cnt > MAX_TRACER_SIZE) 4469 cnt = MAX_TRACER_SIZE; 4470 4471 if (copy_from_user(&buf, ubuf, cnt)) 4472 return -EFAULT; 4473 4474 buf[cnt] = 0; 4475 4476 /* strip ending whitespace. */ 4477 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--) 4478 buf[i] = 0; 4479 4480 err = tracing_set_tracer(tr, buf); 4481 if (err) 4482 return err; 4483 4484 *ppos += ret; 4485 4486 return ret; 4487 } 4488 4489 static ssize_t 4490 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf, 4491 size_t cnt, loff_t *ppos) 4492 { 4493 char buf[64]; 4494 int r; 4495 4496 r = snprintf(buf, sizeof(buf), "%ld\n", 4497 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr)); 4498 if (r > sizeof(buf)) 4499 r = sizeof(buf); 4500 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 4501 } 4502 4503 static ssize_t 4504 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf, 4505 size_t cnt, loff_t *ppos) 4506 { 4507 unsigned long val; 4508 int ret; 4509 4510 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 4511 if (ret) 4512 return ret; 4513 4514 *ptr = val * 1000; 4515 4516 return cnt; 4517 } 4518 4519 static ssize_t 4520 tracing_thresh_read(struct file *filp, char __user *ubuf, 4521 size_t cnt, loff_t *ppos) 4522 { 4523 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos); 4524 } 4525 4526 static ssize_t 4527 tracing_thresh_write(struct file *filp, const char __user *ubuf, 4528 size_t cnt, loff_t *ppos) 4529 { 4530 struct trace_array *tr = filp->private_data; 4531 int ret; 4532 4533 mutex_lock(&trace_types_lock); 4534 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos); 4535 if (ret < 0) 4536 goto out; 4537 4538 if (tr->current_trace->update_thresh) { 4539 ret = tr->current_trace->update_thresh(tr); 4540 if (ret < 0) 4541 goto out; 4542 } 4543 4544 ret = cnt; 4545 out: 4546 mutex_unlock(&trace_types_lock); 4547 4548 return ret; 4549 } 4550 4551 #ifdef CONFIG_TRACER_MAX_TRACE 4552 4553 static ssize_t 4554 tracing_max_lat_read(struct file *filp, char __user *ubuf, 4555 size_t cnt, loff_t *ppos) 4556 { 4557 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos); 4558 } 4559 4560 static ssize_t 4561 tracing_max_lat_write(struct file *filp, const char __user *ubuf, 4562 size_t cnt, loff_t *ppos) 4563 { 4564 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos); 4565 } 4566 4567 #endif 4568 4569 static int tracing_open_pipe(struct inode *inode, struct file *filp) 4570 { 4571 struct trace_array *tr = inode->i_private; 4572 struct trace_iterator *iter; 4573 int ret = 0; 4574 4575 if (tracing_disabled) 4576 return -ENODEV; 4577 4578 if (trace_array_get(tr) < 0) 4579 return -ENODEV; 4580 4581 mutex_lock(&trace_types_lock); 4582 4583 /* create a buffer to store the information to pass to userspace */ 4584 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 4585 if (!iter) { 4586 ret = -ENOMEM; 4587 __trace_array_put(tr); 4588 goto out; 4589 } 4590 4591 trace_seq_init(&iter->seq); 4592 iter->trace = tr->current_trace; 4593 4594 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) { 4595 ret = -ENOMEM; 4596 goto fail; 4597 } 4598 4599 /* trace pipe does not show start of buffer */ 4600 cpumask_setall(iter->started); 4601 4602 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 4603 iter->iter_flags |= TRACE_FILE_LAT_FMT; 4604 4605 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 4606 if (trace_clocks[tr->clock_id].in_ns) 4607 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 4608 4609 iter->tr = tr; 4610 iter->trace_buffer = &tr->trace_buffer; 4611 iter->cpu_file = tracing_get_cpu(inode); 4612 mutex_init(&iter->mutex); 4613 filp->private_data = iter; 4614 4615 if (iter->trace->pipe_open) 4616 iter->trace->pipe_open(iter); 4617 4618 nonseekable_open(inode, filp); 4619 4620 tr->current_trace->ref++; 4621 out: 4622 mutex_unlock(&trace_types_lock); 4623 return ret; 4624 4625 fail: 4626 kfree(iter->trace); 4627 kfree(iter); 4628 __trace_array_put(tr); 4629 mutex_unlock(&trace_types_lock); 4630 return ret; 4631 } 4632 4633 static int tracing_release_pipe(struct inode *inode, struct file *file) 4634 { 4635 struct trace_iterator *iter = file->private_data; 4636 struct trace_array *tr = inode->i_private; 4637 4638 mutex_lock(&trace_types_lock); 4639 4640 tr->current_trace->ref--; 4641 4642 if (iter->trace->pipe_close) 4643 iter->trace->pipe_close(iter); 4644 4645 mutex_unlock(&trace_types_lock); 4646 4647 free_cpumask_var(iter->started); 4648 mutex_destroy(&iter->mutex); 4649 kfree(iter); 4650 4651 trace_array_put(tr); 4652 4653 return 0; 4654 } 4655 4656 static unsigned int 4657 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table) 4658 { 4659 struct trace_array *tr = iter->tr; 4660 4661 /* Iterators are static, they should be filled or empty */ 4662 if (trace_buffer_iter(iter, iter->cpu_file)) 4663 return POLLIN | POLLRDNORM; 4664 4665 if (tr->trace_flags & TRACE_ITER_BLOCK) 4666 /* 4667 * Always select as readable when in blocking mode 4668 */ 4669 return POLLIN | POLLRDNORM; 4670 else 4671 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file, 4672 filp, poll_table); 4673 } 4674 4675 static unsigned int 4676 tracing_poll_pipe(struct file *filp, poll_table *poll_table) 4677 { 4678 struct trace_iterator *iter = filp->private_data; 4679 4680 return trace_poll(iter, filp, poll_table); 4681 } 4682 4683 /* Must be called with iter->mutex held. */ 4684 static int tracing_wait_pipe(struct file *filp) 4685 { 4686 struct trace_iterator *iter = filp->private_data; 4687 int ret; 4688 4689 while (trace_empty(iter)) { 4690 4691 if ((filp->f_flags & O_NONBLOCK)) { 4692 return -EAGAIN; 4693 } 4694 4695 /* 4696 * We block until we read something and tracing is disabled. 4697 * We still block if tracing is disabled, but we have never 4698 * read anything. This allows a user to cat this file, and 4699 * then enable tracing. But after we have read something, 4700 * we give an EOF when tracing is again disabled. 4701 * 4702 * iter->pos will be 0 if we haven't read anything. 4703 */ 4704 if (!tracing_is_on() && iter->pos) 4705 break; 4706 4707 mutex_unlock(&iter->mutex); 4708 4709 ret = wait_on_pipe(iter, false); 4710 4711 mutex_lock(&iter->mutex); 4712 4713 if (ret) 4714 return ret; 4715 } 4716 4717 return 1; 4718 } 4719 4720 /* 4721 * Consumer reader. 4722 */ 4723 static ssize_t 4724 tracing_read_pipe(struct file *filp, char __user *ubuf, 4725 size_t cnt, loff_t *ppos) 4726 { 4727 struct trace_iterator *iter = filp->private_data; 4728 ssize_t sret; 4729 4730 /* return any leftover data */ 4731 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 4732 if (sret != -EBUSY) 4733 return sret; 4734 4735 trace_seq_init(&iter->seq); 4736 4737 /* 4738 * Avoid more than one consumer on a single file descriptor 4739 * This is just a matter of traces coherency, the ring buffer itself 4740 * is protected. 4741 */ 4742 mutex_lock(&iter->mutex); 4743 if (iter->trace->read) { 4744 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); 4745 if (sret) 4746 goto out; 4747 } 4748 4749 waitagain: 4750 sret = tracing_wait_pipe(filp); 4751 if (sret <= 0) 4752 goto out; 4753 4754 /* stop when tracing is finished */ 4755 if (trace_empty(iter)) { 4756 sret = 0; 4757 goto out; 4758 } 4759 4760 if (cnt >= PAGE_SIZE) 4761 cnt = PAGE_SIZE - 1; 4762 4763 /* reset all but tr, trace, and overruns */ 4764 memset(&iter->seq, 0, 4765 sizeof(struct trace_iterator) - 4766 offsetof(struct trace_iterator, seq)); 4767 cpumask_clear(iter->started); 4768 iter->pos = -1; 4769 4770 trace_event_read_lock(); 4771 trace_access_lock(iter->cpu_file); 4772 while (trace_find_next_entry_inc(iter) != NULL) { 4773 enum print_line_t ret; 4774 int save_len = iter->seq.seq.len; 4775 4776 ret = print_trace_line(iter); 4777 if (ret == TRACE_TYPE_PARTIAL_LINE) { 4778 /* don't print partial lines */ 4779 iter->seq.seq.len = save_len; 4780 break; 4781 } 4782 if (ret != TRACE_TYPE_NO_CONSUME) 4783 trace_consume(iter); 4784 4785 if (trace_seq_used(&iter->seq) >= cnt) 4786 break; 4787 4788 /* 4789 * Setting the full flag means we reached the trace_seq buffer 4790 * size and we should leave by partial output condition above. 4791 * One of the trace_seq_* functions is not used properly. 4792 */ 4793 WARN_ONCE(iter->seq.full, "full flag set for trace type %d", 4794 iter->ent->type); 4795 } 4796 trace_access_unlock(iter->cpu_file); 4797 trace_event_read_unlock(); 4798 4799 /* Now copy what we have to the user */ 4800 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 4801 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq)) 4802 trace_seq_init(&iter->seq); 4803 4804 /* 4805 * If there was nothing to send to user, in spite of consuming trace 4806 * entries, go back to wait for more entries. 4807 */ 4808 if (sret == -EBUSY) 4809 goto waitagain; 4810 4811 out: 4812 mutex_unlock(&iter->mutex); 4813 4814 return sret; 4815 } 4816 4817 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd, 4818 unsigned int idx) 4819 { 4820 __free_page(spd->pages[idx]); 4821 } 4822 4823 static const struct pipe_buf_operations tracing_pipe_buf_ops = { 4824 .can_merge = 0, 4825 .confirm = generic_pipe_buf_confirm, 4826 .release = generic_pipe_buf_release, 4827 .steal = generic_pipe_buf_steal, 4828 .get = generic_pipe_buf_get, 4829 }; 4830 4831 static size_t 4832 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter) 4833 { 4834 size_t count; 4835 int save_len; 4836 int ret; 4837 4838 /* Seq buffer is page-sized, exactly what we need. */ 4839 for (;;) { 4840 save_len = iter->seq.seq.len; 4841 ret = print_trace_line(iter); 4842 4843 if (trace_seq_has_overflowed(&iter->seq)) { 4844 iter->seq.seq.len = save_len; 4845 break; 4846 } 4847 4848 /* 4849 * This should not be hit, because it should only 4850 * be set if the iter->seq overflowed. But check it 4851 * anyway to be safe. 4852 */ 4853 if (ret == TRACE_TYPE_PARTIAL_LINE) { 4854 iter->seq.seq.len = save_len; 4855 break; 4856 } 4857 4858 count = trace_seq_used(&iter->seq) - save_len; 4859 if (rem < count) { 4860 rem = 0; 4861 iter->seq.seq.len = save_len; 4862 break; 4863 } 4864 4865 if (ret != TRACE_TYPE_NO_CONSUME) 4866 trace_consume(iter); 4867 rem -= count; 4868 if (!trace_find_next_entry_inc(iter)) { 4869 rem = 0; 4870 iter->ent = NULL; 4871 break; 4872 } 4873 } 4874 4875 return rem; 4876 } 4877 4878 static ssize_t tracing_splice_read_pipe(struct file *filp, 4879 loff_t *ppos, 4880 struct pipe_inode_info *pipe, 4881 size_t len, 4882 unsigned int flags) 4883 { 4884 struct page *pages_def[PIPE_DEF_BUFFERS]; 4885 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 4886 struct trace_iterator *iter = filp->private_data; 4887 struct splice_pipe_desc spd = { 4888 .pages = pages_def, 4889 .partial = partial_def, 4890 .nr_pages = 0, /* This gets updated below. */ 4891 .nr_pages_max = PIPE_DEF_BUFFERS, 4892 .flags = flags, 4893 .ops = &tracing_pipe_buf_ops, 4894 .spd_release = tracing_spd_release_pipe, 4895 }; 4896 ssize_t ret; 4897 size_t rem; 4898 unsigned int i; 4899 4900 if (splice_grow_spd(pipe, &spd)) 4901 return -ENOMEM; 4902 4903 mutex_lock(&iter->mutex); 4904 4905 if (iter->trace->splice_read) { 4906 ret = iter->trace->splice_read(iter, filp, 4907 ppos, pipe, len, flags); 4908 if (ret) 4909 goto out_err; 4910 } 4911 4912 ret = tracing_wait_pipe(filp); 4913 if (ret <= 0) 4914 goto out_err; 4915 4916 if (!iter->ent && !trace_find_next_entry_inc(iter)) { 4917 ret = -EFAULT; 4918 goto out_err; 4919 } 4920 4921 trace_event_read_lock(); 4922 trace_access_lock(iter->cpu_file); 4923 4924 /* Fill as many pages as possible. */ 4925 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) { 4926 spd.pages[i] = alloc_page(GFP_KERNEL); 4927 if (!spd.pages[i]) 4928 break; 4929 4930 rem = tracing_fill_pipe_page(rem, iter); 4931 4932 /* Copy the data into the page, so we can start over. */ 4933 ret = trace_seq_to_buffer(&iter->seq, 4934 page_address(spd.pages[i]), 4935 trace_seq_used(&iter->seq)); 4936 if (ret < 0) { 4937 __free_page(spd.pages[i]); 4938 break; 4939 } 4940 spd.partial[i].offset = 0; 4941 spd.partial[i].len = trace_seq_used(&iter->seq); 4942 4943 trace_seq_init(&iter->seq); 4944 } 4945 4946 trace_access_unlock(iter->cpu_file); 4947 trace_event_read_unlock(); 4948 mutex_unlock(&iter->mutex); 4949 4950 spd.nr_pages = i; 4951 4952 ret = splice_to_pipe(pipe, &spd); 4953 out: 4954 splice_shrink_spd(&spd); 4955 return ret; 4956 4957 out_err: 4958 mutex_unlock(&iter->mutex); 4959 goto out; 4960 } 4961 4962 static ssize_t 4963 tracing_entries_read(struct file *filp, char __user *ubuf, 4964 size_t cnt, loff_t *ppos) 4965 { 4966 struct inode *inode = file_inode(filp); 4967 struct trace_array *tr = inode->i_private; 4968 int cpu = tracing_get_cpu(inode); 4969 char buf[64]; 4970 int r = 0; 4971 ssize_t ret; 4972 4973 mutex_lock(&trace_types_lock); 4974 4975 if (cpu == RING_BUFFER_ALL_CPUS) { 4976 int cpu, buf_size_same; 4977 unsigned long size; 4978 4979 size = 0; 4980 buf_size_same = 1; 4981 /* check if all cpu sizes are same */ 4982 for_each_tracing_cpu(cpu) { 4983 /* fill in the size from first enabled cpu */ 4984 if (size == 0) 4985 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries; 4986 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) { 4987 buf_size_same = 0; 4988 break; 4989 } 4990 } 4991 4992 if (buf_size_same) { 4993 if (!ring_buffer_expanded) 4994 r = sprintf(buf, "%lu (expanded: %lu)\n", 4995 size >> 10, 4996 trace_buf_size >> 10); 4997 else 4998 r = sprintf(buf, "%lu\n", size >> 10); 4999 } else 5000 r = sprintf(buf, "X\n"); 5001 } else 5002 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10); 5003 5004 mutex_unlock(&trace_types_lock); 5005 5006 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5007 return ret; 5008 } 5009 5010 static ssize_t 5011 tracing_entries_write(struct file *filp, const char __user *ubuf, 5012 size_t cnt, loff_t *ppos) 5013 { 5014 struct inode *inode = file_inode(filp); 5015 struct trace_array *tr = inode->i_private; 5016 unsigned long val; 5017 int ret; 5018 5019 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 5020 if (ret) 5021 return ret; 5022 5023 /* must have at least 1 entry */ 5024 if (!val) 5025 return -EINVAL; 5026 5027 /* value is in KB */ 5028 val <<= 10; 5029 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode)); 5030 if (ret < 0) 5031 return ret; 5032 5033 *ppos += cnt; 5034 5035 return cnt; 5036 } 5037 5038 static ssize_t 5039 tracing_total_entries_read(struct file *filp, char __user *ubuf, 5040 size_t cnt, loff_t *ppos) 5041 { 5042 struct trace_array *tr = filp->private_data; 5043 char buf[64]; 5044 int r, cpu; 5045 unsigned long size = 0, expanded_size = 0; 5046 5047 mutex_lock(&trace_types_lock); 5048 for_each_tracing_cpu(cpu) { 5049 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10; 5050 if (!ring_buffer_expanded) 5051 expanded_size += trace_buf_size >> 10; 5052 } 5053 if (ring_buffer_expanded) 5054 r = sprintf(buf, "%lu\n", size); 5055 else 5056 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size); 5057 mutex_unlock(&trace_types_lock); 5058 5059 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5060 } 5061 5062 static ssize_t 5063 tracing_free_buffer_write(struct file *filp, const char __user *ubuf, 5064 size_t cnt, loff_t *ppos) 5065 { 5066 /* 5067 * There is no need to read what the user has written, this function 5068 * is just to make sure that there is no error when "echo" is used 5069 */ 5070 5071 *ppos += cnt; 5072 5073 return cnt; 5074 } 5075 5076 static int 5077 tracing_free_buffer_release(struct inode *inode, struct file *filp) 5078 { 5079 struct trace_array *tr = inode->i_private; 5080 5081 /* disable tracing ? */ 5082 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE) 5083 tracer_tracing_off(tr); 5084 /* resize the ring buffer to 0 */ 5085 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS); 5086 5087 trace_array_put(tr); 5088 5089 return 0; 5090 } 5091 5092 static ssize_t 5093 tracing_mark_write(struct file *filp, const char __user *ubuf, 5094 size_t cnt, loff_t *fpos) 5095 { 5096 unsigned long addr = (unsigned long)ubuf; 5097 struct trace_array *tr = filp->private_data; 5098 struct ring_buffer_event *event; 5099 struct ring_buffer *buffer; 5100 struct print_entry *entry; 5101 unsigned long irq_flags; 5102 struct page *pages[2]; 5103 void *map_page[2]; 5104 int nr_pages = 1; 5105 ssize_t written; 5106 int offset; 5107 int size; 5108 int len; 5109 int ret; 5110 int i; 5111 5112 if (tracing_disabled) 5113 return -EINVAL; 5114 5115 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) 5116 return -EINVAL; 5117 5118 if (cnt > TRACE_BUF_SIZE) 5119 cnt = TRACE_BUF_SIZE; 5120 5121 /* 5122 * Userspace is injecting traces into the kernel trace buffer. 5123 * We want to be as non intrusive as possible. 5124 * To do so, we do not want to allocate any special buffers 5125 * or take any locks, but instead write the userspace data 5126 * straight into the ring buffer. 5127 * 5128 * First we need to pin the userspace buffer into memory, 5129 * which, most likely it is, because it just referenced it. 5130 * But there's no guarantee that it is. By using get_user_pages_fast() 5131 * and kmap_atomic/kunmap_atomic() we can get access to the 5132 * pages directly. We then write the data directly into the 5133 * ring buffer. 5134 */ 5135 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); 5136 5137 /* check if we cross pages */ 5138 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK)) 5139 nr_pages = 2; 5140 5141 offset = addr & (PAGE_SIZE - 1); 5142 addr &= PAGE_MASK; 5143 5144 ret = get_user_pages_fast(addr, nr_pages, 0, pages); 5145 if (ret < nr_pages) { 5146 while (--ret >= 0) 5147 put_page(pages[ret]); 5148 written = -EFAULT; 5149 goto out; 5150 } 5151 5152 for (i = 0; i < nr_pages; i++) 5153 map_page[i] = kmap_atomic(pages[i]); 5154 5155 local_save_flags(irq_flags); 5156 size = sizeof(*entry) + cnt + 2; /* possible \n added */ 5157 buffer = tr->trace_buffer.buffer; 5158 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 5159 irq_flags, preempt_count()); 5160 if (!event) { 5161 /* Ring buffer disabled, return as if not open for write */ 5162 written = -EBADF; 5163 goto out_unlock; 5164 } 5165 5166 entry = ring_buffer_event_data(event); 5167 entry->ip = _THIS_IP_; 5168 5169 if (nr_pages == 2) { 5170 len = PAGE_SIZE - offset; 5171 memcpy(&entry->buf, map_page[0] + offset, len); 5172 memcpy(&entry->buf[len], map_page[1], cnt - len); 5173 } else 5174 memcpy(&entry->buf, map_page[0] + offset, cnt); 5175 5176 if (entry->buf[cnt - 1] != '\n') { 5177 entry->buf[cnt] = '\n'; 5178 entry->buf[cnt + 1] = '\0'; 5179 } else 5180 entry->buf[cnt] = '\0'; 5181 5182 __buffer_unlock_commit(buffer, event); 5183 5184 written = cnt; 5185 5186 *fpos += written; 5187 5188 out_unlock: 5189 for (i = nr_pages - 1; i >= 0; i--) { 5190 kunmap_atomic(map_page[i]); 5191 put_page(pages[i]); 5192 } 5193 out: 5194 return written; 5195 } 5196 5197 static int tracing_clock_show(struct seq_file *m, void *v) 5198 { 5199 struct trace_array *tr = m->private; 5200 int i; 5201 5202 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) 5203 seq_printf(m, 5204 "%s%s%s%s", i ? " " : "", 5205 i == tr->clock_id ? "[" : "", trace_clocks[i].name, 5206 i == tr->clock_id ? "]" : ""); 5207 seq_putc(m, '\n'); 5208 5209 return 0; 5210 } 5211 5212 static int tracing_set_clock(struct trace_array *tr, const char *clockstr) 5213 { 5214 int i; 5215 5216 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) { 5217 if (strcmp(trace_clocks[i].name, clockstr) == 0) 5218 break; 5219 } 5220 if (i == ARRAY_SIZE(trace_clocks)) 5221 return -EINVAL; 5222 5223 mutex_lock(&trace_types_lock); 5224 5225 tr->clock_id = i; 5226 5227 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func); 5228 5229 /* 5230 * New clock may not be consistent with the previous clock. 5231 * Reset the buffer so that it doesn't have incomparable timestamps. 5232 */ 5233 tracing_reset_online_cpus(&tr->trace_buffer); 5234 5235 #ifdef CONFIG_TRACER_MAX_TRACE 5236 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer) 5237 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func); 5238 tracing_reset_online_cpus(&tr->max_buffer); 5239 #endif 5240 5241 mutex_unlock(&trace_types_lock); 5242 5243 return 0; 5244 } 5245 5246 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf, 5247 size_t cnt, loff_t *fpos) 5248 { 5249 struct seq_file *m = filp->private_data; 5250 struct trace_array *tr = m->private; 5251 char buf[64]; 5252 const char *clockstr; 5253 int ret; 5254 5255 if (cnt >= sizeof(buf)) 5256 return -EINVAL; 5257 5258 if (copy_from_user(&buf, ubuf, cnt)) 5259 return -EFAULT; 5260 5261 buf[cnt] = 0; 5262 5263 clockstr = strstrip(buf); 5264 5265 ret = tracing_set_clock(tr, clockstr); 5266 if (ret) 5267 return ret; 5268 5269 *fpos += cnt; 5270 5271 return cnt; 5272 } 5273 5274 static int tracing_clock_open(struct inode *inode, struct file *file) 5275 { 5276 struct trace_array *tr = inode->i_private; 5277 int ret; 5278 5279 if (tracing_disabled) 5280 return -ENODEV; 5281 5282 if (trace_array_get(tr)) 5283 return -ENODEV; 5284 5285 ret = single_open(file, tracing_clock_show, inode->i_private); 5286 if (ret < 0) 5287 trace_array_put(tr); 5288 5289 return ret; 5290 } 5291 5292 struct ftrace_buffer_info { 5293 struct trace_iterator iter; 5294 void *spare; 5295 unsigned int read; 5296 }; 5297 5298 #ifdef CONFIG_TRACER_SNAPSHOT 5299 static int tracing_snapshot_open(struct inode *inode, struct file *file) 5300 { 5301 struct trace_array *tr = inode->i_private; 5302 struct trace_iterator *iter; 5303 struct seq_file *m; 5304 int ret = 0; 5305 5306 if (trace_array_get(tr) < 0) 5307 return -ENODEV; 5308 5309 if (file->f_mode & FMODE_READ) { 5310 iter = __tracing_open(inode, file, true); 5311 if (IS_ERR(iter)) 5312 ret = PTR_ERR(iter); 5313 } else { 5314 /* Writes still need the seq_file to hold the private data */ 5315 ret = -ENOMEM; 5316 m = kzalloc(sizeof(*m), GFP_KERNEL); 5317 if (!m) 5318 goto out; 5319 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 5320 if (!iter) { 5321 kfree(m); 5322 goto out; 5323 } 5324 ret = 0; 5325 5326 iter->tr = tr; 5327 iter->trace_buffer = &tr->max_buffer; 5328 iter->cpu_file = tracing_get_cpu(inode); 5329 m->private = iter; 5330 file->private_data = m; 5331 } 5332 out: 5333 if (ret < 0) 5334 trace_array_put(tr); 5335 5336 return ret; 5337 } 5338 5339 static ssize_t 5340 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt, 5341 loff_t *ppos) 5342 { 5343 struct seq_file *m = filp->private_data; 5344 struct trace_iterator *iter = m->private; 5345 struct trace_array *tr = iter->tr; 5346 unsigned long val; 5347 int ret; 5348 5349 ret = tracing_update_buffers(); 5350 if (ret < 0) 5351 return ret; 5352 5353 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 5354 if (ret) 5355 return ret; 5356 5357 mutex_lock(&trace_types_lock); 5358 5359 if (tr->current_trace->use_max_tr) { 5360 ret = -EBUSY; 5361 goto out; 5362 } 5363 5364 switch (val) { 5365 case 0: 5366 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 5367 ret = -EINVAL; 5368 break; 5369 } 5370 if (tr->allocated_snapshot) 5371 free_snapshot(tr); 5372 break; 5373 case 1: 5374 /* Only allow per-cpu swap if the ring buffer supports it */ 5375 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP 5376 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 5377 ret = -EINVAL; 5378 break; 5379 } 5380 #endif 5381 if (!tr->allocated_snapshot) { 5382 ret = alloc_snapshot(tr); 5383 if (ret < 0) 5384 break; 5385 } 5386 local_irq_disable(); 5387 /* Now, we're going to swap */ 5388 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 5389 update_max_tr(tr, current, smp_processor_id()); 5390 else 5391 update_max_tr_single(tr, current, iter->cpu_file); 5392 local_irq_enable(); 5393 break; 5394 default: 5395 if (tr->allocated_snapshot) { 5396 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 5397 tracing_reset_online_cpus(&tr->max_buffer); 5398 else 5399 tracing_reset(&tr->max_buffer, iter->cpu_file); 5400 } 5401 break; 5402 } 5403 5404 if (ret >= 0) { 5405 *ppos += cnt; 5406 ret = cnt; 5407 } 5408 out: 5409 mutex_unlock(&trace_types_lock); 5410 return ret; 5411 } 5412 5413 static int tracing_snapshot_release(struct inode *inode, struct file *file) 5414 { 5415 struct seq_file *m = file->private_data; 5416 int ret; 5417 5418 ret = tracing_release(inode, file); 5419 5420 if (file->f_mode & FMODE_READ) 5421 return ret; 5422 5423 /* If write only, the seq_file is just a stub */ 5424 if (m) 5425 kfree(m->private); 5426 kfree(m); 5427 5428 return 0; 5429 } 5430 5431 static int tracing_buffers_open(struct inode *inode, struct file *filp); 5432 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, 5433 size_t count, loff_t *ppos); 5434 static int tracing_buffers_release(struct inode *inode, struct file *file); 5435 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos, 5436 struct pipe_inode_info *pipe, size_t len, unsigned int flags); 5437 5438 static int snapshot_raw_open(struct inode *inode, struct file *filp) 5439 { 5440 struct ftrace_buffer_info *info; 5441 int ret; 5442 5443 ret = tracing_buffers_open(inode, filp); 5444 if (ret < 0) 5445 return ret; 5446 5447 info = filp->private_data; 5448 5449 if (info->iter.trace->use_max_tr) { 5450 tracing_buffers_release(inode, filp); 5451 return -EBUSY; 5452 } 5453 5454 info->iter.snapshot = true; 5455 info->iter.trace_buffer = &info->iter.tr->max_buffer; 5456 5457 return ret; 5458 } 5459 5460 #endif /* CONFIG_TRACER_SNAPSHOT */ 5461 5462 5463 static const struct file_operations tracing_thresh_fops = { 5464 .open = tracing_open_generic, 5465 .read = tracing_thresh_read, 5466 .write = tracing_thresh_write, 5467 .llseek = generic_file_llseek, 5468 }; 5469 5470 #ifdef CONFIG_TRACER_MAX_TRACE 5471 static const struct file_operations tracing_max_lat_fops = { 5472 .open = tracing_open_generic, 5473 .read = tracing_max_lat_read, 5474 .write = tracing_max_lat_write, 5475 .llseek = generic_file_llseek, 5476 }; 5477 #endif 5478 5479 static const struct file_operations set_tracer_fops = { 5480 .open = tracing_open_generic, 5481 .read = tracing_set_trace_read, 5482 .write = tracing_set_trace_write, 5483 .llseek = generic_file_llseek, 5484 }; 5485 5486 static const struct file_operations tracing_pipe_fops = { 5487 .open = tracing_open_pipe, 5488 .poll = tracing_poll_pipe, 5489 .read = tracing_read_pipe, 5490 .splice_read = tracing_splice_read_pipe, 5491 .release = tracing_release_pipe, 5492 .llseek = no_llseek, 5493 }; 5494 5495 static const struct file_operations tracing_entries_fops = { 5496 .open = tracing_open_generic_tr, 5497 .read = tracing_entries_read, 5498 .write = tracing_entries_write, 5499 .llseek = generic_file_llseek, 5500 .release = tracing_release_generic_tr, 5501 }; 5502 5503 static const struct file_operations tracing_total_entries_fops = { 5504 .open = tracing_open_generic_tr, 5505 .read = tracing_total_entries_read, 5506 .llseek = generic_file_llseek, 5507 .release = tracing_release_generic_tr, 5508 }; 5509 5510 static const struct file_operations tracing_free_buffer_fops = { 5511 .open = tracing_open_generic_tr, 5512 .write = tracing_free_buffer_write, 5513 .release = tracing_free_buffer_release, 5514 }; 5515 5516 static const struct file_operations tracing_mark_fops = { 5517 .open = tracing_open_generic_tr, 5518 .write = tracing_mark_write, 5519 .llseek = generic_file_llseek, 5520 .release = tracing_release_generic_tr, 5521 }; 5522 5523 static const struct file_operations trace_clock_fops = { 5524 .open = tracing_clock_open, 5525 .read = seq_read, 5526 .llseek = seq_lseek, 5527 .release = tracing_single_release_tr, 5528 .write = tracing_clock_write, 5529 }; 5530 5531 #ifdef CONFIG_TRACER_SNAPSHOT 5532 static const struct file_operations snapshot_fops = { 5533 .open = tracing_snapshot_open, 5534 .read = seq_read, 5535 .write = tracing_snapshot_write, 5536 .llseek = tracing_lseek, 5537 .release = tracing_snapshot_release, 5538 }; 5539 5540 static const struct file_operations snapshot_raw_fops = { 5541 .open = snapshot_raw_open, 5542 .read = tracing_buffers_read, 5543 .release = tracing_buffers_release, 5544 .splice_read = tracing_buffers_splice_read, 5545 .llseek = no_llseek, 5546 }; 5547 5548 #endif /* CONFIG_TRACER_SNAPSHOT */ 5549 5550 static int tracing_buffers_open(struct inode *inode, struct file *filp) 5551 { 5552 struct trace_array *tr = inode->i_private; 5553 struct ftrace_buffer_info *info; 5554 int ret; 5555 5556 if (tracing_disabled) 5557 return -ENODEV; 5558 5559 if (trace_array_get(tr) < 0) 5560 return -ENODEV; 5561 5562 info = kzalloc(sizeof(*info), GFP_KERNEL); 5563 if (!info) { 5564 trace_array_put(tr); 5565 return -ENOMEM; 5566 } 5567 5568 mutex_lock(&trace_types_lock); 5569 5570 info->iter.tr = tr; 5571 info->iter.cpu_file = tracing_get_cpu(inode); 5572 info->iter.trace = tr->current_trace; 5573 info->iter.trace_buffer = &tr->trace_buffer; 5574 info->spare = NULL; 5575 /* Force reading ring buffer for first read */ 5576 info->read = (unsigned int)-1; 5577 5578 filp->private_data = info; 5579 5580 tr->current_trace->ref++; 5581 5582 mutex_unlock(&trace_types_lock); 5583 5584 ret = nonseekable_open(inode, filp); 5585 if (ret < 0) 5586 trace_array_put(tr); 5587 5588 return ret; 5589 } 5590 5591 static unsigned int 5592 tracing_buffers_poll(struct file *filp, poll_table *poll_table) 5593 { 5594 struct ftrace_buffer_info *info = filp->private_data; 5595 struct trace_iterator *iter = &info->iter; 5596 5597 return trace_poll(iter, filp, poll_table); 5598 } 5599 5600 static ssize_t 5601 tracing_buffers_read(struct file *filp, char __user *ubuf, 5602 size_t count, loff_t *ppos) 5603 { 5604 struct ftrace_buffer_info *info = filp->private_data; 5605 struct trace_iterator *iter = &info->iter; 5606 ssize_t ret; 5607 ssize_t size; 5608 5609 if (!count) 5610 return 0; 5611 5612 #ifdef CONFIG_TRACER_MAX_TRACE 5613 if (iter->snapshot && iter->tr->current_trace->use_max_tr) 5614 return -EBUSY; 5615 #endif 5616 5617 if (!info->spare) 5618 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer, 5619 iter->cpu_file); 5620 if (!info->spare) 5621 return -ENOMEM; 5622 5623 /* Do we have previous read data to read? */ 5624 if (info->read < PAGE_SIZE) 5625 goto read; 5626 5627 again: 5628 trace_access_lock(iter->cpu_file); 5629 ret = ring_buffer_read_page(iter->trace_buffer->buffer, 5630 &info->spare, 5631 count, 5632 iter->cpu_file, 0); 5633 trace_access_unlock(iter->cpu_file); 5634 5635 if (ret < 0) { 5636 if (trace_empty(iter)) { 5637 if ((filp->f_flags & O_NONBLOCK)) 5638 return -EAGAIN; 5639 5640 ret = wait_on_pipe(iter, false); 5641 if (ret) 5642 return ret; 5643 5644 goto again; 5645 } 5646 return 0; 5647 } 5648 5649 info->read = 0; 5650 read: 5651 size = PAGE_SIZE - info->read; 5652 if (size > count) 5653 size = count; 5654 5655 ret = copy_to_user(ubuf, info->spare + info->read, size); 5656 if (ret == size) 5657 return -EFAULT; 5658 5659 size -= ret; 5660 5661 *ppos += size; 5662 info->read += size; 5663 5664 return size; 5665 } 5666 5667 static int tracing_buffers_release(struct inode *inode, struct file *file) 5668 { 5669 struct ftrace_buffer_info *info = file->private_data; 5670 struct trace_iterator *iter = &info->iter; 5671 5672 mutex_lock(&trace_types_lock); 5673 5674 iter->tr->current_trace->ref--; 5675 5676 __trace_array_put(iter->tr); 5677 5678 if (info->spare) 5679 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare); 5680 kfree(info); 5681 5682 mutex_unlock(&trace_types_lock); 5683 5684 return 0; 5685 } 5686 5687 struct buffer_ref { 5688 struct ring_buffer *buffer; 5689 void *page; 5690 int ref; 5691 }; 5692 5693 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe, 5694 struct pipe_buffer *buf) 5695 { 5696 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 5697 5698 if (--ref->ref) 5699 return; 5700 5701 ring_buffer_free_read_page(ref->buffer, ref->page); 5702 kfree(ref); 5703 buf->private = 0; 5704 } 5705 5706 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe, 5707 struct pipe_buffer *buf) 5708 { 5709 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 5710 5711 ref->ref++; 5712 } 5713 5714 /* Pipe buffer operations for a buffer. */ 5715 static const struct pipe_buf_operations buffer_pipe_buf_ops = { 5716 .can_merge = 0, 5717 .confirm = generic_pipe_buf_confirm, 5718 .release = buffer_pipe_buf_release, 5719 .steal = generic_pipe_buf_steal, 5720 .get = buffer_pipe_buf_get, 5721 }; 5722 5723 /* 5724 * Callback from splice_to_pipe(), if we need to release some pages 5725 * at the end of the spd in case we error'ed out in filling the pipe. 5726 */ 5727 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i) 5728 { 5729 struct buffer_ref *ref = 5730 (struct buffer_ref *)spd->partial[i].private; 5731 5732 if (--ref->ref) 5733 return; 5734 5735 ring_buffer_free_read_page(ref->buffer, ref->page); 5736 kfree(ref); 5737 spd->partial[i].private = 0; 5738 } 5739 5740 static ssize_t 5741 tracing_buffers_splice_read(struct file *file, loff_t *ppos, 5742 struct pipe_inode_info *pipe, size_t len, 5743 unsigned int flags) 5744 { 5745 struct ftrace_buffer_info *info = file->private_data; 5746 struct trace_iterator *iter = &info->iter; 5747 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 5748 struct page *pages_def[PIPE_DEF_BUFFERS]; 5749 struct splice_pipe_desc spd = { 5750 .pages = pages_def, 5751 .partial = partial_def, 5752 .nr_pages_max = PIPE_DEF_BUFFERS, 5753 .flags = flags, 5754 .ops = &buffer_pipe_buf_ops, 5755 .spd_release = buffer_spd_release, 5756 }; 5757 struct buffer_ref *ref; 5758 int entries, size, i; 5759 ssize_t ret = 0; 5760 5761 #ifdef CONFIG_TRACER_MAX_TRACE 5762 if (iter->snapshot && iter->tr->current_trace->use_max_tr) 5763 return -EBUSY; 5764 #endif 5765 5766 if (splice_grow_spd(pipe, &spd)) 5767 return -ENOMEM; 5768 5769 if (*ppos & (PAGE_SIZE - 1)) 5770 return -EINVAL; 5771 5772 if (len & (PAGE_SIZE - 1)) { 5773 if (len < PAGE_SIZE) 5774 return -EINVAL; 5775 len &= PAGE_MASK; 5776 } 5777 5778 again: 5779 trace_access_lock(iter->cpu_file); 5780 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file); 5781 5782 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) { 5783 struct page *page; 5784 int r; 5785 5786 ref = kzalloc(sizeof(*ref), GFP_KERNEL); 5787 if (!ref) { 5788 ret = -ENOMEM; 5789 break; 5790 } 5791 5792 ref->ref = 1; 5793 ref->buffer = iter->trace_buffer->buffer; 5794 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file); 5795 if (!ref->page) { 5796 ret = -ENOMEM; 5797 kfree(ref); 5798 break; 5799 } 5800 5801 r = ring_buffer_read_page(ref->buffer, &ref->page, 5802 len, iter->cpu_file, 1); 5803 if (r < 0) { 5804 ring_buffer_free_read_page(ref->buffer, ref->page); 5805 kfree(ref); 5806 break; 5807 } 5808 5809 /* 5810 * zero out any left over data, this is going to 5811 * user land. 5812 */ 5813 size = ring_buffer_page_len(ref->page); 5814 if (size < PAGE_SIZE) 5815 memset(ref->page + size, 0, PAGE_SIZE - size); 5816 5817 page = virt_to_page(ref->page); 5818 5819 spd.pages[i] = page; 5820 spd.partial[i].len = PAGE_SIZE; 5821 spd.partial[i].offset = 0; 5822 spd.partial[i].private = (unsigned long)ref; 5823 spd.nr_pages++; 5824 *ppos += PAGE_SIZE; 5825 5826 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file); 5827 } 5828 5829 trace_access_unlock(iter->cpu_file); 5830 spd.nr_pages = i; 5831 5832 /* did we read anything? */ 5833 if (!spd.nr_pages) { 5834 if (ret) 5835 return ret; 5836 5837 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) 5838 return -EAGAIN; 5839 5840 ret = wait_on_pipe(iter, true); 5841 if (ret) 5842 return ret; 5843 5844 goto again; 5845 } 5846 5847 ret = splice_to_pipe(pipe, &spd); 5848 splice_shrink_spd(&spd); 5849 5850 return ret; 5851 } 5852 5853 static const struct file_operations tracing_buffers_fops = { 5854 .open = tracing_buffers_open, 5855 .read = tracing_buffers_read, 5856 .poll = tracing_buffers_poll, 5857 .release = tracing_buffers_release, 5858 .splice_read = tracing_buffers_splice_read, 5859 .llseek = no_llseek, 5860 }; 5861 5862 static ssize_t 5863 tracing_stats_read(struct file *filp, char __user *ubuf, 5864 size_t count, loff_t *ppos) 5865 { 5866 struct inode *inode = file_inode(filp); 5867 struct trace_array *tr = inode->i_private; 5868 struct trace_buffer *trace_buf = &tr->trace_buffer; 5869 int cpu = tracing_get_cpu(inode); 5870 struct trace_seq *s; 5871 unsigned long cnt; 5872 unsigned long long t; 5873 unsigned long usec_rem; 5874 5875 s = kmalloc(sizeof(*s), GFP_KERNEL); 5876 if (!s) 5877 return -ENOMEM; 5878 5879 trace_seq_init(s); 5880 5881 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu); 5882 trace_seq_printf(s, "entries: %ld\n", cnt); 5883 5884 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu); 5885 trace_seq_printf(s, "overrun: %ld\n", cnt); 5886 5887 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu); 5888 trace_seq_printf(s, "commit overrun: %ld\n", cnt); 5889 5890 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu); 5891 trace_seq_printf(s, "bytes: %ld\n", cnt); 5892 5893 if (trace_clocks[tr->clock_id].in_ns) { 5894 /* local or global for trace_clock */ 5895 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); 5896 usec_rem = do_div(t, USEC_PER_SEC); 5897 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", 5898 t, usec_rem); 5899 5900 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu)); 5901 usec_rem = do_div(t, USEC_PER_SEC); 5902 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem); 5903 } else { 5904 /* counter or tsc mode for trace_clock */ 5905 trace_seq_printf(s, "oldest event ts: %llu\n", 5906 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); 5907 5908 trace_seq_printf(s, "now ts: %llu\n", 5909 ring_buffer_time_stamp(trace_buf->buffer, cpu)); 5910 } 5911 5912 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu); 5913 trace_seq_printf(s, "dropped events: %ld\n", cnt); 5914 5915 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu); 5916 trace_seq_printf(s, "read events: %ld\n", cnt); 5917 5918 count = simple_read_from_buffer(ubuf, count, ppos, 5919 s->buffer, trace_seq_used(s)); 5920 5921 kfree(s); 5922 5923 return count; 5924 } 5925 5926 static const struct file_operations tracing_stats_fops = { 5927 .open = tracing_open_generic_tr, 5928 .read = tracing_stats_read, 5929 .llseek = generic_file_llseek, 5930 .release = tracing_release_generic_tr, 5931 }; 5932 5933 #ifdef CONFIG_DYNAMIC_FTRACE 5934 5935 int __weak ftrace_arch_read_dyn_info(char *buf, int size) 5936 { 5937 return 0; 5938 } 5939 5940 static ssize_t 5941 tracing_read_dyn_info(struct file *filp, char __user *ubuf, 5942 size_t cnt, loff_t *ppos) 5943 { 5944 static char ftrace_dyn_info_buffer[1024]; 5945 static DEFINE_MUTEX(dyn_info_mutex); 5946 unsigned long *p = filp->private_data; 5947 char *buf = ftrace_dyn_info_buffer; 5948 int size = ARRAY_SIZE(ftrace_dyn_info_buffer); 5949 int r; 5950 5951 mutex_lock(&dyn_info_mutex); 5952 r = sprintf(buf, "%ld ", *p); 5953 5954 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r); 5955 buf[r++] = '\n'; 5956 5957 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5958 5959 mutex_unlock(&dyn_info_mutex); 5960 5961 return r; 5962 } 5963 5964 static const struct file_operations tracing_dyn_info_fops = { 5965 .open = tracing_open_generic, 5966 .read = tracing_read_dyn_info, 5967 .llseek = generic_file_llseek, 5968 }; 5969 #endif /* CONFIG_DYNAMIC_FTRACE */ 5970 5971 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) 5972 static void 5973 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data) 5974 { 5975 tracing_snapshot(); 5976 } 5977 5978 static void 5979 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data) 5980 { 5981 unsigned long *count = (long *)data; 5982 5983 if (!*count) 5984 return; 5985 5986 if (*count != -1) 5987 (*count)--; 5988 5989 tracing_snapshot(); 5990 } 5991 5992 static int 5993 ftrace_snapshot_print(struct seq_file *m, unsigned long ip, 5994 struct ftrace_probe_ops *ops, void *data) 5995 { 5996 long count = (long)data; 5997 5998 seq_printf(m, "%ps:", (void *)ip); 5999 6000 seq_puts(m, "snapshot"); 6001 6002 if (count == -1) 6003 seq_puts(m, ":unlimited\n"); 6004 else 6005 seq_printf(m, ":count=%ld\n", count); 6006 6007 return 0; 6008 } 6009 6010 static struct ftrace_probe_ops snapshot_probe_ops = { 6011 .func = ftrace_snapshot, 6012 .print = ftrace_snapshot_print, 6013 }; 6014 6015 static struct ftrace_probe_ops snapshot_count_probe_ops = { 6016 .func = ftrace_count_snapshot, 6017 .print = ftrace_snapshot_print, 6018 }; 6019 6020 static int 6021 ftrace_trace_snapshot_callback(struct ftrace_hash *hash, 6022 char *glob, char *cmd, char *param, int enable) 6023 { 6024 struct ftrace_probe_ops *ops; 6025 void *count = (void *)-1; 6026 char *number; 6027 int ret; 6028 6029 /* hash funcs only work with set_ftrace_filter */ 6030 if (!enable) 6031 return -EINVAL; 6032 6033 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops; 6034 6035 if (glob[0] == '!') { 6036 unregister_ftrace_function_probe_func(glob+1, ops); 6037 return 0; 6038 } 6039 6040 if (!param) 6041 goto out_reg; 6042 6043 number = strsep(¶m, ":"); 6044 6045 if (!strlen(number)) 6046 goto out_reg; 6047 6048 /* 6049 * We use the callback data field (which is a pointer) 6050 * as our counter. 6051 */ 6052 ret = kstrtoul(number, 0, (unsigned long *)&count); 6053 if (ret) 6054 return ret; 6055 6056 out_reg: 6057 ret = register_ftrace_function_probe(glob, ops, count); 6058 6059 if (ret >= 0) 6060 alloc_snapshot(&global_trace); 6061 6062 return ret < 0 ? ret : 0; 6063 } 6064 6065 static struct ftrace_func_command ftrace_snapshot_cmd = { 6066 .name = "snapshot", 6067 .func = ftrace_trace_snapshot_callback, 6068 }; 6069 6070 static __init int register_snapshot_cmd(void) 6071 { 6072 return register_ftrace_command(&ftrace_snapshot_cmd); 6073 } 6074 #else 6075 static inline __init int register_snapshot_cmd(void) { return 0; } 6076 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */ 6077 6078 static struct dentry *tracing_get_dentry(struct trace_array *tr) 6079 { 6080 if (WARN_ON(!tr->dir)) 6081 return ERR_PTR(-ENODEV); 6082 6083 /* Top directory uses NULL as the parent */ 6084 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 6085 return NULL; 6086 6087 /* All sub buffers have a descriptor */ 6088 return tr->dir; 6089 } 6090 6091 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu) 6092 { 6093 struct dentry *d_tracer; 6094 6095 if (tr->percpu_dir) 6096 return tr->percpu_dir; 6097 6098 d_tracer = tracing_get_dentry(tr); 6099 if (IS_ERR(d_tracer)) 6100 return NULL; 6101 6102 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer); 6103 6104 WARN_ONCE(!tr->percpu_dir, 6105 "Could not create tracefs directory 'per_cpu/%d'\n", cpu); 6106 6107 return tr->percpu_dir; 6108 } 6109 6110 static struct dentry * 6111 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent, 6112 void *data, long cpu, const struct file_operations *fops) 6113 { 6114 struct dentry *ret = trace_create_file(name, mode, parent, data, fops); 6115 6116 if (ret) /* See tracing_get_cpu() */ 6117 d_inode(ret)->i_cdev = (void *)(cpu + 1); 6118 return ret; 6119 } 6120 6121 static void 6122 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu) 6123 { 6124 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu); 6125 struct dentry *d_cpu; 6126 char cpu_dir[30]; /* 30 characters should be more than enough */ 6127 6128 if (!d_percpu) 6129 return; 6130 6131 snprintf(cpu_dir, 30, "cpu%ld", cpu); 6132 d_cpu = tracefs_create_dir(cpu_dir, d_percpu); 6133 if (!d_cpu) { 6134 pr_warning("Could not create tracefs '%s' entry\n", cpu_dir); 6135 return; 6136 } 6137 6138 /* per cpu trace_pipe */ 6139 trace_create_cpu_file("trace_pipe", 0444, d_cpu, 6140 tr, cpu, &tracing_pipe_fops); 6141 6142 /* per cpu trace */ 6143 trace_create_cpu_file("trace", 0644, d_cpu, 6144 tr, cpu, &tracing_fops); 6145 6146 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu, 6147 tr, cpu, &tracing_buffers_fops); 6148 6149 trace_create_cpu_file("stats", 0444, d_cpu, 6150 tr, cpu, &tracing_stats_fops); 6151 6152 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu, 6153 tr, cpu, &tracing_entries_fops); 6154 6155 #ifdef CONFIG_TRACER_SNAPSHOT 6156 trace_create_cpu_file("snapshot", 0644, d_cpu, 6157 tr, cpu, &snapshot_fops); 6158 6159 trace_create_cpu_file("snapshot_raw", 0444, d_cpu, 6160 tr, cpu, &snapshot_raw_fops); 6161 #endif 6162 } 6163 6164 #ifdef CONFIG_FTRACE_SELFTEST 6165 /* Let selftest have access to static functions in this file */ 6166 #include "trace_selftest.c" 6167 #endif 6168 6169 static ssize_t 6170 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt, 6171 loff_t *ppos) 6172 { 6173 struct trace_option_dentry *topt = filp->private_data; 6174 char *buf; 6175 6176 if (topt->flags->val & topt->opt->bit) 6177 buf = "1\n"; 6178 else 6179 buf = "0\n"; 6180 6181 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 6182 } 6183 6184 static ssize_t 6185 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt, 6186 loff_t *ppos) 6187 { 6188 struct trace_option_dentry *topt = filp->private_data; 6189 unsigned long val; 6190 int ret; 6191 6192 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 6193 if (ret) 6194 return ret; 6195 6196 if (val != 0 && val != 1) 6197 return -EINVAL; 6198 6199 if (!!(topt->flags->val & topt->opt->bit) != val) { 6200 mutex_lock(&trace_types_lock); 6201 ret = __set_tracer_option(topt->tr, topt->flags, 6202 topt->opt, !val); 6203 mutex_unlock(&trace_types_lock); 6204 if (ret) 6205 return ret; 6206 } 6207 6208 *ppos += cnt; 6209 6210 return cnt; 6211 } 6212 6213 6214 static const struct file_operations trace_options_fops = { 6215 .open = tracing_open_generic, 6216 .read = trace_options_read, 6217 .write = trace_options_write, 6218 .llseek = generic_file_llseek, 6219 }; 6220 6221 /* 6222 * In order to pass in both the trace_array descriptor as well as the index 6223 * to the flag that the trace option file represents, the trace_array 6224 * has a character array of trace_flags_index[], which holds the index 6225 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc. 6226 * The address of this character array is passed to the flag option file 6227 * read/write callbacks. 6228 * 6229 * In order to extract both the index and the trace_array descriptor, 6230 * get_tr_index() uses the following algorithm. 6231 * 6232 * idx = *ptr; 6233 * 6234 * As the pointer itself contains the address of the index (remember 6235 * index[1] == 1). 6236 * 6237 * Then to get the trace_array descriptor, by subtracting that index 6238 * from the ptr, we get to the start of the index itself. 6239 * 6240 * ptr - idx == &index[0] 6241 * 6242 * Then a simple container_of() from that pointer gets us to the 6243 * trace_array descriptor. 6244 */ 6245 static void get_tr_index(void *data, struct trace_array **ptr, 6246 unsigned int *pindex) 6247 { 6248 *pindex = *(unsigned char *)data; 6249 6250 *ptr = container_of(data - *pindex, struct trace_array, 6251 trace_flags_index); 6252 } 6253 6254 static ssize_t 6255 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt, 6256 loff_t *ppos) 6257 { 6258 void *tr_index = filp->private_data; 6259 struct trace_array *tr; 6260 unsigned int index; 6261 char *buf; 6262 6263 get_tr_index(tr_index, &tr, &index); 6264 6265 if (tr->trace_flags & (1 << index)) 6266 buf = "1\n"; 6267 else 6268 buf = "0\n"; 6269 6270 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 6271 } 6272 6273 static ssize_t 6274 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt, 6275 loff_t *ppos) 6276 { 6277 void *tr_index = filp->private_data; 6278 struct trace_array *tr; 6279 unsigned int index; 6280 unsigned long val; 6281 int ret; 6282 6283 get_tr_index(tr_index, &tr, &index); 6284 6285 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 6286 if (ret) 6287 return ret; 6288 6289 if (val != 0 && val != 1) 6290 return -EINVAL; 6291 6292 mutex_lock(&trace_types_lock); 6293 ret = set_tracer_flag(tr, 1 << index, val); 6294 mutex_unlock(&trace_types_lock); 6295 6296 if (ret < 0) 6297 return ret; 6298 6299 *ppos += cnt; 6300 6301 return cnt; 6302 } 6303 6304 static const struct file_operations trace_options_core_fops = { 6305 .open = tracing_open_generic, 6306 .read = trace_options_core_read, 6307 .write = trace_options_core_write, 6308 .llseek = generic_file_llseek, 6309 }; 6310 6311 struct dentry *trace_create_file(const char *name, 6312 umode_t mode, 6313 struct dentry *parent, 6314 void *data, 6315 const struct file_operations *fops) 6316 { 6317 struct dentry *ret; 6318 6319 ret = tracefs_create_file(name, mode, parent, data, fops); 6320 if (!ret) 6321 pr_warning("Could not create tracefs '%s' entry\n", name); 6322 6323 return ret; 6324 } 6325 6326 6327 static struct dentry *trace_options_init_dentry(struct trace_array *tr) 6328 { 6329 struct dentry *d_tracer; 6330 6331 if (tr->options) 6332 return tr->options; 6333 6334 d_tracer = tracing_get_dentry(tr); 6335 if (IS_ERR(d_tracer)) 6336 return NULL; 6337 6338 tr->options = tracefs_create_dir("options", d_tracer); 6339 if (!tr->options) { 6340 pr_warning("Could not create tracefs directory 'options'\n"); 6341 return NULL; 6342 } 6343 6344 return tr->options; 6345 } 6346 6347 static void 6348 create_trace_option_file(struct trace_array *tr, 6349 struct trace_option_dentry *topt, 6350 struct tracer_flags *flags, 6351 struct tracer_opt *opt) 6352 { 6353 struct dentry *t_options; 6354 6355 t_options = trace_options_init_dentry(tr); 6356 if (!t_options) 6357 return; 6358 6359 topt->flags = flags; 6360 topt->opt = opt; 6361 topt->tr = tr; 6362 6363 topt->entry = trace_create_file(opt->name, 0644, t_options, topt, 6364 &trace_options_fops); 6365 6366 } 6367 6368 static void 6369 create_trace_option_files(struct trace_array *tr, struct tracer *tracer) 6370 { 6371 struct trace_option_dentry *topts; 6372 struct trace_options *tr_topts; 6373 struct tracer_flags *flags; 6374 struct tracer_opt *opts; 6375 int cnt; 6376 int i; 6377 6378 if (!tracer) 6379 return; 6380 6381 flags = tracer->flags; 6382 6383 if (!flags || !flags->opts) 6384 return; 6385 6386 /* 6387 * If this is an instance, only create flags for tracers 6388 * the instance may have. 6389 */ 6390 if (!trace_ok_for_array(tracer, tr)) 6391 return; 6392 6393 for (i = 0; i < tr->nr_topts; i++) { 6394 /* 6395 * Check if these flags have already been added. 6396 * Some tracers share flags. 6397 */ 6398 if (tr->topts[i].tracer->flags == tracer->flags) 6399 return; 6400 } 6401 6402 opts = flags->opts; 6403 6404 for (cnt = 0; opts[cnt].name; cnt++) 6405 ; 6406 6407 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL); 6408 if (!topts) 6409 return; 6410 6411 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1), 6412 GFP_KERNEL); 6413 if (!tr_topts) { 6414 kfree(topts); 6415 return; 6416 } 6417 6418 tr->topts = tr_topts; 6419 tr->topts[tr->nr_topts].tracer = tracer; 6420 tr->topts[tr->nr_topts].topts = topts; 6421 tr->nr_topts++; 6422 6423 for (cnt = 0; opts[cnt].name; cnt++) { 6424 create_trace_option_file(tr, &topts[cnt], flags, 6425 &opts[cnt]); 6426 WARN_ONCE(topts[cnt].entry == NULL, 6427 "Failed to create trace option: %s", 6428 opts[cnt].name); 6429 } 6430 } 6431 6432 static struct dentry * 6433 create_trace_option_core_file(struct trace_array *tr, 6434 const char *option, long index) 6435 { 6436 struct dentry *t_options; 6437 6438 t_options = trace_options_init_dentry(tr); 6439 if (!t_options) 6440 return NULL; 6441 6442 return trace_create_file(option, 0644, t_options, 6443 (void *)&tr->trace_flags_index[index], 6444 &trace_options_core_fops); 6445 } 6446 6447 static void create_trace_options_dir(struct trace_array *tr) 6448 { 6449 struct dentry *t_options; 6450 bool top_level = tr == &global_trace; 6451 int i; 6452 6453 t_options = trace_options_init_dentry(tr); 6454 if (!t_options) 6455 return; 6456 6457 for (i = 0; trace_options[i]; i++) { 6458 if (top_level || 6459 !((1 << i) & TOP_LEVEL_TRACE_FLAGS)) 6460 create_trace_option_core_file(tr, trace_options[i], i); 6461 } 6462 } 6463 6464 static ssize_t 6465 rb_simple_read(struct file *filp, char __user *ubuf, 6466 size_t cnt, loff_t *ppos) 6467 { 6468 struct trace_array *tr = filp->private_data; 6469 char buf[64]; 6470 int r; 6471 6472 r = tracer_tracing_is_on(tr); 6473 r = sprintf(buf, "%d\n", r); 6474 6475 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 6476 } 6477 6478 static ssize_t 6479 rb_simple_write(struct file *filp, const char __user *ubuf, 6480 size_t cnt, loff_t *ppos) 6481 { 6482 struct trace_array *tr = filp->private_data; 6483 struct ring_buffer *buffer = tr->trace_buffer.buffer; 6484 unsigned long val; 6485 int ret; 6486 6487 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 6488 if (ret) 6489 return ret; 6490 6491 if (buffer) { 6492 mutex_lock(&trace_types_lock); 6493 if (val) { 6494 tracer_tracing_on(tr); 6495 if (tr->current_trace->start) 6496 tr->current_trace->start(tr); 6497 } else { 6498 tracer_tracing_off(tr); 6499 if (tr->current_trace->stop) 6500 tr->current_trace->stop(tr); 6501 } 6502 mutex_unlock(&trace_types_lock); 6503 } 6504 6505 (*ppos)++; 6506 6507 return cnt; 6508 } 6509 6510 static const struct file_operations rb_simple_fops = { 6511 .open = tracing_open_generic_tr, 6512 .read = rb_simple_read, 6513 .write = rb_simple_write, 6514 .release = tracing_release_generic_tr, 6515 .llseek = default_llseek, 6516 }; 6517 6518 struct dentry *trace_instance_dir; 6519 6520 static void 6521 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer); 6522 6523 static int 6524 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size) 6525 { 6526 enum ring_buffer_flags rb_flags; 6527 6528 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0; 6529 6530 buf->tr = tr; 6531 6532 buf->buffer = ring_buffer_alloc(size, rb_flags); 6533 if (!buf->buffer) 6534 return -ENOMEM; 6535 6536 buf->data = alloc_percpu(struct trace_array_cpu); 6537 if (!buf->data) { 6538 ring_buffer_free(buf->buffer); 6539 return -ENOMEM; 6540 } 6541 6542 /* Allocate the first page for all buffers */ 6543 set_buffer_entries(&tr->trace_buffer, 6544 ring_buffer_size(tr->trace_buffer.buffer, 0)); 6545 6546 return 0; 6547 } 6548 6549 static int allocate_trace_buffers(struct trace_array *tr, int size) 6550 { 6551 int ret; 6552 6553 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size); 6554 if (ret) 6555 return ret; 6556 6557 #ifdef CONFIG_TRACER_MAX_TRACE 6558 ret = allocate_trace_buffer(tr, &tr->max_buffer, 6559 allocate_snapshot ? size : 1); 6560 if (WARN_ON(ret)) { 6561 ring_buffer_free(tr->trace_buffer.buffer); 6562 free_percpu(tr->trace_buffer.data); 6563 return -ENOMEM; 6564 } 6565 tr->allocated_snapshot = allocate_snapshot; 6566 6567 /* 6568 * Only the top level trace array gets its snapshot allocated 6569 * from the kernel command line. 6570 */ 6571 allocate_snapshot = false; 6572 #endif 6573 return 0; 6574 } 6575 6576 static void free_trace_buffer(struct trace_buffer *buf) 6577 { 6578 if (buf->buffer) { 6579 ring_buffer_free(buf->buffer); 6580 buf->buffer = NULL; 6581 free_percpu(buf->data); 6582 buf->data = NULL; 6583 } 6584 } 6585 6586 static void free_trace_buffers(struct trace_array *tr) 6587 { 6588 if (!tr) 6589 return; 6590 6591 free_trace_buffer(&tr->trace_buffer); 6592 6593 #ifdef CONFIG_TRACER_MAX_TRACE 6594 free_trace_buffer(&tr->max_buffer); 6595 #endif 6596 } 6597 6598 static void init_trace_flags_index(struct trace_array *tr) 6599 { 6600 int i; 6601 6602 /* Used by the trace options files */ 6603 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) 6604 tr->trace_flags_index[i] = i; 6605 } 6606 6607 static void __update_tracer_options(struct trace_array *tr) 6608 { 6609 struct tracer *t; 6610 6611 for (t = trace_types; t; t = t->next) 6612 add_tracer_options(tr, t); 6613 } 6614 6615 static void update_tracer_options(struct trace_array *tr) 6616 { 6617 mutex_lock(&trace_types_lock); 6618 __update_tracer_options(tr); 6619 mutex_unlock(&trace_types_lock); 6620 } 6621 6622 static int instance_mkdir(const char *name) 6623 { 6624 struct trace_array *tr; 6625 int ret; 6626 6627 mutex_lock(&trace_types_lock); 6628 6629 ret = -EEXIST; 6630 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 6631 if (tr->name && strcmp(tr->name, name) == 0) 6632 goto out_unlock; 6633 } 6634 6635 ret = -ENOMEM; 6636 tr = kzalloc(sizeof(*tr), GFP_KERNEL); 6637 if (!tr) 6638 goto out_unlock; 6639 6640 tr->name = kstrdup(name, GFP_KERNEL); 6641 if (!tr->name) 6642 goto out_free_tr; 6643 6644 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL)) 6645 goto out_free_tr; 6646 6647 tr->trace_flags = global_trace.trace_flags; 6648 6649 cpumask_copy(tr->tracing_cpumask, cpu_all_mask); 6650 6651 raw_spin_lock_init(&tr->start_lock); 6652 6653 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 6654 6655 tr->current_trace = &nop_trace; 6656 6657 INIT_LIST_HEAD(&tr->systems); 6658 INIT_LIST_HEAD(&tr->events); 6659 6660 if (allocate_trace_buffers(tr, trace_buf_size) < 0) 6661 goto out_free_tr; 6662 6663 tr->dir = tracefs_create_dir(name, trace_instance_dir); 6664 if (!tr->dir) 6665 goto out_free_tr; 6666 6667 ret = event_trace_add_tracer(tr->dir, tr); 6668 if (ret) { 6669 tracefs_remove_recursive(tr->dir); 6670 goto out_free_tr; 6671 } 6672 6673 init_tracer_tracefs(tr, tr->dir); 6674 init_trace_flags_index(tr); 6675 __update_tracer_options(tr); 6676 6677 list_add(&tr->list, &ftrace_trace_arrays); 6678 6679 mutex_unlock(&trace_types_lock); 6680 6681 return 0; 6682 6683 out_free_tr: 6684 free_trace_buffers(tr); 6685 free_cpumask_var(tr->tracing_cpumask); 6686 kfree(tr->name); 6687 kfree(tr); 6688 6689 out_unlock: 6690 mutex_unlock(&trace_types_lock); 6691 6692 return ret; 6693 6694 } 6695 6696 static int instance_rmdir(const char *name) 6697 { 6698 struct trace_array *tr; 6699 int found = 0; 6700 int ret; 6701 int i; 6702 6703 mutex_lock(&trace_types_lock); 6704 6705 ret = -ENODEV; 6706 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 6707 if (tr->name && strcmp(tr->name, name) == 0) { 6708 found = 1; 6709 break; 6710 } 6711 } 6712 if (!found) 6713 goto out_unlock; 6714 6715 ret = -EBUSY; 6716 if (tr->ref || (tr->current_trace && tr->current_trace->ref)) 6717 goto out_unlock; 6718 6719 list_del(&tr->list); 6720 6721 tracing_set_nop(tr); 6722 event_trace_del_tracer(tr); 6723 ftrace_destroy_function_files(tr); 6724 tracefs_remove_recursive(tr->dir); 6725 free_trace_buffers(tr); 6726 6727 for (i = 0; i < tr->nr_topts; i++) { 6728 kfree(tr->topts[i].topts); 6729 } 6730 kfree(tr->topts); 6731 6732 kfree(tr->name); 6733 kfree(tr); 6734 6735 ret = 0; 6736 6737 out_unlock: 6738 mutex_unlock(&trace_types_lock); 6739 6740 return ret; 6741 } 6742 6743 static __init void create_trace_instances(struct dentry *d_tracer) 6744 { 6745 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer, 6746 instance_mkdir, 6747 instance_rmdir); 6748 if (WARN_ON(!trace_instance_dir)) 6749 return; 6750 } 6751 6752 static void 6753 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) 6754 { 6755 int cpu; 6756 6757 trace_create_file("available_tracers", 0444, d_tracer, 6758 tr, &show_traces_fops); 6759 6760 trace_create_file("current_tracer", 0644, d_tracer, 6761 tr, &set_tracer_fops); 6762 6763 trace_create_file("tracing_cpumask", 0644, d_tracer, 6764 tr, &tracing_cpumask_fops); 6765 6766 trace_create_file("trace_options", 0644, d_tracer, 6767 tr, &tracing_iter_fops); 6768 6769 trace_create_file("trace", 0644, d_tracer, 6770 tr, &tracing_fops); 6771 6772 trace_create_file("trace_pipe", 0444, d_tracer, 6773 tr, &tracing_pipe_fops); 6774 6775 trace_create_file("buffer_size_kb", 0644, d_tracer, 6776 tr, &tracing_entries_fops); 6777 6778 trace_create_file("buffer_total_size_kb", 0444, d_tracer, 6779 tr, &tracing_total_entries_fops); 6780 6781 trace_create_file("free_buffer", 0200, d_tracer, 6782 tr, &tracing_free_buffer_fops); 6783 6784 trace_create_file("trace_marker", 0220, d_tracer, 6785 tr, &tracing_mark_fops); 6786 6787 trace_create_file("trace_clock", 0644, d_tracer, tr, 6788 &trace_clock_fops); 6789 6790 trace_create_file("tracing_on", 0644, d_tracer, 6791 tr, &rb_simple_fops); 6792 6793 create_trace_options_dir(tr); 6794 6795 #ifdef CONFIG_TRACER_MAX_TRACE 6796 trace_create_file("tracing_max_latency", 0644, d_tracer, 6797 &tr->max_latency, &tracing_max_lat_fops); 6798 #endif 6799 6800 if (ftrace_create_function_files(tr, d_tracer)) 6801 WARN(1, "Could not allocate function filter files"); 6802 6803 #ifdef CONFIG_TRACER_SNAPSHOT 6804 trace_create_file("snapshot", 0644, d_tracer, 6805 tr, &snapshot_fops); 6806 #endif 6807 6808 for_each_tracing_cpu(cpu) 6809 tracing_init_tracefs_percpu(tr, cpu); 6810 6811 } 6812 6813 static struct vfsmount *trace_automount(void *ingore) 6814 { 6815 struct vfsmount *mnt; 6816 struct file_system_type *type; 6817 6818 /* 6819 * To maintain backward compatibility for tools that mount 6820 * debugfs to get to the tracing facility, tracefs is automatically 6821 * mounted to the debugfs/tracing directory. 6822 */ 6823 type = get_fs_type("tracefs"); 6824 if (!type) 6825 return NULL; 6826 mnt = vfs_kern_mount(type, 0, "tracefs", NULL); 6827 put_filesystem(type); 6828 if (IS_ERR(mnt)) 6829 return NULL; 6830 mntget(mnt); 6831 6832 return mnt; 6833 } 6834 6835 /** 6836 * tracing_init_dentry - initialize top level trace array 6837 * 6838 * This is called when creating files or directories in the tracing 6839 * directory. It is called via fs_initcall() by any of the boot up code 6840 * and expects to return the dentry of the top level tracing directory. 6841 */ 6842 struct dentry *tracing_init_dentry(void) 6843 { 6844 struct trace_array *tr = &global_trace; 6845 6846 /* The top level trace array uses NULL as parent */ 6847 if (tr->dir) 6848 return NULL; 6849 6850 if (WARN_ON(!tracefs_initialized()) || 6851 (IS_ENABLED(CONFIG_DEBUG_FS) && 6852 WARN_ON(!debugfs_initialized()))) 6853 return ERR_PTR(-ENODEV); 6854 6855 /* 6856 * As there may still be users that expect the tracing 6857 * files to exist in debugfs/tracing, we must automount 6858 * the tracefs file system there, so older tools still 6859 * work with the newer kerenl. 6860 */ 6861 tr->dir = debugfs_create_automount("tracing", NULL, 6862 trace_automount, NULL); 6863 if (!tr->dir) { 6864 pr_warn_once("Could not create debugfs directory 'tracing'\n"); 6865 return ERR_PTR(-ENOMEM); 6866 } 6867 6868 return NULL; 6869 } 6870 6871 extern struct trace_enum_map *__start_ftrace_enum_maps[]; 6872 extern struct trace_enum_map *__stop_ftrace_enum_maps[]; 6873 6874 static void __init trace_enum_init(void) 6875 { 6876 int len; 6877 6878 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps; 6879 trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len); 6880 } 6881 6882 #ifdef CONFIG_MODULES 6883 static void trace_module_add_enums(struct module *mod) 6884 { 6885 if (!mod->num_trace_enums) 6886 return; 6887 6888 /* 6889 * Modules with bad taint do not have events created, do 6890 * not bother with enums either. 6891 */ 6892 if (trace_module_has_bad_taint(mod)) 6893 return; 6894 6895 trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums); 6896 } 6897 6898 #ifdef CONFIG_TRACE_ENUM_MAP_FILE 6899 static void trace_module_remove_enums(struct module *mod) 6900 { 6901 union trace_enum_map_item *map; 6902 union trace_enum_map_item **last = &trace_enum_maps; 6903 6904 if (!mod->num_trace_enums) 6905 return; 6906 6907 mutex_lock(&trace_enum_mutex); 6908 6909 map = trace_enum_maps; 6910 6911 while (map) { 6912 if (map->head.mod == mod) 6913 break; 6914 map = trace_enum_jmp_to_tail(map); 6915 last = &map->tail.next; 6916 map = map->tail.next; 6917 } 6918 if (!map) 6919 goto out; 6920 6921 *last = trace_enum_jmp_to_tail(map)->tail.next; 6922 kfree(map); 6923 out: 6924 mutex_unlock(&trace_enum_mutex); 6925 } 6926 #else 6927 static inline void trace_module_remove_enums(struct module *mod) { } 6928 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */ 6929 6930 static int trace_module_notify(struct notifier_block *self, 6931 unsigned long val, void *data) 6932 { 6933 struct module *mod = data; 6934 6935 switch (val) { 6936 case MODULE_STATE_COMING: 6937 trace_module_add_enums(mod); 6938 break; 6939 case MODULE_STATE_GOING: 6940 trace_module_remove_enums(mod); 6941 break; 6942 } 6943 6944 return 0; 6945 } 6946 6947 static struct notifier_block trace_module_nb = { 6948 .notifier_call = trace_module_notify, 6949 .priority = 0, 6950 }; 6951 #endif /* CONFIG_MODULES */ 6952 6953 static __init int tracer_init_tracefs(void) 6954 { 6955 struct dentry *d_tracer; 6956 6957 trace_access_lock_init(); 6958 6959 d_tracer = tracing_init_dentry(); 6960 if (IS_ERR(d_tracer)) 6961 return 0; 6962 6963 init_tracer_tracefs(&global_trace, d_tracer); 6964 6965 trace_create_file("tracing_thresh", 0644, d_tracer, 6966 &global_trace, &tracing_thresh_fops); 6967 6968 trace_create_file("README", 0444, d_tracer, 6969 NULL, &tracing_readme_fops); 6970 6971 trace_create_file("saved_cmdlines", 0444, d_tracer, 6972 NULL, &tracing_saved_cmdlines_fops); 6973 6974 trace_create_file("saved_cmdlines_size", 0644, d_tracer, 6975 NULL, &tracing_saved_cmdlines_size_fops); 6976 6977 trace_enum_init(); 6978 6979 trace_create_enum_file(d_tracer); 6980 6981 #ifdef CONFIG_MODULES 6982 register_module_notifier(&trace_module_nb); 6983 #endif 6984 6985 #ifdef CONFIG_DYNAMIC_FTRACE 6986 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer, 6987 &ftrace_update_tot_cnt, &tracing_dyn_info_fops); 6988 #endif 6989 6990 create_trace_instances(d_tracer); 6991 6992 update_tracer_options(&global_trace); 6993 6994 return 0; 6995 } 6996 6997 static int trace_panic_handler(struct notifier_block *this, 6998 unsigned long event, void *unused) 6999 { 7000 if (ftrace_dump_on_oops) 7001 ftrace_dump(ftrace_dump_on_oops); 7002 return NOTIFY_OK; 7003 } 7004 7005 static struct notifier_block trace_panic_notifier = { 7006 .notifier_call = trace_panic_handler, 7007 .next = NULL, 7008 .priority = 150 /* priority: INT_MAX >= x >= 0 */ 7009 }; 7010 7011 static int trace_die_handler(struct notifier_block *self, 7012 unsigned long val, 7013 void *data) 7014 { 7015 switch (val) { 7016 case DIE_OOPS: 7017 if (ftrace_dump_on_oops) 7018 ftrace_dump(ftrace_dump_on_oops); 7019 break; 7020 default: 7021 break; 7022 } 7023 return NOTIFY_OK; 7024 } 7025 7026 static struct notifier_block trace_die_notifier = { 7027 .notifier_call = trace_die_handler, 7028 .priority = 200 7029 }; 7030 7031 /* 7032 * printk is set to max of 1024, we really don't need it that big. 7033 * Nothing should be printing 1000 characters anyway. 7034 */ 7035 #define TRACE_MAX_PRINT 1000 7036 7037 /* 7038 * Define here KERN_TRACE so that we have one place to modify 7039 * it if we decide to change what log level the ftrace dump 7040 * should be at. 7041 */ 7042 #define KERN_TRACE KERN_EMERG 7043 7044 void 7045 trace_printk_seq(struct trace_seq *s) 7046 { 7047 /* Probably should print a warning here. */ 7048 if (s->seq.len >= TRACE_MAX_PRINT) 7049 s->seq.len = TRACE_MAX_PRINT; 7050 7051 /* 7052 * More paranoid code. Although the buffer size is set to 7053 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just 7054 * an extra layer of protection. 7055 */ 7056 if (WARN_ON_ONCE(s->seq.len >= s->seq.size)) 7057 s->seq.len = s->seq.size - 1; 7058 7059 /* should be zero ended, but we are paranoid. */ 7060 s->buffer[s->seq.len] = 0; 7061 7062 printk(KERN_TRACE "%s", s->buffer); 7063 7064 trace_seq_init(s); 7065 } 7066 7067 void trace_init_global_iter(struct trace_iterator *iter) 7068 { 7069 iter->tr = &global_trace; 7070 iter->trace = iter->tr->current_trace; 7071 iter->cpu_file = RING_BUFFER_ALL_CPUS; 7072 iter->trace_buffer = &global_trace.trace_buffer; 7073 7074 if (iter->trace && iter->trace->open) 7075 iter->trace->open(iter); 7076 7077 /* Annotate start of buffers if we had overruns */ 7078 if (ring_buffer_overruns(iter->trace_buffer->buffer)) 7079 iter->iter_flags |= TRACE_FILE_ANNOTATE; 7080 7081 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 7082 if (trace_clocks[iter->tr->clock_id].in_ns) 7083 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 7084 } 7085 7086 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) 7087 { 7088 /* use static because iter can be a bit big for the stack */ 7089 static struct trace_iterator iter; 7090 static atomic_t dump_running; 7091 struct trace_array *tr = &global_trace; 7092 unsigned int old_userobj; 7093 unsigned long flags; 7094 int cnt = 0, cpu; 7095 7096 /* Only allow one dump user at a time. */ 7097 if (atomic_inc_return(&dump_running) != 1) { 7098 atomic_dec(&dump_running); 7099 return; 7100 } 7101 7102 /* 7103 * Always turn off tracing when we dump. 7104 * We don't need to show trace output of what happens 7105 * between multiple crashes. 7106 * 7107 * If the user does a sysrq-z, then they can re-enable 7108 * tracing with echo 1 > tracing_on. 7109 */ 7110 tracing_off(); 7111 7112 local_irq_save(flags); 7113 7114 /* Simulate the iterator */ 7115 trace_init_global_iter(&iter); 7116 7117 for_each_tracing_cpu(cpu) { 7118 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled); 7119 } 7120 7121 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ; 7122 7123 /* don't look at user memory in panic mode */ 7124 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ; 7125 7126 switch (oops_dump_mode) { 7127 case DUMP_ALL: 7128 iter.cpu_file = RING_BUFFER_ALL_CPUS; 7129 break; 7130 case DUMP_ORIG: 7131 iter.cpu_file = raw_smp_processor_id(); 7132 break; 7133 case DUMP_NONE: 7134 goto out_enable; 7135 default: 7136 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n"); 7137 iter.cpu_file = RING_BUFFER_ALL_CPUS; 7138 } 7139 7140 printk(KERN_TRACE "Dumping ftrace buffer:\n"); 7141 7142 /* Did function tracer already get disabled? */ 7143 if (ftrace_is_dead()) { 7144 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n"); 7145 printk("# MAY BE MISSING FUNCTION EVENTS\n"); 7146 } 7147 7148 /* 7149 * We need to stop all tracing on all CPUS to read the 7150 * the next buffer. This is a bit expensive, but is 7151 * not done often. We fill all what we can read, 7152 * and then release the locks again. 7153 */ 7154 7155 while (!trace_empty(&iter)) { 7156 7157 if (!cnt) 7158 printk(KERN_TRACE "---------------------------------\n"); 7159 7160 cnt++; 7161 7162 /* reset all but tr, trace, and overruns */ 7163 memset(&iter.seq, 0, 7164 sizeof(struct trace_iterator) - 7165 offsetof(struct trace_iterator, seq)); 7166 iter.iter_flags |= TRACE_FILE_LAT_FMT; 7167 iter.pos = -1; 7168 7169 if (trace_find_next_entry_inc(&iter) != NULL) { 7170 int ret; 7171 7172 ret = print_trace_line(&iter); 7173 if (ret != TRACE_TYPE_NO_CONSUME) 7174 trace_consume(&iter); 7175 } 7176 touch_nmi_watchdog(); 7177 7178 trace_printk_seq(&iter.seq); 7179 } 7180 7181 if (!cnt) 7182 printk(KERN_TRACE " (ftrace buffer empty)\n"); 7183 else 7184 printk(KERN_TRACE "---------------------------------\n"); 7185 7186 out_enable: 7187 tr->trace_flags |= old_userobj; 7188 7189 for_each_tracing_cpu(cpu) { 7190 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled); 7191 } 7192 atomic_dec(&dump_running); 7193 local_irq_restore(flags); 7194 } 7195 EXPORT_SYMBOL_GPL(ftrace_dump); 7196 7197 __init static int tracer_alloc_buffers(void) 7198 { 7199 int ring_buf_size; 7200 int ret = -ENOMEM; 7201 7202 /* 7203 * Make sure we don't accidently add more trace options 7204 * than we have bits for. 7205 */ 7206 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE); 7207 7208 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL)) 7209 goto out; 7210 7211 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL)) 7212 goto out_free_buffer_mask; 7213 7214 /* Only allocate trace_printk buffers if a trace_printk exists */ 7215 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt) 7216 /* Must be called before global_trace.buffer is allocated */ 7217 trace_printk_init_buffers(); 7218 7219 /* To save memory, keep the ring buffer size to its minimum */ 7220 if (ring_buffer_expanded) 7221 ring_buf_size = trace_buf_size; 7222 else 7223 ring_buf_size = 1; 7224 7225 cpumask_copy(tracing_buffer_mask, cpu_possible_mask); 7226 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask); 7227 7228 raw_spin_lock_init(&global_trace.start_lock); 7229 7230 /* Used for event triggers */ 7231 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE); 7232 if (!temp_buffer) 7233 goto out_free_cpumask; 7234 7235 if (trace_create_savedcmd() < 0) 7236 goto out_free_temp_buffer; 7237 7238 /* TODO: make the number of buffers hot pluggable with CPUS */ 7239 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) { 7240 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n"); 7241 WARN_ON(1); 7242 goto out_free_savedcmd; 7243 } 7244 7245 if (global_trace.buffer_disabled) 7246 tracing_off(); 7247 7248 if (trace_boot_clock) { 7249 ret = tracing_set_clock(&global_trace, trace_boot_clock); 7250 if (ret < 0) 7251 pr_warning("Trace clock %s not defined, going back to default\n", 7252 trace_boot_clock); 7253 } 7254 7255 /* 7256 * register_tracer() might reference current_trace, so it 7257 * needs to be set before we register anything. This is 7258 * just a bootstrap of current_trace anyway. 7259 */ 7260 global_trace.current_trace = &nop_trace; 7261 7262 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 7263 7264 ftrace_init_global_array_ops(&global_trace); 7265 7266 init_trace_flags_index(&global_trace); 7267 7268 register_tracer(&nop_trace); 7269 7270 /* All seems OK, enable tracing */ 7271 tracing_disabled = 0; 7272 7273 atomic_notifier_chain_register(&panic_notifier_list, 7274 &trace_panic_notifier); 7275 7276 register_die_notifier(&trace_die_notifier); 7277 7278 global_trace.flags = TRACE_ARRAY_FL_GLOBAL; 7279 7280 INIT_LIST_HEAD(&global_trace.systems); 7281 INIT_LIST_HEAD(&global_trace.events); 7282 list_add(&global_trace.list, &ftrace_trace_arrays); 7283 7284 apply_trace_boot_options(); 7285 7286 register_snapshot_cmd(); 7287 7288 return 0; 7289 7290 out_free_savedcmd: 7291 free_saved_cmdlines_buffer(savedcmd); 7292 out_free_temp_buffer: 7293 ring_buffer_free(temp_buffer); 7294 out_free_cpumask: 7295 free_cpumask_var(global_trace.tracing_cpumask); 7296 out_free_buffer_mask: 7297 free_cpumask_var(tracing_buffer_mask); 7298 out: 7299 return ret; 7300 } 7301 7302 void __init trace_init(void) 7303 { 7304 if (tracepoint_printk) { 7305 tracepoint_print_iter = 7306 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL); 7307 if (WARN_ON(!tracepoint_print_iter)) 7308 tracepoint_printk = 0; 7309 } 7310 tracer_alloc_buffers(); 7311 trace_event_init(); 7312 } 7313 7314 __init static int clear_boot_tracer(void) 7315 { 7316 /* 7317 * The default tracer at boot buffer is an init section. 7318 * This function is called in lateinit. If we did not 7319 * find the boot tracer, then clear it out, to prevent 7320 * later registration from accessing the buffer that is 7321 * about to be freed. 7322 */ 7323 if (!default_bootup_tracer) 7324 return 0; 7325 7326 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n", 7327 default_bootup_tracer); 7328 default_bootup_tracer = NULL; 7329 7330 return 0; 7331 } 7332 7333 fs_initcall(tracer_init_tracefs); 7334 late_initcall(clear_boot_tracer); 7335