1 /* 2 * ring buffer based function tracer 3 * 4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com> 5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 6 * 7 * Originally taken from the RT patch by: 8 * Arnaldo Carvalho de Melo <acme@redhat.com> 9 * 10 * Based on code from the latency_tracer, that is: 11 * Copyright (C) 2004-2006 Ingo Molnar 12 * Copyright (C) 2004 Nadia Yvette Chambers 13 */ 14 #include <linux/ring_buffer.h> 15 #include <generated/utsrelease.h> 16 #include <linux/stacktrace.h> 17 #include <linux/writeback.h> 18 #include <linux/kallsyms.h> 19 #include <linux/seq_file.h> 20 #include <linux/notifier.h> 21 #include <linux/irqflags.h> 22 #include <linux/debugfs.h> 23 #include <linux/pagemap.h> 24 #include <linux/hardirq.h> 25 #include <linux/linkage.h> 26 #include <linux/uaccess.h> 27 #include <linux/kprobes.h> 28 #include <linux/ftrace.h> 29 #include <linux/module.h> 30 #include <linux/percpu.h> 31 #include <linux/splice.h> 32 #include <linux/kdebug.h> 33 #include <linux/string.h> 34 #include <linux/rwsem.h> 35 #include <linux/slab.h> 36 #include <linux/ctype.h> 37 #include <linux/init.h> 38 #include <linux/poll.h> 39 #include <linux/nmi.h> 40 #include <linux/fs.h> 41 #include <linux/sched/rt.h> 42 43 #include "trace.h" 44 #include "trace_output.h" 45 46 /* 47 * On boot up, the ring buffer is set to the minimum size, so that 48 * we do not waste memory on systems that are not using tracing. 49 */ 50 bool ring_buffer_expanded; 51 52 /* 53 * We need to change this state when a selftest is running. 54 * A selftest will lurk into the ring-buffer to count the 55 * entries inserted during the selftest although some concurrent 56 * insertions into the ring-buffer such as trace_printk could occurred 57 * at the same time, giving false positive or negative results. 58 */ 59 static bool __read_mostly tracing_selftest_running; 60 61 /* 62 * If a tracer is running, we do not want to run SELFTEST. 63 */ 64 bool __read_mostly tracing_selftest_disabled; 65 66 /* For tracers that don't implement custom flags */ 67 static struct tracer_opt dummy_tracer_opt[] = { 68 { } 69 }; 70 71 static struct tracer_flags dummy_tracer_flags = { 72 .val = 0, 73 .opts = dummy_tracer_opt 74 }; 75 76 static int 77 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 78 { 79 return 0; 80 } 81 82 /* 83 * To prevent the comm cache from being overwritten when no 84 * tracing is active, only save the comm when a trace event 85 * occurred. 86 */ 87 static DEFINE_PER_CPU(bool, trace_cmdline_save); 88 89 /* 90 * Kill all tracing for good (never come back). 91 * It is initialized to 1 but will turn to zero if the initialization 92 * of the tracer is successful. But that is the only place that sets 93 * this back to zero. 94 */ 95 static int tracing_disabled = 1; 96 97 DEFINE_PER_CPU(int, ftrace_cpu_disabled); 98 99 cpumask_var_t __read_mostly tracing_buffer_mask; 100 101 /* 102 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops 103 * 104 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops 105 * is set, then ftrace_dump is called. This will output the contents 106 * of the ftrace buffers to the console. This is very useful for 107 * capturing traces that lead to crashes and outputing it to a 108 * serial console. 109 * 110 * It is default off, but you can enable it with either specifying 111 * "ftrace_dump_on_oops" in the kernel command line, or setting 112 * /proc/sys/kernel/ftrace_dump_on_oops 113 * Set 1 if you want to dump buffers of all CPUs 114 * Set 2 if you want to dump the buffer of the CPU that triggered oops 115 */ 116 117 enum ftrace_dump_mode ftrace_dump_on_oops; 118 119 /* When set, tracing will stop when a WARN*() is hit */ 120 int __disable_trace_on_warning; 121 122 static int tracing_set_tracer(struct trace_array *tr, const char *buf); 123 124 #define MAX_TRACER_SIZE 100 125 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; 126 static char *default_bootup_tracer; 127 128 static bool allocate_snapshot; 129 130 static int __init set_cmdline_ftrace(char *str) 131 { 132 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE); 133 default_bootup_tracer = bootup_tracer_buf; 134 /* We are using ftrace early, expand it */ 135 ring_buffer_expanded = true; 136 return 1; 137 } 138 __setup("ftrace=", set_cmdline_ftrace); 139 140 static int __init set_ftrace_dump_on_oops(char *str) 141 { 142 if (*str++ != '=' || !*str) { 143 ftrace_dump_on_oops = DUMP_ALL; 144 return 1; 145 } 146 147 if (!strcmp("orig_cpu", str)) { 148 ftrace_dump_on_oops = DUMP_ORIG; 149 return 1; 150 } 151 152 return 0; 153 } 154 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops); 155 156 static int __init stop_trace_on_warning(char *str) 157 { 158 __disable_trace_on_warning = 1; 159 return 1; 160 } 161 __setup("traceoff_on_warning=", stop_trace_on_warning); 162 163 static int __init boot_alloc_snapshot(char *str) 164 { 165 allocate_snapshot = true; 166 /* We also need the main ring buffer expanded */ 167 ring_buffer_expanded = true; 168 return 1; 169 } 170 __setup("alloc_snapshot", boot_alloc_snapshot); 171 172 173 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata; 174 static char *trace_boot_options __initdata; 175 176 static int __init set_trace_boot_options(char *str) 177 { 178 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE); 179 trace_boot_options = trace_boot_options_buf; 180 return 0; 181 } 182 __setup("trace_options=", set_trace_boot_options); 183 184 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata; 185 static char *trace_boot_clock __initdata; 186 187 static int __init set_trace_boot_clock(char *str) 188 { 189 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE); 190 trace_boot_clock = trace_boot_clock_buf; 191 return 0; 192 } 193 __setup("trace_clock=", set_trace_boot_clock); 194 195 196 unsigned long long ns2usecs(cycle_t nsec) 197 { 198 nsec += 500; 199 do_div(nsec, 1000); 200 return nsec; 201 } 202 203 /* 204 * The global_trace is the descriptor that holds the tracing 205 * buffers for the live tracing. For each CPU, it contains 206 * a link list of pages that will store trace entries. The 207 * page descriptor of the pages in the memory is used to hold 208 * the link list by linking the lru item in the page descriptor 209 * to each of the pages in the buffer per CPU. 210 * 211 * For each active CPU there is a data field that holds the 212 * pages for the buffer for that CPU. Each CPU has the same number 213 * of pages allocated for its buffer. 214 */ 215 static struct trace_array global_trace; 216 217 LIST_HEAD(ftrace_trace_arrays); 218 219 int trace_array_get(struct trace_array *this_tr) 220 { 221 struct trace_array *tr; 222 int ret = -ENODEV; 223 224 mutex_lock(&trace_types_lock); 225 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 226 if (tr == this_tr) { 227 tr->ref++; 228 ret = 0; 229 break; 230 } 231 } 232 mutex_unlock(&trace_types_lock); 233 234 return ret; 235 } 236 237 static void __trace_array_put(struct trace_array *this_tr) 238 { 239 WARN_ON(!this_tr->ref); 240 this_tr->ref--; 241 } 242 243 void trace_array_put(struct trace_array *this_tr) 244 { 245 mutex_lock(&trace_types_lock); 246 __trace_array_put(this_tr); 247 mutex_unlock(&trace_types_lock); 248 } 249 250 int filter_check_discard(struct ftrace_event_file *file, void *rec, 251 struct ring_buffer *buffer, 252 struct ring_buffer_event *event) 253 { 254 if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) && 255 !filter_match_preds(file->filter, rec)) { 256 ring_buffer_discard_commit(buffer, event); 257 return 1; 258 } 259 260 return 0; 261 } 262 EXPORT_SYMBOL_GPL(filter_check_discard); 263 264 int call_filter_check_discard(struct ftrace_event_call *call, void *rec, 265 struct ring_buffer *buffer, 266 struct ring_buffer_event *event) 267 { 268 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) && 269 !filter_match_preds(call->filter, rec)) { 270 ring_buffer_discard_commit(buffer, event); 271 return 1; 272 } 273 274 return 0; 275 } 276 EXPORT_SYMBOL_GPL(call_filter_check_discard); 277 278 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu) 279 { 280 u64 ts; 281 282 /* Early boot up does not have a buffer yet */ 283 if (!buf->buffer) 284 return trace_clock_local(); 285 286 ts = ring_buffer_time_stamp(buf->buffer, cpu); 287 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts); 288 289 return ts; 290 } 291 292 cycle_t ftrace_now(int cpu) 293 { 294 return buffer_ftrace_now(&global_trace.trace_buffer, cpu); 295 } 296 297 /** 298 * tracing_is_enabled - Show if global_trace has been disabled 299 * 300 * Shows if the global trace has been enabled or not. It uses the 301 * mirror flag "buffer_disabled" to be used in fast paths such as for 302 * the irqsoff tracer. But it may be inaccurate due to races. If you 303 * need to know the accurate state, use tracing_is_on() which is a little 304 * slower, but accurate. 305 */ 306 int tracing_is_enabled(void) 307 { 308 /* 309 * For quick access (irqsoff uses this in fast path), just 310 * return the mirror variable of the state of the ring buffer. 311 * It's a little racy, but we don't really care. 312 */ 313 smp_rmb(); 314 return !global_trace.buffer_disabled; 315 } 316 317 /* 318 * trace_buf_size is the size in bytes that is allocated 319 * for a buffer. Note, the number of bytes is always rounded 320 * to page size. 321 * 322 * This number is purposely set to a low number of 16384. 323 * If the dump on oops happens, it will be much appreciated 324 * to not have to wait for all that output. Anyway this can be 325 * boot time and run time configurable. 326 */ 327 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */ 328 329 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT; 330 331 /* trace_types holds a link list of available tracers. */ 332 static struct tracer *trace_types __read_mostly; 333 334 /* 335 * trace_types_lock is used to protect the trace_types list. 336 */ 337 DEFINE_MUTEX(trace_types_lock); 338 339 /* 340 * serialize the access of the ring buffer 341 * 342 * ring buffer serializes readers, but it is low level protection. 343 * The validity of the events (which returns by ring_buffer_peek() ..etc) 344 * are not protected by ring buffer. 345 * 346 * The content of events may become garbage if we allow other process consumes 347 * these events concurrently: 348 * A) the page of the consumed events may become a normal page 349 * (not reader page) in ring buffer, and this page will be rewrited 350 * by events producer. 351 * B) The page of the consumed events may become a page for splice_read, 352 * and this page will be returned to system. 353 * 354 * These primitives allow multi process access to different cpu ring buffer 355 * concurrently. 356 * 357 * These primitives don't distinguish read-only and read-consume access. 358 * Multi read-only access are also serialized. 359 */ 360 361 #ifdef CONFIG_SMP 362 static DECLARE_RWSEM(all_cpu_access_lock); 363 static DEFINE_PER_CPU(struct mutex, cpu_access_lock); 364 365 static inline void trace_access_lock(int cpu) 366 { 367 if (cpu == RING_BUFFER_ALL_CPUS) { 368 /* gain it for accessing the whole ring buffer. */ 369 down_write(&all_cpu_access_lock); 370 } else { 371 /* gain it for accessing a cpu ring buffer. */ 372 373 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */ 374 down_read(&all_cpu_access_lock); 375 376 /* Secondly block other access to this @cpu ring buffer. */ 377 mutex_lock(&per_cpu(cpu_access_lock, cpu)); 378 } 379 } 380 381 static inline void trace_access_unlock(int cpu) 382 { 383 if (cpu == RING_BUFFER_ALL_CPUS) { 384 up_write(&all_cpu_access_lock); 385 } else { 386 mutex_unlock(&per_cpu(cpu_access_lock, cpu)); 387 up_read(&all_cpu_access_lock); 388 } 389 } 390 391 static inline void trace_access_lock_init(void) 392 { 393 int cpu; 394 395 for_each_possible_cpu(cpu) 396 mutex_init(&per_cpu(cpu_access_lock, cpu)); 397 } 398 399 #else 400 401 static DEFINE_MUTEX(access_lock); 402 403 static inline void trace_access_lock(int cpu) 404 { 405 (void)cpu; 406 mutex_lock(&access_lock); 407 } 408 409 static inline void trace_access_unlock(int cpu) 410 { 411 (void)cpu; 412 mutex_unlock(&access_lock); 413 } 414 415 static inline void trace_access_lock_init(void) 416 { 417 } 418 419 #endif 420 421 /* trace_flags holds trace_options default values */ 422 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | 423 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME | 424 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | 425 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION; 426 427 static void tracer_tracing_on(struct trace_array *tr) 428 { 429 if (tr->trace_buffer.buffer) 430 ring_buffer_record_on(tr->trace_buffer.buffer); 431 /* 432 * This flag is looked at when buffers haven't been allocated 433 * yet, or by some tracers (like irqsoff), that just want to 434 * know if the ring buffer has been disabled, but it can handle 435 * races of where it gets disabled but we still do a record. 436 * As the check is in the fast path of the tracers, it is more 437 * important to be fast than accurate. 438 */ 439 tr->buffer_disabled = 0; 440 /* Make the flag seen by readers */ 441 smp_wmb(); 442 } 443 444 /** 445 * tracing_on - enable tracing buffers 446 * 447 * This function enables tracing buffers that may have been 448 * disabled with tracing_off. 449 */ 450 void tracing_on(void) 451 { 452 tracer_tracing_on(&global_trace); 453 } 454 EXPORT_SYMBOL_GPL(tracing_on); 455 456 /** 457 * __trace_puts - write a constant string into the trace buffer. 458 * @ip: The address of the caller 459 * @str: The constant string to write 460 * @size: The size of the string. 461 */ 462 int __trace_puts(unsigned long ip, const char *str, int size) 463 { 464 struct ring_buffer_event *event; 465 struct ring_buffer *buffer; 466 struct print_entry *entry; 467 unsigned long irq_flags; 468 int alloc; 469 470 if (unlikely(tracing_selftest_running || tracing_disabled)) 471 return 0; 472 473 alloc = sizeof(*entry) + size + 2; /* possible \n added */ 474 475 local_save_flags(irq_flags); 476 buffer = global_trace.trace_buffer.buffer; 477 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 478 irq_flags, preempt_count()); 479 if (!event) 480 return 0; 481 482 entry = ring_buffer_event_data(event); 483 entry->ip = ip; 484 485 memcpy(&entry->buf, str, size); 486 487 /* Add a newline if necessary */ 488 if (entry->buf[size - 1] != '\n') { 489 entry->buf[size] = '\n'; 490 entry->buf[size + 1] = '\0'; 491 } else 492 entry->buf[size] = '\0'; 493 494 __buffer_unlock_commit(buffer, event); 495 496 return size; 497 } 498 EXPORT_SYMBOL_GPL(__trace_puts); 499 500 /** 501 * __trace_bputs - write the pointer to a constant string into trace buffer 502 * @ip: The address of the caller 503 * @str: The constant string to write to the buffer to 504 */ 505 int __trace_bputs(unsigned long ip, const char *str) 506 { 507 struct ring_buffer_event *event; 508 struct ring_buffer *buffer; 509 struct bputs_entry *entry; 510 unsigned long irq_flags; 511 int size = sizeof(struct bputs_entry); 512 513 if (unlikely(tracing_selftest_running || tracing_disabled)) 514 return 0; 515 516 local_save_flags(irq_flags); 517 buffer = global_trace.trace_buffer.buffer; 518 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size, 519 irq_flags, preempt_count()); 520 if (!event) 521 return 0; 522 523 entry = ring_buffer_event_data(event); 524 entry->ip = ip; 525 entry->str = str; 526 527 __buffer_unlock_commit(buffer, event); 528 529 return 1; 530 } 531 EXPORT_SYMBOL_GPL(__trace_bputs); 532 533 #ifdef CONFIG_TRACER_SNAPSHOT 534 /** 535 * trace_snapshot - take a snapshot of the current buffer. 536 * 537 * This causes a swap between the snapshot buffer and the current live 538 * tracing buffer. You can use this to take snapshots of the live 539 * trace when some condition is triggered, but continue to trace. 540 * 541 * Note, make sure to allocate the snapshot with either 542 * a tracing_snapshot_alloc(), or by doing it manually 543 * with: echo 1 > /sys/kernel/debug/tracing/snapshot 544 * 545 * If the snapshot buffer is not allocated, it will stop tracing. 546 * Basically making a permanent snapshot. 547 */ 548 void tracing_snapshot(void) 549 { 550 struct trace_array *tr = &global_trace; 551 struct tracer *tracer = tr->current_trace; 552 unsigned long flags; 553 554 if (in_nmi()) { 555 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n"); 556 internal_trace_puts("*** snapshot is being ignored ***\n"); 557 return; 558 } 559 560 if (!tr->allocated_snapshot) { 561 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n"); 562 internal_trace_puts("*** stopping trace here! ***\n"); 563 tracing_off(); 564 return; 565 } 566 567 /* Note, snapshot can not be used when the tracer uses it */ 568 if (tracer->use_max_tr) { 569 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n"); 570 internal_trace_puts("*** Can not use snapshot (sorry) ***\n"); 571 return; 572 } 573 574 local_irq_save(flags); 575 update_max_tr(tr, current, smp_processor_id()); 576 local_irq_restore(flags); 577 } 578 EXPORT_SYMBOL_GPL(tracing_snapshot); 579 580 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf, 581 struct trace_buffer *size_buf, int cpu_id); 582 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val); 583 584 static int alloc_snapshot(struct trace_array *tr) 585 { 586 int ret; 587 588 if (!tr->allocated_snapshot) { 589 590 /* allocate spare buffer */ 591 ret = resize_buffer_duplicate_size(&tr->max_buffer, 592 &tr->trace_buffer, RING_BUFFER_ALL_CPUS); 593 if (ret < 0) 594 return ret; 595 596 tr->allocated_snapshot = true; 597 } 598 599 return 0; 600 } 601 602 static void free_snapshot(struct trace_array *tr) 603 { 604 /* 605 * We don't free the ring buffer. instead, resize it because 606 * The max_tr ring buffer has some state (e.g. ring->clock) and 607 * we want preserve it. 608 */ 609 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS); 610 set_buffer_entries(&tr->max_buffer, 1); 611 tracing_reset_online_cpus(&tr->max_buffer); 612 tr->allocated_snapshot = false; 613 } 614 615 /** 616 * tracing_alloc_snapshot - allocate snapshot buffer. 617 * 618 * This only allocates the snapshot buffer if it isn't already 619 * allocated - it doesn't also take a snapshot. 620 * 621 * This is meant to be used in cases where the snapshot buffer needs 622 * to be set up for events that can't sleep but need to be able to 623 * trigger a snapshot. 624 */ 625 int tracing_alloc_snapshot(void) 626 { 627 struct trace_array *tr = &global_trace; 628 int ret; 629 630 ret = alloc_snapshot(tr); 631 WARN_ON(ret < 0); 632 633 return ret; 634 } 635 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); 636 637 /** 638 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer. 639 * 640 * This is similar to trace_snapshot(), but it will allocate the 641 * snapshot buffer if it isn't already allocated. Use this only 642 * where it is safe to sleep, as the allocation may sleep. 643 * 644 * This causes a swap between the snapshot buffer and the current live 645 * tracing buffer. You can use this to take snapshots of the live 646 * trace when some condition is triggered, but continue to trace. 647 */ 648 void tracing_snapshot_alloc(void) 649 { 650 int ret; 651 652 ret = tracing_alloc_snapshot(); 653 if (ret < 0) 654 return; 655 656 tracing_snapshot(); 657 } 658 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); 659 #else 660 void tracing_snapshot(void) 661 { 662 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used"); 663 } 664 EXPORT_SYMBOL_GPL(tracing_snapshot); 665 int tracing_alloc_snapshot(void) 666 { 667 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used"); 668 return -ENODEV; 669 } 670 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); 671 void tracing_snapshot_alloc(void) 672 { 673 /* Give warning */ 674 tracing_snapshot(); 675 } 676 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); 677 #endif /* CONFIG_TRACER_SNAPSHOT */ 678 679 static void tracer_tracing_off(struct trace_array *tr) 680 { 681 if (tr->trace_buffer.buffer) 682 ring_buffer_record_off(tr->trace_buffer.buffer); 683 /* 684 * This flag is looked at when buffers haven't been allocated 685 * yet, or by some tracers (like irqsoff), that just want to 686 * know if the ring buffer has been disabled, but it can handle 687 * races of where it gets disabled but we still do a record. 688 * As the check is in the fast path of the tracers, it is more 689 * important to be fast than accurate. 690 */ 691 tr->buffer_disabled = 1; 692 /* Make the flag seen by readers */ 693 smp_wmb(); 694 } 695 696 /** 697 * tracing_off - turn off tracing buffers 698 * 699 * This function stops the tracing buffers from recording data. 700 * It does not disable any overhead the tracers themselves may 701 * be causing. This function simply causes all recording to 702 * the ring buffers to fail. 703 */ 704 void tracing_off(void) 705 { 706 tracer_tracing_off(&global_trace); 707 } 708 EXPORT_SYMBOL_GPL(tracing_off); 709 710 void disable_trace_on_warning(void) 711 { 712 if (__disable_trace_on_warning) 713 tracing_off(); 714 } 715 716 /** 717 * tracer_tracing_is_on - show real state of ring buffer enabled 718 * @tr : the trace array to know if ring buffer is enabled 719 * 720 * Shows real state of the ring buffer if it is enabled or not. 721 */ 722 static int tracer_tracing_is_on(struct trace_array *tr) 723 { 724 if (tr->trace_buffer.buffer) 725 return ring_buffer_record_is_on(tr->trace_buffer.buffer); 726 return !tr->buffer_disabled; 727 } 728 729 /** 730 * tracing_is_on - show state of ring buffers enabled 731 */ 732 int tracing_is_on(void) 733 { 734 return tracer_tracing_is_on(&global_trace); 735 } 736 EXPORT_SYMBOL_GPL(tracing_is_on); 737 738 static int __init set_buf_size(char *str) 739 { 740 unsigned long buf_size; 741 742 if (!str) 743 return 0; 744 buf_size = memparse(str, &str); 745 /* nr_entries can not be zero */ 746 if (buf_size == 0) 747 return 0; 748 trace_buf_size = buf_size; 749 return 1; 750 } 751 __setup("trace_buf_size=", set_buf_size); 752 753 static int __init set_tracing_thresh(char *str) 754 { 755 unsigned long threshold; 756 int ret; 757 758 if (!str) 759 return 0; 760 ret = kstrtoul(str, 0, &threshold); 761 if (ret < 0) 762 return 0; 763 tracing_thresh = threshold * 1000; 764 return 1; 765 } 766 __setup("tracing_thresh=", set_tracing_thresh); 767 768 unsigned long nsecs_to_usecs(unsigned long nsecs) 769 { 770 return nsecs / 1000; 771 } 772 773 /* These must match the bit postions in trace_iterator_flags */ 774 static const char *trace_options[] = { 775 "print-parent", 776 "sym-offset", 777 "sym-addr", 778 "verbose", 779 "raw", 780 "hex", 781 "bin", 782 "block", 783 "stacktrace", 784 "trace_printk", 785 "ftrace_preempt", 786 "branch", 787 "annotate", 788 "userstacktrace", 789 "sym-userobj", 790 "printk-msg-only", 791 "context-info", 792 "latency-format", 793 "sleep-time", 794 "graph-time", 795 "record-cmd", 796 "overwrite", 797 "disable_on_free", 798 "irq-info", 799 "markers", 800 "function-trace", 801 NULL 802 }; 803 804 static struct { 805 u64 (*func)(void); 806 const char *name; 807 int in_ns; /* is this clock in nanoseconds? */ 808 } trace_clocks[] = { 809 { trace_clock_local, "local", 1 }, 810 { trace_clock_global, "global", 1 }, 811 { trace_clock_counter, "counter", 0 }, 812 { trace_clock_jiffies, "uptime", 1 }, 813 { trace_clock, "perf", 1 }, 814 ARCH_TRACE_CLOCKS 815 }; 816 817 /* 818 * trace_parser_get_init - gets the buffer for trace parser 819 */ 820 int trace_parser_get_init(struct trace_parser *parser, int size) 821 { 822 memset(parser, 0, sizeof(*parser)); 823 824 parser->buffer = kmalloc(size, GFP_KERNEL); 825 if (!parser->buffer) 826 return 1; 827 828 parser->size = size; 829 return 0; 830 } 831 832 /* 833 * trace_parser_put - frees the buffer for trace parser 834 */ 835 void trace_parser_put(struct trace_parser *parser) 836 { 837 kfree(parser->buffer); 838 } 839 840 /* 841 * trace_get_user - reads the user input string separated by space 842 * (matched by isspace(ch)) 843 * 844 * For each string found the 'struct trace_parser' is updated, 845 * and the function returns. 846 * 847 * Returns number of bytes read. 848 * 849 * See kernel/trace/trace.h for 'struct trace_parser' details. 850 */ 851 int trace_get_user(struct trace_parser *parser, const char __user *ubuf, 852 size_t cnt, loff_t *ppos) 853 { 854 char ch; 855 size_t read = 0; 856 ssize_t ret; 857 858 if (!*ppos) 859 trace_parser_clear(parser); 860 861 ret = get_user(ch, ubuf++); 862 if (ret) 863 goto out; 864 865 read++; 866 cnt--; 867 868 /* 869 * The parser is not finished with the last write, 870 * continue reading the user input without skipping spaces. 871 */ 872 if (!parser->cont) { 873 /* skip white space */ 874 while (cnt && isspace(ch)) { 875 ret = get_user(ch, ubuf++); 876 if (ret) 877 goto out; 878 read++; 879 cnt--; 880 } 881 882 /* only spaces were written */ 883 if (isspace(ch)) { 884 *ppos += read; 885 ret = read; 886 goto out; 887 } 888 889 parser->idx = 0; 890 } 891 892 /* read the non-space input */ 893 while (cnt && !isspace(ch)) { 894 if (parser->idx < parser->size - 1) 895 parser->buffer[parser->idx++] = ch; 896 else { 897 ret = -EINVAL; 898 goto out; 899 } 900 ret = get_user(ch, ubuf++); 901 if (ret) 902 goto out; 903 read++; 904 cnt--; 905 } 906 907 /* We either got finished input or we have to wait for another call. */ 908 if (isspace(ch)) { 909 parser->buffer[parser->idx] = 0; 910 parser->cont = false; 911 } else if (parser->idx < parser->size - 1) { 912 parser->cont = true; 913 parser->buffer[parser->idx++] = ch; 914 } else { 915 ret = -EINVAL; 916 goto out; 917 } 918 919 *ppos += read; 920 ret = read; 921 922 out: 923 return ret; 924 } 925 926 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt) 927 { 928 int len; 929 int ret; 930 931 if (!cnt) 932 return 0; 933 934 if (s->len <= s->readpos) 935 return -EBUSY; 936 937 len = s->len - s->readpos; 938 if (cnt > len) 939 cnt = len; 940 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt); 941 if (ret == cnt) 942 return -EFAULT; 943 944 cnt -= ret; 945 946 s->readpos += cnt; 947 return cnt; 948 } 949 950 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) 951 { 952 int len; 953 954 if (s->len <= s->readpos) 955 return -EBUSY; 956 957 len = s->len - s->readpos; 958 if (cnt > len) 959 cnt = len; 960 memcpy(buf, s->buffer + s->readpos, cnt); 961 962 s->readpos += cnt; 963 return cnt; 964 } 965 966 unsigned long __read_mostly tracing_thresh; 967 968 #ifdef CONFIG_TRACER_MAX_TRACE 969 /* 970 * Copy the new maximum trace into the separate maximum-trace 971 * structure. (this way the maximum trace is permanently saved, 972 * for later retrieval via /sys/kernel/debug/tracing/latency_trace) 973 */ 974 static void 975 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 976 { 977 struct trace_buffer *trace_buf = &tr->trace_buffer; 978 struct trace_buffer *max_buf = &tr->max_buffer; 979 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu); 980 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu); 981 982 max_buf->cpu = cpu; 983 max_buf->time_start = data->preempt_timestamp; 984 985 max_data->saved_latency = tr->max_latency; 986 max_data->critical_start = data->critical_start; 987 max_data->critical_end = data->critical_end; 988 989 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN); 990 max_data->pid = tsk->pid; 991 /* 992 * If tsk == current, then use current_uid(), as that does not use 993 * RCU. The irq tracer can be called out of RCU scope. 994 */ 995 if (tsk == current) 996 max_data->uid = current_uid(); 997 else 998 max_data->uid = task_uid(tsk); 999 1000 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; 1001 max_data->policy = tsk->policy; 1002 max_data->rt_priority = tsk->rt_priority; 1003 1004 /* record this tasks comm */ 1005 tracing_record_cmdline(tsk); 1006 } 1007 1008 /** 1009 * update_max_tr - snapshot all trace buffers from global_trace to max_tr 1010 * @tr: tracer 1011 * @tsk: the task with the latency 1012 * @cpu: The cpu that initiated the trace. 1013 * 1014 * Flip the buffers between the @tr and the max_tr and record information 1015 * about which task was the cause of this latency. 1016 */ 1017 void 1018 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 1019 { 1020 struct ring_buffer *buf; 1021 1022 if (tr->stop_count) 1023 return; 1024 1025 WARN_ON_ONCE(!irqs_disabled()); 1026 1027 if (!tr->allocated_snapshot) { 1028 /* Only the nop tracer should hit this when disabling */ 1029 WARN_ON_ONCE(tr->current_trace != &nop_trace); 1030 return; 1031 } 1032 1033 arch_spin_lock(&tr->max_lock); 1034 1035 buf = tr->trace_buffer.buffer; 1036 tr->trace_buffer.buffer = tr->max_buffer.buffer; 1037 tr->max_buffer.buffer = buf; 1038 1039 __update_max_tr(tr, tsk, cpu); 1040 arch_spin_unlock(&tr->max_lock); 1041 } 1042 1043 /** 1044 * update_max_tr_single - only copy one trace over, and reset the rest 1045 * @tr - tracer 1046 * @tsk - task with the latency 1047 * @cpu - the cpu of the buffer to copy. 1048 * 1049 * Flip the trace of a single CPU buffer between the @tr and the max_tr. 1050 */ 1051 void 1052 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) 1053 { 1054 int ret; 1055 1056 if (tr->stop_count) 1057 return; 1058 1059 WARN_ON_ONCE(!irqs_disabled()); 1060 if (!tr->allocated_snapshot) { 1061 /* Only the nop tracer should hit this when disabling */ 1062 WARN_ON_ONCE(tr->current_trace != &nop_trace); 1063 return; 1064 } 1065 1066 arch_spin_lock(&tr->max_lock); 1067 1068 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu); 1069 1070 if (ret == -EBUSY) { 1071 /* 1072 * We failed to swap the buffer due to a commit taking 1073 * place on this CPU. We fail to record, but we reset 1074 * the max trace buffer (no one writes directly to it) 1075 * and flag that it failed. 1076 */ 1077 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_, 1078 "Failed to swap buffers due to commit in progress\n"); 1079 } 1080 1081 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); 1082 1083 __update_max_tr(tr, tsk, cpu); 1084 arch_spin_unlock(&tr->max_lock); 1085 } 1086 #endif /* CONFIG_TRACER_MAX_TRACE */ 1087 1088 static int wait_on_pipe(struct trace_iterator *iter) 1089 { 1090 /* Iterators are static, they should be filled or empty */ 1091 if (trace_buffer_iter(iter, iter->cpu_file)) 1092 return 0; 1093 1094 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file); 1095 } 1096 1097 #ifdef CONFIG_FTRACE_STARTUP_TEST 1098 static int run_tracer_selftest(struct tracer *type) 1099 { 1100 struct trace_array *tr = &global_trace; 1101 struct tracer *saved_tracer = tr->current_trace; 1102 int ret; 1103 1104 if (!type->selftest || tracing_selftest_disabled) 1105 return 0; 1106 1107 /* 1108 * Run a selftest on this tracer. 1109 * Here we reset the trace buffer, and set the current 1110 * tracer to be this tracer. The tracer can then run some 1111 * internal tracing to verify that everything is in order. 1112 * If we fail, we do not register this tracer. 1113 */ 1114 tracing_reset_online_cpus(&tr->trace_buffer); 1115 1116 tr->current_trace = type; 1117 1118 #ifdef CONFIG_TRACER_MAX_TRACE 1119 if (type->use_max_tr) { 1120 /* If we expanded the buffers, make sure the max is expanded too */ 1121 if (ring_buffer_expanded) 1122 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size, 1123 RING_BUFFER_ALL_CPUS); 1124 tr->allocated_snapshot = true; 1125 } 1126 #endif 1127 1128 /* the test is responsible for initializing and enabling */ 1129 pr_info("Testing tracer %s: ", type->name); 1130 ret = type->selftest(type, tr); 1131 /* the test is responsible for resetting too */ 1132 tr->current_trace = saved_tracer; 1133 if (ret) { 1134 printk(KERN_CONT "FAILED!\n"); 1135 /* Add the warning after printing 'FAILED' */ 1136 WARN_ON(1); 1137 return -1; 1138 } 1139 /* Only reset on passing, to avoid touching corrupted buffers */ 1140 tracing_reset_online_cpus(&tr->trace_buffer); 1141 1142 #ifdef CONFIG_TRACER_MAX_TRACE 1143 if (type->use_max_tr) { 1144 tr->allocated_snapshot = false; 1145 1146 /* Shrink the max buffer again */ 1147 if (ring_buffer_expanded) 1148 ring_buffer_resize(tr->max_buffer.buffer, 1, 1149 RING_BUFFER_ALL_CPUS); 1150 } 1151 #endif 1152 1153 printk(KERN_CONT "PASSED\n"); 1154 return 0; 1155 } 1156 #else 1157 static inline int run_tracer_selftest(struct tracer *type) 1158 { 1159 return 0; 1160 } 1161 #endif /* CONFIG_FTRACE_STARTUP_TEST */ 1162 1163 /** 1164 * register_tracer - register a tracer with the ftrace system. 1165 * @type - the plugin for the tracer 1166 * 1167 * Register a new plugin tracer. 1168 */ 1169 int register_tracer(struct tracer *type) 1170 { 1171 struct tracer *t; 1172 int ret = 0; 1173 1174 if (!type->name) { 1175 pr_info("Tracer must have a name\n"); 1176 return -1; 1177 } 1178 1179 if (strlen(type->name) >= MAX_TRACER_SIZE) { 1180 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE); 1181 return -1; 1182 } 1183 1184 mutex_lock(&trace_types_lock); 1185 1186 tracing_selftest_running = true; 1187 1188 for (t = trace_types; t; t = t->next) { 1189 if (strcmp(type->name, t->name) == 0) { 1190 /* already found */ 1191 pr_info("Tracer %s already registered\n", 1192 type->name); 1193 ret = -1; 1194 goto out; 1195 } 1196 } 1197 1198 if (!type->set_flag) 1199 type->set_flag = &dummy_set_flag; 1200 if (!type->flags) 1201 type->flags = &dummy_tracer_flags; 1202 else 1203 if (!type->flags->opts) 1204 type->flags->opts = dummy_tracer_opt; 1205 1206 ret = run_tracer_selftest(type); 1207 if (ret < 0) 1208 goto out; 1209 1210 type->next = trace_types; 1211 trace_types = type; 1212 1213 out: 1214 tracing_selftest_running = false; 1215 mutex_unlock(&trace_types_lock); 1216 1217 if (ret || !default_bootup_tracer) 1218 goto out_unlock; 1219 1220 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE)) 1221 goto out_unlock; 1222 1223 printk(KERN_INFO "Starting tracer '%s'\n", type->name); 1224 /* Do we want this tracer to start on bootup? */ 1225 tracing_set_tracer(&global_trace, type->name); 1226 default_bootup_tracer = NULL; 1227 /* disable other selftests, since this will break it. */ 1228 tracing_selftest_disabled = true; 1229 #ifdef CONFIG_FTRACE_STARTUP_TEST 1230 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n", 1231 type->name); 1232 #endif 1233 1234 out_unlock: 1235 return ret; 1236 } 1237 1238 void tracing_reset(struct trace_buffer *buf, int cpu) 1239 { 1240 struct ring_buffer *buffer = buf->buffer; 1241 1242 if (!buffer) 1243 return; 1244 1245 ring_buffer_record_disable(buffer); 1246 1247 /* Make sure all commits have finished */ 1248 synchronize_sched(); 1249 ring_buffer_reset_cpu(buffer, cpu); 1250 1251 ring_buffer_record_enable(buffer); 1252 } 1253 1254 void tracing_reset_online_cpus(struct trace_buffer *buf) 1255 { 1256 struct ring_buffer *buffer = buf->buffer; 1257 int cpu; 1258 1259 if (!buffer) 1260 return; 1261 1262 ring_buffer_record_disable(buffer); 1263 1264 /* Make sure all commits have finished */ 1265 synchronize_sched(); 1266 1267 buf->time_start = buffer_ftrace_now(buf, buf->cpu); 1268 1269 for_each_online_cpu(cpu) 1270 ring_buffer_reset_cpu(buffer, cpu); 1271 1272 ring_buffer_record_enable(buffer); 1273 } 1274 1275 /* Must have trace_types_lock held */ 1276 void tracing_reset_all_online_cpus(void) 1277 { 1278 struct trace_array *tr; 1279 1280 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 1281 tracing_reset_online_cpus(&tr->trace_buffer); 1282 #ifdef CONFIG_TRACER_MAX_TRACE 1283 tracing_reset_online_cpus(&tr->max_buffer); 1284 #endif 1285 } 1286 } 1287 1288 #define SAVED_CMDLINES_DEFAULT 128 1289 #define NO_CMDLINE_MAP UINT_MAX 1290 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED; 1291 struct saved_cmdlines_buffer { 1292 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; 1293 unsigned *map_cmdline_to_pid; 1294 unsigned cmdline_num; 1295 int cmdline_idx; 1296 char *saved_cmdlines; 1297 }; 1298 static struct saved_cmdlines_buffer *savedcmd; 1299 1300 /* temporary disable recording */ 1301 static atomic_t trace_record_cmdline_disabled __read_mostly; 1302 1303 static inline char *get_saved_cmdlines(int idx) 1304 { 1305 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN]; 1306 } 1307 1308 static inline void set_cmdline(int idx, const char *cmdline) 1309 { 1310 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN); 1311 } 1312 1313 static int allocate_cmdlines_buffer(unsigned int val, 1314 struct saved_cmdlines_buffer *s) 1315 { 1316 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid), 1317 GFP_KERNEL); 1318 if (!s->map_cmdline_to_pid) 1319 return -ENOMEM; 1320 1321 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL); 1322 if (!s->saved_cmdlines) { 1323 kfree(s->map_cmdline_to_pid); 1324 return -ENOMEM; 1325 } 1326 1327 s->cmdline_idx = 0; 1328 s->cmdline_num = val; 1329 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP, 1330 sizeof(s->map_pid_to_cmdline)); 1331 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP, 1332 val * sizeof(*s->map_cmdline_to_pid)); 1333 1334 return 0; 1335 } 1336 1337 static int trace_create_savedcmd(void) 1338 { 1339 int ret; 1340 1341 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL); 1342 if (!savedcmd) 1343 return -ENOMEM; 1344 1345 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd); 1346 if (ret < 0) { 1347 kfree(savedcmd); 1348 savedcmd = NULL; 1349 return -ENOMEM; 1350 } 1351 1352 return 0; 1353 } 1354 1355 int is_tracing_stopped(void) 1356 { 1357 return global_trace.stop_count; 1358 } 1359 1360 /** 1361 * tracing_start - quick start of the tracer 1362 * 1363 * If tracing is enabled but was stopped by tracing_stop, 1364 * this will start the tracer back up. 1365 */ 1366 void tracing_start(void) 1367 { 1368 struct ring_buffer *buffer; 1369 unsigned long flags; 1370 1371 if (tracing_disabled) 1372 return; 1373 1374 raw_spin_lock_irqsave(&global_trace.start_lock, flags); 1375 if (--global_trace.stop_count) { 1376 if (global_trace.stop_count < 0) { 1377 /* Someone screwed up their debugging */ 1378 WARN_ON_ONCE(1); 1379 global_trace.stop_count = 0; 1380 } 1381 goto out; 1382 } 1383 1384 /* Prevent the buffers from switching */ 1385 arch_spin_lock(&global_trace.max_lock); 1386 1387 buffer = global_trace.trace_buffer.buffer; 1388 if (buffer) 1389 ring_buffer_record_enable(buffer); 1390 1391 #ifdef CONFIG_TRACER_MAX_TRACE 1392 buffer = global_trace.max_buffer.buffer; 1393 if (buffer) 1394 ring_buffer_record_enable(buffer); 1395 #endif 1396 1397 arch_spin_unlock(&global_trace.max_lock); 1398 1399 ftrace_start(); 1400 out: 1401 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags); 1402 } 1403 1404 static void tracing_start_tr(struct trace_array *tr) 1405 { 1406 struct ring_buffer *buffer; 1407 unsigned long flags; 1408 1409 if (tracing_disabled) 1410 return; 1411 1412 /* If global, we need to also start the max tracer */ 1413 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 1414 return tracing_start(); 1415 1416 raw_spin_lock_irqsave(&tr->start_lock, flags); 1417 1418 if (--tr->stop_count) { 1419 if (tr->stop_count < 0) { 1420 /* Someone screwed up their debugging */ 1421 WARN_ON_ONCE(1); 1422 tr->stop_count = 0; 1423 } 1424 goto out; 1425 } 1426 1427 buffer = tr->trace_buffer.buffer; 1428 if (buffer) 1429 ring_buffer_record_enable(buffer); 1430 1431 out: 1432 raw_spin_unlock_irqrestore(&tr->start_lock, flags); 1433 } 1434 1435 /** 1436 * tracing_stop - quick stop of the tracer 1437 * 1438 * Light weight way to stop tracing. Use in conjunction with 1439 * tracing_start. 1440 */ 1441 void tracing_stop(void) 1442 { 1443 struct ring_buffer *buffer; 1444 unsigned long flags; 1445 1446 ftrace_stop(); 1447 raw_spin_lock_irqsave(&global_trace.start_lock, flags); 1448 if (global_trace.stop_count++) 1449 goto out; 1450 1451 /* Prevent the buffers from switching */ 1452 arch_spin_lock(&global_trace.max_lock); 1453 1454 buffer = global_trace.trace_buffer.buffer; 1455 if (buffer) 1456 ring_buffer_record_disable(buffer); 1457 1458 #ifdef CONFIG_TRACER_MAX_TRACE 1459 buffer = global_trace.max_buffer.buffer; 1460 if (buffer) 1461 ring_buffer_record_disable(buffer); 1462 #endif 1463 1464 arch_spin_unlock(&global_trace.max_lock); 1465 1466 out: 1467 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags); 1468 } 1469 1470 static void tracing_stop_tr(struct trace_array *tr) 1471 { 1472 struct ring_buffer *buffer; 1473 unsigned long flags; 1474 1475 /* If global, we need to also stop the max tracer */ 1476 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 1477 return tracing_stop(); 1478 1479 raw_spin_lock_irqsave(&tr->start_lock, flags); 1480 if (tr->stop_count++) 1481 goto out; 1482 1483 buffer = tr->trace_buffer.buffer; 1484 if (buffer) 1485 ring_buffer_record_disable(buffer); 1486 1487 out: 1488 raw_spin_unlock_irqrestore(&tr->start_lock, flags); 1489 } 1490 1491 void trace_stop_cmdline_recording(void); 1492 1493 static int trace_save_cmdline(struct task_struct *tsk) 1494 { 1495 unsigned pid, idx; 1496 1497 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT)) 1498 return 0; 1499 1500 /* 1501 * It's not the end of the world if we don't get 1502 * the lock, but we also don't want to spin 1503 * nor do we want to disable interrupts, 1504 * so if we miss here, then better luck next time. 1505 */ 1506 if (!arch_spin_trylock(&trace_cmdline_lock)) 1507 return 0; 1508 1509 idx = savedcmd->map_pid_to_cmdline[tsk->pid]; 1510 if (idx == NO_CMDLINE_MAP) { 1511 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num; 1512 1513 /* 1514 * Check whether the cmdline buffer at idx has a pid 1515 * mapped. We are going to overwrite that entry so we 1516 * need to clear the map_pid_to_cmdline. Otherwise we 1517 * would read the new comm for the old pid. 1518 */ 1519 pid = savedcmd->map_cmdline_to_pid[idx]; 1520 if (pid != NO_CMDLINE_MAP) 1521 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP; 1522 1523 savedcmd->map_cmdline_to_pid[idx] = tsk->pid; 1524 savedcmd->map_pid_to_cmdline[tsk->pid] = idx; 1525 1526 savedcmd->cmdline_idx = idx; 1527 } 1528 1529 set_cmdline(idx, tsk->comm); 1530 1531 arch_spin_unlock(&trace_cmdline_lock); 1532 1533 return 1; 1534 } 1535 1536 static void __trace_find_cmdline(int pid, char comm[]) 1537 { 1538 unsigned map; 1539 1540 if (!pid) { 1541 strcpy(comm, "<idle>"); 1542 return; 1543 } 1544 1545 if (WARN_ON_ONCE(pid < 0)) { 1546 strcpy(comm, "<XXX>"); 1547 return; 1548 } 1549 1550 if (pid > PID_MAX_DEFAULT) { 1551 strcpy(comm, "<...>"); 1552 return; 1553 } 1554 1555 map = savedcmd->map_pid_to_cmdline[pid]; 1556 if (map != NO_CMDLINE_MAP) 1557 strcpy(comm, get_saved_cmdlines(map)); 1558 else 1559 strcpy(comm, "<...>"); 1560 } 1561 1562 void trace_find_cmdline(int pid, char comm[]) 1563 { 1564 preempt_disable(); 1565 arch_spin_lock(&trace_cmdline_lock); 1566 1567 __trace_find_cmdline(pid, comm); 1568 1569 arch_spin_unlock(&trace_cmdline_lock); 1570 preempt_enable(); 1571 } 1572 1573 void tracing_record_cmdline(struct task_struct *tsk) 1574 { 1575 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on()) 1576 return; 1577 1578 if (!__this_cpu_read(trace_cmdline_save)) 1579 return; 1580 1581 if (trace_save_cmdline(tsk)) 1582 __this_cpu_write(trace_cmdline_save, false); 1583 } 1584 1585 void 1586 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags, 1587 int pc) 1588 { 1589 struct task_struct *tsk = current; 1590 1591 entry->preempt_count = pc & 0xff; 1592 entry->pid = (tsk) ? tsk->pid : 0; 1593 entry->flags = 1594 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT 1595 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) | 1596 #else 1597 TRACE_FLAG_IRQS_NOSUPPORT | 1598 #endif 1599 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) | 1600 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) | 1601 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) | 1602 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0); 1603 } 1604 EXPORT_SYMBOL_GPL(tracing_generic_entry_update); 1605 1606 struct ring_buffer_event * 1607 trace_buffer_lock_reserve(struct ring_buffer *buffer, 1608 int type, 1609 unsigned long len, 1610 unsigned long flags, int pc) 1611 { 1612 struct ring_buffer_event *event; 1613 1614 event = ring_buffer_lock_reserve(buffer, len); 1615 if (event != NULL) { 1616 struct trace_entry *ent = ring_buffer_event_data(event); 1617 1618 tracing_generic_entry_update(ent, flags, pc); 1619 ent->type = type; 1620 } 1621 1622 return event; 1623 } 1624 1625 void 1626 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event) 1627 { 1628 __this_cpu_write(trace_cmdline_save, true); 1629 ring_buffer_unlock_commit(buffer, event); 1630 } 1631 1632 static inline void 1633 __trace_buffer_unlock_commit(struct ring_buffer *buffer, 1634 struct ring_buffer_event *event, 1635 unsigned long flags, int pc) 1636 { 1637 __buffer_unlock_commit(buffer, event); 1638 1639 ftrace_trace_stack(buffer, flags, 6, pc); 1640 ftrace_trace_userstack(buffer, flags, pc); 1641 } 1642 1643 void trace_buffer_unlock_commit(struct ring_buffer *buffer, 1644 struct ring_buffer_event *event, 1645 unsigned long flags, int pc) 1646 { 1647 __trace_buffer_unlock_commit(buffer, event, flags, pc); 1648 } 1649 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit); 1650 1651 static struct ring_buffer *temp_buffer; 1652 1653 struct ring_buffer_event * 1654 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb, 1655 struct ftrace_event_file *ftrace_file, 1656 int type, unsigned long len, 1657 unsigned long flags, int pc) 1658 { 1659 struct ring_buffer_event *entry; 1660 1661 *current_rb = ftrace_file->tr->trace_buffer.buffer; 1662 entry = trace_buffer_lock_reserve(*current_rb, 1663 type, len, flags, pc); 1664 /* 1665 * If tracing is off, but we have triggers enabled 1666 * we still need to look at the event data. Use the temp_buffer 1667 * to store the trace event for the tigger to use. It's recusive 1668 * safe and will not be recorded anywhere. 1669 */ 1670 if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) { 1671 *current_rb = temp_buffer; 1672 entry = trace_buffer_lock_reserve(*current_rb, 1673 type, len, flags, pc); 1674 } 1675 return entry; 1676 } 1677 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve); 1678 1679 struct ring_buffer_event * 1680 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb, 1681 int type, unsigned long len, 1682 unsigned long flags, int pc) 1683 { 1684 *current_rb = global_trace.trace_buffer.buffer; 1685 return trace_buffer_lock_reserve(*current_rb, 1686 type, len, flags, pc); 1687 } 1688 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve); 1689 1690 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer, 1691 struct ring_buffer_event *event, 1692 unsigned long flags, int pc) 1693 { 1694 __trace_buffer_unlock_commit(buffer, event, flags, pc); 1695 } 1696 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit); 1697 1698 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer, 1699 struct ring_buffer_event *event, 1700 unsigned long flags, int pc, 1701 struct pt_regs *regs) 1702 { 1703 __buffer_unlock_commit(buffer, event); 1704 1705 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs); 1706 ftrace_trace_userstack(buffer, flags, pc); 1707 } 1708 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs); 1709 1710 void trace_current_buffer_discard_commit(struct ring_buffer *buffer, 1711 struct ring_buffer_event *event) 1712 { 1713 ring_buffer_discard_commit(buffer, event); 1714 } 1715 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit); 1716 1717 void 1718 trace_function(struct trace_array *tr, 1719 unsigned long ip, unsigned long parent_ip, unsigned long flags, 1720 int pc) 1721 { 1722 struct ftrace_event_call *call = &event_function; 1723 struct ring_buffer *buffer = tr->trace_buffer.buffer; 1724 struct ring_buffer_event *event; 1725 struct ftrace_entry *entry; 1726 1727 /* If we are reading the ring buffer, don't trace */ 1728 if (unlikely(__this_cpu_read(ftrace_cpu_disabled))) 1729 return; 1730 1731 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), 1732 flags, pc); 1733 if (!event) 1734 return; 1735 entry = ring_buffer_event_data(event); 1736 entry->ip = ip; 1737 entry->parent_ip = parent_ip; 1738 1739 if (!call_filter_check_discard(call, entry, buffer, event)) 1740 __buffer_unlock_commit(buffer, event); 1741 } 1742 1743 #ifdef CONFIG_STACKTRACE 1744 1745 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long)) 1746 struct ftrace_stack { 1747 unsigned long calls[FTRACE_STACK_MAX_ENTRIES]; 1748 }; 1749 1750 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack); 1751 static DEFINE_PER_CPU(int, ftrace_stack_reserve); 1752 1753 static void __ftrace_trace_stack(struct ring_buffer *buffer, 1754 unsigned long flags, 1755 int skip, int pc, struct pt_regs *regs) 1756 { 1757 struct ftrace_event_call *call = &event_kernel_stack; 1758 struct ring_buffer_event *event; 1759 struct stack_entry *entry; 1760 struct stack_trace trace; 1761 int use_stack; 1762 int size = FTRACE_STACK_ENTRIES; 1763 1764 trace.nr_entries = 0; 1765 trace.skip = skip; 1766 1767 /* 1768 * Since events can happen in NMIs there's no safe way to 1769 * use the per cpu ftrace_stacks. We reserve it and if an interrupt 1770 * or NMI comes in, it will just have to use the default 1771 * FTRACE_STACK_SIZE. 1772 */ 1773 preempt_disable_notrace(); 1774 1775 use_stack = __this_cpu_inc_return(ftrace_stack_reserve); 1776 /* 1777 * We don't need any atomic variables, just a barrier. 1778 * If an interrupt comes in, we don't care, because it would 1779 * have exited and put the counter back to what we want. 1780 * We just need a barrier to keep gcc from moving things 1781 * around. 1782 */ 1783 barrier(); 1784 if (use_stack == 1) { 1785 trace.entries = this_cpu_ptr(ftrace_stack.calls); 1786 trace.max_entries = FTRACE_STACK_MAX_ENTRIES; 1787 1788 if (regs) 1789 save_stack_trace_regs(regs, &trace); 1790 else 1791 save_stack_trace(&trace); 1792 1793 if (trace.nr_entries > size) 1794 size = trace.nr_entries; 1795 } else 1796 /* From now on, use_stack is a boolean */ 1797 use_stack = 0; 1798 1799 size *= sizeof(unsigned long); 1800 1801 event = trace_buffer_lock_reserve(buffer, TRACE_STACK, 1802 sizeof(*entry) + size, flags, pc); 1803 if (!event) 1804 goto out; 1805 entry = ring_buffer_event_data(event); 1806 1807 memset(&entry->caller, 0, size); 1808 1809 if (use_stack) 1810 memcpy(&entry->caller, trace.entries, 1811 trace.nr_entries * sizeof(unsigned long)); 1812 else { 1813 trace.max_entries = FTRACE_STACK_ENTRIES; 1814 trace.entries = entry->caller; 1815 if (regs) 1816 save_stack_trace_regs(regs, &trace); 1817 else 1818 save_stack_trace(&trace); 1819 } 1820 1821 entry->size = trace.nr_entries; 1822 1823 if (!call_filter_check_discard(call, entry, buffer, event)) 1824 __buffer_unlock_commit(buffer, event); 1825 1826 out: 1827 /* Again, don't let gcc optimize things here */ 1828 barrier(); 1829 __this_cpu_dec(ftrace_stack_reserve); 1830 preempt_enable_notrace(); 1831 1832 } 1833 1834 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags, 1835 int skip, int pc, struct pt_regs *regs) 1836 { 1837 if (!(trace_flags & TRACE_ITER_STACKTRACE)) 1838 return; 1839 1840 __ftrace_trace_stack(buffer, flags, skip, pc, regs); 1841 } 1842 1843 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags, 1844 int skip, int pc) 1845 { 1846 if (!(trace_flags & TRACE_ITER_STACKTRACE)) 1847 return; 1848 1849 __ftrace_trace_stack(buffer, flags, skip, pc, NULL); 1850 } 1851 1852 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, 1853 int pc) 1854 { 1855 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL); 1856 } 1857 1858 /** 1859 * trace_dump_stack - record a stack back trace in the trace buffer 1860 * @skip: Number of functions to skip (helper handlers) 1861 */ 1862 void trace_dump_stack(int skip) 1863 { 1864 unsigned long flags; 1865 1866 if (tracing_disabled || tracing_selftest_running) 1867 return; 1868 1869 local_save_flags(flags); 1870 1871 /* 1872 * Skip 3 more, seems to get us at the caller of 1873 * this function. 1874 */ 1875 skip += 3; 1876 __ftrace_trace_stack(global_trace.trace_buffer.buffer, 1877 flags, skip, preempt_count(), NULL); 1878 } 1879 1880 static DEFINE_PER_CPU(int, user_stack_count); 1881 1882 void 1883 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc) 1884 { 1885 struct ftrace_event_call *call = &event_user_stack; 1886 struct ring_buffer_event *event; 1887 struct userstack_entry *entry; 1888 struct stack_trace trace; 1889 1890 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE)) 1891 return; 1892 1893 /* 1894 * NMIs can not handle page faults, even with fix ups. 1895 * The save user stack can (and often does) fault. 1896 */ 1897 if (unlikely(in_nmi())) 1898 return; 1899 1900 /* 1901 * prevent recursion, since the user stack tracing may 1902 * trigger other kernel events. 1903 */ 1904 preempt_disable(); 1905 if (__this_cpu_read(user_stack_count)) 1906 goto out; 1907 1908 __this_cpu_inc(user_stack_count); 1909 1910 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, 1911 sizeof(*entry), flags, pc); 1912 if (!event) 1913 goto out_drop_count; 1914 entry = ring_buffer_event_data(event); 1915 1916 entry->tgid = current->tgid; 1917 memset(&entry->caller, 0, sizeof(entry->caller)); 1918 1919 trace.nr_entries = 0; 1920 trace.max_entries = FTRACE_STACK_ENTRIES; 1921 trace.skip = 0; 1922 trace.entries = entry->caller; 1923 1924 save_stack_trace_user(&trace); 1925 if (!call_filter_check_discard(call, entry, buffer, event)) 1926 __buffer_unlock_commit(buffer, event); 1927 1928 out_drop_count: 1929 __this_cpu_dec(user_stack_count); 1930 out: 1931 preempt_enable(); 1932 } 1933 1934 #ifdef UNUSED 1935 static void __trace_userstack(struct trace_array *tr, unsigned long flags) 1936 { 1937 ftrace_trace_userstack(tr, flags, preempt_count()); 1938 } 1939 #endif /* UNUSED */ 1940 1941 #endif /* CONFIG_STACKTRACE */ 1942 1943 /* created for use with alloc_percpu */ 1944 struct trace_buffer_struct { 1945 char buffer[TRACE_BUF_SIZE]; 1946 }; 1947 1948 static struct trace_buffer_struct *trace_percpu_buffer; 1949 static struct trace_buffer_struct *trace_percpu_sirq_buffer; 1950 static struct trace_buffer_struct *trace_percpu_irq_buffer; 1951 static struct trace_buffer_struct *trace_percpu_nmi_buffer; 1952 1953 /* 1954 * The buffer used is dependent on the context. There is a per cpu 1955 * buffer for normal context, softirq contex, hard irq context and 1956 * for NMI context. Thise allows for lockless recording. 1957 * 1958 * Note, if the buffers failed to be allocated, then this returns NULL 1959 */ 1960 static char *get_trace_buf(void) 1961 { 1962 struct trace_buffer_struct *percpu_buffer; 1963 1964 /* 1965 * If we have allocated per cpu buffers, then we do not 1966 * need to do any locking. 1967 */ 1968 if (in_nmi()) 1969 percpu_buffer = trace_percpu_nmi_buffer; 1970 else if (in_irq()) 1971 percpu_buffer = trace_percpu_irq_buffer; 1972 else if (in_softirq()) 1973 percpu_buffer = trace_percpu_sirq_buffer; 1974 else 1975 percpu_buffer = trace_percpu_buffer; 1976 1977 if (!percpu_buffer) 1978 return NULL; 1979 1980 return this_cpu_ptr(&percpu_buffer->buffer[0]); 1981 } 1982 1983 static int alloc_percpu_trace_buffer(void) 1984 { 1985 struct trace_buffer_struct *buffers; 1986 struct trace_buffer_struct *sirq_buffers; 1987 struct trace_buffer_struct *irq_buffers; 1988 struct trace_buffer_struct *nmi_buffers; 1989 1990 buffers = alloc_percpu(struct trace_buffer_struct); 1991 if (!buffers) 1992 goto err_warn; 1993 1994 sirq_buffers = alloc_percpu(struct trace_buffer_struct); 1995 if (!sirq_buffers) 1996 goto err_sirq; 1997 1998 irq_buffers = alloc_percpu(struct trace_buffer_struct); 1999 if (!irq_buffers) 2000 goto err_irq; 2001 2002 nmi_buffers = alloc_percpu(struct trace_buffer_struct); 2003 if (!nmi_buffers) 2004 goto err_nmi; 2005 2006 trace_percpu_buffer = buffers; 2007 trace_percpu_sirq_buffer = sirq_buffers; 2008 trace_percpu_irq_buffer = irq_buffers; 2009 trace_percpu_nmi_buffer = nmi_buffers; 2010 2011 return 0; 2012 2013 err_nmi: 2014 free_percpu(irq_buffers); 2015 err_irq: 2016 free_percpu(sirq_buffers); 2017 err_sirq: 2018 free_percpu(buffers); 2019 err_warn: 2020 WARN(1, "Could not allocate percpu trace_printk buffer"); 2021 return -ENOMEM; 2022 } 2023 2024 static int buffers_allocated; 2025 2026 void trace_printk_init_buffers(void) 2027 { 2028 if (buffers_allocated) 2029 return; 2030 2031 if (alloc_percpu_trace_buffer()) 2032 return; 2033 2034 /* trace_printk() is for debug use only. Don't use it in production. */ 2035 2036 pr_warning("\n**********************************************************\n"); 2037 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 2038 pr_warning("** **\n"); 2039 pr_warning("** trace_printk() being used. Allocating extra memory. **\n"); 2040 pr_warning("** **\n"); 2041 pr_warning("** This means that this is a DEBUG kernel and it is **\n"); 2042 pr_warning("** unsafe for produciton use. **\n"); 2043 pr_warning("** **\n"); 2044 pr_warning("** If you see this message and you are not debugging **\n"); 2045 pr_warning("** the kernel, report this immediately to your vendor! **\n"); 2046 pr_warning("** **\n"); 2047 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 2048 pr_warning("**********************************************************\n"); 2049 2050 /* Expand the buffers to set size */ 2051 tracing_update_buffers(); 2052 2053 buffers_allocated = 1; 2054 2055 /* 2056 * trace_printk_init_buffers() can be called by modules. 2057 * If that happens, then we need to start cmdline recording 2058 * directly here. If the global_trace.buffer is already 2059 * allocated here, then this was called by module code. 2060 */ 2061 if (global_trace.trace_buffer.buffer) 2062 tracing_start_cmdline_record(); 2063 } 2064 2065 void trace_printk_start_comm(void) 2066 { 2067 /* Start tracing comms if trace printk is set */ 2068 if (!buffers_allocated) 2069 return; 2070 tracing_start_cmdline_record(); 2071 } 2072 2073 static void trace_printk_start_stop_comm(int enabled) 2074 { 2075 if (!buffers_allocated) 2076 return; 2077 2078 if (enabled) 2079 tracing_start_cmdline_record(); 2080 else 2081 tracing_stop_cmdline_record(); 2082 } 2083 2084 /** 2085 * trace_vbprintk - write binary msg to tracing buffer 2086 * 2087 */ 2088 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) 2089 { 2090 struct ftrace_event_call *call = &event_bprint; 2091 struct ring_buffer_event *event; 2092 struct ring_buffer *buffer; 2093 struct trace_array *tr = &global_trace; 2094 struct bprint_entry *entry; 2095 unsigned long flags; 2096 char *tbuffer; 2097 int len = 0, size, pc; 2098 2099 if (unlikely(tracing_selftest_running || tracing_disabled)) 2100 return 0; 2101 2102 /* Don't pollute graph traces with trace_vprintk internals */ 2103 pause_graph_tracing(); 2104 2105 pc = preempt_count(); 2106 preempt_disable_notrace(); 2107 2108 tbuffer = get_trace_buf(); 2109 if (!tbuffer) { 2110 len = 0; 2111 goto out; 2112 } 2113 2114 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args); 2115 2116 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0) 2117 goto out; 2118 2119 local_save_flags(flags); 2120 size = sizeof(*entry) + sizeof(u32) * len; 2121 buffer = tr->trace_buffer.buffer; 2122 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, 2123 flags, pc); 2124 if (!event) 2125 goto out; 2126 entry = ring_buffer_event_data(event); 2127 entry->ip = ip; 2128 entry->fmt = fmt; 2129 2130 memcpy(entry->buf, tbuffer, sizeof(u32) * len); 2131 if (!call_filter_check_discard(call, entry, buffer, event)) { 2132 __buffer_unlock_commit(buffer, event); 2133 ftrace_trace_stack(buffer, flags, 6, pc); 2134 } 2135 2136 out: 2137 preempt_enable_notrace(); 2138 unpause_graph_tracing(); 2139 2140 return len; 2141 } 2142 EXPORT_SYMBOL_GPL(trace_vbprintk); 2143 2144 static int 2145 __trace_array_vprintk(struct ring_buffer *buffer, 2146 unsigned long ip, const char *fmt, va_list args) 2147 { 2148 struct ftrace_event_call *call = &event_print; 2149 struct ring_buffer_event *event; 2150 int len = 0, size, pc; 2151 struct print_entry *entry; 2152 unsigned long flags; 2153 char *tbuffer; 2154 2155 if (tracing_disabled || tracing_selftest_running) 2156 return 0; 2157 2158 /* Don't pollute graph traces with trace_vprintk internals */ 2159 pause_graph_tracing(); 2160 2161 pc = preempt_count(); 2162 preempt_disable_notrace(); 2163 2164 2165 tbuffer = get_trace_buf(); 2166 if (!tbuffer) { 2167 len = 0; 2168 goto out; 2169 } 2170 2171 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args); 2172 if (len > TRACE_BUF_SIZE) 2173 goto out; 2174 2175 local_save_flags(flags); 2176 size = sizeof(*entry) + len + 1; 2177 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 2178 flags, pc); 2179 if (!event) 2180 goto out; 2181 entry = ring_buffer_event_data(event); 2182 entry->ip = ip; 2183 2184 memcpy(&entry->buf, tbuffer, len); 2185 entry->buf[len] = '\0'; 2186 if (!call_filter_check_discard(call, entry, buffer, event)) { 2187 __buffer_unlock_commit(buffer, event); 2188 ftrace_trace_stack(buffer, flags, 6, pc); 2189 } 2190 out: 2191 preempt_enable_notrace(); 2192 unpause_graph_tracing(); 2193 2194 return len; 2195 } 2196 2197 int trace_array_vprintk(struct trace_array *tr, 2198 unsigned long ip, const char *fmt, va_list args) 2199 { 2200 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args); 2201 } 2202 2203 int trace_array_printk(struct trace_array *tr, 2204 unsigned long ip, const char *fmt, ...) 2205 { 2206 int ret; 2207 va_list ap; 2208 2209 if (!(trace_flags & TRACE_ITER_PRINTK)) 2210 return 0; 2211 2212 va_start(ap, fmt); 2213 ret = trace_array_vprintk(tr, ip, fmt, ap); 2214 va_end(ap); 2215 return ret; 2216 } 2217 2218 int trace_array_printk_buf(struct ring_buffer *buffer, 2219 unsigned long ip, const char *fmt, ...) 2220 { 2221 int ret; 2222 va_list ap; 2223 2224 if (!(trace_flags & TRACE_ITER_PRINTK)) 2225 return 0; 2226 2227 va_start(ap, fmt); 2228 ret = __trace_array_vprintk(buffer, ip, fmt, ap); 2229 va_end(ap); 2230 return ret; 2231 } 2232 2233 int trace_vprintk(unsigned long ip, const char *fmt, va_list args) 2234 { 2235 return trace_array_vprintk(&global_trace, ip, fmt, args); 2236 } 2237 EXPORT_SYMBOL_GPL(trace_vprintk); 2238 2239 static void trace_iterator_increment(struct trace_iterator *iter) 2240 { 2241 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu); 2242 2243 iter->idx++; 2244 if (buf_iter) 2245 ring_buffer_read(buf_iter, NULL); 2246 } 2247 2248 static struct trace_entry * 2249 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts, 2250 unsigned long *lost_events) 2251 { 2252 struct ring_buffer_event *event; 2253 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu); 2254 2255 if (buf_iter) 2256 event = ring_buffer_iter_peek(buf_iter, ts); 2257 else 2258 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts, 2259 lost_events); 2260 2261 if (event) { 2262 iter->ent_size = ring_buffer_event_length(event); 2263 return ring_buffer_event_data(event); 2264 } 2265 iter->ent_size = 0; 2266 return NULL; 2267 } 2268 2269 static struct trace_entry * 2270 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, 2271 unsigned long *missing_events, u64 *ent_ts) 2272 { 2273 struct ring_buffer *buffer = iter->trace_buffer->buffer; 2274 struct trace_entry *ent, *next = NULL; 2275 unsigned long lost_events = 0, next_lost = 0; 2276 int cpu_file = iter->cpu_file; 2277 u64 next_ts = 0, ts; 2278 int next_cpu = -1; 2279 int next_size = 0; 2280 int cpu; 2281 2282 /* 2283 * If we are in a per_cpu trace file, don't bother by iterating over 2284 * all cpu and peek directly. 2285 */ 2286 if (cpu_file > RING_BUFFER_ALL_CPUS) { 2287 if (ring_buffer_empty_cpu(buffer, cpu_file)) 2288 return NULL; 2289 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events); 2290 if (ent_cpu) 2291 *ent_cpu = cpu_file; 2292 2293 return ent; 2294 } 2295 2296 for_each_tracing_cpu(cpu) { 2297 2298 if (ring_buffer_empty_cpu(buffer, cpu)) 2299 continue; 2300 2301 ent = peek_next_entry(iter, cpu, &ts, &lost_events); 2302 2303 /* 2304 * Pick the entry with the smallest timestamp: 2305 */ 2306 if (ent && (!next || ts < next_ts)) { 2307 next = ent; 2308 next_cpu = cpu; 2309 next_ts = ts; 2310 next_lost = lost_events; 2311 next_size = iter->ent_size; 2312 } 2313 } 2314 2315 iter->ent_size = next_size; 2316 2317 if (ent_cpu) 2318 *ent_cpu = next_cpu; 2319 2320 if (ent_ts) 2321 *ent_ts = next_ts; 2322 2323 if (missing_events) 2324 *missing_events = next_lost; 2325 2326 return next; 2327 } 2328 2329 /* Find the next real entry, without updating the iterator itself */ 2330 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter, 2331 int *ent_cpu, u64 *ent_ts) 2332 { 2333 return __find_next_entry(iter, ent_cpu, NULL, ent_ts); 2334 } 2335 2336 /* Find the next real entry, and increment the iterator to the next entry */ 2337 void *trace_find_next_entry_inc(struct trace_iterator *iter) 2338 { 2339 iter->ent = __find_next_entry(iter, &iter->cpu, 2340 &iter->lost_events, &iter->ts); 2341 2342 if (iter->ent) 2343 trace_iterator_increment(iter); 2344 2345 return iter->ent ? iter : NULL; 2346 } 2347 2348 static void trace_consume(struct trace_iterator *iter) 2349 { 2350 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts, 2351 &iter->lost_events); 2352 } 2353 2354 static void *s_next(struct seq_file *m, void *v, loff_t *pos) 2355 { 2356 struct trace_iterator *iter = m->private; 2357 int i = (int)*pos; 2358 void *ent; 2359 2360 WARN_ON_ONCE(iter->leftover); 2361 2362 (*pos)++; 2363 2364 /* can't go backwards */ 2365 if (iter->idx > i) 2366 return NULL; 2367 2368 if (iter->idx < 0) 2369 ent = trace_find_next_entry_inc(iter); 2370 else 2371 ent = iter; 2372 2373 while (ent && iter->idx < i) 2374 ent = trace_find_next_entry_inc(iter); 2375 2376 iter->pos = *pos; 2377 2378 return ent; 2379 } 2380 2381 void tracing_iter_reset(struct trace_iterator *iter, int cpu) 2382 { 2383 struct ring_buffer_event *event; 2384 struct ring_buffer_iter *buf_iter; 2385 unsigned long entries = 0; 2386 u64 ts; 2387 2388 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0; 2389 2390 buf_iter = trace_buffer_iter(iter, cpu); 2391 if (!buf_iter) 2392 return; 2393 2394 ring_buffer_iter_reset(buf_iter); 2395 2396 /* 2397 * We could have the case with the max latency tracers 2398 * that a reset never took place on a cpu. This is evident 2399 * by the timestamp being before the start of the buffer. 2400 */ 2401 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) { 2402 if (ts >= iter->trace_buffer->time_start) 2403 break; 2404 entries++; 2405 ring_buffer_read(buf_iter, NULL); 2406 } 2407 2408 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries; 2409 } 2410 2411 /* 2412 * The current tracer is copied to avoid a global locking 2413 * all around. 2414 */ 2415 static void *s_start(struct seq_file *m, loff_t *pos) 2416 { 2417 struct trace_iterator *iter = m->private; 2418 struct trace_array *tr = iter->tr; 2419 int cpu_file = iter->cpu_file; 2420 void *p = NULL; 2421 loff_t l = 0; 2422 int cpu; 2423 2424 /* 2425 * copy the tracer to avoid using a global lock all around. 2426 * iter->trace is a copy of current_trace, the pointer to the 2427 * name may be used instead of a strcmp(), as iter->trace->name 2428 * will point to the same string as current_trace->name. 2429 */ 2430 mutex_lock(&trace_types_lock); 2431 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name)) 2432 *iter->trace = *tr->current_trace; 2433 mutex_unlock(&trace_types_lock); 2434 2435 #ifdef CONFIG_TRACER_MAX_TRACE 2436 if (iter->snapshot && iter->trace->use_max_tr) 2437 return ERR_PTR(-EBUSY); 2438 #endif 2439 2440 if (!iter->snapshot) 2441 atomic_inc(&trace_record_cmdline_disabled); 2442 2443 if (*pos != iter->pos) { 2444 iter->ent = NULL; 2445 iter->cpu = 0; 2446 iter->idx = -1; 2447 2448 if (cpu_file == RING_BUFFER_ALL_CPUS) { 2449 for_each_tracing_cpu(cpu) 2450 tracing_iter_reset(iter, cpu); 2451 } else 2452 tracing_iter_reset(iter, cpu_file); 2453 2454 iter->leftover = 0; 2455 for (p = iter; p && l < *pos; p = s_next(m, p, &l)) 2456 ; 2457 2458 } else { 2459 /* 2460 * If we overflowed the seq_file before, then we want 2461 * to just reuse the trace_seq buffer again. 2462 */ 2463 if (iter->leftover) 2464 p = iter; 2465 else { 2466 l = *pos - 1; 2467 p = s_next(m, p, &l); 2468 } 2469 } 2470 2471 trace_event_read_lock(); 2472 trace_access_lock(cpu_file); 2473 return p; 2474 } 2475 2476 static void s_stop(struct seq_file *m, void *p) 2477 { 2478 struct trace_iterator *iter = m->private; 2479 2480 #ifdef CONFIG_TRACER_MAX_TRACE 2481 if (iter->snapshot && iter->trace->use_max_tr) 2482 return; 2483 #endif 2484 2485 if (!iter->snapshot) 2486 atomic_dec(&trace_record_cmdline_disabled); 2487 2488 trace_access_unlock(iter->cpu_file); 2489 trace_event_read_unlock(); 2490 } 2491 2492 static void 2493 get_total_entries(struct trace_buffer *buf, 2494 unsigned long *total, unsigned long *entries) 2495 { 2496 unsigned long count; 2497 int cpu; 2498 2499 *total = 0; 2500 *entries = 0; 2501 2502 for_each_tracing_cpu(cpu) { 2503 count = ring_buffer_entries_cpu(buf->buffer, cpu); 2504 /* 2505 * If this buffer has skipped entries, then we hold all 2506 * entries for the trace and we need to ignore the 2507 * ones before the time stamp. 2508 */ 2509 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) { 2510 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries; 2511 /* total is the same as the entries */ 2512 *total += count; 2513 } else 2514 *total += count + 2515 ring_buffer_overrun_cpu(buf->buffer, cpu); 2516 *entries += count; 2517 } 2518 } 2519 2520 static void print_lat_help_header(struct seq_file *m) 2521 { 2522 seq_puts(m, "# _------=> CPU# \n"); 2523 seq_puts(m, "# / _-----=> irqs-off \n"); 2524 seq_puts(m, "# | / _----=> need-resched \n"); 2525 seq_puts(m, "# || / _---=> hardirq/softirq \n"); 2526 seq_puts(m, "# ||| / _--=> preempt-depth \n"); 2527 seq_puts(m, "# |||| / delay \n"); 2528 seq_puts(m, "# cmd pid ||||| time | caller \n"); 2529 seq_puts(m, "# \\ / ||||| \\ | / \n"); 2530 } 2531 2532 static void print_event_info(struct trace_buffer *buf, struct seq_file *m) 2533 { 2534 unsigned long total; 2535 unsigned long entries; 2536 2537 get_total_entries(buf, &total, &entries); 2538 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n", 2539 entries, total, num_online_cpus()); 2540 seq_puts(m, "#\n"); 2541 } 2542 2543 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m) 2544 { 2545 print_event_info(buf, m); 2546 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"); 2547 seq_puts(m, "# | | | | |\n"); 2548 } 2549 2550 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m) 2551 { 2552 print_event_info(buf, m); 2553 seq_puts(m, "# _-----=> irqs-off\n"); 2554 seq_puts(m, "# / _----=> need-resched\n"); 2555 seq_puts(m, "# | / _---=> hardirq/softirq\n"); 2556 seq_puts(m, "# || / _--=> preempt-depth\n"); 2557 seq_puts(m, "# ||| / delay\n"); 2558 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"); 2559 seq_puts(m, "# | | | |||| | |\n"); 2560 } 2561 2562 void 2563 print_trace_header(struct seq_file *m, struct trace_iterator *iter) 2564 { 2565 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK); 2566 struct trace_buffer *buf = iter->trace_buffer; 2567 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu); 2568 struct tracer *type = iter->trace; 2569 unsigned long entries; 2570 unsigned long total; 2571 const char *name = "preemption"; 2572 2573 name = type->name; 2574 2575 get_total_entries(buf, &total, &entries); 2576 2577 seq_printf(m, "# %s latency trace v1.1.5 on %s\n", 2578 name, UTS_RELEASE); 2579 seq_puts(m, "# -----------------------------------" 2580 "---------------------------------\n"); 2581 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |" 2582 " (M:%s VP:%d, KP:%d, SP:%d HP:%d", 2583 nsecs_to_usecs(data->saved_latency), 2584 entries, 2585 total, 2586 buf->cpu, 2587 #if defined(CONFIG_PREEMPT_NONE) 2588 "server", 2589 #elif defined(CONFIG_PREEMPT_VOLUNTARY) 2590 "desktop", 2591 #elif defined(CONFIG_PREEMPT) 2592 "preempt", 2593 #else 2594 "unknown", 2595 #endif 2596 /* These are reserved for later use */ 2597 0, 0, 0, 0); 2598 #ifdef CONFIG_SMP 2599 seq_printf(m, " #P:%d)\n", num_online_cpus()); 2600 #else 2601 seq_puts(m, ")\n"); 2602 #endif 2603 seq_puts(m, "# -----------------\n"); 2604 seq_printf(m, "# | task: %.16s-%d " 2605 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n", 2606 data->comm, data->pid, 2607 from_kuid_munged(seq_user_ns(m), data->uid), data->nice, 2608 data->policy, data->rt_priority); 2609 seq_puts(m, "# -----------------\n"); 2610 2611 if (data->critical_start) { 2612 seq_puts(m, "# => started at: "); 2613 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags); 2614 trace_print_seq(m, &iter->seq); 2615 seq_puts(m, "\n# => ended at: "); 2616 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); 2617 trace_print_seq(m, &iter->seq); 2618 seq_puts(m, "\n#\n"); 2619 } 2620 2621 seq_puts(m, "#\n"); 2622 } 2623 2624 static void test_cpu_buff_start(struct trace_iterator *iter) 2625 { 2626 struct trace_seq *s = &iter->seq; 2627 2628 if (!(trace_flags & TRACE_ITER_ANNOTATE)) 2629 return; 2630 2631 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE)) 2632 return; 2633 2634 if (cpumask_test_cpu(iter->cpu, iter->started)) 2635 return; 2636 2637 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries) 2638 return; 2639 2640 cpumask_set_cpu(iter->cpu, iter->started); 2641 2642 /* Don't print started cpu buffer for the first entry of the trace */ 2643 if (iter->idx > 1) 2644 trace_seq_printf(s, "##### CPU %u buffer started ####\n", 2645 iter->cpu); 2646 } 2647 2648 static enum print_line_t print_trace_fmt(struct trace_iterator *iter) 2649 { 2650 struct trace_seq *s = &iter->seq; 2651 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK); 2652 struct trace_entry *entry; 2653 struct trace_event *event; 2654 2655 entry = iter->ent; 2656 2657 test_cpu_buff_start(iter); 2658 2659 event = ftrace_find_event(entry->type); 2660 2661 if (trace_flags & TRACE_ITER_CONTEXT_INFO) { 2662 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { 2663 if (!trace_print_lat_context(iter)) 2664 goto partial; 2665 } else { 2666 if (!trace_print_context(iter)) 2667 goto partial; 2668 } 2669 } 2670 2671 if (event) 2672 return event->funcs->trace(iter, sym_flags, event); 2673 2674 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type)) 2675 goto partial; 2676 2677 return TRACE_TYPE_HANDLED; 2678 partial: 2679 return TRACE_TYPE_PARTIAL_LINE; 2680 } 2681 2682 static enum print_line_t print_raw_fmt(struct trace_iterator *iter) 2683 { 2684 struct trace_seq *s = &iter->seq; 2685 struct trace_entry *entry; 2686 struct trace_event *event; 2687 2688 entry = iter->ent; 2689 2690 if (trace_flags & TRACE_ITER_CONTEXT_INFO) { 2691 if (!trace_seq_printf(s, "%d %d %llu ", 2692 entry->pid, iter->cpu, iter->ts)) 2693 goto partial; 2694 } 2695 2696 event = ftrace_find_event(entry->type); 2697 if (event) 2698 return event->funcs->raw(iter, 0, event); 2699 2700 if (!trace_seq_printf(s, "%d ?\n", entry->type)) 2701 goto partial; 2702 2703 return TRACE_TYPE_HANDLED; 2704 partial: 2705 return TRACE_TYPE_PARTIAL_LINE; 2706 } 2707 2708 static enum print_line_t print_hex_fmt(struct trace_iterator *iter) 2709 { 2710 struct trace_seq *s = &iter->seq; 2711 unsigned char newline = '\n'; 2712 struct trace_entry *entry; 2713 struct trace_event *event; 2714 2715 entry = iter->ent; 2716 2717 if (trace_flags & TRACE_ITER_CONTEXT_INFO) { 2718 SEQ_PUT_HEX_FIELD_RET(s, entry->pid); 2719 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu); 2720 SEQ_PUT_HEX_FIELD_RET(s, iter->ts); 2721 } 2722 2723 event = ftrace_find_event(entry->type); 2724 if (event) { 2725 enum print_line_t ret = event->funcs->hex(iter, 0, event); 2726 if (ret != TRACE_TYPE_HANDLED) 2727 return ret; 2728 } 2729 2730 SEQ_PUT_FIELD_RET(s, newline); 2731 2732 return TRACE_TYPE_HANDLED; 2733 } 2734 2735 static enum print_line_t print_bin_fmt(struct trace_iterator *iter) 2736 { 2737 struct trace_seq *s = &iter->seq; 2738 struct trace_entry *entry; 2739 struct trace_event *event; 2740 2741 entry = iter->ent; 2742 2743 if (trace_flags & TRACE_ITER_CONTEXT_INFO) { 2744 SEQ_PUT_FIELD_RET(s, entry->pid); 2745 SEQ_PUT_FIELD_RET(s, iter->cpu); 2746 SEQ_PUT_FIELD_RET(s, iter->ts); 2747 } 2748 2749 event = ftrace_find_event(entry->type); 2750 return event ? event->funcs->binary(iter, 0, event) : 2751 TRACE_TYPE_HANDLED; 2752 } 2753 2754 int trace_empty(struct trace_iterator *iter) 2755 { 2756 struct ring_buffer_iter *buf_iter; 2757 int cpu; 2758 2759 /* If we are looking at one CPU buffer, only check that one */ 2760 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 2761 cpu = iter->cpu_file; 2762 buf_iter = trace_buffer_iter(iter, cpu); 2763 if (buf_iter) { 2764 if (!ring_buffer_iter_empty(buf_iter)) 2765 return 0; 2766 } else { 2767 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu)) 2768 return 0; 2769 } 2770 return 1; 2771 } 2772 2773 for_each_tracing_cpu(cpu) { 2774 buf_iter = trace_buffer_iter(iter, cpu); 2775 if (buf_iter) { 2776 if (!ring_buffer_iter_empty(buf_iter)) 2777 return 0; 2778 } else { 2779 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu)) 2780 return 0; 2781 } 2782 } 2783 2784 return 1; 2785 } 2786 2787 /* Called with trace_event_read_lock() held. */ 2788 enum print_line_t print_trace_line(struct trace_iterator *iter) 2789 { 2790 enum print_line_t ret; 2791 2792 if (iter->lost_events && 2793 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", 2794 iter->cpu, iter->lost_events)) 2795 return TRACE_TYPE_PARTIAL_LINE; 2796 2797 if (iter->trace && iter->trace->print_line) { 2798 ret = iter->trace->print_line(iter); 2799 if (ret != TRACE_TYPE_UNHANDLED) 2800 return ret; 2801 } 2802 2803 if (iter->ent->type == TRACE_BPUTS && 2804 trace_flags & TRACE_ITER_PRINTK && 2805 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 2806 return trace_print_bputs_msg_only(iter); 2807 2808 if (iter->ent->type == TRACE_BPRINT && 2809 trace_flags & TRACE_ITER_PRINTK && 2810 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 2811 return trace_print_bprintk_msg_only(iter); 2812 2813 if (iter->ent->type == TRACE_PRINT && 2814 trace_flags & TRACE_ITER_PRINTK && 2815 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 2816 return trace_print_printk_msg_only(iter); 2817 2818 if (trace_flags & TRACE_ITER_BIN) 2819 return print_bin_fmt(iter); 2820 2821 if (trace_flags & TRACE_ITER_HEX) 2822 return print_hex_fmt(iter); 2823 2824 if (trace_flags & TRACE_ITER_RAW) 2825 return print_raw_fmt(iter); 2826 2827 return print_trace_fmt(iter); 2828 } 2829 2830 void trace_latency_header(struct seq_file *m) 2831 { 2832 struct trace_iterator *iter = m->private; 2833 2834 /* print nothing if the buffers are empty */ 2835 if (trace_empty(iter)) 2836 return; 2837 2838 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 2839 print_trace_header(m, iter); 2840 2841 if (!(trace_flags & TRACE_ITER_VERBOSE)) 2842 print_lat_help_header(m); 2843 } 2844 2845 void trace_default_header(struct seq_file *m) 2846 { 2847 struct trace_iterator *iter = m->private; 2848 2849 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO)) 2850 return; 2851 2852 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { 2853 /* print nothing if the buffers are empty */ 2854 if (trace_empty(iter)) 2855 return; 2856 print_trace_header(m, iter); 2857 if (!(trace_flags & TRACE_ITER_VERBOSE)) 2858 print_lat_help_header(m); 2859 } else { 2860 if (!(trace_flags & TRACE_ITER_VERBOSE)) { 2861 if (trace_flags & TRACE_ITER_IRQ_INFO) 2862 print_func_help_header_irq(iter->trace_buffer, m); 2863 else 2864 print_func_help_header(iter->trace_buffer, m); 2865 } 2866 } 2867 } 2868 2869 static void test_ftrace_alive(struct seq_file *m) 2870 { 2871 if (!ftrace_is_dead()) 2872 return; 2873 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"); 2874 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n"); 2875 } 2876 2877 #ifdef CONFIG_TRACER_MAX_TRACE 2878 static void show_snapshot_main_help(struct seq_file *m) 2879 { 2880 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"); 2881 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"); 2882 seq_printf(m, "# Takes a snapshot of the main buffer.\n"); 2883 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"); 2884 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n"); 2885 seq_printf(m, "# is not a '0' or '1')\n"); 2886 } 2887 2888 static void show_snapshot_percpu_help(struct seq_file *m) 2889 { 2890 seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n"); 2891 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP 2892 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"); 2893 seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n"); 2894 #else 2895 seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n"); 2896 seq_printf(m, "# Must use main snapshot file to allocate.\n"); 2897 #endif 2898 seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"); 2899 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n"); 2900 seq_printf(m, "# is not a '0' or '1')\n"); 2901 } 2902 2903 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) 2904 { 2905 if (iter->tr->allocated_snapshot) 2906 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n"); 2907 else 2908 seq_printf(m, "#\n# * Snapshot is freed *\n#\n"); 2909 2910 seq_printf(m, "# Snapshot commands:\n"); 2911 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 2912 show_snapshot_main_help(m); 2913 else 2914 show_snapshot_percpu_help(m); 2915 } 2916 #else 2917 /* Should never be called */ 2918 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { } 2919 #endif 2920 2921 static int s_show(struct seq_file *m, void *v) 2922 { 2923 struct trace_iterator *iter = v; 2924 int ret; 2925 2926 if (iter->ent == NULL) { 2927 if (iter->tr) { 2928 seq_printf(m, "# tracer: %s\n", iter->trace->name); 2929 seq_puts(m, "#\n"); 2930 test_ftrace_alive(m); 2931 } 2932 if (iter->snapshot && trace_empty(iter)) 2933 print_snapshot_help(m, iter); 2934 else if (iter->trace && iter->trace->print_header) 2935 iter->trace->print_header(m); 2936 else 2937 trace_default_header(m); 2938 2939 } else if (iter->leftover) { 2940 /* 2941 * If we filled the seq_file buffer earlier, we 2942 * want to just show it now. 2943 */ 2944 ret = trace_print_seq(m, &iter->seq); 2945 2946 /* ret should this time be zero, but you never know */ 2947 iter->leftover = ret; 2948 2949 } else { 2950 print_trace_line(iter); 2951 ret = trace_print_seq(m, &iter->seq); 2952 /* 2953 * If we overflow the seq_file buffer, then it will 2954 * ask us for this data again at start up. 2955 * Use that instead. 2956 * ret is 0 if seq_file write succeeded. 2957 * -1 otherwise. 2958 */ 2959 iter->leftover = ret; 2960 } 2961 2962 return 0; 2963 } 2964 2965 /* 2966 * Should be used after trace_array_get(), trace_types_lock 2967 * ensures that i_cdev was already initialized. 2968 */ 2969 static inline int tracing_get_cpu(struct inode *inode) 2970 { 2971 if (inode->i_cdev) /* See trace_create_cpu_file() */ 2972 return (long)inode->i_cdev - 1; 2973 return RING_BUFFER_ALL_CPUS; 2974 } 2975 2976 static const struct seq_operations tracer_seq_ops = { 2977 .start = s_start, 2978 .next = s_next, 2979 .stop = s_stop, 2980 .show = s_show, 2981 }; 2982 2983 static struct trace_iterator * 2984 __tracing_open(struct inode *inode, struct file *file, bool snapshot) 2985 { 2986 struct trace_array *tr = inode->i_private; 2987 struct trace_iterator *iter; 2988 int cpu; 2989 2990 if (tracing_disabled) 2991 return ERR_PTR(-ENODEV); 2992 2993 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter)); 2994 if (!iter) 2995 return ERR_PTR(-ENOMEM); 2996 2997 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(), 2998 GFP_KERNEL); 2999 if (!iter->buffer_iter) 3000 goto release; 3001 3002 /* 3003 * We make a copy of the current tracer to avoid concurrent 3004 * changes on it while we are reading. 3005 */ 3006 mutex_lock(&trace_types_lock); 3007 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL); 3008 if (!iter->trace) 3009 goto fail; 3010 3011 *iter->trace = *tr->current_trace; 3012 3013 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL)) 3014 goto fail; 3015 3016 iter->tr = tr; 3017 3018 #ifdef CONFIG_TRACER_MAX_TRACE 3019 /* Currently only the top directory has a snapshot */ 3020 if (tr->current_trace->print_max || snapshot) 3021 iter->trace_buffer = &tr->max_buffer; 3022 else 3023 #endif 3024 iter->trace_buffer = &tr->trace_buffer; 3025 iter->snapshot = snapshot; 3026 iter->pos = -1; 3027 iter->cpu_file = tracing_get_cpu(inode); 3028 mutex_init(&iter->mutex); 3029 3030 /* Notify the tracer early; before we stop tracing. */ 3031 if (iter->trace && iter->trace->open) 3032 iter->trace->open(iter); 3033 3034 /* Annotate start of buffers if we had overruns */ 3035 if (ring_buffer_overruns(iter->trace_buffer->buffer)) 3036 iter->iter_flags |= TRACE_FILE_ANNOTATE; 3037 3038 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 3039 if (trace_clocks[tr->clock_id].in_ns) 3040 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 3041 3042 /* stop the trace while dumping if we are not opening "snapshot" */ 3043 if (!iter->snapshot) 3044 tracing_stop_tr(tr); 3045 3046 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { 3047 for_each_tracing_cpu(cpu) { 3048 iter->buffer_iter[cpu] = 3049 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu); 3050 } 3051 ring_buffer_read_prepare_sync(); 3052 for_each_tracing_cpu(cpu) { 3053 ring_buffer_read_start(iter->buffer_iter[cpu]); 3054 tracing_iter_reset(iter, cpu); 3055 } 3056 } else { 3057 cpu = iter->cpu_file; 3058 iter->buffer_iter[cpu] = 3059 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu); 3060 ring_buffer_read_prepare_sync(); 3061 ring_buffer_read_start(iter->buffer_iter[cpu]); 3062 tracing_iter_reset(iter, cpu); 3063 } 3064 3065 mutex_unlock(&trace_types_lock); 3066 3067 return iter; 3068 3069 fail: 3070 mutex_unlock(&trace_types_lock); 3071 kfree(iter->trace); 3072 kfree(iter->buffer_iter); 3073 release: 3074 seq_release_private(inode, file); 3075 return ERR_PTR(-ENOMEM); 3076 } 3077 3078 int tracing_open_generic(struct inode *inode, struct file *filp) 3079 { 3080 if (tracing_disabled) 3081 return -ENODEV; 3082 3083 filp->private_data = inode->i_private; 3084 return 0; 3085 } 3086 3087 bool tracing_is_disabled(void) 3088 { 3089 return (tracing_disabled) ? true: false; 3090 } 3091 3092 /* 3093 * Open and update trace_array ref count. 3094 * Must have the current trace_array passed to it. 3095 */ 3096 static int tracing_open_generic_tr(struct inode *inode, struct file *filp) 3097 { 3098 struct trace_array *tr = inode->i_private; 3099 3100 if (tracing_disabled) 3101 return -ENODEV; 3102 3103 if (trace_array_get(tr) < 0) 3104 return -ENODEV; 3105 3106 filp->private_data = inode->i_private; 3107 3108 return 0; 3109 } 3110 3111 static int tracing_release(struct inode *inode, struct file *file) 3112 { 3113 struct trace_array *tr = inode->i_private; 3114 struct seq_file *m = file->private_data; 3115 struct trace_iterator *iter; 3116 int cpu; 3117 3118 if (!(file->f_mode & FMODE_READ)) { 3119 trace_array_put(tr); 3120 return 0; 3121 } 3122 3123 /* Writes do not use seq_file */ 3124 iter = m->private; 3125 mutex_lock(&trace_types_lock); 3126 3127 for_each_tracing_cpu(cpu) { 3128 if (iter->buffer_iter[cpu]) 3129 ring_buffer_read_finish(iter->buffer_iter[cpu]); 3130 } 3131 3132 if (iter->trace && iter->trace->close) 3133 iter->trace->close(iter); 3134 3135 if (!iter->snapshot) 3136 /* reenable tracing if it was previously enabled */ 3137 tracing_start_tr(tr); 3138 3139 __trace_array_put(tr); 3140 3141 mutex_unlock(&trace_types_lock); 3142 3143 mutex_destroy(&iter->mutex); 3144 free_cpumask_var(iter->started); 3145 kfree(iter->trace); 3146 kfree(iter->buffer_iter); 3147 seq_release_private(inode, file); 3148 3149 return 0; 3150 } 3151 3152 static int tracing_release_generic_tr(struct inode *inode, struct file *file) 3153 { 3154 struct trace_array *tr = inode->i_private; 3155 3156 trace_array_put(tr); 3157 return 0; 3158 } 3159 3160 static int tracing_single_release_tr(struct inode *inode, struct file *file) 3161 { 3162 struct trace_array *tr = inode->i_private; 3163 3164 trace_array_put(tr); 3165 3166 return single_release(inode, file); 3167 } 3168 3169 static int tracing_open(struct inode *inode, struct file *file) 3170 { 3171 struct trace_array *tr = inode->i_private; 3172 struct trace_iterator *iter; 3173 int ret = 0; 3174 3175 if (trace_array_get(tr) < 0) 3176 return -ENODEV; 3177 3178 /* If this file was open for write, then erase contents */ 3179 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 3180 int cpu = tracing_get_cpu(inode); 3181 3182 if (cpu == RING_BUFFER_ALL_CPUS) 3183 tracing_reset_online_cpus(&tr->trace_buffer); 3184 else 3185 tracing_reset(&tr->trace_buffer, cpu); 3186 } 3187 3188 if (file->f_mode & FMODE_READ) { 3189 iter = __tracing_open(inode, file, false); 3190 if (IS_ERR(iter)) 3191 ret = PTR_ERR(iter); 3192 else if (trace_flags & TRACE_ITER_LATENCY_FMT) 3193 iter->iter_flags |= TRACE_FILE_LAT_FMT; 3194 } 3195 3196 if (ret < 0) 3197 trace_array_put(tr); 3198 3199 return ret; 3200 } 3201 3202 /* 3203 * Some tracers are not suitable for instance buffers. 3204 * A tracer is always available for the global array (toplevel) 3205 * or if it explicitly states that it is. 3206 */ 3207 static bool 3208 trace_ok_for_array(struct tracer *t, struct trace_array *tr) 3209 { 3210 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances; 3211 } 3212 3213 /* Find the next tracer that this trace array may use */ 3214 static struct tracer * 3215 get_tracer_for_array(struct trace_array *tr, struct tracer *t) 3216 { 3217 while (t && !trace_ok_for_array(t, tr)) 3218 t = t->next; 3219 3220 return t; 3221 } 3222 3223 static void * 3224 t_next(struct seq_file *m, void *v, loff_t *pos) 3225 { 3226 struct trace_array *tr = m->private; 3227 struct tracer *t = v; 3228 3229 (*pos)++; 3230 3231 if (t) 3232 t = get_tracer_for_array(tr, t->next); 3233 3234 return t; 3235 } 3236 3237 static void *t_start(struct seq_file *m, loff_t *pos) 3238 { 3239 struct trace_array *tr = m->private; 3240 struct tracer *t; 3241 loff_t l = 0; 3242 3243 mutex_lock(&trace_types_lock); 3244 3245 t = get_tracer_for_array(tr, trace_types); 3246 for (; t && l < *pos; t = t_next(m, t, &l)) 3247 ; 3248 3249 return t; 3250 } 3251 3252 static void t_stop(struct seq_file *m, void *p) 3253 { 3254 mutex_unlock(&trace_types_lock); 3255 } 3256 3257 static int t_show(struct seq_file *m, void *v) 3258 { 3259 struct tracer *t = v; 3260 3261 if (!t) 3262 return 0; 3263 3264 seq_printf(m, "%s", t->name); 3265 if (t->next) 3266 seq_putc(m, ' '); 3267 else 3268 seq_putc(m, '\n'); 3269 3270 return 0; 3271 } 3272 3273 static const struct seq_operations show_traces_seq_ops = { 3274 .start = t_start, 3275 .next = t_next, 3276 .stop = t_stop, 3277 .show = t_show, 3278 }; 3279 3280 static int show_traces_open(struct inode *inode, struct file *file) 3281 { 3282 struct trace_array *tr = inode->i_private; 3283 struct seq_file *m; 3284 int ret; 3285 3286 if (tracing_disabled) 3287 return -ENODEV; 3288 3289 ret = seq_open(file, &show_traces_seq_ops); 3290 if (ret) 3291 return ret; 3292 3293 m = file->private_data; 3294 m->private = tr; 3295 3296 return 0; 3297 } 3298 3299 static ssize_t 3300 tracing_write_stub(struct file *filp, const char __user *ubuf, 3301 size_t count, loff_t *ppos) 3302 { 3303 return count; 3304 } 3305 3306 loff_t tracing_lseek(struct file *file, loff_t offset, int whence) 3307 { 3308 int ret; 3309 3310 if (file->f_mode & FMODE_READ) 3311 ret = seq_lseek(file, offset, whence); 3312 else 3313 file->f_pos = ret = 0; 3314 3315 return ret; 3316 } 3317 3318 static const struct file_operations tracing_fops = { 3319 .open = tracing_open, 3320 .read = seq_read, 3321 .write = tracing_write_stub, 3322 .llseek = tracing_lseek, 3323 .release = tracing_release, 3324 }; 3325 3326 static const struct file_operations show_traces_fops = { 3327 .open = show_traces_open, 3328 .read = seq_read, 3329 .release = seq_release, 3330 .llseek = seq_lseek, 3331 }; 3332 3333 /* 3334 * The tracer itself will not take this lock, but still we want 3335 * to provide a consistent cpumask to user-space: 3336 */ 3337 static DEFINE_MUTEX(tracing_cpumask_update_lock); 3338 3339 /* 3340 * Temporary storage for the character representation of the 3341 * CPU bitmask (and one more byte for the newline): 3342 */ 3343 static char mask_str[NR_CPUS + 1]; 3344 3345 static ssize_t 3346 tracing_cpumask_read(struct file *filp, char __user *ubuf, 3347 size_t count, loff_t *ppos) 3348 { 3349 struct trace_array *tr = file_inode(filp)->i_private; 3350 int len; 3351 3352 mutex_lock(&tracing_cpumask_update_lock); 3353 3354 len = cpumask_scnprintf(mask_str, count, tr->tracing_cpumask); 3355 if (count - len < 2) { 3356 count = -EINVAL; 3357 goto out_err; 3358 } 3359 len += sprintf(mask_str + len, "\n"); 3360 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1); 3361 3362 out_err: 3363 mutex_unlock(&tracing_cpumask_update_lock); 3364 3365 return count; 3366 } 3367 3368 static ssize_t 3369 tracing_cpumask_write(struct file *filp, const char __user *ubuf, 3370 size_t count, loff_t *ppos) 3371 { 3372 struct trace_array *tr = file_inode(filp)->i_private; 3373 cpumask_var_t tracing_cpumask_new; 3374 int err, cpu; 3375 3376 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL)) 3377 return -ENOMEM; 3378 3379 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new); 3380 if (err) 3381 goto err_unlock; 3382 3383 mutex_lock(&tracing_cpumask_update_lock); 3384 3385 local_irq_disable(); 3386 arch_spin_lock(&tr->max_lock); 3387 for_each_tracing_cpu(cpu) { 3388 /* 3389 * Increase/decrease the disabled counter if we are 3390 * about to flip a bit in the cpumask: 3391 */ 3392 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) && 3393 !cpumask_test_cpu(cpu, tracing_cpumask_new)) { 3394 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled); 3395 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu); 3396 } 3397 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) && 3398 cpumask_test_cpu(cpu, tracing_cpumask_new)) { 3399 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled); 3400 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu); 3401 } 3402 } 3403 arch_spin_unlock(&tr->max_lock); 3404 local_irq_enable(); 3405 3406 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new); 3407 3408 mutex_unlock(&tracing_cpumask_update_lock); 3409 free_cpumask_var(tracing_cpumask_new); 3410 3411 return count; 3412 3413 err_unlock: 3414 free_cpumask_var(tracing_cpumask_new); 3415 3416 return err; 3417 } 3418 3419 static const struct file_operations tracing_cpumask_fops = { 3420 .open = tracing_open_generic_tr, 3421 .read = tracing_cpumask_read, 3422 .write = tracing_cpumask_write, 3423 .release = tracing_release_generic_tr, 3424 .llseek = generic_file_llseek, 3425 }; 3426 3427 static int tracing_trace_options_show(struct seq_file *m, void *v) 3428 { 3429 struct tracer_opt *trace_opts; 3430 struct trace_array *tr = m->private; 3431 u32 tracer_flags; 3432 int i; 3433 3434 mutex_lock(&trace_types_lock); 3435 tracer_flags = tr->current_trace->flags->val; 3436 trace_opts = tr->current_trace->flags->opts; 3437 3438 for (i = 0; trace_options[i]; i++) { 3439 if (trace_flags & (1 << i)) 3440 seq_printf(m, "%s\n", trace_options[i]); 3441 else 3442 seq_printf(m, "no%s\n", trace_options[i]); 3443 } 3444 3445 for (i = 0; trace_opts[i].name; i++) { 3446 if (tracer_flags & trace_opts[i].bit) 3447 seq_printf(m, "%s\n", trace_opts[i].name); 3448 else 3449 seq_printf(m, "no%s\n", trace_opts[i].name); 3450 } 3451 mutex_unlock(&trace_types_lock); 3452 3453 return 0; 3454 } 3455 3456 static int __set_tracer_option(struct trace_array *tr, 3457 struct tracer_flags *tracer_flags, 3458 struct tracer_opt *opts, int neg) 3459 { 3460 struct tracer *trace = tr->current_trace; 3461 int ret; 3462 3463 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg); 3464 if (ret) 3465 return ret; 3466 3467 if (neg) 3468 tracer_flags->val &= ~opts->bit; 3469 else 3470 tracer_flags->val |= opts->bit; 3471 return 0; 3472 } 3473 3474 /* Try to assign a tracer specific option */ 3475 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg) 3476 { 3477 struct tracer *trace = tr->current_trace; 3478 struct tracer_flags *tracer_flags = trace->flags; 3479 struct tracer_opt *opts = NULL; 3480 int i; 3481 3482 for (i = 0; tracer_flags->opts[i].name; i++) { 3483 opts = &tracer_flags->opts[i]; 3484 3485 if (strcmp(cmp, opts->name) == 0) 3486 return __set_tracer_option(tr, trace->flags, opts, neg); 3487 } 3488 3489 return -EINVAL; 3490 } 3491 3492 /* Some tracers require overwrite to stay enabled */ 3493 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set) 3494 { 3495 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set) 3496 return -1; 3497 3498 return 0; 3499 } 3500 3501 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled) 3502 { 3503 /* do nothing if flag is already set */ 3504 if (!!(trace_flags & mask) == !!enabled) 3505 return 0; 3506 3507 /* Give the tracer a chance to approve the change */ 3508 if (tr->current_trace->flag_changed) 3509 if (tr->current_trace->flag_changed(tr, mask, !!enabled)) 3510 return -EINVAL; 3511 3512 if (enabled) 3513 trace_flags |= mask; 3514 else 3515 trace_flags &= ~mask; 3516 3517 if (mask == TRACE_ITER_RECORD_CMD) 3518 trace_event_enable_cmd_record(enabled); 3519 3520 if (mask == TRACE_ITER_OVERWRITE) { 3521 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled); 3522 #ifdef CONFIG_TRACER_MAX_TRACE 3523 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled); 3524 #endif 3525 } 3526 3527 if (mask == TRACE_ITER_PRINTK) 3528 trace_printk_start_stop_comm(enabled); 3529 3530 return 0; 3531 } 3532 3533 static int trace_set_options(struct trace_array *tr, char *option) 3534 { 3535 char *cmp; 3536 int neg = 0; 3537 int ret = -ENODEV; 3538 int i; 3539 3540 cmp = strstrip(option); 3541 3542 if (strncmp(cmp, "no", 2) == 0) { 3543 neg = 1; 3544 cmp += 2; 3545 } 3546 3547 mutex_lock(&trace_types_lock); 3548 3549 for (i = 0; trace_options[i]; i++) { 3550 if (strcmp(cmp, trace_options[i]) == 0) { 3551 ret = set_tracer_flag(tr, 1 << i, !neg); 3552 break; 3553 } 3554 } 3555 3556 /* If no option could be set, test the specific tracer options */ 3557 if (!trace_options[i]) 3558 ret = set_tracer_option(tr, cmp, neg); 3559 3560 mutex_unlock(&trace_types_lock); 3561 3562 return ret; 3563 } 3564 3565 static ssize_t 3566 tracing_trace_options_write(struct file *filp, const char __user *ubuf, 3567 size_t cnt, loff_t *ppos) 3568 { 3569 struct seq_file *m = filp->private_data; 3570 struct trace_array *tr = m->private; 3571 char buf[64]; 3572 int ret; 3573 3574 if (cnt >= sizeof(buf)) 3575 return -EINVAL; 3576 3577 if (copy_from_user(&buf, ubuf, cnt)) 3578 return -EFAULT; 3579 3580 buf[cnt] = 0; 3581 3582 ret = trace_set_options(tr, buf); 3583 if (ret < 0) 3584 return ret; 3585 3586 *ppos += cnt; 3587 3588 return cnt; 3589 } 3590 3591 static int tracing_trace_options_open(struct inode *inode, struct file *file) 3592 { 3593 struct trace_array *tr = inode->i_private; 3594 int ret; 3595 3596 if (tracing_disabled) 3597 return -ENODEV; 3598 3599 if (trace_array_get(tr) < 0) 3600 return -ENODEV; 3601 3602 ret = single_open(file, tracing_trace_options_show, inode->i_private); 3603 if (ret < 0) 3604 trace_array_put(tr); 3605 3606 return ret; 3607 } 3608 3609 static const struct file_operations tracing_iter_fops = { 3610 .open = tracing_trace_options_open, 3611 .read = seq_read, 3612 .llseek = seq_lseek, 3613 .release = tracing_single_release_tr, 3614 .write = tracing_trace_options_write, 3615 }; 3616 3617 static const char readme_msg[] = 3618 "tracing mini-HOWTO:\n\n" 3619 "# echo 0 > tracing_on : quick way to disable tracing\n" 3620 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n" 3621 " Important files:\n" 3622 " trace\t\t\t- The static contents of the buffer\n" 3623 "\t\t\t To clear the buffer write into this file: echo > trace\n" 3624 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n" 3625 " current_tracer\t- function and latency tracers\n" 3626 " available_tracers\t- list of configured tracers for current_tracer\n" 3627 " buffer_size_kb\t- view and modify size of per cpu buffer\n" 3628 " buffer_total_size_kb - view total size of all cpu buffers\n\n" 3629 " trace_clock\t\t-change the clock used to order events\n" 3630 " local: Per cpu clock but may not be synced across CPUs\n" 3631 " global: Synced across CPUs but slows tracing down.\n" 3632 " counter: Not a clock, but just an increment\n" 3633 " uptime: Jiffy counter from time of boot\n" 3634 " perf: Same clock that perf events use\n" 3635 #ifdef CONFIG_X86_64 3636 " x86-tsc: TSC cycle counter\n" 3637 #endif 3638 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n" 3639 " tracing_cpumask\t- Limit which CPUs to trace\n" 3640 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n" 3641 "\t\t\t Remove sub-buffer with rmdir\n" 3642 " trace_options\t\t- Set format or modify how tracing happens\n" 3643 "\t\t\t Disable an option by adding a suffix 'no' to the\n" 3644 "\t\t\t option name\n" 3645 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n" 3646 #ifdef CONFIG_DYNAMIC_FTRACE 3647 "\n available_filter_functions - list of functions that can be filtered on\n" 3648 " set_ftrace_filter\t- echo function name in here to only trace these\n" 3649 "\t\t\t functions\n" 3650 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n" 3651 "\t modules: Can select a group via module\n" 3652 "\t Format: :mod:<module-name>\n" 3653 "\t example: echo :mod:ext3 > set_ftrace_filter\n" 3654 "\t triggers: a command to perform when function is hit\n" 3655 "\t Format: <function>:<trigger>[:count]\n" 3656 "\t trigger: traceon, traceoff\n" 3657 "\t\t enable_event:<system>:<event>\n" 3658 "\t\t disable_event:<system>:<event>\n" 3659 #ifdef CONFIG_STACKTRACE 3660 "\t\t stacktrace\n" 3661 #endif 3662 #ifdef CONFIG_TRACER_SNAPSHOT 3663 "\t\t snapshot\n" 3664 #endif 3665 "\t\t dump\n" 3666 "\t\t cpudump\n" 3667 "\t example: echo do_fault:traceoff > set_ftrace_filter\n" 3668 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n" 3669 "\t The first one will disable tracing every time do_fault is hit\n" 3670 "\t The second will disable tracing at most 3 times when do_trap is hit\n" 3671 "\t The first time do trap is hit and it disables tracing, the\n" 3672 "\t counter will decrement to 2. If tracing is already disabled,\n" 3673 "\t the counter will not decrement. It only decrements when the\n" 3674 "\t trigger did work\n" 3675 "\t To remove trigger without count:\n" 3676 "\t echo '!<function>:<trigger> > set_ftrace_filter\n" 3677 "\t To remove trigger with a count:\n" 3678 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n" 3679 " set_ftrace_notrace\t- echo function name in here to never trace.\n" 3680 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n" 3681 "\t modules: Can select a group via module command :mod:\n" 3682 "\t Does not accept triggers\n" 3683 #endif /* CONFIG_DYNAMIC_FTRACE */ 3684 #ifdef CONFIG_FUNCTION_TRACER 3685 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n" 3686 "\t\t (function)\n" 3687 #endif 3688 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 3689 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n" 3690 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n" 3691 #endif 3692 #ifdef CONFIG_TRACER_SNAPSHOT 3693 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n" 3694 "\t\t\t snapshot buffer. Read the contents for more\n" 3695 "\t\t\t information\n" 3696 #endif 3697 #ifdef CONFIG_STACK_TRACER 3698 " stack_trace\t\t- Shows the max stack trace when active\n" 3699 " stack_max_size\t- Shows current max stack size that was traced\n" 3700 "\t\t\t Write into this file to reset the max size (trigger a\n" 3701 "\t\t\t new trace)\n" 3702 #ifdef CONFIG_DYNAMIC_FTRACE 3703 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n" 3704 "\t\t\t traces\n" 3705 #endif 3706 #endif /* CONFIG_STACK_TRACER */ 3707 " events/\t\t- Directory containing all trace event subsystems:\n" 3708 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n" 3709 " events/<system>/\t- Directory containing all trace events for <system>:\n" 3710 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n" 3711 "\t\t\t events\n" 3712 " filter\t\t- If set, only events passing filter are traced\n" 3713 " events/<system>/<event>/\t- Directory containing control files for\n" 3714 "\t\t\t <event>:\n" 3715 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n" 3716 " filter\t\t- If set, only events passing filter are traced\n" 3717 " trigger\t\t- If set, a command to perform when event is hit\n" 3718 "\t Format: <trigger>[:count][if <filter>]\n" 3719 "\t trigger: traceon, traceoff\n" 3720 "\t enable_event:<system>:<event>\n" 3721 "\t disable_event:<system>:<event>\n" 3722 #ifdef CONFIG_STACKTRACE 3723 "\t\t stacktrace\n" 3724 #endif 3725 #ifdef CONFIG_TRACER_SNAPSHOT 3726 "\t\t snapshot\n" 3727 #endif 3728 "\t example: echo traceoff > events/block/block_unplug/trigger\n" 3729 "\t echo traceoff:3 > events/block/block_unplug/trigger\n" 3730 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n" 3731 "\t events/block/block_unplug/trigger\n" 3732 "\t The first disables tracing every time block_unplug is hit.\n" 3733 "\t The second disables tracing the first 3 times block_unplug is hit.\n" 3734 "\t The third enables the kmalloc event the first 3 times block_unplug\n" 3735 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n" 3736 "\t Like function triggers, the counter is only decremented if it\n" 3737 "\t enabled or disabled tracing.\n" 3738 "\t To remove a trigger without a count:\n" 3739 "\t echo '!<trigger> > <system>/<event>/trigger\n" 3740 "\t To remove a trigger with a count:\n" 3741 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n" 3742 "\t Filters can be ignored when removing a trigger.\n" 3743 ; 3744 3745 static ssize_t 3746 tracing_readme_read(struct file *filp, char __user *ubuf, 3747 size_t cnt, loff_t *ppos) 3748 { 3749 return simple_read_from_buffer(ubuf, cnt, ppos, 3750 readme_msg, strlen(readme_msg)); 3751 } 3752 3753 static const struct file_operations tracing_readme_fops = { 3754 .open = tracing_open_generic, 3755 .read = tracing_readme_read, 3756 .llseek = generic_file_llseek, 3757 }; 3758 3759 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos) 3760 { 3761 unsigned int *ptr = v; 3762 3763 if (*pos || m->count) 3764 ptr++; 3765 3766 (*pos)++; 3767 3768 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num]; 3769 ptr++) { 3770 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP) 3771 continue; 3772 3773 return ptr; 3774 } 3775 3776 return NULL; 3777 } 3778 3779 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos) 3780 { 3781 void *v; 3782 loff_t l = 0; 3783 3784 preempt_disable(); 3785 arch_spin_lock(&trace_cmdline_lock); 3786 3787 v = &savedcmd->map_cmdline_to_pid[0]; 3788 while (l <= *pos) { 3789 v = saved_cmdlines_next(m, v, &l); 3790 if (!v) 3791 return NULL; 3792 } 3793 3794 return v; 3795 } 3796 3797 static void saved_cmdlines_stop(struct seq_file *m, void *v) 3798 { 3799 arch_spin_unlock(&trace_cmdline_lock); 3800 preempt_enable(); 3801 } 3802 3803 static int saved_cmdlines_show(struct seq_file *m, void *v) 3804 { 3805 char buf[TASK_COMM_LEN]; 3806 unsigned int *pid = v; 3807 3808 __trace_find_cmdline(*pid, buf); 3809 seq_printf(m, "%d %s\n", *pid, buf); 3810 return 0; 3811 } 3812 3813 static const struct seq_operations tracing_saved_cmdlines_seq_ops = { 3814 .start = saved_cmdlines_start, 3815 .next = saved_cmdlines_next, 3816 .stop = saved_cmdlines_stop, 3817 .show = saved_cmdlines_show, 3818 }; 3819 3820 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp) 3821 { 3822 if (tracing_disabled) 3823 return -ENODEV; 3824 3825 return seq_open(filp, &tracing_saved_cmdlines_seq_ops); 3826 } 3827 3828 static const struct file_operations tracing_saved_cmdlines_fops = { 3829 .open = tracing_saved_cmdlines_open, 3830 .read = seq_read, 3831 .llseek = seq_lseek, 3832 .release = seq_release, 3833 }; 3834 3835 static ssize_t 3836 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf, 3837 size_t cnt, loff_t *ppos) 3838 { 3839 char buf[64]; 3840 int r; 3841 3842 arch_spin_lock(&trace_cmdline_lock); 3843 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num); 3844 arch_spin_unlock(&trace_cmdline_lock); 3845 3846 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 3847 } 3848 3849 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s) 3850 { 3851 kfree(s->saved_cmdlines); 3852 kfree(s->map_cmdline_to_pid); 3853 kfree(s); 3854 } 3855 3856 static int tracing_resize_saved_cmdlines(unsigned int val) 3857 { 3858 struct saved_cmdlines_buffer *s, *savedcmd_temp; 3859 3860 s = kmalloc(sizeof(*s), GFP_KERNEL); 3861 if (!s) 3862 return -ENOMEM; 3863 3864 if (allocate_cmdlines_buffer(val, s) < 0) { 3865 kfree(s); 3866 return -ENOMEM; 3867 } 3868 3869 arch_spin_lock(&trace_cmdline_lock); 3870 savedcmd_temp = savedcmd; 3871 savedcmd = s; 3872 arch_spin_unlock(&trace_cmdline_lock); 3873 free_saved_cmdlines_buffer(savedcmd_temp); 3874 3875 return 0; 3876 } 3877 3878 static ssize_t 3879 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf, 3880 size_t cnt, loff_t *ppos) 3881 { 3882 unsigned long val; 3883 int ret; 3884 3885 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 3886 if (ret) 3887 return ret; 3888 3889 /* must have at least 1 entry or less than PID_MAX_DEFAULT */ 3890 if (!val || val > PID_MAX_DEFAULT) 3891 return -EINVAL; 3892 3893 ret = tracing_resize_saved_cmdlines((unsigned int)val); 3894 if (ret < 0) 3895 return ret; 3896 3897 *ppos += cnt; 3898 3899 return cnt; 3900 } 3901 3902 static const struct file_operations tracing_saved_cmdlines_size_fops = { 3903 .open = tracing_open_generic, 3904 .read = tracing_saved_cmdlines_size_read, 3905 .write = tracing_saved_cmdlines_size_write, 3906 }; 3907 3908 static ssize_t 3909 tracing_set_trace_read(struct file *filp, char __user *ubuf, 3910 size_t cnt, loff_t *ppos) 3911 { 3912 struct trace_array *tr = filp->private_data; 3913 char buf[MAX_TRACER_SIZE+2]; 3914 int r; 3915 3916 mutex_lock(&trace_types_lock); 3917 r = sprintf(buf, "%s\n", tr->current_trace->name); 3918 mutex_unlock(&trace_types_lock); 3919 3920 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 3921 } 3922 3923 int tracer_init(struct tracer *t, struct trace_array *tr) 3924 { 3925 tracing_reset_online_cpus(&tr->trace_buffer); 3926 return t->init(tr); 3927 } 3928 3929 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val) 3930 { 3931 int cpu; 3932 3933 for_each_tracing_cpu(cpu) 3934 per_cpu_ptr(buf->data, cpu)->entries = val; 3935 } 3936 3937 #ifdef CONFIG_TRACER_MAX_TRACE 3938 /* resize @tr's buffer to the size of @size_tr's entries */ 3939 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf, 3940 struct trace_buffer *size_buf, int cpu_id) 3941 { 3942 int cpu, ret = 0; 3943 3944 if (cpu_id == RING_BUFFER_ALL_CPUS) { 3945 for_each_tracing_cpu(cpu) { 3946 ret = ring_buffer_resize(trace_buf->buffer, 3947 per_cpu_ptr(size_buf->data, cpu)->entries, cpu); 3948 if (ret < 0) 3949 break; 3950 per_cpu_ptr(trace_buf->data, cpu)->entries = 3951 per_cpu_ptr(size_buf->data, cpu)->entries; 3952 } 3953 } else { 3954 ret = ring_buffer_resize(trace_buf->buffer, 3955 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id); 3956 if (ret == 0) 3957 per_cpu_ptr(trace_buf->data, cpu_id)->entries = 3958 per_cpu_ptr(size_buf->data, cpu_id)->entries; 3959 } 3960 3961 return ret; 3962 } 3963 #endif /* CONFIG_TRACER_MAX_TRACE */ 3964 3965 static int __tracing_resize_ring_buffer(struct trace_array *tr, 3966 unsigned long size, int cpu) 3967 { 3968 int ret; 3969 3970 /* 3971 * If kernel or user changes the size of the ring buffer 3972 * we use the size that was given, and we can forget about 3973 * expanding it later. 3974 */ 3975 ring_buffer_expanded = true; 3976 3977 /* May be called before buffers are initialized */ 3978 if (!tr->trace_buffer.buffer) 3979 return 0; 3980 3981 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu); 3982 if (ret < 0) 3983 return ret; 3984 3985 #ifdef CONFIG_TRACER_MAX_TRACE 3986 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) || 3987 !tr->current_trace->use_max_tr) 3988 goto out; 3989 3990 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu); 3991 if (ret < 0) { 3992 int r = resize_buffer_duplicate_size(&tr->trace_buffer, 3993 &tr->trace_buffer, cpu); 3994 if (r < 0) { 3995 /* 3996 * AARGH! We are left with different 3997 * size max buffer!!!! 3998 * The max buffer is our "snapshot" buffer. 3999 * When a tracer needs a snapshot (one of the 4000 * latency tracers), it swaps the max buffer 4001 * with the saved snap shot. We succeeded to 4002 * update the size of the main buffer, but failed to 4003 * update the size of the max buffer. But when we tried 4004 * to reset the main buffer to the original size, we 4005 * failed there too. This is very unlikely to 4006 * happen, but if it does, warn and kill all 4007 * tracing. 4008 */ 4009 WARN_ON(1); 4010 tracing_disabled = 1; 4011 } 4012 return ret; 4013 } 4014 4015 if (cpu == RING_BUFFER_ALL_CPUS) 4016 set_buffer_entries(&tr->max_buffer, size); 4017 else 4018 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size; 4019 4020 out: 4021 #endif /* CONFIG_TRACER_MAX_TRACE */ 4022 4023 if (cpu == RING_BUFFER_ALL_CPUS) 4024 set_buffer_entries(&tr->trace_buffer, size); 4025 else 4026 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size; 4027 4028 return ret; 4029 } 4030 4031 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr, 4032 unsigned long size, int cpu_id) 4033 { 4034 int ret = size; 4035 4036 mutex_lock(&trace_types_lock); 4037 4038 if (cpu_id != RING_BUFFER_ALL_CPUS) { 4039 /* make sure, this cpu is enabled in the mask */ 4040 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) { 4041 ret = -EINVAL; 4042 goto out; 4043 } 4044 } 4045 4046 ret = __tracing_resize_ring_buffer(tr, size, cpu_id); 4047 if (ret < 0) 4048 ret = -ENOMEM; 4049 4050 out: 4051 mutex_unlock(&trace_types_lock); 4052 4053 return ret; 4054 } 4055 4056 4057 /** 4058 * tracing_update_buffers - used by tracing facility to expand ring buffers 4059 * 4060 * To save on memory when the tracing is never used on a system with it 4061 * configured in. The ring buffers are set to a minimum size. But once 4062 * a user starts to use the tracing facility, then they need to grow 4063 * to their default size. 4064 * 4065 * This function is to be called when a tracer is about to be used. 4066 */ 4067 int tracing_update_buffers(void) 4068 { 4069 int ret = 0; 4070 4071 mutex_lock(&trace_types_lock); 4072 if (!ring_buffer_expanded) 4073 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size, 4074 RING_BUFFER_ALL_CPUS); 4075 mutex_unlock(&trace_types_lock); 4076 4077 return ret; 4078 } 4079 4080 struct trace_option_dentry; 4081 4082 static struct trace_option_dentry * 4083 create_trace_option_files(struct trace_array *tr, struct tracer *tracer); 4084 4085 static void 4086 destroy_trace_option_files(struct trace_option_dentry *topts); 4087 4088 /* 4089 * Used to clear out the tracer before deletion of an instance. 4090 * Must have trace_types_lock held. 4091 */ 4092 static void tracing_set_nop(struct trace_array *tr) 4093 { 4094 if (tr->current_trace == &nop_trace) 4095 return; 4096 4097 tr->current_trace->enabled--; 4098 4099 if (tr->current_trace->reset) 4100 tr->current_trace->reset(tr); 4101 4102 tr->current_trace = &nop_trace; 4103 } 4104 4105 static int tracing_set_tracer(struct trace_array *tr, const char *buf) 4106 { 4107 static struct trace_option_dentry *topts; 4108 struct tracer *t; 4109 #ifdef CONFIG_TRACER_MAX_TRACE 4110 bool had_max_tr; 4111 #endif 4112 int ret = 0; 4113 4114 mutex_lock(&trace_types_lock); 4115 4116 if (!ring_buffer_expanded) { 4117 ret = __tracing_resize_ring_buffer(tr, trace_buf_size, 4118 RING_BUFFER_ALL_CPUS); 4119 if (ret < 0) 4120 goto out; 4121 ret = 0; 4122 } 4123 4124 for (t = trace_types; t; t = t->next) { 4125 if (strcmp(t->name, buf) == 0) 4126 break; 4127 } 4128 if (!t) { 4129 ret = -EINVAL; 4130 goto out; 4131 } 4132 if (t == tr->current_trace) 4133 goto out; 4134 4135 /* Some tracers are only allowed for the top level buffer */ 4136 if (!trace_ok_for_array(t, tr)) { 4137 ret = -EINVAL; 4138 goto out; 4139 } 4140 4141 trace_branch_disable(); 4142 4143 tr->current_trace->enabled--; 4144 4145 if (tr->current_trace->reset) 4146 tr->current_trace->reset(tr); 4147 4148 /* Current trace needs to be nop_trace before synchronize_sched */ 4149 tr->current_trace = &nop_trace; 4150 4151 #ifdef CONFIG_TRACER_MAX_TRACE 4152 had_max_tr = tr->allocated_snapshot; 4153 4154 if (had_max_tr && !t->use_max_tr) { 4155 /* 4156 * We need to make sure that the update_max_tr sees that 4157 * current_trace changed to nop_trace to keep it from 4158 * swapping the buffers after we resize it. 4159 * The update_max_tr is called from interrupts disabled 4160 * so a synchronized_sched() is sufficient. 4161 */ 4162 synchronize_sched(); 4163 free_snapshot(tr); 4164 } 4165 #endif 4166 /* Currently, only the top instance has options */ 4167 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) { 4168 destroy_trace_option_files(topts); 4169 topts = create_trace_option_files(tr, t); 4170 } 4171 4172 #ifdef CONFIG_TRACER_MAX_TRACE 4173 if (t->use_max_tr && !had_max_tr) { 4174 ret = alloc_snapshot(tr); 4175 if (ret < 0) 4176 goto out; 4177 } 4178 #endif 4179 4180 if (t->init) { 4181 ret = tracer_init(t, tr); 4182 if (ret) 4183 goto out; 4184 } 4185 4186 tr->current_trace = t; 4187 tr->current_trace->enabled++; 4188 trace_branch_enable(tr); 4189 out: 4190 mutex_unlock(&trace_types_lock); 4191 4192 return ret; 4193 } 4194 4195 static ssize_t 4196 tracing_set_trace_write(struct file *filp, const char __user *ubuf, 4197 size_t cnt, loff_t *ppos) 4198 { 4199 struct trace_array *tr = filp->private_data; 4200 char buf[MAX_TRACER_SIZE+1]; 4201 int i; 4202 size_t ret; 4203 int err; 4204 4205 ret = cnt; 4206 4207 if (cnt > MAX_TRACER_SIZE) 4208 cnt = MAX_TRACER_SIZE; 4209 4210 if (copy_from_user(&buf, ubuf, cnt)) 4211 return -EFAULT; 4212 4213 buf[cnt] = 0; 4214 4215 /* strip ending whitespace. */ 4216 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--) 4217 buf[i] = 0; 4218 4219 err = tracing_set_tracer(tr, buf); 4220 if (err) 4221 return err; 4222 4223 *ppos += ret; 4224 4225 return ret; 4226 } 4227 4228 static ssize_t 4229 tracing_max_lat_read(struct file *filp, char __user *ubuf, 4230 size_t cnt, loff_t *ppos) 4231 { 4232 unsigned long *ptr = filp->private_data; 4233 char buf[64]; 4234 int r; 4235 4236 r = snprintf(buf, sizeof(buf), "%ld\n", 4237 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr)); 4238 if (r > sizeof(buf)) 4239 r = sizeof(buf); 4240 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 4241 } 4242 4243 static ssize_t 4244 tracing_max_lat_write(struct file *filp, const char __user *ubuf, 4245 size_t cnt, loff_t *ppos) 4246 { 4247 unsigned long *ptr = filp->private_data; 4248 unsigned long val; 4249 int ret; 4250 4251 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 4252 if (ret) 4253 return ret; 4254 4255 *ptr = val * 1000; 4256 4257 return cnt; 4258 } 4259 4260 static int tracing_open_pipe(struct inode *inode, struct file *filp) 4261 { 4262 struct trace_array *tr = inode->i_private; 4263 struct trace_iterator *iter; 4264 int ret = 0; 4265 4266 if (tracing_disabled) 4267 return -ENODEV; 4268 4269 if (trace_array_get(tr) < 0) 4270 return -ENODEV; 4271 4272 mutex_lock(&trace_types_lock); 4273 4274 /* create a buffer to store the information to pass to userspace */ 4275 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 4276 if (!iter) { 4277 ret = -ENOMEM; 4278 __trace_array_put(tr); 4279 goto out; 4280 } 4281 4282 /* 4283 * We make a copy of the current tracer to avoid concurrent 4284 * changes on it while we are reading. 4285 */ 4286 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL); 4287 if (!iter->trace) { 4288 ret = -ENOMEM; 4289 goto fail; 4290 } 4291 *iter->trace = *tr->current_trace; 4292 4293 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) { 4294 ret = -ENOMEM; 4295 goto fail; 4296 } 4297 4298 /* trace pipe does not show start of buffer */ 4299 cpumask_setall(iter->started); 4300 4301 if (trace_flags & TRACE_ITER_LATENCY_FMT) 4302 iter->iter_flags |= TRACE_FILE_LAT_FMT; 4303 4304 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 4305 if (trace_clocks[tr->clock_id].in_ns) 4306 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 4307 4308 iter->tr = tr; 4309 iter->trace_buffer = &tr->trace_buffer; 4310 iter->cpu_file = tracing_get_cpu(inode); 4311 mutex_init(&iter->mutex); 4312 filp->private_data = iter; 4313 4314 if (iter->trace->pipe_open) 4315 iter->trace->pipe_open(iter); 4316 4317 nonseekable_open(inode, filp); 4318 out: 4319 mutex_unlock(&trace_types_lock); 4320 return ret; 4321 4322 fail: 4323 kfree(iter->trace); 4324 kfree(iter); 4325 __trace_array_put(tr); 4326 mutex_unlock(&trace_types_lock); 4327 return ret; 4328 } 4329 4330 static int tracing_release_pipe(struct inode *inode, struct file *file) 4331 { 4332 struct trace_iterator *iter = file->private_data; 4333 struct trace_array *tr = inode->i_private; 4334 4335 mutex_lock(&trace_types_lock); 4336 4337 if (iter->trace->pipe_close) 4338 iter->trace->pipe_close(iter); 4339 4340 mutex_unlock(&trace_types_lock); 4341 4342 free_cpumask_var(iter->started); 4343 mutex_destroy(&iter->mutex); 4344 kfree(iter->trace); 4345 kfree(iter); 4346 4347 trace_array_put(tr); 4348 4349 return 0; 4350 } 4351 4352 static unsigned int 4353 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table) 4354 { 4355 /* Iterators are static, they should be filled or empty */ 4356 if (trace_buffer_iter(iter, iter->cpu_file)) 4357 return POLLIN | POLLRDNORM; 4358 4359 if (trace_flags & TRACE_ITER_BLOCK) 4360 /* 4361 * Always select as readable when in blocking mode 4362 */ 4363 return POLLIN | POLLRDNORM; 4364 else 4365 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file, 4366 filp, poll_table); 4367 } 4368 4369 static unsigned int 4370 tracing_poll_pipe(struct file *filp, poll_table *poll_table) 4371 { 4372 struct trace_iterator *iter = filp->private_data; 4373 4374 return trace_poll(iter, filp, poll_table); 4375 } 4376 4377 /* Must be called with trace_types_lock mutex held. */ 4378 static int tracing_wait_pipe(struct file *filp) 4379 { 4380 struct trace_iterator *iter = filp->private_data; 4381 int ret; 4382 4383 while (trace_empty(iter)) { 4384 4385 if ((filp->f_flags & O_NONBLOCK)) { 4386 return -EAGAIN; 4387 } 4388 4389 /* 4390 * We block until we read something and tracing is disabled. 4391 * We still block if tracing is disabled, but we have never 4392 * read anything. This allows a user to cat this file, and 4393 * then enable tracing. But after we have read something, 4394 * we give an EOF when tracing is again disabled. 4395 * 4396 * iter->pos will be 0 if we haven't read anything. 4397 */ 4398 if (!tracing_is_on() && iter->pos) 4399 break; 4400 4401 mutex_unlock(&iter->mutex); 4402 4403 ret = wait_on_pipe(iter); 4404 4405 mutex_lock(&iter->mutex); 4406 4407 if (ret) 4408 return ret; 4409 4410 if (signal_pending(current)) 4411 return -EINTR; 4412 } 4413 4414 return 1; 4415 } 4416 4417 /* 4418 * Consumer reader. 4419 */ 4420 static ssize_t 4421 tracing_read_pipe(struct file *filp, char __user *ubuf, 4422 size_t cnt, loff_t *ppos) 4423 { 4424 struct trace_iterator *iter = filp->private_data; 4425 struct trace_array *tr = iter->tr; 4426 ssize_t sret; 4427 4428 /* return any leftover data */ 4429 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 4430 if (sret != -EBUSY) 4431 return sret; 4432 4433 trace_seq_init(&iter->seq); 4434 4435 /* copy the tracer to avoid using a global lock all around */ 4436 mutex_lock(&trace_types_lock); 4437 if (unlikely(iter->trace->name != tr->current_trace->name)) 4438 *iter->trace = *tr->current_trace; 4439 mutex_unlock(&trace_types_lock); 4440 4441 /* 4442 * Avoid more than one consumer on a single file descriptor 4443 * This is just a matter of traces coherency, the ring buffer itself 4444 * is protected. 4445 */ 4446 mutex_lock(&iter->mutex); 4447 if (iter->trace->read) { 4448 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); 4449 if (sret) 4450 goto out; 4451 } 4452 4453 waitagain: 4454 sret = tracing_wait_pipe(filp); 4455 if (sret <= 0) 4456 goto out; 4457 4458 /* stop when tracing is finished */ 4459 if (trace_empty(iter)) { 4460 sret = 0; 4461 goto out; 4462 } 4463 4464 if (cnt >= PAGE_SIZE) 4465 cnt = PAGE_SIZE - 1; 4466 4467 /* reset all but tr, trace, and overruns */ 4468 memset(&iter->seq, 0, 4469 sizeof(struct trace_iterator) - 4470 offsetof(struct trace_iterator, seq)); 4471 cpumask_clear(iter->started); 4472 iter->pos = -1; 4473 4474 trace_event_read_lock(); 4475 trace_access_lock(iter->cpu_file); 4476 while (trace_find_next_entry_inc(iter) != NULL) { 4477 enum print_line_t ret; 4478 int len = iter->seq.len; 4479 4480 ret = print_trace_line(iter); 4481 if (ret == TRACE_TYPE_PARTIAL_LINE) { 4482 /* don't print partial lines */ 4483 iter->seq.len = len; 4484 break; 4485 } 4486 if (ret != TRACE_TYPE_NO_CONSUME) 4487 trace_consume(iter); 4488 4489 if (iter->seq.len >= cnt) 4490 break; 4491 4492 /* 4493 * Setting the full flag means we reached the trace_seq buffer 4494 * size and we should leave by partial output condition above. 4495 * One of the trace_seq_* functions is not used properly. 4496 */ 4497 WARN_ONCE(iter->seq.full, "full flag set for trace type %d", 4498 iter->ent->type); 4499 } 4500 trace_access_unlock(iter->cpu_file); 4501 trace_event_read_unlock(); 4502 4503 /* Now copy what we have to the user */ 4504 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 4505 if (iter->seq.readpos >= iter->seq.len) 4506 trace_seq_init(&iter->seq); 4507 4508 /* 4509 * If there was nothing to send to user, in spite of consuming trace 4510 * entries, go back to wait for more entries. 4511 */ 4512 if (sret == -EBUSY) 4513 goto waitagain; 4514 4515 out: 4516 mutex_unlock(&iter->mutex); 4517 4518 return sret; 4519 } 4520 4521 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd, 4522 unsigned int idx) 4523 { 4524 __free_page(spd->pages[idx]); 4525 } 4526 4527 static const struct pipe_buf_operations tracing_pipe_buf_ops = { 4528 .can_merge = 0, 4529 .confirm = generic_pipe_buf_confirm, 4530 .release = generic_pipe_buf_release, 4531 .steal = generic_pipe_buf_steal, 4532 .get = generic_pipe_buf_get, 4533 }; 4534 4535 static size_t 4536 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter) 4537 { 4538 size_t count; 4539 int ret; 4540 4541 /* Seq buffer is page-sized, exactly what we need. */ 4542 for (;;) { 4543 count = iter->seq.len; 4544 ret = print_trace_line(iter); 4545 count = iter->seq.len - count; 4546 if (rem < count) { 4547 rem = 0; 4548 iter->seq.len -= count; 4549 break; 4550 } 4551 if (ret == TRACE_TYPE_PARTIAL_LINE) { 4552 iter->seq.len -= count; 4553 break; 4554 } 4555 4556 if (ret != TRACE_TYPE_NO_CONSUME) 4557 trace_consume(iter); 4558 rem -= count; 4559 if (!trace_find_next_entry_inc(iter)) { 4560 rem = 0; 4561 iter->ent = NULL; 4562 break; 4563 } 4564 } 4565 4566 return rem; 4567 } 4568 4569 static ssize_t tracing_splice_read_pipe(struct file *filp, 4570 loff_t *ppos, 4571 struct pipe_inode_info *pipe, 4572 size_t len, 4573 unsigned int flags) 4574 { 4575 struct page *pages_def[PIPE_DEF_BUFFERS]; 4576 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 4577 struct trace_iterator *iter = filp->private_data; 4578 struct splice_pipe_desc spd = { 4579 .pages = pages_def, 4580 .partial = partial_def, 4581 .nr_pages = 0, /* This gets updated below. */ 4582 .nr_pages_max = PIPE_DEF_BUFFERS, 4583 .flags = flags, 4584 .ops = &tracing_pipe_buf_ops, 4585 .spd_release = tracing_spd_release_pipe, 4586 }; 4587 struct trace_array *tr = iter->tr; 4588 ssize_t ret; 4589 size_t rem; 4590 unsigned int i; 4591 4592 if (splice_grow_spd(pipe, &spd)) 4593 return -ENOMEM; 4594 4595 /* copy the tracer to avoid using a global lock all around */ 4596 mutex_lock(&trace_types_lock); 4597 if (unlikely(iter->trace->name != tr->current_trace->name)) 4598 *iter->trace = *tr->current_trace; 4599 mutex_unlock(&trace_types_lock); 4600 4601 mutex_lock(&iter->mutex); 4602 4603 if (iter->trace->splice_read) { 4604 ret = iter->trace->splice_read(iter, filp, 4605 ppos, pipe, len, flags); 4606 if (ret) 4607 goto out_err; 4608 } 4609 4610 ret = tracing_wait_pipe(filp); 4611 if (ret <= 0) 4612 goto out_err; 4613 4614 if (!iter->ent && !trace_find_next_entry_inc(iter)) { 4615 ret = -EFAULT; 4616 goto out_err; 4617 } 4618 4619 trace_event_read_lock(); 4620 trace_access_lock(iter->cpu_file); 4621 4622 /* Fill as many pages as possible. */ 4623 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) { 4624 spd.pages[i] = alloc_page(GFP_KERNEL); 4625 if (!spd.pages[i]) 4626 break; 4627 4628 rem = tracing_fill_pipe_page(rem, iter); 4629 4630 /* Copy the data into the page, so we can start over. */ 4631 ret = trace_seq_to_buffer(&iter->seq, 4632 page_address(spd.pages[i]), 4633 iter->seq.len); 4634 if (ret < 0) { 4635 __free_page(spd.pages[i]); 4636 break; 4637 } 4638 spd.partial[i].offset = 0; 4639 spd.partial[i].len = iter->seq.len; 4640 4641 trace_seq_init(&iter->seq); 4642 } 4643 4644 trace_access_unlock(iter->cpu_file); 4645 trace_event_read_unlock(); 4646 mutex_unlock(&iter->mutex); 4647 4648 spd.nr_pages = i; 4649 4650 ret = splice_to_pipe(pipe, &spd); 4651 out: 4652 splice_shrink_spd(&spd); 4653 return ret; 4654 4655 out_err: 4656 mutex_unlock(&iter->mutex); 4657 goto out; 4658 } 4659 4660 static ssize_t 4661 tracing_entries_read(struct file *filp, char __user *ubuf, 4662 size_t cnt, loff_t *ppos) 4663 { 4664 struct inode *inode = file_inode(filp); 4665 struct trace_array *tr = inode->i_private; 4666 int cpu = tracing_get_cpu(inode); 4667 char buf[64]; 4668 int r = 0; 4669 ssize_t ret; 4670 4671 mutex_lock(&trace_types_lock); 4672 4673 if (cpu == RING_BUFFER_ALL_CPUS) { 4674 int cpu, buf_size_same; 4675 unsigned long size; 4676 4677 size = 0; 4678 buf_size_same = 1; 4679 /* check if all cpu sizes are same */ 4680 for_each_tracing_cpu(cpu) { 4681 /* fill in the size from first enabled cpu */ 4682 if (size == 0) 4683 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries; 4684 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) { 4685 buf_size_same = 0; 4686 break; 4687 } 4688 } 4689 4690 if (buf_size_same) { 4691 if (!ring_buffer_expanded) 4692 r = sprintf(buf, "%lu (expanded: %lu)\n", 4693 size >> 10, 4694 trace_buf_size >> 10); 4695 else 4696 r = sprintf(buf, "%lu\n", size >> 10); 4697 } else 4698 r = sprintf(buf, "X\n"); 4699 } else 4700 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10); 4701 4702 mutex_unlock(&trace_types_lock); 4703 4704 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 4705 return ret; 4706 } 4707 4708 static ssize_t 4709 tracing_entries_write(struct file *filp, const char __user *ubuf, 4710 size_t cnt, loff_t *ppos) 4711 { 4712 struct inode *inode = file_inode(filp); 4713 struct trace_array *tr = inode->i_private; 4714 unsigned long val; 4715 int ret; 4716 4717 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 4718 if (ret) 4719 return ret; 4720 4721 /* must have at least 1 entry */ 4722 if (!val) 4723 return -EINVAL; 4724 4725 /* value is in KB */ 4726 val <<= 10; 4727 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode)); 4728 if (ret < 0) 4729 return ret; 4730 4731 *ppos += cnt; 4732 4733 return cnt; 4734 } 4735 4736 static ssize_t 4737 tracing_total_entries_read(struct file *filp, char __user *ubuf, 4738 size_t cnt, loff_t *ppos) 4739 { 4740 struct trace_array *tr = filp->private_data; 4741 char buf[64]; 4742 int r, cpu; 4743 unsigned long size = 0, expanded_size = 0; 4744 4745 mutex_lock(&trace_types_lock); 4746 for_each_tracing_cpu(cpu) { 4747 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10; 4748 if (!ring_buffer_expanded) 4749 expanded_size += trace_buf_size >> 10; 4750 } 4751 if (ring_buffer_expanded) 4752 r = sprintf(buf, "%lu\n", size); 4753 else 4754 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size); 4755 mutex_unlock(&trace_types_lock); 4756 4757 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 4758 } 4759 4760 static ssize_t 4761 tracing_free_buffer_write(struct file *filp, const char __user *ubuf, 4762 size_t cnt, loff_t *ppos) 4763 { 4764 /* 4765 * There is no need to read what the user has written, this function 4766 * is just to make sure that there is no error when "echo" is used 4767 */ 4768 4769 *ppos += cnt; 4770 4771 return cnt; 4772 } 4773 4774 static int 4775 tracing_free_buffer_release(struct inode *inode, struct file *filp) 4776 { 4777 struct trace_array *tr = inode->i_private; 4778 4779 /* disable tracing ? */ 4780 if (trace_flags & TRACE_ITER_STOP_ON_FREE) 4781 tracer_tracing_off(tr); 4782 /* resize the ring buffer to 0 */ 4783 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS); 4784 4785 trace_array_put(tr); 4786 4787 return 0; 4788 } 4789 4790 static ssize_t 4791 tracing_mark_write(struct file *filp, const char __user *ubuf, 4792 size_t cnt, loff_t *fpos) 4793 { 4794 unsigned long addr = (unsigned long)ubuf; 4795 struct trace_array *tr = filp->private_data; 4796 struct ring_buffer_event *event; 4797 struct ring_buffer *buffer; 4798 struct print_entry *entry; 4799 unsigned long irq_flags; 4800 struct page *pages[2]; 4801 void *map_page[2]; 4802 int nr_pages = 1; 4803 ssize_t written; 4804 int offset; 4805 int size; 4806 int len; 4807 int ret; 4808 int i; 4809 4810 if (tracing_disabled) 4811 return -EINVAL; 4812 4813 if (!(trace_flags & TRACE_ITER_MARKERS)) 4814 return -EINVAL; 4815 4816 if (cnt > TRACE_BUF_SIZE) 4817 cnt = TRACE_BUF_SIZE; 4818 4819 /* 4820 * Userspace is injecting traces into the kernel trace buffer. 4821 * We want to be as non intrusive as possible. 4822 * To do so, we do not want to allocate any special buffers 4823 * or take any locks, but instead write the userspace data 4824 * straight into the ring buffer. 4825 * 4826 * First we need to pin the userspace buffer into memory, 4827 * which, most likely it is, because it just referenced it. 4828 * But there's no guarantee that it is. By using get_user_pages_fast() 4829 * and kmap_atomic/kunmap_atomic() we can get access to the 4830 * pages directly. We then write the data directly into the 4831 * ring buffer. 4832 */ 4833 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); 4834 4835 /* check if we cross pages */ 4836 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK)) 4837 nr_pages = 2; 4838 4839 offset = addr & (PAGE_SIZE - 1); 4840 addr &= PAGE_MASK; 4841 4842 ret = get_user_pages_fast(addr, nr_pages, 0, pages); 4843 if (ret < nr_pages) { 4844 while (--ret >= 0) 4845 put_page(pages[ret]); 4846 written = -EFAULT; 4847 goto out; 4848 } 4849 4850 for (i = 0; i < nr_pages; i++) 4851 map_page[i] = kmap_atomic(pages[i]); 4852 4853 local_save_flags(irq_flags); 4854 size = sizeof(*entry) + cnt + 2; /* possible \n added */ 4855 buffer = tr->trace_buffer.buffer; 4856 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 4857 irq_flags, preempt_count()); 4858 if (!event) { 4859 /* Ring buffer disabled, return as if not open for write */ 4860 written = -EBADF; 4861 goto out_unlock; 4862 } 4863 4864 entry = ring_buffer_event_data(event); 4865 entry->ip = _THIS_IP_; 4866 4867 if (nr_pages == 2) { 4868 len = PAGE_SIZE - offset; 4869 memcpy(&entry->buf, map_page[0] + offset, len); 4870 memcpy(&entry->buf[len], map_page[1], cnt - len); 4871 } else 4872 memcpy(&entry->buf, map_page[0] + offset, cnt); 4873 4874 if (entry->buf[cnt - 1] != '\n') { 4875 entry->buf[cnt] = '\n'; 4876 entry->buf[cnt + 1] = '\0'; 4877 } else 4878 entry->buf[cnt] = '\0'; 4879 4880 __buffer_unlock_commit(buffer, event); 4881 4882 written = cnt; 4883 4884 *fpos += written; 4885 4886 out_unlock: 4887 for (i = 0; i < nr_pages; i++){ 4888 kunmap_atomic(map_page[i]); 4889 put_page(pages[i]); 4890 } 4891 out: 4892 return written; 4893 } 4894 4895 static int tracing_clock_show(struct seq_file *m, void *v) 4896 { 4897 struct trace_array *tr = m->private; 4898 int i; 4899 4900 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) 4901 seq_printf(m, 4902 "%s%s%s%s", i ? " " : "", 4903 i == tr->clock_id ? "[" : "", trace_clocks[i].name, 4904 i == tr->clock_id ? "]" : ""); 4905 seq_putc(m, '\n'); 4906 4907 return 0; 4908 } 4909 4910 static int tracing_set_clock(struct trace_array *tr, const char *clockstr) 4911 { 4912 int i; 4913 4914 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) { 4915 if (strcmp(trace_clocks[i].name, clockstr) == 0) 4916 break; 4917 } 4918 if (i == ARRAY_SIZE(trace_clocks)) 4919 return -EINVAL; 4920 4921 mutex_lock(&trace_types_lock); 4922 4923 tr->clock_id = i; 4924 4925 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func); 4926 4927 /* 4928 * New clock may not be consistent with the previous clock. 4929 * Reset the buffer so that it doesn't have incomparable timestamps. 4930 */ 4931 tracing_reset_online_cpus(&tr->trace_buffer); 4932 4933 #ifdef CONFIG_TRACER_MAX_TRACE 4934 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer) 4935 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func); 4936 tracing_reset_online_cpus(&tr->max_buffer); 4937 #endif 4938 4939 mutex_unlock(&trace_types_lock); 4940 4941 return 0; 4942 } 4943 4944 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf, 4945 size_t cnt, loff_t *fpos) 4946 { 4947 struct seq_file *m = filp->private_data; 4948 struct trace_array *tr = m->private; 4949 char buf[64]; 4950 const char *clockstr; 4951 int ret; 4952 4953 if (cnt >= sizeof(buf)) 4954 return -EINVAL; 4955 4956 if (copy_from_user(&buf, ubuf, cnt)) 4957 return -EFAULT; 4958 4959 buf[cnt] = 0; 4960 4961 clockstr = strstrip(buf); 4962 4963 ret = tracing_set_clock(tr, clockstr); 4964 if (ret) 4965 return ret; 4966 4967 *fpos += cnt; 4968 4969 return cnt; 4970 } 4971 4972 static int tracing_clock_open(struct inode *inode, struct file *file) 4973 { 4974 struct trace_array *tr = inode->i_private; 4975 int ret; 4976 4977 if (tracing_disabled) 4978 return -ENODEV; 4979 4980 if (trace_array_get(tr)) 4981 return -ENODEV; 4982 4983 ret = single_open(file, tracing_clock_show, inode->i_private); 4984 if (ret < 0) 4985 trace_array_put(tr); 4986 4987 return ret; 4988 } 4989 4990 struct ftrace_buffer_info { 4991 struct trace_iterator iter; 4992 void *spare; 4993 unsigned int read; 4994 }; 4995 4996 #ifdef CONFIG_TRACER_SNAPSHOT 4997 static int tracing_snapshot_open(struct inode *inode, struct file *file) 4998 { 4999 struct trace_array *tr = inode->i_private; 5000 struct trace_iterator *iter; 5001 struct seq_file *m; 5002 int ret = 0; 5003 5004 if (trace_array_get(tr) < 0) 5005 return -ENODEV; 5006 5007 if (file->f_mode & FMODE_READ) { 5008 iter = __tracing_open(inode, file, true); 5009 if (IS_ERR(iter)) 5010 ret = PTR_ERR(iter); 5011 } else { 5012 /* Writes still need the seq_file to hold the private data */ 5013 ret = -ENOMEM; 5014 m = kzalloc(sizeof(*m), GFP_KERNEL); 5015 if (!m) 5016 goto out; 5017 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 5018 if (!iter) { 5019 kfree(m); 5020 goto out; 5021 } 5022 ret = 0; 5023 5024 iter->tr = tr; 5025 iter->trace_buffer = &tr->max_buffer; 5026 iter->cpu_file = tracing_get_cpu(inode); 5027 m->private = iter; 5028 file->private_data = m; 5029 } 5030 out: 5031 if (ret < 0) 5032 trace_array_put(tr); 5033 5034 return ret; 5035 } 5036 5037 static ssize_t 5038 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt, 5039 loff_t *ppos) 5040 { 5041 struct seq_file *m = filp->private_data; 5042 struct trace_iterator *iter = m->private; 5043 struct trace_array *tr = iter->tr; 5044 unsigned long val; 5045 int ret; 5046 5047 ret = tracing_update_buffers(); 5048 if (ret < 0) 5049 return ret; 5050 5051 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 5052 if (ret) 5053 return ret; 5054 5055 mutex_lock(&trace_types_lock); 5056 5057 if (tr->current_trace->use_max_tr) { 5058 ret = -EBUSY; 5059 goto out; 5060 } 5061 5062 switch (val) { 5063 case 0: 5064 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 5065 ret = -EINVAL; 5066 break; 5067 } 5068 if (tr->allocated_snapshot) 5069 free_snapshot(tr); 5070 break; 5071 case 1: 5072 /* Only allow per-cpu swap if the ring buffer supports it */ 5073 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP 5074 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 5075 ret = -EINVAL; 5076 break; 5077 } 5078 #endif 5079 if (!tr->allocated_snapshot) { 5080 ret = alloc_snapshot(tr); 5081 if (ret < 0) 5082 break; 5083 } 5084 local_irq_disable(); 5085 /* Now, we're going to swap */ 5086 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 5087 update_max_tr(tr, current, smp_processor_id()); 5088 else 5089 update_max_tr_single(tr, current, iter->cpu_file); 5090 local_irq_enable(); 5091 break; 5092 default: 5093 if (tr->allocated_snapshot) { 5094 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 5095 tracing_reset_online_cpus(&tr->max_buffer); 5096 else 5097 tracing_reset(&tr->max_buffer, iter->cpu_file); 5098 } 5099 break; 5100 } 5101 5102 if (ret >= 0) { 5103 *ppos += cnt; 5104 ret = cnt; 5105 } 5106 out: 5107 mutex_unlock(&trace_types_lock); 5108 return ret; 5109 } 5110 5111 static int tracing_snapshot_release(struct inode *inode, struct file *file) 5112 { 5113 struct seq_file *m = file->private_data; 5114 int ret; 5115 5116 ret = tracing_release(inode, file); 5117 5118 if (file->f_mode & FMODE_READ) 5119 return ret; 5120 5121 /* If write only, the seq_file is just a stub */ 5122 if (m) 5123 kfree(m->private); 5124 kfree(m); 5125 5126 return 0; 5127 } 5128 5129 static int tracing_buffers_open(struct inode *inode, struct file *filp); 5130 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, 5131 size_t count, loff_t *ppos); 5132 static int tracing_buffers_release(struct inode *inode, struct file *file); 5133 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos, 5134 struct pipe_inode_info *pipe, size_t len, unsigned int flags); 5135 5136 static int snapshot_raw_open(struct inode *inode, struct file *filp) 5137 { 5138 struct ftrace_buffer_info *info; 5139 int ret; 5140 5141 ret = tracing_buffers_open(inode, filp); 5142 if (ret < 0) 5143 return ret; 5144 5145 info = filp->private_data; 5146 5147 if (info->iter.trace->use_max_tr) { 5148 tracing_buffers_release(inode, filp); 5149 return -EBUSY; 5150 } 5151 5152 info->iter.snapshot = true; 5153 info->iter.trace_buffer = &info->iter.tr->max_buffer; 5154 5155 return ret; 5156 } 5157 5158 #endif /* CONFIG_TRACER_SNAPSHOT */ 5159 5160 5161 static const struct file_operations tracing_max_lat_fops = { 5162 .open = tracing_open_generic, 5163 .read = tracing_max_lat_read, 5164 .write = tracing_max_lat_write, 5165 .llseek = generic_file_llseek, 5166 }; 5167 5168 static const struct file_operations set_tracer_fops = { 5169 .open = tracing_open_generic, 5170 .read = tracing_set_trace_read, 5171 .write = tracing_set_trace_write, 5172 .llseek = generic_file_llseek, 5173 }; 5174 5175 static const struct file_operations tracing_pipe_fops = { 5176 .open = tracing_open_pipe, 5177 .poll = tracing_poll_pipe, 5178 .read = tracing_read_pipe, 5179 .splice_read = tracing_splice_read_pipe, 5180 .release = tracing_release_pipe, 5181 .llseek = no_llseek, 5182 }; 5183 5184 static const struct file_operations tracing_entries_fops = { 5185 .open = tracing_open_generic_tr, 5186 .read = tracing_entries_read, 5187 .write = tracing_entries_write, 5188 .llseek = generic_file_llseek, 5189 .release = tracing_release_generic_tr, 5190 }; 5191 5192 static const struct file_operations tracing_total_entries_fops = { 5193 .open = tracing_open_generic_tr, 5194 .read = tracing_total_entries_read, 5195 .llseek = generic_file_llseek, 5196 .release = tracing_release_generic_tr, 5197 }; 5198 5199 static const struct file_operations tracing_free_buffer_fops = { 5200 .open = tracing_open_generic_tr, 5201 .write = tracing_free_buffer_write, 5202 .release = tracing_free_buffer_release, 5203 }; 5204 5205 static const struct file_operations tracing_mark_fops = { 5206 .open = tracing_open_generic_tr, 5207 .write = tracing_mark_write, 5208 .llseek = generic_file_llseek, 5209 .release = tracing_release_generic_tr, 5210 }; 5211 5212 static const struct file_operations trace_clock_fops = { 5213 .open = tracing_clock_open, 5214 .read = seq_read, 5215 .llseek = seq_lseek, 5216 .release = tracing_single_release_tr, 5217 .write = tracing_clock_write, 5218 }; 5219 5220 #ifdef CONFIG_TRACER_SNAPSHOT 5221 static const struct file_operations snapshot_fops = { 5222 .open = tracing_snapshot_open, 5223 .read = seq_read, 5224 .write = tracing_snapshot_write, 5225 .llseek = tracing_lseek, 5226 .release = tracing_snapshot_release, 5227 }; 5228 5229 static const struct file_operations snapshot_raw_fops = { 5230 .open = snapshot_raw_open, 5231 .read = tracing_buffers_read, 5232 .release = tracing_buffers_release, 5233 .splice_read = tracing_buffers_splice_read, 5234 .llseek = no_llseek, 5235 }; 5236 5237 #endif /* CONFIG_TRACER_SNAPSHOT */ 5238 5239 static int tracing_buffers_open(struct inode *inode, struct file *filp) 5240 { 5241 struct trace_array *tr = inode->i_private; 5242 struct ftrace_buffer_info *info; 5243 int ret; 5244 5245 if (tracing_disabled) 5246 return -ENODEV; 5247 5248 if (trace_array_get(tr) < 0) 5249 return -ENODEV; 5250 5251 info = kzalloc(sizeof(*info), GFP_KERNEL); 5252 if (!info) { 5253 trace_array_put(tr); 5254 return -ENOMEM; 5255 } 5256 5257 mutex_lock(&trace_types_lock); 5258 5259 info->iter.tr = tr; 5260 info->iter.cpu_file = tracing_get_cpu(inode); 5261 info->iter.trace = tr->current_trace; 5262 info->iter.trace_buffer = &tr->trace_buffer; 5263 info->spare = NULL; 5264 /* Force reading ring buffer for first read */ 5265 info->read = (unsigned int)-1; 5266 5267 filp->private_data = info; 5268 5269 mutex_unlock(&trace_types_lock); 5270 5271 ret = nonseekable_open(inode, filp); 5272 if (ret < 0) 5273 trace_array_put(tr); 5274 5275 return ret; 5276 } 5277 5278 static unsigned int 5279 tracing_buffers_poll(struct file *filp, poll_table *poll_table) 5280 { 5281 struct ftrace_buffer_info *info = filp->private_data; 5282 struct trace_iterator *iter = &info->iter; 5283 5284 return trace_poll(iter, filp, poll_table); 5285 } 5286 5287 static ssize_t 5288 tracing_buffers_read(struct file *filp, char __user *ubuf, 5289 size_t count, loff_t *ppos) 5290 { 5291 struct ftrace_buffer_info *info = filp->private_data; 5292 struct trace_iterator *iter = &info->iter; 5293 ssize_t ret; 5294 ssize_t size; 5295 5296 if (!count) 5297 return 0; 5298 5299 mutex_lock(&trace_types_lock); 5300 5301 #ifdef CONFIG_TRACER_MAX_TRACE 5302 if (iter->snapshot && iter->tr->current_trace->use_max_tr) { 5303 size = -EBUSY; 5304 goto out_unlock; 5305 } 5306 #endif 5307 5308 if (!info->spare) 5309 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer, 5310 iter->cpu_file); 5311 size = -ENOMEM; 5312 if (!info->spare) 5313 goto out_unlock; 5314 5315 /* Do we have previous read data to read? */ 5316 if (info->read < PAGE_SIZE) 5317 goto read; 5318 5319 again: 5320 trace_access_lock(iter->cpu_file); 5321 ret = ring_buffer_read_page(iter->trace_buffer->buffer, 5322 &info->spare, 5323 count, 5324 iter->cpu_file, 0); 5325 trace_access_unlock(iter->cpu_file); 5326 5327 if (ret < 0) { 5328 if (trace_empty(iter)) { 5329 if ((filp->f_flags & O_NONBLOCK)) { 5330 size = -EAGAIN; 5331 goto out_unlock; 5332 } 5333 mutex_unlock(&trace_types_lock); 5334 ret = wait_on_pipe(iter); 5335 mutex_lock(&trace_types_lock); 5336 if (ret) { 5337 size = ret; 5338 goto out_unlock; 5339 } 5340 if (signal_pending(current)) { 5341 size = -EINTR; 5342 goto out_unlock; 5343 } 5344 goto again; 5345 } 5346 size = 0; 5347 goto out_unlock; 5348 } 5349 5350 info->read = 0; 5351 read: 5352 size = PAGE_SIZE - info->read; 5353 if (size > count) 5354 size = count; 5355 5356 ret = copy_to_user(ubuf, info->spare + info->read, size); 5357 if (ret == size) { 5358 size = -EFAULT; 5359 goto out_unlock; 5360 } 5361 size -= ret; 5362 5363 *ppos += size; 5364 info->read += size; 5365 5366 out_unlock: 5367 mutex_unlock(&trace_types_lock); 5368 5369 return size; 5370 } 5371 5372 static int tracing_buffers_release(struct inode *inode, struct file *file) 5373 { 5374 struct ftrace_buffer_info *info = file->private_data; 5375 struct trace_iterator *iter = &info->iter; 5376 5377 mutex_lock(&trace_types_lock); 5378 5379 __trace_array_put(iter->tr); 5380 5381 if (info->spare) 5382 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare); 5383 kfree(info); 5384 5385 mutex_unlock(&trace_types_lock); 5386 5387 return 0; 5388 } 5389 5390 struct buffer_ref { 5391 struct ring_buffer *buffer; 5392 void *page; 5393 int ref; 5394 }; 5395 5396 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe, 5397 struct pipe_buffer *buf) 5398 { 5399 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 5400 5401 if (--ref->ref) 5402 return; 5403 5404 ring_buffer_free_read_page(ref->buffer, ref->page); 5405 kfree(ref); 5406 buf->private = 0; 5407 } 5408 5409 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe, 5410 struct pipe_buffer *buf) 5411 { 5412 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 5413 5414 ref->ref++; 5415 } 5416 5417 /* Pipe buffer operations for a buffer. */ 5418 static const struct pipe_buf_operations buffer_pipe_buf_ops = { 5419 .can_merge = 0, 5420 .confirm = generic_pipe_buf_confirm, 5421 .release = buffer_pipe_buf_release, 5422 .steal = generic_pipe_buf_steal, 5423 .get = buffer_pipe_buf_get, 5424 }; 5425 5426 /* 5427 * Callback from splice_to_pipe(), if we need to release some pages 5428 * at the end of the spd in case we error'ed out in filling the pipe. 5429 */ 5430 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i) 5431 { 5432 struct buffer_ref *ref = 5433 (struct buffer_ref *)spd->partial[i].private; 5434 5435 if (--ref->ref) 5436 return; 5437 5438 ring_buffer_free_read_page(ref->buffer, ref->page); 5439 kfree(ref); 5440 spd->partial[i].private = 0; 5441 } 5442 5443 static ssize_t 5444 tracing_buffers_splice_read(struct file *file, loff_t *ppos, 5445 struct pipe_inode_info *pipe, size_t len, 5446 unsigned int flags) 5447 { 5448 struct ftrace_buffer_info *info = file->private_data; 5449 struct trace_iterator *iter = &info->iter; 5450 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 5451 struct page *pages_def[PIPE_DEF_BUFFERS]; 5452 struct splice_pipe_desc spd = { 5453 .pages = pages_def, 5454 .partial = partial_def, 5455 .nr_pages_max = PIPE_DEF_BUFFERS, 5456 .flags = flags, 5457 .ops = &buffer_pipe_buf_ops, 5458 .spd_release = buffer_spd_release, 5459 }; 5460 struct buffer_ref *ref; 5461 int entries, size, i; 5462 ssize_t ret; 5463 5464 mutex_lock(&trace_types_lock); 5465 5466 #ifdef CONFIG_TRACER_MAX_TRACE 5467 if (iter->snapshot && iter->tr->current_trace->use_max_tr) { 5468 ret = -EBUSY; 5469 goto out; 5470 } 5471 #endif 5472 5473 if (splice_grow_spd(pipe, &spd)) { 5474 ret = -ENOMEM; 5475 goto out; 5476 } 5477 5478 if (*ppos & (PAGE_SIZE - 1)) { 5479 ret = -EINVAL; 5480 goto out; 5481 } 5482 5483 if (len & (PAGE_SIZE - 1)) { 5484 if (len < PAGE_SIZE) { 5485 ret = -EINVAL; 5486 goto out; 5487 } 5488 len &= PAGE_MASK; 5489 } 5490 5491 again: 5492 trace_access_lock(iter->cpu_file); 5493 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file); 5494 5495 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) { 5496 struct page *page; 5497 int r; 5498 5499 ref = kzalloc(sizeof(*ref), GFP_KERNEL); 5500 if (!ref) 5501 break; 5502 5503 ref->ref = 1; 5504 ref->buffer = iter->trace_buffer->buffer; 5505 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file); 5506 if (!ref->page) { 5507 kfree(ref); 5508 break; 5509 } 5510 5511 r = ring_buffer_read_page(ref->buffer, &ref->page, 5512 len, iter->cpu_file, 1); 5513 if (r < 0) { 5514 ring_buffer_free_read_page(ref->buffer, ref->page); 5515 kfree(ref); 5516 break; 5517 } 5518 5519 /* 5520 * zero out any left over data, this is going to 5521 * user land. 5522 */ 5523 size = ring_buffer_page_len(ref->page); 5524 if (size < PAGE_SIZE) 5525 memset(ref->page + size, 0, PAGE_SIZE - size); 5526 5527 page = virt_to_page(ref->page); 5528 5529 spd.pages[i] = page; 5530 spd.partial[i].len = PAGE_SIZE; 5531 spd.partial[i].offset = 0; 5532 spd.partial[i].private = (unsigned long)ref; 5533 spd.nr_pages++; 5534 *ppos += PAGE_SIZE; 5535 5536 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file); 5537 } 5538 5539 trace_access_unlock(iter->cpu_file); 5540 spd.nr_pages = i; 5541 5542 /* did we read anything? */ 5543 if (!spd.nr_pages) { 5544 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) { 5545 ret = -EAGAIN; 5546 goto out; 5547 } 5548 mutex_unlock(&trace_types_lock); 5549 ret = wait_on_pipe(iter); 5550 mutex_lock(&trace_types_lock); 5551 if (ret) 5552 goto out; 5553 if (signal_pending(current)) { 5554 ret = -EINTR; 5555 goto out; 5556 } 5557 goto again; 5558 } 5559 5560 ret = splice_to_pipe(pipe, &spd); 5561 splice_shrink_spd(&spd); 5562 out: 5563 mutex_unlock(&trace_types_lock); 5564 5565 return ret; 5566 } 5567 5568 static const struct file_operations tracing_buffers_fops = { 5569 .open = tracing_buffers_open, 5570 .read = tracing_buffers_read, 5571 .poll = tracing_buffers_poll, 5572 .release = tracing_buffers_release, 5573 .splice_read = tracing_buffers_splice_read, 5574 .llseek = no_llseek, 5575 }; 5576 5577 static ssize_t 5578 tracing_stats_read(struct file *filp, char __user *ubuf, 5579 size_t count, loff_t *ppos) 5580 { 5581 struct inode *inode = file_inode(filp); 5582 struct trace_array *tr = inode->i_private; 5583 struct trace_buffer *trace_buf = &tr->trace_buffer; 5584 int cpu = tracing_get_cpu(inode); 5585 struct trace_seq *s; 5586 unsigned long cnt; 5587 unsigned long long t; 5588 unsigned long usec_rem; 5589 5590 s = kmalloc(sizeof(*s), GFP_KERNEL); 5591 if (!s) 5592 return -ENOMEM; 5593 5594 trace_seq_init(s); 5595 5596 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu); 5597 trace_seq_printf(s, "entries: %ld\n", cnt); 5598 5599 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu); 5600 trace_seq_printf(s, "overrun: %ld\n", cnt); 5601 5602 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu); 5603 trace_seq_printf(s, "commit overrun: %ld\n", cnt); 5604 5605 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu); 5606 trace_seq_printf(s, "bytes: %ld\n", cnt); 5607 5608 if (trace_clocks[tr->clock_id].in_ns) { 5609 /* local or global for trace_clock */ 5610 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); 5611 usec_rem = do_div(t, USEC_PER_SEC); 5612 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", 5613 t, usec_rem); 5614 5615 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu)); 5616 usec_rem = do_div(t, USEC_PER_SEC); 5617 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem); 5618 } else { 5619 /* counter or tsc mode for trace_clock */ 5620 trace_seq_printf(s, "oldest event ts: %llu\n", 5621 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); 5622 5623 trace_seq_printf(s, "now ts: %llu\n", 5624 ring_buffer_time_stamp(trace_buf->buffer, cpu)); 5625 } 5626 5627 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu); 5628 trace_seq_printf(s, "dropped events: %ld\n", cnt); 5629 5630 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu); 5631 trace_seq_printf(s, "read events: %ld\n", cnt); 5632 5633 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len); 5634 5635 kfree(s); 5636 5637 return count; 5638 } 5639 5640 static const struct file_operations tracing_stats_fops = { 5641 .open = tracing_open_generic_tr, 5642 .read = tracing_stats_read, 5643 .llseek = generic_file_llseek, 5644 .release = tracing_release_generic_tr, 5645 }; 5646 5647 #ifdef CONFIG_DYNAMIC_FTRACE 5648 5649 int __weak ftrace_arch_read_dyn_info(char *buf, int size) 5650 { 5651 return 0; 5652 } 5653 5654 static ssize_t 5655 tracing_read_dyn_info(struct file *filp, char __user *ubuf, 5656 size_t cnt, loff_t *ppos) 5657 { 5658 static char ftrace_dyn_info_buffer[1024]; 5659 static DEFINE_MUTEX(dyn_info_mutex); 5660 unsigned long *p = filp->private_data; 5661 char *buf = ftrace_dyn_info_buffer; 5662 int size = ARRAY_SIZE(ftrace_dyn_info_buffer); 5663 int r; 5664 5665 mutex_lock(&dyn_info_mutex); 5666 r = sprintf(buf, "%ld ", *p); 5667 5668 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r); 5669 buf[r++] = '\n'; 5670 5671 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5672 5673 mutex_unlock(&dyn_info_mutex); 5674 5675 return r; 5676 } 5677 5678 static const struct file_operations tracing_dyn_info_fops = { 5679 .open = tracing_open_generic, 5680 .read = tracing_read_dyn_info, 5681 .llseek = generic_file_llseek, 5682 }; 5683 #endif /* CONFIG_DYNAMIC_FTRACE */ 5684 5685 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) 5686 static void 5687 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data) 5688 { 5689 tracing_snapshot(); 5690 } 5691 5692 static void 5693 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data) 5694 { 5695 unsigned long *count = (long *)data; 5696 5697 if (!*count) 5698 return; 5699 5700 if (*count != -1) 5701 (*count)--; 5702 5703 tracing_snapshot(); 5704 } 5705 5706 static int 5707 ftrace_snapshot_print(struct seq_file *m, unsigned long ip, 5708 struct ftrace_probe_ops *ops, void *data) 5709 { 5710 long count = (long)data; 5711 5712 seq_printf(m, "%ps:", (void *)ip); 5713 5714 seq_printf(m, "snapshot"); 5715 5716 if (count == -1) 5717 seq_printf(m, ":unlimited\n"); 5718 else 5719 seq_printf(m, ":count=%ld\n", count); 5720 5721 return 0; 5722 } 5723 5724 static struct ftrace_probe_ops snapshot_probe_ops = { 5725 .func = ftrace_snapshot, 5726 .print = ftrace_snapshot_print, 5727 }; 5728 5729 static struct ftrace_probe_ops snapshot_count_probe_ops = { 5730 .func = ftrace_count_snapshot, 5731 .print = ftrace_snapshot_print, 5732 }; 5733 5734 static int 5735 ftrace_trace_snapshot_callback(struct ftrace_hash *hash, 5736 char *glob, char *cmd, char *param, int enable) 5737 { 5738 struct ftrace_probe_ops *ops; 5739 void *count = (void *)-1; 5740 char *number; 5741 int ret; 5742 5743 /* hash funcs only work with set_ftrace_filter */ 5744 if (!enable) 5745 return -EINVAL; 5746 5747 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops; 5748 5749 if (glob[0] == '!') { 5750 unregister_ftrace_function_probe_func(glob+1, ops); 5751 return 0; 5752 } 5753 5754 if (!param) 5755 goto out_reg; 5756 5757 number = strsep(¶m, ":"); 5758 5759 if (!strlen(number)) 5760 goto out_reg; 5761 5762 /* 5763 * We use the callback data field (which is a pointer) 5764 * as our counter. 5765 */ 5766 ret = kstrtoul(number, 0, (unsigned long *)&count); 5767 if (ret) 5768 return ret; 5769 5770 out_reg: 5771 ret = register_ftrace_function_probe(glob, ops, count); 5772 5773 if (ret >= 0) 5774 alloc_snapshot(&global_trace); 5775 5776 return ret < 0 ? ret : 0; 5777 } 5778 5779 static struct ftrace_func_command ftrace_snapshot_cmd = { 5780 .name = "snapshot", 5781 .func = ftrace_trace_snapshot_callback, 5782 }; 5783 5784 static __init int register_snapshot_cmd(void) 5785 { 5786 return register_ftrace_command(&ftrace_snapshot_cmd); 5787 } 5788 #else 5789 static inline __init int register_snapshot_cmd(void) { return 0; } 5790 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */ 5791 5792 struct dentry *tracing_init_dentry_tr(struct trace_array *tr) 5793 { 5794 if (tr->dir) 5795 return tr->dir; 5796 5797 if (!debugfs_initialized()) 5798 return NULL; 5799 5800 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 5801 tr->dir = debugfs_create_dir("tracing", NULL); 5802 5803 if (!tr->dir) 5804 pr_warn_once("Could not create debugfs directory 'tracing'\n"); 5805 5806 return tr->dir; 5807 } 5808 5809 struct dentry *tracing_init_dentry(void) 5810 { 5811 return tracing_init_dentry_tr(&global_trace); 5812 } 5813 5814 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu) 5815 { 5816 struct dentry *d_tracer; 5817 5818 if (tr->percpu_dir) 5819 return tr->percpu_dir; 5820 5821 d_tracer = tracing_init_dentry_tr(tr); 5822 if (!d_tracer) 5823 return NULL; 5824 5825 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer); 5826 5827 WARN_ONCE(!tr->percpu_dir, 5828 "Could not create debugfs directory 'per_cpu/%d'\n", cpu); 5829 5830 return tr->percpu_dir; 5831 } 5832 5833 static struct dentry * 5834 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent, 5835 void *data, long cpu, const struct file_operations *fops) 5836 { 5837 struct dentry *ret = trace_create_file(name, mode, parent, data, fops); 5838 5839 if (ret) /* See tracing_get_cpu() */ 5840 ret->d_inode->i_cdev = (void *)(cpu + 1); 5841 return ret; 5842 } 5843 5844 static void 5845 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu) 5846 { 5847 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu); 5848 struct dentry *d_cpu; 5849 char cpu_dir[30]; /* 30 characters should be more than enough */ 5850 5851 if (!d_percpu) 5852 return; 5853 5854 snprintf(cpu_dir, 30, "cpu%ld", cpu); 5855 d_cpu = debugfs_create_dir(cpu_dir, d_percpu); 5856 if (!d_cpu) { 5857 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir); 5858 return; 5859 } 5860 5861 /* per cpu trace_pipe */ 5862 trace_create_cpu_file("trace_pipe", 0444, d_cpu, 5863 tr, cpu, &tracing_pipe_fops); 5864 5865 /* per cpu trace */ 5866 trace_create_cpu_file("trace", 0644, d_cpu, 5867 tr, cpu, &tracing_fops); 5868 5869 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu, 5870 tr, cpu, &tracing_buffers_fops); 5871 5872 trace_create_cpu_file("stats", 0444, d_cpu, 5873 tr, cpu, &tracing_stats_fops); 5874 5875 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu, 5876 tr, cpu, &tracing_entries_fops); 5877 5878 #ifdef CONFIG_TRACER_SNAPSHOT 5879 trace_create_cpu_file("snapshot", 0644, d_cpu, 5880 tr, cpu, &snapshot_fops); 5881 5882 trace_create_cpu_file("snapshot_raw", 0444, d_cpu, 5883 tr, cpu, &snapshot_raw_fops); 5884 #endif 5885 } 5886 5887 #ifdef CONFIG_FTRACE_SELFTEST 5888 /* Let selftest have access to static functions in this file */ 5889 #include "trace_selftest.c" 5890 #endif 5891 5892 struct trace_option_dentry { 5893 struct tracer_opt *opt; 5894 struct tracer_flags *flags; 5895 struct trace_array *tr; 5896 struct dentry *entry; 5897 }; 5898 5899 static ssize_t 5900 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt, 5901 loff_t *ppos) 5902 { 5903 struct trace_option_dentry *topt = filp->private_data; 5904 char *buf; 5905 5906 if (topt->flags->val & topt->opt->bit) 5907 buf = "1\n"; 5908 else 5909 buf = "0\n"; 5910 5911 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 5912 } 5913 5914 static ssize_t 5915 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt, 5916 loff_t *ppos) 5917 { 5918 struct trace_option_dentry *topt = filp->private_data; 5919 unsigned long val; 5920 int ret; 5921 5922 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 5923 if (ret) 5924 return ret; 5925 5926 if (val != 0 && val != 1) 5927 return -EINVAL; 5928 5929 if (!!(topt->flags->val & topt->opt->bit) != val) { 5930 mutex_lock(&trace_types_lock); 5931 ret = __set_tracer_option(topt->tr, topt->flags, 5932 topt->opt, !val); 5933 mutex_unlock(&trace_types_lock); 5934 if (ret) 5935 return ret; 5936 } 5937 5938 *ppos += cnt; 5939 5940 return cnt; 5941 } 5942 5943 5944 static const struct file_operations trace_options_fops = { 5945 .open = tracing_open_generic, 5946 .read = trace_options_read, 5947 .write = trace_options_write, 5948 .llseek = generic_file_llseek, 5949 }; 5950 5951 static ssize_t 5952 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt, 5953 loff_t *ppos) 5954 { 5955 long index = (long)filp->private_data; 5956 char *buf; 5957 5958 if (trace_flags & (1 << index)) 5959 buf = "1\n"; 5960 else 5961 buf = "0\n"; 5962 5963 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 5964 } 5965 5966 static ssize_t 5967 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt, 5968 loff_t *ppos) 5969 { 5970 struct trace_array *tr = &global_trace; 5971 long index = (long)filp->private_data; 5972 unsigned long val; 5973 int ret; 5974 5975 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 5976 if (ret) 5977 return ret; 5978 5979 if (val != 0 && val != 1) 5980 return -EINVAL; 5981 5982 mutex_lock(&trace_types_lock); 5983 ret = set_tracer_flag(tr, 1 << index, val); 5984 mutex_unlock(&trace_types_lock); 5985 5986 if (ret < 0) 5987 return ret; 5988 5989 *ppos += cnt; 5990 5991 return cnt; 5992 } 5993 5994 static const struct file_operations trace_options_core_fops = { 5995 .open = tracing_open_generic, 5996 .read = trace_options_core_read, 5997 .write = trace_options_core_write, 5998 .llseek = generic_file_llseek, 5999 }; 6000 6001 struct dentry *trace_create_file(const char *name, 6002 umode_t mode, 6003 struct dentry *parent, 6004 void *data, 6005 const struct file_operations *fops) 6006 { 6007 struct dentry *ret; 6008 6009 ret = debugfs_create_file(name, mode, parent, data, fops); 6010 if (!ret) 6011 pr_warning("Could not create debugfs '%s' entry\n", name); 6012 6013 return ret; 6014 } 6015 6016 6017 static struct dentry *trace_options_init_dentry(struct trace_array *tr) 6018 { 6019 struct dentry *d_tracer; 6020 6021 if (tr->options) 6022 return tr->options; 6023 6024 d_tracer = tracing_init_dentry_tr(tr); 6025 if (!d_tracer) 6026 return NULL; 6027 6028 tr->options = debugfs_create_dir("options", d_tracer); 6029 if (!tr->options) { 6030 pr_warning("Could not create debugfs directory 'options'\n"); 6031 return NULL; 6032 } 6033 6034 return tr->options; 6035 } 6036 6037 static void 6038 create_trace_option_file(struct trace_array *tr, 6039 struct trace_option_dentry *topt, 6040 struct tracer_flags *flags, 6041 struct tracer_opt *opt) 6042 { 6043 struct dentry *t_options; 6044 6045 t_options = trace_options_init_dentry(tr); 6046 if (!t_options) 6047 return; 6048 6049 topt->flags = flags; 6050 topt->opt = opt; 6051 topt->tr = tr; 6052 6053 topt->entry = trace_create_file(opt->name, 0644, t_options, topt, 6054 &trace_options_fops); 6055 6056 } 6057 6058 static struct trace_option_dentry * 6059 create_trace_option_files(struct trace_array *tr, struct tracer *tracer) 6060 { 6061 struct trace_option_dentry *topts; 6062 struct tracer_flags *flags; 6063 struct tracer_opt *opts; 6064 int cnt; 6065 6066 if (!tracer) 6067 return NULL; 6068 6069 flags = tracer->flags; 6070 6071 if (!flags || !flags->opts) 6072 return NULL; 6073 6074 opts = flags->opts; 6075 6076 for (cnt = 0; opts[cnt].name; cnt++) 6077 ; 6078 6079 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL); 6080 if (!topts) 6081 return NULL; 6082 6083 for (cnt = 0; opts[cnt].name; cnt++) 6084 create_trace_option_file(tr, &topts[cnt], flags, 6085 &opts[cnt]); 6086 6087 return topts; 6088 } 6089 6090 static void 6091 destroy_trace_option_files(struct trace_option_dentry *topts) 6092 { 6093 int cnt; 6094 6095 if (!topts) 6096 return; 6097 6098 for (cnt = 0; topts[cnt].opt; cnt++) { 6099 if (topts[cnt].entry) 6100 debugfs_remove(topts[cnt].entry); 6101 } 6102 6103 kfree(topts); 6104 } 6105 6106 static struct dentry * 6107 create_trace_option_core_file(struct trace_array *tr, 6108 const char *option, long index) 6109 { 6110 struct dentry *t_options; 6111 6112 t_options = trace_options_init_dentry(tr); 6113 if (!t_options) 6114 return NULL; 6115 6116 return trace_create_file(option, 0644, t_options, (void *)index, 6117 &trace_options_core_fops); 6118 } 6119 6120 static __init void create_trace_options_dir(struct trace_array *tr) 6121 { 6122 struct dentry *t_options; 6123 int i; 6124 6125 t_options = trace_options_init_dentry(tr); 6126 if (!t_options) 6127 return; 6128 6129 for (i = 0; trace_options[i]; i++) 6130 create_trace_option_core_file(tr, trace_options[i], i); 6131 } 6132 6133 static ssize_t 6134 rb_simple_read(struct file *filp, char __user *ubuf, 6135 size_t cnt, loff_t *ppos) 6136 { 6137 struct trace_array *tr = filp->private_data; 6138 char buf[64]; 6139 int r; 6140 6141 r = tracer_tracing_is_on(tr); 6142 r = sprintf(buf, "%d\n", r); 6143 6144 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 6145 } 6146 6147 static ssize_t 6148 rb_simple_write(struct file *filp, const char __user *ubuf, 6149 size_t cnt, loff_t *ppos) 6150 { 6151 struct trace_array *tr = filp->private_data; 6152 struct ring_buffer *buffer = tr->trace_buffer.buffer; 6153 unsigned long val; 6154 int ret; 6155 6156 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 6157 if (ret) 6158 return ret; 6159 6160 if (buffer) { 6161 mutex_lock(&trace_types_lock); 6162 if (val) { 6163 tracer_tracing_on(tr); 6164 if (tr->current_trace->start) 6165 tr->current_trace->start(tr); 6166 } else { 6167 tracer_tracing_off(tr); 6168 if (tr->current_trace->stop) 6169 tr->current_trace->stop(tr); 6170 } 6171 mutex_unlock(&trace_types_lock); 6172 } 6173 6174 (*ppos)++; 6175 6176 return cnt; 6177 } 6178 6179 static const struct file_operations rb_simple_fops = { 6180 .open = tracing_open_generic_tr, 6181 .read = rb_simple_read, 6182 .write = rb_simple_write, 6183 .release = tracing_release_generic_tr, 6184 .llseek = default_llseek, 6185 }; 6186 6187 struct dentry *trace_instance_dir; 6188 6189 static void 6190 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer); 6191 6192 static int 6193 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size) 6194 { 6195 enum ring_buffer_flags rb_flags; 6196 6197 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0; 6198 6199 buf->tr = tr; 6200 6201 buf->buffer = ring_buffer_alloc(size, rb_flags); 6202 if (!buf->buffer) 6203 return -ENOMEM; 6204 6205 buf->data = alloc_percpu(struct trace_array_cpu); 6206 if (!buf->data) { 6207 ring_buffer_free(buf->buffer); 6208 return -ENOMEM; 6209 } 6210 6211 /* Allocate the first page for all buffers */ 6212 set_buffer_entries(&tr->trace_buffer, 6213 ring_buffer_size(tr->trace_buffer.buffer, 0)); 6214 6215 return 0; 6216 } 6217 6218 static int allocate_trace_buffers(struct trace_array *tr, int size) 6219 { 6220 int ret; 6221 6222 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size); 6223 if (ret) 6224 return ret; 6225 6226 #ifdef CONFIG_TRACER_MAX_TRACE 6227 ret = allocate_trace_buffer(tr, &tr->max_buffer, 6228 allocate_snapshot ? size : 1); 6229 if (WARN_ON(ret)) { 6230 ring_buffer_free(tr->trace_buffer.buffer); 6231 free_percpu(tr->trace_buffer.data); 6232 return -ENOMEM; 6233 } 6234 tr->allocated_snapshot = allocate_snapshot; 6235 6236 /* 6237 * Only the top level trace array gets its snapshot allocated 6238 * from the kernel command line. 6239 */ 6240 allocate_snapshot = false; 6241 #endif 6242 return 0; 6243 } 6244 6245 static void free_trace_buffer(struct trace_buffer *buf) 6246 { 6247 if (buf->buffer) { 6248 ring_buffer_free(buf->buffer); 6249 buf->buffer = NULL; 6250 free_percpu(buf->data); 6251 buf->data = NULL; 6252 } 6253 } 6254 6255 static void free_trace_buffers(struct trace_array *tr) 6256 { 6257 if (!tr) 6258 return; 6259 6260 free_trace_buffer(&tr->trace_buffer); 6261 6262 #ifdef CONFIG_TRACER_MAX_TRACE 6263 free_trace_buffer(&tr->max_buffer); 6264 #endif 6265 } 6266 6267 static int new_instance_create(const char *name) 6268 { 6269 struct trace_array *tr; 6270 int ret; 6271 6272 mutex_lock(&trace_types_lock); 6273 6274 ret = -EEXIST; 6275 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 6276 if (tr->name && strcmp(tr->name, name) == 0) 6277 goto out_unlock; 6278 } 6279 6280 ret = -ENOMEM; 6281 tr = kzalloc(sizeof(*tr), GFP_KERNEL); 6282 if (!tr) 6283 goto out_unlock; 6284 6285 tr->name = kstrdup(name, GFP_KERNEL); 6286 if (!tr->name) 6287 goto out_free_tr; 6288 6289 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL)) 6290 goto out_free_tr; 6291 6292 cpumask_copy(tr->tracing_cpumask, cpu_all_mask); 6293 6294 raw_spin_lock_init(&tr->start_lock); 6295 6296 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 6297 6298 tr->current_trace = &nop_trace; 6299 6300 INIT_LIST_HEAD(&tr->systems); 6301 INIT_LIST_HEAD(&tr->events); 6302 6303 if (allocate_trace_buffers(tr, trace_buf_size) < 0) 6304 goto out_free_tr; 6305 6306 tr->dir = debugfs_create_dir(name, trace_instance_dir); 6307 if (!tr->dir) 6308 goto out_free_tr; 6309 6310 ret = event_trace_add_tracer(tr->dir, tr); 6311 if (ret) { 6312 debugfs_remove_recursive(tr->dir); 6313 goto out_free_tr; 6314 } 6315 6316 init_tracer_debugfs(tr, tr->dir); 6317 6318 list_add(&tr->list, &ftrace_trace_arrays); 6319 6320 mutex_unlock(&trace_types_lock); 6321 6322 return 0; 6323 6324 out_free_tr: 6325 free_trace_buffers(tr); 6326 free_cpumask_var(tr->tracing_cpumask); 6327 kfree(tr->name); 6328 kfree(tr); 6329 6330 out_unlock: 6331 mutex_unlock(&trace_types_lock); 6332 6333 return ret; 6334 6335 } 6336 6337 static int instance_delete(const char *name) 6338 { 6339 struct trace_array *tr; 6340 int found = 0; 6341 int ret; 6342 6343 mutex_lock(&trace_types_lock); 6344 6345 ret = -ENODEV; 6346 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 6347 if (tr->name && strcmp(tr->name, name) == 0) { 6348 found = 1; 6349 break; 6350 } 6351 } 6352 if (!found) 6353 goto out_unlock; 6354 6355 ret = -EBUSY; 6356 if (tr->ref) 6357 goto out_unlock; 6358 6359 list_del(&tr->list); 6360 6361 tracing_set_nop(tr); 6362 event_trace_del_tracer(tr); 6363 ftrace_destroy_function_files(tr); 6364 debugfs_remove_recursive(tr->dir); 6365 free_trace_buffers(tr); 6366 6367 kfree(tr->name); 6368 kfree(tr); 6369 6370 ret = 0; 6371 6372 out_unlock: 6373 mutex_unlock(&trace_types_lock); 6374 6375 return ret; 6376 } 6377 6378 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode) 6379 { 6380 struct dentry *parent; 6381 int ret; 6382 6383 /* Paranoid: Make sure the parent is the "instances" directory */ 6384 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias); 6385 if (WARN_ON_ONCE(parent != trace_instance_dir)) 6386 return -ENOENT; 6387 6388 /* 6389 * The inode mutex is locked, but debugfs_create_dir() will also 6390 * take the mutex. As the instances directory can not be destroyed 6391 * or changed in any other way, it is safe to unlock it, and 6392 * let the dentry try. If two users try to make the same dir at 6393 * the same time, then the new_instance_create() will determine the 6394 * winner. 6395 */ 6396 mutex_unlock(&inode->i_mutex); 6397 6398 ret = new_instance_create(dentry->d_iname); 6399 6400 mutex_lock(&inode->i_mutex); 6401 6402 return ret; 6403 } 6404 6405 static int instance_rmdir(struct inode *inode, struct dentry *dentry) 6406 { 6407 struct dentry *parent; 6408 int ret; 6409 6410 /* Paranoid: Make sure the parent is the "instances" directory */ 6411 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias); 6412 if (WARN_ON_ONCE(parent != trace_instance_dir)) 6413 return -ENOENT; 6414 6415 /* The caller did a dget() on dentry */ 6416 mutex_unlock(&dentry->d_inode->i_mutex); 6417 6418 /* 6419 * The inode mutex is locked, but debugfs_create_dir() will also 6420 * take the mutex. As the instances directory can not be destroyed 6421 * or changed in any other way, it is safe to unlock it, and 6422 * let the dentry try. If two users try to make the same dir at 6423 * the same time, then the instance_delete() will determine the 6424 * winner. 6425 */ 6426 mutex_unlock(&inode->i_mutex); 6427 6428 ret = instance_delete(dentry->d_iname); 6429 6430 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT); 6431 mutex_lock(&dentry->d_inode->i_mutex); 6432 6433 return ret; 6434 } 6435 6436 static const struct inode_operations instance_dir_inode_operations = { 6437 .lookup = simple_lookup, 6438 .mkdir = instance_mkdir, 6439 .rmdir = instance_rmdir, 6440 }; 6441 6442 static __init void create_trace_instances(struct dentry *d_tracer) 6443 { 6444 trace_instance_dir = debugfs_create_dir("instances", d_tracer); 6445 if (WARN_ON(!trace_instance_dir)) 6446 return; 6447 6448 /* Hijack the dir inode operations, to allow mkdir */ 6449 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations; 6450 } 6451 6452 static void 6453 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer) 6454 { 6455 int cpu; 6456 6457 trace_create_file("available_tracers", 0444, d_tracer, 6458 tr, &show_traces_fops); 6459 6460 trace_create_file("current_tracer", 0644, d_tracer, 6461 tr, &set_tracer_fops); 6462 6463 trace_create_file("tracing_cpumask", 0644, d_tracer, 6464 tr, &tracing_cpumask_fops); 6465 6466 trace_create_file("trace_options", 0644, d_tracer, 6467 tr, &tracing_iter_fops); 6468 6469 trace_create_file("trace", 0644, d_tracer, 6470 tr, &tracing_fops); 6471 6472 trace_create_file("trace_pipe", 0444, d_tracer, 6473 tr, &tracing_pipe_fops); 6474 6475 trace_create_file("buffer_size_kb", 0644, d_tracer, 6476 tr, &tracing_entries_fops); 6477 6478 trace_create_file("buffer_total_size_kb", 0444, d_tracer, 6479 tr, &tracing_total_entries_fops); 6480 6481 trace_create_file("free_buffer", 0200, d_tracer, 6482 tr, &tracing_free_buffer_fops); 6483 6484 trace_create_file("trace_marker", 0220, d_tracer, 6485 tr, &tracing_mark_fops); 6486 6487 trace_create_file("trace_clock", 0644, d_tracer, tr, 6488 &trace_clock_fops); 6489 6490 trace_create_file("tracing_on", 0644, d_tracer, 6491 tr, &rb_simple_fops); 6492 6493 #ifdef CONFIG_TRACER_MAX_TRACE 6494 trace_create_file("tracing_max_latency", 0644, d_tracer, 6495 &tr->max_latency, &tracing_max_lat_fops); 6496 #endif 6497 6498 if (ftrace_create_function_files(tr, d_tracer)) 6499 WARN(1, "Could not allocate function filter files"); 6500 6501 #ifdef CONFIG_TRACER_SNAPSHOT 6502 trace_create_file("snapshot", 0644, d_tracer, 6503 tr, &snapshot_fops); 6504 #endif 6505 6506 for_each_tracing_cpu(cpu) 6507 tracing_init_debugfs_percpu(tr, cpu); 6508 6509 } 6510 6511 static __init int tracer_init_debugfs(void) 6512 { 6513 struct dentry *d_tracer; 6514 6515 trace_access_lock_init(); 6516 6517 d_tracer = tracing_init_dentry(); 6518 if (!d_tracer) 6519 return 0; 6520 6521 init_tracer_debugfs(&global_trace, d_tracer); 6522 6523 trace_create_file("tracing_thresh", 0644, d_tracer, 6524 &tracing_thresh, &tracing_max_lat_fops); 6525 6526 trace_create_file("README", 0444, d_tracer, 6527 NULL, &tracing_readme_fops); 6528 6529 trace_create_file("saved_cmdlines", 0444, d_tracer, 6530 NULL, &tracing_saved_cmdlines_fops); 6531 6532 trace_create_file("saved_cmdlines_size", 0644, d_tracer, 6533 NULL, &tracing_saved_cmdlines_size_fops); 6534 6535 #ifdef CONFIG_DYNAMIC_FTRACE 6536 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer, 6537 &ftrace_update_tot_cnt, &tracing_dyn_info_fops); 6538 #endif 6539 6540 create_trace_instances(d_tracer); 6541 6542 create_trace_options_dir(&global_trace); 6543 6544 return 0; 6545 } 6546 6547 static int trace_panic_handler(struct notifier_block *this, 6548 unsigned long event, void *unused) 6549 { 6550 if (ftrace_dump_on_oops) 6551 ftrace_dump(ftrace_dump_on_oops); 6552 return NOTIFY_OK; 6553 } 6554 6555 static struct notifier_block trace_panic_notifier = { 6556 .notifier_call = trace_panic_handler, 6557 .next = NULL, 6558 .priority = 150 /* priority: INT_MAX >= x >= 0 */ 6559 }; 6560 6561 static int trace_die_handler(struct notifier_block *self, 6562 unsigned long val, 6563 void *data) 6564 { 6565 switch (val) { 6566 case DIE_OOPS: 6567 if (ftrace_dump_on_oops) 6568 ftrace_dump(ftrace_dump_on_oops); 6569 break; 6570 default: 6571 break; 6572 } 6573 return NOTIFY_OK; 6574 } 6575 6576 static struct notifier_block trace_die_notifier = { 6577 .notifier_call = trace_die_handler, 6578 .priority = 200 6579 }; 6580 6581 /* 6582 * printk is set to max of 1024, we really don't need it that big. 6583 * Nothing should be printing 1000 characters anyway. 6584 */ 6585 #define TRACE_MAX_PRINT 1000 6586 6587 /* 6588 * Define here KERN_TRACE so that we have one place to modify 6589 * it if we decide to change what log level the ftrace dump 6590 * should be at. 6591 */ 6592 #define KERN_TRACE KERN_EMERG 6593 6594 void 6595 trace_printk_seq(struct trace_seq *s) 6596 { 6597 /* Probably should print a warning here. */ 6598 if (s->len >= TRACE_MAX_PRINT) 6599 s->len = TRACE_MAX_PRINT; 6600 6601 /* should be zero ended, but we are paranoid. */ 6602 s->buffer[s->len] = 0; 6603 6604 printk(KERN_TRACE "%s", s->buffer); 6605 6606 trace_seq_init(s); 6607 } 6608 6609 void trace_init_global_iter(struct trace_iterator *iter) 6610 { 6611 iter->tr = &global_trace; 6612 iter->trace = iter->tr->current_trace; 6613 iter->cpu_file = RING_BUFFER_ALL_CPUS; 6614 iter->trace_buffer = &global_trace.trace_buffer; 6615 6616 if (iter->trace && iter->trace->open) 6617 iter->trace->open(iter); 6618 6619 /* Annotate start of buffers if we had overruns */ 6620 if (ring_buffer_overruns(iter->trace_buffer->buffer)) 6621 iter->iter_flags |= TRACE_FILE_ANNOTATE; 6622 6623 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 6624 if (trace_clocks[iter->tr->clock_id].in_ns) 6625 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 6626 } 6627 6628 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) 6629 { 6630 /* use static because iter can be a bit big for the stack */ 6631 static struct trace_iterator iter; 6632 static atomic_t dump_running; 6633 unsigned int old_userobj; 6634 unsigned long flags; 6635 int cnt = 0, cpu; 6636 6637 /* Only allow one dump user at a time. */ 6638 if (atomic_inc_return(&dump_running) != 1) { 6639 atomic_dec(&dump_running); 6640 return; 6641 } 6642 6643 /* 6644 * Always turn off tracing when we dump. 6645 * We don't need to show trace output of what happens 6646 * between multiple crashes. 6647 * 6648 * If the user does a sysrq-z, then they can re-enable 6649 * tracing with echo 1 > tracing_on. 6650 */ 6651 tracing_off(); 6652 6653 local_irq_save(flags); 6654 6655 /* Simulate the iterator */ 6656 trace_init_global_iter(&iter); 6657 6658 for_each_tracing_cpu(cpu) { 6659 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled); 6660 } 6661 6662 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ; 6663 6664 /* don't look at user memory in panic mode */ 6665 trace_flags &= ~TRACE_ITER_SYM_USEROBJ; 6666 6667 switch (oops_dump_mode) { 6668 case DUMP_ALL: 6669 iter.cpu_file = RING_BUFFER_ALL_CPUS; 6670 break; 6671 case DUMP_ORIG: 6672 iter.cpu_file = raw_smp_processor_id(); 6673 break; 6674 case DUMP_NONE: 6675 goto out_enable; 6676 default: 6677 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n"); 6678 iter.cpu_file = RING_BUFFER_ALL_CPUS; 6679 } 6680 6681 printk(KERN_TRACE "Dumping ftrace buffer:\n"); 6682 6683 /* Did function tracer already get disabled? */ 6684 if (ftrace_is_dead()) { 6685 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n"); 6686 printk("# MAY BE MISSING FUNCTION EVENTS\n"); 6687 } 6688 6689 /* 6690 * We need to stop all tracing on all CPUS to read the 6691 * the next buffer. This is a bit expensive, but is 6692 * not done often. We fill all what we can read, 6693 * and then release the locks again. 6694 */ 6695 6696 while (!trace_empty(&iter)) { 6697 6698 if (!cnt) 6699 printk(KERN_TRACE "---------------------------------\n"); 6700 6701 cnt++; 6702 6703 /* reset all but tr, trace, and overruns */ 6704 memset(&iter.seq, 0, 6705 sizeof(struct trace_iterator) - 6706 offsetof(struct trace_iterator, seq)); 6707 iter.iter_flags |= TRACE_FILE_LAT_FMT; 6708 iter.pos = -1; 6709 6710 if (trace_find_next_entry_inc(&iter) != NULL) { 6711 int ret; 6712 6713 ret = print_trace_line(&iter); 6714 if (ret != TRACE_TYPE_NO_CONSUME) 6715 trace_consume(&iter); 6716 } 6717 touch_nmi_watchdog(); 6718 6719 trace_printk_seq(&iter.seq); 6720 } 6721 6722 if (!cnt) 6723 printk(KERN_TRACE " (ftrace buffer empty)\n"); 6724 else 6725 printk(KERN_TRACE "---------------------------------\n"); 6726 6727 out_enable: 6728 trace_flags |= old_userobj; 6729 6730 for_each_tracing_cpu(cpu) { 6731 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled); 6732 } 6733 atomic_dec(&dump_running); 6734 local_irq_restore(flags); 6735 } 6736 EXPORT_SYMBOL_GPL(ftrace_dump); 6737 6738 __init static int tracer_alloc_buffers(void) 6739 { 6740 int ring_buf_size; 6741 int ret = -ENOMEM; 6742 6743 6744 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL)) 6745 goto out; 6746 6747 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL)) 6748 goto out_free_buffer_mask; 6749 6750 /* Only allocate trace_printk buffers if a trace_printk exists */ 6751 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt) 6752 /* Must be called before global_trace.buffer is allocated */ 6753 trace_printk_init_buffers(); 6754 6755 /* To save memory, keep the ring buffer size to its minimum */ 6756 if (ring_buffer_expanded) 6757 ring_buf_size = trace_buf_size; 6758 else 6759 ring_buf_size = 1; 6760 6761 cpumask_copy(tracing_buffer_mask, cpu_possible_mask); 6762 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask); 6763 6764 raw_spin_lock_init(&global_trace.start_lock); 6765 6766 /* Used for event triggers */ 6767 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE); 6768 if (!temp_buffer) 6769 goto out_free_cpumask; 6770 6771 if (trace_create_savedcmd() < 0) 6772 goto out_free_temp_buffer; 6773 6774 /* TODO: make the number of buffers hot pluggable with CPUS */ 6775 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) { 6776 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n"); 6777 WARN_ON(1); 6778 goto out_free_savedcmd; 6779 } 6780 6781 if (global_trace.buffer_disabled) 6782 tracing_off(); 6783 6784 if (trace_boot_clock) { 6785 ret = tracing_set_clock(&global_trace, trace_boot_clock); 6786 if (ret < 0) 6787 pr_warning("Trace clock %s not defined, going back to default\n", 6788 trace_boot_clock); 6789 } 6790 6791 /* 6792 * register_tracer() might reference current_trace, so it 6793 * needs to be set before we register anything. This is 6794 * just a bootstrap of current_trace anyway. 6795 */ 6796 global_trace.current_trace = &nop_trace; 6797 6798 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 6799 6800 ftrace_init_global_array_ops(&global_trace); 6801 6802 register_tracer(&nop_trace); 6803 6804 /* All seems OK, enable tracing */ 6805 tracing_disabled = 0; 6806 6807 atomic_notifier_chain_register(&panic_notifier_list, 6808 &trace_panic_notifier); 6809 6810 register_die_notifier(&trace_die_notifier); 6811 6812 global_trace.flags = TRACE_ARRAY_FL_GLOBAL; 6813 6814 INIT_LIST_HEAD(&global_trace.systems); 6815 INIT_LIST_HEAD(&global_trace.events); 6816 list_add(&global_trace.list, &ftrace_trace_arrays); 6817 6818 while (trace_boot_options) { 6819 char *option; 6820 6821 option = strsep(&trace_boot_options, ","); 6822 trace_set_options(&global_trace, option); 6823 } 6824 6825 register_snapshot_cmd(); 6826 6827 return 0; 6828 6829 out_free_savedcmd: 6830 free_saved_cmdlines_buffer(savedcmd); 6831 out_free_temp_buffer: 6832 ring_buffer_free(temp_buffer); 6833 out_free_cpumask: 6834 free_cpumask_var(global_trace.tracing_cpumask); 6835 out_free_buffer_mask: 6836 free_cpumask_var(tracing_buffer_mask); 6837 out: 6838 return ret; 6839 } 6840 6841 __init static int clear_boot_tracer(void) 6842 { 6843 /* 6844 * The default tracer at boot buffer is an init section. 6845 * This function is called in lateinit. If we did not 6846 * find the boot tracer, then clear it out, to prevent 6847 * later registration from accessing the buffer that is 6848 * about to be freed. 6849 */ 6850 if (!default_bootup_tracer) 6851 return 0; 6852 6853 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n", 6854 default_bootup_tracer); 6855 default_bootup_tracer = NULL; 6856 6857 return 0; 6858 } 6859 6860 early_initcall(tracer_alloc_buffers); 6861 fs_initcall(tracer_init_debugfs); 6862 late_initcall(clear_boot_tracer); 6863