1 /* 2 * ring buffer based function tracer 3 * 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 6 * 7 * Originally taken from the RT patch by: 8 * Arnaldo Carvalho de Melo <acme@redhat.com> 9 * 10 * Based on code from the latency_tracer, that is: 11 * Copyright (C) 2004-2006 Ingo Molnar 12 * Copyright (C) 2004 Nadia Yvette Chambers 13 */ 14 #include <linux/ring_buffer.h> 15 #include <generated/utsrelease.h> 16 #include <linux/stacktrace.h> 17 #include <linux/writeback.h> 18 #include <linux/kallsyms.h> 19 #include <linux/seq_file.h> 20 #include <linux/notifier.h> 21 #include <linux/irqflags.h> 22 #include <linux/irq_work.h> 23 #include <linux/debugfs.h> 24 #include <linux/pagemap.h> 25 #include <linux/hardirq.h> 26 #include <linux/linkage.h> 27 #include <linux/uaccess.h> 28 #include <linux/kprobes.h> 29 #include <linux/ftrace.h> 30 #include <linux/module.h> 31 #include <linux/percpu.h> 32 #include <linux/splice.h> 33 #include <linux/kdebug.h> 34 #include <linux/string.h> 35 #include <linux/rwsem.h> 36 #include <linux/slab.h> 37 #include <linux/ctype.h> 38 #include <linux/init.h> 39 #include <linux/poll.h> 40 #include <linux/nmi.h> 41 #include <linux/fs.h> 42 43 #include "trace.h" 44 #include "trace_output.h" 45 46 /* 47 * On boot up, the ring buffer is set to the minimum size, so that 48 * we do not waste memory on systems that are not using tracing. 49 */ 50 int ring_buffer_expanded; 51 52 /* 53 * We need to change this state when a selftest is running. 54 * A selftest will lurk into the ring-buffer to count the 55 * entries inserted during the selftest although some concurrent 56 * insertions into the ring-buffer such as trace_printk could occurred 57 * at the same time, giving false positive or negative results. 58 */ 59 static bool __read_mostly tracing_selftest_running; 60 61 /* 62 * If a tracer is running, we do not want to run SELFTEST. 63 */ 64 bool __read_mostly tracing_selftest_disabled; 65 66 /* For tracers that don't implement custom flags */ 67 static struct tracer_opt dummy_tracer_opt[] = { 68 { } 69 }; 70 71 static struct tracer_flags dummy_tracer_flags = { 72 .val = 0, 73 .opts = dummy_tracer_opt 74 }; 75 76 static int dummy_set_flag(u32 old_flags, u32 bit, int set) 77 { 78 return 0; 79 } 80 81 /* 82 * To prevent the comm cache from being overwritten when no 83 * tracing is active, only save the comm when a trace event 84 * occurred. 85 */ 86 static DEFINE_PER_CPU(bool, trace_cmdline_save); 87 88 /* 89 * When a reader is waiting for data, then this variable is 90 * set to true. 91 */ 92 static bool trace_wakeup_needed; 93 94 static struct irq_work trace_work_wakeup; 95 96 /* 97 * Kill all tracing for good (never come back). 98 * It is initialized to 1 but will turn to zero if the initialization 99 * of the tracer is successful. But that is the only place that sets 100 * this back to zero. 101 */ 102 static int tracing_disabled = 1; 103 104 DEFINE_PER_CPU(int, ftrace_cpu_disabled); 105 106 cpumask_var_t __read_mostly tracing_buffer_mask; 107 108 /* 109 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops 110 * 111 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops 112 * is set, then ftrace_dump is called. This will output the contents 113 * of the ftrace buffers to the console. This is very useful for 114 * capturing traces that lead to crashes and outputing it to a 115 * serial console. 116 * 117 * It is default off, but you can enable it with either specifying 118 * "ftrace_dump_on_oops" in the kernel command line, or setting 119 * /proc/sys/kernel/ftrace_dump_on_oops 120 * Set 1 if you want to dump buffers of all CPUs 121 * Set 2 if you want to dump the buffer of the CPU that triggered oops 122 */ 123 124 enum ftrace_dump_mode ftrace_dump_on_oops; 125 126 static int tracing_set_tracer(const char *buf); 127 128 #define MAX_TRACER_SIZE 100 129 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; 130 static char *default_bootup_tracer; 131 132 static int __init set_cmdline_ftrace(char *str) 133 { 134 strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE); 135 default_bootup_tracer = bootup_tracer_buf; 136 /* We are using ftrace early, expand it */ 137 ring_buffer_expanded = 1; 138 return 1; 139 } 140 __setup("ftrace=", set_cmdline_ftrace); 141 142 static int __init set_ftrace_dump_on_oops(char *str) 143 { 144 if (*str++ != '=' || !*str) { 145 ftrace_dump_on_oops = DUMP_ALL; 146 return 1; 147 } 148 149 if (!strcmp("orig_cpu", str)) { 150 ftrace_dump_on_oops = DUMP_ORIG; 151 return 1; 152 } 153 154 return 0; 155 } 156 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops); 157 158 159 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata; 160 static char *trace_boot_options __initdata; 161 162 static int __init set_trace_boot_options(char *str) 163 { 164 strncpy(trace_boot_options_buf, str, MAX_TRACER_SIZE); 165 trace_boot_options = trace_boot_options_buf; 166 return 0; 167 } 168 __setup("trace_options=", set_trace_boot_options); 169 170 unsigned long long ns2usecs(cycle_t nsec) 171 { 172 nsec += 500; 173 do_div(nsec, 1000); 174 return nsec; 175 } 176 177 /* 178 * The global_trace is the descriptor that holds the tracing 179 * buffers for the live tracing. For each CPU, it contains 180 * a link list of pages that will store trace entries. The 181 * page descriptor of the pages in the memory is used to hold 182 * the link list by linking the lru item in the page descriptor 183 * to each of the pages in the buffer per CPU. 184 * 185 * For each active CPU there is a data field that holds the 186 * pages for the buffer for that CPU. Each CPU has the same number 187 * of pages allocated for its buffer. 188 */ 189 static struct trace_array global_trace; 190 191 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu); 192 193 int filter_current_check_discard(struct ring_buffer *buffer, 194 struct ftrace_event_call *call, void *rec, 195 struct ring_buffer_event *event) 196 { 197 return filter_check_discard(call, rec, buffer, event); 198 } 199 EXPORT_SYMBOL_GPL(filter_current_check_discard); 200 201 cycle_t ftrace_now(int cpu) 202 { 203 u64 ts; 204 205 /* Early boot up does not have a buffer yet */ 206 if (!global_trace.buffer) 207 return trace_clock_local(); 208 209 ts = ring_buffer_time_stamp(global_trace.buffer, cpu); 210 ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts); 211 212 return ts; 213 } 214 215 /* 216 * The max_tr is used to snapshot the global_trace when a maximum 217 * latency is reached. Some tracers will use this to store a maximum 218 * trace while it continues examining live traces. 219 * 220 * The buffers for the max_tr are set up the same as the global_trace. 221 * When a snapshot is taken, the link list of the max_tr is swapped 222 * with the link list of the global_trace and the buffers are reset for 223 * the global_trace so the tracing can continue. 224 */ 225 static struct trace_array max_tr; 226 227 static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data); 228 229 int tracing_is_enabled(void) 230 { 231 return tracing_is_on(); 232 } 233 234 /* 235 * trace_buf_size is the size in bytes that is allocated 236 * for a buffer. Note, the number of bytes is always rounded 237 * to page size. 238 * 239 * This number is purposely set to a low number of 16384. 240 * If the dump on oops happens, it will be much appreciated 241 * to not have to wait for all that output. Anyway this can be 242 * boot time and run time configurable. 243 */ 244 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */ 245 246 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT; 247 248 /* trace_types holds a link list of available tracers. */ 249 static struct tracer *trace_types __read_mostly; 250 251 /* current_trace points to the tracer that is currently active */ 252 static struct tracer *current_trace __read_mostly; 253 254 /* 255 * trace_types_lock is used to protect the trace_types list. 256 */ 257 static DEFINE_MUTEX(trace_types_lock); 258 259 /* 260 * serialize the access of the ring buffer 261 * 262 * ring buffer serializes readers, but it is low level protection. 263 * The validity of the events (which returns by ring_buffer_peek() ..etc) 264 * are not protected by ring buffer. 265 * 266 * The content of events may become garbage if we allow other process consumes 267 * these events concurrently: 268 * A) the page of the consumed events may become a normal page 269 * (not reader page) in ring buffer, and this page will be rewrited 270 * by events producer. 271 * B) The page of the consumed events may become a page for splice_read, 272 * and this page will be returned to system. 273 * 274 * These primitives allow multi process access to different cpu ring buffer 275 * concurrently. 276 * 277 * These primitives don't distinguish read-only and read-consume access. 278 * Multi read-only access are also serialized. 279 */ 280 281 #ifdef CONFIG_SMP 282 static DECLARE_RWSEM(all_cpu_access_lock); 283 static DEFINE_PER_CPU(struct mutex, cpu_access_lock); 284 285 static inline void trace_access_lock(int cpu) 286 { 287 if (cpu == TRACE_PIPE_ALL_CPU) { 288 /* gain it for accessing the whole ring buffer. */ 289 down_write(&all_cpu_access_lock); 290 } else { 291 /* gain it for accessing a cpu ring buffer. */ 292 293 /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */ 294 down_read(&all_cpu_access_lock); 295 296 /* Secondly block other access to this @cpu ring buffer. */ 297 mutex_lock(&per_cpu(cpu_access_lock, cpu)); 298 } 299 } 300 301 static inline void trace_access_unlock(int cpu) 302 { 303 if (cpu == TRACE_PIPE_ALL_CPU) { 304 up_write(&all_cpu_access_lock); 305 } else { 306 mutex_unlock(&per_cpu(cpu_access_lock, cpu)); 307 up_read(&all_cpu_access_lock); 308 } 309 } 310 311 static inline void trace_access_lock_init(void) 312 { 313 int cpu; 314 315 for_each_possible_cpu(cpu) 316 mutex_init(&per_cpu(cpu_access_lock, cpu)); 317 } 318 319 #else 320 321 static DEFINE_MUTEX(access_lock); 322 323 static inline void trace_access_lock(int cpu) 324 { 325 (void)cpu; 326 mutex_lock(&access_lock); 327 } 328 329 static inline void trace_access_unlock(int cpu) 330 { 331 (void)cpu; 332 mutex_unlock(&access_lock); 333 } 334 335 static inline void trace_access_lock_init(void) 336 { 337 } 338 339 #endif 340 341 /* trace_wait is a waitqueue for tasks blocked on trace_poll */ 342 static DECLARE_WAIT_QUEUE_HEAD(trace_wait); 343 344 /* trace_flags holds trace_options default values */ 345 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | 346 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME | 347 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | 348 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS; 349 350 static int trace_stop_count; 351 static DEFINE_RAW_SPINLOCK(tracing_start_lock); 352 353 /** 354 * trace_wake_up - wake up tasks waiting for trace input 355 * 356 * Schedules a delayed work to wake up any task that is blocked on the 357 * trace_wait queue. These is used with trace_poll for tasks polling the 358 * trace. 359 */ 360 static void trace_wake_up(struct irq_work *work) 361 { 362 wake_up_all(&trace_wait); 363 364 } 365 366 /** 367 * tracing_on - enable tracing buffers 368 * 369 * This function enables tracing buffers that may have been 370 * disabled with tracing_off. 371 */ 372 void tracing_on(void) 373 { 374 if (global_trace.buffer) 375 ring_buffer_record_on(global_trace.buffer); 376 /* 377 * This flag is only looked at when buffers haven't been 378 * allocated yet. We don't really care about the race 379 * between setting this flag and actually turning 380 * on the buffer. 381 */ 382 global_trace.buffer_disabled = 0; 383 } 384 EXPORT_SYMBOL_GPL(tracing_on); 385 386 /** 387 * tracing_off - turn off tracing buffers 388 * 389 * This function stops the tracing buffers from recording data. 390 * It does not disable any overhead the tracers themselves may 391 * be causing. This function simply causes all recording to 392 * the ring buffers to fail. 393 */ 394 void tracing_off(void) 395 { 396 if (global_trace.buffer) 397 ring_buffer_record_off(global_trace.buffer); 398 /* 399 * This flag is only looked at when buffers haven't been 400 * allocated yet. We don't really care about the race 401 * between setting this flag and actually turning 402 * on the buffer. 403 */ 404 global_trace.buffer_disabled = 1; 405 } 406 EXPORT_SYMBOL_GPL(tracing_off); 407 408 /** 409 * tracing_is_on - show state of ring buffers enabled 410 */ 411 int tracing_is_on(void) 412 { 413 if (global_trace.buffer) 414 return ring_buffer_record_is_on(global_trace.buffer); 415 return !global_trace.buffer_disabled; 416 } 417 EXPORT_SYMBOL_GPL(tracing_is_on); 418 419 static int __init set_buf_size(char *str) 420 { 421 unsigned long buf_size; 422 423 if (!str) 424 return 0; 425 buf_size = memparse(str, &str); 426 /* nr_entries can not be zero */ 427 if (buf_size == 0) 428 return 0; 429 trace_buf_size = buf_size; 430 return 1; 431 } 432 __setup("trace_buf_size=", set_buf_size); 433 434 static int __init set_tracing_thresh(char *str) 435 { 436 unsigned long threshold; 437 int ret; 438 439 if (!str) 440 return 0; 441 ret = kstrtoul(str, 0, &threshold); 442 if (ret < 0) 443 return 0; 444 tracing_thresh = threshold * 1000; 445 return 1; 446 } 447 __setup("tracing_thresh=", set_tracing_thresh); 448 449 unsigned long nsecs_to_usecs(unsigned long nsecs) 450 { 451 return nsecs / 1000; 452 } 453 454 /* These must match the bit postions in trace_iterator_flags */ 455 static const char *trace_options[] = { 456 "print-parent", 457 "sym-offset", 458 "sym-addr", 459 "verbose", 460 "raw", 461 "hex", 462 "bin", 463 "block", 464 "stacktrace", 465 "trace_printk", 466 "ftrace_preempt", 467 "branch", 468 "annotate", 469 "userstacktrace", 470 "sym-userobj", 471 "printk-msg-only", 472 "context-info", 473 "latency-format", 474 "sleep-time", 475 "graph-time", 476 "record-cmd", 477 "overwrite", 478 "disable_on_free", 479 "irq-info", 480 "markers", 481 NULL 482 }; 483 484 static struct { 485 u64 (*func)(void); 486 const char *name; 487 int in_ns; /* is this clock in nanoseconds? */ 488 } trace_clocks[] = { 489 { trace_clock_local, "local", 1 }, 490 { trace_clock_global, "global", 1 }, 491 { trace_clock_counter, "counter", 0 }, 492 ARCH_TRACE_CLOCKS 493 }; 494 495 int trace_clock_id; 496 497 /* 498 * trace_parser_get_init - gets the buffer for trace parser 499 */ 500 int trace_parser_get_init(struct trace_parser *parser, int size) 501 { 502 memset(parser, 0, sizeof(*parser)); 503 504 parser->buffer = kmalloc(size, GFP_KERNEL); 505 if (!parser->buffer) 506 return 1; 507 508 parser->size = size; 509 return 0; 510 } 511 512 /* 513 * trace_parser_put - frees the buffer for trace parser 514 */ 515 void trace_parser_put(struct trace_parser *parser) 516 { 517 kfree(parser->buffer); 518 } 519 520 /* 521 * trace_get_user - reads the user input string separated by space 522 * (matched by isspace(ch)) 523 * 524 * For each string found the 'struct trace_parser' is updated, 525 * and the function returns. 526 * 527 * Returns number of bytes read. 528 * 529 * See kernel/trace/trace.h for 'struct trace_parser' details. 530 */ 531 int trace_get_user(struct trace_parser *parser, const char __user *ubuf, 532 size_t cnt, loff_t *ppos) 533 { 534 char ch; 535 size_t read = 0; 536 ssize_t ret; 537 538 if (!*ppos) 539 trace_parser_clear(parser); 540 541 ret = get_user(ch, ubuf++); 542 if (ret) 543 goto out; 544 545 read++; 546 cnt--; 547 548 /* 549 * The parser is not finished with the last write, 550 * continue reading the user input without skipping spaces. 551 */ 552 if (!parser->cont) { 553 /* skip white space */ 554 while (cnt && isspace(ch)) { 555 ret = get_user(ch, ubuf++); 556 if (ret) 557 goto out; 558 read++; 559 cnt--; 560 } 561 562 /* only spaces were written */ 563 if (isspace(ch)) { 564 *ppos += read; 565 ret = read; 566 goto out; 567 } 568 569 parser->idx = 0; 570 } 571 572 /* read the non-space input */ 573 while (cnt && !isspace(ch)) { 574 if (parser->idx < parser->size - 1) 575 parser->buffer[parser->idx++] = ch; 576 else { 577 ret = -EINVAL; 578 goto out; 579 } 580 ret = get_user(ch, ubuf++); 581 if (ret) 582 goto out; 583 read++; 584 cnt--; 585 } 586 587 /* We either got finished input or we have to wait for another call. */ 588 if (isspace(ch)) { 589 parser->buffer[parser->idx] = 0; 590 parser->cont = false; 591 } else { 592 parser->cont = true; 593 parser->buffer[parser->idx++] = ch; 594 } 595 596 *ppos += read; 597 ret = read; 598 599 out: 600 return ret; 601 } 602 603 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt) 604 { 605 int len; 606 int ret; 607 608 if (!cnt) 609 return 0; 610 611 if (s->len <= s->readpos) 612 return -EBUSY; 613 614 len = s->len - s->readpos; 615 if (cnt > len) 616 cnt = len; 617 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt); 618 if (ret == cnt) 619 return -EFAULT; 620 621 cnt -= ret; 622 623 s->readpos += cnt; 624 return cnt; 625 } 626 627 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) 628 { 629 int len; 630 631 if (s->len <= s->readpos) 632 return -EBUSY; 633 634 len = s->len - s->readpos; 635 if (cnt > len) 636 cnt = len; 637 memcpy(buf, s->buffer + s->readpos, cnt); 638 639 s->readpos += cnt; 640 return cnt; 641 } 642 643 /* 644 * ftrace_max_lock is used to protect the swapping of buffers 645 * when taking a max snapshot. The buffers themselves are 646 * protected by per_cpu spinlocks. But the action of the swap 647 * needs its own lock. 648 * 649 * This is defined as a arch_spinlock_t in order to help 650 * with performance when lockdep debugging is enabled. 651 * 652 * It is also used in other places outside the update_max_tr 653 * so it needs to be defined outside of the 654 * CONFIG_TRACER_MAX_TRACE. 655 */ 656 static arch_spinlock_t ftrace_max_lock = 657 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 658 659 unsigned long __read_mostly tracing_thresh; 660 661 #ifdef CONFIG_TRACER_MAX_TRACE 662 unsigned long __read_mostly tracing_max_latency; 663 664 /* 665 * Copy the new maximum trace into the separate maximum-trace 666 * structure. (this way the maximum trace is permanently saved, 667 * for later retrieval via /sys/kernel/debug/tracing/latency_trace) 668 */ 669 static void 670 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 671 { 672 struct trace_array_cpu *data = tr->data[cpu]; 673 struct trace_array_cpu *max_data; 674 675 max_tr.cpu = cpu; 676 max_tr.time_start = data->preempt_timestamp; 677 678 max_data = max_tr.data[cpu]; 679 max_data->saved_latency = tracing_max_latency; 680 max_data->critical_start = data->critical_start; 681 max_data->critical_end = data->critical_end; 682 683 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN); 684 max_data->pid = tsk->pid; 685 max_data->uid = task_uid(tsk); 686 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; 687 max_data->policy = tsk->policy; 688 max_data->rt_priority = tsk->rt_priority; 689 690 /* record this tasks comm */ 691 tracing_record_cmdline(tsk); 692 } 693 694 /** 695 * update_max_tr - snapshot all trace buffers from global_trace to max_tr 696 * @tr: tracer 697 * @tsk: the task with the latency 698 * @cpu: The cpu that initiated the trace. 699 * 700 * Flip the buffers between the @tr and the max_tr and record information 701 * about which task was the cause of this latency. 702 */ 703 void 704 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 705 { 706 struct ring_buffer *buf = tr->buffer; 707 708 if (trace_stop_count) 709 return; 710 711 WARN_ON_ONCE(!irqs_disabled()); 712 if (!current_trace->use_max_tr) { 713 WARN_ON_ONCE(1); 714 return; 715 } 716 arch_spin_lock(&ftrace_max_lock); 717 718 tr->buffer = max_tr.buffer; 719 max_tr.buffer = buf; 720 721 __update_max_tr(tr, tsk, cpu); 722 arch_spin_unlock(&ftrace_max_lock); 723 } 724 725 /** 726 * update_max_tr_single - only copy one trace over, and reset the rest 727 * @tr - tracer 728 * @tsk - task with the latency 729 * @cpu - the cpu of the buffer to copy. 730 * 731 * Flip the trace of a single CPU buffer between the @tr and the max_tr. 732 */ 733 void 734 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) 735 { 736 int ret; 737 738 if (trace_stop_count) 739 return; 740 741 WARN_ON_ONCE(!irqs_disabled()); 742 if (!current_trace->use_max_tr) { 743 WARN_ON_ONCE(1); 744 return; 745 } 746 747 arch_spin_lock(&ftrace_max_lock); 748 749 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu); 750 751 if (ret == -EBUSY) { 752 /* 753 * We failed to swap the buffer due to a commit taking 754 * place on this CPU. We fail to record, but we reset 755 * the max trace buffer (no one writes directly to it) 756 * and flag that it failed. 757 */ 758 trace_array_printk(&max_tr, _THIS_IP_, 759 "Failed to swap buffers due to commit in progress\n"); 760 } 761 762 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); 763 764 __update_max_tr(tr, tsk, cpu); 765 arch_spin_unlock(&ftrace_max_lock); 766 } 767 #endif /* CONFIG_TRACER_MAX_TRACE */ 768 769 static void default_wait_pipe(struct trace_iterator *iter) 770 { 771 DEFINE_WAIT(wait); 772 773 prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE); 774 775 /* 776 * The events can happen in critical sections where 777 * checking a work queue can cause deadlocks. 778 * After adding a task to the queue, this flag is set 779 * only to notify events to try to wake up the queue 780 * using irq_work. 781 * 782 * We don't clear it even if the buffer is no longer 783 * empty. The flag only causes the next event to run 784 * irq_work to do the work queue wake up. The worse 785 * that can happen if we race with !trace_empty() is that 786 * an event will cause an irq_work to try to wake up 787 * an empty queue. 788 * 789 * There's no reason to protect this flag either, as 790 * the work queue and irq_work logic will do the necessary 791 * synchronization for the wake ups. The only thing 792 * that is necessary is that the wake up happens after 793 * a task has been queued. It's OK for spurious wake ups. 794 */ 795 trace_wakeup_needed = true; 796 797 if (trace_empty(iter)) 798 schedule(); 799 800 finish_wait(&trace_wait, &wait); 801 } 802 803 /** 804 * register_tracer - register a tracer with the ftrace system. 805 * @type - the plugin for the tracer 806 * 807 * Register a new plugin tracer. 808 */ 809 int register_tracer(struct tracer *type) 810 { 811 struct tracer *t; 812 int ret = 0; 813 814 if (!type->name) { 815 pr_info("Tracer must have a name\n"); 816 return -1; 817 } 818 819 if (strlen(type->name) >= MAX_TRACER_SIZE) { 820 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE); 821 return -1; 822 } 823 824 mutex_lock(&trace_types_lock); 825 826 tracing_selftest_running = true; 827 828 for (t = trace_types; t; t = t->next) { 829 if (strcmp(type->name, t->name) == 0) { 830 /* already found */ 831 pr_info("Tracer %s already registered\n", 832 type->name); 833 ret = -1; 834 goto out; 835 } 836 } 837 838 if (!type->set_flag) 839 type->set_flag = &dummy_set_flag; 840 if (!type->flags) 841 type->flags = &dummy_tracer_flags; 842 else 843 if (!type->flags->opts) 844 type->flags->opts = dummy_tracer_opt; 845 if (!type->wait_pipe) 846 type->wait_pipe = default_wait_pipe; 847 848 849 #ifdef CONFIG_FTRACE_STARTUP_TEST 850 if (type->selftest && !tracing_selftest_disabled) { 851 struct tracer *saved_tracer = current_trace; 852 struct trace_array *tr = &global_trace; 853 854 /* 855 * Run a selftest on this tracer. 856 * Here we reset the trace buffer, and set the current 857 * tracer to be this tracer. The tracer can then run some 858 * internal tracing to verify that everything is in order. 859 * If we fail, we do not register this tracer. 860 */ 861 tracing_reset_online_cpus(tr); 862 863 current_trace = type; 864 865 /* If we expanded the buffers, make sure the max is expanded too */ 866 if (ring_buffer_expanded && type->use_max_tr) 867 ring_buffer_resize(max_tr.buffer, trace_buf_size, 868 RING_BUFFER_ALL_CPUS); 869 870 /* the test is responsible for initializing and enabling */ 871 pr_info("Testing tracer %s: ", type->name); 872 ret = type->selftest(type, tr); 873 /* the test is responsible for resetting too */ 874 current_trace = saved_tracer; 875 if (ret) { 876 printk(KERN_CONT "FAILED!\n"); 877 /* Add the warning after printing 'FAILED' */ 878 WARN_ON(1); 879 goto out; 880 } 881 /* Only reset on passing, to avoid touching corrupted buffers */ 882 tracing_reset_online_cpus(tr); 883 884 /* Shrink the max buffer again */ 885 if (ring_buffer_expanded && type->use_max_tr) 886 ring_buffer_resize(max_tr.buffer, 1, 887 RING_BUFFER_ALL_CPUS); 888 889 printk(KERN_CONT "PASSED\n"); 890 } 891 #endif 892 893 type->next = trace_types; 894 trace_types = type; 895 896 out: 897 tracing_selftest_running = false; 898 mutex_unlock(&trace_types_lock); 899 900 if (ret || !default_bootup_tracer) 901 goto out_unlock; 902 903 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE)) 904 goto out_unlock; 905 906 printk(KERN_INFO "Starting tracer '%s'\n", type->name); 907 /* Do we want this tracer to start on bootup? */ 908 tracing_set_tracer(type->name); 909 default_bootup_tracer = NULL; 910 /* disable other selftests, since this will break it. */ 911 tracing_selftest_disabled = 1; 912 #ifdef CONFIG_FTRACE_STARTUP_TEST 913 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n", 914 type->name); 915 #endif 916 917 out_unlock: 918 return ret; 919 } 920 921 void tracing_reset(struct trace_array *tr, int cpu) 922 { 923 struct ring_buffer *buffer = tr->buffer; 924 925 ring_buffer_record_disable(buffer); 926 927 /* Make sure all commits have finished */ 928 synchronize_sched(); 929 ring_buffer_reset_cpu(buffer, cpu); 930 931 ring_buffer_record_enable(buffer); 932 } 933 934 void tracing_reset_online_cpus(struct trace_array *tr) 935 { 936 struct ring_buffer *buffer = tr->buffer; 937 int cpu; 938 939 ring_buffer_record_disable(buffer); 940 941 /* Make sure all commits have finished */ 942 synchronize_sched(); 943 944 tr->time_start = ftrace_now(tr->cpu); 945 946 for_each_online_cpu(cpu) 947 ring_buffer_reset_cpu(buffer, cpu); 948 949 ring_buffer_record_enable(buffer); 950 } 951 952 void tracing_reset_current(int cpu) 953 { 954 tracing_reset(&global_trace, cpu); 955 } 956 957 void tracing_reset_current_online_cpus(void) 958 { 959 tracing_reset_online_cpus(&global_trace); 960 } 961 962 #define SAVED_CMDLINES 128 963 #define NO_CMDLINE_MAP UINT_MAX 964 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; 965 static unsigned map_cmdline_to_pid[SAVED_CMDLINES]; 966 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN]; 967 static int cmdline_idx; 968 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED; 969 970 /* temporary disable recording */ 971 static atomic_t trace_record_cmdline_disabled __read_mostly; 972 973 static void trace_init_cmdlines(void) 974 { 975 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline)); 976 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid)); 977 cmdline_idx = 0; 978 } 979 980 int is_tracing_stopped(void) 981 { 982 return trace_stop_count; 983 } 984 985 /** 986 * ftrace_off_permanent - disable all ftrace code permanently 987 * 988 * This should only be called when a serious anomally has 989 * been detected. This will turn off the function tracing, 990 * ring buffers, and other tracing utilites. It takes no 991 * locks and can be called from any context. 992 */ 993 void ftrace_off_permanent(void) 994 { 995 tracing_disabled = 1; 996 ftrace_stop(); 997 tracing_off_permanent(); 998 } 999 1000 /** 1001 * tracing_start - quick start of the tracer 1002 * 1003 * If tracing is enabled but was stopped by tracing_stop, 1004 * this will start the tracer back up. 1005 */ 1006 void tracing_start(void) 1007 { 1008 struct ring_buffer *buffer; 1009 unsigned long flags; 1010 1011 if (tracing_disabled) 1012 return; 1013 1014 raw_spin_lock_irqsave(&tracing_start_lock, flags); 1015 if (--trace_stop_count) { 1016 if (trace_stop_count < 0) { 1017 /* Someone screwed up their debugging */ 1018 WARN_ON_ONCE(1); 1019 trace_stop_count = 0; 1020 } 1021 goto out; 1022 } 1023 1024 /* Prevent the buffers from switching */ 1025 arch_spin_lock(&ftrace_max_lock); 1026 1027 buffer = global_trace.buffer; 1028 if (buffer) 1029 ring_buffer_record_enable(buffer); 1030 1031 buffer = max_tr.buffer; 1032 if (buffer) 1033 ring_buffer_record_enable(buffer); 1034 1035 arch_spin_unlock(&ftrace_max_lock); 1036 1037 ftrace_start(); 1038 out: 1039 raw_spin_unlock_irqrestore(&tracing_start_lock, flags); 1040 } 1041 1042 /** 1043 * tracing_stop - quick stop of the tracer 1044 * 1045 * Light weight way to stop tracing. Use in conjunction with 1046 * tracing_start. 1047 */ 1048 void tracing_stop(void) 1049 { 1050 struct ring_buffer *buffer; 1051 unsigned long flags; 1052 1053 ftrace_stop(); 1054 raw_spin_lock_irqsave(&tracing_start_lock, flags); 1055 if (trace_stop_count++) 1056 goto out; 1057 1058 /* Prevent the buffers from switching */ 1059 arch_spin_lock(&ftrace_max_lock); 1060 1061 buffer = global_trace.buffer; 1062 if (buffer) 1063 ring_buffer_record_disable(buffer); 1064 1065 buffer = max_tr.buffer; 1066 if (buffer) 1067 ring_buffer_record_disable(buffer); 1068 1069 arch_spin_unlock(&ftrace_max_lock); 1070 1071 out: 1072 raw_spin_unlock_irqrestore(&tracing_start_lock, flags); 1073 } 1074 1075 void trace_stop_cmdline_recording(void); 1076 1077 static void trace_save_cmdline(struct task_struct *tsk) 1078 { 1079 unsigned pid, idx; 1080 1081 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT)) 1082 return; 1083 1084 /* 1085 * It's not the end of the world if we don't get 1086 * the lock, but we also don't want to spin 1087 * nor do we want to disable interrupts, 1088 * so if we miss here, then better luck next time. 1089 */ 1090 if (!arch_spin_trylock(&trace_cmdline_lock)) 1091 return; 1092 1093 idx = map_pid_to_cmdline[tsk->pid]; 1094 if (idx == NO_CMDLINE_MAP) { 1095 idx = (cmdline_idx + 1) % SAVED_CMDLINES; 1096 1097 /* 1098 * Check whether the cmdline buffer at idx has a pid 1099 * mapped. We are going to overwrite that entry so we 1100 * need to clear the map_pid_to_cmdline. Otherwise we 1101 * would read the new comm for the old pid. 1102 */ 1103 pid = map_cmdline_to_pid[idx]; 1104 if (pid != NO_CMDLINE_MAP) 1105 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP; 1106 1107 map_cmdline_to_pid[idx] = tsk->pid; 1108 map_pid_to_cmdline[tsk->pid] = idx; 1109 1110 cmdline_idx = idx; 1111 } 1112 1113 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN); 1114 1115 arch_spin_unlock(&trace_cmdline_lock); 1116 } 1117 1118 void trace_find_cmdline(int pid, char comm[]) 1119 { 1120 unsigned map; 1121 1122 if (!pid) { 1123 strcpy(comm, "<idle>"); 1124 return; 1125 } 1126 1127 if (WARN_ON_ONCE(pid < 0)) { 1128 strcpy(comm, "<XXX>"); 1129 return; 1130 } 1131 1132 if (pid > PID_MAX_DEFAULT) { 1133 strcpy(comm, "<...>"); 1134 return; 1135 } 1136 1137 preempt_disable(); 1138 arch_spin_lock(&trace_cmdline_lock); 1139 map = map_pid_to_cmdline[pid]; 1140 if (map != NO_CMDLINE_MAP) 1141 strcpy(comm, saved_cmdlines[map]); 1142 else 1143 strcpy(comm, "<...>"); 1144 1145 arch_spin_unlock(&trace_cmdline_lock); 1146 preempt_enable(); 1147 } 1148 1149 void tracing_record_cmdline(struct task_struct *tsk) 1150 { 1151 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on()) 1152 return; 1153 1154 if (!__this_cpu_read(trace_cmdline_save)) 1155 return; 1156 1157 __this_cpu_write(trace_cmdline_save, false); 1158 1159 trace_save_cmdline(tsk); 1160 } 1161 1162 void 1163 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags, 1164 int pc) 1165 { 1166 struct task_struct *tsk = current; 1167 1168 entry->preempt_count = pc & 0xff; 1169 entry->pid = (tsk) ? tsk->pid : 0; 1170 entry->padding = 0; 1171 entry->flags = 1172 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT 1173 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) | 1174 #else 1175 TRACE_FLAG_IRQS_NOSUPPORT | 1176 #endif 1177 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) | 1178 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) | 1179 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0); 1180 } 1181 EXPORT_SYMBOL_GPL(tracing_generic_entry_update); 1182 1183 struct ring_buffer_event * 1184 trace_buffer_lock_reserve(struct ring_buffer *buffer, 1185 int type, 1186 unsigned long len, 1187 unsigned long flags, int pc) 1188 { 1189 struct ring_buffer_event *event; 1190 1191 event = ring_buffer_lock_reserve(buffer, len); 1192 if (event != NULL) { 1193 struct trace_entry *ent = ring_buffer_event_data(event); 1194 1195 tracing_generic_entry_update(ent, flags, pc); 1196 ent->type = type; 1197 } 1198 1199 return event; 1200 } 1201 1202 void 1203 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event) 1204 { 1205 __this_cpu_write(trace_cmdline_save, true); 1206 if (trace_wakeup_needed) { 1207 trace_wakeup_needed = false; 1208 /* irq_work_queue() supplies it's own memory barriers */ 1209 irq_work_queue(&trace_work_wakeup); 1210 } 1211 ring_buffer_unlock_commit(buffer, event); 1212 } 1213 1214 static inline void 1215 __trace_buffer_unlock_commit(struct ring_buffer *buffer, 1216 struct ring_buffer_event *event, 1217 unsigned long flags, int pc) 1218 { 1219 __buffer_unlock_commit(buffer, event); 1220 1221 ftrace_trace_stack(buffer, flags, 6, pc); 1222 ftrace_trace_userstack(buffer, flags, pc); 1223 } 1224 1225 void trace_buffer_unlock_commit(struct ring_buffer *buffer, 1226 struct ring_buffer_event *event, 1227 unsigned long flags, int pc) 1228 { 1229 __trace_buffer_unlock_commit(buffer, event, flags, pc); 1230 } 1231 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit); 1232 1233 struct ring_buffer_event * 1234 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb, 1235 int type, unsigned long len, 1236 unsigned long flags, int pc) 1237 { 1238 *current_rb = global_trace.buffer; 1239 return trace_buffer_lock_reserve(*current_rb, 1240 type, len, flags, pc); 1241 } 1242 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve); 1243 1244 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer, 1245 struct ring_buffer_event *event, 1246 unsigned long flags, int pc) 1247 { 1248 __trace_buffer_unlock_commit(buffer, event, flags, pc); 1249 } 1250 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit); 1251 1252 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer, 1253 struct ring_buffer_event *event, 1254 unsigned long flags, int pc, 1255 struct pt_regs *regs) 1256 { 1257 __buffer_unlock_commit(buffer, event); 1258 1259 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs); 1260 ftrace_trace_userstack(buffer, flags, pc); 1261 } 1262 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs); 1263 1264 void trace_current_buffer_discard_commit(struct ring_buffer *buffer, 1265 struct ring_buffer_event *event) 1266 { 1267 ring_buffer_discard_commit(buffer, event); 1268 } 1269 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit); 1270 1271 void 1272 trace_function(struct trace_array *tr, 1273 unsigned long ip, unsigned long parent_ip, unsigned long flags, 1274 int pc) 1275 { 1276 struct ftrace_event_call *call = &event_function; 1277 struct ring_buffer *buffer = tr->buffer; 1278 struct ring_buffer_event *event; 1279 struct ftrace_entry *entry; 1280 1281 /* If we are reading the ring buffer, don't trace */ 1282 if (unlikely(__this_cpu_read(ftrace_cpu_disabled))) 1283 return; 1284 1285 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), 1286 flags, pc); 1287 if (!event) 1288 return; 1289 entry = ring_buffer_event_data(event); 1290 entry->ip = ip; 1291 entry->parent_ip = parent_ip; 1292 1293 if (!filter_check_discard(call, entry, buffer, event)) 1294 __buffer_unlock_commit(buffer, event); 1295 } 1296 1297 void 1298 ftrace(struct trace_array *tr, struct trace_array_cpu *data, 1299 unsigned long ip, unsigned long parent_ip, unsigned long flags, 1300 int pc) 1301 { 1302 if (likely(!atomic_read(&data->disabled))) 1303 trace_function(tr, ip, parent_ip, flags, pc); 1304 } 1305 1306 #ifdef CONFIG_STACKTRACE 1307 1308 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long)) 1309 struct ftrace_stack { 1310 unsigned long calls[FTRACE_STACK_MAX_ENTRIES]; 1311 }; 1312 1313 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack); 1314 static DEFINE_PER_CPU(int, ftrace_stack_reserve); 1315 1316 static void __ftrace_trace_stack(struct ring_buffer *buffer, 1317 unsigned long flags, 1318 int skip, int pc, struct pt_regs *regs) 1319 { 1320 struct ftrace_event_call *call = &event_kernel_stack; 1321 struct ring_buffer_event *event; 1322 struct stack_entry *entry; 1323 struct stack_trace trace; 1324 int use_stack; 1325 int size = FTRACE_STACK_ENTRIES; 1326 1327 trace.nr_entries = 0; 1328 trace.skip = skip; 1329 1330 /* 1331 * Since events can happen in NMIs there's no safe way to 1332 * use the per cpu ftrace_stacks. We reserve it and if an interrupt 1333 * or NMI comes in, it will just have to use the default 1334 * FTRACE_STACK_SIZE. 1335 */ 1336 preempt_disable_notrace(); 1337 1338 use_stack = ++__get_cpu_var(ftrace_stack_reserve); 1339 /* 1340 * We don't need any atomic variables, just a barrier. 1341 * If an interrupt comes in, we don't care, because it would 1342 * have exited and put the counter back to what we want. 1343 * We just need a barrier to keep gcc from moving things 1344 * around. 1345 */ 1346 barrier(); 1347 if (use_stack == 1) { 1348 trace.entries = &__get_cpu_var(ftrace_stack).calls[0]; 1349 trace.max_entries = FTRACE_STACK_MAX_ENTRIES; 1350 1351 if (regs) 1352 save_stack_trace_regs(regs, &trace); 1353 else 1354 save_stack_trace(&trace); 1355 1356 if (trace.nr_entries > size) 1357 size = trace.nr_entries; 1358 } else 1359 /* From now on, use_stack is a boolean */ 1360 use_stack = 0; 1361 1362 size *= sizeof(unsigned long); 1363 1364 event = trace_buffer_lock_reserve(buffer, TRACE_STACK, 1365 sizeof(*entry) + size, flags, pc); 1366 if (!event) 1367 goto out; 1368 entry = ring_buffer_event_data(event); 1369 1370 memset(&entry->caller, 0, size); 1371 1372 if (use_stack) 1373 memcpy(&entry->caller, trace.entries, 1374 trace.nr_entries * sizeof(unsigned long)); 1375 else { 1376 trace.max_entries = FTRACE_STACK_ENTRIES; 1377 trace.entries = entry->caller; 1378 if (regs) 1379 save_stack_trace_regs(regs, &trace); 1380 else 1381 save_stack_trace(&trace); 1382 } 1383 1384 entry->size = trace.nr_entries; 1385 1386 if (!filter_check_discard(call, entry, buffer, event)) 1387 __buffer_unlock_commit(buffer, event); 1388 1389 out: 1390 /* Again, don't let gcc optimize things here */ 1391 barrier(); 1392 __get_cpu_var(ftrace_stack_reserve)--; 1393 preempt_enable_notrace(); 1394 1395 } 1396 1397 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags, 1398 int skip, int pc, struct pt_regs *regs) 1399 { 1400 if (!(trace_flags & TRACE_ITER_STACKTRACE)) 1401 return; 1402 1403 __ftrace_trace_stack(buffer, flags, skip, pc, regs); 1404 } 1405 1406 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags, 1407 int skip, int pc) 1408 { 1409 if (!(trace_flags & TRACE_ITER_STACKTRACE)) 1410 return; 1411 1412 __ftrace_trace_stack(buffer, flags, skip, pc, NULL); 1413 } 1414 1415 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, 1416 int pc) 1417 { 1418 __ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL); 1419 } 1420 1421 /** 1422 * trace_dump_stack - record a stack back trace in the trace buffer 1423 */ 1424 void trace_dump_stack(void) 1425 { 1426 unsigned long flags; 1427 1428 if (tracing_disabled || tracing_selftest_running) 1429 return; 1430 1431 local_save_flags(flags); 1432 1433 /* skipping 3 traces, seems to get us at the caller of this function */ 1434 __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL); 1435 } 1436 1437 static DEFINE_PER_CPU(int, user_stack_count); 1438 1439 void 1440 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc) 1441 { 1442 struct ftrace_event_call *call = &event_user_stack; 1443 struct ring_buffer_event *event; 1444 struct userstack_entry *entry; 1445 struct stack_trace trace; 1446 1447 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE)) 1448 return; 1449 1450 /* 1451 * NMIs can not handle page faults, even with fix ups. 1452 * The save user stack can (and often does) fault. 1453 */ 1454 if (unlikely(in_nmi())) 1455 return; 1456 1457 /* 1458 * prevent recursion, since the user stack tracing may 1459 * trigger other kernel events. 1460 */ 1461 preempt_disable(); 1462 if (__this_cpu_read(user_stack_count)) 1463 goto out; 1464 1465 __this_cpu_inc(user_stack_count); 1466 1467 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, 1468 sizeof(*entry), flags, pc); 1469 if (!event) 1470 goto out_drop_count; 1471 entry = ring_buffer_event_data(event); 1472 1473 entry->tgid = current->tgid; 1474 memset(&entry->caller, 0, sizeof(entry->caller)); 1475 1476 trace.nr_entries = 0; 1477 trace.max_entries = FTRACE_STACK_ENTRIES; 1478 trace.skip = 0; 1479 trace.entries = entry->caller; 1480 1481 save_stack_trace_user(&trace); 1482 if (!filter_check_discard(call, entry, buffer, event)) 1483 __buffer_unlock_commit(buffer, event); 1484 1485 out_drop_count: 1486 __this_cpu_dec(user_stack_count); 1487 out: 1488 preempt_enable(); 1489 } 1490 1491 #ifdef UNUSED 1492 static void __trace_userstack(struct trace_array *tr, unsigned long flags) 1493 { 1494 ftrace_trace_userstack(tr, flags, preempt_count()); 1495 } 1496 #endif /* UNUSED */ 1497 1498 #endif /* CONFIG_STACKTRACE */ 1499 1500 /* created for use with alloc_percpu */ 1501 struct trace_buffer_struct { 1502 char buffer[TRACE_BUF_SIZE]; 1503 }; 1504 1505 static struct trace_buffer_struct *trace_percpu_buffer; 1506 static struct trace_buffer_struct *trace_percpu_sirq_buffer; 1507 static struct trace_buffer_struct *trace_percpu_irq_buffer; 1508 static struct trace_buffer_struct *trace_percpu_nmi_buffer; 1509 1510 /* 1511 * The buffer used is dependent on the context. There is a per cpu 1512 * buffer for normal context, softirq contex, hard irq context and 1513 * for NMI context. Thise allows for lockless recording. 1514 * 1515 * Note, if the buffers failed to be allocated, then this returns NULL 1516 */ 1517 static char *get_trace_buf(void) 1518 { 1519 struct trace_buffer_struct *percpu_buffer; 1520 struct trace_buffer_struct *buffer; 1521 1522 /* 1523 * If we have allocated per cpu buffers, then we do not 1524 * need to do any locking. 1525 */ 1526 if (in_nmi()) 1527 percpu_buffer = trace_percpu_nmi_buffer; 1528 else if (in_irq()) 1529 percpu_buffer = trace_percpu_irq_buffer; 1530 else if (in_softirq()) 1531 percpu_buffer = trace_percpu_sirq_buffer; 1532 else 1533 percpu_buffer = trace_percpu_buffer; 1534 1535 if (!percpu_buffer) 1536 return NULL; 1537 1538 buffer = per_cpu_ptr(percpu_buffer, smp_processor_id()); 1539 1540 return buffer->buffer; 1541 } 1542 1543 static int alloc_percpu_trace_buffer(void) 1544 { 1545 struct trace_buffer_struct *buffers; 1546 struct trace_buffer_struct *sirq_buffers; 1547 struct trace_buffer_struct *irq_buffers; 1548 struct trace_buffer_struct *nmi_buffers; 1549 1550 buffers = alloc_percpu(struct trace_buffer_struct); 1551 if (!buffers) 1552 goto err_warn; 1553 1554 sirq_buffers = alloc_percpu(struct trace_buffer_struct); 1555 if (!sirq_buffers) 1556 goto err_sirq; 1557 1558 irq_buffers = alloc_percpu(struct trace_buffer_struct); 1559 if (!irq_buffers) 1560 goto err_irq; 1561 1562 nmi_buffers = alloc_percpu(struct trace_buffer_struct); 1563 if (!nmi_buffers) 1564 goto err_nmi; 1565 1566 trace_percpu_buffer = buffers; 1567 trace_percpu_sirq_buffer = sirq_buffers; 1568 trace_percpu_irq_buffer = irq_buffers; 1569 trace_percpu_nmi_buffer = nmi_buffers; 1570 1571 return 0; 1572 1573 err_nmi: 1574 free_percpu(irq_buffers); 1575 err_irq: 1576 free_percpu(sirq_buffers); 1577 err_sirq: 1578 free_percpu(buffers); 1579 err_warn: 1580 WARN(1, "Could not allocate percpu trace_printk buffer"); 1581 return -ENOMEM; 1582 } 1583 1584 static int buffers_allocated; 1585 1586 void trace_printk_init_buffers(void) 1587 { 1588 if (buffers_allocated) 1589 return; 1590 1591 if (alloc_percpu_trace_buffer()) 1592 return; 1593 1594 pr_info("ftrace: Allocated trace_printk buffers\n"); 1595 1596 /* Expand the buffers to set size */ 1597 tracing_update_buffers(); 1598 1599 buffers_allocated = 1; 1600 1601 /* 1602 * trace_printk_init_buffers() can be called by modules. 1603 * If that happens, then we need to start cmdline recording 1604 * directly here. If the global_trace.buffer is already 1605 * allocated here, then this was called by module code. 1606 */ 1607 if (global_trace.buffer) 1608 tracing_start_cmdline_record(); 1609 } 1610 1611 void trace_printk_start_comm(void) 1612 { 1613 /* Start tracing comms if trace printk is set */ 1614 if (!buffers_allocated) 1615 return; 1616 tracing_start_cmdline_record(); 1617 } 1618 1619 static void trace_printk_start_stop_comm(int enabled) 1620 { 1621 if (!buffers_allocated) 1622 return; 1623 1624 if (enabled) 1625 tracing_start_cmdline_record(); 1626 else 1627 tracing_stop_cmdline_record(); 1628 } 1629 1630 /** 1631 * trace_vbprintk - write binary msg to tracing buffer 1632 * 1633 */ 1634 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) 1635 { 1636 struct ftrace_event_call *call = &event_bprint; 1637 struct ring_buffer_event *event; 1638 struct ring_buffer *buffer; 1639 struct trace_array *tr = &global_trace; 1640 struct bprint_entry *entry; 1641 unsigned long flags; 1642 char *tbuffer; 1643 int len = 0, size, pc; 1644 1645 if (unlikely(tracing_selftest_running || tracing_disabled)) 1646 return 0; 1647 1648 /* Don't pollute graph traces with trace_vprintk internals */ 1649 pause_graph_tracing(); 1650 1651 pc = preempt_count(); 1652 preempt_disable_notrace(); 1653 1654 tbuffer = get_trace_buf(); 1655 if (!tbuffer) { 1656 len = 0; 1657 goto out; 1658 } 1659 1660 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args); 1661 1662 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0) 1663 goto out; 1664 1665 local_save_flags(flags); 1666 size = sizeof(*entry) + sizeof(u32) * len; 1667 buffer = tr->buffer; 1668 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, 1669 flags, pc); 1670 if (!event) 1671 goto out; 1672 entry = ring_buffer_event_data(event); 1673 entry->ip = ip; 1674 entry->fmt = fmt; 1675 1676 memcpy(entry->buf, tbuffer, sizeof(u32) * len); 1677 if (!filter_check_discard(call, entry, buffer, event)) { 1678 __buffer_unlock_commit(buffer, event); 1679 ftrace_trace_stack(buffer, flags, 6, pc); 1680 } 1681 1682 out: 1683 preempt_enable_notrace(); 1684 unpause_graph_tracing(); 1685 1686 return len; 1687 } 1688 EXPORT_SYMBOL_GPL(trace_vbprintk); 1689 1690 int trace_array_printk(struct trace_array *tr, 1691 unsigned long ip, const char *fmt, ...) 1692 { 1693 int ret; 1694 va_list ap; 1695 1696 if (!(trace_flags & TRACE_ITER_PRINTK)) 1697 return 0; 1698 1699 va_start(ap, fmt); 1700 ret = trace_array_vprintk(tr, ip, fmt, ap); 1701 va_end(ap); 1702 return ret; 1703 } 1704 1705 int trace_array_vprintk(struct trace_array *tr, 1706 unsigned long ip, const char *fmt, va_list args) 1707 { 1708 struct ftrace_event_call *call = &event_print; 1709 struct ring_buffer_event *event; 1710 struct ring_buffer *buffer; 1711 int len = 0, size, pc; 1712 struct print_entry *entry; 1713 unsigned long flags; 1714 char *tbuffer; 1715 1716 if (tracing_disabled || tracing_selftest_running) 1717 return 0; 1718 1719 /* Don't pollute graph traces with trace_vprintk internals */ 1720 pause_graph_tracing(); 1721 1722 pc = preempt_count(); 1723 preempt_disable_notrace(); 1724 1725 1726 tbuffer = get_trace_buf(); 1727 if (!tbuffer) { 1728 len = 0; 1729 goto out; 1730 } 1731 1732 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args); 1733 if (len > TRACE_BUF_SIZE) 1734 goto out; 1735 1736 local_save_flags(flags); 1737 size = sizeof(*entry) + len + 1; 1738 buffer = tr->buffer; 1739 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 1740 flags, pc); 1741 if (!event) 1742 goto out; 1743 entry = ring_buffer_event_data(event); 1744 entry->ip = ip; 1745 1746 memcpy(&entry->buf, tbuffer, len); 1747 entry->buf[len] = '\0'; 1748 if (!filter_check_discard(call, entry, buffer, event)) { 1749 __buffer_unlock_commit(buffer, event); 1750 ftrace_trace_stack(buffer, flags, 6, pc); 1751 } 1752 out: 1753 preempt_enable_notrace(); 1754 unpause_graph_tracing(); 1755 1756 return len; 1757 } 1758 1759 int trace_vprintk(unsigned long ip, const char *fmt, va_list args) 1760 { 1761 return trace_array_vprintk(&global_trace, ip, fmt, args); 1762 } 1763 EXPORT_SYMBOL_GPL(trace_vprintk); 1764 1765 static void trace_iterator_increment(struct trace_iterator *iter) 1766 { 1767 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu); 1768 1769 iter->idx++; 1770 if (buf_iter) 1771 ring_buffer_read(buf_iter, NULL); 1772 } 1773 1774 static struct trace_entry * 1775 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts, 1776 unsigned long *lost_events) 1777 { 1778 struct ring_buffer_event *event; 1779 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu); 1780 1781 if (buf_iter) 1782 event = ring_buffer_iter_peek(buf_iter, ts); 1783 else 1784 event = ring_buffer_peek(iter->tr->buffer, cpu, ts, 1785 lost_events); 1786 1787 if (event) { 1788 iter->ent_size = ring_buffer_event_length(event); 1789 return ring_buffer_event_data(event); 1790 } 1791 iter->ent_size = 0; 1792 return NULL; 1793 } 1794 1795 static struct trace_entry * 1796 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, 1797 unsigned long *missing_events, u64 *ent_ts) 1798 { 1799 struct ring_buffer *buffer = iter->tr->buffer; 1800 struct trace_entry *ent, *next = NULL; 1801 unsigned long lost_events = 0, next_lost = 0; 1802 int cpu_file = iter->cpu_file; 1803 u64 next_ts = 0, ts; 1804 int next_cpu = -1; 1805 int next_size = 0; 1806 int cpu; 1807 1808 /* 1809 * If we are in a per_cpu trace file, don't bother by iterating over 1810 * all cpu and peek directly. 1811 */ 1812 if (cpu_file > TRACE_PIPE_ALL_CPU) { 1813 if (ring_buffer_empty_cpu(buffer, cpu_file)) 1814 return NULL; 1815 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events); 1816 if (ent_cpu) 1817 *ent_cpu = cpu_file; 1818 1819 return ent; 1820 } 1821 1822 for_each_tracing_cpu(cpu) { 1823 1824 if (ring_buffer_empty_cpu(buffer, cpu)) 1825 continue; 1826 1827 ent = peek_next_entry(iter, cpu, &ts, &lost_events); 1828 1829 /* 1830 * Pick the entry with the smallest timestamp: 1831 */ 1832 if (ent && (!next || ts < next_ts)) { 1833 next = ent; 1834 next_cpu = cpu; 1835 next_ts = ts; 1836 next_lost = lost_events; 1837 next_size = iter->ent_size; 1838 } 1839 } 1840 1841 iter->ent_size = next_size; 1842 1843 if (ent_cpu) 1844 *ent_cpu = next_cpu; 1845 1846 if (ent_ts) 1847 *ent_ts = next_ts; 1848 1849 if (missing_events) 1850 *missing_events = next_lost; 1851 1852 return next; 1853 } 1854 1855 /* Find the next real entry, without updating the iterator itself */ 1856 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter, 1857 int *ent_cpu, u64 *ent_ts) 1858 { 1859 return __find_next_entry(iter, ent_cpu, NULL, ent_ts); 1860 } 1861 1862 /* Find the next real entry, and increment the iterator to the next entry */ 1863 void *trace_find_next_entry_inc(struct trace_iterator *iter) 1864 { 1865 iter->ent = __find_next_entry(iter, &iter->cpu, 1866 &iter->lost_events, &iter->ts); 1867 1868 if (iter->ent) 1869 trace_iterator_increment(iter); 1870 1871 return iter->ent ? iter : NULL; 1872 } 1873 1874 static void trace_consume(struct trace_iterator *iter) 1875 { 1876 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts, 1877 &iter->lost_events); 1878 } 1879 1880 static void *s_next(struct seq_file *m, void *v, loff_t *pos) 1881 { 1882 struct trace_iterator *iter = m->private; 1883 int i = (int)*pos; 1884 void *ent; 1885 1886 WARN_ON_ONCE(iter->leftover); 1887 1888 (*pos)++; 1889 1890 /* can't go backwards */ 1891 if (iter->idx > i) 1892 return NULL; 1893 1894 if (iter->idx < 0) 1895 ent = trace_find_next_entry_inc(iter); 1896 else 1897 ent = iter; 1898 1899 while (ent && iter->idx < i) 1900 ent = trace_find_next_entry_inc(iter); 1901 1902 iter->pos = *pos; 1903 1904 return ent; 1905 } 1906 1907 void tracing_iter_reset(struct trace_iterator *iter, int cpu) 1908 { 1909 struct trace_array *tr = iter->tr; 1910 struct ring_buffer_event *event; 1911 struct ring_buffer_iter *buf_iter; 1912 unsigned long entries = 0; 1913 u64 ts; 1914 1915 tr->data[cpu]->skipped_entries = 0; 1916 1917 buf_iter = trace_buffer_iter(iter, cpu); 1918 if (!buf_iter) 1919 return; 1920 1921 ring_buffer_iter_reset(buf_iter); 1922 1923 /* 1924 * We could have the case with the max latency tracers 1925 * that a reset never took place on a cpu. This is evident 1926 * by the timestamp being before the start of the buffer. 1927 */ 1928 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) { 1929 if (ts >= iter->tr->time_start) 1930 break; 1931 entries++; 1932 ring_buffer_read(buf_iter, NULL); 1933 } 1934 1935 tr->data[cpu]->skipped_entries = entries; 1936 } 1937 1938 /* 1939 * The current tracer is copied to avoid a global locking 1940 * all around. 1941 */ 1942 static void *s_start(struct seq_file *m, loff_t *pos) 1943 { 1944 struct trace_iterator *iter = m->private; 1945 static struct tracer *old_tracer; 1946 int cpu_file = iter->cpu_file; 1947 void *p = NULL; 1948 loff_t l = 0; 1949 int cpu; 1950 1951 /* copy the tracer to avoid using a global lock all around */ 1952 mutex_lock(&trace_types_lock); 1953 if (unlikely(old_tracer != current_trace && current_trace)) { 1954 old_tracer = current_trace; 1955 *iter->trace = *current_trace; 1956 } 1957 mutex_unlock(&trace_types_lock); 1958 1959 atomic_inc(&trace_record_cmdline_disabled); 1960 1961 if (*pos != iter->pos) { 1962 iter->ent = NULL; 1963 iter->cpu = 0; 1964 iter->idx = -1; 1965 1966 if (cpu_file == TRACE_PIPE_ALL_CPU) { 1967 for_each_tracing_cpu(cpu) 1968 tracing_iter_reset(iter, cpu); 1969 } else 1970 tracing_iter_reset(iter, cpu_file); 1971 1972 iter->leftover = 0; 1973 for (p = iter; p && l < *pos; p = s_next(m, p, &l)) 1974 ; 1975 1976 } else { 1977 /* 1978 * If we overflowed the seq_file before, then we want 1979 * to just reuse the trace_seq buffer again. 1980 */ 1981 if (iter->leftover) 1982 p = iter; 1983 else { 1984 l = *pos - 1; 1985 p = s_next(m, p, &l); 1986 } 1987 } 1988 1989 trace_event_read_lock(); 1990 trace_access_lock(cpu_file); 1991 return p; 1992 } 1993 1994 static void s_stop(struct seq_file *m, void *p) 1995 { 1996 struct trace_iterator *iter = m->private; 1997 1998 atomic_dec(&trace_record_cmdline_disabled); 1999 trace_access_unlock(iter->cpu_file); 2000 trace_event_read_unlock(); 2001 } 2002 2003 static void 2004 get_total_entries(struct trace_array *tr, unsigned long *total, unsigned long *entries) 2005 { 2006 unsigned long count; 2007 int cpu; 2008 2009 *total = 0; 2010 *entries = 0; 2011 2012 for_each_tracing_cpu(cpu) { 2013 count = ring_buffer_entries_cpu(tr->buffer, cpu); 2014 /* 2015 * If this buffer has skipped entries, then we hold all 2016 * entries for the trace and we need to ignore the 2017 * ones before the time stamp. 2018 */ 2019 if (tr->data[cpu]->skipped_entries) { 2020 count -= tr->data[cpu]->skipped_entries; 2021 /* total is the same as the entries */ 2022 *total += count; 2023 } else 2024 *total += count + 2025 ring_buffer_overrun_cpu(tr->buffer, cpu); 2026 *entries += count; 2027 } 2028 } 2029 2030 static void print_lat_help_header(struct seq_file *m) 2031 { 2032 seq_puts(m, "# _------=> CPU# \n"); 2033 seq_puts(m, "# / _-----=> irqs-off \n"); 2034 seq_puts(m, "# | / _----=> need-resched \n"); 2035 seq_puts(m, "# || / _---=> hardirq/softirq \n"); 2036 seq_puts(m, "# ||| / _--=> preempt-depth \n"); 2037 seq_puts(m, "# |||| / delay \n"); 2038 seq_puts(m, "# cmd pid ||||| time | caller \n"); 2039 seq_puts(m, "# \\ / ||||| \\ | / \n"); 2040 } 2041 2042 static void print_event_info(struct trace_array *tr, struct seq_file *m) 2043 { 2044 unsigned long total; 2045 unsigned long entries; 2046 2047 get_total_entries(tr, &total, &entries); 2048 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n", 2049 entries, total, num_online_cpus()); 2050 seq_puts(m, "#\n"); 2051 } 2052 2053 static void print_func_help_header(struct trace_array *tr, struct seq_file *m) 2054 { 2055 print_event_info(tr, m); 2056 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"); 2057 seq_puts(m, "# | | | | |\n"); 2058 } 2059 2060 static void print_func_help_header_irq(struct trace_array *tr, struct seq_file *m) 2061 { 2062 print_event_info(tr, m); 2063 seq_puts(m, "# _-----=> irqs-off\n"); 2064 seq_puts(m, "# / _----=> need-resched\n"); 2065 seq_puts(m, "# | / _---=> hardirq/softirq\n"); 2066 seq_puts(m, "# || / _--=> preempt-depth\n"); 2067 seq_puts(m, "# ||| / delay\n"); 2068 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"); 2069 seq_puts(m, "# | | | |||| | |\n"); 2070 } 2071 2072 void 2073 print_trace_header(struct seq_file *m, struct trace_iterator *iter) 2074 { 2075 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK); 2076 struct trace_array *tr = iter->tr; 2077 struct trace_array_cpu *data = tr->data[tr->cpu]; 2078 struct tracer *type = current_trace; 2079 unsigned long entries; 2080 unsigned long total; 2081 const char *name = "preemption"; 2082 2083 if (type) 2084 name = type->name; 2085 2086 get_total_entries(tr, &total, &entries); 2087 2088 seq_printf(m, "# %s latency trace v1.1.5 on %s\n", 2089 name, UTS_RELEASE); 2090 seq_puts(m, "# -----------------------------------" 2091 "---------------------------------\n"); 2092 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |" 2093 " (M:%s VP:%d, KP:%d, SP:%d HP:%d", 2094 nsecs_to_usecs(data->saved_latency), 2095 entries, 2096 total, 2097 tr->cpu, 2098 #if defined(CONFIG_PREEMPT_NONE) 2099 "server", 2100 #elif defined(CONFIG_PREEMPT_VOLUNTARY) 2101 "desktop", 2102 #elif defined(CONFIG_PREEMPT) 2103 "preempt", 2104 #else 2105 "unknown", 2106 #endif 2107 /* These are reserved for later use */ 2108 0, 0, 0, 0); 2109 #ifdef CONFIG_SMP 2110 seq_printf(m, " #P:%d)\n", num_online_cpus()); 2111 #else 2112 seq_puts(m, ")\n"); 2113 #endif 2114 seq_puts(m, "# -----------------\n"); 2115 seq_printf(m, "# | task: %.16s-%d " 2116 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n", 2117 data->comm, data->pid, 2118 from_kuid_munged(seq_user_ns(m), data->uid), data->nice, 2119 data->policy, data->rt_priority); 2120 seq_puts(m, "# -----------------\n"); 2121 2122 if (data->critical_start) { 2123 seq_puts(m, "# => started at: "); 2124 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags); 2125 trace_print_seq(m, &iter->seq); 2126 seq_puts(m, "\n# => ended at: "); 2127 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); 2128 trace_print_seq(m, &iter->seq); 2129 seq_puts(m, "\n#\n"); 2130 } 2131 2132 seq_puts(m, "#\n"); 2133 } 2134 2135 static void test_cpu_buff_start(struct trace_iterator *iter) 2136 { 2137 struct trace_seq *s = &iter->seq; 2138 2139 if (!(trace_flags & TRACE_ITER_ANNOTATE)) 2140 return; 2141 2142 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE)) 2143 return; 2144 2145 if (cpumask_test_cpu(iter->cpu, iter->started)) 2146 return; 2147 2148 if (iter->tr->data[iter->cpu]->skipped_entries) 2149 return; 2150 2151 cpumask_set_cpu(iter->cpu, iter->started); 2152 2153 /* Don't print started cpu buffer for the first entry of the trace */ 2154 if (iter->idx > 1) 2155 trace_seq_printf(s, "##### CPU %u buffer started ####\n", 2156 iter->cpu); 2157 } 2158 2159 static enum print_line_t print_trace_fmt(struct trace_iterator *iter) 2160 { 2161 struct trace_seq *s = &iter->seq; 2162 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK); 2163 struct trace_entry *entry; 2164 struct trace_event *event; 2165 2166 entry = iter->ent; 2167 2168 test_cpu_buff_start(iter); 2169 2170 event = ftrace_find_event(entry->type); 2171 2172 if (trace_flags & TRACE_ITER_CONTEXT_INFO) { 2173 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { 2174 if (!trace_print_lat_context(iter)) 2175 goto partial; 2176 } else { 2177 if (!trace_print_context(iter)) 2178 goto partial; 2179 } 2180 } 2181 2182 if (event) 2183 return event->funcs->trace(iter, sym_flags, event); 2184 2185 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type)) 2186 goto partial; 2187 2188 return TRACE_TYPE_HANDLED; 2189 partial: 2190 return TRACE_TYPE_PARTIAL_LINE; 2191 } 2192 2193 static enum print_line_t print_raw_fmt(struct trace_iterator *iter) 2194 { 2195 struct trace_seq *s = &iter->seq; 2196 struct trace_entry *entry; 2197 struct trace_event *event; 2198 2199 entry = iter->ent; 2200 2201 if (trace_flags & TRACE_ITER_CONTEXT_INFO) { 2202 if (!trace_seq_printf(s, "%d %d %llu ", 2203 entry->pid, iter->cpu, iter->ts)) 2204 goto partial; 2205 } 2206 2207 event = ftrace_find_event(entry->type); 2208 if (event) 2209 return event->funcs->raw(iter, 0, event); 2210 2211 if (!trace_seq_printf(s, "%d ?\n", entry->type)) 2212 goto partial; 2213 2214 return TRACE_TYPE_HANDLED; 2215 partial: 2216 return TRACE_TYPE_PARTIAL_LINE; 2217 } 2218 2219 static enum print_line_t print_hex_fmt(struct trace_iterator *iter) 2220 { 2221 struct trace_seq *s = &iter->seq; 2222 unsigned char newline = '\n'; 2223 struct trace_entry *entry; 2224 struct trace_event *event; 2225 2226 entry = iter->ent; 2227 2228 if (trace_flags & TRACE_ITER_CONTEXT_INFO) { 2229 SEQ_PUT_HEX_FIELD_RET(s, entry->pid); 2230 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu); 2231 SEQ_PUT_HEX_FIELD_RET(s, iter->ts); 2232 } 2233 2234 event = ftrace_find_event(entry->type); 2235 if (event) { 2236 enum print_line_t ret = event->funcs->hex(iter, 0, event); 2237 if (ret != TRACE_TYPE_HANDLED) 2238 return ret; 2239 } 2240 2241 SEQ_PUT_FIELD_RET(s, newline); 2242 2243 return TRACE_TYPE_HANDLED; 2244 } 2245 2246 static enum print_line_t print_bin_fmt(struct trace_iterator *iter) 2247 { 2248 struct trace_seq *s = &iter->seq; 2249 struct trace_entry *entry; 2250 struct trace_event *event; 2251 2252 entry = iter->ent; 2253 2254 if (trace_flags & TRACE_ITER_CONTEXT_INFO) { 2255 SEQ_PUT_FIELD_RET(s, entry->pid); 2256 SEQ_PUT_FIELD_RET(s, iter->cpu); 2257 SEQ_PUT_FIELD_RET(s, iter->ts); 2258 } 2259 2260 event = ftrace_find_event(entry->type); 2261 return event ? event->funcs->binary(iter, 0, event) : 2262 TRACE_TYPE_HANDLED; 2263 } 2264 2265 int trace_empty(struct trace_iterator *iter) 2266 { 2267 struct ring_buffer_iter *buf_iter; 2268 int cpu; 2269 2270 /* If we are looking at one CPU buffer, only check that one */ 2271 if (iter->cpu_file != TRACE_PIPE_ALL_CPU) { 2272 cpu = iter->cpu_file; 2273 buf_iter = trace_buffer_iter(iter, cpu); 2274 if (buf_iter) { 2275 if (!ring_buffer_iter_empty(buf_iter)) 2276 return 0; 2277 } else { 2278 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu)) 2279 return 0; 2280 } 2281 return 1; 2282 } 2283 2284 for_each_tracing_cpu(cpu) { 2285 buf_iter = trace_buffer_iter(iter, cpu); 2286 if (buf_iter) { 2287 if (!ring_buffer_iter_empty(buf_iter)) 2288 return 0; 2289 } else { 2290 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu)) 2291 return 0; 2292 } 2293 } 2294 2295 return 1; 2296 } 2297 2298 /* Called with trace_event_read_lock() held. */ 2299 enum print_line_t print_trace_line(struct trace_iterator *iter) 2300 { 2301 enum print_line_t ret; 2302 2303 if (iter->lost_events && 2304 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", 2305 iter->cpu, iter->lost_events)) 2306 return TRACE_TYPE_PARTIAL_LINE; 2307 2308 if (iter->trace && iter->trace->print_line) { 2309 ret = iter->trace->print_line(iter); 2310 if (ret != TRACE_TYPE_UNHANDLED) 2311 return ret; 2312 } 2313 2314 if (iter->ent->type == TRACE_BPRINT && 2315 trace_flags & TRACE_ITER_PRINTK && 2316 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 2317 return trace_print_bprintk_msg_only(iter); 2318 2319 if (iter->ent->type == TRACE_PRINT && 2320 trace_flags & TRACE_ITER_PRINTK && 2321 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 2322 return trace_print_printk_msg_only(iter); 2323 2324 if (trace_flags & TRACE_ITER_BIN) 2325 return print_bin_fmt(iter); 2326 2327 if (trace_flags & TRACE_ITER_HEX) 2328 return print_hex_fmt(iter); 2329 2330 if (trace_flags & TRACE_ITER_RAW) 2331 return print_raw_fmt(iter); 2332 2333 return print_trace_fmt(iter); 2334 } 2335 2336 void trace_latency_header(struct seq_file *m) 2337 { 2338 struct trace_iterator *iter = m->private; 2339 2340 /* print nothing if the buffers are empty */ 2341 if (trace_empty(iter)) 2342 return; 2343 2344 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 2345 print_trace_header(m, iter); 2346 2347 if (!(trace_flags & TRACE_ITER_VERBOSE)) 2348 print_lat_help_header(m); 2349 } 2350 2351 void trace_default_header(struct seq_file *m) 2352 { 2353 struct trace_iterator *iter = m->private; 2354 2355 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO)) 2356 return; 2357 2358 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { 2359 /* print nothing if the buffers are empty */ 2360 if (trace_empty(iter)) 2361 return; 2362 print_trace_header(m, iter); 2363 if (!(trace_flags & TRACE_ITER_VERBOSE)) 2364 print_lat_help_header(m); 2365 } else { 2366 if (!(trace_flags & TRACE_ITER_VERBOSE)) { 2367 if (trace_flags & TRACE_ITER_IRQ_INFO) 2368 print_func_help_header_irq(iter->tr, m); 2369 else 2370 print_func_help_header(iter->tr, m); 2371 } 2372 } 2373 } 2374 2375 static void test_ftrace_alive(struct seq_file *m) 2376 { 2377 if (!ftrace_is_dead()) 2378 return; 2379 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"); 2380 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n"); 2381 } 2382 2383 static int s_show(struct seq_file *m, void *v) 2384 { 2385 struct trace_iterator *iter = v; 2386 int ret; 2387 2388 if (iter->ent == NULL) { 2389 if (iter->tr) { 2390 seq_printf(m, "# tracer: %s\n", iter->trace->name); 2391 seq_puts(m, "#\n"); 2392 test_ftrace_alive(m); 2393 } 2394 if (iter->trace && iter->trace->print_header) 2395 iter->trace->print_header(m); 2396 else 2397 trace_default_header(m); 2398 2399 } else if (iter->leftover) { 2400 /* 2401 * If we filled the seq_file buffer earlier, we 2402 * want to just show it now. 2403 */ 2404 ret = trace_print_seq(m, &iter->seq); 2405 2406 /* ret should this time be zero, but you never know */ 2407 iter->leftover = ret; 2408 2409 } else { 2410 print_trace_line(iter); 2411 ret = trace_print_seq(m, &iter->seq); 2412 /* 2413 * If we overflow the seq_file buffer, then it will 2414 * ask us for this data again at start up. 2415 * Use that instead. 2416 * ret is 0 if seq_file write succeeded. 2417 * -1 otherwise. 2418 */ 2419 iter->leftover = ret; 2420 } 2421 2422 return 0; 2423 } 2424 2425 static const struct seq_operations tracer_seq_ops = { 2426 .start = s_start, 2427 .next = s_next, 2428 .stop = s_stop, 2429 .show = s_show, 2430 }; 2431 2432 static struct trace_iterator * 2433 __tracing_open(struct inode *inode, struct file *file) 2434 { 2435 long cpu_file = (long) inode->i_private; 2436 struct trace_iterator *iter; 2437 int cpu; 2438 2439 if (tracing_disabled) 2440 return ERR_PTR(-ENODEV); 2441 2442 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter)); 2443 if (!iter) 2444 return ERR_PTR(-ENOMEM); 2445 2446 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(), 2447 GFP_KERNEL); 2448 if (!iter->buffer_iter) 2449 goto release; 2450 2451 /* 2452 * We make a copy of the current tracer to avoid concurrent 2453 * changes on it while we are reading. 2454 */ 2455 mutex_lock(&trace_types_lock); 2456 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL); 2457 if (!iter->trace) 2458 goto fail; 2459 2460 if (current_trace) 2461 *iter->trace = *current_trace; 2462 2463 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL)) 2464 goto fail; 2465 2466 if (current_trace && current_trace->print_max) 2467 iter->tr = &max_tr; 2468 else 2469 iter->tr = &global_trace; 2470 iter->pos = -1; 2471 mutex_init(&iter->mutex); 2472 iter->cpu_file = cpu_file; 2473 2474 /* Notify the tracer early; before we stop tracing. */ 2475 if (iter->trace && iter->trace->open) 2476 iter->trace->open(iter); 2477 2478 /* Annotate start of buffers if we had overruns */ 2479 if (ring_buffer_overruns(iter->tr->buffer)) 2480 iter->iter_flags |= TRACE_FILE_ANNOTATE; 2481 2482 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 2483 if (trace_clocks[trace_clock_id].in_ns) 2484 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 2485 2486 /* stop the trace while dumping */ 2487 tracing_stop(); 2488 2489 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) { 2490 for_each_tracing_cpu(cpu) { 2491 iter->buffer_iter[cpu] = 2492 ring_buffer_read_prepare(iter->tr->buffer, cpu); 2493 } 2494 ring_buffer_read_prepare_sync(); 2495 for_each_tracing_cpu(cpu) { 2496 ring_buffer_read_start(iter->buffer_iter[cpu]); 2497 tracing_iter_reset(iter, cpu); 2498 } 2499 } else { 2500 cpu = iter->cpu_file; 2501 iter->buffer_iter[cpu] = 2502 ring_buffer_read_prepare(iter->tr->buffer, cpu); 2503 ring_buffer_read_prepare_sync(); 2504 ring_buffer_read_start(iter->buffer_iter[cpu]); 2505 tracing_iter_reset(iter, cpu); 2506 } 2507 2508 mutex_unlock(&trace_types_lock); 2509 2510 return iter; 2511 2512 fail: 2513 mutex_unlock(&trace_types_lock); 2514 kfree(iter->trace); 2515 kfree(iter->buffer_iter); 2516 release: 2517 seq_release_private(inode, file); 2518 return ERR_PTR(-ENOMEM); 2519 } 2520 2521 int tracing_open_generic(struct inode *inode, struct file *filp) 2522 { 2523 if (tracing_disabled) 2524 return -ENODEV; 2525 2526 filp->private_data = inode->i_private; 2527 return 0; 2528 } 2529 2530 static int tracing_release(struct inode *inode, struct file *file) 2531 { 2532 struct seq_file *m = file->private_data; 2533 struct trace_iterator *iter; 2534 int cpu; 2535 2536 if (!(file->f_mode & FMODE_READ)) 2537 return 0; 2538 2539 iter = m->private; 2540 2541 mutex_lock(&trace_types_lock); 2542 for_each_tracing_cpu(cpu) { 2543 if (iter->buffer_iter[cpu]) 2544 ring_buffer_read_finish(iter->buffer_iter[cpu]); 2545 } 2546 2547 if (iter->trace && iter->trace->close) 2548 iter->trace->close(iter); 2549 2550 /* reenable tracing if it was previously enabled */ 2551 tracing_start(); 2552 mutex_unlock(&trace_types_lock); 2553 2554 mutex_destroy(&iter->mutex); 2555 free_cpumask_var(iter->started); 2556 kfree(iter->trace); 2557 kfree(iter->buffer_iter); 2558 seq_release_private(inode, file); 2559 return 0; 2560 } 2561 2562 static int tracing_open(struct inode *inode, struct file *file) 2563 { 2564 struct trace_iterator *iter; 2565 int ret = 0; 2566 2567 /* If this file was open for write, then erase contents */ 2568 if ((file->f_mode & FMODE_WRITE) && 2569 (file->f_flags & O_TRUNC)) { 2570 long cpu = (long) inode->i_private; 2571 2572 if (cpu == TRACE_PIPE_ALL_CPU) 2573 tracing_reset_online_cpus(&global_trace); 2574 else 2575 tracing_reset(&global_trace, cpu); 2576 } 2577 2578 if (file->f_mode & FMODE_READ) { 2579 iter = __tracing_open(inode, file); 2580 if (IS_ERR(iter)) 2581 ret = PTR_ERR(iter); 2582 else if (trace_flags & TRACE_ITER_LATENCY_FMT) 2583 iter->iter_flags |= TRACE_FILE_LAT_FMT; 2584 } 2585 return ret; 2586 } 2587 2588 static void * 2589 t_next(struct seq_file *m, void *v, loff_t *pos) 2590 { 2591 struct tracer *t = v; 2592 2593 (*pos)++; 2594 2595 if (t) 2596 t = t->next; 2597 2598 return t; 2599 } 2600 2601 static void *t_start(struct seq_file *m, loff_t *pos) 2602 { 2603 struct tracer *t; 2604 loff_t l = 0; 2605 2606 mutex_lock(&trace_types_lock); 2607 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l)) 2608 ; 2609 2610 return t; 2611 } 2612 2613 static void t_stop(struct seq_file *m, void *p) 2614 { 2615 mutex_unlock(&trace_types_lock); 2616 } 2617 2618 static int t_show(struct seq_file *m, void *v) 2619 { 2620 struct tracer *t = v; 2621 2622 if (!t) 2623 return 0; 2624 2625 seq_printf(m, "%s", t->name); 2626 if (t->next) 2627 seq_putc(m, ' '); 2628 else 2629 seq_putc(m, '\n'); 2630 2631 return 0; 2632 } 2633 2634 static const struct seq_operations show_traces_seq_ops = { 2635 .start = t_start, 2636 .next = t_next, 2637 .stop = t_stop, 2638 .show = t_show, 2639 }; 2640 2641 static int show_traces_open(struct inode *inode, struct file *file) 2642 { 2643 if (tracing_disabled) 2644 return -ENODEV; 2645 2646 return seq_open(file, &show_traces_seq_ops); 2647 } 2648 2649 static ssize_t 2650 tracing_write_stub(struct file *filp, const char __user *ubuf, 2651 size_t count, loff_t *ppos) 2652 { 2653 return count; 2654 } 2655 2656 static loff_t tracing_seek(struct file *file, loff_t offset, int origin) 2657 { 2658 if (file->f_mode & FMODE_READ) 2659 return seq_lseek(file, offset, origin); 2660 else 2661 return 0; 2662 } 2663 2664 static const struct file_operations tracing_fops = { 2665 .open = tracing_open, 2666 .read = seq_read, 2667 .write = tracing_write_stub, 2668 .llseek = tracing_seek, 2669 .release = tracing_release, 2670 }; 2671 2672 static const struct file_operations show_traces_fops = { 2673 .open = show_traces_open, 2674 .read = seq_read, 2675 .release = seq_release, 2676 .llseek = seq_lseek, 2677 }; 2678 2679 /* 2680 * Only trace on a CPU if the bitmask is set: 2681 */ 2682 static cpumask_var_t tracing_cpumask; 2683 2684 /* 2685 * The tracer itself will not take this lock, but still we want 2686 * to provide a consistent cpumask to user-space: 2687 */ 2688 static DEFINE_MUTEX(tracing_cpumask_update_lock); 2689 2690 /* 2691 * Temporary storage for the character representation of the 2692 * CPU bitmask (and one more byte for the newline): 2693 */ 2694 static char mask_str[NR_CPUS + 1]; 2695 2696 static ssize_t 2697 tracing_cpumask_read(struct file *filp, char __user *ubuf, 2698 size_t count, loff_t *ppos) 2699 { 2700 int len; 2701 2702 mutex_lock(&tracing_cpumask_update_lock); 2703 2704 len = cpumask_scnprintf(mask_str, count, tracing_cpumask); 2705 if (count - len < 2) { 2706 count = -EINVAL; 2707 goto out_err; 2708 } 2709 len += sprintf(mask_str + len, "\n"); 2710 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1); 2711 2712 out_err: 2713 mutex_unlock(&tracing_cpumask_update_lock); 2714 2715 return count; 2716 } 2717 2718 static ssize_t 2719 tracing_cpumask_write(struct file *filp, const char __user *ubuf, 2720 size_t count, loff_t *ppos) 2721 { 2722 int err, cpu; 2723 cpumask_var_t tracing_cpumask_new; 2724 2725 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL)) 2726 return -ENOMEM; 2727 2728 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new); 2729 if (err) 2730 goto err_unlock; 2731 2732 mutex_lock(&tracing_cpumask_update_lock); 2733 2734 local_irq_disable(); 2735 arch_spin_lock(&ftrace_max_lock); 2736 for_each_tracing_cpu(cpu) { 2737 /* 2738 * Increase/decrease the disabled counter if we are 2739 * about to flip a bit in the cpumask: 2740 */ 2741 if (cpumask_test_cpu(cpu, tracing_cpumask) && 2742 !cpumask_test_cpu(cpu, tracing_cpumask_new)) { 2743 atomic_inc(&global_trace.data[cpu]->disabled); 2744 ring_buffer_record_disable_cpu(global_trace.buffer, cpu); 2745 } 2746 if (!cpumask_test_cpu(cpu, tracing_cpumask) && 2747 cpumask_test_cpu(cpu, tracing_cpumask_new)) { 2748 atomic_dec(&global_trace.data[cpu]->disabled); 2749 ring_buffer_record_enable_cpu(global_trace.buffer, cpu); 2750 } 2751 } 2752 arch_spin_unlock(&ftrace_max_lock); 2753 local_irq_enable(); 2754 2755 cpumask_copy(tracing_cpumask, tracing_cpumask_new); 2756 2757 mutex_unlock(&tracing_cpumask_update_lock); 2758 free_cpumask_var(tracing_cpumask_new); 2759 2760 return count; 2761 2762 err_unlock: 2763 free_cpumask_var(tracing_cpumask_new); 2764 2765 return err; 2766 } 2767 2768 static const struct file_operations tracing_cpumask_fops = { 2769 .open = tracing_open_generic, 2770 .read = tracing_cpumask_read, 2771 .write = tracing_cpumask_write, 2772 .llseek = generic_file_llseek, 2773 }; 2774 2775 static int tracing_trace_options_show(struct seq_file *m, void *v) 2776 { 2777 struct tracer_opt *trace_opts; 2778 u32 tracer_flags; 2779 int i; 2780 2781 mutex_lock(&trace_types_lock); 2782 tracer_flags = current_trace->flags->val; 2783 trace_opts = current_trace->flags->opts; 2784 2785 for (i = 0; trace_options[i]; i++) { 2786 if (trace_flags & (1 << i)) 2787 seq_printf(m, "%s\n", trace_options[i]); 2788 else 2789 seq_printf(m, "no%s\n", trace_options[i]); 2790 } 2791 2792 for (i = 0; trace_opts[i].name; i++) { 2793 if (tracer_flags & trace_opts[i].bit) 2794 seq_printf(m, "%s\n", trace_opts[i].name); 2795 else 2796 seq_printf(m, "no%s\n", trace_opts[i].name); 2797 } 2798 mutex_unlock(&trace_types_lock); 2799 2800 return 0; 2801 } 2802 2803 static int __set_tracer_option(struct tracer *trace, 2804 struct tracer_flags *tracer_flags, 2805 struct tracer_opt *opts, int neg) 2806 { 2807 int ret; 2808 2809 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg); 2810 if (ret) 2811 return ret; 2812 2813 if (neg) 2814 tracer_flags->val &= ~opts->bit; 2815 else 2816 tracer_flags->val |= opts->bit; 2817 return 0; 2818 } 2819 2820 /* Try to assign a tracer specific option */ 2821 static int set_tracer_option(struct tracer *trace, char *cmp, int neg) 2822 { 2823 struct tracer_flags *tracer_flags = trace->flags; 2824 struct tracer_opt *opts = NULL; 2825 int i; 2826 2827 for (i = 0; tracer_flags->opts[i].name; i++) { 2828 opts = &tracer_flags->opts[i]; 2829 2830 if (strcmp(cmp, opts->name) == 0) 2831 return __set_tracer_option(trace, trace->flags, 2832 opts, neg); 2833 } 2834 2835 return -EINVAL; 2836 } 2837 2838 static void set_tracer_flags(unsigned int mask, int enabled) 2839 { 2840 /* do nothing if flag is already set */ 2841 if (!!(trace_flags & mask) == !!enabled) 2842 return; 2843 2844 if (enabled) 2845 trace_flags |= mask; 2846 else 2847 trace_flags &= ~mask; 2848 2849 if (mask == TRACE_ITER_RECORD_CMD) 2850 trace_event_enable_cmd_record(enabled); 2851 2852 if (mask == TRACE_ITER_OVERWRITE) 2853 ring_buffer_change_overwrite(global_trace.buffer, enabled); 2854 2855 if (mask == TRACE_ITER_PRINTK) 2856 trace_printk_start_stop_comm(enabled); 2857 } 2858 2859 static int trace_set_options(char *option) 2860 { 2861 char *cmp; 2862 int neg = 0; 2863 int ret = 0; 2864 int i; 2865 2866 cmp = strstrip(option); 2867 2868 if (strncmp(cmp, "no", 2) == 0) { 2869 neg = 1; 2870 cmp += 2; 2871 } 2872 2873 for (i = 0; trace_options[i]; i++) { 2874 if (strcmp(cmp, trace_options[i]) == 0) { 2875 set_tracer_flags(1 << i, !neg); 2876 break; 2877 } 2878 } 2879 2880 /* If no option could be set, test the specific tracer options */ 2881 if (!trace_options[i]) { 2882 mutex_lock(&trace_types_lock); 2883 ret = set_tracer_option(current_trace, cmp, neg); 2884 mutex_unlock(&trace_types_lock); 2885 } 2886 2887 return ret; 2888 } 2889 2890 static ssize_t 2891 tracing_trace_options_write(struct file *filp, const char __user *ubuf, 2892 size_t cnt, loff_t *ppos) 2893 { 2894 char buf[64]; 2895 2896 if (cnt >= sizeof(buf)) 2897 return -EINVAL; 2898 2899 if (copy_from_user(&buf, ubuf, cnt)) 2900 return -EFAULT; 2901 2902 trace_set_options(buf); 2903 2904 *ppos += cnt; 2905 2906 return cnt; 2907 } 2908 2909 static int tracing_trace_options_open(struct inode *inode, struct file *file) 2910 { 2911 if (tracing_disabled) 2912 return -ENODEV; 2913 return single_open(file, tracing_trace_options_show, NULL); 2914 } 2915 2916 static const struct file_operations tracing_iter_fops = { 2917 .open = tracing_trace_options_open, 2918 .read = seq_read, 2919 .llseek = seq_lseek, 2920 .release = single_release, 2921 .write = tracing_trace_options_write, 2922 }; 2923 2924 static const char readme_msg[] = 2925 "tracing mini-HOWTO:\n\n" 2926 "# mount -t debugfs nodev /sys/kernel/debug\n\n" 2927 "# cat /sys/kernel/debug/tracing/available_tracers\n" 2928 "wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n" 2929 "# cat /sys/kernel/debug/tracing/current_tracer\n" 2930 "nop\n" 2931 "# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n" 2932 "# cat /sys/kernel/debug/tracing/current_tracer\n" 2933 "wakeup\n" 2934 "# cat /sys/kernel/debug/tracing/trace_options\n" 2935 "noprint-parent nosym-offset nosym-addr noverbose\n" 2936 "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n" 2937 "# echo 1 > /sys/kernel/debug/tracing/tracing_on\n" 2938 "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n" 2939 "# echo 0 > /sys/kernel/debug/tracing/tracing_on\n" 2940 ; 2941 2942 static ssize_t 2943 tracing_readme_read(struct file *filp, char __user *ubuf, 2944 size_t cnt, loff_t *ppos) 2945 { 2946 return simple_read_from_buffer(ubuf, cnt, ppos, 2947 readme_msg, strlen(readme_msg)); 2948 } 2949 2950 static const struct file_operations tracing_readme_fops = { 2951 .open = tracing_open_generic, 2952 .read = tracing_readme_read, 2953 .llseek = generic_file_llseek, 2954 }; 2955 2956 static ssize_t 2957 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf, 2958 size_t cnt, loff_t *ppos) 2959 { 2960 char *buf_comm; 2961 char *file_buf; 2962 char *buf; 2963 int len = 0; 2964 int pid; 2965 int i; 2966 2967 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL); 2968 if (!file_buf) 2969 return -ENOMEM; 2970 2971 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL); 2972 if (!buf_comm) { 2973 kfree(file_buf); 2974 return -ENOMEM; 2975 } 2976 2977 buf = file_buf; 2978 2979 for (i = 0; i < SAVED_CMDLINES; i++) { 2980 int r; 2981 2982 pid = map_cmdline_to_pid[i]; 2983 if (pid == -1 || pid == NO_CMDLINE_MAP) 2984 continue; 2985 2986 trace_find_cmdline(pid, buf_comm); 2987 r = sprintf(buf, "%d %s\n", pid, buf_comm); 2988 buf += r; 2989 len += r; 2990 } 2991 2992 len = simple_read_from_buffer(ubuf, cnt, ppos, 2993 file_buf, len); 2994 2995 kfree(file_buf); 2996 kfree(buf_comm); 2997 2998 return len; 2999 } 3000 3001 static const struct file_operations tracing_saved_cmdlines_fops = { 3002 .open = tracing_open_generic, 3003 .read = tracing_saved_cmdlines_read, 3004 .llseek = generic_file_llseek, 3005 }; 3006 3007 static ssize_t 3008 tracing_set_trace_read(struct file *filp, char __user *ubuf, 3009 size_t cnt, loff_t *ppos) 3010 { 3011 char buf[MAX_TRACER_SIZE+2]; 3012 int r; 3013 3014 mutex_lock(&trace_types_lock); 3015 if (current_trace) 3016 r = sprintf(buf, "%s\n", current_trace->name); 3017 else 3018 r = sprintf(buf, "\n"); 3019 mutex_unlock(&trace_types_lock); 3020 3021 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 3022 } 3023 3024 int tracer_init(struct tracer *t, struct trace_array *tr) 3025 { 3026 tracing_reset_online_cpus(tr); 3027 return t->init(tr); 3028 } 3029 3030 static void set_buffer_entries(struct trace_array *tr, unsigned long val) 3031 { 3032 int cpu; 3033 for_each_tracing_cpu(cpu) 3034 tr->data[cpu]->entries = val; 3035 } 3036 3037 /* resize @tr's buffer to the size of @size_tr's entries */ 3038 static int resize_buffer_duplicate_size(struct trace_array *tr, 3039 struct trace_array *size_tr, int cpu_id) 3040 { 3041 int cpu, ret = 0; 3042 3043 if (cpu_id == RING_BUFFER_ALL_CPUS) { 3044 for_each_tracing_cpu(cpu) { 3045 ret = ring_buffer_resize(tr->buffer, 3046 size_tr->data[cpu]->entries, cpu); 3047 if (ret < 0) 3048 break; 3049 tr->data[cpu]->entries = size_tr->data[cpu]->entries; 3050 } 3051 } else { 3052 ret = ring_buffer_resize(tr->buffer, 3053 size_tr->data[cpu_id]->entries, cpu_id); 3054 if (ret == 0) 3055 tr->data[cpu_id]->entries = 3056 size_tr->data[cpu_id]->entries; 3057 } 3058 3059 return ret; 3060 } 3061 3062 static int __tracing_resize_ring_buffer(unsigned long size, int cpu) 3063 { 3064 int ret; 3065 3066 /* 3067 * If kernel or user changes the size of the ring buffer 3068 * we use the size that was given, and we can forget about 3069 * expanding it later. 3070 */ 3071 ring_buffer_expanded = 1; 3072 3073 /* May be called before buffers are initialized */ 3074 if (!global_trace.buffer) 3075 return 0; 3076 3077 ret = ring_buffer_resize(global_trace.buffer, size, cpu); 3078 if (ret < 0) 3079 return ret; 3080 3081 if (!current_trace->use_max_tr) 3082 goto out; 3083 3084 ret = ring_buffer_resize(max_tr.buffer, size, cpu); 3085 if (ret < 0) { 3086 int r = resize_buffer_duplicate_size(&global_trace, 3087 &global_trace, cpu); 3088 if (r < 0) { 3089 /* 3090 * AARGH! We are left with different 3091 * size max buffer!!!! 3092 * The max buffer is our "snapshot" buffer. 3093 * When a tracer needs a snapshot (one of the 3094 * latency tracers), it swaps the max buffer 3095 * with the saved snap shot. We succeeded to 3096 * update the size of the main buffer, but failed to 3097 * update the size of the max buffer. But when we tried 3098 * to reset the main buffer to the original size, we 3099 * failed there too. This is very unlikely to 3100 * happen, but if it does, warn and kill all 3101 * tracing. 3102 */ 3103 WARN_ON(1); 3104 tracing_disabled = 1; 3105 } 3106 return ret; 3107 } 3108 3109 if (cpu == RING_BUFFER_ALL_CPUS) 3110 set_buffer_entries(&max_tr, size); 3111 else 3112 max_tr.data[cpu]->entries = size; 3113 3114 out: 3115 if (cpu == RING_BUFFER_ALL_CPUS) 3116 set_buffer_entries(&global_trace, size); 3117 else 3118 global_trace.data[cpu]->entries = size; 3119 3120 return ret; 3121 } 3122 3123 static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id) 3124 { 3125 int ret = size; 3126 3127 mutex_lock(&trace_types_lock); 3128 3129 if (cpu_id != RING_BUFFER_ALL_CPUS) { 3130 /* make sure, this cpu is enabled in the mask */ 3131 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) { 3132 ret = -EINVAL; 3133 goto out; 3134 } 3135 } 3136 3137 ret = __tracing_resize_ring_buffer(size, cpu_id); 3138 if (ret < 0) 3139 ret = -ENOMEM; 3140 3141 out: 3142 mutex_unlock(&trace_types_lock); 3143 3144 return ret; 3145 } 3146 3147 3148 /** 3149 * tracing_update_buffers - used by tracing facility to expand ring buffers 3150 * 3151 * To save on memory when the tracing is never used on a system with it 3152 * configured in. The ring buffers are set to a minimum size. But once 3153 * a user starts to use the tracing facility, then they need to grow 3154 * to their default size. 3155 * 3156 * This function is to be called when a tracer is about to be used. 3157 */ 3158 int tracing_update_buffers(void) 3159 { 3160 int ret = 0; 3161 3162 mutex_lock(&trace_types_lock); 3163 if (!ring_buffer_expanded) 3164 ret = __tracing_resize_ring_buffer(trace_buf_size, 3165 RING_BUFFER_ALL_CPUS); 3166 mutex_unlock(&trace_types_lock); 3167 3168 return ret; 3169 } 3170 3171 struct trace_option_dentry; 3172 3173 static struct trace_option_dentry * 3174 create_trace_option_files(struct tracer *tracer); 3175 3176 static void 3177 destroy_trace_option_files(struct trace_option_dentry *topts); 3178 3179 static int tracing_set_tracer(const char *buf) 3180 { 3181 static struct trace_option_dentry *topts; 3182 struct trace_array *tr = &global_trace; 3183 struct tracer *t; 3184 int ret = 0; 3185 3186 mutex_lock(&trace_types_lock); 3187 3188 if (!ring_buffer_expanded) { 3189 ret = __tracing_resize_ring_buffer(trace_buf_size, 3190 RING_BUFFER_ALL_CPUS); 3191 if (ret < 0) 3192 goto out; 3193 ret = 0; 3194 } 3195 3196 for (t = trace_types; t; t = t->next) { 3197 if (strcmp(t->name, buf) == 0) 3198 break; 3199 } 3200 if (!t) { 3201 ret = -EINVAL; 3202 goto out; 3203 } 3204 if (t == current_trace) 3205 goto out; 3206 3207 trace_branch_disable(); 3208 if (current_trace && current_trace->reset) 3209 current_trace->reset(tr); 3210 if (current_trace && current_trace->use_max_tr) { 3211 /* 3212 * We don't free the ring buffer. instead, resize it because 3213 * The max_tr ring buffer has some state (e.g. ring->clock) and 3214 * we want preserve it. 3215 */ 3216 ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS); 3217 set_buffer_entries(&max_tr, 1); 3218 } 3219 destroy_trace_option_files(topts); 3220 3221 current_trace = &nop_trace; 3222 3223 topts = create_trace_option_files(t); 3224 if (t->use_max_tr) { 3225 /* we need to make per cpu buffer sizes equivalent */ 3226 ret = resize_buffer_duplicate_size(&max_tr, &global_trace, 3227 RING_BUFFER_ALL_CPUS); 3228 if (ret < 0) 3229 goto out; 3230 } 3231 3232 if (t->init) { 3233 ret = tracer_init(t, tr); 3234 if (ret) 3235 goto out; 3236 } 3237 3238 current_trace = t; 3239 trace_branch_enable(tr); 3240 out: 3241 mutex_unlock(&trace_types_lock); 3242 3243 return ret; 3244 } 3245 3246 static ssize_t 3247 tracing_set_trace_write(struct file *filp, const char __user *ubuf, 3248 size_t cnt, loff_t *ppos) 3249 { 3250 char buf[MAX_TRACER_SIZE+1]; 3251 int i; 3252 size_t ret; 3253 int err; 3254 3255 ret = cnt; 3256 3257 if (cnt > MAX_TRACER_SIZE) 3258 cnt = MAX_TRACER_SIZE; 3259 3260 if (copy_from_user(&buf, ubuf, cnt)) 3261 return -EFAULT; 3262 3263 buf[cnt] = 0; 3264 3265 /* strip ending whitespace. */ 3266 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--) 3267 buf[i] = 0; 3268 3269 err = tracing_set_tracer(buf); 3270 if (err) 3271 return err; 3272 3273 *ppos += ret; 3274 3275 return ret; 3276 } 3277 3278 static ssize_t 3279 tracing_max_lat_read(struct file *filp, char __user *ubuf, 3280 size_t cnt, loff_t *ppos) 3281 { 3282 unsigned long *ptr = filp->private_data; 3283 char buf[64]; 3284 int r; 3285 3286 r = snprintf(buf, sizeof(buf), "%ld\n", 3287 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr)); 3288 if (r > sizeof(buf)) 3289 r = sizeof(buf); 3290 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 3291 } 3292 3293 static ssize_t 3294 tracing_max_lat_write(struct file *filp, const char __user *ubuf, 3295 size_t cnt, loff_t *ppos) 3296 { 3297 unsigned long *ptr = filp->private_data; 3298 unsigned long val; 3299 int ret; 3300 3301 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 3302 if (ret) 3303 return ret; 3304 3305 *ptr = val * 1000; 3306 3307 return cnt; 3308 } 3309 3310 static int tracing_open_pipe(struct inode *inode, struct file *filp) 3311 { 3312 long cpu_file = (long) inode->i_private; 3313 struct trace_iterator *iter; 3314 int ret = 0; 3315 3316 if (tracing_disabled) 3317 return -ENODEV; 3318 3319 mutex_lock(&trace_types_lock); 3320 3321 /* create a buffer to store the information to pass to userspace */ 3322 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 3323 if (!iter) { 3324 ret = -ENOMEM; 3325 goto out; 3326 } 3327 3328 /* 3329 * We make a copy of the current tracer to avoid concurrent 3330 * changes on it while we are reading. 3331 */ 3332 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL); 3333 if (!iter->trace) { 3334 ret = -ENOMEM; 3335 goto fail; 3336 } 3337 if (current_trace) 3338 *iter->trace = *current_trace; 3339 3340 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) { 3341 ret = -ENOMEM; 3342 goto fail; 3343 } 3344 3345 /* trace pipe does not show start of buffer */ 3346 cpumask_setall(iter->started); 3347 3348 if (trace_flags & TRACE_ITER_LATENCY_FMT) 3349 iter->iter_flags |= TRACE_FILE_LAT_FMT; 3350 3351 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 3352 if (trace_clocks[trace_clock_id].in_ns) 3353 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 3354 3355 iter->cpu_file = cpu_file; 3356 iter->tr = &global_trace; 3357 mutex_init(&iter->mutex); 3358 filp->private_data = iter; 3359 3360 if (iter->trace->pipe_open) 3361 iter->trace->pipe_open(iter); 3362 3363 nonseekable_open(inode, filp); 3364 out: 3365 mutex_unlock(&trace_types_lock); 3366 return ret; 3367 3368 fail: 3369 kfree(iter->trace); 3370 kfree(iter); 3371 mutex_unlock(&trace_types_lock); 3372 return ret; 3373 } 3374 3375 static int tracing_release_pipe(struct inode *inode, struct file *file) 3376 { 3377 struct trace_iterator *iter = file->private_data; 3378 3379 mutex_lock(&trace_types_lock); 3380 3381 if (iter->trace->pipe_close) 3382 iter->trace->pipe_close(iter); 3383 3384 mutex_unlock(&trace_types_lock); 3385 3386 free_cpumask_var(iter->started); 3387 mutex_destroy(&iter->mutex); 3388 kfree(iter->trace); 3389 kfree(iter); 3390 3391 return 0; 3392 } 3393 3394 static unsigned int 3395 tracing_poll_pipe(struct file *filp, poll_table *poll_table) 3396 { 3397 struct trace_iterator *iter = filp->private_data; 3398 3399 if (trace_flags & TRACE_ITER_BLOCK) { 3400 /* 3401 * Always select as readable when in blocking mode 3402 */ 3403 return POLLIN | POLLRDNORM; 3404 } else { 3405 if (!trace_empty(iter)) 3406 return POLLIN | POLLRDNORM; 3407 poll_wait(filp, &trace_wait, poll_table); 3408 if (!trace_empty(iter)) 3409 return POLLIN | POLLRDNORM; 3410 3411 return 0; 3412 } 3413 } 3414 3415 /* 3416 * This is a make-shift waitqueue. 3417 * A tracer might use this callback on some rare cases: 3418 * 3419 * 1) the current tracer might hold the runqueue lock when it wakes up 3420 * a reader, hence a deadlock (sched, function, and function graph tracers) 3421 * 2) the function tracers, trace all functions, we don't want 3422 * the overhead of calling wake_up and friends 3423 * (and tracing them too) 3424 * 3425 * Anyway, this is really very primitive wakeup. 3426 */ 3427 void poll_wait_pipe(struct trace_iterator *iter) 3428 { 3429 set_current_state(TASK_INTERRUPTIBLE); 3430 /* sleep for 100 msecs, and try again. */ 3431 schedule_timeout(HZ / 10); 3432 } 3433 3434 /* Must be called with trace_types_lock mutex held. */ 3435 static int tracing_wait_pipe(struct file *filp) 3436 { 3437 struct trace_iterator *iter = filp->private_data; 3438 3439 while (trace_empty(iter)) { 3440 3441 if ((filp->f_flags & O_NONBLOCK)) { 3442 return -EAGAIN; 3443 } 3444 3445 mutex_unlock(&iter->mutex); 3446 3447 iter->trace->wait_pipe(iter); 3448 3449 mutex_lock(&iter->mutex); 3450 3451 if (signal_pending(current)) 3452 return -EINTR; 3453 3454 /* 3455 * We block until we read something and tracing is enabled. 3456 * We still block if tracing is disabled, but we have never 3457 * read anything. This allows a user to cat this file, and 3458 * then enable tracing. But after we have read something, 3459 * we give an EOF when tracing is again disabled. 3460 * 3461 * iter->pos will be 0 if we haven't read anything. 3462 */ 3463 if (tracing_is_enabled() && iter->pos) 3464 break; 3465 } 3466 3467 return 1; 3468 } 3469 3470 /* 3471 * Consumer reader. 3472 */ 3473 static ssize_t 3474 tracing_read_pipe(struct file *filp, char __user *ubuf, 3475 size_t cnt, loff_t *ppos) 3476 { 3477 struct trace_iterator *iter = filp->private_data; 3478 static struct tracer *old_tracer; 3479 ssize_t sret; 3480 3481 /* return any leftover data */ 3482 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 3483 if (sret != -EBUSY) 3484 return sret; 3485 3486 trace_seq_init(&iter->seq); 3487 3488 /* copy the tracer to avoid using a global lock all around */ 3489 mutex_lock(&trace_types_lock); 3490 if (unlikely(old_tracer != current_trace && current_trace)) { 3491 old_tracer = current_trace; 3492 *iter->trace = *current_trace; 3493 } 3494 mutex_unlock(&trace_types_lock); 3495 3496 /* 3497 * Avoid more than one consumer on a single file descriptor 3498 * This is just a matter of traces coherency, the ring buffer itself 3499 * is protected. 3500 */ 3501 mutex_lock(&iter->mutex); 3502 if (iter->trace->read) { 3503 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); 3504 if (sret) 3505 goto out; 3506 } 3507 3508 waitagain: 3509 sret = tracing_wait_pipe(filp); 3510 if (sret <= 0) 3511 goto out; 3512 3513 /* stop when tracing is finished */ 3514 if (trace_empty(iter)) { 3515 sret = 0; 3516 goto out; 3517 } 3518 3519 if (cnt >= PAGE_SIZE) 3520 cnt = PAGE_SIZE - 1; 3521 3522 /* reset all but tr, trace, and overruns */ 3523 memset(&iter->seq, 0, 3524 sizeof(struct trace_iterator) - 3525 offsetof(struct trace_iterator, seq)); 3526 iter->pos = -1; 3527 3528 trace_event_read_lock(); 3529 trace_access_lock(iter->cpu_file); 3530 while (trace_find_next_entry_inc(iter) != NULL) { 3531 enum print_line_t ret; 3532 int len = iter->seq.len; 3533 3534 ret = print_trace_line(iter); 3535 if (ret == TRACE_TYPE_PARTIAL_LINE) { 3536 /* don't print partial lines */ 3537 iter->seq.len = len; 3538 break; 3539 } 3540 if (ret != TRACE_TYPE_NO_CONSUME) 3541 trace_consume(iter); 3542 3543 if (iter->seq.len >= cnt) 3544 break; 3545 3546 /* 3547 * Setting the full flag means we reached the trace_seq buffer 3548 * size and we should leave by partial output condition above. 3549 * One of the trace_seq_* functions is not used properly. 3550 */ 3551 WARN_ONCE(iter->seq.full, "full flag set for trace type %d", 3552 iter->ent->type); 3553 } 3554 trace_access_unlock(iter->cpu_file); 3555 trace_event_read_unlock(); 3556 3557 /* Now copy what we have to the user */ 3558 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 3559 if (iter->seq.readpos >= iter->seq.len) 3560 trace_seq_init(&iter->seq); 3561 3562 /* 3563 * If there was nothing to send to user, in spite of consuming trace 3564 * entries, go back to wait for more entries. 3565 */ 3566 if (sret == -EBUSY) 3567 goto waitagain; 3568 3569 out: 3570 mutex_unlock(&iter->mutex); 3571 3572 return sret; 3573 } 3574 3575 static void tracing_pipe_buf_release(struct pipe_inode_info *pipe, 3576 struct pipe_buffer *buf) 3577 { 3578 __free_page(buf->page); 3579 } 3580 3581 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd, 3582 unsigned int idx) 3583 { 3584 __free_page(spd->pages[idx]); 3585 } 3586 3587 static const struct pipe_buf_operations tracing_pipe_buf_ops = { 3588 .can_merge = 0, 3589 .map = generic_pipe_buf_map, 3590 .unmap = generic_pipe_buf_unmap, 3591 .confirm = generic_pipe_buf_confirm, 3592 .release = tracing_pipe_buf_release, 3593 .steal = generic_pipe_buf_steal, 3594 .get = generic_pipe_buf_get, 3595 }; 3596 3597 static size_t 3598 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter) 3599 { 3600 size_t count; 3601 int ret; 3602 3603 /* Seq buffer is page-sized, exactly what we need. */ 3604 for (;;) { 3605 count = iter->seq.len; 3606 ret = print_trace_line(iter); 3607 count = iter->seq.len - count; 3608 if (rem < count) { 3609 rem = 0; 3610 iter->seq.len -= count; 3611 break; 3612 } 3613 if (ret == TRACE_TYPE_PARTIAL_LINE) { 3614 iter->seq.len -= count; 3615 break; 3616 } 3617 3618 if (ret != TRACE_TYPE_NO_CONSUME) 3619 trace_consume(iter); 3620 rem -= count; 3621 if (!trace_find_next_entry_inc(iter)) { 3622 rem = 0; 3623 iter->ent = NULL; 3624 break; 3625 } 3626 } 3627 3628 return rem; 3629 } 3630 3631 static ssize_t tracing_splice_read_pipe(struct file *filp, 3632 loff_t *ppos, 3633 struct pipe_inode_info *pipe, 3634 size_t len, 3635 unsigned int flags) 3636 { 3637 struct page *pages_def[PIPE_DEF_BUFFERS]; 3638 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 3639 struct trace_iterator *iter = filp->private_data; 3640 struct splice_pipe_desc spd = { 3641 .pages = pages_def, 3642 .partial = partial_def, 3643 .nr_pages = 0, /* This gets updated below. */ 3644 .nr_pages_max = PIPE_DEF_BUFFERS, 3645 .flags = flags, 3646 .ops = &tracing_pipe_buf_ops, 3647 .spd_release = tracing_spd_release_pipe, 3648 }; 3649 static struct tracer *old_tracer; 3650 ssize_t ret; 3651 size_t rem; 3652 unsigned int i; 3653 3654 if (splice_grow_spd(pipe, &spd)) 3655 return -ENOMEM; 3656 3657 /* copy the tracer to avoid using a global lock all around */ 3658 mutex_lock(&trace_types_lock); 3659 if (unlikely(old_tracer != current_trace && current_trace)) { 3660 old_tracer = current_trace; 3661 *iter->trace = *current_trace; 3662 } 3663 mutex_unlock(&trace_types_lock); 3664 3665 mutex_lock(&iter->mutex); 3666 3667 if (iter->trace->splice_read) { 3668 ret = iter->trace->splice_read(iter, filp, 3669 ppos, pipe, len, flags); 3670 if (ret) 3671 goto out_err; 3672 } 3673 3674 ret = tracing_wait_pipe(filp); 3675 if (ret <= 0) 3676 goto out_err; 3677 3678 if (!iter->ent && !trace_find_next_entry_inc(iter)) { 3679 ret = -EFAULT; 3680 goto out_err; 3681 } 3682 3683 trace_event_read_lock(); 3684 trace_access_lock(iter->cpu_file); 3685 3686 /* Fill as many pages as possible. */ 3687 for (i = 0, rem = len; i < pipe->buffers && rem; i++) { 3688 spd.pages[i] = alloc_page(GFP_KERNEL); 3689 if (!spd.pages[i]) 3690 break; 3691 3692 rem = tracing_fill_pipe_page(rem, iter); 3693 3694 /* Copy the data into the page, so we can start over. */ 3695 ret = trace_seq_to_buffer(&iter->seq, 3696 page_address(spd.pages[i]), 3697 iter->seq.len); 3698 if (ret < 0) { 3699 __free_page(spd.pages[i]); 3700 break; 3701 } 3702 spd.partial[i].offset = 0; 3703 spd.partial[i].len = iter->seq.len; 3704 3705 trace_seq_init(&iter->seq); 3706 } 3707 3708 trace_access_unlock(iter->cpu_file); 3709 trace_event_read_unlock(); 3710 mutex_unlock(&iter->mutex); 3711 3712 spd.nr_pages = i; 3713 3714 ret = splice_to_pipe(pipe, &spd); 3715 out: 3716 splice_shrink_spd(&spd); 3717 return ret; 3718 3719 out_err: 3720 mutex_unlock(&iter->mutex); 3721 goto out; 3722 } 3723 3724 struct ftrace_entries_info { 3725 struct trace_array *tr; 3726 int cpu; 3727 }; 3728 3729 static int tracing_entries_open(struct inode *inode, struct file *filp) 3730 { 3731 struct ftrace_entries_info *info; 3732 3733 if (tracing_disabled) 3734 return -ENODEV; 3735 3736 info = kzalloc(sizeof(*info), GFP_KERNEL); 3737 if (!info) 3738 return -ENOMEM; 3739 3740 info->tr = &global_trace; 3741 info->cpu = (unsigned long)inode->i_private; 3742 3743 filp->private_data = info; 3744 3745 return 0; 3746 } 3747 3748 static ssize_t 3749 tracing_entries_read(struct file *filp, char __user *ubuf, 3750 size_t cnt, loff_t *ppos) 3751 { 3752 struct ftrace_entries_info *info = filp->private_data; 3753 struct trace_array *tr = info->tr; 3754 char buf[64]; 3755 int r = 0; 3756 ssize_t ret; 3757 3758 mutex_lock(&trace_types_lock); 3759 3760 if (info->cpu == RING_BUFFER_ALL_CPUS) { 3761 int cpu, buf_size_same; 3762 unsigned long size; 3763 3764 size = 0; 3765 buf_size_same = 1; 3766 /* check if all cpu sizes are same */ 3767 for_each_tracing_cpu(cpu) { 3768 /* fill in the size from first enabled cpu */ 3769 if (size == 0) 3770 size = tr->data[cpu]->entries; 3771 if (size != tr->data[cpu]->entries) { 3772 buf_size_same = 0; 3773 break; 3774 } 3775 } 3776 3777 if (buf_size_same) { 3778 if (!ring_buffer_expanded) 3779 r = sprintf(buf, "%lu (expanded: %lu)\n", 3780 size >> 10, 3781 trace_buf_size >> 10); 3782 else 3783 r = sprintf(buf, "%lu\n", size >> 10); 3784 } else 3785 r = sprintf(buf, "X\n"); 3786 } else 3787 r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10); 3788 3789 mutex_unlock(&trace_types_lock); 3790 3791 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 3792 return ret; 3793 } 3794 3795 static ssize_t 3796 tracing_entries_write(struct file *filp, const char __user *ubuf, 3797 size_t cnt, loff_t *ppos) 3798 { 3799 struct ftrace_entries_info *info = filp->private_data; 3800 unsigned long val; 3801 int ret; 3802 3803 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 3804 if (ret) 3805 return ret; 3806 3807 /* must have at least 1 entry */ 3808 if (!val) 3809 return -EINVAL; 3810 3811 /* value is in KB */ 3812 val <<= 10; 3813 3814 ret = tracing_resize_ring_buffer(val, info->cpu); 3815 if (ret < 0) 3816 return ret; 3817 3818 *ppos += cnt; 3819 3820 return cnt; 3821 } 3822 3823 static int 3824 tracing_entries_release(struct inode *inode, struct file *filp) 3825 { 3826 struct ftrace_entries_info *info = filp->private_data; 3827 3828 kfree(info); 3829 3830 return 0; 3831 } 3832 3833 static ssize_t 3834 tracing_total_entries_read(struct file *filp, char __user *ubuf, 3835 size_t cnt, loff_t *ppos) 3836 { 3837 struct trace_array *tr = filp->private_data; 3838 char buf[64]; 3839 int r, cpu; 3840 unsigned long size = 0, expanded_size = 0; 3841 3842 mutex_lock(&trace_types_lock); 3843 for_each_tracing_cpu(cpu) { 3844 size += tr->data[cpu]->entries >> 10; 3845 if (!ring_buffer_expanded) 3846 expanded_size += trace_buf_size >> 10; 3847 } 3848 if (ring_buffer_expanded) 3849 r = sprintf(buf, "%lu\n", size); 3850 else 3851 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size); 3852 mutex_unlock(&trace_types_lock); 3853 3854 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 3855 } 3856 3857 static ssize_t 3858 tracing_free_buffer_write(struct file *filp, const char __user *ubuf, 3859 size_t cnt, loff_t *ppos) 3860 { 3861 /* 3862 * There is no need to read what the user has written, this function 3863 * is just to make sure that there is no error when "echo" is used 3864 */ 3865 3866 *ppos += cnt; 3867 3868 return cnt; 3869 } 3870 3871 static int 3872 tracing_free_buffer_release(struct inode *inode, struct file *filp) 3873 { 3874 /* disable tracing ? */ 3875 if (trace_flags & TRACE_ITER_STOP_ON_FREE) 3876 tracing_off(); 3877 /* resize the ring buffer to 0 */ 3878 tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS); 3879 3880 return 0; 3881 } 3882 3883 static ssize_t 3884 tracing_mark_write(struct file *filp, const char __user *ubuf, 3885 size_t cnt, loff_t *fpos) 3886 { 3887 unsigned long addr = (unsigned long)ubuf; 3888 struct ring_buffer_event *event; 3889 struct ring_buffer *buffer; 3890 struct print_entry *entry; 3891 unsigned long irq_flags; 3892 struct page *pages[2]; 3893 void *map_page[2]; 3894 int nr_pages = 1; 3895 ssize_t written; 3896 int offset; 3897 int size; 3898 int len; 3899 int ret; 3900 int i; 3901 3902 if (tracing_disabled) 3903 return -EINVAL; 3904 3905 if (!(trace_flags & TRACE_ITER_MARKERS)) 3906 return -EINVAL; 3907 3908 if (cnt > TRACE_BUF_SIZE) 3909 cnt = TRACE_BUF_SIZE; 3910 3911 /* 3912 * Userspace is injecting traces into the kernel trace buffer. 3913 * We want to be as non intrusive as possible. 3914 * To do so, we do not want to allocate any special buffers 3915 * or take any locks, but instead write the userspace data 3916 * straight into the ring buffer. 3917 * 3918 * First we need to pin the userspace buffer into memory, 3919 * which, most likely it is, because it just referenced it. 3920 * But there's no guarantee that it is. By using get_user_pages_fast() 3921 * and kmap_atomic/kunmap_atomic() we can get access to the 3922 * pages directly. We then write the data directly into the 3923 * ring buffer. 3924 */ 3925 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); 3926 3927 /* check if we cross pages */ 3928 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK)) 3929 nr_pages = 2; 3930 3931 offset = addr & (PAGE_SIZE - 1); 3932 addr &= PAGE_MASK; 3933 3934 ret = get_user_pages_fast(addr, nr_pages, 0, pages); 3935 if (ret < nr_pages) { 3936 while (--ret >= 0) 3937 put_page(pages[ret]); 3938 written = -EFAULT; 3939 goto out; 3940 } 3941 3942 for (i = 0; i < nr_pages; i++) 3943 map_page[i] = kmap_atomic(pages[i]); 3944 3945 local_save_flags(irq_flags); 3946 size = sizeof(*entry) + cnt + 2; /* possible \n added */ 3947 buffer = global_trace.buffer; 3948 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 3949 irq_flags, preempt_count()); 3950 if (!event) { 3951 /* Ring buffer disabled, return as if not open for write */ 3952 written = -EBADF; 3953 goto out_unlock; 3954 } 3955 3956 entry = ring_buffer_event_data(event); 3957 entry->ip = _THIS_IP_; 3958 3959 if (nr_pages == 2) { 3960 len = PAGE_SIZE - offset; 3961 memcpy(&entry->buf, map_page[0] + offset, len); 3962 memcpy(&entry->buf[len], map_page[1], cnt - len); 3963 } else 3964 memcpy(&entry->buf, map_page[0] + offset, cnt); 3965 3966 if (entry->buf[cnt - 1] != '\n') { 3967 entry->buf[cnt] = '\n'; 3968 entry->buf[cnt + 1] = '\0'; 3969 } else 3970 entry->buf[cnt] = '\0'; 3971 3972 __buffer_unlock_commit(buffer, event); 3973 3974 written = cnt; 3975 3976 *fpos += written; 3977 3978 out_unlock: 3979 for (i = 0; i < nr_pages; i++){ 3980 kunmap_atomic(map_page[i]); 3981 put_page(pages[i]); 3982 } 3983 out: 3984 return written; 3985 } 3986 3987 static int tracing_clock_show(struct seq_file *m, void *v) 3988 { 3989 int i; 3990 3991 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) 3992 seq_printf(m, 3993 "%s%s%s%s", i ? " " : "", 3994 i == trace_clock_id ? "[" : "", trace_clocks[i].name, 3995 i == trace_clock_id ? "]" : ""); 3996 seq_putc(m, '\n'); 3997 3998 return 0; 3999 } 4000 4001 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf, 4002 size_t cnt, loff_t *fpos) 4003 { 4004 char buf[64]; 4005 const char *clockstr; 4006 int i; 4007 4008 if (cnt >= sizeof(buf)) 4009 return -EINVAL; 4010 4011 if (copy_from_user(&buf, ubuf, cnt)) 4012 return -EFAULT; 4013 4014 buf[cnt] = 0; 4015 4016 clockstr = strstrip(buf); 4017 4018 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) { 4019 if (strcmp(trace_clocks[i].name, clockstr) == 0) 4020 break; 4021 } 4022 if (i == ARRAY_SIZE(trace_clocks)) 4023 return -EINVAL; 4024 4025 trace_clock_id = i; 4026 4027 mutex_lock(&trace_types_lock); 4028 4029 ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func); 4030 if (max_tr.buffer) 4031 ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func); 4032 4033 /* 4034 * New clock may not be consistent with the previous clock. 4035 * Reset the buffer so that it doesn't have incomparable timestamps. 4036 */ 4037 tracing_reset_online_cpus(&global_trace); 4038 if (max_tr.buffer) 4039 tracing_reset_online_cpus(&max_tr); 4040 4041 mutex_unlock(&trace_types_lock); 4042 4043 *fpos += cnt; 4044 4045 return cnt; 4046 } 4047 4048 static int tracing_clock_open(struct inode *inode, struct file *file) 4049 { 4050 if (tracing_disabled) 4051 return -ENODEV; 4052 return single_open(file, tracing_clock_show, NULL); 4053 } 4054 4055 static const struct file_operations tracing_max_lat_fops = { 4056 .open = tracing_open_generic, 4057 .read = tracing_max_lat_read, 4058 .write = tracing_max_lat_write, 4059 .llseek = generic_file_llseek, 4060 }; 4061 4062 static const struct file_operations set_tracer_fops = { 4063 .open = tracing_open_generic, 4064 .read = tracing_set_trace_read, 4065 .write = tracing_set_trace_write, 4066 .llseek = generic_file_llseek, 4067 }; 4068 4069 static const struct file_operations tracing_pipe_fops = { 4070 .open = tracing_open_pipe, 4071 .poll = tracing_poll_pipe, 4072 .read = tracing_read_pipe, 4073 .splice_read = tracing_splice_read_pipe, 4074 .release = tracing_release_pipe, 4075 .llseek = no_llseek, 4076 }; 4077 4078 static const struct file_operations tracing_entries_fops = { 4079 .open = tracing_entries_open, 4080 .read = tracing_entries_read, 4081 .write = tracing_entries_write, 4082 .release = tracing_entries_release, 4083 .llseek = generic_file_llseek, 4084 }; 4085 4086 static const struct file_operations tracing_total_entries_fops = { 4087 .open = tracing_open_generic, 4088 .read = tracing_total_entries_read, 4089 .llseek = generic_file_llseek, 4090 }; 4091 4092 static const struct file_operations tracing_free_buffer_fops = { 4093 .write = tracing_free_buffer_write, 4094 .release = tracing_free_buffer_release, 4095 }; 4096 4097 static const struct file_operations tracing_mark_fops = { 4098 .open = tracing_open_generic, 4099 .write = tracing_mark_write, 4100 .llseek = generic_file_llseek, 4101 }; 4102 4103 static const struct file_operations trace_clock_fops = { 4104 .open = tracing_clock_open, 4105 .read = seq_read, 4106 .llseek = seq_lseek, 4107 .release = single_release, 4108 .write = tracing_clock_write, 4109 }; 4110 4111 struct ftrace_buffer_info { 4112 struct trace_array *tr; 4113 void *spare; 4114 int cpu; 4115 unsigned int read; 4116 }; 4117 4118 static int tracing_buffers_open(struct inode *inode, struct file *filp) 4119 { 4120 int cpu = (int)(long)inode->i_private; 4121 struct ftrace_buffer_info *info; 4122 4123 if (tracing_disabled) 4124 return -ENODEV; 4125 4126 info = kzalloc(sizeof(*info), GFP_KERNEL); 4127 if (!info) 4128 return -ENOMEM; 4129 4130 info->tr = &global_trace; 4131 info->cpu = cpu; 4132 info->spare = NULL; 4133 /* Force reading ring buffer for first read */ 4134 info->read = (unsigned int)-1; 4135 4136 filp->private_data = info; 4137 4138 return nonseekable_open(inode, filp); 4139 } 4140 4141 static ssize_t 4142 tracing_buffers_read(struct file *filp, char __user *ubuf, 4143 size_t count, loff_t *ppos) 4144 { 4145 struct ftrace_buffer_info *info = filp->private_data; 4146 ssize_t ret; 4147 size_t size; 4148 4149 if (!count) 4150 return 0; 4151 4152 if (!info->spare) 4153 info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu); 4154 if (!info->spare) 4155 return -ENOMEM; 4156 4157 /* Do we have previous read data to read? */ 4158 if (info->read < PAGE_SIZE) 4159 goto read; 4160 4161 trace_access_lock(info->cpu); 4162 ret = ring_buffer_read_page(info->tr->buffer, 4163 &info->spare, 4164 count, 4165 info->cpu, 0); 4166 trace_access_unlock(info->cpu); 4167 if (ret < 0) 4168 return 0; 4169 4170 info->read = 0; 4171 4172 read: 4173 size = PAGE_SIZE - info->read; 4174 if (size > count) 4175 size = count; 4176 4177 ret = copy_to_user(ubuf, info->spare + info->read, size); 4178 if (ret == size) 4179 return -EFAULT; 4180 size -= ret; 4181 4182 *ppos += size; 4183 info->read += size; 4184 4185 return size; 4186 } 4187 4188 static int tracing_buffers_release(struct inode *inode, struct file *file) 4189 { 4190 struct ftrace_buffer_info *info = file->private_data; 4191 4192 if (info->spare) 4193 ring_buffer_free_read_page(info->tr->buffer, info->spare); 4194 kfree(info); 4195 4196 return 0; 4197 } 4198 4199 struct buffer_ref { 4200 struct ring_buffer *buffer; 4201 void *page; 4202 int ref; 4203 }; 4204 4205 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe, 4206 struct pipe_buffer *buf) 4207 { 4208 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 4209 4210 if (--ref->ref) 4211 return; 4212 4213 ring_buffer_free_read_page(ref->buffer, ref->page); 4214 kfree(ref); 4215 buf->private = 0; 4216 } 4217 4218 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe, 4219 struct pipe_buffer *buf) 4220 { 4221 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 4222 4223 ref->ref++; 4224 } 4225 4226 /* Pipe buffer operations for a buffer. */ 4227 static const struct pipe_buf_operations buffer_pipe_buf_ops = { 4228 .can_merge = 0, 4229 .map = generic_pipe_buf_map, 4230 .unmap = generic_pipe_buf_unmap, 4231 .confirm = generic_pipe_buf_confirm, 4232 .release = buffer_pipe_buf_release, 4233 .steal = generic_pipe_buf_steal, 4234 .get = buffer_pipe_buf_get, 4235 }; 4236 4237 /* 4238 * Callback from splice_to_pipe(), if we need to release some pages 4239 * at the end of the spd in case we error'ed out in filling the pipe. 4240 */ 4241 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i) 4242 { 4243 struct buffer_ref *ref = 4244 (struct buffer_ref *)spd->partial[i].private; 4245 4246 if (--ref->ref) 4247 return; 4248 4249 ring_buffer_free_read_page(ref->buffer, ref->page); 4250 kfree(ref); 4251 spd->partial[i].private = 0; 4252 } 4253 4254 static ssize_t 4255 tracing_buffers_splice_read(struct file *file, loff_t *ppos, 4256 struct pipe_inode_info *pipe, size_t len, 4257 unsigned int flags) 4258 { 4259 struct ftrace_buffer_info *info = file->private_data; 4260 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 4261 struct page *pages_def[PIPE_DEF_BUFFERS]; 4262 struct splice_pipe_desc spd = { 4263 .pages = pages_def, 4264 .partial = partial_def, 4265 .nr_pages_max = PIPE_DEF_BUFFERS, 4266 .flags = flags, 4267 .ops = &buffer_pipe_buf_ops, 4268 .spd_release = buffer_spd_release, 4269 }; 4270 struct buffer_ref *ref; 4271 int entries, size, i; 4272 size_t ret; 4273 4274 if (splice_grow_spd(pipe, &spd)) 4275 return -ENOMEM; 4276 4277 if (*ppos & (PAGE_SIZE - 1)) { 4278 ret = -EINVAL; 4279 goto out; 4280 } 4281 4282 if (len & (PAGE_SIZE - 1)) { 4283 if (len < PAGE_SIZE) { 4284 ret = -EINVAL; 4285 goto out; 4286 } 4287 len &= PAGE_MASK; 4288 } 4289 4290 trace_access_lock(info->cpu); 4291 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu); 4292 4293 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) { 4294 struct page *page; 4295 int r; 4296 4297 ref = kzalloc(sizeof(*ref), GFP_KERNEL); 4298 if (!ref) 4299 break; 4300 4301 ref->ref = 1; 4302 ref->buffer = info->tr->buffer; 4303 ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu); 4304 if (!ref->page) { 4305 kfree(ref); 4306 break; 4307 } 4308 4309 r = ring_buffer_read_page(ref->buffer, &ref->page, 4310 len, info->cpu, 1); 4311 if (r < 0) { 4312 ring_buffer_free_read_page(ref->buffer, ref->page); 4313 kfree(ref); 4314 break; 4315 } 4316 4317 /* 4318 * zero out any left over data, this is going to 4319 * user land. 4320 */ 4321 size = ring_buffer_page_len(ref->page); 4322 if (size < PAGE_SIZE) 4323 memset(ref->page + size, 0, PAGE_SIZE - size); 4324 4325 page = virt_to_page(ref->page); 4326 4327 spd.pages[i] = page; 4328 spd.partial[i].len = PAGE_SIZE; 4329 spd.partial[i].offset = 0; 4330 spd.partial[i].private = (unsigned long)ref; 4331 spd.nr_pages++; 4332 *ppos += PAGE_SIZE; 4333 4334 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu); 4335 } 4336 4337 trace_access_unlock(info->cpu); 4338 spd.nr_pages = i; 4339 4340 /* did we read anything? */ 4341 if (!spd.nr_pages) { 4342 if (flags & SPLICE_F_NONBLOCK) 4343 ret = -EAGAIN; 4344 else 4345 ret = 0; 4346 /* TODO: block */ 4347 goto out; 4348 } 4349 4350 ret = splice_to_pipe(pipe, &spd); 4351 splice_shrink_spd(&spd); 4352 out: 4353 return ret; 4354 } 4355 4356 static const struct file_operations tracing_buffers_fops = { 4357 .open = tracing_buffers_open, 4358 .read = tracing_buffers_read, 4359 .release = tracing_buffers_release, 4360 .splice_read = tracing_buffers_splice_read, 4361 .llseek = no_llseek, 4362 }; 4363 4364 static ssize_t 4365 tracing_stats_read(struct file *filp, char __user *ubuf, 4366 size_t count, loff_t *ppos) 4367 { 4368 unsigned long cpu = (unsigned long)filp->private_data; 4369 struct trace_array *tr = &global_trace; 4370 struct trace_seq *s; 4371 unsigned long cnt; 4372 unsigned long long t; 4373 unsigned long usec_rem; 4374 4375 s = kmalloc(sizeof(*s), GFP_KERNEL); 4376 if (!s) 4377 return -ENOMEM; 4378 4379 trace_seq_init(s); 4380 4381 cnt = ring_buffer_entries_cpu(tr->buffer, cpu); 4382 trace_seq_printf(s, "entries: %ld\n", cnt); 4383 4384 cnt = ring_buffer_overrun_cpu(tr->buffer, cpu); 4385 trace_seq_printf(s, "overrun: %ld\n", cnt); 4386 4387 cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu); 4388 trace_seq_printf(s, "commit overrun: %ld\n", cnt); 4389 4390 cnt = ring_buffer_bytes_cpu(tr->buffer, cpu); 4391 trace_seq_printf(s, "bytes: %ld\n", cnt); 4392 4393 if (trace_clocks[trace_clock_id].in_ns) { 4394 /* local or global for trace_clock */ 4395 t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu)); 4396 usec_rem = do_div(t, USEC_PER_SEC); 4397 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", 4398 t, usec_rem); 4399 4400 t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu)); 4401 usec_rem = do_div(t, USEC_PER_SEC); 4402 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem); 4403 } else { 4404 /* counter or tsc mode for trace_clock */ 4405 trace_seq_printf(s, "oldest event ts: %llu\n", 4406 ring_buffer_oldest_event_ts(tr->buffer, cpu)); 4407 4408 trace_seq_printf(s, "now ts: %llu\n", 4409 ring_buffer_time_stamp(tr->buffer, cpu)); 4410 } 4411 4412 cnt = ring_buffer_dropped_events_cpu(tr->buffer, cpu); 4413 trace_seq_printf(s, "dropped events: %ld\n", cnt); 4414 4415 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len); 4416 4417 kfree(s); 4418 4419 return count; 4420 } 4421 4422 static const struct file_operations tracing_stats_fops = { 4423 .open = tracing_open_generic, 4424 .read = tracing_stats_read, 4425 .llseek = generic_file_llseek, 4426 }; 4427 4428 #ifdef CONFIG_DYNAMIC_FTRACE 4429 4430 int __weak ftrace_arch_read_dyn_info(char *buf, int size) 4431 { 4432 return 0; 4433 } 4434 4435 static ssize_t 4436 tracing_read_dyn_info(struct file *filp, char __user *ubuf, 4437 size_t cnt, loff_t *ppos) 4438 { 4439 static char ftrace_dyn_info_buffer[1024]; 4440 static DEFINE_MUTEX(dyn_info_mutex); 4441 unsigned long *p = filp->private_data; 4442 char *buf = ftrace_dyn_info_buffer; 4443 int size = ARRAY_SIZE(ftrace_dyn_info_buffer); 4444 int r; 4445 4446 mutex_lock(&dyn_info_mutex); 4447 r = sprintf(buf, "%ld ", *p); 4448 4449 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r); 4450 buf[r++] = '\n'; 4451 4452 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 4453 4454 mutex_unlock(&dyn_info_mutex); 4455 4456 return r; 4457 } 4458 4459 static const struct file_operations tracing_dyn_info_fops = { 4460 .open = tracing_open_generic, 4461 .read = tracing_read_dyn_info, 4462 .llseek = generic_file_llseek, 4463 }; 4464 #endif 4465 4466 static struct dentry *d_tracer; 4467 4468 struct dentry *tracing_init_dentry(void) 4469 { 4470 static int once; 4471 4472 if (d_tracer) 4473 return d_tracer; 4474 4475 if (!debugfs_initialized()) 4476 return NULL; 4477 4478 d_tracer = debugfs_create_dir("tracing", NULL); 4479 4480 if (!d_tracer && !once) { 4481 once = 1; 4482 pr_warning("Could not create debugfs directory 'tracing'\n"); 4483 return NULL; 4484 } 4485 4486 return d_tracer; 4487 } 4488 4489 static struct dentry *d_percpu; 4490 4491 struct dentry *tracing_dentry_percpu(void) 4492 { 4493 static int once; 4494 struct dentry *d_tracer; 4495 4496 if (d_percpu) 4497 return d_percpu; 4498 4499 d_tracer = tracing_init_dentry(); 4500 4501 if (!d_tracer) 4502 return NULL; 4503 4504 d_percpu = debugfs_create_dir("per_cpu", d_tracer); 4505 4506 if (!d_percpu && !once) { 4507 once = 1; 4508 pr_warning("Could not create debugfs directory 'per_cpu'\n"); 4509 return NULL; 4510 } 4511 4512 return d_percpu; 4513 } 4514 4515 static void tracing_init_debugfs_percpu(long cpu) 4516 { 4517 struct dentry *d_percpu = tracing_dentry_percpu(); 4518 struct dentry *d_cpu; 4519 char cpu_dir[30]; /* 30 characters should be more than enough */ 4520 4521 if (!d_percpu) 4522 return; 4523 4524 snprintf(cpu_dir, 30, "cpu%ld", cpu); 4525 d_cpu = debugfs_create_dir(cpu_dir, d_percpu); 4526 if (!d_cpu) { 4527 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir); 4528 return; 4529 } 4530 4531 /* per cpu trace_pipe */ 4532 trace_create_file("trace_pipe", 0444, d_cpu, 4533 (void *) cpu, &tracing_pipe_fops); 4534 4535 /* per cpu trace */ 4536 trace_create_file("trace", 0644, d_cpu, 4537 (void *) cpu, &tracing_fops); 4538 4539 trace_create_file("trace_pipe_raw", 0444, d_cpu, 4540 (void *) cpu, &tracing_buffers_fops); 4541 4542 trace_create_file("stats", 0444, d_cpu, 4543 (void *) cpu, &tracing_stats_fops); 4544 4545 trace_create_file("buffer_size_kb", 0444, d_cpu, 4546 (void *) cpu, &tracing_entries_fops); 4547 } 4548 4549 #ifdef CONFIG_FTRACE_SELFTEST 4550 /* Let selftest have access to static functions in this file */ 4551 #include "trace_selftest.c" 4552 #endif 4553 4554 struct trace_option_dentry { 4555 struct tracer_opt *opt; 4556 struct tracer_flags *flags; 4557 struct dentry *entry; 4558 }; 4559 4560 static ssize_t 4561 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt, 4562 loff_t *ppos) 4563 { 4564 struct trace_option_dentry *topt = filp->private_data; 4565 char *buf; 4566 4567 if (topt->flags->val & topt->opt->bit) 4568 buf = "1\n"; 4569 else 4570 buf = "0\n"; 4571 4572 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 4573 } 4574 4575 static ssize_t 4576 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt, 4577 loff_t *ppos) 4578 { 4579 struct trace_option_dentry *topt = filp->private_data; 4580 unsigned long val; 4581 int ret; 4582 4583 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 4584 if (ret) 4585 return ret; 4586 4587 if (val != 0 && val != 1) 4588 return -EINVAL; 4589 4590 if (!!(topt->flags->val & topt->opt->bit) != val) { 4591 mutex_lock(&trace_types_lock); 4592 ret = __set_tracer_option(current_trace, topt->flags, 4593 topt->opt, !val); 4594 mutex_unlock(&trace_types_lock); 4595 if (ret) 4596 return ret; 4597 } 4598 4599 *ppos += cnt; 4600 4601 return cnt; 4602 } 4603 4604 4605 static const struct file_operations trace_options_fops = { 4606 .open = tracing_open_generic, 4607 .read = trace_options_read, 4608 .write = trace_options_write, 4609 .llseek = generic_file_llseek, 4610 }; 4611 4612 static ssize_t 4613 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt, 4614 loff_t *ppos) 4615 { 4616 long index = (long)filp->private_data; 4617 char *buf; 4618 4619 if (trace_flags & (1 << index)) 4620 buf = "1\n"; 4621 else 4622 buf = "0\n"; 4623 4624 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 4625 } 4626 4627 static ssize_t 4628 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt, 4629 loff_t *ppos) 4630 { 4631 long index = (long)filp->private_data; 4632 unsigned long val; 4633 int ret; 4634 4635 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 4636 if (ret) 4637 return ret; 4638 4639 if (val != 0 && val != 1) 4640 return -EINVAL; 4641 set_tracer_flags(1 << index, val); 4642 4643 *ppos += cnt; 4644 4645 return cnt; 4646 } 4647 4648 static const struct file_operations trace_options_core_fops = { 4649 .open = tracing_open_generic, 4650 .read = trace_options_core_read, 4651 .write = trace_options_core_write, 4652 .llseek = generic_file_llseek, 4653 }; 4654 4655 struct dentry *trace_create_file(const char *name, 4656 umode_t mode, 4657 struct dentry *parent, 4658 void *data, 4659 const struct file_operations *fops) 4660 { 4661 struct dentry *ret; 4662 4663 ret = debugfs_create_file(name, mode, parent, data, fops); 4664 if (!ret) 4665 pr_warning("Could not create debugfs '%s' entry\n", name); 4666 4667 return ret; 4668 } 4669 4670 4671 static struct dentry *trace_options_init_dentry(void) 4672 { 4673 struct dentry *d_tracer; 4674 static struct dentry *t_options; 4675 4676 if (t_options) 4677 return t_options; 4678 4679 d_tracer = tracing_init_dentry(); 4680 if (!d_tracer) 4681 return NULL; 4682 4683 t_options = debugfs_create_dir("options", d_tracer); 4684 if (!t_options) { 4685 pr_warning("Could not create debugfs directory 'options'\n"); 4686 return NULL; 4687 } 4688 4689 return t_options; 4690 } 4691 4692 static void 4693 create_trace_option_file(struct trace_option_dentry *topt, 4694 struct tracer_flags *flags, 4695 struct tracer_opt *opt) 4696 { 4697 struct dentry *t_options; 4698 4699 t_options = trace_options_init_dentry(); 4700 if (!t_options) 4701 return; 4702 4703 topt->flags = flags; 4704 topt->opt = opt; 4705 4706 topt->entry = trace_create_file(opt->name, 0644, t_options, topt, 4707 &trace_options_fops); 4708 4709 } 4710 4711 static struct trace_option_dentry * 4712 create_trace_option_files(struct tracer *tracer) 4713 { 4714 struct trace_option_dentry *topts; 4715 struct tracer_flags *flags; 4716 struct tracer_opt *opts; 4717 int cnt; 4718 4719 if (!tracer) 4720 return NULL; 4721 4722 flags = tracer->flags; 4723 4724 if (!flags || !flags->opts) 4725 return NULL; 4726 4727 opts = flags->opts; 4728 4729 for (cnt = 0; opts[cnt].name; cnt++) 4730 ; 4731 4732 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL); 4733 if (!topts) 4734 return NULL; 4735 4736 for (cnt = 0; opts[cnt].name; cnt++) 4737 create_trace_option_file(&topts[cnt], flags, 4738 &opts[cnt]); 4739 4740 return topts; 4741 } 4742 4743 static void 4744 destroy_trace_option_files(struct trace_option_dentry *topts) 4745 { 4746 int cnt; 4747 4748 if (!topts) 4749 return; 4750 4751 for (cnt = 0; topts[cnt].opt; cnt++) { 4752 if (topts[cnt].entry) 4753 debugfs_remove(topts[cnt].entry); 4754 } 4755 4756 kfree(topts); 4757 } 4758 4759 static struct dentry * 4760 create_trace_option_core_file(const char *option, long index) 4761 { 4762 struct dentry *t_options; 4763 4764 t_options = trace_options_init_dentry(); 4765 if (!t_options) 4766 return NULL; 4767 4768 return trace_create_file(option, 0644, t_options, (void *)index, 4769 &trace_options_core_fops); 4770 } 4771 4772 static __init void create_trace_options_dir(void) 4773 { 4774 struct dentry *t_options; 4775 int i; 4776 4777 t_options = trace_options_init_dentry(); 4778 if (!t_options) 4779 return; 4780 4781 for (i = 0; trace_options[i]; i++) 4782 create_trace_option_core_file(trace_options[i], i); 4783 } 4784 4785 static ssize_t 4786 rb_simple_read(struct file *filp, char __user *ubuf, 4787 size_t cnt, loff_t *ppos) 4788 { 4789 struct trace_array *tr = filp->private_data; 4790 struct ring_buffer *buffer = tr->buffer; 4791 char buf[64]; 4792 int r; 4793 4794 if (buffer) 4795 r = ring_buffer_record_is_on(buffer); 4796 else 4797 r = 0; 4798 4799 r = sprintf(buf, "%d\n", r); 4800 4801 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 4802 } 4803 4804 static ssize_t 4805 rb_simple_write(struct file *filp, const char __user *ubuf, 4806 size_t cnt, loff_t *ppos) 4807 { 4808 struct trace_array *tr = filp->private_data; 4809 struct ring_buffer *buffer = tr->buffer; 4810 unsigned long val; 4811 int ret; 4812 4813 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 4814 if (ret) 4815 return ret; 4816 4817 if (buffer) { 4818 if (val) 4819 ring_buffer_record_on(buffer); 4820 else 4821 ring_buffer_record_off(buffer); 4822 } 4823 4824 (*ppos)++; 4825 4826 return cnt; 4827 } 4828 4829 static const struct file_operations rb_simple_fops = { 4830 .open = tracing_open_generic, 4831 .read = rb_simple_read, 4832 .write = rb_simple_write, 4833 .llseek = default_llseek, 4834 }; 4835 4836 static __init int tracer_init_debugfs(void) 4837 { 4838 struct dentry *d_tracer; 4839 int cpu; 4840 4841 trace_access_lock_init(); 4842 4843 d_tracer = tracing_init_dentry(); 4844 4845 trace_create_file("trace_options", 0644, d_tracer, 4846 NULL, &tracing_iter_fops); 4847 4848 trace_create_file("tracing_cpumask", 0644, d_tracer, 4849 NULL, &tracing_cpumask_fops); 4850 4851 trace_create_file("trace", 0644, d_tracer, 4852 (void *) TRACE_PIPE_ALL_CPU, &tracing_fops); 4853 4854 trace_create_file("available_tracers", 0444, d_tracer, 4855 &global_trace, &show_traces_fops); 4856 4857 trace_create_file("current_tracer", 0644, d_tracer, 4858 &global_trace, &set_tracer_fops); 4859 4860 #ifdef CONFIG_TRACER_MAX_TRACE 4861 trace_create_file("tracing_max_latency", 0644, d_tracer, 4862 &tracing_max_latency, &tracing_max_lat_fops); 4863 #endif 4864 4865 trace_create_file("tracing_thresh", 0644, d_tracer, 4866 &tracing_thresh, &tracing_max_lat_fops); 4867 4868 trace_create_file("README", 0444, d_tracer, 4869 NULL, &tracing_readme_fops); 4870 4871 trace_create_file("trace_pipe", 0444, d_tracer, 4872 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops); 4873 4874 trace_create_file("buffer_size_kb", 0644, d_tracer, 4875 (void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops); 4876 4877 trace_create_file("buffer_total_size_kb", 0444, d_tracer, 4878 &global_trace, &tracing_total_entries_fops); 4879 4880 trace_create_file("free_buffer", 0644, d_tracer, 4881 &global_trace, &tracing_free_buffer_fops); 4882 4883 trace_create_file("trace_marker", 0220, d_tracer, 4884 NULL, &tracing_mark_fops); 4885 4886 trace_create_file("saved_cmdlines", 0444, d_tracer, 4887 NULL, &tracing_saved_cmdlines_fops); 4888 4889 trace_create_file("trace_clock", 0644, d_tracer, NULL, 4890 &trace_clock_fops); 4891 4892 trace_create_file("tracing_on", 0644, d_tracer, 4893 &global_trace, &rb_simple_fops); 4894 4895 #ifdef CONFIG_DYNAMIC_FTRACE 4896 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer, 4897 &ftrace_update_tot_cnt, &tracing_dyn_info_fops); 4898 #endif 4899 4900 create_trace_options_dir(); 4901 4902 for_each_tracing_cpu(cpu) 4903 tracing_init_debugfs_percpu(cpu); 4904 4905 return 0; 4906 } 4907 4908 static int trace_panic_handler(struct notifier_block *this, 4909 unsigned long event, void *unused) 4910 { 4911 if (ftrace_dump_on_oops) 4912 ftrace_dump(ftrace_dump_on_oops); 4913 return NOTIFY_OK; 4914 } 4915 4916 static struct notifier_block trace_panic_notifier = { 4917 .notifier_call = trace_panic_handler, 4918 .next = NULL, 4919 .priority = 150 /* priority: INT_MAX >= x >= 0 */ 4920 }; 4921 4922 static int trace_die_handler(struct notifier_block *self, 4923 unsigned long val, 4924 void *data) 4925 { 4926 switch (val) { 4927 case DIE_OOPS: 4928 if (ftrace_dump_on_oops) 4929 ftrace_dump(ftrace_dump_on_oops); 4930 break; 4931 default: 4932 break; 4933 } 4934 return NOTIFY_OK; 4935 } 4936 4937 static struct notifier_block trace_die_notifier = { 4938 .notifier_call = trace_die_handler, 4939 .priority = 200 4940 }; 4941 4942 /* 4943 * printk is set to max of 1024, we really don't need it that big. 4944 * Nothing should be printing 1000 characters anyway. 4945 */ 4946 #define TRACE_MAX_PRINT 1000 4947 4948 /* 4949 * Define here KERN_TRACE so that we have one place to modify 4950 * it if we decide to change what log level the ftrace dump 4951 * should be at. 4952 */ 4953 #define KERN_TRACE KERN_EMERG 4954 4955 void 4956 trace_printk_seq(struct trace_seq *s) 4957 { 4958 /* Probably should print a warning here. */ 4959 if (s->len >= 1000) 4960 s->len = 1000; 4961 4962 /* should be zero ended, but we are paranoid. */ 4963 s->buffer[s->len] = 0; 4964 4965 printk(KERN_TRACE "%s", s->buffer); 4966 4967 trace_seq_init(s); 4968 } 4969 4970 void trace_init_global_iter(struct trace_iterator *iter) 4971 { 4972 iter->tr = &global_trace; 4973 iter->trace = current_trace; 4974 iter->cpu_file = TRACE_PIPE_ALL_CPU; 4975 } 4976 4977 static void 4978 __ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode) 4979 { 4980 static arch_spinlock_t ftrace_dump_lock = 4981 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 4982 /* use static because iter can be a bit big for the stack */ 4983 static struct trace_iterator iter; 4984 unsigned int old_userobj; 4985 static int dump_ran; 4986 unsigned long flags; 4987 int cnt = 0, cpu; 4988 4989 /* only one dump */ 4990 local_irq_save(flags); 4991 arch_spin_lock(&ftrace_dump_lock); 4992 if (dump_ran) 4993 goto out; 4994 4995 dump_ran = 1; 4996 4997 tracing_off(); 4998 4999 /* Did function tracer already get disabled? */ 5000 if (ftrace_is_dead()) { 5001 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n"); 5002 printk("# MAY BE MISSING FUNCTION EVENTS\n"); 5003 } 5004 5005 if (disable_tracing) 5006 ftrace_kill(); 5007 5008 trace_init_global_iter(&iter); 5009 5010 for_each_tracing_cpu(cpu) { 5011 atomic_inc(&iter.tr->data[cpu]->disabled); 5012 } 5013 5014 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ; 5015 5016 /* don't look at user memory in panic mode */ 5017 trace_flags &= ~TRACE_ITER_SYM_USEROBJ; 5018 5019 /* Simulate the iterator */ 5020 iter.tr = &global_trace; 5021 iter.trace = current_trace; 5022 5023 switch (oops_dump_mode) { 5024 case DUMP_ALL: 5025 iter.cpu_file = TRACE_PIPE_ALL_CPU; 5026 break; 5027 case DUMP_ORIG: 5028 iter.cpu_file = raw_smp_processor_id(); 5029 break; 5030 case DUMP_NONE: 5031 goto out_enable; 5032 default: 5033 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n"); 5034 iter.cpu_file = TRACE_PIPE_ALL_CPU; 5035 } 5036 5037 printk(KERN_TRACE "Dumping ftrace buffer:\n"); 5038 5039 /* 5040 * We need to stop all tracing on all CPUS to read the 5041 * the next buffer. This is a bit expensive, but is 5042 * not done often. We fill all what we can read, 5043 * and then release the locks again. 5044 */ 5045 5046 while (!trace_empty(&iter)) { 5047 5048 if (!cnt) 5049 printk(KERN_TRACE "---------------------------------\n"); 5050 5051 cnt++; 5052 5053 /* reset all but tr, trace, and overruns */ 5054 memset(&iter.seq, 0, 5055 sizeof(struct trace_iterator) - 5056 offsetof(struct trace_iterator, seq)); 5057 iter.iter_flags |= TRACE_FILE_LAT_FMT; 5058 iter.pos = -1; 5059 5060 if (trace_find_next_entry_inc(&iter) != NULL) { 5061 int ret; 5062 5063 ret = print_trace_line(&iter); 5064 if (ret != TRACE_TYPE_NO_CONSUME) 5065 trace_consume(&iter); 5066 } 5067 touch_nmi_watchdog(); 5068 5069 trace_printk_seq(&iter.seq); 5070 } 5071 5072 if (!cnt) 5073 printk(KERN_TRACE " (ftrace buffer empty)\n"); 5074 else 5075 printk(KERN_TRACE "---------------------------------\n"); 5076 5077 out_enable: 5078 /* Re-enable tracing if requested */ 5079 if (!disable_tracing) { 5080 trace_flags |= old_userobj; 5081 5082 for_each_tracing_cpu(cpu) { 5083 atomic_dec(&iter.tr->data[cpu]->disabled); 5084 } 5085 tracing_on(); 5086 } 5087 5088 out: 5089 arch_spin_unlock(&ftrace_dump_lock); 5090 local_irq_restore(flags); 5091 } 5092 5093 /* By default: disable tracing after the dump */ 5094 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) 5095 { 5096 __ftrace_dump(true, oops_dump_mode); 5097 } 5098 EXPORT_SYMBOL_GPL(ftrace_dump); 5099 5100 __init static int tracer_alloc_buffers(void) 5101 { 5102 int ring_buf_size; 5103 enum ring_buffer_flags rb_flags; 5104 int i; 5105 int ret = -ENOMEM; 5106 5107 5108 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL)) 5109 goto out; 5110 5111 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL)) 5112 goto out_free_buffer_mask; 5113 5114 /* Only allocate trace_printk buffers if a trace_printk exists */ 5115 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt) 5116 /* Must be called before global_trace.buffer is allocated */ 5117 trace_printk_init_buffers(); 5118 5119 /* To save memory, keep the ring buffer size to its minimum */ 5120 if (ring_buffer_expanded) 5121 ring_buf_size = trace_buf_size; 5122 else 5123 ring_buf_size = 1; 5124 5125 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0; 5126 5127 cpumask_copy(tracing_buffer_mask, cpu_possible_mask); 5128 cpumask_copy(tracing_cpumask, cpu_all_mask); 5129 5130 /* TODO: make the number of buffers hot pluggable with CPUS */ 5131 global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags); 5132 if (!global_trace.buffer) { 5133 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n"); 5134 WARN_ON(1); 5135 goto out_free_cpumask; 5136 } 5137 if (global_trace.buffer_disabled) 5138 tracing_off(); 5139 5140 5141 #ifdef CONFIG_TRACER_MAX_TRACE 5142 max_tr.buffer = ring_buffer_alloc(1, rb_flags); 5143 if (!max_tr.buffer) { 5144 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n"); 5145 WARN_ON(1); 5146 ring_buffer_free(global_trace.buffer); 5147 goto out_free_cpumask; 5148 } 5149 #endif 5150 5151 /* Allocate the first page for all buffers */ 5152 for_each_tracing_cpu(i) { 5153 global_trace.data[i] = &per_cpu(global_trace_cpu, i); 5154 max_tr.data[i] = &per_cpu(max_tr_data, i); 5155 } 5156 5157 set_buffer_entries(&global_trace, 5158 ring_buffer_size(global_trace.buffer, 0)); 5159 #ifdef CONFIG_TRACER_MAX_TRACE 5160 set_buffer_entries(&max_tr, 1); 5161 #endif 5162 5163 trace_init_cmdlines(); 5164 init_irq_work(&trace_work_wakeup, trace_wake_up); 5165 5166 register_tracer(&nop_trace); 5167 current_trace = &nop_trace; 5168 /* All seems OK, enable tracing */ 5169 tracing_disabled = 0; 5170 5171 atomic_notifier_chain_register(&panic_notifier_list, 5172 &trace_panic_notifier); 5173 5174 register_die_notifier(&trace_die_notifier); 5175 5176 while (trace_boot_options) { 5177 char *option; 5178 5179 option = strsep(&trace_boot_options, ","); 5180 trace_set_options(option); 5181 } 5182 5183 return 0; 5184 5185 out_free_cpumask: 5186 free_cpumask_var(tracing_cpumask); 5187 out_free_buffer_mask: 5188 free_cpumask_var(tracing_buffer_mask); 5189 out: 5190 return ret; 5191 } 5192 5193 __init static int clear_boot_tracer(void) 5194 { 5195 /* 5196 * The default tracer at boot buffer is an init section. 5197 * This function is called in lateinit. If we did not 5198 * find the boot tracer, then clear it out, to prevent 5199 * later registration from accessing the buffer that is 5200 * about to be freed. 5201 */ 5202 if (!default_bootup_tracer) 5203 return 0; 5204 5205 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n", 5206 default_bootup_tracer); 5207 default_bootup_tracer = NULL; 5208 5209 return 0; 5210 } 5211 5212 early_initcall(tracer_alloc_buffers); 5213 fs_initcall(tracer_init_debugfs); 5214 late_initcall(clear_boot_tracer); 5215