194b80ffdSSteven Rostedt /* 294b80ffdSSteven Rostedt * unlikely profiler 394b80ffdSSteven Rostedt * 494b80ffdSSteven Rostedt * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> 594b80ffdSSteven Rostedt */ 694b80ffdSSteven Rostedt #include <linux/kallsyms.h> 794b80ffdSSteven Rostedt #include <linux/seq_file.h> 894b80ffdSSteven Rostedt #include <linux/spinlock.h> 965c6dc6aSFrederic Weisbecker #include <linux/irqflags.h> 1094b80ffdSSteven Rostedt #include <linux/debugfs.h> 1194b80ffdSSteven Rostedt #include <linux/uaccess.h> 1294b80ffdSSteven Rostedt #include <linux/module.h> 1394b80ffdSSteven Rostedt #include <linux/ftrace.h> 1494b80ffdSSteven Rostedt #include <linux/hash.h> 1594b80ffdSSteven Rostedt #include <linux/fs.h> 1694b80ffdSSteven Rostedt #include <asm/local.h> 1794b80ffdSSteven Rostedt #include "trace.h" 1894b80ffdSSteven Rostedt 1994b80ffdSSteven Rostedt #ifdef CONFIG_BRANCH_TRACER 2094b80ffdSSteven Rostedt 2194b80ffdSSteven Rostedt static int branch_tracing_enabled __read_mostly; 2294b80ffdSSteven Rostedt static DEFINE_MUTEX(branch_tracing_mutex); 2394b80ffdSSteven Rostedt static struct trace_array *branch_tracer; 2494b80ffdSSteven Rostedt 2594b80ffdSSteven Rostedt static void 2694b80ffdSSteven Rostedt probe_likely_condition(struct ftrace_branch_data *f, int val, int expect) 2794b80ffdSSteven Rostedt { 2894b80ffdSSteven Rostedt struct trace_array *tr = branch_tracer; 2994b80ffdSSteven Rostedt struct ring_buffer_event *event; 3094b80ffdSSteven Rostedt struct trace_branch *entry; 3194b80ffdSSteven Rostedt unsigned long flags, irq_flags; 3294b80ffdSSteven Rostedt int cpu, pc; 3394b80ffdSSteven Rostedt const char *p; 3494b80ffdSSteven Rostedt 3594b80ffdSSteven Rostedt /* 3694b80ffdSSteven Rostedt * I would love to save just the ftrace_likely_data pointer, but 3794b80ffdSSteven Rostedt * this code can also be used by modules. Ugly things can happen 3894b80ffdSSteven Rostedt * if the module is unloaded, and then we go and read the 3994b80ffdSSteven Rostedt * pointer. This is slower, but much safer. 4094b80ffdSSteven Rostedt */ 4194b80ffdSSteven Rostedt 4294b80ffdSSteven Rostedt if (unlikely(!tr)) 4394b80ffdSSteven Rostedt return; 4494b80ffdSSteven Rostedt 45a5e25883SSteven Rostedt local_irq_save(flags); 4694b80ffdSSteven Rostedt cpu = raw_smp_processor_id(); 4794b80ffdSSteven Rostedt if (atomic_inc_return(&tr->data[cpu]->disabled) != 1) 4894b80ffdSSteven Rostedt goto out; 4994b80ffdSSteven Rostedt 5094b80ffdSSteven Rostedt event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 5194b80ffdSSteven Rostedt &irq_flags); 5294b80ffdSSteven Rostedt if (!event) 5394b80ffdSSteven Rostedt goto out; 5494b80ffdSSteven Rostedt 5594b80ffdSSteven Rostedt pc = preempt_count(); 5694b80ffdSSteven Rostedt entry = ring_buffer_event_data(event); 5794b80ffdSSteven Rostedt tracing_generic_entry_update(&entry->ent, flags, pc); 5894b80ffdSSteven Rostedt entry->ent.type = TRACE_BRANCH; 5994b80ffdSSteven Rostedt 6094b80ffdSSteven Rostedt /* Strip off the path, only save the file */ 6194b80ffdSSteven Rostedt p = f->file + strlen(f->file); 6294b80ffdSSteven Rostedt while (p >= f->file && *p != '/') 6394b80ffdSSteven Rostedt p--; 6494b80ffdSSteven Rostedt p++; 6594b80ffdSSteven Rostedt 6694b80ffdSSteven Rostedt strncpy(entry->func, f->func, TRACE_FUNC_SIZE); 6794b80ffdSSteven Rostedt strncpy(entry->file, p, TRACE_FILE_SIZE); 6894b80ffdSSteven Rostedt entry->func[TRACE_FUNC_SIZE] = 0; 6994b80ffdSSteven Rostedt entry->file[TRACE_FILE_SIZE] = 0; 7094b80ffdSSteven Rostedt entry->line = f->line; 7194b80ffdSSteven Rostedt entry->correct = val == expect; 7294b80ffdSSteven Rostedt 7394b80ffdSSteven Rostedt ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 7494b80ffdSSteven Rostedt 7594b80ffdSSteven Rostedt out: 7694b80ffdSSteven Rostedt atomic_dec(&tr->data[cpu]->disabled); 77a5e25883SSteven Rostedt local_irq_restore(flags); 7894b80ffdSSteven Rostedt } 7994b80ffdSSteven Rostedt 8094b80ffdSSteven Rostedt static inline 8194b80ffdSSteven Rostedt void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) 8294b80ffdSSteven Rostedt { 8394b80ffdSSteven Rostedt if (!branch_tracing_enabled) 8494b80ffdSSteven Rostedt return; 8594b80ffdSSteven Rostedt 8694b80ffdSSteven Rostedt probe_likely_condition(f, val, expect); 8794b80ffdSSteven Rostedt } 8894b80ffdSSteven Rostedt 8994b80ffdSSteven Rostedt int enable_branch_tracing(struct trace_array *tr) 9094b80ffdSSteven Rostedt { 9194b80ffdSSteven Rostedt int ret = 0; 9294b80ffdSSteven Rostedt 9394b80ffdSSteven Rostedt mutex_lock(&branch_tracing_mutex); 9494b80ffdSSteven Rostedt branch_tracer = tr; 9594b80ffdSSteven Rostedt /* 9694b80ffdSSteven Rostedt * Must be seen before enabling. The reader is a condition 9794b80ffdSSteven Rostedt * where we do not need a matching rmb() 9894b80ffdSSteven Rostedt */ 9994b80ffdSSteven Rostedt smp_wmb(); 10094b80ffdSSteven Rostedt branch_tracing_enabled++; 10194b80ffdSSteven Rostedt mutex_unlock(&branch_tracing_mutex); 10294b80ffdSSteven Rostedt 10394b80ffdSSteven Rostedt return ret; 10494b80ffdSSteven Rostedt } 10594b80ffdSSteven Rostedt 10694b80ffdSSteven Rostedt void disable_branch_tracing(void) 10794b80ffdSSteven Rostedt { 10894b80ffdSSteven Rostedt mutex_lock(&branch_tracing_mutex); 10994b80ffdSSteven Rostedt 11094b80ffdSSteven Rostedt if (!branch_tracing_enabled) 11194b80ffdSSteven Rostedt goto out_unlock; 11294b80ffdSSteven Rostedt 11394b80ffdSSteven Rostedt branch_tracing_enabled--; 11494b80ffdSSteven Rostedt 11594b80ffdSSteven Rostedt out_unlock: 11694b80ffdSSteven Rostedt mutex_unlock(&branch_tracing_mutex); 11794b80ffdSSteven Rostedt } 11894b80ffdSSteven Rostedt 11994b80ffdSSteven Rostedt static void start_branch_trace(struct trace_array *tr) 12094b80ffdSSteven Rostedt { 12194b80ffdSSteven Rostedt enable_branch_tracing(tr); 12294b80ffdSSteven Rostedt } 12394b80ffdSSteven Rostedt 12494b80ffdSSteven Rostedt static void stop_branch_trace(struct trace_array *tr) 12594b80ffdSSteven Rostedt { 12694b80ffdSSteven Rostedt disable_branch_tracing(); 12794b80ffdSSteven Rostedt } 12894b80ffdSSteven Rostedt 1291c80025aSFrederic Weisbecker static int branch_trace_init(struct trace_array *tr) 13094b80ffdSSteven Rostedt { 13194b80ffdSSteven Rostedt int cpu; 13294b80ffdSSteven Rostedt 13394b80ffdSSteven Rostedt for_each_online_cpu(cpu) 13494b80ffdSSteven Rostedt tracing_reset(tr, cpu); 13594b80ffdSSteven Rostedt 13694b80ffdSSteven Rostedt start_branch_trace(tr); 1371c80025aSFrederic Weisbecker return 0; 13894b80ffdSSteven Rostedt } 13994b80ffdSSteven Rostedt 14094b80ffdSSteven Rostedt static void branch_trace_reset(struct trace_array *tr) 14194b80ffdSSteven Rostedt { 14294b80ffdSSteven Rostedt stop_branch_trace(tr); 14394b80ffdSSteven Rostedt } 14494b80ffdSSteven Rostedt 14594b80ffdSSteven Rostedt struct tracer branch_trace __read_mostly = 14694b80ffdSSteven Rostedt { 14794b80ffdSSteven Rostedt .name = "branch", 14894b80ffdSSteven Rostedt .init = branch_trace_init, 14994b80ffdSSteven Rostedt .reset = branch_trace_reset, 15094b80ffdSSteven Rostedt #ifdef CONFIG_FTRACE_SELFTEST 15194b80ffdSSteven Rostedt .selftest = trace_selftest_startup_branch, 15294b80ffdSSteven Rostedt #endif 15394b80ffdSSteven Rostedt }; 15494b80ffdSSteven Rostedt 15594b80ffdSSteven Rostedt __init static int init_branch_trace(void) 15694b80ffdSSteven Rostedt { 15794b80ffdSSteven Rostedt return register_tracer(&branch_trace); 15894b80ffdSSteven Rostedt } 15994b80ffdSSteven Rostedt 16094b80ffdSSteven Rostedt device_initcall(init_branch_trace); 16194b80ffdSSteven Rostedt #else 16294b80ffdSSteven Rostedt static inline 16394b80ffdSSteven Rostedt void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) 16494b80ffdSSteven Rostedt { 16594b80ffdSSteven Rostedt } 16694b80ffdSSteven Rostedt #endif /* CONFIG_BRANCH_TRACER */ 16794b80ffdSSteven Rostedt 16894b80ffdSSteven Rostedt void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect) 16994b80ffdSSteven Rostedt { 17094b80ffdSSteven Rostedt /* 17194b80ffdSSteven Rostedt * I would love to have a trace point here instead, but the 17294b80ffdSSteven Rostedt * trace point code is so inundated with unlikely and likely 17394b80ffdSSteven Rostedt * conditions that the recursive nightmare that exists is too 17494b80ffdSSteven Rostedt * much to try to get working. At least for now. 17594b80ffdSSteven Rostedt */ 17694b80ffdSSteven Rostedt trace_likely_condition(f, val, expect); 17794b80ffdSSteven Rostedt 17894b80ffdSSteven Rostedt /* FIXME: Make this atomic! */ 17994b80ffdSSteven Rostedt if (val == expect) 18094b80ffdSSteven Rostedt f->correct++; 18194b80ffdSSteven Rostedt else 18294b80ffdSSteven Rostedt f->incorrect++; 18394b80ffdSSteven Rostedt } 18494b80ffdSSteven Rostedt EXPORT_SYMBOL(ftrace_likely_update); 18594b80ffdSSteven Rostedt 18694b80ffdSSteven Rostedt struct ftrace_pointer { 18794b80ffdSSteven Rostedt void *start; 18894b80ffdSSteven Rostedt void *stop; 1892bcd521aSSteven Rostedt int hit; 19094b80ffdSSteven Rostedt }; 19194b80ffdSSteven Rostedt 19294b80ffdSSteven Rostedt static void * 19394b80ffdSSteven Rostedt t_next(struct seq_file *m, void *v, loff_t *pos) 19494b80ffdSSteven Rostedt { 1950429149fSSteven Rostedt const struct ftrace_pointer *f = m->private; 19694b80ffdSSteven Rostedt struct ftrace_branch_data *p = v; 19794b80ffdSSteven Rostedt 19894b80ffdSSteven Rostedt (*pos)++; 19994b80ffdSSteven Rostedt 20094b80ffdSSteven Rostedt if (v == (void *)1) 20194b80ffdSSteven Rostedt return f->start; 20294b80ffdSSteven Rostedt 20394b80ffdSSteven Rostedt ++p; 20494b80ffdSSteven Rostedt 20594b80ffdSSteven Rostedt if ((void *)p >= (void *)f->stop) 20694b80ffdSSteven Rostedt return NULL; 20794b80ffdSSteven Rostedt 20894b80ffdSSteven Rostedt return p; 20994b80ffdSSteven Rostedt } 21094b80ffdSSteven Rostedt 21194b80ffdSSteven Rostedt static void *t_start(struct seq_file *m, loff_t *pos) 21294b80ffdSSteven Rostedt { 21394b80ffdSSteven Rostedt void *t = (void *)1; 21494b80ffdSSteven Rostedt loff_t l = 0; 21594b80ffdSSteven Rostedt 21694b80ffdSSteven Rostedt for (; t && l < *pos; t = t_next(m, t, &l)) 21794b80ffdSSteven Rostedt ; 21894b80ffdSSteven Rostedt 21994b80ffdSSteven Rostedt return t; 22094b80ffdSSteven Rostedt } 22194b80ffdSSteven Rostedt 22294b80ffdSSteven Rostedt static void t_stop(struct seq_file *m, void *p) 22394b80ffdSSteven Rostedt { 22494b80ffdSSteven Rostedt } 22594b80ffdSSteven Rostedt 22694b80ffdSSteven Rostedt static int t_show(struct seq_file *m, void *v) 22794b80ffdSSteven Rostedt { 2280429149fSSteven Rostedt const struct ftrace_pointer *fp = m->private; 22994b80ffdSSteven Rostedt struct ftrace_branch_data *p = v; 23094b80ffdSSteven Rostedt const char *f; 231bac28bfeSSteven Rostedt long percent; 23294b80ffdSSteven Rostedt 23394b80ffdSSteven Rostedt if (v == (void *)1) { 2342bcd521aSSteven Rostedt if (fp->hit) 2352bcd521aSSteven Rostedt seq_printf(m, " miss hit %% "); 2362bcd521aSSteven Rostedt else 2372bcd521aSSteven Rostedt seq_printf(m, " correct incorrect %% "); 2382bcd521aSSteven Rostedt seq_printf(m, " Function " 23994b80ffdSSteven Rostedt " File Line\n" 24094b80ffdSSteven Rostedt " ------- --------- - " 24194b80ffdSSteven Rostedt " -------- " 24294b80ffdSSteven Rostedt " ---- ----\n"); 24394b80ffdSSteven Rostedt return 0; 24494b80ffdSSteven Rostedt } 24594b80ffdSSteven Rostedt 24694b80ffdSSteven Rostedt /* Only print the file, not the path */ 24794b80ffdSSteven Rostedt f = p->file + strlen(p->file); 24894b80ffdSSteven Rostedt while (f >= p->file && *f != '/') 24994b80ffdSSteven Rostedt f--; 25094b80ffdSSteven Rostedt f++; 25194b80ffdSSteven Rostedt 2522bcd521aSSteven Rostedt /* 2532bcd521aSSteven Rostedt * The miss is overlayed on correct, and hit on incorrect. 2542bcd521aSSteven Rostedt */ 25594b80ffdSSteven Rostedt if (p->correct) { 25694b80ffdSSteven Rostedt percent = p->incorrect * 100; 25794b80ffdSSteven Rostedt percent /= p->correct + p->incorrect; 25894b80ffdSSteven Rostedt } else 259bac28bfeSSteven Rostedt percent = p->incorrect ? 100 : -1; 26094b80ffdSSteven Rostedt 261bac28bfeSSteven Rostedt seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect); 262bac28bfeSSteven Rostedt if (percent < 0) 263bac28bfeSSteven Rostedt seq_printf(m, " X "); 264bac28bfeSSteven Rostedt else 265bac28bfeSSteven Rostedt seq_printf(m, "%3ld ", percent); 26694b80ffdSSteven Rostedt seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line); 26794b80ffdSSteven Rostedt return 0; 26894b80ffdSSteven Rostedt } 26994b80ffdSSteven Rostedt 27094b80ffdSSteven Rostedt static struct seq_operations tracing_likely_seq_ops = { 27194b80ffdSSteven Rostedt .start = t_start, 27294b80ffdSSteven Rostedt .next = t_next, 27394b80ffdSSteven Rostedt .stop = t_stop, 27494b80ffdSSteven Rostedt .show = t_show, 27594b80ffdSSteven Rostedt }; 27694b80ffdSSteven Rostedt 27745b79749SSteven Rostedt static int tracing_branch_open(struct inode *inode, struct file *file) 27894b80ffdSSteven Rostedt { 27994b80ffdSSteven Rostedt int ret; 28094b80ffdSSteven Rostedt 28194b80ffdSSteven Rostedt ret = seq_open(file, &tracing_likely_seq_ops); 28294b80ffdSSteven Rostedt if (!ret) { 28394b80ffdSSteven Rostedt struct seq_file *m = file->private_data; 28494b80ffdSSteven Rostedt m->private = (void *)inode->i_private; 28594b80ffdSSteven Rostedt } 28694b80ffdSSteven Rostedt 28794b80ffdSSteven Rostedt return ret; 28894b80ffdSSteven Rostedt } 28994b80ffdSSteven Rostedt 29045b79749SSteven Rostedt static const struct file_operations tracing_branch_fops = { 29145b79749SSteven Rostedt .open = tracing_branch_open, 29294b80ffdSSteven Rostedt .read = seq_read, 29394b80ffdSSteven Rostedt .llseek = seq_lseek, 29494b80ffdSSteven Rostedt }; 29594b80ffdSSteven Rostedt 2962bcd521aSSteven Rostedt #ifdef CONFIG_PROFILE_ALL_BRANCHES 2972bcd521aSSteven Rostedt extern unsigned long __start_branch_profile[]; 2982bcd521aSSteven Rostedt extern unsigned long __stop_branch_profile[]; 2992bcd521aSSteven Rostedt 3000429149fSSteven Rostedt static const struct ftrace_pointer ftrace_branch_pos = { 3012bcd521aSSteven Rostedt .start = __start_branch_profile, 3022bcd521aSSteven Rostedt .stop = __stop_branch_profile, 3032bcd521aSSteven Rostedt .hit = 1, 3042bcd521aSSteven Rostedt }; 3052bcd521aSSteven Rostedt 3062bcd521aSSteven Rostedt #endif /* CONFIG_PROFILE_ALL_BRANCHES */ 3072bcd521aSSteven Rostedt 30845b79749SSteven Rostedt extern unsigned long __start_annotated_branch_profile[]; 30945b79749SSteven Rostedt extern unsigned long __stop_annotated_branch_profile[]; 31094b80ffdSSteven Rostedt 31145b79749SSteven Rostedt static const struct ftrace_pointer ftrace_annotated_branch_pos = { 31245b79749SSteven Rostedt .start = __start_annotated_branch_profile, 31345b79749SSteven Rostedt .stop = __stop_annotated_branch_profile, 31494b80ffdSSteven Rostedt }; 31594b80ffdSSteven Rostedt 31694b80ffdSSteven Rostedt static __init int ftrace_branch_init(void) 31794b80ffdSSteven Rostedt { 31894b80ffdSSteven Rostedt struct dentry *d_tracer; 31994b80ffdSSteven Rostedt struct dentry *entry; 32094b80ffdSSteven Rostedt 32194b80ffdSSteven Rostedt d_tracer = tracing_init_dentry(); 32294b80ffdSSteven Rostedt 32345b79749SSteven Rostedt entry = debugfs_create_file("profile_annotated_branch", 0444, d_tracer, 3240429149fSSteven Rostedt (void *)&ftrace_annotated_branch_pos, 32545b79749SSteven Rostedt &tracing_branch_fops); 32694b80ffdSSteven Rostedt if (!entry) 32794b80ffdSSteven Rostedt pr_warning("Could not create debugfs " 32845b79749SSteven Rostedt "'profile_annotatet_branch' entry\n"); 32994b80ffdSSteven Rostedt 3302bcd521aSSteven Rostedt #ifdef CONFIG_PROFILE_ALL_BRANCHES 3312bcd521aSSteven Rostedt entry = debugfs_create_file("profile_branch", 0444, d_tracer, 3320429149fSSteven Rostedt (void *)&ftrace_branch_pos, 3332bcd521aSSteven Rostedt &tracing_branch_fops); 3342bcd521aSSteven Rostedt if (!entry) 3352bcd521aSSteven Rostedt pr_warning("Could not create debugfs" 3362bcd521aSSteven Rostedt " 'profile_branch' entry\n"); 3372bcd521aSSteven Rostedt #endif 3382bcd521aSSteven Rostedt 33994b80ffdSSteven Rostedt return 0; 34094b80ffdSSteven Rostedt } 34194b80ffdSSteven Rostedt 34294b80ffdSSteven Rostedt device_initcall(ftrace_branch_init); 343