194b80ffdSSteven Rostedt /* 294b80ffdSSteven Rostedt * unlikely profiler 394b80ffdSSteven Rostedt * 494b80ffdSSteven Rostedt * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> 594b80ffdSSteven Rostedt */ 694b80ffdSSteven Rostedt #include <linux/kallsyms.h> 794b80ffdSSteven Rostedt #include <linux/seq_file.h> 894b80ffdSSteven Rostedt #include <linux/spinlock.h> 994b80ffdSSteven Rostedt #include <linux/debugfs.h> 1094b80ffdSSteven Rostedt #include <linux/uaccess.h> 1194b80ffdSSteven Rostedt #include <linux/module.h> 1294b80ffdSSteven Rostedt #include <linux/ftrace.h> 1394b80ffdSSteven Rostedt #include <linux/hash.h> 1494b80ffdSSteven Rostedt #include <linux/fs.h> 1594b80ffdSSteven Rostedt #include <asm/local.h> 1694b80ffdSSteven Rostedt #include "trace.h" 1794b80ffdSSteven Rostedt 1894b80ffdSSteven Rostedt #ifdef CONFIG_BRANCH_TRACER 1994b80ffdSSteven Rostedt 2094b80ffdSSteven Rostedt static int branch_tracing_enabled __read_mostly; 2194b80ffdSSteven Rostedt static DEFINE_MUTEX(branch_tracing_mutex); 2294b80ffdSSteven Rostedt static struct trace_array *branch_tracer; 2394b80ffdSSteven Rostedt 2494b80ffdSSteven Rostedt static void 2594b80ffdSSteven Rostedt probe_likely_condition(struct ftrace_branch_data *f, int val, int expect) 2694b80ffdSSteven Rostedt { 2794b80ffdSSteven Rostedt struct trace_array *tr = branch_tracer; 2894b80ffdSSteven Rostedt struct ring_buffer_event *event; 2994b80ffdSSteven Rostedt struct trace_branch *entry; 3094b80ffdSSteven Rostedt unsigned long flags, irq_flags; 3194b80ffdSSteven Rostedt int cpu, pc; 3294b80ffdSSteven Rostedt const char *p; 3394b80ffdSSteven Rostedt 3494b80ffdSSteven Rostedt /* 3594b80ffdSSteven Rostedt * I would love to save just the ftrace_likely_data pointer, but 3694b80ffdSSteven Rostedt * this code can also be used by modules. Ugly things can happen 3794b80ffdSSteven Rostedt * if the module is unloaded, and then we go and read the 3894b80ffdSSteven Rostedt * pointer. This is slower, but much safer. 3994b80ffdSSteven Rostedt */ 4094b80ffdSSteven Rostedt 4194b80ffdSSteven Rostedt if (unlikely(!tr)) 4294b80ffdSSteven Rostedt return; 4394b80ffdSSteven Rostedt 44072b40a1SFrederic Weisbecker raw_local_irq_save(flags); 4594b80ffdSSteven Rostedt cpu = raw_smp_processor_id(); 4694b80ffdSSteven Rostedt if (atomic_inc_return(&tr->data[cpu]->disabled) != 1) 4794b80ffdSSteven Rostedt goto out; 4894b80ffdSSteven Rostedt 4994b80ffdSSteven Rostedt event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 5094b80ffdSSteven Rostedt &irq_flags); 5194b80ffdSSteven Rostedt if (!event) 5294b80ffdSSteven Rostedt goto out; 5394b80ffdSSteven Rostedt 5494b80ffdSSteven Rostedt pc = preempt_count(); 5594b80ffdSSteven Rostedt entry = ring_buffer_event_data(event); 5694b80ffdSSteven Rostedt tracing_generic_entry_update(&entry->ent, flags, pc); 5794b80ffdSSteven Rostedt entry->ent.type = TRACE_BRANCH; 5894b80ffdSSteven Rostedt 5994b80ffdSSteven Rostedt /* Strip off the path, only save the file */ 6094b80ffdSSteven Rostedt p = f->file + strlen(f->file); 6194b80ffdSSteven Rostedt while (p >= f->file && *p != '/') 6294b80ffdSSteven Rostedt p--; 6394b80ffdSSteven Rostedt p++; 6494b80ffdSSteven Rostedt 6594b80ffdSSteven Rostedt strncpy(entry->func, f->func, TRACE_FUNC_SIZE); 6694b80ffdSSteven Rostedt strncpy(entry->file, p, TRACE_FILE_SIZE); 6794b80ffdSSteven Rostedt entry->func[TRACE_FUNC_SIZE] = 0; 6894b80ffdSSteven Rostedt entry->file[TRACE_FILE_SIZE] = 0; 6994b80ffdSSteven Rostedt entry->line = f->line; 7094b80ffdSSteven Rostedt entry->correct = val == expect; 7194b80ffdSSteven Rostedt 7294b80ffdSSteven Rostedt ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 7394b80ffdSSteven Rostedt 7494b80ffdSSteven Rostedt out: 7594b80ffdSSteven Rostedt atomic_dec(&tr->data[cpu]->disabled); 76072b40a1SFrederic Weisbecker raw_local_irq_restore(flags); 7794b80ffdSSteven Rostedt } 7894b80ffdSSteven Rostedt 7994b80ffdSSteven Rostedt static inline 8094b80ffdSSteven Rostedt void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) 8194b80ffdSSteven Rostedt { 8294b80ffdSSteven Rostedt if (!branch_tracing_enabled) 8394b80ffdSSteven Rostedt return; 8494b80ffdSSteven Rostedt 8594b80ffdSSteven Rostedt probe_likely_condition(f, val, expect); 8694b80ffdSSteven Rostedt } 8794b80ffdSSteven Rostedt 8894b80ffdSSteven Rostedt int enable_branch_tracing(struct trace_array *tr) 8994b80ffdSSteven Rostedt { 9094b80ffdSSteven Rostedt int ret = 0; 9194b80ffdSSteven Rostedt 9294b80ffdSSteven Rostedt mutex_lock(&branch_tracing_mutex); 9394b80ffdSSteven Rostedt branch_tracer = tr; 9494b80ffdSSteven Rostedt /* 9594b80ffdSSteven Rostedt * Must be seen before enabling. The reader is a condition 9694b80ffdSSteven Rostedt * where we do not need a matching rmb() 9794b80ffdSSteven Rostedt */ 9894b80ffdSSteven Rostedt smp_wmb(); 9994b80ffdSSteven Rostedt branch_tracing_enabled++; 10094b80ffdSSteven Rostedt mutex_unlock(&branch_tracing_mutex); 10194b80ffdSSteven Rostedt 10294b80ffdSSteven Rostedt return ret; 10394b80ffdSSteven Rostedt } 10494b80ffdSSteven Rostedt 10594b80ffdSSteven Rostedt void disable_branch_tracing(void) 10694b80ffdSSteven Rostedt { 10794b80ffdSSteven Rostedt mutex_lock(&branch_tracing_mutex); 10894b80ffdSSteven Rostedt 10994b80ffdSSteven Rostedt if (!branch_tracing_enabled) 11094b80ffdSSteven Rostedt goto out_unlock; 11194b80ffdSSteven Rostedt 11294b80ffdSSteven Rostedt branch_tracing_enabled--; 11394b80ffdSSteven Rostedt 11494b80ffdSSteven Rostedt out_unlock: 11594b80ffdSSteven Rostedt mutex_unlock(&branch_tracing_mutex); 11694b80ffdSSteven Rostedt } 11794b80ffdSSteven Rostedt 11894b80ffdSSteven Rostedt static void start_branch_trace(struct trace_array *tr) 11994b80ffdSSteven Rostedt { 12094b80ffdSSteven Rostedt enable_branch_tracing(tr); 12194b80ffdSSteven Rostedt } 12294b80ffdSSteven Rostedt 12394b80ffdSSteven Rostedt static void stop_branch_trace(struct trace_array *tr) 12494b80ffdSSteven Rostedt { 12594b80ffdSSteven Rostedt disable_branch_tracing(); 12694b80ffdSSteven Rostedt } 12794b80ffdSSteven Rostedt 1281c80025aSFrederic Weisbecker static int branch_trace_init(struct trace_array *tr) 12994b80ffdSSteven Rostedt { 13094b80ffdSSteven Rostedt int cpu; 13194b80ffdSSteven Rostedt 13294b80ffdSSteven Rostedt for_each_online_cpu(cpu) 13394b80ffdSSteven Rostedt tracing_reset(tr, cpu); 13494b80ffdSSteven Rostedt 13594b80ffdSSteven Rostedt start_branch_trace(tr); 1361c80025aSFrederic Weisbecker return 0; 13794b80ffdSSteven Rostedt } 13894b80ffdSSteven Rostedt 13994b80ffdSSteven Rostedt static void branch_trace_reset(struct trace_array *tr) 14094b80ffdSSteven Rostedt { 14194b80ffdSSteven Rostedt stop_branch_trace(tr); 14294b80ffdSSteven Rostedt } 14394b80ffdSSteven Rostedt 14494b80ffdSSteven Rostedt struct tracer branch_trace __read_mostly = 14594b80ffdSSteven Rostedt { 14694b80ffdSSteven Rostedt .name = "branch", 14794b80ffdSSteven Rostedt .init = branch_trace_init, 14894b80ffdSSteven Rostedt .reset = branch_trace_reset, 14994b80ffdSSteven Rostedt #ifdef CONFIG_FTRACE_SELFTEST 15094b80ffdSSteven Rostedt .selftest = trace_selftest_startup_branch, 15194b80ffdSSteven Rostedt #endif 15294b80ffdSSteven Rostedt }; 15394b80ffdSSteven Rostedt 15494b80ffdSSteven Rostedt __init static int init_branch_trace(void) 15594b80ffdSSteven Rostedt { 15694b80ffdSSteven Rostedt return register_tracer(&branch_trace); 15794b80ffdSSteven Rostedt } 15894b80ffdSSteven Rostedt 15994b80ffdSSteven Rostedt device_initcall(init_branch_trace); 16094b80ffdSSteven Rostedt #else 16194b80ffdSSteven Rostedt static inline 16294b80ffdSSteven Rostedt void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) 16394b80ffdSSteven Rostedt { 16494b80ffdSSteven Rostedt } 16594b80ffdSSteven Rostedt #endif /* CONFIG_BRANCH_TRACER */ 16694b80ffdSSteven Rostedt 16794b80ffdSSteven Rostedt void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect) 16894b80ffdSSteven Rostedt { 16994b80ffdSSteven Rostedt /* 17094b80ffdSSteven Rostedt * I would love to have a trace point here instead, but the 17194b80ffdSSteven Rostedt * trace point code is so inundated with unlikely and likely 17294b80ffdSSteven Rostedt * conditions that the recursive nightmare that exists is too 17394b80ffdSSteven Rostedt * much to try to get working. At least for now. 17494b80ffdSSteven Rostedt */ 17594b80ffdSSteven Rostedt trace_likely_condition(f, val, expect); 17694b80ffdSSteven Rostedt 17794b80ffdSSteven Rostedt /* FIXME: Make this atomic! */ 17894b80ffdSSteven Rostedt if (val == expect) 17994b80ffdSSteven Rostedt f->correct++; 18094b80ffdSSteven Rostedt else 18194b80ffdSSteven Rostedt f->incorrect++; 18294b80ffdSSteven Rostedt } 18394b80ffdSSteven Rostedt EXPORT_SYMBOL(ftrace_likely_update); 18494b80ffdSSteven Rostedt 18594b80ffdSSteven Rostedt struct ftrace_pointer { 18694b80ffdSSteven Rostedt void *start; 18794b80ffdSSteven Rostedt void *stop; 1882bcd521aSSteven Rostedt int hit; 18994b80ffdSSteven Rostedt }; 19094b80ffdSSteven Rostedt 19194b80ffdSSteven Rostedt static void * 19294b80ffdSSteven Rostedt t_next(struct seq_file *m, void *v, loff_t *pos) 19394b80ffdSSteven Rostedt { 1940429149fSSteven Rostedt const struct ftrace_pointer *f = m->private; 19594b80ffdSSteven Rostedt struct ftrace_branch_data *p = v; 19694b80ffdSSteven Rostedt 19794b80ffdSSteven Rostedt (*pos)++; 19894b80ffdSSteven Rostedt 19994b80ffdSSteven Rostedt if (v == (void *)1) 20094b80ffdSSteven Rostedt return f->start; 20194b80ffdSSteven Rostedt 20294b80ffdSSteven Rostedt ++p; 20394b80ffdSSteven Rostedt 20494b80ffdSSteven Rostedt if ((void *)p >= (void *)f->stop) 20594b80ffdSSteven Rostedt return NULL; 20694b80ffdSSteven Rostedt 20794b80ffdSSteven Rostedt return p; 20894b80ffdSSteven Rostedt } 20994b80ffdSSteven Rostedt 21094b80ffdSSteven Rostedt static void *t_start(struct seq_file *m, loff_t *pos) 21194b80ffdSSteven Rostedt { 21294b80ffdSSteven Rostedt void *t = (void *)1; 21394b80ffdSSteven Rostedt loff_t l = 0; 21494b80ffdSSteven Rostedt 21594b80ffdSSteven Rostedt for (; t && l < *pos; t = t_next(m, t, &l)) 21694b80ffdSSteven Rostedt ; 21794b80ffdSSteven Rostedt 21894b80ffdSSteven Rostedt return t; 21994b80ffdSSteven Rostedt } 22094b80ffdSSteven Rostedt 22194b80ffdSSteven Rostedt static void t_stop(struct seq_file *m, void *p) 22294b80ffdSSteven Rostedt { 22394b80ffdSSteven Rostedt } 22494b80ffdSSteven Rostedt 22594b80ffdSSteven Rostedt static int t_show(struct seq_file *m, void *v) 22694b80ffdSSteven Rostedt { 2270429149fSSteven Rostedt const struct ftrace_pointer *fp = m->private; 22894b80ffdSSteven Rostedt struct ftrace_branch_data *p = v; 22994b80ffdSSteven Rostedt const char *f; 230bac28bfeSSteven Rostedt long percent; 23194b80ffdSSteven Rostedt 23294b80ffdSSteven Rostedt if (v == (void *)1) { 2332bcd521aSSteven Rostedt if (fp->hit) 2342bcd521aSSteven Rostedt seq_printf(m, " miss hit %% "); 2352bcd521aSSteven Rostedt else 2362bcd521aSSteven Rostedt seq_printf(m, " correct incorrect %% "); 2372bcd521aSSteven Rostedt seq_printf(m, " Function " 23894b80ffdSSteven Rostedt " File Line\n" 23994b80ffdSSteven Rostedt " ------- --------- - " 24094b80ffdSSteven Rostedt " -------- " 24194b80ffdSSteven Rostedt " ---- ----\n"); 24294b80ffdSSteven Rostedt return 0; 24394b80ffdSSteven Rostedt } 24494b80ffdSSteven Rostedt 24594b80ffdSSteven Rostedt /* Only print the file, not the path */ 24694b80ffdSSteven Rostedt f = p->file + strlen(p->file); 24794b80ffdSSteven Rostedt while (f >= p->file && *f != '/') 24894b80ffdSSteven Rostedt f--; 24994b80ffdSSteven Rostedt f++; 25094b80ffdSSteven Rostedt 2512bcd521aSSteven Rostedt /* 2522bcd521aSSteven Rostedt * The miss is overlayed on correct, and hit on incorrect. 2532bcd521aSSteven Rostedt */ 25494b80ffdSSteven Rostedt if (p->correct) { 25594b80ffdSSteven Rostedt percent = p->incorrect * 100; 25694b80ffdSSteven Rostedt percent /= p->correct + p->incorrect; 25794b80ffdSSteven Rostedt } else 258bac28bfeSSteven Rostedt percent = p->incorrect ? 100 : -1; 25994b80ffdSSteven Rostedt 260bac28bfeSSteven Rostedt seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect); 261bac28bfeSSteven Rostedt if (percent < 0) 262bac28bfeSSteven Rostedt seq_printf(m, " X "); 263bac28bfeSSteven Rostedt else 264bac28bfeSSteven Rostedt seq_printf(m, "%3ld ", percent); 26594b80ffdSSteven Rostedt seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line); 26694b80ffdSSteven Rostedt return 0; 26794b80ffdSSteven Rostedt } 26894b80ffdSSteven Rostedt 26994b80ffdSSteven Rostedt static struct seq_operations tracing_likely_seq_ops = { 27094b80ffdSSteven Rostedt .start = t_start, 27194b80ffdSSteven Rostedt .next = t_next, 27294b80ffdSSteven Rostedt .stop = t_stop, 27394b80ffdSSteven Rostedt .show = t_show, 27494b80ffdSSteven Rostedt }; 27594b80ffdSSteven Rostedt 27645b79749SSteven Rostedt static int tracing_branch_open(struct inode *inode, struct file *file) 27794b80ffdSSteven Rostedt { 27894b80ffdSSteven Rostedt int ret; 27994b80ffdSSteven Rostedt 28094b80ffdSSteven Rostedt ret = seq_open(file, &tracing_likely_seq_ops); 28194b80ffdSSteven Rostedt if (!ret) { 28294b80ffdSSteven Rostedt struct seq_file *m = file->private_data; 28394b80ffdSSteven Rostedt m->private = (void *)inode->i_private; 28494b80ffdSSteven Rostedt } 28594b80ffdSSteven Rostedt 28694b80ffdSSteven Rostedt return ret; 28794b80ffdSSteven Rostedt } 28894b80ffdSSteven Rostedt 28945b79749SSteven Rostedt static const struct file_operations tracing_branch_fops = { 29045b79749SSteven Rostedt .open = tracing_branch_open, 29194b80ffdSSteven Rostedt .read = seq_read, 29294b80ffdSSteven Rostedt .llseek = seq_lseek, 29394b80ffdSSteven Rostedt }; 29494b80ffdSSteven Rostedt 2952bcd521aSSteven Rostedt #ifdef CONFIG_PROFILE_ALL_BRANCHES 2962bcd521aSSteven Rostedt extern unsigned long __start_branch_profile[]; 2972bcd521aSSteven Rostedt extern unsigned long __stop_branch_profile[]; 2982bcd521aSSteven Rostedt 2990429149fSSteven Rostedt static const struct ftrace_pointer ftrace_branch_pos = { 3002bcd521aSSteven Rostedt .start = __start_branch_profile, 3012bcd521aSSteven Rostedt .stop = __stop_branch_profile, 3022bcd521aSSteven Rostedt .hit = 1, 3032bcd521aSSteven Rostedt }; 3042bcd521aSSteven Rostedt 3052bcd521aSSteven Rostedt #endif /* CONFIG_PROFILE_ALL_BRANCHES */ 3062bcd521aSSteven Rostedt 30745b79749SSteven Rostedt extern unsigned long __start_annotated_branch_profile[]; 30845b79749SSteven Rostedt extern unsigned long __stop_annotated_branch_profile[]; 30994b80ffdSSteven Rostedt 31045b79749SSteven Rostedt static const struct ftrace_pointer ftrace_annotated_branch_pos = { 31145b79749SSteven Rostedt .start = __start_annotated_branch_profile, 31245b79749SSteven Rostedt .stop = __stop_annotated_branch_profile, 31394b80ffdSSteven Rostedt }; 31494b80ffdSSteven Rostedt 31594b80ffdSSteven Rostedt static __init int ftrace_branch_init(void) 31694b80ffdSSteven Rostedt { 31794b80ffdSSteven Rostedt struct dentry *d_tracer; 31894b80ffdSSteven Rostedt struct dentry *entry; 31994b80ffdSSteven Rostedt 32094b80ffdSSteven Rostedt d_tracer = tracing_init_dentry(); 32194b80ffdSSteven Rostedt 32245b79749SSteven Rostedt entry = debugfs_create_file("profile_annotated_branch", 0444, d_tracer, 3230429149fSSteven Rostedt (void *)&ftrace_annotated_branch_pos, 32445b79749SSteven Rostedt &tracing_branch_fops); 32594b80ffdSSteven Rostedt if (!entry) 32694b80ffdSSteven Rostedt pr_warning("Could not create debugfs " 32745b79749SSteven Rostedt "'profile_annotatet_branch' entry\n"); 32894b80ffdSSteven Rostedt 3292bcd521aSSteven Rostedt #ifdef CONFIG_PROFILE_ALL_BRANCHES 3302bcd521aSSteven Rostedt entry = debugfs_create_file("profile_branch", 0444, d_tracer, 3310429149fSSteven Rostedt (void *)&ftrace_branch_pos, 3322bcd521aSSteven Rostedt &tracing_branch_fops); 3332bcd521aSSteven Rostedt if (!entry) 3342bcd521aSSteven Rostedt pr_warning("Could not create debugfs" 3352bcd521aSSteven Rostedt " 'profile_branch' entry\n"); 3362bcd521aSSteven Rostedt #endif 3372bcd521aSSteven Rostedt 33894b80ffdSSteven Rostedt return 0; 33994b80ffdSSteven Rostedt } 34094b80ffdSSteven Rostedt 34194b80ffdSSteven Rostedt device_initcall(ftrace_branch_init); 342