135e8e302SSteven Rostedt /* 235e8e302SSteven Rostedt * trace context switch 335e8e302SSteven Rostedt * 435e8e302SSteven Rostedt * Copyright (C) 2007 Steven Rostedt <srostedt@redhat.com> 535e8e302SSteven Rostedt * 635e8e302SSteven Rostedt */ 735e8e302SSteven Rostedt #include <linux/module.h> 835e8e302SSteven Rostedt #include <linux/fs.h> 935e8e302SSteven Rostedt #include <linux/debugfs.h> 1035e8e302SSteven Rostedt #include <linux/kallsyms.h> 1135e8e302SSteven Rostedt #include <linux/uaccess.h> 1235e8e302SSteven Rostedt #include <linux/ftrace.h> 13ad8d75ffSSteven Rostedt #include <trace/events/sched.h> 1435e8e302SSteven Rostedt 1535e8e302SSteven Rostedt #include "trace.h" 1635e8e302SSteven Rostedt 1735e8e302SSteven Rostedt static struct trace_array *ctx_trace; 1835e8e302SSteven Rostedt static int __read_mostly tracer_enabled; 19efade6e7SFrederic Weisbecker static int sched_ref; 20efade6e7SFrederic Weisbecker static DEFINE_MUTEX(sched_register_mutex); 215fec6ddcSSteven Rostedt static int sched_stopped; 2235e8e302SSteven Rostedt 2382e04af4SFrederic Weisbecker 2482e04af4SFrederic Weisbecker void 2582e04af4SFrederic Weisbecker tracing_sched_switch_trace(struct trace_array *tr, 2682e04af4SFrederic Weisbecker struct task_struct *prev, 2782e04af4SFrederic Weisbecker struct task_struct *next, 2882e04af4SFrederic Weisbecker unsigned long flags, int pc) 2982e04af4SFrederic Weisbecker { 3082e04af4SFrederic Weisbecker struct ftrace_event_call *call = &event_context_switch; 31e77405adSSteven Rostedt struct ring_buffer *buffer = tr->buffer; 3282e04af4SFrederic Weisbecker struct ring_buffer_event *event; 3382e04af4SFrederic Weisbecker struct ctx_switch_entry *entry; 3482e04af4SFrederic Weisbecker 35e77405adSSteven Rostedt event = trace_buffer_lock_reserve(buffer, TRACE_CTX, 3682e04af4SFrederic Weisbecker sizeof(*entry), flags, pc); 3782e04af4SFrederic Weisbecker if (!event) 3882e04af4SFrederic Weisbecker return; 3982e04af4SFrederic Weisbecker entry = ring_buffer_event_data(event); 4082e04af4SFrederic Weisbecker entry->prev_pid = prev->pid; 4182e04af4SFrederic Weisbecker entry->prev_prio = prev->prio; 4282e04af4SFrederic Weisbecker entry->prev_state = prev->state; 4382e04af4SFrederic Weisbecker entry->next_pid = next->pid; 4482e04af4SFrederic Weisbecker entry->next_prio = next->prio; 4582e04af4SFrederic Weisbecker entry->next_state = next->state; 4682e04af4SFrederic Weisbecker entry->next_cpu = task_cpu(next); 4782e04af4SFrederic Weisbecker 48e77405adSSteven Rostedt if (!filter_check_discard(call, entry, buffer, event)) 49e77405adSSteven Rostedt trace_buffer_unlock_commit(buffer, event, flags, pc); 5082e04af4SFrederic Weisbecker } 5182e04af4SFrederic Weisbecker 52e309b41dSIngo Molnar static void 5338516ab5SSteven Rostedt probe_sched_switch(void *ignore, struct task_struct *prev, struct task_struct *next) 5435e8e302SSteven Rostedt { 5535e8e302SSteven Rostedt struct trace_array_cpu *data; 5635e8e302SSteven Rostedt unsigned long flags; 5735e8e302SSteven Rostedt int cpu; 5838697053SSteven Rostedt int pc; 5935e8e302SSteven Rostedt 60dcef788eSZhaolei if (unlikely(!sched_ref)) 61b07c3f19SMathieu Desnoyers return; 62b07c3f19SMathieu Desnoyers 6341bc8144SSteven Rostedt tracing_record_cmdline(prev); 6441bc8144SSteven Rostedt tracing_record_cmdline(next); 6541bc8144SSteven Rostedt 66dcef788eSZhaolei if (!tracer_enabled || sched_stopped) 6735e8e302SSteven Rostedt return; 6835e8e302SSteven Rostedt 6938697053SSteven Rostedt pc = preempt_count(); 7018cef379SSteven Rostedt local_irq_save(flags); 7135e8e302SSteven Rostedt cpu = raw_smp_processor_id(); 72b07c3f19SMathieu Desnoyers data = ctx_trace->data[cpu]; 7335e8e302SSteven Rostedt 743ea2e6d7SSteven Rostedt if (likely(!atomic_read(&data->disabled))) 757be42151SArnaldo Carvalho de Melo tracing_sched_switch_trace(ctx_trace, prev, next, flags, pc); 7635e8e302SSteven Rostedt 7718cef379SSteven Rostedt local_irq_restore(flags); 7835e8e302SSteven Rostedt } 7935e8e302SSteven Rostedt 8082e04af4SFrederic Weisbecker void 8182e04af4SFrederic Weisbecker tracing_sched_wakeup_trace(struct trace_array *tr, 8282e04af4SFrederic Weisbecker struct task_struct *wakee, 8382e04af4SFrederic Weisbecker struct task_struct *curr, 8482e04af4SFrederic Weisbecker unsigned long flags, int pc) 8582e04af4SFrederic Weisbecker { 8682e04af4SFrederic Weisbecker struct ftrace_event_call *call = &event_wakeup; 8782e04af4SFrederic Weisbecker struct ring_buffer_event *event; 8882e04af4SFrederic Weisbecker struct ctx_switch_entry *entry; 89e77405adSSteven Rostedt struct ring_buffer *buffer = tr->buffer; 9082e04af4SFrederic Weisbecker 91e77405adSSteven Rostedt event = trace_buffer_lock_reserve(buffer, TRACE_WAKE, 9282e04af4SFrederic Weisbecker sizeof(*entry), flags, pc); 9382e04af4SFrederic Weisbecker if (!event) 9482e04af4SFrederic Weisbecker return; 9582e04af4SFrederic Weisbecker entry = ring_buffer_event_data(event); 9682e04af4SFrederic Weisbecker entry->prev_pid = curr->pid; 9782e04af4SFrederic Weisbecker entry->prev_prio = curr->prio; 9882e04af4SFrederic Weisbecker entry->prev_state = curr->state; 9982e04af4SFrederic Weisbecker entry->next_pid = wakee->pid; 10082e04af4SFrederic Weisbecker entry->next_prio = wakee->prio; 10182e04af4SFrederic Weisbecker entry->next_state = wakee->state; 10282e04af4SFrederic Weisbecker entry->next_cpu = task_cpu(wakee); 10382e04af4SFrederic Weisbecker 104e77405adSSteven Rostedt if (!filter_check_discard(call, entry, buffer, event)) 105e77405adSSteven Rostedt ring_buffer_unlock_commit(buffer, event); 106e77405adSSteven Rostedt ftrace_trace_stack(tr->buffer, flags, 6, pc); 107e77405adSSteven Rostedt ftrace_trace_userstack(tr->buffer, flags, pc); 10882e04af4SFrederic Weisbecker } 10982e04af4SFrederic Weisbecker 1105b82a1b0SMathieu Desnoyers static void 11138516ab5SSteven Rostedt probe_sched_wakeup(void *ignore, struct task_struct *wakee, int success) 1125b82a1b0SMathieu Desnoyers { 11357422797SIngo Molnar struct trace_array_cpu *data; 11457422797SIngo Molnar unsigned long flags; 11538697053SSteven Rostedt int cpu, pc; 11657422797SIngo Molnar 117dcef788eSZhaolei if (unlikely(!sched_ref)) 118dcef788eSZhaolei return; 119dcef788eSZhaolei 120dcef788eSZhaolei tracing_record_cmdline(current); 121dcef788eSZhaolei 122dcef788eSZhaolei if (!tracer_enabled || sched_stopped) 12357422797SIngo Molnar return; 12457422797SIngo Molnar 12538697053SSteven Rostedt pc = preempt_count(); 12657422797SIngo Molnar local_irq_save(flags); 12757422797SIngo Molnar cpu = raw_smp_processor_id(); 128b07c3f19SMathieu Desnoyers data = ctx_trace->data[cpu]; 12957422797SIngo Molnar 1303ea2e6d7SSteven Rostedt if (likely(!atomic_read(&data->disabled))) 1317be42151SArnaldo Carvalho de Melo tracing_sched_wakeup_trace(ctx_trace, wakee, current, 13238697053SSteven Rostedt flags, pc); 13357422797SIngo Molnar 13457422797SIngo Molnar local_irq_restore(flags); 13557422797SIngo Molnar } 13657422797SIngo Molnar 1375b82a1b0SMathieu Desnoyers static int tracing_sched_register(void) 1385b82a1b0SMathieu Desnoyers { 1395b82a1b0SMathieu Desnoyers int ret; 1405b82a1b0SMathieu Desnoyers 14138516ab5SSteven Rostedt ret = register_trace_sched_wakeup(probe_sched_wakeup, NULL); 1425b82a1b0SMathieu Desnoyers if (ret) { 143b07c3f19SMathieu Desnoyers pr_info("wakeup trace: Couldn't activate tracepoint" 1445b82a1b0SMathieu Desnoyers " probe to kernel_sched_wakeup\n"); 1455b82a1b0SMathieu Desnoyers return ret; 1465b82a1b0SMathieu Desnoyers } 1475b82a1b0SMathieu Desnoyers 14838516ab5SSteven Rostedt ret = register_trace_sched_wakeup_new(probe_sched_wakeup, NULL); 1495b82a1b0SMathieu Desnoyers if (ret) { 150b07c3f19SMathieu Desnoyers pr_info("wakeup trace: Couldn't activate tracepoint" 1515b82a1b0SMathieu Desnoyers " probe to kernel_sched_wakeup_new\n"); 1525b82a1b0SMathieu Desnoyers goto fail_deprobe; 1535b82a1b0SMathieu Desnoyers } 1545b82a1b0SMathieu Desnoyers 15538516ab5SSteven Rostedt ret = register_trace_sched_switch(probe_sched_switch, NULL); 1565b82a1b0SMathieu Desnoyers if (ret) { 157b07c3f19SMathieu Desnoyers pr_info("sched trace: Couldn't activate tracepoint" 15873d8b8bcSWenji Huang " probe to kernel_sched_switch\n"); 1595b82a1b0SMathieu Desnoyers goto fail_deprobe_wake_new; 1605b82a1b0SMathieu Desnoyers } 1615b82a1b0SMathieu Desnoyers 1625b82a1b0SMathieu Desnoyers return ret; 1635b82a1b0SMathieu Desnoyers fail_deprobe_wake_new: 16438516ab5SSteven Rostedt unregister_trace_sched_wakeup_new(probe_sched_wakeup, NULL); 1655b82a1b0SMathieu Desnoyers fail_deprobe: 16638516ab5SSteven Rostedt unregister_trace_sched_wakeup(probe_sched_wakeup, NULL); 1675b82a1b0SMathieu Desnoyers return ret; 1685b82a1b0SMathieu Desnoyers } 1695b82a1b0SMathieu Desnoyers 1705b82a1b0SMathieu Desnoyers static void tracing_sched_unregister(void) 1715b82a1b0SMathieu Desnoyers { 17238516ab5SSteven Rostedt unregister_trace_sched_switch(probe_sched_switch, NULL); 17338516ab5SSteven Rostedt unregister_trace_sched_wakeup_new(probe_sched_wakeup, NULL); 17438516ab5SSteven Rostedt unregister_trace_sched_wakeup(probe_sched_wakeup, NULL); 1755b82a1b0SMathieu Desnoyers } 1765b82a1b0SMathieu Desnoyers 177f2252935SIngo Molnar static void tracing_start_sched_switch(void) 1785b82a1b0SMathieu Desnoyers { 179efade6e7SFrederic Weisbecker mutex_lock(&sched_register_mutex); 180e168e051SSteven Rostedt if (!(sched_ref++)) 1815b82a1b0SMathieu Desnoyers tracing_sched_register(); 182efade6e7SFrederic Weisbecker mutex_unlock(&sched_register_mutex); 1835b82a1b0SMathieu Desnoyers } 1845b82a1b0SMathieu Desnoyers 185f2252935SIngo Molnar static void tracing_stop_sched_switch(void) 1865b82a1b0SMathieu Desnoyers { 187efade6e7SFrederic Weisbecker mutex_lock(&sched_register_mutex); 188e168e051SSteven Rostedt if (!(--sched_ref)) 1895b82a1b0SMathieu Desnoyers tracing_sched_unregister(); 190efade6e7SFrederic Weisbecker mutex_unlock(&sched_register_mutex); 1915b82a1b0SMathieu Desnoyers } 1925b82a1b0SMathieu Desnoyers 19341bc8144SSteven Rostedt void tracing_start_cmdline_record(void) 19441bc8144SSteven Rostedt { 19541bc8144SSteven Rostedt tracing_start_sched_switch(); 19641bc8144SSteven Rostedt } 19741bc8144SSteven Rostedt 19841bc8144SSteven Rostedt void tracing_stop_cmdline_record(void) 19941bc8144SSteven Rostedt { 20041bc8144SSteven Rostedt tracing_stop_sched_switch(); 20141bc8144SSteven Rostedt } 20241bc8144SSteven Rostedt 20375f5c47dSSteven Rostedt /** 204e168e051SSteven Rostedt * tracing_start_sched_switch_record - start tracing context switches 205e168e051SSteven Rostedt * 206e168e051SSteven Rostedt * Turns on context switch tracing for a tracer. 207e168e051SSteven Rostedt */ 208e168e051SSteven Rostedt void tracing_start_sched_switch_record(void) 209e168e051SSteven Rostedt { 210e168e051SSteven Rostedt if (unlikely(!ctx_trace)) { 211e168e051SSteven Rostedt WARN_ON(1); 212e168e051SSteven Rostedt return; 213e168e051SSteven Rostedt } 214e168e051SSteven Rostedt 215e168e051SSteven Rostedt tracing_start_sched_switch(); 216e168e051SSteven Rostedt 217e168e051SSteven Rostedt mutex_lock(&sched_register_mutex); 218e168e051SSteven Rostedt tracer_enabled++; 219e168e051SSteven Rostedt mutex_unlock(&sched_register_mutex); 220e168e051SSteven Rostedt } 221e168e051SSteven Rostedt 222e168e051SSteven Rostedt /** 223e168e051SSteven Rostedt * tracing_stop_sched_switch_record - start tracing context switches 224e168e051SSteven Rostedt * 225e168e051SSteven Rostedt * Turns off context switch tracing for a tracer. 226e168e051SSteven Rostedt */ 227e168e051SSteven Rostedt void tracing_stop_sched_switch_record(void) 228e168e051SSteven Rostedt { 229e168e051SSteven Rostedt mutex_lock(&sched_register_mutex); 230e168e051SSteven Rostedt tracer_enabled--; 231e168e051SSteven Rostedt WARN_ON(tracer_enabled < 0); 232e168e051SSteven Rostedt mutex_unlock(&sched_register_mutex); 233e168e051SSteven Rostedt 234e168e051SSteven Rostedt tracing_stop_sched_switch(); 235e168e051SSteven Rostedt } 236e168e051SSteven Rostedt 237e168e051SSteven Rostedt /** 238e168e051SSteven Rostedt * tracing_sched_switch_assign_trace - assign a trace array for ctx switch 23975f5c47dSSteven Rostedt * @tr: trace array pointer to assign 24075f5c47dSSteven Rostedt * 24175f5c47dSSteven Rostedt * Some tracers might want to record the context switches in their 24275f5c47dSSteven Rostedt * trace. This function lets those tracers assign the trace array 24375f5c47dSSteven Rostedt * to use. 24475f5c47dSSteven Rostedt */ 245e168e051SSteven Rostedt void tracing_sched_switch_assign_trace(struct trace_array *tr) 24675f5c47dSSteven Rostedt { 24775f5c47dSSteven Rostedt ctx_trace = tr; 24875f5c47dSSteven Rostedt } 24975f5c47dSSteven Rostedt 250e309b41dSIngo Molnar static void stop_sched_trace(struct trace_array *tr) 25135e8e302SSteven Rostedt { 252e168e051SSteven Rostedt tracing_stop_sched_switch_record(); 25335e8e302SSteven Rostedt } 25435e8e302SSteven Rostedt 2551c80025aSFrederic Weisbecker static int sched_switch_trace_init(struct trace_array *tr) 25635e8e302SSteven Rostedt { 25735e8e302SSteven Rostedt ctx_trace = tr; 2585fec6ddcSSteven Rostedt tracing_reset_online_cpus(tr); 259b6f11df2SArnaldo Carvalho de Melo tracing_start_sched_switch_record(); 2601c80025aSFrederic Weisbecker return 0; 26135e8e302SSteven Rostedt } 26235e8e302SSteven Rostedt 263e309b41dSIngo Molnar static void sched_switch_trace_reset(struct trace_array *tr) 26435e8e302SSteven Rostedt { 265c76f0694SSteven Rostedt if (sched_ref) 26635e8e302SSteven Rostedt stop_sched_trace(tr); 26735e8e302SSteven Rostedt } 26835e8e302SSteven Rostedt 2699036990dSSteven Rostedt static void sched_switch_trace_start(struct trace_array *tr) 2709036990dSSteven Rostedt { 2715fec6ddcSSteven Rostedt sched_stopped = 0; 2729036990dSSteven Rostedt } 2739036990dSSteven Rostedt 2749036990dSSteven Rostedt static void sched_switch_trace_stop(struct trace_array *tr) 2759036990dSSteven Rostedt { 2765fec6ddcSSteven Rostedt sched_stopped = 1; 2779036990dSSteven Rostedt } 2789036990dSSteven Rostedt 27975f5c47dSSteven Rostedt static struct tracer sched_switch_trace __read_mostly = 28035e8e302SSteven Rostedt { 28135e8e302SSteven Rostedt .name = "sched_switch", 28235e8e302SSteven Rostedt .init = sched_switch_trace_init, 28335e8e302SSteven Rostedt .reset = sched_switch_trace_reset, 2849036990dSSteven Rostedt .start = sched_switch_trace_start, 2859036990dSSteven Rostedt .stop = sched_switch_trace_stop, 2866eaaa5d5SFrederic Weisbecker .wait_pipe = poll_wait_pipe, 28760a11774SSteven Rostedt #ifdef CONFIG_FTRACE_SELFTEST 28860a11774SSteven Rostedt .selftest = trace_selftest_startup_sched_switch, 28960a11774SSteven Rostedt #endif 29035e8e302SSteven Rostedt }; 29135e8e302SSteven Rostedt 29235e8e302SSteven Rostedt __init static int init_sched_switch_trace(void) 29335e8e302SSteven Rostedt { 29435e8e302SSteven Rostedt return register_tracer(&sched_switch_trace); 29535e8e302SSteven Rostedt } 29635e8e302SSteven Rostedt device_initcall(init_sched_switch_trace); 297c71dd42dSIngo Molnar 298