1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * preemptoff and irqoff tracepoints 4 * 5 * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org> 6 */ 7 8 #include <linux/kallsyms.h> 9 #include <linux/uaccess.h> 10 #include <linux/module.h> 11 #include <linux/ftrace.h> 12 #include <linux/kprobes.h> 13 #include "trace.h" 14 15 #define CREATE_TRACE_POINTS 16 #include <trace/events/preemptirq.h> 17 18 #ifdef CONFIG_TRACE_IRQFLAGS 19 /* Per-cpu variable to prevent redundant calls when IRQs already off */ 20 static DEFINE_PER_CPU(int, tracing_irq_cpu); 21 22 /* 23 * Like trace_hardirqs_on() but without the lockdep invocation. This is 24 * used in the low level entry code where the ordering vs. RCU is important 25 * and lockdep uses a staged approach which splits the lockdep hardirq 26 * tracking into a RCU on and a RCU off section. 27 */ 28 void trace_hardirqs_on_prepare(void) 29 { 30 if (this_cpu_read(tracing_irq_cpu)) { 31 if (!in_nmi()) 32 trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1); 33 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); 34 this_cpu_write(tracing_irq_cpu, 0); 35 } 36 } 37 EXPORT_SYMBOL(trace_hardirqs_on_prepare); 38 NOKPROBE_SYMBOL(trace_hardirqs_on_prepare); 39 40 void trace_hardirqs_on(void) 41 { 42 if (this_cpu_read(tracing_irq_cpu)) { 43 if (!in_nmi()) 44 trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1); 45 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); 46 this_cpu_write(tracing_irq_cpu, 0); 47 } 48 49 lockdep_hardirqs_on_prepare(); 50 lockdep_hardirqs_on(CALLER_ADDR0); 51 } 52 EXPORT_SYMBOL(trace_hardirqs_on); 53 NOKPROBE_SYMBOL(trace_hardirqs_on); 54 55 /* 56 * Like trace_hardirqs_off() but without the lockdep invocation. This is 57 * used in the low level entry code where the ordering vs. RCU is important 58 * and lockdep uses a staged approach which splits the lockdep hardirq 59 * tracking into a RCU on and a RCU off section. 60 */ 61 void trace_hardirqs_off_finish(void) 62 { 63 if (!this_cpu_read(tracing_irq_cpu)) { 64 this_cpu_write(tracing_irq_cpu, 1); 65 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); 66 if (!in_nmi()) 67 trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1); 68 } 69 70 } 71 EXPORT_SYMBOL(trace_hardirqs_off_finish); 72 NOKPROBE_SYMBOL(trace_hardirqs_off_finish); 73 74 void trace_hardirqs_off(void) 75 { 76 lockdep_hardirqs_off(CALLER_ADDR0); 77 78 if (!this_cpu_read(tracing_irq_cpu)) { 79 this_cpu_write(tracing_irq_cpu, 1); 80 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); 81 if (!in_nmi()) 82 trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1); 83 } 84 } 85 EXPORT_SYMBOL(trace_hardirqs_off); 86 NOKPROBE_SYMBOL(trace_hardirqs_off); 87 88 __visible void trace_hardirqs_on_caller(unsigned long caller_addr) 89 { 90 if (this_cpu_read(tracing_irq_cpu)) { 91 if (!in_nmi()) 92 trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr); 93 tracer_hardirqs_on(CALLER_ADDR0, caller_addr); 94 this_cpu_write(tracing_irq_cpu, 0); 95 } 96 97 lockdep_hardirqs_on_prepare(); 98 lockdep_hardirqs_on(CALLER_ADDR0); 99 } 100 EXPORT_SYMBOL(trace_hardirqs_on_caller); 101 NOKPROBE_SYMBOL(trace_hardirqs_on_caller); 102 103 __visible void trace_hardirqs_off_caller(unsigned long caller_addr) 104 { 105 lockdep_hardirqs_off(CALLER_ADDR0); 106 107 if (!this_cpu_read(tracing_irq_cpu)) { 108 this_cpu_write(tracing_irq_cpu, 1); 109 tracer_hardirqs_off(CALLER_ADDR0, caller_addr); 110 if (!in_nmi()) 111 trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr); 112 } 113 } 114 EXPORT_SYMBOL(trace_hardirqs_off_caller); 115 NOKPROBE_SYMBOL(trace_hardirqs_off_caller); 116 #endif /* CONFIG_TRACE_IRQFLAGS */ 117 118 #ifdef CONFIG_TRACE_PREEMPT_TOGGLE 119 120 void trace_preempt_on(unsigned long a0, unsigned long a1) 121 { 122 if (!in_nmi()) 123 trace_preempt_enable_rcuidle(a0, a1); 124 tracer_preempt_on(a0, a1); 125 } 126 127 void trace_preempt_off(unsigned long a0, unsigned long a1) 128 { 129 if (!in_nmi()) 130 trace_preempt_disable_rcuidle(a0, a1); 131 tracer_preempt_off(a0, a1); 132 } 133 #endif 134