1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/spinlock.h> 3 #include <linux/hardirq.h> 4 #include <linux/ftrace.h> 5 #include <linux/percpu.h> 6 #include <linux/init.h> 7 #include <linux/list.h> 8 #include <trace/syscall.h> 9 10 #include <asm/ftrace.h> 11 12 #ifdef CONFIG_DYNAMIC_FTRACE 13 static const u32 ftrace_nop = 0x01000000; 14 15 static u32 ftrace_call_replace(unsigned long ip, unsigned long addr) 16 { 17 u32 call; 18 s32 off; 19 20 off = ((s32)addr - (s32)ip); 21 call = 0x40000000 | ((u32)off >> 2); 22 23 return call; 24 } 25 26 static int ftrace_modify_code(unsigned long ip, u32 old, u32 new) 27 { 28 u32 replaced; 29 int faulted; 30 31 __asm__ __volatile__( 32 "1: cas [%[ip]], %[old], %[new]\n" 33 " flush %[ip]\n" 34 " mov 0, %[faulted]\n" 35 "2:\n" 36 " .section .fixup,#alloc,#execinstr\n" 37 " .align 4\n" 38 "3: sethi %%hi(2b), %[faulted]\n" 39 " jmpl %[faulted] + %%lo(2b), %%g0\n" 40 " mov 1, %[faulted]\n" 41 " .previous\n" 42 " .section __ex_table,\"a\"\n" 43 " .align 4\n" 44 " .word 1b, 3b\n" 45 " .previous\n" 46 : "=r" (replaced), [faulted] "=r" (faulted) 47 : [new] "0" (new), [old] "r" (old), [ip] "r" (ip) 48 : "memory"); 49 50 if (replaced != old && replaced != new) 51 faulted = 2; 52 53 return faulted; 54 } 55 56 int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) 57 { 58 unsigned long ip = rec->ip; 59 u32 old, new; 60 61 old = ftrace_call_replace(ip, addr); 62 new = ftrace_nop; 63 return ftrace_modify_code(ip, old, new); 64 } 65 66 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 67 { 68 unsigned long ip = rec->ip; 69 u32 old, new; 70 71 old = ftrace_nop; 72 new = ftrace_call_replace(ip, addr); 73 return ftrace_modify_code(ip, old, new); 74 } 75 76 int ftrace_update_ftrace_func(ftrace_func_t func) 77 { 78 unsigned long ip = (unsigned long)(&ftrace_call); 79 u32 old, new; 80 81 old = *(u32 *) &ftrace_call; 82 new = ftrace_call_replace(ip, (unsigned long)func); 83 return ftrace_modify_code(ip, old, new); 84 } 85 #endif 86 87 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 88 89 #ifdef CONFIG_DYNAMIC_FTRACE 90 extern void ftrace_graph_call(void); 91 92 int ftrace_enable_ftrace_graph_caller(void) 93 { 94 unsigned long ip = (unsigned long)(&ftrace_graph_call); 95 u32 old, new; 96 97 old = *(u32 *) &ftrace_graph_call; 98 new = ftrace_call_replace(ip, (unsigned long) &ftrace_graph_caller); 99 return ftrace_modify_code(ip, old, new); 100 } 101 102 int ftrace_disable_ftrace_graph_caller(void) 103 { 104 unsigned long ip = (unsigned long)(&ftrace_graph_call); 105 u32 old, new; 106 107 old = *(u32 *) &ftrace_graph_call; 108 new = ftrace_call_replace(ip, (unsigned long) &ftrace_stub); 109 110 return ftrace_modify_code(ip, old, new); 111 } 112 113 #endif /* !CONFIG_DYNAMIC_FTRACE */ 114 115 /* 116 * Hook the return address and push it in the stack of return addrs 117 * in current thread info. 118 */ 119 unsigned long prepare_ftrace_return(unsigned long parent, 120 unsigned long self_addr, 121 unsigned long frame_pointer) 122 { 123 unsigned long return_hooker = (unsigned long) &return_to_handler; 124 125 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 126 return parent + 8UL; 127 128 if (function_graph_enter(parent, self_addr, frame_pointer, NULL)) 129 return parent + 8UL; 130 131 return return_hooker; 132 } 133 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 134