1 /* 2 * Dynamic function tracing support. 3 * 4 * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com> 5 * Copyright (C) 2010 Rabin Vincent <rabin@rab.in> 6 * 7 * For licencing details, see COPYING. 8 * 9 * Defines low-level handling of mcount calls when the kernel 10 * is compiled with the -pg flag. When using dynamic ftrace, the 11 * mcount call-sites get patched with NOP till they are enabled. 12 * All code mutation routines here are called under stop_machine(). 13 */ 14 15 #include <linux/ftrace.h> 16 #include <linux/uaccess.h> 17 #include <linux/module.h> 18 #include <linux/stop_machine.h> 19 20 #include <asm/cacheflush.h> 21 #include <asm/opcodes.h> 22 #include <asm/ftrace.h> 23 #include <asm/insn.h> 24 #include <asm/set_memory.h> 25 26 #ifdef CONFIG_THUMB2_KERNEL 27 #define NOP 0xf85deb04 /* pop.w {lr} */ 28 #else 29 #define NOP 0xe8bd4000 /* pop {lr} */ 30 #endif 31 32 #ifdef CONFIG_DYNAMIC_FTRACE 33 34 static int __ftrace_modify_code(void *data) 35 { 36 int *command = data; 37 38 set_kernel_text_rw(); 39 ftrace_modify_all_code(*command); 40 set_kernel_text_ro(); 41 42 return 0; 43 } 44 45 void arch_ftrace_update_code(int command) 46 { 47 stop_machine(__ftrace_modify_code, &command, NULL); 48 } 49 50 static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) 51 { 52 return NOP; 53 } 54 55 static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) 56 { 57 return addr; 58 } 59 60 int ftrace_arch_code_modify_prepare(void) 61 { 62 set_all_modules_text_rw(); 63 return 0; 64 } 65 66 int ftrace_arch_code_modify_post_process(void) 67 { 68 set_all_modules_text_ro(); 69 /* Make sure any TLB misses during machine stop are cleared. */ 70 flush_tlb_all(); 71 return 0; 72 } 73 74 static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) 75 { 76 return arm_gen_branch_link(pc, addr); 77 } 78 79 static int ftrace_modify_code(unsigned long pc, unsigned long old, 80 unsigned long new, bool validate) 81 { 82 unsigned long replaced; 83 84 if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) { 85 old = __opcode_to_mem_thumb32(old); 86 new = __opcode_to_mem_thumb32(new); 87 } else { 88 old = __opcode_to_mem_arm(old); 89 new = __opcode_to_mem_arm(new); 90 } 91 92 if (validate) { 93 if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE)) 94 return -EFAULT; 95 96 if (replaced != old) 97 return -EINVAL; 98 } 99 100 if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE)) 101 return -EPERM; 102 103 flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); 104 105 return 0; 106 } 107 108 int ftrace_update_ftrace_func(ftrace_func_t func) 109 { 110 unsigned long pc; 111 unsigned long new; 112 int ret; 113 114 pc = (unsigned long)&ftrace_call; 115 new = ftrace_call_replace(pc, (unsigned long)func); 116 117 ret = ftrace_modify_code(pc, 0, new, false); 118 119 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 120 if (!ret) { 121 pc = (unsigned long)&ftrace_regs_call; 122 new = ftrace_call_replace(pc, (unsigned long)func); 123 124 ret = ftrace_modify_code(pc, 0, new, false); 125 } 126 #endif 127 128 return ret; 129 } 130 131 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 132 { 133 unsigned long new, old; 134 unsigned long ip = rec->ip; 135 136 old = ftrace_nop_replace(rec); 137 138 new = ftrace_call_replace(ip, adjust_address(rec, addr)); 139 140 return ftrace_modify_code(rec->ip, old, new, true); 141 } 142 143 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 144 145 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 146 unsigned long addr) 147 { 148 unsigned long new, old; 149 unsigned long ip = rec->ip; 150 151 old = ftrace_call_replace(ip, adjust_address(rec, old_addr)); 152 153 new = ftrace_call_replace(ip, adjust_address(rec, addr)); 154 155 return ftrace_modify_code(rec->ip, old, new, true); 156 } 157 158 #endif 159 160 int ftrace_make_nop(struct module *mod, 161 struct dyn_ftrace *rec, unsigned long addr) 162 { 163 unsigned long ip = rec->ip; 164 unsigned long old; 165 unsigned long new; 166 int ret; 167 168 old = ftrace_call_replace(ip, adjust_address(rec, addr)); 169 new = ftrace_nop_replace(rec); 170 ret = ftrace_modify_code(ip, old, new, true); 171 172 return ret; 173 } 174 175 int __init ftrace_dyn_arch_init(void) 176 { 177 return 0; 178 } 179 #endif /* CONFIG_DYNAMIC_FTRACE */ 180 181 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 182 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, 183 unsigned long frame_pointer) 184 { 185 unsigned long return_hooker = (unsigned long) &return_to_handler; 186 unsigned long old; 187 188 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 189 return; 190 191 old = *parent; 192 *parent = return_hooker; 193 194 if (function_graph_enter(old, self_addr, frame_pointer, NULL)) 195 *parent = old; 196 } 197 198 #ifdef CONFIG_DYNAMIC_FTRACE 199 extern unsigned long ftrace_graph_call; 200 extern unsigned long ftrace_graph_call_old; 201 extern void ftrace_graph_caller_old(void); 202 extern unsigned long ftrace_graph_regs_call; 203 extern void ftrace_graph_regs_caller(void); 204 205 static int __ftrace_modify_caller(unsigned long *callsite, 206 void (*func) (void), bool enable) 207 { 208 unsigned long caller_fn = (unsigned long) func; 209 unsigned long pc = (unsigned long) callsite; 210 unsigned long branch = arm_gen_branch(pc, caller_fn); 211 unsigned long nop = 0xe1a00000; /* mov r0, r0 */ 212 unsigned long old = enable ? nop : branch; 213 unsigned long new = enable ? branch : nop; 214 215 return ftrace_modify_code(pc, old, new, true); 216 } 217 218 static int ftrace_modify_graph_caller(bool enable) 219 { 220 int ret; 221 222 ret = __ftrace_modify_caller(&ftrace_graph_call, 223 ftrace_graph_caller, 224 enable); 225 226 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 227 if (!ret) 228 ret = __ftrace_modify_caller(&ftrace_graph_regs_call, 229 ftrace_graph_regs_caller, 230 enable); 231 #endif 232 233 234 return ret; 235 } 236 237 int ftrace_enable_ftrace_graph_caller(void) 238 { 239 return ftrace_modify_graph_caller(true); 240 } 241 242 int ftrace_disable_ftrace_graph_caller(void) 243 { 244 return ftrace_modify_graph_caller(false); 245 } 246 #endif /* CONFIG_DYNAMIC_FTRACE */ 247 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 248