1 /* 2 * Dynamic function tracing support. 3 * 4 * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com> 5 * Copyright (C) 2010 Rabin Vincent <rabin@rab.in> 6 * 7 * For licencing details, see COPYING. 8 * 9 * Defines low-level handling of mcount calls when the kernel 10 * is compiled with the -pg flag. When using dynamic ftrace, the 11 * mcount call-sites get patched with NOP till they are enabled. 12 * All code mutation routines here are called under stop_machine(). 13 */ 14 15 #include <linux/ftrace.h> 16 #include <linux/uaccess.h> 17 #include <linux/module.h> 18 #include <linux/stop_machine.h> 19 20 #include <asm/cacheflush.h> 21 #include <asm/opcodes.h> 22 #include <asm/ftrace.h> 23 #include <asm/insn.h> 24 #include <asm/set_memory.h> 25 #include <asm/stacktrace.h> 26 #include <asm/patch.h> 27 28 /* 29 * The compiler emitted profiling hook consists of 30 * 31 * PUSH {LR} 32 * BL __gnu_mcount_nc 33 * 34 * To turn this combined sequence into a NOP, we need to restore the value of 35 * SP before the PUSH. Let's use an ADD rather than a POP into LR, as LR is not 36 * modified anyway, and reloading LR from memory is highly likely to be less 37 * efficient. 38 */ 39 #ifdef CONFIG_THUMB2_KERNEL 40 #define NOP 0xf10d0d04 /* add.w sp, sp, #4 */ 41 #else 42 #define NOP 0xe28dd004 /* add sp, sp, #4 */ 43 #endif 44 45 #ifdef CONFIG_DYNAMIC_FTRACE 46 47 static int __ftrace_modify_code(void *data) 48 { 49 int *command = data; 50 51 ftrace_modify_all_code(*command); 52 53 return 0; 54 } 55 56 void arch_ftrace_update_code(int command) 57 { 58 stop_machine(__ftrace_modify_code, &command, NULL); 59 } 60 61 static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) 62 { 63 return NOP; 64 } 65 66 void ftrace_caller_from_init(void); 67 void ftrace_regs_caller_from_init(void); 68 69 static unsigned long __ref adjust_address(struct dyn_ftrace *rec, 70 unsigned long addr) 71 { 72 if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE) || 73 system_state >= SYSTEM_FREEING_INITMEM || 74 likely(!is_kernel_inittext(rec->ip))) 75 return addr; 76 if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) || 77 addr == (unsigned long)&ftrace_caller) 78 return (unsigned long)&ftrace_caller_from_init; 79 return (unsigned long)&ftrace_regs_caller_from_init; 80 } 81 82 int ftrace_arch_code_modify_prepare(void) 83 { 84 return 0; 85 } 86 87 int ftrace_arch_code_modify_post_process(void) 88 { 89 /* Make sure any TLB misses during machine stop are cleared. */ 90 flush_tlb_all(); 91 return 0; 92 } 93 94 static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr, 95 bool warn) 96 { 97 return arm_gen_branch_link(pc, addr, warn); 98 } 99 100 static int ftrace_modify_code(unsigned long pc, unsigned long old, 101 unsigned long new, bool validate) 102 { 103 unsigned long replaced; 104 105 if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) 106 old = __opcode_to_mem_thumb32(old); 107 else 108 old = __opcode_to_mem_arm(old); 109 110 if (validate) { 111 if (copy_from_kernel_nofault(&replaced, (void *)pc, 112 MCOUNT_INSN_SIZE)) 113 return -EFAULT; 114 115 if (replaced != old) 116 return -EINVAL; 117 } 118 119 __patch_text((void *)pc, new); 120 121 return 0; 122 } 123 124 int ftrace_update_ftrace_func(ftrace_func_t func) 125 { 126 unsigned long pc; 127 unsigned long new; 128 int ret; 129 130 pc = (unsigned long)&ftrace_call; 131 new = ftrace_call_replace(pc, (unsigned long)func, true); 132 133 ret = ftrace_modify_code(pc, 0, new, false); 134 135 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 136 if (!ret) { 137 pc = (unsigned long)&ftrace_regs_call; 138 new = ftrace_call_replace(pc, (unsigned long)func, true); 139 140 ret = ftrace_modify_code(pc, 0, new, false); 141 } 142 #endif 143 144 return ret; 145 } 146 147 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 148 { 149 unsigned long new, old; 150 unsigned long ip = rec->ip; 151 unsigned long aaddr = adjust_address(rec, addr); 152 struct module *mod = NULL; 153 154 #ifdef CONFIG_ARM_MODULE_PLTS 155 mod = rec->arch.mod; 156 #endif 157 158 old = ftrace_nop_replace(rec); 159 160 new = ftrace_call_replace(ip, aaddr, !mod); 161 #ifdef CONFIG_ARM_MODULE_PLTS 162 if (!new && mod) { 163 aaddr = get_module_plt(mod, ip, aaddr); 164 new = ftrace_call_replace(ip, aaddr, true); 165 } 166 #endif 167 168 return ftrace_modify_code(rec->ip, old, new, true); 169 } 170 171 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 172 173 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 174 unsigned long addr) 175 { 176 unsigned long new, old; 177 unsigned long ip = rec->ip; 178 179 old = ftrace_call_replace(ip, adjust_address(rec, old_addr), true); 180 181 new = ftrace_call_replace(ip, adjust_address(rec, addr), true); 182 183 return ftrace_modify_code(rec->ip, old, new, true); 184 } 185 186 #endif 187 188 int ftrace_make_nop(struct module *mod, 189 struct dyn_ftrace *rec, unsigned long addr) 190 { 191 unsigned long aaddr = adjust_address(rec, addr); 192 unsigned long ip = rec->ip; 193 unsigned long old; 194 unsigned long new; 195 int ret; 196 197 #ifdef CONFIG_ARM_MODULE_PLTS 198 /* mod is only supplied during module loading */ 199 if (!mod) 200 mod = rec->arch.mod; 201 else 202 rec->arch.mod = mod; 203 #endif 204 205 old = ftrace_call_replace(ip, aaddr, 206 !IS_ENABLED(CONFIG_ARM_MODULE_PLTS) || !mod); 207 #ifdef CONFIG_ARM_MODULE_PLTS 208 if (!old && mod) { 209 aaddr = get_module_plt(mod, ip, aaddr); 210 old = ftrace_call_replace(ip, aaddr, true); 211 } 212 #endif 213 214 new = ftrace_nop_replace(rec); 215 /* 216 * Locations in .init.text may call __gnu_mcount_mc via a linker 217 * emitted veneer if they are too far away from its implementation, and 218 * so validation may fail spuriously in such cases. Let's work around 219 * this by omitting those from validation. 220 */ 221 ret = ftrace_modify_code(ip, old, new, !is_kernel_inittext(ip)); 222 223 return ret; 224 } 225 #endif /* CONFIG_DYNAMIC_FTRACE */ 226 227 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 228 asmlinkage 229 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, 230 unsigned long frame_pointer, 231 unsigned long stack_pointer) 232 { 233 unsigned long return_hooker = (unsigned long) &return_to_handler; 234 unsigned long old; 235 236 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 237 return; 238 239 if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) { 240 /* FP points one word below parent's top of stack */ 241 frame_pointer += 4; 242 } else { 243 struct stackframe frame = { 244 .fp = frame_pointer, 245 .sp = stack_pointer, 246 .lr = self_addr, 247 .pc = self_addr, 248 }; 249 if (unwind_frame(&frame) < 0) 250 return; 251 if (frame.lr != self_addr) 252 parent = frame.lr_addr; 253 frame_pointer = frame.sp; 254 } 255 256 old = *parent; 257 *parent = return_hooker; 258 259 if (function_graph_enter(old, self_addr, frame_pointer, NULL)) 260 *parent = old; 261 } 262 263 #ifdef CONFIG_DYNAMIC_FTRACE 264 extern unsigned long ftrace_graph_call; 265 extern unsigned long ftrace_graph_call_old; 266 extern void ftrace_graph_caller_old(void); 267 extern unsigned long ftrace_graph_regs_call; 268 extern void ftrace_graph_regs_caller(void); 269 270 static int __ftrace_modify_caller(unsigned long *callsite, 271 void (*func) (void), bool enable) 272 { 273 unsigned long caller_fn = (unsigned long) func; 274 unsigned long pc = (unsigned long) callsite; 275 unsigned long branch = arm_gen_branch(pc, caller_fn); 276 unsigned long nop = arm_gen_nop(); 277 unsigned long old = enable ? nop : branch; 278 unsigned long new = enable ? branch : nop; 279 280 return ftrace_modify_code(pc, old, new, true); 281 } 282 283 static int ftrace_modify_graph_caller(bool enable) 284 { 285 int ret; 286 287 ret = __ftrace_modify_caller(&ftrace_graph_call, 288 ftrace_graph_caller, 289 enable); 290 291 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 292 if (!ret) 293 ret = __ftrace_modify_caller(&ftrace_graph_regs_call, 294 ftrace_graph_regs_caller, 295 enable); 296 #endif 297 298 299 return ret; 300 } 301 302 int ftrace_enable_ftrace_graph_caller(void) 303 { 304 return ftrace_modify_graph_caller(true); 305 } 306 307 int ftrace_disable_ftrace_graph_caller(void) 308 { 309 return ftrace_modify_graph_caller(false); 310 } 311 #endif /* CONFIG_DYNAMIC_FTRACE */ 312 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 313