1 /* 2 * Dynamic function tracing support. 3 * 4 * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com> 5 * Copyright (C) 2010 Rabin Vincent <rabin@rab.in> 6 * 7 * For licencing details, see COPYING. 8 * 9 * Defines low-level handling of mcount calls when the kernel 10 * is compiled with the -pg flag. When using dynamic ftrace, the 11 * mcount call-sites get patched with NOP till they are enabled. 12 * All code mutation routines here are called under stop_machine(). 13 */ 14 15 #include <linux/ftrace.h> 16 #include <linux/uaccess.h> 17 #include <linux/module.h> 18 #include <linux/stop_machine.h> 19 20 #include <asm/cacheflush.h> 21 #include <asm/opcodes.h> 22 #include <asm/ftrace.h> 23 #include <asm/insn.h> 24 #include <asm/set_memory.h> 25 26 #ifdef CONFIG_THUMB2_KERNEL 27 #define NOP 0xf85deb04 /* pop.w {lr} */ 28 #else 29 #define NOP 0xe8bd4000 /* pop {lr} */ 30 #endif 31 32 #ifdef CONFIG_DYNAMIC_FTRACE 33 34 static int __ftrace_modify_code(void *data) 35 { 36 int *command = data; 37 38 set_kernel_text_rw(); 39 ftrace_modify_all_code(*command); 40 set_kernel_text_ro(); 41 42 return 0; 43 } 44 45 void arch_ftrace_update_code(int command) 46 { 47 stop_machine(__ftrace_modify_code, &command, NULL); 48 } 49 50 static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) 51 { 52 return NOP; 53 } 54 55 static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) 56 { 57 return addr; 58 } 59 60 int ftrace_arch_code_modify_prepare(void) 61 { 62 set_all_modules_text_rw(); 63 return 0; 64 } 65 66 int ftrace_arch_code_modify_post_process(void) 67 { 68 set_all_modules_text_ro(); 69 /* Make sure any TLB misses during machine stop are cleared. */ 70 flush_tlb_all(); 71 return 0; 72 } 73 74 static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) 75 { 76 return arm_gen_branch_link(pc, addr); 77 } 78 79 static int ftrace_modify_code(unsigned long pc, unsigned long old, 80 unsigned long new, bool validate) 81 { 82 unsigned long replaced; 83 84 if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) { 85 old = __opcode_to_mem_thumb32(old); 86 new = __opcode_to_mem_thumb32(new); 87 } else { 88 old = __opcode_to_mem_arm(old); 89 new = __opcode_to_mem_arm(new); 90 } 91 92 if (validate) { 93 if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE)) 94 return -EFAULT; 95 96 if (replaced != old) 97 return -EINVAL; 98 } 99 100 if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE)) 101 return -EPERM; 102 103 flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); 104 105 return 0; 106 } 107 108 int ftrace_update_ftrace_func(ftrace_func_t func) 109 { 110 unsigned long pc; 111 unsigned long new; 112 int ret; 113 114 pc = (unsigned long)&ftrace_call; 115 new = ftrace_call_replace(pc, (unsigned long)func); 116 117 ret = ftrace_modify_code(pc, 0, new, false); 118 119 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 120 if (!ret) { 121 pc = (unsigned long)&ftrace_regs_call; 122 new = ftrace_call_replace(pc, (unsigned long)func); 123 124 ret = ftrace_modify_code(pc, 0, new, false); 125 } 126 #endif 127 128 return ret; 129 } 130 131 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 132 { 133 unsigned long new, old; 134 unsigned long ip = rec->ip; 135 136 old = ftrace_nop_replace(rec); 137 138 new = ftrace_call_replace(ip, adjust_address(rec, addr)); 139 140 return ftrace_modify_code(rec->ip, old, new, true); 141 } 142 143 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 144 145 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 146 unsigned long addr) 147 { 148 unsigned long new, old; 149 unsigned long ip = rec->ip; 150 151 old = ftrace_call_replace(ip, adjust_address(rec, old_addr)); 152 153 new = ftrace_call_replace(ip, adjust_address(rec, addr)); 154 155 return ftrace_modify_code(rec->ip, old, new, true); 156 } 157 158 #endif 159 160 int ftrace_make_nop(struct module *mod, 161 struct dyn_ftrace *rec, unsigned long addr) 162 { 163 unsigned long ip = rec->ip; 164 unsigned long old; 165 unsigned long new; 166 int ret; 167 168 old = ftrace_call_replace(ip, adjust_address(rec, addr)); 169 new = ftrace_nop_replace(rec); 170 ret = ftrace_modify_code(ip, old, new, true); 171 172 return ret; 173 } 174 175 int __init ftrace_dyn_arch_init(void) 176 { 177 return 0; 178 } 179 #endif /* CONFIG_DYNAMIC_FTRACE */ 180 181 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 182 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, 183 unsigned long frame_pointer) 184 { 185 unsigned long return_hooker = (unsigned long) &return_to_handler; 186 struct ftrace_graph_ent trace; 187 unsigned long old; 188 int err; 189 190 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 191 return; 192 193 old = *parent; 194 *parent = return_hooker; 195 196 trace.func = self_addr; 197 trace.depth = current->curr_ret_stack + 1; 198 199 /* Only trace if the calling function expects to */ 200 if (!ftrace_graph_entry(&trace)) { 201 *parent = old; 202 return; 203 } 204 205 err = ftrace_push_return_trace(old, self_addr, &trace.depth, 206 frame_pointer, NULL); 207 if (err == -EBUSY) { 208 *parent = old; 209 return; 210 } 211 } 212 213 #ifdef CONFIG_DYNAMIC_FTRACE 214 extern unsigned long ftrace_graph_call; 215 extern unsigned long ftrace_graph_call_old; 216 extern void ftrace_graph_caller_old(void); 217 extern unsigned long ftrace_graph_regs_call; 218 extern void ftrace_graph_regs_caller(void); 219 220 static int __ftrace_modify_caller(unsigned long *callsite, 221 void (*func) (void), bool enable) 222 { 223 unsigned long caller_fn = (unsigned long) func; 224 unsigned long pc = (unsigned long) callsite; 225 unsigned long branch = arm_gen_branch(pc, caller_fn); 226 unsigned long nop = 0xe1a00000; /* mov r0, r0 */ 227 unsigned long old = enable ? nop : branch; 228 unsigned long new = enable ? branch : nop; 229 230 return ftrace_modify_code(pc, old, new, true); 231 } 232 233 static int ftrace_modify_graph_caller(bool enable) 234 { 235 int ret; 236 237 ret = __ftrace_modify_caller(&ftrace_graph_call, 238 ftrace_graph_caller, 239 enable); 240 241 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 242 if (!ret) 243 ret = __ftrace_modify_caller(&ftrace_graph_regs_call, 244 ftrace_graph_regs_caller, 245 enable); 246 #endif 247 248 249 return ret; 250 } 251 252 int ftrace_enable_ftrace_graph_caller(void) 253 { 254 return ftrace_modify_graph_caller(true); 255 } 256 257 int ftrace_disable_ftrace_graph_caller(void) 258 { 259 return ftrace_modify_graph_caller(false); 260 } 261 #endif /* CONFIG_DYNAMIC_FTRACE */ 262 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 263