1 /* 2 * Dynamic function tracing support. 3 * 4 * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com> 5 * Copyright (C) 2010 Rabin Vincent <rabin@rab.in> 6 * 7 * For licencing details, see COPYING. 8 * 9 * Defines low-level handling of mcount calls when the kernel 10 * is compiled with the -pg flag. When using dynamic ftrace, the 11 * mcount call-sites get patched with NOP till they are enabled. 12 * All code mutation routines here are called under stop_machine(). 13 */ 14 15 #include <linux/ftrace.h> 16 #include <linux/uaccess.h> 17 #include <linux/module.h> 18 #include <linux/stop_machine.h> 19 20 #include <asm/cacheflush.h> 21 #include <asm/opcodes.h> 22 #include <asm/ftrace.h> 23 24 #include "insn.h" 25 26 #ifdef CONFIG_THUMB2_KERNEL 27 #define NOP 0xf85deb04 /* pop.w {lr} */ 28 #else 29 #define NOP 0xe8bd4000 /* pop {lr} */ 30 #endif 31 32 #ifdef CONFIG_DYNAMIC_FTRACE 33 #ifdef CONFIG_OLD_MCOUNT 34 #define OLD_MCOUNT_ADDR ((unsigned long) mcount) 35 #define OLD_FTRACE_ADDR ((unsigned long) ftrace_caller_old) 36 37 #define OLD_NOP 0xe1a00000 /* mov r0, r0 */ 38 39 static int __ftrace_modify_code(void *data) 40 { 41 int *command = data; 42 43 set_kernel_text_rw(); 44 ftrace_modify_all_code(*command); 45 set_kernel_text_ro(); 46 47 return 0; 48 } 49 50 void arch_ftrace_update_code(int command) 51 { 52 stop_machine(__ftrace_modify_code, &command, NULL); 53 } 54 55 static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) 56 { 57 return rec->arch.old_mcount ? OLD_NOP : NOP; 58 } 59 60 static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) 61 { 62 if (!rec->arch.old_mcount) 63 return addr; 64 65 if (addr == MCOUNT_ADDR) 66 addr = OLD_MCOUNT_ADDR; 67 else if (addr == FTRACE_ADDR) 68 addr = OLD_FTRACE_ADDR; 69 70 return addr; 71 } 72 #else 73 static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) 74 { 75 return NOP; 76 } 77 78 static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) 79 { 80 return addr; 81 } 82 #endif 83 84 int ftrace_arch_code_modify_prepare(void) 85 { 86 set_all_modules_text_rw(); 87 return 0; 88 } 89 90 int ftrace_arch_code_modify_post_process(void) 91 { 92 set_all_modules_text_ro(); 93 /* Make sure any TLB misses during machine stop are cleared. */ 94 flush_tlb_all(); 95 return 0; 96 } 97 98 static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) 99 { 100 return arm_gen_branch_link(pc, addr); 101 } 102 103 static int ftrace_modify_code(unsigned long pc, unsigned long old, 104 unsigned long new, bool validate) 105 { 106 unsigned long replaced; 107 108 if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) { 109 old = __opcode_to_mem_thumb32(old); 110 new = __opcode_to_mem_thumb32(new); 111 } else { 112 old = __opcode_to_mem_arm(old); 113 new = __opcode_to_mem_arm(new); 114 } 115 116 if (validate) { 117 if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE)) 118 return -EFAULT; 119 120 if (replaced != old) 121 return -EINVAL; 122 } 123 124 if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE)) 125 return -EPERM; 126 127 flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); 128 129 return 0; 130 } 131 132 int ftrace_update_ftrace_func(ftrace_func_t func) 133 { 134 unsigned long pc; 135 unsigned long new; 136 int ret; 137 138 pc = (unsigned long)&ftrace_call; 139 new = ftrace_call_replace(pc, (unsigned long)func); 140 141 ret = ftrace_modify_code(pc, 0, new, false); 142 143 #ifdef CONFIG_OLD_MCOUNT 144 if (!ret) { 145 pc = (unsigned long)&ftrace_call_old; 146 new = ftrace_call_replace(pc, (unsigned long)func); 147 148 ret = ftrace_modify_code(pc, 0, new, false); 149 } 150 #endif 151 152 return ret; 153 } 154 155 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 156 { 157 unsigned long new, old; 158 unsigned long ip = rec->ip; 159 160 old = ftrace_nop_replace(rec); 161 new = ftrace_call_replace(ip, adjust_address(rec, addr)); 162 163 return ftrace_modify_code(rec->ip, old, new, true); 164 } 165 166 int ftrace_make_nop(struct module *mod, 167 struct dyn_ftrace *rec, unsigned long addr) 168 { 169 unsigned long ip = rec->ip; 170 unsigned long old; 171 unsigned long new; 172 int ret; 173 174 old = ftrace_call_replace(ip, adjust_address(rec, addr)); 175 new = ftrace_nop_replace(rec); 176 ret = ftrace_modify_code(ip, old, new, true); 177 178 #ifdef CONFIG_OLD_MCOUNT 179 if (ret == -EINVAL && addr == MCOUNT_ADDR) { 180 rec->arch.old_mcount = true; 181 182 old = ftrace_call_replace(ip, adjust_address(rec, addr)); 183 new = ftrace_nop_replace(rec); 184 ret = ftrace_modify_code(ip, old, new, true); 185 } 186 #endif 187 188 return ret; 189 } 190 191 int __init ftrace_dyn_arch_init(void) 192 { 193 return 0; 194 } 195 #endif /* CONFIG_DYNAMIC_FTRACE */ 196 197 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 198 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, 199 unsigned long frame_pointer) 200 { 201 unsigned long return_hooker = (unsigned long) &return_to_handler; 202 struct ftrace_graph_ent trace; 203 unsigned long old; 204 int err; 205 206 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 207 return; 208 209 old = *parent; 210 *parent = return_hooker; 211 212 trace.func = self_addr; 213 trace.depth = current->curr_ret_stack + 1; 214 215 /* Only trace if the calling function expects to */ 216 if (!ftrace_graph_entry(&trace)) { 217 *parent = old; 218 return; 219 } 220 221 err = ftrace_push_return_trace(old, self_addr, &trace.depth, 222 frame_pointer); 223 if (err == -EBUSY) { 224 *parent = old; 225 return; 226 } 227 } 228 229 #ifdef CONFIG_DYNAMIC_FTRACE 230 extern unsigned long ftrace_graph_call; 231 extern unsigned long ftrace_graph_call_old; 232 extern void ftrace_graph_caller_old(void); 233 234 static int __ftrace_modify_caller(unsigned long *callsite, 235 void (*func) (void), bool enable) 236 { 237 unsigned long caller_fn = (unsigned long) func; 238 unsigned long pc = (unsigned long) callsite; 239 unsigned long branch = arm_gen_branch(pc, caller_fn); 240 unsigned long nop = 0xe1a00000; /* mov r0, r0 */ 241 unsigned long old = enable ? nop : branch; 242 unsigned long new = enable ? branch : nop; 243 244 return ftrace_modify_code(pc, old, new, true); 245 } 246 247 static int ftrace_modify_graph_caller(bool enable) 248 { 249 int ret; 250 251 ret = __ftrace_modify_caller(&ftrace_graph_call, 252 ftrace_graph_caller, 253 enable); 254 255 #ifdef CONFIG_OLD_MCOUNT 256 if (!ret) 257 ret = __ftrace_modify_caller(&ftrace_graph_call_old, 258 ftrace_graph_caller_old, 259 enable); 260 #endif 261 262 return ret; 263 } 264 265 int ftrace_enable_ftrace_graph_caller(void) 266 { 267 return ftrace_modify_graph_caller(true); 268 } 269 270 int ftrace_disable_ftrace_graph_caller(void) 271 { 272 return ftrace_modify_graph_caller(false); 273 } 274 #endif /* CONFIG_DYNAMIC_FTRACE */ 275 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 276