1 /* 2 * Dynamic function tracing support. 3 * 4 * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com> 5 * Copyright (C) 2010 Rabin Vincent <rabin@rab.in> 6 * 7 * For licencing details, see COPYING. 8 * 9 * Defines low-level handling of mcount calls when the kernel 10 * is compiled with the -pg flag. When using dynamic ftrace, the 11 * mcount call-sites get patched with NOP till they are enabled. 12 * All code mutation routines here are called under stop_machine(). 13 */ 14 15 #include <linux/ftrace.h> 16 #include <linux/uaccess.h> 17 #include <linux/module.h> 18 #include <linux/stop_machine.h> 19 20 #include <asm/cacheflush.h> 21 #include <asm/opcodes.h> 22 #include <asm/ftrace.h> 23 #include <asm/insn.h> 24 25 #ifdef CONFIG_THUMB2_KERNEL 26 #define NOP 0xf85deb04 /* pop.w {lr} */ 27 #else 28 #define NOP 0xe8bd4000 /* pop {lr} */ 29 #endif 30 31 #ifdef CONFIG_DYNAMIC_FTRACE 32 #ifdef CONFIG_OLD_MCOUNT 33 #define OLD_MCOUNT_ADDR ((unsigned long) mcount) 34 #define OLD_FTRACE_ADDR ((unsigned long) ftrace_caller_old) 35 36 #define OLD_NOP 0xe1a00000 /* mov r0, r0 */ 37 38 static int __ftrace_modify_code(void *data) 39 { 40 int *command = data; 41 42 set_kernel_text_rw(); 43 ftrace_modify_all_code(*command); 44 set_kernel_text_ro(); 45 46 return 0; 47 } 48 49 void arch_ftrace_update_code(int command) 50 { 51 stop_machine(__ftrace_modify_code, &command, NULL); 52 } 53 54 static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) 55 { 56 return rec->arch.old_mcount ? OLD_NOP : NOP; 57 } 58 59 static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) 60 { 61 if (!rec->arch.old_mcount) 62 return addr; 63 64 if (addr == MCOUNT_ADDR) 65 addr = OLD_MCOUNT_ADDR; 66 else if (addr == FTRACE_ADDR) 67 addr = OLD_FTRACE_ADDR; 68 69 return addr; 70 } 71 #else 72 static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) 73 { 74 return NOP; 75 } 76 77 static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) 78 { 79 return addr; 80 } 81 #endif 82 83 int ftrace_arch_code_modify_prepare(void) 84 { 85 set_all_modules_text_rw(); 86 return 0; 87 } 88 89 int ftrace_arch_code_modify_post_process(void) 90 { 91 set_all_modules_text_ro(); 92 /* Make sure any TLB misses during machine stop are cleared. */ 93 flush_tlb_all(); 94 return 0; 95 } 96 97 static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) 98 { 99 return arm_gen_branch_link(pc, addr); 100 } 101 102 static int ftrace_modify_code(unsigned long pc, unsigned long old, 103 unsigned long new, bool validate) 104 { 105 unsigned long replaced; 106 107 if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) { 108 old = __opcode_to_mem_thumb32(old); 109 new = __opcode_to_mem_thumb32(new); 110 } else { 111 old = __opcode_to_mem_arm(old); 112 new = __opcode_to_mem_arm(new); 113 } 114 115 if (validate) { 116 if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE)) 117 return -EFAULT; 118 119 if (replaced != old) 120 return -EINVAL; 121 } 122 123 if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE)) 124 return -EPERM; 125 126 flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); 127 128 return 0; 129 } 130 131 int ftrace_update_ftrace_func(ftrace_func_t func) 132 { 133 unsigned long pc; 134 unsigned long new; 135 int ret; 136 137 pc = (unsigned long)&ftrace_call; 138 new = ftrace_call_replace(pc, (unsigned long)func); 139 140 ret = ftrace_modify_code(pc, 0, new, false); 141 142 #ifdef CONFIG_OLD_MCOUNT 143 if (!ret) { 144 pc = (unsigned long)&ftrace_call_old; 145 new = ftrace_call_replace(pc, (unsigned long)func); 146 147 ret = ftrace_modify_code(pc, 0, new, false); 148 } 149 #endif 150 151 return ret; 152 } 153 154 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 155 { 156 unsigned long new, old; 157 unsigned long ip = rec->ip; 158 159 old = ftrace_nop_replace(rec); 160 new = ftrace_call_replace(ip, adjust_address(rec, addr)); 161 162 return ftrace_modify_code(rec->ip, old, new, true); 163 } 164 165 int ftrace_make_nop(struct module *mod, 166 struct dyn_ftrace *rec, unsigned long addr) 167 { 168 unsigned long ip = rec->ip; 169 unsigned long old; 170 unsigned long new; 171 int ret; 172 173 old = ftrace_call_replace(ip, adjust_address(rec, addr)); 174 new = ftrace_nop_replace(rec); 175 ret = ftrace_modify_code(ip, old, new, true); 176 177 #ifdef CONFIG_OLD_MCOUNT 178 if (ret == -EINVAL && addr == MCOUNT_ADDR) { 179 rec->arch.old_mcount = true; 180 181 old = ftrace_call_replace(ip, adjust_address(rec, addr)); 182 new = ftrace_nop_replace(rec); 183 ret = ftrace_modify_code(ip, old, new, true); 184 } 185 #endif 186 187 return ret; 188 } 189 190 int __init ftrace_dyn_arch_init(void) 191 { 192 return 0; 193 } 194 #endif /* CONFIG_DYNAMIC_FTRACE */ 195 196 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 197 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, 198 unsigned long frame_pointer) 199 { 200 unsigned long return_hooker = (unsigned long) &return_to_handler; 201 struct ftrace_graph_ent trace; 202 unsigned long old; 203 int err; 204 205 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 206 return; 207 208 old = *parent; 209 *parent = return_hooker; 210 211 trace.func = self_addr; 212 trace.depth = current->curr_ret_stack + 1; 213 214 /* Only trace if the calling function expects to */ 215 if (!ftrace_graph_entry(&trace)) { 216 *parent = old; 217 return; 218 } 219 220 err = ftrace_push_return_trace(old, self_addr, &trace.depth, 221 frame_pointer); 222 if (err == -EBUSY) { 223 *parent = old; 224 return; 225 } 226 } 227 228 #ifdef CONFIG_DYNAMIC_FTRACE 229 extern unsigned long ftrace_graph_call; 230 extern unsigned long ftrace_graph_call_old; 231 extern void ftrace_graph_caller_old(void); 232 233 static int __ftrace_modify_caller(unsigned long *callsite, 234 void (*func) (void), bool enable) 235 { 236 unsigned long caller_fn = (unsigned long) func; 237 unsigned long pc = (unsigned long) callsite; 238 unsigned long branch = arm_gen_branch(pc, caller_fn); 239 unsigned long nop = 0xe1a00000; /* mov r0, r0 */ 240 unsigned long old = enable ? nop : branch; 241 unsigned long new = enable ? branch : nop; 242 243 return ftrace_modify_code(pc, old, new, true); 244 } 245 246 static int ftrace_modify_graph_caller(bool enable) 247 { 248 int ret; 249 250 ret = __ftrace_modify_caller(&ftrace_graph_call, 251 ftrace_graph_caller, 252 enable); 253 254 #ifdef CONFIG_OLD_MCOUNT 255 if (!ret) 256 ret = __ftrace_modify_caller(&ftrace_graph_call_old, 257 ftrace_graph_caller_old, 258 enable); 259 #endif 260 261 return ret; 262 } 263 264 int ftrace_enable_ftrace_graph_caller(void) 265 { 266 return ftrace_modify_graph_caller(true); 267 } 268 269 int ftrace_disable_ftrace_graph_caller(void) 270 { 271 return ftrace_modify_graph_caller(false); 272 } 273 #endif /* CONFIG_DYNAMIC_FTRACE */ 274 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 275