1 /* 2 * Dynamic function tracing support. 3 * 4 * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com> 5 * Copyright (C) 2010 Rabin Vincent <rabin@rab.in> 6 * 7 * For licencing details, see COPYING. 8 * 9 * Defines low-level handling of mcount calls when the kernel 10 * is compiled with the -pg flag. When using dynamic ftrace, the 11 * mcount call-sites get patched with NOP till they are enabled. 12 * All code mutation routines here are called under stop_machine(). 13 */ 14 15 #include <linux/ftrace.h> 16 #include <linux/uaccess.h> 17 #include <linux/module.h> 18 19 #include <asm/cacheflush.h> 20 #include <asm/opcodes.h> 21 #include <asm/ftrace.h> 22 23 #include "insn.h" 24 25 #ifdef CONFIG_THUMB2_KERNEL 26 #define NOP 0xf85deb04 /* pop.w {lr} */ 27 #else 28 #define NOP 0xe8bd4000 /* pop {lr} */ 29 #endif 30 31 #ifdef CONFIG_DYNAMIC_FTRACE 32 #ifdef CONFIG_OLD_MCOUNT 33 #define OLD_MCOUNT_ADDR ((unsigned long) mcount) 34 #define OLD_FTRACE_ADDR ((unsigned long) ftrace_caller_old) 35 36 #define OLD_NOP 0xe1a00000 /* mov r0, r0 */ 37 38 static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) 39 { 40 return rec->arch.old_mcount ? OLD_NOP : NOP; 41 } 42 43 static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) 44 { 45 if (!rec->arch.old_mcount) 46 return addr; 47 48 if (addr == MCOUNT_ADDR) 49 addr = OLD_MCOUNT_ADDR; 50 else if (addr == FTRACE_ADDR) 51 addr = OLD_FTRACE_ADDR; 52 53 return addr; 54 } 55 #else 56 static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) 57 { 58 return NOP; 59 } 60 61 static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) 62 { 63 return addr; 64 } 65 #endif 66 67 int ftrace_arch_code_modify_prepare(void) 68 { 69 set_all_modules_text_rw(); 70 return 0; 71 } 72 73 int ftrace_arch_code_modify_post_process(void) 74 { 75 set_all_modules_text_ro(); 76 return 0; 77 } 78 79 static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) 80 { 81 return arm_gen_branch_link(pc, addr); 82 } 83 84 static int ftrace_modify_code(unsigned long pc, unsigned long old, 85 unsigned long new, bool validate) 86 { 87 unsigned long replaced; 88 89 if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) { 90 old = __opcode_to_mem_thumb32(old); 91 new = __opcode_to_mem_thumb32(new); 92 } else { 93 old = __opcode_to_mem_arm(old); 94 new = __opcode_to_mem_arm(new); 95 } 96 97 if (validate) { 98 if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE)) 99 return -EFAULT; 100 101 if (replaced != old) 102 return -EINVAL; 103 } 104 105 if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE)) 106 return -EPERM; 107 108 flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); 109 110 return 0; 111 } 112 113 int ftrace_update_ftrace_func(ftrace_func_t func) 114 { 115 unsigned long pc; 116 unsigned long new; 117 int ret; 118 119 pc = (unsigned long)&ftrace_call; 120 new = ftrace_call_replace(pc, (unsigned long)func); 121 122 ret = ftrace_modify_code(pc, 0, new, false); 123 124 #ifdef CONFIG_OLD_MCOUNT 125 if (!ret) { 126 pc = (unsigned long)&ftrace_call_old; 127 new = ftrace_call_replace(pc, (unsigned long)func); 128 129 ret = ftrace_modify_code(pc, 0, new, false); 130 } 131 #endif 132 133 return ret; 134 } 135 136 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 137 { 138 unsigned long new, old; 139 unsigned long ip = rec->ip; 140 141 old = ftrace_nop_replace(rec); 142 new = ftrace_call_replace(ip, adjust_address(rec, addr)); 143 144 return ftrace_modify_code(rec->ip, old, new, true); 145 } 146 147 int ftrace_make_nop(struct module *mod, 148 struct dyn_ftrace *rec, unsigned long addr) 149 { 150 unsigned long ip = rec->ip; 151 unsigned long old; 152 unsigned long new; 153 int ret; 154 155 old = ftrace_call_replace(ip, adjust_address(rec, addr)); 156 new = ftrace_nop_replace(rec); 157 ret = ftrace_modify_code(ip, old, new, true); 158 159 #ifdef CONFIG_OLD_MCOUNT 160 if (ret == -EINVAL && addr == MCOUNT_ADDR) { 161 rec->arch.old_mcount = true; 162 163 old = ftrace_call_replace(ip, adjust_address(rec, addr)); 164 new = ftrace_nop_replace(rec); 165 ret = ftrace_modify_code(ip, old, new, true); 166 } 167 #endif 168 169 return ret; 170 } 171 172 int __init ftrace_dyn_arch_init(void) 173 { 174 return 0; 175 } 176 #endif /* CONFIG_DYNAMIC_FTRACE */ 177 178 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 179 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, 180 unsigned long frame_pointer) 181 { 182 unsigned long return_hooker = (unsigned long) &return_to_handler; 183 struct ftrace_graph_ent trace; 184 unsigned long old; 185 int err; 186 187 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 188 return; 189 190 old = *parent; 191 *parent = return_hooker; 192 193 trace.func = self_addr; 194 trace.depth = current->curr_ret_stack + 1; 195 196 /* Only trace if the calling function expects to */ 197 if (!ftrace_graph_entry(&trace)) { 198 *parent = old; 199 return; 200 } 201 202 err = ftrace_push_return_trace(old, self_addr, &trace.depth, 203 frame_pointer); 204 if (err == -EBUSY) { 205 *parent = old; 206 return; 207 } 208 } 209 210 #ifdef CONFIG_DYNAMIC_FTRACE 211 extern unsigned long ftrace_graph_call; 212 extern unsigned long ftrace_graph_call_old; 213 extern void ftrace_graph_caller_old(void); 214 215 static int __ftrace_modify_caller(unsigned long *callsite, 216 void (*func) (void), bool enable) 217 { 218 unsigned long caller_fn = (unsigned long) func; 219 unsigned long pc = (unsigned long) callsite; 220 unsigned long branch = arm_gen_branch(pc, caller_fn); 221 unsigned long nop = 0xe1a00000; /* mov r0, r0 */ 222 unsigned long old = enable ? nop : branch; 223 unsigned long new = enable ? branch : nop; 224 225 return ftrace_modify_code(pc, old, new, true); 226 } 227 228 static int ftrace_modify_graph_caller(bool enable) 229 { 230 int ret; 231 232 ret = __ftrace_modify_caller(&ftrace_graph_call, 233 ftrace_graph_caller, 234 enable); 235 236 #ifdef CONFIG_OLD_MCOUNT 237 if (!ret) 238 ret = __ftrace_modify_caller(&ftrace_graph_call_old, 239 ftrace_graph_caller_old, 240 enable); 241 #endif 242 243 return ret; 244 } 245 246 int ftrace_enable_ftrace_graph_caller(void) 247 { 248 return ftrace_modify_graph_caller(true); 249 } 250 251 int ftrace_disable_ftrace_graph_caller(void) 252 { 253 return ftrace_modify_graph_caller(false); 254 } 255 #endif /* CONFIG_DYNAMIC_FTRACE */ 256 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 257