1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * arch/arm64/kernel/ftrace.c 4 * 5 * Copyright (C) 2013 Linaro Limited 6 * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> 7 */ 8 9 #include <linux/ftrace.h> 10 #include <linux/module.h> 11 #include <linux/swab.h> 12 #include <linux/uaccess.h> 13 14 #include <asm/cacheflush.h> 15 #include <asm/debug-monitors.h> 16 #include <asm/ftrace.h> 17 #include <asm/insn.h> 18 19 #ifdef CONFIG_DYNAMIC_FTRACE 20 /* 21 * Replace a single instruction, which may be a branch or NOP. 22 * If @validate == true, a replaced instruction is checked against 'old'. 23 */ 24 static int ftrace_modify_code(unsigned long pc, u32 old, u32 new, 25 bool validate) 26 { 27 u32 replaced; 28 29 /* 30 * Note: 31 * We are paranoid about modifying text, as if a bug were to happen, it 32 * could cause us to read or write to someplace that could cause harm. 33 * Carefully read and modify the code with aarch64_insn_*() which uses 34 * probe_kernel_*(), and make sure what we read is what we expected it 35 * to be before modifying it. 36 */ 37 if (validate) { 38 if (aarch64_insn_read((void *)pc, &replaced)) 39 return -EFAULT; 40 41 if (replaced != old) 42 return -EINVAL; 43 } 44 if (aarch64_insn_patch_text_nosync((void *)pc, new)) 45 return -EPERM; 46 47 return 0; 48 } 49 50 /* 51 * Replace tracer function in ftrace_caller() 52 */ 53 int ftrace_update_ftrace_func(ftrace_func_t func) 54 { 55 unsigned long pc; 56 u32 new; 57 58 pc = (unsigned long)&ftrace_call; 59 new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func, 60 AARCH64_INSN_BRANCH_LINK); 61 62 return ftrace_modify_code(pc, 0, new, false); 63 } 64 65 /* 66 * Turn on the call to ftrace_caller() in instrumented function 67 */ 68 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 69 { 70 unsigned long pc = rec->ip; 71 u32 old, new; 72 long offset = (long)pc - (long)addr; 73 74 if (offset < -SZ_128M || offset >= SZ_128M) { 75 #ifdef CONFIG_ARM64_MODULE_PLTS 76 struct plt_entry trampoline; 77 struct module *mod; 78 79 /* 80 * On kernels that support module PLTs, the offset between the 81 * branch instruction and its target may legally exceed the 82 * range of an ordinary relative 'bl' opcode. In this case, we 83 * need to branch via a trampoline in the module. 84 * 85 * NOTE: __module_text_address() must be called with preemption 86 * disabled, but we can rely on ftrace_lock to ensure that 'mod' 87 * retains its validity throughout the remainder of this code. 88 */ 89 preempt_disable(); 90 mod = __module_text_address(pc); 91 preempt_enable(); 92 93 if (WARN_ON(!mod)) 94 return -EINVAL; 95 96 /* 97 * There is only one ftrace trampoline per module. For now, 98 * this is not a problem since on arm64, all dynamic ftrace 99 * invocations are routed via ftrace_caller(). This will need 100 * to be revisited if support for multiple ftrace entry points 101 * is added in the future, but for now, the pr_err() below 102 * deals with a theoretical issue only. 103 * 104 * Note that PLTs are place relative, and plt_entries_equal() 105 * checks whether they point to the same target. Here, we need 106 * to check if the actual opcodes are in fact identical, 107 * regardless of the offset in memory so use memcmp() instead. 108 */ 109 trampoline = get_plt_entry(addr, mod->arch.ftrace_trampoline); 110 if (memcmp(mod->arch.ftrace_trampoline, &trampoline, 111 sizeof(trampoline))) { 112 if (plt_entry_is_initialized(mod->arch.ftrace_trampoline)) { 113 pr_err("ftrace: far branches to multiple entry points unsupported inside a single module\n"); 114 return -EINVAL; 115 } 116 117 /* point the trampoline to our ftrace entry point */ 118 module_disable_ro(mod); 119 *mod->arch.ftrace_trampoline = trampoline; 120 module_enable_ro(mod, true); 121 122 /* update trampoline before patching in the branch */ 123 smp_wmb(); 124 } 125 addr = (unsigned long)(void *)mod->arch.ftrace_trampoline; 126 #else /* CONFIG_ARM64_MODULE_PLTS */ 127 return -EINVAL; 128 #endif /* CONFIG_ARM64_MODULE_PLTS */ 129 } 130 131 old = aarch64_insn_gen_nop(); 132 new = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK); 133 134 return ftrace_modify_code(pc, old, new, true); 135 } 136 137 /* 138 * Turn off the call to ftrace_caller() in instrumented function 139 */ 140 int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, 141 unsigned long addr) 142 { 143 unsigned long pc = rec->ip; 144 bool validate = true; 145 u32 old = 0, new; 146 long offset = (long)pc - (long)addr; 147 148 if (offset < -SZ_128M || offset >= SZ_128M) { 149 #ifdef CONFIG_ARM64_MODULE_PLTS 150 u32 replaced; 151 152 /* 153 * 'mod' is only set at module load time, but if we end up 154 * dealing with an out-of-range condition, we can assume it 155 * is due to a module being loaded far away from the kernel. 156 */ 157 if (!mod) { 158 preempt_disable(); 159 mod = __module_text_address(pc); 160 preempt_enable(); 161 162 if (WARN_ON(!mod)) 163 return -EINVAL; 164 } 165 166 /* 167 * The instruction we are about to patch may be a branch and 168 * link instruction that was redirected via a PLT entry. In 169 * this case, the normal validation will fail, but we can at 170 * least check that we are dealing with a branch and link 171 * instruction that points into the right module. 172 */ 173 if (aarch64_insn_read((void *)pc, &replaced)) 174 return -EFAULT; 175 176 if (!aarch64_insn_is_bl(replaced) || 177 !within_module(pc + aarch64_get_branch_offset(replaced), 178 mod)) 179 return -EINVAL; 180 181 validate = false; 182 #else /* CONFIG_ARM64_MODULE_PLTS */ 183 return -EINVAL; 184 #endif /* CONFIG_ARM64_MODULE_PLTS */ 185 } else { 186 old = aarch64_insn_gen_branch_imm(pc, addr, 187 AARCH64_INSN_BRANCH_LINK); 188 } 189 190 new = aarch64_insn_gen_nop(); 191 192 return ftrace_modify_code(pc, old, new, validate); 193 } 194 195 void arch_ftrace_update_code(int command) 196 { 197 command |= FTRACE_MAY_SLEEP; 198 ftrace_modify_all_code(command); 199 } 200 201 int __init ftrace_dyn_arch_init(void) 202 { 203 return 0; 204 } 205 #endif /* CONFIG_DYNAMIC_FTRACE */ 206 207 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 208 /* 209 * function_graph tracer expects ftrace_return_to_handler() to be called 210 * on the way back to parent. For this purpose, this function is called 211 * in _mcount() or ftrace_caller() to replace return address (*parent) on 212 * the call stack to return_to_handler. 213 * 214 * Note that @frame_pointer is used only for sanity check later. 215 */ 216 void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent, 217 unsigned long frame_pointer) 218 { 219 unsigned long return_hooker = (unsigned long)&return_to_handler; 220 unsigned long old; 221 222 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 223 return; 224 225 /* 226 * Note: 227 * No protection against faulting at *parent, which may be seen 228 * on other archs. It's unlikely on AArch64. 229 */ 230 old = *parent; 231 232 if (!function_graph_enter(old, self_addr, frame_pointer, NULL)) 233 *parent = return_hooker; 234 } 235 236 #ifdef CONFIG_DYNAMIC_FTRACE 237 /* 238 * Turn on/off the call to ftrace_graph_caller() in ftrace_caller() 239 * depending on @enable. 240 */ 241 static int ftrace_modify_graph_caller(bool enable) 242 { 243 unsigned long pc = (unsigned long)&ftrace_graph_call; 244 u32 branch, nop; 245 246 branch = aarch64_insn_gen_branch_imm(pc, 247 (unsigned long)ftrace_graph_caller, 248 AARCH64_INSN_BRANCH_NOLINK); 249 nop = aarch64_insn_gen_nop(); 250 251 if (enable) 252 return ftrace_modify_code(pc, nop, branch, true); 253 else 254 return ftrace_modify_code(pc, branch, nop, true); 255 } 256 257 int ftrace_enable_ftrace_graph_caller(void) 258 { 259 return ftrace_modify_graph_caller(true); 260 } 261 262 int ftrace_disable_ftrace_graph_caller(void) 263 { 264 return ftrace_modify_graph_caller(false); 265 } 266 #endif /* CONFIG_DYNAMIC_FTRACE */ 267 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 268