1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Based on arch/arm64/kernel/ftrace.c 4 * 5 * Copyright (C) 2022 Loongson Technology Corporation Limited 6 */ 7 8 #include <linux/ftrace.h> 9 #include <linux/kprobes.h> 10 #include <linux/uaccess.h> 11 12 #include <asm/inst.h> 13 #include <asm/module.h> 14 15 static int ftrace_modify_code(unsigned long pc, u32 old, u32 new, bool validate) 16 { 17 u32 replaced; 18 19 if (validate) { 20 if (larch_insn_read((void *)pc, &replaced)) 21 return -EFAULT; 22 23 if (replaced != old) 24 return -EINVAL; 25 } 26 27 if (larch_insn_patch_text((void *)pc, new)) 28 return -EPERM; 29 30 return 0; 31 } 32 33 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 34 35 #ifdef CONFIG_MODULES 36 static inline int __get_mod(struct module **mod, unsigned long addr) 37 { 38 preempt_disable(); 39 *mod = __module_text_address(addr); 40 preempt_enable(); 41 42 if (WARN_ON(!(*mod))) 43 return -EINVAL; 44 45 return 0; 46 } 47 48 static struct plt_entry *get_ftrace_plt(struct module *mod, unsigned long addr) 49 { 50 struct plt_entry *plt = mod->arch.ftrace_trampolines; 51 52 if (addr == FTRACE_ADDR) 53 return &plt[FTRACE_PLT_IDX]; 54 if (addr == FTRACE_REGS_ADDR && 55 IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) 56 return &plt[FTRACE_REGS_PLT_IDX]; 57 58 return NULL; 59 } 60 61 static unsigned long get_plt_addr(struct module *mod, unsigned long addr) 62 { 63 struct plt_entry *plt; 64 65 plt = get_ftrace_plt(mod, addr); 66 if (!plt) { 67 pr_err("ftrace: no module PLT for %ps\n", (void *)addr); 68 return -EINVAL; 69 } 70 71 return (unsigned long)plt; 72 } 73 #endif 74 75 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) 76 { 77 u32 old, new; 78 unsigned long pc; 79 long offset __maybe_unused; 80 81 pc = rec->ip + LOONGARCH_INSN_SIZE; 82 83 #ifdef CONFIG_MODULES 84 offset = (long)pc - (long)addr; 85 86 if (offset < -SZ_128M || offset >= SZ_128M) { 87 int ret; 88 struct module *mod; 89 90 ret = __get_mod(&mod, pc); 91 if (ret) 92 return ret; 93 94 addr = get_plt_addr(mod, addr); 95 96 old_addr = get_plt_addr(mod, old_addr); 97 } 98 #endif 99 100 new = larch_insn_gen_bl(pc, addr); 101 old = larch_insn_gen_bl(pc, old_addr); 102 103 return ftrace_modify_code(pc, old, new, true); 104 } 105 106 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ 107 108 int ftrace_update_ftrace_func(ftrace_func_t func) 109 { 110 u32 new; 111 unsigned long pc; 112 113 pc = (unsigned long)&ftrace_call; 114 new = larch_insn_gen_bl(pc, (unsigned long)func); 115 116 return ftrace_modify_code(pc, 0, new, false); 117 } 118 119 /* 120 * The compiler has inserted 2 NOPs before the regular function prologue. 121 * T series registers are available and safe because of LoongArch's psABI. 122 * 123 * At runtime, we can replace nop with bl to enable ftrace call and replace bl 124 * with nop to disable ftrace call. The bl requires us to save the original RA 125 * value, so it saves RA at t0 here. 126 * 127 * Details are: 128 * 129 * | Compiled | Disabled | Enabled | 130 * +------------+------------------------+------------------------+ 131 * | nop | move t0, ra | move t0, ra | 132 * | nop | nop | bl ftrace_caller | 133 * | func_body | func_body | func_body | 134 * 135 * The RA value will be recovered by ftrace_regs_entry, and restored into RA 136 * before returning to the regular function prologue. When a function is not 137 * being traced, the "move t0, ra" is not harmful. 138 */ 139 140 int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) 141 { 142 u32 old, new; 143 unsigned long pc; 144 145 pc = rec->ip; 146 old = larch_insn_gen_nop(); 147 new = larch_insn_gen_move(LOONGARCH_GPR_T0, LOONGARCH_GPR_RA); 148 149 return ftrace_modify_code(pc, old, new, true); 150 } 151 152 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 153 { 154 u32 old, new; 155 unsigned long pc; 156 long offset __maybe_unused; 157 158 pc = rec->ip + LOONGARCH_INSN_SIZE; 159 160 #ifdef CONFIG_MODULES 161 offset = (long)pc - (long)addr; 162 163 if (offset < -SZ_128M || offset >= SZ_128M) { 164 int ret; 165 struct module *mod; 166 167 ret = __get_mod(&mod, pc); 168 if (ret) 169 return ret; 170 171 addr = get_plt_addr(mod, addr); 172 } 173 #endif 174 175 old = larch_insn_gen_nop(); 176 new = larch_insn_gen_bl(pc, addr); 177 178 return ftrace_modify_code(pc, old, new, true); 179 } 180 181 int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) 182 { 183 u32 old, new; 184 unsigned long pc; 185 long offset __maybe_unused; 186 187 pc = rec->ip + LOONGARCH_INSN_SIZE; 188 189 #ifdef CONFIG_MODULES 190 offset = (long)pc - (long)addr; 191 192 if (offset < -SZ_128M || offset >= SZ_128M) { 193 int ret; 194 struct module *mod; 195 196 ret = __get_mod(&mod, pc); 197 if (ret) 198 return ret; 199 200 addr = get_plt_addr(mod, addr); 201 } 202 #endif 203 204 new = larch_insn_gen_nop(); 205 old = larch_insn_gen_bl(pc, addr); 206 207 return ftrace_modify_code(pc, old, new, true); 208 } 209 210 void arch_ftrace_update_code(int command) 211 { 212 command |= FTRACE_MAY_SLEEP; 213 ftrace_modify_all_code(command); 214 } 215 216 int __init ftrace_dyn_arch_init(void) 217 { 218 return 0; 219 } 220 221 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 222 void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent) 223 { 224 unsigned long old; 225 unsigned long return_hooker = (unsigned long)&return_to_handler; 226 227 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 228 return; 229 230 old = *parent; 231 232 if (!function_graph_enter(old, self_addr, 0, parent)) 233 *parent = return_hooker; 234 } 235 236 #ifdef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS 237 void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, 238 struct ftrace_ops *op, struct ftrace_regs *fregs) 239 { 240 struct pt_regs *regs = &fregs->regs; 241 unsigned long *parent = (unsigned long *)®s->regs[1]; 242 243 prepare_ftrace_return(ip, (unsigned long *)parent); 244 } 245 #else 246 static int ftrace_modify_graph_caller(bool enable) 247 { 248 u32 branch, nop; 249 unsigned long pc, func; 250 extern void ftrace_graph_call(void); 251 252 pc = (unsigned long)&ftrace_graph_call; 253 func = (unsigned long)&ftrace_graph_caller; 254 255 nop = larch_insn_gen_nop(); 256 branch = larch_insn_gen_b(pc, func); 257 258 if (enable) 259 return ftrace_modify_code(pc, nop, branch, true); 260 else 261 return ftrace_modify_code(pc, branch, nop, true); 262 } 263 264 int ftrace_enable_ftrace_graph_caller(void) 265 { 266 return ftrace_modify_graph_caller(true); 267 } 268 269 int ftrace_disable_ftrace_graph_caller(void) 270 { 271 return ftrace_modify_graph_caller(false); 272 } 273 #endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */ 274 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 275 276 #ifdef CONFIG_KPROBES_ON_FTRACE 277 /* Ftrace callback handler for kprobes -- called under preepmt disabled */ 278 void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, 279 struct ftrace_ops *ops, struct ftrace_regs *fregs) 280 { 281 int bit; 282 struct pt_regs *regs; 283 struct kprobe *p; 284 struct kprobe_ctlblk *kcb; 285 286 bit = ftrace_test_recursion_trylock(ip, parent_ip); 287 if (bit < 0) 288 return; 289 290 p = get_kprobe((kprobe_opcode_t *)ip); 291 if (unlikely(!p) || kprobe_disabled(p)) 292 goto out; 293 294 regs = ftrace_get_regs(fregs); 295 if (!regs) 296 goto out; 297 298 kcb = get_kprobe_ctlblk(); 299 if (kprobe_running()) { 300 kprobes_inc_nmissed_count(p); 301 } else { 302 unsigned long orig_ip = instruction_pointer(regs); 303 304 instruction_pointer_set(regs, ip); 305 306 __this_cpu_write(current_kprobe, p); 307 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 308 if (!p->pre_handler || !p->pre_handler(p, regs)) { 309 /* 310 * Emulate singlestep (and also recover regs->csr_era) 311 * as if there is a nop 312 */ 313 instruction_pointer_set(regs, (unsigned long)p->addr + MCOUNT_INSN_SIZE); 314 if (unlikely(p->post_handler)) { 315 kcb->kprobe_status = KPROBE_HIT_SSDONE; 316 p->post_handler(p, regs, 0); 317 } 318 instruction_pointer_set(regs, orig_ip); 319 } 320 321 /* 322 * If pre_handler returns !0, it changes regs->csr_era. We have to 323 * skip emulating post_handler. 324 */ 325 __this_cpu_write(current_kprobe, NULL); 326 } 327 out: 328 ftrace_test_recursion_unlock(bit); 329 } 330 NOKPROBE_SYMBOL(kprobe_ftrace_handler); 331 332 int arch_prepare_kprobe_ftrace(struct kprobe *p) 333 { 334 p->ainsn.insn = NULL; 335 return 0; 336 } 337 #endif /* CONFIG_KPROBES_ON_FTRACE */ 338