1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Kernel Probes (KProbes) 4 * 5 * Copyright (C) IBM Corporation, 2002, 2004 6 * 7 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 8 * Probes initial implementation ( includes contributions from 9 * Rusty Russell). 10 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 11 * interface to access function arguments. 12 * 2004-Nov Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port 13 * for PPC64 14 */ 15 16 #include <linux/kprobes.h> 17 #include <linux/ptrace.h> 18 #include <linux/preempt.h> 19 #include <linux/extable.h> 20 #include <linux/kdebug.h> 21 #include <linux/slab.h> 22 #include <linux/moduleloader.h> 23 #include <asm/code-patching.h> 24 #include <asm/cacheflush.h> 25 #include <asm/sstep.h> 26 #include <asm/sections.h> 27 #include <asm/inst.h> 28 #include <asm/set_memory.h> 29 #include <linux/uaccess.h> 30 31 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; 32 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 33 34 struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}}; 35 36 bool arch_within_kprobe_blacklist(unsigned long addr) 37 { 38 return (addr >= (unsigned long)__kprobes_text_start && 39 addr < (unsigned long)__kprobes_text_end) || 40 (addr >= (unsigned long)_stext && 41 addr < (unsigned long)__head_end); 42 } 43 44 kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset) 45 { 46 kprobe_opcode_t *addr = NULL; 47 48 #ifdef CONFIG_PPC64_ELF_ABI_V2 49 /* PPC64 ABIv2 needs local entry point */ 50 addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); 51 if (addr && !offset) { 52 #ifdef CONFIG_KPROBES_ON_FTRACE 53 unsigned long faddr; 54 /* 55 * Per livepatch.h, ftrace location is always within the first 56 * 16 bytes of a function on powerpc with -mprofile-kernel. 57 */ 58 faddr = ftrace_location_range((unsigned long)addr, 59 (unsigned long)addr + 16); 60 if (faddr) 61 addr = (kprobe_opcode_t *)faddr; 62 else 63 #endif 64 addr = (kprobe_opcode_t *)ppc_function_entry(addr); 65 } 66 #elif defined(CONFIG_PPC64_ELF_ABI_V1) 67 /* 68 * 64bit powerpc ABIv1 uses function descriptors: 69 * - Check for the dot variant of the symbol first. 70 * - If that fails, try looking up the symbol provided. 71 * 72 * This ensures we always get to the actual symbol and not 73 * the descriptor. 74 * 75 * Also handle <module:symbol> format. 76 */ 77 char dot_name[MODULE_NAME_LEN + 1 + KSYM_NAME_LEN]; 78 bool dot_appended = false; 79 const char *c; 80 ssize_t ret = 0; 81 int len = 0; 82 83 if ((c = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) { 84 c++; 85 len = c - name; 86 memcpy(dot_name, name, len); 87 } else 88 c = name; 89 90 if (*c != '\0' && *c != '.') { 91 dot_name[len++] = '.'; 92 dot_appended = true; 93 } 94 ret = strscpy(dot_name + len, c, KSYM_NAME_LEN); 95 if (ret > 0) 96 addr = (kprobe_opcode_t *)kallsyms_lookup_name(dot_name); 97 98 /* Fallback to the original non-dot symbol lookup */ 99 if (!addr && dot_appended) 100 addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); 101 #else 102 addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); 103 #endif 104 105 return addr; 106 } 107 108 static bool arch_kprobe_on_func_entry(unsigned long offset) 109 { 110 #ifdef CONFIG_PPC64_ELF_ABI_V2 111 #ifdef CONFIG_KPROBES_ON_FTRACE 112 return offset <= 16; 113 #else 114 return offset <= 8; 115 #endif 116 #else 117 return !offset; 118 #endif 119 } 120 121 /* XXX try and fold the magic of kprobe_lookup_name() in this */ 122 kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset, 123 bool *on_func_entry) 124 { 125 *on_func_entry = arch_kprobe_on_func_entry(offset); 126 return (kprobe_opcode_t *)(addr + offset); 127 } 128 129 void *alloc_insn_page(void) 130 { 131 void *page; 132 133 page = module_alloc(PAGE_SIZE); 134 if (!page) 135 return NULL; 136 137 if (strict_module_rwx_enabled()) { 138 set_memory_ro((unsigned long)page, 1); 139 set_memory_x((unsigned long)page, 1); 140 } 141 return page; 142 } 143 144 int arch_prepare_kprobe(struct kprobe *p) 145 { 146 int ret = 0; 147 struct kprobe *prev; 148 ppc_inst_t insn = ppc_inst_read(p->addr); 149 150 if ((unsigned long)p->addr & 0x03) { 151 printk("Attempt to register kprobe at an unaligned address\n"); 152 ret = -EINVAL; 153 } else if (!can_single_step(ppc_inst_val(insn))) { 154 printk("Cannot register a kprobe on instructions that can't be single stepped\n"); 155 ret = -EINVAL; 156 } else if ((unsigned long)p->addr & ~PAGE_MASK && 157 ppc_inst_prefixed(ppc_inst_read(p->addr - 1))) { 158 printk("Cannot register a kprobe on the second word of prefixed instruction\n"); 159 ret = -EINVAL; 160 } 161 prev = get_kprobe(p->addr - 1); 162 163 /* 164 * When prev is a ftrace-based kprobe, we don't have an insn, and it 165 * doesn't probe for prefixed instruction. 166 */ 167 if (prev && !kprobe_ftrace(prev) && 168 ppc_inst_prefixed(ppc_inst_read(prev->ainsn.insn))) { 169 printk("Cannot register a kprobe on the second word of prefixed instruction\n"); 170 ret = -EINVAL; 171 } 172 173 /* insn must be on a special executable page on ppc64. This is 174 * not explicitly required on ppc32 (right now), but it doesn't hurt */ 175 if (!ret) { 176 p->ainsn.insn = get_insn_slot(); 177 if (!p->ainsn.insn) 178 ret = -ENOMEM; 179 } 180 181 if (!ret) { 182 patch_instruction(p->ainsn.insn, insn); 183 p->opcode = ppc_inst_val(insn); 184 } 185 186 p->ainsn.boostable = 0; 187 return ret; 188 } 189 NOKPROBE_SYMBOL(arch_prepare_kprobe); 190 191 void arch_arm_kprobe(struct kprobe *p) 192 { 193 WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(BREAKPOINT_INSTRUCTION))); 194 } 195 NOKPROBE_SYMBOL(arch_arm_kprobe); 196 197 void arch_disarm_kprobe(struct kprobe *p) 198 { 199 WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(p->opcode))); 200 } 201 NOKPROBE_SYMBOL(arch_disarm_kprobe); 202 203 void arch_remove_kprobe(struct kprobe *p) 204 { 205 if (p->ainsn.insn) { 206 free_insn_slot(p->ainsn.insn, 0); 207 p->ainsn.insn = NULL; 208 } 209 } 210 NOKPROBE_SYMBOL(arch_remove_kprobe); 211 212 static nokprobe_inline void prepare_singlestep(struct kprobe *p, struct pt_regs *regs) 213 { 214 enable_single_step(regs); 215 216 /* 217 * On powerpc we should single step on the original 218 * instruction even if the probed insn is a trap 219 * variant as values in regs could play a part in 220 * if the trap is taken or not 221 */ 222 regs_set_return_ip(regs, (unsigned long)p->ainsn.insn); 223 } 224 225 static nokprobe_inline void save_previous_kprobe(struct kprobe_ctlblk *kcb) 226 { 227 kcb->prev_kprobe.kp = kprobe_running(); 228 kcb->prev_kprobe.status = kcb->kprobe_status; 229 kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr; 230 } 231 232 static nokprobe_inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb) 233 { 234 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); 235 kcb->kprobe_status = kcb->prev_kprobe.status; 236 kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr; 237 } 238 239 static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs *regs, 240 struct kprobe_ctlblk *kcb) 241 { 242 __this_cpu_write(current_kprobe, p); 243 kcb->kprobe_saved_msr = regs->msr; 244 } 245 246 void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) 247 { 248 ri->ret_addr = (kprobe_opcode_t *)regs->link; 249 ri->fp = NULL; 250 251 /* Replace the return addr with trampoline addr */ 252 regs->link = (unsigned long)__kretprobe_trampoline; 253 } 254 NOKPROBE_SYMBOL(arch_prepare_kretprobe); 255 256 static int try_to_emulate(struct kprobe *p, struct pt_regs *regs) 257 { 258 int ret; 259 ppc_inst_t insn = ppc_inst_read(p->ainsn.insn); 260 261 /* regs->nip is also adjusted if emulate_step returns 1 */ 262 ret = emulate_step(regs, insn); 263 if (ret > 0) { 264 /* 265 * Once this instruction has been boosted 266 * successfully, set the boostable flag 267 */ 268 if (unlikely(p->ainsn.boostable == 0)) 269 p->ainsn.boostable = 1; 270 } else if (ret < 0) { 271 /* 272 * We don't allow kprobes on mtmsr(d)/rfi(d), etc. 273 * So, we should never get here... but, its still 274 * good to catch them, just in case... 275 */ 276 printk("Can't step on instruction %08lx\n", ppc_inst_as_ulong(insn)); 277 BUG(); 278 } else { 279 /* 280 * If we haven't previously emulated this instruction, then it 281 * can't be boosted. Note it down so we don't try to do so again. 282 * 283 * If, however, we had emulated this instruction in the past, 284 * then this is just an error with the current run (for 285 * instance, exceptions due to a load/store). We return 0 so 286 * that this is now single-stepped, but continue to try 287 * emulating it in subsequent probe hits. 288 */ 289 if (unlikely(p->ainsn.boostable != 1)) 290 p->ainsn.boostable = -1; 291 } 292 293 return ret; 294 } 295 NOKPROBE_SYMBOL(try_to_emulate); 296 297 int kprobe_handler(struct pt_regs *regs) 298 { 299 struct kprobe *p; 300 int ret = 0; 301 unsigned int *addr = (unsigned int *)regs->nip; 302 struct kprobe_ctlblk *kcb; 303 304 if (user_mode(regs)) 305 return 0; 306 307 if (!IS_ENABLED(CONFIG_BOOKE) && 308 (!(regs->msr & MSR_IR) || !(regs->msr & MSR_DR))) 309 return 0; 310 311 /* 312 * We don't want to be preempted for the entire 313 * duration of kprobe processing 314 */ 315 preempt_disable(); 316 kcb = get_kprobe_ctlblk(); 317 318 p = get_kprobe(addr); 319 if (!p) { 320 unsigned int instr; 321 322 if (get_kernel_nofault(instr, addr)) 323 goto no_kprobe; 324 325 if (instr != BREAKPOINT_INSTRUCTION) { 326 /* 327 * PowerPC has multiple variants of the "trap" 328 * instruction. If the current instruction is a 329 * trap variant, it could belong to someone else 330 */ 331 if (is_trap(instr)) 332 goto no_kprobe; 333 /* 334 * The breakpoint instruction was removed right 335 * after we hit it. Another cpu has removed 336 * either a probepoint or a debugger breakpoint 337 * at this address. In either case, no further 338 * handling of this interrupt is appropriate. 339 */ 340 ret = 1; 341 } 342 /* Not one of ours: let kernel handle it */ 343 goto no_kprobe; 344 } 345 346 /* Check we're not actually recursing */ 347 if (kprobe_running()) { 348 kprobe_opcode_t insn = *p->ainsn.insn; 349 if (kcb->kprobe_status == KPROBE_HIT_SS && is_trap(insn)) { 350 /* Turn off 'trace' bits */ 351 regs_set_return_msr(regs, 352 (regs->msr & ~MSR_SINGLESTEP) | 353 kcb->kprobe_saved_msr); 354 goto no_kprobe; 355 } 356 357 /* 358 * We have reentered the kprobe_handler(), since another probe 359 * was hit while within the handler. We here save the original 360 * kprobes variables and just single step on the instruction of 361 * the new probe without calling any user handlers. 362 */ 363 save_previous_kprobe(kcb); 364 set_current_kprobe(p, regs, kcb); 365 kprobes_inc_nmissed_count(p); 366 kcb->kprobe_status = KPROBE_REENTER; 367 if (p->ainsn.boostable >= 0) { 368 ret = try_to_emulate(p, regs); 369 370 if (ret > 0) { 371 restore_previous_kprobe(kcb); 372 preempt_enable(); 373 return 1; 374 } 375 } 376 prepare_singlestep(p, regs); 377 return 1; 378 } 379 380 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 381 set_current_kprobe(p, regs, kcb); 382 if (p->pre_handler && p->pre_handler(p, regs)) { 383 /* handler changed execution path, so skip ss setup */ 384 reset_current_kprobe(); 385 preempt_enable(); 386 return 1; 387 } 388 389 if (p->ainsn.boostable >= 0) { 390 ret = try_to_emulate(p, regs); 391 392 if (ret > 0) { 393 if (p->post_handler) 394 p->post_handler(p, regs, 0); 395 396 kcb->kprobe_status = KPROBE_HIT_SSDONE; 397 reset_current_kprobe(); 398 preempt_enable(); 399 return 1; 400 } 401 } 402 prepare_singlestep(p, regs); 403 kcb->kprobe_status = KPROBE_HIT_SS; 404 return 1; 405 406 no_kprobe: 407 preempt_enable(); 408 return ret; 409 } 410 NOKPROBE_SYMBOL(kprobe_handler); 411 412 /* 413 * Function return probe trampoline: 414 * - init_kprobes() establishes a probepoint here 415 * - When the probed function returns, this probe 416 * causes the handlers to fire 417 */ 418 asm(".global __kretprobe_trampoline\n" 419 ".type __kretprobe_trampoline, @function\n" 420 "__kretprobe_trampoline:\n" 421 "nop\n" 422 "blr\n" 423 ".size __kretprobe_trampoline, .-__kretprobe_trampoline\n"); 424 425 /* 426 * Called when the probe at kretprobe trampoline is hit 427 */ 428 static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) 429 { 430 unsigned long orig_ret_address; 431 432 orig_ret_address = __kretprobe_trampoline_handler(regs, NULL); 433 /* 434 * We get here through one of two paths: 435 * 1. by taking a trap -> kprobe_handler() -> here 436 * 2. by optprobe branch -> optimized_callback() -> opt_pre_handler() -> here 437 * 438 * When going back through (1), we need regs->nip to be setup properly 439 * as it is used to determine the return address from the trap. 440 * For (2), since nip is not honoured with optprobes, we instead setup 441 * the link register properly so that the subsequent 'blr' in 442 * __kretprobe_trampoline jumps back to the right instruction. 443 * 444 * For nip, we should set the address to the previous instruction since 445 * we end up emulating it in kprobe_handler(), which increments the nip 446 * again. 447 */ 448 regs_set_return_ip(regs, orig_ret_address - 4); 449 regs->link = orig_ret_address; 450 451 return 0; 452 } 453 NOKPROBE_SYMBOL(trampoline_probe_handler); 454 455 /* 456 * Called after single-stepping. p->addr is the address of the 457 * instruction whose first byte has been replaced by the "breakpoint" 458 * instruction. To avoid the SMP problems that can occur when we 459 * temporarily put back the original opcode to single-step, we 460 * single-stepped a copy of the instruction. The address of this 461 * copy is p->ainsn.insn. 462 */ 463 int kprobe_post_handler(struct pt_regs *regs) 464 { 465 int len; 466 struct kprobe *cur = kprobe_running(); 467 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 468 469 if (!cur || user_mode(regs)) 470 return 0; 471 472 len = ppc_inst_len(ppc_inst_read(cur->ainsn.insn)); 473 /* make sure we got here for instruction we have a kprobe on */ 474 if (((unsigned long)cur->ainsn.insn + len) != regs->nip) 475 return 0; 476 477 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { 478 kcb->kprobe_status = KPROBE_HIT_SSDONE; 479 cur->post_handler(cur, regs, 0); 480 } 481 482 /* Adjust nip to after the single-stepped instruction */ 483 regs_set_return_ip(regs, (unsigned long)cur->addr + len); 484 regs_set_return_msr(regs, regs->msr | kcb->kprobe_saved_msr); 485 486 /*Restore back the original saved kprobes variables and continue. */ 487 if (kcb->kprobe_status == KPROBE_REENTER) { 488 restore_previous_kprobe(kcb); 489 goto out; 490 } 491 reset_current_kprobe(); 492 out: 493 preempt_enable(); 494 495 /* 496 * if somebody else is singlestepping across a probe point, msr 497 * will have DE/SE set, in which case, continue the remaining processing 498 * of do_debug, as if this is not a probe hit. 499 */ 500 if (regs->msr & MSR_SINGLESTEP) 501 return 0; 502 503 return 1; 504 } 505 NOKPROBE_SYMBOL(kprobe_post_handler); 506 507 int kprobe_fault_handler(struct pt_regs *regs, int trapnr) 508 { 509 struct kprobe *cur = kprobe_running(); 510 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 511 const struct exception_table_entry *entry; 512 513 switch(kcb->kprobe_status) { 514 case KPROBE_HIT_SS: 515 case KPROBE_REENTER: 516 /* 517 * We are here because the instruction being single 518 * stepped caused a page fault. We reset the current 519 * kprobe and the nip points back to the probe address 520 * and allow the page fault handler to continue as a 521 * normal page fault. 522 */ 523 regs_set_return_ip(regs, (unsigned long)cur->addr); 524 /* Turn off 'trace' bits */ 525 regs_set_return_msr(regs, 526 (regs->msr & ~MSR_SINGLESTEP) | 527 kcb->kprobe_saved_msr); 528 if (kcb->kprobe_status == KPROBE_REENTER) 529 restore_previous_kprobe(kcb); 530 else 531 reset_current_kprobe(); 532 preempt_enable(); 533 break; 534 case KPROBE_HIT_ACTIVE: 535 case KPROBE_HIT_SSDONE: 536 /* 537 * In case the user-specified fault handler returned 538 * zero, try to fix up. 539 */ 540 if ((entry = search_exception_tables(regs->nip)) != NULL) { 541 regs_set_return_ip(regs, extable_fixup(entry)); 542 return 1; 543 } 544 545 /* 546 * fixup_exception() could not handle it, 547 * Let do_page_fault() fix it. 548 */ 549 break; 550 default: 551 break; 552 } 553 return 0; 554 } 555 NOKPROBE_SYMBOL(kprobe_fault_handler); 556 557 static struct kprobe trampoline_p = { 558 .addr = (kprobe_opcode_t *) &__kretprobe_trampoline, 559 .pre_handler = trampoline_probe_handler 560 }; 561 562 int __init arch_init_kprobes(void) 563 { 564 return register_kprobe(&trampoline_p); 565 } 566 567 int arch_trampoline_kprobe(struct kprobe *p) 568 { 569 if (p->addr == (kprobe_opcode_t *)&__kretprobe_trampoline) 570 return 1; 571 572 return 0; 573 } 574 NOKPROBE_SYMBOL(arch_trampoline_kprobe); 575