1 /* 2 * Kernel Probes (KProbes) 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 * 18 * Copyright (C) IBM Corporation, 2002, 2004 19 * 20 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 21 * Probes initial implementation ( includes contributions from 22 * Rusty Russell). 23 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 24 * interface to access function arguments. 25 * 2004-Nov Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port 26 * for PPC64 27 */ 28 29 #include <linux/kprobes.h> 30 #include <linux/ptrace.h> 31 #include <linux/preempt.h> 32 #include <linux/module.h> 33 #include <linux/kdebug.h> 34 #include <linux/slab.h> 35 #include <asm/cacheflush.h> 36 #include <asm/sstep.h> 37 #include <asm/uaccess.h> 38 #include <asm/system.h> 39 40 #ifdef CONFIG_PPC_ADV_DEBUG_REGS 41 #define MSR_SINGLESTEP (MSR_DE) 42 #else 43 #define MSR_SINGLESTEP (MSR_SE) 44 #endif 45 46 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; 47 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 48 49 struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}}; 50 51 int __kprobes arch_prepare_kprobe(struct kprobe *p) 52 { 53 int ret = 0; 54 kprobe_opcode_t insn = *p->addr; 55 56 if ((unsigned long)p->addr & 0x03) { 57 printk("Attempt to register kprobe at an unaligned address\n"); 58 ret = -EINVAL; 59 } else if (IS_MTMSRD(insn) || IS_RFID(insn) || IS_RFI(insn)) { 60 printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n"); 61 ret = -EINVAL; 62 } 63 64 /* insn must be on a special executable page on ppc64. This is 65 * not explicitly required on ppc32 (right now), but it doesn't hurt */ 66 if (!ret) { 67 p->ainsn.insn = get_insn_slot(); 68 if (!p->ainsn.insn) 69 ret = -ENOMEM; 70 } 71 72 if (!ret) { 73 memcpy(p->ainsn.insn, p->addr, 74 MAX_INSN_SIZE * sizeof(kprobe_opcode_t)); 75 p->opcode = *p->addr; 76 flush_icache_range((unsigned long)p->ainsn.insn, 77 (unsigned long)p->ainsn.insn + sizeof(kprobe_opcode_t)); 78 } 79 80 p->ainsn.boostable = 0; 81 return ret; 82 } 83 84 void __kprobes arch_arm_kprobe(struct kprobe *p) 85 { 86 *p->addr = BREAKPOINT_INSTRUCTION; 87 flush_icache_range((unsigned long) p->addr, 88 (unsigned long) p->addr + sizeof(kprobe_opcode_t)); 89 } 90 91 void __kprobes arch_disarm_kprobe(struct kprobe *p) 92 { 93 *p->addr = p->opcode; 94 flush_icache_range((unsigned long) p->addr, 95 (unsigned long) p->addr + sizeof(kprobe_opcode_t)); 96 } 97 98 void __kprobes arch_remove_kprobe(struct kprobe *p) 99 { 100 if (p->ainsn.insn) { 101 free_insn_slot(p->ainsn.insn, 0); 102 p->ainsn.insn = NULL; 103 } 104 } 105 106 static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs) 107 { 108 /* We turn off async exceptions to ensure that the single step will 109 * be for the instruction we have the kprobe on, if we dont its 110 * possible we'd get the single step reported for an exception handler 111 * like Decrementer or External Interrupt */ 112 regs->msr &= ~MSR_EE; 113 regs->msr |= MSR_SINGLESTEP; 114 #ifdef CONFIG_PPC_ADV_DEBUG_REGS 115 regs->msr &= ~MSR_CE; 116 mtspr(SPRN_DBCR0, mfspr(SPRN_DBCR0) | DBCR0_IC | DBCR0_IDM); 117 #ifdef CONFIG_PPC_47x 118 isync(); 119 #endif 120 #endif 121 122 /* 123 * On powerpc we should single step on the original 124 * instruction even if the probed insn is a trap 125 * variant as values in regs could play a part in 126 * if the trap is taken or not 127 */ 128 regs->nip = (unsigned long)p->ainsn.insn; 129 } 130 131 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) 132 { 133 kcb->prev_kprobe.kp = kprobe_running(); 134 kcb->prev_kprobe.status = kcb->kprobe_status; 135 kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr; 136 } 137 138 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) 139 { 140 __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp; 141 kcb->kprobe_status = kcb->prev_kprobe.status; 142 kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr; 143 } 144 145 static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs, 146 struct kprobe_ctlblk *kcb) 147 { 148 __get_cpu_var(current_kprobe) = p; 149 kcb->kprobe_saved_msr = regs->msr; 150 } 151 152 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, 153 struct pt_regs *regs) 154 { 155 ri->ret_addr = (kprobe_opcode_t *)regs->link; 156 157 /* Replace the return addr with trampoline addr */ 158 regs->link = (unsigned long)kretprobe_trampoline; 159 } 160 161 static int __kprobes kprobe_handler(struct pt_regs *regs) 162 { 163 struct kprobe *p; 164 int ret = 0; 165 unsigned int *addr = (unsigned int *)regs->nip; 166 struct kprobe_ctlblk *kcb; 167 168 /* 169 * We don't want to be preempted for the entire 170 * duration of kprobe processing 171 */ 172 preempt_disable(); 173 kcb = get_kprobe_ctlblk(); 174 175 /* Check we're not actually recursing */ 176 if (kprobe_running()) { 177 p = get_kprobe(addr); 178 if (p) { 179 kprobe_opcode_t insn = *p->ainsn.insn; 180 if (kcb->kprobe_status == KPROBE_HIT_SS && 181 is_trap(insn)) { 182 /* Turn off 'trace' bits */ 183 regs->msr &= ~MSR_SINGLESTEP; 184 regs->msr |= kcb->kprobe_saved_msr; 185 goto no_kprobe; 186 } 187 /* We have reentered the kprobe_handler(), since 188 * another probe was hit while within the handler. 189 * We here save the original kprobes variables and 190 * just single step on the instruction of the new probe 191 * without calling any user handlers. 192 */ 193 save_previous_kprobe(kcb); 194 set_current_kprobe(p, regs, kcb); 195 kcb->kprobe_saved_msr = regs->msr; 196 kprobes_inc_nmissed_count(p); 197 prepare_singlestep(p, regs); 198 kcb->kprobe_status = KPROBE_REENTER; 199 return 1; 200 } else { 201 if (*addr != BREAKPOINT_INSTRUCTION) { 202 /* If trap variant, then it belongs not to us */ 203 kprobe_opcode_t cur_insn = *addr; 204 if (is_trap(cur_insn)) 205 goto no_kprobe; 206 /* The breakpoint instruction was removed by 207 * another cpu right after we hit, no further 208 * handling of this interrupt is appropriate 209 */ 210 ret = 1; 211 goto no_kprobe; 212 } 213 p = __get_cpu_var(current_kprobe); 214 if (p->break_handler && p->break_handler(p, regs)) { 215 goto ss_probe; 216 } 217 } 218 goto no_kprobe; 219 } 220 221 p = get_kprobe(addr); 222 if (!p) { 223 if (*addr != BREAKPOINT_INSTRUCTION) { 224 /* 225 * PowerPC has multiple variants of the "trap" 226 * instruction. If the current instruction is a 227 * trap variant, it could belong to someone else 228 */ 229 kprobe_opcode_t cur_insn = *addr; 230 if (is_trap(cur_insn)) 231 goto no_kprobe; 232 /* 233 * The breakpoint instruction was removed right 234 * after we hit it. Another cpu has removed 235 * either a probepoint or a debugger breakpoint 236 * at this address. In either case, no further 237 * handling of this interrupt is appropriate. 238 */ 239 ret = 1; 240 } 241 /* Not one of ours: let kernel handle it */ 242 goto no_kprobe; 243 } 244 245 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 246 set_current_kprobe(p, regs, kcb); 247 if (p->pre_handler && p->pre_handler(p, regs)) 248 /* handler has already set things up, so skip ss setup */ 249 return 1; 250 251 ss_probe: 252 if (p->ainsn.boostable >= 0) { 253 unsigned int insn = *p->ainsn.insn; 254 255 /* regs->nip is also adjusted if emulate_step returns 1 */ 256 ret = emulate_step(regs, insn); 257 if (ret > 0) { 258 /* 259 * Once this instruction has been boosted 260 * successfully, set the boostable flag 261 */ 262 if (unlikely(p->ainsn.boostable == 0)) 263 p->ainsn.boostable = 1; 264 265 if (p->post_handler) 266 p->post_handler(p, regs, 0); 267 268 kcb->kprobe_status = KPROBE_HIT_SSDONE; 269 reset_current_kprobe(); 270 preempt_enable_no_resched(); 271 return 1; 272 } else if (ret < 0) { 273 /* 274 * We don't allow kprobes on mtmsr(d)/rfi(d), etc. 275 * So, we should never get here... but, its still 276 * good to catch them, just in case... 277 */ 278 printk("Can't step on instruction %x\n", insn); 279 BUG(); 280 } else if (ret == 0) 281 /* This instruction can't be boosted */ 282 p->ainsn.boostable = -1; 283 } 284 prepare_singlestep(p, regs); 285 kcb->kprobe_status = KPROBE_HIT_SS; 286 return 1; 287 288 no_kprobe: 289 preempt_enable_no_resched(); 290 return ret; 291 } 292 293 /* 294 * Function return probe trampoline: 295 * - init_kprobes() establishes a probepoint here 296 * - When the probed function returns, this probe 297 * causes the handlers to fire 298 */ 299 static void __used kretprobe_trampoline_holder(void) 300 { 301 asm volatile(".global kretprobe_trampoline\n" 302 "kretprobe_trampoline:\n" 303 "nop\n"); 304 } 305 306 /* 307 * Called when the probe at kretprobe trampoline is hit 308 */ 309 static int __kprobes trampoline_probe_handler(struct kprobe *p, 310 struct pt_regs *regs) 311 { 312 struct kretprobe_instance *ri = NULL; 313 struct hlist_head *head, empty_rp; 314 struct hlist_node *node, *tmp; 315 unsigned long flags, orig_ret_address = 0; 316 unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline; 317 318 INIT_HLIST_HEAD(&empty_rp); 319 kretprobe_hash_lock(current, &head, &flags); 320 321 /* 322 * It is possible to have multiple instances associated with a given 323 * task either because an multiple functions in the call path 324 * have a return probe installed on them, and/or more than one return 325 * return probe was registered for a target function. 326 * 327 * We can handle this because: 328 * - instances are always inserted at the head of the list 329 * - when multiple return probes are registered for the same 330 * function, the first instance's ret_addr will point to the 331 * real return address, and all the rest will point to 332 * kretprobe_trampoline 333 */ 334 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { 335 if (ri->task != current) 336 /* another task is sharing our hash bucket */ 337 continue; 338 339 if (ri->rp && ri->rp->handler) 340 ri->rp->handler(ri, regs); 341 342 orig_ret_address = (unsigned long)ri->ret_addr; 343 recycle_rp_inst(ri, &empty_rp); 344 345 if (orig_ret_address != trampoline_address) 346 /* 347 * This is the real return address. Any other 348 * instances associated with this task are for 349 * other calls deeper on the call stack 350 */ 351 break; 352 } 353 354 kretprobe_assert(ri, orig_ret_address, trampoline_address); 355 regs->nip = orig_ret_address; 356 357 reset_current_kprobe(); 358 kretprobe_hash_unlock(current, &flags); 359 preempt_enable_no_resched(); 360 361 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) { 362 hlist_del(&ri->hlist); 363 kfree(ri); 364 } 365 /* 366 * By returning a non-zero value, we are telling 367 * kprobe_handler() that we don't want the post_handler 368 * to run (and have re-enabled preemption) 369 */ 370 return 1; 371 } 372 373 /* 374 * Called after single-stepping. p->addr is the address of the 375 * instruction whose first byte has been replaced by the "breakpoint" 376 * instruction. To avoid the SMP problems that can occur when we 377 * temporarily put back the original opcode to single-step, we 378 * single-stepped a copy of the instruction. The address of this 379 * copy is p->ainsn.insn. 380 */ 381 static int __kprobes post_kprobe_handler(struct pt_regs *regs) 382 { 383 struct kprobe *cur = kprobe_running(); 384 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 385 386 if (!cur) 387 return 0; 388 389 /* make sure we got here for instruction we have a kprobe on */ 390 if (((unsigned long)cur->ainsn.insn + 4) != regs->nip) 391 return 0; 392 393 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { 394 kcb->kprobe_status = KPROBE_HIT_SSDONE; 395 cur->post_handler(cur, regs, 0); 396 } 397 398 /* Adjust nip to after the single-stepped instruction */ 399 regs->nip = (unsigned long)cur->addr + 4; 400 regs->msr |= kcb->kprobe_saved_msr; 401 402 /*Restore back the original saved kprobes variables and continue. */ 403 if (kcb->kprobe_status == KPROBE_REENTER) { 404 restore_previous_kprobe(kcb); 405 goto out; 406 } 407 reset_current_kprobe(); 408 out: 409 preempt_enable_no_resched(); 410 411 /* 412 * if somebody else is singlestepping across a probe point, msr 413 * will have DE/SE set, in which case, continue the remaining processing 414 * of do_debug, as if this is not a probe hit. 415 */ 416 if (regs->msr & MSR_SINGLESTEP) 417 return 0; 418 419 return 1; 420 } 421 422 int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) 423 { 424 struct kprobe *cur = kprobe_running(); 425 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 426 const struct exception_table_entry *entry; 427 428 switch(kcb->kprobe_status) { 429 case KPROBE_HIT_SS: 430 case KPROBE_REENTER: 431 /* 432 * We are here because the instruction being single 433 * stepped caused a page fault. We reset the current 434 * kprobe and the nip points back to the probe address 435 * and allow the page fault handler to continue as a 436 * normal page fault. 437 */ 438 regs->nip = (unsigned long)cur->addr; 439 regs->msr &= ~MSR_SINGLESTEP; /* Turn off 'trace' bits */ 440 regs->msr |= kcb->kprobe_saved_msr; 441 if (kcb->kprobe_status == KPROBE_REENTER) 442 restore_previous_kprobe(kcb); 443 else 444 reset_current_kprobe(); 445 preempt_enable_no_resched(); 446 break; 447 case KPROBE_HIT_ACTIVE: 448 case KPROBE_HIT_SSDONE: 449 /* 450 * We increment the nmissed count for accounting, 451 * we can also use npre/npostfault count for accouting 452 * these specific fault cases. 453 */ 454 kprobes_inc_nmissed_count(cur); 455 456 /* 457 * We come here because instructions in the pre/post 458 * handler caused the page_fault, this could happen 459 * if handler tries to access user space by 460 * copy_from_user(), get_user() etc. Let the 461 * user-specified handler try to fix it first. 462 */ 463 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) 464 return 1; 465 466 /* 467 * In case the user-specified fault handler returned 468 * zero, try to fix up. 469 */ 470 if ((entry = search_exception_tables(regs->nip)) != NULL) { 471 regs->nip = entry->fixup; 472 return 1; 473 } 474 475 /* 476 * fixup_exception() could not handle it, 477 * Let do_page_fault() fix it. 478 */ 479 break; 480 default: 481 break; 482 } 483 return 0; 484 } 485 486 /* 487 * Wrapper routine to for handling exceptions. 488 */ 489 int __kprobes kprobe_exceptions_notify(struct notifier_block *self, 490 unsigned long val, void *data) 491 { 492 struct die_args *args = (struct die_args *)data; 493 int ret = NOTIFY_DONE; 494 495 if (args->regs && user_mode(args->regs)) 496 return ret; 497 498 switch (val) { 499 case DIE_BPT: 500 if (kprobe_handler(args->regs)) 501 ret = NOTIFY_STOP; 502 break; 503 case DIE_SSTEP: 504 if (post_kprobe_handler(args->regs)) 505 ret = NOTIFY_STOP; 506 break; 507 default: 508 break; 509 } 510 return ret; 511 } 512 513 #ifdef CONFIG_PPC64 514 unsigned long arch_deref_entry_point(void *entry) 515 { 516 return ((func_descr_t *)entry)->entry; 517 } 518 #endif 519 520 int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) 521 { 522 struct jprobe *jp = container_of(p, struct jprobe, kp); 523 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 524 525 memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs)); 526 527 /* setup return addr to the jprobe handler routine */ 528 regs->nip = arch_deref_entry_point(jp->entry); 529 #ifdef CONFIG_PPC64 530 regs->gpr[2] = (unsigned long)(((func_descr_t *)jp->entry)->toc); 531 #endif 532 533 return 1; 534 } 535 536 void __used __kprobes jprobe_return(void) 537 { 538 asm volatile("trap" ::: "memory"); 539 } 540 541 static void __used __kprobes jprobe_return_end(void) 542 { 543 }; 544 545 int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) 546 { 547 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 548 549 /* 550 * FIXME - we should ideally be validating that we got here 'cos 551 * of the "trap" in jprobe_return() above, before restoring the 552 * saved regs... 553 */ 554 memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs)); 555 preempt_enable_no_resched(); 556 return 1; 557 } 558 559 static struct kprobe trampoline_p = { 560 .addr = (kprobe_opcode_t *) &kretprobe_trampoline, 561 .pre_handler = trampoline_probe_handler 562 }; 563 564 int __init arch_init_kprobes(void) 565 { 566 return register_kprobe(&trampoline_p); 567 } 568 569 int __kprobes arch_trampoline_kprobe(struct kprobe *p) 570 { 571 if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline) 572 return 1; 573 574 return 0; 575 } 576