1 /* 2 * Kernel Probes (KProbes) 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 * 18 * Copyright (C) IBM Corporation, 2002, 2004 19 * 20 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 21 * Probes initial implementation ( includes contributions from 22 * Rusty Russell). 23 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 24 * interface to access function arguments. 25 * 2004-Nov Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port 26 * for PPC64 27 */ 28 29 #include <linux/kprobes.h> 30 #include <linux/ptrace.h> 31 #include <linux/preempt.h> 32 #include <linux/extable.h> 33 #include <linux/kdebug.h> 34 #include <linux/slab.h> 35 #include <asm/code-patching.h> 36 #include <asm/cacheflush.h> 37 #include <asm/sstep.h> 38 #include <asm/uaccess.h> 39 40 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; 41 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 42 43 struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}}; 44 45 int __kprobes arch_prepare_kprobe(struct kprobe *p) 46 { 47 int ret = 0; 48 kprobe_opcode_t insn = *p->addr; 49 50 if ((unsigned long)p->addr & 0x03) { 51 printk("Attempt to register kprobe at an unaligned address\n"); 52 ret = -EINVAL; 53 } else if (IS_MTMSRD(insn) || IS_RFID(insn) || IS_RFI(insn)) { 54 printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n"); 55 ret = -EINVAL; 56 } 57 58 /* insn must be on a special executable page on ppc64. This is 59 * not explicitly required on ppc32 (right now), but it doesn't hurt */ 60 if (!ret) { 61 p->ainsn.insn = get_insn_slot(); 62 if (!p->ainsn.insn) 63 ret = -ENOMEM; 64 } 65 66 if (!ret) { 67 memcpy(p->ainsn.insn, p->addr, 68 MAX_INSN_SIZE * sizeof(kprobe_opcode_t)); 69 p->opcode = *p->addr; 70 flush_icache_range((unsigned long)p->ainsn.insn, 71 (unsigned long)p->ainsn.insn + sizeof(kprobe_opcode_t)); 72 } 73 74 p->ainsn.boostable = 0; 75 return ret; 76 } 77 78 void __kprobes arch_arm_kprobe(struct kprobe *p) 79 { 80 *p->addr = BREAKPOINT_INSTRUCTION; 81 flush_icache_range((unsigned long) p->addr, 82 (unsigned long) p->addr + sizeof(kprobe_opcode_t)); 83 } 84 85 void __kprobes arch_disarm_kprobe(struct kprobe *p) 86 { 87 *p->addr = p->opcode; 88 flush_icache_range((unsigned long) p->addr, 89 (unsigned long) p->addr + sizeof(kprobe_opcode_t)); 90 } 91 92 void __kprobes arch_remove_kprobe(struct kprobe *p) 93 { 94 if (p->ainsn.insn) { 95 free_insn_slot(p->ainsn.insn, 0); 96 p->ainsn.insn = NULL; 97 } 98 } 99 100 static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs) 101 { 102 enable_single_step(regs); 103 104 /* 105 * On powerpc we should single step on the original 106 * instruction even if the probed insn is a trap 107 * variant as values in regs could play a part in 108 * if the trap is taken or not 109 */ 110 regs->nip = (unsigned long)p->ainsn.insn; 111 } 112 113 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) 114 { 115 kcb->prev_kprobe.kp = kprobe_running(); 116 kcb->prev_kprobe.status = kcb->kprobe_status; 117 kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr; 118 } 119 120 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) 121 { 122 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); 123 kcb->kprobe_status = kcb->prev_kprobe.status; 124 kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr; 125 } 126 127 static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs, 128 struct kprobe_ctlblk *kcb) 129 { 130 __this_cpu_write(current_kprobe, p); 131 kcb->kprobe_saved_msr = regs->msr; 132 } 133 134 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, 135 struct pt_regs *regs) 136 { 137 ri->ret_addr = (kprobe_opcode_t *)regs->link; 138 139 /* Replace the return addr with trampoline addr */ 140 regs->link = (unsigned long)kretprobe_trampoline; 141 } 142 143 static int __kprobes kprobe_handler(struct pt_regs *regs) 144 { 145 struct kprobe *p; 146 int ret = 0; 147 unsigned int *addr = (unsigned int *)regs->nip; 148 struct kprobe_ctlblk *kcb; 149 150 /* 151 * We don't want to be preempted for the entire 152 * duration of kprobe processing 153 */ 154 preempt_disable(); 155 kcb = get_kprobe_ctlblk(); 156 157 /* Check we're not actually recursing */ 158 if (kprobe_running()) { 159 p = get_kprobe(addr); 160 if (p) { 161 kprobe_opcode_t insn = *p->ainsn.insn; 162 if (kcb->kprobe_status == KPROBE_HIT_SS && 163 is_trap(insn)) { 164 /* Turn off 'trace' bits */ 165 regs->msr &= ~MSR_SINGLESTEP; 166 regs->msr |= kcb->kprobe_saved_msr; 167 goto no_kprobe; 168 } 169 /* We have reentered the kprobe_handler(), since 170 * another probe was hit while within the handler. 171 * We here save the original kprobes variables and 172 * just single step on the instruction of the new probe 173 * without calling any user handlers. 174 */ 175 save_previous_kprobe(kcb); 176 set_current_kprobe(p, regs, kcb); 177 kcb->kprobe_saved_msr = regs->msr; 178 kprobes_inc_nmissed_count(p); 179 prepare_singlestep(p, regs); 180 kcb->kprobe_status = KPROBE_REENTER; 181 return 1; 182 } else { 183 if (*addr != BREAKPOINT_INSTRUCTION) { 184 /* If trap variant, then it belongs not to us */ 185 kprobe_opcode_t cur_insn = *addr; 186 if (is_trap(cur_insn)) 187 goto no_kprobe; 188 /* The breakpoint instruction was removed by 189 * another cpu right after we hit, no further 190 * handling of this interrupt is appropriate 191 */ 192 ret = 1; 193 goto no_kprobe; 194 } 195 p = __this_cpu_read(current_kprobe); 196 if (p->break_handler && p->break_handler(p, regs)) { 197 goto ss_probe; 198 } 199 } 200 goto no_kprobe; 201 } 202 203 p = get_kprobe(addr); 204 if (!p) { 205 if (*addr != BREAKPOINT_INSTRUCTION) { 206 /* 207 * PowerPC has multiple variants of the "trap" 208 * instruction. If the current instruction is a 209 * trap variant, it could belong to someone else 210 */ 211 kprobe_opcode_t cur_insn = *addr; 212 if (is_trap(cur_insn)) 213 goto no_kprobe; 214 /* 215 * The breakpoint instruction was removed right 216 * after we hit it. Another cpu has removed 217 * either a probepoint or a debugger breakpoint 218 * at this address. In either case, no further 219 * handling of this interrupt is appropriate. 220 */ 221 ret = 1; 222 } 223 /* Not one of ours: let kernel handle it */ 224 goto no_kprobe; 225 } 226 227 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 228 set_current_kprobe(p, regs, kcb); 229 if (p->pre_handler && p->pre_handler(p, regs)) 230 /* handler has already set things up, so skip ss setup */ 231 return 1; 232 233 ss_probe: 234 if (p->ainsn.boostable >= 0) { 235 unsigned int insn = *p->ainsn.insn; 236 237 /* regs->nip is also adjusted if emulate_step returns 1 */ 238 ret = emulate_step(regs, insn); 239 if (ret > 0) { 240 /* 241 * Once this instruction has been boosted 242 * successfully, set the boostable flag 243 */ 244 if (unlikely(p->ainsn.boostable == 0)) 245 p->ainsn.boostable = 1; 246 247 if (p->post_handler) 248 p->post_handler(p, regs, 0); 249 250 kcb->kprobe_status = KPROBE_HIT_SSDONE; 251 reset_current_kprobe(); 252 preempt_enable_no_resched(); 253 return 1; 254 } else if (ret < 0) { 255 /* 256 * We don't allow kprobes on mtmsr(d)/rfi(d), etc. 257 * So, we should never get here... but, its still 258 * good to catch them, just in case... 259 */ 260 printk("Can't step on instruction %x\n", insn); 261 BUG(); 262 } else if (ret == 0) 263 /* This instruction can't be boosted */ 264 p->ainsn.boostable = -1; 265 } 266 prepare_singlestep(p, regs); 267 kcb->kprobe_status = KPROBE_HIT_SS; 268 return 1; 269 270 no_kprobe: 271 preempt_enable_no_resched(); 272 return ret; 273 } 274 275 /* 276 * Function return probe trampoline: 277 * - init_kprobes() establishes a probepoint here 278 * - When the probed function returns, this probe 279 * causes the handlers to fire 280 */ 281 asm(".global kretprobe_trampoline\n" 282 ".type kretprobe_trampoline, @function\n" 283 "kretprobe_trampoline:\n" 284 "nop\n" 285 ".size kretprobe_trampoline, .-kretprobe_trampoline\n"); 286 287 /* 288 * Called when the probe at kretprobe trampoline is hit 289 */ 290 static int __kprobes trampoline_probe_handler(struct kprobe *p, 291 struct pt_regs *regs) 292 { 293 struct kretprobe_instance *ri = NULL; 294 struct hlist_head *head, empty_rp; 295 struct hlist_node *tmp; 296 unsigned long flags, orig_ret_address = 0; 297 unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline; 298 299 INIT_HLIST_HEAD(&empty_rp); 300 kretprobe_hash_lock(current, &head, &flags); 301 302 /* 303 * It is possible to have multiple instances associated with a given 304 * task either because an multiple functions in the call path 305 * have a return probe installed on them, and/or more than one return 306 * return probe was registered for a target function. 307 * 308 * We can handle this because: 309 * - instances are always inserted at the head of the list 310 * - when multiple return probes are registered for the same 311 * function, the first instance's ret_addr will point to the 312 * real return address, and all the rest will point to 313 * kretprobe_trampoline 314 */ 315 hlist_for_each_entry_safe(ri, tmp, head, hlist) { 316 if (ri->task != current) 317 /* another task is sharing our hash bucket */ 318 continue; 319 320 if (ri->rp && ri->rp->handler) 321 ri->rp->handler(ri, regs); 322 323 orig_ret_address = (unsigned long)ri->ret_addr; 324 recycle_rp_inst(ri, &empty_rp); 325 326 if (orig_ret_address != trampoline_address) 327 /* 328 * This is the real return address. Any other 329 * instances associated with this task are for 330 * other calls deeper on the call stack 331 */ 332 break; 333 } 334 335 kretprobe_assert(ri, orig_ret_address, trampoline_address); 336 regs->nip = orig_ret_address; 337 338 reset_current_kprobe(); 339 kretprobe_hash_unlock(current, &flags); 340 preempt_enable_no_resched(); 341 342 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { 343 hlist_del(&ri->hlist); 344 kfree(ri); 345 } 346 /* 347 * By returning a non-zero value, we are telling 348 * kprobe_handler() that we don't want the post_handler 349 * to run (and have re-enabled preemption) 350 */ 351 return 1; 352 } 353 354 /* 355 * Called after single-stepping. p->addr is the address of the 356 * instruction whose first byte has been replaced by the "breakpoint" 357 * instruction. To avoid the SMP problems that can occur when we 358 * temporarily put back the original opcode to single-step, we 359 * single-stepped a copy of the instruction. The address of this 360 * copy is p->ainsn.insn. 361 */ 362 static int __kprobes post_kprobe_handler(struct pt_regs *regs) 363 { 364 struct kprobe *cur = kprobe_running(); 365 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 366 367 if (!cur) 368 return 0; 369 370 /* make sure we got here for instruction we have a kprobe on */ 371 if (((unsigned long)cur->ainsn.insn + 4) != regs->nip) 372 return 0; 373 374 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { 375 kcb->kprobe_status = KPROBE_HIT_SSDONE; 376 cur->post_handler(cur, regs, 0); 377 } 378 379 /* Adjust nip to after the single-stepped instruction */ 380 regs->nip = (unsigned long)cur->addr + 4; 381 regs->msr |= kcb->kprobe_saved_msr; 382 383 /*Restore back the original saved kprobes variables and continue. */ 384 if (kcb->kprobe_status == KPROBE_REENTER) { 385 restore_previous_kprobe(kcb); 386 goto out; 387 } 388 reset_current_kprobe(); 389 out: 390 preempt_enable_no_resched(); 391 392 /* 393 * if somebody else is singlestepping across a probe point, msr 394 * will have DE/SE set, in which case, continue the remaining processing 395 * of do_debug, as if this is not a probe hit. 396 */ 397 if (regs->msr & MSR_SINGLESTEP) 398 return 0; 399 400 return 1; 401 } 402 403 int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) 404 { 405 struct kprobe *cur = kprobe_running(); 406 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 407 const struct exception_table_entry *entry; 408 409 switch(kcb->kprobe_status) { 410 case KPROBE_HIT_SS: 411 case KPROBE_REENTER: 412 /* 413 * We are here because the instruction being single 414 * stepped caused a page fault. We reset the current 415 * kprobe and the nip points back to the probe address 416 * and allow the page fault handler to continue as a 417 * normal page fault. 418 */ 419 regs->nip = (unsigned long)cur->addr; 420 regs->msr &= ~MSR_SINGLESTEP; /* Turn off 'trace' bits */ 421 regs->msr |= kcb->kprobe_saved_msr; 422 if (kcb->kprobe_status == KPROBE_REENTER) 423 restore_previous_kprobe(kcb); 424 else 425 reset_current_kprobe(); 426 preempt_enable_no_resched(); 427 break; 428 case KPROBE_HIT_ACTIVE: 429 case KPROBE_HIT_SSDONE: 430 /* 431 * We increment the nmissed count for accounting, 432 * we can also use npre/npostfault count for accounting 433 * these specific fault cases. 434 */ 435 kprobes_inc_nmissed_count(cur); 436 437 /* 438 * We come here because instructions in the pre/post 439 * handler caused the page_fault, this could happen 440 * if handler tries to access user space by 441 * copy_from_user(), get_user() etc. Let the 442 * user-specified handler try to fix it first. 443 */ 444 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) 445 return 1; 446 447 /* 448 * In case the user-specified fault handler returned 449 * zero, try to fix up. 450 */ 451 if ((entry = search_exception_tables(regs->nip)) != NULL) { 452 regs->nip = entry->fixup; 453 return 1; 454 } 455 456 /* 457 * fixup_exception() could not handle it, 458 * Let do_page_fault() fix it. 459 */ 460 break; 461 default: 462 break; 463 } 464 return 0; 465 } 466 467 /* 468 * Wrapper routine to for handling exceptions. 469 */ 470 int __kprobes kprobe_exceptions_notify(struct notifier_block *self, 471 unsigned long val, void *data) 472 { 473 struct die_args *args = (struct die_args *)data; 474 int ret = NOTIFY_DONE; 475 476 if (args->regs && user_mode(args->regs)) 477 return ret; 478 479 switch (val) { 480 case DIE_BPT: 481 if (kprobe_handler(args->regs)) 482 ret = NOTIFY_STOP; 483 break; 484 case DIE_SSTEP: 485 if (post_kprobe_handler(args->regs)) 486 ret = NOTIFY_STOP; 487 break; 488 default: 489 break; 490 } 491 return ret; 492 } 493 494 unsigned long arch_deref_entry_point(void *entry) 495 { 496 return ppc_global_function_entry(entry); 497 } 498 499 int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) 500 { 501 struct jprobe *jp = container_of(p, struct jprobe, kp); 502 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 503 504 memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs)); 505 506 /* setup return addr to the jprobe handler routine */ 507 regs->nip = arch_deref_entry_point(jp->entry); 508 #ifdef PPC64_ELF_ABI_v2 509 regs->gpr[12] = (unsigned long)jp->entry; 510 #elif defined(PPC64_ELF_ABI_v1) 511 regs->gpr[2] = (unsigned long)(((func_descr_t *)jp->entry)->toc); 512 #endif 513 514 return 1; 515 } 516 517 void __used __kprobes jprobe_return(void) 518 { 519 asm volatile("trap" ::: "memory"); 520 } 521 522 static void __used __kprobes jprobe_return_end(void) 523 { 524 }; 525 526 int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) 527 { 528 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 529 530 /* 531 * FIXME - we should ideally be validating that we got here 'cos 532 * of the "trap" in jprobe_return() above, before restoring the 533 * saved regs... 534 */ 535 memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs)); 536 preempt_enable_no_resched(); 537 return 1; 538 } 539 540 static struct kprobe trampoline_p = { 541 .addr = (kprobe_opcode_t *) &kretprobe_trampoline, 542 .pre_handler = trampoline_probe_handler 543 }; 544 545 int __init arch_init_kprobes(void) 546 { 547 return register_kprobe(&trampoline_p); 548 } 549 550 int __kprobes arch_trampoline_kprobe(struct kprobe *p) 551 { 552 if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline) 553 return 1; 554 555 return 0; 556 } 557