1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Kernel Probes (KProbes) 4 * arch/ia64/kernel/kprobes.c 5 * 6 * Copyright (C) IBM Corporation, 2002, 2004 7 * Copyright (C) Intel Corporation, 2005 8 * 9 * 2005-Apr Rusty Lynch <rusty.lynch@intel.com> and Anil S Keshavamurthy 10 * <anil.s.keshavamurthy@intel.com> adapted from i386 11 */ 12 13 #include <linux/kprobes.h> 14 #include <linux/ptrace.h> 15 #include <linux/string.h> 16 #include <linux/slab.h> 17 #include <linux/preempt.h> 18 #include <linux/extable.h> 19 #include <linux/kdebug.h> 20 #include <linux/pgtable.h> 21 22 #include <asm/sections.h> 23 #include <asm/exception.h> 24 25 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; 26 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 27 28 struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}}; 29 30 enum instruction_type {A, I, M, F, B, L, X, u}; 31 static enum instruction_type bundle_encoding[32][3] = { 32 { M, I, I }, /* 00 */ 33 { M, I, I }, /* 01 */ 34 { M, I, I }, /* 02 */ 35 { M, I, I }, /* 03 */ 36 { M, L, X }, /* 04 */ 37 { M, L, X }, /* 05 */ 38 { u, u, u }, /* 06 */ 39 { u, u, u }, /* 07 */ 40 { M, M, I }, /* 08 */ 41 { M, M, I }, /* 09 */ 42 { M, M, I }, /* 0A */ 43 { M, M, I }, /* 0B */ 44 { M, F, I }, /* 0C */ 45 { M, F, I }, /* 0D */ 46 { M, M, F }, /* 0E */ 47 { M, M, F }, /* 0F */ 48 { M, I, B }, /* 10 */ 49 { M, I, B }, /* 11 */ 50 { M, B, B }, /* 12 */ 51 { M, B, B }, /* 13 */ 52 { u, u, u }, /* 14 */ 53 { u, u, u }, /* 15 */ 54 { B, B, B }, /* 16 */ 55 { B, B, B }, /* 17 */ 56 { M, M, B }, /* 18 */ 57 { M, M, B }, /* 19 */ 58 { u, u, u }, /* 1A */ 59 { u, u, u }, /* 1B */ 60 { M, F, B }, /* 1C */ 61 { M, F, B }, /* 1D */ 62 { u, u, u }, /* 1E */ 63 { u, u, u }, /* 1F */ 64 }; 65 66 /* Insert a long branch code */ 67 static void __kprobes set_brl_inst(void *from, void *to) 68 { 69 s64 rel = ((s64) to - (s64) from) >> 4; 70 bundle_t *brl; 71 brl = (bundle_t *) ((u64) from & ~0xf); 72 brl->quad0.template = 0x05; /* [MLX](stop) */ 73 brl->quad0.slot0 = NOP_M_INST; /* nop.m 0x0 */ 74 brl->quad0.slot1_p0 = ((rel >> 20) & 0x7fffffffff) << 2; 75 brl->quad1.slot1_p1 = (((rel >> 20) & 0x7fffffffff) << 2) >> (64 - 46); 76 /* brl.cond.sptk.many.clr rel<<4 (qp=0) */ 77 brl->quad1.slot2 = BRL_INST(rel >> 59, rel & 0xfffff); 78 } 79 80 /* 81 * In this function we check to see if the instruction 82 * is IP relative instruction and update the kprobe 83 * inst flag accordingly 84 */ 85 static void __kprobes update_kprobe_inst_flag(uint template, uint slot, 86 uint major_opcode, 87 unsigned long kprobe_inst, 88 struct kprobe *p) 89 { 90 p->ainsn.inst_flag = 0; 91 p->ainsn.target_br_reg = 0; 92 p->ainsn.slot = slot; 93 94 /* Check for Break instruction 95 * Bits 37:40 Major opcode to be zero 96 * Bits 27:32 X6 to be zero 97 * Bits 32:35 X3 to be zero 98 */ 99 if ((!major_opcode) && (!((kprobe_inst >> 27) & 0x1FF)) ) { 100 /* is a break instruction */ 101 p->ainsn.inst_flag |= INST_FLAG_BREAK_INST; 102 return; 103 } 104 105 if (bundle_encoding[template][slot] == B) { 106 switch (major_opcode) { 107 case INDIRECT_CALL_OPCODE: 108 p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG; 109 p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7); 110 break; 111 case IP_RELATIVE_PREDICT_OPCODE: 112 case IP_RELATIVE_BRANCH_OPCODE: 113 p->ainsn.inst_flag |= INST_FLAG_FIX_RELATIVE_IP_ADDR; 114 break; 115 case IP_RELATIVE_CALL_OPCODE: 116 p->ainsn.inst_flag |= INST_FLAG_FIX_RELATIVE_IP_ADDR; 117 p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG; 118 p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7); 119 break; 120 } 121 } else if (bundle_encoding[template][slot] == X) { 122 switch (major_opcode) { 123 case LONG_CALL_OPCODE: 124 p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG; 125 p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7); 126 break; 127 } 128 } 129 return; 130 } 131 132 /* 133 * In this function we check to see if the instruction 134 * (qp) cmpx.crel.ctype p1,p2=r2,r3 135 * on which we are inserting kprobe is cmp instruction 136 * with ctype as unc. 137 */ 138 static uint __kprobes is_cmp_ctype_unc_inst(uint template, uint slot, 139 uint major_opcode, 140 unsigned long kprobe_inst) 141 { 142 cmp_inst_t cmp_inst; 143 uint ctype_unc = 0; 144 145 if (!((bundle_encoding[template][slot] == I) || 146 (bundle_encoding[template][slot] == M))) 147 goto out; 148 149 if (!((major_opcode == 0xC) || (major_opcode == 0xD) || 150 (major_opcode == 0xE))) 151 goto out; 152 153 cmp_inst.l = kprobe_inst; 154 if ((cmp_inst.f.x2 == 0) || (cmp_inst.f.x2 == 1)) { 155 /* Integer compare - Register Register (A6 type)*/ 156 if ((cmp_inst.f.tb == 0) && (cmp_inst.f.ta == 0) 157 &&(cmp_inst.f.c == 1)) 158 ctype_unc = 1; 159 } else if ((cmp_inst.f.x2 == 2)||(cmp_inst.f.x2 == 3)) { 160 /* Integer compare - Immediate Register (A8 type)*/ 161 if ((cmp_inst.f.ta == 0) &&(cmp_inst.f.c == 1)) 162 ctype_unc = 1; 163 } 164 out: 165 return ctype_unc; 166 } 167 168 /* 169 * In this function we check to see if the instruction 170 * on which we are inserting kprobe is supported. 171 * Returns qp value if supported 172 * Returns -EINVAL if unsupported 173 */ 174 static int __kprobes unsupported_inst(uint template, uint slot, 175 uint major_opcode, 176 unsigned long kprobe_inst, 177 unsigned long addr) 178 { 179 int qp; 180 181 qp = kprobe_inst & 0x3f; 182 if (is_cmp_ctype_unc_inst(template, slot, major_opcode, kprobe_inst)) { 183 if (slot == 1 && qp) { 184 printk(KERN_WARNING "Kprobes on cmp unc " 185 "instruction on slot 1 at <0x%lx> " 186 "is not supported\n", addr); 187 return -EINVAL; 188 189 } 190 qp = 0; 191 } 192 else if (bundle_encoding[template][slot] == I) { 193 if (major_opcode == 0) { 194 /* 195 * Check for Integer speculation instruction 196 * - Bit 33-35 to be equal to 0x1 197 */ 198 if (((kprobe_inst >> 33) & 0x7) == 1) { 199 printk(KERN_WARNING 200 "Kprobes on speculation inst at <0x%lx> not supported\n", 201 addr); 202 return -EINVAL; 203 } 204 /* 205 * IP relative mov instruction 206 * - Bit 27-35 to be equal to 0x30 207 */ 208 if (((kprobe_inst >> 27) & 0x1FF) == 0x30) { 209 printk(KERN_WARNING 210 "Kprobes on \"mov r1=ip\" at <0x%lx> not supported\n", 211 addr); 212 return -EINVAL; 213 214 } 215 } 216 else if ((major_opcode == 5) && !(kprobe_inst & (0xFUl << 33)) && 217 (kprobe_inst & (0x1UL << 12))) { 218 /* test bit instructions, tbit,tnat,tf 219 * bit 33-36 to be equal to 0 220 * bit 12 to be equal to 1 221 */ 222 if (slot == 1 && qp) { 223 printk(KERN_WARNING "Kprobes on test bit " 224 "instruction on slot at <0x%lx> " 225 "is not supported\n", addr); 226 return -EINVAL; 227 } 228 qp = 0; 229 } 230 } 231 else if (bundle_encoding[template][slot] == B) { 232 if (major_opcode == 7) { 233 /* IP-Relative Predict major code is 7 */ 234 printk(KERN_WARNING "Kprobes on IP-Relative" 235 "Predict is not supported\n"); 236 return -EINVAL; 237 } 238 else if (major_opcode == 2) { 239 /* Indirect Predict, major code is 2 240 * bit 27-32 to be equal to 10 or 11 241 */ 242 int x6=(kprobe_inst >> 27) & 0x3F; 243 if ((x6 == 0x10) || (x6 == 0x11)) { 244 printk(KERN_WARNING "Kprobes on " 245 "Indirect Predict is not supported\n"); 246 return -EINVAL; 247 } 248 } 249 } 250 /* kernel does not use float instruction, here for safety kprobe 251 * will judge whether it is fcmp/flass/float approximation instruction 252 */ 253 else if (unlikely(bundle_encoding[template][slot] == F)) { 254 if ((major_opcode == 4 || major_opcode == 5) && 255 (kprobe_inst & (0x1 << 12))) { 256 /* fcmp/fclass unc instruction */ 257 if (slot == 1 && qp) { 258 printk(KERN_WARNING "Kprobes on fcmp/fclass " 259 "instruction on slot at <0x%lx> " 260 "is not supported\n", addr); 261 return -EINVAL; 262 263 } 264 qp = 0; 265 } 266 if ((major_opcode == 0 || major_opcode == 1) && 267 (kprobe_inst & (0x1UL << 33))) { 268 /* float Approximation instruction */ 269 if (slot == 1 && qp) { 270 printk(KERN_WARNING "Kprobes on float Approx " 271 "instr at <0x%lx> is not supported\n", 272 addr); 273 return -EINVAL; 274 } 275 qp = 0; 276 } 277 } 278 return qp; 279 } 280 281 /* 282 * In this function we override the bundle with 283 * the break instruction at the given slot. 284 */ 285 static void __kprobes prepare_break_inst(uint template, uint slot, 286 uint major_opcode, 287 unsigned long kprobe_inst, 288 struct kprobe *p, 289 int qp) 290 { 291 unsigned long break_inst = BREAK_INST; 292 bundle_t *bundle = &p->opcode.bundle; 293 294 /* 295 * Copy the original kprobe_inst qualifying predicate(qp) 296 * to the break instruction 297 */ 298 break_inst |= qp; 299 300 switch (slot) { 301 case 0: 302 bundle->quad0.slot0 = break_inst; 303 break; 304 case 1: 305 bundle->quad0.slot1_p0 = break_inst; 306 bundle->quad1.slot1_p1 = break_inst >> (64-46); 307 break; 308 case 2: 309 bundle->quad1.slot2 = break_inst; 310 break; 311 } 312 313 /* 314 * Update the instruction flag, so that we can 315 * emulate the instruction properly after we 316 * single step on original instruction 317 */ 318 update_kprobe_inst_flag(template, slot, major_opcode, kprobe_inst, p); 319 } 320 321 static void __kprobes get_kprobe_inst(bundle_t *bundle, uint slot, 322 unsigned long *kprobe_inst, uint *major_opcode) 323 { 324 unsigned long kprobe_inst_p0, kprobe_inst_p1; 325 unsigned int template; 326 327 template = bundle->quad0.template; 328 329 switch (slot) { 330 case 0: 331 *major_opcode = (bundle->quad0.slot0 >> SLOT0_OPCODE_SHIFT); 332 *kprobe_inst = bundle->quad0.slot0; 333 break; 334 case 1: 335 *major_opcode = (bundle->quad1.slot1_p1 >> SLOT1_p1_OPCODE_SHIFT); 336 kprobe_inst_p0 = bundle->quad0.slot1_p0; 337 kprobe_inst_p1 = bundle->quad1.slot1_p1; 338 *kprobe_inst = kprobe_inst_p0 | (kprobe_inst_p1 << (64-46)); 339 break; 340 case 2: 341 *major_opcode = (bundle->quad1.slot2 >> SLOT2_OPCODE_SHIFT); 342 *kprobe_inst = bundle->quad1.slot2; 343 break; 344 } 345 } 346 347 /* Returns non-zero if the addr is in the Interrupt Vector Table */ 348 static int __kprobes in_ivt_functions(unsigned long addr) 349 { 350 return (addr >= (unsigned long)__start_ivt_text 351 && addr < (unsigned long)__end_ivt_text); 352 } 353 354 static int __kprobes valid_kprobe_addr(int template, int slot, 355 unsigned long addr) 356 { 357 if ((slot > 2) || ((bundle_encoding[template][1] == L) && slot > 1)) { 358 printk(KERN_WARNING "Attempting to insert unaligned kprobe " 359 "at 0x%lx\n", addr); 360 return -EINVAL; 361 } 362 363 if (in_ivt_functions(addr)) { 364 printk(KERN_WARNING "Kprobes can't be inserted inside " 365 "IVT functions at 0x%lx\n", addr); 366 return -EINVAL; 367 } 368 369 return 0; 370 } 371 372 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) 373 { 374 unsigned int i; 375 i = atomic_add_return(1, &kcb->prev_kprobe_index); 376 kcb->prev_kprobe[i-1].kp = kprobe_running(); 377 kcb->prev_kprobe[i-1].status = kcb->kprobe_status; 378 } 379 380 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) 381 { 382 unsigned int i; 383 i = atomic_read(&kcb->prev_kprobe_index); 384 __this_cpu_write(current_kprobe, kcb->prev_kprobe[i-1].kp); 385 kcb->kprobe_status = kcb->prev_kprobe[i-1].status; 386 atomic_sub(1, &kcb->prev_kprobe_index); 387 } 388 389 static void __kprobes set_current_kprobe(struct kprobe *p, 390 struct kprobe_ctlblk *kcb) 391 { 392 __this_cpu_write(current_kprobe, p); 393 } 394 395 static void kretprobe_trampoline(void) 396 { 397 } 398 399 int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) 400 { 401 regs->cr_iip = __kretprobe_trampoline_handler(regs, kretprobe_trampoline, NULL); 402 /* 403 * By returning a non-zero value, we are telling 404 * kprobe_handler() that we don't want the post_handler 405 * to run (and have re-enabled preemption) 406 */ 407 return 1; 408 } 409 410 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, 411 struct pt_regs *regs) 412 { 413 ri->ret_addr = (kprobe_opcode_t *)regs->b0; 414 ri->fp = NULL; 415 416 /* Replace the return addr with trampoline addr */ 417 regs->b0 = ((struct fnptr *)kretprobe_trampoline)->ip; 418 } 419 420 /* Check the instruction in the slot is break */ 421 static int __kprobes __is_ia64_break_inst(bundle_t *bundle, uint slot) 422 { 423 unsigned int major_opcode; 424 unsigned int template = bundle->quad0.template; 425 unsigned long kprobe_inst; 426 427 /* Move to slot 2, if bundle is MLX type and kprobe slot is 1 */ 428 if (slot == 1 && bundle_encoding[template][1] == L) 429 slot++; 430 431 /* Get Kprobe probe instruction at given slot*/ 432 get_kprobe_inst(bundle, slot, &kprobe_inst, &major_opcode); 433 434 /* For break instruction, 435 * Bits 37:40 Major opcode to be zero 436 * Bits 27:32 X6 to be zero 437 * Bits 32:35 X3 to be zero 438 */ 439 if (major_opcode || ((kprobe_inst >> 27) & 0x1FF)) { 440 /* Not a break instruction */ 441 return 0; 442 } 443 444 /* Is a break instruction */ 445 return 1; 446 } 447 448 /* 449 * In this function, we check whether the target bundle modifies IP or 450 * it triggers an exception. If so, it cannot be boostable. 451 */ 452 static int __kprobes can_boost(bundle_t *bundle, uint slot, 453 unsigned long bundle_addr) 454 { 455 unsigned int template = bundle->quad0.template; 456 457 do { 458 if (search_exception_tables(bundle_addr + slot) || 459 __is_ia64_break_inst(bundle, slot)) 460 return 0; /* exception may occur in this bundle*/ 461 } while ((++slot) < 3); 462 template &= 0x1e; 463 if (template >= 0x10 /* including B unit */ || 464 template == 0x04 /* including X unit */ || 465 template == 0x06) /* undefined */ 466 return 0; 467 468 return 1; 469 } 470 471 /* Prepare long jump bundle and disables other boosters if need */ 472 static void __kprobes prepare_booster(struct kprobe *p) 473 { 474 unsigned long addr = (unsigned long)p->addr & ~0xFULL; 475 unsigned int slot = (unsigned long)p->addr & 0xf; 476 struct kprobe *other_kp; 477 478 if (can_boost(&p->ainsn.insn[0].bundle, slot, addr)) { 479 set_brl_inst(&p->ainsn.insn[1].bundle, (bundle_t *)addr + 1); 480 p->ainsn.inst_flag |= INST_FLAG_BOOSTABLE; 481 } 482 483 /* disables boosters in previous slots */ 484 for (; addr < (unsigned long)p->addr; addr++) { 485 other_kp = get_kprobe((void *)addr); 486 if (other_kp) 487 other_kp->ainsn.inst_flag &= ~INST_FLAG_BOOSTABLE; 488 } 489 } 490 491 int __kprobes arch_prepare_kprobe(struct kprobe *p) 492 { 493 unsigned long addr = (unsigned long) p->addr; 494 unsigned long *kprobe_addr = (unsigned long *)(addr & ~0xFULL); 495 unsigned long kprobe_inst=0; 496 unsigned int slot = addr & 0xf, template, major_opcode = 0; 497 bundle_t *bundle; 498 int qp; 499 500 bundle = &((kprobe_opcode_t *)kprobe_addr)->bundle; 501 template = bundle->quad0.template; 502 503 if(valid_kprobe_addr(template, slot, addr)) 504 return -EINVAL; 505 506 /* Move to slot 2, if bundle is MLX type and kprobe slot is 1 */ 507 if (slot == 1 && bundle_encoding[template][1] == L) 508 slot++; 509 510 /* Get kprobe_inst and major_opcode from the bundle */ 511 get_kprobe_inst(bundle, slot, &kprobe_inst, &major_opcode); 512 513 qp = unsupported_inst(template, slot, major_opcode, kprobe_inst, addr); 514 if (qp < 0) 515 return -EINVAL; 516 517 p->ainsn.insn = get_insn_slot(); 518 if (!p->ainsn.insn) 519 return -ENOMEM; 520 memcpy(&p->opcode, kprobe_addr, sizeof(kprobe_opcode_t)); 521 memcpy(p->ainsn.insn, kprobe_addr, sizeof(kprobe_opcode_t)); 522 523 prepare_break_inst(template, slot, major_opcode, kprobe_inst, p, qp); 524 525 prepare_booster(p); 526 527 return 0; 528 } 529 530 void __kprobes arch_arm_kprobe(struct kprobe *p) 531 { 532 unsigned long arm_addr; 533 bundle_t *src, *dest; 534 535 arm_addr = ((unsigned long)p->addr) & ~0xFUL; 536 dest = &((kprobe_opcode_t *)arm_addr)->bundle; 537 src = &p->opcode.bundle; 538 539 flush_icache_range((unsigned long)p->ainsn.insn, 540 (unsigned long)p->ainsn.insn + 541 sizeof(kprobe_opcode_t) * MAX_INSN_SIZE); 542 543 switch (p->ainsn.slot) { 544 case 0: 545 dest->quad0.slot0 = src->quad0.slot0; 546 break; 547 case 1: 548 dest->quad1.slot1_p1 = src->quad1.slot1_p1; 549 break; 550 case 2: 551 dest->quad1.slot2 = src->quad1.slot2; 552 break; 553 } 554 flush_icache_range(arm_addr, arm_addr + sizeof(kprobe_opcode_t)); 555 } 556 557 void __kprobes arch_disarm_kprobe(struct kprobe *p) 558 { 559 unsigned long arm_addr; 560 bundle_t *src, *dest; 561 562 arm_addr = ((unsigned long)p->addr) & ~0xFUL; 563 dest = &((kprobe_opcode_t *)arm_addr)->bundle; 564 /* p->ainsn.insn contains the original unaltered kprobe_opcode_t */ 565 src = &p->ainsn.insn->bundle; 566 switch (p->ainsn.slot) { 567 case 0: 568 dest->quad0.slot0 = src->quad0.slot0; 569 break; 570 case 1: 571 dest->quad1.slot1_p1 = src->quad1.slot1_p1; 572 break; 573 case 2: 574 dest->quad1.slot2 = src->quad1.slot2; 575 break; 576 } 577 flush_icache_range(arm_addr, arm_addr + sizeof(kprobe_opcode_t)); 578 } 579 580 void __kprobes arch_remove_kprobe(struct kprobe *p) 581 { 582 if (p->ainsn.insn) { 583 free_insn_slot(p->ainsn.insn, 584 p->ainsn.inst_flag & INST_FLAG_BOOSTABLE); 585 p->ainsn.insn = NULL; 586 } 587 } 588 /* 589 * We are resuming execution after a single step fault, so the pt_regs 590 * structure reflects the register state after we executed the instruction 591 * located in the kprobe (p->ainsn.insn->bundle). We still need to adjust 592 * the ip to point back to the original stack address. To set the IP address 593 * to original stack address, handle the case where we need to fixup the 594 * relative IP address and/or fixup branch register. 595 */ 596 static void __kprobes resume_execution(struct kprobe *p, struct pt_regs *regs) 597 { 598 unsigned long bundle_addr = (unsigned long) (&p->ainsn.insn->bundle); 599 unsigned long resume_addr = (unsigned long)p->addr & ~0xFULL; 600 unsigned long template; 601 int slot = ((unsigned long)p->addr & 0xf); 602 603 template = p->ainsn.insn->bundle.quad0.template; 604 605 if (slot == 1 && bundle_encoding[template][1] == L) 606 slot = 2; 607 608 if (p->ainsn.inst_flag & ~INST_FLAG_BOOSTABLE) { 609 610 if (p->ainsn.inst_flag & INST_FLAG_FIX_RELATIVE_IP_ADDR) { 611 /* Fix relative IP address */ 612 regs->cr_iip = (regs->cr_iip - bundle_addr) + 613 resume_addr; 614 } 615 616 if (p->ainsn.inst_flag & INST_FLAG_FIX_BRANCH_REG) { 617 /* 618 * Fix target branch register, software convention is 619 * to use either b0 or b6 or b7, so just checking 620 * only those registers 621 */ 622 switch (p->ainsn.target_br_reg) { 623 case 0: 624 if ((regs->b0 == bundle_addr) || 625 (regs->b0 == bundle_addr + 0x10)) { 626 regs->b0 = (regs->b0 - bundle_addr) + 627 resume_addr; 628 } 629 break; 630 case 6: 631 if ((regs->b6 == bundle_addr) || 632 (regs->b6 == bundle_addr + 0x10)) { 633 regs->b6 = (regs->b6 - bundle_addr) + 634 resume_addr; 635 } 636 break; 637 case 7: 638 if ((regs->b7 == bundle_addr) || 639 (regs->b7 == bundle_addr + 0x10)) { 640 regs->b7 = (regs->b7 - bundle_addr) + 641 resume_addr; 642 } 643 break; 644 } /* end switch */ 645 } 646 goto turn_ss_off; 647 } 648 649 if (slot == 2) { 650 if (regs->cr_iip == bundle_addr + 0x10) { 651 regs->cr_iip = resume_addr + 0x10; 652 } 653 } else { 654 if (regs->cr_iip == bundle_addr) { 655 regs->cr_iip = resume_addr; 656 } 657 } 658 659 turn_ss_off: 660 /* Turn off Single Step bit */ 661 ia64_psr(regs)->ss = 0; 662 } 663 664 static void __kprobes prepare_ss(struct kprobe *p, struct pt_regs *regs) 665 { 666 unsigned long bundle_addr = (unsigned long) &p->ainsn.insn->bundle; 667 unsigned long slot = (unsigned long)p->addr & 0xf; 668 669 /* single step inline if break instruction */ 670 if (p->ainsn.inst_flag == INST_FLAG_BREAK_INST) 671 regs->cr_iip = (unsigned long)p->addr & ~0xFULL; 672 else 673 regs->cr_iip = bundle_addr & ~0xFULL; 674 675 if (slot > 2) 676 slot = 0; 677 678 ia64_psr(regs)->ri = slot; 679 680 /* turn on single stepping */ 681 ia64_psr(regs)->ss = 1; 682 } 683 684 static int __kprobes is_ia64_break_inst(struct pt_regs *regs) 685 { 686 unsigned int slot = ia64_psr(regs)->ri; 687 unsigned long *kprobe_addr = (unsigned long *)regs->cr_iip; 688 bundle_t bundle; 689 690 memcpy(&bundle, kprobe_addr, sizeof(bundle_t)); 691 692 return __is_ia64_break_inst(&bundle, slot); 693 } 694 695 static int __kprobes pre_kprobes_handler(struct die_args *args) 696 { 697 struct kprobe *p; 698 int ret = 0; 699 struct pt_regs *regs = args->regs; 700 kprobe_opcode_t *addr = (kprobe_opcode_t *)instruction_pointer(regs); 701 struct kprobe_ctlblk *kcb; 702 703 /* 704 * We don't want to be preempted for the entire 705 * duration of kprobe processing 706 */ 707 preempt_disable(); 708 kcb = get_kprobe_ctlblk(); 709 710 /* Handle recursion cases */ 711 if (kprobe_running()) { 712 p = get_kprobe(addr); 713 if (p) { 714 if ((kcb->kprobe_status == KPROBE_HIT_SS) && 715 (p->ainsn.inst_flag == INST_FLAG_BREAK_INST)) { 716 ia64_psr(regs)->ss = 0; 717 goto no_kprobe; 718 } 719 /* We have reentered the pre_kprobe_handler(), since 720 * another probe was hit while within the handler. 721 * We here save the original kprobes variables and 722 * just single step on the instruction of the new probe 723 * without calling any user handlers. 724 */ 725 save_previous_kprobe(kcb); 726 set_current_kprobe(p, kcb); 727 kprobes_inc_nmissed_count(p); 728 prepare_ss(p, regs); 729 kcb->kprobe_status = KPROBE_REENTER; 730 return 1; 731 } else if (!is_ia64_break_inst(regs)) { 732 /* The breakpoint instruction was removed by 733 * another cpu right after we hit, no further 734 * handling of this interrupt is appropriate 735 */ 736 ret = 1; 737 goto no_kprobe; 738 } else { 739 /* Not our break */ 740 goto no_kprobe; 741 } 742 } 743 744 p = get_kprobe(addr); 745 if (!p) { 746 if (!is_ia64_break_inst(regs)) { 747 /* 748 * The breakpoint instruction was removed right 749 * after we hit it. Another cpu has removed 750 * either a probepoint or a debugger breakpoint 751 * at this address. In either case, no further 752 * handling of this interrupt is appropriate. 753 */ 754 ret = 1; 755 756 } 757 758 /* Not one of our break, let kernel handle it */ 759 goto no_kprobe; 760 } 761 762 set_current_kprobe(p, kcb); 763 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 764 765 if (p->pre_handler && p->pre_handler(p, regs)) { 766 reset_current_kprobe(); 767 preempt_enable_no_resched(); 768 return 1; 769 } 770 771 #if !defined(CONFIG_PREEMPTION) 772 if (p->ainsn.inst_flag == INST_FLAG_BOOSTABLE && !p->post_handler) { 773 /* Boost up -- we can execute copied instructions directly */ 774 ia64_psr(regs)->ri = p->ainsn.slot; 775 regs->cr_iip = (unsigned long)&p->ainsn.insn->bundle & ~0xFULL; 776 /* turn single stepping off */ 777 ia64_psr(regs)->ss = 0; 778 779 reset_current_kprobe(); 780 preempt_enable_no_resched(); 781 return 1; 782 } 783 #endif 784 prepare_ss(p, regs); 785 kcb->kprobe_status = KPROBE_HIT_SS; 786 return 1; 787 788 no_kprobe: 789 preempt_enable_no_resched(); 790 return ret; 791 } 792 793 static int __kprobes post_kprobes_handler(struct pt_regs *regs) 794 { 795 struct kprobe *cur = kprobe_running(); 796 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 797 798 if (!cur) 799 return 0; 800 801 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { 802 kcb->kprobe_status = KPROBE_HIT_SSDONE; 803 cur->post_handler(cur, regs, 0); 804 } 805 806 resume_execution(cur, regs); 807 808 /*Restore back the original saved kprobes variables and continue. */ 809 if (kcb->kprobe_status == KPROBE_REENTER) { 810 restore_previous_kprobe(kcb); 811 goto out; 812 } 813 reset_current_kprobe(); 814 815 out: 816 preempt_enable_no_resched(); 817 return 1; 818 } 819 820 int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) 821 { 822 struct kprobe *cur = kprobe_running(); 823 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 824 825 826 switch(kcb->kprobe_status) { 827 case KPROBE_HIT_SS: 828 case KPROBE_REENTER: 829 /* 830 * We are here because the instruction being single 831 * stepped caused a page fault. We reset the current 832 * kprobe and the instruction pointer points back to 833 * the probe address and allow the page fault handler 834 * to continue as a normal page fault. 835 */ 836 regs->cr_iip = ((unsigned long)cur->addr) & ~0xFULL; 837 ia64_psr(regs)->ri = ((unsigned long)cur->addr) & 0xf; 838 if (kcb->kprobe_status == KPROBE_REENTER) 839 restore_previous_kprobe(kcb); 840 else 841 reset_current_kprobe(); 842 preempt_enable_no_resched(); 843 break; 844 case KPROBE_HIT_ACTIVE: 845 case KPROBE_HIT_SSDONE: 846 /* 847 * We increment the nmissed count for accounting, 848 * we can also use npre/npostfault count for accounting 849 * these specific fault cases. 850 */ 851 kprobes_inc_nmissed_count(cur); 852 853 /* 854 * We come here because instructions in the pre/post 855 * handler caused the page_fault, this could happen 856 * if handler tries to access user space by 857 * copy_from_user(), get_user() etc. Let the 858 * user-specified handler try to fix it first. 859 */ 860 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) 861 return 1; 862 /* 863 * In case the user-specified fault handler returned 864 * zero, try to fix up. 865 */ 866 if (ia64_done_with_exception(regs)) 867 return 1; 868 869 /* 870 * Let ia64_do_page_fault() fix it. 871 */ 872 break; 873 default: 874 break; 875 } 876 877 return 0; 878 } 879 880 int __kprobes kprobe_exceptions_notify(struct notifier_block *self, 881 unsigned long val, void *data) 882 { 883 struct die_args *args = (struct die_args *)data; 884 int ret = NOTIFY_DONE; 885 886 if (args->regs && user_mode(args->regs)) 887 return ret; 888 889 switch(val) { 890 case DIE_BREAK: 891 /* err is break number from ia64_bad_break() */ 892 if ((args->err >> 12) == (__IA64_BREAK_KPROBE >> 12) 893 || args->err == 0) 894 if (pre_kprobes_handler(args)) 895 ret = NOTIFY_STOP; 896 break; 897 case DIE_FAULT: 898 /* err is vector number from ia64_fault() */ 899 if (args->err == 36) 900 if (post_kprobes_handler(args->regs)) 901 ret = NOTIFY_STOP; 902 break; 903 default: 904 break; 905 } 906 return ret; 907 } 908 909 unsigned long arch_deref_entry_point(void *entry) 910 { 911 return ((struct fnptr *)entry)->ip; 912 } 913 914 static struct kprobe trampoline_p = { 915 .pre_handler = trampoline_probe_handler 916 }; 917 918 int __init arch_init_kprobes(void) 919 { 920 trampoline_p.addr = 921 (kprobe_opcode_t *)((struct fnptr *)kretprobe_trampoline)->ip; 922 return register_kprobe(&trampoline_p); 923 } 924 925 int __kprobes arch_trampoline_kprobe(struct kprobe *p) 926 { 927 if (p->addr == 928 (kprobe_opcode_t *)((struct fnptr *)kretprobe_trampoline)->ip) 929 return 1; 930 931 return 0; 932 } 933