1 // SPDX-License-Identifier: GPL-2.0 2 /* arch/sparc64/kernel/kprobes.c 3 * 4 * Copyright (C) 2004 David S. Miller <davem@davemloft.net> 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/kprobes.h> 9 #include <linux/extable.h> 10 #include <linux/kdebug.h> 11 #include <linux/slab.h> 12 #include <linux/context_tracking.h> 13 #include <asm/signal.h> 14 #include <asm/cacheflush.h> 15 #include <linux/uaccess.h> 16 17 /* We do not have hardware single-stepping on sparc64. 18 * So we implement software single-stepping with breakpoint 19 * traps. The top-level scheme is similar to that used 20 * in the x86 kprobes implementation. 21 * 22 * In the kprobe->ainsn.insn[] array we store the original 23 * instruction at index zero and a break instruction at 24 * index one. 25 * 26 * When we hit a kprobe we: 27 * - Run the pre-handler 28 * - Remember "regs->tnpc" and interrupt level stored in 29 * "regs->tstate" so we can restore them later 30 * - Disable PIL interrupts 31 * - Set regs->tpc to point to kprobe->ainsn.insn[0] 32 * - Set regs->tnpc to point to kprobe->ainsn.insn[1] 33 * - Mark that we are actively in a kprobe 34 * 35 * At this point we wait for the second breakpoint at 36 * kprobe->ainsn.insn[1] to hit. When it does we: 37 * - Run the post-handler 38 * - Set regs->tpc to "remembered" regs->tnpc stored above, 39 * restore the PIL interrupt level in "regs->tstate" as well 40 * - Make any adjustments necessary to regs->tnpc in order 41 * to handle relative branches correctly. See below. 42 * - Mark that we are no longer actively in a kprobe. 43 */ 44 45 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; 46 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 47 48 struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}}; 49 50 int __kprobes arch_prepare_kprobe(struct kprobe *p) 51 { 52 if ((unsigned long) p->addr & 0x3UL) 53 return -EILSEQ; 54 55 p->ainsn.insn[0] = *p->addr; 56 flushi(&p->ainsn.insn[0]); 57 58 p->ainsn.insn[1] = BREAKPOINT_INSTRUCTION_2; 59 flushi(&p->ainsn.insn[1]); 60 61 p->opcode = *p->addr; 62 return 0; 63 } 64 65 void __kprobes arch_arm_kprobe(struct kprobe *p) 66 { 67 *p->addr = BREAKPOINT_INSTRUCTION; 68 flushi(p->addr); 69 } 70 71 void __kprobes arch_disarm_kprobe(struct kprobe *p) 72 { 73 *p->addr = p->opcode; 74 flushi(p->addr); 75 } 76 77 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) 78 { 79 kcb->prev_kprobe.kp = kprobe_running(); 80 kcb->prev_kprobe.status = kcb->kprobe_status; 81 kcb->prev_kprobe.orig_tnpc = kcb->kprobe_orig_tnpc; 82 kcb->prev_kprobe.orig_tstate_pil = kcb->kprobe_orig_tstate_pil; 83 } 84 85 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) 86 { 87 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); 88 kcb->kprobe_status = kcb->prev_kprobe.status; 89 kcb->kprobe_orig_tnpc = kcb->prev_kprobe.orig_tnpc; 90 kcb->kprobe_orig_tstate_pil = kcb->prev_kprobe.orig_tstate_pil; 91 } 92 93 static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs, 94 struct kprobe_ctlblk *kcb) 95 { 96 __this_cpu_write(current_kprobe, p); 97 kcb->kprobe_orig_tnpc = regs->tnpc; 98 kcb->kprobe_orig_tstate_pil = (regs->tstate & TSTATE_PIL); 99 } 100 101 static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs, 102 struct kprobe_ctlblk *kcb) 103 { 104 regs->tstate |= TSTATE_PIL; 105 106 /*single step inline, if it a breakpoint instruction*/ 107 if (p->opcode == BREAKPOINT_INSTRUCTION) { 108 regs->tpc = (unsigned long) p->addr; 109 regs->tnpc = kcb->kprobe_orig_tnpc; 110 } else { 111 regs->tpc = (unsigned long) &p->ainsn.insn[0]; 112 regs->tnpc = (unsigned long) &p->ainsn.insn[1]; 113 } 114 } 115 116 static int __kprobes kprobe_handler(struct pt_regs *regs) 117 { 118 struct kprobe *p; 119 void *addr = (void *) regs->tpc; 120 int ret = 0; 121 struct kprobe_ctlblk *kcb; 122 123 /* 124 * We don't want to be preempted for the entire 125 * duration of kprobe processing 126 */ 127 preempt_disable(); 128 kcb = get_kprobe_ctlblk(); 129 130 if (kprobe_running()) { 131 p = get_kprobe(addr); 132 if (p) { 133 if (kcb->kprobe_status == KPROBE_HIT_SS) { 134 regs->tstate = ((regs->tstate & ~TSTATE_PIL) | 135 kcb->kprobe_orig_tstate_pil); 136 goto no_kprobe; 137 } 138 /* We have reentered the kprobe_handler(), since 139 * another probe was hit while within the handler. 140 * We here save the original kprobes variables and 141 * just single step on the instruction of the new probe 142 * without calling any user handlers. 143 */ 144 save_previous_kprobe(kcb); 145 set_current_kprobe(p, regs, kcb); 146 kprobes_inc_nmissed_count(p); 147 kcb->kprobe_status = KPROBE_REENTER; 148 prepare_singlestep(p, regs, kcb); 149 return 1; 150 } else if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { 151 /* The breakpoint instruction was removed by 152 * another cpu right after we hit, no further 153 * handling of this interrupt is appropriate 154 */ 155 ret = 1; 156 } 157 goto no_kprobe; 158 } 159 160 p = get_kprobe(addr); 161 if (!p) { 162 if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { 163 /* 164 * The breakpoint instruction was removed right 165 * after we hit it. Another cpu has removed 166 * either a probepoint or a debugger breakpoint 167 * at this address. In either case, no further 168 * handling of this interrupt is appropriate. 169 */ 170 ret = 1; 171 } 172 /* Not one of ours: let kernel handle it */ 173 goto no_kprobe; 174 } 175 176 set_current_kprobe(p, regs, kcb); 177 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 178 if (p->pre_handler && p->pre_handler(p, regs)) { 179 reset_current_kprobe(); 180 preempt_enable_no_resched(); 181 return 1; 182 } 183 184 prepare_singlestep(p, regs, kcb); 185 kcb->kprobe_status = KPROBE_HIT_SS; 186 return 1; 187 188 no_kprobe: 189 preempt_enable_no_resched(); 190 return ret; 191 } 192 193 /* If INSN is a relative control transfer instruction, 194 * return the corrected branch destination value. 195 * 196 * regs->tpc and regs->tnpc still hold the values of the 197 * program counters at the time of trap due to the execution 198 * of the BREAKPOINT_INSTRUCTION_2 at p->ainsn.insn[1] 199 * 200 */ 201 static unsigned long __kprobes relbranch_fixup(u32 insn, struct kprobe *p, 202 struct pt_regs *regs) 203 { 204 unsigned long real_pc = (unsigned long) p->addr; 205 206 /* Branch not taken, no mods necessary. */ 207 if (regs->tnpc == regs->tpc + 0x4UL) 208 return real_pc + 0x8UL; 209 210 /* The three cases are call, branch w/prediction, 211 * and traditional branch. 212 */ 213 if ((insn & 0xc0000000) == 0x40000000 || 214 (insn & 0xc1c00000) == 0x00400000 || 215 (insn & 0xc1c00000) == 0x00800000) { 216 unsigned long ainsn_addr; 217 218 ainsn_addr = (unsigned long) &p->ainsn.insn[0]; 219 220 /* The instruction did all the work for us 221 * already, just apply the offset to the correct 222 * instruction location. 223 */ 224 return (real_pc + (regs->tnpc - ainsn_addr)); 225 } 226 227 /* It is jmpl or some other absolute PC modification instruction, 228 * leave NPC as-is. 229 */ 230 return regs->tnpc; 231 } 232 233 /* If INSN is an instruction which writes it's PC location 234 * into a destination register, fix that up. 235 */ 236 static void __kprobes retpc_fixup(struct pt_regs *regs, u32 insn, 237 unsigned long real_pc) 238 { 239 unsigned long *slot = NULL; 240 241 /* Simplest case is 'call', which always uses %o7 */ 242 if ((insn & 0xc0000000) == 0x40000000) { 243 slot = ®s->u_regs[UREG_I7]; 244 } 245 246 /* 'jmpl' encodes the register inside of the opcode */ 247 if ((insn & 0xc1f80000) == 0x81c00000) { 248 unsigned long rd = ((insn >> 25) & 0x1f); 249 250 if (rd <= 15) { 251 slot = ®s->u_regs[rd]; 252 } else { 253 /* Hard case, it goes onto the stack. */ 254 flushw_all(); 255 256 rd -= 16; 257 slot = (unsigned long *) 258 (regs->u_regs[UREG_FP] + STACK_BIAS); 259 slot += rd; 260 } 261 } 262 if (slot != NULL) 263 *slot = real_pc; 264 } 265 266 /* 267 * Called after single-stepping. p->addr is the address of the 268 * instruction which has been replaced by the breakpoint 269 * instruction. To avoid the SMP problems that can occur when we 270 * temporarily put back the original opcode to single-step, we 271 * single-stepped a copy of the instruction. The address of this 272 * copy is &p->ainsn.insn[0]. 273 * 274 * This function prepares to return from the post-single-step 275 * breakpoint trap. 276 */ 277 static void __kprobes resume_execution(struct kprobe *p, 278 struct pt_regs *regs, struct kprobe_ctlblk *kcb) 279 { 280 u32 insn = p->ainsn.insn[0]; 281 282 regs->tnpc = relbranch_fixup(insn, p, regs); 283 284 /* This assignment must occur after relbranch_fixup() */ 285 regs->tpc = kcb->kprobe_orig_tnpc; 286 287 retpc_fixup(regs, insn, (unsigned long) p->addr); 288 289 regs->tstate = ((regs->tstate & ~TSTATE_PIL) | 290 kcb->kprobe_orig_tstate_pil); 291 } 292 293 static int __kprobes post_kprobe_handler(struct pt_regs *regs) 294 { 295 struct kprobe *cur = kprobe_running(); 296 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 297 298 if (!cur) 299 return 0; 300 301 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { 302 kcb->kprobe_status = KPROBE_HIT_SSDONE; 303 cur->post_handler(cur, regs, 0); 304 } 305 306 resume_execution(cur, regs, kcb); 307 308 /*Restore back the original saved kprobes variables and continue. */ 309 if (kcb->kprobe_status == KPROBE_REENTER) { 310 restore_previous_kprobe(kcb); 311 goto out; 312 } 313 reset_current_kprobe(); 314 out: 315 preempt_enable_no_resched(); 316 317 return 1; 318 } 319 320 int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) 321 { 322 struct kprobe *cur = kprobe_running(); 323 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 324 const struct exception_table_entry *entry; 325 326 switch(kcb->kprobe_status) { 327 case KPROBE_HIT_SS: 328 case KPROBE_REENTER: 329 /* 330 * We are here because the instruction being single 331 * stepped caused a page fault. We reset the current 332 * kprobe and the tpc points back to the probe address 333 * and allow the page fault handler to continue as a 334 * normal page fault. 335 */ 336 regs->tpc = (unsigned long)cur->addr; 337 regs->tnpc = kcb->kprobe_orig_tnpc; 338 regs->tstate = ((regs->tstate & ~TSTATE_PIL) | 339 kcb->kprobe_orig_tstate_pil); 340 if (kcb->kprobe_status == KPROBE_REENTER) 341 restore_previous_kprobe(kcb); 342 else 343 reset_current_kprobe(); 344 preempt_enable_no_resched(); 345 break; 346 case KPROBE_HIT_ACTIVE: 347 case KPROBE_HIT_SSDONE: 348 /* 349 * We increment the nmissed count for accounting, 350 * we can also use npre/npostfault count for accounting 351 * these specific fault cases. 352 */ 353 kprobes_inc_nmissed_count(cur); 354 355 /* 356 * We come here because instructions in the pre/post 357 * handler caused the page_fault, this could happen 358 * if handler tries to access user space by 359 * copy_from_user(), get_user() etc. Let the 360 * user-specified handler try to fix it first. 361 */ 362 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) 363 return 1; 364 365 /* 366 * In case the user-specified fault handler returned 367 * zero, try to fix up. 368 */ 369 370 entry = search_exception_tables(regs->tpc); 371 if (entry) { 372 regs->tpc = entry->fixup; 373 regs->tnpc = regs->tpc + 4; 374 return 1; 375 } 376 377 /* 378 * fixup_exception() could not handle it, 379 * Let do_page_fault() fix it. 380 */ 381 break; 382 default: 383 break; 384 } 385 386 return 0; 387 } 388 389 /* 390 * Wrapper routine to for handling exceptions. 391 */ 392 int __kprobes kprobe_exceptions_notify(struct notifier_block *self, 393 unsigned long val, void *data) 394 { 395 struct die_args *args = (struct die_args *)data; 396 int ret = NOTIFY_DONE; 397 398 if (args->regs && user_mode(args->regs)) 399 return ret; 400 401 switch (val) { 402 case DIE_DEBUG: 403 if (kprobe_handler(args->regs)) 404 ret = NOTIFY_STOP; 405 break; 406 case DIE_DEBUG_2: 407 if (post_kprobe_handler(args->regs)) 408 ret = NOTIFY_STOP; 409 break; 410 default: 411 break; 412 } 413 return ret; 414 } 415 416 asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, 417 struct pt_regs *regs) 418 { 419 enum ctx_state prev_state = exception_enter(); 420 421 BUG_ON(trap_level != 0x170 && trap_level != 0x171); 422 423 if (user_mode(regs)) { 424 local_irq_enable(); 425 bad_trap(regs, trap_level); 426 goto out; 427 } 428 429 /* trap_level == 0x170 --> ta 0x70 430 * trap_level == 0x171 --> ta 0x71 431 */ 432 if (notify_die((trap_level == 0x170) ? DIE_DEBUG : DIE_DEBUG_2, 433 (trap_level == 0x170) ? "debug" : "debug_2", 434 regs, 0, trap_level, SIGTRAP) != NOTIFY_STOP) 435 bad_trap(regs, trap_level); 436 out: 437 exception_exit(prev_state); 438 } 439 440 /* The value stored in the return address register is actually 2 441 * instructions before where the callee will return to. 442 * Sequences usually look something like this 443 * 444 * call some_function <--- return register points here 445 * nop <--- call delay slot 446 * whatever <--- where callee returns to 447 * 448 * To keep trampoline_probe_handler logic simpler, we normalize the 449 * value kept in ri->ret_addr so we don't need to keep adjusting it 450 * back and forth. 451 */ 452 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, 453 struct pt_regs *regs) 454 { 455 ri->ret_addr = (kprobe_opcode_t *)(regs->u_regs[UREG_RETPC] + 8); 456 457 /* Replace the return addr with trampoline addr */ 458 regs->u_regs[UREG_RETPC] = 459 ((unsigned long)kretprobe_trampoline) - 8; 460 } 461 462 /* 463 * Called when the probe at kretprobe trampoline is hit 464 */ 465 static int __kprobes trampoline_probe_handler(struct kprobe *p, 466 struct pt_regs *regs) 467 { 468 struct kretprobe_instance *ri = NULL; 469 struct hlist_head *head, empty_rp; 470 struct hlist_node *tmp; 471 unsigned long flags, orig_ret_address = 0; 472 unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline; 473 474 INIT_HLIST_HEAD(&empty_rp); 475 kretprobe_hash_lock(current, &head, &flags); 476 477 /* 478 * It is possible to have multiple instances associated with a given 479 * task either because an multiple functions in the call path 480 * have a return probe installed on them, and/or more than one return 481 * return probe was registered for a target function. 482 * 483 * We can handle this because: 484 * - instances are always inserted at the head of the list 485 * - when multiple return probes are registered for the same 486 * function, the first instance's ret_addr will point to the 487 * real return address, and all the rest will point to 488 * kretprobe_trampoline 489 */ 490 hlist_for_each_entry_safe(ri, tmp, head, hlist) { 491 if (ri->task != current) 492 /* another task is sharing our hash bucket */ 493 continue; 494 495 if (ri->rp && ri->rp->handler) 496 ri->rp->handler(ri, regs); 497 498 orig_ret_address = (unsigned long)ri->ret_addr; 499 recycle_rp_inst(ri, &empty_rp); 500 501 if (orig_ret_address != trampoline_address) 502 /* 503 * This is the real return address. Any other 504 * instances associated with this task are for 505 * other calls deeper on the call stack 506 */ 507 break; 508 } 509 510 kretprobe_assert(ri, orig_ret_address, trampoline_address); 511 regs->tpc = orig_ret_address; 512 regs->tnpc = orig_ret_address + 4; 513 514 kretprobe_hash_unlock(current, &flags); 515 516 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { 517 hlist_del(&ri->hlist); 518 kfree(ri); 519 } 520 /* 521 * By returning a non-zero value, we are telling 522 * kprobe_handler() that we don't want the post_handler 523 * to run (and have re-enabled preemption) 524 */ 525 return 1; 526 } 527 528 static void __used kretprobe_trampoline_holder(void) 529 { 530 asm volatile(".global kretprobe_trampoline\n" 531 "kretprobe_trampoline:\n" 532 "\tnop\n" 533 "\tnop\n"); 534 } 535 static struct kprobe trampoline_p = { 536 .addr = (kprobe_opcode_t *) &kretprobe_trampoline, 537 .pre_handler = trampoline_probe_handler 538 }; 539 540 int __init arch_init_kprobes(void) 541 { 542 return register_kprobe(&trampoline_p); 543 } 544 545 int __kprobes arch_trampoline_kprobe(struct kprobe *p) 546 { 547 if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline) 548 return 1; 549 550 return 0; 551 } 552