1 /* 2 * arch/arm64/kernel/probes/kprobes.c 3 * 4 * Kprobes support for ARM64 5 * 6 * Copyright (C) 2013 Linaro Limited. 7 * Author: Sandeepa Prabhu <sandeepa.prabhu@linaro.org> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 */ 19 #include <linux/kasan.h> 20 #include <linux/kernel.h> 21 #include <linux/kprobes.h> 22 #include <linux/extable.h> 23 #include <linux/slab.h> 24 #include <linux/stop_machine.h> 25 #include <linux/sched/debug.h> 26 #include <linux/set_memory.h> 27 #include <linux/stringify.h> 28 #include <linux/vmalloc.h> 29 #include <asm/traps.h> 30 #include <asm/ptrace.h> 31 #include <asm/cacheflush.h> 32 #include <asm/debug-monitors.h> 33 #include <asm/system_misc.h> 34 #include <asm/insn.h> 35 #include <linux/uaccess.h> 36 #include <asm/irq.h> 37 #include <asm/sections.h> 38 39 #include "decode-insn.h" 40 41 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; 42 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 43 44 static void __kprobes 45 post_kprobe_handler(struct kprobe_ctlblk *, struct pt_regs *); 46 47 static int __kprobes patch_text(kprobe_opcode_t *addr, u32 opcode) 48 { 49 void *addrs[1]; 50 u32 insns[1]; 51 52 addrs[0] = addr; 53 insns[0] = opcode; 54 55 return aarch64_insn_patch_text(addrs, insns, 1); 56 } 57 58 static void __kprobes arch_prepare_ss_slot(struct kprobe *p) 59 { 60 /* prepare insn slot */ 61 patch_text(p->ainsn.api.insn, p->opcode); 62 63 flush_icache_range((uintptr_t) (p->ainsn.api.insn), 64 (uintptr_t) (p->ainsn.api.insn) + 65 MAX_INSN_SIZE * sizeof(kprobe_opcode_t)); 66 67 /* 68 * Needs restoring of return address after stepping xol. 69 */ 70 p->ainsn.api.restore = (unsigned long) p->addr + 71 sizeof(kprobe_opcode_t); 72 } 73 74 static void __kprobes arch_prepare_simulate(struct kprobe *p) 75 { 76 /* This instructions is not executed xol. No need to adjust the PC */ 77 p->ainsn.api.restore = 0; 78 } 79 80 static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs) 81 { 82 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 83 84 if (p->ainsn.api.handler) 85 p->ainsn.api.handler((u32)p->opcode, (long)p->addr, regs); 86 87 /* single step simulated, now go for post processing */ 88 post_kprobe_handler(kcb, regs); 89 } 90 91 int __kprobes arch_prepare_kprobe(struct kprobe *p) 92 { 93 unsigned long probe_addr = (unsigned long)p->addr; 94 extern char __start_rodata[]; 95 extern char __end_rodata[]; 96 97 if (probe_addr & 0x3) 98 return -EINVAL; 99 100 /* copy instruction */ 101 p->opcode = le32_to_cpu(*p->addr); 102 103 if (in_exception_text(probe_addr)) 104 return -EINVAL; 105 if (probe_addr >= (unsigned long) __start_rodata && 106 probe_addr <= (unsigned long) __end_rodata) 107 return -EINVAL; 108 109 /* decode instruction */ 110 switch (arm_kprobe_decode_insn(p->addr, &p->ainsn)) { 111 case INSN_REJECTED: /* insn not supported */ 112 return -EINVAL; 113 114 case INSN_GOOD_NO_SLOT: /* insn need simulation */ 115 p->ainsn.api.insn = NULL; 116 break; 117 118 case INSN_GOOD: /* instruction uses slot */ 119 p->ainsn.api.insn = get_insn_slot(); 120 if (!p->ainsn.api.insn) 121 return -ENOMEM; 122 break; 123 } 124 125 /* prepare the instruction */ 126 if (p->ainsn.api.insn) 127 arch_prepare_ss_slot(p); 128 else 129 arch_prepare_simulate(p); 130 131 return 0; 132 } 133 134 void *alloc_insn_page(void) 135 { 136 void *page; 137 138 page = vmalloc_exec(PAGE_SIZE); 139 if (page) 140 set_memory_ro((unsigned long)page, 1); 141 142 return page; 143 } 144 145 /* arm kprobe: install breakpoint in text */ 146 void __kprobes arch_arm_kprobe(struct kprobe *p) 147 { 148 patch_text(p->addr, BRK64_OPCODE_KPROBES); 149 } 150 151 /* disarm kprobe: remove breakpoint from text */ 152 void __kprobes arch_disarm_kprobe(struct kprobe *p) 153 { 154 patch_text(p->addr, p->opcode); 155 } 156 157 void __kprobes arch_remove_kprobe(struct kprobe *p) 158 { 159 if (p->ainsn.api.insn) { 160 free_insn_slot(p->ainsn.api.insn, 0); 161 p->ainsn.api.insn = NULL; 162 } 163 } 164 165 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) 166 { 167 kcb->prev_kprobe.kp = kprobe_running(); 168 kcb->prev_kprobe.status = kcb->kprobe_status; 169 } 170 171 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) 172 { 173 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); 174 kcb->kprobe_status = kcb->prev_kprobe.status; 175 } 176 177 static void __kprobes set_current_kprobe(struct kprobe *p) 178 { 179 __this_cpu_write(current_kprobe, p); 180 } 181 182 /* 183 * When PSTATE.D is set (masked), then software step exceptions can not be 184 * generated. 185 * SPSR's D bit shows the value of PSTATE.D immediately before the 186 * exception was taken. PSTATE.D is set while entering into any exception 187 * mode, however software clears it for any normal (none-debug-exception) 188 * mode in the exception entry. Therefore, when we are entering into kprobe 189 * breakpoint handler from any normal mode then SPSR.D bit is already 190 * cleared, however it is set when we are entering from any debug exception 191 * mode. 192 * Since we always need to generate single step exception after a kprobe 193 * breakpoint exception therefore we need to clear it unconditionally, when 194 * we become sure that the current breakpoint exception is for kprobe. 195 */ 196 static void __kprobes 197 spsr_set_debug_flag(struct pt_regs *regs, int mask) 198 { 199 unsigned long spsr = regs->pstate; 200 201 if (mask) 202 spsr |= PSR_D_BIT; 203 else 204 spsr &= ~PSR_D_BIT; 205 206 regs->pstate = spsr; 207 } 208 209 /* 210 * Interrupts need to be disabled before single-step mode is set, and not 211 * reenabled until after single-step mode ends. 212 * Without disabling interrupt on local CPU, there is a chance of 213 * interrupt occurrence in the period of exception return and start of 214 * out-of-line single-step, that result in wrongly single stepping 215 * into the interrupt handler. 216 */ 217 static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb, 218 struct pt_regs *regs) 219 { 220 kcb->saved_irqflag = regs->pstate; 221 regs->pstate |= PSR_I_BIT; 222 } 223 224 static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb, 225 struct pt_regs *regs) 226 { 227 if (kcb->saved_irqflag & PSR_I_BIT) 228 regs->pstate |= PSR_I_BIT; 229 else 230 regs->pstate &= ~PSR_I_BIT; 231 } 232 233 static void __kprobes 234 set_ss_context(struct kprobe_ctlblk *kcb, unsigned long addr) 235 { 236 kcb->ss_ctx.ss_pending = true; 237 kcb->ss_ctx.match_addr = addr + sizeof(kprobe_opcode_t); 238 } 239 240 static void __kprobes clear_ss_context(struct kprobe_ctlblk *kcb) 241 { 242 kcb->ss_ctx.ss_pending = false; 243 kcb->ss_ctx.match_addr = 0; 244 } 245 246 static void __kprobes setup_singlestep(struct kprobe *p, 247 struct pt_regs *regs, 248 struct kprobe_ctlblk *kcb, int reenter) 249 { 250 unsigned long slot; 251 252 if (reenter) { 253 save_previous_kprobe(kcb); 254 set_current_kprobe(p); 255 kcb->kprobe_status = KPROBE_REENTER; 256 } else { 257 kcb->kprobe_status = KPROBE_HIT_SS; 258 } 259 260 261 if (p->ainsn.api.insn) { 262 /* prepare for single stepping */ 263 slot = (unsigned long)p->ainsn.api.insn; 264 265 set_ss_context(kcb, slot); /* mark pending ss */ 266 267 spsr_set_debug_flag(regs, 0); 268 269 /* IRQs and single stepping do not mix well. */ 270 kprobes_save_local_irqflag(kcb, regs); 271 kernel_enable_single_step(regs); 272 instruction_pointer_set(regs, slot); 273 } else { 274 /* insn simulation */ 275 arch_simulate_insn(p, regs); 276 } 277 } 278 279 static int __kprobes reenter_kprobe(struct kprobe *p, 280 struct pt_regs *regs, 281 struct kprobe_ctlblk *kcb) 282 { 283 switch (kcb->kprobe_status) { 284 case KPROBE_HIT_SSDONE: 285 case KPROBE_HIT_ACTIVE: 286 kprobes_inc_nmissed_count(p); 287 setup_singlestep(p, regs, kcb, 1); 288 break; 289 case KPROBE_HIT_SS: 290 case KPROBE_REENTER: 291 pr_warn("Unrecoverable kprobe detected.\n"); 292 dump_kprobe(p); 293 BUG(); 294 break; 295 default: 296 WARN_ON(1); 297 return 0; 298 } 299 300 return 1; 301 } 302 303 static void __kprobes 304 post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs) 305 { 306 struct kprobe *cur = kprobe_running(); 307 308 if (!cur) 309 return; 310 311 /* return addr restore if non-branching insn */ 312 if (cur->ainsn.api.restore != 0) 313 instruction_pointer_set(regs, cur->ainsn.api.restore); 314 315 /* restore back original saved kprobe variables and continue */ 316 if (kcb->kprobe_status == KPROBE_REENTER) { 317 restore_previous_kprobe(kcb); 318 return; 319 } 320 /* call post handler */ 321 kcb->kprobe_status = KPROBE_HIT_SSDONE; 322 if (cur->post_handler) { 323 /* post_handler can hit breakpoint and single step 324 * again, so we enable D-flag for recursive exception. 325 */ 326 cur->post_handler(cur, regs, 0); 327 } 328 329 reset_current_kprobe(); 330 } 331 332 int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr) 333 { 334 struct kprobe *cur = kprobe_running(); 335 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 336 337 switch (kcb->kprobe_status) { 338 case KPROBE_HIT_SS: 339 case KPROBE_REENTER: 340 /* 341 * We are here because the instruction being single 342 * stepped caused a page fault. We reset the current 343 * kprobe and the ip points back to the probe address 344 * and allow the page fault handler to continue as a 345 * normal page fault. 346 */ 347 instruction_pointer_set(regs, (unsigned long) cur->addr); 348 if (!instruction_pointer(regs)) 349 BUG(); 350 351 kernel_disable_single_step(); 352 353 if (kcb->kprobe_status == KPROBE_REENTER) 354 restore_previous_kprobe(kcb); 355 else 356 reset_current_kprobe(); 357 358 break; 359 case KPROBE_HIT_ACTIVE: 360 case KPROBE_HIT_SSDONE: 361 /* 362 * We increment the nmissed count for accounting, 363 * we can also use npre/npostfault count for accounting 364 * these specific fault cases. 365 */ 366 kprobes_inc_nmissed_count(cur); 367 368 /* 369 * We come here because instructions in the pre/post 370 * handler caused the page_fault, this could happen 371 * if handler tries to access user space by 372 * copy_from_user(), get_user() etc. Let the 373 * user-specified handler try to fix it first. 374 */ 375 if (cur->fault_handler && cur->fault_handler(cur, regs, fsr)) 376 return 1; 377 378 /* 379 * In case the user-specified fault handler returned 380 * zero, try to fix up. 381 */ 382 if (fixup_exception(regs)) 383 return 1; 384 } 385 return 0; 386 } 387 388 static void __kprobes kprobe_handler(struct pt_regs *regs) 389 { 390 struct kprobe *p, *cur_kprobe; 391 struct kprobe_ctlblk *kcb; 392 unsigned long addr = instruction_pointer(regs); 393 394 kcb = get_kprobe_ctlblk(); 395 cur_kprobe = kprobe_running(); 396 397 p = get_kprobe((kprobe_opcode_t *) addr); 398 399 if (p) { 400 if (cur_kprobe) { 401 if (reenter_kprobe(p, regs, kcb)) 402 return; 403 } else { 404 /* Probe hit */ 405 set_current_kprobe(p); 406 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 407 408 /* 409 * If we have no pre-handler or it returned 0, we 410 * continue with normal processing. If we have a 411 * pre-handler and it returned non-zero, it will 412 * modify the execution path and no need to single 413 * stepping. Let's just reset current kprobe and exit. 414 * 415 * pre_handler can hit a breakpoint and can step thru 416 * before return, keep PSTATE D-flag enabled until 417 * pre_handler return back. 418 */ 419 if (!p->pre_handler || !p->pre_handler(p, regs)) { 420 setup_singlestep(p, regs, kcb, 0); 421 } else 422 reset_current_kprobe(); 423 } 424 } 425 /* 426 * The breakpoint instruction was removed right 427 * after we hit it. Another cpu has removed 428 * either a probepoint or a debugger breakpoint 429 * at this address. In either case, no further 430 * handling of this interrupt is appropriate. 431 * Return back to original instruction, and continue. 432 */ 433 } 434 435 static int __kprobes 436 kprobe_ss_hit(struct kprobe_ctlblk *kcb, unsigned long addr) 437 { 438 if ((kcb->ss_ctx.ss_pending) 439 && (kcb->ss_ctx.match_addr == addr)) { 440 clear_ss_context(kcb); /* clear pending ss */ 441 return DBG_HOOK_HANDLED; 442 } 443 /* not ours, kprobes should ignore it */ 444 return DBG_HOOK_ERROR; 445 } 446 447 int __kprobes 448 kprobe_single_step_handler(struct pt_regs *regs, unsigned int esr) 449 { 450 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 451 int retval; 452 453 /* return error if this is not our step */ 454 retval = kprobe_ss_hit(kcb, instruction_pointer(regs)); 455 456 if (retval == DBG_HOOK_HANDLED) { 457 kprobes_restore_local_irqflag(kcb, regs); 458 kernel_disable_single_step(); 459 460 post_kprobe_handler(kcb, regs); 461 } 462 463 return retval; 464 } 465 466 int __kprobes 467 kprobe_breakpoint_handler(struct pt_regs *regs, unsigned int esr) 468 { 469 kprobe_handler(regs); 470 return DBG_HOOK_HANDLED; 471 } 472 473 bool arch_within_kprobe_blacklist(unsigned long addr) 474 { 475 if ((addr >= (unsigned long)__kprobes_text_start && 476 addr < (unsigned long)__kprobes_text_end) || 477 (addr >= (unsigned long)__entry_text_start && 478 addr < (unsigned long)__entry_text_end) || 479 (addr >= (unsigned long)__idmap_text_start && 480 addr < (unsigned long)__idmap_text_end) || 481 !!search_exception_tables(addr)) 482 return true; 483 484 if (!is_kernel_in_hyp_mode()) { 485 if ((addr >= (unsigned long)__hyp_text_start && 486 addr < (unsigned long)__hyp_text_end) || 487 (addr >= (unsigned long)__hyp_idmap_text_start && 488 addr < (unsigned long)__hyp_idmap_text_end)) 489 return true; 490 } 491 492 return false; 493 } 494 495 void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs) 496 { 497 struct kretprobe_instance *ri = NULL; 498 struct hlist_head *head, empty_rp; 499 struct hlist_node *tmp; 500 unsigned long flags, orig_ret_address = 0; 501 unsigned long trampoline_address = 502 (unsigned long)&kretprobe_trampoline; 503 kprobe_opcode_t *correct_ret_addr = NULL; 504 505 INIT_HLIST_HEAD(&empty_rp); 506 kretprobe_hash_lock(current, &head, &flags); 507 508 /* 509 * It is possible to have multiple instances associated with a given 510 * task either because multiple functions in the call path have 511 * return probes installed on them, and/or more than one 512 * return probe was registered for a target function. 513 * 514 * We can handle this because: 515 * - instances are always pushed into the head of the list 516 * - when multiple return probes are registered for the same 517 * function, the (chronologically) first instance's ret_addr 518 * will be the real return address, and all the rest will 519 * point to kretprobe_trampoline. 520 */ 521 hlist_for_each_entry_safe(ri, tmp, head, hlist) { 522 if (ri->task != current) 523 /* another task is sharing our hash bucket */ 524 continue; 525 526 orig_ret_address = (unsigned long)ri->ret_addr; 527 528 if (orig_ret_address != trampoline_address) 529 /* 530 * This is the real return address. Any other 531 * instances associated with this task are for 532 * other calls deeper on the call stack 533 */ 534 break; 535 } 536 537 kretprobe_assert(ri, orig_ret_address, trampoline_address); 538 539 correct_ret_addr = ri->ret_addr; 540 hlist_for_each_entry_safe(ri, tmp, head, hlist) { 541 if (ri->task != current) 542 /* another task is sharing our hash bucket */ 543 continue; 544 545 orig_ret_address = (unsigned long)ri->ret_addr; 546 if (ri->rp && ri->rp->handler) { 547 __this_cpu_write(current_kprobe, &ri->rp->kp); 548 get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE; 549 ri->ret_addr = correct_ret_addr; 550 ri->rp->handler(ri, regs); 551 __this_cpu_write(current_kprobe, NULL); 552 } 553 554 recycle_rp_inst(ri, &empty_rp); 555 556 if (orig_ret_address != trampoline_address) 557 /* 558 * This is the real return address. Any other 559 * instances associated with this task are for 560 * other calls deeper on the call stack 561 */ 562 break; 563 } 564 565 kretprobe_hash_unlock(current, &flags); 566 567 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { 568 hlist_del(&ri->hlist); 569 kfree(ri); 570 } 571 return (void *)orig_ret_address; 572 } 573 574 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, 575 struct pt_regs *regs) 576 { 577 ri->ret_addr = (kprobe_opcode_t *)regs->regs[30]; 578 579 /* replace return addr (x30) with trampoline */ 580 regs->regs[30] = (long)&kretprobe_trampoline; 581 } 582 583 int __kprobes arch_trampoline_kprobe(struct kprobe *p) 584 { 585 return 0; 586 } 587 588 int __init arch_init_kprobes(void) 589 { 590 return 0; 591 } 592