1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, 4 * using the CPU's debug registers. Derived from 5 * "arch/x86/kernel/hw_breakpoint.c" 6 * 7 * Copyright 2010 IBM Corporation 8 * Author: K.Prasad <prasad@linux.vnet.ibm.com> 9 */ 10 11 #include <linux/hw_breakpoint.h> 12 #include <linux/notifier.h> 13 #include <linux/kprobes.h> 14 #include <linux/percpu.h> 15 #include <linux/kernel.h> 16 #include <linux/sched.h> 17 #include <linux/smp.h> 18 #include <linux/spinlock.h> 19 #include <linux/debugfs.h> 20 #include <linux/init.h> 21 22 #include <asm/hw_breakpoint.h> 23 #include <asm/processor.h> 24 #include <asm/sstep.h> 25 #include <asm/debug.h> 26 #include <asm/hvcall.h> 27 #include <asm/inst.h> 28 #include <linux/uaccess.h> 29 30 /* 31 * Stores the breakpoints currently in use on each breakpoint address 32 * register for every cpu 33 */ 34 static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM_MAX]); 35 36 /* 37 * Returns total number of data or instruction breakpoints available. 38 */ 39 int hw_breakpoint_slots(int type) 40 { 41 if (type == TYPE_DATA) 42 return nr_wp_slots(); 43 return 0; /* no instruction breakpoints available */ 44 } 45 46 47 /* 48 * Install a perf counter breakpoint. 49 * 50 * We seek a free debug address register and use it for this 51 * breakpoint. 52 * 53 * Atomic: we hold the counter->ctx->lock and we only handle variables 54 * and registers local to this cpu. 55 */ 56 int arch_install_hw_breakpoint(struct perf_event *bp) 57 { 58 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 59 struct perf_event **slot; 60 int i; 61 62 for (i = 0; i < nr_wp_slots(); i++) { 63 slot = this_cpu_ptr(&bp_per_reg[i]); 64 if (!*slot) { 65 *slot = bp; 66 break; 67 } 68 } 69 70 if (WARN_ONCE(i == nr_wp_slots(), "Can't find any breakpoint slot")) 71 return -EBUSY; 72 73 /* 74 * Do not install DABR values if the instruction must be single-stepped. 75 * If so, DABR will be populated in single_step_dabr_instruction(). 76 */ 77 if (!info->perf_single_step) 78 __set_breakpoint(i, info); 79 80 return 0; 81 } 82 83 /* 84 * Uninstall the breakpoint contained in the given counter. 85 * 86 * First we search the debug address register it uses and then we disable 87 * it. 88 * 89 * Atomic: we hold the counter->ctx->lock and we only handle variables 90 * and registers local to this cpu. 91 */ 92 void arch_uninstall_hw_breakpoint(struct perf_event *bp) 93 { 94 struct arch_hw_breakpoint null_brk = {0}; 95 struct perf_event **slot; 96 int i; 97 98 for (i = 0; i < nr_wp_slots(); i++) { 99 slot = this_cpu_ptr(&bp_per_reg[i]); 100 if (*slot == bp) { 101 *slot = NULL; 102 break; 103 } 104 } 105 106 if (WARN_ONCE(i == nr_wp_slots(), "Can't find any breakpoint slot")) 107 return; 108 109 __set_breakpoint(i, &null_brk); 110 } 111 112 static bool is_ptrace_bp(struct perf_event *bp) 113 { 114 return bp->overflow_handler == ptrace_triggered; 115 } 116 117 /* 118 * Check for virtual address in kernel space. 119 */ 120 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) 121 { 122 return is_kernel_addr(hw->address); 123 } 124 125 int arch_bp_generic_fields(int type, int *gen_bp_type) 126 { 127 *gen_bp_type = 0; 128 if (type & HW_BRK_TYPE_READ) 129 *gen_bp_type |= HW_BREAKPOINT_R; 130 if (type & HW_BRK_TYPE_WRITE) 131 *gen_bp_type |= HW_BREAKPOINT_W; 132 if (*gen_bp_type == 0) 133 return -EINVAL; 134 return 0; 135 } 136 137 /* 138 * Watchpoint match range is always doubleword(8 bytes) aligned on 139 * powerpc. If the given range is crossing doubleword boundary, we 140 * need to increase the length such that next doubleword also get 141 * covered. Ex, 142 * 143 * address len = 6 bytes 144 * |=========. 145 * |------------v--|------v--------| 146 * | | | | | | | | | | | | | | | | | 147 * |---------------|---------------| 148 * <---8 bytes---> 149 * 150 * In this case, we should configure hw as: 151 * start_addr = address & ~(HW_BREAKPOINT_SIZE - 1) 152 * len = 16 bytes 153 * 154 * @start_addr is inclusive but @end_addr is exclusive. 155 */ 156 static int hw_breakpoint_validate_len(struct arch_hw_breakpoint *hw) 157 { 158 u16 max_len = DABR_MAX_LEN; 159 u16 hw_len; 160 unsigned long start_addr, end_addr; 161 162 start_addr = ALIGN_DOWN(hw->address, HW_BREAKPOINT_SIZE); 163 end_addr = ALIGN(hw->address + hw->len, HW_BREAKPOINT_SIZE); 164 hw_len = end_addr - start_addr; 165 166 if (dawr_enabled()) { 167 max_len = DAWR_MAX_LEN; 168 /* DAWR region can't cross 512 bytes boundary on p10 predecessors */ 169 if (!cpu_has_feature(CPU_FTR_ARCH_31) && 170 (ALIGN_DOWN(start_addr, SZ_512) != ALIGN_DOWN(end_addr - 1, SZ_512))) 171 return -EINVAL; 172 } else if (IS_ENABLED(CONFIG_PPC_8xx)) { 173 /* 8xx can setup a range without limitation */ 174 max_len = U16_MAX; 175 } 176 177 if (hw_len > max_len) 178 return -EINVAL; 179 180 hw->hw_len = hw_len; 181 return 0; 182 } 183 184 /* 185 * Validate the arch-specific HW Breakpoint register settings 186 */ 187 int hw_breakpoint_arch_parse(struct perf_event *bp, 188 const struct perf_event_attr *attr, 189 struct arch_hw_breakpoint *hw) 190 { 191 int ret = -EINVAL; 192 193 if (!bp || !attr->bp_len) 194 return ret; 195 196 hw->type = HW_BRK_TYPE_TRANSLATE; 197 if (attr->bp_type & HW_BREAKPOINT_R) 198 hw->type |= HW_BRK_TYPE_READ; 199 if (attr->bp_type & HW_BREAKPOINT_W) 200 hw->type |= HW_BRK_TYPE_WRITE; 201 if (hw->type == HW_BRK_TYPE_TRANSLATE) 202 /* must set alteast read or write */ 203 return ret; 204 if (!attr->exclude_user) 205 hw->type |= HW_BRK_TYPE_USER; 206 if (!attr->exclude_kernel) 207 hw->type |= HW_BRK_TYPE_KERNEL; 208 if (!attr->exclude_hv) 209 hw->type |= HW_BRK_TYPE_HYP; 210 hw->address = attr->bp_addr; 211 hw->len = attr->bp_len; 212 213 if (!ppc_breakpoint_available()) 214 return -ENODEV; 215 216 return hw_breakpoint_validate_len(hw); 217 } 218 219 /* 220 * Restores the breakpoint on the debug registers. 221 * Invoke this function if it is known that the execution context is 222 * about to change to cause loss of MSR_SE settings. 223 * 224 * The perf watchpoint will simply re-trigger once the thread is started again, 225 * and the watchpoint handler will set up MSR_SE and perf_single_step as 226 * needed. 227 */ 228 void thread_change_pc(struct task_struct *tsk, struct pt_regs *regs) 229 { 230 struct arch_hw_breakpoint *info; 231 int i; 232 233 preempt_disable(); 234 235 for (i = 0; i < nr_wp_slots(); i++) { 236 struct perf_event *bp = __this_cpu_read(bp_per_reg[i]); 237 238 if (unlikely(bp && counter_arch_bp(bp)->perf_single_step)) 239 goto reset; 240 } 241 goto out; 242 243 reset: 244 regs_set_return_msr(regs, regs->msr & ~MSR_SE); 245 for (i = 0; i < nr_wp_slots(); i++) { 246 info = counter_arch_bp(__this_cpu_read(bp_per_reg[i])); 247 __set_breakpoint(i, info); 248 info->perf_single_step = false; 249 } 250 251 out: 252 preempt_enable(); 253 } 254 255 static bool is_larx_stcx_instr(int type) 256 { 257 return type == LARX || type == STCX; 258 } 259 260 static bool is_octword_vsx_instr(int type, int size) 261 { 262 return ((type == LOAD_VSX || type == STORE_VSX) && size == 32); 263 } 264 265 /* 266 * We've failed in reliably handling the hw-breakpoint. Unregister 267 * it and throw a warning message to let the user know about it. 268 */ 269 static void handler_error(struct perf_event *bp) 270 { 271 WARN(1, "Unable to handle hardware breakpoint. Breakpoint at 0x%lx will be disabled.", 272 counter_arch_bp(bp)->address); 273 perf_event_disable_inatomic(bp); 274 } 275 276 static void larx_stcx_err(struct perf_event *bp) 277 { 278 printk_ratelimited("Breakpoint hit on instruction that can't be emulated. Breakpoint at 0x%lx will be disabled.\n", 279 counter_arch_bp(bp)->address); 280 perf_event_disable_inatomic(bp); 281 } 282 283 static bool stepping_handler(struct pt_regs *regs, struct perf_event **bp, 284 int *hit, ppc_inst_t instr) 285 { 286 int i; 287 int stepped; 288 289 /* Do not emulate user-space instructions, instead single-step them */ 290 if (user_mode(regs)) { 291 for (i = 0; i < nr_wp_slots(); i++) { 292 if (!hit[i]) 293 continue; 294 295 counter_arch_bp(bp[i])->perf_single_step = true; 296 bp[i] = NULL; 297 } 298 regs_set_return_msr(regs, regs->msr | MSR_SE); 299 return false; 300 } 301 302 stepped = emulate_step(regs, instr); 303 if (!stepped) { 304 for (i = 0; i < nr_wp_slots(); i++) { 305 if (!hit[i]) 306 continue; 307 handler_error(bp[i]); 308 bp[i] = NULL; 309 } 310 return false; 311 } 312 return true; 313 } 314 315 static void handle_p10dd1_spurious_exception(struct perf_event **bp, 316 int *hit, unsigned long ea) 317 { 318 int i; 319 unsigned long hw_end_addr; 320 321 /* 322 * Handle spurious exception only when any bp_per_reg is set. 323 * Otherwise this might be created by xmon and not actually a 324 * spurious exception. 325 */ 326 for (i = 0; i < nr_wp_slots(); i++) { 327 struct arch_hw_breakpoint *info; 328 329 if (!bp[i]) 330 continue; 331 332 info = counter_arch_bp(bp[i]); 333 334 hw_end_addr = ALIGN(info->address + info->len, HW_BREAKPOINT_SIZE); 335 336 /* 337 * Ending address of DAWR range is less than starting 338 * address of op. 339 */ 340 if ((hw_end_addr - 1) >= ea) 341 continue; 342 343 /* 344 * Those addresses need to be in the same or in two 345 * consecutive 512B blocks; 346 */ 347 if (((hw_end_addr - 1) >> 10) != (ea >> 10)) 348 continue; 349 350 /* 351 * 'op address + 64B' generates an address that has a 352 * carry into bit 52 (crosses 2K boundary). 353 */ 354 if ((ea & 0x800) == ((ea + 64) & 0x800)) 355 continue; 356 357 break; 358 } 359 360 if (i == nr_wp_slots()) 361 return; 362 363 for (i = 0; i < nr_wp_slots(); i++) { 364 if (bp[i]) { 365 hit[i] = 1; 366 counter_arch_bp(bp[i])->type |= HW_BRK_TYPE_EXTRANEOUS_IRQ; 367 } 368 } 369 } 370 371 /* 372 * Handle a DABR or DAWR exception. 373 * 374 * Called in atomic context. 375 */ 376 int hw_breakpoint_handler(struct die_args *args) 377 { 378 bool err = false; 379 int rc = NOTIFY_STOP; 380 struct perf_event *bp[HBP_NUM_MAX] = { NULL }; 381 struct pt_regs *regs = args->regs; 382 int i; 383 int hit[HBP_NUM_MAX] = {0}; 384 int nr_hit = 0; 385 bool ptrace_bp = false; 386 ppc_inst_t instr = ppc_inst(0); 387 int type = 0; 388 int size = 0; 389 unsigned long ea = 0; 390 391 /* Disable breakpoints during exception handling */ 392 hw_breakpoint_disable(); 393 394 /* 395 * The counter may be concurrently released but that can only 396 * occur from a call_rcu() path. We can then safely fetch 397 * the breakpoint, use its callback, touch its counter 398 * while we are in an rcu_read_lock() path. 399 */ 400 rcu_read_lock(); 401 402 if (!IS_ENABLED(CONFIG_PPC_8xx)) 403 wp_get_instr_detail(regs, &instr, &type, &size, &ea); 404 405 for (i = 0; i < nr_wp_slots(); i++) { 406 struct arch_hw_breakpoint *info; 407 408 bp[i] = __this_cpu_read(bp_per_reg[i]); 409 if (!bp[i]) 410 continue; 411 412 info = counter_arch_bp(bp[i]); 413 info->type &= ~HW_BRK_TYPE_EXTRANEOUS_IRQ; 414 415 if (wp_check_constraints(regs, instr, ea, type, size, info)) { 416 if (!IS_ENABLED(CONFIG_PPC_8xx) && 417 ppc_inst_equal(instr, ppc_inst(0))) { 418 handler_error(bp[i]); 419 bp[i] = NULL; 420 err = 1; 421 continue; 422 } 423 424 if (is_ptrace_bp(bp[i])) 425 ptrace_bp = true; 426 hit[i] = 1; 427 nr_hit++; 428 } 429 } 430 431 if (err) 432 goto reset; 433 434 if (!nr_hit) { 435 /* Workaround for Power10 DD1 */ 436 if (!IS_ENABLED(CONFIG_PPC_8xx) && mfspr(SPRN_PVR) == 0x800100 && 437 is_octword_vsx_instr(type, size)) { 438 handle_p10dd1_spurious_exception(bp, hit, ea); 439 } else { 440 rc = NOTIFY_DONE; 441 goto out; 442 } 443 } 444 445 /* 446 * Return early after invoking user-callback function without restoring 447 * DABR if the breakpoint is from ptrace which always operates in 448 * one-shot mode. The ptrace-ed process will receive the SIGTRAP signal 449 * generated in do_dabr(). 450 */ 451 if (ptrace_bp) { 452 for (i = 0; i < nr_wp_slots(); i++) { 453 if (!hit[i] || !is_ptrace_bp(bp[i])) 454 continue; 455 perf_bp_event(bp[i], regs); 456 bp[i] = NULL; 457 } 458 rc = NOTIFY_DONE; 459 goto reset; 460 } 461 462 if (!IS_ENABLED(CONFIG_PPC_8xx)) { 463 if (is_larx_stcx_instr(type)) { 464 for (i = 0; i < nr_wp_slots(); i++) { 465 if (!hit[i]) 466 continue; 467 larx_stcx_err(bp[i]); 468 bp[i] = NULL; 469 } 470 goto reset; 471 } 472 473 if (!stepping_handler(regs, bp, hit, instr)) 474 goto reset; 475 } 476 477 /* 478 * As a policy, the callback is invoked in a 'trigger-after-execute' 479 * fashion 480 */ 481 for (i = 0; i < nr_wp_slots(); i++) { 482 if (!hit[i]) 483 continue; 484 if (!(counter_arch_bp(bp[i])->type & HW_BRK_TYPE_EXTRANEOUS_IRQ)) 485 perf_bp_event(bp[i], regs); 486 } 487 488 reset: 489 for (i = 0; i < nr_wp_slots(); i++) { 490 if (!bp[i]) 491 continue; 492 __set_breakpoint(i, counter_arch_bp(bp[i])); 493 } 494 495 out: 496 rcu_read_unlock(); 497 return rc; 498 } 499 NOKPROBE_SYMBOL(hw_breakpoint_handler); 500 501 /* 502 * Handle single-step exceptions following a DABR hit. 503 * 504 * Called in atomic context. 505 */ 506 static int single_step_dabr_instruction(struct die_args *args) 507 { 508 struct pt_regs *regs = args->regs; 509 bool found = false; 510 511 /* 512 * Check if we are single-stepping as a result of a 513 * previous HW Breakpoint exception 514 */ 515 for (int i = 0; i < nr_wp_slots(); i++) { 516 struct perf_event *bp; 517 struct arch_hw_breakpoint *info; 518 519 bp = __this_cpu_read(bp_per_reg[i]); 520 521 if (!bp) 522 continue; 523 524 info = counter_arch_bp(bp); 525 526 if (!info->perf_single_step) 527 continue; 528 529 found = true; 530 531 /* 532 * We shall invoke the user-defined callback function in the 533 * single stepping handler to confirm to 'trigger-after-execute' 534 * semantics 535 */ 536 if (!(info->type & HW_BRK_TYPE_EXTRANEOUS_IRQ)) 537 perf_bp_event(bp, regs); 538 539 info->perf_single_step = false; 540 __set_breakpoint(i, counter_arch_bp(bp)); 541 } 542 543 /* 544 * If the process was being single-stepped by ptrace, let the 545 * other single-step actions occur (e.g. generate SIGTRAP). 546 */ 547 if (!found || test_thread_flag(TIF_SINGLESTEP)) 548 return NOTIFY_DONE; 549 550 return NOTIFY_STOP; 551 } 552 NOKPROBE_SYMBOL(single_step_dabr_instruction); 553 554 /* 555 * Handle debug exception notifications. 556 * 557 * Called in atomic context. 558 */ 559 int hw_breakpoint_exceptions_notify( 560 struct notifier_block *unused, unsigned long val, void *data) 561 { 562 int ret = NOTIFY_DONE; 563 564 switch (val) { 565 case DIE_DABR_MATCH: 566 ret = hw_breakpoint_handler(data); 567 break; 568 case DIE_SSTEP: 569 ret = single_step_dabr_instruction(data); 570 break; 571 } 572 573 return ret; 574 } 575 NOKPROBE_SYMBOL(hw_breakpoint_exceptions_notify); 576 577 /* 578 * Release the user breakpoints used by ptrace 579 */ 580 void flush_ptrace_hw_breakpoint(struct task_struct *tsk) 581 { 582 int i; 583 struct thread_struct *t = &tsk->thread; 584 585 for (i = 0; i < nr_wp_slots(); i++) { 586 unregister_hw_breakpoint(t->ptrace_bps[i]); 587 t->ptrace_bps[i] = NULL; 588 } 589 } 590 591 void hw_breakpoint_pmu_read(struct perf_event *bp) 592 { 593 /* TODO */ 594 } 595 596 void ptrace_triggered(struct perf_event *bp, 597 struct perf_sample_data *data, struct pt_regs *regs) 598 { 599 struct perf_event_attr attr; 600 601 /* 602 * Disable the breakpoint request here since ptrace has defined a 603 * one-shot behaviour for breakpoint exceptions in PPC64. 604 * The SIGTRAP signal is generated automatically for us in do_dabr(). 605 * We don't have to do anything about that here 606 */ 607 attr = bp->attr; 608 attr.disabled = true; 609 modify_user_hw_breakpoint(bp, &attr); 610 } 611