1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, 4 * using the CPU's debug registers. Derived from 5 * "arch/x86/kernel/hw_breakpoint.c" 6 * 7 * Copyright 2010 IBM Corporation 8 * Author: K.Prasad <prasad@linux.vnet.ibm.com> 9 */ 10 11 #include <linux/hw_breakpoint.h> 12 #include <linux/notifier.h> 13 #include <linux/kprobes.h> 14 #include <linux/percpu.h> 15 #include <linux/kernel.h> 16 #include <linux/sched.h> 17 #include <linux/smp.h> 18 #include <linux/debugfs.h> 19 #include <linux/init.h> 20 21 #include <asm/hw_breakpoint.h> 22 #include <asm/processor.h> 23 #include <asm/sstep.h> 24 #include <asm/debug.h> 25 #include <asm/debugfs.h> 26 #include <asm/hvcall.h> 27 #include <asm/inst.h> 28 #include <linux/uaccess.h> 29 30 /* 31 * Stores the breakpoints currently in use on each breakpoint address 32 * register for every cpu 33 */ 34 static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM_MAX]); 35 36 /* 37 * Returns total number of data or instruction breakpoints available. 38 */ 39 int hw_breakpoint_slots(int type) 40 { 41 if (type == TYPE_DATA) 42 return nr_wp_slots(); 43 return 0; /* no instruction breakpoints available */ 44 } 45 46 static bool single_step_pending(void) 47 { 48 int i; 49 50 for (i = 0; i < nr_wp_slots(); i++) { 51 if (current->thread.last_hit_ubp[i]) 52 return true; 53 } 54 return false; 55 } 56 57 /* 58 * Install a perf counter breakpoint. 59 * 60 * We seek a free debug address register and use it for this 61 * breakpoint. 62 * 63 * Atomic: we hold the counter->ctx->lock and we only handle variables 64 * and registers local to this cpu. 65 */ 66 int arch_install_hw_breakpoint(struct perf_event *bp) 67 { 68 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 69 struct perf_event **slot; 70 int i; 71 72 for (i = 0; i < nr_wp_slots(); i++) { 73 slot = this_cpu_ptr(&bp_per_reg[i]); 74 if (!*slot) { 75 *slot = bp; 76 break; 77 } 78 } 79 80 if (WARN_ONCE(i == nr_wp_slots(), "Can't find any breakpoint slot")) 81 return -EBUSY; 82 83 /* 84 * Do not install DABR values if the instruction must be single-stepped. 85 * If so, DABR will be populated in single_step_dabr_instruction(). 86 */ 87 if (!single_step_pending()) 88 __set_breakpoint(i, info); 89 90 return 0; 91 } 92 93 /* 94 * Uninstall the breakpoint contained in the given counter. 95 * 96 * First we search the debug address register it uses and then we disable 97 * it. 98 * 99 * Atomic: we hold the counter->ctx->lock and we only handle variables 100 * and registers local to this cpu. 101 */ 102 void arch_uninstall_hw_breakpoint(struct perf_event *bp) 103 { 104 struct arch_hw_breakpoint null_brk = {0}; 105 struct perf_event **slot; 106 int i; 107 108 for (i = 0; i < nr_wp_slots(); i++) { 109 slot = this_cpu_ptr(&bp_per_reg[i]); 110 if (*slot == bp) { 111 *slot = NULL; 112 break; 113 } 114 } 115 116 if (WARN_ONCE(i == nr_wp_slots(), "Can't find any breakpoint slot")) 117 return; 118 119 __set_breakpoint(i, &null_brk); 120 } 121 122 static bool is_ptrace_bp(struct perf_event *bp) 123 { 124 return bp->overflow_handler == ptrace_triggered; 125 } 126 127 struct breakpoint { 128 struct list_head list; 129 struct perf_event *bp; 130 bool ptrace_bp; 131 }; 132 133 static DEFINE_PER_CPU(struct breakpoint *, cpu_bps[HBP_NUM_MAX]); 134 static LIST_HEAD(task_bps); 135 136 static struct breakpoint *alloc_breakpoint(struct perf_event *bp) 137 { 138 struct breakpoint *tmp; 139 140 tmp = kzalloc(sizeof(*tmp), GFP_KERNEL); 141 if (!tmp) 142 return ERR_PTR(-ENOMEM); 143 tmp->bp = bp; 144 tmp->ptrace_bp = is_ptrace_bp(bp); 145 return tmp; 146 } 147 148 static bool bp_addr_range_overlap(struct perf_event *bp1, struct perf_event *bp2) 149 { 150 __u64 bp1_saddr, bp1_eaddr, bp2_saddr, bp2_eaddr; 151 152 bp1_saddr = ALIGN_DOWN(bp1->attr.bp_addr, HW_BREAKPOINT_SIZE); 153 bp1_eaddr = ALIGN(bp1->attr.bp_addr + bp1->attr.bp_len, HW_BREAKPOINT_SIZE); 154 bp2_saddr = ALIGN_DOWN(bp2->attr.bp_addr, HW_BREAKPOINT_SIZE); 155 bp2_eaddr = ALIGN(bp2->attr.bp_addr + bp2->attr.bp_len, HW_BREAKPOINT_SIZE); 156 157 return (bp1_saddr < bp2_eaddr && bp1_eaddr > bp2_saddr); 158 } 159 160 static bool alternate_infra_bp(struct breakpoint *b, struct perf_event *bp) 161 { 162 return is_ptrace_bp(bp) ? !b->ptrace_bp : b->ptrace_bp; 163 } 164 165 static bool can_co_exist(struct breakpoint *b, struct perf_event *bp) 166 { 167 return !(alternate_infra_bp(b, bp) && bp_addr_range_overlap(b->bp, bp)); 168 } 169 170 static int task_bps_add(struct perf_event *bp) 171 { 172 struct breakpoint *tmp; 173 174 tmp = alloc_breakpoint(bp); 175 if (IS_ERR(tmp)) 176 return PTR_ERR(tmp); 177 178 list_add(&tmp->list, &task_bps); 179 return 0; 180 } 181 182 static void task_bps_remove(struct perf_event *bp) 183 { 184 struct list_head *pos, *q; 185 186 list_for_each_safe(pos, q, &task_bps) { 187 struct breakpoint *tmp = list_entry(pos, struct breakpoint, list); 188 189 if (tmp->bp == bp) { 190 list_del(&tmp->list); 191 kfree(tmp); 192 break; 193 } 194 } 195 } 196 197 /* 198 * If any task has breakpoint from alternate infrastructure, 199 * return true. Otherwise return false. 200 */ 201 static bool all_task_bps_check(struct perf_event *bp) 202 { 203 struct breakpoint *tmp; 204 205 list_for_each_entry(tmp, &task_bps, list) { 206 if (!can_co_exist(tmp, bp)) 207 return true; 208 } 209 return false; 210 } 211 212 /* 213 * If same task has breakpoint from alternate infrastructure, 214 * return true. Otherwise return false. 215 */ 216 static bool same_task_bps_check(struct perf_event *bp) 217 { 218 struct breakpoint *tmp; 219 220 list_for_each_entry(tmp, &task_bps, list) { 221 if (tmp->bp->hw.target == bp->hw.target && 222 !can_co_exist(tmp, bp)) 223 return true; 224 } 225 return false; 226 } 227 228 static int cpu_bps_add(struct perf_event *bp) 229 { 230 struct breakpoint **cpu_bp; 231 struct breakpoint *tmp; 232 int i = 0; 233 234 tmp = alloc_breakpoint(bp); 235 if (IS_ERR(tmp)) 236 return PTR_ERR(tmp); 237 238 cpu_bp = per_cpu_ptr(cpu_bps, bp->cpu); 239 for (i = 0; i < nr_wp_slots(); i++) { 240 if (!cpu_bp[i]) { 241 cpu_bp[i] = tmp; 242 break; 243 } 244 } 245 return 0; 246 } 247 248 static void cpu_bps_remove(struct perf_event *bp) 249 { 250 struct breakpoint **cpu_bp; 251 int i = 0; 252 253 cpu_bp = per_cpu_ptr(cpu_bps, bp->cpu); 254 for (i = 0; i < nr_wp_slots(); i++) { 255 if (!cpu_bp[i]) 256 continue; 257 258 if (cpu_bp[i]->bp == bp) { 259 kfree(cpu_bp[i]); 260 cpu_bp[i] = NULL; 261 break; 262 } 263 } 264 } 265 266 static bool cpu_bps_check(int cpu, struct perf_event *bp) 267 { 268 struct breakpoint **cpu_bp; 269 int i; 270 271 cpu_bp = per_cpu_ptr(cpu_bps, cpu); 272 for (i = 0; i < nr_wp_slots(); i++) { 273 if (cpu_bp[i] && !can_co_exist(cpu_bp[i], bp)) 274 return true; 275 } 276 return false; 277 } 278 279 static bool all_cpu_bps_check(struct perf_event *bp) 280 { 281 int cpu; 282 283 for_each_online_cpu(cpu) { 284 if (cpu_bps_check(cpu, bp)) 285 return true; 286 } 287 return false; 288 } 289 290 /* 291 * We don't use any locks to serialize accesses to cpu_bps or task_bps 292 * because are already inside nr_bp_mutex. 293 */ 294 int arch_reserve_bp_slot(struct perf_event *bp) 295 { 296 int ret; 297 298 /* ptrace breakpoint */ 299 if (is_ptrace_bp(bp)) { 300 if (all_cpu_bps_check(bp)) 301 return -ENOSPC; 302 303 if (same_task_bps_check(bp)) 304 return -ENOSPC; 305 306 return task_bps_add(bp); 307 } 308 309 /* perf breakpoint */ 310 if (is_kernel_addr(bp->attr.bp_addr)) 311 return 0; 312 313 if (bp->hw.target && bp->cpu == -1) { 314 if (same_task_bps_check(bp)) 315 return -ENOSPC; 316 317 return task_bps_add(bp); 318 } else if (!bp->hw.target && bp->cpu != -1) { 319 if (all_task_bps_check(bp)) 320 return -ENOSPC; 321 322 return cpu_bps_add(bp); 323 } 324 325 if (same_task_bps_check(bp)) 326 return -ENOSPC; 327 328 ret = cpu_bps_add(bp); 329 if (ret) 330 return ret; 331 ret = task_bps_add(bp); 332 if (ret) 333 cpu_bps_remove(bp); 334 335 return ret; 336 } 337 338 void arch_release_bp_slot(struct perf_event *bp) 339 { 340 if (!is_kernel_addr(bp->attr.bp_addr)) { 341 if (bp->hw.target) 342 task_bps_remove(bp); 343 if (bp->cpu != -1) 344 cpu_bps_remove(bp); 345 } 346 } 347 348 /* 349 * Perform cleanup of arch-specific counters during unregistration 350 * of the perf-event 351 */ 352 void arch_unregister_hw_breakpoint(struct perf_event *bp) 353 { 354 /* 355 * If the breakpoint is unregistered between a hw_breakpoint_handler() 356 * and the single_step_dabr_instruction(), then cleanup the breakpoint 357 * restoration variables to prevent dangling pointers. 358 * FIXME, this should not be using bp->ctx at all! Sayeth peterz. 359 */ 360 if (bp->ctx && bp->ctx->task && bp->ctx->task != ((void *)-1L)) { 361 int i; 362 363 for (i = 0; i < nr_wp_slots(); i++) { 364 if (bp->ctx->task->thread.last_hit_ubp[i] == bp) 365 bp->ctx->task->thread.last_hit_ubp[i] = NULL; 366 } 367 } 368 } 369 370 /* 371 * Check for virtual address in kernel space. 372 */ 373 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) 374 { 375 return is_kernel_addr(hw->address); 376 } 377 378 int arch_bp_generic_fields(int type, int *gen_bp_type) 379 { 380 *gen_bp_type = 0; 381 if (type & HW_BRK_TYPE_READ) 382 *gen_bp_type |= HW_BREAKPOINT_R; 383 if (type & HW_BRK_TYPE_WRITE) 384 *gen_bp_type |= HW_BREAKPOINT_W; 385 if (*gen_bp_type == 0) 386 return -EINVAL; 387 return 0; 388 } 389 390 /* 391 * Watchpoint match range is always doubleword(8 bytes) aligned on 392 * powerpc. If the given range is crossing doubleword boundary, we 393 * need to increase the length such that next doubleword also get 394 * covered. Ex, 395 * 396 * address len = 6 bytes 397 * |=========. 398 * |------------v--|------v--------| 399 * | | | | | | | | | | | | | | | | | 400 * |---------------|---------------| 401 * <---8 bytes---> 402 * 403 * In this case, we should configure hw as: 404 * start_addr = address & ~(HW_BREAKPOINT_SIZE - 1) 405 * len = 16 bytes 406 * 407 * @start_addr is inclusive but @end_addr is exclusive. 408 */ 409 static int hw_breakpoint_validate_len(struct arch_hw_breakpoint *hw) 410 { 411 u16 max_len = DABR_MAX_LEN; 412 u16 hw_len; 413 unsigned long start_addr, end_addr; 414 415 start_addr = ALIGN_DOWN(hw->address, HW_BREAKPOINT_SIZE); 416 end_addr = ALIGN(hw->address + hw->len, HW_BREAKPOINT_SIZE); 417 hw_len = end_addr - start_addr; 418 419 if (dawr_enabled()) { 420 max_len = DAWR_MAX_LEN; 421 /* DAWR region can't cross 512 bytes boundary */ 422 if (ALIGN(start_addr, SZ_512M) != ALIGN(end_addr - 1, SZ_512M)) 423 return -EINVAL; 424 } else if (IS_ENABLED(CONFIG_PPC_8xx)) { 425 /* 8xx can setup a range without limitation */ 426 max_len = U16_MAX; 427 } 428 429 if (hw_len > max_len) 430 return -EINVAL; 431 432 hw->hw_len = hw_len; 433 return 0; 434 } 435 436 /* 437 * Validate the arch-specific HW Breakpoint register settings 438 */ 439 int hw_breakpoint_arch_parse(struct perf_event *bp, 440 const struct perf_event_attr *attr, 441 struct arch_hw_breakpoint *hw) 442 { 443 int ret = -EINVAL; 444 445 if (!bp || !attr->bp_len) 446 return ret; 447 448 hw->type = HW_BRK_TYPE_TRANSLATE; 449 if (attr->bp_type & HW_BREAKPOINT_R) 450 hw->type |= HW_BRK_TYPE_READ; 451 if (attr->bp_type & HW_BREAKPOINT_W) 452 hw->type |= HW_BRK_TYPE_WRITE; 453 if (hw->type == HW_BRK_TYPE_TRANSLATE) 454 /* must set alteast read or write */ 455 return ret; 456 if (!attr->exclude_user) 457 hw->type |= HW_BRK_TYPE_USER; 458 if (!attr->exclude_kernel) 459 hw->type |= HW_BRK_TYPE_KERNEL; 460 if (!attr->exclude_hv) 461 hw->type |= HW_BRK_TYPE_HYP; 462 hw->address = attr->bp_addr; 463 hw->len = attr->bp_len; 464 465 if (!ppc_breakpoint_available()) 466 return -ENODEV; 467 468 return hw_breakpoint_validate_len(hw); 469 } 470 471 /* 472 * Restores the breakpoint on the debug registers. 473 * Invoke this function if it is known that the execution context is 474 * about to change to cause loss of MSR_SE settings. 475 */ 476 void thread_change_pc(struct task_struct *tsk, struct pt_regs *regs) 477 { 478 struct arch_hw_breakpoint *info; 479 int i; 480 481 for (i = 0; i < nr_wp_slots(); i++) { 482 if (unlikely(tsk->thread.last_hit_ubp[i])) 483 goto reset; 484 } 485 return; 486 487 reset: 488 regs->msr &= ~MSR_SE; 489 for (i = 0; i < nr_wp_slots(); i++) { 490 info = counter_arch_bp(__this_cpu_read(bp_per_reg[i])); 491 __set_breakpoint(i, info); 492 tsk->thread.last_hit_ubp[i] = NULL; 493 } 494 } 495 496 static bool dar_in_user_range(unsigned long dar, struct arch_hw_breakpoint *info) 497 { 498 return ((info->address <= dar) && (dar - info->address < info->len)); 499 } 500 501 static bool dar_user_range_overlaps(unsigned long dar, int size, 502 struct arch_hw_breakpoint *info) 503 { 504 return ((dar < info->address + info->len) && 505 (dar + size > info->address)); 506 } 507 508 static bool dar_in_hw_range(unsigned long dar, struct arch_hw_breakpoint *info) 509 { 510 unsigned long hw_start_addr, hw_end_addr; 511 512 hw_start_addr = ALIGN_DOWN(info->address, HW_BREAKPOINT_SIZE); 513 hw_end_addr = ALIGN(info->address + info->len, HW_BREAKPOINT_SIZE); 514 515 return ((hw_start_addr <= dar) && (hw_end_addr > dar)); 516 } 517 518 static bool dar_hw_range_overlaps(unsigned long dar, int size, 519 struct arch_hw_breakpoint *info) 520 { 521 unsigned long hw_start_addr, hw_end_addr; 522 523 hw_start_addr = ALIGN_DOWN(info->address, HW_BREAKPOINT_SIZE); 524 hw_end_addr = ALIGN(info->address + info->len, HW_BREAKPOINT_SIZE); 525 526 return ((dar < hw_end_addr) && (dar + size > hw_start_addr)); 527 } 528 529 /* 530 * If hw has multiple DAWR registers, we also need to check all 531 * dawrx constraint bits to confirm this is _really_ a valid event. 532 */ 533 static bool check_dawrx_constraints(struct pt_regs *regs, int type, 534 struct arch_hw_breakpoint *info) 535 { 536 if (OP_IS_LOAD(type) && !(info->type & HW_BRK_TYPE_READ)) 537 return false; 538 539 if (OP_IS_STORE(type) && !(info->type & HW_BRK_TYPE_WRITE)) 540 return false; 541 542 if (is_kernel_addr(regs->nip) && !(info->type & HW_BRK_TYPE_KERNEL)) 543 return false; 544 545 if (user_mode(regs) && !(info->type & HW_BRK_TYPE_USER)) 546 return false; 547 548 return true; 549 } 550 551 /* 552 * Return true if the event is valid wrt dawr configuration, 553 * including extraneous exception. Otherwise return false. 554 */ 555 static bool check_constraints(struct pt_regs *regs, struct ppc_inst instr, 556 int type, int size, struct arch_hw_breakpoint *info) 557 { 558 bool in_user_range = dar_in_user_range(regs->dar, info); 559 bool dawrx_constraints; 560 561 /* 562 * 8xx supports only one breakpoint and thus we can 563 * unconditionally return true. 564 */ 565 if (IS_ENABLED(CONFIG_PPC_8xx)) { 566 if (!in_user_range) 567 info->type |= HW_BRK_TYPE_EXTRANEOUS_IRQ; 568 return true; 569 } 570 571 if (unlikely(ppc_inst_equal(instr, ppc_inst(0)))) { 572 if (in_user_range) 573 return true; 574 575 if (dar_in_hw_range(regs->dar, info)) { 576 info->type |= HW_BRK_TYPE_EXTRANEOUS_IRQ; 577 return true; 578 } 579 return false; 580 } 581 582 dawrx_constraints = check_dawrx_constraints(regs, type, info); 583 584 if (dar_user_range_overlaps(regs->dar, size, info)) 585 return dawrx_constraints; 586 587 if (dar_hw_range_overlaps(regs->dar, size, info)) { 588 if (dawrx_constraints) { 589 info->type |= HW_BRK_TYPE_EXTRANEOUS_IRQ; 590 return true; 591 } 592 } 593 return false; 594 } 595 596 static void get_instr_detail(struct pt_regs *regs, struct ppc_inst *instr, 597 int *type, int *size, bool *larx_stcx) 598 { 599 struct instruction_op op; 600 601 if (__get_user_instr_inatomic(*instr, (void __user *)regs->nip)) 602 return; 603 604 analyse_instr(&op, regs, *instr); 605 606 /* 607 * Set size = 8 if analyse_instr() fails. If it's a userspace 608 * watchpoint(valid or extraneous), we can notify user about it. 609 * If it's a kernel watchpoint, instruction emulation will fail 610 * in stepping_handler() and watchpoint will be disabled. 611 */ 612 *type = GETTYPE(op.type); 613 *size = !(*type == UNKNOWN) ? GETSIZE(op.type) : 8; 614 *larx_stcx = (*type == LARX || *type == STCX); 615 } 616 617 /* 618 * We've failed in reliably handling the hw-breakpoint. Unregister 619 * it and throw a warning message to let the user know about it. 620 */ 621 static void handler_error(struct perf_event *bp, struct arch_hw_breakpoint *info) 622 { 623 WARN(1, "Unable to handle hardware breakpoint. Breakpoint at 0x%lx will be disabled.", 624 info->address); 625 perf_event_disable_inatomic(bp); 626 } 627 628 static void larx_stcx_err(struct perf_event *bp, struct arch_hw_breakpoint *info) 629 { 630 printk_ratelimited("Breakpoint hit on instruction that can't be emulated. Breakpoint at 0x%lx will be disabled.\n", 631 info->address); 632 perf_event_disable_inatomic(bp); 633 } 634 635 static bool stepping_handler(struct pt_regs *regs, struct perf_event **bp, 636 struct arch_hw_breakpoint **info, int *hit, 637 struct ppc_inst instr) 638 { 639 int i; 640 int stepped; 641 642 /* Do not emulate user-space instructions, instead single-step them */ 643 if (user_mode(regs)) { 644 for (i = 0; i < nr_wp_slots(); i++) { 645 if (!hit[i]) 646 continue; 647 current->thread.last_hit_ubp[i] = bp[i]; 648 info[i] = NULL; 649 } 650 regs->msr |= MSR_SE; 651 return false; 652 } 653 654 stepped = emulate_step(regs, instr); 655 if (!stepped) { 656 for (i = 0; i < nr_wp_slots(); i++) { 657 if (!hit[i]) 658 continue; 659 handler_error(bp[i], info[i]); 660 info[i] = NULL; 661 } 662 return false; 663 } 664 return true; 665 } 666 667 int hw_breakpoint_handler(struct die_args *args) 668 { 669 bool err = false; 670 int rc = NOTIFY_STOP; 671 struct perf_event *bp[HBP_NUM_MAX] = { NULL }; 672 struct pt_regs *regs = args->regs; 673 struct arch_hw_breakpoint *info[HBP_NUM_MAX] = { NULL }; 674 int i; 675 int hit[HBP_NUM_MAX] = {0}; 676 int nr_hit = 0; 677 bool ptrace_bp = false; 678 struct ppc_inst instr = ppc_inst(0); 679 int type = 0; 680 int size = 0; 681 bool larx_stcx = false; 682 683 /* Disable breakpoints during exception handling */ 684 hw_breakpoint_disable(); 685 686 /* 687 * The counter may be concurrently released but that can only 688 * occur from a call_rcu() path. We can then safely fetch 689 * the breakpoint, use its callback, touch its counter 690 * while we are in an rcu_read_lock() path. 691 */ 692 rcu_read_lock(); 693 694 if (!IS_ENABLED(CONFIG_PPC_8xx)) 695 get_instr_detail(regs, &instr, &type, &size, &larx_stcx); 696 697 for (i = 0; i < nr_wp_slots(); i++) { 698 bp[i] = __this_cpu_read(bp_per_reg[i]); 699 if (!bp[i]) 700 continue; 701 702 info[i] = counter_arch_bp(bp[i]); 703 info[i]->type &= ~HW_BRK_TYPE_EXTRANEOUS_IRQ; 704 705 if (check_constraints(regs, instr, type, size, info[i])) { 706 if (!IS_ENABLED(CONFIG_PPC_8xx) && 707 ppc_inst_equal(instr, ppc_inst(0))) { 708 handler_error(bp[i], info[i]); 709 info[i] = NULL; 710 err = 1; 711 continue; 712 } 713 714 if (is_ptrace_bp(bp[i])) 715 ptrace_bp = true; 716 hit[i] = 1; 717 nr_hit++; 718 } 719 } 720 721 if (err) 722 goto reset; 723 724 if (!nr_hit) { 725 rc = NOTIFY_DONE; 726 goto out; 727 } 728 729 /* 730 * Return early after invoking user-callback function without restoring 731 * DABR if the breakpoint is from ptrace which always operates in 732 * one-shot mode. The ptrace-ed process will receive the SIGTRAP signal 733 * generated in do_dabr(). 734 */ 735 if (ptrace_bp) { 736 for (i = 0; i < nr_wp_slots(); i++) { 737 if (!hit[i]) 738 continue; 739 perf_bp_event(bp[i], regs); 740 info[i] = NULL; 741 } 742 rc = NOTIFY_DONE; 743 goto reset; 744 } 745 746 if (!IS_ENABLED(CONFIG_PPC_8xx)) { 747 if (larx_stcx) { 748 for (i = 0; i < nr_wp_slots(); i++) { 749 if (!hit[i]) 750 continue; 751 larx_stcx_err(bp[i], info[i]); 752 info[i] = NULL; 753 } 754 goto reset; 755 } 756 757 if (!stepping_handler(regs, bp, info, hit, instr)) 758 goto reset; 759 } 760 761 /* 762 * As a policy, the callback is invoked in a 'trigger-after-execute' 763 * fashion 764 */ 765 for (i = 0; i < nr_wp_slots(); i++) { 766 if (!hit[i]) 767 continue; 768 if (!(info[i]->type & HW_BRK_TYPE_EXTRANEOUS_IRQ)) 769 perf_bp_event(bp[i], regs); 770 } 771 772 reset: 773 for (i = 0; i < nr_wp_slots(); i++) { 774 if (!info[i]) 775 continue; 776 __set_breakpoint(i, info[i]); 777 } 778 779 out: 780 rcu_read_unlock(); 781 return rc; 782 } 783 NOKPROBE_SYMBOL(hw_breakpoint_handler); 784 785 /* 786 * Handle single-step exceptions following a DABR hit. 787 */ 788 static int single_step_dabr_instruction(struct die_args *args) 789 { 790 struct pt_regs *regs = args->regs; 791 struct perf_event *bp = NULL; 792 struct arch_hw_breakpoint *info; 793 int i; 794 bool found = false; 795 796 /* 797 * Check if we are single-stepping as a result of a 798 * previous HW Breakpoint exception 799 */ 800 for (i = 0; i < nr_wp_slots(); i++) { 801 bp = current->thread.last_hit_ubp[i]; 802 803 if (!bp) 804 continue; 805 806 found = true; 807 info = counter_arch_bp(bp); 808 809 /* 810 * We shall invoke the user-defined callback function in the 811 * single stepping handler to confirm to 'trigger-after-execute' 812 * semantics 813 */ 814 if (!(info->type & HW_BRK_TYPE_EXTRANEOUS_IRQ)) 815 perf_bp_event(bp, regs); 816 current->thread.last_hit_ubp[i] = NULL; 817 } 818 819 if (!found) 820 return NOTIFY_DONE; 821 822 for (i = 0; i < nr_wp_slots(); i++) { 823 bp = __this_cpu_read(bp_per_reg[i]); 824 if (!bp) 825 continue; 826 827 info = counter_arch_bp(bp); 828 __set_breakpoint(i, info); 829 } 830 831 /* 832 * If the process was being single-stepped by ptrace, let the 833 * other single-step actions occur (e.g. generate SIGTRAP). 834 */ 835 if (test_thread_flag(TIF_SINGLESTEP)) 836 return NOTIFY_DONE; 837 838 return NOTIFY_STOP; 839 } 840 NOKPROBE_SYMBOL(single_step_dabr_instruction); 841 842 /* 843 * Handle debug exception notifications. 844 */ 845 int hw_breakpoint_exceptions_notify( 846 struct notifier_block *unused, unsigned long val, void *data) 847 { 848 int ret = NOTIFY_DONE; 849 850 switch (val) { 851 case DIE_DABR_MATCH: 852 ret = hw_breakpoint_handler(data); 853 break; 854 case DIE_SSTEP: 855 ret = single_step_dabr_instruction(data); 856 break; 857 } 858 859 return ret; 860 } 861 NOKPROBE_SYMBOL(hw_breakpoint_exceptions_notify); 862 863 /* 864 * Release the user breakpoints used by ptrace 865 */ 866 void flush_ptrace_hw_breakpoint(struct task_struct *tsk) 867 { 868 int i; 869 struct thread_struct *t = &tsk->thread; 870 871 for (i = 0; i < nr_wp_slots(); i++) { 872 unregister_hw_breakpoint(t->ptrace_bps[i]); 873 t->ptrace_bps[i] = NULL; 874 } 875 } 876 877 void hw_breakpoint_pmu_read(struct perf_event *bp) 878 { 879 /* TODO */ 880 } 881 882 void ptrace_triggered(struct perf_event *bp, 883 struct perf_sample_data *data, struct pt_regs *regs) 884 { 885 struct perf_event_attr attr; 886 887 /* 888 * Disable the breakpoint request here since ptrace has defined a 889 * one-shot behaviour for breakpoint exceptions in PPC64. 890 * The SIGTRAP signal is generated automatically for us in do_dabr(). 891 * We don't have to do anything about that here 892 */ 893 attr = bp->attr; 894 attr.disabled = true; 895 modify_user_hw_breakpoint(bp, &attr); 896 } 897