1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * 4 * Copyright (C) 2009, 2010 ARM Limited 5 * 6 * Author: Will Deacon <will.deacon@arm.com> 7 */ 8 9 /* 10 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, 11 * using the CPU's debug registers. 12 */ 13 #define pr_fmt(fmt) "hw-breakpoint: " fmt 14 15 #include <linux/errno.h> 16 #include <linux/hardirq.h> 17 #include <linux/perf_event.h> 18 #include <linux/hw_breakpoint.h> 19 #include <linux/smp.h> 20 #include <linux/cpu_pm.h> 21 #include <linux/coresight.h> 22 23 #include <asm/cacheflush.h> 24 #include <asm/cputype.h> 25 #include <asm/current.h> 26 #include <asm/hw_breakpoint.h> 27 #include <asm/traps.h> 28 29 /* Breakpoint currently in use for each BRP. */ 30 static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]); 31 32 /* Watchpoint currently in use for each WRP. */ 33 static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]); 34 35 /* Number of BRP/WRP registers on this CPU. */ 36 static int core_num_brps __ro_after_init; 37 static int core_num_wrps __ro_after_init; 38 39 /* Debug architecture version. */ 40 static u8 debug_arch __ro_after_init; 41 42 /* Does debug architecture support OS Save and Restore? */ 43 static bool has_ossr __ro_after_init; 44 45 /* Maximum supported watchpoint length. */ 46 static u8 max_watchpoint_len __ro_after_init; 47 48 #define READ_WB_REG_CASE(OP2, M, VAL) \ 49 case ((OP2 << 4) + M): \ 50 ARM_DBG_READ(c0, c ## M, OP2, VAL); \ 51 break 52 53 #define WRITE_WB_REG_CASE(OP2, M, VAL) \ 54 case ((OP2 << 4) + M): \ 55 ARM_DBG_WRITE(c0, c ## M, OP2, VAL); \ 56 break 57 58 #define GEN_READ_WB_REG_CASES(OP2, VAL) \ 59 READ_WB_REG_CASE(OP2, 0, VAL); \ 60 READ_WB_REG_CASE(OP2, 1, VAL); \ 61 READ_WB_REG_CASE(OP2, 2, VAL); \ 62 READ_WB_REG_CASE(OP2, 3, VAL); \ 63 READ_WB_REG_CASE(OP2, 4, VAL); \ 64 READ_WB_REG_CASE(OP2, 5, VAL); \ 65 READ_WB_REG_CASE(OP2, 6, VAL); \ 66 READ_WB_REG_CASE(OP2, 7, VAL); \ 67 READ_WB_REG_CASE(OP2, 8, VAL); \ 68 READ_WB_REG_CASE(OP2, 9, VAL); \ 69 READ_WB_REG_CASE(OP2, 10, VAL); \ 70 READ_WB_REG_CASE(OP2, 11, VAL); \ 71 READ_WB_REG_CASE(OP2, 12, VAL); \ 72 READ_WB_REG_CASE(OP2, 13, VAL); \ 73 READ_WB_REG_CASE(OP2, 14, VAL); \ 74 READ_WB_REG_CASE(OP2, 15, VAL) 75 76 #define GEN_WRITE_WB_REG_CASES(OP2, VAL) \ 77 WRITE_WB_REG_CASE(OP2, 0, VAL); \ 78 WRITE_WB_REG_CASE(OP2, 1, VAL); \ 79 WRITE_WB_REG_CASE(OP2, 2, VAL); \ 80 WRITE_WB_REG_CASE(OP2, 3, VAL); \ 81 WRITE_WB_REG_CASE(OP2, 4, VAL); \ 82 WRITE_WB_REG_CASE(OP2, 5, VAL); \ 83 WRITE_WB_REG_CASE(OP2, 6, VAL); \ 84 WRITE_WB_REG_CASE(OP2, 7, VAL); \ 85 WRITE_WB_REG_CASE(OP2, 8, VAL); \ 86 WRITE_WB_REG_CASE(OP2, 9, VAL); \ 87 WRITE_WB_REG_CASE(OP2, 10, VAL); \ 88 WRITE_WB_REG_CASE(OP2, 11, VAL); \ 89 WRITE_WB_REG_CASE(OP2, 12, VAL); \ 90 WRITE_WB_REG_CASE(OP2, 13, VAL); \ 91 WRITE_WB_REG_CASE(OP2, 14, VAL); \ 92 WRITE_WB_REG_CASE(OP2, 15, VAL) 93 94 static u32 read_wb_reg(int n) 95 { 96 u32 val = 0; 97 98 switch (n) { 99 GEN_READ_WB_REG_CASES(ARM_OP2_BVR, val); 100 GEN_READ_WB_REG_CASES(ARM_OP2_BCR, val); 101 GEN_READ_WB_REG_CASES(ARM_OP2_WVR, val); 102 GEN_READ_WB_REG_CASES(ARM_OP2_WCR, val); 103 default: 104 pr_warn("attempt to read from unknown breakpoint register %d\n", 105 n); 106 } 107 108 return val; 109 } 110 111 static void write_wb_reg(int n, u32 val) 112 { 113 switch (n) { 114 GEN_WRITE_WB_REG_CASES(ARM_OP2_BVR, val); 115 GEN_WRITE_WB_REG_CASES(ARM_OP2_BCR, val); 116 GEN_WRITE_WB_REG_CASES(ARM_OP2_WVR, val); 117 GEN_WRITE_WB_REG_CASES(ARM_OP2_WCR, val); 118 default: 119 pr_warn("attempt to write to unknown breakpoint register %d\n", 120 n); 121 } 122 isb(); 123 } 124 125 /* Determine debug architecture. */ 126 static u8 get_debug_arch(void) 127 { 128 u32 didr; 129 130 /* Do we implement the extended CPUID interface? */ 131 if (((read_cpuid_id() >> 16) & 0xf) != 0xf) { 132 pr_warn_once("CPUID feature registers not supported. " 133 "Assuming v6 debug is present.\n"); 134 return ARM_DEBUG_ARCH_V6; 135 } 136 137 ARM_DBG_READ(c0, c0, 0, didr); 138 return (didr >> 16) & 0xf; 139 } 140 141 u8 arch_get_debug_arch(void) 142 { 143 return debug_arch; 144 } 145 146 static int debug_arch_supported(void) 147 { 148 u8 arch = get_debug_arch(); 149 150 /* We don't support the memory-mapped interface. */ 151 return (arch >= ARM_DEBUG_ARCH_V6 && arch <= ARM_DEBUG_ARCH_V7_ECP14) || 152 arch >= ARM_DEBUG_ARCH_V7_1; 153 } 154 155 /* Can we determine the watchpoint access type from the fsr? */ 156 static int debug_exception_updates_fsr(void) 157 { 158 return get_debug_arch() >= ARM_DEBUG_ARCH_V8; 159 } 160 161 /* Determine number of WRP registers available. */ 162 static int get_num_wrp_resources(void) 163 { 164 u32 didr; 165 ARM_DBG_READ(c0, c0, 0, didr); 166 return ((didr >> 28) & 0xf) + 1; 167 } 168 169 /* Determine number of BRP registers available. */ 170 static int get_num_brp_resources(void) 171 { 172 u32 didr; 173 ARM_DBG_READ(c0, c0, 0, didr); 174 return ((didr >> 24) & 0xf) + 1; 175 } 176 177 /* Does this core support mismatch breakpoints? */ 178 static int core_has_mismatch_brps(void) 179 { 180 return (get_debug_arch() >= ARM_DEBUG_ARCH_V7_ECP14 && 181 get_num_brp_resources() > 1); 182 } 183 184 /* Determine number of usable WRPs available. */ 185 static int get_num_wrps(void) 186 { 187 /* 188 * On debug architectures prior to 7.1, when a watchpoint fires, the 189 * only way to work out which watchpoint it was is by disassembling 190 * the faulting instruction and working out the address of the memory 191 * access. 192 * 193 * Furthermore, we can only do this if the watchpoint was precise 194 * since imprecise watchpoints prevent us from calculating register 195 * based addresses. 196 * 197 * Providing we have more than 1 breakpoint register, we only report 198 * a single watchpoint register for the time being. This way, we always 199 * know which watchpoint fired. In the future we can either add a 200 * disassembler and address generation emulator, or we can insert a 201 * check to see if the DFAR is set on watchpoint exception entry 202 * [the ARM ARM states that the DFAR is UNKNOWN, but experience shows 203 * that it is set on some implementations]. 204 */ 205 if (get_debug_arch() < ARM_DEBUG_ARCH_V7_1) 206 return 1; 207 208 return get_num_wrp_resources(); 209 } 210 211 /* Determine number of usable BRPs available. */ 212 static int get_num_brps(void) 213 { 214 int brps = get_num_brp_resources(); 215 return core_has_mismatch_brps() ? brps - 1 : brps; 216 } 217 218 /* 219 * In order to access the breakpoint/watchpoint control registers, 220 * we must be running in debug monitor mode. Unfortunately, we can 221 * be put into halting debug mode at any time by an external debugger 222 * but there is nothing we can do to prevent that. 223 */ 224 static int monitor_mode_enabled(void) 225 { 226 u32 dscr; 227 ARM_DBG_READ(c0, c1, 0, dscr); 228 return !!(dscr & ARM_DSCR_MDBGEN); 229 } 230 231 static int enable_monitor_mode(void) 232 { 233 u32 dscr; 234 ARM_DBG_READ(c0, c1, 0, dscr); 235 236 /* If monitor mode is already enabled, just return. */ 237 if (dscr & ARM_DSCR_MDBGEN) 238 goto out; 239 240 /* Write to the corresponding DSCR. */ 241 switch (get_debug_arch()) { 242 case ARM_DEBUG_ARCH_V6: 243 case ARM_DEBUG_ARCH_V6_1: 244 ARM_DBG_WRITE(c0, c1, 0, (dscr | ARM_DSCR_MDBGEN)); 245 break; 246 case ARM_DEBUG_ARCH_V7_ECP14: 247 case ARM_DEBUG_ARCH_V7_1: 248 case ARM_DEBUG_ARCH_V8: 249 case ARM_DEBUG_ARCH_V8_1: 250 case ARM_DEBUG_ARCH_V8_2: 251 case ARM_DEBUG_ARCH_V8_4: 252 ARM_DBG_WRITE(c0, c2, 2, (dscr | ARM_DSCR_MDBGEN)); 253 isb(); 254 break; 255 default: 256 return -ENODEV; 257 } 258 259 /* Check that the write made it through. */ 260 ARM_DBG_READ(c0, c1, 0, dscr); 261 if (!(dscr & ARM_DSCR_MDBGEN)) { 262 pr_warn_once("Failed to enable monitor mode on CPU %d.\n", 263 smp_processor_id()); 264 return -EPERM; 265 } 266 267 out: 268 return 0; 269 } 270 271 int hw_breakpoint_slots(int type) 272 { 273 if (!debug_arch_supported()) 274 return 0; 275 276 /* 277 * We can be called early, so don't rely on 278 * our static variables being initialised. 279 */ 280 switch (type) { 281 case TYPE_INST: 282 return get_num_brps(); 283 case TYPE_DATA: 284 return get_num_wrps(); 285 default: 286 pr_warn("unknown slot type: %d\n", type); 287 return 0; 288 } 289 } 290 291 /* 292 * Check if 8-bit byte-address select is available. 293 * This clobbers WRP 0. 294 */ 295 static u8 get_max_wp_len(void) 296 { 297 u32 ctrl_reg; 298 struct arch_hw_breakpoint_ctrl ctrl; 299 u8 size = 4; 300 301 if (debug_arch < ARM_DEBUG_ARCH_V7_ECP14) 302 goto out; 303 304 memset(&ctrl, 0, sizeof(ctrl)); 305 ctrl.len = ARM_BREAKPOINT_LEN_8; 306 ctrl_reg = encode_ctrl_reg(ctrl); 307 308 write_wb_reg(ARM_BASE_WVR, 0); 309 write_wb_reg(ARM_BASE_WCR, ctrl_reg); 310 if ((read_wb_reg(ARM_BASE_WCR) & ctrl_reg) == ctrl_reg) 311 size = 8; 312 313 out: 314 return size; 315 } 316 317 u8 arch_get_max_wp_len(void) 318 { 319 return max_watchpoint_len; 320 } 321 322 /* 323 * Install a perf counter breakpoint. 324 */ 325 int arch_install_hw_breakpoint(struct perf_event *bp) 326 { 327 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 328 struct perf_event **slot, **slots; 329 int i, max_slots, ctrl_base, val_base; 330 u32 addr, ctrl; 331 332 addr = info->address; 333 ctrl = encode_ctrl_reg(info->ctrl) | 0x1; 334 335 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { 336 /* Breakpoint */ 337 ctrl_base = ARM_BASE_BCR; 338 val_base = ARM_BASE_BVR; 339 slots = this_cpu_ptr(bp_on_reg); 340 max_slots = core_num_brps; 341 } else { 342 /* Watchpoint */ 343 ctrl_base = ARM_BASE_WCR; 344 val_base = ARM_BASE_WVR; 345 slots = this_cpu_ptr(wp_on_reg); 346 max_slots = core_num_wrps; 347 } 348 349 for (i = 0; i < max_slots; ++i) { 350 slot = &slots[i]; 351 352 if (!*slot) { 353 *slot = bp; 354 break; 355 } 356 } 357 358 if (i == max_slots) { 359 pr_warn("Can't find any breakpoint slot\n"); 360 return -EBUSY; 361 } 362 363 /* Override the breakpoint data with the step data. */ 364 if (info->step_ctrl.enabled) { 365 addr = info->trigger & ~0x3; 366 ctrl = encode_ctrl_reg(info->step_ctrl); 367 if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE) { 368 i = 0; 369 ctrl_base = ARM_BASE_BCR + core_num_brps; 370 val_base = ARM_BASE_BVR + core_num_brps; 371 } 372 } 373 374 /* Setup the address register. */ 375 write_wb_reg(val_base + i, addr); 376 377 /* Setup the control register. */ 378 write_wb_reg(ctrl_base + i, ctrl); 379 return 0; 380 } 381 382 void arch_uninstall_hw_breakpoint(struct perf_event *bp) 383 { 384 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 385 struct perf_event **slot, **slots; 386 int i, max_slots, base; 387 388 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { 389 /* Breakpoint */ 390 base = ARM_BASE_BCR; 391 slots = this_cpu_ptr(bp_on_reg); 392 max_slots = core_num_brps; 393 } else { 394 /* Watchpoint */ 395 base = ARM_BASE_WCR; 396 slots = this_cpu_ptr(wp_on_reg); 397 max_slots = core_num_wrps; 398 } 399 400 /* Remove the breakpoint. */ 401 for (i = 0; i < max_slots; ++i) { 402 slot = &slots[i]; 403 404 if (*slot == bp) { 405 *slot = NULL; 406 break; 407 } 408 } 409 410 if (i == max_slots) { 411 pr_warn("Can't find any breakpoint slot\n"); 412 return; 413 } 414 415 /* Ensure that we disable the mismatch breakpoint. */ 416 if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE && 417 info->step_ctrl.enabled) { 418 i = 0; 419 base = ARM_BASE_BCR + core_num_brps; 420 } 421 422 /* Reset the control register. */ 423 write_wb_reg(base + i, 0); 424 } 425 426 static int get_hbp_len(u8 hbp_len) 427 { 428 unsigned int len_in_bytes = 0; 429 430 switch (hbp_len) { 431 case ARM_BREAKPOINT_LEN_1: 432 len_in_bytes = 1; 433 break; 434 case ARM_BREAKPOINT_LEN_2: 435 len_in_bytes = 2; 436 break; 437 case ARM_BREAKPOINT_LEN_4: 438 len_in_bytes = 4; 439 break; 440 case ARM_BREAKPOINT_LEN_8: 441 len_in_bytes = 8; 442 break; 443 } 444 445 return len_in_bytes; 446 } 447 448 /* 449 * Check whether bp virtual address is in kernel space. 450 */ 451 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) 452 { 453 unsigned int len; 454 unsigned long va; 455 456 va = hw->address; 457 len = get_hbp_len(hw->ctrl.len); 458 459 return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); 460 } 461 462 /* 463 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl. 464 * Hopefully this will disappear when ptrace can bypass the conversion 465 * to generic breakpoint descriptions. 466 */ 467 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, 468 int *gen_len, int *gen_type) 469 { 470 /* Type */ 471 switch (ctrl.type) { 472 case ARM_BREAKPOINT_EXECUTE: 473 *gen_type = HW_BREAKPOINT_X; 474 break; 475 case ARM_BREAKPOINT_LOAD: 476 *gen_type = HW_BREAKPOINT_R; 477 break; 478 case ARM_BREAKPOINT_STORE: 479 *gen_type = HW_BREAKPOINT_W; 480 break; 481 case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE: 482 *gen_type = HW_BREAKPOINT_RW; 483 break; 484 default: 485 return -EINVAL; 486 } 487 488 /* Len */ 489 switch (ctrl.len) { 490 case ARM_BREAKPOINT_LEN_1: 491 *gen_len = HW_BREAKPOINT_LEN_1; 492 break; 493 case ARM_BREAKPOINT_LEN_2: 494 *gen_len = HW_BREAKPOINT_LEN_2; 495 break; 496 case ARM_BREAKPOINT_LEN_4: 497 *gen_len = HW_BREAKPOINT_LEN_4; 498 break; 499 case ARM_BREAKPOINT_LEN_8: 500 *gen_len = HW_BREAKPOINT_LEN_8; 501 break; 502 default: 503 return -EINVAL; 504 } 505 506 return 0; 507 } 508 509 /* 510 * Construct an arch_hw_breakpoint from a perf_event. 511 */ 512 static int arch_build_bp_info(struct perf_event *bp, 513 const struct perf_event_attr *attr, 514 struct arch_hw_breakpoint *hw) 515 { 516 /* Type */ 517 switch (attr->bp_type) { 518 case HW_BREAKPOINT_X: 519 hw->ctrl.type = ARM_BREAKPOINT_EXECUTE; 520 break; 521 case HW_BREAKPOINT_R: 522 hw->ctrl.type = ARM_BREAKPOINT_LOAD; 523 break; 524 case HW_BREAKPOINT_W: 525 hw->ctrl.type = ARM_BREAKPOINT_STORE; 526 break; 527 case HW_BREAKPOINT_RW: 528 hw->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE; 529 break; 530 default: 531 return -EINVAL; 532 } 533 534 /* Len */ 535 switch (attr->bp_len) { 536 case HW_BREAKPOINT_LEN_1: 537 hw->ctrl.len = ARM_BREAKPOINT_LEN_1; 538 break; 539 case HW_BREAKPOINT_LEN_2: 540 hw->ctrl.len = ARM_BREAKPOINT_LEN_2; 541 break; 542 case HW_BREAKPOINT_LEN_4: 543 hw->ctrl.len = ARM_BREAKPOINT_LEN_4; 544 break; 545 case HW_BREAKPOINT_LEN_8: 546 hw->ctrl.len = ARM_BREAKPOINT_LEN_8; 547 if ((hw->ctrl.type != ARM_BREAKPOINT_EXECUTE) 548 && max_watchpoint_len >= 8) 549 break; 550 /* Else, fall through */ 551 default: 552 return -EINVAL; 553 } 554 555 /* 556 * Breakpoints must be of length 2 (thumb) or 4 (ARM) bytes. 557 * Watchpoints can be of length 1, 2, 4 or 8 bytes if supported 558 * by the hardware and must be aligned to the appropriate number of 559 * bytes. 560 */ 561 if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE && 562 hw->ctrl.len != ARM_BREAKPOINT_LEN_2 && 563 hw->ctrl.len != ARM_BREAKPOINT_LEN_4) 564 return -EINVAL; 565 566 /* Address */ 567 hw->address = attr->bp_addr; 568 569 /* Privilege */ 570 hw->ctrl.privilege = ARM_BREAKPOINT_USER; 571 if (arch_check_bp_in_kernelspace(hw)) 572 hw->ctrl.privilege |= ARM_BREAKPOINT_PRIV; 573 574 /* Enabled? */ 575 hw->ctrl.enabled = !attr->disabled; 576 577 /* Mismatch */ 578 hw->ctrl.mismatch = 0; 579 580 return 0; 581 } 582 583 /* 584 * Validate the arch-specific HW Breakpoint register settings. 585 */ 586 int hw_breakpoint_arch_parse(struct perf_event *bp, 587 const struct perf_event_attr *attr, 588 struct arch_hw_breakpoint *hw) 589 { 590 int ret = 0; 591 u32 offset, alignment_mask = 0x3; 592 593 /* Ensure that we are in monitor debug mode. */ 594 if (!monitor_mode_enabled()) 595 return -ENODEV; 596 597 /* Build the arch_hw_breakpoint. */ 598 ret = arch_build_bp_info(bp, attr, hw); 599 if (ret) 600 goto out; 601 602 /* Check address alignment. */ 603 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_8) 604 alignment_mask = 0x7; 605 offset = hw->address & alignment_mask; 606 switch (offset) { 607 case 0: 608 /* Aligned */ 609 break; 610 case 1: 611 case 2: 612 /* Allow halfword watchpoints and breakpoints. */ 613 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_2) 614 break; 615 /* Else, fall through */ 616 case 3: 617 /* Allow single byte watchpoint. */ 618 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_1) 619 break; 620 /* Else, fall through */ 621 default: 622 ret = -EINVAL; 623 goto out; 624 } 625 626 hw->address &= ~alignment_mask; 627 hw->ctrl.len <<= offset; 628 629 if (is_default_overflow_handler(bp)) { 630 /* 631 * Mismatch breakpoints are required for single-stepping 632 * breakpoints. 633 */ 634 if (!core_has_mismatch_brps()) 635 return -EINVAL; 636 637 /* We don't allow mismatch breakpoints in kernel space. */ 638 if (arch_check_bp_in_kernelspace(hw)) 639 return -EPERM; 640 641 /* 642 * Per-cpu breakpoints are not supported by our stepping 643 * mechanism. 644 */ 645 if (!bp->hw.target) 646 return -EINVAL; 647 648 /* 649 * We only support specific access types if the fsr 650 * reports them. 651 */ 652 if (!debug_exception_updates_fsr() && 653 (hw->ctrl.type == ARM_BREAKPOINT_LOAD || 654 hw->ctrl.type == ARM_BREAKPOINT_STORE)) 655 return -EINVAL; 656 } 657 658 out: 659 return ret; 660 } 661 662 /* 663 * Enable/disable single-stepping over the breakpoint bp at address addr. 664 */ 665 static void enable_single_step(struct perf_event *bp, u32 addr) 666 { 667 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 668 669 arch_uninstall_hw_breakpoint(bp); 670 info->step_ctrl.mismatch = 1; 671 info->step_ctrl.len = ARM_BREAKPOINT_LEN_4; 672 info->step_ctrl.type = ARM_BREAKPOINT_EXECUTE; 673 info->step_ctrl.privilege = info->ctrl.privilege; 674 info->step_ctrl.enabled = 1; 675 info->trigger = addr; 676 arch_install_hw_breakpoint(bp); 677 } 678 679 static void disable_single_step(struct perf_event *bp) 680 { 681 arch_uninstall_hw_breakpoint(bp); 682 counter_arch_bp(bp)->step_ctrl.enabled = 0; 683 arch_install_hw_breakpoint(bp); 684 } 685 686 static int watchpoint_fault_on_uaccess(struct pt_regs *regs, 687 struct arch_hw_breakpoint *info) 688 { 689 return !user_mode(regs) && info->ctrl.privilege == ARM_BREAKPOINT_USER; 690 } 691 692 static void watchpoint_handler(unsigned long addr, unsigned int fsr, 693 struct pt_regs *regs) 694 { 695 int i, access; 696 u32 val, ctrl_reg, alignment_mask; 697 struct perf_event *wp, **slots; 698 struct arch_hw_breakpoint *info; 699 struct arch_hw_breakpoint_ctrl ctrl; 700 701 slots = this_cpu_ptr(wp_on_reg); 702 703 for (i = 0; i < core_num_wrps; ++i) { 704 rcu_read_lock(); 705 706 wp = slots[i]; 707 708 if (wp == NULL) 709 goto unlock; 710 711 info = counter_arch_bp(wp); 712 /* 713 * The DFAR is an unknown value on debug architectures prior 714 * to 7.1. Since we only allow a single watchpoint on these 715 * older CPUs, we can set the trigger to the lowest possible 716 * faulting address. 717 */ 718 if (debug_arch < ARM_DEBUG_ARCH_V7_1) { 719 BUG_ON(i > 0); 720 info->trigger = wp->attr.bp_addr; 721 } else { 722 if (info->ctrl.len == ARM_BREAKPOINT_LEN_8) 723 alignment_mask = 0x7; 724 else 725 alignment_mask = 0x3; 726 727 /* Check if the watchpoint value matches. */ 728 val = read_wb_reg(ARM_BASE_WVR + i); 729 if (val != (addr & ~alignment_mask)) 730 goto unlock; 731 732 /* Possible match, check the byte address select. */ 733 ctrl_reg = read_wb_reg(ARM_BASE_WCR + i); 734 decode_ctrl_reg(ctrl_reg, &ctrl); 735 if (!((1 << (addr & alignment_mask)) & ctrl.len)) 736 goto unlock; 737 738 /* Check that the access type matches. */ 739 if (debug_exception_updates_fsr()) { 740 access = (fsr & ARM_FSR_ACCESS_MASK) ? 741 HW_BREAKPOINT_W : HW_BREAKPOINT_R; 742 if (!(access & hw_breakpoint_type(wp))) 743 goto unlock; 744 } 745 746 /* We have a winner. */ 747 info->trigger = addr; 748 } 749 750 pr_debug("watchpoint fired: address = 0x%x\n", info->trigger); 751 752 /* 753 * If we triggered a user watchpoint from a uaccess routine, 754 * then handle the stepping ourselves since userspace really 755 * can't help us with this. 756 */ 757 if (watchpoint_fault_on_uaccess(regs, info)) 758 goto step; 759 760 perf_bp_event(wp, regs); 761 762 /* 763 * Defer stepping to the overflow handler if one is installed. 764 * Otherwise, insert a temporary mismatch breakpoint so that 765 * we can single-step over the watchpoint trigger. 766 */ 767 if (!is_default_overflow_handler(wp)) 768 goto unlock; 769 770 step: 771 enable_single_step(wp, instruction_pointer(regs)); 772 unlock: 773 rcu_read_unlock(); 774 } 775 } 776 777 static void watchpoint_single_step_handler(unsigned long pc) 778 { 779 int i; 780 struct perf_event *wp, **slots; 781 struct arch_hw_breakpoint *info; 782 783 slots = this_cpu_ptr(wp_on_reg); 784 785 for (i = 0; i < core_num_wrps; ++i) { 786 rcu_read_lock(); 787 788 wp = slots[i]; 789 790 if (wp == NULL) 791 goto unlock; 792 793 info = counter_arch_bp(wp); 794 if (!info->step_ctrl.enabled) 795 goto unlock; 796 797 /* 798 * Restore the original watchpoint if we've completed the 799 * single-step. 800 */ 801 if (info->trigger != pc) 802 disable_single_step(wp); 803 804 unlock: 805 rcu_read_unlock(); 806 } 807 } 808 809 static void breakpoint_handler(unsigned long unknown, struct pt_regs *regs) 810 { 811 int i; 812 u32 ctrl_reg, val, addr; 813 struct perf_event *bp, **slots; 814 struct arch_hw_breakpoint *info; 815 struct arch_hw_breakpoint_ctrl ctrl; 816 817 slots = this_cpu_ptr(bp_on_reg); 818 819 /* The exception entry code places the amended lr in the PC. */ 820 addr = regs->ARM_pc; 821 822 /* Check the currently installed breakpoints first. */ 823 for (i = 0; i < core_num_brps; ++i) { 824 rcu_read_lock(); 825 826 bp = slots[i]; 827 828 if (bp == NULL) 829 goto unlock; 830 831 info = counter_arch_bp(bp); 832 833 /* Check if the breakpoint value matches. */ 834 val = read_wb_reg(ARM_BASE_BVR + i); 835 if (val != (addr & ~0x3)) 836 goto mismatch; 837 838 /* Possible match, check the byte address select to confirm. */ 839 ctrl_reg = read_wb_reg(ARM_BASE_BCR + i); 840 decode_ctrl_reg(ctrl_reg, &ctrl); 841 if ((1 << (addr & 0x3)) & ctrl.len) { 842 info->trigger = addr; 843 pr_debug("breakpoint fired: address = 0x%x\n", addr); 844 perf_bp_event(bp, regs); 845 if (!bp->overflow_handler) 846 enable_single_step(bp, addr); 847 goto unlock; 848 } 849 850 mismatch: 851 /* If we're stepping a breakpoint, it can now be restored. */ 852 if (info->step_ctrl.enabled) 853 disable_single_step(bp); 854 unlock: 855 rcu_read_unlock(); 856 } 857 858 /* Handle any pending watchpoint single-step breakpoints. */ 859 watchpoint_single_step_handler(addr); 860 } 861 862 /* 863 * Called from either the Data Abort Handler [watchpoint] or the 864 * Prefetch Abort Handler [breakpoint] with interrupts disabled. 865 */ 866 static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr, 867 struct pt_regs *regs) 868 { 869 int ret = 0; 870 u32 dscr; 871 872 preempt_disable(); 873 874 if (interrupts_enabled(regs)) 875 local_irq_enable(); 876 877 /* We only handle watchpoints and hardware breakpoints. */ 878 ARM_DBG_READ(c0, c1, 0, dscr); 879 880 /* Perform perf callbacks. */ 881 switch (ARM_DSCR_MOE(dscr)) { 882 case ARM_ENTRY_BREAKPOINT: 883 breakpoint_handler(addr, regs); 884 break; 885 case ARM_ENTRY_ASYNC_WATCHPOINT: 886 WARN(1, "Asynchronous watchpoint exception taken. Debugging results may be unreliable\n"); 887 /* Fall through */ 888 case ARM_ENTRY_SYNC_WATCHPOINT: 889 watchpoint_handler(addr, fsr, regs); 890 break; 891 default: 892 ret = 1; /* Unhandled fault. */ 893 } 894 895 preempt_enable(); 896 897 return ret; 898 } 899 900 /* 901 * One-time initialisation. 902 */ 903 static cpumask_t debug_err_mask; 904 905 static int debug_reg_trap(struct pt_regs *regs, unsigned int instr) 906 { 907 int cpu = smp_processor_id(); 908 909 pr_warn("Debug register access (0x%x) caused undefined instruction on CPU %d\n", 910 instr, cpu); 911 912 /* Set the error flag for this CPU and skip the faulting instruction. */ 913 cpumask_set_cpu(cpu, &debug_err_mask); 914 instruction_pointer(regs) += 4; 915 return 0; 916 } 917 918 static struct undef_hook debug_reg_hook = { 919 .instr_mask = 0x0fe80f10, 920 .instr_val = 0x0e000e10, 921 .fn = debug_reg_trap, 922 }; 923 924 /* Does this core support OS Save and Restore? */ 925 static bool core_has_os_save_restore(void) 926 { 927 u32 oslsr; 928 929 switch (get_debug_arch()) { 930 case ARM_DEBUG_ARCH_V7_1: 931 return true; 932 case ARM_DEBUG_ARCH_V7_ECP14: 933 ARM_DBG_READ(c1, c1, 4, oslsr); 934 if (oslsr & ARM_OSLSR_OSLM0) 935 return true; 936 /* Else, fall through */ 937 default: 938 return false; 939 } 940 } 941 942 static void reset_ctrl_regs(unsigned int cpu) 943 { 944 int i, raw_num_brps, err = 0; 945 u32 val; 946 947 /* 948 * v7 debug contains save and restore registers so that debug state 949 * can be maintained across low-power modes without leaving the debug 950 * logic powered up. It is IMPLEMENTATION DEFINED whether we can access 951 * the debug registers out of reset, so we must unlock the OS Lock 952 * Access Register to avoid taking undefined instruction exceptions 953 * later on. 954 */ 955 switch (debug_arch) { 956 case ARM_DEBUG_ARCH_V6: 957 case ARM_DEBUG_ARCH_V6_1: 958 /* ARMv6 cores clear the registers out of reset. */ 959 goto out_mdbgen; 960 case ARM_DEBUG_ARCH_V7_ECP14: 961 /* 962 * Ensure sticky power-down is clear (i.e. debug logic is 963 * powered up). 964 */ 965 ARM_DBG_READ(c1, c5, 4, val); 966 if ((val & 0x1) == 0) 967 err = -EPERM; 968 969 if (!has_ossr) 970 goto clear_vcr; 971 break; 972 case ARM_DEBUG_ARCH_V7_1: 973 /* 974 * Ensure the OS double lock is clear. 975 */ 976 ARM_DBG_READ(c1, c3, 4, val); 977 if ((val & 0x1) == 1) 978 err = -EPERM; 979 break; 980 } 981 982 if (err) { 983 pr_warn_once("CPU %d debug is powered down!\n", cpu); 984 cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu)); 985 return; 986 } 987 988 /* 989 * Unconditionally clear the OS lock by writing a value 990 * other than CS_LAR_KEY to the access register. 991 */ 992 ARM_DBG_WRITE(c1, c0, 4, ~CORESIGHT_UNLOCK); 993 isb(); 994 995 /* 996 * Clear any configured vector-catch events before 997 * enabling monitor mode. 998 */ 999 clear_vcr: 1000 ARM_DBG_WRITE(c0, c7, 0, 0); 1001 isb(); 1002 1003 if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) { 1004 pr_warn_once("CPU %d failed to disable vector catch\n", cpu); 1005 return; 1006 } 1007 1008 /* 1009 * The control/value register pairs are UNKNOWN out of reset so 1010 * clear them to avoid spurious debug events. 1011 */ 1012 raw_num_brps = get_num_brp_resources(); 1013 for (i = 0; i < raw_num_brps; ++i) { 1014 write_wb_reg(ARM_BASE_BCR + i, 0UL); 1015 write_wb_reg(ARM_BASE_BVR + i, 0UL); 1016 } 1017 1018 for (i = 0; i < core_num_wrps; ++i) { 1019 write_wb_reg(ARM_BASE_WCR + i, 0UL); 1020 write_wb_reg(ARM_BASE_WVR + i, 0UL); 1021 } 1022 1023 if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) { 1024 pr_warn_once("CPU %d failed to clear debug register pairs\n", cpu); 1025 return; 1026 } 1027 1028 /* 1029 * Have a crack at enabling monitor mode. We don't actually need 1030 * it yet, but reporting an error early is useful if it fails. 1031 */ 1032 out_mdbgen: 1033 if (enable_monitor_mode()) 1034 cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu)); 1035 } 1036 1037 static int dbg_reset_online(unsigned int cpu) 1038 { 1039 local_irq_disable(); 1040 reset_ctrl_regs(cpu); 1041 local_irq_enable(); 1042 return 0; 1043 } 1044 1045 #ifdef CONFIG_CPU_PM 1046 static int dbg_cpu_pm_notify(struct notifier_block *self, unsigned long action, 1047 void *v) 1048 { 1049 if (action == CPU_PM_EXIT) 1050 reset_ctrl_regs(smp_processor_id()); 1051 1052 return NOTIFY_OK; 1053 } 1054 1055 static struct notifier_block dbg_cpu_pm_nb = { 1056 .notifier_call = dbg_cpu_pm_notify, 1057 }; 1058 1059 static void __init pm_init(void) 1060 { 1061 cpu_pm_register_notifier(&dbg_cpu_pm_nb); 1062 } 1063 #else 1064 static inline void pm_init(void) 1065 { 1066 } 1067 #endif 1068 1069 static int __init arch_hw_breakpoint_init(void) 1070 { 1071 int ret; 1072 1073 debug_arch = get_debug_arch(); 1074 1075 if (!debug_arch_supported()) { 1076 pr_info("debug architecture 0x%x unsupported.\n", debug_arch); 1077 return 0; 1078 } 1079 1080 /* 1081 * Scorpion CPUs (at least those in APQ8060) seem to set DBGPRSR.SPD 1082 * whenever a WFI is issued, even if the core is not powered down, in 1083 * violation of the architecture. When DBGPRSR.SPD is set, accesses to 1084 * breakpoint and watchpoint registers are treated as undefined, so 1085 * this results in boot time and runtime failures when these are 1086 * accessed and we unexpectedly take a trap. 1087 * 1088 * It's not clear if/how this can be worked around, so we blacklist 1089 * Scorpion CPUs to avoid these issues. 1090 */ 1091 if (read_cpuid_part() == ARM_CPU_PART_SCORPION) { 1092 pr_info("Scorpion CPU detected. Hardware breakpoints and watchpoints disabled\n"); 1093 return 0; 1094 } 1095 1096 has_ossr = core_has_os_save_restore(); 1097 1098 /* Determine how many BRPs/WRPs are available. */ 1099 core_num_brps = get_num_brps(); 1100 core_num_wrps = get_num_wrps(); 1101 1102 /* 1103 * We need to tread carefully here because DBGSWENABLE may be 1104 * driven low on this core and there isn't an architected way to 1105 * determine that. 1106 */ 1107 cpus_read_lock(); 1108 register_undef_hook(&debug_reg_hook); 1109 1110 /* 1111 * Register CPU notifier which resets the breakpoint resources. We 1112 * assume that a halting debugger will leave the world in a nice state 1113 * for us. 1114 */ 1115 ret = cpuhp_setup_state_cpuslocked(CPUHP_AP_ONLINE_DYN, 1116 "arm/hw_breakpoint:online", 1117 dbg_reset_online, NULL); 1118 unregister_undef_hook(&debug_reg_hook); 1119 if (WARN_ON(ret < 0) || !cpumask_empty(&debug_err_mask)) { 1120 core_num_brps = 0; 1121 core_num_wrps = 0; 1122 if (ret > 0) 1123 cpuhp_remove_state_nocalls_cpuslocked(ret); 1124 cpus_read_unlock(); 1125 return 0; 1126 } 1127 1128 pr_info("found %d " "%s" "breakpoint and %d watchpoint registers.\n", 1129 core_num_brps, core_has_mismatch_brps() ? "(+1 reserved) " : 1130 "", core_num_wrps); 1131 1132 /* Work out the maximum supported watchpoint length. */ 1133 max_watchpoint_len = get_max_wp_len(); 1134 pr_info("maximum watchpoint size is %u bytes.\n", 1135 max_watchpoint_len); 1136 1137 /* Register debug fault handler. */ 1138 hook_fault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP, 1139 TRAP_HWBKPT, "watchpoint debug exception"); 1140 hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP, 1141 TRAP_HWBKPT, "breakpoint debug exception"); 1142 cpus_read_unlock(); 1143 1144 /* Register PM notifiers. */ 1145 pm_init(); 1146 return 0; 1147 } 1148 arch_initcall(arch_hw_breakpoint_init); 1149 1150 void hw_breakpoint_pmu_read(struct perf_event *bp) 1151 { 1152 } 1153 1154 /* 1155 * Dummy function to register with die_notifier. 1156 */ 1157 int hw_breakpoint_exceptions_notify(struct notifier_block *unused, 1158 unsigned long val, void *data) 1159 { 1160 return NOTIFY_DONE; 1161 } 1162