1 /* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License version 2 as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * You should have received a copy of the GNU General Public License 12 * along with this program; if not, write to the Free Software 13 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 14 * 15 * Copyright (C) 2009, 2010 ARM Limited 16 * 17 * Author: Will Deacon <will.deacon@arm.com> 18 */ 19 20 /* 21 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, 22 * using the CPU's debug registers. 23 */ 24 #define pr_fmt(fmt) "hw-breakpoint: " fmt 25 26 #include <linux/errno.h> 27 #include <linux/hardirq.h> 28 #include <linux/perf_event.h> 29 #include <linux/hw_breakpoint.h> 30 #include <linux/smp.h> 31 #include <linux/cpu_pm.h> 32 #include <linux/coresight.h> 33 34 #include <asm/cacheflush.h> 35 #include <asm/cputype.h> 36 #include <asm/current.h> 37 #include <asm/hw_breakpoint.h> 38 #include <asm/traps.h> 39 40 /* Breakpoint currently in use for each BRP. */ 41 static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]); 42 43 /* Watchpoint currently in use for each WRP. */ 44 static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]); 45 46 /* Number of BRP/WRP registers on this CPU. */ 47 static int core_num_brps __ro_after_init; 48 static int core_num_wrps __ro_after_init; 49 50 /* Debug architecture version. */ 51 static u8 debug_arch __ro_after_init; 52 53 /* Does debug architecture support OS Save and Restore? */ 54 static bool has_ossr __ro_after_init; 55 56 /* Maximum supported watchpoint length. */ 57 static u8 max_watchpoint_len __ro_after_init; 58 59 #define READ_WB_REG_CASE(OP2, M, VAL) \ 60 case ((OP2 << 4) + M): \ 61 ARM_DBG_READ(c0, c ## M, OP2, VAL); \ 62 break 63 64 #define WRITE_WB_REG_CASE(OP2, M, VAL) \ 65 case ((OP2 << 4) + M): \ 66 ARM_DBG_WRITE(c0, c ## M, OP2, VAL); \ 67 break 68 69 #define GEN_READ_WB_REG_CASES(OP2, VAL) \ 70 READ_WB_REG_CASE(OP2, 0, VAL); \ 71 READ_WB_REG_CASE(OP2, 1, VAL); \ 72 READ_WB_REG_CASE(OP2, 2, VAL); \ 73 READ_WB_REG_CASE(OP2, 3, VAL); \ 74 READ_WB_REG_CASE(OP2, 4, VAL); \ 75 READ_WB_REG_CASE(OP2, 5, VAL); \ 76 READ_WB_REG_CASE(OP2, 6, VAL); \ 77 READ_WB_REG_CASE(OP2, 7, VAL); \ 78 READ_WB_REG_CASE(OP2, 8, VAL); \ 79 READ_WB_REG_CASE(OP2, 9, VAL); \ 80 READ_WB_REG_CASE(OP2, 10, VAL); \ 81 READ_WB_REG_CASE(OP2, 11, VAL); \ 82 READ_WB_REG_CASE(OP2, 12, VAL); \ 83 READ_WB_REG_CASE(OP2, 13, VAL); \ 84 READ_WB_REG_CASE(OP2, 14, VAL); \ 85 READ_WB_REG_CASE(OP2, 15, VAL) 86 87 #define GEN_WRITE_WB_REG_CASES(OP2, VAL) \ 88 WRITE_WB_REG_CASE(OP2, 0, VAL); \ 89 WRITE_WB_REG_CASE(OP2, 1, VAL); \ 90 WRITE_WB_REG_CASE(OP2, 2, VAL); \ 91 WRITE_WB_REG_CASE(OP2, 3, VAL); \ 92 WRITE_WB_REG_CASE(OP2, 4, VAL); \ 93 WRITE_WB_REG_CASE(OP2, 5, VAL); \ 94 WRITE_WB_REG_CASE(OP2, 6, VAL); \ 95 WRITE_WB_REG_CASE(OP2, 7, VAL); \ 96 WRITE_WB_REG_CASE(OP2, 8, VAL); \ 97 WRITE_WB_REG_CASE(OP2, 9, VAL); \ 98 WRITE_WB_REG_CASE(OP2, 10, VAL); \ 99 WRITE_WB_REG_CASE(OP2, 11, VAL); \ 100 WRITE_WB_REG_CASE(OP2, 12, VAL); \ 101 WRITE_WB_REG_CASE(OP2, 13, VAL); \ 102 WRITE_WB_REG_CASE(OP2, 14, VAL); \ 103 WRITE_WB_REG_CASE(OP2, 15, VAL) 104 105 static u32 read_wb_reg(int n) 106 { 107 u32 val = 0; 108 109 switch (n) { 110 GEN_READ_WB_REG_CASES(ARM_OP2_BVR, val); 111 GEN_READ_WB_REG_CASES(ARM_OP2_BCR, val); 112 GEN_READ_WB_REG_CASES(ARM_OP2_WVR, val); 113 GEN_READ_WB_REG_CASES(ARM_OP2_WCR, val); 114 default: 115 pr_warn("attempt to read from unknown breakpoint register %d\n", 116 n); 117 } 118 119 return val; 120 } 121 122 static void write_wb_reg(int n, u32 val) 123 { 124 switch (n) { 125 GEN_WRITE_WB_REG_CASES(ARM_OP2_BVR, val); 126 GEN_WRITE_WB_REG_CASES(ARM_OP2_BCR, val); 127 GEN_WRITE_WB_REG_CASES(ARM_OP2_WVR, val); 128 GEN_WRITE_WB_REG_CASES(ARM_OP2_WCR, val); 129 default: 130 pr_warn("attempt to write to unknown breakpoint register %d\n", 131 n); 132 } 133 isb(); 134 } 135 136 /* Determine debug architecture. */ 137 static u8 get_debug_arch(void) 138 { 139 u32 didr; 140 141 /* Do we implement the extended CPUID interface? */ 142 if (((read_cpuid_id() >> 16) & 0xf) != 0xf) { 143 pr_warn_once("CPUID feature registers not supported. " 144 "Assuming v6 debug is present.\n"); 145 return ARM_DEBUG_ARCH_V6; 146 } 147 148 ARM_DBG_READ(c0, c0, 0, didr); 149 return (didr >> 16) & 0xf; 150 } 151 152 u8 arch_get_debug_arch(void) 153 { 154 return debug_arch; 155 } 156 157 static int debug_arch_supported(void) 158 { 159 u8 arch = get_debug_arch(); 160 161 /* We don't support the memory-mapped interface. */ 162 return (arch >= ARM_DEBUG_ARCH_V6 && arch <= ARM_DEBUG_ARCH_V7_ECP14) || 163 arch >= ARM_DEBUG_ARCH_V7_1; 164 } 165 166 /* Can we determine the watchpoint access type from the fsr? */ 167 static int debug_exception_updates_fsr(void) 168 { 169 return get_debug_arch() >= ARM_DEBUG_ARCH_V8; 170 } 171 172 /* Determine number of WRP registers available. */ 173 static int get_num_wrp_resources(void) 174 { 175 u32 didr; 176 ARM_DBG_READ(c0, c0, 0, didr); 177 return ((didr >> 28) & 0xf) + 1; 178 } 179 180 /* Determine number of BRP registers available. */ 181 static int get_num_brp_resources(void) 182 { 183 u32 didr; 184 ARM_DBG_READ(c0, c0, 0, didr); 185 return ((didr >> 24) & 0xf) + 1; 186 } 187 188 /* Does this core support mismatch breakpoints? */ 189 static int core_has_mismatch_brps(void) 190 { 191 return (get_debug_arch() >= ARM_DEBUG_ARCH_V7_ECP14 && 192 get_num_brp_resources() > 1); 193 } 194 195 /* Determine number of usable WRPs available. */ 196 static int get_num_wrps(void) 197 { 198 /* 199 * On debug architectures prior to 7.1, when a watchpoint fires, the 200 * only way to work out which watchpoint it was is by disassembling 201 * the faulting instruction and working out the address of the memory 202 * access. 203 * 204 * Furthermore, we can only do this if the watchpoint was precise 205 * since imprecise watchpoints prevent us from calculating register 206 * based addresses. 207 * 208 * Providing we have more than 1 breakpoint register, we only report 209 * a single watchpoint register for the time being. This way, we always 210 * know which watchpoint fired. In the future we can either add a 211 * disassembler and address generation emulator, or we can insert a 212 * check to see if the DFAR is set on watchpoint exception entry 213 * [the ARM ARM states that the DFAR is UNKNOWN, but experience shows 214 * that it is set on some implementations]. 215 */ 216 if (get_debug_arch() < ARM_DEBUG_ARCH_V7_1) 217 return 1; 218 219 return get_num_wrp_resources(); 220 } 221 222 /* Determine number of usable BRPs available. */ 223 static int get_num_brps(void) 224 { 225 int brps = get_num_brp_resources(); 226 return core_has_mismatch_brps() ? brps - 1 : brps; 227 } 228 229 /* 230 * In order to access the breakpoint/watchpoint control registers, 231 * we must be running in debug monitor mode. Unfortunately, we can 232 * be put into halting debug mode at any time by an external debugger 233 * but there is nothing we can do to prevent that. 234 */ 235 static int monitor_mode_enabled(void) 236 { 237 u32 dscr; 238 ARM_DBG_READ(c0, c1, 0, dscr); 239 return !!(dscr & ARM_DSCR_MDBGEN); 240 } 241 242 static int enable_monitor_mode(void) 243 { 244 u32 dscr; 245 ARM_DBG_READ(c0, c1, 0, dscr); 246 247 /* If monitor mode is already enabled, just return. */ 248 if (dscr & ARM_DSCR_MDBGEN) 249 goto out; 250 251 /* Write to the corresponding DSCR. */ 252 switch (get_debug_arch()) { 253 case ARM_DEBUG_ARCH_V6: 254 case ARM_DEBUG_ARCH_V6_1: 255 ARM_DBG_WRITE(c0, c1, 0, (dscr | ARM_DSCR_MDBGEN)); 256 break; 257 case ARM_DEBUG_ARCH_V7_ECP14: 258 case ARM_DEBUG_ARCH_V7_1: 259 case ARM_DEBUG_ARCH_V8: 260 ARM_DBG_WRITE(c0, c2, 2, (dscr | ARM_DSCR_MDBGEN)); 261 isb(); 262 break; 263 default: 264 return -ENODEV; 265 } 266 267 /* Check that the write made it through. */ 268 ARM_DBG_READ(c0, c1, 0, dscr); 269 if (!(dscr & ARM_DSCR_MDBGEN)) { 270 pr_warn_once("Failed to enable monitor mode on CPU %d.\n", 271 smp_processor_id()); 272 return -EPERM; 273 } 274 275 out: 276 return 0; 277 } 278 279 int hw_breakpoint_slots(int type) 280 { 281 if (!debug_arch_supported()) 282 return 0; 283 284 /* 285 * We can be called early, so don't rely on 286 * our static variables being initialised. 287 */ 288 switch (type) { 289 case TYPE_INST: 290 return get_num_brps(); 291 case TYPE_DATA: 292 return get_num_wrps(); 293 default: 294 pr_warn("unknown slot type: %d\n", type); 295 return 0; 296 } 297 } 298 299 /* 300 * Check if 8-bit byte-address select is available. 301 * This clobbers WRP 0. 302 */ 303 static u8 get_max_wp_len(void) 304 { 305 u32 ctrl_reg; 306 struct arch_hw_breakpoint_ctrl ctrl; 307 u8 size = 4; 308 309 if (debug_arch < ARM_DEBUG_ARCH_V7_ECP14) 310 goto out; 311 312 memset(&ctrl, 0, sizeof(ctrl)); 313 ctrl.len = ARM_BREAKPOINT_LEN_8; 314 ctrl_reg = encode_ctrl_reg(ctrl); 315 316 write_wb_reg(ARM_BASE_WVR, 0); 317 write_wb_reg(ARM_BASE_WCR, ctrl_reg); 318 if ((read_wb_reg(ARM_BASE_WCR) & ctrl_reg) == ctrl_reg) 319 size = 8; 320 321 out: 322 return size; 323 } 324 325 u8 arch_get_max_wp_len(void) 326 { 327 return max_watchpoint_len; 328 } 329 330 /* 331 * Install a perf counter breakpoint. 332 */ 333 int arch_install_hw_breakpoint(struct perf_event *bp) 334 { 335 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 336 struct perf_event **slot, **slots; 337 int i, max_slots, ctrl_base, val_base; 338 u32 addr, ctrl; 339 340 addr = info->address; 341 ctrl = encode_ctrl_reg(info->ctrl) | 0x1; 342 343 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { 344 /* Breakpoint */ 345 ctrl_base = ARM_BASE_BCR; 346 val_base = ARM_BASE_BVR; 347 slots = this_cpu_ptr(bp_on_reg); 348 max_slots = core_num_brps; 349 } else { 350 /* Watchpoint */ 351 ctrl_base = ARM_BASE_WCR; 352 val_base = ARM_BASE_WVR; 353 slots = this_cpu_ptr(wp_on_reg); 354 max_slots = core_num_wrps; 355 } 356 357 for (i = 0; i < max_slots; ++i) { 358 slot = &slots[i]; 359 360 if (!*slot) { 361 *slot = bp; 362 break; 363 } 364 } 365 366 if (i == max_slots) { 367 pr_warn("Can't find any breakpoint slot\n"); 368 return -EBUSY; 369 } 370 371 /* Override the breakpoint data with the step data. */ 372 if (info->step_ctrl.enabled) { 373 addr = info->trigger & ~0x3; 374 ctrl = encode_ctrl_reg(info->step_ctrl); 375 if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE) { 376 i = 0; 377 ctrl_base = ARM_BASE_BCR + core_num_brps; 378 val_base = ARM_BASE_BVR + core_num_brps; 379 } 380 } 381 382 /* Setup the address register. */ 383 write_wb_reg(val_base + i, addr); 384 385 /* Setup the control register. */ 386 write_wb_reg(ctrl_base + i, ctrl); 387 return 0; 388 } 389 390 void arch_uninstall_hw_breakpoint(struct perf_event *bp) 391 { 392 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 393 struct perf_event **slot, **slots; 394 int i, max_slots, base; 395 396 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { 397 /* Breakpoint */ 398 base = ARM_BASE_BCR; 399 slots = this_cpu_ptr(bp_on_reg); 400 max_slots = core_num_brps; 401 } else { 402 /* Watchpoint */ 403 base = ARM_BASE_WCR; 404 slots = this_cpu_ptr(wp_on_reg); 405 max_slots = core_num_wrps; 406 } 407 408 /* Remove the breakpoint. */ 409 for (i = 0; i < max_slots; ++i) { 410 slot = &slots[i]; 411 412 if (*slot == bp) { 413 *slot = NULL; 414 break; 415 } 416 } 417 418 if (i == max_slots) { 419 pr_warn("Can't find any breakpoint slot\n"); 420 return; 421 } 422 423 /* Ensure that we disable the mismatch breakpoint. */ 424 if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE && 425 info->step_ctrl.enabled) { 426 i = 0; 427 base = ARM_BASE_BCR + core_num_brps; 428 } 429 430 /* Reset the control register. */ 431 write_wb_reg(base + i, 0); 432 } 433 434 static int get_hbp_len(u8 hbp_len) 435 { 436 unsigned int len_in_bytes = 0; 437 438 switch (hbp_len) { 439 case ARM_BREAKPOINT_LEN_1: 440 len_in_bytes = 1; 441 break; 442 case ARM_BREAKPOINT_LEN_2: 443 len_in_bytes = 2; 444 break; 445 case ARM_BREAKPOINT_LEN_4: 446 len_in_bytes = 4; 447 break; 448 case ARM_BREAKPOINT_LEN_8: 449 len_in_bytes = 8; 450 break; 451 } 452 453 return len_in_bytes; 454 } 455 456 /* 457 * Check whether bp virtual address is in kernel space. 458 */ 459 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) 460 { 461 unsigned int len; 462 unsigned long va; 463 464 va = hw->address; 465 len = get_hbp_len(hw->ctrl.len); 466 467 return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); 468 } 469 470 /* 471 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl. 472 * Hopefully this will disappear when ptrace can bypass the conversion 473 * to generic breakpoint descriptions. 474 */ 475 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, 476 int *gen_len, int *gen_type) 477 { 478 /* Type */ 479 switch (ctrl.type) { 480 case ARM_BREAKPOINT_EXECUTE: 481 *gen_type = HW_BREAKPOINT_X; 482 break; 483 case ARM_BREAKPOINT_LOAD: 484 *gen_type = HW_BREAKPOINT_R; 485 break; 486 case ARM_BREAKPOINT_STORE: 487 *gen_type = HW_BREAKPOINT_W; 488 break; 489 case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE: 490 *gen_type = HW_BREAKPOINT_RW; 491 break; 492 default: 493 return -EINVAL; 494 } 495 496 /* Len */ 497 switch (ctrl.len) { 498 case ARM_BREAKPOINT_LEN_1: 499 *gen_len = HW_BREAKPOINT_LEN_1; 500 break; 501 case ARM_BREAKPOINT_LEN_2: 502 *gen_len = HW_BREAKPOINT_LEN_2; 503 break; 504 case ARM_BREAKPOINT_LEN_4: 505 *gen_len = HW_BREAKPOINT_LEN_4; 506 break; 507 case ARM_BREAKPOINT_LEN_8: 508 *gen_len = HW_BREAKPOINT_LEN_8; 509 break; 510 default: 511 return -EINVAL; 512 } 513 514 return 0; 515 } 516 517 /* 518 * Construct an arch_hw_breakpoint from a perf_event. 519 */ 520 static int arch_build_bp_info(struct perf_event *bp, 521 const struct perf_event_attr *attr, 522 struct arch_hw_breakpoint *hw) 523 { 524 /* Type */ 525 switch (attr->bp_type) { 526 case HW_BREAKPOINT_X: 527 hw->ctrl.type = ARM_BREAKPOINT_EXECUTE; 528 break; 529 case HW_BREAKPOINT_R: 530 hw->ctrl.type = ARM_BREAKPOINT_LOAD; 531 break; 532 case HW_BREAKPOINT_W: 533 hw->ctrl.type = ARM_BREAKPOINT_STORE; 534 break; 535 case HW_BREAKPOINT_RW: 536 hw->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE; 537 break; 538 default: 539 return -EINVAL; 540 } 541 542 /* Len */ 543 switch (attr->bp_len) { 544 case HW_BREAKPOINT_LEN_1: 545 hw->ctrl.len = ARM_BREAKPOINT_LEN_1; 546 break; 547 case HW_BREAKPOINT_LEN_2: 548 hw->ctrl.len = ARM_BREAKPOINT_LEN_2; 549 break; 550 case HW_BREAKPOINT_LEN_4: 551 hw->ctrl.len = ARM_BREAKPOINT_LEN_4; 552 break; 553 case HW_BREAKPOINT_LEN_8: 554 hw->ctrl.len = ARM_BREAKPOINT_LEN_8; 555 if ((hw->ctrl.type != ARM_BREAKPOINT_EXECUTE) 556 && max_watchpoint_len >= 8) 557 break; 558 default: 559 return -EINVAL; 560 } 561 562 /* 563 * Breakpoints must be of length 2 (thumb) or 4 (ARM) bytes. 564 * Watchpoints can be of length 1, 2, 4 or 8 bytes if supported 565 * by the hardware and must be aligned to the appropriate number of 566 * bytes. 567 */ 568 if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE && 569 hw->ctrl.len != ARM_BREAKPOINT_LEN_2 && 570 hw->ctrl.len != ARM_BREAKPOINT_LEN_4) 571 return -EINVAL; 572 573 /* Address */ 574 hw->address = attr->bp_addr; 575 576 /* Privilege */ 577 hw->ctrl.privilege = ARM_BREAKPOINT_USER; 578 if (arch_check_bp_in_kernelspace(hw)) 579 hw->ctrl.privilege |= ARM_BREAKPOINT_PRIV; 580 581 /* Enabled? */ 582 hw->ctrl.enabled = !attr->disabled; 583 584 /* Mismatch */ 585 hw->ctrl.mismatch = 0; 586 587 return 0; 588 } 589 590 /* 591 * Validate the arch-specific HW Breakpoint register settings. 592 */ 593 int hw_breakpoint_arch_parse(struct perf_event *bp, 594 const struct perf_event_attr *attr, 595 struct arch_hw_breakpoint *hw) 596 { 597 int ret = 0; 598 u32 offset, alignment_mask = 0x3; 599 600 /* Ensure that we are in monitor debug mode. */ 601 if (!monitor_mode_enabled()) 602 return -ENODEV; 603 604 /* Build the arch_hw_breakpoint. */ 605 ret = arch_build_bp_info(bp, attr, hw); 606 if (ret) 607 goto out; 608 609 /* Check address alignment. */ 610 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_8) 611 alignment_mask = 0x7; 612 offset = hw->address & alignment_mask; 613 switch (offset) { 614 case 0: 615 /* Aligned */ 616 break; 617 case 1: 618 case 2: 619 /* Allow halfword watchpoints and breakpoints. */ 620 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_2) 621 break; 622 case 3: 623 /* Allow single byte watchpoint. */ 624 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_1) 625 break; 626 default: 627 ret = -EINVAL; 628 goto out; 629 } 630 631 hw->address &= ~alignment_mask; 632 hw->ctrl.len <<= offset; 633 634 if (is_default_overflow_handler(bp)) { 635 /* 636 * Mismatch breakpoints are required for single-stepping 637 * breakpoints. 638 */ 639 if (!core_has_mismatch_brps()) 640 return -EINVAL; 641 642 /* We don't allow mismatch breakpoints in kernel space. */ 643 if (arch_check_bp_in_kernelspace(hw)) 644 return -EPERM; 645 646 /* 647 * Per-cpu breakpoints are not supported by our stepping 648 * mechanism. 649 */ 650 if (!bp->hw.target) 651 return -EINVAL; 652 653 /* 654 * We only support specific access types if the fsr 655 * reports them. 656 */ 657 if (!debug_exception_updates_fsr() && 658 (hw->ctrl.type == ARM_BREAKPOINT_LOAD || 659 hw->ctrl.type == ARM_BREAKPOINT_STORE)) 660 return -EINVAL; 661 } 662 663 out: 664 return ret; 665 } 666 667 /* 668 * Enable/disable single-stepping over the breakpoint bp at address addr. 669 */ 670 static void enable_single_step(struct perf_event *bp, u32 addr) 671 { 672 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 673 674 arch_uninstall_hw_breakpoint(bp); 675 info->step_ctrl.mismatch = 1; 676 info->step_ctrl.len = ARM_BREAKPOINT_LEN_4; 677 info->step_ctrl.type = ARM_BREAKPOINT_EXECUTE; 678 info->step_ctrl.privilege = info->ctrl.privilege; 679 info->step_ctrl.enabled = 1; 680 info->trigger = addr; 681 arch_install_hw_breakpoint(bp); 682 } 683 684 static void disable_single_step(struct perf_event *bp) 685 { 686 arch_uninstall_hw_breakpoint(bp); 687 counter_arch_bp(bp)->step_ctrl.enabled = 0; 688 arch_install_hw_breakpoint(bp); 689 } 690 691 static void watchpoint_handler(unsigned long addr, unsigned int fsr, 692 struct pt_regs *regs) 693 { 694 int i, access; 695 u32 val, ctrl_reg, alignment_mask; 696 struct perf_event *wp, **slots; 697 struct arch_hw_breakpoint *info; 698 struct arch_hw_breakpoint_ctrl ctrl; 699 700 slots = this_cpu_ptr(wp_on_reg); 701 702 for (i = 0; i < core_num_wrps; ++i) { 703 rcu_read_lock(); 704 705 wp = slots[i]; 706 707 if (wp == NULL) 708 goto unlock; 709 710 info = counter_arch_bp(wp); 711 /* 712 * The DFAR is an unknown value on debug architectures prior 713 * to 7.1. Since we only allow a single watchpoint on these 714 * older CPUs, we can set the trigger to the lowest possible 715 * faulting address. 716 */ 717 if (debug_arch < ARM_DEBUG_ARCH_V7_1) { 718 BUG_ON(i > 0); 719 info->trigger = wp->attr.bp_addr; 720 } else { 721 if (info->ctrl.len == ARM_BREAKPOINT_LEN_8) 722 alignment_mask = 0x7; 723 else 724 alignment_mask = 0x3; 725 726 /* Check if the watchpoint value matches. */ 727 val = read_wb_reg(ARM_BASE_WVR + i); 728 if (val != (addr & ~alignment_mask)) 729 goto unlock; 730 731 /* Possible match, check the byte address select. */ 732 ctrl_reg = read_wb_reg(ARM_BASE_WCR + i); 733 decode_ctrl_reg(ctrl_reg, &ctrl); 734 if (!((1 << (addr & alignment_mask)) & ctrl.len)) 735 goto unlock; 736 737 /* Check that the access type matches. */ 738 if (debug_exception_updates_fsr()) { 739 access = (fsr & ARM_FSR_ACCESS_MASK) ? 740 HW_BREAKPOINT_W : HW_BREAKPOINT_R; 741 if (!(access & hw_breakpoint_type(wp))) 742 goto unlock; 743 } 744 745 /* We have a winner. */ 746 info->trigger = addr; 747 } 748 749 pr_debug("watchpoint fired: address = 0x%x\n", info->trigger); 750 perf_bp_event(wp, regs); 751 752 /* 753 * If no overflow handler is present, insert a temporary 754 * mismatch breakpoint so we can single-step over the 755 * watchpoint trigger. 756 */ 757 if (is_default_overflow_handler(wp)) 758 enable_single_step(wp, instruction_pointer(regs)); 759 760 unlock: 761 rcu_read_unlock(); 762 } 763 } 764 765 static void watchpoint_single_step_handler(unsigned long pc) 766 { 767 int i; 768 struct perf_event *wp, **slots; 769 struct arch_hw_breakpoint *info; 770 771 slots = this_cpu_ptr(wp_on_reg); 772 773 for (i = 0; i < core_num_wrps; ++i) { 774 rcu_read_lock(); 775 776 wp = slots[i]; 777 778 if (wp == NULL) 779 goto unlock; 780 781 info = counter_arch_bp(wp); 782 if (!info->step_ctrl.enabled) 783 goto unlock; 784 785 /* 786 * Restore the original watchpoint if we've completed the 787 * single-step. 788 */ 789 if (info->trigger != pc) 790 disable_single_step(wp); 791 792 unlock: 793 rcu_read_unlock(); 794 } 795 } 796 797 static void breakpoint_handler(unsigned long unknown, struct pt_regs *regs) 798 { 799 int i; 800 u32 ctrl_reg, val, addr; 801 struct perf_event *bp, **slots; 802 struct arch_hw_breakpoint *info; 803 struct arch_hw_breakpoint_ctrl ctrl; 804 805 slots = this_cpu_ptr(bp_on_reg); 806 807 /* The exception entry code places the amended lr in the PC. */ 808 addr = regs->ARM_pc; 809 810 /* Check the currently installed breakpoints first. */ 811 for (i = 0; i < core_num_brps; ++i) { 812 rcu_read_lock(); 813 814 bp = slots[i]; 815 816 if (bp == NULL) 817 goto unlock; 818 819 info = counter_arch_bp(bp); 820 821 /* Check if the breakpoint value matches. */ 822 val = read_wb_reg(ARM_BASE_BVR + i); 823 if (val != (addr & ~0x3)) 824 goto mismatch; 825 826 /* Possible match, check the byte address select to confirm. */ 827 ctrl_reg = read_wb_reg(ARM_BASE_BCR + i); 828 decode_ctrl_reg(ctrl_reg, &ctrl); 829 if ((1 << (addr & 0x3)) & ctrl.len) { 830 info->trigger = addr; 831 pr_debug("breakpoint fired: address = 0x%x\n", addr); 832 perf_bp_event(bp, regs); 833 if (!bp->overflow_handler) 834 enable_single_step(bp, addr); 835 goto unlock; 836 } 837 838 mismatch: 839 /* If we're stepping a breakpoint, it can now be restored. */ 840 if (info->step_ctrl.enabled) 841 disable_single_step(bp); 842 unlock: 843 rcu_read_unlock(); 844 } 845 846 /* Handle any pending watchpoint single-step breakpoints. */ 847 watchpoint_single_step_handler(addr); 848 } 849 850 /* 851 * Called from either the Data Abort Handler [watchpoint] or the 852 * Prefetch Abort Handler [breakpoint] with interrupts disabled. 853 */ 854 static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr, 855 struct pt_regs *regs) 856 { 857 int ret = 0; 858 u32 dscr; 859 860 preempt_disable(); 861 862 if (interrupts_enabled(regs)) 863 local_irq_enable(); 864 865 /* We only handle watchpoints and hardware breakpoints. */ 866 ARM_DBG_READ(c0, c1, 0, dscr); 867 868 /* Perform perf callbacks. */ 869 switch (ARM_DSCR_MOE(dscr)) { 870 case ARM_ENTRY_BREAKPOINT: 871 breakpoint_handler(addr, regs); 872 break; 873 case ARM_ENTRY_ASYNC_WATCHPOINT: 874 WARN(1, "Asynchronous watchpoint exception taken. Debugging results may be unreliable\n"); 875 case ARM_ENTRY_SYNC_WATCHPOINT: 876 watchpoint_handler(addr, fsr, regs); 877 break; 878 default: 879 ret = 1; /* Unhandled fault. */ 880 } 881 882 preempt_enable(); 883 884 return ret; 885 } 886 887 /* 888 * One-time initialisation. 889 */ 890 static cpumask_t debug_err_mask; 891 892 static int debug_reg_trap(struct pt_regs *regs, unsigned int instr) 893 { 894 int cpu = smp_processor_id(); 895 896 pr_warn("Debug register access (0x%x) caused undefined instruction on CPU %d\n", 897 instr, cpu); 898 899 /* Set the error flag for this CPU and skip the faulting instruction. */ 900 cpumask_set_cpu(cpu, &debug_err_mask); 901 instruction_pointer(regs) += 4; 902 return 0; 903 } 904 905 static struct undef_hook debug_reg_hook = { 906 .instr_mask = 0x0fe80f10, 907 .instr_val = 0x0e000e10, 908 .fn = debug_reg_trap, 909 }; 910 911 /* Does this core support OS Save and Restore? */ 912 static bool core_has_os_save_restore(void) 913 { 914 u32 oslsr; 915 916 switch (get_debug_arch()) { 917 case ARM_DEBUG_ARCH_V7_1: 918 return true; 919 case ARM_DEBUG_ARCH_V7_ECP14: 920 ARM_DBG_READ(c1, c1, 4, oslsr); 921 if (oslsr & ARM_OSLSR_OSLM0) 922 return true; 923 default: 924 return false; 925 } 926 } 927 928 static void reset_ctrl_regs(unsigned int cpu) 929 { 930 int i, raw_num_brps, err = 0; 931 u32 val; 932 933 /* 934 * v7 debug contains save and restore registers so that debug state 935 * can be maintained across low-power modes without leaving the debug 936 * logic powered up. It is IMPLEMENTATION DEFINED whether we can access 937 * the debug registers out of reset, so we must unlock the OS Lock 938 * Access Register to avoid taking undefined instruction exceptions 939 * later on. 940 */ 941 switch (debug_arch) { 942 case ARM_DEBUG_ARCH_V6: 943 case ARM_DEBUG_ARCH_V6_1: 944 /* ARMv6 cores clear the registers out of reset. */ 945 goto out_mdbgen; 946 case ARM_DEBUG_ARCH_V7_ECP14: 947 /* 948 * Ensure sticky power-down is clear (i.e. debug logic is 949 * powered up). 950 */ 951 ARM_DBG_READ(c1, c5, 4, val); 952 if ((val & 0x1) == 0) 953 err = -EPERM; 954 955 if (!has_ossr) 956 goto clear_vcr; 957 break; 958 case ARM_DEBUG_ARCH_V7_1: 959 /* 960 * Ensure the OS double lock is clear. 961 */ 962 ARM_DBG_READ(c1, c3, 4, val); 963 if ((val & 0x1) == 1) 964 err = -EPERM; 965 break; 966 } 967 968 if (err) { 969 pr_warn_once("CPU %d debug is powered down!\n", cpu); 970 cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu)); 971 return; 972 } 973 974 /* 975 * Unconditionally clear the OS lock by writing a value 976 * other than CS_LAR_KEY to the access register. 977 */ 978 ARM_DBG_WRITE(c1, c0, 4, ~CORESIGHT_UNLOCK); 979 isb(); 980 981 /* 982 * Clear any configured vector-catch events before 983 * enabling monitor mode. 984 */ 985 clear_vcr: 986 ARM_DBG_WRITE(c0, c7, 0, 0); 987 isb(); 988 989 if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) { 990 pr_warn_once("CPU %d failed to disable vector catch\n", cpu); 991 return; 992 } 993 994 /* 995 * The control/value register pairs are UNKNOWN out of reset so 996 * clear them to avoid spurious debug events. 997 */ 998 raw_num_brps = get_num_brp_resources(); 999 for (i = 0; i < raw_num_brps; ++i) { 1000 write_wb_reg(ARM_BASE_BCR + i, 0UL); 1001 write_wb_reg(ARM_BASE_BVR + i, 0UL); 1002 } 1003 1004 for (i = 0; i < core_num_wrps; ++i) { 1005 write_wb_reg(ARM_BASE_WCR + i, 0UL); 1006 write_wb_reg(ARM_BASE_WVR + i, 0UL); 1007 } 1008 1009 if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) { 1010 pr_warn_once("CPU %d failed to clear debug register pairs\n", cpu); 1011 return; 1012 } 1013 1014 /* 1015 * Have a crack at enabling monitor mode. We don't actually need 1016 * it yet, but reporting an error early is useful if it fails. 1017 */ 1018 out_mdbgen: 1019 if (enable_monitor_mode()) 1020 cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu)); 1021 } 1022 1023 static int dbg_reset_online(unsigned int cpu) 1024 { 1025 local_irq_disable(); 1026 reset_ctrl_regs(cpu); 1027 local_irq_enable(); 1028 return 0; 1029 } 1030 1031 #ifdef CONFIG_CPU_PM 1032 static int dbg_cpu_pm_notify(struct notifier_block *self, unsigned long action, 1033 void *v) 1034 { 1035 if (action == CPU_PM_EXIT) 1036 reset_ctrl_regs(smp_processor_id()); 1037 1038 return NOTIFY_OK; 1039 } 1040 1041 static struct notifier_block dbg_cpu_pm_nb = { 1042 .notifier_call = dbg_cpu_pm_notify, 1043 }; 1044 1045 static void __init pm_init(void) 1046 { 1047 cpu_pm_register_notifier(&dbg_cpu_pm_nb); 1048 } 1049 #else 1050 static inline void pm_init(void) 1051 { 1052 } 1053 #endif 1054 1055 static int __init arch_hw_breakpoint_init(void) 1056 { 1057 int ret; 1058 1059 debug_arch = get_debug_arch(); 1060 1061 if (!debug_arch_supported()) { 1062 pr_info("debug architecture 0x%x unsupported.\n", debug_arch); 1063 return 0; 1064 } 1065 1066 /* 1067 * Scorpion CPUs (at least those in APQ8060) seem to set DBGPRSR.SPD 1068 * whenever a WFI is issued, even if the core is not powered down, in 1069 * violation of the architecture. When DBGPRSR.SPD is set, accesses to 1070 * breakpoint and watchpoint registers are treated as undefined, so 1071 * this results in boot time and runtime failures when these are 1072 * accessed and we unexpectedly take a trap. 1073 * 1074 * It's not clear if/how this can be worked around, so we blacklist 1075 * Scorpion CPUs to avoid these issues. 1076 */ 1077 if (read_cpuid_part() == ARM_CPU_PART_SCORPION) { 1078 pr_info("Scorpion CPU detected. Hardware breakpoints and watchpoints disabled\n"); 1079 return 0; 1080 } 1081 1082 has_ossr = core_has_os_save_restore(); 1083 1084 /* Determine how many BRPs/WRPs are available. */ 1085 core_num_brps = get_num_brps(); 1086 core_num_wrps = get_num_wrps(); 1087 1088 /* 1089 * We need to tread carefully here because DBGSWENABLE may be 1090 * driven low on this core and there isn't an architected way to 1091 * determine that. 1092 */ 1093 cpus_read_lock(); 1094 register_undef_hook(&debug_reg_hook); 1095 1096 /* 1097 * Register CPU notifier which resets the breakpoint resources. We 1098 * assume that a halting debugger will leave the world in a nice state 1099 * for us. 1100 */ 1101 ret = cpuhp_setup_state_cpuslocked(CPUHP_AP_ONLINE_DYN, 1102 "arm/hw_breakpoint:online", 1103 dbg_reset_online, NULL); 1104 unregister_undef_hook(&debug_reg_hook); 1105 if (WARN_ON(ret < 0) || !cpumask_empty(&debug_err_mask)) { 1106 core_num_brps = 0; 1107 core_num_wrps = 0; 1108 if (ret > 0) 1109 cpuhp_remove_state_nocalls_cpuslocked(ret); 1110 cpus_read_unlock(); 1111 return 0; 1112 } 1113 1114 pr_info("found %d " "%s" "breakpoint and %d watchpoint registers.\n", 1115 core_num_brps, core_has_mismatch_brps() ? "(+1 reserved) " : 1116 "", core_num_wrps); 1117 1118 /* Work out the maximum supported watchpoint length. */ 1119 max_watchpoint_len = get_max_wp_len(); 1120 pr_info("maximum watchpoint size is %u bytes.\n", 1121 max_watchpoint_len); 1122 1123 /* Register debug fault handler. */ 1124 hook_fault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP, 1125 TRAP_HWBKPT, "watchpoint debug exception"); 1126 hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP, 1127 TRAP_HWBKPT, "breakpoint debug exception"); 1128 cpus_read_unlock(); 1129 1130 /* Register PM notifiers. */ 1131 pm_init(); 1132 return 0; 1133 } 1134 arch_initcall(arch_hw_breakpoint_init); 1135 1136 void hw_breakpoint_pmu_read(struct perf_event *bp) 1137 { 1138 } 1139 1140 /* 1141 * Dummy function to register with die_notifier. 1142 */ 1143 int hw_breakpoint_exceptions_notify(struct notifier_block *unused, 1144 unsigned long val, void *data) 1145 { 1146 return NOTIFY_DONE; 1147 } 1148