1 /* 2 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, 3 * using the CPU's debug registers. 4 * 5 * Copyright (C) 2012 ARM Limited 6 * Author: Will Deacon <will.deacon@arm.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program. If not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #define pr_fmt(fmt) "hw-breakpoint: " fmt 22 23 #include <linux/compat.h> 24 #include <linux/cpu_pm.h> 25 #include <linux/errno.h> 26 #include <linux/hw_breakpoint.h> 27 #include <linux/kprobes.h> 28 #include <linux/perf_event.h> 29 #include <linux/ptrace.h> 30 #include <linux/smp.h> 31 #include <linux/uaccess.h> 32 33 #include <asm/compat.h> 34 #include <asm/current.h> 35 #include <asm/debug-monitors.h> 36 #include <asm/hw_breakpoint.h> 37 #include <asm/traps.h> 38 #include <asm/cputype.h> 39 #include <asm/system_misc.h> 40 41 /* Breakpoint currently in use for each BRP. */ 42 static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]); 43 44 /* Watchpoint currently in use for each WRP. */ 45 static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]); 46 47 /* Currently stepping a per-CPU kernel breakpoint. */ 48 static DEFINE_PER_CPU(int, stepping_kernel_bp); 49 50 /* Number of BRP/WRP registers on this CPU. */ 51 static int core_num_brps; 52 static int core_num_wrps; 53 54 int hw_breakpoint_slots(int type) 55 { 56 /* 57 * We can be called early, so don't rely on 58 * our static variables being initialised. 59 */ 60 switch (type) { 61 case TYPE_INST: 62 return get_num_brps(); 63 case TYPE_DATA: 64 return get_num_wrps(); 65 default: 66 pr_warning("unknown slot type: %d\n", type); 67 return 0; 68 } 69 } 70 71 #define READ_WB_REG_CASE(OFF, N, REG, VAL) \ 72 case (OFF + N): \ 73 AARCH64_DBG_READ(N, REG, VAL); \ 74 break 75 76 #define WRITE_WB_REG_CASE(OFF, N, REG, VAL) \ 77 case (OFF + N): \ 78 AARCH64_DBG_WRITE(N, REG, VAL); \ 79 break 80 81 #define GEN_READ_WB_REG_CASES(OFF, REG, VAL) \ 82 READ_WB_REG_CASE(OFF, 0, REG, VAL); \ 83 READ_WB_REG_CASE(OFF, 1, REG, VAL); \ 84 READ_WB_REG_CASE(OFF, 2, REG, VAL); \ 85 READ_WB_REG_CASE(OFF, 3, REG, VAL); \ 86 READ_WB_REG_CASE(OFF, 4, REG, VAL); \ 87 READ_WB_REG_CASE(OFF, 5, REG, VAL); \ 88 READ_WB_REG_CASE(OFF, 6, REG, VAL); \ 89 READ_WB_REG_CASE(OFF, 7, REG, VAL); \ 90 READ_WB_REG_CASE(OFF, 8, REG, VAL); \ 91 READ_WB_REG_CASE(OFF, 9, REG, VAL); \ 92 READ_WB_REG_CASE(OFF, 10, REG, VAL); \ 93 READ_WB_REG_CASE(OFF, 11, REG, VAL); \ 94 READ_WB_REG_CASE(OFF, 12, REG, VAL); \ 95 READ_WB_REG_CASE(OFF, 13, REG, VAL); \ 96 READ_WB_REG_CASE(OFF, 14, REG, VAL); \ 97 READ_WB_REG_CASE(OFF, 15, REG, VAL) 98 99 #define GEN_WRITE_WB_REG_CASES(OFF, REG, VAL) \ 100 WRITE_WB_REG_CASE(OFF, 0, REG, VAL); \ 101 WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \ 102 WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \ 103 WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \ 104 WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \ 105 WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \ 106 WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \ 107 WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \ 108 WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \ 109 WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \ 110 WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \ 111 WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \ 112 WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \ 113 WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \ 114 WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \ 115 WRITE_WB_REG_CASE(OFF, 15, REG, VAL) 116 117 static u64 read_wb_reg(int reg, int n) 118 { 119 u64 val = 0; 120 121 switch (reg + n) { 122 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); 123 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); 124 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val); 125 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val); 126 default: 127 pr_warning("attempt to read from unknown breakpoint register %d\n", n); 128 } 129 130 return val; 131 } 132 NOKPROBE_SYMBOL(read_wb_reg); 133 134 static void write_wb_reg(int reg, int n, u64 val) 135 { 136 switch (reg + n) { 137 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); 138 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); 139 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val); 140 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val); 141 default: 142 pr_warning("attempt to write to unknown breakpoint register %d\n", n); 143 } 144 isb(); 145 } 146 NOKPROBE_SYMBOL(write_wb_reg); 147 148 /* 149 * Convert a breakpoint privilege level to the corresponding exception 150 * level. 151 */ 152 static enum dbg_active_el debug_exception_level(int privilege) 153 { 154 switch (privilege) { 155 case AARCH64_BREAKPOINT_EL0: 156 return DBG_ACTIVE_EL0; 157 case AARCH64_BREAKPOINT_EL1: 158 return DBG_ACTIVE_EL1; 159 default: 160 pr_warning("invalid breakpoint privilege level %d\n", privilege); 161 return -EINVAL; 162 } 163 } 164 NOKPROBE_SYMBOL(debug_exception_level); 165 166 enum hw_breakpoint_ops { 167 HW_BREAKPOINT_INSTALL, 168 HW_BREAKPOINT_UNINSTALL, 169 HW_BREAKPOINT_RESTORE 170 }; 171 172 static int is_compat_bp(struct perf_event *bp) 173 { 174 struct task_struct *tsk = bp->hw.target; 175 176 /* 177 * tsk can be NULL for per-cpu (non-ptrace) breakpoints. 178 * In this case, use the native interface, since we don't have 179 * the notion of a "compat CPU" and could end up relying on 180 * deprecated behaviour if we use unaligned watchpoints in 181 * AArch64 state. 182 */ 183 return tsk && is_compat_thread(task_thread_info(tsk)); 184 } 185 186 /** 187 * hw_breakpoint_slot_setup - Find and setup a perf slot according to 188 * operations 189 * 190 * @slots: pointer to array of slots 191 * @max_slots: max number of slots 192 * @bp: perf_event to setup 193 * @ops: operation to be carried out on the slot 194 * 195 * Return: 196 * slot index on success 197 * -ENOSPC if no slot is available/matches 198 * -EINVAL on wrong operations parameter 199 */ 200 static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots, 201 struct perf_event *bp, 202 enum hw_breakpoint_ops ops) 203 { 204 int i; 205 struct perf_event **slot; 206 207 for (i = 0; i < max_slots; ++i) { 208 slot = &slots[i]; 209 switch (ops) { 210 case HW_BREAKPOINT_INSTALL: 211 if (!*slot) { 212 *slot = bp; 213 return i; 214 } 215 break; 216 case HW_BREAKPOINT_UNINSTALL: 217 if (*slot == bp) { 218 *slot = NULL; 219 return i; 220 } 221 break; 222 case HW_BREAKPOINT_RESTORE: 223 if (*slot == bp) 224 return i; 225 break; 226 default: 227 pr_warn_once("Unhandled hw breakpoint ops %d\n", ops); 228 return -EINVAL; 229 } 230 } 231 return -ENOSPC; 232 } 233 234 static int hw_breakpoint_control(struct perf_event *bp, 235 enum hw_breakpoint_ops ops) 236 { 237 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 238 struct perf_event **slots; 239 struct debug_info *debug_info = ¤t->thread.debug; 240 int i, max_slots, ctrl_reg, val_reg, reg_enable; 241 enum dbg_active_el dbg_el = debug_exception_level(info->ctrl.privilege); 242 u32 ctrl; 243 244 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { 245 /* Breakpoint */ 246 ctrl_reg = AARCH64_DBG_REG_BCR; 247 val_reg = AARCH64_DBG_REG_BVR; 248 slots = this_cpu_ptr(bp_on_reg); 249 max_slots = core_num_brps; 250 reg_enable = !debug_info->bps_disabled; 251 } else { 252 /* Watchpoint */ 253 ctrl_reg = AARCH64_DBG_REG_WCR; 254 val_reg = AARCH64_DBG_REG_WVR; 255 slots = this_cpu_ptr(wp_on_reg); 256 max_slots = core_num_wrps; 257 reg_enable = !debug_info->wps_disabled; 258 } 259 260 i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops); 261 262 if (WARN_ONCE(i < 0, "Can't find any breakpoint slot")) 263 return i; 264 265 switch (ops) { 266 case HW_BREAKPOINT_INSTALL: 267 /* 268 * Ensure debug monitors are enabled at the correct exception 269 * level. 270 */ 271 enable_debug_monitors(dbg_el); 272 /* Fall through */ 273 case HW_BREAKPOINT_RESTORE: 274 /* Setup the address register. */ 275 write_wb_reg(val_reg, i, info->address); 276 277 /* Setup the control register. */ 278 ctrl = encode_ctrl_reg(info->ctrl); 279 write_wb_reg(ctrl_reg, i, 280 reg_enable ? ctrl | 0x1 : ctrl & ~0x1); 281 break; 282 case HW_BREAKPOINT_UNINSTALL: 283 /* Reset the control register. */ 284 write_wb_reg(ctrl_reg, i, 0); 285 286 /* 287 * Release the debug monitors for the correct exception 288 * level. 289 */ 290 disable_debug_monitors(dbg_el); 291 break; 292 } 293 294 return 0; 295 } 296 297 /* 298 * Install a perf counter breakpoint. 299 */ 300 int arch_install_hw_breakpoint(struct perf_event *bp) 301 { 302 return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL); 303 } 304 305 void arch_uninstall_hw_breakpoint(struct perf_event *bp) 306 { 307 hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL); 308 } 309 310 static int get_hbp_len(u8 hbp_len) 311 { 312 unsigned int len_in_bytes = 0; 313 314 switch (hbp_len) { 315 case ARM_BREAKPOINT_LEN_1: 316 len_in_bytes = 1; 317 break; 318 case ARM_BREAKPOINT_LEN_2: 319 len_in_bytes = 2; 320 break; 321 case ARM_BREAKPOINT_LEN_3: 322 len_in_bytes = 3; 323 break; 324 case ARM_BREAKPOINT_LEN_4: 325 len_in_bytes = 4; 326 break; 327 case ARM_BREAKPOINT_LEN_5: 328 len_in_bytes = 5; 329 break; 330 case ARM_BREAKPOINT_LEN_6: 331 len_in_bytes = 6; 332 break; 333 case ARM_BREAKPOINT_LEN_7: 334 len_in_bytes = 7; 335 break; 336 case ARM_BREAKPOINT_LEN_8: 337 len_in_bytes = 8; 338 break; 339 } 340 341 return len_in_bytes; 342 } 343 344 /* 345 * Check whether bp virtual address is in kernel space. 346 */ 347 int arch_check_bp_in_kernelspace(struct perf_event *bp) 348 { 349 unsigned int len; 350 unsigned long va; 351 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 352 353 va = info->address; 354 len = get_hbp_len(info->ctrl.len); 355 356 return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); 357 } 358 359 /* 360 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl. 361 * Hopefully this will disappear when ptrace can bypass the conversion 362 * to generic breakpoint descriptions. 363 */ 364 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, 365 int *gen_len, int *gen_type, int *offset) 366 { 367 /* Type */ 368 switch (ctrl.type) { 369 case ARM_BREAKPOINT_EXECUTE: 370 *gen_type = HW_BREAKPOINT_X; 371 break; 372 case ARM_BREAKPOINT_LOAD: 373 *gen_type = HW_BREAKPOINT_R; 374 break; 375 case ARM_BREAKPOINT_STORE: 376 *gen_type = HW_BREAKPOINT_W; 377 break; 378 case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE: 379 *gen_type = HW_BREAKPOINT_RW; 380 break; 381 default: 382 return -EINVAL; 383 } 384 385 if (!ctrl.len) 386 return -EINVAL; 387 *offset = __ffs(ctrl.len); 388 389 /* Len */ 390 switch (ctrl.len >> *offset) { 391 case ARM_BREAKPOINT_LEN_1: 392 *gen_len = HW_BREAKPOINT_LEN_1; 393 break; 394 case ARM_BREAKPOINT_LEN_2: 395 *gen_len = HW_BREAKPOINT_LEN_2; 396 break; 397 case ARM_BREAKPOINT_LEN_3: 398 *gen_len = HW_BREAKPOINT_LEN_3; 399 break; 400 case ARM_BREAKPOINT_LEN_4: 401 *gen_len = HW_BREAKPOINT_LEN_4; 402 break; 403 case ARM_BREAKPOINT_LEN_5: 404 *gen_len = HW_BREAKPOINT_LEN_5; 405 break; 406 case ARM_BREAKPOINT_LEN_6: 407 *gen_len = HW_BREAKPOINT_LEN_6; 408 break; 409 case ARM_BREAKPOINT_LEN_7: 410 *gen_len = HW_BREAKPOINT_LEN_7; 411 break; 412 case ARM_BREAKPOINT_LEN_8: 413 *gen_len = HW_BREAKPOINT_LEN_8; 414 break; 415 default: 416 return -EINVAL; 417 } 418 419 return 0; 420 } 421 422 /* 423 * Construct an arch_hw_breakpoint from a perf_event. 424 */ 425 static int arch_build_bp_info(struct perf_event *bp) 426 { 427 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 428 429 /* Type */ 430 switch (bp->attr.bp_type) { 431 case HW_BREAKPOINT_X: 432 info->ctrl.type = ARM_BREAKPOINT_EXECUTE; 433 break; 434 case HW_BREAKPOINT_R: 435 info->ctrl.type = ARM_BREAKPOINT_LOAD; 436 break; 437 case HW_BREAKPOINT_W: 438 info->ctrl.type = ARM_BREAKPOINT_STORE; 439 break; 440 case HW_BREAKPOINT_RW: 441 info->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE; 442 break; 443 default: 444 return -EINVAL; 445 } 446 447 /* Len */ 448 switch (bp->attr.bp_len) { 449 case HW_BREAKPOINT_LEN_1: 450 info->ctrl.len = ARM_BREAKPOINT_LEN_1; 451 break; 452 case HW_BREAKPOINT_LEN_2: 453 info->ctrl.len = ARM_BREAKPOINT_LEN_2; 454 break; 455 case HW_BREAKPOINT_LEN_3: 456 info->ctrl.len = ARM_BREAKPOINT_LEN_3; 457 break; 458 case HW_BREAKPOINT_LEN_4: 459 info->ctrl.len = ARM_BREAKPOINT_LEN_4; 460 break; 461 case HW_BREAKPOINT_LEN_5: 462 info->ctrl.len = ARM_BREAKPOINT_LEN_5; 463 break; 464 case HW_BREAKPOINT_LEN_6: 465 info->ctrl.len = ARM_BREAKPOINT_LEN_6; 466 break; 467 case HW_BREAKPOINT_LEN_7: 468 info->ctrl.len = ARM_BREAKPOINT_LEN_7; 469 break; 470 case HW_BREAKPOINT_LEN_8: 471 info->ctrl.len = ARM_BREAKPOINT_LEN_8; 472 break; 473 default: 474 return -EINVAL; 475 } 476 477 /* 478 * On AArch64, we only permit breakpoints of length 4, whereas 479 * AArch32 also requires breakpoints of length 2 for Thumb. 480 * Watchpoints can be of length 1, 2, 4 or 8 bytes. 481 */ 482 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { 483 if (is_compat_bp(bp)) { 484 if (info->ctrl.len != ARM_BREAKPOINT_LEN_2 && 485 info->ctrl.len != ARM_BREAKPOINT_LEN_4) 486 return -EINVAL; 487 } else if (info->ctrl.len != ARM_BREAKPOINT_LEN_4) { 488 /* 489 * FIXME: Some tools (I'm looking at you perf) assume 490 * that breakpoints should be sizeof(long). This 491 * is nonsense. For now, we fix up the parameter 492 * but we should probably return -EINVAL instead. 493 */ 494 info->ctrl.len = ARM_BREAKPOINT_LEN_4; 495 } 496 } 497 498 /* Address */ 499 info->address = bp->attr.bp_addr; 500 501 /* 502 * Privilege 503 * Note that we disallow combined EL0/EL1 breakpoints because 504 * that would complicate the stepping code. 505 */ 506 if (arch_check_bp_in_kernelspace(bp)) 507 info->ctrl.privilege = AARCH64_BREAKPOINT_EL1; 508 else 509 info->ctrl.privilege = AARCH64_BREAKPOINT_EL0; 510 511 /* Enabled? */ 512 info->ctrl.enabled = !bp->attr.disabled; 513 514 return 0; 515 } 516 517 /* 518 * Validate the arch-specific HW Breakpoint register settings. 519 */ 520 int arch_validate_hwbkpt_settings(struct perf_event *bp) 521 { 522 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 523 int ret; 524 u64 alignment_mask, offset; 525 526 /* Build the arch_hw_breakpoint. */ 527 ret = arch_build_bp_info(bp); 528 if (ret) 529 return ret; 530 531 /* 532 * Check address alignment. 533 * We don't do any clever alignment correction for watchpoints 534 * because using 64-bit unaligned addresses is deprecated for 535 * AArch64. 536 * 537 * AArch32 tasks expect some simple alignment fixups, so emulate 538 * that here. 539 */ 540 if (is_compat_bp(bp)) { 541 if (info->ctrl.len == ARM_BREAKPOINT_LEN_8) 542 alignment_mask = 0x7; 543 else 544 alignment_mask = 0x3; 545 offset = info->address & alignment_mask; 546 switch (offset) { 547 case 0: 548 /* Aligned */ 549 break; 550 case 1: 551 /* Allow single byte watchpoint. */ 552 if (info->ctrl.len == ARM_BREAKPOINT_LEN_1) 553 break; 554 case 2: 555 /* Allow halfword watchpoints and breakpoints. */ 556 if (info->ctrl.len == ARM_BREAKPOINT_LEN_2) 557 break; 558 default: 559 return -EINVAL; 560 } 561 } else { 562 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) 563 alignment_mask = 0x3; 564 else 565 alignment_mask = 0x7; 566 offset = info->address & alignment_mask; 567 } 568 569 info->address &= ~alignment_mask; 570 info->ctrl.len <<= offset; 571 572 /* 573 * Disallow per-task kernel breakpoints since these would 574 * complicate the stepping code. 575 */ 576 if (info->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.target) 577 return -EINVAL; 578 579 return 0; 580 } 581 582 /* 583 * Enable/disable all of the breakpoints active at the specified 584 * exception level at the register level. 585 * This is used when single-stepping after a breakpoint exception. 586 */ 587 static void toggle_bp_registers(int reg, enum dbg_active_el el, int enable) 588 { 589 int i, max_slots, privilege; 590 u32 ctrl; 591 struct perf_event **slots; 592 593 switch (reg) { 594 case AARCH64_DBG_REG_BCR: 595 slots = this_cpu_ptr(bp_on_reg); 596 max_slots = core_num_brps; 597 break; 598 case AARCH64_DBG_REG_WCR: 599 slots = this_cpu_ptr(wp_on_reg); 600 max_slots = core_num_wrps; 601 break; 602 default: 603 return; 604 } 605 606 for (i = 0; i < max_slots; ++i) { 607 if (!slots[i]) 608 continue; 609 610 privilege = counter_arch_bp(slots[i])->ctrl.privilege; 611 if (debug_exception_level(privilege) != el) 612 continue; 613 614 ctrl = read_wb_reg(reg, i); 615 if (enable) 616 ctrl |= 0x1; 617 else 618 ctrl &= ~0x1; 619 write_wb_reg(reg, i, ctrl); 620 } 621 } 622 NOKPROBE_SYMBOL(toggle_bp_registers); 623 624 /* 625 * Debug exception handlers. 626 */ 627 static int breakpoint_handler(unsigned long unused, unsigned int esr, 628 struct pt_regs *regs) 629 { 630 int i, step = 0, *kernel_step; 631 u32 ctrl_reg; 632 u64 addr, val; 633 struct perf_event *bp, **slots; 634 struct debug_info *debug_info; 635 struct arch_hw_breakpoint_ctrl ctrl; 636 637 slots = this_cpu_ptr(bp_on_reg); 638 addr = instruction_pointer(regs); 639 debug_info = ¤t->thread.debug; 640 641 for (i = 0; i < core_num_brps; ++i) { 642 rcu_read_lock(); 643 644 bp = slots[i]; 645 646 if (bp == NULL) 647 goto unlock; 648 649 /* Check if the breakpoint value matches. */ 650 val = read_wb_reg(AARCH64_DBG_REG_BVR, i); 651 if (val != (addr & ~0x3)) 652 goto unlock; 653 654 /* Possible match, check the byte address select to confirm. */ 655 ctrl_reg = read_wb_reg(AARCH64_DBG_REG_BCR, i); 656 decode_ctrl_reg(ctrl_reg, &ctrl); 657 if (!((1 << (addr & 0x3)) & ctrl.len)) 658 goto unlock; 659 660 counter_arch_bp(bp)->trigger = addr; 661 perf_bp_event(bp, regs); 662 663 /* Do we need to handle the stepping? */ 664 if (is_default_overflow_handler(bp)) 665 step = 1; 666 unlock: 667 rcu_read_unlock(); 668 } 669 670 if (!step) 671 return 0; 672 673 if (user_mode(regs)) { 674 debug_info->bps_disabled = 1; 675 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 0); 676 677 /* If we're already stepping a watchpoint, just return. */ 678 if (debug_info->wps_disabled) 679 return 0; 680 681 if (test_thread_flag(TIF_SINGLESTEP)) 682 debug_info->suspended_step = 1; 683 else 684 user_enable_single_step(current); 685 } else { 686 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 0); 687 kernel_step = this_cpu_ptr(&stepping_kernel_bp); 688 689 if (*kernel_step != ARM_KERNEL_STEP_NONE) 690 return 0; 691 692 if (kernel_active_single_step()) { 693 *kernel_step = ARM_KERNEL_STEP_SUSPEND; 694 } else { 695 *kernel_step = ARM_KERNEL_STEP_ACTIVE; 696 kernel_enable_single_step(regs); 697 } 698 } 699 700 return 0; 701 } 702 NOKPROBE_SYMBOL(breakpoint_handler); 703 704 /* 705 * Arm64 hardware does not always report a watchpoint hit address that matches 706 * one of the watchpoints set. It can also report an address "near" the 707 * watchpoint if a single instruction access both watched and unwatched 708 * addresses. There is no straight-forward way, short of disassembling the 709 * offending instruction, to map that address back to the watchpoint. This 710 * function computes the distance of the memory access from the watchpoint as a 711 * heuristic for the likelyhood that a given access triggered the watchpoint. 712 * 713 * See Section D2.10.5 "Determining the memory location that caused a Watchpoint 714 * exception" of ARMv8 Architecture Reference Manual for details. 715 * 716 * The function returns the distance of the address from the bytes watched by 717 * the watchpoint. In case of an exact match, it returns 0. 718 */ 719 static u64 get_distance_from_watchpoint(unsigned long addr, u64 val, 720 struct arch_hw_breakpoint_ctrl *ctrl) 721 { 722 u64 wp_low, wp_high; 723 u32 lens, lene; 724 725 addr = untagged_addr(addr); 726 727 lens = __ffs(ctrl->len); 728 lene = __fls(ctrl->len); 729 730 wp_low = val + lens; 731 wp_high = val + lene; 732 if (addr < wp_low) 733 return wp_low - addr; 734 else if (addr > wp_high) 735 return addr - wp_high; 736 else 737 return 0; 738 } 739 740 static int watchpoint_handler(unsigned long addr, unsigned int esr, 741 struct pt_regs *regs) 742 { 743 int i, step = 0, *kernel_step, access, closest_match = 0; 744 u64 min_dist = -1, dist; 745 u32 ctrl_reg; 746 u64 val; 747 struct perf_event *wp, **slots; 748 struct debug_info *debug_info; 749 struct arch_hw_breakpoint *info; 750 struct arch_hw_breakpoint_ctrl ctrl; 751 752 slots = this_cpu_ptr(wp_on_reg); 753 debug_info = ¤t->thread.debug; 754 755 /* 756 * Find all watchpoints that match the reported address. If no exact 757 * match is found. Attribute the hit to the closest watchpoint. 758 */ 759 rcu_read_lock(); 760 for (i = 0; i < core_num_wrps; ++i) { 761 wp = slots[i]; 762 if (wp == NULL) 763 continue; 764 765 /* 766 * Check that the access type matches. 767 * 0 => load, otherwise => store 768 */ 769 access = (esr & AARCH64_ESR_ACCESS_MASK) ? HW_BREAKPOINT_W : 770 HW_BREAKPOINT_R; 771 if (!(access & hw_breakpoint_type(wp))) 772 continue; 773 774 /* Check if the watchpoint value and byte select match. */ 775 val = read_wb_reg(AARCH64_DBG_REG_WVR, i); 776 ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i); 777 decode_ctrl_reg(ctrl_reg, &ctrl); 778 dist = get_distance_from_watchpoint(addr, val, &ctrl); 779 if (dist < min_dist) { 780 min_dist = dist; 781 closest_match = i; 782 } 783 /* Is this an exact match? */ 784 if (dist != 0) 785 continue; 786 787 info = counter_arch_bp(wp); 788 info->trigger = addr; 789 perf_bp_event(wp, regs); 790 791 /* Do we need to handle the stepping? */ 792 if (is_default_overflow_handler(wp)) 793 step = 1; 794 } 795 if (min_dist > 0 && min_dist != -1) { 796 /* No exact match found. */ 797 wp = slots[closest_match]; 798 info = counter_arch_bp(wp); 799 info->trigger = addr; 800 perf_bp_event(wp, regs); 801 802 /* Do we need to handle the stepping? */ 803 if (is_default_overflow_handler(wp)) 804 step = 1; 805 } 806 rcu_read_unlock(); 807 808 if (!step) 809 return 0; 810 811 /* 812 * We always disable EL0 watchpoints because the kernel can 813 * cause these to fire via an unprivileged access. 814 */ 815 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 0); 816 817 if (user_mode(regs)) { 818 debug_info->wps_disabled = 1; 819 820 /* If we're already stepping a breakpoint, just return. */ 821 if (debug_info->bps_disabled) 822 return 0; 823 824 if (test_thread_flag(TIF_SINGLESTEP)) 825 debug_info->suspended_step = 1; 826 else 827 user_enable_single_step(current); 828 } else { 829 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 0); 830 kernel_step = this_cpu_ptr(&stepping_kernel_bp); 831 832 if (*kernel_step != ARM_KERNEL_STEP_NONE) 833 return 0; 834 835 if (kernel_active_single_step()) { 836 *kernel_step = ARM_KERNEL_STEP_SUSPEND; 837 } else { 838 *kernel_step = ARM_KERNEL_STEP_ACTIVE; 839 kernel_enable_single_step(regs); 840 } 841 } 842 843 return 0; 844 } 845 NOKPROBE_SYMBOL(watchpoint_handler); 846 847 /* 848 * Handle single-step exception. 849 */ 850 int reinstall_suspended_bps(struct pt_regs *regs) 851 { 852 struct debug_info *debug_info = ¤t->thread.debug; 853 int handled_exception = 0, *kernel_step; 854 855 kernel_step = this_cpu_ptr(&stepping_kernel_bp); 856 857 /* 858 * Called from single-step exception handler. 859 * Return 0 if execution can resume, 1 if a SIGTRAP should be 860 * reported. 861 */ 862 if (user_mode(regs)) { 863 if (debug_info->bps_disabled) { 864 debug_info->bps_disabled = 0; 865 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 1); 866 handled_exception = 1; 867 } 868 869 if (debug_info->wps_disabled) { 870 debug_info->wps_disabled = 0; 871 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1); 872 handled_exception = 1; 873 } 874 875 if (handled_exception) { 876 if (debug_info->suspended_step) { 877 debug_info->suspended_step = 0; 878 /* Allow exception handling to fall-through. */ 879 handled_exception = 0; 880 } else { 881 user_disable_single_step(current); 882 } 883 } 884 } else if (*kernel_step != ARM_KERNEL_STEP_NONE) { 885 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 1); 886 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 1); 887 888 if (!debug_info->wps_disabled) 889 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1); 890 891 if (*kernel_step != ARM_KERNEL_STEP_SUSPEND) { 892 kernel_disable_single_step(); 893 handled_exception = 1; 894 } else { 895 handled_exception = 0; 896 } 897 898 *kernel_step = ARM_KERNEL_STEP_NONE; 899 } 900 901 return !handled_exception; 902 } 903 NOKPROBE_SYMBOL(reinstall_suspended_bps); 904 905 /* 906 * Context-switcher for restoring suspended breakpoints. 907 */ 908 void hw_breakpoint_thread_switch(struct task_struct *next) 909 { 910 /* 911 * current next 912 * disabled: 0 0 => The usual case, NOTIFY_DONE 913 * 0 1 => Disable the registers 914 * 1 0 => Enable the registers 915 * 1 1 => NOTIFY_DONE. per-task bps will 916 * get taken care of by perf. 917 */ 918 919 struct debug_info *current_debug_info, *next_debug_info; 920 921 current_debug_info = ¤t->thread.debug; 922 next_debug_info = &next->thread.debug; 923 924 /* Update breakpoints. */ 925 if (current_debug_info->bps_disabled != next_debug_info->bps_disabled) 926 toggle_bp_registers(AARCH64_DBG_REG_BCR, 927 DBG_ACTIVE_EL0, 928 !next_debug_info->bps_disabled); 929 930 /* Update watchpoints. */ 931 if (current_debug_info->wps_disabled != next_debug_info->wps_disabled) 932 toggle_bp_registers(AARCH64_DBG_REG_WCR, 933 DBG_ACTIVE_EL0, 934 !next_debug_info->wps_disabled); 935 } 936 937 /* 938 * CPU initialisation. 939 */ 940 static int hw_breakpoint_reset(unsigned int cpu) 941 { 942 int i; 943 struct perf_event **slots; 944 /* 945 * When a CPU goes through cold-boot, it does not have any installed 946 * slot, so it is safe to share the same function for restoring and 947 * resetting breakpoints; when a CPU is hotplugged in, it goes 948 * through the slots, which are all empty, hence it just resets control 949 * and value for debug registers. 950 * When this function is triggered on warm-boot through a CPU PM 951 * notifier some slots might be initialized; if so they are 952 * reprogrammed according to the debug slots content. 953 */ 954 for (slots = this_cpu_ptr(bp_on_reg), i = 0; i < core_num_brps; ++i) { 955 if (slots[i]) { 956 hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE); 957 } else { 958 write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL); 959 write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL); 960 } 961 } 962 963 for (slots = this_cpu_ptr(wp_on_reg), i = 0; i < core_num_wrps; ++i) { 964 if (slots[i]) { 965 hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE); 966 } else { 967 write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL); 968 write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL); 969 } 970 } 971 972 return 0; 973 } 974 975 #ifdef CONFIG_CPU_PM 976 extern void cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int)); 977 #else 978 static inline void cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int)) 979 { 980 } 981 #endif 982 983 /* 984 * One-time initialisation. 985 */ 986 static int __init arch_hw_breakpoint_init(void) 987 { 988 int ret; 989 990 core_num_brps = get_num_brps(); 991 core_num_wrps = get_num_wrps(); 992 993 pr_info("found %d breakpoint and %d watchpoint registers.\n", 994 core_num_brps, core_num_wrps); 995 996 /* Register debug fault handlers. */ 997 hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP, 998 TRAP_HWBKPT, "hw-breakpoint handler"); 999 hook_debug_fault_code(DBG_ESR_EVT_HWWP, watchpoint_handler, SIGTRAP, 1000 TRAP_HWBKPT, "hw-watchpoint handler"); 1001 1002 /* 1003 * Reset the breakpoint resources. We assume that a halting 1004 * debugger will leave the world in a nice state for us. 1005 */ 1006 ret = cpuhp_setup_state(CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING, 1007 "perf/arm64/hw_breakpoint:starting", 1008 hw_breakpoint_reset, NULL); 1009 if (ret) 1010 pr_err("failed to register CPU hotplug notifier: %d\n", ret); 1011 1012 /* Register cpu_suspend hw breakpoint restore hook */ 1013 cpu_suspend_set_dbg_restorer(hw_breakpoint_reset); 1014 1015 return ret; 1016 } 1017 arch_initcall(arch_hw_breakpoint_init); 1018 1019 void hw_breakpoint_pmu_read(struct perf_event *bp) 1020 { 1021 } 1022 1023 /* 1024 * Dummy function to register with die_notifier. 1025 */ 1026 int hw_breakpoint_exceptions_notify(struct notifier_block *unused, 1027 unsigned long val, void *data) 1028 { 1029 return NOTIFY_DONE; 1030 } 1031