1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2023 Loongson Technology Corporation Limited 4 */ 5 #define pr_fmt(fmt) "hw-breakpoint: " fmt 6 7 #include <linux/hw_breakpoint.h> 8 #include <linux/kprobes.h> 9 #include <linux/perf_event.h> 10 11 #include <asm/hw_breakpoint.h> 12 13 /* Breakpoint currently in use for each BRP. */ 14 static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[LOONGARCH_MAX_BRP]); 15 16 /* Watchpoint currently in use for each WRP. */ 17 static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[LOONGARCH_MAX_WRP]); 18 19 int hw_breakpoint_slots(int type) 20 { 21 /* 22 * We can be called early, so don't rely on 23 * our static variables being initialised. 24 */ 25 switch (type) { 26 case TYPE_INST: 27 return get_num_brps(); 28 case TYPE_DATA: 29 return get_num_wrps(); 30 default: 31 pr_warn("unknown slot type: %d\n", type); 32 return 0; 33 } 34 } 35 36 #define READ_WB_REG_CASE(OFF, N, REG, T, VAL) \ 37 case (OFF + N): \ 38 LOONGARCH_CSR_WATCH_READ(N, REG, T, VAL); \ 39 break 40 41 #define WRITE_WB_REG_CASE(OFF, N, REG, T, VAL) \ 42 case (OFF + N): \ 43 LOONGARCH_CSR_WATCH_WRITE(N, REG, T, VAL); \ 44 break 45 46 #define GEN_READ_WB_REG_CASES(OFF, REG, T, VAL) \ 47 READ_WB_REG_CASE(OFF, 0, REG, T, VAL); \ 48 READ_WB_REG_CASE(OFF, 1, REG, T, VAL); \ 49 READ_WB_REG_CASE(OFF, 2, REG, T, VAL); \ 50 READ_WB_REG_CASE(OFF, 3, REG, T, VAL); \ 51 READ_WB_REG_CASE(OFF, 4, REG, T, VAL); \ 52 READ_WB_REG_CASE(OFF, 5, REG, T, VAL); \ 53 READ_WB_REG_CASE(OFF, 6, REG, T, VAL); \ 54 READ_WB_REG_CASE(OFF, 7, REG, T, VAL); 55 56 #define GEN_WRITE_WB_REG_CASES(OFF, REG, T, VAL) \ 57 WRITE_WB_REG_CASE(OFF, 0, REG, T, VAL); \ 58 WRITE_WB_REG_CASE(OFF, 1, REG, T, VAL); \ 59 WRITE_WB_REG_CASE(OFF, 2, REG, T, VAL); \ 60 WRITE_WB_REG_CASE(OFF, 3, REG, T, VAL); \ 61 WRITE_WB_REG_CASE(OFF, 4, REG, T, VAL); \ 62 WRITE_WB_REG_CASE(OFF, 5, REG, T, VAL); \ 63 WRITE_WB_REG_CASE(OFF, 6, REG, T, VAL); \ 64 WRITE_WB_REG_CASE(OFF, 7, REG, T, VAL); 65 66 static u64 read_wb_reg(int reg, int n, int t) 67 { 68 u64 val = 0; 69 70 switch (reg + n) { 71 GEN_READ_WB_REG_CASES(CSR_CFG_ADDR, ADDR, t, val); 72 GEN_READ_WB_REG_CASES(CSR_CFG_MASK, MASK, t, val); 73 GEN_READ_WB_REG_CASES(CSR_CFG_CTRL, CTRL, t, val); 74 GEN_READ_WB_REG_CASES(CSR_CFG_ASID, ASID, t, val); 75 default: 76 pr_warn("Attempt to read from unknown breakpoint register %d\n", n); 77 } 78 79 return val; 80 } 81 NOKPROBE_SYMBOL(read_wb_reg); 82 83 static void write_wb_reg(int reg, int n, int t, u64 val) 84 { 85 switch (reg + n) { 86 GEN_WRITE_WB_REG_CASES(CSR_CFG_ADDR, ADDR, t, val); 87 GEN_WRITE_WB_REG_CASES(CSR_CFG_MASK, MASK, t, val); 88 GEN_WRITE_WB_REG_CASES(CSR_CFG_CTRL, CTRL, t, val); 89 GEN_WRITE_WB_REG_CASES(CSR_CFG_ASID, ASID, t, val); 90 default: 91 pr_warn("Attempt to write to unknown breakpoint register %d\n", n); 92 } 93 } 94 NOKPROBE_SYMBOL(write_wb_reg); 95 96 enum hw_breakpoint_ops { 97 HW_BREAKPOINT_INSTALL, 98 HW_BREAKPOINT_UNINSTALL, 99 }; 100 101 /* 102 * hw_breakpoint_slot_setup - Find and setup a perf slot according to operations 103 * 104 * @slots: pointer to array of slots 105 * @max_slots: max number of slots 106 * @bp: perf_event to setup 107 * @ops: operation to be carried out on the slot 108 * 109 * Return: 110 * slot index on success 111 * -ENOSPC if no slot is available/matches 112 * -EINVAL on wrong operations parameter 113 */ 114 115 static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots, 116 struct perf_event *bp, enum hw_breakpoint_ops ops) 117 { 118 int i; 119 struct perf_event **slot; 120 121 for (i = 0; i < max_slots; ++i) { 122 slot = &slots[i]; 123 switch (ops) { 124 case HW_BREAKPOINT_INSTALL: 125 if (!*slot) { 126 *slot = bp; 127 return i; 128 } 129 break; 130 case HW_BREAKPOINT_UNINSTALL: 131 if (*slot == bp) { 132 *slot = NULL; 133 return i; 134 } 135 break; 136 default: 137 pr_warn_once("Unhandled hw breakpoint ops %d\n", ops); 138 return -EINVAL; 139 } 140 } 141 142 return -ENOSPC; 143 } 144 145 void ptrace_hw_copy_thread(struct task_struct *tsk) 146 { 147 memset(tsk->thread.hbp_break, 0, sizeof(tsk->thread.hbp_break)); 148 memset(tsk->thread.hbp_watch, 0, sizeof(tsk->thread.hbp_watch)); 149 } 150 151 /* 152 * Unregister breakpoints from this task and reset the pointers in the thread_struct. 153 */ 154 void flush_ptrace_hw_breakpoint(struct task_struct *tsk) 155 { 156 int i; 157 struct thread_struct *t = &tsk->thread; 158 159 for (i = 0; i < LOONGARCH_MAX_BRP; i++) { 160 if (t->hbp_break[i]) { 161 unregister_hw_breakpoint(t->hbp_break[i]); 162 t->hbp_break[i] = NULL; 163 } 164 } 165 166 for (i = 0; i < LOONGARCH_MAX_WRP; i++) { 167 if (t->hbp_watch[i]) { 168 unregister_hw_breakpoint(t->hbp_watch[i]); 169 t->hbp_watch[i] = NULL; 170 } 171 } 172 } 173 174 static int hw_breakpoint_control(struct perf_event *bp, 175 enum hw_breakpoint_ops ops) 176 { 177 u32 ctrl, privilege; 178 int i, max_slots, enable; 179 struct pt_regs *regs; 180 struct perf_event **slots; 181 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 182 183 if (arch_check_bp_in_kernelspace(info)) 184 privilege = CTRL_PLV0_ENABLE; 185 else 186 privilege = CTRL_PLV3_ENABLE; 187 188 /* Whether bp belongs to a task. */ 189 if (bp->hw.target) 190 regs = task_pt_regs(bp->hw.target); 191 192 if (info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) { 193 /* Breakpoint */ 194 slots = this_cpu_ptr(bp_on_reg); 195 max_slots = boot_cpu_data.watch_ireg_count; 196 } else { 197 /* Watchpoint */ 198 slots = this_cpu_ptr(wp_on_reg); 199 max_slots = boot_cpu_data.watch_dreg_count; 200 } 201 202 i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops); 203 204 if (WARN_ONCE(i < 0, "Can't find any breakpoint slot")) 205 return i; 206 207 switch (ops) { 208 case HW_BREAKPOINT_INSTALL: 209 /* Set the FWPnCFG/MWPnCFG 1~4 register. */ 210 if (info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) { 211 write_wb_reg(CSR_CFG_ADDR, i, 0, info->address); 212 write_wb_reg(CSR_CFG_MASK, i, 0, info->mask); 213 write_wb_reg(CSR_CFG_ASID, i, 0, 0); 214 write_wb_reg(CSR_CFG_CTRL, i, 0, privilege); 215 } else { 216 write_wb_reg(CSR_CFG_ADDR, i, 1, info->address); 217 write_wb_reg(CSR_CFG_MASK, i, 1, info->mask); 218 write_wb_reg(CSR_CFG_ASID, i, 1, 0); 219 ctrl = encode_ctrl_reg(info->ctrl); 220 write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl | privilege); 221 } 222 enable = csr_read64(LOONGARCH_CSR_CRMD); 223 csr_write64(CSR_CRMD_WE | enable, LOONGARCH_CSR_CRMD); 224 if (bp->hw.target) 225 regs->csr_prmd |= CSR_PRMD_PWE; 226 break; 227 case HW_BREAKPOINT_UNINSTALL: 228 /* Reset the FWPnCFG/MWPnCFG 1~4 register. */ 229 if (info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) { 230 write_wb_reg(CSR_CFG_ADDR, i, 0, 0); 231 write_wb_reg(CSR_CFG_MASK, i, 0, 0); 232 write_wb_reg(CSR_CFG_CTRL, i, 0, 0); 233 write_wb_reg(CSR_CFG_ASID, i, 0, 0); 234 } else { 235 write_wb_reg(CSR_CFG_ADDR, i, 1, 0); 236 write_wb_reg(CSR_CFG_MASK, i, 1, 0); 237 write_wb_reg(CSR_CFG_CTRL, i, 1, 0); 238 write_wb_reg(CSR_CFG_ASID, i, 1, 0); 239 } 240 if (bp->hw.target) 241 regs->csr_prmd &= ~CSR_PRMD_PWE; 242 break; 243 } 244 245 return 0; 246 } 247 248 /* 249 * Install a perf counter breakpoint. 250 */ 251 int arch_install_hw_breakpoint(struct perf_event *bp) 252 { 253 return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL); 254 } 255 256 void arch_uninstall_hw_breakpoint(struct perf_event *bp) 257 { 258 hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL); 259 } 260 261 static int get_hbp_len(u8 hbp_len) 262 { 263 unsigned int len_in_bytes = 0; 264 265 switch (hbp_len) { 266 case LOONGARCH_BREAKPOINT_LEN_1: 267 len_in_bytes = 1; 268 break; 269 case LOONGARCH_BREAKPOINT_LEN_2: 270 len_in_bytes = 2; 271 break; 272 case LOONGARCH_BREAKPOINT_LEN_4: 273 len_in_bytes = 4; 274 break; 275 case LOONGARCH_BREAKPOINT_LEN_8: 276 len_in_bytes = 8; 277 break; 278 } 279 280 return len_in_bytes; 281 } 282 283 /* 284 * Check whether bp virtual address is in kernel space. 285 */ 286 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) 287 { 288 unsigned int len; 289 unsigned long va; 290 291 va = hw->address; 292 len = get_hbp_len(hw->ctrl.len); 293 294 return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); 295 } 296 297 /* 298 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl. 299 * Hopefully this will disappear when ptrace can bypass the conversion 300 * to generic breakpoint descriptions. 301 */ 302 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, 303 int *gen_len, int *gen_type) 304 { 305 /* Type */ 306 switch (ctrl.type) { 307 case LOONGARCH_BREAKPOINT_EXECUTE: 308 *gen_type = HW_BREAKPOINT_X; 309 break; 310 case LOONGARCH_BREAKPOINT_LOAD: 311 *gen_type = HW_BREAKPOINT_R; 312 break; 313 case LOONGARCH_BREAKPOINT_STORE: 314 *gen_type = HW_BREAKPOINT_W; 315 break; 316 case LOONGARCH_BREAKPOINT_LOAD | LOONGARCH_BREAKPOINT_STORE: 317 *gen_type = HW_BREAKPOINT_RW; 318 break; 319 default: 320 return -EINVAL; 321 } 322 323 /* Len */ 324 switch (ctrl.len) { 325 case LOONGARCH_BREAKPOINT_LEN_1: 326 *gen_len = HW_BREAKPOINT_LEN_1; 327 break; 328 case LOONGARCH_BREAKPOINT_LEN_2: 329 *gen_len = HW_BREAKPOINT_LEN_2; 330 break; 331 case LOONGARCH_BREAKPOINT_LEN_4: 332 *gen_len = HW_BREAKPOINT_LEN_4; 333 break; 334 case LOONGARCH_BREAKPOINT_LEN_8: 335 *gen_len = HW_BREAKPOINT_LEN_8; 336 break; 337 default: 338 return -EINVAL; 339 } 340 341 return 0; 342 } 343 344 /* 345 * Construct an arch_hw_breakpoint from a perf_event. 346 */ 347 static int arch_build_bp_info(struct perf_event *bp, 348 const struct perf_event_attr *attr, 349 struct arch_hw_breakpoint *hw) 350 { 351 /* Type */ 352 switch (attr->bp_type) { 353 case HW_BREAKPOINT_X: 354 hw->ctrl.type = LOONGARCH_BREAKPOINT_EXECUTE; 355 break; 356 case HW_BREAKPOINT_R: 357 hw->ctrl.type = LOONGARCH_BREAKPOINT_LOAD; 358 break; 359 case HW_BREAKPOINT_W: 360 hw->ctrl.type = LOONGARCH_BREAKPOINT_STORE; 361 break; 362 case HW_BREAKPOINT_RW: 363 hw->ctrl.type = LOONGARCH_BREAKPOINT_LOAD | LOONGARCH_BREAKPOINT_STORE; 364 break; 365 default: 366 return -EINVAL; 367 } 368 369 /* Len */ 370 switch (attr->bp_len) { 371 case HW_BREAKPOINT_LEN_1: 372 hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_1; 373 break; 374 case HW_BREAKPOINT_LEN_2: 375 hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_2; 376 break; 377 case HW_BREAKPOINT_LEN_4: 378 hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_4; 379 break; 380 case HW_BREAKPOINT_LEN_8: 381 hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_8; 382 break; 383 default: 384 return -EINVAL; 385 } 386 387 /* Address */ 388 hw->address = attr->bp_addr; 389 390 return 0; 391 } 392 393 /* 394 * Validate the arch-specific HW Breakpoint register settings. 395 */ 396 int hw_breakpoint_arch_parse(struct perf_event *bp, 397 const struct perf_event_attr *attr, 398 struct arch_hw_breakpoint *hw) 399 { 400 int ret; 401 u64 alignment_mask; 402 403 /* Build the arch_hw_breakpoint. */ 404 ret = arch_build_bp_info(bp, attr, hw); 405 if (ret) 406 return ret; 407 408 if (hw->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) { 409 alignment_mask = 0x3; 410 hw->address &= ~alignment_mask; 411 } 412 413 return 0; 414 } 415 416 static void update_bp_registers(struct pt_regs *regs, int enable, int type) 417 { 418 u32 ctrl; 419 int i, max_slots; 420 struct perf_event **slots; 421 struct arch_hw_breakpoint *info; 422 423 switch (type) { 424 case 0: 425 slots = this_cpu_ptr(bp_on_reg); 426 max_slots = boot_cpu_data.watch_ireg_count; 427 break; 428 case 1: 429 slots = this_cpu_ptr(wp_on_reg); 430 max_slots = boot_cpu_data.watch_dreg_count; 431 break; 432 default: 433 return; 434 } 435 436 for (i = 0; i < max_slots; ++i) { 437 if (!slots[i]) 438 continue; 439 440 info = counter_arch_bp(slots[i]); 441 if (enable) { 442 if ((info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) && (type == 0)) { 443 write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE); 444 write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE); 445 } else { 446 ctrl = read_wb_reg(CSR_CFG_CTRL, i, 1); 447 if (info->ctrl.type == LOONGARCH_BREAKPOINT_LOAD) 448 ctrl |= 0x1 << MWPnCFG3_LoadEn; 449 if (info->ctrl.type == LOONGARCH_BREAKPOINT_STORE) 450 ctrl |= 0x1 << MWPnCFG3_StoreEn; 451 write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl); 452 } 453 regs->csr_prmd |= CSR_PRMD_PWE; 454 } else { 455 if ((info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) && (type == 0)) { 456 write_wb_reg(CSR_CFG_CTRL, i, 0, 0); 457 } else { 458 ctrl = read_wb_reg(CSR_CFG_CTRL, i, 1); 459 if (info->ctrl.type == LOONGARCH_BREAKPOINT_LOAD) 460 ctrl &= ~0x1 << MWPnCFG3_LoadEn; 461 if (info->ctrl.type == LOONGARCH_BREAKPOINT_STORE) 462 ctrl &= ~0x1 << MWPnCFG3_StoreEn; 463 write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl); 464 } 465 regs->csr_prmd &= ~CSR_PRMD_PWE; 466 } 467 } 468 } 469 NOKPROBE_SYMBOL(update_bp_registers); 470 471 /* 472 * Debug exception handlers. 473 */ 474 void breakpoint_handler(struct pt_regs *regs) 475 { 476 int i; 477 struct perf_event *bp, **slots; 478 479 slots = this_cpu_ptr(bp_on_reg); 480 481 for (i = 0; i < boot_cpu_data.watch_ireg_count; ++i) { 482 if ((csr_read32(LOONGARCH_CSR_FWPS) & (0x1 << i))) { 483 bp = slots[i]; 484 if (bp == NULL) 485 continue; 486 perf_bp_event(bp, regs); 487 csr_write32(0x1 << i, LOONGARCH_CSR_FWPS); 488 update_bp_registers(regs, 0, 0); 489 } 490 } 491 } 492 NOKPROBE_SYMBOL(breakpoint_handler); 493 494 void watchpoint_handler(struct pt_regs *regs) 495 { 496 int i; 497 struct perf_event *wp, **slots; 498 499 slots = this_cpu_ptr(wp_on_reg); 500 501 for (i = 0; i < boot_cpu_data.watch_dreg_count; ++i) { 502 if ((csr_read32(LOONGARCH_CSR_MWPS) & (0x1 << i))) { 503 wp = slots[i]; 504 if (wp == NULL) 505 continue; 506 perf_bp_event(wp, regs); 507 csr_write32(0x1 << i, LOONGARCH_CSR_MWPS); 508 update_bp_registers(regs, 0, 1); 509 } 510 } 511 } 512 NOKPROBE_SYMBOL(watchpoint_handler); 513 514 static int __init arch_hw_breakpoint_init(void) 515 { 516 int cpu; 517 518 boot_cpu_data.watch_ireg_count = get_num_brps(); 519 boot_cpu_data.watch_dreg_count = get_num_wrps(); 520 521 pr_info("Found %d breakpoint and %d watchpoint registers.\n", 522 boot_cpu_data.watch_ireg_count, boot_cpu_data.watch_dreg_count); 523 524 for (cpu = 1; cpu < NR_CPUS; cpu++) { 525 cpu_data[cpu].watch_ireg_count = boot_cpu_data.watch_ireg_count; 526 cpu_data[cpu].watch_dreg_count = boot_cpu_data.watch_dreg_count; 527 } 528 529 return 0; 530 } 531 arch_initcall(arch_hw_breakpoint_init); 532 533 void hw_breakpoint_thread_switch(struct task_struct *next) 534 { 535 u64 addr, mask; 536 struct pt_regs *regs = task_pt_regs(next); 537 538 if (test_tsk_thread_flag(next, TIF_SINGLESTEP)) { 539 addr = read_wb_reg(CSR_CFG_ADDR, 0, 0); 540 mask = read_wb_reg(CSR_CFG_MASK, 0, 0); 541 if (!((regs->csr_era ^ addr) & ~mask)) 542 csr_write32(CSR_FWPC_SKIP, LOONGARCH_CSR_FWPS); 543 regs->csr_prmd |= CSR_PRMD_PWE; 544 } else { 545 /* Update breakpoints */ 546 update_bp_registers(regs, 1, 0); 547 /* Update watchpoints */ 548 update_bp_registers(regs, 1, 1); 549 } 550 } 551 552 void hw_breakpoint_pmu_read(struct perf_event *bp) 553 { 554 } 555 556 /* 557 * Dummy function to register with die_notifier. 558 */ 559 int hw_breakpoint_exceptions_notify(struct notifier_block *unused, 560 unsigned long val, void *data) 561 { 562 return NOTIFY_DONE; 563 } 564