1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2023 Loongson Technology Corporation Limited 4 */ 5 #define pr_fmt(fmt) "hw-breakpoint: " fmt 6 7 #include <linux/hw_breakpoint.h> 8 #include <linux/kprobes.h> 9 #include <linux/perf_event.h> 10 11 #include <asm/hw_breakpoint.h> 12 13 /* Breakpoint currently in use for each BRP. */ 14 static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[LOONGARCH_MAX_BRP]); 15 16 /* Watchpoint currently in use for each WRP. */ 17 static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[LOONGARCH_MAX_WRP]); 18 19 int hw_breakpoint_slots(int type) 20 { 21 /* 22 * We can be called early, so don't rely on 23 * our static variables being initialised. 24 */ 25 switch (type) { 26 case TYPE_INST: 27 return get_num_brps(); 28 case TYPE_DATA: 29 return get_num_wrps(); 30 default: 31 pr_warn("unknown slot type: %d\n", type); 32 return 0; 33 } 34 } 35 36 #define READ_WB_REG_CASE(OFF, N, REG, T, VAL) \ 37 case (OFF + N): \ 38 LOONGARCH_CSR_WATCH_READ(N, REG, T, VAL); \ 39 break 40 41 #define WRITE_WB_REG_CASE(OFF, N, REG, T, VAL) \ 42 case (OFF + N): \ 43 LOONGARCH_CSR_WATCH_WRITE(N, REG, T, VAL); \ 44 break 45 46 #define GEN_READ_WB_REG_CASES(OFF, REG, T, VAL) \ 47 READ_WB_REG_CASE(OFF, 0, REG, T, VAL); \ 48 READ_WB_REG_CASE(OFF, 1, REG, T, VAL); \ 49 READ_WB_REG_CASE(OFF, 2, REG, T, VAL); \ 50 READ_WB_REG_CASE(OFF, 3, REG, T, VAL); \ 51 READ_WB_REG_CASE(OFF, 4, REG, T, VAL); \ 52 READ_WB_REG_CASE(OFF, 5, REG, T, VAL); \ 53 READ_WB_REG_CASE(OFF, 6, REG, T, VAL); \ 54 READ_WB_REG_CASE(OFF, 7, REG, T, VAL); 55 56 #define GEN_WRITE_WB_REG_CASES(OFF, REG, T, VAL) \ 57 WRITE_WB_REG_CASE(OFF, 0, REG, T, VAL); \ 58 WRITE_WB_REG_CASE(OFF, 1, REG, T, VAL); \ 59 WRITE_WB_REG_CASE(OFF, 2, REG, T, VAL); \ 60 WRITE_WB_REG_CASE(OFF, 3, REG, T, VAL); \ 61 WRITE_WB_REG_CASE(OFF, 4, REG, T, VAL); \ 62 WRITE_WB_REG_CASE(OFF, 5, REG, T, VAL); \ 63 WRITE_WB_REG_CASE(OFF, 6, REG, T, VAL); \ 64 WRITE_WB_REG_CASE(OFF, 7, REG, T, VAL); 65 66 static u64 read_wb_reg(int reg, int n, int t) 67 { 68 u64 val = 0; 69 70 switch (reg + n) { 71 GEN_READ_WB_REG_CASES(CSR_CFG_ADDR, ADDR, t, val); 72 GEN_READ_WB_REG_CASES(CSR_CFG_MASK, MASK, t, val); 73 GEN_READ_WB_REG_CASES(CSR_CFG_CTRL, CTRL, t, val); 74 GEN_READ_WB_REG_CASES(CSR_CFG_ASID, ASID, t, val); 75 default: 76 pr_warn("Attempt to read from unknown breakpoint register %d\n", n); 77 } 78 79 return val; 80 } 81 NOKPROBE_SYMBOL(read_wb_reg); 82 83 static void write_wb_reg(int reg, int n, int t, u64 val) 84 { 85 switch (reg + n) { 86 GEN_WRITE_WB_REG_CASES(CSR_CFG_ADDR, ADDR, t, val); 87 GEN_WRITE_WB_REG_CASES(CSR_CFG_MASK, MASK, t, val); 88 GEN_WRITE_WB_REG_CASES(CSR_CFG_CTRL, CTRL, t, val); 89 GEN_WRITE_WB_REG_CASES(CSR_CFG_ASID, ASID, t, val); 90 default: 91 pr_warn("Attempt to write to unknown breakpoint register %d\n", n); 92 } 93 } 94 NOKPROBE_SYMBOL(write_wb_reg); 95 96 enum hw_breakpoint_ops { 97 HW_BREAKPOINT_INSTALL, 98 HW_BREAKPOINT_UNINSTALL, 99 }; 100 101 /* 102 * hw_breakpoint_slot_setup - Find and setup a perf slot according to operations 103 * 104 * @slots: pointer to array of slots 105 * @max_slots: max number of slots 106 * @bp: perf_event to setup 107 * @ops: operation to be carried out on the slot 108 * 109 * Return: 110 * slot index on success 111 * -ENOSPC if no slot is available/matches 112 * -EINVAL on wrong operations parameter 113 */ 114 115 static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots, 116 struct perf_event *bp, enum hw_breakpoint_ops ops) 117 { 118 int i; 119 struct perf_event **slot; 120 121 for (i = 0; i < max_slots; ++i) { 122 slot = &slots[i]; 123 switch (ops) { 124 case HW_BREAKPOINT_INSTALL: 125 if (!*slot) { 126 *slot = bp; 127 return i; 128 } 129 break; 130 case HW_BREAKPOINT_UNINSTALL: 131 if (*slot == bp) { 132 *slot = NULL; 133 return i; 134 } 135 break; 136 default: 137 pr_warn_once("Unhandled hw breakpoint ops %d\n", ops); 138 return -EINVAL; 139 } 140 } 141 142 return -ENOSPC; 143 } 144 145 void ptrace_hw_copy_thread(struct task_struct *tsk) 146 { 147 memset(tsk->thread.hbp_break, 0, sizeof(tsk->thread.hbp_break)); 148 memset(tsk->thread.hbp_watch, 0, sizeof(tsk->thread.hbp_watch)); 149 } 150 151 /* 152 * Unregister breakpoints from this task and reset the pointers in the thread_struct. 153 */ 154 void flush_ptrace_hw_breakpoint(struct task_struct *tsk) 155 { 156 int i; 157 struct thread_struct *t = &tsk->thread; 158 159 for (i = 0; i < LOONGARCH_MAX_BRP; i++) { 160 if (t->hbp_break[i]) { 161 unregister_hw_breakpoint(t->hbp_break[i]); 162 t->hbp_break[i] = NULL; 163 } 164 } 165 166 for (i = 0; i < LOONGARCH_MAX_WRP; i++) { 167 if (t->hbp_watch[i]) { 168 unregister_hw_breakpoint(t->hbp_watch[i]); 169 t->hbp_watch[i] = NULL; 170 } 171 } 172 } 173 174 static int hw_breakpoint_control(struct perf_event *bp, 175 enum hw_breakpoint_ops ops) 176 { 177 u32 ctrl; 178 int i, max_slots, enable; 179 struct perf_event **slots; 180 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 181 182 if (info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) { 183 /* Breakpoint */ 184 slots = this_cpu_ptr(bp_on_reg); 185 max_slots = boot_cpu_data.watch_ireg_count; 186 } else { 187 /* Watchpoint */ 188 slots = this_cpu_ptr(wp_on_reg); 189 max_slots = boot_cpu_data.watch_dreg_count; 190 } 191 192 i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops); 193 194 if (WARN_ONCE(i < 0, "Can't find any breakpoint slot")) 195 return i; 196 197 switch (ops) { 198 case HW_BREAKPOINT_INSTALL: 199 /* Set the FWPnCFG/MWPnCFG 1~4 register. */ 200 write_wb_reg(CSR_CFG_ADDR, i, 0, info->address); 201 write_wb_reg(CSR_CFG_ADDR, i, 1, info->address); 202 write_wb_reg(CSR_CFG_MASK, i, 0, info->mask); 203 write_wb_reg(CSR_CFG_MASK, i, 1, info->mask); 204 write_wb_reg(CSR_CFG_ASID, i, 0, 0); 205 write_wb_reg(CSR_CFG_ASID, i, 1, 0); 206 if (info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) { 207 write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE); 208 } else { 209 ctrl = encode_ctrl_reg(info->ctrl); 210 write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl | CTRL_PLV_ENABLE); 211 } 212 enable = csr_read64(LOONGARCH_CSR_CRMD); 213 csr_write64(CSR_CRMD_WE | enable, LOONGARCH_CSR_CRMD); 214 break; 215 case HW_BREAKPOINT_UNINSTALL: 216 /* Reset the FWPnCFG/MWPnCFG 1~4 register. */ 217 write_wb_reg(CSR_CFG_ADDR, i, 0, 0); 218 write_wb_reg(CSR_CFG_ADDR, i, 1, 0); 219 write_wb_reg(CSR_CFG_MASK, i, 0, 0); 220 write_wb_reg(CSR_CFG_MASK, i, 1, 0); 221 write_wb_reg(CSR_CFG_CTRL, i, 0, 0); 222 write_wb_reg(CSR_CFG_CTRL, i, 1, 0); 223 write_wb_reg(CSR_CFG_ASID, i, 0, 0); 224 write_wb_reg(CSR_CFG_ASID, i, 1, 0); 225 break; 226 } 227 228 return 0; 229 } 230 231 /* 232 * Install a perf counter breakpoint. 233 */ 234 int arch_install_hw_breakpoint(struct perf_event *bp) 235 { 236 return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL); 237 } 238 239 void arch_uninstall_hw_breakpoint(struct perf_event *bp) 240 { 241 hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL); 242 } 243 244 static int get_hbp_len(u8 hbp_len) 245 { 246 unsigned int len_in_bytes = 0; 247 248 switch (hbp_len) { 249 case LOONGARCH_BREAKPOINT_LEN_1: 250 len_in_bytes = 1; 251 break; 252 case LOONGARCH_BREAKPOINT_LEN_2: 253 len_in_bytes = 2; 254 break; 255 case LOONGARCH_BREAKPOINT_LEN_4: 256 len_in_bytes = 4; 257 break; 258 case LOONGARCH_BREAKPOINT_LEN_8: 259 len_in_bytes = 8; 260 break; 261 } 262 263 return len_in_bytes; 264 } 265 266 /* 267 * Check whether bp virtual address is in kernel space. 268 */ 269 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) 270 { 271 unsigned int len; 272 unsigned long va; 273 274 va = hw->address; 275 len = get_hbp_len(hw->ctrl.len); 276 277 return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); 278 } 279 280 /* 281 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl. 282 * Hopefully this will disappear when ptrace can bypass the conversion 283 * to generic breakpoint descriptions. 284 */ 285 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, 286 int *gen_len, int *gen_type, int *offset) 287 { 288 /* Type */ 289 switch (ctrl.type) { 290 case LOONGARCH_BREAKPOINT_EXECUTE: 291 *gen_type = HW_BREAKPOINT_X; 292 break; 293 case LOONGARCH_BREAKPOINT_LOAD: 294 *gen_type = HW_BREAKPOINT_R; 295 break; 296 case LOONGARCH_BREAKPOINT_STORE: 297 *gen_type = HW_BREAKPOINT_W; 298 break; 299 case LOONGARCH_BREAKPOINT_LOAD | LOONGARCH_BREAKPOINT_STORE: 300 *gen_type = HW_BREAKPOINT_RW; 301 break; 302 default: 303 return -EINVAL; 304 } 305 306 if (!ctrl.len) 307 return -EINVAL; 308 309 *offset = __ffs(ctrl.len); 310 311 /* Len */ 312 switch (ctrl.len) { 313 case LOONGARCH_BREAKPOINT_LEN_1: 314 *gen_len = HW_BREAKPOINT_LEN_1; 315 break; 316 case LOONGARCH_BREAKPOINT_LEN_2: 317 *gen_len = HW_BREAKPOINT_LEN_2; 318 break; 319 case LOONGARCH_BREAKPOINT_LEN_4: 320 *gen_len = HW_BREAKPOINT_LEN_4; 321 break; 322 case LOONGARCH_BREAKPOINT_LEN_8: 323 *gen_len = HW_BREAKPOINT_LEN_8; 324 break; 325 default: 326 return -EINVAL; 327 } 328 329 return 0; 330 } 331 332 /* 333 * Construct an arch_hw_breakpoint from a perf_event. 334 */ 335 static int arch_build_bp_info(struct perf_event *bp, 336 const struct perf_event_attr *attr, 337 struct arch_hw_breakpoint *hw) 338 { 339 /* Type */ 340 switch (attr->bp_type) { 341 case HW_BREAKPOINT_X: 342 hw->ctrl.type = LOONGARCH_BREAKPOINT_EXECUTE; 343 break; 344 case HW_BREAKPOINT_R: 345 hw->ctrl.type = LOONGARCH_BREAKPOINT_LOAD; 346 break; 347 case HW_BREAKPOINT_W: 348 hw->ctrl.type = LOONGARCH_BREAKPOINT_STORE; 349 break; 350 case HW_BREAKPOINT_RW: 351 hw->ctrl.type = LOONGARCH_BREAKPOINT_LOAD | LOONGARCH_BREAKPOINT_STORE; 352 break; 353 default: 354 return -EINVAL; 355 } 356 357 /* Len */ 358 switch (attr->bp_len) { 359 case HW_BREAKPOINT_LEN_1: 360 hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_1; 361 break; 362 case HW_BREAKPOINT_LEN_2: 363 hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_2; 364 break; 365 case HW_BREAKPOINT_LEN_4: 366 hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_4; 367 break; 368 case HW_BREAKPOINT_LEN_8: 369 hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_8; 370 break; 371 default: 372 return -EINVAL; 373 } 374 375 /* Address */ 376 hw->address = attr->bp_addr; 377 378 return 0; 379 } 380 381 /* 382 * Validate the arch-specific HW Breakpoint register settings. 383 */ 384 int hw_breakpoint_arch_parse(struct perf_event *bp, 385 const struct perf_event_attr *attr, 386 struct arch_hw_breakpoint *hw) 387 { 388 int ret; 389 u64 alignment_mask, offset; 390 391 /* Build the arch_hw_breakpoint. */ 392 ret = arch_build_bp_info(bp, attr, hw); 393 if (ret) 394 return ret; 395 396 if (hw->ctrl.type != LOONGARCH_BREAKPOINT_EXECUTE) 397 alignment_mask = 0x7; 398 else 399 alignment_mask = 0x3; 400 offset = hw->address & alignment_mask; 401 402 hw->address &= ~alignment_mask; 403 hw->ctrl.len <<= offset; 404 405 return 0; 406 } 407 408 static void update_bp_registers(struct pt_regs *regs, int enable, int type) 409 { 410 u32 ctrl; 411 int i, max_slots; 412 struct perf_event **slots; 413 struct arch_hw_breakpoint *info; 414 415 switch (type) { 416 case 0: 417 slots = this_cpu_ptr(bp_on_reg); 418 max_slots = boot_cpu_data.watch_ireg_count; 419 break; 420 case 1: 421 slots = this_cpu_ptr(wp_on_reg); 422 max_slots = boot_cpu_data.watch_dreg_count; 423 break; 424 default: 425 return; 426 } 427 428 for (i = 0; i < max_slots; ++i) { 429 if (!slots[i]) 430 continue; 431 432 info = counter_arch_bp(slots[i]); 433 if (enable) { 434 if ((info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) && (type == 0)) { 435 write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE); 436 write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE); 437 } else { 438 ctrl = read_wb_reg(CSR_CFG_CTRL, i, 1); 439 if (info->ctrl.type == LOONGARCH_BREAKPOINT_LOAD) 440 ctrl |= 0x1 << MWPnCFG3_LoadEn; 441 if (info->ctrl.type == LOONGARCH_BREAKPOINT_STORE) 442 ctrl |= 0x1 << MWPnCFG3_StoreEn; 443 write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl); 444 } 445 regs->csr_prmd |= CSR_PRMD_PWE; 446 } else { 447 if ((info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) && (type == 0)) { 448 write_wb_reg(CSR_CFG_CTRL, i, 0, 0); 449 } else { 450 ctrl = read_wb_reg(CSR_CFG_CTRL, i, 1); 451 if (info->ctrl.type == LOONGARCH_BREAKPOINT_LOAD) 452 ctrl &= ~0x1 << MWPnCFG3_LoadEn; 453 if (info->ctrl.type == LOONGARCH_BREAKPOINT_STORE) 454 ctrl &= ~0x1 << MWPnCFG3_StoreEn; 455 write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl); 456 } 457 regs->csr_prmd &= ~CSR_PRMD_PWE; 458 } 459 } 460 } 461 NOKPROBE_SYMBOL(update_bp_registers); 462 463 /* 464 * Debug exception handlers. 465 */ 466 void breakpoint_handler(struct pt_regs *regs) 467 { 468 int i; 469 struct perf_event *bp, **slots; 470 471 slots = this_cpu_ptr(bp_on_reg); 472 473 for (i = 0; i < boot_cpu_data.watch_ireg_count; ++i) { 474 bp = slots[i]; 475 if (bp == NULL) 476 continue; 477 perf_bp_event(bp, regs); 478 } 479 update_bp_registers(regs, 0, 0); 480 } 481 NOKPROBE_SYMBOL(breakpoint_handler); 482 483 void watchpoint_handler(struct pt_regs *regs) 484 { 485 int i; 486 struct perf_event *wp, **slots; 487 488 slots = this_cpu_ptr(wp_on_reg); 489 490 for (i = 0; i < boot_cpu_data.watch_dreg_count; ++i) { 491 wp = slots[i]; 492 if (wp == NULL) 493 continue; 494 perf_bp_event(wp, regs); 495 } 496 update_bp_registers(regs, 0, 1); 497 } 498 NOKPROBE_SYMBOL(watchpoint_handler); 499 500 static int __init arch_hw_breakpoint_init(void) 501 { 502 int cpu; 503 504 boot_cpu_data.watch_ireg_count = get_num_brps(); 505 boot_cpu_data.watch_dreg_count = get_num_wrps(); 506 507 pr_info("Found %d breakpoint and %d watchpoint registers.\n", 508 boot_cpu_data.watch_ireg_count, boot_cpu_data.watch_dreg_count); 509 510 for (cpu = 1; cpu < NR_CPUS; cpu++) { 511 cpu_data[cpu].watch_ireg_count = boot_cpu_data.watch_ireg_count; 512 cpu_data[cpu].watch_dreg_count = boot_cpu_data.watch_dreg_count; 513 } 514 515 return 0; 516 } 517 arch_initcall(arch_hw_breakpoint_init); 518 519 void hw_breakpoint_thread_switch(struct task_struct *next) 520 { 521 u64 addr, mask; 522 struct pt_regs *regs = task_pt_regs(next); 523 524 if (test_tsk_thread_flag(next, TIF_SINGLESTEP)) { 525 addr = read_wb_reg(CSR_CFG_ADDR, 0, 0); 526 mask = read_wb_reg(CSR_CFG_MASK, 0, 0); 527 if (!((regs->csr_era ^ addr) & ~mask)) 528 csr_write32(CSR_FWPC_SKIP, LOONGARCH_CSR_FWPS); 529 regs->csr_prmd |= CSR_PRMD_PWE; 530 } else { 531 /* Update breakpoints */ 532 update_bp_registers(regs, 1, 0); 533 /* Update watchpoints */ 534 update_bp_registers(regs, 1, 1); 535 } 536 } 537 538 void hw_breakpoint_pmu_read(struct perf_event *bp) 539 { 540 } 541 542 /* 543 * Dummy function to register with die_notifier. 544 */ 545 int hw_breakpoint_exceptions_notify(struct notifier_block *unused, 546 unsigned long val, void *data) 547 { 548 return NOTIFY_DONE; 549 } 550