1 /* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License as published by 4 * the Free Software Foundation; either version 2 of the License, or 5 * (at your option) any later version. 6 * 7 * This program is distributed in the hope that it will be useful, 8 * but WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 * GNU General Public License for more details. 11 * 12 * You should have received a copy of the GNU General Public License 13 * along with this program; if not, write to the Free Software 14 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 15 * 16 * Copyright (C) 2007 Alan Stern 17 * Copyright (C) 2009 IBM Corporation 18 * Copyright (C) 2009 Frederic Weisbecker <fweisbec@gmail.com> 19 * 20 * Authors: Alan Stern <stern@rowland.harvard.edu> 21 * K.Prasad <prasad@linux.vnet.ibm.com> 22 * Frederic Weisbecker <fweisbec@gmail.com> 23 */ 24 25 /* 26 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, 27 * using the CPU's debug registers. 28 */ 29 30 #include <linux/perf_event.h> 31 #include <linux/hw_breakpoint.h> 32 #include <linux/irqflags.h> 33 #include <linux/notifier.h> 34 #include <linux/kallsyms.h> 35 #include <linux/kprobes.h> 36 #include <linux/percpu.h> 37 #include <linux/kdebug.h> 38 #include <linux/kernel.h> 39 #include <linux/export.h> 40 #include <linux/sched.h> 41 #include <linux/smp.h> 42 43 #include <asm/hw_breakpoint.h> 44 #include <asm/processor.h> 45 #include <asm/debugreg.h> 46 #include <asm/user.h> 47 48 /* Per cpu debug control register value */ 49 DEFINE_PER_CPU(unsigned long, cpu_dr7); 50 EXPORT_PER_CPU_SYMBOL(cpu_dr7); 51 52 /* Per cpu debug address registers values */ 53 static DEFINE_PER_CPU(unsigned long, cpu_debugreg[HBP_NUM]); 54 55 /* 56 * Stores the breakpoints currently in use on each breakpoint address 57 * register for each cpus 58 */ 59 static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM]); 60 61 62 static inline unsigned long 63 __encode_dr7(int drnum, unsigned int len, unsigned int type) 64 { 65 unsigned long bp_info; 66 67 bp_info = (len | type) & 0xf; 68 bp_info <<= (DR_CONTROL_SHIFT + drnum * DR_CONTROL_SIZE); 69 bp_info |= (DR_GLOBAL_ENABLE << (drnum * DR_ENABLE_SIZE)); 70 71 return bp_info; 72 } 73 74 /* 75 * Encode the length, type, Exact, and Enable bits for a particular breakpoint 76 * as stored in debug register 7. 77 */ 78 unsigned long encode_dr7(int drnum, unsigned int len, unsigned int type) 79 { 80 return __encode_dr7(drnum, len, type) | DR_GLOBAL_SLOWDOWN; 81 } 82 83 /* 84 * Decode the length and type bits for a particular breakpoint as 85 * stored in debug register 7. Return the "enabled" status. 86 */ 87 int decode_dr7(unsigned long dr7, int bpnum, unsigned *len, unsigned *type) 88 { 89 int bp_info = dr7 >> (DR_CONTROL_SHIFT + bpnum * DR_CONTROL_SIZE); 90 91 *len = (bp_info & 0xc) | 0x40; 92 *type = (bp_info & 0x3) | 0x80; 93 94 return (dr7 >> (bpnum * DR_ENABLE_SIZE)) & 0x3; 95 } 96 97 /* 98 * Install a perf counter breakpoint. 99 * 100 * We seek a free debug address register and use it for this 101 * breakpoint. Eventually we enable it in the debug control register. 102 * 103 * Atomic: we hold the counter->ctx->lock and we only handle variables 104 * and registers local to this cpu. 105 */ 106 int arch_install_hw_breakpoint(struct perf_event *bp) 107 { 108 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 109 unsigned long *dr7; 110 int i; 111 112 for (i = 0; i < HBP_NUM; i++) { 113 struct perf_event **slot = this_cpu_ptr(&bp_per_reg[i]); 114 115 if (!*slot) { 116 *slot = bp; 117 break; 118 } 119 } 120 121 if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot")) 122 return -EBUSY; 123 124 set_debugreg(info->address, i); 125 __this_cpu_write(cpu_debugreg[i], info->address); 126 127 dr7 = this_cpu_ptr(&cpu_dr7); 128 *dr7 |= encode_dr7(i, info->len, info->type); 129 130 set_debugreg(*dr7, 7); 131 if (info->mask) 132 set_dr_addr_mask(info->mask, i); 133 134 return 0; 135 } 136 137 /* 138 * Uninstall the breakpoint contained in the given counter. 139 * 140 * First we search the debug address register it uses and then we disable 141 * it. 142 * 143 * Atomic: we hold the counter->ctx->lock and we only handle variables 144 * and registers local to this cpu. 145 */ 146 void arch_uninstall_hw_breakpoint(struct perf_event *bp) 147 { 148 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 149 unsigned long *dr7; 150 int i; 151 152 for (i = 0; i < HBP_NUM; i++) { 153 struct perf_event **slot = this_cpu_ptr(&bp_per_reg[i]); 154 155 if (*slot == bp) { 156 *slot = NULL; 157 break; 158 } 159 } 160 161 if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot")) 162 return; 163 164 dr7 = this_cpu_ptr(&cpu_dr7); 165 *dr7 &= ~__encode_dr7(i, info->len, info->type); 166 167 set_debugreg(*dr7, 7); 168 if (info->mask) 169 set_dr_addr_mask(0, i); 170 } 171 172 static int arch_bp_generic_len(int x86_len) 173 { 174 switch (x86_len) { 175 case X86_BREAKPOINT_LEN_1: 176 return HW_BREAKPOINT_LEN_1; 177 case X86_BREAKPOINT_LEN_2: 178 return HW_BREAKPOINT_LEN_2; 179 case X86_BREAKPOINT_LEN_4: 180 return HW_BREAKPOINT_LEN_4; 181 #ifdef CONFIG_X86_64 182 case X86_BREAKPOINT_LEN_8: 183 return HW_BREAKPOINT_LEN_8; 184 #endif 185 default: 186 return -EINVAL; 187 } 188 } 189 190 int arch_bp_generic_fields(int x86_len, int x86_type, 191 int *gen_len, int *gen_type) 192 { 193 int len; 194 195 /* Type */ 196 switch (x86_type) { 197 case X86_BREAKPOINT_EXECUTE: 198 if (x86_len != X86_BREAKPOINT_LEN_X) 199 return -EINVAL; 200 201 *gen_type = HW_BREAKPOINT_X; 202 *gen_len = sizeof(long); 203 return 0; 204 case X86_BREAKPOINT_WRITE: 205 *gen_type = HW_BREAKPOINT_W; 206 break; 207 case X86_BREAKPOINT_RW: 208 *gen_type = HW_BREAKPOINT_W | HW_BREAKPOINT_R; 209 break; 210 default: 211 return -EINVAL; 212 } 213 214 /* Len */ 215 len = arch_bp_generic_len(x86_len); 216 if (len < 0) 217 return -EINVAL; 218 *gen_len = len; 219 220 return 0; 221 } 222 223 /* 224 * Check for virtual address in kernel space. 225 */ 226 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) 227 { 228 unsigned long va; 229 int len; 230 231 va = hw->address; 232 len = arch_bp_generic_len(hw->len); 233 WARN_ON_ONCE(len < 0); 234 235 /* 236 * We don't need to worry about va + len - 1 overflowing: 237 * we already require that va is aligned to a multiple of len. 238 */ 239 return (va >= TASK_SIZE_MAX) || ((va + len - 1) >= TASK_SIZE_MAX); 240 } 241 242 static int arch_build_bp_info(struct perf_event *bp, 243 const struct perf_event_attr *attr, 244 struct arch_hw_breakpoint *hw) 245 { 246 hw->address = attr->bp_addr; 247 hw->mask = 0; 248 249 /* Type */ 250 switch (attr->bp_type) { 251 case HW_BREAKPOINT_W: 252 hw->type = X86_BREAKPOINT_WRITE; 253 break; 254 case HW_BREAKPOINT_W | HW_BREAKPOINT_R: 255 hw->type = X86_BREAKPOINT_RW; 256 break; 257 case HW_BREAKPOINT_X: 258 /* 259 * We don't allow kernel breakpoints in places that are not 260 * acceptable for kprobes. On non-kprobes kernels, we don't 261 * allow kernel breakpoints at all. 262 */ 263 if (attr->bp_addr >= TASK_SIZE_MAX) { 264 if (within_kprobe_blacklist(attr->bp_addr)) 265 return -EINVAL; 266 } 267 268 hw->type = X86_BREAKPOINT_EXECUTE; 269 /* 270 * x86 inst breakpoints need to have a specific undefined len. 271 * But we still need to check userspace is not trying to setup 272 * an unsupported length, to get a range breakpoint for example. 273 */ 274 if (attr->bp_len == sizeof(long)) { 275 hw->len = X86_BREAKPOINT_LEN_X; 276 return 0; 277 } 278 /* fall through */ 279 default: 280 return -EINVAL; 281 } 282 283 /* Len */ 284 switch (attr->bp_len) { 285 case HW_BREAKPOINT_LEN_1: 286 hw->len = X86_BREAKPOINT_LEN_1; 287 break; 288 case HW_BREAKPOINT_LEN_2: 289 hw->len = X86_BREAKPOINT_LEN_2; 290 break; 291 case HW_BREAKPOINT_LEN_4: 292 hw->len = X86_BREAKPOINT_LEN_4; 293 break; 294 #ifdef CONFIG_X86_64 295 case HW_BREAKPOINT_LEN_8: 296 hw->len = X86_BREAKPOINT_LEN_8; 297 break; 298 #endif 299 default: 300 /* AMD range breakpoint */ 301 if (!is_power_of_2(attr->bp_len)) 302 return -EINVAL; 303 if (attr->bp_addr & (attr->bp_len - 1)) 304 return -EINVAL; 305 306 if (!boot_cpu_has(X86_FEATURE_BPEXT)) 307 return -EOPNOTSUPP; 308 309 /* 310 * It's impossible to use a range breakpoint to fake out 311 * user vs kernel detection because bp_len - 1 can't 312 * have the high bit set. If we ever allow range instruction 313 * breakpoints, then we'll have to check for kprobe-blacklisted 314 * addresses anywhere in the range. 315 */ 316 hw->mask = attr->bp_len - 1; 317 hw->len = X86_BREAKPOINT_LEN_1; 318 } 319 320 return 0; 321 } 322 323 /* 324 * Validate the arch-specific HW Breakpoint register settings 325 */ 326 int hw_breakpoint_arch_parse(struct perf_event *bp, 327 const struct perf_event_attr *attr, 328 struct arch_hw_breakpoint *hw) 329 { 330 unsigned int align; 331 int ret; 332 333 334 ret = arch_build_bp_info(bp, attr, hw); 335 if (ret) 336 return ret; 337 338 switch (hw->len) { 339 case X86_BREAKPOINT_LEN_1: 340 align = 0; 341 if (hw->mask) 342 align = hw->mask; 343 break; 344 case X86_BREAKPOINT_LEN_2: 345 align = 1; 346 break; 347 case X86_BREAKPOINT_LEN_4: 348 align = 3; 349 break; 350 #ifdef CONFIG_X86_64 351 case X86_BREAKPOINT_LEN_8: 352 align = 7; 353 break; 354 #endif 355 default: 356 WARN_ON_ONCE(1); 357 } 358 359 /* 360 * Check that the low-order bits of the address are appropriate 361 * for the alignment implied by len. 362 */ 363 if (hw->address & align) 364 return -EINVAL; 365 366 return 0; 367 } 368 369 /* 370 * Dump the debug register contents to the user. 371 * We can't dump our per cpu values because it 372 * may contain cpu wide breakpoint, something that 373 * doesn't belong to the current task. 374 * 375 * TODO: include non-ptrace user breakpoints (perf) 376 */ 377 void aout_dump_debugregs(struct user *dump) 378 { 379 int i; 380 int dr7 = 0; 381 struct perf_event *bp; 382 struct arch_hw_breakpoint *info; 383 struct thread_struct *thread = ¤t->thread; 384 385 for (i = 0; i < HBP_NUM; i++) { 386 bp = thread->ptrace_bps[i]; 387 388 if (bp && !bp->attr.disabled) { 389 dump->u_debugreg[i] = bp->attr.bp_addr; 390 info = counter_arch_bp(bp); 391 dr7 |= encode_dr7(i, info->len, info->type); 392 } else { 393 dump->u_debugreg[i] = 0; 394 } 395 } 396 397 dump->u_debugreg[4] = 0; 398 dump->u_debugreg[5] = 0; 399 dump->u_debugreg[6] = current->thread.debugreg6; 400 401 dump->u_debugreg[7] = dr7; 402 } 403 EXPORT_SYMBOL_GPL(aout_dump_debugregs); 404 405 /* 406 * Release the user breakpoints used by ptrace 407 */ 408 void flush_ptrace_hw_breakpoint(struct task_struct *tsk) 409 { 410 int i; 411 struct thread_struct *t = &tsk->thread; 412 413 for (i = 0; i < HBP_NUM; i++) { 414 unregister_hw_breakpoint(t->ptrace_bps[i]); 415 t->ptrace_bps[i] = NULL; 416 } 417 418 t->debugreg6 = 0; 419 t->ptrace_dr7 = 0; 420 } 421 422 void hw_breakpoint_restore(void) 423 { 424 set_debugreg(__this_cpu_read(cpu_debugreg[0]), 0); 425 set_debugreg(__this_cpu_read(cpu_debugreg[1]), 1); 426 set_debugreg(__this_cpu_read(cpu_debugreg[2]), 2); 427 set_debugreg(__this_cpu_read(cpu_debugreg[3]), 3); 428 set_debugreg(current->thread.debugreg6, 6); 429 set_debugreg(__this_cpu_read(cpu_dr7), 7); 430 } 431 EXPORT_SYMBOL_GPL(hw_breakpoint_restore); 432 433 /* 434 * Handle debug exception notifications. 435 * 436 * Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below. 437 * 438 * NOTIFY_DONE returned if one of the following conditions is true. 439 * i) When the causative address is from user-space and the exception 440 * is a valid one, i.e. not triggered as a result of lazy debug register 441 * switching 442 * ii) When there are more bits than trap<n> set in DR6 register (such 443 * as BD, BS or BT) indicating that more than one debug condition is 444 * met and requires some more action in do_debug(). 445 * 446 * NOTIFY_STOP returned for all other cases 447 * 448 */ 449 static int hw_breakpoint_handler(struct die_args *args) 450 { 451 int i, cpu, rc = NOTIFY_STOP; 452 struct perf_event *bp; 453 unsigned long dr7, dr6; 454 unsigned long *dr6_p; 455 456 /* The DR6 value is pointed by args->err */ 457 dr6_p = (unsigned long *)ERR_PTR(args->err); 458 dr6 = *dr6_p; 459 460 /* If it's a single step, TRAP bits are random */ 461 if (dr6 & DR_STEP) 462 return NOTIFY_DONE; 463 464 /* Do an early return if no trap bits are set in DR6 */ 465 if ((dr6 & DR_TRAP_BITS) == 0) 466 return NOTIFY_DONE; 467 468 get_debugreg(dr7, 7); 469 /* Disable breakpoints during exception handling */ 470 set_debugreg(0UL, 7); 471 /* 472 * Assert that local interrupts are disabled 473 * Reset the DRn bits in the virtualized register value. 474 * The ptrace trigger routine will add in whatever is needed. 475 */ 476 current->thread.debugreg6 &= ~DR_TRAP_BITS; 477 cpu = get_cpu(); 478 479 /* Handle all the breakpoints that were triggered */ 480 for (i = 0; i < HBP_NUM; ++i) { 481 if (likely(!(dr6 & (DR_TRAP0 << i)))) 482 continue; 483 484 /* 485 * The counter may be concurrently released but that can only 486 * occur from a call_rcu() path. We can then safely fetch 487 * the breakpoint, use its callback, touch its counter 488 * while we are in an rcu_read_lock() path. 489 */ 490 rcu_read_lock(); 491 492 bp = per_cpu(bp_per_reg[i], cpu); 493 /* 494 * Reset the 'i'th TRAP bit in dr6 to denote completion of 495 * exception handling 496 */ 497 (*dr6_p) &= ~(DR_TRAP0 << i); 498 /* 499 * bp can be NULL due to lazy debug register switching 500 * or due to concurrent perf counter removing. 501 */ 502 if (!bp) { 503 rcu_read_unlock(); 504 break; 505 } 506 507 perf_bp_event(bp, args->regs); 508 509 /* 510 * Set up resume flag to avoid breakpoint recursion when 511 * returning back to origin. 512 */ 513 if (bp->hw.info.type == X86_BREAKPOINT_EXECUTE) 514 args->regs->flags |= X86_EFLAGS_RF; 515 516 rcu_read_unlock(); 517 } 518 /* 519 * Further processing in do_debug() is needed for a) user-space 520 * breakpoints (to generate signals) and b) when the system has 521 * taken exception due to multiple causes 522 */ 523 if ((current->thread.debugreg6 & DR_TRAP_BITS) || 524 (dr6 & (~DR_TRAP_BITS))) 525 rc = NOTIFY_DONE; 526 527 set_debugreg(dr7, 7); 528 put_cpu(); 529 530 return rc; 531 } 532 533 /* 534 * Handle debug exception notifications. 535 */ 536 int hw_breakpoint_exceptions_notify( 537 struct notifier_block *unused, unsigned long val, void *data) 538 { 539 if (val != DIE_DEBUG) 540 return NOTIFY_DONE; 541 542 return hw_breakpoint_handler(data); 543 } 544 545 void hw_breakpoint_pmu_read(struct perf_event *bp) 546 { 547 /* TODO */ 548 } 549