11a59d1b8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 20067f129SK.Prasad /* 30067f129SK.Prasad * 40067f129SK.Prasad * Copyright (C) 2007 Alan Stern 50067f129SK.Prasad * Copyright (C) 2009 IBM Corporation 624f1e32cSFrederic Weisbecker * Copyright (C) 2009 Frederic Weisbecker <fweisbec@gmail.com> 7ba6909b7SK.Prasad * 8ba6909b7SK.Prasad * Authors: Alan Stern <stern@rowland.harvard.edu> 9ba6909b7SK.Prasad * K.Prasad <prasad@linux.vnet.ibm.com> 10ba6909b7SK.Prasad * Frederic Weisbecker <fweisbec@gmail.com> 110067f129SK.Prasad */ 120067f129SK.Prasad 130067f129SK.Prasad /* 140067f129SK.Prasad * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, 150067f129SK.Prasad * using the CPU's debug registers. 160067f129SK.Prasad */ 170067f129SK.Prasad 1824f1e32cSFrederic Weisbecker #include <linux/perf_event.h> 1924f1e32cSFrederic Weisbecker #include <linux/hw_breakpoint.h> 200067f129SK.Prasad #include <linux/irqflags.h> 210067f129SK.Prasad #include <linux/notifier.h> 220067f129SK.Prasad #include <linux/kallsyms.h> 23e5779e8eSAndy Lutomirski #include <linux/kprobes.h> 240067f129SK.Prasad #include <linux/percpu.h> 250067f129SK.Prasad #include <linux/kdebug.h> 260067f129SK.Prasad #include <linux/kernel.h> 27186f4360SPaul Gortmaker #include <linux/export.h> 280067f129SK.Prasad #include <linux/sched.h> 290067f129SK.Prasad #include <linux/smp.h> 300067f129SK.Prasad 310067f129SK.Prasad #include <asm/hw_breakpoint.h> 320067f129SK.Prasad #include <asm/processor.h> 330067f129SK.Prasad #include <asm/debugreg.h> 34186f4360SPaul Gortmaker #include <asm/user.h> 3597417cb9SLai Jiangshan #include <asm/desc.h> 36fdef24dfSLai Jiangshan #include <asm/tlbflush.h> 370067f129SK.Prasad 3824f1e32cSFrederic Weisbecker /* Per cpu debug control register value */ 3928b4e0d8STejun Heo DEFINE_PER_CPU(unsigned long, cpu_dr7); 4028b4e0d8STejun Heo EXPORT_PER_CPU_SYMBOL(cpu_dr7); 4124f1e32cSFrederic Weisbecker 4224f1e32cSFrederic Weisbecker /* Per cpu debug address registers values */ 4324f1e32cSFrederic Weisbecker static DEFINE_PER_CPU(unsigned long, cpu_debugreg[HBP_NUM]); 440067f129SK.Prasad 450067f129SK.Prasad /* 4624f1e32cSFrederic Weisbecker * Stores the breakpoints currently in use on each breakpoint address 4724f1e32cSFrederic Weisbecker * register for each cpus 480067f129SK.Prasad */ 4924f1e32cSFrederic Weisbecker static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM]); 500067f129SK.Prasad 510067f129SK.Prasad 522c31b795SFrederic Weisbecker static inline unsigned long 532c31b795SFrederic Weisbecker __encode_dr7(int drnum, unsigned int len, unsigned int type) 542c31b795SFrederic Weisbecker { 552c31b795SFrederic Weisbecker unsigned long bp_info; 562c31b795SFrederic Weisbecker 572c31b795SFrederic Weisbecker bp_info = (len | type) & 0xf; 582c31b795SFrederic Weisbecker bp_info <<= (DR_CONTROL_SHIFT + drnum * DR_CONTROL_SIZE); 592c31b795SFrederic Weisbecker bp_info |= (DR_GLOBAL_ENABLE << (drnum * DR_ENABLE_SIZE)); 602c31b795SFrederic Weisbecker 612c31b795SFrederic Weisbecker return bp_info; 622c31b795SFrederic Weisbecker } 632c31b795SFrederic Weisbecker 640067f129SK.Prasad /* 650067f129SK.Prasad * Encode the length, type, Exact, and Enable bits for a particular breakpoint 660067f129SK.Prasad * as stored in debug register 7. 670067f129SK.Prasad */ 6824f1e32cSFrederic Weisbecker unsigned long encode_dr7(int drnum, unsigned int len, unsigned int type) 690067f129SK.Prasad { 702c31b795SFrederic Weisbecker return __encode_dr7(drnum, len, type) | DR_GLOBAL_SLOWDOWN; 710067f129SK.Prasad } 720067f129SK.Prasad 7324f1e32cSFrederic Weisbecker /* 7424f1e32cSFrederic Weisbecker * Decode the length and type bits for a particular breakpoint as 7524f1e32cSFrederic Weisbecker * stored in debug register 7. Return the "enabled" status. 760067f129SK.Prasad */ 7724f1e32cSFrederic Weisbecker int decode_dr7(unsigned long dr7, int bpnum, unsigned *len, unsigned *type) 7824f1e32cSFrederic Weisbecker { 7924f1e32cSFrederic Weisbecker int bp_info = dr7 >> (DR_CONTROL_SHIFT + bpnum * DR_CONTROL_SIZE); 8024f1e32cSFrederic Weisbecker 8124f1e32cSFrederic Weisbecker *len = (bp_info & 0xc) | 0x40; 8224f1e32cSFrederic Weisbecker *type = (bp_info & 0x3) | 0x80; 8324f1e32cSFrederic Weisbecker 8424f1e32cSFrederic Weisbecker return (dr7 >> (bpnum * DR_ENABLE_SIZE)) & 0x3; 850067f129SK.Prasad } 860067f129SK.Prasad 870067f129SK.Prasad /* 8824f1e32cSFrederic Weisbecker * Install a perf counter breakpoint. 8924f1e32cSFrederic Weisbecker * 9024f1e32cSFrederic Weisbecker * We seek a free debug address register and use it for this 9124f1e32cSFrederic Weisbecker * breakpoint. Eventually we enable it in the debug control register. 9224f1e32cSFrederic Weisbecker * 9324f1e32cSFrederic Weisbecker * Atomic: we hold the counter->ctx->lock and we only handle variables 9424f1e32cSFrederic Weisbecker * and registers local to this cpu. 950067f129SK.Prasad */ 9624f1e32cSFrederic Weisbecker int arch_install_hw_breakpoint(struct perf_event *bp) 970067f129SK.Prasad { 9824f1e32cSFrederic Weisbecker struct arch_hw_breakpoint *info = counter_arch_bp(bp); 9924f1e32cSFrederic Weisbecker unsigned long *dr7; 10024f1e32cSFrederic Weisbecker int i; 1010067f129SK.Prasad 10284b6a349SPeter Zijlstra lockdep_assert_irqs_disabled(); 10384b6a349SPeter Zijlstra 10424f1e32cSFrederic Weisbecker for (i = 0; i < HBP_NUM; i++) { 10589cbc767SChristoph Lameter struct perf_event **slot = this_cpu_ptr(&bp_per_reg[i]); 10624f1e32cSFrederic Weisbecker 10724f1e32cSFrederic Weisbecker if (!*slot) { 10824f1e32cSFrederic Weisbecker *slot = bp; 1090067f129SK.Prasad break; 1100067f129SK.Prasad } 11124f1e32cSFrederic Weisbecker } 1120067f129SK.Prasad 11324f1e32cSFrederic Weisbecker if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot")) 11424f1e32cSFrederic Weisbecker return -EBUSY; 11524f1e32cSFrederic Weisbecker 11624f1e32cSFrederic Weisbecker set_debugreg(info->address, i); 1170a3aee0dSTejun Heo __this_cpu_write(cpu_debugreg[i], info->address); 11824f1e32cSFrederic Weisbecker 11989cbc767SChristoph Lameter dr7 = this_cpu_ptr(&cpu_dr7); 12024f1e32cSFrederic Weisbecker *dr7 |= encode_dr7(i, info->len, info->type); 12124f1e32cSFrederic Weisbecker 12284b6a349SPeter Zijlstra /* 12384b6a349SPeter Zijlstra * Ensure we first write cpu_dr7 before we set the DR7 register. 12484b6a349SPeter Zijlstra * This ensures an NMI never see cpu_dr7 0 when DR7 is not. 12584b6a349SPeter Zijlstra */ 12684b6a349SPeter Zijlstra barrier(); 12784b6a349SPeter Zijlstra 12824f1e32cSFrederic Weisbecker set_debugreg(*dr7, 7); 129d6d55f0bSJacob Shin if (info->mask) 130d6d55f0bSJacob Shin set_dr_addr_mask(info->mask, i); 13124f1e32cSFrederic Weisbecker 13224f1e32cSFrederic Weisbecker return 0; 1330067f129SK.Prasad } 1340067f129SK.Prasad 1350067f129SK.Prasad /* 13624f1e32cSFrederic Weisbecker * Uninstall the breakpoint contained in the given counter. 13724f1e32cSFrederic Weisbecker * 13824f1e32cSFrederic Weisbecker * First we search the debug address register it uses and then we disable 13924f1e32cSFrederic Weisbecker * it. 14024f1e32cSFrederic Weisbecker * 14124f1e32cSFrederic Weisbecker * Atomic: we hold the counter->ctx->lock and we only handle variables 14224f1e32cSFrederic Weisbecker * and registers local to this cpu. 1430067f129SK.Prasad */ 14424f1e32cSFrederic Weisbecker void arch_uninstall_hw_breakpoint(struct perf_event *bp) 1450067f129SK.Prasad { 14624f1e32cSFrederic Weisbecker struct arch_hw_breakpoint *info = counter_arch_bp(bp); 14784b6a349SPeter Zijlstra unsigned long dr7; 14824f1e32cSFrederic Weisbecker int i; 1490067f129SK.Prasad 15084b6a349SPeter Zijlstra lockdep_assert_irqs_disabled(); 15184b6a349SPeter Zijlstra 15224f1e32cSFrederic Weisbecker for (i = 0; i < HBP_NUM; i++) { 15389cbc767SChristoph Lameter struct perf_event **slot = this_cpu_ptr(&bp_per_reg[i]); 15424f1e32cSFrederic Weisbecker 15524f1e32cSFrederic Weisbecker if (*slot == bp) { 15624f1e32cSFrederic Weisbecker *slot = NULL; 15724f1e32cSFrederic Weisbecker break; 15824f1e32cSFrederic Weisbecker } 15924f1e32cSFrederic Weisbecker } 16024f1e32cSFrederic Weisbecker 16124f1e32cSFrederic Weisbecker if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot")) 16224f1e32cSFrederic Weisbecker return; 16324f1e32cSFrederic Weisbecker 16484b6a349SPeter Zijlstra dr7 = this_cpu_read(cpu_dr7); 16584b6a349SPeter Zijlstra dr7 &= ~__encode_dr7(i, info->len, info->type); 16624f1e32cSFrederic Weisbecker 16784b6a349SPeter Zijlstra set_debugreg(dr7, 7); 168d6d55f0bSJacob Shin if (info->mask) 169d6d55f0bSJacob Shin set_dr_addr_mask(0, i); 17084b6a349SPeter Zijlstra 17184b6a349SPeter Zijlstra /* 17284b6a349SPeter Zijlstra * Ensure the write to cpu_dr7 is after we've set the DR7 register. 17384b6a349SPeter Zijlstra * This ensures an NMI never see cpu_dr7 0 when DR7 is not. 17484b6a349SPeter Zijlstra */ 17584b6a349SPeter Zijlstra barrier(); 17684b6a349SPeter Zijlstra 17784b6a349SPeter Zijlstra this_cpu_write(cpu_dr7, dr7); 1780067f129SK.Prasad } 1790067f129SK.Prasad 1808e983ff9SFrederic Weisbecker static int arch_bp_generic_len(int x86_len) 1810067f129SK.Prasad { 1828e983ff9SFrederic Weisbecker switch (x86_len) { 1838e983ff9SFrederic Weisbecker case X86_BREAKPOINT_LEN_1: 1848e983ff9SFrederic Weisbecker return HW_BREAKPOINT_LEN_1; 1858e983ff9SFrederic Weisbecker case X86_BREAKPOINT_LEN_2: 1868e983ff9SFrederic Weisbecker return HW_BREAKPOINT_LEN_2; 1878e983ff9SFrederic Weisbecker case X86_BREAKPOINT_LEN_4: 1888e983ff9SFrederic Weisbecker return HW_BREAKPOINT_LEN_4; 1898e983ff9SFrederic Weisbecker #ifdef CONFIG_X86_64 1908e983ff9SFrederic Weisbecker case X86_BREAKPOINT_LEN_8: 1918e983ff9SFrederic Weisbecker return HW_BREAKPOINT_LEN_8; 1928e983ff9SFrederic Weisbecker #endif 1938e983ff9SFrederic Weisbecker default: 1948e983ff9SFrederic Weisbecker return -EINVAL; 1958e983ff9SFrederic Weisbecker } 1960067f129SK.Prasad } 1970067f129SK.Prasad 19824f1e32cSFrederic Weisbecker int arch_bp_generic_fields(int x86_len, int x86_type, 19924f1e32cSFrederic Weisbecker int *gen_len, int *gen_type) 20024f1e32cSFrederic Weisbecker { 2018e983ff9SFrederic Weisbecker int len; 2028e983ff9SFrederic Weisbecker 20389e45aacSFrederic Weisbecker /* Type */ 20489e45aacSFrederic Weisbecker switch (x86_type) { 20589e45aacSFrederic Weisbecker case X86_BREAKPOINT_EXECUTE: 20689e45aacSFrederic Weisbecker if (x86_len != X86_BREAKPOINT_LEN_X) 20789e45aacSFrederic Weisbecker return -EINVAL; 20889e45aacSFrederic Weisbecker 20989e45aacSFrederic Weisbecker *gen_type = HW_BREAKPOINT_X; 21089e45aacSFrederic Weisbecker *gen_len = sizeof(long); 21189e45aacSFrederic Weisbecker return 0; 21289e45aacSFrederic Weisbecker case X86_BREAKPOINT_WRITE: 21389e45aacSFrederic Weisbecker *gen_type = HW_BREAKPOINT_W; 21489e45aacSFrederic Weisbecker break; 21589e45aacSFrederic Weisbecker case X86_BREAKPOINT_RW: 21689e45aacSFrederic Weisbecker *gen_type = HW_BREAKPOINT_W | HW_BREAKPOINT_R; 21789e45aacSFrederic Weisbecker break; 21889e45aacSFrederic Weisbecker default: 21989e45aacSFrederic Weisbecker return -EINVAL; 22089e45aacSFrederic Weisbecker } 22189e45aacSFrederic Weisbecker 22224f1e32cSFrederic Weisbecker /* Len */ 2238e983ff9SFrederic Weisbecker len = arch_bp_generic_len(x86_len); 2248e983ff9SFrederic Weisbecker if (len < 0) 22524f1e32cSFrederic Weisbecker return -EINVAL; 2268e983ff9SFrederic Weisbecker *gen_len = len; 22724f1e32cSFrederic Weisbecker 22824f1e32cSFrederic Weisbecker return 0; 22924f1e32cSFrederic Weisbecker } 23024f1e32cSFrederic Weisbecker 2318e983ff9SFrederic Weisbecker /* 2328e983ff9SFrederic Weisbecker * Check for virtual address in kernel space. 2338e983ff9SFrederic Weisbecker */ 2348e983ff9SFrederic Weisbecker int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) 2358e983ff9SFrederic Weisbecker { 2368e983ff9SFrederic Weisbecker unsigned long va; 2378e983ff9SFrederic Weisbecker int len; 2388e983ff9SFrederic Weisbecker 2398e983ff9SFrederic Weisbecker va = hw->address; 2408e983ff9SFrederic Weisbecker len = arch_bp_generic_len(hw->len); 2418e983ff9SFrederic Weisbecker WARN_ON_ONCE(len < 0); 2428e983ff9SFrederic Weisbecker 2438e983ff9SFrederic Weisbecker /* 2448e983ff9SFrederic Weisbecker * We don't need to worry about va + len - 1 overflowing: 2458e983ff9SFrederic Weisbecker * we already require that va is aligned to a multiple of len. 2468e983ff9SFrederic Weisbecker */ 2478e983ff9SFrederic Weisbecker return (va >= TASK_SIZE_MAX) || ((va + len - 1) >= TASK_SIZE_MAX); 2488e983ff9SFrederic Weisbecker } 24924f1e32cSFrederic Weisbecker 25024ae0c91SAndy Lutomirski /* 251d390e6deSLai Jiangshan * Checks whether the range [addr, end], overlaps the area [base, base + size). 252d390e6deSLai Jiangshan */ 253d390e6deSLai Jiangshan static inline bool within_area(unsigned long addr, unsigned long end, 254d390e6deSLai Jiangshan unsigned long base, unsigned long size) 255d390e6deSLai Jiangshan { 256d390e6deSLai Jiangshan return end >= base && addr < (base + size); 257d390e6deSLai Jiangshan } 258d390e6deSLai Jiangshan 259d390e6deSLai Jiangshan /* 26097417cb9SLai Jiangshan * Checks whether the range from addr to end, inclusive, overlaps the fixed 26197417cb9SLai Jiangshan * mapped CPU entry area range or other ranges used for CPU entry. 26224ae0c91SAndy Lutomirski */ 26397417cb9SLai Jiangshan static inline bool within_cpu_entry(unsigned long addr, unsigned long end) 26424ae0c91SAndy Lutomirski { 26597417cb9SLai Jiangshan int cpu; 26697417cb9SLai Jiangshan 26797417cb9SLai Jiangshan /* CPU entry erea is always used for CPU entry */ 26897417cb9SLai Jiangshan if (within_area(addr, end, CPU_ENTRY_AREA_BASE, 269*97e3d26bSPeter Zijlstra CPU_ENTRY_AREA_MAP_SIZE)) 27097417cb9SLai Jiangshan return true; 27197417cb9SLai Jiangshan 272c4bed4b9SLai Jiangshan /* 273c4bed4b9SLai Jiangshan * When FSGSBASE is enabled, paranoid_entry() fetches the per-CPU 274c4bed4b9SLai Jiangshan * GSBASE value via __per_cpu_offset or pcpu_unit_offsets. 275c4bed4b9SLai Jiangshan */ 276c4bed4b9SLai Jiangshan #ifdef CONFIG_SMP 277c4bed4b9SLai Jiangshan if (within_area(addr, end, (unsigned long)__per_cpu_offset, 278c4bed4b9SLai Jiangshan sizeof(unsigned long) * nr_cpu_ids)) 279c4bed4b9SLai Jiangshan return true; 280c4bed4b9SLai Jiangshan #else 281c4bed4b9SLai Jiangshan if (within_area(addr, end, (unsigned long)&pcpu_unit_offsets, 282c4bed4b9SLai Jiangshan sizeof(pcpu_unit_offsets))) 283c4bed4b9SLai Jiangshan return true; 284c4bed4b9SLai Jiangshan #endif 285c4bed4b9SLai Jiangshan 28697417cb9SLai Jiangshan for_each_possible_cpu(cpu) { 28797417cb9SLai Jiangshan /* The original rw GDT is being used after load_direct_gdt() */ 28897417cb9SLai Jiangshan if (within_area(addr, end, (unsigned long)get_cpu_gdt_rw(cpu), 28997417cb9SLai Jiangshan GDT_SIZE)) 29097417cb9SLai Jiangshan return true; 291f9fe0b89SLai Jiangshan 292f9fe0b89SLai Jiangshan /* 293f9fe0b89SLai Jiangshan * cpu_tss_rw is not directly referenced by hardware, but 294f9fe0b89SLai Jiangshan * cpu_tss_rw is also used in CPU entry code, 295f9fe0b89SLai Jiangshan */ 296f9fe0b89SLai Jiangshan if (within_area(addr, end, 297f9fe0b89SLai Jiangshan (unsigned long)&per_cpu(cpu_tss_rw, cpu), 298f9fe0b89SLai Jiangshan sizeof(struct tss_struct))) 299f9fe0b89SLai Jiangshan return true; 300fdef24dfSLai Jiangshan 301fdef24dfSLai Jiangshan /* 302fdef24dfSLai Jiangshan * cpu_tlbstate.user_pcid_flush_mask is used for CPU entry. 303fdef24dfSLai Jiangshan * If a data breakpoint on it, it will cause an unwanted #DB. 304fdef24dfSLai Jiangshan * Protect the full cpu_tlbstate structure to be sure. 305fdef24dfSLai Jiangshan */ 306fdef24dfSLai Jiangshan if (within_area(addr, end, 307fdef24dfSLai Jiangshan (unsigned long)&per_cpu(cpu_tlbstate, cpu), 308fdef24dfSLai Jiangshan sizeof(struct tlb_state))) 309fdef24dfSLai Jiangshan return true; 3103943abf2SLai Jiangshan 3113943abf2SLai Jiangshan /* 3123943abf2SLai Jiangshan * When in guest (X86_FEATURE_HYPERVISOR), local_db_save() 3133943abf2SLai Jiangshan * will read per-cpu cpu_dr7 before clear dr7 register. 3143943abf2SLai Jiangshan */ 3153943abf2SLai Jiangshan if (within_area(addr, end, (unsigned long)&per_cpu(cpu_dr7, cpu), 3163943abf2SLai Jiangshan sizeof(cpu_dr7))) 3173943abf2SLai Jiangshan return true; 31897417cb9SLai Jiangshan } 31997417cb9SLai Jiangshan 32097417cb9SLai Jiangshan return false; 32124ae0c91SAndy Lutomirski } 32224ae0c91SAndy Lutomirski 323a0baf043SFrederic Weisbecker static int arch_build_bp_info(struct perf_event *bp, 324a0baf043SFrederic Weisbecker const struct perf_event_attr *attr, 325a0baf043SFrederic Weisbecker struct arch_hw_breakpoint *hw) 32624f1e32cSFrederic Weisbecker { 32724ae0c91SAndy Lutomirski unsigned long bp_end; 32824ae0c91SAndy Lutomirski 32924ae0c91SAndy Lutomirski bp_end = attr->bp_addr + attr->bp_len - 1; 33024ae0c91SAndy Lutomirski if (bp_end < attr->bp_addr) 33124ae0c91SAndy Lutomirski return -EINVAL; 33224ae0c91SAndy Lutomirski 33324ae0c91SAndy Lutomirski /* 33497417cb9SLai Jiangshan * Prevent any breakpoint of any type that overlaps the CPU 33597417cb9SLai Jiangshan * entry area and data. This protects the IST stacks and also 33624ae0c91SAndy Lutomirski * reduces the chance that we ever find out what happens if 33724ae0c91SAndy Lutomirski * there's a data breakpoint on the GDT, IDT, or TSS. 33824ae0c91SAndy Lutomirski */ 33997417cb9SLai Jiangshan if (within_cpu_entry(attr->bp_addr, bp_end)) 34024ae0c91SAndy Lutomirski return -EINVAL; 34124ae0c91SAndy Lutomirski 342a0baf043SFrederic Weisbecker hw->address = attr->bp_addr; 343a0baf043SFrederic Weisbecker hw->mask = 0; 34424f1e32cSFrederic Weisbecker 345f7809dafSFrederic Weisbecker /* Type */ 346a0baf043SFrederic Weisbecker switch (attr->bp_type) { 347f7809dafSFrederic Weisbecker case HW_BREAKPOINT_W: 348a0baf043SFrederic Weisbecker hw->type = X86_BREAKPOINT_WRITE; 349f7809dafSFrederic Weisbecker break; 350f7809dafSFrederic Weisbecker case HW_BREAKPOINT_W | HW_BREAKPOINT_R: 351a0baf043SFrederic Weisbecker hw->type = X86_BREAKPOINT_RW; 352f7809dafSFrederic Weisbecker break; 353f7809dafSFrederic Weisbecker case HW_BREAKPOINT_X: 354e5779e8eSAndy Lutomirski /* 355e5779e8eSAndy Lutomirski * We don't allow kernel breakpoints in places that are not 356e5779e8eSAndy Lutomirski * acceptable for kprobes. On non-kprobes kernels, we don't 357e5779e8eSAndy Lutomirski * allow kernel breakpoints at all. 358e5779e8eSAndy Lutomirski */ 359a0baf043SFrederic Weisbecker if (attr->bp_addr >= TASK_SIZE_MAX) { 360a0baf043SFrederic Weisbecker if (within_kprobe_blacklist(attr->bp_addr)) 361e5779e8eSAndy Lutomirski return -EINVAL; 362e5779e8eSAndy Lutomirski } 363e5779e8eSAndy Lutomirski 364a0baf043SFrederic Weisbecker hw->type = X86_BREAKPOINT_EXECUTE; 365f7809dafSFrederic Weisbecker /* 366f7809dafSFrederic Weisbecker * x86 inst breakpoints need to have a specific undefined len. 367f7809dafSFrederic Weisbecker * But we still need to check userspace is not trying to setup 368f7809dafSFrederic Weisbecker * an unsupported length, to get a range breakpoint for example. 369f7809dafSFrederic Weisbecker */ 370a0baf043SFrederic Weisbecker if (attr->bp_len == sizeof(long)) { 371a0baf043SFrederic Weisbecker hw->len = X86_BREAKPOINT_LEN_X; 372f7809dafSFrederic Weisbecker return 0; 373f7809dafSFrederic Weisbecker } 374df561f66SGustavo A. R. Silva fallthrough; 375f7809dafSFrederic Weisbecker default: 376f7809dafSFrederic Weisbecker return -EINVAL; 377f7809dafSFrederic Weisbecker } 378f7809dafSFrederic Weisbecker 37924f1e32cSFrederic Weisbecker /* Len */ 380a0baf043SFrederic Weisbecker switch (attr->bp_len) { 38124f1e32cSFrederic Weisbecker case HW_BREAKPOINT_LEN_1: 382a0baf043SFrederic Weisbecker hw->len = X86_BREAKPOINT_LEN_1; 38324f1e32cSFrederic Weisbecker break; 38424f1e32cSFrederic Weisbecker case HW_BREAKPOINT_LEN_2: 385a0baf043SFrederic Weisbecker hw->len = X86_BREAKPOINT_LEN_2; 38624f1e32cSFrederic Weisbecker break; 38724f1e32cSFrederic Weisbecker case HW_BREAKPOINT_LEN_4: 388a0baf043SFrederic Weisbecker hw->len = X86_BREAKPOINT_LEN_4; 38924f1e32cSFrederic Weisbecker break; 39024f1e32cSFrederic Weisbecker #ifdef CONFIG_X86_64 39124f1e32cSFrederic Weisbecker case HW_BREAKPOINT_LEN_8: 392a0baf043SFrederic Weisbecker hw->len = X86_BREAKPOINT_LEN_8; 39324f1e32cSFrederic Weisbecker break; 39424f1e32cSFrederic Weisbecker #endif 39524f1e32cSFrederic Weisbecker default: 396ab513927SAndy Lutomirski /* AMD range breakpoint */ 397a0baf043SFrederic Weisbecker if (!is_power_of_2(attr->bp_len)) 39824f1e32cSFrederic Weisbecker return -EINVAL; 399a0baf043SFrederic Weisbecker if (attr->bp_addr & (attr->bp_len - 1)) 400ab513927SAndy Lutomirski return -EINVAL; 401362f924bSBorislav Petkov 402362f924bSBorislav Petkov if (!boot_cpu_has(X86_FEATURE_BPEXT)) 403362f924bSBorislav Petkov return -EOPNOTSUPP; 404362f924bSBorislav Petkov 405ab513927SAndy Lutomirski /* 406ab513927SAndy Lutomirski * It's impossible to use a range breakpoint to fake out 407ab513927SAndy Lutomirski * user vs kernel detection because bp_len - 1 can't 408ab513927SAndy Lutomirski * have the high bit set. If we ever allow range instruction 409ab513927SAndy Lutomirski * breakpoints, then we'll have to check for kprobe-blacklisted 410ab513927SAndy Lutomirski * addresses anywhere in the range. 411ab513927SAndy Lutomirski */ 412a0baf043SFrederic Weisbecker hw->mask = attr->bp_len - 1; 413a0baf043SFrederic Weisbecker hw->len = X86_BREAKPOINT_LEN_1; 41424f1e32cSFrederic Weisbecker } 41524f1e32cSFrederic Weisbecker 41624f1e32cSFrederic Weisbecker return 0; 41724f1e32cSFrederic Weisbecker } 418d6d55f0bSJacob Shin 4190067f129SK.Prasad /* 4200067f129SK.Prasad * Validate the arch-specific HW Breakpoint register settings 4210067f129SK.Prasad */ 422a0baf043SFrederic Weisbecker int hw_breakpoint_arch_parse(struct perf_event *bp, 423a0baf043SFrederic Weisbecker const struct perf_event_attr *attr, 424a0baf043SFrederic Weisbecker struct arch_hw_breakpoint *hw) 4250067f129SK.Prasad { 4260067f129SK.Prasad unsigned int align; 42724f1e32cSFrederic Weisbecker int ret; 4280067f129SK.Prasad 42924f1e32cSFrederic Weisbecker 430a0baf043SFrederic Weisbecker ret = arch_build_bp_info(bp, attr, hw); 43124f1e32cSFrederic Weisbecker if (ret) 43224f1e32cSFrederic Weisbecker return ret; 43324f1e32cSFrederic Weisbecker 434a0baf043SFrederic Weisbecker switch (hw->len) { 43524f1e32cSFrederic Weisbecker case X86_BREAKPOINT_LEN_1: 4360067f129SK.Prasad align = 0; 437a0baf043SFrederic Weisbecker if (hw->mask) 438a0baf043SFrederic Weisbecker align = hw->mask; 4390067f129SK.Prasad break; 44024f1e32cSFrederic Weisbecker case X86_BREAKPOINT_LEN_2: 4410067f129SK.Prasad align = 1; 4420067f129SK.Prasad break; 44324f1e32cSFrederic Weisbecker case X86_BREAKPOINT_LEN_4: 4440067f129SK.Prasad align = 3; 4450067f129SK.Prasad break; 4460067f129SK.Prasad #ifdef CONFIG_X86_64 44724f1e32cSFrederic Weisbecker case X86_BREAKPOINT_LEN_8: 4480067f129SK.Prasad align = 7; 4490067f129SK.Prasad break; 4500067f129SK.Prasad #endif 4510067f129SK.Prasad default: 452d6d55f0bSJacob Shin WARN_ON_ONCE(1); 453e898e69dSNathan Chancellor return -EINVAL; 4540067f129SK.Prasad } 4550067f129SK.Prasad 45684d71092SFrederic Weisbecker /* 4570067f129SK.Prasad * Check that the low-order bits of the address are appropriate 4580067f129SK.Prasad * for the alignment implied by len. 4590067f129SK.Prasad */ 460a0baf043SFrederic Weisbecker if (hw->address & align) 4610067f129SK.Prasad return -EINVAL; 4620067f129SK.Prasad 4630067f129SK.Prasad return 0; 4640067f129SK.Prasad } 4650067f129SK.Prasad 46624f1e32cSFrederic Weisbecker /* 46724f1e32cSFrederic Weisbecker * Release the user breakpoints used by ptrace 46824f1e32cSFrederic Weisbecker */ 46924f1e32cSFrederic Weisbecker void flush_ptrace_hw_breakpoint(struct task_struct *tsk) 4700067f129SK.Prasad { 4710067f129SK.Prasad int i; 47224f1e32cSFrederic Weisbecker struct thread_struct *t = &tsk->thread; 4730067f129SK.Prasad 47424f1e32cSFrederic Weisbecker for (i = 0; i < HBP_NUM; i++) { 47524f1e32cSFrederic Weisbecker unregister_hw_breakpoint(t->ptrace_bps[i]); 47624f1e32cSFrederic Weisbecker t->ptrace_bps[i] = NULL; 4770067f129SK.Prasad } 478f7da04c9SOleg Nesterov 479d53d9bc0SPeter Zijlstra t->virtual_dr6 = 0; 480f7da04c9SOleg Nesterov t->ptrace_dr7 = 0; 48124f1e32cSFrederic Weisbecker } 48224f1e32cSFrederic Weisbecker 48324f1e32cSFrederic Weisbecker void hw_breakpoint_restore(void) 48424f1e32cSFrederic Weisbecker { 4850a3aee0dSTejun Heo set_debugreg(__this_cpu_read(cpu_debugreg[0]), 0); 4860a3aee0dSTejun Heo set_debugreg(__this_cpu_read(cpu_debugreg[1]), 1); 4870a3aee0dSTejun Heo set_debugreg(__this_cpu_read(cpu_debugreg[2]), 2); 4880a3aee0dSTejun Heo set_debugreg(__this_cpu_read(cpu_debugreg[3]), 3); 489f4956cf8SPeter Zijlstra set_debugreg(DR6_RESERVED, 6); 4900a3aee0dSTejun Heo set_debugreg(__this_cpu_read(cpu_dr7), 7); 49124f1e32cSFrederic Weisbecker } 49224f1e32cSFrederic Weisbecker EXPORT_SYMBOL_GPL(hw_breakpoint_restore); 4930067f129SK.Prasad 4940067f129SK.Prasad /* 4950067f129SK.Prasad * Handle debug exception notifications. 4960067f129SK.Prasad * 4970067f129SK.Prasad * Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below. 4980067f129SK.Prasad * 4990067f129SK.Prasad * NOTIFY_DONE returned if one of the following conditions is true. 5000067f129SK.Prasad * i) When the causative address is from user-space and the exception 5010067f129SK.Prasad * is a valid one, i.e. not triggered as a result of lazy debug register 5020067f129SK.Prasad * switching 5030067f129SK.Prasad * ii) When there are more bits than trap<n> set in DR6 register (such 5040067f129SK.Prasad * as BD, BS or BT) indicating that more than one debug condition is 5050067f129SK.Prasad * met and requires some more action in do_debug(). 5060067f129SK.Prasad * 5070067f129SK.Prasad * NOTIFY_STOP returned for all other cases 5080067f129SK.Prasad * 5090067f129SK.Prasad */ 5109c54b616SMasami Hiramatsu static int hw_breakpoint_handler(struct die_args *args) 5110067f129SK.Prasad { 51221d44be7SPeter Zijlstra int i, rc = NOTIFY_STOP; 51324f1e32cSFrederic Weisbecker struct perf_event *bp; 51462edab90SK.Prasad unsigned long *dr6_p; 515d53d9bc0SPeter Zijlstra unsigned long dr6; 5169ad22e16SPeter Zijlstra bool bpx; 51762edab90SK.Prasad 51862edab90SK.Prasad /* The DR6 value is pointed by args->err */ 51962edab90SK.Prasad dr6_p = (unsigned long *)ERR_PTR(args->err); 52062edab90SK.Prasad dr6 = *dr6_p; 5210067f129SK.Prasad 5220067f129SK.Prasad /* Do an early return if no trap bits are set in DR6 */ 5230067f129SK.Prasad if ((dr6 & DR_TRAP_BITS) == 0) 5240067f129SK.Prasad return NOTIFY_DONE; 5250067f129SK.Prasad 5260067f129SK.Prasad /* Handle all the breakpoints that were triggered */ 5270067f129SK.Prasad for (i = 0; i < HBP_NUM; ++i) { 5280067f129SK.Prasad if (likely(!(dr6 & (DR_TRAP0 << i)))) 5290067f129SK.Prasad continue; 53024f1e32cSFrederic Weisbecker 53121d44be7SPeter Zijlstra bp = this_cpu_read(bp_per_reg[i]); 5329ad22e16SPeter Zijlstra if (!bp) 5339ad22e16SPeter Zijlstra continue; 5349ad22e16SPeter Zijlstra 5359ad22e16SPeter Zijlstra bpx = bp->hw.info.type == X86_BREAKPOINT_EXECUTE; 5369ad22e16SPeter Zijlstra 5379ad22e16SPeter Zijlstra /* 5389ad22e16SPeter Zijlstra * TF and data breakpoints are traps and can be merged, however 5399ad22e16SPeter Zijlstra * instruction breakpoints are faults and will be raised 5409ad22e16SPeter Zijlstra * separately. 5419ad22e16SPeter Zijlstra * 5429ad22e16SPeter Zijlstra * However DR6 can indicate both TF and instruction 5439ad22e16SPeter Zijlstra * breakpoints. In that case take TF as that has precedence and 5449ad22e16SPeter Zijlstra * delay the instruction breakpoint for the next exception. 5459ad22e16SPeter Zijlstra */ 5469ad22e16SPeter Zijlstra if (bpx && (dr6 & DR_STEP)) 5479ad22e16SPeter Zijlstra continue; 5489ad22e16SPeter Zijlstra 5490067f129SK.Prasad /* 55062edab90SK.Prasad * Reset the 'i'th TRAP bit in dr6 to denote completion of 55162edab90SK.Prasad * exception handling 55262edab90SK.Prasad */ 55362edab90SK.Prasad (*dr6_p) &= ~(DR_TRAP0 << i); 5540067f129SK.Prasad 555b326e956SFrederic Weisbecker perf_bp_event(bp, args->regs); 55624f1e32cSFrederic Weisbecker 5570c4519e8SFrederic Weisbecker /* 5580c4519e8SFrederic Weisbecker * Set up resume flag to avoid breakpoint recursion when 5590c4519e8SFrederic Weisbecker * returning back to origin. 5600c4519e8SFrederic Weisbecker */ 5619ad22e16SPeter Zijlstra if (bpx) 5620c4519e8SFrederic Weisbecker args->regs->flags |= X86_EFLAGS_RF; 5630067f129SK.Prasad } 5649ad22e16SPeter Zijlstra 565e0e53db6SK.Prasad /* 566e0e53db6SK.Prasad * Further processing in do_debug() is needed for a) user-space 567e0e53db6SK.Prasad * breakpoints (to generate signals) and b) when the system has 568e0e53db6SK.Prasad * taken exception due to multiple causes 569e0e53db6SK.Prasad */ 570d53d9bc0SPeter Zijlstra if ((current->thread.virtual_dr6 & DR_TRAP_BITS) || 571e0e53db6SK.Prasad (dr6 & (~DR_TRAP_BITS))) 5720067f129SK.Prasad rc = NOTIFY_DONE; 5730067f129SK.Prasad 5740067f129SK.Prasad return rc; 5750067f129SK.Prasad } 5760067f129SK.Prasad 5770067f129SK.Prasad /* 5780067f129SK.Prasad * Handle debug exception notifications. 5790067f129SK.Prasad */ 5809c54b616SMasami Hiramatsu int hw_breakpoint_exceptions_notify( 5810067f129SK.Prasad struct notifier_block *unused, unsigned long val, void *data) 5820067f129SK.Prasad { 5830067f129SK.Prasad if (val != DIE_DEBUG) 5840067f129SK.Prasad return NOTIFY_DONE; 5850067f129SK.Prasad 5860067f129SK.Prasad return hw_breakpoint_handler(data); 5870067f129SK.Prasad } 58824f1e32cSFrederic Weisbecker 58924f1e32cSFrederic Weisbecker void hw_breakpoint_pmu_read(struct perf_event *bp) 59024f1e32cSFrederic Weisbecker { 59124f1e32cSFrederic Weisbecker /* TODO */ 59224f1e32cSFrederic Weisbecker } 593