1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * local apic based NMI watchdog for various CPUs.
4  *
5  * This file also handles reservation of performance counters for coordination
6  * with other users (like oprofile).
7  *
8  * Note that these events normally don't tick when the CPU idles. This means
9  * the frequency varies with CPU load.
10  *
11  * Original code for K7/P6 written by Keith Owens
12  *
13  */
14 
15 #include <linux/percpu.h>
16 #include <linux/export.h>
17 #include <linux/kernel.h>
18 #include <linux/bitops.h>
19 #include <linux/smp.h>
20 #include <asm/nmi.h>
21 #include <linux/kprobes.h>
22 
23 #include <asm/apic.h>
24 #include <asm/perf_event.h>
25 
26 /*
27  * this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's
28  * offset from MSR_P4_BSU_ESCR0.
29  *
30  * It will be the max for all platforms (for now)
31  */
32 #define NMI_MAX_COUNTER_BITS 66
33 
34 /*
35  * perfctr_nmi_owner tracks the ownership of the perfctr registers:
36  * evtsel_nmi_owner tracks the ownership of the event selection
37  * - different performance counters/ event selection may be reserved for
38  *   different subsystems this reservation system just tries to coordinate
39  *   things a little
40  */
41 static DECLARE_BITMAP(perfctr_nmi_owner, NMI_MAX_COUNTER_BITS);
42 static DECLARE_BITMAP(evntsel_nmi_owner, NMI_MAX_COUNTER_BITS);
43 
44 /* converts an msr to an appropriate reservation bit */
45 static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr)
46 {
47 	/* returns the bit offset of the performance counter register */
48 	switch (boot_cpu_data.x86_vendor) {
49 	case X86_VENDOR_HYGON:
50 	case X86_VENDOR_AMD:
51 		if (msr >= MSR_F15H_PERF_CTR)
52 			return (msr - MSR_F15H_PERF_CTR) >> 1;
53 		return msr - MSR_K7_PERFCTR0;
54 	case X86_VENDOR_INTEL:
55 		if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
56 			return msr - MSR_ARCH_PERFMON_PERFCTR0;
57 
58 		switch (boot_cpu_data.x86) {
59 		case 6:
60 			return msr - MSR_P6_PERFCTR0;
61 		case 11:
62 			return msr - MSR_KNC_PERFCTR0;
63 		case 15:
64 			return msr - MSR_P4_BPU_PERFCTR0;
65 		}
66 	}
67 	return 0;
68 }
69 
70 /*
71  * converts an msr to an appropriate reservation bit
72  * returns the bit offset of the event selection register
73  */
74 static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
75 {
76 	/* returns the bit offset of the event selection register */
77 	switch (boot_cpu_data.x86_vendor) {
78 	case X86_VENDOR_HYGON:
79 	case X86_VENDOR_AMD:
80 		if (msr >= MSR_F15H_PERF_CTL)
81 			return (msr - MSR_F15H_PERF_CTL) >> 1;
82 		return msr - MSR_K7_EVNTSEL0;
83 	case X86_VENDOR_INTEL:
84 		if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
85 			return msr - MSR_ARCH_PERFMON_EVENTSEL0;
86 
87 		switch (boot_cpu_data.x86) {
88 		case 6:
89 			return msr - MSR_P6_EVNTSEL0;
90 		case 11:
91 			return msr - MSR_KNC_EVNTSEL0;
92 		case 15:
93 			return msr - MSR_P4_BSU_ESCR0;
94 		}
95 	}
96 	return 0;
97 
98 }
99 
100 /* checks for a bit availability (hack for oprofile) */
101 int avail_to_resrv_perfctr_nmi_bit(unsigned int counter)
102 {
103 	BUG_ON(counter > NMI_MAX_COUNTER_BITS);
104 
105 	return !test_bit(counter, perfctr_nmi_owner);
106 }
107 EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit);
108 
109 int reserve_perfctr_nmi(unsigned int msr)
110 {
111 	unsigned int counter;
112 
113 	counter = nmi_perfctr_msr_to_bit(msr);
114 	/* register not managed by the allocator? */
115 	if (counter > NMI_MAX_COUNTER_BITS)
116 		return 1;
117 
118 	if (!test_and_set_bit(counter, perfctr_nmi_owner))
119 		return 1;
120 	return 0;
121 }
122 EXPORT_SYMBOL(reserve_perfctr_nmi);
123 
124 void release_perfctr_nmi(unsigned int msr)
125 {
126 	unsigned int counter;
127 
128 	counter = nmi_perfctr_msr_to_bit(msr);
129 	/* register not managed by the allocator? */
130 	if (counter > NMI_MAX_COUNTER_BITS)
131 		return;
132 
133 	clear_bit(counter, perfctr_nmi_owner);
134 }
135 EXPORT_SYMBOL(release_perfctr_nmi);
136 
137 int reserve_evntsel_nmi(unsigned int msr)
138 {
139 	unsigned int counter;
140 
141 	counter = nmi_evntsel_msr_to_bit(msr);
142 	/* register not managed by the allocator? */
143 	if (counter > NMI_MAX_COUNTER_BITS)
144 		return 1;
145 
146 	if (!test_and_set_bit(counter, evntsel_nmi_owner))
147 		return 1;
148 	return 0;
149 }
150 EXPORT_SYMBOL(reserve_evntsel_nmi);
151 
152 void release_evntsel_nmi(unsigned int msr)
153 {
154 	unsigned int counter;
155 
156 	counter = nmi_evntsel_msr_to_bit(msr);
157 	/* register not managed by the allocator? */
158 	if (counter > NMI_MAX_COUNTER_BITS)
159 		return;
160 
161 	clear_bit(counter, evntsel_nmi_owner);
162 }
163 EXPORT_SYMBOL(release_evntsel_nmi);
164