xref: /openbmc/linux/arch/arm/kernel/perf_event_v7.c (revision a2cce7a9)
1 /*
2  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
3  *
4  * ARMv7 support: Jean Pihet <jpihet@mvista.com>
5  * 2010 (c) MontaVista Software, LLC.
6  *
7  * Copied from ARMv6 code, with the low level code inspired
8  *  by the ARMv7 Oprofile code.
9  *
10  * Cortex-A8 has up to 4 configurable performance counters and
11  *  a single cycle counter.
12  * Cortex-A9 has up to 31 configurable performance counters and
13  *  a single cycle counter.
14  *
15  * All counters can be enabled/disabled and IRQ masked separately. The cycle
16  *  counter and all 4 performance counters together can be reset separately.
17  */
18 
19 #ifdef CONFIG_CPU_V7
20 
21 #include <asm/cp15.h>
22 #include <asm/cputype.h>
23 #include <asm/irq_regs.h>
24 #include <asm/vfp.h>
25 #include "../vfp/vfpinstr.h"
26 
27 #include <linux/of.h>
28 #include <linux/perf/arm_pmu.h>
29 #include <linux/platform_device.h>
30 
31 /*
32  * Common ARMv7 event types
33  *
34  * Note: An implementation may not be able to count all of these events
35  * but the encodings are considered to be `reserved' in the case that
36  * they are not available.
37  */
38 enum armv7_perf_types {
39 	ARMV7_PERFCTR_PMNC_SW_INCR			= 0x00,
40 	ARMV7_PERFCTR_L1_ICACHE_REFILL			= 0x01,
41 	ARMV7_PERFCTR_ITLB_REFILL			= 0x02,
42 	ARMV7_PERFCTR_L1_DCACHE_REFILL			= 0x03,
43 	ARMV7_PERFCTR_L1_DCACHE_ACCESS			= 0x04,
44 	ARMV7_PERFCTR_DTLB_REFILL			= 0x05,
45 	ARMV7_PERFCTR_MEM_READ				= 0x06,
46 	ARMV7_PERFCTR_MEM_WRITE				= 0x07,
47 	ARMV7_PERFCTR_INSTR_EXECUTED			= 0x08,
48 	ARMV7_PERFCTR_EXC_TAKEN				= 0x09,
49 	ARMV7_PERFCTR_EXC_EXECUTED			= 0x0A,
50 	ARMV7_PERFCTR_CID_WRITE				= 0x0B,
51 
52 	/*
53 	 * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
54 	 * It counts:
55 	 *  - all (taken) branch instructions,
56 	 *  - instructions that explicitly write the PC,
57 	 *  - exception generating instructions.
58 	 */
59 	ARMV7_PERFCTR_PC_WRITE				= 0x0C,
60 	ARMV7_PERFCTR_PC_IMM_BRANCH			= 0x0D,
61 	ARMV7_PERFCTR_PC_PROC_RETURN			= 0x0E,
62 	ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS		= 0x0F,
63 	ARMV7_PERFCTR_PC_BRANCH_MIS_PRED		= 0x10,
64 	ARMV7_PERFCTR_CLOCK_CYCLES			= 0x11,
65 	ARMV7_PERFCTR_PC_BRANCH_PRED			= 0x12,
66 
67 	/* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
68 	ARMV7_PERFCTR_MEM_ACCESS			= 0x13,
69 	ARMV7_PERFCTR_L1_ICACHE_ACCESS			= 0x14,
70 	ARMV7_PERFCTR_L1_DCACHE_WB			= 0x15,
71 	ARMV7_PERFCTR_L2_CACHE_ACCESS			= 0x16,
72 	ARMV7_PERFCTR_L2_CACHE_REFILL			= 0x17,
73 	ARMV7_PERFCTR_L2_CACHE_WB			= 0x18,
74 	ARMV7_PERFCTR_BUS_ACCESS			= 0x19,
75 	ARMV7_PERFCTR_MEM_ERROR				= 0x1A,
76 	ARMV7_PERFCTR_INSTR_SPEC			= 0x1B,
77 	ARMV7_PERFCTR_TTBR_WRITE			= 0x1C,
78 	ARMV7_PERFCTR_BUS_CYCLES			= 0x1D,
79 
80 	ARMV7_PERFCTR_CPU_CYCLES			= 0xFF
81 };
82 
83 /* ARMv7 Cortex-A8 specific event types */
84 enum armv7_a8_perf_types {
85 	ARMV7_A8_PERFCTR_L2_CACHE_ACCESS		= 0x43,
86 	ARMV7_A8_PERFCTR_L2_CACHE_REFILL		= 0x44,
87 	ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS		= 0x50,
88 	ARMV7_A8_PERFCTR_STALL_ISIDE			= 0x56,
89 };
90 
91 /* ARMv7 Cortex-A9 specific event types */
92 enum armv7_a9_perf_types {
93 	ARMV7_A9_PERFCTR_INSTR_CORE_RENAME		= 0x68,
94 	ARMV7_A9_PERFCTR_STALL_ICACHE			= 0x60,
95 	ARMV7_A9_PERFCTR_STALL_DISPATCH			= 0x66,
96 };
97 
98 /* ARMv7 Cortex-A5 specific event types */
99 enum armv7_a5_perf_types {
100 	ARMV7_A5_PERFCTR_PREFETCH_LINEFILL		= 0xc2,
101 	ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP		= 0xc3,
102 };
103 
104 /* ARMv7 Cortex-A15 specific event types */
105 enum armv7_a15_perf_types {
106 	ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ		= 0x40,
107 	ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE	= 0x41,
108 	ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ		= 0x42,
109 	ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE	= 0x43,
110 
111 	ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ		= 0x4C,
112 	ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE		= 0x4D,
113 
114 	ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ		= 0x50,
115 	ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE		= 0x51,
116 	ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ		= 0x52,
117 	ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE		= 0x53,
118 
119 	ARMV7_A15_PERFCTR_PC_WRITE_SPEC			= 0x76,
120 };
121 
122 /* ARMv7 Cortex-A12 specific event types */
123 enum armv7_a12_perf_types {
124 	ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ		= 0x40,
125 	ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE	= 0x41,
126 
127 	ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ		= 0x50,
128 	ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE		= 0x51,
129 
130 	ARMV7_A12_PERFCTR_PC_WRITE_SPEC			= 0x76,
131 
132 	ARMV7_A12_PERFCTR_PF_TLB_REFILL			= 0xe7,
133 };
134 
135 /* ARMv7 Krait specific event types */
136 enum krait_perf_types {
137 	KRAIT_PMRESR0_GROUP0				= 0xcc,
138 	KRAIT_PMRESR1_GROUP0				= 0xd0,
139 	KRAIT_PMRESR2_GROUP0				= 0xd4,
140 	KRAIT_VPMRESR0_GROUP0				= 0xd8,
141 
142 	KRAIT_PERFCTR_L1_ICACHE_ACCESS			= 0x10011,
143 	KRAIT_PERFCTR_L1_ICACHE_MISS			= 0x10010,
144 
145 	KRAIT_PERFCTR_L1_ITLB_ACCESS			= 0x12222,
146 	KRAIT_PERFCTR_L1_DTLB_ACCESS			= 0x12210,
147 };
148 
149 /* ARMv7 Scorpion specific event types */
150 enum scorpion_perf_types {
151 	SCORPION_LPM0_GROUP0				= 0x4c,
152 	SCORPION_LPM1_GROUP0				= 0x50,
153 	SCORPION_LPM2_GROUP0				= 0x54,
154 	SCORPION_L2LPM_GROUP0				= 0x58,
155 	SCORPION_VLPM_GROUP0				= 0x5c,
156 
157 	SCORPION_ICACHE_ACCESS				= 0x10053,
158 	SCORPION_ICACHE_MISS				= 0x10052,
159 
160 	SCORPION_DTLB_ACCESS				= 0x12013,
161 	SCORPION_DTLB_MISS				= 0x12012,
162 
163 	SCORPION_ITLB_MISS				= 0x12021,
164 };
165 
166 /*
167  * Cortex-A8 HW events mapping
168  *
169  * The hardware events that we support. We do support cache operations but
170  * we have harvard caches and no way to combine instruction and data
171  * accesses/misses in hardware.
172  */
173 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
174 	PERF_MAP_ALL_UNSUPPORTED,
175 	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV7_PERFCTR_CPU_CYCLES,
176 	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV7_PERFCTR_INSTR_EXECUTED,
177 	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
178 	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV7_PERFCTR_L1_DCACHE_REFILL,
179 	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= ARMV7_PERFCTR_PC_WRITE,
180 	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
181 	[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND]	= ARMV7_A8_PERFCTR_STALL_ISIDE,
182 };
183 
184 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
185 					  [PERF_COUNT_HW_CACHE_OP_MAX]
186 					  [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
187 	PERF_CACHE_MAP_ALL_UNSUPPORTED,
188 
189 	/*
190 	 * The performance counters don't differentiate between read and write
191 	 * accesses/misses so this isn't strictly correct, but it's the best we
192 	 * can do. Writes and reads get combined.
193 	 */
194 	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
195 	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
196 	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
197 	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
198 
199 	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
200 	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_ICACHE_REFILL,
201 
202 	[C(LL)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
203 	[C(LL)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
204 	[C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
205 	[C(LL)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
206 
207 	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
208 	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
209 
210 	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
211 	[C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
212 
213 	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
214 	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
215 	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
216 	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
217 };
218 
219 /*
220  * Cortex-A9 HW events mapping
221  */
222 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
223 	PERF_MAP_ALL_UNSUPPORTED,
224 	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV7_PERFCTR_CPU_CYCLES,
225 	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
226 	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
227 	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV7_PERFCTR_L1_DCACHE_REFILL,
228 	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= ARMV7_PERFCTR_PC_WRITE,
229 	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
230 	[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND]	= ARMV7_A9_PERFCTR_STALL_ICACHE,
231 	[PERF_COUNT_HW_STALLED_CYCLES_BACKEND]	= ARMV7_A9_PERFCTR_STALL_DISPATCH,
232 };
233 
234 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
235 					  [PERF_COUNT_HW_CACHE_OP_MAX]
236 					  [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
237 	PERF_CACHE_MAP_ALL_UNSUPPORTED,
238 
239 	/*
240 	 * The performance counters don't differentiate between read and write
241 	 * accesses/misses so this isn't strictly correct, but it's the best we
242 	 * can do. Writes and reads get combined.
243 	 */
244 	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
245 	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
246 	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
247 	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
248 
249 	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_ICACHE_REFILL,
250 
251 	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
252 	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
253 
254 	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
255 	[C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
256 
257 	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
258 	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
259 	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
260 	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
261 };
262 
263 /*
264  * Cortex-A5 HW events mapping
265  */
266 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
267 	PERF_MAP_ALL_UNSUPPORTED,
268 	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV7_PERFCTR_CPU_CYCLES,
269 	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV7_PERFCTR_INSTR_EXECUTED,
270 	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
271 	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV7_PERFCTR_L1_DCACHE_REFILL,
272 	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= ARMV7_PERFCTR_PC_WRITE,
273 	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
274 };
275 
276 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
277 					[PERF_COUNT_HW_CACHE_OP_MAX]
278 					[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
279 	PERF_CACHE_MAP_ALL_UNSUPPORTED,
280 
281 	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
282 	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
283 	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
284 	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
285 	[C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)]	= ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
286 	[C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)]	= ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
287 
288 	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_ICACHE_ACCESS,
289 	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_ICACHE_REFILL,
290 	/*
291 	 * The prefetch counters don't differentiate between the I side and the
292 	 * D side.
293 	 */
294 	[C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)]	= ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
295 	[C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)]	= ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
296 
297 	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
298 	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
299 
300 	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
301 	[C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
302 
303 	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
304 	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
305 	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
306 	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
307 };
308 
309 /*
310  * Cortex-A15 HW events mapping
311  */
312 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
313 	PERF_MAP_ALL_UNSUPPORTED,
314 	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV7_PERFCTR_CPU_CYCLES,
315 	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV7_PERFCTR_INSTR_EXECUTED,
316 	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
317 	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV7_PERFCTR_L1_DCACHE_REFILL,
318 	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
319 	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
320 	[PERF_COUNT_HW_BUS_CYCLES]		= ARMV7_PERFCTR_BUS_CYCLES,
321 };
322 
323 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
324 					[PERF_COUNT_HW_CACHE_OP_MAX]
325 					[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
326 	PERF_CACHE_MAP_ALL_UNSUPPORTED,
327 
328 	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
329 	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
330 	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
331 	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
332 
333 	/*
334 	 * Not all performance counters differentiate between read and write
335 	 * accesses/misses so we're not always strictly correct, but it's the
336 	 * best we can do. Writes and reads get combined in these cases.
337 	 */
338 	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_ICACHE_ACCESS,
339 	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_ICACHE_REFILL,
340 
341 	[C(LL)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
342 	[C(LL)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
343 	[C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
344 	[C(LL)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
345 
346 	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
347 	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
348 
349 	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
350 	[C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
351 
352 	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
353 	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
354 	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
355 	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
356 };
357 
358 /*
359  * Cortex-A7 HW events mapping
360  */
361 static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {
362 	PERF_MAP_ALL_UNSUPPORTED,
363 	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV7_PERFCTR_CPU_CYCLES,
364 	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV7_PERFCTR_INSTR_EXECUTED,
365 	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
366 	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV7_PERFCTR_L1_DCACHE_REFILL,
367 	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= ARMV7_PERFCTR_PC_WRITE,
368 	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
369 	[PERF_COUNT_HW_BUS_CYCLES]		= ARMV7_PERFCTR_BUS_CYCLES,
370 };
371 
372 static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
373 					[PERF_COUNT_HW_CACHE_OP_MAX]
374 					[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
375 	PERF_CACHE_MAP_ALL_UNSUPPORTED,
376 
377 	/*
378 	 * The performance counters don't differentiate between read and write
379 	 * accesses/misses so this isn't strictly correct, but it's the best we
380 	 * can do. Writes and reads get combined.
381 	 */
382 	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
383 	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
384 	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
385 	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
386 
387 	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_ICACHE_ACCESS,
388 	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_ICACHE_REFILL,
389 
390 	[C(LL)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L2_CACHE_ACCESS,
391 	[C(LL)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L2_CACHE_REFILL,
392 	[C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L2_CACHE_ACCESS,
393 	[C(LL)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L2_CACHE_REFILL,
394 
395 	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
396 	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
397 
398 	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
399 	[C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
400 
401 	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
402 	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
403 	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
404 	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
405 };
406 
407 /*
408  * Cortex-A12 HW events mapping
409  */
410 static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = {
411 	PERF_MAP_ALL_UNSUPPORTED,
412 	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV7_PERFCTR_CPU_CYCLES,
413 	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV7_PERFCTR_INSTR_EXECUTED,
414 	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
415 	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV7_PERFCTR_L1_DCACHE_REFILL,
416 	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= ARMV7_A12_PERFCTR_PC_WRITE_SPEC,
417 	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
418 	[PERF_COUNT_HW_BUS_CYCLES]		= ARMV7_PERFCTR_BUS_CYCLES,
419 };
420 
421 static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
422 					[PERF_COUNT_HW_CACHE_OP_MAX]
423 					[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
424 	PERF_CACHE_MAP_ALL_UNSUPPORTED,
425 
426 	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ,
427 	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
428 	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE,
429 	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
430 
431 	/*
432 	 * Not all performance counters differentiate between read and write
433 	 * accesses/misses so we're not always strictly correct, but it's the
434 	 * best we can do. Writes and reads get combined in these cases.
435 	 */
436 	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_ICACHE_ACCESS,
437 	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_ICACHE_REFILL,
438 
439 	[C(LL)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ,
440 	[C(LL)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L2_CACHE_REFILL,
441 	[C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE,
442 	[C(LL)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L2_CACHE_REFILL,
443 
444 	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
445 	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_DTLB_REFILL,
446 	[C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)]	= ARMV7_A12_PERFCTR_PF_TLB_REFILL,
447 
448 	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
449 	[C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_ITLB_REFILL,
450 
451 	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
452 	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
453 	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
454 	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
455 };
456 
457 /*
458  * Krait HW events mapping
459  */
460 static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = {
461 	PERF_MAP_ALL_UNSUPPORTED,
462 	[PERF_COUNT_HW_CPU_CYCLES]	    = ARMV7_PERFCTR_CPU_CYCLES,
463 	[PERF_COUNT_HW_INSTRUCTIONS]	    = ARMV7_PERFCTR_INSTR_EXECUTED,
464 	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
465 	[PERF_COUNT_HW_BRANCH_MISSES]	    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
466 	[PERF_COUNT_HW_BUS_CYCLES]	    = ARMV7_PERFCTR_CLOCK_CYCLES,
467 };
468 
469 static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = {
470 	PERF_MAP_ALL_UNSUPPORTED,
471 	[PERF_COUNT_HW_CPU_CYCLES]	    = ARMV7_PERFCTR_CPU_CYCLES,
472 	[PERF_COUNT_HW_INSTRUCTIONS]	    = ARMV7_PERFCTR_INSTR_EXECUTED,
473 	[PERF_COUNT_HW_BRANCH_MISSES]	    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
474 	[PERF_COUNT_HW_BUS_CYCLES]	    = ARMV7_PERFCTR_CLOCK_CYCLES,
475 };
476 
477 static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
478 					  [PERF_COUNT_HW_CACHE_OP_MAX]
479 					  [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
480 	PERF_CACHE_MAP_ALL_UNSUPPORTED,
481 
482 	/*
483 	 * The performance counters don't differentiate between read and write
484 	 * accesses/misses so this isn't strictly correct, but it's the best we
485 	 * can do. Writes and reads get combined.
486 	 */
487 	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
488 	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
489 	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_L1_DCACHE_ACCESS,
490 	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_L1_DCACHE_REFILL,
491 
492 	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= KRAIT_PERFCTR_L1_ICACHE_ACCESS,
493 	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= KRAIT_PERFCTR_L1_ICACHE_MISS,
494 
495 	[C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)]	= KRAIT_PERFCTR_L1_DTLB_ACCESS,
496 	[C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)]	= KRAIT_PERFCTR_L1_DTLB_ACCESS,
497 
498 	[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)]	= KRAIT_PERFCTR_L1_ITLB_ACCESS,
499 	[C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)]	= KRAIT_PERFCTR_L1_ITLB_ACCESS,
500 
501 	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
502 	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
503 	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV7_PERFCTR_PC_BRANCH_PRED,
504 	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
505 };
506 
507 /*
508  * Scorpion HW events mapping
509  */
510 static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = {
511 	PERF_MAP_ALL_UNSUPPORTED,
512 	[PERF_COUNT_HW_CPU_CYCLES]	    = ARMV7_PERFCTR_CPU_CYCLES,
513 	[PERF_COUNT_HW_INSTRUCTIONS]	    = ARMV7_PERFCTR_INSTR_EXECUTED,
514 	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
515 	[PERF_COUNT_HW_BRANCH_MISSES]	    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
516 	[PERF_COUNT_HW_BUS_CYCLES]	    = ARMV7_PERFCTR_CLOCK_CYCLES,
517 };
518 
519 static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
520 					    [PERF_COUNT_HW_CACHE_OP_MAX]
521 					    [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
522 	PERF_CACHE_MAP_ALL_UNSUPPORTED,
523 	/*
524 	 * The performance counters don't differentiate between read and write
525 	 * accesses/misses so this isn't strictly correct, but it's the best we
526 	 * can do. Writes and reads get combined.
527 	 */
528 	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
529 	[C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
530 	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
531 	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
532 	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS,
533 	[C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS,
534 	/*
535 	 * Only ITLB misses and DTLB refills are supported.  If users want the
536 	 * DTLB refills misses a raw counter must be used.
537 	 */
538 	[C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
539 	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
540 	[C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
541 	[C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
542 	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
543 	[C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
544 	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
545 	[C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
546 	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
547 	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
548 };
549 
550 /*
551  * Perf Events' indices
552  */
553 #define	ARMV7_IDX_CYCLE_COUNTER	0
554 #define	ARMV7_IDX_COUNTER0	1
555 #define	ARMV7_IDX_COUNTER_LAST(cpu_pmu) \
556 	(ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
557 
558 #define	ARMV7_MAX_COUNTERS	32
559 #define	ARMV7_COUNTER_MASK	(ARMV7_MAX_COUNTERS - 1)
560 
561 /*
562  * ARMv7 low level PMNC access
563  */
564 
565 /*
566  * Perf Event to low level counters mapping
567  */
568 #define	ARMV7_IDX_TO_COUNTER(x)	\
569 	(((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
570 
571 /*
572  * Per-CPU PMNC: config reg
573  */
574 #define ARMV7_PMNC_E		(1 << 0) /* Enable all counters */
575 #define ARMV7_PMNC_P		(1 << 1) /* Reset all counters */
576 #define ARMV7_PMNC_C		(1 << 2) /* Cycle counter reset */
577 #define ARMV7_PMNC_D		(1 << 3) /* CCNT counts every 64th cpu cycle */
578 #define ARMV7_PMNC_X		(1 << 4) /* Export to ETM */
579 #define ARMV7_PMNC_DP		(1 << 5) /* Disable CCNT if non-invasive debug*/
580 #define	ARMV7_PMNC_N_SHIFT	11	 /* Number of counters supported */
581 #define	ARMV7_PMNC_N_MASK	0x1f
582 #define	ARMV7_PMNC_MASK		0x3f	 /* Mask for writable bits */
583 
584 /*
585  * FLAG: counters overflow flag status reg
586  */
587 #define	ARMV7_FLAG_MASK		0xffffffff	/* Mask for writable bits */
588 #define	ARMV7_OVERFLOWED_MASK	ARMV7_FLAG_MASK
589 
590 /*
591  * PMXEVTYPER: Event selection reg
592  */
593 #define	ARMV7_EVTYPE_MASK	0xc80000ff	/* Mask for writable bits */
594 #define	ARMV7_EVTYPE_EVENT	0xff		/* Mask for EVENT bits */
595 
596 /*
597  * Event filters for PMUv2
598  */
599 #define	ARMV7_EXCLUDE_PL1	(1 << 31)
600 #define	ARMV7_EXCLUDE_USER	(1 << 30)
601 #define	ARMV7_INCLUDE_HYP	(1 << 27)
602 
603 static inline u32 armv7_pmnc_read(void)
604 {
605 	u32 val;
606 	asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
607 	return val;
608 }
609 
610 static inline void armv7_pmnc_write(u32 val)
611 {
612 	val &= ARMV7_PMNC_MASK;
613 	isb();
614 	asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
615 }
616 
617 static inline int armv7_pmnc_has_overflowed(u32 pmnc)
618 {
619 	return pmnc & ARMV7_OVERFLOWED_MASK;
620 }
621 
622 static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx)
623 {
624 	return idx >= ARMV7_IDX_CYCLE_COUNTER &&
625 		idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu);
626 }
627 
628 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
629 {
630 	return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx));
631 }
632 
633 static inline void armv7_pmnc_select_counter(int idx)
634 {
635 	u32 counter = ARMV7_IDX_TO_COUNTER(idx);
636 	asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
637 	isb();
638 }
639 
640 static inline u32 armv7pmu_read_counter(struct perf_event *event)
641 {
642 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
643 	struct hw_perf_event *hwc = &event->hw;
644 	int idx = hwc->idx;
645 	u32 value = 0;
646 
647 	if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
648 		pr_err("CPU%u reading wrong counter %d\n",
649 			smp_processor_id(), idx);
650 	} else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
651 		asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
652 	} else {
653 		armv7_pmnc_select_counter(idx);
654 		asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
655 	}
656 
657 	return value;
658 }
659 
660 static inline void armv7pmu_write_counter(struct perf_event *event, u32 value)
661 {
662 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
663 	struct hw_perf_event *hwc = &event->hw;
664 	int idx = hwc->idx;
665 
666 	if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
667 		pr_err("CPU%u writing wrong counter %d\n",
668 			smp_processor_id(), idx);
669 	} else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
670 		asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
671 	} else {
672 		armv7_pmnc_select_counter(idx);
673 		asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
674 	}
675 }
676 
677 static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
678 {
679 	armv7_pmnc_select_counter(idx);
680 	val &= ARMV7_EVTYPE_MASK;
681 	asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
682 }
683 
684 static inline void armv7_pmnc_enable_counter(int idx)
685 {
686 	u32 counter = ARMV7_IDX_TO_COUNTER(idx);
687 	asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
688 }
689 
690 static inline void armv7_pmnc_disable_counter(int idx)
691 {
692 	u32 counter = ARMV7_IDX_TO_COUNTER(idx);
693 	asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
694 }
695 
696 static inline void armv7_pmnc_enable_intens(int idx)
697 {
698 	u32 counter = ARMV7_IDX_TO_COUNTER(idx);
699 	asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
700 }
701 
702 static inline void armv7_pmnc_disable_intens(int idx)
703 {
704 	u32 counter = ARMV7_IDX_TO_COUNTER(idx);
705 	asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
706 	isb();
707 	/* Clear the overflow flag in case an interrupt is pending. */
708 	asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
709 	isb();
710 }
711 
712 static inline u32 armv7_pmnc_getreset_flags(void)
713 {
714 	u32 val;
715 
716 	/* Read */
717 	asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
718 
719 	/* Write to clear flags */
720 	val &= ARMV7_FLAG_MASK;
721 	asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
722 
723 	return val;
724 }
725 
726 #ifdef DEBUG
727 static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu)
728 {
729 	u32 val;
730 	unsigned int cnt;
731 
732 	pr_info("PMNC registers dump:\n");
733 
734 	asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
735 	pr_info("PMNC  =0x%08x\n", val);
736 
737 	asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
738 	pr_info("CNTENS=0x%08x\n", val);
739 
740 	asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
741 	pr_info("INTENS=0x%08x\n", val);
742 
743 	asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
744 	pr_info("FLAGS =0x%08x\n", val);
745 
746 	asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
747 	pr_info("SELECT=0x%08x\n", val);
748 
749 	asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
750 	pr_info("CCNT  =0x%08x\n", val);
751 
752 	for (cnt = ARMV7_IDX_COUNTER0;
753 			cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) {
754 		armv7_pmnc_select_counter(cnt);
755 		asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
756 		pr_info("CNT[%d] count =0x%08x\n",
757 			ARMV7_IDX_TO_COUNTER(cnt), val);
758 		asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
759 		pr_info("CNT[%d] evtsel=0x%08x\n",
760 			ARMV7_IDX_TO_COUNTER(cnt), val);
761 	}
762 }
763 #endif
764 
765 static void armv7pmu_enable_event(struct perf_event *event)
766 {
767 	unsigned long flags;
768 	struct hw_perf_event *hwc = &event->hw;
769 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
770 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
771 	int idx = hwc->idx;
772 
773 	if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
774 		pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
775 			smp_processor_id(), idx);
776 		return;
777 	}
778 
779 	/*
780 	 * Enable counter and interrupt, and set the counter to count
781 	 * the event that we're interested in.
782 	 */
783 	raw_spin_lock_irqsave(&events->pmu_lock, flags);
784 
785 	/*
786 	 * Disable counter
787 	 */
788 	armv7_pmnc_disable_counter(idx);
789 
790 	/*
791 	 * Set event (if destined for PMNx counters)
792 	 * We only need to set the event for the cycle counter if we
793 	 * have the ability to perform event filtering.
794 	 */
795 	if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
796 		armv7_pmnc_write_evtsel(idx, hwc->config_base);
797 
798 	/*
799 	 * Enable interrupt for this counter
800 	 */
801 	armv7_pmnc_enable_intens(idx);
802 
803 	/*
804 	 * Enable counter
805 	 */
806 	armv7_pmnc_enable_counter(idx);
807 
808 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
809 }
810 
811 static void armv7pmu_disable_event(struct perf_event *event)
812 {
813 	unsigned long flags;
814 	struct hw_perf_event *hwc = &event->hw;
815 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
816 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
817 	int idx = hwc->idx;
818 
819 	if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
820 		pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
821 			smp_processor_id(), idx);
822 		return;
823 	}
824 
825 	/*
826 	 * Disable counter and interrupt
827 	 */
828 	raw_spin_lock_irqsave(&events->pmu_lock, flags);
829 
830 	/*
831 	 * Disable counter
832 	 */
833 	armv7_pmnc_disable_counter(idx);
834 
835 	/*
836 	 * Disable interrupt for this counter
837 	 */
838 	armv7_pmnc_disable_intens(idx);
839 
840 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
841 }
842 
843 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
844 {
845 	u32 pmnc;
846 	struct perf_sample_data data;
847 	struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
848 	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
849 	struct pt_regs *regs;
850 	int idx;
851 
852 	/*
853 	 * Get and reset the IRQ flags
854 	 */
855 	pmnc = armv7_pmnc_getreset_flags();
856 
857 	/*
858 	 * Did an overflow occur?
859 	 */
860 	if (!armv7_pmnc_has_overflowed(pmnc))
861 		return IRQ_NONE;
862 
863 	/*
864 	 * Handle the counter(s) overflow(s)
865 	 */
866 	regs = get_irq_regs();
867 
868 	for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
869 		struct perf_event *event = cpuc->events[idx];
870 		struct hw_perf_event *hwc;
871 
872 		/* Ignore if we don't have an event. */
873 		if (!event)
874 			continue;
875 
876 		/*
877 		 * We have a single interrupt for all counters. Check that
878 		 * each counter has overflowed before we process it.
879 		 */
880 		if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
881 			continue;
882 
883 		hwc = &event->hw;
884 		armpmu_event_update(event);
885 		perf_sample_data_init(&data, 0, hwc->last_period);
886 		if (!armpmu_event_set_period(event))
887 			continue;
888 
889 		if (perf_event_overflow(event, &data, regs))
890 			cpu_pmu->disable(event);
891 	}
892 
893 	/*
894 	 * Handle the pending perf events.
895 	 *
896 	 * Note: this call *must* be run with interrupts disabled. For
897 	 * platforms that can have the PMU interrupts raised as an NMI, this
898 	 * will not work.
899 	 */
900 	irq_work_run();
901 
902 	return IRQ_HANDLED;
903 }
904 
905 static void armv7pmu_start(struct arm_pmu *cpu_pmu)
906 {
907 	unsigned long flags;
908 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
909 
910 	raw_spin_lock_irqsave(&events->pmu_lock, flags);
911 	/* Enable all counters */
912 	armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
913 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
914 }
915 
916 static void armv7pmu_stop(struct arm_pmu *cpu_pmu)
917 {
918 	unsigned long flags;
919 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
920 
921 	raw_spin_lock_irqsave(&events->pmu_lock, flags);
922 	/* Disable all counters */
923 	armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
924 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
925 }
926 
927 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
928 				  struct perf_event *event)
929 {
930 	int idx;
931 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
932 	struct hw_perf_event *hwc = &event->hw;
933 	unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;
934 
935 	/* Always place a cycle counter into the cycle counter. */
936 	if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
937 		if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
938 			return -EAGAIN;
939 
940 		return ARMV7_IDX_CYCLE_COUNTER;
941 	}
942 
943 	/*
944 	 * For anything other than a cycle counter, try and use
945 	 * the events counters
946 	 */
947 	for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
948 		if (!test_and_set_bit(idx, cpuc->used_mask))
949 			return idx;
950 	}
951 
952 	/* The counters are all in use. */
953 	return -EAGAIN;
954 }
955 
956 /*
957  * Add an event filter to a given event. This will only work for PMUv2 PMUs.
958  */
959 static int armv7pmu_set_event_filter(struct hw_perf_event *event,
960 				     struct perf_event_attr *attr)
961 {
962 	unsigned long config_base = 0;
963 
964 	if (attr->exclude_idle)
965 		return -EPERM;
966 	if (attr->exclude_user)
967 		config_base |= ARMV7_EXCLUDE_USER;
968 	if (attr->exclude_kernel)
969 		config_base |= ARMV7_EXCLUDE_PL1;
970 	if (!attr->exclude_hv)
971 		config_base |= ARMV7_INCLUDE_HYP;
972 
973 	/*
974 	 * Install the filter into config_base as this is used to
975 	 * construct the event type.
976 	 */
977 	event->config_base = config_base;
978 
979 	return 0;
980 }
981 
982 static void armv7pmu_reset(void *info)
983 {
984 	struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
985 	u32 idx, nb_cnt = cpu_pmu->num_events;
986 
987 	/* The counter and interrupt enable registers are unknown at reset. */
988 	for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
989 		armv7_pmnc_disable_counter(idx);
990 		armv7_pmnc_disable_intens(idx);
991 	}
992 
993 	/* Initialize & Reset PMNC: C and P bits */
994 	armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
995 }
996 
997 static int armv7_a8_map_event(struct perf_event *event)
998 {
999 	return armpmu_map_event(event, &armv7_a8_perf_map,
1000 				&armv7_a8_perf_cache_map, 0xFF);
1001 }
1002 
1003 static int armv7_a9_map_event(struct perf_event *event)
1004 {
1005 	return armpmu_map_event(event, &armv7_a9_perf_map,
1006 				&armv7_a9_perf_cache_map, 0xFF);
1007 }
1008 
1009 static int armv7_a5_map_event(struct perf_event *event)
1010 {
1011 	return armpmu_map_event(event, &armv7_a5_perf_map,
1012 				&armv7_a5_perf_cache_map, 0xFF);
1013 }
1014 
1015 static int armv7_a15_map_event(struct perf_event *event)
1016 {
1017 	return armpmu_map_event(event, &armv7_a15_perf_map,
1018 				&armv7_a15_perf_cache_map, 0xFF);
1019 }
1020 
1021 static int armv7_a7_map_event(struct perf_event *event)
1022 {
1023 	return armpmu_map_event(event, &armv7_a7_perf_map,
1024 				&armv7_a7_perf_cache_map, 0xFF);
1025 }
1026 
1027 static int armv7_a12_map_event(struct perf_event *event)
1028 {
1029 	return armpmu_map_event(event, &armv7_a12_perf_map,
1030 				&armv7_a12_perf_cache_map, 0xFF);
1031 }
1032 
1033 static int krait_map_event(struct perf_event *event)
1034 {
1035 	return armpmu_map_event(event, &krait_perf_map,
1036 				&krait_perf_cache_map, 0xFFFFF);
1037 }
1038 
1039 static int krait_map_event_no_branch(struct perf_event *event)
1040 {
1041 	return armpmu_map_event(event, &krait_perf_map_no_branch,
1042 				&krait_perf_cache_map, 0xFFFFF);
1043 }
1044 
1045 static int scorpion_map_event(struct perf_event *event)
1046 {
1047 	return armpmu_map_event(event, &scorpion_perf_map,
1048 				&scorpion_perf_cache_map, 0xFFFFF);
1049 }
1050 
1051 static void armv7pmu_init(struct arm_pmu *cpu_pmu)
1052 {
1053 	cpu_pmu->handle_irq	= armv7pmu_handle_irq;
1054 	cpu_pmu->enable		= armv7pmu_enable_event;
1055 	cpu_pmu->disable	= armv7pmu_disable_event;
1056 	cpu_pmu->read_counter	= armv7pmu_read_counter;
1057 	cpu_pmu->write_counter	= armv7pmu_write_counter;
1058 	cpu_pmu->get_event_idx	= armv7pmu_get_event_idx;
1059 	cpu_pmu->start		= armv7pmu_start;
1060 	cpu_pmu->stop		= armv7pmu_stop;
1061 	cpu_pmu->reset		= armv7pmu_reset;
1062 	cpu_pmu->max_period	= (1LLU << 32) - 1;
1063 };
1064 
1065 static void armv7_read_num_pmnc_events(void *info)
1066 {
1067 	int *nb_cnt = info;
1068 
1069 	/* Read the nb of CNTx counters supported from PMNC */
1070 	*nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1071 
1072 	/* Add the CPU cycles counter */
1073 	*nb_cnt += 1;
1074 }
1075 
1076 static int armv7_probe_num_events(struct arm_pmu *arm_pmu)
1077 {
1078 	return smp_call_function_any(&arm_pmu->supported_cpus,
1079 				     armv7_read_num_pmnc_events,
1080 				     &arm_pmu->num_events, 1);
1081 }
1082 
1083 static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1084 {
1085 	armv7pmu_init(cpu_pmu);
1086 	cpu_pmu->name		= "armv7_cortex_a8";
1087 	cpu_pmu->map_event	= armv7_a8_map_event;
1088 	return armv7_probe_num_events(cpu_pmu);
1089 }
1090 
1091 static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1092 {
1093 	armv7pmu_init(cpu_pmu);
1094 	cpu_pmu->name		= "armv7_cortex_a9";
1095 	cpu_pmu->map_event	= armv7_a9_map_event;
1096 	return armv7_probe_num_events(cpu_pmu);
1097 }
1098 
1099 static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1100 {
1101 	armv7pmu_init(cpu_pmu);
1102 	cpu_pmu->name		= "armv7_cortex_a5";
1103 	cpu_pmu->map_event	= armv7_a5_map_event;
1104 	return armv7_probe_num_events(cpu_pmu);
1105 }
1106 
1107 static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1108 {
1109 	armv7pmu_init(cpu_pmu);
1110 	cpu_pmu->name		= "armv7_cortex_a15";
1111 	cpu_pmu->map_event	= armv7_a15_map_event;
1112 	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1113 	return armv7_probe_num_events(cpu_pmu);
1114 }
1115 
1116 static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1117 {
1118 	armv7pmu_init(cpu_pmu);
1119 	cpu_pmu->name		= "armv7_cortex_a7";
1120 	cpu_pmu->map_event	= armv7_a7_map_event;
1121 	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1122 	return armv7_probe_num_events(cpu_pmu);
1123 }
1124 
1125 static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)
1126 {
1127 	armv7pmu_init(cpu_pmu);
1128 	cpu_pmu->name		= "armv7_cortex_a12";
1129 	cpu_pmu->map_event	= armv7_a12_map_event;
1130 	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1131 	return armv7_probe_num_events(cpu_pmu);
1132 }
1133 
1134 static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
1135 {
1136 	int ret = armv7_a12_pmu_init(cpu_pmu);
1137 	cpu_pmu->name = "armv7_cortex_a17";
1138 	return ret;
1139 }
1140 
1141 /*
1142  * Krait Performance Monitor Region Event Selection Register (PMRESRn)
1143  *
1144  *            31   30     24     16     8      0
1145  *            +--------------------------------+
1146  *  PMRESR0   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 0
1147  *            +--------------------------------+
1148  *  PMRESR1   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 1
1149  *            +--------------------------------+
1150  *  PMRESR2   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 2
1151  *            +--------------------------------+
1152  *  VPMRESR0  | EN |  CC  |  CC  |  CC  |  CC  |   N = 2, R = ?
1153  *            +--------------------------------+
1154  *              EN | G=3  | G=2  | G=1  | G=0
1155  *
1156  *  Event Encoding:
1157  *
1158  *      hwc->config_base = 0xNRCCG
1159  *
1160  *      N  = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR)
1161  *      R  = region register
1162  *      CC = class of events the group G is choosing from
1163  *      G  = group or particular event
1164  *
1165  *  Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2
1166  *
1167  *  A region (R) corresponds to a piece of the CPU (execution unit, instruction
1168  *  unit, etc.) while the event code (CC) corresponds to a particular class of
1169  *  events (interrupts for example). An event code is broken down into
1170  *  groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1171  *  example).
1172  */
1173 
1174 #define KRAIT_EVENT		(1 << 16)
1175 #define VENUM_EVENT		(2 << 16)
1176 #define KRAIT_EVENT_MASK	(KRAIT_EVENT | VENUM_EVENT)
1177 #define PMRESRn_EN		BIT(31)
1178 
1179 #define EVENT_REGION(event)	(((event) >> 12) & 0xf)		/* R */
1180 #define EVENT_GROUP(event)	((event) & 0xf)			/* G */
1181 #define EVENT_CODE(event)	(((event) >> 4) & 0xff)		/* CC */
1182 #define EVENT_VENUM(event)	(!!(event & VENUM_EVENT))	/* N=2 */
1183 #define EVENT_CPU(event)	(!!(event & KRAIT_EVENT))	/* N=1 */
1184 
1185 static u32 krait_read_pmresrn(int n)
1186 {
1187 	u32 val;
1188 
1189 	switch (n) {
1190 	case 0:
1191 		asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val));
1192 		break;
1193 	case 1:
1194 		asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val));
1195 		break;
1196 	case 2:
1197 		asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val));
1198 		break;
1199 	default:
1200 		BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1201 	}
1202 
1203 	return val;
1204 }
1205 
1206 static void krait_write_pmresrn(int n, u32 val)
1207 {
1208 	switch (n) {
1209 	case 0:
1210 		asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val));
1211 		break;
1212 	case 1:
1213 		asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val));
1214 		break;
1215 	case 2:
1216 		asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val));
1217 		break;
1218 	default:
1219 		BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1220 	}
1221 }
1222 
1223 static u32 venum_read_pmresr(void)
1224 {
1225 	u32 val;
1226 	asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val));
1227 	return val;
1228 }
1229 
1230 static void venum_write_pmresr(u32 val)
1231 {
1232 	asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val));
1233 }
1234 
1235 static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val)
1236 {
1237 	u32 venum_new_val;
1238 	u32 fp_new_val;
1239 
1240 	BUG_ON(preemptible());
1241 	/* CPACR Enable CP10 and CP11 access */
1242 	*venum_orig_val = get_copro_access();
1243 	venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11);
1244 	set_copro_access(venum_new_val);
1245 
1246 	/* Enable FPEXC */
1247 	*fp_orig_val = fmrx(FPEXC);
1248 	fp_new_val = *fp_orig_val | FPEXC_EN;
1249 	fmxr(FPEXC, fp_new_val);
1250 }
1251 
1252 static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val)
1253 {
1254 	BUG_ON(preemptible());
1255 	/* Restore FPEXC */
1256 	fmxr(FPEXC, fp_orig_val);
1257 	isb();
1258 	/* Restore CPACR */
1259 	set_copro_access(venum_orig_val);
1260 }
1261 
1262 static u32 krait_get_pmresrn_event(unsigned int region)
1263 {
1264 	static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0,
1265 					     KRAIT_PMRESR1_GROUP0,
1266 					     KRAIT_PMRESR2_GROUP0 };
1267 	return pmresrn_table[region];
1268 }
1269 
1270 static void krait_evt_setup(int idx, u32 config_base)
1271 {
1272 	u32 val;
1273 	u32 mask;
1274 	u32 vval, fval;
1275 	unsigned int region = EVENT_REGION(config_base);
1276 	unsigned int group = EVENT_GROUP(config_base);
1277 	unsigned int code = EVENT_CODE(config_base);
1278 	unsigned int group_shift;
1279 	bool venum_event = EVENT_VENUM(config_base);
1280 
1281 	group_shift = group * 8;
1282 	mask = 0xff << group_shift;
1283 
1284 	/* Configure evtsel for the region and group */
1285 	if (venum_event)
1286 		val = KRAIT_VPMRESR0_GROUP0;
1287 	else
1288 		val = krait_get_pmresrn_event(region);
1289 	val += group;
1290 	/* Mix in mode-exclusion bits */
1291 	val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1292 	armv7_pmnc_write_evtsel(idx, val);
1293 
1294 	if (venum_event) {
1295 		venum_pre_pmresr(&vval, &fval);
1296 		val = venum_read_pmresr();
1297 		val &= ~mask;
1298 		val |= code << group_shift;
1299 		val |= PMRESRn_EN;
1300 		venum_write_pmresr(val);
1301 		venum_post_pmresr(vval, fval);
1302 	} else {
1303 		val = krait_read_pmresrn(region);
1304 		val &= ~mask;
1305 		val |= code << group_shift;
1306 		val |= PMRESRn_EN;
1307 		krait_write_pmresrn(region, val);
1308 	}
1309 }
1310 
1311 static u32 clear_pmresrn_group(u32 val, int group)
1312 {
1313 	u32 mask;
1314 	int group_shift;
1315 
1316 	group_shift = group * 8;
1317 	mask = 0xff << group_shift;
1318 	val &= ~mask;
1319 
1320 	/* Don't clear enable bit if entire region isn't disabled */
1321 	if (val & ~PMRESRn_EN)
1322 		return val |= PMRESRn_EN;
1323 
1324 	return 0;
1325 }
1326 
1327 static void krait_clearpmu(u32 config_base)
1328 {
1329 	u32 val;
1330 	u32 vval, fval;
1331 	unsigned int region = EVENT_REGION(config_base);
1332 	unsigned int group = EVENT_GROUP(config_base);
1333 	bool venum_event = EVENT_VENUM(config_base);
1334 
1335 	if (venum_event) {
1336 		venum_pre_pmresr(&vval, &fval);
1337 		val = venum_read_pmresr();
1338 		val = clear_pmresrn_group(val, group);
1339 		venum_write_pmresr(val);
1340 		venum_post_pmresr(vval, fval);
1341 	} else {
1342 		val = krait_read_pmresrn(region);
1343 		val = clear_pmresrn_group(val, group);
1344 		krait_write_pmresrn(region, val);
1345 	}
1346 }
1347 
1348 static void krait_pmu_disable_event(struct perf_event *event)
1349 {
1350 	unsigned long flags;
1351 	struct hw_perf_event *hwc = &event->hw;
1352 	int idx = hwc->idx;
1353 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1354 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1355 
1356 	/* Disable counter and interrupt */
1357 	raw_spin_lock_irqsave(&events->pmu_lock, flags);
1358 
1359 	/* Disable counter */
1360 	armv7_pmnc_disable_counter(idx);
1361 
1362 	/*
1363 	 * Clear pmresr code (if destined for PMNx counters)
1364 	 */
1365 	if (hwc->config_base & KRAIT_EVENT_MASK)
1366 		krait_clearpmu(hwc->config_base);
1367 
1368 	/* Disable interrupt for this counter */
1369 	armv7_pmnc_disable_intens(idx);
1370 
1371 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1372 }
1373 
1374 static void krait_pmu_enable_event(struct perf_event *event)
1375 {
1376 	unsigned long flags;
1377 	struct hw_perf_event *hwc = &event->hw;
1378 	int idx = hwc->idx;
1379 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1380 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1381 
1382 	/*
1383 	 * Enable counter and interrupt, and set the counter to count
1384 	 * the event that we're interested in.
1385 	 */
1386 	raw_spin_lock_irqsave(&events->pmu_lock, flags);
1387 
1388 	/* Disable counter */
1389 	armv7_pmnc_disable_counter(idx);
1390 
1391 	/*
1392 	 * Set event (if destined for PMNx counters)
1393 	 * We set the event for the cycle counter because we
1394 	 * have the ability to perform event filtering.
1395 	 */
1396 	if (hwc->config_base & KRAIT_EVENT_MASK)
1397 		krait_evt_setup(idx, hwc->config_base);
1398 	else
1399 		armv7_pmnc_write_evtsel(idx, hwc->config_base);
1400 
1401 	/* Enable interrupt for this counter */
1402 	armv7_pmnc_enable_intens(idx);
1403 
1404 	/* Enable counter */
1405 	armv7_pmnc_enable_counter(idx);
1406 
1407 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1408 }
1409 
1410 static void krait_pmu_reset(void *info)
1411 {
1412 	u32 vval, fval;
1413 	struct arm_pmu *cpu_pmu = info;
1414 	u32 idx, nb_cnt = cpu_pmu->num_events;
1415 
1416 	armv7pmu_reset(info);
1417 
1418 	/* Clear all pmresrs */
1419 	krait_write_pmresrn(0, 0);
1420 	krait_write_pmresrn(1, 0);
1421 	krait_write_pmresrn(2, 0);
1422 
1423 	venum_pre_pmresr(&vval, &fval);
1424 	venum_write_pmresr(0);
1425 	venum_post_pmresr(vval, fval);
1426 
1427 	/* Reset PMxEVNCTCR to sane default */
1428 	for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1429 		armv7_pmnc_select_counter(idx);
1430 		asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1431 	}
1432 
1433 }
1434 
1435 static int krait_event_to_bit(struct perf_event *event, unsigned int region,
1436 			      unsigned int group)
1437 {
1438 	int bit;
1439 	struct hw_perf_event *hwc = &event->hw;
1440 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1441 
1442 	if (hwc->config_base & VENUM_EVENT)
1443 		bit = KRAIT_VPMRESR0_GROUP0;
1444 	else
1445 		bit = krait_get_pmresrn_event(region);
1446 	bit -= krait_get_pmresrn_event(0);
1447 	bit += group;
1448 	/*
1449 	 * Lower bits are reserved for use by the counters (see
1450 	 * armv7pmu_get_event_idx() for more info)
1451 	 */
1452 	bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1453 
1454 	return bit;
1455 }
1456 
1457 /*
1458  * We check for column exclusion constraints here.
1459  * Two events cant use the same group within a pmresr register.
1460  */
1461 static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1462 				   struct perf_event *event)
1463 {
1464 	int idx;
1465 	int bit = -1;
1466 	struct hw_perf_event *hwc = &event->hw;
1467 	unsigned int region = EVENT_REGION(hwc->config_base);
1468 	unsigned int code = EVENT_CODE(hwc->config_base);
1469 	unsigned int group = EVENT_GROUP(hwc->config_base);
1470 	bool venum_event = EVENT_VENUM(hwc->config_base);
1471 	bool krait_event = EVENT_CPU(hwc->config_base);
1472 
1473 	if (venum_event || krait_event) {
1474 		/* Ignore invalid events */
1475 		if (group > 3 || region > 2)
1476 			return -EINVAL;
1477 		if (venum_event && (code & 0xe0))
1478 			return -EINVAL;
1479 
1480 		bit = krait_event_to_bit(event, region, group);
1481 		if (test_and_set_bit(bit, cpuc->used_mask))
1482 			return -EAGAIN;
1483 	}
1484 
1485 	idx = armv7pmu_get_event_idx(cpuc, event);
1486 	if (idx < 0 && bit >= 0)
1487 		clear_bit(bit, cpuc->used_mask);
1488 
1489 	return idx;
1490 }
1491 
1492 static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1493 				      struct perf_event *event)
1494 {
1495 	int bit;
1496 	struct hw_perf_event *hwc = &event->hw;
1497 	unsigned int region = EVENT_REGION(hwc->config_base);
1498 	unsigned int group = EVENT_GROUP(hwc->config_base);
1499 	bool venum_event = EVENT_VENUM(hwc->config_base);
1500 	bool krait_event = EVENT_CPU(hwc->config_base);
1501 
1502 	if (venum_event || krait_event) {
1503 		bit = krait_event_to_bit(event, region, group);
1504 		clear_bit(bit, cpuc->used_mask);
1505 	}
1506 }
1507 
1508 static int krait_pmu_init(struct arm_pmu *cpu_pmu)
1509 {
1510 	armv7pmu_init(cpu_pmu);
1511 	cpu_pmu->name		= "armv7_krait";
1512 	/* Some early versions of Krait don't support PC write events */
1513 	if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node,
1514 				  "qcom,no-pc-write"))
1515 		cpu_pmu->map_event = krait_map_event_no_branch;
1516 	else
1517 		cpu_pmu->map_event = krait_map_event;
1518 	cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1519 	cpu_pmu->reset		= krait_pmu_reset;
1520 	cpu_pmu->enable		= krait_pmu_enable_event;
1521 	cpu_pmu->disable	= krait_pmu_disable_event;
1522 	cpu_pmu->get_event_idx	= krait_pmu_get_event_idx;
1523 	cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;
1524 	return armv7_probe_num_events(cpu_pmu);
1525 }
1526 
1527 /*
1528  * Scorpion Local Performance Monitor Register (LPMn)
1529  *
1530  *            31   30     24     16     8      0
1531  *            +--------------------------------+
1532  *  LPM0      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 0
1533  *            +--------------------------------+
1534  *  LPM1      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 1
1535  *            +--------------------------------+
1536  *  LPM2      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 2
1537  *            +--------------------------------+
1538  *  L2LPM     | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 3
1539  *            +--------------------------------+
1540  *  VLPM      | EN |  CC  |  CC  |  CC  |  CC  |   N = 2, R = ?
1541  *            +--------------------------------+
1542  *              EN | G=3  | G=2  | G=1  | G=0
1543  *
1544  *
1545  *  Event Encoding:
1546  *
1547  *      hwc->config_base = 0xNRCCG
1548  *
1549  *      N  = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM)
1550  *      R  = region register
1551  *      CC = class of events the group G is choosing from
1552  *      G  = group or particular event
1553  *
1554  *  Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2
1555  *
1556  *  A region (R) corresponds to a piece of the CPU (execution unit, instruction
1557  *  unit, etc.) while the event code (CC) corresponds to a particular class of
1558  *  events (interrupts for example). An event code is broken down into
1559  *  groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1560  *  example).
1561  */
1562 
1563 static u32 scorpion_read_pmresrn(int n)
1564 {
1565 	u32 val;
1566 
1567 	switch (n) {
1568 	case 0:
1569 		asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val));
1570 		break;
1571 	case 1:
1572 		asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val));
1573 		break;
1574 	case 2:
1575 		asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val));
1576 		break;
1577 	case 3:
1578 		asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val));
1579 		break;
1580 	default:
1581 		BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1582 	}
1583 
1584 	return val;
1585 }
1586 
1587 static void scorpion_write_pmresrn(int n, u32 val)
1588 {
1589 	switch (n) {
1590 	case 0:
1591 		asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val));
1592 		break;
1593 	case 1:
1594 		asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val));
1595 		break;
1596 	case 2:
1597 		asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val));
1598 		break;
1599 	case 3:
1600 		asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val));
1601 		break;
1602 	default:
1603 		BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1604 	}
1605 }
1606 
1607 static u32 scorpion_get_pmresrn_event(unsigned int region)
1608 {
1609 	static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0,
1610 					     SCORPION_LPM1_GROUP0,
1611 					     SCORPION_LPM2_GROUP0,
1612 					     SCORPION_L2LPM_GROUP0 };
1613 	return pmresrn_table[region];
1614 }
1615 
1616 static void scorpion_evt_setup(int idx, u32 config_base)
1617 {
1618 	u32 val;
1619 	u32 mask;
1620 	u32 vval, fval;
1621 	unsigned int region = EVENT_REGION(config_base);
1622 	unsigned int group = EVENT_GROUP(config_base);
1623 	unsigned int code = EVENT_CODE(config_base);
1624 	unsigned int group_shift;
1625 	bool venum_event = EVENT_VENUM(config_base);
1626 
1627 	group_shift = group * 8;
1628 	mask = 0xff << group_shift;
1629 
1630 	/* Configure evtsel for the region and group */
1631 	if (venum_event)
1632 		val = SCORPION_VLPM_GROUP0;
1633 	else
1634 		val = scorpion_get_pmresrn_event(region);
1635 	val += group;
1636 	/* Mix in mode-exclusion bits */
1637 	val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1638 	armv7_pmnc_write_evtsel(idx, val);
1639 
1640 	asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1641 
1642 	if (venum_event) {
1643 		venum_pre_pmresr(&vval, &fval);
1644 		val = venum_read_pmresr();
1645 		val &= ~mask;
1646 		val |= code << group_shift;
1647 		val |= PMRESRn_EN;
1648 		venum_write_pmresr(val);
1649 		venum_post_pmresr(vval, fval);
1650 	} else {
1651 		val = scorpion_read_pmresrn(region);
1652 		val &= ~mask;
1653 		val |= code << group_shift;
1654 		val |= PMRESRn_EN;
1655 		scorpion_write_pmresrn(region, val);
1656 	}
1657 }
1658 
1659 static void scorpion_clearpmu(u32 config_base)
1660 {
1661 	u32 val;
1662 	u32 vval, fval;
1663 	unsigned int region = EVENT_REGION(config_base);
1664 	unsigned int group = EVENT_GROUP(config_base);
1665 	bool venum_event = EVENT_VENUM(config_base);
1666 
1667 	if (venum_event) {
1668 		venum_pre_pmresr(&vval, &fval);
1669 		val = venum_read_pmresr();
1670 		val = clear_pmresrn_group(val, group);
1671 		venum_write_pmresr(val);
1672 		venum_post_pmresr(vval, fval);
1673 	} else {
1674 		val = scorpion_read_pmresrn(region);
1675 		val = clear_pmresrn_group(val, group);
1676 		scorpion_write_pmresrn(region, val);
1677 	}
1678 }
1679 
1680 static void scorpion_pmu_disable_event(struct perf_event *event)
1681 {
1682 	unsigned long flags;
1683 	struct hw_perf_event *hwc = &event->hw;
1684 	int idx = hwc->idx;
1685 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1686 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1687 
1688 	/* Disable counter and interrupt */
1689 	raw_spin_lock_irqsave(&events->pmu_lock, flags);
1690 
1691 	/* Disable counter */
1692 	armv7_pmnc_disable_counter(idx);
1693 
1694 	/*
1695 	 * Clear pmresr code (if destined for PMNx counters)
1696 	 */
1697 	if (hwc->config_base & KRAIT_EVENT_MASK)
1698 		scorpion_clearpmu(hwc->config_base);
1699 
1700 	/* Disable interrupt for this counter */
1701 	armv7_pmnc_disable_intens(idx);
1702 
1703 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1704 }
1705 
1706 static void scorpion_pmu_enable_event(struct perf_event *event)
1707 {
1708 	unsigned long flags;
1709 	struct hw_perf_event *hwc = &event->hw;
1710 	int idx = hwc->idx;
1711 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1712 	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1713 
1714 	/*
1715 	 * Enable counter and interrupt, and set the counter to count
1716 	 * the event that we're interested in.
1717 	 */
1718 	raw_spin_lock_irqsave(&events->pmu_lock, flags);
1719 
1720 	/* Disable counter */
1721 	armv7_pmnc_disable_counter(idx);
1722 
1723 	/*
1724 	 * Set event (if destined for PMNx counters)
1725 	 * We don't set the event for the cycle counter because we
1726 	 * don't have the ability to perform event filtering.
1727 	 */
1728 	if (hwc->config_base & KRAIT_EVENT_MASK)
1729 		scorpion_evt_setup(idx, hwc->config_base);
1730 	else if (idx != ARMV7_IDX_CYCLE_COUNTER)
1731 		armv7_pmnc_write_evtsel(idx, hwc->config_base);
1732 
1733 	/* Enable interrupt for this counter */
1734 	armv7_pmnc_enable_intens(idx);
1735 
1736 	/* Enable counter */
1737 	armv7_pmnc_enable_counter(idx);
1738 
1739 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1740 }
1741 
1742 static void scorpion_pmu_reset(void *info)
1743 {
1744 	u32 vval, fval;
1745 	struct arm_pmu *cpu_pmu = info;
1746 	u32 idx, nb_cnt = cpu_pmu->num_events;
1747 
1748 	armv7pmu_reset(info);
1749 
1750 	/* Clear all pmresrs */
1751 	scorpion_write_pmresrn(0, 0);
1752 	scorpion_write_pmresrn(1, 0);
1753 	scorpion_write_pmresrn(2, 0);
1754 	scorpion_write_pmresrn(3, 0);
1755 
1756 	venum_pre_pmresr(&vval, &fval);
1757 	venum_write_pmresr(0);
1758 	venum_post_pmresr(vval, fval);
1759 
1760 	/* Reset PMxEVNCTCR to sane default */
1761 	for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1762 		armv7_pmnc_select_counter(idx);
1763 		asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1764 	}
1765 }
1766 
1767 static int scorpion_event_to_bit(struct perf_event *event, unsigned int region,
1768 			      unsigned int group)
1769 {
1770 	int bit;
1771 	struct hw_perf_event *hwc = &event->hw;
1772 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1773 
1774 	if (hwc->config_base & VENUM_EVENT)
1775 		bit = SCORPION_VLPM_GROUP0;
1776 	else
1777 		bit = scorpion_get_pmresrn_event(region);
1778 	bit -= scorpion_get_pmresrn_event(0);
1779 	bit += group;
1780 	/*
1781 	 * Lower bits are reserved for use by the counters (see
1782 	 * armv7pmu_get_event_idx() for more info)
1783 	 */
1784 	bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1785 
1786 	return bit;
1787 }
1788 
1789 /*
1790  * We check for column exclusion constraints here.
1791  * Two events cant use the same group within a pmresr register.
1792  */
1793 static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1794 				   struct perf_event *event)
1795 {
1796 	int idx;
1797 	int bit = -1;
1798 	struct hw_perf_event *hwc = &event->hw;
1799 	unsigned int region = EVENT_REGION(hwc->config_base);
1800 	unsigned int group = EVENT_GROUP(hwc->config_base);
1801 	bool venum_event = EVENT_VENUM(hwc->config_base);
1802 	bool scorpion_event = EVENT_CPU(hwc->config_base);
1803 
1804 	if (venum_event || scorpion_event) {
1805 		/* Ignore invalid events */
1806 		if (group > 3 || region > 3)
1807 			return -EINVAL;
1808 
1809 		bit = scorpion_event_to_bit(event, region, group);
1810 		if (test_and_set_bit(bit, cpuc->used_mask))
1811 			return -EAGAIN;
1812 	}
1813 
1814 	idx = armv7pmu_get_event_idx(cpuc, event);
1815 	if (idx < 0 && bit >= 0)
1816 		clear_bit(bit, cpuc->used_mask);
1817 
1818 	return idx;
1819 }
1820 
1821 static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1822 				      struct perf_event *event)
1823 {
1824 	int bit;
1825 	struct hw_perf_event *hwc = &event->hw;
1826 	unsigned int region = EVENT_REGION(hwc->config_base);
1827 	unsigned int group = EVENT_GROUP(hwc->config_base);
1828 	bool venum_event = EVENT_VENUM(hwc->config_base);
1829 	bool scorpion_event = EVENT_CPU(hwc->config_base);
1830 
1831 	if (venum_event || scorpion_event) {
1832 		bit = scorpion_event_to_bit(event, region, group);
1833 		clear_bit(bit, cpuc->used_mask);
1834 	}
1835 }
1836 
1837 static int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1838 {
1839 	armv7pmu_init(cpu_pmu);
1840 	cpu_pmu->name		= "armv7_scorpion";
1841 	cpu_pmu->map_event	= scorpion_map_event;
1842 	cpu_pmu->reset		= scorpion_pmu_reset;
1843 	cpu_pmu->enable		= scorpion_pmu_enable_event;
1844 	cpu_pmu->disable	= scorpion_pmu_disable_event;
1845 	cpu_pmu->get_event_idx	= scorpion_pmu_get_event_idx;
1846 	cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1847 	return armv7_probe_num_events(cpu_pmu);
1848 }
1849 
1850 static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1851 {
1852 	armv7pmu_init(cpu_pmu);
1853 	cpu_pmu->name		= "armv7_scorpion_mp";
1854 	cpu_pmu->map_event	= scorpion_map_event;
1855 	cpu_pmu->reset		= scorpion_pmu_reset;
1856 	cpu_pmu->enable		= scorpion_pmu_enable_event;
1857 	cpu_pmu->disable	= scorpion_pmu_disable_event;
1858 	cpu_pmu->get_event_idx	= scorpion_pmu_get_event_idx;
1859 	cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1860 	return armv7_probe_num_events(cpu_pmu);
1861 }
1862 
1863 static const struct of_device_id armv7_pmu_of_device_ids[] = {
1864 	{.compatible = "arm,cortex-a17-pmu",	.data = armv7_a17_pmu_init},
1865 	{.compatible = "arm,cortex-a15-pmu",	.data = armv7_a15_pmu_init},
1866 	{.compatible = "arm,cortex-a12-pmu",	.data = armv7_a12_pmu_init},
1867 	{.compatible = "arm,cortex-a9-pmu",	.data = armv7_a9_pmu_init},
1868 	{.compatible = "arm,cortex-a8-pmu",	.data = armv7_a8_pmu_init},
1869 	{.compatible = "arm,cortex-a7-pmu",	.data = armv7_a7_pmu_init},
1870 	{.compatible = "arm,cortex-a5-pmu",	.data = armv7_a5_pmu_init},
1871 	{.compatible = "qcom,krait-pmu",	.data = krait_pmu_init},
1872 	{.compatible = "qcom,scorpion-pmu",	.data = scorpion_pmu_init},
1873 	{.compatible = "qcom,scorpion-mp-pmu",	.data = scorpion_mp_pmu_init},
1874 	{},
1875 };
1876 
1877 static const struct pmu_probe_info armv7_pmu_probe_table[] = {
1878 	ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A8, armv7_a8_pmu_init),
1879 	ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A9, armv7_a9_pmu_init),
1880 	{ /* sentinel value */ }
1881 };
1882 
1883 
1884 static int armv7_pmu_device_probe(struct platform_device *pdev)
1885 {
1886 	return arm_pmu_device_probe(pdev, armv7_pmu_of_device_ids,
1887 				    armv7_pmu_probe_table);
1888 }
1889 
1890 static struct platform_driver armv7_pmu_driver = {
1891 	.driver		= {
1892 		.name	= "armv7-pmu",
1893 		.of_match_table = armv7_pmu_of_device_ids,
1894 	},
1895 	.probe		= armv7_pmu_device_probe,
1896 };
1897 
1898 static int __init register_armv7_pmu_driver(void)
1899 {
1900 	return platform_driver_register(&armv7_pmu_driver);
1901 }
1902 device_initcall(register_armv7_pmu_driver);
1903 #endif	/* CONFIG_CPU_V7 */
1904