xref: /openbmc/linux/drivers/perf/qcom_l2_pmu.c (revision 7bd42f12)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2015-2017 The Linux Foundation. All rights reserved.
3  */
4 #include <linux/acpi.h>
5 #include <linux/bitops.h>
6 #include <linux/bug.h>
7 #include <linux/cpuhotplug.h>
8 #include <linux/cpumask.h>
9 #include <linux/device.h>
10 #include <linux/errno.h>
11 #include <linux/interrupt.h>
12 #include <linux/irq.h>
13 #include <linux/kernel.h>
14 #include <linux/list.h>
15 #include <linux/percpu.h>
16 #include <linux/perf_event.h>
17 #include <linux/platform_device.h>
18 #include <linux/smp.h>
19 #include <linux/spinlock.h>
20 #include <linux/sysfs.h>
21 #include <linux/types.h>
22 
23 #include <asm/barrier.h>
24 #include <asm/local64.h>
25 #include <asm/sysreg.h>
26 #include <soc/qcom/kryo-l2-accessors.h>
27 
28 #define MAX_L2_CTRS             9
29 
30 #define L2PMCR_NUM_EV_SHIFT     11
31 #define L2PMCR_NUM_EV_MASK      0x1F
32 
33 #define L2PMCR                  0x400
34 #define L2PMCNTENCLR            0x403
35 #define L2PMCNTENSET            0x404
36 #define L2PMINTENCLR            0x405
37 #define L2PMINTENSET            0x406
38 #define L2PMOVSCLR              0x407
39 #define L2PMOVSSET              0x408
40 #define L2PMCCNTCR              0x409
41 #define L2PMCCNTR               0x40A
42 #define L2PMCCNTSR              0x40C
43 #define L2PMRESR                0x410
44 #define IA_L2PMXEVCNTCR_BASE    0x420
45 #define IA_L2PMXEVCNTR_BASE     0x421
46 #define IA_L2PMXEVFILTER_BASE   0x423
47 #define IA_L2PMXEVTYPER_BASE    0x424
48 
49 #define IA_L2_REG_OFFSET        0x10
50 
51 #define L2PMXEVFILTER_SUFILTER_ALL      0x000E0000
52 #define L2PMXEVFILTER_ORGFILTER_IDINDEP 0x00000004
53 #define L2PMXEVFILTER_ORGFILTER_ALL     0x00000003
54 
55 #define L2EVTYPER_REG_SHIFT     3
56 
57 #define L2PMRESR_GROUP_BITS     8
58 #define L2PMRESR_GROUP_MASK     GENMASK(7, 0)
59 
60 #define L2CYCLE_CTR_BIT         31
61 #define L2CYCLE_CTR_RAW_CODE    0xFE
62 
63 #define L2PMCR_RESET_ALL        0x6
64 #define L2PMCR_COUNTERS_ENABLE  0x1
65 #define L2PMCR_COUNTERS_DISABLE 0x0
66 
67 #define L2PMRESR_EN             BIT_ULL(63)
68 
69 #define L2_EVT_MASK             0x00000FFF
70 #define L2_EVT_CODE_MASK        0x00000FF0
71 #define L2_EVT_GRP_MASK         0x0000000F
72 #define L2_EVT_CODE_SHIFT       4
73 #define L2_EVT_GRP_SHIFT        0
74 
75 #define L2_EVT_CODE(event)   (((event) & L2_EVT_CODE_MASK) >> L2_EVT_CODE_SHIFT)
76 #define L2_EVT_GROUP(event)  (((event) & L2_EVT_GRP_MASK) >> L2_EVT_GRP_SHIFT)
77 
78 #define L2_EVT_GROUP_MAX        7
79 
80 #define L2_COUNTER_RELOAD       BIT_ULL(31)
81 #define L2_CYCLE_COUNTER_RELOAD BIT_ULL(63)
82 
83 
84 #define reg_idx(reg, i)         (((i) * IA_L2_REG_OFFSET) + reg##_BASE)
85 
86 /*
87  * Events
88  */
89 #define L2_EVENT_CYCLES                    0xfe
90 #define L2_EVENT_DCACHE_OPS                0x400
91 #define L2_EVENT_ICACHE_OPS                0x401
92 #define L2_EVENT_TLBI                      0x402
93 #define L2_EVENT_BARRIERS                  0x403
94 #define L2_EVENT_TOTAL_READS               0x405
95 #define L2_EVENT_TOTAL_WRITES              0x406
96 #define L2_EVENT_TOTAL_REQUESTS            0x407
97 #define L2_EVENT_LDREX                     0x420
98 #define L2_EVENT_STREX                     0x421
99 #define L2_EVENT_CLREX                     0x422
100 
101 
102 
103 struct cluster_pmu;
104 
105 /*
106  * Aggregate PMU. Implements the core pmu functions and manages
107  * the hardware PMUs.
108  */
109 struct l2cache_pmu {
110 	struct hlist_node node;
111 	u32 num_pmus;
112 	struct pmu pmu;
113 	int num_counters;
114 	cpumask_t cpumask;
115 	struct platform_device *pdev;
116 	struct cluster_pmu * __percpu *pmu_cluster;
117 	struct list_head clusters;
118 };
119 
120 /*
121  * The cache is made up of one or more clusters, each cluster has its own PMU.
122  * Each cluster is associated with one or more CPUs.
123  * This structure represents one of the hardware PMUs.
124  *
125  * Events can be envisioned as a 2-dimensional array. Each column represents
126  * a group of events. There are 8 groups. Only one entry from each
127  * group can be in use at a time.
128  *
129  * Events are specified as 0xCCG, where CC is 2 hex digits specifying
130  * the code (array row) and G specifies the group (column).
131  *
132  * In addition there is a cycle counter event specified by L2CYCLE_CTR_RAW_CODE
133  * which is outside the above scheme.
134  */
135 struct cluster_pmu {
136 	struct list_head next;
137 	struct perf_event *events[MAX_L2_CTRS];
138 	struct l2cache_pmu *l2cache_pmu;
139 	DECLARE_BITMAP(used_counters, MAX_L2_CTRS);
140 	DECLARE_BITMAP(used_groups, L2_EVT_GROUP_MAX + 1);
141 	int irq;
142 	int cluster_id;
143 	/* The CPU that is used for collecting events on this cluster */
144 	int on_cpu;
145 	/* All the CPUs associated with this cluster */
146 	cpumask_t cluster_cpus;
147 	spinlock_t pmu_lock;
148 };
149 
150 #define to_l2cache_pmu(p) (container_of(p, struct l2cache_pmu, pmu))
151 
152 static u32 l2_cycle_ctr_idx;
153 static u32 l2_counter_present_mask;
154 
idx_to_reg_bit(u32 idx)155 static inline u32 idx_to_reg_bit(u32 idx)
156 {
157 	if (idx == l2_cycle_ctr_idx)
158 		return BIT(L2CYCLE_CTR_BIT);
159 
160 	return BIT(idx);
161 }
162 
get_cluster_pmu(struct l2cache_pmu * l2cache_pmu,int cpu)163 static inline struct cluster_pmu *get_cluster_pmu(
164 	struct l2cache_pmu *l2cache_pmu, int cpu)
165 {
166 	return *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu);
167 }
168 
cluster_pmu_reset(void)169 static void cluster_pmu_reset(void)
170 {
171 	/* Reset all counters */
172 	kryo_l2_set_indirect_reg(L2PMCR, L2PMCR_RESET_ALL);
173 	kryo_l2_set_indirect_reg(L2PMCNTENCLR, l2_counter_present_mask);
174 	kryo_l2_set_indirect_reg(L2PMINTENCLR, l2_counter_present_mask);
175 	kryo_l2_set_indirect_reg(L2PMOVSCLR, l2_counter_present_mask);
176 }
177 
cluster_pmu_enable(void)178 static inline void cluster_pmu_enable(void)
179 {
180 	kryo_l2_set_indirect_reg(L2PMCR, L2PMCR_COUNTERS_ENABLE);
181 }
182 
cluster_pmu_disable(void)183 static inline void cluster_pmu_disable(void)
184 {
185 	kryo_l2_set_indirect_reg(L2PMCR, L2PMCR_COUNTERS_DISABLE);
186 }
187 
cluster_pmu_counter_set_value(u32 idx,u64 value)188 static inline void cluster_pmu_counter_set_value(u32 idx, u64 value)
189 {
190 	if (idx == l2_cycle_ctr_idx)
191 		kryo_l2_set_indirect_reg(L2PMCCNTR, value);
192 	else
193 		kryo_l2_set_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx), value);
194 }
195 
cluster_pmu_counter_get_value(u32 idx)196 static inline u64 cluster_pmu_counter_get_value(u32 idx)
197 {
198 	u64 value;
199 
200 	if (idx == l2_cycle_ctr_idx)
201 		value = kryo_l2_get_indirect_reg(L2PMCCNTR);
202 	else
203 		value = kryo_l2_get_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx));
204 
205 	return value;
206 }
207 
cluster_pmu_counter_enable(u32 idx)208 static inline void cluster_pmu_counter_enable(u32 idx)
209 {
210 	kryo_l2_set_indirect_reg(L2PMCNTENSET, idx_to_reg_bit(idx));
211 }
212 
cluster_pmu_counter_disable(u32 idx)213 static inline void cluster_pmu_counter_disable(u32 idx)
214 {
215 	kryo_l2_set_indirect_reg(L2PMCNTENCLR, idx_to_reg_bit(idx));
216 }
217 
cluster_pmu_counter_enable_interrupt(u32 idx)218 static inline void cluster_pmu_counter_enable_interrupt(u32 idx)
219 {
220 	kryo_l2_set_indirect_reg(L2PMINTENSET, idx_to_reg_bit(idx));
221 }
222 
cluster_pmu_counter_disable_interrupt(u32 idx)223 static inline void cluster_pmu_counter_disable_interrupt(u32 idx)
224 {
225 	kryo_l2_set_indirect_reg(L2PMINTENCLR, idx_to_reg_bit(idx));
226 }
227 
cluster_pmu_set_evccntcr(u32 val)228 static inline void cluster_pmu_set_evccntcr(u32 val)
229 {
230 	kryo_l2_set_indirect_reg(L2PMCCNTCR, val);
231 }
232 
cluster_pmu_set_evcntcr(u32 ctr,u32 val)233 static inline void cluster_pmu_set_evcntcr(u32 ctr, u32 val)
234 {
235 	kryo_l2_set_indirect_reg(reg_idx(IA_L2PMXEVCNTCR, ctr), val);
236 }
237 
cluster_pmu_set_evtyper(u32 ctr,u32 val)238 static inline void cluster_pmu_set_evtyper(u32 ctr, u32 val)
239 {
240 	kryo_l2_set_indirect_reg(reg_idx(IA_L2PMXEVTYPER, ctr), val);
241 }
242 
cluster_pmu_set_resr(struct cluster_pmu * cluster,u32 event_group,u32 event_cc)243 static void cluster_pmu_set_resr(struct cluster_pmu *cluster,
244 			       u32 event_group, u32 event_cc)
245 {
246 	u64 field;
247 	u64 resr_val;
248 	u32 shift;
249 	unsigned long flags;
250 
251 	shift = L2PMRESR_GROUP_BITS * event_group;
252 	field = ((u64)(event_cc & L2PMRESR_GROUP_MASK) << shift);
253 
254 	spin_lock_irqsave(&cluster->pmu_lock, flags);
255 
256 	resr_val = kryo_l2_get_indirect_reg(L2PMRESR);
257 	resr_val &= ~(L2PMRESR_GROUP_MASK << shift);
258 	resr_val |= field;
259 	resr_val |= L2PMRESR_EN;
260 	kryo_l2_set_indirect_reg(L2PMRESR, resr_val);
261 
262 	spin_unlock_irqrestore(&cluster->pmu_lock, flags);
263 }
264 
265 /*
266  * Hardware allows filtering of events based on the originating
267  * CPU. Turn this off by setting filter bits to allow events from
268  * all CPUS, subunits and ID independent events in this cluster.
269  */
cluster_pmu_set_evfilter_sys_mode(u32 ctr)270 static inline void cluster_pmu_set_evfilter_sys_mode(u32 ctr)
271 {
272 	u32 val =  L2PMXEVFILTER_SUFILTER_ALL |
273 		   L2PMXEVFILTER_ORGFILTER_IDINDEP |
274 		   L2PMXEVFILTER_ORGFILTER_ALL;
275 
276 	kryo_l2_set_indirect_reg(reg_idx(IA_L2PMXEVFILTER, ctr), val);
277 }
278 
cluster_pmu_getreset_ovsr(void)279 static inline u32 cluster_pmu_getreset_ovsr(void)
280 {
281 	u32 result = kryo_l2_get_indirect_reg(L2PMOVSSET);
282 
283 	kryo_l2_set_indirect_reg(L2PMOVSCLR, result);
284 	return result;
285 }
286 
cluster_pmu_has_overflowed(u32 ovsr)287 static inline bool cluster_pmu_has_overflowed(u32 ovsr)
288 {
289 	return !!(ovsr & l2_counter_present_mask);
290 }
291 
cluster_pmu_counter_has_overflowed(u32 ovsr,u32 idx)292 static inline bool cluster_pmu_counter_has_overflowed(u32 ovsr, u32 idx)
293 {
294 	return !!(ovsr & idx_to_reg_bit(idx));
295 }
296 
l2_cache_event_update(struct perf_event * event)297 static void l2_cache_event_update(struct perf_event *event)
298 {
299 	struct hw_perf_event *hwc = &event->hw;
300 	u64 delta, prev, now;
301 	u32 idx = hwc->idx;
302 
303 	do {
304 		prev = local64_read(&hwc->prev_count);
305 		now = cluster_pmu_counter_get_value(idx);
306 	} while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
307 
308 	/*
309 	 * The cycle counter is 64-bit, but all other counters are
310 	 * 32-bit, and we must handle 32-bit overflow explicitly.
311 	 */
312 	delta = now - prev;
313 	if (idx != l2_cycle_ctr_idx)
314 		delta &= 0xffffffff;
315 
316 	local64_add(delta, &event->count);
317 }
318 
l2_cache_cluster_set_period(struct cluster_pmu * cluster,struct hw_perf_event * hwc)319 static void l2_cache_cluster_set_period(struct cluster_pmu *cluster,
320 				       struct hw_perf_event *hwc)
321 {
322 	u32 idx = hwc->idx;
323 	u64 new;
324 
325 	/*
326 	 * We limit the max period to half the max counter value so
327 	 * that even in the case of extreme interrupt latency the
328 	 * counter will (hopefully) not wrap past its initial value.
329 	 */
330 	if (idx == l2_cycle_ctr_idx)
331 		new = L2_CYCLE_COUNTER_RELOAD;
332 	else
333 		new = L2_COUNTER_RELOAD;
334 
335 	local64_set(&hwc->prev_count, new);
336 	cluster_pmu_counter_set_value(idx, new);
337 }
338 
l2_cache_get_event_idx(struct cluster_pmu * cluster,struct perf_event * event)339 static int l2_cache_get_event_idx(struct cluster_pmu *cluster,
340 				   struct perf_event *event)
341 {
342 	struct hw_perf_event *hwc = &event->hw;
343 	int idx;
344 	int num_ctrs = cluster->l2cache_pmu->num_counters - 1;
345 	unsigned int group;
346 
347 	if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) {
348 		if (test_and_set_bit(l2_cycle_ctr_idx, cluster->used_counters))
349 			return -EAGAIN;
350 
351 		return l2_cycle_ctr_idx;
352 	}
353 
354 	idx = find_first_zero_bit(cluster->used_counters, num_ctrs);
355 	if (idx == num_ctrs)
356 		/* The counters are all in use. */
357 		return -EAGAIN;
358 
359 	/*
360 	 * Check for column exclusion: event column already in use by another
361 	 * event. This is for events which are not in the same group.
362 	 * Conflicting events in the same group are detected in event_init.
363 	 */
364 	group = L2_EVT_GROUP(hwc->config_base);
365 	if (test_bit(group, cluster->used_groups))
366 		return -EAGAIN;
367 
368 	set_bit(idx, cluster->used_counters);
369 	set_bit(group, cluster->used_groups);
370 
371 	return idx;
372 }
373 
l2_cache_clear_event_idx(struct cluster_pmu * cluster,struct perf_event * event)374 static void l2_cache_clear_event_idx(struct cluster_pmu *cluster,
375 				      struct perf_event *event)
376 {
377 	struct hw_perf_event *hwc = &event->hw;
378 	int idx = hwc->idx;
379 
380 	clear_bit(idx, cluster->used_counters);
381 	if (hwc->config_base != L2CYCLE_CTR_RAW_CODE)
382 		clear_bit(L2_EVT_GROUP(hwc->config_base), cluster->used_groups);
383 }
384 
l2_cache_handle_irq(int irq_num,void * data)385 static irqreturn_t l2_cache_handle_irq(int irq_num, void *data)
386 {
387 	struct cluster_pmu *cluster = data;
388 	int num_counters = cluster->l2cache_pmu->num_counters;
389 	u32 ovsr;
390 	int idx;
391 
392 	ovsr = cluster_pmu_getreset_ovsr();
393 	if (!cluster_pmu_has_overflowed(ovsr))
394 		return IRQ_NONE;
395 
396 	for_each_set_bit(idx, cluster->used_counters, num_counters) {
397 		struct perf_event *event = cluster->events[idx];
398 		struct hw_perf_event *hwc;
399 
400 		if (WARN_ON_ONCE(!event))
401 			continue;
402 
403 		if (!cluster_pmu_counter_has_overflowed(ovsr, idx))
404 			continue;
405 
406 		l2_cache_event_update(event);
407 		hwc = &event->hw;
408 
409 		l2_cache_cluster_set_period(cluster, hwc);
410 	}
411 
412 	return IRQ_HANDLED;
413 }
414 
415 /*
416  * Implementation of abstract pmu functionality required by
417  * the core perf events code.
418  */
419 
l2_cache_pmu_enable(struct pmu * pmu)420 static void l2_cache_pmu_enable(struct pmu *pmu)
421 {
422 	/*
423 	 * Although there is only one PMU (per socket) controlling multiple
424 	 * physical PMUs (per cluster), because we do not support per-task mode
425 	 * each event is associated with a CPU. Each event has pmu_enable
426 	 * called on its CPU, so here it is only necessary to enable the
427 	 * counters for the current CPU.
428 	 */
429 
430 	cluster_pmu_enable();
431 }
432 
l2_cache_pmu_disable(struct pmu * pmu)433 static void l2_cache_pmu_disable(struct pmu *pmu)
434 {
435 	cluster_pmu_disable();
436 }
437 
l2_cache_event_init(struct perf_event * event)438 static int l2_cache_event_init(struct perf_event *event)
439 {
440 	struct hw_perf_event *hwc = &event->hw;
441 	struct cluster_pmu *cluster;
442 	struct perf_event *sibling;
443 	struct l2cache_pmu *l2cache_pmu;
444 
445 	if (event->attr.type != event->pmu->type)
446 		return -ENOENT;
447 
448 	l2cache_pmu = to_l2cache_pmu(event->pmu);
449 
450 	if (hwc->sample_period) {
451 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
452 				    "Sampling not supported\n");
453 		return -EOPNOTSUPP;
454 	}
455 
456 	if (event->cpu < 0) {
457 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
458 				    "Per-task mode not supported\n");
459 		return -EOPNOTSUPP;
460 	}
461 
462 	if (((L2_EVT_GROUP(event->attr.config) > L2_EVT_GROUP_MAX) ||
463 	     ((event->attr.config & ~L2_EVT_MASK) != 0)) &&
464 	    (event->attr.config != L2CYCLE_CTR_RAW_CODE)) {
465 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
466 				    "Invalid config %llx\n",
467 				    event->attr.config);
468 		return -EINVAL;
469 	}
470 
471 	/* Don't allow groups with mixed PMUs, except for s/w events */
472 	if (event->group_leader->pmu != event->pmu &&
473 	    !is_software_event(event->group_leader)) {
474 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
475 			 "Can't create mixed PMU group\n");
476 		return -EINVAL;
477 	}
478 
479 	for_each_sibling_event(sibling, event->group_leader) {
480 		if (sibling->pmu != event->pmu &&
481 		    !is_software_event(sibling)) {
482 			dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
483 				 "Can't create mixed PMU group\n");
484 			return -EINVAL;
485 		}
486 	}
487 
488 	cluster = get_cluster_pmu(l2cache_pmu, event->cpu);
489 	if (!cluster) {
490 		/* CPU has not been initialised */
491 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
492 			"CPU%d not associated with L2 cluster\n", event->cpu);
493 		return -EINVAL;
494 	}
495 
496 	/* Ensure all events in a group are on the same cpu */
497 	if ((event->group_leader != event) &&
498 	    (cluster->on_cpu != event->group_leader->cpu)) {
499 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
500 			 "Can't create group on CPUs %d and %d",
501 			 event->cpu, event->group_leader->cpu);
502 		return -EINVAL;
503 	}
504 
505 	if ((event != event->group_leader) &&
506 	    !is_software_event(event->group_leader) &&
507 	    (L2_EVT_GROUP(event->group_leader->attr.config) ==
508 	     L2_EVT_GROUP(event->attr.config))) {
509 		dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
510 			 "Column exclusion: conflicting events %llx %llx\n",
511 		       event->group_leader->attr.config,
512 		       event->attr.config);
513 		return -EINVAL;
514 	}
515 
516 	for_each_sibling_event(sibling, event->group_leader) {
517 		if ((sibling != event) &&
518 		    !is_software_event(sibling) &&
519 		    (L2_EVT_GROUP(sibling->attr.config) ==
520 		     L2_EVT_GROUP(event->attr.config))) {
521 			dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
522 			     "Column exclusion: conflicting events %llx %llx\n",
523 					    sibling->attr.config,
524 					    event->attr.config);
525 			return -EINVAL;
526 		}
527 	}
528 
529 	hwc->idx = -1;
530 	hwc->config_base = event->attr.config;
531 
532 	/*
533 	 * Ensure all events are on the same cpu so all events are in the
534 	 * same cpu context, to avoid races on pmu_enable etc.
535 	 */
536 	event->cpu = cluster->on_cpu;
537 
538 	return 0;
539 }
540 
l2_cache_event_start(struct perf_event * event,int flags)541 static void l2_cache_event_start(struct perf_event *event, int flags)
542 {
543 	struct cluster_pmu *cluster;
544 	struct hw_perf_event *hwc = &event->hw;
545 	int idx = hwc->idx;
546 	u32 config;
547 	u32 event_cc, event_group;
548 
549 	hwc->state = 0;
550 
551 	cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
552 
553 	l2_cache_cluster_set_period(cluster, hwc);
554 
555 	if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) {
556 		cluster_pmu_set_evccntcr(0);
557 	} else {
558 		config = hwc->config_base;
559 		event_cc    = L2_EVT_CODE(config);
560 		event_group = L2_EVT_GROUP(config);
561 
562 		cluster_pmu_set_evcntcr(idx, 0);
563 		cluster_pmu_set_evtyper(idx, event_group);
564 		cluster_pmu_set_resr(cluster, event_group, event_cc);
565 		cluster_pmu_set_evfilter_sys_mode(idx);
566 	}
567 
568 	cluster_pmu_counter_enable_interrupt(idx);
569 	cluster_pmu_counter_enable(idx);
570 }
571 
l2_cache_event_stop(struct perf_event * event,int flags)572 static void l2_cache_event_stop(struct perf_event *event, int flags)
573 {
574 	struct hw_perf_event *hwc = &event->hw;
575 	int idx = hwc->idx;
576 
577 	if (hwc->state & PERF_HES_STOPPED)
578 		return;
579 
580 	cluster_pmu_counter_disable_interrupt(idx);
581 	cluster_pmu_counter_disable(idx);
582 
583 	if (flags & PERF_EF_UPDATE)
584 		l2_cache_event_update(event);
585 	hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
586 }
587 
l2_cache_event_add(struct perf_event * event,int flags)588 static int l2_cache_event_add(struct perf_event *event, int flags)
589 {
590 	struct hw_perf_event *hwc = &event->hw;
591 	int idx;
592 	int err = 0;
593 	struct cluster_pmu *cluster;
594 
595 	cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
596 
597 	idx = l2_cache_get_event_idx(cluster, event);
598 	if (idx < 0)
599 		return idx;
600 
601 	hwc->idx = idx;
602 	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
603 	cluster->events[idx] = event;
604 	local64_set(&hwc->prev_count, 0);
605 
606 	if (flags & PERF_EF_START)
607 		l2_cache_event_start(event, flags);
608 
609 	/* Propagate changes to the userspace mapping. */
610 	perf_event_update_userpage(event);
611 
612 	return err;
613 }
614 
l2_cache_event_del(struct perf_event * event,int flags)615 static void l2_cache_event_del(struct perf_event *event, int flags)
616 {
617 	struct hw_perf_event *hwc = &event->hw;
618 	struct cluster_pmu *cluster;
619 	int idx = hwc->idx;
620 
621 	cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
622 
623 	l2_cache_event_stop(event, flags | PERF_EF_UPDATE);
624 	cluster->events[idx] = NULL;
625 	l2_cache_clear_event_idx(cluster, event);
626 
627 	perf_event_update_userpage(event);
628 }
629 
l2_cache_event_read(struct perf_event * event)630 static void l2_cache_event_read(struct perf_event *event)
631 {
632 	l2_cache_event_update(event);
633 }
634 
l2_cache_pmu_cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)635 static ssize_t l2_cache_pmu_cpumask_show(struct device *dev,
636 					 struct device_attribute *attr,
637 					 char *buf)
638 {
639 	struct l2cache_pmu *l2cache_pmu = to_l2cache_pmu(dev_get_drvdata(dev));
640 
641 	return cpumap_print_to_pagebuf(true, buf, &l2cache_pmu->cpumask);
642 }
643 
644 static struct device_attribute l2_cache_pmu_cpumask_attr =
645 		__ATTR(cpumask, S_IRUGO, l2_cache_pmu_cpumask_show, NULL);
646 
647 static struct attribute *l2_cache_pmu_cpumask_attrs[] = {
648 	&l2_cache_pmu_cpumask_attr.attr,
649 	NULL,
650 };
651 
652 static const struct attribute_group l2_cache_pmu_cpumask_group = {
653 	.attrs = l2_cache_pmu_cpumask_attrs,
654 };
655 
656 /* CCG format for perf RAW codes. */
657 PMU_FORMAT_ATTR(l2_code,   "config:4-11");
658 PMU_FORMAT_ATTR(l2_group,  "config:0-3");
659 PMU_FORMAT_ATTR(event,     "config:0-11");
660 
661 static struct attribute *l2_cache_pmu_formats[] = {
662 	&format_attr_l2_code.attr,
663 	&format_attr_l2_group.attr,
664 	&format_attr_event.attr,
665 	NULL,
666 };
667 
668 static const struct attribute_group l2_cache_pmu_format_group = {
669 	.name = "format",
670 	.attrs = l2_cache_pmu_formats,
671 };
672 
l2cache_pmu_event_show(struct device * dev,struct device_attribute * attr,char * page)673 static ssize_t l2cache_pmu_event_show(struct device *dev,
674 				      struct device_attribute *attr, char *page)
675 {
676 	struct perf_pmu_events_attr *pmu_attr;
677 
678 	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
679 	return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
680 }
681 
682 #define L2CACHE_EVENT_ATTR(_name, _id)			    \
683 	PMU_EVENT_ATTR_ID(_name, l2cache_pmu_event_show, _id)
684 
685 static struct attribute *l2_cache_pmu_events[] = {
686 	L2CACHE_EVENT_ATTR(cycles, L2_EVENT_CYCLES),
687 	L2CACHE_EVENT_ATTR(dcache-ops, L2_EVENT_DCACHE_OPS),
688 	L2CACHE_EVENT_ATTR(icache-ops, L2_EVENT_ICACHE_OPS),
689 	L2CACHE_EVENT_ATTR(tlbi, L2_EVENT_TLBI),
690 	L2CACHE_EVENT_ATTR(barriers, L2_EVENT_BARRIERS),
691 	L2CACHE_EVENT_ATTR(total-reads, L2_EVENT_TOTAL_READS),
692 	L2CACHE_EVENT_ATTR(total-writes, L2_EVENT_TOTAL_WRITES),
693 	L2CACHE_EVENT_ATTR(total-requests, L2_EVENT_TOTAL_REQUESTS),
694 	L2CACHE_EVENT_ATTR(ldrex, L2_EVENT_LDREX),
695 	L2CACHE_EVENT_ATTR(strex, L2_EVENT_STREX),
696 	L2CACHE_EVENT_ATTR(clrex, L2_EVENT_CLREX),
697 	NULL
698 };
699 
700 static const struct attribute_group l2_cache_pmu_events_group = {
701 	.name = "events",
702 	.attrs = l2_cache_pmu_events,
703 };
704 
705 static const struct attribute_group *l2_cache_pmu_attr_grps[] = {
706 	&l2_cache_pmu_format_group,
707 	&l2_cache_pmu_cpumask_group,
708 	&l2_cache_pmu_events_group,
709 	NULL,
710 };
711 
712 /*
713  * Generic device handlers
714  */
715 
716 static const struct acpi_device_id l2_cache_pmu_acpi_match[] = {
717 	{ "QCOM8130", },
718 	{ }
719 };
720 
get_num_counters(void)721 static int get_num_counters(void)
722 {
723 	int val;
724 
725 	val = kryo_l2_get_indirect_reg(L2PMCR);
726 
727 	/*
728 	 * Read number of counters from L2PMCR and add 1
729 	 * for the cycle counter.
730 	 */
731 	return ((val >> L2PMCR_NUM_EV_SHIFT) & L2PMCR_NUM_EV_MASK) + 1;
732 }
733 
l2_cache_associate_cpu_with_cluster(struct l2cache_pmu * l2cache_pmu,int cpu)734 static struct cluster_pmu *l2_cache_associate_cpu_with_cluster(
735 	struct l2cache_pmu *l2cache_pmu, int cpu)
736 {
737 	u64 mpidr;
738 	int cpu_cluster_id;
739 	struct cluster_pmu *cluster;
740 
741 	/*
742 	 * This assumes that the cluster_id is in MPIDR[aff1] for
743 	 * single-threaded cores, and MPIDR[aff2] for multi-threaded
744 	 * cores. This logic will have to be updated if this changes.
745 	 */
746 	mpidr = read_cpuid_mpidr();
747 	if (mpidr & MPIDR_MT_BITMASK)
748 		cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
749 	else
750 		cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
751 
752 	list_for_each_entry(cluster, &l2cache_pmu->clusters, next) {
753 		if (cluster->cluster_id != cpu_cluster_id)
754 			continue;
755 
756 		dev_info(&l2cache_pmu->pdev->dev,
757 			 "CPU%d associated with cluster %d\n", cpu,
758 			 cluster->cluster_id);
759 		cpumask_set_cpu(cpu, &cluster->cluster_cpus);
760 		*per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu) = cluster;
761 		return cluster;
762 	}
763 
764 	return NULL;
765 }
766 
l2cache_pmu_online_cpu(unsigned int cpu,struct hlist_node * node)767 static int l2cache_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
768 {
769 	struct cluster_pmu *cluster;
770 	struct l2cache_pmu *l2cache_pmu;
771 
772 	l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node);
773 	cluster = get_cluster_pmu(l2cache_pmu, cpu);
774 	if (!cluster) {
775 		/* First time this CPU has come online */
776 		cluster = l2_cache_associate_cpu_with_cluster(l2cache_pmu, cpu);
777 		if (!cluster) {
778 			/* Only if broken firmware doesn't list every cluster */
779 			WARN_ONCE(1, "No L2 cache cluster for CPU%d\n", cpu);
780 			return 0;
781 		}
782 	}
783 
784 	/* If another CPU is managing this cluster, we're done */
785 	if (cluster->on_cpu != -1)
786 		return 0;
787 
788 	/*
789 	 * All CPUs on this cluster were down, use this one.
790 	 * Reset to put it into sane state.
791 	 */
792 	cluster->on_cpu = cpu;
793 	cpumask_set_cpu(cpu, &l2cache_pmu->cpumask);
794 	cluster_pmu_reset();
795 
796 	WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(cpu)));
797 	enable_irq(cluster->irq);
798 
799 	return 0;
800 }
801 
l2cache_pmu_offline_cpu(unsigned int cpu,struct hlist_node * node)802 static int l2cache_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
803 {
804 	struct cluster_pmu *cluster;
805 	struct l2cache_pmu *l2cache_pmu;
806 	cpumask_t cluster_online_cpus;
807 	unsigned int target;
808 
809 	l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node);
810 	cluster = get_cluster_pmu(l2cache_pmu, cpu);
811 	if (!cluster)
812 		return 0;
813 
814 	/* If this CPU is not managing the cluster, we're done */
815 	if (cluster->on_cpu != cpu)
816 		return 0;
817 
818 	/* Give up ownership of cluster */
819 	cpumask_clear_cpu(cpu, &l2cache_pmu->cpumask);
820 	cluster->on_cpu = -1;
821 
822 	/* Any other CPU for this cluster which is still online */
823 	cpumask_and(&cluster_online_cpus, &cluster->cluster_cpus,
824 		    cpu_online_mask);
825 	target = cpumask_any_but(&cluster_online_cpus, cpu);
826 	if (target >= nr_cpu_ids) {
827 		disable_irq(cluster->irq);
828 		return 0;
829 	}
830 
831 	perf_pmu_migrate_context(&l2cache_pmu->pmu, cpu, target);
832 	cluster->on_cpu = target;
833 	cpumask_set_cpu(target, &l2cache_pmu->cpumask);
834 	WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(target)));
835 
836 	return 0;
837 }
838 
l2_cache_pmu_probe_cluster(struct device * dev,void * data)839 static int l2_cache_pmu_probe_cluster(struct device *dev, void *data)
840 {
841 	struct platform_device *pdev = to_platform_device(dev->parent);
842 	struct platform_device *sdev = to_platform_device(dev);
843 	struct l2cache_pmu *l2cache_pmu = data;
844 	struct cluster_pmu *cluster;
845 	u64 fw_cluster_id;
846 	int err;
847 	int irq;
848 
849 	err = acpi_dev_uid_to_integer(ACPI_COMPANION(dev), &fw_cluster_id);
850 	if (err) {
851 		dev_err(&pdev->dev, "unable to read ACPI uid\n");
852 		return err;
853 	}
854 
855 	cluster = devm_kzalloc(&pdev->dev, sizeof(*cluster), GFP_KERNEL);
856 	if (!cluster)
857 		return -ENOMEM;
858 
859 	INIT_LIST_HEAD(&cluster->next);
860 	cluster->cluster_id = fw_cluster_id;
861 
862 	irq = platform_get_irq(sdev, 0);
863 	if (irq < 0)
864 		return irq;
865 	cluster->irq = irq;
866 
867 	cluster->l2cache_pmu = l2cache_pmu;
868 	cluster->on_cpu = -1;
869 
870 	err = devm_request_irq(&pdev->dev, irq, l2_cache_handle_irq,
871 			       IRQF_NOBALANCING | IRQF_NO_THREAD |
872 			       IRQF_NO_AUTOEN,
873 			       "l2-cache-pmu", cluster);
874 	if (err) {
875 		dev_err(&pdev->dev,
876 			"Unable to request IRQ%d for L2 PMU counters\n", irq);
877 		return err;
878 	}
879 
880 	dev_info(&pdev->dev,
881 		 "Registered L2 cache PMU cluster %lld\n", fw_cluster_id);
882 
883 	spin_lock_init(&cluster->pmu_lock);
884 
885 	list_add(&cluster->next, &l2cache_pmu->clusters);
886 	l2cache_pmu->num_pmus++;
887 
888 	return 0;
889 }
890 
l2_cache_pmu_probe(struct platform_device * pdev)891 static int l2_cache_pmu_probe(struct platform_device *pdev)
892 {
893 	int err;
894 	struct l2cache_pmu *l2cache_pmu;
895 
896 	l2cache_pmu =
897 		devm_kzalloc(&pdev->dev, sizeof(*l2cache_pmu), GFP_KERNEL);
898 	if (!l2cache_pmu)
899 		return -ENOMEM;
900 
901 	INIT_LIST_HEAD(&l2cache_pmu->clusters);
902 
903 	platform_set_drvdata(pdev, l2cache_pmu);
904 	l2cache_pmu->pmu = (struct pmu) {
905 		/* suffix is instance id for future use with multiple sockets */
906 		.name		= "l2cache_0",
907 		.task_ctx_nr    = perf_invalid_context,
908 		.pmu_enable	= l2_cache_pmu_enable,
909 		.pmu_disable	= l2_cache_pmu_disable,
910 		.event_init	= l2_cache_event_init,
911 		.add		= l2_cache_event_add,
912 		.del		= l2_cache_event_del,
913 		.start		= l2_cache_event_start,
914 		.stop		= l2_cache_event_stop,
915 		.read		= l2_cache_event_read,
916 		.attr_groups	= l2_cache_pmu_attr_grps,
917 		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
918 	};
919 
920 	l2cache_pmu->num_counters = get_num_counters();
921 	l2cache_pmu->pdev = pdev;
922 	l2cache_pmu->pmu_cluster = devm_alloc_percpu(&pdev->dev,
923 						     struct cluster_pmu *);
924 	if (!l2cache_pmu->pmu_cluster)
925 		return -ENOMEM;
926 
927 	l2_cycle_ctr_idx = l2cache_pmu->num_counters - 1;
928 	l2_counter_present_mask = GENMASK(l2cache_pmu->num_counters - 2, 0) |
929 		BIT(L2CYCLE_CTR_BIT);
930 
931 	cpumask_clear(&l2cache_pmu->cpumask);
932 
933 	/* Read cluster info and initialize each cluster */
934 	err = device_for_each_child(&pdev->dev, l2cache_pmu,
935 				    l2_cache_pmu_probe_cluster);
936 	if (err)
937 		return err;
938 
939 	if (l2cache_pmu->num_pmus == 0) {
940 		dev_err(&pdev->dev, "No hardware L2 cache PMUs found\n");
941 		return -ENODEV;
942 	}
943 
944 	err = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
945 				       &l2cache_pmu->node);
946 	if (err) {
947 		dev_err(&pdev->dev, "Error %d registering hotplug", err);
948 		return err;
949 	}
950 
951 	err = perf_pmu_register(&l2cache_pmu->pmu, l2cache_pmu->pmu.name, -1);
952 	if (err) {
953 		dev_err(&pdev->dev, "Error %d registering L2 cache PMU\n", err);
954 		goto out_unregister;
955 	}
956 
957 	dev_info(&pdev->dev, "Registered L2 cache PMU using %d HW PMUs\n",
958 		 l2cache_pmu->num_pmus);
959 
960 	return err;
961 
962 out_unregister:
963 	cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
964 				    &l2cache_pmu->node);
965 	return err;
966 }
967 
l2_cache_pmu_remove(struct platform_device * pdev)968 static int l2_cache_pmu_remove(struct platform_device *pdev)
969 {
970 	struct l2cache_pmu *l2cache_pmu =
971 		to_l2cache_pmu(platform_get_drvdata(pdev));
972 
973 	perf_pmu_unregister(&l2cache_pmu->pmu);
974 	cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
975 				    &l2cache_pmu->node);
976 	return 0;
977 }
978 
979 static struct platform_driver l2_cache_pmu_driver = {
980 	.driver = {
981 		.name = "qcom-l2cache-pmu",
982 		.acpi_match_table = ACPI_PTR(l2_cache_pmu_acpi_match),
983 		.suppress_bind_attrs = true,
984 	},
985 	.probe = l2_cache_pmu_probe,
986 	.remove = l2_cache_pmu_remove,
987 };
988 
register_l2_cache_pmu_driver(void)989 static int __init register_l2_cache_pmu_driver(void)
990 {
991 	int err;
992 
993 	err = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
994 				      "AP_PERF_ARM_QCOM_L2_ONLINE",
995 				      l2cache_pmu_online_cpu,
996 				      l2cache_pmu_offline_cpu);
997 	if (err)
998 		return err;
999 
1000 	return platform_driver_register(&l2_cache_pmu_driver);
1001 }
1002 device_initcall(register_l2_cache_pmu_driver);
1003