1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright 2017 NXP
4  * Copyright 2016 Freescale Semiconductor, Inc.
5  */
6 
7 #include <linux/bitfield.h>
8 #include <linux/init.h>
9 #include <linux/interrupt.h>
10 #include <linux/io.h>
11 #include <linux/module.h>
12 #include <linux/of.h>
13 #include <linux/of_address.h>
14 #include <linux/of_device.h>
15 #include <linux/of_irq.h>
16 #include <linux/perf_event.h>
17 #include <linux/slab.h>
18 
19 #define COUNTER_CNTL		0x0
20 #define COUNTER_READ		0x20
21 
22 #define COUNTER_DPCR1		0x30
23 
24 #define CNTL_OVER		0x1
25 #define CNTL_CLEAR		0x2
26 #define CNTL_EN			0x4
27 #define CNTL_EN_MASK		0xFFFFFFFB
28 #define CNTL_CLEAR_MASK		0xFFFFFFFD
29 #define CNTL_OVER_MASK		0xFFFFFFFE
30 
31 #define CNTL_CSV_SHIFT		24
32 #define CNTL_CSV_MASK		(0xFF << CNTL_CSV_SHIFT)
33 
34 #define EVENT_CYCLES_ID		0
35 #define EVENT_CYCLES_COUNTER	0
36 #define NUM_COUNTERS		4
37 
38 #define AXI_MASKING_REVERT	0xffff0000	/* AXI_MASKING(MSB 16bits) + AXI_ID(LSB 16bits) */
39 
40 #define to_ddr_pmu(p)		container_of(p, struct ddr_pmu, pmu)
41 
42 #define DDR_PERF_DEV_NAME	"imx8_ddr"
43 #define DDR_CPUHP_CB_NAME	DDR_PERF_DEV_NAME "_perf_pmu"
44 
45 static DEFINE_IDA(ddr_ida);
46 
47 /* DDR Perf hardware feature */
48 #define DDR_CAP_AXI_ID_FILTER			0x1     /* support AXI ID filter */
49 #define DDR_CAP_AXI_ID_FILTER_ENHANCED		0x3     /* support enhanced AXI ID filter */
50 
51 struct fsl_ddr_devtype_data {
52 	unsigned int quirks;    /* quirks needed for different DDR Perf core */
53 	const char *identifier;	/* system PMU identifier for userspace */
54 };
55 
56 static const struct fsl_ddr_devtype_data imx8_devtype_data;
57 
58 static const struct fsl_ddr_devtype_data imx8m_devtype_data = {
59 	.quirks = DDR_CAP_AXI_ID_FILTER,
60 };
61 
62 static const struct fsl_ddr_devtype_data imx8mq_devtype_data = {
63 	.quirks = DDR_CAP_AXI_ID_FILTER,
64 	.identifier = "i.MX8MQ",
65 };
66 
67 static const struct fsl_ddr_devtype_data imx8mm_devtype_data = {
68 	.quirks = DDR_CAP_AXI_ID_FILTER,
69 	.identifier = "i.MX8MM",
70 };
71 
72 static const struct fsl_ddr_devtype_data imx8mn_devtype_data = {
73 	.quirks = DDR_CAP_AXI_ID_FILTER,
74 	.identifier = "i.MX8MN",
75 };
76 
77 static const struct fsl_ddr_devtype_data imx8mp_devtype_data = {
78 	.quirks = DDR_CAP_AXI_ID_FILTER_ENHANCED,
79 	.identifier = "i.MX8MP",
80 };
81 
82 static const struct of_device_id imx_ddr_pmu_dt_ids[] = {
83 	{ .compatible = "fsl,imx8-ddr-pmu", .data = &imx8_devtype_data},
84 	{ .compatible = "fsl,imx8m-ddr-pmu", .data = &imx8m_devtype_data},
85 	{ .compatible = "fsl,imx8mq-ddr-pmu", .data = &imx8mq_devtype_data},
86 	{ .compatible = "fsl,imx8mm-ddr-pmu", .data = &imx8mm_devtype_data},
87 	{ .compatible = "fsl,imx8mn-ddr-pmu", .data = &imx8mn_devtype_data},
88 	{ .compatible = "fsl,imx8mp-ddr-pmu", .data = &imx8mp_devtype_data},
89 	{ /* sentinel */ }
90 };
91 MODULE_DEVICE_TABLE(of, imx_ddr_pmu_dt_ids);
92 
93 struct ddr_pmu {
94 	struct pmu pmu;
95 	void __iomem *base;
96 	unsigned int cpu;
97 	struct	hlist_node node;
98 	struct	device *dev;
99 	struct perf_event *events[NUM_COUNTERS];
100 	int active_events;
101 	enum cpuhp_state cpuhp_state;
102 	const struct fsl_ddr_devtype_data *devtype_data;
103 	int irq;
104 	int id;
105 };
106 
107 static ssize_t ddr_perf_identifier_show(struct device *dev,
108 					struct device_attribute *attr,
109 					char *page)
110 {
111 	struct ddr_pmu *pmu = dev_get_drvdata(dev);
112 
113 	return sprintf(page, "%s\n", pmu->devtype_data->identifier);
114 }
115 
116 static umode_t ddr_perf_identifier_attr_visible(struct kobject *kobj,
117 						struct attribute *attr,
118 						int n)
119 {
120 	struct device *dev = kobj_to_dev(kobj);
121 	struct ddr_pmu *pmu = dev_get_drvdata(dev);
122 
123 	if (!pmu->devtype_data->identifier)
124 		return 0;
125 	return attr->mode;
126 };
127 
128 static struct device_attribute ddr_perf_identifier_attr =
129 	__ATTR(identifier, 0444, ddr_perf_identifier_show, NULL);
130 
131 static struct attribute *ddr_perf_identifier_attrs[] = {
132 	&ddr_perf_identifier_attr.attr,
133 	NULL,
134 };
135 
136 static struct attribute_group ddr_perf_identifier_attr_group = {
137 	.attrs = ddr_perf_identifier_attrs,
138 	.is_visible = ddr_perf_identifier_attr_visible,
139 };
140 
141 enum ddr_perf_filter_capabilities {
142 	PERF_CAP_AXI_ID_FILTER = 0,
143 	PERF_CAP_AXI_ID_FILTER_ENHANCED,
144 	PERF_CAP_AXI_ID_FEAT_MAX,
145 };
146 
147 static u32 ddr_perf_filter_cap_get(struct ddr_pmu *pmu, int cap)
148 {
149 	u32 quirks = pmu->devtype_data->quirks;
150 
151 	switch (cap) {
152 	case PERF_CAP_AXI_ID_FILTER:
153 		return !!(quirks & DDR_CAP_AXI_ID_FILTER);
154 	case PERF_CAP_AXI_ID_FILTER_ENHANCED:
155 		quirks &= DDR_CAP_AXI_ID_FILTER_ENHANCED;
156 		return quirks == DDR_CAP_AXI_ID_FILTER_ENHANCED;
157 	default:
158 		WARN(1, "unknown filter cap %d\n", cap);
159 	}
160 
161 	return 0;
162 }
163 
164 static ssize_t ddr_perf_filter_cap_show(struct device *dev,
165 					struct device_attribute *attr,
166 					char *buf)
167 {
168 	struct ddr_pmu *pmu = dev_get_drvdata(dev);
169 	struct dev_ext_attribute *ea =
170 		container_of(attr, struct dev_ext_attribute, attr);
171 	int cap = (long)ea->var;
172 
173 	return snprintf(buf, PAGE_SIZE, "%u\n",
174 			ddr_perf_filter_cap_get(pmu, cap));
175 }
176 
177 #define PERF_EXT_ATTR_ENTRY(_name, _func, _var)				\
178 	(&((struct dev_ext_attribute) {					\
179 		__ATTR(_name, 0444, _func, NULL), (void *)_var		\
180 	}).attr.attr)
181 
182 #define PERF_FILTER_EXT_ATTR_ENTRY(_name, _var)				\
183 	PERF_EXT_ATTR_ENTRY(_name, ddr_perf_filter_cap_show, _var)
184 
185 static struct attribute *ddr_perf_filter_cap_attr[] = {
186 	PERF_FILTER_EXT_ATTR_ENTRY(filter, PERF_CAP_AXI_ID_FILTER),
187 	PERF_FILTER_EXT_ATTR_ENTRY(enhanced_filter, PERF_CAP_AXI_ID_FILTER_ENHANCED),
188 	NULL,
189 };
190 
191 static struct attribute_group ddr_perf_filter_cap_attr_group = {
192 	.name = "caps",
193 	.attrs = ddr_perf_filter_cap_attr,
194 };
195 
196 static ssize_t ddr_perf_cpumask_show(struct device *dev,
197 				struct device_attribute *attr, char *buf)
198 {
199 	struct ddr_pmu *pmu = dev_get_drvdata(dev);
200 
201 	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
202 }
203 
204 static struct device_attribute ddr_perf_cpumask_attr =
205 	__ATTR(cpumask, 0444, ddr_perf_cpumask_show, NULL);
206 
207 static struct attribute *ddr_perf_cpumask_attrs[] = {
208 	&ddr_perf_cpumask_attr.attr,
209 	NULL,
210 };
211 
212 static struct attribute_group ddr_perf_cpumask_attr_group = {
213 	.attrs = ddr_perf_cpumask_attrs,
214 };
215 
216 static ssize_t
217 ddr_pmu_event_show(struct device *dev, struct device_attribute *attr,
218 		   char *page)
219 {
220 	struct perf_pmu_events_attr *pmu_attr;
221 
222 	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
223 	return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
224 }
225 
226 #define IMX8_DDR_PMU_EVENT_ATTR(_name, _id)				\
227 	(&((struct perf_pmu_events_attr[]) {				\
228 		{ .attr = __ATTR(_name, 0444, ddr_pmu_event_show, NULL),\
229 		  .id = _id, }						\
230 	})[0].attr.attr)
231 
232 static struct attribute *ddr_perf_events_attrs[] = {
233 	IMX8_DDR_PMU_EVENT_ATTR(cycles, EVENT_CYCLES_ID),
234 	IMX8_DDR_PMU_EVENT_ATTR(selfresh, 0x01),
235 	IMX8_DDR_PMU_EVENT_ATTR(read-accesses, 0x04),
236 	IMX8_DDR_PMU_EVENT_ATTR(write-accesses, 0x05),
237 	IMX8_DDR_PMU_EVENT_ATTR(read-queue-depth, 0x08),
238 	IMX8_DDR_PMU_EVENT_ATTR(write-queue-depth, 0x09),
239 	IMX8_DDR_PMU_EVENT_ATTR(lp-read-credit-cnt, 0x10),
240 	IMX8_DDR_PMU_EVENT_ATTR(hp-read-credit-cnt, 0x11),
241 	IMX8_DDR_PMU_EVENT_ATTR(write-credit-cnt, 0x12),
242 	IMX8_DDR_PMU_EVENT_ATTR(read-command, 0x20),
243 	IMX8_DDR_PMU_EVENT_ATTR(write-command, 0x21),
244 	IMX8_DDR_PMU_EVENT_ATTR(read-modify-write-command, 0x22),
245 	IMX8_DDR_PMU_EVENT_ATTR(hp-read, 0x23),
246 	IMX8_DDR_PMU_EVENT_ATTR(hp-req-nocredit, 0x24),
247 	IMX8_DDR_PMU_EVENT_ATTR(hp-xact-credit, 0x25),
248 	IMX8_DDR_PMU_EVENT_ATTR(lp-req-nocredit, 0x26),
249 	IMX8_DDR_PMU_EVENT_ATTR(lp-xact-credit, 0x27),
250 	IMX8_DDR_PMU_EVENT_ATTR(wr-xact-credit, 0x29),
251 	IMX8_DDR_PMU_EVENT_ATTR(read-cycles, 0x2a),
252 	IMX8_DDR_PMU_EVENT_ATTR(write-cycles, 0x2b),
253 	IMX8_DDR_PMU_EVENT_ATTR(read-write-transition, 0x30),
254 	IMX8_DDR_PMU_EVENT_ATTR(precharge, 0x31),
255 	IMX8_DDR_PMU_EVENT_ATTR(activate, 0x32),
256 	IMX8_DDR_PMU_EVENT_ATTR(load-mode, 0x33),
257 	IMX8_DDR_PMU_EVENT_ATTR(perf-mwr, 0x34),
258 	IMX8_DDR_PMU_EVENT_ATTR(read, 0x35),
259 	IMX8_DDR_PMU_EVENT_ATTR(read-activate, 0x36),
260 	IMX8_DDR_PMU_EVENT_ATTR(refresh, 0x37),
261 	IMX8_DDR_PMU_EVENT_ATTR(write, 0x38),
262 	IMX8_DDR_PMU_EVENT_ATTR(raw-hazard, 0x39),
263 	IMX8_DDR_PMU_EVENT_ATTR(axid-read, 0x41),
264 	IMX8_DDR_PMU_EVENT_ATTR(axid-write, 0x42),
265 	NULL,
266 };
267 
268 static struct attribute_group ddr_perf_events_attr_group = {
269 	.name = "events",
270 	.attrs = ddr_perf_events_attrs,
271 };
272 
273 PMU_FORMAT_ATTR(event, "config:0-7");
274 PMU_FORMAT_ATTR(axi_id, "config1:0-15");
275 PMU_FORMAT_ATTR(axi_mask, "config1:16-31");
276 
277 static struct attribute *ddr_perf_format_attrs[] = {
278 	&format_attr_event.attr,
279 	&format_attr_axi_id.attr,
280 	&format_attr_axi_mask.attr,
281 	NULL,
282 };
283 
284 static struct attribute_group ddr_perf_format_attr_group = {
285 	.name = "format",
286 	.attrs = ddr_perf_format_attrs,
287 };
288 
289 static const struct attribute_group *attr_groups[] = {
290 	&ddr_perf_events_attr_group,
291 	&ddr_perf_format_attr_group,
292 	&ddr_perf_cpumask_attr_group,
293 	&ddr_perf_filter_cap_attr_group,
294 	&ddr_perf_identifier_attr_group,
295 	NULL,
296 };
297 
298 static bool ddr_perf_is_filtered(struct perf_event *event)
299 {
300 	return event->attr.config == 0x41 || event->attr.config == 0x42;
301 }
302 
303 static u32 ddr_perf_filter_val(struct perf_event *event)
304 {
305 	return event->attr.config1;
306 }
307 
308 static bool ddr_perf_filters_compatible(struct perf_event *a,
309 					struct perf_event *b)
310 {
311 	if (!ddr_perf_is_filtered(a))
312 		return true;
313 	if (!ddr_perf_is_filtered(b))
314 		return true;
315 	return ddr_perf_filter_val(a) == ddr_perf_filter_val(b);
316 }
317 
318 static bool ddr_perf_is_enhanced_filtered(struct perf_event *event)
319 {
320 	unsigned int filt;
321 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
322 
323 	filt = pmu->devtype_data->quirks & DDR_CAP_AXI_ID_FILTER_ENHANCED;
324 	return (filt == DDR_CAP_AXI_ID_FILTER_ENHANCED) &&
325 		ddr_perf_is_filtered(event);
326 }
327 
328 static u32 ddr_perf_alloc_counter(struct ddr_pmu *pmu, int event)
329 {
330 	int i;
331 
332 	/*
333 	 * Always map cycle event to counter 0
334 	 * Cycles counter is dedicated for cycle event
335 	 * can't used for the other events
336 	 */
337 	if (event == EVENT_CYCLES_ID) {
338 		if (pmu->events[EVENT_CYCLES_COUNTER] == NULL)
339 			return EVENT_CYCLES_COUNTER;
340 		else
341 			return -ENOENT;
342 	}
343 
344 	for (i = 1; i < NUM_COUNTERS; i++) {
345 		if (pmu->events[i] == NULL)
346 			return i;
347 	}
348 
349 	return -ENOENT;
350 }
351 
352 static void ddr_perf_free_counter(struct ddr_pmu *pmu, int counter)
353 {
354 	pmu->events[counter] = NULL;
355 }
356 
357 static u32 ddr_perf_read_counter(struct ddr_pmu *pmu, int counter)
358 {
359 	struct perf_event *event = pmu->events[counter];
360 	void __iomem *base = pmu->base;
361 
362 	/*
363 	 * return bytes instead of bursts from ddr transaction for
364 	 * axid-read and axid-write event if PMU core supports enhanced
365 	 * filter.
366 	 */
367 	base += ddr_perf_is_enhanced_filtered(event) ? COUNTER_DPCR1 :
368 						       COUNTER_READ;
369 	return readl_relaxed(base + counter * 4);
370 }
371 
372 static int ddr_perf_event_init(struct perf_event *event)
373 {
374 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
375 	struct hw_perf_event *hwc = &event->hw;
376 	struct perf_event *sibling;
377 
378 	if (event->attr.type != event->pmu->type)
379 		return -ENOENT;
380 
381 	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
382 		return -EOPNOTSUPP;
383 
384 	if (event->cpu < 0) {
385 		dev_warn(pmu->dev, "Can't provide per-task data!\n");
386 		return -EOPNOTSUPP;
387 	}
388 
389 	/*
390 	 * We must NOT create groups containing mixed PMUs, although software
391 	 * events are acceptable (for example to create a CCN group
392 	 * periodically read when a hrtimer aka cpu-clock leader triggers).
393 	 */
394 	if (event->group_leader->pmu != event->pmu &&
395 			!is_software_event(event->group_leader))
396 		return -EINVAL;
397 
398 	if (pmu->devtype_data->quirks & DDR_CAP_AXI_ID_FILTER) {
399 		if (!ddr_perf_filters_compatible(event, event->group_leader))
400 			return -EINVAL;
401 		for_each_sibling_event(sibling, event->group_leader) {
402 			if (!ddr_perf_filters_compatible(event, sibling))
403 				return -EINVAL;
404 		}
405 	}
406 
407 	for_each_sibling_event(sibling, event->group_leader) {
408 		if (sibling->pmu != event->pmu &&
409 				!is_software_event(sibling))
410 			return -EINVAL;
411 	}
412 
413 	event->cpu = pmu->cpu;
414 	hwc->idx = -1;
415 
416 	return 0;
417 }
418 
419 static void ddr_perf_counter_enable(struct ddr_pmu *pmu, int config,
420 				  int counter, bool enable)
421 {
422 	u8 reg = counter * 4 + COUNTER_CNTL;
423 	int val;
424 
425 	if (enable) {
426 		/*
427 		 * cycle counter is special which should firstly write 0 then
428 		 * write 1 into CLEAR bit to clear it. Other counters only
429 		 * need write 0 into CLEAR bit and it turns out to be 1 by
430 		 * hardware. Below enable flow is harmless for all counters.
431 		 */
432 		writel(0, pmu->base + reg);
433 		val = CNTL_EN | CNTL_CLEAR;
434 		val |= FIELD_PREP(CNTL_CSV_MASK, config);
435 		writel(val, pmu->base + reg);
436 	} else {
437 		/* Disable counter */
438 		val = readl_relaxed(pmu->base + reg) & CNTL_EN_MASK;
439 		writel(val, pmu->base + reg);
440 	}
441 }
442 
443 static bool ddr_perf_counter_overflow(struct ddr_pmu *pmu, int counter)
444 {
445 	int val;
446 
447 	val = readl_relaxed(pmu->base + counter * 4 + COUNTER_CNTL);
448 
449 	return val & CNTL_OVER;
450 }
451 
452 static void ddr_perf_counter_clear(struct ddr_pmu *pmu, int counter)
453 {
454 	u8 reg = counter * 4 + COUNTER_CNTL;
455 	int val;
456 
457 	val = readl_relaxed(pmu->base + reg);
458 	val &= ~CNTL_CLEAR;
459 	writel(val, pmu->base + reg);
460 
461 	val |= CNTL_CLEAR;
462 	writel(val, pmu->base + reg);
463 }
464 
465 static void ddr_perf_event_update(struct perf_event *event)
466 {
467 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
468 	struct hw_perf_event *hwc = &event->hw;
469 	u64 new_raw_count;
470 	int counter = hwc->idx;
471 	int ret;
472 
473 	new_raw_count = ddr_perf_read_counter(pmu, counter);
474 	local64_add(new_raw_count, &event->count);
475 
476 	/*
477 	 * For legacy SoCs: event counter continue counting when overflow,
478 	 *                  no need to clear the counter.
479 	 * For new SoCs: event counter stop counting when overflow, need
480 	 *               clear counter to let it count again.
481 	 */
482 	if (counter != EVENT_CYCLES_COUNTER) {
483 		ret = ddr_perf_counter_overflow(pmu, counter);
484 		if (ret)
485 			dev_warn_ratelimited(pmu->dev,  "events lost due to counter overflow (config 0x%llx)\n",
486 					     event->attr.config);
487 	}
488 
489 	/* clear counter every time for both cycle counter and event counter */
490 	ddr_perf_counter_clear(pmu, counter);
491 }
492 
493 static void ddr_perf_event_start(struct perf_event *event, int flags)
494 {
495 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
496 	struct hw_perf_event *hwc = &event->hw;
497 	int counter = hwc->idx;
498 
499 	local64_set(&hwc->prev_count, 0);
500 
501 	ddr_perf_counter_enable(pmu, event->attr.config, counter, true);
502 
503 	hwc->state = 0;
504 }
505 
506 static int ddr_perf_event_add(struct perf_event *event, int flags)
507 {
508 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
509 	struct hw_perf_event *hwc = &event->hw;
510 	int counter;
511 	int cfg = event->attr.config;
512 	int cfg1 = event->attr.config1;
513 
514 	if (pmu->devtype_data->quirks & DDR_CAP_AXI_ID_FILTER) {
515 		int i;
516 
517 		for (i = 1; i < NUM_COUNTERS; i++) {
518 			if (pmu->events[i] &&
519 			    !ddr_perf_filters_compatible(event, pmu->events[i]))
520 				return -EINVAL;
521 		}
522 
523 		if (ddr_perf_is_filtered(event)) {
524 			/* revert axi id masking(axi_mask) value */
525 			cfg1 ^= AXI_MASKING_REVERT;
526 			writel(cfg1, pmu->base + COUNTER_DPCR1);
527 		}
528 	}
529 
530 	counter = ddr_perf_alloc_counter(pmu, cfg);
531 	if (counter < 0) {
532 		dev_dbg(pmu->dev, "There are not enough counters\n");
533 		return -EOPNOTSUPP;
534 	}
535 
536 	pmu->events[counter] = event;
537 	pmu->active_events++;
538 	hwc->idx = counter;
539 
540 	hwc->state |= PERF_HES_STOPPED;
541 
542 	if (flags & PERF_EF_START)
543 		ddr_perf_event_start(event, flags);
544 
545 	return 0;
546 }
547 
548 static void ddr_perf_event_stop(struct perf_event *event, int flags)
549 {
550 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
551 	struct hw_perf_event *hwc = &event->hw;
552 	int counter = hwc->idx;
553 
554 	ddr_perf_counter_enable(pmu, event->attr.config, counter, false);
555 	ddr_perf_event_update(event);
556 
557 	hwc->state |= PERF_HES_STOPPED;
558 }
559 
560 static void ddr_perf_event_del(struct perf_event *event, int flags)
561 {
562 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
563 	struct hw_perf_event *hwc = &event->hw;
564 	int counter = hwc->idx;
565 
566 	ddr_perf_event_stop(event, PERF_EF_UPDATE);
567 
568 	ddr_perf_free_counter(pmu, counter);
569 	pmu->active_events--;
570 	hwc->idx = -1;
571 }
572 
573 static void ddr_perf_pmu_enable(struct pmu *pmu)
574 {
575 	struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
576 
577 	/* enable cycle counter if cycle is not active event list */
578 	if (ddr_pmu->events[EVENT_CYCLES_COUNTER] == NULL)
579 		ddr_perf_counter_enable(ddr_pmu,
580 				      EVENT_CYCLES_ID,
581 				      EVENT_CYCLES_COUNTER,
582 				      true);
583 }
584 
585 static void ddr_perf_pmu_disable(struct pmu *pmu)
586 {
587 	struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
588 
589 	if (ddr_pmu->events[EVENT_CYCLES_COUNTER] == NULL)
590 		ddr_perf_counter_enable(ddr_pmu,
591 				      EVENT_CYCLES_ID,
592 				      EVENT_CYCLES_COUNTER,
593 				      false);
594 }
595 
596 static int ddr_perf_init(struct ddr_pmu *pmu, void __iomem *base,
597 			 struct device *dev)
598 {
599 	*pmu = (struct ddr_pmu) {
600 		.pmu = (struct pmu) {
601 			.module	      = THIS_MODULE,
602 			.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
603 			.task_ctx_nr = perf_invalid_context,
604 			.attr_groups = attr_groups,
605 			.event_init  = ddr_perf_event_init,
606 			.add	     = ddr_perf_event_add,
607 			.del	     = ddr_perf_event_del,
608 			.start	     = ddr_perf_event_start,
609 			.stop	     = ddr_perf_event_stop,
610 			.read	     = ddr_perf_event_update,
611 			.pmu_enable  = ddr_perf_pmu_enable,
612 			.pmu_disable = ddr_perf_pmu_disable,
613 		},
614 		.base = base,
615 		.dev = dev,
616 	};
617 
618 	pmu->id = ida_simple_get(&ddr_ida, 0, 0, GFP_KERNEL);
619 	return pmu->id;
620 }
621 
622 static irqreturn_t ddr_perf_irq_handler(int irq, void *p)
623 {
624 	int i;
625 	struct ddr_pmu *pmu = (struct ddr_pmu *) p;
626 	struct perf_event *event;
627 
628 	/* all counter will stop if cycle counter disabled */
629 	ddr_perf_counter_enable(pmu,
630 			      EVENT_CYCLES_ID,
631 			      EVENT_CYCLES_COUNTER,
632 			      false);
633 	/*
634 	 * When the cycle counter overflows, all counters are stopped,
635 	 * and an IRQ is raised. If any other counter overflows, it
636 	 * continues counting, and no IRQ is raised. But for new SoCs,
637 	 * such as i.MX8MP, event counter would stop when overflow, so
638 	 * we need use cycle counter to stop overflow of event counter.
639 	 *
640 	 * Cycles occur at least 4 times as often as other events, so we
641 	 * can update all events on a cycle counter overflow and not
642 	 * lose events.
643 	 *
644 	 */
645 	for (i = 0; i < NUM_COUNTERS; i++) {
646 
647 		if (!pmu->events[i])
648 			continue;
649 
650 		event = pmu->events[i];
651 
652 		ddr_perf_event_update(event);
653 	}
654 
655 	ddr_perf_counter_enable(pmu,
656 			      EVENT_CYCLES_ID,
657 			      EVENT_CYCLES_COUNTER,
658 			      true);
659 
660 	return IRQ_HANDLED;
661 }
662 
663 static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)
664 {
665 	struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node);
666 	int target;
667 
668 	if (cpu != pmu->cpu)
669 		return 0;
670 
671 	target = cpumask_any_but(cpu_online_mask, cpu);
672 	if (target >= nr_cpu_ids)
673 		return 0;
674 
675 	perf_pmu_migrate_context(&pmu->pmu, cpu, target);
676 	pmu->cpu = target;
677 
678 	WARN_ON(irq_set_affinity_hint(pmu->irq, cpumask_of(pmu->cpu)));
679 
680 	return 0;
681 }
682 
683 static int ddr_perf_probe(struct platform_device *pdev)
684 {
685 	struct ddr_pmu *pmu;
686 	struct device_node *np;
687 	void __iomem *base;
688 	char *name;
689 	int num;
690 	int ret;
691 	int irq;
692 
693 	base = devm_platform_ioremap_resource(pdev, 0);
694 	if (IS_ERR(base))
695 		return PTR_ERR(base);
696 
697 	np = pdev->dev.of_node;
698 
699 	pmu = devm_kzalloc(&pdev->dev, sizeof(*pmu), GFP_KERNEL);
700 	if (!pmu)
701 		return -ENOMEM;
702 
703 	num = ddr_perf_init(pmu, base, &pdev->dev);
704 
705 	platform_set_drvdata(pdev, pmu);
706 
707 	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d",
708 			      num);
709 	if (!name)
710 		return -ENOMEM;
711 
712 	pmu->devtype_data = of_device_get_match_data(&pdev->dev);
713 
714 	pmu->cpu = raw_smp_processor_id();
715 	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
716 				      DDR_CPUHP_CB_NAME,
717 				      NULL,
718 				      ddr_perf_offline_cpu);
719 
720 	if (ret < 0) {
721 		dev_err(&pdev->dev, "cpuhp_setup_state_multi failed\n");
722 		goto cpuhp_state_err;
723 	}
724 
725 	pmu->cpuhp_state = ret;
726 
727 	/* Register the pmu instance for cpu hotplug */
728 	ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);
729 	if (ret) {
730 		dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
731 		goto cpuhp_instance_err;
732 	}
733 
734 	/* Request irq */
735 	irq = of_irq_get(np, 0);
736 	if (irq < 0) {
737 		dev_err(&pdev->dev, "Failed to get irq: %d", irq);
738 		ret = irq;
739 		goto ddr_perf_err;
740 	}
741 
742 	ret = devm_request_irq(&pdev->dev, irq,
743 					ddr_perf_irq_handler,
744 					IRQF_NOBALANCING | IRQF_NO_THREAD,
745 					DDR_CPUHP_CB_NAME,
746 					pmu);
747 	if (ret < 0) {
748 		dev_err(&pdev->dev, "Request irq failed: %d", ret);
749 		goto ddr_perf_err;
750 	}
751 
752 	pmu->irq = irq;
753 	ret = irq_set_affinity_hint(pmu->irq, cpumask_of(pmu->cpu));
754 	if (ret) {
755 		dev_err(pmu->dev, "Failed to set interrupt affinity!\n");
756 		goto ddr_perf_err;
757 	}
758 
759 	ret = perf_pmu_register(&pmu->pmu, name, -1);
760 	if (ret)
761 		goto ddr_perf_err;
762 
763 	return 0;
764 
765 ddr_perf_err:
766 	cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
767 cpuhp_instance_err:
768 	cpuhp_remove_multi_state(pmu->cpuhp_state);
769 cpuhp_state_err:
770 	ida_simple_remove(&ddr_ida, pmu->id);
771 	dev_warn(&pdev->dev, "i.MX8 DDR Perf PMU failed (%d), disabled\n", ret);
772 	return ret;
773 }
774 
775 static int ddr_perf_remove(struct platform_device *pdev)
776 {
777 	struct ddr_pmu *pmu = platform_get_drvdata(pdev);
778 
779 	cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
780 	cpuhp_remove_multi_state(pmu->cpuhp_state);
781 	irq_set_affinity_hint(pmu->irq, NULL);
782 
783 	perf_pmu_unregister(&pmu->pmu);
784 
785 	ida_simple_remove(&ddr_ida, pmu->id);
786 	return 0;
787 }
788 
789 static struct platform_driver imx_ddr_pmu_driver = {
790 	.driver         = {
791 		.name   = "imx-ddr-pmu",
792 		.of_match_table = imx_ddr_pmu_dt_ids,
793 		.suppress_bind_attrs = true,
794 	},
795 	.probe          = ddr_perf_probe,
796 	.remove         = ddr_perf_remove,
797 };
798 
799 module_platform_driver(imx_ddr_pmu_driver);
800 MODULE_LICENSE("GPL v2");
801