1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Alibaba DDR Sub-System Driveway PMU driver
4  *
5  * Copyright (C) 2022 Alibaba Inc
6  */
7 
8 #define ALI_DRW_PMUNAME		"ali_drw"
9 #define ALI_DRW_DRVNAME		ALI_DRW_PMUNAME "_pmu"
10 #define pr_fmt(fmt)		ALI_DRW_DRVNAME ": " fmt
11 
12 #include <linux/acpi.h>
13 #include <linux/bitfield.h>
14 #include <linux/bitmap.h>
15 #include <linux/bitops.h>
16 #include <linux/cpuhotplug.h>
17 #include <linux/cpumask.h>
18 #include <linux/device.h>
19 #include <linux/errno.h>
20 #include <linux/interrupt.h>
21 #include <linux/irq.h>
22 #include <linux/kernel.h>
23 #include <linux/list.h>
24 #include <linux/module.h>
25 #include <linux/mutex.h>
26 #include <linux/perf_event.h>
27 #include <linux/platform_device.h>
28 #include <linux/printk.h>
29 #include <linux/rculist.h>
30 #include <linux/refcount.h>
31 
32 
33 #define ALI_DRW_PMU_COMMON_MAX_COUNTERS			16
34 #define ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE	19
35 
36 #define ALI_DRW_PMU_PA_SHIFT			12
37 #define ALI_DRW_PMU_CNT_INIT			0x00000000
38 #define ALI_DRW_CNT_MAX_PERIOD			0xffffffff
39 #define ALI_DRW_PMU_CYCLE_EVT_ID		0x80
40 
41 #define ALI_DRW_PMU_CNT_CTRL			0xC00
42 #define ALI_DRW_PMU_CNT_RST			BIT(2)
43 #define ALI_DRW_PMU_CNT_STOP			BIT(1)
44 #define ALI_DRW_PMU_CNT_START			BIT(0)
45 
46 #define ALI_DRW_PMU_CNT_STATE			0xC04
47 #define ALI_DRW_PMU_TEST_CTRL			0xC08
48 #define ALI_DRW_PMU_CNT_PRELOAD			0xC0C
49 
50 #define ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK		GENMASK(23, 0)
51 #define ALI_DRW_PMU_CYCLE_CNT_LOW_MASK		GENMASK(31, 0)
52 #define ALI_DRW_PMU_CYCLE_CNT_HIGH		0xC10
53 #define ALI_DRW_PMU_CYCLE_CNT_LOW		0xC14
54 
55 /* PMU EVENT SEL 0-3 are paired in 32-bit registers on a 4-byte stride */
56 #define ALI_DRW_PMU_EVENT_SEL0			0xC68
57 /* counter 0-3 use sel0, counter 4-7 use sel1...*/
58 #define ALI_DRW_PMU_EVENT_SELn(n) \
59 	(ALI_DRW_PMU_EVENT_SEL0 + (n / 4) * 0x4)
60 #define ALI_DRW_PMCOM_CNT_EN			BIT(7)
61 #define ALI_DRW_PMCOM_CNT_EVENT_MASK		GENMASK(5, 0)
62 #define ALI_DRW_PMCOM_CNT_EVENT_OFFSET(n) \
63 	(8 * (n % 4))
64 
65 /* PMU COMMON COUNTER 0-15, are paired in 32-bit registers on a 4-byte stride */
66 #define ALI_DRW_PMU_COMMON_COUNTER0		0xC78
67 #define ALI_DRW_PMU_COMMON_COUNTERn(n) \
68 	(ALI_DRW_PMU_COMMON_COUNTER0 + 0x4 * (n))
69 
70 #define ALI_DRW_PMU_OV_INTR_ENABLE_CTL		0xCB8
71 #define ALI_DRW_PMU_OV_INTR_DISABLE_CTL		0xCBC
72 #define ALI_DRW_PMU_OV_INTR_ENABLE_STATUS	0xCC0
73 #define ALI_DRW_PMU_OV_INTR_CLR			0xCC4
74 #define ALI_DRW_PMU_OV_INTR_STATUS		0xCC8
75 #define ALI_DRW_PMCOM_CNT_OV_INTR_MASK		GENMASK(23, 8)
76 #define ALI_DRW_PMBW_CNT_OV_INTR_MASK		GENMASK(7, 0)
77 #define ALI_DRW_PMU_OV_INTR_MASK		GENMASK_ULL(63, 0)
78 
79 static int ali_drw_cpuhp_state_num;
80 
81 static LIST_HEAD(ali_drw_pmu_irqs);
82 static DEFINE_MUTEX(ali_drw_pmu_irqs_lock);
83 
84 struct ali_drw_pmu_irq {
85 	struct hlist_node node;
86 	struct list_head irqs_node;
87 	struct list_head pmus_node;
88 	int irq_num;
89 	int cpu;
90 	refcount_t refcount;
91 };
92 
93 struct ali_drw_pmu {
94 	void __iomem *cfg_base;
95 	struct device *dev;
96 
97 	struct list_head pmus_node;
98 	struct ali_drw_pmu_irq *irq;
99 	int irq_num;
100 	int cpu;
101 	DECLARE_BITMAP(used_mask, ALI_DRW_PMU_COMMON_MAX_COUNTERS);
102 	struct perf_event *events[ALI_DRW_PMU_COMMON_MAX_COUNTERS];
103 	int evtids[ALI_DRW_PMU_COMMON_MAX_COUNTERS];
104 
105 	struct pmu pmu;
106 };
107 
108 #define to_ali_drw_pmu(p) (container_of(p, struct ali_drw_pmu, pmu))
109 
110 #define DRW_CONFIG_EVENTID		GENMASK(7, 0)
111 #define GET_DRW_EVENTID(event)	FIELD_GET(DRW_CONFIG_EVENTID, (event)->attr.config)
112 
ali_drw_pmu_format_show(struct device * dev,struct device_attribute * attr,char * buf)113 static ssize_t ali_drw_pmu_format_show(struct device *dev,
114 				struct device_attribute *attr, char *buf)
115 {
116 	struct dev_ext_attribute *eattr;
117 
118 	eattr = container_of(attr, struct dev_ext_attribute, attr);
119 
120 	return sprintf(buf, "%s\n", (char *)eattr->var);
121 }
122 
123 /*
124  * PMU event attributes
125  */
ali_drw_pmu_event_show(struct device * dev,struct device_attribute * attr,char * page)126 static ssize_t ali_drw_pmu_event_show(struct device *dev,
127 			       struct device_attribute *attr, char *page)
128 {
129 	struct dev_ext_attribute *eattr;
130 
131 	eattr = container_of(attr, struct dev_ext_attribute, attr);
132 
133 	return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);
134 }
135 
136 #define ALI_DRW_PMU_ATTR(_name, _func, _config)                            \
137 		(&((struct dev_ext_attribute[]) {                               \
138 				{ __ATTR(_name, 0444, _func, NULL), (void *)_config }   \
139 		})[0].attr.attr)
140 
141 #define ALI_DRW_PMU_FORMAT_ATTR(_name, _config)            \
142 	ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_format_show, (void *)_config)
143 #define ALI_DRW_PMU_EVENT_ATTR(_name, _config)             \
144 	ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_event_show, (unsigned long)_config)
145 
146 static struct attribute *ali_drw_pmu_events_attrs[] = {
147 	ALI_DRW_PMU_EVENT_ATTR(hif_rd_or_wr,			0x0),
148 	ALI_DRW_PMU_EVENT_ATTR(hif_wr,				0x1),
149 	ALI_DRW_PMU_EVENT_ATTR(hif_rd,				0x2),
150 	ALI_DRW_PMU_EVENT_ATTR(hif_rmw,				0x3),
151 	ALI_DRW_PMU_EVENT_ATTR(hif_hi_pri_rd,			0x4),
152 	ALI_DRW_PMU_EVENT_ATTR(dfi_wr_data_cycles,		0x7),
153 	ALI_DRW_PMU_EVENT_ATTR(dfi_rd_data_cycles,		0x8),
154 	ALI_DRW_PMU_EVENT_ATTR(hpr_xact_when_critical,		0x9),
155 	ALI_DRW_PMU_EVENT_ATTR(lpr_xact_when_critical,		0xA),
156 	ALI_DRW_PMU_EVENT_ATTR(wr_xact_when_critical,		0xB),
157 	ALI_DRW_PMU_EVENT_ATTR(op_is_activate,			0xC),
158 	ALI_DRW_PMU_EVENT_ATTR(op_is_rd_or_wr,			0xD),
159 	ALI_DRW_PMU_EVENT_ATTR(op_is_rd_activate,		0xE),
160 	ALI_DRW_PMU_EVENT_ATTR(op_is_rd,			0xF),
161 	ALI_DRW_PMU_EVENT_ATTR(op_is_wr,			0x10),
162 	ALI_DRW_PMU_EVENT_ATTR(op_is_mwr,			0x11),
163 	ALI_DRW_PMU_EVENT_ATTR(op_is_precharge,			0x12),
164 	ALI_DRW_PMU_EVENT_ATTR(precharge_for_rdwr,		0x13),
165 	ALI_DRW_PMU_EVENT_ATTR(precharge_for_other,		0x14),
166 	ALI_DRW_PMU_EVENT_ATTR(rdwr_transitions,		0x15),
167 	ALI_DRW_PMU_EVENT_ATTR(write_combine,			0x16),
168 	ALI_DRW_PMU_EVENT_ATTR(war_hazard,			0x17),
169 	ALI_DRW_PMU_EVENT_ATTR(raw_hazard,			0x18),
170 	ALI_DRW_PMU_EVENT_ATTR(waw_hazard,			0x19),
171 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk0,		0x1A),
172 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk1,		0x1B),
173 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk2,		0x1C),
174 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk3,		0x1D),
175 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk0,	0x1E),
176 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk1,	0x1F),
177 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk2,	0x20),
178 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk3,	0x21),
179 	ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk0,		0x26),
180 	ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk1,		0x27),
181 	ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk2,		0x28),
182 	ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk3,		0x29),
183 	ALI_DRW_PMU_EVENT_ATTR(op_is_refresh,			0x2A),
184 	ALI_DRW_PMU_EVENT_ATTR(op_is_crit_ref,			0x2B),
185 	ALI_DRW_PMU_EVENT_ATTR(op_is_load_mode,			0x2D),
186 	ALI_DRW_PMU_EVENT_ATTR(op_is_zqcl,			0x2E),
187 	ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_rd, 0x30),
188 	ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_wr, 0x31),
189 	ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mpc,		0x34),
190 	ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mrr,		0x35),
191 	ALI_DRW_PMU_EVENT_ATTR(op_is_tcr_mrr,			0x36),
192 	ALI_DRW_PMU_EVENT_ATTR(op_is_zqstart,			0x37),
193 	ALI_DRW_PMU_EVENT_ATTR(op_is_zqlatch,			0x38),
194 	ALI_DRW_PMU_EVENT_ATTR(chi_txreq,			0x39),
195 	ALI_DRW_PMU_EVENT_ATTR(chi_txdat,			0x3A),
196 	ALI_DRW_PMU_EVENT_ATTR(chi_rxdat,			0x3B),
197 	ALI_DRW_PMU_EVENT_ATTR(chi_rxrsp,			0x3C),
198 	ALI_DRW_PMU_EVENT_ATTR(tsz_vio,				0x3D),
199 	ALI_DRW_PMU_EVENT_ATTR(cycle,				0x80),
200 	NULL,
201 };
202 
203 static struct attribute_group ali_drw_pmu_events_attr_group = {
204 	.name = "events",
205 	.attrs = ali_drw_pmu_events_attrs,
206 };
207 
208 static struct attribute *ali_drw_pmu_format_attr[] = {
209 	ALI_DRW_PMU_FORMAT_ATTR(event, "config:0-7"),
210 	NULL,
211 };
212 
213 static const struct attribute_group ali_drw_pmu_format_group = {
214 	.name = "format",
215 	.attrs = ali_drw_pmu_format_attr,
216 };
217 
ali_drw_pmu_cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)218 static ssize_t ali_drw_pmu_cpumask_show(struct device *dev,
219 					struct device_attribute *attr,
220 					char *buf)
221 {
222 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(dev_get_drvdata(dev));
223 
224 	return cpumap_print_to_pagebuf(true, buf, cpumask_of(drw_pmu->cpu));
225 }
226 
227 static struct device_attribute ali_drw_pmu_cpumask_attr =
228 		__ATTR(cpumask, 0444, ali_drw_pmu_cpumask_show, NULL);
229 
230 static struct attribute *ali_drw_pmu_cpumask_attrs[] = {
231 	&ali_drw_pmu_cpumask_attr.attr,
232 	NULL,
233 };
234 
235 static const struct attribute_group ali_drw_pmu_cpumask_attr_group = {
236 	.attrs = ali_drw_pmu_cpumask_attrs,
237 };
238 
ali_drw_pmu_identifier_show(struct device * dev,struct device_attribute * attr,char * page)239 static ssize_t ali_drw_pmu_identifier_show(struct device *dev,
240 					struct device_attribute *attr,
241 					char *page)
242 {
243 	return sysfs_emit(page, "%s\n", "ali_drw_pmu");
244 }
245 
ali_drw_pmu_identifier_attr_visible(struct kobject * kobj,struct attribute * attr,int n)246 static umode_t ali_drw_pmu_identifier_attr_visible(struct kobject *kobj,
247 						struct attribute *attr, int n)
248 {
249 	return attr->mode;
250 }
251 
252 static struct device_attribute ali_drw_pmu_identifier_attr =
253 	__ATTR(identifier, 0444, ali_drw_pmu_identifier_show, NULL);
254 
255 static struct attribute *ali_drw_pmu_identifier_attrs[] = {
256 	&ali_drw_pmu_identifier_attr.attr,
257 	NULL
258 };
259 
260 static const struct attribute_group ali_drw_pmu_identifier_attr_group = {
261 	.attrs = ali_drw_pmu_identifier_attrs,
262 	.is_visible = ali_drw_pmu_identifier_attr_visible
263 };
264 
265 static const struct attribute_group *ali_drw_pmu_attr_groups[] = {
266 	&ali_drw_pmu_events_attr_group,
267 	&ali_drw_pmu_cpumask_attr_group,
268 	&ali_drw_pmu_format_group,
269 	&ali_drw_pmu_identifier_attr_group,
270 	NULL,
271 };
272 
273 /* find a counter for event, then in add func, hw.idx will equal to counter */
ali_drw_get_counter_idx(struct perf_event * event)274 static int ali_drw_get_counter_idx(struct perf_event *event)
275 {
276 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
277 	int idx;
278 
279 	for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; ++idx) {
280 		if (!test_and_set_bit(idx, drw_pmu->used_mask))
281 			return idx;
282 	}
283 
284 	/* The counters are all in use. */
285 	return -EBUSY;
286 }
287 
ali_drw_pmu_read_counter(struct perf_event * event)288 static u64 ali_drw_pmu_read_counter(struct perf_event *event)
289 {
290 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
291 	u64 cycle_high, cycle_low;
292 
293 	if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {
294 		cycle_high = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_HIGH);
295 		cycle_high &= ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK;
296 		cycle_low = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_LOW);
297 		cycle_low &= ALI_DRW_PMU_CYCLE_CNT_LOW_MASK;
298 		return (cycle_high << 32 | cycle_low);
299 	}
300 
301 	return readl(drw_pmu->cfg_base +
302 		     ALI_DRW_PMU_COMMON_COUNTERn(event->hw.idx));
303 }
304 
ali_drw_pmu_event_update(struct perf_event * event)305 static void ali_drw_pmu_event_update(struct perf_event *event)
306 {
307 	struct hw_perf_event *hwc = &event->hw;
308 	u64 delta, prev, now;
309 
310 	do {
311 		prev = local64_read(&hwc->prev_count);
312 		now = ali_drw_pmu_read_counter(event);
313 	} while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
314 
315 	/* handle overflow. */
316 	delta = now - prev;
317 	if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID)
318 		delta &= ALI_DRW_PMU_OV_INTR_MASK;
319 	else
320 		delta &= ALI_DRW_CNT_MAX_PERIOD;
321 	local64_add(delta, &event->count);
322 }
323 
ali_drw_pmu_event_set_period(struct perf_event * event)324 static void ali_drw_pmu_event_set_period(struct perf_event *event)
325 {
326 	u64 pre_val;
327 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
328 
329 	/* set a preload counter for test purpose */
330 	writel(ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE + event->hw.idx,
331 	       drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);
332 
333 	/* set conunter initial value */
334 	pre_val = ALI_DRW_PMU_CNT_INIT;
335 	writel(pre_val, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);
336 	local64_set(&event->hw.prev_count, pre_val);
337 
338 	/* set sel mode to zero to start test */
339 	writel(0x0, drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);
340 }
341 
ali_drw_pmu_enable_counter(struct perf_event * event)342 static void ali_drw_pmu_enable_counter(struct perf_event *event)
343 {
344 	u32 val, subval, reg, shift;
345 	int counter = event->hw.idx;
346 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
347 
348 	reg = ALI_DRW_PMU_EVENT_SELn(counter);
349 	val = readl(drw_pmu->cfg_base + reg);
350 	subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 1) |
351 		 FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, drw_pmu->evtids[counter]);
352 
353 	shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);
354 	val &= ~(GENMASK(7, 0) << shift);
355 	val |= subval << shift;
356 
357 	writel(val, drw_pmu->cfg_base + reg);
358 }
359 
ali_drw_pmu_disable_counter(struct perf_event * event)360 static void ali_drw_pmu_disable_counter(struct perf_event *event)
361 {
362 	u32 val, reg, subval, shift;
363 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
364 	int counter = event->hw.idx;
365 
366 	reg = ALI_DRW_PMU_EVENT_SELn(counter);
367 	val = readl(drw_pmu->cfg_base + reg);
368 	subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 0) |
369 		 FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, 0);
370 
371 	shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);
372 	val &= ~(GENMASK(7, 0) << shift);
373 	val |= subval << shift;
374 
375 	writel(val, drw_pmu->cfg_base + reg);
376 }
377 
ali_drw_pmu_isr(int irq_num,void * data)378 static irqreturn_t ali_drw_pmu_isr(int irq_num, void *data)
379 {
380 	struct ali_drw_pmu_irq *irq = data;
381 	struct ali_drw_pmu *drw_pmu;
382 	irqreturn_t ret = IRQ_NONE;
383 
384 	rcu_read_lock();
385 	list_for_each_entry_rcu(drw_pmu, &irq->pmus_node, pmus_node) {
386 		unsigned long status, clr_status;
387 		struct perf_event *event;
388 		unsigned int idx;
389 
390 		for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {
391 			event = drw_pmu->events[idx];
392 			if (!event)
393 				continue;
394 			ali_drw_pmu_disable_counter(event);
395 		}
396 
397 		/* common counter intr status */
398 		status = readl(drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_STATUS);
399 		status = FIELD_GET(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status);
400 		if (status) {
401 			for_each_set_bit(idx, &status,
402 					 ALI_DRW_PMU_COMMON_MAX_COUNTERS) {
403 				event = drw_pmu->events[idx];
404 				if (WARN_ON_ONCE(!event))
405 					continue;
406 				ali_drw_pmu_event_update(event);
407 				ali_drw_pmu_event_set_period(event);
408 			}
409 
410 			/* clear common counter intr status */
411 			clr_status = FIELD_PREP(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, 1);
412 			writel(clr_status,
413 			       drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);
414 		}
415 
416 		for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {
417 			event = drw_pmu->events[idx];
418 			if (!event)
419 				continue;
420 			if (!(event->hw.state & PERF_HES_STOPPED))
421 				ali_drw_pmu_enable_counter(event);
422 		}
423 		if (status)
424 			ret = IRQ_HANDLED;
425 	}
426 	rcu_read_unlock();
427 	return ret;
428 }
429 
__ali_drw_pmu_init_irq(struct platform_device * pdev,int irq_num)430 static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device
431 						      *pdev, int irq_num)
432 {
433 	int ret;
434 	struct ali_drw_pmu_irq *irq;
435 
436 	list_for_each_entry(irq, &ali_drw_pmu_irqs, irqs_node) {
437 		if (irq->irq_num == irq_num
438 		    && refcount_inc_not_zero(&irq->refcount))
439 			return irq;
440 	}
441 
442 	irq = kzalloc(sizeof(*irq), GFP_KERNEL);
443 	if (!irq)
444 		return ERR_PTR(-ENOMEM);
445 
446 	INIT_LIST_HEAD(&irq->pmus_node);
447 
448 	/* Pick one CPU to be the preferred one to use */
449 	irq->cpu = smp_processor_id();
450 	refcount_set(&irq->refcount, 1);
451 
452 	/*
453 	 * FIXME: one of DDRSS Driveway PMU overflow interrupt shares the same
454 	 * irq number with MPAM ERR_IRQ. To register DDRSS PMU and MPAM drivers
455 	 * successfully, add IRQF_SHARED flag. Howerer, PMU interrupt should not
456 	 * share with other component.
457 	 */
458 	ret = devm_request_irq(&pdev->dev, irq_num, ali_drw_pmu_isr,
459 			       IRQF_SHARED, dev_name(&pdev->dev), irq);
460 	if (ret < 0) {
461 		dev_err(&pdev->dev,
462 			"Fail to request IRQ:%d ret:%d\n", irq_num, ret);
463 		goto out_free;
464 	}
465 
466 	ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu));
467 	if (ret)
468 		goto out_free;
469 
470 	ret = cpuhp_state_add_instance_nocalls(ali_drw_cpuhp_state_num,
471 					     &irq->node);
472 	if (ret)
473 		goto out_free;
474 
475 	irq->irq_num = irq_num;
476 	list_add(&irq->irqs_node, &ali_drw_pmu_irqs);
477 
478 	return irq;
479 
480 out_free:
481 	kfree(irq);
482 	return ERR_PTR(ret);
483 }
484 
ali_drw_pmu_init_irq(struct ali_drw_pmu * drw_pmu,struct platform_device * pdev)485 static int ali_drw_pmu_init_irq(struct ali_drw_pmu *drw_pmu,
486 				struct platform_device *pdev)
487 {
488 	int irq_num;
489 	struct ali_drw_pmu_irq *irq;
490 
491 	/* Read and init IRQ */
492 	irq_num = platform_get_irq(pdev, 0);
493 	if (irq_num < 0)
494 		return irq_num;
495 
496 	mutex_lock(&ali_drw_pmu_irqs_lock);
497 	irq = __ali_drw_pmu_init_irq(pdev, irq_num);
498 	mutex_unlock(&ali_drw_pmu_irqs_lock);
499 
500 	if (IS_ERR(irq))
501 		return PTR_ERR(irq);
502 
503 	drw_pmu->irq = irq;
504 
505 	mutex_lock(&ali_drw_pmu_irqs_lock);
506 	list_add_rcu(&drw_pmu->pmus_node, &irq->pmus_node);
507 	mutex_unlock(&ali_drw_pmu_irqs_lock);
508 
509 	return 0;
510 }
511 
ali_drw_pmu_uninit_irq(struct ali_drw_pmu * drw_pmu)512 static void ali_drw_pmu_uninit_irq(struct ali_drw_pmu *drw_pmu)
513 {
514 	struct ali_drw_pmu_irq *irq = drw_pmu->irq;
515 
516 	mutex_lock(&ali_drw_pmu_irqs_lock);
517 	list_del_rcu(&drw_pmu->pmus_node);
518 
519 	if (!refcount_dec_and_test(&irq->refcount)) {
520 		mutex_unlock(&ali_drw_pmu_irqs_lock);
521 		return;
522 	}
523 
524 	list_del(&irq->irqs_node);
525 	mutex_unlock(&ali_drw_pmu_irqs_lock);
526 
527 	WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL));
528 	cpuhp_state_remove_instance_nocalls(ali_drw_cpuhp_state_num,
529 					    &irq->node);
530 	kfree(irq);
531 }
532 
ali_drw_pmu_event_init(struct perf_event * event)533 static int ali_drw_pmu_event_init(struct perf_event *event)
534 {
535 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
536 	struct hw_perf_event *hwc = &event->hw;
537 	struct perf_event *sibling;
538 	struct device *dev = drw_pmu->pmu.dev;
539 
540 	if (event->attr.type != event->pmu->type)
541 		return -ENOENT;
542 
543 	if (is_sampling_event(event)) {
544 		dev_err(dev, "Sampling not supported!\n");
545 		return -EOPNOTSUPP;
546 	}
547 
548 	if (event->attach_state & PERF_ATTACH_TASK) {
549 		dev_err(dev, "Per-task counter cannot allocate!\n");
550 		return -EOPNOTSUPP;
551 	}
552 
553 	event->cpu = drw_pmu->cpu;
554 	if (event->cpu < 0) {
555 		dev_err(dev, "Per-task mode not supported!\n");
556 		return -EOPNOTSUPP;
557 	}
558 
559 	if (event->group_leader != event &&
560 	    !is_software_event(event->group_leader)) {
561 		dev_err(dev, "driveway only allow one event!\n");
562 		return -EINVAL;
563 	}
564 
565 	for_each_sibling_event(sibling, event->group_leader) {
566 		if (sibling != event && !is_software_event(sibling)) {
567 			dev_err(dev, "driveway event not allowed!\n");
568 			return -EINVAL;
569 		}
570 	}
571 
572 	/* reset all the pmu counters */
573 	writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
574 
575 	hwc->idx = -1;
576 
577 	return 0;
578 }
579 
ali_drw_pmu_start(struct perf_event * event,int flags)580 static void ali_drw_pmu_start(struct perf_event *event, int flags)
581 {
582 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
583 
584 	event->hw.state = 0;
585 
586 	if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {
587 		writel(ALI_DRW_PMU_CNT_START,
588 		       drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
589 		return;
590 	}
591 
592 	ali_drw_pmu_event_set_period(event);
593 	if (flags & PERF_EF_RELOAD) {
594 		unsigned long prev_raw_count =
595 		    local64_read(&event->hw.prev_count);
596 		writel(prev_raw_count,
597 		       drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);
598 	}
599 
600 	ali_drw_pmu_enable_counter(event);
601 
602 	writel(ALI_DRW_PMU_CNT_START, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
603 }
604 
ali_drw_pmu_stop(struct perf_event * event,int flags)605 static void ali_drw_pmu_stop(struct perf_event *event, int flags)
606 {
607 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
608 
609 	if (event->hw.state & PERF_HES_STOPPED)
610 		return;
611 
612 	if (GET_DRW_EVENTID(event) != ALI_DRW_PMU_CYCLE_EVT_ID)
613 		ali_drw_pmu_disable_counter(event);
614 
615 	writel(ALI_DRW_PMU_CNT_STOP, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
616 
617 	ali_drw_pmu_event_update(event);
618 	event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
619 }
620 
ali_drw_pmu_add(struct perf_event * event,int flags)621 static int ali_drw_pmu_add(struct perf_event *event, int flags)
622 {
623 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
624 	struct hw_perf_event *hwc = &event->hw;
625 	int idx = -1;
626 	int evtid;
627 
628 	evtid = GET_DRW_EVENTID(event);
629 
630 	if (evtid != ALI_DRW_PMU_CYCLE_EVT_ID) {
631 		idx = ali_drw_get_counter_idx(event);
632 		if (idx < 0)
633 			return idx;
634 		drw_pmu->events[idx] = event;
635 		drw_pmu->evtids[idx] = evtid;
636 	}
637 	hwc->idx = idx;
638 
639 	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
640 
641 	if (flags & PERF_EF_START)
642 		ali_drw_pmu_start(event, PERF_EF_RELOAD);
643 
644 	/* Propagate our changes to the userspace mapping. */
645 	perf_event_update_userpage(event);
646 
647 	return 0;
648 }
649 
ali_drw_pmu_del(struct perf_event * event,int flags)650 static void ali_drw_pmu_del(struct perf_event *event, int flags)
651 {
652 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
653 	struct hw_perf_event *hwc = &event->hw;
654 	int idx = hwc->idx;
655 
656 	ali_drw_pmu_stop(event, PERF_EF_UPDATE);
657 
658 	if (idx >= 0 && idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS) {
659 		drw_pmu->events[idx] = NULL;
660 		drw_pmu->evtids[idx] = 0;
661 		clear_bit(idx, drw_pmu->used_mask);
662 	}
663 
664 	perf_event_update_userpage(event);
665 }
666 
ali_drw_pmu_read(struct perf_event * event)667 static void ali_drw_pmu_read(struct perf_event *event)
668 {
669 	ali_drw_pmu_event_update(event);
670 }
671 
ali_drw_pmu_probe(struct platform_device * pdev)672 static int ali_drw_pmu_probe(struct platform_device *pdev)
673 {
674 	struct ali_drw_pmu *drw_pmu;
675 	struct resource *res;
676 	char *name;
677 	int ret;
678 
679 	drw_pmu = devm_kzalloc(&pdev->dev, sizeof(*drw_pmu), GFP_KERNEL);
680 	if (!drw_pmu)
681 		return -ENOMEM;
682 
683 	drw_pmu->dev = &pdev->dev;
684 	platform_set_drvdata(pdev, drw_pmu);
685 
686 	drw_pmu->cfg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
687 	if (IS_ERR(drw_pmu->cfg_base))
688 		return PTR_ERR(drw_pmu->cfg_base);
689 
690 	name = devm_kasprintf(drw_pmu->dev, GFP_KERNEL, "ali_drw_%llx",
691 			      (u64) (res->start >> ALI_DRW_PMU_PA_SHIFT));
692 	if (!name)
693 		return -ENOMEM;
694 
695 	writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
696 
697 	/* enable the generation of interrupt by all common counters */
698 	writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,
699 	       drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_ENABLE_CTL);
700 
701 	/* clearing interrupt status */
702 	writel(0xffffff, drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);
703 
704 	drw_pmu->cpu = smp_processor_id();
705 
706 	ret = ali_drw_pmu_init_irq(drw_pmu, pdev);
707 	if (ret)
708 		return ret;
709 
710 	drw_pmu->pmu = (struct pmu) {
711 		.module		= THIS_MODULE,
712 		.task_ctx_nr	= perf_invalid_context,
713 		.event_init	= ali_drw_pmu_event_init,
714 		.add		= ali_drw_pmu_add,
715 		.del		= ali_drw_pmu_del,
716 		.start		= ali_drw_pmu_start,
717 		.stop		= ali_drw_pmu_stop,
718 		.read		= ali_drw_pmu_read,
719 		.attr_groups	= ali_drw_pmu_attr_groups,
720 		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
721 	};
722 
723 	ret = perf_pmu_register(&drw_pmu->pmu, name, -1);
724 	if (ret) {
725 		dev_err(drw_pmu->dev, "DRW Driveway PMU PMU register failed!\n");
726 		ali_drw_pmu_uninit_irq(drw_pmu);
727 	}
728 
729 	return ret;
730 }
731 
ali_drw_pmu_remove(struct platform_device * pdev)732 static int ali_drw_pmu_remove(struct platform_device *pdev)
733 {
734 	struct ali_drw_pmu *drw_pmu = platform_get_drvdata(pdev);
735 
736 	/* disable the generation of interrupt by all common counters */
737 	writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,
738 	       drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_DISABLE_CTL);
739 
740 	ali_drw_pmu_uninit_irq(drw_pmu);
741 	perf_pmu_unregister(&drw_pmu->pmu);
742 
743 	return 0;
744 }
745 
ali_drw_pmu_offline_cpu(unsigned int cpu,struct hlist_node * node)746 static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
747 {
748 	struct ali_drw_pmu_irq *irq;
749 	struct ali_drw_pmu *drw_pmu;
750 	unsigned int target;
751 	int ret;
752 	cpumask_t node_online_cpus;
753 
754 	irq = hlist_entry_safe(node, struct ali_drw_pmu_irq, node);
755 	if (cpu != irq->cpu)
756 		return 0;
757 
758 	ret = cpumask_and(&node_online_cpus,
759 			  cpumask_of_node(cpu_to_node(cpu)), cpu_online_mask);
760 	if (ret)
761 		target = cpumask_any_but(&node_online_cpus, cpu);
762 	else
763 		target = cpumask_any_but(cpu_online_mask, cpu);
764 
765 	if (target >= nr_cpu_ids)
766 		return 0;
767 
768 	/* We're only reading, but this isn't the place to be involving RCU */
769 	mutex_lock(&ali_drw_pmu_irqs_lock);
770 	list_for_each_entry(drw_pmu, &irq->pmus_node, pmus_node)
771 		perf_pmu_migrate_context(&drw_pmu->pmu, irq->cpu, target);
772 	mutex_unlock(&ali_drw_pmu_irqs_lock);
773 
774 	WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target)));
775 	irq->cpu = target;
776 
777 	return 0;
778 }
779 
780 /*
781  * Due to historical reasons, the HID used in the production environment is
782  * ARMHD700, so we leave ARMHD700 as Compatible ID.
783  */
784 static const struct acpi_device_id ali_drw_acpi_match[] = {
785 	{"BABA5000", 0},
786 	{"ARMHD700", 0},
787 	{}
788 };
789 
790 MODULE_DEVICE_TABLE(acpi, ali_drw_acpi_match);
791 
792 static struct platform_driver ali_drw_pmu_driver = {
793 	.driver = {
794 		   .name = "ali_drw_pmu",
795 		   .acpi_match_table = ali_drw_acpi_match,
796 		   },
797 	.probe = ali_drw_pmu_probe,
798 	.remove = ali_drw_pmu_remove,
799 };
800 
ali_drw_pmu_init(void)801 static int __init ali_drw_pmu_init(void)
802 {
803 	int ret;
804 
805 	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
806 				      "ali_drw_pmu:online",
807 				      NULL, ali_drw_pmu_offline_cpu);
808 
809 	if (ret < 0) {
810 		pr_err("DRW Driveway PMU: setup hotplug failed, ret = %d\n",
811 		       ret);
812 		return ret;
813 	}
814 	ali_drw_cpuhp_state_num = ret;
815 
816 	ret = platform_driver_register(&ali_drw_pmu_driver);
817 	if (ret)
818 		cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);
819 
820 	return ret;
821 }
822 
ali_drw_pmu_exit(void)823 static void __exit ali_drw_pmu_exit(void)
824 {
825 	platform_driver_unregister(&ali_drw_pmu_driver);
826 	cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);
827 }
828 
829 module_init(ali_drw_pmu_init);
830 module_exit(ali_drw_pmu_exit);
831 
832 MODULE_AUTHOR("Hongbo Yao <yaohongbo@linux.alibaba.com>");
833 MODULE_AUTHOR("Neng Chen <nengchen@linux.alibaba.com>");
834 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>");
835 MODULE_DESCRIPTION("Alibaba DDR Sub-System Driveway PMU driver");
836 MODULE_LICENSE("GPL v2");
837