1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Alibaba DDR Sub-System Driveway PMU driver
4  *
5  * Copyright (C) 2022 Alibaba Inc
6  */
7 
8 #define ALI_DRW_PMUNAME		"ali_drw"
9 #define ALI_DRW_DRVNAME		ALI_DRW_PMUNAME "_pmu"
10 #define pr_fmt(fmt)		ALI_DRW_DRVNAME ": " fmt
11 
12 #include <linux/acpi.h>
13 #include <linux/bitfield.h>
14 #include <linux/bitmap.h>
15 #include <linux/bitops.h>
16 #include <linux/cpuhotplug.h>
17 #include <linux/cpumask.h>
18 #include <linux/device.h>
19 #include <linux/errno.h>
20 #include <linux/interrupt.h>
21 #include <linux/irq.h>
22 #include <linux/kernel.h>
23 #include <linux/list.h>
24 #include <linux/module.h>
25 #include <linux/mutex.h>
26 #include <linux/perf_event.h>
27 #include <linux/platform_device.h>
28 #include <linux/printk.h>
29 #include <linux/rculist.h>
30 #include <linux/refcount.h>
31 
32 
33 #define ALI_DRW_PMU_COMMON_MAX_COUNTERS			16
34 #define ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE	19
35 
36 #define ALI_DRW_PMU_PA_SHIFT			12
37 #define ALI_DRW_PMU_CNT_INIT			0x00000000
38 #define ALI_DRW_CNT_MAX_PERIOD			0xffffffff
39 #define ALI_DRW_PMU_CYCLE_EVT_ID		0x80
40 
41 #define ALI_DRW_PMU_CNT_CTRL			0xC00
42 #define ALI_DRW_PMU_CNT_RST			BIT(2)
43 #define ALI_DRW_PMU_CNT_STOP			BIT(1)
44 #define ALI_DRW_PMU_CNT_START			BIT(0)
45 
46 #define ALI_DRW_PMU_CNT_STATE			0xC04
47 #define ALI_DRW_PMU_TEST_CTRL			0xC08
48 #define ALI_DRW_PMU_CNT_PRELOAD			0xC0C
49 
50 #define ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK		GENMASK(23, 0)
51 #define ALI_DRW_PMU_CYCLE_CNT_LOW_MASK		GENMASK(31, 0)
52 #define ALI_DRW_PMU_CYCLE_CNT_HIGH		0xC10
53 #define ALI_DRW_PMU_CYCLE_CNT_LOW		0xC14
54 
55 /* PMU EVENT SEL 0-3 are paired in 32-bit registers on a 4-byte stride */
56 #define ALI_DRW_PMU_EVENT_SEL0			0xC68
57 /* counter 0-3 use sel0, counter 4-7 use sel1...*/
58 #define ALI_DRW_PMU_EVENT_SELn(n) \
59 	(ALI_DRW_PMU_EVENT_SEL0 + (n / 4) * 0x4)
60 #define ALI_DRW_PMCOM_CNT_EN			BIT(7)
61 #define ALI_DRW_PMCOM_CNT_EVENT_MASK		GENMASK(5, 0)
62 #define ALI_DRW_PMCOM_CNT_EVENT_OFFSET(n) \
63 	(8 * (n % 4))
64 
65 /* PMU COMMON COUNTER 0-15, are paired in 32-bit registers on a 4-byte stride */
66 #define ALI_DRW_PMU_COMMON_COUNTER0		0xC78
67 #define ALI_DRW_PMU_COMMON_COUNTERn(n) \
68 	(ALI_DRW_PMU_COMMON_COUNTER0 + 0x4 * (n))
69 
70 #define ALI_DRW_PMU_OV_INTR_ENABLE_CTL		0xCB8
71 #define ALI_DRW_PMU_OV_INTR_DISABLE_CTL		0xCBC
72 #define ALI_DRW_PMU_OV_INTR_ENABLE_STATUS	0xCC0
73 #define ALI_DRW_PMU_OV_INTR_CLR			0xCC4
74 #define ALI_DRW_PMU_OV_INTR_STATUS		0xCC8
75 #define ALI_DRW_PMCOM_CNT_OV_INTR_MASK		GENMASK(23, 8)
76 #define ALI_DRW_PMBW_CNT_OV_INTR_MASK		GENMASK(7, 0)
77 #define ALI_DRW_PMU_OV_INTR_MASK		GENMASK_ULL(63, 0)
78 
79 static int ali_drw_cpuhp_state_num;
80 
81 static LIST_HEAD(ali_drw_pmu_irqs);
82 static DEFINE_MUTEX(ali_drw_pmu_irqs_lock);
83 
84 struct ali_drw_pmu_irq {
85 	struct hlist_node node;
86 	struct list_head irqs_node;
87 	struct list_head pmus_node;
88 	int irq_num;
89 	int cpu;
90 	refcount_t refcount;
91 };
92 
93 struct ali_drw_pmu {
94 	void __iomem *cfg_base;
95 	struct device *dev;
96 
97 	struct list_head pmus_node;
98 	struct ali_drw_pmu_irq *irq;
99 	int irq_num;
100 	int cpu;
101 	DECLARE_BITMAP(used_mask, ALI_DRW_PMU_COMMON_MAX_COUNTERS);
102 	struct perf_event *events[ALI_DRW_PMU_COMMON_MAX_COUNTERS];
103 	int evtids[ALI_DRW_PMU_COMMON_MAX_COUNTERS];
104 
105 	struct pmu pmu;
106 };
107 
108 #define to_ali_drw_pmu(p) (container_of(p, struct ali_drw_pmu, pmu))
109 
110 #define DRW_CONFIG_EVENTID		GENMASK(7, 0)
111 #define GET_DRW_EVENTID(event)	FIELD_GET(DRW_CONFIG_EVENTID, (event)->attr.config)
112 
113 static ssize_t ali_drw_pmu_format_show(struct device *dev,
114 				struct device_attribute *attr, char *buf)
115 {
116 	struct dev_ext_attribute *eattr;
117 
118 	eattr = container_of(attr, struct dev_ext_attribute, attr);
119 
120 	return sprintf(buf, "%s\n", (char *)eattr->var);
121 }
122 
123 /*
124  * PMU event attributes
125  */
126 static ssize_t ali_drw_pmu_event_show(struct device *dev,
127 			       struct device_attribute *attr, char *page)
128 {
129 	struct dev_ext_attribute *eattr;
130 
131 	eattr = container_of(attr, struct dev_ext_attribute, attr);
132 
133 	return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);
134 }
135 
136 #define ALI_DRW_PMU_ATTR(_name, _func, _config)                            \
137 		(&((struct dev_ext_attribute[]) {                               \
138 				{ __ATTR(_name, 0444, _func, NULL), (void *)_config }   \
139 		})[0].attr.attr)
140 
141 #define ALI_DRW_PMU_FORMAT_ATTR(_name, _config)            \
142 	ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_format_show, (void *)_config)
143 #define ALI_DRW_PMU_EVENT_ATTR(_name, _config)             \
144 	ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_event_show, (unsigned long)_config)
145 
146 static struct attribute *ali_drw_pmu_events_attrs[] = {
147 	ALI_DRW_PMU_EVENT_ATTR(hif_rd_or_wr,			0x0),
148 	ALI_DRW_PMU_EVENT_ATTR(hif_wr,				0x1),
149 	ALI_DRW_PMU_EVENT_ATTR(hif_rd,				0x2),
150 	ALI_DRW_PMU_EVENT_ATTR(hif_rmw,				0x3),
151 	ALI_DRW_PMU_EVENT_ATTR(hif_hi_pri_rd,			0x4),
152 	ALI_DRW_PMU_EVENT_ATTR(dfi_wr_data_cycles,		0x7),
153 	ALI_DRW_PMU_EVENT_ATTR(dfi_rd_data_cycles,		0x8),
154 	ALI_DRW_PMU_EVENT_ATTR(hpr_xact_when_critical,		0x9),
155 	ALI_DRW_PMU_EVENT_ATTR(lpr_xact_when_critical,		0xA),
156 	ALI_DRW_PMU_EVENT_ATTR(wr_xact_when_critical,		0xB),
157 	ALI_DRW_PMU_EVENT_ATTR(op_is_activate,			0xC),
158 	ALI_DRW_PMU_EVENT_ATTR(op_is_rd_or_wr,			0xD),
159 	ALI_DRW_PMU_EVENT_ATTR(op_is_rd_activate,		0xE),
160 	ALI_DRW_PMU_EVENT_ATTR(op_is_rd,			0xF),
161 	ALI_DRW_PMU_EVENT_ATTR(op_is_wr,			0x10),
162 	ALI_DRW_PMU_EVENT_ATTR(op_is_mwr,			0x11),
163 	ALI_DRW_PMU_EVENT_ATTR(op_is_precharge,			0x12),
164 	ALI_DRW_PMU_EVENT_ATTR(precharge_for_rdwr,		0x13),
165 	ALI_DRW_PMU_EVENT_ATTR(precharge_for_other,		0x14),
166 	ALI_DRW_PMU_EVENT_ATTR(rdwr_transitions,		0x15),
167 	ALI_DRW_PMU_EVENT_ATTR(write_combine,			0x16),
168 	ALI_DRW_PMU_EVENT_ATTR(war_hazard,			0x17),
169 	ALI_DRW_PMU_EVENT_ATTR(raw_hazard,			0x18),
170 	ALI_DRW_PMU_EVENT_ATTR(waw_hazard,			0x19),
171 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk0,		0x1A),
172 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk1,		0x1B),
173 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk2,		0x1C),
174 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk3,		0x1D),
175 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk0,	0x1E),
176 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk1,	0x1F),
177 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk2,	0x20),
178 	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk3,	0x21),
179 	ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk0,		0x26),
180 	ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk1,		0x27),
181 	ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk2,		0x28),
182 	ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk3,		0x29),
183 	ALI_DRW_PMU_EVENT_ATTR(op_is_refresh,			0x2A),
184 	ALI_DRW_PMU_EVENT_ATTR(op_is_crit_ref,			0x2B),
185 	ALI_DRW_PMU_EVENT_ATTR(op_is_load_mode,			0x2D),
186 	ALI_DRW_PMU_EVENT_ATTR(op_is_zqcl,			0x2E),
187 	ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_rd, 0x30),
188 	ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_wr, 0x31),
189 	ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mpc,		0x34),
190 	ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mrr,		0x35),
191 	ALI_DRW_PMU_EVENT_ATTR(op_is_tcr_mrr,			0x36),
192 	ALI_DRW_PMU_EVENT_ATTR(op_is_zqstart,			0x37),
193 	ALI_DRW_PMU_EVENT_ATTR(op_is_zqlatch,			0x38),
194 	ALI_DRW_PMU_EVENT_ATTR(chi_txreq,			0x39),
195 	ALI_DRW_PMU_EVENT_ATTR(chi_txdat,			0x3A),
196 	ALI_DRW_PMU_EVENT_ATTR(chi_rxdat,			0x3B),
197 	ALI_DRW_PMU_EVENT_ATTR(chi_rxrsp,			0x3C),
198 	ALI_DRW_PMU_EVENT_ATTR(tsz_vio,				0x3D),
199 	ALI_DRW_PMU_EVENT_ATTR(cycle,				0x80),
200 	NULL,
201 };
202 
203 static struct attribute_group ali_drw_pmu_events_attr_group = {
204 	.name = "events",
205 	.attrs = ali_drw_pmu_events_attrs,
206 };
207 
208 static struct attribute *ali_drw_pmu_format_attr[] = {
209 	ALI_DRW_PMU_FORMAT_ATTR(event, "config:0-7"),
210 	NULL,
211 };
212 
213 static const struct attribute_group ali_drw_pmu_format_group = {
214 	.name = "format",
215 	.attrs = ali_drw_pmu_format_attr,
216 };
217 
218 static ssize_t ali_drw_pmu_cpumask_show(struct device *dev,
219 					struct device_attribute *attr,
220 					char *buf)
221 {
222 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(dev_get_drvdata(dev));
223 
224 	return cpumap_print_to_pagebuf(true, buf, cpumask_of(drw_pmu->cpu));
225 }
226 
227 static struct device_attribute ali_drw_pmu_cpumask_attr =
228 		__ATTR(cpumask, 0444, ali_drw_pmu_cpumask_show, NULL);
229 
230 static struct attribute *ali_drw_pmu_cpumask_attrs[] = {
231 	&ali_drw_pmu_cpumask_attr.attr,
232 	NULL,
233 };
234 
235 static const struct attribute_group ali_drw_pmu_cpumask_attr_group = {
236 	.attrs = ali_drw_pmu_cpumask_attrs,
237 };
238 
239 static const struct attribute_group *ali_drw_pmu_attr_groups[] = {
240 	&ali_drw_pmu_events_attr_group,
241 	&ali_drw_pmu_cpumask_attr_group,
242 	&ali_drw_pmu_format_group,
243 	NULL,
244 };
245 
246 /* find a counter for event, then in add func, hw.idx will equal to counter */
247 static int ali_drw_get_counter_idx(struct perf_event *event)
248 {
249 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
250 	int idx;
251 
252 	for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; ++idx) {
253 		if (!test_and_set_bit(idx, drw_pmu->used_mask))
254 			return idx;
255 	}
256 
257 	/* The counters are all in use. */
258 	return -EBUSY;
259 }
260 
261 static u64 ali_drw_pmu_read_counter(struct perf_event *event)
262 {
263 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
264 	u64 cycle_high, cycle_low;
265 
266 	if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {
267 		cycle_high = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_HIGH);
268 		cycle_high &= ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK;
269 		cycle_low = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_LOW);
270 		cycle_low &= ALI_DRW_PMU_CYCLE_CNT_LOW_MASK;
271 		return (cycle_high << 32 | cycle_low);
272 	}
273 
274 	return readl(drw_pmu->cfg_base +
275 		     ALI_DRW_PMU_COMMON_COUNTERn(event->hw.idx));
276 }
277 
278 static void ali_drw_pmu_event_update(struct perf_event *event)
279 {
280 	struct hw_perf_event *hwc = &event->hw;
281 	u64 delta, prev, now;
282 
283 	do {
284 		prev = local64_read(&hwc->prev_count);
285 		now = ali_drw_pmu_read_counter(event);
286 	} while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
287 
288 	/* handle overflow. */
289 	delta = now - prev;
290 	if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID)
291 		delta &= ALI_DRW_PMU_OV_INTR_MASK;
292 	else
293 		delta &= ALI_DRW_CNT_MAX_PERIOD;
294 	local64_add(delta, &event->count);
295 }
296 
297 static void ali_drw_pmu_event_set_period(struct perf_event *event)
298 {
299 	u64 pre_val;
300 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
301 
302 	/* set a preload counter for test purpose */
303 	writel(ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE + event->hw.idx,
304 	       drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);
305 
306 	/* set conunter initial value */
307 	pre_val = ALI_DRW_PMU_CNT_INIT;
308 	writel(pre_val, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);
309 	local64_set(&event->hw.prev_count, pre_val);
310 
311 	/* set sel mode to zero to start test */
312 	writel(0x0, drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);
313 }
314 
315 static void ali_drw_pmu_enable_counter(struct perf_event *event)
316 {
317 	u32 val, subval, reg, shift;
318 	int counter = event->hw.idx;
319 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
320 
321 	reg = ALI_DRW_PMU_EVENT_SELn(counter);
322 	val = readl(drw_pmu->cfg_base + reg);
323 	subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 1) |
324 		 FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, drw_pmu->evtids[counter]);
325 
326 	shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);
327 	val &= ~(GENMASK(7, 0) << shift);
328 	val |= subval << shift;
329 
330 	writel(val, drw_pmu->cfg_base + reg);
331 }
332 
333 static void ali_drw_pmu_disable_counter(struct perf_event *event)
334 {
335 	u32 val, reg, subval, shift;
336 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
337 	int counter = event->hw.idx;
338 
339 	reg = ALI_DRW_PMU_EVENT_SELn(counter);
340 	val = readl(drw_pmu->cfg_base + reg);
341 	subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 0) |
342 		 FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, 0);
343 
344 	shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);
345 	val &= ~(GENMASK(7, 0) << shift);
346 	val |= subval << shift;
347 
348 	writel(val, drw_pmu->cfg_base + reg);
349 }
350 
351 static irqreturn_t ali_drw_pmu_isr(int irq_num, void *data)
352 {
353 	struct ali_drw_pmu_irq *irq = data;
354 	struct ali_drw_pmu *drw_pmu;
355 	irqreturn_t ret = IRQ_NONE;
356 
357 	rcu_read_lock();
358 	list_for_each_entry_rcu(drw_pmu, &irq->pmus_node, pmus_node) {
359 		unsigned long status, clr_status;
360 		struct perf_event *event;
361 		unsigned int idx;
362 
363 		for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {
364 			event = drw_pmu->events[idx];
365 			if (!event)
366 				continue;
367 			ali_drw_pmu_disable_counter(event);
368 		}
369 
370 		/* common counter intr status */
371 		status = readl(drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_STATUS);
372 		status = FIELD_GET(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status);
373 		if (status) {
374 			for_each_set_bit(idx, &status,
375 					 ALI_DRW_PMU_COMMON_MAX_COUNTERS) {
376 				event = drw_pmu->events[idx];
377 				if (WARN_ON_ONCE(!event))
378 					continue;
379 				ali_drw_pmu_event_update(event);
380 				ali_drw_pmu_event_set_period(event);
381 			}
382 
383 			/* clear common counter intr status */
384 			clr_status = FIELD_PREP(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, 1);
385 			writel(clr_status,
386 			       drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);
387 		}
388 
389 		for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {
390 			event = drw_pmu->events[idx];
391 			if (!event)
392 				continue;
393 			if (!(event->hw.state & PERF_HES_STOPPED))
394 				ali_drw_pmu_enable_counter(event);
395 		}
396 		if (status)
397 			ret = IRQ_HANDLED;
398 	}
399 	rcu_read_unlock();
400 	return ret;
401 }
402 
403 static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device
404 						      *pdev, int irq_num)
405 {
406 	int ret;
407 	struct ali_drw_pmu_irq *irq;
408 
409 	list_for_each_entry(irq, &ali_drw_pmu_irqs, irqs_node) {
410 		if (irq->irq_num == irq_num
411 		    && refcount_inc_not_zero(&irq->refcount))
412 			return irq;
413 	}
414 
415 	irq = kzalloc(sizeof(*irq), GFP_KERNEL);
416 	if (!irq)
417 		return ERR_PTR(-ENOMEM);
418 
419 	INIT_LIST_HEAD(&irq->pmus_node);
420 
421 	/* Pick one CPU to be the preferred one to use */
422 	irq->cpu = smp_processor_id();
423 	refcount_set(&irq->refcount, 1);
424 
425 	/*
426 	 * FIXME: one of DDRSS Driveway PMU overflow interrupt shares the same
427 	 * irq number with MPAM ERR_IRQ. To register DDRSS PMU and MPAM drivers
428 	 * successfully, add IRQF_SHARED flag. Howerer, PMU interrupt should not
429 	 * share with other component.
430 	 */
431 	ret = devm_request_irq(&pdev->dev, irq_num, ali_drw_pmu_isr,
432 			       IRQF_SHARED, dev_name(&pdev->dev), irq);
433 	if (ret < 0) {
434 		dev_err(&pdev->dev,
435 			"Fail to request IRQ:%d ret:%d\n", irq_num, ret);
436 		goto out_free;
437 	}
438 
439 	ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu));
440 	if (ret)
441 		goto out_free;
442 
443 	ret = cpuhp_state_add_instance_nocalls(ali_drw_cpuhp_state_num,
444 					     &irq->node);
445 	if (ret)
446 		goto out_free;
447 
448 	irq->irq_num = irq_num;
449 	list_add(&irq->irqs_node, &ali_drw_pmu_irqs);
450 
451 	return irq;
452 
453 out_free:
454 	kfree(irq);
455 	return ERR_PTR(ret);
456 }
457 
458 static int ali_drw_pmu_init_irq(struct ali_drw_pmu *drw_pmu,
459 				struct platform_device *pdev)
460 {
461 	int irq_num;
462 	struct ali_drw_pmu_irq *irq;
463 
464 	/* Read and init IRQ */
465 	irq_num = platform_get_irq(pdev, 0);
466 	if (irq_num < 0)
467 		return irq_num;
468 
469 	mutex_lock(&ali_drw_pmu_irqs_lock);
470 	irq = __ali_drw_pmu_init_irq(pdev, irq_num);
471 	mutex_unlock(&ali_drw_pmu_irqs_lock);
472 
473 	if (IS_ERR(irq))
474 		return PTR_ERR(irq);
475 
476 	drw_pmu->irq = irq;
477 
478 	mutex_lock(&ali_drw_pmu_irqs_lock);
479 	list_add_rcu(&drw_pmu->pmus_node, &irq->pmus_node);
480 	mutex_unlock(&ali_drw_pmu_irqs_lock);
481 
482 	return 0;
483 }
484 
485 static void ali_drw_pmu_uninit_irq(struct ali_drw_pmu *drw_pmu)
486 {
487 	struct ali_drw_pmu_irq *irq = drw_pmu->irq;
488 
489 	mutex_lock(&ali_drw_pmu_irqs_lock);
490 	list_del_rcu(&drw_pmu->pmus_node);
491 
492 	if (!refcount_dec_and_test(&irq->refcount)) {
493 		mutex_unlock(&ali_drw_pmu_irqs_lock);
494 		return;
495 	}
496 
497 	list_del(&irq->irqs_node);
498 	mutex_unlock(&ali_drw_pmu_irqs_lock);
499 
500 	WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL));
501 	cpuhp_state_remove_instance_nocalls(ali_drw_cpuhp_state_num,
502 					    &irq->node);
503 	kfree(irq);
504 }
505 
506 static int ali_drw_pmu_event_init(struct perf_event *event)
507 {
508 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
509 	struct hw_perf_event *hwc = &event->hw;
510 	struct perf_event *sibling;
511 	struct device *dev = drw_pmu->pmu.dev;
512 
513 	if (event->attr.type != event->pmu->type)
514 		return -ENOENT;
515 
516 	if (is_sampling_event(event)) {
517 		dev_err(dev, "Sampling not supported!\n");
518 		return -EOPNOTSUPP;
519 	}
520 
521 	if (event->attach_state & PERF_ATTACH_TASK) {
522 		dev_err(dev, "Per-task counter cannot allocate!\n");
523 		return -EOPNOTSUPP;
524 	}
525 
526 	event->cpu = drw_pmu->cpu;
527 	if (event->cpu < 0) {
528 		dev_err(dev, "Per-task mode not supported!\n");
529 		return -EOPNOTSUPP;
530 	}
531 
532 	if (event->group_leader != event &&
533 	    !is_software_event(event->group_leader)) {
534 		dev_err(dev, "driveway only allow one event!\n");
535 		return -EINVAL;
536 	}
537 
538 	for_each_sibling_event(sibling, event->group_leader) {
539 		if (sibling != event && !is_software_event(sibling)) {
540 			dev_err(dev, "driveway event not allowed!\n");
541 			return -EINVAL;
542 		}
543 	}
544 
545 	/* reset all the pmu counters */
546 	writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
547 
548 	hwc->idx = -1;
549 
550 	return 0;
551 }
552 
553 static void ali_drw_pmu_start(struct perf_event *event, int flags)
554 {
555 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
556 
557 	event->hw.state = 0;
558 
559 	if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {
560 		writel(ALI_DRW_PMU_CNT_START,
561 		       drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
562 		return;
563 	}
564 
565 	ali_drw_pmu_event_set_period(event);
566 	if (flags & PERF_EF_RELOAD) {
567 		unsigned long prev_raw_count =
568 		    local64_read(&event->hw.prev_count);
569 		writel(prev_raw_count,
570 		       drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);
571 	}
572 
573 	ali_drw_pmu_enable_counter(event);
574 
575 	writel(ALI_DRW_PMU_CNT_START, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
576 }
577 
578 static void ali_drw_pmu_stop(struct perf_event *event, int flags)
579 {
580 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
581 
582 	if (event->hw.state & PERF_HES_STOPPED)
583 		return;
584 
585 	if (GET_DRW_EVENTID(event) != ALI_DRW_PMU_CYCLE_EVT_ID)
586 		ali_drw_pmu_disable_counter(event);
587 
588 	writel(ALI_DRW_PMU_CNT_STOP, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
589 
590 	ali_drw_pmu_event_update(event);
591 	event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
592 }
593 
594 static int ali_drw_pmu_add(struct perf_event *event, int flags)
595 {
596 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
597 	struct hw_perf_event *hwc = &event->hw;
598 	int idx = -1;
599 	int evtid;
600 
601 	evtid = GET_DRW_EVENTID(event);
602 
603 	if (evtid != ALI_DRW_PMU_CYCLE_EVT_ID) {
604 		idx = ali_drw_get_counter_idx(event);
605 		if (idx < 0)
606 			return idx;
607 		drw_pmu->events[idx] = event;
608 		drw_pmu->evtids[idx] = evtid;
609 	}
610 	hwc->idx = idx;
611 
612 	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
613 
614 	if (flags & PERF_EF_START)
615 		ali_drw_pmu_start(event, PERF_EF_RELOAD);
616 
617 	/* Propagate our changes to the userspace mapping. */
618 	perf_event_update_userpage(event);
619 
620 	return 0;
621 }
622 
623 static void ali_drw_pmu_del(struct perf_event *event, int flags)
624 {
625 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
626 	struct hw_perf_event *hwc = &event->hw;
627 	int idx = hwc->idx;
628 
629 	ali_drw_pmu_stop(event, PERF_EF_UPDATE);
630 
631 	if (idx >= 0 && idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS) {
632 		drw_pmu->events[idx] = NULL;
633 		drw_pmu->evtids[idx] = 0;
634 		clear_bit(idx, drw_pmu->used_mask);
635 	}
636 
637 	perf_event_update_userpage(event);
638 }
639 
640 static void ali_drw_pmu_read(struct perf_event *event)
641 {
642 	ali_drw_pmu_event_update(event);
643 }
644 
645 static int ali_drw_pmu_probe(struct platform_device *pdev)
646 {
647 	struct ali_drw_pmu *drw_pmu;
648 	struct resource *res;
649 	char *name;
650 	int ret;
651 
652 	drw_pmu = devm_kzalloc(&pdev->dev, sizeof(*drw_pmu), GFP_KERNEL);
653 	if (!drw_pmu)
654 		return -ENOMEM;
655 
656 	drw_pmu->dev = &pdev->dev;
657 	platform_set_drvdata(pdev, drw_pmu);
658 
659 	drw_pmu->cfg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
660 	if (IS_ERR(drw_pmu->cfg_base))
661 		return PTR_ERR(drw_pmu->cfg_base);
662 
663 	name = devm_kasprintf(drw_pmu->dev, GFP_KERNEL, "ali_drw_%llx",
664 			      (u64) (res->start >> ALI_DRW_PMU_PA_SHIFT));
665 	if (!name)
666 		return -ENOMEM;
667 
668 	writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
669 
670 	/* enable the generation of interrupt by all common counters */
671 	writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,
672 	       drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_ENABLE_CTL);
673 
674 	/* clearing interrupt status */
675 	writel(0xffffff, drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);
676 
677 	drw_pmu->cpu = smp_processor_id();
678 
679 	ret = ali_drw_pmu_init_irq(drw_pmu, pdev);
680 	if (ret)
681 		return ret;
682 
683 	drw_pmu->pmu = (struct pmu) {
684 		.module		= THIS_MODULE,
685 		.task_ctx_nr	= perf_invalid_context,
686 		.event_init	= ali_drw_pmu_event_init,
687 		.add		= ali_drw_pmu_add,
688 		.del		= ali_drw_pmu_del,
689 		.start		= ali_drw_pmu_start,
690 		.stop		= ali_drw_pmu_stop,
691 		.read		= ali_drw_pmu_read,
692 		.attr_groups	= ali_drw_pmu_attr_groups,
693 		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
694 	};
695 
696 	ret = perf_pmu_register(&drw_pmu->pmu, name, -1);
697 	if (ret) {
698 		dev_err(drw_pmu->dev, "DRW Driveway PMU PMU register failed!\n");
699 		ali_drw_pmu_uninit_irq(drw_pmu);
700 	}
701 
702 	return ret;
703 }
704 
705 static int ali_drw_pmu_remove(struct platform_device *pdev)
706 {
707 	struct ali_drw_pmu *drw_pmu = platform_get_drvdata(pdev);
708 
709 	/* disable the generation of interrupt by all common counters */
710 	writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,
711 	       drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_DISABLE_CTL);
712 
713 	ali_drw_pmu_uninit_irq(drw_pmu);
714 	perf_pmu_unregister(&drw_pmu->pmu);
715 
716 	return 0;
717 }
718 
719 static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
720 {
721 	struct ali_drw_pmu_irq *irq;
722 	struct ali_drw_pmu *drw_pmu;
723 	unsigned int target;
724 	int ret;
725 	cpumask_t node_online_cpus;
726 
727 	irq = hlist_entry_safe(node, struct ali_drw_pmu_irq, node);
728 	if (cpu != irq->cpu)
729 		return 0;
730 
731 	ret = cpumask_and(&node_online_cpus,
732 			  cpumask_of_node(cpu_to_node(cpu)), cpu_online_mask);
733 	if (ret)
734 		target = cpumask_any_but(&node_online_cpus, cpu);
735 	else
736 		target = cpumask_any_but(cpu_online_mask, cpu);
737 
738 	if (target >= nr_cpu_ids)
739 		return 0;
740 
741 	/* We're only reading, but this isn't the place to be involving RCU */
742 	mutex_lock(&ali_drw_pmu_irqs_lock);
743 	list_for_each_entry(drw_pmu, &irq->pmus_node, pmus_node)
744 		perf_pmu_migrate_context(&drw_pmu->pmu, irq->cpu, target);
745 	mutex_unlock(&ali_drw_pmu_irqs_lock);
746 
747 	WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target)));
748 	irq->cpu = target;
749 
750 	return 0;
751 }
752 
753 /*
754  * Due to historical reasons, the HID used in the production environment is
755  * ARMHD700, so we leave ARMHD700 as Compatible ID.
756  */
757 static const struct acpi_device_id ali_drw_acpi_match[] = {
758 	{"BABA5000", 0},
759 	{"ARMHD700", 0},
760 	{}
761 };
762 
763 MODULE_DEVICE_TABLE(acpi, ali_drw_acpi_match);
764 
765 static struct platform_driver ali_drw_pmu_driver = {
766 	.driver = {
767 		   .name = "ali_drw_pmu",
768 		   .acpi_match_table = ali_drw_acpi_match,
769 		   },
770 	.probe = ali_drw_pmu_probe,
771 	.remove = ali_drw_pmu_remove,
772 };
773 
774 static int __init ali_drw_pmu_init(void)
775 {
776 	int ret;
777 
778 	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
779 				      "ali_drw_pmu:online",
780 				      NULL, ali_drw_pmu_offline_cpu);
781 
782 	if (ret < 0) {
783 		pr_err("DRW Driveway PMU: setup hotplug failed, ret = %d\n",
784 		       ret);
785 		return ret;
786 	}
787 	ali_drw_cpuhp_state_num = ret;
788 
789 	ret = platform_driver_register(&ali_drw_pmu_driver);
790 	if (ret)
791 		cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);
792 
793 	return ret;
794 }
795 
796 static void __exit ali_drw_pmu_exit(void)
797 {
798 	platform_driver_unregister(&ali_drw_pmu_driver);
799 	cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);
800 }
801 
802 module_init(ali_drw_pmu_init);
803 module_exit(ali_drw_pmu_exit);
804 
805 MODULE_AUTHOR("Hongbo Yao <yaohongbo@linux.alibaba.com>");
806 MODULE_AUTHOR("Neng Chen <nengchen@linux.alibaba.com>");
807 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>");
808 MODULE_DESCRIPTION("Alibaba DDR Sub-System Driveway PMU driver");
809 MODULE_LICENSE("GPL v2");
810