xref: /openbmc/linux/drivers/iommu/intel/perfmon.c (revision f3ccbb6b)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Support Intel IOMMU PerfMon
4  * Copyright(c) 2023 Intel Corporation.
5  */
6 #define pr_fmt(fmt)	"DMAR: " fmt
7 #define dev_fmt(fmt)	pr_fmt(fmt)
8 
9 #include <linux/dmar.h>
10 #include "iommu.h"
11 #include "perfmon.h"
12 
13 PMU_FORMAT_ATTR(event,		"config:0-27");		/* ES: Events Select */
14 PMU_FORMAT_ATTR(event_group,	"config:28-31");	/* EGI: Event Group Index */
15 
16 static struct attribute *iommu_pmu_format_attrs[] = {
17 	&format_attr_event_group.attr,
18 	&format_attr_event.attr,
19 	NULL
20 };
21 
22 static struct attribute_group iommu_pmu_format_attr_group = {
23 	.name = "format",
24 	.attrs = iommu_pmu_format_attrs,
25 };
26 
27 /* The available events are added in attr_update later */
28 static struct attribute *attrs_empty[] = {
29 	NULL
30 };
31 
32 static struct attribute_group iommu_pmu_events_attr_group = {
33 	.name = "events",
34 	.attrs = attrs_empty,
35 };
36 
37 static cpumask_t iommu_pmu_cpu_mask;
38 
39 static ssize_t
cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)40 cpumask_show(struct device *dev, struct device_attribute *attr, char *buf)
41 {
42 	return cpumap_print_to_pagebuf(true, buf, &iommu_pmu_cpu_mask);
43 }
44 static DEVICE_ATTR_RO(cpumask);
45 
46 static struct attribute *iommu_pmu_cpumask_attrs[] = {
47 	&dev_attr_cpumask.attr,
48 	NULL
49 };
50 
51 static struct attribute_group iommu_pmu_cpumask_attr_group = {
52 	.attrs = iommu_pmu_cpumask_attrs,
53 };
54 
55 static const struct attribute_group *iommu_pmu_attr_groups[] = {
56 	&iommu_pmu_format_attr_group,
57 	&iommu_pmu_events_attr_group,
58 	&iommu_pmu_cpumask_attr_group,
59 	NULL
60 };
61 
dev_to_iommu_pmu(struct device * dev)62 static inline struct iommu_pmu *dev_to_iommu_pmu(struct device *dev)
63 {
64 	/*
65 	 * The perf_event creates its own dev for each PMU.
66 	 * See pmu_dev_alloc()
67 	 */
68 	return container_of(dev_get_drvdata(dev), struct iommu_pmu, pmu);
69 }
70 
71 #define IOMMU_PMU_ATTR(_name, _format, _filter)				\
72 	PMU_FORMAT_ATTR(_name, _format);				\
73 									\
74 static struct attribute *_name##_attr[] = {				\
75 	&format_attr_##_name.attr,					\
76 	NULL								\
77 };									\
78 									\
79 static umode_t								\
80 _name##_is_visible(struct kobject *kobj, struct attribute *attr, int i)	\
81 {									\
82 	struct device *dev = kobj_to_dev(kobj);				\
83 	struct iommu_pmu *iommu_pmu = dev_to_iommu_pmu(dev);		\
84 									\
85 	if (!iommu_pmu)							\
86 		return 0;						\
87 	return (iommu_pmu->filter & _filter) ? attr->mode : 0;		\
88 }									\
89 									\
90 static struct attribute_group _name = {					\
91 	.name		= "format",					\
92 	.attrs		= _name##_attr,					\
93 	.is_visible	= _name##_is_visible,				\
94 };
95 
96 IOMMU_PMU_ATTR(filter_requester_id_en,	"config1:0",		IOMMU_PMU_FILTER_REQUESTER_ID);
97 IOMMU_PMU_ATTR(filter_domain_en,	"config1:1",		IOMMU_PMU_FILTER_DOMAIN);
98 IOMMU_PMU_ATTR(filter_pasid_en,		"config1:2",		IOMMU_PMU_FILTER_PASID);
99 IOMMU_PMU_ATTR(filter_ats_en,		"config1:3",		IOMMU_PMU_FILTER_ATS);
100 IOMMU_PMU_ATTR(filter_page_table_en,	"config1:4",		IOMMU_PMU_FILTER_PAGE_TABLE);
101 IOMMU_PMU_ATTR(filter_requester_id,	"config1:16-31",	IOMMU_PMU_FILTER_REQUESTER_ID);
102 IOMMU_PMU_ATTR(filter_domain,		"config1:32-47",	IOMMU_PMU_FILTER_DOMAIN);
103 IOMMU_PMU_ATTR(filter_pasid,		"config2:0-21",		IOMMU_PMU_FILTER_PASID);
104 IOMMU_PMU_ATTR(filter_ats,		"config2:24-28",	IOMMU_PMU_FILTER_ATS);
105 IOMMU_PMU_ATTR(filter_page_table,	"config2:32-36",	IOMMU_PMU_FILTER_PAGE_TABLE);
106 
107 #define iommu_pmu_en_requester_id(e)		((e) & 0x1)
108 #define iommu_pmu_en_domain(e)			(((e) >> 1) & 0x1)
109 #define iommu_pmu_en_pasid(e)			(((e) >> 2) & 0x1)
110 #define iommu_pmu_en_ats(e)			(((e) >> 3) & 0x1)
111 #define iommu_pmu_en_page_table(e)		(((e) >> 4) & 0x1)
112 #define iommu_pmu_get_requester_id(filter)	(((filter) >> 16) & 0xffff)
113 #define iommu_pmu_get_domain(filter)		(((filter) >> 32) & 0xffff)
114 #define iommu_pmu_get_pasid(filter)		((filter) & 0x3fffff)
115 #define iommu_pmu_get_ats(filter)		(((filter) >> 24) & 0x1f)
116 #define iommu_pmu_get_page_table(filter)	(((filter) >> 32) & 0x1f)
117 
118 #define iommu_pmu_set_filter(_name, _config, _filter, _idx, _econfig)		\
119 {										\
120 	if ((iommu_pmu->filter & _filter) && iommu_pmu_en_##_name(_econfig)) {	\
121 		dmar_writel(iommu_pmu->cfg_reg + _idx * IOMMU_PMU_CFG_OFFSET +	\
122 			    IOMMU_PMU_CFG_SIZE +				\
123 			    (ffs(_filter) - 1) * IOMMU_PMU_CFG_FILTERS_OFFSET,	\
124 			    iommu_pmu_get_##_name(_config) | IOMMU_PMU_FILTER_EN);\
125 	}									\
126 }
127 
128 #define iommu_pmu_clear_filter(_filter, _idx)					\
129 {										\
130 	if (iommu_pmu->filter & _filter) {					\
131 		dmar_writel(iommu_pmu->cfg_reg + _idx * IOMMU_PMU_CFG_OFFSET +	\
132 			    IOMMU_PMU_CFG_SIZE +				\
133 			    (ffs(_filter) - 1) * IOMMU_PMU_CFG_FILTERS_OFFSET,	\
134 			    0);							\
135 	}									\
136 }
137 
138 /*
139  * Define the event attr related functions
140  * Input: _name: event attr name
141  *        _string: string of the event in sysfs
142  *        _g_idx: event group encoding
143  *        _event: event encoding
144  */
145 #define IOMMU_PMU_EVENT_ATTR(_name, _string, _g_idx, _event)			\
146 	PMU_EVENT_ATTR_STRING(_name, event_attr_##_name, _string)		\
147 										\
148 static struct attribute *_name##_attr[] = {					\
149 	&event_attr_##_name.attr.attr,						\
150 	NULL									\
151 };										\
152 										\
153 static umode_t									\
154 _name##_is_visible(struct kobject *kobj, struct attribute *attr, int i)		\
155 {										\
156 	struct device *dev = kobj_to_dev(kobj);					\
157 	struct iommu_pmu *iommu_pmu = dev_to_iommu_pmu(dev);			\
158 										\
159 	if (!iommu_pmu)								\
160 		return 0;							\
161 	return (iommu_pmu->evcap[_g_idx] & _event) ? attr->mode : 0;		\
162 }										\
163 										\
164 static struct attribute_group _name = {						\
165 	.name		= "events",						\
166 	.attrs		= _name##_attr,						\
167 	.is_visible	= _name##_is_visible,					\
168 };
169 
170 IOMMU_PMU_EVENT_ATTR(iommu_clocks,		"event_group=0x0,event=0x001", 0x0, 0x001)
171 IOMMU_PMU_EVENT_ATTR(iommu_requests,		"event_group=0x0,event=0x002", 0x0, 0x002)
172 IOMMU_PMU_EVENT_ATTR(pw_occupancy,		"event_group=0x0,event=0x004", 0x0, 0x004)
173 IOMMU_PMU_EVENT_ATTR(ats_blocked,		"event_group=0x0,event=0x008", 0x0, 0x008)
174 IOMMU_PMU_EVENT_ATTR(iommu_mrds,		"event_group=0x1,event=0x001", 0x1, 0x001)
175 IOMMU_PMU_EVENT_ATTR(iommu_mem_blocked,		"event_group=0x1,event=0x020", 0x1, 0x020)
176 IOMMU_PMU_EVENT_ATTR(pg_req_posted,		"event_group=0x1,event=0x040", 0x1, 0x040)
177 IOMMU_PMU_EVENT_ATTR(ctxt_cache_lookup,		"event_group=0x2,event=0x001", 0x2, 0x001)
178 IOMMU_PMU_EVENT_ATTR(ctxt_cache_hit,		"event_group=0x2,event=0x002", 0x2, 0x002)
179 IOMMU_PMU_EVENT_ATTR(pasid_cache_lookup,	"event_group=0x2,event=0x004", 0x2, 0x004)
180 IOMMU_PMU_EVENT_ATTR(pasid_cache_hit,		"event_group=0x2,event=0x008", 0x2, 0x008)
181 IOMMU_PMU_EVENT_ATTR(ss_nonleaf_lookup,		"event_group=0x2,event=0x010", 0x2, 0x010)
182 IOMMU_PMU_EVENT_ATTR(ss_nonleaf_hit,		"event_group=0x2,event=0x020", 0x2, 0x020)
183 IOMMU_PMU_EVENT_ATTR(fs_nonleaf_lookup,		"event_group=0x2,event=0x040", 0x2, 0x040)
184 IOMMU_PMU_EVENT_ATTR(fs_nonleaf_hit,		"event_group=0x2,event=0x080", 0x2, 0x080)
185 IOMMU_PMU_EVENT_ATTR(hpt_nonleaf_lookup,	"event_group=0x2,event=0x100", 0x2, 0x100)
186 IOMMU_PMU_EVENT_ATTR(hpt_nonleaf_hit,		"event_group=0x2,event=0x200", 0x2, 0x200)
187 IOMMU_PMU_EVENT_ATTR(iotlb_lookup,		"event_group=0x3,event=0x001", 0x3, 0x001)
188 IOMMU_PMU_EVENT_ATTR(iotlb_hit,			"event_group=0x3,event=0x002", 0x3, 0x002)
189 IOMMU_PMU_EVENT_ATTR(hpt_leaf_lookup,		"event_group=0x3,event=0x004", 0x3, 0x004)
190 IOMMU_PMU_EVENT_ATTR(hpt_leaf_hit,		"event_group=0x3,event=0x008", 0x3, 0x008)
191 IOMMU_PMU_EVENT_ATTR(int_cache_lookup,		"event_group=0x4,event=0x001", 0x4, 0x001)
192 IOMMU_PMU_EVENT_ATTR(int_cache_hit_nonposted,	"event_group=0x4,event=0x002", 0x4, 0x002)
193 IOMMU_PMU_EVENT_ATTR(int_cache_hit_posted,	"event_group=0x4,event=0x004", 0x4, 0x004)
194 
195 static const struct attribute_group *iommu_pmu_attr_update[] = {
196 	&filter_requester_id_en,
197 	&filter_domain_en,
198 	&filter_pasid_en,
199 	&filter_ats_en,
200 	&filter_page_table_en,
201 	&filter_requester_id,
202 	&filter_domain,
203 	&filter_pasid,
204 	&filter_ats,
205 	&filter_page_table,
206 	&iommu_clocks,
207 	&iommu_requests,
208 	&pw_occupancy,
209 	&ats_blocked,
210 	&iommu_mrds,
211 	&iommu_mem_blocked,
212 	&pg_req_posted,
213 	&ctxt_cache_lookup,
214 	&ctxt_cache_hit,
215 	&pasid_cache_lookup,
216 	&pasid_cache_hit,
217 	&ss_nonleaf_lookup,
218 	&ss_nonleaf_hit,
219 	&fs_nonleaf_lookup,
220 	&fs_nonleaf_hit,
221 	&hpt_nonleaf_lookup,
222 	&hpt_nonleaf_hit,
223 	&iotlb_lookup,
224 	&iotlb_hit,
225 	&hpt_leaf_lookup,
226 	&hpt_leaf_hit,
227 	&int_cache_lookup,
228 	&int_cache_hit_nonposted,
229 	&int_cache_hit_posted,
230 	NULL
231 };
232 
233 static inline void __iomem *
iommu_event_base(struct iommu_pmu * iommu_pmu,int idx)234 iommu_event_base(struct iommu_pmu *iommu_pmu, int idx)
235 {
236 	return iommu_pmu->cntr_reg + idx * iommu_pmu->cntr_stride;
237 }
238 
239 static inline void __iomem *
iommu_config_base(struct iommu_pmu * iommu_pmu,int idx)240 iommu_config_base(struct iommu_pmu *iommu_pmu, int idx)
241 {
242 	return iommu_pmu->cfg_reg + idx * IOMMU_PMU_CFG_OFFSET;
243 }
244 
iommu_event_to_pmu(struct perf_event * event)245 static inline struct iommu_pmu *iommu_event_to_pmu(struct perf_event *event)
246 {
247 	return container_of(event->pmu, struct iommu_pmu, pmu);
248 }
249 
iommu_event_config(struct perf_event * event)250 static inline u64 iommu_event_config(struct perf_event *event)
251 {
252 	u64 config = event->attr.config;
253 
254 	return (iommu_event_select(config) << IOMMU_EVENT_CFG_ES_SHIFT) |
255 	       (iommu_event_group(config) << IOMMU_EVENT_CFG_EGI_SHIFT) |
256 	       IOMMU_EVENT_CFG_INT;
257 }
258 
is_iommu_pmu_event(struct iommu_pmu * iommu_pmu,struct perf_event * event)259 static inline bool is_iommu_pmu_event(struct iommu_pmu *iommu_pmu,
260 				      struct perf_event *event)
261 {
262 	return event->pmu == &iommu_pmu->pmu;
263 }
264 
iommu_pmu_validate_event(struct perf_event * event)265 static int iommu_pmu_validate_event(struct perf_event *event)
266 {
267 	struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event);
268 	u32 event_group = iommu_event_group(event->attr.config);
269 
270 	if (event_group >= iommu_pmu->num_eg)
271 		return -EINVAL;
272 
273 	return 0;
274 }
275 
iommu_pmu_validate_group(struct perf_event * event)276 static int iommu_pmu_validate_group(struct perf_event *event)
277 {
278 	struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event);
279 	struct perf_event *sibling;
280 	int nr = 0;
281 
282 	/*
283 	 * All events in a group must be scheduled simultaneously.
284 	 * Check whether there is enough counters for all the events.
285 	 */
286 	for_each_sibling_event(sibling, event->group_leader) {
287 		if (!is_iommu_pmu_event(iommu_pmu, sibling) ||
288 		    sibling->state <= PERF_EVENT_STATE_OFF)
289 			continue;
290 
291 		if (++nr > iommu_pmu->num_cntr)
292 			return -EINVAL;
293 	}
294 
295 	return 0;
296 }
297 
iommu_pmu_event_init(struct perf_event * event)298 static int iommu_pmu_event_init(struct perf_event *event)
299 {
300 	struct hw_perf_event *hwc = &event->hw;
301 
302 	if (event->attr.type != event->pmu->type)
303 		return -ENOENT;
304 
305 	/* sampling not supported */
306 	if (event->attr.sample_period)
307 		return -EINVAL;
308 
309 	if (event->cpu < 0)
310 		return -EINVAL;
311 
312 	if (iommu_pmu_validate_event(event))
313 		return -EINVAL;
314 
315 	hwc->config = iommu_event_config(event);
316 
317 	return iommu_pmu_validate_group(event);
318 }
319 
iommu_pmu_event_update(struct perf_event * event)320 static void iommu_pmu_event_update(struct perf_event *event)
321 {
322 	struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event);
323 	struct hw_perf_event *hwc = &event->hw;
324 	u64 prev_count, new_count, delta;
325 	int shift = 64 - iommu_pmu->cntr_width;
326 
327 again:
328 	prev_count = local64_read(&hwc->prev_count);
329 	new_count = dmar_readq(iommu_event_base(iommu_pmu, hwc->idx));
330 	if (local64_xchg(&hwc->prev_count, new_count) != prev_count)
331 		goto again;
332 
333 	/*
334 	 * The counter width is enumerated. Always shift the counter
335 	 * before using it.
336 	 */
337 	delta = (new_count << shift) - (prev_count << shift);
338 	delta >>= shift;
339 
340 	local64_add(delta, &event->count);
341 }
342 
iommu_pmu_start(struct perf_event * event,int flags)343 static void iommu_pmu_start(struct perf_event *event, int flags)
344 {
345 	struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event);
346 	struct intel_iommu *iommu = iommu_pmu->iommu;
347 	struct hw_perf_event *hwc = &event->hw;
348 	u64 count;
349 
350 	if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
351 		return;
352 
353 	if (WARN_ON_ONCE(hwc->idx < 0 || hwc->idx >= IOMMU_PMU_IDX_MAX))
354 		return;
355 
356 	if (flags & PERF_EF_RELOAD)
357 		WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
358 
359 	hwc->state = 0;
360 
361 	/* Always reprogram the period */
362 	count = dmar_readq(iommu_event_base(iommu_pmu, hwc->idx));
363 	local64_set((&hwc->prev_count), count);
364 
365 	/*
366 	 * The error of ecmd will be ignored.
367 	 * - The existing perf_event subsystem doesn't handle the error.
368 	 *   Only IOMMU PMU returns runtime HW error. We don't want to
369 	 *   change the existing generic interfaces for the specific case.
370 	 * - It's a corner case caused by HW, which is very unlikely to
371 	 *   happen. There is nothing SW can do.
372 	 * - The worst case is that the user will get <not count> with
373 	 *   perf command, which can give the user some hints.
374 	 */
375 	ecmd_submit_sync(iommu, DMA_ECMD_ENABLE, hwc->idx, 0);
376 
377 	perf_event_update_userpage(event);
378 }
379 
iommu_pmu_stop(struct perf_event * event,int flags)380 static void iommu_pmu_stop(struct perf_event *event, int flags)
381 {
382 	struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event);
383 	struct intel_iommu *iommu = iommu_pmu->iommu;
384 	struct hw_perf_event *hwc = &event->hw;
385 
386 	if (!(hwc->state & PERF_HES_STOPPED)) {
387 		ecmd_submit_sync(iommu, DMA_ECMD_DISABLE, hwc->idx, 0);
388 
389 		iommu_pmu_event_update(event);
390 
391 		hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
392 	}
393 }
394 
395 static inline int
iommu_pmu_validate_per_cntr_event(struct iommu_pmu * iommu_pmu,int idx,struct perf_event * event)396 iommu_pmu_validate_per_cntr_event(struct iommu_pmu *iommu_pmu,
397 				  int idx, struct perf_event *event)
398 {
399 	u32 event_group = iommu_event_group(event->attr.config);
400 	u32 select = iommu_event_select(event->attr.config);
401 
402 	if (!(iommu_pmu->cntr_evcap[idx][event_group] & select))
403 		return -EINVAL;
404 
405 	return 0;
406 }
407 
iommu_pmu_assign_event(struct iommu_pmu * iommu_pmu,struct perf_event * event)408 static int iommu_pmu_assign_event(struct iommu_pmu *iommu_pmu,
409 				  struct perf_event *event)
410 {
411 	struct hw_perf_event *hwc = &event->hw;
412 	int idx;
413 
414 	/*
415 	 * The counters which support limited events are usually at the end.
416 	 * Schedule them first to accommodate more events.
417 	 */
418 	for (idx = iommu_pmu->num_cntr - 1; idx >= 0; idx--) {
419 		if (test_and_set_bit(idx, iommu_pmu->used_mask))
420 			continue;
421 		/* Check per-counter event capabilities */
422 		if (!iommu_pmu_validate_per_cntr_event(iommu_pmu, idx, event))
423 			break;
424 		clear_bit(idx, iommu_pmu->used_mask);
425 	}
426 	if (idx < 0)
427 		return -EINVAL;
428 
429 	iommu_pmu->event_list[idx] = event;
430 	hwc->idx = idx;
431 
432 	/* config events */
433 	dmar_writeq(iommu_config_base(iommu_pmu, idx), hwc->config);
434 
435 	iommu_pmu_set_filter(requester_id, event->attr.config1,
436 			     IOMMU_PMU_FILTER_REQUESTER_ID, idx,
437 			     event->attr.config1);
438 	iommu_pmu_set_filter(domain, event->attr.config1,
439 			     IOMMU_PMU_FILTER_DOMAIN, idx,
440 			     event->attr.config1);
441 	iommu_pmu_set_filter(pasid, event->attr.config2,
442 			     IOMMU_PMU_FILTER_PASID, idx,
443 			     event->attr.config1);
444 	iommu_pmu_set_filter(ats, event->attr.config2,
445 			     IOMMU_PMU_FILTER_ATS, idx,
446 			     event->attr.config1);
447 	iommu_pmu_set_filter(page_table, event->attr.config2,
448 			     IOMMU_PMU_FILTER_PAGE_TABLE, idx,
449 			     event->attr.config1);
450 
451 	return 0;
452 }
453 
iommu_pmu_add(struct perf_event * event,int flags)454 static int iommu_pmu_add(struct perf_event *event, int flags)
455 {
456 	struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event);
457 	struct hw_perf_event *hwc = &event->hw;
458 	int ret;
459 
460 	ret = iommu_pmu_assign_event(iommu_pmu, event);
461 	if (ret < 0)
462 		return ret;
463 
464 	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
465 
466 	if (flags & PERF_EF_START)
467 		iommu_pmu_start(event, 0);
468 
469 	return 0;
470 }
471 
iommu_pmu_del(struct perf_event * event,int flags)472 static void iommu_pmu_del(struct perf_event *event, int flags)
473 {
474 	struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event);
475 	int idx = event->hw.idx;
476 
477 	iommu_pmu_stop(event, PERF_EF_UPDATE);
478 
479 	iommu_pmu_clear_filter(IOMMU_PMU_FILTER_REQUESTER_ID, idx);
480 	iommu_pmu_clear_filter(IOMMU_PMU_FILTER_DOMAIN, idx);
481 	iommu_pmu_clear_filter(IOMMU_PMU_FILTER_PASID, idx);
482 	iommu_pmu_clear_filter(IOMMU_PMU_FILTER_ATS, idx);
483 	iommu_pmu_clear_filter(IOMMU_PMU_FILTER_PAGE_TABLE, idx);
484 
485 	iommu_pmu->event_list[idx] = NULL;
486 	event->hw.idx = -1;
487 	clear_bit(idx, iommu_pmu->used_mask);
488 
489 	perf_event_update_userpage(event);
490 }
491 
iommu_pmu_enable(struct pmu * pmu)492 static void iommu_pmu_enable(struct pmu *pmu)
493 {
494 	struct iommu_pmu *iommu_pmu = container_of(pmu, struct iommu_pmu, pmu);
495 	struct intel_iommu *iommu = iommu_pmu->iommu;
496 
497 	ecmd_submit_sync(iommu, DMA_ECMD_UNFREEZE, 0, 0);
498 }
499 
iommu_pmu_disable(struct pmu * pmu)500 static void iommu_pmu_disable(struct pmu *pmu)
501 {
502 	struct iommu_pmu *iommu_pmu = container_of(pmu, struct iommu_pmu, pmu);
503 	struct intel_iommu *iommu = iommu_pmu->iommu;
504 
505 	ecmd_submit_sync(iommu, DMA_ECMD_FREEZE, 0, 0);
506 }
507 
iommu_pmu_counter_overflow(struct iommu_pmu * iommu_pmu)508 static void iommu_pmu_counter_overflow(struct iommu_pmu *iommu_pmu)
509 {
510 	struct perf_event *event;
511 	u64 status;
512 	int i;
513 
514 	/*
515 	 * Two counters may be overflowed very close. Always check
516 	 * whether there are more to handle.
517 	 */
518 	while ((status = dmar_readq(iommu_pmu->overflow))) {
519 		for_each_set_bit(i, (unsigned long *)&status, iommu_pmu->num_cntr) {
520 			/*
521 			 * Find the assigned event of the counter.
522 			 * Accumulate the value into the event->count.
523 			 */
524 			event = iommu_pmu->event_list[i];
525 			if (!event) {
526 				pr_warn_once("Cannot find the assigned event for counter %d\n", i);
527 				continue;
528 			}
529 			iommu_pmu_event_update(event);
530 		}
531 
532 		dmar_writeq(iommu_pmu->overflow, status);
533 	}
534 }
535 
iommu_pmu_irq_handler(int irq,void * dev_id)536 static irqreturn_t iommu_pmu_irq_handler(int irq, void *dev_id)
537 {
538 	struct intel_iommu *iommu = dev_id;
539 
540 	if (!dmar_readl(iommu->reg + DMAR_PERFINTRSTS_REG))
541 		return IRQ_NONE;
542 
543 	iommu_pmu_counter_overflow(iommu->pmu);
544 
545 	/* Clear the status bit */
546 	dmar_writel(iommu->reg + DMAR_PERFINTRSTS_REG, DMA_PERFINTRSTS_PIS);
547 
548 	return IRQ_HANDLED;
549 }
550 
__iommu_pmu_register(struct intel_iommu * iommu)551 static int __iommu_pmu_register(struct intel_iommu *iommu)
552 {
553 	struct iommu_pmu *iommu_pmu = iommu->pmu;
554 
555 	iommu_pmu->pmu.name		= iommu->name;
556 	iommu_pmu->pmu.task_ctx_nr	= perf_invalid_context;
557 	iommu_pmu->pmu.event_init	= iommu_pmu_event_init;
558 	iommu_pmu->pmu.pmu_enable	= iommu_pmu_enable;
559 	iommu_pmu->pmu.pmu_disable	= iommu_pmu_disable;
560 	iommu_pmu->pmu.add		= iommu_pmu_add;
561 	iommu_pmu->pmu.del		= iommu_pmu_del;
562 	iommu_pmu->pmu.start		= iommu_pmu_start;
563 	iommu_pmu->pmu.stop		= iommu_pmu_stop;
564 	iommu_pmu->pmu.read		= iommu_pmu_event_update;
565 	iommu_pmu->pmu.attr_groups	= iommu_pmu_attr_groups;
566 	iommu_pmu->pmu.attr_update	= iommu_pmu_attr_update;
567 	iommu_pmu->pmu.capabilities	= PERF_PMU_CAP_NO_EXCLUDE;
568 	iommu_pmu->pmu.module		= THIS_MODULE;
569 
570 	return perf_pmu_register(&iommu_pmu->pmu, iommu_pmu->pmu.name, -1);
571 }
572 
573 static inline void __iomem *
get_perf_reg_address(struct intel_iommu * iommu,u32 offset)574 get_perf_reg_address(struct intel_iommu *iommu, u32 offset)
575 {
576 	u32 off = dmar_readl(iommu->reg + offset);
577 
578 	return iommu->reg + off;
579 }
580 
alloc_iommu_pmu(struct intel_iommu * iommu)581 int alloc_iommu_pmu(struct intel_iommu *iommu)
582 {
583 	struct iommu_pmu *iommu_pmu;
584 	int i, j, ret;
585 	u64 perfcap;
586 	u32 cap;
587 
588 	if (!ecap_pms(iommu->ecap))
589 		return 0;
590 
591 	/* The IOMMU PMU requires the ECMD support as well */
592 	if (!cap_ecmds(iommu->cap))
593 		return -ENODEV;
594 
595 	perfcap = dmar_readq(iommu->reg + DMAR_PERFCAP_REG);
596 	/* The performance monitoring is not supported. */
597 	if (!perfcap)
598 		return -ENODEV;
599 
600 	/* Sanity check for the number of the counters and event groups */
601 	if (!pcap_num_cntr(perfcap) || !pcap_num_event_group(perfcap))
602 		return -ENODEV;
603 
604 	/* The interrupt on overflow is required */
605 	if (!pcap_interrupt(perfcap))
606 		return -ENODEV;
607 
608 	/* Check required Enhanced Command Capability */
609 	if (!ecmd_has_pmu_essential(iommu))
610 		return -ENODEV;
611 
612 	iommu_pmu = kzalloc(sizeof(*iommu_pmu), GFP_KERNEL);
613 	if (!iommu_pmu)
614 		return -ENOMEM;
615 
616 	iommu_pmu->num_cntr = pcap_num_cntr(perfcap);
617 	if (iommu_pmu->num_cntr > IOMMU_PMU_IDX_MAX) {
618 		pr_warn_once("The number of IOMMU counters %d > max(%d), clipping!",
619 			     iommu_pmu->num_cntr, IOMMU_PMU_IDX_MAX);
620 		iommu_pmu->num_cntr = IOMMU_PMU_IDX_MAX;
621 	}
622 
623 	iommu_pmu->cntr_width = pcap_cntr_width(perfcap);
624 	iommu_pmu->filter = pcap_filters_mask(perfcap);
625 	iommu_pmu->cntr_stride = pcap_cntr_stride(perfcap);
626 	iommu_pmu->num_eg = pcap_num_event_group(perfcap);
627 
628 	iommu_pmu->evcap = kcalloc(iommu_pmu->num_eg, sizeof(u64), GFP_KERNEL);
629 	if (!iommu_pmu->evcap) {
630 		ret = -ENOMEM;
631 		goto free_pmu;
632 	}
633 
634 	/* Parse event group capabilities */
635 	for (i = 0; i < iommu_pmu->num_eg; i++) {
636 		u64 pcap;
637 
638 		pcap = dmar_readq(iommu->reg + DMAR_PERFEVNTCAP_REG +
639 				  i * IOMMU_PMU_CAP_REGS_STEP);
640 		iommu_pmu->evcap[i] = pecap_es(pcap);
641 	}
642 
643 	iommu_pmu->cntr_evcap = kcalloc(iommu_pmu->num_cntr, sizeof(u32 *), GFP_KERNEL);
644 	if (!iommu_pmu->cntr_evcap) {
645 		ret = -ENOMEM;
646 		goto free_pmu_evcap;
647 	}
648 	for (i = 0; i < iommu_pmu->num_cntr; i++) {
649 		iommu_pmu->cntr_evcap[i] = kcalloc(iommu_pmu->num_eg, sizeof(u32), GFP_KERNEL);
650 		if (!iommu_pmu->cntr_evcap[i]) {
651 			ret = -ENOMEM;
652 			goto free_pmu_cntr_evcap;
653 		}
654 		/*
655 		 * Set to the global capabilities, will adjust according
656 		 * to per-counter capabilities later.
657 		 */
658 		for (j = 0; j < iommu_pmu->num_eg; j++)
659 			iommu_pmu->cntr_evcap[i][j] = (u32)iommu_pmu->evcap[j];
660 	}
661 
662 	iommu_pmu->cfg_reg = get_perf_reg_address(iommu, DMAR_PERFCFGOFF_REG);
663 	iommu_pmu->cntr_reg = get_perf_reg_address(iommu, DMAR_PERFCNTROFF_REG);
664 	iommu_pmu->overflow = get_perf_reg_address(iommu, DMAR_PERFOVFOFF_REG);
665 
666 	/*
667 	 * Check per-counter capabilities. All counters should have the
668 	 * same capabilities on Interrupt on Overflow Support and Counter
669 	 * Width.
670 	 */
671 	for (i = 0; i < iommu_pmu->num_cntr; i++) {
672 		cap = dmar_readl(iommu_pmu->cfg_reg +
673 				 i * IOMMU_PMU_CFG_OFFSET +
674 				 IOMMU_PMU_CFG_CNTRCAP_OFFSET);
675 		if (!iommu_cntrcap_pcc(cap))
676 			continue;
677 
678 		/*
679 		 * It's possible that some counters have a different
680 		 * capability because of e.g., HW bug. Check the corner
681 		 * case here and simply drop those counters.
682 		 */
683 		if ((iommu_cntrcap_cw(cap) != iommu_pmu->cntr_width) ||
684 		    !iommu_cntrcap_ios(cap)) {
685 			iommu_pmu->num_cntr = i;
686 			pr_warn("PMU counter capability inconsistent, counter number reduced to %d\n",
687 				iommu_pmu->num_cntr);
688 		}
689 
690 		/* Clear the pre-defined events group */
691 		for (j = 0; j < iommu_pmu->num_eg; j++)
692 			iommu_pmu->cntr_evcap[i][j] = 0;
693 
694 		/* Override with per-counter event capabilities */
695 		for (j = 0; j < iommu_cntrcap_egcnt(cap); j++) {
696 			cap = dmar_readl(iommu_pmu->cfg_reg + i * IOMMU_PMU_CFG_OFFSET +
697 					 IOMMU_PMU_CFG_CNTREVCAP_OFFSET +
698 					 (j * IOMMU_PMU_OFF_REGS_STEP));
699 			iommu_pmu->cntr_evcap[i][iommu_event_group(cap)] = iommu_event_select(cap);
700 			/*
701 			 * Some events may only be supported by a specific counter.
702 			 * Track them in the evcap as well.
703 			 */
704 			iommu_pmu->evcap[iommu_event_group(cap)] |= iommu_event_select(cap);
705 		}
706 	}
707 
708 	iommu_pmu->iommu = iommu;
709 	iommu->pmu = iommu_pmu;
710 
711 	return 0;
712 
713 free_pmu_cntr_evcap:
714 	for (i = 0; i < iommu_pmu->num_cntr; i++)
715 		kfree(iommu_pmu->cntr_evcap[i]);
716 	kfree(iommu_pmu->cntr_evcap);
717 free_pmu_evcap:
718 	kfree(iommu_pmu->evcap);
719 free_pmu:
720 	kfree(iommu_pmu);
721 
722 	return ret;
723 }
724 
free_iommu_pmu(struct intel_iommu * iommu)725 void free_iommu_pmu(struct intel_iommu *iommu)
726 {
727 	struct iommu_pmu *iommu_pmu = iommu->pmu;
728 
729 	if (!iommu_pmu)
730 		return;
731 
732 	if (iommu_pmu->evcap) {
733 		int i;
734 
735 		for (i = 0; i < iommu_pmu->num_cntr; i++)
736 			kfree(iommu_pmu->cntr_evcap[i]);
737 		kfree(iommu_pmu->cntr_evcap);
738 	}
739 	kfree(iommu_pmu->evcap);
740 	kfree(iommu_pmu);
741 	iommu->pmu = NULL;
742 }
743 
iommu_pmu_set_interrupt(struct intel_iommu * iommu)744 static int iommu_pmu_set_interrupt(struct intel_iommu *iommu)
745 {
746 	struct iommu_pmu *iommu_pmu = iommu->pmu;
747 	int irq, ret;
748 
749 	irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PERF + iommu->seq_id, iommu->node, iommu);
750 	if (irq <= 0)
751 		return -EINVAL;
752 
753 	snprintf(iommu_pmu->irq_name, sizeof(iommu_pmu->irq_name), "dmar%d-perf", iommu->seq_id);
754 
755 	iommu->perf_irq = irq;
756 	ret = request_threaded_irq(irq, NULL, iommu_pmu_irq_handler,
757 				   IRQF_ONESHOT, iommu_pmu->irq_name, iommu);
758 	if (ret) {
759 		dmar_free_hwirq(irq);
760 		iommu->perf_irq = 0;
761 		return ret;
762 	}
763 	return 0;
764 }
765 
iommu_pmu_unset_interrupt(struct intel_iommu * iommu)766 static void iommu_pmu_unset_interrupt(struct intel_iommu *iommu)
767 {
768 	if (!iommu->perf_irq)
769 		return;
770 
771 	free_irq(iommu->perf_irq, iommu);
772 	dmar_free_hwirq(iommu->perf_irq);
773 	iommu->perf_irq = 0;
774 }
775 
iommu_pmu_cpu_online(unsigned int cpu,struct hlist_node * node)776 static int iommu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
777 {
778 	struct iommu_pmu *iommu_pmu = hlist_entry_safe(node, typeof(*iommu_pmu), cpuhp_node);
779 
780 	if (cpumask_empty(&iommu_pmu_cpu_mask))
781 		cpumask_set_cpu(cpu, &iommu_pmu_cpu_mask);
782 
783 	if (cpumask_test_cpu(cpu, &iommu_pmu_cpu_mask))
784 		iommu_pmu->cpu = cpu;
785 
786 	return 0;
787 }
788 
iommu_pmu_cpu_offline(unsigned int cpu,struct hlist_node * node)789 static int iommu_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node)
790 {
791 	struct iommu_pmu *iommu_pmu = hlist_entry_safe(node, typeof(*iommu_pmu), cpuhp_node);
792 	int target = cpumask_first(&iommu_pmu_cpu_mask);
793 
794 	/*
795 	 * The iommu_pmu_cpu_mask has been updated when offline the CPU
796 	 * for the first iommu_pmu. Migrate the other iommu_pmu to the
797 	 * new target.
798 	 */
799 	if (target < nr_cpu_ids && target != iommu_pmu->cpu) {
800 		perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target);
801 		iommu_pmu->cpu = target;
802 		return 0;
803 	}
804 
805 	if (!cpumask_test_and_clear_cpu(cpu, &iommu_pmu_cpu_mask))
806 		return 0;
807 
808 	target = cpumask_any_but(cpu_online_mask, cpu);
809 
810 	if (target < nr_cpu_ids)
811 		cpumask_set_cpu(target, &iommu_pmu_cpu_mask);
812 	else
813 		return 0;
814 
815 	perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target);
816 	iommu_pmu->cpu = target;
817 
818 	return 0;
819 }
820 
821 static int nr_iommu_pmu;
822 static enum cpuhp_state iommu_cpuhp_slot;
823 
iommu_pmu_cpuhp_setup(struct iommu_pmu * iommu_pmu)824 static int iommu_pmu_cpuhp_setup(struct iommu_pmu *iommu_pmu)
825 {
826 	int ret;
827 
828 	if (!nr_iommu_pmu) {
829 		ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
830 					      "driver/iommu/intel/perfmon:online",
831 					      iommu_pmu_cpu_online,
832 					      iommu_pmu_cpu_offline);
833 		if (ret < 0)
834 			return ret;
835 		iommu_cpuhp_slot = ret;
836 	}
837 
838 	ret = cpuhp_state_add_instance(iommu_cpuhp_slot, &iommu_pmu->cpuhp_node);
839 	if (ret) {
840 		if (!nr_iommu_pmu)
841 			cpuhp_remove_multi_state(iommu_cpuhp_slot);
842 		return ret;
843 	}
844 	nr_iommu_pmu++;
845 
846 	return 0;
847 }
848 
iommu_pmu_cpuhp_free(struct iommu_pmu * iommu_pmu)849 static void iommu_pmu_cpuhp_free(struct iommu_pmu *iommu_pmu)
850 {
851 	cpuhp_state_remove_instance(iommu_cpuhp_slot, &iommu_pmu->cpuhp_node);
852 
853 	if (--nr_iommu_pmu)
854 		return;
855 
856 	cpuhp_remove_multi_state(iommu_cpuhp_slot);
857 }
858 
iommu_pmu_register(struct intel_iommu * iommu)859 void iommu_pmu_register(struct intel_iommu *iommu)
860 {
861 	struct iommu_pmu *iommu_pmu = iommu->pmu;
862 
863 	if (!iommu_pmu)
864 		return;
865 
866 	if (__iommu_pmu_register(iommu))
867 		goto err;
868 
869 	if (iommu_pmu_cpuhp_setup(iommu_pmu))
870 		goto unregister;
871 
872 	/* Set interrupt for overflow */
873 	if (iommu_pmu_set_interrupt(iommu))
874 		goto cpuhp_free;
875 
876 	return;
877 
878 cpuhp_free:
879 	iommu_pmu_cpuhp_free(iommu_pmu);
880 unregister:
881 	perf_pmu_unregister(&iommu_pmu->pmu);
882 err:
883 	pr_err("Failed to register PMU for iommu (seq_id = %d)\n", iommu->seq_id);
884 	free_iommu_pmu(iommu);
885 }
886 
iommu_pmu_unregister(struct intel_iommu * iommu)887 void iommu_pmu_unregister(struct intel_iommu *iommu)
888 {
889 	struct iommu_pmu *iommu_pmu = iommu->pmu;
890 
891 	if (!iommu_pmu)
892 		return;
893 
894 	iommu_pmu_unset_interrupt(iommu);
895 	iommu_pmu_cpuhp_free(iommu_pmu);
896 	perf_pmu_unregister(&iommu_pmu->pmu);
897 }
898