80 static bool is_idxd_event(struct idxd_pmu *idxd_pmu, struct perf_event *event)  in is_idxd_event()  argument
82 	return &idxd_pmu->pmu == event->pmu;  in is_idxd_event()
85 static int perfmon_collect_events(struct idxd_pmu *idxd_pmu,  in perfmon_collect_events()  argument
92 	max_count = idxd_pmu->n_counters;  in perfmon_collect_events()
93 	n = idxd_pmu->n_events;  in perfmon_collect_events()
98 	if (is_idxd_event(idxd_pmu, leader)) {  in perfmon_collect_events()
99 		idxd_pmu->event_list[n] = leader;  in perfmon_collect_events()
100 		idxd_pmu->event_list[n]->hw.idx = n;  in perfmon_collect_events()
108 		if (!is_idxd_event(idxd_pmu, event) ||  in perfmon_collect_events()
115 		idxd_pmu->event_list[n] = event;  in perfmon_collect_events()
116 		idxd_pmu->event_list[n]->hw.idx = n;  in perfmon_collect_events()
123 static void perfmon_assign_hw_event(struct idxd_pmu *idxd_pmu,  in perfmon_assign_hw_event()  argument
126 	struct idxd_device *idxd = idxd_pmu->idxd;  in perfmon_assign_hw_event()
134 static int perfmon_assign_event(struct idxd_pmu *idxd_pmu,  in perfmon_assign_event()  argument
140 		if (!test_and_set_bit(i, idxd_pmu->used_mask))  in perfmon_assign_event()
154 static int perfmon_validate_group(struct idxd_pmu *pmu,  in perfmon_validate_group()
158 	struct idxd_pmu *fake_pmu;  in perfmon_validate_group()
216 	if (event->pmu != &idxd->idxd_pmu->pmu)  in perfmon_pmu_event_init()
220 	event->cpu = idxd->idxd_pmu->cpu;  in perfmon_pmu_event_init()
225 		ret = perfmon_validate_group(idxd->idxd_pmu, event);  in perfmon_pmu_event_init()
245 	int shift = 64 - idxd->idxd_pmu->counter_width;  in perfmon_pmu_event_update()
267 	n_counters = min(idxd->idxd_pmu->n_counters, OVERFLOW_SIZE);  in perfmon_counter_overflow()
285 			event = idxd->idxd_pmu->event_list[i];  in perfmon_counter_overflow()
348 	if (flt_wq && test_bit(FLT_WQ, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
350 	if (flt_tc && test_bit(FLT_TC, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
352 	if (flt_pg_sz && test_bit(FLT_PG_SZ, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
354 	if (flt_xfer_sz && test_bit(FLT_XFER_SZ, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
356 	if (flt_eng && test_bit(FLT_ENG, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
382 	for (i = 0; i < idxd->idxd_pmu->n_events; i++) {  in perfmon_pmu_event_stop()
383 		if (event != idxd->idxd_pmu->event_list[i])  in perfmon_pmu_event_stop()
386 		for (++i; i < idxd->idxd_pmu->n_events; i++)  in perfmon_pmu_event_stop()
387 			idxd->idxd_pmu->event_list[i - 1] = idxd->idxd_pmu->event_list[i];  in perfmon_pmu_event_stop()
388 		--idxd->idxd_pmu->n_events;  in perfmon_pmu_event_stop()
400 	clear_bit(cntr, idxd->idxd_pmu->used_mask);  in perfmon_pmu_event_stop()
411 	struct idxd_pmu *idxd_pmu = idxd->idxd_pmu;  in perfmon_pmu_event_add()  local
415 	n = perfmon_collect_events(idxd_pmu, event, false);  in perfmon_pmu_event_add()
423 	idx = perfmon_assign_event(idxd_pmu, event);  in perfmon_pmu_event_add()
427 	perfmon_assign_hw_event(idxd_pmu, event, idx);  in perfmon_pmu_event_add()
432 	idxd_pmu->n_events = n;  in perfmon_pmu_event_add()
470 static void idxd_pmu_init(struct idxd_pmu *idxd_pmu)  in idxd_pmu_init()  argument
475 		if (!test_bit(i, &idxd_pmu->supported_filters))  in idxd_pmu_init()
479 	idxd_pmu->pmu.name		= idxd_pmu->name;  in idxd_pmu_init()
480 	idxd_pmu->pmu.attr_groups	= perfmon_attr_groups;  in idxd_pmu_init()
481 	idxd_pmu->pmu.task_ctx_nr	= perf_invalid_context;  in idxd_pmu_init()
482 	idxd_pmu->pmu.event_init	= perfmon_pmu_event_init;  in idxd_pmu_init()
483 	idxd_pmu->pmu.pmu_enable	= perfmon_pmu_enable,  in idxd_pmu_init()
484 	idxd_pmu->pmu.pmu_disable	= perfmon_pmu_disable,  in idxd_pmu_init()
485 	idxd_pmu->pmu.add		= perfmon_pmu_event_add;  in idxd_pmu_init()
486 	idxd_pmu->pmu.del		= perfmon_pmu_event_del;  in idxd_pmu_init()
487 	idxd_pmu->pmu.start		= perfmon_pmu_event_start;  in idxd_pmu_init()
488 	idxd_pmu->pmu.stop		= perfmon_pmu_event_stop;  in idxd_pmu_init()
489 	idxd_pmu->pmu.read		= perfmon_pmu_event_update;  in idxd_pmu_init()
490 	idxd_pmu->pmu.capabilities	= PERF_PMU_CAP_NO_EXCLUDE;  in idxd_pmu_init()
491 	idxd_pmu->pmu.module		= THIS_MODULE;  in idxd_pmu_init()
496 	if (!idxd->idxd_pmu)  in perfmon_pmu_remove()
499 	cpuhp_state_remove_instance(cpuhp_slot, &idxd->idxd_pmu->cpuhp_node);  in perfmon_pmu_remove()
500 	perf_pmu_unregister(&idxd->idxd_pmu->pmu);  in perfmon_pmu_remove()
501 	kfree(idxd->idxd_pmu);  in perfmon_pmu_remove()
502 	idxd->idxd_pmu = NULL;  in perfmon_pmu_remove()
507 	struct idxd_pmu *idxd_pmu;  in perf_event_cpu_online()  local
509 	idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node);  in perf_event_cpu_online()
514 		idxd_pmu->cpu = cpu;  in perf_event_cpu_online()
522 	struct idxd_pmu *idxd_pmu;  in perf_event_cpu_offline()  local
525 	idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node);  in perf_event_cpu_offline()
534 		perf_pmu_migrate_context(&idxd_pmu->pmu, cpu, target);  in perf_event_cpu_offline()
543 	struct idxd_pmu *idxd_pmu;  in perfmon_pmu_init()  local
559 	idxd_pmu = kzalloc(sizeof(*idxd_pmu), GFP_KERNEL);  in perfmon_pmu_init()
560 	if (!idxd_pmu)  in perfmon_pmu_init()
563 	idxd_pmu->idxd = idxd;  in perfmon_pmu_init()
564 	idxd->idxd_pmu = idxd_pmu;  in perfmon_pmu_init()
567 		rc = sprintf(idxd_pmu->name, "dsa%d", idxd->id);  in perfmon_pmu_init()
571 		rc = sprintf(idxd_pmu->name, "iax%d", idxd->id);  in perfmon_pmu_init()
608 	idxd_pmu->n_event_categories = perfcap.num_event_category;  in perfmon_pmu_init()
609 	idxd_pmu->supported_event_categories = perfcap.global_event_category;  in perfmon_pmu_init()
610 	idxd_pmu->per_counter_caps_supported = perfcap.cap_per_counter;  in perfmon_pmu_init()
613 	idxd_pmu->supported_filters = perfcap.filter;  in perfmon_pmu_init()
615 		idxd_pmu->n_filters = hweight8(perfcap.filter);  in perfmon_pmu_init()
618 	idxd_pmu->n_counters = perfcap.num_perf_counter;  in perfmon_pmu_init()
619 	idxd_pmu->counter_width = perfcap.counter_width;  in perfmon_pmu_init()
621 	idxd_pmu_init(idxd_pmu);  in perfmon_pmu_init()
623 	rc = perf_pmu_register(&idxd_pmu->pmu, idxd_pmu->name, -1);  in perfmon_pmu_init()
627 	rc = cpuhp_state_add_instance(cpuhp_slot, &idxd_pmu->cpuhp_node);  in perfmon_pmu_init()
629 		perf_pmu_unregister(&idxd->idxd_pmu->pmu);  in perfmon_pmu_init()
635 	kfree(idxd_pmu);  in perfmon_pmu_init()
636 	idxd->idxd_pmu = NULL;  in perfmon_pmu_init()