124 static u32 arm_spe_pmu_cap_get(struct arm_spe_pmu *spe_pmu, int cap)  in arm_spe_pmu_cap_get()  argument
127 		return !!(spe_pmu->features & arm_spe_pmu_feat_caps[cap]);  in arm_spe_pmu_cap_get()
131 		return spe_pmu->counter_sz;  in arm_spe_pmu_cap_get()
133 		return spe_pmu->min_period;  in arm_spe_pmu_cap_get()
145 	struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev);  in arm_spe_pmu_cap_show()  local
150 	return sysfs_emit(buf, "%u\n", arm_spe_pmu_cap_get(spe_pmu, cap));  in arm_spe_pmu_cap_show()
261 	struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev);  in arm_spe_pmu_format_attr_is_visible()  local
263 	if (attr == &format_attr_inv_event_filter.attr && !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT))  in arm_spe_pmu_format_attr_is_visible()
278 	struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev);  in cpumask_show()  local
280 	return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus);  in cpumask_show()
324 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_event_sanitise_period()  local
328 	if (period < spe_pmu->min_period)  in arm_spe_event_sanitise_period()
329 		period = spe_pmu->min_period;  in arm_spe_event_sanitise_period()
406 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in arm_spe_pmu_next_snapshot_off()  local
423 	if (limit - head < spe_pmu->max_record_sz) {  in arm_spe_pmu_next_snapshot_off()
434 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in __arm_spe_pmu_next_off()  local
455 	if (!IS_ALIGNED(head, spe_pmu->align)) {  in __arm_spe_pmu_next_off()
456 		unsigned long delta = roundup(head, spe_pmu->align) - head;  in __arm_spe_pmu_next_off()
505 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);  in arm_spe_pmu_next_off()  local
513 	if (limit && (limit - head < spe_pmu->max_record_sz)) {  in arm_spe_pmu_next_off()
722 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_event_init()  local
729 	    !cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_event_init()
732 	if (arm_spe_event_to_pmsevfr(event) & arm_spe_pmsevfr_res0(spe_pmu->pmsver))  in arm_spe_pmu_event_init()
735 	if (arm_spe_event_to_pmsnevfr(event) & arm_spe_pmsevfr_res0(spe_pmu->pmsver))  in arm_spe_pmu_event_init()
753 	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_EVT))  in arm_spe_pmu_event_init()
757 	    !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT))  in arm_spe_pmu_event_init()
761 	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_TYP))  in arm_spe_pmu_event_init()
765 	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_LAT))  in arm_spe_pmu_event_init()
779 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_start()  local
781 	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);  in arm_spe_pmu_start()
794 	if (spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT) {  in arm_spe_pmu_start()
817 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_stop()  local
819 	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);  in arm_spe_pmu_stop()
859 	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);  in arm_spe_pmu_add()  local
863 	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_add()
946 static int arm_spe_pmu_perf_init(struct arm_spe_pmu *spe_pmu)  in arm_spe_pmu_perf_init()  argument
952 	struct device *dev = &spe_pmu->pdev->dev;  in arm_spe_pmu_perf_init()
954 	spe_pmu->pmu = (struct pmu) {  in arm_spe_pmu_perf_init()
986 	return perf_pmu_register(&spe_pmu->pmu, name, -1);  in arm_spe_pmu_perf_init()
989 static void arm_spe_pmu_perf_destroy(struct arm_spe_pmu *spe_pmu)  in arm_spe_pmu_perf_destroy()  argument
991 	perf_pmu_unregister(&spe_pmu->pmu);  in arm_spe_pmu_perf_destroy()
998 	struct arm_spe_pmu *spe_pmu = info;  in __arm_spe_pmu_dev_probe()  local
999 	struct device *dev = &spe_pmu->pdev->dev;  in __arm_spe_pmu_dev_probe()
1009 	spe_pmu->pmsver = (u16)fld;  in __arm_spe_pmu_dev_probe()
1021 	spe_pmu->align = 1 << fld;  in __arm_spe_pmu_dev_probe()
1022 	if (spe_pmu->align > SZ_2K) {  in __arm_spe_pmu_dev_probe()
1031 		spe_pmu->features |= SPE_PMU_FEAT_FILT_EVT;  in __arm_spe_pmu_dev_probe()
1034 		spe_pmu->features |= SPE_PMU_FEAT_INV_FILT_EVT;  in __arm_spe_pmu_dev_probe()
1037 		spe_pmu->features |= SPE_PMU_FEAT_FILT_TYP;  in __arm_spe_pmu_dev_probe()
1040 		spe_pmu->features |= SPE_PMU_FEAT_FILT_LAT;  in __arm_spe_pmu_dev_probe()
1043 		spe_pmu->features |= SPE_PMU_FEAT_ARCH_INST;  in __arm_spe_pmu_dev_probe()
1046 		spe_pmu->features |= SPE_PMU_FEAT_LDS;  in __arm_spe_pmu_dev_probe()
1049 		spe_pmu->features |= SPE_PMU_FEAT_ERND;  in __arm_spe_pmu_dev_probe()
1055 		spe_pmu->min_period = 256;  in __arm_spe_pmu_dev_probe()
1058 		spe_pmu->min_period = 512;  in __arm_spe_pmu_dev_probe()
1061 		spe_pmu->min_period = 768;  in __arm_spe_pmu_dev_probe()
1064 		spe_pmu->min_period = 1024;  in __arm_spe_pmu_dev_probe()
1067 		spe_pmu->min_period = 1536;  in __arm_spe_pmu_dev_probe()
1070 		spe_pmu->min_period = 2048;  in __arm_spe_pmu_dev_probe()
1073 		spe_pmu->min_period = 3072;  in __arm_spe_pmu_dev_probe()
1080 		spe_pmu->min_period = 4096;  in __arm_spe_pmu_dev_probe()
1085 	spe_pmu->max_record_sz = 1 << fld;  in __arm_spe_pmu_dev_probe()
1086 	if (spe_pmu->max_record_sz > SZ_2K || spe_pmu->max_record_sz < 16) {  in __arm_spe_pmu_dev_probe()
1099 		spe_pmu->counter_sz = 12;  in __arm_spe_pmu_dev_probe()
1102 		spe_pmu->counter_sz = 16;  in __arm_spe_pmu_dev_probe()
1107 		 spe_pmu->pmsver - 1, cpumask_pr_args(&spe_pmu->supported_cpus),  in __arm_spe_pmu_dev_probe()
1108 		 spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features);  in __arm_spe_pmu_dev_probe()
1110 	spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED;  in __arm_spe_pmu_dev_probe()
1132 	struct arm_spe_pmu *spe_pmu = info;  in __arm_spe_pmu_setup_one()  local
1135 	enable_percpu_irq(spe_pmu->irq, IRQ_TYPE_NONE);  in __arm_spe_pmu_setup_one()
1140 	struct arm_spe_pmu *spe_pmu = info;  in __arm_spe_pmu_stop_one()  local
1142 	disable_percpu_irq(spe_pmu->irq);  in __arm_spe_pmu_stop_one()
1148 	struct arm_spe_pmu *spe_pmu;  in arm_spe_pmu_cpu_startup()  local
1150 	spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node);  in arm_spe_pmu_cpu_startup()
1151 	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_cpu_startup()
1154 	__arm_spe_pmu_setup_one(spe_pmu);  in arm_spe_pmu_cpu_startup()
1160 	struct arm_spe_pmu *spe_pmu;  in arm_spe_pmu_cpu_teardown()  local
1162 	spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node);  in arm_spe_pmu_cpu_teardown()
1163 	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))  in arm_spe_pmu_cpu_teardown()
1166 	__arm_spe_pmu_stop_one(spe_pmu);  in arm_spe_pmu_cpu_teardown()
1170 static int arm_spe_pmu_dev_init(struct arm_spe_pmu *spe_pmu)  in arm_spe_pmu_dev_init()  argument
1173 	cpumask_t *mask = &spe_pmu->supported_cpus;  in arm_spe_pmu_dev_init()
1176 	ret = smp_call_function_any(mask,  __arm_spe_pmu_dev_probe, spe_pmu, 1);  in arm_spe_pmu_dev_init()
1177 	if (ret || !(spe_pmu->features & SPE_PMU_FEAT_DEV_PROBED))  in arm_spe_pmu_dev_init()
1181 	ret = request_percpu_irq(spe_pmu->irq, arm_spe_pmu_irq_handler, DRVNAME,  in arm_spe_pmu_dev_init()
1182 				 spe_pmu->handle);  in arm_spe_pmu_dev_init()
1192 				       &spe_pmu->hotplug_node);  in arm_spe_pmu_dev_init()
1194 		free_percpu_irq(spe_pmu->irq, spe_pmu->handle);  in arm_spe_pmu_dev_init()
1199 static void arm_spe_pmu_dev_teardown(struct arm_spe_pmu *spe_pmu)  in arm_spe_pmu_dev_teardown()  argument
1201 	cpuhp_state_remove_instance(arm_spe_pmu_online, &spe_pmu->hotplug_node);  in arm_spe_pmu_dev_teardown()
1202 	free_percpu_irq(spe_pmu->irq, spe_pmu->handle);  in arm_spe_pmu_dev_teardown()
1206 static int arm_spe_pmu_irq_probe(struct arm_spe_pmu *spe_pmu)  in arm_spe_pmu_irq_probe()  argument
1208 	struct platform_device *pdev = spe_pmu->pdev;  in arm_spe_pmu_irq_probe()
1219 	if (irq_get_percpu_devid_partition(irq, &spe_pmu->supported_cpus)) {  in arm_spe_pmu_irq_probe()
1224 	spe_pmu->irq = irq;  in arm_spe_pmu_irq_probe()
1243 	struct arm_spe_pmu *spe_pmu;  in arm_spe_pmu_device_probe()  local
1255 	spe_pmu = devm_kzalloc(dev, sizeof(*spe_pmu), GFP_KERNEL);  in arm_spe_pmu_device_probe()
1256 	if (!spe_pmu)  in arm_spe_pmu_device_probe()
1259 	spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle));  in arm_spe_pmu_device_probe()
1260 	if (!spe_pmu->handle)  in arm_spe_pmu_device_probe()
1263 	spe_pmu->pdev = pdev;  in arm_spe_pmu_device_probe()
1264 	platform_set_drvdata(pdev, spe_pmu);  in arm_spe_pmu_device_probe()
1266 	ret = arm_spe_pmu_irq_probe(spe_pmu);  in arm_spe_pmu_device_probe()
1270 	ret = arm_spe_pmu_dev_init(spe_pmu);  in arm_spe_pmu_device_probe()
1274 	ret = arm_spe_pmu_perf_init(spe_pmu);  in arm_spe_pmu_device_probe()
1281 	arm_spe_pmu_dev_teardown(spe_pmu);  in arm_spe_pmu_device_probe()
1283 	free_percpu(spe_pmu->handle);  in arm_spe_pmu_device_probe()
1289 	struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev);  in arm_spe_pmu_device_remove()  local
1291 	arm_spe_pmu_perf_destroy(spe_pmu);  in arm_spe_pmu_device_remove()
1292 	arm_spe_pmu_dev_teardown(spe_pmu);  in arm_spe_pmu_device_remove()
1293 	free_percpu(spe_pmu->handle);  in arm_spe_pmu_device_remove()