160 static inline bool supports_cycle_counter(const struct arm_cspmu *cspmu)  in supports_cycle_counter()  argument
162 	return (cspmu->pmcfgr & PMCFGR_CC);  in supports_cycle_counter()
166 static inline u32 counter_size(const struct arm_cspmu *cspmu)  in counter_size()  argument
168 	return FIELD_GET(PMCFGR_SIZE, cspmu->pmcfgr) + 1;  in counter_size()
172 static inline u64 counter_mask(const struct arm_cspmu *cspmu)  in counter_mask()  argument
174 	return GENMASK_ULL(counter_size(cspmu) - 1, 0);  in counter_mask()
178 static inline bool use_64b_counter_reg(const struct arm_cspmu *cspmu)  in use_64b_counter_reg()  argument
180 	return (counter_size(cspmu) > 32);  in use_64b_counter_reg()
200 arm_cspmu_get_event_attrs(const struct arm_cspmu *cspmu)  in arm_cspmu_get_event_attrs()  argument
204 	attrs = devm_kmemdup(cspmu->dev, arm_cspmu_event_attrs,  in arm_cspmu_get_event_attrs()
215 	struct arm_cspmu *cspmu = to_arm_cspmu(dev_get_drvdata(dev));  in arm_cspmu_event_attr_is_visible()  local
221 	if (!supports_cycle_counter(cspmu) &&  in arm_cspmu_event_attr_is_visible()
245 arm_cspmu_get_format_attrs(const struct arm_cspmu *cspmu)  in arm_cspmu_get_format_attrs()  argument
249 	attrs = devm_kmemdup(cspmu->dev, arm_cspmu_format_attrs,  in arm_cspmu_get_format_attrs()
274 	struct arm_cspmu *cspmu = to_arm_cspmu(dev_get_drvdata(dev));  in arm_cspmu_identifier_show()  local
276 	return sysfs_emit(page, "%s\n", cspmu->identifier);  in arm_cspmu_identifier_show()
291 static const char *arm_cspmu_get_identifier(const struct arm_cspmu *cspmu)  in arm_cspmu_get_identifier()  argument
294 		devm_kasprintf(cspmu->dev, GFP_KERNEL, "%x",  in arm_cspmu_get_identifier()
295 			       cspmu->impl.pmiidr);  in arm_cspmu_get_identifier()
307 static const char *arm_cspmu_get_name(const struct arm_cspmu *cspmu)  in arm_cspmu_get_name()  argument
316 	dev = cspmu->dev;  in arm_cspmu_get_name()
347 	struct arm_cspmu *cspmu = to_arm_cspmu(pmu);  in arm_cspmu_cpumask_show()  local
355 		cpumask = &cspmu->active_cpu;  in arm_cspmu_cpumask_show()
358 		cpumask = &cspmu->associated_cpus;  in arm_cspmu_cpumask_show()
379 	int (*impl_init_ops)(struct arm_cspmu *cspmu);
391 static int arm_cspmu_init_impl_ops(struct arm_cspmu *cspmu)  in arm_cspmu_init_impl_ops()  argument
394 	struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;  in arm_cspmu_init_impl_ops()
395 	struct acpi_apmt_node *apmt_node = arm_cspmu_apmt_node(cspmu->dev);  in arm_cspmu_init_impl_ops()
403 	cspmu->impl.pmiidr =  in arm_cspmu_init_impl_ops()
405 				       readl(cspmu->base0 + PMIIDR);  in arm_cspmu_init_impl_ops()
411 		if ((match->pmiidr & mask) == (cspmu->impl.pmiidr & mask)) {  in arm_cspmu_init_impl_ops()
412 			ret = match->impl_init_ops(cspmu);  in arm_cspmu_init_impl_ops()
434 arm_cspmu_alloc_event_attr_group(struct arm_cspmu *cspmu)  in arm_cspmu_alloc_event_attr_group()  argument
437 	struct device *dev = cspmu->dev;  in arm_cspmu_alloc_event_attr_group()
438 	const struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;  in arm_cspmu_alloc_event_attr_group()
447 	event_group->attrs = impl_ops->get_event_attrs(cspmu);  in arm_cspmu_alloc_event_attr_group()
456 arm_cspmu_alloc_format_attr_group(struct arm_cspmu *cspmu)  in arm_cspmu_alloc_format_attr_group()  argument
459 	struct device *dev = cspmu->dev;  in arm_cspmu_alloc_format_attr_group()
467 	format_group->attrs = cspmu->impl.ops.get_format_attrs(cspmu);  in arm_cspmu_alloc_format_attr_group()
476 arm_cspmu_alloc_attr_group(struct arm_cspmu *cspmu)  in arm_cspmu_alloc_attr_group()  argument
479 	struct device *dev = cspmu->dev;  in arm_cspmu_alloc_attr_group()
480 	const struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;  in arm_cspmu_alloc_attr_group()
483 	ret = arm_cspmu_init_impl_ops(cspmu);  in arm_cspmu_alloc_attr_group()
487 	cspmu->identifier = impl_ops->get_identifier(cspmu);  in arm_cspmu_alloc_attr_group()
488 	cspmu->name = impl_ops->get_name(cspmu);  in arm_cspmu_alloc_attr_group()
490 	if (!cspmu->identifier || !cspmu->name)  in arm_cspmu_alloc_attr_group()
498 	attr_groups[0] = arm_cspmu_alloc_event_attr_group(cspmu);  in arm_cspmu_alloc_attr_group()
499 	attr_groups[1] = arm_cspmu_alloc_format_attr_group(cspmu);  in arm_cspmu_alloc_attr_group()
509 static inline void arm_cspmu_reset_counters(struct arm_cspmu *cspmu)  in arm_cspmu_reset_counters()  argument
515 	writel(pmcr, cspmu->base0 + PMCR);  in arm_cspmu_reset_counters()
518 static inline void arm_cspmu_start_counters(struct arm_cspmu *cspmu)  in arm_cspmu_start_counters()  argument
520 	writel(PMCR_E, cspmu->base0 + PMCR);  in arm_cspmu_start_counters()
523 static inline void arm_cspmu_stop_counters(struct arm_cspmu *cspmu)  in arm_cspmu_stop_counters()  argument
525 	writel(0, cspmu->base0 + PMCR);  in arm_cspmu_stop_counters()
531 	struct arm_cspmu *cspmu = to_arm_cspmu(pmu);  in arm_cspmu_enable()  local
533 	disabled = bitmap_empty(cspmu->hw_events.used_ctrs,  in arm_cspmu_enable()
534 				cspmu->num_logical_ctrs);  in arm_cspmu_enable()
539 	arm_cspmu_start_counters(cspmu);  in arm_cspmu_enable()
544 	struct arm_cspmu *cspmu = to_arm_cspmu(pmu);  in arm_cspmu_disable()  local
546 	arm_cspmu_stop_counters(cspmu);  in arm_cspmu_disable()
553 	struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);  in arm_cspmu_get_event_idx()  local
555 	if (supports_cycle_counter(cspmu)) {  in arm_cspmu_get_event_idx()
556 		if (cspmu->impl.ops.is_cycle_counter_event(event)) {  in arm_cspmu_get_event_idx()
558 			if (test_and_set_bit(cspmu->cycle_counter_logical_idx,  in arm_cspmu_get_event_idx()
562 			return cspmu->cycle_counter_logical_idx;  in arm_cspmu_get_event_idx()
571 					  cspmu->cycle_counter_logical_idx);  in arm_cspmu_get_event_idx()
572 		if (idx >= cspmu->cycle_counter_logical_idx) {  in arm_cspmu_get_event_idx()
575 				cspmu->num_logical_ctrs,  in arm_cspmu_get_event_idx()
576 				cspmu->cycle_counter_logical_idx + 1);  in arm_cspmu_get_event_idx()
580 					  cspmu->num_logical_ctrs);  in arm_cspmu_get_event_idx()
583 	if (idx >= cspmu->num_logical_ctrs)  in arm_cspmu_get_event_idx()
633 	struct arm_cspmu *cspmu;  in arm_cspmu_event_init()  local
636 	cspmu = to_arm_cspmu(event->pmu);  in arm_cspmu_event_init()
646 		dev_dbg(cspmu->pmu.dev,  in arm_cspmu_event_init()
652 		dev_dbg(cspmu->pmu.dev,  in arm_cspmu_event_init()
661 	if (!cpumask_test_cpu(event->cpu, &cspmu->associated_cpus)) {  in arm_cspmu_event_init()
662 		dev_dbg(cspmu->pmu.dev,  in arm_cspmu_event_init()
668 	event->cpu = cpumask_first(&cspmu->active_cpu);  in arm_cspmu_event_init()
684 	hwc->config = cspmu->impl.ops.event_type(event);  in arm_cspmu_event_init()
697 	struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);  in arm_cspmu_write_counter()  local
699 	if (use_64b_counter_reg(cspmu)) {  in arm_cspmu_write_counter()
702 		writeq(val, cspmu->base1 + offset);  in arm_cspmu_write_counter()
706 		writel(lower_32_bits(val), cspmu->base1 + offset);  in arm_cspmu_write_counter()
714 	struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);  in arm_cspmu_read_counter()  local
716 	if (use_64b_counter_reg(cspmu)) {  in arm_cspmu_read_counter()
718 		counter_addr = cspmu->base1 + offset;  in arm_cspmu_read_counter()
720 		return cspmu->has_atomic_dword ?  in arm_cspmu_read_counter()
726 	return readl(cspmu->base1 + offset);  in arm_cspmu_read_counter()
737 	struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);  in arm_cspmu_set_event_period()  local
738 	u64 val = counter_mask(cspmu) >> 1ULL;  in arm_cspmu_set_event_period()
744 static void arm_cspmu_enable_counter(struct arm_cspmu *cspmu, int idx)  in arm_cspmu_enable_counter()  argument
754 	writel(BIT(reg_bit), cspmu->base0 + inten_off);  in arm_cspmu_enable_counter()
755 	writel(BIT(reg_bit), cspmu->base0 + cnten_off);  in arm_cspmu_enable_counter()
758 static void arm_cspmu_disable_counter(struct arm_cspmu *cspmu, int idx)  in arm_cspmu_disable_counter()  argument
768 	writel(BIT(reg_bit), cspmu->base0 + cnten_off);  in arm_cspmu_disable_counter()
769 	writel(BIT(reg_bit), cspmu->base0 + inten_off);  in arm_cspmu_disable_counter()
774 	struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);  in arm_cspmu_event_update()  local
783 	delta = (now - prev) & counter_mask(cspmu);  in arm_cspmu_event_update()
787 static inline void arm_cspmu_set_event(struct arm_cspmu *cspmu,  in arm_cspmu_set_event()  argument
792 	writel(hwc->config, cspmu->base0 + offset);  in arm_cspmu_set_event()
795 static inline void arm_cspmu_set_ev_filter(struct arm_cspmu *cspmu,  in arm_cspmu_set_ev_filter()  argument
801 	writel(filter, cspmu->base0 + offset);  in arm_cspmu_set_ev_filter()
804 static inline void arm_cspmu_set_cc_filter(struct arm_cspmu *cspmu, u32 filter)  in arm_cspmu_set_cc_filter()  argument
808 	writel(filter, cspmu->base0 + offset);  in arm_cspmu_set_cc_filter()
813 	struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);  in arm_cspmu_start()  local
823 	filter = cspmu->impl.ops.event_filter(event);  in arm_cspmu_start()
825 	if (event->hw.extra_reg.idx == cspmu->cycle_counter_logical_idx) {  in arm_cspmu_start()
826 		arm_cspmu_set_cc_filter(cspmu, filter);  in arm_cspmu_start()
828 		arm_cspmu_set_event(cspmu, hwc);  in arm_cspmu_start()
829 		arm_cspmu_set_ev_filter(cspmu, hwc, filter);  in arm_cspmu_start()
834 	arm_cspmu_enable_counter(cspmu, hwc->idx);  in arm_cspmu_start()
839 	struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);  in arm_cspmu_stop()  local
845 	arm_cspmu_disable_counter(cspmu, hwc->idx);  in arm_cspmu_stop()
851 static inline u32 to_phys_idx(struct arm_cspmu *cspmu, u32 idx)  in to_phys_idx()  argument
853 	return (idx == cspmu->cycle_counter_logical_idx) ?  in to_phys_idx()
859 	struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);  in arm_cspmu_add()  local
860 	struct arm_cspmu_hw_events *hw_events = &cspmu->hw_events;  in arm_cspmu_add()
865 					   &cspmu->associated_cpus)))  in arm_cspmu_add()
873 	hwc->idx = to_phys_idx(cspmu, idx);  in arm_cspmu_add()
888 	struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);  in arm_cspmu_del()  local
889 	struct arm_cspmu_hw_events *hw_events = &cspmu->hw_events;  in arm_cspmu_del()
910 	struct arm_cspmu *cspmu;  in arm_cspmu_alloc()  local
913 	cspmu = devm_kzalloc(dev, sizeof(*cspmu), GFP_KERNEL);  in arm_cspmu_alloc()
914 	if (!cspmu)  in arm_cspmu_alloc()
917 	cspmu->dev = dev;  in arm_cspmu_alloc()
918 	platform_set_drvdata(pdev, cspmu);  in arm_cspmu_alloc()
921 	cspmu->has_atomic_dword = apmt_node->flags & ACPI_APMT_FLAGS_ATOMIC;  in arm_cspmu_alloc()
923 	return cspmu;  in arm_cspmu_alloc()
926 static int arm_cspmu_init_mmio(struct arm_cspmu *cspmu)  in arm_cspmu_init_mmio()  argument
931 	dev = cspmu->dev;  in arm_cspmu_init_mmio()
935 	cspmu->base0 = devm_platform_ioremap_resource(pdev, 0);  in arm_cspmu_init_mmio()
936 	if (IS_ERR(cspmu->base0)) {  in arm_cspmu_init_mmio()
938 		return PTR_ERR(cspmu->base0);  in arm_cspmu_init_mmio()
942 	cspmu->base1 = cspmu->base0;  in arm_cspmu_init_mmio()
944 		cspmu->base1 = devm_platform_ioremap_resource(pdev, 1);  in arm_cspmu_init_mmio()
945 		if (IS_ERR(cspmu->base1)) {  in arm_cspmu_init_mmio()
947 			return PTR_ERR(cspmu->base1);  in arm_cspmu_init_mmio()
951 	cspmu->pmcfgr = readl(cspmu->base0 + PMCFGR);  in arm_cspmu_init_mmio()
953 	cspmu->num_logical_ctrs = FIELD_GET(PMCFGR_N, cspmu->pmcfgr) + 1;  in arm_cspmu_init_mmio()
955 	cspmu->cycle_counter_logical_idx = ARM_CSPMU_MAX_HW_CNTRS;  in arm_cspmu_init_mmio()
957 	if (supports_cycle_counter(cspmu)) {  in arm_cspmu_init_mmio()
963 		cspmu->cycle_counter_logical_idx =  in arm_cspmu_init_mmio()
964 			(cspmu->num_logical_ctrs <= ARM_CSPMU_CYCLE_CNTR_IDX) ?  in arm_cspmu_init_mmio()
965 				cspmu->num_logical_ctrs - 1 :  in arm_cspmu_init_mmio()
969 	cspmu->num_set_clr_reg =  in arm_cspmu_init_mmio()
970 		DIV_ROUND_UP(cspmu->num_logical_ctrs,  in arm_cspmu_init_mmio()
973 	cspmu->hw_events.events =  in arm_cspmu_init_mmio()
974 		devm_kcalloc(dev, cspmu->num_logical_ctrs,  in arm_cspmu_init_mmio()
975 			     sizeof(*cspmu->hw_events.events), GFP_KERNEL);  in arm_cspmu_init_mmio()
977 	if (!cspmu->hw_events.events)  in arm_cspmu_init_mmio()
983 static inline int arm_cspmu_get_reset_overflow(struct arm_cspmu *cspmu,  in arm_cspmu_get_reset_overflow()  argument
990 	for (i = 0; i < cspmu->num_set_clr_reg; ++i) {  in arm_cspmu_get_reset_overflow()
991 		pmovs[i] = readl(cspmu->base1 + pmovclr_offset);  in arm_cspmu_get_reset_overflow()
993 		writel(pmovs[i], cspmu->base1 + pmovclr_offset);  in arm_cspmu_get_reset_overflow()
1004 	struct arm_cspmu *cspmu = dev;  in arm_cspmu_handle_irq()  local
1008 	arm_cspmu_stop_counters(cspmu);  in arm_cspmu_handle_irq()
1010 	has_overflowed = arm_cspmu_get_reset_overflow(cspmu, (u32 *)pmovs);  in arm_cspmu_handle_irq()
1014 	for_each_set_bit(idx, cspmu->hw_events.used_ctrs,  in arm_cspmu_handle_irq()
1015 			cspmu->num_logical_ctrs) {  in arm_cspmu_handle_irq()
1016 		event = cspmu->hw_events.events[idx];  in arm_cspmu_handle_irq()
1031 	arm_cspmu_start_counters(cspmu);  in arm_cspmu_handle_irq()
1035 static int arm_cspmu_request_irq(struct arm_cspmu *cspmu)  in arm_cspmu_request_irq()  argument
1041 	dev = cspmu->dev;  in arm_cspmu_request_irq()
1051 			       cspmu);  in arm_cspmu_request_irq()
1057 	cspmu->irq = irq;  in arm_cspmu_request_irq()
1089 static int arm_cspmu_acpi_get_cpus(struct arm_cspmu *cspmu)  in arm_cspmu_acpi_get_cpus()  argument
1095 	apmt_node = arm_cspmu_apmt_node(cspmu->dev);  in arm_cspmu_acpi_get_cpus()
1102 				cpumask_set_cpu(cpu, &cspmu->associated_cpus);  in arm_cspmu_acpi_get_cpus()
1112 			cpumask_set_cpu(cpu, &cspmu->associated_cpus);  in arm_cspmu_acpi_get_cpus()
1116 	if (cpumask_empty(&cspmu->associated_cpus)) {  in arm_cspmu_acpi_get_cpus()
1117 		dev_dbg(cspmu->dev, "No cpu associated with the PMU\n");  in arm_cspmu_acpi_get_cpus()
1124 static int arm_cspmu_acpi_get_cpus(struct arm_cspmu *cspmu)  in arm_cspmu_acpi_get_cpus()  argument
1130 static int arm_cspmu_get_cpus(struct arm_cspmu *cspmu)  in arm_cspmu_get_cpus()  argument
1132 	return arm_cspmu_acpi_get_cpus(cspmu);  in arm_cspmu_get_cpus()
1135 static int arm_cspmu_register_pmu(struct arm_cspmu *cspmu)  in arm_cspmu_register_pmu()  argument
1140 	attr_groups = arm_cspmu_alloc_attr_group(cspmu);  in arm_cspmu_register_pmu()
1145 				       &cspmu->cpuhp_node);  in arm_cspmu_register_pmu()
1150 	if (cspmu->irq == 0)  in arm_cspmu_register_pmu()
1153 	cspmu->pmu = (struct pmu){  in arm_cspmu_register_pmu()
1169 	arm_cspmu_stop_counters(cspmu);  in arm_cspmu_register_pmu()
1170 	arm_cspmu_reset_counters(cspmu);  in arm_cspmu_register_pmu()
1172 	ret = perf_pmu_register(&cspmu->pmu, cspmu->name, -1);  in arm_cspmu_register_pmu()
1175 					    &cspmu->cpuhp_node);  in arm_cspmu_register_pmu()
1184 	struct arm_cspmu *cspmu;  in arm_cspmu_device_probe()  local
1186 	cspmu = arm_cspmu_alloc(pdev);  in arm_cspmu_device_probe()
1187 	if (!cspmu)  in arm_cspmu_device_probe()
1190 	ret = arm_cspmu_init_mmio(cspmu);  in arm_cspmu_device_probe()
1194 	ret = arm_cspmu_request_irq(cspmu);  in arm_cspmu_device_probe()
1198 	ret = arm_cspmu_get_cpus(cspmu);  in arm_cspmu_device_probe()
1202 	ret = arm_cspmu_register_pmu(cspmu);  in arm_cspmu_device_probe()
1211 	struct arm_cspmu *cspmu = platform_get_drvdata(pdev);  in arm_cspmu_device_remove()  local
1213 	perf_pmu_unregister(&cspmu->pmu);  in arm_cspmu_device_remove()
1214 	cpuhp_state_remove_instance(arm_cspmu_cpuhp_state, &cspmu->cpuhp_node);  in arm_cspmu_device_remove()
1235 static void arm_cspmu_set_active_cpu(int cpu, struct arm_cspmu *cspmu)  in arm_cspmu_set_active_cpu()  argument
1237 	cpumask_set_cpu(cpu, &cspmu->active_cpu);  in arm_cspmu_set_active_cpu()
1238 	if (cspmu->irq)  in arm_cspmu_set_active_cpu()
1239 		WARN_ON(irq_set_affinity(cspmu->irq, &cspmu->active_cpu));  in arm_cspmu_set_active_cpu()
1244 	struct arm_cspmu *cspmu =  in arm_cspmu_cpu_online()  local
1247 	if (!cpumask_test_cpu(cpu, &cspmu->associated_cpus))  in arm_cspmu_cpu_online()
1251 	if (!cpumask_empty(&cspmu->active_cpu))  in arm_cspmu_cpu_online()
1255 	arm_cspmu_set_active_cpu(cpu, cspmu);  in arm_cspmu_cpu_online()
1265 	struct arm_cspmu *cspmu =  in arm_cspmu_cpu_teardown()  local
1269 	if (!cpumask_test_and_clear_cpu(cpu, &cspmu->active_cpu))  in arm_cspmu_cpu_teardown()
1273 	cpumask_and(&online_supported, &cspmu->associated_cpus,  in arm_cspmu_cpu_teardown()
1280 	perf_pmu_migrate_context(&cspmu->pmu, cpu, dst);  in arm_cspmu_cpu_teardown()
1281 	arm_cspmu_set_active_cpu(dst, cspmu);  in arm_cspmu_cpu_teardown()