45 #define to_ddr_pmu(p)		container_of(p, struct ddr_pmu, pmu)
57 	struct pmu pmu;  member
84 	struct ddr_pmu *pmu = dev_get_drvdata(dev);  in ddr_perf_identifier_show()  local
86 	return sysfs_emit(page, "%s\n", pmu->devtype_data->identifier);  in ddr_perf_identifier_show()
104 	struct ddr_pmu *pmu = dev_get_drvdata(dev);  in ddr_perf_cpumask_show()  local
106 	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));  in ddr_perf_cpumask_show()
271 static void ddr_perf_clear_counter(struct ddr_pmu *pmu, int counter)  in ddr_perf_clear_counter()  argument
274 		writel(0, pmu->base + PMC(counter) + 0x4);  in ddr_perf_clear_counter()
275 		writel(0, pmu->base + PMC(counter));  in ddr_perf_clear_counter()
277 		writel(0, pmu->base + PMC(counter));  in ddr_perf_clear_counter()
281 static u64 ddr_perf_read_counter(struct ddr_pmu *pmu, int counter)  in ddr_perf_read_counter()  argument
287 		val = readl_relaxed(pmu->base + PMC(counter));  in ddr_perf_read_counter()
293 		val_upper = readl_relaxed(pmu->base + PMC(counter) + 0x4);  in ddr_perf_read_counter()
294 		val_lower = readl_relaxed(pmu->base + PMC(counter));  in ddr_perf_read_counter()
295 	} while (val_upper != readl_relaxed(pmu->base + PMC(counter) + 0x4));  in ddr_perf_read_counter()
304 static void ddr_perf_counter_global_config(struct ddr_pmu *pmu, bool enable)  in ddr_perf_counter_global_config()  argument
308 	ctrl = readl_relaxed(pmu->base + PMGC0);  in ddr_perf_counter_global_config()
322 		writel(ctrl, pmu->base + PMGC0);  in ddr_perf_counter_global_config()
330 		writel(ctrl, pmu->base + PMGC0);  in ddr_perf_counter_global_config()
334 		writel(ctrl, pmu->base + PMGC0);  in ddr_perf_counter_global_config()
338 static void ddr_perf_counter_local_config(struct ddr_pmu *pmu, int config,  in ddr_perf_counter_local_config()  argument
343 	ctrl_a = readl_relaxed(pmu->base + PMLCA(counter));  in ddr_perf_counter_local_config()
347 		writel(ctrl_a, pmu->base + PMLCA(counter));  in ddr_perf_counter_local_config()
349 		ddr_perf_clear_counter(pmu, counter);  in ddr_perf_counter_local_config()
356 		writel(ctrl_a, pmu->base + PMLCA(counter));  in ddr_perf_counter_local_config()
360 		writel(ctrl_a, pmu->base + PMLCA(counter));  in ddr_perf_counter_local_config()
364 static void ddr_perf_monitor_config(struct ddr_pmu *pmu, int cfg, int cfg1, int cfg2)  in ddr_perf_monitor_config()  argument
372 	pmcfg1 = readl_relaxed(pmu->base + PMCFG1);  in ddr_perf_monitor_config()
391 	writel(pmcfg1, pmu->base + PMCFG1);  in ddr_perf_monitor_config()
393 	pmcfg2 = readl_relaxed(pmu->base + PMCFG2);  in ddr_perf_monitor_config()
396 	writel(pmcfg2, pmu->base + PMCFG2);  in ddr_perf_monitor_config()
401 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_update()  local
406 	new_raw_count = ddr_perf_read_counter(pmu, counter);  in ddr_perf_event_update()
410 	ddr_perf_clear_counter(pmu, counter);  in ddr_perf_event_update()
415 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_init()  local
419 	if (event->attr.type != event->pmu->type)  in ddr_perf_event_init()
426 		dev_warn(pmu->dev, "Can't provide per-task data!\n");  in ddr_perf_event_init()
435 	if (event->group_leader->pmu != event->pmu &&  in ddr_perf_event_init()
440 		if (sibling->pmu != event->pmu &&  in ddr_perf_event_init()
445 	event->cpu = pmu->cpu;  in ddr_perf_event_init()
453 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_start()  local
459 	ddr_perf_counter_local_config(pmu, event->attr.config, counter, true);  in ddr_perf_event_start()
465 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_add()  local
474 	pmu->events[counter] = event;  in ddr_perf_event_add()
475 	pmu->active_events++;  in ddr_perf_event_add()
480 	ddr_perf_monitor_config(pmu, cfg, cfg1, cfg2);  in ddr_perf_event_add()
490 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_stop()  local
494 	ddr_perf_counter_local_config(pmu, event->attr.config, counter, false);  in ddr_perf_event_stop()
502 	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);  in ddr_perf_event_del()  local
507 	pmu->active_events--;  in ddr_perf_event_del()
511 static void ddr_perf_pmu_enable(struct pmu *pmu)  in ddr_perf_pmu_enable()  argument
513 	struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);  in ddr_perf_pmu_enable()
518 static void ddr_perf_pmu_disable(struct pmu *pmu)  in ddr_perf_pmu_disable()  argument
520 	struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);  in ddr_perf_pmu_disable()
525 static void ddr_perf_init(struct ddr_pmu *pmu, void __iomem *base,  in ddr_perf_init()  argument
528 	*pmu = (struct ddr_pmu) {  in ddr_perf_init()
529 		.pmu = (struct pmu) {  in ddr_perf_init()
550 	struct ddr_pmu *pmu = (struct ddr_pmu *)p;  in ddr_perf_irq_handler()  local
566 		if (!pmu->events[i])  in ddr_perf_irq_handler()
569 		event = pmu->events[i];  in ddr_perf_irq_handler()
574 	ddr_perf_counter_global_config(pmu, true);  in ddr_perf_irq_handler()
581 	struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node);  in ddr_perf_offline_cpu()  local
584 	if (cpu != pmu->cpu)  in ddr_perf_offline_cpu()
591 	perf_pmu_migrate_context(&pmu->pmu, cpu, target);  in ddr_perf_offline_cpu()
592 	pmu->cpu = target;  in ddr_perf_offline_cpu()
594 	WARN_ON(irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)));  in ddr_perf_offline_cpu()
601 	struct ddr_pmu *pmu;  in ddr_perf_probe()  local
610 	pmu = devm_kzalloc(&pdev->dev, sizeof(*pmu), GFP_KERNEL);  in ddr_perf_probe()
611 	if (!pmu)  in ddr_perf_probe()
614 	ddr_perf_init(pmu, base, &pdev->dev);  in ddr_perf_probe()
616 	pmu->devtype_data = of_device_get_match_data(&pdev->dev);  in ddr_perf_probe()
618 	platform_set_drvdata(pdev, pmu);  in ddr_perf_probe()
620 	pmu->id = ida_simple_get(&ddr_ida, 0, 0, GFP_KERNEL);  in ddr_perf_probe()
621 	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d", pmu->id);  in ddr_perf_probe()
627 	pmu->cpu = raw_smp_processor_id();  in ddr_perf_probe()
634 	pmu->cpuhp_state = ret;  in ddr_perf_probe()
637 	ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_probe()
652 			       DDR_CPUHP_CB_NAME, pmu);  in ddr_perf_probe()
658 	pmu->irq = irq;  in ddr_perf_probe()
659 	ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu));  in ddr_perf_probe()
661 		dev_err(pmu->dev, "Failed to set interrupt affinity\n");  in ddr_perf_probe()
665 	ret = perf_pmu_register(&pmu->pmu, name, -1);  in ddr_perf_probe()
672 	cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_probe()
674 	cpuhp_remove_multi_state(pmu->cpuhp_state);  in ddr_perf_probe()
677 	ida_simple_remove(&ddr_ida, pmu->id);  in ddr_perf_probe()
684 	struct ddr_pmu *pmu = platform_get_drvdata(pdev);  in ddr_perf_remove()  local
686 	cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);  in ddr_perf_remove()
687 	cpuhp_remove_multi_state(pmu->cpuhp_state);  in ddr_perf_remove()
689 	perf_pmu_unregister(&pmu->pmu);  in ddr_perf_remove()
691 	ida_simple_remove(&ddr_ida, pmu->id);  in ddr_perf_remove()