1009f1315SGregory CLEMENT // SPDX-License-Identifier: GPL-2.0-only
2e12f12acSThomas Petazzoni /*
3e12f12acSThomas Petazzoni * Coherency fabric (Aurora) support for Armada 370, 375, 38x and XP
4009f1315SGregory CLEMENT * platforms.
5009f1315SGregory CLEMENT *
6009f1315SGregory CLEMENT * Copyright (C) 2012 Marvell
7009f1315SGregory CLEMENT *
8009f1315SGregory CLEMENT * Yehuda Yitschak <yehuday@marvell.com>
9009f1315SGregory CLEMENT * Gregory Clement <gregory.clement@free-electrons.com>
10009f1315SGregory CLEMENT * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
11009f1315SGregory CLEMENT *
12009f1315SGregory CLEMENT * The Armada 370, 375, 38x and XP SOCs have a coherency fabric which is
13009f1315SGregory CLEMENT * responsible for ensuring hardware coherency between all CPUs and between
14009f1315SGregory CLEMENT * CPUs and I/O masters. This file initializes the coherency fabric and
15e12f12acSThomas Petazzoni * supplies basic routines for configuring and controlling hardware coherency
16009f1315SGregory CLEMENT */
17009f1315SGregory CLEMENT
18009f1315SGregory CLEMENT #define pr_fmt(fmt) "mvebu-coherency: " fmt
19009f1315SGregory CLEMENT
20009f1315SGregory CLEMENT #include <linux/kernel.h>
215ab5afd8SThomas Petazzoni #include <linux/init.h>
225ab5afd8SThomas Petazzoni #include <linux/of_address.h>
23009f1315SGregory CLEMENT #include <linux/io.h>
24009f1315SGregory CLEMENT #include <linux/smp.h>
25009f1315SGregory CLEMENT #include <linux/dma-map-ops.h>
26009f1315SGregory CLEMENT #include <linux/platform_device.h>
27009f1315SGregory CLEMENT #include <linux/slab.h>
280a0f0d8bSChristoph Hellwig #include <linux/mbus.h>
29e60304f8SGregory CLEMENT #include <linux/pci.h>
305ab5afd8SThomas Petazzoni #include <asm/smp_plat.h>
315ab5afd8SThomas Petazzoni #include <asm/cacheflush.h>
32b0063aadSThomas Petazzoni #include <asm/mach/map.h>
33009f1315SGregory CLEMENT #include <asm/dma-mapping.h>
34580ff0eeSThomas Petazzoni #include "coherency.h"
35497a9230SThomas Petazzoni #include "mvebu-soc-id.h"
361bd4d8a6SThomas Petazzoni
37b12634e3SJisheng Zhang unsigned long coherency_phys_base;
3839438567SThomas Petazzoni void __iomem *coherency_base;
39009f1315SGregory CLEMENT static void __iomem *coherency_cpu_base;
408bd26e3aSPaul Gortmaker static void __iomem *cpu_config_base;
41ccd6a131SGregory CLEMENT
42e60304f8SGregory CLEMENT /* Coherency fabric registers */
43d492cccaSThomas Petazzoni #define IO_SYNC_BARRIER_CTL_OFFSET 0x0
44009f1315SGregory CLEMENT
45009f1315SGregory CLEMENT enum {
46e60304f8SGregory CLEMENT COHERENCY_FABRIC_TYPE_NONE,
47e60304f8SGregory CLEMENT COHERENCY_FABRIC_TYPE_ARMADA_370_XP,
48924d38f4SThomas Petazzoni COHERENCY_FABRIC_TYPE_ARMADA_375,
49501f928eSThomas Petazzoni COHERENCY_FABRIC_TYPE_ARMADA_380,
50924d38f4SThomas Petazzoni };
5177fa4b9aSThomas Petazzoni
52d0de9323SThomas Petazzoni static const struct of_device_id of_coherency_table[] = {
53924d38f4SThomas Petazzoni {.compatible = "marvell,coherency-fabric",
54924d38f4SThomas Petazzoni .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_370_XP },
55444d2d33SUwe Kleine-König {.compatible = "marvell,armada-375-coherency-fabric",
56924d38f4SThomas Petazzoni .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_375 },
57924d38f4SThomas Petazzoni {.compatible = "marvell,armada-380-coherency-fabric",
5877fa4b9aSThomas Petazzoni .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_380 },
5977fa4b9aSThomas Petazzoni { /* end of list */ },
60d0de9323SThomas Petazzoni };
61d0de9323SThomas Petazzoni
62009f1315SGregory CLEMENT /* Functions defined in coherency_ll.S */
63009f1315SGregory CLEMENT int ll_enable_coherency(void);
64009f1315SGregory CLEMENT void ll_add_cpu_to_smp_group(void);
652e8a5942SGregory CLEMENT
662e8a5942SGregory CLEMENT #define CPU_CONFIG_SHARED_L2 BIT(16)
672e8a5942SGregory CLEMENT
68009f1315SGregory CLEMENT /*
69d492cccaSThomas Petazzoni * Disable the "Shared L2 Present" bit in CPU Configuration register
70d492cccaSThomas Petazzoni * on Armada XP.
71d492cccaSThomas Petazzoni *
72d492cccaSThomas Petazzoni * The "Shared L2 Present" bit affects the "level of coherence" value
73d492cccaSThomas Petazzoni * in the clidr CP15 register. Cache operation functions such as
74d492cccaSThomas Petazzoni * "flush all" and "invalidate all" operate on all the cache levels
75d492cccaSThomas Petazzoni * that included in the defined level of coherence. When HW I/O
76d492cccaSThomas Petazzoni * coherency is used, this bit causes unnecessary flushes of the L2
77d492cccaSThomas Petazzoni * cache.
78d492cccaSThomas Petazzoni */
armada_xp_clear_shared_l2(void)79d492cccaSThomas Petazzoni static void armada_xp_clear_shared_l2(void)
80d492cccaSThomas Petazzoni {
81d492cccaSThomas Petazzoni u32 reg;
82d492cccaSThomas Petazzoni
83d492cccaSThomas Petazzoni if (!cpu_config_base)
84d492cccaSThomas Petazzoni return;
85d492cccaSThomas Petazzoni
86d492cccaSThomas Petazzoni reg = readl(cpu_config_base);
87d492cccaSThomas Petazzoni reg &= ~CPU_CONFIG_SHARED_L2;
88d492cccaSThomas Petazzoni writel(reg, cpu_config_base);
89d492cccaSThomas Petazzoni }
90d492cccaSThomas Petazzoni
mvebu_hwcc_notifier(struct notifier_block * nb,unsigned long event,void * __dev)91d492cccaSThomas Petazzoni static int mvebu_hwcc_notifier(struct notifier_block *nb,
92d492cccaSThomas Petazzoni unsigned long event, void *__dev)
93d492cccaSThomas Petazzoni {
94b0063aadSThomas Petazzoni struct device *dev = __dev;
95e60304f8SGregory CLEMENT
96e60304f8SGregory CLEMENT if (event != BUS_NOTIFY_ADD_DEVICE)
97e60304f8SGregory CLEMENT return NOTIFY_DONE;
98e60304f8SGregory CLEMENT dev->dma_coherent = true;
99e60304f8SGregory CLEMENT
100e60304f8SGregory CLEMENT return NOTIFY_OK;
101*ae626eb9SChristoph Hellwig }
102e60304f8SGregory CLEMENT
103e60304f8SGregory CLEMENT static struct notifier_block mvebu_hwcc_nb = {
104e60304f8SGregory CLEMENT .notifier_call = mvebu_hwcc_notifier,
105e60304f8SGregory CLEMENT };
106b0063aadSThomas Petazzoni
107b0063aadSThomas Petazzoni static struct notifier_block mvebu_hwcc_pci_nb __maybe_unused = {
108e60304f8SGregory CLEMENT .notifier_call = mvebu_hwcc_notifier,
109e60304f8SGregory CLEMENT };
11060f23952SArnd Bergmann
armada_xp_clear_l2_starting(unsigned int cpu)111a728b977SEzequiel Garcia static int armada_xp_clear_l2_starting(unsigned int cpu)
112a728b977SEzequiel Garcia {
113a728b977SEzequiel Garcia armada_xp_clear_shared_l2();
1147fbbaebfSSebastian Andrzej Siewior return 0;
115d492cccaSThomas Petazzoni }
116d492cccaSThomas Petazzoni
armada_370_coherency_init(struct device_node * np)1177fbbaebfSSebastian Andrzej Siewior static void __init armada_370_coherency_init(struct device_node *np)
118d492cccaSThomas Petazzoni {
119d492cccaSThomas Petazzoni struct resource res;
120924d38f4SThomas Petazzoni struct device_node *cpu_config_np;
121009f1315SGregory CLEMENT
122580ff0eeSThomas Petazzoni of_address_to_resource(np, 0, &res);
123d492cccaSThomas Petazzoni coherency_phys_base = res.start;
124924d38f4SThomas Petazzoni /*
125580ff0eeSThomas Petazzoni * Ensure secondary CPUs will see the updated value,
126580ff0eeSThomas Petazzoni * which they read before they join the coherency
127580ff0eeSThomas Petazzoni * fabric, and therefore before they are coherent with
128580ff0eeSThomas Petazzoni * the boot CPU cache.
129580ff0eeSThomas Petazzoni */
130580ff0eeSThomas Petazzoni sync_cache_w(&coherency_phys_base);
131580ff0eeSThomas Petazzoni coherency_base = of_iomap(np, 0);
132580ff0eeSThomas Petazzoni coherency_cpu_base = of_iomap(np, 1);
133580ff0eeSThomas Petazzoni
134009f1315SGregory CLEMENT cpu_config_np = of_find_compatible_node(NULL, NULL,
135e60304f8SGregory CLEMENT "marvell,armada-xp-cpu-config");
136d492cccaSThomas Petazzoni if (!cpu_config_np)
137d492cccaSThomas Petazzoni goto exit;
138d492cccaSThomas Petazzoni
139d492cccaSThomas Petazzoni cpu_config_base = of_iomap(cpu_config_np, 0);
140d492cccaSThomas Petazzoni if (!cpu_config_base) {
141d492cccaSThomas Petazzoni of_node_put(cpu_config_np);
142d492cccaSThomas Petazzoni goto exit;
143d492cccaSThomas Petazzoni }
144d492cccaSThomas Petazzoni
145d492cccaSThomas Petazzoni of_node_put(cpu_config_np);
146d492cccaSThomas Petazzoni
147d492cccaSThomas Petazzoni cpuhp_setup_state_nocalls(CPUHP_AP_ARM_MVEBU_COHERENCY,
148d492cccaSThomas Petazzoni "arm/mvebu/coherency:starting",
149d492cccaSThomas Petazzoni armada_xp_clear_l2_starting, NULL);
1507fbbaebfSSebastian Andrzej Siewior exit:
15173c1b41eSThomas Gleixner set_cpu_coherent();
1527fbbaebfSSebastian Andrzej Siewior }
153d492cccaSThomas Petazzoni
154952f4ca7SGregory CLEMENT /*
155924d38f4SThomas Petazzoni * This ioremap hook is used on Armada 375/38x to ensure that all MMIO
156924d38f4SThomas Petazzoni * areas are mapped as MT_UNCACHED instead of MT_DEVICE. This is
157497a9230SThomas Petazzoni * needed for the HW I/O coherency mechanism to work properly without
158c5379ba8SThomas Petazzoni * deadlock.
159c5379ba8SThomas Petazzoni */
160c5379ba8SThomas Petazzoni static void __iomem *
armada_wa_ioremap_caller(phys_addr_t phys_addr,size_t size,unsigned int mtype,void * caller)161c5379ba8SThomas Petazzoni armada_wa_ioremap_caller(phys_addr_t phys_addr, size_t size,
162497a9230SThomas Petazzoni unsigned int mtype, void *caller)
163497a9230SThomas Petazzoni {
164c5379ba8SThomas Petazzoni mtype = MT_UNCACHED;
165497a9230SThomas Petazzoni return __arm_ioremap_caller(phys_addr, size, mtype, caller);
166497a9230SThomas Petazzoni }
167497a9230SThomas Petazzoni
armada_375_380_coherency_init(struct device_node * np)168497a9230SThomas Petazzoni static void __init armada_375_380_coherency_init(struct device_node *np)
169497a9230SThomas Petazzoni {
170497a9230SThomas Petazzoni struct device_node *cache_dn;
171d0de9323SThomas Petazzoni
17277fa4b9aSThomas Petazzoni coherency_cpu_base = of_iomap(np, 0);
173497a9230SThomas Petazzoni arch_ioremap_caller = armada_wa_ioremap_caller;
174497a9230SThomas Petazzoni pci_ioremap_set_mem_type(MT_UNCACHED);
17577fa4b9aSThomas Petazzoni
176c5379ba8SThomas Petazzoni /*
1776a02734dSThomas Petazzoni * We should switch the PL310 to I/O coherency mode only if
178497a9230SThomas Petazzoni * I/O coherency is actually enabled.
179497a9230SThomas Petazzoni */
180dcad6887SThomas Petazzoni if (!coherency_available())
181dcad6887SThomas Petazzoni return;
182dcad6887SThomas Petazzoni
183dcad6887SThomas Petazzoni /*
184dcad6887SThomas Petazzoni * Add the PL310 property "arm,io-coherent". This makes sure the
185dcad6887SThomas Petazzoni * outer sync operation is not used, which allows to
186dcad6887SThomas Petazzoni * workaround the system erratum that causes deadlocks when
187497a9230SThomas Petazzoni * doing PCIe in an SMP situation on Armada 375 and Armada
188497a9230SThomas Petazzoni * 38x.
189497a9230SThomas Petazzoni */
190497a9230SThomas Petazzoni for_each_compatible_node(cache_dn, NULL, "arm,pl310-cache") {
191497a9230SThomas Petazzoni struct property *p;
192497a9230SThomas Petazzoni
193497a9230SThomas Petazzoni p = kzalloc(sizeof(*p), GFP_KERNEL);
194497a9230SThomas Petazzoni p->name = kstrdup("arm,io-coherent", GFP_KERNEL);
195497a9230SThomas Petazzoni of_add_property(cache_dn, p);
196497a9230SThomas Petazzoni }
197497a9230SThomas Petazzoni }
198497a9230SThomas Petazzoni
coherency_type(void)199497a9230SThomas Petazzoni static int coherency_type(void)
20077fa4b9aSThomas Petazzoni {
20177fa4b9aSThomas Petazzoni struct device_node *np;
202501f928eSThomas Petazzoni const struct of_device_id *match;
203924d38f4SThomas Petazzoni int type;
204924d38f4SThomas Petazzoni
2055fbba080SThomas Petazzoni /*
206e5535545SThomas Petazzoni * The coherency fabric is needed:
207e5535545SThomas Petazzoni * - For coherency between processors on Armada XP, so only
208e5535545SThomas Petazzoni * when SMP is enabled.
209e5535545SThomas Petazzoni * - For coherency between the processor and I/O devices, but
210e5535545SThomas Petazzoni * this coherency requires many pre-requisites (write
211e5535545SThomas Petazzoni * allocate cache policy, shareable pages, SMP bit set) that
212e5535545SThomas Petazzoni * are only meant in SMP situations.
213e5535545SThomas Petazzoni *
214e5535545SThomas Petazzoni * Note that this means that on Armada 370, there is currently
215e5535545SThomas Petazzoni * no way to use hardware I/O coherency, because even when
216e5535545SThomas Petazzoni * CONFIG_SMP is enabled, is_smp() returns false due to the
217e5535545SThomas Petazzoni * Armada 370 being a single-core processor. To lift this
218e5535545SThomas Petazzoni * limitation, we would have to find a way to make the cache
219e5535545SThomas Petazzoni * policy set to write-allocate (on all Armada SoCs), and to
220e5535545SThomas Petazzoni * set the shareable attribute in page tables (on all Armada
221e5535545SThomas Petazzoni * SoCs except the Armada 370). Unfortunately, such decisions
222e5535545SThomas Petazzoni * are taken very early in the kernel boot process, at a point
223e5535545SThomas Petazzoni * where we don't know yet on which SoC we are running.
224e5535545SThomas Petazzoni
225e5535545SThomas Petazzoni */
226e5535545SThomas Petazzoni if (!is_smp())
227e5535545SThomas Petazzoni return COHERENCY_FABRIC_TYPE_NONE;
228e5535545SThomas Petazzoni
229e5535545SThomas Petazzoni np = of_find_matching_node_and_match(NULL, of_coherency_table, &match);
230e5535545SThomas Petazzoni if (!np)
231924d38f4SThomas Petazzoni return COHERENCY_FABRIC_TYPE_NONE;
2325fbba080SThomas Petazzoni
233e5535545SThomas Petazzoni type = (int) match->data;
234501f928eSThomas Petazzoni
235e5535545SThomas Petazzoni of_node_put(np);
236e5535545SThomas Petazzoni
237e5535545SThomas Petazzoni return type;
238e5535545SThomas Petazzoni }
239e5535545SThomas Petazzoni
set_cpu_coherent(void)240e5535545SThomas Petazzoni int set_cpu_coherent(void)
241501f928eSThomas Petazzoni {
242501f928eSThomas Petazzoni int type = coherency_type();
24301049a5dSNadav Haklai
24401049a5dSNadav Haklai if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP) {
24501049a5dSNadav Haklai if (!coherency_base) {
24601049a5dSNadav Haklai pr_warn("Can't make current CPU cache coherent.\n");
24701049a5dSNadav Haklai pr_warn("Coherency fabric is not initialized\n");
24801049a5dSNadav Haklai return 1;
24901049a5dSNadav Haklai }
25001049a5dSNadav Haklai
25101049a5dSNadav Haklai armada_xp_clear_shared_l2();
25201049a5dSNadav Haklai ll_add_cpu_to_smp_group();
253d492cccaSThomas Petazzoni return ll_enable_coherency();
254d492cccaSThomas Petazzoni }
25501049a5dSNadav Haklai
25601049a5dSNadav Haklai return 0;
25701049a5dSNadav Haklai }
25801049a5dSNadav Haklai
coherency_available(void)25901049a5dSNadav Haklai int coherency_available(void)
26001049a5dSNadav Haklai {
26101049a5dSNadav Haklai return coherency_type() != COHERENCY_FABRIC_TYPE_NONE;
262501f928eSThomas Petazzoni }
263501f928eSThomas Petazzoni
coherency_init(void)2641bd4d8a6SThomas Petazzoni int __init coherency_init(void)
265501f928eSThomas Petazzoni {
266501f928eSThomas Petazzoni int type = coherency_type();
267501f928eSThomas Petazzoni struct device_node *np;
268501f928eSThomas Petazzoni
269501f928eSThomas Petazzoni np = of_find_matching_node(NULL, of_coherency_table);
270501f928eSThomas Petazzoni
271501f928eSThomas Petazzoni if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP)
272501f928eSThomas Petazzoni armada_370_coherency_init(np);
273501f928eSThomas Petazzoni else if (type == COHERENCY_FABRIC_TYPE_ARMADA_375 ||
274501f928eSThomas Petazzoni type == COHERENCY_FABRIC_TYPE_ARMADA_380)
275501f928eSThomas Petazzoni armada_375_380_coherency_init(np);
276d0de9323SThomas Petazzoni
277d0de9323SThomas Petazzoni of_node_put(np);
278d0de9323SThomas Petazzoni
279501f928eSThomas Petazzoni return 0;
2802eb04ae0SThomas Petazzoni }
2812eb04ae0SThomas Petazzoni
coherency_late_init(void)282009f1315SGregory CLEMENT static int __init coherency_late_init(void)
283009f1315SGregory CLEMENT {
284865e0527SThomas Petazzoni if (coherency_available())
285865e0527SThomas Petazzoni bus_register_notifier(&platform_bus_type,
286865e0527SThomas Petazzoni &mvebu_hwcc_nb);
287ef01c6c3SThomas Petazzoni return 0;
288865e0527SThomas Petazzoni }
289b0063aadSThomas Petazzoni
290865e0527SThomas Petazzoni postcore_initcall(coherency_late_init);
291865e0527SThomas Petazzoni
292865e0527SThomas Petazzoni #if IS_ENABLED(CONFIG_PCI)
coherency_pci_init(void)293865e0527SThomas Petazzoni static int __init coherency_pci_init(void)
294b0063aadSThomas Petazzoni {
2958828ccc3SThomas Petazzoni if (coherency_available())
296b0063aadSThomas Petazzoni bus_register_notifier(&pci_bus_type,
297b0063aadSThomas Petazzoni &mvebu_hwcc_pci_nb);
298b0063aadSThomas Petazzoni return 0;
299b0063aadSThomas Petazzoni }
300a728b977SEzequiel Garcia
301b0063aadSThomas Petazzoni arch_initcall(coherency_pci_init);
302b0063aadSThomas Petazzoni #endif
303b0063aadSThomas Petazzoni