1009f1315SGregory CLEMENT /* 2e12f12acSThomas Petazzoni * Coherency fabric (Aurora) support for Armada 370, 375, 38x and XP 3e12f12acSThomas Petazzoni * platforms. 4009f1315SGregory CLEMENT * 5009f1315SGregory CLEMENT * Copyright (C) 2012 Marvell 6009f1315SGregory CLEMENT * 7009f1315SGregory CLEMENT * Yehuda Yitschak <yehuday@marvell.com> 8009f1315SGregory CLEMENT * Gregory Clement <gregory.clement@free-electrons.com> 9009f1315SGregory CLEMENT * Thomas Petazzoni <thomas.petazzoni@free-electrons.com> 10009f1315SGregory CLEMENT * 11009f1315SGregory CLEMENT * This file is licensed under the terms of the GNU General Public 12009f1315SGregory CLEMENT * License version 2. This program is licensed "as is" without any 13009f1315SGregory CLEMENT * warranty of any kind, whether express or implied. 14009f1315SGregory CLEMENT * 15e12f12acSThomas Petazzoni * The Armada 370, 375, 38x and XP SOCs have a coherency fabric which is 16009f1315SGregory CLEMENT * responsible for ensuring hardware coherency between all CPUs and between 17009f1315SGregory CLEMENT * CPUs and I/O masters. This file initializes the coherency fabric and 18009f1315SGregory CLEMENT * supplies basic routines for configuring and controlling hardware coherency 19009f1315SGregory CLEMENT */ 20009f1315SGregory CLEMENT 215ab5afd8SThomas Petazzoni #define pr_fmt(fmt) "mvebu-coherency: " fmt 225ab5afd8SThomas Petazzoni 23009f1315SGregory CLEMENT #include <linux/kernel.h> 24009f1315SGregory CLEMENT #include <linux/init.h> 25009f1315SGregory CLEMENT #include <linux/of_address.h> 26009f1315SGregory CLEMENT #include <linux/io.h> 27009f1315SGregory CLEMENT #include <linux/smp.h> 28e60304f8SGregory CLEMENT #include <linux/dma-mapping.h> 29e60304f8SGregory CLEMENT #include <linux/platform_device.h> 305ab5afd8SThomas Petazzoni #include <linux/slab.h> 315ab5afd8SThomas Petazzoni #include <linux/mbus.h> 32b0063aadSThomas Petazzoni #include <linux/pci.h> 33009f1315SGregory CLEMENT #include <asm/smp_plat.h> 34580ff0eeSThomas Petazzoni #include <asm/cacheflush.h> 35497a9230SThomas Petazzoni #include <asm/mach/map.h> 361bd4d8a6SThomas Petazzoni #include <asm/dma-mapping.h> 37b12634e3SJisheng Zhang #include "coherency.h" 3839438567SThomas Petazzoni #include "mvebu-soc-id.h" 39009f1315SGregory CLEMENT 408bd26e3aSPaul Gortmaker unsigned long coherency_phys_base; 41ccd6a131SGregory CLEMENT void __iomem *coherency_base; 42e60304f8SGregory CLEMENT static void __iomem *coherency_cpu_base; 43d492cccaSThomas Petazzoni static void __iomem *cpu_config_base; 44009f1315SGregory CLEMENT 45009f1315SGregory CLEMENT /* Coherency fabric registers */ 46e60304f8SGregory CLEMENT #define IO_SYNC_BARRIER_CTL_OFFSET 0x0 47e60304f8SGregory CLEMENT 48924d38f4SThomas Petazzoni enum { 49501f928eSThomas Petazzoni COHERENCY_FABRIC_TYPE_NONE, 50924d38f4SThomas Petazzoni COHERENCY_FABRIC_TYPE_ARMADA_370_XP, 5177fa4b9aSThomas Petazzoni COHERENCY_FABRIC_TYPE_ARMADA_375, 52d0de9323SThomas Petazzoni COHERENCY_FABRIC_TYPE_ARMADA_380, 53924d38f4SThomas Petazzoni }; 54924d38f4SThomas Petazzoni 55444d2d33SUwe Kleine-König static const struct of_device_id of_coherency_table[] = { 56924d38f4SThomas Petazzoni {.compatible = "marvell,coherency-fabric", 57924d38f4SThomas Petazzoni .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_370_XP }, 5877fa4b9aSThomas Petazzoni {.compatible = "marvell,armada-375-coherency-fabric", 5977fa4b9aSThomas Petazzoni .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_375 }, 60d0de9323SThomas Petazzoni {.compatible = "marvell,armada-380-coherency-fabric", 61d0de9323SThomas Petazzoni .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_380 }, 62009f1315SGregory CLEMENT { /* end of list */ }, 63009f1315SGregory CLEMENT }; 64009f1315SGregory CLEMENT 652e8a5942SGregory CLEMENT /* Functions defined in coherency_ll.S */ 662e8a5942SGregory CLEMENT int ll_enable_coherency(void); 672e8a5942SGregory CLEMENT void ll_add_cpu_to_smp_group(void); 68009f1315SGregory CLEMENT 69d492cccaSThomas Petazzoni #define CPU_CONFIG_SHARED_L2 BIT(16) 70d492cccaSThomas Petazzoni 71d492cccaSThomas Petazzoni /* 72d492cccaSThomas Petazzoni * Disable the "Shared L2 Present" bit in CPU Configuration register 73d492cccaSThomas Petazzoni * on Armada XP. 74d492cccaSThomas Petazzoni * 75d492cccaSThomas Petazzoni * The "Shared L2 Present" bit affects the "level of coherence" value 76d492cccaSThomas Petazzoni * in the clidr CP15 register. Cache operation functions such as 77d492cccaSThomas Petazzoni * "flush all" and "invalidate all" operate on all the cache levels 78d492cccaSThomas Petazzoni * that included in the defined level of coherence. When HW I/O 79d492cccaSThomas Petazzoni * coherency is used, this bit causes unnecessary flushes of the L2 80d492cccaSThomas Petazzoni * cache. 81d492cccaSThomas Petazzoni */ 82d492cccaSThomas Petazzoni static void armada_xp_clear_shared_l2(void) 83d492cccaSThomas Petazzoni { 84d492cccaSThomas Petazzoni u32 reg; 85d492cccaSThomas Petazzoni 86d492cccaSThomas Petazzoni if (!cpu_config_base) 87d492cccaSThomas Petazzoni return; 88d492cccaSThomas Petazzoni 89d492cccaSThomas Petazzoni reg = readl(cpu_config_base); 90d492cccaSThomas Petazzoni reg &= ~CPU_CONFIG_SHARED_L2; 91d492cccaSThomas Petazzoni writel(reg, cpu_config_base); 92d492cccaSThomas Petazzoni } 93d492cccaSThomas Petazzoni 94b0063aadSThomas Petazzoni static int mvebu_hwcc_notifier(struct notifier_block *nb, 95e60304f8SGregory CLEMENT unsigned long event, void *__dev) 96e60304f8SGregory CLEMENT { 97e60304f8SGregory CLEMENT struct device *dev = __dev; 98e60304f8SGregory CLEMENT 99e60304f8SGregory CLEMENT if (event != BUS_NOTIFY_ADD_DEVICE) 100e60304f8SGregory CLEMENT return NOTIFY_DONE; 1011bd4d8a6SThomas Petazzoni set_dma_ops(dev, &arm_coherent_dma_ops); 102e60304f8SGregory CLEMENT 103e60304f8SGregory CLEMENT return NOTIFY_OK; 104e60304f8SGregory CLEMENT } 105e60304f8SGregory CLEMENT 106b0063aadSThomas Petazzoni static struct notifier_block mvebu_hwcc_nb = { 107b0063aadSThomas Petazzoni .notifier_call = mvebu_hwcc_notifier, 108e60304f8SGregory CLEMENT }; 109e60304f8SGregory CLEMENT 11060f23952SArnd Bergmann static struct notifier_block mvebu_hwcc_pci_nb __maybe_unused = { 111a728b977SEzequiel Garcia .notifier_call = mvebu_hwcc_notifier, 112a728b977SEzequiel Garcia }; 113a728b977SEzequiel Garcia 1147fbbaebfSSebastian Andrzej Siewior static int armada_xp_clear_l2_starting(unsigned int cpu) 115d492cccaSThomas Petazzoni { 116d492cccaSThomas Petazzoni armada_xp_clear_shared_l2(); 1177fbbaebfSSebastian Andrzej Siewior return 0; 118d492cccaSThomas Petazzoni } 119d492cccaSThomas Petazzoni 120924d38f4SThomas Petazzoni static void __init armada_370_coherency_init(struct device_node *np) 121009f1315SGregory CLEMENT { 122580ff0eeSThomas Petazzoni struct resource res; 123d492cccaSThomas Petazzoni struct device_node *cpu_config_np; 124924d38f4SThomas Petazzoni 125580ff0eeSThomas Petazzoni of_address_to_resource(np, 0, &res); 126580ff0eeSThomas Petazzoni coherency_phys_base = res.start; 127580ff0eeSThomas Petazzoni /* 128580ff0eeSThomas Petazzoni * Ensure secondary CPUs will see the updated value, 129580ff0eeSThomas Petazzoni * which they read before they join the coherency 130580ff0eeSThomas Petazzoni * fabric, and therefore before they are coherent with 131580ff0eeSThomas Petazzoni * the boot CPU cache. 132580ff0eeSThomas Petazzoni */ 133580ff0eeSThomas Petazzoni sync_cache_w(&coherency_phys_base); 134009f1315SGregory CLEMENT coherency_base = of_iomap(np, 0); 135e60304f8SGregory CLEMENT coherency_cpu_base = of_iomap(np, 1); 136d492cccaSThomas Petazzoni 137d492cccaSThomas Petazzoni cpu_config_np = of_find_compatible_node(NULL, NULL, 138d492cccaSThomas Petazzoni "marvell,armada-xp-cpu-config"); 139d492cccaSThomas Petazzoni if (!cpu_config_np) 140d492cccaSThomas Petazzoni goto exit; 141d492cccaSThomas Petazzoni 142d492cccaSThomas Petazzoni cpu_config_base = of_iomap(cpu_config_np, 0); 143d492cccaSThomas Petazzoni if (!cpu_config_base) { 144d492cccaSThomas Petazzoni of_node_put(cpu_config_np); 145d492cccaSThomas Petazzoni goto exit; 146d492cccaSThomas Petazzoni } 147d492cccaSThomas Petazzoni 148d492cccaSThomas Petazzoni of_node_put(cpu_config_np); 149d492cccaSThomas Petazzoni 1507fbbaebfSSebastian Andrzej Siewior cpuhp_setup_state_nocalls(CPUHP_AP_ARM_MVEBU_COHERENCY, 15173c1b41eSThomas Gleixner "arm/mvebu/coherency:starting", 1527fbbaebfSSebastian Andrzej Siewior armada_xp_clear_l2_starting, NULL); 153d492cccaSThomas Petazzoni exit: 154952f4ca7SGregory CLEMENT set_cpu_coherent(); 155924d38f4SThomas Petazzoni } 156924d38f4SThomas Petazzoni 157497a9230SThomas Petazzoni /* 158c5379ba8SThomas Petazzoni * This ioremap hook is used on Armada 375/38x to ensure that all MMIO 159c5379ba8SThomas Petazzoni * areas are mapped as MT_UNCACHED instead of MT_DEVICE. This is 160c5379ba8SThomas Petazzoni * needed for the HW I/O coherency mechanism to work properly without 161c5379ba8SThomas Petazzoni * deadlock. 162497a9230SThomas Petazzoni */ 163497a9230SThomas Petazzoni static void __iomem * 164c5379ba8SThomas Petazzoni armada_wa_ioremap_caller(phys_addr_t phys_addr, size_t size, 165497a9230SThomas Petazzoni unsigned int mtype, void *caller) 166497a9230SThomas Petazzoni { 167497a9230SThomas Petazzoni mtype = MT_UNCACHED; 168497a9230SThomas Petazzoni return __arm_ioremap_caller(phys_addr, size, mtype, caller); 169497a9230SThomas Petazzoni } 170497a9230SThomas Petazzoni 171d0de9323SThomas Petazzoni static void __init armada_375_380_coherency_init(struct device_node *np) 17277fa4b9aSThomas Petazzoni { 173497a9230SThomas Petazzoni struct device_node *cache_dn; 174497a9230SThomas Petazzoni 17577fa4b9aSThomas Petazzoni coherency_cpu_base = of_iomap(np, 0); 176c5379ba8SThomas Petazzoni arch_ioremap_caller = armada_wa_ioremap_caller; 1776a02734dSThomas Petazzoni pci_ioremap_set_mem_type(MT_UNCACHED); 178497a9230SThomas Petazzoni 179497a9230SThomas Petazzoni /* 180dcad6887SThomas Petazzoni * We should switch the PL310 to I/O coherency mode only if 181dcad6887SThomas Petazzoni * I/O coherency is actually enabled. 182dcad6887SThomas Petazzoni */ 183dcad6887SThomas Petazzoni if (!coherency_available()) 184dcad6887SThomas Petazzoni return; 185dcad6887SThomas Petazzoni 186dcad6887SThomas Petazzoni /* 187497a9230SThomas Petazzoni * Add the PL310 property "arm,io-coherent". This makes sure the 188497a9230SThomas Petazzoni * outer sync operation is not used, which allows to 189497a9230SThomas Petazzoni * workaround the system erratum that causes deadlocks when 190497a9230SThomas Petazzoni * doing PCIe in an SMP situation on Armada 375 and Armada 191497a9230SThomas Petazzoni * 38x. 192497a9230SThomas Petazzoni */ 193497a9230SThomas Petazzoni for_each_compatible_node(cache_dn, NULL, "arm,pl310-cache") { 194497a9230SThomas Petazzoni struct property *p; 195497a9230SThomas Petazzoni 196497a9230SThomas Petazzoni p = kzalloc(sizeof(*p), GFP_KERNEL); 197497a9230SThomas Petazzoni p->name = kstrdup("arm,io-coherent", GFP_KERNEL); 198497a9230SThomas Petazzoni of_add_property(cache_dn, p); 199497a9230SThomas Petazzoni } 20077fa4b9aSThomas Petazzoni } 20177fa4b9aSThomas Petazzoni 202501f928eSThomas Petazzoni static int coherency_type(void) 203924d38f4SThomas Petazzoni { 204924d38f4SThomas Petazzoni struct device_node *np; 2055fbba080SThomas Petazzoni const struct of_device_id *match; 206e5535545SThomas Petazzoni int type; 207e5535545SThomas Petazzoni 208e5535545SThomas Petazzoni /* 209e5535545SThomas Petazzoni * The coherency fabric is needed: 210e5535545SThomas Petazzoni * - For coherency between processors on Armada XP, so only 211e5535545SThomas Petazzoni * when SMP is enabled. 212e5535545SThomas Petazzoni * - For coherency between the processor and I/O devices, but 213e5535545SThomas Petazzoni * this coherency requires many pre-requisites (write 214e5535545SThomas Petazzoni * allocate cache policy, shareable pages, SMP bit set) that 215e5535545SThomas Petazzoni * are only meant in SMP situations. 216e5535545SThomas Petazzoni * 217e5535545SThomas Petazzoni * Note that this means that on Armada 370, there is currently 218e5535545SThomas Petazzoni * no way to use hardware I/O coherency, because even when 219e5535545SThomas Petazzoni * CONFIG_SMP is enabled, is_smp() returns false due to the 220e5535545SThomas Petazzoni * Armada 370 being a single-core processor. To lift this 221e5535545SThomas Petazzoni * limitation, we would have to find a way to make the cache 222e5535545SThomas Petazzoni * policy set to write-allocate (on all Armada SoCs), and to 223e5535545SThomas Petazzoni * set the shareable attribute in page tables (on all Armada 224e5535545SThomas Petazzoni * SoCs except the Armada 370). Unfortunately, such decisions 225e5535545SThomas Petazzoni * are taken very early in the kernel boot process, at a point 226e5535545SThomas Petazzoni * where we don't know yet on which SoC we are running. 227e5535545SThomas Petazzoni 228e5535545SThomas Petazzoni */ 229e5535545SThomas Petazzoni if (!is_smp()) 230e5535545SThomas Petazzoni return COHERENCY_FABRIC_TYPE_NONE; 231924d38f4SThomas Petazzoni 2325fbba080SThomas Petazzoni np = of_find_matching_node_and_match(NULL, of_coherency_table, &match); 233e5535545SThomas Petazzoni if (!np) 234501f928eSThomas Petazzoni return COHERENCY_FABRIC_TYPE_NONE; 235e5535545SThomas Petazzoni 236e5535545SThomas Petazzoni type = (int) match->data; 237e5535545SThomas Petazzoni 238e5535545SThomas Petazzoni of_node_put(np); 239e5535545SThomas Petazzoni 240e5535545SThomas Petazzoni return type; 241501f928eSThomas Petazzoni } 242501f928eSThomas Petazzoni 24301049a5dSNadav Haklai int set_cpu_coherent(void) 24401049a5dSNadav Haklai { 24501049a5dSNadav Haklai int type = coherency_type(); 24601049a5dSNadav Haklai 24701049a5dSNadav Haklai if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP) { 24801049a5dSNadav Haklai if (!coherency_base) { 24901049a5dSNadav Haklai pr_warn("Can't make current CPU cache coherent.\n"); 25001049a5dSNadav Haklai pr_warn("Coherency fabric is not initialized\n"); 25101049a5dSNadav Haklai return 1; 25201049a5dSNadav Haklai } 253d492cccaSThomas Petazzoni 254d492cccaSThomas Petazzoni armada_xp_clear_shared_l2(); 25501049a5dSNadav Haklai ll_add_cpu_to_smp_group(); 25601049a5dSNadav Haklai return ll_enable_coherency(); 25701049a5dSNadav Haklai } 25801049a5dSNadav Haklai 25901049a5dSNadav Haklai return 0; 26001049a5dSNadav Haklai } 26101049a5dSNadav Haklai 262501f928eSThomas Petazzoni int coherency_available(void) 263501f928eSThomas Petazzoni { 2641bd4d8a6SThomas Petazzoni return coherency_type() != COHERENCY_FABRIC_TYPE_NONE; 265501f928eSThomas Petazzoni } 266501f928eSThomas Petazzoni 267501f928eSThomas Petazzoni int __init coherency_init(void) 268501f928eSThomas Petazzoni { 269501f928eSThomas Petazzoni int type = coherency_type(); 270501f928eSThomas Petazzoni struct device_node *np; 271501f928eSThomas Petazzoni 272501f928eSThomas Petazzoni np = of_find_matching_node(NULL, of_coherency_table); 273501f928eSThomas Petazzoni 274501f928eSThomas Petazzoni if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP) 275501f928eSThomas Petazzoni armada_370_coherency_init(np); 276d0de9323SThomas Petazzoni else if (type == COHERENCY_FABRIC_TYPE_ARMADA_375 || 277d0de9323SThomas Petazzoni type == COHERENCY_FABRIC_TYPE_ARMADA_380) 278d0de9323SThomas Petazzoni armada_375_380_coherency_init(np); 279501f928eSThomas Petazzoni 2802eb04ae0SThomas Petazzoni of_node_put(np); 2812eb04ae0SThomas Petazzoni 282009f1315SGregory CLEMENT return 0; 283009f1315SGregory CLEMENT } 284865e0527SThomas Petazzoni 285865e0527SThomas Petazzoni static int __init coherency_late_init(void) 286865e0527SThomas Petazzoni { 287ef01c6c3SThomas Petazzoni if (coherency_available()) 288865e0527SThomas Petazzoni bus_register_notifier(&platform_bus_type, 289b0063aadSThomas Petazzoni &mvebu_hwcc_nb); 290865e0527SThomas Petazzoni return 0; 291865e0527SThomas Petazzoni } 292865e0527SThomas Petazzoni 293865e0527SThomas Petazzoni postcore_initcall(coherency_late_init); 294b0063aadSThomas Petazzoni 2958828ccc3SThomas Petazzoni #if IS_ENABLED(CONFIG_PCI) 296b0063aadSThomas Petazzoni static int __init coherency_pci_init(void) 297b0063aadSThomas Petazzoni { 298b0063aadSThomas Petazzoni if (coherency_available()) 299b0063aadSThomas Petazzoni bus_register_notifier(&pci_bus_type, 300a728b977SEzequiel Garcia &mvebu_hwcc_pci_nb); 301b0063aadSThomas Petazzoni return 0; 302b0063aadSThomas Petazzoni } 303b0063aadSThomas Petazzoni 304b0063aadSThomas Petazzoni arch_initcall(coherency_pci_init); 3058828ccc3SThomas Petazzoni #endif 306