1f6f9be1cSFlorian Fainelli /* 2f6f9be1cSFlorian Fainelli * Broadcom Brahma-B15 CPU read-ahead cache management functions 3f6f9be1cSFlorian Fainelli * 4f6f9be1cSFlorian Fainelli * Copyright (C) 2015-2016 Broadcom 5f6f9be1cSFlorian Fainelli * 6f6f9be1cSFlorian Fainelli * This program is free software; you can redistribute it and/or modify 7f6f9be1cSFlorian Fainelli * it under the terms of the GNU General Public License version 2 as 8f6f9be1cSFlorian Fainelli * published by the Free Software Foundation. 9f6f9be1cSFlorian Fainelli */ 10f6f9be1cSFlorian Fainelli 11f6f9be1cSFlorian Fainelli #include <linux/err.h> 12f6f9be1cSFlorian Fainelli #include <linux/spinlock.h> 13f6f9be1cSFlorian Fainelli #include <linux/io.h> 14f6f9be1cSFlorian Fainelli #include <linux/bitops.h> 15f6f9be1cSFlorian Fainelli #include <linux/of_address.h> 16f6f9be1cSFlorian Fainelli 17f6f9be1cSFlorian Fainelli #include <asm/cacheflush.h> 18f6f9be1cSFlorian Fainelli #include <asm/hardware/cache-b15-rac.h> 19f6f9be1cSFlorian Fainelli 20f6f9be1cSFlorian Fainelli extern void v7_flush_kern_cache_all(void); 21f6f9be1cSFlorian Fainelli 22f6f9be1cSFlorian Fainelli /* RAC register offsets, relative to the HIF_CPU_BIUCTRL register base */ 23f6f9be1cSFlorian Fainelli #define RAC_CONFIG0_REG (0x78) 24f6f9be1cSFlorian Fainelli #define RACENPREF_MASK (0x3) 25f6f9be1cSFlorian Fainelli #define RACPREFINST_SHIFT (0) 26f6f9be1cSFlorian Fainelli #define RACENINST_SHIFT (2) 27f6f9be1cSFlorian Fainelli #define RACPREFDATA_SHIFT (4) 28f6f9be1cSFlorian Fainelli #define RACENDATA_SHIFT (6) 29f6f9be1cSFlorian Fainelli #define RAC_CPU_SHIFT (8) 30f6f9be1cSFlorian Fainelli #define RACCFG_MASK (0xff) 31f6f9be1cSFlorian Fainelli #define RAC_CONFIG1_REG (0x7c) 32f6f9be1cSFlorian Fainelli #define RAC_FLUSH_REG (0x80) 33f6f9be1cSFlorian Fainelli #define FLUSH_RAC (1 << 0) 34f6f9be1cSFlorian Fainelli 35f6f9be1cSFlorian Fainelli /* Bitmask to enable instruction and data prefetching with a 256-bytes stride */ 36f6f9be1cSFlorian Fainelli #define RAC_DATA_INST_EN_MASK (1 << RACPREFINST_SHIFT | \ 37f6f9be1cSFlorian Fainelli RACENPREF_MASK << RACENINST_SHIFT | \ 38f6f9be1cSFlorian Fainelli 1 << RACPREFDATA_SHIFT | \ 39f6f9be1cSFlorian Fainelli RACENPREF_MASK << RACENDATA_SHIFT) 40f6f9be1cSFlorian Fainelli 41f6f9be1cSFlorian Fainelli #define RAC_ENABLED 0 42f6f9be1cSFlorian Fainelli 43f6f9be1cSFlorian Fainelli static void __iomem *b15_rac_base; 44f6f9be1cSFlorian Fainelli static DEFINE_SPINLOCK(rac_lock); 45f6f9be1cSFlorian Fainelli 46f6f9be1cSFlorian Fainelli /* Initialization flag to avoid checking for b15_rac_base, and to prevent 47f6f9be1cSFlorian Fainelli * multi-platform kernels from crashing here as well. 48f6f9be1cSFlorian Fainelli */ 49f6f9be1cSFlorian Fainelli static unsigned long b15_rac_flags; 50f6f9be1cSFlorian Fainelli 51f6f9be1cSFlorian Fainelli static inline u32 __b15_rac_disable(void) 52f6f9be1cSFlorian Fainelli { 53f6f9be1cSFlorian Fainelli u32 val = __raw_readl(b15_rac_base + RAC_CONFIG0_REG); 54f6f9be1cSFlorian Fainelli __raw_writel(0, b15_rac_base + RAC_CONFIG0_REG); 55f6f9be1cSFlorian Fainelli dmb(); 56f6f9be1cSFlorian Fainelli return val; 57f6f9be1cSFlorian Fainelli } 58f6f9be1cSFlorian Fainelli 59f6f9be1cSFlorian Fainelli static inline void __b15_rac_flush(void) 60f6f9be1cSFlorian Fainelli { 61f6f9be1cSFlorian Fainelli u32 reg; 62f6f9be1cSFlorian Fainelli 63f6f9be1cSFlorian Fainelli __raw_writel(FLUSH_RAC, b15_rac_base + RAC_FLUSH_REG); 64f6f9be1cSFlorian Fainelli do { 65f6f9be1cSFlorian Fainelli /* This dmb() is required to force the Bus Interface Unit 66f6f9be1cSFlorian Fainelli * to clean oustanding writes, and forces an idle cycle 67f6f9be1cSFlorian Fainelli * to be inserted. 68f6f9be1cSFlorian Fainelli */ 69f6f9be1cSFlorian Fainelli dmb(); 70f6f9be1cSFlorian Fainelli reg = __raw_readl(b15_rac_base + RAC_FLUSH_REG); 71f6f9be1cSFlorian Fainelli } while (reg & FLUSH_RAC); 72f6f9be1cSFlorian Fainelli } 73f6f9be1cSFlorian Fainelli 74f6f9be1cSFlorian Fainelli static inline u32 b15_rac_disable_and_flush(void) 75f6f9be1cSFlorian Fainelli { 76f6f9be1cSFlorian Fainelli u32 reg; 77f6f9be1cSFlorian Fainelli 78f6f9be1cSFlorian Fainelli reg = __b15_rac_disable(); 79f6f9be1cSFlorian Fainelli __b15_rac_flush(); 80f6f9be1cSFlorian Fainelli return reg; 81f6f9be1cSFlorian Fainelli } 82f6f9be1cSFlorian Fainelli 83f6f9be1cSFlorian Fainelli static inline void __b15_rac_enable(u32 val) 84f6f9be1cSFlorian Fainelli { 85f6f9be1cSFlorian Fainelli __raw_writel(val, b15_rac_base + RAC_CONFIG0_REG); 86f6f9be1cSFlorian Fainelli /* dsb() is required here to be consistent with __flush_icache_all() */ 87f6f9be1cSFlorian Fainelli dsb(); 88f6f9be1cSFlorian Fainelli } 89f6f9be1cSFlorian Fainelli 90f6f9be1cSFlorian Fainelli #define BUILD_RAC_CACHE_OP(name, bar) \ 91f6f9be1cSFlorian Fainelli void b15_flush_##name(void) \ 92f6f9be1cSFlorian Fainelli { \ 93f6f9be1cSFlorian Fainelli unsigned int do_flush; \ 94f6f9be1cSFlorian Fainelli u32 val = 0; \ 95f6f9be1cSFlorian Fainelli \ 96f6f9be1cSFlorian Fainelli spin_lock(&rac_lock); \ 97f6f9be1cSFlorian Fainelli do_flush = test_bit(RAC_ENABLED, &b15_rac_flags); \ 98f6f9be1cSFlorian Fainelli if (do_flush) \ 99f6f9be1cSFlorian Fainelli val = b15_rac_disable_and_flush(); \ 100f6f9be1cSFlorian Fainelli v7_flush_##name(); \ 101f6f9be1cSFlorian Fainelli if (!do_flush) \ 102f6f9be1cSFlorian Fainelli bar; \ 103f6f9be1cSFlorian Fainelli else \ 104f6f9be1cSFlorian Fainelli __b15_rac_enable(val); \ 105f6f9be1cSFlorian Fainelli spin_unlock(&rac_lock); \ 106f6f9be1cSFlorian Fainelli } 107f6f9be1cSFlorian Fainelli 108f6f9be1cSFlorian Fainelli #define nobarrier 109f6f9be1cSFlorian Fainelli 110f6f9be1cSFlorian Fainelli /* The readahead cache present in the Brahma-B15 CPU is a special piece of 111f6f9be1cSFlorian Fainelli * hardware after the integrated L2 cache of the B15 CPU complex whose purpose 112f6f9be1cSFlorian Fainelli * is to prefetch instruction and/or data with a line size of either 64 bytes 113f6f9be1cSFlorian Fainelli * or 256 bytes. The rationale is that the data-bus of the CPU interface is 114f6f9be1cSFlorian Fainelli * optimized for 256-bytes transactions, and enabling the readahead cache 115f6f9be1cSFlorian Fainelli * provides a significant performance boost we want it enabled (typically 116f6f9be1cSFlorian Fainelli * twice the performance for a memcpy benchmark application). 117f6f9be1cSFlorian Fainelli * 118f6f9be1cSFlorian Fainelli * The readahead cache is transparent for Modified Virtual Addresses 119f6f9be1cSFlorian Fainelli * cache maintenance operations: ICIMVAU, DCIMVAC, DCCMVAC, DCCMVAU and 120f6f9be1cSFlorian Fainelli * DCCIMVAC. 121f6f9be1cSFlorian Fainelli * 122f6f9be1cSFlorian Fainelli * It is however not transparent for the following cache maintenance 123f6f9be1cSFlorian Fainelli * operations: DCISW, DCCSW, DCCISW, ICIALLUIS and ICIALLU which is precisely 124f6f9be1cSFlorian Fainelli * what we are patching here with our BUILD_RAC_CACHE_OP here. 125f6f9be1cSFlorian Fainelli */ 126f6f9be1cSFlorian Fainelli BUILD_RAC_CACHE_OP(kern_cache_all, nobarrier); 127f6f9be1cSFlorian Fainelli 128f6f9be1cSFlorian Fainelli static void b15_rac_enable(void) 129f6f9be1cSFlorian Fainelli { 130f6f9be1cSFlorian Fainelli unsigned int cpu; 131f6f9be1cSFlorian Fainelli u32 enable = 0; 132f6f9be1cSFlorian Fainelli 133f6f9be1cSFlorian Fainelli for_each_possible_cpu(cpu) 134f6f9be1cSFlorian Fainelli enable |= (RAC_DATA_INST_EN_MASK << (cpu * RAC_CPU_SHIFT)); 135f6f9be1cSFlorian Fainelli 136f6f9be1cSFlorian Fainelli b15_rac_disable_and_flush(); 137f6f9be1cSFlorian Fainelli __b15_rac_enable(enable); 138f6f9be1cSFlorian Fainelli } 139f6f9be1cSFlorian Fainelli 140f6f9be1cSFlorian Fainelli static int __init b15_rac_init(void) 141f6f9be1cSFlorian Fainelli { 142f6f9be1cSFlorian Fainelli struct device_node *dn; 143f6f9be1cSFlorian Fainelli int ret = 0, cpu; 144f6f9be1cSFlorian Fainelli u32 reg, en_mask = 0; 145f6f9be1cSFlorian Fainelli 146f6f9be1cSFlorian Fainelli dn = of_find_compatible_node(NULL, NULL, "brcm,brcmstb-cpu-biu-ctrl"); 147f6f9be1cSFlorian Fainelli if (!dn) 148f6f9be1cSFlorian Fainelli return -ENODEV; 149f6f9be1cSFlorian Fainelli 150f6f9be1cSFlorian Fainelli if (WARN(num_possible_cpus() > 4, "RAC only supports 4 CPUs\n")) 151f6f9be1cSFlorian Fainelli goto out; 152f6f9be1cSFlorian Fainelli 153f6f9be1cSFlorian Fainelli b15_rac_base = of_iomap(dn, 0); 154f6f9be1cSFlorian Fainelli if (!b15_rac_base) { 155f6f9be1cSFlorian Fainelli pr_err("failed to remap BIU control base\n"); 156f6f9be1cSFlorian Fainelli ret = -ENOMEM; 157f6f9be1cSFlorian Fainelli goto out; 158f6f9be1cSFlorian Fainelli } 159f6f9be1cSFlorian Fainelli 160f6f9be1cSFlorian Fainelli spin_lock(&rac_lock); 161f6f9be1cSFlorian Fainelli reg = __raw_readl(b15_rac_base + RAC_CONFIG0_REG); 162f6f9be1cSFlorian Fainelli for_each_possible_cpu(cpu) 163f6f9be1cSFlorian Fainelli en_mask |= ((1 << RACPREFDATA_SHIFT) << (cpu * RAC_CPU_SHIFT)); 164f6f9be1cSFlorian Fainelli WARN(reg & en_mask, "Read-ahead cache not previously disabled\n"); 165f6f9be1cSFlorian Fainelli 166f6f9be1cSFlorian Fainelli b15_rac_enable(); 167f6f9be1cSFlorian Fainelli set_bit(RAC_ENABLED, &b15_rac_flags); 168f6f9be1cSFlorian Fainelli spin_unlock(&rac_lock); 169f6f9be1cSFlorian Fainelli 170f6f9be1cSFlorian Fainelli pr_info("Broadcom Brahma-B15 readahead cache at: 0x%p\n", 171f6f9be1cSFlorian Fainelli b15_rac_base + RAC_CONFIG0_REG); 172f6f9be1cSFlorian Fainelli 173f6f9be1cSFlorian Fainelli out: 174f6f9be1cSFlorian Fainelli of_node_put(dn); 175f6f9be1cSFlorian Fainelli return ret; 176f6f9be1cSFlorian Fainelli } 177f6f9be1cSFlorian Fainelli arch_initcall(b15_rac_init); 178