1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Broadcom STB SoCs Bus Unit Interface controls 4 * 5 * Copyright (C) 2015, Broadcom Corporation 6 */ 7 8 #define pr_fmt(fmt) "brcmstb: " KBUILD_MODNAME ": " fmt 9 10 #include <linux/kernel.h> 11 #include <linux/io.h> 12 #include <linux/of_address.h> 13 #include <linux/syscore_ops.h> 14 #include <linux/soc/brcmstb/brcmstb.h> 15 16 #define RACENPREF_MASK 0x3 17 #define RACPREFINST_SHIFT 0 18 #define RACENINST_SHIFT 2 19 #define RACPREFDATA_SHIFT 4 20 #define RACENDATA_SHIFT 6 21 #define RAC_CPU_SHIFT 8 22 #define RACCFG_MASK 0xff 23 24 /* Bitmask to enable instruction and data prefetching with a 256-bytes stride */ 25 #define RAC_DATA_INST_EN_MASK (1 << RACPREFINST_SHIFT | \ 26 RACENPREF_MASK << RACENINST_SHIFT | \ 27 1 << RACPREFDATA_SHIFT | \ 28 RACENPREF_MASK << RACENDATA_SHIFT) 29 30 #define CPU_CREDIT_REG_MCPx_WR_PAIRING_EN_MASK 0x70000000 31 #define CPU_CREDIT_REG_MCPx_READ_CRED_MASK 0xf 32 #define CPU_CREDIT_REG_MCPx_WRITE_CRED_MASK 0xf 33 #define CPU_CREDIT_REG_MCPx_READ_CRED_SHIFT(x) ((x) * 8) 34 #define CPU_CREDIT_REG_MCPx_WRITE_CRED_SHIFT(x) (((x) * 8) + 4) 35 36 #define CPU_MCP_FLOW_REG_MCPx_RDBUFF_CRED_SHIFT(x) ((x) * 8) 37 #define CPU_MCP_FLOW_REG_MCPx_RDBUFF_CRED_MASK 0xff 38 39 #define CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_THRESHOLD_MASK 0xf 40 #define CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_TIMEOUT_MASK 0xf 41 #define CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_TIMEOUT_SHIFT 4 42 #define CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_ENABLE BIT(8) 43 44 static void __iomem *cpubiuctrl_base; 45 static bool mcp_wr_pairing_en; 46 static const int *cpubiuctrl_regs; 47 48 enum cpubiuctrl_regs { 49 CPU_CREDIT_REG = 0, 50 CPU_MCP_FLOW_REG, 51 CPU_WRITEBACK_CTRL_REG, 52 RAC_CONFIG0_REG, 53 NUM_CPU_BIUCTRL_REGS, 54 }; 55 56 static inline u32 cbc_readl(int reg) 57 { 58 int offset = cpubiuctrl_regs[reg]; 59 60 if (offset == -1 || 61 (IS_ENABLED(CONFIG_CACHE_B15_RAC) && reg == RAC_CONFIG0_REG)) 62 return (u32)-1; 63 64 return readl_relaxed(cpubiuctrl_base + offset); 65 } 66 67 static inline void cbc_writel(u32 val, int reg) 68 { 69 int offset = cpubiuctrl_regs[reg]; 70 71 if (offset == -1 || 72 (IS_ENABLED(CONFIG_CACHE_B15_RAC) && reg == RAC_CONFIG0_REG)) 73 return; 74 75 writel(val, cpubiuctrl_base + offset); 76 } 77 78 static const int b15_cpubiuctrl_regs[] = { 79 [CPU_CREDIT_REG] = 0x184, 80 [CPU_MCP_FLOW_REG] = -1, 81 [CPU_WRITEBACK_CTRL_REG] = -1, 82 [RAC_CONFIG0_REG] = -1, 83 }; 84 85 /* Odd cases, e.g: 7260A0 */ 86 static const int b53_cpubiuctrl_no_wb_regs[] = { 87 [CPU_CREDIT_REG] = 0x0b0, 88 [CPU_MCP_FLOW_REG] = 0x0b4, 89 [CPU_WRITEBACK_CTRL_REG] = -1, 90 [RAC_CONFIG0_REG] = 0x78, 91 }; 92 93 static const int b53_cpubiuctrl_regs[] = { 94 [CPU_CREDIT_REG] = 0x0b0, 95 [CPU_MCP_FLOW_REG] = 0x0b4, 96 [CPU_WRITEBACK_CTRL_REG] = 0x22c, 97 [RAC_CONFIG0_REG] = 0x78, 98 }; 99 100 static const int a72_cpubiuctrl_regs[] = { 101 [CPU_CREDIT_REG] = 0x18, 102 [CPU_MCP_FLOW_REG] = 0x1c, 103 [CPU_WRITEBACK_CTRL_REG] = 0x20, 104 [RAC_CONFIG0_REG] = 0x08, 105 }; 106 107 static int __init mcp_write_pairing_set(void) 108 { 109 u32 creds = 0; 110 111 if (!cpubiuctrl_base) 112 return -1; 113 114 creds = cbc_readl(CPU_CREDIT_REG); 115 if (mcp_wr_pairing_en) { 116 pr_info("MCP: Enabling write pairing\n"); 117 cbc_writel(creds | CPU_CREDIT_REG_MCPx_WR_PAIRING_EN_MASK, 118 CPU_CREDIT_REG); 119 } else if (creds & CPU_CREDIT_REG_MCPx_WR_PAIRING_EN_MASK) { 120 pr_info("MCP: Disabling write pairing\n"); 121 cbc_writel(creds & ~CPU_CREDIT_REG_MCPx_WR_PAIRING_EN_MASK, 122 CPU_CREDIT_REG); 123 } else { 124 pr_info("MCP: Write pairing already disabled\n"); 125 } 126 127 return 0; 128 } 129 130 static const u32 a72_b53_mach_compat[] = { 131 0x7211, 132 0x7216, 133 0x72164, 134 0x7255, 135 0x7260, 136 0x7268, 137 0x7271, 138 0x7278, 139 }; 140 141 /* The read-ahead cache present in the Brahma-B53 CPU is a special piece of 142 * hardware after the integrated L2 cache of the B53 CPU complex whose purpose 143 * is to prefetch instruction and/or data with a line size of either 64 bytes 144 * or 256 bytes. The rationale is that the data-bus of the CPU interface is 145 * optimized for 256-byte transactions, and enabling the read-ahead cache 146 * provides a significant performance boost (typically twice the performance 147 * for a memcpy benchmark application). 148 * 149 * The read-ahead cache is transparent for Virtual Address cache maintenance 150 * operations: IC IVAU, DC IVAC, DC CVAC, DC CVAU and DC CIVAC. So no special 151 * handling is needed for the DMA API above and beyond what is included in the 152 * arm64 implementation. 153 * 154 * In addition, since the Point of Unification is typically between L1 and L2 155 * for the Brahma-B53 processor no special read-ahead cache handling is needed 156 * for the IC IALLU and IC IALLUIS cache maintenance operations. 157 * 158 * However, it is not possible to specify the cache level (L3) for the cache 159 * maintenance instructions operating by set/way to operate on the read-ahead 160 * cache. The read-ahead cache will maintain coherency when inner cache lines 161 * are cleaned by set/way, but if it is necessary to invalidate inner cache 162 * lines by set/way to maintain coherency with system masters operating on 163 * shared memory that does not have hardware support for coherency, then it 164 * will also be necessary to explicitly invalidate the read-ahead cache. 165 */ 166 static void __init a72_b53_rac_enable_all(struct device_node *np) 167 { 168 unsigned int cpu; 169 u32 enable = 0; 170 171 if (IS_ENABLED(CONFIG_CACHE_B15_RAC)) 172 return; 173 174 if (WARN(num_possible_cpus() > 4, "RAC only supports 4 CPUs\n")) 175 return; 176 177 for_each_possible_cpu(cpu) 178 enable |= RAC_DATA_INST_EN_MASK << (cpu * RAC_CPU_SHIFT); 179 180 cbc_writel(enable, RAC_CONFIG0_REG); 181 182 pr_info("%pOF: Broadcom %s read-ahead cache\n", 183 np, cpubiuctrl_regs == a72_cpubiuctrl_regs ? 184 "Cortex-A72" : "Brahma-B53"); 185 } 186 187 static void __init mcp_a72_b53_set(void) 188 { 189 unsigned int i; 190 u32 reg; 191 192 reg = brcmstb_get_family_id(); 193 194 for (i = 0; i < ARRAY_SIZE(a72_b53_mach_compat); i++) { 195 if (BRCM_ID(reg) == a72_b53_mach_compat[i]) 196 break; 197 } 198 199 if (i == ARRAY_SIZE(a72_b53_mach_compat)) 200 return; 201 202 /* Set all 3 MCP interfaces to 8 credits */ 203 reg = cbc_readl(CPU_CREDIT_REG); 204 for (i = 0; i < 3; i++) { 205 reg &= ~(CPU_CREDIT_REG_MCPx_WRITE_CRED_MASK << 206 CPU_CREDIT_REG_MCPx_WRITE_CRED_SHIFT(i)); 207 reg &= ~(CPU_CREDIT_REG_MCPx_READ_CRED_MASK << 208 CPU_CREDIT_REG_MCPx_READ_CRED_SHIFT(i)); 209 reg |= 8 << CPU_CREDIT_REG_MCPx_WRITE_CRED_SHIFT(i); 210 reg |= 8 << CPU_CREDIT_REG_MCPx_READ_CRED_SHIFT(i); 211 } 212 cbc_writel(reg, CPU_CREDIT_REG); 213 214 /* Max out the number of in-flight Jwords reads on the MCP interface */ 215 reg = cbc_readl(CPU_MCP_FLOW_REG); 216 for (i = 0; i < 3; i++) 217 reg |= CPU_MCP_FLOW_REG_MCPx_RDBUFF_CRED_MASK << 218 CPU_MCP_FLOW_REG_MCPx_RDBUFF_CRED_SHIFT(i); 219 cbc_writel(reg, CPU_MCP_FLOW_REG); 220 221 /* Enable writeback throttling, set timeout to 128 cycles, 256 cycles 222 * threshold 223 */ 224 reg = cbc_readl(CPU_WRITEBACK_CTRL_REG); 225 reg |= CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_ENABLE; 226 reg &= ~CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_THRESHOLD_MASK; 227 reg &= ~(CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_TIMEOUT_MASK << 228 CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_TIMEOUT_SHIFT); 229 reg |= 8; 230 reg |= 7 << CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_TIMEOUT_SHIFT; 231 cbc_writel(reg, CPU_WRITEBACK_CTRL_REG); 232 } 233 234 static int __init setup_hifcpubiuctrl_regs(struct device_node *np) 235 { 236 struct device_node *cpu_dn; 237 u32 family_id; 238 int ret = 0; 239 240 cpubiuctrl_base = of_iomap(np, 0); 241 if (!cpubiuctrl_base) { 242 pr_err("failed to remap BIU control base\n"); 243 ret = -ENOMEM; 244 goto out; 245 } 246 247 mcp_wr_pairing_en = of_property_read_bool(np, "brcm,write-pairing"); 248 249 cpu_dn = of_get_cpu_node(0, NULL); 250 if (!cpu_dn) { 251 pr_err("failed to obtain CPU device node\n"); 252 ret = -ENODEV; 253 goto out; 254 } 255 256 if (of_device_is_compatible(cpu_dn, "brcm,brahma-b15")) 257 cpubiuctrl_regs = b15_cpubiuctrl_regs; 258 else if (of_device_is_compatible(cpu_dn, "brcm,brahma-b53")) 259 cpubiuctrl_regs = b53_cpubiuctrl_regs; 260 else if (of_device_is_compatible(cpu_dn, "arm,cortex-a72")) 261 cpubiuctrl_regs = a72_cpubiuctrl_regs; 262 else { 263 pr_err("unsupported CPU\n"); 264 ret = -EINVAL; 265 } 266 of_node_put(cpu_dn); 267 268 family_id = brcmstb_get_family_id(); 269 if (BRCM_ID(family_id) == 0x7260 && BRCM_REV(family_id) == 0) 270 cpubiuctrl_regs = b53_cpubiuctrl_no_wb_regs; 271 out: 272 of_node_put(np); 273 return ret; 274 } 275 276 #ifdef CONFIG_PM_SLEEP 277 static u32 cpubiuctrl_reg_save[NUM_CPU_BIUCTRL_REGS]; 278 279 static int brcmstb_cpu_credit_reg_suspend(void) 280 { 281 unsigned int i; 282 283 if (!cpubiuctrl_base) 284 return 0; 285 286 for (i = 0; i < NUM_CPU_BIUCTRL_REGS; i++) 287 cpubiuctrl_reg_save[i] = cbc_readl(i); 288 289 return 0; 290 } 291 292 static void brcmstb_cpu_credit_reg_resume(void) 293 { 294 unsigned int i; 295 296 if (!cpubiuctrl_base) 297 return; 298 299 for (i = 0; i < NUM_CPU_BIUCTRL_REGS; i++) 300 cbc_writel(cpubiuctrl_reg_save[i], i); 301 } 302 303 static struct syscore_ops brcmstb_cpu_credit_syscore_ops = { 304 .suspend = brcmstb_cpu_credit_reg_suspend, 305 .resume = brcmstb_cpu_credit_reg_resume, 306 }; 307 #endif 308 309 310 static int __init brcmstb_biuctrl_init(void) 311 { 312 struct device_node *np; 313 int ret; 314 315 /* We might be running on a multi-platform kernel, don't make this a 316 * fatal error, just bail out early 317 */ 318 np = of_find_compatible_node(NULL, NULL, "brcm,brcmstb-cpu-biu-ctrl"); 319 if (!np) 320 return 0; 321 322 ret = setup_hifcpubiuctrl_regs(np); 323 if (ret) 324 return ret; 325 326 ret = mcp_write_pairing_set(); 327 if (ret) { 328 pr_err("MCP: Unable to disable write pairing!\n"); 329 return ret; 330 } 331 332 a72_b53_rac_enable_all(np); 333 mcp_a72_b53_set(); 334 #ifdef CONFIG_PM_SLEEP 335 register_syscore_ops(&brcmstb_cpu_credit_syscore_ops); 336 #endif 337 return 0; 338 } 339 early_initcall(brcmstb_biuctrl_init); 340