1 /* 2 * Copyright (C) 2013-2014 Synopsys, Inc. All rights reserved. 3 * 4 * SPDX-License-Identifier: GPL-2.0+ 5 */ 6 7 #include <config.h> 8 #include <common.h> 9 #include <linux/compiler.h> 10 #include <linux/kernel.h> 11 #include <asm/arcregs.h> 12 #include <asm/cache.h> 13 14 /* Bit values in IC_CTRL */ 15 #define IC_CTRL_CACHE_DISABLE (1 << 0) 16 17 /* Bit values in DC_CTRL */ 18 #define DC_CTRL_CACHE_DISABLE (1 << 0) 19 #define DC_CTRL_INV_MODE_FLUSH (1 << 6) 20 #define DC_CTRL_FLUSH_STATUS (1 << 8) 21 #define CACHE_VER_NUM_MASK 0xF 22 #define SLC_CTRL_SB (1 << 2) 23 24 #define OP_INV 0x1 25 #define OP_FLUSH 0x2 26 #define OP_INV_IC 0x3 27 28 /* 29 * By default that variable will fall into .bss section. 30 * But .bss section is not relocated and so it will be initilized before 31 * relocation but will be used after being zeroed. 32 */ 33 int l1_line_sz __section(".data"); 34 int dcache_exists __section(".data"); 35 int icache_exists __section(".data"); 36 37 #define CACHE_LINE_MASK (~(l1_line_sz - 1)) 38 39 #ifdef CONFIG_ISA_ARCV2 40 int slc_line_sz __section(".data"); 41 int slc_exists __section(".data"); 42 int ioc_exists __section(".data"); 43 44 static unsigned int __before_slc_op(const int op) 45 { 46 unsigned int reg = reg; 47 48 if (op == OP_INV) { 49 /* 50 * IM is set by default and implies Flush-n-inv 51 * Clear it here for vanilla inv 52 */ 53 reg = read_aux_reg(ARC_AUX_SLC_CTRL); 54 write_aux_reg(ARC_AUX_SLC_CTRL, reg & ~DC_CTRL_INV_MODE_FLUSH); 55 } 56 57 return reg; 58 } 59 60 static void __after_slc_op(const int op, unsigned int reg) 61 { 62 if (op & OP_FLUSH) /* flush / flush-n-inv both wait */ 63 while (read_aux_reg(ARC_AUX_SLC_CTRL) & 64 DC_CTRL_FLUSH_STATUS) 65 ; 66 67 /* Switch back to default Invalidate mode */ 68 if (op == OP_INV) 69 write_aux_reg(ARC_AUX_SLC_CTRL, reg | DC_CTRL_INV_MODE_FLUSH); 70 } 71 72 static inline void __slc_line_loop(unsigned long paddr, unsigned long sz, 73 const int op) 74 { 75 unsigned int aux_cmd; 76 int num_lines; 77 78 #define SLC_LINE_MASK (~(slc_line_sz - 1)) 79 80 aux_cmd = op & OP_INV ? ARC_AUX_SLC_IVDL : ARC_AUX_SLC_FLDL; 81 82 sz += paddr & ~SLC_LINE_MASK; 83 paddr &= SLC_LINE_MASK; 84 85 num_lines = DIV_ROUND_UP(sz, slc_line_sz); 86 87 while (num_lines-- > 0) { 88 write_aux_reg(aux_cmd, paddr); 89 paddr += slc_line_sz; 90 } 91 } 92 93 static inline void __slc_entire_op(const int cacheop) 94 { 95 int aux; 96 unsigned int ctrl_reg = __before_slc_op(cacheop); 97 98 if (cacheop & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 99 aux = ARC_AUX_SLC_INVALIDATE; 100 else 101 aux = ARC_AUX_SLC_FLUSH; 102 103 write_aux_reg(aux, 0x1); 104 105 __after_slc_op(cacheop, ctrl_reg); 106 } 107 108 static inline void __slc_line_op(unsigned long paddr, unsigned long sz, 109 const int cacheop) 110 { 111 unsigned int ctrl_reg = __before_slc_op(cacheop); 112 __slc_line_loop(paddr, sz, cacheop); 113 __after_slc_op(cacheop, ctrl_reg); 114 } 115 #else 116 #define __slc_entire_op(cacheop) 117 #define __slc_line_op(paddr, sz, cacheop) 118 #endif 119 120 #ifdef CONFIG_ISA_ARCV2 121 static void read_decode_cache_bcr_arcv2(void) 122 { 123 union { 124 struct { 125 #ifdef CONFIG_CPU_BIG_ENDIAN 126 unsigned int pad:24, way:2, lsz:2, sz:4; 127 #else 128 unsigned int sz:4, lsz:2, way:2, pad:24; 129 #endif 130 } fields; 131 unsigned int word; 132 } slc_cfg; 133 134 union { 135 struct { 136 #ifdef CONFIG_CPU_BIG_ENDIAN 137 unsigned int pad:24, ver:8; 138 #else 139 unsigned int ver:8, pad:24; 140 #endif 141 } fields; 142 unsigned int word; 143 } sbcr; 144 145 sbcr.word = read_aux_reg(ARC_BCR_SLC); 146 if (sbcr.fields.ver) { 147 slc_cfg.word = read_aux_reg(ARC_AUX_SLC_CONFIG); 148 slc_exists = 1; 149 slc_line_sz = (slc_cfg.fields.lsz == 0) ? 128 : 64; 150 } 151 152 union { 153 struct bcr_clust_cfg { 154 #ifdef CONFIG_CPU_BIG_ENDIAN 155 unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8; 156 #else 157 unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7; 158 #endif 159 } fields; 160 unsigned int word; 161 } cbcr; 162 163 cbcr.word = read_aux_reg(ARC_BCR_CLUSTER); 164 if (cbcr.fields.c) 165 ioc_exists = 1; 166 } 167 #endif 168 169 void read_decode_cache_bcr(void) 170 { 171 int dc_line_sz = 0, ic_line_sz = 0; 172 173 union { 174 struct { 175 #ifdef CONFIG_CPU_BIG_ENDIAN 176 unsigned int pad:12, line_len:4, sz:4, config:4, ver:8; 177 #else 178 unsigned int ver:8, config:4, sz:4, line_len:4, pad:12; 179 #endif 180 } fields; 181 unsigned int word; 182 } ibcr, dbcr; 183 184 ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD); 185 if (ibcr.fields.ver) { 186 icache_exists = 1; 187 l1_line_sz = ic_line_sz = 8 << ibcr.fields.line_len; 188 if (!ic_line_sz) 189 panic("Instruction exists but line length is 0\n"); 190 } 191 192 dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD); 193 if (dbcr.fields.ver){ 194 dcache_exists = 1; 195 l1_line_sz = dc_line_sz = 16 << dbcr.fields.line_len; 196 if (!dc_line_sz) 197 panic("Data cache exists but line length is 0\n"); 198 } 199 200 if (ic_line_sz && dc_line_sz && (ic_line_sz != dc_line_sz)) 201 panic("Instruction and data cache line lengths differ\n"); 202 } 203 204 void cache_init(void) 205 { 206 read_decode_cache_bcr(); 207 208 #ifdef CONFIG_ISA_ARCV2 209 read_decode_cache_bcr_arcv2(); 210 211 if (ioc_exists) { 212 flush_dcache_all(); 213 invalidate_dcache_all(); 214 215 /* IO coherency base - 0x8z */ 216 write_aux_reg(ARC_AUX_IO_COH_AP0_BASE, 0x80000); 217 /* IO coherency aperture size - 512Mb: 0x8z-0xAz */ 218 write_aux_reg(ARC_AUX_IO_COH_AP0_SIZE, 0x11); 219 /* Enable partial writes */ 220 write_aux_reg(ARC_AUX_IO_COH_PARTIAL, 1); 221 /* Enable IO coherency */ 222 write_aux_reg(ARC_AUX_IO_COH_ENABLE, 1); 223 } 224 #endif 225 } 226 227 int icache_status(void) 228 { 229 if (!icache_exists) 230 return 0; 231 232 if (read_aux_reg(ARC_AUX_IC_CTRL) & IC_CTRL_CACHE_DISABLE) 233 return 0; 234 else 235 return 1; 236 } 237 238 void icache_enable(void) 239 { 240 if (icache_exists) 241 write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) & 242 ~IC_CTRL_CACHE_DISABLE); 243 } 244 245 void icache_disable(void) 246 { 247 if (icache_exists) 248 write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) | 249 IC_CTRL_CACHE_DISABLE); 250 } 251 252 #ifndef CONFIG_SYS_DCACHE_OFF 253 void invalidate_icache_all(void) 254 { 255 /* Any write to IC_IVIC register triggers invalidation of entire I$ */ 256 if (icache_status()) { 257 write_aux_reg(ARC_AUX_IC_IVIC, 1); 258 read_aux_reg(ARC_AUX_IC_CTRL); /* blocks */ 259 } 260 } 261 #else 262 void invalidate_icache_all(void) 263 { 264 } 265 #endif 266 267 int dcache_status(void) 268 { 269 if (!dcache_exists) 270 return 0; 271 272 if (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_CACHE_DISABLE) 273 return 0; 274 else 275 return 1; 276 } 277 278 void dcache_enable(void) 279 { 280 if (!dcache_exists) 281 return; 282 283 write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) & 284 ~(DC_CTRL_INV_MODE_FLUSH | DC_CTRL_CACHE_DISABLE)); 285 } 286 287 void dcache_disable(void) 288 { 289 if (!dcache_exists) 290 return; 291 292 write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) | 293 DC_CTRL_CACHE_DISABLE); 294 } 295 296 #ifndef CONFIG_SYS_DCACHE_OFF 297 /* 298 * Common Helper for Line Operations on {I,D}-Cache 299 */ 300 static inline void __cache_line_loop(unsigned long paddr, unsigned long sz, 301 const int cacheop) 302 { 303 unsigned int aux_cmd; 304 #if (CONFIG_ARC_MMU_VER == 3) 305 unsigned int aux_tag; 306 #endif 307 int num_lines; 308 309 if (cacheop == OP_INV_IC) { 310 aux_cmd = ARC_AUX_IC_IVIL; 311 #if (CONFIG_ARC_MMU_VER == 3) 312 aux_tag = ARC_AUX_IC_PTAG; 313 #endif 314 } else { 315 /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 316 aux_cmd = cacheop & OP_INV ? ARC_AUX_DC_IVDL : ARC_AUX_DC_FLDL; 317 #if (CONFIG_ARC_MMU_VER == 3) 318 aux_tag = ARC_AUX_DC_PTAG; 319 #endif 320 } 321 322 sz += paddr & ~CACHE_LINE_MASK; 323 paddr &= CACHE_LINE_MASK; 324 325 num_lines = DIV_ROUND_UP(sz, l1_line_sz); 326 327 while (num_lines-- > 0) { 328 #if (CONFIG_ARC_MMU_VER == 3) 329 write_aux_reg(aux_tag, paddr); 330 #endif 331 write_aux_reg(aux_cmd, paddr); 332 paddr += l1_line_sz; 333 } 334 } 335 336 static unsigned int __before_dc_op(const int op) 337 { 338 unsigned int reg; 339 340 if (op == OP_INV) { 341 /* 342 * IM is set by default and implies Flush-n-inv 343 * Clear it here for vanilla inv 344 */ 345 reg = read_aux_reg(ARC_AUX_DC_CTRL); 346 write_aux_reg(ARC_AUX_DC_CTRL, reg & ~DC_CTRL_INV_MODE_FLUSH); 347 } 348 349 return reg; 350 } 351 352 static void __after_dc_op(const int op, unsigned int reg) 353 { 354 if (op & OP_FLUSH) /* flush / flush-n-inv both wait */ 355 while (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_FLUSH_STATUS) 356 ; 357 358 /* Switch back to default Invalidate mode */ 359 if (op == OP_INV) 360 write_aux_reg(ARC_AUX_DC_CTRL, reg | DC_CTRL_INV_MODE_FLUSH); 361 } 362 363 static inline void __dc_entire_op(const int cacheop) 364 { 365 int aux; 366 unsigned int ctrl_reg = __before_dc_op(cacheop); 367 368 if (cacheop & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 369 aux = ARC_AUX_DC_IVDC; 370 else 371 aux = ARC_AUX_DC_FLSH; 372 373 write_aux_reg(aux, 0x1); 374 375 __after_dc_op(cacheop, ctrl_reg); 376 } 377 378 static inline void __dc_line_op(unsigned long paddr, unsigned long sz, 379 const int cacheop) 380 { 381 unsigned int ctrl_reg = __before_dc_op(cacheop); 382 __cache_line_loop(paddr, sz, cacheop); 383 __after_dc_op(cacheop, ctrl_reg); 384 } 385 #else 386 #define __dc_entire_op(cacheop) 387 #define __dc_line_op(paddr, sz, cacheop) 388 #endif /* !CONFIG_SYS_DCACHE_OFF */ 389 390 void invalidate_dcache_range(unsigned long start, unsigned long end) 391 { 392 #ifdef CONFIG_ISA_ARCV2 393 if (!ioc_exists) 394 #endif 395 __dc_line_op(start, end - start, OP_INV); 396 397 #ifdef CONFIG_ISA_ARCV2 398 if (slc_exists && !ioc_exists) 399 __slc_line_op(start, end - start, OP_INV); 400 #endif 401 } 402 403 void flush_dcache_range(unsigned long start, unsigned long end) 404 { 405 #ifdef CONFIG_ISA_ARCV2 406 if (!ioc_exists) 407 #endif 408 __dc_line_op(start, end - start, OP_FLUSH); 409 410 #ifdef CONFIG_ISA_ARCV2 411 if (slc_exists && !ioc_exists) 412 __slc_line_op(start, end - start, OP_FLUSH); 413 #endif 414 } 415 416 void flush_cache(unsigned long start, unsigned long size) 417 { 418 flush_dcache_range(start, start + size); 419 } 420 421 void invalidate_dcache_all(void) 422 { 423 __dc_entire_op(OP_INV); 424 425 #ifdef CONFIG_ISA_ARCV2 426 if (slc_exists) 427 __slc_entire_op(OP_INV); 428 #endif 429 } 430 431 void flush_dcache_all(void) 432 { 433 __dc_entire_op(OP_FLUSH); 434 435 #ifdef CONFIG_ISA_ARCV2 436 if (slc_exists) 437 __slc_entire_op(OP_FLUSH); 438 #endif 439 } 440