1 /* 2 * Copyright (C) 2013-2014 Synopsys, Inc. All rights reserved. 3 * 4 * SPDX-License-Identifier: GPL-2.0+ 5 */ 6 7 #include <config.h> 8 #include <common.h> 9 #include <linux/compiler.h> 10 #include <linux/kernel.h> 11 #include <asm/arcregs.h> 12 #include <asm/cache.h> 13 14 /* Bit values in IC_CTRL */ 15 #define IC_CTRL_CACHE_DISABLE (1 << 0) 16 17 /* Bit values in DC_CTRL */ 18 #define DC_CTRL_CACHE_DISABLE (1 << 0) 19 #define DC_CTRL_INV_MODE_FLUSH (1 << 6) 20 #define DC_CTRL_FLUSH_STATUS (1 << 8) 21 #define CACHE_VER_NUM_MASK 0xF 22 #define SLC_CTRL_SB (1 << 2) 23 24 #define OP_INV 0x1 25 #define OP_FLUSH 0x2 26 #define OP_INV_IC 0x3 27 28 /* 29 * By default that variable will fall into .bss section. 30 * But .bss section is not relocated and so it will be initilized before 31 * relocation but will be used after being zeroed. 32 */ 33 int l1_line_sz __section(".data"); 34 int dcache_exists __section(".data"); 35 int icache_exists __section(".data"); 36 37 #define CACHE_LINE_MASK (~(l1_line_sz - 1)) 38 39 #ifdef CONFIG_ISA_ARCV2 40 int slc_line_sz __section(".data"); 41 int slc_exists __section(".data"); 42 int ioc_exists __section(".data"); 43 44 static unsigned int __before_slc_op(const int op) 45 { 46 unsigned int reg = reg; 47 48 if (op == OP_INV) { 49 /* 50 * IM is set by default and implies Flush-n-inv 51 * Clear it here for vanilla inv 52 */ 53 reg = read_aux_reg(ARC_AUX_SLC_CTRL); 54 write_aux_reg(ARC_AUX_SLC_CTRL, reg & ~DC_CTRL_INV_MODE_FLUSH); 55 } 56 57 return reg; 58 } 59 60 static void __after_slc_op(const int op, unsigned int reg) 61 { 62 if (op & OP_FLUSH) { /* flush / flush-n-inv both wait */ 63 /* 64 * Make sure "busy" bit reports correct status, 65 * see STAR 9001165532 66 */ 67 read_aux_reg(ARC_AUX_SLC_CTRL); 68 while (read_aux_reg(ARC_AUX_SLC_CTRL) & 69 DC_CTRL_FLUSH_STATUS) 70 ; 71 } 72 73 /* Switch back to default Invalidate mode */ 74 if (op == OP_INV) 75 write_aux_reg(ARC_AUX_SLC_CTRL, reg | DC_CTRL_INV_MODE_FLUSH); 76 } 77 78 static inline void __slc_line_loop(unsigned long paddr, unsigned long sz, 79 const int op) 80 { 81 unsigned int aux_cmd; 82 int num_lines; 83 84 #define SLC_LINE_MASK (~(slc_line_sz - 1)) 85 86 aux_cmd = op & OP_INV ? ARC_AUX_SLC_IVDL : ARC_AUX_SLC_FLDL; 87 88 sz += paddr & ~SLC_LINE_MASK; 89 paddr &= SLC_LINE_MASK; 90 91 num_lines = DIV_ROUND_UP(sz, slc_line_sz); 92 93 while (num_lines-- > 0) { 94 write_aux_reg(aux_cmd, paddr); 95 paddr += slc_line_sz; 96 } 97 } 98 99 static inline void __slc_entire_op(const int cacheop) 100 { 101 int aux; 102 unsigned int ctrl_reg = __before_slc_op(cacheop); 103 104 if (cacheop & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 105 aux = ARC_AUX_SLC_INVALIDATE; 106 else 107 aux = ARC_AUX_SLC_FLUSH; 108 109 write_aux_reg(aux, 0x1); 110 111 __after_slc_op(cacheop, ctrl_reg); 112 } 113 114 static inline void __slc_line_op(unsigned long paddr, unsigned long sz, 115 const int cacheop) 116 { 117 unsigned int ctrl_reg = __before_slc_op(cacheop); 118 __slc_line_loop(paddr, sz, cacheop); 119 __after_slc_op(cacheop, ctrl_reg); 120 } 121 #else 122 #define __slc_entire_op(cacheop) 123 #define __slc_line_op(paddr, sz, cacheop) 124 #endif 125 126 #ifdef CONFIG_ISA_ARCV2 127 static void read_decode_cache_bcr_arcv2(void) 128 { 129 union { 130 struct { 131 #ifdef CONFIG_CPU_BIG_ENDIAN 132 unsigned int pad:24, way:2, lsz:2, sz:4; 133 #else 134 unsigned int sz:4, lsz:2, way:2, pad:24; 135 #endif 136 } fields; 137 unsigned int word; 138 } slc_cfg; 139 140 union { 141 struct { 142 #ifdef CONFIG_CPU_BIG_ENDIAN 143 unsigned int pad:24, ver:8; 144 #else 145 unsigned int ver:8, pad:24; 146 #endif 147 } fields; 148 unsigned int word; 149 } sbcr; 150 151 sbcr.word = read_aux_reg(ARC_BCR_SLC); 152 if (sbcr.fields.ver) { 153 slc_cfg.word = read_aux_reg(ARC_AUX_SLC_CONFIG); 154 slc_exists = 1; 155 slc_line_sz = (slc_cfg.fields.lsz == 0) ? 128 : 64; 156 } 157 158 union { 159 struct bcr_clust_cfg { 160 #ifdef CONFIG_CPU_BIG_ENDIAN 161 unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8; 162 #else 163 unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7; 164 #endif 165 } fields; 166 unsigned int word; 167 } cbcr; 168 169 cbcr.word = read_aux_reg(ARC_BCR_CLUSTER); 170 if (cbcr.fields.c) 171 ioc_exists = 1; 172 } 173 #endif 174 175 void read_decode_cache_bcr(void) 176 { 177 int dc_line_sz = 0, ic_line_sz = 0; 178 179 union { 180 struct { 181 #ifdef CONFIG_CPU_BIG_ENDIAN 182 unsigned int pad:12, line_len:4, sz:4, config:4, ver:8; 183 #else 184 unsigned int ver:8, config:4, sz:4, line_len:4, pad:12; 185 #endif 186 } fields; 187 unsigned int word; 188 } ibcr, dbcr; 189 190 ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD); 191 if (ibcr.fields.ver) { 192 icache_exists = 1; 193 l1_line_sz = ic_line_sz = 8 << ibcr.fields.line_len; 194 if (!ic_line_sz) 195 panic("Instruction exists but line length is 0\n"); 196 } 197 198 dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD); 199 if (dbcr.fields.ver){ 200 dcache_exists = 1; 201 l1_line_sz = dc_line_sz = 16 << dbcr.fields.line_len; 202 if (!dc_line_sz) 203 panic("Data cache exists but line length is 0\n"); 204 } 205 206 if (ic_line_sz && dc_line_sz && (ic_line_sz != dc_line_sz)) 207 panic("Instruction and data cache line lengths differ\n"); 208 } 209 210 void cache_init(void) 211 { 212 read_decode_cache_bcr(); 213 214 #ifdef CONFIG_ISA_ARCV2 215 read_decode_cache_bcr_arcv2(); 216 217 if (ioc_exists) { 218 flush_dcache_all(); 219 invalidate_dcache_all(); 220 221 /* IO coherency base - 0x8z */ 222 write_aux_reg(ARC_AUX_IO_COH_AP0_BASE, 0x80000); 223 /* IO coherency aperture size - 512Mb: 0x8z-0xAz */ 224 write_aux_reg(ARC_AUX_IO_COH_AP0_SIZE, 0x11); 225 /* Enable partial writes */ 226 write_aux_reg(ARC_AUX_IO_COH_PARTIAL, 1); 227 /* Enable IO coherency */ 228 write_aux_reg(ARC_AUX_IO_COH_ENABLE, 1); 229 } 230 #endif 231 } 232 233 int icache_status(void) 234 { 235 if (!icache_exists) 236 return 0; 237 238 if (read_aux_reg(ARC_AUX_IC_CTRL) & IC_CTRL_CACHE_DISABLE) 239 return 0; 240 else 241 return 1; 242 } 243 244 void icache_enable(void) 245 { 246 if (icache_exists) 247 write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) & 248 ~IC_CTRL_CACHE_DISABLE); 249 } 250 251 void icache_disable(void) 252 { 253 if (icache_exists) 254 write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) | 255 IC_CTRL_CACHE_DISABLE); 256 } 257 258 #ifndef CONFIG_SYS_DCACHE_OFF 259 void invalidate_icache_all(void) 260 { 261 /* Any write to IC_IVIC register triggers invalidation of entire I$ */ 262 if (icache_status()) { 263 write_aux_reg(ARC_AUX_IC_IVIC, 1); 264 read_aux_reg(ARC_AUX_IC_CTRL); /* blocks */ 265 } 266 } 267 #else 268 void invalidate_icache_all(void) 269 { 270 } 271 #endif 272 273 int dcache_status(void) 274 { 275 if (!dcache_exists) 276 return 0; 277 278 if (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_CACHE_DISABLE) 279 return 0; 280 else 281 return 1; 282 } 283 284 void dcache_enable(void) 285 { 286 if (!dcache_exists) 287 return; 288 289 write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) & 290 ~(DC_CTRL_INV_MODE_FLUSH | DC_CTRL_CACHE_DISABLE)); 291 } 292 293 void dcache_disable(void) 294 { 295 if (!dcache_exists) 296 return; 297 298 write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) | 299 DC_CTRL_CACHE_DISABLE); 300 } 301 302 #ifndef CONFIG_SYS_DCACHE_OFF 303 /* 304 * Common Helper for Line Operations on {I,D}-Cache 305 */ 306 static inline void __cache_line_loop(unsigned long paddr, unsigned long sz, 307 const int cacheop) 308 { 309 unsigned int aux_cmd; 310 #if (CONFIG_ARC_MMU_VER == 3) 311 unsigned int aux_tag; 312 #endif 313 int num_lines; 314 315 if (cacheop == OP_INV_IC) { 316 aux_cmd = ARC_AUX_IC_IVIL; 317 #if (CONFIG_ARC_MMU_VER == 3) 318 aux_tag = ARC_AUX_IC_PTAG; 319 #endif 320 } else { 321 /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 322 aux_cmd = cacheop & OP_INV ? ARC_AUX_DC_IVDL : ARC_AUX_DC_FLDL; 323 #if (CONFIG_ARC_MMU_VER == 3) 324 aux_tag = ARC_AUX_DC_PTAG; 325 #endif 326 } 327 328 sz += paddr & ~CACHE_LINE_MASK; 329 paddr &= CACHE_LINE_MASK; 330 331 num_lines = DIV_ROUND_UP(sz, l1_line_sz); 332 333 while (num_lines-- > 0) { 334 #if (CONFIG_ARC_MMU_VER == 3) 335 write_aux_reg(aux_tag, paddr); 336 #endif 337 write_aux_reg(aux_cmd, paddr); 338 paddr += l1_line_sz; 339 } 340 } 341 342 static unsigned int __before_dc_op(const int op) 343 { 344 unsigned int reg; 345 346 if (op == OP_INV) { 347 /* 348 * IM is set by default and implies Flush-n-inv 349 * Clear it here for vanilla inv 350 */ 351 reg = read_aux_reg(ARC_AUX_DC_CTRL); 352 write_aux_reg(ARC_AUX_DC_CTRL, reg & ~DC_CTRL_INV_MODE_FLUSH); 353 } 354 355 return reg; 356 } 357 358 static void __after_dc_op(const int op, unsigned int reg) 359 { 360 if (op & OP_FLUSH) /* flush / flush-n-inv both wait */ 361 while (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_FLUSH_STATUS) 362 ; 363 364 /* Switch back to default Invalidate mode */ 365 if (op == OP_INV) 366 write_aux_reg(ARC_AUX_DC_CTRL, reg | DC_CTRL_INV_MODE_FLUSH); 367 } 368 369 static inline void __dc_entire_op(const int cacheop) 370 { 371 int aux; 372 unsigned int ctrl_reg = __before_dc_op(cacheop); 373 374 if (cacheop & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 375 aux = ARC_AUX_DC_IVDC; 376 else 377 aux = ARC_AUX_DC_FLSH; 378 379 write_aux_reg(aux, 0x1); 380 381 __after_dc_op(cacheop, ctrl_reg); 382 } 383 384 static inline void __dc_line_op(unsigned long paddr, unsigned long sz, 385 const int cacheop) 386 { 387 unsigned int ctrl_reg = __before_dc_op(cacheop); 388 __cache_line_loop(paddr, sz, cacheop); 389 __after_dc_op(cacheop, ctrl_reg); 390 } 391 #else 392 #define __dc_entire_op(cacheop) 393 #define __dc_line_op(paddr, sz, cacheop) 394 #endif /* !CONFIG_SYS_DCACHE_OFF */ 395 396 void invalidate_dcache_range(unsigned long start, unsigned long end) 397 { 398 #ifdef CONFIG_ISA_ARCV2 399 if (!ioc_exists) 400 #endif 401 __dc_line_op(start, end - start, OP_INV); 402 403 #ifdef CONFIG_ISA_ARCV2 404 if (slc_exists && !ioc_exists) 405 __slc_line_op(start, end - start, OP_INV); 406 #endif 407 } 408 409 void flush_dcache_range(unsigned long start, unsigned long end) 410 { 411 #ifdef CONFIG_ISA_ARCV2 412 if (!ioc_exists) 413 #endif 414 __dc_line_op(start, end - start, OP_FLUSH); 415 416 #ifdef CONFIG_ISA_ARCV2 417 if (slc_exists && !ioc_exists) 418 __slc_line_op(start, end - start, OP_FLUSH); 419 #endif 420 } 421 422 void flush_cache(unsigned long start, unsigned long size) 423 { 424 flush_dcache_range(start, start + size); 425 } 426 427 void invalidate_dcache_all(void) 428 { 429 __dc_entire_op(OP_INV); 430 431 #ifdef CONFIG_ISA_ARCV2 432 if (slc_exists) 433 __slc_entire_op(OP_INV); 434 #endif 435 } 436 437 void flush_dcache_all(void) 438 { 439 __dc_entire_op(OP_FLUSH); 440 441 #ifdef CONFIG_ISA_ARCV2 442 if (slc_exists) 443 __slc_entire_op(OP_FLUSH); 444 #endif 445 } 446