1 /* 2 * Copyright (C) 2013-2014 Synopsys, Inc. All rights reserved. 3 * 4 * SPDX-License-Identifier: GPL-2.0+ 5 */ 6 7 #include <config.h> 8 #include <common.h> 9 #include <linux/compiler.h> 10 #include <linux/kernel.h> 11 #include <linux/log2.h> 12 #include <asm/arcregs.h> 13 #include <asm/arc-bcr.h> 14 #include <asm/cache.h> 15 16 /* 17 * [ NOTE 1 ]: 18 * Data cache (L1 D$ or SL$) entire invalidate operation or data cache disable 19 * operation may result in unexpected behavior and data loss even if we flush 20 * data cache right before invalidation. That may happens if we store any context 21 * on stack (like we store BLINK register on stack before function call). 22 * BLINK register is the register where return address is automatically saved 23 * when we do function call with instructions like 'bl'. 24 * 25 * There is the real example: 26 * We may hang in the next code as we store any BLINK register on stack in 27 * invalidate_dcache_all() function. 28 * 29 * void flush_dcache_all() { 30 * __dc_entire_op(OP_FLUSH); 31 * // Other code // 32 * } 33 * 34 * void invalidate_dcache_all() { 35 * __dc_entire_op(OP_INV); 36 * // Other code // 37 * } 38 * 39 * void foo(void) { 40 * flush_dcache_all(); 41 * invalidate_dcache_all(); 42 * } 43 * 44 * Now let's see what really happens during that code execution: 45 * 46 * foo() 47 * |->> call flush_dcache_all 48 * [return address is saved to BLINK register] 49 * [push BLINK] (save to stack) ![point 1] 50 * |->> call __dc_entire_op(OP_FLUSH) 51 * [return address is saved to BLINK register] 52 * [flush L1 D$] 53 * return [jump to BLINK] 54 * <<------ 55 * [other flush_dcache_all code] 56 * [pop BLINK] (get from stack) 57 * return [jump to BLINK] 58 * <<------ 59 * |->> call invalidate_dcache_all 60 * [return address is saved to BLINK register] 61 * [push BLINK] (save to stack) ![point 2] 62 * |->> call __dc_entire_op(OP_FLUSH) 63 * [return address is saved to BLINK register] 64 * [invalidate L1 D$] ![point 3] 65 * // Oops!!! 66 * // We lose return address from invalidate_dcache_all function: 67 * // we save it to stack and invalidate L1 D$ after that! 68 * return [jump to BLINK] 69 * <<------ 70 * [other invalidate_dcache_all code] 71 * [pop BLINK] (get from stack) 72 * // we don't have this data in L1 dcache as we invalidated it in [point 3] 73 * // so we get it from next memory level (for example DDR memory) 74 * // but in the memory we have value which we save in [point 1], which 75 * // is return address from flush_dcache_all function (instead of 76 * // address from current invalidate_dcache_all function which we 77 * // saved in [point 2] !) 78 * return [jump to BLINK] 79 * <<------ 80 * // As BLINK points to invalidate_dcache_all, we call it again and 81 * // loop forever. 82 * 83 * Fortunately we may fix that by using flush & invalidation of D$ with a single 84 * one instruction (instead of flush and invalidation instructions pair) and 85 * enabling force function inline with '__attribute__((always_inline))' gcc 86 * attribute to avoid any function call (and BLINK store) between cache flush 87 * and disable. 88 */ 89 90 DECLARE_GLOBAL_DATA_PTR; 91 92 /* Bit values in IC_CTRL */ 93 #define IC_CTRL_CACHE_DISABLE BIT(0) 94 95 /* Bit values in DC_CTRL */ 96 #define DC_CTRL_CACHE_DISABLE BIT(0) 97 #define DC_CTRL_INV_MODE_FLUSH BIT(6) 98 #define DC_CTRL_FLUSH_STATUS BIT(8) 99 100 #define OP_INV BIT(0) 101 #define OP_FLUSH BIT(1) 102 #define OP_FLUSH_N_INV (OP_FLUSH | OP_INV) 103 104 /* Bit val in SLC_CONTROL */ 105 #define SLC_CTRL_DIS 0x001 106 #define SLC_CTRL_IM 0x040 107 #define SLC_CTRL_BUSY 0x100 108 #define SLC_CTRL_RGN_OP_INV 0x200 109 110 /* 111 * By default that variable will fall into .bss section. 112 * But .bss section is not relocated and so it will be initilized before 113 * relocation but will be used after being zeroed. 114 */ 115 #define CACHE_LINE_MASK (~(gd->arch.l1_line_sz - 1)) 116 117 bool ioc_exists __section(".data") = false; 118 119 /* To force enable IOC set ioc_enable to 'true' */ 120 bool ioc_enable __section(".data") = false; 121 122 static inline bool pae_exists(void) 123 { 124 /* TODO: should we compare mmu version from BCR and from CONFIG? */ 125 #if (CONFIG_ARC_MMU_VER >= 4) 126 union bcr_mmu_4 mmu4; 127 128 mmu4.word = read_aux_reg(ARC_AUX_MMU_BCR); 129 130 if (mmu4.fields.pae) 131 return true; 132 #endif /* (CONFIG_ARC_MMU_VER >= 4) */ 133 134 return false; 135 } 136 137 static inline bool icache_exists(void) 138 { 139 union bcr_di_cache ibcr; 140 141 ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD); 142 return !!ibcr.fields.ver; 143 } 144 145 static inline bool dcache_exists(void) 146 { 147 union bcr_di_cache dbcr; 148 149 dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD); 150 return !!dbcr.fields.ver; 151 } 152 153 static inline bool slc_exists(void) 154 { 155 if (is_isa_arcv2()) { 156 union bcr_generic sbcr; 157 158 sbcr.word = read_aux_reg(ARC_BCR_SLC); 159 return !!sbcr.fields.ver; 160 } 161 162 return false; 163 } 164 165 static void __slc_entire_op(const int op) 166 { 167 unsigned int ctrl; 168 169 if (!slc_exists()) 170 return; 171 172 ctrl = read_aux_reg(ARC_AUX_SLC_CTRL); 173 174 if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 175 ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 176 else 177 ctrl |= SLC_CTRL_IM; 178 179 write_aux_reg(ARC_AUX_SLC_CTRL, ctrl); 180 181 if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 182 write_aux_reg(ARC_AUX_SLC_INVALIDATE, 0x1); 183 else 184 write_aux_reg(ARC_AUX_SLC_FLUSH, 0x1); 185 186 /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 187 read_aux_reg(ARC_AUX_SLC_CTRL); 188 189 /* Important to wait for flush to complete */ 190 while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY); 191 } 192 193 static void slc_upper_region_init(void) 194 { 195 /* 196 * ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1 are always == 0 197 * as we don't use PAE40. 198 */ 199 write_aux_reg(ARC_AUX_SLC_RGN_END1, 0); 200 write_aux_reg(ARC_AUX_SLC_RGN_START1, 0); 201 } 202 203 static void __slc_rgn_op(unsigned long paddr, unsigned long sz, const int op) 204 { 205 #ifdef CONFIG_ISA_ARCV2 206 207 unsigned int ctrl; 208 unsigned long end; 209 210 if (!slc_exists()) 211 return; 212 213 /* 214 * The Region Flush operation is specified by CTRL.RGN_OP[11..9] 215 * - b'000 (default) is Flush, 216 * - b'001 is Invalidate if CTRL.IM == 0 217 * - b'001 is Flush-n-Invalidate if CTRL.IM == 1 218 */ 219 ctrl = read_aux_reg(ARC_AUX_SLC_CTRL); 220 221 /* Don't rely on default value of IM bit */ 222 if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 223 ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 224 else 225 ctrl |= SLC_CTRL_IM; 226 227 if (op & OP_INV) 228 ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */ 229 else 230 ctrl &= ~SLC_CTRL_RGN_OP_INV; 231 232 write_aux_reg(ARC_AUX_SLC_CTRL, ctrl); 233 234 /* 235 * Lower bits are ignored, no need to clip 236 * END needs to be setup before START (latter triggers the operation) 237 * END can't be same as START, so add (l2_line_sz - 1) to sz 238 */ 239 end = paddr + sz + gd->arch.slc_line_sz - 1; 240 241 /* 242 * Upper addresses (ARC_AUX_SLC_RGN_END1 and ARC_AUX_SLC_RGN_START1) 243 * are always == 0 as we don't use PAE40, so we only setup lower ones 244 * (ARC_AUX_SLC_RGN_END and ARC_AUX_SLC_RGN_START) 245 */ 246 write_aux_reg(ARC_AUX_SLC_RGN_END, end); 247 write_aux_reg(ARC_AUX_SLC_RGN_START, paddr); 248 249 /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 250 read_aux_reg(ARC_AUX_SLC_CTRL); 251 252 while (read_aux_reg(ARC_AUX_SLC_CTRL) & SLC_CTRL_BUSY); 253 254 #endif /* CONFIG_ISA_ARCV2 */ 255 } 256 257 static void arc_ioc_setup(void) 258 { 259 /* IOC Aperture start is equal to DDR start */ 260 unsigned int ap_base = CONFIG_SYS_SDRAM_BASE; 261 /* IOC Aperture size is equal to DDR size */ 262 long ap_size = CONFIG_SYS_SDRAM_SIZE; 263 264 flush_n_invalidate_dcache_all(); 265 266 if (!is_power_of_2(ap_size) || ap_size < 4096) 267 panic("IOC Aperture size must be power of 2 and bigger 4Kib"); 268 269 /* 270 * IOC Aperture size decoded as 2 ^ (SIZE + 2) KB, 271 * so setting 0x11 implies 512M, 0x12 implies 1G... 272 */ 273 write_aux_reg(ARC_AUX_IO_COH_AP0_SIZE, 274 order_base_2(ap_size / 1024) - 2); 275 276 /* IOC Aperture start must be aligned to the size of the aperture */ 277 if (ap_base % ap_size != 0) 278 panic("IOC Aperture start must be aligned to the size of the aperture"); 279 280 write_aux_reg(ARC_AUX_IO_COH_AP0_BASE, ap_base >> 12); 281 write_aux_reg(ARC_AUX_IO_COH_PARTIAL, 1); 282 write_aux_reg(ARC_AUX_IO_COH_ENABLE, 1); 283 } 284 285 static void read_decode_cache_bcr_arcv2(void) 286 { 287 #ifdef CONFIG_ISA_ARCV2 288 289 union bcr_slc_cfg slc_cfg; 290 union bcr_clust_cfg cbcr; 291 292 if (slc_exists()) { 293 slc_cfg.word = read_aux_reg(ARC_AUX_SLC_CONFIG); 294 gd->arch.slc_line_sz = (slc_cfg.fields.lsz == 0) ? 128 : 64; 295 } 296 297 cbcr.word = read_aux_reg(ARC_BCR_CLUSTER); 298 if (cbcr.fields.c && ioc_enable) 299 ioc_exists = true; 300 301 #endif /* CONFIG_ISA_ARCV2 */ 302 } 303 304 void read_decode_cache_bcr(void) 305 { 306 int dc_line_sz = 0, ic_line_sz = 0; 307 union bcr_di_cache ibcr, dbcr; 308 309 ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD); 310 if (ibcr.fields.ver) { 311 gd->arch.l1_line_sz = ic_line_sz = 8 << ibcr.fields.line_len; 312 if (!ic_line_sz) 313 panic("Instruction exists but line length is 0\n"); 314 } 315 316 dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD); 317 if (dbcr.fields.ver) { 318 gd->arch.l1_line_sz = dc_line_sz = 16 << dbcr.fields.line_len; 319 if (!dc_line_sz) 320 panic("Data cache exists but line length is 0\n"); 321 } 322 323 if (ic_line_sz && dc_line_sz && (ic_line_sz != dc_line_sz)) 324 panic("Instruction and data cache line lengths differ\n"); 325 } 326 327 void cache_init(void) 328 { 329 read_decode_cache_bcr(); 330 331 if (is_isa_arcv2()) 332 read_decode_cache_bcr_arcv2(); 333 334 if (is_isa_arcv2() && ioc_exists) 335 arc_ioc_setup(); 336 337 /* 338 * ARC_AUX_SLC_RGN_START1 and ARC_AUX_SLC_RGN_END1 register exist 339 * only if PAE exists in current HW. So we had to check pae_exist 340 * before using them. 341 */ 342 if (is_isa_arcv2() && slc_exists() && pae_exists()) 343 slc_upper_region_init(); 344 } 345 346 int icache_status(void) 347 { 348 if (!icache_exists()) 349 return 0; 350 351 if (read_aux_reg(ARC_AUX_IC_CTRL) & IC_CTRL_CACHE_DISABLE) 352 return 0; 353 else 354 return 1; 355 } 356 357 void icache_enable(void) 358 { 359 if (icache_exists()) 360 write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) & 361 ~IC_CTRL_CACHE_DISABLE); 362 } 363 364 void icache_disable(void) 365 { 366 if (icache_exists()) 367 write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) | 368 IC_CTRL_CACHE_DISABLE); 369 } 370 371 /* IC supports only invalidation */ 372 static inline void __ic_entire_invalidate(void) 373 { 374 if (!icache_status()) 375 return; 376 377 /* Any write to IC_IVIC register triggers invalidation of entire I$ */ 378 write_aux_reg(ARC_AUX_IC_IVIC, 1); 379 /* 380 * As per ARC HS databook (see chapter 5.3.3.2) 381 * it is required to add 3 NOPs after each write to IC_IVIC. 382 */ 383 __builtin_arc_nop(); 384 __builtin_arc_nop(); 385 __builtin_arc_nop(); 386 read_aux_reg(ARC_AUX_IC_CTRL); /* blocks */ 387 } 388 389 void invalidate_icache_all(void) 390 { 391 __ic_entire_invalidate(); 392 393 if (is_isa_arcv2()) 394 __slc_entire_op(OP_INV); 395 } 396 397 int dcache_status(void) 398 { 399 if (!dcache_exists()) 400 return 0; 401 402 if (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_CACHE_DISABLE) 403 return 0; 404 else 405 return 1; 406 } 407 408 void dcache_enable(void) 409 { 410 if (!dcache_exists()) 411 return; 412 413 write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) & 414 ~(DC_CTRL_INV_MODE_FLUSH | DC_CTRL_CACHE_DISABLE)); 415 } 416 417 void dcache_disable(void) 418 { 419 if (!dcache_exists()) 420 return; 421 422 write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) | 423 DC_CTRL_CACHE_DISABLE); 424 } 425 426 /* Common Helper for Line Operations on D-cache */ 427 static inline void __dcache_line_loop(unsigned long paddr, unsigned long sz, 428 const int cacheop) 429 { 430 unsigned int aux_cmd; 431 int num_lines; 432 433 /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 434 aux_cmd = cacheop & OP_INV ? ARC_AUX_DC_IVDL : ARC_AUX_DC_FLDL; 435 436 sz += paddr & ~CACHE_LINE_MASK; 437 paddr &= CACHE_LINE_MASK; 438 439 num_lines = DIV_ROUND_UP(sz, gd->arch.l1_line_sz); 440 441 while (num_lines-- > 0) { 442 #if (CONFIG_ARC_MMU_VER == 3) 443 write_aux_reg(ARC_AUX_DC_PTAG, paddr); 444 #endif 445 write_aux_reg(aux_cmd, paddr); 446 paddr += gd->arch.l1_line_sz; 447 } 448 } 449 450 static void __before_dc_op(const int op) 451 { 452 unsigned int ctrl; 453 454 ctrl = read_aux_reg(ARC_AUX_DC_CTRL); 455 456 /* IM bit implies flush-n-inv, instead of vanilla inv */ 457 if (op == OP_INV) 458 ctrl &= ~DC_CTRL_INV_MODE_FLUSH; 459 else 460 ctrl |= DC_CTRL_INV_MODE_FLUSH; 461 462 write_aux_reg(ARC_AUX_DC_CTRL, ctrl); 463 } 464 465 static void __after_dc_op(const int op) 466 { 467 if (op & OP_FLUSH) /* flush / flush-n-inv both wait */ 468 while (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_FLUSH_STATUS); 469 } 470 471 static inline void __dc_entire_op(const int cacheop) 472 { 473 int aux; 474 475 if (!dcache_status()) 476 return; 477 478 __before_dc_op(cacheop); 479 480 if (cacheop & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 481 aux = ARC_AUX_DC_IVDC; 482 else 483 aux = ARC_AUX_DC_FLSH; 484 485 write_aux_reg(aux, 0x1); 486 487 __after_dc_op(cacheop); 488 } 489 490 static inline void __dc_line_op(unsigned long paddr, unsigned long sz, 491 const int cacheop) 492 { 493 if (!dcache_status()) 494 return; 495 496 __before_dc_op(cacheop); 497 __dcache_line_loop(paddr, sz, cacheop); 498 __after_dc_op(cacheop); 499 } 500 501 void invalidate_dcache_range(unsigned long start, unsigned long end) 502 { 503 if (start >= end) 504 return; 505 506 /* 507 * ARCv1 -> call __dc_line_op 508 * ARCv2 && no IOC -> call __dc_line_op; call __slc_rgn_op 509 * ARCv2 && IOC enabled -> nothing 510 */ 511 if (!is_isa_arcv2() || !ioc_exists) 512 __dc_line_op(start, end - start, OP_INV); 513 514 if (is_isa_arcv2() && !ioc_exists) 515 __slc_rgn_op(start, end - start, OP_INV); 516 } 517 518 void flush_dcache_range(unsigned long start, unsigned long end) 519 { 520 if (start >= end) 521 return; 522 523 /* 524 * ARCv1 -> call __dc_line_op 525 * ARCv2 && no IOC -> call __dc_line_op; call __slc_rgn_op 526 * ARCv2 && IOC enabled -> nothing 527 */ 528 if (!is_isa_arcv2() || !ioc_exists) 529 __dc_line_op(start, end - start, OP_FLUSH); 530 531 if (is_isa_arcv2() && !ioc_exists) 532 __slc_rgn_op(start, end - start, OP_FLUSH); 533 } 534 535 void flush_cache(unsigned long start, unsigned long size) 536 { 537 flush_dcache_range(start, start + size); 538 } 539 540 /* 541 * As invalidate_dcache_all() is not used in generic U-Boot code and as we 542 * don't need it in arch/arc code alone (invalidate without flush) we implement 543 * flush_n_invalidate_dcache_all (flush and invalidate in 1 operation) because 544 * it's much safer. See [ NOTE 1 ] for more details. 545 */ 546 void flush_n_invalidate_dcache_all(void) 547 { 548 __dc_entire_op(OP_FLUSH_N_INV); 549 550 if (is_isa_arcv2()) 551 __slc_entire_op(OP_FLUSH_N_INV); 552 } 553 554 void flush_dcache_all(void) 555 { 556 __dc_entire_op(OP_FLUSH); 557 558 if (is_isa_arcv2()) 559 __slc_entire_op(OP_FLUSH); 560 } 561