1 /* 2 * ARC Cache Management 3 * 4 * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com) 5 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 12 #include <linux/module.h> 13 #include <linux/mm.h> 14 #include <linux/sched.h> 15 #include <linux/cache.h> 16 #include <linux/mmu_context.h> 17 #include <linux/syscalls.h> 18 #include <linux/uaccess.h> 19 #include <linux/pagemap.h> 20 #include <asm/cacheflush.h> 21 #include <asm/cachectl.h> 22 #include <asm/setup.h> 23 24 #ifdef CONFIG_ISA_ARCV2 25 #define USE_RGN_FLSH 1 26 #endif 27 28 static int l2_line_sz; 29 static int ioc_exists; 30 int slc_enable = 1, ioc_enable = 1; 31 unsigned long perip_base = ARC_UNCACHED_ADDR_SPACE; /* legacy value for boot */ 32 unsigned long perip_end = 0xFFFFFFFF; /* legacy value */ 33 34 void (*_cache_line_loop_ic_fn)(phys_addr_t paddr, unsigned long vaddr, 35 unsigned long sz, const int op, const int full_page); 36 37 void (*__dma_cache_wback_inv)(phys_addr_t start, unsigned long sz); 38 void (*__dma_cache_inv)(phys_addr_t start, unsigned long sz); 39 void (*__dma_cache_wback)(phys_addr_t start, unsigned long sz); 40 41 char *arc_cache_mumbojumbo(int c, char *buf, int len) 42 { 43 int n = 0; 44 struct cpuinfo_arc_cache *p; 45 46 #define PR_CACHE(p, cfg, str) \ 47 if (!(p)->line_len) \ 48 n += scnprintf(buf + n, len - n, str"\t\t: N/A\n"); \ 49 else \ 50 n += scnprintf(buf + n, len - n, \ 51 str"\t\t: %uK, %dway/set, %uB Line, %s%s%s\n", \ 52 (p)->sz_k, (p)->assoc, (p)->line_len, \ 53 (p)->vipt ? "VIPT" : "PIPT", \ 54 (p)->alias ? " aliasing" : "", \ 55 IS_USED_CFG(cfg)); 56 57 PR_CACHE(&cpuinfo_arc700[c].icache, CONFIG_ARC_HAS_ICACHE, "I-Cache"); 58 PR_CACHE(&cpuinfo_arc700[c].dcache, CONFIG_ARC_HAS_DCACHE, "D-Cache"); 59 60 p = &cpuinfo_arc700[c].slc; 61 if (p->line_len) 62 n += scnprintf(buf + n, len - n, 63 "SLC\t\t: %uK, %uB Line%s\n", 64 p->sz_k, p->line_len, IS_USED_RUN(slc_enable)); 65 66 n += scnprintf(buf + n, len - n, "Peripherals\t: %#lx%s%s\n", 67 perip_base, 68 IS_AVAIL3(ioc_exists, ioc_enable, ", IO-Coherency (per-device) ")); 69 70 return buf; 71 } 72 73 /* 74 * Read the Cache Build Confuration Registers, Decode them and save into 75 * the cpuinfo structure for later use. 76 * No Validation done here, simply read/convert the BCRs 77 */ 78 static void read_decode_cache_bcr_arcv2(int cpu) 79 { 80 struct cpuinfo_arc_cache *p_slc = &cpuinfo_arc700[cpu].slc; 81 struct bcr_generic sbcr; 82 83 struct bcr_slc_cfg { 84 #ifdef CONFIG_CPU_BIG_ENDIAN 85 unsigned int pad:24, way:2, lsz:2, sz:4; 86 #else 87 unsigned int sz:4, lsz:2, way:2, pad:24; 88 #endif 89 } slc_cfg; 90 91 struct bcr_clust_cfg { 92 #ifdef CONFIG_CPU_BIG_ENDIAN 93 unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8; 94 #else 95 unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7; 96 #endif 97 } cbcr; 98 99 struct bcr_volatile { 100 #ifdef CONFIG_CPU_BIG_ENDIAN 101 unsigned int start:4, limit:4, pad:22, order:1, disable:1; 102 #else 103 unsigned int disable:1, order:1, pad:22, limit:4, start:4; 104 #endif 105 } vol; 106 107 108 READ_BCR(ARC_REG_SLC_BCR, sbcr); 109 if (sbcr.ver) { 110 READ_BCR(ARC_REG_SLC_CFG, slc_cfg); 111 p_slc->sz_k = 128 << slc_cfg.sz; 112 l2_line_sz = p_slc->line_len = (slc_cfg.lsz == 0) ? 128 : 64; 113 } 114 115 READ_BCR(ARC_REG_CLUSTER_BCR, cbcr); 116 if (cbcr.c) { 117 ioc_exists = 1; 118 119 /* 120 * As for today we don't support both IOC and ZONE_HIGHMEM enabled 121 * simultaneously. This happens because as of today IOC aperture covers 122 * only ZONE_NORMAL (low mem) and any dma transactions outside this 123 * region won't be HW coherent. 124 * If we want to use both IOC and ZONE_HIGHMEM we can use 125 * bounce_buffer to handle dma transactions to HIGHMEM. 126 * Also it is possible to modify dma_direct cache ops or increase IOC 127 * aperture size if we are planning to use HIGHMEM without PAE. 128 */ 129 if (IS_ENABLED(CONFIG_HIGHMEM) || is_pae40_enabled()) 130 ioc_enable = 0; 131 } else { 132 ioc_enable = 0; 133 } 134 135 /* HS 2.0 didn't have AUX_VOL */ 136 if (cpuinfo_arc700[cpu].core.family > 0x51) { 137 READ_BCR(AUX_VOL, vol); 138 perip_base = vol.start << 28; 139 /* HS 3.0 has limit and strict-ordering fields */ 140 if (cpuinfo_arc700[cpu].core.family > 0x52) 141 perip_end = (vol.limit << 28) - 1; 142 } 143 } 144 145 void read_decode_cache_bcr(void) 146 { 147 struct cpuinfo_arc_cache *p_ic, *p_dc; 148 unsigned int cpu = smp_processor_id(); 149 struct bcr_cache { 150 #ifdef CONFIG_CPU_BIG_ENDIAN 151 unsigned int pad:12, line_len:4, sz:4, config:4, ver:8; 152 #else 153 unsigned int ver:8, config:4, sz:4, line_len:4, pad:12; 154 #endif 155 } ibcr, dbcr; 156 157 p_ic = &cpuinfo_arc700[cpu].icache; 158 READ_BCR(ARC_REG_IC_BCR, ibcr); 159 160 if (!ibcr.ver) 161 goto dc_chk; 162 163 if (ibcr.ver <= 3) { 164 BUG_ON(ibcr.config != 3); 165 p_ic->assoc = 2; /* Fixed to 2w set assoc */ 166 } else if (ibcr.ver >= 4) { 167 p_ic->assoc = 1 << ibcr.config; /* 1,2,4,8 */ 168 } 169 170 p_ic->line_len = 8 << ibcr.line_len; 171 p_ic->sz_k = 1 << (ibcr.sz - 1); 172 p_ic->vipt = 1; 173 p_ic->alias = p_ic->sz_k/p_ic->assoc/TO_KB(PAGE_SIZE) > 1; 174 175 dc_chk: 176 p_dc = &cpuinfo_arc700[cpu].dcache; 177 READ_BCR(ARC_REG_DC_BCR, dbcr); 178 179 if (!dbcr.ver) 180 goto slc_chk; 181 182 if (dbcr.ver <= 3) { 183 BUG_ON(dbcr.config != 2); 184 p_dc->assoc = 4; /* Fixed to 4w set assoc */ 185 p_dc->vipt = 1; 186 p_dc->alias = p_dc->sz_k/p_dc->assoc/TO_KB(PAGE_SIZE) > 1; 187 } else if (dbcr.ver >= 4) { 188 p_dc->assoc = 1 << dbcr.config; /* 1,2,4,8 */ 189 p_dc->vipt = 0; 190 p_dc->alias = 0; /* PIPT so can't VIPT alias */ 191 } 192 193 p_dc->line_len = 16 << dbcr.line_len; 194 p_dc->sz_k = 1 << (dbcr.sz - 1); 195 196 slc_chk: 197 if (is_isa_arcv2()) 198 read_decode_cache_bcr_arcv2(cpu); 199 } 200 201 /* 202 * Line Operation on {I,D}-Cache 203 */ 204 205 #define OP_INV 0x1 206 #define OP_FLUSH 0x2 207 #define OP_FLUSH_N_INV 0x3 208 #define OP_INV_IC 0x4 209 210 /* 211 * I-Cache Aliasing in ARC700 VIPT caches (MMU v1-v3) 212 * 213 * ARC VIPT I-cache uses vaddr to index into cache and paddr to match the tag. 214 * The orig Cache Management Module "CDU" only required paddr to invalidate a 215 * certain line since it sufficed as index in Non-Aliasing VIPT cache-geometry. 216 * Infact for distinct V1,V2,P: all of {V1-P},{V2-P},{P-P} would end up fetching 217 * the exact same line. 218 * 219 * However for larger Caches (way-size > page-size) - i.e. in Aliasing config, 220 * paddr alone could not be used to correctly index the cache. 221 * 222 * ------------------ 223 * MMU v1/v2 (Fixed Page Size 8k) 224 * ------------------ 225 * The solution was to provide CDU with these additonal vaddr bits. These 226 * would be bits [x:13], x would depend on cache-geometry, 13 comes from 227 * standard page size of 8k. 228 * H/w folks chose [17:13] to be a future safe range, and moreso these 5 bits 229 * of vaddr could easily be "stuffed" in the paddr as bits [4:0] since the 230 * orig 5 bits of paddr were anyways ignored by CDU line ops, as they 231 * represent the offset within cache-line. The adv of using this "clumsy" 232 * interface for additional info was no new reg was needed in CDU programming 233 * model. 234 * 235 * 17:13 represented the max num of bits passable, actual bits needed were 236 * fewer, based on the num-of-aliases possible. 237 * -for 2 alias possibility, only bit 13 needed (32K cache) 238 * -for 4 alias possibility, bits 14:13 needed (64K cache) 239 * 240 * ------------------ 241 * MMU v3 242 * ------------------ 243 * This ver of MMU supports variable page sizes (1k-16k): although Linux will 244 * only support 8k (default), 16k and 4k. 245 * However from hardware perspective, smaller page sizes aggravate aliasing 246 * meaning more vaddr bits needed to disambiguate the cache-line-op ; 247 * the existing scheme of piggybacking won't work for certain configurations. 248 * Two new registers IC_PTAG and DC_PTAG inttoduced. 249 * "tag" bits are provided in PTAG, index bits in existing IVIL/IVDL/FLDL regs 250 */ 251 252 static inline 253 void __cache_line_loop_v2(phys_addr_t paddr, unsigned long vaddr, 254 unsigned long sz, const int op, const int full_page) 255 { 256 unsigned int aux_cmd; 257 int num_lines; 258 259 if (op == OP_INV_IC) { 260 aux_cmd = ARC_REG_IC_IVIL; 261 } else { 262 /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 263 aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 264 } 265 266 /* Ensure we properly floor/ceil the non-line aligned/sized requests 267 * and have @paddr - aligned to cache line and integral @num_lines. 268 * This however can be avoided for page sized since: 269 * -@paddr will be cache-line aligned already (being page aligned) 270 * -@sz will be integral multiple of line size (being page sized). 271 */ 272 if (!full_page) { 273 sz += paddr & ~CACHE_LINE_MASK; 274 paddr &= CACHE_LINE_MASK; 275 vaddr &= CACHE_LINE_MASK; 276 } 277 278 num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 279 280 /* MMUv2 and before: paddr contains stuffed vaddrs bits */ 281 paddr |= (vaddr >> PAGE_SHIFT) & 0x1F; 282 283 while (num_lines-- > 0) { 284 write_aux_reg(aux_cmd, paddr); 285 paddr += L1_CACHE_BYTES; 286 } 287 } 288 289 /* 290 * For ARC700 MMUv3 I-cache and D-cache flushes 291 * - ARC700 programming model requires paddr and vaddr be passed in seperate 292 * AUX registers (*_IV*L and *_PTAG respectively) irrespective of whether the 293 * caches actually alias or not. 294 * - For HS38, only the aliasing I-cache configuration uses the PTAG reg 295 * (non aliasing I-cache version doesn't; while D-cache can't possibly alias) 296 */ 297 static inline 298 void __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr, 299 unsigned long sz, const int op, const int full_page) 300 { 301 unsigned int aux_cmd, aux_tag; 302 int num_lines; 303 304 if (op == OP_INV_IC) { 305 aux_cmd = ARC_REG_IC_IVIL; 306 aux_tag = ARC_REG_IC_PTAG; 307 } else { 308 aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 309 aux_tag = ARC_REG_DC_PTAG; 310 } 311 312 /* Ensure we properly floor/ceil the non-line aligned/sized requests 313 * and have @paddr - aligned to cache line and integral @num_lines. 314 * This however can be avoided for page sized since: 315 * -@paddr will be cache-line aligned already (being page aligned) 316 * -@sz will be integral multiple of line size (being page sized). 317 */ 318 if (!full_page) { 319 sz += paddr & ~CACHE_LINE_MASK; 320 paddr &= CACHE_LINE_MASK; 321 vaddr &= CACHE_LINE_MASK; 322 } 323 num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 324 325 /* 326 * MMUv3, cache ops require paddr in PTAG reg 327 * if V-P const for loop, PTAG can be written once outside loop 328 */ 329 if (full_page) 330 write_aux_reg(aux_tag, paddr); 331 332 /* 333 * This is technically for MMU v4, using the MMU v3 programming model 334 * Special work for HS38 aliasing I-cache configuration with PAE40 335 * - upper 8 bits of paddr need to be written into PTAG_HI 336 * - (and needs to be written before the lower 32 bits) 337 * Note that PTAG_HI is hoisted outside the line loop 338 */ 339 if (is_pae40_enabled() && op == OP_INV_IC) 340 write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 341 342 while (num_lines-- > 0) { 343 if (!full_page) { 344 write_aux_reg(aux_tag, paddr); 345 paddr += L1_CACHE_BYTES; 346 } 347 348 write_aux_reg(aux_cmd, vaddr); 349 vaddr += L1_CACHE_BYTES; 350 } 351 } 352 353 #ifndef USE_RGN_FLSH 354 355 /* 356 * In HS38x (MMU v4), I-cache is VIPT (can alias), D-cache is PIPT 357 * Here's how cache ops are implemented 358 * 359 * - D-cache: only paddr needed (in DC_IVDL/DC_FLDL) 360 * - I-cache Non Aliasing: Despite VIPT, only paddr needed (in IC_IVIL) 361 * - I-cache Aliasing: Both vaddr and paddr needed (in IC_IVIL, IC_PTAG 362 * respectively, similar to MMU v3 programming model, hence 363 * __cache_line_loop_v3() is used) 364 * 365 * If PAE40 is enabled, independent of aliasing considerations, the higher bits 366 * needs to be written into PTAG_HI 367 */ 368 static inline 369 void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, 370 unsigned long sz, const int op, const int full_page) 371 { 372 unsigned int aux_cmd; 373 int num_lines; 374 375 if (op == OP_INV_IC) { 376 aux_cmd = ARC_REG_IC_IVIL; 377 } else { 378 /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 379 aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 380 } 381 382 /* Ensure we properly floor/ceil the non-line aligned/sized requests 383 * and have @paddr - aligned to cache line and integral @num_lines. 384 * This however can be avoided for page sized since: 385 * -@paddr will be cache-line aligned already (being page aligned) 386 * -@sz will be integral multiple of line size (being page sized). 387 */ 388 if (!full_page) { 389 sz += paddr & ~CACHE_LINE_MASK; 390 paddr &= CACHE_LINE_MASK; 391 } 392 393 num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 394 395 /* 396 * For HS38 PAE40 configuration 397 * - upper 8 bits of paddr need to be written into PTAG_HI 398 * - (and needs to be written before the lower 32 bits) 399 */ 400 if (is_pae40_enabled()) { 401 if (op == OP_INV_IC) 402 /* 403 * Non aliasing I-cache in HS38, 404 * aliasing I-cache handled in __cache_line_loop_v3() 405 */ 406 write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 407 else 408 write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32); 409 } 410 411 while (num_lines-- > 0) { 412 write_aux_reg(aux_cmd, paddr); 413 paddr += L1_CACHE_BYTES; 414 } 415 } 416 417 #else 418 419 /* 420 * optimized flush operation which takes a region as opposed to iterating per line 421 */ 422 static inline 423 void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, 424 unsigned long sz, const int op, const int full_page) 425 { 426 unsigned int s, e; 427 428 /* Only for Non aliasing I-cache in HS38 */ 429 if (op == OP_INV_IC) { 430 s = ARC_REG_IC_IVIR; 431 e = ARC_REG_IC_ENDR; 432 } else { 433 s = ARC_REG_DC_STARTR; 434 e = ARC_REG_DC_ENDR; 435 } 436 437 if (!full_page) { 438 /* for any leading gap between @paddr and start of cache line */ 439 sz += paddr & ~CACHE_LINE_MASK; 440 paddr &= CACHE_LINE_MASK; 441 442 /* 443 * account for any trailing gap to end of cache line 444 * this is equivalent to DIV_ROUND_UP() in line ops above 445 */ 446 sz += L1_CACHE_BYTES - 1; 447 } 448 449 if (is_pae40_enabled()) { 450 /* TBD: check if crossing 4TB boundary */ 451 if (op == OP_INV_IC) 452 write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 453 else 454 write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32); 455 } 456 457 /* ENDR needs to be set ahead of START */ 458 write_aux_reg(e, paddr + sz); /* ENDR is exclusive */ 459 write_aux_reg(s, paddr); 460 461 /* caller waits on DC_CTRL.FS */ 462 } 463 464 #endif 465 466 #if (CONFIG_ARC_MMU_VER < 3) 467 #define __cache_line_loop __cache_line_loop_v2 468 #elif (CONFIG_ARC_MMU_VER == 3) 469 #define __cache_line_loop __cache_line_loop_v3 470 #elif (CONFIG_ARC_MMU_VER > 3) 471 #define __cache_line_loop __cache_line_loop_v4 472 #endif 473 474 #ifdef CONFIG_ARC_HAS_DCACHE 475 476 /*************************************************************** 477 * Machine specific helpers for Entire D-Cache or Per Line ops 478 */ 479 480 #ifndef USE_RGN_FLSH 481 /* 482 * this version avoids extra read/write of DC_CTRL for flush or invalid ops 483 * in the non region flush regime (such as for ARCompact) 484 */ 485 static inline void __before_dc_op(const int op) 486 { 487 if (op == OP_FLUSH_N_INV) { 488 /* Dcache provides 2 cmd: FLUSH or INV 489 * INV inturn has sub-modes: DISCARD or FLUSH-BEFORE 490 * flush-n-inv is achieved by INV cmd but with IM=1 491 * So toggle INV sub-mode depending on op request and default 492 */ 493 const unsigned int ctl = ARC_REG_DC_CTRL; 494 write_aux_reg(ctl, read_aux_reg(ctl) | DC_CTRL_INV_MODE_FLUSH); 495 } 496 } 497 498 #else 499 500 static inline void __before_dc_op(const int op) 501 { 502 const unsigned int ctl = ARC_REG_DC_CTRL; 503 unsigned int val = read_aux_reg(ctl); 504 505 if (op == OP_FLUSH_N_INV) { 506 val |= DC_CTRL_INV_MODE_FLUSH; 507 } 508 509 if (op != OP_INV_IC) { 510 /* 511 * Flush / Invalidate is provided by DC_CTRL.RNG_OP 0 or 1 512 * combined Flush-n-invalidate uses DC_CTRL.IM = 1 set above 513 */ 514 val &= ~DC_CTRL_RGN_OP_MSK; 515 if (op & OP_INV) 516 val |= DC_CTRL_RGN_OP_INV; 517 } 518 write_aux_reg(ctl, val); 519 } 520 521 #endif 522 523 524 static inline void __after_dc_op(const int op) 525 { 526 if (op & OP_FLUSH) { 527 const unsigned int ctl = ARC_REG_DC_CTRL; 528 unsigned int reg; 529 530 /* flush / flush-n-inv both wait */ 531 while ((reg = read_aux_reg(ctl)) & DC_CTRL_FLUSH_STATUS) 532 ; 533 534 /* Switch back to default Invalidate mode */ 535 if (op == OP_FLUSH_N_INV) 536 write_aux_reg(ctl, reg & ~DC_CTRL_INV_MODE_FLUSH); 537 } 538 } 539 540 /* 541 * Operation on Entire D-Cache 542 * @op = {OP_INV, OP_FLUSH, OP_FLUSH_N_INV} 543 * Note that constant propagation ensures all the checks are gone 544 * in generated code 545 */ 546 static inline void __dc_entire_op(const int op) 547 { 548 int aux; 549 550 __before_dc_op(op); 551 552 if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 553 aux = ARC_REG_DC_IVDC; 554 else 555 aux = ARC_REG_DC_FLSH; 556 557 write_aux_reg(aux, 0x1); 558 559 __after_dc_op(op); 560 } 561 562 static inline void __dc_disable(void) 563 { 564 const int r = ARC_REG_DC_CTRL; 565 566 __dc_entire_op(OP_FLUSH_N_INV); 567 write_aux_reg(r, read_aux_reg(r) | DC_CTRL_DIS); 568 } 569 570 static void __dc_enable(void) 571 { 572 const int r = ARC_REG_DC_CTRL; 573 574 write_aux_reg(r, read_aux_reg(r) & ~DC_CTRL_DIS); 575 } 576 577 /* For kernel mappings cache operation: index is same as paddr */ 578 #define __dc_line_op_k(p, sz, op) __dc_line_op(p, p, sz, op) 579 580 /* 581 * D-Cache Line ops: Per Line INV (discard or wback+discard) or FLUSH (wback) 582 */ 583 static inline void __dc_line_op(phys_addr_t paddr, unsigned long vaddr, 584 unsigned long sz, const int op) 585 { 586 const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 587 unsigned long flags; 588 589 local_irq_save(flags); 590 591 __before_dc_op(op); 592 593 __cache_line_loop(paddr, vaddr, sz, op, full_page); 594 595 __after_dc_op(op); 596 597 local_irq_restore(flags); 598 } 599 600 #else 601 602 #define __dc_entire_op(op) 603 #define __dc_disable() 604 #define __dc_enable() 605 #define __dc_line_op(paddr, vaddr, sz, op) 606 #define __dc_line_op_k(paddr, sz, op) 607 608 #endif /* CONFIG_ARC_HAS_DCACHE */ 609 610 #ifdef CONFIG_ARC_HAS_ICACHE 611 612 static inline void __ic_entire_inv(void) 613 { 614 write_aux_reg(ARC_REG_IC_IVIC, 1); 615 read_aux_reg(ARC_REG_IC_CTRL); /* blocks */ 616 } 617 618 static inline void 619 __ic_line_inv_vaddr_local(phys_addr_t paddr, unsigned long vaddr, 620 unsigned long sz) 621 { 622 const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 623 unsigned long flags; 624 625 local_irq_save(flags); 626 (*_cache_line_loop_ic_fn)(paddr, vaddr, sz, OP_INV_IC, full_page); 627 local_irq_restore(flags); 628 } 629 630 #ifndef CONFIG_SMP 631 632 #define __ic_line_inv_vaddr(p, v, s) __ic_line_inv_vaddr_local(p, v, s) 633 634 #else 635 636 struct ic_inv_args { 637 phys_addr_t paddr, vaddr; 638 int sz; 639 }; 640 641 static void __ic_line_inv_vaddr_helper(void *info) 642 { 643 struct ic_inv_args *ic_inv = info; 644 645 __ic_line_inv_vaddr_local(ic_inv->paddr, ic_inv->vaddr, ic_inv->sz); 646 } 647 648 static void __ic_line_inv_vaddr(phys_addr_t paddr, unsigned long vaddr, 649 unsigned long sz) 650 { 651 struct ic_inv_args ic_inv = { 652 .paddr = paddr, 653 .vaddr = vaddr, 654 .sz = sz 655 }; 656 657 on_each_cpu(__ic_line_inv_vaddr_helper, &ic_inv, 1); 658 } 659 660 #endif /* CONFIG_SMP */ 661 662 #else /* !CONFIG_ARC_HAS_ICACHE */ 663 664 #define __ic_entire_inv() 665 #define __ic_line_inv_vaddr(pstart, vstart, sz) 666 667 #endif /* CONFIG_ARC_HAS_ICACHE */ 668 669 noinline void slc_op_rgn(phys_addr_t paddr, unsigned long sz, const int op) 670 { 671 #ifdef CONFIG_ISA_ARCV2 672 /* 673 * SLC is shared between all cores and concurrent aux operations from 674 * multiple cores need to be serialized using a spinlock 675 * A concurrent operation can be silently ignored and/or the old/new 676 * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop 677 * below) 678 */ 679 static DEFINE_SPINLOCK(lock); 680 unsigned long flags; 681 unsigned int ctrl; 682 phys_addr_t end; 683 684 spin_lock_irqsave(&lock, flags); 685 686 /* 687 * The Region Flush operation is specified by CTRL.RGN_OP[11..9] 688 * - b'000 (default) is Flush, 689 * - b'001 is Invalidate if CTRL.IM == 0 690 * - b'001 is Flush-n-Invalidate if CTRL.IM == 1 691 */ 692 ctrl = read_aux_reg(ARC_REG_SLC_CTRL); 693 694 /* Don't rely on default value of IM bit */ 695 if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 696 ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 697 else 698 ctrl |= SLC_CTRL_IM; 699 700 if (op & OP_INV) 701 ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */ 702 else 703 ctrl &= ~SLC_CTRL_RGN_OP_INV; 704 705 write_aux_reg(ARC_REG_SLC_CTRL, ctrl); 706 707 /* 708 * Lower bits are ignored, no need to clip 709 * END needs to be setup before START (latter triggers the operation) 710 * END can't be same as START, so add (l2_line_sz - 1) to sz 711 */ 712 end = paddr + sz + l2_line_sz - 1; 713 if (is_pae40_enabled()) 714 write_aux_reg(ARC_REG_SLC_RGN_END1, upper_32_bits(end)); 715 716 write_aux_reg(ARC_REG_SLC_RGN_END, lower_32_bits(end)); 717 718 if (is_pae40_enabled()) 719 write_aux_reg(ARC_REG_SLC_RGN_START1, upper_32_bits(paddr)); 720 721 write_aux_reg(ARC_REG_SLC_RGN_START, lower_32_bits(paddr)); 722 723 /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 724 read_aux_reg(ARC_REG_SLC_CTRL); 725 726 while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY); 727 728 spin_unlock_irqrestore(&lock, flags); 729 #endif 730 } 731 732 noinline void slc_op_line(phys_addr_t paddr, unsigned long sz, const int op) 733 { 734 #ifdef CONFIG_ISA_ARCV2 735 /* 736 * SLC is shared between all cores and concurrent aux operations from 737 * multiple cores need to be serialized using a spinlock 738 * A concurrent operation can be silently ignored and/or the old/new 739 * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop 740 * below) 741 */ 742 static DEFINE_SPINLOCK(lock); 743 744 const unsigned long SLC_LINE_MASK = ~(l2_line_sz - 1); 745 unsigned int ctrl, cmd; 746 unsigned long flags; 747 int num_lines; 748 749 spin_lock_irqsave(&lock, flags); 750 751 ctrl = read_aux_reg(ARC_REG_SLC_CTRL); 752 753 /* Don't rely on default value of IM bit */ 754 if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 755 ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 756 else 757 ctrl |= SLC_CTRL_IM; 758 759 write_aux_reg(ARC_REG_SLC_CTRL, ctrl); 760 761 cmd = op & OP_INV ? ARC_AUX_SLC_IVDL : ARC_AUX_SLC_FLDL; 762 763 sz += paddr & ~SLC_LINE_MASK; 764 paddr &= SLC_LINE_MASK; 765 766 num_lines = DIV_ROUND_UP(sz, l2_line_sz); 767 768 while (num_lines-- > 0) { 769 write_aux_reg(cmd, paddr); 770 paddr += l2_line_sz; 771 } 772 773 /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 774 read_aux_reg(ARC_REG_SLC_CTRL); 775 776 while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY); 777 778 spin_unlock_irqrestore(&lock, flags); 779 #endif 780 } 781 782 #define slc_op(paddr, sz, op) slc_op_rgn(paddr, sz, op) 783 784 noinline static void slc_entire_op(const int op) 785 { 786 unsigned int ctrl, r = ARC_REG_SLC_CTRL; 787 788 ctrl = read_aux_reg(r); 789 790 if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 791 ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 792 else 793 ctrl |= SLC_CTRL_IM; 794 795 write_aux_reg(r, ctrl); 796 797 if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 798 write_aux_reg(ARC_REG_SLC_INVALIDATE, 0x1); 799 else 800 write_aux_reg(ARC_REG_SLC_FLUSH, 0x1); 801 802 /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 803 read_aux_reg(r); 804 805 /* Important to wait for flush to complete */ 806 while (read_aux_reg(r) & SLC_CTRL_BUSY); 807 } 808 809 static inline void arc_slc_disable(void) 810 { 811 const int r = ARC_REG_SLC_CTRL; 812 813 slc_entire_op(OP_FLUSH_N_INV); 814 write_aux_reg(r, read_aux_reg(r) | SLC_CTRL_DIS); 815 } 816 817 static inline void arc_slc_enable(void) 818 { 819 const int r = ARC_REG_SLC_CTRL; 820 821 write_aux_reg(r, read_aux_reg(r) & ~SLC_CTRL_DIS); 822 } 823 824 /*********************************************************** 825 * Exported APIs 826 */ 827 828 /* 829 * Handle cache congruency of kernel and userspace mappings of page when kernel 830 * writes-to/reads-from 831 * 832 * The idea is to defer flushing of kernel mapping after a WRITE, possible if: 833 * -dcache is NOT aliasing, hence any U/K-mappings of page are congruent 834 * -U-mapping doesn't exist yet for page (finalised in update_mmu_cache) 835 * -In SMP, if hardware caches are coherent 836 * 837 * There's a corollary case, where kernel READs from a userspace mapped page. 838 * If the U-mapping is not congruent to to K-mapping, former needs flushing. 839 */ 840 void flush_dcache_page(struct page *page) 841 { 842 struct address_space *mapping; 843 844 if (!cache_is_vipt_aliasing()) { 845 clear_bit(PG_dc_clean, &page->flags); 846 return; 847 } 848 849 /* don't handle anon pages here */ 850 mapping = page_mapping_file(page); 851 if (!mapping) 852 return; 853 854 /* 855 * pagecache page, file not yet mapped to userspace 856 * Make a note that K-mapping is dirty 857 */ 858 if (!mapping_mapped(mapping)) { 859 clear_bit(PG_dc_clean, &page->flags); 860 } else if (page_mapcount(page)) { 861 862 /* kernel reading from page with U-mapping */ 863 phys_addr_t paddr = (unsigned long)page_address(page); 864 unsigned long vaddr = page->index << PAGE_SHIFT; 865 866 if (addr_not_cache_congruent(paddr, vaddr)) 867 __flush_dcache_page(paddr, vaddr); 868 } 869 } 870 EXPORT_SYMBOL(flush_dcache_page); 871 872 /* 873 * DMA ops for systems with L1 cache only 874 * Make memory coherent with L1 cache by flushing/invalidating L1 lines 875 */ 876 static void __dma_cache_wback_inv_l1(phys_addr_t start, unsigned long sz) 877 { 878 __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 879 } 880 881 static void __dma_cache_inv_l1(phys_addr_t start, unsigned long sz) 882 { 883 __dc_line_op_k(start, sz, OP_INV); 884 } 885 886 static void __dma_cache_wback_l1(phys_addr_t start, unsigned long sz) 887 { 888 __dc_line_op_k(start, sz, OP_FLUSH); 889 } 890 891 /* 892 * DMA ops for systems with both L1 and L2 caches, but without IOC 893 * Both L1 and L2 lines need to be explicitly flushed/invalidated 894 */ 895 static void __dma_cache_wback_inv_slc(phys_addr_t start, unsigned long sz) 896 { 897 __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 898 slc_op(start, sz, OP_FLUSH_N_INV); 899 } 900 901 static void __dma_cache_inv_slc(phys_addr_t start, unsigned long sz) 902 { 903 __dc_line_op_k(start, sz, OP_INV); 904 slc_op(start, sz, OP_INV); 905 } 906 907 static void __dma_cache_wback_slc(phys_addr_t start, unsigned long sz) 908 { 909 __dc_line_op_k(start, sz, OP_FLUSH); 910 slc_op(start, sz, OP_FLUSH); 911 } 912 913 /* 914 * Exported DMA API 915 */ 916 void dma_cache_wback_inv(phys_addr_t start, unsigned long sz) 917 { 918 __dma_cache_wback_inv(start, sz); 919 } 920 EXPORT_SYMBOL(dma_cache_wback_inv); 921 922 void dma_cache_inv(phys_addr_t start, unsigned long sz) 923 { 924 __dma_cache_inv(start, sz); 925 } 926 EXPORT_SYMBOL(dma_cache_inv); 927 928 void dma_cache_wback(phys_addr_t start, unsigned long sz) 929 { 930 __dma_cache_wback(start, sz); 931 } 932 EXPORT_SYMBOL(dma_cache_wback); 933 934 /* 935 * This is API for making I/D Caches consistent when modifying 936 * kernel code (loadable modules, kprobes, kgdb...) 937 * This is called on insmod, with kernel virtual address for CODE of 938 * the module. ARC cache maintenance ops require PHY address thus we 939 * need to convert vmalloc addr to PHY addr 940 */ 941 void flush_icache_range(unsigned long kstart, unsigned long kend) 942 { 943 unsigned int tot_sz; 944 945 WARN(kstart < TASK_SIZE, "%s() can't handle user vaddr", __func__); 946 947 /* Shortcut for bigger flush ranges. 948 * Here we don't care if this was kernel virtual or phy addr 949 */ 950 tot_sz = kend - kstart; 951 if (tot_sz > PAGE_SIZE) { 952 flush_cache_all(); 953 return; 954 } 955 956 /* Case: Kernel Phy addr (0x8000_0000 onwards) */ 957 if (likely(kstart > PAGE_OFFSET)) { 958 /* 959 * The 2nd arg despite being paddr will be used to index icache 960 * This is OK since no alternate virtual mappings will exist 961 * given the callers for this case: kprobe/kgdb in built-in 962 * kernel code only. 963 */ 964 __sync_icache_dcache(kstart, kstart, kend - kstart); 965 return; 966 } 967 968 /* 969 * Case: Kernel Vaddr (0x7000_0000 to 0x7fff_ffff) 970 * (1) ARC Cache Maintenance ops only take Phy addr, hence special 971 * handling of kernel vaddr. 972 * 973 * (2) Despite @tot_sz being < PAGE_SIZE (bigger cases handled already), 974 * it still needs to handle a 2 page scenario, where the range 975 * straddles across 2 virtual pages and hence need for loop 976 */ 977 while (tot_sz > 0) { 978 unsigned int off, sz; 979 unsigned long phy, pfn; 980 981 off = kstart % PAGE_SIZE; 982 pfn = vmalloc_to_pfn((void *)kstart); 983 phy = (pfn << PAGE_SHIFT) + off; 984 sz = min_t(unsigned int, tot_sz, PAGE_SIZE - off); 985 __sync_icache_dcache(phy, kstart, sz); 986 kstart += sz; 987 tot_sz -= sz; 988 } 989 } 990 EXPORT_SYMBOL(flush_icache_range); 991 992 /* 993 * General purpose helper to make I and D cache lines consistent. 994 * @paddr is phy addr of region 995 * @vaddr is typically user vaddr (breakpoint) or kernel vaddr (vmalloc) 996 * However in one instance, when called by kprobe (for a breakpt in 997 * builtin kernel code) @vaddr will be paddr only, meaning CDU operation will 998 * use a paddr to index the cache (despite VIPT). This is fine since since a 999 * builtin kernel page will not have any virtual mappings. 1000 * kprobe on loadable module will be kernel vaddr. 1001 */ 1002 void __sync_icache_dcache(phys_addr_t paddr, unsigned long vaddr, int len) 1003 { 1004 __dc_line_op(paddr, vaddr, len, OP_FLUSH_N_INV); 1005 __ic_line_inv_vaddr(paddr, vaddr, len); 1006 } 1007 1008 /* wrapper to compile time eliminate alignment checks in flush loop */ 1009 void __inv_icache_page(phys_addr_t paddr, unsigned long vaddr) 1010 { 1011 __ic_line_inv_vaddr(paddr, vaddr, PAGE_SIZE); 1012 } 1013 1014 /* 1015 * wrapper to clearout kernel or userspace mappings of a page 1016 * For kernel mappings @vaddr == @paddr 1017 */ 1018 void __flush_dcache_page(phys_addr_t paddr, unsigned long vaddr) 1019 { 1020 __dc_line_op(paddr, vaddr & PAGE_MASK, PAGE_SIZE, OP_FLUSH_N_INV); 1021 } 1022 1023 noinline void flush_cache_all(void) 1024 { 1025 unsigned long flags; 1026 1027 local_irq_save(flags); 1028 1029 __ic_entire_inv(); 1030 __dc_entire_op(OP_FLUSH_N_INV); 1031 1032 local_irq_restore(flags); 1033 1034 } 1035 1036 #ifdef CONFIG_ARC_CACHE_VIPT_ALIASING 1037 1038 void flush_cache_mm(struct mm_struct *mm) 1039 { 1040 flush_cache_all(); 1041 } 1042 1043 void flush_cache_page(struct vm_area_struct *vma, unsigned long u_vaddr, 1044 unsigned long pfn) 1045 { 1046 phys_addr_t paddr = pfn << PAGE_SHIFT; 1047 1048 u_vaddr &= PAGE_MASK; 1049 1050 __flush_dcache_page(paddr, u_vaddr); 1051 1052 if (vma->vm_flags & VM_EXEC) 1053 __inv_icache_page(paddr, u_vaddr); 1054 } 1055 1056 void flush_cache_range(struct vm_area_struct *vma, unsigned long start, 1057 unsigned long end) 1058 { 1059 flush_cache_all(); 1060 } 1061 1062 void flush_anon_page(struct vm_area_struct *vma, struct page *page, 1063 unsigned long u_vaddr) 1064 { 1065 /* TBD: do we really need to clear the kernel mapping */ 1066 __flush_dcache_page((phys_addr_t)page_address(page), u_vaddr); 1067 __flush_dcache_page((phys_addr_t)page_address(page), 1068 (phys_addr_t)page_address(page)); 1069 1070 } 1071 1072 #endif 1073 1074 void copy_user_highpage(struct page *to, struct page *from, 1075 unsigned long u_vaddr, struct vm_area_struct *vma) 1076 { 1077 void *kfrom = kmap_atomic(from); 1078 void *kto = kmap_atomic(to); 1079 int clean_src_k_mappings = 0; 1080 1081 /* 1082 * If SRC page was already mapped in userspace AND it's U-mapping is 1083 * not congruent with K-mapping, sync former to physical page so that 1084 * K-mapping in memcpy below, sees the right data 1085 * 1086 * Note that while @u_vaddr refers to DST page's userspace vaddr, it is 1087 * equally valid for SRC page as well 1088 * 1089 * For !VIPT cache, all of this gets compiled out as 1090 * addr_not_cache_congruent() is 0 1091 */ 1092 if (page_mapcount(from) && addr_not_cache_congruent(kfrom, u_vaddr)) { 1093 __flush_dcache_page((unsigned long)kfrom, u_vaddr); 1094 clean_src_k_mappings = 1; 1095 } 1096 1097 copy_page(kto, kfrom); 1098 1099 /* 1100 * Mark DST page K-mapping as dirty for a later finalization by 1101 * update_mmu_cache(). Although the finalization could have been done 1102 * here as well (given that both vaddr/paddr are available). 1103 * But update_mmu_cache() already has code to do that for other 1104 * non copied user pages (e.g. read faults which wire in pagecache page 1105 * directly). 1106 */ 1107 clear_bit(PG_dc_clean, &to->flags); 1108 1109 /* 1110 * if SRC was already usermapped and non-congruent to kernel mapping 1111 * sync the kernel mapping back to physical page 1112 */ 1113 if (clean_src_k_mappings) { 1114 __flush_dcache_page((unsigned long)kfrom, (unsigned long)kfrom); 1115 set_bit(PG_dc_clean, &from->flags); 1116 } else { 1117 clear_bit(PG_dc_clean, &from->flags); 1118 } 1119 1120 kunmap_atomic(kto); 1121 kunmap_atomic(kfrom); 1122 } 1123 1124 void clear_user_page(void *to, unsigned long u_vaddr, struct page *page) 1125 { 1126 clear_page(to); 1127 clear_bit(PG_dc_clean, &page->flags); 1128 } 1129 1130 1131 /********************************************************************** 1132 * Explicit Cache flush request from user space via syscall 1133 * Needed for JITs which generate code on the fly 1134 */ 1135 SYSCALL_DEFINE3(cacheflush, uint32_t, start, uint32_t, sz, uint32_t, flags) 1136 { 1137 /* TBD: optimize this */ 1138 flush_cache_all(); 1139 return 0; 1140 } 1141 1142 /* 1143 * IO-Coherency (IOC) setup rules: 1144 * 1145 * 1. Needs to be at system level, so only once by Master core 1146 * Non-Masters need not be accessing caches at that time 1147 * - They are either HALT_ON_RESET and kick started much later or 1148 * - if run on reset, need to ensure that arc_platform_smp_wait_to_boot() 1149 * doesn't perturb caches or coherency unit 1150 * 1151 * 2. caches (L1 and SLC) need to be purged (flush+inv) before setting up IOC, 1152 * otherwise any straggler data might behave strangely post IOC enabling 1153 * 1154 * 3. All Caches need to be disabled when setting up IOC to elide any in-flight 1155 * Coherency transactions 1156 */ 1157 noinline void __init arc_ioc_setup(void) 1158 { 1159 unsigned int ioc_base, mem_sz; 1160 1161 /* 1162 * If IOC was already enabled (due to bootloader) it technically needs to 1163 * be reconfigured with aperture base,size corresponding to Linux memory map 1164 * which will certainly be different than uboot's. But disabling and 1165 * reenabling IOC when DMA might be potentially active is tricky business. 1166 * To avoid random memory issues later, just panic here and ask user to 1167 * upgrade bootloader to one which doesn't enable IOC 1168 */ 1169 if (read_aux_reg(ARC_REG_IO_COH_ENABLE) & ARC_IO_COH_ENABLE_BIT) 1170 panic("IOC already enabled, please upgrade bootloader!\n"); 1171 1172 if (!ioc_enable) 1173 return; 1174 1175 /* Flush + invalidate + disable L1 dcache */ 1176 __dc_disable(); 1177 1178 /* Flush + invalidate SLC */ 1179 if (read_aux_reg(ARC_REG_SLC_BCR)) 1180 slc_entire_op(OP_FLUSH_N_INV); 1181 1182 /* 1183 * currently IOC Aperture covers entire DDR 1184 * TBD: fix for PGU + 1GB of low mem 1185 * TBD: fix for PAE 1186 */ 1187 mem_sz = arc_get_mem_sz(); 1188 1189 if (!is_power_of_2(mem_sz) || mem_sz < 4096) 1190 panic("IOC Aperture size must be power of 2 larger than 4KB"); 1191 1192 /* 1193 * IOC Aperture size decoded as 2 ^ (SIZE + 2) KB, 1194 * so setting 0x11 implies 512MB, 0x12 implies 1GB... 1195 */ 1196 write_aux_reg(ARC_REG_IO_COH_AP0_SIZE, order_base_2(mem_sz >> 10) - 2); 1197 1198 /* for now assume kernel base is start of IOC aperture */ 1199 ioc_base = CONFIG_LINUX_RAM_BASE; 1200 1201 if (ioc_base % mem_sz != 0) 1202 panic("IOC Aperture start must be aligned to the size of the aperture"); 1203 1204 write_aux_reg(ARC_REG_IO_COH_AP0_BASE, ioc_base >> 12); 1205 write_aux_reg(ARC_REG_IO_COH_PARTIAL, ARC_IO_COH_PARTIAL_BIT); 1206 write_aux_reg(ARC_REG_IO_COH_ENABLE, ARC_IO_COH_ENABLE_BIT); 1207 1208 /* Re-enable L1 dcache */ 1209 __dc_enable(); 1210 } 1211 1212 /* 1213 * Cache related boot time checks/setups only needed on master CPU: 1214 * - Geometry checks (kernel build and hardware agree: e.g. L1_CACHE_BYTES) 1215 * Assume SMP only, so all cores will have same cache config. A check on 1216 * one core suffices for all 1217 * - IOC setup / dma callbacks only need to be done once 1218 */ 1219 void __init arc_cache_init_master(void) 1220 { 1221 unsigned int __maybe_unused cpu = smp_processor_id(); 1222 1223 if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) { 1224 struct cpuinfo_arc_cache *ic = &cpuinfo_arc700[cpu].icache; 1225 1226 if (!ic->line_len) 1227 panic("cache support enabled but non-existent cache\n"); 1228 1229 if (ic->line_len != L1_CACHE_BYTES) 1230 panic("ICache line [%d] != kernel Config [%d]", 1231 ic->line_len, L1_CACHE_BYTES); 1232 1233 /* 1234 * In MMU v4 (HS38x) the aliasing icache config uses IVIL/PTAG 1235 * pair to provide vaddr/paddr respectively, just as in MMU v3 1236 */ 1237 if (is_isa_arcv2() && ic->alias) 1238 _cache_line_loop_ic_fn = __cache_line_loop_v3; 1239 else 1240 _cache_line_loop_ic_fn = __cache_line_loop; 1241 } 1242 1243 if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) { 1244 struct cpuinfo_arc_cache *dc = &cpuinfo_arc700[cpu].dcache; 1245 1246 if (!dc->line_len) 1247 panic("cache support enabled but non-existent cache\n"); 1248 1249 if (dc->line_len != L1_CACHE_BYTES) 1250 panic("DCache line [%d] != kernel Config [%d]", 1251 dc->line_len, L1_CACHE_BYTES); 1252 1253 /* check for D-Cache aliasing on ARCompact: ARCv2 has PIPT */ 1254 if (is_isa_arcompact()) { 1255 int handled = IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING); 1256 int num_colors = dc->sz_k/dc->assoc/TO_KB(PAGE_SIZE); 1257 1258 if (dc->alias) { 1259 if (!handled) 1260 panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 1261 if (CACHE_COLORS_NUM != num_colors) 1262 panic("CACHE_COLORS_NUM not optimized for config\n"); 1263 } else if (!dc->alias && handled) { 1264 panic("Disable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 1265 } 1266 } 1267 } 1268 1269 /* 1270 * Check that SMP_CACHE_BYTES (and hence ARCH_DMA_MINALIGN) is larger 1271 * or equal to any cache line length. 1272 */ 1273 BUILD_BUG_ON_MSG(L1_CACHE_BYTES > SMP_CACHE_BYTES, 1274 "SMP_CACHE_BYTES must be >= any cache line length"); 1275 if (is_isa_arcv2() && (l2_line_sz > SMP_CACHE_BYTES)) 1276 panic("L2 Cache line [%d] > kernel Config [%d]\n", 1277 l2_line_sz, SMP_CACHE_BYTES); 1278 1279 /* Note that SLC disable not formally supported till HS 3.0 */ 1280 if (is_isa_arcv2() && l2_line_sz && !slc_enable) 1281 arc_slc_disable(); 1282 1283 if (is_isa_arcv2() && ioc_exists) 1284 arc_ioc_setup(); 1285 1286 if (is_isa_arcv2() && l2_line_sz && slc_enable) { 1287 __dma_cache_wback_inv = __dma_cache_wback_inv_slc; 1288 __dma_cache_inv = __dma_cache_inv_slc; 1289 __dma_cache_wback = __dma_cache_wback_slc; 1290 } else { 1291 __dma_cache_wback_inv = __dma_cache_wback_inv_l1; 1292 __dma_cache_inv = __dma_cache_inv_l1; 1293 __dma_cache_wback = __dma_cache_wback_l1; 1294 } 1295 /* 1296 * In case of IOC (say IOC+SLC case), pointers above could still be set 1297 * but end up not being relevant as the first function in chain is not 1298 * called at all for devices using coherent DMA. 1299 * arch_sync_dma_for_cpu() -> dma_cache_*() -> __dma_cache_*() 1300 */ 1301 } 1302 1303 void __ref arc_cache_init(void) 1304 { 1305 unsigned int __maybe_unused cpu = smp_processor_id(); 1306 char str[256]; 1307 1308 pr_info("%s", arc_cache_mumbojumbo(0, str, sizeof(str))); 1309 1310 if (!cpu) 1311 arc_cache_init_master(); 1312 1313 /* 1314 * In PAE regime, TLB and cache maintenance ops take wider addresses 1315 * And even if PAE is not enabled in kernel, the upper 32-bits still need 1316 * to be zeroed to keep the ops sane. 1317 * As an optimization for more common !PAE enabled case, zero them out 1318 * once at init, rather than checking/setting to 0 for every runtime op 1319 */ 1320 if (is_isa_arcv2() && pae40_exist_but_not_enab()) { 1321 1322 if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) 1323 write_aux_reg(ARC_REG_IC_PTAG_HI, 0); 1324 1325 if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) 1326 write_aux_reg(ARC_REG_DC_PTAG_HI, 0); 1327 1328 if (l2_line_sz) { 1329 write_aux_reg(ARC_REG_SLC_RGN_END1, 0); 1330 write_aux_reg(ARC_REG_SLC_RGN_START1, 0); 1331 } 1332 } 1333 } 1334