18362c389SVineet Gupta /* 28ea2ddffSVineet Gupta * ARC Cache Management 38362c389SVineet Gupta * 48ea2ddffSVineet Gupta * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com) 58362c389SVineet Gupta * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 68362c389SVineet Gupta * 78362c389SVineet Gupta * This program is free software; you can redistribute it and/or modify 88362c389SVineet Gupta * it under the terms of the GNU General Public License version 2 as 98362c389SVineet Gupta * published by the Free Software Foundation. 108362c389SVineet Gupta */ 118362c389SVineet Gupta 128362c389SVineet Gupta #include <linux/module.h> 138362c389SVineet Gupta #include <linux/mm.h> 148362c389SVineet Gupta #include <linux/sched.h> 158362c389SVineet Gupta #include <linux/cache.h> 168362c389SVineet Gupta #include <linux/mmu_context.h> 178362c389SVineet Gupta #include <linux/syscalls.h> 188362c389SVineet Gupta #include <linux/uaccess.h> 198362c389SVineet Gupta #include <linux/pagemap.h> 208362c389SVineet Gupta #include <asm/cacheflush.h> 218362c389SVineet Gupta #include <asm/cachectl.h> 228362c389SVineet Gupta #include <asm/setup.h> 238362c389SVineet Gupta 24795f4558SVineet Gupta static int l2_line_sz; 25f2b0b25aSAlexey Brodkin int ioc_exists; 261648c70dSAlexey Brodkin volatile int slc_enable = 1, ioc_enable = 1; 27795f4558SVineet Gupta 28bcc4d65aSVineet Gupta void (*_cache_line_loop_ic_fn)(unsigned long paddr, unsigned long vaddr, 29bcc4d65aSVineet Gupta unsigned long sz, const int cacheop); 30bcc4d65aSVineet Gupta 31f2b0b25aSAlexey Brodkin void (*__dma_cache_wback_inv)(unsigned long start, unsigned long sz); 32f2b0b25aSAlexey Brodkin void (*__dma_cache_inv)(unsigned long start, unsigned long sz); 33f2b0b25aSAlexey Brodkin void (*__dma_cache_wback)(unsigned long start, unsigned long sz); 34f2b0b25aSAlexey Brodkin 358362c389SVineet Gupta char *arc_cache_mumbojumbo(int c, char *buf, int len) 368362c389SVineet Gupta { 378362c389SVineet Gupta int n = 0; 38d1f317d8SVineet Gupta struct cpuinfo_arc_cache *p; 398362c389SVineet Gupta 4079335a2cSVineet Gupta #define IS_USED_RUN(v) ((v) ? "" : "(disabled) ") 418362c389SVineet Gupta #define PR_CACHE(p, cfg, str) \ 428362c389SVineet Gupta if (!(p)->ver) \ 438362c389SVineet Gupta n += scnprintf(buf + n, len - n, str"\t\t: N/A\n"); \ 448362c389SVineet Gupta else \ 458362c389SVineet Gupta n += scnprintf(buf + n, len - n, \ 468362c389SVineet Gupta str"\t\t: %uK, %dway/set, %uB Line, %s%s%s\n", \ 478362c389SVineet Gupta (p)->sz_k, (p)->assoc, (p)->line_len, \ 488362c389SVineet Gupta (p)->vipt ? "VIPT" : "PIPT", \ 498362c389SVineet Gupta (p)->alias ? " aliasing" : "", \ 508362c389SVineet Gupta IS_ENABLED(cfg) ? "" : " (not used)"); 518362c389SVineet Gupta 528362c389SVineet Gupta PR_CACHE(&cpuinfo_arc700[c].icache, CONFIG_ARC_HAS_ICACHE, "I-Cache"); 538362c389SVineet Gupta PR_CACHE(&cpuinfo_arc700[c].dcache, CONFIG_ARC_HAS_DCACHE, "D-Cache"); 548362c389SVineet Gupta 55fd0881a2SVineet Gupta if (!is_isa_arcv2()) 56fd0881a2SVineet Gupta return buf; 57fd0881a2SVineet Gupta 58d1f317d8SVineet Gupta p = &cpuinfo_arc700[c].slc; 59d1f317d8SVineet Gupta if (p->ver) 60d1f317d8SVineet Gupta n += scnprintf(buf + n, len - n, 6179335a2cSVineet Gupta "SLC\t\t: %uK, %uB Line%s\n", 6279335a2cSVineet Gupta p->sz_k, p->line_len, IS_USED_RUN(slc_enable)); 63d1f317d8SVineet Gupta 64f2b0b25aSAlexey Brodkin if (ioc_exists) 651648c70dSAlexey Brodkin n += scnprintf(buf + n, len - n, "IOC\t\t:%s\n", 661648c70dSAlexey Brodkin IS_USED_RUN(ioc_enable)); 67f2b0b25aSAlexey Brodkin 688362c389SVineet Gupta return buf; 698362c389SVineet Gupta } 708362c389SVineet Gupta 718362c389SVineet Gupta /* 728362c389SVineet Gupta * Read the Cache Build Confuration Registers, Decode them and save into 738362c389SVineet Gupta * the cpuinfo structure for later use. 748362c389SVineet Gupta * No Validation done here, simply read/convert the BCRs 758362c389SVineet Gupta */ 76fd0881a2SVineet Gupta static void read_decode_cache_bcr_arcv2(int cpu) 778362c389SVineet Gupta { 78fd0881a2SVineet Gupta struct cpuinfo_arc_cache *p_slc = &cpuinfo_arc700[cpu].slc; 79d1f317d8SVineet Gupta struct bcr_generic sbcr; 80d1f317d8SVineet Gupta 81d1f317d8SVineet Gupta struct bcr_slc_cfg { 82d1f317d8SVineet Gupta #ifdef CONFIG_CPU_BIG_ENDIAN 83d1f317d8SVineet Gupta unsigned int pad:24, way:2, lsz:2, sz:4; 84d1f317d8SVineet Gupta #else 85d1f317d8SVineet Gupta unsigned int sz:4, lsz:2, way:2, pad:24; 86d1f317d8SVineet Gupta #endif 87d1f317d8SVineet Gupta } slc_cfg; 88d1f317d8SVineet Gupta 89f2b0b25aSAlexey Brodkin struct bcr_clust_cfg { 90f2b0b25aSAlexey Brodkin #ifdef CONFIG_CPU_BIG_ENDIAN 91f2b0b25aSAlexey Brodkin unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8; 92f2b0b25aSAlexey Brodkin #else 93f2b0b25aSAlexey Brodkin unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7; 94f2b0b25aSAlexey Brodkin #endif 95f2b0b25aSAlexey Brodkin } cbcr; 96f2b0b25aSAlexey Brodkin 97fd0881a2SVineet Gupta READ_BCR(ARC_REG_SLC_BCR, sbcr); 98fd0881a2SVineet Gupta if (sbcr.ver) { 99fd0881a2SVineet Gupta READ_BCR(ARC_REG_SLC_CFG, slc_cfg); 100fd0881a2SVineet Gupta p_slc->ver = sbcr.ver; 101fd0881a2SVineet Gupta p_slc->sz_k = 128 << slc_cfg.sz; 102fd0881a2SVineet Gupta l2_line_sz = p_slc->line_len = (slc_cfg.lsz == 0) ? 128 : 64; 103fd0881a2SVineet Gupta } 104fd0881a2SVineet Gupta 105fd0881a2SVineet Gupta READ_BCR(ARC_REG_CLUSTER_BCR, cbcr); 106fd0881a2SVineet Gupta if (cbcr.c && ioc_enable) 107fd0881a2SVineet Gupta ioc_exists = 1; 108fd0881a2SVineet Gupta } 109fd0881a2SVineet Gupta 110fd0881a2SVineet Gupta void read_decode_cache_bcr(void) 111fd0881a2SVineet Gupta { 112fd0881a2SVineet Gupta struct cpuinfo_arc_cache *p_ic, *p_dc; 113fd0881a2SVineet Gupta unsigned int cpu = smp_processor_id(); 114fd0881a2SVineet Gupta struct bcr_cache { 115fd0881a2SVineet Gupta #ifdef CONFIG_CPU_BIG_ENDIAN 116fd0881a2SVineet Gupta unsigned int pad:12, line_len:4, sz:4, config:4, ver:8; 117fd0881a2SVineet Gupta #else 118fd0881a2SVineet Gupta unsigned int ver:8, config:4, sz:4, line_len:4, pad:12; 119fd0881a2SVineet Gupta #endif 120fd0881a2SVineet Gupta } ibcr, dbcr; 121fd0881a2SVineet Gupta 1228362c389SVineet Gupta p_ic = &cpuinfo_arc700[cpu].icache; 1238362c389SVineet Gupta READ_BCR(ARC_REG_IC_BCR, ibcr); 1248362c389SVineet Gupta 1258362c389SVineet Gupta if (!ibcr.ver) 1268362c389SVineet Gupta goto dc_chk; 1278362c389SVineet Gupta 128d1f317d8SVineet Gupta if (ibcr.ver <= 3) { 1298362c389SVineet Gupta BUG_ON(ibcr.config != 3); 1308362c389SVineet Gupta p_ic->assoc = 2; /* Fixed to 2w set assoc */ 131d1f317d8SVineet Gupta } else if (ibcr.ver >= 4) { 132d1f317d8SVineet Gupta p_ic->assoc = 1 << ibcr.config; /* 1,2,4,8 */ 133d1f317d8SVineet Gupta } 134d1f317d8SVineet Gupta 1358362c389SVineet Gupta p_ic->line_len = 8 << ibcr.line_len; 1368362c389SVineet Gupta p_ic->sz_k = 1 << (ibcr.sz - 1); 1378362c389SVineet Gupta p_ic->ver = ibcr.ver; 1388362c389SVineet Gupta p_ic->vipt = 1; 1398362c389SVineet Gupta p_ic->alias = p_ic->sz_k/p_ic->assoc/TO_KB(PAGE_SIZE) > 1; 1408362c389SVineet Gupta 1418362c389SVineet Gupta dc_chk: 1428362c389SVineet Gupta p_dc = &cpuinfo_arc700[cpu].dcache; 1438362c389SVineet Gupta READ_BCR(ARC_REG_DC_BCR, dbcr); 1448362c389SVineet Gupta 1458362c389SVineet Gupta if (!dbcr.ver) 146d1f317d8SVineet Gupta goto slc_chk; 1478362c389SVineet Gupta 148d1f317d8SVineet Gupta if (dbcr.ver <= 3) { 1498362c389SVineet Gupta BUG_ON(dbcr.config != 2); 1508362c389SVineet Gupta p_dc->assoc = 4; /* Fixed to 4w set assoc */ 151d1f317d8SVineet Gupta p_dc->vipt = 1; 152d1f317d8SVineet Gupta p_dc->alias = p_dc->sz_k/p_dc->assoc/TO_KB(PAGE_SIZE) > 1; 153d1f317d8SVineet Gupta } else if (dbcr.ver >= 4) { 154d1f317d8SVineet Gupta p_dc->assoc = 1 << dbcr.config; /* 1,2,4,8 */ 155d1f317d8SVineet Gupta p_dc->vipt = 0; 156d1f317d8SVineet Gupta p_dc->alias = 0; /* PIPT so can't VIPT alias */ 157d1f317d8SVineet Gupta } 158d1f317d8SVineet Gupta 1598362c389SVineet Gupta p_dc->line_len = 16 << dbcr.line_len; 1608362c389SVineet Gupta p_dc->sz_k = 1 << (dbcr.sz - 1); 1618362c389SVineet Gupta p_dc->ver = dbcr.ver; 162d1f317d8SVineet Gupta 163d1f317d8SVineet Gupta slc_chk: 164fd0881a2SVineet Gupta if (is_isa_arcv2()) 165fd0881a2SVineet Gupta read_decode_cache_bcr_arcv2(cpu); 1668362c389SVineet Gupta } 1678362c389SVineet Gupta 1688362c389SVineet Gupta /* 1698ea2ddffSVineet Gupta * Line Operation on {I,D}-Cache 1708362c389SVineet Gupta */ 1718362c389SVineet Gupta 1728362c389SVineet Gupta #define OP_INV 0x1 1738362c389SVineet Gupta #define OP_FLUSH 0x2 1748362c389SVineet Gupta #define OP_FLUSH_N_INV 0x3 1758362c389SVineet Gupta #define OP_INV_IC 0x4 1768362c389SVineet Gupta 1778362c389SVineet Gupta /* 1788ea2ddffSVineet Gupta * I-Cache Aliasing in ARC700 VIPT caches (MMU v1-v3) 1798ea2ddffSVineet Gupta * 1808ea2ddffSVineet Gupta * ARC VIPT I-cache uses vaddr to index into cache and paddr to match the tag. 1818ea2ddffSVineet Gupta * The orig Cache Management Module "CDU" only required paddr to invalidate a 1828ea2ddffSVineet Gupta * certain line since it sufficed as index in Non-Aliasing VIPT cache-geometry. 1838ea2ddffSVineet Gupta * Infact for distinct V1,V2,P: all of {V1-P},{V2-P},{P-P} would end up fetching 1848ea2ddffSVineet Gupta * the exact same line. 1858ea2ddffSVineet Gupta * 1868ea2ddffSVineet Gupta * However for larger Caches (way-size > page-size) - i.e. in Aliasing config, 1878ea2ddffSVineet Gupta * paddr alone could not be used to correctly index the cache. 1888ea2ddffSVineet Gupta * 1898ea2ddffSVineet Gupta * ------------------ 1908ea2ddffSVineet Gupta * MMU v1/v2 (Fixed Page Size 8k) 1918ea2ddffSVineet Gupta * ------------------ 1928ea2ddffSVineet Gupta * The solution was to provide CDU with these additonal vaddr bits. These 1938ea2ddffSVineet Gupta * would be bits [x:13], x would depend on cache-geometry, 13 comes from 1948ea2ddffSVineet Gupta * standard page size of 8k. 1958ea2ddffSVineet Gupta * H/w folks chose [17:13] to be a future safe range, and moreso these 5 bits 1968ea2ddffSVineet Gupta * of vaddr could easily be "stuffed" in the paddr as bits [4:0] since the 1978ea2ddffSVineet Gupta * orig 5 bits of paddr were anyways ignored by CDU line ops, as they 1988ea2ddffSVineet Gupta * represent the offset within cache-line. The adv of using this "clumsy" 1998ea2ddffSVineet Gupta * interface for additional info was no new reg was needed in CDU programming 2008ea2ddffSVineet Gupta * model. 2018ea2ddffSVineet Gupta * 2028ea2ddffSVineet Gupta * 17:13 represented the max num of bits passable, actual bits needed were 2038ea2ddffSVineet Gupta * fewer, based on the num-of-aliases possible. 2048ea2ddffSVineet Gupta * -for 2 alias possibility, only bit 13 needed (32K cache) 2058ea2ddffSVineet Gupta * -for 4 alias possibility, bits 14:13 needed (64K cache) 2068ea2ddffSVineet Gupta * 2078ea2ddffSVineet Gupta * ------------------ 2088ea2ddffSVineet Gupta * MMU v3 2098ea2ddffSVineet Gupta * ------------------ 2108ea2ddffSVineet Gupta * This ver of MMU supports variable page sizes (1k-16k): although Linux will 2118ea2ddffSVineet Gupta * only support 8k (default), 16k and 4k. 2128ea2ddffSVineet Gupta * However from hardware perspective, smaller page sizes aggrevate aliasing 2138ea2ddffSVineet Gupta * meaning more vaddr bits needed to disambiguate the cache-line-op ; 2148ea2ddffSVineet Gupta * the existing scheme of piggybacking won't work for certain configurations. 2158ea2ddffSVineet Gupta * Two new registers IC_PTAG and DC_PTAG inttoduced. 2168ea2ddffSVineet Gupta * "tag" bits are provided in PTAG, index bits in existing IVIL/IVDL/FLDL regs 2178362c389SVineet Gupta */ 2188ea2ddffSVineet Gupta 21911e14896SVineet Gupta static inline 22011e14896SVineet Gupta void __cache_line_loop_v2(unsigned long paddr, unsigned long vaddr, 2218ea2ddffSVineet Gupta unsigned long sz, const int op) 2228362c389SVineet Gupta { 22311e14896SVineet Gupta unsigned int aux_cmd; 2248362c389SVineet Gupta int num_lines; 22511e14896SVineet Gupta const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 2268362c389SVineet Gupta 2278ea2ddffSVineet Gupta if (op == OP_INV_IC) { 2288362c389SVineet Gupta aux_cmd = ARC_REG_IC_IVIL; 22911e14896SVineet Gupta } else { 2308362c389SVineet Gupta /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 2318ea2ddffSVineet Gupta aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 2328362c389SVineet Gupta } 2338362c389SVineet Gupta 2348362c389SVineet Gupta /* Ensure we properly floor/ceil the non-line aligned/sized requests 2358362c389SVineet Gupta * and have @paddr - aligned to cache line and integral @num_lines. 2368362c389SVineet Gupta * This however can be avoided for page sized since: 2378362c389SVineet Gupta * -@paddr will be cache-line aligned already (being page aligned) 2388362c389SVineet Gupta * -@sz will be integral multiple of line size (being page sized). 2398362c389SVineet Gupta */ 24011e14896SVineet Gupta if (!full_page) { 2418362c389SVineet Gupta sz += paddr & ~CACHE_LINE_MASK; 2428362c389SVineet Gupta paddr &= CACHE_LINE_MASK; 2438362c389SVineet Gupta vaddr &= CACHE_LINE_MASK; 2448362c389SVineet Gupta } 2458362c389SVineet Gupta 2468362c389SVineet Gupta num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 2478362c389SVineet Gupta 2488362c389SVineet Gupta /* MMUv2 and before: paddr contains stuffed vaddrs bits */ 2498362c389SVineet Gupta paddr |= (vaddr >> PAGE_SHIFT) & 0x1F; 2508362c389SVineet Gupta 2518362c389SVineet Gupta while (num_lines-- > 0) { 25211e14896SVineet Gupta write_aux_reg(aux_cmd, paddr); 25311e14896SVineet Gupta paddr += L1_CACHE_BYTES; 25411e14896SVineet Gupta } 25511e14896SVineet Gupta } 25611e14896SVineet Gupta 25711e14896SVineet Gupta static inline 25811e14896SVineet Gupta void __cache_line_loop_v3(unsigned long paddr, unsigned long vaddr, 25911e14896SVineet Gupta unsigned long sz, const int op) 26011e14896SVineet Gupta { 26111e14896SVineet Gupta unsigned int aux_cmd, aux_tag; 26211e14896SVineet Gupta int num_lines; 26311e14896SVineet Gupta const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 26411e14896SVineet Gupta 26511e14896SVineet Gupta if (op == OP_INV_IC) { 26611e14896SVineet Gupta aux_cmd = ARC_REG_IC_IVIL; 26711e14896SVineet Gupta aux_tag = ARC_REG_IC_PTAG; 26811e14896SVineet Gupta } else { 26911e14896SVineet Gupta aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 27011e14896SVineet Gupta aux_tag = ARC_REG_DC_PTAG; 27111e14896SVineet Gupta } 27211e14896SVineet Gupta 27311e14896SVineet Gupta /* Ensure we properly floor/ceil the non-line aligned/sized requests 27411e14896SVineet Gupta * and have @paddr - aligned to cache line and integral @num_lines. 27511e14896SVineet Gupta * This however can be avoided for page sized since: 27611e14896SVineet Gupta * -@paddr will be cache-line aligned already (being page aligned) 27711e14896SVineet Gupta * -@sz will be integral multiple of line size (being page sized). 27811e14896SVineet Gupta */ 27911e14896SVineet Gupta if (!full_page) { 28011e14896SVineet Gupta sz += paddr & ~CACHE_LINE_MASK; 28111e14896SVineet Gupta paddr &= CACHE_LINE_MASK; 28211e14896SVineet Gupta vaddr &= CACHE_LINE_MASK; 28311e14896SVineet Gupta } 28411e14896SVineet Gupta num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 28511e14896SVineet Gupta 28611e14896SVineet Gupta /* 28711e14896SVineet Gupta * MMUv3, cache ops require paddr in PTAG reg 28811e14896SVineet Gupta * if V-P const for loop, PTAG can be written once outside loop 28911e14896SVineet Gupta */ 29011e14896SVineet Gupta if (full_page) 29111e14896SVineet Gupta write_aux_reg(aux_tag, paddr); 29211e14896SVineet Gupta 29311e14896SVineet Gupta while (num_lines-- > 0) { 29411e14896SVineet Gupta if (!full_page) { 2958362c389SVineet Gupta write_aux_reg(aux_tag, paddr); 2968362c389SVineet Gupta paddr += L1_CACHE_BYTES; 2978362c389SVineet Gupta } 2988362c389SVineet Gupta 2998362c389SVineet Gupta write_aux_reg(aux_cmd, vaddr); 3008362c389SVineet Gupta vaddr += L1_CACHE_BYTES; 30111e14896SVineet Gupta } 30211e14896SVineet Gupta } 30311e14896SVineet Gupta 304d1f317d8SVineet Gupta /* 305d1f317d8SVineet Gupta * In HS38x (MMU v4), although icache is VIPT, only paddr is needed for cache 306d1f317d8SVineet Gupta * maintenance ops (in IVIL reg), as long as icache doesn't alias. 307d1f317d8SVineet Gupta * 308d1f317d8SVineet Gupta * For Aliasing icache, vaddr is also needed (in IVIL), while paddr is 309d1f317d8SVineet Gupta * specified in PTAG (similar to MMU v3) 310d1f317d8SVineet Gupta */ 311d1f317d8SVineet Gupta static inline 312d1f317d8SVineet Gupta void __cache_line_loop_v4(unsigned long paddr, unsigned long vaddr, 313d1f317d8SVineet Gupta unsigned long sz, const int cacheop) 314d1f317d8SVineet Gupta { 315d1f317d8SVineet Gupta unsigned int aux_cmd; 316d1f317d8SVineet Gupta int num_lines; 317d1f317d8SVineet Gupta const int full_page_op = __builtin_constant_p(sz) && sz == PAGE_SIZE; 318d1f317d8SVineet Gupta 319d1f317d8SVineet Gupta if (cacheop == OP_INV_IC) { 320d1f317d8SVineet Gupta aux_cmd = ARC_REG_IC_IVIL; 321d1f317d8SVineet Gupta } else { 322d1f317d8SVineet Gupta /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 323d1f317d8SVineet Gupta aux_cmd = cacheop & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 324d1f317d8SVineet Gupta } 325d1f317d8SVineet Gupta 326d1f317d8SVineet Gupta /* Ensure we properly floor/ceil the non-line aligned/sized requests 327d1f317d8SVineet Gupta * and have @paddr - aligned to cache line and integral @num_lines. 328d1f317d8SVineet Gupta * This however can be avoided for page sized since: 329d1f317d8SVineet Gupta * -@paddr will be cache-line aligned already (being page aligned) 330d1f317d8SVineet Gupta * -@sz will be integral multiple of line size (being page sized). 331d1f317d8SVineet Gupta */ 332d1f317d8SVineet Gupta if (!full_page_op) { 333d1f317d8SVineet Gupta sz += paddr & ~CACHE_LINE_MASK; 334d1f317d8SVineet Gupta paddr &= CACHE_LINE_MASK; 335d1f317d8SVineet Gupta } 336d1f317d8SVineet Gupta 337d1f317d8SVineet Gupta num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 338d1f317d8SVineet Gupta 339d1f317d8SVineet Gupta while (num_lines-- > 0) { 340d1f317d8SVineet Gupta write_aux_reg(aux_cmd, paddr); 341d1f317d8SVineet Gupta paddr += L1_CACHE_BYTES; 342d1f317d8SVineet Gupta } 343d1f317d8SVineet Gupta } 344d1f317d8SVineet Gupta 34511e14896SVineet Gupta #if (CONFIG_ARC_MMU_VER < 3) 34611e14896SVineet Gupta #define __cache_line_loop __cache_line_loop_v2 34711e14896SVineet Gupta #elif (CONFIG_ARC_MMU_VER == 3) 34811e14896SVineet Gupta #define __cache_line_loop __cache_line_loop_v3 349d1f317d8SVineet Gupta #elif (CONFIG_ARC_MMU_VER > 3) 350d1f317d8SVineet Gupta #define __cache_line_loop __cache_line_loop_v4 3518362c389SVineet Gupta #endif 3528362c389SVineet Gupta 3538362c389SVineet Gupta #ifdef CONFIG_ARC_HAS_DCACHE 3548362c389SVineet Gupta 3558362c389SVineet Gupta /*************************************************************** 3568362c389SVineet Gupta * Machine specific helpers for Entire D-Cache or Per Line ops 3578362c389SVineet Gupta */ 3588362c389SVineet Gupta 3596c310681SVineet Gupta static inline void __before_dc_op(const int op) 3608362c389SVineet Gupta { 3618362c389SVineet Gupta if (op == OP_FLUSH_N_INV) { 3628362c389SVineet Gupta /* Dcache provides 2 cmd: FLUSH or INV 3638362c389SVineet Gupta * INV inturn has sub-modes: DISCARD or FLUSH-BEFORE 3648362c389SVineet Gupta * flush-n-inv is achieved by INV cmd but with IM=1 3658362c389SVineet Gupta * So toggle INV sub-mode depending on op request and default 3668362c389SVineet Gupta */ 3676c310681SVineet Gupta const unsigned int ctl = ARC_REG_DC_CTRL; 3686c310681SVineet Gupta write_aux_reg(ctl, read_aux_reg(ctl) | DC_CTRL_INV_MODE_FLUSH); 3696c310681SVineet Gupta } 3708362c389SVineet Gupta } 3718362c389SVineet Gupta 3726c310681SVineet Gupta static inline void __after_dc_op(const int op) 3738362c389SVineet Gupta { 3746c310681SVineet Gupta if (op & OP_FLUSH) { 3756c310681SVineet Gupta const unsigned int ctl = ARC_REG_DC_CTRL; 3766c310681SVineet Gupta unsigned int reg; 3776c310681SVineet Gupta 3786c310681SVineet Gupta /* flush / flush-n-inv both wait */ 3796c310681SVineet Gupta while ((reg = read_aux_reg(ctl)) & DC_CTRL_FLUSH_STATUS) 3806c310681SVineet Gupta ; 3818362c389SVineet Gupta 3828362c389SVineet Gupta /* Switch back to default Invalidate mode */ 3838362c389SVineet Gupta if (op == OP_FLUSH_N_INV) 3846c310681SVineet Gupta write_aux_reg(ctl, reg & ~DC_CTRL_INV_MODE_FLUSH); 3856c310681SVineet Gupta } 3868362c389SVineet Gupta } 3878362c389SVineet Gupta 3888362c389SVineet Gupta /* 3898362c389SVineet Gupta * Operation on Entire D-Cache 3908ea2ddffSVineet Gupta * @op = {OP_INV, OP_FLUSH, OP_FLUSH_N_INV} 3918362c389SVineet Gupta * Note that constant propagation ensures all the checks are gone 3928362c389SVineet Gupta * in generated code 3938362c389SVineet Gupta */ 3948ea2ddffSVineet Gupta static inline void __dc_entire_op(const int op) 3958362c389SVineet Gupta { 3968362c389SVineet Gupta int aux; 3978362c389SVineet Gupta 3986c310681SVineet Gupta __before_dc_op(op); 3998362c389SVineet Gupta 4008ea2ddffSVineet Gupta if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 4018362c389SVineet Gupta aux = ARC_REG_DC_IVDC; 4028362c389SVineet Gupta else 4038362c389SVineet Gupta aux = ARC_REG_DC_FLSH; 4048362c389SVineet Gupta 4058362c389SVineet Gupta write_aux_reg(aux, 0x1); 4068362c389SVineet Gupta 4076c310681SVineet Gupta __after_dc_op(op); 4088362c389SVineet Gupta } 4098362c389SVineet Gupta 4108362c389SVineet Gupta /* For kernel mappings cache operation: index is same as paddr */ 4118362c389SVineet Gupta #define __dc_line_op_k(p, sz, op) __dc_line_op(p, p, sz, op) 4128362c389SVineet Gupta 4138362c389SVineet Gupta /* 4148ea2ddffSVineet Gupta * D-Cache Line ops: Per Line INV (discard or wback+discard) or FLUSH (wback) 4158362c389SVineet Gupta */ 4168362c389SVineet Gupta static inline void __dc_line_op(unsigned long paddr, unsigned long vaddr, 4178ea2ddffSVineet Gupta unsigned long sz, const int op) 4188362c389SVineet Gupta { 4198362c389SVineet Gupta unsigned long flags; 4208362c389SVineet Gupta 4218362c389SVineet Gupta local_irq_save(flags); 4228362c389SVineet Gupta 4236c310681SVineet Gupta __before_dc_op(op); 4248362c389SVineet Gupta 4258ea2ddffSVineet Gupta __cache_line_loop(paddr, vaddr, sz, op); 4268362c389SVineet Gupta 4276c310681SVineet Gupta __after_dc_op(op); 4288362c389SVineet Gupta 4298362c389SVineet Gupta local_irq_restore(flags); 4308362c389SVineet Gupta } 4318362c389SVineet Gupta 4328362c389SVineet Gupta #else 4338362c389SVineet Gupta 4348ea2ddffSVineet Gupta #define __dc_entire_op(op) 4358ea2ddffSVineet Gupta #define __dc_line_op(paddr, vaddr, sz, op) 4368ea2ddffSVineet Gupta #define __dc_line_op_k(paddr, sz, op) 4378362c389SVineet Gupta 4388362c389SVineet Gupta #endif /* CONFIG_ARC_HAS_DCACHE */ 4398362c389SVineet Gupta 4408362c389SVineet Gupta #ifdef CONFIG_ARC_HAS_ICACHE 4418362c389SVineet Gupta 4428362c389SVineet Gupta static inline void __ic_entire_inv(void) 4438362c389SVineet Gupta { 4448362c389SVineet Gupta write_aux_reg(ARC_REG_IC_IVIC, 1); 4458362c389SVineet Gupta read_aux_reg(ARC_REG_IC_CTRL); /* blocks */ 4468362c389SVineet Gupta } 4478362c389SVineet Gupta 4488362c389SVineet Gupta static inline void 4498362c389SVineet Gupta __ic_line_inv_vaddr_local(unsigned long paddr, unsigned long vaddr, 4508362c389SVineet Gupta unsigned long sz) 4518362c389SVineet Gupta { 4528362c389SVineet Gupta unsigned long flags; 4538362c389SVineet Gupta 4548362c389SVineet Gupta local_irq_save(flags); 455bcc4d65aSVineet Gupta (*_cache_line_loop_ic_fn)(paddr, vaddr, sz, OP_INV_IC); 4568362c389SVineet Gupta local_irq_restore(flags); 4578362c389SVineet Gupta } 4588362c389SVineet Gupta 4598362c389SVineet Gupta #ifndef CONFIG_SMP 4608362c389SVineet Gupta 4618362c389SVineet Gupta #define __ic_line_inv_vaddr(p, v, s) __ic_line_inv_vaddr_local(p, v, s) 4628362c389SVineet Gupta 4638362c389SVineet Gupta #else 4648362c389SVineet Gupta 4658362c389SVineet Gupta struct ic_inv_args { 4668362c389SVineet Gupta unsigned long paddr, vaddr; 4678362c389SVineet Gupta int sz; 4688362c389SVineet Gupta }; 4698362c389SVineet Gupta 4708362c389SVineet Gupta static void __ic_line_inv_vaddr_helper(void *info) 4718362c389SVineet Gupta { 4728362c389SVineet Gupta struct ic_inv_args *ic_inv = info; 4738362c389SVineet Gupta 4748362c389SVineet Gupta __ic_line_inv_vaddr_local(ic_inv->paddr, ic_inv->vaddr, ic_inv->sz); 4758362c389SVineet Gupta } 4768362c389SVineet Gupta 4778362c389SVineet Gupta static void __ic_line_inv_vaddr(unsigned long paddr, unsigned long vaddr, 4788362c389SVineet Gupta unsigned long sz) 4798362c389SVineet Gupta { 4808362c389SVineet Gupta struct ic_inv_args ic_inv = { 4818362c389SVineet Gupta .paddr = paddr, 4828362c389SVineet Gupta .vaddr = vaddr, 4838362c389SVineet Gupta .sz = sz 4848362c389SVineet Gupta }; 4858362c389SVineet Gupta 4868362c389SVineet Gupta on_each_cpu(__ic_line_inv_vaddr_helper, &ic_inv, 1); 4878362c389SVineet Gupta } 4888362c389SVineet Gupta 4898362c389SVineet Gupta #endif /* CONFIG_SMP */ 4908362c389SVineet Gupta 4918362c389SVineet Gupta #else /* !CONFIG_ARC_HAS_ICACHE */ 4928362c389SVineet Gupta 4938362c389SVineet Gupta #define __ic_entire_inv() 4948362c389SVineet Gupta #define __ic_line_inv_vaddr(pstart, vstart, sz) 4958362c389SVineet Gupta 4968362c389SVineet Gupta #endif /* CONFIG_ARC_HAS_ICACHE */ 4978362c389SVineet Gupta 498795f4558SVineet Gupta noinline void slc_op(unsigned long paddr, unsigned long sz, const int op) 499795f4558SVineet Gupta { 500795f4558SVineet Gupta #ifdef CONFIG_ISA_ARCV2 501b607edddSAlexey Brodkin /* 502b607edddSAlexey Brodkin * SLC is shared between all cores and concurrent aux operations from 503b607edddSAlexey Brodkin * multiple cores need to be serialized using a spinlock 504b607edddSAlexey Brodkin * A concurrent operation can be silently ignored and/or the old/new 505b607edddSAlexey Brodkin * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop 506b607edddSAlexey Brodkin * below) 507b607edddSAlexey Brodkin */ 508b607edddSAlexey Brodkin static DEFINE_SPINLOCK(lock); 509795f4558SVineet Gupta unsigned long flags; 510795f4558SVineet Gupta unsigned int ctrl; 511795f4558SVineet Gupta 512b607edddSAlexey Brodkin spin_lock_irqsave(&lock, flags); 513795f4558SVineet Gupta 514795f4558SVineet Gupta /* 515795f4558SVineet Gupta * The Region Flush operation is specified by CTRL.RGN_OP[11..9] 516795f4558SVineet Gupta * - b'000 (default) is Flush, 517795f4558SVineet Gupta * - b'001 is Invalidate if CTRL.IM == 0 518795f4558SVineet Gupta * - b'001 is Flush-n-Invalidate if CTRL.IM == 1 519795f4558SVineet Gupta */ 520795f4558SVineet Gupta ctrl = read_aux_reg(ARC_REG_SLC_CTRL); 521795f4558SVineet Gupta 522795f4558SVineet Gupta /* Don't rely on default value of IM bit */ 523795f4558SVineet Gupta if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 524795f4558SVineet Gupta ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 525795f4558SVineet Gupta else 526795f4558SVineet Gupta ctrl |= SLC_CTRL_IM; 527795f4558SVineet Gupta 528795f4558SVineet Gupta if (op & OP_INV) 529795f4558SVineet Gupta ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */ 530795f4558SVineet Gupta else 531795f4558SVineet Gupta ctrl &= ~SLC_CTRL_RGN_OP_INV; 532795f4558SVineet Gupta 533795f4558SVineet Gupta write_aux_reg(ARC_REG_SLC_CTRL, ctrl); 534795f4558SVineet Gupta 535795f4558SVineet Gupta /* 536795f4558SVineet Gupta * Lower bits are ignored, no need to clip 537795f4558SVineet Gupta * END needs to be setup before START (latter triggers the operation) 538795f4558SVineet Gupta * END can't be same as START, so add (l2_line_sz - 1) to sz 539795f4558SVineet Gupta */ 540795f4558SVineet Gupta write_aux_reg(ARC_REG_SLC_RGN_END, (paddr + sz + l2_line_sz - 1)); 541795f4558SVineet Gupta write_aux_reg(ARC_REG_SLC_RGN_START, paddr); 542795f4558SVineet Gupta 543795f4558SVineet Gupta while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY); 544795f4558SVineet Gupta 545b607edddSAlexey Brodkin spin_unlock_irqrestore(&lock, flags); 546795f4558SVineet Gupta #endif 547795f4558SVineet Gupta } 548795f4558SVineet Gupta 5498362c389SVineet Gupta /*********************************************************** 5508362c389SVineet Gupta * Exported APIs 5518362c389SVineet Gupta */ 5528362c389SVineet Gupta 5538362c389SVineet Gupta /* 5548362c389SVineet Gupta * Handle cache congruency of kernel and userspace mappings of page when kernel 5558362c389SVineet Gupta * writes-to/reads-from 5568362c389SVineet Gupta * 5578362c389SVineet Gupta * The idea is to defer flushing of kernel mapping after a WRITE, possible if: 5588362c389SVineet Gupta * -dcache is NOT aliasing, hence any U/K-mappings of page are congruent 5598362c389SVineet Gupta * -U-mapping doesn't exist yet for page (finalised in update_mmu_cache) 5608362c389SVineet Gupta * -In SMP, if hardware caches are coherent 5618362c389SVineet Gupta * 5628362c389SVineet Gupta * There's a corollary case, where kernel READs from a userspace mapped page. 5638362c389SVineet Gupta * If the U-mapping is not congruent to to K-mapping, former needs flushing. 5648362c389SVineet Gupta */ 5658362c389SVineet Gupta void flush_dcache_page(struct page *page) 5668362c389SVineet Gupta { 5678362c389SVineet Gupta struct address_space *mapping; 5688362c389SVineet Gupta 5698362c389SVineet Gupta if (!cache_is_vipt_aliasing()) { 5708362c389SVineet Gupta clear_bit(PG_dc_clean, &page->flags); 5718362c389SVineet Gupta return; 5728362c389SVineet Gupta } 5738362c389SVineet Gupta 5748362c389SVineet Gupta /* don't handle anon pages here */ 5758362c389SVineet Gupta mapping = page_mapping(page); 5768362c389SVineet Gupta if (!mapping) 5778362c389SVineet Gupta return; 5788362c389SVineet Gupta 5798362c389SVineet Gupta /* 5808362c389SVineet Gupta * pagecache page, file not yet mapped to userspace 5818362c389SVineet Gupta * Make a note that K-mapping is dirty 5828362c389SVineet Gupta */ 5838362c389SVineet Gupta if (!mapping_mapped(mapping)) { 5848362c389SVineet Gupta clear_bit(PG_dc_clean, &page->flags); 5858362c389SVineet Gupta } else if (page_mapped(page)) { 5868362c389SVineet Gupta 5878362c389SVineet Gupta /* kernel reading from page with U-mapping */ 5888362c389SVineet Gupta unsigned long paddr = (unsigned long)page_address(page); 5898362c389SVineet Gupta unsigned long vaddr = page->index << PAGE_CACHE_SHIFT; 5908362c389SVineet Gupta 5918362c389SVineet Gupta if (addr_not_cache_congruent(paddr, vaddr)) 5928362c389SVineet Gupta __flush_dcache_page(paddr, vaddr); 5938362c389SVineet Gupta } 5948362c389SVineet Gupta } 5958362c389SVineet Gupta EXPORT_SYMBOL(flush_dcache_page); 5968362c389SVineet Gupta 597f2b0b25aSAlexey Brodkin /* 598f2b0b25aSAlexey Brodkin * DMA ops for systems with L1 cache only 599f2b0b25aSAlexey Brodkin * Make memory coherent with L1 cache by flushing/invalidating L1 lines 600f2b0b25aSAlexey Brodkin */ 601f2b0b25aSAlexey Brodkin static void __dma_cache_wback_inv_l1(unsigned long start, unsigned long sz) 6028362c389SVineet Gupta { 6038362c389SVineet Gupta __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 604f2b0b25aSAlexey Brodkin } 605795f4558SVineet Gupta 606f2b0b25aSAlexey Brodkin static void __dma_cache_inv_l1(unsigned long start, unsigned long sz) 607f2b0b25aSAlexey Brodkin { 608f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_INV); 609f2b0b25aSAlexey Brodkin } 610f2b0b25aSAlexey Brodkin 611f2b0b25aSAlexey Brodkin static void __dma_cache_wback_l1(unsigned long start, unsigned long sz) 612f2b0b25aSAlexey Brodkin { 613f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_FLUSH); 614f2b0b25aSAlexey Brodkin } 615f2b0b25aSAlexey Brodkin 616f2b0b25aSAlexey Brodkin /* 617f2b0b25aSAlexey Brodkin * DMA ops for systems with both L1 and L2 caches, but without IOC 618f2b0b25aSAlexey Brodkin * Both L1 and L2 lines need to be explicity flushed/invalidated 619f2b0b25aSAlexey Brodkin */ 620f2b0b25aSAlexey Brodkin static void __dma_cache_wback_inv_slc(unsigned long start, unsigned long sz) 621f2b0b25aSAlexey Brodkin { 622f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 623795f4558SVineet Gupta slc_op(start, sz, OP_FLUSH_N_INV); 6248362c389SVineet Gupta } 625f2b0b25aSAlexey Brodkin 626f2b0b25aSAlexey Brodkin static void __dma_cache_inv_slc(unsigned long start, unsigned long sz) 627f2b0b25aSAlexey Brodkin { 628f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_INV); 629f2b0b25aSAlexey Brodkin slc_op(start, sz, OP_INV); 630f2b0b25aSAlexey Brodkin } 631f2b0b25aSAlexey Brodkin 632f2b0b25aSAlexey Brodkin static void __dma_cache_wback_slc(unsigned long start, unsigned long sz) 633f2b0b25aSAlexey Brodkin { 634f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_FLUSH); 635f2b0b25aSAlexey Brodkin slc_op(start, sz, OP_FLUSH); 636f2b0b25aSAlexey Brodkin } 637f2b0b25aSAlexey Brodkin 638f2b0b25aSAlexey Brodkin /* 639f2b0b25aSAlexey Brodkin * DMA ops for systems with IOC 640f2b0b25aSAlexey Brodkin * IOC hardware snoops all DMA traffic keeping the caches consistent with 641f2b0b25aSAlexey Brodkin * memory - eliding need for any explicit cache maintenance of DMA buffers 642f2b0b25aSAlexey Brodkin */ 643f2b0b25aSAlexey Brodkin static void __dma_cache_wback_inv_ioc(unsigned long start, unsigned long sz) {} 644f2b0b25aSAlexey Brodkin static void __dma_cache_inv_ioc(unsigned long start, unsigned long sz) {} 645f2b0b25aSAlexey Brodkin static void __dma_cache_wback_ioc(unsigned long start, unsigned long sz) {} 646f2b0b25aSAlexey Brodkin 647f2b0b25aSAlexey Brodkin /* 648f2b0b25aSAlexey Brodkin * Exported DMA API 649f2b0b25aSAlexey Brodkin */ 650f2b0b25aSAlexey Brodkin void dma_cache_wback_inv(unsigned long start, unsigned long sz) 651f2b0b25aSAlexey Brodkin { 652f2b0b25aSAlexey Brodkin __dma_cache_wback_inv(start, sz); 653f2b0b25aSAlexey Brodkin } 6548362c389SVineet Gupta EXPORT_SYMBOL(dma_cache_wback_inv); 6558362c389SVineet Gupta 6568362c389SVineet Gupta void dma_cache_inv(unsigned long start, unsigned long sz) 6578362c389SVineet Gupta { 658f2b0b25aSAlexey Brodkin __dma_cache_inv(start, sz); 6598362c389SVineet Gupta } 6608362c389SVineet Gupta EXPORT_SYMBOL(dma_cache_inv); 6618362c389SVineet Gupta 6628362c389SVineet Gupta void dma_cache_wback(unsigned long start, unsigned long sz) 6638362c389SVineet Gupta { 664f2b0b25aSAlexey Brodkin __dma_cache_wback(start, sz); 6658362c389SVineet Gupta } 6668362c389SVineet Gupta EXPORT_SYMBOL(dma_cache_wback); 6678362c389SVineet Gupta 6688362c389SVineet Gupta /* 6698362c389SVineet Gupta * This is API for making I/D Caches consistent when modifying 6708362c389SVineet Gupta * kernel code (loadable modules, kprobes, kgdb...) 6718362c389SVineet Gupta * This is called on insmod, with kernel virtual address for CODE of 6728362c389SVineet Gupta * the module. ARC cache maintenance ops require PHY address thus we 6738362c389SVineet Gupta * need to convert vmalloc addr to PHY addr 6748362c389SVineet Gupta */ 6758362c389SVineet Gupta void flush_icache_range(unsigned long kstart, unsigned long kend) 6768362c389SVineet Gupta { 6778362c389SVineet Gupta unsigned int tot_sz; 6788362c389SVineet Gupta 6798362c389SVineet Gupta WARN(kstart < TASK_SIZE, "%s() can't handle user vaddr", __func__); 6808362c389SVineet Gupta 6818362c389SVineet Gupta /* Shortcut for bigger flush ranges. 6828362c389SVineet Gupta * Here we don't care if this was kernel virtual or phy addr 6838362c389SVineet Gupta */ 6848362c389SVineet Gupta tot_sz = kend - kstart; 6858362c389SVineet Gupta if (tot_sz > PAGE_SIZE) { 6868362c389SVineet Gupta flush_cache_all(); 6878362c389SVineet Gupta return; 6888362c389SVineet Gupta } 6898362c389SVineet Gupta 6908362c389SVineet Gupta /* Case: Kernel Phy addr (0x8000_0000 onwards) */ 6918362c389SVineet Gupta if (likely(kstart > PAGE_OFFSET)) { 6928362c389SVineet Gupta /* 6938362c389SVineet Gupta * The 2nd arg despite being paddr will be used to index icache 6948362c389SVineet Gupta * This is OK since no alternate virtual mappings will exist 6958362c389SVineet Gupta * given the callers for this case: kprobe/kgdb in built-in 6968362c389SVineet Gupta * kernel code only. 6978362c389SVineet Gupta */ 6988362c389SVineet Gupta __sync_icache_dcache(kstart, kstart, kend - kstart); 6998362c389SVineet Gupta return; 7008362c389SVineet Gupta } 7018362c389SVineet Gupta 7028362c389SVineet Gupta /* 7038362c389SVineet Gupta * Case: Kernel Vaddr (0x7000_0000 to 0x7fff_ffff) 7048362c389SVineet Gupta * (1) ARC Cache Maintenance ops only take Phy addr, hence special 7058362c389SVineet Gupta * handling of kernel vaddr. 7068362c389SVineet Gupta * 7078362c389SVineet Gupta * (2) Despite @tot_sz being < PAGE_SIZE (bigger cases handled already), 7088362c389SVineet Gupta * it still needs to handle a 2 page scenario, where the range 7098362c389SVineet Gupta * straddles across 2 virtual pages and hence need for loop 7108362c389SVineet Gupta */ 7118362c389SVineet Gupta while (tot_sz > 0) { 7128362c389SVineet Gupta unsigned int off, sz; 7138362c389SVineet Gupta unsigned long phy, pfn; 7148362c389SVineet Gupta 7158362c389SVineet Gupta off = kstart % PAGE_SIZE; 7168362c389SVineet Gupta pfn = vmalloc_to_pfn((void *)kstart); 7178362c389SVineet Gupta phy = (pfn << PAGE_SHIFT) + off; 7188362c389SVineet Gupta sz = min_t(unsigned int, tot_sz, PAGE_SIZE - off); 7198362c389SVineet Gupta __sync_icache_dcache(phy, kstart, sz); 7208362c389SVineet Gupta kstart += sz; 7218362c389SVineet Gupta tot_sz -= sz; 7228362c389SVineet Gupta } 7238362c389SVineet Gupta } 7248362c389SVineet Gupta EXPORT_SYMBOL(flush_icache_range); 7258362c389SVineet Gupta 7268362c389SVineet Gupta /* 7278362c389SVineet Gupta * General purpose helper to make I and D cache lines consistent. 7288362c389SVineet Gupta * @paddr is phy addr of region 7298362c389SVineet Gupta * @vaddr is typically user vaddr (breakpoint) or kernel vaddr (vmalloc) 7308362c389SVineet Gupta * However in one instance, when called by kprobe (for a breakpt in 7318362c389SVineet Gupta * builtin kernel code) @vaddr will be paddr only, meaning CDU operation will 7328362c389SVineet Gupta * use a paddr to index the cache (despite VIPT). This is fine since since a 7338362c389SVineet Gupta * builtin kernel page will not have any virtual mappings. 7348362c389SVineet Gupta * kprobe on loadable module will be kernel vaddr. 7358362c389SVineet Gupta */ 7368362c389SVineet Gupta void __sync_icache_dcache(unsigned long paddr, unsigned long vaddr, int len) 7378362c389SVineet Gupta { 7388362c389SVineet Gupta __dc_line_op(paddr, vaddr, len, OP_FLUSH_N_INV); 7398362c389SVineet Gupta __ic_line_inv_vaddr(paddr, vaddr, len); 7408362c389SVineet Gupta } 7418362c389SVineet Gupta 7428362c389SVineet Gupta /* wrapper to compile time eliminate alignment checks in flush loop */ 7438362c389SVineet Gupta void __inv_icache_page(unsigned long paddr, unsigned long vaddr) 7448362c389SVineet Gupta { 7458362c389SVineet Gupta __ic_line_inv_vaddr(paddr, vaddr, PAGE_SIZE); 7468362c389SVineet Gupta } 7478362c389SVineet Gupta 7488362c389SVineet Gupta /* 7498362c389SVineet Gupta * wrapper to clearout kernel or userspace mappings of a page 7508362c389SVineet Gupta * For kernel mappings @vaddr == @paddr 7518362c389SVineet Gupta */ 7528362c389SVineet Gupta void __flush_dcache_page(unsigned long paddr, unsigned long vaddr) 7538362c389SVineet Gupta { 7548362c389SVineet Gupta __dc_line_op(paddr, vaddr & PAGE_MASK, PAGE_SIZE, OP_FLUSH_N_INV); 7558362c389SVineet Gupta } 7568362c389SVineet Gupta 7578362c389SVineet Gupta noinline void flush_cache_all(void) 7588362c389SVineet Gupta { 7598362c389SVineet Gupta unsigned long flags; 7608362c389SVineet Gupta 7618362c389SVineet Gupta local_irq_save(flags); 7628362c389SVineet Gupta 7638362c389SVineet Gupta __ic_entire_inv(); 7648362c389SVineet Gupta __dc_entire_op(OP_FLUSH_N_INV); 7658362c389SVineet Gupta 7668362c389SVineet Gupta local_irq_restore(flags); 7678362c389SVineet Gupta 7688362c389SVineet Gupta } 7698362c389SVineet Gupta 7708362c389SVineet Gupta #ifdef CONFIG_ARC_CACHE_VIPT_ALIASING 7718362c389SVineet Gupta 7728362c389SVineet Gupta void flush_cache_mm(struct mm_struct *mm) 7738362c389SVineet Gupta { 7748362c389SVineet Gupta flush_cache_all(); 7758362c389SVineet Gupta } 7768362c389SVineet Gupta 7778362c389SVineet Gupta void flush_cache_page(struct vm_area_struct *vma, unsigned long u_vaddr, 7788362c389SVineet Gupta unsigned long pfn) 7798362c389SVineet Gupta { 7808362c389SVineet Gupta unsigned int paddr = pfn << PAGE_SHIFT; 7818362c389SVineet Gupta 7828362c389SVineet Gupta u_vaddr &= PAGE_MASK; 7838362c389SVineet Gupta 7848362c389SVineet Gupta __flush_dcache_page(paddr, u_vaddr); 7858362c389SVineet Gupta 7868362c389SVineet Gupta if (vma->vm_flags & VM_EXEC) 7878362c389SVineet Gupta __inv_icache_page(paddr, u_vaddr); 7888362c389SVineet Gupta } 7898362c389SVineet Gupta 7908362c389SVineet Gupta void flush_cache_range(struct vm_area_struct *vma, unsigned long start, 7918362c389SVineet Gupta unsigned long end) 7928362c389SVineet Gupta { 7938362c389SVineet Gupta flush_cache_all(); 7948362c389SVineet Gupta } 7958362c389SVineet Gupta 7968362c389SVineet Gupta void flush_anon_page(struct vm_area_struct *vma, struct page *page, 7978362c389SVineet Gupta unsigned long u_vaddr) 7988362c389SVineet Gupta { 7998362c389SVineet Gupta /* TBD: do we really need to clear the kernel mapping */ 8008362c389SVineet Gupta __flush_dcache_page(page_address(page), u_vaddr); 8018362c389SVineet Gupta __flush_dcache_page(page_address(page), page_address(page)); 8028362c389SVineet Gupta 8038362c389SVineet Gupta } 8048362c389SVineet Gupta 8058362c389SVineet Gupta #endif 8068362c389SVineet Gupta 8078362c389SVineet Gupta void copy_user_highpage(struct page *to, struct page *from, 8088362c389SVineet Gupta unsigned long u_vaddr, struct vm_area_struct *vma) 8098362c389SVineet Gupta { 8108362c389SVineet Gupta unsigned long kfrom = (unsigned long)page_address(from); 8118362c389SVineet Gupta unsigned long kto = (unsigned long)page_address(to); 8128362c389SVineet Gupta int clean_src_k_mappings = 0; 8138362c389SVineet Gupta 8148362c389SVineet Gupta /* 8158362c389SVineet Gupta * If SRC page was already mapped in userspace AND it's U-mapping is 8168362c389SVineet Gupta * not congruent with K-mapping, sync former to physical page so that 8178362c389SVineet Gupta * K-mapping in memcpy below, sees the right data 8188362c389SVineet Gupta * 8198362c389SVineet Gupta * Note that while @u_vaddr refers to DST page's userspace vaddr, it is 8208362c389SVineet Gupta * equally valid for SRC page as well 8218362c389SVineet Gupta */ 8228362c389SVineet Gupta if (page_mapped(from) && addr_not_cache_congruent(kfrom, u_vaddr)) { 8238362c389SVineet Gupta __flush_dcache_page(kfrom, u_vaddr); 8248362c389SVineet Gupta clean_src_k_mappings = 1; 8258362c389SVineet Gupta } 8268362c389SVineet Gupta 8278362c389SVineet Gupta copy_page((void *)kto, (void *)kfrom); 8288362c389SVineet Gupta 8298362c389SVineet Gupta /* 8308362c389SVineet Gupta * Mark DST page K-mapping as dirty for a later finalization by 8318362c389SVineet Gupta * update_mmu_cache(). Although the finalization could have been done 8328362c389SVineet Gupta * here as well (given that both vaddr/paddr are available). 8338362c389SVineet Gupta * But update_mmu_cache() already has code to do that for other 8348362c389SVineet Gupta * non copied user pages (e.g. read faults which wire in pagecache page 8358362c389SVineet Gupta * directly). 8368362c389SVineet Gupta */ 8378362c389SVineet Gupta clear_bit(PG_dc_clean, &to->flags); 8388362c389SVineet Gupta 8398362c389SVineet Gupta /* 8408362c389SVineet Gupta * if SRC was already usermapped and non-congruent to kernel mapping 8418362c389SVineet Gupta * sync the kernel mapping back to physical page 8428362c389SVineet Gupta */ 8438362c389SVineet Gupta if (clean_src_k_mappings) { 8448362c389SVineet Gupta __flush_dcache_page(kfrom, kfrom); 8458362c389SVineet Gupta set_bit(PG_dc_clean, &from->flags); 8468362c389SVineet Gupta } else { 8478362c389SVineet Gupta clear_bit(PG_dc_clean, &from->flags); 8488362c389SVineet Gupta } 8498362c389SVineet Gupta } 8508362c389SVineet Gupta 8518362c389SVineet Gupta void clear_user_page(void *to, unsigned long u_vaddr, struct page *page) 8528362c389SVineet Gupta { 8538362c389SVineet Gupta clear_page(to); 8548362c389SVineet Gupta clear_bit(PG_dc_clean, &page->flags); 8558362c389SVineet Gupta } 8568362c389SVineet Gupta 8578362c389SVineet Gupta 8588362c389SVineet Gupta /********************************************************************** 8598362c389SVineet Gupta * Explicit Cache flush request from user space via syscall 8608362c389SVineet Gupta * Needed for JITs which generate code on the fly 8618362c389SVineet Gupta */ 8628362c389SVineet Gupta SYSCALL_DEFINE3(cacheflush, uint32_t, start, uint32_t, sz, uint32_t, flags) 8638362c389SVineet Gupta { 8648362c389SVineet Gupta /* TBD: optimize this */ 8658362c389SVineet Gupta flush_cache_all(); 8668362c389SVineet Gupta return 0; 8678362c389SVineet Gupta } 8688ea2ddffSVineet Gupta 8698ea2ddffSVineet Gupta void arc_cache_init(void) 8708ea2ddffSVineet Gupta { 8718ea2ddffSVineet Gupta unsigned int __maybe_unused cpu = smp_processor_id(); 8728ea2ddffSVineet Gupta char str[256]; 8738ea2ddffSVineet Gupta 8748ea2ddffSVineet Gupta printk(arc_cache_mumbojumbo(0, str, sizeof(str))); 8758ea2ddffSVineet Gupta 8768ea2ddffSVineet Gupta if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) { 8778ea2ddffSVineet Gupta struct cpuinfo_arc_cache *ic = &cpuinfo_arc700[cpu].icache; 8788ea2ddffSVineet Gupta 8798ea2ddffSVineet Gupta if (!ic->ver) 8808ea2ddffSVineet Gupta panic("cache support enabled but non-existent cache\n"); 8818ea2ddffSVineet Gupta 8828ea2ddffSVineet Gupta if (ic->line_len != L1_CACHE_BYTES) 8838ea2ddffSVineet Gupta panic("ICache line [%d] != kernel Config [%d]", 8848ea2ddffSVineet Gupta ic->line_len, L1_CACHE_BYTES); 8858ea2ddffSVineet Gupta 8868ea2ddffSVineet Gupta if (ic->ver != CONFIG_ARC_MMU_VER) 8878ea2ddffSVineet Gupta panic("Cache ver [%d] doesn't match MMU ver [%d]\n", 8888ea2ddffSVineet Gupta ic->ver, CONFIG_ARC_MMU_VER); 889bcc4d65aSVineet Gupta 890bcc4d65aSVineet Gupta /* 891bcc4d65aSVineet Gupta * In MMU v4 (HS38x) the alising icache config uses IVIL/PTAG 892bcc4d65aSVineet Gupta * pair to provide vaddr/paddr respectively, just as in MMU v3 893bcc4d65aSVineet Gupta */ 894bcc4d65aSVineet Gupta if (is_isa_arcv2() && ic->alias) 895bcc4d65aSVineet Gupta _cache_line_loop_ic_fn = __cache_line_loop_v3; 896bcc4d65aSVineet Gupta else 897bcc4d65aSVineet Gupta _cache_line_loop_ic_fn = __cache_line_loop; 8988ea2ddffSVineet Gupta } 8998ea2ddffSVineet Gupta 9008ea2ddffSVineet Gupta if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) { 9018ea2ddffSVineet Gupta struct cpuinfo_arc_cache *dc = &cpuinfo_arc700[cpu].dcache; 9028ea2ddffSVineet Gupta 9038ea2ddffSVineet Gupta if (!dc->ver) 9048ea2ddffSVineet Gupta panic("cache support enabled but non-existent cache\n"); 9058ea2ddffSVineet Gupta 9068ea2ddffSVineet Gupta if (dc->line_len != L1_CACHE_BYTES) 9078ea2ddffSVineet Gupta panic("DCache line [%d] != kernel Config [%d]", 9088ea2ddffSVineet Gupta dc->line_len, L1_CACHE_BYTES); 9098ea2ddffSVineet Gupta 910d1f317d8SVineet Gupta /* check for D-Cache aliasing on ARCompact: ARCv2 has PIPT */ 911d1f317d8SVineet Gupta if (is_isa_arcompact()) { 912d1f317d8SVineet Gupta int handled = IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING); 9138ea2ddffSVineet Gupta 9148ea2ddffSVineet Gupta if (dc->alias && !handled) 9158ea2ddffSVineet Gupta panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 9168ea2ddffSVineet Gupta else if (!dc->alias && handled) 9178ea2ddffSVineet Gupta panic("Disable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 9188ea2ddffSVineet Gupta } 9198ea2ddffSVineet Gupta } 920f2b0b25aSAlexey Brodkin 92179335a2cSVineet Gupta if (is_isa_arcv2() && l2_line_sz && !slc_enable) { 92279335a2cSVineet Gupta 92379335a2cSVineet Gupta /* IM set : flush before invalidate */ 92479335a2cSVineet Gupta write_aux_reg(ARC_REG_SLC_CTRL, 92579335a2cSVineet Gupta read_aux_reg(ARC_REG_SLC_CTRL) | SLC_CTRL_IM); 92679335a2cSVineet Gupta 92779335a2cSVineet Gupta write_aux_reg(ARC_REG_SLC_INVALIDATE, 1); 92879335a2cSVineet Gupta 92979335a2cSVineet Gupta /* Important to wait for flush to complete */ 93079335a2cSVineet Gupta while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY); 93179335a2cSVineet Gupta write_aux_reg(ARC_REG_SLC_CTRL, 93279335a2cSVineet Gupta read_aux_reg(ARC_REG_SLC_CTRL) | SLC_CTRL_DISABLE); 93379335a2cSVineet Gupta } 93479335a2cSVineet Gupta 935f2b0b25aSAlexey Brodkin if (is_isa_arcv2() && ioc_exists) { 936f2b0b25aSAlexey Brodkin /* IO coherency base - 0x8z */ 937f2b0b25aSAlexey Brodkin write_aux_reg(ARC_REG_IO_COH_AP0_BASE, 0x80000); 938f2b0b25aSAlexey Brodkin /* IO coherency aperture size - 512Mb: 0x8z-0xAz */ 939f2b0b25aSAlexey Brodkin write_aux_reg(ARC_REG_IO_COH_AP0_SIZE, 0x11); 940f2b0b25aSAlexey Brodkin /* Enable partial writes */ 941f2b0b25aSAlexey Brodkin write_aux_reg(ARC_REG_IO_COH_PARTIAL, 1); 942f2b0b25aSAlexey Brodkin /* Enable IO coherency */ 943f2b0b25aSAlexey Brodkin write_aux_reg(ARC_REG_IO_COH_ENABLE, 1); 944f2b0b25aSAlexey Brodkin 945f2b0b25aSAlexey Brodkin __dma_cache_wback_inv = __dma_cache_wback_inv_ioc; 946f2b0b25aSAlexey Brodkin __dma_cache_inv = __dma_cache_inv_ioc; 947f2b0b25aSAlexey Brodkin __dma_cache_wback = __dma_cache_wback_ioc; 94879335a2cSVineet Gupta } else if (is_isa_arcv2() && l2_line_sz && slc_enable) { 949f2b0b25aSAlexey Brodkin __dma_cache_wback_inv = __dma_cache_wback_inv_slc; 950f2b0b25aSAlexey Brodkin __dma_cache_inv = __dma_cache_inv_slc; 951f2b0b25aSAlexey Brodkin __dma_cache_wback = __dma_cache_wback_slc; 952f2b0b25aSAlexey Brodkin } else { 953f2b0b25aSAlexey Brodkin __dma_cache_wback_inv = __dma_cache_wback_inv_l1; 954f2b0b25aSAlexey Brodkin __dma_cache_inv = __dma_cache_inv_l1; 955f2b0b25aSAlexey Brodkin __dma_cache_wback = __dma_cache_wback_l1; 956f2b0b25aSAlexey Brodkin } 957d1f317d8SVineet Gupta } 958