18362c389SVineet Gupta /* 28ea2ddffSVineet Gupta * ARC Cache Management 38362c389SVineet Gupta * 48ea2ddffSVineet Gupta * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com) 58362c389SVineet Gupta * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 68362c389SVineet Gupta * 78362c389SVineet Gupta * This program is free software; you can redistribute it and/or modify 88362c389SVineet Gupta * it under the terms of the GNU General Public License version 2 as 98362c389SVineet Gupta * published by the Free Software Foundation. 108362c389SVineet Gupta */ 118362c389SVineet Gupta 128362c389SVineet Gupta #include <linux/module.h> 138362c389SVineet Gupta #include <linux/mm.h> 148362c389SVineet Gupta #include <linux/sched.h> 158362c389SVineet Gupta #include <linux/cache.h> 168362c389SVineet Gupta #include <linux/mmu_context.h> 178362c389SVineet Gupta #include <linux/syscalls.h> 188362c389SVineet Gupta #include <linux/uaccess.h> 198362c389SVineet Gupta #include <linux/pagemap.h> 208362c389SVineet Gupta #include <asm/cacheflush.h> 218362c389SVineet Gupta #include <asm/cachectl.h> 228362c389SVineet Gupta #include <asm/setup.h> 238362c389SVineet Gupta 24795f4558SVineet Gupta static int l2_line_sz; 25f2b0b25aSAlexey Brodkin int ioc_exists; 261648c70dSAlexey Brodkin volatile int slc_enable = 1, ioc_enable = 1; 27795f4558SVineet Gupta 2828b4af72SVineet Gupta void (*_cache_line_loop_ic_fn)(phys_addr_t paddr, unsigned long vaddr, 29bcc4d65aSVineet Gupta unsigned long sz, const int cacheop); 30bcc4d65aSVineet Gupta 31f2b0b25aSAlexey Brodkin void (*__dma_cache_wback_inv)(unsigned long start, unsigned long sz); 32f2b0b25aSAlexey Brodkin void (*__dma_cache_inv)(unsigned long start, unsigned long sz); 33f2b0b25aSAlexey Brodkin void (*__dma_cache_wback)(unsigned long start, unsigned long sz); 34f2b0b25aSAlexey Brodkin 358362c389SVineet Gupta char *arc_cache_mumbojumbo(int c, char *buf, int len) 368362c389SVineet Gupta { 378362c389SVineet Gupta int n = 0; 38d1f317d8SVineet Gupta struct cpuinfo_arc_cache *p; 398362c389SVineet Gupta 408362c389SVineet Gupta #define PR_CACHE(p, cfg, str) \ 418362c389SVineet Gupta if (!(p)->ver) \ 428362c389SVineet Gupta n += scnprintf(buf + n, len - n, str"\t\t: N/A\n"); \ 438362c389SVineet Gupta else \ 448362c389SVineet Gupta n += scnprintf(buf + n, len - n, \ 458362c389SVineet Gupta str"\t\t: %uK, %dway/set, %uB Line, %s%s%s\n", \ 468362c389SVineet Gupta (p)->sz_k, (p)->assoc, (p)->line_len, \ 478362c389SVineet Gupta (p)->vipt ? "VIPT" : "PIPT", \ 488362c389SVineet Gupta (p)->alias ? " aliasing" : "", \ 49964cf28fSVineet Gupta IS_USED_CFG(cfg)); 508362c389SVineet Gupta 518362c389SVineet Gupta PR_CACHE(&cpuinfo_arc700[c].icache, CONFIG_ARC_HAS_ICACHE, "I-Cache"); 528362c389SVineet Gupta PR_CACHE(&cpuinfo_arc700[c].dcache, CONFIG_ARC_HAS_DCACHE, "D-Cache"); 538362c389SVineet Gupta 54fd0881a2SVineet Gupta if (!is_isa_arcv2()) 55fd0881a2SVineet Gupta return buf; 56fd0881a2SVineet Gupta 57d1f317d8SVineet Gupta p = &cpuinfo_arc700[c].slc; 58d1f317d8SVineet Gupta if (p->ver) 59d1f317d8SVineet Gupta n += scnprintf(buf + n, len - n, 6079335a2cSVineet Gupta "SLC\t\t: %uK, %uB Line%s\n", 6179335a2cSVineet Gupta p->sz_k, p->line_len, IS_USED_RUN(slc_enable)); 62d1f317d8SVineet Gupta 63f2b0b25aSAlexey Brodkin if (ioc_exists) 641648c70dSAlexey Brodkin n += scnprintf(buf + n, len - n, "IOC\t\t:%s\n", 65964cf28fSVineet Gupta IS_DISABLED_RUN(ioc_enable)); 66f2b0b25aSAlexey Brodkin 678362c389SVineet Gupta return buf; 688362c389SVineet Gupta } 698362c389SVineet Gupta 708362c389SVineet Gupta /* 718362c389SVineet Gupta * Read the Cache Build Confuration Registers, Decode them and save into 728362c389SVineet Gupta * the cpuinfo structure for later use. 738362c389SVineet Gupta * No Validation done here, simply read/convert the BCRs 748362c389SVineet Gupta */ 75fd0881a2SVineet Gupta static void read_decode_cache_bcr_arcv2(int cpu) 768362c389SVineet Gupta { 77fd0881a2SVineet Gupta struct cpuinfo_arc_cache *p_slc = &cpuinfo_arc700[cpu].slc; 78d1f317d8SVineet Gupta struct bcr_generic sbcr; 79d1f317d8SVineet Gupta 80d1f317d8SVineet Gupta struct bcr_slc_cfg { 81d1f317d8SVineet Gupta #ifdef CONFIG_CPU_BIG_ENDIAN 82d1f317d8SVineet Gupta unsigned int pad:24, way:2, lsz:2, sz:4; 83d1f317d8SVineet Gupta #else 84d1f317d8SVineet Gupta unsigned int sz:4, lsz:2, way:2, pad:24; 85d1f317d8SVineet Gupta #endif 86d1f317d8SVineet Gupta } slc_cfg; 87d1f317d8SVineet Gupta 88f2b0b25aSAlexey Brodkin struct bcr_clust_cfg { 89f2b0b25aSAlexey Brodkin #ifdef CONFIG_CPU_BIG_ENDIAN 90f2b0b25aSAlexey Brodkin unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8; 91f2b0b25aSAlexey Brodkin #else 92f2b0b25aSAlexey Brodkin unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7; 93f2b0b25aSAlexey Brodkin #endif 94f2b0b25aSAlexey Brodkin } cbcr; 95f2b0b25aSAlexey Brodkin 96fd0881a2SVineet Gupta READ_BCR(ARC_REG_SLC_BCR, sbcr); 97fd0881a2SVineet Gupta if (sbcr.ver) { 98fd0881a2SVineet Gupta READ_BCR(ARC_REG_SLC_CFG, slc_cfg); 99fd0881a2SVineet Gupta p_slc->ver = sbcr.ver; 100fd0881a2SVineet Gupta p_slc->sz_k = 128 << slc_cfg.sz; 101fd0881a2SVineet Gupta l2_line_sz = p_slc->line_len = (slc_cfg.lsz == 0) ? 128 : 64; 102fd0881a2SVineet Gupta } 103fd0881a2SVineet Gupta 104fd0881a2SVineet Gupta READ_BCR(ARC_REG_CLUSTER_BCR, cbcr); 105fd0881a2SVineet Gupta if (cbcr.c && ioc_enable) 106fd0881a2SVineet Gupta ioc_exists = 1; 107fd0881a2SVineet Gupta } 108fd0881a2SVineet Gupta 109fd0881a2SVineet Gupta void read_decode_cache_bcr(void) 110fd0881a2SVineet Gupta { 111fd0881a2SVineet Gupta struct cpuinfo_arc_cache *p_ic, *p_dc; 112fd0881a2SVineet Gupta unsigned int cpu = smp_processor_id(); 113fd0881a2SVineet Gupta struct bcr_cache { 114fd0881a2SVineet Gupta #ifdef CONFIG_CPU_BIG_ENDIAN 115fd0881a2SVineet Gupta unsigned int pad:12, line_len:4, sz:4, config:4, ver:8; 116fd0881a2SVineet Gupta #else 117fd0881a2SVineet Gupta unsigned int ver:8, config:4, sz:4, line_len:4, pad:12; 118fd0881a2SVineet Gupta #endif 119fd0881a2SVineet Gupta } ibcr, dbcr; 120fd0881a2SVineet Gupta 1218362c389SVineet Gupta p_ic = &cpuinfo_arc700[cpu].icache; 1228362c389SVineet Gupta READ_BCR(ARC_REG_IC_BCR, ibcr); 1238362c389SVineet Gupta 1248362c389SVineet Gupta if (!ibcr.ver) 1258362c389SVineet Gupta goto dc_chk; 1268362c389SVineet Gupta 127d1f317d8SVineet Gupta if (ibcr.ver <= 3) { 1288362c389SVineet Gupta BUG_ON(ibcr.config != 3); 1298362c389SVineet Gupta p_ic->assoc = 2; /* Fixed to 2w set assoc */ 130d1f317d8SVineet Gupta } else if (ibcr.ver >= 4) { 131d1f317d8SVineet Gupta p_ic->assoc = 1 << ibcr.config; /* 1,2,4,8 */ 132d1f317d8SVineet Gupta } 133d1f317d8SVineet Gupta 1348362c389SVineet Gupta p_ic->line_len = 8 << ibcr.line_len; 1358362c389SVineet Gupta p_ic->sz_k = 1 << (ibcr.sz - 1); 1368362c389SVineet Gupta p_ic->ver = ibcr.ver; 1378362c389SVineet Gupta p_ic->vipt = 1; 1388362c389SVineet Gupta p_ic->alias = p_ic->sz_k/p_ic->assoc/TO_KB(PAGE_SIZE) > 1; 1398362c389SVineet Gupta 1408362c389SVineet Gupta dc_chk: 1418362c389SVineet Gupta p_dc = &cpuinfo_arc700[cpu].dcache; 1428362c389SVineet Gupta READ_BCR(ARC_REG_DC_BCR, dbcr); 1438362c389SVineet Gupta 1448362c389SVineet Gupta if (!dbcr.ver) 145d1f317d8SVineet Gupta goto slc_chk; 1468362c389SVineet Gupta 147d1f317d8SVineet Gupta if (dbcr.ver <= 3) { 1488362c389SVineet Gupta BUG_ON(dbcr.config != 2); 1498362c389SVineet Gupta p_dc->assoc = 4; /* Fixed to 4w set assoc */ 150d1f317d8SVineet Gupta p_dc->vipt = 1; 151d1f317d8SVineet Gupta p_dc->alias = p_dc->sz_k/p_dc->assoc/TO_KB(PAGE_SIZE) > 1; 152d1f317d8SVineet Gupta } else if (dbcr.ver >= 4) { 153d1f317d8SVineet Gupta p_dc->assoc = 1 << dbcr.config; /* 1,2,4,8 */ 154d1f317d8SVineet Gupta p_dc->vipt = 0; 155d1f317d8SVineet Gupta p_dc->alias = 0; /* PIPT so can't VIPT alias */ 156d1f317d8SVineet Gupta } 157d1f317d8SVineet Gupta 1588362c389SVineet Gupta p_dc->line_len = 16 << dbcr.line_len; 1598362c389SVineet Gupta p_dc->sz_k = 1 << (dbcr.sz - 1); 1608362c389SVineet Gupta p_dc->ver = dbcr.ver; 161d1f317d8SVineet Gupta 162d1f317d8SVineet Gupta slc_chk: 163fd0881a2SVineet Gupta if (is_isa_arcv2()) 164fd0881a2SVineet Gupta read_decode_cache_bcr_arcv2(cpu); 1658362c389SVineet Gupta } 1668362c389SVineet Gupta 1678362c389SVineet Gupta /* 1688ea2ddffSVineet Gupta * Line Operation on {I,D}-Cache 1698362c389SVineet Gupta */ 1708362c389SVineet Gupta 1718362c389SVineet Gupta #define OP_INV 0x1 1728362c389SVineet Gupta #define OP_FLUSH 0x2 1738362c389SVineet Gupta #define OP_FLUSH_N_INV 0x3 1748362c389SVineet Gupta #define OP_INV_IC 0x4 1758362c389SVineet Gupta 1768362c389SVineet Gupta /* 1778ea2ddffSVineet Gupta * I-Cache Aliasing in ARC700 VIPT caches (MMU v1-v3) 1788ea2ddffSVineet Gupta * 1798ea2ddffSVineet Gupta * ARC VIPT I-cache uses vaddr to index into cache and paddr to match the tag. 1808ea2ddffSVineet Gupta * The orig Cache Management Module "CDU" only required paddr to invalidate a 1818ea2ddffSVineet Gupta * certain line since it sufficed as index in Non-Aliasing VIPT cache-geometry. 1828ea2ddffSVineet Gupta * Infact for distinct V1,V2,P: all of {V1-P},{V2-P},{P-P} would end up fetching 1838ea2ddffSVineet Gupta * the exact same line. 1848ea2ddffSVineet Gupta * 1858ea2ddffSVineet Gupta * However for larger Caches (way-size > page-size) - i.e. in Aliasing config, 1868ea2ddffSVineet Gupta * paddr alone could not be used to correctly index the cache. 1878ea2ddffSVineet Gupta * 1888ea2ddffSVineet Gupta * ------------------ 1898ea2ddffSVineet Gupta * MMU v1/v2 (Fixed Page Size 8k) 1908ea2ddffSVineet Gupta * ------------------ 1918ea2ddffSVineet Gupta * The solution was to provide CDU with these additonal vaddr bits. These 1928ea2ddffSVineet Gupta * would be bits [x:13], x would depend on cache-geometry, 13 comes from 1938ea2ddffSVineet Gupta * standard page size of 8k. 1948ea2ddffSVineet Gupta * H/w folks chose [17:13] to be a future safe range, and moreso these 5 bits 1958ea2ddffSVineet Gupta * of vaddr could easily be "stuffed" in the paddr as bits [4:0] since the 1968ea2ddffSVineet Gupta * orig 5 bits of paddr were anyways ignored by CDU line ops, as they 1978ea2ddffSVineet Gupta * represent the offset within cache-line. The adv of using this "clumsy" 1988ea2ddffSVineet Gupta * interface for additional info was no new reg was needed in CDU programming 1998ea2ddffSVineet Gupta * model. 2008ea2ddffSVineet Gupta * 2018ea2ddffSVineet Gupta * 17:13 represented the max num of bits passable, actual bits needed were 2028ea2ddffSVineet Gupta * fewer, based on the num-of-aliases possible. 2038ea2ddffSVineet Gupta * -for 2 alias possibility, only bit 13 needed (32K cache) 2048ea2ddffSVineet Gupta * -for 4 alias possibility, bits 14:13 needed (64K cache) 2058ea2ddffSVineet Gupta * 2068ea2ddffSVineet Gupta * ------------------ 2078ea2ddffSVineet Gupta * MMU v3 2088ea2ddffSVineet Gupta * ------------------ 2098ea2ddffSVineet Gupta * This ver of MMU supports variable page sizes (1k-16k): although Linux will 2108ea2ddffSVineet Gupta * only support 8k (default), 16k and 4k. 2118ea2ddffSVineet Gupta * However from hardware perspective, smaller page sizes aggrevate aliasing 2128ea2ddffSVineet Gupta * meaning more vaddr bits needed to disambiguate the cache-line-op ; 2138ea2ddffSVineet Gupta * the existing scheme of piggybacking won't work for certain configurations. 2148ea2ddffSVineet Gupta * Two new registers IC_PTAG and DC_PTAG inttoduced. 2158ea2ddffSVineet Gupta * "tag" bits are provided in PTAG, index bits in existing IVIL/IVDL/FLDL regs 2168362c389SVineet Gupta */ 2178ea2ddffSVineet Gupta 21811e14896SVineet Gupta static inline 21928b4af72SVineet Gupta void __cache_line_loop_v2(phys_addr_t paddr, unsigned long vaddr, 2208ea2ddffSVineet Gupta unsigned long sz, const int op) 2218362c389SVineet Gupta { 22211e14896SVineet Gupta unsigned int aux_cmd; 2238362c389SVineet Gupta int num_lines; 22411e14896SVineet Gupta const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 2258362c389SVineet Gupta 2268ea2ddffSVineet Gupta if (op == OP_INV_IC) { 2278362c389SVineet Gupta aux_cmd = ARC_REG_IC_IVIL; 22811e14896SVineet Gupta } else { 2298362c389SVineet Gupta /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 2308ea2ddffSVineet Gupta aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 2318362c389SVineet Gupta } 2328362c389SVineet Gupta 2338362c389SVineet Gupta /* Ensure we properly floor/ceil the non-line aligned/sized requests 2348362c389SVineet Gupta * and have @paddr - aligned to cache line and integral @num_lines. 2358362c389SVineet Gupta * This however can be avoided for page sized since: 2368362c389SVineet Gupta * -@paddr will be cache-line aligned already (being page aligned) 2378362c389SVineet Gupta * -@sz will be integral multiple of line size (being page sized). 2388362c389SVineet Gupta */ 23911e14896SVineet Gupta if (!full_page) { 2408362c389SVineet Gupta sz += paddr & ~CACHE_LINE_MASK; 2418362c389SVineet Gupta paddr &= CACHE_LINE_MASK; 2428362c389SVineet Gupta vaddr &= CACHE_LINE_MASK; 2438362c389SVineet Gupta } 2448362c389SVineet Gupta 2458362c389SVineet Gupta num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 2468362c389SVineet Gupta 2478362c389SVineet Gupta /* MMUv2 and before: paddr contains stuffed vaddrs bits */ 2488362c389SVineet Gupta paddr |= (vaddr >> PAGE_SHIFT) & 0x1F; 2498362c389SVineet Gupta 2508362c389SVineet Gupta while (num_lines-- > 0) { 25111e14896SVineet Gupta write_aux_reg(aux_cmd, paddr); 25211e14896SVineet Gupta paddr += L1_CACHE_BYTES; 25311e14896SVineet Gupta } 25411e14896SVineet Gupta } 25511e14896SVineet Gupta 2565a364c2aSVineet Gupta /* 2575a364c2aSVineet Gupta * For ARC700 MMUv3 I-cache and D-cache flushes 2585a364c2aSVineet Gupta * Also reused for HS38 aliasing I-cache configuration 2595a364c2aSVineet Gupta */ 26011e14896SVineet Gupta static inline 26128b4af72SVineet Gupta void __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr, 26211e14896SVineet Gupta unsigned long sz, const int op) 26311e14896SVineet Gupta { 26411e14896SVineet Gupta unsigned int aux_cmd, aux_tag; 26511e14896SVineet Gupta int num_lines; 26611e14896SVineet Gupta const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 26711e14896SVineet Gupta 26811e14896SVineet Gupta if (op == OP_INV_IC) { 26911e14896SVineet Gupta aux_cmd = ARC_REG_IC_IVIL; 27011e14896SVineet Gupta aux_tag = ARC_REG_IC_PTAG; 27111e14896SVineet Gupta } else { 27211e14896SVineet Gupta aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 27311e14896SVineet Gupta aux_tag = ARC_REG_DC_PTAG; 27411e14896SVineet Gupta } 27511e14896SVineet Gupta 27611e14896SVineet Gupta /* Ensure we properly floor/ceil the non-line aligned/sized requests 27711e14896SVineet Gupta * and have @paddr - aligned to cache line and integral @num_lines. 27811e14896SVineet Gupta * This however can be avoided for page sized since: 27911e14896SVineet Gupta * -@paddr will be cache-line aligned already (being page aligned) 28011e14896SVineet Gupta * -@sz will be integral multiple of line size (being page sized). 28111e14896SVineet Gupta */ 28211e14896SVineet Gupta if (!full_page) { 28311e14896SVineet Gupta sz += paddr & ~CACHE_LINE_MASK; 28411e14896SVineet Gupta paddr &= CACHE_LINE_MASK; 28511e14896SVineet Gupta vaddr &= CACHE_LINE_MASK; 28611e14896SVineet Gupta } 28711e14896SVineet Gupta num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 28811e14896SVineet Gupta 28911e14896SVineet Gupta /* 29011e14896SVineet Gupta * MMUv3, cache ops require paddr in PTAG reg 29111e14896SVineet Gupta * if V-P const for loop, PTAG can be written once outside loop 29211e14896SVineet Gupta */ 29311e14896SVineet Gupta if (full_page) 29411e14896SVineet Gupta write_aux_reg(aux_tag, paddr); 29511e14896SVineet Gupta 2965a364c2aSVineet Gupta /* 2975a364c2aSVineet Gupta * This is technically for MMU v4, using the MMU v3 programming model 2985a364c2aSVineet Gupta * Special work for HS38 aliasing I-cache configuratino with PAE40 2995a364c2aSVineet Gupta * - upper 8 bits of paddr need to be written into PTAG_HI 3005a364c2aSVineet Gupta * - (and needs to be written before the lower 32 bits) 3015a364c2aSVineet Gupta * Note that PTAG_HI is hoisted outside the line loop 3025a364c2aSVineet Gupta */ 3035a364c2aSVineet Gupta if (is_pae40_enabled() && op == OP_INV_IC) 3045a364c2aSVineet Gupta write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 3055a364c2aSVineet Gupta 30611e14896SVineet Gupta while (num_lines-- > 0) { 30711e14896SVineet Gupta if (!full_page) { 3088362c389SVineet Gupta write_aux_reg(aux_tag, paddr); 3098362c389SVineet Gupta paddr += L1_CACHE_BYTES; 3108362c389SVineet Gupta } 3118362c389SVineet Gupta 3128362c389SVineet Gupta write_aux_reg(aux_cmd, vaddr); 3138362c389SVineet Gupta vaddr += L1_CACHE_BYTES; 31411e14896SVineet Gupta } 31511e14896SVineet Gupta } 31611e14896SVineet Gupta 317d1f317d8SVineet Gupta /* 3185a364c2aSVineet Gupta * In HS38x (MMU v4), I-cache is VIPT (can alias), D-cache is PIPT 3195a364c2aSVineet Gupta * Here's how cache ops are implemented 320d1f317d8SVineet Gupta * 3215a364c2aSVineet Gupta * - D-cache: only paddr needed (in DC_IVDL/DC_FLDL) 3225a364c2aSVineet Gupta * - I-cache Non Aliasing: Despite VIPT, only paddr needed (in IC_IVIL) 3235a364c2aSVineet Gupta * - I-cache Aliasing: Both vaddr and paddr needed (in IC_IVIL, IC_PTAG 3245a364c2aSVineet Gupta * respectively, similar to MMU v3 programming model, hence 3255a364c2aSVineet Gupta * __cache_line_loop_v3() is used) 3265a364c2aSVineet Gupta * 3275a364c2aSVineet Gupta * If PAE40 is enabled, independent of aliasing considerations, the higher bits 3285a364c2aSVineet Gupta * needs to be written into PTAG_HI 329d1f317d8SVineet Gupta */ 330d1f317d8SVineet Gupta static inline 33128b4af72SVineet Gupta void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, 332d1f317d8SVineet Gupta unsigned long sz, const int cacheop) 333d1f317d8SVineet Gupta { 334d1f317d8SVineet Gupta unsigned int aux_cmd; 335d1f317d8SVineet Gupta int num_lines; 336d1f317d8SVineet Gupta const int full_page_op = __builtin_constant_p(sz) && sz == PAGE_SIZE; 337d1f317d8SVineet Gupta 338d1f317d8SVineet Gupta if (cacheop == OP_INV_IC) { 339d1f317d8SVineet Gupta aux_cmd = ARC_REG_IC_IVIL; 340d1f317d8SVineet Gupta } else { 341d1f317d8SVineet Gupta /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 342d1f317d8SVineet Gupta aux_cmd = cacheop & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 343d1f317d8SVineet Gupta } 344d1f317d8SVineet Gupta 345d1f317d8SVineet Gupta /* Ensure we properly floor/ceil the non-line aligned/sized requests 346d1f317d8SVineet Gupta * and have @paddr - aligned to cache line and integral @num_lines. 347d1f317d8SVineet Gupta * This however can be avoided for page sized since: 348d1f317d8SVineet Gupta * -@paddr will be cache-line aligned already (being page aligned) 349d1f317d8SVineet Gupta * -@sz will be integral multiple of line size (being page sized). 350d1f317d8SVineet Gupta */ 351d1f317d8SVineet Gupta if (!full_page_op) { 352d1f317d8SVineet Gupta sz += paddr & ~CACHE_LINE_MASK; 353d1f317d8SVineet Gupta paddr &= CACHE_LINE_MASK; 354d1f317d8SVineet Gupta } 355d1f317d8SVineet Gupta 356d1f317d8SVineet Gupta num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 357d1f317d8SVineet Gupta 3585a364c2aSVineet Gupta /* 3595a364c2aSVineet Gupta * For HS38 PAE40 configuration 3605a364c2aSVineet Gupta * - upper 8 bits of paddr need to be written into PTAG_HI 3615a364c2aSVineet Gupta * - (and needs to be written before the lower 32 bits) 3625a364c2aSVineet Gupta */ 3635a364c2aSVineet Gupta if (is_pae40_enabled()) { 3645a364c2aSVineet Gupta if (cacheop == OP_INV_IC) 3655a364c2aSVineet Gupta /* 3665a364c2aSVineet Gupta * Non aliasing I-cache in HS38, 3675a364c2aSVineet Gupta * aliasing I-cache handled in __cache_line_loop_v3() 3685a364c2aSVineet Gupta */ 3695a364c2aSVineet Gupta write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 3705a364c2aSVineet Gupta else 3715a364c2aSVineet Gupta write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32); 3725a364c2aSVineet Gupta } 3735a364c2aSVineet Gupta 374d1f317d8SVineet Gupta while (num_lines-- > 0) { 375d1f317d8SVineet Gupta write_aux_reg(aux_cmd, paddr); 376d1f317d8SVineet Gupta paddr += L1_CACHE_BYTES; 377d1f317d8SVineet Gupta } 378d1f317d8SVineet Gupta } 379d1f317d8SVineet Gupta 38011e14896SVineet Gupta #if (CONFIG_ARC_MMU_VER < 3) 38111e14896SVineet Gupta #define __cache_line_loop __cache_line_loop_v2 38211e14896SVineet Gupta #elif (CONFIG_ARC_MMU_VER == 3) 38311e14896SVineet Gupta #define __cache_line_loop __cache_line_loop_v3 384d1f317d8SVineet Gupta #elif (CONFIG_ARC_MMU_VER > 3) 385d1f317d8SVineet Gupta #define __cache_line_loop __cache_line_loop_v4 3868362c389SVineet Gupta #endif 3878362c389SVineet Gupta 3888362c389SVineet Gupta #ifdef CONFIG_ARC_HAS_DCACHE 3898362c389SVineet Gupta 3908362c389SVineet Gupta /*************************************************************** 3918362c389SVineet Gupta * Machine specific helpers for Entire D-Cache or Per Line ops 3928362c389SVineet Gupta */ 3938362c389SVineet Gupta 3946c310681SVineet Gupta static inline void __before_dc_op(const int op) 3958362c389SVineet Gupta { 3968362c389SVineet Gupta if (op == OP_FLUSH_N_INV) { 3978362c389SVineet Gupta /* Dcache provides 2 cmd: FLUSH or INV 3988362c389SVineet Gupta * INV inturn has sub-modes: DISCARD or FLUSH-BEFORE 3998362c389SVineet Gupta * flush-n-inv is achieved by INV cmd but with IM=1 4008362c389SVineet Gupta * So toggle INV sub-mode depending on op request and default 4018362c389SVineet Gupta */ 4026c310681SVineet Gupta const unsigned int ctl = ARC_REG_DC_CTRL; 4036c310681SVineet Gupta write_aux_reg(ctl, read_aux_reg(ctl) | DC_CTRL_INV_MODE_FLUSH); 4046c310681SVineet Gupta } 4058362c389SVineet Gupta } 4068362c389SVineet Gupta 4076c310681SVineet Gupta static inline void __after_dc_op(const int op) 4088362c389SVineet Gupta { 4096c310681SVineet Gupta if (op & OP_FLUSH) { 4106c310681SVineet Gupta const unsigned int ctl = ARC_REG_DC_CTRL; 4116c310681SVineet Gupta unsigned int reg; 4126c310681SVineet Gupta 4136c310681SVineet Gupta /* flush / flush-n-inv both wait */ 4146c310681SVineet Gupta while ((reg = read_aux_reg(ctl)) & DC_CTRL_FLUSH_STATUS) 4156c310681SVineet Gupta ; 4168362c389SVineet Gupta 4178362c389SVineet Gupta /* Switch back to default Invalidate mode */ 4188362c389SVineet Gupta if (op == OP_FLUSH_N_INV) 4196c310681SVineet Gupta write_aux_reg(ctl, reg & ~DC_CTRL_INV_MODE_FLUSH); 4206c310681SVineet Gupta } 4218362c389SVineet Gupta } 4228362c389SVineet Gupta 4238362c389SVineet Gupta /* 4248362c389SVineet Gupta * Operation on Entire D-Cache 4258ea2ddffSVineet Gupta * @op = {OP_INV, OP_FLUSH, OP_FLUSH_N_INV} 4268362c389SVineet Gupta * Note that constant propagation ensures all the checks are gone 4278362c389SVineet Gupta * in generated code 4288362c389SVineet Gupta */ 4298ea2ddffSVineet Gupta static inline void __dc_entire_op(const int op) 4308362c389SVineet Gupta { 4318362c389SVineet Gupta int aux; 4328362c389SVineet Gupta 4336c310681SVineet Gupta __before_dc_op(op); 4348362c389SVineet Gupta 4358ea2ddffSVineet Gupta if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 4368362c389SVineet Gupta aux = ARC_REG_DC_IVDC; 4378362c389SVineet Gupta else 4388362c389SVineet Gupta aux = ARC_REG_DC_FLSH; 4398362c389SVineet Gupta 4408362c389SVineet Gupta write_aux_reg(aux, 0x1); 4418362c389SVineet Gupta 4426c310681SVineet Gupta __after_dc_op(op); 4438362c389SVineet Gupta } 4448362c389SVineet Gupta 4458362c389SVineet Gupta /* For kernel mappings cache operation: index is same as paddr */ 4468362c389SVineet Gupta #define __dc_line_op_k(p, sz, op) __dc_line_op(p, p, sz, op) 4478362c389SVineet Gupta 4488362c389SVineet Gupta /* 4498ea2ddffSVineet Gupta * D-Cache Line ops: Per Line INV (discard or wback+discard) or FLUSH (wback) 4508362c389SVineet Gupta */ 45128b4af72SVineet Gupta static inline void __dc_line_op(phys_addr_t paddr, unsigned long vaddr, 4528ea2ddffSVineet Gupta unsigned long sz, const int op) 4538362c389SVineet Gupta { 4548362c389SVineet Gupta unsigned long flags; 4558362c389SVineet Gupta 4568362c389SVineet Gupta local_irq_save(flags); 4578362c389SVineet Gupta 4586c310681SVineet Gupta __before_dc_op(op); 4598362c389SVineet Gupta 4608ea2ddffSVineet Gupta __cache_line_loop(paddr, vaddr, sz, op); 4618362c389SVineet Gupta 4626c310681SVineet Gupta __after_dc_op(op); 4638362c389SVineet Gupta 4648362c389SVineet Gupta local_irq_restore(flags); 4658362c389SVineet Gupta } 4668362c389SVineet Gupta 4678362c389SVineet Gupta #else 4688362c389SVineet Gupta 4698ea2ddffSVineet Gupta #define __dc_entire_op(op) 4708ea2ddffSVineet Gupta #define __dc_line_op(paddr, vaddr, sz, op) 4718ea2ddffSVineet Gupta #define __dc_line_op_k(paddr, sz, op) 4728362c389SVineet Gupta 4738362c389SVineet Gupta #endif /* CONFIG_ARC_HAS_DCACHE */ 4748362c389SVineet Gupta 4758362c389SVineet Gupta #ifdef CONFIG_ARC_HAS_ICACHE 4768362c389SVineet Gupta 4778362c389SVineet Gupta static inline void __ic_entire_inv(void) 4788362c389SVineet Gupta { 4798362c389SVineet Gupta write_aux_reg(ARC_REG_IC_IVIC, 1); 4808362c389SVineet Gupta read_aux_reg(ARC_REG_IC_CTRL); /* blocks */ 4818362c389SVineet Gupta } 4828362c389SVineet Gupta 4838362c389SVineet Gupta static inline void 48428b4af72SVineet Gupta __ic_line_inv_vaddr_local(phys_addr_t paddr, unsigned long vaddr, 4858362c389SVineet Gupta unsigned long sz) 4868362c389SVineet Gupta { 4878362c389SVineet Gupta unsigned long flags; 4888362c389SVineet Gupta 4898362c389SVineet Gupta local_irq_save(flags); 490bcc4d65aSVineet Gupta (*_cache_line_loop_ic_fn)(paddr, vaddr, sz, OP_INV_IC); 4918362c389SVineet Gupta local_irq_restore(flags); 4928362c389SVineet Gupta } 4938362c389SVineet Gupta 4948362c389SVineet Gupta #ifndef CONFIG_SMP 4958362c389SVineet Gupta 4968362c389SVineet Gupta #define __ic_line_inv_vaddr(p, v, s) __ic_line_inv_vaddr_local(p, v, s) 4978362c389SVineet Gupta 4988362c389SVineet Gupta #else 4998362c389SVineet Gupta 5008362c389SVineet Gupta struct ic_inv_args { 50128b4af72SVineet Gupta phys_addr_t paddr, vaddr; 5028362c389SVineet Gupta int sz; 5038362c389SVineet Gupta }; 5048362c389SVineet Gupta 5058362c389SVineet Gupta static void __ic_line_inv_vaddr_helper(void *info) 5068362c389SVineet Gupta { 5078362c389SVineet Gupta struct ic_inv_args *ic_inv = info; 5088362c389SVineet Gupta 5098362c389SVineet Gupta __ic_line_inv_vaddr_local(ic_inv->paddr, ic_inv->vaddr, ic_inv->sz); 5108362c389SVineet Gupta } 5118362c389SVineet Gupta 51228b4af72SVineet Gupta static void __ic_line_inv_vaddr(phys_addr_t paddr, unsigned long vaddr, 5138362c389SVineet Gupta unsigned long sz) 5148362c389SVineet Gupta { 5158362c389SVineet Gupta struct ic_inv_args ic_inv = { 5168362c389SVineet Gupta .paddr = paddr, 5178362c389SVineet Gupta .vaddr = vaddr, 5188362c389SVineet Gupta .sz = sz 5198362c389SVineet Gupta }; 5208362c389SVineet Gupta 5218362c389SVineet Gupta on_each_cpu(__ic_line_inv_vaddr_helper, &ic_inv, 1); 5228362c389SVineet Gupta } 5238362c389SVineet Gupta 5248362c389SVineet Gupta #endif /* CONFIG_SMP */ 5258362c389SVineet Gupta 5268362c389SVineet Gupta #else /* !CONFIG_ARC_HAS_ICACHE */ 5278362c389SVineet Gupta 5288362c389SVineet Gupta #define __ic_entire_inv() 5298362c389SVineet Gupta #define __ic_line_inv_vaddr(pstart, vstart, sz) 5308362c389SVineet Gupta 5318362c389SVineet Gupta #endif /* CONFIG_ARC_HAS_ICACHE */ 5328362c389SVineet Gupta 53328b4af72SVineet Gupta noinline void slc_op(phys_addr_t paddr, unsigned long sz, const int op) 534795f4558SVineet Gupta { 535795f4558SVineet Gupta #ifdef CONFIG_ISA_ARCV2 536b607edddSAlexey Brodkin /* 537b607edddSAlexey Brodkin * SLC is shared between all cores and concurrent aux operations from 538b607edddSAlexey Brodkin * multiple cores need to be serialized using a spinlock 539b607edddSAlexey Brodkin * A concurrent operation can be silently ignored and/or the old/new 540b607edddSAlexey Brodkin * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop 541b607edddSAlexey Brodkin * below) 542b607edddSAlexey Brodkin */ 543b607edddSAlexey Brodkin static DEFINE_SPINLOCK(lock); 544795f4558SVineet Gupta unsigned long flags; 545795f4558SVineet Gupta unsigned int ctrl; 546795f4558SVineet Gupta 547b607edddSAlexey Brodkin spin_lock_irqsave(&lock, flags); 548795f4558SVineet Gupta 549795f4558SVineet Gupta /* 550795f4558SVineet Gupta * The Region Flush operation is specified by CTRL.RGN_OP[11..9] 551795f4558SVineet Gupta * - b'000 (default) is Flush, 552795f4558SVineet Gupta * - b'001 is Invalidate if CTRL.IM == 0 553795f4558SVineet Gupta * - b'001 is Flush-n-Invalidate if CTRL.IM == 1 554795f4558SVineet Gupta */ 555795f4558SVineet Gupta ctrl = read_aux_reg(ARC_REG_SLC_CTRL); 556795f4558SVineet Gupta 557795f4558SVineet Gupta /* Don't rely on default value of IM bit */ 558795f4558SVineet Gupta if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 559795f4558SVineet Gupta ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 560795f4558SVineet Gupta else 561795f4558SVineet Gupta ctrl |= SLC_CTRL_IM; 562795f4558SVineet Gupta 563795f4558SVineet Gupta if (op & OP_INV) 564795f4558SVineet Gupta ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */ 565795f4558SVineet Gupta else 566795f4558SVineet Gupta ctrl &= ~SLC_CTRL_RGN_OP_INV; 567795f4558SVineet Gupta 568795f4558SVineet Gupta write_aux_reg(ARC_REG_SLC_CTRL, ctrl); 569795f4558SVineet Gupta 570795f4558SVineet Gupta /* 571795f4558SVineet Gupta * Lower bits are ignored, no need to clip 572795f4558SVineet Gupta * END needs to be setup before START (latter triggers the operation) 573795f4558SVineet Gupta * END can't be same as START, so add (l2_line_sz - 1) to sz 574795f4558SVineet Gupta */ 575795f4558SVineet Gupta write_aux_reg(ARC_REG_SLC_RGN_END, (paddr + sz + l2_line_sz - 1)); 576795f4558SVineet Gupta write_aux_reg(ARC_REG_SLC_RGN_START, paddr); 577795f4558SVineet Gupta 578795f4558SVineet Gupta while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY); 579795f4558SVineet Gupta 580b607edddSAlexey Brodkin spin_unlock_irqrestore(&lock, flags); 581795f4558SVineet Gupta #endif 582795f4558SVineet Gupta } 583795f4558SVineet Gupta 5848362c389SVineet Gupta /*********************************************************** 5858362c389SVineet Gupta * Exported APIs 5868362c389SVineet Gupta */ 5878362c389SVineet Gupta 5888362c389SVineet Gupta /* 5898362c389SVineet Gupta * Handle cache congruency of kernel and userspace mappings of page when kernel 5908362c389SVineet Gupta * writes-to/reads-from 5918362c389SVineet Gupta * 5928362c389SVineet Gupta * The idea is to defer flushing of kernel mapping after a WRITE, possible if: 5938362c389SVineet Gupta * -dcache is NOT aliasing, hence any U/K-mappings of page are congruent 5948362c389SVineet Gupta * -U-mapping doesn't exist yet for page (finalised in update_mmu_cache) 5958362c389SVineet Gupta * -In SMP, if hardware caches are coherent 5968362c389SVineet Gupta * 5978362c389SVineet Gupta * There's a corollary case, where kernel READs from a userspace mapped page. 5988362c389SVineet Gupta * If the U-mapping is not congruent to to K-mapping, former needs flushing. 5998362c389SVineet Gupta */ 6008362c389SVineet Gupta void flush_dcache_page(struct page *page) 6018362c389SVineet Gupta { 6028362c389SVineet Gupta struct address_space *mapping; 6038362c389SVineet Gupta 6048362c389SVineet Gupta if (!cache_is_vipt_aliasing()) { 6058362c389SVineet Gupta clear_bit(PG_dc_clean, &page->flags); 6068362c389SVineet Gupta return; 6078362c389SVineet Gupta } 6088362c389SVineet Gupta 6098362c389SVineet Gupta /* don't handle anon pages here */ 6108362c389SVineet Gupta mapping = page_mapping(page); 6118362c389SVineet Gupta if (!mapping) 6128362c389SVineet Gupta return; 6138362c389SVineet Gupta 6148362c389SVineet Gupta /* 6158362c389SVineet Gupta * pagecache page, file not yet mapped to userspace 6168362c389SVineet Gupta * Make a note that K-mapping is dirty 6178362c389SVineet Gupta */ 6188362c389SVineet Gupta if (!mapping_mapped(mapping)) { 6198362c389SVineet Gupta clear_bit(PG_dc_clean, &page->flags); 620e1534ae9SKirill A. Shutemov } else if (page_mapcount(page)) { 6218362c389SVineet Gupta 6228362c389SVineet Gupta /* kernel reading from page with U-mapping */ 62328b4af72SVineet Gupta phys_addr_t paddr = (unsigned long)page_address(page); 6248362c389SVineet Gupta unsigned long vaddr = page->index << PAGE_CACHE_SHIFT; 6258362c389SVineet Gupta 6268362c389SVineet Gupta if (addr_not_cache_congruent(paddr, vaddr)) 6278362c389SVineet Gupta __flush_dcache_page(paddr, vaddr); 6288362c389SVineet Gupta } 6298362c389SVineet Gupta } 6308362c389SVineet Gupta EXPORT_SYMBOL(flush_dcache_page); 6318362c389SVineet Gupta 632f2b0b25aSAlexey Brodkin /* 633f2b0b25aSAlexey Brodkin * DMA ops for systems with L1 cache only 634f2b0b25aSAlexey Brodkin * Make memory coherent with L1 cache by flushing/invalidating L1 lines 635f2b0b25aSAlexey Brodkin */ 636f2b0b25aSAlexey Brodkin static void __dma_cache_wback_inv_l1(unsigned long start, unsigned long sz) 6378362c389SVineet Gupta { 6388362c389SVineet Gupta __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 639f2b0b25aSAlexey Brodkin } 640795f4558SVineet Gupta 641f2b0b25aSAlexey Brodkin static void __dma_cache_inv_l1(unsigned long start, unsigned long sz) 642f2b0b25aSAlexey Brodkin { 643f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_INV); 644f2b0b25aSAlexey Brodkin } 645f2b0b25aSAlexey Brodkin 646f2b0b25aSAlexey Brodkin static void __dma_cache_wback_l1(unsigned long start, unsigned long sz) 647f2b0b25aSAlexey Brodkin { 648f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_FLUSH); 649f2b0b25aSAlexey Brodkin } 650f2b0b25aSAlexey Brodkin 651f2b0b25aSAlexey Brodkin /* 652f2b0b25aSAlexey Brodkin * DMA ops for systems with both L1 and L2 caches, but without IOC 653f2b0b25aSAlexey Brodkin * Both L1 and L2 lines need to be explicity flushed/invalidated 654f2b0b25aSAlexey Brodkin */ 655f2b0b25aSAlexey Brodkin static void __dma_cache_wback_inv_slc(unsigned long start, unsigned long sz) 656f2b0b25aSAlexey Brodkin { 657f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 658795f4558SVineet Gupta slc_op(start, sz, OP_FLUSH_N_INV); 6598362c389SVineet Gupta } 660f2b0b25aSAlexey Brodkin 661f2b0b25aSAlexey Brodkin static void __dma_cache_inv_slc(unsigned long start, unsigned long sz) 662f2b0b25aSAlexey Brodkin { 663f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_INV); 664f2b0b25aSAlexey Brodkin slc_op(start, sz, OP_INV); 665f2b0b25aSAlexey Brodkin } 666f2b0b25aSAlexey Brodkin 667f2b0b25aSAlexey Brodkin static void __dma_cache_wback_slc(unsigned long start, unsigned long sz) 668f2b0b25aSAlexey Brodkin { 669f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_FLUSH); 670f2b0b25aSAlexey Brodkin slc_op(start, sz, OP_FLUSH); 671f2b0b25aSAlexey Brodkin } 672f2b0b25aSAlexey Brodkin 673f2b0b25aSAlexey Brodkin /* 674f2b0b25aSAlexey Brodkin * DMA ops for systems with IOC 675f2b0b25aSAlexey Brodkin * IOC hardware snoops all DMA traffic keeping the caches consistent with 676f2b0b25aSAlexey Brodkin * memory - eliding need for any explicit cache maintenance of DMA buffers 677f2b0b25aSAlexey Brodkin */ 678f2b0b25aSAlexey Brodkin static void __dma_cache_wback_inv_ioc(unsigned long start, unsigned long sz) {} 679f2b0b25aSAlexey Brodkin static void __dma_cache_inv_ioc(unsigned long start, unsigned long sz) {} 680f2b0b25aSAlexey Brodkin static void __dma_cache_wback_ioc(unsigned long start, unsigned long sz) {} 681f2b0b25aSAlexey Brodkin 682f2b0b25aSAlexey Brodkin /* 683f2b0b25aSAlexey Brodkin * Exported DMA API 684f2b0b25aSAlexey Brodkin */ 685f2b0b25aSAlexey Brodkin void dma_cache_wback_inv(unsigned long start, unsigned long sz) 686f2b0b25aSAlexey Brodkin { 687f2b0b25aSAlexey Brodkin __dma_cache_wback_inv(start, sz); 688f2b0b25aSAlexey Brodkin } 6898362c389SVineet Gupta EXPORT_SYMBOL(dma_cache_wback_inv); 6908362c389SVineet Gupta 6918362c389SVineet Gupta void dma_cache_inv(unsigned long start, unsigned long sz) 6928362c389SVineet Gupta { 693f2b0b25aSAlexey Brodkin __dma_cache_inv(start, sz); 6948362c389SVineet Gupta } 6958362c389SVineet Gupta EXPORT_SYMBOL(dma_cache_inv); 6968362c389SVineet Gupta 6978362c389SVineet Gupta void dma_cache_wback(unsigned long start, unsigned long sz) 6988362c389SVineet Gupta { 699f2b0b25aSAlexey Brodkin __dma_cache_wback(start, sz); 7008362c389SVineet Gupta } 7018362c389SVineet Gupta EXPORT_SYMBOL(dma_cache_wback); 7028362c389SVineet Gupta 7038362c389SVineet Gupta /* 7048362c389SVineet Gupta * This is API for making I/D Caches consistent when modifying 7058362c389SVineet Gupta * kernel code (loadable modules, kprobes, kgdb...) 7068362c389SVineet Gupta * This is called on insmod, with kernel virtual address for CODE of 7078362c389SVineet Gupta * the module. ARC cache maintenance ops require PHY address thus we 7088362c389SVineet Gupta * need to convert vmalloc addr to PHY addr 7098362c389SVineet Gupta */ 7108362c389SVineet Gupta void flush_icache_range(unsigned long kstart, unsigned long kend) 7118362c389SVineet Gupta { 7128362c389SVineet Gupta unsigned int tot_sz; 7138362c389SVineet Gupta 7148362c389SVineet Gupta WARN(kstart < TASK_SIZE, "%s() can't handle user vaddr", __func__); 7158362c389SVineet Gupta 7168362c389SVineet Gupta /* Shortcut for bigger flush ranges. 7178362c389SVineet Gupta * Here we don't care if this was kernel virtual or phy addr 7188362c389SVineet Gupta */ 7198362c389SVineet Gupta tot_sz = kend - kstart; 7208362c389SVineet Gupta if (tot_sz > PAGE_SIZE) { 7218362c389SVineet Gupta flush_cache_all(); 7228362c389SVineet Gupta return; 7238362c389SVineet Gupta } 7248362c389SVineet Gupta 7258362c389SVineet Gupta /* Case: Kernel Phy addr (0x8000_0000 onwards) */ 7268362c389SVineet Gupta if (likely(kstart > PAGE_OFFSET)) { 7278362c389SVineet Gupta /* 7288362c389SVineet Gupta * The 2nd arg despite being paddr will be used to index icache 7298362c389SVineet Gupta * This is OK since no alternate virtual mappings will exist 7308362c389SVineet Gupta * given the callers for this case: kprobe/kgdb in built-in 7318362c389SVineet Gupta * kernel code only. 7328362c389SVineet Gupta */ 7338362c389SVineet Gupta __sync_icache_dcache(kstart, kstart, kend - kstart); 7348362c389SVineet Gupta return; 7358362c389SVineet Gupta } 7368362c389SVineet Gupta 7378362c389SVineet Gupta /* 7388362c389SVineet Gupta * Case: Kernel Vaddr (0x7000_0000 to 0x7fff_ffff) 7398362c389SVineet Gupta * (1) ARC Cache Maintenance ops only take Phy addr, hence special 7408362c389SVineet Gupta * handling of kernel vaddr. 7418362c389SVineet Gupta * 7428362c389SVineet Gupta * (2) Despite @tot_sz being < PAGE_SIZE (bigger cases handled already), 7438362c389SVineet Gupta * it still needs to handle a 2 page scenario, where the range 7448362c389SVineet Gupta * straddles across 2 virtual pages and hence need for loop 7458362c389SVineet Gupta */ 7468362c389SVineet Gupta while (tot_sz > 0) { 7478362c389SVineet Gupta unsigned int off, sz; 7488362c389SVineet Gupta unsigned long phy, pfn; 7498362c389SVineet Gupta 7508362c389SVineet Gupta off = kstart % PAGE_SIZE; 7518362c389SVineet Gupta pfn = vmalloc_to_pfn((void *)kstart); 7528362c389SVineet Gupta phy = (pfn << PAGE_SHIFT) + off; 7538362c389SVineet Gupta sz = min_t(unsigned int, tot_sz, PAGE_SIZE - off); 7548362c389SVineet Gupta __sync_icache_dcache(phy, kstart, sz); 7558362c389SVineet Gupta kstart += sz; 7568362c389SVineet Gupta tot_sz -= sz; 7578362c389SVineet Gupta } 7588362c389SVineet Gupta } 7598362c389SVineet Gupta EXPORT_SYMBOL(flush_icache_range); 7608362c389SVineet Gupta 7618362c389SVineet Gupta /* 7628362c389SVineet Gupta * General purpose helper to make I and D cache lines consistent. 7638362c389SVineet Gupta * @paddr is phy addr of region 7648362c389SVineet Gupta * @vaddr is typically user vaddr (breakpoint) or kernel vaddr (vmalloc) 7658362c389SVineet Gupta * However in one instance, when called by kprobe (for a breakpt in 7668362c389SVineet Gupta * builtin kernel code) @vaddr will be paddr only, meaning CDU operation will 7678362c389SVineet Gupta * use a paddr to index the cache (despite VIPT). This is fine since since a 7688362c389SVineet Gupta * builtin kernel page will not have any virtual mappings. 7698362c389SVineet Gupta * kprobe on loadable module will be kernel vaddr. 7708362c389SVineet Gupta */ 77128b4af72SVineet Gupta void __sync_icache_dcache(phys_addr_t paddr, unsigned long vaddr, int len) 7728362c389SVineet Gupta { 7738362c389SVineet Gupta __dc_line_op(paddr, vaddr, len, OP_FLUSH_N_INV); 7748362c389SVineet Gupta __ic_line_inv_vaddr(paddr, vaddr, len); 7758362c389SVineet Gupta } 7768362c389SVineet Gupta 7778362c389SVineet Gupta /* wrapper to compile time eliminate alignment checks in flush loop */ 77828b4af72SVineet Gupta void __inv_icache_page(phys_addr_t paddr, unsigned long vaddr) 7798362c389SVineet Gupta { 7808362c389SVineet Gupta __ic_line_inv_vaddr(paddr, vaddr, PAGE_SIZE); 7818362c389SVineet Gupta } 7828362c389SVineet Gupta 7838362c389SVineet Gupta /* 7848362c389SVineet Gupta * wrapper to clearout kernel or userspace mappings of a page 7858362c389SVineet Gupta * For kernel mappings @vaddr == @paddr 7868362c389SVineet Gupta */ 78728b4af72SVineet Gupta void __flush_dcache_page(phys_addr_t paddr, unsigned long vaddr) 7888362c389SVineet Gupta { 7898362c389SVineet Gupta __dc_line_op(paddr, vaddr & PAGE_MASK, PAGE_SIZE, OP_FLUSH_N_INV); 7908362c389SVineet Gupta } 7918362c389SVineet Gupta 7928362c389SVineet Gupta noinline void flush_cache_all(void) 7938362c389SVineet Gupta { 7948362c389SVineet Gupta unsigned long flags; 7958362c389SVineet Gupta 7968362c389SVineet Gupta local_irq_save(flags); 7978362c389SVineet Gupta 7988362c389SVineet Gupta __ic_entire_inv(); 7998362c389SVineet Gupta __dc_entire_op(OP_FLUSH_N_INV); 8008362c389SVineet Gupta 8018362c389SVineet Gupta local_irq_restore(flags); 8028362c389SVineet Gupta 8038362c389SVineet Gupta } 8048362c389SVineet Gupta 8058362c389SVineet Gupta #ifdef CONFIG_ARC_CACHE_VIPT_ALIASING 8068362c389SVineet Gupta 8078362c389SVineet Gupta void flush_cache_mm(struct mm_struct *mm) 8088362c389SVineet Gupta { 8098362c389SVineet Gupta flush_cache_all(); 8108362c389SVineet Gupta } 8118362c389SVineet Gupta 8128362c389SVineet Gupta void flush_cache_page(struct vm_area_struct *vma, unsigned long u_vaddr, 8138362c389SVineet Gupta unsigned long pfn) 8148362c389SVineet Gupta { 8158362c389SVineet Gupta unsigned int paddr = pfn << PAGE_SHIFT; 8168362c389SVineet Gupta 8178362c389SVineet Gupta u_vaddr &= PAGE_MASK; 8188362c389SVineet Gupta 8198362c389SVineet Gupta __flush_dcache_page(paddr, u_vaddr); 8208362c389SVineet Gupta 8218362c389SVineet Gupta if (vma->vm_flags & VM_EXEC) 8228362c389SVineet Gupta __inv_icache_page(paddr, u_vaddr); 8238362c389SVineet Gupta } 8248362c389SVineet Gupta 8258362c389SVineet Gupta void flush_cache_range(struct vm_area_struct *vma, unsigned long start, 8268362c389SVineet Gupta unsigned long end) 8278362c389SVineet Gupta { 8288362c389SVineet Gupta flush_cache_all(); 8298362c389SVineet Gupta } 8308362c389SVineet Gupta 8318362c389SVineet Gupta void flush_anon_page(struct vm_area_struct *vma, struct page *page, 8328362c389SVineet Gupta unsigned long u_vaddr) 8338362c389SVineet Gupta { 8348362c389SVineet Gupta /* TBD: do we really need to clear the kernel mapping */ 8358362c389SVineet Gupta __flush_dcache_page(page_address(page), u_vaddr); 8368362c389SVineet Gupta __flush_dcache_page(page_address(page), page_address(page)); 8378362c389SVineet Gupta 8388362c389SVineet Gupta } 8398362c389SVineet Gupta 8408362c389SVineet Gupta #endif 8418362c389SVineet Gupta 8428362c389SVineet Gupta void copy_user_highpage(struct page *to, struct page *from, 8438362c389SVineet Gupta unsigned long u_vaddr, struct vm_area_struct *vma) 8448362c389SVineet Gupta { 845336e2136SVineet Gupta void *kfrom = kmap_atomic(from); 846336e2136SVineet Gupta void *kto = kmap_atomic(to); 8478362c389SVineet Gupta int clean_src_k_mappings = 0; 8488362c389SVineet Gupta 8498362c389SVineet Gupta /* 8508362c389SVineet Gupta * If SRC page was already mapped in userspace AND it's U-mapping is 8518362c389SVineet Gupta * not congruent with K-mapping, sync former to physical page so that 8528362c389SVineet Gupta * K-mapping in memcpy below, sees the right data 8538362c389SVineet Gupta * 8548362c389SVineet Gupta * Note that while @u_vaddr refers to DST page's userspace vaddr, it is 8558362c389SVineet Gupta * equally valid for SRC page as well 856336e2136SVineet Gupta * 857336e2136SVineet Gupta * For !VIPT cache, all of this gets compiled out as 858336e2136SVineet Gupta * addr_not_cache_congruent() is 0 8598362c389SVineet Gupta */ 860e1534ae9SKirill A. Shutemov if (page_mapcount(from) && addr_not_cache_congruent(kfrom, u_vaddr)) { 861336e2136SVineet Gupta __flush_dcache_page((unsigned long)kfrom, u_vaddr); 8628362c389SVineet Gupta clean_src_k_mappings = 1; 8638362c389SVineet Gupta } 8648362c389SVineet Gupta 865336e2136SVineet Gupta copy_page(kto, kfrom); 8668362c389SVineet Gupta 8678362c389SVineet Gupta /* 8688362c389SVineet Gupta * Mark DST page K-mapping as dirty for a later finalization by 8698362c389SVineet Gupta * update_mmu_cache(). Although the finalization could have been done 8708362c389SVineet Gupta * here as well (given that both vaddr/paddr are available). 8718362c389SVineet Gupta * But update_mmu_cache() already has code to do that for other 8728362c389SVineet Gupta * non copied user pages (e.g. read faults which wire in pagecache page 8738362c389SVineet Gupta * directly). 8748362c389SVineet Gupta */ 8758362c389SVineet Gupta clear_bit(PG_dc_clean, &to->flags); 8768362c389SVineet Gupta 8778362c389SVineet Gupta /* 8788362c389SVineet Gupta * if SRC was already usermapped and non-congruent to kernel mapping 8798362c389SVineet Gupta * sync the kernel mapping back to physical page 8808362c389SVineet Gupta */ 8818362c389SVineet Gupta if (clean_src_k_mappings) { 882336e2136SVineet Gupta __flush_dcache_page((unsigned long)kfrom, (unsigned long)kfrom); 8838362c389SVineet Gupta set_bit(PG_dc_clean, &from->flags); 8848362c389SVineet Gupta } else { 8858362c389SVineet Gupta clear_bit(PG_dc_clean, &from->flags); 8868362c389SVineet Gupta } 887336e2136SVineet Gupta 888336e2136SVineet Gupta kunmap_atomic(kto); 889336e2136SVineet Gupta kunmap_atomic(kfrom); 8908362c389SVineet Gupta } 8918362c389SVineet Gupta 8928362c389SVineet Gupta void clear_user_page(void *to, unsigned long u_vaddr, struct page *page) 8938362c389SVineet Gupta { 8948362c389SVineet Gupta clear_page(to); 8958362c389SVineet Gupta clear_bit(PG_dc_clean, &page->flags); 8968362c389SVineet Gupta } 8978362c389SVineet Gupta 8988362c389SVineet Gupta 8998362c389SVineet Gupta /********************************************************************** 9008362c389SVineet Gupta * Explicit Cache flush request from user space via syscall 9018362c389SVineet Gupta * Needed for JITs which generate code on the fly 9028362c389SVineet Gupta */ 9038362c389SVineet Gupta SYSCALL_DEFINE3(cacheflush, uint32_t, start, uint32_t, sz, uint32_t, flags) 9048362c389SVineet Gupta { 9058362c389SVineet Gupta /* TBD: optimize this */ 9068362c389SVineet Gupta flush_cache_all(); 9078362c389SVineet Gupta return 0; 9088362c389SVineet Gupta } 9098ea2ddffSVineet Gupta 9108ea2ddffSVineet Gupta void arc_cache_init(void) 9118ea2ddffSVineet Gupta { 9128ea2ddffSVineet Gupta unsigned int __maybe_unused cpu = smp_processor_id(); 9138ea2ddffSVineet Gupta char str[256]; 9148ea2ddffSVineet Gupta 9158ea2ddffSVineet Gupta printk(arc_cache_mumbojumbo(0, str, sizeof(str))); 9168ea2ddffSVineet Gupta 9178ea2ddffSVineet Gupta if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) { 9188ea2ddffSVineet Gupta struct cpuinfo_arc_cache *ic = &cpuinfo_arc700[cpu].icache; 9198ea2ddffSVineet Gupta 9208ea2ddffSVineet Gupta if (!ic->ver) 9218ea2ddffSVineet Gupta panic("cache support enabled but non-existent cache\n"); 9228ea2ddffSVineet Gupta 9238ea2ddffSVineet Gupta if (ic->line_len != L1_CACHE_BYTES) 9248ea2ddffSVineet Gupta panic("ICache line [%d] != kernel Config [%d]", 9258ea2ddffSVineet Gupta ic->line_len, L1_CACHE_BYTES); 9268ea2ddffSVineet Gupta 9278ea2ddffSVineet Gupta if (ic->ver != CONFIG_ARC_MMU_VER) 9288ea2ddffSVineet Gupta panic("Cache ver [%d] doesn't match MMU ver [%d]\n", 9298ea2ddffSVineet Gupta ic->ver, CONFIG_ARC_MMU_VER); 930bcc4d65aSVineet Gupta 931bcc4d65aSVineet Gupta /* 932bcc4d65aSVineet Gupta * In MMU v4 (HS38x) the alising icache config uses IVIL/PTAG 933bcc4d65aSVineet Gupta * pair to provide vaddr/paddr respectively, just as in MMU v3 934bcc4d65aSVineet Gupta */ 935bcc4d65aSVineet Gupta if (is_isa_arcv2() && ic->alias) 936bcc4d65aSVineet Gupta _cache_line_loop_ic_fn = __cache_line_loop_v3; 937bcc4d65aSVineet Gupta else 938bcc4d65aSVineet Gupta _cache_line_loop_ic_fn = __cache_line_loop; 9398ea2ddffSVineet Gupta } 9408ea2ddffSVineet Gupta 9418ea2ddffSVineet Gupta if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) { 9428ea2ddffSVineet Gupta struct cpuinfo_arc_cache *dc = &cpuinfo_arc700[cpu].dcache; 9438ea2ddffSVineet Gupta 9448ea2ddffSVineet Gupta if (!dc->ver) 9458ea2ddffSVineet Gupta panic("cache support enabled but non-existent cache\n"); 9468ea2ddffSVineet Gupta 9478ea2ddffSVineet Gupta if (dc->line_len != L1_CACHE_BYTES) 9488ea2ddffSVineet Gupta panic("DCache line [%d] != kernel Config [%d]", 9498ea2ddffSVineet Gupta dc->line_len, L1_CACHE_BYTES); 9508ea2ddffSVineet Gupta 951d1f317d8SVineet Gupta /* check for D-Cache aliasing on ARCompact: ARCv2 has PIPT */ 952d1f317d8SVineet Gupta if (is_isa_arcompact()) { 953d1f317d8SVineet Gupta int handled = IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING); 9548ea2ddffSVineet Gupta 9558ea2ddffSVineet Gupta if (dc->alias && !handled) 9568ea2ddffSVineet Gupta panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 9578ea2ddffSVineet Gupta else if (!dc->alias && handled) 9588ea2ddffSVineet Gupta panic("Disable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 9598ea2ddffSVineet Gupta } 9608ea2ddffSVineet Gupta } 961f2b0b25aSAlexey Brodkin 96279335a2cSVineet Gupta if (is_isa_arcv2() && l2_line_sz && !slc_enable) { 96379335a2cSVineet Gupta 96479335a2cSVineet Gupta /* IM set : flush before invalidate */ 96579335a2cSVineet Gupta write_aux_reg(ARC_REG_SLC_CTRL, 96679335a2cSVineet Gupta read_aux_reg(ARC_REG_SLC_CTRL) | SLC_CTRL_IM); 96779335a2cSVineet Gupta 96879335a2cSVineet Gupta write_aux_reg(ARC_REG_SLC_INVALIDATE, 1); 96979335a2cSVineet Gupta 97079335a2cSVineet Gupta /* Important to wait for flush to complete */ 97179335a2cSVineet Gupta while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY); 97279335a2cSVineet Gupta write_aux_reg(ARC_REG_SLC_CTRL, 97379335a2cSVineet Gupta read_aux_reg(ARC_REG_SLC_CTRL) | SLC_CTRL_DISABLE); 97479335a2cSVineet Gupta } 97579335a2cSVineet Gupta 976f2b0b25aSAlexey Brodkin if (is_isa_arcv2() && ioc_exists) { 977f2b0b25aSAlexey Brodkin /* IO coherency base - 0x8z */ 978f2b0b25aSAlexey Brodkin write_aux_reg(ARC_REG_IO_COH_AP0_BASE, 0x80000); 979f2b0b25aSAlexey Brodkin /* IO coherency aperture size - 512Mb: 0x8z-0xAz */ 980f2b0b25aSAlexey Brodkin write_aux_reg(ARC_REG_IO_COH_AP0_SIZE, 0x11); 981f2b0b25aSAlexey Brodkin /* Enable partial writes */ 982f2b0b25aSAlexey Brodkin write_aux_reg(ARC_REG_IO_COH_PARTIAL, 1); 983f2b0b25aSAlexey Brodkin /* Enable IO coherency */ 984f2b0b25aSAlexey Brodkin write_aux_reg(ARC_REG_IO_COH_ENABLE, 1); 985f2b0b25aSAlexey Brodkin 986f2b0b25aSAlexey Brodkin __dma_cache_wback_inv = __dma_cache_wback_inv_ioc; 987f2b0b25aSAlexey Brodkin __dma_cache_inv = __dma_cache_inv_ioc; 988f2b0b25aSAlexey Brodkin __dma_cache_wback = __dma_cache_wback_ioc; 98979335a2cSVineet Gupta } else if (is_isa_arcv2() && l2_line_sz && slc_enable) { 990f2b0b25aSAlexey Brodkin __dma_cache_wback_inv = __dma_cache_wback_inv_slc; 991f2b0b25aSAlexey Brodkin __dma_cache_inv = __dma_cache_inv_slc; 992f2b0b25aSAlexey Brodkin __dma_cache_wback = __dma_cache_wback_slc; 993f2b0b25aSAlexey Brodkin } else { 994f2b0b25aSAlexey Brodkin __dma_cache_wback_inv = __dma_cache_wback_inv_l1; 995f2b0b25aSAlexey Brodkin __dma_cache_inv = __dma_cache_inv_l1; 996f2b0b25aSAlexey Brodkin __dma_cache_wback = __dma_cache_wback_l1; 997f2b0b25aSAlexey Brodkin } 998d1f317d8SVineet Gupta } 999