18362c389SVineet Gupta /* 28ea2ddffSVineet Gupta * ARC Cache Management 38362c389SVineet Gupta * 48ea2ddffSVineet Gupta * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com) 58362c389SVineet Gupta * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 68362c389SVineet Gupta * 78362c389SVineet Gupta * This program is free software; you can redistribute it and/or modify 88362c389SVineet Gupta * it under the terms of the GNU General Public License version 2 as 98362c389SVineet Gupta * published by the Free Software Foundation. 108362c389SVineet Gupta */ 118362c389SVineet Gupta 128362c389SVineet Gupta #include <linux/module.h> 138362c389SVineet Gupta #include <linux/mm.h> 148362c389SVineet Gupta #include <linux/sched.h> 158362c389SVineet Gupta #include <linux/cache.h> 168362c389SVineet Gupta #include <linux/mmu_context.h> 178362c389SVineet Gupta #include <linux/syscalls.h> 188362c389SVineet Gupta #include <linux/uaccess.h> 198362c389SVineet Gupta #include <linux/pagemap.h> 208362c389SVineet Gupta #include <asm/cacheflush.h> 218362c389SVineet Gupta #include <asm/cachectl.h> 228362c389SVineet Gupta #include <asm/setup.h> 238362c389SVineet Gupta 24795f4558SVineet Gupta static int l2_line_sz; 25cf986d47SVineet Gupta static int ioc_exists; 26cf986d47SVineet Gupta int slc_enable = 1, ioc_enable = 1; 27deaf7565SVineet Gupta unsigned long perip_base = ARC_UNCACHED_ADDR_SPACE; /* legacy value for boot */ 2826c01c49SVineet Gupta unsigned long perip_end = 0xFFFFFFFF; /* legacy value */ 29795f4558SVineet Gupta 3028b4af72SVineet Gupta void (*_cache_line_loop_ic_fn)(phys_addr_t paddr, unsigned long vaddr, 31bcc4d65aSVineet Gupta unsigned long sz, const int cacheop); 32bcc4d65aSVineet Gupta 33f5db19e9SVineet Gupta void (*__dma_cache_wback_inv)(phys_addr_t start, unsigned long sz); 34f5db19e9SVineet Gupta void (*__dma_cache_inv)(phys_addr_t start, unsigned long sz); 35f5db19e9SVineet Gupta void (*__dma_cache_wback)(phys_addr_t start, unsigned long sz); 36f2b0b25aSAlexey Brodkin 378362c389SVineet Gupta char *arc_cache_mumbojumbo(int c, char *buf, int len) 388362c389SVineet Gupta { 398362c389SVineet Gupta int n = 0; 40d1f317d8SVineet Gupta struct cpuinfo_arc_cache *p; 418362c389SVineet Gupta 428362c389SVineet Gupta #define PR_CACHE(p, cfg, str) \ 438362c389SVineet Gupta if (!(p)->ver) \ 448362c389SVineet Gupta n += scnprintf(buf + n, len - n, str"\t\t: N/A\n"); \ 458362c389SVineet Gupta else \ 468362c389SVineet Gupta n += scnprintf(buf + n, len - n, \ 478362c389SVineet Gupta str"\t\t: %uK, %dway/set, %uB Line, %s%s%s\n", \ 488362c389SVineet Gupta (p)->sz_k, (p)->assoc, (p)->line_len, \ 498362c389SVineet Gupta (p)->vipt ? "VIPT" : "PIPT", \ 508362c389SVineet Gupta (p)->alias ? " aliasing" : "", \ 51964cf28fSVineet Gupta IS_USED_CFG(cfg)); 528362c389SVineet Gupta 538362c389SVineet Gupta PR_CACHE(&cpuinfo_arc700[c].icache, CONFIG_ARC_HAS_ICACHE, "I-Cache"); 548362c389SVineet Gupta PR_CACHE(&cpuinfo_arc700[c].dcache, CONFIG_ARC_HAS_DCACHE, "D-Cache"); 558362c389SVineet Gupta 56fd0881a2SVineet Gupta if (!is_isa_arcv2()) 57fd0881a2SVineet Gupta return buf; 58fd0881a2SVineet Gupta 59d1f317d8SVineet Gupta p = &cpuinfo_arc700[c].slc; 60d1f317d8SVineet Gupta if (p->ver) 61d1f317d8SVineet Gupta n += scnprintf(buf + n, len - n, 6279335a2cSVineet Gupta "SLC\t\t: %uK, %uB Line%s\n", 6379335a2cSVineet Gupta p->sz_k, p->line_len, IS_USED_RUN(slc_enable)); 64d1f317d8SVineet Gupta 65f2b0b25aSAlexey Brodkin if (ioc_exists) 661648c70dSAlexey Brodkin n += scnprintf(buf + n, len - n, "IOC\t\t:%s\n", 67964cf28fSVineet Gupta IS_DISABLED_RUN(ioc_enable)); 68f2b0b25aSAlexey Brodkin 698362c389SVineet Gupta return buf; 708362c389SVineet Gupta } 718362c389SVineet Gupta 728362c389SVineet Gupta /* 738362c389SVineet Gupta * Read the Cache Build Confuration Registers, Decode them and save into 748362c389SVineet Gupta * the cpuinfo structure for later use. 758362c389SVineet Gupta * No Validation done here, simply read/convert the BCRs 768362c389SVineet Gupta */ 77fd0881a2SVineet Gupta static void read_decode_cache_bcr_arcv2(int cpu) 788362c389SVineet Gupta { 79fd0881a2SVineet Gupta struct cpuinfo_arc_cache *p_slc = &cpuinfo_arc700[cpu].slc; 80d1f317d8SVineet Gupta struct bcr_generic sbcr; 81d1f317d8SVineet Gupta 82d1f317d8SVineet Gupta struct bcr_slc_cfg { 83d1f317d8SVineet Gupta #ifdef CONFIG_CPU_BIG_ENDIAN 84d1f317d8SVineet Gupta unsigned int pad:24, way:2, lsz:2, sz:4; 85d1f317d8SVineet Gupta #else 86d1f317d8SVineet Gupta unsigned int sz:4, lsz:2, way:2, pad:24; 87d1f317d8SVineet Gupta #endif 88d1f317d8SVineet Gupta } slc_cfg; 89d1f317d8SVineet Gupta 90f2b0b25aSAlexey Brodkin struct bcr_clust_cfg { 91f2b0b25aSAlexey Brodkin #ifdef CONFIG_CPU_BIG_ENDIAN 92f2b0b25aSAlexey Brodkin unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8; 93f2b0b25aSAlexey Brodkin #else 94f2b0b25aSAlexey Brodkin unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7; 95f2b0b25aSAlexey Brodkin #endif 96f2b0b25aSAlexey Brodkin } cbcr; 97f2b0b25aSAlexey Brodkin 9826c01c49SVineet Gupta struct bcr_volatile { 9926c01c49SVineet Gupta #ifdef CONFIG_CPU_BIG_ENDIAN 10026c01c49SVineet Gupta unsigned int start:4, limit:4, pad:22, order:1, disable:1; 10126c01c49SVineet Gupta #else 10226c01c49SVineet Gupta unsigned int disable:1, order:1, pad:22, limit:4, start:4; 10326c01c49SVineet Gupta #endif 10426c01c49SVineet Gupta } vol; 10526c01c49SVineet Gupta 10626c01c49SVineet Gupta 107fd0881a2SVineet Gupta READ_BCR(ARC_REG_SLC_BCR, sbcr); 108fd0881a2SVineet Gupta if (sbcr.ver) { 109fd0881a2SVineet Gupta READ_BCR(ARC_REG_SLC_CFG, slc_cfg); 110fd0881a2SVineet Gupta p_slc->ver = sbcr.ver; 111fd0881a2SVineet Gupta p_slc->sz_k = 128 << slc_cfg.sz; 112fd0881a2SVineet Gupta l2_line_sz = p_slc->line_len = (slc_cfg.lsz == 0) ? 128 : 64; 113fd0881a2SVineet Gupta } 114fd0881a2SVineet Gupta 115fd0881a2SVineet Gupta READ_BCR(ARC_REG_CLUSTER_BCR, cbcr); 116cf986d47SVineet Gupta if (cbcr.c) 117fd0881a2SVineet Gupta ioc_exists = 1; 118cf986d47SVineet Gupta else 119cf986d47SVineet Gupta ioc_enable = 0; 120deaf7565SVineet Gupta 12126c01c49SVineet Gupta /* HS 2.0 didn't have AUX_VOL */ 12226c01c49SVineet Gupta if (cpuinfo_arc700[cpu].core.family > 0x51) { 12326c01c49SVineet Gupta READ_BCR(AUX_VOL, vol); 12426c01c49SVineet Gupta perip_base = vol.start << 28; 12526c01c49SVineet Gupta /* HS 3.0 has limit and strict-ordering fields */ 12626c01c49SVineet Gupta if (cpuinfo_arc700[cpu].core.family > 0x52) 12726c01c49SVineet Gupta perip_end = (vol.limit << 28) - 1; 12826c01c49SVineet Gupta } 129fd0881a2SVineet Gupta } 130fd0881a2SVineet Gupta 131fd0881a2SVineet Gupta void read_decode_cache_bcr(void) 132fd0881a2SVineet Gupta { 133fd0881a2SVineet Gupta struct cpuinfo_arc_cache *p_ic, *p_dc; 134fd0881a2SVineet Gupta unsigned int cpu = smp_processor_id(); 135fd0881a2SVineet Gupta struct bcr_cache { 136fd0881a2SVineet Gupta #ifdef CONFIG_CPU_BIG_ENDIAN 137fd0881a2SVineet Gupta unsigned int pad:12, line_len:4, sz:4, config:4, ver:8; 138fd0881a2SVineet Gupta #else 139fd0881a2SVineet Gupta unsigned int ver:8, config:4, sz:4, line_len:4, pad:12; 140fd0881a2SVineet Gupta #endif 141fd0881a2SVineet Gupta } ibcr, dbcr; 142fd0881a2SVineet Gupta 1438362c389SVineet Gupta p_ic = &cpuinfo_arc700[cpu].icache; 1448362c389SVineet Gupta READ_BCR(ARC_REG_IC_BCR, ibcr); 1458362c389SVineet Gupta 1468362c389SVineet Gupta if (!ibcr.ver) 1478362c389SVineet Gupta goto dc_chk; 1488362c389SVineet Gupta 149d1f317d8SVineet Gupta if (ibcr.ver <= 3) { 1508362c389SVineet Gupta BUG_ON(ibcr.config != 3); 1518362c389SVineet Gupta p_ic->assoc = 2; /* Fixed to 2w set assoc */ 152d1f317d8SVineet Gupta } else if (ibcr.ver >= 4) { 153d1f317d8SVineet Gupta p_ic->assoc = 1 << ibcr.config; /* 1,2,4,8 */ 154d1f317d8SVineet Gupta } 155d1f317d8SVineet Gupta 1568362c389SVineet Gupta p_ic->line_len = 8 << ibcr.line_len; 1578362c389SVineet Gupta p_ic->sz_k = 1 << (ibcr.sz - 1); 1588362c389SVineet Gupta p_ic->ver = ibcr.ver; 1598362c389SVineet Gupta p_ic->vipt = 1; 1608362c389SVineet Gupta p_ic->alias = p_ic->sz_k/p_ic->assoc/TO_KB(PAGE_SIZE) > 1; 1618362c389SVineet Gupta 1628362c389SVineet Gupta dc_chk: 1638362c389SVineet Gupta p_dc = &cpuinfo_arc700[cpu].dcache; 1648362c389SVineet Gupta READ_BCR(ARC_REG_DC_BCR, dbcr); 1658362c389SVineet Gupta 1668362c389SVineet Gupta if (!dbcr.ver) 167d1f317d8SVineet Gupta goto slc_chk; 1688362c389SVineet Gupta 169d1f317d8SVineet Gupta if (dbcr.ver <= 3) { 1708362c389SVineet Gupta BUG_ON(dbcr.config != 2); 1718362c389SVineet Gupta p_dc->assoc = 4; /* Fixed to 4w set assoc */ 172d1f317d8SVineet Gupta p_dc->vipt = 1; 173d1f317d8SVineet Gupta p_dc->alias = p_dc->sz_k/p_dc->assoc/TO_KB(PAGE_SIZE) > 1; 174d1f317d8SVineet Gupta } else if (dbcr.ver >= 4) { 175d1f317d8SVineet Gupta p_dc->assoc = 1 << dbcr.config; /* 1,2,4,8 */ 176d1f317d8SVineet Gupta p_dc->vipt = 0; 177d1f317d8SVineet Gupta p_dc->alias = 0; /* PIPT so can't VIPT alias */ 178d1f317d8SVineet Gupta } 179d1f317d8SVineet Gupta 1808362c389SVineet Gupta p_dc->line_len = 16 << dbcr.line_len; 1818362c389SVineet Gupta p_dc->sz_k = 1 << (dbcr.sz - 1); 1828362c389SVineet Gupta p_dc->ver = dbcr.ver; 183d1f317d8SVineet Gupta 184d1f317d8SVineet Gupta slc_chk: 185fd0881a2SVineet Gupta if (is_isa_arcv2()) 186fd0881a2SVineet Gupta read_decode_cache_bcr_arcv2(cpu); 1878362c389SVineet Gupta } 1888362c389SVineet Gupta 1898362c389SVineet Gupta /* 1908ea2ddffSVineet Gupta * Line Operation on {I,D}-Cache 1918362c389SVineet Gupta */ 1928362c389SVineet Gupta 1938362c389SVineet Gupta #define OP_INV 0x1 1948362c389SVineet Gupta #define OP_FLUSH 0x2 1958362c389SVineet Gupta #define OP_FLUSH_N_INV 0x3 1968362c389SVineet Gupta #define OP_INV_IC 0x4 1978362c389SVineet Gupta 1988362c389SVineet Gupta /* 1998ea2ddffSVineet Gupta * I-Cache Aliasing in ARC700 VIPT caches (MMU v1-v3) 2008ea2ddffSVineet Gupta * 2018ea2ddffSVineet Gupta * ARC VIPT I-cache uses vaddr to index into cache and paddr to match the tag. 2028ea2ddffSVineet Gupta * The orig Cache Management Module "CDU" only required paddr to invalidate a 2038ea2ddffSVineet Gupta * certain line since it sufficed as index in Non-Aliasing VIPT cache-geometry. 2048ea2ddffSVineet Gupta * Infact for distinct V1,V2,P: all of {V1-P},{V2-P},{P-P} would end up fetching 2058ea2ddffSVineet Gupta * the exact same line. 2068ea2ddffSVineet Gupta * 2078ea2ddffSVineet Gupta * However for larger Caches (way-size > page-size) - i.e. in Aliasing config, 2088ea2ddffSVineet Gupta * paddr alone could not be used to correctly index the cache. 2098ea2ddffSVineet Gupta * 2108ea2ddffSVineet Gupta * ------------------ 2118ea2ddffSVineet Gupta * MMU v1/v2 (Fixed Page Size 8k) 2128ea2ddffSVineet Gupta * ------------------ 2138ea2ddffSVineet Gupta * The solution was to provide CDU with these additonal vaddr bits. These 2148ea2ddffSVineet Gupta * would be bits [x:13], x would depend on cache-geometry, 13 comes from 2158ea2ddffSVineet Gupta * standard page size of 8k. 2168ea2ddffSVineet Gupta * H/w folks chose [17:13] to be a future safe range, and moreso these 5 bits 2178ea2ddffSVineet Gupta * of vaddr could easily be "stuffed" in the paddr as bits [4:0] since the 2188ea2ddffSVineet Gupta * orig 5 bits of paddr were anyways ignored by CDU line ops, as they 2198ea2ddffSVineet Gupta * represent the offset within cache-line. The adv of using this "clumsy" 2208ea2ddffSVineet Gupta * interface for additional info was no new reg was needed in CDU programming 2218ea2ddffSVineet Gupta * model. 2228ea2ddffSVineet Gupta * 2238ea2ddffSVineet Gupta * 17:13 represented the max num of bits passable, actual bits needed were 2248ea2ddffSVineet Gupta * fewer, based on the num-of-aliases possible. 2258ea2ddffSVineet Gupta * -for 2 alias possibility, only bit 13 needed (32K cache) 2268ea2ddffSVineet Gupta * -for 4 alias possibility, bits 14:13 needed (64K cache) 2278ea2ddffSVineet Gupta * 2288ea2ddffSVineet Gupta * ------------------ 2298ea2ddffSVineet Gupta * MMU v3 2308ea2ddffSVineet Gupta * ------------------ 2318ea2ddffSVineet Gupta * This ver of MMU supports variable page sizes (1k-16k): although Linux will 2328ea2ddffSVineet Gupta * only support 8k (default), 16k and 4k. 2332547476aSAndrea Gelmini * However from hardware perspective, smaller page sizes aggravate aliasing 2348ea2ddffSVineet Gupta * meaning more vaddr bits needed to disambiguate the cache-line-op ; 2358ea2ddffSVineet Gupta * the existing scheme of piggybacking won't work for certain configurations. 2368ea2ddffSVineet Gupta * Two new registers IC_PTAG and DC_PTAG inttoduced. 2378ea2ddffSVineet Gupta * "tag" bits are provided in PTAG, index bits in existing IVIL/IVDL/FLDL regs 2388362c389SVineet Gupta */ 2398ea2ddffSVineet Gupta 24011e14896SVineet Gupta static inline 24128b4af72SVineet Gupta void __cache_line_loop_v2(phys_addr_t paddr, unsigned long vaddr, 2428ea2ddffSVineet Gupta unsigned long sz, const int op) 2438362c389SVineet Gupta { 24411e14896SVineet Gupta unsigned int aux_cmd; 2458362c389SVineet Gupta int num_lines; 24611e14896SVineet Gupta const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 2478362c389SVineet Gupta 2488ea2ddffSVineet Gupta if (op == OP_INV_IC) { 2498362c389SVineet Gupta aux_cmd = ARC_REG_IC_IVIL; 25011e14896SVineet Gupta } else { 2518362c389SVineet Gupta /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 2528ea2ddffSVineet Gupta aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 2538362c389SVineet Gupta } 2548362c389SVineet Gupta 2558362c389SVineet Gupta /* Ensure we properly floor/ceil the non-line aligned/sized requests 2568362c389SVineet Gupta * and have @paddr - aligned to cache line and integral @num_lines. 2578362c389SVineet Gupta * This however can be avoided for page sized since: 2588362c389SVineet Gupta * -@paddr will be cache-line aligned already (being page aligned) 2598362c389SVineet Gupta * -@sz will be integral multiple of line size (being page sized). 2608362c389SVineet Gupta */ 26111e14896SVineet Gupta if (!full_page) { 2628362c389SVineet Gupta sz += paddr & ~CACHE_LINE_MASK; 2638362c389SVineet Gupta paddr &= CACHE_LINE_MASK; 2648362c389SVineet Gupta vaddr &= CACHE_LINE_MASK; 2658362c389SVineet Gupta } 2668362c389SVineet Gupta 2678362c389SVineet Gupta num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 2688362c389SVineet Gupta 2698362c389SVineet Gupta /* MMUv2 and before: paddr contains stuffed vaddrs bits */ 2708362c389SVineet Gupta paddr |= (vaddr >> PAGE_SHIFT) & 0x1F; 2718362c389SVineet Gupta 2728362c389SVineet Gupta while (num_lines-- > 0) { 27311e14896SVineet Gupta write_aux_reg(aux_cmd, paddr); 27411e14896SVineet Gupta paddr += L1_CACHE_BYTES; 27511e14896SVineet Gupta } 27611e14896SVineet Gupta } 27711e14896SVineet Gupta 2785a364c2aSVineet Gupta /* 2795a364c2aSVineet Gupta * For ARC700 MMUv3 I-cache and D-cache flushes 2805a364c2aSVineet Gupta * Also reused for HS38 aliasing I-cache configuration 2815a364c2aSVineet Gupta */ 28211e14896SVineet Gupta static inline 28328b4af72SVineet Gupta void __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr, 28411e14896SVineet Gupta unsigned long sz, const int op) 28511e14896SVineet Gupta { 28611e14896SVineet Gupta unsigned int aux_cmd, aux_tag; 28711e14896SVineet Gupta int num_lines; 28811e14896SVineet Gupta const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 28911e14896SVineet Gupta 29011e14896SVineet Gupta if (op == OP_INV_IC) { 29111e14896SVineet Gupta aux_cmd = ARC_REG_IC_IVIL; 29211e14896SVineet Gupta aux_tag = ARC_REG_IC_PTAG; 29311e14896SVineet Gupta } else { 29411e14896SVineet Gupta aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 29511e14896SVineet Gupta aux_tag = ARC_REG_DC_PTAG; 29611e14896SVineet Gupta } 29711e14896SVineet Gupta 29811e14896SVineet Gupta /* Ensure we properly floor/ceil the non-line aligned/sized requests 29911e14896SVineet Gupta * and have @paddr - aligned to cache line and integral @num_lines. 30011e14896SVineet Gupta * This however can be avoided for page sized since: 30111e14896SVineet Gupta * -@paddr will be cache-line aligned already (being page aligned) 30211e14896SVineet Gupta * -@sz will be integral multiple of line size (being page sized). 30311e14896SVineet Gupta */ 30411e14896SVineet Gupta if (!full_page) { 30511e14896SVineet Gupta sz += paddr & ~CACHE_LINE_MASK; 30611e14896SVineet Gupta paddr &= CACHE_LINE_MASK; 30711e14896SVineet Gupta vaddr &= CACHE_LINE_MASK; 30811e14896SVineet Gupta } 30911e14896SVineet Gupta num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 31011e14896SVineet Gupta 31111e14896SVineet Gupta /* 31211e14896SVineet Gupta * MMUv3, cache ops require paddr in PTAG reg 31311e14896SVineet Gupta * if V-P const for loop, PTAG can be written once outside loop 31411e14896SVineet Gupta */ 31511e14896SVineet Gupta if (full_page) 31611e14896SVineet Gupta write_aux_reg(aux_tag, paddr); 31711e14896SVineet Gupta 3185a364c2aSVineet Gupta /* 3195a364c2aSVineet Gupta * This is technically for MMU v4, using the MMU v3 programming model 3202547476aSAndrea Gelmini * Special work for HS38 aliasing I-cache configuration with PAE40 3215a364c2aSVineet Gupta * - upper 8 bits of paddr need to be written into PTAG_HI 3225a364c2aSVineet Gupta * - (and needs to be written before the lower 32 bits) 3235a364c2aSVineet Gupta * Note that PTAG_HI is hoisted outside the line loop 3245a364c2aSVineet Gupta */ 3255a364c2aSVineet Gupta if (is_pae40_enabled() && op == OP_INV_IC) 3265a364c2aSVineet Gupta write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 3275a364c2aSVineet Gupta 32811e14896SVineet Gupta while (num_lines-- > 0) { 32911e14896SVineet Gupta if (!full_page) { 3308362c389SVineet Gupta write_aux_reg(aux_tag, paddr); 3318362c389SVineet Gupta paddr += L1_CACHE_BYTES; 3328362c389SVineet Gupta } 3338362c389SVineet Gupta 3348362c389SVineet Gupta write_aux_reg(aux_cmd, vaddr); 3358362c389SVineet Gupta vaddr += L1_CACHE_BYTES; 33611e14896SVineet Gupta } 33711e14896SVineet Gupta } 33811e14896SVineet Gupta 339d1f317d8SVineet Gupta /* 3405a364c2aSVineet Gupta * In HS38x (MMU v4), I-cache is VIPT (can alias), D-cache is PIPT 3415a364c2aSVineet Gupta * Here's how cache ops are implemented 342d1f317d8SVineet Gupta * 3435a364c2aSVineet Gupta * - D-cache: only paddr needed (in DC_IVDL/DC_FLDL) 3445a364c2aSVineet Gupta * - I-cache Non Aliasing: Despite VIPT, only paddr needed (in IC_IVIL) 3455a364c2aSVineet Gupta * - I-cache Aliasing: Both vaddr and paddr needed (in IC_IVIL, IC_PTAG 3465a364c2aSVineet Gupta * respectively, similar to MMU v3 programming model, hence 3475a364c2aSVineet Gupta * __cache_line_loop_v3() is used) 3485a364c2aSVineet Gupta * 3495a364c2aSVineet Gupta * If PAE40 is enabled, independent of aliasing considerations, the higher bits 3505a364c2aSVineet Gupta * needs to be written into PTAG_HI 351d1f317d8SVineet Gupta */ 352d1f317d8SVineet Gupta static inline 35328b4af72SVineet Gupta void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, 354d1f317d8SVineet Gupta unsigned long sz, const int cacheop) 355d1f317d8SVineet Gupta { 356d1f317d8SVineet Gupta unsigned int aux_cmd; 357d1f317d8SVineet Gupta int num_lines; 358d1f317d8SVineet Gupta const int full_page_op = __builtin_constant_p(sz) && sz == PAGE_SIZE; 359d1f317d8SVineet Gupta 360d1f317d8SVineet Gupta if (cacheop == OP_INV_IC) { 361d1f317d8SVineet Gupta aux_cmd = ARC_REG_IC_IVIL; 362d1f317d8SVineet Gupta } else { 363d1f317d8SVineet Gupta /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 364d1f317d8SVineet Gupta aux_cmd = cacheop & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 365d1f317d8SVineet Gupta } 366d1f317d8SVineet Gupta 367d1f317d8SVineet Gupta /* Ensure we properly floor/ceil the non-line aligned/sized requests 368d1f317d8SVineet Gupta * and have @paddr - aligned to cache line and integral @num_lines. 369d1f317d8SVineet Gupta * This however can be avoided for page sized since: 370d1f317d8SVineet Gupta * -@paddr will be cache-line aligned already (being page aligned) 371d1f317d8SVineet Gupta * -@sz will be integral multiple of line size (being page sized). 372d1f317d8SVineet Gupta */ 373d1f317d8SVineet Gupta if (!full_page_op) { 374d1f317d8SVineet Gupta sz += paddr & ~CACHE_LINE_MASK; 375d1f317d8SVineet Gupta paddr &= CACHE_LINE_MASK; 376d1f317d8SVineet Gupta } 377d1f317d8SVineet Gupta 378d1f317d8SVineet Gupta num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 379d1f317d8SVineet Gupta 3805a364c2aSVineet Gupta /* 3815a364c2aSVineet Gupta * For HS38 PAE40 configuration 3825a364c2aSVineet Gupta * - upper 8 bits of paddr need to be written into PTAG_HI 3835a364c2aSVineet Gupta * - (and needs to be written before the lower 32 bits) 3845a364c2aSVineet Gupta */ 3855a364c2aSVineet Gupta if (is_pae40_enabled()) { 3865a364c2aSVineet Gupta if (cacheop == OP_INV_IC) 3875a364c2aSVineet Gupta /* 3885a364c2aSVineet Gupta * Non aliasing I-cache in HS38, 3895a364c2aSVineet Gupta * aliasing I-cache handled in __cache_line_loop_v3() 3905a364c2aSVineet Gupta */ 3915a364c2aSVineet Gupta write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 3925a364c2aSVineet Gupta else 3935a364c2aSVineet Gupta write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32); 3945a364c2aSVineet Gupta } 3955a364c2aSVineet Gupta 396d1f317d8SVineet Gupta while (num_lines-- > 0) { 397d1f317d8SVineet Gupta write_aux_reg(aux_cmd, paddr); 398d1f317d8SVineet Gupta paddr += L1_CACHE_BYTES; 399d1f317d8SVineet Gupta } 400d1f317d8SVineet Gupta } 401d1f317d8SVineet Gupta 40211e14896SVineet Gupta #if (CONFIG_ARC_MMU_VER < 3) 40311e14896SVineet Gupta #define __cache_line_loop __cache_line_loop_v2 40411e14896SVineet Gupta #elif (CONFIG_ARC_MMU_VER == 3) 40511e14896SVineet Gupta #define __cache_line_loop __cache_line_loop_v3 406d1f317d8SVineet Gupta #elif (CONFIG_ARC_MMU_VER > 3) 407d1f317d8SVineet Gupta #define __cache_line_loop __cache_line_loop_v4 4088362c389SVineet Gupta #endif 4098362c389SVineet Gupta 4108362c389SVineet Gupta #ifdef CONFIG_ARC_HAS_DCACHE 4118362c389SVineet Gupta 4128362c389SVineet Gupta /*************************************************************** 4138362c389SVineet Gupta * Machine specific helpers for Entire D-Cache or Per Line ops 4148362c389SVineet Gupta */ 4158362c389SVineet Gupta 4166c310681SVineet Gupta static inline void __before_dc_op(const int op) 4178362c389SVineet Gupta { 4188362c389SVineet Gupta if (op == OP_FLUSH_N_INV) { 4198362c389SVineet Gupta /* Dcache provides 2 cmd: FLUSH or INV 4208362c389SVineet Gupta * INV inturn has sub-modes: DISCARD or FLUSH-BEFORE 4218362c389SVineet Gupta * flush-n-inv is achieved by INV cmd but with IM=1 4228362c389SVineet Gupta * So toggle INV sub-mode depending on op request and default 4238362c389SVineet Gupta */ 4246c310681SVineet Gupta const unsigned int ctl = ARC_REG_DC_CTRL; 4256c310681SVineet Gupta write_aux_reg(ctl, read_aux_reg(ctl) | DC_CTRL_INV_MODE_FLUSH); 4266c310681SVineet Gupta } 4278362c389SVineet Gupta } 4288362c389SVineet Gupta 4296c310681SVineet Gupta static inline void __after_dc_op(const int op) 4308362c389SVineet Gupta { 4316c310681SVineet Gupta if (op & OP_FLUSH) { 4326c310681SVineet Gupta const unsigned int ctl = ARC_REG_DC_CTRL; 4336c310681SVineet Gupta unsigned int reg; 4346c310681SVineet Gupta 4356c310681SVineet Gupta /* flush / flush-n-inv both wait */ 4366c310681SVineet Gupta while ((reg = read_aux_reg(ctl)) & DC_CTRL_FLUSH_STATUS) 4376c310681SVineet Gupta ; 4388362c389SVineet Gupta 4398362c389SVineet Gupta /* Switch back to default Invalidate mode */ 4408362c389SVineet Gupta if (op == OP_FLUSH_N_INV) 4416c310681SVineet Gupta write_aux_reg(ctl, reg & ~DC_CTRL_INV_MODE_FLUSH); 4426c310681SVineet Gupta } 4438362c389SVineet Gupta } 4448362c389SVineet Gupta 4458362c389SVineet Gupta /* 4468362c389SVineet Gupta * Operation on Entire D-Cache 4478ea2ddffSVineet Gupta * @op = {OP_INV, OP_FLUSH, OP_FLUSH_N_INV} 4488362c389SVineet Gupta * Note that constant propagation ensures all the checks are gone 4498362c389SVineet Gupta * in generated code 4508362c389SVineet Gupta */ 4518ea2ddffSVineet Gupta static inline void __dc_entire_op(const int op) 4528362c389SVineet Gupta { 4538362c389SVineet Gupta int aux; 4548362c389SVineet Gupta 4556c310681SVineet Gupta __before_dc_op(op); 4568362c389SVineet Gupta 4578ea2ddffSVineet Gupta if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 4588362c389SVineet Gupta aux = ARC_REG_DC_IVDC; 4598362c389SVineet Gupta else 4608362c389SVineet Gupta aux = ARC_REG_DC_FLSH; 4618362c389SVineet Gupta 4628362c389SVineet Gupta write_aux_reg(aux, 0x1); 4638362c389SVineet Gupta 4646c310681SVineet Gupta __after_dc_op(op); 4658362c389SVineet Gupta } 4668362c389SVineet Gupta 4678362c389SVineet Gupta /* For kernel mappings cache operation: index is same as paddr */ 4688362c389SVineet Gupta #define __dc_line_op_k(p, sz, op) __dc_line_op(p, p, sz, op) 4698362c389SVineet Gupta 4708362c389SVineet Gupta /* 4718ea2ddffSVineet Gupta * D-Cache Line ops: Per Line INV (discard or wback+discard) or FLUSH (wback) 4728362c389SVineet Gupta */ 47328b4af72SVineet Gupta static inline void __dc_line_op(phys_addr_t paddr, unsigned long vaddr, 4748ea2ddffSVineet Gupta unsigned long sz, const int op) 4758362c389SVineet Gupta { 4768362c389SVineet Gupta unsigned long flags; 4778362c389SVineet Gupta 4788362c389SVineet Gupta local_irq_save(flags); 4798362c389SVineet Gupta 4806c310681SVineet Gupta __before_dc_op(op); 4818362c389SVineet Gupta 4828ea2ddffSVineet Gupta __cache_line_loop(paddr, vaddr, sz, op); 4838362c389SVineet Gupta 4846c310681SVineet Gupta __after_dc_op(op); 4858362c389SVineet Gupta 4868362c389SVineet Gupta local_irq_restore(flags); 4878362c389SVineet Gupta } 4888362c389SVineet Gupta 4898362c389SVineet Gupta #else 4908362c389SVineet Gupta 4918ea2ddffSVineet Gupta #define __dc_entire_op(op) 4928ea2ddffSVineet Gupta #define __dc_line_op(paddr, vaddr, sz, op) 4938ea2ddffSVineet Gupta #define __dc_line_op_k(paddr, sz, op) 4948362c389SVineet Gupta 4958362c389SVineet Gupta #endif /* CONFIG_ARC_HAS_DCACHE */ 4968362c389SVineet Gupta 4978362c389SVineet Gupta #ifdef CONFIG_ARC_HAS_ICACHE 4988362c389SVineet Gupta 4998362c389SVineet Gupta static inline void __ic_entire_inv(void) 5008362c389SVineet Gupta { 5018362c389SVineet Gupta write_aux_reg(ARC_REG_IC_IVIC, 1); 5028362c389SVineet Gupta read_aux_reg(ARC_REG_IC_CTRL); /* blocks */ 5038362c389SVineet Gupta } 5048362c389SVineet Gupta 5058362c389SVineet Gupta static inline void 50628b4af72SVineet Gupta __ic_line_inv_vaddr_local(phys_addr_t paddr, unsigned long vaddr, 5078362c389SVineet Gupta unsigned long sz) 5088362c389SVineet Gupta { 5098362c389SVineet Gupta unsigned long flags; 5108362c389SVineet Gupta 5118362c389SVineet Gupta local_irq_save(flags); 512bcc4d65aSVineet Gupta (*_cache_line_loop_ic_fn)(paddr, vaddr, sz, OP_INV_IC); 5138362c389SVineet Gupta local_irq_restore(flags); 5148362c389SVineet Gupta } 5158362c389SVineet Gupta 5168362c389SVineet Gupta #ifndef CONFIG_SMP 5178362c389SVineet Gupta 5188362c389SVineet Gupta #define __ic_line_inv_vaddr(p, v, s) __ic_line_inv_vaddr_local(p, v, s) 5198362c389SVineet Gupta 5208362c389SVineet Gupta #else 5218362c389SVineet Gupta 5228362c389SVineet Gupta struct ic_inv_args { 52328b4af72SVineet Gupta phys_addr_t paddr, vaddr; 5248362c389SVineet Gupta int sz; 5258362c389SVineet Gupta }; 5268362c389SVineet Gupta 5278362c389SVineet Gupta static void __ic_line_inv_vaddr_helper(void *info) 5288362c389SVineet Gupta { 5298362c389SVineet Gupta struct ic_inv_args *ic_inv = info; 5308362c389SVineet Gupta 5318362c389SVineet Gupta __ic_line_inv_vaddr_local(ic_inv->paddr, ic_inv->vaddr, ic_inv->sz); 5328362c389SVineet Gupta } 5338362c389SVineet Gupta 53428b4af72SVineet Gupta static void __ic_line_inv_vaddr(phys_addr_t paddr, unsigned long vaddr, 5358362c389SVineet Gupta unsigned long sz) 5368362c389SVineet Gupta { 5378362c389SVineet Gupta struct ic_inv_args ic_inv = { 5388362c389SVineet Gupta .paddr = paddr, 5398362c389SVineet Gupta .vaddr = vaddr, 5408362c389SVineet Gupta .sz = sz 5418362c389SVineet Gupta }; 5428362c389SVineet Gupta 5438362c389SVineet Gupta on_each_cpu(__ic_line_inv_vaddr_helper, &ic_inv, 1); 5448362c389SVineet Gupta } 5458362c389SVineet Gupta 5468362c389SVineet Gupta #endif /* CONFIG_SMP */ 5478362c389SVineet Gupta 5488362c389SVineet Gupta #else /* !CONFIG_ARC_HAS_ICACHE */ 5498362c389SVineet Gupta 5508362c389SVineet Gupta #define __ic_entire_inv() 5518362c389SVineet Gupta #define __ic_line_inv_vaddr(pstart, vstart, sz) 5528362c389SVineet Gupta 5538362c389SVineet Gupta #endif /* CONFIG_ARC_HAS_ICACHE */ 5548362c389SVineet Gupta 55528b4af72SVineet Gupta noinline void slc_op(phys_addr_t paddr, unsigned long sz, const int op) 556795f4558SVineet Gupta { 557795f4558SVineet Gupta #ifdef CONFIG_ISA_ARCV2 558b607edddSAlexey Brodkin /* 559b607edddSAlexey Brodkin * SLC is shared between all cores and concurrent aux operations from 560b607edddSAlexey Brodkin * multiple cores need to be serialized using a spinlock 561b607edddSAlexey Brodkin * A concurrent operation can be silently ignored and/or the old/new 562b607edddSAlexey Brodkin * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop 563b607edddSAlexey Brodkin * below) 564b607edddSAlexey Brodkin */ 565b607edddSAlexey Brodkin static DEFINE_SPINLOCK(lock); 566795f4558SVineet Gupta unsigned long flags; 567795f4558SVineet Gupta unsigned int ctrl; 568795f4558SVineet Gupta 569b607edddSAlexey Brodkin spin_lock_irqsave(&lock, flags); 570795f4558SVineet Gupta 571795f4558SVineet Gupta /* 572795f4558SVineet Gupta * The Region Flush operation is specified by CTRL.RGN_OP[11..9] 573795f4558SVineet Gupta * - b'000 (default) is Flush, 574795f4558SVineet Gupta * - b'001 is Invalidate if CTRL.IM == 0 575795f4558SVineet Gupta * - b'001 is Flush-n-Invalidate if CTRL.IM == 1 576795f4558SVineet Gupta */ 577795f4558SVineet Gupta ctrl = read_aux_reg(ARC_REG_SLC_CTRL); 578795f4558SVineet Gupta 579795f4558SVineet Gupta /* Don't rely on default value of IM bit */ 580795f4558SVineet Gupta if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 581795f4558SVineet Gupta ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 582795f4558SVineet Gupta else 583795f4558SVineet Gupta ctrl |= SLC_CTRL_IM; 584795f4558SVineet Gupta 585795f4558SVineet Gupta if (op & OP_INV) 586795f4558SVineet Gupta ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */ 587795f4558SVineet Gupta else 588795f4558SVineet Gupta ctrl &= ~SLC_CTRL_RGN_OP_INV; 589795f4558SVineet Gupta 590795f4558SVineet Gupta write_aux_reg(ARC_REG_SLC_CTRL, ctrl); 591795f4558SVineet Gupta 592795f4558SVineet Gupta /* 593795f4558SVineet Gupta * Lower bits are ignored, no need to clip 594795f4558SVineet Gupta * END needs to be setup before START (latter triggers the operation) 595795f4558SVineet Gupta * END can't be same as START, so add (l2_line_sz - 1) to sz 596795f4558SVineet Gupta */ 597795f4558SVineet Gupta write_aux_reg(ARC_REG_SLC_RGN_END, (paddr + sz + l2_line_sz - 1)); 598795f4558SVineet Gupta write_aux_reg(ARC_REG_SLC_RGN_START, paddr); 599795f4558SVineet Gupta 600795f4558SVineet Gupta while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY); 601795f4558SVineet Gupta 602b607edddSAlexey Brodkin spin_unlock_irqrestore(&lock, flags); 603795f4558SVineet Gupta #endif 604795f4558SVineet Gupta } 605795f4558SVineet Gupta 6068362c389SVineet Gupta /*********************************************************** 6078362c389SVineet Gupta * Exported APIs 6088362c389SVineet Gupta */ 6098362c389SVineet Gupta 6108362c389SVineet Gupta /* 6118362c389SVineet Gupta * Handle cache congruency of kernel and userspace mappings of page when kernel 6128362c389SVineet Gupta * writes-to/reads-from 6138362c389SVineet Gupta * 6148362c389SVineet Gupta * The idea is to defer flushing of kernel mapping after a WRITE, possible if: 6158362c389SVineet Gupta * -dcache is NOT aliasing, hence any U/K-mappings of page are congruent 6168362c389SVineet Gupta * -U-mapping doesn't exist yet for page (finalised in update_mmu_cache) 6178362c389SVineet Gupta * -In SMP, if hardware caches are coherent 6188362c389SVineet Gupta * 6198362c389SVineet Gupta * There's a corollary case, where kernel READs from a userspace mapped page. 6208362c389SVineet Gupta * If the U-mapping is not congruent to to K-mapping, former needs flushing. 6218362c389SVineet Gupta */ 6228362c389SVineet Gupta void flush_dcache_page(struct page *page) 6238362c389SVineet Gupta { 6248362c389SVineet Gupta struct address_space *mapping; 6258362c389SVineet Gupta 6268362c389SVineet Gupta if (!cache_is_vipt_aliasing()) { 6278362c389SVineet Gupta clear_bit(PG_dc_clean, &page->flags); 6288362c389SVineet Gupta return; 6298362c389SVineet Gupta } 6308362c389SVineet Gupta 6318362c389SVineet Gupta /* don't handle anon pages here */ 6328362c389SVineet Gupta mapping = page_mapping(page); 6338362c389SVineet Gupta if (!mapping) 6348362c389SVineet Gupta return; 6358362c389SVineet Gupta 6368362c389SVineet Gupta /* 6378362c389SVineet Gupta * pagecache page, file not yet mapped to userspace 6388362c389SVineet Gupta * Make a note that K-mapping is dirty 6398362c389SVineet Gupta */ 6408362c389SVineet Gupta if (!mapping_mapped(mapping)) { 6418362c389SVineet Gupta clear_bit(PG_dc_clean, &page->flags); 642e1534ae9SKirill A. Shutemov } else if (page_mapcount(page)) { 6438362c389SVineet Gupta 6448362c389SVineet Gupta /* kernel reading from page with U-mapping */ 64528b4af72SVineet Gupta phys_addr_t paddr = (unsigned long)page_address(page); 64609cbfeafSKirill A. Shutemov unsigned long vaddr = page->index << PAGE_SHIFT; 6478362c389SVineet Gupta 6488362c389SVineet Gupta if (addr_not_cache_congruent(paddr, vaddr)) 6498362c389SVineet Gupta __flush_dcache_page(paddr, vaddr); 6508362c389SVineet Gupta } 6518362c389SVineet Gupta } 6528362c389SVineet Gupta EXPORT_SYMBOL(flush_dcache_page); 6538362c389SVineet Gupta 654f2b0b25aSAlexey Brodkin /* 655f2b0b25aSAlexey Brodkin * DMA ops for systems with L1 cache only 656f2b0b25aSAlexey Brodkin * Make memory coherent with L1 cache by flushing/invalidating L1 lines 657f2b0b25aSAlexey Brodkin */ 658f5db19e9SVineet Gupta static void __dma_cache_wback_inv_l1(phys_addr_t start, unsigned long sz) 6598362c389SVineet Gupta { 6608362c389SVineet Gupta __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 661f2b0b25aSAlexey Brodkin } 662795f4558SVineet Gupta 663f5db19e9SVineet Gupta static void __dma_cache_inv_l1(phys_addr_t start, unsigned long sz) 664f2b0b25aSAlexey Brodkin { 665f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_INV); 666f2b0b25aSAlexey Brodkin } 667f2b0b25aSAlexey Brodkin 668f5db19e9SVineet Gupta static void __dma_cache_wback_l1(phys_addr_t start, unsigned long sz) 669f2b0b25aSAlexey Brodkin { 670f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_FLUSH); 671f2b0b25aSAlexey Brodkin } 672f2b0b25aSAlexey Brodkin 673f2b0b25aSAlexey Brodkin /* 674f2b0b25aSAlexey Brodkin * DMA ops for systems with both L1 and L2 caches, but without IOC 6757423cc0cSAdam Buchbinder * Both L1 and L2 lines need to be explicitly flushed/invalidated 676f2b0b25aSAlexey Brodkin */ 677f5db19e9SVineet Gupta static void __dma_cache_wback_inv_slc(phys_addr_t start, unsigned long sz) 678f2b0b25aSAlexey Brodkin { 679f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 680795f4558SVineet Gupta slc_op(start, sz, OP_FLUSH_N_INV); 6818362c389SVineet Gupta } 682f2b0b25aSAlexey Brodkin 683f5db19e9SVineet Gupta static void __dma_cache_inv_slc(phys_addr_t start, unsigned long sz) 684f2b0b25aSAlexey Brodkin { 685f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_INV); 686f2b0b25aSAlexey Brodkin slc_op(start, sz, OP_INV); 687f2b0b25aSAlexey Brodkin } 688f2b0b25aSAlexey Brodkin 689f5db19e9SVineet Gupta static void __dma_cache_wback_slc(phys_addr_t start, unsigned long sz) 690f2b0b25aSAlexey Brodkin { 691f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_FLUSH); 692f2b0b25aSAlexey Brodkin slc_op(start, sz, OP_FLUSH); 693f2b0b25aSAlexey Brodkin } 694f2b0b25aSAlexey Brodkin 695f2b0b25aSAlexey Brodkin /* 696f2b0b25aSAlexey Brodkin * DMA ops for systems with IOC 697f2b0b25aSAlexey Brodkin * IOC hardware snoops all DMA traffic keeping the caches consistent with 698f2b0b25aSAlexey Brodkin * memory - eliding need for any explicit cache maintenance of DMA buffers 699f2b0b25aSAlexey Brodkin */ 700f5db19e9SVineet Gupta static void __dma_cache_wback_inv_ioc(phys_addr_t start, unsigned long sz) {} 701f5db19e9SVineet Gupta static void __dma_cache_inv_ioc(phys_addr_t start, unsigned long sz) {} 702f5db19e9SVineet Gupta static void __dma_cache_wback_ioc(phys_addr_t start, unsigned long sz) {} 703f2b0b25aSAlexey Brodkin 704f2b0b25aSAlexey Brodkin /* 705f2b0b25aSAlexey Brodkin * Exported DMA API 706f2b0b25aSAlexey Brodkin */ 707f5db19e9SVineet Gupta void dma_cache_wback_inv(phys_addr_t start, unsigned long sz) 708f2b0b25aSAlexey Brodkin { 709f2b0b25aSAlexey Brodkin __dma_cache_wback_inv(start, sz); 710f2b0b25aSAlexey Brodkin } 7118362c389SVineet Gupta EXPORT_SYMBOL(dma_cache_wback_inv); 7128362c389SVineet Gupta 713f5db19e9SVineet Gupta void dma_cache_inv(phys_addr_t start, unsigned long sz) 7148362c389SVineet Gupta { 715f2b0b25aSAlexey Brodkin __dma_cache_inv(start, sz); 7168362c389SVineet Gupta } 7178362c389SVineet Gupta EXPORT_SYMBOL(dma_cache_inv); 7188362c389SVineet Gupta 719f5db19e9SVineet Gupta void dma_cache_wback(phys_addr_t start, unsigned long sz) 7208362c389SVineet Gupta { 721f2b0b25aSAlexey Brodkin __dma_cache_wback(start, sz); 7228362c389SVineet Gupta } 7238362c389SVineet Gupta EXPORT_SYMBOL(dma_cache_wback); 7248362c389SVineet Gupta 7258362c389SVineet Gupta /* 7268362c389SVineet Gupta * This is API for making I/D Caches consistent when modifying 7278362c389SVineet Gupta * kernel code (loadable modules, kprobes, kgdb...) 7288362c389SVineet Gupta * This is called on insmod, with kernel virtual address for CODE of 7298362c389SVineet Gupta * the module. ARC cache maintenance ops require PHY address thus we 7308362c389SVineet Gupta * need to convert vmalloc addr to PHY addr 7318362c389SVineet Gupta */ 7328362c389SVineet Gupta void flush_icache_range(unsigned long kstart, unsigned long kend) 7338362c389SVineet Gupta { 7348362c389SVineet Gupta unsigned int tot_sz; 7358362c389SVineet Gupta 7368362c389SVineet Gupta WARN(kstart < TASK_SIZE, "%s() can't handle user vaddr", __func__); 7378362c389SVineet Gupta 7388362c389SVineet Gupta /* Shortcut for bigger flush ranges. 7398362c389SVineet Gupta * Here we don't care if this was kernel virtual or phy addr 7408362c389SVineet Gupta */ 7418362c389SVineet Gupta tot_sz = kend - kstart; 7428362c389SVineet Gupta if (tot_sz > PAGE_SIZE) { 7438362c389SVineet Gupta flush_cache_all(); 7448362c389SVineet Gupta return; 7458362c389SVineet Gupta } 7468362c389SVineet Gupta 7478362c389SVineet Gupta /* Case: Kernel Phy addr (0x8000_0000 onwards) */ 7488362c389SVineet Gupta if (likely(kstart > PAGE_OFFSET)) { 7498362c389SVineet Gupta /* 7508362c389SVineet Gupta * The 2nd arg despite being paddr will be used to index icache 7518362c389SVineet Gupta * This is OK since no alternate virtual mappings will exist 7528362c389SVineet Gupta * given the callers for this case: kprobe/kgdb in built-in 7538362c389SVineet Gupta * kernel code only. 7548362c389SVineet Gupta */ 7558362c389SVineet Gupta __sync_icache_dcache(kstart, kstart, kend - kstart); 7568362c389SVineet Gupta return; 7578362c389SVineet Gupta } 7588362c389SVineet Gupta 7598362c389SVineet Gupta /* 7608362c389SVineet Gupta * Case: Kernel Vaddr (0x7000_0000 to 0x7fff_ffff) 7618362c389SVineet Gupta * (1) ARC Cache Maintenance ops only take Phy addr, hence special 7628362c389SVineet Gupta * handling of kernel vaddr. 7638362c389SVineet Gupta * 7648362c389SVineet Gupta * (2) Despite @tot_sz being < PAGE_SIZE (bigger cases handled already), 7658362c389SVineet Gupta * it still needs to handle a 2 page scenario, where the range 7668362c389SVineet Gupta * straddles across 2 virtual pages and hence need for loop 7678362c389SVineet Gupta */ 7688362c389SVineet Gupta while (tot_sz > 0) { 7698362c389SVineet Gupta unsigned int off, sz; 7708362c389SVineet Gupta unsigned long phy, pfn; 7718362c389SVineet Gupta 7728362c389SVineet Gupta off = kstart % PAGE_SIZE; 7738362c389SVineet Gupta pfn = vmalloc_to_pfn((void *)kstart); 7748362c389SVineet Gupta phy = (pfn << PAGE_SHIFT) + off; 7758362c389SVineet Gupta sz = min_t(unsigned int, tot_sz, PAGE_SIZE - off); 7768362c389SVineet Gupta __sync_icache_dcache(phy, kstart, sz); 7778362c389SVineet Gupta kstart += sz; 7788362c389SVineet Gupta tot_sz -= sz; 7798362c389SVineet Gupta } 7808362c389SVineet Gupta } 7818362c389SVineet Gupta EXPORT_SYMBOL(flush_icache_range); 7828362c389SVineet Gupta 7838362c389SVineet Gupta /* 7848362c389SVineet Gupta * General purpose helper to make I and D cache lines consistent. 7858362c389SVineet Gupta * @paddr is phy addr of region 7868362c389SVineet Gupta * @vaddr is typically user vaddr (breakpoint) or kernel vaddr (vmalloc) 7878362c389SVineet Gupta * However in one instance, when called by kprobe (for a breakpt in 7888362c389SVineet Gupta * builtin kernel code) @vaddr will be paddr only, meaning CDU operation will 7898362c389SVineet Gupta * use a paddr to index the cache (despite VIPT). This is fine since since a 7908362c389SVineet Gupta * builtin kernel page will not have any virtual mappings. 7918362c389SVineet Gupta * kprobe on loadable module will be kernel vaddr. 7928362c389SVineet Gupta */ 79328b4af72SVineet Gupta void __sync_icache_dcache(phys_addr_t paddr, unsigned long vaddr, int len) 7948362c389SVineet Gupta { 7958362c389SVineet Gupta __dc_line_op(paddr, vaddr, len, OP_FLUSH_N_INV); 7968362c389SVineet Gupta __ic_line_inv_vaddr(paddr, vaddr, len); 7978362c389SVineet Gupta } 7988362c389SVineet Gupta 7998362c389SVineet Gupta /* wrapper to compile time eliminate alignment checks in flush loop */ 80028b4af72SVineet Gupta void __inv_icache_page(phys_addr_t paddr, unsigned long vaddr) 8018362c389SVineet Gupta { 8028362c389SVineet Gupta __ic_line_inv_vaddr(paddr, vaddr, PAGE_SIZE); 8038362c389SVineet Gupta } 8048362c389SVineet Gupta 8058362c389SVineet Gupta /* 8068362c389SVineet Gupta * wrapper to clearout kernel or userspace mappings of a page 8078362c389SVineet Gupta * For kernel mappings @vaddr == @paddr 8088362c389SVineet Gupta */ 80928b4af72SVineet Gupta void __flush_dcache_page(phys_addr_t paddr, unsigned long vaddr) 8108362c389SVineet Gupta { 8118362c389SVineet Gupta __dc_line_op(paddr, vaddr & PAGE_MASK, PAGE_SIZE, OP_FLUSH_N_INV); 8128362c389SVineet Gupta } 8138362c389SVineet Gupta 8148362c389SVineet Gupta noinline void flush_cache_all(void) 8158362c389SVineet Gupta { 8168362c389SVineet Gupta unsigned long flags; 8178362c389SVineet Gupta 8188362c389SVineet Gupta local_irq_save(flags); 8198362c389SVineet Gupta 8208362c389SVineet Gupta __ic_entire_inv(); 8218362c389SVineet Gupta __dc_entire_op(OP_FLUSH_N_INV); 8228362c389SVineet Gupta 8238362c389SVineet Gupta local_irq_restore(flags); 8248362c389SVineet Gupta 8258362c389SVineet Gupta } 8268362c389SVineet Gupta 8278362c389SVineet Gupta #ifdef CONFIG_ARC_CACHE_VIPT_ALIASING 8288362c389SVineet Gupta 8298362c389SVineet Gupta void flush_cache_mm(struct mm_struct *mm) 8308362c389SVineet Gupta { 8318362c389SVineet Gupta flush_cache_all(); 8328362c389SVineet Gupta } 8338362c389SVineet Gupta 8348362c389SVineet Gupta void flush_cache_page(struct vm_area_struct *vma, unsigned long u_vaddr, 8358362c389SVineet Gupta unsigned long pfn) 8368362c389SVineet Gupta { 8378362c389SVineet Gupta unsigned int paddr = pfn << PAGE_SHIFT; 8388362c389SVineet Gupta 8398362c389SVineet Gupta u_vaddr &= PAGE_MASK; 8408362c389SVineet Gupta 8418362c389SVineet Gupta __flush_dcache_page(paddr, u_vaddr); 8428362c389SVineet Gupta 8438362c389SVineet Gupta if (vma->vm_flags & VM_EXEC) 8448362c389SVineet Gupta __inv_icache_page(paddr, u_vaddr); 8458362c389SVineet Gupta } 8468362c389SVineet Gupta 8478362c389SVineet Gupta void flush_cache_range(struct vm_area_struct *vma, unsigned long start, 8488362c389SVineet Gupta unsigned long end) 8498362c389SVineet Gupta { 8508362c389SVineet Gupta flush_cache_all(); 8518362c389SVineet Gupta } 8528362c389SVineet Gupta 8538362c389SVineet Gupta void flush_anon_page(struct vm_area_struct *vma, struct page *page, 8548362c389SVineet Gupta unsigned long u_vaddr) 8558362c389SVineet Gupta { 8568362c389SVineet Gupta /* TBD: do we really need to clear the kernel mapping */ 8578362c389SVineet Gupta __flush_dcache_page(page_address(page), u_vaddr); 8588362c389SVineet Gupta __flush_dcache_page(page_address(page), page_address(page)); 8598362c389SVineet Gupta 8608362c389SVineet Gupta } 8618362c389SVineet Gupta 8628362c389SVineet Gupta #endif 8638362c389SVineet Gupta 8648362c389SVineet Gupta void copy_user_highpage(struct page *to, struct page *from, 8658362c389SVineet Gupta unsigned long u_vaddr, struct vm_area_struct *vma) 8668362c389SVineet Gupta { 867336e2136SVineet Gupta void *kfrom = kmap_atomic(from); 868336e2136SVineet Gupta void *kto = kmap_atomic(to); 8698362c389SVineet Gupta int clean_src_k_mappings = 0; 8708362c389SVineet Gupta 8718362c389SVineet Gupta /* 8728362c389SVineet Gupta * If SRC page was already mapped in userspace AND it's U-mapping is 8738362c389SVineet Gupta * not congruent with K-mapping, sync former to physical page so that 8748362c389SVineet Gupta * K-mapping in memcpy below, sees the right data 8758362c389SVineet Gupta * 8768362c389SVineet Gupta * Note that while @u_vaddr refers to DST page's userspace vaddr, it is 8778362c389SVineet Gupta * equally valid for SRC page as well 878336e2136SVineet Gupta * 879336e2136SVineet Gupta * For !VIPT cache, all of this gets compiled out as 880336e2136SVineet Gupta * addr_not_cache_congruent() is 0 8818362c389SVineet Gupta */ 882e1534ae9SKirill A. Shutemov if (page_mapcount(from) && addr_not_cache_congruent(kfrom, u_vaddr)) { 883336e2136SVineet Gupta __flush_dcache_page((unsigned long)kfrom, u_vaddr); 8848362c389SVineet Gupta clean_src_k_mappings = 1; 8858362c389SVineet Gupta } 8868362c389SVineet Gupta 887336e2136SVineet Gupta copy_page(kto, kfrom); 8888362c389SVineet Gupta 8898362c389SVineet Gupta /* 8908362c389SVineet Gupta * Mark DST page K-mapping as dirty for a later finalization by 8918362c389SVineet Gupta * update_mmu_cache(). Although the finalization could have been done 8928362c389SVineet Gupta * here as well (given that both vaddr/paddr are available). 8938362c389SVineet Gupta * But update_mmu_cache() already has code to do that for other 8948362c389SVineet Gupta * non copied user pages (e.g. read faults which wire in pagecache page 8958362c389SVineet Gupta * directly). 8968362c389SVineet Gupta */ 8978362c389SVineet Gupta clear_bit(PG_dc_clean, &to->flags); 8988362c389SVineet Gupta 8998362c389SVineet Gupta /* 9008362c389SVineet Gupta * if SRC was already usermapped and non-congruent to kernel mapping 9018362c389SVineet Gupta * sync the kernel mapping back to physical page 9028362c389SVineet Gupta */ 9038362c389SVineet Gupta if (clean_src_k_mappings) { 904336e2136SVineet Gupta __flush_dcache_page((unsigned long)kfrom, (unsigned long)kfrom); 9058362c389SVineet Gupta set_bit(PG_dc_clean, &from->flags); 9068362c389SVineet Gupta } else { 9078362c389SVineet Gupta clear_bit(PG_dc_clean, &from->flags); 9088362c389SVineet Gupta } 909336e2136SVineet Gupta 910336e2136SVineet Gupta kunmap_atomic(kto); 911336e2136SVineet Gupta kunmap_atomic(kfrom); 9128362c389SVineet Gupta } 9138362c389SVineet Gupta 9148362c389SVineet Gupta void clear_user_page(void *to, unsigned long u_vaddr, struct page *page) 9158362c389SVineet Gupta { 9168362c389SVineet Gupta clear_page(to); 9178362c389SVineet Gupta clear_bit(PG_dc_clean, &page->flags); 9188362c389SVineet Gupta } 9198362c389SVineet Gupta 9208362c389SVineet Gupta 9218362c389SVineet Gupta /********************************************************************** 9228362c389SVineet Gupta * Explicit Cache flush request from user space via syscall 9238362c389SVineet Gupta * Needed for JITs which generate code on the fly 9248362c389SVineet Gupta */ 9258362c389SVineet Gupta SYSCALL_DEFINE3(cacheflush, uint32_t, start, uint32_t, sz, uint32_t, flags) 9268362c389SVineet Gupta { 9278362c389SVineet Gupta /* TBD: optimize this */ 9288362c389SVineet Gupta flush_cache_all(); 9298362c389SVineet Gupta return 0; 9308362c389SVineet Gupta } 9318ea2ddffSVineet Gupta 9328ea2ddffSVineet Gupta void arc_cache_init(void) 9338ea2ddffSVineet Gupta { 9348ea2ddffSVineet Gupta unsigned int __maybe_unused cpu = smp_processor_id(); 9358ea2ddffSVineet Gupta char str[256]; 9368ea2ddffSVineet Gupta 9378ea2ddffSVineet Gupta printk(arc_cache_mumbojumbo(0, str, sizeof(str))); 9388ea2ddffSVineet Gupta 93945c3b08aSVineet Gupta /* 94045c3b08aSVineet Gupta * Only master CPU needs to execute rest of function: 94145c3b08aSVineet Gupta * - Assume SMP so all cores will have same cache config so 94245c3b08aSVineet Gupta * any geomtry checks will be same for all 94345c3b08aSVineet Gupta * - IOC setup / dma callbacks only need to be setup once 94445c3b08aSVineet Gupta */ 94545c3b08aSVineet Gupta if (cpu) 94645c3b08aSVineet Gupta return; 94745c3b08aSVineet Gupta 9488ea2ddffSVineet Gupta if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) { 9498ea2ddffSVineet Gupta struct cpuinfo_arc_cache *ic = &cpuinfo_arc700[cpu].icache; 9508ea2ddffSVineet Gupta 9518ea2ddffSVineet Gupta if (!ic->ver) 9528ea2ddffSVineet Gupta panic("cache support enabled but non-existent cache\n"); 9538ea2ddffSVineet Gupta 9548ea2ddffSVineet Gupta if (ic->line_len != L1_CACHE_BYTES) 9558ea2ddffSVineet Gupta panic("ICache line [%d] != kernel Config [%d]", 9568ea2ddffSVineet Gupta ic->line_len, L1_CACHE_BYTES); 9578ea2ddffSVineet Gupta 9588ea2ddffSVineet Gupta if (ic->ver != CONFIG_ARC_MMU_VER) 9598ea2ddffSVineet Gupta panic("Cache ver [%d] doesn't match MMU ver [%d]\n", 9608ea2ddffSVineet Gupta ic->ver, CONFIG_ARC_MMU_VER); 961bcc4d65aSVineet Gupta 962bcc4d65aSVineet Gupta /* 9632547476aSAndrea Gelmini * In MMU v4 (HS38x) the aliasing icache config uses IVIL/PTAG 964bcc4d65aSVineet Gupta * pair to provide vaddr/paddr respectively, just as in MMU v3 965bcc4d65aSVineet Gupta */ 966bcc4d65aSVineet Gupta if (is_isa_arcv2() && ic->alias) 967bcc4d65aSVineet Gupta _cache_line_loop_ic_fn = __cache_line_loop_v3; 968bcc4d65aSVineet Gupta else 969bcc4d65aSVineet Gupta _cache_line_loop_ic_fn = __cache_line_loop; 9708ea2ddffSVineet Gupta } 9718ea2ddffSVineet Gupta 9728ea2ddffSVineet Gupta if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) { 9738ea2ddffSVineet Gupta struct cpuinfo_arc_cache *dc = &cpuinfo_arc700[cpu].dcache; 9748ea2ddffSVineet Gupta 9758ea2ddffSVineet Gupta if (!dc->ver) 9768ea2ddffSVineet Gupta panic("cache support enabled but non-existent cache\n"); 9778ea2ddffSVineet Gupta 9788ea2ddffSVineet Gupta if (dc->line_len != L1_CACHE_BYTES) 9798ea2ddffSVineet Gupta panic("DCache line [%d] != kernel Config [%d]", 9808ea2ddffSVineet Gupta dc->line_len, L1_CACHE_BYTES); 9818ea2ddffSVineet Gupta 982d1f317d8SVineet Gupta /* check for D-Cache aliasing on ARCompact: ARCv2 has PIPT */ 983d1f317d8SVineet Gupta if (is_isa_arcompact()) { 984d1f317d8SVineet Gupta int handled = IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING); 9858ea2ddffSVineet Gupta 9868ea2ddffSVineet Gupta if (dc->alias && !handled) 9878ea2ddffSVineet Gupta panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 9888ea2ddffSVineet Gupta else if (!dc->alias && handled) 9898ea2ddffSVineet Gupta panic("Disable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 9908ea2ddffSVineet Gupta } 9918ea2ddffSVineet Gupta } 992f2b0b25aSAlexey Brodkin 99379335a2cSVineet Gupta if (is_isa_arcv2() && l2_line_sz && !slc_enable) { 99479335a2cSVineet Gupta 99579335a2cSVineet Gupta /* IM set : flush before invalidate */ 99679335a2cSVineet Gupta write_aux_reg(ARC_REG_SLC_CTRL, 99779335a2cSVineet Gupta read_aux_reg(ARC_REG_SLC_CTRL) | SLC_CTRL_IM); 99879335a2cSVineet Gupta 99979335a2cSVineet Gupta write_aux_reg(ARC_REG_SLC_INVALIDATE, 1); 100079335a2cSVineet Gupta 100179335a2cSVineet Gupta /* Important to wait for flush to complete */ 100279335a2cSVineet Gupta while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY); 100379335a2cSVineet Gupta write_aux_reg(ARC_REG_SLC_CTRL, 100479335a2cSVineet Gupta read_aux_reg(ARC_REG_SLC_CTRL) | SLC_CTRL_DISABLE); 100579335a2cSVineet Gupta } 100679335a2cSVineet Gupta 1007cf986d47SVineet Gupta if (is_isa_arcv2() && ioc_enable) { 1008f2b0b25aSAlexey Brodkin /* IO coherency base - 0x8z */ 1009f2b0b25aSAlexey Brodkin write_aux_reg(ARC_REG_IO_COH_AP0_BASE, 0x80000); 1010f2b0b25aSAlexey Brodkin /* IO coherency aperture size - 512Mb: 0x8z-0xAz */ 1011f2b0b25aSAlexey Brodkin write_aux_reg(ARC_REG_IO_COH_AP0_SIZE, 0x11); 1012f2b0b25aSAlexey Brodkin /* Enable partial writes */ 1013f2b0b25aSAlexey Brodkin write_aux_reg(ARC_REG_IO_COH_PARTIAL, 1); 1014f2b0b25aSAlexey Brodkin /* Enable IO coherency */ 1015f2b0b25aSAlexey Brodkin write_aux_reg(ARC_REG_IO_COH_ENABLE, 1); 1016f2b0b25aSAlexey Brodkin 1017f2b0b25aSAlexey Brodkin __dma_cache_wback_inv = __dma_cache_wback_inv_ioc; 1018f2b0b25aSAlexey Brodkin __dma_cache_inv = __dma_cache_inv_ioc; 1019f2b0b25aSAlexey Brodkin __dma_cache_wback = __dma_cache_wback_ioc; 102079335a2cSVineet Gupta } else if (is_isa_arcv2() && l2_line_sz && slc_enable) { 1021f2b0b25aSAlexey Brodkin __dma_cache_wback_inv = __dma_cache_wback_inv_slc; 1022f2b0b25aSAlexey Brodkin __dma_cache_inv = __dma_cache_inv_slc; 1023f2b0b25aSAlexey Brodkin __dma_cache_wback = __dma_cache_wback_slc; 1024f2b0b25aSAlexey Brodkin } else { 1025f2b0b25aSAlexey Brodkin __dma_cache_wback_inv = __dma_cache_wback_inv_l1; 1026f2b0b25aSAlexey Brodkin __dma_cache_inv = __dma_cache_inv_l1; 1027f2b0b25aSAlexey Brodkin __dma_cache_wback = __dma_cache_wback_l1; 1028f2b0b25aSAlexey Brodkin } 1029d1f317d8SVineet Gupta } 1030