18362c389SVineet Gupta /* 28ea2ddffSVineet Gupta * ARC Cache Management 38362c389SVineet Gupta * 48ea2ddffSVineet Gupta * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com) 58362c389SVineet Gupta * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 68362c389SVineet Gupta * 78362c389SVineet Gupta * This program is free software; you can redistribute it and/or modify 88362c389SVineet Gupta * it under the terms of the GNU General Public License version 2 as 98362c389SVineet Gupta * published by the Free Software Foundation. 108362c389SVineet Gupta */ 118362c389SVineet Gupta 128362c389SVineet Gupta #include <linux/module.h> 138362c389SVineet Gupta #include <linux/mm.h> 148362c389SVineet Gupta #include <linux/sched.h> 158362c389SVineet Gupta #include <linux/cache.h> 168362c389SVineet Gupta #include <linux/mmu_context.h> 178362c389SVineet Gupta #include <linux/syscalls.h> 188362c389SVineet Gupta #include <linux/uaccess.h> 198362c389SVineet Gupta #include <linux/pagemap.h> 208362c389SVineet Gupta #include <asm/cacheflush.h> 218362c389SVineet Gupta #include <asm/cachectl.h> 228362c389SVineet Gupta #include <asm/setup.h> 238362c389SVineet Gupta 240d77117fSVineet Gupta #ifdef CONFIG_ISA_ARCV2 250d77117fSVineet Gupta #define USE_RGN_FLSH 1 260d77117fSVineet Gupta #endif 270d77117fSVineet Gupta 28795f4558SVineet Gupta static int l2_line_sz; 29cf986d47SVineet Gupta static int ioc_exists; 30d0e73e2aSVineet Gupta int slc_enable = 1, ioc_enable = 1; 31deaf7565SVineet Gupta unsigned long perip_base = ARC_UNCACHED_ADDR_SPACE; /* legacy value for boot */ 3226c01c49SVineet Gupta unsigned long perip_end = 0xFFFFFFFF; /* legacy value */ 33795f4558SVineet Gupta 3428b4af72SVineet Gupta void (*_cache_line_loop_ic_fn)(phys_addr_t paddr, unsigned long vaddr, 357d3d162bSVineet Gupta unsigned long sz, const int op, const int full_page); 36bcc4d65aSVineet Gupta 37f5db19e9SVineet Gupta void (*__dma_cache_wback_inv)(phys_addr_t start, unsigned long sz); 38f5db19e9SVineet Gupta void (*__dma_cache_inv)(phys_addr_t start, unsigned long sz); 39f5db19e9SVineet Gupta void (*__dma_cache_wback)(phys_addr_t start, unsigned long sz); 40f2b0b25aSAlexey Brodkin 418362c389SVineet Gupta char *arc_cache_mumbojumbo(int c, char *buf, int len) 428362c389SVineet Gupta { 438362c389SVineet Gupta int n = 0; 44d1f317d8SVineet Gupta struct cpuinfo_arc_cache *p; 458362c389SVineet Gupta 468362c389SVineet Gupta #define PR_CACHE(p, cfg, str) \ 47f64915beSVineet Gupta if (!(p)->line_len) \ 488362c389SVineet Gupta n += scnprintf(buf + n, len - n, str"\t\t: N/A\n"); \ 498362c389SVineet Gupta else \ 508362c389SVineet Gupta n += scnprintf(buf + n, len - n, \ 518362c389SVineet Gupta str"\t\t: %uK, %dway/set, %uB Line, %s%s%s\n", \ 528362c389SVineet Gupta (p)->sz_k, (p)->assoc, (p)->line_len, \ 538362c389SVineet Gupta (p)->vipt ? "VIPT" : "PIPT", \ 548362c389SVineet Gupta (p)->alias ? " aliasing" : "", \ 55964cf28fSVineet Gupta IS_USED_CFG(cfg)); 568362c389SVineet Gupta 578362c389SVineet Gupta PR_CACHE(&cpuinfo_arc700[c].icache, CONFIG_ARC_HAS_ICACHE, "I-Cache"); 588362c389SVineet Gupta PR_CACHE(&cpuinfo_arc700[c].dcache, CONFIG_ARC_HAS_DCACHE, "D-Cache"); 598362c389SVineet Gupta 60d1f317d8SVineet Gupta p = &cpuinfo_arc700[c].slc; 61f64915beSVineet Gupta if (p->line_len) 62d1f317d8SVineet Gupta n += scnprintf(buf + n, len - n, 6379335a2cSVineet Gupta "SLC\t\t: %uK, %uB Line%s\n", 6479335a2cSVineet Gupta p->sz_k, p->line_len, IS_USED_RUN(slc_enable)); 65d1f317d8SVineet Gupta 66711c1f26SVineet Gupta n += scnprintf(buf + n, len - n, "Peripherals\t: %#lx%s%s\n", 67711c1f26SVineet Gupta perip_base, 68711c1f26SVineet Gupta IS_AVAIL3(ioc_exists, ioc_enable, ", IO-Coherency ")); 69f2b0b25aSAlexey Brodkin 708362c389SVineet Gupta return buf; 718362c389SVineet Gupta } 728362c389SVineet Gupta 738362c389SVineet Gupta /* 748362c389SVineet Gupta * Read the Cache Build Confuration Registers, Decode them and save into 758362c389SVineet Gupta * the cpuinfo structure for later use. 768362c389SVineet Gupta * No Validation done here, simply read/convert the BCRs 778362c389SVineet Gupta */ 78fd0881a2SVineet Gupta static void read_decode_cache_bcr_arcv2(int cpu) 798362c389SVineet Gupta { 80fd0881a2SVineet Gupta struct cpuinfo_arc_cache *p_slc = &cpuinfo_arc700[cpu].slc; 81d1f317d8SVineet Gupta struct bcr_generic sbcr; 82d1f317d8SVineet Gupta 83d1f317d8SVineet Gupta struct bcr_slc_cfg { 84d1f317d8SVineet Gupta #ifdef CONFIG_CPU_BIG_ENDIAN 85d1f317d8SVineet Gupta unsigned int pad:24, way:2, lsz:2, sz:4; 86d1f317d8SVineet Gupta #else 87d1f317d8SVineet Gupta unsigned int sz:4, lsz:2, way:2, pad:24; 88d1f317d8SVineet Gupta #endif 89d1f317d8SVineet Gupta } slc_cfg; 90d1f317d8SVineet Gupta 91f2b0b25aSAlexey Brodkin struct bcr_clust_cfg { 92f2b0b25aSAlexey Brodkin #ifdef CONFIG_CPU_BIG_ENDIAN 93f2b0b25aSAlexey Brodkin unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8; 94f2b0b25aSAlexey Brodkin #else 95f2b0b25aSAlexey Brodkin unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7; 96f2b0b25aSAlexey Brodkin #endif 97f2b0b25aSAlexey Brodkin } cbcr; 98f2b0b25aSAlexey Brodkin 9926c01c49SVineet Gupta struct bcr_volatile { 10026c01c49SVineet Gupta #ifdef CONFIG_CPU_BIG_ENDIAN 10126c01c49SVineet Gupta unsigned int start:4, limit:4, pad:22, order:1, disable:1; 10226c01c49SVineet Gupta #else 10326c01c49SVineet Gupta unsigned int disable:1, order:1, pad:22, limit:4, start:4; 10426c01c49SVineet Gupta #endif 10526c01c49SVineet Gupta } vol; 10626c01c49SVineet Gupta 10726c01c49SVineet Gupta 108fd0881a2SVineet Gupta READ_BCR(ARC_REG_SLC_BCR, sbcr); 109fd0881a2SVineet Gupta if (sbcr.ver) { 110fd0881a2SVineet Gupta READ_BCR(ARC_REG_SLC_CFG, slc_cfg); 111fd0881a2SVineet Gupta p_slc->sz_k = 128 << slc_cfg.sz; 112fd0881a2SVineet Gupta l2_line_sz = p_slc->line_len = (slc_cfg.lsz == 0) ? 128 : 64; 113fd0881a2SVineet Gupta } 114fd0881a2SVineet Gupta 115fd0881a2SVineet Gupta READ_BCR(ARC_REG_CLUSTER_BCR, cbcr); 116cf986d47SVineet Gupta if (cbcr.c) 117fd0881a2SVineet Gupta ioc_exists = 1; 118cf986d47SVineet Gupta else 119cf986d47SVineet Gupta ioc_enable = 0; 120deaf7565SVineet Gupta 12126c01c49SVineet Gupta /* HS 2.0 didn't have AUX_VOL */ 12226c01c49SVineet Gupta if (cpuinfo_arc700[cpu].core.family > 0x51) { 12326c01c49SVineet Gupta READ_BCR(AUX_VOL, vol); 12426c01c49SVineet Gupta perip_base = vol.start << 28; 12526c01c49SVineet Gupta /* HS 3.0 has limit and strict-ordering fields */ 12626c01c49SVineet Gupta if (cpuinfo_arc700[cpu].core.family > 0x52) 12726c01c49SVineet Gupta perip_end = (vol.limit << 28) - 1; 12826c01c49SVineet Gupta } 129fd0881a2SVineet Gupta } 130fd0881a2SVineet Gupta 131fd0881a2SVineet Gupta void read_decode_cache_bcr(void) 132fd0881a2SVineet Gupta { 133fd0881a2SVineet Gupta struct cpuinfo_arc_cache *p_ic, *p_dc; 134fd0881a2SVineet Gupta unsigned int cpu = smp_processor_id(); 135fd0881a2SVineet Gupta struct bcr_cache { 136fd0881a2SVineet Gupta #ifdef CONFIG_CPU_BIG_ENDIAN 137fd0881a2SVineet Gupta unsigned int pad:12, line_len:4, sz:4, config:4, ver:8; 138fd0881a2SVineet Gupta #else 139fd0881a2SVineet Gupta unsigned int ver:8, config:4, sz:4, line_len:4, pad:12; 140fd0881a2SVineet Gupta #endif 141fd0881a2SVineet Gupta } ibcr, dbcr; 142fd0881a2SVineet Gupta 1438362c389SVineet Gupta p_ic = &cpuinfo_arc700[cpu].icache; 1448362c389SVineet Gupta READ_BCR(ARC_REG_IC_BCR, ibcr); 1458362c389SVineet Gupta 1468362c389SVineet Gupta if (!ibcr.ver) 1478362c389SVineet Gupta goto dc_chk; 1488362c389SVineet Gupta 149d1f317d8SVineet Gupta if (ibcr.ver <= 3) { 1508362c389SVineet Gupta BUG_ON(ibcr.config != 3); 1518362c389SVineet Gupta p_ic->assoc = 2; /* Fixed to 2w set assoc */ 152d1f317d8SVineet Gupta } else if (ibcr.ver >= 4) { 153d1f317d8SVineet Gupta p_ic->assoc = 1 << ibcr.config; /* 1,2,4,8 */ 154d1f317d8SVineet Gupta } 155d1f317d8SVineet Gupta 1568362c389SVineet Gupta p_ic->line_len = 8 << ibcr.line_len; 1578362c389SVineet Gupta p_ic->sz_k = 1 << (ibcr.sz - 1); 1588362c389SVineet Gupta p_ic->vipt = 1; 1598362c389SVineet Gupta p_ic->alias = p_ic->sz_k/p_ic->assoc/TO_KB(PAGE_SIZE) > 1; 1608362c389SVineet Gupta 1618362c389SVineet Gupta dc_chk: 1628362c389SVineet Gupta p_dc = &cpuinfo_arc700[cpu].dcache; 1638362c389SVineet Gupta READ_BCR(ARC_REG_DC_BCR, dbcr); 1648362c389SVineet Gupta 1658362c389SVineet Gupta if (!dbcr.ver) 166d1f317d8SVineet Gupta goto slc_chk; 1678362c389SVineet Gupta 168d1f317d8SVineet Gupta if (dbcr.ver <= 3) { 1698362c389SVineet Gupta BUG_ON(dbcr.config != 2); 1708362c389SVineet Gupta p_dc->assoc = 4; /* Fixed to 4w set assoc */ 171d1f317d8SVineet Gupta p_dc->vipt = 1; 172d1f317d8SVineet Gupta p_dc->alias = p_dc->sz_k/p_dc->assoc/TO_KB(PAGE_SIZE) > 1; 173d1f317d8SVineet Gupta } else if (dbcr.ver >= 4) { 174d1f317d8SVineet Gupta p_dc->assoc = 1 << dbcr.config; /* 1,2,4,8 */ 175d1f317d8SVineet Gupta p_dc->vipt = 0; 176d1f317d8SVineet Gupta p_dc->alias = 0; /* PIPT so can't VIPT alias */ 177d1f317d8SVineet Gupta } 178d1f317d8SVineet Gupta 1798362c389SVineet Gupta p_dc->line_len = 16 << dbcr.line_len; 1808362c389SVineet Gupta p_dc->sz_k = 1 << (dbcr.sz - 1); 181d1f317d8SVineet Gupta 182d1f317d8SVineet Gupta slc_chk: 183fd0881a2SVineet Gupta if (is_isa_arcv2()) 184fd0881a2SVineet Gupta read_decode_cache_bcr_arcv2(cpu); 1858362c389SVineet Gupta } 1868362c389SVineet Gupta 1878362c389SVineet Gupta /* 1888ea2ddffSVineet Gupta * Line Operation on {I,D}-Cache 1898362c389SVineet Gupta */ 1908362c389SVineet Gupta 1918362c389SVineet Gupta #define OP_INV 0x1 1928362c389SVineet Gupta #define OP_FLUSH 0x2 1938362c389SVineet Gupta #define OP_FLUSH_N_INV 0x3 1948362c389SVineet Gupta #define OP_INV_IC 0x4 1958362c389SVineet Gupta 1968362c389SVineet Gupta /* 1978ea2ddffSVineet Gupta * I-Cache Aliasing in ARC700 VIPT caches (MMU v1-v3) 1988ea2ddffSVineet Gupta * 1998ea2ddffSVineet Gupta * ARC VIPT I-cache uses vaddr to index into cache and paddr to match the tag. 2008ea2ddffSVineet Gupta * The orig Cache Management Module "CDU" only required paddr to invalidate a 2018ea2ddffSVineet Gupta * certain line since it sufficed as index in Non-Aliasing VIPT cache-geometry. 2028ea2ddffSVineet Gupta * Infact for distinct V1,V2,P: all of {V1-P},{V2-P},{P-P} would end up fetching 2038ea2ddffSVineet Gupta * the exact same line. 2048ea2ddffSVineet Gupta * 2058ea2ddffSVineet Gupta * However for larger Caches (way-size > page-size) - i.e. in Aliasing config, 2068ea2ddffSVineet Gupta * paddr alone could not be used to correctly index the cache. 2078ea2ddffSVineet Gupta * 2088ea2ddffSVineet Gupta * ------------------ 2098ea2ddffSVineet Gupta * MMU v1/v2 (Fixed Page Size 8k) 2108ea2ddffSVineet Gupta * ------------------ 2118ea2ddffSVineet Gupta * The solution was to provide CDU with these additonal vaddr bits. These 2128ea2ddffSVineet Gupta * would be bits [x:13], x would depend on cache-geometry, 13 comes from 2138ea2ddffSVineet Gupta * standard page size of 8k. 2148ea2ddffSVineet Gupta * H/w folks chose [17:13] to be a future safe range, and moreso these 5 bits 2158ea2ddffSVineet Gupta * of vaddr could easily be "stuffed" in the paddr as bits [4:0] since the 2168ea2ddffSVineet Gupta * orig 5 bits of paddr were anyways ignored by CDU line ops, as they 2178ea2ddffSVineet Gupta * represent the offset within cache-line. The adv of using this "clumsy" 2188ea2ddffSVineet Gupta * interface for additional info was no new reg was needed in CDU programming 2198ea2ddffSVineet Gupta * model. 2208ea2ddffSVineet Gupta * 2218ea2ddffSVineet Gupta * 17:13 represented the max num of bits passable, actual bits needed were 2228ea2ddffSVineet Gupta * fewer, based on the num-of-aliases possible. 2238ea2ddffSVineet Gupta * -for 2 alias possibility, only bit 13 needed (32K cache) 2248ea2ddffSVineet Gupta * -for 4 alias possibility, bits 14:13 needed (64K cache) 2258ea2ddffSVineet Gupta * 2268ea2ddffSVineet Gupta * ------------------ 2278ea2ddffSVineet Gupta * MMU v3 2288ea2ddffSVineet Gupta * ------------------ 2298ea2ddffSVineet Gupta * This ver of MMU supports variable page sizes (1k-16k): although Linux will 2308ea2ddffSVineet Gupta * only support 8k (default), 16k and 4k. 2312547476aSAndrea Gelmini * However from hardware perspective, smaller page sizes aggravate aliasing 2328ea2ddffSVineet Gupta * meaning more vaddr bits needed to disambiguate the cache-line-op ; 2338ea2ddffSVineet Gupta * the existing scheme of piggybacking won't work for certain configurations. 2348ea2ddffSVineet Gupta * Two new registers IC_PTAG and DC_PTAG inttoduced. 2358ea2ddffSVineet Gupta * "tag" bits are provided in PTAG, index bits in existing IVIL/IVDL/FLDL regs 2368362c389SVineet Gupta */ 2378ea2ddffSVineet Gupta 23811e14896SVineet Gupta static inline 23928b4af72SVineet Gupta void __cache_line_loop_v2(phys_addr_t paddr, unsigned long vaddr, 2407d3d162bSVineet Gupta unsigned long sz, const int op, const int full_page) 2418362c389SVineet Gupta { 24211e14896SVineet Gupta unsigned int aux_cmd; 2438362c389SVineet Gupta int num_lines; 2448362c389SVineet Gupta 2458ea2ddffSVineet Gupta if (op == OP_INV_IC) { 2468362c389SVineet Gupta aux_cmd = ARC_REG_IC_IVIL; 24711e14896SVineet Gupta } else { 2488362c389SVineet Gupta /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 2498ea2ddffSVineet Gupta aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 2508362c389SVineet Gupta } 2518362c389SVineet Gupta 2528362c389SVineet Gupta /* Ensure we properly floor/ceil the non-line aligned/sized requests 2538362c389SVineet Gupta * and have @paddr - aligned to cache line and integral @num_lines. 2548362c389SVineet Gupta * This however can be avoided for page sized since: 2558362c389SVineet Gupta * -@paddr will be cache-line aligned already (being page aligned) 2568362c389SVineet Gupta * -@sz will be integral multiple of line size (being page sized). 2578362c389SVineet Gupta */ 25811e14896SVineet Gupta if (!full_page) { 2598362c389SVineet Gupta sz += paddr & ~CACHE_LINE_MASK; 2608362c389SVineet Gupta paddr &= CACHE_LINE_MASK; 2618362c389SVineet Gupta vaddr &= CACHE_LINE_MASK; 2628362c389SVineet Gupta } 2638362c389SVineet Gupta 2648362c389SVineet Gupta num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 2658362c389SVineet Gupta 2668362c389SVineet Gupta /* MMUv2 and before: paddr contains stuffed vaddrs bits */ 2678362c389SVineet Gupta paddr |= (vaddr >> PAGE_SHIFT) & 0x1F; 2688362c389SVineet Gupta 2698362c389SVineet Gupta while (num_lines-- > 0) { 27011e14896SVineet Gupta write_aux_reg(aux_cmd, paddr); 27111e14896SVineet Gupta paddr += L1_CACHE_BYTES; 27211e14896SVineet Gupta } 27311e14896SVineet Gupta } 27411e14896SVineet Gupta 2755a364c2aSVineet Gupta /* 2765a364c2aSVineet Gupta * For ARC700 MMUv3 I-cache and D-cache flushes 277fa84d731SVineet Gupta * - ARC700 programming model requires paddr and vaddr be passed in seperate 278fa84d731SVineet Gupta * AUX registers (*_IV*L and *_PTAG respectively) irrespective of whether the 279fa84d731SVineet Gupta * caches actually alias or not. 280fa84d731SVineet Gupta * - For HS38, only the aliasing I-cache configuration uses the PTAG reg 281fa84d731SVineet Gupta * (non aliasing I-cache version doesn't; while D-cache can't possibly alias) 2825a364c2aSVineet Gupta */ 28311e14896SVineet Gupta static inline 28428b4af72SVineet Gupta void __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr, 2857d3d162bSVineet Gupta unsigned long sz, const int op, const int full_page) 28611e14896SVineet Gupta { 28711e14896SVineet Gupta unsigned int aux_cmd, aux_tag; 28811e14896SVineet Gupta int num_lines; 28911e14896SVineet Gupta 29011e14896SVineet Gupta if (op == OP_INV_IC) { 29111e14896SVineet Gupta aux_cmd = ARC_REG_IC_IVIL; 29211e14896SVineet Gupta aux_tag = ARC_REG_IC_PTAG; 29311e14896SVineet Gupta } else { 29411e14896SVineet Gupta aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 29511e14896SVineet Gupta aux_tag = ARC_REG_DC_PTAG; 29611e14896SVineet Gupta } 29711e14896SVineet Gupta 29811e14896SVineet Gupta /* Ensure we properly floor/ceil the non-line aligned/sized requests 29911e14896SVineet Gupta * and have @paddr - aligned to cache line and integral @num_lines. 30011e14896SVineet Gupta * This however can be avoided for page sized since: 30111e14896SVineet Gupta * -@paddr will be cache-line aligned already (being page aligned) 30211e14896SVineet Gupta * -@sz will be integral multiple of line size (being page sized). 30311e14896SVineet Gupta */ 30411e14896SVineet Gupta if (!full_page) { 30511e14896SVineet Gupta sz += paddr & ~CACHE_LINE_MASK; 30611e14896SVineet Gupta paddr &= CACHE_LINE_MASK; 30711e14896SVineet Gupta vaddr &= CACHE_LINE_MASK; 30811e14896SVineet Gupta } 30911e14896SVineet Gupta num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 31011e14896SVineet Gupta 31111e14896SVineet Gupta /* 31211e14896SVineet Gupta * MMUv3, cache ops require paddr in PTAG reg 31311e14896SVineet Gupta * if V-P const for loop, PTAG can be written once outside loop 31411e14896SVineet Gupta */ 31511e14896SVineet Gupta if (full_page) 31611e14896SVineet Gupta write_aux_reg(aux_tag, paddr); 31711e14896SVineet Gupta 3185a364c2aSVineet Gupta /* 3195a364c2aSVineet Gupta * This is technically for MMU v4, using the MMU v3 programming model 3202547476aSAndrea Gelmini * Special work for HS38 aliasing I-cache configuration with PAE40 3215a364c2aSVineet Gupta * - upper 8 bits of paddr need to be written into PTAG_HI 3225a364c2aSVineet Gupta * - (and needs to be written before the lower 32 bits) 3235a364c2aSVineet Gupta * Note that PTAG_HI is hoisted outside the line loop 3245a364c2aSVineet Gupta */ 3255a364c2aSVineet Gupta if (is_pae40_enabled() && op == OP_INV_IC) 3265a364c2aSVineet Gupta write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 3275a364c2aSVineet Gupta 32811e14896SVineet Gupta while (num_lines-- > 0) { 32911e14896SVineet Gupta if (!full_page) { 3308362c389SVineet Gupta write_aux_reg(aux_tag, paddr); 3318362c389SVineet Gupta paddr += L1_CACHE_BYTES; 3328362c389SVineet Gupta } 3338362c389SVineet Gupta 3348362c389SVineet Gupta write_aux_reg(aux_cmd, vaddr); 3358362c389SVineet Gupta vaddr += L1_CACHE_BYTES; 33611e14896SVineet Gupta } 33711e14896SVineet Gupta } 33811e14896SVineet Gupta 3390d77117fSVineet Gupta #ifndef USE_RGN_FLSH 3400d77117fSVineet Gupta 341d1f317d8SVineet Gupta /* 3425a364c2aSVineet Gupta * In HS38x (MMU v4), I-cache is VIPT (can alias), D-cache is PIPT 3435a364c2aSVineet Gupta * Here's how cache ops are implemented 344d1f317d8SVineet Gupta * 3455a364c2aSVineet Gupta * - D-cache: only paddr needed (in DC_IVDL/DC_FLDL) 3465a364c2aSVineet Gupta * - I-cache Non Aliasing: Despite VIPT, only paddr needed (in IC_IVIL) 3475a364c2aSVineet Gupta * - I-cache Aliasing: Both vaddr and paddr needed (in IC_IVIL, IC_PTAG 3485a364c2aSVineet Gupta * respectively, similar to MMU v3 programming model, hence 3495a364c2aSVineet Gupta * __cache_line_loop_v3() is used) 3505a364c2aSVineet Gupta * 3515a364c2aSVineet Gupta * If PAE40 is enabled, independent of aliasing considerations, the higher bits 3525a364c2aSVineet Gupta * needs to be written into PTAG_HI 353d1f317d8SVineet Gupta */ 354d1f317d8SVineet Gupta static inline 35528b4af72SVineet Gupta void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, 3567d3d162bSVineet Gupta unsigned long sz, const int op, const int full_page) 357d1f317d8SVineet Gupta { 358d1f317d8SVineet Gupta unsigned int aux_cmd; 359d1f317d8SVineet Gupta int num_lines; 360d1f317d8SVineet Gupta 3617d3d162bSVineet Gupta if (op == OP_INV_IC) { 362d1f317d8SVineet Gupta aux_cmd = ARC_REG_IC_IVIL; 363d1f317d8SVineet Gupta } else { 364d1f317d8SVineet Gupta /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 3657d3d162bSVineet Gupta aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 366d1f317d8SVineet Gupta } 367d1f317d8SVineet Gupta 368d1f317d8SVineet Gupta /* Ensure we properly floor/ceil the non-line aligned/sized requests 369d1f317d8SVineet Gupta * and have @paddr - aligned to cache line and integral @num_lines. 370d1f317d8SVineet Gupta * This however can be avoided for page sized since: 371d1f317d8SVineet Gupta * -@paddr will be cache-line aligned already (being page aligned) 372d1f317d8SVineet Gupta * -@sz will be integral multiple of line size (being page sized). 373d1f317d8SVineet Gupta */ 3747d3d162bSVineet Gupta if (!full_page) { 375d1f317d8SVineet Gupta sz += paddr & ~CACHE_LINE_MASK; 376d1f317d8SVineet Gupta paddr &= CACHE_LINE_MASK; 377d1f317d8SVineet Gupta } 378d1f317d8SVineet Gupta 379d1f317d8SVineet Gupta num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 380d1f317d8SVineet Gupta 3815a364c2aSVineet Gupta /* 3825a364c2aSVineet Gupta * For HS38 PAE40 configuration 3835a364c2aSVineet Gupta * - upper 8 bits of paddr need to be written into PTAG_HI 3845a364c2aSVineet Gupta * - (and needs to be written before the lower 32 bits) 3855a364c2aSVineet Gupta */ 3865a364c2aSVineet Gupta if (is_pae40_enabled()) { 3877d3d162bSVineet Gupta if (op == OP_INV_IC) 3885a364c2aSVineet Gupta /* 3895a364c2aSVineet Gupta * Non aliasing I-cache in HS38, 3905a364c2aSVineet Gupta * aliasing I-cache handled in __cache_line_loop_v3() 3915a364c2aSVineet Gupta */ 3925a364c2aSVineet Gupta write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 3935a364c2aSVineet Gupta else 3945a364c2aSVineet Gupta write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32); 3955a364c2aSVineet Gupta } 3965a364c2aSVineet Gupta 397d1f317d8SVineet Gupta while (num_lines-- > 0) { 398d1f317d8SVineet Gupta write_aux_reg(aux_cmd, paddr); 399d1f317d8SVineet Gupta paddr += L1_CACHE_BYTES; 400d1f317d8SVineet Gupta } 401d1f317d8SVineet Gupta } 402d1f317d8SVineet Gupta 4030d77117fSVineet Gupta #else 4040d77117fSVineet Gupta 4050d77117fSVineet Gupta /* 4060d77117fSVineet Gupta * optimized flush operation which takes a region as opposed to iterating per line 4070d77117fSVineet Gupta */ 4080d77117fSVineet Gupta static inline 4090d77117fSVineet Gupta void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, 4100d77117fSVineet Gupta unsigned long sz, const int op, const int full_page) 4110d77117fSVineet Gupta { 412ee40bd1eSVineet Gupta unsigned int s, e; 4130d77117fSVineet Gupta 4140d77117fSVineet Gupta /* Only for Non aliasing I-cache in HS38 */ 4150d77117fSVineet Gupta if (op == OP_INV_IC) { 4160d77117fSVineet Gupta s = ARC_REG_IC_IVIR; 4170d77117fSVineet Gupta e = ARC_REG_IC_ENDR; 4180d77117fSVineet Gupta } else { 4190d77117fSVineet Gupta s = ARC_REG_DC_STARTR; 4200d77117fSVineet Gupta e = ARC_REG_DC_ENDR; 4210d77117fSVineet Gupta } 4220d77117fSVineet Gupta 4230d77117fSVineet Gupta if (!full_page) { 4240d77117fSVineet Gupta /* for any leading gap between @paddr and start of cache line */ 4250d77117fSVineet Gupta sz += paddr & ~CACHE_LINE_MASK; 4260d77117fSVineet Gupta paddr &= CACHE_LINE_MASK; 4270d77117fSVineet Gupta 4280d77117fSVineet Gupta /* 4290d77117fSVineet Gupta * account for any trailing gap to end of cache line 4300d77117fSVineet Gupta * this is equivalent to DIV_ROUND_UP() in line ops above 4310d77117fSVineet Gupta */ 4320d77117fSVineet Gupta sz += L1_CACHE_BYTES - 1; 4330d77117fSVineet Gupta } 4340d77117fSVineet Gupta 4350d77117fSVineet Gupta if (is_pae40_enabled()) { 4360d77117fSVineet Gupta /* TBD: check if crossing 4TB boundary */ 4370d77117fSVineet Gupta if (op == OP_INV_IC) 4380d77117fSVineet Gupta write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 4390d77117fSVineet Gupta else 4400d77117fSVineet Gupta write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32); 4410d77117fSVineet Gupta } 4420d77117fSVineet Gupta 4430d77117fSVineet Gupta /* ENDR needs to be set ahead of START */ 4440d77117fSVineet Gupta write_aux_reg(e, paddr + sz); /* ENDR is exclusive */ 4450d77117fSVineet Gupta write_aux_reg(s, paddr); 4460d77117fSVineet Gupta 4470d77117fSVineet Gupta /* caller waits on DC_CTRL.FS */ 4480d77117fSVineet Gupta } 4490d77117fSVineet Gupta 4500d77117fSVineet Gupta #endif 4510d77117fSVineet Gupta 45211e14896SVineet Gupta #if (CONFIG_ARC_MMU_VER < 3) 45311e14896SVineet Gupta #define __cache_line_loop __cache_line_loop_v2 45411e14896SVineet Gupta #elif (CONFIG_ARC_MMU_VER == 3) 45511e14896SVineet Gupta #define __cache_line_loop __cache_line_loop_v3 456d1f317d8SVineet Gupta #elif (CONFIG_ARC_MMU_VER > 3) 457d1f317d8SVineet Gupta #define __cache_line_loop __cache_line_loop_v4 4588362c389SVineet Gupta #endif 4598362c389SVineet Gupta 4608362c389SVineet Gupta #ifdef CONFIG_ARC_HAS_DCACHE 4618362c389SVineet Gupta 4628362c389SVineet Gupta /*************************************************************** 4638362c389SVineet Gupta * Machine specific helpers for Entire D-Cache or Per Line ops 4648362c389SVineet Gupta */ 4658362c389SVineet Gupta 466ee40bd1eSVineet Gupta #ifndef USE_RGN_FLSH 467ee40bd1eSVineet Gupta /* 468ee40bd1eSVineet Gupta * this version avoids extra read/write of DC_CTRL for flush or invalid ops 469ee40bd1eSVineet Gupta * in the non region flush regime (such as for ARCompact) 470ee40bd1eSVineet Gupta */ 4716c310681SVineet Gupta static inline void __before_dc_op(const int op) 4728362c389SVineet Gupta { 4738362c389SVineet Gupta if (op == OP_FLUSH_N_INV) { 4748362c389SVineet Gupta /* Dcache provides 2 cmd: FLUSH or INV 4758362c389SVineet Gupta * INV inturn has sub-modes: DISCARD or FLUSH-BEFORE 4768362c389SVineet Gupta * flush-n-inv is achieved by INV cmd but with IM=1 4778362c389SVineet Gupta * So toggle INV sub-mode depending on op request and default 4788362c389SVineet Gupta */ 4796c310681SVineet Gupta const unsigned int ctl = ARC_REG_DC_CTRL; 4806c310681SVineet Gupta write_aux_reg(ctl, read_aux_reg(ctl) | DC_CTRL_INV_MODE_FLUSH); 4816c310681SVineet Gupta } 4828362c389SVineet Gupta } 4838362c389SVineet Gupta 484ee40bd1eSVineet Gupta #else 485ee40bd1eSVineet Gupta 486ee40bd1eSVineet Gupta static inline void __before_dc_op(const int op) 487ee40bd1eSVineet Gupta { 488ee40bd1eSVineet Gupta const unsigned int ctl = ARC_REG_DC_CTRL; 489ee40bd1eSVineet Gupta unsigned int val = read_aux_reg(ctl); 490ee40bd1eSVineet Gupta 491ee40bd1eSVineet Gupta if (op == OP_FLUSH_N_INV) { 492ee40bd1eSVineet Gupta val |= DC_CTRL_INV_MODE_FLUSH; 493ee40bd1eSVineet Gupta } 494ee40bd1eSVineet Gupta 495ee40bd1eSVineet Gupta if (op != OP_INV_IC) { 496ee40bd1eSVineet Gupta /* 497ee40bd1eSVineet Gupta * Flush / Invalidate is provided by DC_CTRL.RNG_OP 0 or 1 498ee40bd1eSVineet Gupta * combined Flush-n-invalidate uses DC_CTRL.IM = 1 set above 499ee40bd1eSVineet Gupta */ 500ee40bd1eSVineet Gupta val &= ~DC_CTRL_RGN_OP_MSK; 501ee40bd1eSVineet Gupta if (op & OP_INV) 502ee40bd1eSVineet Gupta val |= DC_CTRL_RGN_OP_INV; 503ee40bd1eSVineet Gupta } 504ee40bd1eSVineet Gupta write_aux_reg(ctl, val); 505ee40bd1eSVineet Gupta } 506ee40bd1eSVineet Gupta 507ee40bd1eSVineet Gupta #endif 508ee40bd1eSVineet Gupta 509ee40bd1eSVineet Gupta 5106c310681SVineet Gupta static inline void __after_dc_op(const int op) 5118362c389SVineet Gupta { 5126c310681SVineet Gupta if (op & OP_FLUSH) { 5136c310681SVineet Gupta const unsigned int ctl = ARC_REG_DC_CTRL; 5146c310681SVineet Gupta unsigned int reg; 5156c310681SVineet Gupta 5166c310681SVineet Gupta /* flush / flush-n-inv both wait */ 5176c310681SVineet Gupta while ((reg = read_aux_reg(ctl)) & DC_CTRL_FLUSH_STATUS) 5186c310681SVineet Gupta ; 5198362c389SVineet Gupta 5208362c389SVineet Gupta /* Switch back to default Invalidate mode */ 5218362c389SVineet Gupta if (op == OP_FLUSH_N_INV) 5226c310681SVineet Gupta write_aux_reg(ctl, reg & ~DC_CTRL_INV_MODE_FLUSH); 5236c310681SVineet Gupta } 5248362c389SVineet Gupta } 5258362c389SVineet Gupta 5268362c389SVineet Gupta /* 5278362c389SVineet Gupta * Operation on Entire D-Cache 5288ea2ddffSVineet Gupta * @op = {OP_INV, OP_FLUSH, OP_FLUSH_N_INV} 5298362c389SVineet Gupta * Note that constant propagation ensures all the checks are gone 5308362c389SVineet Gupta * in generated code 5318362c389SVineet Gupta */ 5328ea2ddffSVineet Gupta static inline void __dc_entire_op(const int op) 5338362c389SVineet Gupta { 5348362c389SVineet Gupta int aux; 5358362c389SVineet Gupta 5366c310681SVineet Gupta __before_dc_op(op); 5378362c389SVineet Gupta 5388ea2ddffSVineet Gupta if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 5398362c389SVineet Gupta aux = ARC_REG_DC_IVDC; 5408362c389SVineet Gupta else 5418362c389SVineet Gupta aux = ARC_REG_DC_FLSH; 5428362c389SVineet Gupta 5438362c389SVineet Gupta write_aux_reg(aux, 0x1); 5448362c389SVineet Gupta 5456c310681SVineet Gupta __after_dc_op(op); 5468362c389SVineet Gupta } 5478362c389SVineet Gupta 5488c47f83bSVineet Gupta static inline void __dc_disable(void) 5498c47f83bSVineet Gupta { 5508c47f83bSVineet Gupta const int r = ARC_REG_DC_CTRL; 5518c47f83bSVineet Gupta 5528c47f83bSVineet Gupta __dc_entire_op(OP_FLUSH_N_INV); 5538c47f83bSVineet Gupta write_aux_reg(r, read_aux_reg(r) | DC_CTRL_DIS); 5548c47f83bSVineet Gupta } 5558c47f83bSVineet Gupta 5568c47f83bSVineet Gupta static void __dc_enable(void) 5578c47f83bSVineet Gupta { 5588c47f83bSVineet Gupta const int r = ARC_REG_DC_CTRL; 5598c47f83bSVineet Gupta 5608c47f83bSVineet Gupta write_aux_reg(r, read_aux_reg(r) & ~DC_CTRL_DIS); 5618c47f83bSVineet Gupta } 5628c47f83bSVineet Gupta 5638362c389SVineet Gupta /* For kernel mappings cache operation: index is same as paddr */ 5648362c389SVineet Gupta #define __dc_line_op_k(p, sz, op) __dc_line_op(p, p, sz, op) 5658362c389SVineet Gupta 5668362c389SVineet Gupta /* 5678ea2ddffSVineet Gupta * D-Cache Line ops: Per Line INV (discard or wback+discard) or FLUSH (wback) 5688362c389SVineet Gupta */ 56928b4af72SVineet Gupta static inline void __dc_line_op(phys_addr_t paddr, unsigned long vaddr, 5708ea2ddffSVineet Gupta unsigned long sz, const int op) 5718362c389SVineet Gupta { 5727d3d162bSVineet Gupta const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 5738362c389SVineet Gupta unsigned long flags; 5748362c389SVineet Gupta 5758362c389SVineet Gupta local_irq_save(flags); 5768362c389SVineet Gupta 5776c310681SVineet Gupta __before_dc_op(op); 5788362c389SVineet Gupta 5797d3d162bSVineet Gupta __cache_line_loop(paddr, vaddr, sz, op, full_page); 5808362c389SVineet Gupta 5816c310681SVineet Gupta __after_dc_op(op); 5828362c389SVineet Gupta 5838362c389SVineet Gupta local_irq_restore(flags); 5848362c389SVineet Gupta } 5858362c389SVineet Gupta 5868362c389SVineet Gupta #else 5878362c389SVineet Gupta 5888ea2ddffSVineet Gupta #define __dc_entire_op(op) 5898c47f83bSVineet Gupta #define __dc_disable() 5908c47f83bSVineet Gupta #define __dc_enable() 5918ea2ddffSVineet Gupta #define __dc_line_op(paddr, vaddr, sz, op) 5928ea2ddffSVineet Gupta #define __dc_line_op_k(paddr, sz, op) 5938362c389SVineet Gupta 5948362c389SVineet Gupta #endif /* CONFIG_ARC_HAS_DCACHE */ 5958362c389SVineet Gupta 5968362c389SVineet Gupta #ifdef CONFIG_ARC_HAS_ICACHE 5978362c389SVineet Gupta 5988362c389SVineet Gupta static inline void __ic_entire_inv(void) 5998362c389SVineet Gupta { 6008362c389SVineet Gupta write_aux_reg(ARC_REG_IC_IVIC, 1); 6018362c389SVineet Gupta read_aux_reg(ARC_REG_IC_CTRL); /* blocks */ 6028362c389SVineet Gupta } 6038362c389SVineet Gupta 6048362c389SVineet Gupta static inline void 60528b4af72SVineet Gupta __ic_line_inv_vaddr_local(phys_addr_t paddr, unsigned long vaddr, 6068362c389SVineet Gupta unsigned long sz) 6078362c389SVineet Gupta { 6087d3d162bSVineet Gupta const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 6098362c389SVineet Gupta unsigned long flags; 6108362c389SVineet Gupta 6118362c389SVineet Gupta local_irq_save(flags); 6127d3d162bSVineet Gupta (*_cache_line_loop_ic_fn)(paddr, vaddr, sz, OP_INV_IC, full_page); 6138362c389SVineet Gupta local_irq_restore(flags); 6148362c389SVineet Gupta } 6158362c389SVineet Gupta 6168362c389SVineet Gupta #ifndef CONFIG_SMP 6178362c389SVineet Gupta 6188362c389SVineet Gupta #define __ic_line_inv_vaddr(p, v, s) __ic_line_inv_vaddr_local(p, v, s) 6198362c389SVineet Gupta 6208362c389SVineet Gupta #else 6218362c389SVineet Gupta 6228362c389SVineet Gupta struct ic_inv_args { 62328b4af72SVineet Gupta phys_addr_t paddr, vaddr; 6248362c389SVineet Gupta int sz; 6258362c389SVineet Gupta }; 6268362c389SVineet Gupta 6278362c389SVineet Gupta static void __ic_line_inv_vaddr_helper(void *info) 6288362c389SVineet Gupta { 6298362c389SVineet Gupta struct ic_inv_args *ic_inv = info; 6308362c389SVineet Gupta 6318362c389SVineet Gupta __ic_line_inv_vaddr_local(ic_inv->paddr, ic_inv->vaddr, ic_inv->sz); 6328362c389SVineet Gupta } 6338362c389SVineet Gupta 63428b4af72SVineet Gupta static void __ic_line_inv_vaddr(phys_addr_t paddr, unsigned long vaddr, 6358362c389SVineet Gupta unsigned long sz) 6368362c389SVineet Gupta { 6378362c389SVineet Gupta struct ic_inv_args ic_inv = { 6388362c389SVineet Gupta .paddr = paddr, 6398362c389SVineet Gupta .vaddr = vaddr, 6408362c389SVineet Gupta .sz = sz 6418362c389SVineet Gupta }; 6428362c389SVineet Gupta 6438362c389SVineet Gupta on_each_cpu(__ic_line_inv_vaddr_helper, &ic_inv, 1); 6448362c389SVineet Gupta } 6458362c389SVineet Gupta 6468362c389SVineet Gupta #endif /* CONFIG_SMP */ 6478362c389SVineet Gupta 6488362c389SVineet Gupta #else /* !CONFIG_ARC_HAS_ICACHE */ 6498362c389SVineet Gupta 6508362c389SVineet Gupta #define __ic_entire_inv() 6518362c389SVineet Gupta #define __ic_line_inv_vaddr(pstart, vstart, sz) 6528362c389SVineet Gupta 6538362c389SVineet Gupta #endif /* CONFIG_ARC_HAS_ICACHE */ 6548362c389SVineet Gupta 65528b4af72SVineet Gupta noinline void slc_op(phys_addr_t paddr, unsigned long sz, const int op) 656795f4558SVineet Gupta { 657795f4558SVineet Gupta #ifdef CONFIG_ISA_ARCV2 658b607edddSAlexey Brodkin /* 659b607edddSAlexey Brodkin * SLC is shared between all cores and concurrent aux operations from 660b607edddSAlexey Brodkin * multiple cores need to be serialized using a spinlock 661b607edddSAlexey Brodkin * A concurrent operation can be silently ignored and/or the old/new 662b607edddSAlexey Brodkin * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop 663b607edddSAlexey Brodkin * below) 664b607edddSAlexey Brodkin */ 665b607edddSAlexey Brodkin static DEFINE_SPINLOCK(lock); 666795f4558SVineet Gupta unsigned long flags; 667795f4558SVineet Gupta unsigned int ctrl; 6687d79cee2SAlexey Brodkin phys_addr_t end; 669795f4558SVineet Gupta 670b607edddSAlexey Brodkin spin_lock_irqsave(&lock, flags); 671795f4558SVineet Gupta 672795f4558SVineet Gupta /* 673795f4558SVineet Gupta * The Region Flush operation is specified by CTRL.RGN_OP[11..9] 674795f4558SVineet Gupta * - b'000 (default) is Flush, 675795f4558SVineet Gupta * - b'001 is Invalidate if CTRL.IM == 0 676795f4558SVineet Gupta * - b'001 is Flush-n-Invalidate if CTRL.IM == 1 677795f4558SVineet Gupta */ 678795f4558SVineet Gupta ctrl = read_aux_reg(ARC_REG_SLC_CTRL); 679795f4558SVineet Gupta 680795f4558SVineet Gupta /* Don't rely on default value of IM bit */ 681795f4558SVineet Gupta if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 682795f4558SVineet Gupta ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 683795f4558SVineet Gupta else 684795f4558SVineet Gupta ctrl |= SLC_CTRL_IM; 685795f4558SVineet Gupta 686795f4558SVineet Gupta if (op & OP_INV) 687795f4558SVineet Gupta ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */ 688795f4558SVineet Gupta else 689795f4558SVineet Gupta ctrl &= ~SLC_CTRL_RGN_OP_INV; 690795f4558SVineet Gupta 691795f4558SVineet Gupta write_aux_reg(ARC_REG_SLC_CTRL, ctrl); 692795f4558SVineet Gupta 693795f4558SVineet Gupta /* 694795f4558SVineet Gupta * Lower bits are ignored, no need to clip 695795f4558SVineet Gupta * END needs to be setup before START (latter triggers the operation) 696795f4558SVineet Gupta * END can't be same as START, so add (l2_line_sz - 1) to sz 697795f4558SVineet Gupta */ 6987d79cee2SAlexey Brodkin end = paddr + sz + l2_line_sz - 1; 6997d79cee2SAlexey Brodkin if (is_pae40_enabled()) 7007d79cee2SAlexey Brodkin write_aux_reg(ARC_REG_SLC_RGN_END1, upper_32_bits(end)); 7017d79cee2SAlexey Brodkin 7027d79cee2SAlexey Brodkin write_aux_reg(ARC_REG_SLC_RGN_END, lower_32_bits(end)); 7037d79cee2SAlexey Brodkin 7047d79cee2SAlexey Brodkin if (is_pae40_enabled()) 7057d79cee2SAlexey Brodkin write_aux_reg(ARC_REG_SLC_RGN_START1, upper_32_bits(paddr)); 7067d79cee2SAlexey Brodkin 7077d79cee2SAlexey Brodkin write_aux_reg(ARC_REG_SLC_RGN_START, lower_32_bits(paddr)); 708795f4558SVineet Gupta 709b37174d9SAlexey Brodkin /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 710b37174d9SAlexey Brodkin read_aux_reg(ARC_REG_SLC_CTRL); 711b37174d9SAlexey Brodkin 712795f4558SVineet Gupta while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY); 713795f4558SVineet Gupta 714b607edddSAlexey Brodkin spin_unlock_irqrestore(&lock, flags); 715795f4558SVineet Gupta #endif 716795f4558SVineet Gupta } 717795f4558SVineet Gupta 718d4911cddSVineet Gupta noinline static void slc_entire_op(const int op) 719d4911cddSVineet Gupta { 720d4911cddSVineet Gupta unsigned int ctrl, r = ARC_REG_SLC_CTRL; 721d4911cddSVineet Gupta 722d4911cddSVineet Gupta ctrl = read_aux_reg(r); 723d4911cddSVineet Gupta 724d4911cddSVineet Gupta if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 725d4911cddSVineet Gupta ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 726d4911cddSVineet Gupta else 727d4911cddSVineet Gupta ctrl |= SLC_CTRL_IM; 728d4911cddSVineet Gupta 729d4911cddSVineet Gupta write_aux_reg(r, ctrl); 730d4911cddSVineet Gupta 731d4911cddSVineet Gupta write_aux_reg(ARC_REG_SLC_INVALIDATE, 1); 732d4911cddSVineet Gupta 733c70c4733SAlexey Brodkin /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 734c70c4733SAlexey Brodkin read_aux_reg(r); 735c70c4733SAlexey Brodkin 736d4911cddSVineet Gupta /* Important to wait for flush to complete */ 737d4911cddSVineet Gupta while (read_aux_reg(r) & SLC_CTRL_BUSY); 738d4911cddSVineet Gupta } 739d4911cddSVineet Gupta 740d4911cddSVineet Gupta static inline void arc_slc_disable(void) 741d4911cddSVineet Gupta { 742d4911cddSVineet Gupta const int r = ARC_REG_SLC_CTRL; 743d4911cddSVineet Gupta 744d4911cddSVineet Gupta slc_entire_op(OP_FLUSH_N_INV); 745d4911cddSVineet Gupta write_aux_reg(r, read_aux_reg(r) | SLC_CTRL_DIS); 746d4911cddSVineet Gupta } 747d4911cddSVineet Gupta 748d4911cddSVineet Gupta static inline void arc_slc_enable(void) 749d4911cddSVineet Gupta { 750d4911cddSVineet Gupta const int r = ARC_REG_SLC_CTRL; 751d4911cddSVineet Gupta 752d4911cddSVineet Gupta write_aux_reg(r, read_aux_reg(r) & ~SLC_CTRL_DIS); 753d4911cddSVineet Gupta } 754d4911cddSVineet Gupta 7558362c389SVineet Gupta /*********************************************************** 7568362c389SVineet Gupta * Exported APIs 7578362c389SVineet Gupta */ 7588362c389SVineet Gupta 7598362c389SVineet Gupta /* 7608362c389SVineet Gupta * Handle cache congruency of kernel and userspace mappings of page when kernel 7618362c389SVineet Gupta * writes-to/reads-from 7628362c389SVineet Gupta * 7638362c389SVineet Gupta * The idea is to defer flushing of kernel mapping after a WRITE, possible if: 7648362c389SVineet Gupta * -dcache is NOT aliasing, hence any U/K-mappings of page are congruent 7658362c389SVineet Gupta * -U-mapping doesn't exist yet for page (finalised in update_mmu_cache) 7668362c389SVineet Gupta * -In SMP, if hardware caches are coherent 7678362c389SVineet Gupta * 7688362c389SVineet Gupta * There's a corollary case, where kernel READs from a userspace mapped page. 7698362c389SVineet Gupta * If the U-mapping is not congruent to to K-mapping, former needs flushing. 7708362c389SVineet Gupta */ 7718362c389SVineet Gupta void flush_dcache_page(struct page *page) 7728362c389SVineet Gupta { 7738362c389SVineet Gupta struct address_space *mapping; 7748362c389SVineet Gupta 7758362c389SVineet Gupta if (!cache_is_vipt_aliasing()) { 7768362c389SVineet Gupta clear_bit(PG_dc_clean, &page->flags); 7778362c389SVineet Gupta return; 7788362c389SVineet Gupta } 7798362c389SVineet Gupta 7808362c389SVineet Gupta /* don't handle anon pages here */ 7818362c389SVineet Gupta mapping = page_mapping(page); 7828362c389SVineet Gupta if (!mapping) 7838362c389SVineet Gupta return; 7848362c389SVineet Gupta 7858362c389SVineet Gupta /* 7868362c389SVineet Gupta * pagecache page, file not yet mapped to userspace 7878362c389SVineet Gupta * Make a note that K-mapping is dirty 7888362c389SVineet Gupta */ 7898362c389SVineet Gupta if (!mapping_mapped(mapping)) { 7908362c389SVineet Gupta clear_bit(PG_dc_clean, &page->flags); 791e1534ae9SKirill A. Shutemov } else if (page_mapcount(page)) { 7928362c389SVineet Gupta 7938362c389SVineet Gupta /* kernel reading from page with U-mapping */ 79428b4af72SVineet Gupta phys_addr_t paddr = (unsigned long)page_address(page); 79509cbfeafSKirill A. Shutemov unsigned long vaddr = page->index << PAGE_SHIFT; 7968362c389SVineet Gupta 7978362c389SVineet Gupta if (addr_not_cache_congruent(paddr, vaddr)) 7988362c389SVineet Gupta __flush_dcache_page(paddr, vaddr); 7998362c389SVineet Gupta } 8008362c389SVineet Gupta } 8018362c389SVineet Gupta EXPORT_SYMBOL(flush_dcache_page); 8028362c389SVineet Gupta 803f2b0b25aSAlexey Brodkin /* 804f2b0b25aSAlexey Brodkin * DMA ops for systems with L1 cache only 805f2b0b25aSAlexey Brodkin * Make memory coherent with L1 cache by flushing/invalidating L1 lines 806f2b0b25aSAlexey Brodkin */ 807f5db19e9SVineet Gupta static void __dma_cache_wback_inv_l1(phys_addr_t start, unsigned long sz) 8088362c389SVineet Gupta { 8098362c389SVineet Gupta __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 810f2b0b25aSAlexey Brodkin } 811795f4558SVineet Gupta 812f5db19e9SVineet Gupta static void __dma_cache_inv_l1(phys_addr_t start, unsigned long sz) 813f2b0b25aSAlexey Brodkin { 814f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_INV); 815f2b0b25aSAlexey Brodkin } 816f2b0b25aSAlexey Brodkin 817f5db19e9SVineet Gupta static void __dma_cache_wback_l1(phys_addr_t start, unsigned long sz) 818f2b0b25aSAlexey Brodkin { 819f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_FLUSH); 820f2b0b25aSAlexey Brodkin } 821f2b0b25aSAlexey Brodkin 822f2b0b25aSAlexey Brodkin /* 823f2b0b25aSAlexey Brodkin * DMA ops for systems with both L1 and L2 caches, but without IOC 8247423cc0cSAdam Buchbinder * Both L1 and L2 lines need to be explicitly flushed/invalidated 825f2b0b25aSAlexey Brodkin */ 826f5db19e9SVineet Gupta static void __dma_cache_wback_inv_slc(phys_addr_t start, unsigned long sz) 827f2b0b25aSAlexey Brodkin { 828f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 829795f4558SVineet Gupta slc_op(start, sz, OP_FLUSH_N_INV); 8308362c389SVineet Gupta } 831f2b0b25aSAlexey Brodkin 832f5db19e9SVineet Gupta static void __dma_cache_inv_slc(phys_addr_t start, unsigned long sz) 833f2b0b25aSAlexey Brodkin { 834f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_INV); 835f2b0b25aSAlexey Brodkin slc_op(start, sz, OP_INV); 836f2b0b25aSAlexey Brodkin } 837f2b0b25aSAlexey Brodkin 838f5db19e9SVineet Gupta static void __dma_cache_wback_slc(phys_addr_t start, unsigned long sz) 839f2b0b25aSAlexey Brodkin { 840f2b0b25aSAlexey Brodkin __dc_line_op_k(start, sz, OP_FLUSH); 841f2b0b25aSAlexey Brodkin slc_op(start, sz, OP_FLUSH); 842f2b0b25aSAlexey Brodkin } 843f2b0b25aSAlexey Brodkin 844f2b0b25aSAlexey Brodkin /* 845f2b0b25aSAlexey Brodkin * DMA ops for systems with IOC 846f2b0b25aSAlexey Brodkin * IOC hardware snoops all DMA traffic keeping the caches consistent with 847f2b0b25aSAlexey Brodkin * memory - eliding need for any explicit cache maintenance of DMA buffers 848f2b0b25aSAlexey Brodkin */ 849f5db19e9SVineet Gupta static void __dma_cache_wback_inv_ioc(phys_addr_t start, unsigned long sz) {} 850f5db19e9SVineet Gupta static void __dma_cache_inv_ioc(phys_addr_t start, unsigned long sz) {} 851f5db19e9SVineet Gupta static void __dma_cache_wback_ioc(phys_addr_t start, unsigned long sz) {} 852f2b0b25aSAlexey Brodkin 853f2b0b25aSAlexey Brodkin /* 854f2b0b25aSAlexey Brodkin * Exported DMA API 855f2b0b25aSAlexey Brodkin */ 856f5db19e9SVineet Gupta void dma_cache_wback_inv(phys_addr_t start, unsigned long sz) 857f2b0b25aSAlexey Brodkin { 858f2b0b25aSAlexey Brodkin __dma_cache_wback_inv(start, sz); 859f2b0b25aSAlexey Brodkin } 8608362c389SVineet Gupta EXPORT_SYMBOL(dma_cache_wback_inv); 8618362c389SVineet Gupta 862f5db19e9SVineet Gupta void dma_cache_inv(phys_addr_t start, unsigned long sz) 8638362c389SVineet Gupta { 864f2b0b25aSAlexey Brodkin __dma_cache_inv(start, sz); 8658362c389SVineet Gupta } 8668362c389SVineet Gupta EXPORT_SYMBOL(dma_cache_inv); 8678362c389SVineet Gupta 868f5db19e9SVineet Gupta void dma_cache_wback(phys_addr_t start, unsigned long sz) 8698362c389SVineet Gupta { 870f2b0b25aSAlexey Brodkin __dma_cache_wback(start, sz); 8718362c389SVineet Gupta } 8728362c389SVineet Gupta EXPORT_SYMBOL(dma_cache_wback); 8738362c389SVineet Gupta 8748362c389SVineet Gupta /* 8758362c389SVineet Gupta * This is API for making I/D Caches consistent when modifying 8768362c389SVineet Gupta * kernel code (loadable modules, kprobes, kgdb...) 8778362c389SVineet Gupta * This is called on insmod, with kernel virtual address for CODE of 8788362c389SVineet Gupta * the module. ARC cache maintenance ops require PHY address thus we 8798362c389SVineet Gupta * need to convert vmalloc addr to PHY addr 8808362c389SVineet Gupta */ 8818362c389SVineet Gupta void flush_icache_range(unsigned long kstart, unsigned long kend) 8828362c389SVineet Gupta { 8838362c389SVineet Gupta unsigned int tot_sz; 8848362c389SVineet Gupta 8858362c389SVineet Gupta WARN(kstart < TASK_SIZE, "%s() can't handle user vaddr", __func__); 8868362c389SVineet Gupta 8878362c389SVineet Gupta /* Shortcut for bigger flush ranges. 8888362c389SVineet Gupta * Here we don't care if this was kernel virtual or phy addr 8898362c389SVineet Gupta */ 8908362c389SVineet Gupta tot_sz = kend - kstart; 8918362c389SVineet Gupta if (tot_sz > PAGE_SIZE) { 8928362c389SVineet Gupta flush_cache_all(); 8938362c389SVineet Gupta return; 8948362c389SVineet Gupta } 8958362c389SVineet Gupta 8968362c389SVineet Gupta /* Case: Kernel Phy addr (0x8000_0000 onwards) */ 8978362c389SVineet Gupta if (likely(kstart > PAGE_OFFSET)) { 8988362c389SVineet Gupta /* 8998362c389SVineet Gupta * The 2nd arg despite being paddr will be used to index icache 9008362c389SVineet Gupta * This is OK since no alternate virtual mappings will exist 9018362c389SVineet Gupta * given the callers for this case: kprobe/kgdb in built-in 9028362c389SVineet Gupta * kernel code only. 9038362c389SVineet Gupta */ 9048362c389SVineet Gupta __sync_icache_dcache(kstart, kstart, kend - kstart); 9058362c389SVineet Gupta return; 9068362c389SVineet Gupta } 9078362c389SVineet Gupta 9088362c389SVineet Gupta /* 9098362c389SVineet Gupta * Case: Kernel Vaddr (0x7000_0000 to 0x7fff_ffff) 9108362c389SVineet Gupta * (1) ARC Cache Maintenance ops only take Phy addr, hence special 9118362c389SVineet Gupta * handling of kernel vaddr. 9128362c389SVineet Gupta * 9138362c389SVineet Gupta * (2) Despite @tot_sz being < PAGE_SIZE (bigger cases handled already), 9148362c389SVineet Gupta * it still needs to handle a 2 page scenario, where the range 9158362c389SVineet Gupta * straddles across 2 virtual pages and hence need for loop 9168362c389SVineet Gupta */ 9178362c389SVineet Gupta while (tot_sz > 0) { 9188362c389SVineet Gupta unsigned int off, sz; 9198362c389SVineet Gupta unsigned long phy, pfn; 9208362c389SVineet Gupta 9218362c389SVineet Gupta off = kstart % PAGE_SIZE; 9228362c389SVineet Gupta pfn = vmalloc_to_pfn((void *)kstart); 9238362c389SVineet Gupta phy = (pfn << PAGE_SHIFT) + off; 9248362c389SVineet Gupta sz = min_t(unsigned int, tot_sz, PAGE_SIZE - off); 9258362c389SVineet Gupta __sync_icache_dcache(phy, kstart, sz); 9268362c389SVineet Gupta kstart += sz; 9278362c389SVineet Gupta tot_sz -= sz; 9288362c389SVineet Gupta } 9298362c389SVineet Gupta } 9308362c389SVineet Gupta EXPORT_SYMBOL(flush_icache_range); 9318362c389SVineet Gupta 9328362c389SVineet Gupta /* 9338362c389SVineet Gupta * General purpose helper to make I and D cache lines consistent. 9348362c389SVineet Gupta * @paddr is phy addr of region 9358362c389SVineet Gupta * @vaddr is typically user vaddr (breakpoint) or kernel vaddr (vmalloc) 9368362c389SVineet Gupta * However in one instance, when called by kprobe (for a breakpt in 9378362c389SVineet Gupta * builtin kernel code) @vaddr will be paddr only, meaning CDU operation will 9388362c389SVineet Gupta * use a paddr to index the cache (despite VIPT). This is fine since since a 9398362c389SVineet Gupta * builtin kernel page will not have any virtual mappings. 9408362c389SVineet Gupta * kprobe on loadable module will be kernel vaddr. 9418362c389SVineet Gupta */ 94228b4af72SVineet Gupta void __sync_icache_dcache(phys_addr_t paddr, unsigned long vaddr, int len) 9438362c389SVineet Gupta { 9448362c389SVineet Gupta __dc_line_op(paddr, vaddr, len, OP_FLUSH_N_INV); 9458362c389SVineet Gupta __ic_line_inv_vaddr(paddr, vaddr, len); 9468362c389SVineet Gupta } 9478362c389SVineet Gupta 9488362c389SVineet Gupta /* wrapper to compile time eliminate alignment checks in flush loop */ 94928b4af72SVineet Gupta void __inv_icache_page(phys_addr_t paddr, unsigned long vaddr) 9508362c389SVineet Gupta { 9518362c389SVineet Gupta __ic_line_inv_vaddr(paddr, vaddr, PAGE_SIZE); 9528362c389SVineet Gupta } 9538362c389SVineet Gupta 9548362c389SVineet Gupta /* 9558362c389SVineet Gupta * wrapper to clearout kernel or userspace mappings of a page 9568362c389SVineet Gupta * For kernel mappings @vaddr == @paddr 9578362c389SVineet Gupta */ 95828b4af72SVineet Gupta void __flush_dcache_page(phys_addr_t paddr, unsigned long vaddr) 9598362c389SVineet Gupta { 9608362c389SVineet Gupta __dc_line_op(paddr, vaddr & PAGE_MASK, PAGE_SIZE, OP_FLUSH_N_INV); 9618362c389SVineet Gupta } 9628362c389SVineet Gupta 9638362c389SVineet Gupta noinline void flush_cache_all(void) 9648362c389SVineet Gupta { 9658362c389SVineet Gupta unsigned long flags; 9668362c389SVineet Gupta 9678362c389SVineet Gupta local_irq_save(flags); 9688362c389SVineet Gupta 9698362c389SVineet Gupta __ic_entire_inv(); 9708362c389SVineet Gupta __dc_entire_op(OP_FLUSH_N_INV); 9718362c389SVineet Gupta 9728362c389SVineet Gupta local_irq_restore(flags); 9738362c389SVineet Gupta 9748362c389SVineet Gupta } 9758362c389SVineet Gupta 9768362c389SVineet Gupta #ifdef CONFIG_ARC_CACHE_VIPT_ALIASING 9778362c389SVineet Gupta 9788362c389SVineet Gupta void flush_cache_mm(struct mm_struct *mm) 9798362c389SVineet Gupta { 9808362c389SVineet Gupta flush_cache_all(); 9818362c389SVineet Gupta } 9828362c389SVineet Gupta 9838362c389SVineet Gupta void flush_cache_page(struct vm_area_struct *vma, unsigned long u_vaddr, 9848362c389SVineet Gupta unsigned long pfn) 9858362c389SVineet Gupta { 9868362c389SVineet Gupta unsigned int paddr = pfn << PAGE_SHIFT; 9878362c389SVineet Gupta 9888362c389SVineet Gupta u_vaddr &= PAGE_MASK; 9898362c389SVineet Gupta 9908362c389SVineet Gupta __flush_dcache_page(paddr, u_vaddr); 9918362c389SVineet Gupta 9928362c389SVineet Gupta if (vma->vm_flags & VM_EXEC) 9938362c389SVineet Gupta __inv_icache_page(paddr, u_vaddr); 9948362c389SVineet Gupta } 9958362c389SVineet Gupta 9968362c389SVineet Gupta void flush_cache_range(struct vm_area_struct *vma, unsigned long start, 9978362c389SVineet Gupta unsigned long end) 9988362c389SVineet Gupta { 9998362c389SVineet Gupta flush_cache_all(); 10008362c389SVineet Gupta } 10018362c389SVineet Gupta 10028362c389SVineet Gupta void flush_anon_page(struct vm_area_struct *vma, struct page *page, 10038362c389SVineet Gupta unsigned long u_vaddr) 10048362c389SVineet Gupta { 10058362c389SVineet Gupta /* TBD: do we really need to clear the kernel mapping */ 10068362c389SVineet Gupta __flush_dcache_page(page_address(page), u_vaddr); 10078362c389SVineet Gupta __flush_dcache_page(page_address(page), page_address(page)); 10088362c389SVineet Gupta 10098362c389SVineet Gupta } 10108362c389SVineet Gupta 10118362c389SVineet Gupta #endif 10128362c389SVineet Gupta 10138362c389SVineet Gupta void copy_user_highpage(struct page *to, struct page *from, 10148362c389SVineet Gupta unsigned long u_vaddr, struct vm_area_struct *vma) 10158362c389SVineet Gupta { 1016336e2136SVineet Gupta void *kfrom = kmap_atomic(from); 1017336e2136SVineet Gupta void *kto = kmap_atomic(to); 10188362c389SVineet Gupta int clean_src_k_mappings = 0; 10198362c389SVineet Gupta 10208362c389SVineet Gupta /* 10218362c389SVineet Gupta * If SRC page was already mapped in userspace AND it's U-mapping is 10228362c389SVineet Gupta * not congruent with K-mapping, sync former to physical page so that 10238362c389SVineet Gupta * K-mapping in memcpy below, sees the right data 10248362c389SVineet Gupta * 10258362c389SVineet Gupta * Note that while @u_vaddr refers to DST page's userspace vaddr, it is 10268362c389SVineet Gupta * equally valid for SRC page as well 1027336e2136SVineet Gupta * 1028336e2136SVineet Gupta * For !VIPT cache, all of this gets compiled out as 1029336e2136SVineet Gupta * addr_not_cache_congruent() is 0 10308362c389SVineet Gupta */ 1031e1534ae9SKirill A. Shutemov if (page_mapcount(from) && addr_not_cache_congruent(kfrom, u_vaddr)) { 1032336e2136SVineet Gupta __flush_dcache_page((unsigned long)kfrom, u_vaddr); 10338362c389SVineet Gupta clean_src_k_mappings = 1; 10348362c389SVineet Gupta } 10358362c389SVineet Gupta 1036336e2136SVineet Gupta copy_page(kto, kfrom); 10378362c389SVineet Gupta 10388362c389SVineet Gupta /* 10398362c389SVineet Gupta * Mark DST page K-mapping as dirty for a later finalization by 10408362c389SVineet Gupta * update_mmu_cache(). Although the finalization could have been done 10418362c389SVineet Gupta * here as well (given that both vaddr/paddr are available). 10428362c389SVineet Gupta * But update_mmu_cache() already has code to do that for other 10438362c389SVineet Gupta * non copied user pages (e.g. read faults which wire in pagecache page 10448362c389SVineet Gupta * directly). 10458362c389SVineet Gupta */ 10468362c389SVineet Gupta clear_bit(PG_dc_clean, &to->flags); 10478362c389SVineet Gupta 10488362c389SVineet Gupta /* 10498362c389SVineet Gupta * if SRC was already usermapped and non-congruent to kernel mapping 10508362c389SVineet Gupta * sync the kernel mapping back to physical page 10518362c389SVineet Gupta */ 10528362c389SVineet Gupta if (clean_src_k_mappings) { 1053336e2136SVineet Gupta __flush_dcache_page((unsigned long)kfrom, (unsigned long)kfrom); 10548362c389SVineet Gupta set_bit(PG_dc_clean, &from->flags); 10558362c389SVineet Gupta } else { 10568362c389SVineet Gupta clear_bit(PG_dc_clean, &from->flags); 10578362c389SVineet Gupta } 1058336e2136SVineet Gupta 1059336e2136SVineet Gupta kunmap_atomic(kto); 1060336e2136SVineet Gupta kunmap_atomic(kfrom); 10618362c389SVineet Gupta } 10628362c389SVineet Gupta 10638362c389SVineet Gupta void clear_user_page(void *to, unsigned long u_vaddr, struct page *page) 10648362c389SVineet Gupta { 10658362c389SVineet Gupta clear_page(to); 10668362c389SVineet Gupta clear_bit(PG_dc_clean, &page->flags); 10678362c389SVineet Gupta } 10688362c389SVineet Gupta 10698362c389SVineet Gupta 10708362c389SVineet Gupta /********************************************************************** 10718362c389SVineet Gupta * Explicit Cache flush request from user space via syscall 10728362c389SVineet Gupta * Needed for JITs which generate code on the fly 10738362c389SVineet Gupta */ 10748362c389SVineet Gupta SYSCALL_DEFINE3(cacheflush, uint32_t, start, uint32_t, sz, uint32_t, flags) 10758362c389SVineet Gupta { 10768362c389SVineet Gupta /* TBD: optimize this */ 10778362c389SVineet Gupta flush_cache_all(); 10788362c389SVineet Gupta return 0; 10798362c389SVineet Gupta } 10808ea2ddffSVineet Gupta 10818c47f83bSVineet Gupta /* 10828c47f83bSVineet Gupta * IO-Coherency (IOC) setup rules: 10838c47f83bSVineet Gupta * 10848c47f83bSVineet Gupta * 1. Needs to be at system level, so only once by Master core 10858c47f83bSVineet Gupta * Non-Masters need not be accessing caches at that time 10868c47f83bSVineet Gupta * - They are either HALT_ON_RESET and kick started much later or 10878c47f83bSVineet Gupta * - if run on reset, need to ensure that arc_platform_smp_wait_to_boot() 10888c47f83bSVineet Gupta * doesn't perturb caches or coherency unit 10898c47f83bSVineet Gupta * 10908c47f83bSVineet Gupta * 2. caches (L1 and SLC) need to be purged (flush+inv) before setting up IOC, 10918c47f83bSVineet Gupta * otherwise any straggler data might behave strangely post IOC enabling 10928c47f83bSVineet Gupta * 10938c47f83bSVineet Gupta * 3. All Caches need to be disabled when setting up IOC to elide any in-flight 10948c47f83bSVineet Gupta * Coherency transactions 10958c47f83bSVineet Gupta */ 109676894a72SVineet Gupta noinline void __init arc_ioc_setup(void) 1097d4911cddSVineet Gupta { 1098e497c8e5SVineet Gupta unsigned int ap_sz; 1099e497c8e5SVineet Gupta 11008c47f83bSVineet Gupta /* Flush + invalidate + disable L1 dcache */ 11018c47f83bSVineet Gupta __dc_disable(); 11028c47f83bSVineet Gupta 11038c47f83bSVineet Gupta /* Flush + invalidate SLC */ 11048c47f83bSVineet Gupta if (read_aux_reg(ARC_REG_SLC_BCR)) 11058c47f83bSVineet Gupta slc_entire_op(OP_FLUSH_N_INV); 11068c47f83bSVineet Gupta 11078c47f83bSVineet Gupta /* IOC Aperture start: TDB: handle non default CONFIG_LINUX_LINK_BASE */ 1108d4911cddSVineet Gupta write_aux_reg(ARC_REG_IO_COH_AP0_BASE, 0x80000); 11098c47f83bSVineet Gupta 1110e497c8e5SVineet Gupta /* 1111e497c8e5SVineet Gupta * IOC Aperture size: 1112e497c8e5SVineet Gupta * decoded as 2 ^ (SIZE + 2) KB: so setting 0x11 implies 512M 1113e497c8e5SVineet Gupta * TBD: fix for PGU + 1GB of low mem 1114e497c8e5SVineet Gupta * TBD: fix for PAE 1115e497c8e5SVineet Gupta */ 1116e497c8e5SVineet Gupta ap_sz = order_base_2(arc_get_mem_sz()/1024) - 2; 1117e497c8e5SVineet Gupta write_aux_reg(ARC_REG_IO_COH_AP0_SIZE, ap_sz); 11188c47f83bSVineet Gupta 1119d4911cddSVineet Gupta write_aux_reg(ARC_REG_IO_COH_PARTIAL, 1); 1120d4911cddSVineet Gupta write_aux_reg(ARC_REG_IO_COH_ENABLE, 1); 11218c47f83bSVineet Gupta 11228c47f83bSVineet Gupta /* Re-enable L1 dcache */ 11238c47f83bSVineet Gupta __dc_enable(); 1124d4911cddSVineet Gupta } 1125d4911cddSVineet Gupta 1126b5ddb6d5SVineet Gupta /* 1127b5ddb6d5SVineet Gupta * Cache related boot time checks/setups only needed on master CPU: 1128b5ddb6d5SVineet Gupta * - Geometry checks (kernel build and hardware agree: e.g. L1_CACHE_BYTES) 1129b5ddb6d5SVineet Gupta * Assume SMP only, so all cores will have same cache config. A check on 1130b5ddb6d5SVineet Gupta * one core suffices for all 1131b5ddb6d5SVineet Gupta * - IOC setup / dma callbacks only need to be done once 1132b5ddb6d5SVineet Gupta */ 113376894a72SVineet Gupta void __init arc_cache_init_master(void) 11348ea2ddffSVineet Gupta { 11358ea2ddffSVineet Gupta unsigned int __maybe_unused cpu = smp_processor_id(); 113645c3b08aSVineet Gupta 11378ea2ddffSVineet Gupta if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) { 11388ea2ddffSVineet Gupta struct cpuinfo_arc_cache *ic = &cpuinfo_arc700[cpu].icache; 11398ea2ddffSVineet Gupta 1140f64915beSVineet Gupta if (!ic->line_len) 11418ea2ddffSVineet Gupta panic("cache support enabled but non-existent cache\n"); 11428ea2ddffSVineet Gupta 11438ea2ddffSVineet Gupta if (ic->line_len != L1_CACHE_BYTES) 11448ea2ddffSVineet Gupta panic("ICache line [%d] != kernel Config [%d]", 11458ea2ddffSVineet Gupta ic->line_len, L1_CACHE_BYTES); 11468ea2ddffSVineet Gupta 1147bcc4d65aSVineet Gupta /* 11482547476aSAndrea Gelmini * In MMU v4 (HS38x) the aliasing icache config uses IVIL/PTAG 1149bcc4d65aSVineet Gupta * pair to provide vaddr/paddr respectively, just as in MMU v3 1150bcc4d65aSVineet Gupta */ 1151bcc4d65aSVineet Gupta if (is_isa_arcv2() && ic->alias) 1152bcc4d65aSVineet Gupta _cache_line_loop_ic_fn = __cache_line_loop_v3; 1153bcc4d65aSVineet Gupta else 1154bcc4d65aSVineet Gupta _cache_line_loop_ic_fn = __cache_line_loop; 11558ea2ddffSVineet Gupta } 11568ea2ddffSVineet Gupta 11578ea2ddffSVineet Gupta if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) { 11588ea2ddffSVineet Gupta struct cpuinfo_arc_cache *dc = &cpuinfo_arc700[cpu].dcache; 11598ea2ddffSVineet Gupta 1160f64915beSVineet Gupta if (!dc->line_len) 11618ea2ddffSVineet Gupta panic("cache support enabled but non-existent cache\n"); 11628ea2ddffSVineet Gupta 11638ea2ddffSVineet Gupta if (dc->line_len != L1_CACHE_BYTES) 11648ea2ddffSVineet Gupta panic("DCache line [%d] != kernel Config [%d]", 11658ea2ddffSVineet Gupta dc->line_len, L1_CACHE_BYTES); 11668ea2ddffSVineet Gupta 1167d1f317d8SVineet Gupta /* check for D-Cache aliasing on ARCompact: ARCv2 has PIPT */ 1168d1f317d8SVineet Gupta if (is_isa_arcompact()) { 1169d1f317d8SVineet Gupta int handled = IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING); 117008fe0079SVineet Gupta int num_colors = dc->sz_k/dc->assoc/TO_KB(PAGE_SIZE); 11718ea2ddffSVineet Gupta 117208fe0079SVineet Gupta if (dc->alias) { 117308fe0079SVineet Gupta if (!handled) 11748ea2ddffSVineet Gupta panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 117508fe0079SVineet Gupta if (CACHE_COLORS_NUM != num_colors) 117608fe0079SVineet Gupta panic("CACHE_COLORS_NUM not optimized for config\n"); 117708fe0079SVineet Gupta } else if (!dc->alias && handled) { 11788ea2ddffSVineet Gupta panic("Disable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 11798ea2ddffSVineet Gupta } 11808ea2ddffSVineet Gupta } 118108fe0079SVineet Gupta } 1182f2b0b25aSAlexey Brodkin 1183d4911cddSVineet Gupta /* Note that SLC disable not formally supported till HS 3.0 */ 1184d4911cddSVineet Gupta if (is_isa_arcv2() && l2_line_sz && !slc_enable) 1185d4911cddSVineet Gupta arc_slc_disable(); 118679335a2cSVineet Gupta 1187d4911cddSVineet Gupta if (is_isa_arcv2() && ioc_enable) 1188d4911cddSVineet Gupta arc_ioc_setup(); 118979335a2cSVineet Gupta 1190cf986d47SVineet Gupta if (is_isa_arcv2() && ioc_enable) { 1191f2b0b25aSAlexey Brodkin __dma_cache_wback_inv = __dma_cache_wback_inv_ioc; 1192f2b0b25aSAlexey Brodkin __dma_cache_inv = __dma_cache_inv_ioc; 1193f2b0b25aSAlexey Brodkin __dma_cache_wback = __dma_cache_wback_ioc; 119479335a2cSVineet Gupta } else if (is_isa_arcv2() && l2_line_sz && slc_enable) { 1195f2b0b25aSAlexey Brodkin __dma_cache_wback_inv = __dma_cache_wback_inv_slc; 1196f2b0b25aSAlexey Brodkin __dma_cache_inv = __dma_cache_inv_slc; 1197f2b0b25aSAlexey Brodkin __dma_cache_wback = __dma_cache_wback_slc; 1198f2b0b25aSAlexey Brodkin } else { 1199f2b0b25aSAlexey Brodkin __dma_cache_wback_inv = __dma_cache_wback_inv_l1; 1200f2b0b25aSAlexey Brodkin __dma_cache_inv = __dma_cache_inv_l1; 1201f2b0b25aSAlexey Brodkin __dma_cache_wback = __dma_cache_wback_l1; 1202f2b0b25aSAlexey Brodkin } 1203d1f317d8SVineet Gupta } 120476894a72SVineet Gupta 120576894a72SVineet Gupta void __ref arc_cache_init(void) 120676894a72SVineet Gupta { 120776894a72SVineet Gupta unsigned int __maybe_unused cpu = smp_processor_id(); 120876894a72SVineet Gupta char str[256]; 120976894a72SVineet Gupta 121018ee4becSNoam Camus pr_info("%s", arc_cache_mumbojumbo(0, str, sizeof(str))); 121176894a72SVineet Gupta 121276894a72SVineet Gupta if (!cpu) 121376894a72SVineet Gupta arc_cache_init_master(); 1214b5ddb6d5SVineet Gupta 1215b5ddb6d5SVineet Gupta /* 1216b5ddb6d5SVineet Gupta * In PAE regime, TLB and cache maintenance ops take wider addresses 1217b5ddb6d5SVineet Gupta * And even if PAE is not enabled in kernel, the upper 32-bits still need 1218b5ddb6d5SVineet Gupta * to be zeroed to keep the ops sane. 1219b5ddb6d5SVineet Gupta * As an optimization for more common !PAE enabled case, zero them out 1220b5ddb6d5SVineet Gupta * once at init, rather than checking/setting to 0 for every runtime op 1221b5ddb6d5SVineet Gupta */ 1222b5ddb6d5SVineet Gupta if (is_isa_arcv2() && pae40_exist_but_not_enab()) { 1223b5ddb6d5SVineet Gupta 1224b5ddb6d5SVineet Gupta if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) 1225b5ddb6d5SVineet Gupta write_aux_reg(ARC_REG_IC_PTAG_HI, 0); 1226b5ddb6d5SVineet Gupta 1227b5ddb6d5SVineet Gupta if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) 1228b5ddb6d5SVineet Gupta write_aux_reg(ARC_REG_DC_PTAG_HI, 0); 1229b5ddb6d5SVineet Gupta 1230b5ddb6d5SVineet Gupta if (l2_line_sz) { 1231b5ddb6d5SVineet Gupta write_aux_reg(ARC_REG_SLC_RGN_END1, 0); 1232b5ddb6d5SVineet Gupta write_aux_reg(ARC_REG_SLC_RGN_START1, 0); 1233b5ddb6d5SVineet Gupta } 1234b5ddb6d5SVineet Gupta } 123576894a72SVineet Gupta } 1236