109cfefb7SHuacai Chen // SPDX-License-Identifier: GPL-2.0 209cfefb7SHuacai Chen /* 309cfefb7SHuacai Chen * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 409cfefb7SHuacai Chen * 509cfefb7SHuacai Chen * Derived from MIPS: 609cfefb7SHuacai Chen * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org) 709cfefb7SHuacai Chen * Copyright (C) 2007 MIPS Technologies, Inc. 809cfefb7SHuacai Chen */ 9*b61a40afSHuacai Chen #include <linux/cacheinfo.h> 1009cfefb7SHuacai Chen #include <linux/export.h> 1109cfefb7SHuacai Chen #include <linux/fs.h> 1209cfefb7SHuacai Chen #include <linux/highmem.h> 1309cfefb7SHuacai Chen #include <linux/kernel.h> 1409cfefb7SHuacai Chen #include <linux/linkage.h> 1509cfefb7SHuacai Chen #include <linux/mm.h> 1609cfefb7SHuacai Chen #include <linux/sched.h> 1709cfefb7SHuacai Chen #include <linux/syscalls.h> 1809cfefb7SHuacai Chen 19*b61a40afSHuacai Chen #include <asm/bootinfo.h> 2009cfefb7SHuacai Chen #include <asm/cacheflush.h> 2109cfefb7SHuacai Chen #include <asm/cpu.h> 2209cfefb7SHuacai Chen #include <asm/cpu-features.h> 2309cfefb7SHuacai Chen #include <asm/loongarch.h> 24*b61a40afSHuacai Chen #include <asm/numa.h> 2509cfefb7SHuacai Chen #include <asm/processor.h> 2609cfefb7SHuacai Chen #include <asm/setup.h> 2709cfefb7SHuacai Chen 28*b61a40afSHuacai Chen void cache_error_setup(void) 29*b61a40afSHuacai Chen { 30*b61a40afSHuacai Chen extern char __weak except_vec_cex; 31*b61a40afSHuacai Chen set_merr_handler(0x0, &except_vec_cex, 0x80); 32*b61a40afSHuacai Chen } 33*b61a40afSHuacai Chen 3409cfefb7SHuacai Chen /* 3509cfefb7SHuacai Chen * LoongArch maintains ICache/DCache coherency by hardware, 3609cfefb7SHuacai Chen * we just need "ibar" to avoid instruction hazard here. 3709cfefb7SHuacai Chen */ 3809cfefb7SHuacai Chen void local_flush_icache_range(unsigned long start, unsigned long end) 3909cfefb7SHuacai Chen { 4009cfefb7SHuacai Chen asm volatile ("\tibar 0\n"::); 4109cfefb7SHuacai Chen } 4209cfefb7SHuacai Chen EXPORT_SYMBOL(local_flush_icache_range); 4309cfefb7SHuacai Chen 44*b61a40afSHuacai Chen static void flush_cache_leaf(unsigned int leaf) 4509cfefb7SHuacai Chen { 46*b61a40afSHuacai Chen int i, j, nr_nodes; 47*b61a40afSHuacai Chen uint64_t addr = CSR_DMW0_BASE; 48*b61a40afSHuacai Chen struct cache_desc *cdesc = current_cpu_data.cache_leaves + leaf; 49*b61a40afSHuacai Chen 50*b61a40afSHuacai Chen nr_nodes = cache_private(cdesc) ? 1 : loongson_sysconf.nr_nodes; 51*b61a40afSHuacai Chen 52*b61a40afSHuacai Chen do { 53*b61a40afSHuacai Chen for (i = 0; i < cdesc->sets; i++) { 54*b61a40afSHuacai Chen for (j = 0; j < cdesc->ways; j++) { 55*b61a40afSHuacai Chen flush_cache_line(leaf, addr); 56*b61a40afSHuacai Chen addr++; 5709cfefb7SHuacai Chen } 5809cfefb7SHuacai Chen 59*b61a40afSHuacai Chen addr -= cdesc->ways; 60*b61a40afSHuacai Chen addr += cdesc->linesz; 61*b61a40afSHuacai Chen } 62*b61a40afSHuacai Chen addr += (1ULL << NODE_ADDRSPACE_SHIFT); 63*b61a40afSHuacai Chen } while (--nr_nodes > 0); 6409cfefb7SHuacai Chen } 6509cfefb7SHuacai Chen 66*b61a40afSHuacai Chen asmlinkage __visible void __flush_cache_all(void) 6709cfefb7SHuacai Chen { 68*b61a40afSHuacai Chen int leaf; 69*b61a40afSHuacai Chen struct cache_desc *cdesc = current_cpu_data.cache_leaves; 70*b61a40afSHuacai Chen unsigned int cache_present = current_cpu_data.cache_leaves_present; 7109cfefb7SHuacai Chen 72*b61a40afSHuacai Chen leaf = cache_present - 1; 73*b61a40afSHuacai Chen if (cache_inclusive(cdesc + leaf)) { 74*b61a40afSHuacai Chen flush_cache_leaf(leaf); 75*b61a40afSHuacai Chen return; 7609cfefb7SHuacai Chen } 7709cfefb7SHuacai Chen 78*b61a40afSHuacai Chen for (leaf = 0; leaf < cache_present; leaf++) 79*b61a40afSHuacai Chen flush_cache_leaf(leaf); 8009cfefb7SHuacai Chen } 8109cfefb7SHuacai Chen 82*b61a40afSHuacai Chen #define L1IUPRE (1 << 0) 83*b61a40afSHuacai Chen #define L1IUUNIFY (1 << 1) 84*b61a40afSHuacai Chen #define L1DPRE (1 << 2) 85*b61a40afSHuacai Chen 86*b61a40afSHuacai Chen #define LXIUPRE (1 << 0) 87*b61a40afSHuacai Chen #define LXIUUNIFY (1 << 1) 88*b61a40afSHuacai Chen #define LXIUPRIV (1 << 2) 89*b61a40afSHuacai Chen #define LXIUINCL (1 << 3) 90*b61a40afSHuacai Chen #define LXDPRE (1 << 4) 91*b61a40afSHuacai Chen #define LXDPRIV (1 << 5) 92*b61a40afSHuacai Chen #define LXDINCL (1 << 6) 93*b61a40afSHuacai Chen 94*b61a40afSHuacai Chen #define populate_cache_properties(cfg0, cdesc, level, leaf) \ 95*b61a40afSHuacai Chen do { \ 96*b61a40afSHuacai Chen unsigned int cfg1; \ 97*b61a40afSHuacai Chen \ 98*b61a40afSHuacai Chen cfg1 = read_cpucfg(LOONGARCH_CPUCFG17 + leaf); \ 99*b61a40afSHuacai Chen if (level == 1) { \ 100*b61a40afSHuacai Chen cdesc->flags |= CACHE_PRIVATE; \ 101*b61a40afSHuacai Chen } else { \ 102*b61a40afSHuacai Chen if (cfg0 & LXIUPRIV) \ 103*b61a40afSHuacai Chen cdesc->flags |= CACHE_PRIVATE; \ 104*b61a40afSHuacai Chen if (cfg0 & LXIUINCL) \ 105*b61a40afSHuacai Chen cdesc->flags |= CACHE_INCLUSIVE; \ 106*b61a40afSHuacai Chen } \ 107*b61a40afSHuacai Chen cdesc->level = level; \ 108*b61a40afSHuacai Chen cdesc->flags |= CACHE_PRESENT; \ 109*b61a40afSHuacai Chen cdesc->ways = ((cfg1 & CPUCFG_CACHE_WAYS_M) >> CPUCFG_CACHE_WAYS) + 1; \ 110*b61a40afSHuacai Chen cdesc->sets = 1 << ((cfg1 & CPUCFG_CACHE_SETS_M) >> CPUCFG_CACHE_SETS); \ 111*b61a40afSHuacai Chen cdesc->linesz = 1 << ((cfg1 & CPUCFG_CACHE_LSIZE_M) >> CPUCFG_CACHE_LSIZE); \ 112*b61a40afSHuacai Chen cdesc++; leaf++; \ 113*b61a40afSHuacai Chen } while (0) 114*b61a40afSHuacai Chen 11509cfefb7SHuacai Chen void cpu_cache_init(void) 11609cfefb7SHuacai Chen { 117*b61a40afSHuacai Chen unsigned int leaf = 0, level = 1; 118*b61a40afSHuacai Chen unsigned int config = read_cpucfg(LOONGARCH_CPUCFG16); 119*b61a40afSHuacai Chen struct cache_desc *cdesc = current_cpu_data.cache_leaves; 12009cfefb7SHuacai Chen 121*b61a40afSHuacai Chen if (config & L1IUPRE) { 122*b61a40afSHuacai Chen if (config & L1IUUNIFY) 123*b61a40afSHuacai Chen cdesc->type = CACHE_TYPE_UNIFIED; 124*b61a40afSHuacai Chen else 125*b61a40afSHuacai Chen cdesc->type = CACHE_TYPE_INST; 126*b61a40afSHuacai Chen populate_cache_properties(config, cdesc, level, leaf); 127*b61a40afSHuacai Chen } 128*b61a40afSHuacai Chen 129*b61a40afSHuacai Chen if (config & L1DPRE) { 130*b61a40afSHuacai Chen cdesc->type = CACHE_TYPE_DATA; 131*b61a40afSHuacai Chen populate_cache_properties(config, cdesc, level, leaf); 132*b61a40afSHuacai Chen } 133*b61a40afSHuacai Chen 134*b61a40afSHuacai Chen config = config >> 3; 135*b61a40afSHuacai Chen for (level = 2; level <= CACHE_LEVEL_MAX; level++) { 136*b61a40afSHuacai Chen if (!config) 137*b61a40afSHuacai Chen break; 138*b61a40afSHuacai Chen 139*b61a40afSHuacai Chen if (config & LXIUPRE) { 140*b61a40afSHuacai Chen if (config & LXIUUNIFY) 141*b61a40afSHuacai Chen cdesc->type = CACHE_TYPE_UNIFIED; 142*b61a40afSHuacai Chen else 143*b61a40afSHuacai Chen cdesc->type = CACHE_TYPE_INST; 144*b61a40afSHuacai Chen populate_cache_properties(config, cdesc, level, leaf); 145*b61a40afSHuacai Chen } 146*b61a40afSHuacai Chen 147*b61a40afSHuacai Chen if (config & LXDPRE) { 148*b61a40afSHuacai Chen cdesc->type = CACHE_TYPE_DATA; 149*b61a40afSHuacai Chen populate_cache_properties(config, cdesc, level, leaf); 150*b61a40afSHuacai Chen } 151*b61a40afSHuacai Chen 152*b61a40afSHuacai Chen config = config >> 7; 153*b61a40afSHuacai Chen } 154*b61a40afSHuacai Chen 155*b61a40afSHuacai Chen BUG_ON(leaf > CACHE_LEAVES_MAX); 156*b61a40afSHuacai Chen 157*b61a40afSHuacai Chen current_cpu_data.cache_leaves_present = leaf; 158*b61a40afSHuacai Chen current_cpu_data.options |= LOONGARCH_CPU_PREFETCH; 15909cfefb7SHuacai Chen shm_align_mask = PAGE_SIZE - 1; 16009cfefb7SHuacai Chen } 161f6d1e19cSAnshuman Khandual 162f6d1e19cSAnshuman Khandual static const pgprot_t protection_map[16] = { 163f6d1e19cSAnshuman Khandual [VM_NONE] = __pgprot(_CACHE_CC | _PAGE_USER | 164f6d1e19cSAnshuman Khandual _PAGE_PROTNONE | _PAGE_NO_EXEC | 165f6d1e19cSAnshuman Khandual _PAGE_NO_READ), 166f6d1e19cSAnshuman Khandual [VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 167f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT | 168f6d1e19cSAnshuman Khandual _PAGE_NO_EXEC), 169f6d1e19cSAnshuman Khandual [VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 170f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT | 171f6d1e19cSAnshuman Khandual _PAGE_NO_EXEC), 172f6d1e19cSAnshuman Khandual [VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 173f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT | 174f6d1e19cSAnshuman Khandual _PAGE_NO_EXEC), 175f6d1e19cSAnshuman Khandual [VM_EXEC] = __pgprot(_CACHE_CC | _PAGE_VALID | 176f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT), 177f6d1e19cSAnshuman Khandual [VM_EXEC | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 178f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT), 179f6d1e19cSAnshuman Khandual [VM_EXEC | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 180f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT), 181f6d1e19cSAnshuman Khandual [VM_EXEC | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 182f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT), 183f6d1e19cSAnshuman Khandual [VM_SHARED] = __pgprot(_CACHE_CC | _PAGE_USER | 184f6d1e19cSAnshuman Khandual _PAGE_PROTNONE | _PAGE_NO_EXEC | 185f6d1e19cSAnshuman Khandual _PAGE_NO_READ), 186f6d1e19cSAnshuman Khandual [VM_SHARED | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 187f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT | 188f6d1e19cSAnshuman Khandual _PAGE_NO_EXEC), 189f6d1e19cSAnshuman Khandual [VM_SHARED | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 190f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT | 191f6d1e19cSAnshuman Khandual _PAGE_NO_EXEC | _PAGE_WRITE), 192f6d1e19cSAnshuman Khandual [VM_SHARED | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 193f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT | 194f6d1e19cSAnshuman Khandual _PAGE_NO_EXEC | _PAGE_WRITE), 195f6d1e19cSAnshuman Khandual [VM_SHARED | VM_EXEC] = __pgprot(_CACHE_CC | _PAGE_VALID | 196f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT), 197f6d1e19cSAnshuman Khandual [VM_SHARED | VM_EXEC | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 198f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT), 199f6d1e19cSAnshuman Khandual [VM_SHARED | VM_EXEC | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 200f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT | 201f6d1e19cSAnshuman Khandual _PAGE_WRITE), 202f6d1e19cSAnshuman Khandual [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 203f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT | 204f6d1e19cSAnshuman Khandual _PAGE_WRITE) 205f6d1e19cSAnshuman Khandual }; 206f6d1e19cSAnshuman Khandual DECLARE_VM_GET_PAGE_PROT 207