1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 * 5 * Derived from MIPS: 6 * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org) 7 * Copyright (C) 2007 MIPS Technologies, Inc. 8 */ 9 #include <linux/cacheinfo.h> 10 #include <linux/export.h> 11 #include <linux/fs.h> 12 #include <linux/highmem.h> 13 #include <linux/kernel.h> 14 #include <linux/linkage.h> 15 #include <linux/mm.h> 16 #include <linux/sched.h> 17 #include <linux/syscalls.h> 18 19 #include <asm/bootinfo.h> 20 #include <asm/cacheflush.h> 21 #include <asm/cpu.h> 22 #include <asm/cpu-features.h> 23 #include <asm/loongarch.h> 24 #include <asm/numa.h> 25 #include <asm/processor.h> 26 #include <asm/setup.h> 27 28 void cache_error_setup(void) 29 { 30 extern char __weak except_vec_cex; 31 set_merr_handler(0x0, &except_vec_cex, 0x80); 32 } 33 34 /* 35 * LoongArch maintains ICache/DCache coherency by hardware, 36 * we just need "ibar" to avoid instruction hazard here. 37 */ 38 void local_flush_icache_range(unsigned long start, unsigned long end) 39 { 40 asm volatile ("\tibar 0\n"::); 41 } 42 EXPORT_SYMBOL(local_flush_icache_range); 43 44 static void flush_cache_leaf(unsigned int leaf) 45 { 46 int i, j, nr_nodes; 47 uint64_t addr = CSR_DMW0_BASE; 48 struct cache_desc *cdesc = current_cpu_data.cache_leaves + leaf; 49 50 nr_nodes = cache_private(cdesc) ? 1 : loongson_sysconf.nr_nodes; 51 52 do { 53 for (i = 0; i < cdesc->sets; i++) { 54 for (j = 0; j < cdesc->ways; j++) { 55 flush_cache_line(leaf, addr); 56 addr++; 57 } 58 59 addr -= cdesc->ways; 60 addr += cdesc->linesz; 61 } 62 addr += (1ULL << NODE_ADDRSPACE_SHIFT); 63 } while (--nr_nodes > 0); 64 } 65 66 asmlinkage __visible void __flush_cache_all(void) 67 { 68 int leaf; 69 struct cache_desc *cdesc = current_cpu_data.cache_leaves; 70 unsigned int cache_present = current_cpu_data.cache_leaves_present; 71 72 leaf = cache_present - 1; 73 if (cache_inclusive(cdesc + leaf)) { 74 flush_cache_leaf(leaf); 75 return; 76 } 77 78 for (leaf = 0; leaf < cache_present; leaf++) 79 flush_cache_leaf(leaf); 80 } 81 82 #define L1IUPRE (1 << 0) 83 #define L1IUUNIFY (1 << 1) 84 #define L1DPRE (1 << 2) 85 86 #define LXIUPRE (1 << 0) 87 #define LXIUUNIFY (1 << 1) 88 #define LXIUPRIV (1 << 2) 89 #define LXIUINCL (1 << 3) 90 #define LXDPRE (1 << 4) 91 #define LXDPRIV (1 << 5) 92 #define LXDINCL (1 << 6) 93 94 #define populate_cache_properties(cfg0, cdesc, level, leaf) \ 95 do { \ 96 unsigned int cfg1; \ 97 \ 98 cfg1 = read_cpucfg(LOONGARCH_CPUCFG17 + leaf); \ 99 if (level == 1) { \ 100 cdesc->flags |= CACHE_PRIVATE; \ 101 } else { \ 102 if (cfg0 & LXIUPRIV) \ 103 cdesc->flags |= CACHE_PRIVATE; \ 104 if (cfg0 & LXIUINCL) \ 105 cdesc->flags |= CACHE_INCLUSIVE; \ 106 } \ 107 cdesc->level = level; \ 108 cdesc->flags |= CACHE_PRESENT; \ 109 cdesc->ways = ((cfg1 & CPUCFG_CACHE_WAYS_M) >> CPUCFG_CACHE_WAYS) + 1; \ 110 cdesc->sets = 1 << ((cfg1 & CPUCFG_CACHE_SETS_M) >> CPUCFG_CACHE_SETS); \ 111 cdesc->linesz = 1 << ((cfg1 & CPUCFG_CACHE_LSIZE_M) >> CPUCFG_CACHE_LSIZE); \ 112 cdesc++; leaf++; \ 113 } while (0) 114 115 void cpu_cache_init(void) 116 { 117 unsigned int leaf = 0, level = 1; 118 unsigned int config = read_cpucfg(LOONGARCH_CPUCFG16); 119 struct cache_desc *cdesc = current_cpu_data.cache_leaves; 120 121 if (config & L1IUPRE) { 122 if (config & L1IUUNIFY) 123 cdesc->type = CACHE_TYPE_UNIFIED; 124 else 125 cdesc->type = CACHE_TYPE_INST; 126 populate_cache_properties(config, cdesc, level, leaf); 127 } 128 129 if (config & L1DPRE) { 130 cdesc->type = CACHE_TYPE_DATA; 131 populate_cache_properties(config, cdesc, level, leaf); 132 } 133 134 config = config >> 3; 135 for (level = 2; level <= CACHE_LEVEL_MAX; level++) { 136 if (!config) 137 break; 138 139 if (config & LXIUPRE) { 140 if (config & LXIUUNIFY) 141 cdesc->type = CACHE_TYPE_UNIFIED; 142 else 143 cdesc->type = CACHE_TYPE_INST; 144 populate_cache_properties(config, cdesc, level, leaf); 145 } 146 147 if (config & LXDPRE) { 148 cdesc->type = CACHE_TYPE_DATA; 149 populate_cache_properties(config, cdesc, level, leaf); 150 } 151 152 config = config >> 7; 153 } 154 155 BUG_ON(leaf > CACHE_LEAVES_MAX); 156 157 current_cpu_data.cache_leaves_present = leaf; 158 current_cpu_data.options |= LOONGARCH_CPU_PREFETCH; 159 shm_align_mask = PAGE_SIZE - 1; 160 } 161 162 static const pgprot_t protection_map[16] = { 163 [VM_NONE] = __pgprot(_CACHE_CC | _PAGE_USER | 164 _PAGE_PROTNONE | _PAGE_NO_EXEC | 165 _PAGE_NO_READ), 166 [VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 167 _PAGE_USER | _PAGE_PRESENT | 168 _PAGE_NO_EXEC), 169 [VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 170 _PAGE_USER | _PAGE_PRESENT | 171 _PAGE_NO_EXEC), 172 [VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 173 _PAGE_USER | _PAGE_PRESENT | 174 _PAGE_NO_EXEC), 175 [VM_EXEC] = __pgprot(_CACHE_CC | _PAGE_VALID | 176 _PAGE_USER | _PAGE_PRESENT), 177 [VM_EXEC | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 178 _PAGE_USER | _PAGE_PRESENT), 179 [VM_EXEC | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 180 _PAGE_USER | _PAGE_PRESENT), 181 [VM_EXEC | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 182 _PAGE_USER | _PAGE_PRESENT), 183 [VM_SHARED] = __pgprot(_CACHE_CC | _PAGE_USER | 184 _PAGE_PROTNONE | _PAGE_NO_EXEC | 185 _PAGE_NO_READ), 186 [VM_SHARED | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 187 _PAGE_USER | _PAGE_PRESENT | 188 _PAGE_NO_EXEC), 189 [VM_SHARED | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 190 _PAGE_USER | _PAGE_PRESENT | 191 _PAGE_NO_EXEC | _PAGE_WRITE), 192 [VM_SHARED | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 193 _PAGE_USER | _PAGE_PRESENT | 194 _PAGE_NO_EXEC | _PAGE_WRITE), 195 [VM_SHARED | VM_EXEC] = __pgprot(_CACHE_CC | _PAGE_VALID | 196 _PAGE_USER | _PAGE_PRESENT), 197 [VM_SHARED | VM_EXEC | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 198 _PAGE_USER | _PAGE_PRESENT), 199 [VM_SHARED | VM_EXEC | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 200 _PAGE_USER | _PAGE_PRESENT | 201 _PAGE_WRITE), 202 [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 203 _PAGE_USER | _PAGE_PRESENT | 204 _PAGE_WRITE) 205 }; 206 DECLARE_VM_GET_PAGE_PROT 207