109cfefb7SHuacai Chen // SPDX-License-Identifier: GPL-2.0
209cfefb7SHuacai Chen /*
309cfefb7SHuacai Chen * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
409cfefb7SHuacai Chen *
509cfefb7SHuacai Chen * Derived from MIPS:
609cfefb7SHuacai Chen * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org)
709cfefb7SHuacai Chen * Copyright (C) 2007 MIPS Technologies, Inc.
809cfefb7SHuacai Chen */
9*b61a40afSHuacai Chen #include <linux/cacheinfo.h>
1009cfefb7SHuacai Chen #include <linux/export.h>
1109cfefb7SHuacai Chen #include <linux/fs.h>
1209cfefb7SHuacai Chen #include <linux/highmem.h>
1309cfefb7SHuacai Chen #include <linux/kernel.h>
1409cfefb7SHuacai Chen #include <linux/linkage.h>
1509cfefb7SHuacai Chen #include <linux/mm.h>
1609cfefb7SHuacai Chen #include <linux/sched.h>
1709cfefb7SHuacai Chen #include <linux/syscalls.h>
1809cfefb7SHuacai Chen
19*b61a40afSHuacai Chen #include <asm/bootinfo.h>
2009cfefb7SHuacai Chen #include <asm/cacheflush.h>
2109cfefb7SHuacai Chen #include <asm/cpu.h>
2209cfefb7SHuacai Chen #include <asm/cpu-features.h>
2309cfefb7SHuacai Chen #include <asm/loongarch.h>
24*b61a40afSHuacai Chen #include <asm/numa.h>
2509cfefb7SHuacai Chen #include <asm/processor.h>
2609cfefb7SHuacai Chen #include <asm/setup.h>
2709cfefb7SHuacai Chen
cache_error_setup(void)28*b61a40afSHuacai Chen void cache_error_setup(void)
29*b61a40afSHuacai Chen {
30*b61a40afSHuacai Chen extern char __weak except_vec_cex;
31*b61a40afSHuacai Chen set_merr_handler(0x0, &except_vec_cex, 0x80);
32*b61a40afSHuacai Chen }
33*b61a40afSHuacai Chen
3409cfefb7SHuacai Chen /*
3509cfefb7SHuacai Chen * LoongArch maintains ICache/DCache coherency by hardware,
3609cfefb7SHuacai Chen * we just need "ibar" to avoid instruction hazard here.
3709cfefb7SHuacai Chen */
local_flush_icache_range(unsigned long start,unsigned long end)3809cfefb7SHuacai Chen void local_flush_icache_range(unsigned long start, unsigned long end)
3909cfefb7SHuacai Chen {
4009cfefb7SHuacai Chen asm volatile ("\tibar 0\n"::);
4109cfefb7SHuacai Chen }
4209cfefb7SHuacai Chen EXPORT_SYMBOL(local_flush_icache_range);
4309cfefb7SHuacai Chen
flush_cache_leaf(unsigned int leaf)44*b61a40afSHuacai Chen static void flush_cache_leaf(unsigned int leaf)
4509cfefb7SHuacai Chen {
46*b61a40afSHuacai Chen int i, j, nr_nodes;
47*b61a40afSHuacai Chen uint64_t addr = CSR_DMW0_BASE;
48*b61a40afSHuacai Chen struct cache_desc *cdesc = current_cpu_data.cache_leaves + leaf;
49*b61a40afSHuacai Chen
50*b61a40afSHuacai Chen nr_nodes = cache_private(cdesc) ? 1 : loongson_sysconf.nr_nodes;
51*b61a40afSHuacai Chen
52*b61a40afSHuacai Chen do {
53*b61a40afSHuacai Chen for (i = 0; i < cdesc->sets; i++) {
54*b61a40afSHuacai Chen for (j = 0; j < cdesc->ways; j++) {
55*b61a40afSHuacai Chen flush_cache_line(leaf, addr);
56*b61a40afSHuacai Chen addr++;
5709cfefb7SHuacai Chen }
5809cfefb7SHuacai Chen
59*b61a40afSHuacai Chen addr -= cdesc->ways;
60*b61a40afSHuacai Chen addr += cdesc->linesz;
61*b61a40afSHuacai Chen }
62*b61a40afSHuacai Chen addr += (1ULL << NODE_ADDRSPACE_SHIFT);
63*b61a40afSHuacai Chen } while (--nr_nodes > 0);
6409cfefb7SHuacai Chen }
6509cfefb7SHuacai Chen
__flush_cache_all(void)66*b61a40afSHuacai Chen asmlinkage __visible void __flush_cache_all(void)
6709cfefb7SHuacai Chen {
68*b61a40afSHuacai Chen int leaf;
69*b61a40afSHuacai Chen struct cache_desc *cdesc = current_cpu_data.cache_leaves;
70*b61a40afSHuacai Chen unsigned int cache_present = current_cpu_data.cache_leaves_present;
7109cfefb7SHuacai Chen
72*b61a40afSHuacai Chen leaf = cache_present - 1;
73*b61a40afSHuacai Chen if (cache_inclusive(cdesc + leaf)) {
74*b61a40afSHuacai Chen flush_cache_leaf(leaf);
75*b61a40afSHuacai Chen return;
7609cfefb7SHuacai Chen }
7709cfefb7SHuacai Chen
78*b61a40afSHuacai Chen for (leaf = 0; leaf < cache_present; leaf++)
79*b61a40afSHuacai Chen flush_cache_leaf(leaf);
8009cfefb7SHuacai Chen }
8109cfefb7SHuacai Chen
82*b61a40afSHuacai Chen #define L1IUPRE (1 << 0)
83*b61a40afSHuacai Chen #define L1IUUNIFY (1 << 1)
84*b61a40afSHuacai Chen #define L1DPRE (1 << 2)
85*b61a40afSHuacai Chen
86*b61a40afSHuacai Chen #define LXIUPRE (1 << 0)
87*b61a40afSHuacai Chen #define LXIUUNIFY (1 << 1)
88*b61a40afSHuacai Chen #define LXIUPRIV (1 << 2)
89*b61a40afSHuacai Chen #define LXIUINCL (1 << 3)
90*b61a40afSHuacai Chen #define LXDPRE (1 << 4)
91*b61a40afSHuacai Chen #define LXDPRIV (1 << 5)
92*b61a40afSHuacai Chen #define LXDINCL (1 << 6)
93*b61a40afSHuacai Chen
94*b61a40afSHuacai Chen #define populate_cache_properties(cfg0, cdesc, level, leaf) \
95*b61a40afSHuacai Chen do { \
96*b61a40afSHuacai Chen unsigned int cfg1; \
97*b61a40afSHuacai Chen \
98*b61a40afSHuacai Chen cfg1 = read_cpucfg(LOONGARCH_CPUCFG17 + leaf); \
99*b61a40afSHuacai Chen if (level == 1) { \
100*b61a40afSHuacai Chen cdesc->flags |= CACHE_PRIVATE; \
101*b61a40afSHuacai Chen } else { \
102*b61a40afSHuacai Chen if (cfg0 & LXIUPRIV) \
103*b61a40afSHuacai Chen cdesc->flags |= CACHE_PRIVATE; \
104*b61a40afSHuacai Chen if (cfg0 & LXIUINCL) \
105*b61a40afSHuacai Chen cdesc->flags |= CACHE_INCLUSIVE; \
106*b61a40afSHuacai Chen } \
107*b61a40afSHuacai Chen cdesc->level = level; \
108*b61a40afSHuacai Chen cdesc->flags |= CACHE_PRESENT; \
109*b61a40afSHuacai Chen cdesc->ways = ((cfg1 & CPUCFG_CACHE_WAYS_M) >> CPUCFG_CACHE_WAYS) + 1; \
110*b61a40afSHuacai Chen cdesc->sets = 1 << ((cfg1 & CPUCFG_CACHE_SETS_M) >> CPUCFG_CACHE_SETS); \
111*b61a40afSHuacai Chen cdesc->linesz = 1 << ((cfg1 & CPUCFG_CACHE_LSIZE_M) >> CPUCFG_CACHE_LSIZE); \
112*b61a40afSHuacai Chen cdesc++; leaf++; \
113*b61a40afSHuacai Chen } while (0)
114*b61a40afSHuacai Chen
cpu_cache_init(void)11509cfefb7SHuacai Chen void cpu_cache_init(void)
11609cfefb7SHuacai Chen {
117*b61a40afSHuacai Chen unsigned int leaf = 0, level = 1;
118*b61a40afSHuacai Chen unsigned int config = read_cpucfg(LOONGARCH_CPUCFG16);
119*b61a40afSHuacai Chen struct cache_desc *cdesc = current_cpu_data.cache_leaves;
12009cfefb7SHuacai Chen
121*b61a40afSHuacai Chen if (config & L1IUPRE) {
122*b61a40afSHuacai Chen if (config & L1IUUNIFY)
123*b61a40afSHuacai Chen cdesc->type = CACHE_TYPE_UNIFIED;
124*b61a40afSHuacai Chen else
125*b61a40afSHuacai Chen cdesc->type = CACHE_TYPE_INST;
126*b61a40afSHuacai Chen populate_cache_properties(config, cdesc, level, leaf);
127*b61a40afSHuacai Chen }
128*b61a40afSHuacai Chen
129*b61a40afSHuacai Chen if (config & L1DPRE) {
130*b61a40afSHuacai Chen cdesc->type = CACHE_TYPE_DATA;
131*b61a40afSHuacai Chen populate_cache_properties(config, cdesc, level, leaf);
132*b61a40afSHuacai Chen }
133*b61a40afSHuacai Chen
134*b61a40afSHuacai Chen config = config >> 3;
135*b61a40afSHuacai Chen for (level = 2; level <= CACHE_LEVEL_MAX; level++) {
136*b61a40afSHuacai Chen if (!config)
137*b61a40afSHuacai Chen break;
138*b61a40afSHuacai Chen
139*b61a40afSHuacai Chen if (config & LXIUPRE) {
140*b61a40afSHuacai Chen if (config & LXIUUNIFY)
141*b61a40afSHuacai Chen cdesc->type = CACHE_TYPE_UNIFIED;
142*b61a40afSHuacai Chen else
143*b61a40afSHuacai Chen cdesc->type = CACHE_TYPE_INST;
144*b61a40afSHuacai Chen populate_cache_properties(config, cdesc, level, leaf);
145*b61a40afSHuacai Chen }
146*b61a40afSHuacai Chen
147*b61a40afSHuacai Chen if (config & LXDPRE) {
148*b61a40afSHuacai Chen cdesc->type = CACHE_TYPE_DATA;
149*b61a40afSHuacai Chen populate_cache_properties(config, cdesc, level, leaf);
150*b61a40afSHuacai Chen }
151*b61a40afSHuacai Chen
152*b61a40afSHuacai Chen config = config >> 7;
153*b61a40afSHuacai Chen }
154*b61a40afSHuacai Chen
155*b61a40afSHuacai Chen BUG_ON(leaf > CACHE_LEAVES_MAX);
156*b61a40afSHuacai Chen
157*b61a40afSHuacai Chen current_cpu_data.cache_leaves_present = leaf;
158*b61a40afSHuacai Chen current_cpu_data.options |= LOONGARCH_CPU_PREFETCH;
15909cfefb7SHuacai Chen }
160f6d1e19cSAnshuman Khandual
161f6d1e19cSAnshuman Khandual static const pgprot_t protection_map[16] = {
162f6d1e19cSAnshuman Khandual [VM_NONE] = __pgprot(_CACHE_CC | _PAGE_USER |
163f6d1e19cSAnshuman Khandual _PAGE_PROTNONE | _PAGE_NO_EXEC |
164f6d1e19cSAnshuman Khandual _PAGE_NO_READ),
165f6d1e19cSAnshuman Khandual [VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
166f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT |
167f6d1e19cSAnshuman Khandual _PAGE_NO_EXEC),
168f6d1e19cSAnshuman Khandual [VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID |
169f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT |
170f6d1e19cSAnshuman Khandual _PAGE_NO_EXEC),
171f6d1e19cSAnshuman Khandual [VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
172f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT |
173f6d1e19cSAnshuman Khandual _PAGE_NO_EXEC),
174f6d1e19cSAnshuman Khandual [VM_EXEC] = __pgprot(_CACHE_CC | _PAGE_VALID |
175f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT),
176f6d1e19cSAnshuman Khandual [VM_EXEC | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
177f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT),
178f6d1e19cSAnshuman Khandual [VM_EXEC | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID |
179f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT),
180f6d1e19cSAnshuman Khandual [VM_EXEC | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
181f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT),
182f6d1e19cSAnshuman Khandual [VM_SHARED] = __pgprot(_CACHE_CC | _PAGE_USER |
183f6d1e19cSAnshuman Khandual _PAGE_PROTNONE | _PAGE_NO_EXEC |
184f6d1e19cSAnshuman Khandual _PAGE_NO_READ),
185f6d1e19cSAnshuman Khandual [VM_SHARED | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
186f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT |
187f6d1e19cSAnshuman Khandual _PAGE_NO_EXEC),
188f6d1e19cSAnshuman Khandual [VM_SHARED | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID |
189f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT |
190f6d1e19cSAnshuman Khandual _PAGE_NO_EXEC | _PAGE_WRITE),
191f6d1e19cSAnshuman Khandual [VM_SHARED | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
192f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT |
193f6d1e19cSAnshuman Khandual _PAGE_NO_EXEC | _PAGE_WRITE),
194f6d1e19cSAnshuman Khandual [VM_SHARED | VM_EXEC] = __pgprot(_CACHE_CC | _PAGE_VALID |
195f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT),
196f6d1e19cSAnshuman Khandual [VM_SHARED | VM_EXEC | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
197f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT),
198f6d1e19cSAnshuman Khandual [VM_SHARED | VM_EXEC | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID |
199f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT |
200f6d1e19cSAnshuman Khandual _PAGE_WRITE),
201f6d1e19cSAnshuman Khandual [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
202f6d1e19cSAnshuman Khandual _PAGE_USER | _PAGE_PRESENT |
203f6d1e19cSAnshuman Khandual _PAGE_WRITE)
204f6d1e19cSAnshuman Khandual };
205f6d1e19cSAnshuman Khandual DECLARE_VM_GET_PAGE_PROT
206