xref: /openbmc/linux/arch/loongarch/mm/cache.c (revision ad3ff105)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4  *
5  * Derived from MIPS:
6  * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org)
7  * Copyright (C) 2007 MIPS Technologies, Inc.
8  */
9 #include <linux/cacheinfo.h>
10 #include <linux/export.h>
11 #include <linux/fs.h>
12 #include <linux/highmem.h>
13 #include <linux/kernel.h>
14 #include <linux/linkage.h>
15 #include <linux/mm.h>
16 #include <linux/sched.h>
17 #include <linux/syscalls.h>
18 
19 #include <asm/bootinfo.h>
20 #include <asm/cacheflush.h>
21 #include <asm/cpu.h>
22 #include <asm/cpu-features.h>
23 #include <asm/loongarch.h>
24 #include <asm/numa.h>
25 #include <asm/processor.h>
26 #include <asm/setup.h>
27 
cache_error_setup(void)28 void cache_error_setup(void)
29 {
30 	extern char __weak except_vec_cex;
31 	set_merr_handler(0x0, &except_vec_cex, 0x80);
32 }
33 
34 /*
35  * LoongArch maintains ICache/DCache coherency by hardware,
36  * we just need "ibar" to avoid instruction hazard here.
37  */
local_flush_icache_range(unsigned long start,unsigned long end)38 void local_flush_icache_range(unsigned long start, unsigned long end)
39 {
40 	asm volatile ("\tibar 0\n"::);
41 }
42 EXPORT_SYMBOL(local_flush_icache_range);
43 
flush_cache_leaf(unsigned int leaf)44 static void flush_cache_leaf(unsigned int leaf)
45 {
46 	int i, j, nr_nodes;
47 	uint64_t addr = CSR_DMW0_BASE;
48 	struct cache_desc *cdesc = current_cpu_data.cache_leaves + leaf;
49 
50 	nr_nodes = cache_private(cdesc) ? 1 : loongson_sysconf.nr_nodes;
51 
52 	do {
53 		for (i = 0; i < cdesc->sets; i++) {
54 			for (j = 0; j < cdesc->ways; j++) {
55 				flush_cache_line(leaf, addr);
56 				addr++;
57 			}
58 
59 			addr -= cdesc->ways;
60 			addr += cdesc->linesz;
61 		}
62 		addr += (1ULL << NODE_ADDRSPACE_SHIFT);
63 	} while (--nr_nodes > 0);
64 }
65 
__flush_cache_all(void)66 asmlinkage __visible void __flush_cache_all(void)
67 {
68 	int leaf;
69 	struct cache_desc *cdesc = current_cpu_data.cache_leaves;
70 	unsigned int cache_present = current_cpu_data.cache_leaves_present;
71 
72 	leaf = cache_present - 1;
73 	if (cache_inclusive(cdesc + leaf)) {
74 		flush_cache_leaf(leaf);
75 		return;
76 	}
77 
78 	for (leaf = 0; leaf < cache_present; leaf++)
79 		flush_cache_leaf(leaf);
80 }
81 
82 #define L1IUPRE		(1 << 0)
83 #define L1IUUNIFY	(1 << 1)
84 #define L1DPRE		(1 << 2)
85 
86 #define LXIUPRE		(1 << 0)
87 #define LXIUUNIFY	(1 << 1)
88 #define LXIUPRIV	(1 << 2)
89 #define LXIUINCL	(1 << 3)
90 #define LXDPRE		(1 << 4)
91 #define LXDPRIV		(1 << 5)
92 #define LXDINCL		(1 << 6)
93 
94 #define populate_cache_properties(cfg0, cdesc, level, leaf)				\
95 do {											\
96 	unsigned int cfg1;								\
97 											\
98 	cfg1 = read_cpucfg(LOONGARCH_CPUCFG17 + leaf);					\
99 	if (level == 1)	{								\
100 		cdesc->flags |= CACHE_PRIVATE;						\
101 	} else {									\
102 		if (cfg0 & LXIUPRIV)							\
103 			cdesc->flags |= CACHE_PRIVATE;					\
104 		if (cfg0 & LXIUINCL)							\
105 			cdesc->flags |= CACHE_INCLUSIVE;				\
106 	}										\
107 	cdesc->level = level;								\
108 	cdesc->flags |= CACHE_PRESENT;							\
109 	cdesc->ways = ((cfg1 & CPUCFG_CACHE_WAYS_M) >> CPUCFG_CACHE_WAYS) + 1;		\
110 	cdesc->sets = 1 << ((cfg1 & CPUCFG_CACHE_SETS_M) >> CPUCFG_CACHE_SETS);		\
111 	cdesc->linesz = 1 << ((cfg1 & CPUCFG_CACHE_LSIZE_M) >> CPUCFG_CACHE_LSIZE);	\
112 	cdesc++; leaf++;								\
113 } while (0)
114 
cpu_cache_init(void)115 void cpu_cache_init(void)
116 {
117 	unsigned int leaf = 0, level = 1;
118 	unsigned int config = read_cpucfg(LOONGARCH_CPUCFG16);
119 	struct cache_desc *cdesc = current_cpu_data.cache_leaves;
120 
121 	if (config & L1IUPRE) {
122 		if (config & L1IUUNIFY)
123 			cdesc->type = CACHE_TYPE_UNIFIED;
124 		else
125 			cdesc->type = CACHE_TYPE_INST;
126 		populate_cache_properties(config, cdesc, level, leaf);
127 	}
128 
129 	if (config & L1DPRE) {
130 		cdesc->type = CACHE_TYPE_DATA;
131 		populate_cache_properties(config, cdesc, level, leaf);
132 	}
133 
134 	config = config >> 3;
135 	for (level = 2; level <= CACHE_LEVEL_MAX; level++) {
136 		if (!config)
137 			break;
138 
139 		if (config & LXIUPRE) {
140 			if (config & LXIUUNIFY)
141 				cdesc->type = CACHE_TYPE_UNIFIED;
142 			else
143 				cdesc->type = CACHE_TYPE_INST;
144 			populate_cache_properties(config, cdesc, level, leaf);
145 		}
146 
147 		if (config & LXDPRE) {
148 			cdesc->type = CACHE_TYPE_DATA;
149 			populate_cache_properties(config, cdesc, level, leaf);
150 		}
151 
152 		config = config >> 7;
153 	}
154 
155 	BUG_ON(leaf > CACHE_LEAVES_MAX);
156 
157 	current_cpu_data.cache_leaves_present = leaf;
158 	current_cpu_data.options |= LOONGARCH_CPU_PREFETCH;
159 }
160 
161 static const pgprot_t protection_map[16] = {
162 	[VM_NONE]					= __pgprot(_CACHE_CC | _PAGE_USER |
163 								   _PAGE_PROTNONE | _PAGE_NO_EXEC |
164 								   _PAGE_NO_READ),
165 	[VM_READ]					= __pgprot(_CACHE_CC | _PAGE_VALID |
166 								   _PAGE_USER | _PAGE_PRESENT |
167 								   _PAGE_NO_EXEC),
168 	[VM_WRITE]					= __pgprot(_CACHE_CC | _PAGE_VALID |
169 								   _PAGE_USER | _PAGE_PRESENT |
170 								   _PAGE_NO_EXEC),
171 	[VM_WRITE | VM_READ]				= __pgprot(_CACHE_CC | _PAGE_VALID |
172 								   _PAGE_USER | _PAGE_PRESENT |
173 								   _PAGE_NO_EXEC),
174 	[VM_EXEC]					= __pgprot(_CACHE_CC | _PAGE_VALID |
175 								   _PAGE_USER | _PAGE_PRESENT),
176 	[VM_EXEC | VM_READ]				= __pgprot(_CACHE_CC | _PAGE_VALID |
177 								   _PAGE_USER | _PAGE_PRESENT),
178 	[VM_EXEC | VM_WRITE]				= __pgprot(_CACHE_CC | _PAGE_VALID |
179 								   _PAGE_USER | _PAGE_PRESENT),
180 	[VM_EXEC | VM_WRITE | VM_READ]			= __pgprot(_CACHE_CC | _PAGE_VALID |
181 								   _PAGE_USER | _PAGE_PRESENT),
182 	[VM_SHARED]					= __pgprot(_CACHE_CC | _PAGE_USER |
183 								   _PAGE_PROTNONE | _PAGE_NO_EXEC |
184 								   _PAGE_NO_READ),
185 	[VM_SHARED | VM_READ]				= __pgprot(_CACHE_CC | _PAGE_VALID |
186 								   _PAGE_USER | _PAGE_PRESENT |
187 								   _PAGE_NO_EXEC),
188 	[VM_SHARED | VM_WRITE]				= __pgprot(_CACHE_CC | _PAGE_VALID |
189 								   _PAGE_USER | _PAGE_PRESENT |
190 								   _PAGE_NO_EXEC | _PAGE_WRITE),
191 	[VM_SHARED | VM_WRITE | VM_READ]		= __pgprot(_CACHE_CC | _PAGE_VALID |
192 								   _PAGE_USER | _PAGE_PRESENT |
193 								   _PAGE_NO_EXEC | _PAGE_WRITE),
194 	[VM_SHARED | VM_EXEC]				= __pgprot(_CACHE_CC | _PAGE_VALID |
195 								   _PAGE_USER | _PAGE_PRESENT),
196 	[VM_SHARED | VM_EXEC | VM_READ]			= __pgprot(_CACHE_CC | _PAGE_VALID |
197 								   _PAGE_USER | _PAGE_PRESENT),
198 	[VM_SHARED | VM_EXEC | VM_WRITE]		= __pgprot(_CACHE_CC | _PAGE_VALID |
199 								   _PAGE_USER | _PAGE_PRESENT |
200 								   _PAGE_WRITE),
201 	[VM_SHARED | VM_EXEC | VM_WRITE | VM_READ]	= __pgprot(_CACHE_CC | _PAGE_VALID |
202 								   _PAGE_USER | _PAGE_PRESENT |
203 								   _PAGE_WRITE)
204 };
205 DECLARE_VM_GET_PAGE_PROT
206