1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Copyright (C) 2012 Regents of the University of California
4  */
5 
6 #ifndef _ASM_RISCV_PGTABLE_64_H
7 #define _ASM_RISCV_PGTABLE_64_H
8 
9 #include <linux/bits.h>
10 #include <linux/const.h>
11 #include <asm/errata_list.h>
12 
13 extern bool pgtable_l4_enabled;
14 extern bool pgtable_l5_enabled;
15 
16 #define PGDIR_SHIFT_L3  30
17 #define PGDIR_SHIFT_L4  39
18 #define PGDIR_SHIFT_L5  48
19 #define PGDIR_SIZE_L3   (_AC(1, UL) << PGDIR_SHIFT_L3)
20 
21 #define PGDIR_SHIFT     (pgtable_l5_enabled ? PGDIR_SHIFT_L5 : \
22 		(pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3))
23 /* Size of region mapped by a page global directory */
24 #define PGDIR_SIZE      (_AC(1, UL) << PGDIR_SHIFT)
25 #define PGDIR_MASK      (~(PGDIR_SIZE - 1))
26 
27 /* p4d is folded into pgd in case of 4-level page table */
28 #define P4D_SHIFT      39
29 #define P4D_SIZE       (_AC(1, UL) << P4D_SHIFT)
30 #define P4D_MASK       (~(P4D_SIZE - 1))
31 
32 /* pud is folded into pgd in case of 3-level page table */
33 #define PUD_SHIFT      30
34 #define PUD_SIZE       (_AC(1, UL) << PUD_SHIFT)
35 #define PUD_MASK       (~(PUD_SIZE - 1))
36 
37 #define PMD_SHIFT       21
38 /* Size of region mapped by a page middle directory */
39 #define PMD_SIZE        (_AC(1, UL) << PMD_SHIFT)
40 #define PMD_MASK        (~(PMD_SIZE - 1))
41 
42 /* Page 4th Directory entry */
43 typedef struct {
44 	unsigned long p4d;
45 } p4d_t;
46 
47 #define p4d_val(x)	((x).p4d)
48 #define __p4d(x)	((p4d_t) { (x) })
49 #define PTRS_PER_P4D	(PAGE_SIZE / sizeof(p4d_t))
50 
51 /* Page Upper Directory entry */
52 typedef struct {
53 	unsigned long pud;
54 } pud_t;
55 
56 #define pud_val(x)      ((x).pud)
57 #define __pud(x)        ((pud_t) { (x) })
58 #define PTRS_PER_PUD    (PAGE_SIZE / sizeof(pud_t))
59 
60 /* Page Middle Directory entry */
61 typedef struct {
62 	unsigned long pmd;
63 } pmd_t;
64 
65 #define pmd_val(x)      ((x).pmd)
66 #define __pmd(x)        ((pmd_t) { (x) })
67 
68 #define PTRS_PER_PMD    (PAGE_SIZE / sizeof(pmd_t))
69 
70 /*
71  * rv64 PTE format:
72  * | 63 | 62 61 | 60 54 | 53  10 | 9             8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0
73  *   N      MT     RSV    PFN      reserved for SW   D   A   G   U   X   W   R   V
74  */
75 #define _PAGE_PFN_MASK  GENMASK(53, 10)
76 
77 /*
78  * [62:61] Svpbmt Memory Type definitions:
79  *
80  *  00 - PMA    Normal Cacheable, No change to implied PMA memory type
81  *  01 - NC     Non-cacheable, idempotent, weakly-ordered Main Memory
82  *  10 - IO     Non-cacheable, non-idempotent, strongly-ordered I/O memory
83  *  11 - Rsvd   Reserved for future standard use
84  */
85 #define _PAGE_NOCACHE_SVPBMT	(1UL << 61)
86 #define _PAGE_IO_SVPBMT		(1UL << 62)
87 #define _PAGE_MTMASK_SVPBMT	(_PAGE_NOCACHE_SVPBMT | _PAGE_IO_SVPBMT)
88 
89 /*
90  * [63:59] T-Head Memory Type definitions:
91  *
92  * 00000 - NC   Weakly-ordered, Non-cacheable, Non-bufferable, Non-shareable, Non-trustable
93  * 01110 - PMA  Weakly-ordered, Cacheable, Bufferable, Shareable, Non-trustable
94  * 10000 - IO   Strongly-ordered, Non-cacheable, Non-bufferable, Non-shareable, Non-trustable
95  */
96 #define _PAGE_PMA_THEAD		((1UL << 62) | (1UL << 61) | (1UL << 60))
97 #define _PAGE_NOCACHE_THEAD	0UL
98 #define _PAGE_IO_THEAD		(1UL << 63)
99 #define _PAGE_MTMASK_THEAD	(_PAGE_PMA_THEAD | _PAGE_IO_THEAD | (1UL << 59))
100 
101 static inline u64 riscv_page_mtmask(void)
102 {
103 	u64 val;
104 
105 	ALT_SVPBMT(val, _PAGE_MTMASK);
106 	return val;
107 }
108 
109 static inline u64 riscv_page_nocache(void)
110 {
111 	u64 val;
112 
113 	ALT_SVPBMT(val, _PAGE_NOCACHE);
114 	return val;
115 }
116 
117 static inline u64 riscv_page_io(void)
118 {
119 	u64 val;
120 
121 	ALT_SVPBMT(val, _PAGE_IO);
122 	return val;
123 }
124 
125 #define _PAGE_NOCACHE		riscv_page_nocache()
126 #define _PAGE_IO		riscv_page_io()
127 #define _PAGE_MTMASK		riscv_page_mtmask()
128 
129 /* Set of bits to preserve across pte_modify() */
130 #define _PAGE_CHG_MASK  (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ |	\
131 					  _PAGE_WRITE | _PAGE_EXEC |	\
132 					  _PAGE_USER | _PAGE_GLOBAL |	\
133 					  _PAGE_MTMASK))
134 
135 static inline int pud_present(pud_t pud)
136 {
137 	return (pud_val(pud) & _PAGE_PRESENT);
138 }
139 
140 static inline int pud_none(pud_t pud)
141 {
142 	return (pud_val(pud) == 0);
143 }
144 
145 static inline int pud_bad(pud_t pud)
146 {
147 	return !pud_present(pud);
148 }
149 
150 #define pud_leaf	pud_leaf
151 static inline int pud_leaf(pud_t pud)
152 {
153 	return pud_present(pud) && (pud_val(pud) & _PAGE_LEAF);
154 }
155 
156 static inline int pud_user(pud_t pud)
157 {
158 	return pud_val(pud) & _PAGE_USER;
159 }
160 
161 static inline void set_pud(pud_t *pudp, pud_t pud)
162 {
163 	*pudp = pud;
164 }
165 
166 static inline void pud_clear(pud_t *pudp)
167 {
168 	set_pud(pudp, __pud(0));
169 }
170 
171 static inline pud_t pfn_pud(unsigned long pfn, pgprot_t prot)
172 {
173 	return __pud((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
174 }
175 
176 static inline unsigned long _pud_pfn(pud_t pud)
177 {
178 	return pud_val(pud) >> _PAGE_PFN_SHIFT;
179 }
180 
181 static inline pmd_t *pud_pgtable(pud_t pud)
182 {
183 	return (pmd_t *)pfn_to_virt(__page_val_to_pfn(pud_val(pud)));
184 }
185 
186 static inline struct page *pud_page(pud_t pud)
187 {
188 	return pfn_to_page(__page_val_to_pfn(pud_val(pud)));
189 }
190 
191 #define mm_p4d_folded  mm_p4d_folded
192 static inline bool mm_p4d_folded(struct mm_struct *mm)
193 {
194 	if (pgtable_l5_enabled)
195 		return false;
196 
197 	return true;
198 }
199 
200 #define mm_pud_folded  mm_pud_folded
201 static inline bool mm_pud_folded(struct mm_struct *mm)
202 {
203 	if (pgtable_l4_enabled)
204 		return false;
205 
206 	return true;
207 }
208 
209 #define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
210 
211 static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot)
212 {
213 	unsigned long prot_val = pgprot_val(prot);
214 
215 	ALT_THEAD_PMA(prot_val);
216 
217 	return __pmd((pfn << _PAGE_PFN_SHIFT) | prot_val);
218 }
219 
220 static inline unsigned long _pmd_pfn(pmd_t pmd)
221 {
222 	return __page_val_to_pfn(pmd_val(pmd));
223 }
224 
225 #define mk_pmd(page, prot)    pfn_pmd(page_to_pfn(page), prot)
226 
227 #define pmd_ERROR(e) \
228 	pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
229 
230 #define pud_ERROR(e)   \
231 	pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e))
232 
233 #define p4d_ERROR(e)   \
234 	pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e))
235 
236 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
237 {
238 	if (pgtable_l4_enabled)
239 		*p4dp = p4d;
240 	else
241 		set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) });
242 }
243 
244 static inline int p4d_none(p4d_t p4d)
245 {
246 	if (pgtable_l4_enabled)
247 		return (p4d_val(p4d) == 0);
248 
249 	return 0;
250 }
251 
252 static inline int p4d_present(p4d_t p4d)
253 {
254 	if (pgtable_l4_enabled)
255 		return (p4d_val(p4d) & _PAGE_PRESENT);
256 
257 	return 1;
258 }
259 
260 static inline int p4d_bad(p4d_t p4d)
261 {
262 	if (pgtable_l4_enabled)
263 		return !p4d_present(p4d);
264 
265 	return 0;
266 }
267 
268 static inline void p4d_clear(p4d_t *p4d)
269 {
270 	if (pgtable_l4_enabled)
271 		set_p4d(p4d, __p4d(0));
272 }
273 
274 static inline p4d_t pfn_p4d(unsigned long pfn, pgprot_t prot)
275 {
276 	return __p4d((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
277 }
278 
279 static inline unsigned long _p4d_pfn(p4d_t p4d)
280 {
281 	return p4d_val(p4d) >> _PAGE_PFN_SHIFT;
282 }
283 
284 static inline pud_t *p4d_pgtable(p4d_t p4d)
285 {
286 	if (pgtable_l4_enabled)
287 		return (pud_t *)pfn_to_virt(p4d_val(p4d) >> _PAGE_PFN_SHIFT);
288 
289 	return (pud_t *)pud_pgtable((pud_t) { p4d_val(p4d) });
290 }
291 #define p4d_page_vaddr(p4d)	((unsigned long)p4d_pgtable(p4d))
292 
293 static inline struct page *p4d_page(p4d_t p4d)
294 {
295 	return pfn_to_page(p4d_val(p4d) >> _PAGE_PFN_SHIFT);
296 }
297 
298 #define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
299 
300 #define pud_offset pud_offset
301 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
302 {
303 	if (pgtable_l4_enabled)
304 		return p4d_pgtable(*p4d) + pud_index(address);
305 
306 	return (pud_t *)p4d;
307 }
308 
309 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
310 {
311 	if (pgtable_l5_enabled)
312 		*pgdp = pgd;
313 	else
314 		set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) });
315 }
316 
317 static inline int pgd_none(pgd_t pgd)
318 {
319 	if (pgtable_l5_enabled)
320 		return (pgd_val(pgd) == 0);
321 
322 	return 0;
323 }
324 
325 static inline int pgd_present(pgd_t pgd)
326 {
327 	if (pgtable_l5_enabled)
328 		return (pgd_val(pgd) & _PAGE_PRESENT);
329 
330 	return 1;
331 }
332 
333 static inline int pgd_bad(pgd_t pgd)
334 {
335 	if (pgtable_l5_enabled)
336 		return !pgd_present(pgd);
337 
338 	return 0;
339 }
340 
341 static inline void pgd_clear(pgd_t *pgd)
342 {
343 	if (pgtable_l5_enabled)
344 		set_pgd(pgd, __pgd(0));
345 }
346 
347 static inline p4d_t *pgd_pgtable(pgd_t pgd)
348 {
349 	if (pgtable_l5_enabled)
350 		return (p4d_t *)pfn_to_virt(pgd_val(pgd) >> _PAGE_PFN_SHIFT);
351 
352 	return (p4d_t *)p4d_pgtable((p4d_t) { pgd_val(pgd) });
353 }
354 #define pgd_page_vaddr(pgd)	((unsigned long)pgd_pgtable(pgd))
355 
356 static inline struct page *pgd_page(pgd_t pgd)
357 {
358 	return pfn_to_page(pgd_val(pgd) >> _PAGE_PFN_SHIFT);
359 }
360 #define pgd_page(pgd)	pgd_page(pgd)
361 
362 #define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))
363 
364 #define p4d_offset p4d_offset
365 static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
366 {
367 	if (pgtable_l5_enabled)
368 		return pgd_pgtable(*pgd) + p4d_index(address);
369 
370 	return (p4d_t *)pgd;
371 }
372 
373 #endif /* _ASM_RISCV_PGTABLE_64_H */
374