xref: /openbmc/linux/arch/arm64/include/asm/pgtable.h (revision 78289b4a)
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Copyright (C) 2012 ARM Ltd.
4  */
5 #ifndef __ASM_PGTABLE_H
6 #define __ASM_PGTABLE_H
7 
8 #include <asm/bug.h>
9 #include <asm/proc-fns.h>
10 
11 #include <asm/memory.h>
12 #include <asm/pgtable-hwdef.h>
13 #include <asm/pgtable-prot.h>
14 #include <asm/tlbflush.h>
15 
16 /*
17  * VMALLOC range.
18  *
19  * VMALLOC_START: beginning of the kernel vmalloc space
20  * VMALLOC_END: extends to the available space below vmemmap, PCI I/O space
21  *	and fixed mappings
22  */
23 #define VMALLOC_START		(MODULES_END)
24 #define VMALLOC_END		(- PUD_SIZE - VMEMMAP_SIZE - SZ_64K)
25 
26 #define FIRST_USER_ADDRESS	0UL
27 
28 #ifndef __ASSEMBLY__
29 
30 #include <asm/cmpxchg.h>
31 #include <asm/fixmap.h>
32 #include <linux/mmdebug.h>
33 #include <linux/mm_types.h>
34 #include <linux/sched.h>
35 
36 extern struct page *vmemmap;
37 
38 extern void __pte_error(const char *file, int line, unsigned long val);
39 extern void __pmd_error(const char *file, int line, unsigned long val);
40 extern void __pud_error(const char *file, int line, unsigned long val);
41 extern void __pgd_error(const char *file, int line, unsigned long val);
42 
43 /*
44  * ZERO_PAGE is a global shared page that is always zero: used
45  * for zero-mapped memory areas etc..
46  */
47 extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
48 #define ZERO_PAGE(vaddr)	phys_to_page(__pa_symbol(empty_zero_page))
49 
50 #define pte_ERROR(pte)		__pte_error(__FILE__, __LINE__, pte_val(pte))
51 
52 /*
53  * Macros to convert between a physical address and its placement in a
54  * page table entry, taking care of 52-bit addresses.
55  */
56 #ifdef CONFIG_ARM64_PA_BITS_52
57 #define __pte_to_phys(pte)	\
58 	((pte_val(pte) & PTE_ADDR_LOW) | ((pte_val(pte) & PTE_ADDR_HIGH) << 36))
59 #define __phys_to_pte_val(phys)	(((phys) | ((phys) >> 36)) & PTE_ADDR_MASK)
60 #else
61 #define __pte_to_phys(pte)	(pte_val(pte) & PTE_ADDR_MASK)
62 #define __phys_to_pte_val(phys)	(phys)
63 #endif
64 
65 #define pte_pfn(pte)		(__pte_to_phys(pte) >> PAGE_SHIFT)
66 #define pfn_pte(pfn,prot)	\
67 	__pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
68 
69 #define pte_none(pte)		(!pte_val(pte))
70 #define pte_clear(mm,addr,ptep)	set_pte(ptep, __pte(0))
71 #define pte_page(pte)		(pfn_to_page(pte_pfn(pte)))
72 
73 /*
74  * The following only work if pte_present(). Undefined behaviour otherwise.
75  */
76 #define pte_present(pte)	(!!(pte_val(pte) & (PTE_VALID | PTE_PROT_NONE)))
77 #define pte_young(pte)		(!!(pte_val(pte) & PTE_AF))
78 #define pte_special(pte)	(!!(pte_val(pte) & PTE_SPECIAL))
79 #define pte_write(pte)		(!!(pte_val(pte) & PTE_WRITE))
80 #define pte_user_exec(pte)	(!(pte_val(pte) & PTE_UXN))
81 #define pte_cont(pte)		(!!(pte_val(pte) & PTE_CONT))
82 #define pte_devmap(pte)		(!!(pte_val(pte) & PTE_DEVMAP))
83 
84 #define pte_cont_addr_end(addr, end)						\
85 ({	unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK;	\
86 	(__boundary - 1 < (end) - 1) ? __boundary : (end);			\
87 })
88 
89 #define pmd_cont_addr_end(addr, end)						\
90 ({	unsigned long __boundary = ((addr) + CONT_PMD_SIZE) & CONT_PMD_MASK;	\
91 	(__boundary - 1 < (end) - 1) ? __boundary : (end);			\
92 })
93 
94 #define pte_hw_dirty(pte)	(pte_write(pte) && !(pte_val(pte) & PTE_RDONLY))
95 #define pte_sw_dirty(pte)	(!!(pte_val(pte) & PTE_DIRTY))
96 #define pte_dirty(pte)		(pte_sw_dirty(pte) || pte_hw_dirty(pte))
97 
98 #define pte_valid(pte)		(!!(pte_val(pte) & PTE_VALID))
99 #define pte_valid_not_user(pte) \
100 	((pte_val(pte) & (PTE_VALID | PTE_USER)) == PTE_VALID)
101 #define pte_valid_young(pte) \
102 	((pte_val(pte) & (PTE_VALID | PTE_AF)) == (PTE_VALID | PTE_AF))
103 #define pte_valid_user(pte) \
104 	((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER))
105 
106 /*
107  * Could the pte be present in the TLB? We must check mm_tlb_flush_pending
108  * so that we don't erroneously return false for pages that have been
109  * remapped as PROT_NONE but are yet to be flushed from the TLB.
110  */
111 #define pte_accessible(mm, pte)	\
112 	(mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid_young(pte))
113 
114 /*
115  * p??_access_permitted() is true for valid user mappings (subject to the
116  * write permission check). PROT_NONE mappings do not have the PTE_VALID bit
117  * set.
118  */
119 #define pte_access_permitted(pte, write) \
120 	(pte_valid_user(pte) && (!(write) || pte_write(pte)))
121 #define pmd_access_permitted(pmd, write) \
122 	(pte_access_permitted(pmd_pte(pmd), (write)))
123 #define pud_access_permitted(pud, write) \
124 	(pte_access_permitted(pud_pte(pud), (write)))
125 
126 static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
127 {
128 	pte_val(pte) &= ~pgprot_val(prot);
129 	return pte;
130 }
131 
132 static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot)
133 {
134 	pte_val(pte) |= pgprot_val(prot);
135 	return pte;
136 }
137 
138 static inline pte_t pte_wrprotect(pte_t pte)
139 {
140 	pte = clear_pte_bit(pte, __pgprot(PTE_WRITE));
141 	pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
142 	return pte;
143 }
144 
145 static inline pte_t pte_mkwrite(pte_t pte)
146 {
147 	pte = set_pte_bit(pte, __pgprot(PTE_WRITE));
148 	pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
149 	return pte;
150 }
151 
152 static inline pte_t pte_mkclean(pte_t pte)
153 {
154 	pte = clear_pte_bit(pte, __pgprot(PTE_DIRTY));
155 	pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
156 
157 	return pte;
158 }
159 
160 static inline pte_t pte_mkdirty(pte_t pte)
161 {
162 	pte = set_pte_bit(pte, __pgprot(PTE_DIRTY));
163 
164 	if (pte_write(pte))
165 		pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
166 
167 	return pte;
168 }
169 
170 static inline pte_t pte_mkold(pte_t pte)
171 {
172 	return clear_pte_bit(pte, __pgprot(PTE_AF));
173 }
174 
175 static inline pte_t pte_mkyoung(pte_t pte)
176 {
177 	return set_pte_bit(pte, __pgprot(PTE_AF));
178 }
179 
180 static inline pte_t pte_mkspecial(pte_t pte)
181 {
182 	return set_pte_bit(pte, __pgprot(PTE_SPECIAL));
183 }
184 
185 static inline pte_t pte_mkcont(pte_t pte)
186 {
187 	pte = set_pte_bit(pte, __pgprot(PTE_CONT));
188 	return set_pte_bit(pte, __pgprot(PTE_TYPE_PAGE));
189 }
190 
191 static inline pte_t pte_mknoncont(pte_t pte)
192 {
193 	return clear_pte_bit(pte, __pgprot(PTE_CONT));
194 }
195 
196 static inline pte_t pte_mkpresent(pte_t pte)
197 {
198 	return set_pte_bit(pte, __pgprot(PTE_VALID));
199 }
200 
201 static inline pmd_t pmd_mkcont(pmd_t pmd)
202 {
203 	return __pmd(pmd_val(pmd) | PMD_SECT_CONT);
204 }
205 
206 static inline pte_t pte_mkdevmap(pte_t pte)
207 {
208 	return set_pte_bit(pte, __pgprot(PTE_DEVMAP | PTE_SPECIAL));
209 }
210 
211 static inline void set_pte(pte_t *ptep, pte_t pte)
212 {
213 	WRITE_ONCE(*ptep, pte);
214 
215 	/*
216 	 * Only if the new pte is valid and kernel, otherwise TLB maintenance
217 	 * or update_mmu_cache() have the necessary barriers.
218 	 */
219 	if (pte_valid_not_user(pte)) {
220 		dsb(ishst);
221 		isb();
222 	}
223 }
224 
225 extern void __sync_icache_dcache(pte_t pteval);
226 
227 /*
228  * PTE bits configuration in the presence of hardware Dirty Bit Management
229  * (PTE_WRITE == PTE_DBM):
230  *
231  * Dirty  Writable | PTE_RDONLY  PTE_WRITE  PTE_DIRTY (sw)
232  *   0      0      |   1           0          0
233  *   0      1      |   1           1          0
234  *   1      0      |   1           0          1
235  *   1      1      |   0           1          x
236  *
237  * When hardware DBM is not present, the sofware PTE_DIRTY bit is updated via
238  * the page fault mechanism. Checking the dirty status of a pte becomes:
239  *
240  *   PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY)
241  */
242 
243 static inline void __check_racy_pte_update(struct mm_struct *mm, pte_t *ptep,
244 					   pte_t pte)
245 {
246 	pte_t old_pte;
247 
248 	if (!IS_ENABLED(CONFIG_DEBUG_VM))
249 		return;
250 
251 	old_pte = READ_ONCE(*ptep);
252 
253 	if (!pte_valid(old_pte) || !pte_valid(pte))
254 		return;
255 	if (mm != current->active_mm && atomic_read(&mm->mm_users) <= 1)
256 		return;
257 
258 	/*
259 	 * Check for potential race with hardware updates of the pte
260 	 * (ptep_set_access_flags safely changes valid ptes without going
261 	 * through an invalid entry).
262 	 */
263 	VM_WARN_ONCE(!pte_young(pte),
264 		     "%s: racy access flag clearing: 0x%016llx -> 0x%016llx",
265 		     __func__, pte_val(old_pte), pte_val(pte));
266 	VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte),
267 		     "%s: racy dirty state clearing: 0x%016llx -> 0x%016llx",
268 		     __func__, pte_val(old_pte), pte_val(pte));
269 }
270 
271 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
272 			      pte_t *ptep, pte_t pte)
273 {
274 	if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte))
275 		__sync_icache_dcache(pte);
276 
277 	__check_racy_pte_update(mm, ptep, pte);
278 
279 	set_pte(ptep, pte);
280 }
281 
282 /*
283  * Huge pte definitions.
284  */
285 #define pte_mkhuge(pte)		(__pte(pte_val(pte) & ~PTE_TABLE_BIT))
286 
287 /*
288  * Hugetlb definitions.
289  */
290 #define HUGE_MAX_HSTATE		4
291 #define HPAGE_SHIFT		PMD_SHIFT
292 #define HPAGE_SIZE		(_AC(1, UL) << HPAGE_SHIFT)
293 #define HPAGE_MASK		(~(HPAGE_SIZE - 1))
294 #define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)
295 
296 static inline pte_t pgd_pte(pgd_t pgd)
297 {
298 	return __pte(pgd_val(pgd));
299 }
300 
301 static inline pte_t p4d_pte(p4d_t p4d)
302 {
303 	return __pte(p4d_val(p4d));
304 }
305 
306 static inline pte_t pud_pte(pud_t pud)
307 {
308 	return __pte(pud_val(pud));
309 }
310 
311 static inline pud_t pte_pud(pte_t pte)
312 {
313 	return __pud(pte_val(pte));
314 }
315 
316 static inline pmd_t pud_pmd(pud_t pud)
317 {
318 	return __pmd(pud_val(pud));
319 }
320 
321 static inline pte_t pmd_pte(pmd_t pmd)
322 {
323 	return __pte(pmd_val(pmd));
324 }
325 
326 static inline pmd_t pte_pmd(pte_t pte)
327 {
328 	return __pmd(pte_val(pte));
329 }
330 
331 static inline pgprot_t mk_pud_sect_prot(pgprot_t prot)
332 {
333 	return __pgprot((pgprot_val(prot) & ~PUD_TABLE_BIT) | PUD_TYPE_SECT);
334 }
335 
336 static inline pgprot_t mk_pmd_sect_prot(pgprot_t prot)
337 {
338 	return __pgprot((pgprot_val(prot) & ~PMD_TABLE_BIT) | PMD_TYPE_SECT);
339 }
340 
341 #ifdef CONFIG_NUMA_BALANCING
342 /*
343  * See the comment in include/linux/pgtable.h
344  */
345 static inline int pte_protnone(pte_t pte)
346 {
347 	return (pte_val(pte) & (PTE_VALID | PTE_PROT_NONE)) == PTE_PROT_NONE;
348 }
349 
350 static inline int pmd_protnone(pmd_t pmd)
351 {
352 	return pte_protnone(pmd_pte(pmd));
353 }
354 #endif
355 
356 /*
357  * THP definitions.
358  */
359 
360 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
361 #define pmd_trans_huge(pmd)	(pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT))
362 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
363 
364 #define pmd_present(pmd)	pte_present(pmd_pte(pmd))
365 #define pmd_dirty(pmd)		pte_dirty(pmd_pte(pmd))
366 #define pmd_young(pmd)		pte_young(pmd_pte(pmd))
367 #define pmd_valid(pmd)		pte_valid(pmd_pte(pmd))
368 #define pmd_wrprotect(pmd)	pte_pmd(pte_wrprotect(pmd_pte(pmd)))
369 #define pmd_mkold(pmd)		pte_pmd(pte_mkold(pmd_pte(pmd)))
370 #define pmd_mkwrite(pmd)	pte_pmd(pte_mkwrite(pmd_pte(pmd)))
371 #define pmd_mkclean(pmd)	pte_pmd(pte_mkclean(pmd_pte(pmd)))
372 #define pmd_mkdirty(pmd)	pte_pmd(pte_mkdirty(pmd_pte(pmd)))
373 #define pmd_mkyoung(pmd)	pte_pmd(pte_mkyoung(pmd_pte(pmd)))
374 #define pmd_mkinvalid(pmd)	(__pmd(pmd_val(pmd) & ~PMD_SECT_VALID))
375 
376 #define pmd_thp_or_huge(pmd)	(pmd_huge(pmd) || pmd_trans_huge(pmd))
377 
378 #define pmd_write(pmd)		pte_write(pmd_pte(pmd))
379 
380 #define pmd_mkhuge(pmd)		(__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))
381 
382 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
383 #define pmd_devmap(pmd)		pte_devmap(pmd_pte(pmd))
384 #endif
385 static inline pmd_t pmd_mkdevmap(pmd_t pmd)
386 {
387 	return pte_pmd(set_pte_bit(pmd_pte(pmd), __pgprot(PTE_DEVMAP)));
388 }
389 
390 #define __pmd_to_phys(pmd)	__pte_to_phys(pmd_pte(pmd))
391 #define __phys_to_pmd_val(phys)	__phys_to_pte_val(phys)
392 #define pmd_pfn(pmd)		((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT)
393 #define pfn_pmd(pfn,prot)	__pmd(__phys_to_pmd_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
394 #define mk_pmd(page,prot)	pfn_pmd(page_to_pfn(page),prot)
395 
396 #define pud_young(pud)		pte_young(pud_pte(pud))
397 #define pud_mkyoung(pud)	pte_pud(pte_mkyoung(pud_pte(pud)))
398 #define pud_write(pud)		pte_write(pud_pte(pud))
399 
400 #define pud_mkhuge(pud)		(__pud(pud_val(pud) & ~PUD_TABLE_BIT))
401 
402 #define __pud_to_phys(pud)	__pte_to_phys(pud_pte(pud))
403 #define __phys_to_pud_val(phys)	__phys_to_pte_val(phys)
404 #define pud_pfn(pud)		((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT)
405 #define pfn_pud(pfn,prot)	__pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
406 
407 #define set_pmd_at(mm, addr, pmdp, pmd)	set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd))
408 
409 #define __p4d_to_phys(p4d)	__pte_to_phys(p4d_pte(p4d))
410 #define __phys_to_p4d_val(phys)	__phys_to_pte_val(phys)
411 
412 #define __pgd_to_phys(pgd)	__pte_to_phys(pgd_pte(pgd))
413 #define __phys_to_pgd_val(phys)	__phys_to_pte_val(phys)
414 
415 #define __pgprot_modify(prot,mask,bits) \
416 	__pgprot((pgprot_val(prot) & ~(mask)) | (bits))
417 
418 #define pgprot_nx(prot) \
419 	__pgprot_modify(prot, 0, PTE_PXN)
420 
421 /*
422  * Mark the prot value as uncacheable and unbufferable.
423  */
424 #define pgprot_noncached(prot) \
425 	__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRnE) | PTE_PXN | PTE_UXN)
426 #define pgprot_writecombine(prot) \
427 	__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
428 #define pgprot_device(prot) \
429 	__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRE) | PTE_PXN | PTE_UXN)
430 /*
431  * DMA allocations for non-coherent devices use what the Arm architecture calls
432  * "Normal non-cacheable" memory, which permits speculation, unaligned accesses
433  * and merging of writes.  This is different from "Device-nGnR[nE]" memory which
434  * is intended for MMIO and thus forbids speculation, preserves access size,
435  * requires strict alignment and can also force write responses to come from the
436  * endpoint.
437  */
438 #define pgprot_dmacoherent(prot) \
439 	__pgprot_modify(prot, PTE_ATTRINDX_MASK, \
440 			PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
441 
442 #define __HAVE_PHYS_MEM_ACCESS_PROT
443 struct file;
444 extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
445 				     unsigned long size, pgprot_t vma_prot);
446 
447 #define pmd_none(pmd)		(!pmd_val(pmd))
448 
449 #define pmd_bad(pmd)		(!(pmd_val(pmd) & PMD_TABLE_BIT))
450 
451 #define pmd_table(pmd)		((pmd_val(pmd) & PMD_TYPE_MASK) == \
452 				 PMD_TYPE_TABLE)
453 #define pmd_sect(pmd)		((pmd_val(pmd) & PMD_TYPE_MASK) == \
454 				 PMD_TYPE_SECT)
455 #define pmd_leaf(pmd)		pmd_sect(pmd)
456 
457 #if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS < 3
458 static inline bool pud_sect(pud_t pud) { return false; }
459 static inline bool pud_table(pud_t pud) { return true; }
460 #else
461 #define pud_sect(pud)		((pud_val(pud) & PUD_TYPE_MASK) == \
462 				 PUD_TYPE_SECT)
463 #define pud_table(pud)		((pud_val(pud) & PUD_TYPE_MASK) == \
464 				 PUD_TYPE_TABLE)
465 #endif
466 
467 extern pgd_t init_pg_dir[PTRS_PER_PGD];
468 extern pgd_t init_pg_end[];
469 extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
470 extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
471 extern pgd_t idmap_pg_end[];
472 extern pgd_t tramp_pg_dir[PTRS_PER_PGD];
473 
474 extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd);
475 
476 static inline bool in_swapper_pgdir(void *addr)
477 {
478 	return ((unsigned long)addr & PAGE_MASK) ==
479 	        ((unsigned long)swapper_pg_dir & PAGE_MASK);
480 }
481 
482 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
483 {
484 #ifdef __PAGETABLE_PMD_FOLDED
485 	if (in_swapper_pgdir(pmdp)) {
486 		set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd)));
487 		return;
488 	}
489 #endif /* __PAGETABLE_PMD_FOLDED */
490 
491 	WRITE_ONCE(*pmdp, pmd);
492 
493 	if (pmd_valid(pmd)) {
494 		dsb(ishst);
495 		isb();
496 	}
497 }
498 
499 static inline void pmd_clear(pmd_t *pmdp)
500 {
501 	set_pmd(pmdp, __pmd(0));
502 }
503 
504 static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
505 {
506 	return __pmd_to_phys(pmd);
507 }
508 
509 static inline unsigned long pmd_page_vaddr(pmd_t pmd)
510 {
511 	return (unsigned long)__va(pmd_page_paddr(pmd));
512 }
513 
514 /* Find an entry in the third-level page table. */
515 #define pte_offset_phys(dir,addr)	(pmd_page_paddr(READ_ONCE(*(dir))) + pte_index(addr) * sizeof(pte_t))
516 
517 #define pte_set_fixmap(addr)		((pte_t *)set_fixmap_offset(FIX_PTE, addr))
518 #define pte_set_fixmap_offset(pmd, addr)	pte_set_fixmap(pte_offset_phys(pmd, addr))
519 #define pte_clear_fixmap()		clear_fixmap(FIX_PTE)
520 
521 #define pmd_page(pmd)			phys_to_page(__pmd_to_phys(pmd))
522 
523 /* use ONLY for statically allocated translation tables */
524 #define pte_offset_kimg(dir,addr)	((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr))))
525 
526 /*
527  * Conversion functions: convert a page and protection to a page entry,
528  * and a page entry and page directory to the page they refer to.
529  */
530 #define mk_pte(page,prot)	pfn_pte(page_to_pfn(page),prot)
531 
532 #if CONFIG_PGTABLE_LEVELS > 2
533 
534 #define pmd_ERROR(pmd)		__pmd_error(__FILE__, __LINE__, pmd_val(pmd))
535 
536 #define pud_none(pud)		(!pud_val(pud))
537 #define pud_bad(pud)		(!(pud_val(pud) & PUD_TABLE_BIT))
538 #define pud_present(pud)	pte_present(pud_pte(pud))
539 #define pud_leaf(pud)		pud_sect(pud)
540 #define pud_valid(pud)		pte_valid(pud_pte(pud))
541 
542 static inline void set_pud(pud_t *pudp, pud_t pud)
543 {
544 #ifdef __PAGETABLE_PUD_FOLDED
545 	if (in_swapper_pgdir(pudp)) {
546 		set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud)));
547 		return;
548 	}
549 #endif /* __PAGETABLE_PUD_FOLDED */
550 
551 	WRITE_ONCE(*pudp, pud);
552 
553 	if (pud_valid(pud)) {
554 		dsb(ishst);
555 		isb();
556 	}
557 }
558 
559 static inline void pud_clear(pud_t *pudp)
560 {
561 	set_pud(pudp, __pud(0));
562 }
563 
564 static inline phys_addr_t pud_page_paddr(pud_t pud)
565 {
566 	return __pud_to_phys(pud);
567 }
568 
569 static inline unsigned long pud_page_vaddr(pud_t pud)
570 {
571 	return (unsigned long)__va(pud_page_paddr(pud));
572 }
573 
574 /* Find an entry in the second-level page table. */
575 #define pmd_offset_phys(dir, addr)	(pud_page_paddr(READ_ONCE(*(dir))) + pmd_index(addr) * sizeof(pmd_t))
576 
577 #define pmd_set_fixmap(addr)		((pmd_t *)set_fixmap_offset(FIX_PMD, addr))
578 #define pmd_set_fixmap_offset(pud, addr)	pmd_set_fixmap(pmd_offset_phys(pud, addr))
579 #define pmd_clear_fixmap()		clear_fixmap(FIX_PMD)
580 
581 #define pud_page(pud)			phys_to_page(__pud_to_phys(pud))
582 
583 /* use ONLY for statically allocated translation tables */
584 #define pmd_offset_kimg(dir,addr)	((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr))))
585 
586 #else
587 
588 #define pud_page_paddr(pud)	({ BUILD_BUG(); 0; })
589 
590 /* Match pmd_offset folding in <asm/generic/pgtable-nopmd.h> */
591 #define pmd_set_fixmap(addr)		NULL
592 #define pmd_set_fixmap_offset(pudp, addr)	((pmd_t *)pudp)
593 #define pmd_clear_fixmap()
594 
595 #define pmd_offset_kimg(dir,addr)	((pmd_t *)dir)
596 
597 #endif	/* CONFIG_PGTABLE_LEVELS > 2 */
598 
599 #if CONFIG_PGTABLE_LEVELS > 3
600 
601 #define pud_ERROR(pud)		__pud_error(__FILE__, __LINE__, pud_val(pud))
602 
603 #define p4d_none(p4d)		(!p4d_val(p4d))
604 #define p4d_bad(p4d)		(!(p4d_val(p4d) & 2))
605 #define p4d_present(p4d)	(p4d_val(p4d))
606 
607 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
608 {
609 	if (in_swapper_pgdir(p4dp)) {
610 		set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d)));
611 		return;
612 	}
613 
614 	WRITE_ONCE(*p4dp, p4d);
615 	dsb(ishst);
616 	isb();
617 }
618 
619 static inline void p4d_clear(p4d_t *p4dp)
620 {
621 	set_p4d(p4dp, __p4d(0));
622 }
623 
624 static inline phys_addr_t p4d_page_paddr(p4d_t p4d)
625 {
626 	return __p4d_to_phys(p4d);
627 }
628 
629 static inline unsigned long p4d_page_vaddr(p4d_t p4d)
630 {
631 	return (unsigned long)__va(p4d_page_paddr(p4d));
632 }
633 
634 /* Find an entry in the frst-level page table. */
635 #define pud_offset_phys(dir, addr)	(p4d_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t))
636 
637 #define pud_set_fixmap(addr)		((pud_t *)set_fixmap_offset(FIX_PUD, addr))
638 #define pud_set_fixmap_offset(p4d, addr)	pud_set_fixmap(pud_offset_phys(p4d, addr))
639 #define pud_clear_fixmap()		clear_fixmap(FIX_PUD)
640 
641 #define p4d_page(p4d)		pfn_to_page(__phys_to_pfn(__p4d_to_phys(p4d)))
642 
643 /* use ONLY for statically allocated translation tables */
644 #define pud_offset_kimg(dir,addr)	((pud_t *)__phys_to_kimg(pud_offset_phys((dir), (addr))))
645 
646 #else
647 
648 #define p4d_page_paddr(p4d)	({ BUILD_BUG(); 0;})
649 #define pgd_page_paddr(pgd)	({ BUILD_BUG(); 0;})
650 
651 /* Match pud_offset folding in <asm/generic/pgtable-nopud.h> */
652 #define pud_set_fixmap(addr)		NULL
653 #define pud_set_fixmap_offset(pgdp, addr)	((pud_t *)pgdp)
654 #define pud_clear_fixmap()
655 
656 #define pud_offset_kimg(dir,addr)	((pud_t *)dir)
657 
658 #endif  /* CONFIG_PGTABLE_LEVELS > 3 */
659 
660 #define pgd_ERROR(pgd)		__pgd_error(__FILE__, __LINE__, pgd_val(pgd))
661 
662 #define pgd_set_fixmap(addr)	((pgd_t *)set_fixmap_offset(FIX_PGD, addr))
663 #define pgd_clear_fixmap()	clear_fixmap(FIX_PGD)
664 
665 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
666 {
667 	const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY |
668 			      PTE_PROT_NONE | PTE_VALID | PTE_WRITE | PTE_GP;
669 	/* preserve the hardware dirty information */
670 	if (pte_hw_dirty(pte))
671 		pte = pte_mkdirty(pte);
672 	pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask);
673 	return pte;
674 }
675 
676 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
677 {
678 	return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
679 }
680 
681 #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
682 extern int ptep_set_access_flags(struct vm_area_struct *vma,
683 				 unsigned long address, pte_t *ptep,
684 				 pte_t entry, int dirty);
685 
686 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
687 #define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
688 static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
689 					unsigned long address, pmd_t *pmdp,
690 					pmd_t entry, int dirty)
691 {
692 	return ptep_set_access_flags(vma, address, (pte_t *)pmdp, pmd_pte(entry), dirty);
693 }
694 
695 static inline int pud_devmap(pud_t pud)
696 {
697 	return 0;
698 }
699 
700 static inline int pgd_devmap(pgd_t pgd)
701 {
702 	return 0;
703 }
704 #endif
705 
706 /*
707  * Atomic pte/pmd modifications.
708  */
709 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
710 static inline int __ptep_test_and_clear_young(pte_t *ptep)
711 {
712 	pte_t old_pte, pte;
713 
714 	pte = READ_ONCE(*ptep);
715 	do {
716 		old_pte = pte;
717 		pte = pte_mkold(pte);
718 		pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
719 					       pte_val(old_pte), pte_val(pte));
720 	} while (pte_val(pte) != pte_val(old_pte));
721 
722 	return pte_young(pte);
723 }
724 
725 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
726 					    unsigned long address,
727 					    pte_t *ptep)
728 {
729 	return __ptep_test_and_clear_young(ptep);
730 }
731 
732 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
733 static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
734 					 unsigned long address, pte_t *ptep)
735 {
736 	int young = ptep_test_and_clear_young(vma, address, ptep);
737 
738 	if (young) {
739 		/*
740 		 * We can elide the trailing DSB here since the worst that can
741 		 * happen is that a CPU continues to use the young entry in its
742 		 * TLB and we mistakenly reclaim the associated page. The
743 		 * window for such an event is bounded by the next
744 		 * context-switch, which provides a DSB to complete the TLB
745 		 * invalidation.
746 		 */
747 		flush_tlb_page_nosync(vma, address);
748 	}
749 
750 	return young;
751 }
752 
753 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
754 #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
755 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
756 					    unsigned long address,
757 					    pmd_t *pmdp)
758 {
759 	return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);
760 }
761 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
762 
763 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
764 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
765 				       unsigned long address, pte_t *ptep)
766 {
767 	return __pte(xchg_relaxed(&pte_val(*ptep), 0));
768 }
769 
770 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
771 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
772 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
773 					    unsigned long address, pmd_t *pmdp)
774 {
775 	return pte_pmd(ptep_get_and_clear(mm, address, (pte_t *)pmdp));
776 }
777 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
778 
779 /*
780  * ptep_set_wrprotect - mark read-only while trasferring potential hardware
781  * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit.
782  */
783 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
784 static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
785 {
786 	pte_t old_pte, pte;
787 
788 	pte = READ_ONCE(*ptep);
789 	do {
790 		old_pte = pte;
791 		/*
792 		 * If hardware-dirty (PTE_WRITE/DBM bit set and PTE_RDONLY
793 		 * clear), set the PTE_DIRTY bit.
794 		 */
795 		if (pte_hw_dirty(pte))
796 			pte = pte_mkdirty(pte);
797 		pte = pte_wrprotect(pte);
798 		pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
799 					       pte_val(old_pte), pte_val(pte));
800 	} while (pte_val(pte) != pte_val(old_pte));
801 }
802 
803 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
804 #define __HAVE_ARCH_PMDP_SET_WRPROTECT
805 static inline void pmdp_set_wrprotect(struct mm_struct *mm,
806 				      unsigned long address, pmd_t *pmdp)
807 {
808 	ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
809 }
810 
811 #define pmdp_establish pmdp_establish
812 static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
813 		unsigned long address, pmd_t *pmdp, pmd_t pmd)
814 {
815 	return __pmd(xchg_relaxed(&pmd_val(*pmdp), pmd_val(pmd)));
816 }
817 #endif
818 
819 /*
820  * Encode and decode a swap entry:
821  *	bits 0-1:	present (must be zero)
822  *	bits 2-7:	swap type
823  *	bits 8-57:	swap offset
824  *	bit  58:	PTE_PROT_NONE (must be zero)
825  */
826 #define __SWP_TYPE_SHIFT	2
827 #define __SWP_TYPE_BITS		6
828 #define __SWP_OFFSET_BITS	50
829 #define __SWP_TYPE_MASK		((1 << __SWP_TYPE_BITS) - 1)
830 #define __SWP_OFFSET_SHIFT	(__SWP_TYPE_BITS + __SWP_TYPE_SHIFT)
831 #define __SWP_OFFSET_MASK	((1UL << __SWP_OFFSET_BITS) - 1)
832 
833 #define __swp_type(x)		(((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK)
834 #define __swp_offset(x)		(((x).val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK)
835 #define __swp_entry(type,offset) ((swp_entry_t) { ((type) << __SWP_TYPE_SHIFT) | ((offset) << __SWP_OFFSET_SHIFT) })
836 
837 #define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
838 #define __swp_entry_to_pte(swp)	((pte_t) { (swp).val })
839 
840 /*
841  * Ensure that there are not more swap files than can be encoded in the kernel
842  * PTEs.
843  */
844 #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS)
845 
846 extern int kern_addr_valid(unsigned long addr);
847 
848 /*
849  * On AArch64, the cache coherency is handled via the set_pte_at() function.
850  */
851 static inline void update_mmu_cache(struct vm_area_struct *vma,
852 				    unsigned long addr, pte_t *ptep)
853 {
854 	/*
855 	 * We don't do anything here, so there's a very small chance of
856 	 * us retaking a user fault which we just fixed up. The alternative
857 	 * is doing a dsb(ishst), but that penalises the fastpath.
858 	 */
859 }
860 
861 #define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
862 
863 #ifdef CONFIG_ARM64_PA_BITS_52
864 #define phys_to_ttbr(addr)	(((addr) | ((addr) >> 46)) & TTBR_BADDR_MASK_52)
865 #else
866 #define phys_to_ttbr(addr)	(addr)
867 #endif
868 
869 /*
870  * On arm64 without hardware Access Flag, copying from user will fail because
871  * the pte is old and cannot be marked young. So we always end up with zeroed
872  * page after fork() + CoW for pfn mappings. We don't always have a
873  * hardware-managed access flag on arm64.
874  */
875 static inline bool arch_faults_on_old_pte(void)
876 {
877 	WARN_ON(preemptible());
878 
879 	return !cpu_has_hw_af();
880 }
881 #define arch_faults_on_old_pte arch_faults_on_old_pte
882 
883 #endif /* !__ASSEMBLY__ */
884 
885 #endif /* __ASM_PGTABLE_H */
886