1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_PGALLOC_H 3 #define _ASM_X86_PGALLOC_H 4 5 #include <linux/threads.h> 6 #include <linux/mm.h> /* for struct page */ 7 #include <linux/pagemap.h> 8 9 static inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; } 10 11 #ifdef CONFIG_PARAVIRT_XXL 12 #include <asm/paravirt.h> 13 #else 14 #define paravirt_pgd_alloc(mm) __paravirt_pgd_alloc(mm) 15 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) {} 16 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) {} 17 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) {} 18 static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn, 19 unsigned long start, unsigned long count) {} 20 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) {} 21 static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn) {} 22 static inline void paravirt_release_pte(unsigned long pfn) {} 23 static inline void paravirt_release_pmd(unsigned long pfn) {} 24 static inline void paravirt_release_pud(unsigned long pfn) {} 25 static inline void paravirt_release_p4d(unsigned long pfn) {} 26 #endif 27 28 /* 29 * Flags to use when allocating a user page table page. 30 */ 31 extern gfp_t __userpte_alloc_gfp; 32 33 #ifdef CONFIG_PAGE_TABLE_ISOLATION 34 /* 35 * Instead of one PGD, we acquire two PGDs. Being order-1, it is 36 * both 8k in size and 8k-aligned. That lets us just flip bit 12 37 * in a pointer to swap between the two 4k halves. 38 */ 39 #define PGD_ALLOCATION_ORDER 1 40 #else 41 #define PGD_ALLOCATION_ORDER 0 42 #endif 43 44 /* 45 * Allocate and free page tables. 46 */ 47 extern pgd_t *pgd_alloc(struct mm_struct *); 48 extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); 49 50 extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long); 51 extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long); 52 53 /* Should really implement gc for free page table pages. This could be 54 done with a reference count in struct page. */ 55 56 static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) 57 { 58 BUG_ON((unsigned long)pte & (PAGE_SIZE-1)); 59 free_page((unsigned long)pte); 60 } 61 62 static inline void pte_free(struct mm_struct *mm, struct page *pte) 63 { 64 pgtable_page_dtor(pte); 65 __free_page(pte); 66 } 67 68 extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte); 69 70 static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte, 71 unsigned long address) 72 { 73 ___pte_free_tlb(tlb, pte); 74 } 75 76 static inline void pmd_populate_kernel(struct mm_struct *mm, 77 pmd_t *pmd, pte_t *pte) 78 { 79 paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT); 80 set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); 81 } 82 83 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, 84 struct page *pte) 85 { 86 unsigned long pfn = page_to_pfn(pte); 87 88 paravirt_alloc_pte(mm, pfn); 89 set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE)); 90 } 91 92 #define pmd_pgtable(pmd) pmd_page(pmd) 93 94 #if CONFIG_PGTABLE_LEVELS > 2 95 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) 96 { 97 struct page *page; 98 gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO; 99 100 if (mm == &init_mm) 101 gfp &= ~__GFP_ACCOUNT; 102 page = alloc_pages(gfp, 0); 103 if (!page) 104 return NULL; 105 if (!pgtable_pmd_page_ctor(page)) { 106 __free_pages(page, 0); 107 return NULL; 108 } 109 return (pmd_t *)page_address(page); 110 } 111 112 static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) 113 { 114 BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); 115 pgtable_pmd_page_dtor(virt_to_page(pmd)); 116 free_page((unsigned long)pmd); 117 } 118 119 extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd); 120 121 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, 122 unsigned long address) 123 { 124 ___pmd_free_tlb(tlb, pmd); 125 } 126 127 #ifdef CONFIG_X86_PAE 128 extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd); 129 #else /* !CONFIG_X86_PAE */ 130 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) 131 { 132 paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); 133 set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd))); 134 } 135 #endif /* CONFIG_X86_PAE */ 136 137 #if CONFIG_PGTABLE_LEVELS > 3 138 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) 139 { 140 paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT); 141 set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud))); 142 } 143 144 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) 145 { 146 gfp_t gfp = GFP_KERNEL_ACCOUNT; 147 148 if (mm == &init_mm) 149 gfp &= ~__GFP_ACCOUNT; 150 return (pud_t *)get_zeroed_page(gfp); 151 } 152 153 static inline void pud_free(struct mm_struct *mm, pud_t *pud) 154 { 155 BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); 156 free_page((unsigned long)pud); 157 } 158 159 extern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud); 160 161 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, 162 unsigned long address) 163 { 164 ___pud_free_tlb(tlb, pud); 165 } 166 167 #if CONFIG_PGTABLE_LEVELS > 4 168 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) 169 { 170 if (!pgtable_l5_enabled()) 171 return; 172 paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); 173 set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); 174 } 175 176 static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr) 177 { 178 gfp_t gfp = GFP_KERNEL_ACCOUNT; 179 180 if (mm == &init_mm) 181 gfp &= ~__GFP_ACCOUNT; 182 return (p4d_t *)get_zeroed_page(gfp); 183 } 184 185 static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) 186 { 187 if (!pgtable_l5_enabled()) 188 return; 189 190 BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); 191 free_page((unsigned long)p4d); 192 } 193 194 extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d); 195 196 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, 197 unsigned long address) 198 { 199 if (pgtable_l5_enabled()) 200 ___p4d_free_tlb(tlb, p4d); 201 } 202 203 #endif /* CONFIG_PGTABLE_LEVELS > 4 */ 204 #endif /* CONFIG_PGTABLE_LEVELS > 3 */ 205 #endif /* CONFIG_PGTABLE_LEVELS > 2 */ 206 207 #endif /* _ASM_X86_PGALLOC_H */ 208