1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_PGALLOC_H 3 #define _ASM_X86_PGALLOC_H 4 5 #include <linux/threads.h> 6 #include <linux/mm.h> /* for struct page */ 7 #include <linux/pagemap.h> 8 9 static inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; } 10 11 #ifdef CONFIG_PARAVIRT 12 #include <asm/paravirt.h> 13 #else 14 #define paravirt_pgd_alloc(mm) __paravirt_pgd_alloc(mm) 15 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) {} 16 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) {} 17 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) {} 18 static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn, 19 unsigned long start, unsigned long count) {} 20 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) {} 21 static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn) {} 22 static inline void paravirt_release_pte(unsigned long pfn) {} 23 static inline void paravirt_release_pmd(unsigned long pfn) {} 24 static inline void paravirt_release_pud(unsigned long pfn) {} 25 static inline void paravirt_release_p4d(unsigned long pfn) {} 26 #endif 27 28 /* 29 * Flags to use when allocating a user page table page. 30 */ 31 extern gfp_t __userpte_alloc_gfp; 32 33 /* 34 * Allocate and free page tables. 35 */ 36 extern pgd_t *pgd_alloc(struct mm_struct *); 37 extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); 38 39 extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long); 40 extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long); 41 42 /* Should really implement gc for free page table pages. This could be 43 done with a reference count in struct page. */ 44 45 static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) 46 { 47 BUG_ON((unsigned long)pte & (PAGE_SIZE-1)); 48 free_page((unsigned long)pte); 49 } 50 51 static inline void pte_free(struct mm_struct *mm, struct page *pte) 52 { 53 pgtable_page_dtor(pte); 54 __free_page(pte); 55 } 56 57 extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte); 58 59 static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte, 60 unsigned long address) 61 { 62 ___pte_free_tlb(tlb, pte); 63 } 64 65 static inline void pmd_populate_kernel(struct mm_struct *mm, 66 pmd_t *pmd, pte_t *pte) 67 { 68 paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT); 69 set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); 70 } 71 72 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, 73 struct page *pte) 74 { 75 unsigned long pfn = page_to_pfn(pte); 76 77 paravirt_alloc_pte(mm, pfn); 78 set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE)); 79 } 80 81 #define pmd_pgtable(pmd) pmd_page(pmd) 82 83 #if CONFIG_PGTABLE_LEVELS > 2 84 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) 85 { 86 struct page *page; 87 gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO; 88 89 if (mm == &init_mm) 90 gfp &= ~__GFP_ACCOUNT; 91 page = alloc_pages(gfp, 0); 92 if (!page) 93 return NULL; 94 if (!pgtable_pmd_page_ctor(page)) { 95 __free_pages(page, 0); 96 return NULL; 97 } 98 return (pmd_t *)page_address(page); 99 } 100 101 static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) 102 { 103 BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); 104 pgtable_pmd_page_dtor(virt_to_page(pmd)); 105 free_page((unsigned long)pmd); 106 } 107 108 extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd); 109 110 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, 111 unsigned long address) 112 { 113 ___pmd_free_tlb(tlb, pmd); 114 } 115 116 #ifdef CONFIG_X86_PAE 117 extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd); 118 #else /* !CONFIG_X86_PAE */ 119 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) 120 { 121 paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); 122 set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd))); 123 } 124 #endif /* CONFIG_X86_PAE */ 125 126 #if CONFIG_PGTABLE_LEVELS > 3 127 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) 128 { 129 paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT); 130 set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud))); 131 } 132 133 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) 134 { 135 gfp_t gfp = GFP_KERNEL_ACCOUNT; 136 137 if (mm == &init_mm) 138 gfp &= ~__GFP_ACCOUNT; 139 return (pud_t *)get_zeroed_page(gfp); 140 } 141 142 static inline void pud_free(struct mm_struct *mm, pud_t *pud) 143 { 144 BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); 145 free_page((unsigned long)pud); 146 } 147 148 extern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud); 149 150 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, 151 unsigned long address) 152 { 153 ___pud_free_tlb(tlb, pud); 154 } 155 156 #if CONFIG_PGTABLE_LEVELS > 4 157 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) 158 { 159 paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); 160 set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); 161 } 162 163 static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr) 164 { 165 gfp_t gfp = GFP_KERNEL_ACCOUNT; 166 167 if (mm == &init_mm) 168 gfp &= ~__GFP_ACCOUNT; 169 return (p4d_t *)get_zeroed_page(gfp); 170 } 171 172 static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) 173 { 174 BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); 175 free_page((unsigned long)p4d); 176 } 177 178 extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d); 179 180 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, 181 unsigned long address) 182 { 183 ___p4d_free_tlb(tlb, p4d); 184 } 185 186 #endif /* CONFIG_PGTABLE_LEVELS > 4 */ 187 #endif /* CONFIG_PGTABLE_LEVELS > 3 */ 188 #endif /* CONFIG_PGTABLE_LEVELS > 2 */ 189 190 #endif /* _ASM_X86_PGALLOC_H */ 191