1 #ifndef _ASM_X86_PGALLOC_H 2 #define _ASM_X86_PGALLOC_H 3 4 #include <linux/threads.h> 5 #include <linux/mm.h> /* for struct page */ 6 #include <linux/pagemap.h> 7 8 static inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; } 9 10 #ifdef CONFIG_PARAVIRT 11 #include <asm/paravirt.h> 12 #else 13 #define paravirt_pgd_alloc(mm) __paravirt_pgd_alloc(mm) 14 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) {} 15 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) {} 16 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) {} 17 static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn, 18 unsigned long start, unsigned long count) {} 19 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) {} 20 static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn) {} 21 static inline void paravirt_release_pte(unsigned long pfn) {} 22 static inline void paravirt_release_pmd(unsigned long pfn) {} 23 static inline void paravirt_release_pud(unsigned long pfn) {} 24 static inline void paravirt_release_p4d(unsigned long pfn) {} 25 #endif 26 27 /* 28 * Flags to use when allocating a user page table page. 29 */ 30 extern gfp_t __userpte_alloc_gfp; 31 32 /* 33 * Allocate and free page tables. 34 */ 35 extern pgd_t *pgd_alloc(struct mm_struct *); 36 extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); 37 38 extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long); 39 extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long); 40 41 /* Should really implement gc for free page table pages. This could be 42 done with a reference count in struct page. */ 43 44 static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) 45 { 46 BUG_ON((unsigned long)pte & (PAGE_SIZE-1)); 47 free_page((unsigned long)pte); 48 } 49 50 static inline void pte_free(struct mm_struct *mm, struct page *pte) 51 { 52 pgtable_page_dtor(pte); 53 __free_page(pte); 54 } 55 56 extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte); 57 58 static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte, 59 unsigned long address) 60 { 61 ___pte_free_tlb(tlb, pte); 62 } 63 64 static inline void pmd_populate_kernel(struct mm_struct *mm, 65 pmd_t *pmd, pte_t *pte) 66 { 67 paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT); 68 set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); 69 } 70 71 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, 72 struct page *pte) 73 { 74 unsigned long pfn = page_to_pfn(pte); 75 76 paravirt_alloc_pte(mm, pfn); 77 set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE)); 78 } 79 80 #define pmd_pgtable(pmd) pmd_page(pmd) 81 82 #if CONFIG_PGTABLE_LEVELS > 2 83 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) 84 { 85 struct page *page; 86 gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO; 87 88 if (mm == &init_mm) 89 gfp &= ~__GFP_ACCOUNT; 90 page = alloc_pages(gfp, 0); 91 if (!page) 92 return NULL; 93 if (!pgtable_pmd_page_ctor(page)) { 94 __free_pages(page, 0); 95 return NULL; 96 } 97 return (pmd_t *)page_address(page); 98 } 99 100 static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) 101 { 102 BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); 103 pgtable_pmd_page_dtor(virt_to_page(pmd)); 104 free_page((unsigned long)pmd); 105 } 106 107 extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd); 108 109 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, 110 unsigned long address) 111 { 112 ___pmd_free_tlb(tlb, pmd); 113 } 114 115 #ifdef CONFIG_X86_PAE 116 extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd); 117 #else /* !CONFIG_X86_PAE */ 118 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) 119 { 120 paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); 121 set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd))); 122 } 123 #endif /* CONFIG_X86_PAE */ 124 125 #if CONFIG_PGTABLE_LEVELS > 3 126 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) 127 { 128 paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT); 129 set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud))); 130 } 131 132 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) 133 { 134 gfp_t gfp = GFP_KERNEL_ACCOUNT; 135 136 if (mm == &init_mm) 137 gfp &= ~__GFP_ACCOUNT; 138 return (pud_t *)get_zeroed_page(gfp); 139 } 140 141 static inline void pud_free(struct mm_struct *mm, pud_t *pud) 142 { 143 BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); 144 free_page((unsigned long)pud); 145 } 146 147 extern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud); 148 149 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, 150 unsigned long address) 151 { 152 ___pud_free_tlb(tlb, pud); 153 } 154 155 #if CONFIG_PGTABLE_LEVELS > 4 156 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) 157 { 158 paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); 159 set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); 160 } 161 162 static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr) 163 { 164 gfp_t gfp = GFP_KERNEL_ACCOUNT; 165 166 if (mm == &init_mm) 167 gfp &= ~__GFP_ACCOUNT; 168 return (p4d_t *)get_zeroed_page(gfp); 169 } 170 171 static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) 172 { 173 BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); 174 free_page((unsigned long)p4d); 175 } 176 177 extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d); 178 179 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, 180 unsigned long address) 181 { 182 ___p4d_free_tlb(tlb, p4d); 183 } 184 185 #endif /* CONFIG_PGTABLE_LEVELS > 4 */ 186 #endif /* CONFIG_PGTABLE_LEVELS > 3 */ 187 #endif /* CONFIG_PGTABLE_LEVELS > 2 */ 188 189 #endif /* _ASM_X86_PGALLOC_H */ 190