1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */
25c01b46bSArnd Bergmann #ifndef __ASM_GENERIC_PGALLOC_H
35c01b46bSArnd Bergmann #define __ASM_GENERIC_PGALLOC_H
45fba4af4SMike Rapoport
55c01b46bSArnd Bergmann #ifdef CONFIG_MMU
65fba4af4SMike Rapoport
75fba4af4SMike Rapoport #define GFP_PGTABLE_KERNEL (GFP_KERNEL | __GFP_ZERO)
85fba4af4SMike Rapoport #define GFP_PGTABLE_USER (GFP_PGTABLE_KERNEL | __GFP_ACCOUNT)
95fba4af4SMike Rapoport
105fba4af4SMike Rapoport /**
11*c787ae5bSVishal Moola (Oracle) * __pte_alloc_one_kernel - allocate memory for a PTE-level kernel page table
125fba4af4SMike Rapoport * @mm: the mm_struct of the current context
135fba4af4SMike Rapoport *
145fba4af4SMike Rapoport * This function is intended for architectures that need
155fba4af4SMike Rapoport * anything beyond simple page allocation.
165fba4af4SMike Rapoport *
175fba4af4SMike Rapoport * Return: pointer to the allocated memory or %NULL on error
185fba4af4SMike Rapoport */
__pte_alloc_one_kernel(struct mm_struct * mm)195fba4af4SMike Rapoport static inline pte_t *__pte_alloc_one_kernel(struct mm_struct *mm)
205fba4af4SMike Rapoport {
21*c787ae5bSVishal Moola (Oracle) struct ptdesc *ptdesc = pagetable_alloc(GFP_PGTABLE_KERNEL &
22*c787ae5bSVishal Moola (Oracle) ~__GFP_HIGHMEM, 0);
23*c787ae5bSVishal Moola (Oracle)
24*c787ae5bSVishal Moola (Oracle) if (!ptdesc)
25*c787ae5bSVishal Moola (Oracle) return NULL;
26*c787ae5bSVishal Moola (Oracle) return ptdesc_address(ptdesc);
275fba4af4SMike Rapoport }
285fba4af4SMike Rapoport
295fba4af4SMike Rapoport #ifndef __HAVE_ARCH_PTE_ALLOC_ONE_KERNEL
305fba4af4SMike Rapoport /**
31*c787ae5bSVishal Moola (Oracle) * pte_alloc_one_kernel - allocate memory for a PTE-level kernel page table
325fba4af4SMike Rapoport * @mm: the mm_struct of the current context
335fba4af4SMike Rapoport *
345fba4af4SMike Rapoport * Return: pointer to the allocated memory or %NULL on error
355fba4af4SMike Rapoport */
pte_alloc_one_kernel(struct mm_struct * mm)365fba4af4SMike Rapoport static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
375fba4af4SMike Rapoport {
385fba4af4SMike Rapoport return __pte_alloc_one_kernel(mm);
395fba4af4SMike Rapoport }
405c01b46bSArnd Bergmann #endif
415c01b46bSArnd Bergmann
425fba4af4SMike Rapoport /**
43*c787ae5bSVishal Moola (Oracle) * pte_free_kernel - free PTE-level kernel page table memory
445fba4af4SMike Rapoport * @mm: the mm_struct of the current context
455fba4af4SMike Rapoport * @pte: pointer to the memory containing the page table
465fba4af4SMike Rapoport */
pte_free_kernel(struct mm_struct * mm,pte_t * pte)475fba4af4SMike Rapoport static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
485fba4af4SMike Rapoport {
49*c787ae5bSVishal Moola (Oracle) pagetable_free(virt_to_ptdesc(pte));
505fba4af4SMike Rapoport }
515fba4af4SMike Rapoport
525fba4af4SMike Rapoport /**
53*c787ae5bSVishal Moola (Oracle) * __pte_alloc_one - allocate memory for a PTE-level user page table
545fba4af4SMike Rapoport * @mm: the mm_struct of the current context
555fba4af4SMike Rapoport * @gfp: GFP flags to use for the allocation
565fba4af4SMike Rapoport *
57*c787ae5bSVishal Moola (Oracle) * Allocate memory for a page table and ptdesc and runs pagetable_pte_ctor().
585fba4af4SMike Rapoport *
595fba4af4SMike Rapoport * This function is intended for architectures that need
605fba4af4SMike Rapoport * anything beyond simple page allocation or must have custom GFP flags.
615fba4af4SMike Rapoport *
62*c787ae5bSVishal Moola (Oracle) * Return: `struct page` referencing the ptdesc or %NULL on error
635fba4af4SMike Rapoport */
__pte_alloc_one(struct mm_struct * mm,gfp_t gfp)645fba4af4SMike Rapoport static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp)
655fba4af4SMike Rapoport {
66*c787ae5bSVishal Moola (Oracle) struct ptdesc *ptdesc;
675fba4af4SMike Rapoport
68*c787ae5bSVishal Moola (Oracle) ptdesc = pagetable_alloc(gfp, 0);
69*c787ae5bSVishal Moola (Oracle) if (!ptdesc)
705fba4af4SMike Rapoport return NULL;
71*c787ae5bSVishal Moola (Oracle) if (!pagetable_pte_ctor(ptdesc)) {
72*c787ae5bSVishal Moola (Oracle) pagetable_free(ptdesc);
735fba4af4SMike Rapoport return NULL;
745fba4af4SMike Rapoport }
755fba4af4SMike Rapoport
76*c787ae5bSVishal Moola (Oracle) return ptdesc_page(ptdesc);
775fba4af4SMike Rapoport }
785fba4af4SMike Rapoport
795fba4af4SMike Rapoport #ifndef __HAVE_ARCH_PTE_ALLOC_ONE
805fba4af4SMike Rapoport /**
815fba4af4SMike Rapoport * pte_alloc_one - allocate a page for PTE-level user page table
825fba4af4SMike Rapoport * @mm: the mm_struct of the current context
835fba4af4SMike Rapoport *
84*c787ae5bSVishal Moola (Oracle) * Allocate memory for a page table and ptdesc and runs pagetable_pte_ctor().
855fba4af4SMike Rapoport *
86*c787ae5bSVishal Moola (Oracle) * Return: `struct page` referencing the ptdesc or %NULL on error
875fba4af4SMike Rapoport */
pte_alloc_one(struct mm_struct * mm)885fba4af4SMike Rapoport static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
895fba4af4SMike Rapoport {
905fba4af4SMike Rapoport return __pte_alloc_one(mm, GFP_PGTABLE_USER);
915fba4af4SMike Rapoport }
925fba4af4SMike Rapoport #endif
935fba4af4SMike Rapoport
945fba4af4SMike Rapoport /*
955fba4af4SMike Rapoport * Should really implement gc for free page table pages. This could be
965fba4af4SMike Rapoport * done with a reference count in struct page.
975fba4af4SMike Rapoport */
985fba4af4SMike Rapoport
995fba4af4SMike Rapoport /**
100*c787ae5bSVishal Moola (Oracle) * pte_free - free PTE-level user page table memory
1015fba4af4SMike Rapoport * @mm: the mm_struct of the current context
102*c787ae5bSVishal Moola (Oracle) * @pte_page: the `struct page` referencing the ptdesc
1035fba4af4SMike Rapoport */
pte_free(struct mm_struct * mm,struct page * pte_page)1045fba4af4SMike Rapoport static inline void pte_free(struct mm_struct *mm, struct page *pte_page)
1055fba4af4SMike Rapoport {
106*c787ae5bSVishal Moola (Oracle) struct ptdesc *ptdesc = page_ptdesc(pte_page);
107*c787ae5bSVishal Moola (Oracle)
108*c787ae5bSVishal Moola (Oracle) pagetable_pte_dtor(ptdesc);
109*c787ae5bSVishal Moola (Oracle) pagetable_free(ptdesc);
1105fba4af4SMike Rapoport }
1115fba4af4SMike Rapoport
1121355c31eSMike Rapoport
1131355c31eSMike Rapoport #if CONFIG_PGTABLE_LEVELS > 2
1141355c31eSMike Rapoport
1151355c31eSMike Rapoport #ifndef __HAVE_ARCH_PMD_ALLOC_ONE
1161355c31eSMike Rapoport /**
117*c787ae5bSVishal Moola (Oracle) * pmd_alloc_one - allocate memory for a PMD-level page table
1181355c31eSMike Rapoport * @mm: the mm_struct of the current context
1191355c31eSMike Rapoport *
120*c787ae5bSVishal Moola (Oracle) * Allocate memory for a page table and ptdesc and runs pagetable_pmd_ctor().
121*c787ae5bSVishal Moola (Oracle) *
1221355c31eSMike Rapoport * Allocations use %GFP_PGTABLE_USER in user context and
1231355c31eSMike Rapoport * %GFP_PGTABLE_KERNEL in kernel context.
1241355c31eSMike Rapoport *
1251355c31eSMike Rapoport * Return: pointer to the allocated memory or %NULL on error
1261355c31eSMike Rapoport */
pmd_alloc_one(struct mm_struct * mm,unsigned long addr)1271355c31eSMike Rapoport static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
1281355c31eSMike Rapoport {
129*c787ae5bSVishal Moola (Oracle) struct ptdesc *ptdesc;
1301355c31eSMike Rapoport gfp_t gfp = GFP_PGTABLE_USER;
1311355c31eSMike Rapoport
1321355c31eSMike Rapoport if (mm == &init_mm)
1331355c31eSMike Rapoport gfp = GFP_PGTABLE_KERNEL;
134*c787ae5bSVishal Moola (Oracle) ptdesc = pagetable_alloc(gfp, 0);
135*c787ae5bSVishal Moola (Oracle) if (!ptdesc)
1361355c31eSMike Rapoport return NULL;
137*c787ae5bSVishal Moola (Oracle) if (!pagetable_pmd_ctor(ptdesc)) {
138*c787ae5bSVishal Moola (Oracle) pagetable_free(ptdesc);
1391355c31eSMike Rapoport return NULL;
1401355c31eSMike Rapoport }
141*c787ae5bSVishal Moola (Oracle) return ptdesc_address(ptdesc);
1421355c31eSMike Rapoport }
1431355c31eSMike Rapoport #endif
1441355c31eSMike Rapoport
1451355c31eSMike Rapoport #ifndef __HAVE_ARCH_PMD_FREE
pmd_free(struct mm_struct * mm,pmd_t * pmd)1461355c31eSMike Rapoport static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
1471355c31eSMike Rapoport {
148*c787ae5bSVishal Moola (Oracle) struct ptdesc *ptdesc = virt_to_ptdesc(pmd);
149*c787ae5bSVishal Moola (Oracle)
1501355c31eSMike Rapoport BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
151*c787ae5bSVishal Moola (Oracle) pagetable_pmd_dtor(ptdesc);
152*c787ae5bSVishal Moola (Oracle) pagetable_free(ptdesc);
1531355c31eSMike Rapoport }
1541355c31eSMike Rapoport #endif
1551355c31eSMike Rapoport
1561355c31eSMike Rapoport #endif /* CONFIG_PGTABLE_LEVELS > 2 */
1571355c31eSMike Rapoport
158d9e8b929SMike Rapoport #if CONFIG_PGTABLE_LEVELS > 3
159d9e8b929SMike Rapoport
__pud_alloc_one(struct mm_struct * mm,unsigned long addr)16060639f74SAlexandre Ghiti static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr)
16160639f74SAlexandre Ghiti {
16260639f74SAlexandre Ghiti gfp_t gfp = GFP_PGTABLE_USER;
163*c787ae5bSVishal Moola (Oracle) struct ptdesc *ptdesc;
16460639f74SAlexandre Ghiti
16560639f74SAlexandre Ghiti if (mm == &init_mm)
16660639f74SAlexandre Ghiti gfp = GFP_PGTABLE_KERNEL;
167*c787ae5bSVishal Moola (Oracle) gfp &= ~__GFP_HIGHMEM;
168*c787ae5bSVishal Moola (Oracle)
169*c787ae5bSVishal Moola (Oracle) ptdesc = pagetable_alloc(gfp, 0);
170*c787ae5bSVishal Moola (Oracle) if (!ptdesc)
171*c787ae5bSVishal Moola (Oracle) return NULL;
172*c787ae5bSVishal Moola (Oracle) return ptdesc_address(ptdesc);
17360639f74SAlexandre Ghiti }
17460639f74SAlexandre Ghiti
1759922c1deSMike Rapoport #ifndef __HAVE_ARCH_PUD_ALLOC_ONE
176d9e8b929SMike Rapoport /**
177*c787ae5bSVishal Moola (Oracle) * pud_alloc_one - allocate memory for a PUD-level page table
178d9e8b929SMike Rapoport * @mm: the mm_struct of the current context
179d9e8b929SMike Rapoport *
180*c787ae5bSVishal Moola (Oracle) * Allocate memory for a page table using %GFP_PGTABLE_USER for user context
181*c787ae5bSVishal Moola (Oracle) * and %GFP_PGTABLE_KERNEL for kernel context.
182d9e8b929SMike Rapoport *
183d9e8b929SMike Rapoport * Return: pointer to the allocated memory or %NULL on error
184d9e8b929SMike Rapoport */
pud_alloc_one(struct mm_struct * mm,unsigned long addr)185d9e8b929SMike Rapoport static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
186d9e8b929SMike Rapoport {
18760639f74SAlexandre Ghiti return __pud_alloc_one(mm, addr);
188d9e8b929SMike Rapoport }
189d9e8b929SMike Rapoport #endif
190d9e8b929SMike Rapoport
__pud_free(struct mm_struct * mm,pud_t * pud)19160639f74SAlexandre Ghiti static inline void __pud_free(struct mm_struct *mm, pud_t *pud)
192d9e8b929SMike Rapoport {
193d9e8b929SMike Rapoport BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
194*c787ae5bSVishal Moola (Oracle) pagetable_free(virt_to_ptdesc(pud));
195d9e8b929SMike Rapoport }
196d9e8b929SMike Rapoport
19760639f74SAlexandre Ghiti #ifndef __HAVE_ARCH_PUD_FREE
pud_free(struct mm_struct * mm,pud_t * pud)19860639f74SAlexandre Ghiti static inline void pud_free(struct mm_struct *mm, pud_t *pud)
19960639f74SAlexandre Ghiti {
20060639f74SAlexandre Ghiti __pud_free(mm, pud);
20160639f74SAlexandre Ghiti }
20260639f74SAlexandre Ghiti #endif
20360639f74SAlexandre Ghiti
204d9e8b929SMike Rapoport #endif /* CONFIG_PGTABLE_LEVELS > 3 */
205d9e8b929SMike Rapoport
206f9cb654cSMike Rapoport #ifndef __HAVE_ARCH_PGD_FREE
pgd_free(struct mm_struct * mm,pgd_t * pgd)207f9cb654cSMike Rapoport static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
208f9cb654cSMike Rapoport {
209*c787ae5bSVishal Moola (Oracle) pagetable_free(virt_to_ptdesc(pgd));
210f9cb654cSMike Rapoport }
211f9cb654cSMike Rapoport #endif
212f9cb654cSMike Rapoport
2135fba4af4SMike Rapoport #endif /* CONFIG_MMU */
2145fba4af4SMike Rapoport
2155c01b46bSArnd Bergmann #endif /* __ASM_GENERIC_PGALLOC_H */
216