| /openbmc/linux/arch/powerpc/mm/ |
| H A D | pgtable-frag.c | 21 struct ptdesc *ptdesc; in pte_frag_destroy() local
28 pagetable_pte_dtor(ptdesc); in pte_frag_destroy()
29 pagetable_free(ptdesc); in pte_frag_destroy()
58 struct ptdesc *ptdesc; in __alloc_for_ptecache() local
62 if (!ptdesc) in __alloc_for_ptecache()
65 pagetable_free(ptdesc); in __alloc_for_ptecache()
70 if (!ptdesc) in __alloc_for_ptecache()
111 struct ptdesc *ptdesc; in pte_free_now() local
113 ptdesc = container_of(head, struct ptdesc, pt_rcu_head); in pte_free_now()
115 pagetable_free(ptdesc); in pte_free_now()
[all …]
|
| /openbmc/linux/include/asm-generic/ |
| H A D | pgalloc.h | 21 struct ptdesc *ptdesc = pagetable_alloc(GFP_PGTABLE_KERNEL & in __pte_alloc_one_kernel() local
24 if (!ptdesc) in __pte_alloc_one_kernel()
66 struct ptdesc *ptdesc; in __pte_alloc_one() local
69 if (!ptdesc) in __pte_alloc_one()
106 struct ptdesc *ptdesc = page_ptdesc(pte_page); in pte_free() local
109 pagetable_free(ptdesc); in pte_free()
129 struct ptdesc *ptdesc; in pmd_alloc_one() local
135 if (!ptdesc) in pmd_alloc_one()
148 struct ptdesc *ptdesc = virt_to_ptdesc(pmd); in pmd_free() local
163 struct ptdesc *ptdesc; in __pud_alloc_one() local
[all …]
|
| H A D | tlb.h | 489 static inline void tlb_remove_page_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt) in tlb_remove_page_ptdesc()
|
| /openbmc/linux/arch/m68k/include/asm/ |
| H A D | mcf_pgalloc.h | 17 struct ptdesc *ptdesc = pagetable_alloc((GFP_DMA | __GFP_ZERO) & in pte_alloc_one_kernel() local
20 if (!ptdesc) in pte_alloc_one_kernel()
38 struct ptdesc *ptdesc = virt_to_ptdesc(pgtable); in __pte_free_tlb() local
41 pagetable_free(ptdesc); in __pte_free_tlb()
46 struct ptdesc *ptdesc = pagetable_alloc(GFP_DMA | __GFP_ZERO, 0); in pte_alloc_one() local
49 if (!ptdesc) in pte_alloc_one()
52 pagetable_free(ptdesc); in pte_alloc_one()
62 struct ptdesc *ptdesc = virt_to_ptdesc(pgtable); in pte_free() local
65 pagetable_free(ptdesc); in pte_free()
82 struct ptdesc *ptdesc = pagetable_alloc((GFP_DMA | __GFP_NOWARN) & in pgd_alloc() local
[all …]
|
| /openbmc/linux/arch/s390/mm/ |
| H A D | pgalloc.c | 48 if (!ptdesc) in crst_table_alloc()
145 struct ptdesc *ptdesc; in page_table_alloc_pgste() local
149 if (ptdesc) { in page_table_alloc_pgste()
248 struct ptdesc *ptdesc; in page_table_alloc() local
285 if (!ptdesc) in page_table_alloc()
328 struct ptdesc *ptdesc; in pte_free_now() local
330 ptdesc = container_of(head, struct ptdesc, pt_rcu_head); in pte_free_now()
338 struct ptdesc *ptdesc = virt_to_ptdesc(table); in page_table_free() local
386 struct ptdesc *ptdesc = virt_to_ptdesc(table); in page_table_free_rcu() local
425 struct ptdesc *ptdesc = virt_to_ptdesc(table); in __tlb_remove_table() local
[all …]
|
| /openbmc/linux/arch/loongarch/include/asm/ |
| H A D | pgalloc.h | 58 struct ptdesc *ptdesc; in pmd_alloc_one() local
60 ptdesc = pagetable_alloc(GFP_KERNEL_ACCOUNT, 0); in pmd_alloc_one()
61 if (!ptdesc) in pmd_alloc_one()
64 if (!pagetable_pmd_ctor(ptdesc)) { in pmd_alloc_one()
65 pagetable_free(ptdesc); in pmd_alloc_one()
69 pmd = ptdesc_address(ptdesc); in pmd_alloc_one()
83 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); in pud_alloc_one() local
85 if (!ptdesc) in pud_alloc_one()
87 pud = ptdesc_address(ptdesc); in pud_alloc_one()
|
| /openbmc/linux/arch/mips/include/asm/ |
| H A D | pgalloc.h | 68 struct ptdesc *ptdesc; in pmd_alloc_one() local
70 ptdesc = pagetable_alloc(GFP_KERNEL_ACCOUNT, PMD_TABLE_ORDER); in pmd_alloc_one()
71 if (!ptdesc) in pmd_alloc_one()
74 if (!pagetable_pmd_ctor(ptdesc)) { in pmd_alloc_one()
75 pagetable_free(ptdesc); in pmd_alloc_one()
79 pmd = ptdesc_address(ptdesc); in pmd_alloc_one()
93 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, in pud_alloc_one() local
96 if (!ptdesc) in pud_alloc_one()
98 pud = ptdesc_address(ptdesc); in pud_alloc_one()
|
| /openbmc/linux/arch/arm/include/asm/ |
| H A D | tlb.h | 42 struct ptdesc *ptdesc = page_ptdesc(pte); in __pte_free_tlb() local
44 pagetable_pte_dtor(ptdesc); in __pte_free_tlb()
55 tlb_remove_ptdesc(tlb, ptdesc); in __pte_free_tlb()
62 struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); in __pmd_free_tlb() local
64 pagetable_pmd_dtor(ptdesc); in __pmd_free_tlb()
65 tlb_remove_ptdesc(tlb, ptdesc); in __pmd_free_tlb()
|
| /openbmc/linux/arch/arm64/include/asm/ |
| H A D | tlb.h | 78 struct ptdesc *ptdesc = page_ptdesc(pte); in __pte_free_tlb() local
80 pagetable_pte_dtor(ptdesc); in __pte_free_tlb()
81 tlb_remove_ptdesc(tlb, ptdesc); in __pte_free_tlb()
88 struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); in __pmd_free_tlb() local
90 pagetable_pmd_dtor(ptdesc); in __pmd_free_tlb()
91 tlb_remove_ptdesc(tlb, ptdesc); in __pmd_free_tlb()
|
| /openbmc/linux/arch/x86/mm/ |
| H A D | pgtable.c | 63 struct ptdesc *ptdesc = virt_to_ptdesc(pmd); in ___pmd_free_tlb() local
72 pagetable_pmd_dtor(ptdesc); in ___pmd_free_tlb()
95 struct ptdesc *ptdesc = virt_to_ptdesc(pgd); in pgd_list_add() local
102 struct ptdesc *ptdesc = virt_to_ptdesc(pgd); in pgd_list_del() local
216 struct ptdesc *ptdesc; in free_pmds() local
223 pagetable_free(ptdesc); in free_pmds()
240 struct ptdesc *ptdesc = pagetable_alloc(gfp, 0); in preallocate_pmds() local
242 if (!ptdesc) in preallocate_pmds()
244 if (ptdesc && !pagetable_pmd_ctor(ptdesc)) { in preallocate_pmds()
246 ptdesc = NULL; in preallocate_pmds()
[all …]
|
| /openbmc/linux/arch/mips/mm/ |
| H A D | pgtable.c | 14 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, in pgd_alloc() local
17 if (ptdesc) { in pgd_alloc()
18 ret = ptdesc_address(ptdesc); in pgd_alloc()
|
| /openbmc/linux/arch/powerpc/mm/book3s64/ |
| H A D | pgtable.c | 389 struct ptdesc *ptdesc; in __alloc_for_pmdcache() local
394 ptdesc = pagetable_alloc(gfp, 0); in __alloc_for_pmdcache()
395 if (!ptdesc) in __alloc_for_pmdcache()
397 if (!pagetable_pmd_ctor(ptdesc)) { in __alloc_for_pmdcache()
398 pagetable_free(ptdesc); in __alloc_for_pmdcache()
404 ret = ptdesc_address(ptdesc); in __alloc_for_pmdcache()
440 struct ptdesc *ptdesc = virt_to_ptdesc(pmd); in pmd_fragment_free() local
442 if (pagetable_is_reserved(ptdesc)) in pmd_fragment_free()
443 return free_reserved_ptdesc(ptdesc); in pmd_fragment_free()
447 pagetable_pmd_dtor(ptdesc); in pmd_fragment_free()
[all …]
|
| H A D | mmu_context.c | 249 struct ptdesc *ptdesc; in pmd_frag_destroy() local
251 ptdesc = virt_to_ptdesc(pmd_frag); in pmd_frag_destroy()
255 if (atomic_sub_and_test(PMD_FRAG_NR - count, &ptdesc->pt_frag_refcount)) { in pmd_frag_destroy()
256 pagetable_pmd_dtor(ptdesc); in pmd_frag_destroy()
257 pagetable_free(ptdesc); in pmd_frag_destroy()
|
| /openbmc/linux/include/linux/ |
| H A D | mm.h | 2851 bool ptlock_alloc(struct ptdesc *ptdesc);
2852 void ptlock_free(struct ptdesc *ptdesc);
2854 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) in ptlock_ptr() argument
2863 static inline bool ptlock_alloc(struct ptdesc *ptdesc) in ptlock_alloc() argument
2868 static inline void ptlock_free(struct ptdesc *ptdesc) in ptlock_free() argument
2872 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) in ptlock_ptr() argument
2883 static inline bool ptlock_init(struct ptdesc *ptdesc) in ptlock_init() argument
2909 static inline void ptlock_free(struct ptdesc *ptdesc) {} in ptlock_free() argument
2988 static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) in pmd_ptlock_init() argument
2996 static inline void pmd_ptlock_free(struct ptdesc *ptdesc) in pmd_ptlock_free() argument
[all …]
|
| H A D | mm_types.h | 411 struct ptdesc { struct
445 static_assert(offsetof(struct page, pg) == offsetof(struct ptdesc, pt)) argument
457 static_assert(sizeof(struct ptdesc) <= sizeof(struct page));
460 const struct ptdesc *: (const struct page *)(pt), \
461 struct ptdesc *: (struct page *)(pt)))
464 const struct ptdesc *: (const struct folio *)(pt), \
465 struct ptdesc *: (struct folio *)(pt)))
468 const struct page *: (const struct ptdesc *)(p), \
469 struct page *: (struct ptdesc *)(p)))
|
| /openbmc/linux/arch/loongarch/mm/ |
| H A D | pgtable.c | 27 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); in pgd_alloc() local
29 if (ptdesc) { in pgd_alloc()
30 ret = (pgd_t *)ptdesc_address(ptdesc); in pgd_alloc()
|
| /openbmc/linux/arch/riscv/mm/ |
| H A D | init.c | 375 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); in alloc_pte_late() local
377 BUG_ON(!ptdesc || !pagetable_pte_ctor(ptdesc)); in alloc_pte_late()
378 return __pa((pte_t *)ptdesc_address(ptdesc)); in alloc_pte_late()
456 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); in alloc_pmd_late() local
458 BUG_ON(!ptdesc || !pagetable_pmd_ctor(ptdesc)); in alloc_pmd_late()
459 return __pa((pmd_t *)ptdesc_address(ptdesc)); in alloc_pmd_late()
|
| /openbmc/linux/arch/sparc/mm/ |
| H A D | init_64.c | 2910 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0); in pte_alloc_one() local
2912 if (!ptdesc) in pte_alloc_one()
2914 if (!pagetable_pte_ctor(ptdesc)) { in pte_alloc_one()
2915 pagetable_free(ptdesc); in pte_alloc_one()
2918 return ptdesc_address(ptdesc); in pte_alloc_one()
2928 struct ptdesc *ptdesc = virt_to_ptdesc(pte); in __pte_free() local
2930 pagetable_pte_dtor(ptdesc); in __pte_free()
2931 pagetable_free(ptdesc); in __pte_free()
|
| /openbmc/linux/arch/arm64/mm/ |
| H A D | mmu.c | 429 struct ptdesc *ptdesc = page_ptdesc(phys_to_page(pa)); in pgd_pgtable_alloc() local
440 BUG_ON(!pagetable_pte_ctor(ptdesc)); in pgd_pgtable_alloc()
442 BUG_ON(!pagetable_pmd_ctor(ptdesc)); in pgd_pgtable_alloc()
|
| /openbmc/linux/arch/arm/mm/ |
| H A D | mmu.c | 740 void *ptdesc = pagetable_alloc(GFP_PGTABLE_KERNEL & ~__GFP_HIGHMEM, in late_alloc() local
743 if (!ptdesc || !pagetable_pte_ctor(ptdesc)) in late_alloc()
745 return ptdesc_to_virt(ptdesc); in late_alloc()
|
| /openbmc/linux/mm/ |
| H A D | memory.c | 6145 bool ptlock_alloc(struct ptdesc *ptdesc) in ptlock_alloc() argument
6152 ptdesc->ptl = ptl; in ptlock_alloc()
6156 void ptlock_free(struct ptdesc *ptdesc) in ptlock_free() argument
6158 kmem_cache_free(page_ptl_cachep, ptdesc->ptl); in ptlock_free()
|