/openbmc/linux/mm/ |
H A D | pgtable-generic.c | 138 pmd_t *pmdp) in pmdp_huge_clear_flush() argument 142 VM_BUG_ON(pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) && in pmdp_huge_clear_flush() 143 !pmd_devmap(*pmdp)); in pmdp_huge_clear_flush() 171 if (!pmd_huge_pte(mm, pmdp)) in pgtable_trans_huge_deposit() 175 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit() 188 pgtable = pmd_huge_pte(mm, pmdp); in pgtable_trans_huge_withdraw() 191 if (pmd_huge_pte(mm, pmdp)) in pgtable_trans_huge_withdraw() 199 pmd_t *pmdp) in pmdp_invalidate() argument 202 pmd_t old = pmdp_establish(vma, address, pmdp, pmd_mkinvalid(*pmdp)); in pmdp_invalidate() 210 pmd_t *pmdp) in pmdp_invalidate_ad() argument [all …]
|
/openbmc/linux/arch/powerpc/mm/book3s64/ |
H A D | hash_pgtable.c | 152 pmd_t *pmdp; in hash__map_kernel_page() local 163 if (!pmdp) in hash__map_kernel_page() 165 ptep = pte_alloc_kernel(pmdp, ea); in hash__map_kernel_page() 198 WARN_ON(!hash__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); in hash__pmd_hugepage_update() 211 : "r" (pmdp), "r" (cpu_to_be64(clr)), "m" (*pmdp), in hash__pmd_hugepage_update() 224 pmd_t *pmdp) in hash__pmdp_collapse_flush() argument 229 VM_BUG_ON(pmd_trans_huge(*pmdp)); in hash__pmdp_collapse_flush() 230 VM_BUG_ON(pmd_devmap(*pmdp)); in hash__pmdp_collapse_flush() 232 pmd = *pmdp; in hash__pmdp_collapse_flush() 233 pmd_clear(pmdp); in hash__pmdp_collapse_flush() [all …]
|
H A D | pgtable.c | 49 pmd_t *pmdp, pmd_t entry, int dirty) in pmdp_set_access_flags() argument 53 WARN_ON(!pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); in pmdp_set_access_flags() 56 changed = !pmd_same(*(pmdp), entry); in pmdp_set_access_flags() 90 unsigned long address, pmd_t *pmdp) in pmdp_test_and_clear_young() argument 106 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument 114 WARN_ON(pte_hw_valid(pmd_pte(*pmdp)) && !pte_protnone(pmd_pte(*pmdp))); in set_pmd_at() 115 assert_spin_locked(pmd_lockptr(mm, pmdp)); in set_pmd_at() 169 pmd_t *pmdp) in pmdp_invalidate() argument 173 VM_WARN_ON_ONCE(!pmd_present(*pmdp)); in pmdp_invalidate() 184 VM_BUG_ON((pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) && in pmdp_huge_get_and_clear_full() [all …]
|
H A D | radix_pgtable.c | 79 pmd_t *pmdp; in early_map_kernel_page() local 101 ptep = pmdp_ptep(pmdp); in early_map_kernel_page() 131 pmd_t *pmdp; in __map_kernel_page() local 161 if (!pmdp) in __map_kernel_page() 164 ptep = pmdp_ptep(pmdp); in __map_kernel_page() 192 pmd_t *pmdp; in radix__change_memory_range() local 212 if (!pmdp) in radix__change_memory_range() 1365 WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); in radix__pmd_hugepage_update() 1393 pmd_t *pmdp) in radix__pmdp_collapse_flush() argument 1404 pmd = *pmdp; in radix__pmdp_collapse_flush() [all …]
|
H A D | hash_hugepage.c | 22 pmd_t *pmdp, unsigned long trap, unsigned long flags, in __hash_page_thp() argument 36 pmd_t pmd = READ_ONCE(*pmdp); in __hash_page_thp() 52 } while (!pmd_xchg(pmdp, __pmd(old_pmd), __pmd(new_pmd))); in __hash_page_thp() 80 hpte_slot_array = get_hpte_slot_array(pmdp); in __hash_page_thp() 88 flush_hash_hugepage(vsid, ea, pmdp, MMU_PAGE_64K, in __hash_page_thp() 165 *pmdp = __pmd(old_pmd); in __hash_page_thp() 189 *pmdp = __pmd(new_pmd & ~H_PAGE_BUSY); in __hash_page_thp()
|
/openbmc/linux/arch/arm64/include/asm/ |
H A D | pgalloc.h | 23 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) in __pud_populate() argument 25 set_pud(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); in __pud_populate() 28 static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) in pud_populate() argument 33 __pud_populate(pudp, __pa(pmdp), pudval); in pud_populate() 36 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) in __pud_populate() argument 66 static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t ptep, in __pmd_populate() argument 69 set_pmd(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot)); in __pmd_populate() 77 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_populate_kernel() argument 80 __pmd_populate(pmdp, __pa(ptep), PMD_TYPE_TABLE | PMD_TABLE_UXN); in pmd_populate_kernel() 84 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) in pmd_populate() argument [all …]
|
/openbmc/linux/arch/powerpc/mm/nohash/ |
H A D | 40x.c | 106 pmd_t *pmdp; in mmu_mapin_ram() local 109 pmdp = pmd_off_k(v); in mmu_mapin_ram() 110 *pmdp++ = __pmd(val); in mmu_mapin_ram() 111 *pmdp++ = __pmd(val); in mmu_mapin_ram() 112 *pmdp++ = __pmd(val); in mmu_mapin_ram() 113 *pmdp++ = __pmd(val); in mmu_mapin_ram() 121 pmd_t *pmdp; in mmu_mapin_ram() local 124 pmdp = pmd_off_k(v); in mmu_mapin_ram() 125 *pmdp = __pmd(val); in mmu_mapin_ram()
|
H A D | book3e_pgtable.c | 79 pmd_t *pmdp; in map_kernel_page() local 89 pmdp = pmd_alloc(&init_mm, pudp, ea); in map_kernel_page() 90 if (!pmdp) in map_kernel_page() 92 ptep = pte_alloc_kernel(pmdp, ea); in map_kernel_page() 104 pmdp = early_alloc_pgtable(PMD_TABLE_SIZE); in map_kernel_page() 105 pud_populate(&init_mm, pudp, pmdp); in map_kernel_page() 107 pmdp = pmd_offset(pudp, ea); in map_kernel_page() 108 if (!pmd_present(*pmdp)) { in map_kernel_page() 110 pmd_populate_kernel(&init_mm, pmdp, ptep); in map_kernel_page() 112 ptep = pte_offset_kernel(pmdp, ea); in map_kernel_page()
|
H A D | 8xx.c | 49 static pte_t __init *early_hugepd_alloc_kernel(hugepd_t *pmdp, unsigned long va) in early_hugepd_alloc_kernel() argument 51 if (hpd_val(*pmdp) == 0) { in early_hugepd_alloc_kernel() 57 hugepd_populate_kernel((hugepd_t *)pmdp, ptep, PAGE_SHIFT_8M); in early_hugepd_alloc_kernel() 58 hugepd_populate_kernel((hugepd_t *)pmdp + 1, ptep, PAGE_SHIFT_8M); in early_hugepd_alloc_kernel() 60 return hugepte_offset(*(hugepd_t *)pmdp, va, PGDIR_SHIFT); in early_hugepd_alloc_kernel() 66 pmd_t *pmdp = pmd_off_k(va); in __early_map_kernel_hugepage() local 77 ptep = early_pte_alloc_kernel(pmdp, va); in __early_map_kernel_hugepage() 79 ptep = early_hugepd_alloc_kernel((hugepd_t *)pmdp, va); in __early_map_kernel_hugepage() 82 ptep = pte_offset_kernel(pmdp, va); in __early_map_kernel_hugepage() 84 ptep = hugepte_offset(*(hugepd_t *)pmdp, va, PGDIR_SHIFT); in __early_map_kernel_hugepage()
|
/openbmc/linux/arch/sparc/mm/ |
H A D | tlb.c | 223 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument 225 pmd_t orig = *pmdp; in set_pmd_at() 227 *pmdp = pmd; in set_pmd_at() 237 old = *pmdp; in pmdp_establish() 248 pmd_t *pmdp) in pmdp_invalidate() argument 252 VM_WARN_ON_ONCE(!pmd_present(*pmdp)); in pmdp_invalidate() 253 entry = __pmd(pmd_val(*pmdp) & ~_PAGE_VALID); in pmdp_invalidate() 277 if (!pmd_huge_pte(mm, pmdp)) in pgtable_trans_huge_deposit() 281 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit() 292 pgtable = pmd_huge_pte(mm, pmdp); in pgtable_trans_huge_withdraw() [all …]
|
/openbmc/linux/arch/x86/include/asm/ |
H A D | pgtable-3level.h | 48 static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) in native_set_pmd() argument 50 pxx_xchg64(pmd, pmdp, native_pmd_val(pmd)); in native_set_pmd() 74 static inline void native_pmd_clear(pmd_t *pmdp) in native_pmd_clear() argument 76 WRITE_ONCE(pmdp->pmd_low, 0); in native_pmd_clear() 78 WRITE_ONCE(pmdp->pmd_high, 0); in native_pmd_clear() 110 return pxx_xchg64(pmd, pmdp, 0ULL); in native_pmdp_get_and_clear() 126 unsigned long address, pmd_t *pmdp, pmd_t pmd) in pmdp_establish() argument 137 old.pmd_low = xchg(&pmdp->pmd_low, pmd.pmd_low); in pmdp_establish() 138 old.pmd_high = READ_ONCE(pmdp->pmd_high); in pmdp_establish() 139 WRITE_ONCE(pmdp->pmd_high, pmd.pmd_high); in pmdp_establish() [all …]
|
/openbmc/linux/arch/powerpc/include/asm/nohash/32/ |
H A D | pgalloc.h | 17 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate_kernel() argument 21 *pmdp = __pmd((unsigned long)pte | _PMD_PRESENT); in pmd_populate_kernel() 23 *pmdp = __pmd(__pa(pte) | _PMD_PRESENT); in pmd_populate_kernel() 26 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate() argument 30 *pmdp = __pmd((unsigned long)pte_page | _PMD_PRESENT); in pmd_populate() 32 *pmdp = __pmd(__pa(pte_page) | _PMD_USER | _PMD_PRESENT); in pmd_populate()
|
/openbmc/linux/arch/s390/mm/ |
H A D | pgtable.c | 410 __pmdp_csp(pmdp); in pmdp_idte_global() 421 old = *pmdp; in pmdp_flush_direct() 439 old = *pmdp; in pmdp_flush_lazy() 445 set_pmd(pmdp, set_pmd_bit(*pmdp, __pgprot(_SEGMENT_ENTRY_INVALID))); in pmdp_flush_lazy() 497 set_pmd(pmdp, new); in pmdp_xchg_direct() 510 set_pmd(pmdp, new); in pmdp_xchg_lazy() 817 pmd_t *pmdp; in set_guest_storage_key() local 839 if (pmd_large(*pmdp)) { in set_guest_storage_key() 927 pmd_t *pmdp; in reset_guest_reference_bit() local 950 if (pmd_large(*pmdp)) { in reset_guest_reference_bit() [all …]
|
H A D | pageattr.c | 93 ptep = pte_offset_kernel(pmdp, addr); in walk_pte_level() 154 pmd_t new = *pmdp; in modify_pmd_page() 180 pmd_t *pmdp; in walk_pmd_level() local 183 pmdp = pmd_offset(pudp, addr); in walk_pmd_level() 185 if (pmd_none(*pmdp)) in walk_pmd_level() 188 if (pmd_large(*pmdp)) { in walk_pmd_level() 193 rc = split_pmd_page(pmdp, addr); in walk_pmd_level() 204 pmdp++; in walk_pmd_level() 214 pmd_t *pm_dir, *pmdp; in split_pud_page() local 227 pmdp = pm_dir; in split_pud_page() [all …]
|
H A D | gmap.c | 928 pmd_t *pmdp; in gmap_pmd_op_walk() local 932 if (!pmdp) in gmap_pmd_op_walk() 937 return pmd_none(*pmdp) ? NULL : pmdp; in gmap_pmd_op_walk() 948 return pmdp; in gmap_pmd_op_walk() 1061 pmd_t *pmdp; in gmap_protect_range() local 1068 if (pmdp) { in gmap_protect_range() 2335 pmd_t *pmdp; in gmap_pmdp_clear() local 2344 if (pmdp) { in gmap_pmdp_clear() 2390 pmd_t *pmdp; in gmap_pmdp_idte_local() local 2425 pmd_t *pmdp; in gmap_pmdp_idte_global() local [all …]
|
/openbmc/linux/arch/arm/include/asm/ |
H A D | pgalloc.h | 107 static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte, in __pmd_populate() argument 111 pmdp[0] = __pmd(pmdval); in __pmd_populate() 113 pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t)); in __pmd_populate() 115 flush_pmd_entry(pmdp); in __pmd_populate() 125 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_populate_kernel() argument 130 __pmd_populate(pmdp, __pa(ptep), _PAGE_KERNEL_TABLE); in pmd_populate_kernel() 134 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) in pmd_populate() argument 144 __pmd_populate(pmdp, page_to_phys(ptep), prot); in pmd_populate()
|
/openbmc/linux/arch/arm64/mm/ |
H A D | mmu.c | 215 pmd = READ_ONCE(*pmdp); in alloc_init_cont_pte() 240 pmd_t *pmdp; in init_pmd() local 875 pmd_t *pmdp, pmd; in unmap_hotplug_pmd_range() local 886 pmd_clear(pmdp); in unmap_hotplug_pmd_range() 1013 pmd_clear(pmdp); in free_empty_pte_table() 1022 pmd_t *pmdp, pmd; in free_empty_pmd_table() local 1195 set_pmd(pmdp, new_pmd); in pmd_set_huge() 1211 pmd_clear(pmdp); in pmd_clear_huge() 1228 pmd_clear(pmdp); in pmd_free_pte_page() 1237 pmd_t *pmdp; in pud_free_pmd_page() local [all …]
|
H A D | kasan_init.c | 62 static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node, in kasan_pte_offset() argument 65 if (pmd_none(READ_ONCE(*pmdp))) { in kasan_pte_offset() 69 __pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE); in kasan_pte_offset() 72 return early ? pte_offset_kimg(pmdp, addr) in kasan_pte_offset() 73 : pte_offset_kernel(pmdp, addr); in kasan_pte_offset() 102 static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr, in kasan_pte_populate() argument 106 pte_t *ptep = kasan_pte_offset(pmdp, addr, node, early); in kasan_pte_populate() 123 pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early); in kasan_pmd_populate() local 127 kasan_pte_populate(pmdp, addr, next, node, early); in kasan_pmd_populate() 128 } while (pmdp++, addr = next, addr != end && pmd_none(READ_ONCE(*pmdp))); in kasan_pmd_populate()
|
/openbmc/linux/include/linux/ |
H A D | pgtable.h | 335 pmd_t *pmdp) in pmdp_test_and_clear_young() argument 337 pmd_t pmd = *pmdp; in pmdp_test_and_clear_young() 348 pmd_t *pmdp) in pmdp_test_and_clear_young() argument 522 pmd_t *pmdp) in pmdp_huge_get_and_clear() argument 524 pmd_t pmd = *pmdp; in pmdp_huge_get_and_clear() 526 pmd_clear(pmdp); in pmdp_huge_get_and_clear() 617 pmd_t *pmdp); 705 pmd_t *pmdp) in pmdp_collapse_flush() argument 708 return *pmdp; in pmdp_collapse_flush() 744 pmd_t *pmdp); [all …]
|
/openbmc/linux/arch/powerpc/mm/kasan/ |
H A D | init_book3e_64.c | 37 pmd_t *pmdp; in kasan_map_kernel_page() local 49 pmdp = memblock_alloc(PMD_TABLE_SIZE, PMD_TABLE_SIZE); in kasan_map_kernel_page() 50 memcpy(pmdp, kasan_early_shadow_pmd, PMD_TABLE_SIZE); in kasan_map_kernel_page() 51 pud_populate(&init_mm, pudp, pmdp); in kasan_map_kernel_page() 53 pmdp = pmd_offset(pudp, ea); in kasan_map_kernel_page() 54 if (kasan_pte_table(*pmdp)) { in kasan_map_kernel_page() 57 pmd_populate_kernel(&init_mm, pmdp, ptep); in kasan_map_kernel_page() 59 ptep = pte_offset_kernel(pmdp, ea); in kasan_map_kernel_page()
|
/openbmc/linux/arch/powerpc/mm/ |
H A D | pgtable_32.c | 42 pmd_t *pmdp = pmd_off_k(addr); in early_ioremap_init() local 45 addr += PGDIR_SIZE, ptep += PTRS_PER_PTE, pmdp++) in early_ioremap_init() 46 pmd_populate_kernel(&init_mm, pmdp, ptep); in early_ioremap_init() 62 pte_t __init *early_pte_alloc_kernel(pmd_t *pmdp, unsigned long va) in early_pte_alloc_kernel() argument 64 if (pmd_none(*pmdp)) { in early_pte_alloc_kernel() 67 pmd_populate_kernel(&init_mm, pmdp, ptep); in early_pte_alloc_kernel() 69 return pte_offset_kernel(pmdp, va); in early_pte_alloc_kernel()
|
/openbmc/linux/arch/powerpc/include/asm/book3s/64/ |
H A D | hash-4k.h | 109 static inline char *get_hpte_slot_array(pmd_t *pmdp) in get_hpte_slot_array() argument 146 unsigned long addr, pmd_t *pmdp, 149 unsigned long address, pmd_t *pmdp); 150 extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 152 extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); 154 unsigned long addr, pmd_t *pmdp);
|
/openbmc/linux/arch/riscv/include/asm/ |
H A D | pgtable.h | 251 WRITE_ONCE(*pmdp, pmd); in set_pmd() 254 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument 256 set_pmd(pmdp, __pmd(0)); in pmd_clear() 498 unsigned long address, pmd_t *pmdp) in update_mmu_cache_pmd() argument 500 pte_t *ptep = (pte_t *)pmdp; in update_mmu_cache_pmd() 711 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument 713 page_table_check_pmd_set(mm, pmdp, pmd); in set_pmd_at() 749 unsigned long address, pmd_t *pmdp, in pmdp_set_access_flags() argument 757 unsigned long address, pmd_t *pmdp) in pmdp_test_and_clear_young() argument 764 unsigned long address, pmd_t *pmdp) in pmdp_huge_get_and_clear() argument [all …]
|
/openbmc/linux/arch/xtensa/include/asm/ |
H A D | pgalloc.h | 24 #define pmd_populate_kernel(mm, pmdp, ptep) \ argument 25 (pmd_val(*(pmdp)) = ((unsigned long)ptep)) 26 #define pmd_populate(mm, pmdp, page) \ argument 27 (pmd_val(*(pmdp)) = ((unsigned long)page_to_virt(page)))
|
/openbmc/linux/arch/arm/mm/ |
H A D | kasan_init.c | 37 static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr, in kasan_pte_populate() argument 41 pte_t *ptep = pte_offset_kernel(pmdp, addr); in kasan_pte_populate() 91 pmd_t *pmdp = pmd_offset(pudp, addr); in kasan_pmd_populate() local 94 if (pmd_none(*pmdp)) { in kasan_pmd_populate() 108 pmd_populate_kernel(&init_mm, pmdp, p); in kasan_pmd_populate() 109 flush_pmd_entry(pmdp); in kasan_pmd_populate() 113 kasan_pte_populate(pmdp, addr, next, early); in kasan_pmd_populate() 114 } while (pmdp++, addr = next, addr != end); in kasan_pmd_populate()
|