1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * mm/pgtable-generic.c 4 * 5 * Generic pgtable methods declared in linux/pgtable.h 6 * 7 * Copyright (C) 2010 Linus Torvalds 8 */ 9 10 #include <linux/pagemap.h> 11 #include <linux/hugetlb.h> 12 #include <linux/pgtable.h> 13 #include <linux/swap.h> 14 #include <linux/swapops.h> 15 #include <linux/mm_inline.h> 16 #include <asm/tlb.h> 17 18 /* 19 * If a p?d_bad entry is found while walking page tables, report 20 * the error, before resetting entry to p?d_none. Usually (but 21 * very seldom) called out from the p?d_none_or_clear_bad macros. 22 */ 23 24 void pgd_clear_bad(pgd_t *pgd) 25 { 26 pgd_ERROR(*pgd); 27 pgd_clear(pgd); 28 } 29 30 #ifndef __PAGETABLE_P4D_FOLDED 31 void p4d_clear_bad(p4d_t *p4d) 32 { 33 p4d_ERROR(*p4d); 34 p4d_clear(p4d); 35 } 36 #endif 37 38 #ifndef __PAGETABLE_PUD_FOLDED 39 void pud_clear_bad(pud_t *pud) 40 { 41 pud_ERROR(*pud); 42 pud_clear(pud); 43 } 44 #endif 45 46 /* 47 * Note that the pmd variant below can't be stub'ed out just as for p4d/pud 48 * above. pmd folding is special and typically pmd_* macros refer to upper 49 * level even when folded 50 */ 51 void pmd_clear_bad(pmd_t *pmd) 52 { 53 pmd_ERROR(*pmd); 54 pmd_clear(pmd); 55 } 56 57 #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS 58 /* 59 * Only sets the access flags (dirty, accessed), as well as write 60 * permission. Furthermore, we know it always gets set to a "more 61 * permissive" setting, which allows most architectures to optimize 62 * this. We return whether the PTE actually changed, which in turn 63 * instructs the caller to do things like update__mmu_cache. This 64 * used to be done in the caller, but sparc needs minor faults to 65 * force that call on sun4c so we changed this macro slightly 66 */ 67 int ptep_set_access_flags(struct vm_area_struct *vma, 68 unsigned long address, pte_t *ptep, 69 pte_t entry, int dirty) 70 { 71 int changed = !pte_same(ptep_get(ptep), entry); 72 if (changed) { 73 set_pte_at(vma->vm_mm, address, ptep, entry); 74 flush_tlb_fix_spurious_fault(vma, address, ptep); 75 } 76 return changed; 77 } 78 #endif 79 80 #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH 81 int ptep_clear_flush_young(struct vm_area_struct *vma, 82 unsigned long address, pte_t *ptep) 83 { 84 int young; 85 young = ptep_test_and_clear_young(vma, address, ptep); 86 if (young) 87 flush_tlb_page(vma, address); 88 return young; 89 } 90 #endif 91 92 #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH 93 pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address, 94 pte_t *ptep) 95 { 96 struct mm_struct *mm = (vma)->vm_mm; 97 pte_t pte; 98 pte = ptep_get_and_clear(mm, address, ptep); 99 if (pte_accessible(mm, pte)) 100 flush_tlb_page(vma, address); 101 return pte; 102 } 103 #endif 104 105 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 106 107 #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS 108 int pmdp_set_access_flags(struct vm_area_struct *vma, 109 unsigned long address, pmd_t *pmdp, 110 pmd_t entry, int dirty) 111 { 112 int changed = !pmd_same(*pmdp, entry); 113 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 114 if (changed) { 115 set_pmd_at(vma->vm_mm, address, pmdp, entry); 116 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); 117 } 118 return changed; 119 } 120 #endif 121 122 #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH 123 int pmdp_clear_flush_young(struct vm_area_struct *vma, 124 unsigned long address, pmd_t *pmdp) 125 { 126 int young; 127 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 128 young = pmdp_test_and_clear_young(vma, address, pmdp); 129 if (young) 130 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); 131 return young; 132 } 133 #endif 134 135 #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH 136 pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, 137 pmd_t *pmdp) 138 { 139 pmd_t pmd; 140 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 141 VM_BUG_ON(pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) && 142 !pmd_devmap(*pmdp)); 143 pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); 144 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); 145 return pmd; 146 } 147 148 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 149 pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, 150 pud_t *pudp) 151 { 152 pud_t pud; 153 154 VM_BUG_ON(address & ~HPAGE_PUD_MASK); 155 VM_BUG_ON(!pud_trans_huge(*pudp) && !pud_devmap(*pudp)); 156 pud = pudp_huge_get_and_clear(vma->vm_mm, address, pudp); 157 flush_pud_tlb_range(vma, address, address + HPAGE_PUD_SIZE); 158 return pud; 159 } 160 #endif 161 #endif 162 163 #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT 164 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 165 pgtable_t pgtable) 166 { 167 assert_spin_locked(pmd_lockptr(mm, pmdp)); 168 169 /* FIFO */ 170 if (!pmd_huge_pte(mm, pmdp)) 171 INIT_LIST_HEAD(&pgtable->lru); 172 else 173 list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru); 174 pmd_huge_pte(mm, pmdp) = pgtable; 175 } 176 #endif 177 178 #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW 179 /* no "address" argument so destroys page coloring of some arch */ 180 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) 181 { 182 pgtable_t pgtable; 183 184 assert_spin_locked(pmd_lockptr(mm, pmdp)); 185 186 /* FIFO */ 187 pgtable = pmd_huge_pte(mm, pmdp); 188 pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru, 189 struct page, lru); 190 if (pmd_huge_pte(mm, pmdp)) 191 list_del(&pgtable->lru); 192 return pgtable; 193 } 194 #endif 195 196 #ifndef __HAVE_ARCH_PMDP_INVALIDATE 197 pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, 198 pmd_t *pmdp) 199 { 200 pmd_t old = pmdp_establish(vma, address, pmdp, pmd_mkinvalid(*pmdp)); 201 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); 202 return old; 203 } 204 #endif 205 206 #ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD 207 pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address, 208 pmd_t *pmdp) 209 { 210 return pmdp_invalidate(vma, address, pmdp); 211 } 212 #endif 213 214 #ifndef pmdp_collapse_flush 215 pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, 216 pmd_t *pmdp) 217 { 218 /* 219 * pmd and hugepage pte format are same. So we could 220 * use the same function. 221 */ 222 pmd_t pmd; 223 224 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 225 VM_BUG_ON(pmd_trans_huge(*pmdp)); 226 pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); 227 228 /* collapse entails shooting down ptes not pmd */ 229 flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); 230 return pmd; 231 } 232 #endif 233 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 234 235 pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) 236 { 237 pmd_t pmdval; 238 239 /* rcu_read_lock() to be added later */ 240 pmdval = pmdp_get_lockless(pmd); 241 if (pmdvalp) 242 *pmdvalp = pmdval; 243 if (unlikely(pmd_none(pmdval) || is_pmd_migration_entry(pmdval))) 244 goto nomap; 245 if (unlikely(pmd_trans_huge(pmdval) || pmd_devmap(pmdval))) 246 goto nomap; 247 if (unlikely(pmd_bad(pmdval))) { 248 pmd_clear_bad(pmd); 249 goto nomap; 250 } 251 return __pte_map(&pmdval, addr); 252 nomap: 253 /* rcu_read_unlock() to be added later */ 254 return NULL; 255 } 256 257 pte_t *pte_offset_map_nolock(struct mm_struct *mm, pmd_t *pmd, 258 unsigned long addr, spinlock_t **ptlp) 259 { 260 pmd_t pmdval; 261 pte_t *pte; 262 263 pte = __pte_offset_map(pmd, addr, &pmdval); 264 if (likely(pte)) 265 *ptlp = pte_lockptr(mm, &pmdval); 266 return pte; 267 } 268 269 pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, 270 unsigned long addr, spinlock_t **ptlp) 271 { 272 spinlock_t *ptl; 273 pmd_t pmdval; 274 pte_t *pte; 275 again: 276 pte = __pte_offset_map(pmd, addr, &pmdval); 277 if (unlikely(!pte)) 278 return pte; 279 ptl = pte_lockptr(mm, &pmdval); 280 spin_lock(ptl); 281 if (likely(pmd_same(pmdval, pmdp_get_lockless(pmd)))) { 282 *ptlp = ptl; 283 return pte; 284 } 285 pte_unmap_unlock(pte, ptl); 286 goto again; 287 } 288