1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_PGTABLE_H 3 #define _LINUX_PGTABLE_H 4 5 #include <linux/pfn.h> 6 #include <asm/pgtable.h> 7 8 #ifndef __ASSEMBLY__ 9 #ifdef CONFIG_MMU 10 11 #include <linux/mm_types.h> 12 #include <linux/bug.h> 13 #include <linux/errno.h> 14 #include <asm-generic/pgtable_uffd.h> 15 16 #if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ 17 defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS 18 #error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED 19 #endif 20 21 /* 22 * On almost all architectures and configurations, 0 can be used as the 23 * upper ceiling to free_pgtables(): on many architectures it has the same 24 * effect as using TASK_SIZE. However, there is one configuration which 25 * must impose a more careful limit, to avoid freeing kernel pgtables. 26 */ 27 #ifndef USER_PGTABLES_CEILING 28 #define USER_PGTABLES_CEILING 0UL 29 #endif 30 31 /* 32 * A page table page can be thought of an array like this: pXd_t[PTRS_PER_PxD] 33 * 34 * The pXx_index() functions return the index of the entry in the page 35 * table page which would control the given virtual address 36 * 37 * As these functions may be used by the same code for different levels of 38 * the page table folding, they are always available, regardless of 39 * CONFIG_PGTABLE_LEVELS value. For the folded levels they simply return 0 40 * because in such cases PTRS_PER_PxD equals 1. 41 */ 42 43 static inline unsigned long pte_index(unsigned long address) 44 { 45 return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); 46 } 47 48 #ifndef pmd_index 49 static inline unsigned long pmd_index(unsigned long address) 50 { 51 return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1); 52 } 53 #define pmd_index pmd_index 54 #endif 55 56 #ifndef pud_index 57 static inline unsigned long pud_index(unsigned long address) 58 { 59 return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1); 60 } 61 #define pud_index pud_index 62 #endif 63 64 #ifndef pgd_index 65 /* Must be a compile-time constant, so implement it as a macro */ 66 #define pgd_index(a) (((a) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) 67 #endif 68 69 #ifndef pte_offset_kernel 70 static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address) 71 { 72 return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address); 73 } 74 #define pte_offset_kernel pte_offset_kernel 75 #endif 76 77 #if defined(CONFIG_HIGHPTE) 78 #define pte_offset_map(dir, address) \ 79 ((pte_t *)kmap_atomic(pmd_page(*(dir))) + \ 80 pte_index((address))) 81 #define pte_unmap(pte) kunmap_atomic((pte)) 82 #else 83 #define pte_offset_map(dir, address) pte_offset_kernel((dir), (address)) 84 #define pte_unmap(pte) ((void)(pte)) /* NOP */ 85 #endif 86 87 /* Find an entry in the second-level page table.. */ 88 #ifndef pmd_offset 89 static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) 90 { 91 return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address); 92 } 93 #define pmd_offset pmd_offset 94 #endif 95 96 #ifndef pud_offset 97 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) 98 { 99 return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address); 100 } 101 #define pud_offset pud_offset 102 #endif 103 104 static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address) 105 { 106 return (pgd + pgd_index(address)); 107 }; 108 109 /* 110 * a shortcut to get a pgd_t in a given mm 111 */ 112 #ifndef pgd_offset 113 #define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address)) 114 #endif 115 116 /* 117 * a shortcut which implies the use of the kernel's pgd, instead 118 * of a process's 119 */ 120 #define pgd_offset_k(address) pgd_offset(&init_mm, (address)) 121 122 /* 123 * In many cases it is known that a virtual address is mapped at PMD or PTE 124 * level, so instead of traversing all the page table levels, we can get a 125 * pointer to the PMD entry in user or kernel page table or translate a virtual 126 * address to the pointer in the PTE in the kernel page tables with simple 127 * helpers. 128 */ 129 static inline pmd_t *pmd_off(struct mm_struct *mm, unsigned long va) 130 { 131 return pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, va), va), va), va); 132 } 133 134 static inline pmd_t *pmd_off_k(unsigned long va) 135 { 136 return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va); 137 } 138 139 static inline pte_t *virt_to_kpte(unsigned long vaddr) 140 { 141 pmd_t *pmd = pmd_off_k(vaddr); 142 143 return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr); 144 } 145 146 #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS 147 extern int ptep_set_access_flags(struct vm_area_struct *vma, 148 unsigned long address, pte_t *ptep, 149 pte_t entry, int dirty); 150 #endif 151 152 #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS 153 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 154 extern int pmdp_set_access_flags(struct vm_area_struct *vma, 155 unsigned long address, pmd_t *pmdp, 156 pmd_t entry, int dirty); 157 extern int pudp_set_access_flags(struct vm_area_struct *vma, 158 unsigned long address, pud_t *pudp, 159 pud_t entry, int dirty); 160 #else 161 static inline int pmdp_set_access_flags(struct vm_area_struct *vma, 162 unsigned long address, pmd_t *pmdp, 163 pmd_t entry, int dirty) 164 { 165 BUILD_BUG(); 166 return 0; 167 } 168 static inline int pudp_set_access_flags(struct vm_area_struct *vma, 169 unsigned long address, pud_t *pudp, 170 pud_t entry, int dirty) 171 { 172 BUILD_BUG(); 173 return 0; 174 } 175 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 176 #endif 177 178 #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG 179 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, 180 unsigned long address, 181 pte_t *ptep) 182 { 183 pte_t pte = *ptep; 184 int r = 1; 185 if (!pte_young(pte)) 186 r = 0; 187 else 188 set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte)); 189 return r; 190 } 191 #endif 192 193 #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG 194 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 195 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, 196 unsigned long address, 197 pmd_t *pmdp) 198 { 199 pmd_t pmd = *pmdp; 200 int r = 1; 201 if (!pmd_young(pmd)) 202 r = 0; 203 else 204 set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd)); 205 return r; 206 } 207 #else 208 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, 209 unsigned long address, 210 pmd_t *pmdp) 211 { 212 BUILD_BUG(); 213 return 0; 214 } 215 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 216 #endif 217 218 #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH 219 int ptep_clear_flush_young(struct vm_area_struct *vma, 220 unsigned long address, pte_t *ptep); 221 #endif 222 223 #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH 224 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 225 extern int pmdp_clear_flush_young(struct vm_area_struct *vma, 226 unsigned long address, pmd_t *pmdp); 227 #else 228 /* 229 * Despite relevant to THP only, this API is called from generic rmap code 230 * under PageTransHuge(), hence needs a dummy implementation for !THP 231 */ 232 static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, 233 unsigned long address, pmd_t *pmdp) 234 { 235 BUILD_BUG(); 236 return 0; 237 } 238 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 239 #endif 240 241 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR 242 static inline pte_t ptep_get_and_clear(struct mm_struct *mm, 243 unsigned long address, 244 pte_t *ptep) 245 { 246 pte_t pte = *ptep; 247 pte_clear(mm, address, ptep); 248 return pte; 249 } 250 #endif 251 252 #ifndef __HAVE_ARCH_PTEP_GET 253 static inline pte_t ptep_get(pte_t *ptep) 254 { 255 return READ_ONCE(*ptep); 256 } 257 #endif 258 259 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 260 #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR 261 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, 262 unsigned long address, 263 pmd_t *pmdp) 264 { 265 pmd_t pmd = *pmdp; 266 pmd_clear(pmdp); 267 return pmd; 268 } 269 #endif /* __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR */ 270 #ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR 271 static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, 272 unsigned long address, 273 pud_t *pudp) 274 { 275 pud_t pud = *pudp; 276 277 pud_clear(pudp); 278 return pud; 279 } 280 #endif /* __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR */ 281 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 282 283 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 284 #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL 285 static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma, 286 unsigned long address, pmd_t *pmdp, 287 int full) 288 { 289 return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); 290 } 291 #endif 292 293 #ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULL 294 static inline pud_t pudp_huge_get_and_clear_full(struct mm_struct *mm, 295 unsigned long address, pud_t *pudp, 296 int full) 297 { 298 return pudp_huge_get_and_clear(mm, address, pudp); 299 } 300 #endif 301 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 302 303 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL 304 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, 305 unsigned long address, pte_t *ptep, 306 int full) 307 { 308 pte_t pte; 309 pte = ptep_get_and_clear(mm, address, ptep); 310 return pte; 311 } 312 #endif 313 314 315 /* 316 * If two threads concurrently fault at the same page, the thread that 317 * won the race updates the PTE and its local TLB/Cache. The other thread 318 * gives up, simply does nothing, and continues; on architectures where 319 * software can update TLB, local TLB can be updated here to avoid next page 320 * fault. This function updates TLB only, do nothing with cache or others. 321 * It is the difference with function update_mmu_cache. 322 */ 323 #ifndef __HAVE_ARCH_UPDATE_MMU_TLB 324 static inline void update_mmu_tlb(struct vm_area_struct *vma, 325 unsigned long address, pte_t *ptep) 326 { 327 } 328 #define __HAVE_ARCH_UPDATE_MMU_TLB 329 #endif 330 331 /* 332 * Some architectures may be able to avoid expensive synchronization 333 * primitives when modifications are made to PTE's which are already 334 * not present, or in the process of an address space destruction. 335 */ 336 #ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL 337 static inline void pte_clear_not_present_full(struct mm_struct *mm, 338 unsigned long address, 339 pte_t *ptep, 340 int full) 341 { 342 pte_clear(mm, address, ptep); 343 } 344 #endif 345 346 #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH 347 extern pte_t ptep_clear_flush(struct vm_area_struct *vma, 348 unsigned long address, 349 pte_t *ptep); 350 #endif 351 352 #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH 353 extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, 354 unsigned long address, 355 pmd_t *pmdp); 356 extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, 357 unsigned long address, 358 pud_t *pudp); 359 #endif 360 361 #ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT 362 struct mm_struct; 363 static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) 364 { 365 pte_t old_pte = *ptep; 366 set_pte_at(mm, address, ptep, pte_wrprotect(old_pte)); 367 } 368 #endif 369 370 /* 371 * On some architectures hardware does not set page access bit when accessing 372 * memory page, it is responsibilty of software setting this bit. It brings 373 * out extra page fault penalty to track page access bit. For optimization page 374 * access bit can be set during all page fault flow on these arches. 375 * To be differentiate with macro pte_mkyoung, this macro is used on platforms 376 * where software maintains page access bit. 377 */ 378 #ifndef pte_sw_mkyoung 379 static inline pte_t pte_sw_mkyoung(pte_t pte) 380 { 381 return pte; 382 } 383 #define pte_sw_mkyoung pte_sw_mkyoung 384 #endif 385 386 #ifndef pte_savedwrite 387 #define pte_savedwrite pte_write 388 #endif 389 390 #ifndef pte_mk_savedwrite 391 #define pte_mk_savedwrite pte_mkwrite 392 #endif 393 394 #ifndef pte_clear_savedwrite 395 #define pte_clear_savedwrite pte_wrprotect 396 #endif 397 398 #ifndef pmd_savedwrite 399 #define pmd_savedwrite pmd_write 400 #endif 401 402 #ifndef pmd_mk_savedwrite 403 #define pmd_mk_savedwrite pmd_mkwrite 404 #endif 405 406 #ifndef pmd_clear_savedwrite 407 #define pmd_clear_savedwrite pmd_wrprotect 408 #endif 409 410 #ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT 411 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 412 static inline void pmdp_set_wrprotect(struct mm_struct *mm, 413 unsigned long address, pmd_t *pmdp) 414 { 415 pmd_t old_pmd = *pmdp; 416 set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd)); 417 } 418 #else 419 static inline void pmdp_set_wrprotect(struct mm_struct *mm, 420 unsigned long address, pmd_t *pmdp) 421 { 422 BUILD_BUG(); 423 } 424 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 425 #endif 426 #ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT 427 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 428 static inline void pudp_set_wrprotect(struct mm_struct *mm, 429 unsigned long address, pud_t *pudp) 430 { 431 pud_t old_pud = *pudp; 432 433 set_pud_at(mm, address, pudp, pud_wrprotect(old_pud)); 434 } 435 #else 436 static inline void pudp_set_wrprotect(struct mm_struct *mm, 437 unsigned long address, pud_t *pudp) 438 { 439 BUILD_BUG(); 440 } 441 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 442 #endif 443 444 #ifndef pmdp_collapse_flush 445 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 446 extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, 447 unsigned long address, pmd_t *pmdp); 448 #else 449 static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, 450 unsigned long address, 451 pmd_t *pmdp) 452 { 453 BUILD_BUG(); 454 return *pmdp; 455 } 456 #define pmdp_collapse_flush pmdp_collapse_flush 457 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 458 #endif 459 460 #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT 461 extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 462 pgtable_t pgtable); 463 #endif 464 465 #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW 466 extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); 467 #endif 468 469 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 470 /* 471 * This is an implementation of pmdp_establish() that is only suitable for an 472 * architecture that doesn't have hardware dirty/accessed bits. In this case we 473 * can't race with CPU which sets these bits and non-atomic aproach is fine. 474 */ 475 static inline pmd_t generic_pmdp_establish(struct vm_area_struct *vma, 476 unsigned long address, pmd_t *pmdp, pmd_t pmd) 477 { 478 pmd_t old_pmd = *pmdp; 479 set_pmd_at(vma->vm_mm, address, pmdp, pmd); 480 return old_pmd; 481 } 482 #endif 483 484 #ifndef __HAVE_ARCH_PMDP_INVALIDATE 485 extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, 486 pmd_t *pmdp); 487 #endif 488 489 #ifndef __HAVE_ARCH_PTE_SAME 490 static inline int pte_same(pte_t pte_a, pte_t pte_b) 491 { 492 return pte_val(pte_a) == pte_val(pte_b); 493 } 494 #endif 495 496 #ifndef __HAVE_ARCH_PTE_UNUSED 497 /* 498 * Some architectures provide facilities to virtualization guests 499 * so that they can flag allocated pages as unused. This allows the 500 * host to transparently reclaim unused pages. This function returns 501 * whether the pte's page is unused. 502 */ 503 static inline int pte_unused(pte_t pte) 504 { 505 return 0; 506 } 507 #endif 508 509 #ifndef pte_access_permitted 510 #define pte_access_permitted(pte, write) \ 511 (pte_present(pte) && (!(write) || pte_write(pte))) 512 #endif 513 514 #ifndef pmd_access_permitted 515 #define pmd_access_permitted(pmd, write) \ 516 (pmd_present(pmd) && (!(write) || pmd_write(pmd))) 517 #endif 518 519 #ifndef pud_access_permitted 520 #define pud_access_permitted(pud, write) \ 521 (pud_present(pud) && (!(write) || pud_write(pud))) 522 #endif 523 524 #ifndef p4d_access_permitted 525 #define p4d_access_permitted(p4d, write) \ 526 (p4d_present(p4d) && (!(write) || p4d_write(p4d))) 527 #endif 528 529 #ifndef pgd_access_permitted 530 #define pgd_access_permitted(pgd, write) \ 531 (pgd_present(pgd) && (!(write) || pgd_write(pgd))) 532 #endif 533 534 #ifndef __HAVE_ARCH_PMD_SAME 535 static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) 536 { 537 return pmd_val(pmd_a) == pmd_val(pmd_b); 538 } 539 540 static inline int pud_same(pud_t pud_a, pud_t pud_b) 541 { 542 return pud_val(pud_a) == pud_val(pud_b); 543 } 544 #endif 545 546 #ifndef __HAVE_ARCH_P4D_SAME 547 static inline int p4d_same(p4d_t p4d_a, p4d_t p4d_b) 548 { 549 return p4d_val(p4d_a) == p4d_val(p4d_b); 550 } 551 #endif 552 553 #ifndef __HAVE_ARCH_PGD_SAME 554 static inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b) 555 { 556 return pgd_val(pgd_a) == pgd_val(pgd_b); 557 } 558 #endif 559 560 /* 561 * Use set_p*_safe(), and elide TLB flushing, when confident that *no* 562 * TLB flush will be required as a result of the "set". For example, use 563 * in scenarios where it is known ahead of time that the routine is 564 * setting non-present entries, or re-setting an existing entry to the 565 * same value. Otherwise, use the typical "set" helpers and flush the 566 * TLB. 567 */ 568 #define set_pte_safe(ptep, pte) \ 569 ({ \ 570 WARN_ON_ONCE(pte_present(*ptep) && !pte_same(*ptep, pte)); \ 571 set_pte(ptep, pte); \ 572 }) 573 574 #define set_pmd_safe(pmdp, pmd) \ 575 ({ \ 576 WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \ 577 set_pmd(pmdp, pmd); \ 578 }) 579 580 #define set_pud_safe(pudp, pud) \ 581 ({ \ 582 WARN_ON_ONCE(pud_present(*pudp) && !pud_same(*pudp, pud)); \ 583 set_pud(pudp, pud); \ 584 }) 585 586 #define set_p4d_safe(p4dp, p4d) \ 587 ({ \ 588 WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \ 589 set_p4d(p4dp, p4d); \ 590 }) 591 592 #define set_pgd_safe(pgdp, pgd) \ 593 ({ \ 594 WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \ 595 set_pgd(pgdp, pgd); \ 596 }) 597 598 #ifndef __HAVE_ARCH_DO_SWAP_PAGE 599 /* 600 * Some architectures support metadata associated with a page. When a 601 * page is being swapped out, this metadata must be saved so it can be 602 * restored when the page is swapped back in. SPARC M7 and newer 603 * processors support an ADI (Application Data Integrity) tag for the 604 * page as metadata for the page. arch_do_swap_page() can restore this 605 * metadata when a page is swapped back in. 606 */ 607 static inline void arch_do_swap_page(struct mm_struct *mm, 608 struct vm_area_struct *vma, 609 unsigned long addr, 610 pte_t pte, pte_t oldpte) 611 { 612 613 } 614 #endif 615 616 #ifndef __HAVE_ARCH_UNMAP_ONE 617 /* 618 * Some architectures support metadata associated with a page. When a 619 * page is being swapped out, this metadata must be saved so it can be 620 * restored when the page is swapped back in. SPARC M7 and newer 621 * processors support an ADI (Application Data Integrity) tag for the 622 * page as metadata for the page. arch_unmap_one() can save this 623 * metadata on a swap-out of a page. 624 */ 625 static inline int arch_unmap_one(struct mm_struct *mm, 626 struct vm_area_struct *vma, 627 unsigned long addr, 628 pte_t orig_pte) 629 { 630 return 0; 631 } 632 #endif 633 634 #ifndef __HAVE_ARCH_PGD_OFFSET_GATE 635 #define pgd_offset_gate(mm, addr) pgd_offset(mm, addr) 636 #endif 637 638 #ifndef __HAVE_ARCH_MOVE_PTE 639 #define move_pte(pte, prot, old_addr, new_addr) (pte) 640 #endif 641 642 #ifndef pte_accessible 643 # define pte_accessible(mm, pte) ((void)(pte), 1) 644 #endif 645 646 #ifndef flush_tlb_fix_spurious_fault 647 #define flush_tlb_fix_spurious_fault(vma, address) flush_tlb_page(vma, address) 648 #endif 649 650 /* 651 * When walking page tables, get the address of the next boundary, 652 * or the end address of the range if that comes earlier. Although no 653 * vma end wraps to 0, rounded up __boundary may wrap to 0 throughout. 654 */ 655 656 #define pgd_addr_end(addr, end) \ 657 ({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \ 658 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 659 }) 660 661 #ifndef p4d_addr_end 662 #define p4d_addr_end(addr, end) \ 663 ({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \ 664 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 665 }) 666 #endif 667 668 #ifndef pud_addr_end 669 #define pud_addr_end(addr, end) \ 670 ({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \ 671 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 672 }) 673 #endif 674 675 #ifndef pmd_addr_end 676 #define pmd_addr_end(addr, end) \ 677 ({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \ 678 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 679 }) 680 #endif 681 682 /* 683 * When walking page tables, we usually want to skip any p?d_none entries; 684 * and any p?d_bad entries - reporting the error before resetting to none. 685 * Do the tests inline, but report and clear the bad entry in mm/memory.c. 686 */ 687 void pgd_clear_bad(pgd_t *); 688 689 #ifndef __PAGETABLE_P4D_FOLDED 690 void p4d_clear_bad(p4d_t *); 691 #else 692 #define p4d_clear_bad(p4d) do { } while (0) 693 #endif 694 695 #ifndef __PAGETABLE_PUD_FOLDED 696 void pud_clear_bad(pud_t *); 697 #else 698 #define pud_clear_bad(p4d) do { } while (0) 699 #endif 700 701 void pmd_clear_bad(pmd_t *); 702 703 static inline int pgd_none_or_clear_bad(pgd_t *pgd) 704 { 705 if (pgd_none(*pgd)) 706 return 1; 707 if (unlikely(pgd_bad(*pgd))) { 708 pgd_clear_bad(pgd); 709 return 1; 710 } 711 return 0; 712 } 713 714 static inline int p4d_none_or_clear_bad(p4d_t *p4d) 715 { 716 if (p4d_none(*p4d)) 717 return 1; 718 if (unlikely(p4d_bad(*p4d))) { 719 p4d_clear_bad(p4d); 720 return 1; 721 } 722 return 0; 723 } 724 725 static inline int pud_none_or_clear_bad(pud_t *pud) 726 { 727 if (pud_none(*pud)) 728 return 1; 729 if (unlikely(pud_bad(*pud))) { 730 pud_clear_bad(pud); 731 return 1; 732 } 733 return 0; 734 } 735 736 static inline int pmd_none_or_clear_bad(pmd_t *pmd) 737 { 738 if (pmd_none(*pmd)) 739 return 1; 740 if (unlikely(pmd_bad(*pmd))) { 741 pmd_clear_bad(pmd); 742 return 1; 743 } 744 return 0; 745 } 746 747 static inline pte_t __ptep_modify_prot_start(struct vm_area_struct *vma, 748 unsigned long addr, 749 pte_t *ptep) 750 { 751 /* 752 * Get the current pte state, but zero it out to make it 753 * non-present, preventing the hardware from asynchronously 754 * updating it. 755 */ 756 return ptep_get_and_clear(vma->vm_mm, addr, ptep); 757 } 758 759 static inline void __ptep_modify_prot_commit(struct vm_area_struct *vma, 760 unsigned long addr, 761 pte_t *ptep, pte_t pte) 762 { 763 /* 764 * The pte is non-present, so there's no hardware state to 765 * preserve. 766 */ 767 set_pte_at(vma->vm_mm, addr, ptep, pte); 768 } 769 770 #ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION 771 /* 772 * Start a pte protection read-modify-write transaction, which 773 * protects against asynchronous hardware modifications to the pte. 774 * The intention is not to prevent the hardware from making pte 775 * updates, but to prevent any updates it may make from being lost. 776 * 777 * This does not protect against other software modifications of the 778 * pte; the appropriate pte lock must be held over the transation. 779 * 780 * Note that this interface is intended to be batchable, meaning that 781 * ptep_modify_prot_commit may not actually update the pte, but merely 782 * queue the update to be done at some later time. The update must be 783 * actually committed before the pte lock is released, however. 784 */ 785 static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma, 786 unsigned long addr, 787 pte_t *ptep) 788 { 789 return __ptep_modify_prot_start(vma, addr, ptep); 790 } 791 792 /* 793 * Commit an update to a pte, leaving any hardware-controlled bits in 794 * the PTE unmodified. 795 */ 796 static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, 797 unsigned long addr, 798 pte_t *ptep, pte_t old_pte, pte_t pte) 799 { 800 __ptep_modify_prot_commit(vma, addr, ptep, pte); 801 } 802 #endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */ 803 #endif /* CONFIG_MMU */ 804 805 /* 806 * No-op macros that just return the current protection value. Defined here 807 * because these macros can be used even if CONFIG_MMU is not defined. 808 */ 809 810 #ifndef pgprot_nx 811 #define pgprot_nx(prot) (prot) 812 #endif 813 814 #ifndef pgprot_noncached 815 #define pgprot_noncached(prot) (prot) 816 #endif 817 818 #ifndef pgprot_writecombine 819 #define pgprot_writecombine pgprot_noncached 820 #endif 821 822 #ifndef pgprot_writethrough 823 #define pgprot_writethrough pgprot_noncached 824 #endif 825 826 #ifndef pgprot_device 827 #define pgprot_device pgprot_noncached 828 #endif 829 830 #ifdef CONFIG_MMU 831 #ifndef pgprot_modify 832 #define pgprot_modify pgprot_modify 833 static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) 834 { 835 if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot))) 836 newprot = pgprot_noncached(newprot); 837 if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot))) 838 newprot = pgprot_writecombine(newprot); 839 if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot))) 840 newprot = pgprot_device(newprot); 841 return newprot; 842 } 843 #endif 844 #endif /* CONFIG_MMU */ 845 846 #ifndef pgprot_encrypted 847 #define pgprot_encrypted(prot) (prot) 848 #endif 849 850 #ifndef pgprot_decrypted 851 #define pgprot_decrypted(prot) (prot) 852 #endif 853 854 /* 855 * A facility to provide lazy MMU batching. This allows PTE updates and 856 * page invalidations to be delayed until a call to leave lazy MMU mode 857 * is issued. Some architectures may benefit from doing this, and it is 858 * beneficial for both shadow and direct mode hypervisors, which may batch 859 * the PTE updates which happen during this window. Note that using this 860 * interface requires that read hazards be removed from the code. A read 861 * hazard could result in the direct mode hypervisor case, since the actual 862 * write to the page tables may not yet have taken place, so reads though 863 * a raw PTE pointer after it has been modified are not guaranteed to be 864 * up to date. This mode can only be entered and left under the protection of 865 * the page table locks for all page tables which may be modified. In the UP 866 * case, this is required so that preemption is disabled, and in the SMP case, 867 * it must synchronize the delayed page table writes properly on other CPUs. 868 */ 869 #ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE 870 #define arch_enter_lazy_mmu_mode() do {} while (0) 871 #define arch_leave_lazy_mmu_mode() do {} while (0) 872 #define arch_flush_lazy_mmu_mode() do {} while (0) 873 #endif 874 875 /* 876 * A facility to provide batching of the reload of page tables and 877 * other process state with the actual context switch code for 878 * paravirtualized guests. By convention, only one of the batched 879 * update (lazy) modes (CPU, MMU) should be active at any given time, 880 * entry should never be nested, and entry and exits should always be 881 * paired. This is for sanity of maintaining and reasoning about the 882 * kernel code. In this case, the exit (end of the context switch) is 883 * in architecture-specific code, and so doesn't need a generic 884 * definition. 885 */ 886 #ifndef __HAVE_ARCH_START_CONTEXT_SWITCH 887 #define arch_start_context_switch(prev) do {} while (0) 888 #endif 889 890 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY 891 #ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION 892 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) 893 { 894 return pmd; 895 } 896 897 static inline int pmd_swp_soft_dirty(pmd_t pmd) 898 { 899 return 0; 900 } 901 902 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) 903 { 904 return pmd; 905 } 906 #endif 907 #else /* !CONFIG_HAVE_ARCH_SOFT_DIRTY */ 908 static inline int pte_soft_dirty(pte_t pte) 909 { 910 return 0; 911 } 912 913 static inline int pmd_soft_dirty(pmd_t pmd) 914 { 915 return 0; 916 } 917 918 static inline pte_t pte_mksoft_dirty(pte_t pte) 919 { 920 return pte; 921 } 922 923 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd) 924 { 925 return pmd; 926 } 927 928 static inline pte_t pte_clear_soft_dirty(pte_t pte) 929 { 930 return pte; 931 } 932 933 static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) 934 { 935 return pmd; 936 } 937 938 static inline pte_t pte_swp_mksoft_dirty(pte_t pte) 939 { 940 return pte; 941 } 942 943 static inline int pte_swp_soft_dirty(pte_t pte) 944 { 945 return 0; 946 } 947 948 static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) 949 { 950 return pte; 951 } 952 953 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) 954 { 955 return pmd; 956 } 957 958 static inline int pmd_swp_soft_dirty(pmd_t pmd) 959 { 960 return 0; 961 } 962 963 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) 964 { 965 return pmd; 966 } 967 #endif 968 969 #ifndef __HAVE_PFNMAP_TRACKING 970 /* 971 * Interfaces that can be used by architecture code to keep track of 972 * memory type of pfn mappings specified by the remap_pfn_range, 973 * vmf_insert_pfn. 974 */ 975 976 /* 977 * track_pfn_remap is called when a _new_ pfn mapping is being established 978 * by remap_pfn_range() for physical range indicated by pfn and size. 979 */ 980 static inline int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot, 981 unsigned long pfn, unsigned long addr, 982 unsigned long size) 983 { 984 return 0; 985 } 986 987 /* 988 * track_pfn_insert is called when a _new_ single pfn is established 989 * by vmf_insert_pfn(). 990 */ 991 static inline void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, 992 pfn_t pfn) 993 { 994 } 995 996 /* 997 * track_pfn_copy is called when vma that is covering the pfnmap gets 998 * copied through copy_page_range(). 999 */ 1000 static inline int track_pfn_copy(struct vm_area_struct *vma) 1001 { 1002 return 0; 1003 } 1004 1005 /* 1006 * untrack_pfn is called while unmapping a pfnmap for a region. 1007 * untrack can be called for a specific region indicated by pfn and size or 1008 * can be for the entire vma (in which case pfn, size are zero). 1009 */ 1010 static inline void untrack_pfn(struct vm_area_struct *vma, 1011 unsigned long pfn, unsigned long size) 1012 { 1013 } 1014 1015 /* 1016 * untrack_pfn_moved is called while mremapping a pfnmap for a new region. 1017 */ 1018 static inline void untrack_pfn_moved(struct vm_area_struct *vma) 1019 { 1020 } 1021 #else 1022 extern int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot, 1023 unsigned long pfn, unsigned long addr, 1024 unsigned long size); 1025 extern void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, 1026 pfn_t pfn); 1027 extern int track_pfn_copy(struct vm_area_struct *vma); 1028 extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn, 1029 unsigned long size); 1030 extern void untrack_pfn_moved(struct vm_area_struct *vma); 1031 #endif 1032 1033 #ifdef __HAVE_COLOR_ZERO_PAGE 1034 static inline int is_zero_pfn(unsigned long pfn) 1035 { 1036 extern unsigned long zero_pfn; 1037 unsigned long offset_from_zero_pfn = pfn - zero_pfn; 1038 return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT); 1039 } 1040 1041 #define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr)) 1042 1043 #else 1044 static inline int is_zero_pfn(unsigned long pfn) 1045 { 1046 extern unsigned long zero_pfn; 1047 return pfn == zero_pfn; 1048 } 1049 1050 static inline unsigned long my_zero_pfn(unsigned long addr) 1051 { 1052 extern unsigned long zero_pfn; 1053 return zero_pfn; 1054 } 1055 #endif 1056 1057 #ifdef CONFIG_MMU 1058 1059 #ifndef CONFIG_TRANSPARENT_HUGEPAGE 1060 static inline int pmd_trans_huge(pmd_t pmd) 1061 { 1062 return 0; 1063 } 1064 #ifndef pmd_write 1065 static inline int pmd_write(pmd_t pmd) 1066 { 1067 BUG(); 1068 return 0; 1069 } 1070 #endif /* pmd_write */ 1071 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1072 1073 #ifndef pud_write 1074 static inline int pud_write(pud_t pud) 1075 { 1076 BUG(); 1077 return 0; 1078 } 1079 #endif /* pud_write */ 1080 1081 #if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE) 1082 static inline int pmd_devmap(pmd_t pmd) 1083 { 1084 return 0; 1085 } 1086 static inline int pud_devmap(pud_t pud) 1087 { 1088 return 0; 1089 } 1090 static inline int pgd_devmap(pgd_t pgd) 1091 { 1092 return 0; 1093 } 1094 #endif 1095 1096 #if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \ 1097 (defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ 1098 !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)) 1099 static inline int pud_trans_huge(pud_t pud) 1100 { 1101 return 0; 1102 } 1103 #endif 1104 1105 /* See pmd_none_or_trans_huge_or_clear_bad for discussion. */ 1106 static inline int pud_none_or_trans_huge_or_dev_or_clear_bad(pud_t *pud) 1107 { 1108 pud_t pudval = READ_ONCE(*pud); 1109 1110 if (pud_none(pudval) || pud_trans_huge(pudval) || pud_devmap(pudval)) 1111 return 1; 1112 if (unlikely(pud_bad(pudval))) { 1113 pud_clear_bad(pud); 1114 return 1; 1115 } 1116 return 0; 1117 } 1118 1119 /* See pmd_trans_unstable for discussion. */ 1120 static inline int pud_trans_unstable(pud_t *pud) 1121 { 1122 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ 1123 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) 1124 return pud_none_or_trans_huge_or_dev_or_clear_bad(pud); 1125 #else 1126 return 0; 1127 #endif 1128 } 1129 1130 #ifndef pmd_read_atomic 1131 static inline pmd_t pmd_read_atomic(pmd_t *pmdp) 1132 { 1133 /* 1134 * Depend on compiler for an atomic pmd read. NOTE: this is 1135 * only going to work, if the pmdval_t isn't larger than 1136 * an unsigned long. 1137 */ 1138 return *pmdp; 1139 } 1140 #endif 1141 1142 #ifndef arch_needs_pgtable_deposit 1143 #define arch_needs_pgtable_deposit() (false) 1144 #endif 1145 /* 1146 * This function is meant to be used by sites walking pagetables with 1147 * the mmap_lock held in read mode to protect against MADV_DONTNEED and 1148 * transhuge page faults. MADV_DONTNEED can convert a transhuge pmd 1149 * into a null pmd and the transhuge page fault can convert a null pmd 1150 * into an hugepmd or into a regular pmd (if the hugepage allocation 1151 * fails). While holding the mmap_lock in read mode the pmd becomes 1152 * stable and stops changing under us only if it's not null and not a 1153 * transhuge pmd. When those races occurs and this function makes a 1154 * difference vs the standard pmd_none_or_clear_bad, the result is 1155 * undefined so behaving like if the pmd was none is safe (because it 1156 * can return none anyway). The compiler level barrier() is critically 1157 * important to compute the two checks atomically on the same pmdval. 1158 * 1159 * For 32bit kernels with a 64bit large pmd_t this automatically takes 1160 * care of reading the pmd atomically to avoid SMP race conditions 1161 * against pmd_populate() when the mmap_lock is hold for reading by the 1162 * caller (a special atomic read not done by "gcc" as in the generic 1163 * version above, is also needed when THP is disabled because the page 1164 * fault can populate the pmd from under us). 1165 */ 1166 static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd) 1167 { 1168 pmd_t pmdval = pmd_read_atomic(pmd); 1169 /* 1170 * The barrier will stabilize the pmdval in a register or on 1171 * the stack so that it will stop changing under the code. 1172 * 1173 * When CONFIG_TRANSPARENT_HUGEPAGE=y on x86 32bit PAE, 1174 * pmd_read_atomic is allowed to return a not atomic pmdval 1175 * (for example pointing to an hugepage that has never been 1176 * mapped in the pmd). The below checks will only care about 1177 * the low part of the pmd with 32bit PAE x86 anyway, with the 1178 * exception of pmd_none(). So the important thing is that if 1179 * the low part of the pmd is found null, the high part will 1180 * be also null or the pmd_none() check below would be 1181 * confused. 1182 */ 1183 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1184 barrier(); 1185 #endif 1186 /* 1187 * !pmd_present() checks for pmd migration entries 1188 * 1189 * The complete check uses is_pmd_migration_entry() in linux/swapops.h 1190 * But using that requires moving current function and pmd_trans_unstable() 1191 * to linux/swapops.h to resovle dependency, which is too much code move. 1192 * 1193 * !pmd_present() is equivalent to is_pmd_migration_entry() currently, 1194 * because !pmd_present() pages can only be under migration not swapped 1195 * out. 1196 * 1197 * pmd_none() is preseved for future condition checks on pmd migration 1198 * entries and not confusing with this function name, although it is 1199 * redundant with !pmd_present(). 1200 */ 1201 if (pmd_none(pmdval) || pmd_trans_huge(pmdval) || 1202 (IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION) && !pmd_present(pmdval))) 1203 return 1; 1204 if (unlikely(pmd_bad(pmdval))) { 1205 pmd_clear_bad(pmd); 1206 return 1; 1207 } 1208 return 0; 1209 } 1210 1211 /* 1212 * This is a noop if Transparent Hugepage Support is not built into 1213 * the kernel. Otherwise it is equivalent to 1214 * pmd_none_or_trans_huge_or_clear_bad(), and shall only be called in 1215 * places that already verified the pmd is not none and they want to 1216 * walk ptes while holding the mmap sem in read mode (write mode don't 1217 * need this). If THP is not enabled, the pmd can't go away under the 1218 * code even if MADV_DONTNEED runs, but if THP is enabled we need to 1219 * run a pmd_trans_unstable before walking the ptes after 1220 * split_huge_pmd returns (because it may have run when the pmd become 1221 * null, but then a page fault can map in a THP and not a regular page). 1222 */ 1223 static inline int pmd_trans_unstable(pmd_t *pmd) 1224 { 1225 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1226 return pmd_none_or_trans_huge_or_clear_bad(pmd); 1227 #else 1228 return 0; 1229 #endif 1230 } 1231 1232 #ifndef CONFIG_NUMA_BALANCING 1233 /* 1234 * Technically a PTE can be PROTNONE even when not doing NUMA balancing but 1235 * the only case the kernel cares is for NUMA balancing and is only ever set 1236 * when the VMA is accessible. For PROT_NONE VMAs, the PTEs are not marked 1237 * _PAGE_PROTNONE so by default, implement the helper as "always no". It 1238 * is the responsibility of the caller to distinguish between PROT_NONE 1239 * protections and NUMA hinting fault protections. 1240 */ 1241 static inline int pte_protnone(pte_t pte) 1242 { 1243 return 0; 1244 } 1245 1246 static inline int pmd_protnone(pmd_t pmd) 1247 { 1248 return 0; 1249 } 1250 #endif /* CONFIG_NUMA_BALANCING */ 1251 1252 #endif /* CONFIG_MMU */ 1253 1254 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 1255 1256 #ifndef __PAGETABLE_P4D_FOLDED 1257 int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot); 1258 int p4d_clear_huge(p4d_t *p4d); 1259 #else 1260 static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) 1261 { 1262 return 0; 1263 } 1264 static inline int p4d_clear_huge(p4d_t *p4d) 1265 { 1266 return 0; 1267 } 1268 #endif /* !__PAGETABLE_P4D_FOLDED */ 1269 1270 int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot); 1271 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot); 1272 int pud_clear_huge(pud_t *pud); 1273 int pmd_clear_huge(pmd_t *pmd); 1274 int p4d_free_pud_page(p4d_t *p4d, unsigned long addr); 1275 int pud_free_pmd_page(pud_t *pud, unsigned long addr); 1276 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr); 1277 #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ 1278 static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) 1279 { 1280 return 0; 1281 } 1282 static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot) 1283 { 1284 return 0; 1285 } 1286 static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot) 1287 { 1288 return 0; 1289 } 1290 static inline int p4d_clear_huge(p4d_t *p4d) 1291 { 1292 return 0; 1293 } 1294 static inline int pud_clear_huge(pud_t *pud) 1295 { 1296 return 0; 1297 } 1298 static inline int pmd_clear_huge(pmd_t *pmd) 1299 { 1300 return 0; 1301 } 1302 static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) 1303 { 1304 return 0; 1305 } 1306 static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr) 1307 { 1308 return 0; 1309 } 1310 static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) 1311 { 1312 return 0; 1313 } 1314 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 1315 1316 #ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE 1317 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1318 /* 1319 * ARCHes with special requirements for evicting THP backing TLB entries can 1320 * implement this. Otherwise also, it can help optimize normal TLB flush in 1321 * THP regime. Stock flush_tlb_range() typically has optimization to nuke the 1322 * entire TLB if flush span is greater than a threshold, which will 1323 * likely be true for a single huge page. Thus a single THP flush will 1324 * invalidate the entire TLB which is not desirable. 1325 * e.g. see arch/arc: flush_pmd_tlb_range 1326 */ 1327 #define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 1328 #define flush_pud_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 1329 #else 1330 #define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG() 1331 #define flush_pud_tlb_range(vma, addr, end) BUILD_BUG() 1332 #endif 1333 #endif 1334 1335 struct file; 1336 int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, 1337 unsigned long size, pgprot_t *vma_prot); 1338 1339 #ifndef CONFIG_X86_ESPFIX64 1340 static inline void init_espfix_bsp(void) { } 1341 #endif 1342 1343 extern void __init pgtable_cache_init(void); 1344 1345 #ifndef __HAVE_ARCH_PFN_MODIFY_ALLOWED 1346 static inline bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot) 1347 { 1348 return true; 1349 } 1350 1351 static inline bool arch_has_pfn_modify_check(void) 1352 { 1353 return false; 1354 } 1355 #endif /* !_HAVE_ARCH_PFN_MODIFY_ALLOWED */ 1356 1357 /* 1358 * Architecture PAGE_KERNEL_* fallbacks 1359 * 1360 * Some architectures don't define certain PAGE_KERNEL_* flags. This is either 1361 * because they really don't support them, or the port needs to be updated to 1362 * reflect the required functionality. Below are a set of relatively safe 1363 * fallbacks, as best effort, which we can count on in lieu of the architectures 1364 * not defining them on their own yet. 1365 */ 1366 1367 #ifndef PAGE_KERNEL_RO 1368 # define PAGE_KERNEL_RO PAGE_KERNEL 1369 #endif 1370 1371 #ifndef PAGE_KERNEL_EXEC 1372 # define PAGE_KERNEL_EXEC PAGE_KERNEL 1373 #endif 1374 1375 /* 1376 * Page Table Modification bits for pgtbl_mod_mask. 1377 * 1378 * These are used by the p?d_alloc_track*() set of functions an in the generic 1379 * vmalloc/ioremap code to track at which page-table levels entries have been 1380 * modified. Based on that the code can better decide when vmalloc and ioremap 1381 * mapping changes need to be synchronized to other page-tables in the system. 1382 */ 1383 #define __PGTBL_PGD_MODIFIED 0 1384 #define __PGTBL_P4D_MODIFIED 1 1385 #define __PGTBL_PUD_MODIFIED 2 1386 #define __PGTBL_PMD_MODIFIED 3 1387 #define __PGTBL_PTE_MODIFIED 4 1388 1389 #define PGTBL_PGD_MODIFIED BIT(__PGTBL_PGD_MODIFIED) 1390 #define PGTBL_P4D_MODIFIED BIT(__PGTBL_P4D_MODIFIED) 1391 #define PGTBL_PUD_MODIFIED BIT(__PGTBL_PUD_MODIFIED) 1392 #define PGTBL_PMD_MODIFIED BIT(__PGTBL_PMD_MODIFIED) 1393 #define PGTBL_PTE_MODIFIED BIT(__PGTBL_PTE_MODIFIED) 1394 1395 /* Page-Table Modification Mask */ 1396 typedef unsigned int pgtbl_mod_mask; 1397 1398 #endif /* !__ASSEMBLY__ */ 1399 1400 #ifndef io_remap_pfn_range 1401 #define io_remap_pfn_range remap_pfn_range 1402 #endif 1403 1404 #ifndef has_transparent_hugepage 1405 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1406 #define has_transparent_hugepage() 1 1407 #else 1408 #define has_transparent_hugepage() 0 1409 #endif 1410 #endif 1411 1412 /* 1413 * On some architectures it depends on the mm if the p4d/pud or pmd 1414 * layer of the page table hierarchy is folded or not. 1415 */ 1416 #ifndef mm_p4d_folded 1417 #define mm_p4d_folded(mm) __is_defined(__PAGETABLE_P4D_FOLDED) 1418 #endif 1419 1420 #ifndef mm_pud_folded 1421 #define mm_pud_folded(mm) __is_defined(__PAGETABLE_PUD_FOLDED) 1422 #endif 1423 1424 #ifndef mm_pmd_folded 1425 #define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED) 1426 #endif 1427 1428 /* 1429 * p?d_leaf() - true if this entry is a final mapping to a physical address. 1430 * This differs from p?d_huge() by the fact that they are always available (if 1431 * the architecture supports large pages at the appropriate level) even 1432 * if CONFIG_HUGETLB_PAGE is not defined. 1433 * Only meaningful when called on a valid entry. 1434 */ 1435 #ifndef pgd_leaf 1436 #define pgd_leaf(x) 0 1437 #endif 1438 #ifndef p4d_leaf 1439 #define p4d_leaf(x) 0 1440 #endif 1441 #ifndef pud_leaf 1442 #define pud_leaf(x) 0 1443 #endif 1444 #ifndef pmd_leaf 1445 #define pmd_leaf(x) 0 1446 #endif 1447 1448 #endif /* _LINUX_PGTABLE_H */ 1449