1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_SWAPOPS_H 3 #define _LINUX_SWAPOPS_H 4 5 #include <linux/radix-tree.h> 6 #include <linux/bug.h> 7 #include <linux/mm_types.h> 8 9 #ifdef CONFIG_MMU 10 11 #ifdef CONFIG_SWAP 12 #include <linux/swapfile.h> 13 #endif /* CONFIG_SWAP */ 14 15 /* 16 * swapcache pages are stored in the swapper_space radix tree. We want to 17 * get good packing density in that tree, so the index should be dense in 18 * the low-order bits. 19 * 20 * We arrange the `type' and `offset' fields so that `type' is at the six 21 * high-order bits of the swp_entry_t and `offset' is right-aligned in the 22 * remaining bits. Although `type' itself needs only five bits, we allow for 23 * shmem/tmpfs to shift it all up a further one bit: see swp_to_radix_entry(). 24 * 25 * swp_entry_t's are *never* stored anywhere in their arch-dependent format. 26 */ 27 #define SWP_TYPE_SHIFT (BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT) 28 #define SWP_OFFSET_MASK ((1UL << SWP_TYPE_SHIFT) - 1) 29 30 /* 31 * Definitions only for PFN swap entries (see is_pfn_swap_entry()). To 32 * store PFN, we only need SWP_PFN_BITS bits. Each of the pfn swap entries 33 * can use the extra bits to store other information besides PFN. 34 */ 35 #ifdef MAX_PHYSMEM_BITS 36 #define SWP_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT) 37 #else /* MAX_PHYSMEM_BITS */ 38 #define SWP_PFN_BITS min_t(int, \ 39 sizeof(phys_addr_t) * 8 - PAGE_SHIFT, \ 40 SWP_TYPE_SHIFT) 41 #endif /* MAX_PHYSMEM_BITS */ 42 #define SWP_PFN_MASK (BIT(SWP_PFN_BITS) - 1) 43 44 /** 45 * Migration swap entry specific bitfield definitions. Layout: 46 * 47 * |----------+--------------------| 48 * | swp_type | swp_offset | 49 * |----------+--------+-+-+-------| 50 * | | resv |D|A| PFN | 51 * |----------+--------+-+-+-------| 52 * 53 * @SWP_MIG_YOUNG_BIT: Whether the page used to have young bit set (bit A) 54 * @SWP_MIG_DIRTY_BIT: Whether the page used to have dirty bit set (bit D) 55 * 56 * Note: A/D bits will be stored in migration entries iff there're enough 57 * free bits in arch specific swp offset. By default we'll ignore A/D bits 58 * when migrating a page. Please refer to migration_entry_supports_ad() 59 * for more information. If there're more bits besides PFN and A/D bits, 60 * they should be reserved and always be zeros. 61 */ 62 #define SWP_MIG_YOUNG_BIT (SWP_PFN_BITS) 63 #define SWP_MIG_DIRTY_BIT (SWP_PFN_BITS + 1) 64 #define SWP_MIG_TOTAL_BITS (SWP_PFN_BITS + 2) 65 66 #define SWP_MIG_YOUNG BIT(SWP_MIG_YOUNG_BIT) 67 #define SWP_MIG_DIRTY BIT(SWP_MIG_DIRTY_BIT) 68 69 static inline bool is_pfn_swap_entry(swp_entry_t entry); 70 71 /* Clear all flags but only keep swp_entry_t related information */ 72 static inline pte_t pte_swp_clear_flags(pte_t pte) 73 { 74 if (pte_swp_exclusive(pte)) 75 pte = pte_swp_clear_exclusive(pte); 76 if (pte_swp_soft_dirty(pte)) 77 pte = pte_swp_clear_soft_dirty(pte); 78 if (pte_swp_uffd_wp(pte)) 79 pte = pte_swp_clear_uffd_wp(pte); 80 return pte; 81 } 82 83 /* 84 * Store a type+offset into a swp_entry_t in an arch-independent format 85 */ 86 static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset) 87 { 88 swp_entry_t ret; 89 90 ret.val = (type << SWP_TYPE_SHIFT) | (offset & SWP_OFFSET_MASK); 91 return ret; 92 } 93 94 /* 95 * Extract the `type' field from a swp_entry_t. The swp_entry_t is in 96 * arch-independent format 97 */ 98 static inline unsigned swp_type(swp_entry_t entry) 99 { 100 return (entry.val >> SWP_TYPE_SHIFT); 101 } 102 103 /* 104 * Extract the `offset' field from a swp_entry_t. The swp_entry_t is in 105 * arch-independent format 106 */ 107 static inline pgoff_t swp_offset(swp_entry_t entry) 108 { 109 return entry.val & SWP_OFFSET_MASK; 110 } 111 112 /* 113 * This should only be called upon a pfn swap entry to get the PFN stored 114 * in the swap entry. Please refers to is_pfn_swap_entry() for definition 115 * of pfn swap entry. 116 */ 117 static inline unsigned long swp_offset_pfn(swp_entry_t entry) 118 { 119 VM_BUG_ON(!is_pfn_swap_entry(entry)); 120 return swp_offset(entry) & SWP_PFN_MASK; 121 } 122 123 /* check whether a pte points to a swap entry */ 124 static inline int is_swap_pte(pte_t pte) 125 { 126 return !pte_none(pte) && !pte_present(pte); 127 } 128 129 /* 130 * Convert the arch-dependent pte representation of a swp_entry_t into an 131 * arch-independent swp_entry_t. 132 */ 133 static inline swp_entry_t pte_to_swp_entry(pte_t pte) 134 { 135 swp_entry_t arch_entry; 136 137 pte = pte_swp_clear_flags(pte); 138 arch_entry = __pte_to_swp_entry(pte); 139 return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); 140 } 141 142 /* 143 * Convert the arch-independent representation of a swp_entry_t into the 144 * arch-dependent pte representation. 145 */ 146 static inline pte_t swp_entry_to_pte(swp_entry_t entry) 147 { 148 swp_entry_t arch_entry; 149 150 arch_entry = __swp_entry(swp_type(entry), swp_offset(entry)); 151 return __swp_entry_to_pte(arch_entry); 152 } 153 154 static inline swp_entry_t radix_to_swp_entry(void *arg) 155 { 156 swp_entry_t entry; 157 158 entry.val = xa_to_value(arg); 159 return entry; 160 } 161 162 static inline void *swp_to_radix_entry(swp_entry_t entry) 163 { 164 return xa_mk_value(entry.val); 165 } 166 167 #if IS_ENABLED(CONFIG_DEVICE_PRIVATE) 168 static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset) 169 { 170 return swp_entry(SWP_DEVICE_READ, offset); 171 } 172 173 static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset) 174 { 175 return swp_entry(SWP_DEVICE_WRITE, offset); 176 } 177 178 static inline bool is_device_private_entry(swp_entry_t entry) 179 { 180 int type = swp_type(entry); 181 return type == SWP_DEVICE_READ || type == SWP_DEVICE_WRITE; 182 } 183 184 static inline bool is_writable_device_private_entry(swp_entry_t entry) 185 { 186 return unlikely(swp_type(entry) == SWP_DEVICE_WRITE); 187 } 188 189 static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset) 190 { 191 return swp_entry(SWP_DEVICE_EXCLUSIVE_READ, offset); 192 } 193 194 static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset) 195 { 196 return swp_entry(SWP_DEVICE_EXCLUSIVE_WRITE, offset); 197 } 198 199 static inline bool is_device_exclusive_entry(swp_entry_t entry) 200 { 201 return swp_type(entry) == SWP_DEVICE_EXCLUSIVE_READ || 202 swp_type(entry) == SWP_DEVICE_EXCLUSIVE_WRITE; 203 } 204 205 static inline bool is_writable_device_exclusive_entry(swp_entry_t entry) 206 { 207 return unlikely(swp_type(entry) == SWP_DEVICE_EXCLUSIVE_WRITE); 208 } 209 #else /* CONFIG_DEVICE_PRIVATE */ 210 static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset) 211 { 212 return swp_entry(0, 0); 213 } 214 215 static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset) 216 { 217 return swp_entry(0, 0); 218 } 219 220 static inline bool is_device_private_entry(swp_entry_t entry) 221 { 222 return false; 223 } 224 225 static inline bool is_writable_device_private_entry(swp_entry_t entry) 226 { 227 return false; 228 } 229 230 static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset) 231 { 232 return swp_entry(0, 0); 233 } 234 235 static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset) 236 { 237 return swp_entry(0, 0); 238 } 239 240 static inline bool is_device_exclusive_entry(swp_entry_t entry) 241 { 242 return false; 243 } 244 245 static inline bool is_writable_device_exclusive_entry(swp_entry_t entry) 246 { 247 return false; 248 } 249 #endif /* CONFIG_DEVICE_PRIVATE */ 250 251 #ifdef CONFIG_MIGRATION 252 static inline int is_migration_entry(swp_entry_t entry) 253 { 254 return unlikely(swp_type(entry) == SWP_MIGRATION_READ || 255 swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE || 256 swp_type(entry) == SWP_MIGRATION_WRITE); 257 } 258 259 static inline int is_writable_migration_entry(swp_entry_t entry) 260 { 261 return unlikely(swp_type(entry) == SWP_MIGRATION_WRITE); 262 } 263 264 static inline int is_readable_migration_entry(swp_entry_t entry) 265 { 266 return unlikely(swp_type(entry) == SWP_MIGRATION_READ); 267 } 268 269 static inline int is_readable_exclusive_migration_entry(swp_entry_t entry) 270 { 271 return unlikely(swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE); 272 } 273 274 static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) 275 { 276 return swp_entry(SWP_MIGRATION_READ, offset); 277 } 278 279 static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset) 280 { 281 return swp_entry(SWP_MIGRATION_READ_EXCLUSIVE, offset); 282 } 283 284 static inline swp_entry_t make_writable_migration_entry(pgoff_t offset) 285 { 286 return swp_entry(SWP_MIGRATION_WRITE, offset); 287 } 288 289 /* 290 * Returns whether the host has large enough swap offset field to support 291 * carrying over pgtable A/D bits for page migrations. The result is 292 * pretty much arch specific. 293 */ 294 static inline bool migration_entry_supports_ad(void) 295 { 296 #ifdef CONFIG_SWAP 297 return swap_migration_ad_supported; 298 #else /* CONFIG_SWAP */ 299 return false; 300 #endif /* CONFIG_SWAP */ 301 } 302 303 static inline swp_entry_t make_migration_entry_young(swp_entry_t entry) 304 { 305 if (migration_entry_supports_ad()) 306 return swp_entry(swp_type(entry), 307 swp_offset(entry) | SWP_MIG_YOUNG); 308 return entry; 309 } 310 311 static inline bool is_migration_entry_young(swp_entry_t entry) 312 { 313 if (migration_entry_supports_ad()) 314 return swp_offset(entry) & SWP_MIG_YOUNG; 315 /* Keep the old behavior of aging page after migration */ 316 return false; 317 } 318 319 static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry) 320 { 321 if (migration_entry_supports_ad()) 322 return swp_entry(swp_type(entry), 323 swp_offset(entry) | SWP_MIG_DIRTY); 324 return entry; 325 } 326 327 static inline bool is_migration_entry_dirty(swp_entry_t entry) 328 { 329 if (migration_entry_supports_ad()) 330 return swp_offset(entry) & SWP_MIG_DIRTY; 331 /* Keep the old behavior of clean page after migration */ 332 return false; 333 } 334 335 extern void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, 336 spinlock_t *ptl); 337 extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 338 unsigned long address); 339 #ifdef CONFIG_HUGETLB_PAGE 340 extern void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl); 341 extern void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte); 342 #endif /* CONFIG_HUGETLB_PAGE */ 343 #else /* CONFIG_MIGRATION */ 344 static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) 345 { 346 return swp_entry(0, 0); 347 } 348 349 static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset) 350 { 351 return swp_entry(0, 0); 352 } 353 354 static inline swp_entry_t make_writable_migration_entry(pgoff_t offset) 355 { 356 return swp_entry(0, 0); 357 } 358 359 static inline int is_migration_entry(swp_entry_t swp) 360 { 361 return 0; 362 } 363 364 static inline void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, 365 spinlock_t *ptl) { } 366 static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 367 unsigned long address) { } 368 #ifdef CONFIG_HUGETLB_PAGE 369 static inline void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl) { } 370 static inline void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte) { } 371 #endif /* CONFIG_HUGETLB_PAGE */ 372 static inline int is_writable_migration_entry(swp_entry_t entry) 373 { 374 return 0; 375 } 376 static inline int is_readable_migration_entry(swp_entry_t entry) 377 { 378 return 0; 379 } 380 381 static inline swp_entry_t make_migration_entry_young(swp_entry_t entry) 382 { 383 return entry; 384 } 385 386 static inline bool is_migration_entry_young(swp_entry_t entry) 387 { 388 return false; 389 } 390 391 static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry) 392 { 393 return entry; 394 } 395 396 static inline bool is_migration_entry_dirty(swp_entry_t entry) 397 { 398 return false; 399 } 400 #endif /* CONFIG_MIGRATION */ 401 402 typedef unsigned long pte_marker; 403 404 #define PTE_MARKER_UFFD_WP BIT(0) 405 #define PTE_MARKER_SWAPIN_ERROR BIT(1) 406 #define PTE_MARKER_MASK (BIT(2) - 1) 407 408 static inline swp_entry_t make_pte_marker_entry(pte_marker marker) 409 { 410 return swp_entry(SWP_PTE_MARKER, marker); 411 } 412 413 static inline bool is_pte_marker_entry(swp_entry_t entry) 414 { 415 return swp_type(entry) == SWP_PTE_MARKER; 416 } 417 418 static inline pte_marker pte_marker_get(swp_entry_t entry) 419 { 420 return swp_offset(entry) & PTE_MARKER_MASK; 421 } 422 423 static inline bool is_pte_marker(pte_t pte) 424 { 425 return is_swap_pte(pte) && is_pte_marker_entry(pte_to_swp_entry(pte)); 426 } 427 428 static inline pte_t make_pte_marker(pte_marker marker) 429 { 430 return swp_entry_to_pte(make_pte_marker_entry(marker)); 431 } 432 433 static inline swp_entry_t make_swapin_error_entry(void) 434 { 435 return make_pte_marker_entry(PTE_MARKER_SWAPIN_ERROR); 436 } 437 438 static inline int is_swapin_error_entry(swp_entry_t entry) 439 { 440 return is_pte_marker_entry(entry) && 441 (pte_marker_get(entry) & PTE_MARKER_SWAPIN_ERROR); 442 } 443 444 /* 445 * This is a special version to check pte_none() just to cover the case when 446 * the pte is a pte marker. It existed because in many cases the pte marker 447 * should be seen as a none pte; it's just that we have stored some information 448 * onto the none pte so it becomes not-none any more. 449 * 450 * It should be used when the pte is file-backed, ram-based and backing 451 * userspace pages, like shmem. It is not needed upon pgtables that do not 452 * support pte markers at all. For example, it's not needed on anonymous 453 * memory, kernel-only memory (including when the system is during-boot), 454 * non-ram based generic file-system. It's fine to be used even there, but the 455 * extra pte marker check will be pure overhead. 456 */ 457 static inline int pte_none_mostly(pte_t pte) 458 { 459 return pte_none(pte) || is_pte_marker(pte); 460 } 461 462 static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry) 463 { 464 struct page *p = pfn_to_page(swp_offset_pfn(entry)); 465 466 /* 467 * Any use of migration entries may only occur while the 468 * corresponding page is locked 469 */ 470 BUG_ON(is_migration_entry(entry) && !PageLocked(p)); 471 472 return p; 473 } 474 475 /* 476 * A pfn swap entry is a special type of swap entry that always has a pfn stored 477 * in the swap offset. They are used to represent unaddressable device memory 478 * and to restrict access to a page undergoing migration. 479 */ 480 static inline bool is_pfn_swap_entry(swp_entry_t entry) 481 { 482 /* Make sure the swp offset can always store the needed fields */ 483 BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS); 484 485 return is_migration_entry(entry) || is_device_private_entry(entry) || 486 is_device_exclusive_entry(entry); 487 } 488 489 struct page_vma_mapped_walk; 490 491 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 492 extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 493 struct page *page); 494 495 extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, 496 struct page *new); 497 498 extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd); 499 500 static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd) 501 { 502 swp_entry_t arch_entry; 503 504 if (pmd_swp_soft_dirty(pmd)) 505 pmd = pmd_swp_clear_soft_dirty(pmd); 506 if (pmd_swp_uffd_wp(pmd)) 507 pmd = pmd_swp_clear_uffd_wp(pmd); 508 arch_entry = __pmd_to_swp_entry(pmd); 509 return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); 510 } 511 512 static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) 513 { 514 swp_entry_t arch_entry; 515 516 arch_entry = __swp_entry(swp_type(entry), swp_offset(entry)); 517 return __swp_entry_to_pmd(arch_entry); 518 } 519 520 static inline int is_pmd_migration_entry(pmd_t pmd) 521 { 522 return is_swap_pmd(pmd) && is_migration_entry(pmd_to_swp_entry(pmd)); 523 } 524 #else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ 525 static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 526 struct page *page) 527 { 528 BUILD_BUG(); 529 } 530 531 static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, 532 struct page *new) 533 { 534 BUILD_BUG(); 535 } 536 537 static inline void pmd_migration_entry_wait(struct mm_struct *m, pmd_t *p) { } 538 539 static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd) 540 { 541 return swp_entry(0, 0); 542 } 543 544 static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) 545 { 546 return __pmd(0); 547 } 548 549 static inline int is_pmd_migration_entry(pmd_t pmd) 550 { 551 return 0; 552 } 553 #endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ 554 555 #ifdef CONFIG_MEMORY_FAILURE 556 557 /* 558 * Support for hardware poisoned pages 559 */ 560 static inline swp_entry_t make_hwpoison_entry(struct page *page) 561 { 562 BUG_ON(!PageLocked(page)); 563 return swp_entry(SWP_HWPOISON, page_to_pfn(page)); 564 } 565 566 static inline int is_hwpoison_entry(swp_entry_t entry) 567 { 568 return swp_type(entry) == SWP_HWPOISON; 569 } 570 571 #else 572 573 static inline swp_entry_t make_hwpoison_entry(struct page *page) 574 { 575 return swp_entry(0, 0); 576 } 577 578 static inline int is_hwpoison_entry(swp_entry_t swp) 579 { 580 return 0; 581 } 582 #endif 583 584 static inline int non_swap_entry(swp_entry_t entry) 585 { 586 return swp_type(entry) >= MAX_SWAPFILES; 587 } 588 589 #endif /* CONFIG_MMU */ 590 #endif /* _LINUX_SWAPOPS_H */ 591