1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_SWAPOPS_H 3 #define _LINUX_SWAPOPS_H 4 5 #include <linux/radix-tree.h> 6 #include <linux/bug.h> 7 #include <linux/mm_types.h> 8 9 #ifdef CONFIG_MMU 10 11 /* 12 * swapcache pages are stored in the swapper_space radix tree. We want to 13 * get good packing density in that tree, so the index should be dense in 14 * the low-order bits. 15 * 16 * We arrange the `type' and `offset' fields so that `type' is at the six 17 * high-order bits of the swp_entry_t and `offset' is right-aligned in the 18 * remaining bits. Although `type' itself needs only five bits, we allow for 19 * shmem/tmpfs to shift it all up a further one bit: see swp_to_radix_entry(). 20 * 21 * swp_entry_t's are *never* stored anywhere in their arch-dependent format. 22 */ 23 #define SWP_TYPE_SHIFT (BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT) 24 #define SWP_OFFSET_MASK ((1UL << SWP_TYPE_SHIFT) - 1) 25 26 /* Clear all flags but only keep swp_entry_t related information */ 27 static inline pte_t pte_swp_clear_flags(pte_t pte) 28 { 29 if (pte_swp_exclusive(pte)) 30 pte = pte_swp_clear_exclusive(pte); 31 if (pte_swp_soft_dirty(pte)) 32 pte = pte_swp_clear_soft_dirty(pte); 33 if (pte_swp_uffd_wp(pte)) 34 pte = pte_swp_clear_uffd_wp(pte); 35 return pte; 36 } 37 38 /* 39 * Store a type+offset into a swp_entry_t in an arch-independent format 40 */ 41 static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset) 42 { 43 swp_entry_t ret; 44 45 ret.val = (type << SWP_TYPE_SHIFT) | (offset & SWP_OFFSET_MASK); 46 return ret; 47 } 48 49 /* 50 * Extract the `type' field from a swp_entry_t. The swp_entry_t is in 51 * arch-independent format 52 */ 53 static inline unsigned swp_type(swp_entry_t entry) 54 { 55 return (entry.val >> SWP_TYPE_SHIFT); 56 } 57 58 /* 59 * Extract the `offset' field from a swp_entry_t. The swp_entry_t is in 60 * arch-independent format 61 */ 62 static inline pgoff_t swp_offset(swp_entry_t entry) 63 { 64 return entry.val & SWP_OFFSET_MASK; 65 } 66 67 /* check whether a pte points to a swap entry */ 68 static inline int is_swap_pte(pte_t pte) 69 { 70 return !pte_none(pte) && !pte_present(pte); 71 } 72 73 /* 74 * Convert the arch-dependent pte representation of a swp_entry_t into an 75 * arch-independent swp_entry_t. 76 */ 77 static inline swp_entry_t pte_to_swp_entry(pte_t pte) 78 { 79 swp_entry_t arch_entry; 80 81 pte = pte_swp_clear_flags(pte); 82 arch_entry = __pte_to_swp_entry(pte); 83 return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); 84 } 85 86 /* 87 * Convert the arch-independent representation of a swp_entry_t into the 88 * arch-dependent pte representation. 89 */ 90 static inline pte_t swp_entry_to_pte(swp_entry_t entry) 91 { 92 swp_entry_t arch_entry; 93 94 arch_entry = __swp_entry(swp_type(entry), swp_offset(entry)); 95 return __swp_entry_to_pte(arch_entry); 96 } 97 98 static inline swp_entry_t radix_to_swp_entry(void *arg) 99 { 100 swp_entry_t entry; 101 102 entry.val = xa_to_value(arg); 103 return entry; 104 } 105 106 static inline void *swp_to_radix_entry(swp_entry_t entry) 107 { 108 return xa_mk_value(entry.val); 109 } 110 111 static inline swp_entry_t make_swapin_error_entry(struct page *page) 112 { 113 return swp_entry(SWP_SWAPIN_ERROR, page_to_pfn(page)); 114 } 115 116 static inline int is_swapin_error_entry(swp_entry_t entry) 117 { 118 return swp_type(entry) == SWP_SWAPIN_ERROR; 119 } 120 121 #if IS_ENABLED(CONFIG_DEVICE_PRIVATE) 122 static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset) 123 { 124 return swp_entry(SWP_DEVICE_READ, offset); 125 } 126 127 static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset) 128 { 129 return swp_entry(SWP_DEVICE_WRITE, offset); 130 } 131 132 static inline bool is_device_private_entry(swp_entry_t entry) 133 { 134 int type = swp_type(entry); 135 return type == SWP_DEVICE_READ || type == SWP_DEVICE_WRITE; 136 } 137 138 static inline bool is_writable_device_private_entry(swp_entry_t entry) 139 { 140 return unlikely(swp_type(entry) == SWP_DEVICE_WRITE); 141 } 142 143 static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset) 144 { 145 return swp_entry(SWP_DEVICE_EXCLUSIVE_READ, offset); 146 } 147 148 static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset) 149 { 150 return swp_entry(SWP_DEVICE_EXCLUSIVE_WRITE, offset); 151 } 152 153 static inline bool is_device_exclusive_entry(swp_entry_t entry) 154 { 155 return swp_type(entry) == SWP_DEVICE_EXCLUSIVE_READ || 156 swp_type(entry) == SWP_DEVICE_EXCLUSIVE_WRITE; 157 } 158 159 static inline bool is_writable_device_exclusive_entry(swp_entry_t entry) 160 { 161 return unlikely(swp_type(entry) == SWP_DEVICE_EXCLUSIVE_WRITE); 162 } 163 #else /* CONFIG_DEVICE_PRIVATE */ 164 static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset) 165 { 166 return swp_entry(0, 0); 167 } 168 169 static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset) 170 { 171 return swp_entry(0, 0); 172 } 173 174 static inline bool is_device_private_entry(swp_entry_t entry) 175 { 176 return false; 177 } 178 179 static inline bool is_writable_device_private_entry(swp_entry_t entry) 180 { 181 return false; 182 } 183 184 static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset) 185 { 186 return swp_entry(0, 0); 187 } 188 189 static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset) 190 { 191 return swp_entry(0, 0); 192 } 193 194 static inline bool is_device_exclusive_entry(swp_entry_t entry) 195 { 196 return false; 197 } 198 199 static inline bool is_writable_device_exclusive_entry(swp_entry_t entry) 200 { 201 return false; 202 } 203 #endif /* CONFIG_DEVICE_PRIVATE */ 204 205 #ifdef CONFIG_MIGRATION 206 static inline int is_migration_entry(swp_entry_t entry) 207 { 208 return unlikely(swp_type(entry) == SWP_MIGRATION_READ || 209 swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE || 210 swp_type(entry) == SWP_MIGRATION_WRITE); 211 } 212 213 static inline int is_writable_migration_entry(swp_entry_t entry) 214 { 215 return unlikely(swp_type(entry) == SWP_MIGRATION_WRITE); 216 } 217 218 static inline int is_readable_migration_entry(swp_entry_t entry) 219 { 220 return unlikely(swp_type(entry) == SWP_MIGRATION_READ); 221 } 222 223 static inline int is_readable_exclusive_migration_entry(swp_entry_t entry) 224 { 225 return unlikely(swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE); 226 } 227 228 static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) 229 { 230 return swp_entry(SWP_MIGRATION_READ, offset); 231 } 232 233 static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset) 234 { 235 return swp_entry(SWP_MIGRATION_READ_EXCLUSIVE, offset); 236 } 237 238 static inline swp_entry_t make_writable_migration_entry(pgoff_t offset) 239 { 240 return swp_entry(SWP_MIGRATION_WRITE, offset); 241 } 242 243 extern void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, 244 spinlock_t *ptl); 245 extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 246 unsigned long address); 247 #ifdef CONFIG_HUGETLB_PAGE 248 extern void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl); 249 extern void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte); 250 #endif 251 #else 252 static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) 253 { 254 return swp_entry(0, 0); 255 } 256 257 static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset) 258 { 259 return swp_entry(0, 0); 260 } 261 262 static inline swp_entry_t make_writable_migration_entry(pgoff_t offset) 263 { 264 return swp_entry(0, 0); 265 } 266 267 static inline int is_migration_entry(swp_entry_t swp) 268 { 269 return 0; 270 } 271 272 static inline void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, 273 spinlock_t *ptl) { } 274 static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 275 unsigned long address) { } 276 #ifdef CONFIG_HUGETLB_PAGE 277 static inline void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl) { } 278 static inline void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte) { } 279 #endif 280 static inline int is_writable_migration_entry(swp_entry_t entry) 281 { 282 return 0; 283 } 284 static inline int is_readable_migration_entry(swp_entry_t entry) 285 { 286 return 0; 287 } 288 289 #endif 290 291 typedef unsigned long pte_marker; 292 293 #define PTE_MARKER_UFFD_WP BIT(0) 294 #define PTE_MARKER_MASK (PTE_MARKER_UFFD_WP) 295 296 #ifdef CONFIG_PTE_MARKER 297 298 static inline swp_entry_t make_pte_marker_entry(pte_marker marker) 299 { 300 return swp_entry(SWP_PTE_MARKER, marker); 301 } 302 303 static inline bool is_pte_marker_entry(swp_entry_t entry) 304 { 305 return swp_type(entry) == SWP_PTE_MARKER; 306 } 307 308 static inline pte_marker pte_marker_get(swp_entry_t entry) 309 { 310 return swp_offset(entry) & PTE_MARKER_MASK; 311 } 312 313 static inline bool is_pte_marker(pte_t pte) 314 { 315 return is_swap_pte(pte) && is_pte_marker_entry(pte_to_swp_entry(pte)); 316 } 317 318 #else /* CONFIG_PTE_MARKER */ 319 320 static inline swp_entry_t make_pte_marker_entry(pte_marker marker) 321 { 322 /* This should never be called if !CONFIG_PTE_MARKER */ 323 WARN_ON_ONCE(1); 324 return swp_entry(0, 0); 325 } 326 327 static inline bool is_pte_marker_entry(swp_entry_t entry) 328 { 329 return false; 330 } 331 332 static inline pte_marker pte_marker_get(swp_entry_t entry) 333 { 334 return 0; 335 } 336 337 static inline bool is_pte_marker(pte_t pte) 338 { 339 return false; 340 } 341 342 #endif /* CONFIG_PTE_MARKER */ 343 344 static inline pte_t make_pte_marker(pte_marker marker) 345 { 346 return swp_entry_to_pte(make_pte_marker_entry(marker)); 347 } 348 349 /* 350 * This is a special version to check pte_none() just to cover the case when 351 * the pte is a pte marker. It existed because in many cases the pte marker 352 * should be seen as a none pte; it's just that we have stored some information 353 * onto the none pte so it becomes not-none any more. 354 * 355 * It should be used when the pte is file-backed, ram-based and backing 356 * userspace pages, like shmem. It is not needed upon pgtables that do not 357 * support pte markers at all. For example, it's not needed on anonymous 358 * memory, kernel-only memory (including when the system is during-boot), 359 * non-ram based generic file-system. It's fine to be used even there, but the 360 * extra pte marker check will be pure overhead. 361 * 362 * For systems configured with !CONFIG_PTE_MARKER this will be automatically 363 * optimized to pte_none(). 364 */ 365 static inline int pte_none_mostly(pte_t pte) 366 { 367 return pte_none(pte) || is_pte_marker(pte); 368 } 369 370 static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry) 371 { 372 struct page *p = pfn_to_page(swp_offset(entry)); 373 374 /* 375 * Any use of migration entries may only occur while the 376 * corresponding page is locked 377 */ 378 BUG_ON(is_migration_entry(entry) && !PageLocked(p)); 379 380 return p; 381 } 382 383 /* 384 * A pfn swap entry is a special type of swap entry that always has a pfn stored 385 * in the swap offset. They are used to represent unaddressable device memory 386 * and to restrict access to a page undergoing migration. 387 */ 388 static inline bool is_pfn_swap_entry(swp_entry_t entry) 389 { 390 return is_migration_entry(entry) || is_device_private_entry(entry) || 391 is_device_exclusive_entry(entry); 392 } 393 394 struct page_vma_mapped_walk; 395 396 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 397 extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 398 struct page *page); 399 400 extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, 401 struct page *new); 402 403 extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd); 404 405 static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd) 406 { 407 swp_entry_t arch_entry; 408 409 if (pmd_swp_soft_dirty(pmd)) 410 pmd = pmd_swp_clear_soft_dirty(pmd); 411 if (pmd_swp_uffd_wp(pmd)) 412 pmd = pmd_swp_clear_uffd_wp(pmd); 413 arch_entry = __pmd_to_swp_entry(pmd); 414 return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); 415 } 416 417 static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) 418 { 419 swp_entry_t arch_entry; 420 421 arch_entry = __swp_entry(swp_type(entry), swp_offset(entry)); 422 return __swp_entry_to_pmd(arch_entry); 423 } 424 425 static inline int is_pmd_migration_entry(pmd_t pmd) 426 { 427 return is_swap_pmd(pmd) && is_migration_entry(pmd_to_swp_entry(pmd)); 428 } 429 #else 430 static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 431 struct page *page) 432 { 433 BUILD_BUG(); 434 } 435 436 static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, 437 struct page *new) 438 { 439 BUILD_BUG(); 440 } 441 442 static inline void pmd_migration_entry_wait(struct mm_struct *m, pmd_t *p) { } 443 444 static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd) 445 { 446 return swp_entry(0, 0); 447 } 448 449 static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) 450 { 451 return __pmd(0); 452 } 453 454 static inline int is_pmd_migration_entry(pmd_t pmd) 455 { 456 return 0; 457 } 458 #endif 459 460 #ifdef CONFIG_MEMORY_FAILURE 461 462 extern atomic_long_t num_poisoned_pages __read_mostly; 463 464 /* 465 * Support for hardware poisoned pages 466 */ 467 static inline swp_entry_t make_hwpoison_entry(struct page *page) 468 { 469 BUG_ON(!PageLocked(page)); 470 return swp_entry(SWP_HWPOISON, page_to_pfn(page)); 471 } 472 473 static inline int is_hwpoison_entry(swp_entry_t entry) 474 { 475 return swp_type(entry) == SWP_HWPOISON; 476 } 477 478 static inline unsigned long hwpoison_entry_to_pfn(swp_entry_t entry) 479 { 480 return swp_offset(entry); 481 } 482 483 static inline void num_poisoned_pages_inc(void) 484 { 485 atomic_long_inc(&num_poisoned_pages); 486 } 487 488 static inline void num_poisoned_pages_dec(void) 489 { 490 atomic_long_dec(&num_poisoned_pages); 491 } 492 493 static inline void num_poisoned_pages_sub(long i) 494 { 495 atomic_long_sub(i, &num_poisoned_pages); 496 } 497 498 #else 499 500 static inline swp_entry_t make_hwpoison_entry(struct page *page) 501 { 502 return swp_entry(0, 0); 503 } 504 505 static inline int is_hwpoison_entry(swp_entry_t swp) 506 { 507 return 0; 508 } 509 510 static inline void num_poisoned_pages_inc(void) 511 { 512 } 513 514 static inline void num_poisoned_pages_sub(long i) 515 { 516 } 517 #endif 518 519 static inline int non_swap_entry(swp_entry_t entry) 520 { 521 return swp_type(entry) >= MAX_SWAPFILES; 522 } 523 524 #endif /* CONFIG_MMU */ 525 #endif /* _LINUX_SWAPOPS_H */ 526