1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/swapfile.c 4 * 5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 6 * Swap reorganised 29.12.95, Stephen Tweedie 7 */ 8 9 #include <linux/mm.h> 10 #include <linux/sched/mm.h> 11 #include <linux/sched/task.h> 12 #include <linux/hugetlb.h> 13 #include <linux/mman.h> 14 #include <linux/slab.h> 15 #include <linux/kernel_stat.h> 16 #include <linux/swap.h> 17 #include <linux/vmalloc.h> 18 #include <linux/pagemap.h> 19 #include <linux/namei.h> 20 #include <linux/shmem_fs.h> 21 #include <linux/blkdev.h> 22 #include <linux/random.h> 23 #include <linux/writeback.h> 24 #include <linux/proc_fs.h> 25 #include <linux/seq_file.h> 26 #include <linux/init.h> 27 #include <linux/ksm.h> 28 #include <linux/rmap.h> 29 #include <linux/security.h> 30 #include <linux/backing-dev.h> 31 #include <linux/mutex.h> 32 #include <linux/capability.h> 33 #include <linux/syscalls.h> 34 #include <linux/memcontrol.h> 35 #include <linux/poll.h> 36 #include <linux/oom.h> 37 #include <linux/frontswap.h> 38 #include <linux/swapfile.h> 39 #include <linux/export.h> 40 #include <linux/swap_slots.h> 41 #include <linux/sort.h> 42 43 #include <asm/tlbflush.h> 44 #include <linux/swapops.h> 45 #include <linux/swap_cgroup.h> 46 47 static bool swap_count_continued(struct swap_info_struct *, pgoff_t, 48 unsigned char); 49 static void free_swap_count_continuations(struct swap_info_struct *); 50 51 DEFINE_SPINLOCK(swap_lock); 52 static unsigned int nr_swapfiles; 53 atomic_long_t nr_swap_pages; 54 /* 55 * Some modules use swappable objects and may try to swap them out under 56 * memory pressure (via the shrinker). Before doing so, they may wish to 57 * check to see if any swap space is available. 58 */ 59 EXPORT_SYMBOL_GPL(nr_swap_pages); 60 /* protected with swap_lock. reading in vm_swap_full() doesn't need lock */ 61 long total_swap_pages; 62 static int least_priority = -1; 63 64 static const char Bad_file[] = "Bad swap file entry "; 65 static const char Unused_file[] = "Unused swap file entry "; 66 static const char Bad_offset[] = "Bad swap offset entry "; 67 static const char Unused_offset[] = "Unused swap offset entry "; 68 69 /* 70 * all active swap_info_structs 71 * protected with swap_lock, and ordered by priority. 72 */ 73 PLIST_HEAD(swap_active_head); 74 75 /* 76 * all available (active, not full) swap_info_structs 77 * protected with swap_avail_lock, ordered by priority. 78 * This is used by get_swap_page() instead of swap_active_head 79 * because swap_active_head includes all swap_info_structs, 80 * but get_swap_page() doesn't need to look at full ones. 81 * This uses its own lock instead of swap_lock because when a 82 * swap_info_struct changes between not-full/full, it needs to 83 * add/remove itself to/from this list, but the swap_info_struct->lock 84 * is held and the locking order requires swap_lock to be taken 85 * before any swap_info_struct->lock. 86 */ 87 static struct plist_head *swap_avail_heads; 88 static DEFINE_SPINLOCK(swap_avail_lock); 89 90 struct swap_info_struct *swap_info[MAX_SWAPFILES]; 91 92 static DEFINE_MUTEX(swapon_mutex); 93 94 static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait); 95 /* Activity counter to indicate that a swapon or swapoff has occurred */ 96 static atomic_t proc_poll_event = ATOMIC_INIT(0); 97 98 atomic_t nr_rotate_swap = ATOMIC_INIT(0); 99 100 static struct swap_info_struct *swap_type_to_swap_info(int type) 101 { 102 if (type >= READ_ONCE(nr_swapfiles)) 103 return NULL; 104 105 smp_rmb(); /* Pairs with smp_wmb in alloc_swap_info. */ 106 return READ_ONCE(swap_info[type]); 107 } 108 109 static inline unsigned char swap_count(unsigned char ent) 110 { 111 return ent & ~SWAP_HAS_CACHE; /* may include COUNT_CONTINUED flag */ 112 } 113 114 /* Reclaim the swap entry anyway if possible */ 115 #define TTRS_ANYWAY 0x1 116 /* 117 * Reclaim the swap entry if there are no more mappings of the 118 * corresponding page 119 */ 120 #define TTRS_UNMAPPED 0x2 121 /* Reclaim the swap entry if swap is getting full*/ 122 #define TTRS_FULL 0x4 123 124 /* returns 1 if swap entry is freed */ 125 static int __try_to_reclaim_swap(struct swap_info_struct *si, 126 unsigned long offset, unsigned long flags) 127 { 128 swp_entry_t entry = swp_entry(si->type, offset); 129 struct page *page; 130 int ret = 0; 131 132 page = find_get_page(swap_address_space(entry), offset); 133 if (!page) 134 return 0; 135 /* 136 * When this function is called from scan_swap_map_slots() and it's 137 * called by vmscan.c at reclaiming pages. So, we hold a lock on a page, 138 * here. We have to use trylock for avoiding deadlock. This is a special 139 * case and you should use try_to_free_swap() with explicit lock_page() 140 * in usual operations. 141 */ 142 if (trylock_page(page)) { 143 if ((flags & TTRS_ANYWAY) || 144 ((flags & TTRS_UNMAPPED) && !page_mapped(page)) || 145 ((flags & TTRS_FULL) && mem_cgroup_swap_full(page))) 146 ret = try_to_free_swap(page); 147 unlock_page(page); 148 } 149 put_page(page); 150 return ret; 151 } 152 153 static inline struct swap_extent *first_se(struct swap_info_struct *sis) 154 { 155 struct rb_node *rb = rb_first(&sis->swap_extent_root); 156 return rb_entry(rb, struct swap_extent, rb_node); 157 } 158 159 static inline struct swap_extent *next_se(struct swap_extent *se) 160 { 161 struct rb_node *rb = rb_next(&se->rb_node); 162 return rb ? rb_entry(rb, struct swap_extent, rb_node) : NULL; 163 } 164 165 /* 166 * swapon tell device that all the old swap contents can be discarded, 167 * to allow the swap device to optimize its wear-levelling. 168 */ 169 static int discard_swap(struct swap_info_struct *si) 170 { 171 struct swap_extent *se; 172 sector_t start_block; 173 sector_t nr_blocks; 174 int err = 0; 175 176 /* Do not discard the swap header page! */ 177 se = first_se(si); 178 start_block = (se->start_block + 1) << (PAGE_SHIFT - 9); 179 nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9); 180 if (nr_blocks) { 181 err = blkdev_issue_discard(si->bdev, start_block, 182 nr_blocks, GFP_KERNEL, 0); 183 if (err) 184 return err; 185 cond_resched(); 186 } 187 188 for (se = next_se(se); se; se = next_se(se)) { 189 start_block = se->start_block << (PAGE_SHIFT - 9); 190 nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9); 191 192 err = blkdev_issue_discard(si->bdev, start_block, 193 nr_blocks, GFP_KERNEL, 0); 194 if (err) 195 break; 196 197 cond_resched(); 198 } 199 return err; /* That will often be -EOPNOTSUPP */ 200 } 201 202 static struct swap_extent * 203 offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset) 204 { 205 struct swap_extent *se; 206 struct rb_node *rb; 207 208 rb = sis->swap_extent_root.rb_node; 209 while (rb) { 210 se = rb_entry(rb, struct swap_extent, rb_node); 211 if (offset < se->start_page) 212 rb = rb->rb_left; 213 else if (offset >= se->start_page + se->nr_pages) 214 rb = rb->rb_right; 215 else 216 return se; 217 } 218 /* It *must* be present */ 219 BUG(); 220 } 221 222 /* 223 * swap allocation tell device that a cluster of swap can now be discarded, 224 * to allow the swap device to optimize its wear-levelling. 225 */ 226 static void discard_swap_cluster(struct swap_info_struct *si, 227 pgoff_t start_page, pgoff_t nr_pages) 228 { 229 struct swap_extent *se = offset_to_swap_extent(si, start_page); 230 231 while (nr_pages) { 232 pgoff_t offset = start_page - se->start_page; 233 sector_t start_block = se->start_block + offset; 234 sector_t nr_blocks = se->nr_pages - offset; 235 236 if (nr_blocks > nr_pages) 237 nr_blocks = nr_pages; 238 start_page += nr_blocks; 239 nr_pages -= nr_blocks; 240 241 start_block <<= PAGE_SHIFT - 9; 242 nr_blocks <<= PAGE_SHIFT - 9; 243 if (blkdev_issue_discard(si->bdev, start_block, 244 nr_blocks, GFP_NOIO, 0)) 245 break; 246 247 se = next_se(se); 248 } 249 } 250 251 #ifdef CONFIG_THP_SWAP 252 #define SWAPFILE_CLUSTER HPAGE_PMD_NR 253 254 #define swap_entry_size(size) (size) 255 #else 256 #define SWAPFILE_CLUSTER 256 257 258 /* 259 * Define swap_entry_size() as constant to let compiler to optimize 260 * out some code if !CONFIG_THP_SWAP 261 */ 262 #define swap_entry_size(size) 1 263 #endif 264 #define LATENCY_LIMIT 256 265 266 static inline void cluster_set_flag(struct swap_cluster_info *info, 267 unsigned int flag) 268 { 269 info->flags = flag; 270 } 271 272 static inline unsigned int cluster_count(struct swap_cluster_info *info) 273 { 274 return info->data; 275 } 276 277 static inline void cluster_set_count(struct swap_cluster_info *info, 278 unsigned int c) 279 { 280 info->data = c; 281 } 282 283 static inline void cluster_set_count_flag(struct swap_cluster_info *info, 284 unsigned int c, unsigned int f) 285 { 286 info->flags = f; 287 info->data = c; 288 } 289 290 static inline unsigned int cluster_next(struct swap_cluster_info *info) 291 { 292 return info->data; 293 } 294 295 static inline void cluster_set_next(struct swap_cluster_info *info, 296 unsigned int n) 297 { 298 info->data = n; 299 } 300 301 static inline void cluster_set_next_flag(struct swap_cluster_info *info, 302 unsigned int n, unsigned int f) 303 { 304 info->flags = f; 305 info->data = n; 306 } 307 308 static inline bool cluster_is_free(struct swap_cluster_info *info) 309 { 310 return info->flags & CLUSTER_FLAG_FREE; 311 } 312 313 static inline bool cluster_is_null(struct swap_cluster_info *info) 314 { 315 return info->flags & CLUSTER_FLAG_NEXT_NULL; 316 } 317 318 static inline void cluster_set_null(struct swap_cluster_info *info) 319 { 320 info->flags = CLUSTER_FLAG_NEXT_NULL; 321 info->data = 0; 322 } 323 324 static inline bool cluster_is_huge(struct swap_cluster_info *info) 325 { 326 if (IS_ENABLED(CONFIG_THP_SWAP)) 327 return info->flags & CLUSTER_FLAG_HUGE; 328 return false; 329 } 330 331 static inline void cluster_clear_huge(struct swap_cluster_info *info) 332 { 333 info->flags &= ~CLUSTER_FLAG_HUGE; 334 } 335 336 static inline struct swap_cluster_info *lock_cluster(struct swap_info_struct *si, 337 unsigned long offset) 338 { 339 struct swap_cluster_info *ci; 340 341 ci = si->cluster_info; 342 if (ci) { 343 ci += offset / SWAPFILE_CLUSTER; 344 spin_lock(&ci->lock); 345 } 346 return ci; 347 } 348 349 static inline void unlock_cluster(struct swap_cluster_info *ci) 350 { 351 if (ci) 352 spin_unlock(&ci->lock); 353 } 354 355 /* 356 * Determine the locking method in use for this device. Return 357 * swap_cluster_info if SSD-style cluster-based locking is in place. 358 */ 359 static inline struct swap_cluster_info *lock_cluster_or_swap_info( 360 struct swap_info_struct *si, unsigned long offset) 361 { 362 struct swap_cluster_info *ci; 363 364 /* Try to use fine-grained SSD-style locking if available: */ 365 ci = lock_cluster(si, offset); 366 /* Otherwise, fall back to traditional, coarse locking: */ 367 if (!ci) 368 spin_lock(&si->lock); 369 370 return ci; 371 } 372 373 static inline void unlock_cluster_or_swap_info(struct swap_info_struct *si, 374 struct swap_cluster_info *ci) 375 { 376 if (ci) 377 unlock_cluster(ci); 378 else 379 spin_unlock(&si->lock); 380 } 381 382 static inline bool cluster_list_empty(struct swap_cluster_list *list) 383 { 384 return cluster_is_null(&list->head); 385 } 386 387 static inline unsigned int cluster_list_first(struct swap_cluster_list *list) 388 { 389 return cluster_next(&list->head); 390 } 391 392 static void cluster_list_init(struct swap_cluster_list *list) 393 { 394 cluster_set_null(&list->head); 395 cluster_set_null(&list->tail); 396 } 397 398 static void cluster_list_add_tail(struct swap_cluster_list *list, 399 struct swap_cluster_info *ci, 400 unsigned int idx) 401 { 402 if (cluster_list_empty(list)) { 403 cluster_set_next_flag(&list->head, idx, 0); 404 cluster_set_next_flag(&list->tail, idx, 0); 405 } else { 406 struct swap_cluster_info *ci_tail; 407 unsigned int tail = cluster_next(&list->tail); 408 409 /* 410 * Nested cluster lock, but both cluster locks are 411 * only acquired when we held swap_info_struct->lock 412 */ 413 ci_tail = ci + tail; 414 spin_lock_nested(&ci_tail->lock, SINGLE_DEPTH_NESTING); 415 cluster_set_next(ci_tail, idx); 416 spin_unlock(&ci_tail->lock); 417 cluster_set_next_flag(&list->tail, idx, 0); 418 } 419 } 420 421 static unsigned int cluster_list_del_first(struct swap_cluster_list *list, 422 struct swap_cluster_info *ci) 423 { 424 unsigned int idx; 425 426 idx = cluster_next(&list->head); 427 if (cluster_next(&list->tail) == idx) { 428 cluster_set_null(&list->head); 429 cluster_set_null(&list->tail); 430 } else 431 cluster_set_next_flag(&list->head, 432 cluster_next(&ci[idx]), 0); 433 434 return idx; 435 } 436 437 /* Add a cluster to discard list and schedule it to do discard */ 438 static void swap_cluster_schedule_discard(struct swap_info_struct *si, 439 unsigned int idx) 440 { 441 /* 442 * If scan_swap_map() can't find a free cluster, it will check 443 * si->swap_map directly. To make sure the discarding cluster isn't 444 * taken by scan_swap_map(), mark the swap entries bad (occupied). It 445 * will be cleared after discard 446 */ 447 memset(si->swap_map + idx * SWAPFILE_CLUSTER, 448 SWAP_MAP_BAD, SWAPFILE_CLUSTER); 449 450 cluster_list_add_tail(&si->discard_clusters, si->cluster_info, idx); 451 452 schedule_work(&si->discard_work); 453 } 454 455 static void __free_cluster(struct swap_info_struct *si, unsigned long idx) 456 { 457 struct swap_cluster_info *ci = si->cluster_info; 458 459 cluster_set_flag(ci + idx, CLUSTER_FLAG_FREE); 460 cluster_list_add_tail(&si->free_clusters, ci, idx); 461 } 462 463 /* 464 * Doing discard actually. After a cluster discard is finished, the cluster 465 * will be added to free cluster list. caller should hold si->lock. 466 */ 467 static void swap_do_scheduled_discard(struct swap_info_struct *si) 468 { 469 struct swap_cluster_info *info, *ci; 470 unsigned int idx; 471 472 info = si->cluster_info; 473 474 while (!cluster_list_empty(&si->discard_clusters)) { 475 idx = cluster_list_del_first(&si->discard_clusters, info); 476 spin_unlock(&si->lock); 477 478 discard_swap_cluster(si, idx * SWAPFILE_CLUSTER, 479 SWAPFILE_CLUSTER); 480 481 spin_lock(&si->lock); 482 ci = lock_cluster(si, idx * SWAPFILE_CLUSTER); 483 __free_cluster(si, idx); 484 memset(si->swap_map + idx * SWAPFILE_CLUSTER, 485 0, SWAPFILE_CLUSTER); 486 unlock_cluster(ci); 487 } 488 } 489 490 static void swap_discard_work(struct work_struct *work) 491 { 492 struct swap_info_struct *si; 493 494 si = container_of(work, struct swap_info_struct, discard_work); 495 496 spin_lock(&si->lock); 497 swap_do_scheduled_discard(si); 498 spin_unlock(&si->lock); 499 } 500 501 static void alloc_cluster(struct swap_info_struct *si, unsigned long idx) 502 { 503 struct swap_cluster_info *ci = si->cluster_info; 504 505 VM_BUG_ON(cluster_list_first(&si->free_clusters) != idx); 506 cluster_list_del_first(&si->free_clusters, ci); 507 cluster_set_count_flag(ci + idx, 0, 0); 508 } 509 510 static void free_cluster(struct swap_info_struct *si, unsigned long idx) 511 { 512 struct swap_cluster_info *ci = si->cluster_info + idx; 513 514 VM_BUG_ON(cluster_count(ci) != 0); 515 /* 516 * If the swap is discardable, prepare discard the cluster 517 * instead of free it immediately. The cluster will be freed 518 * after discard. 519 */ 520 if ((si->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) == 521 (SWP_WRITEOK | SWP_PAGE_DISCARD)) { 522 swap_cluster_schedule_discard(si, idx); 523 return; 524 } 525 526 __free_cluster(si, idx); 527 } 528 529 /* 530 * The cluster corresponding to page_nr will be used. The cluster will be 531 * removed from free cluster list and its usage counter will be increased. 532 */ 533 static void inc_cluster_info_page(struct swap_info_struct *p, 534 struct swap_cluster_info *cluster_info, unsigned long page_nr) 535 { 536 unsigned long idx = page_nr / SWAPFILE_CLUSTER; 537 538 if (!cluster_info) 539 return; 540 if (cluster_is_free(&cluster_info[idx])) 541 alloc_cluster(p, idx); 542 543 VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER); 544 cluster_set_count(&cluster_info[idx], 545 cluster_count(&cluster_info[idx]) + 1); 546 } 547 548 /* 549 * The cluster corresponding to page_nr decreases one usage. If the usage 550 * counter becomes 0, which means no page in the cluster is in using, we can 551 * optionally discard the cluster and add it to free cluster list. 552 */ 553 static void dec_cluster_info_page(struct swap_info_struct *p, 554 struct swap_cluster_info *cluster_info, unsigned long page_nr) 555 { 556 unsigned long idx = page_nr / SWAPFILE_CLUSTER; 557 558 if (!cluster_info) 559 return; 560 561 VM_BUG_ON(cluster_count(&cluster_info[idx]) == 0); 562 cluster_set_count(&cluster_info[idx], 563 cluster_count(&cluster_info[idx]) - 1); 564 565 if (cluster_count(&cluster_info[idx]) == 0) 566 free_cluster(p, idx); 567 } 568 569 /* 570 * It's possible scan_swap_map() uses a free cluster in the middle of free 571 * cluster list. Avoiding such abuse to avoid list corruption. 572 */ 573 static bool 574 scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si, 575 unsigned long offset) 576 { 577 struct percpu_cluster *percpu_cluster; 578 bool conflict; 579 580 offset /= SWAPFILE_CLUSTER; 581 conflict = !cluster_list_empty(&si->free_clusters) && 582 offset != cluster_list_first(&si->free_clusters) && 583 cluster_is_free(&si->cluster_info[offset]); 584 585 if (!conflict) 586 return false; 587 588 percpu_cluster = this_cpu_ptr(si->percpu_cluster); 589 cluster_set_null(&percpu_cluster->index); 590 return true; 591 } 592 593 /* 594 * Try to get a swap entry from current cpu's swap entry pool (a cluster). This 595 * might involve allocating a new cluster for current CPU too. 596 */ 597 static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si, 598 unsigned long *offset, unsigned long *scan_base) 599 { 600 struct percpu_cluster *cluster; 601 struct swap_cluster_info *ci; 602 unsigned long tmp, max; 603 604 new_cluster: 605 cluster = this_cpu_ptr(si->percpu_cluster); 606 if (cluster_is_null(&cluster->index)) { 607 if (!cluster_list_empty(&si->free_clusters)) { 608 cluster->index = si->free_clusters.head; 609 cluster->next = cluster_next(&cluster->index) * 610 SWAPFILE_CLUSTER; 611 } else if (!cluster_list_empty(&si->discard_clusters)) { 612 /* 613 * we don't have free cluster but have some clusters in 614 * discarding, do discard now and reclaim them, then 615 * reread cluster_next_cpu since we dropped si->lock 616 */ 617 swap_do_scheduled_discard(si); 618 *scan_base = this_cpu_read(*si->cluster_next_cpu); 619 *offset = *scan_base; 620 goto new_cluster; 621 } else 622 return false; 623 } 624 625 /* 626 * Other CPUs can use our cluster if they can't find a free cluster, 627 * check if there is still free entry in the cluster 628 */ 629 tmp = cluster->next; 630 max = min_t(unsigned long, si->max, 631 (cluster_next(&cluster->index) + 1) * SWAPFILE_CLUSTER); 632 if (tmp < max) { 633 ci = lock_cluster(si, tmp); 634 while (tmp < max) { 635 if (!si->swap_map[tmp]) 636 break; 637 tmp++; 638 } 639 unlock_cluster(ci); 640 } 641 if (tmp >= max) { 642 cluster_set_null(&cluster->index); 643 goto new_cluster; 644 } 645 cluster->next = tmp + 1; 646 *offset = tmp; 647 *scan_base = tmp; 648 return true; 649 } 650 651 static void __del_from_avail_list(struct swap_info_struct *p) 652 { 653 int nid; 654 655 for_each_node(nid) 656 plist_del(&p->avail_lists[nid], &swap_avail_heads[nid]); 657 } 658 659 static void del_from_avail_list(struct swap_info_struct *p) 660 { 661 spin_lock(&swap_avail_lock); 662 __del_from_avail_list(p); 663 spin_unlock(&swap_avail_lock); 664 } 665 666 static void swap_range_alloc(struct swap_info_struct *si, unsigned long offset, 667 unsigned int nr_entries) 668 { 669 unsigned int end = offset + nr_entries - 1; 670 671 if (offset == si->lowest_bit) 672 si->lowest_bit += nr_entries; 673 if (end == si->highest_bit) 674 WRITE_ONCE(si->highest_bit, si->highest_bit - nr_entries); 675 si->inuse_pages += nr_entries; 676 if (si->inuse_pages == si->pages) { 677 si->lowest_bit = si->max; 678 si->highest_bit = 0; 679 del_from_avail_list(si); 680 } 681 } 682 683 static void add_to_avail_list(struct swap_info_struct *p) 684 { 685 int nid; 686 687 spin_lock(&swap_avail_lock); 688 for_each_node(nid) { 689 WARN_ON(!plist_node_empty(&p->avail_lists[nid])); 690 plist_add(&p->avail_lists[nid], &swap_avail_heads[nid]); 691 } 692 spin_unlock(&swap_avail_lock); 693 } 694 695 static void swap_range_free(struct swap_info_struct *si, unsigned long offset, 696 unsigned int nr_entries) 697 { 698 unsigned long begin = offset; 699 unsigned long end = offset + nr_entries - 1; 700 void (*swap_slot_free_notify)(struct block_device *, unsigned long); 701 702 if (offset < si->lowest_bit) 703 si->lowest_bit = offset; 704 if (end > si->highest_bit) { 705 bool was_full = !si->highest_bit; 706 707 WRITE_ONCE(si->highest_bit, end); 708 if (was_full && (si->flags & SWP_WRITEOK)) 709 add_to_avail_list(si); 710 } 711 atomic_long_add(nr_entries, &nr_swap_pages); 712 si->inuse_pages -= nr_entries; 713 if (si->flags & SWP_BLKDEV) 714 swap_slot_free_notify = 715 si->bdev->bd_disk->fops->swap_slot_free_notify; 716 else 717 swap_slot_free_notify = NULL; 718 while (offset <= end) { 719 arch_swap_invalidate_page(si->type, offset); 720 frontswap_invalidate_page(si->type, offset); 721 if (swap_slot_free_notify) 722 swap_slot_free_notify(si->bdev, offset); 723 offset++; 724 } 725 clear_shadow_from_swap_cache(si->type, begin, end); 726 } 727 728 static void set_cluster_next(struct swap_info_struct *si, unsigned long next) 729 { 730 unsigned long prev; 731 732 if (!(si->flags & SWP_SOLIDSTATE)) { 733 si->cluster_next = next; 734 return; 735 } 736 737 prev = this_cpu_read(*si->cluster_next_cpu); 738 /* 739 * Cross the swap address space size aligned trunk, choose 740 * another trunk randomly to avoid lock contention on swap 741 * address space if possible. 742 */ 743 if ((prev >> SWAP_ADDRESS_SPACE_SHIFT) != 744 (next >> SWAP_ADDRESS_SPACE_SHIFT)) { 745 /* No free swap slots available */ 746 if (si->highest_bit <= si->lowest_bit) 747 return; 748 next = si->lowest_bit + 749 prandom_u32_max(si->highest_bit - si->lowest_bit + 1); 750 next = ALIGN_DOWN(next, SWAP_ADDRESS_SPACE_PAGES); 751 next = max_t(unsigned int, next, si->lowest_bit); 752 } 753 this_cpu_write(*si->cluster_next_cpu, next); 754 } 755 756 static int scan_swap_map_slots(struct swap_info_struct *si, 757 unsigned char usage, int nr, 758 swp_entry_t slots[]) 759 { 760 struct swap_cluster_info *ci; 761 unsigned long offset; 762 unsigned long scan_base; 763 unsigned long last_in_cluster = 0; 764 int latency_ration = LATENCY_LIMIT; 765 int n_ret = 0; 766 bool scanned_many = false; 767 768 /* 769 * We try to cluster swap pages by allocating them sequentially 770 * in swap. Once we've allocated SWAPFILE_CLUSTER pages this 771 * way, however, we resort to first-free allocation, starting 772 * a new cluster. This prevents us from scattering swap pages 773 * all over the entire swap partition, so that we reduce 774 * overall disk seek times between swap pages. -- sct 775 * But we do now try to find an empty cluster. -Andrea 776 * And we let swap pages go all over an SSD partition. Hugh 777 */ 778 779 si->flags += SWP_SCANNING; 780 /* 781 * Use percpu scan base for SSD to reduce lock contention on 782 * cluster and swap cache. For HDD, sequential access is more 783 * important. 784 */ 785 if (si->flags & SWP_SOLIDSTATE) 786 scan_base = this_cpu_read(*si->cluster_next_cpu); 787 else 788 scan_base = si->cluster_next; 789 offset = scan_base; 790 791 /* SSD algorithm */ 792 if (si->cluster_info) { 793 if (!scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) 794 goto scan; 795 } else if (unlikely(!si->cluster_nr--)) { 796 if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) { 797 si->cluster_nr = SWAPFILE_CLUSTER - 1; 798 goto checks; 799 } 800 801 spin_unlock(&si->lock); 802 803 /* 804 * If seek is expensive, start searching for new cluster from 805 * start of partition, to minimize the span of allocated swap. 806 * If seek is cheap, that is the SWP_SOLIDSTATE si->cluster_info 807 * case, just handled by scan_swap_map_try_ssd_cluster() above. 808 */ 809 scan_base = offset = si->lowest_bit; 810 last_in_cluster = offset + SWAPFILE_CLUSTER - 1; 811 812 /* Locate the first empty (unaligned) cluster */ 813 for (; last_in_cluster <= si->highest_bit; offset++) { 814 if (si->swap_map[offset]) 815 last_in_cluster = offset + SWAPFILE_CLUSTER; 816 else if (offset == last_in_cluster) { 817 spin_lock(&si->lock); 818 offset -= SWAPFILE_CLUSTER - 1; 819 si->cluster_next = offset; 820 si->cluster_nr = SWAPFILE_CLUSTER - 1; 821 goto checks; 822 } 823 if (unlikely(--latency_ration < 0)) { 824 cond_resched(); 825 latency_ration = LATENCY_LIMIT; 826 } 827 } 828 829 offset = scan_base; 830 spin_lock(&si->lock); 831 si->cluster_nr = SWAPFILE_CLUSTER - 1; 832 } 833 834 checks: 835 if (si->cluster_info) { 836 while (scan_swap_map_ssd_cluster_conflict(si, offset)) { 837 /* take a break if we already got some slots */ 838 if (n_ret) 839 goto done; 840 if (!scan_swap_map_try_ssd_cluster(si, &offset, 841 &scan_base)) 842 goto scan; 843 } 844 } 845 if (!(si->flags & SWP_WRITEOK)) 846 goto no_page; 847 if (!si->highest_bit) 848 goto no_page; 849 if (offset > si->highest_bit) 850 scan_base = offset = si->lowest_bit; 851 852 ci = lock_cluster(si, offset); 853 /* reuse swap entry of cache-only swap if not busy. */ 854 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) { 855 int swap_was_freed; 856 unlock_cluster(ci); 857 spin_unlock(&si->lock); 858 swap_was_freed = __try_to_reclaim_swap(si, offset, TTRS_ANYWAY); 859 spin_lock(&si->lock); 860 /* entry was freed successfully, try to use this again */ 861 if (swap_was_freed) 862 goto checks; 863 goto scan; /* check next one */ 864 } 865 866 if (si->swap_map[offset]) { 867 unlock_cluster(ci); 868 if (!n_ret) 869 goto scan; 870 else 871 goto done; 872 } 873 WRITE_ONCE(si->swap_map[offset], usage); 874 inc_cluster_info_page(si, si->cluster_info, offset); 875 unlock_cluster(ci); 876 877 swap_range_alloc(si, offset, 1); 878 slots[n_ret++] = swp_entry(si->type, offset); 879 880 /* got enough slots or reach max slots? */ 881 if ((n_ret == nr) || (offset >= si->highest_bit)) 882 goto done; 883 884 /* search for next available slot */ 885 886 /* time to take a break? */ 887 if (unlikely(--latency_ration < 0)) { 888 if (n_ret) 889 goto done; 890 spin_unlock(&si->lock); 891 cond_resched(); 892 spin_lock(&si->lock); 893 latency_ration = LATENCY_LIMIT; 894 } 895 896 /* try to get more slots in cluster */ 897 if (si->cluster_info) { 898 if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) 899 goto checks; 900 } else if (si->cluster_nr && !si->swap_map[++offset]) { 901 /* non-ssd case, still more slots in cluster? */ 902 --si->cluster_nr; 903 goto checks; 904 } 905 906 /* 907 * Even if there's no free clusters available (fragmented), 908 * try to scan a little more quickly with lock held unless we 909 * have scanned too many slots already. 910 */ 911 if (!scanned_many) { 912 unsigned long scan_limit; 913 914 if (offset < scan_base) 915 scan_limit = scan_base; 916 else 917 scan_limit = si->highest_bit; 918 for (; offset <= scan_limit && --latency_ration > 0; 919 offset++) { 920 if (!si->swap_map[offset]) 921 goto checks; 922 } 923 } 924 925 done: 926 set_cluster_next(si, offset + 1); 927 si->flags -= SWP_SCANNING; 928 return n_ret; 929 930 scan: 931 spin_unlock(&si->lock); 932 while (++offset <= READ_ONCE(si->highest_bit)) { 933 if (data_race(!si->swap_map[offset])) { 934 spin_lock(&si->lock); 935 goto checks; 936 } 937 if (vm_swap_full() && 938 READ_ONCE(si->swap_map[offset]) == SWAP_HAS_CACHE) { 939 spin_lock(&si->lock); 940 goto checks; 941 } 942 if (unlikely(--latency_ration < 0)) { 943 cond_resched(); 944 latency_ration = LATENCY_LIMIT; 945 scanned_many = true; 946 } 947 } 948 offset = si->lowest_bit; 949 while (offset < scan_base) { 950 if (data_race(!si->swap_map[offset])) { 951 spin_lock(&si->lock); 952 goto checks; 953 } 954 if (vm_swap_full() && 955 READ_ONCE(si->swap_map[offset]) == SWAP_HAS_CACHE) { 956 spin_lock(&si->lock); 957 goto checks; 958 } 959 if (unlikely(--latency_ration < 0)) { 960 cond_resched(); 961 latency_ration = LATENCY_LIMIT; 962 scanned_many = true; 963 } 964 offset++; 965 } 966 spin_lock(&si->lock); 967 968 no_page: 969 si->flags -= SWP_SCANNING; 970 return n_ret; 971 } 972 973 static int swap_alloc_cluster(struct swap_info_struct *si, swp_entry_t *slot) 974 { 975 unsigned long idx; 976 struct swap_cluster_info *ci; 977 unsigned long offset; 978 979 /* 980 * Should not even be attempting cluster allocations when huge 981 * page swap is disabled. Warn and fail the allocation. 982 */ 983 if (!IS_ENABLED(CONFIG_THP_SWAP)) { 984 VM_WARN_ON_ONCE(1); 985 return 0; 986 } 987 988 if (cluster_list_empty(&si->free_clusters)) 989 return 0; 990 991 idx = cluster_list_first(&si->free_clusters); 992 offset = idx * SWAPFILE_CLUSTER; 993 ci = lock_cluster(si, offset); 994 alloc_cluster(si, idx); 995 cluster_set_count_flag(ci, SWAPFILE_CLUSTER, CLUSTER_FLAG_HUGE); 996 997 memset(si->swap_map + offset, SWAP_HAS_CACHE, SWAPFILE_CLUSTER); 998 unlock_cluster(ci); 999 swap_range_alloc(si, offset, SWAPFILE_CLUSTER); 1000 *slot = swp_entry(si->type, offset); 1001 1002 return 1; 1003 } 1004 1005 static void swap_free_cluster(struct swap_info_struct *si, unsigned long idx) 1006 { 1007 unsigned long offset = idx * SWAPFILE_CLUSTER; 1008 struct swap_cluster_info *ci; 1009 1010 ci = lock_cluster(si, offset); 1011 memset(si->swap_map + offset, 0, SWAPFILE_CLUSTER); 1012 cluster_set_count_flag(ci, 0, 0); 1013 free_cluster(si, idx); 1014 unlock_cluster(ci); 1015 swap_range_free(si, offset, SWAPFILE_CLUSTER); 1016 } 1017 1018 static unsigned long scan_swap_map(struct swap_info_struct *si, 1019 unsigned char usage) 1020 { 1021 swp_entry_t entry; 1022 int n_ret; 1023 1024 n_ret = scan_swap_map_slots(si, usage, 1, &entry); 1025 1026 if (n_ret) 1027 return swp_offset(entry); 1028 else 1029 return 0; 1030 1031 } 1032 1033 int get_swap_pages(int n_goal, swp_entry_t swp_entries[], int entry_size) 1034 { 1035 unsigned long size = swap_entry_size(entry_size); 1036 struct swap_info_struct *si, *next; 1037 long avail_pgs; 1038 int n_ret = 0; 1039 int node; 1040 1041 /* Only single cluster request supported */ 1042 WARN_ON_ONCE(n_goal > 1 && size == SWAPFILE_CLUSTER); 1043 1044 spin_lock(&swap_avail_lock); 1045 1046 avail_pgs = atomic_long_read(&nr_swap_pages) / size; 1047 if (avail_pgs <= 0) { 1048 spin_unlock(&swap_avail_lock); 1049 goto noswap; 1050 } 1051 1052 n_goal = min3((long)n_goal, (long)SWAP_BATCH, avail_pgs); 1053 1054 atomic_long_sub(n_goal * size, &nr_swap_pages); 1055 1056 start_over: 1057 node = numa_node_id(); 1058 plist_for_each_entry_safe(si, next, &swap_avail_heads[node], avail_lists[node]) { 1059 /* requeue si to after same-priority siblings */ 1060 plist_requeue(&si->avail_lists[node], &swap_avail_heads[node]); 1061 spin_unlock(&swap_avail_lock); 1062 spin_lock(&si->lock); 1063 if (!si->highest_bit || !(si->flags & SWP_WRITEOK)) { 1064 spin_lock(&swap_avail_lock); 1065 if (plist_node_empty(&si->avail_lists[node])) { 1066 spin_unlock(&si->lock); 1067 goto nextsi; 1068 } 1069 WARN(!si->highest_bit, 1070 "swap_info %d in list but !highest_bit\n", 1071 si->type); 1072 WARN(!(si->flags & SWP_WRITEOK), 1073 "swap_info %d in list but !SWP_WRITEOK\n", 1074 si->type); 1075 __del_from_avail_list(si); 1076 spin_unlock(&si->lock); 1077 goto nextsi; 1078 } 1079 if (size == SWAPFILE_CLUSTER) { 1080 if (si->flags & SWP_BLKDEV) 1081 n_ret = swap_alloc_cluster(si, swp_entries); 1082 } else 1083 n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE, 1084 n_goal, swp_entries); 1085 spin_unlock(&si->lock); 1086 if (n_ret || size == SWAPFILE_CLUSTER) 1087 goto check_out; 1088 pr_debug("scan_swap_map of si %d failed to find offset\n", 1089 si->type); 1090 1091 spin_lock(&swap_avail_lock); 1092 nextsi: 1093 /* 1094 * if we got here, it's likely that si was almost full before, 1095 * and since scan_swap_map() can drop the si->lock, multiple 1096 * callers probably all tried to get a page from the same si 1097 * and it filled up before we could get one; or, the si filled 1098 * up between us dropping swap_avail_lock and taking si->lock. 1099 * Since we dropped the swap_avail_lock, the swap_avail_head 1100 * list may have been modified; so if next is still in the 1101 * swap_avail_head list then try it, otherwise start over 1102 * if we have not gotten any slots. 1103 */ 1104 if (plist_node_empty(&next->avail_lists[node])) 1105 goto start_over; 1106 } 1107 1108 spin_unlock(&swap_avail_lock); 1109 1110 check_out: 1111 if (n_ret < n_goal) 1112 atomic_long_add((long)(n_goal - n_ret) * size, 1113 &nr_swap_pages); 1114 noswap: 1115 return n_ret; 1116 } 1117 1118 /* The only caller of this function is now suspend routine */ 1119 swp_entry_t get_swap_page_of_type(int type) 1120 { 1121 struct swap_info_struct *si = swap_type_to_swap_info(type); 1122 pgoff_t offset; 1123 1124 if (!si) 1125 goto fail; 1126 1127 spin_lock(&si->lock); 1128 if (si->flags & SWP_WRITEOK) { 1129 /* This is called for allocating swap entry, not cache */ 1130 offset = scan_swap_map(si, 1); 1131 if (offset) { 1132 atomic_long_dec(&nr_swap_pages); 1133 spin_unlock(&si->lock); 1134 return swp_entry(type, offset); 1135 } 1136 } 1137 spin_unlock(&si->lock); 1138 fail: 1139 return (swp_entry_t) {0}; 1140 } 1141 1142 static struct swap_info_struct *__swap_info_get(swp_entry_t entry) 1143 { 1144 struct swap_info_struct *p; 1145 unsigned long offset; 1146 1147 if (!entry.val) 1148 goto out; 1149 p = swp_swap_info(entry); 1150 if (!p) 1151 goto bad_nofile; 1152 if (data_race(!(p->flags & SWP_USED))) 1153 goto bad_device; 1154 offset = swp_offset(entry); 1155 if (offset >= p->max) 1156 goto bad_offset; 1157 return p; 1158 1159 bad_offset: 1160 pr_err("swap_info_get: %s%08lx\n", Bad_offset, entry.val); 1161 goto out; 1162 bad_device: 1163 pr_err("swap_info_get: %s%08lx\n", Unused_file, entry.val); 1164 goto out; 1165 bad_nofile: 1166 pr_err("swap_info_get: %s%08lx\n", Bad_file, entry.val); 1167 out: 1168 return NULL; 1169 } 1170 1171 static struct swap_info_struct *_swap_info_get(swp_entry_t entry) 1172 { 1173 struct swap_info_struct *p; 1174 1175 p = __swap_info_get(entry); 1176 if (!p) 1177 goto out; 1178 if (data_race(!p->swap_map[swp_offset(entry)])) 1179 goto bad_free; 1180 return p; 1181 1182 bad_free: 1183 pr_err("swap_info_get: %s%08lx\n", Unused_offset, entry.val); 1184 out: 1185 return NULL; 1186 } 1187 1188 static struct swap_info_struct *swap_info_get(swp_entry_t entry) 1189 { 1190 struct swap_info_struct *p; 1191 1192 p = _swap_info_get(entry); 1193 if (p) 1194 spin_lock(&p->lock); 1195 return p; 1196 } 1197 1198 static struct swap_info_struct *swap_info_get_cont(swp_entry_t entry, 1199 struct swap_info_struct *q) 1200 { 1201 struct swap_info_struct *p; 1202 1203 p = _swap_info_get(entry); 1204 1205 if (p != q) { 1206 if (q != NULL) 1207 spin_unlock(&q->lock); 1208 if (p != NULL) 1209 spin_lock(&p->lock); 1210 } 1211 return p; 1212 } 1213 1214 static unsigned char __swap_entry_free_locked(struct swap_info_struct *p, 1215 unsigned long offset, 1216 unsigned char usage) 1217 { 1218 unsigned char count; 1219 unsigned char has_cache; 1220 1221 count = p->swap_map[offset]; 1222 1223 has_cache = count & SWAP_HAS_CACHE; 1224 count &= ~SWAP_HAS_CACHE; 1225 1226 if (usage == SWAP_HAS_CACHE) { 1227 VM_BUG_ON(!has_cache); 1228 has_cache = 0; 1229 } else if (count == SWAP_MAP_SHMEM) { 1230 /* 1231 * Or we could insist on shmem.c using a special 1232 * swap_shmem_free() and free_shmem_swap_and_cache()... 1233 */ 1234 count = 0; 1235 } else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) { 1236 if (count == COUNT_CONTINUED) { 1237 if (swap_count_continued(p, offset, count)) 1238 count = SWAP_MAP_MAX | COUNT_CONTINUED; 1239 else 1240 count = SWAP_MAP_MAX; 1241 } else 1242 count--; 1243 } 1244 1245 usage = count | has_cache; 1246 if (usage) 1247 WRITE_ONCE(p->swap_map[offset], usage); 1248 else 1249 WRITE_ONCE(p->swap_map[offset], SWAP_HAS_CACHE); 1250 1251 return usage; 1252 } 1253 1254 /* 1255 * Check whether swap entry is valid in the swap device. If so, 1256 * return pointer to swap_info_struct, and keep the swap entry valid 1257 * via preventing the swap device from being swapoff, until 1258 * put_swap_device() is called. Otherwise return NULL. 1259 * 1260 * The entirety of the RCU read critical section must come before the 1261 * return from or after the call to synchronize_rcu() in 1262 * enable_swap_info() or swapoff(). So if "si->flags & SWP_VALID" is 1263 * true, the si->map, si->cluster_info, etc. must be valid in the 1264 * critical section. 1265 * 1266 * Notice that swapoff or swapoff+swapon can still happen before the 1267 * rcu_read_lock() in get_swap_device() or after the rcu_read_unlock() 1268 * in put_swap_device() if there isn't any other way to prevent 1269 * swapoff, such as page lock, page table lock, etc. The caller must 1270 * be prepared for that. For example, the following situation is 1271 * possible. 1272 * 1273 * CPU1 CPU2 1274 * do_swap_page() 1275 * ... swapoff+swapon 1276 * __read_swap_cache_async() 1277 * swapcache_prepare() 1278 * __swap_duplicate() 1279 * // check swap_map 1280 * // verify PTE not changed 1281 * 1282 * In __swap_duplicate(), the swap_map need to be checked before 1283 * changing partly because the specified swap entry may be for another 1284 * swap device which has been swapoff. And in do_swap_page(), after 1285 * the page is read from the swap device, the PTE is verified not 1286 * changed with the page table locked to check whether the swap device 1287 * has been swapoff or swapoff+swapon. 1288 */ 1289 struct swap_info_struct *get_swap_device(swp_entry_t entry) 1290 { 1291 struct swap_info_struct *si; 1292 unsigned long offset; 1293 1294 if (!entry.val) 1295 goto out; 1296 si = swp_swap_info(entry); 1297 if (!si) 1298 goto bad_nofile; 1299 1300 rcu_read_lock(); 1301 if (data_race(!(si->flags & SWP_VALID))) 1302 goto unlock_out; 1303 offset = swp_offset(entry); 1304 if (offset >= si->max) 1305 goto unlock_out; 1306 1307 return si; 1308 bad_nofile: 1309 pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val); 1310 out: 1311 return NULL; 1312 unlock_out: 1313 rcu_read_unlock(); 1314 return NULL; 1315 } 1316 1317 static unsigned char __swap_entry_free(struct swap_info_struct *p, 1318 swp_entry_t entry) 1319 { 1320 struct swap_cluster_info *ci; 1321 unsigned long offset = swp_offset(entry); 1322 unsigned char usage; 1323 1324 ci = lock_cluster_or_swap_info(p, offset); 1325 usage = __swap_entry_free_locked(p, offset, 1); 1326 unlock_cluster_or_swap_info(p, ci); 1327 if (!usage) 1328 free_swap_slot(entry); 1329 1330 return usage; 1331 } 1332 1333 static void swap_entry_free(struct swap_info_struct *p, swp_entry_t entry) 1334 { 1335 struct swap_cluster_info *ci; 1336 unsigned long offset = swp_offset(entry); 1337 unsigned char count; 1338 1339 ci = lock_cluster(p, offset); 1340 count = p->swap_map[offset]; 1341 VM_BUG_ON(count != SWAP_HAS_CACHE); 1342 p->swap_map[offset] = 0; 1343 dec_cluster_info_page(p, p->cluster_info, offset); 1344 unlock_cluster(ci); 1345 1346 mem_cgroup_uncharge_swap(entry, 1); 1347 swap_range_free(p, offset, 1); 1348 } 1349 1350 /* 1351 * Caller has made sure that the swap device corresponding to entry 1352 * is still around or has not been recycled. 1353 */ 1354 void swap_free(swp_entry_t entry) 1355 { 1356 struct swap_info_struct *p; 1357 1358 p = _swap_info_get(entry); 1359 if (p) 1360 __swap_entry_free(p, entry); 1361 } 1362 1363 /* 1364 * Called after dropping swapcache to decrease refcnt to swap entries. 1365 */ 1366 void put_swap_page(struct page *page, swp_entry_t entry) 1367 { 1368 unsigned long offset = swp_offset(entry); 1369 unsigned long idx = offset / SWAPFILE_CLUSTER; 1370 struct swap_cluster_info *ci; 1371 struct swap_info_struct *si; 1372 unsigned char *map; 1373 unsigned int i, free_entries = 0; 1374 unsigned char val; 1375 int size = swap_entry_size(thp_nr_pages(page)); 1376 1377 si = _swap_info_get(entry); 1378 if (!si) 1379 return; 1380 1381 ci = lock_cluster_or_swap_info(si, offset); 1382 if (size == SWAPFILE_CLUSTER) { 1383 VM_BUG_ON(!cluster_is_huge(ci)); 1384 map = si->swap_map + offset; 1385 for (i = 0; i < SWAPFILE_CLUSTER; i++) { 1386 val = map[i]; 1387 VM_BUG_ON(!(val & SWAP_HAS_CACHE)); 1388 if (val == SWAP_HAS_CACHE) 1389 free_entries++; 1390 } 1391 cluster_clear_huge(ci); 1392 if (free_entries == SWAPFILE_CLUSTER) { 1393 unlock_cluster_or_swap_info(si, ci); 1394 spin_lock(&si->lock); 1395 mem_cgroup_uncharge_swap(entry, SWAPFILE_CLUSTER); 1396 swap_free_cluster(si, idx); 1397 spin_unlock(&si->lock); 1398 return; 1399 } 1400 } 1401 for (i = 0; i < size; i++, entry.val++) { 1402 if (!__swap_entry_free_locked(si, offset + i, SWAP_HAS_CACHE)) { 1403 unlock_cluster_or_swap_info(si, ci); 1404 free_swap_slot(entry); 1405 if (i == size - 1) 1406 return; 1407 lock_cluster_or_swap_info(si, offset); 1408 } 1409 } 1410 unlock_cluster_or_swap_info(si, ci); 1411 } 1412 1413 #ifdef CONFIG_THP_SWAP 1414 int split_swap_cluster(swp_entry_t entry) 1415 { 1416 struct swap_info_struct *si; 1417 struct swap_cluster_info *ci; 1418 unsigned long offset = swp_offset(entry); 1419 1420 si = _swap_info_get(entry); 1421 if (!si) 1422 return -EBUSY; 1423 ci = lock_cluster(si, offset); 1424 cluster_clear_huge(ci); 1425 unlock_cluster(ci); 1426 return 0; 1427 } 1428 #endif 1429 1430 static int swp_entry_cmp(const void *ent1, const void *ent2) 1431 { 1432 const swp_entry_t *e1 = ent1, *e2 = ent2; 1433 1434 return (int)swp_type(*e1) - (int)swp_type(*e2); 1435 } 1436 1437 void swapcache_free_entries(swp_entry_t *entries, int n) 1438 { 1439 struct swap_info_struct *p, *prev; 1440 int i; 1441 1442 if (n <= 0) 1443 return; 1444 1445 prev = NULL; 1446 p = NULL; 1447 1448 /* 1449 * Sort swap entries by swap device, so each lock is only taken once. 1450 * nr_swapfiles isn't absolutely correct, but the overhead of sort() is 1451 * so low that it isn't necessary to optimize further. 1452 */ 1453 if (nr_swapfiles > 1) 1454 sort(entries, n, sizeof(entries[0]), swp_entry_cmp, NULL); 1455 for (i = 0; i < n; ++i) { 1456 p = swap_info_get_cont(entries[i], prev); 1457 if (p) 1458 swap_entry_free(p, entries[i]); 1459 prev = p; 1460 } 1461 if (p) 1462 spin_unlock(&p->lock); 1463 } 1464 1465 /* 1466 * How many references to page are currently swapped out? 1467 * This does not give an exact answer when swap count is continued, 1468 * but does include the high COUNT_CONTINUED flag to allow for that. 1469 */ 1470 int page_swapcount(struct page *page) 1471 { 1472 int count = 0; 1473 struct swap_info_struct *p; 1474 struct swap_cluster_info *ci; 1475 swp_entry_t entry; 1476 unsigned long offset; 1477 1478 entry.val = page_private(page); 1479 p = _swap_info_get(entry); 1480 if (p) { 1481 offset = swp_offset(entry); 1482 ci = lock_cluster_or_swap_info(p, offset); 1483 count = swap_count(p->swap_map[offset]); 1484 unlock_cluster_or_swap_info(p, ci); 1485 } 1486 return count; 1487 } 1488 1489 int __swap_count(swp_entry_t entry) 1490 { 1491 struct swap_info_struct *si; 1492 pgoff_t offset = swp_offset(entry); 1493 int count = 0; 1494 1495 si = get_swap_device(entry); 1496 if (si) { 1497 count = swap_count(si->swap_map[offset]); 1498 put_swap_device(si); 1499 } 1500 return count; 1501 } 1502 1503 static int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry) 1504 { 1505 int count = 0; 1506 pgoff_t offset = swp_offset(entry); 1507 struct swap_cluster_info *ci; 1508 1509 ci = lock_cluster_or_swap_info(si, offset); 1510 count = swap_count(si->swap_map[offset]); 1511 unlock_cluster_or_swap_info(si, ci); 1512 return count; 1513 } 1514 1515 /* 1516 * How many references to @entry are currently swapped out? 1517 * This does not give an exact answer when swap count is continued, 1518 * but does include the high COUNT_CONTINUED flag to allow for that. 1519 */ 1520 int __swp_swapcount(swp_entry_t entry) 1521 { 1522 int count = 0; 1523 struct swap_info_struct *si; 1524 1525 si = get_swap_device(entry); 1526 if (si) { 1527 count = swap_swapcount(si, entry); 1528 put_swap_device(si); 1529 } 1530 return count; 1531 } 1532 1533 /* 1534 * How many references to @entry are currently swapped out? 1535 * This considers COUNT_CONTINUED so it returns exact answer. 1536 */ 1537 int swp_swapcount(swp_entry_t entry) 1538 { 1539 int count, tmp_count, n; 1540 struct swap_info_struct *p; 1541 struct swap_cluster_info *ci; 1542 struct page *page; 1543 pgoff_t offset; 1544 unsigned char *map; 1545 1546 p = _swap_info_get(entry); 1547 if (!p) 1548 return 0; 1549 1550 offset = swp_offset(entry); 1551 1552 ci = lock_cluster_or_swap_info(p, offset); 1553 1554 count = swap_count(p->swap_map[offset]); 1555 if (!(count & COUNT_CONTINUED)) 1556 goto out; 1557 1558 count &= ~COUNT_CONTINUED; 1559 n = SWAP_MAP_MAX + 1; 1560 1561 page = vmalloc_to_page(p->swap_map + offset); 1562 offset &= ~PAGE_MASK; 1563 VM_BUG_ON(page_private(page) != SWP_CONTINUED); 1564 1565 do { 1566 page = list_next_entry(page, lru); 1567 map = kmap_atomic(page); 1568 tmp_count = map[offset]; 1569 kunmap_atomic(map); 1570 1571 count += (tmp_count & ~COUNT_CONTINUED) * n; 1572 n *= (SWAP_CONT_MAX + 1); 1573 } while (tmp_count & COUNT_CONTINUED); 1574 out: 1575 unlock_cluster_or_swap_info(p, ci); 1576 return count; 1577 } 1578 1579 static bool swap_page_trans_huge_swapped(struct swap_info_struct *si, 1580 swp_entry_t entry) 1581 { 1582 struct swap_cluster_info *ci; 1583 unsigned char *map = si->swap_map; 1584 unsigned long roffset = swp_offset(entry); 1585 unsigned long offset = round_down(roffset, SWAPFILE_CLUSTER); 1586 int i; 1587 bool ret = false; 1588 1589 ci = lock_cluster_or_swap_info(si, offset); 1590 if (!ci || !cluster_is_huge(ci)) { 1591 if (swap_count(map[roffset])) 1592 ret = true; 1593 goto unlock_out; 1594 } 1595 for (i = 0; i < SWAPFILE_CLUSTER; i++) { 1596 if (swap_count(map[offset + i])) { 1597 ret = true; 1598 break; 1599 } 1600 } 1601 unlock_out: 1602 unlock_cluster_or_swap_info(si, ci); 1603 return ret; 1604 } 1605 1606 static bool page_swapped(struct page *page) 1607 { 1608 swp_entry_t entry; 1609 struct swap_info_struct *si; 1610 1611 if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!PageTransCompound(page))) 1612 return page_swapcount(page) != 0; 1613 1614 page = compound_head(page); 1615 entry.val = page_private(page); 1616 si = _swap_info_get(entry); 1617 if (si) 1618 return swap_page_trans_huge_swapped(si, entry); 1619 return false; 1620 } 1621 1622 static int page_trans_huge_map_swapcount(struct page *page, int *total_mapcount, 1623 int *total_swapcount) 1624 { 1625 int i, map_swapcount, _total_mapcount, _total_swapcount; 1626 unsigned long offset = 0; 1627 struct swap_info_struct *si; 1628 struct swap_cluster_info *ci = NULL; 1629 unsigned char *map = NULL; 1630 int mapcount, swapcount = 0; 1631 1632 /* hugetlbfs shouldn't call it */ 1633 VM_BUG_ON_PAGE(PageHuge(page), page); 1634 1635 if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!PageTransCompound(page))) { 1636 mapcount = page_trans_huge_mapcount(page, total_mapcount); 1637 if (PageSwapCache(page)) 1638 swapcount = page_swapcount(page); 1639 if (total_swapcount) 1640 *total_swapcount = swapcount; 1641 return mapcount + swapcount; 1642 } 1643 1644 page = compound_head(page); 1645 1646 _total_mapcount = _total_swapcount = map_swapcount = 0; 1647 if (PageSwapCache(page)) { 1648 swp_entry_t entry; 1649 1650 entry.val = page_private(page); 1651 si = _swap_info_get(entry); 1652 if (si) { 1653 map = si->swap_map; 1654 offset = swp_offset(entry); 1655 } 1656 } 1657 if (map) 1658 ci = lock_cluster(si, offset); 1659 for (i = 0; i < HPAGE_PMD_NR; i++) { 1660 mapcount = atomic_read(&page[i]._mapcount) + 1; 1661 _total_mapcount += mapcount; 1662 if (map) { 1663 swapcount = swap_count(map[offset + i]); 1664 _total_swapcount += swapcount; 1665 } 1666 map_swapcount = max(map_swapcount, mapcount + swapcount); 1667 } 1668 unlock_cluster(ci); 1669 if (PageDoubleMap(page)) { 1670 map_swapcount -= 1; 1671 _total_mapcount -= HPAGE_PMD_NR; 1672 } 1673 mapcount = compound_mapcount(page); 1674 map_swapcount += mapcount; 1675 _total_mapcount += mapcount; 1676 if (total_mapcount) 1677 *total_mapcount = _total_mapcount; 1678 if (total_swapcount) 1679 *total_swapcount = _total_swapcount; 1680 1681 return map_swapcount; 1682 } 1683 1684 /* 1685 * We can write to an anon page without COW if there are no other references 1686 * to it. And as a side-effect, free up its swap: because the old content 1687 * on disk will never be read, and seeking back there to write new content 1688 * later would only waste time away from clustering. 1689 * 1690 * NOTE: total_map_swapcount should not be relied upon by the caller if 1691 * reuse_swap_page() returns false, but it may be always overwritten 1692 * (see the other implementation for CONFIG_SWAP=n). 1693 */ 1694 bool reuse_swap_page(struct page *page, int *total_map_swapcount) 1695 { 1696 int count, total_mapcount, total_swapcount; 1697 1698 VM_BUG_ON_PAGE(!PageLocked(page), page); 1699 if (unlikely(PageKsm(page))) 1700 return false; 1701 count = page_trans_huge_map_swapcount(page, &total_mapcount, 1702 &total_swapcount); 1703 if (total_map_swapcount) 1704 *total_map_swapcount = total_mapcount + total_swapcount; 1705 if (count == 1 && PageSwapCache(page) && 1706 (likely(!PageTransCompound(page)) || 1707 /* The remaining swap count will be freed soon */ 1708 total_swapcount == page_swapcount(page))) { 1709 if (!PageWriteback(page)) { 1710 page = compound_head(page); 1711 delete_from_swap_cache(page); 1712 SetPageDirty(page); 1713 } else { 1714 swp_entry_t entry; 1715 struct swap_info_struct *p; 1716 1717 entry.val = page_private(page); 1718 p = swap_info_get(entry); 1719 if (p->flags & SWP_STABLE_WRITES) { 1720 spin_unlock(&p->lock); 1721 return false; 1722 } 1723 spin_unlock(&p->lock); 1724 } 1725 } 1726 1727 return count <= 1; 1728 } 1729 1730 /* 1731 * If swap is getting full, or if there are no more mappings of this page, 1732 * then try_to_free_swap is called to free its swap space. 1733 */ 1734 int try_to_free_swap(struct page *page) 1735 { 1736 VM_BUG_ON_PAGE(!PageLocked(page), page); 1737 1738 if (!PageSwapCache(page)) 1739 return 0; 1740 if (PageWriteback(page)) 1741 return 0; 1742 if (page_swapped(page)) 1743 return 0; 1744 1745 /* 1746 * Once hibernation has begun to create its image of memory, 1747 * there's a danger that one of the calls to try_to_free_swap() 1748 * - most probably a call from __try_to_reclaim_swap() while 1749 * hibernation is allocating its own swap pages for the image, 1750 * but conceivably even a call from memory reclaim - will free 1751 * the swap from a page which has already been recorded in the 1752 * image as a clean swapcache page, and then reuse its swap for 1753 * another page of the image. On waking from hibernation, the 1754 * original page might be freed under memory pressure, then 1755 * later read back in from swap, now with the wrong data. 1756 * 1757 * Hibernation suspends storage while it is writing the image 1758 * to disk so check that here. 1759 */ 1760 if (pm_suspended_storage()) 1761 return 0; 1762 1763 page = compound_head(page); 1764 delete_from_swap_cache(page); 1765 SetPageDirty(page); 1766 return 1; 1767 } 1768 1769 /* 1770 * Free the swap entry like above, but also try to 1771 * free the page cache entry if it is the last user. 1772 */ 1773 int free_swap_and_cache(swp_entry_t entry) 1774 { 1775 struct swap_info_struct *p; 1776 unsigned char count; 1777 1778 if (non_swap_entry(entry)) 1779 return 1; 1780 1781 p = _swap_info_get(entry); 1782 if (p) { 1783 count = __swap_entry_free(p, entry); 1784 if (count == SWAP_HAS_CACHE && 1785 !swap_page_trans_huge_swapped(p, entry)) 1786 __try_to_reclaim_swap(p, swp_offset(entry), 1787 TTRS_UNMAPPED | TTRS_FULL); 1788 } 1789 return p != NULL; 1790 } 1791 1792 #ifdef CONFIG_HIBERNATION 1793 /* 1794 * Find the swap type that corresponds to given device (if any). 1795 * 1796 * @offset - number of the PAGE_SIZE-sized block of the device, starting 1797 * from 0, in which the swap header is expected to be located. 1798 * 1799 * This is needed for the suspend to disk (aka swsusp). 1800 */ 1801 int swap_type_of(dev_t device, sector_t offset) 1802 { 1803 int type; 1804 1805 if (!device) 1806 return -1; 1807 1808 spin_lock(&swap_lock); 1809 for (type = 0; type < nr_swapfiles; type++) { 1810 struct swap_info_struct *sis = swap_info[type]; 1811 1812 if (!(sis->flags & SWP_WRITEOK)) 1813 continue; 1814 1815 if (device == sis->bdev->bd_dev) { 1816 struct swap_extent *se = first_se(sis); 1817 1818 if (se->start_block == offset) { 1819 spin_unlock(&swap_lock); 1820 return type; 1821 } 1822 } 1823 } 1824 spin_unlock(&swap_lock); 1825 return -ENODEV; 1826 } 1827 1828 int find_first_swap(dev_t *device) 1829 { 1830 int type; 1831 1832 spin_lock(&swap_lock); 1833 for (type = 0; type < nr_swapfiles; type++) { 1834 struct swap_info_struct *sis = swap_info[type]; 1835 1836 if (!(sis->flags & SWP_WRITEOK)) 1837 continue; 1838 *device = sis->bdev->bd_dev; 1839 spin_unlock(&swap_lock); 1840 return type; 1841 } 1842 spin_unlock(&swap_lock); 1843 return -ENODEV; 1844 } 1845 1846 /* 1847 * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev 1848 * corresponding to given index in swap_info (swap type). 1849 */ 1850 sector_t swapdev_block(int type, pgoff_t offset) 1851 { 1852 struct swap_info_struct *si = swap_type_to_swap_info(type); 1853 struct swap_extent *se; 1854 1855 if (!si || !(si->flags & SWP_WRITEOK)) 1856 return 0; 1857 se = offset_to_swap_extent(si, offset); 1858 return se->start_block + (offset - se->start_page); 1859 } 1860 1861 /* 1862 * Return either the total number of swap pages of given type, or the number 1863 * of free pages of that type (depending on @free) 1864 * 1865 * This is needed for software suspend 1866 */ 1867 unsigned int count_swap_pages(int type, int free) 1868 { 1869 unsigned int n = 0; 1870 1871 spin_lock(&swap_lock); 1872 if ((unsigned int)type < nr_swapfiles) { 1873 struct swap_info_struct *sis = swap_info[type]; 1874 1875 spin_lock(&sis->lock); 1876 if (sis->flags & SWP_WRITEOK) { 1877 n = sis->pages; 1878 if (free) 1879 n -= sis->inuse_pages; 1880 } 1881 spin_unlock(&sis->lock); 1882 } 1883 spin_unlock(&swap_lock); 1884 return n; 1885 } 1886 #endif /* CONFIG_HIBERNATION */ 1887 1888 static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte) 1889 { 1890 return pte_same(pte_swp_clear_soft_dirty(pte), swp_pte); 1891 } 1892 1893 /* 1894 * No need to decide whether this PTE shares the swap entry with others, 1895 * just let do_wp_page work it out if a write is requested later - to 1896 * force COW, vm_page_prot omits write permission from any private vma. 1897 */ 1898 static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, 1899 unsigned long addr, swp_entry_t entry, struct page *page) 1900 { 1901 struct page *swapcache; 1902 spinlock_t *ptl; 1903 pte_t *pte; 1904 int ret = 1; 1905 1906 swapcache = page; 1907 page = ksm_might_need_to_copy(page, vma, addr); 1908 if (unlikely(!page)) 1909 return -ENOMEM; 1910 1911 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 1912 if (unlikely(!pte_same_as_swp(*pte, swp_entry_to_pte(entry)))) { 1913 ret = 0; 1914 goto out; 1915 } 1916 1917 dec_mm_counter(vma->vm_mm, MM_SWAPENTS); 1918 inc_mm_counter(vma->vm_mm, MM_ANONPAGES); 1919 get_page(page); 1920 set_pte_at(vma->vm_mm, addr, pte, 1921 pte_mkold(mk_pte(page, vma->vm_page_prot))); 1922 if (page == swapcache) { 1923 page_add_anon_rmap(page, vma, addr, false); 1924 } else { /* ksm created a completely new copy */ 1925 page_add_new_anon_rmap(page, vma, addr, false); 1926 lru_cache_add_inactive_or_unevictable(page, vma); 1927 } 1928 swap_free(entry); 1929 out: 1930 pte_unmap_unlock(pte, ptl); 1931 if (page != swapcache) { 1932 unlock_page(page); 1933 put_page(page); 1934 } 1935 return ret; 1936 } 1937 1938 static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 1939 unsigned long addr, unsigned long end, 1940 unsigned int type, bool frontswap, 1941 unsigned long *fs_pages_to_unuse) 1942 { 1943 struct page *page; 1944 swp_entry_t entry; 1945 pte_t *pte; 1946 struct swap_info_struct *si; 1947 unsigned long offset; 1948 int ret = 0; 1949 volatile unsigned char *swap_map; 1950 1951 si = swap_info[type]; 1952 pte = pte_offset_map(pmd, addr); 1953 do { 1954 if (!is_swap_pte(*pte)) 1955 continue; 1956 1957 entry = pte_to_swp_entry(*pte); 1958 if (swp_type(entry) != type) 1959 continue; 1960 1961 offset = swp_offset(entry); 1962 if (frontswap && !frontswap_test(si, offset)) 1963 continue; 1964 1965 pte_unmap(pte); 1966 swap_map = &si->swap_map[offset]; 1967 page = lookup_swap_cache(entry, vma, addr); 1968 if (!page) { 1969 struct vm_fault vmf = { 1970 .vma = vma, 1971 .address = addr, 1972 .pmd = pmd, 1973 }; 1974 1975 page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, 1976 &vmf); 1977 } 1978 if (!page) { 1979 if (*swap_map == 0 || *swap_map == SWAP_MAP_BAD) 1980 goto try_next; 1981 return -ENOMEM; 1982 } 1983 1984 lock_page(page); 1985 wait_on_page_writeback(page); 1986 ret = unuse_pte(vma, pmd, addr, entry, page); 1987 if (ret < 0) { 1988 unlock_page(page); 1989 put_page(page); 1990 goto out; 1991 } 1992 1993 try_to_free_swap(page); 1994 unlock_page(page); 1995 put_page(page); 1996 1997 if (*fs_pages_to_unuse && !--(*fs_pages_to_unuse)) { 1998 ret = FRONTSWAP_PAGES_UNUSED; 1999 goto out; 2000 } 2001 try_next: 2002 pte = pte_offset_map(pmd, addr); 2003 } while (pte++, addr += PAGE_SIZE, addr != end); 2004 pte_unmap(pte - 1); 2005 2006 ret = 0; 2007 out: 2008 return ret; 2009 } 2010 2011 static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, 2012 unsigned long addr, unsigned long end, 2013 unsigned int type, bool frontswap, 2014 unsigned long *fs_pages_to_unuse) 2015 { 2016 pmd_t *pmd; 2017 unsigned long next; 2018 int ret; 2019 2020 pmd = pmd_offset(pud, addr); 2021 do { 2022 cond_resched(); 2023 next = pmd_addr_end(addr, end); 2024 if (pmd_none_or_trans_huge_or_clear_bad(pmd)) 2025 continue; 2026 ret = unuse_pte_range(vma, pmd, addr, next, type, 2027 frontswap, fs_pages_to_unuse); 2028 if (ret) 2029 return ret; 2030 } while (pmd++, addr = next, addr != end); 2031 return 0; 2032 } 2033 2034 static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, 2035 unsigned long addr, unsigned long end, 2036 unsigned int type, bool frontswap, 2037 unsigned long *fs_pages_to_unuse) 2038 { 2039 pud_t *pud; 2040 unsigned long next; 2041 int ret; 2042 2043 pud = pud_offset(p4d, addr); 2044 do { 2045 next = pud_addr_end(addr, end); 2046 if (pud_none_or_clear_bad(pud)) 2047 continue; 2048 ret = unuse_pmd_range(vma, pud, addr, next, type, 2049 frontswap, fs_pages_to_unuse); 2050 if (ret) 2051 return ret; 2052 } while (pud++, addr = next, addr != end); 2053 return 0; 2054 } 2055 2056 static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, 2057 unsigned long addr, unsigned long end, 2058 unsigned int type, bool frontswap, 2059 unsigned long *fs_pages_to_unuse) 2060 { 2061 p4d_t *p4d; 2062 unsigned long next; 2063 int ret; 2064 2065 p4d = p4d_offset(pgd, addr); 2066 do { 2067 next = p4d_addr_end(addr, end); 2068 if (p4d_none_or_clear_bad(p4d)) 2069 continue; 2070 ret = unuse_pud_range(vma, p4d, addr, next, type, 2071 frontswap, fs_pages_to_unuse); 2072 if (ret) 2073 return ret; 2074 } while (p4d++, addr = next, addr != end); 2075 return 0; 2076 } 2077 2078 static int unuse_vma(struct vm_area_struct *vma, unsigned int type, 2079 bool frontswap, unsigned long *fs_pages_to_unuse) 2080 { 2081 pgd_t *pgd; 2082 unsigned long addr, end, next; 2083 int ret; 2084 2085 addr = vma->vm_start; 2086 end = vma->vm_end; 2087 2088 pgd = pgd_offset(vma->vm_mm, addr); 2089 do { 2090 next = pgd_addr_end(addr, end); 2091 if (pgd_none_or_clear_bad(pgd)) 2092 continue; 2093 ret = unuse_p4d_range(vma, pgd, addr, next, type, 2094 frontswap, fs_pages_to_unuse); 2095 if (ret) 2096 return ret; 2097 } while (pgd++, addr = next, addr != end); 2098 return 0; 2099 } 2100 2101 static int unuse_mm(struct mm_struct *mm, unsigned int type, 2102 bool frontswap, unsigned long *fs_pages_to_unuse) 2103 { 2104 struct vm_area_struct *vma; 2105 int ret = 0; 2106 2107 mmap_read_lock(mm); 2108 for (vma = mm->mmap; vma; vma = vma->vm_next) { 2109 if (vma->anon_vma) { 2110 ret = unuse_vma(vma, type, frontswap, 2111 fs_pages_to_unuse); 2112 if (ret) 2113 break; 2114 } 2115 cond_resched(); 2116 } 2117 mmap_read_unlock(mm); 2118 return ret; 2119 } 2120 2121 /* 2122 * Scan swap_map (or frontswap_map if frontswap parameter is true) 2123 * from current position to next entry still in use. Return 0 2124 * if there are no inuse entries after prev till end of the map. 2125 */ 2126 static unsigned int find_next_to_unuse(struct swap_info_struct *si, 2127 unsigned int prev, bool frontswap) 2128 { 2129 unsigned int i; 2130 unsigned char count; 2131 2132 /* 2133 * No need for swap_lock here: we're just looking 2134 * for whether an entry is in use, not modifying it; false 2135 * hits are okay, and sys_swapoff() has already prevented new 2136 * allocations from this area (while holding swap_lock). 2137 */ 2138 for (i = prev + 1; i < si->max; i++) { 2139 count = READ_ONCE(si->swap_map[i]); 2140 if (count && swap_count(count) != SWAP_MAP_BAD) 2141 if (!frontswap || frontswap_test(si, i)) 2142 break; 2143 if ((i % LATENCY_LIMIT) == 0) 2144 cond_resched(); 2145 } 2146 2147 if (i == si->max) 2148 i = 0; 2149 2150 return i; 2151 } 2152 2153 /* 2154 * If the boolean frontswap is true, only unuse pages_to_unuse pages; 2155 * pages_to_unuse==0 means all pages; ignored if frontswap is false 2156 */ 2157 int try_to_unuse(unsigned int type, bool frontswap, 2158 unsigned long pages_to_unuse) 2159 { 2160 struct mm_struct *prev_mm; 2161 struct mm_struct *mm; 2162 struct list_head *p; 2163 int retval = 0; 2164 struct swap_info_struct *si = swap_info[type]; 2165 struct page *page; 2166 swp_entry_t entry; 2167 unsigned int i; 2168 2169 if (!READ_ONCE(si->inuse_pages)) 2170 return 0; 2171 2172 if (!frontswap) 2173 pages_to_unuse = 0; 2174 2175 retry: 2176 retval = shmem_unuse(type, frontswap, &pages_to_unuse); 2177 if (retval) 2178 goto out; 2179 2180 prev_mm = &init_mm; 2181 mmget(prev_mm); 2182 2183 spin_lock(&mmlist_lock); 2184 p = &init_mm.mmlist; 2185 while (READ_ONCE(si->inuse_pages) && 2186 !signal_pending(current) && 2187 (p = p->next) != &init_mm.mmlist) { 2188 2189 mm = list_entry(p, struct mm_struct, mmlist); 2190 if (!mmget_not_zero(mm)) 2191 continue; 2192 spin_unlock(&mmlist_lock); 2193 mmput(prev_mm); 2194 prev_mm = mm; 2195 retval = unuse_mm(mm, type, frontswap, &pages_to_unuse); 2196 2197 if (retval) { 2198 mmput(prev_mm); 2199 goto out; 2200 } 2201 2202 /* 2203 * Make sure that we aren't completely killing 2204 * interactive performance. 2205 */ 2206 cond_resched(); 2207 spin_lock(&mmlist_lock); 2208 } 2209 spin_unlock(&mmlist_lock); 2210 2211 mmput(prev_mm); 2212 2213 i = 0; 2214 while (READ_ONCE(si->inuse_pages) && 2215 !signal_pending(current) && 2216 (i = find_next_to_unuse(si, i, frontswap)) != 0) { 2217 2218 entry = swp_entry(type, i); 2219 page = find_get_page(swap_address_space(entry), i); 2220 if (!page) 2221 continue; 2222 2223 /* 2224 * It is conceivable that a racing task removed this page from 2225 * swap cache just before we acquired the page lock. The page 2226 * might even be back in swap cache on another swap area. But 2227 * that is okay, try_to_free_swap() only removes stale pages. 2228 */ 2229 lock_page(page); 2230 wait_on_page_writeback(page); 2231 try_to_free_swap(page); 2232 unlock_page(page); 2233 put_page(page); 2234 2235 /* 2236 * For frontswap, we just need to unuse pages_to_unuse, if 2237 * it was specified. Need not check frontswap again here as 2238 * we already zeroed out pages_to_unuse if not frontswap. 2239 */ 2240 if (pages_to_unuse && --pages_to_unuse == 0) 2241 goto out; 2242 } 2243 2244 /* 2245 * Lets check again to see if there are still swap entries in the map. 2246 * If yes, we would need to do retry the unuse logic again. 2247 * Under global memory pressure, swap entries can be reinserted back 2248 * into process space after the mmlist loop above passes over them. 2249 * 2250 * Limit the number of retries? No: when mmget_not_zero() above fails, 2251 * that mm is likely to be freeing swap from exit_mmap(), which proceeds 2252 * at its own independent pace; and even shmem_writepage() could have 2253 * been preempted after get_swap_page(), temporarily hiding that swap. 2254 * It's easy and robust (though cpu-intensive) just to keep retrying. 2255 */ 2256 if (READ_ONCE(si->inuse_pages)) { 2257 if (!signal_pending(current)) 2258 goto retry; 2259 retval = -EINTR; 2260 } 2261 out: 2262 return (retval == FRONTSWAP_PAGES_UNUSED) ? 0 : retval; 2263 } 2264 2265 /* 2266 * After a successful try_to_unuse, if no swap is now in use, we know 2267 * we can empty the mmlist. swap_lock must be held on entry and exit. 2268 * Note that mmlist_lock nests inside swap_lock, and an mm must be 2269 * added to the mmlist just after page_duplicate - before would be racy. 2270 */ 2271 static void drain_mmlist(void) 2272 { 2273 struct list_head *p, *next; 2274 unsigned int type; 2275 2276 for (type = 0; type < nr_swapfiles; type++) 2277 if (swap_info[type]->inuse_pages) 2278 return; 2279 spin_lock(&mmlist_lock); 2280 list_for_each_safe(p, next, &init_mm.mmlist) 2281 list_del_init(p); 2282 spin_unlock(&mmlist_lock); 2283 } 2284 2285 /* 2286 * Free all of a swapdev's extent information 2287 */ 2288 static void destroy_swap_extents(struct swap_info_struct *sis) 2289 { 2290 while (!RB_EMPTY_ROOT(&sis->swap_extent_root)) { 2291 struct rb_node *rb = sis->swap_extent_root.rb_node; 2292 struct swap_extent *se = rb_entry(rb, struct swap_extent, rb_node); 2293 2294 rb_erase(rb, &sis->swap_extent_root); 2295 kfree(se); 2296 } 2297 2298 if (sis->flags & SWP_ACTIVATED) { 2299 struct file *swap_file = sis->swap_file; 2300 struct address_space *mapping = swap_file->f_mapping; 2301 2302 sis->flags &= ~SWP_ACTIVATED; 2303 if (mapping->a_ops->swap_deactivate) 2304 mapping->a_ops->swap_deactivate(swap_file); 2305 } 2306 } 2307 2308 /* 2309 * Add a block range (and the corresponding page range) into this swapdev's 2310 * extent tree. 2311 * 2312 * This function rather assumes that it is called in ascending page order. 2313 */ 2314 int 2315 add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, 2316 unsigned long nr_pages, sector_t start_block) 2317 { 2318 struct rb_node **link = &sis->swap_extent_root.rb_node, *parent = NULL; 2319 struct swap_extent *se; 2320 struct swap_extent *new_se; 2321 2322 /* 2323 * place the new node at the right most since the 2324 * function is called in ascending page order. 2325 */ 2326 while (*link) { 2327 parent = *link; 2328 link = &parent->rb_right; 2329 } 2330 2331 if (parent) { 2332 se = rb_entry(parent, struct swap_extent, rb_node); 2333 BUG_ON(se->start_page + se->nr_pages != start_page); 2334 if (se->start_block + se->nr_pages == start_block) { 2335 /* Merge it */ 2336 se->nr_pages += nr_pages; 2337 return 0; 2338 } 2339 } 2340 2341 /* No merge, insert a new extent. */ 2342 new_se = kmalloc(sizeof(*se), GFP_KERNEL); 2343 if (new_se == NULL) 2344 return -ENOMEM; 2345 new_se->start_page = start_page; 2346 new_se->nr_pages = nr_pages; 2347 new_se->start_block = start_block; 2348 2349 rb_link_node(&new_se->rb_node, parent, link); 2350 rb_insert_color(&new_se->rb_node, &sis->swap_extent_root); 2351 return 1; 2352 } 2353 EXPORT_SYMBOL_GPL(add_swap_extent); 2354 2355 /* 2356 * A `swap extent' is a simple thing which maps a contiguous range of pages 2357 * onto a contiguous range of disk blocks. An ordered list of swap extents 2358 * is built at swapon time and is then used at swap_writepage/swap_readpage 2359 * time for locating where on disk a page belongs. 2360 * 2361 * If the swapfile is an S_ISBLK block device, a single extent is installed. 2362 * This is done so that the main operating code can treat S_ISBLK and S_ISREG 2363 * swap files identically. 2364 * 2365 * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap 2366 * extent list operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK 2367 * swapfiles are handled *identically* after swapon time. 2368 * 2369 * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks 2370 * and will parse them into an ordered extent list, in PAGE_SIZE chunks. If 2371 * some stray blocks are found which do not fall within the PAGE_SIZE alignment 2372 * requirements, they are simply tossed out - we will never use those blocks 2373 * for swapping. 2374 * 2375 * For all swap devices we set S_SWAPFILE across the life of the swapon. This 2376 * prevents users from writing to the swap device, which will corrupt memory. 2377 * 2378 * The amount of disk space which a single swap extent represents varies. 2379 * Typically it is in the 1-4 megabyte range. So we can have hundreds of 2380 * extents in the list. To avoid much list walking, we cache the previous 2381 * search location in `curr_swap_extent', and start new searches from there. 2382 * This is extremely effective. The average number of iterations in 2383 * map_swap_page() has been measured at about 0.3 per page. - akpm. 2384 */ 2385 static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span) 2386 { 2387 struct file *swap_file = sis->swap_file; 2388 struct address_space *mapping = swap_file->f_mapping; 2389 struct inode *inode = mapping->host; 2390 int ret; 2391 2392 if (S_ISBLK(inode->i_mode)) { 2393 ret = add_swap_extent(sis, 0, sis->max, 0); 2394 *span = sis->pages; 2395 return ret; 2396 } 2397 2398 if (mapping->a_ops->swap_activate) { 2399 ret = mapping->a_ops->swap_activate(sis, swap_file, span); 2400 if (ret >= 0) 2401 sis->flags |= SWP_ACTIVATED; 2402 if (!ret) { 2403 sis->flags |= SWP_FS_OPS; 2404 ret = add_swap_extent(sis, 0, sis->max, 0); 2405 *span = sis->pages; 2406 } 2407 return ret; 2408 } 2409 2410 return generic_swapfile_activate(sis, swap_file, span); 2411 } 2412 2413 static int swap_node(struct swap_info_struct *p) 2414 { 2415 struct block_device *bdev; 2416 2417 if (p->bdev) 2418 bdev = p->bdev; 2419 else 2420 bdev = p->swap_file->f_inode->i_sb->s_bdev; 2421 2422 return bdev ? bdev->bd_disk->node_id : NUMA_NO_NODE; 2423 } 2424 2425 static void setup_swap_info(struct swap_info_struct *p, int prio, 2426 unsigned char *swap_map, 2427 struct swap_cluster_info *cluster_info) 2428 { 2429 int i; 2430 2431 if (prio >= 0) 2432 p->prio = prio; 2433 else 2434 p->prio = --least_priority; 2435 /* 2436 * the plist prio is negated because plist ordering is 2437 * low-to-high, while swap ordering is high-to-low 2438 */ 2439 p->list.prio = -p->prio; 2440 for_each_node(i) { 2441 if (p->prio >= 0) 2442 p->avail_lists[i].prio = -p->prio; 2443 else { 2444 if (swap_node(p) == i) 2445 p->avail_lists[i].prio = 1; 2446 else 2447 p->avail_lists[i].prio = -p->prio; 2448 } 2449 } 2450 p->swap_map = swap_map; 2451 p->cluster_info = cluster_info; 2452 } 2453 2454 static void _enable_swap_info(struct swap_info_struct *p) 2455 { 2456 p->flags |= SWP_WRITEOK | SWP_VALID; 2457 atomic_long_add(p->pages, &nr_swap_pages); 2458 total_swap_pages += p->pages; 2459 2460 assert_spin_locked(&swap_lock); 2461 /* 2462 * both lists are plists, and thus priority ordered. 2463 * swap_active_head needs to be priority ordered for swapoff(), 2464 * which on removal of any swap_info_struct with an auto-assigned 2465 * (i.e. negative) priority increments the auto-assigned priority 2466 * of any lower-priority swap_info_structs. 2467 * swap_avail_head needs to be priority ordered for get_swap_page(), 2468 * which allocates swap pages from the highest available priority 2469 * swap_info_struct. 2470 */ 2471 plist_add(&p->list, &swap_active_head); 2472 add_to_avail_list(p); 2473 } 2474 2475 static void enable_swap_info(struct swap_info_struct *p, int prio, 2476 unsigned char *swap_map, 2477 struct swap_cluster_info *cluster_info, 2478 unsigned long *frontswap_map) 2479 { 2480 frontswap_init(p->type, frontswap_map); 2481 spin_lock(&swap_lock); 2482 spin_lock(&p->lock); 2483 setup_swap_info(p, prio, swap_map, cluster_info); 2484 spin_unlock(&p->lock); 2485 spin_unlock(&swap_lock); 2486 /* 2487 * Guarantee swap_map, cluster_info, etc. fields are valid 2488 * between get/put_swap_device() if SWP_VALID bit is set 2489 */ 2490 synchronize_rcu(); 2491 spin_lock(&swap_lock); 2492 spin_lock(&p->lock); 2493 _enable_swap_info(p); 2494 spin_unlock(&p->lock); 2495 spin_unlock(&swap_lock); 2496 } 2497 2498 static void reinsert_swap_info(struct swap_info_struct *p) 2499 { 2500 spin_lock(&swap_lock); 2501 spin_lock(&p->lock); 2502 setup_swap_info(p, p->prio, p->swap_map, p->cluster_info); 2503 _enable_swap_info(p); 2504 spin_unlock(&p->lock); 2505 spin_unlock(&swap_lock); 2506 } 2507 2508 bool has_usable_swap(void) 2509 { 2510 bool ret = true; 2511 2512 spin_lock(&swap_lock); 2513 if (plist_head_empty(&swap_active_head)) 2514 ret = false; 2515 spin_unlock(&swap_lock); 2516 return ret; 2517 } 2518 2519 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) 2520 { 2521 struct swap_info_struct *p = NULL; 2522 unsigned char *swap_map; 2523 struct swap_cluster_info *cluster_info; 2524 unsigned long *frontswap_map; 2525 struct file *swap_file, *victim; 2526 struct address_space *mapping; 2527 struct inode *inode; 2528 struct filename *pathname; 2529 int err, found = 0; 2530 unsigned int old_block_size; 2531 2532 if (!capable(CAP_SYS_ADMIN)) 2533 return -EPERM; 2534 2535 BUG_ON(!current->mm); 2536 2537 pathname = getname(specialfile); 2538 if (IS_ERR(pathname)) 2539 return PTR_ERR(pathname); 2540 2541 victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0); 2542 err = PTR_ERR(victim); 2543 if (IS_ERR(victim)) 2544 goto out; 2545 2546 mapping = victim->f_mapping; 2547 spin_lock(&swap_lock); 2548 plist_for_each_entry(p, &swap_active_head, list) { 2549 if (p->flags & SWP_WRITEOK) { 2550 if (p->swap_file->f_mapping == mapping) { 2551 found = 1; 2552 break; 2553 } 2554 } 2555 } 2556 if (!found) { 2557 err = -EINVAL; 2558 spin_unlock(&swap_lock); 2559 goto out_dput; 2560 } 2561 if (!security_vm_enough_memory_mm(current->mm, p->pages)) 2562 vm_unacct_memory(p->pages); 2563 else { 2564 err = -ENOMEM; 2565 spin_unlock(&swap_lock); 2566 goto out_dput; 2567 } 2568 del_from_avail_list(p); 2569 spin_lock(&p->lock); 2570 if (p->prio < 0) { 2571 struct swap_info_struct *si = p; 2572 int nid; 2573 2574 plist_for_each_entry_continue(si, &swap_active_head, list) { 2575 si->prio++; 2576 si->list.prio--; 2577 for_each_node(nid) { 2578 if (si->avail_lists[nid].prio != 1) 2579 si->avail_lists[nid].prio--; 2580 } 2581 } 2582 least_priority++; 2583 } 2584 plist_del(&p->list, &swap_active_head); 2585 atomic_long_sub(p->pages, &nr_swap_pages); 2586 total_swap_pages -= p->pages; 2587 p->flags &= ~SWP_WRITEOK; 2588 spin_unlock(&p->lock); 2589 spin_unlock(&swap_lock); 2590 2591 disable_swap_slots_cache_lock(); 2592 2593 set_current_oom_origin(); 2594 err = try_to_unuse(p->type, false, 0); /* force unuse all pages */ 2595 clear_current_oom_origin(); 2596 2597 if (err) { 2598 /* re-insert swap space back into swap_list */ 2599 reinsert_swap_info(p); 2600 reenable_swap_slots_cache_unlock(); 2601 goto out_dput; 2602 } 2603 2604 reenable_swap_slots_cache_unlock(); 2605 2606 spin_lock(&swap_lock); 2607 spin_lock(&p->lock); 2608 p->flags &= ~SWP_VALID; /* mark swap device as invalid */ 2609 spin_unlock(&p->lock); 2610 spin_unlock(&swap_lock); 2611 /* 2612 * wait for swap operations protected by get/put_swap_device() 2613 * to complete 2614 */ 2615 synchronize_rcu(); 2616 2617 flush_work(&p->discard_work); 2618 2619 destroy_swap_extents(p); 2620 if (p->flags & SWP_CONTINUED) 2621 free_swap_count_continuations(p); 2622 2623 if (!p->bdev || !blk_queue_nonrot(bdev_get_queue(p->bdev))) 2624 atomic_dec(&nr_rotate_swap); 2625 2626 mutex_lock(&swapon_mutex); 2627 spin_lock(&swap_lock); 2628 spin_lock(&p->lock); 2629 drain_mmlist(); 2630 2631 /* wait for anyone still in scan_swap_map */ 2632 p->highest_bit = 0; /* cuts scans short */ 2633 while (p->flags >= SWP_SCANNING) { 2634 spin_unlock(&p->lock); 2635 spin_unlock(&swap_lock); 2636 schedule_timeout_uninterruptible(1); 2637 spin_lock(&swap_lock); 2638 spin_lock(&p->lock); 2639 } 2640 2641 swap_file = p->swap_file; 2642 old_block_size = p->old_block_size; 2643 p->swap_file = NULL; 2644 p->max = 0; 2645 swap_map = p->swap_map; 2646 p->swap_map = NULL; 2647 cluster_info = p->cluster_info; 2648 p->cluster_info = NULL; 2649 frontswap_map = frontswap_map_get(p); 2650 spin_unlock(&p->lock); 2651 spin_unlock(&swap_lock); 2652 arch_swap_invalidate_area(p->type); 2653 frontswap_invalidate_area(p->type); 2654 frontswap_map_set(p, NULL); 2655 mutex_unlock(&swapon_mutex); 2656 free_percpu(p->percpu_cluster); 2657 p->percpu_cluster = NULL; 2658 free_percpu(p->cluster_next_cpu); 2659 p->cluster_next_cpu = NULL; 2660 vfree(swap_map); 2661 kvfree(cluster_info); 2662 kvfree(frontswap_map); 2663 /* Destroy swap account information */ 2664 swap_cgroup_swapoff(p->type); 2665 exit_swap_address_space(p->type); 2666 2667 inode = mapping->host; 2668 if (S_ISBLK(inode->i_mode)) { 2669 struct block_device *bdev = I_BDEV(inode); 2670 2671 set_blocksize(bdev, old_block_size); 2672 blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); 2673 } 2674 2675 inode_lock(inode); 2676 inode->i_flags &= ~S_SWAPFILE; 2677 inode_unlock(inode); 2678 filp_close(swap_file, NULL); 2679 2680 /* 2681 * Clear the SWP_USED flag after all resources are freed so that swapon 2682 * can reuse this swap_info in alloc_swap_info() safely. It is ok to 2683 * not hold p->lock after we cleared its SWP_WRITEOK. 2684 */ 2685 spin_lock(&swap_lock); 2686 p->flags = 0; 2687 spin_unlock(&swap_lock); 2688 2689 err = 0; 2690 atomic_inc(&proc_poll_event); 2691 wake_up_interruptible(&proc_poll_wait); 2692 2693 out_dput: 2694 filp_close(victim, NULL); 2695 out: 2696 putname(pathname); 2697 return err; 2698 } 2699 2700 #ifdef CONFIG_PROC_FS 2701 static __poll_t swaps_poll(struct file *file, poll_table *wait) 2702 { 2703 struct seq_file *seq = file->private_data; 2704 2705 poll_wait(file, &proc_poll_wait, wait); 2706 2707 if (seq->poll_event != atomic_read(&proc_poll_event)) { 2708 seq->poll_event = atomic_read(&proc_poll_event); 2709 return EPOLLIN | EPOLLRDNORM | EPOLLERR | EPOLLPRI; 2710 } 2711 2712 return EPOLLIN | EPOLLRDNORM; 2713 } 2714 2715 /* iterator */ 2716 static void *swap_start(struct seq_file *swap, loff_t *pos) 2717 { 2718 struct swap_info_struct *si; 2719 int type; 2720 loff_t l = *pos; 2721 2722 mutex_lock(&swapon_mutex); 2723 2724 if (!l) 2725 return SEQ_START_TOKEN; 2726 2727 for (type = 0; (si = swap_type_to_swap_info(type)); type++) { 2728 if (!(si->flags & SWP_USED) || !si->swap_map) 2729 continue; 2730 if (!--l) 2731 return si; 2732 } 2733 2734 return NULL; 2735 } 2736 2737 static void *swap_next(struct seq_file *swap, void *v, loff_t *pos) 2738 { 2739 struct swap_info_struct *si = v; 2740 int type; 2741 2742 if (v == SEQ_START_TOKEN) 2743 type = 0; 2744 else 2745 type = si->type + 1; 2746 2747 ++(*pos); 2748 for (; (si = swap_type_to_swap_info(type)); type++) { 2749 if (!(si->flags & SWP_USED) || !si->swap_map) 2750 continue; 2751 return si; 2752 } 2753 2754 return NULL; 2755 } 2756 2757 static void swap_stop(struct seq_file *swap, void *v) 2758 { 2759 mutex_unlock(&swapon_mutex); 2760 } 2761 2762 static int swap_show(struct seq_file *swap, void *v) 2763 { 2764 struct swap_info_struct *si = v; 2765 struct file *file; 2766 int len; 2767 unsigned int bytes, inuse; 2768 2769 if (si == SEQ_START_TOKEN) { 2770 seq_puts(swap,"Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n"); 2771 return 0; 2772 } 2773 2774 bytes = si->pages << (PAGE_SHIFT - 10); 2775 inuse = si->inuse_pages << (PAGE_SHIFT - 10); 2776 2777 file = si->swap_file; 2778 len = seq_file_path(swap, file, " \t\n\\"); 2779 seq_printf(swap, "%*s%s\t%u\t%s%u\t%s%d\n", 2780 len < 40 ? 40 - len : 1, " ", 2781 S_ISBLK(file_inode(file)->i_mode) ? 2782 "partition" : "file\t", 2783 bytes, bytes < 10000000 ? "\t" : "", 2784 inuse, inuse < 10000000 ? "\t" : "", 2785 si->prio); 2786 return 0; 2787 } 2788 2789 static const struct seq_operations swaps_op = { 2790 .start = swap_start, 2791 .next = swap_next, 2792 .stop = swap_stop, 2793 .show = swap_show 2794 }; 2795 2796 static int swaps_open(struct inode *inode, struct file *file) 2797 { 2798 struct seq_file *seq; 2799 int ret; 2800 2801 ret = seq_open(file, &swaps_op); 2802 if (ret) 2803 return ret; 2804 2805 seq = file->private_data; 2806 seq->poll_event = atomic_read(&proc_poll_event); 2807 return 0; 2808 } 2809 2810 static const struct proc_ops swaps_proc_ops = { 2811 .proc_flags = PROC_ENTRY_PERMANENT, 2812 .proc_open = swaps_open, 2813 .proc_read = seq_read, 2814 .proc_lseek = seq_lseek, 2815 .proc_release = seq_release, 2816 .proc_poll = swaps_poll, 2817 }; 2818 2819 static int __init procswaps_init(void) 2820 { 2821 proc_create("swaps", 0, NULL, &swaps_proc_ops); 2822 return 0; 2823 } 2824 __initcall(procswaps_init); 2825 #endif /* CONFIG_PROC_FS */ 2826 2827 #ifdef MAX_SWAPFILES_CHECK 2828 static int __init max_swapfiles_check(void) 2829 { 2830 MAX_SWAPFILES_CHECK(); 2831 return 0; 2832 } 2833 late_initcall(max_swapfiles_check); 2834 #endif 2835 2836 static struct swap_info_struct *alloc_swap_info(void) 2837 { 2838 struct swap_info_struct *p; 2839 struct swap_info_struct *defer = NULL; 2840 unsigned int type; 2841 int i; 2842 2843 p = kvzalloc(struct_size(p, avail_lists, nr_node_ids), GFP_KERNEL); 2844 if (!p) 2845 return ERR_PTR(-ENOMEM); 2846 2847 spin_lock(&swap_lock); 2848 for (type = 0; type < nr_swapfiles; type++) { 2849 if (!(swap_info[type]->flags & SWP_USED)) 2850 break; 2851 } 2852 if (type >= MAX_SWAPFILES) { 2853 spin_unlock(&swap_lock); 2854 kvfree(p); 2855 return ERR_PTR(-EPERM); 2856 } 2857 if (type >= nr_swapfiles) { 2858 p->type = type; 2859 WRITE_ONCE(swap_info[type], p); 2860 /* 2861 * Write swap_info[type] before nr_swapfiles, in case a 2862 * racing procfs swap_start() or swap_next() is reading them. 2863 * (We never shrink nr_swapfiles, we never free this entry.) 2864 */ 2865 smp_wmb(); 2866 WRITE_ONCE(nr_swapfiles, nr_swapfiles + 1); 2867 } else { 2868 defer = p; 2869 p = swap_info[type]; 2870 /* 2871 * Do not memset this entry: a racing procfs swap_next() 2872 * would be relying on p->type to remain valid. 2873 */ 2874 } 2875 p->swap_extent_root = RB_ROOT; 2876 plist_node_init(&p->list, 0); 2877 for_each_node(i) 2878 plist_node_init(&p->avail_lists[i], 0); 2879 p->flags = SWP_USED; 2880 spin_unlock(&swap_lock); 2881 kvfree(defer); 2882 spin_lock_init(&p->lock); 2883 spin_lock_init(&p->cont_lock); 2884 2885 return p; 2886 } 2887 2888 static int claim_swapfile(struct swap_info_struct *p, struct inode *inode) 2889 { 2890 int error; 2891 2892 if (S_ISBLK(inode->i_mode)) { 2893 p->bdev = blkdev_get_by_dev(inode->i_rdev, 2894 FMODE_READ | FMODE_WRITE | FMODE_EXCL, p); 2895 if (IS_ERR(p->bdev)) { 2896 error = PTR_ERR(p->bdev); 2897 p->bdev = NULL; 2898 return error; 2899 } 2900 p->old_block_size = block_size(p->bdev); 2901 error = set_blocksize(p->bdev, PAGE_SIZE); 2902 if (error < 0) 2903 return error; 2904 /* 2905 * Zoned block devices contain zones that have a sequential 2906 * write only restriction. Hence zoned block devices are not 2907 * suitable for swapping. Disallow them here. 2908 */ 2909 if (blk_queue_is_zoned(p->bdev->bd_disk->queue)) 2910 return -EINVAL; 2911 p->flags |= SWP_BLKDEV; 2912 } else if (S_ISREG(inode->i_mode)) { 2913 p->bdev = inode->i_sb->s_bdev; 2914 } 2915 2916 return 0; 2917 } 2918 2919 2920 /* 2921 * Find out how many pages are allowed for a single swap device. There 2922 * are two limiting factors: 2923 * 1) the number of bits for the swap offset in the swp_entry_t type, and 2924 * 2) the number of bits in the swap pte, as defined by the different 2925 * architectures. 2926 * 2927 * In order to find the largest possible bit mask, a swap entry with 2928 * swap type 0 and swap offset ~0UL is created, encoded to a swap pte, 2929 * decoded to a swp_entry_t again, and finally the swap offset is 2930 * extracted. 2931 * 2932 * This will mask all the bits from the initial ~0UL mask that can't 2933 * be encoded in either the swp_entry_t or the architecture definition 2934 * of a swap pte. 2935 */ 2936 unsigned long generic_max_swapfile_size(void) 2937 { 2938 return swp_offset(pte_to_swp_entry( 2939 swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1; 2940 } 2941 2942 /* Can be overridden by an architecture for additional checks. */ 2943 __weak unsigned long max_swapfile_size(void) 2944 { 2945 return generic_max_swapfile_size(); 2946 } 2947 2948 static unsigned long read_swap_header(struct swap_info_struct *p, 2949 union swap_header *swap_header, 2950 struct inode *inode) 2951 { 2952 int i; 2953 unsigned long maxpages; 2954 unsigned long swapfilepages; 2955 unsigned long last_page; 2956 2957 if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) { 2958 pr_err("Unable to find swap-space signature\n"); 2959 return 0; 2960 } 2961 2962 /* swap partition endianess hack... */ 2963 if (swab32(swap_header->info.version) == 1) { 2964 swab32s(&swap_header->info.version); 2965 swab32s(&swap_header->info.last_page); 2966 swab32s(&swap_header->info.nr_badpages); 2967 if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES) 2968 return 0; 2969 for (i = 0; i < swap_header->info.nr_badpages; i++) 2970 swab32s(&swap_header->info.badpages[i]); 2971 } 2972 /* Check the swap header's sub-version */ 2973 if (swap_header->info.version != 1) { 2974 pr_warn("Unable to handle swap header version %d\n", 2975 swap_header->info.version); 2976 return 0; 2977 } 2978 2979 p->lowest_bit = 1; 2980 p->cluster_next = 1; 2981 p->cluster_nr = 0; 2982 2983 maxpages = max_swapfile_size(); 2984 last_page = swap_header->info.last_page; 2985 if (!last_page) { 2986 pr_warn("Empty swap-file\n"); 2987 return 0; 2988 } 2989 if (last_page > maxpages) { 2990 pr_warn("Truncating oversized swap area, only using %luk out of %luk\n", 2991 maxpages << (PAGE_SHIFT - 10), 2992 last_page << (PAGE_SHIFT - 10)); 2993 } 2994 if (maxpages > last_page) { 2995 maxpages = last_page + 1; 2996 /* p->max is an unsigned int: don't overflow it */ 2997 if ((unsigned int)maxpages == 0) 2998 maxpages = UINT_MAX; 2999 } 3000 p->highest_bit = maxpages - 1; 3001 3002 if (!maxpages) 3003 return 0; 3004 swapfilepages = i_size_read(inode) >> PAGE_SHIFT; 3005 if (swapfilepages && maxpages > swapfilepages) { 3006 pr_warn("Swap area shorter than signature indicates\n"); 3007 return 0; 3008 } 3009 if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode)) 3010 return 0; 3011 if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES) 3012 return 0; 3013 3014 return maxpages; 3015 } 3016 3017 #define SWAP_CLUSTER_INFO_COLS \ 3018 DIV_ROUND_UP(L1_CACHE_BYTES, sizeof(struct swap_cluster_info)) 3019 #define SWAP_CLUSTER_SPACE_COLS \ 3020 DIV_ROUND_UP(SWAP_ADDRESS_SPACE_PAGES, SWAPFILE_CLUSTER) 3021 #define SWAP_CLUSTER_COLS \ 3022 max_t(unsigned int, SWAP_CLUSTER_INFO_COLS, SWAP_CLUSTER_SPACE_COLS) 3023 3024 static int setup_swap_map_and_extents(struct swap_info_struct *p, 3025 union swap_header *swap_header, 3026 unsigned char *swap_map, 3027 struct swap_cluster_info *cluster_info, 3028 unsigned long maxpages, 3029 sector_t *span) 3030 { 3031 unsigned int j, k; 3032 unsigned int nr_good_pages; 3033 int nr_extents; 3034 unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER); 3035 unsigned long col = p->cluster_next / SWAPFILE_CLUSTER % SWAP_CLUSTER_COLS; 3036 unsigned long i, idx; 3037 3038 nr_good_pages = maxpages - 1; /* omit header page */ 3039 3040 cluster_list_init(&p->free_clusters); 3041 cluster_list_init(&p->discard_clusters); 3042 3043 for (i = 0; i < swap_header->info.nr_badpages; i++) { 3044 unsigned int page_nr = swap_header->info.badpages[i]; 3045 if (page_nr == 0 || page_nr > swap_header->info.last_page) 3046 return -EINVAL; 3047 if (page_nr < maxpages) { 3048 swap_map[page_nr] = SWAP_MAP_BAD; 3049 nr_good_pages--; 3050 /* 3051 * Haven't marked the cluster free yet, no list 3052 * operation involved 3053 */ 3054 inc_cluster_info_page(p, cluster_info, page_nr); 3055 } 3056 } 3057 3058 /* Haven't marked the cluster free yet, no list operation involved */ 3059 for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++) 3060 inc_cluster_info_page(p, cluster_info, i); 3061 3062 if (nr_good_pages) { 3063 swap_map[0] = SWAP_MAP_BAD; 3064 /* 3065 * Not mark the cluster free yet, no list 3066 * operation involved 3067 */ 3068 inc_cluster_info_page(p, cluster_info, 0); 3069 p->max = maxpages; 3070 p->pages = nr_good_pages; 3071 nr_extents = setup_swap_extents(p, span); 3072 if (nr_extents < 0) 3073 return nr_extents; 3074 nr_good_pages = p->pages; 3075 } 3076 if (!nr_good_pages) { 3077 pr_warn("Empty swap-file\n"); 3078 return -EINVAL; 3079 } 3080 3081 if (!cluster_info) 3082 return nr_extents; 3083 3084 3085 /* 3086 * Reduce false cache line sharing between cluster_info and 3087 * sharing same address space. 3088 */ 3089 for (k = 0; k < SWAP_CLUSTER_COLS; k++) { 3090 j = (k + col) % SWAP_CLUSTER_COLS; 3091 for (i = 0; i < DIV_ROUND_UP(nr_clusters, SWAP_CLUSTER_COLS); i++) { 3092 idx = i * SWAP_CLUSTER_COLS + j; 3093 if (idx >= nr_clusters) 3094 continue; 3095 if (cluster_count(&cluster_info[idx])) 3096 continue; 3097 cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE); 3098 cluster_list_add_tail(&p->free_clusters, cluster_info, 3099 idx); 3100 } 3101 } 3102 return nr_extents; 3103 } 3104 3105 /* 3106 * Helper to sys_swapon determining if a given swap 3107 * backing device queue supports DISCARD operations. 3108 */ 3109 static bool swap_discardable(struct swap_info_struct *si) 3110 { 3111 struct request_queue *q = bdev_get_queue(si->bdev); 3112 3113 if (!q || !blk_queue_discard(q)) 3114 return false; 3115 3116 return true; 3117 } 3118 3119 SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) 3120 { 3121 struct swap_info_struct *p; 3122 struct filename *name; 3123 struct file *swap_file = NULL; 3124 struct address_space *mapping; 3125 int prio; 3126 int error; 3127 union swap_header *swap_header; 3128 int nr_extents; 3129 sector_t span; 3130 unsigned long maxpages; 3131 unsigned char *swap_map = NULL; 3132 struct swap_cluster_info *cluster_info = NULL; 3133 unsigned long *frontswap_map = NULL; 3134 struct page *page = NULL; 3135 struct inode *inode = NULL; 3136 bool inced_nr_rotate_swap = false; 3137 3138 if (swap_flags & ~SWAP_FLAGS_VALID) 3139 return -EINVAL; 3140 3141 if (!capable(CAP_SYS_ADMIN)) 3142 return -EPERM; 3143 3144 if (!swap_avail_heads) 3145 return -ENOMEM; 3146 3147 p = alloc_swap_info(); 3148 if (IS_ERR(p)) 3149 return PTR_ERR(p); 3150 3151 INIT_WORK(&p->discard_work, swap_discard_work); 3152 3153 name = getname(specialfile); 3154 if (IS_ERR(name)) { 3155 error = PTR_ERR(name); 3156 name = NULL; 3157 goto bad_swap; 3158 } 3159 swap_file = file_open_name(name, O_RDWR|O_LARGEFILE, 0); 3160 if (IS_ERR(swap_file)) { 3161 error = PTR_ERR(swap_file); 3162 swap_file = NULL; 3163 goto bad_swap; 3164 } 3165 3166 p->swap_file = swap_file; 3167 mapping = swap_file->f_mapping; 3168 inode = mapping->host; 3169 3170 error = claim_swapfile(p, inode); 3171 if (unlikely(error)) 3172 goto bad_swap; 3173 3174 inode_lock(inode); 3175 if (IS_SWAPFILE(inode)) { 3176 error = -EBUSY; 3177 goto bad_swap_unlock_inode; 3178 } 3179 3180 /* 3181 * Read the swap header. 3182 */ 3183 if (!mapping->a_ops->readpage) { 3184 error = -EINVAL; 3185 goto bad_swap_unlock_inode; 3186 } 3187 page = read_mapping_page(mapping, 0, swap_file); 3188 if (IS_ERR(page)) { 3189 error = PTR_ERR(page); 3190 goto bad_swap_unlock_inode; 3191 } 3192 swap_header = kmap(page); 3193 3194 maxpages = read_swap_header(p, swap_header, inode); 3195 if (unlikely(!maxpages)) { 3196 error = -EINVAL; 3197 goto bad_swap_unlock_inode; 3198 } 3199 3200 /* OK, set up the swap map and apply the bad block list */ 3201 swap_map = vzalloc(maxpages); 3202 if (!swap_map) { 3203 error = -ENOMEM; 3204 goto bad_swap_unlock_inode; 3205 } 3206 3207 if (p->bdev && blk_queue_stable_writes(p->bdev->bd_disk->queue)) 3208 p->flags |= SWP_STABLE_WRITES; 3209 3210 if (p->bdev && p->bdev->bd_disk->fops->rw_page) 3211 p->flags |= SWP_SYNCHRONOUS_IO; 3212 3213 if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) { 3214 int cpu; 3215 unsigned long ci, nr_cluster; 3216 3217 p->flags |= SWP_SOLIDSTATE; 3218 p->cluster_next_cpu = alloc_percpu(unsigned int); 3219 if (!p->cluster_next_cpu) { 3220 error = -ENOMEM; 3221 goto bad_swap_unlock_inode; 3222 } 3223 /* 3224 * select a random position to start with to help wear leveling 3225 * SSD 3226 */ 3227 for_each_possible_cpu(cpu) { 3228 per_cpu(*p->cluster_next_cpu, cpu) = 3229 1 + prandom_u32_max(p->highest_bit); 3230 } 3231 nr_cluster = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER); 3232 3233 cluster_info = kvcalloc(nr_cluster, sizeof(*cluster_info), 3234 GFP_KERNEL); 3235 if (!cluster_info) { 3236 error = -ENOMEM; 3237 goto bad_swap_unlock_inode; 3238 } 3239 3240 for (ci = 0; ci < nr_cluster; ci++) 3241 spin_lock_init(&((cluster_info + ci)->lock)); 3242 3243 p->percpu_cluster = alloc_percpu(struct percpu_cluster); 3244 if (!p->percpu_cluster) { 3245 error = -ENOMEM; 3246 goto bad_swap_unlock_inode; 3247 } 3248 for_each_possible_cpu(cpu) { 3249 struct percpu_cluster *cluster; 3250 cluster = per_cpu_ptr(p->percpu_cluster, cpu); 3251 cluster_set_null(&cluster->index); 3252 } 3253 } else { 3254 atomic_inc(&nr_rotate_swap); 3255 inced_nr_rotate_swap = true; 3256 } 3257 3258 error = swap_cgroup_swapon(p->type, maxpages); 3259 if (error) 3260 goto bad_swap_unlock_inode; 3261 3262 nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map, 3263 cluster_info, maxpages, &span); 3264 if (unlikely(nr_extents < 0)) { 3265 error = nr_extents; 3266 goto bad_swap_unlock_inode; 3267 } 3268 /* frontswap enabled? set up bit-per-page map for frontswap */ 3269 if (IS_ENABLED(CONFIG_FRONTSWAP)) 3270 frontswap_map = kvcalloc(BITS_TO_LONGS(maxpages), 3271 sizeof(long), 3272 GFP_KERNEL); 3273 3274 if (p->bdev &&(swap_flags & SWAP_FLAG_DISCARD) && swap_discardable(p)) { 3275 /* 3276 * When discard is enabled for swap with no particular 3277 * policy flagged, we set all swap discard flags here in 3278 * order to sustain backward compatibility with older 3279 * swapon(8) releases. 3280 */ 3281 p->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD | 3282 SWP_PAGE_DISCARD); 3283 3284 /* 3285 * By flagging sys_swapon, a sysadmin can tell us to 3286 * either do single-time area discards only, or to just 3287 * perform discards for released swap page-clusters. 3288 * Now it's time to adjust the p->flags accordingly. 3289 */ 3290 if (swap_flags & SWAP_FLAG_DISCARD_ONCE) 3291 p->flags &= ~SWP_PAGE_DISCARD; 3292 else if (swap_flags & SWAP_FLAG_DISCARD_PAGES) 3293 p->flags &= ~SWP_AREA_DISCARD; 3294 3295 /* issue a swapon-time discard if it's still required */ 3296 if (p->flags & SWP_AREA_DISCARD) { 3297 int err = discard_swap(p); 3298 if (unlikely(err)) 3299 pr_err("swapon: discard_swap(%p): %d\n", 3300 p, err); 3301 } 3302 } 3303 3304 error = init_swap_address_space(p->type, maxpages); 3305 if (error) 3306 goto bad_swap_unlock_inode; 3307 3308 /* 3309 * Flush any pending IO and dirty mappings before we start using this 3310 * swap device. 3311 */ 3312 inode->i_flags |= S_SWAPFILE; 3313 error = inode_drain_writes(inode); 3314 if (error) { 3315 inode->i_flags &= ~S_SWAPFILE; 3316 goto free_swap_address_space; 3317 } 3318 3319 mutex_lock(&swapon_mutex); 3320 prio = -1; 3321 if (swap_flags & SWAP_FLAG_PREFER) 3322 prio = 3323 (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT; 3324 enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map); 3325 3326 pr_info("Adding %uk swap on %s. Priority:%d extents:%d across:%lluk %s%s%s%s%s\n", 3327 p->pages<<(PAGE_SHIFT-10), name->name, p->prio, 3328 nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10), 3329 (p->flags & SWP_SOLIDSTATE) ? "SS" : "", 3330 (p->flags & SWP_DISCARDABLE) ? "D" : "", 3331 (p->flags & SWP_AREA_DISCARD) ? "s" : "", 3332 (p->flags & SWP_PAGE_DISCARD) ? "c" : "", 3333 (frontswap_map) ? "FS" : ""); 3334 3335 mutex_unlock(&swapon_mutex); 3336 atomic_inc(&proc_poll_event); 3337 wake_up_interruptible(&proc_poll_wait); 3338 3339 error = 0; 3340 goto out; 3341 free_swap_address_space: 3342 exit_swap_address_space(p->type); 3343 bad_swap_unlock_inode: 3344 inode_unlock(inode); 3345 bad_swap: 3346 free_percpu(p->percpu_cluster); 3347 p->percpu_cluster = NULL; 3348 free_percpu(p->cluster_next_cpu); 3349 p->cluster_next_cpu = NULL; 3350 if (inode && S_ISBLK(inode->i_mode) && p->bdev) { 3351 set_blocksize(p->bdev, p->old_block_size); 3352 blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); 3353 } 3354 inode = NULL; 3355 destroy_swap_extents(p); 3356 swap_cgroup_swapoff(p->type); 3357 spin_lock(&swap_lock); 3358 p->swap_file = NULL; 3359 p->flags = 0; 3360 spin_unlock(&swap_lock); 3361 vfree(swap_map); 3362 kvfree(cluster_info); 3363 kvfree(frontswap_map); 3364 if (inced_nr_rotate_swap) 3365 atomic_dec(&nr_rotate_swap); 3366 if (swap_file) 3367 filp_close(swap_file, NULL); 3368 out: 3369 if (page && !IS_ERR(page)) { 3370 kunmap(page); 3371 put_page(page); 3372 } 3373 if (name) 3374 putname(name); 3375 if (inode) 3376 inode_unlock(inode); 3377 if (!error) 3378 enable_swap_slots_cache(); 3379 return error; 3380 } 3381 3382 void si_swapinfo(struct sysinfo *val) 3383 { 3384 unsigned int type; 3385 unsigned long nr_to_be_unused = 0; 3386 3387 spin_lock(&swap_lock); 3388 for (type = 0; type < nr_swapfiles; type++) { 3389 struct swap_info_struct *si = swap_info[type]; 3390 3391 if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK)) 3392 nr_to_be_unused += si->inuse_pages; 3393 } 3394 val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused; 3395 val->totalswap = total_swap_pages + nr_to_be_unused; 3396 spin_unlock(&swap_lock); 3397 } 3398 3399 /* 3400 * Verify that a swap entry is valid and increment its swap map count. 3401 * 3402 * Returns error code in following case. 3403 * - success -> 0 3404 * - swp_entry is invalid -> EINVAL 3405 * - swp_entry is migration entry -> EINVAL 3406 * - swap-cache reference is requested but there is already one. -> EEXIST 3407 * - swap-cache reference is requested but the entry is not used. -> ENOENT 3408 * - swap-mapped reference requested but needs continued swap count. -> ENOMEM 3409 */ 3410 static int __swap_duplicate(swp_entry_t entry, unsigned char usage) 3411 { 3412 struct swap_info_struct *p; 3413 struct swap_cluster_info *ci; 3414 unsigned long offset; 3415 unsigned char count; 3416 unsigned char has_cache; 3417 int err; 3418 3419 p = get_swap_device(entry); 3420 if (!p) 3421 return -EINVAL; 3422 3423 offset = swp_offset(entry); 3424 ci = lock_cluster_or_swap_info(p, offset); 3425 3426 count = p->swap_map[offset]; 3427 3428 /* 3429 * swapin_readahead() doesn't check if a swap entry is valid, so the 3430 * swap entry could be SWAP_MAP_BAD. Check here with lock held. 3431 */ 3432 if (unlikely(swap_count(count) == SWAP_MAP_BAD)) { 3433 err = -ENOENT; 3434 goto unlock_out; 3435 } 3436 3437 has_cache = count & SWAP_HAS_CACHE; 3438 count &= ~SWAP_HAS_CACHE; 3439 err = 0; 3440 3441 if (usage == SWAP_HAS_CACHE) { 3442 3443 /* set SWAP_HAS_CACHE if there is no cache and entry is used */ 3444 if (!has_cache && count) 3445 has_cache = SWAP_HAS_CACHE; 3446 else if (has_cache) /* someone else added cache */ 3447 err = -EEXIST; 3448 else /* no users remaining */ 3449 err = -ENOENT; 3450 3451 } else if (count || has_cache) { 3452 3453 if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX) 3454 count += usage; 3455 else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX) 3456 err = -EINVAL; 3457 else if (swap_count_continued(p, offset, count)) 3458 count = COUNT_CONTINUED; 3459 else 3460 err = -ENOMEM; 3461 } else 3462 err = -ENOENT; /* unused swap entry */ 3463 3464 WRITE_ONCE(p->swap_map[offset], count | has_cache); 3465 3466 unlock_out: 3467 unlock_cluster_or_swap_info(p, ci); 3468 if (p) 3469 put_swap_device(p); 3470 return err; 3471 } 3472 3473 /* 3474 * Help swapoff by noting that swap entry belongs to shmem/tmpfs 3475 * (in which case its reference count is never incremented). 3476 */ 3477 void swap_shmem_alloc(swp_entry_t entry) 3478 { 3479 __swap_duplicate(entry, SWAP_MAP_SHMEM); 3480 } 3481 3482 /* 3483 * Increase reference count of swap entry by 1. 3484 * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required 3485 * but could not be atomically allocated. Returns 0, just as if it succeeded, 3486 * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which 3487 * might occur if a page table entry has got corrupted. 3488 */ 3489 int swap_duplicate(swp_entry_t entry) 3490 { 3491 int err = 0; 3492 3493 while (!err && __swap_duplicate(entry, 1) == -ENOMEM) 3494 err = add_swap_count_continuation(entry, GFP_ATOMIC); 3495 return err; 3496 } 3497 3498 /* 3499 * @entry: swap entry for which we allocate swap cache. 3500 * 3501 * Called when allocating swap cache for existing swap entry, 3502 * This can return error codes. Returns 0 at success. 3503 * -EEXIST means there is a swap cache. 3504 * Note: return code is different from swap_duplicate(). 3505 */ 3506 int swapcache_prepare(swp_entry_t entry) 3507 { 3508 return __swap_duplicate(entry, SWAP_HAS_CACHE); 3509 } 3510 3511 struct swap_info_struct *swp_swap_info(swp_entry_t entry) 3512 { 3513 return swap_type_to_swap_info(swp_type(entry)); 3514 } 3515 3516 struct swap_info_struct *page_swap_info(struct page *page) 3517 { 3518 swp_entry_t entry = { .val = page_private(page) }; 3519 return swp_swap_info(entry); 3520 } 3521 3522 /* 3523 * out-of-line __page_file_ methods to avoid include hell. 3524 */ 3525 struct address_space *__page_file_mapping(struct page *page) 3526 { 3527 return page_swap_info(page)->swap_file->f_mapping; 3528 } 3529 EXPORT_SYMBOL_GPL(__page_file_mapping); 3530 3531 pgoff_t __page_file_index(struct page *page) 3532 { 3533 swp_entry_t swap = { .val = page_private(page) }; 3534 return swp_offset(swap); 3535 } 3536 EXPORT_SYMBOL_GPL(__page_file_index); 3537 3538 /* 3539 * add_swap_count_continuation - called when a swap count is duplicated 3540 * beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's 3541 * page of the original vmalloc'ed swap_map, to hold the continuation count 3542 * (for that entry and for its neighbouring PAGE_SIZE swap entries). Called 3543 * again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc. 3544 * 3545 * These continuation pages are seldom referenced: the common paths all work 3546 * on the original swap_map, only referring to a continuation page when the 3547 * low "digit" of a count is incremented or decremented through SWAP_MAP_MAX. 3548 * 3549 * add_swap_count_continuation(, GFP_ATOMIC) can be called while holding 3550 * page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL) 3551 * can be called after dropping locks. 3552 */ 3553 int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask) 3554 { 3555 struct swap_info_struct *si; 3556 struct swap_cluster_info *ci; 3557 struct page *head; 3558 struct page *page; 3559 struct page *list_page; 3560 pgoff_t offset; 3561 unsigned char count; 3562 int ret = 0; 3563 3564 /* 3565 * When debugging, it's easier to use __GFP_ZERO here; but it's better 3566 * for latency not to zero a page while GFP_ATOMIC and holding locks. 3567 */ 3568 page = alloc_page(gfp_mask | __GFP_HIGHMEM); 3569 3570 si = get_swap_device(entry); 3571 if (!si) { 3572 /* 3573 * An acceptable race has occurred since the failing 3574 * __swap_duplicate(): the swap device may be swapoff 3575 */ 3576 goto outer; 3577 } 3578 spin_lock(&si->lock); 3579 3580 offset = swp_offset(entry); 3581 3582 ci = lock_cluster(si, offset); 3583 3584 count = swap_count(si->swap_map[offset]); 3585 3586 if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) { 3587 /* 3588 * The higher the swap count, the more likely it is that tasks 3589 * will race to add swap count continuation: we need to avoid 3590 * over-provisioning. 3591 */ 3592 goto out; 3593 } 3594 3595 if (!page) { 3596 ret = -ENOMEM; 3597 goto out; 3598 } 3599 3600 /* 3601 * We are fortunate that although vmalloc_to_page uses pte_offset_map, 3602 * no architecture is using highmem pages for kernel page tables: so it 3603 * will not corrupt the GFP_ATOMIC caller's atomic page table kmaps. 3604 */ 3605 head = vmalloc_to_page(si->swap_map + offset); 3606 offset &= ~PAGE_MASK; 3607 3608 spin_lock(&si->cont_lock); 3609 /* 3610 * Page allocation does not initialize the page's lru field, 3611 * but it does always reset its private field. 3612 */ 3613 if (!page_private(head)) { 3614 BUG_ON(count & COUNT_CONTINUED); 3615 INIT_LIST_HEAD(&head->lru); 3616 set_page_private(head, SWP_CONTINUED); 3617 si->flags |= SWP_CONTINUED; 3618 } 3619 3620 list_for_each_entry(list_page, &head->lru, lru) { 3621 unsigned char *map; 3622 3623 /* 3624 * If the previous map said no continuation, but we've found 3625 * a continuation page, free our allocation and use this one. 3626 */ 3627 if (!(count & COUNT_CONTINUED)) 3628 goto out_unlock_cont; 3629 3630 map = kmap_atomic(list_page) + offset; 3631 count = *map; 3632 kunmap_atomic(map); 3633 3634 /* 3635 * If this continuation count now has some space in it, 3636 * free our allocation and use this one. 3637 */ 3638 if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX) 3639 goto out_unlock_cont; 3640 } 3641 3642 list_add_tail(&page->lru, &head->lru); 3643 page = NULL; /* now it's attached, don't free it */ 3644 out_unlock_cont: 3645 spin_unlock(&si->cont_lock); 3646 out: 3647 unlock_cluster(ci); 3648 spin_unlock(&si->lock); 3649 put_swap_device(si); 3650 outer: 3651 if (page) 3652 __free_page(page); 3653 return ret; 3654 } 3655 3656 /* 3657 * swap_count_continued - when the original swap_map count is incremented 3658 * from SWAP_MAP_MAX, check if there is already a continuation page to carry 3659 * into, carry if so, or else fail until a new continuation page is allocated; 3660 * when the original swap_map count is decremented from 0 with continuation, 3661 * borrow from the continuation and report whether it still holds more. 3662 * Called while __swap_duplicate() or swap_entry_free() holds swap or cluster 3663 * lock. 3664 */ 3665 static bool swap_count_continued(struct swap_info_struct *si, 3666 pgoff_t offset, unsigned char count) 3667 { 3668 struct page *head; 3669 struct page *page; 3670 unsigned char *map; 3671 bool ret; 3672 3673 head = vmalloc_to_page(si->swap_map + offset); 3674 if (page_private(head) != SWP_CONTINUED) { 3675 BUG_ON(count & COUNT_CONTINUED); 3676 return false; /* need to add count continuation */ 3677 } 3678 3679 spin_lock(&si->cont_lock); 3680 offset &= ~PAGE_MASK; 3681 page = list_next_entry(head, lru); 3682 map = kmap_atomic(page) + offset; 3683 3684 if (count == SWAP_MAP_MAX) /* initial increment from swap_map */ 3685 goto init_map; /* jump over SWAP_CONT_MAX checks */ 3686 3687 if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */ 3688 /* 3689 * Think of how you add 1 to 999 3690 */ 3691 while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) { 3692 kunmap_atomic(map); 3693 page = list_next_entry(page, lru); 3694 BUG_ON(page == head); 3695 map = kmap_atomic(page) + offset; 3696 } 3697 if (*map == SWAP_CONT_MAX) { 3698 kunmap_atomic(map); 3699 page = list_next_entry(page, lru); 3700 if (page == head) { 3701 ret = false; /* add count continuation */ 3702 goto out; 3703 } 3704 map = kmap_atomic(page) + offset; 3705 init_map: *map = 0; /* we didn't zero the page */ 3706 } 3707 *map += 1; 3708 kunmap_atomic(map); 3709 while ((page = list_prev_entry(page, lru)) != head) { 3710 map = kmap_atomic(page) + offset; 3711 *map = COUNT_CONTINUED; 3712 kunmap_atomic(map); 3713 } 3714 ret = true; /* incremented */ 3715 3716 } else { /* decrementing */ 3717 /* 3718 * Think of how you subtract 1 from 1000 3719 */ 3720 BUG_ON(count != COUNT_CONTINUED); 3721 while (*map == COUNT_CONTINUED) { 3722 kunmap_atomic(map); 3723 page = list_next_entry(page, lru); 3724 BUG_ON(page == head); 3725 map = kmap_atomic(page) + offset; 3726 } 3727 BUG_ON(*map == 0); 3728 *map -= 1; 3729 if (*map == 0) 3730 count = 0; 3731 kunmap_atomic(map); 3732 while ((page = list_prev_entry(page, lru)) != head) { 3733 map = kmap_atomic(page) + offset; 3734 *map = SWAP_CONT_MAX | count; 3735 count = COUNT_CONTINUED; 3736 kunmap_atomic(map); 3737 } 3738 ret = count == COUNT_CONTINUED; 3739 } 3740 out: 3741 spin_unlock(&si->cont_lock); 3742 return ret; 3743 } 3744 3745 /* 3746 * free_swap_count_continuations - swapoff free all the continuation pages 3747 * appended to the swap_map, after swap_map is quiesced, before vfree'ing it. 3748 */ 3749 static void free_swap_count_continuations(struct swap_info_struct *si) 3750 { 3751 pgoff_t offset; 3752 3753 for (offset = 0; offset < si->max; offset += PAGE_SIZE) { 3754 struct page *head; 3755 head = vmalloc_to_page(si->swap_map + offset); 3756 if (page_private(head)) { 3757 struct page *page, *next; 3758 3759 list_for_each_entry_safe(page, next, &head->lru, lru) { 3760 list_del(&page->lru); 3761 __free_page(page); 3762 } 3763 } 3764 } 3765 } 3766 3767 #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) 3768 void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) 3769 { 3770 struct swap_info_struct *si, *next; 3771 int nid = page_to_nid(page); 3772 3773 if (!(gfp_mask & __GFP_IO)) 3774 return; 3775 3776 if (!blk_cgroup_congested()) 3777 return; 3778 3779 /* 3780 * We've already scheduled a throttle, avoid taking the global swap 3781 * lock. 3782 */ 3783 if (current->throttle_queue) 3784 return; 3785 3786 spin_lock(&swap_avail_lock); 3787 plist_for_each_entry_safe(si, next, &swap_avail_heads[nid], 3788 avail_lists[nid]) { 3789 if (si->bdev) { 3790 blkcg_schedule_throttle(bdev_get_queue(si->bdev), true); 3791 break; 3792 } 3793 } 3794 spin_unlock(&swap_avail_lock); 3795 } 3796 #endif 3797 3798 static int __init swapfile_init(void) 3799 { 3800 int nid; 3801 3802 swap_avail_heads = kmalloc_array(nr_node_ids, sizeof(struct plist_head), 3803 GFP_KERNEL); 3804 if (!swap_avail_heads) { 3805 pr_emerg("Not enough memory for swap heads, swap is disabled\n"); 3806 return -ENOMEM; 3807 } 3808 3809 for_each_node(nid) 3810 plist_head_init(&swap_avail_heads[nid]); 3811 3812 return 0; 3813 } 3814 subsys_initcall(swapfile_init); 3815