1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/swapfile.c 4 * 5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 6 * Swap reorganised 29.12.95, Stephen Tweedie 7 */ 8 9 #include <linux/blkdev.h> 10 #include <linux/mm.h> 11 #include <linux/sched/mm.h> 12 #include <linux/sched/task.h> 13 #include <linux/hugetlb.h> 14 #include <linux/mman.h> 15 #include <linux/slab.h> 16 #include <linux/kernel_stat.h> 17 #include <linux/swap.h> 18 #include <linux/vmalloc.h> 19 #include <linux/pagemap.h> 20 #include <linux/namei.h> 21 #include <linux/shmem_fs.h> 22 #include <linux/blk-cgroup.h> 23 #include <linux/random.h> 24 #include <linux/writeback.h> 25 #include <linux/proc_fs.h> 26 #include <linux/seq_file.h> 27 #include <linux/init.h> 28 #include <linux/ksm.h> 29 #include <linux/rmap.h> 30 #include <linux/security.h> 31 #include <linux/backing-dev.h> 32 #include <linux/mutex.h> 33 #include <linux/capability.h> 34 #include <linux/syscalls.h> 35 #include <linux/memcontrol.h> 36 #include <linux/poll.h> 37 #include <linux/oom.h> 38 #include <linux/frontswap.h> 39 #include <linux/swapfile.h> 40 #include <linux/export.h> 41 #include <linux/swap_slots.h> 42 #include <linux/sort.h> 43 #include <linux/completion.h> 44 45 #include <asm/tlbflush.h> 46 #include <linux/swapops.h> 47 #include <linux/swap_cgroup.h> 48 49 static bool swap_count_continued(struct swap_info_struct *, pgoff_t, 50 unsigned char); 51 static void free_swap_count_continuations(struct swap_info_struct *); 52 53 static DEFINE_SPINLOCK(swap_lock); 54 static unsigned int nr_swapfiles; 55 atomic_long_t nr_swap_pages; 56 /* 57 * Some modules use swappable objects and may try to swap them out under 58 * memory pressure (via the shrinker). Before doing so, they may wish to 59 * check to see if any swap space is available. 60 */ 61 EXPORT_SYMBOL_GPL(nr_swap_pages); 62 /* protected with swap_lock. reading in vm_swap_full() doesn't need lock */ 63 long total_swap_pages; 64 static int least_priority = -1; 65 66 static const char Bad_file[] = "Bad swap file entry "; 67 static const char Unused_file[] = "Unused swap file entry "; 68 static const char Bad_offset[] = "Bad swap offset entry "; 69 static const char Unused_offset[] = "Unused swap offset entry "; 70 71 /* 72 * all active swap_info_structs 73 * protected with swap_lock, and ordered by priority. 74 */ 75 static PLIST_HEAD(swap_active_head); 76 77 /* 78 * all available (active, not full) swap_info_structs 79 * protected with swap_avail_lock, ordered by priority. 80 * This is used by get_swap_page() instead of swap_active_head 81 * because swap_active_head includes all swap_info_structs, 82 * but get_swap_page() doesn't need to look at full ones. 83 * This uses its own lock instead of swap_lock because when a 84 * swap_info_struct changes between not-full/full, it needs to 85 * add/remove itself to/from this list, but the swap_info_struct->lock 86 * is held and the locking order requires swap_lock to be taken 87 * before any swap_info_struct->lock. 88 */ 89 static struct plist_head *swap_avail_heads; 90 static DEFINE_SPINLOCK(swap_avail_lock); 91 92 struct swap_info_struct *swap_info[MAX_SWAPFILES]; 93 94 static DEFINE_MUTEX(swapon_mutex); 95 96 static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait); 97 /* Activity counter to indicate that a swapon or swapoff has occurred */ 98 static atomic_t proc_poll_event = ATOMIC_INIT(0); 99 100 atomic_t nr_rotate_swap = ATOMIC_INIT(0); 101 102 static struct swap_info_struct *swap_type_to_swap_info(int type) 103 { 104 if (type >= MAX_SWAPFILES) 105 return NULL; 106 107 return READ_ONCE(swap_info[type]); /* rcu_dereference() */ 108 } 109 110 static inline unsigned char swap_count(unsigned char ent) 111 { 112 return ent & ~SWAP_HAS_CACHE; /* may include COUNT_CONTINUED flag */ 113 } 114 115 /* Reclaim the swap entry anyway if possible */ 116 #define TTRS_ANYWAY 0x1 117 /* 118 * Reclaim the swap entry if there are no more mappings of the 119 * corresponding page 120 */ 121 #define TTRS_UNMAPPED 0x2 122 /* Reclaim the swap entry if swap is getting full*/ 123 #define TTRS_FULL 0x4 124 125 /* returns 1 if swap entry is freed */ 126 static int __try_to_reclaim_swap(struct swap_info_struct *si, 127 unsigned long offset, unsigned long flags) 128 { 129 swp_entry_t entry = swp_entry(si->type, offset); 130 struct page *page; 131 int ret = 0; 132 133 page = find_get_page(swap_address_space(entry), offset); 134 if (!page) 135 return 0; 136 /* 137 * When this function is called from scan_swap_map_slots() and it's 138 * called by vmscan.c at reclaiming pages. So, we hold a lock on a page, 139 * here. We have to use trylock for avoiding deadlock. This is a special 140 * case and you should use try_to_free_swap() with explicit lock_page() 141 * in usual operations. 142 */ 143 if (trylock_page(page)) { 144 if ((flags & TTRS_ANYWAY) || 145 ((flags & TTRS_UNMAPPED) && !page_mapped(page)) || 146 ((flags & TTRS_FULL) && mem_cgroup_swap_full(page))) 147 ret = try_to_free_swap(page); 148 unlock_page(page); 149 } 150 put_page(page); 151 return ret; 152 } 153 154 static inline struct swap_extent *first_se(struct swap_info_struct *sis) 155 { 156 struct rb_node *rb = rb_first(&sis->swap_extent_root); 157 return rb_entry(rb, struct swap_extent, rb_node); 158 } 159 160 static inline struct swap_extent *next_se(struct swap_extent *se) 161 { 162 struct rb_node *rb = rb_next(&se->rb_node); 163 return rb ? rb_entry(rb, struct swap_extent, rb_node) : NULL; 164 } 165 166 /* 167 * swapon tell device that all the old swap contents can be discarded, 168 * to allow the swap device to optimize its wear-levelling. 169 */ 170 static int discard_swap(struct swap_info_struct *si) 171 { 172 struct swap_extent *se; 173 sector_t start_block; 174 sector_t nr_blocks; 175 int err = 0; 176 177 /* Do not discard the swap header page! */ 178 se = first_se(si); 179 start_block = (se->start_block + 1) << (PAGE_SHIFT - 9); 180 nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9); 181 if (nr_blocks) { 182 err = blkdev_issue_discard(si->bdev, start_block, 183 nr_blocks, GFP_KERNEL); 184 if (err) 185 return err; 186 cond_resched(); 187 } 188 189 for (se = next_se(se); se; se = next_se(se)) { 190 start_block = se->start_block << (PAGE_SHIFT - 9); 191 nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9); 192 193 err = blkdev_issue_discard(si->bdev, start_block, 194 nr_blocks, GFP_KERNEL); 195 if (err) 196 break; 197 198 cond_resched(); 199 } 200 return err; /* That will often be -EOPNOTSUPP */ 201 } 202 203 static struct swap_extent * 204 offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset) 205 { 206 struct swap_extent *se; 207 struct rb_node *rb; 208 209 rb = sis->swap_extent_root.rb_node; 210 while (rb) { 211 se = rb_entry(rb, struct swap_extent, rb_node); 212 if (offset < se->start_page) 213 rb = rb->rb_left; 214 else if (offset >= se->start_page + se->nr_pages) 215 rb = rb->rb_right; 216 else 217 return se; 218 } 219 /* It *must* be present */ 220 BUG(); 221 } 222 223 sector_t swap_page_sector(struct page *page) 224 { 225 struct swap_info_struct *sis = page_swap_info(page); 226 struct swap_extent *se; 227 sector_t sector; 228 pgoff_t offset; 229 230 offset = __page_file_index(page); 231 se = offset_to_swap_extent(sis, offset); 232 sector = se->start_block + (offset - se->start_page); 233 return sector << (PAGE_SHIFT - 9); 234 } 235 236 /* 237 * swap allocation tell device that a cluster of swap can now be discarded, 238 * to allow the swap device to optimize its wear-levelling. 239 */ 240 static void discard_swap_cluster(struct swap_info_struct *si, 241 pgoff_t start_page, pgoff_t nr_pages) 242 { 243 struct swap_extent *se = offset_to_swap_extent(si, start_page); 244 245 while (nr_pages) { 246 pgoff_t offset = start_page - se->start_page; 247 sector_t start_block = se->start_block + offset; 248 sector_t nr_blocks = se->nr_pages - offset; 249 250 if (nr_blocks > nr_pages) 251 nr_blocks = nr_pages; 252 start_page += nr_blocks; 253 nr_pages -= nr_blocks; 254 255 start_block <<= PAGE_SHIFT - 9; 256 nr_blocks <<= PAGE_SHIFT - 9; 257 if (blkdev_issue_discard(si->bdev, start_block, 258 nr_blocks, GFP_NOIO)) 259 break; 260 261 se = next_se(se); 262 } 263 } 264 265 #ifdef CONFIG_THP_SWAP 266 #define SWAPFILE_CLUSTER HPAGE_PMD_NR 267 268 #define swap_entry_size(size) (size) 269 #else 270 #define SWAPFILE_CLUSTER 256 271 272 /* 273 * Define swap_entry_size() as constant to let compiler to optimize 274 * out some code if !CONFIG_THP_SWAP 275 */ 276 #define swap_entry_size(size) 1 277 #endif 278 #define LATENCY_LIMIT 256 279 280 static inline void cluster_set_flag(struct swap_cluster_info *info, 281 unsigned int flag) 282 { 283 info->flags = flag; 284 } 285 286 static inline unsigned int cluster_count(struct swap_cluster_info *info) 287 { 288 return info->data; 289 } 290 291 static inline void cluster_set_count(struct swap_cluster_info *info, 292 unsigned int c) 293 { 294 info->data = c; 295 } 296 297 static inline void cluster_set_count_flag(struct swap_cluster_info *info, 298 unsigned int c, unsigned int f) 299 { 300 info->flags = f; 301 info->data = c; 302 } 303 304 static inline unsigned int cluster_next(struct swap_cluster_info *info) 305 { 306 return info->data; 307 } 308 309 static inline void cluster_set_next(struct swap_cluster_info *info, 310 unsigned int n) 311 { 312 info->data = n; 313 } 314 315 static inline void cluster_set_next_flag(struct swap_cluster_info *info, 316 unsigned int n, unsigned int f) 317 { 318 info->flags = f; 319 info->data = n; 320 } 321 322 static inline bool cluster_is_free(struct swap_cluster_info *info) 323 { 324 return info->flags & CLUSTER_FLAG_FREE; 325 } 326 327 static inline bool cluster_is_null(struct swap_cluster_info *info) 328 { 329 return info->flags & CLUSTER_FLAG_NEXT_NULL; 330 } 331 332 static inline void cluster_set_null(struct swap_cluster_info *info) 333 { 334 info->flags = CLUSTER_FLAG_NEXT_NULL; 335 info->data = 0; 336 } 337 338 static inline bool cluster_is_huge(struct swap_cluster_info *info) 339 { 340 if (IS_ENABLED(CONFIG_THP_SWAP)) 341 return info->flags & CLUSTER_FLAG_HUGE; 342 return false; 343 } 344 345 static inline void cluster_clear_huge(struct swap_cluster_info *info) 346 { 347 info->flags &= ~CLUSTER_FLAG_HUGE; 348 } 349 350 static inline struct swap_cluster_info *lock_cluster(struct swap_info_struct *si, 351 unsigned long offset) 352 { 353 struct swap_cluster_info *ci; 354 355 ci = si->cluster_info; 356 if (ci) { 357 ci += offset / SWAPFILE_CLUSTER; 358 spin_lock(&ci->lock); 359 } 360 return ci; 361 } 362 363 static inline void unlock_cluster(struct swap_cluster_info *ci) 364 { 365 if (ci) 366 spin_unlock(&ci->lock); 367 } 368 369 /* 370 * Determine the locking method in use for this device. Return 371 * swap_cluster_info if SSD-style cluster-based locking is in place. 372 */ 373 static inline struct swap_cluster_info *lock_cluster_or_swap_info( 374 struct swap_info_struct *si, unsigned long offset) 375 { 376 struct swap_cluster_info *ci; 377 378 /* Try to use fine-grained SSD-style locking if available: */ 379 ci = lock_cluster(si, offset); 380 /* Otherwise, fall back to traditional, coarse locking: */ 381 if (!ci) 382 spin_lock(&si->lock); 383 384 return ci; 385 } 386 387 static inline void unlock_cluster_or_swap_info(struct swap_info_struct *si, 388 struct swap_cluster_info *ci) 389 { 390 if (ci) 391 unlock_cluster(ci); 392 else 393 spin_unlock(&si->lock); 394 } 395 396 static inline bool cluster_list_empty(struct swap_cluster_list *list) 397 { 398 return cluster_is_null(&list->head); 399 } 400 401 static inline unsigned int cluster_list_first(struct swap_cluster_list *list) 402 { 403 return cluster_next(&list->head); 404 } 405 406 static void cluster_list_init(struct swap_cluster_list *list) 407 { 408 cluster_set_null(&list->head); 409 cluster_set_null(&list->tail); 410 } 411 412 static void cluster_list_add_tail(struct swap_cluster_list *list, 413 struct swap_cluster_info *ci, 414 unsigned int idx) 415 { 416 if (cluster_list_empty(list)) { 417 cluster_set_next_flag(&list->head, idx, 0); 418 cluster_set_next_flag(&list->tail, idx, 0); 419 } else { 420 struct swap_cluster_info *ci_tail; 421 unsigned int tail = cluster_next(&list->tail); 422 423 /* 424 * Nested cluster lock, but both cluster locks are 425 * only acquired when we held swap_info_struct->lock 426 */ 427 ci_tail = ci + tail; 428 spin_lock_nested(&ci_tail->lock, SINGLE_DEPTH_NESTING); 429 cluster_set_next(ci_tail, idx); 430 spin_unlock(&ci_tail->lock); 431 cluster_set_next_flag(&list->tail, idx, 0); 432 } 433 } 434 435 static unsigned int cluster_list_del_first(struct swap_cluster_list *list, 436 struct swap_cluster_info *ci) 437 { 438 unsigned int idx; 439 440 idx = cluster_next(&list->head); 441 if (cluster_next(&list->tail) == idx) { 442 cluster_set_null(&list->head); 443 cluster_set_null(&list->tail); 444 } else 445 cluster_set_next_flag(&list->head, 446 cluster_next(&ci[idx]), 0); 447 448 return idx; 449 } 450 451 /* Add a cluster to discard list and schedule it to do discard */ 452 static void swap_cluster_schedule_discard(struct swap_info_struct *si, 453 unsigned int idx) 454 { 455 /* 456 * If scan_swap_map_slots() can't find a free cluster, it will check 457 * si->swap_map directly. To make sure the discarding cluster isn't 458 * taken by scan_swap_map_slots(), mark the swap entries bad (occupied). 459 * It will be cleared after discard 460 */ 461 memset(si->swap_map + idx * SWAPFILE_CLUSTER, 462 SWAP_MAP_BAD, SWAPFILE_CLUSTER); 463 464 cluster_list_add_tail(&si->discard_clusters, si->cluster_info, idx); 465 466 schedule_work(&si->discard_work); 467 } 468 469 static void __free_cluster(struct swap_info_struct *si, unsigned long idx) 470 { 471 struct swap_cluster_info *ci = si->cluster_info; 472 473 cluster_set_flag(ci + idx, CLUSTER_FLAG_FREE); 474 cluster_list_add_tail(&si->free_clusters, ci, idx); 475 } 476 477 /* 478 * Doing discard actually. After a cluster discard is finished, the cluster 479 * will be added to free cluster list. caller should hold si->lock. 480 */ 481 static void swap_do_scheduled_discard(struct swap_info_struct *si) 482 { 483 struct swap_cluster_info *info, *ci; 484 unsigned int idx; 485 486 info = si->cluster_info; 487 488 while (!cluster_list_empty(&si->discard_clusters)) { 489 idx = cluster_list_del_first(&si->discard_clusters, info); 490 spin_unlock(&si->lock); 491 492 discard_swap_cluster(si, idx * SWAPFILE_CLUSTER, 493 SWAPFILE_CLUSTER); 494 495 spin_lock(&si->lock); 496 ci = lock_cluster(si, idx * SWAPFILE_CLUSTER); 497 __free_cluster(si, idx); 498 memset(si->swap_map + idx * SWAPFILE_CLUSTER, 499 0, SWAPFILE_CLUSTER); 500 unlock_cluster(ci); 501 } 502 } 503 504 static void swap_discard_work(struct work_struct *work) 505 { 506 struct swap_info_struct *si; 507 508 si = container_of(work, struct swap_info_struct, discard_work); 509 510 spin_lock(&si->lock); 511 swap_do_scheduled_discard(si); 512 spin_unlock(&si->lock); 513 } 514 515 static void swap_users_ref_free(struct percpu_ref *ref) 516 { 517 struct swap_info_struct *si; 518 519 si = container_of(ref, struct swap_info_struct, users); 520 complete(&si->comp); 521 } 522 523 static void alloc_cluster(struct swap_info_struct *si, unsigned long idx) 524 { 525 struct swap_cluster_info *ci = si->cluster_info; 526 527 VM_BUG_ON(cluster_list_first(&si->free_clusters) != idx); 528 cluster_list_del_first(&si->free_clusters, ci); 529 cluster_set_count_flag(ci + idx, 0, 0); 530 } 531 532 static void free_cluster(struct swap_info_struct *si, unsigned long idx) 533 { 534 struct swap_cluster_info *ci = si->cluster_info + idx; 535 536 VM_BUG_ON(cluster_count(ci) != 0); 537 /* 538 * If the swap is discardable, prepare discard the cluster 539 * instead of free it immediately. The cluster will be freed 540 * after discard. 541 */ 542 if ((si->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) == 543 (SWP_WRITEOK | SWP_PAGE_DISCARD)) { 544 swap_cluster_schedule_discard(si, idx); 545 return; 546 } 547 548 __free_cluster(si, idx); 549 } 550 551 /* 552 * The cluster corresponding to page_nr will be used. The cluster will be 553 * removed from free cluster list and its usage counter will be increased. 554 */ 555 static void inc_cluster_info_page(struct swap_info_struct *p, 556 struct swap_cluster_info *cluster_info, unsigned long page_nr) 557 { 558 unsigned long idx = page_nr / SWAPFILE_CLUSTER; 559 560 if (!cluster_info) 561 return; 562 if (cluster_is_free(&cluster_info[idx])) 563 alloc_cluster(p, idx); 564 565 VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER); 566 cluster_set_count(&cluster_info[idx], 567 cluster_count(&cluster_info[idx]) + 1); 568 } 569 570 /* 571 * The cluster corresponding to page_nr decreases one usage. If the usage 572 * counter becomes 0, which means no page in the cluster is in using, we can 573 * optionally discard the cluster and add it to free cluster list. 574 */ 575 static void dec_cluster_info_page(struct swap_info_struct *p, 576 struct swap_cluster_info *cluster_info, unsigned long page_nr) 577 { 578 unsigned long idx = page_nr / SWAPFILE_CLUSTER; 579 580 if (!cluster_info) 581 return; 582 583 VM_BUG_ON(cluster_count(&cluster_info[idx]) == 0); 584 cluster_set_count(&cluster_info[idx], 585 cluster_count(&cluster_info[idx]) - 1); 586 587 if (cluster_count(&cluster_info[idx]) == 0) 588 free_cluster(p, idx); 589 } 590 591 /* 592 * It's possible scan_swap_map_slots() uses a free cluster in the middle of free 593 * cluster list. Avoiding such abuse to avoid list corruption. 594 */ 595 static bool 596 scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si, 597 unsigned long offset) 598 { 599 struct percpu_cluster *percpu_cluster; 600 bool conflict; 601 602 offset /= SWAPFILE_CLUSTER; 603 conflict = !cluster_list_empty(&si->free_clusters) && 604 offset != cluster_list_first(&si->free_clusters) && 605 cluster_is_free(&si->cluster_info[offset]); 606 607 if (!conflict) 608 return false; 609 610 percpu_cluster = this_cpu_ptr(si->percpu_cluster); 611 cluster_set_null(&percpu_cluster->index); 612 return true; 613 } 614 615 /* 616 * Try to get a swap entry from current cpu's swap entry pool (a cluster). This 617 * might involve allocating a new cluster for current CPU too. 618 */ 619 static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si, 620 unsigned long *offset, unsigned long *scan_base) 621 { 622 struct percpu_cluster *cluster; 623 struct swap_cluster_info *ci; 624 unsigned long tmp, max; 625 626 new_cluster: 627 cluster = this_cpu_ptr(si->percpu_cluster); 628 if (cluster_is_null(&cluster->index)) { 629 if (!cluster_list_empty(&si->free_clusters)) { 630 cluster->index = si->free_clusters.head; 631 cluster->next = cluster_next(&cluster->index) * 632 SWAPFILE_CLUSTER; 633 } else if (!cluster_list_empty(&si->discard_clusters)) { 634 /* 635 * we don't have free cluster but have some clusters in 636 * discarding, do discard now and reclaim them, then 637 * reread cluster_next_cpu since we dropped si->lock 638 */ 639 swap_do_scheduled_discard(si); 640 *scan_base = this_cpu_read(*si->cluster_next_cpu); 641 *offset = *scan_base; 642 goto new_cluster; 643 } else 644 return false; 645 } 646 647 /* 648 * Other CPUs can use our cluster if they can't find a free cluster, 649 * check if there is still free entry in the cluster 650 */ 651 tmp = cluster->next; 652 max = min_t(unsigned long, si->max, 653 (cluster_next(&cluster->index) + 1) * SWAPFILE_CLUSTER); 654 if (tmp < max) { 655 ci = lock_cluster(si, tmp); 656 while (tmp < max) { 657 if (!si->swap_map[tmp]) 658 break; 659 tmp++; 660 } 661 unlock_cluster(ci); 662 } 663 if (tmp >= max) { 664 cluster_set_null(&cluster->index); 665 goto new_cluster; 666 } 667 cluster->next = tmp + 1; 668 *offset = tmp; 669 *scan_base = tmp; 670 return true; 671 } 672 673 static void __del_from_avail_list(struct swap_info_struct *p) 674 { 675 int nid; 676 677 for_each_node(nid) 678 plist_del(&p->avail_lists[nid], &swap_avail_heads[nid]); 679 } 680 681 static void del_from_avail_list(struct swap_info_struct *p) 682 { 683 spin_lock(&swap_avail_lock); 684 __del_from_avail_list(p); 685 spin_unlock(&swap_avail_lock); 686 } 687 688 static void swap_range_alloc(struct swap_info_struct *si, unsigned long offset, 689 unsigned int nr_entries) 690 { 691 unsigned int end = offset + nr_entries - 1; 692 693 if (offset == si->lowest_bit) 694 si->lowest_bit += nr_entries; 695 if (end == si->highest_bit) 696 WRITE_ONCE(si->highest_bit, si->highest_bit - nr_entries); 697 si->inuse_pages += nr_entries; 698 if (si->inuse_pages == si->pages) { 699 si->lowest_bit = si->max; 700 si->highest_bit = 0; 701 del_from_avail_list(si); 702 } 703 } 704 705 static void add_to_avail_list(struct swap_info_struct *p) 706 { 707 int nid; 708 709 spin_lock(&swap_avail_lock); 710 for_each_node(nid) { 711 WARN_ON(!plist_node_empty(&p->avail_lists[nid])); 712 plist_add(&p->avail_lists[nid], &swap_avail_heads[nid]); 713 } 714 spin_unlock(&swap_avail_lock); 715 } 716 717 static void swap_range_free(struct swap_info_struct *si, unsigned long offset, 718 unsigned int nr_entries) 719 { 720 unsigned long begin = offset; 721 unsigned long end = offset + nr_entries - 1; 722 void (*swap_slot_free_notify)(struct block_device *, unsigned long); 723 724 if (offset < si->lowest_bit) 725 si->lowest_bit = offset; 726 if (end > si->highest_bit) { 727 bool was_full = !si->highest_bit; 728 729 WRITE_ONCE(si->highest_bit, end); 730 if (was_full && (si->flags & SWP_WRITEOK)) 731 add_to_avail_list(si); 732 } 733 atomic_long_add(nr_entries, &nr_swap_pages); 734 si->inuse_pages -= nr_entries; 735 if (si->flags & SWP_BLKDEV) 736 swap_slot_free_notify = 737 si->bdev->bd_disk->fops->swap_slot_free_notify; 738 else 739 swap_slot_free_notify = NULL; 740 while (offset <= end) { 741 arch_swap_invalidate_page(si->type, offset); 742 frontswap_invalidate_page(si->type, offset); 743 if (swap_slot_free_notify) 744 swap_slot_free_notify(si->bdev, offset); 745 offset++; 746 } 747 clear_shadow_from_swap_cache(si->type, begin, end); 748 } 749 750 static void set_cluster_next(struct swap_info_struct *si, unsigned long next) 751 { 752 unsigned long prev; 753 754 if (!(si->flags & SWP_SOLIDSTATE)) { 755 si->cluster_next = next; 756 return; 757 } 758 759 prev = this_cpu_read(*si->cluster_next_cpu); 760 /* 761 * Cross the swap address space size aligned trunk, choose 762 * another trunk randomly to avoid lock contention on swap 763 * address space if possible. 764 */ 765 if ((prev >> SWAP_ADDRESS_SPACE_SHIFT) != 766 (next >> SWAP_ADDRESS_SPACE_SHIFT)) { 767 /* No free swap slots available */ 768 if (si->highest_bit <= si->lowest_bit) 769 return; 770 next = si->lowest_bit + 771 prandom_u32_max(si->highest_bit - si->lowest_bit + 1); 772 next = ALIGN_DOWN(next, SWAP_ADDRESS_SPACE_PAGES); 773 next = max_t(unsigned int, next, si->lowest_bit); 774 } 775 this_cpu_write(*si->cluster_next_cpu, next); 776 } 777 778 static int scan_swap_map_slots(struct swap_info_struct *si, 779 unsigned char usage, int nr, 780 swp_entry_t slots[]) 781 { 782 struct swap_cluster_info *ci; 783 unsigned long offset; 784 unsigned long scan_base; 785 unsigned long last_in_cluster = 0; 786 int latency_ration = LATENCY_LIMIT; 787 int n_ret = 0; 788 bool scanned_many = false; 789 790 /* 791 * We try to cluster swap pages by allocating them sequentially 792 * in swap. Once we've allocated SWAPFILE_CLUSTER pages this 793 * way, however, we resort to first-free allocation, starting 794 * a new cluster. This prevents us from scattering swap pages 795 * all over the entire swap partition, so that we reduce 796 * overall disk seek times between swap pages. -- sct 797 * But we do now try to find an empty cluster. -Andrea 798 * And we let swap pages go all over an SSD partition. Hugh 799 */ 800 801 si->flags += SWP_SCANNING; 802 /* 803 * Use percpu scan base for SSD to reduce lock contention on 804 * cluster and swap cache. For HDD, sequential access is more 805 * important. 806 */ 807 if (si->flags & SWP_SOLIDSTATE) 808 scan_base = this_cpu_read(*si->cluster_next_cpu); 809 else 810 scan_base = si->cluster_next; 811 offset = scan_base; 812 813 /* SSD algorithm */ 814 if (si->cluster_info) { 815 if (!scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) 816 goto scan; 817 } else if (unlikely(!si->cluster_nr--)) { 818 if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) { 819 si->cluster_nr = SWAPFILE_CLUSTER - 1; 820 goto checks; 821 } 822 823 spin_unlock(&si->lock); 824 825 /* 826 * If seek is expensive, start searching for new cluster from 827 * start of partition, to minimize the span of allocated swap. 828 * If seek is cheap, that is the SWP_SOLIDSTATE si->cluster_info 829 * case, just handled by scan_swap_map_try_ssd_cluster() above. 830 */ 831 scan_base = offset = si->lowest_bit; 832 last_in_cluster = offset + SWAPFILE_CLUSTER - 1; 833 834 /* Locate the first empty (unaligned) cluster */ 835 for (; last_in_cluster <= si->highest_bit; offset++) { 836 if (si->swap_map[offset]) 837 last_in_cluster = offset + SWAPFILE_CLUSTER; 838 else if (offset == last_in_cluster) { 839 spin_lock(&si->lock); 840 offset -= SWAPFILE_CLUSTER - 1; 841 si->cluster_next = offset; 842 si->cluster_nr = SWAPFILE_CLUSTER - 1; 843 goto checks; 844 } 845 if (unlikely(--latency_ration < 0)) { 846 cond_resched(); 847 latency_ration = LATENCY_LIMIT; 848 } 849 } 850 851 offset = scan_base; 852 spin_lock(&si->lock); 853 si->cluster_nr = SWAPFILE_CLUSTER - 1; 854 } 855 856 checks: 857 if (si->cluster_info) { 858 while (scan_swap_map_ssd_cluster_conflict(si, offset)) { 859 /* take a break if we already got some slots */ 860 if (n_ret) 861 goto done; 862 if (!scan_swap_map_try_ssd_cluster(si, &offset, 863 &scan_base)) 864 goto scan; 865 } 866 } 867 if (!(si->flags & SWP_WRITEOK)) 868 goto no_page; 869 if (!si->highest_bit) 870 goto no_page; 871 if (offset > si->highest_bit) 872 scan_base = offset = si->lowest_bit; 873 874 ci = lock_cluster(si, offset); 875 /* reuse swap entry of cache-only swap if not busy. */ 876 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) { 877 int swap_was_freed; 878 unlock_cluster(ci); 879 spin_unlock(&si->lock); 880 swap_was_freed = __try_to_reclaim_swap(si, offset, TTRS_ANYWAY); 881 spin_lock(&si->lock); 882 /* entry was freed successfully, try to use this again */ 883 if (swap_was_freed) 884 goto checks; 885 goto scan; /* check next one */ 886 } 887 888 if (si->swap_map[offset]) { 889 unlock_cluster(ci); 890 if (!n_ret) 891 goto scan; 892 else 893 goto done; 894 } 895 WRITE_ONCE(si->swap_map[offset], usage); 896 inc_cluster_info_page(si, si->cluster_info, offset); 897 unlock_cluster(ci); 898 899 swap_range_alloc(si, offset, 1); 900 slots[n_ret++] = swp_entry(si->type, offset); 901 902 /* got enough slots or reach max slots? */ 903 if ((n_ret == nr) || (offset >= si->highest_bit)) 904 goto done; 905 906 /* search for next available slot */ 907 908 /* time to take a break? */ 909 if (unlikely(--latency_ration < 0)) { 910 if (n_ret) 911 goto done; 912 spin_unlock(&si->lock); 913 cond_resched(); 914 spin_lock(&si->lock); 915 latency_ration = LATENCY_LIMIT; 916 } 917 918 /* try to get more slots in cluster */ 919 if (si->cluster_info) { 920 if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) 921 goto checks; 922 } else if (si->cluster_nr && !si->swap_map[++offset]) { 923 /* non-ssd case, still more slots in cluster? */ 924 --si->cluster_nr; 925 goto checks; 926 } 927 928 /* 929 * Even if there's no free clusters available (fragmented), 930 * try to scan a little more quickly with lock held unless we 931 * have scanned too many slots already. 932 */ 933 if (!scanned_many) { 934 unsigned long scan_limit; 935 936 if (offset < scan_base) 937 scan_limit = scan_base; 938 else 939 scan_limit = si->highest_bit; 940 for (; offset <= scan_limit && --latency_ration > 0; 941 offset++) { 942 if (!si->swap_map[offset]) 943 goto checks; 944 } 945 } 946 947 done: 948 set_cluster_next(si, offset + 1); 949 si->flags -= SWP_SCANNING; 950 return n_ret; 951 952 scan: 953 spin_unlock(&si->lock); 954 while (++offset <= READ_ONCE(si->highest_bit)) { 955 if (data_race(!si->swap_map[offset])) { 956 spin_lock(&si->lock); 957 goto checks; 958 } 959 if (vm_swap_full() && 960 READ_ONCE(si->swap_map[offset]) == SWAP_HAS_CACHE) { 961 spin_lock(&si->lock); 962 goto checks; 963 } 964 if (unlikely(--latency_ration < 0)) { 965 cond_resched(); 966 latency_ration = LATENCY_LIMIT; 967 scanned_many = true; 968 } 969 } 970 offset = si->lowest_bit; 971 while (offset < scan_base) { 972 if (data_race(!si->swap_map[offset])) { 973 spin_lock(&si->lock); 974 goto checks; 975 } 976 if (vm_swap_full() && 977 READ_ONCE(si->swap_map[offset]) == SWAP_HAS_CACHE) { 978 spin_lock(&si->lock); 979 goto checks; 980 } 981 if (unlikely(--latency_ration < 0)) { 982 cond_resched(); 983 latency_ration = LATENCY_LIMIT; 984 scanned_many = true; 985 } 986 offset++; 987 } 988 spin_lock(&si->lock); 989 990 no_page: 991 si->flags -= SWP_SCANNING; 992 return n_ret; 993 } 994 995 static int swap_alloc_cluster(struct swap_info_struct *si, swp_entry_t *slot) 996 { 997 unsigned long idx; 998 struct swap_cluster_info *ci; 999 unsigned long offset; 1000 1001 /* 1002 * Should not even be attempting cluster allocations when huge 1003 * page swap is disabled. Warn and fail the allocation. 1004 */ 1005 if (!IS_ENABLED(CONFIG_THP_SWAP)) { 1006 VM_WARN_ON_ONCE(1); 1007 return 0; 1008 } 1009 1010 if (cluster_list_empty(&si->free_clusters)) 1011 return 0; 1012 1013 idx = cluster_list_first(&si->free_clusters); 1014 offset = idx * SWAPFILE_CLUSTER; 1015 ci = lock_cluster(si, offset); 1016 alloc_cluster(si, idx); 1017 cluster_set_count_flag(ci, SWAPFILE_CLUSTER, CLUSTER_FLAG_HUGE); 1018 1019 memset(si->swap_map + offset, SWAP_HAS_CACHE, SWAPFILE_CLUSTER); 1020 unlock_cluster(ci); 1021 swap_range_alloc(si, offset, SWAPFILE_CLUSTER); 1022 *slot = swp_entry(si->type, offset); 1023 1024 return 1; 1025 } 1026 1027 static void swap_free_cluster(struct swap_info_struct *si, unsigned long idx) 1028 { 1029 unsigned long offset = idx * SWAPFILE_CLUSTER; 1030 struct swap_cluster_info *ci; 1031 1032 ci = lock_cluster(si, offset); 1033 memset(si->swap_map + offset, 0, SWAPFILE_CLUSTER); 1034 cluster_set_count_flag(ci, 0, 0); 1035 free_cluster(si, idx); 1036 unlock_cluster(ci); 1037 swap_range_free(si, offset, SWAPFILE_CLUSTER); 1038 } 1039 1040 int get_swap_pages(int n_goal, swp_entry_t swp_entries[], int entry_size) 1041 { 1042 unsigned long size = swap_entry_size(entry_size); 1043 struct swap_info_struct *si, *next; 1044 long avail_pgs; 1045 int n_ret = 0; 1046 int node; 1047 1048 /* Only single cluster request supported */ 1049 WARN_ON_ONCE(n_goal > 1 && size == SWAPFILE_CLUSTER); 1050 1051 spin_lock(&swap_avail_lock); 1052 1053 avail_pgs = atomic_long_read(&nr_swap_pages) / size; 1054 if (avail_pgs <= 0) { 1055 spin_unlock(&swap_avail_lock); 1056 goto noswap; 1057 } 1058 1059 n_goal = min3((long)n_goal, (long)SWAP_BATCH, avail_pgs); 1060 1061 atomic_long_sub(n_goal * size, &nr_swap_pages); 1062 1063 start_over: 1064 node = numa_node_id(); 1065 plist_for_each_entry_safe(si, next, &swap_avail_heads[node], avail_lists[node]) { 1066 /* requeue si to after same-priority siblings */ 1067 plist_requeue(&si->avail_lists[node], &swap_avail_heads[node]); 1068 spin_unlock(&swap_avail_lock); 1069 spin_lock(&si->lock); 1070 if (!si->highest_bit || !(si->flags & SWP_WRITEOK)) { 1071 spin_lock(&swap_avail_lock); 1072 if (plist_node_empty(&si->avail_lists[node])) { 1073 spin_unlock(&si->lock); 1074 goto nextsi; 1075 } 1076 WARN(!si->highest_bit, 1077 "swap_info %d in list but !highest_bit\n", 1078 si->type); 1079 WARN(!(si->flags & SWP_WRITEOK), 1080 "swap_info %d in list but !SWP_WRITEOK\n", 1081 si->type); 1082 __del_from_avail_list(si); 1083 spin_unlock(&si->lock); 1084 goto nextsi; 1085 } 1086 if (size == SWAPFILE_CLUSTER) { 1087 if (si->flags & SWP_BLKDEV) 1088 n_ret = swap_alloc_cluster(si, swp_entries); 1089 } else 1090 n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE, 1091 n_goal, swp_entries); 1092 spin_unlock(&si->lock); 1093 if (n_ret || size == SWAPFILE_CLUSTER) 1094 goto check_out; 1095 pr_debug("scan_swap_map of si %d failed to find offset\n", 1096 si->type); 1097 1098 spin_lock(&swap_avail_lock); 1099 nextsi: 1100 /* 1101 * if we got here, it's likely that si was almost full before, 1102 * and since scan_swap_map_slots() can drop the si->lock, 1103 * multiple callers probably all tried to get a page from the 1104 * same si and it filled up before we could get one; or, the si 1105 * filled up between us dropping swap_avail_lock and taking 1106 * si->lock. Since we dropped the swap_avail_lock, the 1107 * swap_avail_head list may have been modified; so if next is 1108 * still in the swap_avail_head list then try it, otherwise 1109 * start over if we have not gotten any slots. 1110 */ 1111 if (plist_node_empty(&next->avail_lists[node])) 1112 goto start_over; 1113 } 1114 1115 spin_unlock(&swap_avail_lock); 1116 1117 check_out: 1118 if (n_ret < n_goal) 1119 atomic_long_add((long)(n_goal - n_ret) * size, 1120 &nr_swap_pages); 1121 noswap: 1122 return n_ret; 1123 } 1124 1125 static struct swap_info_struct *__swap_info_get(swp_entry_t entry) 1126 { 1127 struct swap_info_struct *p; 1128 unsigned long offset; 1129 1130 if (!entry.val) 1131 goto out; 1132 p = swp_swap_info(entry); 1133 if (!p) 1134 goto bad_nofile; 1135 if (data_race(!(p->flags & SWP_USED))) 1136 goto bad_device; 1137 offset = swp_offset(entry); 1138 if (offset >= p->max) 1139 goto bad_offset; 1140 return p; 1141 1142 bad_offset: 1143 pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val); 1144 goto out; 1145 bad_device: 1146 pr_err("%s: %s%08lx\n", __func__, Unused_file, entry.val); 1147 goto out; 1148 bad_nofile: 1149 pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val); 1150 out: 1151 return NULL; 1152 } 1153 1154 static struct swap_info_struct *_swap_info_get(swp_entry_t entry) 1155 { 1156 struct swap_info_struct *p; 1157 1158 p = __swap_info_get(entry); 1159 if (!p) 1160 goto out; 1161 if (data_race(!p->swap_map[swp_offset(entry)])) 1162 goto bad_free; 1163 return p; 1164 1165 bad_free: 1166 pr_err("%s: %s%08lx\n", __func__, Unused_offset, entry.val); 1167 out: 1168 return NULL; 1169 } 1170 1171 static struct swap_info_struct *swap_info_get_cont(swp_entry_t entry, 1172 struct swap_info_struct *q) 1173 { 1174 struct swap_info_struct *p; 1175 1176 p = _swap_info_get(entry); 1177 1178 if (p != q) { 1179 if (q != NULL) 1180 spin_unlock(&q->lock); 1181 if (p != NULL) 1182 spin_lock(&p->lock); 1183 } 1184 return p; 1185 } 1186 1187 static unsigned char __swap_entry_free_locked(struct swap_info_struct *p, 1188 unsigned long offset, 1189 unsigned char usage) 1190 { 1191 unsigned char count; 1192 unsigned char has_cache; 1193 1194 count = p->swap_map[offset]; 1195 1196 has_cache = count & SWAP_HAS_CACHE; 1197 count &= ~SWAP_HAS_CACHE; 1198 1199 if (usage == SWAP_HAS_CACHE) { 1200 VM_BUG_ON(!has_cache); 1201 has_cache = 0; 1202 } else if (count == SWAP_MAP_SHMEM) { 1203 /* 1204 * Or we could insist on shmem.c using a special 1205 * swap_shmem_free() and free_shmem_swap_and_cache()... 1206 */ 1207 count = 0; 1208 } else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) { 1209 if (count == COUNT_CONTINUED) { 1210 if (swap_count_continued(p, offset, count)) 1211 count = SWAP_MAP_MAX | COUNT_CONTINUED; 1212 else 1213 count = SWAP_MAP_MAX; 1214 } else 1215 count--; 1216 } 1217 1218 usage = count | has_cache; 1219 if (usage) 1220 WRITE_ONCE(p->swap_map[offset], usage); 1221 else 1222 WRITE_ONCE(p->swap_map[offset], SWAP_HAS_CACHE); 1223 1224 return usage; 1225 } 1226 1227 /* 1228 * Check whether swap entry is valid in the swap device. If so, 1229 * return pointer to swap_info_struct, and keep the swap entry valid 1230 * via preventing the swap device from being swapoff, until 1231 * put_swap_device() is called. Otherwise return NULL. 1232 * 1233 * Notice that swapoff or swapoff+swapon can still happen before the 1234 * percpu_ref_tryget_live() in get_swap_device() or after the 1235 * percpu_ref_put() in put_swap_device() if there isn't any other way 1236 * to prevent swapoff, such as page lock, page table lock, etc. The 1237 * caller must be prepared for that. For example, the following 1238 * situation is possible. 1239 * 1240 * CPU1 CPU2 1241 * do_swap_page() 1242 * ... swapoff+swapon 1243 * __read_swap_cache_async() 1244 * swapcache_prepare() 1245 * __swap_duplicate() 1246 * // check swap_map 1247 * // verify PTE not changed 1248 * 1249 * In __swap_duplicate(), the swap_map need to be checked before 1250 * changing partly because the specified swap entry may be for another 1251 * swap device which has been swapoff. And in do_swap_page(), after 1252 * the page is read from the swap device, the PTE is verified not 1253 * changed with the page table locked to check whether the swap device 1254 * has been swapoff or swapoff+swapon. 1255 */ 1256 struct swap_info_struct *get_swap_device(swp_entry_t entry) 1257 { 1258 struct swap_info_struct *si; 1259 unsigned long offset; 1260 1261 if (!entry.val) 1262 goto out; 1263 si = swp_swap_info(entry); 1264 if (!si) 1265 goto bad_nofile; 1266 if (!percpu_ref_tryget_live(&si->users)) 1267 goto out; 1268 /* 1269 * Guarantee the si->users are checked before accessing other 1270 * fields of swap_info_struct. 1271 * 1272 * Paired with the spin_unlock() after setup_swap_info() in 1273 * enable_swap_info(). 1274 */ 1275 smp_rmb(); 1276 offset = swp_offset(entry); 1277 if (offset >= si->max) 1278 goto put_out; 1279 1280 return si; 1281 bad_nofile: 1282 pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val); 1283 out: 1284 return NULL; 1285 put_out: 1286 percpu_ref_put(&si->users); 1287 return NULL; 1288 } 1289 1290 static unsigned char __swap_entry_free(struct swap_info_struct *p, 1291 swp_entry_t entry) 1292 { 1293 struct swap_cluster_info *ci; 1294 unsigned long offset = swp_offset(entry); 1295 unsigned char usage; 1296 1297 ci = lock_cluster_or_swap_info(p, offset); 1298 usage = __swap_entry_free_locked(p, offset, 1); 1299 unlock_cluster_or_swap_info(p, ci); 1300 if (!usage) 1301 free_swap_slot(entry); 1302 1303 return usage; 1304 } 1305 1306 static void swap_entry_free(struct swap_info_struct *p, swp_entry_t entry) 1307 { 1308 struct swap_cluster_info *ci; 1309 unsigned long offset = swp_offset(entry); 1310 unsigned char count; 1311 1312 ci = lock_cluster(p, offset); 1313 count = p->swap_map[offset]; 1314 VM_BUG_ON(count != SWAP_HAS_CACHE); 1315 p->swap_map[offset] = 0; 1316 dec_cluster_info_page(p, p->cluster_info, offset); 1317 unlock_cluster(ci); 1318 1319 mem_cgroup_uncharge_swap(entry, 1); 1320 swap_range_free(p, offset, 1); 1321 } 1322 1323 /* 1324 * Caller has made sure that the swap device corresponding to entry 1325 * is still around or has not been recycled. 1326 */ 1327 void swap_free(swp_entry_t entry) 1328 { 1329 struct swap_info_struct *p; 1330 1331 p = _swap_info_get(entry); 1332 if (p) 1333 __swap_entry_free(p, entry); 1334 } 1335 1336 /* 1337 * Called after dropping swapcache to decrease refcnt to swap entries. 1338 */ 1339 void put_swap_page(struct page *page, swp_entry_t entry) 1340 { 1341 unsigned long offset = swp_offset(entry); 1342 unsigned long idx = offset / SWAPFILE_CLUSTER; 1343 struct swap_cluster_info *ci; 1344 struct swap_info_struct *si; 1345 unsigned char *map; 1346 unsigned int i, free_entries = 0; 1347 unsigned char val; 1348 int size = swap_entry_size(thp_nr_pages(page)); 1349 1350 si = _swap_info_get(entry); 1351 if (!si) 1352 return; 1353 1354 ci = lock_cluster_or_swap_info(si, offset); 1355 if (size == SWAPFILE_CLUSTER) { 1356 VM_BUG_ON(!cluster_is_huge(ci)); 1357 map = si->swap_map + offset; 1358 for (i = 0; i < SWAPFILE_CLUSTER; i++) { 1359 val = map[i]; 1360 VM_BUG_ON(!(val & SWAP_HAS_CACHE)); 1361 if (val == SWAP_HAS_CACHE) 1362 free_entries++; 1363 } 1364 cluster_clear_huge(ci); 1365 if (free_entries == SWAPFILE_CLUSTER) { 1366 unlock_cluster_or_swap_info(si, ci); 1367 spin_lock(&si->lock); 1368 mem_cgroup_uncharge_swap(entry, SWAPFILE_CLUSTER); 1369 swap_free_cluster(si, idx); 1370 spin_unlock(&si->lock); 1371 return; 1372 } 1373 } 1374 for (i = 0; i < size; i++, entry.val++) { 1375 if (!__swap_entry_free_locked(si, offset + i, SWAP_HAS_CACHE)) { 1376 unlock_cluster_or_swap_info(si, ci); 1377 free_swap_slot(entry); 1378 if (i == size - 1) 1379 return; 1380 lock_cluster_or_swap_info(si, offset); 1381 } 1382 } 1383 unlock_cluster_or_swap_info(si, ci); 1384 } 1385 1386 #ifdef CONFIG_THP_SWAP 1387 int split_swap_cluster(swp_entry_t entry) 1388 { 1389 struct swap_info_struct *si; 1390 struct swap_cluster_info *ci; 1391 unsigned long offset = swp_offset(entry); 1392 1393 si = _swap_info_get(entry); 1394 if (!si) 1395 return -EBUSY; 1396 ci = lock_cluster(si, offset); 1397 cluster_clear_huge(ci); 1398 unlock_cluster(ci); 1399 return 0; 1400 } 1401 #endif 1402 1403 static int swp_entry_cmp(const void *ent1, const void *ent2) 1404 { 1405 const swp_entry_t *e1 = ent1, *e2 = ent2; 1406 1407 return (int)swp_type(*e1) - (int)swp_type(*e2); 1408 } 1409 1410 void swapcache_free_entries(swp_entry_t *entries, int n) 1411 { 1412 struct swap_info_struct *p, *prev; 1413 int i; 1414 1415 if (n <= 0) 1416 return; 1417 1418 prev = NULL; 1419 p = NULL; 1420 1421 /* 1422 * Sort swap entries by swap device, so each lock is only taken once. 1423 * nr_swapfiles isn't absolutely correct, but the overhead of sort() is 1424 * so low that it isn't necessary to optimize further. 1425 */ 1426 if (nr_swapfiles > 1) 1427 sort(entries, n, sizeof(entries[0]), swp_entry_cmp, NULL); 1428 for (i = 0; i < n; ++i) { 1429 p = swap_info_get_cont(entries[i], prev); 1430 if (p) 1431 swap_entry_free(p, entries[i]); 1432 prev = p; 1433 } 1434 if (p) 1435 spin_unlock(&p->lock); 1436 } 1437 1438 /* 1439 * How many references to page are currently swapped out? 1440 * This does not give an exact answer when swap count is continued, 1441 * but does include the high COUNT_CONTINUED flag to allow for that. 1442 */ 1443 int page_swapcount(struct page *page) 1444 { 1445 int count = 0; 1446 struct swap_info_struct *p; 1447 struct swap_cluster_info *ci; 1448 swp_entry_t entry; 1449 unsigned long offset; 1450 1451 entry.val = page_private(page); 1452 p = _swap_info_get(entry); 1453 if (p) { 1454 offset = swp_offset(entry); 1455 ci = lock_cluster_or_swap_info(p, offset); 1456 count = swap_count(p->swap_map[offset]); 1457 unlock_cluster_or_swap_info(p, ci); 1458 } 1459 return count; 1460 } 1461 1462 int __swap_count(swp_entry_t entry) 1463 { 1464 struct swap_info_struct *si; 1465 pgoff_t offset = swp_offset(entry); 1466 int count = 0; 1467 1468 si = get_swap_device(entry); 1469 if (si) { 1470 count = swap_count(si->swap_map[offset]); 1471 put_swap_device(si); 1472 } 1473 return count; 1474 } 1475 1476 static int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry) 1477 { 1478 int count = 0; 1479 pgoff_t offset = swp_offset(entry); 1480 struct swap_cluster_info *ci; 1481 1482 ci = lock_cluster_or_swap_info(si, offset); 1483 count = swap_count(si->swap_map[offset]); 1484 unlock_cluster_or_swap_info(si, ci); 1485 return count; 1486 } 1487 1488 /* 1489 * How many references to @entry are currently swapped out? 1490 * This does not give an exact answer when swap count is continued, 1491 * but does include the high COUNT_CONTINUED flag to allow for that. 1492 */ 1493 int __swp_swapcount(swp_entry_t entry) 1494 { 1495 int count = 0; 1496 struct swap_info_struct *si; 1497 1498 si = get_swap_device(entry); 1499 if (si) { 1500 count = swap_swapcount(si, entry); 1501 put_swap_device(si); 1502 } 1503 return count; 1504 } 1505 1506 /* 1507 * How many references to @entry are currently swapped out? 1508 * This considers COUNT_CONTINUED so it returns exact answer. 1509 */ 1510 int swp_swapcount(swp_entry_t entry) 1511 { 1512 int count, tmp_count, n; 1513 struct swap_info_struct *p; 1514 struct swap_cluster_info *ci; 1515 struct page *page; 1516 pgoff_t offset; 1517 unsigned char *map; 1518 1519 p = _swap_info_get(entry); 1520 if (!p) 1521 return 0; 1522 1523 offset = swp_offset(entry); 1524 1525 ci = lock_cluster_or_swap_info(p, offset); 1526 1527 count = swap_count(p->swap_map[offset]); 1528 if (!(count & COUNT_CONTINUED)) 1529 goto out; 1530 1531 count &= ~COUNT_CONTINUED; 1532 n = SWAP_MAP_MAX + 1; 1533 1534 page = vmalloc_to_page(p->swap_map + offset); 1535 offset &= ~PAGE_MASK; 1536 VM_BUG_ON(page_private(page) != SWP_CONTINUED); 1537 1538 do { 1539 page = list_next_entry(page, lru); 1540 map = kmap_atomic(page); 1541 tmp_count = map[offset]; 1542 kunmap_atomic(map); 1543 1544 count += (tmp_count & ~COUNT_CONTINUED) * n; 1545 n *= (SWAP_CONT_MAX + 1); 1546 } while (tmp_count & COUNT_CONTINUED); 1547 out: 1548 unlock_cluster_or_swap_info(p, ci); 1549 return count; 1550 } 1551 1552 static bool swap_page_trans_huge_swapped(struct swap_info_struct *si, 1553 swp_entry_t entry) 1554 { 1555 struct swap_cluster_info *ci; 1556 unsigned char *map = si->swap_map; 1557 unsigned long roffset = swp_offset(entry); 1558 unsigned long offset = round_down(roffset, SWAPFILE_CLUSTER); 1559 int i; 1560 bool ret = false; 1561 1562 ci = lock_cluster_or_swap_info(si, offset); 1563 if (!ci || !cluster_is_huge(ci)) { 1564 if (swap_count(map[roffset])) 1565 ret = true; 1566 goto unlock_out; 1567 } 1568 for (i = 0; i < SWAPFILE_CLUSTER; i++) { 1569 if (swap_count(map[offset + i])) { 1570 ret = true; 1571 break; 1572 } 1573 } 1574 unlock_out: 1575 unlock_cluster_or_swap_info(si, ci); 1576 return ret; 1577 } 1578 1579 static bool page_swapped(struct page *page) 1580 { 1581 swp_entry_t entry; 1582 struct swap_info_struct *si; 1583 1584 if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!PageTransCompound(page))) 1585 return page_swapcount(page) != 0; 1586 1587 page = compound_head(page); 1588 entry.val = page_private(page); 1589 si = _swap_info_get(entry); 1590 if (si) 1591 return swap_page_trans_huge_swapped(si, entry); 1592 return false; 1593 } 1594 1595 /* 1596 * If swap is getting full, or if there are no more mappings of this page, 1597 * then try_to_free_swap is called to free its swap space. 1598 */ 1599 int try_to_free_swap(struct page *page) 1600 { 1601 VM_BUG_ON_PAGE(!PageLocked(page), page); 1602 1603 if (!PageSwapCache(page)) 1604 return 0; 1605 if (PageWriteback(page)) 1606 return 0; 1607 if (page_swapped(page)) 1608 return 0; 1609 1610 /* 1611 * Once hibernation has begun to create its image of memory, 1612 * there's a danger that one of the calls to try_to_free_swap() 1613 * - most probably a call from __try_to_reclaim_swap() while 1614 * hibernation is allocating its own swap pages for the image, 1615 * but conceivably even a call from memory reclaim - will free 1616 * the swap from a page which has already been recorded in the 1617 * image as a clean swapcache page, and then reuse its swap for 1618 * another page of the image. On waking from hibernation, the 1619 * original page might be freed under memory pressure, then 1620 * later read back in from swap, now with the wrong data. 1621 * 1622 * Hibernation suspends storage while it is writing the image 1623 * to disk so check that here. 1624 */ 1625 if (pm_suspended_storage()) 1626 return 0; 1627 1628 page = compound_head(page); 1629 delete_from_swap_cache(page); 1630 SetPageDirty(page); 1631 return 1; 1632 } 1633 1634 /* 1635 * Free the swap entry like above, but also try to 1636 * free the page cache entry if it is the last user. 1637 */ 1638 int free_swap_and_cache(swp_entry_t entry) 1639 { 1640 struct swap_info_struct *p; 1641 unsigned char count; 1642 1643 if (non_swap_entry(entry)) 1644 return 1; 1645 1646 p = _swap_info_get(entry); 1647 if (p) { 1648 count = __swap_entry_free(p, entry); 1649 if (count == SWAP_HAS_CACHE && 1650 !swap_page_trans_huge_swapped(p, entry)) 1651 __try_to_reclaim_swap(p, swp_offset(entry), 1652 TTRS_UNMAPPED | TTRS_FULL); 1653 } 1654 return p != NULL; 1655 } 1656 1657 #ifdef CONFIG_HIBERNATION 1658 1659 swp_entry_t get_swap_page_of_type(int type) 1660 { 1661 struct swap_info_struct *si = swap_type_to_swap_info(type); 1662 swp_entry_t entry = {0}; 1663 1664 if (!si) 1665 goto fail; 1666 1667 /* This is called for allocating swap entry, not cache */ 1668 spin_lock(&si->lock); 1669 if ((si->flags & SWP_WRITEOK) && scan_swap_map_slots(si, 1, 1, &entry)) 1670 atomic_long_dec(&nr_swap_pages); 1671 spin_unlock(&si->lock); 1672 fail: 1673 return entry; 1674 } 1675 1676 /* 1677 * Find the swap type that corresponds to given device (if any). 1678 * 1679 * @offset - number of the PAGE_SIZE-sized block of the device, starting 1680 * from 0, in which the swap header is expected to be located. 1681 * 1682 * This is needed for the suspend to disk (aka swsusp). 1683 */ 1684 int swap_type_of(dev_t device, sector_t offset) 1685 { 1686 int type; 1687 1688 if (!device) 1689 return -1; 1690 1691 spin_lock(&swap_lock); 1692 for (type = 0; type < nr_swapfiles; type++) { 1693 struct swap_info_struct *sis = swap_info[type]; 1694 1695 if (!(sis->flags & SWP_WRITEOK)) 1696 continue; 1697 1698 if (device == sis->bdev->bd_dev) { 1699 struct swap_extent *se = first_se(sis); 1700 1701 if (se->start_block == offset) { 1702 spin_unlock(&swap_lock); 1703 return type; 1704 } 1705 } 1706 } 1707 spin_unlock(&swap_lock); 1708 return -ENODEV; 1709 } 1710 1711 int find_first_swap(dev_t *device) 1712 { 1713 int type; 1714 1715 spin_lock(&swap_lock); 1716 for (type = 0; type < nr_swapfiles; type++) { 1717 struct swap_info_struct *sis = swap_info[type]; 1718 1719 if (!(sis->flags & SWP_WRITEOK)) 1720 continue; 1721 *device = sis->bdev->bd_dev; 1722 spin_unlock(&swap_lock); 1723 return type; 1724 } 1725 spin_unlock(&swap_lock); 1726 return -ENODEV; 1727 } 1728 1729 /* 1730 * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev 1731 * corresponding to given index in swap_info (swap type). 1732 */ 1733 sector_t swapdev_block(int type, pgoff_t offset) 1734 { 1735 struct swap_info_struct *si = swap_type_to_swap_info(type); 1736 struct swap_extent *se; 1737 1738 if (!si || !(si->flags & SWP_WRITEOK)) 1739 return 0; 1740 se = offset_to_swap_extent(si, offset); 1741 return se->start_block + (offset - se->start_page); 1742 } 1743 1744 /* 1745 * Return either the total number of swap pages of given type, or the number 1746 * of free pages of that type (depending on @free) 1747 * 1748 * This is needed for software suspend 1749 */ 1750 unsigned int count_swap_pages(int type, int free) 1751 { 1752 unsigned int n = 0; 1753 1754 spin_lock(&swap_lock); 1755 if ((unsigned int)type < nr_swapfiles) { 1756 struct swap_info_struct *sis = swap_info[type]; 1757 1758 spin_lock(&sis->lock); 1759 if (sis->flags & SWP_WRITEOK) { 1760 n = sis->pages; 1761 if (free) 1762 n -= sis->inuse_pages; 1763 } 1764 spin_unlock(&sis->lock); 1765 } 1766 spin_unlock(&swap_lock); 1767 return n; 1768 } 1769 #endif /* CONFIG_HIBERNATION */ 1770 1771 static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte) 1772 { 1773 return pte_same(pte_swp_clear_flags(pte), swp_pte); 1774 } 1775 1776 /* 1777 * No need to decide whether this PTE shares the swap entry with others, 1778 * just let do_wp_page work it out if a write is requested later - to 1779 * force COW, vm_page_prot omits write permission from any private vma. 1780 */ 1781 static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, 1782 unsigned long addr, swp_entry_t entry, struct page *page) 1783 { 1784 struct page *swapcache; 1785 spinlock_t *ptl; 1786 pte_t *pte; 1787 int ret = 1; 1788 1789 swapcache = page; 1790 page = ksm_might_need_to_copy(page, vma, addr); 1791 if (unlikely(!page)) 1792 return -ENOMEM; 1793 1794 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 1795 if (unlikely(!pte_same_as_swp(*pte, swp_entry_to_pte(entry)))) { 1796 ret = 0; 1797 goto out; 1798 } 1799 1800 dec_mm_counter(vma->vm_mm, MM_SWAPENTS); 1801 inc_mm_counter(vma->vm_mm, MM_ANONPAGES); 1802 get_page(page); 1803 if (page == swapcache) { 1804 page_add_anon_rmap(page, vma, addr, false); 1805 } else { /* ksm created a completely new copy */ 1806 page_add_new_anon_rmap(page, vma, addr, false); 1807 lru_cache_add_inactive_or_unevictable(page, vma); 1808 } 1809 set_pte_at(vma->vm_mm, addr, pte, 1810 pte_mkold(mk_pte(page, vma->vm_page_prot))); 1811 swap_free(entry); 1812 out: 1813 pte_unmap_unlock(pte, ptl); 1814 if (page != swapcache) { 1815 unlock_page(page); 1816 put_page(page); 1817 } 1818 return ret; 1819 } 1820 1821 static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 1822 unsigned long addr, unsigned long end, 1823 unsigned int type) 1824 { 1825 struct page *page; 1826 swp_entry_t entry; 1827 pte_t *pte; 1828 struct swap_info_struct *si; 1829 unsigned long offset; 1830 int ret = 0; 1831 volatile unsigned char *swap_map; 1832 1833 si = swap_info[type]; 1834 pte = pte_offset_map(pmd, addr); 1835 do { 1836 if (!is_swap_pte(*pte)) 1837 continue; 1838 1839 entry = pte_to_swp_entry(*pte); 1840 if (swp_type(entry) != type) 1841 continue; 1842 1843 offset = swp_offset(entry); 1844 pte_unmap(pte); 1845 swap_map = &si->swap_map[offset]; 1846 page = lookup_swap_cache(entry, vma, addr); 1847 if (!page) { 1848 struct vm_fault vmf = { 1849 .vma = vma, 1850 .address = addr, 1851 .real_address = addr, 1852 .pmd = pmd, 1853 }; 1854 1855 page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, 1856 &vmf); 1857 } 1858 if (!page) { 1859 if (*swap_map == 0 || *swap_map == SWAP_MAP_BAD) 1860 goto try_next; 1861 return -ENOMEM; 1862 } 1863 1864 lock_page(page); 1865 wait_on_page_writeback(page); 1866 ret = unuse_pte(vma, pmd, addr, entry, page); 1867 if (ret < 0) { 1868 unlock_page(page); 1869 put_page(page); 1870 goto out; 1871 } 1872 1873 try_to_free_swap(page); 1874 unlock_page(page); 1875 put_page(page); 1876 try_next: 1877 pte = pte_offset_map(pmd, addr); 1878 } while (pte++, addr += PAGE_SIZE, addr != end); 1879 pte_unmap(pte - 1); 1880 1881 ret = 0; 1882 out: 1883 return ret; 1884 } 1885 1886 static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, 1887 unsigned long addr, unsigned long end, 1888 unsigned int type) 1889 { 1890 pmd_t *pmd; 1891 unsigned long next; 1892 int ret; 1893 1894 pmd = pmd_offset(pud, addr); 1895 do { 1896 cond_resched(); 1897 next = pmd_addr_end(addr, end); 1898 if (pmd_none_or_trans_huge_or_clear_bad(pmd)) 1899 continue; 1900 ret = unuse_pte_range(vma, pmd, addr, next, type); 1901 if (ret) 1902 return ret; 1903 } while (pmd++, addr = next, addr != end); 1904 return 0; 1905 } 1906 1907 static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, 1908 unsigned long addr, unsigned long end, 1909 unsigned int type) 1910 { 1911 pud_t *pud; 1912 unsigned long next; 1913 int ret; 1914 1915 pud = pud_offset(p4d, addr); 1916 do { 1917 next = pud_addr_end(addr, end); 1918 if (pud_none_or_clear_bad(pud)) 1919 continue; 1920 ret = unuse_pmd_range(vma, pud, addr, next, type); 1921 if (ret) 1922 return ret; 1923 } while (pud++, addr = next, addr != end); 1924 return 0; 1925 } 1926 1927 static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, 1928 unsigned long addr, unsigned long end, 1929 unsigned int type) 1930 { 1931 p4d_t *p4d; 1932 unsigned long next; 1933 int ret; 1934 1935 p4d = p4d_offset(pgd, addr); 1936 do { 1937 next = p4d_addr_end(addr, end); 1938 if (p4d_none_or_clear_bad(p4d)) 1939 continue; 1940 ret = unuse_pud_range(vma, p4d, addr, next, type); 1941 if (ret) 1942 return ret; 1943 } while (p4d++, addr = next, addr != end); 1944 return 0; 1945 } 1946 1947 static int unuse_vma(struct vm_area_struct *vma, unsigned int type) 1948 { 1949 pgd_t *pgd; 1950 unsigned long addr, end, next; 1951 int ret; 1952 1953 addr = vma->vm_start; 1954 end = vma->vm_end; 1955 1956 pgd = pgd_offset(vma->vm_mm, addr); 1957 do { 1958 next = pgd_addr_end(addr, end); 1959 if (pgd_none_or_clear_bad(pgd)) 1960 continue; 1961 ret = unuse_p4d_range(vma, pgd, addr, next, type); 1962 if (ret) 1963 return ret; 1964 } while (pgd++, addr = next, addr != end); 1965 return 0; 1966 } 1967 1968 static int unuse_mm(struct mm_struct *mm, unsigned int type) 1969 { 1970 struct vm_area_struct *vma; 1971 int ret = 0; 1972 1973 mmap_read_lock(mm); 1974 for (vma = mm->mmap; vma; vma = vma->vm_next) { 1975 if (vma->anon_vma) { 1976 ret = unuse_vma(vma, type); 1977 if (ret) 1978 break; 1979 } 1980 cond_resched(); 1981 } 1982 mmap_read_unlock(mm); 1983 return ret; 1984 } 1985 1986 /* 1987 * Scan swap_map (or frontswap_map if frontswap parameter is true) 1988 * from current position to next entry still in use. Return 0 1989 * if there are no inuse entries after prev till end of the map. 1990 */ 1991 static unsigned int find_next_to_unuse(struct swap_info_struct *si, 1992 unsigned int prev) 1993 { 1994 unsigned int i; 1995 unsigned char count; 1996 1997 /* 1998 * No need for swap_lock here: we're just looking 1999 * for whether an entry is in use, not modifying it; false 2000 * hits are okay, and sys_swapoff() has already prevented new 2001 * allocations from this area (while holding swap_lock). 2002 */ 2003 for (i = prev + 1; i < si->max; i++) { 2004 count = READ_ONCE(si->swap_map[i]); 2005 if (count && swap_count(count) != SWAP_MAP_BAD) 2006 break; 2007 if ((i % LATENCY_LIMIT) == 0) 2008 cond_resched(); 2009 } 2010 2011 if (i == si->max) 2012 i = 0; 2013 2014 return i; 2015 } 2016 2017 static int try_to_unuse(unsigned int type) 2018 { 2019 struct mm_struct *prev_mm; 2020 struct mm_struct *mm; 2021 struct list_head *p; 2022 int retval = 0; 2023 struct swap_info_struct *si = swap_info[type]; 2024 struct page *page; 2025 swp_entry_t entry; 2026 unsigned int i; 2027 2028 if (!READ_ONCE(si->inuse_pages)) 2029 return 0; 2030 2031 retry: 2032 retval = shmem_unuse(type); 2033 if (retval) 2034 return retval; 2035 2036 prev_mm = &init_mm; 2037 mmget(prev_mm); 2038 2039 spin_lock(&mmlist_lock); 2040 p = &init_mm.mmlist; 2041 while (READ_ONCE(si->inuse_pages) && 2042 !signal_pending(current) && 2043 (p = p->next) != &init_mm.mmlist) { 2044 2045 mm = list_entry(p, struct mm_struct, mmlist); 2046 if (!mmget_not_zero(mm)) 2047 continue; 2048 spin_unlock(&mmlist_lock); 2049 mmput(prev_mm); 2050 prev_mm = mm; 2051 retval = unuse_mm(mm, type); 2052 if (retval) { 2053 mmput(prev_mm); 2054 return retval; 2055 } 2056 2057 /* 2058 * Make sure that we aren't completely killing 2059 * interactive performance. 2060 */ 2061 cond_resched(); 2062 spin_lock(&mmlist_lock); 2063 } 2064 spin_unlock(&mmlist_lock); 2065 2066 mmput(prev_mm); 2067 2068 i = 0; 2069 while (READ_ONCE(si->inuse_pages) && 2070 !signal_pending(current) && 2071 (i = find_next_to_unuse(si, i)) != 0) { 2072 2073 entry = swp_entry(type, i); 2074 page = find_get_page(swap_address_space(entry), i); 2075 if (!page) 2076 continue; 2077 2078 /* 2079 * It is conceivable that a racing task removed this page from 2080 * swap cache just before we acquired the page lock. The page 2081 * might even be back in swap cache on another swap area. But 2082 * that is okay, try_to_free_swap() only removes stale pages. 2083 */ 2084 lock_page(page); 2085 wait_on_page_writeback(page); 2086 try_to_free_swap(page); 2087 unlock_page(page); 2088 put_page(page); 2089 } 2090 2091 /* 2092 * Lets check again to see if there are still swap entries in the map. 2093 * If yes, we would need to do retry the unuse logic again. 2094 * Under global memory pressure, swap entries can be reinserted back 2095 * into process space after the mmlist loop above passes over them. 2096 * 2097 * Limit the number of retries? No: when mmget_not_zero() above fails, 2098 * that mm is likely to be freeing swap from exit_mmap(), which proceeds 2099 * at its own independent pace; and even shmem_writepage() could have 2100 * been preempted after get_swap_page(), temporarily hiding that swap. 2101 * It's easy and robust (though cpu-intensive) just to keep retrying. 2102 */ 2103 if (READ_ONCE(si->inuse_pages)) { 2104 if (!signal_pending(current)) 2105 goto retry; 2106 return -EINTR; 2107 } 2108 2109 return 0; 2110 } 2111 2112 /* 2113 * After a successful try_to_unuse, if no swap is now in use, we know 2114 * we can empty the mmlist. swap_lock must be held on entry and exit. 2115 * Note that mmlist_lock nests inside swap_lock, and an mm must be 2116 * added to the mmlist just after page_duplicate - before would be racy. 2117 */ 2118 static void drain_mmlist(void) 2119 { 2120 struct list_head *p, *next; 2121 unsigned int type; 2122 2123 for (type = 0; type < nr_swapfiles; type++) 2124 if (swap_info[type]->inuse_pages) 2125 return; 2126 spin_lock(&mmlist_lock); 2127 list_for_each_safe(p, next, &init_mm.mmlist) 2128 list_del_init(p); 2129 spin_unlock(&mmlist_lock); 2130 } 2131 2132 /* 2133 * Free all of a swapdev's extent information 2134 */ 2135 static void destroy_swap_extents(struct swap_info_struct *sis) 2136 { 2137 while (!RB_EMPTY_ROOT(&sis->swap_extent_root)) { 2138 struct rb_node *rb = sis->swap_extent_root.rb_node; 2139 struct swap_extent *se = rb_entry(rb, struct swap_extent, rb_node); 2140 2141 rb_erase(rb, &sis->swap_extent_root); 2142 kfree(se); 2143 } 2144 2145 if (sis->flags & SWP_ACTIVATED) { 2146 struct file *swap_file = sis->swap_file; 2147 struct address_space *mapping = swap_file->f_mapping; 2148 2149 sis->flags &= ~SWP_ACTIVATED; 2150 if (mapping->a_ops->swap_deactivate) 2151 mapping->a_ops->swap_deactivate(swap_file); 2152 } 2153 } 2154 2155 /* 2156 * Add a block range (and the corresponding page range) into this swapdev's 2157 * extent tree. 2158 * 2159 * This function rather assumes that it is called in ascending page order. 2160 */ 2161 int 2162 add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, 2163 unsigned long nr_pages, sector_t start_block) 2164 { 2165 struct rb_node **link = &sis->swap_extent_root.rb_node, *parent = NULL; 2166 struct swap_extent *se; 2167 struct swap_extent *new_se; 2168 2169 /* 2170 * place the new node at the right most since the 2171 * function is called in ascending page order. 2172 */ 2173 while (*link) { 2174 parent = *link; 2175 link = &parent->rb_right; 2176 } 2177 2178 if (parent) { 2179 se = rb_entry(parent, struct swap_extent, rb_node); 2180 BUG_ON(se->start_page + se->nr_pages != start_page); 2181 if (se->start_block + se->nr_pages == start_block) { 2182 /* Merge it */ 2183 se->nr_pages += nr_pages; 2184 return 0; 2185 } 2186 } 2187 2188 /* No merge, insert a new extent. */ 2189 new_se = kmalloc(sizeof(*se), GFP_KERNEL); 2190 if (new_se == NULL) 2191 return -ENOMEM; 2192 new_se->start_page = start_page; 2193 new_se->nr_pages = nr_pages; 2194 new_se->start_block = start_block; 2195 2196 rb_link_node(&new_se->rb_node, parent, link); 2197 rb_insert_color(&new_se->rb_node, &sis->swap_extent_root); 2198 return 1; 2199 } 2200 EXPORT_SYMBOL_GPL(add_swap_extent); 2201 2202 /* 2203 * A `swap extent' is a simple thing which maps a contiguous range of pages 2204 * onto a contiguous range of disk blocks. An ordered list of swap extents 2205 * is built at swapon time and is then used at swap_writepage/swap_readpage 2206 * time for locating where on disk a page belongs. 2207 * 2208 * If the swapfile is an S_ISBLK block device, a single extent is installed. 2209 * This is done so that the main operating code can treat S_ISBLK and S_ISREG 2210 * swap files identically. 2211 * 2212 * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap 2213 * extent list operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK 2214 * swapfiles are handled *identically* after swapon time. 2215 * 2216 * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks 2217 * and will parse them into an ordered extent list, in PAGE_SIZE chunks. If 2218 * some stray blocks are found which do not fall within the PAGE_SIZE alignment 2219 * requirements, they are simply tossed out - we will never use those blocks 2220 * for swapping. 2221 * 2222 * For all swap devices we set S_SWAPFILE across the life of the swapon. This 2223 * prevents users from writing to the swap device, which will corrupt memory. 2224 * 2225 * The amount of disk space which a single swap extent represents varies. 2226 * Typically it is in the 1-4 megabyte range. So we can have hundreds of 2227 * extents in the list. To avoid much list walking, we cache the previous 2228 * search location in `curr_swap_extent', and start new searches from there. 2229 * This is extremely effective. The average number of iterations in 2230 * map_swap_page() has been measured at about 0.3 per page. - akpm. 2231 */ 2232 static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span) 2233 { 2234 struct file *swap_file = sis->swap_file; 2235 struct address_space *mapping = swap_file->f_mapping; 2236 struct inode *inode = mapping->host; 2237 int ret; 2238 2239 if (S_ISBLK(inode->i_mode)) { 2240 ret = add_swap_extent(sis, 0, sis->max, 0); 2241 *span = sis->pages; 2242 return ret; 2243 } 2244 2245 if (mapping->a_ops->swap_activate) { 2246 ret = mapping->a_ops->swap_activate(sis, swap_file, span); 2247 if (ret >= 0) 2248 sis->flags |= SWP_ACTIVATED; 2249 if (!ret) { 2250 sis->flags |= SWP_FS_OPS; 2251 ret = add_swap_extent(sis, 0, sis->max, 0); 2252 *span = sis->pages; 2253 } 2254 return ret; 2255 } 2256 2257 return generic_swapfile_activate(sis, swap_file, span); 2258 } 2259 2260 static int swap_node(struct swap_info_struct *p) 2261 { 2262 struct block_device *bdev; 2263 2264 if (p->bdev) 2265 bdev = p->bdev; 2266 else 2267 bdev = p->swap_file->f_inode->i_sb->s_bdev; 2268 2269 return bdev ? bdev->bd_disk->node_id : NUMA_NO_NODE; 2270 } 2271 2272 static void setup_swap_info(struct swap_info_struct *p, int prio, 2273 unsigned char *swap_map, 2274 struct swap_cluster_info *cluster_info) 2275 { 2276 int i; 2277 2278 if (prio >= 0) 2279 p->prio = prio; 2280 else 2281 p->prio = --least_priority; 2282 /* 2283 * the plist prio is negated because plist ordering is 2284 * low-to-high, while swap ordering is high-to-low 2285 */ 2286 p->list.prio = -p->prio; 2287 for_each_node(i) { 2288 if (p->prio >= 0) 2289 p->avail_lists[i].prio = -p->prio; 2290 else { 2291 if (swap_node(p) == i) 2292 p->avail_lists[i].prio = 1; 2293 else 2294 p->avail_lists[i].prio = -p->prio; 2295 } 2296 } 2297 p->swap_map = swap_map; 2298 p->cluster_info = cluster_info; 2299 } 2300 2301 static void _enable_swap_info(struct swap_info_struct *p) 2302 { 2303 p->flags |= SWP_WRITEOK; 2304 atomic_long_add(p->pages, &nr_swap_pages); 2305 total_swap_pages += p->pages; 2306 2307 assert_spin_locked(&swap_lock); 2308 /* 2309 * both lists are plists, and thus priority ordered. 2310 * swap_active_head needs to be priority ordered for swapoff(), 2311 * which on removal of any swap_info_struct with an auto-assigned 2312 * (i.e. negative) priority increments the auto-assigned priority 2313 * of any lower-priority swap_info_structs. 2314 * swap_avail_head needs to be priority ordered for get_swap_page(), 2315 * which allocates swap pages from the highest available priority 2316 * swap_info_struct. 2317 */ 2318 plist_add(&p->list, &swap_active_head); 2319 add_to_avail_list(p); 2320 } 2321 2322 static void enable_swap_info(struct swap_info_struct *p, int prio, 2323 unsigned char *swap_map, 2324 struct swap_cluster_info *cluster_info, 2325 unsigned long *frontswap_map) 2326 { 2327 if (IS_ENABLED(CONFIG_FRONTSWAP)) 2328 frontswap_init(p->type, frontswap_map); 2329 spin_lock(&swap_lock); 2330 spin_lock(&p->lock); 2331 setup_swap_info(p, prio, swap_map, cluster_info); 2332 spin_unlock(&p->lock); 2333 spin_unlock(&swap_lock); 2334 /* 2335 * Finished initializing swap device, now it's safe to reference it. 2336 */ 2337 percpu_ref_resurrect(&p->users); 2338 spin_lock(&swap_lock); 2339 spin_lock(&p->lock); 2340 _enable_swap_info(p); 2341 spin_unlock(&p->lock); 2342 spin_unlock(&swap_lock); 2343 } 2344 2345 static void reinsert_swap_info(struct swap_info_struct *p) 2346 { 2347 spin_lock(&swap_lock); 2348 spin_lock(&p->lock); 2349 setup_swap_info(p, p->prio, p->swap_map, p->cluster_info); 2350 _enable_swap_info(p); 2351 spin_unlock(&p->lock); 2352 spin_unlock(&swap_lock); 2353 } 2354 2355 bool has_usable_swap(void) 2356 { 2357 bool ret = true; 2358 2359 spin_lock(&swap_lock); 2360 if (plist_head_empty(&swap_active_head)) 2361 ret = false; 2362 spin_unlock(&swap_lock); 2363 return ret; 2364 } 2365 2366 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) 2367 { 2368 struct swap_info_struct *p = NULL; 2369 unsigned char *swap_map; 2370 struct swap_cluster_info *cluster_info; 2371 unsigned long *frontswap_map; 2372 struct file *swap_file, *victim; 2373 struct address_space *mapping; 2374 struct inode *inode; 2375 struct filename *pathname; 2376 int err, found = 0; 2377 unsigned int old_block_size; 2378 2379 if (!capable(CAP_SYS_ADMIN)) 2380 return -EPERM; 2381 2382 BUG_ON(!current->mm); 2383 2384 pathname = getname(specialfile); 2385 if (IS_ERR(pathname)) 2386 return PTR_ERR(pathname); 2387 2388 victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0); 2389 err = PTR_ERR(victim); 2390 if (IS_ERR(victim)) 2391 goto out; 2392 2393 mapping = victim->f_mapping; 2394 spin_lock(&swap_lock); 2395 plist_for_each_entry(p, &swap_active_head, list) { 2396 if (p->flags & SWP_WRITEOK) { 2397 if (p->swap_file->f_mapping == mapping) { 2398 found = 1; 2399 break; 2400 } 2401 } 2402 } 2403 if (!found) { 2404 err = -EINVAL; 2405 spin_unlock(&swap_lock); 2406 goto out_dput; 2407 } 2408 if (!security_vm_enough_memory_mm(current->mm, p->pages)) 2409 vm_unacct_memory(p->pages); 2410 else { 2411 err = -ENOMEM; 2412 spin_unlock(&swap_lock); 2413 goto out_dput; 2414 } 2415 del_from_avail_list(p); 2416 spin_lock(&p->lock); 2417 if (p->prio < 0) { 2418 struct swap_info_struct *si = p; 2419 int nid; 2420 2421 plist_for_each_entry_continue(si, &swap_active_head, list) { 2422 si->prio++; 2423 si->list.prio--; 2424 for_each_node(nid) { 2425 if (si->avail_lists[nid].prio != 1) 2426 si->avail_lists[nid].prio--; 2427 } 2428 } 2429 least_priority++; 2430 } 2431 plist_del(&p->list, &swap_active_head); 2432 atomic_long_sub(p->pages, &nr_swap_pages); 2433 total_swap_pages -= p->pages; 2434 p->flags &= ~SWP_WRITEOK; 2435 spin_unlock(&p->lock); 2436 spin_unlock(&swap_lock); 2437 2438 disable_swap_slots_cache_lock(); 2439 2440 set_current_oom_origin(); 2441 err = try_to_unuse(p->type); 2442 clear_current_oom_origin(); 2443 2444 if (err) { 2445 /* re-insert swap space back into swap_list */ 2446 reinsert_swap_info(p); 2447 reenable_swap_slots_cache_unlock(); 2448 goto out_dput; 2449 } 2450 2451 reenable_swap_slots_cache_unlock(); 2452 2453 /* 2454 * Wait for swap operations protected by get/put_swap_device() 2455 * to complete. 2456 * 2457 * We need synchronize_rcu() here to protect the accessing to 2458 * the swap cache data structure. 2459 */ 2460 percpu_ref_kill(&p->users); 2461 synchronize_rcu(); 2462 wait_for_completion(&p->comp); 2463 2464 flush_work(&p->discard_work); 2465 2466 destroy_swap_extents(p); 2467 if (p->flags & SWP_CONTINUED) 2468 free_swap_count_continuations(p); 2469 2470 if (!p->bdev || !bdev_nonrot(p->bdev)) 2471 atomic_dec(&nr_rotate_swap); 2472 2473 mutex_lock(&swapon_mutex); 2474 spin_lock(&swap_lock); 2475 spin_lock(&p->lock); 2476 drain_mmlist(); 2477 2478 /* wait for anyone still in scan_swap_map_slots */ 2479 p->highest_bit = 0; /* cuts scans short */ 2480 while (p->flags >= SWP_SCANNING) { 2481 spin_unlock(&p->lock); 2482 spin_unlock(&swap_lock); 2483 schedule_timeout_uninterruptible(1); 2484 spin_lock(&swap_lock); 2485 spin_lock(&p->lock); 2486 } 2487 2488 swap_file = p->swap_file; 2489 old_block_size = p->old_block_size; 2490 p->swap_file = NULL; 2491 p->max = 0; 2492 swap_map = p->swap_map; 2493 p->swap_map = NULL; 2494 cluster_info = p->cluster_info; 2495 p->cluster_info = NULL; 2496 frontswap_map = frontswap_map_get(p); 2497 spin_unlock(&p->lock); 2498 spin_unlock(&swap_lock); 2499 arch_swap_invalidate_area(p->type); 2500 frontswap_invalidate_area(p->type); 2501 frontswap_map_set(p, NULL); 2502 mutex_unlock(&swapon_mutex); 2503 free_percpu(p->percpu_cluster); 2504 p->percpu_cluster = NULL; 2505 free_percpu(p->cluster_next_cpu); 2506 p->cluster_next_cpu = NULL; 2507 vfree(swap_map); 2508 kvfree(cluster_info); 2509 kvfree(frontswap_map); 2510 /* Destroy swap account information */ 2511 swap_cgroup_swapoff(p->type); 2512 exit_swap_address_space(p->type); 2513 2514 inode = mapping->host; 2515 if (S_ISBLK(inode->i_mode)) { 2516 struct block_device *bdev = I_BDEV(inode); 2517 2518 set_blocksize(bdev, old_block_size); 2519 blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); 2520 } 2521 2522 inode_lock(inode); 2523 inode->i_flags &= ~S_SWAPFILE; 2524 inode_unlock(inode); 2525 filp_close(swap_file, NULL); 2526 2527 /* 2528 * Clear the SWP_USED flag after all resources are freed so that swapon 2529 * can reuse this swap_info in alloc_swap_info() safely. It is ok to 2530 * not hold p->lock after we cleared its SWP_WRITEOK. 2531 */ 2532 spin_lock(&swap_lock); 2533 p->flags = 0; 2534 spin_unlock(&swap_lock); 2535 2536 err = 0; 2537 atomic_inc(&proc_poll_event); 2538 wake_up_interruptible(&proc_poll_wait); 2539 2540 out_dput: 2541 filp_close(victim, NULL); 2542 out: 2543 putname(pathname); 2544 return err; 2545 } 2546 2547 #ifdef CONFIG_PROC_FS 2548 static __poll_t swaps_poll(struct file *file, poll_table *wait) 2549 { 2550 struct seq_file *seq = file->private_data; 2551 2552 poll_wait(file, &proc_poll_wait, wait); 2553 2554 if (seq->poll_event != atomic_read(&proc_poll_event)) { 2555 seq->poll_event = atomic_read(&proc_poll_event); 2556 return EPOLLIN | EPOLLRDNORM | EPOLLERR | EPOLLPRI; 2557 } 2558 2559 return EPOLLIN | EPOLLRDNORM; 2560 } 2561 2562 /* iterator */ 2563 static void *swap_start(struct seq_file *swap, loff_t *pos) 2564 { 2565 struct swap_info_struct *si; 2566 int type; 2567 loff_t l = *pos; 2568 2569 mutex_lock(&swapon_mutex); 2570 2571 if (!l) 2572 return SEQ_START_TOKEN; 2573 2574 for (type = 0; (si = swap_type_to_swap_info(type)); type++) { 2575 if (!(si->flags & SWP_USED) || !si->swap_map) 2576 continue; 2577 if (!--l) 2578 return si; 2579 } 2580 2581 return NULL; 2582 } 2583 2584 static void *swap_next(struct seq_file *swap, void *v, loff_t *pos) 2585 { 2586 struct swap_info_struct *si = v; 2587 int type; 2588 2589 if (v == SEQ_START_TOKEN) 2590 type = 0; 2591 else 2592 type = si->type + 1; 2593 2594 ++(*pos); 2595 for (; (si = swap_type_to_swap_info(type)); type++) { 2596 if (!(si->flags & SWP_USED) || !si->swap_map) 2597 continue; 2598 return si; 2599 } 2600 2601 return NULL; 2602 } 2603 2604 static void swap_stop(struct seq_file *swap, void *v) 2605 { 2606 mutex_unlock(&swapon_mutex); 2607 } 2608 2609 static int swap_show(struct seq_file *swap, void *v) 2610 { 2611 struct swap_info_struct *si = v; 2612 struct file *file; 2613 int len; 2614 unsigned long bytes, inuse; 2615 2616 if (si == SEQ_START_TOKEN) { 2617 seq_puts(swap, "Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n"); 2618 return 0; 2619 } 2620 2621 bytes = si->pages << (PAGE_SHIFT - 10); 2622 inuse = si->inuse_pages << (PAGE_SHIFT - 10); 2623 2624 file = si->swap_file; 2625 len = seq_file_path(swap, file, " \t\n\\"); 2626 seq_printf(swap, "%*s%s\t%lu\t%s%lu\t%s%d\n", 2627 len < 40 ? 40 - len : 1, " ", 2628 S_ISBLK(file_inode(file)->i_mode) ? 2629 "partition" : "file\t", 2630 bytes, bytes < 10000000 ? "\t" : "", 2631 inuse, inuse < 10000000 ? "\t" : "", 2632 si->prio); 2633 return 0; 2634 } 2635 2636 static const struct seq_operations swaps_op = { 2637 .start = swap_start, 2638 .next = swap_next, 2639 .stop = swap_stop, 2640 .show = swap_show 2641 }; 2642 2643 static int swaps_open(struct inode *inode, struct file *file) 2644 { 2645 struct seq_file *seq; 2646 int ret; 2647 2648 ret = seq_open(file, &swaps_op); 2649 if (ret) 2650 return ret; 2651 2652 seq = file->private_data; 2653 seq->poll_event = atomic_read(&proc_poll_event); 2654 return 0; 2655 } 2656 2657 static const struct proc_ops swaps_proc_ops = { 2658 .proc_flags = PROC_ENTRY_PERMANENT, 2659 .proc_open = swaps_open, 2660 .proc_read = seq_read, 2661 .proc_lseek = seq_lseek, 2662 .proc_release = seq_release, 2663 .proc_poll = swaps_poll, 2664 }; 2665 2666 static int __init procswaps_init(void) 2667 { 2668 proc_create("swaps", 0, NULL, &swaps_proc_ops); 2669 return 0; 2670 } 2671 __initcall(procswaps_init); 2672 #endif /* CONFIG_PROC_FS */ 2673 2674 #ifdef MAX_SWAPFILES_CHECK 2675 static int __init max_swapfiles_check(void) 2676 { 2677 MAX_SWAPFILES_CHECK(); 2678 return 0; 2679 } 2680 late_initcall(max_swapfiles_check); 2681 #endif 2682 2683 static struct swap_info_struct *alloc_swap_info(void) 2684 { 2685 struct swap_info_struct *p; 2686 struct swap_info_struct *defer = NULL; 2687 unsigned int type; 2688 int i; 2689 2690 p = kvzalloc(struct_size(p, avail_lists, nr_node_ids), GFP_KERNEL); 2691 if (!p) 2692 return ERR_PTR(-ENOMEM); 2693 2694 if (percpu_ref_init(&p->users, swap_users_ref_free, 2695 PERCPU_REF_INIT_DEAD, GFP_KERNEL)) { 2696 kvfree(p); 2697 return ERR_PTR(-ENOMEM); 2698 } 2699 2700 spin_lock(&swap_lock); 2701 for (type = 0; type < nr_swapfiles; type++) { 2702 if (!(swap_info[type]->flags & SWP_USED)) 2703 break; 2704 } 2705 if (type >= MAX_SWAPFILES) { 2706 spin_unlock(&swap_lock); 2707 percpu_ref_exit(&p->users); 2708 kvfree(p); 2709 return ERR_PTR(-EPERM); 2710 } 2711 if (type >= nr_swapfiles) { 2712 p->type = type; 2713 /* 2714 * Publish the swap_info_struct after initializing it. 2715 * Note that kvzalloc() above zeroes all its fields. 2716 */ 2717 smp_store_release(&swap_info[type], p); /* rcu_assign_pointer() */ 2718 nr_swapfiles++; 2719 } else { 2720 defer = p; 2721 p = swap_info[type]; 2722 /* 2723 * Do not memset this entry: a racing procfs swap_next() 2724 * would be relying on p->type to remain valid. 2725 */ 2726 } 2727 p->swap_extent_root = RB_ROOT; 2728 plist_node_init(&p->list, 0); 2729 for_each_node(i) 2730 plist_node_init(&p->avail_lists[i], 0); 2731 p->flags = SWP_USED; 2732 spin_unlock(&swap_lock); 2733 if (defer) { 2734 percpu_ref_exit(&defer->users); 2735 kvfree(defer); 2736 } 2737 spin_lock_init(&p->lock); 2738 spin_lock_init(&p->cont_lock); 2739 init_completion(&p->comp); 2740 2741 return p; 2742 } 2743 2744 static int claim_swapfile(struct swap_info_struct *p, struct inode *inode) 2745 { 2746 int error; 2747 2748 if (S_ISBLK(inode->i_mode)) { 2749 p->bdev = blkdev_get_by_dev(inode->i_rdev, 2750 FMODE_READ | FMODE_WRITE | FMODE_EXCL, p); 2751 if (IS_ERR(p->bdev)) { 2752 error = PTR_ERR(p->bdev); 2753 p->bdev = NULL; 2754 return error; 2755 } 2756 p->old_block_size = block_size(p->bdev); 2757 error = set_blocksize(p->bdev, PAGE_SIZE); 2758 if (error < 0) 2759 return error; 2760 /* 2761 * Zoned block devices contain zones that have a sequential 2762 * write only restriction. Hence zoned block devices are not 2763 * suitable for swapping. Disallow them here. 2764 */ 2765 if (bdev_is_zoned(p->bdev)) 2766 return -EINVAL; 2767 p->flags |= SWP_BLKDEV; 2768 } else if (S_ISREG(inode->i_mode)) { 2769 p->bdev = inode->i_sb->s_bdev; 2770 } 2771 2772 return 0; 2773 } 2774 2775 2776 /* 2777 * Find out how many pages are allowed for a single swap device. There 2778 * are two limiting factors: 2779 * 1) the number of bits for the swap offset in the swp_entry_t type, and 2780 * 2) the number of bits in the swap pte, as defined by the different 2781 * architectures. 2782 * 2783 * In order to find the largest possible bit mask, a swap entry with 2784 * swap type 0 and swap offset ~0UL is created, encoded to a swap pte, 2785 * decoded to a swp_entry_t again, and finally the swap offset is 2786 * extracted. 2787 * 2788 * This will mask all the bits from the initial ~0UL mask that can't 2789 * be encoded in either the swp_entry_t or the architecture definition 2790 * of a swap pte. 2791 */ 2792 unsigned long generic_max_swapfile_size(void) 2793 { 2794 return swp_offset(pte_to_swp_entry( 2795 swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1; 2796 } 2797 2798 /* Can be overridden by an architecture for additional checks. */ 2799 __weak unsigned long max_swapfile_size(void) 2800 { 2801 return generic_max_swapfile_size(); 2802 } 2803 2804 static unsigned long read_swap_header(struct swap_info_struct *p, 2805 union swap_header *swap_header, 2806 struct inode *inode) 2807 { 2808 int i; 2809 unsigned long maxpages; 2810 unsigned long swapfilepages; 2811 unsigned long last_page; 2812 2813 if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) { 2814 pr_err("Unable to find swap-space signature\n"); 2815 return 0; 2816 } 2817 2818 /* swap partition endianness hack... */ 2819 if (swab32(swap_header->info.version) == 1) { 2820 swab32s(&swap_header->info.version); 2821 swab32s(&swap_header->info.last_page); 2822 swab32s(&swap_header->info.nr_badpages); 2823 if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES) 2824 return 0; 2825 for (i = 0; i < swap_header->info.nr_badpages; i++) 2826 swab32s(&swap_header->info.badpages[i]); 2827 } 2828 /* Check the swap header's sub-version */ 2829 if (swap_header->info.version != 1) { 2830 pr_warn("Unable to handle swap header version %d\n", 2831 swap_header->info.version); 2832 return 0; 2833 } 2834 2835 p->lowest_bit = 1; 2836 p->cluster_next = 1; 2837 p->cluster_nr = 0; 2838 2839 maxpages = max_swapfile_size(); 2840 last_page = swap_header->info.last_page; 2841 if (!last_page) { 2842 pr_warn("Empty swap-file\n"); 2843 return 0; 2844 } 2845 if (last_page > maxpages) { 2846 pr_warn("Truncating oversized swap area, only using %luk out of %luk\n", 2847 maxpages << (PAGE_SHIFT - 10), 2848 last_page << (PAGE_SHIFT - 10)); 2849 } 2850 if (maxpages > last_page) { 2851 maxpages = last_page + 1; 2852 /* p->max is an unsigned int: don't overflow it */ 2853 if ((unsigned int)maxpages == 0) 2854 maxpages = UINT_MAX; 2855 } 2856 p->highest_bit = maxpages - 1; 2857 2858 if (!maxpages) 2859 return 0; 2860 swapfilepages = i_size_read(inode) >> PAGE_SHIFT; 2861 if (swapfilepages && maxpages > swapfilepages) { 2862 pr_warn("Swap area shorter than signature indicates\n"); 2863 return 0; 2864 } 2865 if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode)) 2866 return 0; 2867 if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES) 2868 return 0; 2869 2870 return maxpages; 2871 } 2872 2873 #define SWAP_CLUSTER_INFO_COLS \ 2874 DIV_ROUND_UP(L1_CACHE_BYTES, sizeof(struct swap_cluster_info)) 2875 #define SWAP_CLUSTER_SPACE_COLS \ 2876 DIV_ROUND_UP(SWAP_ADDRESS_SPACE_PAGES, SWAPFILE_CLUSTER) 2877 #define SWAP_CLUSTER_COLS \ 2878 max_t(unsigned int, SWAP_CLUSTER_INFO_COLS, SWAP_CLUSTER_SPACE_COLS) 2879 2880 static int setup_swap_map_and_extents(struct swap_info_struct *p, 2881 union swap_header *swap_header, 2882 unsigned char *swap_map, 2883 struct swap_cluster_info *cluster_info, 2884 unsigned long maxpages, 2885 sector_t *span) 2886 { 2887 unsigned int j, k; 2888 unsigned int nr_good_pages; 2889 int nr_extents; 2890 unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER); 2891 unsigned long col = p->cluster_next / SWAPFILE_CLUSTER % SWAP_CLUSTER_COLS; 2892 unsigned long i, idx; 2893 2894 nr_good_pages = maxpages - 1; /* omit header page */ 2895 2896 cluster_list_init(&p->free_clusters); 2897 cluster_list_init(&p->discard_clusters); 2898 2899 for (i = 0; i < swap_header->info.nr_badpages; i++) { 2900 unsigned int page_nr = swap_header->info.badpages[i]; 2901 if (page_nr == 0 || page_nr > swap_header->info.last_page) 2902 return -EINVAL; 2903 if (page_nr < maxpages) { 2904 swap_map[page_nr] = SWAP_MAP_BAD; 2905 nr_good_pages--; 2906 /* 2907 * Haven't marked the cluster free yet, no list 2908 * operation involved 2909 */ 2910 inc_cluster_info_page(p, cluster_info, page_nr); 2911 } 2912 } 2913 2914 /* Haven't marked the cluster free yet, no list operation involved */ 2915 for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++) 2916 inc_cluster_info_page(p, cluster_info, i); 2917 2918 if (nr_good_pages) { 2919 swap_map[0] = SWAP_MAP_BAD; 2920 /* 2921 * Not mark the cluster free yet, no list 2922 * operation involved 2923 */ 2924 inc_cluster_info_page(p, cluster_info, 0); 2925 p->max = maxpages; 2926 p->pages = nr_good_pages; 2927 nr_extents = setup_swap_extents(p, span); 2928 if (nr_extents < 0) 2929 return nr_extents; 2930 nr_good_pages = p->pages; 2931 } 2932 if (!nr_good_pages) { 2933 pr_warn("Empty swap-file\n"); 2934 return -EINVAL; 2935 } 2936 2937 if (!cluster_info) 2938 return nr_extents; 2939 2940 2941 /* 2942 * Reduce false cache line sharing between cluster_info and 2943 * sharing same address space. 2944 */ 2945 for (k = 0; k < SWAP_CLUSTER_COLS; k++) { 2946 j = (k + col) % SWAP_CLUSTER_COLS; 2947 for (i = 0; i < DIV_ROUND_UP(nr_clusters, SWAP_CLUSTER_COLS); i++) { 2948 idx = i * SWAP_CLUSTER_COLS + j; 2949 if (idx >= nr_clusters) 2950 continue; 2951 if (cluster_count(&cluster_info[idx])) 2952 continue; 2953 cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE); 2954 cluster_list_add_tail(&p->free_clusters, cluster_info, 2955 idx); 2956 } 2957 } 2958 return nr_extents; 2959 } 2960 2961 SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) 2962 { 2963 struct swap_info_struct *p; 2964 struct filename *name; 2965 struct file *swap_file = NULL; 2966 struct address_space *mapping; 2967 struct dentry *dentry; 2968 int prio; 2969 int error; 2970 union swap_header *swap_header; 2971 int nr_extents; 2972 sector_t span; 2973 unsigned long maxpages; 2974 unsigned char *swap_map = NULL; 2975 struct swap_cluster_info *cluster_info = NULL; 2976 unsigned long *frontswap_map = NULL; 2977 struct page *page = NULL; 2978 struct inode *inode = NULL; 2979 bool inced_nr_rotate_swap = false; 2980 2981 if (swap_flags & ~SWAP_FLAGS_VALID) 2982 return -EINVAL; 2983 2984 if (!capable(CAP_SYS_ADMIN)) 2985 return -EPERM; 2986 2987 if (!swap_avail_heads) 2988 return -ENOMEM; 2989 2990 p = alloc_swap_info(); 2991 if (IS_ERR(p)) 2992 return PTR_ERR(p); 2993 2994 INIT_WORK(&p->discard_work, swap_discard_work); 2995 2996 name = getname(specialfile); 2997 if (IS_ERR(name)) { 2998 error = PTR_ERR(name); 2999 name = NULL; 3000 goto bad_swap; 3001 } 3002 swap_file = file_open_name(name, O_RDWR|O_LARGEFILE, 0); 3003 if (IS_ERR(swap_file)) { 3004 error = PTR_ERR(swap_file); 3005 swap_file = NULL; 3006 goto bad_swap; 3007 } 3008 3009 p->swap_file = swap_file; 3010 mapping = swap_file->f_mapping; 3011 dentry = swap_file->f_path.dentry; 3012 inode = mapping->host; 3013 3014 error = claim_swapfile(p, inode); 3015 if (unlikely(error)) 3016 goto bad_swap; 3017 3018 inode_lock(inode); 3019 if (d_unlinked(dentry) || cant_mount(dentry)) { 3020 error = -ENOENT; 3021 goto bad_swap_unlock_inode; 3022 } 3023 if (IS_SWAPFILE(inode)) { 3024 error = -EBUSY; 3025 goto bad_swap_unlock_inode; 3026 } 3027 3028 /* 3029 * Read the swap header. 3030 */ 3031 if (!mapping->a_ops->readpage) { 3032 error = -EINVAL; 3033 goto bad_swap_unlock_inode; 3034 } 3035 page = read_mapping_page(mapping, 0, swap_file); 3036 if (IS_ERR(page)) { 3037 error = PTR_ERR(page); 3038 goto bad_swap_unlock_inode; 3039 } 3040 swap_header = kmap(page); 3041 3042 maxpages = read_swap_header(p, swap_header, inode); 3043 if (unlikely(!maxpages)) { 3044 error = -EINVAL; 3045 goto bad_swap_unlock_inode; 3046 } 3047 3048 /* OK, set up the swap map and apply the bad block list */ 3049 swap_map = vzalloc(maxpages); 3050 if (!swap_map) { 3051 error = -ENOMEM; 3052 goto bad_swap_unlock_inode; 3053 } 3054 3055 if (p->bdev && bdev_stable_writes(p->bdev)) 3056 p->flags |= SWP_STABLE_WRITES; 3057 3058 if (p->bdev && p->bdev->bd_disk->fops->rw_page) 3059 p->flags |= SWP_SYNCHRONOUS_IO; 3060 3061 if (p->bdev && bdev_nonrot(p->bdev)) { 3062 int cpu; 3063 unsigned long ci, nr_cluster; 3064 3065 p->flags |= SWP_SOLIDSTATE; 3066 p->cluster_next_cpu = alloc_percpu(unsigned int); 3067 if (!p->cluster_next_cpu) { 3068 error = -ENOMEM; 3069 goto bad_swap_unlock_inode; 3070 } 3071 /* 3072 * select a random position to start with to help wear leveling 3073 * SSD 3074 */ 3075 for_each_possible_cpu(cpu) { 3076 per_cpu(*p->cluster_next_cpu, cpu) = 3077 1 + prandom_u32_max(p->highest_bit); 3078 } 3079 nr_cluster = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER); 3080 3081 cluster_info = kvcalloc(nr_cluster, sizeof(*cluster_info), 3082 GFP_KERNEL); 3083 if (!cluster_info) { 3084 error = -ENOMEM; 3085 goto bad_swap_unlock_inode; 3086 } 3087 3088 for (ci = 0; ci < nr_cluster; ci++) 3089 spin_lock_init(&((cluster_info + ci)->lock)); 3090 3091 p->percpu_cluster = alloc_percpu(struct percpu_cluster); 3092 if (!p->percpu_cluster) { 3093 error = -ENOMEM; 3094 goto bad_swap_unlock_inode; 3095 } 3096 for_each_possible_cpu(cpu) { 3097 struct percpu_cluster *cluster; 3098 cluster = per_cpu_ptr(p->percpu_cluster, cpu); 3099 cluster_set_null(&cluster->index); 3100 } 3101 } else { 3102 atomic_inc(&nr_rotate_swap); 3103 inced_nr_rotate_swap = true; 3104 } 3105 3106 error = swap_cgroup_swapon(p->type, maxpages); 3107 if (error) 3108 goto bad_swap_unlock_inode; 3109 3110 nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map, 3111 cluster_info, maxpages, &span); 3112 if (unlikely(nr_extents < 0)) { 3113 error = nr_extents; 3114 goto bad_swap_unlock_inode; 3115 } 3116 /* frontswap enabled? set up bit-per-page map for frontswap */ 3117 if (IS_ENABLED(CONFIG_FRONTSWAP)) 3118 frontswap_map = kvcalloc(BITS_TO_LONGS(maxpages), 3119 sizeof(long), 3120 GFP_KERNEL); 3121 3122 if ((swap_flags & SWAP_FLAG_DISCARD) && 3123 p->bdev && bdev_max_discard_sectors(p->bdev)) { 3124 /* 3125 * When discard is enabled for swap with no particular 3126 * policy flagged, we set all swap discard flags here in 3127 * order to sustain backward compatibility with older 3128 * swapon(8) releases. 3129 */ 3130 p->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD | 3131 SWP_PAGE_DISCARD); 3132 3133 /* 3134 * By flagging sys_swapon, a sysadmin can tell us to 3135 * either do single-time area discards only, or to just 3136 * perform discards for released swap page-clusters. 3137 * Now it's time to adjust the p->flags accordingly. 3138 */ 3139 if (swap_flags & SWAP_FLAG_DISCARD_ONCE) 3140 p->flags &= ~SWP_PAGE_DISCARD; 3141 else if (swap_flags & SWAP_FLAG_DISCARD_PAGES) 3142 p->flags &= ~SWP_AREA_DISCARD; 3143 3144 /* issue a swapon-time discard if it's still required */ 3145 if (p->flags & SWP_AREA_DISCARD) { 3146 int err = discard_swap(p); 3147 if (unlikely(err)) 3148 pr_err("swapon: discard_swap(%p): %d\n", 3149 p, err); 3150 } 3151 } 3152 3153 error = init_swap_address_space(p->type, maxpages); 3154 if (error) 3155 goto bad_swap_unlock_inode; 3156 3157 /* 3158 * Flush any pending IO and dirty mappings before we start using this 3159 * swap device. 3160 */ 3161 inode->i_flags |= S_SWAPFILE; 3162 error = inode_drain_writes(inode); 3163 if (error) { 3164 inode->i_flags &= ~S_SWAPFILE; 3165 goto free_swap_address_space; 3166 } 3167 3168 mutex_lock(&swapon_mutex); 3169 prio = -1; 3170 if (swap_flags & SWAP_FLAG_PREFER) 3171 prio = 3172 (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT; 3173 enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map); 3174 3175 pr_info("Adding %uk swap on %s. Priority:%d extents:%d across:%lluk %s%s%s%s%s\n", 3176 p->pages<<(PAGE_SHIFT-10), name->name, p->prio, 3177 nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10), 3178 (p->flags & SWP_SOLIDSTATE) ? "SS" : "", 3179 (p->flags & SWP_DISCARDABLE) ? "D" : "", 3180 (p->flags & SWP_AREA_DISCARD) ? "s" : "", 3181 (p->flags & SWP_PAGE_DISCARD) ? "c" : "", 3182 (frontswap_map) ? "FS" : ""); 3183 3184 mutex_unlock(&swapon_mutex); 3185 atomic_inc(&proc_poll_event); 3186 wake_up_interruptible(&proc_poll_wait); 3187 3188 error = 0; 3189 goto out; 3190 free_swap_address_space: 3191 exit_swap_address_space(p->type); 3192 bad_swap_unlock_inode: 3193 inode_unlock(inode); 3194 bad_swap: 3195 free_percpu(p->percpu_cluster); 3196 p->percpu_cluster = NULL; 3197 free_percpu(p->cluster_next_cpu); 3198 p->cluster_next_cpu = NULL; 3199 if (inode && S_ISBLK(inode->i_mode) && p->bdev) { 3200 set_blocksize(p->bdev, p->old_block_size); 3201 blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); 3202 } 3203 inode = NULL; 3204 destroy_swap_extents(p); 3205 swap_cgroup_swapoff(p->type); 3206 spin_lock(&swap_lock); 3207 p->swap_file = NULL; 3208 p->flags = 0; 3209 spin_unlock(&swap_lock); 3210 vfree(swap_map); 3211 kvfree(cluster_info); 3212 kvfree(frontswap_map); 3213 if (inced_nr_rotate_swap) 3214 atomic_dec(&nr_rotate_swap); 3215 if (swap_file) 3216 filp_close(swap_file, NULL); 3217 out: 3218 if (page && !IS_ERR(page)) { 3219 kunmap(page); 3220 put_page(page); 3221 } 3222 if (name) 3223 putname(name); 3224 if (inode) 3225 inode_unlock(inode); 3226 if (!error) 3227 enable_swap_slots_cache(); 3228 return error; 3229 } 3230 3231 void si_swapinfo(struct sysinfo *val) 3232 { 3233 unsigned int type; 3234 unsigned long nr_to_be_unused = 0; 3235 3236 spin_lock(&swap_lock); 3237 for (type = 0; type < nr_swapfiles; type++) { 3238 struct swap_info_struct *si = swap_info[type]; 3239 3240 if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK)) 3241 nr_to_be_unused += si->inuse_pages; 3242 } 3243 val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused; 3244 val->totalswap = total_swap_pages + nr_to_be_unused; 3245 spin_unlock(&swap_lock); 3246 } 3247 3248 /* 3249 * Verify that a swap entry is valid and increment its swap map count. 3250 * 3251 * Returns error code in following case. 3252 * - success -> 0 3253 * - swp_entry is invalid -> EINVAL 3254 * - swp_entry is migration entry -> EINVAL 3255 * - swap-cache reference is requested but there is already one. -> EEXIST 3256 * - swap-cache reference is requested but the entry is not used. -> ENOENT 3257 * - swap-mapped reference requested but needs continued swap count. -> ENOMEM 3258 */ 3259 static int __swap_duplicate(swp_entry_t entry, unsigned char usage) 3260 { 3261 struct swap_info_struct *p; 3262 struct swap_cluster_info *ci; 3263 unsigned long offset; 3264 unsigned char count; 3265 unsigned char has_cache; 3266 int err; 3267 3268 p = get_swap_device(entry); 3269 if (!p) 3270 return -EINVAL; 3271 3272 offset = swp_offset(entry); 3273 ci = lock_cluster_or_swap_info(p, offset); 3274 3275 count = p->swap_map[offset]; 3276 3277 /* 3278 * swapin_readahead() doesn't check if a swap entry is valid, so the 3279 * swap entry could be SWAP_MAP_BAD. Check here with lock held. 3280 */ 3281 if (unlikely(swap_count(count) == SWAP_MAP_BAD)) { 3282 err = -ENOENT; 3283 goto unlock_out; 3284 } 3285 3286 has_cache = count & SWAP_HAS_CACHE; 3287 count &= ~SWAP_HAS_CACHE; 3288 err = 0; 3289 3290 if (usage == SWAP_HAS_CACHE) { 3291 3292 /* set SWAP_HAS_CACHE if there is no cache and entry is used */ 3293 if (!has_cache && count) 3294 has_cache = SWAP_HAS_CACHE; 3295 else if (has_cache) /* someone else added cache */ 3296 err = -EEXIST; 3297 else /* no users remaining */ 3298 err = -ENOENT; 3299 3300 } else if (count || has_cache) { 3301 3302 if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX) 3303 count += usage; 3304 else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX) 3305 err = -EINVAL; 3306 else if (swap_count_continued(p, offset, count)) 3307 count = COUNT_CONTINUED; 3308 else 3309 err = -ENOMEM; 3310 } else 3311 err = -ENOENT; /* unused swap entry */ 3312 3313 WRITE_ONCE(p->swap_map[offset], count | has_cache); 3314 3315 unlock_out: 3316 unlock_cluster_or_swap_info(p, ci); 3317 if (p) 3318 put_swap_device(p); 3319 return err; 3320 } 3321 3322 /* 3323 * Help swapoff by noting that swap entry belongs to shmem/tmpfs 3324 * (in which case its reference count is never incremented). 3325 */ 3326 void swap_shmem_alloc(swp_entry_t entry) 3327 { 3328 __swap_duplicate(entry, SWAP_MAP_SHMEM); 3329 } 3330 3331 /* 3332 * Increase reference count of swap entry by 1. 3333 * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required 3334 * but could not be atomically allocated. Returns 0, just as if it succeeded, 3335 * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which 3336 * might occur if a page table entry has got corrupted. 3337 */ 3338 int swap_duplicate(swp_entry_t entry) 3339 { 3340 int err = 0; 3341 3342 while (!err && __swap_duplicate(entry, 1) == -ENOMEM) 3343 err = add_swap_count_continuation(entry, GFP_ATOMIC); 3344 return err; 3345 } 3346 3347 /* 3348 * @entry: swap entry for which we allocate swap cache. 3349 * 3350 * Called when allocating swap cache for existing swap entry, 3351 * This can return error codes. Returns 0 at success. 3352 * -EEXIST means there is a swap cache. 3353 * Note: return code is different from swap_duplicate(). 3354 */ 3355 int swapcache_prepare(swp_entry_t entry) 3356 { 3357 return __swap_duplicate(entry, SWAP_HAS_CACHE); 3358 } 3359 3360 struct swap_info_struct *swp_swap_info(swp_entry_t entry) 3361 { 3362 return swap_type_to_swap_info(swp_type(entry)); 3363 } 3364 3365 struct swap_info_struct *page_swap_info(struct page *page) 3366 { 3367 swp_entry_t entry = { .val = page_private(page) }; 3368 return swp_swap_info(entry); 3369 } 3370 3371 /* 3372 * out-of-line methods to avoid include hell. 3373 */ 3374 struct address_space *swapcache_mapping(struct folio *folio) 3375 { 3376 return page_swap_info(&folio->page)->swap_file->f_mapping; 3377 } 3378 EXPORT_SYMBOL_GPL(swapcache_mapping); 3379 3380 pgoff_t __page_file_index(struct page *page) 3381 { 3382 swp_entry_t swap = { .val = page_private(page) }; 3383 return swp_offset(swap); 3384 } 3385 EXPORT_SYMBOL_GPL(__page_file_index); 3386 3387 /* 3388 * add_swap_count_continuation - called when a swap count is duplicated 3389 * beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's 3390 * page of the original vmalloc'ed swap_map, to hold the continuation count 3391 * (for that entry and for its neighbouring PAGE_SIZE swap entries). Called 3392 * again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc. 3393 * 3394 * These continuation pages are seldom referenced: the common paths all work 3395 * on the original swap_map, only referring to a continuation page when the 3396 * low "digit" of a count is incremented or decremented through SWAP_MAP_MAX. 3397 * 3398 * add_swap_count_continuation(, GFP_ATOMIC) can be called while holding 3399 * page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL) 3400 * can be called after dropping locks. 3401 */ 3402 int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask) 3403 { 3404 struct swap_info_struct *si; 3405 struct swap_cluster_info *ci; 3406 struct page *head; 3407 struct page *page; 3408 struct page *list_page; 3409 pgoff_t offset; 3410 unsigned char count; 3411 int ret = 0; 3412 3413 /* 3414 * When debugging, it's easier to use __GFP_ZERO here; but it's better 3415 * for latency not to zero a page while GFP_ATOMIC and holding locks. 3416 */ 3417 page = alloc_page(gfp_mask | __GFP_HIGHMEM); 3418 3419 si = get_swap_device(entry); 3420 if (!si) { 3421 /* 3422 * An acceptable race has occurred since the failing 3423 * __swap_duplicate(): the swap device may be swapoff 3424 */ 3425 goto outer; 3426 } 3427 spin_lock(&si->lock); 3428 3429 offset = swp_offset(entry); 3430 3431 ci = lock_cluster(si, offset); 3432 3433 count = swap_count(si->swap_map[offset]); 3434 3435 if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) { 3436 /* 3437 * The higher the swap count, the more likely it is that tasks 3438 * will race to add swap count continuation: we need to avoid 3439 * over-provisioning. 3440 */ 3441 goto out; 3442 } 3443 3444 if (!page) { 3445 ret = -ENOMEM; 3446 goto out; 3447 } 3448 3449 /* 3450 * We are fortunate that although vmalloc_to_page uses pte_offset_map, 3451 * no architecture is using highmem pages for kernel page tables: so it 3452 * will not corrupt the GFP_ATOMIC caller's atomic page table kmaps. 3453 */ 3454 head = vmalloc_to_page(si->swap_map + offset); 3455 offset &= ~PAGE_MASK; 3456 3457 spin_lock(&si->cont_lock); 3458 /* 3459 * Page allocation does not initialize the page's lru field, 3460 * but it does always reset its private field. 3461 */ 3462 if (!page_private(head)) { 3463 BUG_ON(count & COUNT_CONTINUED); 3464 INIT_LIST_HEAD(&head->lru); 3465 set_page_private(head, SWP_CONTINUED); 3466 si->flags |= SWP_CONTINUED; 3467 } 3468 3469 list_for_each_entry(list_page, &head->lru, lru) { 3470 unsigned char *map; 3471 3472 /* 3473 * If the previous map said no continuation, but we've found 3474 * a continuation page, free our allocation and use this one. 3475 */ 3476 if (!(count & COUNT_CONTINUED)) 3477 goto out_unlock_cont; 3478 3479 map = kmap_atomic(list_page) + offset; 3480 count = *map; 3481 kunmap_atomic(map); 3482 3483 /* 3484 * If this continuation count now has some space in it, 3485 * free our allocation and use this one. 3486 */ 3487 if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX) 3488 goto out_unlock_cont; 3489 } 3490 3491 list_add_tail(&page->lru, &head->lru); 3492 page = NULL; /* now it's attached, don't free it */ 3493 out_unlock_cont: 3494 spin_unlock(&si->cont_lock); 3495 out: 3496 unlock_cluster(ci); 3497 spin_unlock(&si->lock); 3498 put_swap_device(si); 3499 outer: 3500 if (page) 3501 __free_page(page); 3502 return ret; 3503 } 3504 3505 /* 3506 * swap_count_continued - when the original swap_map count is incremented 3507 * from SWAP_MAP_MAX, check if there is already a continuation page to carry 3508 * into, carry if so, or else fail until a new continuation page is allocated; 3509 * when the original swap_map count is decremented from 0 with continuation, 3510 * borrow from the continuation and report whether it still holds more. 3511 * Called while __swap_duplicate() or swap_entry_free() holds swap or cluster 3512 * lock. 3513 */ 3514 static bool swap_count_continued(struct swap_info_struct *si, 3515 pgoff_t offset, unsigned char count) 3516 { 3517 struct page *head; 3518 struct page *page; 3519 unsigned char *map; 3520 bool ret; 3521 3522 head = vmalloc_to_page(si->swap_map + offset); 3523 if (page_private(head) != SWP_CONTINUED) { 3524 BUG_ON(count & COUNT_CONTINUED); 3525 return false; /* need to add count continuation */ 3526 } 3527 3528 spin_lock(&si->cont_lock); 3529 offset &= ~PAGE_MASK; 3530 page = list_next_entry(head, lru); 3531 map = kmap_atomic(page) + offset; 3532 3533 if (count == SWAP_MAP_MAX) /* initial increment from swap_map */ 3534 goto init_map; /* jump over SWAP_CONT_MAX checks */ 3535 3536 if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */ 3537 /* 3538 * Think of how you add 1 to 999 3539 */ 3540 while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) { 3541 kunmap_atomic(map); 3542 page = list_next_entry(page, lru); 3543 BUG_ON(page == head); 3544 map = kmap_atomic(page) + offset; 3545 } 3546 if (*map == SWAP_CONT_MAX) { 3547 kunmap_atomic(map); 3548 page = list_next_entry(page, lru); 3549 if (page == head) { 3550 ret = false; /* add count continuation */ 3551 goto out; 3552 } 3553 map = kmap_atomic(page) + offset; 3554 init_map: *map = 0; /* we didn't zero the page */ 3555 } 3556 *map += 1; 3557 kunmap_atomic(map); 3558 while ((page = list_prev_entry(page, lru)) != head) { 3559 map = kmap_atomic(page) + offset; 3560 *map = COUNT_CONTINUED; 3561 kunmap_atomic(map); 3562 } 3563 ret = true; /* incremented */ 3564 3565 } else { /* decrementing */ 3566 /* 3567 * Think of how you subtract 1 from 1000 3568 */ 3569 BUG_ON(count != COUNT_CONTINUED); 3570 while (*map == COUNT_CONTINUED) { 3571 kunmap_atomic(map); 3572 page = list_next_entry(page, lru); 3573 BUG_ON(page == head); 3574 map = kmap_atomic(page) + offset; 3575 } 3576 BUG_ON(*map == 0); 3577 *map -= 1; 3578 if (*map == 0) 3579 count = 0; 3580 kunmap_atomic(map); 3581 while ((page = list_prev_entry(page, lru)) != head) { 3582 map = kmap_atomic(page) + offset; 3583 *map = SWAP_CONT_MAX | count; 3584 count = COUNT_CONTINUED; 3585 kunmap_atomic(map); 3586 } 3587 ret = count == COUNT_CONTINUED; 3588 } 3589 out: 3590 spin_unlock(&si->cont_lock); 3591 return ret; 3592 } 3593 3594 /* 3595 * free_swap_count_continuations - swapoff free all the continuation pages 3596 * appended to the swap_map, after swap_map is quiesced, before vfree'ing it. 3597 */ 3598 static void free_swap_count_continuations(struct swap_info_struct *si) 3599 { 3600 pgoff_t offset; 3601 3602 for (offset = 0; offset < si->max; offset += PAGE_SIZE) { 3603 struct page *head; 3604 head = vmalloc_to_page(si->swap_map + offset); 3605 if (page_private(head)) { 3606 struct page *page, *next; 3607 3608 list_for_each_entry_safe(page, next, &head->lru, lru) { 3609 list_del(&page->lru); 3610 __free_page(page); 3611 } 3612 } 3613 } 3614 } 3615 3616 #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) 3617 void __cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) 3618 { 3619 struct swap_info_struct *si, *next; 3620 int nid = page_to_nid(page); 3621 3622 if (!(gfp_mask & __GFP_IO)) 3623 return; 3624 3625 if (!blk_cgroup_congested()) 3626 return; 3627 3628 /* 3629 * We've already scheduled a throttle, avoid taking the global swap 3630 * lock. 3631 */ 3632 if (current->throttle_queue) 3633 return; 3634 3635 spin_lock(&swap_avail_lock); 3636 plist_for_each_entry_safe(si, next, &swap_avail_heads[nid], 3637 avail_lists[nid]) { 3638 if (si->bdev) { 3639 blkcg_schedule_throttle(bdev_get_queue(si->bdev), true); 3640 break; 3641 } 3642 } 3643 spin_unlock(&swap_avail_lock); 3644 } 3645 #endif 3646 3647 static int __init swapfile_init(void) 3648 { 3649 int nid; 3650 3651 swap_avail_heads = kmalloc_array(nr_node_ids, sizeof(struct plist_head), 3652 GFP_KERNEL); 3653 if (!swap_avail_heads) { 3654 pr_emerg("Not enough memory for swap heads, swap is disabled\n"); 3655 return -ENOMEM; 3656 } 3657 3658 for_each_node(nid) 3659 plist_head_init(&swap_avail_heads[nid]); 3660 3661 return 0; 3662 } 3663 subsys_initcall(swapfile_init); 3664