1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/swapfile.c 4 * 5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 6 * Swap reorganised 29.12.95, Stephen Tweedie 7 */ 8 9 #include <linux/mm.h> 10 #include <linux/sched/mm.h> 11 #include <linux/sched/task.h> 12 #include <linux/hugetlb.h> 13 #include <linux/mman.h> 14 #include <linux/slab.h> 15 #include <linux/kernel_stat.h> 16 #include <linux/swap.h> 17 #include <linux/vmalloc.h> 18 #include <linux/pagemap.h> 19 #include <linux/namei.h> 20 #include <linux/shmem_fs.h> 21 #include <linux/blkdev.h> 22 #include <linux/random.h> 23 #include <linux/writeback.h> 24 #include <linux/proc_fs.h> 25 #include <linux/seq_file.h> 26 #include <linux/init.h> 27 #include <linux/ksm.h> 28 #include <linux/rmap.h> 29 #include <linux/security.h> 30 #include <linux/backing-dev.h> 31 #include <linux/mutex.h> 32 #include <linux/capability.h> 33 #include <linux/syscalls.h> 34 #include <linux/memcontrol.h> 35 #include <linux/poll.h> 36 #include <linux/oom.h> 37 #include <linux/frontswap.h> 38 #include <linux/swapfile.h> 39 #include <linux/export.h> 40 #include <linux/swap_slots.h> 41 #include <linux/sort.h> 42 43 #include <asm/tlbflush.h> 44 #include <linux/swapops.h> 45 #include <linux/swap_cgroup.h> 46 47 static bool swap_count_continued(struct swap_info_struct *, pgoff_t, 48 unsigned char); 49 static void free_swap_count_continuations(struct swap_info_struct *); 50 static sector_t map_swap_entry(swp_entry_t, struct block_device**); 51 52 DEFINE_SPINLOCK(swap_lock); 53 static unsigned int nr_swapfiles; 54 atomic_long_t nr_swap_pages; 55 /* 56 * Some modules use swappable objects and may try to swap them out under 57 * memory pressure (via the shrinker). Before doing so, they may wish to 58 * check to see if any swap space is available. 59 */ 60 EXPORT_SYMBOL_GPL(nr_swap_pages); 61 /* protected with swap_lock. reading in vm_swap_full() doesn't need lock */ 62 long total_swap_pages; 63 static int least_priority = -1; 64 65 static const char Bad_file[] = "Bad swap file entry "; 66 static const char Unused_file[] = "Unused swap file entry "; 67 static const char Bad_offset[] = "Bad swap offset entry "; 68 static const char Unused_offset[] = "Unused swap offset entry "; 69 70 /* 71 * all active swap_info_structs 72 * protected with swap_lock, and ordered by priority. 73 */ 74 PLIST_HEAD(swap_active_head); 75 76 /* 77 * all available (active, not full) swap_info_structs 78 * protected with swap_avail_lock, ordered by priority. 79 * This is used by get_swap_page() instead of swap_active_head 80 * because swap_active_head includes all swap_info_structs, 81 * but get_swap_page() doesn't need to look at full ones. 82 * This uses its own lock instead of swap_lock because when a 83 * swap_info_struct changes between not-full/full, it needs to 84 * add/remove itself to/from this list, but the swap_info_struct->lock 85 * is held and the locking order requires swap_lock to be taken 86 * before any swap_info_struct->lock. 87 */ 88 static struct plist_head *swap_avail_heads; 89 static DEFINE_SPINLOCK(swap_avail_lock); 90 91 struct swap_info_struct *swap_info[MAX_SWAPFILES]; 92 93 static DEFINE_MUTEX(swapon_mutex); 94 95 static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait); 96 /* Activity counter to indicate that a swapon or swapoff has occurred */ 97 static atomic_t proc_poll_event = ATOMIC_INIT(0); 98 99 atomic_t nr_rotate_swap = ATOMIC_INIT(0); 100 101 static struct swap_info_struct *swap_type_to_swap_info(int type) 102 { 103 if (type >= READ_ONCE(nr_swapfiles)) 104 return NULL; 105 106 smp_rmb(); /* Pairs with smp_wmb in alloc_swap_info. */ 107 return READ_ONCE(swap_info[type]); 108 } 109 110 static inline unsigned char swap_count(unsigned char ent) 111 { 112 return ent & ~SWAP_HAS_CACHE; /* may include COUNT_CONTINUED flag */ 113 } 114 115 /* Reclaim the swap entry anyway if possible */ 116 #define TTRS_ANYWAY 0x1 117 /* 118 * Reclaim the swap entry if there are no more mappings of the 119 * corresponding page 120 */ 121 #define TTRS_UNMAPPED 0x2 122 /* Reclaim the swap entry if swap is getting full*/ 123 #define TTRS_FULL 0x4 124 125 /* returns 1 if swap entry is freed */ 126 static int __try_to_reclaim_swap(struct swap_info_struct *si, 127 unsigned long offset, unsigned long flags) 128 { 129 swp_entry_t entry = swp_entry(si->type, offset); 130 struct page *page; 131 int ret = 0; 132 133 page = find_get_page(swap_address_space(entry), offset); 134 if (!page) 135 return 0; 136 /* 137 * When this function is called from scan_swap_map_slots() and it's 138 * called by vmscan.c at reclaiming pages. So, we hold a lock on a page, 139 * here. We have to use trylock for avoiding deadlock. This is a special 140 * case and you should use try_to_free_swap() with explicit lock_page() 141 * in usual operations. 142 */ 143 if (trylock_page(page)) { 144 if ((flags & TTRS_ANYWAY) || 145 ((flags & TTRS_UNMAPPED) && !page_mapped(page)) || 146 ((flags & TTRS_FULL) && mem_cgroup_swap_full(page))) 147 ret = try_to_free_swap(page); 148 unlock_page(page); 149 } 150 put_page(page); 151 return ret; 152 } 153 154 static inline struct swap_extent *first_se(struct swap_info_struct *sis) 155 { 156 struct rb_node *rb = rb_first(&sis->swap_extent_root); 157 return rb_entry(rb, struct swap_extent, rb_node); 158 } 159 160 static inline struct swap_extent *next_se(struct swap_extent *se) 161 { 162 struct rb_node *rb = rb_next(&se->rb_node); 163 return rb ? rb_entry(rb, struct swap_extent, rb_node) : NULL; 164 } 165 166 /* 167 * swapon tell device that all the old swap contents can be discarded, 168 * to allow the swap device to optimize its wear-levelling. 169 */ 170 static int discard_swap(struct swap_info_struct *si) 171 { 172 struct swap_extent *se; 173 sector_t start_block; 174 sector_t nr_blocks; 175 int err = 0; 176 177 /* Do not discard the swap header page! */ 178 se = first_se(si); 179 start_block = (se->start_block + 1) << (PAGE_SHIFT - 9); 180 nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9); 181 if (nr_blocks) { 182 err = blkdev_issue_discard(si->bdev, start_block, 183 nr_blocks, GFP_KERNEL, 0); 184 if (err) 185 return err; 186 cond_resched(); 187 } 188 189 for (se = next_se(se); se; se = next_se(se)) { 190 start_block = se->start_block << (PAGE_SHIFT - 9); 191 nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9); 192 193 err = blkdev_issue_discard(si->bdev, start_block, 194 nr_blocks, GFP_KERNEL, 0); 195 if (err) 196 break; 197 198 cond_resched(); 199 } 200 return err; /* That will often be -EOPNOTSUPP */ 201 } 202 203 static struct swap_extent * 204 offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset) 205 { 206 struct swap_extent *se; 207 struct rb_node *rb; 208 209 rb = sis->swap_extent_root.rb_node; 210 while (rb) { 211 se = rb_entry(rb, struct swap_extent, rb_node); 212 if (offset < se->start_page) 213 rb = rb->rb_left; 214 else if (offset >= se->start_page + se->nr_pages) 215 rb = rb->rb_right; 216 else 217 return se; 218 } 219 /* It *must* be present */ 220 BUG(); 221 } 222 223 /* 224 * swap allocation tell device that a cluster of swap can now be discarded, 225 * to allow the swap device to optimize its wear-levelling. 226 */ 227 static void discard_swap_cluster(struct swap_info_struct *si, 228 pgoff_t start_page, pgoff_t nr_pages) 229 { 230 struct swap_extent *se = offset_to_swap_extent(si, start_page); 231 232 while (nr_pages) { 233 pgoff_t offset = start_page - se->start_page; 234 sector_t start_block = se->start_block + offset; 235 sector_t nr_blocks = se->nr_pages - offset; 236 237 if (nr_blocks > nr_pages) 238 nr_blocks = nr_pages; 239 start_page += nr_blocks; 240 nr_pages -= nr_blocks; 241 242 start_block <<= PAGE_SHIFT - 9; 243 nr_blocks <<= PAGE_SHIFT - 9; 244 if (blkdev_issue_discard(si->bdev, start_block, 245 nr_blocks, GFP_NOIO, 0)) 246 break; 247 248 se = next_se(se); 249 } 250 } 251 252 #ifdef CONFIG_THP_SWAP 253 #define SWAPFILE_CLUSTER HPAGE_PMD_NR 254 255 #define swap_entry_size(size) (size) 256 #else 257 #define SWAPFILE_CLUSTER 256 258 259 /* 260 * Define swap_entry_size() as constant to let compiler to optimize 261 * out some code if !CONFIG_THP_SWAP 262 */ 263 #define swap_entry_size(size) 1 264 #endif 265 #define LATENCY_LIMIT 256 266 267 static inline void cluster_set_flag(struct swap_cluster_info *info, 268 unsigned int flag) 269 { 270 info->flags = flag; 271 } 272 273 static inline unsigned int cluster_count(struct swap_cluster_info *info) 274 { 275 return info->data; 276 } 277 278 static inline void cluster_set_count(struct swap_cluster_info *info, 279 unsigned int c) 280 { 281 info->data = c; 282 } 283 284 static inline void cluster_set_count_flag(struct swap_cluster_info *info, 285 unsigned int c, unsigned int f) 286 { 287 info->flags = f; 288 info->data = c; 289 } 290 291 static inline unsigned int cluster_next(struct swap_cluster_info *info) 292 { 293 return info->data; 294 } 295 296 static inline void cluster_set_next(struct swap_cluster_info *info, 297 unsigned int n) 298 { 299 info->data = n; 300 } 301 302 static inline void cluster_set_next_flag(struct swap_cluster_info *info, 303 unsigned int n, unsigned int f) 304 { 305 info->flags = f; 306 info->data = n; 307 } 308 309 static inline bool cluster_is_free(struct swap_cluster_info *info) 310 { 311 return info->flags & CLUSTER_FLAG_FREE; 312 } 313 314 static inline bool cluster_is_null(struct swap_cluster_info *info) 315 { 316 return info->flags & CLUSTER_FLAG_NEXT_NULL; 317 } 318 319 static inline void cluster_set_null(struct swap_cluster_info *info) 320 { 321 info->flags = CLUSTER_FLAG_NEXT_NULL; 322 info->data = 0; 323 } 324 325 static inline bool cluster_is_huge(struct swap_cluster_info *info) 326 { 327 if (IS_ENABLED(CONFIG_THP_SWAP)) 328 return info->flags & CLUSTER_FLAG_HUGE; 329 return false; 330 } 331 332 static inline void cluster_clear_huge(struct swap_cluster_info *info) 333 { 334 info->flags &= ~CLUSTER_FLAG_HUGE; 335 } 336 337 static inline struct swap_cluster_info *lock_cluster(struct swap_info_struct *si, 338 unsigned long offset) 339 { 340 struct swap_cluster_info *ci; 341 342 ci = si->cluster_info; 343 if (ci) { 344 ci += offset / SWAPFILE_CLUSTER; 345 spin_lock(&ci->lock); 346 } 347 return ci; 348 } 349 350 static inline void unlock_cluster(struct swap_cluster_info *ci) 351 { 352 if (ci) 353 spin_unlock(&ci->lock); 354 } 355 356 /* 357 * Determine the locking method in use for this device. Return 358 * swap_cluster_info if SSD-style cluster-based locking is in place. 359 */ 360 static inline struct swap_cluster_info *lock_cluster_or_swap_info( 361 struct swap_info_struct *si, unsigned long offset) 362 { 363 struct swap_cluster_info *ci; 364 365 /* Try to use fine-grained SSD-style locking if available: */ 366 ci = lock_cluster(si, offset); 367 /* Otherwise, fall back to traditional, coarse locking: */ 368 if (!ci) 369 spin_lock(&si->lock); 370 371 return ci; 372 } 373 374 static inline void unlock_cluster_or_swap_info(struct swap_info_struct *si, 375 struct swap_cluster_info *ci) 376 { 377 if (ci) 378 unlock_cluster(ci); 379 else 380 spin_unlock(&si->lock); 381 } 382 383 static inline bool cluster_list_empty(struct swap_cluster_list *list) 384 { 385 return cluster_is_null(&list->head); 386 } 387 388 static inline unsigned int cluster_list_first(struct swap_cluster_list *list) 389 { 390 return cluster_next(&list->head); 391 } 392 393 static void cluster_list_init(struct swap_cluster_list *list) 394 { 395 cluster_set_null(&list->head); 396 cluster_set_null(&list->tail); 397 } 398 399 static void cluster_list_add_tail(struct swap_cluster_list *list, 400 struct swap_cluster_info *ci, 401 unsigned int idx) 402 { 403 if (cluster_list_empty(list)) { 404 cluster_set_next_flag(&list->head, idx, 0); 405 cluster_set_next_flag(&list->tail, idx, 0); 406 } else { 407 struct swap_cluster_info *ci_tail; 408 unsigned int tail = cluster_next(&list->tail); 409 410 /* 411 * Nested cluster lock, but both cluster locks are 412 * only acquired when we held swap_info_struct->lock 413 */ 414 ci_tail = ci + tail; 415 spin_lock_nested(&ci_tail->lock, SINGLE_DEPTH_NESTING); 416 cluster_set_next(ci_tail, idx); 417 spin_unlock(&ci_tail->lock); 418 cluster_set_next_flag(&list->tail, idx, 0); 419 } 420 } 421 422 static unsigned int cluster_list_del_first(struct swap_cluster_list *list, 423 struct swap_cluster_info *ci) 424 { 425 unsigned int idx; 426 427 idx = cluster_next(&list->head); 428 if (cluster_next(&list->tail) == idx) { 429 cluster_set_null(&list->head); 430 cluster_set_null(&list->tail); 431 } else 432 cluster_set_next_flag(&list->head, 433 cluster_next(&ci[idx]), 0); 434 435 return idx; 436 } 437 438 /* Add a cluster to discard list and schedule it to do discard */ 439 static void swap_cluster_schedule_discard(struct swap_info_struct *si, 440 unsigned int idx) 441 { 442 /* 443 * If scan_swap_map() can't find a free cluster, it will check 444 * si->swap_map directly. To make sure the discarding cluster isn't 445 * taken by scan_swap_map(), mark the swap entries bad (occupied). It 446 * will be cleared after discard 447 */ 448 memset(si->swap_map + idx * SWAPFILE_CLUSTER, 449 SWAP_MAP_BAD, SWAPFILE_CLUSTER); 450 451 cluster_list_add_tail(&si->discard_clusters, si->cluster_info, idx); 452 453 schedule_work(&si->discard_work); 454 } 455 456 static void __free_cluster(struct swap_info_struct *si, unsigned long idx) 457 { 458 struct swap_cluster_info *ci = si->cluster_info; 459 460 cluster_set_flag(ci + idx, CLUSTER_FLAG_FREE); 461 cluster_list_add_tail(&si->free_clusters, ci, idx); 462 } 463 464 /* 465 * Doing discard actually. After a cluster discard is finished, the cluster 466 * will be added to free cluster list. caller should hold si->lock. 467 */ 468 static void swap_do_scheduled_discard(struct swap_info_struct *si) 469 { 470 struct swap_cluster_info *info, *ci; 471 unsigned int idx; 472 473 info = si->cluster_info; 474 475 while (!cluster_list_empty(&si->discard_clusters)) { 476 idx = cluster_list_del_first(&si->discard_clusters, info); 477 spin_unlock(&si->lock); 478 479 discard_swap_cluster(si, idx * SWAPFILE_CLUSTER, 480 SWAPFILE_CLUSTER); 481 482 spin_lock(&si->lock); 483 ci = lock_cluster(si, idx * SWAPFILE_CLUSTER); 484 __free_cluster(si, idx); 485 memset(si->swap_map + idx * SWAPFILE_CLUSTER, 486 0, SWAPFILE_CLUSTER); 487 unlock_cluster(ci); 488 } 489 } 490 491 static void swap_discard_work(struct work_struct *work) 492 { 493 struct swap_info_struct *si; 494 495 si = container_of(work, struct swap_info_struct, discard_work); 496 497 spin_lock(&si->lock); 498 swap_do_scheduled_discard(si); 499 spin_unlock(&si->lock); 500 } 501 502 static void alloc_cluster(struct swap_info_struct *si, unsigned long idx) 503 { 504 struct swap_cluster_info *ci = si->cluster_info; 505 506 VM_BUG_ON(cluster_list_first(&si->free_clusters) != idx); 507 cluster_list_del_first(&si->free_clusters, ci); 508 cluster_set_count_flag(ci + idx, 0, 0); 509 } 510 511 static void free_cluster(struct swap_info_struct *si, unsigned long idx) 512 { 513 struct swap_cluster_info *ci = si->cluster_info + idx; 514 515 VM_BUG_ON(cluster_count(ci) != 0); 516 /* 517 * If the swap is discardable, prepare discard the cluster 518 * instead of free it immediately. The cluster will be freed 519 * after discard. 520 */ 521 if ((si->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) == 522 (SWP_WRITEOK | SWP_PAGE_DISCARD)) { 523 swap_cluster_schedule_discard(si, idx); 524 return; 525 } 526 527 __free_cluster(si, idx); 528 } 529 530 /* 531 * The cluster corresponding to page_nr will be used. The cluster will be 532 * removed from free cluster list and its usage counter will be increased. 533 */ 534 static void inc_cluster_info_page(struct swap_info_struct *p, 535 struct swap_cluster_info *cluster_info, unsigned long page_nr) 536 { 537 unsigned long idx = page_nr / SWAPFILE_CLUSTER; 538 539 if (!cluster_info) 540 return; 541 if (cluster_is_free(&cluster_info[idx])) 542 alloc_cluster(p, idx); 543 544 VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER); 545 cluster_set_count(&cluster_info[idx], 546 cluster_count(&cluster_info[idx]) + 1); 547 } 548 549 /* 550 * The cluster corresponding to page_nr decreases one usage. If the usage 551 * counter becomes 0, which means no page in the cluster is in using, we can 552 * optionally discard the cluster and add it to free cluster list. 553 */ 554 static void dec_cluster_info_page(struct swap_info_struct *p, 555 struct swap_cluster_info *cluster_info, unsigned long page_nr) 556 { 557 unsigned long idx = page_nr / SWAPFILE_CLUSTER; 558 559 if (!cluster_info) 560 return; 561 562 VM_BUG_ON(cluster_count(&cluster_info[idx]) == 0); 563 cluster_set_count(&cluster_info[idx], 564 cluster_count(&cluster_info[idx]) - 1); 565 566 if (cluster_count(&cluster_info[idx]) == 0) 567 free_cluster(p, idx); 568 } 569 570 /* 571 * It's possible scan_swap_map() uses a free cluster in the middle of free 572 * cluster list. Avoiding such abuse to avoid list corruption. 573 */ 574 static bool 575 scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si, 576 unsigned long offset) 577 { 578 struct percpu_cluster *percpu_cluster; 579 bool conflict; 580 581 offset /= SWAPFILE_CLUSTER; 582 conflict = !cluster_list_empty(&si->free_clusters) && 583 offset != cluster_list_first(&si->free_clusters) && 584 cluster_is_free(&si->cluster_info[offset]); 585 586 if (!conflict) 587 return false; 588 589 percpu_cluster = this_cpu_ptr(si->percpu_cluster); 590 cluster_set_null(&percpu_cluster->index); 591 return true; 592 } 593 594 /* 595 * Try to get a swap entry from current cpu's swap entry pool (a cluster). This 596 * might involve allocating a new cluster for current CPU too. 597 */ 598 static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si, 599 unsigned long *offset, unsigned long *scan_base) 600 { 601 struct percpu_cluster *cluster; 602 struct swap_cluster_info *ci; 603 unsigned long tmp, max; 604 605 new_cluster: 606 cluster = this_cpu_ptr(si->percpu_cluster); 607 if (cluster_is_null(&cluster->index)) { 608 if (!cluster_list_empty(&si->free_clusters)) { 609 cluster->index = si->free_clusters.head; 610 cluster->next = cluster_next(&cluster->index) * 611 SWAPFILE_CLUSTER; 612 } else if (!cluster_list_empty(&si->discard_clusters)) { 613 /* 614 * we don't have free cluster but have some clusters in 615 * discarding, do discard now and reclaim them, then 616 * reread cluster_next_cpu since we dropped si->lock 617 */ 618 swap_do_scheduled_discard(si); 619 *scan_base = this_cpu_read(*si->cluster_next_cpu); 620 *offset = *scan_base; 621 goto new_cluster; 622 } else 623 return false; 624 } 625 626 /* 627 * Other CPUs can use our cluster if they can't find a free cluster, 628 * check if there is still free entry in the cluster 629 */ 630 tmp = cluster->next; 631 max = min_t(unsigned long, si->max, 632 (cluster_next(&cluster->index) + 1) * SWAPFILE_CLUSTER); 633 if (tmp < max) { 634 ci = lock_cluster(si, tmp); 635 while (tmp < max) { 636 if (!si->swap_map[tmp]) 637 break; 638 tmp++; 639 } 640 unlock_cluster(ci); 641 } 642 if (tmp >= max) { 643 cluster_set_null(&cluster->index); 644 goto new_cluster; 645 } 646 cluster->next = tmp + 1; 647 *offset = tmp; 648 *scan_base = tmp; 649 return true; 650 } 651 652 static void __del_from_avail_list(struct swap_info_struct *p) 653 { 654 int nid; 655 656 for_each_node(nid) 657 plist_del(&p->avail_lists[nid], &swap_avail_heads[nid]); 658 } 659 660 static void del_from_avail_list(struct swap_info_struct *p) 661 { 662 spin_lock(&swap_avail_lock); 663 __del_from_avail_list(p); 664 spin_unlock(&swap_avail_lock); 665 } 666 667 static void swap_range_alloc(struct swap_info_struct *si, unsigned long offset, 668 unsigned int nr_entries) 669 { 670 unsigned int end = offset + nr_entries - 1; 671 672 if (offset == si->lowest_bit) 673 si->lowest_bit += nr_entries; 674 if (end == si->highest_bit) 675 si->highest_bit -= nr_entries; 676 si->inuse_pages += nr_entries; 677 if (si->inuse_pages == si->pages) { 678 si->lowest_bit = si->max; 679 si->highest_bit = 0; 680 del_from_avail_list(si); 681 } 682 } 683 684 static void add_to_avail_list(struct swap_info_struct *p) 685 { 686 int nid; 687 688 spin_lock(&swap_avail_lock); 689 for_each_node(nid) { 690 WARN_ON(!plist_node_empty(&p->avail_lists[nid])); 691 plist_add(&p->avail_lists[nid], &swap_avail_heads[nid]); 692 } 693 spin_unlock(&swap_avail_lock); 694 } 695 696 static void swap_range_free(struct swap_info_struct *si, unsigned long offset, 697 unsigned int nr_entries) 698 { 699 unsigned long begin = offset; 700 unsigned long end = offset + nr_entries - 1; 701 void (*swap_slot_free_notify)(struct block_device *, unsigned long); 702 703 if (offset < si->lowest_bit) 704 si->lowest_bit = offset; 705 if (end > si->highest_bit) { 706 bool was_full = !si->highest_bit; 707 708 si->highest_bit = end; 709 if (was_full && (si->flags & SWP_WRITEOK)) 710 add_to_avail_list(si); 711 } 712 atomic_long_add(nr_entries, &nr_swap_pages); 713 si->inuse_pages -= nr_entries; 714 if (si->flags & SWP_BLKDEV) 715 swap_slot_free_notify = 716 si->bdev->bd_disk->fops->swap_slot_free_notify; 717 else 718 swap_slot_free_notify = NULL; 719 while (offset <= end) { 720 frontswap_invalidate_page(si->type, offset); 721 if (swap_slot_free_notify) 722 swap_slot_free_notify(si->bdev, offset); 723 offset++; 724 } 725 clear_shadow_from_swap_cache(si->type, begin, end); 726 } 727 728 static void set_cluster_next(struct swap_info_struct *si, unsigned long next) 729 { 730 unsigned long prev; 731 732 if (!(si->flags & SWP_SOLIDSTATE)) { 733 si->cluster_next = next; 734 return; 735 } 736 737 prev = this_cpu_read(*si->cluster_next_cpu); 738 /* 739 * Cross the swap address space size aligned trunk, choose 740 * another trunk randomly to avoid lock contention on swap 741 * address space if possible. 742 */ 743 if ((prev >> SWAP_ADDRESS_SPACE_SHIFT) != 744 (next >> SWAP_ADDRESS_SPACE_SHIFT)) { 745 /* No free swap slots available */ 746 if (si->highest_bit <= si->lowest_bit) 747 return; 748 next = si->lowest_bit + 749 prandom_u32_max(si->highest_bit - si->lowest_bit + 1); 750 next = ALIGN_DOWN(next, SWAP_ADDRESS_SPACE_PAGES); 751 next = max_t(unsigned int, next, si->lowest_bit); 752 } 753 this_cpu_write(*si->cluster_next_cpu, next); 754 } 755 756 static int scan_swap_map_slots(struct swap_info_struct *si, 757 unsigned char usage, int nr, 758 swp_entry_t slots[]) 759 { 760 struct swap_cluster_info *ci; 761 unsigned long offset; 762 unsigned long scan_base; 763 unsigned long last_in_cluster = 0; 764 int latency_ration = LATENCY_LIMIT; 765 int n_ret = 0; 766 bool scanned_many = false; 767 768 /* 769 * We try to cluster swap pages by allocating them sequentially 770 * in swap. Once we've allocated SWAPFILE_CLUSTER pages this 771 * way, however, we resort to first-free allocation, starting 772 * a new cluster. This prevents us from scattering swap pages 773 * all over the entire swap partition, so that we reduce 774 * overall disk seek times between swap pages. -- sct 775 * But we do now try to find an empty cluster. -Andrea 776 * And we let swap pages go all over an SSD partition. Hugh 777 */ 778 779 si->flags += SWP_SCANNING; 780 /* 781 * Use percpu scan base for SSD to reduce lock contention on 782 * cluster and swap cache. For HDD, sequential access is more 783 * important. 784 */ 785 if (si->flags & SWP_SOLIDSTATE) 786 scan_base = this_cpu_read(*si->cluster_next_cpu); 787 else 788 scan_base = si->cluster_next; 789 offset = scan_base; 790 791 /* SSD algorithm */ 792 if (si->cluster_info) { 793 if (!scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) 794 goto scan; 795 } else if (unlikely(!si->cluster_nr--)) { 796 if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) { 797 si->cluster_nr = SWAPFILE_CLUSTER - 1; 798 goto checks; 799 } 800 801 spin_unlock(&si->lock); 802 803 /* 804 * If seek is expensive, start searching for new cluster from 805 * start of partition, to minimize the span of allocated swap. 806 * If seek is cheap, that is the SWP_SOLIDSTATE si->cluster_info 807 * case, just handled by scan_swap_map_try_ssd_cluster() above. 808 */ 809 scan_base = offset = si->lowest_bit; 810 last_in_cluster = offset + SWAPFILE_CLUSTER - 1; 811 812 /* Locate the first empty (unaligned) cluster */ 813 for (; last_in_cluster <= si->highest_bit; offset++) { 814 if (si->swap_map[offset]) 815 last_in_cluster = offset + SWAPFILE_CLUSTER; 816 else if (offset == last_in_cluster) { 817 spin_lock(&si->lock); 818 offset -= SWAPFILE_CLUSTER - 1; 819 si->cluster_next = offset; 820 si->cluster_nr = SWAPFILE_CLUSTER - 1; 821 goto checks; 822 } 823 if (unlikely(--latency_ration < 0)) { 824 cond_resched(); 825 latency_ration = LATENCY_LIMIT; 826 } 827 } 828 829 offset = scan_base; 830 spin_lock(&si->lock); 831 si->cluster_nr = SWAPFILE_CLUSTER - 1; 832 } 833 834 checks: 835 if (si->cluster_info) { 836 while (scan_swap_map_ssd_cluster_conflict(si, offset)) { 837 /* take a break if we already got some slots */ 838 if (n_ret) 839 goto done; 840 if (!scan_swap_map_try_ssd_cluster(si, &offset, 841 &scan_base)) 842 goto scan; 843 } 844 } 845 if (!(si->flags & SWP_WRITEOK)) 846 goto no_page; 847 if (!si->highest_bit) 848 goto no_page; 849 if (offset > si->highest_bit) 850 scan_base = offset = si->lowest_bit; 851 852 ci = lock_cluster(si, offset); 853 /* reuse swap entry of cache-only swap if not busy. */ 854 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) { 855 int swap_was_freed; 856 unlock_cluster(ci); 857 spin_unlock(&si->lock); 858 swap_was_freed = __try_to_reclaim_swap(si, offset, TTRS_ANYWAY); 859 spin_lock(&si->lock); 860 /* entry was freed successfully, try to use this again */ 861 if (swap_was_freed) 862 goto checks; 863 goto scan; /* check next one */ 864 } 865 866 if (si->swap_map[offset]) { 867 unlock_cluster(ci); 868 if (!n_ret) 869 goto scan; 870 else 871 goto done; 872 } 873 si->swap_map[offset] = usage; 874 inc_cluster_info_page(si, si->cluster_info, offset); 875 unlock_cluster(ci); 876 877 swap_range_alloc(si, offset, 1); 878 slots[n_ret++] = swp_entry(si->type, offset); 879 880 /* got enough slots or reach max slots? */ 881 if ((n_ret == nr) || (offset >= si->highest_bit)) 882 goto done; 883 884 /* search for next available slot */ 885 886 /* time to take a break? */ 887 if (unlikely(--latency_ration < 0)) { 888 if (n_ret) 889 goto done; 890 spin_unlock(&si->lock); 891 cond_resched(); 892 spin_lock(&si->lock); 893 latency_ration = LATENCY_LIMIT; 894 } 895 896 /* try to get more slots in cluster */ 897 if (si->cluster_info) { 898 if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) 899 goto checks; 900 } else if (si->cluster_nr && !si->swap_map[++offset]) { 901 /* non-ssd case, still more slots in cluster? */ 902 --si->cluster_nr; 903 goto checks; 904 } 905 906 /* 907 * Even if there's no free clusters available (fragmented), 908 * try to scan a little more quickly with lock held unless we 909 * have scanned too many slots already. 910 */ 911 if (!scanned_many) { 912 unsigned long scan_limit; 913 914 if (offset < scan_base) 915 scan_limit = scan_base; 916 else 917 scan_limit = si->highest_bit; 918 for (; offset <= scan_limit && --latency_ration > 0; 919 offset++) { 920 if (!si->swap_map[offset]) 921 goto checks; 922 } 923 } 924 925 done: 926 set_cluster_next(si, offset + 1); 927 si->flags -= SWP_SCANNING; 928 return n_ret; 929 930 scan: 931 spin_unlock(&si->lock); 932 while (++offset <= si->highest_bit) { 933 if (!si->swap_map[offset]) { 934 spin_lock(&si->lock); 935 goto checks; 936 } 937 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) { 938 spin_lock(&si->lock); 939 goto checks; 940 } 941 if (unlikely(--latency_ration < 0)) { 942 cond_resched(); 943 latency_ration = LATENCY_LIMIT; 944 scanned_many = true; 945 } 946 } 947 offset = si->lowest_bit; 948 while (offset < scan_base) { 949 if (!si->swap_map[offset]) { 950 spin_lock(&si->lock); 951 goto checks; 952 } 953 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) { 954 spin_lock(&si->lock); 955 goto checks; 956 } 957 if (unlikely(--latency_ration < 0)) { 958 cond_resched(); 959 latency_ration = LATENCY_LIMIT; 960 scanned_many = true; 961 } 962 offset++; 963 } 964 spin_lock(&si->lock); 965 966 no_page: 967 si->flags -= SWP_SCANNING; 968 return n_ret; 969 } 970 971 static int swap_alloc_cluster(struct swap_info_struct *si, swp_entry_t *slot) 972 { 973 unsigned long idx; 974 struct swap_cluster_info *ci; 975 unsigned long offset, i; 976 unsigned char *map; 977 978 /* 979 * Should not even be attempting cluster allocations when huge 980 * page swap is disabled. Warn and fail the allocation. 981 */ 982 if (!IS_ENABLED(CONFIG_THP_SWAP)) { 983 VM_WARN_ON_ONCE(1); 984 return 0; 985 } 986 987 if (cluster_list_empty(&si->free_clusters)) 988 return 0; 989 990 idx = cluster_list_first(&si->free_clusters); 991 offset = idx * SWAPFILE_CLUSTER; 992 ci = lock_cluster(si, offset); 993 alloc_cluster(si, idx); 994 cluster_set_count_flag(ci, SWAPFILE_CLUSTER, CLUSTER_FLAG_HUGE); 995 996 map = si->swap_map + offset; 997 for (i = 0; i < SWAPFILE_CLUSTER; i++) 998 map[i] = SWAP_HAS_CACHE; 999 unlock_cluster(ci); 1000 swap_range_alloc(si, offset, SWAPFILE_CLUSTER); 1001 *slot = swp_entry(si->type, offset); 1002 1003 return 1; 1004 } 1005 1006 static void swap_free_cluster(struct swap_info_struct *si, unsigned long idx) 1007 { 1008 unsigned long offset = idx * SWAPFILE_CLUSTER; 1009 struct swap_cluster_info *ci; 1010 1011 ci = lock_cluster(si, offset); 1012 memset(si->swap_map + offset, 0, SWAPFILE_CLUSTER); 1013 cluster_set_count_flag(ci, 0, 0); 1014 free_cluster(si, idx); 1015 unlock_cluster(ci); 1016 swap_range_free(si, offset, SWAPFILE_CLUSTER); 1017 } 1018 1019 static unsigned long scan_swap_map(struct swap_info_struct *si, 1020 unsigned char usage) 1021 { 1022 swp_entry_t entry; 1023 int n_ret; 1024 1025 n_ret = scan_swap_map_slots(si, usage, 1, &entry); 1026 1027 if (n_ret) 1028 return swp_offset(entry); 1029 else 1030 return 0; 1031 1032 } 1033 1034 int get_swap_pages(int n_goal, swp_entry_t swp_entries[], int entry_size) 1035 { 1036 unsigned long size = swap_entry_size(entry_size); 1037 struct swap_info_struct *si, *next; 1038 long avail_pgs; 1039 int n_ret = 0; 1040 int node; 1041 1042 /* Only single cluster request supported */ 1043 WARN_ON_ONCE(n_goal > 1 && size == SWAPFILE_CLUSTER); 1044 1045 avail_pgs = atomic_long_read(&nr_swap_pages) / size; 1046 if (avail_pgs <= 0) 1047 goto noswap; 1048 1049 n_goal = min3((long)n_goal, (long)SWAP_BATCH, avail_pgs); 1050 1051 atomic_long_sub(n_goal * size, &nr_swap_pages); 1052 1053 spin_lock(&swap_avail_lock); 1054 1055 start_over: 1056 node = numa_node_id(); 1057 plist_for_each_entry_safe(si, next, &swap_avail_heads[node], avail_lists[node]) { 1058 /* requeue si to after same-priority siblings */ 1059 plist_requeue(&si->avail_lists[node], &swap_avail_heads[node]); 1060 spin_unlock(&swap_avail_lock); 1061 spin_lock(&si->lock); 1062 if (!si->highest_bit || !(si->flags & SWP_WRITEOK)) { 1063 spin_lock(&swap_avail_lock); 1064 if (plist_node_empty(&si->avail_lists[node])) { 1065 spin_unlock(&si->lock); 1066 goto nextsi; 1067 } 1068 WARN(!si->highest_bit, 1069 "swap_info %d in list but !highest_bit\n", 1070 si->type); 1071 WARN(!(si->flags & SWP_WRITEOK), 1072 "swap_info %d in list but !SWP_WRITEOK\n", 1073 si->type); 1074 __del_from_avail_list(si); 1075 spin_unlock(&si->lock); 1076 goto nextsi; 1077 } 1078 if (size == SWAPFILE_CLUSTER) { 1079 if (!(si->flags & SWP_FS)) 1080 n_ret = swap_alloc_cluster(si, swp_entries); 1081 } else 1082 n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE, 1083 n_goal, swp_entries); 1084 spin_unlock(&si->lock); 1085 if (n_ret || size == SWAPFILE_CLUSTER) 1086 goto check_out; 1087 pr_debug("scan_swap_map of si %d failed to find offset\n", 1088 si->type); 1089 1090 spin_lock(&swap_avail_lock); 1091 nextsi: 1092 /* 1093 * if we got here, it's likely that si was almost full before, 1094 * and since scan_swap_map() can drop the si->lock, multiple 1095 * callers probably all tried to get a page from the same si 1096 * and it filled up before we could get one; or, the si filled 1097 * up between us dropping swap_avail_lock and taking si->lock. 1098 * Since we dropped the swap_avail_lock, the swap_avail_head 1099 * list may have been modified; so if next is still in the 1100 * swap_avail_head list then try it, otherwise start over 1101 * if we have not gotten any slots. 1102 */ 1103 if (plist_node_empty(&next->avail_lists[node])) 1104 goto start_over; 1105 } 1106 1107 spin_unlock(&swap_avail_lock); 1108 1109 check_out: 1110 if (n_ret < n_goal) 1111 atomic_long_add((long)(n_goal - n_ret) * size, 1112 &nr_swap_pages); 1113 noswap: 1114 return n_ret; 1115 } 1116 1117 /* The only caller of this function is now suspend routine */ 1118 swp_entry_t get_swap_page_of_type(int type) 1119 { 1120 struct swap_info_struct *si = swap_type_to_swap_info(type); 1121 pgoff_t offset; 1122 1123 if (!si) 1124 goto fail; 1125 1126 spin_lock(&si->lock); 1127 if (si->flags & SWP_WRITEOK) { 1128 atomic_long_dec(&nr_swap_pages); 1129 /* This is called for allocating swap entry, not cache */ 1130 offset = scan_swap_map(si, 1); 1131 if (offset) { 1132 spin_unlock(&si->lock); 1133 return swp_entry(type, offset); 1134 } 1135 atomic_long_inc(&nr_swap_pages); 1136 } 1137 spin_unlock(&si->lock); 1138 fail: 1139 return (swp_entry_t) {0}; 1140 } 1141 1142 static struct swap_info_struct *__swap_info_get(swp_entry_t entry) 1143 { 1144 struct swap_info_struct *p; 1145 unsigned long offset; 1146 1147 if (!entry.val) 1148 goto out; 1149 p = swp_swap_info(entry); 1150 if (!p) 1151 goto bad_nofile; 1152 if (!(p->flags & SWP_USED)) 1153 goto bad_device; 1154 offset = swp_offset(entry); 1155 if (offset >= p->max) 1156 goto bad_offset; 1157 return p; 1158 1159 bad_offset: 1160 pr_err("swap_info_get: %s%08lx\n", Bad_offset, entry.val); 1161 goto out; 1162 bad_device: 1163 pr_err("swap_info_get: %s%08lx\n", Unused_file, entry.val); 1164 goto out; 1165 bad_nofile: 1166 pr_err("swap_info_get: %s%08lx\n", Bad_file, entry.val); 1167 out: 1168 return NULL; 1169 } 1170 1171 static struct swap_info_struct *_swap_info_get(swp_entry_t entry) 1172 { 1173 struct swap_info_struct *p; 1174 1175 p = __swap_info_get(entry); 1176 if (!p) 1177 goto out; 1178 if (!p->swap_map[swp_offset(entry)]) 1179 goto bad_free; 1180 return p; 1181 1182 bad_free: 1183 pr_err("swap_info_get: %s%08lx\n", Unused_offset, entry.val); 1184 goto out; 1185 out: 1186 return NULL; 1187 } 1188 1189 static struct swap_info_struct *swap_info_get(swp_entry_t entry) 1190 { 1191 struct swap_info_struct *p; 1192 1193 p = _swap_info_get(entry); 1194 if (p) 1195 spin_lock(&p->lock); 1196 return p; 1197 } 1198 1199 static struct swap_info_struct *swap_info_get_cont(swp_entry_t entry, 1200 struct swap_info_struct *q) 1201 { 1202 struct swap_info_struct *p; 1203 1204 p = _swap_info_get(entry); 1205 1206 if (p != q) { 1207 if (q != NULL) 1208 spin_unlock(&q->lock); 1209 if (p != NULL) 1210 spin_lock(&p->lock); 1211 } 1212 return p; 1213 } 1214 1215 static unsigned char __swap_entry_free_locked(struct swap_info_struct *p, 1216 unsigned long offset, 1217 unsigned char usage) 1218 { 1219 unsigned char count; 1220 unsigned char has_cache; 1221 1222 count = p->swap_map[offset]; 1223 1224 has_cache = count & SWAP_HAS_CACHE; 1225 count &= ~SWAP_HAS_CACHE; 1226 1227 if (usage == SWAP_HAS_CACHE) { 1228 VM_BUG_ON(!has_cache); 1229 has_cache = 0; 1230 } else if (count == SWAP_MAP_SHMEM) { 1231 /* 1232 * Or we could insist on shmem.c using a special 1233 * swap_shmem_free() and free_shmem_swap_and_cache()... 1234 */ 1235 count = 0; 1236 } else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) { 1237 if (count == COUNT_CONTINUED) { 1238 if (swap_count_continued(p, offset, count)) 1239 count = SWAP_MAP_MAX | COUNT_CONTINUED; 1240 else 1241 count = SWAP_MAP_MAX; 1242 } else 1243 count--; 1244 } 1245 1246 usage = count | has_cache; 1247 p->swap_map[offset] = usage ? : SWAP_HAS_CACHE; 1248 1249 return usage; 1250 } 1251 1252 /* 1253 * Check whether swap entry is valid in the swap device. If so, 1254 * return pointer to swap_info_struct, and keep the swap entry valid 1255 * via preventing the swap device from being swapoff, until 1256 * put_swap_device() is called. Otherwise return NULL. 1257 * 1258 * The entirety of the RCU read critical section must come before the 1259 * return from or after the call to synchronize_rcu() in 1260 * enable_swap_info() or swapoff(). So if "si->flags & SWP_VALID" is 1261 * true, the si->map, si->cluster_info, etc. must be valid in the 1262 * critical section. 1263 * 1264 * Notice that swapoff or swapoff+swapon can still happen before the 1265 * rcu_read_lock() in get_swap_device() or after the rcu_read_unlock() 1266 * in put_swap_device() if there isn't any other way to prevent 1267 * swapoff, such as page lock, page table lock, etc. The caller must 1268 * be prepared for that. For example, the following situation is 1269 * possible. 1270 * 1271 * CPU1 CPU2 1272 * do_swap_page() 1273 * ... swapoff+swapon 1274 * __read_swap_cache_async() 1275 * swapcache_prepare() 1276 * __swap_duplicate() 1277 * // check swap_map 1278 * // verify PTE not changed 1279 * 1280 * In __swap_duplicate(), the swap_map need to be checked before 1281 * changing partly because the specified swap entry may be for another 1282 * swap device which has been swapoff. And in do_swap_page(), after 1283 * the page is read from the swap device, the PTE is verified not 1284 * changed with the page table locked to check whether the swap device 1285 * has been swapoff or swapoff+swapon. 1286 */ 1287 struct swap_info_struct *get_swap_device(swp_entry_t entry) 1288 { 1289 struct swap_info_struct *si; 1290 unsigned long offset; 1291 1292 if (!entry.val) 1293 goto out; 1294 si = swp_swap_info(entry); 1295 if (!si) 1296 goto bad_nofile; 1297 1298 rcu_read_lock(); 1299 if (!(si->flags & SWP_VALID)) 1300 goto unlock_out; 1301 offset = swp_offset(entry); 1302 if (offset >= si->max) 1303 goto unlock_out; 1304 1305 return si; 1306 bad_nofile: 1307 pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val); 1308 out: 1309 return NULL; 1310 unlock_out: 1311 rcu_read_unlock(); 1312 return NULL; 1313 } 1314 1315 static unsigned char __swap_entry_free(struct swap_info_struct *p, 1316 swp_entry_t entry) 1317 { 1318 struct swap_cluster_info *ci; 1319 unsigned long offset = swp_offset(entry); 1320 unsigned char usage; 1321 1322 ci = lock_cluster_or_swap_info(p, offset); 1323 usage = __swap_entry_free_locked(p, offset, 1); 1324 unlock_cluster_or_swap_info(p, ci); 1325 if (!usage) 1326 free_swap_slot(entry); 1327 1328 return usage; 1329 } 1330 1331 static void swap_entry_free(struct swap_info_struct *p, swp_entry_t entry) 1332 { 1333 struct swap_cluster_info *ci; 1334 unsigned long offset = swp_offset(entry); 1335 unsigned char count; 1336 1337 ci = lock_cluster(p, offset); 1338 count = p->swap_map[offset]; 1339 VM_BUG_ON(count != SWAP_HAS_CACHE); 1340 p->swap_map[offset] = 0; 1341 dec_cluster_info_page(p, p->cluster_info, offset); 1342 unlock_cluster(ci); 1343 1344 mem_cgroup_uncharge_swap(entry, 1); 1345 swap_range_free(p, offset, 1); 1346 } 1347 1348 /* 1349 * Caller has made sure that the swap device corresponding to entry 1350 * is still around or has not been recycled. 1351 */ 1352 void swap_free(swp_entry_t entry) 1353 { 1354 struct swap_info_struct *p; 1355 1356 p = _swap_info_get(entry); 1357 if (p) 1358 __swap_entry_free(p, entry); 1359 } 1360 1361 /* 1362 * Called after dropping swapcache to decrease refcnt to swap entries. 1363 */ 1364 void put_swap_page(struct page *page, swp_entry_t entry) 1365 { 1366 unsigned long offset = swp_offset(entry); 1367 unsigned long idx = offset / SWAPFILE_CLUSTER; 1368 struct swap_cluster_info *ci; 1369 struct swap_info_struct *si; 1370 unsigned char *map; 1371 unsigned int i, free_entries = 0; 1372 unsigned char val; 1373 int size = swap_entry_size(hpage_nr_pages(page)); 1374 1375 si = _swap_info_get(entry); 1376 if (!si) 1377 return; 1378 1379 ci = lock_cluster_or_swap_info(si, offset); 1380 if (size == SWAPFILE_CLUSTER) { 1381 VM_BUG_ON(!cluster_is_huge(ci)); 1382 map = si->swap_map + offset; 1383 for (i = 0; i < SWAPFILE_CLUSTER; i++) { 1384 val = map[i]; 1385 VM_BUG_ON(!(val & SWAP_HAS_CACHE)); 1386 if (val == SWAP_HAS_CACHE) 1387 free_entries++; 1388 } 1389 cluster_clear_huge(ci); 1390 if (free_entries == SWAPFILE_CLUSTER) { 1391 unlock_cluster_or_swap_info(si, ci); 1392 spin_lock(&si->lock); 1393 mem_cgroup_uncharge_swap(entry, SWAPFILE_CLUSTER); 1394 swap_free_cluster(si, idx); 1395 spin_unlock(&si->lock); 1396 return; 1397 } 1398 } 1399 for (i = 0; i < size; i++, entry.val++) { 1400 if (!__swap_entry_free_locked(si, offset + i, SWAP_HAS_CACHE)) { 1401 unlock_cluster_or_swap_info(si, ci); 1402 free_swap_slot(entry); 1403 if (i == size - 1) 1404 return; 1405 lock_cluster_or_swap_info(si, offset); 1406 } 1407 } 1408 unlock_cluster_or_swap_info(si, ci); 1409 } 1410 1411 #ifdef CONFIG_THP_SWAP 1412 int split_swap_cluster(swp_entry_t entry) 1413 { 1414 struct swap_info_struct *si; 1415 struct swap_cluster_info *ci; 1416 unsigned long offset = swp_offset(entry); 1417 1418 si = _swap_info_get(entry); 1419 if (!si) 1420 return -EBUSY; 1421 ci = lock_cluster(si, offset); 1422 cluster_clear_huge(ci); 1423 unlock_cluster(ci); 1424 return 0; 1425 } 1426 #endif 1427 1428 static int swp_entry_cmp(const void *ent1, const void *ent2) 1429 { 1430 const swp_entry_t *e1 = ent1, *e2 = ent2; 1431 1432 return (int)swp_type(*e1) - (int)swp_type(*e2); 1433 } 1434 1435 void swapcache_free_entries(swp_entry_t *entries, int n) 1436 { 1437 struct swap_info_struct *p, *prev; 1438 int i; 1439 1440 if (n <= 0) 1441 return; 1442 1443 prev = NULL; 1444 p = NULL; 1445 1446 /* 1447 * Sort swap entries by swap device, so each lock is only taken once. 1448 * nr_swapfiles isn't absolutely correct, but the overhead of sort() is 1449 * so low that it isn't necessary to optimize further. 1450 */ 1451 if (nr_swapfiles > 1) 1452 sort(entries, n, sizeof(entries[0]), swp_entry_cmp, NULL); 1453 for (i = 0; i < n; ++i) { 1454 p = swap_info_get_cont(entries[i], prev); 1455 if (p) 1456 swap_entry_free(p, entries[i]); 1457 prev = p; 1458 } 1459 if (p) 1460 spin_unlock(&p->lock); 1461 } 1462 1463 /* 1464 * How many references to page are currently swapped out? 1465 * This does not give an exact answer when swap count is continued, 1466 * but does include the high COUNT_CONTINUED flag to allow for that. 1467 */ 1468 int page_swapcount(struct page *page) 1469 { 1470 int count = 0; 1471 struct swap_info_struct *p; 1472 struct swap_cluster_info *ci; 1473 swp_entry_t entry; 1474 unsigned long offset; 1475 1476 entry.val = page_private(page); 1477 p = _swap_info_get(entry); 1478 if (p) { 1479 offset = swp_offset(entry); 1480 ci = lock_cluster_or_swap_info(p, offset); 1481 count = swap_count(p->swap_map[offset]); 1482 unlock_cluster_or_swap_info(p, ci); 1483 } 1484 return count; 1485 } 1486 1487 int __swap_count(swp_entry_t entry) 1488 { 1489 struct swap_info_struct *si; 1490 pgoff_t offset = swp_offset(entry); 1491 int count = 0; 1492 1493 si = get_swap_device(entry); 1494 if (si) { 1495 count = swap_count(si->swap_map[offset]); 1496 put_swap_device(si); 1497 } 1498 return count; 1499 } 1500 1501 static int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry) 1502 { 1503 int count = 0; 1504 pgoff_t offset = swp_offset(entry); 1505 struct swap_cluster_info *ci; 1506 1507 ci = lock_cluster_or_swap_info(si, offset); 1508 count = swap_count(si->swap_map[offset]); 1509 unlock_cluster_or_swap_info(si, ci); 1510 return count; 1511 } 1512 1513 /* 1514 * How many references to @entry are currently swapped out? 1515 * This does not give an exact answer when swap count is continued, 1516 * but does include the high COUNT_CONTINUED flag to allow for that. 1517 */ 1518 int __swp_swapcount(swp_entry_t entry) 1519 { 1520 int count = 0; 1521 struct swap_info_struct *si; 1522 1523 si = get_swap_device(entry); 1524 if (si) { 1525 count = swap_swapcount(si, entry); 1526 put_swap_device(si); 1527 } 1528 return count; 1529 } 1530 1531 /* 1532 * How many references to @entry are currently swapped out? 1533 * This considers COUNT_CONTINUED so it returns exact answer. 1534 */ 1535 int swp_swapcount(swp_entry_t entry) 1536 { 1537 int count, tmp_count, n; 1538 struct swap_info_struct *p; 1539 struct swap_cluster_info *ci; 1540 struct page *page; 1541 pgoff_t offset; 1542 unsigned char *map; 1543 1544 p = _swap_info_get(entry); 1545 if (!p) 1546 return 0; 1547 1548 offset = swp_offset(entry); 1549 1550 ci = lock_cluster_or_swap_info(p, offset); 1551 1552 count = swap_count(p->swap_map[offset]); 1553 if (!(count & COUNT_CONTINUED)) 1554 goto out; 1555 1556 count &= ~COUNT_CONTINUED; 1557 n = SWAP_MAP_MAX + 1; 1558 1559 page = vmalloc_to_page(p->swap_map + offset); 1560 offset &= ~PAGE_MASK; 1561 VM_BUG_ON(page_private(page) != SWP_CONTINUED); 1562 1563 do { 1564 page = list_next_entry(page, lru); 1565 map = kmap_atomic(page); 1566 tmp_count = map[offset]; 1567 kunmap_atomic(map); 1568 1569 count += (tmp_count & ~COUNT_CONTINUED) * n; 1570 n *= (SWAP_CONT_MAX + 1); 1571 } while (tmp_count & COUNT_CONTINUED); 1572 out: 1573 unlock_cluster_or_swap_info(p, ci); 1574 return count; 1575 } 1576 1577 static bool swap_page_trans_huge_swapped(struct swap_info_struct *si, 1578 swp_entry_t entry) 1579 { 1580 struct swap_cluster_info *ci; 1581 unsigned char *map = si->swap_map; 1582 unsigned long roffset = swp_offset(entry); 1583 unsigned long offset = round_down(roffset, SWAPFILE_CLUSTER); 1584 int i; 1585 bool ret = false; 1586 1587 ci = lock_cluster_or_swap_info(si, offset); 1588 if (!ci || !cluster_is_huge(ci)) { 1589 if (swap_count(map[roffset])) 1590 ret = true; 1591 goto unlock_out; 1592 } 1593 for (i = 0; i < SWAPFILE_CLUSTER; i++) { 1594 if (swap_count(map[offset + i])) { 1595 ret = true; 1596 break; 1597 } 1598 } 1599 unlock_out: 1600 unlock_cluster_or_swap_info(si, ci); 1601 return ret; 1602 } 1603 1604 static bool page_swapped(struct page *page) 1605 { 1606 swp_entry_t entry; 1607 struct swap_info_struct *si; 1608 1609 if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!PageTransCompound(page))) 1610 return page_swapcount(page) != 0; 1611 1612 page = compound_head(page); 1613 entry.val = page_private(page); 1614 si = _swap_info_get(entry); 1615 if (si) 1616 return swap_page_trans_huge_swapped(si, entry); 1617 return false; 1618 } 1619 1620 static int page_trans_huge_map_swapcount(struct page *page, int *total_mapcount, 1621 int *total_swapcount) 1622 { 1623 int i, map_swapcount, _total_mapcount, _total_swapcount; 1624 unsigned long offset = 0; 1625 struct swap_info_struct *si; 1626 struct swap_cluster_info *ci = NULL; 1627 unsigned char *map = NULL; 1628 int mapcount, swapcount = 0; 1629 1630 /* hugetlbfs shouldn't call it */ 1631 VM_BUG_ON_PAGE(PageHuge(page), page); 1632 1633 if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!PageTransCompound(page))) { 1634 mapcount = page_trans_huge_mapcount(page, total_mapcount); 1635 if (PageSwapCache(page)) 1636 swapcount = page_swapcount(page); 1637 if (total_swapcount) 1638 *total_swapcount = swapcount; 1639 return mapcount + swapcount; 1640 } 1641 1642 page = compound_head(page); 1643 1644 _total_mapcount = _total_swapcount = map_swapcount = 0; 1645 if (PageSwapCache(page)) { 1646 swp_entry_t entry; 1647 1648 entry.val = page_private(page); 1649 si = _swap_info_get(entry); 1650 if (si) { 1651 map = si->swap_map; 1652 offset = swp_offset(entry); 1653 } 1654 } 1655 if (map) 1656 ci = lock_cluster(si, offset); 1657 for (i = 0; i < HPAGE_PMD_NR; i++) { 1658 mapcount = atomic_read(&page[i]._mapcount) + 1; 1659 _total_mapcount += mapcount; 1660 if (map) { 1661 swapcount = swap_count(map[offset + i]); 1662 _total_swapcount += swapcount; 1663 } 1664 map_swapcount = max(map_swapcount, mapcount + swapcount); 1665 } 1666 unlock_cluster(ci); 1667 if (PageDoubleMap(page)) { 1668 map_swapcount -= 1; 1669 _total_mapcount -= HPAGE_PMD_NR; 1670 } 1671 mapcount = compound_mapcount(page); 1672 map_swapcount += mapcount; 1673 _total_mapcount += mapcount; 1674 if (total_mapcount) 1675 *total_mapcount = _total_mapcount; 1676 if (total_swapcount) 1677 *total_swapcount = _total_swapcount; 1678 1679 return map_swapcount; 1680 } 1681 1682 /* 1683 * We can write to an anon page without COW if there are no other references 1684 * to it. And as a side-effect, free up its swap: because the old content 1685 * on disk will never be read, and seeking back there to write new content 1686 * later would only waste time away from clustering. 1687 * 1688 * NOTE: total_map_swapcount should not be relied upon by the caller if 1689 * reuse_swap_page() returns false, but it may be always overwritten 1690 * (see the other implementation for CONFIG_SWAP=n). 1691 */ 1692 bool reuse_swap_page(struct page *page, int *total_map_swapcount) 1693 { 1694 int count, total_mapcount, total_swapcount; 1695 1696 VM_BUG_ON_PAGE(!PageLocked(page), page); 1697 if (unlikely(PageKsm(page))) 1698 return false; 1699 count = page_trans_huge_map_swapcount(page, &total_mapcount, 1700 &total_swapcount); 1701 if (total_map_swapcount) 1702 *total_map_swapcount = total_mapcount + total_swapcount; 1703 if (count == 1 && PageSwapCache(page) && 1704 (likely(!PageTransCompound(page)) || 1705 /* The remaining swap count will be freed soon */ 1706 total_swapcount == page_swapcount(page))) { 1707 if (!PageWriteback(page)) { 1708 page = compound_head(page); 1709 delete_from_swap_cache(page); 1710 SetPageDirty(page); 1711 } else { 1712 swp_entry_t entry; 1713 struct swap_info_struct *p; 1714 1715 entry.val = page_private(page); 1716 p = swap_info_get(entry); 1717 if (p->flags & SWP_STABLE_WRITES) { 1718 spin_unlock(&p->lock); 1719 return false; 1720 } 1721 spin_unlock(&p->lock); 1722 } 1723 } 1724 1725 return count <= 1; 1726 } 1727 1728 /* 1729 * If swap is getting full, or if there are no more mappings of this page, 1730 * then try_to_free_swap is called to free its swap space. 1731 */ 1732 int try_to_free_swap(struct page *page) 1733 { 1734 VM_BUG_ON_PAGE(!PageLocked(page), page); 1735 1736 if (!PageSwapCache(page)) 1737 return 0; 1738 if (PageWriteback(page)) 1739 return 0; 1740 if (page_swapped(page)) 1741 return 0; 1742 1743 /* 1744 * Once hibernation has begun to create its image of memory, 1745 * there's a danger that one of the calls to try_to_free_swap() 1746 * - most probably a call from __try_to_reclaim_swap() while 1747 * hibernation is allocating its own swap pages for the image, 1748 * but conceivably even a call from memory reclaim - will free 1749 * the swap from a page which has already been recorded in the 1750 * image as a clean swapcache page, and then reuse its swap for 1751 * another page of the image. On waking from hibernation, the 1752 * original page might be freed under memory pressure, then 1753 * later read back in from swap, now with the wrong data. 1754 * 1755 * Hibernation suspends storage while it is writing the image 1756 * to disk so check that here. 1757 */ 1758 if (pm_suspended_storage()) 1759 return 0; 1760 1761 page = compound_head(page); 1762 delete_from_swap_cache(page); 1763 SetPageDirty(page); 1764 return 1; 1765 } 1766 1767 /* 1768 * Free the swap entry like above, but also try to 1769 * free the page cache entry if it is the last user. 1770 */ 1771 int free_swap_and_cache(swp_entry_t entry) 1772 { 1773 struct swap_info_struct *p; 1774 unsigned char count; 1775 1776 if (non_swap_entry(entry)) 1777 return 1; 1778 1779 p = _swap_info_get(entry); 1780 if (p) { 1781 count = __swap_entry_free(p, entry); 1782 if (count == SWAP_HAS_CACHE && 1783 !swap_page_trans_huge_swapped(p, entry)) 1784 __try_to_reclaim_swap(p, swp_offset(entry), 1785 TTRS_UNMAPPED | TTRS_FULL); 1786 } 1787 return p != NULL; 1788 } 1789 1790 #ifdef CONFIG_HIBERNATION 1791 /* 1792 * Find the swap type that corresponds to given device (if any). 1793 * 1794 * @offset - number of the PAGE_SIZE-sized block of the device, starting 1795 * from 0, in which the swap header is expected to be located. 1796 * 1797 * This is needed for the suspend to disk (aka swsusp). 1798 */ 1799 int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p) 1800 { 1801 struct block_device *bdev = NULL; 1802 int type; 1803 1804 if (device) 1805 bdev = bdget(device); 1806 1807 spin_lock(&swap_lock); 1808 for (type = 0; type < nr_swapfiles; type++) { 1809 struct swap_info_struct *sis = swap_info[type]; 1810 1811 if (!(sis->flags & SWP_WRITEOK)) 1812 continue; 1813 1814 if (!bdev) { 1815 if (bdev_p) 1816 *bdev_p = bdgrab(sis->bdev); 1817 1818 spin_unlock(&swap_lock); 1819 return type; 1820 } 1821 if (bdev == sis->bdev) { 1822 struct swap_extent *se = first_se(sis); 1823 1824 if (se->start_block == offset) { 1825 if (bdev_p) 1826 *bdev_p = bdgrab(sis->bdev); 1827 1828 spin_unlock(&swap_lock); 1829 bdput(bdev); 1830 return type; 1831 } 1832 } 1833 } 1834 spin_unlock(&swap_lock); 1835 if (bdev) 1836 bdput(bdev); 1837 1838 return -ENODEV; 1839 } 1840 1841 /* 1842 * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev 1843 * corresponding to given index in swap_info (swap type). 1844 */ 1845 sector_t swapdev_block(int type, pgoff_t offset) 1846 { 1847 struct block_device *bdev; 1848 struct swap_info_struct *si = swap_type_to_swap_info(type); 1849 1850 if (!si || !(si->flags & SWP_WRITEOK)) 1851 return 0; 1852 return map_swap_entry(swp_entry(type, offset), &bdev); 1853 } 1854 1855 /* 1856 * Return either the total number of swap pages of given type, or the number 1857 * of free pages of that type (depending on @free) 1858 * 1859 * This is needed for software suspend 1860 */ 1861 unsigned int count_swap_pages(int type, int free) 1862 { 1863 unsigned int n = 0; 1864 1865 spin_lock(&swap_lock); 1866 if ((unsigned int)type < nr_swapfiles) { 1867 struct swap_info_struct *sis = swap_info[type]; 1868 1869 spin_lock(&sis->lock); 1870 if (sis->flags & SWP_WRITEOK) { 1871 n = sis->pages; 1872 if (free) 1873 n -= sis->inuse_pages; 1874 } 1875 spin_unlock(&sis->lock); 1876 } 1877 spin_unlock(&swap_lock); 1878 return n; 1879 } 1880 #endif /* CONFIG_HIBERNATION */ 1881 1882 static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte) 1883 { 1884 return pte_same(pte_swp_clear_soft_dirty(pte), swp_pte); 1885 } 1886 1887 /* 1888 * No need to decide whether this PTE shares the swap entry with others, 1889 * just let do_wp_page work it out if a write is requested later - to 1890 * force COW, vm_page_prot omits write permission from any private vma. 1891 */ 1892 static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, 1893 unsigned long addr, swp_entry_t entry, struct page *page) 1894 { 1895 struct page *swapcache; 1896 spinlock_t *ptl; 1897 pte_t *pte; 1898 int ret = 1; 1899 1900 swapcache = page; 1901 page = ksm_might_need_to_copy(page, vma, addr); 1902 if (unlikely(!page)) 1903 return -ENOMEM; 1904 1905 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 1906 if (unlikely(!pte_same_as_swp(*pte, swp_entry_to_pte(entry)))) { 1907 ret = 0; 1908 goto out; 1909 } 1910 1911 dec_mm_counter(vma->vm_mm, MM_SWAPENTS); 1912 inc_mm_counter(vma->vm_mm, MM_ANONPAGES); 1913 get_page(page); 1914 set_pte_at(vma->vm_mm, addr, pte, 1915 pte_mkold(mk_pte(page, vma->vm_page_prot))); 1916 if (page == swapcache) { 1917 page_add_anon_rmap(page, vma, addr, false); 1918 } else { /* ksm created a completely new copy */ 1919 page_add_new_anon_rmap(page, vma, addr, false); 1920 lru_cache_add_inactive_or_unevictable(page, vma); 1921 } 1922 swap_free(entry); 1923 /* 1924 * Move the page to the active list so it is not 1925 * immediately swapped out again after swapon. 1926 */ 1927 activate_page(page); 1928 out: 1929 pte_unmap_unlock(pte, ptl); 1930 if (page != swapcache) { 1931 unlock_page(page); 1932 put_page(page); 1933 } 1934 return ret; 1935 } 1936 1937 static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 1938 unsigned long addr, unsigned long end, 1939 unsigned int type, bool frontswap, 1940 unsigned long *fs_pages_to_unuse) 1941 { 1942 struct page *page; 1943 swp_entry_t entry; 1944 pte_t *pte; 1945 struct swap_info_struct *si; 1946 unsigned long offset; 1947 int ret = 0; 1948 volatile unsigned char *swap_map; 1949 1950 si = swap_info[type]; 1951 pte = pte_offset_map(pmd, addr); 1952 do { 1953 struct vm_fault vmf; 1954 1955 if (!is_swap_pte(*pte)) 1956 continue; 1957 1958 entry = pte_to_swp_entry(*pte); 1959 if (swp_type(entry) != type) 1960 continue; 1961 1962 offset = swp_offset(entry); 1963 if (frontswap && !frontswap_test(si, offset)) 1964 continue; 1965 1966 pte_unmap(pte); 1967 swap_map = &si->swap_map[offset]; 1968 page = lookup_swap_cache(entry, vma, addr); 1969 if (!page) { 1970 vmf.vma = vma; 1971 vmf.address = addr; 1972 vmf.pmd = pmd; 1973 page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, 1974 &vmf); 1975 } 1976 if (!page) { 1977 if (*swap_map == 0 || *swap_map == SWAP_MAP_BAD) 1978 goto try_next; 1979 return -ENOMEM; 1980 } 1981 1982 lock_page(page); 1983 wait_on_page_writeback(page); 1984 ret = unuse_pte(vma, pmd, addr, entry, page); 1985 if (ret < 0) { 1986 unlock_page(page); 1987 put_page(page); 1988 goto out; 1989 } 1990 1991 try_to_free_swap(page); 1992 unlock_page(page); 1993 put_page(page); 1994 1995 if (*fs_pages_to_unuse && !--(*fs_pages_to_unuse)) { 1996 ret = FRONTSWAP_PAGES_UNUSED; 1997 goto out; 1998 } 1999 try_next: 2000 pte = pte_offset_map(pmd, addr); 2001 } while (pte++, addr += PAGE_SIZE, addr != end); 2002 pte_unmap(pte - 1); 2003 2004 ret = 0; 2005 out: 2006 return ret; 2007 } 2008 2009 static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, 2010 unsigned long addr, unsigned long end, 2011 unsigned int type, bool frontswap, 2012 unsigned long *fs_pages_to_unuse) 2013 { 2014 pmd_t *pmd; 2015 unsigned long next; 2016 int ret; 2017 2018 pmd = pmd_offset(pud, addr); 2019 do { 2020 cond_resched(); 2021 next = pmd_addr_end(addr, end); 2022 if (pmd_none_or_trans_huge_or_clear_bad(pmd)) 2023 continue; 2024 ret = unuse_pte_range(vma, pmd, addr, next, type, 2025 frontswap, fs_pages_to_unuse); 2026 if (ret) 2027 return ret; 2028 } while (pmd++, addr = next, addr != end); 2029 return 0; 2030 } 2031 2032 static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, 2033 unsigned long addr, unsigned long end, 2034 unsigned int type, bool frontswap, 2035 unsigned long *fs_pages_to_unuse) 2036 { 2037 pud_t *pud; 2038 unsigned long next; 2039 int ret; 2040 2041 pud = pud_offset(p4d, addr); 2042 do { 2043 next = pud_addr_end(addr, end); 2044 if (pud_none_or_clear_bad(pud)) 2045 continue; 2046 ret = unuse_pmd_range(vma, pud, addr, next, type, 2047 frontswap, fs_pages_to_unuse); 2048 if (ret) 2049 return ret; 2050 } while (pud++, addr = next, addr != end); 2051 return 0; 2052 } 2053 2054 static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, 2055 unsigned long addr, unsigned long end, 2056 unsigned int type, bool frontswap, 2057 unsigned long *fs_pages_to_unuse) 2058 { 2059 p4d_t *p4d; 2060 unsigned long next; 2061 int ret; 2062 2063 p4d = p4d_offset(pgd, addr); 2064 do { 2065 next = p4d_addr_end(addr, end); 2066 if (p4d_none_or_clear_bad(p4d)) 2067 continue; 2068 ret = unuse_pud_range(vma, p4d, addr, next, type, 2069 frontswap, fs_pages_to_unuse); 2070 if (ret) 2071 return ret; 2072 } while (p4d++, addr = next, addr != end); 2073 return 0; 2074 } 2075 2076 static int unuse_vma(struct vm_area_struct *vma, unsigned int type, 2077 bool frontswap, unsigned long *fs_pages_to_unuse) 2078 { 2079 pgd_t *pgd; 2080 unsigned long addr, end, next; 2081 int ret; 2082 2083 addr = vma->vm_start; 2084 end = vma->vm_end; 2085 2086 pgd = pgd_offset(vma->vm_mm, addr); 2087 do { 2088 next = pgd_addr_end(addr, end); 2089 if (pgd_none_or_clear_bad(pgd)) 2090 continue; 2091 ret = unuse_p4d_range(vma, pgd, addr, next, type, 2092 frontswap, fs_pages_to_unuse); 2093 if (ret) 2094 return ret; 2095 } while (pgd++, addr = next, addr != end); 2096 return 0; 2097 } 2098 2099 static int unuse_mm(struct mm_struct *mm, unsigned int type, 2100 bool frontswap, unsigned long *fs_pages_to_unuse) 2101 { 2102 struct vm_area_struct *vma; 2103 int ret = 0; 2104 2105 mmap_read_lock(mm); 2106 for (vma = mm->mmap; vma; vma = vma->vm_next) { 2107 if (vma->anon_vma) { 2108 ret = unuse_vma(vma, type, frontswap, 2109 fs_pages_to_unuse); 2110 if (ret) 2111 break; 2112 } 2113 cond_resched(); 2114 } 2115 mmap_read_unlock(mm); 2116 return ret; 2117 } 2118 2119 /* 2120 * Scan swap_map (or frontswap_map if frontswap parameter is true) 2121 * from current position to next entry still in use. Return 0 2122 * if there are no inuse entries after prev till end of the map. 2123 */ 2124 static unsigned int find_next_to_unuse(struct swap_info_struct *si, 2125 unsigned int prev, bool frontswap) 2126 { 2127 unsigned int i; 2128 unsigned char count; 2129 2130 /* 2131 * No need for swap_lock here: we're just looking 2132 * for whether an entry is in use, not modifying it; false 2133 * hits are okay, and sys_swapoff() has already prevented new 2134 * allocations from this area (while holding swap_lock). 2135 */ 2136 for (i = prev + 1; i < si->max; i++) { 2137 count = READ_ONCE(si->swap_map[i]); 2138 if (count && swap_count(count) != SWAP_MAP_BAD) 2139 if (!frontswap || frontswap_test(si, i)) 2140 break; 2141 if ((i % LATENCY_LIMIT) == 0) 2142 cond_resched(); 2143 } 2144 2145 if (i == si->max) 2146 i = 0; 2147 2148 return i; 2149 } 2150 2151 /* 2152 * If the boolean frontswap is true, only unuse pages_to_unuse pages; 2153 * pages_to_unuse==0 means all pages; ignored if frontswap is false 2154 */ 2155 int try_to_unuse(unsigned int type, bool frontswap, 2156 unsigned long pages_to_unuse) 2157 { 2158 struct mm_struct *prev_mm; 2159 struct mm_struct *mm; 2160 struct list_head *p; 2161 int retval = 0; 2162 struct swap_info_struct *si = swap_info[type]; 2163 struct page *page; 2164 swp_entry_t entry; 2165 unsigned int i; 2166 2167 if (!READ_ONCE(si->inuse_pages)) 2168 return 0; 2169 2170 if (!frontswap) 2171 pages_to_unuse = 0; 2172 2173 retry: 2174 retval = shmem_unuse(type, frontswap, &pages_to_unuse); 2175 if (retval) 2176 goto out; 2177 2178 prev_mm = &init_mm; 2179 mmget(prev_mm); 2180 2181 spin_lock(&mmlist_lock); 2182 p = &init_mm.mmlist; 2183 while (READ_ONCE(si->inuse_pages) && 2184 !signal_pending(current) && 2185 (p = p->next) != &init_mm.mmlist) { 2186 2187 mm = list_entry(p, struct mm_struct, mmlist); 2188 if (!mmget_not_zero(mm)) 2189 continue; 2190 spin_unlock(&mmlist_lock); 2191 mmput(prev_mm); 2192 prev_mm = mm; 2193 retval = unuse_mm(mm, type, frontswap, &pages_to_unuse); 2194 2195 if (retval) { 2196 mmput(prev_mm); 2197 goto out; 2198 } 2199 2200 /* 2201 * Make sure that we aren't completely killing 2202 * interactive performance. 2203 */ 2204 cond_resched(); 2205 spin_lock(&mmlist_lock); 2206 } 2207 spin_unlock(&mmlist_lock); 2208 2209 mmput(prev_mm); 2210 2211 i = 0; 2212 while (READ_ONCE(si->inuse_pages) && 2213 !signal_pending(current) && 2214 (i = find_next_to_unuse(si, i, frontswap)) != 0) { 2215 2216 entry = swp_entry(type, i); 2217 page = find_get_page(swap_address_space(entry), i); 2218 if (!page) 2219 continue; 2220 2221 /* 2222 * It is conceivable that a racing task removed this page from 2223 * swap cache just before we acquired the page lock. The page 2224 * might even be back in swap cache on another swap area. But 2225 * that is okay, try_to_free_swap() only removes stale pages. 2226 */ 2227 lock_page(page); 2228 wait_on_page_writeback(page); 2229 try_to_free_swap(page); 2230 unlock_page(page); 2231 put_page(page); 2232 2233 /* 2234 * For frontswap, we just need to unuse pages_to_unuse, if 2235 * it was specified. Need not check frontswap again here as 2236 * we already zeroed out pages_to_unuse if not frontswap. 2237 */ 2238 if (pages_to_unuse && --pages_to_unuse == 0) 2239 goto out; 2240 } 2241 2242 /* 2243 * Lets check again to see if there are still swap entries in the map. 2244 * If yes, we would need to do retry the unuse logic again. 2245 * Under global memory pressure, swap entries can be reinserted back 2246 * into process space after the mmlist loop above passes over them. 2247 * 2248 * Limit the number of retries? No: when mmget_not_zero() above fails, 2249 * that mm is likely to be freeing swap from exit_mmap(), which proceeds 2250 * at its own independent pace; and even shmem_writepage() could have 2251 * been preempted after get_swap_page(), temporarily hiding that swap. 2252 * It's easy and robust (though cpu-intensive) just to keep retrying. 2253 */ 2254 if (READ_ONCE(si->inuse_pages)) { 2255 if (!signal_pending(current)) 2256 goto retry; 2257 retval = -EINTR; 2258 } 2259 out: 2260 return (retval == FRONTSWAP_PAGES_UNUSED) ? 0 : retval; 2261 } 2262 2263 /* 2264 * After a successful try_to_unuse, if no swap is now in use, we know 2265 * we can empty the mmlist. swap_lock must be held on entry and exit. 2266 * Note that mmlist_lock nests inside swap_lock, and an mm must be 2267 * added to the mmlist just after page_duplicate - before would be racy. 2268 */ 2269 static void drain_mmlist(void) 2270 { 2271 struct list_head *p, *next; 2272 unsigned int type; 2273 2274 for (type = 0; type < nr_swapfiles; type++) 2275 if (swap_info[type]->inuse_pages) 2276 return; 2277 spin_lock(&mmlist_lock); 2278 list_for_each_safe(p, next, &init_mm.mmlist) 2279 list_del_init(p); 2280 spin_unlock(&mmlist_lock); 2281 } 2282 2283 /* 2284 * Use this swapdev's extent info to locate the (PAGE_SIZE) block which 2285 * corresponds to page offset for the specified swap entry. 2286 * Note that the type of this function is sector_t, but it returns page offset 2287 * into the bdev, not sector offset. 2288 */ 2289 static sector_t map_swap_entry(swp_entry_t entry, struct block_device **bdev) 2290 { 2291 struct swap_info_struct *sis; 2292 struct swap_extent *se; 2293 pgoff_t offset; 2294 2295 sis = swp_swap_info(entry); 2296 *bdev = sis->bdev; 2297 2298 offset = swp_offset(entry); 2299 se = offset_to_swap_extent(sis, offset); 2300 return se->start_block + (offset - se->start_page); 2301 } 2302 2303 /* 2304 * Returns the page offset into bdev for the specified page's swap entry. 2305 */ 2306 sector_t map_swap_page(struct page *page, struct block_device **bdev) 2307 { 2308 swp_entry_t entry; 2309 entry.val = page_private(page); 2310 return map_swap_entry(entry, bdev); 2311 } 2312 2313 /* 2314 * Free all of a swapdev's extent information 2315 */ 2316 static void destroy_swap_extents(struct swap_info_struct *sis) 2317 { 2318 while (!RB_EMPTY_ROOT(&sis->swap_extent_root)) { 2319 struct rb_node *rb = sis->swap_extent_root.rb_node; 2320 struct swap_extent *se = rb_entry(rb, struct swap_extent, rb_node); 2321 2322 rb_erase(rb, &sis->swap_extent_root); 2323 kfree(se); 2324 } 2325 2326 if (sis->flags & SWP_ACTIVATED) { 2327 struct file *swap_file = sis->swap_file; 2328 struct address_space *mapping = swap_file->f_mapping; 2329 2330 sis->flags &= ~SWP_ACTIVATED; 2331 if (mapping->a_ops->swap_deactivate) 2332 mapping->a_ops->swap_deactivate(swap_file); 2333 } 2334 } 2335 2336 /* 2337 * Add a block range (and the corresponding page range) into this swapdev's 2338 * extent tree. 2339 * 2340 * This function rather assumes that it is called in ascending page order. 2341 */ 2342 int 2343 add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, 2344 unsigned long nr_pages, sector_t start_block) 2345 { 2346 struct rb_node **link = &sis->swap_extent_root.rb_node, *parent = NULL; 2347 struct swap_extent *se; 2348 struct swap_extent *new_se; 2349 2350 /* 2351 * place the new node at the right most since the 2352 * function is called in ascending page order. 2353 */ 2354 while (*link) { 2355 parent = *link; 2356 link = &parent->rb_right; 2357 } 2358 2359 if (parent) { 2360 se = rb_entry(parent, struct swap_extent, rb_node); 2361 BUG_ON(se->start_page + se->nr_pages != start_page); 2362 if (se->start_block + se->nr_pages == start_block) { 2363 /* Merge it */ 2364 se->nr_pages += nr_pages; 2365 return 0; 2366 } 2367 } 2368 2369 /* No merge, insert a new extent. */ 2370 new_se = kmalloc(sizeof(*se), GFP_KERNEL); 2371 if (new_se == NULL) 2372 return -ENOMEM; 2373 new_se->start_page = start_page; 2374 new_se->nr_pages = nr_pages; 2375 new_se->start_block = start_block; 2376 2377 rb_link_node(&new_se->rb_node, parent, link); 2378 rb_insert_color(&new_se->rb_node, &sis->swap_extent_root); 2379 return 1; 2380 } 2381 EXPORT_SYMBOL_GPL(add_swap_extent); 2382 2383 /* 2384 * A `swap extent' is a simple thing which maps a contiguous range of pages 2385 * onto a contiguous range of disk blocks. An ordered list of swap extents 2386 * is built at swapon time and is then used at swap_writepage/swap_readpage 2387 * time for locating where on disk a page belongs. 2388 * 2389 * If the swapfile is an S_ISBLK block device, a single extent is installed. 2390 * This is done so that the main operating code can treat S_ISBLK and S_ISREG 2391 * swap files identically. 2392 * 2393 * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap 2394 * extent list operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK 2395 * swapfiles are handled *identically* after swapon time. 2396 * 2397 * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks 2398 * and will parse them into an ordered extent list, in PAGE_SIZE chunks. If 2399 * some stray blocks are found which do not fall within the PAGE_SIZE alignment 2400 * requirements, they are simply tossed out - we will never use those blocks 2401 * for swapping. 2402 * 2403 * For all swap devices we set S_SWAPFILE across the life of the swapon. This 2404 * prevents users from writing to the swap device, which will corrupt memory. 2405 * 2406 * The amount of disk space which a single swap extent represents varies. 2407 * Typically it is in the 1-4 megabyte range. So we can have hundreds of 2408 * extents in the list. To avoid much list walking, we cache the previous 2409 * search location in `curr_swap_extent', and start new searches from there. 2410 * This is extremely effective. The average number of iterations in 2411 * map_swap_page() has been measured at about 0.3 per page. - akpm. 2412 */ 2413 static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span) 2414 { 2415 struct file *swap_file = sis->swap_file; 2416 struct address_space *mapping = swap_file->f_mapping; 2417 struct inode *inode = mapping->host; 2418 int ret; 2419 2420 if (S_ISBLK(inode->i_mode)) { 2421 ret = add_swap_extent(sis, 0, sis->max, 0); 2422 *span = sis->pages; 2423 return ret; 2424 } 2425 2426 if (mapping->a_ops->swap_activate) { 2427 ret = mapping->a_ops->swap_activate(sis, swap_file, span); 2428 if (ret >= 0) 2429 sis->flags |= SWP_ACTIVATED; 2430 if (!ret) { 2431 sis->flags |= SWP_FS; 2432 ret = add_swap_extent(sis, 0, sis->max, 0); 2433 *span = sis->pages; 2434 } 2435 return ret; 2436 } 2437 2438 return generic_swapfile_activate(sis, swap_file, span); 2439 } 2440 2441 static int swap_node(struct swap_info_struct *p) 2442 { 2443 struct block_device *bdev; 2444 2445 if (p->bdev) 2446 bdev = p->bdev; 2447 else 2448 bdev = p->swap_file->f_inode->i_sb->s_bdev; 2449 2450 return bdev ? bdev->bd_disk->node_id : NUMA_NO_NODE; 2451 } 2452 2453 static void setup_swap_info(struct swap_info_struct *p, int prio, 2454 unsigned char *swap_map, 2455 struct swap_cluster_info *cluster_info) 2456 { 2457 int i; 2458 2459 if (prio >= 0) 2460 p->prio = prio; 2461 else 2462 p->prio = --least_priority; 2463 /* 2464 * the plist prio is negated because plist ordering is 2465 * low-to-high, while swap ordering is high-to-low 2466 */ 2467 p->list.prio = -p->prio; 2468 for_each_node(i) { 2469 if (p->prio >= 0) 2470 p->avail_lists[i].prio = -p->prio; 2471 else { 2472 if (swap_node(p) == i) 2473 p->avail_lists[i].prio = 1; 2474 else 2475 p->avail_lists[i].prio = -p->prio; 2476 } 2477 } 2478 p->swap_map = swap_map; 2479 p->cluster_info = cluster_info; 2480 } 2481 2482 static void _enable_swap_info(struct swap_info_struct *p) 2483 { 2484 p->flags |= SWP_WRITEOK | SWP_VALID; 2485 atomic_long_add(p->pages, &nr_swap_pages); 2486 total_swap_pages += p->pages; 2487 2488 assert_spin_locked(&swap_lock); 2489 /* 2490 * both lists are plists, and thus priority ordered. 2491 * swap_active_head needs to be priority ordered for swapoff(), 2492 * which on removal of any swap_info_struct with an auto-assigned 2493 * (i.e. negative) priority increments the auto-assigned priority 2494 * of any lower-priority swap_info_structs. 2495 * swap_avail_head needs to be priority ordered for get_swap_page(), 2496 * which allocates swap pages from the highest available priority 2497 * swap_info_struct. 2498 */ 2499 plist_add(&p->list, &swap_active_head); 2500 add_to_avail_list(p); 2501 } 2502 2503 static void enable_swap_info(struct swap_info_struct *p, int prio, 2504 unsigned char *swap_map, 2505 struct swap_cluster_info *cluster_info, 2506 unsigned long *frontswap_map) 2507 { 2508 frontswap_init(p->type, frontswap_map); 2509 spin_lock(&swap_lock); 2510 spin_lock(&p->lock); 2511 setup_swap_info(p, prio, swap_map, cluster_info); 2512 spin_unlock(&p->lock); 2513 spin_unlock(&swap_lock); 2514 /* 2515 * Guarantee swap_map, cluster_info, etc. fields are valid 2516 * between get/put_swap_device() if SWP_VALID bit is set 2517 */ 2518 synchronize_rcu(); 2519 spin_lock(&swap_lock); 2520 spin_lock(&p->lock); 2521 _enable_swap_info(p); 2522 spin_unlock(&p->lock); 2523 spin_unlock(&swap_lock); 2524 } 2525 2526 static void reinsert_swap_info(struct swap_info_struct *p) 2527 { 2528 spin_lock(&swap_lock); 2529 spin_lock(&p->lock); 2530 setup_swap_info(p, p->prio, p->swap_map, p->cluster_info); 2531 _enable_swap_info(p); 2532 spin_unlock(&p->lock); 2533 spin_unlock(&swap_lock); 2534 } 2535 2536 bool has_usable_swap(void) 2537 { 2538 bool ret = true; 2539 2540 spin_lock(&swap_lock); 2541 if (plist_head_empty(&swap_active_head)) 2542 ret = false; 2543 spin_unlock(&swap_lock); 2544 return ret; 2545 } 2546 2547 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) 2548 { 2549 struct swap_info_struct *p = NULL; 2550 unsigned char *swap_map; 2551 struct swap_cluster_info *cluster_info; 2552 unsigned long *frontswap_map; 2553 struct file *swap_file, *victim; 2554 struct address_space *mapping; 2555 struct inode *inode; 2556 struct filename *pathname; 2557 int err, found = 0; 2558 unsigned int old_block_size; 2559 2560 if (!capable(CAP_SYS_ADMIN)) 2561 return -EPERM; 2562 2563 BUG_ON(!current->mm); 2564 2565 pathname = getname(specialfile); 2566 if (IS_ERR(pathname)) 2567 return PTR_ERR(pathname); 2568 2569 victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0); 2570 err = PTR_ERR(victim); 2571 if (IS_ERR(victim)) 2572 goto out; 2573 2574 mapping = victim->f_mapping; 2575 spin_lock(&swap_lock); 2576 plist_for_each_entry(p, &swap_active_head, list) { 2577 if (p->flags & SWP_WRITEOK) { 2578 if (p->swap_file->f_mapping == mapping) { 2579 found = 1; 2580 break; 2581 } 2582 } 2583 } 2584 if (!found) { 2585 err = -EINVAL; 2586 spin_unlock(&swap_lock); 2587 goto out_dput; 2588 } 2589 if (!security_vm_enough_memory_mm(current->mm, p->pages)) 2590 vm_unacct_memory(p->pages); 2591 else { 2592 err = -ENOMEM; 2593 spin_unlock(&swap_lock); 2594 goto out_dput; 2595 } 2596 del_from_avail_list(p); 2597 spin_lock(&p->lock); 2598 if (p->prio < 0) { 2599 struct swap_info_struct *si = p; 2600 int nid; 2601 2602 plist_for_each_entry_continue(si, &swap_active_head, list) { 2603 si->prio++; 2604 si->list.prio--; 2605 for_each_node(nid) { 2606 if (si->avail_lists[nid].prio != 1) 2607 si->avail_lists[nid].prio--; 2608 } 2609 } 2610 least_priority++; 2611 } 2612 plist_del(&p->list, &swap_active_head); 2613 atomic_long_sub(p->pages, &nr_swap_pages); 2614 total_swap_pages -= p->pages; 2615 p->flags &= ~SWP_WRITEOK; 2616 spin_unlock(&p->lock); 2617 spin_unlock(&swap_lock); 2618 2619 disable_swap_slots_cache_lock(); 2620 2621 set_current_oom_origin(); 2622 err = try_to_unuse(p->type, false, 0); /* force unuse all pages */ 2623 clear_current_oom_origin(); 2624 2625 if (err) { 2626 /* re-insert swap space back into swap_list */ 2627 reinsert_swap_info(p); 2628 reenable_swap_slots_cache_unlock(); 2629 goto out_dput; 2630 } 2631 2632 reenable_swap_slots_cache_unlock(); 2633 2634 spin_lock(&swap_lock); 2635 spin_lock(&p->lock); 2636 p->flags &= ~SWP_VALID; /* mark swap device as invalid */ 2637 spin_unlock(&p->lock); 2638 spin_unlock(&swap_lock); 2639 /* 2640 * wait for swap operations protected by get/put_swap_device() 2641 * to complete 2642 */ 2643 synchronize_rcu(); 2644 2645 flush_work(&p->discard_work); 2646 2647 destroy_swap_extents(p); 2648 if (p->flags & SWP_CONTINUED) 2649 free_swap_count_continuations(p); 2650 2651 if (!p->bdev || !blk_queue_nonrot(bdev_get_queue(p->bdev))) 2652 atomic_dec(&nr_rotate_swap); 2653 2654 mutex_lock(&swapon_mutex); 2655 spin_lock(&swap_lock); 2656 spin_lock(&p->lock); 2657 drain_mmlist(); 2658 2659 /* wait for anyone still in scan_swap_map */ 2660 p->highest_bit = 0; /* cuts scans short */ 2661 while (p->flags >= SWP_SCANNING) { 2662 spin_unlock(&p->lock); 2663 spin_unlock(&swap_lock); 2664 schedule_timeout_uninterruptible(1); 2665 spin_lock(&swap_lock); 2666 spin_lock(&p->lock); 2667 } 2668 2669 swap_file = p->swap_file; 2670 old_block_size = p->old_block_size; 2671 p->swap_file = NULL; 2672 p->max = 0; 2673 swap_map = p->swap_map; 2674 p->swap_map = NULL; 2675 cluster_info = p->cluster_info; 2676 p->cluster_info = NULL; 2677 frontswap_map = frontswap_map_get(p); 2678 spin_unlock(&p->lock); 2679 spin_unlock(&swap_lock); 2680 frontswap_invalidate_area(p->type); 2681 frontswap_map_set(p, NULL); 2682 mutex_unlock(&swapon_mutex); 2683 free_percpu(p->percpu_cluster); 2684 p->percpu_cluster = NULL; 2685 free_percpu(p->cluster_next_cpu); 2686 p->cluster_next_cpu = NULL; 2687 vfree(swap_map); 2688 kvfree(cluster_info); 2689 kvfree(frontswap_map); 2690 /* Destroy swap account information */ 2691 swap_cgroup_swapoff(p->type); 2692 exit_swap_address_space(p->type); 2693 2694 inode = mapping->host; 2695 if (S_ISBLK(inode->i_mode)) { 2696 struct block_device *bdev = I_BDEV(inode); 2697 2698 set_blocksize(bdev, old_block_size); 2699 blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); 2700 } 2701 2702 inode_lock(inode); 2703 inode->i_flags &= ~S_SWAPFILE; 2704 inode_unlock(inode); 2705 filp_close(swap_file, NULL); 2706 2707 /* 2708 * Clear the SWP_USED flag after all resources are freed so that swapon 2709 * can reuse this swap_info in alloc_swap_info() safely. It is ok to 2710 * not hold p->lock after we cleared its SWP_WRITEOK. 2711 */ 2712 spin_lock(&swap_lock); 2713 p->flags = 0; 2714 spin_unlock(&swap_lock); 2715 2716 err = 0; 2717 atomic_inc(&proc_poll_event); 2718 wake_up_interruptible(&proc_poll_wait); 2719 2720 out_dput: 2721 filp_close(victim, NULL); 2722 out: 2723 putname(pathname); 2724 return err; 2725 } 2726 2727 #ifdef CONFIG_PROC_FS 2728 static __poll_t swaps_poll(struct file *file, poll_table *wait) 2729 { 2730 struct seq_file *seq = file->private_data; 2731 2732 poll_wait(file, &proc_poll_wait, wait); 2733 2734 if (seq->poll_event != atomic_read(&proc_poll_event)) { 2735 seq->poll_event = atomic_read(&proc_poll_event); 2736 return EPOLLIN | EPOLLRDNORM | EPOLLERR | EPOLLPRI; 2737 } 2738 2739 return EPOLLIN | EPOLLRDNORM; 2740 } 2741 2742 /* iterator */ 2743 static void *swap_start(struct seq_file *swap, loff_t *pos) 2744 { 2745 struct swap_info_struct *si; 2746 int type; 2747 loff_t l = *pos; 2748 2749 mutex_lock(&swapon_mutex); 2750 2751 if (!l) 2752 return SEQ_START_TOKEN; 2753 2754 for (type = 0; (si = swap_type_to_swap_info(type)); type++) { 2755 if (!(si->flags & SWP_USED) || !si->swap_map) 2756 continue; 2757 if (!--l) 2758 return si; 2759 } 2760 2761 return NULL; 2762 } 2763 2764 static void *swap_next(struct seq_file *swap, void *v, loff_t *pos) 2765 { 2766 struct swap_info_struct *si = v; 2767 int type; 2768 2769 if (v == SEQ_START_TOKEN) 2770 type = 0; 2771 else 2772 type = si->type + 1; 2773 2774 ++(*pos); 2775 for (; (si = swap_type_to_swap_info(type)); type++) { 2776 if (!(si->flags & SWP_USED) || !si->swap_map) 2777 continue; 2778 return si; 2779 } 2780 2781 return NULL; 2782 } 2783 2784 static void swap_stop(struct seq_file *swap, void *v) 2785 { 2786 mutex_unlock(&swapon_mutex); 2787 } 2788 2789 static int swap_show(struct seq_file *swap, void *v) 2790 { 2791 struct swap_info_struct *si = v; 2792 struct file *file; 2793 int len; 2794 unsigned int bytes, inuse; 2795 2796 if (si == SEQ_START_TOKEN) { 2797 seq_puts(swap,"Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n"); 2798 return 0; 2799 } 2800 2801 bytes = si->pages << (PAGE_SHIFT - 10); 2802 inuse = si->inuse_pages << (PAGE_SHIFT - 10); 2803 2804 file = si->swap_file; 2805 len = seq_file_path(swap, file, " \t\n\\"); 2806 seq_printf(swap, "%*s%s\t%u\t%s%u\t%s%d\n", 2807 len < 40 ? 40 - len : 1, " ", 2808 S_ISBLK(file_inode(file)->i_mode) ? 2809 "partition" : "file\t", 2810 bytes, bytes < 10000000 ? "\t" : "", 2811 inuse, inuse < 10000000 ? "\t" : "", 2812 si->prio); 2813 return 0; 2814 } 2815 2816 static const struct seq_operations swaps_op = { 2817 .start = swap_start, 2818 .next = swap_next, 2819 .stop = swap_stop, 2820 .show = swap_show 2821 }; 2822 2823 static int swaps_open(struct inode *inode, struct file *file) 2824 { 2825 struct seq_file *seq; 2826 int ret; 2827 2828 ret = seq_open(file, &swaps_op); 2829 if (ret) 2830 return ret; 2831 2832 seq = file->private_data; 2833 seq->poll_event = atomic_read(&proc_poll_event); 2834 return 0; 2835 } 2836 2837 static const struct proc_ops swaps_proc_ops = { 2838 .proc_flags = PROC_ENTRY_PERMANENT, 2839 .proc_open = swaps_open, 2840 .proc_read = seq_read, 2841 .proc_lseek = seq_lseek, 2842 .proc_release = seq_release, 2843 .proc_poll = swaps_poll, 2844 }; 2845 2846 static int __init procswaps_init(void) 2847 { 2848 proc_create("swaps", 0, NULL, &swaps_proc_ops); 2849 return 0; 2850 } 2851 __initcall(procswaps_init); 2852 #endif /* CONFIG_PROC_FS */ 2853 2854 #ifdef MAX_SWAPFILES_CHECK 2855 static int __init max_swapfiles_check(void) 2856 { 2857 MAX_SWAPFILES_CHECK(); 2858 return 0; 2859 } 2860 late_initcall(max_swapfiles_check); 2861 #endif 2862 2863 static struct swap_info_struct *alloc_swap_info(void) 2864 { 2865 struct swap_info_struct *p; 2866 unsigned int type; 2867 int i; 2868 2869 p = kvzalloc(struct_size(p, avail_lists, nr_node_ids), GFP_KERNEL); 2870 if (!p) 2871 return ERR_PTR(-ENOMEM); 2872 2873 spin_lock(&swap_lock); 2874 for (type = 0; type < nr_swapfiles; type++) { 2875 if (!(swap_info[type]->flags & SWP_USED)) 2876 break; 2877 } 2878 if (type >= MAX_SWAPFILES) { 2879 spin_unlock(&swap_lock); 2880 kvfree(p); 2881 return ERR_PTR(-EPERM); 2882 } 2883 if (type >= nr_swapfiles) { 2884 p->type = type; 2885 WRITE_ONCE(swap_info[type], p); 2886 /* 2887 * Write swap_info[type] before nr_swapfiles, in case a 2888 * racing procfs swap_start() or swap_next() is reading them. 2889 * (We never shrink nr_swapfiles, we never free this entry.) 2890 */ 2891 smp_wmb(); 2892 WRITE_ONCE(nr_swapfiles, nr_swapfiles + 1); 2893 } else { 2894 kvfree(p); 2895 p = swap_info[type]; 2896 /* 2897 * Do not memset this entry: a racing procfs swap_next() 2898 * would be relying on p->type to remain valid. 2899 */ 2900 } 2901 p->swap_extent_root = RB_ROOT; 2902 plist_node_init(&p->list, 0); 2903 for_each_node(i) 2904 plist_node_init(&p->avail_lists[i], 0); 2905 p->flags = SWP_USED; 2906 spin_unlock(&swap_lock); 2907 spin_lock_init(&p->lock); 2908 spin_lock_init(&p->cont_lock); 2909 2910 return p; 2911 } 2912 2913 static int claim_swapfile(struct swap_info_struct *p, struct inode *inode) 2914 { 2915 int error; 2916 2917 if (S_ISBLK(inode->i_mode)) { 2918 p->bdev = bdgrab(I_BDEV(inode)); 2919 error = blkdev_get(p->bdev, 2920 FMODE_READ | FMODE_WRITE | FMODE_EXCL, p); 2921 if (error < 0) { 2922 p->bdev = NULL; 2923 return error; 2924 } 2925 p->old_block_size = block_size(p->bdev); 2926 error = set_blocksize(p->bdev, PAGE_SIZE); 2927 if (error < 0) 2928 return error; 2929 /* 2930 * Zoned block devices contain zones that have a sequential 2931 * write only restriction. Hence zoned block devices are not 2932 * suitable for swapping. Disallow them here. 2933 */ 2934 if (blk_queue_is_zoned(p->bdev->bd_disk->queue)) 2935 return -EINVAL; 2936 p->flags |= SWP_BLKDEV; 2937 } else if (S_ISREG(inode->i_mode)) { 2938 p->bdev = inode->i_sb->s_bdev; 2939 } 2940 2941 return 0; 2942 } 2943 2944 2945 /* 2946 * Find out how many pages are allowed for a single swap device. There 2947 * are two limiting factors: 2948 * 1) the number of bits for the swap offset in the swp_entry_t type, and 2949 * 2) the number of bits in the swap pte, as defined by the different 2950 * architectures. 2951 * 2952 * In order to find the largest possible bit mask, a swap entry with 2953 * swap type 0 and swap offset ~0UL is created, encoded to a swap pte, 2954 * decoded to a swp_entry_t again, and finally the swap offset is 2955 * extracted. 2956 * 2957 * This will mask all the bits from the initial ~0UL mask that can't 2958 * be encoded in either the swp_entry_t or the architecture definition 2959 * of a swap pte. 2960 */ 2961 unsigned long generic_max_swapfile_size(void) 2962 { 2963 return swp_offset(pte_to_swp_entry( 2964 swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1; 2965 } 2966 2967 /* Can be overridden by an architecture for additional checks. */ 2968 __weak unsigned long max_swapfile_size(void) 2969 { 2970 return generic_max_swapfile_size(); 2971 } 2972 2973 static unsigned long read_swap_header(struct swap_info_struct *p, 2974 union swap_header *swap_header, 2975 struct inode *inode) 2976 { 2977 int i; 2978 unsigned long maxpages; 2979 unsigned long swapfilepages; 2980 unsigned long last_page; 2981 2982 if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) { 2983 pr_err("Unable to find swap-space signature\n"); 2984 return 0; 2985 } 2986 2987 /* swap partition endianess hack... */ 2988 if (swab32(swap_header->info.version) == 1) { 2989 swab32s(&swap_header->info.version); 2990 swab32s(&swap_header->info.last_page); 2991 swab32s(&swap_header->info.nr_badpages); 2992 if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES) 2993 return 0; 2994 for (i = 0; i < swap_header->info.nr_badpages; i++) 2995 swab32s(&swap_header->info.badpages[i]); 2996 } 2997 /* Check the swap header's sub-version */ 2998 if (swap_header->info.version != 1) { 2999 pr_warn("Unable to handle swap header version %d\n", 3000 swap_header->info.version); 3001 return 0; 3002 } 3003 3004 p->lowest_bit = 1; 3005 p->cluster_next = 1; 3006 p->cluster_nr = 0; 3007 3008 maxpages = max_swapfile_size(); 3009 last_page = swap_header->info.last_page; 3010 if (!last_page) { 3011 pr_warn("Empty swap-file\n"); 3012 return 0; 3013 } 3014 if (last_page > maxpages) { 3015 pr_warn("Truncating oversized swap area, only using %luk out of %luk\n", 3016 maxpages << (PAGE_SHIFT - 10), 3017 last_page << (PAGE_SHIFT - 10)); 3018 } 3019 if (maxpages > last_page) { 3020 maxpages = last_page + 1; 3021 /* p->max is an unsigned int: don't overflow it */ 3022 if ((unsigned int)maxpages == 0) 3023 maxpages = UINT_MAX; 3024 } 3025 p->highest_bit = maxpages - 1; 3026 3027 if (!maxpages) 3028 return 0; 3029 swapfilepages = i_size_read(inode) >> PAGE_SHIFT; 3030 if (swapfilepages && maxpages > swapfilepages) { 3031 pr_warn("Swap area shorter than signature indicates\n"); 3032 return 0; 3033 } 3034 if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode)) 3035 return 0; 3036 if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES) 3037 return 0; 3038 3039 return maxpages; 3040 } 3041 3042 #define SWAP_CLUSTER_INFO_COLS \ 3043 DIV_ROUND_UP(L1_CACHE_BYTES, sizeof(struct swap_cluster_info)) 3044 #define SWAP_CLUSTER_SPACE_COLS \ 3045 DIV_ROUND_UP(SWAP_ADDRESS_SPACE_PAGES, SWAPFILE_CLUSTER) 3046 #define SWAP_CLUSTER_COLS \ 3047 max_t(unsigned int, SWAP_CLUSTER_INFO_COLS, SWAP_CLUSTER_SPACE_COLS) 3048 3049 static int setup_swap_map_and_extents(struct swap_info_struct *p, 3050 union swap_header *swap_header, 3051 unsigned char *swap_map, 3052 struct swap_cluster_info *cluster_info, 3053 unsigned long maxpages, 3054 sector_t *span) 3055 { 3056 unsigned int j, k; 3057 unsigned int nr_good_pages; 3058 int nr_extents; 3059 unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER); 3060 unsigned long col = p->cluster_next / SWAPFILE_CLUSTER % SWAP_CLUSTER_COLS; 3061 unsigned long i, idx; 3062 3063 nr_good_pages = maxpages - 1; /* omit header page */ 3064 3065 cluster_list_init(&p->free_clusters); 3066 cluster_list_init(&p->discard_clusters); 3067 3068 for (i = 0; i < swap_header->info.nr_badpages; i++) { 3069 unsigned int page_nr = swap_header->info.badpages[i]; 3070 if (page_nr == 0 || page_nr > swap_header->info.last_page) 3071 return -EINVAL; 3072 if (page_nr < maxpages) { 3073 swap_map[page_nr] = SWAP_MAP_BAD; 3074 nr_good_pages--; 3075 /* 3076 * Haven't marked the cluster free yet, no list 3077 * operation involved 3078 */ 3079 inc_cluster_info_page(p, cluster_info, page_nr); 3080 } 3081 } 3082 3083 /* Haven't marked the cluster free yet, no list operation involved */ 3084 for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++) 3085 inc_cluster_info_page(p, cluster_info, i); 3086 3087 if (nr_good_pages) { 3088 swap_map[0] = SWAP_MAP_BAD; 3089 /* 3090 * Not mark the cluster free yet, no list 3091 * operation involved 3092 */ 3093 inc_cluster_info_page(p, cluster_info, 0); 3094 p->max = maxpages; 3095 p->pages = nr_good_pages; 3096 nr_extents = setup_swap_extents(p, span); 3097 if (nr_extents < 0) 3098 return nr_extents; 3099 nr_good_pages = p->pages; 3100 } 3101 if (!nr_good_pages) { 3102 pr_warn("Empty swap-file\n"); 3103 return -EINVAL; 3104 } 3105 3106 if (!cluster_info) 3107 return nr_extents; 3108 3109 3110 /* 3111 * Reduce false cache line sharing between cluster_info and 3112 * sharing same address space. 3113 */ 3114 for (k = 0; k < SWAP_CLUSTER_COLS; k++) { 3115 j = (k + col) % SWAP_CLUSTER_COLS; 3116 for (i = 0; i < DIV_ROUND_UP(nr_clusters, SWAP_CLUSTER_COLS); i++) { 3117 idx = i * SWAP_CLUSTER_COLS + j; 3118 if (idx >= nr_clusters) 3119 continue; 3120 if (cluster_count(&cluster_info[idx])) 3121 continue; 3122 cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE); 3123 cluster_list_add_tail(&p->free_clusters, cluster_info, 3124 idx); 3125 } 3126 } 3127 return nr_extents; 3128 } 3129 3130 /* 3131 * Helper to sys_swapon determining if a given swap 3132 * backing device queue supports DISCARD operations. 3133 */ 3134 static bool swap_discardable(struct swap_info_struct *si) 3135 { 3136 struct request_queue *q = bdev_get_queue(si->bdev); 3137 3138 if (!q || !blk_queue_discard(q)) 3139 return false; 3140 3141 return true; 3142 } 3143 3144 SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) 3145 { 3146 struct swap_info_struct *p; 3147 struct filename *name; 3148 struct file *swap_file = NULL; 3149 struct address_space *mapping; 3150 int prio; 3151 int error; 3152 union swap_header *swap_header; 3153 int nr_extents; 3154 sector_t span; 3155 unsigned long maxpages; 3156 unsigned char *swap_map = NULL; 3157 struct swap_cluster_info *cluster_info = NULL; 3158 unsigned long *frontswap_map = NULL; 3159 struct page *page = NULL; 3160 struct inode *inode = NULL; 3161 bool inced_nr_rotate_swap = false; 3162 3163 if (swap_flags & ~SWAP_FLAGS_VALID) 3164 return -EINVAL; 3165 3166 if (!capable(CAP_SYS_ADMIN)) 3167 return -EPERM; 3168 3169 if (!swap_avail_heads) 3170 return -ENOMEM; 3171 3172 p = alloc_swap_info(); 3173 if (IS_ERR(p)) 3174 return PTR_ERR(p); 3175 3176 INIT_WORK(&p->discard_work, swap_discard_work); 3177 3178 name = getname(specialfile); 3179 if (IS_ERR(name)) { 3180 error = PTR_ERR(name); 3181 name = NULL; 3182 goto bad_swap; 3183 } 3184 swap_file = file_open_name(name, O_RDWR|O_LARGEFILE, 0); 3185 if (IS_ERR(swap_file)) { 3186 error = PTR_ERR(swap_file); 3187 swap_file = NULL; 3188 goto bad_swap; 3189 } 3190 3191 p->swap_file = swap_file; 3192 mapping = swap_file->f_mapping; 3193 inode = mapping->host; 3194 3195 error = claim_swapfile(p, inode); 3196 if (unlikely(error)) 3197 goto bad_swap; 3198 3199 inode_lock(inode); 3200 if (IS_SWAPFILE(inode)) { 3201 error = -EBUSY; 3202 goto bad_swap_unlock_inode; 3203 } 3204 3205 /* 3206 * Read the swap header. 3207 */ 3208 if (!mapping->a_ops->readpage) { 3209 error = -EINVAL; 3210 goto bad_swap_unlock_inode; 3211 } 3212 page = read_mapping_page(mapping, 0, swap_file); 3213 if (IS_ERR(page)) { 3214 error = PTR_ERR(page); 3215 goto bad_swap_unlock_inode; 3216 } 3217 swap_header = kmap(page); 3218 3219 maxpages = read_swap_header(p, swap_header, inode); 3220 if (unlikely(!maxpages)) { 3221 error = -EINVAL; 3222 goto bad_swap_unlock_inode; 3223 } 3224 3225 /* OK, set up the swap map and apply the bad block list */ 3226 swap_map = vzalloc(maxpages); 3227 if (!swap_map) { 3228 error = -ENOMEM; 3229 goto bad_swap_unlock_inode; 3230 } 3231 3232 if (bdi_cap_stable_pages_required(inode_to_bdi(inode))) 3233 p->flags |= SWP_STABLE_WRITES; 3234 3235 if (bdi_cap_synchronous_io(inode_to_bdi(inode))) 3236 p->flags |= SWP_SYNCHRONOUS_IO; 3237 3238 if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) { 3239 int cpu; 3240 unsigned long ci, nr_cluster; 3241 3242 p->flags |= SWP_SOLIDSTATE; 3243 p->cluster_next_cpu = alloc_percpu(unsigned int); 3244 if (!p->cluster_next_cpu) { 3245 error = -ENOMEM; 3246 goto bad_swap_unlock_inode; 3247 } 3248 /* 3249 * select a random position to start with to help wear leveling 3250 * SSD 3251 */ 3252 for_each_possible_cpu(cpu) { 3253 per_cpu(*p->cluster_next_cpu, cpu) = 3254 1 + prandom_u32_max(p->highest_bit); 3255 } 3256 nr_cluster = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER); 3257 3258 cluster_info = kvcalloc(nr_cluster, sizeof(*cluster_info), 3259 GFP_KERNEL); 3260 if (!cluster_info) { 3261 error = -ENOMEM; 3262 goto bad_swap_unlock_inode; 3263 } 3264 3265 for (ci = 0; ci < nr_cluster; ci++) 3266 spin_lock_init(&((cluster_info + ci)->lock)); 3267 3268 p->percpu_cluster = alloc_percpu(struct percpu_cluster); 3269 if (!p->percpu_cluster) { 3270 error = -ENOMEM; 3271 goto bad_swap_unlock_inode; 3272 } 3273 for_each_possible_cpu(cpu) { 3274 struct percpu_cluster *cluster; 3275 cluster = per_cpu_ptr(p->percpu_cluster, cpu); 3276 cluster_set_null(&cluster->index); 3277 } 3278 } else { 3279 atomic_inc(&nr_rotate_swap); 3280 inced_nr_rotate_swap = true; 3281 } 3282 3283 error = swap_cgroup_swapon(p->type, maxpages); 3284 if (error) 3285 goto bad_swap_unlock_inode; 3286 3287 nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map, 3288 cluster_info, maxpages, &span); 3289 if (unlikely(nr_extents < 0)) { 3290 error = nr_extents; 3291 goto bad_swap_unlock_inode; 3292 } 3293 /* frontswap enabled? set up bit-per-page map for frontswap */ 3294 if (IS_ENABLED(CONFIG_FRONTSWAP)) 3295 frontswap_map = kvcalloc(BITS_TO_LONGS(maxpages), 3296 sizeof(long), 3297 GFP_KERNEL); 3298 3299 if (p->bdev &&(swap_flags & SWAP_FLAG_DISCARD) && swap_discardable(p)) { 3300 /* 3301 * When discard is enabled for swap with no particular 3302 * policy flagged, we set all swap discard flags here in 3303 * order to sustain backward compatibility with older 3304 * swapon(8) releases. 3305 */ 3306 p->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD | 3307 SWP_PAGE_DISCARD); 3308 3309 /* 3310 * By flagging sys_swapon, a sysadmin can tell us to 3311 * either do single-time area discards only, or to just 3312 * perform discards for released swap page-clusters. 3313 * Now it's time to adjust the p->flags accordingly. 3314 */ 3315 if (swap_flags & SWAP_FLAG_DISCARD_ONCE) 3316 p->flags &= ~SWP_PAGE_DISCARD; 3317 else if (swap_flags & SWAP_FLAG_DISCARD_PAGES) 3318 p->flags &= ~SWP_AREA_DISCARD; 3319 3320 /* issue a swapon-time discard if it's still required */ 3321 if (p->flags & SWP_AREA_DISCARD) { 3322 int err = discard_swap(p); 3323 if (unlikely(err)) 3324 pr_err("swapon: discard_swap(%p): %d\n", 3325 p, err); 3326 } 3327 } 3328 3329 error = init_swap_address_space(p->type, maxpages); 3330 if (error) 3331 goto bad_swap_unlock_inode; 3332 3333 /* 3334 * Flush any pending IO and dirty mappings before we start using this 3335 * swap device. 3336 */ 3337 inode->i_flags |= S_SWAPFILE; 3338 error = inode_drain_writes(inode); 3339 if (error) { 3340 inode->i_flags &= ~S_SWAPFILE; 3341 goto bad_swap_unlock_inode; 3342 } 3343 3344 mutex_lock(&swapon_mutex); 3345 prio = -1; 3346 if (swap_flags & SWAP_FLAG_PREFER) 3347 prio = 3348 (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT; 3349 enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map); 3350 3351 pr_info("Adding %uk swap on %s. Priority:%d extents:%d across:%lluk %s%s%s%s%s\n", 3352 p->pages<<(PAGE_SHIFT-10), name->name, p->prio, 3353 nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10), 3354 (p->flags & SWP_SOLIDSTATE) ? "SS" : "", 3355 (p->flags & SWP_DISCARDABLE) ? "D" : "", 3356 (p->flags & SWP_AREA_DISCARD) ? "s" : "", 3357 (p->flags & SWP_PAGE_DISCARD) ? "c" : "", 3358 (frontswap_map) ? "FS" : ""); 3359 3360 mutex_unlock(&swapon_mutex); 3361 atomic_inc(&proc_poll_event); 3362 wake_up_interruptible(&proc_poll_wait); 3363 3364 error = 0; 3365 goto out; 3366 bad_swap_unlock_inode: 3367 inode_unlock(inode); 3368 bad_swap: 3369 free_percpu(p->percpu_cluster); 3370 p->percpu_cluster = NULL; 3371 free_percpu(p->cluster_next_cpu); 3372 p->cluster_next_cpu = NULL; 3373 if (inode && S_ISBLK(inode->i_mode) && p->bdev) { 3374 set_blocksize(p->bdev, p->old_block_size); 3375 blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); 3376 } 3377 inode = NULL; 3378 destroy_swap_extents(p); 3379 swap_cgroup_swapoff(p->type); 3380 spin_lock(&swap_lock); 3381 p->swap_file = NULL; 3382 p->flags = 0; 3383 spin_unlock(&swap_lock); 3384 vfree(swap_map); 3385 kvfree(cluster_info); 3386 kvfree(frontswap_map); 3387 if (inced_nr_rotate_swap) 3388 atomic_dec(&nr_rotate_swap); 3389 if (swap_file) 3390 filp_close(swap_file, NULL); 3391 out: 3392 if (page && !IS_ERR(page)) { 3393 kunmap(page); 3394 put_page(page); 3395 } 3396 if (name) 3397 putname(name); 3398 if (inode) 3399 inode_unlock(inode); 3400 if (!error) 3401 enable_swap_slots_cache(); 3402 return error; 3403 } 3404 3405 void si_swapinfo(struct sysinfo *val) 3406 { 3407 unsigned int type; 3408 unsigned long nr_to_be_unused = 0; 3409 3410 spin_lock(&swap_lock); 3411 for (type = 0; type < nr_swapfiles; type++) { 3412 struct swap_info_struct *si = swap_info[type]; 3413 3414 if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK)) 3415 nr_to_be_unused += si->inuse_pages; 3416 } 3417 val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused; 3418 val->totalswap = total_swap_pages + nr_to_be_unused; 3419 spin_unlock(&swap_lock); 3420 } 3421 3422 /* 3423 * Verify that a swap entry is valid and increment its swap map count. 3424 * 3425 * Returns error code in following case. 3426 * - success -> 0 3427 * - swp_entry is invalid -> EINVAL 3428 * - swp_entry is migration entry -> EINVAL 3429 * - swap-cache reference is requested but there is already one. -> EEXIST 3430 * - swap-cache reference is requested but the entry is not used. -> ENOENT 3431 * - swap-mapped reference requested but needs continued swap count. -> ENOMEM 3432 */ 3433 static int __swap_duplicate(swp_entry_t entry, unsigned char usage) 3434 { 3435 struct swap_info_struct *p; 3436 struct swap_cluster_info *ci; 3437 unsigned long offset; 3438 unsigned char count; 3439 unsigned char has_cache; 3440 int err = -EINVAL; 3441 3442 p = get_swap_device(entry); 3443 if (!p) 3444 goto out; 3445 3446 offset = swp_offset(entry); 3447 ci = lock_cluster_or_swap_info(p, offset); 3448 3449 count = p->swap_map[offset]; 3450 3451 /* 3452 * swapin_readahead() doesn't check if a swap entry is valid, so the 3453 * swap entry could be SWAP_MAP_BAD. Check here with lock held. 3454 */ 3455 if (unlikely(swap_count(count) == SWAP_MAP_BAD)) { 3456 err = -ENOENT; 3457 goto unlock_out; 3458 } 3459 3460 has_cache = count & SWAP_HAS_CACHE; 3461 count &= ~SWAP_HAS_CACHE; 3462 err = 0; 3463 3464 if (usage == SWAP_HAS_CACHE) { 3465 3466 /* set SWAP_HAS_CACHE if there is no cache and entry is used */ 3467 if (!has_cache && count) 3468 has_cache = SWAP_HAS_CACHE; 3469 else if (has_cache) /* someone else added cache */ 3470 err = -EEXIST; 3471 else /* no users remaining */ 3472 err = -ENOENT; 3473 3474 } else if (count || has_cache) { 3475 3476 if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX) 3477 count += usage; 3478 else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX) 3479 err = -EINVAL; 3480 else if (swap_count_continued(p, offset, count)) 3481 count = COUNT_CONTINUED; 3482 else 3483 err = -ENOMEM; 3484 } else 3485 err = -ENOENT; /* unused swap entry */ 3486 3487 p->swap_map[offset] = count | has_cache; 3488 3489 unlock_out: 3490 unlock_cluster_or_swap_info(p, ci); 3491 out: 3492 if (p) 3493 put_swap_device(p); 3494 return err; 3495 } 3496 3497 /* 3498 * Help swapoff by noting that swap entry belongs to shmem/tmpfs 3499 * (in which case its reference count is never incremented). 3500 */ 3501 void swap_shmem_alloc(swp_entry_t entry) 3502 { 3503 __swap_duplicate(entry, SWAP_MAP_SHMEM); 3504 } 3505 3506 /* 3507 * Increase reference count of swap entry by 1. 3508 * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required 3509 * but could not be atomically allocated. Returns 0, just as if it succeeded, 3510 * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which 3511 * might occur if a page table entry has got corrupted. 3512 */ 3513 int swap_duplicate(swp_entry_t entry) 3514 { 3515 int err = 0; 3516 3517 while (!err && __swap_duplicate(entry, 1) == -ENOMEM) 3518 err = add_swap_count_continuation(entry, GFP_ATOMIC); 3519 return err; 3520 } 3521 3522 /* 3523 * @entry: swap entry for which we allocate swap cache. 3524 * 3525 * Called when allocating swap cache for existing swap entry, 3526 * This can return error codes. Returns 0 at success. 3527 * -EEXIST means there is a swap cache. 3528 * Note: return code is different from swap_duplicate(). 3529 */ 3530 int swapcache_prepare(swp_entry_t entry) 3531 { 3532 return __swap_duplicate(entry, SWAP_HAS_CACHE); 3533 } 3534 3535 struct swap_info_struct *swp_swap_info(swp_entry_t entry) 3536 { 3537 return swap_type_to_swap_info(swp_type(entry)); 3538 } 3539 3540 struct swap_info_struct *page_swap_info(struct page *page) 3541 { 3542 swp_entry_t entry = { .val = page_private(page) }; 3543 return swp_swap_info(entry); 3544 } 3545 3546 /* 3547 * out-of-line __page_file_ methods to avoid include hell. 3548 */ 3549 struct address_space *__page_file_mapping(struct page *page) 3550 { 3551 return page_swap_info(page)->swap_file->f_mapping; 3552 } 3553 EXPORT_SYMBOL_GPL(__page_file_mapping); 3554 3555 pgoff_t __page_file_index(struct page *page) 3556 { 3557 swp_entry_t swap = { .val = page_private(page) }; 3558 return swp_offset(swap); 3559 } 3560 EXPORT_SYMBOL_GPL(__page_file_index); 3561 3562 /* 3563 * add_swap_count_continuation - called when a swap count is duplicated 3564 * beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's 3565 * page of the original vmalloc'ed swap_map, to hold the continuation count 3566 * (for that entry and for its neighbouring PAGE_SIZE swap entries). Called 3567 * again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc. 3568 * 3569 * These continuation pages are seldom referenced: the common paths all work 3570 * on the original swap_map, only referring to a continuation page when the 3571 * low "digit" of a count is incremented or decremented through SWAP_MAP_MAX. 3572 * 3573 * add_swap_count_continuation(, GFP_ATOMIC) can be called while holding 3574 * page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL) 3575 * can be called after dropping locks. 3576 */ 3577 int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask) 3578 { 3579 struct swap_info_struct *si; 3580 struct swap_cluster_info *ci; 3581 struct page *head; 3582 struct page *page; 3583 struct page *list_page; 3584 pgoff_t offset; 3585 unsigned char count; 3586 int ret = 0; 3587 3588 /* 3589 * When debugging, it's easier to use __GFP_ZERO here; but it's better 3590 * for latency not to zero a page while GFP_ATOMIC and holding locks. 3591 */ 3592 page = alloc_page(gfp_mask | __GFP_HIGHMEM); 3593 3594 si = get_swap_device(entry); 3595 if (!si) { 3596 /* 3597 * An acceptable race has occurred since the failing 3598 * __swap_duplicate(): the swap device may be swapoff 3599 */ 3600 goto outer; 3601 } 3602 spin_lock(&si->lock); 3603 3604 offset = swp_offset(entry); 3605 3606 ci = lock_cluster(si, offset); 3607 3608 count = si->swap_map[offset] & ~SWAP_HAS_CACHE; 3609 3610 if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) { 3611 /* 3612 * The higher the swap count, the more likely it is that tasks 3613 * will race to add swap count continuation: we need to avoid 3614 * over-provisioning. 3615 */ 3616 goto out; 3617 } 3618 3619 if (!page) { 3620 ret = -ENOMEM; 3621 goto out; 3622 } 3623 3624 /* 3625 * We are fortunate that although vmalloc_to_page uses pte_offset_map, 3626 * no architecture is using highmem pages for kernel page tables: so it 3627 * will not corrupt the GFP_ATOMIC caller's atomic page table kmaps. 3628 */ 3629 head = vmalloc_to_page(si->swap_map + offset); 3630 offset &= ~PAGE_MASK; 3631 3632 spin_lock(&si->cont_lock); 3633 /* 3634 * Page allocation does not initialize the page's lru field, 3635 * but it does always reset its private field. 3636 */ 3637 if (!page_private(head)) { 3638 BUG_ON(count & COUNT_CONTINUED); 3639 INIT_LIST_HEAD(&head->lru); 3640 set_page_private(head, SWP_CONTINUED); 3641 si->flags |= SWP_CONTINUED; 3642 } 3643 3644 list_for_each_entry(list_page, &head->lru, lru) { 3645 unsigned char *map; 3646 3647 /* 3648 * If the previous map said no continuation, but we've found 3649 * a continuation page, free our allocation and use this one. 3650 */ 3651 if (!(count & COUNT_CONTINUED)) 3652 goto out_unlock_cont; 3653 3654 map = kmap_atomic(list_page) + offset; 3655 count = *map; 3656 kunmap_atomic(map); 3657 3658 /* 3659 * If this continuation count now has some space in it, 3660 * free our allocation and use this one. 3661 */ 3662 if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX) 3663 goto out_unlock_cont; 3664 } 3665 3666 list_add_tail(&page->lru, &head->lru); 3667 page = NULL; /* now it's attached, don't free it */ 3668 out_unlock_cont: 3669 spin_unlock(&si->cont_lock); 3670 out: 3671 unlock_cluster(ci); 3672 spin_unlock(&si->lock); 3673 put_swap_device(si); 3674 outer: 3675 if (page) 3676 __free_page(page); 3677 return ret; 3678 } 3679 3680 /* 3681 * swap_count_continued - when the original swap_map count is incremented 3682 * from SWAP_MAP_MAX, check if there is already a continuation page to carry 3683 * into, carry if so, or else fail until a new continuation page is allocated; 3684 * when the original swap_map count is decremented from 0 with continuation, 3685 * borrow from the continuation and report whether it still holds more. 3686 * Called while __swap_duplicate() or swap_entry_free() holds swap or cluster 3687 * lock. 3688 */ 3689 static bool swap_count_continued(struct swap_info_struct *si, 3690 pgoff_t offset, unsigned char count) 3691 { 3692 struct page *head; 3693 struct page *page; 3694 unsigned char *map; 3695 bool ret; 3696 3697 head = vmalloc_to_page(si->swap_map + offset); 3698 if (page_private(head) != SWP_CONTINUED) { 3699 BUG_ON(count & COUNT_CONTINUED); 3700 return false; /* need to add count continuation */ 3701 } 3702 3703 spin_lock(&si->cont_lock); 3704 offset &= ~PAGE_MASK; 3705 page = list_next_entry(head, lru); 3706 map = kmap_atomic(page) + offset; 3707 3708 if (count == SWAP_MAP_MAX) /* initial increment from swap_map */ 3709 goto init_map; /* jump over SWAP_CONT_MAX checks */ 3710 3711 if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */ 3712 /* 3713 * Think of how you add 1 to 999 3714 */ 3715 while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) { 3716 kunmap_atomic(map); 3717 page = list_next_entry(page, lru); 3718 BUG_ON(page == head); 3719 map = kmap_atomic(page) + offset; 3720 } 3721 if (*map == SWAP_CONT_MAX) { 3722 kunmap_atomic(map); 3723 page = list_next_entry(page, lru); 3724 if (page == head) { 3725 ret = false; /* add count continuation */ 3726 goto out; 3727 } 3728 map = kmap_atomic(page) + offset; 3729 init_map: *map = 0; /* we didn't zero the page */ 3730 } 3731 *map += 1; 3732 kunmap_atomic(map); 3733 while ((page = list_prev_entry(page, lru)) != head) { 3734 map = kmap_atomic(page) + offset; 3735 *map = COUNT_CONTINUED; 3736 kunmap_atomic(map); 3737 } 3738 ret = true; /* incremented */ 3739 3740 } else { /* decrementing */ 3741 /* 3742 * Think of how you subtract 1 from 1000 3743 */ 3744 BUG_ON(count != COUNT_CONTINUED); 3745 while (*map == COUNT_CONTINUED) { 3746 kunmap_atomic(map); 3747 page = list_next_entry(page, lru); 3748 BUG_ON(page == head); 3749 map = kmap_atomic(page) + offset; 3750 } 3751 BUG_ON(*map == 0); 3752 *map -= 1; 3753 if (*map == 0) 3754 count = 0; 3755 kunmap_atomic(map); 3756 while ((page = list_prev_entry(page, lru)) != head) { 3757 map = kmap_atomic(page) + offset; 3758 *map = SWAP_CONT_MAX | count; 3759 count = COUNT_CONTINUED; 3760 kunmap_atomic(map); 3761 } 3762 ret = count == COUNT_CONTINUED; 3763 } 3764 out: 3765 spin_unlock(&si->cont_lock); 3766 return ret; 3767 } 3768 3769 /* 3770 * free_swap_count_continuations - swapoff free all the continuation pages 3771 * appended to the swap_map, after swap_map is quiesced, before vfree'ing it. 3772 */ 3773 static void free_swap_count_continuations(struct swap_info_struct *si) 3774 { 3775 pgoff_t offset; 3776 3777 for (offset = 0; offset < si->max; offset += PAGE_SIZE) { 3778 struct page *head; 3779 head = vmalloc_to_page(si->swap_map + offset); 3780 if (page_private(head)) { 3781 struct page *page, *next; 3782 3783 list_for_each_entry_safe(page, next, &head->lru, lru) { 3784 list_del(&page->lru); 3785 __free_page(page); 3786 } 3787 } 3788 } 3789 } 3790 3791 #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) 3792 void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) 3793 { 3794 struct swap_info_struct *si, *next; 3795 int nid = page_to_nid(page); 3796 3797 if (!(gfp_mask & __GFP_IO)) 3798 return; 3799 3800 if (!blk_cgroup_congested()) 3801 return; 3802 3803 /* 3804 * We've already scheduled a throttle, avoid taking the global swap 3805 * lock. 3806 */ 3807 if (current->throttle_queue) 3808 return; 3809 3810 spin_lock(&swap_avail_lock); 3811 plist_for_each_entry_safe(si, next, &swap_avail_heads[nid], 3812 avail_lists[nid]) { 3813 if (si->bdev) { 3814 blkcg_schedule_throttle(bdev_get_queue(si->bdev), true); 3815 break; 3816 } 3817 } 3818 spin_unlock(&swap_avail_lock); 3819 } 3820 #endif 3821 3822 static int __init swapfile_init(void) 3823 { 3824 int nid; 3825 3826 swap_avail_heads = kmalloc_array(nr_node_ids, sizeof(struct plist_head), 3827 GFP_KERNEL); 3828 if (!swap_avail_heads) { 3829 pr_emerg("Not enough memory for swap heads, swap is disabled\n"); 3830 return -ENOMEM; 3831 } 3832 3833 for_each_node(nid) 3834 plist_head_init(&swap_avail_heads[nid]); 3835 3836 return 0; 3837 } 3838 subsys_initcall(swapfile_init); 3839