1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/mm/swap_state.c 4 * 5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 6 * Swap reorganised 29.12.95, Stephen Tweedie 7 * 8 * Rewritten to use page cache, (C) 1998 Stephen Tweedie 9 */ 10 #include <linux/mm.h> 11 #include <linux/gfp.h> 12 #include <linux/kernel_stat.h> 13 #include <linux/swap.h> 14 #include <linux/swapops.h> 15 #include <linux/init.h> 16 #include <linux/pagemap.h> 17 #include <linux/backing-dev.h> 18 #include <linux/blkdev.h> 19 #include <linux/pagevec.h> 20 #include <linux/migrate.h> 21 #include <linux/vmalloc.h> 22 #include <linux/swap_slots.h> 23 #include <linux/huge_mm.h> 24 25 #include <asm/pgtable.h> 26 27 /* 28 * swapper_space is a fiction, retained to simplify the path through 29 * vmscan's shrink_page_list. 30 */ 31 static const struct address_space_operations swap_aops = { 32 .writepage = swap_writepage, 33 .set_page_dirty = swap_set_page_dirty, 34 #ifdef CONFIG_MIGRATION 35 .migratepage = migrate_page, 36 #endif 37 }; 38 39 struct address_space *swapper_spaces[MAX_SWAPFILES] __read_mostly; 40 static unsigned int nr_swapper_spaces[MAX_SWAPFILES] __read_mostly; 41 static bool enable_vma_readahead __read_mostly = true; 42 43 #define SWAP_RA_WIN_SHIFT (PAGE_SHIFT / 2) 44 #define SWAP_RA_HITS_MASK ((1UL << SWAP_RA_WIN_SHIFT) - 1) 45 #define SWAP_RA_HITS_MAX SWAP_RA_HITS_MASK 46 #define SWAP_RA_WIN_MASK (~PAGE_MASK & ~SWAP_RA_HITS_MASK) 47 48 #define SWAP_RA_HITS(v) ((v) & SWAP_RA_HITS_MASK) 49 #define SWAP_RA_WIN(v) (((v) & SWAP_RA_WIN_MASK) >> SWAP_RA_WIN_SHIFT) 50 #define SWAP_RA_ADDR(v) ((v) & PAGE_MASK) 51 52 #define SWAP_RA_VAL(addr, win, hits) \ 53 (((addr) & PAGE_MASK) | \ 54 (((win) << SWAP_RA_WIN_SHIFT) & SWAP_RA_WIN_MASK) | \ 55 ((hits) & SWAP_RA_HITS_MASK)) 56 57 /* Initial readahead hits is 4 to start up with a small window */ 58 #define GET_SWAP_RA_VAL(vma) \ 59 (atomic_long_read(&(vma)->swap_readahead_info) ? : 4) 60 61 #define INC_CACHE_INFO(x) do { swap_cache_info.x++; } while (0) 62 #define ADD_CACHE_INFO(x, nr) do { swap_cache_info.x += (nr); } while (0) 63 64 static struct { 65 unsigned long add_total; 66 unsigned long del_total; 67 unsigned long find_success; 68 unsigned long find_total; 69 } swap_cache_info; 70 71 unsigned long total_swapcache_pages(void) 72 { 73 unsigned int i, j, nr; 74 unsigned long ret = 0; 75 struct address_space *spaces; 76 77 rcu_read_lock(); 78 for (i = 0; i < MAX_SWAPFILES; i++) { 79 /* 80 * The corresponding entries in nr_swapper_spaces and 81 * swapper_spaces will be reused only after at least 82 * one grace period. So it is impossible for them 83 * belongs to different usage. 84 */ 85 nr = nr_swapper_spaces[i]; 86 spaces = rcu_dereference(swapper_spaces[i]); 87 if (!nr || !spaces) 88 continue; 89 for (j = 0; j < nr; j++) 90 ret += spaces[j].nrpages; 91 } 92 rcu_read_unlock(); 93 return ret; 94 } 95 96 static atomic_t swapin_readahead_hits = ATOMIC_INIT(4); 97 98 void show_swap_cache_info(void) 99 { 100 printk("%lu pages in swap cache\n", total_swapcache_pages()); 101 printk("Swap cache stats: add %lu, delete %lu, find %lu/%lu\n", 102 swap_cache_info.add_total, swap_cache_info.del_total, 103 swap_cache_info.find_success, swap_cache_info.find_total); 104 printk("Free swap = %ldkB\n", 105 get_nr_swap_pages() << (PAGE_SHIFT - 10)); 106 printk("Total swap = %lukB\n", total_swap_pages << (PAGE_SHIFT - 10)); 107 } 108 109 /* 110 * add_to_swap_cache resembles add_to_page_cache_locked on swapper_space, 111 * but sets SwapCache flag and private instead of mapping and index. 112 */ 113 int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp) 114 { 115 struct address_space *address_space = swap_address_space(entry); 116 pgoff_t idx = swp_offset(entry); 117 XA_STATE_ORDER(xas, &address_space->i_pages, idx, compound_order(page)); 118 unsigned long i, nr = 1UL << compound_order(page); 119 120 VM_BUG_ON_PAGE(!PageLocked(page), page); 121 VM_BUG_ON_PAGE(PageSwapCache(page), page); 122 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 123 124 page_ref_add(page, nr); 125 SetPageSwapCache(page); 126 127 do { 128 xas_lock_irq(&xas); 129 xas_create_range(&xas); 130 if (xas_error(&xas)) 131 goto unlock; 132 for (i = 0; i < nr; i++) { 133 VM_BUG_ON_PAGE(xas.xa_index != idx + i, page); 134 set_page_private(page + i, entry.val + i); 135 xas_store(&xas, page); 136 xas_next(&xas); 137 } 138 address_space->nrpages += nr; 139 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); 140 ADD_CACHE_INFO(add_total, nr); 141 unlock: 142 xas_unlock_irq(&xas); 143 } while (xas_nomem(&xas, gfp)); 144 145 if (!xas_error(&xas)) 146 return 0; 147 148 ClearPageSwapCache(page); 149 page_ref_sub(page, nr); 150 return xas_error(&xas); 151 } 152 153 /* 154 * This must be called only on pages that have 155 * been verified to be in the swap cache. 156 */ 157 void __delete_from_swap_cache(struct page *page, swp_entry_t entry) 158 { 159 struct address_space *address_space = swap_address_space(entry); 160 int i, nr = hpage_nr_pages(page); 161 pgoff_t idx = swp_offset(entry); 162 XA_STATE(xas, &address_space->i_pages, idx); 163 164 VM_BUG_ON_PAGE(!PageLocked(page), page); 165 VM_BUG_ON_PAGE(!PageSwapCache(page), page); 166 VM_BUG_ON_PAGE(PageWriteback(page), page); 167 168 for (i = 0; i < nr; i++) { 169 void *entry = xas_store(&xas, NULL); 170 VM_BUG_ON_PAGE(entry != page, entry); 171 set_page_private(page + i, 0); 172 xas_next(&xas); 173 } 174 ClearPageSwapCache(page); 175 address_space->nrpages -= nr; 176 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr); 177 ADD_CACHE_INFO(del_total, nr); 178 } 179 180 /** 181 * add_to_swap - allocate swap space for a page 182 * @page: page we want to move to swap 183 * 184 * Allocate swap space for the page and add the page to the 185 * swap cache. Caller needs to hold the page lock. 186 */ 187 int add_to_swap(struct page *page) 188 { 189 swp_entry_t entry; 190 int err; 191 192 VM_BUG_ON_PAGE(!PageLocked(page), page); 193 VM_BUG_ON_PAGE(!PageUptodate(page), page); 194 195 entry = get_swap_page(page); 196 if (!entry.val) 197 return 0; 198 199 /* 200 * XArray node allocations from PF_MEMALLOC contexts could 201 * completely exhaust the page allocator. __GFP_NOMEMALLOC 202 * stops emergency reserves from being allocated. 203 * 204 * TODO: this could cause a theoretical memory reclaim 205 * deadlock in the swap out path. 206 */ 207 /* 208 * Add it to the swap cache. 209 */ 210 err = add_to_swap_cache(page, entry, 211 __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN); 212 if (err) 213 /* 214 * add_to_swap_cache() doesn't return -EEXIST, so we can safely 215 * clear SWAP_HAS_CACHE flag. 216 */ 217 goto fail; 218 /* 219 * Normally the page will be dirtied in unmap because its pte should be 220 * dirty. A special case is MADV_FREE page. The page'e pte could have 221 * dirty bit cleared but the page's SwapBacked bit is still set because 222 * clearing the dirty bit and SwapBacked bit has no lock protected. For 223 * such page, unmap will not set dirty bit for it, so page reclaim will 224 * not write the page out. This can cause data corruption when the page 225 * is swap in later. Always setting the dirty bit for the page solves 226 * the problem. 227 */ 228 set_page_dirty(page); 229 230 return 1; 231 232 fail: 233 put_swap_page(page, entry); 234 return 0; 235 } 236 237 /* 238 * This must be called only on pages that have 239 * been verified to be in the swap cache and locked. 240 * It will never put the page into the free list, 241 * the caller has a reference on the page. 242 */ 243 void delete_from_swap_cache(struct page *page) 244 { 245 swp_entry_t entry = { .val = page_private(page) }; 246 struct address_space *address_space = swap_address_space(entry); 247 248 xa_lock_irq(&address_space->i_pages); 249 __delete_from_swap_cache(page, entry); 250 xa_unlock_irq(&address_space->i_pages); 251 252 put_swap_page(page, entry); 253 page_ref_sub(page, hpage_nr_pages(page)); 254 } 255 256 /* 257 * If we are the only user, then try to free up the swap cache. 258 * 259 * Its ok to check for PageSwapCache without the page lock 260 * here because we are going to recheck again inside 261 * try_to_free_swap() _with_ the lock. 262 * - Marcelo 263 */ 264 static inline void free_swap_cache(struct page *page) 265 { 266 if (PageSwapCache(page) && !page_mapped(page) && trylock_page(page)) { 267 try_to_free_swap(page); 268 unlock_page(page); 269 } 270 } 271 272 /* 273 * Perform a free_page(), also freeing any swap cache associated with 274 * this page if it is the last user of the page. 275 */ 276 void free_page_and_swap_cache(struct page *page) 277 { 278 free_swap_cache(page); 279 if (!is_huge_zero_page(page)) 280 put_page(page); 281 } 282 283 /* 284 * Passed an array of pages, drop them all from swapcache and then release 285 * them. They are removed from the LRU and freed if this is their last use. 286 */ 287 void free_pages_and_swap_cache(struct page **pages, int nr) 288 { 289 struct page **pagep = pages; 290 int i; 291 292 lru_add_drain(); 293 for (i = 0; i < nr; i++) 294 free_swap_cache(pagep[i]); 295 release_pages(pagep, nr); 296 } 297 298 static inline bool swap_use_vma_readahead(void) 299 { 300 return READ_ONCE(enable_vma_readahead) && !atomic_read(&nr_rotate_swap); 301 } 302 303 /* 304 * Lookup a swap entry in the swap cache. A found page will be returned 305 * unlocked and with its refcount incremented - we rely on the kernel 306 * lock getting page table operations atomic even if we drop the page 307 * lock before returning. 308 */ 309 struct page *lookup_swap_cache(swp_entry_t entry, struct vm_area_struct *vma, 310 unsigned long addr) 311 { 312 struct page *page; 313 314 page = find_get_page(swap_address_space(entry), swp_offset(entry)); 315 316 INC_CACHE_INFO(find_total); 317 if (page) { 318 bool vma_ra = swap_use_vma_readahead(); 319 bool readahead; 320 321 INC_CACHE_INFO(find_success); 322 /* 323 * At the moment, we don't support PG_readahead for anon THP 324 * so let's bail out rather than confusing the readahead stat. 325 */ 326 if (unlikely(PageTransCompound(page))) 327 return page; 328 329 readahead = TestClearPageReadahead(page); 330 if (vma && vma_ra) { 331 unsigned long ra_val; 332 int win, hits; 333 334 ra_val = GET_SWAP_RA_VAL(vma); 335 win = SWAP_RA_WIN(ra_val); 336 hits = SWAP_RA_HITS(ra_val); 337 if (readahead) 338 hits = min_t(int, hits + 1, SWAP_RA_HITS_MAX); 339 atomic_long_set(&vma->swap_readahead_info, 340 SWAP_RA_VAL(addr, win, hits)); 341 } 342 343 if (readahead) { 344 count_vm_event(SWAP_RA_HIT); 345 if (!vma || !vma_ra) 346 atomic_inc(&swapin_readahead_hits); 347 } 348 } 349 350 return page; 351 } 352 353 struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, 354 struct vm_area_struct *vma, unsigned long addr, 355 bool *new_page_allocated) 356 { 357 struct page *found_page, *new_page = NULL; 358 struct address_space *swapper_space = swap_address_space(entry); 359 int err; 360 *new_page_allocated = false; 361 362 do { 363 /* 364 * First check the swap cache. Since this is normally 365 * called after lookup_swap_cache() failed, re-calling 366 * that would confuse statistics. 367 */ 368 found_page = find_get_page(swapper_space, swp_offset(entry)); 369 if (found_page) 370 break; 371 372 /* 373 * Just skip read ahead for unused swap slot. 374 * During swap_off when swap_slot_cache is disabled, 375 * we have to handle the race between putting 376 * swap entry in swap cache and marking swap slot 377 * as SWAP_HAS_CACHE. That's done in later part of code or 378 * else swap_off will be aborted if we return NULL. 379 */ 380 if (!__swp_swapcount(entry) && swap_slot_cache_enabled) 381 break; 382 383 /* 384 * Get a new page to read into from swap. 385 */ 386 if (!new_page) { 387 new_page = alloc_page_vma(gfp_mask, vma, addr); 388 if (!new_page) 389 break; /* Out of memory */ 390 } 391 392 /* 393 * Swap entry may have been freed since our caller observed it. 394 */ 395 err = swapcache_prepare(entry); 396 if (err == -EEXIST) { 397 /* 398 * We might race against get_swap_page() and stumble 399 * across a SWAP_HAS_CACHE swap_map entry whose page 400 * has not been brought into the swapcache yet. 401 */ 402 cond_resched(); 403 continue; 404 } else if (err) /* swp entry is obsolete ? */ 405 break; 406 407 /* May fail (-ENOMEM) if XArray node allocation failed. */ 408 __SetPageLocked(new_page); 409 __SetPageSwapBacked(new_page); 410 err = add_to_swap_cache(new_page, entry, gfp_mask & GFP_KERNEL); 411 if (likely(!err)) { 412 /* Initiate read into locked page */ 413 SetPageWorkingset(new_page); 414 lru_cache_add_anon(new_page); 415 *new_page_allocated = true; 416 return new_page; 417 } 418 __ClearPageLocked(new_page); 419 /* 420 * add_to_swap_cache() doesn't return -EEXIST, so we can safely 421 * clear SWAP_HAS_CACHE flag. 422 */ 423 put_swap_page(new_page, entry); 424 } while (err != -ENOMEM); 425 426 if (new_page) 427 put_page(new_page); 428 return found_page; 429 } 430 431 /* 432 * Locate a page of swap in physical memory, reserving swap cache space 433 * and reading the disk if it is not already cached. 434 * A failure return means that either the page allocation failed or that 435 * the swap entry is no longer in use. 436 */ 437 struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, 438 struct vm_area_struct *vma, unsigned long addr, bool do_poll) 439 { 440 bool page_was_allocated; 441 struct page *retpage = __read_swap_cache_async(entry, gfp_mask, 442 vma, addr, &page_was_allocated); 443 444 if (page_was_allocated) 445 swap_readpage(retpage, do_poll); 446 447 return retpage; 448 } 449 450 static unsigned int __swapin_nr_pages(unsigned long prev_offset, 451 unsigned long offset, 452 int hits, 453 int max_pages, 454 int prev_win) 455 { 456 unsigned int pages, last_ra; 457 458 /* 459 * This heuristic has been found to work well on both sequential and 460 * random loads, swapping to hard disk or to SSD: please don't ask 461 * what the "+ 2" means, it just happens to work well, that's all. 462 */ 463 pages = hits + 2; 464 if (pages == 2) { 465 /* 466 * We can have no readahead hits to judge by: but must not get 467 * stuck here forever, so check for an adjacent offset instead 468 * (and don't even bother to check whether swap type is same). 469 */ 470 if (offset != prev_offset + 1 && offset != prev_offset - 1) 471 pages = 1; 472 } else { 473 unsigned int roundup = 4; 474 while (roundup < pages) 475 roundup <<= 1; 476 pages = roundup; 477 } 478 479 if (pages > max_pages) 480 pages = max_pages; 481 482 /* Don't shrink readahead too fast */ 483 last_ra = prev_win / 2; 484 if (pages < last_ra) 485 pages = last_ra; 486 487 return pages; 488 } 489 490 static unsigned long swapin_nr_pages(unsigned long offset) 491 { 492 static unsigned long prev_offset; 493 unsigned int hits, pages, max_pages; 494 static atomic_t last_readahead_pages; 495 496 max_pages = 1 << READ_ONCE(page_cluster); 497 if (max_pages <= 1) 498 return 1; 499 500 hits = atomic_xchg(&swapin_readahead_hits, 0); 501 pages = __swapin_nr_pages(prev_offset, offset, hits, max_pages, 502 atomic_read(&last_readahead_pages)); 503 if (!hits) 504 prev_offset = offset; 505 atomic_set(&last_readahead_pages, pages); 506 507 return pages; 508 } 509 510 /** 511 * swap_cluster_readahead - swap in pages in hope we need them soon 512 * @entry: swap entry of this memory 513 * @gfp_mask: memory allocation flags 514 * @vmf: fault information 515 * 516 * Returns the struct page for entry and addr, after queueing swapin. 517 * 518 * Primitive swap readahead code. We simply read an aligned block of 519 * (1 << page_cluster) entries in the swap area. This method is chosen 520 * because it doesn't cost us any seek time. We also make sure to queue 521 * the 'original' request together with the readahead ones... 522 * 523 * This has been extended to use the NUMA policies from the mm triggering 524 * the readahead. 525 * 526 * Caller must hold read mmap_sem if vmf->vma is not NULL. 527 */ 528 struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, 529 struct vm_fault *vmf) 530 { 531 struct page *page; 532 unsigned long entry_offset = swp_offset(entry); 533 unsigned long offset = entry_offset; 534 unsigned long start_offset, end_offset; 535 unsigned long mask; 536 struct swap_info_struct *si = swp_swap_info(entry); 537 struct blk_plug plug; 538 bool do_poll = true, page_allocated; 539 struct vm_area_struct *vma = vmf->vma; 540 unsigned long addr = vmf->address; 541 542 mask = swapin_nr_pages(offset) - 1; 543 if (!mask) 544 goto skip; 545 546 /* Test swap type to make sure the dereference is safe */ 547 if (likely(si->flags & (SWP_BLKDEV | SWP_FS))) { 548 struct inode *inode = si->swap_file->f_mapping->host; 549 if (inode_read_congested(inode)) 550 goto skip; 551 } 552 553 do_poll = false; 554 /* Read a page_cluster sized and aligned cluster around offset. */ 555 start_offset = offset & ~mask; 556 end_offset = offset | mask; 557 if (!start_offset) /* First page is swap header. */ 558 start_offset++; 559 if (end_offset >= si->max) 560 end_offset = si->max - 1; 561 562 blk_start_plug(&plug); 563 for (offset = start_offset; offset <= end_offset ; offset++) { 564 /* Ok, do the async read-ahead now */ 565 page = __read_swap_cache_async( 566 swp_entry(swp_type(entry), offset), 567 gfp_mask, vma, addr, &page_allocated); 568 if (!page) 569 continue; 570 if (page_allocated) { 571 swap_readpage(page, false); 572 if (offset != entry_offset) { 573 SetPageReadahead(page); 574 count_vm_event(SWAP_RA); 575 } 576 } 577 put_page(page); 578 } 579 blk_finish_plug(&plug); 580 581 lru_add_drain(); /* Push any new pages onto the LRU now */ 582 skip: 583 return read_swap_cache_async(entry, gfp_mask, vma, addr, do_poll); 584 } 585 586 int init_swap_address_space(unsigned int type, unsigned long nr_pages) 587 { 588 struct address_space *spaces, *space; 589 unsigned int i, nr; 590 591 nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES); 592 spaces = kvcalloc(nr, sizeof(struct address_space), GFP_KERNEL); 593 if (!spaces) 594 return -ENOMEM; 595 for (i = 0; i < nr; i++) { 596 space = spaces + i; 597 xa_init_flags(&space->i_pages, XA_FLAGS_LOCK_IRQ); 598 atomic_set(&space->i_mmap_writable, 0); 599 space->a_ops = &swap_aops; 600 /* swap cache doesn't use writeback related tags */ 601 mapping_set_no_writeback_tags(space); 602 } 603 nr_swapper_spaces[type] = nr; 604 rcu_assign_pointer(swapper_spaces[type], spaces); 605 606 return 0; 607 } 608 609 void exit_swap_address_space(unsigned int type) 610 { 611 struct address_space *spaces; 612 613 spaces = swapper_spaces[type]; 614 nr_swapper_spaces[type] = 0; 615 rcu_assign_pointer(swapper_spaces[type], NULL); 616 synchronize_rcu(); 617 kvfree(spaces); 618 } 619 620 static inline void swap_ra_clamp_pfn(struct vm_area_struct *vma, 621 unsigned long faddr, 622 unsigned long lpfn, 623 unsigned long rpfn, 624 unsigned long *start, 625 unsigned long *end) 626 { 627 *start = max3(lpfn, PFN_DOWN(vma->vm_start), 628 PFN_DOWN(faddr & PMD_MASK)); 629 *end = min3(rpfn, PFN_DOWN(vma->vm_end), 630 PFN_DOWN((faddr & PMD_MASK) + PMD_SIZE)); 631 } 632 633 static void swap_ra_info(struct vm_fault *vmf, 634 struct vma_swap_readahead *ra_info) 635 { 636 struct vm_area_struct *vma = vmf->vma; 637 unsigned long ra_val; 638 swp_entry_t entry; 639 unsigned long faddr, pfn, fpfn; 640 unsigned long start, end; 641 pte_t *pte, *orig_pte; 642 unsigned int max_win, hits, prev_win, win, left; 643 #ifndef CONFIG_64BIT 644 pte_t *tpte; 645 #endif 646 647 max_win = 1 << min_t(unsigned int, READ_ONCE(page_cluster), 648 SWAP_RA_ORDER_CEILING); 649 if (max_win == 1) { 650 ra_info->win = 1; 651 return; 652 } 653 654 faddr = vmf->address; 655 orig_pte = pte = pte_offset_map(vmf->pmd, faddr); 656 entry = pte_to_swp_entry(*pte); 657 if ((unlikely(non_swap_entry(entry)))) { 658 pte_unmap(orig_pte); 659 return; 660 } 661 662 fpfn = PFN_DOWN(faddr); 663 ra_val = GET_SWAP_RA_VAL(vma); 664 pfn = PFN_DOWN(SWAP_RA_ADDR(ra_val)); 665 prev_win = SWAP_RA_WIN(ra_val); 666 hits = SWAP_RA_HITS(ra_val); 667 ra_info->win = win = __swapin_nr_pages(pfn, fpfn, hits, 668 max_win, prev_win); 669 atomic_long_set(&vma->swap_readahead_info, 670 SWAP_RA_VAL(faddr, win, 0)); 671 672 if (win == 1) { 673 pte_unmap(orig_pte); 674 return; 675 } 676 677 /* Copy the PTEs because the page table may be unmapped */ 678 if (fpfn == pfn + 1) 679 swap_ra_clamp_pfn(vma, faddr, fpfn, fpfn + win, &start, &end); 680 else if (pfn == fpfn + 1) 681 swap_ra_clamp_pfn(vma, faddr, fpfn - win + 1, fpfn + 1, 682 &start, &end); 683 else { 684 left = (win - 1) / 2; 685 swap_ra_clamp_pfn(vma, faddr, fpfn - left, fpfn + win - left, 686 &start, &end); 687 } 688 ra_info->nr_pte = end - start; 689 ra_info->offset = fpfn - start; 690 pte -= ra_info->offset; 691 #ifdef CONFIG_64BIT 692 ra_info->ptes = pte; 693 #else 694 tpte = ra_info->ptes; 695 for (pfn = start; pfn != end; pfn++) 696 *tpte++ = *pte++; 697 #endif 698 pte_unmap(orig_pte); 699 } 700 701 /** 702 * swap_vma_readahead - swap in pages in hope we need them soon 703 * @entry: swap entry of this memory 704 * @gfp_mask: memory allocation flags 705 * @vmf: fault information 706 * 707 * Returns the struct page for entry and addr, after queueing swapin. 708 * 709 * Primitive swap readahead code. We simply read in a few pages whoes 710 * virtual addresses are around the fault address in the same vma. 711 * 712 * Caller must hold read mmap_sem if vmf->vma is not NULL. 713 * 714 */ 715 static struct page *swap_vma_readahead(swp_entry_t fentry, gfp_t gfp_mask, 716 struct vm_fault *vmf) 717 { 718 struct blk_plug plug; 719 struct vm_area_struct *vma = vmf->vma; 720 struct page *page; 721 pte_t *pte, pentry; 722 swp_entry_t entry; 723 unsigned int i; 724 bool page_allocated; 725 struct vma_swap_readahead ra_info = {0,}; 726 727 swap_ra_info(vmf, &ra_info); 728 if (ra_info.win == 1) 729 goto skip; 730 731 blk_start_plug(&plug); 732 for (i = 0, pte = ra_info.ptes; i < ra_info.nr_pte; 733 i++, pte++) { 734 pentry = *pte; 735 if (pte_none(pentry)) 736 continue; 737 if (pte_present(pentry)) 738 continue; 739 entry = pte_to_swp_entry(pentry); 740 if (unlikely(non_swap_entry(entry))) 741 continue; 742 page = __read_swap_cache_async(entry, gfp_mask, vma, 743 vmf->address, &page_allocated); 744 if (!page) 745 continue; 746 if (page_allocated) { 747 swap_readpage(page, false); 748 if (i != ra_info.offset) { 749 SetPageReadahead(page); 750 count_vm_event(SWAP_RA); 751 } 752 } 753 put_page(page); 754 } 755 blk_finish_plug(&plug); 756 lru_add_drain(); 757 skip: 758 return read_swap_cache_async(fentry, gfp_mask, vma, vmf->address, 759 ra_info.win == 1); 760 } 761 762 /** 763 * swapin_readahead - swap in pages in hope we need them soon 764 * @entry: swap entry of this memory 765 * @gfp_mask: memory allocation flags 766 * @vmf: fault information 767 * 768 * Returns the struct page for entry and addr, after queueing swapin. 769 * 770 * It's a main entry function for swap readahead. By the configuration, 771 * it will read ahead blocks by cluster-based(ie, physical disk based) 772 * or vma-based(ie, virtual address based on faulty address) readahead. 773 */ 774 struct page *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, 775 struct vm_fault *vmf) 776 { 777 return swap_use_vma_readahead() ? 778 swap_vma_readahead(entry, gfp_mask, vmf) : 779 swap_cluster_readahead(entry, gfp_mask, vmf); 780 } 781 782 #ifdef CONFIG_SYSFS 783 static ssize_t vma_ra_enabled_show(struct kobject *kobj, 784 struct kobj_attribute *attr, char *buf) 785 { 786 return sprintf(buf, "%s\n", enable_vma_readahead ? "true" : "false"); 787 } 788 static ssize_t vma_ra_enabled_store(struct kobject *kobj, 789 struct kobj_attribute *attr, 790 const char *buf, size_t count) 791 { 792 if (!strncmp(buf, "true", 4) || !strncmp(buf, "1", 1)) 793 enable_vma_readahead = true; 794 else if (!strncmp(buf, "false", 5) || !strncmp(buf, "0", 1)) 795 enable_vma_readahead = false; 796 else 797 return -EINVAL; 798 799 return count; 800 } 801 static struct kobj_attribute vma_ra_enabled_attr = 802 __ATTR(vma_ra_enabled, 0644, vma_ra_enabled_show, 803 vma_ra_enabled_store); 804 805 static struct attribute *swap_attrs[] = { 806 &vma_ra_enabled_attr.attr, 807 NULL, 808 }; 809 810 static struct attribute_group swap_attr_group = { 811 .attrs = swap_attrs, 812 }; 813 814 static int __init swap_init_sysfs(void) 815 { 816 int err; 817 struct kobject *swap_kobj; 818 819 swap_kobj = kobject_create_and_add("swap", mm_kobj); 820 if (!swap_kobj) { 821 pr_err("failed to create swap kobject\n"); 822 return -ENOMEM; 823 } 824 err = sysfs_create_group(swap_kobj, &swap_attr_group); 825 if (err) { 826 pr_err("failed to register swap group\n"); 827 goto delete_obj; 828 } 829 return 0; 830 831 delete_obj: 832 kobject_put(swap_kobj); 833 return err; 834 } 835 subsys_initcall(swap_init_sysfs); 836 #endif 837