1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/swap.c 4 * 5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 6 */ 7 8 /* 9 * This file contains the default values for the operation of the 10 * Linux VM subsystem. Fine-tuning documentation can be found in 11 * Documentation/admin-guide/sysctl/vm.rst. 12 * Started 18.12.91 13 * Swap aging added 23.2.95, Stephen Tweedie. 14 * Buffermem limits added 12.3.98, Rik van Riel. 15 */ 16 17 #include <linux/mm.h> 18 #include <linux/sched.h> 19 #include <linux/kernel_stat.h> 20 #include <linux/swap.h> 21 #include <linux/mman.h> 22 #include <linux/pagemap.h> 23 #include <linux/pagevec.h> 24 #include <linux/init.h> 25 #include <linux/export.h> 26 #include <linux/mm_inline.h> 27 #include <linux/percpu_counter.h> 28 #include <linux/memremap.h> 29 #include <linux/percpu.h> 30 #include <linux/cpu.h> 31 #include <linux/notifier.h> 32 #include <linux/backing-dev.h> 33 #include <linux/memcontrol.h> 34 #include <linux/gfp.h> 35 #include <linux/uio.h> 36 #include <linux/hugetlb.h> 37 #include <linux/page_idle.h> 38 #include <linux/local_lock.h> 39 40 #include "internal.h" 41 42 #define CREATE_TRACE_POINTS 43 #include <trace/events/pagemap.h> 44 45 /* How many pages do we try to swap or page in/out together? */ 46 int page_cluster; 47 48 /* Protecting only lru_rotate.pvec which requires disabling interrupts */ 49 struct lru_rotate { 50 local_lock_t lock; 51 struct pagevec pvec; 52 }; 53 static DEFINE_PER_CPU(struct lru_rotate, lru_rotate) = { 54 .lock = INIT_LOCAL_LOCK(lock), 55 }; 56 57 /* 58 * The following struct pagevec are grouped together because they are protected 59 * by disabling preemption (and interrupts remain enabled). 60 */ 61 struct lru_pvecs { 62 local_lock_t lock; 63 struct pagevec lru_add; 64 struct pagevec lru_deactivate_file; 65 struct pagevec lru_deactivate; 66 struct pagevec lru_lazyfree; 67 #ifdef CONFIG_SMP 68 struct pagevec activate_page; 69 #endif 70 }; 71 static DEFINE_PER_CPU(struct lru_pvecs, lru_pvecs) = { 72 .lock = INIT_LOCAL_LOCK(lock), 73 }; 74 75 /* 76 * This path almost never happens for VM activity - pages are normally 77 * freed via pagevecs. But it gets used by networking. 78 */ 79 static void __page_cache_release(struct page *page) 80 { 81 if (PageLRU(page)) { 82 pg_data_t *pgdat = page_pgdat(page); 83 struct lruvec *lruvec; 84 unsigned long flags; 85 86 spin_lock_irqsave(&pgdat->lru_lock, flags); 87 lruvec = mem_cgroup_page_lruvec(page, pgdat); 88 VM_BUG_ON_PAGE(!PageLRU(page), page); 89 __ClearPageLRU(page); 90 del_page_from_lru_list(page, lruvec, page_off_lru(page)); 91 spin_unlock_irqrestore(&pgdat->lru_lock, flags); 92 } 93 __ClearPageWaiters(page); 94 } 95 96 static void __put_single_page(struct page *page) 97 { 98 __page_cache_release(page); 99 mem_cgroup_uncharge(page); 100 free_unref_page(page); 101 } 102 103 static void __put_compound_page(struct page *page) 104 { 105 /* 106 * __page_cache_release() is supposed to be called for thp, not for 107 * hugetlb. This is because hugetlb page does never have PageLRU set 108 * (it's never listed to any LRU lists) and no memcg routines should 109 * be called for hugetlb (it has a separate hugetlb_cgroup.) 110 */ 111 if (!PageHuge(page)) 112 __page_cache_release(page); 113 destroy_compound_page(page); 114 } 115 116 void __put_page(struct page *page) 117 { 118 if (is_zone_device_page(page)) { 119 put_dev_pagemap(page->pgmap); 120 121 /* 122 * The page belongs to the device that created pgmap. Do 123 * not return it to page allocator. 124 */ 125 return; 126 } 127 128 if (unlikely(PageCompound(page))) 129 __put_compound_page(page); 130 else 131 __put_single_page(page); 132 } 133 EXPORT_SYMBOL(__put_page); 134 135 /** 136 * put_pages_list() - release a list of pages 137 * @pages: list of pages threaded on page->lru 138 * 139 * Release a list of pages which are strung together on page.lru. Currently 140 * used by read_cache_pages() and related error recovery code. 141 */ 142 void put_pages_list(struct list_head *pages) 143 { 144 while (!list_empty(pages)) { 145 struct page *victim; 146 147 victim = lru_to_page(pages); 148 list_del(&victim->lru); 149 put_page(victim); 150 } 151 } 152 EXPORT_SYMBOL(put_pages_list); 153 154 /* 155 * get_kernel_pages() - pin kernel pages in memory 156 * @kiov: An array of struct kvec structures 157 * @nr_segs: number of segments to pin 158 * @write: pinning for read/write, currently ignored 159 * @pages: array that receives pointers to the pages pinned. 160 * Should be at least nr_segs long. 161 * 162 * Returns number of pages pinned. This may be fewer than the number 163 * requested. If nr_pages is 0 or negative, returns 0. If no pages 164 * were pinned, returns -errno. Each page returned must be released 165 * with a put_page() call when it is finished with. 166 */ 167 int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write, 168 struct page **pages) 169 { 170 int seg; 171 172 for (seg = 0; seg < nr_segs; seg++) { 173 if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE)) 174 return seg; 175 176 pages[seg] = kmap_to_page(kiov[seg].iov_base); 177 get_page(pages[seg]); 178 } 179 180 return seg; 181 } 182 EXPORT_SYMBOL_GPL(get_kernel_pages); 183 184 /* 185 * get_kernel_page() - pin a kernel page in memory 186 * @start: starting kernel address 187 * @write: pinning for read/write, currently ignored 188 * @pages: array that receives pointer to the page pinned. 189 * Must be at least nr_segs long. 190 * 191 * Returns 1 if page is pinned. If the page was not pinned, returns 192 * -errno. The page returned must be released with a put_page() call 193 * when it is finished with. 194 */ 195 int get_kernel_page(unsigned long start, int write, struct page **pages) 196 { 197 const struct kvec kiov = { 198 .iov_base = (void *)start, 199 .iov_len = PAGE_SIZE 200 }; 201 202 return get_kernel_pages(&kiov, 1, write, pages); 203 } 204 EXPORT_SYMBOL_GPL(get_kernel_page); 205 206 static void pagevec_lru_move_fn(struct pagevec *pvec, 207 void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg), 208 void *arg) 209 { 210 int i; 211 struct pglist_data *pgdat = NULL; 212 struct lruvec *lruvec; 213 unsigned long flags = 0; 214 215 for (i = 0; i < pagevec_count(pvec); i++) { 216 struct page *page = pvec->pages[i]; 217 struct pglist_data *pagepgdat = page_pgdat(page); 218 219 if (pagepgdat != pgdat) { 220 if (pgdat) 221 spin_unlock_irqrestore(&pgdat->lru_lock, flags); 222 pgdat = pagepgdat; 223 spin_lock_irqsave(&pgdat->lru_lock, flags); 224 } 225 226 lruvec = mem_cgroup_page_lruvec(page, pgdat); 227 (*move_fn)(page, lruvec, arg); 228 } 229 if (pgdat) 230 spin_unlock_irqrestore(&pgdat->lru_lock, flags); 231 release_pages(pvec->pages, pvec->nr); 232 pagevec_reinit(pvec); 233 } 234 235 static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec, 236 void *arg) 237 { 238 int *pgmoved = arg; 239 240 if (PageLRU(page) && !PageUnevictable(page)) { 241 del_page_from_lru_list(page, lruvec, page_lru(page)); 242 ClearPageActive(page); 243 add_page_to_lru_list_tail(page, lruvec, page_lru(page)); 244 (*pgmoved) += thp_nr_pages(page); 245 } 246 } 247 248 /* 249 * pagevec_move_tail() must be called with IRQ disabled. 250 * Otherwise this may cause nasty races. 251 */ 252 static void pagevec_move_tail(struct pagevec *pvec) 253 { 254 int pgmoved = 0; 255 256 pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved); 257 __count_vm_events(PGROTATED, pgmoved); 258 } 259 260 /* 261 * Writeback is about to end against a page which has been marked for immediate 262 * reclaim. If it still appears to be reclaimable, move it to the tail of the 263 * inactive list. 264 */ 265 void rotate_reclaimable_page(struct page *page) 266 { 267 if (!PageLocked(page) && !PageDirty(page) && 268 !PageUnevictable(page) && PageLRU(page)) { 269 struct pagevec *pvec; 270 unsigned long flags; 271 272 get_page(page); 273 local_lock_irqsave(&lru_rotate.lock, flags); 274 pvec = this_cpu_ptr(&lru_rotate.pvec); 275 if (!pagevec_add(pvec, page) || PageCompound(page)) 276 pagevec_move_tail(pvec); 277 local_unlock_irqrestore(&lru_rotate.lock, flags); 278 } 279 } 280 281 void lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_pages) 282 { 283 do { 284 unsigned long lrusize; 285 286 /* Record cost event */ 287 if (file) 288 lruvec->file_cost += nr_pages; 289 else 290 lruvec->anon_cost += nr_pages; 291 292 /* 293 * Decay previous events 294 * 295 * Because workloads change over time (and to avoid 296 * overflow) we keep these statistics as a floating 297 * average, which ends up weighing recent refaults 298 * more than old ones. 299 */ 300 lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) + 301 lruvec_page_state(lruvec, NR_ACTIVE_ANON) + 302 lruvec_page_state(lruvec, NR_INACTIVE_FILE) + 303 lruvec_page_state(lruvec, NR_ACTIVE_FILE); 304 305 if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) { 306 lruvec->file_cost /= 2; 307 lruvec->anon_cost /= 2; 308 } 309 } while ((lruvec = parent_lruvec(lruvec))); 310 } 311 312 void lru_note_cost_page(struct page *page) 313 { 314 lru_note_cost(mem_cgroup_page_lruvec(page, page_pgdat(page)), 315 page_is_file_lru(page), thp_nr_pages(page)); 316 } 317 318 static void __activate_page(struct page *page, struct lruvec *lruvec, 319 void *arg) 320 { 321 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 322 int lru = page_lru_base_type(page); 323 int nr_pages = thp_nr_pages(page); 324 325 del_page_from_lru_list(page, lruvec, lru); 326 SetPageActive(page); 327 lru += LRU_ACTIVE; 328 add_page_to_lru_list(page, lruvec, lru); 329 trace_mm_lru_activate(page); 330 331 __count_vm_events(PGACTIVATE, nr_pages); 332 __count_memcg_events(lruvec_memcg(lruvec), PGACTIVATE, 333 nr_pages); 334 } 335 } 336 337 #ifdef CONFIG_SMP 338 static void activate_page_drain(int cpu) 339 { 340 struct pagevec *pvec = &per_cpu(lru_pvecs.activate_page, cpu); 341 342 if (pagevec_count(pvec)) 343 pagevec_lru_move_fn(pvec, __activate_page, NULL); 344 } 345 346 static bool need_activate_page_drain(int cpu) 347 { 348 return pagevec_count(&per_cpu(lru_pvecs.activate_page, cpu)) != 0; 349 } 350 351 void activate_page(struct page *page) 352 { 353 page = compound_head(page); 354 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 355 struct pagevec *pvec; 356 357 local_lock(&lru_pvecs.lock); 358 pvec = this_cpu_ptr(&lru_pvecs.activate_page); 359 get_page(page); 360 if (!pagevec_add(pvec, page) || PageCompound(page)) 361 pagevec_lru_move_fn(pvec, __activate_page, NULL); 362 local_unlock(&lru_pvecs.lock); 363 } 364 } 365 366 #else 367 static inline void activate_page_drain(int cpu) 368 { 369 } 370 371 void activate_page(struct page *page) 372 { 373 pg_data_t *pgdat = page_pgdat(page); 374 375 page = compound_head(page); 376 spin_lock_irq(&pgdat->lru_lock); 377 __activate_page(page, mem_cgroup_page_lruvec(page, pgdat), NULL); 378 spin_unlock_irq(&pgdat->lru_lock); 379 } 380 #endif 381 382 static void __lru_cache_activate_page(struct page *page) 383 { 384 struct pagevec *pvec; 385 int i; 386 387 local_lock(&lru_pvecs.lock); 388 pvec = this_cpu_ptr(&lru_pvecs.lru_add); 389 390 /* 391 * Search backwards on the optimistic assumption that the page being 392 * activated has just been added to this pagevec. Note that only 393 * the local pagevec is examined as a !PageLRU page could be in the 394 * process of being released, reclaimed, migrated or on a remote 395 * pagevec that is currently being drained. Furthermore, marking 396 * a remote pagevec's page PageActive potentially hits a race where 397 * a page is marked PageActive just after it is added to the inactive 398 * list causing accounting errors and BUG_ON checks to trigger. 399 */ 400 for (i = pagevec_count(pvec) - 1; i >= 0; i--) { 401 struct page *pagevec_page = pvec->pages[i]; 402 403 if (pagevec_page == page) { 404 SetPageActive(page); 405 break; 406 } 407 } 408 409 local_unlock(&lru_pvecs.lock); 410 } 411 412 /* 413 * Mark a page as having seen activity. 414 * 415 * inactive,unreferenced -> inactive,referenced 416 * inactive,referenced -> active,unreferenced 417 * active,unreferenced -> active,referenced 418 * 419 * When a newly allocated page is not yet visible, so safe for non-atomic ops, 420 * __SetPageReferenced(page) may be substituted for mark_page_accessed(page). 421 */ 422 void mark_page_accessed(struct page *page) 423 { 424 page = compound_head(page); 425 426 if (!PageReferenced(page)) { 427 SetPageReferenced(page); 428 } else if (PageUnevictable(page)) { 429 /* 430 * Unevictable pages are on the "LRU_UNEVICTABLE" list. But, 431 * this list is never rotated or maintained, so marking an 432 * evictable page accessed has no effect. 433 */ 434 } else if (!PageActive(page)) { 435 /* 436 * If the page is on the LRU, queue it for activation via 437 * lru_pvecs.activate_page. Otherwise, assume the page is on a 438 * pagevec, mark it active and it'll be moved to the active 439 * LRU on the next drain. 440 */ 441 if (PageLRU(page)) 442 activate_page(page); 443 else 444 __lru_cache_activate_page(page); 445 ClearPageReferenced(page); 446 workingset_activation(page); 447 } 448 if (page_is_idle(page)) 449 clear_page_idle(page); 450 } 451 EXPORT_SYMBOL(mark_page_accessed); 452 453 /** 454 * lru_cache_add - add a page to a page list 455 * @page: the page to be added to the LRU. 456 * 457 * Queue the page for addition to the LRU via pagevec. The decision on whether 458 * to add the page to the [in]active [file|anon] list is deferred until the 459 * pagevec is drained. This gives a chance for the caller of lru_cache_add() 460 * have the page added to the active list using mark_page_accessed(). 461 */ 462 void lru_cache_add(struct page *page) 463 { 464 struct pagevec *pvec; 465 466 VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); 467 VM_BUG_ON_PAGE(PageLRU(page), page); 468 469 get_page(page); 470 local_lock(&lru_pvecs.lock); 471 pvec = this_cpu_ptr(&lru_pvecs.lru_add); 472 if (!pagevec_add(pvec, page) || PageCompound(page)) 473 __pagevec_lru_add(pvec); 474 local_unlock(&lru_pvecs.lock); 475 } 476 EXPORT_SYMBOL(lru_cache_add); 477 478 /** 479 * lru_cache_add_inactive_or_unevictable 480 * @page: the page to be added to LRU 481 * @vma: vma in which page is mapped for determining reclaimability 482 * 483 * Place @page on the inactive or unevictable LRU list, depending on its 484 * evictability. Note that if the page is not evictable, it goes 485 * directly back onto it's zone's unevictable list, it does NOT use a 486 * per cpu pagevec. 487 */ 488 void lru_cache_add_inactive_or_unevictable(struct page *page, 489 struct vm_area_struct *vma) 490 { 491 bool unevictable; 492 493 VM_BUG_ON_PAGE(PageLRU(page), page); 494 495 unevictable = (vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED; 496 if (unlikely(unevictable) && !TestSetPageMlocked(page)) { 497 int nr_pages = thp_nr_pages(page); 498 /* 499 * We use the irq-unsafe __mod_zone_page_stat because this 500 * counter is not modified from interrupt context, and the pte 501 * lock is held(spinlock), which implies preemption disabled. 502 */ 503 __mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages); 504 count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages); 505 } 506 lru_cache_add(page); 507 } 508 509 /* 510 * If the page can not be invalidated, it is moved to the 511 * inactive list to speed up its reclaim. It is moved to the 512 * head of the list, rather than the tail, to give the flusher 513 * threads some time to write it out, as this is much more 514 * effective than the single-page writeout from reclaim. 515 * 516 * If the page isn't page_mapped and dirty/writeback, the page 517 * could reclaim asap using PG_reclaim. 518 * 519 * 1. active, mapped page -> none 520 * 2. active, dirty/writeback page -> inactive, head, PG_reclaim 521 * 3. inactive, mapped page -> none 522 * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim 523 * 5. inactive, clean -> inactive, tail 524 * 6. Others -> none 525 * 526 * In 4, why it moves inactive's head, the VM expects the page would 527 * be write it out by flusher threads as this is much more effective 528 * than the single-page writeout from reclaim. 529 */ 530 static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, 531 void *arg) 532 { 533 int lru; 534 bool active; 535 int nr_pages = thp_nr_pages(page); 536 537 if (!PageLRU(page)) 538 return; 539 540 if (PageUnevictable(page)) 541 return; 542 543 /* Some processes are using the page */ 544 if (page_mapped(page)) 545 return; 546 547 active = PageActive(page); 548 lru = page_lru_base_type(page); 549 550 del_page_from_lru_list(page, lruvec, lru + active); 551 ClearPageActive(page); 552 ClearPageReferenced(page); 553 554 if (PageWriteback(page) || PageDirty(page)) { 555 /* 556 * PG_reclaim could be raced with end_page_writeback 557 * It can make readahead confusing. But race window 558 * is _really_ small and it's non-critical problem. 559 */ 560 add_page_to_lru_list(page, lruvec, lru); 561 SetPageReclaim(page); 562 } else { 563 /* 564 * The page's writeback ends up during pagevec 565 * We moves tha page into tail of inactive. 566 */ 567 add_page_to_lru_list_tail(page, lruvec, lru); 568 __count_vm_events(PGROTATED, nr_pages); 569 } 570 571 if (active) { 572 __count_vm_events(PGDEACTIVATE, nr_pages); 573 __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, 574 nr_pages); 575 } 576 } 577 578 static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec, 579 void *arg) 580 { 581 if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { 582 int lru = page_lru_base_type(page); 583 int nr_pages = thp_nr_pages(page); 584 585 del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE); 586 ClearPageActive(page); 587 ClearPageReferenced(page); 588 add_page_to_lru_list(page, lruvec, lru); 589 590 __count_vm_events(PGDEACTIVATE, nr_pages); 591 __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, 592 nr_pages); 593 } 594 } 595 596 static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec, 597 void *arg) 598 { 599 if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && 600 !PageSwapCache(page) && !PageUnevictable(page)) { 601 bool active = PageActive(page); 602 int nr_pages = thp_nr_pages(page); 603 604 del_page_from_lru_list(page, lruvec, 605 LRU_INACTIVE_ANON + active); 606 ClearPageActive(page); 607 ClearPageReferenced(page); 608 /* 609 * Lazyfree pages are clean anonymous pages. They have 610 * PG_swapbacked flag cleared, to distinguish them from normal 611 * anonymous pages 612 */ 613 ClearPageSwapBacked(page); 614 add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE); 615 616 __count_vm_events(PGLAZYFREE, nr_pages); 617 __count_memcg_events(lruvec_memcg(lruvec), PGLAZYFREE, 618 nr_pages); 619 } 620 } 621 622 /* 623 * Drain pages out of the cpu's pagevecs. 624 * Either "cpu" is the current CPU, and preemption has already been 625 * disabled; or "cpu" is being hot-unplugged, and is already dead. 626 */ 627 void lru_add_drain_cpu(int cpu) 628 { 629 struct pagevec *pvec = &per_cpu(lru_pvecs.lru_add, cpu); 630 631 if (pagevec_count(pvec)) 632 __pagevec_lru_add(pvec); 633 634 pvec = &per_cpu(lru_rotate.pvec, cpu); 635 /* Disabling interrupts below acts as a compiler barrier. */ 636 if (data_race(pagevec_count(pvec))) { 637 unsigned long flags; 638 639 /* No harm done if a racing interrupt already did this */ 640 local_lock_irqsave(&lru_rotate.lock, flags); 641 pagevec_move_tail(pvec); 642 local_unlock_irqrestore(&lru_rotate.lock, flags); 643 } 644 645 pvec = &per_cpu(lru_pvecs.lru_deactivate_file, cpu); 646 if (pagevec_count(pvec)) 647 pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); 648 649 pvec = &per_cpu(lru_pvecs.lru_deactivate, cpu); 650 if (pagevec_count(pvec)) 651 pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); 652 653 pvec = &per_cpu(lru_pvecs.lru_lazyfree, cpu); 654 if (pagevec_count(pvec)) 655 pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL); 656 657 activate_page_drain(cpu); 658 } 659 660 /** 661 * deactivate_file_page - forcefully deactivate a file page 662 * @page: page to deactivate 663 * 664 * This function hints the VM that @page is a good reclaim candidate, 665 * for example if its invalidation fails due to the page being dirty 666 * or under writeback. 667 */ 668 void deactivate_file_page(struct page *page) 669 { 670 /* 671 * In a workload with many unevictable page such as mprotect, 672 * unevictable page deactivation for accelerating reclaim is pointless. 673 */ 674 if (PageUnevictable(page)) 675 return; 676 677 if (likely(get_page_unless_zero(page))) { 678 struct pagevec *pvec; 679 680 local_lock(&lru_pvecs.lock); 681 pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate_file); 682 683 if (!pagevec_add(pvec, page) || PageCompound(page)) 684 pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); 685 local_unlock(&lru_pvecs.lock); 686 } 687 } 688 689 /* 690 * deactivate_page - deactivate a page 691 * @page: page to deactivate 692 * 693 * deactivate_page() moves @page to the inactive list if @page was on the active 694 * list and was not an unevictable page. This is done to accelerate the reclaim 695 * of @page. 696 */ 697 void deactivate_page(struct page *page) 698 { 699 if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { 700 struct pagevec *pvec; 701 702 local_lock(&lru_pvecs.lock); 703 pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate); 704 get_page(page); 705 if (!pagevec_add(pvec, page) || PageCompound(page)) 706 pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); 707 local_unlock(&lru_pvecs.lock); 708 } 709 } 710 711 /** 712 * mark_page_lazyfree - make an anon page lazyfree 713 * @page: page to deactivate 714 * 715 * mark_page_lazyfree() moves @page to the inactive file list. 716 * This is done to accelerate the reclaim of @page. 717 */ 718 void mark_page_lazyfree(struct page *page) 719 { 720 if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && 721 !PageSwapCache(page) && !PageUnevictable(page)) { 722 struct pagevec *pvec; 723 724 local_lock(&lru_pvecs.lock); 725 pvec = this_cpu_ptr(&lru_pvecs.lru_lazyfree); 726 get_page(page); 727 if (!pagevec_add(pvec, page) || PageCompound(page)) 728 pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL); 729 local_unlock(&lru_pvecs.lock); 730 } 731 } 732 733 void lru_add_drain(void) 734 { 735 local_lock(&lru_pvecs.lock); 736 lru_add_drain_cpu(smp_processor_id()); 737 local_unlock(&lru_pvecs.lock); 738 } 739 740 void lru_add_drain_cpu_zone(struct zone *zone) 741 { 742 local_lock(&lru_pvecs.lock); 743 lru_add_drain_cpu(smp_processor_id()); 744 drain_local_pages(zone); 745 local_unlock(&lru_pvecs.lock); 746 } 747 748 #ifdef CONFIG_SMP 749 750 static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); 751 752 static void lru_add_drain_per_cpu(struct work_struct *dummy) 753 { 754 lru_add_drain(); 755 } 756 757 /* 758 * Doesn't need any cpu hotplug locking because we do rely on per-cpu 759 * kworkers being shut down before our page_alloc_cpu_dead callback is 760 * executed on the offlined cpu. 761 * Calling this function with cpu hotplug locks held can actually lead 762 * to obscure indirect dependencies via WQ context. 763 */ 764 void lru_add_drain_all(void) 765 { 766 static seqcount_t seqcount = SEQCNT_ZERO(seqcount); 767 static DEFINE_MUTEX(lock); 768 static struct cpumask has_work; 769 int cpu, seq; 770 771 /* 772 * Make sure nobody triggers this path before mm_percpu_wq is fully 773 * initialized. 774 */ 775 if (WARN_ON(!mm_percpu_wq)) 776 return; 777 778 seq = raw_read_seqcount_latch(&seqcount); 779 780 mutex_lock(&lock); 781 782 /* 783 * Piggyback on drain started and finished while we waited for lock: 784 * all pages pended at the time of our enter were drained from vectors. 785 */ 786 if (__read_seqcount_retry(&seqcount, seq)) 787 goto done; 788 789 raw_write_seqcount_latch(&seqcount); 790 791 cpumask_clear(&has_work); 792 793 for_each_online_cpu(cpu) { 794 struct work_struct *work = &per_cpu(lru_add_drain_work, cpu); 795 796 if (pagevec_count(&per_cpu(lru_pvecs.lru_add, cpu)) || 797 data_race(pagevec_count(&per_cpu(lru_rotate.pvec, cpu))) || 798 pagevec_count(&per_cpu(lru_pvecs.lru_deactivate_file, cpu)) || 799 pagevec_count(&per_cpu(lru_pvecs.lru_deactivate, cpu)) || 800 pagevec_count(&per_cpu(lru_pvecs.lru_lazyfree, cpu)) || 801 need_activate_page_drain(cpu)) { 802 INIT_WORK(work, lru_add_drain_per_cpu); 803 queue_work_on(cpu, mm_percpu_wq, work); 804 cpumask_set_cpu(cpu, &has_work); 805 } 806 } 807 808 for_each_cpu(cpu, &has_work) 809 flush_work(&per_cpu(lru_add_drain_work, cpu)); 810 811 done: 812 mutex_unlock(&lock); 813 } 814 #else 815 void lru_add_drain_all(void) 816 { 817 lru_add_drain(); 818 } 819 #endif 820 821 /** 822 * release_pages - batched put_page() 823 * @pages: array of pages to release 824 * @nr: number of pages 825 * 826 * Decrement the reference count on all the pages in @pages. If it 827 * fell to zero, remove the page from the LRU and free it. 828 */ 829 void release_pages(struct page **pages, int nr) 830 { 831 int i; 832 LIST_HEAD(pages_to_free); 833 struct pglist_data *locked_pgdat = NULL; 834 struct lruvec *lruvec; 835 unsigned long flags; 836 unsigned int lock_batch; 837 838 for (i = 0; i < nr; i++) { 839 struct page *page = pages[i]; 840 841 /* 842 * Make sure the IRQ-safe lock-holding time does not get 843 * excessive with a continuous string of pages from the 844 * same pgdat. The lock is held only if pgdat != NULL. 845 */ 846 if (locked_pgdat && ++lock_batch == SWAP_CLUSTER_MAX) { 847 spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); 848 locked_pgdat = NULL; 849 } 850 851 if (is_huge_zero_page(page)) 852 continue; 853 854 if (is_zone_device_page(page)) { 855 if (locked_pgdat) { 856 spin_unlock_irqrestore(&locked_pgdat->lru_lock, 857 flags); 858 locked_pgdat = NULL; 859 } 860 /* 861 * ZONE_DEVICE pages that return 'false' from 862 * put_devmap_managed_page() do not require special 863 * processing, and instead, expect a call to 864 * put_page_testzero(). 865 */ 866 if (page_is_devmap_managed(page)) { 867 put_devmap_managed_page(page); 868 continue; 869 } 870 } 871 872 page = compound_head(page); 873 if (!put_page_testzero(page)) 874 continue; 875 876 if (PageCompound(page)) { 877 if (locked_pgdat) { 878 spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); 879 locked_pgdat = NULL; 880 } 881 __put_compound_page(page); 882 continue; 883 } 884 885 if (PageLRU(page)) { 886 struct pglist_data *pgdat = page_pgdat(page); 887 888 if (pgdat != locked_pgdat) { 889 if (locked_pgdat) 890 spin_unlock_irqrestore(&locked_pgdat->lru_lock, 891 flags); 892 lock_batch = 0; 893 locked_pgdat = pgdat; 894 spin_lock_irqsave(&locked_pgdat->lru_lock, flags); 895 } 896 897 lruvec = mem_cgroup_page_lruvec(page, locked_pgdat); 898 VM_BUG_ON_PAGE(!PageLRU(page), page); 899 __ClearPageLRU(page); 900 del_page_from_lru_list(page, lruvec, page_off_lru(page)); 901 } 902 903 /* Clear Active bit in case of parallel mark_page_accessed */ 904 __ClearPageActive(page); 905 __ClearPageWaiters(page); 906 907 list_add(&page->lru, &pages_to_free); 908 } 909 if (locked_pgdat) 910 spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); 911 912 mem_cgroup_uncharge_list(&pages_to_free); 913 free_unref_page_list(&pages_to_free); 914 } 915 EXPORT_SYMBOL(release_pages); 916 917 /* 918 * The pages which we're about to release may be in the deferred lru-addition 919 * queues. That would prevent them from really being freed right now. That's 920 * OK from a correctness point of view but is inefficient - those pages may be 921 * cache-warm and we want to give them back to the page allocator ASAP. 922 * 923 * So __pagevec_release() will drain those queues here. __pagevec_lru_add() 924 * and __pagevec_lru_add_active() call release_pages() directly to avoid 925 * mutual recursion. 926 */ 927 void __pagevec_release(struct pagevec *pvec) 928 { 929 if (!pvec->percpu_pvec_drained) { 930 lru_add_drain(); 931 pvec->percpu_pvec_drained = true; 932 } 933 release_pages(pvec->pages, pagevec_count(pvec)); 934 pagevec_reinit(pvec); 935 } 936 EXPORT_SYMBOL(__pagevec_release); 937 938 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 939 /* used by __split_huge_page_refcount() */ 940 void lru_add_page_tail(struct page *page, struct page *page_tail, 941 struct lruvec *lruvec, struct list_head *list) 942 { 943 VM_BUG_ON_PAGE(!PageHead(page), page); 944 VM_BUG_ON_PAGE(PageCompound(page_tail), page); 945 VM_BUG_ON_PAGE(PageLRU(page_tail), page); 946 lockdep_assert_held(&lruvec_pgdat(lruvec)->lru_lock); 947 948 if (!list) 949 SetPageLRU(page_tail); 950 951 if (likely(PageLRU(page))) 952 list_add_tail(&page_tail->lru, &page->lru); 953 else if (list) { 954 /* page reclaim is reclaiming a huge page */ 955 get_page(page_tail); 956 list_add_tail(&page_tail->lru, list); 957 } else { 958 /* 959 * Head page has not yet been counted, as an hpage, 960 * so we must account for each subpage individually. 961 * 962 * Put page_tail on the list at the correct position 963 * so they all end up in order. 964 */ 965 add_page_to_lru_list_tail(page_tail, lruvec, 966 page_lru(page_tail)); 967 } 968 } 969 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 970 971 static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, 972 void *arg) 973 { 974 enum lru_list lru; 975 int was_unevictable = TestClearPageUnevictable(page); 976 int nr_pages = thp_nr_pages(page); 977 978 VM_BUG_ON_PAGE(PageLRU(page), page); 979 980 /* 981 * Page becomes evictable in two ways: 982 * 1) Within LRU lock [munlock_vma_page() and __munlock_pagevec()]. 983 * 2) Before acquiring LRU lock to put the page to correct LRU and then 984 * a) do PageLRU check with lock [check_move_unevictable_pages] 985 * b) do PageLRU check before lock [clear_page_mlock] 986 * 987 * (1) & (2a) are ok as LRU lock will serialize them. For (2b), we need 988 * following strict ordering: 989 * 990 * #0: __pagevec_lru_add_fn #1: clear_page_mlock 991 * 992 * SetPageLRU() TestClearPageMlocked() 993 * smp_mb() // explicit ordering // above provides strict 994 * // ordering 995 * PageMlocked() PageLRU() 996 * 997 * 998 * if '#1' does not observe setting of PG_lru by '#0' and fails 999 * isolation, the explicit barrier will make sure that page_evictable 1000 * check will put the page in correct LRU. Without smp_mb(), SetPageLRU 1001 * can be reordered after PageMlocked check and can make '#1' to fail 1002 * the isolation of the page whose Mlocked bit is cleared (#0 is also 1003 * looking at the same page) and the evictable page will be stranded 1004 * in an unevictable LRU. 1005 */ 1006 SetPageLRU(page); 1007 smp_mb__after_atomic(); 1008 1009 if (page_evictable(page)) { 1010 lru = page_lru(page); 1011 if (was_unevictable) 1012 __count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages); 1013 } else { 1014 lru = LRU_UNEVICTABLE; 1015 ClearPageActive(page); 1016 SetPageUnevictable(page); 1017 if (!was_unevictable) 1018 __count_vm_events(UNEVICTABLE_PGCULLED, nr_pages); 1019 } 1020 1021 add_page_to_lru_list(page, lruvec, lru); 1022 trace_mm_lru_insertion(page, lru); 1023 } 1024 1025 /* 1026 * Add the passed pages to the LRU, then drop the caller's refcount 1027 * on them. Reinitialises the caller's pagevec. 1028 */ 1029 void __pagevec_lru_add(struct pagevec *pvec) 1030 { 1031 pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL); 1032 } 1033 1034 /** 1035 * pagevec_lookup_entries - gang pagecache lookup 1036 * @pvec: Where the resulting entries are placed 1037 * @mapping: The address_space to search 1038 * @start: The starting entry index 1039 * @nr_entries: The maximum number of pages 1040 * @indices: The cache indices corresponding to the entries in @pvec 1041 * 1042 * pagevec_lookup_entries() will search for and return a group of up 1043 * to @nr_pages pages and shadow entries in the mapping. All 1044 * entries are placed in @pvec. pagevec_lookup_entries() takes a 1045 * reference against actual pages in @pvec. 1046 * 1047 * The search returns a group of mapping-contiguous entries with 1048 * ascending indexes. There may be holes in the indices due to 1049 * not-present entries. 1050 * 1051 * Only one subpage of a Transparent Huge Page is returned in one call: 1052 * allowing truncate_inode_pages_range() to evict the whole THP without 1053 * cycling through a pagevec of extra references. 1054 * 1055 * pagevec_lookup_entries() returns the number of entries which were 1056 * found. 1057 */ 1058 unsigned pagevec_lookup_entries(struct pagevec *pvec, 1059 struct address_space *mapping, 1060 pgoff_t start, unsigned nr_entries, 1061 pgoff_t *indices) 1062 { 1063 pvec->nr = find_get_entries(mapping, start, nr_entries, 1064 pvec->pages, indices); 1065 return pagevec_count(pvec); 1066 } 1067 1068 /** 1069 * pagevec_remove_exceptionals - pagevec exceptionals pruning 1070 * @pvec: The pagevec to prune 1071 * 1072 * pagevec_lookup_entries() fills both pages and exceptional radix 1073 * tree entries into the pagevec. This function prunes all 1074 * exceptionals from @pvec without leaving holes, so that it can be 1075 * passed on to page-only pagevec operations. 1076 */ 1077 void pagevec_remove_exceptionals(struct pagevec *pvec) 1078 { 1079 int i, j; 1080 1081 for (i = 0, j = 0; i < pagevec_count(pvec); i++) { 1082 struct page *page = pvec->pages[i]; 1083 if (!xa_is_value(page)) 1084 pvec->pages[j++] = page; 1085 } 1086 pvec->nr = j; 1087 } 1088 1089 /** 1090 * pagevec_lookup_range - gang pagecache lookup 1091 * @pvec: Where the resulting pages are placed 1092 * @mapping: The address_space to search 1093 * @start: The starting page index 1094 * @end: The final page index 1095 * 1096 * pagevec_lookup_range() will search for & return a group of up to PAGEVEC_SIZE 1097 * pages in the mapping starting from index @start and upto index @end 1098 * (inclusive). The pages are placed in @pvec. pagevec_lookup() takes a 1099 * reference against the pages in @pvec. 1100 * 1101 * The search returns a group of mapping-contiguous pages with ascending 1102 * indexes. There may be holes in the indices due to not-present pages. We 1103 * also update @start to index the next page for the traversal. 1104 * 1105 * pagevec_lookup_range() returns the number of pages which were found. If this 1106 * number is smaller than PAGEVEC_SIZE, the end of specified range has been 1107 * reached. 1108 */ 1109 unsigned pagevec_lookup_range(struct pagevec *pvec, 1110 struct address_space *mapping, pgoff_t *start, pgoff_t end) 1111 { 1112 pvec->nr = find_get_pages_range(mapping, start, end, PAGEVEC_SIZE, 1113 pvec->pages); 1114 return pagevec_count(pvec); 1115 } 1116 EXPORT_SYMBOL(pagevec_lookup_range); 1117 1118 unsigned pagevec_lookup_range_tag(struct pagevec *pvec, 1119 struct address_space *mapping, pgoff_t *index, pgoff_t end, 1120 xa_mark_t tag) 1121 { 1122 pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, 1123 PAGEVEC_SIZE, pvec->pages); 1124 return pagevec_count(pvec); 1125 } 1126 EXPORT_SYMBOL(pagevec_lookup_range_tag); 1127 1128 unsigned pagevec_lookup_range_nr_tag(struct pagevec *pvec, 1129 struct address_space *mapping, pgoff_t *index, pgoff_t end, 1130 xa_mark_t tag, unsigned max_pages) 1131 { 1132 pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, 1133 min_t(unsigned int, max_pages, PAGEVEC_SIZE), pvec->pages); 1134 return pagevec_count(pvec); 1135 } 1136 EXPORT_SYMBOL(pagevec_lookup_range_nr_tag); 1137 /* 1138 * Perform any setup for the swap system 1139 */ 1140 void __init swap_setup(void) 1141 { 1142 unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT); 1143 1144 /* Use a smaller cluster for small-memory machines */ 1145 if (megs < 16) 1146 page_cluster = 2; 1147 else 1148 page_cluster = 3; 1149 /* 1150 * Right now other parts of the system means that we 1151 * _really_ don't want to cluster much more 1152 */ 1153 } 1154 1155 #ifdef CONFIG_DEV_PAGEMAP_OPS 1156 void put_devmap_managed_page(struct page *page) 1157 { 1158 int count; 1159 1160 if (WARN_ON_ONCE(!page_is_devmap_managed(page))) 1161 return; 1162 1163 count = page_ref_dec_return(page); 1164 1165 /* 1166 * devmap page refcounts are 1-based, rather than 0-based: if 1167 * refcount is 1, then the page is free and the refcount is 1168 * stable because nobody holds a reference on the page. 1169 */ 1170 if (count == 1) 1171 free_devmap_managed_page(page); 1172 else if (!count) 1173 __put_page(page); 1174 } 1175 EXPORT_SYMBOL(put_devmap_managed_page); 1176 #endif 1177