1 /* 2 * linux/mm/swap.c 3 * 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 5 */ 6 7 /* 8 * This file contains the default values for the operation of the 9 * Linux VM subsystem. Fine-tuning documentation can be found in 10 * Documentation/sysctl/vm.txt. 11 * Started 18.12.91 12 * Swap aging added 23.2.95, Stephen Tweedie. 13 * Buffermem limits added 12.3.98, Rik van Riel. 14 */ 15 16 #include <linux/mm.h> 17 #include <linux/sched.h> 18 #include <linux/kernel_stat.h> 19 #include <linux/swap.h> 20 #include <linux/mman.h> 21 #include <linux/pagemap.h> 22 #include <linux/pagevec.h> 23 #include <linux/init.h> 24 #include <linux/export.h> 25 #include <linux/mm_inline.h> 26 #include <linux/percpu_counter.h> 27 #include <linux/percpu.h> 28 #include <linux/cpu.h> 29 #include <linux/notifier.h> 30 #include <linux/backing-dev.h> 31 #include <linux/memcontrol.h> 32 #include <linux/gfp.h> 33 #include <linux/uio.h> 34 35 #include "internal.h" 36 37 #define CREATE_TRACE_POINTS 38 #include <trace/events/pagemap.h> 39 40 /* How many pages do we try to swap or page in/out together? */ 41 int page_cluster; 42 43 static DEFINE_PER_CPU(struct pagevec, lru_add_pvec); 44 static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs); 45 static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs); 46 47 /* 48 * This path almost never happens for VM activity - pages are normally 49 * freed via pagevecs. But it gets used by networking. 50 */ 51 static void __page_cache_release(struct page *page) 52 { 53 if (PageLRU(page)) { 54 struct zone *zone = page_zone(page); 55 struct lruvec *lruvec; 56 unsigned long flags; 57 58 spin_lock_irqsave(&zone->lru_lock, flags); 59 lruvec = mem_cgroup_page_lruvec(page, zone); 60 VM_BUG_ON_PAGE(!PageLRU(page), page); 61 __ClearPageLRU(page); 62 del_page_from_lru_list(page, lruvec, page_off_lru(page)); 63 spin_unlock_irqrestore(&zone->lru_lock, flags); 64 } 65 mem_cgroup_uncharge(page); 66 } 67 68 static void __put_single_page(struct page *page) 69 { 70 __page_cache_release(page); 71 free_hot_cold_page(page, false); 72 } 73 74 static void __put_compound_page(struct page *page) 75 { 76 compound_page_dtor *dtor; 77 78 __page_cache_release(page); 79 dtor = get_compound_page_dtor(page); 80 (*dtor)(page); 81 } 82 83 /** 84 * Two special cases here: we could avoid taking compound_lock_irqsave 85 * and could skip the tail refcounting(in _mapcount). 86 * 87 * 1. Hugetlbfs page: 88 * 89 * PageHeadHuge will remain true until the compound page 90 * is released and enters the buddy allocator, and it could 91 * not be split by __split_huge_page_refcount(). 92 * 93 * So if we see PageHeadHuge set, and we have the tail page pin, 94 * then we could safely put head page. 95 * 96 * 2. Slab THP page: 97 * 98 * PG_slab is cleared before the slab frees the head page, and 99 * tail pin cannot be the last reference left on the head page, 100 * because the slab code is free to reuse the compound page 101 * after a kfree/kmem_cache_free without having to check if 102 * there's any tail pin left. In turn all tail pinsmust be always 103 * released while the head is still pinned by the slab code 104 * and so we know PG_slab will be still set too. 105 * 106 * So if we see PageSlab set, and we have the tail page pin, 107 * then we could safely put head page. 108 */ 109 static __always_inline 110 void put_unrefcounted_compound_page(struct page *page_head, struct page *page) 111 { 112 /* 113 * If @page is a THP tail, we must read the tail page 114 * flags after the head page flags. The 115 * __split_huge_page_refcount side enforces write memory barriers 116 * between clearing PageTail and before the head page 117 * can be freed and reallocated. 118 */ 119 smp_rmb(); 120 if (likely(PageTail(page))) { 121 /* 122 * __split_huge_page_refcount cannot race 123 * here, see the comment above this function. 124 */ 125 VM_BUG_ON_PAGE(!PageHead(page_head), page_head); 126 VM_BUG_ON_PAGE(page_mapcount(page) != 0, page); 127 if (put_page_testzero(page_head)) { 128 /* 129 * If this is the tail of a slab THP page, 130 * the tail pin must not be the last reference 131 * held on the page, because the PG_slab cannot 132 * be cleared before all tail pins (which skips 133 * the _mapcount tail refcounting) have been 134 * released. 135 * 136 * If this is the tail of a hugetlbfs page, 137 * the tail pin may be the last reference on 138 * the page instead, because PageHeadHuge will 139 * not go away until the compound page enters 140 * the buddy allocator. 141 */ 142 VM_BUG_ON_PAGE(PageSlab(page_head), page_head); 143 __put_compound_page(page_head); 144 } 145 } else 146 /* 147 * __split_huge_page_refcount run before us, 148 * @page was a THP tail. The split @page_head 149 * has been freed and reallocated as slab or 150 * hugetlbfs page of smaller order (only 151 * possible if reallocated as slab on x86). 152 */ 153 if (put_page_testzero(page)) 154 __put_single_page(page); 155 } 156 157 static __always_inline 158 void put_refcounted_compound_page(struct page *page_head, struct page *page) 159 { 160 if (likely(page != page_head && get_page_unless_zero(page_head))) { 161 unsigned long flags; 162 163 /* 164 * @page_head wasn't a dangling pointer but it may not 165 * be a head page anymore by the time we obtain the 166 * lock. That is ok as long as it can't be freed from 167 * under us. 168 */ 169 flags = compound_lock_irqsave(page_head); 170 if (unlikely(!PageTail(page))) { 171 /* __split_huge_page_refcount run before us */ 172 compound_unlock_irqrestore(page_head, flags); 173 if (put_page_testzero(page_head)) { 174 /* 175 * The @page_head may have been freed 176 * and reallocated as a compound page 177 * of smaller order and then freed 178 * again. All we know is that it 179 * cannot have become: a THP page, a 180 * compound page of higher order, a 181 * tail page. That is because we 182 * still hold the refcount of the 183 * split THP tail and page_head was 184 * the THP head before the split. 185 */ 186 if (PageHead(page_head)) 187 __put_compound_page(page_head); 188 else 189 __put_single_page(page_head); 190 } 191 out_put_single: 192 if (put_page_testzero(page)) 193 __put_single_page(page); 194 return; 195 } 196 VM_BUG_ON_PAGE(page_head != page->first_page, page); 197 /* 198 * We can release the refcount taken by 199 * get_page_unless_zero() now that 200 * __split_huge_page_refcount() is blocked on the 201 * compound_lock. 202 */ 203 if (put_page_testzero(page_head)) 204 VM_BUG_ON_PAGE(1, page_head); 205 /* __split_huge_page_refcount will wait now */ 206 VM_BUG_ON_PAGE(page_mapcount(page) <= 0, page); 207 atomic_dec(&page->_mapcount); 208 VM_BUG_ON_PAGE(atomic_read(&page_head->_count) <= 0, page_head); 209 VM_BUG_ON_PAGE(atomic_read(&page->_count) != 0, page); 210 compound_unlock_irqrestore(page_head, flags); 211 212 if (put_page_testzero(page_head)) { 213 if (PageHead(page_head)) 214 __put_compound_page(page_head); 215 else 216 __put_single_page(page_head); 217 } 218 } else { 219 /* @page_head is a dangling pointer */ 220 VM_BUG_ON_PAGE(PageTail(page), page); 221 goto out_put_single; 222 } 223 } 224 225 static void put_compound_page(struct page *page) 226 { 227 struct page *page_head; 228 229 /* 230 * We see the PageCompound set and PageTail not set, so @page maybe: 231 * 1. hugetlbfs head page, or 232 * 2. THP head page. 233 */ 234 if (likely(!PageTail(page))) { 235 if (put_page_testzero(page)) { 236 /* 237 * By the time all refcounts have been released 238 * split_huge_page cannot run anymore from under us. 239 */ 240 if (PageHead(page)) 241 __put_compound_page(page); 242 else 243 __put_single_page(page); 244 } 245 return; 246 } 247 248 /* 249 * We see the PageCompound set and PageTail set, so @page maybe: 250 * 1. a tail hugetlbfs page, or 251 * 2. a tail THP page, or 252 * 3. a split THP page. 253 * 254 * Case 3 is possible, as we may race with 255 * __split_huge_page_refcount tearing down a THP page. 256 */ 257 page_head = compound_head_by_tail(page); 258 if (!__compound_tail_refcounted(page_head)) 259 put_unrefcounted_compound_page(page_head, page); 260 else 261 put_refcounted_compound_page(page_head, page); 262 } 263 264 void put_page(struct page *page) 265 { 266 if (unlikely(PageCompound(page))) 267 put_compound_page(page); 268 else if (put_page_testzero(page)) 269 __put_single_page(page); 270 } 271 EXPORT_SYMBOL(put_page); 272 273 /* 274 * This function is exported but must not be called by anything other 275 * than get_page(). It implements the slow path of get_page(). 276 */ 277 bool __get_page_tail(struct page *page) 278 { 279 /* 280 * This takes care of get_page() if run on a tail page 281 * returned by one of the get_user_pages/follow_page variants. 282 * get_user_pages/follow_page itself doesn't need the compound 283 * lock because it runs __get_page_tail_foll() under the 284 * proper PT lock that already serializes against 285 * split_huge_page(). 286 */ 287 unsigned long flags; 288 bool got; 289 struct page *page_head = compound_head(page); 290 291 /* Ref to put_compound_page() comment. */ 292 if (!__compound_tail_refcounted(page_head)) { 293 smp_rmb(); 294 if (likely(PageTail(page))) { 295 /* 296 * This is a hugetlbfs page or a slab 297 * page. __split_huge_page_refcount 298 * cannot race here. 299 */ 300 VM_BUG_ON_PAGE(!PageHead(page_head), page_head); 301 __get_page_tail_foll(page, true); 302 return true; 303 } else { 304 /* 305 * __split_huge_page_refcount run 306 * before us, "page" was a THP 307 * tail. The split page_head has been 308 * freed and reallocated as slab or 309 * hugetlbfs page of smaller order 310 * (only possible if reallocated as 311 * slab on x86). 312 */ 313 return false; 314 } 315 } 316 317 got = false; 318 if (likely(page != page_head && get_page_unless_zero(page_head))) { 319 /* 320 * page_head wasn't a dangling pointer but it 321 * may not be a head page anymore by the time 322 * we obtain the lock. That is ok as long as it 323 * can't be freed from under us. 324 */ 325 flags = compound_lock_irqsave(page_head); 326 /* here __split_huge_page_refcount won't run anymore */ 327 if (likely(PageTail(page))) { 328 __get_page_tail_foll(page, false); 329 got = true; 330 } 331 compound_unlock_irqrestore(page_head, flags); 332 if (unlikely(!got)) 333 put_page(page_head); 334 } 335 return got; 336 } 337 EXPORT_SYMBOL(__get_page_tail); 338 339 /** 340 * put_pages_list() - release a list of pages 341 * @pages: list of pages threaded on page->lru 342 * 343 * Release a list of pages which are strung together on page.lru. Currently 344 * used by read_cache_pages() and related error recovery code. 345 */ 346 void put_pages_list(struct list_head *pages) 347 { 348 while (!list_empty(pages)) { 349 struct page *victim; 350 351 victim = list_entry(pages->prev, struct page, lru); 352 list_del(&victim->lru); 353 page_cache_release(victim); 354 } 355 } 356 EXPORT_SYMBOL(put_pages_list); 357 358 /* 359 * get_kernel_pages() - pin kernel pages in memory 360 * @kiov: An array of struct kvec structures 361 * @nr_segs: number of segments to pin 362 * @write: pinning for read/write, currently ignored 363 * @pages: array that receives pointers to the pages pinned. 364 * Should be at least nr_segs long. 365 * 366 * Returns number of pages pinned. This may be fewer than the number 367 * requested. If nr_pages is 0 or negative, returns 0. If no pages 368 * were pinned, returns -errno. Each page returned must be released 369 * with a put_page() call when it is finished with. 370 */ 371 int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write, 372 struct page **pages) 373 { 374 int seg; 375 376 for (seg = 0; seg < nr_segs; seg++) { 377 if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE)) 378 return seg; 379 380 pages[seg] = kmap_to_page(kiov[seg].iov_base); 381 page_cache_get(pages[seg]); 382 } 383 384 return seg; 385 } 386 EXPORT_SYMBOL_GPL(get_kernel_pages); 387 388 /* 389 * get_kernel_page() - pin a kernel page in memory 390 * @start: starting kernel address 391 * @write: pinning for read/write, currently ignored 392 * @pages: array that receives pointer to the page pinned. 393 * Must be at least nr_segs long. 394 * 395 * Returns 1 if page is pinned. If the page was not pinned, returns 396 * -errno. The page returned must be released with a put_page() call 397 * when it is finished with. 398 */ 399 int get_kernel_page(unsigned long start, int write, struct page **pages) 400 { 401 const struct kvec kiov = { 402 .iov_base = (void *)start, 403 .iov_len = PAGE_SIZE 404 }; 405 406 return get_kernel_pages(&kiov, 1, write, pages); 407 } 408 EXPORT_SYMBOL_GPL(get_kernel_page); 409 410 static void pagevec_lru_move_fn(struct pagevec *pvec, 411 void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg), 412 void *arg) 413 { 414 int i; 415 struct zone *zone = NULL; 416 struct lruvec *lruvec; 417 unsigned long flags = 0; 418 419 for (i = 0; i < pagevec_count(pvec); i++) { 420 struct page *page = pvec->pages[i]; 421 struct zone *pagezone = page_zone(page); 422 423 if (pagezone != zone) { 424 if (zone) 425 spin_unlock_irqrestore(&zone->lru_lock, flags); 426 zone = pagezone; 427 spin_lock_irqsave(&zone->lru_lock, flags); 428 } 429 430 lruvec = mem_cgroup_page_lruvec(page, zone); 431 (*move_fn)(page, lruvec, arg); 432 } 433 if (zone) 434 spin_unlock_irqrestore(&zone->lru_lock, flags); 435 release_pages(pvec->pages, pvec->nr, pvec->cold); 436 pagevec_reinit(pvec); 437 } 438 439 static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec, 440 void *arg) 441 { 442 int *pgmoved = arg; 443 444 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 445 enum lru_list lru = page_lru_base_type(page); 446 list_move_tail(&page->lru, &lruvec->lists[lru]); 447 (*pgmoved)++; 448 } 449 } 450 451 /* 452 * pagevec_move_tail() must be called with IRQ disabled. 453 * Otherwise this may cause nasty races. 454 */ 455 static void pagevec_move_tail(struct pagevec *pvec) 456 { 457 int pgmoved = 0; 458 459 pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved); 460 __count_vm_events(PGROTATED, pgmoved); 461 } 462 463 /* 464 * Writeback is about to end against a page which has been marked for immediate 465 * reclaim. If it still appears to be reclaimable, move it to the tail of the 466 * inactive list. 467 */ 468 void rotate_reclaimable_page(struct page *page) 469 { 470 if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) && 471 !PageUnevictable(page) && PageLRU(page)) { 472 struct pagevec *pvec; 473 unsigned long flags; 474 475 page_cache_get(page); 476 local_irq_save(flags); 477 pvec = this_cpu_ptr(&lru_rotate_pvecs); 478 if (!pagevec_add(pvec, page)) 479 pagevec_move_tail(pvec); 480 local_irq_restore(flags); 481 } 482 } 483 484 static void update_page_reclaim_stat(struct lruvec *lruvec, 485 int file, int rotated) 486 { 487 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; 488 489 reclaim_stat->recent_scanned[file]++; 490 if (rotated) 491 reclaim_stat->recent_rotated[file]++; 492 } 493 494 static void __activate_page(struct page *page, struct lruvec *lruvec, 495 void *arg) 496 { 497 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 498 int file = page_is_file_cache(page); 499 int lru = page_lru_base_type(page); 500 501 del_page_from_lru_list(page, lruvec, lru); 502 SetPageActive(page); 503 lru += LRU_ACTIVE; 504 add_page_to_lru_list(page, lruvec, lru); 505 trace_mm_lru_activate(page); 506 507 __count_vm_event(PGACTIVATE); 508 update_page_reclaim_stat(lruvec, file, 1); 509 } 510 } 511 512 #ifdef CONFIG_SMP 513 static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs); 514 515 static void activate_page_drain(int cpu) 516 { 517 struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu); 518 519 if (pagevec_count(pvec)) 520 pagevec_lru_move_fn(pvec, __activate_page, NULL); 521 } 522 523 static bool need_activate_page_drain(int cpu) 524 { 525 return pagevec_count(&per_cpu(activate_page_pvecs, cpu)) != 0; 526 } 527 528 void activate_page(struct page *page) 529 { 530 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { 531 struct pagevec *pvec = &get_cpu_var(activate_page_pvecs); 532 533 page_cache_get(page); 534 if (!pagevec_add(pvec, page)) 535 pagevec_lru_move_fn(pvec, __activate_page, NULL); 536 put_cpu_var(activate_page_pvecs); 537 } 538 } 539 540 #else 541 static inline void activate_page_drain(int cpu) 542 { 543 } 544 545 static bool need_activate_page_drain(int cpu) 546 { 547 return false; 548 } 549 550 void activate_page(struct page *page) 551 { 552 struct zone *zone = page_zone(page); 553 554 spin_lock_irq(&zone->lru_lock); 555 __activate_page(page, mem_cgroup_page_lruvec(page, zone), NULL); 556 spin_unlock_irq(&zone->lru_lock); 557 } 558 #endif 559 560 static void __lru_cache_activate_page(struct page *page) 561 { 562 struct pagevec *pvec = &get_cpu_var(lru_add_pvec); 563 int i; 564 565 /* 566 * Search backwards on the optimistic assumption that the page being 567 * activated has just been added to this pagevec. Note that only 568 * the local pagevec is examined as a !PageLRU page could be in the 569 * process of being released, reclaimed, migrated or on a remote 570 * pagevec that is currently being drained. Furthermore, marking 571 * a remote pagevec's page PageActive potentially hits a race where 572 * a page is marked PageActive just after it is added to the inactive 573 * list causing accounting errors and BUG_ON checks to trigger. 574 */ 575 for (i = pagevec_count(pvec) - 1; i >= 0; i--) { 576 struct page *pagevec_page = pvec->pages[i]; 577 578 if (pagevec_page == page) { 579 SetPageActive(page); 580 break; 581 } 582 } 583 584 put_cpu_var(lru_add_pvec); 585 } 586 587 /* 588 * Mark a page as having seen activity. 589 * 590 * inactive,unreferenced -> inactive,referenced 591 * inactive,referenced -> active,unreferenced 592 * active,unreferenced -> active,referenced 593 * 594 * When a newly allocated page is not yet visible, so safe for non-atomic ops, 595 * __SetPageReferenced(page) may be substituted for mark_page_accessed(page). 596 */ 597 void mark_page_accessed(struct page *page) 598 { 599 if (!PageActive(page) && !PageUnevictable(page) && 600 PageReferenced(page)) { 601 602 /* 603 * If the page is on the LRU, queue it for activation via 604 * activate_page_pvecs. Otherwise, assume the page is on a 605 * pagevec, mark it active and it'll be moved to the active 606 * LRU on the next drain. 607 */ 608 if (PageLRU(page)) 609 activate_page(page); 610 else 611 __lru_cache_activate_page(page); 612 ClearPageReferenced(page); 613 if (page_is_file_cache(page)) 614 workingset_activation(page); 615 } else if (!PageReferenced(page)) { 616 SetPageReferenced(page); 617 } 618 } 619 EXPORT_SYMBOL(mark_page_accessed); 620 621 static void __lru_cache_add(struct page *page) 622 { 623 struct pagevec *pvec = &get_cpu_var(lru_add_pvec); 624 625 page_cache_get(page); 626 if (!pagevec_space(pvec)) 627 __pagevec_lru_add(pvec); 628 pagevec_add(pvec, page); 629 put_cpu_var(lru_add_pvec); 630 } 631 632 /** 633 * lru_cache_add: add a page to the page lists 634 * @page: the page to add 635 */ 636 void lru_cache_add_anon(struct page *page) 637 { 638 if (PageActive(page)) 639 ClearPageActive(page); 640 __lru_cache_add(page); 641 } 642 643 void lru_cache_add_file(struct page *page) 644 { 645 if (PageActive(page)) 646 ClearPageActive(page); 647 __lru_cache_add(page); 648 } 649 EXPORT_SYMBOL(lru_cache_add_file); 650 651 /** 652 * lru_cache_add - add a page to a page list 653 * @page: the page to be added to the LRU. 654 * 655 * Queue the page for addition to the LRU via pagevec. The decision on whether 656 * to add the page to the [in]active [file|anon] list is deferred until the 657 * pagevec is drained. This gives a chance for the caller of lru_cache_add() 658 * have the page added to the active list using mark_page_accessed(). 659 */ 660 void lru_cache_add(struct page *page) 661 { 662 VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); 663 VM_BUG_ON_PAGE(PageLRU(page), page); 664 __lru_cache_add(page); 665 } 666 667 /** 668 * add_page_to_unevictable_list - add a page to the unevictable list 669 * @page: the page to be added to the unevictable list 670 * 671 * Add page directly to its zone's unevictable list. To avoid races with 672 * tasks that might be making the page evictable, through eg. munlock, 673 * munmap or exit, while it's not on the lru, we want to add the page 674 * while it's locked or otherwise "invisible" to other tasks. This is 675 * difficult to do when using the pagevec cache, so bypass that. 676 */ 677 void add_page_to_unevictable_list(struct page *page) 678 { 679 struct zone *zone = page_zone(page); 680 struct lruvec *lruvec; 681 682 spin_lock_irq(&zone->lru_lock); 683 lruvec = mem_cgroup_page_lruvec(page, zone); 684 ClearPageActive(page); 685 SetPageUnevictable(page); 686 SetPageLRU(page); 687 add_page_to_lru_list(page, lruvec, LRU_UNEVICTABLE); 688 spin_unlock_irq(&zone->lru_lock); 689 } 690 691 /** 692 * lru_cache_add_active_or_unevictable 693 * @page: the page to be added to LRU 694 * @vma: vma in which page is mapped for determining reclaimability 695 * 696 * Place @page on the active or unevictable LRU list, depending on its 697 * evictability. Note that if the page is not evictable, it goes 698 * directly back onto it's zone's unevictable list, it does NOT use a 699 * per cpu pagevec. 700 */ 701 void lru_cache_add_active_or_unevictable(struct page *page, 702 struct vm_area_struct *vma) 703 { 704 VM_BUG_ON_PAGE(PageLRU(page), page); 705 706 if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED)) { 707 SetPageActive(page); 708 lru_cache_add(page); 709 return; 710 } 711 712 if (!TestSetPageMlocked(page)) { 713 /* 714 * We use the irq-unsafe __mod_zone_page_stat because this 715 * counter is not modified from interrupt context, and the pte 716 * lock is held(spinlock), which implies preemption disabled. 717 */ 718 __mod_zone_page_state(page_zone(page), NR_MLOCK, 719 hpage_nr_pages(page)); 720 count_vm_event(UNEVICTABLE_PGMLOCKED); 721 } 722 add_page_to_unevictable_list(page); 723 } 724 725 /* 726 * If the page can not be invalidated, it is moved to the 727 * inactive list to speed up its reclaim. It is moved to the 728 * head of the list, rather than the tail, to give the flusher 729 * threads some time to write it out, as this is much more 730 * effective than the single-page writeout from reclaim. 731 * 732 * If the page isn't page_mapped and dirty/writeback, the page 733 * could reclaim asap using PG_reclaim. 734 * 735 * 1. active, mapped page -> none 736 * 2. active, dirty/writeback page -> inactive, head, PG_reclaim 737 * 3. inactive, mapped page -> none 738 * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim 739 * 5. inactive, clean -> inactive, tail 740 * 6. Others -> none 741 * 742 * In 4, why it moves inactive's head, the VM expects the page would 743 * be write it out by flusher threads as this is much more effective 744 * than the single-page writeout from reclaim. 745 */ 746 static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec, 747 void *arg) 748 { 749 int lru, file; 750 bool active; 751 752 if (!PageLRU(page)) 753 return; 754 755 if (PageUnevictable(page)) 756 return; 757 758 /* Some processes are using the page */ 759 if (page_mapped(page)) 760 return; 761 762 active = PageActive(page); 763 file = page_is_file_cache(page); 764 lru = page_lru_base_type(page); 765 766 del_page_from_lru_list(page, lruvec, lru + active); 767 ClearPageActive(page); 768 ClearPageReferenced(page); 769 add_page_to_lru_list(page, lruvec, lru); 770 771 if (PageWriteback(page) || PageDirty(page)) { 772 /* 773 * PG_reclaim could be raced with end_page_writeback 774 * It can make readahead confusing. But race window 775 * is _really_ small and it's non-critical problem. 776 */ 777 SetPageReclaim(page); 778 } else { 779 /* 780 * The page's writeback ends up during pagevec 781 * We moves tha page into tail of inactive. 782 */ 783 list_move_tail(&page->lru, &lruvec->lists[lru]); 784 __count_vm_event(PGROTATED); 785 } 786 787 if (active) 788 __count_vm_event(PGDEACTIVATE); 789 update_page_reclaim_stat(lruvec, file, 0); 790 } 791 792 /* 793 * Drain pages out of the cpu's pagevecs. 794 * Either "cpu" is the current CPU, and preemption has already been 795 * disabled; or "cpu" is being hot-unplugged, and is already dead. 796 */ 797 void lru_add_drain_cpu(int cpu) 798 { 799 struct pagevec *pvec = &per_cpu(lru_add_pvec, cpu); 800 801 if (pagevec_count(pvec)) 802 __pagevec_lru_add(pvec); 803 804 pvec = &per_cpu(lru_rotate_pvecs, cpu); 805 if (pagevec_count(pvec)) { 806 unsigned long flags; 807 808 /* No harm done if a racing interrupt already did this */ 809 local_irq_save(flags); 810 pagevec_move_tail(pvec); 811 local_irq_restore(flags); 812 } 813 814 pvec = &per_cpu(lru_deactivate_pvecs, cpu); 815 if (pagevec_count(pvec)) 816 pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); 817 818 activate_page_drain(cpu); 819 } 820 821 /** 822 * deactivate_page - forcefully deactivate a page 823 * @page: page to deactivate 824 * 825 * This function hints the VM that @page is a good reclaim candidate, 826 * for example if its invalidation fails due to the page being dirty 827 * or under writeback. 828 */ 829 void deactivate_page(struct page *page) 830 { 831 /* 832 * In a workload with many unevictable page such as mprotect, unevictable 833 * page deactivation for accelerating reclaim is pointless. 834 */ 835 if (PageUnevictable(page)) 836 return; 837 838 if (likely(get_page_unless_zero(page))) { 839 struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs); 840 841 if (!pagevec_add(pvec, page)) 842 pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); 843 put_cpu_var(lru_deactivate_pvecs); 844 } 845 } 846 847 void lru_add_drain(void) 848 { 849 lru_add_drain_cpu(get_cpu()); 850 put_cpu(); 851 } 852 853 static void lru_add_drain_per_cpu(struct work_struct *dummy) 854 { 855 lru_add_drain(); 856 } 857 858 static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); 859 860 void lru_add_drain_all(void) 861 { 862 static DEFINE_MUTEX(lock); 863 static struct cpumask has_work; 864 int cpu; 865 866 mutex_lock(&lock); 867 get_online_cpus(); 868 cpumask_clear(&has_work); 869 870 for_each_online_cpu(cpu) { 871 struct work_struct *work = &per_cpu(lru_add_drain_work, cpu); 872 873 if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) || 874 pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) || 875 pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) || 876 need_activate_page_drain(cpu)) { 877 INIT_WORK(work, lru_add_drain_per_cpu); 878 schedule_work_on(cpu, work); 879 cpumask_set_cpu(cpu, &has_work); 880 } 881 } 882 883 for_each_cpu(cpu, &has_work) 884 flush_work(&per_cpu(lru_add_drain_work, cpu)); 885 886 put_online_cpus(); 887 mutex_unlock(&lock); 888 } 889 890 /* 891 * Batched page_cache_release(). Decrement the reference count on all the 892 * passed pages. If it fell to zero then remove the page from the LRU and 893 * free it. 894 * 895 * Avoid taking zone->lru_lock if possible, but if it is taken, retain it 896 * for the remainder of the operation. 897 * 898 * The locking in this function is against shrink_inactive_list(): we recheck 899 * the page count inside the lock to see whether shrink_inactive_list() 900 * grabbed the page via the LRU. If it did, give up: shrink_inactive_list() 901 * will free it. 902 */ 903 void release_pages(struct page **pages, int nr, bool cold) 904 { 905 int i; 906 LIST_HEAD(pages_to_free); 907 struct zone *zone = NULL; 908 struct lruvec *lruvec; 909 unsigned long uninitialized_var(flags); 910 911 for (i = 0; i < nr; i++) { 912 struct page *page = pages[i]; 913 914 if (unlikely(PageCompound(page))) { 915 if (zone) { 916 spin_unlock_irqrestore(&zone->lru_lock, flags); 917 zone = NULL; 918 } 919 put_compound_page(page); 920 continue; 921 } 922 923 if (!put_page_testzero(page)) 924 continue; 925 926 if (PageLRU(page)) { 927 struct zone *pagezone = page_zone(page); 928 929 if (pagezone != zone) { 930 if (zone) 931 spin_unlock_irqrestore(&zone->lru_lock, 932 flags); 933 zone = pagezone; 934 spin_lock_irqsave(&zone->lru_lock, flags); 935 } 936 937 lruvec = mem_cgroup_page_lruvec(page, zone); 938 VM_BUG_ON_PAGE(!PageLRU(page), page); 939 __ClearPageLRU(page); 940 del_page_from_lru_list(page, lruvec, page_off_lru(page)); 941 } 942 943 /* Clear Active bit in case of parallel mark_page_accessed */ 944 __ClearPageActive(page); 945 946 list_add(&page->lru, &pages_to_free); 947 } 948 if (zone) 949 spin_unlock_irqrestore(&zone->lru_lock, flags); 950 951 mem_cgroup_uncharge_list(&pages_to_free); 952 free_hot_cold_page_list(&pages_to_free, cold); 953 } 954 EXPORT_SYMBOL(release_pages); 955 956 /* 957 * The pages which we're about to release may be in the deferred lru-addition 958 * queues. That would prevent them from really being freed right now. That's 959 * OK from a correctness point of view but is inefficient - those pages may be 960 * cache-warm and we want to give them back to the page allocator ASAP. 961 * 962 * So __pagevec_release() will drain those queues here. __pagevec_lru_add() 963 * and __pagevec_lru_add_active() call release_pages() directly to avoid 964 * mutual recursion. 965 */ 966 void __pagevec_release(struct pagevec *pvec) 967 { 968 lru_add_drain(); 969 release_pages(pvec->pages, pagevec_count(pvec), pvec->cold); 970 pagevec_reinit(pvec); 971 } 972 EXPORT_SYMBOL(__pagevec_release); 973 974 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 975 /* used by __split_huge_page_refcount() */ 976 void lru_add_page_tail(struct page *page, struct page *page_tail, 977 struct lruvec *lruvec, struct list_head *list) 978 { 979 const int file = 0; 980 981 VM_BUG_ON_PAGE(!PageHead(page), page); 982 VM_BUG_ON_PAGE(PageCompound(page_tail), page); 983 VM_BUG_ON_PAGE(PageLRU(page_tail), page); 984 VM_BUG_ON(NR_CPUS != 1 && 985 !spin_is_locked(&lruvec_zone(lruvec)->lru_lock)); 986 987 if (!list) 988 SetPageLRU(page_tail); 989 990 if (likely(PageLRU(page))) 991 list_add_tail(&page_tail->lru, &page->lru); 992 else if (list) { 993 /* page reclaim is reclaiming a huge page */ 994 get_page(page_tail); 995 list_add_tail(&page_tail->lru, list); 996 } else { 997 struct list_head *list_head; 998 /* 999 * Head page has not yet been counted, as an hpage, 1000 * so we must account for each subpage individually. 1001 * 1002 * Use the standard add function to put page_tail on the list, 1003 * but then correct its position so they all end up in order. 1004 */ 1005 add_page_to_lru_list(page_tail, lruvec, page_lru(page_tail)); 1006 list_head = page_tail->lru.prev; 1007 list_move_tail(&page_tail->lru, list_head); 1008 } 1009 1010 if (!PageUnevictable(page)) 1011 update_page_reclaim_stat(lruvec, file, PageActive(page_tail)); 1012 } 1013 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1014 1015 static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, 1016 void *arg) 1017 { 1018 int file = page_is_file_cache(page); 1019 int active = PageActive(page); 1020 enum lru_list lru = page_lru(page); 1021 1022 VM_BUG_ON_PAGE(PageLRU(page), page); 1023 1024 SetPageLRU(page); 1025 add_page_to_lru_list(page, lruvec, lru); 1026 update_page_reclaim_stat(lruvec, file, active); 1027 trace_mm_lru_insertion(page, lru); 1028 } 1029 1030 /* 1031 * Add the passed pages to the LRU, then drop the caller's refcount 1032 * on them. Reinitialises the caller's pagevec. 1033 */ 1034 void __pagevec_lru_add(struct pagevec *pvec) 1035 { 1036 pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL); 1037 } 1038 EXPORT_SYMBOL(__pagevec_lru_add); 1039 1040 /** 1041 * pagevec_lookup_entries - gang pagecache lookup 1042 * @pvec: Where the resulting entries are placed 1043 * @mapping: The address_space to search 1044 * @start: The starting entry index 1045 * @nr_entries: The maximum number of entries 1046 * @indices: The cache indices corresponding to the entries in @pvec 1047 * 1048 * pagevec_lookup_entries() will search for and return a group of up 1049 * to @nr_entries pages and shadow entries in the mapping. All 1050 * entries are placed in @pvec. pagevec_lookup_entries() takes a 1051 * reference against actual pages in @pvec. 1052 * 1053 * The search returns a group of mapping-contiguous entries with 1054 * ascending indexes. There may be holes in the indices due to 1055 * not-present entries. 1056 * 1057 * pagevec_lookup_entries() returns the number of entries which were 1058 * found. 1059 */ 1060 unsigned pagevec_lookup_entries(struct pagevec *pvec, 1061 struct address_space *mapping, 1062 pgoff_t start, unsigned nr_pages, 1063 pgoff_t *indices) 1064 { 1065 pvec->nr = find_get_entries(mapping, start, nr_pages, 1066 pvec->pages, indices); 1067 return pagevec_count(pvec); 1068 } 1069 1070 /** 1071 * pagevec_remove_exceptionals - pagevec exceptionals pruning 1072 * @pvec: The pagevec to prune 1073 * 1074 * pagevec_lookup_entries() fills both pages and exceptional radix 1075 * tree entries into the pagevec. This function prunes all 1076 * exceptionals from @pvec without leaving holes, so that it can be 1077 * passed on to page-only pagevec operations. 1078 */ 1079 void pagevec_remove_exceptionals(struct pagevec *pvec) 1080 { 1081 int i, j; 1082 1083 for (i = 0, j = 0; i < pagevec_count(pvec); i++) { 1084 struct page *page = pvec->pages[i]; 1085 if (!radix_tree_exceptional_entry(page)) 1086 pvec->pages[j++] = page; 1087 } 1088 pvec->nr = j; 1089 } 1090 1091 /** 1092 * pagevec_lookup - gang pagecache lookup 1093 * @pvec: Where the resulting pages are placed 1094 * @mapping: The address_space to search 1095 * @start: The starting page index 1096 * @nr_pages: The maximum number of pages 1097 * 1098 * pagevec_lookup() will search for and return a group of up to @nr_pages pages 1099 * in the mapping. The pages are placed in @pvec. pagevec_lookup() takes a 1100 * reference against the pages in @pvec. 1101 * 1102 * The search returns a group of mapping-contiguous pages with ascending 1103 * indexes. There may be holes in the indices due to not-present pages. 1104 * 1105 * pagevec_lookup() returns the number of pages which were found. 1106 */ 1107 unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping, 1108 pgoff_t start, unsigned nr_pages) 1109 { 1110 pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages); 1111 return pagevec_count(pvec); 1112 } 1113 EXPORT_SYMBOL(pagevec_lookup); 1114 1115 unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping, 1116 pgoff_t *index, int tag, unsigned nr_pages) 1117 { 1118 pvec->nr = find_get_pages_tag(mapping, index, tag, 1119 nr_pages, pvec->pages); 1120 return pagevec_count(pvec); 1121 } 1122 EXPORT_SYMBOL(pagevec_lookup_tag); 1123 1124 /* 1125 * Perform any setup for the swap system 1126 */ 1127 void __init swap_setup(void) 1128 { 1129 unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT); 1130 #ifdef CONFIG_SWAP 1131 int i; 1132 1133 if (bdi_init(swapper_spaces[0].backing_dev_info)) 1134 panic("Failed to init swap bdi"); 1135 for (i = 0; i < MAX_SWAPFILES; i++) { 1136 spin_lock_init(&swapper_spaces[i].tree_lock); 1137 INIT_LIST_HEAD(&swapper_spaces[i].i_mmap_nonlinear); 1138 } 1139 #endif 1140 1141 /* Use a smaller cluster for small-memory machines */ 1142 if (megs < 16) 1143 page_cluster = 2; 1144 else 1145 page_cluster = 3; 1146 /* 1147 * Right now other parts of the system means that we 1148 * _really_ don't want to cluster much more 1149 */ 1150 } 1151