1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 4 * 5 * Swap reorganised 29.12.95, Stephen Tweedie. 6 * kswapd added: 7.1.96 sct 7 * Removed kswapd_ctl limits, and swap out as many pages as needed 8 * to bring the system back to freepages.high: 2.4.97, Rik van Riel. 9 * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com). 10 * Multiqueue VM started 5.8.00, Rik van Riel. 11 */ 12 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15 #include <linux/mm.h> 16 #include <linux/sched/mm.h> 17 #include <linux/module.h> 18 #include <linux/gfp.h> 19 #include <linux/kernel_stat.h> 20 #include <linux/swap.h> 21 #include <linux/pagemap.h> 22 #include <linux/init.h> 23 #include <linux/highmem.h> 24 #include <linux/vmpressure.h> 25 #include <linux/vmstat.h> 26 #include <linux/file.h> 27 #include <linux/writeback.h> 28 #include <linux/blkdev.h> 29 #include <linux/buffer_head.h> /* for buffer_heads_over_limit */ 30 #include <linux/mm_inline.h> 31 #include <linux/backing-dev.h> 32 #include <linux/rmap.h> 33 #include <linux/topology.h> 34 #include <linux/cpu.h> 35 #include <linux/cpuset.h> 36 #include <linux/compaction.h> 37 #include <linux/notifier.h> 38 #include <linux/rwsem.h> 39 #include <linux/delay.h> 40 #include <linux/kthread.h> 41 #include <linux/freezer.h> 42 #include <linux/memcontrol.h> 43 #include <linux/migrate.h> 44 #include <linux/delayacct.h> 45 #include <linux/sysctl.h> 46 #include <linux/memory-tiers.h> 47 #include <linux/oom.h> 48 #include <linux/pagevec.h> 49 #include <linux/prefetch.h> 50 #include <linux/printk.h> 51 #include <linux/dax.h> 52 #include <linux/psi.h> 53 #include <linux/pagewalk.h> 54 #include <linux/shmem_fs.h> 55 #include <linux/ctype.h> 56 #include <linux/debugfs.h> 57 #include <linux/khugepaged.h> 58 59 #include <asm/tlbflush.h> 60 #include <asm/div64.h> 61 62 #include <linux/swapops.h> 63 #include <linux/balloon_compaction.h> 64 #include <linux/sched/sysctl.h> 65 66 #include "internal.h" 67 #include "swap.h" 68 69 #define CREATE_TRACE_POINTS 70 #include <trace/events/vmscan.h> 71 72 struct scan_control { 73 /* How many pages shrink_list() should reclaim */ 74 unsigned long nr_to_reclaim; 75 76 /* 77 * Nodemask of nodes allowed by the caller. If NULL, all nodes 78 * are scanned. 79 */ 80 nodemask_t *nodemask; 81 82 /* 83 * The memory cgroup that hit its limit and as a result is the 84 * primary target of this reclaim invocation. 85 */ 86 struct mem_cgroup *target_mem_cgroup; 87 88 /* 89 * Scan pressure balancing between anon and file LRUs 90 */ 91 unsigned long anon_cost; 92 unsigned long file_cost; 93 94 /* Can active folios be deactivated as part of reclaim? */ 95 #define DEACTIVATE_ANON 1 96 #define DEACTIVATE_FILE 2 97 unsigned int may_deactivate:2; 98 unsigned int force_deactivate:1; 99 unsigned int skipped_deactivate:1; 100 101 /* Writepage batching in laptop mode; RECLAIM_WRITE */ 102 unsigned int may_writepage:1; 103 104 /* Can mapped folios be reclaimed? */ 105 unsigned int may_unmap:1; 106 107 /* Can folios be swapped as part of reclaim? */ 108 unsigned int may_swap:1; 109 110 /* Proactive reclaim invoked by userspace through memory.reclaim */ 111 unsigned int proactive:1; 112 113 /* 114 * Cgroup memory below memory.low is protected as long as we 115 * don't threaten to OOM. If any cgroup is reclaimed at 116 * reduced force or passed over entirely due to its memory.low 117 * setting (memcg_low_skipped), and nothing is reclaimed as a 118 * result, then go back for one more cycle that reclaims the protected 119 * memory (memcg_low_reclaim) to avert OOM. 120 */ 121 unsigned int memcg_low_reclaim:1; 122 unsigned int memcg_low_skipped:1; 123 124 unsigned int hibernation_mode:1; 125 126 /* One of the zones is ready for compaction */ 127 unsigned int compaction_ready:1; 128 129 /* There is easily reclaimable cold cache in the current node */ 130 unsigned int cache_trim_mode:1; 131 132 /* The file folios on the current node are dangerously low */ 133 unsigned int file_is_tiny:1; 134 135 /* Always discard instead of demoting to lower tier memory */ 136 unsigned int no_demotion:1; 137 138 #ifdef CONFIG_LRU_GEN 139 /* help kswapd make better choices among multiple memcgs */ 140 unsigned int memcgs_need_aging:1; 141 unsigned long last_reclaimed; 142 #endif 143 144 /* Allocation order */ 145 s8 order; 146 147 /* Scan (total_size >> priority) pages at once */ 148 s8 priority; 149 150 /* The highest zone to isolate folios for reclaim from */ 151 s8 reclaim_idx; 152 153 /* This context's GFP mask */ 154 gfp_t gfp_mask; 155 156 /* Incremented by the number of inactive pages that were scanned */ 157 unsigned long nr_scanned; 158 159 /* Number of pages freed so far during a call to shrink_zones() */ 160 unsigned long nr_reclaimed; 161 162 struct { 163 unsigned int dirty; 164 unsigned int unqueued_dirty; 165 unsigned int congested; 166 unsigned int writeback; 167 unsigned int immediate; 168 unsigned int file_taken; 169 unsigned int taken; 170 } nr; 171 172 /* for recording the reclaimed slab by now */ 173 struct reclaim_state reclaim_state; 174 }; 175 176 #ifdef ARCH_HAS_PREFETCHW 177 #define prefetchw_prev_lru_folio(_folio, _base, _field) \ 178 do { \ 179 if ((_folio)->lru.prev != _base) { \ 180 struct folio *prev; \ 181 \ 182 prev = lru_to_folio(&(_folio->lru)); \ 183 prefetchw(&prev->_field); \ 184 } \ 185 } while (0) 186 #else 187 #define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0) 188 #endif 189 190 /* 191 * From 0 .. 200. Higher means more swappy. 192 */ 193 int vm_swappiness = 60; 194 195 static void set_task_reclaim_state(struct task_struct *task, 196 struct reclaim_state *rs) 197 { 198 /* Check for an overwrite */ 199 WARN_ON_ONCE(rs && task->reclaim_state); 200 201 /* Check for the nulling of an already-nulled member */ 202 WARN_ON_ONCE(!rs && !task->reclaim_state); 203 204 task->reclaim_state = rs; 205 } 206 207 LIST_HEAD(shrinker_list); 208 DECLARE_RWSEM(shrinker_rwsem); 209 210 #ifdef CONFIG_MEMCG 211 static int shrinker_nr_max; 212 213 /* The shrinker_info is expanded in a batch of BITS_PER_LONG */ 214 static inline int shrinker_map_size(int nr_items) 215 { 216 return (DIV_ROUND_UP(nr_items, BITS_PER_LONG) * sizeof(unsigned long)); 217 } 218 219 static inline int shrinker_defer_size(int nr_items) 220 { 221 return (round_up(nr_items, BITS_PER_LONG) * sizeof(atomic_long_t)); 222 } 223 224 static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg, 225 int nid) 226 { 227 return rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info, 228 lockdep_is_held(&shrinker_rwsem)); 229 } 230 231 static int expand_one_shrinker_info(struct mem_cgroup *memcg, 232 int map_size, int defer_size, 233 int old_map_size, int old_defer_size) 234 { 235 struct shrinker_info *new, *old; 236 struct mem_cgroup_per_node *pn; 237 int nid; 238 int size = map_size + defer_size; 239 240 for_each_node(nid) { 241 pn = memcg->nodeinfo[nid]; 242 old = shrinker_info_protected(memcg, nid); 243 /* Not yet online memcg */ 244 if (!old) 245 return 0; 246 247 new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid); 248 if (!new) 249 return -ENOMEM; 250 251 new->nr_deferred = (atomic_long_t *)(new + 1); 252 new->map = (void *)new->nr_deferred + defer_size; 253 254 /* map: set all old bits, clear all new bits */ 255 memset(new->map, (int)0xff, old_map_size); 256 memset((void *)new->map + old_map_size, 0, map_size - old_map_size); 257 /* nr_deferred: copy old values, clear all new values */ 258 memcpy(new->nr_deferred, old->nr_deferred, old_defer_size); 259 memset((void *)new->nr_deferred + old_defer_size, 0, 260 defer_size - old_defer_size); 261 262 rcu_assign_pointer(pn->shrinker_info, new); 263 kvfree_rcu(old, rcu); 264 } 265 266 return 0; 267 } 268 269 void free_shrinker_info(struct mem_cgroup *memcg) 270 { 271 struct mem_cgroup_per_node *pn; 272 struct shrinker_info *info; 273 int nid; 274 275 for_each_node(nid) { 276 pn = memcg->nodeinfo[nid]; 277 info = rcu_dereference_protected(pn->shrinker_info, true); 278 kvfree(info); 279 rcu_assign_pointer(pn->shrinker_info, NULL); 280 } 281 } 282 283 int alloc_shrinker_info(struct mem_cgroup *memcg) 284 { 285 struct shrinker_info *info; 286 int nid, size, ret = 0; 287 int map_size, defer_size = 0; 288 289 down_write(&shrinker_rwsem); 290 map_size = shrinker_map_size(shrinker_nr_max); 291 defer_size = shrinker_defer_size(shrinker_nr_max); 292 size = map_size + defer_size; 293 for_each_node(nid) { 294 info = kvzalloc_node(sizeof(*info) + size, GFP_KERNEL, nid); 295 if (!info) { 296 free_shrinker_info(memcg); 297 ret = -ENOMEM; 298 break; 299 } 300 info->nr_deferred = (atomic_long_t *)(info + 1); 301 info->map = (void *)info->nr_deferred + defer_size; 302 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info); 303 } 304 up_write(&shrinker_rwsem); 305 306 return ret; 307 } 308 309 static inline bool need_expand(int nr_max) 310 { 311 return round_up(nr_max, BITS_PER_LONG) > 312 round_up(shrinker_nr_max, BITS_PER_LONG); 313 } 314 315 static int expand_shrinker_info(int new_id) 316 { 317 int ret = 0; 318 int new_nr_max = new_id + 1; 319 int map_size, defer_size = 0; 320 int old_map_size, old_defer_size = 0; 321 struct mem_cgroup *memcg; 322 323 if (!need_expand(new_nr_max)) 324 goto out; 325 326 if (!root_mem_cgroup) 327 goto out; 328 329 lockdep_assert_held(&shrinker_rwsem); 330 331 map_size = shrinker_map_size(new_nr_max); 332 defer_size = shrinker_defer_size(new_nr_max); 333 old_map_size = shrinker_map_size(shrinker_nr_max); 334 old_defer_size = shrinker_defer_size(shrinker_nr_max); 335 336 memcg = mem_cgroup_iter(NULL, NULL, NULL); 337 do { 338 ret = expand_one_shrinker_info(memcg, map_size, defer_size, 339 old_map_size, old_defer_size); 340 if (ret) { 341 mem_cgroup_iter_break(NULL, memcg); 342 goto out; 343 } 344 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); 345 out: 346 if (!ret) 347 shrinker_nr_max = new_nr_max; 348 349 return ret; 350 } 351 352 void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id) 353 { 354 if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) { 355 struct shrinker_info *info; 356 357 rcu_read_lock(); 358 info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info); 359 /* Pairs with smp mb in shrink_slab() */ 360 smp_mb__before_atomic(); 361 set_bit(shrinker_id, info->map); 362 rcu_read_unlock(); 363 } 364 } 365 366 static DEFINE_IDR(shrinker_idr); 367 368 static int prealloc_memcg_shrinker(struct shrinker *shrinker) 369 { 370 int id, ret = -ENOMEM; 371 372 if (mem_cgroup_disabled()) 373 return -ENOSYS; 374 375 down_write(&shrinker_rwsem); 376 /* This may call shrinker, so it must use down_read_trylock() */ 377 id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL); 378 if (id < 0) 379 goto unlock; 380 381 if (id >= shrinker_nr_max) { 382 if (expand_shrinker_info(id)) { 383 idr_remove(&shrinker_idr, id); 384 goto unlock; 385 } 386 } 387 shrinker->id = id; 388 ret = 0; 389 unlock: 390 up_write(&shrinker_rwsem); 391 return ret; 392 } 393 394 static void unregister_memcg_shrinker(struct shrinker *shrinker) 395 { 396 int id = shrinker->id; 397 398 BUG_ON(id < 0); 399 400 lockdep_assert_held(&shrinker_rwsem); 401 402 idr_remove(&shrinker_idr, id); 403 } 404 405 static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker, 406 struct mem_cgroup *memcg) 407 { 408 struct shrinker_info *info; 409 410 info = shrinker_info_protected(memcg, nid); 411 return atomic_long_xchg(&info->nr_deferred[shrinker->id], 0); 412 } 413 414 static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker, 415 struct mem_cgroup *memcg) 416 { 417 struct shrinker_info *info; 418 419 info = shrinker_info_protected(memcg, nid); 420 return atomic_long_add_return(nr, &info->nr_deferred[shrinker->id]); 421 } 422 423 void reparent_shrinker_deferred(struct mem_cgroup *memcg) 424 { 425 int i, nid; 426 long nr; 427 struct mem_cgroup *parent; 428 struct shrinker_info *child_info, *parent_info; 429 430 parent = parent_mem_cgroup(memcg); 431 if (!parent) 432 parent = root_mem_cgroup; 433 434 /* Prevent from concurrent shrinker_info expand */ 435 down_read(&shrinker_rwsem); 436 for_each_node(nid) { 437 child_info = shrinker_info_protected(memcg, nid); 438 parent_info = shrinker_info_protected(parent, nid); 439 for (i = 0; i < shrinker_nr_max; i++) { 440 nr = atomic_long_read(&child_info->nr_deferred[i]); 441 atomic_long_add(nr, &parent_info->nr_deferred[i]); 442 } 443 } 444 up_read(&shrinker_rwsem); 445 } 446 447 static bool cgroup_reclaim(struct scan_control *sc) 448 { 449 return sc->target_mem_cgroup; 450 } 451 452 /** 453 * writeback_throttling_sane - is the usual dirty throttling mechanism available? 454 * @sc: scan_control in question 455 * 456 * The normal page dirty throttling mechanism in balance_dirty_pages() is 457 * completely broken with the legacy memcg and direct stalling in 458 * shrink_folio_list() is used for throttling instead, which lacks all the 459 * niceties such as fairness, adaptive pausing, bandwidth proportional 460 * allocation and configurability. 461 * 462 * This function tests whether the vmscan currently in progress can assume 463 * that the normal dirty throttling mechanism is operational. 464 */ 465 static bool writeback_throttling_sane(struct scan_control *sc) 466 { 467 if (!cgroup_reclaim(sc)) 468 return true; 469 #ifdef CONFIG_CGROUP_WRITEBACK 470 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 471 return true; 472 #endif 473 return false; 474 } 475 #else 476 static int prealloc_memcg_shrinker(struct shrinker *shrinker) 477 { 478 return -ENOSYS; 479 } 480 481 static void unregister_memcg_shrinker(struct shrinker *shrinker) 482 { 483 } 484 485 static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker, 486 struct mem_cgroup *memcg) 487 { 488 return 0; 489 } 490 491 static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker, 492 struct mem_cgroup *memcg) 493 { 494 return 0; 495 } 496 497 static bool cgroup_reclaim(struct scan_control *sc) 498 { 499 return false; 500 } 501 502 static bool writeback_throttling_sane(struct scan_control *sc) 503 { 504 return true; 505 } 506 #endif 507 508 static long xchg_nr_deferred(struct shrinker *shrinker, 509 struct shrink_control *sc) 510 { 511 int nid = sc->nid; 512 513 if (!(shrinker->flags & SHRINKER_NUMA_AWARE)) 514 nid = 0; 515 516 if (sc->memcg && 517 (shrinker->flags & SHRINKER_MEMCG_AWARE)) 518 return xchg_nr_deferred_memcg(nid, shrinker, 519 sc->memcg); 520 521 return atomic_long_xchg(&shrinker->nr_deferred[nid], 0); 522 } 523 524 525 static long add_nr_deferred(long nr, struct shrinker *shrinker, 526 struct shrink_control *sc) 527 { 528 int nid = sc->nid; 529 530 if (!(shrinker->flags & SHRINKER_NUMA_AWARE)) 531 nid = 0; 532 533 if (sc->memcg && 534 (shrinker->flags & SHRINKER_MEMCG_AWARE)) 535 return add_nr_deferred_memcg(nr, nid, shrinker, 536 sc->memcg); 537 538 return atomic_long_add_return(nr, &shrinker->nr_deferred[nid]); 539 } 540 541 static bool can_demote(int nid, struct scan_control *sc) 542 { 543 if (!numa_demotion_enabled) 544 return false; 545 if (sc && sc->no_demotion) 546 return false; 547 if (next_demotion_node(nid) == NUMA_NO_NODE) 548 return false; 549 550 return true; 551 } 552 553 static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg, 554 int nid, 555 struct scan_control *sc) 556 { 557 if (memcg == NULL) { 558 /* 559 * For non-memcg reclaim, is there 560 * space in any swap device? 561 */ 562 if (get_nr_swap_pages() > 0) 563 return true; 564 } else { 565 /* Is the memcg below its swap limit? */ 566 if (mem_cgroup_get_nr_swap_pages(memcg) > 0) 567 return true; 568 } 569 570 /* 571 * The page can not be swapped. 572 * 573 * Can it be reclaimed from this node via demotion? 574 */ 575 return can_demote(nid, sc); 576 } 577 578 /* 579 * This misses isolated folios which are not accounted for to save counters. 580 * As the data only determines if reclaim or compaction continues, it is 581 * not expected that isolated folios will be a dominating factor. 582 */ 583 unsigned long zone_reclaimable_pages(struct zone *zone) 584 { 585 unsigned long nr; 586 587 nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) + 588 zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE); 589 if (can_reclaim_anon_pages(NULL, zone_to_nid(zone), NULL)) 590 nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) + 591 zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON); 592 593 return nr; 594 } 595 596 /** 597 * lruvec_lru_size - Returns the number of pages on the given LRU list. 598 * @lruvec: lru vector 599 * @lru: lru to use 600 * @zone_idx: zones to consider (use MAX_NR_ZONES - 1 for the whole LRU list) 601 */ 602 static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, 603 int zone_idx) 604 { 605 unsigned long size = 0; 606 int zid; 607 608 for (zid = 0; zid <= zone_idx; zid++) { 609 struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid]; 610 611 if (!managed_zone(zone)) 612 continue; 613 614 if (!mem_cgroup_disabled()) 615 size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid); 616 else 617 size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru); 618 } 619 return size; 620 } 621 622 /* 623 * Add a shrinker callback to be called from the vm. 624 */ 625 static int __prealloc_shrinker(struct shrinker *shrinker) 626 { 627 unsigned int size; 628 int err; 629 630 if (shrinker->flags & SHRINKER_MEMCG_AWARE) { 631 err = prealloc_memcg_shrinker(shrinker); 632 if (err != -ENOSYS) 633 return err; 634 635 shrinker->flags &= ~SHRINKER_MEMCG_AWARE; 636 } 637 638 size = sizeof(*shrinker->nr_deferred); 639 if (shrinker->flags & SHRINKER_NUMA_AWARE) 640 size *= nr_node_ids; 641 642 shrinker->nr_deferred = kzalloc(size, GFP_KERNEL); 643 if (!shrinker->nr_deferred) 644 return -ENOMEM; 645 646 return 0; 647 } 648 649 #ifdef CONFIG_SHRINKER_DEBUG 650 int prealloc_shrinker(struct shrinker *shrinker, const char *fmt, ...) 651 { 652 va_list ap; 653 int err; 654 655 va_start(ap, fmt); 656 shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap); 657 va_end(ap); 658 if (!shrinker->name) 659 return -ENOMEM; 660 661 err = __prealloc_shrinker(shrinker); 662 if (err) { 663 kfree_const(shrinker->name); 664 shrinker->name = NULL; 665 } 666 667 return err; 668 } 669 #else 670 int prealloc_shrinker(struct shrinker *shrinker, const char *fmt, ...) 671 { 672 return __prealloc_shrinker(shrinker); 673 } 674 #endif 675 676 void free_prealloced_shrinker(struct shrinker *shrinker) 677 { 678 #ifdef CONFIG_SHRINKER_DEBUG 679 kfree_const(shrinker->name); 680 shrinker->name = NULL; 681 #endif 682 if (shrinker->flags & SHRINKER_MEMCG_AWARE) { 683 down_write(&shrinker_rwsem); 684 unregister_memcg_shrinker(shrinker); 685 up_write(&shrinker_rwsem); 686 return; 687 } 688 689 kfree(shrinker->nr_deferred); 690 shrinker->nr_deferred = NULL; 691 } 692 693 void register_shrinker_prepared(struct shrinker *shrinker) 694 { 695 down_write(&shrinker_rwsem); 696 list_add_tail(&shrinker->list, &shrinker_list); 697 shrinker->flags |= SHRINKER_REGISTERED; 698 shrinker_debugfs_add(shrinker); 699 up_write(&shrinker_rwsem); 700 } 701 702 static int __register_shrinker(struct shrinker *shrinker) 703 { 704 int err = __prealloc_shrinker(shrinker); 705 706 if (err) 707 return err; 708 register_shrinker_prepared(shrinker); 709 return 0; 710 } 711 712 #ifdef CONFIG_SHRINKER_DEBUG 713 int register_shrinker(struct shrinker *shrinker, const char *fmt, ...) 714 { 715 va_list ap; 716 int err; 717 718 va_start(ap, fmt); 719 shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap); 720 va_end(ap); 721 if (!shrinker->name) 722 return -ENOMEM; 723 724 err = __register_shrinker(shrinker); 725 if (err) { 726 kfree_const(shrinker->name); 727 shrinker->name = NULL; 728 } 729 return err; 730 } 731 #else 732 int register_shrinker(struct shrinker *shrinker, const char *fmt, ...) 733 { 734 return __register_shrinker(shrinker); 735 } 736 #endif 737 EXPORT_SYMBOL(register_shrinker); 738 739 /* 740 * Remove one 741 */ 742 void unregister_shrinker(struct shrinker *shrinker) 743 { 744 struct dentry *debugfs_entry; 745 746 if (!(shrinker->flags & SHRINKER_REGISTERED)) 747 return; 748 749 down_write(&shrinker_rwsem); 750 list_del(&shrinker->list); 751 shrinker->flags &= ~SHRINKER_REGISTERED; 752 if (shrinker->flags & SHRINKER_MEMCG_AWARE) 753 unregister_memcg_shrinker(shrinker); 754 debugfs_entry = shrinker_debugfs_remove(shrinker); 755 up_write(&shrinker_rwsem); 756 757 debugfs_remove_recursive(debugfs_entry); 758 759 kfree(shrinker->nr_deferred); 760 shrinker->nr_deferred = NULL; 761 } 762 EXPORT_SYMBOL(unregister_shrinker); 763 764 /** 765 * synchronize_shrinkers - Wait for all running shrinkers to complete. 766 * 767 * This is equivalent to calling unregister_shrink() and register_shrinker(), 768 * but atomically and with less overhead. This is useful to guarantee that all 769 * shrinker invocations have seen an update, before freeing memory, similar to 770 * rcu. 771 */ 772 void synchronize_shrinkers(void) 773 { 774 down_write(&shrinker_rwsem); 775 up_write(&shrinker_rwsem); 776 } 777 EXPORT_SYMBOL(synchronize_shrinkers); 778 779 #define SHRINK_BATCH 128 780 781 static unsigned long do_shrink_slab(struct shrink_control *shrinkctl, 782 struct shrinker *shrinker, int priority) 783 { 784 unsigned long freed = 0; 785 unsigned long long delta; 786 long total_scan; 787 long freeable; 788 long nr; 789 long new_nr; 790 long batch_size = shrinker->batch ? shrinker->batch 791 : SHRINK_BATCH; 792 long scanned = 0, next_deferred; 793 794 freeable = shrinker->count_objects(shrinker, shrinkctl); 795 if (freeable == 0 || freeable == SHRINK_EMPTY) 796 return freeable; 797 798 /* 799 * copy the current shrinker scan count into a local variable 800 * and zero it so that other concurrent shrinker invocations 801 * don't also do this scanning work. 802 */ 803 nr = xchg_nr_deferred(shrinker, shrinkctl); 804 805 if (shrinker->seeks) { 806 delta = freeable >> priority; 807 delta *= 4; 808 do_div(delta, shrinker->seeks); 809 } else { 810 /* 811 * These objects don't require any IO to create. Trim 812 * them aggressively under memory pressure to keep 813 * them from causing refetches in the IO caches. 814 */ 815 delta = freeable / 2; 816 } 817 818 total_scan = nr >> priority; 819 total_scan += delta; 820 total_scan = min(total_scan, (2 * freeable)); 821 822 trace_mm_shrink_slab_start(shrinker, shrinkctl, nr, 823 freeable, delta, total_scan, priority); 824 825 /* 826 * Normally, we should not scan less than batch_size objects in one 827 * pass to avoid too frequent shrinker calls, but if the slab has less 828 * than batch_size objects in total and we are really tight on memory, 829 * we will try to reclaim all available objects, otherwise we can end 830 * up failing allocations although there are plenty of reclaimable 831 * objects spread over several slabs with usage less than the 832 * batch_size. 833 * 834 * We detect the "tight on memory" situations by looking at the total 835 * number of objects we want to scan (total_scan). If it is greater 836 * than the total number of objects on slab (freeable), we must be 837 * scanning at high prio and therefore should try to reclaim as much as 838 * possible. 839 */ 840 while (total_scan >= batch_size || 841 total_scan >= freeable) { 842 unsigned long ret; 843 unsigned long nr_to_scan = min(batch_size, total_scan); 844 845 shrinkctl->nr_to_scan = nr_to_scan; 846 shrinkctl->nr_scanned = nr_to_scan; 847 ret = shrinker->scan_objects(shrinker, shrinkctl); 848 if (ret == SHRINK_STOP) 849 break; 850 freed += ret; 851 852 count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned); 853 total_scan -= shrinkctl->nr_scanned; 854 scanned += shrinkctl->nr_scanned; 855 856 cond_resched(); 857 } 858 859 /* 860 * The deferred work is increased by any new work (delta) that wasn't 861 * done, decreased by old deferred work that was done now. 862 * 863 * And it is capped to two times of the freeable items. 864 */ 865 next_deferred = max_t(long, (nr + delta - scanned), 0); 866 next_deferred = min(next_deferred, (2 * freeable)); 867 868 /* 869 * move the unused scan count back into the shrinker in a 870 * manner that handles concurrent updates. 871 */ 872 new_nr = add_nr_deferred(next_deferred, shrinker, shrinkctl); 873 874 trace_mm_shrink_slab_end(shrinker, shrinkctl->nid, freed, nr, new_nr, total_scan); 875 return freed; 876 } 877 878 #ifdef CONFIG_MEMCG 879 static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, 880 struct mem_cgroup *memcg, int priority) 881 { 882 struct shrinker_info *info; 883 unsigned long ret, freed = 0; 884 int i; 885 886 if (!mem_cgroup_online(memcg)) 887 return 0; 888 889 if (!down_read_trylock(&shrinker_rwsem)) 890 return 0; 891 892 info = shrinker_info_protected(memcg, nid); 893 if (unlikely(!info)) 894 goto unlock; 895 896 for_each_set_bit(i, info->map, shrinker_nr_max) { 897 struct shrink_control sc = { 898 .gfp_mask = gfp_mask, 899 .nid = nid, 900 .memcg = memcg, 901 }; 902 struct shrinker *shrinker; 903 904 shrinker = idr_find(&shrinker_idr, i); 905 if (unlikely(!shrinker || !(shrinker->flags & SHRINKER_REGISTERED))) { 906 if (!shrinker) 907 clear_bit(i, info->map); 908 continue; 909 } 910 911 /* Call non-slab shrinkers even though kmem is disabled */ 912 if (!memcg_kmem_enabled() && 913 !(shrinker->flags & SHRINKER_NONSLAB)) 914 continue; 915 916 ret = do_shrink_slab(&sc, shrinker, priority); 917 if (ret == SHRINK_EMPTY) { 918 clear_bit(i, info->map); 919 /* 920 * After the shrinker reported that it had no objects to 921 * free, but before we cleared the corresponding bit in 922 * the memcg shrinker map, a new object might have been 923 * added. To make sure, we have the bit set in this 924 * case, we invoke the shrinker one more time and reset 925 * the bit if it reports that it is not empty anymore. 926 * The memory barrier here pairs with the barrier in 927 * set_shrinker_bit(): 928 * 929 * list_lru_add() shrink_slab_memcg() 930 * list_add_tail() clear_bit() 931 * <MB> <MB> 932 * set_bit() do_shrink_slab() 933 */ 934 smp_mb__after_atomic(); 935 ret = do_shrink_slab(&sc, shrinker, priority); 936 if (ret == SHRINK_EMPTY) 937 ret = 0; 938 else 939 set_shrinker_bit(memcg, nid, i); 940 } 941 freed += ret; 942 943 if (rwsem_is_contended(&shrinker_rwsem)) { 944 freed = freed ? : 1; 945 break; 946 } 947 } 948 unlock: 949 up_read(&shrinker_rwsem); 950 return freed; 951 } 952 #else /* CONFIG_MEMCG */ 953 static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, 954 struct mem_cgroup *memcg, int priority) 955 { 956 return 0; 957 } 958 #endif /* CONFIG_MEMCG */ 959 960 /** 961 * shrink_slab - shrink slab caches 962 * @gfp_mask: allocation context 963 * @nid: node whose slab caches to target 964 * @memcg: memory cgroup whose slab caches to target 965 * @priority: the reclaim priority 966 * 967 * Call the shrink functions to age shrinkable caches. 968 * 969 * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set, 970 * unaware shrinkers will receive a node id of 0 instead. 971 * 972 * @memcg specifies the memory cgroup to target. Unaware shrinkers 973 * are called only if it is the root cgroup. 974 * 975 * @priority is sc->priority, we take the number of objects and >> by priority 976 * in order to get the scan target. 977 * 978 * Returns the number of reclaimed slab objects. 979 */ 980 static unsigned long shrink_slab(gfp_t gfp_mask, int nid, 981 struct mem_cgroup *memcg, 982 int priority) 983 { 984 unsigned long ret, freed = 0; 985 struct shrinker *shrinker; 986 987 /* 988 * The root memcg might be allocated even though memcg is disabled 989 * via "cgroup_disable=memory" boot parameter. This could make 990 * mem_cgroup_is_root() return false, then just run memcg slab 991 * shrink, but skip global shrink. This may result in premature 992 * oom. 993 */ 994 if (!mem_cgroup_disabled() && !mem_cgroup_is_root(memcg)) 995 return shrink_slab_memcg(gfp_mask, nid, memcg, priority); 996 997 if (!down_read_trylock(&shrinker_rwsem)) 998 goto out; 999 1000 list_for_each_entry(shrinker, &shrinker_list, list) { 1001 struct shrink_control sc = { 1002 .gfp_mask = gfp_mask, 1003 .nid = nid, 1004 .memcg = memcg, 1005 }; 1006 1007 ret = do_shrink_slab(&sc, shrinker, priority); 1008 if (ret == SHRINK_EMPTY) 1009 ret = 0; 1010 freed += ret; 1011 /* 1012 * Bail out if someone want to register a new shrinker to 1013 * prevent the registration from being stalled for long periods 1014 * by parallel ongoing shrinking. 1015 */ 1016 if (rwsem_is_contended(&shrinker_rwsem)) { 1017 freed = freed ? : 1; 1018 break; 1019 } 1020 } 1021 1022 up_read(&shrinker_rwsem); 1023 out: 1024 cond_resched(); 1025 return freed; 1026 } 1027 1028 static unsigned long drop_slab_node(int nid) 1029 { 1030 unsigned long freed = 0; 1031 struct mem_cgroup *memcg = NULL; 1032 1033 memcg = mem_cgroup_iter(NULL, NULL, NULL); 1034 do { 1035 freed += shrink_slab(GFP_KERNEL, nid, memcg, 0); 1036 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); 1037 1038 return freed; 1039 } 1040 1041 void drop_slab(void) 1042 { 1043 int nid; 1044 int shift = 0; 1045 unsigned long freed; 1046 1047 do { 1048 freed = 0; 1049 for_each_online_node(nid) { 1050 if (fatal_signal_pending(current)) 1051 return; 1052 1053 freed += drop_slab_node(nid); 1054 } 1055 } while ((freed >> shift++) > 1); 1056 } 1057 1058 static int reclaimer_offset(void) 1059 { 1060 BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD != 1061 PGDEMOTE_DIRECT - PGDEMOTE_KSWAPD); 1062 BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD != 1063 PGSCAN_DIRECT - PGSCAN_KSWAPD); 1064 BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD != 1065 PGDEMOTE_KHUGEPAGED - PGDEMOTE_KSWAPD); 1066 BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD != 1067 PGSCAN_KHUGEPAGED - PGSCAN_KSWAPD); 1068 1069 if (current_is_kswapd()) 1070 return 0; 1071 if (current_is_khugepaged()) 1072 return PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD; 1073 return PGSTEAL_DIRECT - PGSTEAL_KSWAPD; 1074 } 1075 1076 static inline int is_page_cache_freeable(struct folio *folio) 1077 { 1078 /* 1079 * A freeable page cache folio is referenced only by the caller 1080 * that isolated the folio, the page cache and optional filesystem 1081 * private data at folio->private. 1082 */ 1083 return folio_ref_count(folio) - folio_test_private(folio) == 1084 1 + folio_nr_pages(folio); 1085 } 1086 1087 /* 1088 * We detected a synchronous write error writing a folio out. Probably 1089 * -ENOSPC. We need to propagate that into the address_space for a subsequent 1090 * fsync(), msync() or close(). 1091 * 1092 * The tricky part is that after writepage we cannot touch the mapping: nothing 1093 * prevents it from being freed up. But we have a ref on the folio and once 1094 * that folio is locked, the mapping is pinned. 1095 * 1096 * We're allowed to run sleeping folio_lock() here because we know the caller has 1097 * __GFP_FS. 1098 */ 1099 static void handle_write_error(struct address_space *mapping, 1100 struct folio *folio, int error) 1101 { 1102 folio_lock(folio); 1103 if (folio_mapping(folio) == mapping) 1104 mapping_set_error(mapping, error); 1105 folio_unlock(folio); 1106 } 1107 1108 static bool skip_throttle_noprogress(pg_data_t *pgdat) 1109 { 1110 int reclaimable = 0, write_pending = 0; 1111 int i; 1112 1113 /* 1114 * If kswapd is disabled, reschedule if necessary but do not 1115 * throttle as the system is likely near OOM. 1116 */ 1117 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) 1118 return true; 1119 1120 /* 1121 * If there are a lot of dirty/writeback folios then do not 1122 * throttle as throttling will occur when the folios cycle 1123 * towards the end of the LRU if still under writeback. 1124 */ 1125 for (i = 0; i < MAX_NR_ZONES; i++) { 1126 struct zone *zone = pgdat->node_zones + i; 1127 1128 if (!managed_zone(zone)) 1129 continue; 1130 1131 reclaimable += zone_reclaimable_pages(zone); 1132 write_pending += zone_page_state_snapshot(zone, 1133 NR_ZONE_WRITE_PENDING); 1134 } 1135 if (2 * write_pending <= reclaimable) 1136 return true; 1137 1138 return false; 1139 } 1140 1141 void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason) 1142 { 1143 wait_queue_head_t *wqh = &pgdat->reclaim_wait[reason]; 1144 long timeout, ret; 1145 DEFINE_WAIT(wait); 1146 1147 /* 1148 * Do not throttle IO workers, kthreads other than kswapd or 1149 * workqueues. They may be required for reclaim to make 1150 * forward progress (e.g. journalling workqueues or kthreads). 1151 */ 1152 if (!current_is_kswapd() && 1153 current->flags & (PF_IO_WORKER|PF_KTHREAD)) { 1154 cond_resched(); 1155 return; 1156 } 1157 1158 /* 1159 * These figures are pulled out of thin air. 1160 * VMSCAN_THROTTLE_ISOLATED is a transient condition based on too many 1161 * parallel reclaimers which is a short-lived event so the timeout is 1162 * short. Failing to make progress or waiting on writeback are 1163 * potentially long-lived events so use a longer timeout. This is shaky 1164 * logic as a failure to make progress could be due to anything from 1165 * writeback to a slow device to excessive referenced folios at the tail 1166 * of the inactive LRU. 1167 */ 1168 switch(reason) { 1169 case VMSCAN_THROTTLE_WRITEBACK: 1170 timeout = HZ/10; 1171 1172 if (atomic_inc_return(&pgdat->nr_writeback_throttled) == 1) { 1173 WRITE_ONCE(pgdat->nr_reclaim_start, 1174 node_page_state(pgdat, NR_THROTTLED_WRITTEN)); 1175 } 1176 1177 break; 1178 case VMSCAN_THROTTLE_CONGESTED: 1179 fallthrough; 1180 case VMSCAN_THROTTLE_NOPROGRESS: 1181 if (skip_throttle_noprogress(pgdat)) { 1182 cond_resched(); 1183 return; 1184 } 1185 1186 timeout = 1; 1187 1188 break; 1189 case VMSCAN_THROTTLE_ISOLATED: 1190 timeout = HZ/50; 1191 break; 1192 default: 1193 WARN_ON_ONCE(1); 1194 timeout = HZ; 1195 break; 1196 } 1197 1198 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); 1199 ret = schedule_timeout(timeout); 1200 finish_wait(wqh, &wait); 1201 1202 if (reason == VMSCAN_THROTTLE_WRITEBACK) 1203 atomic_dec(&pgdat->nr_writeback_throttled); 1204 1205 trace_mm_vmscan_throttled(pgdat->node_id, jiffies_to_usecs(timeout), 1206 jiffies_to_usecs(timeout - ret), 1207 reason); 1208 } 1209 1210 /* 1211 * Account for folios written if tasks are throttled waiting on dirty 1212 * folios to clean. If enough folios have been cleaned since throttling 1213 * started then wakeup the throttled tasks. 1214 */ 1215 void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, 1216 int nr_throttled) 1217 { 1218 unsigned long nr_written; 1219 1220 node_stat_add_folio(folio, NR_THROTTLED_WRITTEN); 1221 1222 /* 1223 * This is an inaccurate read as the per-cpu deltas may not 1224 * be synchronised. However, given that the system is 1225 * writeback throttled, it is not worth taking the penalty 1226 * of getting an accurate count. At worst, the throttle 1227 * timeout guarantees forward progress. 1228 */ 1229 nr_written = node_page_state(pgdat, NR_THROTTLED_WRITTEN) - 1230 READ_ONCE(pgdat->nr_reclaim_start); 1231 1232 if (nr_written > SWAP_CLUSTER_MAX * nr_throttled) 1233 wake_up(&pgdat->reclaim_wait[VMSCAN_THROTTLE_WRITEBACK]); 1234 } 1235 1236 /* possible outcome of pageout() */ 1237 typedef enum { 1238 /* failed to write folio out, folio is locked */ 1239 PAGE_KEEP, 1240 /* move folio to the active list, folio is locked */ 1241 PAGE_ACTIVATE, 1242 /* folio has been sent to the disk successfully, folio is unlocked */ 1243 PAGE_SUCCESS, 1244 /* folio is clean and locked */ 1245 PAGE_CLEAN, 1246 } pageout_t; 1247 1248 /* 1249 * pageout is called by shrink_folio_list() for each dirty folio. 1250 * Calls ->writepage(). 1251 */ 1252 static pageout_t pageout(struct folio *folio, struct address_space *mapping, 1253 struct swap_iocb **plug) 1254 { 1255 /* 1256 * If the folio is dirty, only perform writeback if that write 1257 * will be non-blocking. To prevent this allocation from being 1258 * stalled by pagecache activity. But note that there may be 1259 * stalls if we need to run get_block(). We could test 1260 * PagePrivate for that. 1261 * 1262 * If this process is currently in __generic_file_write_iter() against 1263 * this folio's queue, we can perform writeback even if that 1264 * will block. 1265 * 1266 * If the folio is swapcache, write it back even if that would 1267 * block, for some throttling. This happens by accident, because 1268 * swap_backing_dev_info is bust: it doesn't reflect the 1269 * congestion state of the swapdevs. Easy to fix, if needed. 1270 */ 1271 if (!is_page_cache_freeable(folio)) 1272 return PAGE_KEEP; 1273 if (!mapping) { 1274 /* 1275 * Some data journaling orphaned folios can have 1276 * folio->mapping == NULL while being dirty with clean buffers. 1277 */ 1278 if (folio_test_private(folio)) { 1279 if (try_to_free_buffers(folio)) { 1280 folio_clear_dirty(folio); 1281 pr_info("%s: orphaned folio\n", __func__); 1282 return PAGE_CLEAN; 1283 } 1284 } 1285 return PAGE_KEEP; 1286 } 1287 if (mapping->a_ops->writepage == NULL) 1288 return PAGE_ACTIVATE; 1289 1290 if (folio_clear_dirty_for_io(folio)) { 1291 int res; 1292 struct writeback_control wbc = { 1293 .sync_mode = WB_SYNC_NONE, 1294 .nr_to_write = SWAP_CLUSTER_MAX, 1295 .range_start = 0, 1296 .range_end = LLONG_MAX, 1297 .for_reclaim = 1, 1298 .swap_plug = plug, 1299 }; 1300 1301 folio_set_reclaim(folio); 1302 res = mapping->a_ops->writepage(&folio->page, &wbc); 1303 if (res < 0) 1304 handle_write_error(mapping, folio, res); 1305 if (res == AOP_WRITEPAGE_ACTIVATE) { 1306 folio_clear_reclaim(folio); 1307 return PAGE_ACTIVATE; 1308 } 1309 1310 if (!folio_test_writeback(folio)) { 1311 /* synchronous write or broken a_ops? */ 1312 folio_clear_reclaim(folio); 1313 } 1314 trace_mm_vmscan_write_folio(folio); 1315 node_stat_add_folio(folio, NR_VMSCAN_WRITE); 1316 return PAGE_SUCCESS; 1317 } 1318 1319 return PAGE_CLEAN; 1320 } 1321 1322 /* 1323 * Same as remove_mapping, but if the folio is removed from the mapping, it 1324 * gets returned with a refcount of 0. 1325 */ 1326 static int __remove_mapping(struct address_space *mapping, struct folio *folio, 1327 bool reclaimed, struct mem_cgroup *target_memcg) 1328 { 1329 int refcount; 1330 void *shadow = NULL; 1331 1332 BUG_ON(!folio_test_locked(folio)); 1333 BUG_ON(mapping != folio_mapping(folio)); 1334 1335 if (!folio_test_swapcache(folio)) 1336 spin_lock(&mapping->host->i_lock); 1337 xa_lock_irq(&mapping->i_pages); 1338 /* 1339 * The non racy check for a busy folio. 1340 * 1341 * Must be careful with the order of the tests. When someone has 1342 * a ref to the folio, it may be possible that they dirty it then 1343 * drop the reference. So if the dirty flag is tested before the 1344 * refcount here, then the following race may occur: 1345 * 1346 * get_user_pages(&page); 1347 * [user mapping goes away] 1348 * write_to(page); 1349 * !folio_test_dirty(folio) [good] 1350 * folio_set_dirty(folio); 1351 * folio_put(folio); 1352 * !refcount(folio) [good, discard it] 1353 * 1354 * [oops, our write_to data is lost] 1355 * 1356 * Reversing the order of the tests ensures such a situation cannot 1357 * escape unnoticed. The smp_rmb is needed to ensure the folio->flags 1358 * load is not satisfied before that of folio->_refcount. 1359 * 1360 * Note that if the dirty flag is always set via folio_mark_dirty, 1361 * and thus under the i_pages lock, then this ordering is not required. 1362 */ 1363 refcount = 1 + folio_nr_pages(folio); 1364 if (!folio_ref_freeze(folio, refcount)) 1365 goto cannot_free; 1366 /* note: atomic_cmpxchg in folio_ref_freeze provides the smp_rmb */ 1367 if (unlikely(folio_test_dirty(folio))) { 1368 folio_ref_unfreeze(folio, refcount); 1369 goto cannot_free; 1370 } 1371 1372 if (folio_test_swapcache(folio)) { 1373 swp_entry_t swap = folio_swap_entry(folio); 1374 1375 if (reclaimed && !mapping_exiting(mapping)) 1376 shadow = workingset_eviction(folio, target_memcg); 1377 __delete_from_swap_cache(folio, swap, shadow); 1378 mem_cgroup_swapout(folio, swap); 1379 xa_unlock_irq(&mapping->i_pages); 1380 put_swap_folio(folio, swap); 1381 } else { 1382 void (*free_folio)(struct folio *); 1383 1384 free_folio = mapping->a_ops->free_folio; 1385 /* 1386 * Remember a shadow entry for reclaimed file cache in 1387 * order to detect refaults, thus thrashing, later on. 1388 * 1389 * But don't store shadows in an address space that is 1390 * already exiting. This is not just an optimization, 1391 * inode reclaim needs to empty out the radix tree or 1392 * the nodes are lost. Don't plant shadows behind its 1393 * back. 1394 * 1395 * We also don't store shadows for DAX mappings because the 1396 * only page cache folios found in these are zero pages 1397 * covering holes, and because we don't want to mix DAX 1398 * exceptional entries and shadow exceptional entries in the 1399 * same address_space. 1400 */ 1401 if (reclaimed && folio_is_file_lru(folio) && 1402 !mapping_exiting(mapping) && !dax_mapping(mapping)) 1403 shadow = workingset_eviction(folio, target_memcg); 1404 __filemap_remove_folio(folio, shadow); 1405 xa_unlock_irq(&mapping->i_pages); 1406 if (mapping_shrinkable(mapping)) 1407 inode_add_lru(mapping->host); 1408 spin_unlock(&mapping->host->i_lock); 1409 1410 if (free_folio) 1411 free_folio(folio); 1412 } 1413 1414 return 1; 1415 1416 cannot_free: 1417 xa_unlock_irq(&mapping->i_pages); 1418 if (!folio_test_swapcache(folio)) 1419 spin_unlock(&mapping->host->i_lock); 1420 return 0; 1421 } 1422 1423 /** 1424 * remove_mapping() - Attempt to remove a folio from its mapping. 1425 * @mapping: The address space. 1426 * @folio: The folio to remove. 1427 * 1428 * If the folio is dirty, under writeback or if someone else has a ref 1429 * on it, removal will fail. 1430 * Return: The number of pages removed from the mapping. 0 if the folio 1431 * could not be removed. 1432 * Context: The caller should have a single refcount on the folio and 1433 * hold its lock. 1434 */ 1435 long remove_mapping(struct address_space *mapping, struct folio *folio) 1436 { 1437 if (__remove_mapping(mapping, folio, false, NULL)) { 1438 /* 1439 * Unfreezing the refcount with 1 effectively 1440 * drops the pagecache ref for us without requiring another 1441 * atomic operation. 1442 */ 1443 folio_ref_unfreeze(folio, 1); 1444 return folio_nr_pages(folio); 1445 } 1446 return 0; 1447 } 1448 1449 /** 1450 * folio_putback_lru - Put previously isolated folio onto appropriate LRU list. 1451 * @folio: Folio to be returned to an LRU list. 1452 * 1453 * Add previously isolated @folio to appropriate LRU list. 1454 * The folio may still be unevictable for other reasons. 1455 * 1456 * Context: lru_lock must not be held, interrupts must be enabled. 1457 */ 1458 void folio_putback_lru(struct folio *folio) 1459 { 1460 folio_add_lru(folio); 1461 folio_put(folio); /* drop ref from isolate */ 1462 } 1463 1464 enum folio_references { 1465 FOLIOREF_RECLAIM, 1466 FOLIOREF_RECLAIM_CLEAN, 1467 FOLIOREF_KEEP, 1468 FOLIOREF_ACTIVATE, 1469 }; 1470 1471 static enum folio_references folio_check_references(struct folio *folio, 1472 struct scan_control *sc) 1473 { 1474 int referenced_ptes, referenced_folio; 1475 unsigned long vm_flags; 1476 1477 referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup, 1478 &vm_flags); 1479 referenced_folio = folio_test_clear_referenced(folio); 1480 1481 /* 1482 * The supposedly reclaimable folio was found to be in a VM_LOCKED vma. 1483 * Let the folio, now marked Mlocked, be moved to the unevictable list. 1484 */ 1485 if (vm_flags & VM_LOCKED) 1486 return FOLIOREF_ACTIVATE; 1487 1488 /* rmap lock contention: rotate */ 1489 if (referenced_ptes == -1) 1490 return FOLIOREF_KEEP; 1491 1492 if (referenced_ptes) { 1493 /* 1494 * All mapped folios start out with page table 1495 * references from the instantiating fault, so we need 1496 * to look twice if a mapped file/anon folio is used more 1497 * than once. 1498 * 1499 * Mark it and spare it for another trip around the 1500 * inactive list. Another page table reference will 1501 * lead to its activation. 1502 * 1503 * Note: the mark is set for activated folios as well 1504 * so that recently deactivated but used folios are 1505 * quickly recovered. 1506 */ 1507 folio_set_referenced(folio); 1508 1509 if (referenced_folio || referenced_ptes > 1) 1510 return FOLIOREF_ACTIVATE; 1511 1512 /* 1513 * Activate file-backed executable folios after first usage. 1514 */ 1515 if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) 1516 return FOLIOREF_ACTIVATE; 1517 1518 return FOLIOREF_KEEP; 1519 } 1520 1521 /* Reclaim if clean, defer dirty folios to writeback */ 1522 if (referenced_folio && folio_is_file_lru(folio)) 1523 return FOLIOREF_RECLAIM_CLEAN; 1524 1525 return FOLIOREF_RECLAIM; 1526 } 1527 1528 /* Check if a folio is dirty or under writeback */ 1529 static void folio_check_dirty_writeback(struct folio *folio, 1530 bool *dirty, bool *writeback) 1531 { 1532 struct address_space *mapping; 1533 1534 /* 1535 * Anonymous folios are not handled by flushers and must be written 1536 * from reclaim context. Do not stall reclaim based on them. 1537 * MADV_FREE anonymous folios are put into inactive file list too. 1538 * They could be mistakenly treated as file lru. So further anon 1539 * test is needed. 1540 */ 1541 if (!folio_is_file_lru(folio) || 1542 (folio_test_anon(folio) && !folio_test_swapbacked(folio))) { 1543 *dirty = false; 1544 *writeback = false; 1545 return; 1546 } 1547 1548 /* By default assume that the folio flags are accurate */ 1549 *dirty = folio_test_dirty(folio); 1550 *writeback = folio_test_writeback(folio); 1551 1552 /* Verify dirty/writeback state if the filesystem supports it */ 1553 if (!folio_test_private(folio)) 1554 return; 1555 1556 mapping = folio_mapping(folio); 1557 if (mapping && mapping->a_ops->is_dirty_writeback) 1558 mapping->a_ops->is_dirty_writeback(folio, dirty, writeback); 1559 } 1560 1561 static struct page *alloc_demote_page(struct page *page, unsigned long private) 1562 { 1563 struct page *target_page; 1564 nodemask_t *allowed_mask; 1565 struct migration_target_control *mtc; 1566 1567 mtc = (struct migration_target_control *)private; 1568 1569 allowed_mask = mtc->nmask; 1570 /* 1571 * make sure we allocate from the target node first also trying to 1572 * demote or reclaim pages from the target node via kswapd if we are 1573 * low on free memory on target node. If we don't do this and if 1574 * we have free memory on the slower(lower) memtier, we would start 1575 * allocating pages from slower(lower) memory tiers without even forcing 1576 * a demotion of cold pages from the target memtier. This can result 1577 * in the kernel placing hot pages in slower(lower) memory tiers. 1578 */ 1579 mtc->nmask = NULL; 1580 mtc->gfp_mask |= __GFP_THISNODE; 1581 target_page = alloc_migration_target(page, (unsigned long)mtc); 1582 if (target_page) 1583 return target_page; 1584 1585 mtc->gfp_mask &= ~__GFP_THISNODE; 1586 mtc->nmask = allowed_mask; 1587 1588 return alloc_migration_target(page, (unsigned long)mtc); 1589 } 1590 1591 /* 1592 * Take folios on @demote_folios and attempt to demote them to another node. 1593 * Folios which are not demoted are left on @demote_folios. 1594 */ 1595 static unsigned int demote_folio_list(struct list_head *demote_folios, 1596 struct pglist_data *pgdat) 1597 { 1598 int target_nid = next_demotion_node(pgdat->node_id); 1599 unsigned int nr_succeeded; 1600 nodemask_t allowed_mask; 1601 1602 struct migration_target_control mtc = { 1603 /* 1604 * Allocate from 'node', or fail quickly and quietly. 1605 * When this happens, 'page' will likely just be discarded 1606 * instead of migrated. 1607 */ 1608 .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN | 1609 __GFP_NOMEMALLOC | GFP_NOWAIT, 1610 .nid = target_nid, 1611 .nmask = &allowed_mask 1612 }; 1613 1614 if (list_empty(demote_folios)) 1615 return 0; 1616 1617 if (target_nid == NUMA_NO_NODE) 1618 return 0; 1619 1620 node_get_allowed_targets(pgdat, &allowed_mask); 1621 1622 /* Demotion ignores all cpuset and mempolicy settings */ 1623 migrate_pages(demote_folios, alloc_demote_page, NULL, 1624 (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION, 1625 &nr_succeeded); 1626 1627 __count_vm_events(PGDEMOTE_KSWAPD + reclaimer_offset(), nr_succeeded); 1628 1629 return nr_succeeded; 1630 } 1631 1632 static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask) 1633 { 1634 if (gfp_mask & __GFP_FS) 1635 return true; 1636 if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO)) 1637 return false; 1638 /* 1639 * We can "enter_fs" for swap-cache with only __GFP_IO 1640 * providing this isn't SWP_FS_OPS. 1641 * ->flags can be updated non-atomicially (scan_swap_map_slots), 1642 * but that will never affect SWP_FS_OPS, so the data_race 1643 * is safe. 1644 */ 1645 return !data_race(folio_swap_flags(folio) & SWP_FS_OPS); 1646 } 1647 1648 /* 1649 * shrink_folio_list() returns the number of reclaimed pages 1650 */ 1651 static unsigned int shrink_folio_list(struct list_head *folio_list, 1652 struct pglist_data *pgdat, struct scan_control *sc, 1653 struct reclaim_stat *stat, bool ignore_references) 1654 { 1655 LIST_HEAD(ret_folios); 1656 LIST_HEAD(free_folios); 1657 LIST_HEAD(demote_folios); 1658 unsigned int nr_reclaimed = 0; 1659 unsigned int pgactivate = 0; 1660 bool do_demote_pass; 1661 struct swap_iocb *plug = NULL; 1662 1663 memset(stat, 0, sizeof(*stat)); 1664 cond_resched(); 1665 do_demote_pass = can_demote(pgdat->node_id, sc); 1666 1667 retry: 1668 while (!list_empty(folio_list)) { 1669 struct address_space *mapping; 1670 struct folio *folio; 1671 enum folio_references references = FOLIOREF_RECLAIM; 1672 bool dirty, writeback; 1673 unsigned int nr_pages; 1674 1675 cond_resched(); 1676 1677 folio = lru_to_folio(folio_list); 1678 list_del(&folio->lru); 1679 1680 if (!folio_trylock(folio)) 1681 goto keep; 1682 1683 VM_BUG_ON_FOLIO(folio_test_active(folio), folio); 1684 1685 nr_pages = folio_nr_pages(folio); 1686 1687 /* Account the number of base pages */ 1688 sc->nr_scanned += nr_pages; 1689 1690 if (unlikely(!folio_evictable(folio))) 1691 goto activate_locked; 1692 1693 if (!sc->may_unmap && folio_mapped(folio)) 1694 goto keep_locked; 1695 1696 /* folio_update_gen() tried to promote this page? */ 1697 if (lru_gen_enabled() && !ignore_references && 1698 folio_mapped(folio) && folio_test_referenced(folio)) 1699 goto keep_locked; 1700 1701 /* 1702 * The number of dirty pages determines if a node is marked 1703 * reclaim_congested. kswapd will stall and start writing 1704 * folios if the tail of the LRU is all dirty unqueued folios. 1705 */ 1706 folio_check_dirty_writeback(folio, &dirty, &writeback); 1707 if (dirty || writeback) 1708 stat->nr_dirty += nr_pages; 1709 1710 if (dirty && !writeback) 1711 stat->nr_unqueued_dirty += nr_pages; 1712 1713 /* 1714 * Treat this folio as congested if folios are cycling 1715 * through the LRU so quickly that the folios marked 1716 * for immediate reclaim are making it to the end of 1717 * the LRU a second time. 1718 */ 1719 if (writeback && folio_test_reclaim(folio)) 1720 stat->nr_congested += nr_pages; 1721 1722 /* 1723 * If a folio at the tail of the LRU is under writeback, there 1724 * are three cases to consider. 1725 * 1726 * 1) If reclaim is encountering an excessive number 1727 * of folios under writeback and this folio has both 1728 * the writeback and reclaim flags set, then it 1729 * indicates that folios are being queued for I/O but 1730 * are being recycled through the LRU before the I/O 1731 * can complete. Waiting on the folio itself risks an 1732 * indefinite stall if it is impossible to writeback 1733 * the folio due to I/O error or disconnected storage 1734 * so instead note that the LRU is being scanned too 1735 * quickly and the caller can stall after the folio 1736 * list has been processed. 1737 * 1738 * 2) Global or new memcg reclaim encounters a folio that is 1739 * not marked for immediate reclaim, or the caller does not 1740 * have __GFP_FS (or __GFP_IO if it's simply going to swap, 1741 * not to fs). In this case mark the folio for immediate 1742 * reclaim and continue scanning. 1743 * 1744 * Require may_enter_fs() because we would wait on fs, which 1745 * may not have submitted I/O yet. And the loop driver might 1746 * enter reclaim, and deadlock if it waits on a folio for 1747 * which it is needed to do the write (loop masks off 1748 * __GFP_IO|__GFP_FS for this reason); but more thought 1749 * would probably show more reasons. 1750 * 1751 * 3) Legacy memcg encounters a folio that already has the 1752 * reclaim flag set. memcg does not have any dirty folio 1753 * throttling so we could easily OOM just because too many 1754 * folios are in writeback and there is nothing else to 1755 * reclaim. Wait for the writeback to complete. 1756 * 1757 * In cases 1) and 2) we activate the folios to get them out of 1758 * the way while we continue scanning for clean folios on the 1759 * inactive list and refilling from the active list. The 1760 * observation here is that waiting for disk writes is more 1761 * expensive than potentially causing reloads down the line. 1762 * Since they're marked for immediate reclaim, they won't put 1763 * memory pressure on the cache working set any longer than it 1764 * takes to write them to disk. 1765 */ 1766 if (folio_test_writeback(folio)) { 1767 /* Case 1 above */ 1768 if (current_is_kswapd() && 1769 folio_test_reclaim(folio) && 1770 test_bit(PGDAT_WRITEBACK, &pgdat->flags)) { 1771 stat->nr_immediate += nr_pages; 1772 goto activate_locked; 1773 1774 /* Case 2 above */ 1775 } else if (writeback_throttling_sane(sc) || 1776 !folio_test_reclaim(folio) || 1777 !may_enter_fs(folio, sc->gfp_mask)) { 1778 /* 1779 * This is slightly racy - 1780 * folio_end_writeback() might have 1781 * just cleared the reclaim flag, then 1782 * setting the reclaim flag here ends up 1783 * interpreted as the readahead flag - but 1784 * that does not matter enough to care. 1785 * What we do want is for this folio to 1786 * have the reclaim flag set next time 1787 * memcg reclaim reaches the tests above, 1788 * so it will then wait for writeback to 1789 * avoid OOM; and it's also appropriate 1790 * in global reclaim. 1791 */ 1792 folio_set_reclaim(folio); 1793 stat->nr_writeback += nr_pages; 1794 goto activate_locked; 1795 1796 /* Case 3 above */ 1797 } else { 1798 folio_unlock(folio); 1799 folio_wait_writeback(folio); 1800 /* then go back and try same folio again */ 1801 list_add_tail(&folio->lru, folio_list); 1802 continue; 1803 } 1804 } 1805 1806 if (!ignore_references) 1807 references = folio_check_references(folio, sc); 1808 1809 switch (references) { 1810 case FOLIOREF_ACTIVATE: 1811 goto activate_locked; 1812 case FOLIOREF_KEEP: 1813 stat->nr_ref_keep += nr_pages; 1814 goto keep_locked; 1815 case FOLIOREF_RECLAIM: 1816 case FOLIOREF_RECLAIM_CLEAN: 1817 ; /* try to reclaim the folio below */ 1818 } 1819 1820 /* 1821 * Before reclaiming the folio, try to relocate 1822 * its contents to another node. 1823 */ 1824 if (do_demote_pass && 1825 (thp_migration_supported() || !folio_test_large(folio))) { 1826 list_add(&folio->lru, &demote_folios); 1827 folio_unlock(folio); 1828 continue; 1829 } 1830 1831 /* 1832 * Anonymous process memory has backing store? 1833 * Try to allocate it some swap space here. 1834 * Lazyfree folio could be freed directly 1835 */ 1836 if (folio_test_anon(folio) && folio_test_swapbacked(folio)) { 1837 if (!folio_test_swapcache(folio)) { 1838 if (!(sc->gfp_mask & __GFP_IO)) 1839 goto keep_locked; 1840 if (folio_maybe_dma_pinned(folio)) 1841 goto keep_locked; 1842 if (folio_test_large(folio)) { 1843 /* cannot split folio, skip it */ 1844 if (!can_split_folio(folio, NULL)) 1845 goto activate_locked; 1846 /* 1847 * Split folios without a PMD map right 1848 * away. Chances are some or all of the 1849 * tail pages can be freed without IO. 1850 */ 1851 if (!folio_entire_mapcount(folio) && 1852 split_folio_to_list(folio, 1853 folio_list)) 1854 goto activate_locked; 1855 } 1856 if (!add_to_swap(folio)) { 1857 if (!folio_test_large(folio)) 1858 goto activate_locked_split; 1859 /* Fallback to swap normal pages */ 1860 if (split_folio_to_list(folio, 1861 folio_list)) 1862 goto activate_locked; 1863 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1864 count_vm_event(THP_SWPOUT_FALLBACK); 1865 #endif 1866 if (!add_to_swap(folio)) 1867 goto activate_locked_split; 1868 } 1869 } 1870 } else if (folio_test_swapbacked(folio) && 1871 folio_test_large(folio)) { 1872 /* Split shmem folio */ 1873 if (split_folio_to_list(folio, folio_list)) 1874 goto keep_locked; 1875 } 1876 1877 /* 1878 * If the folio was split above, the tail pages will make 1879 * their own pass through this function and be accounted 1880 * then. 1881 */ 1882 if ((nr_pages > 1) && !folio_test_large(folio)) { 1883 sc->nr_scanned -= (nr_pages - 1); 1884 nr_pages = 1; 1885 } 1886 1887 /* 1888 * The folio is mapped into the page tables of one or more 1889 * processes. Try to unmap it here. 1890 */ 1891 if (folio_mapped(folio)) { 1892 enum ttu_flags flags = TTU_BATCH_FLUSH; 1893 bool was_swapbacked = folio_test_swapbacked(folio); 1894 1895 if (folio_test_pmd_mappable(folio)) 1896 flags |= TTU_SPLIT_HUGE_PMD; 1897 1898 try_to_unmap(folio, flags); 1899 if (folio_mapped(folio)) { 1900 stat->nr_unmap_fail += nr_pages; 1901 if (!was_swapbacked && 1902 folio_test_swapbacked(folio)) 1903 stat->nr_lazyfree_fail += nr_pages; 1904 goto activate_locked; 1905 } 1906 } 1907 1908 mapping = folio_mapping(folio); 1909 if (folio_test_dirty(folio)) { 1910 /* 1911 * Only kswapd can writeback filesystem folios 1912 * to avoid risk of stack overflow. But avoid 1913 * injecting inefficient single-folio I/O into 1914 * flusher writeback as much as possible: only 1915 * write folios when we've encountered many 1916 * dirty folios, and when we've already scanned 1917 * the rest of the LRU for clean folios and see 1918 * the same dirty folios again (with the reclaim 1919 * flag set). 1920 */ 1921 if (folio_is_file_lru(folio) && 1922 (!current_is_kswapd() || 1923 !folio_test_reclaim(folio) || 1924 !test_bit(PGDAT_DIRTY, &pgdat->flags))) { 1925 /* 1926 * Immediately reclaim when written back. 1927 * Similar in principle to deactivate_page() 1928 * except we already have the folio isolated 1929 * and know it's dirty 1930 */ 1931 node_stat_mod_folio(folio, NR_VMSCAN_IMMEDIATE, 1932 nr_pages); 1933 folio_set_reclaim(folio); 1934 1935 goto activate_locked; 1936 } 1937 1938 if (references == FOLIOREF_RECLAIM_CLEAN) 1939 goto keep_locked; 1940 if (!may_enter_fs(folio, sc->gfp_mask)) 1941 goto keep_locked; 1942 if (!sc->may_writepage) 1943 goto keep_locked; 1944 1945 /* 1946 * Folio is dirty. Flush the TLB if a writable entry 1947 * potentially exists to avoid CPU writes after I/O 1948 * starts and then write it out here. 1949 */ 1950 try_to_unmap_flush_dirty(); 1951 switch (pageout(folio, mapping, &plug)) { 1952 case PAGE_KEEP: 1953 goto keep_locked; 1954 case PAGE_ACTIVATE: 1955 goto activate_locked; 1956 case PAGE_SUCCESS: 1957 stat->nr_pageout += nr_pages; 1958 1959 if (folio_test_writeback(folio)) 1960 goto keep; 1961 if (folio_test_dirty(folio)) 1962 goto keep; 1963 1964 /* 1965 * A synchronous write - probably a ramdisk. Go 1966 * ahead and try to reclaim the folio. 1967 */ 1968 if (!folio_trylock(folio)) 1969 goto keep; 1970 if (folio_test_dirty(folio) || 1971 folio_test_writeback(folio)) 1972 goto keep_locked; 1973 mapping = folio_mapping(folio); 1974 fallthrough; 1975 case PAGE_CLEAN: 1976 ; /* try to free the folio below */ 1977 } 1978 } 1979 1980 /* 1981 * If the folio has buffers, try to free the buffer 1982 * mappings associated with this folio. If we succeed 1983 * we try to free the folio as well. 1984 * 1985 * We do this even if the folio is dirty. 1986 * filemap_release_folio() does not perform I/O, but it 1987 * is possible for a folio to have the dirty flag set, 1988 * but it is actually clean (all its buffers are clean). 1989 * This happens if the buffers were written out directly, 1990 * with submit_bh(). ext3 will do this, as well as 1991 * the blockdev mapping. filemap_release_folio() will 1992 * discover that cleanness and will drop the buffers 1993 * and mark the folio clean - it can be freed. 1994 * 1995 * Rarely, folios can have buffers and no ->mapping. 1996 * These are the folios which were not successfully 1997 * invalidated in truncate_cleanup_folio(). We try to 1998 * drop those buffers here and if that worked, and the 1999 * folio is no longer mapped into process address space 2000 * (refcount == 1) it can be freed. Otherwise, leave 2001 * the folio on the LRU so it is swappable. 2002 */ 2003 if (folio_has_private(folio)) { 2004 if (!filemap_release_folio(folio, sc->gfp_mask)) 2005 goto activate_locked; 2006 if (!mapping && folio_ref_count(folio) == 1) { 2007 folio_unlock(folio); 2008 if (folio_put_testzero(folio)) 2009 goto free_it; 2010 else { 2011 /* 2012 * rare race with speculative reference. 2013 * the speculative reference will free 2014 * this folio shortly, so we may 2015 * increment nr_reclaimed here (and 2016 * leave it off the LRU). 2017 */ 2018 nr_reclaimed += nr_pages; 2019 continue; 2020 } 2021 } 2022 } 2023 2024 if (folio_test_anon(folio) && !folio_test_swapbacked(folio)) { 2025 /* follow __remove_mapping for reference */ 2026 if (!folio_ref_freeze(folio, 1)) 2027 goto keep_locked; 2028 /* 2029 * The folio has only one reference left, which is 2030 * from the isolation. After the caller puts the 2031 * folio back on the lru and drops the reference, the 2032 * folio will be freed anyway. It doesn't matter 2033 * which lru it goes on. So we don't bother checking 2034 * the dirty flag here. 2035 */ 2036 count_vm_events(PGLAZYFREED, nr_pages); 2037 count_memcg_folio_events(folio, PGLAZYFREED, nr_pages); 2038 } else if (!mapping || !__remove_mapping(mapping, folio, true, 2039 sc->target_mem_cgroup)) 2040 goto keep_locked; 2041 2042 folio_unlock(folio); 2043 free_it: 2044 /* 2045 * Folio may get swapped out as a whole, need to account 2046 * all pages in it. 2047 */ 2048 nr_reclaimed += nr_pages; 2049 2050 /* 2051 * Is there need to periodically free_folio_list? It would 2052 * appear not as the counts should be low 2053 */ 2054 if (unlikely(folio_test_large(folio))) 2055 destroy_large_folio(folio); 2056 else 2057 list_add(&folio->lru, &free_folios); 2058 continue; 2059 2060 activate_locked_split: 2061 /* 2062 * The tail pages that are failed to add into swap cache 2063 * reach here. Fixup nr_scanned and nr_pages. 2064 */ 2065 if (nr_pages > 1) { 2066 sc->nr_scanned -= (nr_pages - 1); 2067 nr_pages = 1; 2068 } 2069 activate_locked: 2070 /* Not a candidate for swapping, so reclaim swap space. */ 2071 if (folio_test_swapcache(folio) && 2072 (mem_cgroup_swap_full(folio) || folio_test_mlocked(folio))) 2073 folio_free_swap(folio); 2074 VM_BUG_ON_FOLIO(folio_test_active(folio), folio); 2075 if (!folio_test_mlocked(folio)) { 2076 int type = folio_is_file_lru(folio); 2077 folio_set_active(folio); 2078 stat->nr_activate[type] += nr_pages; 2079 count_memcg_folio_events(folio, PGACTIVATE, nr_pages); 2080 } 2081 keep_locked: 2082 folio_unlock(folio); 2083 keep: 2084 list_add(&folio->lru, &ret_folios); 2085 VM_BUG_ON_FOLIO(folio_test_lru(folio) || 2086 folio_test_unevictable(folio), folio); 2087 } 2088 /* 'folio_list' is always empty here */ 2089 2090 /* Migrate folios selected for demotion */ 2091 nr_reclaimed += demote_folio_list(&demote_folios, pgdat); 2092 /* Folios that could not be demoted are still in @demote_folios */ 2093 if (!list_empty(&demote_folios)) { 2094 /* Folios which weren't demoted go back on @folio_list */ 2095 list_splice_init(&demote_folios, folio_list); 2096 2097 /* 2098 * goto retry to reclaim the undemoted folios in folio_list if 2099 * desired. 2100 * 2101 * Reclaiming directly from top tier nodes is not often desired 2102 * due to it breaking the LRU ordering: in general memory 2103 * should be reclaimed from lower tier nodes and demoted from 2104 * top tier nodes. 2105 * 2106 * However, disabling reclaim from top tier nodes entirely 2107 * would cause ooms in edge scenarios where lower tier memory 2108 * is unreclaimable for whatever reason, eg memory being 2109 * mlocked or too hot to reclaim. We can disable reclaim 2110 * from top tier nodes in proactive reclaim though as that is 2111 * not real memory pressure. 2112 */ 2113 if (!sc->proactive) { 2114 do_demote_pass = false; 2115 goto retry; 2116 } 2117 } 2118 2119 pgactivate = stat->nr_activate[0] + stat->nr_activate[1]; 2120 2121 mem_cgroup_uncharge_list(&free_folios); 2122 try_to_unmap_flush(); 2123 free_unref_page_list(&free_folios); 2124 2125 list_splice(&ret_folios, folio_list); 2126 count_vm_events(PGACTIVATE, pgactivate); 2127 2128 if (plug) 2129 swap_write_unplug(plug); 2130 return nr_reclaimed; 2131 } 2132 2133 unsigned int reclaim_clean_pages_from_list(struct zone *zone, 2134 struct list_head *folio_list) 2135 { 2136 struct scan_control sc = { 2137 .gfp_mask = GFP_KERNEL, 2138 .may_unmap = 1, 2139 }; 2140 struct reclaim_stat stat; 2141 unsigned int nr_reclaimed; 2142 struct folio *folio, *next; 2143 LIST_HEAD(clean_folios); 2144 unsigned int noreclaim_flag; 2145 2146 list_for_each_entry_safe(folio, next, folio_list, lru) { 2147 if (!folio_test_hugetlb(folio) && folio_is_file_lru(folio) && 2148 !folio_test_dirty(folio) && !__folio_test_movable(folio) && 2149 !folio_test_unevictable(folio)) { 2150 folio_clear_active(folio); 2151 list_move(&folio->lru, &clean_folios); 2152 } 2153 } 2154 2155 /* 2156 * We should be safe here since we are only dealing with file pages and 2157 * we are not kswapd and therefore cannot write dirty file pages. But 2158 * call memalloc_noreclaim_save() anyway, just in case these conditions 2159 * change in the future. 2160 */ 2161 noreclaim_flag = memalloc_noreclaim_save(); 2162 nr_reclaimed = shrink_folio_list(&clean_folios, zone->zone_pgdat, &sc, 2163 &stat, true); 2164 memalloc_noreclaim_restore(noreclaim_flag); 2165 2166 list_splice(&clean_folios, folio_list); 2167 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, 2168 -(long)nr_reclaimed); 2169 /* 2170 * Since lazyfree pages are isolated from file LRU from the beginning, 2171 * they will rotate back to anonymous LRU in the end if it failed to 2172 * discard so isolated count will be mismatched. 2173 * Compensate the isolated count for both LRU lists. 2174 */ 2175 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON, 2176 stat.nr_lazyfree_fail); 2177 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, 2178 -(long)stat.nr_lazyfree_fail); 2179 return nr_reclaimed; 2180 } 2181 2182 /* 2183 * Update LRU sizes after isolating pages. The LRU size updates must 2184 * be complete before mem_cgroup_update_lru_size due to a sanity check. 2185 */ 2186 static __always_inline void update_lru_sizes(struct lruvec *lruvec, 2187 enum lru_list lru, unsigned long *nr_zone_taken) 2188 { 2189 int zid; 2190 2191 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 2192 if (!nr_zone_taken[zid]) 2193 continue; 2194 2195 update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); 2196 } 2197 2198 } 2199 2200 /* 2201 * Isolating page from the lruvec to fill in @dst list by nr_to_scan times. 2202 * 2203 * lruvec->lru_lock is heavily contended. Some of the functions that 2204 * shrink the lists perform better by taking out a batch of pages 2205 * and working on them outside the LRU lock. 2206 * 2207 * For pagecache intensive workloads, this function is the hottest 2208 * spot in the kernel (apart from copy_*_user functions). 2209 * 2210 * Lru_lock must be held before calling this function. 2211 * 2212 * @nr_to_scan: The number of eligible pages to look through on the list. 2213 * @lruvec: The LRU vector to pull pages from. 2214 * @dst: The temp list to put pages on to. 2215 * @nr_scanned: The number of pages that were scanned. 2216 * @sc: The scan_control struct for this reclaim session 2217 * @lru: LRU list id for isolating 2218 * 2219 * returns how many pages were moved onto *@dst. 2220 */ 2221 static unsigned long isolate_lru_folios(unsigned long nr_to_scan, 2222 struct lruvec *lruvec, struct list_head *dst, 2223 unsigned long *nr_scanned, struct scan_control *sc, 2224 enum lru_list lru) 2225 { 2226 struct list_head *src = &lruvec->lists[lru]; 2227 unsigned long nr_taken = 0; 2228 unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 }; 2229 unsigned long nr_skipped[MAX_NR_ZONES] = { 0, }; 2230 unsigned long skipped = 0; 2231 unsigned long scan, total_scan, nr_pages; 2232 LIST_HEAD(folios_skipped); 2233 2234 total_scan = 0; 2235 scan = 0; 2236 while (scan < nr_to_scan && !list_empty(src)) { 2237 struct list_head *move_to = src; 2238 struct folio *folio; 2239 2240 folio = lru_to_folio(src); 2241 prefetchw_prev_lru_folio(folio, src, flags); 2242 2243 nr_pages = folio_nr_pages(folio); 2244 total_scan += nr_pages; 2245 2246 if (folio_zonenum(folio) > sc->reclaim_idx) { 2247 nr_skipped[folio_zonenum(folio)] += nr_pages; 2248 move_to = &folios_skipped; 2249 goto move; 2250 } 2251 2252 /* 2253 * Do not count skipped folios because that makes the function 2254 * return with no isolated folios if the LRU mostly contains 2255 * ineligible folios. This causes the VM to not reclaim any 2256 * folios, triggering a premature OOM. 2257 * Account all pages in a folio. 2258 */ 2259 scan += nr_pages; 2260 2261 if (!folio_test_lru(folio)) 2262 goto move; 2263 if (!sc->may_unmap && folio_mapped(folio)) 2264 goto move; 2265 2266 /* 2267 * Be careful not to clear the lru flag until after we're 2268 * sure the folio is not being freed elsewhere -- the 2269 * folio release code relies on it. 2270 */ 2271 if (unlikely(!folio_try_get(folio))) 2272 goto move; 2273 2274 if (!folio_test_clear_lru(folio)) { 2275 /* Another thread is already isolating this folio */ 2276 folio_put(folio); 2277 goto move; 2278 } 2279 2280 nr_taken += nr_pages; 2281 nr_zone_taken[folio_zonenum(folio)] += nr_pages; 2282 move_to = dst; 2283 move: 2284 list_move(&folio->lru, move_to); 2285 } 2286 2287 /* 2288 * Splice any skipped folios to the start of the LRU list. Note that 2289 * this disrupts the LRU order when reclaiming for lower zones but 2290 * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX 2291 * scanning would soon rescan the same folios to skip and waste lots 2292 * of cpu cycles. 2293 */ 2294 if (!list_empty(&folios_skipped)) { 2295 int zid; 2296 2297 list_splice(&folios_skipped, src); 2298 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 2299 if (!nr_skipped[zid]) 2300 continue; 2301 2302 __count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]); 2303 skipped += nr_skipped[zid]; 2304 } 2305 } 2306 *nr_scanned = total_scan; 2307 trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan, 2308 total_scan, skipped, nr_taken, 2309 sc->may_unmap ? 0 : ISOLATE_UNMAPPED, lru); 2310 update_lru_sizes(lruvec, lru, nr_zone_taken); 2311 return nr_taken; 2312 } 2313 2314 /** 2315 * folio_isolate_lru() - Try to isolate a folio from its LRU list. 2316 * @folio: Folio to isolate from its LRU list. 2317 * 2318 * Isolate a @folio from an LRU list and adjust the vmstat statistic 2319 * corresponding to whatever LRU list the folio was on. 2320 * 2321 * The folio will have its LRU flag cleared. If it was found on the 2322 * active list, it will have the Active flag set. If it was found on the 2323 * unevictable list, it will have the Unevictable flag set. These flags 2324 * may need to be cleared by the caller before letting the page go. 2325 * 2326 * Context: 2327 * 2328 * (1) Must be called with an elevated refcount on the folio. This is a 2329 * fundamental difference from isolate_lru_folios() (which is called 2330 * without a stable reference). 2331 * (2) The lru_lock must not be held. 2332 * (3) Interrupts must be enabled. 2333 * 2334 * Return: 0 if the folio was removed from an LRU list. 2335 * -EBUSY if the folio was not on an LRU list. 2336 */ 2337 int folio_isolate_lru(struct folio *folio) 2338 { 2339 int ret = -EBUSY; 2340 2341 VM_BUG_ON_FOLIO(!folio_ref_count(folio), folio); 2342 2343 if (folio_test_clear_lru(folio)) { 2344 struct lruvec *lruvec; 2345 2346 folio_get(folio); 2347 lruvec = folio_lruvec_lock_irq(folio); 2348 lruvec_del_folio(lruvec, folio); 2349 unlock_page_lruvec_irq(lruvec); 2350 ret = 0; 2351 } 2352 2353 return ret; 2354 } 2355 2356 /* 2357 * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and 2358 * then get rescheduled. When there are massive number of tasks doing page 2359 * allocation, such sleeping direct reclaimers may keep piling up on each CPU, 2360 * the LRU list will go small and be scanned faster than necessary, leading to 2361 * unnecessary swapping, thrashing and OOM. 2362 */ 2363 static int too_many_isolated(struct pglist_data *pgdat, int file, 2364 struct scan_control *sc) 2365 { 2366 unsigned long inactive, isolated; 2367 bool too_many; 2368 2369 if (current_is_kswapd()) 2370 return 0; 2371 2372 if (!writeback_throttling_sane(sc)) 2373 return 0; 2374 2375 if (file) { 2376 inactive = node_page_state(pgdat, NR_INACTIVE_FILE); 2377 isolated = node_page_state(pgdat, NR_ISOLATED_FILE); 2378 } else { 2379 inactive = node_page_state(pgdat, NR_INACTIVE_ANON); 2380 isolated = node_page_state(pgdat, NR_ISOLATED_ANON); 2381 } 2382 2383 /* 2384 * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they 2385 * won't get blocked by normal direct-reclaimers, forming a circular 2386 * deadlock. 2387 */ 2388 if ((sc->gfp_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 2389 inactive >>= 3; 2390 2391 too_many = isolated > inactive; 2392 2393 /* Wake up tasks throttled due to too_many_isolated. */ 2394 if (!too_many) 2395 wake_throttle_isolated(pgdat); 2396 2397 return too_many; 2398 } 2399 2400 /* 2401 * move_folios_to_lru() moves folios from private @list to appropriate LRU list. 2402 * On return, @list is reused as a list of folios to be freed by the caller. 2403 * 2404 * Returns the number of pages moved to the given lruvec. 2405 */ 2406 static unsigned int move_folios_to_lru(struct lruvec *lruvec, 2407 struct list_head *list) 2408 { 2409 int nr_pages, nr_moved = 0; 2410 LIST_HEAD(folios_to_free); 2411 2412 while (!list_empty(list)) { 2413 struct folio *folio = lru_to_folio(list); 2414 2415 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 2416 list_del(&folio->lru); 2417 if (unlikely(!folio_evictable(folio))) { 2418 spin_unlock_irq(&lruvec->lru_lock); 2419 folio_putback_lru(folio); 2420 spin_lock_irq(&lruvec->lru_lock); 2421 continue; 2422 } 2423 2424 /* 2425 * The folio_set_lru needs to be kept here for list integrity. 2426 * Otherwise: 2427 * #0 move_folios_to_lru #1 release_pages 2428 * if (!folio_put_testzero()) 2429 * if (folio_put_testzero()) 2430 * !lru //skip lru_lock 2431 * folio_set_lru() 2432 * list_add(&folio->lru,) 2433 * list_add(&folio->lru,) 2434 */ 2435 folio_set_lru(folio); 2436 2437 if (unlikely(folio_put_testzero(folio))) { 2438 __folio_clear_lru_flags(folio); 2439 2440 if (unlikely(folio_test_large(folio))) { 2441 spin_unlock_irq(&lruvec->lru_lock); 2442 destroy_large_folio(folio); 2443 spin_lock_irq(&lruvec->lru_lock); 2444 } else 2445 list_add(&folio->lru, &folios_to_free); 2446 2447 continue; 2448 } 2449 2450 /* 2451 * All pages were isolated from the same lruvec (and isolation 2452 * inhibits memcg migration). 2453 */ 2454 VM_BUG_ON_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); 2455 lruvec_add_folio(lruvec, folio); 2456 nr_pages = folio_nr_pages(folio); 2457 nr_moved += nr_pages; 2458 if (folio_test_active(folio)) 2459 workingset_age_nonresident(lruvec, nr_pages); 2460 } 2461 2462 /* 2463 * To save our caller's stack, now use input list for pages to free. 2464 */ 2465 list_splice(&folios_to_free, list); 2466 2467 return nr_moved; 2468 } 2469 2470 /* 2471 * If a kernel thread (such as nfsd for loop-back mounts) services a backing 2472 * device by writing to the page cache it sets PF_LOCAL_THROTTLE. In this case 2473 * we should not throttle. Otherwise it is safe to do so. 2474 */ 2475 static int current_may_throttle(void) 2476 { 2477 return !(current->flags & PF_LOCAL_THROTTLE); 2478 } 2479 2480 /* 2481 * shrink_inactive_list() is a helper for shrink_node(). It returns the number 2482 * of reclaimed pages 2483 */ 2484 static unsigned long shrink_inactive_list(unsigned long nr_to_scan, 2485 struct lruvec *lruvec, struct scan_control *sc, 2486 enum lru_list lru) 2487 { 2488 LIST_HEAD(folio_list); 2489 unsigned long nr_scanned; 2490 unsigned int nr_reclaimed = 0; 2491 unsigned long nr_taken; 2492 struct reclaim_stat stat; 2493 bool file = is_file_lru(lru); 2494 enum vm_event_item item; 2495 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2496 bool stalled = false; 2497 2498 while (unlikely(too_many_isolated(pgdat, file, sc))) { 2499 if (stalled) 2500 return 0; 2501 2502 /* wait a bit for the reclaimer. */ 2503 stalled = true; 2504 reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED); 2505 2506 /* We are about to die and free our memory. Return now. */ 2507 if (fatal_signal_pending(current)) 2508 return SWAP_CLUSTER_MAX; 2509 } 2510 2511 lru_add_drain(); 2512 2513 spin_lock_irq(&lruvec->lru_lock); 2514 2515 nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list, 2516 &nr_scanned, sc, lru); 2517 2518 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); 2519 item = PGSCAN_KSWAPD + reclaimer_offset(); 2520 if (!cgroup_reclaim(sc)) 2521 __count_vm_events(item, nr_scanned); 2522 __count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned); 2523 __count_vm_events(PGSCAN_ANON + file, nr_scanned); 2524 2525 spin_unlock_irq(&lruvec->lru_lock); 2526 2527 if (nr_taken == 0) 2528 return 0; 2529 2530 nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false); 2531 2532 spin_lock_irq(&lruvec->lru_lock); 2533 move_folios_to_lru(lruvec, &folio_list); 2534 2535 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); 2536 item = PGSTEAL_KSWAPD + reclaimer_offset(); 2537 if (!cgroup_reclaim(sc)) 2538 __count_vm_events(item, nr_reclaimed); 2539 __count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed); 2540 __count_vm_events(PGSTEAL_ANON + file, nr_reclaimed); 2541 spin_unlock_irq(&lruvec->lru_lock); 2542 2543 lru_note_cost(lruvec, file, stat.nr_pageout, nr_scanned - nr_reclaimed); 2544 mem_cgroup_uncharge_list(&folio_list); 2545 free_unref_page_list(&folio_list); 2546 2547 /* 2548 * If dirty folios are scanned that are not queued for IO, it 2549 * implies that flushers are not doing their job. This can 2550 * happen when memory pressure pushes dirty folios to the end of 2551 * the LRU before the dirty limits are breached and the dirty 2552 * data has expired. It can also happen when the proportion of 2553 * dirty folios grows not through writes but through memory 2554 * pressure reclaiming all the clean cache. And in some cases, 2555 * the flushers simply cannot keep up with the allocation 2556 * rate. Nudge the flusher threads in case they are asleep. 2557 */ 2558 if (stat.nr_unqueued_dirty == nr_taken) { 2559 wakeup_flusher_threads(WB_REASON_VMSCAN); 2560 /* 2561 * For cgroupv1 dirty throttling is achieved by waking up 2562 * the kernel flusher here and later waiting on folios 2563 * which are in writeback to finish (see shrink_folio_list()). 2564 * 2565 * Flusher may not be able to issue writeback quickly 2566 * enough for cgroupv1 writeback throttling to work 2567 * on a large system. 2568 */ 2569 if (!writeback_throttling_sane(sc)) 2570 reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK); 2571 } 2572 2573 sc->nr.dirty += stat.nr_dirty; 2574 sc->nr.congested += stat.nr_congested; 2575 sc->nr.unqueued_dirty += stat.nr_unqueued_dirty; 2576 sc->nr.writeback += stat.nr_writeback; 2577 sc->nr.immediate += stat.nr_immediate; 2578 sc->nr.taken += nr_taken; 2579 if (file) 2580 sc->nr.file_taken += nr_taken; 2581 2582 trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id, 2583 nr_scanned, nr_reclaimed, &stat, sc->priority, file); 2584 return nr_reclaimed; 2585 } 2586 2587 /* 2588 * shrink_active_list() moves folios from the active LRU to the inactive LRU. 2589 * 2590 * We move them the other way if the folio is referenced by one or more 2591 * processes. 2592 * 2593 * If the folios are mostly unmapped, the processing is fast and it is 2594 * appropriate to hold lru_lock across the whole operation. But if 2595 * the folios are mapped, the processing is slow (folio_referenced()), so 2596 * we should drop lru_lock around each folio. It's impossible to balance 2597 * this, so instead we remove the folios from the LRU while processing them. 2598 * It is safe to rely on the active flag against the non-LRU folios in here 2599 * because nobody will play with that bit on a non-LRU folio. 2600 * 2601 * The downside is that we have to touch folio->_refcount against each folio. 2602 * But we had to alter folio->flags anyway. 2603 */ 2604 static void shrink_active_list(unsigned long nr_to_scan, 2605 struct lruvec *lruvec, 2606 struct scan_control *sc, 2607 enum lru_list lru) 2608 { 2609 unsigned long nr_taken; 2610 unsigned long nr_scanned; 2611 unsigned long vm_flags; 2612 LIST_HEAD(l_hold); /* The folios which were snipped off */ 2613 LIST_HEAD(l_active); 2614 LIST_HEAD(l_inactive); 2615 unsigned nr_deactivate, nr_activate; 2616 unsigned nr_rotated = 0; 2617 int file = is_file_lru(lru); 2618 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2619 2620 lru_add_drain(); 2621 2622 spin_lock_irq(&lruvec->lru_lock); 2623 2624 nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold, 2625 &nr_scanned, sc, lru); 2626 2627 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); 2628 2629 if (!cgroup_reclaim(sc)) 2630 __count_vm_events(PGREFILL, nr_scanned); 2631 __count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned); 2632 2633 spin_unlock_irq(&lruvec->lru_lock); 2634 2635 while (!list_empty(&l_hold)) { 2636 struct folio *folio; 2637 2638 cond_resched(); 2639 folio = lru_to_folio(&l_hold); 2640 list_del(&folio->lru); 2641 2642 if (unlikely(!folio_evictable(folio))) { 2643 folio_putback_lru(folio); 2644 continue; 2645 } 2646 2647 if (unlikely(buffer_heads_over_limit)) { 2648 if (folio_test_private(folio) && folio_trylock(folio)) { 2649 if (folio_test_private(folio)) 2650 filemap_release_folio(folio, 0); 2651 folio_unlock(folio); 2652 } 2653 } 2654 2655 /* Referenced or rmap lock contention: rotate */ 2656 if (folio_referenced(folio, 0, sc->target_mem_cgroup, 2657 &vm_flags) != 0) { 2658 /* 2659 * Identify referenced, file-backed active folios and 2660 * give them one more trip around the active list. So 2661 * that executable code get better chances to stay in 2662 * memory under moderate memory pressure. Anon folios 2663 * are not likely to be evicted by use-once streaming 2664 * IO, plus JVM can create lots of anon VM_EXEC folios, 2665 * so we ignore them here. 2666 */ 2667 if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) { 2668 nr_rotated += folio_nr_pages(folio); 2669 list_add(&folio->lru, &l_active); 2670 continue; 2671 } 2672 } 2673 2674 folio_clear_active(folio); /* we are de-activating */ 2675 folio_set_workingset(folio); 2676 list_add(&folio->lru, &l_inactive); 2677 } 2678 2679 /* 2680 * Move folios back to the lru list. 2681 */ 2682 spin_lock_irq(&lruvec->lru_lock); 2683 2684 nr_activate = move_folios_to_lru(lruvec, &l_active); 2685 nr_deactivate = move_folios_to_lru(lruvec, &l_inactive); 2686 /* Keep all free folios in l_active list */ 2687 list_splice(&l_inactive, &l_active); 2688 2689 __count_vm_events(PGDEACTIVATE, nr_deactivate); 2690 __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); 2691 2692 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); 2693 spin_unlock_irq(&lruvec->lru_lock); 2694 2695 if (nr_rotated) 2696 lru_note_cost(lruvec, file, 0, nr_rotated); 2697 mem_cgroup_uncharge_list(&l_active); 2698 free_unref_page_list(&l_active); 2699 trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate, 2700 nr_deactivate, nr_rotated, sc->priority, file); 2701 } 2702 2703 static unsigned int reclaim_folio_list(struct list_head *folio_list, 2704 struct pglist_data *pgdat) 2705 { 2706 struct reclaim_stat dummy_stat; 2707 unsigned int nr_reclaimed; 2708 struct folio *folio; 2709 struct scan_control sc = { 2710 .gfp_mask = GFP_KERNEL, 2711 .may_writepage = 1, 2712 .may_unmap = 1, 2713 .may_swap = 1, 2714 .no_demotion = 1, 2715 }; 2716 2717 nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, false); 2718 while (!list_empty(folio_list)) { 2719 folio = lru_to_folio(folio_list); 2720 list_del(&folio->lru); 2721 folio_putback_lru(folio); 2722 } 2723 2724 return nr_reclaimed; 2725 } 2726 2727 unsigned long reclaim_pages(struct list_head *folio_list) 2728 { 2729 int nid; 2730 unsigned int nr_reclaimed = 0; 2731 LIST_HEAD(node_folio_list); 2732 unsigned int noreclaim_flag; 2733 2734 if (list_empty(folio_list)) 2735 return nr_reclaimed; 2736 2737 noreclaim_flag = memalloc_noreclaim_save(); 2738 2739 nid = folio_nid(lru_to_folio(folio_list)); 2740 do { 2741 struct folio *folio = lru_to_folio(folio_list); 2742 2743 if (nid == folio_nid(folio)) { 2744 folio_clear_active(folio); 2745 list_move(&folio->lru, &node_folio_list); 2746 continue; 2747 } 2748 2749 nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); 2750 nid = folio_nid(lru_to_folio(folio_list)); 2751 } while (!list_empty(folio_list)); 2752 2753 nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); 2754 2755 memalloc_noreclaim_restore(noreclaim_flag); 2756 2757 return nr_reclaimed; 2758 } 2759 2760 static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan, 2761 struct lruvec *lruvec, struct scan_control *sc) 2762 { 2763 if (is_active_lru(lru)) { 2764 if (sc->may_deactivate & (1 << is_file_lru(lru))) 2765 shrink_active_list(nr_to_scan, lruvec, sc, lru); 2766 else 2767 sc->skipped_deactivate = 1; 2768 return 0; 2769 } 2770 2771 return shrink_inactive_list(nr_to_scan, lruvec, sc, lru); 2772 } 2773 2774 /* 2775 * The inactive anon list should be small enough that the VM never has 2776 * to do too much work. 2777 * 2778 * The inactive file list should be small enough to leave most memory 2779 * to the established workingset on the scan-resistant active list, 2780 * but large enough to avoid thrashing the aggregate readahead window. 2781 * 2782 * Both inactive lists should also be large enough that each inactive 2783 * folio has a chance to be referenced again before it is reclaimed. 2784 * 2785 * If that fails and refaulting is observed, the inactive list grows. 2786 * 2787 * The inactive_ratio is the target ratio of ACTIVE to INACTIVE folios 2788 * on this LRU, maintained by the pageout code. An inactive_ratio 2789 * of 3 means 3:1 or 25% of the folios are kept on the inactive list. 2790 * 2791 * total target max 2792 * memory ratio inactive 2793 * ------------------------------------- 2794 * 10MB 1 5MB 2795 * 100MB 1 50MB 2796 * 1GB 3 250MB 2797 * 10GB 10 0.9GB 2798 * 100GB 31 3GB 2799 * 1TB 101 10GB 2800 * 10TB 320 32GB 2801 */ 2802 static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru) 2803 { 2804 enum lru_list active_lru = inactive_lru + LRU_ACTIVE; 2805 unsigned long inactive, active; 2806 unsigned long inactive_ratio; 2807 unsigned long gb; 2808 2809 inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru); 2810 active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru); 2811 2812 gb = (inactive + active) >> (30 - PAGE_SHIFT); 2813 if (gb) 2814 inactive_ratio = int_sqrt(10 * gb); 2815 else 2816 inactive_ratio = 1; 2817 2818 return inactive * inactive_ratio < active; 2819 } 2820 2821 enum scan_balance { 2822 SCAN_EQUAL, 2823 SCAN_FRACT, 2824 SCAN_ANON, 2825 SCAN_FILE, 2826 }; 2827 2828 static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc) 2829 { 2830 unsigned long file; 2831 struct lruvec *target_lruvec; 2832 2833 if (lru_gen_enabled()) 2834 return; 2835 2836 target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); 2837 2838 /* 2839 * Flush the memory cgroup stats, so that we read accurate per-memcg 2840 * lruvec stats for heuristics. 2841 */ 2842 mem_cgroup_flush_stats(); 2843 2844 /* 2845 * Determine the scan balance between anon and file LRUs. 2846 */ 2847 spin_lock_irq(&target_lruvec->lru_lock); 2848 sc->anon_cost = target_lruvec->anon_cost; 2849 sc->file_cost = target_lruvec->file_cost; 2850 spin_unlock_irq(&target_lruvec->lru_lock); 2851 2852 /* 2853 * Target desirable inactive:active list ratios for the anon 2854 * and file LRU lists. 2855 */ 2856 if (!sc->force_deactivate) { 2857 unsigned long refaults; 2858 2859 /* 2860 * When refaults are being observed, it means a new 2861 * workingset is being established. Deactivate to get 2862 * rid of any stale active pages quickly. 2863 */ 2864 refaults = lruvec_page_state(target_lruvec, 2865 WORKINGSET_ACTIVATE_ANON); 2866 if (refaults != target_lruvec->refaults[WORKINGSET_ANON] || 2867 inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) 2868 sc->may_deactivate |= DEACTIVATE_ANON; 2869 else 2870 sc->may_deactivate &= ~DEACTIVATE_ANON; 2871 2872 refaults = lruvec_page_state(target_lruvec, 2873 WORKINGSET_ACTIVATE_FILE); 2874 if (refaults != target_lruvec->refaults[WORKINGSET_FILE] || 2875 inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) 2876 sc->may_deactivate |= DEACTIVATE_FILE; 2877 else 2878 sc->may_deactivate &= ~DEACTIVATE_FILE; 2879 } else 2880 sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; 2881 2882 /* 2883 * If we have plenty of inactive file pages that aren't 2884 * thrashing, try to reclaim those first before touching 2885 * anonymous pages. 2886 */ 2887 file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); 2888 if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) 2889 sc->cache_trim_mode = 1; 2890 else 2891 sc->cache_trim_mode = 0; 2892 2893 /* 2894 * Prevent the reclaimer from falling into the cache trap: as 2895 * cache pages start out inactive, every cache fault will tip 2896 * the scan balance towards the file LRU. And as the file LRU 2897 * shrinks, so does the window for rotation from references. 2898 * This means we have a runaway feedback loop where a tiny 2899 * thrashing file LRU becomes infinitely more attractive than 2900 * anon pages. Try to detect this based on file LRU size. 2901 */ 2902 if (!cgroup_reclaim(sc)) { 2903 unsigned long total_high_wmark = 0; 2904 unsigned long free, anon; 2905 int z; 2906 2907 free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); 2908 file = node_page_state(pgdat, NR_ACTIVE_FILE) + 2909 node_page_state(pgdat, NR_INACTIVE_FILE); 2910 2911 for (z = 0; z < MAX_NR_ZONES; z++) { 2912 struct zone *zone = &pgdat->node_zones[z]; 2913 2914 if (!managed_zone(zone)) 2915 continue; 2916 2917 total_high_wmark += high_wmark_pages(zone); 2918 } 2919 2920 /* 2921 * Consider anon: if that's low too, this isn't a 2922 * runaway file reclaim problem, but rather just 2923 * extreme pressure. Reclaim as per usual then. 2924 */ 2925 anon = node_page_state(pgdat, NR_INACTIVE_ANON); 2926 2927 sc->file_is_tiny = 2928 file + free <= total_high_wmark && 2929 !(sc->may_deactivate & DEACTIVATE_ANON) && 2930 anon >> sc->priority; 2931 } 2932 } 2933 2934 /* 2935 * Determine how aggressively the anon and file LRU lists should be 2936 * scanned. 2937 * 2938 * nr[0] = anon inactive folios to scan; nr[1] = anon active folios to scan 2939 * nr[2] = file inactive folios to scan; nr[3] = file active folios to scan 2940 */ 2941 static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, 2942 unsigned long *nr) 2943 { 2944 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2945 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 2946 unsigned long anon_cost, file_cost, total_cost; 2947 int swappiness = mem_cgroup_swappiness(memcg); 2948 u64 fraction[ANON_AND_FILE]; 2949 u64 denominator = 0; /* gcc */ 2950 enum scan_balance scan_balance; 2951 unsigned long ap, fp; 2952 enum lru_list lru; 2953 2954 /* If we have no swap space, do not bother scanning anon folios. */ 2955 if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { 2956 scan_balance = SCAN_FILE; 2957 goto out; 2958 } 2959 2960 /* 2961 * Global reclaim will swap to prevent OOM even with no 2962 * swappiness, but memcg users want to use this knob to 2963 * disable swapping for individual groups completely when 2964 * using the memory controller's swap limit feature would be 2965 * too expensive. 2966 */ 2967 if (cgroup_reclaim(sc) && !swappiness) { 2968 scan_balance = SCAN_FILE; 2969 goto out; 2970 } 2971 2972 /* 2973 * Do not apply any pressure balancing cleverness when the 2974 * system is close to OOM, scan both anon and file equally 2975 * (unless the swappiness setting disagrees with swapping). 2976 */ 2977 if (!sc->priority && swappiness) { 2978 scan_balance = SCAN_EQUAL; 2979 goto out; 2980 } 2981 2982 /* 2983 * If the system is almost out of file pages, force-scan anon. 2984 */ 2985 if (sc->file_is_tiny) { 2986 scan_balance = SCAN_ANON; 2987 goto out; 2988 } 2989 2990 /* 2991 * If there is enough inactive page cache, we do not reclaim 2992 * anything from the anonymous working right now. 2993 */ 2994 if (sc->cache_trim_mode) { 2995 scan_balance = SCAN_FILE; 2996 goto out; 2997 } 2998 2999 scan_balance = SCAN_FRACT; 3000 /* 3001 * Calculate the pressure balance between anon and file pages. 3002 * 3003 * The amount of pressure we put on each LRU is inversely 3004 * proportional to the cost of reclaiming each list, as 3005 * determined by the share of pages that are refaulting, times 3006 * the relative IO cost of bringing back a swapped out 3007 * anonymous page vs reloading a filesystem page (swappiness). 3008 * 3009 * Although we limit that influence to ensure no list gets 3010 * left behind completely: at least a third of the pressure is 3011 * applied, before swappiness. 3012 * 3013 * With swappiness at 100, anon and file have equal IO cost. 3014 */ 3015 total_cost = sc->anon_cost + sc->file_cost; 3016 anon_cost = total_cost + sc->anon_cost; 3017 file_cost = total_cost + sc->file_cost; 3018 total_cost = anon_cost + file_cost; 3019 3020 ap = swappiness * (total_cost + 1); 3021 ap /= anon_cost + 1; 3022 3023 fp = (200 - swappiness) * (total_cost + 1); 3024 fp /= file_cost + 1; 3025 3026 fraction[0] = ap; 3027 fraction[1] = fp; 3028 denominator = ap + fp; 3029 out: 3030 for_each_evictable_lru(lru) { 3031 int file = is_file_lru(lru); 3032 unsigned long lruvec_size; 3033 unsigned long low, min; 3034 unsigned long scan; 3035 3036 lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx); 3037 mem_cgroup_protection(sc->target_mem_cgroup, memcg, 3038 &min, &low); 3039 3040 if (min || low) { 3041 /* 3042 * Scale a cgroup's reclaim pressure by proportioning 3043 * its current usage to its memory.low or memory.min 3044 * setting. 3045 * 3046 * This is important, as otherwise scanning aggression 3047 * becomes extremely binary -- from nothing as we 3048 * approach the memory protection threshold, to totally 3049 * nominal as we exceed it. This results in requiring 3050 * setting extremely liberal protection thresholds. It 3051 * also means we simply get no protection at all if we 3052 * set it too low, which is not ideal. 3053 * 3054 * If there is any protection in place, we reduce scan 3055 * pressure by how much of the total memory used is 3056 * within protection thresholds. 3057 * 3058 * There is one special case: in the first reclaim pass, 3059 * we skip over all groups that are within their low 3060 * protection. If that fails to reclaim enough pages to 3061 * satisfy the reclaim goal, we come back and override 3062 * the best-effort low protection. However, we still 3063 * ideally want to honor how well-behaved groups are in 3064 * that case instead of simply punishing them all 3065 * equally. As such, we reclaim them based on how much 3066 * memory they are using, reducing the scan pressure 3067 * again by how much of the total memory used is under 3068 * hard protection. 3069 */ 3070 unsigned long cgroup_size = mem_cgroup_size(memcg); 3071 unsigned long protection; 3072 3073 /* memory.low scaling, make sure we retry before OOM */ 3074 if (!sc->memcg_low_reclaim && low > min) { 3075 protection = low; 3076 sc->memcg_low_skipped = 1; 3077 } else { 3078 protection = min; 3079 } 3080 3081 /* Avoid TOCTOU with earlier protection check */ 3082 cgroup_size = max(cgroup_size, protection); 3083 3084 scan = lruvec_size - lruvec_size * protection / 3085 (cgroup_size + 1); 3086 3087 /* 3088 * Minimally target SWAP_CLUSTER_MAX pages to keep 3089 * reclaim moving forwards, avoiding decrementing 3090 * sc->priority further than desirable. 3091 */ 3092 scan = max(scan, SWAP_CLUSTER_MAX); 3093 } else { 3094 scan = lruvec_size; 3095 } 3096 3097 scan >>= sc->priority; 3098 3099 /* 3100 * If the cgroup's already been deleted, make sure to 3101 * scrape out the remaining cache. 3102 */ 3103 if (!scan && !mem_cgroup_online(memcg)) 3104 scan = min(lruvec_size, SWAP_CLUSTER_MAX); 3105 3106 switch (scan_balance) { 3107 case SCAN_EQUAL: 3108 /* Scan lists relative to size */ 3109 break; 3110 case SCAN_FRACT: 3111 /* 3112 * Scan types proportional to swappiness and 3113 * their relative recent reclaim efficiency. 3114 * Make sure we don't miss the last page on 3115 * the offlined memory cgroups because of a 3116 * round-off error. 3117 */ 3118 scan = mem_cgroup_online(memcg) ? 3119 div64_u64(scan * fraction[file], denominator) : 3120 DIV64_U64_ROUND_UP(scan * fraction[file], 3121 denominator); 3122 break; 3123 case SCAN_FILE: 3124 case SCAN_ANON: 3125 /* Scan one type exclusively */ 3126 if ((scan_balance == SCAN_FILE) != file) 3127 scan = 0; 3128 break; 3129 default: 3130 /* Look ma, no brain */ 3131 BUG(); 3132 } 3133 3134 nr[lru] = scan; 3135 } 3136 } 3137 3138 /* 3139 * Anonymous LRU management is a waste if there is 3140 * ultimately no way to reclaim the memory. 3141 */ 3142 static bool can_age_anon_pages(struct pglist_data *pgdat, 3143 struct scan_control *sc) 3144 { 3145 /* Aging the anon LRU is valuable if swap is present: */ 3146 if (total_swap_pages > 0) 3147 return true; 3148 3149 /* Also valuable if anon pages can be demoted: */ 3150 return can_demote(pgdat->node_id, sc); 3151 } 3152 3153 #ifdef CONFIG_LRU_GEN 3154 3155 #ifdef CONFIG_LRU_GEN_ENABLED 3156 DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); 3157 #define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) 3158 #else 3159 DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); 3160 #define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) 3161 #endif 3162 3163 /****************************************************************************** 3164 * shorthand helpers 3165 ******************************************************************************/ 3166 3167 #define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset)) 3168 3169 #define DEFINE_MAX_SEQ(lruvec) \ 3170 unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) 3171 3172 #define DEFINE_MIN_SEQ(lruvec) \ 3173 unsigned long min_seq[ANON_AND_FILE] = { \ 3174 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \ 3175 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \ 3176 } 3177 3178 #define for_each_gen_type_zone(gen, type, zone) \ 3179 for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ 3180 for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ 3181 for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) 3182 3183 static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) 3184 { 3185 struct pglist_data *pgdat = NODE_DATA(nid); 3186 3187 #ifdef CONFIG_MEMCG 3188 if (memcg) { 3189 struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; 3190 3191 /* see the comment in mem_cgroup_lruvec() */ 3192 if (!lruvec->pgdat) 3193 lruvec->pgdat = pgdat; 3194 3195 return lruvec; 3196 } 3197 #endif 3198 VM_WARN_ON_ONCE(!mem_cgroup_disabled()); 3199 3200 return &pgdat->__lruvec; 3201 } 3202 3203 static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) 3204 { 3205 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 3206 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 3207 3208 if (!can_demote(pgdat->node_id, sc) && 3209 mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) 3210 return 0; 3211 3212 return mem_cgroup_swappiness(memcg); 3213 } 3214 3215 static int get_nr_gens(struct lruvec *lruvec, int type) 3216 { 3217 return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; 3218 } 3219 3220 static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) 3221 { 3222 /* see the comment on lru_gen_struct */ 3223 return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS && 3224 get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) && 3225 get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS; 3226 } 3227 3228 /****************************************************************************** 3229 * mm_struct list 3230 ******************************************************************************/ 3231 3232 static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) 3233 { 3234 static struct lru_gen_mm_list mm_list = { 3235 .fifo = LIST_HEAD_INIT(mm_list.fifo), 3236 .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock), 3237 }; 3238 3239 #ifdef CONFIG_MEMCG 3240 if (memcg) 3241 return &memcg->mm_list; 3242 #endif 3243 VM_WARN_ON_ONCE(!mem_cgroup_disabled()); 3244 3245 return &mm_list; 3246 } 3247 3248 void lru_gen_add_mm(struct mm_struct *mm) 3249 { 3250 int nid; 3251 struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); 3252 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 3253 3254 VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); 3255 #ifdef CONFIG_MEMCG 3256 VM_WARN_ON_ONCE(mm->lru_gen.memcg); 3257 mm->lru_gen.memcg = memcg; 3258 #endif 3259 spin_lock(&mm_list->lock); 3260 3261 for_each_node_state(nid, N_MEMORY) { 3262 struct lruvec *lruvec = get_lruvec(memcg, nid); 3263 3264 /* the first addition since the last iteration */ 3265 if (lruvec->mm_state.tail == &mm_list->fifo) 3266 lruvec->mm_state.tail = &mm->lru_gen.list; 3267 } 3268 3269 list_add_tail(&mm->lru_gen.list, &mm_list->fifo); 3270 3271 spin_unlock(&mm_list->lock); 3272 } 3273 3274 void lru_gen_del_mm(struct mm_struct *mm) 3275 { 3276 int nid; 3277 struct lru_gen_mm_list *mm_list; 3278 struct mem_cgroup *memcg = NULL; 3279 3280 if (list_empty(&mm->lru_gen.list)) 3281 return; 3282 3283 #ifdef CONFIG_MEMCG 3284 memcg = mm->lru_gen.memcg; 3285 #endif 3286 mm_list = get_mm_list(memcg); 3287 3288 spin_lock(&mm_list->lock); 3289 3290 for_each_node(nid) { 3291 struct lruvec *lruvec = get_lruvec(memcg, nid); 3292 3293 /* where the last iteration ended (exclusive) */ 3294 if (lruvec->mm_state.tail == &mm->lru_gen.list) 3295 lruvec->mm_state.tail = lruvec->mm_state.tail->next; 3296 3297 /* where the current iteration continues (inclusive) */ 3298 if (lruvec->mm_state.head != &mm->lru_gen.list) 3299 continue; 3300 3301 lruvec->mm_state.head = lruvec->mm_state.head->next; 3302 /* the deletion ends the current iteration */ 3303 if (lruvec->mm_state.head == &mm_list->fifo) 3304 WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1); 3305 } 3306 3307 list_del_init(&mm->lru_gen.list); 3308 3309 spin_unlock(&mm_list->lock); 3310 3311 #ifdef CONFIG_MEMCG 3312 mem_cgroup_put(mm->lru_gen.memcg); 3313 mm->lru_gen.memcg = NULL; 3314 #endif 3315 } 3316 3317 #ifdef CONFIG_MEMCG 3318 void lru_gen_migrate_mm(struct mm_struct *mm) 3319 { 3320 struct mem_cgroup *memcg; 3321 struct task_struct *task = rcu_dereference_protected(mm->owner, true); 3322 3323 VM_WARN_ON_ONCE(task->mm != mm); 3324 lockdep_assert_held(&task->alloc_lock); 3325 3326 /* for mm_update_next_owner() */ 3327 if (mem_cgroup_disabled()) 3328 return; 3329 3330 /* migration can happen before addition */ 3331 if (!mm->lru_gen.memcg) 3332 return; 3333 3334 rcu_read_lock(); 3335 memcg = mem_cgroup_from_task(task); 3336 rcu_read_unlock(); 3337 if (memcg == mm->lru_gen.memcg) 3338 return; 3339 3340 VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); 3341 3342 lru_gen_del_mm(mm); 3343 lru_gen_add_mm(mm); 3344 } 3345 #endif 3346 3347 /* 3348 * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when 3349 * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of 3350 * bits in a bitmap, k is the number of hash functions and n is the number of 3351 * inserted items. 3352 * 3353 * Page table walkers use one of the two filters to reduce their search space. 3354 * To get rid of non-leaf entries that no longer have enough leaf entries, the 3355 * aging uses the double-buffering technique to flip to the other filter each 3356 * time it produces a new generation. For non-leaf entries that have enough 3357 * leaf entries, the aging carries them over to the next generation in 3358 * walk_pmd_range(); the eviction also report them when walking the rmap 3359 * in lru_gen_look_around(). 3360 * 3361 * For future optimizations: 3362 * 1. It's not necessary to keep both filters all the time. The spare one can be 3363 * freed after the RCU grace period and reallocated if needed again. 3364 * 2. And when reallocating, it's worth scaling its size according to the number 3365 * of inserted entries in the other filter, to reduce the memory overhead on 3366 * small systems and false positives on large systems. 3367 * 3. Jenkins' hash function is an alternative to Knuth's. 3368 */ 3369 #define BLOOM_FILTER_SHIFT 15 3370 3371 static inline int filter_gen_from_seq(unsigned long seq) 3372 { 3373 return seq % NR_BLOOM_FILTERS; 3374 } 3375 3376 static void get_item_key(void *item, int *key) 3377 { 3378 u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2); 3379 3380 BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32)); 3381 3382 key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1); 3383 key[1] = hash >> BLOOM_FILTER_SHIFT; 3384 } 3385 3386 static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq) 3387 { 3388 unsigned long *filter; 3389 int gen = filter_gen_from_seq(seq); 3390 3391 filter = lruvec->mm_state.filters[gen]; 3392 if (filter) { 3393 bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); 3394 return; 3395 } 3396 3397 filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), 3398 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); 3399 WRITE_ONCE(lruvec->mm_state.filters[gen], filter); 3400 } 3401 3402 static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) 3403 { 3404 int key[2]; 3405 unsigned long *filter; 3406 int gen = filter_gen_from_seq(seq); 3407 3408 filter = READ_ONCE(lruvec->mm_state.filters[gen]); 3409 if (!filter) 3410 return; 3411 3412 get_item_key(item, key); 3413 3414 if (!test_bit(key[0], filter)) 3415 set_bit(key[0], filter); 3416 if (!test_bit(key[1], filter)) 3417 set_bit(key[1], filter); 3418 } 3419 3420 static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) 3421 { 3422 int key[2]; 3423 unsigned long *filter; 3424 int gen = filter_gen_from_seq(seq); 3425 3426 filter = READ_ONCE(lruvec->mm_state.filters[gen]); 3427 if (!filter) 3428 return true; 3429 3430 get_item_key(item, key); 3431 3432 return test_bit(key[0], filter) && test_bit(key[1], filter); 3433 } 3434 3435 static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last) 3436 { 3437 int i; 3438 int hist; 3439 3440 lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); 3441 3442 if (walk) { 3443 hist = lru_hist_from_seq(walk->max_seq); 3444 3445 for (i = 0; i < NR_MM_STATS; i++) { 3446 WRITE_ONCE(lruvec->mm_state.stats[hist][i], 3447 lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]); 3448 walk->mm_stats[i] = 0; 3449 } 3450 } 3451 3452 if (NR_HIST_GENS > 1 && last) { 3453 hist = lru_hist_from_seq(lruvec->mm_state.seq + 1); 3454 3455 for (i = 0; i < NR_MM_STATS; i++) 3456 WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0); 3457 } 3458 } 3459 3460 static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) 3461 { 3462 int type; 3463 unsigned long size = 0; 3464 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3465 int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); 3466 3467 if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) 3468 return true; 3469 3470 clear_bit(key, &mm->lru_gen.bitmap); 3471 3472 for (type = !walk->can_swap; type < ANON_AND_FILE; type++) { 3473 size += type ? get_mm_counter(mm, MM_FILEPAGES) : 3474 get_mm_counter(mm, MM_ANONPAGES) + 3475 get_mm_counter(mm, MM_SHMEMPAGES); 3476 } 3477 3478 if (size < MIN_LRU_BATCH) 3479 return true; 3480 3481 return !mmget_not_zero(mm); 3482 } 3483 3484 static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, 3485 struct mm_struct **iter) 3486 { 3487 bool first = false; 3488 bool last = true; 3489 struct mm_struct *mm = NULL; 3490 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 3491 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 3492 struct lru_gen_mm_state *mm_state = &lruvec->mm_state; 3493 3494 /* 3495 * There are four interesting cases for this page table walker: 3496 * 1. It tries to start a new iteration of mm_list with a stale max_seq; 3497 * there is nothing left to do. 3498 * 2. It's the first of the current generation, and it needs to reset 3499 * the Bloom filter for the next generation. 3500 * 3. It reaches the end of mm_list, and it needs to increment 3501 * mm_state->seq; the iteration is done. 3502 * 4. It's the last of the current generation, and it needs to reset the 3503 * mm stats counters for the next generation. 3504 */ 3505 spin_lock(&mm_list->lock); 3506 3507 VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); 3508 VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq); 3509 VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers); 3510 3511 if (walk->max_seq <= mm_state->seq) { 3512 if (!*iter) 3513 last = false; 3514 goto done; 3515 } 3516 3517 if (!mm_state->nr_walkers) { 3518 VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); 3519 3520 mm_state->head = mm_list->fifo.next; 3521 first = true; 3522 } 3523 3524 while (!mm && mm_state->head != &mm_list->fifo) { 3525 mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); 3526 3527 mm_state->head = mm_state->head->next; 3528 3529 /* force scan for those added after the last iteration */ 3530 if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) { 3531 mm_state->tail = mm_state->head; 3532 walk->force_scan = true; 3533 } 3534 3535 if (should_skip_mm(mm, walk)) 3536 mm = NULL; 3537 } 3538 3539 if (mm_state->head == &mm_list->fifo) 3540 WRITE_ONCE(mm_state->seq, mm_state->seq + 1); 3541 done: 3542 if (*iter && !mm) 3543 mm_state->nr_walkers--; 3544 if (!*iter && mm) 3545 mm_state->nr_walkers++; 3546 3547 if (mm_state->nr_walkers) 3548 last = false; 3549 3550 if (*iter || last) 3551 reset_mm_stats(lruvec, walk, last); 3552 3553 spin_unlock(&mm_list->lock); 3554 3555 if (mm && first) 3556 reset_bloom_filter(lruvec, walk->max_seq + 1); 3557 3558 if (*iter) 3559 mmput_async(*iter); 3560 3561 *iter = mm; 3562 3563 return last; 3564 } 3565 3566 static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) 3567 { 3568 bool success = false; 3569 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 3570 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 3571 struct lru_gen_mm_state *mm_state = &lruvec->mm_state; 3572 3573 spin_lock(&mm_list->lock); 3574 3575 VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); 3576 3577 if (max_seq > mm_state->seq && !mm_state->nr_walkers) { 3578 VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); 3579 3580 WRITE_ONCE(mm_state->seq, mm_state->seq + 1); 3581 reset_mm_stats(lruvec, NULL, true); 3582 success = true; 3583 } 3584 3585 spin_unlock(&mm_list->lock); 3586 3587 return success; 3588 } 3589 3590 /****************************************************************************** 3591 * refault feedback loop 3592 ******************************************************************************/ 3593 3594 /* 3595 * A feedback loop based on Proportional-Integral-Derivative (PID) controller. 3596 * 3597 * The P term is refaulted/(evicted+protected) from a tier in the generation 3598 * currently being evicted; the I term is the exponential moving average of the 3599 * P term over the generations previously evicted, using the smoothing factor 3600 * 1/2; the D term isn't supported. 3601 * 3602 * The setpoint (SP) is always the first tier of one type; the process variable 3603 * (PV) is either any tier of the other type or any other tier of the same 3604 * type. 3605 * 3606 * The error is the difference between the SP and the PV; the correction is to 3607 * turn off protection when SP>PV or turn on protection when SP<PV. 3608 * 3609 * For future optimizations: 3610 * 1. The D term may discount the other two terms over time so that long-lived 3611 * generations can resist stale information. 3612 */ 3613 struct ctrl_pos { 3614 unsigned long refaulted; 3615 unsigned long total; 3616 int gain; 3617 }; 3618 3619 static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain, 3620 struct ctrl_pos *pos) 3621 { 3622 struct lru_gen_struct *lrugen = &lruvec->lrugen; 3623 int hist = lru_hist_from_seq(lrugen->min_seq[type]); 3624 3625 pos->refaulted = lrugen->avg_refaulted[type][tier] + 3626 atomic_long_read(&lrugen->refaulted[hist][type][tier]); 3627 pos->total = lrugen->avg_total[type][tier] + 3628 atomic_long_read(&lrugen->evicted[hist][type][tier]); 3629 if (tier) 3630 pos->total += lrugen->protected[hist][type][tier - 1]; 3631 pos->gain = gain; 3632 } 3633 3634 static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) 3635 { 3636 int hist, tier; 3637 struct lru_gen_struct *lrugen = &lruvec->lrugen; 3638 bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1; 3639 unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; 3640 3641 lockdep_assert_held(&lruvec->lru_lock); 3642 3643 if (!carryover && !clear) 3644 return; 3645 3646 hist = lru_hist_from_seq(seq); 3647 3648 for (tier = 0; tier < MAX_NR_TIERS; tier++) { 3649 if (carryover) { 3650 unsigned long sum; 3651 3652 sum = lrugen->avg_refaulted[type][tier] + 3653 atomic_long_read(&lrugen->refaulted[hist][type][tier]); 3654 WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); 3655 3656 sum = lrugen->avg_total[type][tier] + 3657 atomic_long_read(&lrugen->evicted[hist][type][tier]); 3658 if (tier) 3659 sum += lrugen->protected[hist][type][tier - 1]; 3660 WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); 3661 } 3662 3663 if (clear) { 3664 atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); 3665 atomic_long_set(&lrugen->evicted[hist][type][tier], 0); 3666 if (tier) 3667 WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0); 3668 } 3669 } 3670 } 3671 3672 static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) 3673 { 3674 /* 3675 * Return true if the PV has a limited number of refaults or a lower 3676 * refaulted/total than the SP. 3677 */ 3678 return pv->refaulted < MIN_LRU_BATCH || 3679 pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <= 3680 (sp->refaulted + 1) * pv->total * pv->gain; 3681 } 3682 3683 /****************************************************************************** 3684 * the aging 3685 ******************************************************************************/ 3686 3687 /* promote pages accessed through page tables */ 3688 static int folio_update_gen(struct folio *folio, int gen) 3689 { 3690 unsigned long new_flags, old_flags = READ_ONCE(folio->flags); 3691 3692 VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); 3693 VM_WARN_ON_ONCE(!rcu_read_lock_held()); 3694 3695 do { 3696 /* lru_gen_del_folio() has isolated this page? */ 3697 if (!(old_flags & LRU_GEN_MASK)) { 3698 /* for shrink_folio_list() */ 3699 new_flags = old_flags | BIT(PG_referenced); 3700 continue; 3701 } 3702 3703 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); 3704 new_flags |= (gen + 1UL) << LRU_GEN_PGOFF; 3705 } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); 3706 3707 return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; 3708 } 3709 3710 /* protect pages accessed multiple times through file descriptors */ 3711 static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) 3712 { 3713 int type = folio_is_file_lru(folio); 3714 struct lru_gen_struct *lrugen = &lruvec->lrugen; 3715 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); 3716 unsigned long new_flags, old_flags = READ_ONCE(folio->flags); 3717 3718 VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); 3719 3720 do { 3721 new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; 3722 /* folio_update_gen() has promoted this page? */ 3723 if (new_gen >= 0 && new_gen != old_gen) 3724 return new_gen; 3725 3726 new_gen = (old_gen + 1) % MAX_NR_GENS; 3727 3728 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); 3729 new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; 3730 /* for folio_end_writeback() */ 3731 if (reclaiming) 3732 new_flags |= BIT(PG_reclaim); 3733 } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); 3734 3735 lru_gen_update_size(lruvec, folio, old_gen, new_gen); 3736 3737 return new_gen; 3738 } 3739 3740 static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, 3741 int old_gen, int new_gen) 3742 { 3743 int type = folio_is_file_lru(folio); 3744 int zone = folio_zonenum(folio); 3745 int delta = folio_nr_pages(folio); 3746 3747 VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); 3748 VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); 3749 3750 walk->batched++; 3751 3752 walk->nr_pages[old_gen][type][zone] -= delta; 3753 walk->nr_pages[new_gen][type][zone] += delta; 3754 } 3755 3756 static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk) 3757 { 3758 int gen, type, zone; 3759 struct lru_gen_struct *lrugen = &lruvec->lrugen; 3760 3761 walk->batched = 0; 3762 3763 for_each_gen_type_zone(gen, type, zone) { 3764 enum lru_list lru = type * LRU_INACTIVE_FILE; 3765 int delta = walk->nr_pages[gen][type][zone]; 3766 3767 if (!delta) 3768 continue; 3769 3770 walk->nr_pages[gen][type][zone] = 0; 3771 WRITE_ONCE(lrugen->nr_pages[gen][type][zone], 3772 lrugen->nr_pages[gen][type][zone] + delta); 3773 3774 if (lru_gen_is_active(lruvec, gen)) 3775 lru += LRU_ACTIVE; 3776 __update_lru_size(lruvec, lru, zone, delta); 3777 } 3778 } 3779 3780 static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) 3781 { 3782 struct address_space *mapping; 3783 struct vm_area_struct *vma = args->vma; 3784 struct lru_gen_mm_walk *walk = args->private; 3785 3786 if (!vma_is_accessible(vma)) 3787 return true; 3788 3789 if (is_vm_hugetlb_page(vma)) 3790 return true; 3791 3792 if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL | VM_SEQ_READ | VM_RAND_READ)) 3793 return true; 3794 3795 if (vma == get_gate_vma(vma->vm_mm)) 3796 return true; 3797 3798 if (vma_is_anonymous(vma)) 3799 return !walk->can_swap; 3800 3801 if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) 3802 return true; 3803 3804 mapping = vma->vm_file->f_mapping; 3805 if (mapping_unevictable(mapping)) 3806 return true; 3807 3808 if (shmem_mapping(mapping)) 3809 return !walk->can_swap; 3810 3811 /* to exclude special mappings like dax, etc. */ 3812 return !mapping->a_ops->read_folio; 3813 } 3814 3815 /* 3816 * Some userspace memory allocators map many single-page VMAs. Instead of 3817 * returning back to the PGD table for each of such VMAs, finish an entire PMD 3818 * table to reduce zigzags and improve cache performance. 3819 */ 3820 static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, 3821 unsigned long *vm_start, unsigned long *vm_end) 3822 { 3823 unsigned long start = round_up(*vm_end, size); 3824 unsigned long end = (start | ~mask) + 1; 3825 VMA_ITERATOR(vmi, args->mm, start); 3826 3827 VM_WARN_ON_ONCE(mask & size); 3828 VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); 3829 3830 for_each_vma(vmi, args->vma) { 3831 if (end && end <= args->vma->vm_start) 3832 return false; 3833 3834 if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) 3835 continue; 3836 3837 *vm_start = max(start, args->vma->vm_start); 3838 *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; 3839 3840 return true; 3841 } 3842 3843 return false; 3844 } 3845 3846 static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr) 3847 { 3848 unsigned long pfn = pte_pfn(pte); 3849 3850 VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); 3851 3852 if (!pte_present(pte) || is_zero_pfn(pfn)) 3853 return -1; 3854 3855 if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte))) 3856 return -1; 3857 3858 if (WARN_ON_ONCE(!pfn_valid(pfn))) 3859 return -1; 3860 3861 return pfn; 3862 } 3863 3864 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) 3865 static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr) 3866 { 3867 unsigned long pfn = pmd_pfn(pmd); 3868 3869 VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); 3870 3871 if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) 3872 return -1; 3873 3874 if (WARN_ON_ONCE(pmd_devmap(pmd))) 3875 return -1; 3876 3877 if (WARN_ON_ONCE(!pfn_valid(pfn))) 3878 return -1; 3879 3880 return pfn; 3881 } 3882 #endif 3883 3884 static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg, 3885 struct pglist_data *pgdat, bool can_swap) 3886 { 3887 struct folio *folio; 3888 3889 /* try to avoid unnecessary memory loads */ 3890 if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) 3891 return NULL; 3892 3893 folio = pfn_folio(pfn); 3894 if (folio_nid(folio) != pgdat->node_id) 3895 return NULL; 3896 3897 if (folio_memcg_rcu(folio) != memcg) 3898 return NULL; 3899 3900 /* file VMAs can contain anon pages from COW */ 3901 if (!folio_is_file_lru(folio) && !can_swap) 3902 return NULL; 3903 3904 return folio; 3905 } 3906 3907 static bool suitable_to_scan(int total, int young) 3908 { 3909 int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); 3910 3911 /* suitable if the average number of young PTEs per cacheline is >=1 */ 3912 return young * n >= total; 3913 } 3914 3915 static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, 3916 struct mm_walk *args) 3917 { 3918 int i; 3919 pte_t *pte; 3920 spinlock_t *ptl; 3921 unsigned long addr; 3922 int total = 0; 3923 int young = 0; 3924 struct lru_gen_mm_walk *walk = args->private; 3925 struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); 3926 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3927 int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); 3928 3929 VM_WARN_ON_ONCE(pmd_leaf(*pmd)); 3930 3931 ptl = pte_lockptr(args->mm, pmd); 3932 if (!spin_trylock(ptl)) 3933 return false; 3934 3935 arch_enter_lazy_mmu_mode(); 3936 3937 pte = pte_offset_map(pmd, start & PMD_MASK); 3938 restart: 3939 for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { 3940 unsigned long pfn; 3941 struct folio *folio; 3942 3943 total++; 3944 walk->mm_stats[MM_LEAF_TOTAL]++; 3945 3946 pfn = get_pte_pfn(pte[i], args->vma, addr); 3947 if (pfn == -1) 3948 continue; 3949 3950 if (!pte_young(pte[i])) { 3951 walk->mm_stats[MM_LEAF_OLD]++; 3952 continue; 3953 } 3954 3955 folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); 3956 if (!folio) 3957 continue; 3958 3959 if (!ptep_test_and_clear_young(args->vma, addr, pte + i)) 3960 VM_WARN_ON_ONCE(true); 3961 3962 young++; 3963 walk->mm_stats[MM_LEAF_YOUNG]++; 3964 3965 if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && 3966 !(folio_test_anon(folio) && folio_test_swapbacked(folio) && 3967 !folio_test_swapcache(folio))) 3968 folio_mark_dirty(folio); 3969 3970 old_gen = folio_update_gen(folio, new_gen); 3971 if (old_gen >= 0 && old_gen != new_gen) 3972 update_batch_size(walk, folio, old_gen, new_gen); 3973 } 3974 3975 if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) 3976 goto restart; 3977 3978 pte_unmap(pte); 3979 3980 arch_leave_lazy_mmu_mode(); 3981 spin_unlock(ptl); 3982 3983 return suitable_to_scan(total, young); 3984 } 3985 3986 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) 3987 static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, 3988 struct mm_walk *args, unsigned long *bitmap, unsigned long *start) 3989 { 3990 int i; 3991 pmd_t *pmd; 3992 spinlock_t *ptl; 3993 struct lru_gen_mm_walk *walk = args->private; 3994 struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); 3995 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3996 int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); 3997 3998 VM_WARN_ON_ONCE(pud_leaf(*pud)); 3999 4000 /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ 4001 if (*start == -1) { 4002 *start = next; 4003 return; 4004 } 4005 4006 i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start); 4007 if (i && i <= MIN_LRU_BATCH) { 4008 __set_bit(i - 1, bitmap); 4009 return; 4010 } 4011 4012 pmd = pmd_offset(pud, *start); 4013 4014 ptl = pmd_lockptr(args->mm, pmd); 4015 if (!spin_trylock(ptl)) 4016 goto done; 4017 4018 arch_enter_lazy_mmu_mode(); 4019 4020 do { 4021 unsigned long pfn; 4022 struct folio *folio; 4023 unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start; 4024 4025 pfn = get_pmd_pfn(pmd[i], vma, addr); 4026 if (pfn == -1) 4027 goto next; 4028 4029 if (!pmd_trans_huge(pmd[i])) { 4030 if (arch_has_hw_nonleaf_pmd_young() && 4031 get_cap(LRU_GEN_NONLEAF_YOUNG)) 4032 pmdp_test_and_clear_young(vma, addr, pmd + i); 4033 goto next; 4034 } 4035 4036 folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); 4037 if (!folio) 4038 goto next; 4039 4040 if (!pmdp_test_and_clear_young(vma, addr, pmd + i)) 4041 goto next; 4042 4043 walk->mm_stats[MM_LEAF_YOUNG]++; 4044 4045 if (pmd_dirty(pmd[i]) && !folio_test_dirty(folio) && 4046 !(folio_test_anon(folio) && folio_test_swapbacked(folio) && 4047 !folio_test_swapcache(folio))) 4048 folio_mark_dirty(folio); 4049 4050 old_gen = folio_update_gen(folio, new_gen); 4051 if (old_gen >= 0 && old_gen != new_gen) 4052 update_batch_size(walk, folio, old_gen, new_gen); 4053 next: 4054 i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; 4055 } while (i <= MIN_LRU_BATCH); 4056 4057 arch_leave_lazy_mmu_mode(); 4058 spin_unlock(ptl); 4059 done: 4060 *start = -1; 4061 bitmap_zero(bitmap, MIN_LRU_BATCH); 4062 } 4063 #else 4064 static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, 4065 struct mm_walk *args, unsigned long *bitmap, unsigned long *start) 4066 { 4067 } 4068 #endif 4069 4070 static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, 4071 struct mm_walk *args) 4072 { 4073 int i; 4074 pmd_t *pmd; 4075 unsigned long next; 4076 unsigned long addr; 4077 struct vm_area_struct *vma; 4078 unsigned long pos = -1; 4079 struct lru_gen_mm_walk *walk = args->private; 4080 unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; 4081 4082 VM_WARN_ON_ONCE(pud_leaf(*pud)); 4083 4084 /* 4085 * Finish an entire PMD in two passes: the first only reaches to PTE 4086 * tables to avoid taking the PMD lock; the second, if necessary, takes 4087 * the PMD lock to clear the accessed bit in PMD entries. 4088 */ 4089 pmd = pmd_offset(pud, start & PUD_MASK); 4090 restart: 4091 /* walk_pte_range() may call get_next_vma() */ 4092 vma = args->vma; 4093 for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { 4094 pmd_t val = pmdp_get_lockless(pmd + i); 4095 4096 next = pmd_addr_end(addr, end); 4097 4098 if (!pmd_present(val) || is_huge_zero_pmd(val)) { 4099 walk->mm_stats[MM_LEAF_TOTAL]++; 4100 continue; 4101 } 4102 4103 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4104 if (pmd_trans_huge(val)) { 4105 unsigned long pfn = pmd_pfn(val); 4106 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 4107 4108 walk->mm_stats[MM_LEAF_TOTAL]++; 4109 4110 if (!pmd_young(val)) { 4111 walk->mm_stats[MM_LEAF_OLD]++; 4112 continue; 4113 } 4114 4115 /* try to avoid unnecessary memory loads */ 4116 if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) 4117 continue; 4118 4119 walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); 4120 continue; 4121 } 4122 #endif 4123 walk->mm_stats[MM_NONLEAF_TOTAL]++; 4124 4125 if (arch_has_hw_nonleaf_pmd_young() && 4126 get_cap(LRU_GEN_NONLEAF_YOUNG)) { 4127 if (!pmd_young(val)) 4128 continue; 4129 4130 walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); 4131 } 4132 4133 if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) 4134 continue; 4135 4136 walk->mm_stats[MM_NONLEAF_FOUND]++; 4137 4138 if (!walk_pte_range(&val, addr, next, args)) 4139 continue; 4140 4141 walk->mm_stats[MM_NONLEAF_ADDED]++; 4142 4143 /* carry over to the next generation */ 4144 update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i); 4145 } 4146 4147 walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos); 4148 4149 if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) 4150 goto restart; 4151 } 4152 4153 static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, 4154 struct mm_walk *args) 4155 { 4156 int i; 4157 pud_t *pud; 4158 unsigned long addr; 4159 unsigned long next; 4160 struct lru_gen_mm_walk *walk = args->private; 4161 4162 VM_WARN_ON_ONCE(p4d_leaf(*p4d)); 4163 4164 pud = pud_offset(p4d, start & P4D_MASK); 4165 restart: 4166 for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { 4167 pud_t val = READ_ONCE(pud[i]); 4168 4169 next = pud_addr_end(addr, end); 4170 4171 if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) 4172 continue; 4173 4174 walk_pmd_range(&val, addr, next, args); 4175 4176 /* a racy check to curtail the waiting time */ 4177 if (wq_has_sleeper(&walk->lruvec->mm_state.wait)) 4178 return 1; 4179 4180 if (need_resched() || walk->batched >= MAX_LRU_BATCH) { 4181 end = (addr | ~PUD_MASK) + 1; 4182 goto done; 4183 } 4184 } 4185 4186 if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) 4187 goto restart; 4188 4189 end = round_up(end, P4D_SIZE); 4190 done: 4191 if (!end || !args->vma) 4192 return 1; 4193 4194 walk->next_addr = max(end, args->vma->vm_start); 4195 4196 return -EAGAIN; 4197 } 4198 4199 static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk) 4200 { 4201 static const struct mm_walk_ops mm_walk_ops = { 4202 .test_walk = should_skip_vma, 4203 .p4d_entry = walk_pud_range, 4204 }; 4205 4206 int err; 4207 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4208 4209 walk->next_addr = FIRST_USER_ADDRESS; 4210 4211 do { 4212 err = -EBUSY; 4213 4214 /* folio_update_gen() requires stable folio_memcg() */ 4215 if (!mem_cgroup_trylock_pages(memcg)) 4216 break; 4217 4218 /* the caller might be holding the lock for write */ 4219 if (mmap_read_trylock(mm)) { 4220 err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); 4221 4222 mmap_read_unlock(mm); 4223 } 4224 4225 mem_cgroup_unlock_pages(); 4226 4227 if (walk->batched) { 4228 spin_lock_irq(&lruvec->lru_lock); 4229 reset_batch_size(lruvec, walk); 4230 spin_unlock_irq(&lruvec->lru_lock); 4231 } 4232 4233 cond_resched(); 4234 } while (err == -EAGAIN); 4235 } 4236 4237 static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat) 4238 { 4239 struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; 4240 4241 if (pgdat && current_is_kswapd()) { 4242 VM_WARN_ON_ONCE(walk); 4243 4244 walk = &pgdat->mm_walk; 4245 } else if (!pgdat && !walk) { 4246 VM_WARN_ON_ONCE(current_is_kswapd()); 4247 4248 walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); 4249 } 4250 4251 current->reclaim_state->mm_walk = walk; 4252 4253 return walk; 4254 } 4255 4256 static void clear_mm_walk(void) 4257 { 4258 struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; 4259 4260 VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); 4261 VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); 4262 4263 current->reclaim_state->mm_walk = NULL; 4264 4265 if (!current_is_kswapd()) 4266 kfree(walk); 4267 } 4268 4269 static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) 4270 { 4271 int zone; 4272 int remaining = MAX_LRU_BATCH; 4273 struct lru_gen_struct *lrugen = &lruvec->lrugen; 4274 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); 4275 4276 if (type == LRU_GEN_ANON && !can_swap) 4277 goto done; 4278 4279 /* prevent cold/hot inversion if force_scan is true */ 4280 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4281 struct list_head *head = &lrugen->lists[old_gen][type][zone]; 4282 4283 while (!list_empty(head)) { 4284 struct folio *folio = lru_to_folio(head); 4285 4286 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 4287 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 4288 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 4289 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 4290 4291 new_gen = folio_inc_gen(lruvec, folio, false); 4292 list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]); 4293 4294 if (!--remaining) 4295 return false; 4296 } 4297 } 4298 done: 4299 reset_ctrl_pos(lruvec, type, true); 4300 WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); 4301 4302 return true; 4303 } 4304 4305 static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) 4306 { 4307 int gen, type, zone; 4308 bool success = false; 4309 struct lru_gen_struct *lrugen = &lruvec->lrugen; 4310 DEFINE_MIN_SEQ(lruvec); 4311 4312 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 4313 4314 /* find the oldest populated generation */ 4315 for (type = !can_swap; type < ANON_AND_FILE; type++) { 4316 while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { 4317 gen = lru_gen_from_seq(min_seq[type]); 4318 4319 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4320 if (!list_empty(&lrugen->lists[gen][type][zone])) 4321 goto next; 4322 } 4323 4324 min_seq[type]++; 4325 } 4326 next: 4327 ; 4328 } 4329 4330 /* see the comment on lru_gen_struct */ 4331 if (can_swap) { 4332 min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]); 4333 min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]); 4334 } 4335 4336 for (type = !can_swap; type < ANON_AND_FILE; type++) { 4337 if (min_seq[type] == lrugen->min_seq[type]) 4338 continue; 4339 4340 reset_ctrl_pos(lruvec, type, true); 4341 WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); 4342 success = true; 4343 } 4344 4345 return success; 4346 } 4347 4348 static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) 4349 { 4350 int prev, next; 4351 int type, zone; 4352 struct lru_gen_struct *lrugen = &lruvec->lrugen; 4353 4354 spin_lock_irq(&lruvec->lru_lock); 4355 4356 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 4357 4358 for (type = ANON_AND_FILE - 1; type >= 0; type--) { 4359 if (get_nr_gens(lruvec, type) != MAX_NR_GENS) 4360 continue; 4361 4362 VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap)); 4363 4364 while (!inc_min_seq(lruvec, type, can_swap)) { 4365 spin_unlock_irq(&lruvec->lru_lock); 4366 cond_resched(); 4367 spin_lock_irq(&lruvec->lru_lock); 4368 } 4369 } 4370 4371 /* 4372 * Update the active/inactive LRU sizes for compatibility. Both sides of 4373 * the current max_seq need to be covered, since max_seq+1 can overlap 4374 * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do 4375 * overlap, cold/hot inversion happens. 4376 */ 4377 prev = lru_gen_from_seq(lrugen->max_seq - 1); 4378 next = lru_gen_from_seq(lrugen->max_seq + 1); 4379 4380 for (type = 0; type < ANON_AND_FILE; type++) { 4381 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4382 enum lru_list lru = type * LRU_INACTIVE_FILE; 4383 long delta = lrugen->nr_pages[prev][type][zone] - 4384 lrugen->nr_pages[next][type][zone]; 4385 4386 if (!delta) 4387 continue; 4388 4389 __update_lru_size(lruvec, lru, zone, delta); 4390 __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); 4391 } 4392 } 4393 4394 for (type = 0; type < ANON_AND_FILE; type++) 4395 reset_ctrl_pos(lruvec, type, false); 4396 4397 WRITE_ONCE(lrugen->timestamps[next], jiffies); 4398 /* make sure preceding modifications appear */ 4399 smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); 4400 4401 spin_unlock_irq(&lruvec->lru_lock); 4402 } 4403 4404 static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, 4405 struct scan_control *sc, bool can_swap, bool force_scan) 4406 { 4407 bool success; 4408 struct lru_gen_mm_walk *walk; 4409 struct mm_struct *mm = NULL; 4410 struct lru_gen_struct *lrugen = &lruvec->lrugen; 4411 4412 VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq)); 4413 4414 /* see the comment in iterate_mm_list() */ 4415 if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) { 4416 success = false; 4417 goto done; 4418 } 4419 4420 /* 4421 * If the hardware doesn't automatically set the accessed bit, fallback 4422 * to lru_gen_look_around(), which only clears the accessed bit in a 4423 * handful of PTEs. Spreading the work out over a period of time usually 4424 * is less efficient, but it avoids bursty page faults. 4425 */ 4426 if (!force_scan && !(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) { 4427 success = iterate_mm_list_nowalk(lruvec, max_seq); 4428 goto done; 4429 } 4430 4431 walk = set_mm_walk(NULL); 4432 if (!walk) { 4433 success = iterate_mm_list_nowalk(lruvec, max_seq); 4434 goto done; 4435 } 4436 4437 walk->lruvec = lruvec; 4438 walk->max_seq = max_seq; 4439 walk->can_swap = can_swap; 4440 walk->force_scan = force_scan; 4441 4442 do { 4443 success = iterate_mm_list(lruvec, walk, &mm); 4444 if (mm) 4445 walk_mm(lruvec, mm, walk); 4446 4447 cond_resched(); 4448 } while (mm); 4449 done: 4450 if (!success) { 4451 if (sc->priority <= DEF_PRIORITY - 2) 4452 wait_event_killable(lruvec->mm_state.wait, 4453 max_seq < READ_ONCE(lrugen->max_seq)); 4454 4455 return max_seq < READ_ONCE(lrugen->max_seq); 4456 } 4457 4458 VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq)); 4459 4460 inc_max_seq(lruvec, can_swap, force_scan); 4461 /* either this sees any waiters or they will see updated max_seq */ 4462 if (wq_has_sleeper(&lruvec->mm_state.wait)) 4463 wake_up_all(&lruvec->mm_state.wait); 4464 4465 return true; 4466 } 4467 4468 static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq, 4469 struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan) 4470 { 4471 int gen, type, zone; 4472 unsigned long old = 0; 4473 unsigned long young = 0; 4474 unsigned long total = 0; 4475 struct lru_gen_struct *lrugen = &lruvec->lrugen; 4476 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4477 4478 for (type = !can_swap; type < ANON_AND_FILE; type++) { 4479 unsigned long seq; 4480 4481 for (seq = min_seq[type]; seq <= max_seq; seq++) { 4482 unsigned long size = 0; 4483 4484 gen = lru_gen_from_seq(seq); 4485 4486 for (zone = 0; zone < MAX_NR_ZONES; zone++) 4487 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); 4488 4489 total += size; 4490 if (seq == max_seq) 4491 young += size; 4492 else if (seq + MIN_NR_GENS == max_seq) 4493 old += size; 4494 } 4495 } 4496 4497 /* try to scrape all its memory if this memcg was deleted */ 4498 *nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total; 4499 4500 /* 4501 * The aging tries to be lazy to reduce the overhead, while the eviction 4502 * stalls when the number of generations reaches MIN_NR_GENS. Hence, the 4503 * ideal number of generations is MIN_NR_GENS+1. 4504 */ 4505 if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) 4506 return true; 4507 if (min_seq[!can_swap] + MIN_NR_GENS < max_seq) 4508 return false; 4509 4510 /* 4511 * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1) 4512 * of the total number of pages for each generation. A reasonable range 4513 * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The 4514 * aging cares about the upper bound of hot pages, while the eviction 4515 * cares about the lower bound of cold pages. 4516 */ 4517 if (young * MIN_NR_GENS > total) 4518 return true; 4519 if (old * (MIN_NR_GENS + 2) < total) 4520 return true; 4521 4522 return false; 4523 } 4524 4525 static bool age_lruvec(struct lruvec *lruvec, struct scan_control *sc, unsigned long min_ttl) 4526 { 4527 bool need_aging; 4528 unsigned long nr_to_scan; 4529 int swappiness = get_swappiness(lruvec, sc); 4530 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4531 DEFINE_MAX_SEQ(lruvec); 4532 DEFINE_MIN_SEQ(lruvec); 4533 4534 VM_WARN_ON_ONCE(sc->memcg_low_reclaim); 4535 4536 mem_cgroup_calculate_protection(NULL, memcg); 4537 4538 if (mem_cgroup_below_min(NULL, memcg)) 4539 return false; 4540 4541 need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, swappiness, &nr_to_scan); 4542 4543 if (min_ttl) { 4544 int gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]); 4545 unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); 4546 4547 if (time_is_after_jiffies(birth + min_ttl)) 4548 return false; 4549 4550 /* the size is likely too small to be helpful */ 4551 if (!nr_to_scan && sc->priority != DEF_PRIORITY) 4552 return false; 4553 } 4554 4555 if (need_aging) 4556 try_to_inc_max_seq(lruvec, max_seq, sc, swappiness, false); 4557 4558 return true; 4559 } 4560 4561 /* to protect the working set of the last N jiffies */ 4562 static unsigned long lru_gen_min_ttl __read_mostly; 4563 4564 static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) 4565 { 4566 struct mem_cgroup *memcg; 4567 bool success = false; 4568 unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); 4569 4570 VM_WARN_ON_ONCE(!current_is_kswapd()); 4571 4572 sc->last_reclaimed = sc->nr_reclaimed; 4573 4574 /* 4575 * To reduce the chance of going into the aging path, which can be 4576 * costly, optimistically skip it if the flag below was cleared in the 4577 * eviction path. This improves the overall performance when multiple 4578 * memcgs are available. 4579 */ 4580 if (!sc->memcgs_need_aging) { 4581 sc->memcgs_need_aging = true; 4582 return; 4583 } 4584 4585 set_mm_walk(pgdat); 4586 4587 memcg = mem_cgroup_iter(NULL, NULL, NULL); 4588 do { 4589 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 4590 4591 if (age_lruvec(lruvec, sc, min_ttl)) 4592 success = true; 4593 4594 cond_resched(); 4595 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 4596 4597 clear_mm_walk(); 4598 4599 /* check the order to exclude compaction-induced reclaim */ 4600 if (success || !min_ttl || sc->order) 4601 return; 4602 4603 /* 4604 * The main goal is to OOM kill if every generation from all memcgs is 4605 * younger than min_ttl. However, another possibility is all memcgs are 4606 * either below min or empty. 4607 */ 4608 if (mutex_trylock(&oom_lock)) { 4609 struct oom_control oc = { 4610 .gfp_mask = sc->gfp_mask, 4611 }; 4612 4613 out_of_memory(&oc); 4614 4615 mutex_unlock(&oom_lock); 4616 } 4617 } 4618 4619 /* 4620 * This function exploits spatial locality when shrink_folio_list() walks the 4621 * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If 4622 * the scan was done cacheline efficiently, it adds the PMD entry pointing to 4623 * the PTE table to the Bloom filter. This forms a feedback loop between the 4624 * eviction and the aging. 4625 */ 4626 void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) 4627 { 4628 int i; 4629 pte_t *pte; 4630 unsigned long start; 4631 unsigned long end; 4632 unsigned long addr; 4633 struct lru_gen_mm_walk *walk; 4634 int young = 0; 4635 unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; 4636 struct folio *folio = pfn_folio(pvmw->pfn); 4637 struct mem_cgroup *memcg = folio_memcg(folio); 4638 struct pglist_data *pgdat = folio_pgdat(folio); 4639 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 4640 DEFINE_MAX_SEQ(lruvec); 4641 int old_gen, new_gen = lru_gen_from_seq(max_seq); 4642 4643 lockdep_assert_held(pvmw->ptl); 4644 VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); 4645 4646 if (spin_is_contended(pvmw->ptl)) 4647 return; 4648 4649 /* avoid taking the LRU lock under the PTL when possible */ 4650 walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; 4651 4652 start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start); 4653 end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1; 4654 4655 if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { 4656 if (pvmw->address - start < MIN_LRU_BATCH * PAGE_SIZE / 2) 4657 end = start + MIN_LRU_BATCH * PAGE_SIZE; 4658 else if (end - pvmw->address < MIN_LRU_BATCH * PAGE_SIZE / 2) 4659 start = end - MIN_LRU_BATCH * PAGE_SIZE; 4660 else { 4661 start = pvmw->address - MIN_LRU_BATCH * PAGE_SIZE / 2; 4662 end = pvmw->address + MIN_LRU_BATCH * PAGE_SIZE / 2; 4663 } 4664 } 4665 4666 pte = pvmw->pte - (pvmw->address - start) / PAGE_SIZE; 4667 4668 rcu_read_lock(); 4669 arch_enter_lazy_mmu_mode(); 4670 4671 for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { 4672 unsigned long pfn; 4673 4674 pfn = get_pte_pfn(pte[i], pvmw->vma, addr); 4675 if (pfn == -1) 4676 continue; 4677 4678 if (!pte_young(pte[i])) 4679 continue; 4680 4681 folio = get_pfn_folio(pfn, memcg, pgdat, !walk || walk->can_swap); 4682 if (!folio) 4683 continue; 4684 4685 if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i)) 4686 VM_WARN_ON_ONCE(true); 4687 4688 young++; 4689 4690 if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && 4691 !(folio_test_anon(folio) && folio_test_swapbacked(folio) && 4692 !folio_test_swapcache(folio))) 4693 folio_mark_dirty(folio); 4694 4695 old_gen = folio_lru_gen(folio); 4696 if (old_gen < 0) 4697 folio_set_referenced(folio); 4698 else if (old_gen != new_gen) 4699 __set_bit(i, bitmap); 4700 } 4701 4702 arch_leave_lazy_mmu_mode(); 4703 rcu_read_unlock(); 4704 4705 /* feedback from rmap walkers to page table walkers */ 4706 if (suitable_to_scan(i, young)) 4707 update_bloom_filter(lruvec, max_seq, pvmw->pmd); 4708 4709 if (!walk && bitmap_weight(bitmap, MIN_LRU_BATCH) < PAGEVEC_SIZE) { 4710 for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { 4711 folio = pfn_folio(pte_pfn(pte[i])); 4712 folio_activate(folio); 4713 } 4714 return; 4715 } 4716 4717 /* folio_update_gen() requires stable folio_memcg() */ 4718 if (!mem_cgroup_trylock_pages(memcg)) 4719 return; 4720 4721 if (!walk) { 4722 spin_lock_irq(&lruvec->lru_lock); 4723 new_gen = lru_gen_from_seq(lruvec->lrugen.max_seq); 4724 } 4725 4726 for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { 4727 folio = pfn_folio(pte_pfn(pte[i])); 4728 if (folio_memcg_rcu(folio) != memcg) 4729 continue; 4730 4731 old_gen = folio_update_gen(folio, new_gen); 4732 if (old_gen < 0 || old_gen == new_gen) 4733 continue; 4734 4735 if (walk) 4736 update_batch_size(walk, folio, old_gen, new_gen); 4737 else 4738 lru_gen_update_size(lruvec, folio, old_gen, new_gen); 4739 } 4740 4741 if (!walk) 4742 spin_unlock_irq(&lruvec->lru_lock); 4743 4744 mem_cgroup_unlock_pages(); 4745 } 4746 4747 /****************************************************************************** 4748 * the eviction 4749 ******************************************************************************/ 4750 4751 static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx) 4752 { 4753 bool success; 4754 int gen = folio_lru_gen(folio); 4755 int type = folio_is_file_lru(folio); 4756 int zone = folio_zonenum(folio); 4757 int delta = folio_nr_pages(folio); 4758 int refs = folio_lru_refs(folio); 4759 int tier = lru_tier_from_refs(refs); 4760 struct lru_gen_struct *lrugen = &lruvec->lrugen; 4761 4762 VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); 4763 4764 /* unevictable */ 4765 if (!folio_evictable(folio)) { 4766 success = lru_gen_del_folio(lruvec, folio, true); 4767 VM_WARN_ON_ONCE_FOLIO(!success, folio); 4768 folio_set_unevictable(folio); 4769 lruvec_add_folio(lruvec, folio); 4770 __count_vm_events(UNEVICTABLE_PGCULLED, delta); 4771 return true; 4772 } 4773 4774 /* dirty lazyfree */ 4775 if (type == LRU_GEN_FILE && folio_test_anon(folio) && folio_test_dirty(folio)) { 4776 success = lru_gen_del_folio(lruvec, folio, true); 4777 VM_WARN_ON_ONCE_FOLIO(!success, folio); 4778 folio_set_swapbacked(folio); 4779 lruvec_add_folio_tail(lruvec, folio); 4780 return true; 4781 } 4782 4783 /* promoted */ 4784 if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { 4785 list_move(&folio->lru, &lrugen->lists[gen][type][zone]); 4786 return true; 4787 } 4788 4789 /* protected */ 4790 if (tier > tier_idx) { 4791 int hist = lru_hist_from_seq(lrugen->min_seq[type]); 4792 4793 gen = folio_inc_gen(lruvec, folio, false); 4794 list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]); 4795 4796 WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 4797 lrugen->protected[hist][type][tier - 1] + delta); 4798 __mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta); 4799 return true; 4800 } 4801 4802 /* waiting for writeback */ 4803 if (folio_test_locked(folio) || folio_test_writeback(folio) || 4804 (type == LRU_GEN_FILE && folio_test_dirty(folio))) { 4805 gen = folio_inc_gen(lruvec, folio, true); 4806 list_move(&folio->lru, &lrugen->lists[gen][type][zone]); 4807 return true; 4808 } 4809 4810 return false; 4811 } 4812 4813 static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) 4814 { 4815 bool success; 4816 4817 /* unmapping inhibited */ 4818 if (!sc->may_unmap && folio_mapped(folio)) 4819 return false; 4820 4821 /* swapping inhibited */ 4822 if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) && 4823 (folio_test_dirty(folio) || 4824 (folio_test_anon(folio) && !folio_test_swapcache(folio)))) 4825 return false; 4826 4827 /* raced with release_pages() */ 4828 if (!folio_try_get(folio)) 4829 return false; 4830 4831 /* raced with another isolation */ 4832 if (!folio_test_clear_lru(folio)) { 4833 folio_put(folio); 4834 return false; 4835 } 4836 4837 /* see the comment on MAX_NR_TIERS */ 4838 if (!folio_test_referenced(folio)) 4839 set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0); 4840 4841 /* for shrink_folio_list() */ 4842 folio_clear_reclaim(folio); 4843 folio_clear_referenced(folio); 4844 4845 success = lru_gen_del_folio(lruvec, folio, true); 4846 VM_WARN_ON_ONCE_FOLIO(!success, folio); 4847 4848 return true; 4849 } 4850 4851 static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, 4852 int type, int tier, struct list_head *list) 4853 { 4854 int gen, zone; 4855 enum vm_event_item item; 4856 int sorted = 0; 4857 int scanned = 0; 4858 int isolated = 0; 4859 int remaining = MAX_LRU_BATCH; 4860 struct lru_gen_struct *lrugen = &lruvec->lrugen; 4861 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4862 4863 VM_WARN_ON_ONCE(!list_empty(list)); 4864 4865 if (get_nr_gens(lruvec, type) == MIN_NR_GENS) 4866 return 0; 4867 4868 gen = lru_gen_from_seq(lrugen->min_seq[type]); 4869 4870 for (zone = sc->reclaim_idx; zone >= 0; zone--) { 4871 LIST_HEAD(moved); 4872 int skipped = 0; 4873 struct list_head *head = &lrugen->lists[gen][type][zone]; 4874 4875 while (!list_empty(head)) { 4876 struct folio *folio = lru_to_folio(head); 4877 int delta = folio_nr_pages(folio); 4878 4879 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 4880 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 4881 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 4882 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 4883 4884 scanned += delta; 4885 4886 if (sort_folio(lruvec, folio, tier)) 4887 sorted += delta; 4888 else if (isolate_folio(lruvec, folio, sc)) { 4889 list_add(&folio->lru, list); 4890 isolated += delta; 4891 } else { 4892 list_move(&folio->lru, &moved); 4893 skipped += delta; 4894 } 4895 4896 if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH) 4897 break; 4898 } 4899 4900 if (skipped) { 4901 list_splice(&moved, head); 4902 __count_zid_vm_events(PGSCAN_SKIP, zone, skipped); 4903 } 4904 4905 if (!remaining || isolated >= MIN_LRU_BATCH) 4906 break; 4907 } 4908 4909 item = PGSCAN_KSWAPD + reclaimer_offset(); 4910 if (!cgroup_reclaim(sc)) { 4911 __count_vm_events(item, isolated); 4912 __count_vm_events(PGREFILL, sorted); 4913 } 4914 __count_memcg_events(memcg, item, isolated); 4915 __count_memcg_events(memcg, PGREFILL, sorted); 4916 __count_vm_events(PGSCAN_ANON + type, isolated); 4917 4918 /* 4919 * There might not be eligible pages due to reclaim_idx, may_unmap and 4920 * may_writepage. Check the remaining to prevent livelock if it's not 4921 * making progress. 4922 */ 4923 return isolated || !remaining ? scanned : 0; 4924 } 4925 4926 static int get_tier_idx(struct lruvec *lruvec, int type) 4927 { 4928 int tier; 4929 struct ctrl_pos sp, pv; 4930 4931 /* 4932 * To leave a margin for fluctuations, use a larger gain factor (1:2). 4933 * This value is chosen because any other tier would have at least twice 4934 * as many refaults as the first tier. 4935 */ 4936 read_ctrl_pos(lruvec, type, 0, 1, &sp); 4937 for (tier = 1; tier < MAX_NR_TIERS; tier++) { 4938 read_ctrl_pos(lruvec, type, tier, 2, &pv); 4939 if (!positive_ctrl_err(&sp, &pv)) 4940 break; 4941 } 4942 4943 return tier - 1; 4944 } 4945 4946 static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx) 4947 { 4948 int type, tier; 4949 struct ctrl_pos sp, pv; 4950 int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness }; 4951 4952 /* 4953 * Compare the first tier of anon with that of file to determine which 4954 * type to scan. Also need to compare other tiers of the selected type 4955 * with the first tier of the other type to determine the last tier (of 4956 * the selected type) to evict. 4957 */ 4958 read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp); 4959 read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv); 4960 type = positive_ctrl_err(&sp, &pv); 4961 4962 read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp); 4963 for (tier = 1; tier < MAX_NR_TIERS; tier++) { 4964 read_ctrl_pos(lruvec, type, tier, gain[type], &pv); 4965 if (!positive_ctrl_err(&sp, &pv)) 4966 break; 4967 } 4968 4969 *tier_idx = tier - 1; 4970 4971 return type; 4972 } 4973 4974 static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, 4975 int *type_scanned, struct list_head *list) 4976 { 4977 int i; 4978 int type; 4979 int scanned; 4980 int tier = -1; 4981 DEFINE_MIN_SEQ(lruvec); 4982 4983 /* 4984 * Try to make the obvious choice first. When anon and file are both 4985 * available from the same generation, interpret swappiness 1 as file 4986 * first and 200 as anon first. 4987 */ 4988 if (!swappiness) 4989 type = LRU_GEN_FILE; 4990 else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE]) 4991 type = LRU_GEN_ANON; 4992 else if (swappiness == 1) 4993 type = LRU_GEN_FILE; 4994 else if (swappiness == 200) 4995 type = LRU_GEN_ANON; 4996 else 4997 type = get_type_to_scan(lruvec, swappiness, &tier); 4998 4999 for (i = !swappiness; i < ANON_AND_FILE; i++) { 5000 if (tier < 0) 5001 tier = get_tier_idx(lruvec, type); 5002 5003 scanned = scan_folios(lruvec, sc, type, tier, list); 5004 if (scanned) 5005 break; 5006 5007 type = !type; 5008 tier = -1; 5009 } 5010 5011 *type_scanned = type; 5012 5013 return scanned; 5014 } 5015 5016 static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, 5017 bool *need_swapping) 5018 { 5019 int type; 5020 int scanned; 5021 int reclaimed; 5022 LIST_HEAD(list); 5023 LIST_HEAD(clean); 5024 struct folio *folio; 5025 struct folio *next; 5026 enum vm_event_item item; 5027 struct reclaim_stat stat; 5028 struct lru_gen_mm_walk *walk; 5029 bool skip_retry = false; 5030 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 5031 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 5032 5033 spin_lock_irq(&lruvec->lru_lock); 5034 5035 scanned = isolate_folios(lruvec, sc, swappiness, &type, &list); 5036 5037 scanned += try_to_inc_min_seq(lruvec, swappiness); 5038 5039 if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS) 5040 scanned = 0; 5041 5042 spin_unlock_irq(&lruvec->lru_lock); 5043 5044 if (list_empty(&list)) 5045 return scanned; 5046 retry: 5047 reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false); 5048 sc->nr_reclaimed += reclaimed; 5049 5050 list_for_each_entry_safe_reverse(folio, next, &list, lru) { 5051 if (!folio_evictable(folio)) { 5052 list_del(&folio->lru); 5053 folio_putback_lru(folio); 5054 continue; 5055 } 5056 5057 if (folio_test_reclaim(folio) && 5058 (folio_test_dirty(folio) || folio_test_writeback(folio))) { 5059 /* restore LRU_REFS_FLAGS cleared by isolate_folio() */ 5060 if (folio_test_workingset(folio)) 5061 folio_set_referenced(folio); 5062 continue; 5063 } 5064 5065 if (skip_retry || folio_test_active(folio) || folio_test_referenced(folio) || 5066 folio_mapped(folio) || folio_test_locked(folio) || 5067 folio_test_dirty(folio) || folio_test_writeback(folio)) { 5068 /* don't add rejected folios to the oldest generation */ 5069 set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 5070 BIT(PG_active)); 5071 continue; 5072 } 5073 5074 /* retry folios that may have missed folio_rotate_reclaimable() */ 5075 list_move(&folio->lru, &clean); 5076 sc->nr_scanned -= folio_nr_pages(folio); 5077 } 5078 5079 spin_lock_irq(&lruvec->lru_lock); 5080 5081 move_folios_to_lru(lruvec, &list); 5082 5083 walk = current->reclaim_state->mm_walk; 5084 if (walk && walk->batched) 5085 reset_batch_size(lruvec, walk); 5086 5087 item = PGSTEAL_KSWAPD + reclaimer_offset(); 5088 if (!cgroup_reclaim(sc)) 5089 __count_vm_events(item, reclaimed); 5090 __count_memcg_events(memcg, item, reclaimed); 5091 __count_vm_events(PGSTEAL_ANON + type, reclaimed); 5092 5093 spin_unlock_irq(&lruvec->lru_lock); 5094 5095 mem_cgroup_uncharge_list(&list); 5096 free_unref_page_list(&list); 5097 5098 INIT_LIST_HEAD(&list); 5099 list_splice_init(&clean, &list); 5100 5101 if (!list_empty(&list)) { 5102 skip_retry = true; 5103 goto retry; 5104 } 5105 5106 if (need_swapping && type == LRU_GEN_ANON) 5107 *need_swapping = true; 5108 5109 return scanned; 5110 } 5111 5112 /* 5113 * For future optimizations: 5114 * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg 5115 * reclaim. 5116 */ 5117 static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, 5118 bool can_swap, bool *need_aging) 5119 { 5120 unsigned long nr_to_scan; 5121 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 5122 DEFINE_MAX_SEQ(lruvec); 5123 DEFINE_MIN_SEQ(lruvec); 5124 5125 if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg) || 5126 (mem_cgroup_below_low(sc->target_mem_cgroup, memcg) && 5127 !sc->memcg_low_reclaim)) 5128 return 0; 5129 5130 *need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, can_swap, &nr_to_scan); 5131 if (!*need_aging) 5132 return nr_to_scan; 5133 5134 /* skip the aging path at the default priority */ 5135 if (sc->priority == DEF_PRIORITY) 5136 goto done; 5137 5138 /* leave the work to lru_gen_age_node() */ 5139 if (current_is_kswapd()) 5140 return 0; 5141 5142 if (try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false)) 5143 return nr_to_scan; 5144 done: 5145 return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; 5146 } 5147 5148 static bool should_abort_scan(struct lruvec *lruvec, unsigned long seq, 5149 struct scan_control *sc, bool need_swapping) 5150 { 5151 int i; 5152 DEFINE_MAX_SEQ(lruvec); 5153 5154 if (!current_is_kswapd()) { 5155 /* age each memcg at most once to ensure fairness */ 5156 if (max_seq - seq > 1) 5157 return true; 5158 5159 /* over-swapping can increase allocation latency */ 5160 if (sc->nr_reclaimed >= sc->nr_to_reclaim && need_swapping) 5161 return true; 5162 5163 /* give this thread a chance to exit and free its memory */ 5164 if (fatal_signal_pending(current)) { 5165 sc->nr_reclaimed += MIN_LRU_BATCH; 5166 return true; 5167 } 5168 5169 if (cgroup_reclaim(sc)) 5170 return false; 5171 } else if (sc->nr_reclaimed - sc->last_reclaimed < sc->nr_to_reclaim) 5172 return false; 5173 5174 /* keep scanning at low priorities to ensure fairness */ 5175 if (sc->priority > DEF_PRIORITY - 2) 5176 return false; 5177 5178 /* 5179 * A minimum amount of work was done under global memory pressure. For 5180 * kswapd, it may be overshooting. For direct reclaim, the allocation 5181 * may succeed if all suitable zones are somewhat safe. In either case, 5182 * it's better to stop now, and restart later if necessary. 5183 */ 5184 for (i = 0; i <= sc->reclaim_idx; i++) { 5185 unsigned long wmark; 5186 struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i; 5187 5188 if (!managed_zone(zone)) 5189 continue; 5190 5191 wmark = current_is_kswapd() ? high_wmark_pages(zone) : low_wmark_pages(zone); 5192 if (wmark > zone_page_state(zone, NR_FREE_PAGES)) 5193 return false; 5194 } 5195 5196 sc->nr_reclaimed += MIN_LRU_BATCH; 5197 5198 return true; 5199 } 5200 5201 static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5202 { 5203 struct blk_plug plug; 5204 bool need_aging = false; 5205 bool need_swapping = false; 5206 unsigned long scanned = 0; 5207 unsigned long reclaimed = sc->nr_reclaimed; 5208 DEFINE_MAX_SEQ(lruvec); 5209 5210 lru_add_drain(); 5211 5212 blk_start_plug(&plug); 5213 5214 set_mm_walk(lruvec_pgdat(lruvec)); 5215 5216 while (true) { 5217 int delta; 5218 int swappiness; 5219 unsigned long nr_to_scan; 5220 5221 if (sc->may_swap) 5222 swappiness = get_swappiness(lruvec, sc); 5223 else if (!cgroup_reclaim(sc) && get_swappiness(lruvec, sc)) 5224 swappiness = 1; 5225 else 5226 swappiness = 0; 5227 5228 nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness, &need_aging); 5229 if (!nr_to_scan) 5230 goto done; 5231 5232 delta = evict_folios(lruvec, sc, swappiness, &need_swapping); 5233 if (!delta) 5234 goto done; 5235 5236 scanned += delta; 5237 if (scanned >= nr_to_scan) 5238 break; 5239 5240 if (should_abort_scan(lruvec, max_seq, sc, need_swapping)) 5241 break; 5242 5243 cond_resched(); 5244 } 5245 5246 /* see the comment in lru_gen_age_node() */ 5247 if (sc->nr_reclaimed - reclaimed >= MIN_LRU_BATCH && !need_aging) 5248 sc->memcgs_need_aging = false; 5249 done: 5250 clear_mm_walk(); 5251 5252 blk_finish_plug(&plug); 5253 } 5254 5255 /****************************************************************************** 5256 * state change 5257 ******************************************************************************/ 5258 5259 static bool __maybe_unused state_is_valid(struct lruvec *lruvec) 5260 { 5261 struct lru_gen_struct *lrugen = &lruvec->lrugen; 5262 5263 if (lrugen->enabled) { 5264 enum lru_list lru; 5265 5266 for_each_evictable_lru(lru) { 5267 if (!list_empty(&lruvec->lists[lru])) 5268 return false; 5269 } 5270 } else { 5271 int gen, type, zone; 5272 5273 for_each_gen_type_zone(gen, type, zone) { 5274 if (!list_empty(&lrugen->lists[gen][type][zone])) 5275 return false; 5276 } 5277 } 5278 5279 return true; 5280 } 5281 5282 static bool fill_evictable(struct lruvec *lruvec) 5283 { 5284 enum lru_list lru; 5285 int remaining = MAX_LRU_BATCH; 5286 5287 for_each_evictable_lru(lru) { 5288 int type = is_file_lru(lru); 5289 bool active = is_active_lru(lru); 5290 struct list_head *head = &lruvec->lists[lru]; 5291 5292 while (!list_empty(head)) { 5293 bool success; 5294 struct folio *folio = lru_to_folio(head); 5295 5296 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 5297 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); 5298 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 5299 VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); 5300 5301 lruvec_del_folio(lruvec, folio); 5302 success = lru_gen_add_folio(lruvec, folio, false); 5303 VM_WARN_ON_ONCE(!success); 5304 5305 if (!--remaining) 5306 return false; 5307 } 5308 } 5309 5310 return true; 5311 } 5312 5313 static bool drain_evictable(struct lruvec *lruvec) 5314 { 5315 int gen, type, zone; 5316 int remaining = MAX_LRU_BATCH; 5317 5318 for_each_gen_type_zone(gen, type, zone) { 5319 struct list_head *head = &lruvec->lrugen.lists[gen][type][zone]; 5320 5321 while (!list_empty(head)) { 5322 bool success; 5323 struct folio *folio = lru_to_folio(head); 5324 5325 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 5326 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 5327 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 5328 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 5329 5330 success = lru_gen_del_folio(lruvec, folio, false); 5331 VM_WARN_ON_ONCE(!success); 5332 lruvec_add_folio(lruvec, folio); 5333 5334 if (!--remaining) 5335 return false; 5336 } 5337 } 5338 5339 return true; 5340 } 5341 5342 static void lru_gen_change_state(bool enabled) 5343 { 5344 static DEFINE_MUTEX(state_mutex); 5345 5346 struct mem_cgroup *memcg; 5347 5348 cgroup_lock(); 5349 cpus_read_lock(); 5350 get_online_mems(); 5351 mutex_lock(&state_mutex); 5352 5353 if (enabled == lru_gen_enabled()) 5354 goto unlock; 5355 5356 if (enabled) 5357 static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); 5358 else 5359 static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); 5360 5361 memcg = mem_cgroup_iter(NULL, NULL, NULL); 5362 do { 5363 int nid; 5364 5365 for_each_node(nid) { 5366 struct lruvec *lruvec = get_lruvec(memcg, nid); 5367 5368 spin_lock_irq(&lruvec->lru_lock); 5369 5370 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 5371 VM_WARN_ON_ONCE(!state_is_valid(lruvec)); 5372 5373 lruvec->lrugen.enabled = enabled; 5374 5375 while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { 5376 spin_unlock_irq(&lruvec->lru_lock); 5377 cond_resched(); 5378 spin_lock_irq(&lruvec->lru_lock); 5379 } 5380 5381 spin_unlock_irq(&lruvec->lru_lock); 5382 } 5383 5384 cond_resched(); 5385 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 5386 unlock: 5387 mutex_unlock(&state_mutex); 5388 put_online_mems(); 5389 cpus_read_unlock(); 5390 cgroup_unlock(); 5391 } 5392 5393 /****************************************************************************** 5394 * sysfs interface 5395 ******************************************************************************/ 5396 5397 static ssize_t show_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, char *buf) 5398 { 5399 return sprintf(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); 5400 } 5401 5402 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5403 static ssize_t store_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, 5404 const char *buf, size_t len) 5405 { 5406 unsigned int msecs; 5407 5408 if (kstrtouint(buf, 0, &msecs)) 5409 return -EINVAL; 5410 5411 WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs)); 5412 5413 return len; 5414 } 5415 5416 static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR( 5417 min_ttl_ms, 0644, show_min_ttl, store_min_ttl 5418 ); 5419 5420 static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, char *buf) 5421 { 5422 unsigned int caps = 0; 5423 5424 if (get_cap(LRU_GEN_CORE)) 5425 caps |= BIT(LRU_GEN_CORE); 5426 5427 if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK)) 5428 caps |= BIT(LRU_GEN_MM_WALK); 5429 5430 if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG)) 5431 caps |= BIT(LRU_GEN_NONLEAF_YOUNG); 5432 5433 return sysfs_emit(buf, "0x%04x\n", caps); 5434 } 5435 5436 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5437 static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr, 5438 const char *buf, size_t len) 5439 { 5440 int i; 5441 unsigned int caps; 5442 5443 if (tolower(*buf) == 'n') 5444 caps = 0; 5445 else if (tolower(*buf) == 'y') 5446 caps = -1; 5447 else if (kstrtouint(buf, 0, &caps)) 5448 return -EINVAL; 5449 5450 for (i = 0; i < NR_LRU_GEN_CAPS; i++) { 5451 bool enabled = caps & BIT(i); 5452 5453 if (i == LRU_GEN_CORE) 5454 lru_gen_change_state(enabled); 5455 else if (enabled) 5456 static_branch_enable(&lru_gen_caps[i]); 5457 else 5458 static_branch_disable(&lru_gen_caps[i]); 5459 } 5460 5461 return len; 5462 } 5463 5464 static struct kobj_attribute lru_gen_enabled_attr = __ATTR( 5465 enabled, 0644, show_enabled, store_enabled 5466 ); 5467 5468 static struct attribute *lru_gen_attrs[] = { 5469 &lru_gen_min_ttl_attr.attr, 5470 &lru_gen_enabled_attr.attr, 5471 NULL 5472 }; 5473 5474 static struct attribute_group lru_gen_attr_group = { 5475 .name = "lru_gen", 5476 .attrs = lru_gen_attrs, 5477 }; 5478 5479 /****************************************************************************** 5480 * debugfs interface 5481 ******************************************************************************/ 5482 5483 static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) 5484 { 5485 struct mem_cgroup *memcg; 5486 loff_t nr_to_skip = *pos; 5487 5488 m->private = kvmalloc(PATH_MAX, GFP_KERNEL); 5489 if (!m->private) 5490 return ERR_PTR(-ENOMEM); 5491 5492 memcg = mem_cgroup_iter(NULL, NULL, NULL); 5493 do { 5494 int nid; 5495 5496 for_each_node_state(nid, N_MEMORY) { 5497 if (!nr_to_skip--) 5498 return get_lruvec(memcg, nid); 5499 } 5500 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 5501 5502 return NULL; 5503 } 5504 5505 static void lru_gen_seq_stop(struct seq_file *m, void *v) 5506 { 5507 if (!IS_ERR_OR_NULL(v)) 5508 mem_cgroup_iter_break(NULL, lruvec_memcg(v)); 5509 5510 kvfree(m->private); 5511 m->private = NULL; 5512 } 5513 5514 static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) 5515 { 5516 int nid = lruvec_pgdat(v)->node_id; 5517 struct mem_cgroup *memcg = lruvec_memcg(v); 5518 5519 ++*pos; 5520 5521 nid = next_memory_node(nid); 5522 if (nid == MAX_NUMNODES) { 5523 memcg = mem_cgroup_iter(NULL, memcg, NULL); 5524 if (!memcg) 5525 return NULL; 5526 5527 nid = first_memory_node; 5528 } 5529 5530 return get_lruvec(memcg, nid); 5531 } 5532 5533 static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, 5534 unsigned long max_seq, unsigned long *min_seq, 5535 unsigned long seq) 5536 { 5537 int i; 5538 int type, tier; 5539 int hist = lru_hist_from_seq(seq); 5540 struct lru_gen_struct *lrugen = &lruvec->lrugen; 5541 5542 for (tier = 0; tier < MAX_NR_TIERS; tier++) { 5543 seq_printf(m, " %10d", tier); 5544 for (type = 0; type < ANON_AND_FILE; type++) { 5545 const char *s = " "; 5546 unsigned long n[3] = {}; 5547 5548 if (seq == max_seq) { 5549 s = "RT "; 5550 n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); 5551 n[1] = READ_ONCE(lrugen->avg_total[type][tier]); 5552 } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { 5553 s = "rep"; 5554 n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); 5555 n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); 5556 if (tier) 5557 n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]); 5558 } 5559 5560 for (i = 0; i < 3; i++) 5561 seq_printf(m, " %10lu%c", n[i], s[i]); 5562 } 5563 seq_putc(m, '\n'); 5564 } 5565 5566 seq_puts(m, " "); 5567 for (i = 0; i < NR_MM_STATS; i++) { 5568 const char *s = " "; 5569 unsigned long n = 0; 5570 5571 if (seq == max_seq && NR_HIST_GENS == 1) { 5572 s = "LOYNFA"; 5573 n = READ_ONCE(lruvec->mm_state.stats[hist][i]); 5574 } else if (seq != max_seq && NR_HIST_GENS > 1) { 5575 s = "loynfa"; 5576 n = READ_ONCE(lruvec->mm_state.stats[hist][i]); 5577 } 5578 5579 seq_printf(m, " %10lu%c", n, s[i]); 5580 } 5581 seq_putc(m, '\n'); 5582 } 5583 5584 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5585 static int lru_gen_seq_show(struct seq_file *m, void *v) 5586 { 5587 unsigned long seq; 5588 bool full = !debugfs_real_fops(m->file)->write; 5589 struct lruvec *lruvec = v; 5590 struct lru_gen_struct *lrugen = &lruvec->lrugen; 5591 int nid = lruvec_pgdat(lruvec)->node_id; 5592 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 5593 DEFINE_MAX_SEQ(lruvec); 5594 DEFINE_MIN_SEQ(lruvec); 5595 5596 if (nid == first_memory_node) { 5597 const char *path = memcg ? m->private : ""; 5598 5599 #ifdef CONFIG_MEMCG 5600 if (memcg) 5601 cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); 5602 #endif 5603 seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path); 5604 } 5605 5606 seq_printf(m, " node %5d\n", nid); 5607 5608 if (!full) 5609 seq = min_seq[LRU_GEN_ANON]; 5610 else if (max_seq >= MAX_NR_GENS) 5611 seq = max_seq - MAX_NR_GENS + 1; 5612 else 5613 seq = 0; 5614 5615 for (; seq <= max_seq; seq++) { 5616 int type, zone; 5617 int gen = lru_gen_from_seq(seq); 5618 unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); 5619 5620 seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); 5621 5622 for (type = 0; type < ANON_AND_FILE; type++) { 5623 unsigned long size = 0; 5624 char mark = full && seq < min_seq[type] ? 'x' : ' '; 5625 5626 for (zone = 0; zone < MAX_NR_ZONES; zone++) 5627 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); 5628 5629 seq_printf(m, " %10lu%c", size, mark); 5630 } 5631 5632 seq_putc(m, '\n'); 5633 5634 if (full) 5635 lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); 5636 } 5637 5638 return 0; 5639 } 5640 5641 static const struct seq_operations lru_gen_seq_ops = { 5642 .start = lru_gen_seq_start, 5643 .stop = lru_gen_seq_stop, 5644 .next = lru_gen_seq_next, 5645 .show = lru_gen_seq_show, 5646 }; 5647 5648 static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, 5649 bool can_swap, bool force_scan) 5650 { 5651 DEFINE_MAX_SEQ(lruvec); 5652 DEFINE_MIN_SEQ(lruvec); 5653 5654 if (seq < max_seq) 5655 return 0; 5656 5657 if (seq > max_seq) 5658 return -EINVAL; 5659 5660 if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq) 5661 return -ERANGE; 5662 5663 try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan); 5664 5665 return 0; 5666 } 5667 5668 static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, 5669 int swappiness, unsigned long nr_to_reclaim) 5670 { 5671 DEFINE_MAX_SEQ(lruvec); 5672 5673 if (seq + MIN_NR_GENS > max_seq) 5674 return -EINVAL; 5675 5676 sc->nr_reclaimed = 0; 5677 5678 while (!signal_pending(current)) { 5679 DEFINE_MIN_SEQ(lruvec); 5680 5681 if (seq < min_seq[!swappiness]) 5682 return 0; 5683 5684 if (sc->nr_reclaimed >= nr_to_reclaim) 5685 return 0; 5686 5687 if (!evict_folios(lruvec, sc, swappiness, NULL)) 5688 return 0; 5689 5690 cond_resched(); 5691 } 5692 5693 return -EINTR; 5694 } 5695 5696 static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, 5697 struct scan_control *sc, int swappiness, unsigned long opt) 5698 { 5699 struct lruvec *lruvec; 5700 int err = -EINVAL; 5701 struct mem_cgroup *memcg = NULL; 5702 5703 if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) 5704 return -EINVAL; 5705 5706 if (!mem_cgroup_disabled()) { 5707 rcu_read_lock(); 5708 memcg = mem_cgroup_from_id(memcg_id); 5709 #ifdef CONFIG_MEMCG 5710 if (memcg && !css_tryget(&memcg->css)) 5711 memcg = NULL; 5712 #endif 5713 rcu_read_unlock(); 5714 5715 if (!memcg) 5716 return -EINVAL; 5717 } 5718 5719 if (memcg_id != mem_cgroup_id(memcg)) 5720 goto done; 5721 5722 lruvec = get_lruvec(memcg, nid); 5723 5724 if (swappiness < 0) 5725 swappiness = get_swappiness(lruvec, sc); 5726 else if (swappiness > 200) 5727 goto done; 5728 5729 switch (cmd) { 5730 case '+': 5731 err = run_aging(lruvec, seq, sc, swappiness, opt); 5732 break; 5733 case '-': 5734 err = run_eviction(lruvec, seq, sc, swappiness, opt); 5735 break; 5736 } 5737 done: 5738 mem_cgroup_put(memcg); 5739 5740 return err; 5741 } 5742 5743 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5744 static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, 5745 size_t len, loff_t *pos) 5746 { 5747 void *buf; 5748 char *cur, *next; 5749 unsigned int flags; 5750 struct blk_plug plug; 5751 int err = -EINVAL; 5752 struct scan_control sc = { 5753 .may_writepage = true, 5754 .may_unmap = true, 5755 .may_swap = true, 5756 .reclaim_idx = MAX_NR_ZONES - 1, 5757 .gfp_mask = GFP_KERNEL, 5758 }; 5759 5760 buf = kvmalloc(len + 1, GFP_KERNEL); 5761 if (!buf) 5762 return -ENOMEM; 5763 5764 if (copy_from_user(buf, src, len)) { 5765 kvfree(buf); 5766 return -EFAULT; 5767 } 5768 5769 set_task_reclaim_state(current, &sc.reclaim_state); 5770 flags = memalloc_noreclaim_save(); 5771 blk_start_plug(&plug); 5772 if (!set_mm_walk(NULL)) { 5773 err = -ENOMEM; 5774 goto done; 5775 } 5776 5777 next = buf; 5778 next[len] = '\0'; 5779 5780 while ((cur = strsep(&next, ",;\n"))) { 5781 int n; 5782 int end; 5783 char cmd; 5784 unsigned int memcg_id; 5785 unsigned int nid; 5786 unsigned long seq; 5787 unsigned int swappiness = -1; 5788 unsigned long opt = -1; 5789 5790 cur = skip_spaces(cur); 5791 if (!*cur) 5792 continue; 5793 5794 n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid, 5795 &seq, &end, &swappiness, &end, &opt, &end); 5796 if (n < 4 || cur[end]) { 5797 err = -EINVAL; 5798 break; 5799 } 5800 5801 err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt); 5802 if (err) 5803 break; 5804 } 5805 done: 5806 clear_mm_walk(); 5807 blk_finish_plug(&plug); 5808 memalloc_noreclaim_restore(flags); 5809 set_task_reclaim_state(current, NULL); 5810 5811 kvfree(buf); 5812 5813 return err ? : len; 5814 } 5815 5816 static int lru_gen_seq_open(struct inode *inode, struct file *file) 5817 { 5818 return seq_open(file, &lru_gen_seq_ops); 5819 } 5820 5821 static const struct file_operations lru_gen_rw_fops = { 5822 .open = lru_gen_seq_open, 5823 .read = seq_read, 5824 .write = lru_gen_seq_write, 5825 .llseek = seq_lseek, 5826 .release = seq_release, 5827 }; 5828 5829 static const struct file_operations lru_gen_ro_fops = { 5830 .open = lru_gen_seq_open, 5831 .read = seq_read, 5832 .llseek = seq_lseek, 5833 .release = seq_release, 5834 }; 5835 5836 /****************************************************************************** 5837 * initialization 5838 ******************************************************************************/ 5839 5840 void lru_gen_init_lruvec(struct lruvec *lruvec) 5841 { 5842 int i; 5843 int gen, type, zone; 5844 struct lru_gen_struct *lrugen = &lruvec->lrugen; 5845 5846 lrugen->max_seq = MIN_NR_GENS + 1; 5847 lrugen->enabled = lru_gen_enabled(); 5848 5849 for (i = 0; i <= MIN_NR_GENS + 1; i++) 5850 lrugen->timestamps[i] = jiffies; 5851 5852 for_each_gen_type_zone(gen, type, zone) 5853 INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]); 5854 5855 lruvec->mm_state.seq = MIN_NR_GENS; 5856 init_waitqueue_head(&lruvec->mm_state.wait); 5857 } 5858 5859 #ifdef CONFIG_MEMCG 5860 void lru_gen_init_memcg(struct mem_cgroup *memcg) 5861 { 5862 INIT_LIST_HEAD(&memcg->mm_list.fifo); 5863 spin_lock_init(&memcg->mm_list.lock); 5864 } 5865 5866 void lru_gen_exit_memcg(struct mem_cgroup *memcg) 5867 { 5868 int i; 5869 int nid; 5870 5871 for_each_node(nid) { 5872 struct lruvec *lruvec = get_lruvec(memcg, nid); 5873 5874 VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, 5875 sizeof(lruvec->lrugen.nr_pages))); 5876 5877 for (i = 0; i < NR_BLOOM_FILTERS; i++) { 5878 bitmap_free(lruvec->mm_state.filters[i]); 5879 lruvec->mm_state.filters[i] = NULL; 5880 } 5881 } 5882 } 5883 #endif 5884 5885 static int __init init_lru_gen(void) 5886 { 5887 BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); 5888 BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); 5889 5890 if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) 5891 pr_err("lru_gen: failed to create sysfs group\n"); 5892 5893 debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops); 5894 debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops); 5895 5896 return 0; 5897 }; 5898 late_initcall(init_lru_gen); 5899 5900 #else /* !CONFIG_LRU_GEN */ 5901 5902 static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) 5903 { 5904 } 5905 5906 static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5907 { 5908 } 5909 5910 #endif /* CONFIG_LRU_GEN */ 5911 5912 static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5913 { 5914 unsigned long nr[NR_LRU_LISTS]; 5915 unsigned long targets[NR_LRU_LISTS]; 5916 unsigned long nr_to_scan; 5917 enum lru_list lru; 5918 unsigned long nr_reclaimed = 0; 5919 unsigned long nr_to_reclaim = sc->nr_to_reclaim; 5920 bool proportional_reclaim; 5921 struct blk_plug plug; 5922 5923 if (lru_gen_enabled()) { 5924 lru_gen_shrink_lruvec(lruvec, sc); 5925 return; 5926 } 5927 5928 get_scan_count(lruvec, sc, nr); 5929 5930 /* Record the original scan target for proportional adjustments later */ 5931 memcpy(targets, nr, sizeof(nr)); 5932 5933 /* 5934 * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal 5935 * event that can occur when there is little memory pressure e.g. 5936 * multiple streaming readers/writers. Hence, we do not abort scanning 5937 * when the requested number of pages are reclaimed when scanning at 5938 * DEF_PRIORITY on the assumption that the fact we are direct 5939 * reclaiming implies that kswapd is not keeping up and it is best to 5940 * do a batch of work at once. For memcg reclaim one check is made to 5941 * abort proportional reclaim if either the file or anon lru has already 5942 * dropped to zero at the first pass. 5943 */ 5944 proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() && 5945 sc->priority == DEF_PRIORITY); 5946 5947 blk_start_plug(&plug); 5948 while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || 5949 nr[LRU_INACTIVE_FILE]) { 5950 unsigned long nr_anon, nr_file, percentage; 5951 unsigned long nr_scanned; 5952 5953 for_each_evictable_lru(lru) { 5954 if (nr[lru]) { 5955 nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); 5956 nr[lru] -= nr_to_scan; 5957 5958 nr_reclaimed += shrink_list(lru, nr_to_scan, 5959 lruvec, sc); 5960 } 5961 } 5962 5963 cond_resched(); 5964 5965 if (nr_reclaimed < nr_to_reclaim || proportional_reclaim) 5966 continue; 5967 5968 /* 5969 * For kswapd and memcg, reclaim at least the number of pages 5970 * requested. Ensure that the anon and file LRUs are scanned 5971 * proportionally what was requested by get_scan_count(). We 5972 * stop reclaiming one LRU and reduce the amount scanning 5973 * proportional to the original scan target. 5974 */ 5975 nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; 5976 nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; 5977 5978 /* 5979 * It's just vindictive to attack the larger once the smaller 5980 * has gone to zero. And given the way we stop scanning the 5981 * smaller below, this makes sure that we only make one nudge 5982 * towards proportionality once we've got nr_to_reclaim. 5983 */ 5984 if (!nr_file || !nr_anon) 5985 break; 5986 5987 if (nr_file > nr_anon) { 5988 unsigned long scan_target = targets[LRU_INACTIVE_ANON] + 5989 targets[LRU_ACTIVE_ANON] + 1; 5990 lru = LRU_BASE; 5991 percentage = nr_anon * 100 / scan_target; 5992 } else { 5993 unsigned long scan_target = targets[LRU_INACTIVE_FILE] + 5994 targets[LRU_ACTIVE_FILE] + 1; 5995 lru = LRU_FILE; 5996 percentage = nr_file * 100 / scan_target; 5997 } 5998 5999 /* Stop scanning the smaller of the LRU */ 6000 nr[lru] = 0; 6001 nr[lru + LRU_ACTIVE] = 0; 6002 6003 /* 6004 * Recalculate the other LRU scan count based on its original 6005 * scan target and the percentage scanning already complete 6006 */ 6007 lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; 6008 nr_scanned = targets[lru] - nr[lru]; 6009 nr[lru] = targets[lru] * (100 - percentage) / 100; 6010 nr[lru] -= min(nr[lru], nr_scanned); 6011 6012 lru += LRU_ACTIVE; 6013 nr_scanned = targets[lru] - nr[lru]; 6014 nr[lru] = targets[lru] * (100 - percentage) / 100; 6015 nr[lru] -= min(nr[lru], nr_scanned); 6016 } 6017 blk_finish_plug(&plug); 6018 sc->nr_reclaimed += nr_reclaimed; 6019 6020 /* 6021 * Even if we did not try to evict anon pages at all, we want to 6022 * rebalance the anon lru active/inactive ratio. 6023 */ 6024 if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && 6025 inactive_is_low(lruvec, LRU_INACTIVE_ANON)) 6026 shrink_active_list(SWAP_CLUSTER_MAX, lruvec, 6027 sc, LRU_ACTIVE_ANON); 6028 } 6029 6030 /* Use reclaim/compaction for costly allocs or under memory pressure */ 6031 static bool in_reclaim_compaction(struct scan_control *sc) 6032 { 6033 if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && 6034 (sc->order > PAGE_ALLOC_COSTLY_ORDER || 6035 sc->priority < DEF_PRIORITY - 2)) 6036 return true; 6037 6038 return false; 6039 } 6040 6041 /* 6042 * Reclaim/compaction is used for high-order allocation requests. It reclaims 6043 * order-0 pages before compacting the zone. should_continue_reclaim() returns 6044 * true if more pages should be reclaimed such that when the page allocator 6045 * calls try_to_compact_pages() that it will have enough free pages to succeed. 6046 * It will give up earlier than that if there is difficulty reclaiming pages. 6047 */ 6048 static inline bool should_continue_reclaim(struct pglist_data *pgdat, 6049 unsigned long nr_reclaimed, 6050 struct scan_control *sc) 6051 { 6052 unsigned long pages_for_compaction; 6053 unsigned long inactive_lru_pages; 6054 int z; 6055 6056 /* If not in reclaim/compaction mode, stop */ 6057 if (!in_reclaim_compaction(sc)) 6058 return false; 6059 6060 /* 6061 * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX 6062 * number of pages that were scanned. This will return to the caller 6063 * with the risk reclaim/compaction and the resulting allocation attempt 6064 * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL 6065 * allocations through requiring that the full LRU list has been scanned 6066 * first, by assuming that zero delta of sc->nr_scanned means full LRU 6067 * scan, but that approximation was wrong, and there were corner cases 6068 * where always a non-zero amount of pages were scanned. 6069 */ 6070 if (!nr_reclaimed) 6071 return false; 6072 6073 /* If compaction would go ahead or the allocation would succeed, stop */ 6074 for (z = 0; z <= sc->reclaim_idx; z++) { 6075 struct zone *zone = &pgdat->node_zones[z]; 6076 if (!managed_zone(zone)) 6077 continue; 6078 6079 switch (compaction_suitable(zone, sc->order, 0, sc->reclaim_idx)) { 6080 case COMPACT_SUCCESS: 6081 case COMPACT_CONTINUE: 6082 return false; 6083 default: 6084 /* check next zone */ 6085 ; 6086 } 6087 } 6088 6089 /* 6090 * If we have not reclaimed enough pages for compaction and the 6091 * inactive lists are large enough, continue reclaiming 6092 */ 6093 pages_for_compaction = compact_gap(sc->order); 6094 inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE); 6095 if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc)) 6096 inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON); 6097 6098 return inactive_lru_pages > pages_for_compaction; 6099 } 6100 6101 static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc) 6102 { 6103 struct mem_cgroup *target_memcg = sc->target_mem_cgroup; 6104 struct mem_cgroup *memcg; 6105 6106 memcg = mem_cgroup_iter(target_memcg, NULL, NULL); 6107 do { 6108 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 6109 unsigned long reclaimed; 6110 unsigned long scanned; 6111 6112 /* 6113 * This loop can become CPU-bound when target memcgs 6114 * aren't eligible for reclaim - either because they 6115 * don't have any reclaimable pages, or because their 6116 * memory is explicitly protected. Avoid soft lockups. 6117 */ 6118 cond_resched(); 6119 6120 mem_cgroup_calculate_protection(target_memcg, memcg); 6121 6122 if (mem_cgroup_below_min(target_memcg, memcg)) { 6123 /* 6124 * Hard protection. 6125 * If there is no reclaimable memory, OOM. 6126 */ 6127 continue; 6128 } else if (mem_cgroup_below_low(target_memcg, memcg)) { 6129 /* 6130 * Soft protection. 6131 * Respect the protection only as long as 6132 * there is an unprotected supply 6133 * of reclaimable memory from other cgroups. 6134 */ 6135 if (!sc->memcg_low_reclaim) { 6136 sc->memcg_low_skipped = 1; 6137 continue; 6138 } 6139 memcg_memory_event(memcg, MEMCG_LOW); 6140 } 6141 6142 reclaimed = sc->nr_reclaimed; 6143 scanned = sc->nr_scanned; 6144 6145 shrink_lruvec(lruvec, sc); 6146 6147 shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, 6148 sc->priority); 6149 6150 /* Record the group's reclaim efficiency */ 6151 if (!sc->proactive) 6152 vmpressure(sc->gfp_mask, memcg, false, 6153 sc->nr_scanned - scanned, 6154 sc->nr_reclaimed - reclaimed); 6155 6156 } while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL))); 6157 } 6158 6159 static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) 6160 { 6161 struct reclaim_state *reclaim_state = current->reclaim_state; 6162 unsigned long nr_reclaimed, nr_scanned; 6163 struct lruvec *target_lruvec; 6164 bool reclaimable = false; 6165 6166 target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); 6167 6168 again: 6169 memset(&sc->nr, 0, sizeof(sc->nr)); 6170 6171 nr_reclaimed = sc->nr_reclaimed; 6172 nr_scanned = sc->nr_scanned; 6173 6174 prepare_scan_count(pgdat, sc); 6175 6176 shrink_node_memcgs(pgdat, sc); 6177 6178 if (reclaim_state) { 6179 sc->nr_reclaimed += reclaim_state->reclaimed_slab; 6180 reclaim_state->reclaimed_slab = 0; 6181 } 6182 6183 /* Record the subtree's reclaim efficiency */ 6184 if (!sc->proactive) 6185 vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true, 6186 sc->nr_scanned - nr_scanned, 6187 sc->nr_reclaimed - nr_reclaimed); 6188 6189 if (sc->nr_reclaimed - nr_reclaimed) 6190 reclaimable = true; 6191 6192 if (current_is_kswapd()) { 6193 /* 6194 * If reclaim is isolating dirty pages under writeback, 6195 * it implies that the long-lived page allocation rate 6196 * is exceeding the page laundering rate. Either the 6197 * global limits are not being effective at throttling 6198 * processes due to the page distribution throughout 6199 * zones or there is heavy usage of a slow backing 6200 * device. The only option is to throttle from reclaim 6201 * context which is not ideal as there is no guarantee 6202 * the dirtying process is throttled in the same way 6203 * balance_dirty_pages() manages. 6204 * 6205 * Once a node is flagged PGDAT_WRITEBACK, kswapd will 6206 * count the number of pages under pages flagged for 6207 * immediate reclaim and stall if any are encountered 6208 * in the nr_immediate check below. 6209 */ 6210 if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken) 6211 set_bit(PGDAT_WRITEBACK, &pgdat->flags); 6212 6213 /* Allow kswapd to start writing pages during reclaim.*/ 6214 if (sc->nr.unqueued_dirty == sc->nr.file_taken) 6215 set_bit(PGDAT_DIRTY, &pgdat->flags); 6216 6217 /* 6218 * If kswapd scans pages marked for immediate 6219 * reclaim and under writeback (nr_immediate), it 6220 * implies that pages are cycling through the LRU 6221 * faster than they are written so forcibly stall 6222 * until some pages complete writeback. 6223 */ 6224 if (sc->nr.immediate) 6225 reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK); 6226 } 6227 6228 /* 6229 * Tag a node/memcg as congested if all the dirty pages were marked 6230 * for writeback and immediate reclaim (counted in nr.congested). 6231 * 6232 * Legacy memcg will stall in page writeback so avoid forcibly 6233 * stalling in reclaim_throttle(). 6234 */ 6235 if ((current_is_kswapd() || 6236 (cgroup_reclaim(sc) && writeback_throttling_sane(sc))) && 6237 sc->nr.dirty && sc->nr.dirty == sc->nr.congested) 6238 set_bit(LRUVEC_CONGESTED, &target_lruvec->flags); 6239 6240 /* 6241 * Stall direct reclaim for IO completions if the lruvec is 6242 * node is congested. Allow kswapd to continue until it 6243 * starts encountering unqueued dirty pages or cycling through 6244 * the LRU too quickly. 6245 */ 6246 if (!current_is_kswapd() && current_may_throttle() && 6247 !sc->hibernation_mode && 6248 test_bit(LRUVEC_CONGESTED, &target_lruvec->flags)) 6249 reclaim_throttle(pgdat, VMSCAN_THROTTLE_CONGESTED); 6250 6251 if (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed, 6252 sc)) 6253 goto again; 6254 6255 /* 6256 * Kswapd gives up on balancing particular nodes after too 6257 * many failures to reclaim anything from them and goes to 6258 * sleep. On reclaim progress, reset the failure counter. A 6259 * successful direct reclaim run will revive a dormant kswapd. 6260 */ 6261 if (reclaimable) 6262 pgdat->kswapd_failures = 0; 6263 } 6264 6265 /* 6266 * Returns true if compaction should go ahead for a costly-order request, or 6267 * the allocation would already succeed without compaction. Return false if we 6268 * should reclaim first. 6269 */ 6270 static inline bool compaction_ready(struct zone *zone, struct scan_control *sc) 6271 { 6272 unsigned long watermark; 6273 enum compact_result suitable; 6274 6275 suitable = compaction_suitable(zone, sc->order, 0, sc->reclaim_idx); 6276 if (suitable == COMPACT_SUCCESS) 6277 /* Allocation should succeed already. Don't reclaim. */ 6278 return true; 6279 if (suitable == COMPACT_SKIPPED) 6280 /* Compaction cannot yet proceed. Do reclaim. */ 6281 return false; 6282 6283 /* 6284 * Compaction is already possible, but it takes time to run and there 6285 * are potentially other callers using the pages just freed. So proceed 6286 * with reclaim to make a buffer of free pages available to give 6287 * compaction a reasonable chance of completing and allocating the page. 6288 * Note that we won't actually reclaim the whole buffer in one attempt 6289 * as the target watermark in should_continue_reclaim() is lower. But if 6290 * we are already above the high+gap watermark, don't reclaim at all. 6291 */ 6292 watermark = high_wmark_pages(zone) + compact_gap(sc->order); 6293 6294 return zone_watermark_ok_safe(zone, 0, watermark, sc->reclaim_idx); 6295 } 6296 6297 static void consider_reclaim_throttle(pg_data_t *pgdat, struct scan_control *sc) 6298 { 6299 /* 6300 * If reclaim is making progress greater than 12% efficiency then 6301 * wake all the NOPROGRESS throttled tasks. 6302 */ 6303 if (sc->nr_reclaimed > (sc->nr_scanned >> 3)) { 6304 wait_queue_head_t *wqh; 6305 6306 wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_NOPROGRESS]; 6307 if (waitqueue_active(wqh)) 6308 wake_up(wqh); 6309 6310 return; 6311 } 6312 6313 /* 6314 * Do not throttle kswapd or cgroup reclaim on NOPROGRESS as it will 6315 * throttle on VMSCAN_THROTTLE_WRITEBACK if there are too many pages 6316 * under writeback and marked for immediate reclaim at the tail of the 6317 * LRU. 6318 */ 6319 if (current_is_kswapd() || cgroup_reclaim(sc)) 6320 return; 6321 6322 /* Throttle if making no progress at high prioities. */ 6323 if (sc->priority == 1 && !sc->nr_reclaimed) 6324 reclaim_throttle(pgdat, VMSCAN_THROTTLE_NOPROGRESS); 6325 } 6326 6327 /* 6328 * This is the direct reclaim path, for page-allocating processes. We only 6329 * try to reclaim pages from zones which will satisfy the caller's allocation 6330 * request. 6331 * 6332 * If a zone is deemed to be full of pinned pages then just give it a light 6333 * scan then give up on it. 6334 */ 6335 static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc) 6336 { 6337 struct zoneref *z; 6338 struct zone *zone; 6339 unsigned long nr_soft_reclaimed; 6340 unsigned long nr_soft_scanned; 6341 gfp_t orig_mask; 6342 pg_data_t *last_pgdat = NULL; 6343 pg_data_t *first_pgdat = NULL; 6344 6345 /* 6346 * If the number of buffer_heads in the machine exceeds the maximum 6347 * allowed level, force direct reclaim to scan the highmem zone as 6348 * highmem pages could be pinning lowmem pages storing buffer_heads 6349 */ 6350 orig_mask = sc->gfp_mask; 6351 if (buffer_heads_over_limit) { 6352 sc->gfp_mask |= __GFP_HIGHMEM; 6353 sc->reclaim_idx = gfp_zone(sc->gfp_mask); 6354 } 6355 6356 for_each_zone_zonelist_nodemask(zone, z, zonelist, 6357 sc->reclaim_idx, sc->nodemask) { 6358 /* 6359 * Take care memory controller reclaiming has small influence 6360 * to global LRU. 6361 */ 6362 if (!cgroup_reclaim(sc)) { 6363 if (!cpuset_zone_allowed(zone, 6364 GFP_KERNEL | __GFP_HARDWALL)) 6365 continue; 6366 6367 /* 6368 * If we already have plenty of memory free for 6369 * compaction in this zone, don't free any more. 6370 * Even though compaction is invoked for any 6371 * non-zero order, only frequent costly order 6372 * reclamation is disruptive enough to become a 6373 * noticeable problem, like transparent huge 6374 * page allocations. 6375 */ 6376 if (IS_ENABLED(CONFIG_COMPACTION) && 6377 sc->order > PAGE_ALLOC_COSTLY_ORDER && 6378 compaction_ready(zone, sc)) { 6379 sc->compaction_ready = true; 6380 continue; 6381 } 6382 6383 /* 6384 * Shrink each node in the zonelist once. If the 6385 * zonelist is ordered by zone (not the default) then a 6386 * node may be shrunk multiple times but in that case 6387 * the user prefers lower zones being preserved. 6388 */ 6389 if (zone->zone_pgdat == last_pgdat) 6390 continue; 6391 6392 /* 6393 * This steals pages from memory cgroups over softlimit 6394 * and returns the number of reclaimed pages and 6395 * scanned pages. This works for global memory pressure 6396 * and balancing, not for a memcg's limit. 6397 */ 6398 nr_soft_scanned = 0; 6399 nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone->zone_pgdat, 6400 sc->order, sc->gfp_mask, 6401 &nr_soft_scanned); 6402 sc->nr_reclaimed += nr_soft_reclaimed; 6403 sc->nr_scanned += nr_soft_scanned; 6404 /* need some check for avoid more shrink_zone() */ 6405 } 6406 6407 if (!first_pgdat) 6408 first_pgdat = zone->zone_pgdat; 6409 6410 /* See comment about same check for global reclaim above */ 6411 if (zone->zone_pgdat == last_pgdat) 6412 continue; 6413 last_pgdat = zone->zone_pgdat; 6414 shrink_node(zone->zone_pgdat, sc); 6415 } 6416 6417 if (first_pgdat) 6418 consider_reclaim_throttle(first_pgdat, sc); 6419 6420 /* 6421 * Restore to original mask to avoid the impact on the caller if we 6422 * promoted it to __GFP_HIGHMEM. 6423 */ 6424 sc->gfp_mask = orig_mask; 6425 } 6426 6427 static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) 6428 { 6429 struct lruvec *target_lruvec; 6430 unsigned long refaults; 6431 6432 if (lru_gen_enabled()) 6433 return; 6434 6435 target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat); 6436 refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON); 6437 target_lruvec->refaults[WORKINGSET_ANON] = refaults; 6438 refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE); 6439 target_lruvec->refaults[WORKINGSET_FILE] = refaults; 6440 } 6441 6442 /* 6443 * This is the main entry point to direct page reclaim. 6444 * 6445 * If a full scan of the inactive list fails to free enough memory then we 6446 * are "out of memory" and something needs to be killed. 6447 * 6448 * If the caller is !__GFP_FS then the probability of a failure is reasonably 6449 * high - the zone may be full of dirty or under-writeback pages, which this 6450 * caller can't do much about. We kick the writeback threads and take explicit 6451 * naps in the hope that some of these pages can be written. But if the 6452 * allocating task holds filesystem locks which prevent writeout this might not 6453 * work, and the allocation attempt will fail. 6454 * 6455 * returns: 0, if no pages reclaimed 6456 * else, the number of pages reclaimed 6457 */ 6458 static unsigned long do_try_to_free_pages(struct zonelist *zonelist, 6459 struct scan_control *sc) 6460 { 6461 int initial_priority = sc->priority; 6462 pg_data_t *last_pgdat; 6463 struct zoneref *z; 6464 struct zone *zone; 6465 retry: 6466 delayacct_freepages_start(); 6467 6468 if (!cgroup_reclaim(sc)) 6469 __count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1); 6470 6471 do { 6472 if (!sc->proactive) 6473 vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup, 6474 sc->priority); 6475 sc->nr_scanned = 0; 6476 shrink_zones(zonelist, sc); 6477 6478 if (sc->nr_reclaimed >= sc->nr_to_reclaim) 6479 break; 6480 6481 if (sc->compaction_ready) 6482 break; 6483 6484 /* 6485 * If we're getting trouble reclaiming, start doing 6486 * writepage even in laptop mode. 6487 */ 6488 if (sc->priority < DEF_PRIORITY - 2) 6489 sc->may_writepage = 1; 6490 } while (--sc->priority >= 0); 6491 6492 last_pgdat = NULL; 6493 for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx, 6494 sc->nodemask) { 6495 if (zone->zone_pgdat == last_pgdat) 6496 continue; 6497 last_pgdat = zone->zone_pgdat; 6498 6499 snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat); 6500 6501 if (cgroup_reclaim(sc)) { 6502 struct lruvec *lruvec; 6503 6504 lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, 6505 zone->zone_pgdat); 6506 clear_bit(LRUVEC_CONGESTED, &lruvec->flags); 6507 } 6508 } 6509 6510 delayacct_freepages_end(); 6511 6512 if (sc->nr_reclaimed) 6513 return sc->nr_reclaimed; 6514 6515 /* Aborted reclaim to try compaction? don't OOM, then */ 6516 if (sc->compaction_ready) 6517 return 1; 6518 6519 /* 6520 * We make inactive:active ratio decisions based on the node's 6521 * composition of memory, but a restrictive reclaim_idx or a 6522 * memory.low cgroup setting can exempt large amounts of 6523 * memory from reclaim. Neither of which are very common, so 6524 * instead of doing costly eligibility calculations of the 6525 * entire cgroup subtree up front, we assume the estimates are 6526 * good, and retry with forcible deactivation if that fails. 6527 */ 6528 if (sc->skipped_deactivate) { 6529 sc->priority = initial_priority; 6530 sc->force_deactivate = 1; 6531 sc->skipped_deactivate = 0; 6532 goto retry; 6533 } 6534 6535 /* Untapped cgroup reserves? Don't OOM, retry. */ 6536 if (sc->memcg_low_skipped) { 6537 sc->priority = initial_priority; 6538 sc->force_deactivate = 0; 6539 sc->memcg_low_reclaim = 1; 6540 sc->memcg_low_skipped = 0; 6541 goto retry; 6542 } 6543 6544 return 0; 6545 } 6546 6547 static bool allow_direct_reclaim(pg_data_t *pgdat) 6548 { 6549 struct zone *zone; 6550 unsigned long pfmemalloc_reserve = 0; 6551 unsigned long free_pages = 0; 6552 int i; 6553 bool wmark_ok; 6554 6555 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) 6556 return true; 6557 6558 for (i = 0; i <= ZONE_NORMAL; i++) { 6559 zone = &pgdat->node_zones[i]; 6560 if (!managed_zone(zone)) 6561 continue; 6562 6563 if (!zone_reclaimable_pages(zone)) 6564 continue; 6565 6566 pfmemalloc_reserve += min_wmark_pages(zone); 6567 free_pages += zone_page_state(zone, NR_FREE_PAGES); 6568 } 6569 6570 /* If there are no reserves (unexpected config) then do not throttle */ 6571 if (!pfmemalloc_reserve) 6572 return true; 6573 6574 wmark_ok = free_pages > pfmemalloc_reserve / 2; 6575 6576 /* kswapd must be awake if processes are being throttled */ 6577 if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) { 6578 if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL) 6579 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL); 6580 6581 wake_up_interruptible(&pgdat->kswapd_wait); 6582 } 6583 6584 return wmark_ok; 6585 } 6586 6587 /* 6588 * Throttle direct reclaimers if backing storage is backed by the network 6589 * and the PFMEMALLOC reserve for the preferred node is getting dangerously 6590 * depleted. kswapd will continue to make progress and wake the processes 6591 * when the low watermark is reached. 6592 * 6593 * Returns true if a fatal signal was delivered during throttling. If this 6594 * happens, the page allocator should not consider triggering the OOM killer. 6595 */ 6596 static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, 6597 nodemask_t *nodemask) 6598 { 6599 struct zoneref *z; 6600 struct zone *zone; 6601 pg_data_t *pgdat = NULL; 6602 6603 /* 6604 * Kernel threads should not be throttled as they may be indirectly 6605 * responsible for cleaning pages necessary for reclaim to make forward 6606 * progress. kjournald for example may enter direct reclaim while 6607 * committing a transaction where throttling it could forcing other 6608 * processes to block on log_wait_commit(). 6609 */ 6610 if (current->flags & PF_KTHREAD) 6611 goto out; 6612 6613 /* 6614 * If a fatal signal is pending, this process should not throttle. 6615 * It should return quickly so it can exit and free its memory 6616 */ 6617 if (fatal_signal_pending(current)) 6618 goto out; 6619 6620 /* 6621 * Check if the pfmemalloc reserves are ok by finding the first node 6622 * with a usable ZONE_NORMAL or lower zone. The expectation is that 6623 * GFP_KERNEL will be required for allocating network buffers when 6624 * swapping over the network so ZONE_HIGHMEM is unusable. 6625 * 6626 * Throttling is based on the first usable node and throttled processes 6627 * wait on a queue until kswapd makes progress and wakes them. There 6628 * is an affinity then between processes waking up and where reclaim 6629 * progress has been made assuming the process wakes on the same node. 6630 * More importantly, processes running on remote nodes will not compete 6631 * for remote pfmemalloc reserves and processes on different nodes 6632 * should make reasonable progress. 6633 */ 6634 for_each_zone_zonelist_nodemask(zone, z, zonelist, 6635 gfp_zone(gfp_mask), nodemask) { 6636 if (zone_idx(zone) > ZONE_NORMAL) 6637 continue; 6638 6639 /* Throttle based on the first usable node */ 6640 pgdat = zone->zone_pgdat; 6641 if (allow_direct_reclaim(pgdat)) 6642 goto out; 6643 break; 6644 } 6645 6646 /* If no zone was usable by the allocation flags then do not throttle */ 6647 if (!pgdat) 6648 goto out; 6649 6650 /* Account for the throttling */ 6651 count_vm_event(PGSCAN_DIRECT_THROTTLE); 6652 6653 /* 6654 * If the caller cannot enter the filesystem, it's possible that it 6655 * is due to the caller holding an FS lock or performing a journal 6656 * transaction in the case of a filesystem like ext[3|4]. In this case, 6657 * it is not safe to block on pfmemalloc_wait as kswapd could be 6658 * blocked waiting on the same lock. Instead, throttle for up to a 6659 * second before continuing. 6660 */ 6661 if (!(gfp_mask & __GFP_FS)) 6662 wait_event_interruptible_timeout(pgdat->pfmemalloc_wait, 6663 allow_direct_reclaim(pgdat), HZ); 6664 else 6665 /* Throttle until kswapd wakes the process */ 6666 wait_event_killable(zone->zone_pgdat->pfmemalloc_wait, 6667 allow_direct_reclaim(pgdat)); 6668 6669 if (fatal_signal_pending(current)) 6670 return true; 6671 6672 out: 6673 return false; 6674 } 6675 6676 unsigned long try_to_free_pages(struct zonelist *zonelist, int order, 6677 gfp_t gfp_mask, nodemask_t *nodemask) 6678 { 6679 unsigned long nr_reclaimed; 6680 struct scan_control sc = { 6681 .nr_to_reclaim = SWAP_CLUSTER_MAX, 6682 .gfp_mask = current_gfp_context(gfp_mask), 6683 .reclaim_idx = gfp_zone(gfp_mask), 6684 .order = order, 6685 .nodemask = nodemask, 6686 .priority = DEF_PRIORITY, 6687 .may_writepage = !laptop_mode, 6688 .may_unmap = 1, 6689 .may_swap = 1, 6690 }; 6691 6692 /* 6693 * scan_control uses s8 fields for order, priority, and reclaim_idx. 6694 * Confirm they are large enough for max values. 6695 */ 6696 BUILD_BUG_ON(MAX_ORDER > S8_MAX); 6697 BUILD_BUG_ON(DEF_PRIORITY > S8_MAX); 6698 BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX); 6699 6700 /* 6701 * Do not enter reclaim if fatal signal was delivered while throttled. 6702 * 1 is returned so that the page allocator does not OOM kill at this 6703 * point. 6704 */ 6705 if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask)) 6706 return 1; 6707 6708 set_task_reclaim_state(current, &sc.reclaim_state); 6709 trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask); 6710 6711 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 6712 6713 trace_mm_vmscan_direct_reclaim_end(nr_reclaimed); 6714 set_task_reclaim_state(current, NULL); 6715 6716 return nr_reclaimed; 6717 } 6718 6719 #ifdef CONFIG_MEMCG 6720 6721 /* Only used by soft limit reclaim. Do not reuse for anything else. */ 6722 unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, 6723 gfp_t gfp_mask, bool noswap, 6724 pg_data_t *pgdat, 6725 unsigned long *nr_scanned) 6726 { 6727 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 6728 struct scan_control sc = { 6729 .nr_to_reclaim = SWAP_CLUSTER_MAX, 6730 .target_mem_cgroup = memcg, 6731 .may_writepage = !laptop_mode, 6732 .may_unmap = 1, 6733 .reclaim_idx = MAX_NR_ZONES - 1, 6734 .may_swap = !noswap, 6735 }; 6736 6737 WARN_ON_ONCE(!current->reclaim_state); 6738 6739 sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | 6740 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK); 6741 6742 trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order, 6743 sc.gfp_mask); 6744 6745 /* 6746 * NOTE: Although we can get the priority field, using it 6747 * here is not a good idea, since it limits the pages we can scan. 6748 * if we don't reclaim here, the shrink_node from balance_pgdat 6749 * will pick up pages from other mem cgroup's as well. We hack 6750 * the priority and make it zero. 6751 */ 6752 shrink_lruvec(lruvec, &sc); 6753 6754 trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed); 6755 6756 *nr_scanned = sc.nr_scanned; 6757 6758 return sc.nr_reclaimed; 6759 } 6760 6761 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, 6762 unsigned long nr_pages, 6763 gfp_t gfp_mask, 6764 unsigned int reclaim_options) 6765 { 6766 unsigned long nr_reclaimed; 6767 unsigned int noreclaim_flag; 6768 struct scan_control sc = { 6769 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), 6770 .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) | 6771 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK), 6772 .reclaim_idx = MAX_NR_ZONES - 1, 6773 .target_mem_cgroup = memcg, 6774 .priority = DEF_PRIORITY, 6775 .may_writepage = !laptop_mode, 6776 .may_unmap = 1, 6777 .may_swap = !!(reclaim_options & MEMCG_RECLAIM_MAY_SWAP), 6778 .proactive = !!(reclaim_options & MEMCG_RECLAIM_PROACTIVE), 6779 }; 6780 /* 6781 * Traverse the ZONELIST_FALLBACK zonelist of the current node to put 6782 * equal pressure on all the nodes. This is based on the assumption that 6783 * the reclaim does not bail out early. 6784 */ 6785 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); 6786 6787 set_task_reclaim_state(current, &sc.reclaim_state); 6788 trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask); 6789 noreclaim_flag = memalloc_noreclaim_save(); 6790 6791 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 6792 6793 memalloc_noreclaim_restore(noreclaim_flag); 6794 trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed); 6795 set_task_reclaim_state(current, NULL); 6796 6797 return nr_reclaimed; 6798 } 6799 #endif 6800 6801 static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) 6802 { 6803 struct mem_cgroup *memcg; 6804 struct lruvec *lruvec; 6805 6806 if (lru_gen_enabled()) { 6807 lru_gen_age_node(pgdat, sc); 6808 return; 6809 } 6810 6811 if (!can_age_anon_pages(pgdat, sc)) 6812 return; 6813 6814 lruvec = mem_cgroup_lruvec(NULL, pgdat); 6815 if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON)) 6816 return; 6817 6818 memcg = mem_cgroup_iter(NULL, NULL, NULL); 6819 do { 6820 lruvec = mem_cgroup_lruvec(memcg, pgdat); 6821 shrink_active_list(SWAP_CLUSTER_MAX, lruvec, 6822 sc, LRU_ACTIVE_ANON); 6823 memcg = mem_cgroup_iter(NULL, memcg, NULL); 6824 } while (memcg); 6825 } 6826 6827 static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx) 6828 { 6829 int i; 6830 struct zone *zone; 6831 6832 /* 6833 * Check for watermark boosts top-down as the higher zones 6834 * are more likely to be boosted. Both watermarks and boosts 6835 * should not be checked at the same time as reclaim would 6836 * start prematurely when there is no boosting and a lower 6837 * zone is balanced. 6838 */ 6839 for (i = highest_zoneidx; i >= 0; i--) { 6840 zone = pgdat->node_zones + i; 6841 if (!managed_zone(zone)) 6842 continue; 6843 6844 if (zone->watermark_boost) 6845 return true; 6846 } 6847 6848 return false; 6849 } 6850 6851 /* 6852 * Returns true if there is an eligible zone balanced for the request order 6853 * and highest_zoneidx 6854 */ 6855 static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx) 6856 { 6857 int i; 6858 unsigned long mark = -1; 6859 struct zone *zone; 6860 6861 /* 6862 * Check watermarks bottom-up as lower zones are more likely to 6863 * meet watermarks. 6864 */ 6865 for (i = 0; i <= highest_zoneidx; i++) { 6866 zone = pgdat->node_zones + i; 6867 6868 if (!managed_zone(zone)) 6869 continue; 6870 6871 if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) 6872 mark = wmark_pages(zone, WMARK_PROMO); 6873 else 6874 mark = high_wmark_pages(zone); 6875 if (zone_watermark_ok_safe(zone, order, mark, highest_zoneidx)) 6876 return true; 6877 } 6878 6879 /* 6880 * If a node has no managed zone within highest_zoneidx, it does not 6881 * need balancing by definition. This can happen if a zone-restricted 6882 * allocation tries to wake a remote kswapd. 6883 */ 6884 if (mark == -1) 6885 return true; 6886 6887 return false; 6888 } 6889 6890 /* Clear pgdat state for congested, dirty or under writeback. */ 6891 static void clear_pgdat_congested(pg_data_t *pgdat) 6892 { 6893 struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat); 6894 6895 clear_bit(LRUVEC_CONGESTED, &lruvec->flags); 6896 clear_bit(PGDAT_DIRTY, &pgdat->flags); 6897 clear_bit(PGDAT_WRITEBACK, &pgdat->flags); 6898 } 6899 6900 /* 6901 * Prepare kswapd for sleeping. This verifies that there are no processes 6902 * waiting in throttle_direct_reclaim() and that watermarks have been met. 6903 * 6904 * Returns true if kswapd is ready to sleep 6905 */ 6906 static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, 6907 int highest_zoneidx) 6908 { 6909 /* 6910 * The throttled processes are normally woken up in balance_pgdat() as 6911 * soon as allow_direct_reclaim() is true. But there is a potential 6912 * race between when kswapd checks the watermarks and a process gets 6913 * throttled. There is also a potential race if processes get 6914 * throttled, kswapd wakes, a large process exits thereby balancing the 6915 * zones, which causes kswapd to exit balance_pgdat() before reaching 6916 * the wake up checks. If kswapd is going to sleep, no process should 6917 * be sleeping on pfmemalloc_wait, so wake them now if necessary. If 6918 * the wake up is premature, processes will wake kswapd and get 6919 * throttled again. The difference from wake ups in balance_pgdat() is 6920 * that here we are under prepare_to_wait(). 6921 */ 6922 if (waitqueue_active(&pgdat->pfmemalloc_wait)) 6923 wake_up_all(&pgdat->pfmemalloc_wait); 6924 6925 /* Hopeless node, leave it to direct reclaim */ 6926 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) 6927 return true; 6928 6929 if (pgdat_balanced(pgdat, order, highest_zoneidx)) { 6930 clear_pgdat_congested(pgdat); 6931 return true; 6932 } 6933 6934 return false; 6935 } 6936 6937 /* 6938 * kswapd shrinks a node of pages that are at or below the highest usable 6939 * zone that is currently unbalanced. 6940 * 6941 * Returns true if kswapd scanned at least the requested number of pages to 6942 * reclaim or if the lack of progress was due to pages under writeback. 6943 * This is used to determine if the scanning priority needs to be raised. 6944 */ 6945 static bool kswapd_shrink_node(pg_data_t *pgdat, 6946 struct scan_control *sc) 6947 { 6948 struct zone *zone; 6949 int z; 6950 6951 /* Reclaim a number of pages proportional to the number of zones */ 6952 sc->nr_to_reclaim = 0; 6953 for (z = 0; z <= sc->reclaim_idx; z++) { 6954 zone = pgdat->node_zones + z; 6955 if (!managed_zone(zone)) 6956 continue; 6957 6958 sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX); 6959 } 6960 6961 /* 6962 * Historically care was taken to put equal pressure on all zones but 6963 * now pressure is applied based on node LRU order. 6964 */ 6965 shrink_node(pgdat, sc); 6966 6967 /* 6968 * Fragmentation may mean that the system cannot be rebalanced for 6969 * high-order allocations. If twice the allocation size has been 6970 * reclaimed then recheck watermarks only at order-0 to prevent 6971 * excessive reclaim. Assume that a process requested a high-order 6972 * can direct reclaim/compact. 6973 */ 6974 if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order)) 6975 sc->order = 0; 6976 6977 return sc->nr_scanned >= sc->nr_to_reclaim; 6978 } 6979 6980 /* Page allocator PCP high watermark is lowered if reclaim is active. */ 6981 static inline void 6982 update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active) 6983 { 6984 int i; 6985 struct zone *zone; 6986 6987 for (i = 0; i <= highest_zoneidx; i++) { 6988 zone = pgdat->node_zones + i; 6989 6990 if (!managed_zone(zone)) 6991 continue; 6992 6993 if (active) 6994 set_bit(ZONE_RECLAIM_ACTIVE, &zone->flags); 6995 else 6996 clear_bit(ZONE_RECLAIM_ACTIVE, &zone->flags); 6997 } 6998 } 6999 7000 static inline void 7001 set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) 7002 { 7003 update_reclaim_active(pgdat, highest_zoneidx, true); 7004 } 7005 7006 static inline void 7007 clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) 7008 { 7009 update_reclaim_active(pgdat, highest_zoneidx, false); 7010 } 7011 7012 /* 7013 * For kswapd, balance_pgdat() will reclaim pages across a node from zones 7014 * that are eligible for use by the caller until at least one zone is 7015 * balanced. 7016 * 7017 * Returns the order kswapd finished reclaiming at. 7018 * 7019 * kswapd scans the zones in the highmem->normal->dma direction. It skips 7020 * zones which have free_pages > high_wmark_pages(zone), but once a zone is 7021 * found to have free_pages <= high_wmark_pages(zone), any page in that zone 7022 * or lower is eligible for reclaim until at least one usable zone is 7023 * balanced. 7024 */ 7025 static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) 7026 { 7027 int i; 7028 unsigned long nr_soft_reclaimed; 7029 unsigned long nr_soft_scanned; 7030 unsigned long pflags; 7031 unsigned long nr_boost_reclaim; 7032 unsigned long zone_boosts[MAX_NR_ZONES] = { 0, }; 7033 bool boosted; 7034 struct zone *zone; 7035 struct scan_control sc = { 7036 .gfp_mask = GFP_KERNEL, 7037 .order = order, 7038 .may_unmap = 1, 7039 }; 7040 7041 set_task_reclaim_state(current, &sc.reclaim_state); 7042 psi_memstall_enter(&pflags); 7043 __fs_reclaim_acquire(_THIS_IP_); 7044 7045 count_vm_event(PAGEOUTRUN); 7046 7047 /* 7048 * Account for the reclaim boost. Note that the zone boost is left in 7049 * place so that parallel allocations that are near the watermark will 7050 * stall or direct reclaim until kswapd is finished. 7051 */ 7052 nr_boost_reclaim = 0; 7053 for (i = 0; i <= highest_zoneidx; i++) { 7054 zone = pgdat->node_zones + i; 7055 if (!managed_zone(zone)) 7056 continue; 7057 7058 nr_boost_reclaim += zone->watermark_boost; 7059 zone_boosts[i] = zone->watermark_boost; 7060 } 7061 boosted = nr_boost_reclaim; 7062 7063 restart: 7064 set_reclaim_active(pgdat, highest_zoneidx); 7065 sc.priority = DEF_PRIORITY; 7066 do { 7067 unsigned long nr_reclaimed = sc.nr_reclaimed; 7068 bool raise_priority = true; 7069 bool balanced; 7070 bool ret; 7071 7072 sc.reclaim_idx = highest_zoneidx; 7073 7074 /* 7075 * If the number of buffer_heads exceeds the maximum allowed 7076 * then consider reclaiming from all zones. This has a dual 7077 * purpose -- on 64-bit systems it is expected that 7078 * buffer_heads are stripped during active rotation. On 32-bit 7079 * systems, highmem pages can pin lowmem memory and shrinking 7080 * buffers can relieve lowmem pressure. Reclaim may still not 7081 * go ahead if all eligible zones for the original allocation 7082 * request are balanced to avoid excessive reclaim from kswapd. 7083 */ 7084 if (buffer_heads_over_limit) { 7085 for (i = MAX_NR_ZONES - 1; i >= 0; i--) { 7086 zone = pgdat->node_zones + i; 7087 if (!managed_zone(zone)) 7088 continue; 7089 7090 sc.reclaim_idx = i; 7091 break; 7092 } 7093 } 7094 7095 /* 7096 * If the pgdat is imbalanced then ignore boosting and preserve 7097 * the watermarks for a later time and restart. Note that the 7098 * zone watermarks will be still reset at the end of balancing 7099 * on the grounds that the normal reclaim should be enough to 7100 * re-evaluate if boosting is required when kswapd next wakes. 7101 */ 7102 balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx); 7103 if (!balanced && nr_boost_reclaim) { 7104 nr_boost_reclaim = 0; 7105 goto restart; 7106 } 7107 7108 /* 7109 * If boosting is not active then only reclaim if there are no 7110 * eligible zones. Note that sc.reclaim_idx is not used as 7111 * buffer_heads_over_limit may have adjusted it. 7112 */ 7113 if (!nr_boost_reclaim && balanced) 7114 goto out; 7115 7116 /* Limit the priority of boosting to avoid reclaim writeback */ 7117 if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2) 7118 raise_priority = false; 7119 7120 /* 7121 * Do not writeback or swap pages for boosted reclaim. The 7122 * intent is to relieve pressure not issue sub-optimal IO 7123 * from reclaim context. If no pages are reclaimed, the 7124 * reclaim will be aborted. 7125 */ 7126 sc.may_writepage = !laptop_mode && !nr_boost_reclaim; 7127 sc.may_swap = !nr_boost_reclaim; 7128 7129 /* 7130 * Do some background aging, to give pages a chance to be 7131 * referenced before reclaiming. All pages are rotated 7132 * regardless of classzone as this is about consistent aging. 7133 */ 7134 kswapd_age_node(pgdat, &sc); 7135 7136 /* 7137 * If we're getting trouble reclaiming, start doing writepage 7138 * even in laptop mode. 7139 */ 7140 if (sc.priority < DEF_PRIORITY - 2) 7141 sc.may_writepage = 1; 7142 7143 /* Call soft limit reclaim before calling shrink_node. */ 7144 sc.nr_scanned = 0; 7145 nr_soft_scanned = 0; 7146 nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order, 7147 sc.gfp_mask, &nr_soft_scanned); 7148 sc.nr_reclaimed += nr_soft_reclaimed; 7149 7150 /* 7151 * There should be no need to raise the scanning priority if 7152 * enough pages are already being scanned that that high 7153 * watermark would be met at 100% efficiency. 7154 */ 7155 if (kswapd_shrink_node(pgdat, &sc)) 7156 raise_priority = false; 7157 7158 /* 7159 * If the low watermark is met there is no need for processes 7160 * to be throttled on pfmemalloc_wait as they should not be 7161 * able to safely make forward progress. Wake them 7162 */ 7163 if (waitqueue_active(&pgdat->pfmemalloc_wait) && 7164 allow_direct_reclaim(pgdat)) 7165 wake_up_all(&pgdat->pfmemalloc_wait); 7166 7167 /* Check if kswapd should be suspending */ 7168 __fs_reclaim_release(_THIS_IP_); 7169 ret = try_to_freeze(); 7170 __fs_reclaim_acquire(_THIS_IP_); 7171 if (ret || kthread_should_stop()) 7172 break; 7173 7174 /* 7175 * Raise priority if scanning rate is too low or there was no 7176 * progress in reclaiming pages 7177 */ 7178 nr_reclaimed = sc.nr_reclaimed - nr_reclaimed; 7179 nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed); 7180 7181 /* 7182 * If reclaim made no progress for a boost, stop reclaim as 7183 * IO cannot be queued and it could be an infinite loop in 7184 * extreme circumstances. 7185 */ 7186 if (nr_boost_reclaim && !nr_reclaimed) 7187 break; 7188 7189 if (raise_priority || !nr_reclaimed) 7190 sc.priority--; 7191 } while (sc.priority >= 1); 7192 7193 if (!sc.nr_reclaimed) 7194 pgdat->kswapd_failures++; 7195 7196 out: 7197 clear_reclaim_active(pgdat, highest_zoneidx); 7198 7199 /* If reclaim was boosted, account for the reclaim done in this pass */ 7200 if (boosted) { 7201 unsigned long flags; 7202 7203 for (i = 0; i <= highest_zoneidx; i++) { 7204 if (!zone_boosts[i]) 7205 continue; 7206 7207 /* Increments are under the zone lock */ 7208 zone = pgdat->node_zones + i; 7209 spin_lock_irqsave(&zone->lock, flags); 7210 zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]); 7211 spin_unlock_irqrestore(&zone->lock, flags); 7212 } 7213 7214 /* 7215 * As there is now likely space, wakeup kcompact to defragment 7216 * pageblocks. 7217 */ 7218 wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx); 7219 } 7220 7221 snapshot_refaults(NULL, pgdat); 7222 __fs_reclaim_release(_THIS_IP_); 7223 psi_memstall_leave(&pflags); 7224 set_task_reclaim_state(current, NULL); 7225 7226 /* 7227 * Return the order kswapd stopped reclaiming at as 7228 * prepare_kswapd_sleep() takes it into account. If another caller 7229 * entered the allocator slow path while kswapd was awake, order will 7230 * remain at the higher level. 7231 */ 7232 return sc.order; 7233 } 7234 7235 /* 7236 * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to 7237 * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is 7238 * not a valid index then either kswapd runs for first time or kswapd couldn't 7239 * sleep after previous reclaim attempt (node is still unbalanced). In that 7240 * case return the zone index of the previous kswapd reclaim cycle. 7241 */ 7242 static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat, 7243 enum zone_type prev_highest_zoneidx) 7244 { 7245 enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx); 7246 7247 return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx; 7248 } 7249 7250 static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order, 7251 unsigned int highest_zoneidx) 7252 { 7253 long remaining = 0; 7254 DEFINE_WAIT(wait); 7255 7256 if (freezing(current) || kthread_should_stop()) 7257 return; 7258 7259 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); 7260 7261 /* 7262 * Try to sleep for a short interval. Note that kcompactd will only be 7263 * woken if it is possible to sleep for a short interval. This is 7264 * deliberate on the assumption that if reclaim cannot keep an 7265 * eligible zone balanced that it's also unlikely that compaction will 7266 * succeed. 7267 */ 7268 if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) { 7269 /* 7270 * Compaction records what page blocks it recently failed to 7271 * isolate pages from and skips them in the future scanning. 7272 * When kswapd is going to sleep, it is reasonable to assume 7273 * that pages and compaction may succeed so reset the cache. 7274 */ 7275 reset_isolation_suitable(pgdat); 7276 7277 /* 7278 * We have freed the memory, now we should compact it to make 7279 * allocation of the requested order possible. 7280 */ 7281 wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx); 7282 7283 remaining = schedule_timeout(HZ/10); 7284 7285 /* 7286 * If woken prematurely then reset kswapd_highest_zoneidx and 7287 * order. The values will either be from a wakeup request or 7288 * the previous request that slept prematurely. 7289 */ 7290 if (remaining) { 7291 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, 7292 kswapd_highest_zoneidx(pgdat, 7293 highest_zoneidx)); 7294 7295 if (READ_ONCE(pgdat->kswapd_order) < reclaim_order) 7296 WRITE_ONCE(pgdat->kswapd_order, reclaim_order); 7297 } 7298 7299 finish_wait(&pgdat->kswapd_wait, &wait); 7300 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); 7301 } 7302 7303 /* 7304 * After a short sleep, check if it was a premature sleep. If not, then 7305 * go fully to sleep until explicitly woken up. 7306 */ 7307 if (!remaining && 7308 prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) { 7309 trace_mm_vmscan_kswapd_sleep(pgdat->node_id); 7310 7311 /* 7312 * vmstat counters are not perfectly accurate and the estimated 7313 * value for counters such as NR_FREE_PAGES can deviate from the 7314 * true value by nr_online_cpus * threshold. To avoid the zone 7315 * watermarks being breached while under pressure, we reduce the 7316 * per-cpu vmstat threshold while kswapd is awake and restore 7317 * them before going back to sleep. 7318 */ 7319 set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold); 7320 7321 if (!kthread_should_stop()) 7322 schedule(); 7323 7324 set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold); 7325 } else { 7326 if (remaining) 7327 count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY); 7328 else 7329 count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY); 7330 } 7331 finish_wait(&pgdat->kswapd_wait, &wait); 7332 } 7333 7334 /* 7335 * The background pageout daemon, started as a kernel thread 7336 * from the init process. 7337 * 7338 * This basically trickles out pages so that we have _some_ 7339 * free memory available even if there is no other activity 7340 * that frees anything up. This is needed for things like routing 7341 * etc, where we otherwise might have all activity going on in 7342 * asynchronous contexts that cannot page things out. 7343 * 7344 * If there are applications that are active memory-allocators 7345 * (most normal use), this basically shouldn't matter. 7346 */ 7347 static int kswapd(void *p) 7348 { 7349 unsigned int alloc_order, reclaim_order; 7350 unsigned int highest_zoneidx = MAX_NR_ZONES - 1; 7351 pg_data_t *pgdat = (pg_data_t *)p; 7352 struct task_struct *tsk = current; 7353 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 7354 7355 if (!cpumask_empty(cpumask)) 7356 set_cpus_allowed_ptr(tsk, cpumask); 7357 7358 /* 7359 * Tell the memory management that we're a "memory allocator", 7360 * and that if we need more memory we should get access to it 7361 * regardless (see "__alloc_pages()"). "kswapd" should 7362 * never get caught in the normal page freeing logic. 7363 * 7364 * (Kswapd normally doesn't need memory anyway, but sometimes 7365 * you need a small amount of memory in order to be able to 7366 * page out something else, and this flag essentially protects 7367 * us from recursively trying to free more memory as we're 7368 * trying to free the first piece of memory in the first place). 7369 */ 7370 tsk->flags |= PF_MEMALLOC | PF_KSWAPD; 7371 set_freezable(); 7372 7373 WRITE_ONCE(pgdat->kswapd_order, 0); 7374 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); 7375 atomic_set(&pgdat->nr_writeback_throttled, 0); 7376 for ( ; ; ) { 7377 bool ret; 7378 7379 alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order); 7380 highest_zoneidx = kswapd_highest_zoneidx(pgdat, 7381 highest_zoneidx); 7382 7383 kswapd_try_sleep: 7384 kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order, 7385 highest_zoneidx); 7386 7387 /* Read the new order and highest_zoneidx */ 7388 alloc_order = READ_ONCE(pgdat->kswapd_order); 7389 highest_zoneidx = kswapd_highest_zoneidx(pgdat, 7390 highest_zoneidx); 7391 WRITE_ONCE(pgdat->kswapd_order, 0); 7392 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); 7393 7394 ret = try_to_freeze(); 7395 if (kthread_should_stop()) 7396 break; 7397 7398 /* 7399 * We can speed up thawing tasks if we don't call balance_pgdat 7400 * after returning from the refrigerator 7401 */ 7402 if (ret) 7403 continue; 7404 7405 /* 7406 * Reclaim begins at the requested order but if a high-order 7407 * reclaim fails then kswapd falls back to reclaiming for 7408 * order-0. If that happens, kswapd will consider sleeping 7409 * for the order it finished reclaiming at (reclaim_order) 7410 * but kcompactd is woken to compact for the original 7411 * request (alloc_order). 7412 */ 7413 trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx, 7414 alloc_order); 7415 reclaim_order = balance_pgdat(pgdat, alloc_order, 7416 highest_zoneidx); 7417 if (reclaim_order < alloc_order) 7418 goto kswapd_try_sleep; 7419 } 7420 7421 tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD); 7422 7423 return 0; 7424 } 7425 7426 /* 7427 * A zone is low on free memory or too fragmented for high-order memory. If 7428 * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's 7429 * pgdat. It will wake up kcompactd after reclaiming memory. If kswapd reclaim 7430 * has failed or is not needed, still wake up kcompactd if only compaction is 7431 * needed. 7432 */ 7433 void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order, 7434 enum zone_type highest_zoneidx) 7435 { 7436 pg_data_t *pgdat; 7437 enum zone_type curr_idx; 7438 7439 if (!managed_zone(zone)) 7440 return; 7441 7442 if (!cpuset_zone_allowed(zone, gfp_flags)) 7443 return; 7444 7445 pgdat = zone->zone_pgdat; 7446 curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx); 7447 7448 if (curr_idx == MAX_NR_ZONES || curr_idx < highest_zoneidx) 7449 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx); 7450 7451 if (READ_ONCE(pgdat->kswapd_order) < order) 7452 WRITE_ONCE(pgdat->kswapd_order, order); 7453 7454 if (!waitqueue_active(&pgdat->kswapd_wait)) 7455 return; 7456 7457 /* Hopeless node, leave it to direct reclaim if possible */ 7458 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES || 7459 (pgdat_balanced(pgdat, order, highest_zoneidx) && 7460 !pgdat_watermark_boosted(pgdat, highest_zoneidx))) { 7461 /* 7462 * There may be plenty of free memory available, but it's too 7463 * fragmented for high-order allocations. Wake up kcompactd 7464 * and rely on compaction_suitable() to determine if it's 7465 * needed. If it fails, it will defer subsequent attempts to 7466 * ratelimit its work. 7467 */ 7468 if (!(gfp_flags & __GFP_DIRECT_RECLAIM)) 7469 wakeup_kcompactd(pgdat, order, highest_zoneidx); 7470 return; 7471 } 7472 7473 trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order, 7474 gfp_flags); 7475 wake_up_interruptible(&pgdat->kswapd_wait); 7476 } 7477 7478 #ifdef CONFIG_HIBERNATION 7479 /* 7480 * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of 7481 * freed pages. 7482 * 7483 * Rather than trying to age LRUs the aim is to preserve the overall 7484 * LRU order by reclaiming preferentially 7485 * inactive > active > active referenced > active mapped 7486 */ 7487 unsigned long shrink_all_memory(unsigned long nr_to_reclaim) 7488 { 7489 struct scan_control sc = { 7490 .nr_to_reclaim = nr_to_reclaim, 7491 .gfp_mask = GFP_HIGHUSER_MOVABLE, 7492 .reclaim_idx = MAX_NR_ZONES - 1, 7493 .priority = DEF_PRIORITY, 7494 .may_writepage = 1, 7495 .may_unmap = 1, 7496 .may_swap = 1, 7497 .hibernation_mode = 1, 7498 }; 7499 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); 7500 unsigned long nr_reclaimed; 7501 unsigned int noreclaim_flag; 7502 7503 fs_reclaim_acquire(sc.gfp_mask); 7504 noreclaim_flag = memalloc_noreclaim_save(); 7505 set_task_reclaim_state(current, &sc.reclaim_state); 7506 7507 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 7508 7509 set_task_reclaim_state(current, NULL); 7510 memalloc_noreclaim_restore(noreclaim_flag); 7511 fs_reclaim_release(sc.gfp_mask); 7512 7513 return nr_reclaimed; 7514 } 7515 #endif /* CONFIG_HIBERNATION */ 7516 7517 /* 7518 * This kswapd start function will be called by init and node-hot-add. 7519 */ 7520 void kswapd_run(int nid) 7521 { 7522 pg_data_t *pgdat = NODE_DATA(nid); 7523 7524 pgdat_kswapd_lock(pgdat); 7525 if (!pgdat->kswapd) { 7526 pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid); 7527 if (IS_ERR(pgdat->kswapd)) { 7528 /* failure at boot is fatal */ 7529 BUG_ON(system_state < SYSTEM_RUNNING); 7530 pr_err("Failed to start kswapd on node %d\n", nid); 7531 pgdat->kswapd = NULL; 7532 } 7533 } 7534 pgdat_kswapd_unlock(pgdat); 7535 } 7536 7537 /* 7538 * Called by memory hotplug when all memory in a node is offlined. Caller must 7539 * be holding mem_hotplug_begin/done(). 7540 */ 7541 void kswapd_stop(int nid) 7542 { 7543 pg_data_t *pgdat = NODE_DATA(nid); 7544 struct task_struct *kswapd; 7545 7546 pgdat_kswapd_lock(pgdat); 7547 kswapd = pgdat->kswapd; 7548 if (kswapd) { 7549 kthread_stop(kswapd); 7550 pgdat->kswapd = NULL; 7551 } 7552 pgdat_kswapd_unlock(pgdat); 7553 } 7554 7555 static int __init kswapd_init(void) 7556 { 7557 int nid; 7558 7559 swap_setup(); 7560 for_each_node_state(nid, N_MEMORY) 7561 kswapd_run(nid); 7562 return 0; 7563 } 7564 7565 module_init(kswapd_init) 7566 7567 #ifdef CONFIG_NUMA 7568 /* 7569 * Node reclaim mode 7570 * 7571 * If non-zero call node_reclaim when the number of free pages falls below 7572 * the watermarks. 7573 */ 7574 int node_reclaim_mode __read_mostly; 7575 7576 /* 7577 * Priority for NODE_RECLAIM. This determines the fraction of pages 7578 * of a node considered for each zone_reclaim. 4 scans 1/16th of 7579 * a zone. 7580 */ 7581 #define NODE_RECLAIM_PRIORITY 4 7582 7583 /* 7584 * Percentage of pages in a zone that must be unmapped for node_reclaim to 7585 * occur. 7586 */ 7587 int sysctl_min_unmapped_ratio = 1; 7588 7589 /* 7590 * If the number of slab pages in a zone grows beyond this percentage then 7591 * slab reclaim needs to occur. 7592 */ 7593 int sysctl_min_slab_ratio = 5; 7594 7595 static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat) 7596 { 7597 unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED); 7598 unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) + 7599 node_page_state(pgdat, NR_ACTIVE_FILE); 7600 7601 /* 7602 * It's possible for there to be more file mapped pages than 7603 * accounted for by the pages on the file LRU lists because 7604 * tmpfs pages accounted for as ANON can also be FILE_MAPPED 7605 */ 7606 return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0; 7607 } 7608 7609 /* Work out how many page cache pages we can reclaim in this reclaim_mode */ 7610 static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat) 7611 { 7612 unsigned long nr_pagecache_reclaimable; 7613 unsigned long delta = 0; 7614 7615 /* 7616 * If RECLAIM_UNMAP is set, then all file pages are considered 7617 * potentially reclaimable. Otherwise, we have to worry about 7618 * pages like swapcache and node_unmapped_file_pages() provides 7619 * a better estimate 7620 */ 7621 if (node_reclaim_mode & RECLAIM_UNMAP) 7622 nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES); 7623 else 7624 nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat); 7625 7626 /* If we can't clean pages, remove dirty pages from consideration */ 7627 if (!(node_reclaim_mode & RECLAIM_WRITE)) 7628 delta += node_page_state(pgdat, NR_FILE_DIRTY); 7629 7630 /* Watch for any possible underflows due to delta */ 7631 if (unlikely(delta > nr_pagecache_reclaimable)) 7632 delta = nr_pagecache_reclaimable; 7633 7634 return nr_pagecache_reclaimable - delta; 7635 } 7636 7637 /* 7638 * Try to free up some pages from this node through reclaim. 7639 */ 7640 static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) 7641 { 7642 /* Minimum pages needed in order to stay on node */ 7643 const unsigned long nr_pages = 1 << order; 7644 struct task_struct *p = current; 7645 unsigned int noreclaim_flag; 7646 struct scan_control sc = { 7647 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), 7648 .gfp_mask = current_gfp_context(gfp_mask), 7649 .order = order, 7650 .priority = NODE_RECLAIM_PRIORITY, 7651 .may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE), 7652 .may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP), 7653 .may_swap = 1, 7654 .reclaim_idx = gfp_zone(gfp_mask), 7655 }; 7656 unsigned long pflags; 7657 7658 trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order, 7659 sc.gfp_mask); 7660 7661 cond_resched(); 7662 psi_memstall_enter(&pflags); 7663 fs_reclaim_acquire(sc.gfp_mask); 7664 /* 7665 * We need to be able to allocate from the reserves for RECLAIM_UNMAP 7666 */ 7667 noreclaim_flag = memalloc_noreclaim_save(); 7668 set_task_reclaim_state(p, &sc.reclaim_state); 7669 7670 if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages || 7671 node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) { 7672 /* 7673 * Free memory by calling shrink node with increasing 7674 * priorities until we have enough memory freed. 7675 */ 7676 do { 7677 shrink_node(pgdat, &sc); 7678 } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0); 7679 } 7680 7681 set_task_reclaim_state(p, NULL); 7682 memalloc_noreclaim_restore(noreclaim_flag); 7683 fs_reclaim_release(sc.gfp_mask); 7684 psi_memstall_leave(&pflags); 7685 7686 trace_mm_vmscan_node_reclaim_end(sc.nr_reclaimed); 7687 7688 return sc.nr_reclaimed >= nr_pages; 7689 } 7690 7691 int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) 7692 { 7693 int ret; 7694 7695 /* 7696 * Node reclaim reclaims unmapped file backed pages and 7697 * slab pages if we are over the defined limits. 7698 * 7699 * A small portion of unmapped file backed pages is needed for 7700 * file I/O otherwise pages read by file I/O will be immediately 7701 * thrown out if the node is overallocated. So we do not reclaim 7702 * if less than a specified percentage of the node is used by 7703 * unmapped file backed pages. 7704 */ 7705 if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages && 7706 node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <= 7707 pgdat->min_slab_pages) 7708 return NODE_RECLAIM_FULL; 7709 7710 /* 7711 * Do not scan if the allocation should not be delayed. 7712 */ 7713 if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC)) 7714 return NODE_RECLAIM_NOSCAN; 7715 7716 /* 7717 * Only run node reclaim on the local node or on nodes that do not 7718 * have associated processors. This will favor the local processor 7719 * over remote processors and spread off node memory allocations 7720 * as wide as possible. 7721 */ 7722 if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id()) 7723 return NODE_RECLAIM_NOSCAN; 7724 7725 if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags)) 7726 return NODE_RECLAIM_NOSCAN; 7727 7728 ret = __node_reclaim(pgdat, gfp_mask, order); 7729 clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags); 7730 7731 if (!ret) 7732 count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED); 7733 7734 return ret; 7735 } 7736 #endif 7737 7738 void check_move_unevictable_pages(struct pagevec *pvec) 7739 { 7740 struct folio_batch fbatch; 7741 unsigned i; 7742 7743 folio_batch_init(&fbatch); 7744 for (i = 0; i < pvec->nr; i++) { 7745 struct page *page = pvec->pages[i]; 7746 7747 if (PageTransTail(page)) 7748 continue; 7749 folio_batch_add(&fbatch, page_folio(page)); 7750 } 7751 check_move_unevictable_folios(&fbatch); 7752 } 7753 EXPORT_SYMBOL_GPL(check_move_unevictable_pages); 7754 7755 /** 7756 * check_move_unevictable_folios - Move evictable folios to appropriate zone 7757 * lru list 7758 * @fbatch: Batch of lru folios to check. 7759 * 7760 * Checks folios for evictability, if an evictable folio is in the unevictable 7761 * lru list, moves it to the appropriate evictable lru list. This function 7762 * should be only used for lru folios. 7763 */ 7764 void check_move_unevictable_folios(struct folio_batch *fbatch) 7765 { 7766 struct lruvec *lruvec = NULL; 7767 int pgscanned = 0; 7768 int pgrescued = 0; 7769 int i; 7770 7771 for (i = 0; i < fbatch->nr; i++) { 7772 struct folio *folio = fbatch->folios[i]; 7773 int nr_pages = folio_nr_pages(folio); 7774 7775 pgscanned += nr_pages; 7776 7777 /* block memcg migration while the folio moves between lrus */ 7778 if (!folio_test_clear_lru(folio)) 7779 continue; 7780 7781 lruvec = folio_lruvec_relock_irq(folio, lruvec); 7782 if (folio_evictable(folio) && folio_test_unevictable(folio)) { 7783 lruvec_del_folio(lruvec, folio); 7784 folio_clear_unevictable(folio); 7785 lruvec_add_folio(lruvec, folio); 7786 pgrescued += nr_pages; 7787 } 7788 folio_set_lru(folio); 7789 } 7790 7791 if (lruvec) { 7792 __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued); 7793 __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); 7794 unlock_page_lruvec_irq(lruvec); 7795 } else if (pgscanned) { 7796 count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); 7797 } 7798 } 7799 EXPORT_SYMBOL_GPL(check_move_unevictable_folios); 7800