1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 4 * 5 * Swap reorganised 29.12.95, Stephen Tweedie. 6 * kswapd added: 7.1.96 sct 7 * Removed kswapd_ctl limits, and swap out as many pages as needed 8 * to bring the system back to freepages.high: 2.4.97, Rik van Riel. 9 * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com). 10 * Multiqueue VM started 5.8.00, Rik van Riel. 11 */ 12 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15 #include <linux/mm.h> 16 #include <linux/sched/mm.h> 17 #include <linux/module.h> 18 #include <linux/gfp.h> 19 #include <linux/kernel_stat.h> 20 #include <linux/swap.h> 21 #include <linux/pagemap.h> 22 #include <linux/init.h> 23 #include <linux/highmem.h> 24 #include <linux/vmpressure.h> 25 #include <linux/vmstat.h> 26 #include <linux/file.h> 27 #include <linux/writeback.h> 28 #include <linux/blkdev.h> 29 #include <linux/buffer_head.h> /* for buffer_heads_over_limit */ 30 #include <linux/mm_inline.h> 31 #include <linux/backing-dev.h> 32 #include <linux/rmap.h> 33 #include <linux/topology.h> 34 #include <linux/cpu.h> 35 #include <linux/cpuset.h> 36 #include <linux/compaction.h> 37 #include <linux/notifier.h> 38 #include <linux/rwsem.h> 39 #include <linux/delay.h> 40 #include <linux/kthread.h> 41 #include <linux/freezer.h> 42 #include <linux/memcontrol.h> 43 #include <linux/migrate.h> 44 #include <linux/delayacct.h> 45 #include <linux/sysctl.h> 46 #include <linux/memory-tiers.h> 47 #include <linux/oom.h> 48 #include <linux/pagevec.h> 49 #include <linux/prefetch.h> 50 #include <linux/printk.h> 51 #include <linux/dax.h> 52 #include <linux/psi.h> 53 #include <linux/pagewalk.h> 54 #include <linux/shmem_fs.h> 55 #include <linux/ctype.h> 56 #include <linux/debugfs.h> 57 58 #include <asm/tlbflush.h> 59 #include <asm/div64.h> 60 61 #include <linux/swapops.h> 62 #include <linux/balloon_compaction.h> 63 #include <linux/sched/sysctl.h> 64 65 #include "internal.h" 66 #include "swap.h" 67 68 #define CREATE_TRACE_POINTS 69 #include <trace/events/vmscan.h> 70 71 struct scan_control { 72 /* How many pages shrink_list() should reclaim */ 73 unsigned long nr_to_reclaim; 74 75 /* 76 * Nodemask of nodes allowed by the caller. If NULL, all nodes 77 * are scanned. 78 */ 79 nodemask_t *nodemask; 80 81 /* 82 * The memory cgroup that hit its limit and as a result is the 83 * primary target of this reclaim invocation. 84 */ 85 struct mem_cgroup *target_mem_cgroup; 86 87 /* 88 * Scan pressure balancing between anon and file LRUs 89 */ 90 unsigned long anon_cost; 91 unsigned long file_cost; 92 93 /* Can active pages be deactivated as part of reclaim? */ 94 #define DEACTIVATE_ANON 1 95 #define DEACTIVATE_FILE 2 96 unsigned int may_deactivate:2; 97 unsigned int force_deactivate:1; 98 unsigned int skipped_deactivate:1; 99 100 /* Writepage batching in laptop mode; RECLAIM_WRITE */ 101 unsigned int may_writepage:1; 102 103 /* Can mapped pages be reclaimed? */ 104 unsigned int may_unmap:1; 105 106 /* Can pages be swapped as part of reclaim? */ 107 unsigned int may_swap:1; 108 109 /* Proactive reclaim invoked by userspace through memory.reclaim */ 110 unsigned int proactive:1; 111 112 /* 113 * Cgroup memory below memory.low is protected as long as we 114 * don't threaten to OOM. If any cgroup is reclaimed at 115 * reduced force or passed over entirely due to its memory.low 116 * setting (memcg_low_skipped), and nothing is reclaimed as a 117 * result, then go back for one more cycle that reclaims the protected 118 * memory (memcg_low_reclaim) to avert OOM. 119 */ 120 unsigned int memcg_low_reclaim:1; 121 unsigned int memcg_low_skipped:1; 122 123 unsigned int hibernation_mode:1; 124 125 /* One of the zones is ready for compaction */ 126 unsigned int compaction_ready:1; 127 128 /* There is easily reclaimable cold cache in the current node */ 129 unsigned int cache_trim_mode:1; 130 131 /* The file pages on the current node are dangerously low */ 132 unsigned int file_is_tiny:1; 133 134 /* Always discard instead of demoting to lower tier memory */ 135 unsigned int no_demotion:1; 136 137 #ifdef CONFIG_LRU_GEN 138 /* help kswapd make better choices among multiple memcgs */ 139 unsigned int memcgs_need_aging:1; 140 unsigned long last_reclaimed; 141 #endif 142 143 /* Allocation order */ 144 s8 order; 145 146 /* Scan (total_size >> priority) pages at once */ 147 s8 priority; 148 149 /* The highest zone to isolate pages for reclaim from */ 150 s8 reclaim_idx; 151 152 /* This context's GFP mask */ 153 gfp_t gfp_mask; 154 155 /* Incremented by the number of inactive pages that were scanned */ 156 unsigned long nr_scanned; 157 158 /* Number of pages freed so far during a call to shrink_zones() */ 159 unsigned long nr_reclaimed; 160 161 struct { 162 unsigned int dirty; 163 unsigned int unqueued_dirty; 164 unsigned int congested; 165 unsigned int writeback; 166 unsigned int immediate; 167 unsigned int file_taken; 168 unsigned int taken; 169 } nr; 170 171 /* for recording the reclaimed slab by now */ 172 struct reclaim_state reclaim_state; 173 }; 174 175 #ifdef ARCH_HAS_PREFETCHW 176 #define prefetchw_prev_lru_folio(_folio, _base, _field) \ 177 do { \ 178 if ((_folio)->lru.prev != _base) { \ 179 struct folio *prev; \ 180 \ 181 prev = lru_to_folio(&(_folio->lru)); \ 182 prefetchw(&prev->_field); \ 183 } \ 184 } while (0) 185 #else 186 #define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0) 187 #endif 188 189 /* 190 * From 0 .. 200. Higher means more swappy. 191 */ 192 int vm_swappiness = 60; 193 194 static void set_task_reclaim_state(struct task_struct *task, 195 struct reclaim_state *rs) 196 { 197 /* Check for an overwrite */ 198 WARN_ON_ONCE(rs && task->reclaim_state); 199 200 /* Check for the nulling of an already-nulled member */ 201 WARN_ON_ONCE(!rs && !task->reclaim_state); 202 203 task->reclaim_state = rs; 204 } 205 206 LIST_HEAD(shrinker_list); 207 DECLARE_RWSEM(shrinker_rwsem); 208 209 #ifdef CONFIG_MEMCG 210 static int shrinker_nr_max; 211 212 /* The shrinker_info is expanded in a batch of BITS_PER_LONG */ 213 static inline int shrinker_map_size(int nr_items) 214 { 215 return (DIV_ROUND_UP(nr_items, BITS_PER_LONG) * sizeof(unsigned long)); 216 } 217 218 static inline int shrinker_defer_size(int nr_items) 219 { 220 return (round_up(nr_items, BITS_PER_LONG) * sizeof(atomic_long_t)); 221 } 222 223 static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg, 224 int nid) 225 { 226 return rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info, 227 lockdep_is_held(&shrinker_rwsem)); 228 } 229 230 static int expand_one_shrinker_info(struct mem_cgroup *memcg, 231 int map_size, int defer_size, 232 int old_map_size, int old_defer_size) 233 { 234 struct shrinker_info *new, *old; 235 struct mem_cgroup_per_node *pn; 236 int nid; 237 int size = map_size + defer_size; 238 239 for_each_node(nid) { 240 pn = memcg->nodeinfo[nid]; 241 old = shrinker_info_protected(memcg, nid); 242 /* Not yet online memcg */ 243 if (!old) 244 return 0; 245 246 new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid); 247 if (!new) 248 return -ENOMEM; 249 250 new->nr_deferred = (atomic_long_t *)(new + 1); 251 new->map = (void *)new->nr_deferred + defer_size; 252 253 /* map: set all old bits, clear all new bits */ 254 memset(new->map, (int)0xff, old_map_size); 255 memset((void *)new->map + old_map_size, 0, map_size - old_map_size); 256 /* nr_deferred: copy old values, clear all new values */ 257 memcpy(new->nr_deferred, old->nr_deferred, old_defer_size); 258 memset((void *)new->nr_deferred + old_defer_size, 0, 259 defer_size - old_defer_size); 260 261 rcu_assign_pointer(pn->shrinker_info, new); 262 kvfree_rcu(old, rcu); 263 } 264 265 return 0; 266 } 267 268 void free_shrinker_info(struct mem_cgroup *memcg) 269 { 270 struct mem_cgroup_per_node *pn; 271 struct shrinker_info *info; 272 int nid; 273 274 for_each_node(nid) { 275 pn = memcg->nodeinfo[nid]; 276 info = rcu_dereference_protected(pn->shrinker_info, true); 277 kvfree(info); 278 rcu_assign_pointer(pn->shrinker_info, NULL); 279 } 280 } 281 282 int alloc_shrinker_info(struct mem_cgroup *memcg) 283 { 284 struct shrinker_info *info; 285 int nid, size, ret = 0; 286 int map_size, defer_size = 0; 287 288 down_write(&shrinker_rwsem); 289 map_size = shrinker_map_size(shrinker_nr_max); 290 defer_size = shrinker_defer_size(shrinker_nr_max); 291 size = map_size + defer_size; 292 for_each_node(nid) { 293 info = kvzalloc_node(sizeof(*info) + size, GFP_KERNEL, nid); 294 if (!info) { 295 free_shrinker_info(memcg); 296 ret = -ENOMEM; 297 break; 298 } 299 info->nr_deferred = (atomic_long_t *)(info + 1); 300 info->map = (void *)info->nr_deferred + defer_size; 301 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info); 302 } 303 up_write(&shrinker_rwsem); 304 305 return ret; 306 } 307 308 static inline bool need_expand(int nr_max) 309 { 310 return round_up(nr_max, BITS_PER_LONG) > 311 round_up(shrinker_nr_max, BITS_PER_LONG); 312 } 313 314 static int expand_shrinker_info(int new_id) 315 { 316 int ret = 0; 317 int new_nr_max = new_id + 1; 318 int map_size, defer_size = 0; 319 int old_map_size, old_defer_size = 0; 320 struct mem_cgroup *memcg; 321 322 if (!need_expand(new_nr_max)) 323 goto out; 324 325 if (!root_mem_cgroup) 326 goto out; 327 328 lockdep_assert_held(&shrinker_rwsem); 329 330 map_size = shrinker_map_size(new_nr_max); 331 defer_size = shrinker_defer_size(new_nr_max); 332 old_map_size = shrinker_map_size(shrinker_nr_max); 333 old_defer_size = shrinker_defer_size(shrinker_nr_max); 334 335 memcg = mem_cgroup_iter(NULL, NULL, NULL); 336 do { 337 ret = expand_one_shrinker_info(memcg, map_size, defer_size, 338 old_map_size, old_defer_size); 339 if (ret) { 340 mem_cgroup_iter_break(NULL, memcg); 341 goto out; 342 } 343 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); 344 out: 345 if (!ret) 346 shrinker_nr_max = new_nr_max; 347 348 return ret; 349 } 350 351 void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id) 352 { 353 if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) { 354 struct shrinker_info *info; 355 356 rcu_read_lock(); 357 info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info); 358 /* Pairs with smp mb in shrink_slab() */ 359 smp_mb__before_atomic(); 360 set_bit(shrinker_id, info->map); 361 rcu_read_unlock(); 362 } 363 } 364 365 static DEFINE_IDR(shrinker_idr); 366 367 static int prealloc_memcg_shrinker(struct shrinker *shrinker) 368 { 369 int id, ret = -ENOMEM; 370 371 if (mem_cgroup_disabled()) 372 return -ENOSYS; 373 374 down_write(&shrinker_rwsem); 375 /* This may call shrinker, so it must use down_read_trylock() */ 376 id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL); 377 if (id < 0) 378 goto unlock; 379 380 if (id >= shrinker_nr_max) { 381 if (expand_shrinker_info(id)) { 382 idr_remove(&shrinker_idr, id); 383 goto unlock; 384 } 385 } 386 shrinker->id = id; 387 ret = 0; 388 unlock: 389 up_write(&shrinker_rwsem); 390 return ret; 391 } 392 393 static void unregister_memcg_shrinker(struct shrinker *shrinker) 394 { 395 int id = shrinker->id; 396 397 BUG_ON(id < 0); 398 399 lockdep_assert_held(&shrinker_rwsem); 400 401 idr_remove(&shrinker_idr, id); 402 } 403 404 static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker, 405 struct mem_cgroup *memcg) 406 { 407 struct shrinker_info *info; 408 409 info = shrinker_info_protected(memcg, nid); 410 return atomic_long_xchg(&info->nr_deferred[shrinker->id], 0); 411 } 412 413 static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker, 414 struct mem_cgroup *memcg) 415 { 416 struct shrinker_info *info; 417 418 info = shrinker_info_protected(memcg, nid); 419 return atomic_long_add_return(nr, &info->nr_deferred[shrinker->id]); 420 } 421 422 void reparent_shrinker_deferred(struct mem_cgroup *memcg) 423 { 424 int i, nid; 425 long nr; 426 struct mem_cgroup *parent; 427 struct shrinker_info *child_info, *parent_info; 428 429 parent = parent_mem_cgroup(memcg); 430 if (!parent) 431 parent = root_mem_cgroup; 432 433 /* Prevent from concurrent shrinker_info expand */ 434 down_read(&shrinker_rwsem); 435 for_each_node(nid) { 436 child_info = shrinker_info_protected(memcg, nid); 437 parent_info = shrinker_info_protected(parent, nid); 438 for (i = 0; i < shrinker_nr_max; i++) { 439 nr = atomic_long_read(&child_info->nr_deferred[i]); 440 atomic_long_add(nr, &parent_info->nr_deferred[i]); 441 } 442 } 443 up_read(&shrinker_rwsem); 444 } 445 446 static bool cgroup_reclaim(struct scan_control *sc) 447 { 448 return sc->target_mem_cgroup; 449 } 450 451 /** 452 * writeback_throttling_sane - is the usual dirty throttling mechanism available? 453 * @sc: scan_control in question 454 * 455 * The normal page dirty throttling mechanism in balance_dirty_pages() is 456 * completely broken with the legacy memcg and direct stalling in 457 * shrink_page_list() is used for throttling instead, which lacks all the 458 * niceties such as fairness, adaptive pausing, bandwidth proportional 459 * allocation and configurability. 460 * 461 * This function tests whether the vmscan currently in progress can assume 462 * that the normal dirty throttling mechanism is operational. 463 */ 464 static bool writeback_throttling_sane(struct scan_control *sc) 465 { 466 if (!cgroup_reclaim(sc)) 467 return true; 468 #ifdef CONFIG_CGROUP_WRITEBACK 469 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 470 return true; 471 #endif 472 return false; 473 } 474 #else 475 static int prealloc_memcg_shrinker(struct shrinker *shrinker) 476 { 477 return -ENOSYS; 478 } 479 480 static void unregister_memcg_shrinker(struct shrinker *shrinker) 481 { 482 } 483 484 static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker, 485 struct mem_cgroup *memcg) 486 { 487 return 0; 488 } 489 490 static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker, 491 struct mem_cgroup *memcg) 492 { 493 return 0; 494 } 495 496 static bool cgroup_reclaim(struct scan_control *sc) 497 { 498 return false; 499 } 500 501 static bool writeback_throttling_sane(struct scan_control *sc) 502 { 503 return true; 504 } 505 #endif 506 507 static long xchg_nr_deferred(struct shrinker *shrinker, 508 struct shrink_control *sc) 509 { 510 int nid = sc->nid; 511 512 if (!(shrinker->flags & SHRINKER_NUMA_AWARE)) 513 nid = 0; 514 515 if (sc->memcg && 516 (shrinker->flags & SHRINKER_MEMCG_AWARE)) 517 return xchg_nr_deferred_memcg(nid, shrinker, 518 sc->memcg); 519 520 return atomic_long_xchg(&shrinker->nr_deferred[nid], 0); 521 } 522 523 524 static long add_nr_deferred(long nr, struct shrinker *shrinker, 525 struct shrink_control *sc) 526 { 527 int nid = sc->nid; 528 529 if (!(shrinker->flags & SHRINKER_NUMA_AWARE)) 530 nid = 0; 531 532 if (sc->memcg && 533 (shrinker->flags & SHRINKER_MEMCG_AWARE)) 534 return add_nr_deferred_memcg(nr, nid, shrinker, 535 sc->memcg); 536 537 return atomic_long_add_return(nr, &shrinker->nr_deferred[nid]); 538 } 539 540 static bool can_demote(int nid, struct scan_control *sc) 541 { 542 if (!numa_demotion_enabled) 543 return false; 544 if (sc && sc->no_demotion) 545 return false; 546 if (next_demotion_node(nid) == NUMA_NO_NODE) 547 return false; 548 549 return true; 550 } 551 552 static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg, 553 int nid, 554 struct scan_control *sc) 555 { 556 if (memcg == NULL) { 557 /* 558 * For non-memcg reclaim, is there 559 * space in any swap device? 560 */ 561 if (get_nr_swap_pages() > 0) 562 return true; 563 } else { 564 /* Is the memcg below its swap limit? */ 565 if (mem_cgroup_get_nr_swap_pages(memcg) > 0) 566 return true; 567 } 568 569 /* 570 * The page can not be swapped. 571 * 572 * Can it be reclaimed from this node via demotion? 573 */ 574 return can_demote(nid, sc); 575 } 576 577 /* 578 * This misses isolated pages which are not accounted for to save counters. 579 * As the data only determines if reclaim or compaction continues, it is 580 * not expected that isolated pages will be a dominating factor. 581 */ 582 unsigned long zone_reclaimable_pages(struct zone *zone) 583 { 584 unsigned long nr; 585 586 nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) + 587 zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE); 588 if (can_reclaim_anon_pages(NULL, zone_to_nid(zone), NULL)) 589 nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) + 590 zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON); 591 592 return nr; 593 } 594 595 /** 596 * lruvec_lru_size - Returns the number of pages on the given LRU list. 597 * @lruvec: lru vector 598 * @lru: lru to use 599 * @zone_idx: zones to consider (use MAX_NR_ZONES - 1 for the whole LRU list) 600 */ 601 static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, 602 int zone_idx) 603 { 604 unsigned long size = 0; 605 int zid; 606 607 for (zid = 0; zid <= zone_idx; zid++) { 608 struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid]; 609 610 if (!managed_zone(zone)) 611 continue; 612 613 if (!mem_cgroup_disabled()) 614 size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid); 615 else 616 size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru); 617 } 618 return size; 619 } 620 621 /* 622 * Add a shrinker callback to be called from the vm. 623 */ 624 static int __prealloc_shrinker(struct shrinker *shrinker) 625 { 626 unsigned int size; 627 int err; 628 629 if (shrinker->flags & SHRINKER_MEMCG_AWARE) { 630 err = prealloc_memcg_shrinker(shrinker); 631 if (err != -ENOSYS) 632 return err; 633 634 shrinker->flags &= ~SHRINKER_MEMCG_AWARE; 635 } 636 637 size = sizeof(*shrinker->nr_deferred); 638 if (shrinker->flags & SHRINKER_NUMA_AWARE) 639 size *= nr_node_ids; 640 641 shrinker->nr_deferred = kzalloc(size, GFP_KERNEL); 642 if (!shrinker->nr_deferred) 643 return -ENOMEM; 644 645 return 0; 646 } 647 648 #ifdef CONFIG_SHRINKER_DEBUG 649 int prealloc_shrinker(struct shrinker *shrinker, const char *fmt, ...) 650 { 651 va_list ap; 652 int err; 653 654 va_start(ap, fmt); 655 shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap); 656 va_end(ap); 657 if (!shrinker->name) 658 return -ENOMEM; 659 660 err = __prealloc_shrinker(shrinker); 661 if (err) { 662 kfree_const(shrinker->name); 663 shrinker->name = NULL; 664 } 665 666 return err; 667 } 668 #else 669 int prealloc_shrinker(struct shrinker *shrinker, const char *fmt, ...) 670 { 671 return __prealloc_shrinker(shrinker); 672 } 673 #endif 674 675 void free_prealloced_shrinker(struct shrinker *shrinker) 676 { 677 #ifdef CONFIG_SHRINKER_DEBUG 678 kfree_const(shrinker->name); 679 shrinker->name = NULL; 680 #endif 681 if (shrinker->flags & SHRINKER_MEMCG_AWARE) { 682 down_write(&shrinker_rwsem); 683 unregister_memcg_shrinker(shrinker); 684 up_write(&shrinker_rwsem); 685 return; 686 } 687 688 kfree(shrinker->nr_deferred); 689 shrinker->nr_deferred = NULL; 690 } 691 692 void register_shrinker_prepared(struct shrinker *shrinker) 693 { 694 down_write(&shrinker_rwsem); 695 list_add_tail(&shrinker->list, &shrinker_list); 696 shrinker->flags |= SHRINKER_REGISTERED; 697 shrinker_debugfs_add(shrinker); 698 up_write(&shrinker_rwsem); 699 } 700 701 static int __register_shrinker(struct shrinker *shrinker) 702 { 703 int err = __prealloc_shrinker(shrinker); 704 705 if (err) 706 return err; 707 register_shrinker_prepared(shrinker); 708 return 0; 709 } 710 711 #ifdef CONFIG_SHRINKER_DEBUG 712 int register_shrinker(struct shrinker *shrinker, const char *fmt, ...) 713 { 714 va_list ap; 715 int err; 716 717 va_start(ap, fmt); 718 shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap); 719 va_end(ap); 720 if (!shrinker->name) 721 return -ENOMEM; 722 723 err = __register_shrinker(shrinker); 724 if (err) { 725 kfree_const(shrinker->name); 726 shrinker->name = NULL; 727 } 728 return err; 729 } 730 #else 731 int register_shrinker(struct shrinker *shrinker, const char *fmt, ...) 732 { 733 return __register_shrinker(shrinker); 734 } 735 #endif 736 EXPORT_SYMBOL(register_shrinker); 737 738 /* 739 * Remove one 740 */ 741 void unregister_shrinker(struct shrinker *shrinker) 742 { 743 if (!(shrinker->flags & SHRINKER_REGISTERED)) 744 return; 745 746 down_write(&shrinker_rwsem); 747 list_del(&shrinker->list); 748 shrinker->flags &= ~SHRINKER_REGISTERED; 749 if (shrinker->flags & SHRINKER_MEMCG_AWARE) 750 unregister_memcg_shrinker(shrinker); 751 shrinker_debugfs_remove(shrinker); 752 up_write(&shrinker_rwsem); 753 754 kfree(shrinker->nr_deferred); 755 shrinker->nr_deferred = NULL; 756 } 757 EXPORT_SYMBOL(unregister_shrinker); 758 759 /** 760 * synchronize_shrinkers - Wait for all running shrinkers to complete. 761 * 762 * This is equivalent to calling unregister_shrink() and register_shrinker(), 763 * but atomically and with less overhead. This is useful to guarantee that all 764 * shrinker invocations have seen an update, before freeing memory, similar to 765 * rcu. 766 */ 767 void synchronize_shrinkers(void) 768 { 769 down_write(&shrinker_rwsem); 770 up_write(&shrinker_rwsem); 771 } 772 EXPORT_SYMBOL(synchronize_shrinkers); 773 774 #define SHRINK_BATCH 128 775 776 static unsigned long do_shrink_slab(struct shrink_control *shrinkctl, 777 struct shrinker *shrinker, int priority) 778 { 779 unsigned long freed = 0; 780 unsigned long long delta; 781 long total_scan; 782 long freeable; 783 long nr; 784 long new_nr; 785 long batch_size = shrinker->batch ? shrinker->batch 786 : SHRINK_BATCH; 787 long scanned = 0, next_deferred; 788 789 freeable = shrinker->count_objects(shrinker, shrinkctl); 790 if (freeable == 0 || freeable == SHRINK_EMPTY) 791 return freeable; 792 793 /* 794 * copy the current shrinker scan count into a local variable 795 * and zero it so that other concurrent shrinker invocations 796 * don't also do this scanning work. 797 */ 798 nr = xchg_nr_deferred(shrinker, shrinkctl); 799 800 if (shrinker->seeks) { 801 delta = freeable >> priority; 802 delta *= 4; 803 do_div(delta, shrinker->seeks); 804 } else { 805 /* 806 * These objects don't require any IO to create. Trim 807 * them aggressively under memory pressure to keep 808 * them from causing refetches in the IO caches. 809 */ 810 delta = freeable / 2; 811 } 812 813 total_scan = nr >> priority; 814 total_scan += delta; 815 total_scan = min(total_scan, (2 * freeable)); 816 817 trace_mm_shrink_slab_start(shrinker, shrinkctl, nr, 818 freeable, delta, total_scan, priority); 819 820 /* 821 * Normally, we should not scan less than batch_size objects in one 822 * pass to avoid too frequent shrinker calls, but if the slab has less 823 * than batch_size objects in total and we are really tight on memory, 824 * we will try to reclaim all available objects, otherwise we can end 825 * up failing allocations although there are plenty of reclaimable 826 * objects spread over several slabs with usage less than the 827 * batch_size. 828 * 829 * We detect the "tight on memory" situations by looking at the total 830 * number of objects we want to scan (total_scan). If it is greater 831 * than the total number of objects on slab (freeable), we must be 832 * scanning at high prio and therefore should try to reclaim as much as 833 * possible. 834 */ 835 while (total_scan >= batch_size || 836 total_scan >= freeable) { 837 unsigned long ret; 838 unsigned long nr_to_scan = min(batch_size, total_scan); 839 840 shrinkctl->nr_to_scan = nr_to_scan; 841 shrinkctl->nr_scanned = nr_to_scan; 842 ret = shrinker->scan_objects(shrinker, shrinkctl); 843 if (ret == SHRINK_STOP) 844 break; 845 freed += ret; 846 847 count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned); 848 total_scan -= shrinkctl->nr_scanned; 849 scanned += shrinkctl->nr_scanned; 850 851 cond_resched(); 852 } 853 854 /* 855 * The deferred work is increased by any new work (delta) that wasn't 856 * done, decreased by old deferred work that was done now. 857 * 858 * And it is capped to two times of the freeable items. 859 */ 860 next_deferred = max_t(long, (nr + delta - scanned), 0); 861 next_deferred = min(next_deferred, (2 * freeable)); 862 863 /* 864 * move the unused scan count back into the shrinker in a 865 * manner that handles concurrent updates. 866 */ 867 new_nr = add_nr_deferred(next_deferred, shrinker, shrinkctl); 868 869 trace_mm_shrink_slab_end(shrinker, shrinkctl->nid, freed, nr, new_nr, total_scan); 870 return freed; 871 } 872 873 #ifdef CONFIG_MEMCG 874 static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, 875 struct mem_cgroup *memcg, int priority) 876 { 877 struct shrinker_info *info; 878 unsigned long ret, freed = 0; 879 int i; 880 881 if (!mem_cgroup_online(memcg)) 882 return 0; 883 884 if (!down_read_trylock(&shrinker_rwsem)) 885 return 0; 886 887 info = shrinker_info_protected(memcg, nid); 888 if (unlikely(!info)) 889 goto unlock; 890 891 for_each_set_bit(i, info->map, shrinker_nr_max) { 892 struct shrink_control sc = { 893 .gfp_mask = gfp_mask, 894 .nid = nid, 895 .memcg = memcg, 896 }; 897 struct shrinker *shrinker; 898 899 shrinker = idr_find(&shrinker_idr, i); 900 if (unlikely(!shrinker || !(shrinker->flags & SHRINKER_REGISTERED))) { 901 if (!shrinker) 902 clear_bit(i, info->map); 903 continue; 904 } 905 906 /* Call non-slab shrinkers even though kmem is disabled */ 907 if (!memcg_kmem_enabled() && 908 !(shrinker->flags & SHRINKER_NONSLAB)) 909 continue; 910 911 ret = do_shrink_slab(&sc, shrinker, priority); 912 if (ret == SHRINK_EMPTY) { 913 clear_bit(i, info->map); 914 /* 915 * After the shrinker reported that it had no objects to 916 * free, but before we cleared the corresponding bit in 917 * the memcg shrinker map, a new object might have been 918 * added. To make sure, we have the bit set in this 919 * case, we invoke the shrinker one more time and reset 920 * the bit if it reports that it is not empty anymore. 921 * The memory barrier here pairs with the barrier in 922 * set_shrinker_bit(): 923 * 924 * list_lru_add() shrink_slab_memcg() 925 * list_add_tail() clear_bit() 926 * <MB> <MB> 927 * set_bit() do_shrink_slab() 928 */ 929 smp_mb__after_atomic(); 930 ret = do_shrink_slab(&sc, shrinker, priority); 931 if (ret == SHRINK_EMPTY) 932 ret = 0; 933 else 934 set_shrinker_bit(memcg, nid, i); 935 } 936 freed += ret; 937 938 if (rwsem_is_contended(&shrinker_rwsem)) { 939 freed = freed ? : 1; 940 break; 941 } 942 } 943 unlock: 944 up_read(&shrinker_rwsem); 945 return freed; 946 } 947 #else /* CONFIG_MEMCG */ 948 static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, 949 struct mem_cgroup *memcg, int priority) 950 { 951 return 0; 952 } 953 #endif /* CONFIG_MEMCG */ 954 955 /** 956 * shrink_slab - shrink slab caches 957 * @gfp_mask: allocation context 958 * @nid: node whose slab caches to target 959 * @memcg: memory cgroup whose slab caches to target 960 * @priority: the reclaim priority 961 * 962 * Call the shrink functions to age shrinkable caches. 963 * 964 * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set, 965 * unaware shrinkers will receive a node id of 0 instead. 966 * 967 * @memcg specifies the memory cgroup to target. Unaware shrinkers 968 * are called only if it is the root cgroup. 969 * 970 * @priority is sc->priority, we take the number of objects and >> by priority 971 * in order to get the scan target. 972 * 973 * Returns the number of reclaimed slab objects. 974 */ 975 static unsigned long shrink_slab(gfp_t gfp_mask, int nid, 976 struct mem_cgroup *memcg, 977 int priority) 978 { 979 unsigned long ret, freed = 0; 980 struct shrinker *shrinker; 981 982 /* 983 * The root memcg might be allocated even though memcg is disabled 984 * via "cgroup_disable=memory" boot parameter. This could make 985 * mem_cgroup_is_root() return false, then just run memcg slab 986 * shrink, but skip global shrink. This may result in premature 987 * oom. 988 */ 989 if (!mem_cgroup_disabled() && !mem_cgroup_is_root(memcg)) 990 return shrink_slab_memcg(gfp_mask, nid, memcg, priority); 991 992 if (!down_read_trylock(&shrinker_rwsem)) 993 goto out; 994 995 list_for_each_entry(shrinker, &shrinker_list, list) { 996 struct shrink_control sc = { 997 .gfp_mask = gfp_mask, 998 .nid = nid, 999 .memcg = memcg, 1000 }; 1001 1002 ret = do_shrink_slab(&sc, shrinker, priority); 1003 if (ret == SHRINK_EMPTY) 1004 ret = 0; 1005 freed += ret; 1006 /* 1007 * Bail out if someone want to register a new shrinker to 1008 * prevent the registration from being stalled for long periods 1009 * by parallel ongoing shrinking. 1010 */ 1011 if (rwsem_is_contended(&shrinker_rwsem)) { 1012 freed = freed ? : 1; 1013 break; 1014 } 1015 } 1016 1017 up_read(&shrinker_rwsem); 1018 out: 1019 cond_resched(); 1020 return freed; 1021 } 1022 1023 static void drop_slab_node(int nid) 1024 { 1025 unsigned long freed; 1026 int shift = 0; 1027 1028 do { 1029 struct mem_cgroup *memcg = NULL; 1030 1031 if (fatal_signal_pending(current)) 1032 return; 1033 1034 freed = 0; 1035 memcg = mem_cgroup_iter(NULL, NULL, NULL); 1036 do { 1037 freed += shrink_slab(GFP_KERNEL, nid, memcg, 0); 1038 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); 1039 } while ((freed >> shift++) > 1); 1040 } 1041 1042 void drop_slab(void) 1043 { 1044 int nid; 1045 1046 for_each_online_node(nid) 1047 drop_slab_node(nid); 1048 } 1049 1050 static inline int is_page_cache_freeable(struct folio *folio) 1051 { 1052 /* 1053 * A freeable page cache page is referenced only by the caller 1054 * that isolated the page, the page cache and optional buffer 1055 * heads at page->private. 1056 */ 1057 return folio_ref_count(folio) - folio_test_private(folio) == 1058 1 + folio_nr_pages(folio); 1059 } 1060 1061 /* 1062 * We detected a synchronous write error writing a folio out. Probably 1063 * -ENOSPC. We need to propagate that into the address_space for a subsequent 1064 * fsync(), msync() or close(). 1065 * 1066 * The tricky part is that after writepage we cannot touch the mapping: nothing 1067 * prevents it from being freed up. But we have a ref on the folio and once 1068 * that folio is locked, the mapping is pinned. 1069 * 1070 * We're allowed to run sleeping folio_lock() here because we know the caller has 1071 * __GFP_FS. 1072 */ 1073 static void handle_write_error(struct address_space *mapping, 1074 struct folio *folio, int error) 1075 { 1076 folio_lock(folio); 1077 if (folio_mapping(folio) == mapping) 1078 mapping_set_error(mapping, error); 1079 folio_unlock(folio); 1080 } 1081 1082 static bool skip_throttle_noprogress(pg_data_t *pgdat) 1083 { 1084 int reclaimable = 0, write_pending = 0; 1085 int i; 1086 1087 /* 1088 * If kswapd is disabled, reschedule if necessary but do not 1089 * throttle as the system is likely near OOM. 1090 */ 1091 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) 1092 return true; 1093 1094 /* 1095 * If there are a lot of dirty/writeback pages then do not 1096 * throttle as throttling will occur when the pages cycle 1097 * towards the end of the LRU if still under writeback. 1098 */ 1099 for (i = 0; i < MAX_NR_ZONES; i++) { 1100 struct zone *zone = pgdat->node_zones + i; 1101 1102 if (!managed_zone(zone)) 1103 continue; 1104 1105 reclaimable += zone_reclaimable_pages(zone); 1106 write_pending += zone_page_state_snapshot(zone, 1107 NR_ZONE_WRITE_PENDING); 1108 } 1109 if (2 * write_pending <= reclaimable) 1110 return true; 1111 1112 return false; 1113 } 1114 1115 void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason) 1116 { 1117 wait_queue_head_t *wqh = &pgdat->reclaim_wait[reason]; 1118 long timeout, ret; 1119 DEFINE_WAIT(wait); 1120 1121 /* 1122 * Do not throttle IO workers, kthreads other than kswapd or 1123 * workqueues. They may be required for reclaim to make 1124 * forward progress (e.g. journalling workqueues or kthreads). 1125 */ 1126 if (!current_is_kswapd() && 1127 current->flags & (PF_IO_WORKER|PF_KTHREAD)) { 1128 cond_resched(); 1129 return; 1130 } 1131 1132 /* 1133 * These figures are pulled out of thin air. 1134 * VMSCAN_THROTTLE_ISOLATED is a transient condition based on too many 1135 * parallel reclaimers which is a short-lived event so the timeout is 1136 * short. Failing to make progress or waiting on writeback are 1137 * potentially long-lived events so use a longer timeout. This is shaky 1138 * logic as a failure to make progress could be due to anything from 1139 * writeback to a slow device to excessive references pages at the tail 1140 * of the inactive LRU. 1141 */ 1142 switch(reason) { 1143 case VMSCAN_THROTTLE_WRITEBACK: 1144 timeout = HZ/10; 1145 1146 if (atomic_inc_return(&pgdat->nr_writeback_throttled) == 1) { 1147 WRITE_ONCE(pgdat->nr_reclaim_start, 1148 node_page_state(pgdat, NR_THROTTLED_WRITTEN)); 1149 } 1150 1151 break; 1152 case VMSCAN_THROTTLE_CONGESTED: 1153 fallthrough; 1154 case VMSCAN_THROTTLE_NOPROGRESS: 1155 if (skip_throttle_noprogress(pgdat)) { 1156 cond_resched(); 1157 return; 1158 } 1159 1160 timeout = 1; 1161 1162 break; 1163 case VMSCAN_THROTTLE_ISOLATED: 1164 timeout = HZ/50; 1165 break; 1166 default: 1167 WARN_ON_ONCE(1); 1168 timeout = HZ; 1169 break; 1170 } 1171 1172 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); 1173 ret = schedule_timeout(timeout); 1174 finish_wait(wqh, &wait); 1175 1176 if (reason == VMSCAN_THROTTLE_WRITEBACK) 1177 atomic_dec(&pgdat->nr_writeback_throttled); 1178 1179 trace_mm_vmscan_throttled(pgdat->node_id, jiffies_to_usecs(timeout), 1180 jiffies_to_usecs(timeout - ret), 1181 reason); 1182 } 1183 1184 /* 1185 * Account for pages written if tasks are throttled waiting on dirty 1186 * pages to clean. If enough pages have been cleaned since throttling 1187 * started then wakeup the throttled tasks. 1188 */ 1189 void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, 1190 int nr_throttled) 1191 { 1192 unsigned long nr_written; 1193 1194 node_stat_add_folio(folio, NR_THROTTLED_WRITTEN); 1195 1196 /* 1197 * This is an inaccurate read as the per-cpu deltas may not 1198 * be synchronised. However, given that the system is 1199 * writeback throttled, it is not worth taking the penalty 1200 * of getting an accurate count. At worst, the throttle 1201 * timeout guarantees forward progress. 1202 */ 1203 nr_written = node_page_state(pgdat, NR_THROTTLED_WRITTEN) - 1204 READ_ONCE(pgdat->nr_reclaim_start); 1205 1206 if (nr_written > SWAP_CLUSTER_MAX * nr_throttled) 1207 wake_up(&pgdat->reclaim_wait[VMSCAN_THROTTLE_WRITEBACK]); 1208 } 1209 1210 /* possible outcome of pageout() */ 1211 typedef enum { 1212 /* failed to write page out, page is locked */ 1213 PAGE_KEEP, 1214 /* move page to the active list, page is locked */ 1215 PAGE_ACTIVATE, 1216 /* page has been sent to the disk successfully, page is unlocked */ 1217 PAGE_SUCCESS, 1218 /* page is clean and locked */ 1219 PAGE_CLEAN, 1220 } pageout_t; 1221 1222 /* 1223 * pageout is called by shrink_page_list() for each dirty page. 1224 * Calls ->writepage(). 1225 */ 1226 static pageout_t pageout(struct folio *folio, struct address_space *mapping, 1227 struct swap_iocb **plug) 1228 { 1229 /* 1230 * If the folio is dirty, only perform writeback if that write 1231 * will be non-blocking. To prevent this allocation from being 1232 * stalled by pagecache activity. But note that there may be 1233 * stalls if we need to run get_block(). We could test 1234 * PagePrivate for that. 1235 * 1236 * If this process is currently in __generic_file_write_iter() against 1237 * this folio's queue, we can perform writeback even if that 1238 * will block. 1239 * 1240 * If the folio is swapcache, write it back even if that would 1241 * block, for some throttling. This happens by accident, because 1242 * swap_backing_dev_info is bust: it doesn't reflect the 1243 * congestion state of the swapdevs. Easy to fix, if needed. 1244 */ 1245 if (!is_page_cache_freeable(folio)) 1246 return PAGE_KEEP; 1247 if (!mapping) { 1248 /* 1249 * Some data journaling orphaned folios can have 1250 * folio->mapping == NULL while being dirty with clean buffers. 1251 */ 1252 if (folio_test_private(folio)) { 1253 if (try_to_free_buffers(folio)) { 1254 folio_clear_dirty(folio); 1255 pr_info("%s: orphaned folio\n", __func__); 1256 return PAGE_CLEAN; 1257 } 1258 } 1259 return PAGE_KEEP; 1260 } 1261 if (mapping->a_ops->writepage == NULL) 1262 return PAGE_ACTIVATE; 1263 1264 if (folio_clear_dirty_for_io(folio)) { 1265 int res; 1266 struct writeback_control wbc = { 1267 .sync_mode = WB_SYNC_NONE, 1268 .nr_to_write = SWAP_CLUSTER_MAX, 1269 .range_start = 0, 1270 .range_end = LLONG_MAX, 1271 .for_reclaim = 1, 1272 .swap_plug = plug, 1273 }; 1274 1275 folio_set_reclaim(folio); 1276 res = mapping->a_ops->writepage(&folio->page, &wbc); 1277 if (res < 0) 1278 handle_write_error(mapping, folio, res); 1279 if (res == AOP_WRITEPAGE_ACTIVATE) { 1280 folio_clear_reclaim(folio); 1281 return PAGE_ACTIVATE; 1282 } 1283 1284 if (!folio_test_writeback(folio)) { 1285 /* synchronous write or broken a_ops? */ 1286 folio_clear_reclaim(folio); 1287 } 1288 trace_mm_vmscan_write_folio(folio); 1289 node_stat_add_folio(folio, NR_VMSCAN_WRITE); 1290 return PAGE_SUCCESS; 1291 } 1292 1293 return PAGE_CLEAN; 1294 } 1295 1296 /* 1297 * Same as remove_mapping, but if the page is removed from the mapping, it 1298 * gets returned with a refcount of 0. 1299 */ 1300 static int __remove_mapping(struct address_space *mapping, struct folio *folio, 1301 bool reclaimed, struct mem_cgroup *target_memcg) 1302 { 1303 int refcount; 1304 void *shadow = NULL; 1305 1306 BUG_ON(!folio_test_locked(folio)); 1307 BUG_ON(mapping != folio_mapping(folio)); 1308 1309 if (!folio_test_swapcache(folio)) 1310 spin_lock(&mapping->host->i_lock); 1311 xa_lock_irq(&mapping->i_pages); 1312 /* 1313 * The non racy check for a busy page. 1314 * 1315 * Must be careful with the order of the tests. When someone has 1316 * a ref to the page, it may be possible that they dirty it then 1317 * drop the reference. So if PageDirty is tested before page_count 1318 * here, then the following race may occur: 1319 * 1320 * get_user_pages(&page); 1321 * [user mapping goes away] 1322 * write_to(page); 1323 * !PageDirty(page) [good] 1324 * SetPageDirty(page); 1325 * put_page(page); 1326 * !page_count(page) [good, discard it] 1327 * 1328 * [oops, our write_to data is lost] 1329 * 1330 * Reversing the order of the tests ensures such a situation cannot 1331 * escape unnoticed. The smp_rmb is needed to ensure the page->flags 1332 * load is not satisfied before that of page->_refcount. 1333 * 1334 * Note that if SetPageDirty is always performed via set_page_dirty, 1335 * and thus under the i_pages lock, then this ordering is not required. 1336 */ 1337 refcount = 1 + folio_nr_pages(folio); 1338 if (!folio_ref_freeze(folio, refcount)) 1339 goto cannot_free; 1340 /* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */ 1341 if (unlikely(folio_test_dirty(folio))) { 1342 folio_ref_unfreeze(folio, refcount); 1343 goto cannot_free; 1344 } 1345 1346 if (folio_test_swapcache(folio)) { 1347 swp_entry_t swap = folio_swap_entry(folio); 1348 1349 /* get a shadow entry before mem_cgroup_swapout() clears folio_memcg() */ 1350 if (reclaimed && !mapping_exiting(mapping)) 1351 shadow = workingset_eviction(folio, target_memcg); 1352 mem_cgroup_swapout(folio, swap); 1353 __delete_from_swap_cache(folio, swap, shadow); 1354 xa_unlock_irq(&mapping->i_pages); 1355 put_swap_page(&folio->page, swap); 1356 } else { 1357 void (*free_folio)(struct folio *); 1358 1359 free_folio = mapping->a_ops->free_folio; 1360 /* 1361 * Remember a shadow entry for reclaimed file cache in 1362 * order to detect refaults, thus thrashing, later on. 1363 * 1364 * But don't store shadows in an address space that is 1365 * already exiting. This is not just an optimization, 1366 * inode reclaim needs to empty out the radix tree or 1367 * the nodes are lost. Don't plant shadows behind its 1368 * back. 1369 * 1370 * We also don't store shadows for DAX mappings because the 1371 * only page cache pages found in these are zero pages 1372 * covering holes, and because we don't want to mix DAX 1373 * exceptional entries and shadow exceptional entries in the 1374 * same address_space. 1375 */ 1376 if (reclaimed && folio_is_file_lru(folio) && 1377 !mapping_exiting(mapping) && !dax_mapping(mapping)) 1378 shadow = workingset_eviction(folio, target_memcg); 1379 __filemap_remove_folio(folio, shadow); 1380 xa_unlock_irq(&mapping->i_pages); 1381 if (mapping_shrinkable(mapping)) 1382 inode_add_lru(mapping->host); 1383 spin_unlock(&mapping->host->i_lock); 1384 1385 if (free_folio) 1386 free_folio(folio); 1387 } 1388 1389 return 1; 1390 1391 cannot_free: 1392 xa_unlock_irq(&mapping->i_pages); 1393 if (!folio_test_swapcache(folio)) 1394 spin_unlock(&mapping->host->i_lock); 1395 return 0; 1396 } 1397 1398 /** 1399 * remove_mapping() - Attempt to remove a folio from its mapping. 1400 * @mapping: The address space. 1401 * @folio: The folio to remove. 1402 * 1403 * If the folio is dirty, under writeback or if someone else has a ref 1404 * on it, removal will fail. 1405 * Return: The number of pages removed from the mapping. 0 if the folio 1406 * could not be removed. 1407 * Context: The caller should have a single refcount on the folio and 1408 * hold its lock. 1409 */ 1410 long remove_mapping(struct address_space *mapping, struct folio *folio) 1411 { 1412 if (__remove_mapping(mapping, folio, false, NULL)) { 1413 /* 1414 * Unfreezing the refcount with 1 effectively 1415 * drops the pagecache ref for us without requiring another 1416 * atomic operation. 1417 */ 1418 folio_ref_unfreeze(folio, 1); 1419 return folio_nr_pages(folio); 1420 } 1421 return 0; 1422 } 1423 1424 /** 1425 * folio_putback_lru - Put previously isolated folio onto appropriate LRU list. 1426 * @folio: Folio to be returned to an LRU list. 1427 * 1428 * Add previously isolated @folio to appropriate LRU list. 1429 * The folio may still be unevictable for other reasons. 1430 * 1431 * Context: lru_lock must not be held, interrupts must be enabled. 1432 */ 1433 void folio_putback_lru(struct folio *folio) 1434 { 1435 folio_add_lru(folio); 1436 folio_put(folio); /* drop ref from isolate */ 1437 } 1438 1439 enum page_references { 1440 PAGEREF_RECLAIM, 1441 PAGEREF_RECLAIM_CLEAN, 1442 PAGEREF_KEEP, 1443 PAGEREF_ACTIVATE, 1444 }; 1445 1446 static enum page_references folio_check_references(struct folio *folio, 1447 struct scan_control *sc) 1448 { 1449 int referenced_ptes, referenced_folio; 1450 unsigned long vm_flags; 1451 1452 referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup, 1453 &vm_flags); 1454 referenced_folio = folio_test_clear_referenced(folio); 1455 1456 /* 1457 * The supposedly reclaimable folio was found to be in a VM_LOCKED vma. 1458 * Let the folio, now marked Mlocked, be moved to the unevictable list. 1459 */ 1460 if (vm_flags & VM_LOCKED) 1461 return PAGEREF_ACTIVATE; 1462 1463 /* rmap lock contention: rotate */ 1464 if (referenced_ptes == -1) 1465 return PAGEREF_KEEP; 1466 1467 if (referenced_ptes) { 1468 /* 1469 * All mapped folios start out with page table 1470 * references from the instantiating fault, so we need 1471 * to look twice if a mapped file/anon folio is used more 1472 * than once. 1473 * 1474 * Mark it and spare it for another trip around the 1475 * inactive list. Another page table reference will 1476 * lead to its activation. 1477 * 1478 * Note: the mark is set for activated folios as well 1479 * so that recently deactivated but used folios are 1480 * quickly recovered. 1481 */ 1482 folio_set_referenced(folio); 1483 1484 if (referenced_folio || referenced_ptes > 1) 1485 return PAGEREF_ACTIVATE; 1486 1487 /* 1488 * Activate file-backed executable folios after first usage. 1489 */ 1490 if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) 1491 return PAGEREF_ACTIVATE; 1492 1493 return PAGEREF_KEEP; 1494 } 1495 1496 /* Reclaim if clean, defer dirty folios to writeback */ 1497 if (referenced_folio && folio_is_file_lru(folio)) 1498 return PAGEREF_RECLAIM_CLEAN; 1499 1500 return PAGEREF_RECLAIM; 1501 } 1502 1503 /* Check if a page is dirty or under writeback */ 1504 static void folio_check_dirty_writeback(struct folio *folio, 1505 bool *dirty, bool *writeback) 1506 { 1507 struct address_space *mapping; 1508 1509 /* 1510 * Anonymous pages are not handled by flushers and must be written 1511 * from reclaim context. Do not stall reclaim based on them. 1512 * MADV_FREE anonymous pages are put into inactive file list too. 1513 * They could be mistakenly treated as file lru. So further anon 1514 * test is needed. 1515 */ 1516 if (!folio_is_file_lru(folio) || 1517 (folio_test_anon(folio) && !folio_test_swapbacked(folio))) { 1518 *dirty = false; 1519 *writeback = false; 1520 return; 1521 } 1522 1523 /* By default assume that the folio flags are accurate */ 1524 *dirty = folio_test_dirty(folio); 1525 *writeback = folio_test_writeback(folio); 1526 1527 /* Verify dirty/writeback state if the filesystem supports it */ 1528 if (!folio_test_private(folio)) 1529 return; 1530 1531 mapping = folio_mapping(folio); 1532 if (mapping && mapping->a_ops->is_dirty_writeback) 1533 mapping->a_ops->is_dirty_writeback(folio, dirty, writeback); 1534 } 1535 1536 static struct page *alloc_demote_page(struct page *page, unsigned long private) 1537 { 1538 struct page *target_page; 1539 nodemask_t *allowed_mask; 1540 struct migration_target_control *mtc; 1541 1542 mtc = (struct migration_target_control *)private; 1543 1544 allowed_mask = mtc->nmask; 1545 /* 1546 * make sure we allocate from the target node first also trying to 1547 * demote or reclaim pages from the target node via kswapd if we are 1548 * low on free memory on target node. If we don't do this and if 1549 * we have free memory on the slower(lower) memtier, we would start 1550 * allocating pages from slower(lower) memory tiers without even forcing 1551 * a demotion of cold pages from the target memtier. This can result 1552 * in the kernel placing hot pages in slower(lower) memory tiers. 1553 */ 1554 mtc->nmask = NULL; 1555 mtc->gfp_mask |= __GFP_THISNODE; 1556 target_page = alloc_migration_target(page, (unsigned long)mtc); 1557 if (target_page) 1558 return target_page; 1559 1560 mtc->gfp_mask &= ~__GFP_THISNODE; 1561 mtc->nmask = allowed_mask; 1562 1563 return alloc_migration_target(page, (unsigned long)mtc); 1564 } 1565 1566 /* 1567 * Take pages on @demote_list and attempt to demote them to 1568 * another node. Pages which are not demoted are left on 1569 * @demote_pages. 1570 */ 1571 static unsigned int demote_page_list(struct list_head *demote_pages, 1572 struct pglist_data *pgdat) 1573 { 1574 int target_nid = next_demotion_node(pgdat->node_id); 1575 unsigned int nr_succeeded; 1576 nodemask_t allowed_mask; 1577 1578 struct migration_target_control mtc = { 1579 /* 1580 * Allocate from 'node', or fail quickly and quietly. 1581 * When this happens, 'page' will likely just be discarded 1582 * instead of migrated. 1583 */ 1584 .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN | 1585 __GFP_NOMEMALLOC | GFP_NOWAIT, 1586 .nid = target_nid, 1587 .nmask = &allowed_mask 1588 }; 1589 1590 if (list_empty(demote_pages)) 1591 return 0; 1592 1593 if (target_nid == NUMA_NO_NODE) 1594 return 0; 1595 1596 node_get_allowed_targets(pgdat, &allowed_mask); 1597 1598 /* Demotion ignores all cpuset and mempolicy settings */ 1599 migrate_pages(demote_pages, alloc_demote_page, NULL, 1600 (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION, 1601 &nr_succeeded); 1602 1603 if (current_is_kswapd()) 1604 __count_vm_events(PGDEMOTE_KSWAPD, nr_succeeded); 1605 else 1606 __count_vm_events(PGDEMOTE_DIRECT, nr_succeeded); 1607 1608 return nr_succeeded; 1609 } 1610 1611 static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask) 1612 { 1613 if (gfp_mask & __GFP_FS) 1614 return true; 1615 if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO)) 1616 return false; 1617 /* 1618 * We can "enter_fs" for swap-cache with only __GFP_IO 1619 * providing this isn't SWP_FS_OPS. 1620 * ->flags can be updated non-atomicially (scan_swap_map_slots), 1621 * but that will never affect SWP_FS_OPS, so the data_race 1622 * is safe. 1623 */ 1624 return !data_race(folio_swap_flags(folio) & SWP_FS_OPS); 1625 } 1626 1627 /* 1628 * shrink_page_list() returns the number of reclaimed pages 1629 */ 1630 static unsigned int shrink_page_list(struct list_head *page_list, 1631 struct pglist_data *pgdat, 1632 struct scan_control *sc, 1633 struct reclaim_stat *stat, 1634 bool ignore_references) 1635 { 1636 LIST_HEAD(ret_pages); 1637 LIST_HEAD(free_pages); 1638 LIST_HEAD(demote_pages); 1639 unsigned int nr_reclaimed = 0; 1640 unsigned int pgactivate = 0; 1641 bool do_demote_pass; 1642 struct swap_iocb *plug = NULL; 1643 1644 memset(stat, 0, sizeof(*stat)); 1645 cond_resched(); 1646 do_demote_pass = can_demote(pgdat->node_id, sc); 1647 1648 retry: 1649 while (!list_empty(page_list)) { 1650 struct address_space *mapping; 1651 struct folio *folio; 1652 enum page_references references = PAGEREF_RECLAIM; 1653 bool dirty, writeback; 1654 unsigned int nr_pages; 1655 1656 cond_resched(); 1657 1658 folio = lru_to_folio(page_list); 1659 list_del(&folio->lru); 1660 1661 if (!folio_trylock(folio)) 1662 goto keep; 1663 1664 VM_BUG_ON_FOLIO(folio_test_active(folio), folio); 1665 1666 nr_pages = folio_nr_pages(folio); 1667 1668 /* Account the number of base pages */ 1669 sc->nr_scanned += nr_pages; 1670 1671 if (unlikely(!folio_evictable(folio))) 1672 goto activate_locked; 1673 1674 if (!sc->may_unmap && folio_mapped(folio)) 1675 goto keep_locked; 1676 1677 /* folio_update_gen() tried to promote this page? */ 1678 if (lru_gen_enabled() && !ignore_references && 1679 folio_mapped(folio) && folio_test_referenced(folio)) 1680 goto keep_locked; 1681 1682 /* 1683 * The number of dirty pages determines if a node is marked 1684 * reclaim_congested. kswapd will stall and start writing 1685 * folios if the tail of the LRU is all dirty unqueued folios. 1686 */ 1687 folio_check_dirty_writeback(folio, &dirty, &writeback); 1688 if (dirty || writeback) 1689 stat->nr_dirty += nr_pages; 1690 1691 if (dirty && !writeback) 1692 stat->nr_unqueued_dirty += nr_pages; 1693 1694 /* 1695 * Treat this folio as congested if folios are cycling 1696 * through the LRU so quickly that the folios marked 1697 * for immediate reclaim are making it to the end of 1698 * the LRU a second time. 1699 */ 1700 if (writeback && folio_test_reclaim(folio)) 1701 stat->nr_congested += nr_pages; 1702 1703 /* 1704 * If a folio at the tail of the LRU is under writeback, there 1705 * are three cases to consider. 1706 * 1707 * 1) If reclaim is encountering an excessive number 1708 * of folios under writeback and this folio has both 1709 * the writeback and reclaim flags set, then it 1710 * indicates that folios are being queued for I/O but 1711 * are being recycled through the LRU before the I/O 1712 * can complete. Waiting on the folio itself risks an 1713 * indefinite stall if it is impossible to writeback 1714 * the folio due to I/O error or disconnected storage 1715 * so instead note that the LRU is being scanned too 1716 * quickly and the caller can stall after the folio 1717 * list has been processed. 1718 * 1719 * 2) Global or new memcg reclaim encounters a folio that is 1720 * not marked for immediate reclaim, or the caller does not 1721 * have __GFP_FS (or __GFP_IO if it's simply going to swap, 1722 * not to fs). In this case mark the folio for immediate 1723 * reclaim and continue scanning. 1724 * 1725 * Require may_enter_fs() because we would wait on fs, which 1726 * may not have submitted I/O yet. And the loop driver might 1727 * enter reclaim, and deadlock if it waits on a folio for 1728 * which it is needed to do the write (loop masks off 1729 * __GFP_IO|__GFP_FS for this reason); but more thought 1730 * would probably show more reasons. 1731 * 1732 * 3) Legacy memcg encounters a folio that already has the 1733 * reclaim flag set. memcg does not have any dirty folio 1734 * throttling so we could easily OOM just because too many 1735 * folios are in writeback and there is nothing else to 1736 * reclaim. Wait for the writeback to complete. 1737 * 1738 * In cases 1) and 2) we activate the folios to get them out of 1739 * the way while we continue scanning for clean folios on the 1740 * inactive list and refilling from the active list. The 1741 * observation here is that waiting for disk writes is more 1742 * expensive than potentially causing reloads down the line. 1743 * Since they're marked for immediate reclaim, they won't put 1744 * memory pressure on the cache working set any longer than it 1745 * takes to write them to disk. 1746 */ 1747 if (folio_test_writeback(folio)) { 1748 /* Case 1 above */ 1749 if (current_is_kswapd() && 1750 folio_test_reclaim(folio) && 1751 test_bit(PGDAT_WRITEBACK, &pgdat->flags)) { 1752 stat->nr_immediate += nr_pages; 1753 goto activate_locked; 1754 1755 /* Case 2 above */ 1756 } else if (writeback_throttling_sane(sc) || 1757 !folio_test_reclaim(folio) || 1758 !may_enter_fs(folio, sc->gfp_mask)) { 1759 /* 1760 * This is slightly racy - 1761 * folio_end_writeback() might have 1762 * just cleared the reclaim flag, then 1763 * setting the reclaim flag here ends up 1764 * interpreted as the readahead flag - but 1765 * that does not matter enough to care. 1766 * What we do want is for this folio to 1767 * have the reclaim flag set next time 1768 * memcg reclaim reaches the tests above, 1769 * so it will then wait for writeback to 1770 * avoid OOM; and it's also appropriate 1771 * in global reclaim. 1772 */ 1773 folio_set_reclaim(folio); 1774 stat->nr_writeback += nr_pages; 1775 goto activate_locked; 1776 1777 /* Case 3 above */ 1778 } else { 1779 folio_unlock(folio); 1780 folio_wait_writeback(folio); 1781 /* then go back and try same folio again */ 1782 list_add_tail(&folio->lru, page_list); 1783 continue; 1784 } 1785 } 1786 1787 if (!ignore_references) 1788 references = folio_check_references(folio, sc); 1789 1790 switch (references) { 1791 case PAGEREF_ACTIVATE: 1792 goto activate_locked; 1793 case PAGEREF_KEEP: 1794 stat->nr_ref_keep += nr_pages; 1795 goto keep_locked; 1796 case PAGEREF_RECLAIM: 1797 case PAGEREF_RECLAIM_CLEAN: 1798 ; /* try to reclaim the folio below */ 1799 } 1800 1801 /* 1802 * Before reclaiming the folio, try to relocate 1803 * its contents to another node. 1804 */ 1805 if (do_demote_pass && 1806 (thp_migration_supported() || !folio_test_large(folio))) { 1807 list_add(&folio->lru, &demote_pages); 1808 folio_unlock(folio); 1809 continue; 1810 } 1811 1812 /* 1813 * Anonymous process memory has backing store? 1814 * Try to allocate it some swap space here. 1815 * Lazyfree folio could be freed directly 1816 */ 1817 if (folio_test_anon(folio) && folio_test_swapbacked(folio)) { 1818 if (!folio_test_swapcache(folio)) { 1819 if (!(sc->gfp_mask & __GFP_IO)) 1820 goto keep_locked; 1821 if (folio_maybe_dma_pinned(folio)) 1822 goto keep_locked; 1823 if (folio_test_large(folio)) { 1824 /* cannot split folio, skip it */ 1825 if (!can_split_folio(folio, NULL)) 1826 goto activate_locked; 1827 /* 1828 * Split folios without a PMD map right 1829 * away. Chances are some or all of the 1830 * tail pages can be freed without IO. 1831 */ 1832 if (!folio_entire_mapcount(folio) && 1833 split_folio_to_list(folio, 1834 page_list)) 1835 goto activate_locked; 1836 } 1837 if (!add_to_swap(folio)) { 1838 if (!folio_test_large(folio)) 1839 goto activate_locked_split; 1840 /* Fallback to swap normal pages */ 1841 if (split_folio_to_list(folio, 1842 page_list)) 1843 goto activate_locked; 1844 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1845 count_vm_event(THP_SWPOUT_FALLBACK); 1846 #endif 1847 if (!add_to_swap(folio)) 1848 goto activate_locked_split; 1849 } 1850 } 1851 } else if (folio_test_swapbacked(folio) && 1852 folio_test_large(folio)) { 1853 /* Split shmem folio */ 1854 if (split_folio_to_list(folio, page_list)) 1855 goto keep_locked; 1856 } 1857 1858 /* 1859 * If the folio was split above, the tail pages will make 1860 * their own pass through this function and be accounted 1861 * then. 1862 */ 1863 if ((nr_pages > 1) && !folio_test_large(folio)) { 1864 sc->nr_scanned -= (nr_pages - 1); 1865 nr_pages = 1; 1866 } 1867 1868 /* 1869 * The folio is mapped into the page tables of one or more 1870 * processes. Try to unmap it here. 1871 */ 1872 if (folio_mapped(folio)) { 1873 enum ttu_flags flags = TTU_BATCH_FLUSH; 1874 bool was_swapbacked = folio_test_swapbacked(folio); 1875 1876 if (folio_test_pmd_mappable(folio)) 1877 flags |= TTU_SPLIT_HUGE_PMD; 1878 1879 try_to_unmap(folio, flags); 1880 if (folio_mapped(folio)) { 1881 stat->nr_unmap_fail += nr_pages; 1882 if (!was_swapbacked && 1883 folio_test_swapbacked(folio)) 1884 stat->nr_lazyfree_fail += nr_pages; 1885 goto activate_locked; 1886 } 1887 } 1888 1889 mapping = folio_mapping(folio); 1890 if (folio_test_dirty(folio)) { 1891 /* 1892 * Only kswapd can writeback filesystem folios 1893 * to avoid risk of stack overflow. But avoid 1894 * injecting inefficient single-folio I/O into 1895 * flusher writeback as much as possible: only 1896 * write folios when we've encountered many 1897 * dirty folios, and when we've already scanned 1898 * the rest of the LRU for clean folios and see 1899 * the same dirty folios again (with the reclaim 1900 * flag set). 1901 */ 1902 if (folio_is_file_lru(folio) && 1903 (!current_is_kswapd() || 1904 !folio_test_reclaim(folio) || 1905 !test_bit(PGDAT_DIRTY, &pgdat->flags))) { 1906 /* 1907 * Immediately reclaim when written back. 1908 * Similar in principle to deactivate_page() 1909 * except we already have the folio isolated 1910 * and know it's dirty 1911 */ 1912 node_stat_mod_folio(folio, NR_VMSCAN_IMMEDIATE, 1913 nr_pages); 1914 folio_set_reclaim(folio); 1915 1916 goto activate_locked; 1917 } 1918 1919 if (references == PAGEREF_RECLAIM_CLEAN) 1920 goto keep_locked; 1921 if (!may_enter_fs(folio, sc->gfp_mask)) 1922 goto keep_locked; 1923 if (!sc->may_writepage) 1924 goto keep_locked; 1925 1926 /* 1927 * Folio is dirty. Flush the TLB if a writable entry 1928 * potentially exists to avoid CPU writes after I/O 1929 * starts and then write it out here. 1930 */ 1931 try_to_unmap_flush_dirty(); 1932 switch (pageout(folio, mapping, &plug)) { 1933 case PAGE_KEEP: 1934 goto keep_locked; 1935 case PAGE_ACTIVATE: 1936 goto activate_locked; 1937 case PAGE_SUCCESS: 1938 stat->nr_pageout += nr_pages; 1939 1940 if (folio_test_writeback(folio)) 1941 goto keep; 1942 if (folio_test_dirty(folio)) 1943 goto keep; 1944 1945 /* 1946 * A synchronous write - probably a ramdisk. Go 1947 * ahead and try to reclaim the folio. 1948 */ 1949 if (!folio_trylock(folio)) 1950 goto keep; 1951 if (folio_test_dirty(folio) || 1952 folio_test_writeback(folio)) 1953 goto keep_locked; 1954 mapping = folio_mapping(folio); 1955 fallthrough; 1956 case PAGE_CLEAN: 1957 ; /* try to free the folio below */ 1958 } 1959 } 1960 1961 /* 1962 * If the folio has buffers, try to free the buffer 1963 * mappings associated with this folio. If we succeed 1964 * we try to free the folio as well. 1965 * 1966 * We do this even if the folio is dirty. 1967 * filemap_release_folio() does not perform I/O, but it 1968 * is possible for a folio to have the dirty flag set, 1969 * but it is actually clean (all its buffers are clean). 1970 * This happens if the buffers were written out directly, 1971 * with submit_bh(). ext3 will do this, as well as 1972 * the blockdev mapping. filemap_release_folio() will 1973 * discover that cleanness and will drop the buffers 1974 * and mark the folio clean - it can be freed. 1975 * 1976 * Rarely, folios can have buffers and no ->mapping. 1977 * These are the folios which were not successfully 1978 * invalidated in truncate_cleanup_folio(). We try to 1979 * drop those buffers here and if that worked, and the 1980 * folio is no longer mapped into process address space 1981 * (refcount == 1) it can be freed. Otherwise, leave 1982 * the folio on the LRU so it is swappable. 1983 */ 1984 if (folio_has_private(folio)) { 1985 if (!filemap_release_folio(folio, sc->gfp_mask)) 1986 goto activate_locked; 1987 if (!mapping && folio_ref_count(folio) == 1) { 1988 folio_unlock(folio); 1989 if (folio_put_testzero(folio)) 1990 goto free_it; 1991 else { 1992 /* 1993 * rare race with speculative reference. 1994 * the speculative reference will free 1995 * this folio shortly, so we may 1996 * increment nr_reclaimed here (and 1997 * leave it off the LRU). 1998 */ 1999 nr_reclaimed += nr_pages; 2000 continue; 2001 } 2002 } 2003 } 2004 2005 if (folio_test_anon(folio) && !folio_test_swapbacked(folio)) { 2006 /* follow __remove_mapping for reference */ 2007 if (!folio_ref_freeze(folio, 1)) 2008 goto keep_locked; 2009 /* 2010 * The folio has only one reference left, which is 2011 * from the isolation. After the caller puts the 2012 * folio back on the lru and drops the reference, the 2013 * folio will be freed anyway. It doesn't matter 2014 * which lru it goes on. So we don't bother checking 2015 * the dirty flag here. 2016 */ 2017 count_vm_events(PGLAZYFREED, nr_pages); 2018 count_memcg_folio_events(folio, PGLAZYFREED, nr_pages); 2019 } else if (!mapping || !__remove_mapping(mapping, folio, true, 2020 sc->target_mem_cgroup)) 2021 goto keep_locked; 2022 2023 folio_unlock(folio); 2024 free_it: 2025 /* 2026 * Folio may get swapped out as a whole, need to account 2027 * all pages in it. 2028 */ 2029 nr_reclaimed += nr_pages; 2030 2031 /* 2032 * Is there need to periodically free_page_list? It would 2033 * appear not as the counts should be low 2034 */ 2035 if (unlikely(folio_test_large(folio))) 2036 destroy_large_folio(folio); 2037 else 2038 list_add(&folio->lru, &free_pages); 2039 continue; 2040 2041 activate_locked_split: 2042 /* 2043 * The tail pages that are failed to add into swap cache 2044 * reach here. Fixup nr_scanned and nr_pages. 2045 */ 2046 if (nr_pages > 1) { 2047 sc->nr_scanned -= (nr_pages - 1); 2048 nr_pages = 1; 2049 } 2050 activate_locked: 2051 /* Not a candidate for swapping, so reclaim swap space. */ 2052 if (folio_test_swapcache(folio) && 2053 (mem_cgroup_swap_full(&folio->page) || 2054 folio_test_mlocked(folio))) 2055 try_to_free_swap(&folio->page); 2056 VM_BUG_ON_FOLIO(folio_test_active(folio), folio); 2057 if (!folio_test_mlocked(folio)) { 2058 int type = folio_is_file_lru(folio); 2059 folio_set_active(folio); 2060 stat->nr_activate[type] += nr_pages; 2061 count_memcg_folio_events(folio, PGACTIVATE, nr_pages); 2062 } 2063 keep_locked: 2064 folio_unlock(folio); 2065 keep: 2066 list_add(&folio->lru, &ret_pages); 2067 VM_BUG_ON_FOLIO(folio_test_lru(folio) || 2068 folio_test_unevictable(folio), folio); 2069 } 2070 /* 'page_list' is always empty here */ 2071 2072 /* Migrate folios selected for demotion */ 2073 nr_reclaimed += demote_page_list(&demote_pages, pgdat); 2074 /* Folios that could not be demoted are still in @demote_pages */ 2075 if (!list_empty(&demote_pages)) { 2076 /* Folios which weren't demoted go back on @page_list for retry: */ 2077 list_splice_init(&demote_pages, page_list); 2078 do_demote_pass = false; 2079 goto retry; 2080 } 2081 2082 pgactivate = stat->nr_activate[0] + stat->nr_activate[1]; 2083 2084 mem_cgroup_uncharge_list(&free_pages); 2085 try_to_unmap_flush(); 2086 free_unref_page_list(&free_pages); 2087 2088 list_splice(&ret_pages, page_list); 2089 count_vm_events(PGACTIVATE, pgactivate); 2090 2091 if (plug) 2092 swap_write_unplug(plug); 2093 return nr_reclaimed; 2094 } 2095 2096 unsigned int reclaim_clean_pages_from_list(struct zone *zone, 2097 struct list_head *folio_list) 2098 { 2099 struct scan_control sc = { 2100 .gfp_mask = GFP_KERNEL, 2101 .may_unmap = 1, 2102 }; 2103 struct reclaim_stat stat; 2104 unsigned int nr_reclaimed; 2105 struct folio *folio, *next; 2106 LIST_HEAD(clean_folios); 2107 unsigned int noreclaim_flag; 2108 2109 list_for_each_entry_safe(folio, next, folio_list, lru) { 2110 if (!folio_test_hugetlb(folio) && folio_is_file_lru(folio) && 2111 !folio_test_dirty(folio) && !__folio_test_movable(folio) && 2112 !folio_test_unevictable(folio)) { 2113 folio_clear_active(folio); 2114 list_move(&folio->lru, &clean_folios); 2115 } 2116 } 2117 2118 /* 2119 * We should be safe here since we are only dealing with file pages and 2120 * we are not kswapd and therefore cannot write dirty file pages. But 2121 * call memalloc_noreclaim_save() anyway, just in case these conditions 2122 * change in the future. 2123 */ 2124 noreclaim_flag = memalloc_noreclaim_save(); 2125 nr_reclaimed = shrink_page_list(&clean_folios, zone->zone_pgdat, &sc, 2126 &stat, true); 2127 memalloc_noreclaim_restore(noreclaim_flag); 2128 2129 list_splice(&clean_folios, folio_list); 2130 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, 2131 -(long)nr_reclaimed); 2132 /* 2133 * Since lazyfree pages are isolated from file LRU from the beginning, 2134 * they will rotate back to anonymous LRU in the end if it failed to 2135 * discard so isolated count will be mismatched. 2136 * Compensate the isolated count for both LRU lists. 2137 */ 2138 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON, 2139 stat.nr_lazyfree_fail); 2140 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, 2141 -(long)stat.nr_lazyfree_fail); 2142 return nr_reclaimed; 2143 } 2144 2145 /* 2146 * Update LRU sizes after isolating pages. The LRU size updates must 2147 * be complete before mem_cgroup_update_lru_size due to a sanity check. 2148 */ 2149 static __always_inline void update_lru_sizes(struct lruvec *lruvec, 2150 enum lru_list lru, unsigned long *nr_zone_taken) 2151 { 2152 int zid; 2153 2154 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 2155 if (!nr_zone_taken[zid]) 2156 continue; 2157 2158 update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); 2159 } 2160 2161 } 2162 2163 /* 2164 * Isolating page from the lruvec to fill in @dst list by nr_to_scan times. 2165 * 2166 * lruvec->lru_lock is heavily contended. Some of the functions that 2167 * shrink the lists perform better by taking out a batch of pages 2168 * and working on them outside the LRU lock. 2169 * 2170 * For pagecache intensive workloads, this function is the hottest 2171 * spot in the kernel (apart from copy_*_user functions). 2172 * 2173 * Lru_lock must be held before calling this function. 2174 * 2175 * @nr_to_scan: The number of eligible pages to look through on the list. 2176 * @lruvec: The LRU vector to pull pages from. 2177 * @dst: The temp list to put pages on to. 2178 * @nr_scanned: The number of pages that were scanned. 2179 * @sc: The scan_control struct for this reclaim session 2180 * @lru: LRU list id for isolating 2181 * 2182 * returns how many pages were moved onto *@dst. 2183 */ 2184 static unsigned long isolate_lru_pages(unsigned long nr_to_scan, 2185 struct lruvec *lruvec, struct list_head *dst, 2186 unsigned long *nr_scanned, struct scan_control *sc, 2187 enum lru_list lru) 2188 { 2189 struct list_head *src = &lruvec->lists[lru]; 2190 unsigned long nr_taken = 0; 2191 unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 }; 2192 unsigned long nr_skipped[MAX_NR_ZONES] = { 0, }; 2193 unsigned long skipped = 0; 2194 unsigned long scan, total_scan, nr_pages; 2195 LIST_HEAD(folios_skipped); 2196 2197 total_scan = 0; 2198 scan = 0; 2199 while (scan < nr_to_scan && !list_empty(src)) { 2200 struct list_head *move_to = src; 2201 struct folio *folio; 2202 2203 folio = lru_to_folio(src); 2204 prefetchw_prev_lru_folio(folio, src, flags); 2205 2206 nr_pages = folio_nr_pages(folio); 2207 total_scan += nr_pages; 2208 2209 if (folio_zonenum(folio) > sc->reclaim_idx) { 2210 nr_skipped[folio_zonenum(folio)] += nr_pages; 2211 move_to = &folios_skipped; 2212 goto move; 2213 } 2214 2215 /* 2216 * Do not count skipped folios because that makes the function 2217 * return with no isolated folios if the LRU mostly contains 2218 * ineligible folios. This causes the VM to not reclaim any 2219 * folios, triggering a premature OOM. 2220 * Account all pages in a folio. 2221 */ 2222 scan += nr_pages; 2223 2224 if (!folio_test_lru(folio)) 2225 goto move; 2226 if (!sc->may_unmap && folio_mapped(folio)) 2227 goto move; 2228 2229 /* 2230 * Be careful not to clear the lru flag until after we're 2231 * sure the folio is not being freed elsewhere -- the 2232 * folio release code relies on it. 2233 */ 2234 if (unlikely(!folio_try_get(folio))) 2235 goto move; 2236 2237 if (!folio_test_clear_lru(folio)) { 2238 /* Another thread is already isolating this folio */ 2239 folio_put(folio); 2240 goto move; 2241 } 2242 2243 nr_taken += nr_pages; 2244 nr_zone_taken[folio_zonenum(folio)] += nr_pages; 2245 move_to = dst; 2246 move: 2247 list_move(&folio->lru, move_to); 2248 } 2249 2250 /* 2251 * Splice any skipped folios to the start of the LRU list. Note that 2252 * this disrupts the LRU order when reclaiming for lower zones but 2253 * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX 2254 * scanning would soon rescan the same folios to skip and waste lots 2255 * of cpu cycles. 2256 */ 2257 if (!list_empty(&folios_skipped)) { 2258 int zid; 2259 2260 list_splice(&folios_skipped, src); 2261 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 2262 if (!nr_skipped[zid]) 2263 continue; 2264 2265 __count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]); 2266 skipped += nr_skipped[zid]; 2267 } 2268 } 2269 *nr_scanned = total_scan; 2270 trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan, 2271 total_scan, skipped, nr_taken, 2272 sc->may_unmap ? 0 : ISOLATE_UNMAPPED, lru); 2273 update_lru_sizes(lruvec, lru, nr_zone_taken); 2274 return nr_taken; 2275 } 2276 2277 /** 2278 * folio_isolate_lru() - Try to isolate a folio from its LRU list. 2279 * @folio: Folio to isolate from its LRU list. 2280 * 2281 * Isolate a @folio from an LRU list and adjust the vmstat statistic 2282 * corresponding to whatever LRU list the folio was on. 2283 * 2284 * The folio will have its LRU flag cleared. If it was found on the 2285 * active list, it will have the Active flag set. If it was found on the 2286 * unevictable list, it will have the Unevictable flag set. These flags 2287 * may need to be cleared by the caller before letting the page go. 2288 * 2289 * Context: 2290 * 2291 * (1) Must be called with an elevated refcount on the page. This is a 2292 * fundamental difference from isolate_lru_pages() (which is called 2293 * without a stable reference). 2294 * (2) The lru_lock must not be held. 2295 * (3) Interrupts must be enabled. 2296 * 2297 * Return: 0 if the folio was removed from an LRU list. 2298 * -EBUSY if the folio was not on an LRU list. 2299 */ 2300 int folio_isolate_lru(struct folio *folio) 2301 { 2302 int ret = -EBUSY; 2303 2304 VM_BUG_ON_FOLIO(!folio_ref_count(folio), folio); 2305 2306 if (folio_test_clear_lru(folio)) { 2307 struct lruvec *lruvec; 2308 2309 folio_get(folio); 2310 lruvec = folio_lruvec_lock_irq(folio); 2311 lruvec_del_folio(lruvec, folio); 2312 unlock_page_lruvec_irq(lruvec); 2313 ret = 0; 2314 } 2315 2316 return ret; 2317 } 2318 2319 /* 2320 * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and 2321 * then get rescheduled. When there are massive number of tasks doing page 2322 * allocation, such sleeping direct reclaimers may keep piling up on each CPU, 2323 * the LRU list will go small and be scanned faster than necessary, leading to 2324 * unnecessary swapping, thrashing and OOM. 2325 */ 2326 static int too_many_isolated(struct pglist_data *pgdat, int file, 2327 struct scan_control *sc) 2328 { 2329 unsigned long inactive, isolated; 2330 bool too_many; 2331 2332 if (current_is_kswapd()) 2333 return 0; 2334 2335 if (!writeback_throttling_sane(sc)) 2336 return 0; 2337 2338 if (file) { 2339 inactive = node_page_state(pgdat, NR_INACTIVE_FILE); 2340 isolated = node_page_state(pgdat, NR_ISOLATED_FILE); 2341 } else { 2342 inactive = node_page_state(pgdat, NR_INACTIVE_ANON); 2343 isolated = node_page_state(pgdat, NR_ISOLATED_ANON); 2344 } 2345 2346 /* 2347 * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they 2348 * won't get blocked by normal direct-reclaimers, forming a circular 2349 * deadlock. 2350 */ 2351 if ((sc->gfp_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 2352 inactive >>= 3; 2353 2354 too_many = isolated > inactive; 2355 2356 /* Wake up tasks throttled due to too_many_isolated. */ 2357 if (!too_many) 2358 wake_throttle_isolated(pgdat); 2359 2360 return too_many; 2361 } 2362 2363 /* 2364 * move_pages_to_lru() moves folios from private @list to appropriate LRU list. 2365 * On return, @list is reused as a list of folios to be freed by the caller. 2366 * 2367 * Returns the number of pages moved to the given lruvec. 2368 */ 2369 static unsigned int move_pages_to_lru(struct lruvec *lruvec, 2370 struct list_head *list) 2371 { 2372 int nr_pages, nr_moved = 0; 2373 LIST_HEAD(folios_to_free); 2374 2375 while (!list_empty(list)) { 2376 struct folio *folio = lru_to_folio(list); 2377 2378 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 2379 list_del(&folio->lru); 2380 if (unlikely(!folio_evictable(folio))) { 2381 spin_unlock_irq(&lruvec->lru_lock); 2382 folio_putback_lru(folio); 2383 spin_lock_irq(&lruvec->lru_lock); 2384 continue; 2385 } 2386 2387 /* 2388 * The folio_set_lru needs to be kept here for list integrity. 2389 * Otherwise: 2390 * #0 move_pages_to_lru #1 release_pages 2391 * if (!folio_put_testzero()) 2392 * if (folio_put_testzero()) 2393 * !lru //skip lru_lock 2394 * folio_set_lru() 2395 * list_add(&folio->lru,) 2396 * list_add(&folio->lru,) 2397 */ 2398 folio_set_lru(folio); 2399 2400 if (unlikely(folio_put_testzero(folio))) { 2401 __folio_clear_lru_flags(folio); 2402 2403 if (unlikely(folio_test_large(folio))) { 2404 spin_unlock_irq(&lruvec->lru_lock); 2405 destroy_large_folio(folio); 2406 spin_lock_irq(&lruvec->lru_lock); 2407 } else 2408 list_add(&folio->lru, &folios_to_free); 2409 2410 continue; 2411 } 2412 2413 /* 2414 * All pages were isolated from the same lruvec (and isolation 2415 * inhibits memcg migration). 2416 */ 2417 VM_BUG_ON_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); 2418 lruvec_add_folio(lruvec, folio); 2419 nr_pages = folio_nr_pages(folio); 2420 nr_moved += nr_pages; 2421 if (folio_test_active(folio)) 2422 workingset_age_nonresident(lruvec, nr_pages); 2423 } 2424 2425 /* 2426 * To save our caller's stack, now use input list for pages to free. 2427 */ 2428 list_splice(&folios_to_free, list); 2429 2430 return nr_moved; 2431 } 2432 2433 /* 2434 * If a kernel thread (such as nfsd for loop-back mounts) services a backing 2435 * device by writing to the page cache it sets PF_LOCAL_THROTTLE. In this case 2436 * we should not throttle. Otherwise it is safe to do so. 2437 */ 2438 static int current_may_throttle(void) 2439 { 2440 return !(current->flags & PF_LOCAL_THROTTLE); 2441 } 2442 2443 /* 2444 * shrink_inactive_list() is a helper for shrink_node(). It returns the number 2445 * of reclaimed pages 2446 */ 2447 static unsigned long 2448 shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, 2449 struct scan_control *sc, enum lru_list lru) 2450 { 2451 LIST_HEAD(page_list); 2452 unsigned long nr_scanned; 2453 unsigned int nr_reclaimed = 0; 2454 unsigned long nr_taken; 2455 struct reclaim_stat stat; 2456 bool file = is_file_lru(lru); 2457 enum vm_event_item item; 2458 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2459 bool stalled = false; 2460 2461 while (unlikely(too_many_isolated(pgdat, file, sc))) { 2462 if (stalled) 2463 return 0; 2464 2465 /* wait a bit for the reclaimer. */ 2466 stalled = true; 2467 reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED); 2468 2469 /* We are about to die and free our memory. Return now. */ 2470 if (fatal_signal_pending(current)) 2471 return SWAP_CLUSTER_MAX; 2472 } 2473 2474 lru_add_drain(); 2475 2476 spin_lock_irq(&lruvec->lru_lock); 2477 2478 nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list, 2479 &nr_scanned, sc, lru); 2480 2481 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); 2482 item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT; 2483 if (!cgroup_reclaim(sc)) 2484 __count_vm_events(item, nr_scanned); 2485 __count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned); 2486 __count_vm_events(PGSCAN_ANON + file, nr_scanned); 2487 2488 spin_unlock_irq(&lruvec->lru_lock); 2489 2490 if (nr_taken == 0) 2491 return 0; 2492 2493 nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, &stat, false); 2494 2495 spin_lock_irq(&lruvec->lru_lock); 2496 move_pages_to_lru(lruvec, &page_list); 2497 2498 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); 2499 item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT; 2500 if (!cgroup_reclaim(sc)) 2501 __count_vm_events(item, nr_reclaimed); 2502 __count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed); 2503 __count_vm_events(PGSTEAL_ANON + file, nr_reclaimed); 2504 spin_unlock_irq(&lruvec->lru_lock); 2505 2506 lru_note_cost(lruvec, file, stat.nr_pageout); 2507 mem_cgroup_uncharge_list(&page_list); 2508 free_unref_page_list(&page_list); 2509 2510 /* 2511 * If dirty pages are scanned that are not queued for IO, it 2512 * implies that flushers are not doing their job. This can 2513 * happen when memory pressure pushes dirty pages to the end of 2514 * the LRU before the dirty limits are breached and the dirty 2515 * data has expired. It can also happen when the proportion of 2516 * dirty pages grows not through writes but through memory 2517 * pressure reclaiming all the clean cache. And in some cases, 2518 * the flushers simply cannot keep up with the allocation 2519 * rate. Nudge the flusher threads in case they are asleep. 2520 */ 2521 if (stat.nr_unqueued_dirty == nr_taken) 2522 wakeup_flusher_threads(WB_REASON_VMSCAN); 2523 2524 sc->nr.dirty += stat.nr_dirty; 2525 sc->nr.congested += stat.nr_congested; 2526 sc->nr.unqueued_dirty += stat.nr_unqueued_dirty; 2527 sc->nr.writeback += stat.nr_writeback; 2528 sc->nr.immediate += stat.nr_immediate; 2529 sc->nr.taken += nr_taken; 2530 if (file) 2531 sc->nr.file_taken += nr_taken; 2532 2533 trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id, 2534 nr_scanned, nr_reclaimed, &stat, sc->priority, file); 2535 return nr_reclaimed; 2536 } 2537 2538 /* 2539 * shrink_active_list() moves folios from the active LRU to the inactive LRU. 2540 * 2541 * We move them the other way if the folio is referenced by one or more 2542 * processes. 2543 * 2544 * If the folios are mostly unmapped, the processing is fast and it is 2545 * appropriate to hold lru_lock across the whole operation. But if 2546 * the folios are mapped, the processing is slow (folio_referenced()), so 2547 * we should drop lru_lock around each folio. It's impossible to balance 2548 * this, so instead we remove the folios from the LRU while processing them. 2549 * It is safe to rely on the active flag against the non-LRU folios in here 2550 * because nobody will play with that bit on a non-LRU folio. 2551 * 2552 * The downside is that we have to touch folio->_refcount against each folio. 2553 * But we had to alter folio->flags anyway. 2554 */ 2555 static void shrink_active_list(unsigned long nr_to_scan, 2556 struct lruvec *lruvec, 2557 struct scan_control *sc, 2558 enum lru_list lru) 2559 { 2560 unsigned long nr_taken; 2561 unsigned long nr_scanned; 2562 unsigned long vm_flags; 2563 LIST_HEAD(l_hold); /* The folios which were snipped off */ 2564 LIST_HEAD(l_active); 2565 LIST_HEAD(l_inactive); 2566 unsigned nr_deactivate, nr_activate; 2567 unsigned nr_rotated = 0; 2568 int file = is_file_lru(lru); 2569 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2570 2571 lru_add_drain(); 2572 2573 spin_lock_irq(&lruvec->lru_lock); 2574 2575 nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold, 2576 &nr_scanned, sc, lru); 2577 2578 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); 2579 2580 if (!cgroup_reclaim(sc)) 2581 __count_vm_events(PGREFILL, nr_scanned); 2582 __count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned); 2583 2584 spin_unlock_irq(&lruvec->lru_lock); 2585 2586 while (!list_empty(&l_hold)) { 2587 struct folio *folio; 2588 2589 cond_resched(); 2590 folio = lru_to_folio(&l_hold); 2591 list_del(&folio->lru); 2592 2593 if (unlikely(!folio_evictable(folio))) { 2594 folio_putback_lru(folio); 2595 continue; 2596 } 2597 2598 if (unlikely(buffer_heads_over_limit)) { 2599 if (folio_test_private(folio) && folio_trylock(folio)) { 2600 if (folio_test_private(folio)) 2601 filemap_release_folio(folio, 0); 2602 folio_unlock(folio); 2603 } 2604 } 2605 2606 /* Referenced or rmap lock contention: rotate */ 2607 if (folio_referenced(folio, 0, sc->target_mem_cgroup, 2608 &vm_flags) != 0) { 2609 /* 2610 * Identify referenced, file-backed active folios and 2611 * give them one more trip around the active list. So 2612 * that executable code get better chances to stay in 2613 * memory under moderate memory pressure. Anon folios 2614 * are not likely to be evicted by use-once streaming 2615 * IO, plus JVM can create lots of anon VM_EXEC folios, 2616 * so we ignore them here. 2617 */ 2618 if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) { 2619 nr_rotated += folio_nr_pages(folio); 2620 list_add(&folio->lru, &l_active); 2621 continue; 2622 } 2623 } 2624 2625 folio_clear_active(folio); /* we are de-activating */ 2626 folio_set_workingset(folio); 2627 list_add(&folio->lru, &l_inactive); 2628 } 2629 2630 /* 2631 * Move folios back to the lru list. 2632 */ 2633 spin_lock_irq(&lruvec->lru_lock); 2634 2635 nr_activate = move_pages_to_lru(lruvec, &l_active); 2636 nr_deactivate = move_pages_to_lru(lruvec, &l_inactive); 2637 /* Keep all free folios in l_active list */ 2638 list_splice(&l_inactive, &l_active); 2639 2640 __count_vm_events(PGDEACTIVATE, nr_deactivate); 2641 __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); 2642 2643 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); 2644 spin_unlock_irq(&lruvec->lru_lock); 2645 2646 mem_cgroup_uncharge_list(&l_active); 2647 free_unref_page_list(&l_active); 2648 trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate, 2649 nr_deactivate, nr_rotated, sc->priority, file); 2650 } 2651 2652 static unsigned int reclaim_page_list(struct list_head *page_list, 2653 struct pglist_data *pgdat) 2654 { 2655 struct reclaim_stat dummy_stat; 2656 unsigned int nr_reclaimed; 2657 struct folio *folio; 2658 struct scan_control sc = { 2659 .gfp_mask = GFP_KERNEL, 2660 .may_writepage = 1, 2661 .may_unmap = 1, 2662 .may_swap = 1, 2663 .no_demotion = 1, 2664 }; 2665 2666 nr_reclaimed = shrink_page_list(page_list, pgdat, &sc, &dummy_stat, false); 2667 while (!list_empty(page_list)) { 2668 folio = lru_to_folio(page_list); 2669 list_del(&folio->lru); 2670 folio_putback_lru(folio); 2671 } 2672 2673 return nr_reclaimed; 2674 } 2675 2676 unsigned long reclaim_pages(struct list_head *folio_list) 2677 { 2678 int nid; 2679 unsigned int nr_reclaimed = 0; 2680 LIST_HEAD(node_folio_list); 2681 unsigned int noreclaim_flag; 2682 2683 if (list_empty(folio_list)) 2684 return nr_reclaimed; 2685 2686 noreclaim_flag = memalloc_noreclaim_save(); 2687 2688 nid = folio_nid(lru_to_folio(folio_list)); 2689 do { 2690 struct folio *folio = lru_to_folio(folio_list); 2691 2692 if (nid == folio_nid(folio)) { 2693 folio_clear_active(folio); 2694 list_move(&folio->lru, &node_folio_list); 2695 continue; 2696 } 2697 2698 nr_reclaimed += reclaim_page_list(&node_folio_list, NODE_DATA(nid)); 2699 nid = folio_nid(lru_to_folio(folio_list)); 2700 } while (!list_empty(folio_list)); 2701 2702 nr_reclaimed += reclaim_page_list(&node_folio_list, NODE_DATA(nid)); 2703 2704 memalloc_noreclaim_restore(noreclaim_flag); 2705 2706 return nr_reclaimed; 2707 } 2708 2709 static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan, 2710 struct lruvec *lruvec, struct scan_control *sc) 2711 { 2712 if (is_active_lru(lru)) { 2713 if (sc->may_deactivate & (1 << is_file_lru(lru))) 2714 shrink_active_list(nr_to_scan, lruvec, sc, lru); 2715 else 2716 sc->skipped_deactivate = 1; 2717 return 0; 2718 } 2719 2720 return shrink_inactive_list(nr_to_scan, lruvec, sc, lru); 2721 } 2722 2723 /* 2724 * The inactive anon list should be small enough that the VM never has 2725 * to do too much work. 2726 * 2727 * The inactive file list should be small enough to leave most memory 2728 * to the established workingset on the scan-resistant active list, 2729 * but large enough to avoid thrashing the aggregate readahead window. 2730 * 2731 * Both inactive lists should also be large enough that each inactive 2732 * page has a chance to be referenced again before it is reclaimed. 2733 * 2734 * If that fails and refaulting is observed, the inactive list grows. 2735 * 2736 * The inactive_ratio is the target ratio of ACTIVE to INACTIVE pages 2737 * on this LRU, maintained by the pageout code. An inactive_ratio 2738 * of 3 means 3:1 or 25% of the pages are kept on the inactive list. 2739 * 2740 * total target max 2741 * memory ratio inactive 2742 * ------------------------------------- 2743 * 10MB 1 5MB 2744 * 100MB 1 50MB 2745 * 1GB 3 250MB 2746 * 10GB 10 0.9GB 2747 * 100GB 31 3GB 2748 * 1TB 101 10GB 2749 * 10TB 320 32GB 2750 */ 2751 static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru) 2752 { 2753 enum lru_list active_lru = inactive_lru + LRU_ACTIVE; 2754 unsigned long inactive, active; 2755 unsigned long inactive_ratio; 2756 unsigned long gb; 2757 2758 inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru); 2759 active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru); 2760 2761 gb = (inactive + active) >> (30 - PAGE_SHIFT); 2762 if (gb) 2763 inactive_ratio = int_sqrt(10 * gb); 2764 else 2765 inactive_ratio = 1; 2766 2767 return inactive * inactive_ratio < active; 2768 } 2769 2770 enum scan_balance { 2771 SCAN_EQUAL, 2772 SCAN_FRACT, 2773 SCAN_ANON, 2774 SCAN_FILE, 2775 }; 2776 2777 static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc) 2778 { 2779 unsigned long file; 2780 struct lruvec *target_lruvec; 2781 2782 if (lru_gen_enabled()) 2783 return; 2784 2785 target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); 2786 2787 /* 2788 * Flush the memory cgroup stats, so that we read accurate per-memcg 2789 * lruvec stats for heuristics. 2790 */ 2791 mem_cgroup_flush_stats(); 2792 2793 /* 2794 * Determine the scan balance between anon and file LRUs. 2795 */ 2796 spin_lock_irq(&target_lruvec->lru_lock); 2797 sc->anon_cost = target_lruvec->anon_cost; 2798 sc->file_cost = target_lruvec->file_cost; 2799 spin_unlock_irq(&target_lruvec->lru_lock); 2800 2801 /* 2802 * Target desirable inactive:active list ratios for the anon 2803 * and file LRU lists. 2804 */ 2805 if (!sc->force_deactivate) { 2806 unsigned long refaults; 2807 2808 /* 2809 * When refaults are being observed, it means a new 2810 * workingset is being established. Deactivate to get 2811 * rid of any stale active pages quickly. 2812 */ 2813 refaults = lruvec_page_state(target_lruvec, 2814 WORKINGSET_ACTIVATE_ANON); 2815 if (refaults != target_lruvec->refaults[WORKINGSET_ANON] || 2816 inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) 2817 sc->may_deactivate |= DEACTIVATE_ANON; 2818 else 2819 sc->may_deactivate &= ~DEACTIVATE_ANON; 2820 2821 refaults = lruvec_page_state(target_lruvec, 2822 WORKINGSET_ACTIVATE_FILE); 2823 if (refaults != target_lruvec->refaults[WORKINGSET_FILE] || 2824 inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) 2825 sc->may_deactivate |= DEACTIVATE_FILE; 2826 else 2827 sc->may_deactivate &= ~DEACTIVATE_FILE; 2828 } else 2829 sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; 2830 2831 /* 2832 * If we have plenty of inactive file pages that aren't 2833 * thrashing, try to reclaim those first before touching 2834 * anonymous pages. 2835 */ 2836 file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); 2837 if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) 2838 sc->cache_trim_mode = 1; 2839 else 2840 sc->cache_trim_mode = 0; 2841 2842 /* 2843 * Prevent the reclaimer from falling into the cache trap: as 2844 * cache pages start out inactive, every cache fault will tip 2845 * the scan balance towards the file LRU. And as the file LRU 2846 * shrinks, so does the window for rotation from references. 2847 * This means we have a runaway feedback loop where a tiny 2848 * thrashing file LRU becomes infinitely more attractive than 2849 * anon pages. Try to detect this based on file LRU size. 2850 */ 2851 if (!cgroup_reclaim(sc)) { 2852 unsigned long total_high_wmark = 0; 2853 unsigned long free, anon; 2854 int z; 2855 2856 free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); 2857 file = node_page_state(pgdat, NR_ACTIVE_FILE) + 2858 node_page_state(pgdat, NR_INACTIVE_FILE); 2859 2860 for (z = 0; z < MAX_NR_ZONES; z++) { 2861 struct zone *zone = &pgdat->node_zones[z]; 2862 2863 if (!managed_zone(zone)) 2864 continue; 2865 2866 total_high_wmark += high_wmark_pages(zone); 2867 } 2868 2869 /* 2870 * Consider anon: if that's low too, this isn't a 2871 * runaway file reclaim problem, but rather just 2872 * extreme pressure. Reclaim as per usual then. 2873 */ 2874 anon = node_page_state(pgdat, NR_INACTIVE_ANON); 2875 2876 sc->file_is_tiny = 2877 file + free <= total_high_wmark && 2878 !(sc->may_deactivate & DEACTIVATE_ANON) && 2879 anon >> sc->priority; 2880 } 2881 } 2882 2883 /* 2884 * Determine how aggressively the anon and file LRU lists should be 2885 * scanned. 2886 * 2887 * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan 2888 * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan 2889 */ 2890 static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, 2891 unsigned long *nr) 2892 { 2893 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2894 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 2895 unsigned long anon_cost, file_cost, total_cost; 2896 int swappiness = mem_cgroup_swappiness(memcg); 2897 u64 fraction[ANON_AND_FILE]; 2898 u64 denominator = 0; /* gcc */ 2899 enum scan_balance scan_balance; 2900 unsigned long ap, fp; 2901 enum lru_list lru; 2902 2903 /* If we have no swap space, do not bother scanning anon pages. */ 2904 if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { 2905 scan_balance = SCAN_FILE; 2906 goto out; 2907 } 2908 2909 /* 2910 * Global reclaim will swap to prevent OOM even with no 2911 * swappiness, but memcg users want to use this knob to 2912 * disable swapping for individual groups completely when 2913 * using the memory controller's swap limit feature would be 2914 * too expensive. 2915 */ 2916 if (cgroup_reclaim(sc) && !swappiness) { 2917 scan_balance = SCAN_FILE; 2918 goto out; 2919 } 2920 2921 /* 2922 * Do not apply any pressure balancing cleverness when the 2923 * system is close to OOM, scan both anon and file equally 2924 * (unless the swappiness setting disagrees with swapping). 2925 */ 2926 if (!sc->priority && swappiness) { 2927 scan_balance = SCAN_EQUAL; 2928 goto out; 2929 } 2930 2931 /* 2932 * If the system is almost out of file pages, force-scan anon. 2933 */ 2934 if (sc->file_is_tiny) { 2935 scan_balance = SCAN_ANON; 2936 goto out; 2937 } 2938 2939 /* 2940 * If there is enough inactive page cache, we do not reclaim 2941 * anything from the anonymous working right now. 2942 */ 2943 if (sc->cache_trim_mode) { 2944 scan_balance = SCAN_FILE; 2945 goto out; 2946 } 2947 2948 scan_balance = SCAN_FRACT; 2949 /* 2950 * Calculate the pressure balance between anon and file pages. 2951 * 2952 * The amount of pressure we put on each LRU is inversely 2953 * proportional to the cost of reclaiming each list, as 2954 * determined by the share of pages that are refaulting, times 2955 * the relative IO cost of bringing back a swapped out 2956 * anonymous page vs reloading a filesystem page (swappiness). 2957 * 2958 * Although we limit that influence to ensure no list gets 2959 * left behind completely: at least a third of the pressure is 2960 * applied, before swappiness. 2961 * 2962 * With swappiness at 100, anon and file have equal IO cost. 2963 */ 2964 total_cost = sc->anon_cost + sc->file_cost; 2965 anon_cost = total_cost + sc->anon_cost; 2966 file_cost = total_cost + sc->file_cost; 2967 total_cost = anon_cost + file_cost; 2968 2969 ap = swappiness * (total_cost + 1); 2970 ap /= anon_cost + 1; 2971 2972 fp = (200 - swappiness) * (total_cost + 1); 2973 fp /= file_cost + 1; 2974 2975 fraction[0] = ap; 2976 fraction[1] = fp; 2977 denominator = ap + fp; 2978 out: 2979 for_each_evictable_lru(lru) { 2980 int file = is_file_lru(lru); 2981 unsigned long lruvec_size; 2982 unsigned long low, min; 2983 unsigned long scan; 2984 2985 lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx); 2986 mem_cgroup_protection(sc->target_mem_cgroup, memcg, 2987 &min, &low); 2988 2989 if (min || low) { 2990 /* 2991 * Scale a cgroup's reclaim pressure by proportioning 2992 * its current usage to its memory.low or memory.min 2993 * setting. 2994 * 2995 * This is important, as otherwise scanning aggression 2996 * becomes extremely binary -- from nothing as we 2997 * approach the memory protection threshold, to totally 2998 * nominal as we exceed it. This results in requiring 2999 * setting extremely liberal protection thresholds. It 3000 * also means we simply get no protection at all if we 3001 * set it too low, which is not ideal. 3002 * 3003 * If there is any protection in place, we reduce scan 3004 * pressure by how much of the total memory used is 3005 * within protection thresholds. 3006 * 3007 * There is one special case: in the first reclaim pass, 3008 * we skip over all groups that are within their low 3009 * protection. If that fails to reclaim enough pages to 3010 * satisfy the reclaim goal, we come back and override 3011 * the best-effort low protection. However, we still 3012 * ideally want to honor how well-behaved groups are in 3013 * that case instead of simply punishing them all 3014 * equally. As such, we reclaim them based on how much 3015 * memory they are using, reducing the scan pressure 3016 * again by how much of the total memory used is under 3017 * hard protection. 3018 */ 3019 unsigned long cgroup_size = mem_cgroup_size(memcg); 3020 unsigned long protection; 3021 3022 /* memory.low scaling, make sure we retry before OOM */ 3023 if (!sc->memcg_low_reclaim && low > min) { 3024 protection = low; 3025 sc->memcg_low_skipped = 1; 3026 } else { 3027 protection = min; 3028 } 3029 3030 /* Avoid TOCTOU with earlier protection check */ 3031 cgroup_size = max(cgroup_size, protection); 3032 3033 scan = lruvec_size - lruvec_size * protection / 3034 (cgroup_size + 1); 3035 3036 /* 3037 * Minimally target SWAP_CLUSTER_MAX pages to keep 3038 * reclaim moving forwards, avoiding decrementing 3039 * sc->priority further than desirable. 3040 */ 3041 scan = max(scan, SWAP_CLUSTER_MAX); 3042 } else { 3043 scan = lruvec_size; 3044 } 3045 3046 scan >>= sc->priority; 3047 3048 /* 3049 * If the cgroup's already been deleted, make sure to 3050 * scrape out the remaining cache. 3051 */ 3052 if (!scan && !mem_cgroup_online(memcg)) 3053 scan = min(lruvec_size, SWAP_CLUSTER_MAX); 3054 3055 switch (scan_balance) { 3056 case SCAN_EQUAL: 3057 /* Scan lists relative to size */ 3058 break; 3059 case SCAN_FRACT: 3060 /* 3061 * Scan types proportional to swappiness and 3062 * their relative recent reclaim efficiency. 3063 * Make sure we don't miss the last page on 3064 * the offlined memory cgroups because of a 3065 * round-off error. 3066 */ 3067 scan = mem_cgroup_online(memcg) ? 3068 div64_u64(scan * fraction[file], denominator) : 3069 DIV64_U64_ROUND_UP(scan * fraction[file], 3070 denominator); 3071 break; 3072 case SCAN_FILE: 3073 case SCAN_ANON: 3074 /* Scan one type exclusively */ 3075 if ((scan_balance == SCAN_FILE) != file) 3076 scan = 0; 3077 break; 3078 default: 3079 /* Look ma, no brain */ 3080 BUG(); 3081 } 3082 3083 nr[lru] = scan; 3084 } 3085 } 3086 3087 /* 3088 * Anonymous LRU management is a waste if there is 3089 * ultimately no way to reclaim the memory. 3090 */ 3091 static bool can_age_anon_pages(struct pglist_data *pgdat, 3092 struct scan_control *sc) 3093 { 3094 /* Aging the anon LRU is valuable if swap is present: */ 3095 if (total_swap_pages > 0) 3096 return true; 3097 3098 /* Also valuable if anon pages can be demoted: */ 3099 return can_demote(pgdat->node_id, sc); 3100 } 3101 3102 #ifdef CONFIG_LRU_GEN 3103 3104 #ifdef CONFIG_LRU_GEN_ENABLED 3105 DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); 3106 #define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) 3107 #else 3108 DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); 3109 #define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) 3110 #endif 3111 3112 /****************************************************************************** 3113 * shorthand helpers 3114 ******************************************************************************/ 3115 3116 #define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset)) 3117 3118 #define DEFINE_MAX_SEQ(lruvec) \ 3119 unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) 3120 3121 #define DEFINE_MIN_SEQ(lruvec) \ 3122 unsigned long min_seq[ANON_AND_FILE] = { \ 3123 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \ 3124 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \ 3125 } 3126 3127 #define for_each_gen_type_zone(gen, type, zone) \ 3128 for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ 3129 for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ 3130 for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) 3131 3132 static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) 3133 { 3134 struct pglist_data *pgdat = NODE_DATA(nid); 3135 3136 #ifdef CONFIG_MEMCG 3137 if (memcg) { 3138 struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; 3139 3140 /* for hotadd_new_pgdat() */ 3141 if (!lruvec->pgdat) 3142 lruvec->pgdat = pgdat; 3143 3144 return lruvec; 3145 } 3146 #endif 3147 VM_WARN_ON_ONCE(!mem_cgroup_disabled()); 3148 3149 return pgdat ? &pgdat->__lruvec : NULL; 3150 } 3151 3152 static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) 3153 { 3154 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 3155 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 3156 3157 if (!can_demote(pgdat->node_id, sc) && 3158 mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) 3159 return 0; 3160 3161 return mem_cgroup_swappiness(memcg); 3162 } 3163 3164 static int get_nr_gens(struct lruvec *lruvec, int type) 3165 { 3166 return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; 3167 } 3168 3169 static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) 3170 { 3171 /* see the comment on lru_gen_struct */ 3172 return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS && 3173 get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) && 3174 get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS; 3175 } 3176 3177 /****************************************************************************** 3178 * mm_struct list 3179 ******************************************************************************/ 3180 3181 static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) 3182 { 3183 static struct lru_gen_mm_list mm_list = { 3184 .fifo = LIST_HEAD_INIT(mm_list.fifo), 3185 .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock), 3186 }; 3187 3188 #ifdef CONFIG_MEMCG 3189 if (memcg) 3190 return &memcg->mm_list; 3191 #endif 3192 VM_WARN_ON_ONCE(!mem_cgroup_disabled()); 3193 3194 return &mm_list; 3195 } 3196 3197 void lru_gen_add_mm(struct mm_struct *mm) 3198 { 3199 int nid; 3200 struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); 3201 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 3202 3203 VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); 3204 #ifdef CONFIG_MEMCG 3205 VM_WARN_ON_ONCE(mm->lru_gen.memcg); 3206 mm->lru_gen.memcg = memcg; 3207 #endif 3208 spin_lock(&mm_list->lock); 3209 3210 for_each_node_state(nid, N_MEMORY) { 3211 struct lruvec *lruvec = get_lruvec(memcg, nid); 3212 3213 if (!lruvec) 3214 continue; 3215 3216 /* the first addition since the last iteration */ 3217 if (lruvec->mm_state.tail == &mm_list->fifo) 3218 lruvec->mm_state.tail = &mm->lru_gen.list; 3219 } 3220 3221 list_add_tail(&mm->lru_gen.list, &mm_list->fifo); 3222 3223 spin_unlock(&mm_list->lock); 3224 } 3225 3226 void lru_gen_del_mm(struct mm_struct *mm) 3227 { 3228 int nid; 3229 struct lru_gen_mm_list *mm_list; 3230 struct mem_cgroup *memcg = NULL; 3231 3232 if (list_empty(&mm->lru_gen.list)) 3233 return; 3234 3235 #ifdef CONFIG_MEMCG 3236 memcg = mm->lru_gen.memcg; 3237 #endif 3238 mm_list = get_mm_list(memcg); 3239 3240 spin_lock(&mm_list->lock); 3241 3242 for_each_node(nid) { 3243 struct lruvec *lruvec = get_lruvec(memcg, nid); 3244 3245 if (!lruvec) 3246 continue; 3247 3248 /* where the last iteration ended (exclusive) */ 3249 if (lruvec->mm_state.tail == &mm->lru_gen.list) 3250 lruvec->mm_state.tail = lruvec->mm_state.tail->next; 3251 3252 /* where the current iteration continues (inclusive) */ 3253 if (lruvec->mm_state.head != &mm->lru_gen.list) 3254 continue; 3255 3256 lruvec->mm_state.head = lruvec->mm_state.head->next; 3257 /* the deletion ends the current iteration */ 3258 if (lruvec->mm_state.head == &mm_list->fifo) 3259 WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1); 3260 } 3261 3262 list_del_init(&mm->lru_gen.list); 3263 3264 spin_unlock(&mm_list->lock); 3265 3266 #ifdef CONFIG_MEMCG 3267 mem_cgroup_put(mm->lru_gen.memcg); 3268 mm->lru_gen.memcg = NULL; 3269 #endif 3270 } 3271 3272 #ifdef CONFIG_MEMCG 3273 void lru_gen_migrate_mm(struct mm_struct *mm) 3274 { 3275 struct mem_cgroup *memcg; 3276 struct task_struct *task = rcu_dereference_protected(mm->owner, true); 3277 3278 VM_WARN_ON_ONCE(task->mm != mm); 3279 lockdep_assert_held(&task->alloc_lock); 3280 3281 /* for mm_update_next_owner() */ 3282 if (mem_cgroup_disabled()) 3283 return; 3284 3285 rcu_read_lock(); 3286 memcg = mem_cgroup_from_task(task); 3287 rcu_read_unlock(); 3288 if (memcg == mm->lru_gen.memcg) 3289 return; 3290 3291 VM_WARN_ON_ONCE(!mm->lru_gen.memcg); 3292 VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); 3293 3294 lru_gen_del_mm(mm); 3295 lru_gen_add_mm(mm); 3296 } 3297 #endif 3298 3299 /* 3300 * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when 3301 * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of 3302 * bits in a bitmap, k is the number of hash functions and n is the number of 3303 * inserted items. 3304 * 3305 * Page table walkers use one of the two filters to reduce their search space. 3306 * To get rid of non-leaf entries that no longer have enough leaf entries, the 3307 * aging uses the double-buffering technique to flip to the other filter each 3308 * time it produces a new generation. For non-leaf entries that have enough 3309 * leaf entries, the aging carries them over to the next generation in 3310 * walk_pmd_range(); the eviction also report them when walking the rmap 3311 * in lru_gen_look_around(). 3312 * 3313 * For future optimizations: 3314 * 1. It's not necessary to keep both filters all the time. The spare one can be 3315 * freed after the RCU grace period and reallocated if needed again. 3316 * 2. And when reallocating, it's worth scaling its size according to the number 3317 * of inserted entries in the other filter, to reduce the memory overhead on 3318 * small systems and false positives on large systems. 3319 * 3. Jenkins' hash function is an alternative to Knuth's. 3320 */ 3321 #define BLOOM_FILTER_SHIFT 15 3322 3323 static inline int filter_gen_from_seq(unsigned long seq) 3324 { 3325 return seq % NR_BLOOM_FILTERS; 3326 } 3327 3328 static void get_item_key(void *item, int *key) 3329 { 3330 u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2); 3331 3332 BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32)); 3333 3334 key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1); 3335 key[1] = hash >> BLOOM_FILTER_SHIFT; 3336 } 3337 3338 static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq) 3339 { 3340 unsigned long *filter; 3341 int gen = filter_gen_from_seq(seq); 3342 3343 filter = lruvec->mm_state.filters[gen]; 3344 if (filter) { 3345 bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); 3346 return; 3347 } 3348 3349 filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), 3350 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); 3351 WRITE_ONCE(lruvec->mm_state.filters[gen], filter); 3352 } 3353 3354 static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) 3355 { 3356 int key[2]; 3357 unsigned long *filter; 3358 int gen = filter_gen_from_seq(seq); 3359 3360 filter = READ_ONCE(lruvec->mm_state.filters[gen]); 3361 if (!filter) 3362 return; 3363 3364 get_item_key(item, key); 3365 3366 if (!test_bit(key[0], filter)) 3367 set_bit(key[0], filter); 3368 if (!test_bit(key[1], filter)) 3369 set_bit(key[1], filter); 3370 } 3371 3372 static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) 3373 { 3374 int key[2]; 3375 unsigned long *filter; 3376 int gen = filter_gen_from_seq(seq); 3377 3378 filter = READ_ONCE(lruvec->mm_state.filters[gen]); 3379 if (!filter) 3380 return true; 3381 3382 get_item_key(item, key); 3383 3384 return test_bit(key[0], filter) && test_bit(key[1], filter); 3385 } 3386 3387 static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last) 3388 { 3389 int i; 3390 int hist; 3391 3392 lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); 3393 3394 if (walk) { 3395 hist = lru_hist_from_seq(walk->max_seq); 3396 3397 for (i = 0; i < NR_MM_STATS; i++) { 3398 WRITE_ONCE(lruvec->mm_state.stats[hist][i], 3399 lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]); 3400 walk->mm_stats[i] = 0; 3401 } 3402 } 3403 3404 if (NR_HIST_GENS > 1 && last) { 3405 hist = lru_hist_from_seq(lruvec->mm_state.seq + 1); 3406 3407 for (i = 0; i < NR_MM_STATS; i++) 3408 WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0); 3409 } 3410 } 3411 3412 static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) 3413 { 3414 int type; 3415 unsigned long size = 0; 3416 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3417 int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); 3418 3419 if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) 3420 return true; 3421 3422 clear_bit(key, &mm->lru_gen.bitmap); 3423 3424 for (type = !walk->can_swap; type < ANON_AND_FILE; type++) { 3425 size += type ? get_mm_counter(mm, MM_FILEPAGES) : 3426 get_mm_counter(mm, MM_ANONPAGES) + 3427 get_mm_counter(mm, MM_SHMEMPAGES); 3428 } 3429 3430 if (size < MIN_LRU_BATCH) 3431 return true; 3432 3433 return !mmget_not_zero(mm); 3434 } 3435 3436 static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, 3437 struct mm_struct **iter) 3438 { 3439 bool first = false; 3440 bool last = true; 3441 struct mm_struct *mm = NULL; 3442 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 3443 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 3444 struct lru_gen_mm_state *mm_state = &lruvec->mm_state; 3445 3446 /* 3447 * There are four interesting cases for this page table walker: 3448 * 1. It tries to start a new iteration of mm_list with a stale max_seq; 3449 * there is nothing left to do. 3450 * 2. It's the first of the current generation, and it needs to reset 3451 * the Bloom filter for the next generation. 3452 * 3. It reaches the end of mm_list, and it needs to increment 3453 * mm_state->seq; the iteration is done. 3454 * 4. It's the last of the current generation, and it needs to reset the 3455 * mm stats counters for the next generation. 3456 */ 3457 spin_lock(&mm_list->lock); 3458 3459 VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); 3460 VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq); 3461 VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers); 3462 3463 if (walk->max_seq <= mm_state->seq) { 3464 if (!*iter) 3465 last = false; 3466 goto done; 3467 } 3468 3469 if (!mm_state->nr_walkers) { 3470 VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); 3471 3472 mm_state->head = mm_list->fifo.next; 3473 first = true; 3474 } 3475 3476 while (!mm && mm_state->head != &mm_list->fifo) { 3477 mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); 3478 3479 mm_state->head = mm_state->head->next; 3480 3481 /* force scan for those added after the last iteration */ 3482 if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) { 3483 mm_state->tail = mm_state->head; 3484 walk->force_scan = true; 3485 } 3486 3487 if (should_skip_mm(mm, walk)) 3488 mm = NULL; 3489 } 3490 3491 if (mm_state->head == &mm_list->fifo) 3492 WRITE_ONCE(mm_state->seq, mm_state->seq + 1); 3493 done: 3494 if (*iter && !mm) 3495 mm_state->nr_walkers--; 3496 if (!*iter && mm) 3497 mm_state->nr_walkers++; 3498 3499 if (mm_state->nr_walkers) 3500 last = false; 3501 3502 if (*iter || last) 3503 reset_mm_stats(lruvec, walk, last); 3504 3505 spin_unlock(&mm_list->lock); 3506 3507 if (mm && first) 3508 reset_bloom_filter(lruvec, walk->max_seq + 1); 3509 3510 if (*iter) 3511 mmput_async(*iter); 3512 3513 *iter = mm; 3514 3515 return last; 3516 } 3517 3518 static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) 3519 { 3520 bool success = false; 3521 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 3522 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 3523 struct lru_gen_mm_state *mm_state = &lruvec->mm_state; 3524 3525 spin_lock(&mm_list->lock); 3526 3527 VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); 3528 3529 if (max_seq > mm_state->seq && !mm_state->nr_walkers) { 3530 VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); 3531 3532 WRITE_ONCE(mm_state->seq, mm_state->seq + 1); 3533 reset_mm_stats(lruvec, NULL, true); 3534 success = true; 3535 } 3536 3537 spin_unlock(&mm_list->lock); 3538 3539 return success; 3540 } 3541 3542 /****************************************************************************** 3543 * refault feedback loop 3544 ******************************************************************************/ 3545 3546 /* 3547 * A feedback loop based on Proportional-Integral-Derivative (PID) controller. 3548 * 3549 * The P term is refaulted/(evicted+protected) from a tier in the generation 3550 * currently being evicted; the I term is the exponential moving average of the 3551 * P term over the generations previously evicted, using the smoothing factor 3552 * 1/2; the D term isn't supported. 3553 * 3554 * The setpoint (SP) is always the first tier of one type; the process variable 3555 * (PV) is either any tier of the other type or any other tier of the same 3556 * type. 3557 * 3558 * The error is the difference between the SP and the PV; the correction is to 3559 * turn off protection when SP>PV or turn on protection when SP<PV. 3560 * 3561 * For future optimizations: 3562 * 1. The D term may discount the other two terms over time so that long-lived 3563 * generations can resist stale information. 3564 */ 3565 struct ctrl_pos { 3566 unsigned long refaulted; 3567 unsigned long total; 3568 int gain; 3569 }; 3570 3571 static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain, 3572 struct ctrl_pos *pos) 3573 { 3574 struct lru_gen_struct *lrugen = &lruvec->lrugen; 3575 int hist = lru_hist_from_seq(lrugen->min_seq[type]); 3576 3577 pos->refaulted = lrugen->avg_refaulted[type][tier] + 3578 atomic_long_read(&lrugen->refaulted[hist][type][tier]); 3579 pos->total = lrugen->avg_total[type][tier] + 3580 atomic_long_read(&lrugen->evicted[hist][type][tier]); 3581 if (tier) 3582 pos->total += lrugen->protected[hist][type][tier - 1]; 3583 pos->gain = gain; 3584 } 3585 3586 static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) 3587 { 3588 int hist, tier; 3589 struct lru_gen_struct *lrugen = &lruvec->lrugen; 3590 bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1; 3591 unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; 3592 3593 lockdep_assert_held(&lruvec->lru_lock); 3594 3595 if (!carryover && !clear) 3596 return; 3597 3598 hist = lru_hist_from_seq(seq); 3599 3600 for (tier = 0; tier < MAX_NR_TIERS; tier++) { 3601 if (carryover) { 3602 unsigned long sum; 3603 3604 sum = lrugen->avg_refaulted[type][tier] + 3605 atomic_long_read(&lrugen->refaulted[hist][type][tier]); 3606 WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); 3607 3608 sum = lrugen->avg_total[type][tier] + 3609 atomic_long_read(&lrugen->evicted[hist][type][tier]); 3610 if (tier) 3611 sum += lrugen->protected[hist][type][tier - 1]; 3612 WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); 3613 } 3614 3615 if (clear) { 3616 atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); 3617 atomic_long_set(&lrugen->evicted[hist][type][tier], 0); 3618 if (tier) 3619 WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0); 3620 } 3621 } 3622 } 3623 3624 static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) 3625 { 3626 /* 3627 * Return true if the PV has a limited number of refaults or a lower 3628 * refaulted/total than the SP. 3629 */ 3630 return pv->refaulted < MIN_LRU_BATCH || 3631 pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <= 3632 (sp->refaulted + 1) * pv->total * pv->gain; 3633 } 3634 3635 /****************************************************************************** 3636 * the aging 3637 ******************************************************************************/ 3638 3639 /* promote pages accessed through page tables */ 3640 static int folio_update_gen(struct folio *folio, int gen) 3641 { 3642 unsigned long new_flags, old_flags = READ_ONCE(folio->flags); 3643 3644 VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); 3645 VM_WARN_ON_ONCE(!rcu_read_lock_held()); 3646 3647 do { 3648 /* lru_gen_del_folio() has isolated this page? */ 3649 if (!(old_flags & LRU_GEN_MASK)) { 3650 /* for shrink_page_list() */ 3651 new_flags = old_flags | BIT(PG_referenced); 3652 continue; 3653 } 3654 3655 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); 3656 new_flags |= (gen + 1UL) << LRU_GEN_PGOFF; 3657 } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); 3658 3659 return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; 3660 } 3661 3662 /* protect pages accessed multiple times through file descriptors */ 3663 static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) 3664 { 3665 int type = folio_is_file_lru(folio); 3666 struct lru_gen_struct *lrugen = &lruvec->lrugen; 3667 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); 3668 unsigned long new_flags, old_flags = READ_ONCE(folio->flags); 3669 3670 VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); 3671 3672 do { 3673 new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; 3674 /* folio_update_gen() has promoted this page? */ 3675 if (new_gen >= 0 && new_gen != old_gen) 3676 return new_gen; 3677 3678 new_gen = (old_gen + 1) % MAX_NR_GENS; 3679 3680 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); 3681 new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; 3682 /* for folio_end_writeback() */ 3683 if (reclaiming) 3684 new_flags |= BIT(PG_reclaim); 3685 } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); 3686 3687 lru_gen_update_size(lruvec, folio, old_gen, new_gen); 3688 3689 return new_gen; 3690 } 3691 3692 static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, 3693 int old_gen, int new_gen) 3694 { 3695 int type = folio_is_file_lru(folio); 3696 int zone = folio_zonenum(folio); 3697 int delta = folio_nr_pages(folio); 3698 3699 VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); 3700 VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); 3701 3702 walk->batched++; 3703 3704 walk->nr_pages[old_gen][type][zone] -= delta; 3705 walk->nr_pages[new_gen][type][zone] += delta; 3706 } 3707 3708 static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk) 3709 { 3710 int gen, type, zone; 3711 struct lru_gen_struct *lrugen = &lruvec->lrugen; 3712 3713 walk->batched = 0; 3714 3715 for_each_gen_type_zone(gen, type, zone) { 3716 enum lru_list lru = type * LRU_INACTIVE_FILE; 3717 int delta = walk->nr_pages[gen][type][zone]; 3718 3719 if (!delta) 3720 continue; 3721 3722 walk->nr_pages[gen][type][zone] = 0; 3723 WRITE_ONCE(lrugen->nr_pages[gen][type][zone], 3724 lrugen->nr_pages[gen][type][zone] + delta); 3725 3726 if (lru_gen_is_active(lruvec, gen)) 3727 lru += LRU_ACTIVE; 3728 __update_lru_size(lruvec, lru, zone, delta); 3729 } 3730 } 3731 3732 static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) 3733 { 3734 struct address_space *mapping; 3735 struct vm_area_struct *vma = args->vma; 3736 struct lru_gen_mm_walk *walk = args->private; 3737 3738 if (!vma_is_accessible(vma)) 3739 return true; 3740 3741 if (is_vm_hugetlb_page(vma)) 3742 return true; 3743 3744 if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL | VM_SEQ_READ | VM_RAND_READ)) 3745 return true; 3746 3747 if (vma == get_gate_vma(vma->vm_mm)) 3748 return true; 3749 3750 if (vma_is_anonymous(vma)) 3751 return !walk->can_swap; 3752 3753 if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) 3754 return true; 3755 3756 mapping = vma->vm_file->f_mapping; 3757 if (mapping_unevictable(mapping)) 3758 return true; 3759 3760 if (shmem_mapping(mapping)) 3761 return !walk->can_swap; 3762 3763 /* to exclude special mappings like dax, etc. */ 3764 return !mapping->a_ops->read_folio; 3765 } 3766 3767 /* 3768 * Some userspace memory allocators map many single-page VMAs. Instead of 3769 * returning back to the PGD table for each of such VMAs, finish an entire PMD 3770 * table to reduce zigzags and improve cache performance. 3771 */ 3772 static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, 3773 unsigned long *vm_start, unsigned long *vm_end) 3774 { 3775 unsigned long start = round_up(*vm_end, size); 3776 unsigned long end = (start | ~mask) + 1; 3777 3778 VM_WARN_ON_ONCE(mask & size); 3779 VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); 3780 3781 while (args->vma) { 3782 if (start >= args->vma->vm_end) { 3783 args->vma = args->vma->vm_next; 3784 continue; 3785 } 3786 3787 if (end && end <= args->vma->vm_start) 3788 return false; 3789 3790 if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) { 3791 args->vma = args->vma->vm_next; 3792 continue; 3793 } 3794 3795 *vm_start = max(start, args->vma->vm_start); 3796 *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; 3797 3798 return true; 3799 } 3800 3801 return false; 3802 } 3803 3804 static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr) 3805 { 3806 unsigned long pfn = pte_pfn(pte); 3807 3808 VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); 3809 3810 if (!pte_present(pte) || is_zero_pfn(pfn)) 3811 return -1; 3812 3813 if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte))) 3814 return -1; 3815 3816 if (WARN_ON_ONCE(!pfn_valid(pfn))) 3817 return -1; 3818 3819 return pfn; 3820 } 3821 3822 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) 3823 static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr) 3824 { 3825 unsigned long pfn = pmd_pfn(pmd); 3826 3827 VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); 3828 3829 if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) 3830 return -1; 3831 3832 if (WARN_ON_ONCE(pmd_devmap(pmd))) 3833 return -1; 3834 3835 if (WARN_ON_ONCE(!pfn_valid(pfn))) 3836 return -1; 3837 3838 return pfn; 3839 } 3840 #endif 3841 3842 static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg, 3843 struct pglist_data *pgdat, bool can_swap) 3844 { 3845 struct folio *folio; 3846 3847 /* try to avoid unnecessary memory loads */ 3848 if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) 3849 return NULL; 3850 3851 folio = pfn_folio(pfn); 3852 if (folio_nid(folio) != pgdat->node_id) 3853 return NULL; 3854 3855 if (folio_memcg_rcu(folio) != memcg) 3856 return NULL; 3857 3858 /* file VMAs can contain anon pages from COW */ 3859 if (!folio_is_file_lru(folio) && !can_swap) 3860 return NULL; 3861 3862 return folio; 3863 } 3864 3865 static bool suitable_to_scan(int total, int young) 3866 { 3867 int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); 3868 3869 /* suitable if the average number of young PTEs per cacheline is >=1 */ 3870 return young * n >= total; 3871 } 3872 3873 static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, 3874 struct mm_walk *args) 3875 { 3876 int i; 3877 pte_t *pte; 3878 spinlock_t *ptl; 3879 unsigned long addr; 3880 int total = 0; 3881 int young = 0; 3882 struct lru_gen_mm_walk *walk = args->private; 3883 struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); 3884 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3885 int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); 3886 3887 VM_WARN_ON_ONCE(pmd_leaf(*pmd)); 3888 3889 ptl = pte_lockptr(args->mm, pmd); 3890 if (!spin_trylock(ptl)) 3891 return false; 3892 3893 arch_enter_lazy_mmu_mode(); 3894 3895 pte = pte_offset_map(pmd, start & PMD_MASK); 3896 restart: 3897 for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { 3898 unsigned long pfn; 3899 struct folio *folio; 3900 3901 total++; 3902 walk->mm_stats[MM_LEAF_TOTAL]++; 3903 3904 pfn = get_pte_pfn(pte[i], args->vma, addr); 3905 if (pfn == -1) 3906 continue; 3907 3908 if (!pte_young(pte[i])) { 3909 walk->mm_stats[MM_LEAF_OLD]++; 3910 continue; 3911 } 3912 3913 folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); 3914 if (!folio) 3915 continue; 3916 3917 if (!ptep_test_and_clear_young(args->vma, addr, pte + i)) 3918 VM_WARN_ON_ONCE(true); 3919 3920 young++; 3921 walk->mm_stats[MM_LEAF_YOUNG]++; 3922 3923 if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && 3924 !(folio_test_anon(folio) && folio_test_swapbacked(folio) && 3925 !folio_test_swapcache(folio))) 3926 folio_mark_dirty(folio); 3927 3928 old_gen = folio_update_gen(folio, new_gen); 3929 if (old_gen >= 0 && old_gen != new_gen) 3930 update_batch_size(walk, folio, old_gen, new_gen); 3931 } 3932 3933 if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) 3934 goto restart; 3935 3936 pte_unmap(pte); 3937 3938 arch_leave_lazy_mmu_mode(); 3939 spin_unlock(ptl); 3940 3941 return suitable_to_scan(total, young); 3942 } 3943 3944 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) 3945 static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, 3946 struct mm_walk *args, unsigned long *bitmap, unsigned long *start) 3947 { 3948 int i; 3949 pmd_t *pmd; 3950 spinlock_t *ptl; 3951 struct lru_gen_mm_walk *walk = args->private; 3952 struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); 3953 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3954 int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); 3955 3956 VM_WARN_ON_ONCE(pud_leaf(*pud)); 3957 3958 /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ 3959 if (*start == -1) { 3960 *start = next; 3961 return; 3962 } 3963 3964 i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start); 3965 if (i && i <= MIN_LRU_BATCH) { 3966 __set_bit(i - 1, bitmap); 3967 return; 3968 } 3969 3970 pmd = pmd_offset(pud, *start); 3971 3972 ptl = pmd_lockptr(args->mm, pmd); 3973 if (!spin_trylock(ptl)) 3974 goto done; 3975 3976 arch_enter_lazy_mmu_mode(); 3977 3978 do { 3979 unsigned long pfn; 3980 struct folio *folio; 3981 unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start; 3982 3983 pfn = get_pmd_pfn(pmd[i], vma, addr); 3984 if (pfn == -1) 3985 goto next; 3986 3987 if (!pmd_trans_huge(pmd[i])) { 3988 if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && 3989 get_cap(LRU_GEN_NONLEAF_YOUNG)) 3990 pmdp_test_and_clear_young(vma, addr, pmd + i); 3991 goto next; 3992 } 3993 3994 folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); 3995 if (!folio) 3996 goto next; 3997 3998 if (!pmdp_test_and_clear_young(vma, addr, pmd + i)) 3999 goto next; 4000 4001 walk->mm_stats[MM_LEAF_YOUNG]++; 4002 4003 if (pmd_dirty(pmd[i]) && !folio_test_dirty(folio) && 4004 !(folio_test_anon(folio) && folio_test_swapbacked(folio) && 4005 !folio_test_swapcache(folio))) 4006 folio_mark_dirty(folio); 4007 4008 old_gen = folio_update_gen(folio, new_gen); 4009 if (old_gen >= 0 && old_gen != new_gen) 4010 update_batch_size(walk, folio, old_gen, new_gen); 4011 next: 4012 i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; 4013 } while (i <= MIN_LRU_BATCH); 4014 4015 arch_leave_lazy_mmu_mode(); 4016 spin_unlock(ptl); 4017 done: 4018 *start = -1; 4019 bitmap_zero(bitmap, MIN_LRU_BATCH); 4020 } 4021 #else 4022 static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, 4023 struct mm_walk *args, unsigned long *bitmap, unsigned long *start) 4024 { 4025 } 4026 #endif 4027 4028 static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, 4029 struct mm_walk *args) 4030 { 4031 int i; 4032 pmd_t *pmd; 4033 unsigned long next; 4034 unsigned long addr; 4035 struct vm_area_struct *vma; 4036 unsigned long pos = -1; 4037 struct lru_gen_mm_walk *walk = args->private; 4038 unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; 4039 4040 VM_WARN_ON_ONCE(pud_leaf(*pud)); 4041 4042 /* 4043 * Finish an entire PMD in two passes: the first only reaches to PTE 4044 * tables to avoid taking the PMD lock; the second, if necessary, takes 4045 * the PMD lock to clear the accessed bit in PMD entries. 4046 */ 4047 pmd = pmd_offset(pud, start & PUD_MASK); 4048 restart: 4049 /* walk_pte_range() may call get_next_vma() */ 4050 vma = args->vma; 4051 for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { 4052 pmd_t val = pmd_read_atomic(pmd + i); 4053 4054 /* for pmd_read_atomic() */ 4055 barrier(); 4056 4057 next = pmd_addr_end(addr, end); 4058 4059 if (!pmd_present(val) || is_huge_zero_pmd(val)) { 4060 walk->mm_stats[MM_LEAF_TOTAL]++; 4061 continue; 4062 } 4063 4064 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4065 if (pmd_trans_huge(val)) { 4066 unsigned long pfn = pmd_pfn(val); 4067 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 4068 4069 walk->mm_stats[MM_LEAF_TOTAL]++; 4070 4071 if (!pmd_young(val)) { 4072 walk->mm_stats[MM_LEAF_OLD]++; 4073 continue; 4074 } 4075 4076 /* try to avoid unnecessary memory loads */ 4077 if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) 4078 continue; 4079 4080 walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); 4081 continue; 4082 } 4083 #endif 4084 walk->mm_stats[MM_NONLEAF_TOTAL]++; 4085 4086 #ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG 4087 if (get_cap(LRU_GEN_NONLEAF_YOUNG)) { 4088 if (!pmd_young(val)) 4089 continue; 4090 4091 walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); 4092 } 4093 #endif 4094 if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) 4095 continue; 4096 4097 walk->mm_stats[MM_NONLEAF_FOUND]++; 4098 4099 if (!walk_pte_range(&val, addr, next, args)) 4100 continue; 4101 4102 walk->mm_stats[MM_NONLEAF_ADDED]++; 4103 4104 /* carry over to the next generation */ 4105 update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i); 4106 } 4107 4108 walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos); 4109 4110 if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) 4111 goto restart; 4112 } 4113 4114 static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, 4115 struct mm_walk *args) 4116 { 4117 int i; 4118 pud_t *pud; 4119 unsigned long addr; 4120 unsigned long next; 4121 struct lru_gen_mm_walk *walk = args->private; 4122 4123 VM_WARN_ON_ONCE(p4d_leaf(*p4d)); 4124 4125 pud = pud_offset(p4d, start & P4D_MASK); 4126 restart: 4127 for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { 4128 pud_t val = READ_ONCE(pud[i]); 4129 4130 next = pud_addr_end(addr, end); 4131 4132 if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) 4133 continue; 4134 4135 walk_pmd_range(&val, addr, next, args); 4136 4137 /* a racy check to curtail the waiting time */ 4138 if (wq_has_sleeper(&walk->lruvec->mm_state.wait)) 4139 return 1; 4140 4141 if (need_resched() || walk->batched >= MAX_LRU_BATCH) { 4142 end = (addr | ~PUD_MASK) + 1; 4143 goto done; 4144 } 4145 } 4146 4147 if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) 4148 goto restart; 4149 4150 end = round_up(end, P4D_SIZE); 4151 done: 4152 if (!end || !args->vma) 4153 return 1; 4154 4155 walk->next_addr = max(end, args->vma->vm_start); 4156 4157 return -EAGAIN; 4158 } 4159 4160 static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk) 4161 { 4162 static const struct mm_walk_ops mm_walk_ops = { 4163 .test_walk = should_skip_vma, 4164 .p4d_entry = walk_pud_range, 4165 }; 4166 4167 int err; 4168 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4169 4170 walk->next_addr = FIRST_USER_ADDRESS; 4171 4172 do { 4173 err = -EBUSY; 4174 4175 /* folio_update_gen() requires stable folio_memcg() */ 4176 if (!mem_cgroup_trylock_pages(memcg)) 4177 break; 4178 4179 /* the caller might be holding the lock for write */ 4180 if (mmap_read_trylock(mm)) { 4181 err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); 4182 4183 mmap_read_unlock(mm); 4184 } 4185 4186 mem_cgroup_unlock_pages(); 4187 4188 if (walk->batched) { 4189 spin_lock_irq(&lruvec->lru_lock); 4190 reset_batch_size(lruvec, walk); 4191 spin_unlock_irq(&lruvec->lru_lock); 4192 } 4193 4194 cond_resched(); 4195 } while (err == -EAGAIN); 4196 } 4197 4198 static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat) 4199 { 4200 struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; 4201 4202 if (pgdat && current_is_kswapd()) { 4203 VM_WARN_ON_ONCE(walk); 4204 4205 walk = &pgdat->mm_walk; 4206 } else if (!pgdat && !walk) { 4207 VM_WARN_ON_ONCE(current_is_kswapd()); 4208 4209 walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); 4210 } 4211 4212 current->reclaim_state->mm_walk = walk; 4213 4214 return walk; 4215 } 4216 4217 static void clear_mm_walk(void) 4218 { 4219 struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; 4220 4221 VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); 4222 VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); 4223 4224 current->reclaim_state->mm_walk = NULL; 4225 4226 if (!current_is_kswapd()) 4227 kfree(walk); 4228 } 4229 4230 static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) 4231 { 4232 int zone; 4233 int remaining = MAX_LRU_BATCH; 4234 struct lru_gen_struct *lrugen = &lruvec->lrugen; 4235 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); 4236 4237 if (type == LRU_GEN_ANON && !can_swap) 4238 goto done; 4239 4240 /* prevent cold/hot inversion if force_scan is true */ 4241 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4242 struct list_head *head = &lrugen->lists[old_gen][type][zone]; 4243 4244 while (!list_empty(head)) { 4245 struct folio *folio = lru_to_folio(head); 4246 4247 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 4248 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 4249 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 4250 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 4251 4252 new_gen = folio_inc_gen(lruvec, folio, false); 4253 list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]); 4254 4255 if (!--remaining) 4256 return false; 4257 } 4258 } 4259 done: 4260 reset_ctrl_pos(lruvec, type, true); 4261 WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); 4262 4263 return true; 4264 } 4265 4266 static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) 4267 { 4268 int gen, type, zone; 4269 bool success = false; 4270 struct lru_gen_struct *lrugen = &lruvec->lrugen; 4271 DEFINE_MIN_SEQ(lruvec); 4272 4273 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 4274 4275 /* find the oldest populated generation */ 4276 for (type = !can_swap; type < ANON_AND_FILE; type++) { 4277 while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { 4278 gen = lru_gen_from_seq(min_seq[type]); 4279 4280 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4281 if (!list_empty(&lrugen->lists[gen][type][zone])) 4282 goto next; 4283 } 4284 4285 min_seq[type]++; 4286 } 4287 next: 4288 ; 4289 } 4290 4291 /* see the comment on lru_gen_struct */ 4292 if (can_swap) { 4293 min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]); 4294 min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]); 4295 } 4296 4297 for (type = !can_swap; type < ANON_AND_FILE; type++) { 4298 if (min_seq[type] == lrugen->min_seq[type]) 4299 continue; 4300 4301 reset_ctrl_pos(lruvec, type, true); 4302 WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); 4303 success = true; 4304 } 4305 4306 return success; 4307 } 4308 4309 static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) 4310 { 4311 int prev, next; 4312 int type, zone; 4313 struct lru_gen_struct *lrugen = &lruvec->lrugen; 4314 4315 spin_lock_irq(&lruvec->lru_lock); 4316 4317 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 4318 4319 for (type = ANON_AND_FILE - 1; type >= 0; type--) { 4320 if (get_nr_gens(lruvec, type) != MAX_NR_GENS) 4321 continue; 4322 4323 VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap)); 4324 4325 while (!inc_min_seq(lruvec, type, can_swap)) { 4326 spin_unlock_irq(&lruvec->lru_lock); 4327 cond_resched(); 4328 spin_lock_irq(&lruvec->lru_lock); 4329 } 4330 } 4331 4332 /* 4333 * Update the active/inactive LRU sizes for compatibility. Both sides of 4334 * the current max_seq need to be covered, since max_seq+1 can overlap 4335 * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do 4336 * overlap, cold/hot inversion happens. 4337 */ 4338 prev = lru_gen_from_seq(lrugen->max_seq - 1); 4339 next = lru_gen_from_seq(lrugen->max_seq + 1); 4340 4341 for (type = 0; type < ANON_AND_FILE; type++) { 4342 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4343 enum lru_list lru = type * LRU_INACTIVE_FILE; 4344 long delta = lrugen->nr_pages[prev][type][zone] - 4345 lrugen->nr_pages[next][type][zone]; 4346 4347 if (!delta) 4348 continue; 4349 4350 __update_lru_size(lruvec, lru, zone, delta); 4351 __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); 4352 } 4353 } 4354 4355 for (type = 0; type < ANON_AND_FILE; type++) 4356 reset_ctrl_pos(lruvec, type, false); 4357 4358 WRITE_ONCE(lrugen->timestamps[next], jiffies); 4359 /* make sure preceding modifications appear */ 4360 smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); 4361 4362 spin_unlock_irq(&lruvec->lru_lock); 4363 } 4364 4365 static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, 4366 struct scan_control *sc, bool can_swap, bool force_scan) 4367 { 4368 bool success; 4369 struct lru_gen_mm_walk *walk; 4370 struct mm_struct *mm = NULL; 4371 struct lru_gen_struct *lrugen = &lruvec->lrugen; 4372 4373 VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq)); 4374 4375 /* see the comment in iterate_mm_list() */ 4376 if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) { 4377 success = false; 4378 goto done; 4379 } 4380 4381 /* 4382 * If the hardware doesn't automatically set the accessed bit, fallback 4383 * to lru_gen_look_around(), which only clears the accessed bit in a 4384 * handful of PTEs. Spreading the work out over a period of time usually 4385 * is less efficient, but it avoids bursty page faults. 4386 */ 4387 if (!force_scan && !(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) { 4388 success = iterate_mm_list_nowalk(lruvec, max_seq); 4389 goto done; 4390 } 4391 4392 walk = set_mm_walk(NULL); 4393 if (!walk) { 4394 success = iterate_mm_list_nowalk(lruvec, max_seq); 4395 goto done; 4396 } 4397 4398 walk->lruvec = lruvec; 4399 walk->max_seq = max_seq; 4400 walk->can_swap = can_swap; 4401 walk->force_scan = force_scan; 4402 4403 do { 4404 success = iterate_mm_list(lruvec, walk, &mm); 4405 if (mm) 4406 walk_mm(lruvec, mm, walk); 4407 4408 cond_resched(); 4409 } while (mm); 4410 done: 4411 if (!success) { 4412 if (sc->priority <= DEF_PRIORITY - 2) 4413 wait_event_killable(lruvec->mm_state.wait, 4414 max_seq < READ_ONCE(lrugen->max_seq)); 4415 4416 return max_seq < READ_ONCE(lrugen->max_seq); 4417 } 4418 4419 VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq)); 4420 4421 inc_max_seq(lruvec, can_swap, force_scan); 4422 /* either this sees any waiters or they will see updated max_seq */ 4423 if (wq_has_sleeper(&lruvec->mm_state.wait)) 4424 wake_up_all(&lruvec->mm_state.wait); 4425 4426 wakeup_flusher_threads(WB_REASON_VMSCAN); 4427 4428 return true; 4429 } 4430 4431 static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq, 4432 struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan) 4433 { 4434 int gen, type, zone; 4435 unsigned long old = 0; 4436 unsigned long young = 0; 4437 unsigned long total = 0; 4438 struct lru_gen_struct *lrugen = &lruvec->lrugen; 4439 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4440 4441 for (type = !can_swap; type < ANON_AND_FILE; type++) { 4442 unsigned long seq; 4443 4444 for (seq = min_seq[type]; seq <= max_seq; seq++) { 4445 unsigned long size = 0; 4446 4447 gen = lru_gen_from_seq(seq); 4448 4449 for (zone = 0; zone < MAX_NR_ZONES; zone++) 4450 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); 4451 4452 total += size; 4453 if (seq == max_seq) 4454 young += size; 4455 else if (seq + MIN_NR_GENS == max_seq) 4456 old += size; 4457 } 4458 } 4459 4460 /* try to scrape all its memory if this memcg was deleted */ 4461 *nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total; 4462 4463 /* 4464 * The aging tries to be lazy to reduce the overhead, while the eviction 4465 * stalls when the number of generations reaches MIN_NR_GENS. Hence, the 4466 * ideal number of generations is MIN_NR_GENS+1. 4467 */ 4468 if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) 4469 return true; 4470 if (min_seq[!can_swap] + MIN_NR_GENS < max_seq) 4471 return false; 4472 4473 /* 4474 * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1) 4475 * of the total number of pages for each generation. A reasonable range 4476 * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The 4477 * aging cares about the upper bound of hot pages, while the eviction 4478 * cares about the lower bound of cold pages. 4479 */ 4480 if (young * MIN_NR_GENS > total) 4481 return true; 4482 if (old * (MIN_NR_GENS + 2) < total) 4483 return true; 4484 4485 return false; 4486 } 4487 4488 static bool age_lruvec(struct lruvec *lruvec, struct scan_control *sc, unsigned long min_ttl) 4489 { 4490 bool need_aging; 4491 unsigned long nr_to_scan; 4492 int swappiness = get_swappiness(lruvec, sc); 4493 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4494 DEFINE_MAX_SEQ(lruvec); 4495 DEFINE_MIN_SEQ(lruvec); 4496 4497 VM_WARN_ON_ONCE(sc->memcg_low_reclaim); 4498 4499 mem_cgroup_calculate_protection(NULL, memcg); 4500 4501 if (mem_cgroup_below_min(memcg)) 4502 return false; 4503 4504 need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, swappiness, &nr_to_scan); 4505 4506 if (min_ttl) { 4507 int gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]); 4508 unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); 4509 4510 if (time_is_after_jiffies(birth + min_ttl)) 4511 return false; 4512 4513 /* the size is likely too small to be helpful */ 4514 if (!nr_to_scan && sc->priority != DEF_PRIORITY) 4515 return false; 4516 } 4517 4518 if (need_aging) 4519 try_to_inc_max_seq(lruvec, max_seq, sc, swappiness, false); 4520 4521 return true; 4522 } 4523 4524 /* to protect the working set of the last N jiffies */ 4525 static unsigned long lru_gen_min_ttl __read_mostly; 4526 4527 static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) 4528 { 4529 struct mem_cgroup *memcg; 4530 bool success = false; 4531 unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); 4532 4533 VM_WARN_ON_ONCE(!current_is_kswapd()); 4534 4535 sc->last_reclaimed = sc->nr_reclaimed; 4536 4537 /* 4538 * To reduce the chance of going into the aging path, which can be 4539 * costly, optimistically skip it if the flag below was cleared in the 4540 * eviction path. This improves the overall performance when multiple 4541 * memcgs are available. 4542 */ 4543 if (!sc->memcgs_need_aging) { 4544 sc->memcgs_need_aging = true; 4545 return; 4546 } 4547 4548 set_mm_walk(pgdat); 4549 4550 memcg = mem_cgroup_iter(NULL, NULL, NULL); 4551 do { 4552 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 4553 4554 if (age_lruvec(lruvec, sc, min_ttl)) 4555 success = true; 4556 4557 cond_resched(); 4558 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 4559 4560 clear_mm_walk(); 4561 4562 /* check the order to exclude compaction-induced reclaim */ 4563 if (success || !min_ttl || sc->order) 4564 return; 4565 4566 /* 4567 * The main goal is to OOM kill if every generation from all memcgs is 4568 * younger than min_ttl. However, another possibility is all memcgs are 4569 * either below min or empty. 4570 */ 4571 if (mutex_trylock(&oom_lock)) { 4572 struct oom_control oc = { 4573 .gfp_mask = sc->gfp_mask, 4574 }; 4575 4576 out_of_memory(&oc); 4577 4578 mutex_unlock(&oom_lock); 4579 } 4580 } 4581 4582 /* 4583 * This function exploits spatial locality when shrink_page_list() walks the 4584 * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If 4585 * the scan was done cacheline efficiently, it adds the PMD entry pointing to 4586 * the PTE table to the Bloom filter. This forms a feedback loop between the 4587 * eviction and the aging. 4588 */ 4589 void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) 4590 { 4591 int i; 4592 pte_t *pte; 4593 unsigned long start; 4594 unsigned long end; 4595 unsigned long addr; 4596 struct lru_gen_mm_walk *walk; 4597 int young = 0; 4598 unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; 4599 struct folio *folio = pfn_folio(pvmw->pfn); 4600 struct mem_cgroup *memcg = folio_memcg(folio); 4601 struct pglist_data *pgdat = folio_pgdat(folio); 4602 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 4603 DEFINE_MAX_SEQ(lruvec); 4604 int old_gen, new_gen = lru_gen_from_seq(max_seq); 4605 4606 lockdep_assert_held(pvmw->ptl); 4607 VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); 4608 4609 if (spin_is_contended(pvmw->ptl)) 4610 return; 4611 4612 /* avoid taking the LRU lock under the PTL when possible */ 4613 walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; 4614 4615 start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start); 4616 end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1; 4617 4618 if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { 4619 if (pvmw->address - start < MIN_LRU_BATCH * PAGE_SIZE / 2) 4620 end = start + MIN_LRU_BATCH * PAGE_SIZE; 4621 else if (end - pvmw->address < MIN_LRU_BATCH * PAGE_SIZE / 2) 4622 start = end - MIN_LRU_BATCH * PAGE_SIZE; 4623 else { 4624 start = pvmw->address - MIN_LRU_BATCH * PAGE_SIZE / 2; 4625 end = pvmw->address + MIN_LRU_BATCH * PAGE_SIZE / 2; 4626 } 4627 } 4628 4629 pte = pvmw->pte - (pvmw->address - start) / PAGE_SIZE; 4630 4631 rcu_read_lock(); 4632 arch_enter_lazy_mmu_mode(); 4633 4634 for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { 4635 unsigned long pfn; 4636 4637 pfn = get_pte_pfn(pte[i], pvmw->vma, addr); 4638 if (pfn == -1) 4639 continue; 4640 4641 if (!pte_young(pte[i])) 4642 continue; 4643 4644 folio = get_pfn_folio(pfn, memcg, pgdat, !walk || walk->can_swap); 4645 if (!folio) 4646 continue; 4647 4648 if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i)) 4649 VM_WARN_ON_ONCE(true); 4650 4651 young++; 4652 4653 if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && 4654 !(folio_test_anon(folio) && folio_test_swapbacked(folio) && 4655 !folio_test_swapcache(folio))) 4656 folio_mark_dirty(folio); 4657 4658 old_gen = folio_lru_gen(folio); 4659 if (old_gen < 0) 4660 folio_set_referenced(folio); 4661 else if (old_gen != new_gen) 4662 __set_bit(i, bitmap); 4663 } 4664 4665 arch_leave_lazy_mmu_mode(); 4666 rcu_read_unlock(); 4667 4668 /* feedback from rmap walkers to page table walkers */ 4669 if (suitable_to_scan(i, young)) 4670 update_bloom_filter(lruvec, max_seq, pvmw->pmd); 4671 4672 if (!walk && bitmap_weight(bitmap, MIN_LRU_BATCH) < PAGEVEC_SIZE) { 4673 for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { 4674 folio = pfn_folio(pte_pfn(pte[i])); 4675 folio_activate(folio); 4676 } 4677 return; 4678 } 4679 4680 /* folio_update_gen() requires stable folio_memcg() */ 4681 if (!mem_cgroup_trylock_pages(memcg)) 4682 return; 4683 4684 if (!walk) { 4685 spin_lock_irq(&lruvec->lru_lock); 4686 new_gen = lru_gen_from_seq(lruvec->lrugen.max_seq); 4687 } 4688 4689 for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { 4690 folio = pfn_folio(pte_pfn(pte[i])); 4691 if (folio_memcg_rcu(folio) != memcg) 4692 continue; 4693 4694 old_gen = folio_update_gen(folio, new_gen); 4695 if (old_gen < 0 || old_gen == new_gen) 4696 continue; 4697 4698 if (walk) 4699 update_batch_size(walk, folio, old_gen, new_gen); 4700 else 4701 lru_gen_update_size(lruvec, folio, old_gen, new_gen); 4702 } 4703 4704 if (!walk) 4705 spin_unlock_irq(&lruvec->lru_lock); 4706 4707 mem_cgroup_unlock_pages(); 4708 } 4709 4710 /****************************************************************************** 4711 * the eviction 4712 ******************************************************************************/ 4713 4714 static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx) 4715 { 4716 bool success; 4717 int gen = folio_lru_gen(folio); 4718 int type = folio_is_file_lru(folio); 4719 int zone = folio_zonenum(folio); 4720 int delta = folio_nr_pages(folio); 4721 int refs = folio_lru_refs(folio); 4722 int tier = lru_tier_from_refs(refs); 4723 struct lru_gen_struct *lrugen = &lruvec->lrugen; 4724 4725 VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); 4726 4727 /* unevictable */ 4728 if (!folio_evictable(folio)) { 4729 success = lru_gen_del_folio(lruvec, folio, true); 4730 VM_WARN_ON_ONCE_FOLIO(!success, folio); 4731 folio_set_unevictable(folio); 4732 lruvec_add_folio(lruvec, folio); 4733 __count_vm_events(UNEVICTABLE_PGCULLED, delta); 4734 return true; 4735 } 4736 4737 /* dirty lazyfree */ 4738 if (type == LRU_GEN_FILE && folio_test_anon(folio) && folio_test_dirty(folio)) { 4739 success = lru_gen_del_folio(lruvec, folio, true); 4740 VM_WARN_ON_ONCE_FOLIO(!success, folio); 4741 folio_set_swapbacked(folio); 4742 lruvec_add_folio_tail(lruvec, folio); 4743 return true; 4744 } 4745 4746 /* promoted */ 4747 if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { 4748 list_move(&folio->lru, &lrugen->lists[gen][type][zone]); 4749 return true; 4750 } 4751 4752 /* protected */ 4753 if (tier > tier_idx) { 4754 int hist = lru_hist_from_seq(lrugen->min_seq[type]); 4755 4756 gen = folio_inc_gen(lruvec, folio, false); 4757 list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]); 4758 4759 WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 4760 lrugen->protected[hist][type][tier - 1] + delta); 4761 __mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta); 4762 return true; 4763 } 4764 4765 /* waiting for writeback */ 4766 if (folio_test_locked(folio) || folio_test_writeback(folio) || 4767 (type == LRU_GEN_FILE && folio_test_dirty(folio))) { 4768 gen = folio_inc_gen(lruvec, folio, true); 4769 list_move(&folio->lru, &lrugen->lists[gen][type][zone]); 4770 return true; 4771 } 4772 4773 return false; 4774 } 4775 4776 static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) 4777 { 4778 bool success; 4779 4780 /* unmapping inhibited */ 4781 if (!sc->may_unmap && folio_mapped(folio)) 4782 return false; 4783 4784 /* swapping inhibited */ 4785 if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) && 4786 (folio_test_dirty(folio) || 4787 (folio_test_anon(folio) && !folio_test_swapcache(folio)))) 4788 return false; 4789 4790 /* raced with release_pages() */ 4791 if (!folio_try_get(folio)) 4792 return false; 4793 4794 /* raced with another isolation */ 4795 if (!folio_test_clear_lru(folio)) { 4796 folio_put(folio); 4797 return false; 4798 } 4799 4800 /* see the comment on MAX_NR_TIERS */ 4801 if (!folio_test_referenced(folio)) 4802 set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0); 4803 4804 /* for shrink_page_list() */ 4805 folio_clear_reclaim(folio); 4806 folio_clear_referenced(folio); 4807 4808 success = lru_gen_del_folio(lruvec, folio, true); 4809 VM_WARN_ON_ONCE_FOLIO(!success, folio); 4810 4811 return true; 4812 } 4813 4814 static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, 4815 int type, int tier, struct list_head *list) 4816 { 4817 int gen, zone; 4818 enum vm_event_item item; 4819 int sorted = 0; 4820 int scanned = 0; 4821 int isolated = 0; 4822 int remaining = MAX_LRU_BATCH; 4823 struct lru_gen_struct *lrugen = &lruvec->lrugen; 4824 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4825 4826 VM_WARN_ON_ONCE(!list_empty(list)); 4827 4828 if (get_nr_gens(lruvec, type) == MIN_NR_GENS) 4829 return 0; 4830 4831 gen = lru_gen_from_seq(lrugen->min_seq[type]); 4832 4833 for (zone = sc->reclaim_idx; zone >= 0; zone--) { 4834 LIST_HEAD(moved); 4835 int skipped = 0; 4836 struct list_head *head = &lrugen->lists[gen][type][zone]; 4837 4838 while (!list_empty(head)) { 4839 struct folio *folio = lru_to_folio(head); 4840 int delta = folio_nr_pages(folio); 4841 4842 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 4843 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 4844 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 4845 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 4846 4847 scanned += delta; 4848 4849 if (sort_folio(lruvec, folio, tier)) 4850 sorted += delta; 4851 else if (isolate_folio(lruvec, folio, sc)) { 4852 list_add(&folio->lru, list); 4853 isolated += delta; 4854 } else { 4855 list_move(&folio->lru, &moved); 4856 skipped += delta; 4857 } 4858 4859 if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH) 4860 break; 4861 } 4862 4863 if (skipped) { 4864 list_splice(&moved, head); 4865 __count_zid_vm_events(PGSCAN_SKIP, zone, skipped); 4866 } 4867 4868 if (!remaining || isolated >= MIN_LRU_BATCH) 4869 break; 4870 } 4871 4872 item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT; 4873 if (!cgroup_reclaim(sc)) { 4874 __count_vm_events(item, isolated); 4875 __count_vm_events(PGREFILL, sorted); 4876 } 4877 __count_memcg_events(memcg, item, isolated); 4878 __count_memcg_events(memcg, PGREFILL, sorted); 4879 __count_vm_events(PGSCAN_ANON + type, isolated); 4880 4881 /* 4882 * There might not be eligible pages due to reclaim_idx, may_unmap and 4883 * may_writepage. Check the remaining to prevent livelock if it's not 4884 * making progress. 4885 */ 4886 return isolated || !remaining ? scanned : 0; 4887 } 4888 4889 static int get_tier_idx(struct lruvec *lruvec, int type) 4890 { 4891 int tier; 4892 struct ctrl_pos sp, pv; 4893 4894 /* 4895 * To leave a margin for fluctuations, use a larger gain factor (1:2). 4896 * This value is chosen because any other tier would have at least twice 4897 * as many refaults as the first tier. 4898 */ 4899 read_ctrl_pos(lruvec, type, 0, 1, &sp); 4900 for (tier = 1; tier < MAX_NR_TIERS; tier++) { 4901 read_ctrl_pos(lruvec, type, tier, 2, &pv); 4902 if (!positive_ctrl_err(&sp, &pv)) 4903 break; 4904 } 4905 4906 return tier - 1; 4907 } 4908 4909 static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx) 4910 { 4911 int type, tier; 4912 struct ctrl_pos sp, pv; 4913 int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness }; 4914 4915 /* 4916 * Compare the first tier of anon with that of file to determine which 4917 * type to scan. Also need to compare other tiers of the selected type 4918 * with the first tier of the other type to determine the last tier (of 4919 * the selected type) to evict. 4920 */ 4921 read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp); 4922 read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv); 4923 type = positive_ctrl_err(&sp, &pv); 4924 4925 read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp); 4926 for (tier = 1; tier < MAX_NR_TIERS; tier++) { 4927 read_ctrl_pos(lruvec, type, tier, gain[type], &pv); 4928 if (!positive_ctrl_err(&sp, &pv)) 4929 break; 4930 } 4931 4932 *tier_idx = tier - 1; 4933 4934 return type; 4935 } 4936 4937 static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, 4938 int *type_scanned, struct list_head *list) 4939 { 4940 int i; 4941 int type; 4942 int scanned; 4943 int tier = -1; 4944 DEFINE_MIN_SEQ(lruvec); 4945 4946 /* 4947 * Try to make the obvious choice first. When anon and file are both 4948 * available from the same generation, interpret swappiness 1 as file 4949 * first and 200 as anon first. 4950 */ 4951 if (!swappiness) 4952 type = LRU_GEN_FILE; 4953 else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE]) 4954 type = LRU_GEN_ANON; 4955 else if (swappiness == 1) 4956 type = LRU_GEN_FILE; 4957 else if (swappiness == 200) 4958 type = LRU_GEN_ANON; 4959 else 4960 type = get_type_to_scan(lruvec, swappiness, &tier); 4961 4962 for (i = !swappiness; i < ANON_AND_FILE; i++) { 4963 if (tier < 0) 4964 tier = get_tier_idx(lruvec, type); 4965 4966 scanned = scan_folios(lruvec, sc, type, tier, list); 4967 if (scanned) 4968 break; 4969 4970 type = !type; 4971 tier = -1; 4972 } 4973 4974 *type_scanned = type; 4975 4976 return scanned; 4977 } 4978 4979 static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, 4980 bool *need_swapping) 4981 { 4982 int type; 4983 int scanned; 4984 int reclaimed; 4985 LIST_HEAD(list); 4986 struct folio *folio; 4987 enum vm_event_item item; 4988 struct reclaim_stat stat; 4989 struct lru_gen_mm_walk *walk; 4990 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4991 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 4992 4993 spin_lock_irq(&lruvec->lru_lock); 4994 4995 scanned = isolate_folios(lruvec, sc, swappiness, &type, &list); 4996 4997 scanned += try_to_inc_min_seq(lruvec, swappiness); 4998 4999 if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS) 5000 scanned = 0; 5001 5002 spin_unlock_irq(&lruvec->lru_lock); 5003 5004 if (list_empty(&list)) 5005 return scanned; 5006 5007 reclaimed = shrink_page_list(&list, pgdat, sc, &stat, false); 5008 5009 list_for_each_entry(folio, &list, lru) { 5010 /* restore LRU_REFS_FLAGS cleared by isolate_folio() */ 5011 if (folio_test_workingset(folio)) 5012 folio_set_referenced(folio); 5013 5014 /* don't add rejected pages to the oldest generation */ 5015 if (folio_test_reclaim(folio) && 5016 (folio_test_dirty(folio) || folio_test_writeback(folio))) 5017 folio_clear_active(folio); 5018 else 5019 folio_set_active(folio); 5020 } 5021 5022 spin_lock_irq(&lruvec->lru_lock); 5023 5024 move_pages_to_lru(lruvec, &list); 5025 5026 walk = current->reclaim_state->mm_walk; 5027 if (walk && walk->batched) 5028 reset_batch_size(lruvec, walk); 5029 5030 item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT; 5031 if (!cgroup_reclaim(sc)) 5032 __count_vm_events(item, reclaimed); 5033 __count_memcg_events(memcg, item, reclaimed); 5034 __count_vm_events(PGSTEAL_ANON + type, reclaimed); 5035 5036 spin_unlock_irq(&lruvec->lru_lock); 5037 5038 mem_cgroup_uncharge_list(&list); 5039 free_unref_page_list(&list); 5040 5041 sc->nr_reclaimed += reclaimed; 5042 5043 if (need_swapping && type == LRU_GEN_ANON) 5044 *need_swapping = true; 5045 5046 return scanned; 5047 } 5048 5049 /* 5050 * For future optimizations: 5051 * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg 5052 * reclaim. 5053 */ 5054 static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, 5055 bool can_swap, bool *need_aging) 5056 { 5057 unsigned long nr_to_scan; 5058 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 5059 DEFINE_MAX_SEQ(lruvec); 5060 DEFINE_MIN_SEQ(lruvec); 5061 5062 if (mem_cgroup_below_min(memcg) || 5063 (mem_cgroup_below_low(memcg) && !sc->memcg_low_reclaim)) 5064 return 0; 5065 5066 *need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, can_swap, &nr_to_scan); 5067 if (!*need_aging) 5068 return nr_to_scan; 5069 5070 /* skip the aging path at the default priority */ 5071 if (sc->priority == DEF_PRIORITY) 5072 goto done; 5073 5074 /* leave the work to lru_gen_age_node() */ 5075 if (current_is_kswapd()) 5076 return 0; 5077 5078 if (try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false)) 5079 return nr_to_scan; 5080 done: 5081 return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; 5082 } 5083 5084 static bool should_abort_scan(struct lruvec *lruvec, unsigned long seq, 5085 struct scan_control *sc, bool need_swapping) 5086 { 5087 int i; 5088 DEFINE_MAX_SEQ(lruvec); 5089 5090 if (!current_is_kswapd()) { 5091 /* age each memcg once to ensure fairness */ 5092 if (max_seq - seq > 1) 5093 return true; 5094 5095 /* over-swapping can increase allocation latency */ 5096 if (sc->nr_reclaimed >= sc->nr_to_reclaim && need_swapping) 5097 return true; 5098 5099 /* give this thread a chance to exit and free its memory */ 5100 if (fatal_signal_pending(current)) { 5101 sc->nr_reclaimed += MIN_LRU_BATCH; 5102 return true; 5103 } 5104 5105 if (cgroup_reclaim(sc)) 5106 return false; 5107 } else if (sc->nr_reclaimed - sc->last_reclaimed < sc->nr_to_reclaim) 5108 return false; 5109 5110 /* keep scanning at low priorities to ensure fairness */ 5111 if (sc->priority > DEF_PRIORITY - 2) 5112 return false; 5113 5114 /* 5115 * A minimum amount of work was done under global memory pressure. For 5116 * kswapd, it may be overshooting. For direct reclaim, the target isn't 5117 * met, and yet the allocation may still succeed, since kswapd may have 5118 * caught up. In either case, it's better to stop now, and restart if 5119 * necessary. 5120 */ 5121 for (i = 0; i <= sc->reclaim_idx; i++) { 5122 unsigned long wmark; 5123 struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i; 5124 5125 if (!managed_zone(zone)) 5126 continue; 5127 5128 wmark = current_is_kswapd() ? high_wmark_pages(zone) : low_wmark_pages(zone); 5129 if (wmark > zone_page_state(zone, NR_FREE_PAGES)) 5130 return false; 5131 } 5132 5133 sc->nr_reclaimed += MIN_LRU_BATCH; 5134 5135 return true; 5136 } 5137 5138 static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5139 { 5140 struct blk_plug plug; 5141 bool need_aging = false; 5142 bool need_swapping = false; 5143 unsigned long scanned = 0; 5144 unsigned long reclaimed = sc->nr_reclaimed; 5145 DEFINE_MAX_SEQ(lruvec); 5146 5147 lru_add_drain(); 5148 5149 blk_start_plug(&plug); 5150 5151 set_mm_walk(lruvec_pgdat(lruvec)); 5152 5153 while (true) { 5154 int delta; 5155 int swappiness; 5156 unsigned long nr_to_scan; 5157 5158 if (sc->may_swap) 5159 swappiness = get_swappiness(lruvec, sc); 5160 else if (!cgroup_reclaim(sc) && get_swappiness(lruvec, sc)) 5161 swappiness = 1; 5162 else 5163 swappiness = 0; 5164 5165 nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness, &need_aging); 5166 if (!nr_to_scan) 5167 goto done; 5168 5169 delta = evict_folios(lruvec, sc, swappiness, &need_swapping); 5170 if (!delta) 5171 goto done; 5172 5173 scanned += delta; 5174 if (scanned >= nr_to_scan) 5175 break; 5176 5177 if (should_abort_scan(lruvec, max_seq, sc, need_swapping)) 5178 break; 5179 5180 cond_resched(); 5181 } 5182 5183 /* see the comment in lru_gen_age_node() */ 5184 if (sc->nr_reclaimed - reclaimed >= MIN_LRU_BATCH && !need_aging) 5185 sc->memcgs_need_aging = false; 5186 done: 5187 clear_mm_walk(); 5188 5189 blk_finish_plug(&plug); 5190 } 5191 5192 /****************************************************************************** 5193 * state change 5194 ******************************************************************************/ 5195 5196 static bool __maybe_unused state_is_valid(struct lruvec *lruvec) 5197 { 5198 struct lru_gen_struct *lrugen = &lruvec->lrugen; 5199 5200 if (lrugen->enabled) { 5201 enum lru_list lru; 5202 5203 for_each_evictable_lru(lru) { 5204 if (!list_empty(&lruvec->lists[lru])) 5205 return false; 5206 } 5207 } else { 5208 int gen, type, zone; 5209 5210 for_each_gen_type_zone(gen, type, zone) { 5211 if (!list_empty(&lrugen->lists[gen][type][zone])) 5212 return false; 5213 } 5214 } 5215 5216 return true; 5217 } 5218 5219 static bool fill_evictable(struct lruvec *lruvec) 5220 { 5221 enum lru_list lru; 5222 int remaining = MAX_LRU_BATCH; 5223 5224 for_each_evictable_lru(lru) { 5225 int type = is_file_lru(lru); 5226 bool active = is_active_lru(lru); 5227 struct list_head *head = &lruvec->lists[lru]; 5228 5229 while (!list_empty(head)) { 5230 bool success; 5231 struct folio *folio = lru_to_folio(head); 5232 5233 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 5234 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); 5235 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 5236 VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); 5237 5238 lruvec_del_folio(lruvec, folio); 5239 success = lru_gen_add_folio(lruvec, folio, false); 5240 VM_WARN_ON_ONCE(!success); 5241 5242 if (!--remaining) 5243 return false; 5244 } 5245 } 5246 5247 return true; 5248 } 5249 5250 static bool drain_evictable(struct lruvec *lruvec) 5251 { 5252 int gen, type, zone; 5253 int remaining = MAX_LRU_BATCH; 5254 5255 for_each_gen_type_zone(gen, type, zone) { 5256 struct list_head *head = &lruvec->lrugen.lists[gen][type][zone]; 5257 5258 while (!list_empty(head)) { 5259 bool success; 5260 struct folio *folio = lru_to_folio(head); 5261 5262 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 5263 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 5264 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 5265 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 5266 5267 success = lru_gen_del_folio(lruvec, folio, false); 5268 VM_WARN_ON_ONCE(!success); 5269 lruvec_add_folio(lruvec, folio); 5270 5271 if (!--remaining) 5272 return false; 5273 } 5274 } 5275 5276 return true; 5277 } 5278 5279 static void lru_gen_change_state(bool enabled) 5280 { 5281 static DEFINE_MUTEX(state_mutex); 5282 5283 struct mem_cgroup *memcg; 5284 5285 cgroup_lock(); 5286 cpus_read_lock(); 5287 get_online_mems(); 5288 mutex_lock(&state_mutex); 5289 5290 if (enabled == lru_gen_enabled()) 5291 goto unlock; 5292 5293 if (enabled) 5294 static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); 5295 else 5296 static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); 5297 5298 memcg = mem_cgroup_iter(NULL, NULL, NULL); 5299 do { 5300 int nid; 5301 5302 for_each_node(nid) { 5303 struct lruvec *lruvec = get_lruvec(memcg, nid); 5304 5305 if (!lruvec) 5306 continue; 5307 5308 spin_lock_irq(&lruvec->lru_lock); 5309 5310 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 5311 VM_WARN_ON_ONCE(!state_is_valid(lruvec)); 5312 5313 lruvec->lrugen.enabled = enabled; 5314 5315 while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { 5316 spin_unlock_irq(&lruvec->lru_lock); 5317 cond_resched(); 5318 spin_lock_irq(&lruvec->lru_lock); 5319 } 5320 5321 spin_unlock_irq(&lruvec->lru_lock); 5322 } 5323 5324 cond_resched(); 5325 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 5326 unlock: 5327 mutex_unlock(&state_mutex); 5328 put_online_mems(); 5329 cpus_read_unlock(); 5330 cgroup_unlock(); 5331 } 5332 5333 /****************************************************************************** 5334 * sysfs interface 5335 ******************************************************************************/ 5336 5337 static ssize_t show_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, char *buf) 5338 { 5339 return sprintf(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); 5340 } 5341 5342 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5343 static ssize_t store_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, 5344 const char *buf, size_t len) 5345 { 5346 unsigned int msecs; 5347 5348 if (kstrtouint(buf, 0, &msecs)) 5349 return -EINVAL; 5350 5351 WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs)); 5352 5353 return len; 5354 } 5355 5356 static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR( 5357 min_ttl_ms, 0644, show_min_ttl, store_min_ttl 5358 ); 5359 5360 static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, char *buf) 5361 { 5362 unsigned int caps = 0; 5363 5364 if (get_cap(LRU_GEN_CORE)) 5365 caps |= BIT(LRU_GEN_CORE); 5366 5367 if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK)) 5368 caps |= BIT(LRU_GEN_MM_WALK); 5369 5370 if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && get_cap(LRU_GEN_NONLEAF_YOUNG)) 5371 caps |= BIT(LRU_GEN_NONLEAF_YOUNG); 5372 5373 return snprintf(buf, PAGE_SIZE, "0x%04x\n", caps); 5374 } 5375 5376 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5377 static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr, 5378 const char *buf, size_t len) 5379 { 5380 int i; 5381 unsigned int caps; 5382 5383 if (tolower(*buf) == 'n') 5384 caps = 0; 5385 else if (tolower(*buf) == 'y') 5386 caps = -1; 5387 else if (kstrtouint(buf, 0, &caps)) 5388 return -EINVAL; 5389 5390 for (i = 0; i < NR_LRU_GEN_CAPS; i++) { 5391 bool enabled = caps & BIT(i); 5392 5393 if (i == LRU_GEN_CORE) 5394 lru_gen_change_state(enabled); 5395 else if (enabled) 5396 static_branch_enable(&lru_gen_caps[i]); 5397 else 5398 static_branch_disable(&lru_gen_caps[i]); 5399 } 5400 5401 return len; 5402 } 5403 5404 static struct kobj_attribute lru_gen_enabled_attr = __ATTR( 5405 enabled, 0644, show_enabled, store_enabled 5406 ); 5407 5408 static struct attribute *lru_gen_attrs[] = { 5409 &lru_gen_min_ttl_attr.attr, 5410 &lru_gen_enabled_attr.attr, 5411 NULL 5412 }; 5413 5414 static struct attribute_group lru_gen_attr_group = { 5415 .name = "lru_gen", 5416 .attrs = lru_gen_attrs, 5417 }; 5418 5419 /****************************************************************************** 5420 * debugfs interface 5421 ******************************************************************************/ 5422 5423 static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) 5424 { 5425 struct mem_cgroup *memcg; 5426 loff_t nr_to_skip = *pos; 5427 5428 m->private = kvmalloc(PATH_MAX, GFP_KERNEL); 5429 if (!m->private) 5430 return ERR_PTR(-ENOMEM); 5431 5432 memcg = mem_cgroup_iter(NULL, NULL, NULL); 5433 do { 5434 int nid; 5435 5436 for_each_node_state(nid, N_MEMORY) { 5437 if (!nr_to_skip--) 5438 return get_lruvec(memcg, nid); 5439 } 5440 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 5441 5442 return NULL; 5443 } 5444 5445 static void lru_gen_seq_stop(struct seq_file *m, void *v) 5446 { 5447 if (!IS_ERR_OR_NULL(v)) 5448 mem_cgroup_iter_break(NULL, lruvec_memcg(v)); 5449 5450 kvfree(m->private); 5451 m->private = NULL; 5452 } 5453 5454 static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) 5455 { 5456 int nid = lruvec_pgdat(v)->node_id; 5457 struct mem_cgroup *memcg = lruvec_memcg(v); 5458 5459 ++*pos; 5460 5461 nid = next_memory_node(nid); 5462 if (nid == MAX_NUMNODES) { 5463 memcg = mem_cgroup_iter(NULL, memcg, NULL); 5464 if (!memcg) 5465 return NULL; 5466 5467 nid = first_memory_node; 5468 } 5469 5470 return get_lruvec(memcg, nid); 5471 } 5472 5473 static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, 5474 unsigned long max_seq, unsigned long *min_seq, 5475 unsigned long seq) 5476 { 5477 int i; 5478 int type, tier; 5479 int hist = lru_hist_from_seq(seq); 5480 struct lru_gen_struct *lrugen = &lruvec->lrugen; 5481 5482 for (tier = 0; tier < MAX_NR_TIERS; tier++) { 5483 seq_printf(m, " %10d", tier); 5484 for (type = 0; type < ANON_AND_FILE; type++) { 5485 const char *s = " "; 5486 unsigned long n[3] = {}; 5487 5488 if (seq == max_seq) { 5489 s = "RT "; 5490 n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); 5491 n[1] = READ_ONCE(lrugen->avg_total[type][tier]); 5492 } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { 5493 s = "rep"; 5494 n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); 5495 n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); 5496 if (tier) 5497 n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]); 5498 } 5499 5500 for (i = 0; i < 3; i++) 5501 seq_printf(m, " %10lu%c", n[i], s[i]); 5502 } 5503 seq_putc(m, '\n'); 5504 } 5505 5506 seq_puts(m, " "); 5507 for (i = 0; i < NR_MM_STATS; i++) { 5508 const char *s = " "; 5509 unsigned long n = 0; 5510 5511 if (seq == max_seq && NR_HIST_GENS == 1) { 5512 s = "LOYNFA"; 5513 n = READ_ONCE(lruvec->mm_state.stats[hist][i]); 5514 } else if (seq != max_seq && NR_HIST_GENS > 1) { 5515 s = "loynfa"; 5516 n = READ_ONCE(lruvec->mm_state.stats[hist][i]); 5517 } 5518 5519 seq_printf(m, " %10lu%c", n, s[i]); 5520 } 5521 seq_putc(m, '\n'); 5522 } 5523 5524 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5525 static int lru_gen_seq_show(struct seq_file *m, void *v) 5526 { 5527 unsigned long seq; 5528 bool full = !debugfs_real_fops(m->file)->write; 5529 struct lruvec *lruvec = v; 5530 struct lru_gen_struct *lrugen = &lruvec->lrugen; 5531 int nid = lruvec_pgdat(lruvec)->node_id; 5532 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 5533 DEFINE_MAX_SEQ(lruvec); 5534 DEFINE_MIN_SEQ(lruvec); 5535 5536 if (nid == first_memory_node) { 5537 const char *path = memcg ? m->private : ""; 5538 5539 #ifdef CONFIG_MEMCG 5540 if (memcg) 5541 cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); 5542 #endif 5543 seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path); 5544 } 5545 5546 seq_printf(m, " node %5d\n", nid); 5547 5548 if (!full) 5549 seq = min_seq[LRU_GEN_ANON]; 5550 else if (max_seq >= MAX_NR_GENS) 5551 seq = max_seq - MAX_NR_GENS + 1; 5552 else 5553 seq = 0; 5554 5555 for (; seq <= max_seq; seq++) { 5556 int type, zone; 5557 int gen = lru_gen_from_seq(seq); 5558 unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); 5559 5560 seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); 5561 5562 for (type = 0; type < ANON_AND_FILE; type++) { 5563 unsigned long size = 0; 5564 char mark = full && seq < min_seq[type] ? 'x' : ' '; 5565 5566 for (zone = 0; zone < MAX_NR_ZONES; zone++) 5567 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); 5568 5569 seq_printf(m, " %10lu%c", size, mark); 5570 } 5571 5572 seq_putc(m, '\n'); 5573 5574 if (full) 5575 lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); 5576 } 5577 5578 return 0; 5579 } 5580 5581 static const struct seq_operations lru_gen_seq_ops = { 5582 .start = lru_gen_seq_start, 5583 .stop = lru_gen_seq_stop, 5584 .next = lru_gen_seq_next, 5585 .show = lru_gen_seq_show, 5586 }; 5587 5588 static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, 5589 bool can_swap, bool force_scan) 5590 { 5591 DEFINE_MAX_SEQ(lruvec); 5592 DEFINE_MIN_SEQ(lruvec); 5593 5594 if (seq < max_seq) 5595 return 0; 5596 5597 if (seq > max_seq) 5598 return -EINVAL; 5599 5600 if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq) 5601 return -ERANGE; 5602 5603 try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan); 5604 5605 return 0; 5606 } 5607 5608 static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, 5609 int swappiness, unsigned long nr_to_reclaim) 5610 { 5611 DEFINE_MAX_SEQ(lruvec); 5612 5613 if (seq + MIN_NR_GENS > max_seq) 5614 return -EINVAL; 5615 5616 sc->nr_reclaimed = 0; 5617 5618 while (!signal_pending(current)) { 5619 DEFINE_MIN_SEQ(lruvec); 5620 5621 if (seq < min_seq[!swappiness]) 5622 return 0; 5623 5624 if (sc->nr_reclaimed >= nr_to_reclaim) 5625 return 0; 5626 5627 if (!evict_folios(lruvec, sc, swappiness, NULL)) 5628 return 0; 5629 5630 cond_resched(); 5631 } 5632 5633 return -EINTR; 5634 } 5635 5636 static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, 5637 struct scan_control *sc, int swappiness, unsigned long opt) 5638 { 5639 struct lruvec *lruvec; 5640 int err = -EINVAL; 5641 struct mem_cgroup *memcg = NULL; 5642 5643 if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) 5644 return -EINVAL; 5645 5646 if (!mem_cgroup_disabled()) { 5647 rcu_read_lock(); 5648 memcg = mem_cgroup_from_id(memcg_id); 5649 #ifdef CONFIG_MEMCG 5650 if (memcg && !css_tryget(&memcg->css)) 5651 memcg = NULL; 5652 #endif 5653 rcu_read_unlock(); 5654 5655 if (!memcg) 5656 return -EINVAL; 5657 } 5658 5659 if (memcg_id != mem_cgroup_id(memcg)) 5660 goto done; 5661 5662 lruvec = get_lruvec(memcg, nid); 5663 5664 if (swappiness < 0) 5665 swappiness = get_swappiness(lruvec, sc); 5666 else if (swappiness > 200) 5667 goto done; 5668 5669 switch (cmd) { 5670 case '+': 5671 err = run_aging(lruvec, seq, sc, swappiness, opt); 5672 break; 5673 case '-': 5674 err = run_eviction(lruvec, seq, sc, swappiness, opt); 5675 break; 5676 } 5677 done: 5678 mem_cgroup_put(memcg); 5679 5680 return err; 5681 } 5682 5683 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5684 static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, 5685 size_t len, loff_t *pos) 5686 { 5687 void *buf; 5688 char *cur, *next; 5689 unsigned int flags; 5690 struct blk_plug plug; 5691 int err = -EINVAL; 5692 struct scan_control sc = { 5693 .may_writepage = true, 5694 .may_unmap = true, 5695 .may_swap = true, 5696 .reclaim_idx = MAX_NR_ZONES - 1, 5697 .gfp_mask = GFP_KERNEL, 5698 }; 5699 5700 buf = kvmalloc(len + 1, GFP_KERNEL); 5701 if (!buf) 5702 return -ENOMEM; 5703 5704 if (copy_from_user(buf, src, len)) { 5705 kvfree(buf); 5706 return -EFAULT; 5707 } 5708 5709 set_task_reclaim_state(current, &sc.reclaim_state); 5710 flags = memalloc_noreclaim_save(); 5711 blk_start_plug(&plug); 5712 if (!set_mm_walk(NULL)) { 5713 err = -ENOMEM; 5714 goto done; 5715 } 5716 5717 next = buf; 5718 next[len] = '\0'; 5719 5720 while ((cur = strsep(&next, ",;\n"))) { 5721 int n; 5722 int end; 5723 char cmd; 5724 unsigned int memcg_id; 5725 unsigned int nid; 5726 unsigned long seq; 5727 unsigned int swappiness = -1; 5728 unsigned long opt = -1; 5729 5730 cur = skip_spaces(cur); 5731 if (!*cur) 5732 continue; 5733 5734 n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid, 5735 &seq, &end, &swappiness, &end, &opt, &end); 5736 if (n < 4 || cur[end]) { 5737 err = -EINVAL; 5738 break; 5739 } 5740 5741 err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt); 5742 if (err) 5743 break; 5744 } 5745 done: 5746 clear_mm_walk(); 5747 blk_finish_plug(&plug); 5748 memalloc_noreclaim_restore(flags); 5749 set_task_reclaim_state(current, NULL); 5750 5751 kvfree(buf); 5752 5753 return err ? : len; 5754 } 5755 5756 static int lru_gen_seq_open(struct inode *inode, struct file *file) 5757 { 5758 return seq_open(file, &lru_gen_seq_ops); 5759 } 5760 5761 static const struct file_operations lru_gen_rw_fops = { 5762 .open = lru_gen_seq_open, 5763 .read = seq_read, 5764 .write = lru_gen_seq_write, 5765 .llseek = seq_lseek, 5766 .release = seq_release, 5767 }; 5768 5769 static const struct file_operations lru_gen_ro_fops = { 5770 .open = lru_gen_seq_open, 5771 .read = seq_read, 5772 .llseek = seq_lseek, 5773 .release = seq_release, 5774 }; 5775 5776 /****************************************************************************** 5777 * initialization 5778 ******************************************************************************/ 5779 5780 void lru_gen_init_lruvec(struct lruvec *lruvec) 5781 { 5782 int i; 5783 int gen, type, zone; 5784 struct lru_gen_struct *lrugen = &lruvec->lrugen; 5785 5786 lrugen->max_seq = MIN_NR_GENS + 1; 5787 lrugen->enabled = lru_gen_enabled(); 5788 5789 for (i = 0; i <= MIN_NR_GENS + 1; i++) 5790 lrugen->timestamps[i] = jiffies; 5791 5792 for_each_gen_type_zone(gen, type, zone) 5793 INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]); 5794 5795 lruvec->mm_state.seq = MIN_NR_GENS; 5796 init_waitqueue_head(&lruvec->mm_state.wait); 5797 } 5798 5799 #ifdef CONFIG_MEMCG 5800 void lru_gen_init_memcg(struct mem_cgroup *memcg) 5801 { 5802 INIT_LIST_HEAD(&memcg->mm_list.fifo); 5803 spin_lock_init(&memcg->mm_list.lock); 5804 } 5805 5806 void lru_gen_exit_memcg(struct mem_cgroup *memcg) 5807 { 5808 int i; 5809 int nid; 5810 5811 for_each_node(nid) { 5812 struct lruvec *lruvec = get_lruvec(memcg, nid); 5813 5814 VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, 5815 sizeof(lruvec->lrugen.nr_pages))); 5816 5817 for (i = 0; i < NR_BLOOM_FILTERS; i++) { 5818 bitmap_free(lruvec->mm_state.filters[i]); 5819 lruvec->mm_state.filters[i] = NULL; 5820 } 5821 } 5822 } 5823 #endif 5824 5825 static int __init init_lru_gen(void) 5826 { 5827 BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); 5828 BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); 5829 5830 if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) 5831 pr_err("lru_gen: failed to create sysfs group\n"); 5832 5833 debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops); 5834 debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops); 5835 5836 return 0; 5837 }; 5838 late_initcall(init_lru_gen); 5839 5840 #else /* !CONFIG_LRU_GEN */ 5841 5842 static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) 5843 { 5844 } 5845 5846 static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5847 { 5848 } 5849 5850 #endif /* CONFIG_LRU_GEN */ 5851 5852 static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5853 { 5854 unsigned long nr[NR_LRU_LISTS]; 5855 unsigned long targets[NR_LRU_LISTS]; 5856 unsigned long nr_to_scan; 5857 enum lru_list lru; 5858 unsigned long nr_reclaimed = 0; 5859 unsigned long nr_to_reclaim = sc->nr_to_reclaim; 5860 struct blk_plug plug; 5861 bool scan_adjusted; 5862 5863 if (lru_gen_enabled()) { 5864 lru_gen_shrink_lruvec(lruvec, sc); 5865 return; 5866 } 5867 5868 get_scan_count(lruvec, sc, nr); 5869 5870 /* Record the original scan target for proportional adjustments later */ 5871 memcpy(targets, nr, sizeof(nr)); 5872 5873 /* 5874 * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal 5875 * event that can occur when there is little memory pressure e.g. 5876 * multiple streaming readers/writers. Hence, we do not abort scanning 5877 * when the requested number of pages are reclaimed when scanning at 5878 * DEF_PRIORITY on the assumption that the fact we are direct 5879 * reclaiming implies that kswapd is not keeping up and it is best to 5880 * do a batch of work at once. For memcg reclaim one check is made to 5881 * abort proportional reclaim if either the file or anon lru has already 5882 * dropped to zero at the first pass. 5883 */ 5884 scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() && 5885 sc->priority == DEF_PRIORITY); 5886 5887 blk_start_plug(&plug); 5888 while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || 5889 nr[LRU_INACTIVE_FILE]) { 5890 unsigned long nr_anon, nr_file, percentage; 5891 unsigned long nr_scanned; 5892 5893 for_each_evictable_lru(lru) { 5894 if (nr[lru]) { 5895 nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); 5896 nr[lru] -= nr_to_scan; 5897 5898 nr_reclaimed += shrink_list(lru, nr_to_scan, 5899 lruvec, sc); 5900 } 5901 } 5902 5903 cond_resched(); 5904 5905 if (nr_reclaimed < nr_to_reclaim || scan_adjusted) 5906 continue; 5907 5908 /* 5909 * For kswapd and memcg, reclaim at least the number of pages 5910 * requested. Ensure that the anon and file LRUs are scanned 5911 * proportionally what was requested by get_scan_count(). We 5912 * stop reclaiming one LRU and reduce the amount scanning 5913 * proportional to the original scan target. 5914 */ 5915 nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; 5916 nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; 5917 5918 /* 5919 * It's just vindictive to attack the larger once the smaller 5920 * has gone to zero. And given the way we stop scanning the 5921 * smaller below, this makes sure that we only make one nudge 5922 * towards proportionality once we've got nr_to_reclaim. 5923 */ 5924 if (!nr_file || !nr_anon) 5925 break; 5926 5927 if (nr_file > nr_anon) { 5928 unsigned long scan_target = targets[LRU_INACTIVE_ANON] + 5929 targets[LRU_ACTIVE_ANON] + 1; 5930 lru = LRU_BASE; 5931 percentage = nr_anon * 100 / scan_target; 5932 } else { 5933 unsigned long scan_target = targets[LRU_INACTIVE_FILE] + 5934 targets[LRU_ACTIVE_FILE] + 1; 5935 lru = LRU_FILE; 5936 percentage = nr_file * 100 / scan_target; 5937 } 5938 5939 /* Stop scanning the smaller of the LRU */ 5940 nr[lru] = 0; 5941 nr[lru + LRU_ACTIVE] = 0; 5942 5943 /* 5944 * Recalculate the other LRU scan count based on its original 5945 * scan target and the percentage scanning already complete 5946 */ 5947 lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; 5948 nr_scanned = targets[lru] - nr[lru]; 5949 nr[lru] = targets[lru] * (100 - percentage) / 100; 5950 nr[lru] -= min(nr[lru], nr_scanned); 5951 5952 lru += LRU_ACTIVE; 5953 nr_scanned = targets[lru] - nr[lru]; 5954 nr[lru] = targets[lru] * (100 - percentage) / 100; 5955 nr[lru] -= min(nr[lru], nr_scanned); 5956 5957 scan_adjusted = true; 5958 } 5959 blk_finish_plug(&plug); 5960 sc->nr_reclaimed += nr_reclaimed; 5961 5962 /* 5963 * Even if we did not try to evict anon pages at all, we want to 5964 * rebalance the anon lru active/inactive ratio. 5965 */ 5966 if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && 5967 inactive_is_low(lruvec, LRU_INACTIVE_ANON)) 5968 shrink_active_list(SWAP_CLUSTER_MAX, lruvec, 5969 sc, LRU_ACTIVE_ANON); 5970 } 5971 5972 /* Use reclaim/compaction for costly allocs or under memory pressure */ 5973 static bool in_reclaim_compaction(struct scan_control *sc) 5974 { 5975 if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && 5976 (sc->order > PAGE_ALLOC_COSTLY_ORDER || 5977 sc->priority < DEF_PRIORITY - 2)) 5978 return true; 5979 5980 return false; 5981 } 5982 5983 /* 5984 * Reclaim/compaction is used for high-order allocation requests. It reclaims 5985 * order-0 pages before compacting the zone. should_continue_reclaim() returns 5986 * true if more pages should be reclaimed such that when the page allocator 5987 * calls try_to_compact_pages() that it will have enough free pages to succeed. 5988 * It will give up earlier than that if there is difficulty reclaiming pages. 5989 */ 5990 static inline bool should_continue_reclaim(struct pglist_data *pgdat, 5991 unsigned long nr_reclaimed, 5992 struct scan_control *sc) 5993 { 5994 unsigned long pages_for_compaction; 5995 unsigned long inactive_lru_pages; 5996 int z; 5997 5998 /* If not in reclaim/compaction mode, stop */ 5999 if (!in_reclaim_compaction(sc)) 6000 return false; 6001 6002 /* 6003 * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX 6004 * number of pages that were scanned. This will return to the caller 6005 * with the risk reclaim/compaction and the resulting allocation attempt 6006 * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL 6007 * allocations through requiring that the full LRU list has been scanned 6008 * first, by assuming that zero delta of sc->nr_scanned means full LRU 6009 * scan, but that approximation was wrong, and there were corner cases 6010 * where always a non-zero amount of pages were scanned. 6011 */ 6012 if (!nr_reclaimed) 6013 return false; 6014 6015 /* If compaction would go ahead or the allocation would succeed, stop */ 6016 for (z = 0; z <= sc->reclaim_idx; z++) { 6017 struct zone *zone = &pgdat->node_zones[z]; 6018 if (!managed_zone(zone)) 6019 continue; 6020 6021 switch (compaction_suitable(zone, sc->order, 0, sc->reclaim_idx)) { 6022 case COMPACT_SUCCESS: 6023 case COMPACT_CONTINUE: 6024 return false; 6025 default: 6026 /* check next zone */ 6027 ; 6028 } 6029 } 6030 6031 /* 6032 * If we have not reclaimed enough pages for compaction and the 6033 * inactive lists are large enough, continue reclaiming 6034 */ 6035 pages_for_compaction = compact_gap(sc->order); 6036 inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE); 6037 if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc)) 6038 inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON); 6039 6040 return inactive_lru_pages > pages_for_compaction; 6041 } 6042 6043 static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc) 6044 { 6045 struct mem_cgroup *target_memcg = sc->target_mem_cgroup; 6046 struct mem_cgroup *memcg; 6047 6048 memcg = mem_cgroup_iter(target_memcg, NULL, NULL); 6049 do { 6050 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 6051 unsigned long reclaimed; 6052 unsigned long scanned; 6053 6054 /* 6055 * This loop can become CPU-bound when target memcgs 6056 * aren't eligible for reclaim - either because they 6057 * don't have any reclaimable pages, or because their 6058 * memory is explicitly protected. Avoid soft lockups. 6059 */ 6060 cond_resched(); 6061 6062 mem_cgroup_calculate_protection(target_memcg, memcg); 6063 6064 if (mem_cgroup_below_min(memcg)) { 6065 /* 6066 * Hard protection. 6067 * If there is no reclaimable memory, OOM. 6068 */ 6069 continue; 6070 } else if (mem_cgroup_below_low(memcg)) { 6071 /* 6072 * Soft protection. 6073 * Respect the protection only as long as 6074 * there is an unprotected supply 6075 * of reclaimable memory from other cgroups. 6076 */ 6077 if (!sc->memcg_low_reclaim) { 6078 sc->memcg_low_skipped = 1; 6079 continue; 6080 } 6081 memcg_memory_event(memcg, MEMCG_LOW); 6082 } 6083 6084 reclaimed = sc->nr_reclaimed; 6085 scanned = sc->nr_scanned; 6086 6087 shrink_lruvec(lruvec, sc); 6088 6089 shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, 6090 sc->priority); 6091 6092 /* Record the group's reclaim efficiency */ 6093 if (!sc->proactive) 6094 vmpressure(sc->gfp_mask, memcg, false, 6095 sc->nr_scanned - scanned, 6096 sc->nr_reclaimed - reclaimed); 6097 6098 } while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL))); 6099 } 6100 6101 static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) 6102 { 6103 struct reclaim_state *reclaim_state = current->reclaim_state; 6104 unsigned long nr_reclaimed, nr_scanned; 6105 struct lruvec *target_lruvec; 6106 bool reclaimable = false; 6107 6108 target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); 6109 6110 again: 6111 memset(&sc->nr, 0, sizeof(sc->nr)); 6112 6113 nr_reclaimed = sc->nr_reclaimed; 6114 nr_scanned = sc->nr_scanned; 6115 6116 prepare_scan_count(pgdat, sc); 6117 6118 shrink_node_memcgs(pgdat, sc); 6119 6120 if (reclaim_state) { 6121 sc->nr_reclaimed += reclaim_state->reclaimed_slab; 6122 reclaim_state->reclaimed_slab = 0; 6123 } 6124 6125 /* Record the subtree's reclaim efficiency */ 6126 if (!sc->proactive) 6127 vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true, 6128 sc->nr_scanned - nr_scanned, 6129 sc->nr_reclaimed - nr_reclaimed); 6130 6131 if (sc->nr_reclaimed - nr_reclaimed) 6132 reclaimable = true; 6133 6134 if (current_is_kswapd()) { 6135 /* 6136 * If reclaim is isolating dirty pages under writeback, 6137 * it implies that the long-lived page allocation rate 6138 * is exceeding the page laundering rate. Either the 6139 * global limits are not being effective at throttling 6140 * processes due to the page distribution throughout 6141 * zones or there is heavy usage of a slow backing 6142 * device. The only option is to throttle from reclaim 6143 * context which is not ideal as there is no guarantee 6144 * the dirtying process is throttled in the same way 6145 * balance_dirty_pages() manages. 6146 * 6147 * Once a node is flagged PGDAT_WRITEBACK, kswapd will 6148 * count the number of pages under pages flagged for 6149 * immediate reclaim and stall if any are encountered 6150 * in the nr_immediate check below. 6151 */ 6152 if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken) 6153 set_bit(PGDAT_WRITEBACK, &pgdat->flags); 6154 6155 /* Allow kswapd to start writing pages during reclaim.*/ 6156 if (sc->nr.unqueued_dirty == sc->nr.file_taken) 6157 set_bit(PGDAT_DIRTY, &pgdat->flags); 6158 6159 /* 6160 * If kswapd scans pages marked for immediate 6161 * reclaim and under writeback (nr_immediate), it 6162 * implies that pages are cycling through the LRU 6163 * faster than they are written so forcibly stall 6164 * until some pages complete writeback. 6165 */ 6166 if (sc->nr.immediate) 6167 reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK); 6168 } 6169 6170 /* 6171 * Tag a node/memcg as congested if all the dirty pages were marked 6172 * for writeback and immediate reclaim (counted in nr.congested). 6173 * 6174 * Legacy memcg will stall in page writeback so avoid forcibly 6175 * stalling in reclaim_throttle(). 6176 */ 6177 if ((current_is_kswapd() || 6178 (cgroup_reclaim(sc) && writeback_throttling_sane(sc))) && 6179 sc->nr.dirty && sc->nr.dirty == sc->nr.congested) 6180 set_bit(LRUVEC_CONGESTED, &target_lruvec->flags); 6181 6182 /* 6183 * Stall direct reclaim for IO completions if the lruvec is 6184 * node is congested. Allow kswapd to continue until it 6185 * starts encountering unqueued dirty pages or cycling through 6186 * the LRU too quickly. 6187 */ 6188 if (!current_is_kswapd() && current_may_throttle() && 6189 !sc->hibernation_mode && 6190 test_bit(LRUVEC_CONGESTED, &target_lruvec->flags)) 6191 reclaim_throttle(pgdat, VMSCAN_THROTTLE_CONGESTED); 6192 6193 if (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed, 6194 sc)) 6195 goto again; 6196 6197 /* 6198 * Kswapd gives up on balancing particular nodes after too 6199 * many failures to reclaim anything from them and goes to 6200 * sleep. On reclaim progress, reset the failure counter. A 6201 * successful direct reclaim run will revive a dormant kswapd. 6202 */ 6203 if (reclaimable) 6204 pgdat->kswapd_failures = 0; 6205 } 6206 6207 /* 6208 * Returns true if compaction should go ahead for a costly-order request, or 6209 * the allocation would already succeed without compaction. Return false if we 6210 * should reclaim first. 6211 */ 6212 static inline bool compaction_ready(struct zone *zone, struct scan_control *sc) 6213 { 6214 unsigned long watermark; 6215 enum compact_result suitable; 6216 6217 suitable = compaction_suitable(zone, sc->order, 0, sc->reclaim_idx); 6218 if (suitable == COMPACT_SUCCESS) 6219 /* Allocation should succeed already. Don't reclaim. */ 6220 return true; 6221 if (suitable == COMPACT_SKIPPED) 6222 /* Compaction cannot yet proceed. Do reclaim. */ 6223 return false; 6224 6225 /* 6226 * Compaction is already possible, but it takes time to run and there 6227 * are potentially other callers using the pages just freed. So proceed 6228 * with reclaim to make a buffer of free pages available to give 6229 * compaction a reasonable chance of completing and allocating the page. 6230 * Note that we won't actually reclaim the whole buffer in one attempt 6231 * as the target watermark in should_continue_reclaim() is lower. But if 6232 * we are already above the high+gap watermark, don't reclaim at all. 6233 */ 6234 watermark = high_wmark_pages(zone) + compact_gap(sc->order); 6235 6236 return zone_watermark_ok_safe(zone, 0, watermark, sc->reclaim_idx); 6237 } 6238 6239 static void consider_reclaim_throttle(pg_data_t *pgdat, struct scan_control *sc) 6240 { 6241 /* 6242 * If reclaim is making progress greater than 12% efficiency then 6243 * wake all the NOPROGRESS throttled tasks. 6244 */ 6245 if (sc->nr_reclaimed > (sc->nr_scanned >> 3)) { 6246 wait_queue_head_t *wqh; 6247 6248 wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_NOPROGRESS]; 6249 if (waitqueue_active(wqh)) 6250 wake_up(wqh); 6251 6252 return; 6253 } 6254 6255 /* 6256 * Do not throttle kswapd or cgroup reclaim on NOPROGRESS as it will 6257 * throttle on VMSCAN_THROTTLE_WRITEBACK if there are too many pages 6258 * under writeback and marked for immediate reclaim at the tail of the 6259 * LRU. 6260 */ 6261 if (current_is_kswapd() || cgroup_reclaim(sc)) 6262 return; 6263 6264 /* Throttle if making no progress at high prioities. */ 6265 if (sc->priority == 1 && !sc->nr_reclaimed) 6266 reclaim_throttle(pgdat, VMSCAN_THROTTLE_NOPROGRESS); 6267 } 6268 6269 /* 6270 * This is the direct reclaim path, for page-allocating processes. We only 6271 * try to reclaim pages from zones which will satisfy the caller's allocation 6272 * request. 6273 * 6274 * If a zone is deemed to be full of pinned pages then just give it a light 6275 * scan then give up on it. 6276 */ 6277 static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc) 6278 { 6279 struct zoneref *z; 6280 struct zone *zone; 6281 unsigned long nr_soft_reclaimed; 6282 unsigned long nr_soft_scanned; 6283 gfp_t orig_mask; 6284 pg_data_t *last_pgdat = NULL; 6285 pg_data_t *first_pgdat = NULL; 6286 6287 /* 6288 * If the number of buffer_heads in the machine exceeds the maximum 6289 * allowed level, force direct reclaim to scan the highmem zone as 6290 * highmem pages could be pinning lowmem pages storing buffer_heads 6291 */ 6292 orig_mask = sc->gfp_mask; 6293 if (buffer_heads_over_limit) { 6294 sc->gfp_mask |= __GFP_HIGHMEM; 6295 sc->reclaim_idx = gfp_zone(sc->gfp_mask); 6296 } 6297 6298 for_each_zone_zonelist_nodemask(zone, z, zonelist, 6299 sc->reclaim_idx, sc->nodemask) { 6300 /* 6301 * Take care memory controller reclaiming has small influence 6302 * to global LRU. 6303 */ 6304 if (!cgroup_reclaim(sc)) { 6305 if (!cpuset_zone_allowed(zone, 6306 GFP_KERNEL | __GFP_HARDWALL)) 6307 continue; 6308 6309 /* 6310 * If we already have plenty of memory free for 6311 * compaction in this zone, don't free any more. 6312 * Even though compaction is invoked for any 6313 * non-zero order, only frequent costly order 6314 * reclamation is disruptive enough to become a 6315 * noticeable problem, like transparent huge 6316 * page allocations. 6317 */ 6318 if (IS_ENABLED(CONFIG_COMPACTION) && 6319 sc->order > PAGE_ALLOC_COSTLY_ORDER && 6320 compaction_ready(zone, sc)) { 6321 sc->compaction_ready = true; 6322 continue; 6323 } 6324 6325 /* 6326 * Shrink each node in the zonelist once. If the 6327 * zonelist is ordered by zone (not the default) then a 6328 * node may be shrunk multiple times but in that case 6329 * the user prefers lower zones being preserved. 6330 */ 6331 if (zone->zone_pgdat == last_pgdat) 6332 continue; 6333 6334 /* 6335 * This steals pages from memory cgroups over softlimit 6336 * and returns the number of reclaimed pages and 6337 * scanned pages. This works for global memory pressure 6338 * and balancing, not for a memcg's limit. 6339 */ 6340 nr_soft_scanned = 0; 6341 nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone->zone_pgdat, 6342 sc->order, sc->gfp_mask, 6343 &nr_soft_scanned); 6344 sc->nr_reclaimed += nr_soft_reclaimed; 6345 sc->nr_scanned += nr_soft_scanned; 6346 /* need some check for avoid more shrink_zone() */ 6347 } 6348 6349 if (!first_pgdat) 6350 first_pgdat = zone->zone_pgdat; 6351 6352 /* See comment about same check for global reclaim above */ 6353 if (zone->zone_pgdat == last_pgdat) 6354 continue; 6355 last_pgdat = zone->zone_pgdat; 6356 shrink_node(zone->zone_pgdat, sc); 6357 } 6358 6359 if (first_pgdat) 6360 consider_reclaim_throttle(first_pgdat, sc); 6361 6362 /* 6363 * Restore to original mask to avoid the impact on the caller if we 6364 * promoted it to __GFP_HIGHMEM. 6365 */ 6366 sc->gfp_mask = orig_mask; 6367 } 6368 6369 static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) 6370 { 6371 struct lruvec *target_lruvec; 6372 unsigned long refaults; 6373 6374 if (lru_gen_enabled()) 6375 return; 6376 6377 target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat); 6378 refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON); 6379 target_lruvec->refaults[WORKINGSET_ANON] = refaults; 6380 refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE); 6381 target_lruvec->refaults[WORKINGSET_FILE] = refaults; 6382 } 6383 6384 /* 6385 * This is the main entry point to direct page reclaim. 6386 * 6387 * If a full scan of the inactive list fails to free enough memory then we 6388 * are "out of memory" and something needs to be killed. 6389 * 6390 * If the caller is !__GFP_FS then the probability of a failure is reasonably 6391 * high - the zone may be full of dirty or under-writeback pages, which this 6392 * caller can't do much about. We kick the writeback threads and take explicit 6393 * naps in the hope that some of these pages can be written. But if the 6394 * allocating task holds filesystem locks which prevent writeout this might not 6395 * work, and the allocation attempt will fail. 6396 * 6397 * returns: 0, if no pages reclaimed 6398 * else, the number of pages reclaimed 6399 */ 6400 static unsigned long do_try_to_free_pages(struct zonelist *zonelist, 6401 struct scan_control *sc) 6402 { 6403 int initial_priority = sc->priority; 6404 pg_data_t *last_pgdat; 6405 struct zoneref *z; 6406 struct zone *zone; 6407 retry: 6408 delayacct_freepages_start(); 6409 6410 if (!cgroup_reclaim(sc)) 6411 __count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1); 6412 6413 do { 6414 if (!sc->proactive) 6415 vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup, 6416 sc->priority); 6417 sc->nr_scanned = 0; 6418 shrink_zones(zonelist, sc); 6419 6420 if (sc->nr_reclaimed >= sc->nr_to_reclaim) 6421 break; 6422 6423 if (sc->compaction_ready) 6424 break; 6425 6426 /* 6427 * If we're getting trouble reclaiming, start doing 6428 * writepage even in laptop mode. 6429 */ 6430 if (sc->priority < DEF_PRIORITY - 2) 6431 sc->may_writepage = 1; 6432 } while (--sc->priority >= 0); 6433 6434 last_pgdat = NULL; 6435 for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx, 6436 sc->nodemask) { 6437 if (zone->zone_pgdat == last_pgdat) 6438 continue; 6439 last_pgdat = zone->zone_pgdat; 6440 6441 snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat); 6442 6443 if (cgroup_reclaim(sc)) { 6444 struct lruvec *lruvec; 6445 6446 lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, 6447 zone->zone_pgdat); 6448 clear_bit(LRUVEC_CONGESTED, &lruvec->flags); 6449 } 6450 } 6451 6452 delayacct_freepages_end(); 6453 6454 if (sc->nr_reclaimed) 6455 return sc->nr_reclaimed; 6456 6457 /* Aborted reclaim to try compaction? don't OOM, then */ 6458 if (sc->compaction_ready) 6459 return 1; 6460 6461 /* 6462 * We make inactive:active ratio decisions based on the node's 6463 * composition of memory, but a restrictive reclaim_idx or a 6464 * memory.low cgroup setting can exempt large amounts of 6465 * memory from reclaim. Neither of which are very common, so 6466 * instead of doing costly eligibility calculations of the 6467 * entire cgroup subtree up front, we assume the estimates are 6468 * good, and retry with forcible deactivation if that fails. 6469 */ 6470 if (sc->skipped_deactivate) { 6471 sc->priority = initial_priority; 6472 sc->force_deactivate = 1; 6473 sc->skipped_deactivate = 0; 6474 goto retry; 6475 } 6476 6477 /* Untapped cgroup reserves? Don't OOM, retry. */ 6478 if (sc->memcg_low_skipped) { 6479 sc->priority = initial_priority; 6480 sc->force_deactivate = 0; 6481 sc->memcg_low_reclaim = 1; 6482 sc->memcg_low_skipped = 0; 6483 goto retry; 6484 } 6485 6486 return 0; 6487 } 6488 6489 static bool allow_direct_reclaim(pg_data_t *pgdat) 6490 { 6491 struct zone *zone; 6492 unsigned long pfmemalloc_reserve = 0; 6493 unsigned long free_pages = 0; 6494 int i; 6495 bool wmark_ok; 6496 6497 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) 6498 return true; 6499 6500 for (i = 0; i <= ZONE_NORMAL; i++) { 6501 zone = &pgdat->node_zones[i]; 6502 if (!managed_zone(zone)) 6503 continue; 6504 6505 if (!zone_reclaimable_pages(zone)) 6506 continue; 6507 6508 pfmemalloc_reserve += min_wmark_pages(zone); 6509 free_pages += zone_page_state(zone, NR_FREE_PAGES); 6510 } 6511 6512 /* If there are no reserves (unexpected config) then do not throttle */ 6513 if (!pfmemalloc_reserve) 6514 return true; 6515 6516 wmark_ok = free_pages > pfmemalloc_reserve / 2; 6517 6518 /* kswapd must be awake if processes are being throttled */ 6519 if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) { 6520 if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL) 6521 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL); 6522 6523 wake_up_interruptible(&pgdat->kswapd_wait); 6524 } 6525 6526 return wmark_ok; 6527 } 6528 6529 /* 6530 * Throttle direct reclaimers if backing storage is backed by the network 6531 * and the PFMEMALLOC reserve for the preferred node is getting dangerously 6532 * depleted. kswapd will continue to make progress and wake the processes 6533 * when the low watermark is reached. 6534 * 6535 * Returns true if a fatal signal was delivered during throttling. If this 6536 * happens, the page allocator should not consider triggering the OOM killer. 6537 */ 6538 static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, 6539 nodemask_t *nodemask) 6540 { 6541 struct zoneref *z; 6542 struct zone *zone; 6543 pg_data_t *pgdat = NULL; 6544 6545 /* 6546 * Kernel threads should not be throttled as they may be indirectly 6547 * responsible for cleaning pages necessary for reclaim to make forward 6548 * progress. kjournald for example may enter direct reclaim while 6549 * committing a transaction where throttling it could forcing other 6550 * processes to block on log_wait_commit(). 6551 */ 6552 if (current->flags & PF_KTHREAD) 6553 goto out; 6554 6555 /* 6556 * If a fatal signal is pending, this process should not throttle. 6557 * It should return quickly so it can exit and free its memory 6558 */ 6559 if (fatal_signal_pending(current)) 6560 goto out; 6561 6562 /* 6563 * Check if the pfmemalloc reserves are ok by finding the first node 6564 * with a usable ZONE_NORMAL or lower zone. The expectation is that 6565 * GFP_KERNEL will be required for allocating network buffers when 6566 * swapping over the network so ZONE_HIGHMEM is unusable. 6567 * 6568 * Throttling is based on the first usable node and throttled processes 6569 * wait on a queue until kswapd makes progress and wakes them. There 6570 * is an affinity then between processes waking up and where reclaim 6571 * progress has been made assuming the process wakes on the same node. 6572 * More importantly, processes running on remote nodes will not compete 6573 * for remote pfmemalloc reserves and processes on different nodes 6574 * should make reasonable progress. 6575 */ 6576 for_each_zone_zonelist_nodemask(zone, z, zonelist, 6577 gfp_zone(gfp_mask), nodemask) { 6578 if (zone_idx(zone) > ZONE_NORMAL) 6579 continue; 6580 6581 /* Throttle based on the first usable node */ 6582 pgdat = zone->zone_pgdat; 6583 if (allow_direct_reclaim(pgdat)) 6584 goto out; 6585 break; 6586 } 6587 6588 /* If no zone was usable by the allocation flags then do not throttle */ 6589 if (!pgdat) 6590 goto out; 6591 6592 /* Account for the throttling */ 6593 count_vm_event(PGSCAN_DIRECT_THROTTLE); 6594 6595 /* 6596 * If the caller cannot enter the filesystem, it's possible that it 6597 * is due to the caller holding an FS lock or performing a journal 6598 * transaction in the case of a filesystem like ext[3|4]. In this case, 6599 * it is not safe to block on pfmemalloc_wait as kswapd could be 6600 * blocked waiting on the same lock. Instead, throttle for up to a 6601 * second before continuing. 6602 */ 6603 if (!(gfp_mask & __GFP_FS)) 6604 wait_event_interruptible_timeout(pgdat->pfmemalloc_wait, 6605 allow_direct_reclaim(pgdat), HZ); 6606 else 6607 /* Throttle until kswapd wakes the process */ 6608 wait_event_killable(zone->zone_pgdat->pfmemalloc_wait, 6609 allow_direct_reclaim(pgdat)); 6610 6611 if (fatal_signal_pending(current)) 6612 return true; 6613 6614 out: 6615 return false; 6616 } 6617 6618 unsigned long try_to_free_pages(struct zonelist *zonelist, int order, 6619 gfp_t gfp_mask, nodemask_t *nodemask) 6620 { 6621 unsigned long nr_reclaimed; 6622 struct scan_control sc = { 6623 .nr_to_reclaim = SWAP_CLUSTER_MAX, 6624 .gfp_mask = current_gfp_context(gfp_mask), 6625 .reclaim_idx = gfp_zone(gfp_mask), 6626 .order = order, 6627 .nodemask = nodemask, 6628 .priority = DEF_PRIORITY, 6629 .may_writepage = !laptop_mode, 6630 .may_unmap = 1, 6631 .may_swap = 1, 6632 }; 6633 6634 /* 6635 * scan_control uses s8 fields for order, priority, and reclaim_idx. 6636 * Confirm they are large enough for max values. 6637 */ 6638 BUILD_BUG_ON(MAX_ORDER > S8_MAX); 6639 BUILD_BUG_ON(DEF_PRIORITY > S8_MAX); 6640 BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX); 6641 6642 /* 6643 * Do not enter reclaim if fatal signal was delivered while throttled. 6644 * 1 is returned so that the page allocator does not OOM kill at this 6645 * point. 6646 */ 6647 if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask)) 6648 return 1; 6649 6650 set_task_reclaim_state(current, &sc.reclaim_state); 6651 trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask); 6652 6653 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 6654 6655 trace_mm_vmscan_direct_reclaim_end(nr_reclaimed); 6656 set_task_reclaim_state(current, NULL); 6657 6658 return nr_reclaimed; 6659 } 6660 6661 #ifdef CONFIG_MEMCG 6662 6663 /* Only used by soft limit reclaim. Do not reuse for anything else. */ 6664 unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, 6665 gfp_t gfp_mask, bool noswap, 6666 pg_data_t *pgdat, 6667 unsigned long *nr_scanned) 6668 { 6669 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 6670 struct scan_control sc = { 6671 .nr_to_reclaim = SWAP_CLUSTER_MAX, 6672 .target_mem_cgroup = memcg, 6673 .may_writepage = !laptop_mode, 6674 .may_unmap = 1, 6675 .reclaim_idx = MAX_NR_ZONES - 1, 6676 .may_swap = !noswap, 6677 }; 6678 6679 WARN_ON_ONCE(!current->reclaim_state); 6680 6681 sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | 6682 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK); 6683 6684 trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order, 6685 sc.gfp_mask); 6686 6687 /* 6688 * NOTE: Although we can get the priority field, using it 6689 * here is not a good idea, since it limits the pages we can scan. 6690 * if we don't reclaim here, the shrink_node from balance_pgdat 6691 * will pick up pages from other mem cgroup's as well. We hack 6692 * the priority and make it zero. 6693 */ 6694 shrink_lruvec(lruvec, &sc); 6695 6696 trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed); 6697 6698 *nr_scanned = sc.nr_scanned; 6699 6700 return sc.nr_reclaimed; 6701 } 6702 6703 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, 6704 unsigned long nr_pages, 6705 gfp_t gfp_mask, 6706 unsigned int reclaim_options) 6707 { 6708 unsigned long nr_reclaimed; 6709 unsigned int noreclaim_flag; 6710 struct scan_control sc = { 6711 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), 6712 .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) | 6713 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK), 6714 .reclaim_idx = MAX_NR_ZONES - 1, 6715 .target_mem_cgroup = memcg, 6716 .priority = DEF_PRIORITY, 6717 .may_writepage = !laptop_mode, 6718 .may_unmap = 1, 6719 .may_swap = !!(reclaim_options & MEMCG_RECLAIM_MAY_SWAP), 6720 .proactive = !!(reclaim_options & MEMCG_RECLAIM_PROACTIVE), 6721 }; 6722 /* 6723 * Traverse the ZONELIST_FALLBACK zonelist of the current node to put 6724 * equal pressure on all the nodes. This is based on the assumption that 6725 * the reclaim does not bail out early. 6726 */ 6727 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); 6728 6729 set_task_reclaim_state(current, &sc.reclaim_state); 6730 trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask); 6731 noreclaim_flag = memalloc_noreclaim_save(); 6732 6733 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 6734 6735 memalloc_noreclaim_restore(noreclaim_flag); 6736 trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed); 6737 set_task_reclaim_state(current, NULL); 6738 6739 return nr_reclaimed; 6740 } 6741 #endif 6742 6743 static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) 6744 { 6745 struct mem_cgroup *memcg; 6746 struct lruvec *lruvec; 6747 6748 if (lru_gen_enabled()) { 6749 lru_gen_age_node(pgdat, sc); 6750 return; 6751 } 6752 6753 if (!can_age_anon_pages(pgdat, sc)) 6754 return; 6755 6756 lruvec = mem_cgroup_lruvec(NULL, pgdat); 6757 if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON)) 6758 return; 6759 6760 memcg = mem_cgroup_iter(NULL, NULL, NULL); 6761 do { 6762 lruvec = mem_cgroup_lruvec(memcg, pgdat); 6763 shrink_active_list(SWAP_CLUSTER_MAX, lruvec, 6764 sc, LRU_ACTIVE_ANON); 6765 memcg = mem_cgroup_iter(NULL, memcg, NULL); 6766 } while (memcg); 6767 } 6768 6769 static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx) 6770 { 6771 int i; 6772 struct zone *zone; 6773 6774 /* 6775 * Check for watermark boosts top-down as the higher zones 6776 * are more likely to be boosted. Both watermarks and boosts 6777 * should not be checked at the same time as reclaim would 6778 * start prematurely when there is no boosting and a lower 6779 * zone is balanced. 6780 */ 6781 for (i = highest_zoneidx; i >= 0; i--) { 6782 zone = pgdat->node_zones + i; 6783 if (!managed_zone(zone)) 6784 continue; 6785 6786 if (zone->watermark_boost) 6787 return true; 6788 } 6789 6790 return false; 6791 } 6792 6793 /* 6794 * Returns true if there is an eligible zone balanced for the request order 6795 * and highest_zoneidx 6796 */ 6797 static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx) 6798 { 6799 int i; 6800 unsigned long mark = -1; 6801 struct zone *zone; 6802 6803 /* 6804 * Check watermarks bottom-up as lower zones are more likely to 6805 * meet watermarks. 6806 */ 6807 for (i = 0; i <= highest_zoneidx; i++) { 6808 zone = pgdat->node_zones + i; 6809 6810 if (!managed_zone(zone)) 6811 continue; 6812 6813 if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) 6814 mark = wmark_pages(zone, WMARK_PROMO); 6815 else 6816 mark = high_wmark_pages(zone); 6817 if (zone_watermark_ok_safe(zone, order, mark, highest_zoneidx)) 6818 return true; 6819 } 6820 6821 /* 6822 * If a node has no managed zone within highest_zoneidx, it does not 6823 * need balancing by definition. This can happen if a zone-restricted 6824 * allocation tries to wake a remote kswapd. 6825 */ 6826 if (mark == -1) 6827 return true; 6828 6829 return false; 6830 } 6831 6832 /* Clear pgdat state for congested, dirty or under writeback. */ 6833 static void clear_pgdat_congested(pg_data_t *pgdat) 6834 { 6835 struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat); 6836 6837 clear_bit(LRUVEC_CONGESTED, &lruvec->flags); 6838 clear_bit(PGDAT_DIRTY, &pgdat->flags); 6839 clear_bit(PGDAT_WRITEBACK, &pgdat->flags); 6840 } 6841 6842 /* 6843 * Prepare kswapd for sleeping. This verifies that there are no processes 6844 * waiting in throttle_direct_reclaim() and that watermarks have been met. 6845 * 6846 * Returns true if kswapd is ready to sleep 6847 */ 6848 static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, 6849 int highest_zoneidx) 6850 { 6851 /* 6852 * The throttled processes are normally woken up in balance_pgdat() as 6853 * soon as allow_direct_reclaim() is true. But there is a potential 6854 * race between when kswapd checks the watermarks and a process gets 6855 * throttled. There is also a potential race if processes get 6856 * throttled, kswapd wakes, a large process exits thereby balancing the 6857 * zones, which causes kswapd to exit balance_pgdat() before reaching 6858 * the wake up checks. If kswapd is going to sleep, no process should 6859 * be sleeping on pfmemalloc_wait, so wake them now if necessary. If 6860 * the wake up is premature, processes will wake kswapd and get 6861 * throttled again. The difference from wake ups in balance_pgdat() is 6862 * that here we are under prepare_to_wait(). 6863 */ 6864 if (waitqueue_active(&pgdat->pfmemalloc_wait)) 6865 wake_up_all(&pgdat->pfmemalloc_wait); 6866 6867 /* Hopeless node, leave it to direct reclaim */ 6868 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES) 6869 return true; 6870 6871 if (pgdat_balanced(pgdat, order, highest_zoneidx)) { 6872 clear_pgdat_congested(pgdat); 6873 return true; 6874 } 6875 6876 return false; 6877 } 6878 6879 /* 6880 * kswapd shrinks a node of pages that are at or below the highest usable 6881 * zone that is currently unbalanced. 6882 * 6883 * Returns true if kswapd scanned at least the requested number of pages to 6884 * reclaim or if the lack of progress was due to pages under writeback. 6885 * This is used to determine if the scanning priority needs to be raised. 6886 */ 6887 static bool kswapd_shrink_node(pg_data_t *pgdat, 6888 struct scan_control *sc) 6889 { 6890 struct zone *zone; 6891 int z; 6892 6893 /* Reclaim a number of pages proportional to the number of zones */ 6894 sc->nr_to_reclaim = 0; 6895 for (z = 0; z <= sc->reclaim_idx; z++) { 6896 zone = pgdat->node_zones + z; 6897 if (!managed_zone(zone)) 6898 continue; 6899 6900 sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX); 6901 } 6902 6903 /* 6904 * Historically care was taken to put equal pressure on all zones but 6905 * now pressure is applied based on node LRU order. 6906 */ 6907 shrink_node(pgdat, sc); 6908 6909 /* 6910 * Fragmentation may mean that the system cannot be rebalanced for 6911 * high-order allocations. If twice the allocation size has been 6912 * reclaimed then recheck watermarks only at order-0 to prevent 6913 * excessive reclaim. Assume that a process requested a high-order 6914 * can direct reclaim/compact. 6915 */ 6916 if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order)) 6917 sc->order = 0; 6918 6919 return sc->nr_scanned >= sc->nr_to_reclaim; 6920 } 6921 6922 /* Page allocator PCP high watermark is lowered if reclaim is active. */ 6923 static inline void 6924 update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active) 6925 { 6926 int i; 6927 struct zone *zone; 6928 6929 for (i = 0; i <= highest_zoneidx; i++) { 6930 zone = pgdat->node_zones + i; 6931 6932 if (!managed_zone(zone)) 6933 continue; 6934 6935 if (active) 6936 set_bit(ZONE_RECLAIM_ACTIVE, &zone->flags); 6937 else 6938 clear_bit(ZONE_RECLAIM_ACTIVE, &zone->flags); 6939 } 6940 } 6941 6942 static inline void 6943 set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) 6944 { 6945 update_reclaim_active(pgdat, highest_zoneidx, true); 6946 } 6947 6948 static inline void 6949 clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) 6950 { 6951 update_reclaim_active(pgdat, highest_zoneidx, false); 6952 } 6953 6954 /* 6955 * For kswapd, balance_pgdat() will reclaim pages across a node from zones 6956 * that are eligible for use by the caller until at least one zone is 6957 * balanced. 6958 * 6959 * Returns the order kswapd finished reclaiming at. 6960 * 6961 * kswapd scans the zones in the highmem->normal->dma direction. It skips 6962 * zones which have free_pages > high_wmark_pages(zone), but once a zone is 6963 * found to have free_pages <= high_wmark_pages(zone), any page in that zone 6964 * or lower is eligible for reclaim until at least one usable zone is 6965 * balanced. 6966 */ 6967 static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) 6968 { 6969 int i; 6970 unsigned long nr_soft_reclaimed; 6971 unsigned long nr_soft_scanned; 6972 unsigned long pflags; 6973 unsigned long nr_boost_reclaim; 6974 unsigned long zone_boosts[MAX_NR_ZONES] = { 0, }; 6975 bool boosted; 6976 struct zone *zone; 6977 struct scan_control sc = { 6978 .gfp_mask = GFP_KERNEL, 6979 .order = order, 6980 .may_unmap = 1, 6981 }; 6982 6983 set_task_reclaim_state(current, &sc.reclaim_state); 6984 psi_memstall_enter(&pflags); 6985 __fs_reclaim_acquire(_THIS_IP_); 6986 6987 count_vm_event(PAGEOUTRUN); 6988 6989 /* 6990 * Account for the reclaim boost. Note that the zone boost is left in 6991 * place so that parallel allocations that are near the watermark will 6992 * stall or direct reclaim until kswapd is finished. 6993 */ 6994 nr_boost_reclaim = 0; 6995 for (i = 0; i <= highest_zoneidx; i++) { 6996 zone = pgdat->node_zones + i; 6997 if (!managed_zone(zone)) 6998 continue; 6999 7000 nr_boost_reclaim += zone->watermark_boost; 7001 zone_boosts[i] = zone->watermark_boost; 7002 } 7003 boosted = nr_boost_reclaim; 7004 7005 restart: 7006 set_reclaim_active(pgdat, highest_zoneidx); 7007 sc.priority = DEF_PRIORITY; 7008 do { 7009 unsigned long nr_reclaimed = sc.nr_reclaimed; 7010 bool raise_priority = true; 7011 bool balanced; 7012 bool ret; 7013 7014 sc.reclaim_idx = highest_zoneidx; 7015 7016 /* 7017 * If the number of buffer_heads exceeds the maximum allowed 7018 * then consider reclaiming from all zones. This has a dual 7019 * purpose -- on 64-bit systems it is expected that 7020 * buffer_heads are stripped during active rotation. On 32-bit 7021 * systems, highmem pages can pin lowmem memory and shrinking 7022 * buffers can relieve lowmem pressure. Reclaim may still not 7023 * go ahead if all eligible zones for the original allocation 7024 * request are balanced to avoid excessive reclaim from kswapd. 7025 */ 7026 if (buffer_heads_over_limit) { 7027 for (i = MAX_NR_ZONES - 1; i >= 0; i--) { 7028 zone = pgdat->node_zones + i; 7029 if (!managed_zone(zone)) 7030 continue; 7031 7032 sc.reclaim_idx = i; 7033 break; 7034 } 7035 } 7036 7037 /* 7038 * If the pgdat is imbalanced then ignore boosting and preserve 7039 * the watermarks for a later time and restart. Note that the 7040 * zone watermarks will be still reset at the end of balancing 7041 * on the grounds that the normal reclaim should be enough to 7042 * re-evaluate if boosting is required when kswapd next wakes. 7043 */ 7044 balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx); 7045 if (!balanced && nr_boost_reclaim) { 7046 nr_boost_reclaim = 0; 7047 goto restart; 7048 } 7049 7050 /* 7051 * If boosting is not active then only reclaim if there are no 7052 * eligible zones. Note that sc.reclaim_idx is not used as 7053 * buffer_heads_over_limit may have adjusted it. 7054 */ 7055 if (!nr_boost_reclaim && balanced) 7056 goto out; 7057 7058 /* Limit the priority of boosting to avoid reclaim writeback */ 7059 if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2) 7060 raise_priority = false; 7061 7062 /* 7063 * Do not writeback or swap pages for boosted reclaim. The 7064 * intent is to relieve pressure not issue sub-optimal IO 7065 * from reclaim context. If no pages are reclaimed, the 7066 * reclaim will be aborted. 7067 */ 7068 sc.may_writepage = !laptop_mode && !nr_boost_reclaim; 7069 sc.may_swap = !nr_boost_reclaim; 7070 7071 /* 7072 * Do some background aging, to give pages a chance to be 7073 * referenced before reclaiming. All pages are rotated 7074 * regardless of classzone as this is about consistent aging. 7075 */ 7076 kswapd_age_node(pgdat, &sc); 7077 7078 /* 7079 * If we're getting trouble reclaiming, start doing writepage 7080 * even in laptop mode. 7081 */ 7082 if (sc.priority < DEF_PRIORITY - 2) 7083 sc.may_writepage = 1; 7084 7085 /* Call soft limit reclaim before calling shrink_node. */ 7086 sc.nr_scanned = 0; 7087 nr_soft_scanned = 0; 7088 nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order, 7089 sc.gfp_mask, &nr_soft_scanned); 7090 sc.nr_reclaimed += nr_soft_reclaimed; 7091 7092 /* 7093 * There should be no need to raise the scanning priority if 7094 * enough pages are already being scanned that that high 7095 * watermark would be met at 100% efficiency. 7096 */ 7097 if (kswapd_shrink_node(pgdat, &sc)) 7098 raise_priority = false; 7099 7100 /* 7101 * If the low watermark is met there is no need for processes 7102 * to be throttled on pfmemalloc_wait as they should not be 7103 * able to safely make forward progress. Wake them 7104 */ 7105 if (waitqueue_active(&pgdat->pfmemalloc_wait) && 7106 allow_direct_reclaim(pgdat)) 7107 wake_up_all(&pgdat->pfmemalloc_wait); 7108 7109 /* Check if kswapd should be suspending */ 7110 __fs_reclaim_release(_THIS_IP_); 7111 ret = try_to_freeze(); 7112 __fs_reclaim_acquire(_THIS_IP_); 7113 if (ret || kthread_should_stop()) 7114 break; 7115 7116 /* 7117 * Raise priority if scanning rate is too low or there was no 7118 * progress in reclaiming pages 7119 */ 7120 nr_reclaimed = sc.nr_reclaimed - nr_reclaimed; 7121 nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed); 7122 7123 /* 7124 * If reclaim made no progress for a boost, stop reclaim as 7125 * IO cannot be queued and it could be an infinite loop in 7126 * extreme circumstances. 7127 */ 7128 if (nr_boost_reclaim && !nr_reclaimed) 7129 break; 7130 7131 if (raise_priority || !nr_reclaimed) 7132 sc.priority--; 7133 } while (sc.priority >= 1); 7134 7135 if (!sc.nr_reclaimed) 7136 pgdat->kswapd_failures++; 7137 7138 out: 7139 clear_reclaim_active(pgdat, highest_zoneidx); 7140 7141 /* If reclaim was boosted, account for the reclaim done in this pass */ 7142 if (boosted) { 7143 unsigned long flags; 7144 7145 for (i = 0; i <= highest_zoneidx; i++) { 7146 if (!zone_boosts[i]) 7147 continue; 7148 7149 /* Increments are under the zone lock */ 7150 zone = pgdat->node_zones + i; 7151 spin_lock_irqsave(&zone->lock, flags); 7152 zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]); 7153 spin_unlock_irqrestore(&zone->lock, flags); 7154 } 7155 7156 /* 7157 * As there is now likely space, wakeup kcompact to defragment 7158 * pageblocks. 7159 */ 7160 wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx); 7161 } 7162 7163 snapshot_refaults(NULL, pgdat); 7164 __fs_reclaim_release(_THIS_IP_); 7165 psi_memstall_leave(&pflags); 7166 set_task_reclaim_state(current, NULL); 7167 7168 /* 7169 * Return the order kswapd stopped reclaiming at as 7170 * prepare_kswapd_sleep() takes it into account. If another caller 7171 * entered the allocator slow path while kswapd was awake, order will 7172 * remain at the higher level. 7173 */ 7174 return sc.order; 7175 } 7176 7177 /* 7178 * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to 7179 * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is 7180 * not a valid index then either kswapd runs for first time or kswapd couldn't 7181 * sleep after previous reclaim attempt (node is still unbalanced). In that 7182 * case return the zone index of the previous kswapd reclaim cycle. 7183 */ 7184 static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat, 7185 enum zone_type prev_highest_zoneidx) 7186 { 7187 enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx); 7188 7189 return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx; 7190 } 7191 7192 static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order, 7193 unsigned int highest_zoneidx) 7194 { 7195 long remaining = 0; 7196 DEFINE_WAIT(wait); 7197 7198 if (freezing(current) || kthread_should_stop()) 7199 return; 7200 7201 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); 7202 7203 /* 7204 * Try to sleep for a short interval. Note that kcompactd will only be 7205 * woken if it is possible to sleep for a short interval. This is 7206 * deliberate on the assumption that if reclaim cannot keep an 7207 * eligible zone balanced that it's also unlikely that compaction will 7208 * succeed. 7209 */ 7210 if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) { 7211 /* 7212 * Compaction records what page blocks it recently failed to 7213 * isolate pages from and skips them in the future scanning. 7214 * When kswapd is going to sleep, it is reasonable to assume 7215 * that pages and compaction may succeed so reset the cache. 7216 */ 7217 reset_isolation_suitable(pgdat); 7218 7219 /* 7220 * We have freed the memory, now we should compact it to make 7221 * allocation of the requested order possible. 7222 */ 7223 wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx); 7224 7225 remaining = schedule_timeout(HZ/10); 7226 7227 /* 7228 * If woken prematurely then reset kswapd_highest_zoneidx and 7229 * order. The values will either be from a wakeup request or 7230 * the previous request that slept prematurely. 7231 */ 7232 if (remaining) { 7233 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, 7234 kswapd_highest_zoneidx(pgdat, 7235 highest_zoneidx)); 7236 7237 if (READ_ONCE(pgdat->kswapd_order) < reclaim_order) 7238 WRITE_ONCE(pgdat->kswapd_order, reclaim_order); 7239 } 7240 7241 finish_wait(&pgdat->kswapd_wait, &wait); 7242 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); 7243 } 7244 7245 /* 7246 * After a short sleep, check if it was a premature sleep. If not, then 7247 * go fully to sleep until explicitly woken up. 7248 */ 7249 if (!remaining && 7250 prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) { 7251 trace_mm_vmscan_kswapd_sleep(pgdat->node_id); 7252 7253 /* 7254 * vmstat counters are not perfectly accurate and the estimated 7255 * value for counters such as NR_FREE_PAGES can deviate from the 7256 * true value by nr_online_cpus * threshold. To avoid the zone 7257 * watermarks being breached while under pressure, we reduce the 7258 * per-cpu vmstat threshold while kswapd is awake and restore 7259 * them before going back to sleep. 7260 */ 7261 set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold); 7262 7263 if (!kthread_should_stop()) 7264 schedule(); 7265 7266 set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold); 7267 } else { 7268 if (remaining) 7269 count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY); 7270 else 7271 count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY); 7272 } 7273 finish_wait(&pgdat->kswapd_wait, &wait); 7274 } 7275 7276 /* 7277 * The background pageout daemon, started as a kernel thread 7278 * from the init process. 7279 * 7280 * This basically trickles out pages so that we have _some_ 7281 * free memory available even if there is no other activity 7282 * that frees anything up. This is needed for things like routing 7283 * etc, where we otherwise might have all activity going on in 7284 * asynchronous contexts that cannot page things out. 7285 * 7286 * If there are applications that are active memory-allocators 7287 * (most normal use), this basically shouldn't matter. 7288 */ 7289 static int kswapd(void *p) 7290 { 7291 unsigned int alloc_order, reclaim_order; 7292 unsigned int highest_zoneidx = MAX_NR_ZONES - 1; 7293 pg_data_t *pgdat = (pg_data_t *)p; 7294 struct task_struct *tsk = current; 7295 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 7296 7297 if (!cpumask_empty(cpumask)) 7298 set_cpus_allowed_ptr(tsk, cpumask); 7299 7300 /* 7301 * Tell the memory management that we're a "memory allocator", 7302 * and that if we need more memory we should get access to it 7303 * regardless (see "__alloc_pages()"). "kswapd" should 7304 * never get caught in the normal page freeing logic. 7305 * 7306 * (Kswapd normally doesn't need memory anyway, but sometimes 7307 * you need a small amount of memory in order to be able to 7308 * page out something else, and this flag essentially protects 7309 * us from recursively trying to free more memory as we're 7310 * trying to free the first piece of memory in the first place). 7311 */ 7312 tsk->flags |= PF_MEMALLOC | PF_KSWAPD; 7313 set_freezable(); 7314 7315 WRITE_ONCE(pgdat->kswapd_order, 0); 7316 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); 7317 atomic_set(&pgdat->nr_writeback_throttled, 0); 7318 for ( ; ; ) { 7319 bool ret; 7320 7321 alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order); 7322 highest_zoneidx = kswapd_highest_zoneidx(pgdat, 7323 highest_zoneidx); 7324 7325 kswapd_try_sleep: 7326 kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order, 7327 highest_zoneidx); 7328 7329 /* Read the new order and highest_zoneidx */ 7330 alloc_order = READ_ONCE(pgdat->kswapd_order); 7331 highest_zoneidx = kswapd_highest_zoneidx(pgdat, 7332 highest_zoneidx); 7333 WRITE_ONCE(pgdat->kswapd_order, 0); 7334 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); 7335 7336 ret = try_to_freeze(); 7337 if (kthread_should_stop()) 7338 break; 7339 7340 /* 7341 * We can speed up thawing tasks if we don't call balance_pgdat 7342 * after returning from the refrigerator 7343 */ 7344 if (ret) 7345 continue; 7346 7347 /* 7348 * Reclaim begins at the requested order but if a high-order 7349 * reclaim fails then kswapd falls back to reclaiming for 7350 * order-0. If that happens, kswapd will consider sleeping 7351 * for the order it finished reclaiming at (reclaim_order) 7352 * but kcompactd is woken to compact for the original 7353 * request (alloc_order). 7354 */ 7355 trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx, 7356 alloc_order); 7357 reclaim_order = balance_pgdat(pgdat, alloc_order, 7358 highest_zoneidx); 7359 if (reclaim_order < alloc_order) 7360 goto kswapd_try_sleep; 7361 } 7362 7363 tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD); 7364 7365 return 0; 7366 } 7367 7368 /* 7369 * A zone is low on free memory or too fragmented for high-order memory. If 7370 * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's 7371 * pgdat. It will wake up kcompactd after reclaiming memory. If kswapd reclaim 7372 * has failed or is not needed, still wake up kcompactd if only compaction is 7373 * needed. 7374 */ 7375 void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order, 7376 enum zone_type highest_zoneidx) 7377 { 7378 pg_data_t *pgdat; 7379 enum zone_type curr_idx; 7380 7381 if (!managed_zone(zone)) 7382 return; 7383 7384 if (!cpuset_zone_allowed(zone, gfp_flags)) 7385 return; 7386 7387 pgdat = zone->zone_pgdat; 7388 curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx); 7389 7390 if (curr_idx == MAX_NR_ZONES || curr_idx < highest_zoneidx) 7391 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx); 7392 7393 if (READ_ONCE(pgdat->kswapd_order) < order) 7394 WRITE_ONCE(pgdat->kswapd_order, order); 7395 7396 if (!waitqueue_active(&pgdat->kswapd_wait)) 7397 return; 7398 7399 /* Hopeless node, leave it to direct reclaim if possible */ 7400 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES || 7401 (pgdat_balanced(pgdat, order, highest_zoneidx) && 7402 !pgdat_watermark_boosted(pgdat, highest_zoneidx))) { 7403 /* 7404 * There may be plenty of free memory available, but it's too 7405 * fragmented for high-order allocations. Wake up kcompactd 7406 * and rely on compaction_suitable() to determine if it's 7407 * needed. If it fails, it will defer subsequent attempts to 7408 * ratelimit its work. 7409 */ 7410 if (!(gfp_flags & __GFP_DIRECT_RECLAIM)) 7411 wakeup_kcompactd(pgdat, order, highest_zoneidx); 7412 return; 7413 } 7414 7415 trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order, 7416 gfp_flags); 7417 wake_up_interruptible(&pgdat->kswapd_wait); 7418 } 7419 7420 #ifdef CONFIG_HIBERNATION 7421 /* 7422 * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of 7423 * freed pages. 7424 * 7425 * Rather than trying to age LRUs the aim is to preserve the overall 7426 * LRU order by reclaiming preferentially 7427 * inactive > active > active referenced > active mapped 7428 */ 7429 unsigned long shrink_all_memory(unsigned long nr_to_reclaim) 7430 { 7431 struct scan_control sc = { 7432 .nr_to_reclaim = nr_to_reclaim, 7433 .gfp_mask = GFP_HIGHUSER_MOVABLE, 7434 .reclaim_idx = MAX_NR_ZONES - 1, 7435 .priority = DEF_PRIORITY, 7436 .may_writepage = 1, 7437 .may_unmap = 1, 7438 .may_swap = 1, 7439 .hibernation_mode = 1, 7440 }; 7441 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); 7442 unsigned long nr_reclaimed; 7443 unsigned int noreclaim_flag; 7444 7445 fs_reclaim_acquire(sc.gfp_mask); 7446 noreclaim_flag = memalloc_noreclaim_save(); 7447 set_task_reclaim_state(current, &sc.reclaim_state); 7448 7449 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 7450 7451 set_task_reclaim_state(current, NULL); 7452 memalloc_noreclaim_restore(noreclaim_flag); 7453 fs_reclaim_release(sc.gfp_mask); 7454 7455 return nr_reclaimed; 7456 } 7457 #endif /* CONFIG_HIBERNATION */ 7458 7459 /* 7460 * This kswapd start function will be called by init and node-hot-add. 7461 */ 7462 void kswapd_run(int nid) 7463 { 7464 pg_data_t *pgdat = NODE_DATA(nid); 7465 7466 pgdat_kswapd_lock(pgdat); 7467 if (!pgdat->kswapd) { 7468 pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid); 7469 if (IS_ERR(pgdat->kswapd)) { 7470 /* failure at boot is fatal */ 7471 BUG_ON(system_state < SYSTEM_RUNNING); 7472 pr_err("Failed to start kswapd on node %d\n", nid); 7473 pgdat->kswapd = NULL; 7474 } 7475 } 7476 pgdat_kswapd_unlock(pgdat); 7477 } 7478 7479 /* 7480 * Called by memory hotplug when all memory in a node is offlined. Caller must 7481 * be holding mem_hotplug_begin/done(). 7482 */ 7483 void kswapd_stop(int nid) 7484 { 7485 pg_data_t *pgdat = NODE_DATA(nid); 7486 struct task_struct *kswapd; 7487 7488 pgdat_kswapd_lock(pgdat); 7489 kswapd = pgdat->kswapd; 7490 if (kswapd) { 7491 kthread_stop(kswapd); 7492 pgdat->kswapd = NULL; 7493 } 7494 pgdat_kswapd_unlock(pgdat); 7495 } 7496 7497 static int __init kswapd_init(void) 7498 { 7499 int nid; 7500 7501 swap_setup(); 7502 for_each_node_state(nid, N_MEMORY) 7503 kswapd_run(nid); 7504 return 0; 7505 } 7506 7507 module_init(kswapd_init) 7508 7509 #ifdef CONFIG_NUMA 7510 /* 7511 * Node reclaim mode 7512 * 7513 * If non-zero call node_reclaim when the number of free pages falls below 7514 * the watermarks. 7515 */ 7516 int node_reclaim_mode __read_mostly; 7517 7518 /* 7519 * Priority for NODE_RECLAIM. This determines the fraction of pages 7520 * of a node considered for each zone_reclaim. 4 scans 1/16th of 7521 * a zone. 7522 */ 7523 #define NODE_RECLAIM_PRIORITY 4 7524 7525 /* 7526 * Percentage of pages in a zone that must be unmapped for node_reclaim to 7527 * occur. 7528 */ 7529 int sysctl_min_unmapped_ratio = 1; 7530 7531 /* 7532 * If the number of slab pages in a zone grows beyond this percentage then 7533 * slab reclaim needs to occur. 7534 */ 7535 int sysctl_min_slab_ratio = 5; 7536 7537 static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat) 7538 { 7539 unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED); 7540 unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) + 7541 node_page_state(pgdat, NR_ACTIVE_FILE); 7542 7543 /* 7544 * It's possible for there to be more file mapped pages than 7545 * accounted for by the pages on the file LRU lists because 7546 * tmpfs pages accounted for as ANON can also be FILE_MAPPED 7547 */ 7548 return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0; 7549 } 7550 7551 /* Work out how many page cache pages we can reclaim in this reclaim_mode */ 7552 static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat) 7553 { 7554 unsigned long nr_pagecache_reclaimable; 7555 unsigned long delta = 0; 7556 7557 /* 7558 * If RECLAIM_UNMAP is set, then all file pages are considered 7559 * potentially reclaimable. Otherwise, we have to worry about 7560 * pages like swapcache and node_unmapped_file_pages() provides 7561 * a better estimate 7562 */ 7563 if (node_reclaim_mode & RECLAIM_UNMAP) 7564 nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES); 7565 else 7566 nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat); 7567 7568 /* If we can't clean pages, remove dirty pages from consideration */ 7569 if (!(node_reclaim_mode & RECLAIM_WRITE)) 7570 delta += node_page_state(pgdat, NR_FILE_DIRTY); 7571 7572 /* Watch for any possible underflows due to delta */ 7573 if (unlikely(delta > nr_pagecache_reclaimable)) 7574 delta = nr_pagecache_reclaimable; 7575 7576 return nr_pagecache_reclaimable - delta; 7577 } 7578 7579 /* 7580 * Try to free up some pages from this node through reclaim. 7581 */ 7582 static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) 7583 { 7584 /* Minimum pages needed in order to stay on node */ 7585 const unsigned long nr_pages = 1 << order; 7586 struct task_struct *p = current; 7587 unsigned int noreclaim_flag; 7588 struct scan_control sc = { 7589 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), 7590 .gfp_mask = current_gfp_context(gfp_mask), 7591 .order = order, 7592 .priority = NODE_RECLAIM_PRIORITY, 7593 .may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE), 7594 .may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP), 7595 .may_swap = 1, 7596 .reclaim_idx = gfp_zone(gfp_mask), 7597 }; 7598 unsigned long pflags; 7599 7600 trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order, 7601 sc.gfp_mask); 7602 7603 cond_resched(); 7604 psi_memstall_enter(&pflags); 7605 fs_reclaim_acquire(sc.gfp_mask); 7606 /* 7607 * We need to be able to allocate from the reserves for RECLAIM_UNMAP 7608 */ 7609 noreclaim_flag = memalloc_noreclaim_save(); 7610 set_task_reclaim_state(p, &sc.reclaim_state); 7611 7612 if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages || 7613 node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) { 7614 /* 7615 * Free memory by calling shrink node with increasing 7616 * priorities until we have enough memory freed. 7617 */ 7618 do { 7619 shrink_node(pgdat, &sc); 7620 } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0); 7621 } 7622 7623 set_task_reclaim_state(p, NULL); 7624 memalloc_noreclaim_restore(noreclaim_flag); 7625 fs_reclaim_release(sc.gfp_mask); 7626 psi_memstall_leave(&pflags); 7627 7628 trace_mm_vmscan_node_reclaim_end(sc.nr_reclaimed); 7629 7630 return sc.nr_reclaimed >= nr_pages; 7631 } 7632 7633 int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) 7634 { 7635 int ret; 7636 7637 /* 7638 * Node reclaim reclaims unmapped file backed pages and 7639 * slab pages if we are over the defined limits. 7640 * 7641 * A small portion of unmapped file backed pages is needed for 7642 * file I/O otherwise pages read by file I/O will be immediately 7643 * thrown out if the node is overallocated. So we do not reclaim 7644 * if less than a specified percentage of the node is used by 7645 * unmapped file backed pages. 7646 */ 7647 if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages && 7648 node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <= 7649 pgdat->min_slab_pages) 7650 return NODE_RECLAIM_FULL; 7651 7652 /* 7653 * Do not scan if the allocation should not be delayed. 7654 */ 7655 if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC)) 7656 return NODE_RECLAIM_NOSCAN; 7657 7658 /* 7659 * Only run node reclaim on the local node or on nodes that do not 7660 * have associated processors. This will favor the local processor 7661 * over remote processors and spread off node memory allocations 7662 * as wide as possible. 7663 */ 7664 if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id()) 7665 return NODE_RECLAIM_NOSCAN; 7666 7667 if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags)) 7668 return NODE_RECLAIM_NOSCAN; 7669 7670 ret = __node_reclaim(pgdat, gfp_mask, order); 7671 clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags); 7672 7673 if (!ret) 7674 count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED); 7675 7676 return ret; 7677 } 7678 #endif 7679 7680 void check_move_unevictable_pages(struct pagevec *pvec) 7681 { 7682 struct folio_batch fbatch; 7683 unsigned i; 7684 7685 folio_batch_init(&fbatch); 7686 for (i = 0; i < pvec->nr; i++) { 7687 struct page *page = pvec->pages[i]; 7688 7689 if (PageTransTail(page)) 7690 continue; 7691 folio_batch_add(&fbatch, page_folio(page)); 7692 } 7693 check_move_unevictable_folios(&fbatch); 7694 } 7695 EXPORT_SYMBOL_GPL(check_move_unevictable_pages); 7696 7697 /** 7698 * check_move_unevictable_folios - Move evictable folios to appropriate zone 7699 * lru list 7700 * @fbatch: Batch of lru folios to check. 7701 * 7702 * Checks folios for evictability, if an evictable folio is in the unevictable 7703 * lru list, moves it to the appropriate evictable lru list. This function 7704 * should be only used for lru folios. 7705 */ 7706 void check_move_unevictable_folios(struct folio_batch *fbatch) 7707 { 7708 struct lruvec *lruvec = NULL; 7709 int pgscanned = 0; 7710 int pgrescued = 0; 7711 int i; 7712 7713 for (i = 0; i < fbatch->nr; i++) { 7714 struct folio *folio = fbatch->folios[i]; 7715 int nr_pages = folio_nr_pages(folio); 7716 7717 pgscanned += nr_pages; 7718 7719 /* block memcg migration while the folio moves between lrus */ 7720 if (!folio_test_clear_lru(folio)) 7721 continue; 7722 7723 lruvec = folio_lruvec_relock_irq(folio, lruvec); 7724 if (folio_evictable(folio) && folio_test_unevictable(folio)) { 7725 lruvec_del_folio(lruvec, folio); 7726 folio_clear_unevictable(folio); 7727 lruvec_add_folio(lruvec, folio); 7728 pgrescued += nr_pages; 7729 } 7730 folio_set_lru(folio); 7731 } 7732 7733 if (lruvec) { 7734 __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued); 7735 __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); 7736 unlock_page_lruvec_irq(lruvec); 7737 } else if (pgscanned) { 7738 count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); 7739 } 7740 } 7741 EXPORT_SYMBOL_GPL(check_move_unevictable_folios); 7742