xref: /openbmc/linux/mm/vmscan.c (revision a395b8d1)
1  // SPDX-License-Identifier: GPL-2.0
2  /*
3   *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
4   *
5   *  Swap reorganised 29.12.95, Stephen Tweedie.
6   *  kswapd added: 7.1.96  sct
7   *  Removed kswapd_ctl limits, and swap out as many pages as needed
8   *  to bring the system back to freepages.high: 2.4.97, Rik van Riel.
9   *  Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
10   *  Multiqueue VM started 5.8.00, Rik van Riel.
11   */
12  
13  #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14  
15  #include <linux/mm.h>
16  #include <linux/sched/mm.h>
17  #include <linux/module.h>
18  #include <linux/gfp.h>
19  #include <linux/kernel_stat.h>
20  #include <linux/swap.h>
21  #include <linux/pagemap.h>
22  #include <linux/init.h>
23  #include <linux/highmem.h>
24  #include <linux/vmpressure.h>
25  #include <linux/vmstat.h>
26  #include <linux/file.h>
27  #include <linux/writeback.h>
28  #include <linux/blkdev.h>
29  #include <linux/buffer_head.h>	/* for buffer_heads_over_limit */
30  #include <linux/mm_inline.h>
31  #include <linux/backing-dev.h>
32  #include <linux/rmap.h>
33  #include <linux/topology.h>
34  #include <linux/cpu.h>
35  #include <linux/cpuset.h>
36  #include <linux/compaction.h>
37  #include <linux/notifier.h>
38  #include <linux/mutex.h>
39  #include <linux/delay.h>
40  #include <linux/kthread.h>
41  #include <linux/freezer.h>
42  #include <linux/memcontrol.h>
43  #include <linux/migrate.h>
44  #include <linux/delayacct.h>
45  #include <linux/sysctl.h>
46  #include <linux/memory-tiers.h>
47  #include <linux/oom.h>
48  #include <linux/pagevec.h>
49  #include <linux/prefetch.h>
50  #include <linux/printk.h>
51  #include <linux/dax.h>
52  #include <linux/psi.h>
53  #include <linux/pagewalk.h>
54  #include <linux/shmem_fs.h>
55  #include <linux/ctype.h>
56  #include <linux/debugfs.h>
57  #include <linux/khugepaged.h>
58  #include <linux/rculist_nulls.h>
59  #include <linux/random.h>
60  #include <linux/srcu.h>
61  
62  #include <asm/tlbflush.h>
63  #include <asm/div64.h>
64  
65  #include <linux/swapops.h>
66  #include <linux/balloon_compaction.h>
67  #include <linux/sched/sysctl.h>
68  
69  #include "internal.h"
70  #include "swap.h"
71  
72  #define CREATE_TRACE_POINTS
73  #include <trace/events/vmscan.h>
74  
75  struct scan_control {
76  	/* How many pages shrink_list() should reclaim */
77  	unsigned long nr_to_reclaim;
78  
79  	/*
80  	 * Nodemask of nodes allowed by the caller. If NULL, all nodes
81  	 * are scanned.
82  	 */
83  	nodemask_t	*nodemask;
84  
85  	/*
86  	 * The memory cgroup that hit its limit and as a result is the
87  	 * primary target of this reclaim invocation.
88  	 */
89  	struct mem_cgroup *target_mem_cgroup;
90  
91  	/*
92  	 * Scan pressure balancing between anon and file LRUs
93  	 */
94  	unsigned long	anon_cost;
95  	unsigned long	file_cost;
96  
97  	/* Can active folios be deactivated as part of reclaim? */
98  #define DEACTIVATE_ANON 1
99  #define DEACTIVATE_FILE 2
100  	unsigned int may_deactivate:2;
101  	unsigned int force_deactivate:1;
102  	unsigned int skipped_deactivate:1;
103  
104  	/* Writepage batching in laptop mode; RECLAIM_WRITE */
105  	unsigned int may_writepage:1;
106  
107  	/* Can mapped folios be reclaimed? */
108  	unsigned int may_unmap:1;
109  
110  	/* Can folios be swapped as part of reclaim? */
111  	unsigned int may_swap:1;
112  
113  	/* Proactive reclaim invoked by userspace through memory.reclaim */
114  	unsigned int proactive:1;
115  
116  	/*
117  	 * Cgroup memory below memory.low is protected as long as we
118  	 * don't threaten to OOM. If any cgroup is reclaimed at
119  	 * reduced force or passed over entirely due to its memory.low
120  	 * setting (memcg_low_skipped), and nothing is reclaimed as a
121  	 * result, then go back for one more cycle that reclaims the protected
122  	 * memory (memcg_low_reclaim) to avert OOM.
123  	 */
124  	unsigned int memcg_low_reclaim:1;
125  	unsigned int memcg_low_skipped:1;
126  
127  	unsigned int hibernation_mode:1;
128  
129  	/* One of the zones is ready for compaction */
130  	unsigned int compaction_ready:1;
131  
132  	/* There is easily reclaimable cold cache in the current node */
133  	unsigned int cache_trim_mode:1;
134  
135  	/* The file folios on the current node are dangerously low */
136  	unsigned int file_is_tiny:1;
137  
138  	/* Always discard instead of demoting to lower tier memory */
139  	unsigned int no_demotion:1;
140  
141  	/* Allocation order */
142  	s8 order;
143  
144  	/* Scan (total_size >> priority) pages at once */
145  	s8 priority;
146  
147  	/* The highest zone to isolate folios for reclaim from */
148  	s8 reclaim_idx;
149  
150  	/* This context's GFP mask */
151  	gfp_t gfp_mask;
152  
153  	/* Incremented by the number of inactive pages that were scanned */
154  	unsigned long nr_scanned;
155  
156  	/* Number of pages freed so far during a call to shrink_zones() */
157  	unsigned long nr_reclaimed;
158  
159  	struct {
160  		unsigned int dirty;
161  		unsigned int unqueued_dirty;
162  		unsigned int congested;
163  		unsigned int writeback;
164  		unsigned int immediate;
165  		unsigned int file_taken;
166  		unsigned int taken;
167  	} nr;
168  
169  	/* for recording the reclaimed slab by now */
170  	struct reclaim_state reclaim_state;
171  };
172  
173  #ifdef ARCH_HAS_PREFETCHW
174  #define prefetchw_prev_lru_folio(_folio, _base, _field)			\
175  	do {								\
176  		if ((_folio)->lru.prev != _base) {			\
177  			struct folio *prev;				\
178  									\
179  			prev = lru_to_folio(&(_folio->lru));		\
180  			prefetchw(&prev->_field);			\
181  		}							\
182  	} while (0)
183  #else
184  #define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0)
185  #endif
186  
187  /*
188   * From 0 .. 200.  Higher means more swappy.
189   */
190  int vm_swappiness = 60;
191  
192  LIST_HEAD(shrinker_list);
193  DEFINE_MUTEX(shrinker_mutex);
194  DEFINE_SRCU(shrinker_srcu);
195  static atomic_t shrinker_srcu_generation = ATOMIC_INIT(0);
196  
197  #ifdef CONFIG_MEMCG
198  static int shrinker_nr_max;
199  
200  /* The shrinker_info is expanded in a batch of BITS_PER_LONG */
201  static inline int shrinker_map_size(int nr_items)
202  {
203  	return (DIV_ROUND_UP(nr_items, BITS_PER_LONG) * sizeof(unsigned long));
204  }
205  
206  static inline int shrinker_defer_size(int nr_items)
207  {
208  	return (round_up(nr_items, BITS_PER_LONG) * sizeof(atomic_long_t));
209  }
210  
211  static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg,
212  						     int nid)
213  {
214  	return srcu_dereference_check(memcg->nodeinfo[nid]->shrinker_info,
215  				      &shrinker_srcu,
216  				      lockdep_is_held(&shrinker_mutex));
217  }
218  
219  static struct shrinker_info *shrinker_info_srcu(struct mem_cgroup *memcg,
220  						     int nid)
221  {
222  	return srcu_dereference(memcg->nodeinfo[nid]->shrinker_info,
223  				&shrinker_srcu);
224  }
225  
226  static void free_shrinker_info_rcu(struct rcu_head *head)
227  {
228  	kvfree(container_of(head, struct shrinker_info, rcu));
229  }
230  
231  static int expand_one_shrinker_info(struct mem_cgroup *memcg,
232  				    int map_size, int defer_size,
233  				    int old_map_size, int old_defer_size,
234  				    int new_nr_max)
235  {
236  	struct shrinker_info *new, *old;
237  	struct mem_cgroup_per_node *pn;
238  	int nid;
239  	int size = map_size + defer_size;
240  
241  	for_each_node(nid) {
242  		pn = memcg->nodeinfo[nid];
243  		old = shrinker_info_protected(memcg, nid);
244  		/* Not yet online memcg */
245  		if (!old)
246  			return 0;
247  
248  		/* Already expanded this shrinker_info */
249  		if (new_nr_max <= old->map_nr_max)
250  			continue;
251  
252  		new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid);
253  		if (!new)
254  			return -ENOMEM;
255  
256  		new->nr_deferred = (atomic_long_t *)(new + 1);
257  		new->map = (void *)new->nr_deferred + defer_size;
258  		new->map_nr_max = new_nr_max;
259  
260  		/* map: set all old bits, clear all new bits */
261  		memset(new->map, (int)0xff, old_map_size);
262  		memset((void *)new->map + old_map_size, 0, map_size - old_map_size);
263  		/* nr_deferred: copy old values, clear all new values */
264  		memcpy(new->nr_deferred, old->nr_deferred, old_defer_size);
265  		memset((void *)new->nr_deferred + old_defer_size, 0,
266  		       defer_size - old_defer_size);
267  
268  		rcu_assign_pointer(pn->shrinker_info, new);
269  		call_srcu(&shrinker_srcu, &old->rcu, free_shrinker_info_rcu);
270  	}
271  
272  	return 0;
273  }
274  
275  void free_shrinker_info(struct mem_cgroup *memcg)
276  {
277  	struct mem_cgroup_per_node *pn;
278  	struct shrinker_info *info;
279  	int nid;
280  
281  	for_each_node(nid) {
282  		pn = memcg->nodeinfo[nid];
283  		info = rcu_dereference_protected(pn->shrinker_info, true);
284  		kvfree(info);
285  		rcu_assign_pointer(pn->shrinker_info, NULL);
286  	}
287  }
288  
289  int alloc_shrinker_info(struct mem_cgroup *memcg)
290  {
291  	struct shrinker_info *info;
292  	int nid, size, ret = 0;
293  	int map_size, defer_size = 0;
294  
295  	mutex_lock(&shrinker_mutex);
296  	map_size = shrinker_map_size(shrinker_nr_max);
297  	defer_size = shrinker_defer_size(shrinker_nr_max);
298  	size = map_size + defer_size;
299  	for_each_node(nid) {
300  		info = kvzalloc_node(sizeof(*info) + size, GFP_KERNEL, nid);
301  		if (!info) {
302  			free_shrinker_info(memcg);
303  			ret = -ENOMEM;
304  			break;
305  		}
306  		info->nr_deferred = (atomic_long_t *)(info + 1);
307  		info->map = (void *)info->nr_deferred + defer_size;
308  		info->map_nr_max = shrinker_nr_max;
309  		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info);
310  	}
311  	mutex_unlock(&shrinker_mutex);
312  
313  	return ret;
314  }
315  
316  static int expand_shrinker_info(int new_id)
317  {
318  	int ret = 0;
319  	int new_nr_max = round_up(new_id + 1, BITS_PER_LONG);
320  	int map_size, defer_size = 0;
321  	int old_map_size, old_defer_size = 0;
322  	struct mem_cgroup *memcg;
323  
324  	if (!root_mem_cgroup)
325  		goto out;
326  
327  	lockdep_assert_held(&shrinker_mutex);
328  
329  	map_size = shrinker_map_size(new_nr_max);
330  	defer_size = shrinker_defer_size(new_nr_max);
331  	old_map_size = shrinker_map_size(shrinker_nr_max);
332  	old_defer_size = shrinker_defer_size(shrinker_nr_max);
333  
334  	memcg = mem_cgroup_iter(NULL, NULL, NULL);
335  	do {
336  		ret = expand_one_shrinker_info(memcg, map_size, defer_size,
337  					       old_map_size, old_defer_size,
338  					       new_nr_max);
339  		if (ret) {
340  			mem_cgroup_iter_break(NULL, memcg);
341  			goto out;
342  		}
343  	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
344  out:
345  	if (!ret)
346  		shrinker_nr_max = new_nr_max;
347  
348  	return ret;
349  }
350  
351  void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
352  {
353  	if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
354  		struct shrinker_info *info;
355  		int srcu_idx;
356  
357  		srcu_idx = srcu_read_lock(&shrinker_srcu);
358  		info = shrinker_info_srcu(memcg, nid);
359  		if (!WARN_ON_ONCE(shrinker_id >= info->map_nr_max)) {
360  			/* Pairs with smp mb in shrink_slab() */
361  			smp_mb__before_atomic();
362  			set_bit(shrinker_id, info->map);
363  		}
364  		srcu_read_unlock(&shrinker_srcu, srcu_idx);
365  	}
366  }
367  
368  static DEFINE_IDR(shrinker_idr);
369  
370  static int prealloc_memcg_shrinker(struct shrinker *shrinker)
371  {
372  	int id, ret = -ENOMEM;
373  
374  	if (mem_cgroup_disabled())
375  		return -ENOSYS;
376  
377  	mutex_lock(&shrinker_mutex);
378  	id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL);
379  	if (id < 0)
380  		goto unlock;
381  
382  	if (id >= shrinker_nr_max) {
383  		if (expand_shrinker_info(id)) {
384  			idr_remove(&shrinker_idr, id);
385  			goto unlock;
386  		}
387  	}
388  	shrinker->id = id;
389  	ret = 0;
390  unlock:
391  	mutex_unlock(&shrinker_mutex);
392  	return ret;
393  }
394  
395  static void unregister_memcg_shrinker(struct shrinker *shrinker)
396  {
397  	int id = shrinker->id;
398  
399  	BUG_ON(id < 0);
400  
401  	lockdep_assert_held(&shrinker_mutex);
402  
403  	idr_remove(&shrinker_idr, id);
404  }
405  
406  static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
407  				   struct mem_cgroup *memcg)
408  {
409  	struct shrinker_info *info;
410  
411  	info = shrinker_info_srcu(memcg, nid);
412  	return atomic_long_xchg(&info->nr_deferred[shrinker->id], 0);
413  }
414  
415  static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
416  				  struct mem_cgroup *memcg)
417  {
418  	struct shrinker_info *info;
419  
420  	info = shrinker_info_srcu(memcg, nid);
421  	return atomic_long_add_return(nr, &info->nr_deferred[shrinker->id]);
422  }
423  
424  void reparent_shrinker_deferred(struct mem_cgroup *memcg)
425  {
426  	int i, nid;
427  	long nr;
428  	struct mem_cgroup *parent;
429  	struct shrinker_info *child_info, *parent_info;
430  
431  	parent = parent_mem_cgroup(memcg);
432  	if (!parent)
433  		parent = root_mem_cgroup;
434  
435  	/* Prevent from concurrent shrinker_info expand */
436  	mutex_lock(&shrinker_mutex);
437  	for_each_node(nid) {
438  		child_info = shrinker_info_protected(memcg, nid);
439  		parent_info = shrinker_info_protected(parent, nid);
440  		for (i = 0; i < child_info->map_nr_max; i++) {
441  			nr = atomic_long_read(&child_info->nr_deferred[i]);
442  			atomic_long_add(nr, &parent_info->nr_deferred[i]);
443  		}
444  	}
445  	mutex_unlock(&shrinker_mutex);
446  }
447  
448  static bool cgroup_reclaim(struct scan_control *sc)
449  {
450  	return sc->target_mem_cgroup;
451  }
452  
453  static bool global_reclaim(struct scan_control *sc)
454  {
455  	return !sc->target_mem_cgroup || mem_cgroup_is_root(sc->target_mem_cgroup);
456  }
457  
458  /**
459   * writeback_throttling_sane - is the usual dirty throttling mechanism available?
460   * @sc: scan_control in question
461   *
462   * The normal page dirty throttling mechanism in balance_dirty_pages() is
463   * completely broken with the legacy memcg and direct stalling in
464   * shrink_folio_list() is used for throttling instead, which lacks all the
465   * niceties such as fairness, adaptive pausing, bandwidth proportional
466   * allocation and configurability.
467   *
468   * This function tests whether the vmscan currently in progress can assume
469   * that the normal dirty throttling mechanism is operational.
470   */
471  static bool writeback_throttling_sane(struct scan_control *sc)
472  {
473  	if (!cgroup_reclaim(sc))
474  		return true;
475  #ifdef CONFIG_CGROUP_WRITEBACK
476  	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
477  		return true;
478  #endif
479  	return false;
480  }
481  #else
482  static int prealloc_memcg_shrinker(struct shrinker *shrinker)
483  {
484  	return -ENOSYS;
485  }
486  
487  static void unregister_memcg_shrinker(struct shrinker *shrinker)
488  {
489  }
490  
491  static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
492  				   struct mem_cgroup *memcg)
493  {
494  	return 0;
495  }
496  
497  static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
498  				  struct mem_cgroup *memcg)
499  {
500  	return 0;
501  }
502  
503  static bool cgroup_reclaim(struct scan_control *sc)
504  {
505  	return false;
506  }
507  
508  static bool global_reclaim(struct scan_control *sc)
509  {
510  	return true;
511  }
512  
513  static bool writeback_throttling_sane(struct scan_control *sc)
514  {
515  	return true;
516  }
517  #endif
518  
519  static void set_task_reclaim_state(struct task_struct *task,
520  				   struct reclaim_state *rs)
521  {
522  	/* Check for an overwrite */
523  	WARN_ON_ONCE(rs && task->reclaim_state);
524  
525  	/* Check for the nulling of an already-nulled member */
526  	WARN_ON_ONCE(!rs && !task->reclaim_state);
527  
528  	task->reclaim_state = rs;
529  }
530  
531  /*
532   * flush_reclaim_state(): add pages reclaimed outside of LRU-based reclaim to
533   * scan_control->nr_reclaimed.
534   */
535  static void flush_reclaim_state(struct scan_control *sc)
536  {
537  	/*
538  	 * Currently, reclaim_state->reclaimed includes three types of pages
539  	 * freed outside of vmscan:
540  	 * (1) Slab pages.
541  	 * (2) Clean file pages from pruned inodes (on highmem systems).
542  	 * (3) XFS freed buffer pages.
543  	 *
544  	 * For all of these cases, we cannot universally link the pages to a
545  	 * single memcg. For example, a memcg-aware shrinker can free one object
546  	 * charged to the target memcg, causing an entire page to be freed.
547  	 * If we count the entire page as reclaimed from the memcg, we end up
548  	 * overestimating the reclaimed amount (potentially under-reclaiming).
549  	 *
550  	 * Only count such pages for global reclaim to prevent under-reclaiming
551  	 * from the target memcg; preventing unnecessary retries during memcg
552  	 * charging and false positives from proactive reclaim.
553  	 *
554  	 * For uncommon cases where the freed pages were actually mostly
555  	 * charged to the target memcg, we end up underestimating the reclaimed
556  	 * amount. This should be fine. The freed pages will be uncharged
557  	 * anyway, even if they are not counted here properly, and we will be
558  	 * able to make forward progress in charging (which is usually in a
559  	 * retry loop).
560  	 *
561  	 * We can go one step further, and report the uncharged objcg pages in
562  	 * memcg reclaim, to make reporting more accurate and reduce
563  	 * underestimation, but it's probably not worth the complexity for now.
564  	 */
565  	if (current->reclaim_state && global_reclaim(sc)) {
566  		sc->nr_reclaimed += current->reclaim_state->reclaimed;
567  		current->reclaim_state->reclaimed = 0;
568  	}
569  }
570  
571  static long xchg_nr_deferred(struct shrinker *shrinker,
572  			     struct shrink_control *sc)
573  {
574  	int nid = sc->nid;
575  
576  	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
577  		nid = 0;
578  
579  	if (sc->memcg &&
580  	    (shrinker->flags & SHRINKER_MEMCG_AWARE))
581  		return xchg_nr_deferred_memcg(nid, shrinker,
582  					      sc->memcg);
583  
584  	return atomic_long_xchg(&shrinker->nr_deferred[nid], 0);
585  }
586  
587  
588  static long add_nr_deferred(long nr, struct shrinker *shrinker,
589  			    struct shrink_control *sc)
590  {
591  	int nid = sc->nid;
592  
593  	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
594  		nid = 0;
595  
596  	if (sc->memcg &&
597  	    (shrinker->flags & SHRINKER_MEMCG_AWARE))
598  		return add_nr_deferred_memcg(nr, nid, shrinker,
599  					     sc->memcg);
600  
601  	return atomic_long_add_return(nr, &shrinker->nr_deferred[nid]);
602  }
603  
604  static bool can_demote(int nid, struct scan_control *sc)
605  {
606  	if (!numa_demotion_enabled)
607  		return false;
608  	if (sc && sc->no_demotion)
609  		return false;
610  	if (next_demotion_node(nid) == NUMA_NO_NODE)
611  		return false;
612  
613  	return true;
614  }
615  
616  static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg,
617  					  int nid,
618  					  struct scan_control *sc)
619  {
620  	if (memcg == NULL) {
621  		/*
622  		 * For non-memcg reclaim, is there
623  		 * space in any swap device?
624  		 */
625  		if (get_nr_swap_pages() > 0)
626  			return true;
627  	} else {
628  		/* Is the memcg below its swap limit? */
629  		if (mem_cgroup_get_nr_swap_pages(memcg) > 0)
630  			return true;
631  	}
632  
633  	/*
634  	 * The page can not be swapped.
635  	 *
636  	 * Can it be reclaimed from this node via demotion?
637  	 */
638  	return can_demote(nid, sc);
639  }
640  
641  /*
642   * This misses isolated folios which are not accounted for to save counters.
643   * As the data only determines if reclaim or compaction continues, it is
644   * not expected that isolated folios will be a dominating factor.
645   */
646  unsigned long zone_reclaimable_pages(struct zone *zone)
647  {
648  	unsigned long nr;
649  
650  	nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) +
651  		zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE);
652  	if (can_reclaim_anon_pages(NULL, zone_to_nid(zone), NULL))
653  		nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) +
654  			zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON);
655  
656  	return nr;
657  }
658  
659  /**
660   * lruvec_lru_size -  Returns the number of pages on the given LRU list.
661   * @lruvec: lru vector
662   * @lru: lru to use
663   * @zone_idx: zones to consider (use MAX_NR_ZONES - 1 for the whole LRU list)
664   */
665  static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru,
666  				     int zone_idx)
667  {
668  	unsigned long size = 0;
669  	int zid;
670  
671  	for (zid = 0; zid <= zone_idx; zid++) {
672  		struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
673  
674  		if (!managed_zone(zone))
675  			continue;
676  
677  		if (!mem_cgroup_disabled())
678  			size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
679  		else
680  			size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru);
681  	}
682  	return size;
683  }
684  
685  /*
686   * Add a shrinker callback to be called from the vm.
687   */
688  static int __prealloc_shrinker(struct shrinker *shrinker)
689  {
690  	unsigned int size;
691  	int err;
692  
693  	if (shrinker->flags & SHRINKER_MEMCG_AWARE) {
694  		err = prealloc_memcg_shrinker(shrinker);
695  		if (err != -ENOSYS)
696  			return err;
697  
698  		shrinker->flags &= ~SHRINKER_MEMCG_AWARE;
699  	}
700  
701  	size = sizeof(*shrinker->nr_deferred);
702  	if (shrinker->flags & SHRINKER_NUMA_AWARE)
703  		size *= nr_node_ids;
704  
705  	shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
706  	if (!shrinker->nr_deferred)
707  		return -ENOMEM;
708  
709  	return 0;
710  }
711  
712  #ifdef CONFIG_SHRINKER_DEBUG
713  int prealloc_shrinker(struct shrinker *shrinker, const char *fmt, ...)
714  {
715  	va_list ap;
716  	int err;
717  
718  	va_start(ap, fmt);
719  	shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap);
720  	va_end(ap);
721  	if (!shrinker->name)
722  		return -ENOMEM;
723  
724  	err = __prealloc_shrinker(shrinker);
725  	if (err) {
726  		kfree_const(shrinker->name);
727  		shrinker->name = NULL;
728  	}
729  
730  	return err;
731  }
732  #else
733  int prealloc_shrinker(struct shrinker *shrinker, const char *fmt, ...)
734  {
735  	return __prealloc_shrinker(shrinker);
736  }
737  #endif
738  
739  void free_prealloced_shrinker(struct shrinker *shrinker)
740  {
741  #ifdef CONFIG_SHRINKER_DEBUG
742  	kfree_const(shrinker->name);
743  	shrinker->name = NULL;
744  #endif
745  	if (shrinker->flags & SHRINKER_MEMCG_AWARE) {
746  		mutex_lock(&shrinker_mutex);
747  		unregister_memcg_shrinker(shrinker);
748  		mutex_unlock(&shrinker_mutex);
749  		return;
750  	}
751  
752  	kfree(shrinker->nr_deferred);
753  	shrinker->nr_deferred = NULL;
754  }
755  
756  void register_shrinker_prepared(struct shrinker *shrinker)
757  {
758  	mutex_lock(&shrinker_mutex);
759  	list_add_tail_rcu(&shrinker->list, &shrinker_list);
760  	shrinker->flags |= SHRINKER_REGISTERED;
761  	shrinker_debugfs_add(shrinker);
762  	mutex_unlock(&shrinker_mutex);
763  }
764  
765  static int __register_shrinker(struct shrinker *shrinker)
766  {
767  	int err = __prealloc_shrinker(shrinker);
768  
769  	if (err)
770  		return err;
771  	register_shrinker_prepared(shrinker);
772  	return 0;
773  }
774  
775  #ifdef CONFIG_SHRINKER_DEBUG
776  int register_shrinker(struct shrinker *shrinker, const char *fmt, ...)
777  {
778  	va_list ap;
779  	int err;
780  
781  	va_start(ap, fmt);
782  	shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap);
783  	va_end(ap);
784  	if (!shrinker->name)
785  		return -ENOMEM;
786  
787  	err = __register_shrinker(shrinker);
788  	if (err) {
789  		kfree_const(shrinker->name);
790  		shrinker->name = NULL;
791  	}
792  	return err;
793  }
794  #else
795  int register_shrinker(struct shrinker *shrinker, const char *fmt, ...)
796  {
797  	return __register_shrinker(shrinker);
798  }
799  #endif
800  EXPORT_SYMBOL(register_shrinker);
801  
802  /*
803   * Remove one
804   */
805  void unregister_shrinker(struct shrinker *shrinker)
806  {
807  	struct dentry *debugfs_entry;
808  	int debugfs_id;
809  
810  	if (!(shrinker->flags & SHRINKER_REGISTERED))
811  		return;
812  
813  	mutex_lock(&shrinker_mutex);
814  	list_del_rcu(&shrinker->list);
815  	shrinker->flags &= ~SHRINKER_REGISTERED;
816  	if (shrinker->flags & SHRINKER_MEMCG_AWARE)
817  		unregister_memcg_shrinker(shrinker);
818  	debugfs_entry = shrinker_debugfs_detach(shrinker, &debugfs_id);
819  	mutex_unlock(&shrinker_mutex);
820  
821  	atomic_inc(&shrinker_srcu_generation);
822  	synchronize_srcu(&shrinker_srcu);
823  
824  	shrinker_debugfs_remove(debugfs_entry, debugfs_id);
825  
826  	kfree(shrinker->nr_deferred);
827  	shrinker->nr_deferred = NULL;
828  }
829  EXPORT_SYMBOL(unregister_shrinker);
830  
831  /**
832   * synchronize_shrinkers - Wait for all running shrinkers to complete.
833   *
834   * This is useful to guarantee that all shrinker invocations have seen an
835   * update, before freeing memory.
836   */
837  void synchronize_shrinkers(void)
838  {
839  	atomic_inc(&shrinker_srcu_generation);
840  	synchronize_srcu(&shrinker_srcu);
841  }
842  EXPORT_SYMBOL(synchronize_shrinkers);
843  
844  #define SHRINK_BATCH 128
845  
846  static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,
847  				    struct shrinker *shrinker, int priority)
848  {
849  	unsigned long freed = 0;
850  	unsigned long long delta;
851  	long total_scan;
852  	long freeable;
853  	long nr;
854  	long new_nr;
855  	long batch_size = shrinker->batch ? shrinker->batch
856  					  : SHRINK_BATCH;
857  	long scanned = 0, next_deferred;
858  
859  	freeable = shrinker->count_objects(shrinker, shrinkctl);
860  	if (freeable == 0 || freeable == SHRINK_EMPTY)
861  		return freeable;
862  
863  	/*
864  	 * copy the current shrinker scan count into a local variable
865  	 * and zero it so that other concurrent shrinker invocations
866  	 * don't also do this scanning work.
867  	 */
868  	nr = xchg_nr_deferred(shrinker, shrinkctl);
869  
870  	if (shrinker->seeks) {
871  		delta = freeable >> priority;
872  		delta *= 4;
873  		do_div(delta, shrinker->seeks);
874  	} else {
875  		/*
876  		 * These objects don't require any IO to create. Trim
877  		 * them aggressively under memory pressure to keep
878  		 * them from causing refetches in the IO caches.
879  		 */
880  		delta = freeable / 2;
881  	}
882  
883  	total_scan = nr >> priority;
884  	total_scan += delta;
885  	total_scan = min(total_scan, (2 * freeable));
886  
887  	trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
888  				   freeable, delta, total_scan, priority);
889  
890  	/*
891  	 * Normally, we should not scan less than batch_size objects in one
892  	 * pass to avoid too frequent shrinker calls, but if the slab has less
893  	 * than batch_size objects in total and we are really tight on memory,
894  	 * we will try to reclaim all available objects, otherwise we can end
895  	 * up failing allocations although there are plenty of reclaimable
896  	 * objects spread over several slabs with usage less than the
897  	 * batch_size.
898  	 *
899  	 * We detect the "tight on memory" situations by looking at the total
900  	 * number of objects we want to scan (total_scan). If it is greater
901  	 * than the total number of objects on slab (freeable), we must be
902  	 * scanning at high prio and therefore should try to reclaim as much as
903  	 * possible.
904  	 */
905  	while (total_scan >= batch_size ||
906  	       total_scan >= freeable) {
907  		unsigned long ret;
908  		unsigned long nr_to_scan = min(batch_size, total_scan);
909  
910  		shrinkctl->nr_to_scan = nr_to_scan;
911  		shrinkctl->nr_scanned = nr_to_scan;
912  		ret = shrinker->scan_objects(shrinker, shrinkctl);
913  		if (ret == SHRINK_STOP)
914  			break;
915  		freed += ret;
916  
917  		count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned);
918  		total_scan -= shrinkctl->nr_scanned;
919  		scanned += shrinkctl->nr_scanned;
920  
921  		cond_resched();
922  	}
923  
924  	/*
925  	 * The deferred work is increased by any new work (delta) that wasn't
926  	 * done, decreased by old deferred work that was done now.
927  	 *
928  	 * And it is capped to two times of the freeable items.
929  	 */
930  	next_deferred = max_t(long, (nr + delta - scanned), 0);
931  	next_deferred = min(next_deferred, (2 * freeable));
932  
933  	/*
934  	 * move the unused scan count back into the shrinker in a
935  	 * manner that handles concurrent updates.
936  	 */
937  	new_nr = add_nr_deferred(next_deferred, shrinker, shrinkctl);
938  
939  	trace_mm_shrink_slab_end(shrinker, shrinkctl->nid, freed, nr, new_nr, total_scan);
940  	return freed;
941  }
942  
943  #ifdef CONFIG_MEMCG
944  static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
945  			struct mem_cgroup *memcg, int priority)
946  {
947  	struct shrinker_info *info;
948  	unsigned long ret, freed = 0;
949  	int srcu_idx, generation;
950  	int i = 0;
951  
952  	if (!mem_cgroup_online(memcg))
953  		return 0;
954  
955  again:
956  	srcu_idx = srcu_read_lock(&shrinker_srcu);
957  	info = shrinker_info_srcu(memcg, nid);
958  	if (unlikely(!info))
959  		goto unlock;
960  
961  	generation = atomic_read(&shrinker_srcu_generation);
962  	for_each_set_bit_from(i, info->map, info->map_nr_max) {
963  		struct shrink_control sc = {
964  			.gfp_mask = gfp_mask,
965  			.nid = nid,
966  			.memcg = memcg,
967  		};
968  		struct shrinker *shrinker;
969  
970  		shrinker = idr_find(&shrinker_idr, i);
971  		if (unlikely(!shrinker || !(shrinker->flags & SHRINKER_REGISTERED))) {
972  			if (!shrinker)
973  				clear_bit(i, info->map);
974  			continue;
975  		}
976  
977  		/* Call non-slab shrinkers even though kmem is disabled */
978  		if (!memcg_kmem_online() &&
979  		    !(shrinker->flags & SHRINKER_NONSLAB))
980  			continue;
981  
982  		ret = do_shrink_slab(&sc, shrinker, priority);
983  		if (ret == SHRINK_EMPTY) {
984  			clear_bit(i, info->map);
985  			/*
986  			 * After the shrinker reported that it had no objects to
987  			 * free, but before we cleared the corresponding bit in
988  			 * the memcg shrinker map, a new object might have been
989  			 * added. To make sure, we have the bit set in this
990  			 * case, we invoke the shrinker one more time and reset
991  			 * the bit if it reports that it is not empty anymore.
992  			 * The memory barrier here pairs with the barrier in
993  			 * set_shrinker_bit():
994  			 *
995  			 * list_lru_add()     shrink_slab_memcg()
996  			 *   list_add_tail()    clear_bit()
997  			 *   <MB>               <MB>
998  			 *   set_bit()          do_shrink_slab()
999  			 */
1000  			smp_mb__after_atomic();
1001  			ret = do_shrink_slab(&sc, shrinker, priority);
1002  			if (ret == SHRINK_EMPTY)
1003  				ret = 0;
1004  			else
1005  				set_shrinker_bit(memcg, nid, i);
1006  		}
1007  		freed += ret;
1008  		if (atomic_read(&shrinker_srcu_generation) != generation) {
1009  			srcu_read_unlock(&shrinker_srcu, srcu_idx);
1010  			i++;
1011  			goto again;
1012  		}
1013  	}
1014  unlock:
1015  	srcu_read_unlock(&shrinker_srcu, srcu_idx);
1016  	return freed;
1017  }
1018  #else /* CONFIG_MEMCG */
1019  static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
1020  			struct mem_cgroup *memcg, int priority)
1021  {
1022  	return 0;
1023  }
1024  #endif /* CONFIG_MEMCG */
1025  
1026  /**
1027   * shrink_slab - shrink slab caches
1028   * @gfp_mask: allocation context
1029   * @nid: node whose slab caches to target
1030   * @memcg: memory cgroup whose slab caches to target
1031   * @priority: the reclaim priority
1032   *
1033   * Call the shrink functions to age shrinkable caches.
1034   *
1035   * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set,
1036   * unaware shrinkers will receive a node id of 0 instead.
1037   *
1038   * @memcg specifies the memory cgroup to target. Unaware shrinkers
1039   * are called only if it is the root cgroup.
1040   *
1041   * @priority is sc->priority, we take the number of objects and >> by priority
1042   * in order to get the scan target.
1043   *
1044   * Returns the number of reclaimed slab objects.
1045   */
1046  static unsigned long shrink_slab(gfp_t gfp_mask, int nid,
1047  				 struct mem_cgroup *memcg,
1048  				 int priority)
1049  {
1050  	unsigned long ret, freed = 0;
1051  	struct shrinker *shrinker;
1052  	int srcu_idx, generation;
1053  
1054  	/*
1055  	 * The root memcg might be allocated even though memcg is disabled
1056  	 * via "cgroup_disable=memory" boot parameter.  This could make
1057  	 * mem_cgroup_is_root() return false, then just run memcg slab
1058  	 * shrink, but skip global shrink.  This may result in premature
1059  	 * oom.
1060  	 */
1061  	if (!mem_cgroup_disabled() && !mem_cgroup_is_root(memcg))
1062  		return shrink_slab_memcg(gfp_mask, nid, memcg, priority);
1063  
1064  	srcu_idx = srcu_read_lock(&shrinker_srcu);
1065  
1066  	generation = atomic_read(&shrinker_srcu_generation);
1067  	list_for_each_entry_srcu(shrinker, &shrinker_list, list,
1068  				 srcu_read_lock_held(&shrinker_srcu)) {
1069  		struct shrink_control sc = {
1070  			.gfp_mask = gfp_mask,
1071  			.nid = nid,
1072  			.memcg = memcg,
1073  		};
1074  
1075  		ret = do_shrink_slab(&sc, shrinker, priority);
1076  		if (ret == SHRINK_EMPTY)
1077  			ret = 0;
1078  		freed += ret;
1079  
1080  		if (atomic_read(&shrinker_srcu_generation) != generation) {
1081  			freed = freed ? : 1;
1082  			break;
1083  		}
1084  	}
1085  
1086  	srcu_read_unlock(&shrinker_srcu, srcu_idx);
1087  	cond_resched();
1088  	return freed;
1089  }
1090  
1091  static unsigned long drop_slab_node(int nid)
1092  {
1093  	unsigned long freed = 0;
1094  	struct mem_cgroup *memcg = NULL;
1095  
1096  	memcg = mem_cgroup_iter(NULL, NULL, NULL);
1097  	do {
1098  		freed += shrink_slab(GFP_KERNEL, nid, memcg, 0);
1099  	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
1100  
1101  	return freed;
1102  }
1103  
1104  void drop_slab(void)
1105  {
1106  	int nid;
1107  	int shift = 0;
1108  	unsigned long freed;
1109  
1110  	do {
1111  		freed = 0;
1112  		for_each_online_node(nid) {
1113  			if (fatal_signal_pending(current))
1114  				return;
1115  
1116  			freed += drop_slab_node(nid);
1117  		}
1118  	} while ((freed >> shift++) > 1);
1119  }
1120  
1121  static int reclaimer_offset(void)
1122  {
1123  	BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD !=
1124  			PGDEMOTE_DIRECT - PGDEMOTE_KSWAPD);
1125  	BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD !=
1126  			PGSCAN_DIRECT - PGSCAN_KSWAPD);
1127  	BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD !=
1128  			PGDEMOTE_KHUGEPAGED - PGDEMOTE_KSWAPD);
1129  	BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD !=
1130  			PGSCAN_KHUGEPAGED - PGSCAN_KSWAPD);
1131  
1132  	if (current_is_kswapd())
1133  		return 0;
1134  	if (current_is_khugepaged())
1135  		return PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD;
1136  	return PGSTEAL_DIRECT - PGSTEAL_KSWAPD;
1137  }
1138  
1139  static inline int is_page_cache_freeable(struct folio *folio)
1140  {
1141  	/*
1142  	 * A freeable page cache folio is referenced only by the caller
1143  	 * that isolated the folio, the page cache and optional filesystem
1144  	 * private data at folio->private.
1145  	 */
1146  	return folio_ref_count(folio) - folio_test_private(folio) ==
1147  		1 + folio_nr_pages(folio);
1148  }
1149  
1150  /*
1151   * We detected a synchronous write error writing a folio out.  Probably
1152   * -ENOSPC.  We need to propagate that into the address_space for a subsequent
1153   * fsync(), msync() or close().
1154   *
1155   * The tricky part is that after writepage we cannot touch the mapping: nothing
1156   * prevents it from being freed up.  But we have a ref on the folio and once
1157   * that folio is locked, the mapping is pinned.
1158   *
1159   * We're allowed to run sleeping folio_lock() here because we know the caller has
1160   * __GFP_FS.
1161   */
1162  static void handle_write_error(struct address_space *mapping,
1163  				struct folio *folio, int error)
1164  {
1165  	folio_lock(folio);
1166  	if (folio_mapping(folio) == mapping)
1167  		mapping_set_error(mapping, error);
1168  	folio_unlock(folio);
1169  }
1170  
1171  static bool skip_throttle_noprogress(pg_data_t *pgdat)
1172  {
1173  	int reclaimable = 0, write_pending = 0;
1174  	int i;
1175  
1176  	/*
1177  	 * If kswapd is disabled, reschedule if necessary but do not
1178  	 * throttle as the system is likely near OOM.
1179  	 */
1180  	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
1181  		return true;
1182  
1183  	/*
1184  	 * If there are a lot of dirty/writeback folios then do not
1185  	 * throttle as throttling will occur when the folios cycle
1186  	 * towards the end of the LRU if still under writeback.
1187  	 */
1188  	for (i = 0; i < MAX_NR_ZONES; i++) {
1189  		struct zone *zone = pgdat->node_zones + i;
1190  
1191  		if (!managed_zone(zone))
1192  			continue;
1193  
1194  		reclaimable += zone_reclaimable_pages(zone);
1195  		write_pending += zone_page_state_snapshot(zone,
1196  						  NR_ZONE_WRITE_PENDING);
1197  	}
1198  	if (2 * write_pending <= reclaimable)
1199  		return true;
1200  
1201  	return false;
1202  }
1203  
1204  void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason)
1205  {
1206  	wait_queue_head_t *wqh = &pgdat->reclaim_wait[reason];
1207  	long timeout, ret;
1208  	DEFINE_WAIT(wait);
1209  
1210  	/*
1211  	 * Do not throttle user workers, kthreads other than kswapd or
1212  	 * workqueues. They may be required for reclaim to make
1213  	 * forward progress (e.g. journalling workqueues or kthreads).
1214  	 */
1215  	if (!current_is_kswapd() &&
1216  	    current->flags & (PF_USER_WORKER|PF_KTHREAD)) {
1217  		cond_resched();
1218  		return;
1219  	}
1220  
1221  	/*
1222  	 * These figures are pulled out of thin air.
1223  	 * VMSCAN_THROTTLE_ISOLATED is a transient condition based on too many
1224  	 * parallel reclaimers which is a short-lived event so the timeout is
1225  	 * short. Failing to make progress or waiting on writeback are
1226  	 * potentially long-lived events so use a longer timeout. This is shaky
1227  	 * logic as a failure to make progress could be due to anything from
1228  	 * writeback to a slow device to excessive referenced folios at the tail
1229  	 * of the inactive LRU.
1230  	 */
1231  	switch(reason) {
1232  	case VMSCAN_THROTTLE_WRITEBACK:
1233  		timeout = HZ/10;
1234  
1235  		if (atomic_inc_return(&pgdat->nr_writeback_throttled) == 1) {
1236  			WRITE_ONCE(pgdat->nr_reclaim_start,
1237  				node_page_state(pgdat, NR_THROTTLED_WRITTEN));
1238  		}
1239  
1240  		break;
1241  	case VMSCAN_THROTTLE_CONGESTED:
1242  		fallthrough;
1243  	case VMSCAN_THROTTLE_NOPROGRESS:
1244  		if (skip_throttle_noprogress(pgdat)) {
1245  			cond_resched();
1246  			return;
1247  		}
1248  
1249  		timeout = 1;
1250  
1251  		break;
1252  	case VMSCAN_THROTTLE_ISOLATED:
1253  		timeout = HZ/50;
1254  		break;
1255  	default:
1256  		WARN_ON_ONCE(1);
1257  		timeout = HZ;
1258  		break;
1259  	}
1260  
1261  	prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
1262  	ret = schedule_timeout(timeout);
1263  	finish_wait(wqh, &wait);
1264  
1265  	if (reason == VMSCAN_THROTTLE_WRITEBACK)
1266  		atomic_dec(&pgdat->nr_writeback_throttled);
1267  
1268  	trace_mm_vmscan_throttled(pgdat->node_id, jiffies_to_usecs(timeout),
1269  				jiffies_to_usecs(timeout - ret),
1270  				reason);
1271  }
1272  
1273  /*
1274   * Account for folios written if tasks are throttled waiting on dirty
1275   * folios to clean. If enough folios have been cleaned since throttling
1276   * started then wakeup the throttled tasks.
1277   */
1278  void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,
1279  							int nr_throttled)
1280  {
1281  	unsigned long nr_written;
1282  
1283  	node_stat_add_folio(folio, NR_THROTTLED_WRITTEN);
1284  
1285  	/*
1286  	 * This is an inaccurate read as the per-cpu deltas may not
1287  	 * be synchronised. However, given that the system is
1288  	 * writeback throttled, it is not worth taking the penalty
1289  	 * of getting an accurate count. At worst, the throttle
1290  	 * timeout guarantees forward progress.
1291  	 */
1292  	nr_written = node_page_state(pgdat, NR_THROTTLED_WRITTEN) -
1293  		READ_ONCE(pgdat->nr_reclaim_start);
1294  
1295  	if (nr_written > SWAP_CLUSTER_MAX * nr_throttled)
1296  		wake_up(&pgdat->reclaim_wait[VMSCAN_THROTTLE_WRITEBACK]);
1297  }
1298  
1299  /* possible outcome of pageout() */
1300  typedef enum {
1301  	/* failed to write folio out, folio is locked */
1302  	PAGE_KEEP,
1303  	/* move folio to the active list, folio is locked */
1304  	PAGE_ACTIVATE,
1305  	/* folio has been sent to the disk successfully, folio is unlocked */
1306  	PAGE_SUCCESS,
1307  	/* folio is clean and locked */
1308  	PAGE_CLEAN,
1309  } pageout_t;
1310  
1311  /*
1312   * pageout is called by shrink_folio_list() for each dirty folio.
1313   * Calls ->writepage().
1314   */
1315  static pageout_t pageout(struct folio *folio, struct address_space *mapping,
1316  			 struct swap_iocb **plug)
1317  {
1318  	/*
1319  	 * If the folio is dirty, only perform writeback if that write
1320  	 * will be non-blocking.  To prevent this allocation from being
1321  	 * stalled by pagecache activity.  But note that there may be
1322  	 * stalls if we need to run get_block().  We could test
1323  	 * PagePrivate for that.
1324  	 *
1325  	 * If this process is currently in __generic_file_write_iter() against
1326  	 * this folio's queue, we can perform writeback even if that
1327  	 * will block.
1328  	 *
1329  	 * If the folio is swapcache, write it back even if that would
1330  	 * block, for some throttling. This happens by accident, because
1331  	 * swap_backing_dev_info is bust: it doesn't reflect the
1332  	 * congestion state of the swapdevs.  Easy to fix, if needed.
1333  	 */
1334  	if (!is_page_cache_freeable(folio))
1335  		return PAGE_KEEP;
1336  	if (!mapping) {
1337  		/*
1338  		 * Some data journaling orphaned folios can have
1339  		 * folio->mapping == NULL while being dirty with clean buffers.
1340  		 */
1341  		if (folio_test_private(folio)) {
1342  			if (try_to_free_buffers(folio)) {
1343  				folio_clear_dirty(folio);
1344  				pr_info("%s: orphaned folio\n", __func__);
1345  				return PAGE_CLEAN;
1346  			}
1347  		}
1348  		return PAGE_KEEP;
1349  	}
1350  	if (mapping->a_ops->writepage == NULL)
1351  		return PAGE_ACTIVATE;
1352  
1353  	if (folio_clear_dirty_for_io(folio)) {
1354  		int res;
1355  		struct writeback_control wbc = {
1356  			.sync_mode = WB_SYNC_NONE,
1357  			.nr_to_write = SWAP_CLUSTER_MAX,
1358  			.range_start = 0,
1359  			.range_end = LLONG_MAX,
1360  			.for_reclaim = 1,
1361  			.swap_plug = plug,
1362  		};
1363  
1364  		folio_set_reclaim(folio);
1365  		res = mapping->a_ops->writepage(&folio->page, &wbc);
1366  		if (res < 0)
1367  			handle_write_error(mapping, folio, res);
1368  		if (res == AOP_WRITEPAGE_ACTIVATE) {
1369  			folio_clear_reclaim(folio);
1370  			return PAGE_ACTIVATE;
1371  		}
1372  
1373  		if (!folio_test_writeback(folio)) {
1374  			/* synchronous write or broken a_ops? */
1375  			folio_clear_reclaim(folio);
1376  		}
1377  		trace_mm_vmscan_write_folio(folio);
1378  		node_stat_add_folio(folio, NR_VMSCAN_WRITE);
1379  		return PAGE_SUCCESS;
1380  	}
1381  
1382  	return PAGE_CLEAN;
1383  }
1384  
1385  /*
1386   * Same as remove_mapping, but if the folio is removed from the mapping, it
1387   * gets returned with a refcount of 0.
1388   */
1389  static int __remove_mapping(struct address_space *mapping, struct folio *folio,
1390  			    bool reclaimed, struct mem_cgroup *target_memcg)
1391  {
1392  	int refcount;
1393  	void *shadow = NULL;
1394  
1395  	BUG_ON(!folio_test_locked(folio));
1396  	BUG_ON(mapping != folio_mapping(folio));
1397  
1398  	if (!folio_test_swapcache(folio))
1399  		spin_lock(&mapping->host->i_lock);
1400  	xa_lock_irq(&mapping->i_pages);
1401  	/*
1402  	 * The non racy check for a busy folio.
1403  	 *
1404  	 * Must be careful with the order of the tests. When someone has
1405  	 * a ref to the folio, it may be possible that they dirty it then
1406  	 * drop the reference. So if the dirty flag is tested before the
1407  	 * refcount here, then the following race may occur:
1408  	 *
1409  	 * get_user_pages(&page);
1410  	 * [user mapping goes away]
1411  	 * write_to(page);
1412  	 *				!folio_test_dirty(folio)    [good]
1413  	 * folio_set_dirty(folio);
1414  	 * folio_put(folio);
1415  	 *				!refcount(folio)   [good, discard it]
1416  	 *
1417  	 * [oops, our write_to data is lost]
1418  	 *
1419  	 * Reversing the order of the tests ensures such a situation cannot
1420  	 * escape unnoticed. The smp_rmb is needed to ensure the folio->flags
1421  	 * load is not satisfied before that of folio->_refcount.
1422  	 *
1423  	 * Note that if the dirty flag is always set via folio_mark_dirty,
1424  	 * and thus under the i_pages lock, then this ordering is not required.
1425  	 */
1426  	refcount = 1 + folio_nr_pages(folio);
1427  	if (!folio_ref_freeze(folio, refcount))
1428  		goto cannot_free;
1429  	/* note: atomic_cmpxchg in folio_ref_freeze provides the smp_rmb */
1430  	if (unlikely(folio_test_dirty(folio))) {
1431  		folio_ref_unfreeze(folio, refcount);
1432  		goto cannot_free;
1433  	}
1434  
1435  	if (folio_test_swapcache(folio)) {
1436  		swp_entry_t swap = folio_swap_entry(folio);
1437  
1438  		if (reclaimed && !mapping_exiting(mapping))
1439  			shadow = workingset_eviction(folio, target_memcg);
1440  		__delete_from_swap_cache(folio, swap, shadow);
1441  		mem_cgroup_swapout(folio, swap);
1442  		xa_unlock_irq(&mapping->i_pages);
1443  		put_swap_folio(folio, swap);
1444  	} else {
1445  		void (*free_folio)(struct folio *);
1446  
1447  		free_folio = mapping->a_ops->free_folio;
1448  		/*
1449  		 * Remember a shadow entry for reclaimed file cache in
1450  		 * order to detect refaults, thus thrashing, later on.
1451  		 *
1452  		 * But don't store shadows in an address space that is
1453  		 * already exiting.  This is not just an optimization,
1454  		 * inode reclaim needs to empty out the radix tree or
1455  		 * the nodes are lost.  Don't plant shadows behind its
1456  		 * back.
1457  		 *
1458  		 * We also don't store shadows for DAX mappings because the
1459  		 * only page cache folios found in these are zero pages
1460  		 * covering holes, and because we don't want to mix DAX
1461  		 * exceptional entries and shadow exceptional entries in the
1462  		 * same address_space.
1463  		 */
1464  		if (reclaimed && folio_is_file_lru(folio) &&
1465  		    !mapping_exiting(mapping) && !dax_mapping(mapping))
1466  			shadow = workingset_eviction(folio, target_memcg);
1467  		__filemap_remove_folio(folio, shadow);
1468  		xa_unlock_irq(&mapping->i_pages);
1469  		if (mapping_shrinkable(mapping))
1470  			inode_add_lru(mapping->host);
1471  		spin_unlock(&mapping->host->i_lock);
1472  
1473  		if (free_folio)
1474  			free_folio(folio);
1475  	}
1476  
1477  	return 1;
1478  
1479  cannot_free:
1480  	xa_unlock_irq(&mapping->i_pages);
1481  	if (!folio_test_swapcache(folio))
1482  		spin_unlock(&mapping->host->i_lock);
1483  	return 0;
1484  }
1485  
1486  /**
1487   * remove_mapping() - Attempt to remove a folio from its mapping.
1488   * @mapping: The address space.
1489   * @folio: The folio to remove.
1490   *
1491   * If the folio is dirty, under writeback or if someone else has a ref
1492   * on it, removal will fail.
1493   * Return: The number of pages removed from the mapping.  0 if the folio
1494   * could not be removed.
1495   * Context: The caller should have a single refcount on the folio and
1496   * hold its lock.
1497   */
1498  long remove_mapping(struct address_space *mapping, struct folio *folio)
1499  {
1500  	if (__remove_mapping(mapping, folio, false, NULL)) {
1501  		/*
1502  		 * Unfreezing the refcount with 1 effectively
1503  		 * drops the pagecache ref for us without requiring another
1504  		 * atomic operation.
1505  		 */
1506  		folio_ref_unfreeze(folio, 1);
1507  		return folio_nr_pages(folio);
1508  	}
1509  	return 0;
1510  }
1511  
1512  /**
1513   * folio_putback_lru - Put previously isolated folio onto appropriate LRU list.
1514   * @folio: Folio to be returned to an LRU list.
1515   *
1516   * Add previously isolated @folio to appropriate LRU list.
1517   * The folio may still be unevictable for other reasons.
1518   *
1519   * Context: lru_lock must not be held, interrupts must be enabled.
1520   */
1521  void folio_putback_lru(struct folio *folio)
1522  {
1523  	folio_add_lru(folio);
1524  	folio_put(folio);		/* drop ref from isolate */
1525  }
1526  
1527  enum folio_references {
1528  	FOLIOREF_RECLAIM,
1529  	FOLIOREF_RECLAIM_CLEAN,
1530  	FOLIOREF_KEEP,
1531  	FOLIOREF_ACTIVATE,
1532  };
1533  
1534  static enum folio_references folio_check_references(struct folio *folio,
1535  						  struct scan_control *sc)
1536  {
1537  	int referenced_ptes, referenced_folio;
1538  	unsigned long vm_flags;
1539  
1540  	referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup,
1541  					   &vm_flags);
1542  	referenced_folio = folio_test_clear_referenced(folio);
1543  
1544  	/*
1545  	 * The supposedly reclaimable folio was found to be in a VM_LOCKED vma.
1546  	 * Let the folio, now marked Mlocked, be moved to the unevictable list.
1547  	 */
1548  	if (vm_flags & VM_LOCKED)
1549  		return FOLIOREF_ACTIVATE;
1550  
1551  	/* rmap lock contention: rotate */
1552  	if (referenced_ptes == -1)
1553  		return FOLIOREF_KEEP;
1554  
1555  	if (referenced_ptes) {
1556  		/*
1557  		 * All mapped folios start out with page table
1558  		 * references from the instantiating fault, so we need
1559  		 * to look twice if a mapped file/anon folio is used more
1560  		 * than once.
1561  		 *
1562  		 * Mark it and spare it for another trip around the
1563  		 * inactive list.  Another page table reference will
1564  		 * lead to its activation.
1565  		 *
1566  		 * Note: the mark is set for activated folios as well
1567  		 * so that recently deactivated but used folios are
1568  		 * quickly recovered.
1569  		 */
1570  		folio_set_referenced(folio);
1571  
1572  		if (referenced_folio || referenced_ptes > 1)
1573  			return FOLIOREF_ACTIVATE;
1574  
1575  		/*
1576  		 * Activate file-backed executable folios after first usage.
1577  		 */
1578  		if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio))
1579  			return FOLIOREF_ACTIVATE;
1580  
1581  		return FOLIOREF_KEEP;
1582  	}
1583  
1584  	/* Reclaim if clean, defer dirty folios to writeback */
1585  	if (referenced_folio && folio_is_file_lru(folio))
1586  		return FOLIOREF_RECLAIM_CLEAN;
1587  
1588  	return FOLIOREF_RECLAIM;
1589  }
1590  
1591  /* Check if a folio is dirty or under writeback */
1592  static void folio_check_dirty_writeback(struct folio *folio,
1593  				       bool *dirty, bool *writeback)
1594  {
1595  	struct address_space *mapping;
1596  
1597  	/*
1598  	 * Anonymous folios are not handled by flushers and must be written
1599  	 * from reclaim context. Do not stall reclaim based on them.
1600  	 * MADV_FREE anonymous folios are put into inactive file list too.
1601  	 * They could be mistakenly treated as file lru. So further anon
1602  	 * test is needed.
1603  	 */
1604  	if (!folio_is_file_lru(folio) ||
1605  	    (folio_test_anon(folio) && !folio_test_swapbacked(folio))) {
1606  		*dirty = false;
1607  		*writeback = false;
1608  		return;
1609  	}
1610  
1611  	/* By default assume that the folio flags are accurate */
1612  	*dirty = folio_test_dirty(folio);
1613  	*writeback = folio_test_writeback(folio);
1614  
1615  	/* Verify dirty/writeback state if the filesystem supports it */
1616  	if (!folio_test_private(folio))
1617  		return;
1618  
1619  	mapping = folio_mapping(folio);
1620  	if (mapping && mapping->a_ops->is_dirty_writeback)
1621  		mapping->a_ops->is_dirty_writeback(folio, dirty, writeback);
1622  }
1623  
1624  static struct page *alloc_demote_page(struct page *page, unsigned long private)
1625  {
1626  	struct page *target_page;
1627  	nodemask_t *allowed_mask;
1628  	struct migration_target_control *mtc;
1629  
1630  	mtc = (struct migration_target_control *)private;
1631  
1632  	allowed_mask = mtc->nmask;
1633  	/*
1634  	 * make sure we allocate from the target node first also trying to
1635  	 * demote or reclaim pages from the target node via kswapd if we are
1636  	 * low on free memory on target node. If we don't do this and if
1637  	 * we have free memory on the slower(lower) memtier, we would start
1638  	 * allocating pages from slower(lower) memory tiers without even forcing
1639  	 * a demotion of cold pages from the target memtier. This can result
1640  	 * in the kernel placing hot pages in slower(lower) memory tiers.
1641  	 */
1642  	mtc->nmask = NULL;
1643  	mtc->gfp_mask |= __GFP_THISNODE;
1644  	target_page = alloc_migration_target(page, (unsigned long)mtc);
1645  	if (target_page)
1646  		return target_page;
1647  
1648  	mtc->gfp_mask &= ~__GFP_THISNODE;
1649  	mtc->nmask = allowed_mask;
1650  
1651  	return alloc_migration_target(page, (unsigned long)mtc);
1652  }
1653  
1654  /*
1655   * Take folios on @demote_folios and attempt to demote them to another node.
1656   * Folios which are not demoted are left on @demote_folios.
1657   */
1658  static unsigned int demote_folio_list(struct list_head *demote_folios,
1659  				     struct pglist_data *pgdat)
1660  {
1661  	int target_nid = next_demotion_node(pgdat->node_id);
1662  	unsigned int nr_succeeded;
1663  	nodemask_t allowed_mask;
1664  
1665  	struct migration_target_control mtc = {
1666  		/*
1667  		 * Allocate from 'node', or fail quickly and quietly.
1668  		 * When this happens, 'page' will likely just be discarded
1669  		 * instead of migrated.
1670  		 */
1671  		.gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN |
1672  			__GFP_NOMEMALLOC | GFP_NOWAIT,
1673  		.nid = target_nid,
1674  		.nmask = &allowed_mask
1675  	};
1676  
1677  	if (list_empty(demote_folios))
1678  		return 0;
1679  
1680  	if (target_nid == NUMA_NO_NODE)
1681  		return 0;
1682  
1683  	node_get_allowed_targets(pgdat, &allowed_mask);
1684  
1685  	/* Demotion ignores all cpuset and mempolicy settings */
1686  	migrate_pages(demote_folios, alloc_demote_page, NULL,
1687  		      (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION,
1688  		      &nr_succeeded);
1689  
1690  	__count_vm_events(PGDEMOTE_KSWAPD + reclaimer_offset(), nr_succeeded);
1691  
1692  	return nr_succeeded;
1693  }
1694  
1695  static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask)
1696  {
1697  	if (gfp_mask & __GFP_FS)
1698  		return true;
1699  	if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO))
1700  		return false;
1701  	/*
1702  	 * We can "enter_fs" for swap-cache with only __GFP_IO
1703  	 * providing this isn't SWP_FS_OPS.
1704  	 * ->flags can be updated non-atomicially (scan_swap_map_slots),
1705  	 * but that will never affect SWP_FS_OPS, so the data_race
1706  	 * is safe.
1707  	 */
1708  	return !data_race(folio_swap_flags(folio) & SWP_FS_OPS);
1709  }
1710  
1711  /*
1712   * shrink_folio_list() returns the number of reclaimed pages
1713   */
1714  static unsigned int shrink_folio_list(struct list_head *folio_list,
1715  		struct pglist_data *pgdat, struct scan_control *sc,
1716  		struct reclaim_stat *stat, bool ignore_references)
1717  {
1718  	LIST_HEAD(ret_folios);
1719  	LIST_HEAD(free_folios);
1720  	LIST_HEAD(demote_folios);
1721  	unsigned int nr_reclaimed = 0;
1722  	unsigned int pgactivate = 0;
1723  	bool do_demote_pass;
1724  	struct swap_iocb *plug = NULL;
1725  
1726  	memset(stat, 0, sizeof(*stat));
1727  	cond_resched();
1728  	do_demote_pass = can_demote(pgdat->node_id, sc);
1729  
1730  retry:
1731  	while (!list_empty(folio_list)) {
1732  		struct address_space *mapping;
1733  		struct folio *folio;
1734  		enum folio_references references = FOLIOREF_RECLAIM;
1735  		bool dirty, writeback;
1736  		unsigned int nr_pages;
1737  
1738  		cond_resched();
1739  
1740  		folio = lru_to_folio(folio_list);
1741  		list_del(&folio->lru);
1742  
1743  		if (!folio_trylock(folio))
1744  			goto keep;
1745  
1746  		VM_BUG_ON_FOLIO(folio_test_active(folio), folio);
1747  
1748  		nr_pages = folio_nr_pages(folio);
1749  
1750  		/* Account the number of base pages */
1751  		sc->nr_scanned += nr_pages;
1752  
1753  		if (unlikely(!folio_evictable(folio)))
1754  			goto activate_locked;
1755  
1756  		if (!sc->may_unmap && folio_mapped(folio))
1757  			goto keep_locked;
1758  
1759  		/* folio_update_gen() tried to promote this page? */
1760  		if (lru_gen_enabled() && !ignore_references &&
1761  		    folio_mapped(folio) && folio_test_referenced(folio))
1762  			goto keep_locked;
1763  
1764  		/*
1765  		 * The number of dirty pages determines if a node is marked
1766  		 * reclaim_congested. kswapd will stall and start writing
1767  		 * folios if the tail of the LRU is all dirty unqueued folios.
1768  		 */
1769  		folio_check_dirty_writeback(folio, &dirty, &writeback);
1770  		if (dirty || writeback)
1771  			stat->nr_dirty += nr_pages;
1772  
1773  		if (dirty && !writeback)
1774  			stat->nr_unqueued_dirty += nr_pages;
1775  
1776  		/*
1777  		 * Treat this folio as congested if folios are cycling
1778  		 * through the LRU so quickly that the folios marked
1779  		 * for immediate reclaim are making it to the end of
1780  		 * the LRU a second time.
1781  		 */
1782  		if (writeback && folio_test_reclaim(folio))
1783  			stat->nr_congested += nr_pages;
1784  
1785  		/*
1786  		 * If a folio at the tail of the LRU is under writeback, there
1787  		 * are three cases to consider.
1788  		 *
1789  		 * 1) If reclaim is encountering an excessive number
1790  		 *    of folios under writeback and this folio has both
1791  		 *    the writeback and reclaim flags set, then it
1792  		 *    indicates that folios are being queued for I/O but
1793  		 *    are being recycled through the LRU before the I/O
1794  		 *    can complete. Waiting on the folio itself risks an
1795  		 *    indefinite stall if it is impossible to writeback
1796  		 *    the folio due to I/O error or disconnected storage
1797  		 *    so instead note that the LRU is being scanned too
1798  		 *    quickly and the caller can stall after the folio
1799  		 *    list has been processed.
1800  		 *
1801  		 * 2) Global or new memcg reclaim encounters a folio that is
1802  		 *    not marked for immediate reclaim, or the caller does not
1803  		 *    have __GFP_FS (or __GFP_IO if it's simply going to swap,
1804  		 *    not to fs). In this case mark the folio for immediate
1805  		 *    reclaim and continue scanning.
1806  		 *
1807  		 *    Require may_enter_fs() because we would wait on fs, which
1808  		 *    may not have submitted I/O yet. And the loop driver might
1809  		 *    enter reclaim, and deadlock if it waits on a folio for
1810  		 *    which it is needed to do the write (loop masks off
1811  		 *    __GFP_IO|__GFP_FS for this reason); but more thought
1812  		 *    would probably show more reasons.
1813  		 *
1814  		 * 3) Legacy memcg encounters a folio that already has the
1815  		 *    reclaim flag set. memcg does not have any dirty folio
1816  		 *    throttling so we could easily OOM just because too many
1817  		 *    folios are in writeback and there is nothing else to
1818  		 *    reclaim. Wait for the writeback to complete.
1819  		 *
1820  		 * In cases 1) and 2) we activate the folios to get them out of
1821  		 * the way while we continue scanning for clean folios on the
1822  		 * inactive list and refilling from the active list. The
1823  		 * observation here is that waiting for disk writes is more
1824  		 * expensive than potentially causing reloads down the line.
1825  		 * Since they're marked for immediate reclaim, they won't put
1826  		 * memory pressure on the cache working set any longer than it
1827  		 * takes to write them to disk.
1828  		 */
1829  		if (folio_test_writeback(folio)) {
1830  			/* Case 1 above */
1831  			if (current_is_kswapd() &&
1832  			    folio_test_reclaim(folio) &&
1833  			    test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1834  				stat->nr_immediate += nr_pages;
1835  				goto activate_locked;
1836  
1837  			/* Case 2 above */
1838  			} else if (writeback_throttling_sane(sc) ||
1839  			    !folio_test_reclaim(folio) ||
1840  			    !may_enter_fs(folio, sc->gfp_mask)) {
1841  				/*
1842  				 * This is slightly racy -
1843  				 * folio_end_writeback() might have
1844  				 * just cleared the reclaim flag, then
1845  				 * setting the reclaim flag here ends up
1846  				 * interpreted as the readahead flag - but
1847  				 * that does not matter enough to care.
1848  				 * What we do want is for this folio to
1849  				 * have the reclaim flag set next time
1850  				 * memcg reclaim reaches the tests above,
1851  				 * so it will then wait for writeback to
1852  				 * avoid OOM; and it's also appropriate
1853  				 * in global reclaim.
1854  				 */
1855  				folio_set_reclaim(folio);
1856  				stat->nr_writeback += nr_pages;
1857  				goto activate_locked;
1858  
1859  			/* Case 3 above */
1860  			} else {
1861  				folio_unlock(folio);
1862  				folio_wait_writeback(folio);
1863  				/* then go back and try same folio again */
1864  				list_add_tail(&folio->lru, folio_list);
1865  				continue;
1866  			}
1867  		}
1868  
1869  		if (!ignore_references)
1870  			references = folio_check_references(folio, sc);
1871  
1872  		switch (references) {
1873  		case FOLIOREF_ACTIVATE:
1874  			goto activate_locked;
1875  		case FOLIOREF_KEEP:
1876  			stat->nr_ref_keep += nr_pages;
1877  			goto keep_locked;
1878  		case FOLIOREF_RECLAIM:
1879  		case FOLIOREF_RECLAIM_CLEAN:
1880  			; /* try to reclaim the folio below */
1881  		}
1882  
1883  		/*
1884  		 * Before reclaiming the folio, try to relocate
1885  		 * its contents to another node.
1886  		 */
1887  		if (do_demote_pass &&
1888  		    (thp_migration_supported() || !folio_test_large(folio))) {
1889  			list_add(&folio->lru, &demote_folios);
1890  			folio_unlock(folio);
1891  			continue;
1892  		}
1893  
1894  		/*
1895  		 * Anonymous process memory has backing store?
1896  		 * Try to allocate it some swap space here.
1897  		 * Lazyfree folio could be freed directly
1898  		 */
1899  		if (folio_test_anon(folio) && folio_test_swapbacked(folio)) {
1900  			if (!folio_test_swapcache(folio)) {
1901  				if (!(sc->gfp_mask & __GFP_IO))
1902  					goto keep_locked;
1903  				if (folio_maybe_dma_pinned(folio))
1904  					goto keep_locked;
1905  				if (folio_test_large(folio)) {
1906  					/* cannot split folio, skip it */
1907  					if (!can_split_folio(folio, NULL))
1908  						goto activate_locked;
1909  					/*
1910  					 * Split folios without a PMD map right
1911  					 * away. Chances are some or all of the
1912  					 * tail pages can be freed without IO.
1913  					 */
1914  					if (!folio_entire_mapcount(folio) &&
1915  					    split_folio_to_list(folio,
1916  								folio_list))
1917  						goto activate_locked;
1918  				}
1919  				if (!add_to_swap(folio)) {
1920  					if (!folio_test_large(folio))
1921  						goto activate_locked_split;
1922  					/* Fallback to swap normal pages */
1923  					if (split_folio_to_list(folio,
1924  								folio_list))
1925  						goto activate_locked;
1926  #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1927  					count_vm_event(THP_SWPOUT_FALLBACK);
1928  #endif
1929  					if (!add_to_swap(folio))
1930  						goto activate_locked_split;
1931  				}
1932  			}
1933  		} else if (folio_test_swapbacked(folio) &&
1934  			   folio_test_large(folio)) {
1935  			/* Split shmem folio */
1936  			if (split_folio_to_list(folio, folio_list))
1937  				goto keep_locked;
1938  		}
1939  
1940  		/*
1941  		 * If the folio was split above, the tail pages will make
1942  		 * their own pass through this function and be accounted
1943  		 * then.
1944  		 */
1945  		if ((nr_pages > 1) && !folio_test_large(folio)) {
1946  			sc->nr_scanned -= (nr_pages - 1);
1947  			nr_pages = 1;
1948  		}
1949  
1950  		/*
1951  		 * The folio is mapped into the page tables of one or more
1952  		 * processes. Try to unmap it here.
1953  		 */
1954  		if (folio_mapped(folio)) {
1955  			enum ttu_flags flags = TTU_BATCH_FLUSH;
1956  			bool was_swapbacked = folio_test_swapbacked(folio);
1957  
1958  			if (folio_test_pmd_mappable(folio))
1959  				flags |= TTU_SPLIT_HUGE_PMD;
1960  
1961  			try_to_unmap(folio, flags);
1962  			if (folio_mapped(folio)) {
1963  				stat->nr_unmap_fail += nr_pages;
1964  				if (!was_swapbacked &&
1965  				    folio_test_swapbacked(folio))
1966  					stat->nr_lazyfree_fail += nr_pages;
1967  				goto activate_locked;
1968  			}
1969  		}
1970  
1971  		/*
1972  		 * Folio is unmapped now so it cannot be newly pinned anymore.
1973  		 * No point in trying to reclaim folio if it is pinned.
1974  		 * Furthermore we don't want to reclaim underlying fs metadata
1975  		 * if the folio is pinned and thus potentially modified by the
1976  		 * pinning process as that may upset the filesystem.
1977  		 */
1978  		if (folio_maybe_dma_pinned(folio))
1979  			goto activate_locked;
1980  
1981  		mapping = folio_mapping(folio);
1982  		if (folio_test_dirty(folio)) {
1983  			/*
1984  			 * Only kswapd can writeback filesystem folios
1985  			 * to avoid risk of stack overflow. But avoid
1986  			 * injecting inefficient single-folio I/O into
1987  			 * flusher writeback as much as possible: only
1988  			 * write folios when we've encountered many
1989  			 * dirty folios, and when we've already scanned
1990  			 * the rest of the LRU for clean folios and see
1991  			 * the same dirty folios again (with the reclaim
1992  			 * flag set).
1993  			 */
1994  			if (folio_is_file_lru(folio) &&
1995  			    (!current_is_kswapd() ||
1996  			     !folio_test_reclaim(folio) ||
1997  			     !test_bit(PGDAT_DIRTY, &pgdat->flags))) {
1998  				/*
1999  				 * Immediately reclaim when written back.
2000  				 * Similar in principle to folio_deactivate()
2001  				 * except we already have the folio isolated
2002  				 * and know it's dirty
2003  				 */
2004  				node_stat_mod_folio(folio, NR_VMSCAN_IMMEDIATE,
2005  						nr_pages);
2006  				folio_set_reclaim(folio);
2007  
2008  				goto activate_locked;
2009  			}
2010  
2011  			if (references == FOLIOREF_RECLAIM_CLEAN)
2012  				goto keep_locked;
2013  			if (!may_enter_fs(folio, sc->gfp_mask))
2014  				goto keep_locked;
2015  			if (!sc->may_writepage)
2016  				goto keep_locked;
2017  
2018  			/*
2019  			 * Folio is dirty. Flush the TLB if a writable entry
2020  			 * potentially exists to avoid CPU writes after I/O
2021  			 * starts and then write it out here.
2022  			 */
2023  			try_to_unmap_flush_dirty();
2024  			switch (pageout(folio, mapping, &plug)) {
2025  			case PAGE_KEEP:
2026  				goto keep_locked;
2027  			case PAGE_ACTIVATE:
2028  				goto activate_locked;
2029  			case PAGE_SUCCESS:
2030  				stat->nr_pageout += nr_pages;
2031  
2032  				if (folio_test_writeback(folio))
2033  					goto keep;
2034  				if (folio_test_dirty(folio))
2035  					goto keep;
2036  
2037  				/*
2038  				 * A synchronous write - probably a ramdisk.  Go
2039  				 * ahead and try to reclaim the folio.
2040  				 */
2041  				if (!folio_trylock(folio))
2042  					goto keep;
2043  				if (folio_test_dirty(folio) ||
2044  				    folio_test_writeback(folio))
2045  					goto keep_locked;
2046  				mapping = folio_mapping(folio);
2047  				fallthrough;
2048  			case PAGE_CLEAN:
2049  				; /* try to free the folio below */
2050  			}
2051  		}
2052  
2053  		/*
2054  		 * If the folio has buffers, try to free the buffer
2055  		 * mappings associated with this folio. If we succeed
2056  		 * we try to free the folio as well.
2057  		 *
2058  		 * We do this even if the folio is dirty.
2059  		 * filemap_release_folio() does not perform I/O, but it
2060  		 * is possible for a folio to have the dirty flag set,
2061  		 * but it is actually clean (all its buffers are clean).
2062  		 * This happens if the buffers were written out directly,
2063  		 * with submit_bh(). ext3 will do this, as well as
2064  		 * the blockdev mapping.  filemap_release_folio() will
2065  		 * discover that cleanness and will drop the buffers
2066  		 * and mark the folio clean - it can be freed.
2067  		 *
2068  		 * Rarely, folios can have buffers and no ->mapping.
2069  		 * These are the folios which were not successfully
2070  		 * invalidated in truncate_cleanup_folio().  We try to
2071  		 * drop those buffers here and if that worked, and the
2072  		 * folio is no longer mapped into process address space
2073  		 * (refcount == 1) it can be freed.  Otherwise, leave
2074  		 * the folio on the LRU so it is swappable.
2075  		 */
2076  		if (folio_has_private(folio)) {
2077  			if (!filemap_release_folio(folio, sc->gfp_mask))
2078  				goto activate_locked;
2079  			if (!mapping && folio_ref_count(folio) == 1) {
2080  				folio_unlock(folio);
2081  				if (folio_put_testzero(folio))
2082  					goto free_it;
2083  				else {
2084  					/*
2085  					 * rare race with speculative reference.
2086  					 * the speculative reference will free
2087  					 * this folio shortly, so we may
2088  					 * increment nr_reclaimed here (and
2089  					 * leave it off the LRU).
2090  					 */
2091  					nr_reclaimed += nr_pages;
2092  					continue;
2093  				}
2094  			}
2095  		}
2096  
2097  		if (folio_test_anon(folio) && !folio_test_swapbacked(folio)) {
2098  			/* follow __remove_mapping for reference */
2099  			if (!folio_ref_freeze(folio, 1))
2100  				goto keep_locked;
2101  			/*
2102  			 * The folio has only one reference left, which is
2103  			 * from the isolation. After the caller puts the
2104  			 * folio back on the lru and drops the reference, the
2105  			 * folio will be freed anyway. It doesn't matter
2106  			 * which lru it goes on. So we don't bother checking
2107  			 * the dirty flag here.
2108  			 */
2109  			count_vm_events(PGLAZYFREED, nr_pages);
2110  			count_memcg_folio_events(folio, PGLAZYFREED, nr_pages);
2111  		} else if (!mapping || !__remove_mapping(mapping, folio, true,
2112  							 sc->target_mem_cgroup))
2113  			goto keep_locked;
2114  
2115  		folio_unlock(folio);
2116  free_it:
2117  		/*
2118  		 * Folio may get swapped out as a whole, need to account
2119  		 * all pages in it.
2120  		 */
2121  		nr_reclaimed += nr_pages;
2122  
2123  		/*
2124  		 * Is there need to periodically free_folio_list? It would
2125  		 * appear not as the counts should be low
2126  		 */
2127  		if (unlikely(folio_test_large(folio)))
2128  			destroy_large_folio(folio);
2129  		else
2130  			list_add(&folio->lru, &free_folios);
2131  		continue;
2132  
2133  activate_locked_split:
2134  		/*
2135  		 * The tail pages that are failed to add into swap cache
2136  		 * reach here.  Fixup nr_scanned and nr_pages.
2137  		 */
2138  		if (nr_pages > 1) {
2139  			sc->nr_scanned -= (nr_pages - 1);
2140  			nr_pages = 1;
2141  		}
2142  activate_locked:
2143  		/* Not a candidate for swapping, so reclaim swap space. */
2144  		if (folio_test_swapcache(folio) &&
2145  		    (mem_cgroup_swap_full(folio) || folio_test_mlocked(folio)))
2146  			folio_free_swap(folio);
2147  		VM_BUG_ON_FOLIO(folio_test_active(folio), folio);
2148  		if (!folio_test_mlocked(folio)) {
2149  			int type = folio_is_file_lru(folio);
2150  			folio_set_active(folio);
2151  			stat->nr_activate[type] += nr_pages;
2152  			count_memcg_folio_events(folio, PGACTIVATE, nr_pages);
2153  		}
2154  keep_locked:
2155  		folio_unlock(folio);
2156  keep:
2157  		list_add(&folio->lru, &ret_folios);
2158  		VM_BUG_ON_FOLIO(folio_test_lru(folio) ||
2159  				folio_test_unevictable(folio), folio);
2160  	}
2161  	/* 'folio_list' is always empty here */
2162  
2163  	/* Migrate folios selected for demotion */
2164  	nr_reclaimed += demote_folio_list(&demote_folios, pgdat);
2165  	/* Folios that could not be demoted are still in @demote_folios */
2166  	if (!list_empty(&demote_folios)) {
2167  		/* Folios which weren't demoted go back on @folio_list */
2168  		list_splice_init(&demote_folios, folio_list);
2169  
2170  		/*
2171  		 * goto retry to reclaim the undemoted folios in folio_list if
2172  		 * desired.
2173  		 *
2174  		 * Reclaiming directly from top tier nodes is not often desired
2175  		 * due to it breaking the LRU ordering: in general memory
2176  		 * should be reclaimed from lower tier nodes and demoted from
2177  		 * top tier nodes.
2178  		 *
2179  		 * However, disabling reclaim from top tier nodes entirely
2180  		 * would cause ooms in edge scenarios where lower tier memory
2181  		 * is unreclaimable for whatever reason, eg memory being
2182  		 * mlocked or too hot to reclaim. We can disable reclaim
2183  		 * from top tier nodes in proactive reclaim though as that is
2184  		 * not real memory pressure.
2185  		 */
2186  		if (!sc->proactive) {
2187  			do_demote_pass = false;
2188  			goto retry;
2189  		}
2190  	}
2191  
2192  	pgactivate = stat->nr_activate[0] + stat->nr_activate[1];
2193  
2194  	mem_cgroup_uncharge_list(&free_folios);
2195  	try_to_unmap_flush();
2196  	free_unref_page_list(&free_folios);
2197  
2198  	list_splice(&ret_folios, folio_list);
2199  	count_vm_events(PGACTIVATE, pgactivate);
2200  
2201  	if (plug)
2202  		swap_write_unplug(plug);
2203  	return nr_reclaimed;
2204  }
2205  
2206  unsigned int reclaim_clean_pages_from_list(struct zone *zone,
2207  					   struct list_head *folio_list)
2208  {
2209  	struct scan_control sc = {
2210  		.gfp_mask = GFP_KERNEL,
2211  		.may_unmap = 1,
2212  	};
2213  	struct reclaim_stat stat;
2214  	unsigned int nr_reclaimed;
2215  	struct folio *folio, *next;
2216  	LIST_HEAD(clean_folios);
2217  	unsigned int noreclaim_flag;
2218  
2219  	list_for_each_entry_safe(folio, next, folio_list, lru) {
2220  		if (!folio_test_hugetlb(folio) && folio_is_file_lru(folio) &&
2221  		    !folio_test_dirty(folio) && !__folio_test_movable(folio) &&
2222  		    !folio_test_unevictable(folio)) {
2223  			folio_clear_active(folio);
2224  			list_move(&folio->lru, &clean_folios);
2225  		}
2226  	}
2227  
2228  	/*
2229  	 * We should be safe here since we are only dealing with file pages and
2230  	 * we are not kswapd and therefore cannot write dirty file pages. But
2231  	 * call memalloc_noreclaim_save() anyway, just in case these conditions
2232  	 * change in the future.
2233  	 */
2234  	noreclaim_flag = memalloc_noreclaim_save();
2235  	nr_reclaimed = shrink_folio_list(&clean_folios, zone->zone_pgdat, &sc,
2236  					&stat, true);
2237  	memalloc_noreclaim_restore(noreclaim_flag);
2238  
2239  	list_splice(&clean_folios, folio_list);
2240  	mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
2241  			    -(long)nr_reclaimed);
2242  	/*
2243  	 * Since lazyfree pages are isolated from file LRU from the beginning,
2244  	 * they will rotate back to anonymous LRU in the end if it failed to
2245  	 * discard so isolated count will be mismatched.
2246  	 * Compensate the isolated count for both LRU lists.
2247  	 */
2248  	mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON,
2249  			    stat.nr_lazyfree_fail);
2250  	mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
2251  			    -(long)stat.nr_lazyfree_fail);
2252  	return nr_reclaimed;
2253  }
2254  
2255  /*
2256   * Update LRU sizes after isolating pages. The LRU size updates must
2257   * be complete before mem_cgroup_update_lru_size due to a sanity check.
2258   */
2259  static __always_inline void update_lru_sizes(struct lruvec *lruvec,
2260  			enum lru_list lru, unsigned long *nr_zone_taken)
2261  {
2262  	int zid;
2263  
2264  	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2265  		if (!nr_zone_taken[zid])
2266  			continue;
2267  
2268  		update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
2269  	}
2270  
2271  }
2272  
2273  /*
2274   * Isolating page from the lruvec to fill in @dst list by nr_to_scan times.
2275   *
2276   * lruvec->lru_lock is heavily contended.  Some of the functions that
2277   * shrink the lists perform better by taking out a batch of pages
2278   * and working on them outside the LRU lock.
2279   *
2280   * For pagecache intensive workloads, this function is the hottest
2281   * spot in the kernel (apart from copy_*_user functions).
2282   *
2283   * Lru_lock must be held before calling this function.
2284   *
2285   * @nr_to_scan:	The number of eligible pages to look through on the list.
2286   * @lruvec:	The LRU vector to pull pages from.
2287   * @dst:	The temp list to put pages on to.
2288   * @nr_scanned:	The number of pages that were scanned.
2289   * @sc:		The scan_control struct for this reclaim session
2290   * @lru:	LRU list id for isolating
2291   *
2292   * returns how many pages were moved onto *@dst.
2293   */
2294  static unsigned long isolate_lru_folios(unsigned long nr_to_scan,
2295  		struct lruvec *lruvec, struct list_head *dst,
2296  		unsigned long *nr_scanned, struct scan_control *sc,
2297  		enum lru_list lru)
2298  {
2299  	struct list_head *src = &lruvec->lists[lru];
2300  	unsigned long nr_taken = 0;
2301  	unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 };
2302  	unsigned long nr_skipped[MAX_NR_ZONES] = { 0, };
2303  	unsigned long skipped = 0;
2304  	unsigned long scan, total_scan, nr_pages;
2305  	LIST_HEAD(folios_skipped);
2306  
2307  	total_scan = 0;
2308  	scan = 0;
2309  	while (scan < nr_to_scan && !list_empty(src)) {
2310  		struct list_head *move_to = src;
2311  		struct folio *folio;
2312  
2313  		folio = lru_to_folio(src);
2314  		prefetchw_prev_lru_folio(folio, src, flags);
2315  
2316  		nr_pages = folio_nr_pages(folio);
2317  		total_scan += nr_pages;
2318  
2319  		if (folio_zonenum(folio) > sc->reclaim_idx) {
2320  			nr_skipped[folio_zonenum(folio)] += nr_pages;
2321  			move_to = &folios_skipped;
2322  			goto move;
2323  		}
2324  
2325  		/*
2326  		 * Do not count skipped folios because that makes the function
2327  		 * return with no isolated folios if the LRU mostly contains
2328  		 * ineligible folios.  This causes the VM to not reclaim any
2329  		 * folios, triggering a premature OOM.
2330  		 * Account all pages in a folio.
2331  		 */
2332  		scan += nr_pages;
2333  
2334  		if (!folio_test_lru(folio))
2335  			goto move;
2336  		if (!sc->may_unmap && folio_mapped(folio))
2337  			goto move;
2338  
2339  		/*
2340  		 * Be careful not to clear the lru flag until after we're
2341  		 * sure the folio is not being freed elsewhere -- the
2342  		 * folio release code relies on it.
2343  		 */
2344  		if (unlikely(!folio_try_get(folio)))
2345  			goto move;
2346  
2347  		if (!folio_test_clear_lru(folio)) {
2348  			/* Another thread is already isolating this folio */
2349  			folio_put(folio);
2350  			goto move;
2351  		}
2352  
2353  		nr_taken += nr_pages;
2354  		nr_zone_taken[folio_zonenum(folio)] += nr_pages;
2355  		move_to = dst;
2356  move:
2357  		list_move(&folio->lru, move_to);
2358  	}
2359  
2360  	/*
2361  	 * Splice any skipped folios to the start of the LRU list. Note that
2362  	 * this disrupts the LRU order when reclaiming for lower zones but
2363  	 * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX
2364  	 * scanning would soon rescan the same folios to skip and waste lots
2365  	 * of cpu cycles.
2366  	 */
2367  	if (!list_empty(&folios_skipped)) {
2368  		int zid;
2369  
2370  		list_splice(&folios_skipped, src);
2371  		for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2372  			if (!nr_skipped[zid])
2373  				continue;
2374  
2375  			__count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]);
2376  			skipped += nr_skipped[zid];
2377  		}
2378  	}
2379  	*nr_scanned = total_scan;
2380  	trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan,
2381  				    total_scan, skipped, nr_taken,
2382  				    sc->may_unmap ? 0 : ISOLATE_UNMAPPED, lru);
2383  	update_lru_sizes(lruvec, lru, nr_zone_taken);
2384  	return nr_taken;
2385  }
2386  
2387  /**
2388   * folio_isolate_lru() - Try to isolate a folio from its LRU list.
2389   * @folio: Folio to isolate from its LRU list.
2390   *
2391   * Isolate a @folio from an LRU list and adjust the vmstat statistic
2392   * corresponding to whatever LRU list the folio was on.
2393   *
2394   * The folio will have its LRU flag cleared.  If it was found on the
2395   * active list, it will have the Active flag set.  If it was found on the
2396   * unevictable list, it will have the Unevictable flag set.  These flags
2397   * may need to be cleared by the caller before letting the page go.
2398   *
2399   * Context:
2400   *
2401   * (1) Must be called with an elevated refcount on the folio. This is a
2402   *     fundamental difference from isolate_lru_folios() (which is called
2403   *     without a stable reference).
2404   * (2) The lru_lock must not be held.
2405   * (3) Interrupts must be enabled.
2406   *
2407   * Return: true if the folio was removed from an LRU list.
2408   * false if the folio was not on an LRU list.
2409   */
2410  bool folio_isolate_lru(struct folio *folio)
2411  {
2412  	bool ret = false;
2413  
2414  	VM_BUG_ON_FOLIO(!folio_ref_count(folio), folio);
2415  
2416  	if (folio_test_clear_lru(folio)) {
2417  		struct lruvec *lruvec;
2418  
2419  		folio_get(folio);
2420  		lruvec = folio_lruvec_lock_irq(folio);
2421  		lruvec_del_folio(lruvec, folio);
2422  		unlock_page_lruvec_irq(lruvec);
2423  		ret = true;
2424  	}
2425  
2426  	return ret;
2427  }
2428  
2429  /*
2430   * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
2431   * then get rescheduled. When there are massive number of tasks doing page
2432   * allocation, such sleeping direct reclaimers may keep piling up on each CPU,
2433   * the LRU list will go small and be scanned faster than necessary, leading to
2434   * unnecessary swapping, thrashing and OOM.
2435   */
2436  static int too_many_isolated(struct pglist_data *pgdat, int file,
2437  		struct scan_control *sc)
2438  {
2439  	unsigned long inactive, isolated;
2440  	bool too_many;
2441  
2442  	if (current_is_kswapd())
2443  		return 0;
2444  
2445  	if (!writeback_throttling_sane(sc))
2446  		return 0;
2447  
2448  	if (file) {
2449  		inactive = node_page_state(pgdat, NR_INACTIVE_FILE);
2450  		isolated = node_page_state(pgdat, NR_ISOLATED_FILE);
2451  	} else {
2452  		inactive = node_page_state(pgdat, NR_INACTIVE_ANON);
2453  		isolated = node_page_state(pgdat, NR_ISOLATED_ANON);
2454  	}
2455  
2456  	/*
2457  	 * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they
2458  	 * won't get blocked by normal direct-reclaimers, forming a circular
2459  	 * deadlock.
2460  	 */
2461  	if ((sc->gfp_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
2462  		inactive >>= 3;
2463  
2464  	too_many = isolated > inactive;
2465  
2466  	/* Wake up tasks throttled due to too_many_isolated. */
2467  	if (!too_many)
2468  		wake_throttle_isolated(pgdat);
2469  
2470  	return too_many;
2471  }
2472  
2473  /*
2474   * move_folios_to_lru() moves folios from private @list to appropriate LRU list.
2475   * On return, @list is reused as a list of folios to be freed by the caller.
2476   *
2477   * Returns the number of pages moved to the given lruvec.
2478   */
2479  static unsigned int move_folios_to_lru(struct lruvec *lruvec,
2480  		struct list_head *list)
2481  {
2482  	int nr_pages, nr_moved = 0;
2483  	LIST_HEAD(folios_to_free);
2484  
2485  	while (!list_empty(list)) {
2486  		struct folio *folio = lru_to_folio(list);
2487  
2488  		VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
2489  		list_del(&folio->lru);
2490  		if (unlikely(!folio_evictable(folio))) {
2491  			spin_unlock_irq(&lruvec->lru_lock);
2492  			folio_putback_lru(folio);
2493  			spin_lock_irq(&lruvec->lru_lock);
2494  			continue;
2495  		}
2496  
2497  		/*
2498  		 * The folio_set_lru needs to be kept here for list integrity.
2499  		 * Otherwise:
2500  		 *   #0 move_folios_to_lru             #1 release_pages
2501  		 *   if (!folio_put_testzero())
2502  		 *				      if (folio_put_testzero())
2503  		 *				        !lru //skip lru_lock
2504  		 *     folio_set_lru()
2505  		 *     list_add(&folio->lru,)
2506  		 *                                        list_add(&folio->lru,)
2507  		 */
2508  		folio_set_lru(folio);
2509  
2510  		if (unlikely(folio_put_testzero(folio))) {
2511  			__folio_clear_lru_flags(folio);
2512  
2513  			if (unlikely(folio_test_large(folio))) {
2514  				spin_unlock_irq(&lruvec->lru_lock);
2515  				destroy_large_folio(folio);
2516  				spin_lock_irq(&lruvec->lru_lock);
2517  			} else
2518  				list_add(&folio->lru, &folios_to_free);
2519  
2520  			continue;
2521  		}
2522  
2523  		/*
2524  		 * All pages were isolated from the same lruvec (and isolation
2525  		 * inhibits memcg migration).
2526  		 */
2527  		VM_BUG_ON_FOLIO(!folio_matches_lruvec(folio, lruvec), folio);
2528  		lruvec_add_folio(lruvec, folio);
2529  		nr_pages = folio_nr_pages(folio);
2530  		nr_moved += nr_pages;
2531  		if (folio_test_active(folio))
2532  			workingset_age_nonresident(lruvec, nr_pages);
2533  	}
2534  
2535  	/*
2536  	 * To save our caller's stack, now use input list for pages to free.
2537  	 */
2538  	list_splice(&folios_to_free, list);
2539  
2540  	return nr_moved;
2541  }
2542  
2543  /*
2544   * If a kernel thread (such as nfsd for loop-back mounts) services a backing
2545   * device by writing to the page cache it sets PF_LOCAL_THROTTLE. In this case
2546   * we should not throttle.  Otherwise it is safe to do so.
2547   */
2548  static int current_may_throttle(void)
2549  {
2550  	return !(current->flags & PF_LOCAL_THROTTLE);
2551  }
2552  
2553  /*
2554   * shrink_inactive_list() is a helper for shrink_node().  It returns the number
2555   * of reclaimed pages
2556   */
2557  static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
2558  		struct lruvec *lruvec, struct scan_control *sc,
2559  		enum lru_list lru)
2560  {
2561  	LIST_HEAD(folio_list);
2562  	unsigned long nr_scanned;
2563  	unsigned int nr_reclaimed = 0;
2564  	unsigned long nr_taken;
2565  	struct reclaim_stat stat;
2566  	bool file = is_file_lru(lru);
2567  	enum vm_event_item item;
2568  	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2569  	bool stalled = false;
2570  
2571  	while (unlikely(too_many_isolated(pgdat, file, sc))) {
2572  		if (stalled)
2573  			return 0;
2574  
2575  		/* wait a bit for the reclaimer. */
2576  		stalled = true;
2577  		reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED);
2578  
2579  		/* We are about to die and free our memory. Return now. */
2580  		if (fatal_signal_pending(current))
2581  			return SWAP_CLUSTER_MAX;
2582  	}
2583  
2584  	lru_add_drain();
2585  
2586  	spin_lock_irq(&lruvec->lru_lock);
2587  
2588  	nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list,
2589  				     &nr_scanned, sc, lru);
2590  
2591  	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2592  	item = PGSCAN_KSWAPD + reclaimer_offset();
2593  	if (!cgroup_reclaim(sc))
2594  		__count_vm_events(item, nr_scanned);
2595  	__count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned);
2596  	__count_vm_events(PGSCAN_ANON + file, nr_scanned);
2597  
2598  	spin_unlock_irq(&lruvec->lru_lock);
2599  
2600  	if (nr_taken == 0)
2601  		return 0;
2602  
2603  	nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false);
2604  
2605  	spin_lock_irq(&lruvec->lru_lock);
2606  	move_folios_to_lru(lruvec, &folio_list);
2607  
2608  	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2609  	item = PGSTEAL_KSWAPD + reclaimer_offset();
2610  	if (!cgroup_reclaim(sc))
2611  		__count_vm_events(item, nr_reclaimed);
2612  	__count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);
2613  	__count_vm_events(PGSTEAL_ANON + file, nr_reclaimed);
2614  	spin_unlock_irq(&lruvec->lru_lock);
2615  
2616  	lru_note_cost(lruvec, file, stat.nr_pageout, nr_scanned - nr_reclaimed);
2617  	mem_cgroup_uncharge_list(&folio_list);
2618  	free_unref_page_list(&folio_list);
2619  
2620  	/*
2621  	 * If dirty folios are scanned that are not queued for IO, it
2622  	 * implies that flushers are not doing their job. This can
2623  	 * happen when memory pressure pushes dirty folios to the end of
2624  	 * the LRU before the dirty limits are breached and the dirty
2625  	 * data has expired. It can also happen when the proportion of
2626  	 * dirty folios grows not through writes but through memory
2627  	 * pressure reclaiming all the clean cache. And in some cases,
2628  	 * the flushers simply cannot keep up with the allocation
2629  	 * rate. Nudge the flusher threads in case they are asleep.
2630  	 */
2631  	if (stat.nr_unqueued_dirty == nr_taken) {
2632  		wakeup_flusher_threads(WB_REASON_VMSCAN);
2633  		/*
2634  		 * For cgroupv1 dirty throttling is achieved by waking up
2635  		 * the kernel flusher here and later waiting on folios
2636  		 * which are in writeback to finish (see shrink_folio_list()).
2637  		 *
2638  		 * Flusher may not be able to issue writeback quickly
2639  		 * enough for cgroupv1 writeback throttling to work
2640  		 * on a large system.
2641  		 */
2642  		if (!writeback_throttling_sane(sc))
2643  			reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK);
2644  	}
2645  
2646  	sc->nr.dirty += stat.nr_dirty;
2647  	sc->nr.congested += stat.nr_congested;
2648  	sc->nr.unqueued_dirty += stat.nr_unqueued_dirty;
2649  	sc->nr.writeback += stat.nr_writeback;
2650  	sc->nr.immediate += stat.nr_immediate;
2651  	sc->nr.taken += nr_taken;
2652  	if (file)
2653  		sc->nr.file_taken += nr_taken;
2654  
2655  	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
2656  			nr_scanned, nr_reclaimed, &stat, sc->priority, file);
2657  	return nr_reclaimed;
2658  }
2659  
2660  /*
2661   * shrink_active_list() moves folios from the active LRU to the inactive LRU.
2662   *
2663   * We move them the other way if the folio is referenced by one or more
2664   * processes.
2665   *
2666   * If the folios are mostly unmapped, the processing is fast and it is
2667   * appropriate to hold lru_lock across the whole operation.  But if
2668   * the folios are mapped, the processing is slow (folio_referenced()), so
2669   * we should drop lru_lock around each folio.  It's impossible to balance
2670   * this, so instead we remove the folios from the LRU while processing them.
2671   * It is safe to rely on the active flag against the non-LRU folios in here
2672   * because nobody will play with that bit on a non-LRU folio.
2673   *
2674   * The downside is that we have to touch folio->_refcount against each folio.
2675   * But we had to alter folio->flags anyway.
2676   */
2677  static void shrink_active_list(unsigned long nr_to_scan,
2678  			       struct lruvec *lruvec,
2679  			       struct scan_control *sc,
2680  			       enum lru_list lru)
2681  {
2682  	unsigned long nr_taken;
2683  	unsigned long nr_scanned;
2684  	unsigned long vm_flags;
2685  	LIST_HEAD(l_hold);	/* The folios which were snipped off */
2686  	LIST_HEAD(l_active);
2687  	LIST_HEAD(l_inactive);
2688  	unsigned nr_deactivate, nr_activate;
2689  	unsigned nr_rotated = 0;
2690  	int file = is_file_lru(lru);
2691  	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2692  
2693  	lru_add_drain();
2694  
2695  	spin_lock_irq(&lruvec->lru_lock);
2696  
2697  	nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold,
2698  				     &nr_scanned, sc, lru);
2699  
2700  	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2701  
2702  	if (!cgroup_reclaim(sc))
2703  		__count_vm_events(PGREFILL, nr_scanned);
2704  	__count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);
2705  
2706  	spin_unlock_irq(&lruvec->lru_lock);
2707  
2708  	while (!list_empty(&l_hold)) {
2709  		struct folio *folio;
2710  
2711  		cond_resched();
2712  		folio = lru_to_folio(&l_hold);
2713  		list_del(&folio->lru);
2714  
2715  		if (unlikely(!folio_evictable(folio))) {
2716  			folio_putback_lru(folio);
2717  			continue;
2718  		}
2719  
2720  		if (unlikely(buffer_heads_over_limit)) {
2721  			if (folio_test_private(folio) && folio_trylock(folio)) {
2722  				if (folio_test_private(folio))
2723  					filemap_release_folio(folio, 0);
2724  				folio_unlock(folio);
2725  			}
2726  		}
2727  
2728  		/* Referenced or rmap lock contention: rotate */
2729  		if (folio_referenced(folio, 0, sc->target_mem_cgroup,
2730  				     &vm_flags) != 0) {
2731  			/*
2732  			 * Identify referenced, file-backed active folios and
2733  			 * give them one more trip around the active list. So
2734  			 * that executable code get better chances to stay in
2735  			 * memory under moderate memory pressure.  Anon folios
2736  			 * are not likely to be evicted by use-once streaming
2737  			 * IO, plus JVM can create lots of anon VM_EXEC folios,
2738  			 * so we ignore them here.
2739  			 */
2740  			if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) {
2741  				nr_rotated += folio_nr_pages(folio);
2742  				list_add(&folio->lru, &l_active);
2743  				continue;
2744  			}
2745  		}
2746  
2747  		folio_clear_active(folio);	/* we are de-activating */
2748  		folio_set_workingset(folio);
2749  		list_add(&folio->lru, &l_inactive);
2750  	}
2751  
2752  	/*
2753  	 * Move folios back to the lru list.
2754  	 */
2755  	spin_lock_irq(&lruvec->lru_lock);
2756  
2757  	nr_activate = move_folios_to_lru(lruvec, &l_active);
2758  	nr_deactivate = move_folios_to_lru(lruvec, &l_inactive);
2759  	/* Keep all free folios in l_active list */
2760  	list_splice(&l_inactive, &l_active);
2761  
2762  	__count_vm_events(PGDEACTIVATE, nr_deactivate);
2763  	__count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);
2764  
2765  	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2766  	spin_unlock_irq(&lruvec->lru_lock);
2767  
2768  	if (nr_rotated)
2769  		lru_note_cost(lruvec, file, 0, nr_rotated);
2770  	mem_cgroup_uncharge_list(&l_active);
2771  	free_unref_page_list(&l_active);
2772  	trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,
2773  			nr_deactivate, nr_rotated, sc->priority, file);
2774  }
2775  
2776  static unsigned int reclaim_folio_list(struct list_head *folio_list,
2777  				      struct pglist_data *pgdat)
2778  {
2779  	struct reclaim_stat dummy_stat;
2780  	unsigned int nr_reclaimed;
2781  	struct folio *folio;
2782  	struct scan_control sc = {
2783  		.gfp_mask = GFP_KERNEL,
2784  		.may_writepage = 1,
2785  		.may_unmap = 1,
2786  		.may_swap = 1,
2787  		.no_demotion = 1,
2788  	};
2789  
2790  	nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, false);
2791  	while (!list_empty(folio_list)) {
2792  		folio = lru_to_folio(folio_list);
2793  		list_del(&folio->lru);
2794  		folio_putback_lru(folio);
2795  	}
2796  
2797  	return nr_reclaimed;
2798  }
2799  
2800  unsigned long reclaim_pages(struct list_head *folio_list)
2801  {
2802  	int nid;
2803  	unsigned int nr_reclaimed = 0;
2804  	LIST_HEAD(node_folio_list);
2805  	unsigned int noreclaim_flag;
2806  
2807  	if (list_empty(folio_list))
2808  		return nr_reclaimed;
2809  
2810  	noreclaim_flag = memalloc_noreclaim_save();
2811  
2812  	nid = folio_nid(lru_to_folio(folio_list));
2813  	do {
2814  		struct folio *folio = lru_to_folio(folio_list);
2815  
2816  		if (nid == folio_nid(folio)) {
2817  			folio_clear_active(folio);
2818  			list_move(&folio->lru, &node_folio_list);
2819  			continue;
2820  		}
2821  
2822  		nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid));
2823  		nid = folio_nid(lru_to_folio(folio_list));
2824  	} while (!list_empty(folio_list));
2825  
2826  	nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid));
2827  
2828  	memalloc_noreclaim_restore(noreclaim_flag);
2829  
2830  	return nr_reclaimed;
2831  }
2832  
2833  static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2834  				 struct lruvec *lruvec, struct scan_control *sc)
2835  {
2836  	if (is_active_lru(lru)) {
2837  		if (sc->may_deactivate & (1 << is_file_lru(lru)))
2838  			shrink_active_list(nr_to_scan, lruvec, sc, lru);
2839  		else
2840  			sc->skipped_deactivate = 1;
2841  		return 0;
2842  	}
2843  
2844  	return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2845  }
2846  
2847  /*
2848   * The inactive anon list should be small enough that the VM never has
2849   * to do too much work.
2850   *
2851   * The inactive file list should be small enough to leave most memory
2852   * to the established workingset on the scan-resistant active list,
2853   * but large enough to avoid thrashing the aggregate readahead window.
2854   *
2855   * Both inactive lists should also be large enough that each inactive
2856   * folio has a chance to be referenced again before it is reclaimed.
2857   *
2858   * If that fails and refaulting is observed, the inactive list grows.
2859   *
2860   * The inactive_ratio is the target ratio of ACTIVE to INACTIVE folios
2861   * on this LRU, maintained by the pageout code. An inactive_ratio
2862   * of 3 means 3:1 or 25% of the folios are kept on the inactive list.
2863   *
2864   * total     target    max
2865   * memory    ratio     inactive
2866   * -------------------------------------
2867   *   10MB       1         5MB
2868   *  100MB       1        50MB
2869   *    1GB       3       250MB
2870   *   10GB      10       0.9GB
2871   *  100GB      31         3GB
2872   *    1TB     101        10GB
2873   *   10TB     320        32GB
2874   */
2875  static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru)
2876  {
2877  	enum lru_list active_lru = inactive_lru + LRU_ACTIVE;
2878  	unsigned long inactive, active;
2879  	unsigned long inactive_ratio;
2880  	unsigned long gb;
2881  
2882  	inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru);
2883  	active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru);
2884  
2885  	gb = (inactive + active) >> (30 - PAGE_SHIFT);
2886  	if (gb)
2887  		inactive_ratio = int_sqrt(10 * gb);
2888  	else
2889  		inactive_ratio = 1;
2890  
2891  	return inactive * inactive_ratio < active;
2892  }
2893  
2894  enum scan_balance {
2895  	SCAN_EQUAL,
2896  	SCAN_FRACT,
2897  	SCAN_ANON,
2898  	SCAN_FILE,
2899  };
2900  
2901  static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc)
2902  {
2903  	unsigned long file;
2904  	struct lruvec *target_lruvec;
2905  
2906  	if (lru_gen_enabled())
2907  		return;
2908  
2909  	target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
2910  
2911  	/*
2912  	 * Flush the memory cgroup stats, so that we read accurate per-memcg
2913  	 * lruvec stats for heuristics.
2914  	 */
2915  	mem_cgroup_flush_stats();
2916  
2917  	/*
2918  	 * Determine the scan balance between anon and file LRUs.
2919  	 */
2920  	spin_lock_irq(&target_lruvec->lru_lock);
2921  	sc->anon_cost = target_lruvec->anon_cost;
2922  	sc->file_cost = target_lruvec->file_cost;
2923  	spin_unlock_irq(&target_lruvec->lru_lock);
2924  
2925  	/*
2926  	 * Target desirable inactive:active list ratios for the anon
2927  	 * and file LRU lists.
2928  	 */
2929  	if (!sc->force_deactivate) {
2930  		unsigned long refaults;
2931  
2932  		/*
2933  		 * When refaults are being observed, it means a new
2934  		 * workingset is being established. Deactivate to get
2935  		 * rid of any stale active pages quickly.
2936  		 */
2937  		refaults = lruvec_page_state(target_lruvec,
2938  				WORKINGSET_ACTIVATE_ANON);
2939  		if (refaults != target_lruvec->refaults[WORKINGSET_ANON] ||
2940  			inactive_is_low(target_lruvec, LRU_INACTIVE_ANON))
2941  			sc->may_deactivate |= DEACTIVATE_ANON;
2942  		else
2943  			sc->may_deactivate &= ~DEACTIVATE_ANON;
2944  
2945  		refaults = lruvec_page_state(target_lruvec,
2946  				WORKINGSET_ACTIVATE_FILE);
2947  		if (refaults != target_lruvec->refaults[WORKINGSET_FILE] ||
2948  		    inactive_is_low(target_lruvec, LRU_INACTIVE_FILE))
2949  			sc->may_deactivate |= DEACTIVATE_FILE;
2950  		else
2951  			sc->may_deactivate &= ~DEACTIVATE_FILE;
2952  	} else
2953  		sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE;
2954  
2955  	/*
2956  	 * If we have plenty of inactive file pages that aren't
2957  	 * thrashing, try to reclaim those first before touching
2958  	 * anonymous pages.
2959  	 */
2960  	file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE);
2961  	if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE))
2962  		sc->cache_trim_mode = 1;
2963  	else
2964  		sc->cache_trim_mode = 0;
2965  
2966  	/*
2967  	 * Prevent the reclaimer from falling into the cache trap: as
2968  	 * cache pages start out inactive, every cache fault will tip
2969  	 * the scan balance towards the file LRU.  And as the file LRU
2970  	 * shrinks, so does the window for rotation from references.
2971  	 * This means we have a runaway feedback loop where a tiny
2972  	 * thrashing file LRU becomes infinitely more attractive than
2973  	 * anon pages.  Try to detect this based on file LRU size.
2974  	 */
2975  	if (!cgroup_reclaim(sc)) {
2976  		unsigned long total_high_wmark = 0;
2977  		unsigned long free, anon;
2978  		int z;
2979  
2980  		free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
2981  		file = node_page_state(pgdat, NR_ACTIVE_FILE) +
2982  			   node_page_state(pgdat, NR_INACTIVE_FILE);
2983  
2984  		for (z = 0; z < MAX_NR_ZONES; z++) {
2985  			struct zone *zone = &pgdat->node_zones[z];
2986  
2987  			if (!managed_zone(zone))
2988  				continue;
2989  
2990  			total_high_wmark += high_wmark_pages(zone);
2991  		}
2992  
2993  		/*
2994  		 * Consider anon: if that's low too, this isn't a
2995  		 * runaway file reclaim problem, but rather just
2996  		 * extreme pressure. Reclaim as per usual then.
2997  		 */
2998  		anon = node_page_state(pgdat, NR_INACTIVE_ANON);
2999  
3000  		sc->file_is_tiny =
3001  			file + free <= total_high_wmark &&
3002  			!(sc->may_deactivate & DEACTIVATE_ANON) &&
3003  			anon >> sc->priority;
3004  	}
3005  }
3006  
3007  /*
3008   * Determine how aggressively the anon and file LRU lists should be
3009   * scanned.
3010   *
3011   * nr[0] = anon inactive folios to scan; nr[1] = anon active folios to scan
3012   * nr[2] = file inactive folios to scan; nr[3] = file active folios to scan
3013   */
3014  static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
3015  			   unsigned long *nr)
3016  {
3017  	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
3018  	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
3019  	unsigned long anon_cost, file_cost, total_cost;
3020  	int swappiness = mem_cgroup_swappiness(memcg);
3021  	u64 fraction[ANON_AND_FILE];
3022  	u64 denominator = 0;	/* gcc */
3023  	enum scan_balance scan_balance;
3024  	unsigned long ap, fp;
3025  	enum lru_list lru;
3026  
3027  	/* If we have no swap space, do not bother scanning anon folios. */
3028  	if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) {
3029  		scan_balance = SCAN_FILE;
3030  		goto out;
3031  	}
3032  
3033  	/*
3034  	 * Global reclaim will swap to prevent OOM even with no
3035  	 * swappiness, but memcg users want to use this knob to
3036  	 * disable swapping for individual groups completely when
3037  	 * using the memory controller's swap limit feature would be
3038  	 * too expensive.
3039  	 */
3040  	if (cgroup_reclaim(sc) && !swappiness) {
3041  		scan_balance = SCAN_FILE;
3042  		goto out;
3043  	}
3044  
3045  	/*
3046  	 * Do not apply any pressure balancing cleverness when the
3047  	 * system is close to OOM, scan both anon and file equally
3048  	 * (unless the swappiness setting disagrees with swapping).
3049  	 */
3050  	if (!sc->priority && swappiness) {
3051  		scan_balance = SCAN_EQUAL;
3052  		goto out;
3053  	}
3054  
3055  	/*
3056  	 * If the system is almost out of file pages, force-scan anon.
3057  	 */
3058  	if (sc->file_is_tiny) {
3059  		scan_balance = SCAN_ANON;
3060  		goto out;
3061  	}
3062  
3063  	/*
3064  	 * If there is enough inactive page cache, we do not reclaim
3065  	 * anything from the anonymous working right now.
3066  	 */
3067  	if (sc->cache_trim_mode) {
3068  		scan_balance = SCAN_FILE;
3069  		goto out;
3070  	}
3071  
3072  	scan_balance = SCAN_FRACT;
3073  	/*
3074  	 * Calculate the pressure balance between anon and file pages.
3075  	 *
3076  	 * The amount of pressure we put on each LRU is inversely
3077  	 * proportional to the cost of reclaiming each list, as
3078  	 * determined by the share of pages that are refaulting, times
3079  	 * the relative IO cost of bringing back a swapped out
3080  	 * anonymous page vs reloading a filesystem page (swappiness).
3081  	 *
3082  	 * Although we limit that influence to ensure no list gets
3083  	 * left behind completely: at least a third of the pressure is
3084  	 * applied, before swappiness.
3085  	 *
3086  	 * With swappiness at 100, anon and file have equal IO cost.
3087  	 */
3088  	total_cost = sc->anon_cost + sc->file_cost;
3089  	anon_cost = total_cost + sc->anon_cost;
3090  	file_cost = total_cost + sc->file_cost;
3091  	total_cost = anon_cost + file_cost;
3092  
3093  	ap = swappiness * (total_cost + 1);
3094  	ap /= anon_cost + 1;
3095  
3096  	fp = (200 - swappiness) * (total_cost + 1);
3097  	fp /= file_cost + 1;
3098  
3099  	fraction[0] = ap;
3100  	fraction[1] = fp;
3101  	denominator = ap + fp;
3102  out:
3103  	for_each_evictable_lru(lru) {
3104  		int file = is_file_lru(lru);
3105  		unsigned long lruvec_size;
3106  		unsigned long low, min;
3107  		unsigned long scan;
3108  
3109  		lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
3110  		mem_cgroup_protection(sc->target_mem_cgroup, memcg,
3111  				      &min, &low);
3112  
3113  		if (min || low) {
3114  			/*
3115  			 * Scale a cgroup's reclaim pressure by proportioning
3116  			 * its current usage to its memory.low or memory.min
3117  			 * setting.
3118  			 *
3119  			 * This is important, as otherwise scanning aggression
3120  			 * becomes extremely binary -- from nothing as we
3121  			 * approach the memory protection threshold, to totally
3122  			 * nominal as we exceed it.  This results in requiring
3123  			 * setting extremely liberal protection thresholds. It
3124  			 * also means we simply get no protection at all if we
3125  			 * set it too low, which is not ideal.
3126  			 *
3127  			 * If there is any protection in place, we reduce scan
3128  			 * pressure by how much of the total memory used is
3129  			 * within protection thresholds.
3130  			 *
3131  			 * There is one special case: in the first reclaim pass,
3132  			 * we skip over all groups that are within their low
3133  			 * protection. If that fails to reclaim enough pages to
3134  			 * satisfy the reclaim goal, we come back and override
3135  			 * the best-effort low protection. However, we still
3136  			 * ideally want to honor how well-behaved groups are in
3137  			 * that case instead of simply punishing them all
3138  			 * equally. As such, we reclaim them based on how much
3139  			 * memory they are using, reducing the scan pressure
3140  			 * again by how much of the total memory used is under
3141  			 * hard protection.
3142  			 */
3143  			unsigned long cgroup_size = mem_cgroup_size(memcg);
3144  			unsigned long protection;
3145  
3146  			/* memory.low scaling, make sure we retry before OOM */
3147  			if (!sc->memcg_low_reclaim && low > min) {
3148  				protection = low;
3149  				sc->memcg_low_skipped = 1;
3150  			} else {
3151  				protection = min;
3152  			}
3153  
3154  			/* Avoid TOCTOU with earlier protection check */
3155  			cgroup_size = max(cgroup_size, protection);
3156  
3157  			scan = lruvec_size - lruvec_size * protection /
3158  				(cgroup_size + 1);
3159  
3160  			/*
3161  			 * Minimally target SWAP_CLUSTER_MAX pages to keep
3162  			 * reclaim moving forwards, avoiding decrementing
3163  			 * sc->priority further than desirable.
3164  			 */
3165  			scan = max(scan, SWAP_CLUSTER_MAX);
3166  		} else {
3167  			scan = lruvec_size;
3168  		}
3169  
3170  		scan >>= sc->priority;
3171  
3172  		/*
3173  		 * If the cgroup's already been deleted, make sure to
3174  		 * scrape out the remaining cache.
3175  		 */
3176  		if (!scan && !mem_cgroup_online(memcg))
3177  			scan = min(lruvec_size, SWAP_CLUSTER_MAX);
3178  
3179  		switch (scan_balance) {
3180  		case SCAN_EQUAL:
3181  			/* Scan lists relative to size */
3182  			break;
3183  		case SCAN_FRACT:
3184  			/*
3185  			 * Scan types proportional to swappiness and
3186  			 * their relative recent reclaim efficiency.
3187  			 * Make sure we don't miss the last page on
3188  			 * the offlined memory cgroups because of a
3189  			 * round-off error.
3190  			 */
3191  			scan = mem_cgroup_online(memcg) ?
3192  			       div64_u64(scan * fraction[file], denominator) :
3193  			       DIV64_U64_ROUND_UP(scan * fraction[file],
3194  						  denominator);
3195  			break;
3196  		case SCAN_FILE:
3197  		case SCAN_ANON:
3198  			/* Scan one type exclusively */
3199  			if ((scan_balance == SCAN_FILE) != file)
3200  				scan = 0;
3201  			break;
3202  		default:
3203  			/* Look ma, no brain */
3204  			BUG();
3205  		}
3206  
3207  		nr[lru] = scan;
3208  	}
3209  }
3210  
3211  /*
3212   * Anonymous LRU management is a waste if there is
3213   * ultimately no way to reclaim the memory.
3214   */
3215  static bool can_age_anon_pages(struct pglist_data *pgdat,
3216  			       struct scan_control *sc)
3217  {
3218  	/* Aging the anon LRU is valuable if swap is present: */
3219  	if (total_swap_pages > 0)
3220  		return true;
3221  
3222  	/* Also valuable if anon pages can be demoted: */
3223  	return can_demote(pgdat->node_id, sc);
3224  }
3225  
3226  #ifdef CONFIG_LRU_GEN
3227  
3228  #ifdef CONFIG_LRU_GEN_ENABLED
3229  DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS);
3230  #define get_cap(cap)	static_branch_likely(&lru_gen_caps[cap])
3231  #else
3232  DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS);
3233  #define get_cap(cap)	static_branch_unlikely(&lru_gen_caps[cap])
3234  #endif
3235  
3236  /******************************************************************************
3237   *                          shorthand helpers
3238   ******************************************************************************/
3239  
3240  #define LRU_REFS_FLAGS	(BIT(PG_referenced) | BIT(PG_workingset))
3241  
3242  #define DEFINE_MAX_SEQ(lruvec)						\
3243  	unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq)
3244  
3245  #define DEFINE_MIN_SEQ(lruvec)						\
3246  	unsigned long min_seq[ANON_AND_FILE] = {			\
3247  		READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]),	\
3248  		READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]),	\
3249  	}
3250  
3251  #define for_each_gen_type_zone(gen, type, zone)				\
3252  	for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++)			\
3253  		for ((type) = 0; (type) < ANON_AND_FILE; (type)++)	\
3254  			for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++)
3255  
3256  #define get_memcg_gen(seq)	((seq) % MEMCG_NR_GENS)
3257  #define get_memcg_bin(bin)	((bin) % MEMCG_NR_BINS)
3258  
3259  static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)
3260  {
3261  	struct pglist_data *pgdat = NODE_DATA(nid);
3262  
3263  #ifdef CONFIG_MEMCG
3264  	if (memcg) {
3265  		struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec;
3266  
3267  		/* see the comment in mem_cgroup_lruvec() */
3268  		if (!lruvec->pgdat)
3269  			lruvec->pgdat = pgdat;
3270  
3271  		return lruvec;
3272  	}
3273  #endif
3274  	VM_WARN_ON_ONCE(!mem_cgroup_disabled());
3275  
3276  	return &pgdat->__lruvec;
3277  }
3278  
3279  static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc)
3280  {
3281  	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
3282  	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
3283  
3284  	if (!sc->may_swap)
3285  		return 0;
3286  
3287  	if (!can_demote(pgdat->node_id, sc) &&
3288  	    mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH)
3289  		return 0;
3290  
3291  	return mem_cgroup_swappiness(memcg);
3292  }
3293  
3294  static int get_nr_gens(struct lruvec *lruvec, int type)
3295  {
3296  	return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1;
3297  }
3298  
3299  static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
3300  {
3301  	/* see the comment on lru_gen_folio */
3302  	return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS &&
3303  	       get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) &&
3304  	       get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS;
3305  }
3306  
3307  /******************************************************************************
3308   *                          Bloom filters
3309   ******************************************************************************/
3310  
3311  /*
3312   * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when
3313   * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of
3314   * bits in a bitmap, k is the number of hash functions and n is the number of
3315   * inserted items.
3316   *
3317   * Page table walkers use one of the two filters to reduce their search space.
3318   * To get rid of non-leaf entries that no longer have enough leaf entries, the
3319   * aging uses the double-buffering technique to flip to the other filter each
3320   * time it produces a new generation. For non-leaf entries that have enough
3321   * leaf entries, the aging carries them over to the next generation in
3322   * walk_pmd_range(); the eviction also report them when walking the rmap
3323   * in lru_gen_look_around().
3324   *
3325   * For future optimizations:
3326   * 1. It's not necessary to keep both filters all the time. The spare one can be
3327   *    freed after the RCU grace period and reallocated if needed again.
3328   * 2. And when reallocating, it's worth scaling its size according to the number
3329   *    of inserted entries in the other filter, to reduce the memory overhead on
3330   *    small systems and false positives on large systems.
3331   * 3. Jenkins' hash function is an alternative to Knuth's.
3332   */
3333  #define BLOOM_FILTER_SHIFT	15
3334  
3335  static inline int filter_gen_from_seq(unsigned long seq)
3336  {
3337  	return seq % NR_BLOOM_FILTERS;
3338  }
3339  
3340  static void get_item_key(void *item, int *key)
3341  {
3342  	u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2);
3343  
3344  	BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32));
3345  
3346  	key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1);
3347  	key[1] = hash >> BLOOM_FILTER_SHIFT;
3348  }
3349  
3350  static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item)
3351  {
3352  	int key[2];
3353  	unsigned long *filter;
3354  	int gen = filter_gen_from_seq(seq);
3355  
3356  	filter = READ_ONCE(lruvec->mm_state.filters[gen]);
3357  	if (!filter)
3358  		return true;
3359  
3360  	get_item_key(item, key);
3361  
3362  	return test_bit(key[0], filter) && test_bit(key[1], filter);
3363  }
3364  
3365  static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item)
3366  {
3367  	int key[2];
3368  	unsigned long *filter;
3369  	int gen = filter_gen_from_seq(seq);
3370  
3371  	filter = READ_ONCE(lruvec->mm_state.filters[gen]);
3372  	if (!filter)
3373  		return;
3374  
3375  	get_item_key(item, key);
3376  
3377  	if (!test_bit(key[0], filter))
3378  		set_bit(key[0], filter);
3379  	if (!test_bit(key[1], filter))
3380  		set_bit(key[1], filter);
3381  }
3382  
3383  static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq)
3384  {
3385  	unsigned long *filter;
3386  	int gen = filter_gen_from_seq(seq);
3387  
3388  	filter = lruvec->mm_state.filters[gen];
3389  	if (filter) {
3390  		bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT));
3391  		return;
3392  	}
3393  
3394  	filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT),
3395  			       __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
3396  	WRITE_ONCE(lruvec->mm_state.filters[gen], filter);
3397  }
3398  
3399  /******************************************************************************
3400   *                          mm_struct list
3401   ******************************************************************************/
3402  
3403  static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg)
3404  {
3405  	static struct lru_gen_mm_list mm_list = {
3406  		.fifo = LIST_HEAD_INIT(mm_list.fifo),
3407  		.lock = __SPIN_LOCK_UNLOCKED(mm_list.lock),
3408  	};
3409  
3410  #ifdef CONFIG_MEMCG
3411  	if (memcg)
3412  		return &memcg->mm_list;
3413  #endif
3414  	VM_WARN_ON_ONCE(!mem_cgroup_disabled());
3415  
3416  	return &mm_list;
3417  }
3418  
3419  void lru_gen_add_mm(struct mm_struct *mm)
3420  {
3421  	int nid;
3422  	struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm);
3423  	struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
3424  
3425  	VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list));
3426  #ifdef CONFIG_MEMCG
3427  	VM_WARN_ON_ONCE(mm->lru_gen.memcg);
3428  	mm->lru_gen.memcg = memcg;
3429  #endif
3430  	spin_lock(&mm_list->lock);
3431  
3432  	for_each_node_state(nid, N_MEMORY) {
3433  		struct lruvec *lruvec = get_lruvec(memcg, nid);
3434  
3435  		/* the first addition since the last iteration */
3436  		if (lruvec->mm_state.tail == &mm_list->fifo)
3437  			lruvec->mm_state.tail = &mm->lru_gen.list;
3438  	}
3439  
3440  	list_add_tail(&mm->lru_gen.list, &mm_list->fifo);
3441  
3442  	spin_unlock(&mm_list->lock);
3443  }
3444  
3445  void lru_gen_del_mm(struct mm_struct *mm)
3446  {
3447  	int nid;
3448  	struct lru_gen_mm_list *mm_list;
3449  	struct mem_cgroup *memcg = NULL;
3450  
3451  	if (list_empty(&mm->lru_gen.list))
3452  		return;
3453  
3454  #ifdef CONFIG_MEMCG
3455  	memcg = mm->lru_gen.memcg;
3456  #endif
3457  	mm_list = get_mm_list(memcg);
3458  
3459  	spin_lock(&mm_list->lock);
3460  
3461  	for_each_node(nid) {
3462  		struct lruvec *lruvec = get_lruvec(memcg, nid);
3463  
3464  		/* where the current iteration continues after */
3465  		if (lruvec->mm_state.head == &mm->lru_gen.list)
3466  			lruvec->mm_state.head = lruvec->mm_state.head->prev;
3467  
3468  		/* where the last iteration ended before */
3469  		if (lruvec->mm_state.tail == &mm->lru_gen.list)
3470  			lruvec->mm_state.tail = lruvec->mm_state.tail->next;
3471  	}
3472  
3473  	list_del_init(&mm->lru_gen.list);
3474  
3475  	spin_unlock(&mm_list->lock);
3476  
3477  #ifdef CONFIG_MEMCG
3478  	mem_cgroup_put(mm->lru_gen.memcg);
3479  	mm->lru_gen.memcg = NULL;
3480  #endif
3481  }
3482  
3483  #ifdef CONFIG_MEMCG
3484  void lru_gen_migrate_mm(struct mm_struct *mm)
3485  {
3486  	struct mem_cgroup *memcg;
3487  	struct task_struct *task = rcu_dereference_protected(mm->owner, true);
3488  
3489  	VM_WARN_ON_ONCE(task->mm != mm);
3490  	lockdep_assert_held(&task->alloc_lock);
3491  
3492  	/* for mm_update_next_owner() */
3493  	if (mem_cgroup_disabled())
3494  		return;
3495  
3496  	/* migration can happen before addition */
3497  	if (!mm->lru_gen.memcg)
3498  		return;
3499  
3500  	rcu_read_lock();
3501  	memcg = mem_cgroup_from_task(task);
3502  	rcu_read_unlock();
3503  	if (memcg == mm->lru_gen.memcg)
3504  		return;
3505  
3506  	VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list));
3507  
3508  	lru_gen_del_mm(mm);
3509  	lru_gen_add_mm(mm);
3510  }
3511  #endif
3512  
3513  static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last)
3514  {
3515  	int i;
3516  	int hist;
3517  
3518  	lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock);
3519  
3520  	if (walk) {
3521  		hist = lru_hist_from_seq(walk->max_seq);
3522  
3523  		for (i = 0; i < NR_MM_STATS; i++) {
3524  			WRITE_ONCE(lruvec->mm_state.stats[hist][i],
3525  				   lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]);
3526  			walk->mm_stats[i] = 0;
3527  		}
3528  	}
3529  
3530  	if (NR_HIST_GENS > 1 && last) {
3531  		hist = lru_hist_from_seq(lruvec->mm_state.seq + 1);
3532  
3533  		for (i = 0; i < NR_MM_STATS; i++)
3534  			WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0);
3535  	}
3536  }
3537  
3538  static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk)
3539  {
3540  	int type;
3541  	unsigned long size = 0;
3542  	struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
3543  	int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap);
3544  
3545  	if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap))
3546  		return true;
3547  
3548  	clear_bit(key, &mm->lru_gen.bitmap);
3549  
3550  	for (type = !walk->can_swap; type < ANON_AND_FILE; type++) {
3551  		size += type ? get_mm_counter(mm, MM_FILEPAGES) :
3552  			       get_mm_counter(mm, MM_ANONPAGES) +
3553  			       get_mm_counter(mm, MM_SHMEMPAGES);
3554  	}
3555  
3556  	if (size < MIN_LRU_BATCH)
3557  		return true;
3558  
3559  	return !mmget_not_zero(mm);
3560  }
3561  
3562  static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk,
3563  			    struct mm_struct **iter)
3564  {
3565  	bool first = false;
3566  	bool last = false;
3567  	struct mm_struct *mm = NULL;
3568  	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
3569  	struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
3570  	struct lru_gen_mm_state *mm_state = &lruvec->mm_state;
3571  
3572  	/*
3573  	 * mm_state->seq is incremented after each iteration of mm_list. There
3574  	 * are three interesting cases for this page table walker:
3575  	 * 1. It tries to start a new iteration with a stale max_seq: there is
3576  	 *    nothing left to do.
3577  	 * 2. It started the next iteration: it needs to reset the Bloom filter
3578  	 *    so that a fresh set of PTE tables can be recorded.
3579  	 * 3. It ended the current iteration: it needs to reset the mm stats
3580  	 *    counters and tell its caller to increment max_seq.
3581  	 */
3582  	spin_lock(&mm_list->lock);
3583  
3584  	VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq);
3585  
3586  	if (walk->max_seq <= mm_state->seq)
3587  		goto done;
3588  
3589  	if (!mm_state->head)
3590  		mm_state->head = &mm_list->fifo;
3591  
3592  	if (mm_state->head == &mm_list->fifo)
3593  		first = true;
3594  
3595  	do {
3596  		mm_state->head = mm_state->head->next;
3597  		if (mm_state->head == &mm_list->fifo) {
3598  			WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
3599  			last = true;
3600  			break;
3601  		}
3602  
3603  		/* force scan for those added after the last iteration */
3604  		if (!mm_state->tail || mm_state->tail == mm_state->head) {
3605  			mm_state->tail = mm_state->head->next;
3606  			walk->force_scan = true;
3607  		}
3608  
3609  		mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list);
3610  		if (should_skip_mm(mm, walk))
3611  			mm = NULL;
3612  	} while (!mm);
3613  done:
3614  	if (*iter || last)
3615  		reset_mm_stats(lruvec, walk, last);
3616  
3617  	spin_unlock(&mm_list->lock);
3618  
3619  	if (mm && first)
3620  		reset_bloom_filter(lruvec, walk->max_seq + 1);
3621  
3622  	if (*iter)
3623  		mmput_async(*iter);
3624  
3625  	*iter = mm;
3626  
3627  	return last;
3628  }
3629  
3630  static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq)
3631  {
3632  	bool success = false;
3633  	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
3634  	struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
3635  	struct lru_gen_mm_state *mm_state = &lruvec->mm_state;
3636  
3637  	spin_lock(&mm_list->lock);
3638  
3639  	VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq);
3640  
3641  	if (max_seq > mm_state->seq) {
3642  		mm_state->head = NULL;
3643  		mm_state->tail = NULL;
3644  		WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
3645  		reset_mm_stats(lruvec, NULL, true);
3646  		success = true;
3647  	}
3648  
3649  	spin_unlock(&mm_list->lock);
3650  
3651  	return success;
3652  }
3653  
3654  /******************************************************************************
3655   *                          PID controller
3656   ******************************************************************************/
3657  
3658  /*
3659   * A feedback loop based on Proportional-Integral-Derivative (PID) controller.
3660   *
3661   * The P term is refaulted/(evicted+protected) from a tier in the generation
3662   * currently being evicted; the I term is the exponential moving average of the
3663   * P term over the generations previously evicted, using the smoothing factor
3664   * 1/2; the D term isn't supported.
3665   *
3666   * The setpoint (SP) is always the first tier of one type; the process variable
3667   * (PV) is either any tier of the other type or any other tier of the same
3668   * type.
3669   *
3670   * The error is the difference between the SP and the PV; the correction is to
3671   * turn off protection when SP>PV or turn on protection when SP<PV.
3672   *
3673   * For future optimizations:
3674   * 1. The D term may discount the other two terms over time so that long-lived
3675   *    generations can resist stale information.
3676   */
3677  struct ctrl_pos {
3678  	unsigned long refaulted;
3679  	unsigned long total;
3680  	int gain;
3681  };
3682  
3683  static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain,
3684  			  struct ctrl_pos *pos)
3685  {
3686  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
3687  	int hist = lru_hist_from_seq(lrugen->min_seq[type]);
3688  
3689  	pos->refaulted = lrugen->avg_refaulted[type][tier] +
3690  			 atomic_long_read(&lrugen->refaulted[hist][type][tier]);
3691  	pos->total = lrugen->avg_total[type][tier] +
3692  		     atomic_long_read(&lrugen->evicted[hist][type][tier]);
3693  	if (tier)
3694  		pos->total += lrugen->protected[hist][type][tier - 1];
3695  	pos->gain = gain;
3696  }
3697  
3698  static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover)
3699  {
3700  	int hist, tier;
3701  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
3702  	bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1;
3703  	unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1;
3704  
3705  	lockdep_assert_held(&lruvec->lru_lock);
3706  
3707  	if (!carryover && !clear)
3708  		return;
3709  
3710  	hist = lru_hist_from_seq(seq);
3711  
3712  	for (tier = 0; tier < MAX_NR_TIERS; tier++) {
3713  		if (carryover) {
3714  			unsigned long sum;
3715  
3716  			sum = lrugen->avg_refaulted[type][tier] +
3717  			      atomic_long_read(&lrugen->refaulted[hist][type][tier]);
3718  			WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2);
3719  
3720  			sum = lrugen->avg_total[type][tier] +
3721  			      atomic_long_read(&lrugen->evicted[hist][type][tier]);
3722  			if (tier)
3723  				sum += lrugen->protected[hist][type][tier - 1];
3724  			WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2);
3725  		}
3726  
3727  		if (clear) {
3728  			atomic_long_set(&lrugen->refaulted[hist][type][tier], 0);
3729  			atomic_long_set(&lrugen->evicted[hist][type][tier], 0);
3730  			if (tier)
3731  				WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0);
3732  		}
3733  	}
3734  }
3735  
3736  static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv)
3737  {
3738  	/*
3739  	 * Return true if the PV has a limited number of refaults or a lower
3740  	 * refaulted/total than the SP.
3741  	 */
3742  	return pv->refaulted < MIN_LRU_BATCH ||
3743  	       pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <=
3744  	       (sp->refaulted + 1) * pv->total * pv->gain;
3745  }
3746  
3747  /******************************************************************************
3748   *                          the aging
3749   ******************************************************************************/
3750  
3751  /* promote pages accessed through page tables */
3752  static int folio_update_gen(struct folio *folio, int gen)
3753  {
3754  	unsigned long new_flags, old_flags = READ_ONCE(folio->flags);
3755  
3756  	VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
3757  	VM_WARN_ON_ONCE(!rcu_read_lock_held());
3758  
3759  	do {
3760  		/* lru_gen_del_folio() has isolated this page? */
3761  		if (!(old_flags & LRU_GEN_MASK)) {
3762  			/* for shrink_folio_list() */
3763  			new_flags = old_flags | BIT(PG_referenced);
3764  			continue;
3765  		}
3766  
3767  		new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS);
3768  		new_flags |= (gen + 1UL) << LRU_GEN_PGOFF;
3769  	} while (!try_cmpxchg(&folio->flags, &old_flags, new_flags));
3770  
3771  	return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
3772  }
3773  
3774  /* protect pages accessed multiple times through file descriptors */
3775  static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
3776  {
3777  	int type = folio_is_file_lru(folio);
3778  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
3779  	int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
3780  	unsigned long new_flags, old_flags = READ_ONCE(folio->flags);
3781  
3782  	VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio);
3783  
3784  	do {
3785  		new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
3786  		/* folio_update_gen() has promoted this page? */
3787  		if (new_gen >= 0 && new_gen != old_gen)
3788  			return new_gen;
3789  
3790  		new_gen = (old_gen + 1) % MAX_NR_GENS;
3791  
3792  		new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS);
3793  		new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF;
3794  		/* for folio_end_writeback() */
3795  		if (reclaiming)
3796  			new_flags |= BIT(PG_reclaim);
3797  	} while (!try_cmpxchg(&folio->flags, &old_flags, new_flags));
3798  
3799  	lru_gen_update_size(lruvec, folio, old_gen, new_gen);
3800  
3801  	return new_gen;
3802  }
3803  
3804  static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio,
3805  			      int old_gen, int new_gen)
3806  {
3807  	int type = folio_is_file_lru(folio);
3808  	int zone = folio_zonenum(folio);
3809  	int delta = folio_nr_pages(folio);
3810  
3811  	VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS);
3812  	VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS);
3813  
3814  	walk->batched++;
3815  
3816  	walk->nr_pages[old_gen][type][zone] -= delta;
3817  	walk->nr_pages[new_gen][type][zone] += delta;
3818  }
3819  
3820  static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk)
3821  {
3822  	int gen, type, zone;
3823  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
3824  
3825  	walk->batched = 0;
3826  
3827  	for_each_gen_type_zone(gen, type, zone) {
3828  		enum lru_list lru = type * LRU_INACTIVE_FILE;
3829  		int delta = walk->nr_pages[gen][type][zone];
3830  
3831  		if (!delta)
3832  			continue;
3833  
3834  		walk->nr_pages[gen][type][zone] = 0;
3835  		WRITE_ONCE(lrugen->nr_pages[gen][type][zone],
3836  			   lrugen->nr_pages[gen][type][zone] + delta);
3837  
3838  		if (lru_gen_is_active(lruvec, gen))
3839  			lru += LRU_ACTIVE;
3840  		__update_lru_size(lruvec, lru, zone, delta);
3841  	}
3842  }
3843  
3844  static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args)
3845  {
3846  	struct address_space *mapping;
3847  	struct vm_area_struct *vma = args->vma;
3848  	struct lru_gen_mm_walk *walk = args->private;
3849  
3850  	if (!vma_is_accessible(vma))
3851  		return true;
3852  
3853  	if (is_vm_hugetlb_page(vma))
3854  		return true;
3855  
3856  	if (!vma_has_recency(vma))
3857  		return true;
3858  
3859  	if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL))
3860  		return true;
3861  
3862  	if (vma == get_gate_vma(vma->vm_mm))
3863  		return true;
3864  
3865  	if (vma_is_anonymous(vma))
3866  		return !walk->can_swap;
3867  
3868  	if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping))
3869  		return true;
3870  
3871  	mapping = vma->vm_file->f_mapping;
3872  	if (mapping_unevictable(mapping))
3873  		return true;
3874  
3875  	if (shmem_mapping(mapping))
3876  		return !walk->can_swap;
3877  
3878  	/* to exclude special mappings like dax, etc. */
3879  	return !mapping->a_ops->read_folio;
3880  }
3881  
3882  /*
3883   * Some userspace memory allocators map many single-page VMAs. Instead of
3884   * returning back to the PGD table for each of such VMAs, finish an entire PMD
3885   * table to reduce zigzags and improve cache performance.
3886   */
3887  static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args,
3888  			 unsigned long *vm_start, unsigned long *vm_end)
3889  {
3890  	unsigned long start = round_up(*vm_end, size);
3891  	unsigned long end = (start | ~mask) + 1;
3892  	VMA_ITERATOR(vmi, args->mm, start);
3893  
3894  	VM_WARN_ON_ONCE(mask & size);
3895  	VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask));
3896  
3897  	for_each_vma(vmi, args->vma) {
3898  		if (end && end <= args->vma->vm_start)
3899  			return false;
3900  
3901  		if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args))
3902  			continue;
3903  
3904  		*vm_start = max(start, args->vma->vm_start);
3905  		*vm_end = min(end - 1, args->vma->vm_end - 1) + 1;
3906  
3907  		return true;
3908  	}
3909  
3910  	return false;
3911  }
3912  
3913  static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr)
3914  {
3915  	unsigned long pfn = pte_pfn(pte);
3916  
3917  	VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end);
3918  
3919  	if (!pte_present(pte) || is_zero_pfn(pfn))
3920  		return -1;
3921  
3922  	if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte)))
3923  		return -1;
3924  
3925  	if (WARN_ON_ONCE(!pfn_valid(pfn)))
3926  		return -1;
3927  
3928  	return pfn;
3929  }
3930  
3931  #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
3932  static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr)
3933  {
3934  	unsigned long pfn = pmd_pfn(pmd);
3935  
3936  	VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end);
3937  
3938  	if (!pmd_present(pmd) || is_huge_zero_pmd(pmd))
3939  		return -1;
3940  
3941  	if (WARN_ON_ONCE(pmd_devmap(pmd)))
3942  		return -1;
3943  
3944  	if (WARN_ON_ONCE(!pfn_valid(pfn)))
3945  		return -1;
3946  
3947  	return pfn;
3948  }
3949  #endif
3950  
3951  static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg,
3952  				   struct pglist_data *pgdat, bool can_swap)
3953  {
3954  	struct folio *folio;
3955  
3956  	/* try to avoid unnecessary memory loads */
3957  	if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
3958  		return NULL;
3959  
3960  	folio = pfn_folio(pfn);
3961  	if (folio_nid(folio) != pgdat->node_id)
3962  		return NULL;
3963  
3964  	if (folio_memcg_rcu(folio) != memcg)
3965  		return NULL;
3966  
3967  	/* file VMAs can contain anon pages from COW */
3968  	if (!folio_is_file_lru(folio) && !can_swap)
3969  		return NULL;
3970  
3971  	return folio;
3972  }
3973  
3974  static bool suitable_to_scan(int total, int young)
3975  {
3976  	int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8);
3977  
3978  	/* suitable if the average number of young PTEs per cacheline is >=1 */
3979  	return young * n >= total;
3980  }
3981  
3982  static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
3983  			   struct mm_walk *args)
3984  {
3985  	int i;
3986  	pte_t *pte;
3987  	spinlock_t *ptl;
3988  	unsigned long addr;
3989  	int total = 0;
3990  	int young = 0;
3991  	struct lru_gen_mm_walk *walk = args->private;
3992  	struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);
3993  	struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
3994  	int old_gen, new_gen = lru_gen_from_seq(walk->max_seq);
3995  
3996  	VM_WARN_ON_ONCE(pmd_leaf(*pmd));
3997  
3998  	ptl = pte_lockptr(args->mm, pmd);
3999  	if (!spin_trylock(ptl))
4000  		return false;
4001  
4002  	arch_enter_lazy_mmu_mode();
4003  
4004  	pte = pte_offset_map(pmd, start & PMD_MASK);
4005  restart:
4006  	for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) {
4007  		unsigned long pfn;
4008  		struct folio *folio;
4009  
4010  		total++;
4011  		walk->mm_stats[MM_LEAF_TOTAL]++;
4012  
4013  		pfn = get_pte_pfn(pte[i], args->vma, addr);
4014  		if (pfn == -1)
4015  			continue;
4016  
4017  		if (!pte_young(pte[i])) {
4018  			walk->mm_stats[MM_LEAF_OLD]++;
4019  			continue;
4020  		}
4021  
4022  		folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap);
4023  		if (!folio)
4024  			continue;
4025  
4026  		if (!ptep_test_and_clear_young(args->vma, addr, pte + i))
4027  			VM_WARN_ON_ONCE(true);
4028  
4029  		young++;
4030  		walk->mm_stats[MM_LEAF_YOUNG]++;
4031  
4032  		if (pte_dirty(pte[i]) && !folio_test_dirty(folio) &&
4033  		    !(folio_test_anon(folio) && folio_test_swapbacked(folio) &&
4034  		      !folio_test_swapcache(folio)))
4035  			folio_mark_dirty(folio);
4036  
4037  		old_gen = folio_update_gen(folio, new_gen);
4038  		if (old_gen >= 0 && old_gen != new_gen)
4039  			update_batch_size(walk, folio, old_gen, new_gen);
4040  	}
4041  
4042  	if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end))
4043  		goto restart;
4044  
4045  	pte_unmap(pte);
4046  
4047  	arch_leave_lazy_mmu_mode();
4048  	spin_unlock(ptl);
4049  
4050  	return suitable_to_scan(total, young);
4051  }
4052  
4053  #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
4054  static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
4055  				  struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
4056  {
4057  	int i;
4058  	pmd_t *pmd;
4059  	spinlock_t *ptl;
4060  	struct lru_gen_mm_walk *walk = args->private;
4061  	struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);
4062  	struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
4063  	int old_gen, new_gen = lru_gen_from_seq(walk->max_seq);
4064  
4065  	VM_WARN_ON_ONCE(pud_leaf(*pud));
4066  
4067  	/* try to batch at most 1+MIN_LRU_BATCH+1 entries */
4068  	if (*first == -1) {
4069  		*first = addr;
4070  		bitmap_zero(bitmap, MIN_LRU_BATCH);
4071  		return;
4072  	}
4073  
4074  	i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first);
4075  	if (i && i <= MIN_LRU_BATCH) {
4076  		__set_bit(i - 1, bitmap);
4077  		return;
4078  	}
4079  
4080  	pmd = pmd_offset(pud, *first);
4081  
4082  	ptl = pmd_lockptr(args->mm, pmd);
4083  	if (!spin_trylock(ptl))
4084  		goto done;
4085  
4086  	arch_enter_lazy_mmu_mode();
4087  
4088  	do {
4089  		unsigned long pfn;
4090  		struct folio *folio;
4091  
4092  		/* don't round down the first address */
4093  		addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first;
4094  
4095  		pfn = get_pmd_pfn(pmd[i], vma, addr);
4096  		if (pfn == -1)
4097  			goto next;
4098  
4099  		if (!pmd_trans_huge(pmd[i])) {
4100  			if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG))
4101  				pmdp_test_and_clear_young(vma, addr, pmd + i);
4102  			goto next;
4103  		}
4104  
4105  		folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap);
4106  		if (!folio)
4107  			goto next;
4108  
4109  		if (!pmdp_test_and_clear_young(vma, addr, pmd + i))
4110  			goto next;
4111  
4112  		walk->mm_stats[MM_LEAF_YOUNG]++;
4113  
4114  		if (pmd_dirty(pmd[i]) && !folio_test_dirty(folio) &&
4115  		    !(folio_test_anon(folio) && folio_test_swapbacked(folio) &&
4116  		      !folio_test_swapcache(folio)))
4117  			folio_mark_dirty(folio);
4118  
4119  		old_gen = folio_update_gen(folio, new_gen);
4120  		if (old_gen >= 0 && old_gen != new_gen)
4121  			update_batch_size(walk, folio, old_gen, new_gen);
4122  next:
4123  		i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1;
4124  	} while (i <= MIN_LRU_BATCH);
4125  
4126  	arch_leave_lazy_mmu_mode();
4127  	spin_unlock(ptl);
4128  done:
4129  	*first = -1;
4130  }
4131  #else
4132  static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
4133  				  struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
4134  {
4135  }
4136  #endif
4137  
4138  static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
4139  			   struct mm_walk *args)
4140  {
4141  	int i;
4142  	pmd_t *pmd;
4143  	unsigned long next;
4144  	unsigned long addr;
4145  	struct vm_area_struct *vma;
4146  	unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)];
4147  	unsigned long first = -1;
4148  	struct lru_gen_mm_walk *walk = args->private;
4149  
4150  	VM_WARN_ON_ONCE(pud_leaf(*pud));
4151  
4152  	/*
4153  	 * Finish an entire PMD in two passes: the first only reaches to PTE
4154  	 * tables to avoid taking the PMD lock; the second, if necessary, takes
4155  	 * the PMD lock to clear the accessed bit in PMD entries.
4156  	 */
4157  	pmd = pmd_offset(pud, start & PUD_MASK);
4158  restart:
4159  	/* walk_pte_range() may call get_next_vma() */
4160  	vma = args->vma;
4161  	for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) {
4162  		pmd_t val = pmdp_get_lockless(pmd + i);
4163  
4164  		next = pmd_addr_end(addr, end);
4165  
4166  		if (!pmd_present(val) || is_huge_zero_pmd(val)) {
4167  			walk->mm_stats[MM_LEAF_TOTAL]++;
4168  			continue;
4169  		}
4170  
4171  #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4172  		if (pmd_trans_huge(val)) {
4173  			unsigned long pfn = pmd_pfn(val);
4174  			struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
4175  
4176  			walk->mm_stats[MM_LEAF_TOTAL]++;
4177  
4178  			if (!pmd_young(val)) {
4179  				walk->mm_stats[MM_LEAF_OLD]++;
4180  				continue;
4181  			}
4182  
4183  			/* try to avoid unnecessary memory loads */
4184  			if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
4185  				continue;
4186  
4187  			walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
4188  			continue;
4189  		}
4190  #endif
4191  		walk->mm_stats[MM_NONLEAF_TOTAL]++;
4192  
4193  		if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG)) {
4194  			if (!pmd_young(val))
4195  				continue;
4196  
4197  			walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
4198  		}
4199  
4200  		if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i))
4201  			continue;
4202  
4203  		walk->mm_stats[MM_NONLEAF_FOUND]++;
4204  
4205  		if (!walk_pte_range(&val, addr, next, args))
4206  			continue;
4207  
4208  		walk->mm_stats[MM_NONLEAF_ADDED]++;
4209  
4210  		/* carry over to the next generation */
4211  		update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i);
4212  	}
4213  
4214  	walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
4215  
4216  	if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
4217  		goto restart;
4218  }
4219  
4220  static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
4221  			  struct mm_walk *args)
4222  {
4223  	int i;
4224  	pud_t *pud;
4225  	unsigned long addr;
4226  	unsigned long next;
4227  	struct lru_gen_mm_walk *walk = args->private;
4228  
4229  	VM_WARN_ON_ONCE(p4d_leaf(*p4d));
4230  
4231  	pud = pud_offset(p4d, start & P4D_MASK);
4232  restart:
4233  	for (i = pud_index(start), addr = start; addr != end; i++, addr = next) {
4234  		pud_t val = READ_ONCE(pud[i]);
4235  
4236  		next = pud_addr_end(addr, end);
4237  
4238  		if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val)))
4239  			continue;
4240  
4241  		walk_pmd_range(&val, addr, next, args);
4242  
4243  		if (need_resched() || walk->batched >= MAX_LRU_BATCH) {
4244  			end = (addr | ~PUD_MASK) + 1;
4245  			goto done;
4246  		}
4247  	}
4248  
4249  	if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end))
4250  		goto restart;
4251  
4252  	end = round_up(end, P4D_SIZE);
4253  done:
4254  	if (!end || !args->vma)
4255  		return 1;
4256  
4257  	walk->next_addr = max(end, args->vma->vm_start);
4258  
4259  	return -EAGAIN;
4260  }
4261  
4262  static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk)
4263  {
4264  	static const struct mm_walk_ops mm_walk_ops = {
4265  		.test_walk = should_skip_vma,
4266  		.p4d_entry = walk_pud_range,
4267  	};
4268  
4269  	int err;
4270  	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
4271  
4272  	walk->next_addr = FIRST_USER_ADDRESS;
4273  
4274  	do {
4275  		DEFINE_MAX_SEQ(lruvec);
4276  
4277  		err = -EBUSY;
4278  
4279  		/* another thread might have called inc_max_seq() */
4280  		if (walk->max_seq != max_seq)
4281  			break;
4282  
4283  		/* folio_update_gen() requires stable folio_memcg() */
4284  		if (!mem_cgroup_trylock_pages(memcg))
4285  			break;
4286  
4287  		/* the caller might be holding the lock for write */
4288  		if (mmap_read_trylock(mm)) {
4289  			err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk);
4290  
4291  			mmap_read_unlock(mm);
4292  		}
4293  
4294  		mem_cgroup_unlock_pages();
4295  
4296  		if (walk->batched) {
4297  			spin_lock_irq(&lruvec->lru_lock);
4298  			reset_batch_size(lruvec, walk);
4299  			spin_unlock_irq(&lruvec->lru_lock);
4300  		}
4301  
4302  		cond_resched();
4303  	} while (err == -EAGAIN);
4304  }
4305  
4306  static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat, bool force_alloc)
4307  {
4308  	struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk;
4309  
4310  	if (pgdat && current_is_kswapd()) {
4311  		VM_WARN_ON_ONCE(walk);
4312  
4313  		walk = &pgdat->mm_walk;
4314  	} else if (!walk && force_alloc) {
4315  		VM_WARN_ON_ONCE(current_is_kswapd());
4316  
4317  		walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
4318  	}
4319  
4320  	current->reclaim_state->mm_walk = walk;
4321  
4322  	return walk;
4323  }
4324  
4325  static void clear_mm_walk(void)
4326  {
4327  	struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk;
4328  
4329  	VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages)));
4330  	VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats)));
4331  
4332  	current->reclaim_state->mm_walk = NULL;
4333  
4334  	if (!current_is_kswapd())
4335  		kfree(walk);
4336  }
4337  
4338  static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap)
4339  {
4340  	int zone;
4341  	int remaining = MAX_LRU_BATCH;
4342  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
4343  	int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
4344  
4345  	if (type == LRU_GEN_ANON && !can_swap)
4346  		goto done;
4347  
4348  	/* prevent cold/hot inversion if force_scan is true */
4349  	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4350  		struct list_head *head = &lrugen->folios[old_gen][type][zone];
4351  
4352  		while (!list_empty(head)) {
4353  			struct folio *folio = lru_to_folio(head);
4354  
4355  			VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
4356  			VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
4357  			VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
4358  			VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
4359  
4360  			new_gen = folio_inc_gen(lruvec, folio, false);
4361  			list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
4362  
4363  			if (!--remaining)
4364  				return false;
4365  		}
4366  	}
4367  done:
4368  	reset_ctrl_pos(lruvec, type, true);
4369  	WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1);
4370  
4371  	return true;
4372  }
4373  
4374  static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap)
4375  {
4376  	int gen, type, zone;
4377  	bool success = false;
4378  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
4379  	DEFINE_MIN_SEQ(lruvec);
4380  
4381  	VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
4382  
4383  	/* find the oldest populated generation */
4384  	for (type = !can_swap; type < ANON_AND_FILE; type++) {
4385  		while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) {
4386  			gen = lru_gen_from_seq(min_seq[type]);
4387  
4388  			for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4389  				if (!list_empty(&lrugen->folios[gen][type][zone]))
4390  					goto next;
4391  			}
4392  
4393  			min_seq[type]++;
4394  		}
4395  next:
4396  		;
4397  	}
4398  
4399  	/* see the comment on lru_gen_folio */
4400  	if (can_swap) {
4401  		min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]);
4402  		min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]);
4403  	}
4404  
4405  	for (type = !can_swap; type < ANON_AND_FILE; type++) {
4406  		if (min_seq[type] == lrugen->min_seq[type])
4407  			continue;
4408  
4409  		reset_ctrl_pos(lruvec, type, true);
4410  		WRITE_ONCE(lrugen->min_seq[type], min_seq[type]);
4411  		success = true;
4412  	}
4413  
4414  	return success;
4415  }
4416  
4417  static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan)
4418  {
4419  	int prev, next;
4420  	int type, zone;
4421  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
4422  
4423  	spin_lock_irq(&lruvec->lru_lock);
4424  
4425  	VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
4426  
4427  	for (type = ANON_AND_FILE - 1; type >= 0; type--) {
4428  		if (get_nr_gens(lruvec, type) != MAX_NR_GENS)
4429  			continue;
4430  
4431  		VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap));
4432  
4433  		while (!inc_min_seq(lruvec, type, can_swap)) {
4434  			spin_unlock_irq(&lruvec->lru_lock);
4435  			cond_resched();
4436  			spin_lock_irq(&lruvec->lru_lock);
4437  		}
4438  	}
4439  
4440  	/*
4441  	 * Update the active/inactive LRU sizes for compatibility. Both sides of
4442  	 * the current max_seq need to be covered, since max_seq+1 can overlap
4443  	 * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do
4444  	 * overlap, cold/hot inversion happens.
4445  	 */
4446  	prev = lru_gen_from_seq(lrugen->max_seq - 1);
4447  	next = lru_gen_from_seq(lrugen->max_seq + 1);
4448  
4449  	for (type = 0; type < ANON_AND_FILE; type++) {
4450  		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4451  			enum lru_list lru = type * LRU_INACTIVE_FILE;
4452  			long delta = lrugen->nr_pages[prev][type][zone] -
4453  				     lrugen->nr_pages[next][type][zone];
4454  
4455  			if (!delta)
4456  				continue;
4457  
4458  			__update_lru_size(lruvec, lru, zone, delta);
4459  			__update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta);
4460  		}
4461  	}
4462  
4463  	for (type = 0; type < ANON_AND_FILE; type++)
4464  		reset_ctrl_pos(lruvec, type, false);
4465  
4466  	WRITE_ONCE(lrugen->timestamps[next], jiffies);
4467  	/* make sure preceding modifications appear */
4468  	smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1);
4469  
4470  	spin_unlock_irq(&lruvec->lru_lock);
4471  }
4472  
4473  static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
4474  			       struct scan_control *sc, bool can_swap, bool force_scan)
4475  {
4476  	bool success;
4477  	struct lru_gen_mm_walk *walk;
4478  	struct mm_struct *mm = NULL;
4479  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
4480  
4481  	VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq));
4482  
4483  	/* see the comment in iterate_mm_list() */
4484  	if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) {
4485  		success = false;
4486  		goto done;
4487  	}
4488  
4489  	/*
4490  	 * If the hardware doesn't automatically set the accessed bit, fallback
4491  	 * to lru_gen_look_around(), which only clears the accessed bit in a
4492  	 * handful of PTEs. Spreading the work out over a period of time usually
4493  	 * is less efficient, but it avoids bursty page faults.
4494  	 */
4495  	if (!arch_has_hw_pte_young() || !get_cap(LRU_GEN_MM_WALK)) {
4496  		success = iterate_mm_list_nowalk(lruvec, max_seq);
4497  		goto done;
4498  	}
4499  
4500  	walk = set_mm_walk(NULL, true);
4501  	if (!walk) {
4502  		success = iterate_mm_list_nowalk(lruvec, max_seq);
4503  		goto done;
4504  	}
4505  
4506  	walk->lruvec = lruvec;
4507  	walk->max_seq = max_seq;
4508  	walk->can_swap = can_swap;
4509  	walk->force_scan = force_scan;
4510  
4511  	do {
4512  		success = iterate_mm_list(lruvec, walk, &mm);
4513  		if (mm)
4514  			walk_mm(lruvec, mm, walk);
4515  	} while (mm);
4516  done:
4517  	if (success)
4518  		inc_max_seq(lruvec, can_swap, force_scan);
4519  
4520  	return success;
4521  }
4522  
4523  /******************************************************************************
4524   *                          working set protection
4525   ******************************************************************************/
4526  
4527  static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
4528  {
4529  	int gen, type, zone;
4530  	unsigned long total = 0;
4531  	bool can_swap = get_swappiness(lruvec, sc);
4532  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
4533  	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
4534  	DEFINE_MAX_SEQ(lruvec);
4535  	DEFINE_MIN_SEQ(lruvec);
4536  
4537  	for (type = !can_swap; type < ANON_AND_FILE; type++) {
4538  		unsigned long seq;
4539  
4540  		for (seq = min_seq[type]; seq <= max_seq; seq++) {
4541  			gen = lru_gen_from_seq(seq);
4542  
4543  			for (zone = 0; zone < MAX_NR_ZONES; zone++)
4544  				total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
4545  		}
4546  	}
4547  
4548  	/* whether the size is big enough to be helpful */
4549  	return mem_cgroup_online(memcg) ? (total >> sc->priority) : total;
4550  }
4551  
4552  static bool lruvec_is_reclaimable(struct lruvec *lruvec, struct scan_control *sc,
4553  				  unsigned long min_ttl)
4554  {
4555  	int gen;
4556  	unsigned long birth;
4557  	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
4558  	DEFINE_MIN_SEQ(lruvec);
4559  
4560  	/* see the comment on lru_gen_folio */
4561  	gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]);
4562  	birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
4563  
4564  	if (time_is_after_jiffies(birth + min_ttl))
4565  		return false;
4566  
4567  	if (!lruvec_is_sizable(lruvec, sc))
4568  		return false;
4569  
4570  	mem_cgroup_calculate_protection(NULL, memcg);
4571  
4572  	return !mem_cgroup_below_min(NULL, memcg);
4573  }
4574  
4575  /* to protect the working set of the last N jiffies */
4576  static unsigned long lru_gen_min_ttl __read_mostly;
4577  
4578  static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
4579  {
4580  	struct mem_cgroup *memcg;
4581  	unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl);
4582  
4583  	VM_WARN_ON_ONCE(!current_is_kswapd());
4584  
4585  	/* check the order to exclude compaction-induced reclaim */
4586  	if (!min_ttl || sc->order || sc->priority == DEF_PRIORITY)
4587  		return;
4588  
4589  	memcg = mem_cgroup_iter(NULL, NULL, NULL);
4590  	do {
4591  		struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
4592  
4593  		if (lruvec_is_reclaimable(lruvec, sc, min_ttl)) {
4594  			mem_cgroup_iter_break(NULL, memcg);
4595  			return;
4596  		}
4597  
4598  		cond_resched();
4599  	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
4600  
4601  	/*
4602  	 * The main goal is to OOM kill if every generation from all memcgs is
4603  	 * younger than min_ttl. However, another possibility is all memcgs are
4604  	 * either too small or below min.
4605  	 */
4606  	if (mutex_trylock(&oom_lock)) {
4607  		struct oom_control oc = {
4608  			.gfp_mask = sc->gfp_mask,
4609  		};
4610  
4611  		out_of_memory(&oc);
4612  
4613  		mutex_unlock(&oom_lock);
4614  	}
4615  }
4616  
4617  /******************************************************************************
4618   *                          rmap/PT walk feedback
4619   ******************************************************************************/
4620  
4621  /*
4622   * This function exploits spatial locality when shrink_folio_list() walks the
4623   * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If
4624   * the scan was done cacheline efficiently, it adds the PMD entry pointing to
4625   * the PTE table to the Bloom filter. This forms a feedback loop between the
4626   * eviction and the aging.
4627   */
4628  void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
4629  {
4630  	int i;
4631  	unsigned long start;
4632  	unsigned long end;
4633  	struct lru_gen_mm_walk *walk;
4634  	int young = 0;
4635  	pte_t *pte = pvmw->pte;
4636  	unsigned long addr = pvmw->address;
4637  	struct folio *folio = pfn_folio(pvmw->pfn);
4638  	struct mem_cgroup *memcg = folio_memcg(folio);
4639  	struct pglist_data *pgdat = folio_pgdat(folio);
4640  	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
4641  	DEFINE_MAX_SEQ(lruvec);
4642  	int old_gen, new_gen = lru_gen_from_seq(max_seq);
4643  
4644  	lockdep_assert_held(pvmw->ptl);
4645  	VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio);
4646  
4647  	if (spin_is_contended(pvmw->ptl))
4648  		return;
4649  
4650  	/* avoid taking the LRU lock under the PTL when possible */
4651  	walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL;
4652  
4653  	start = max(addr & PMD_MASK, pvmw->vma->vm_start);
4654  	end = min(addr | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1;
4655  
4656  	if (end - start > MIN_LRU_BATCH * PAGE_SIZE) {
4657  		if (addr - start < MIN_LRU_BATCH * PAGE_SIZE / 2)
4658  			end = start + MIN_LRU_BATCH * PAGE_SIZE;
4659  		else if (end - addr < MIN_LRU_BATCH * PAGE_SIZE / 2)
4660  			start = end - MIN_LRU_BATCH * PAGE_SIZE;
4661  		else {
4662  			start = addr - MIN_LRU_BATCH * PAGE_SIZE / 2;
4663  			end = addr + MIN_LRU_BATCH * PAGE_SIZE / 2;
4664  		}
4665  	}
4666  
4667  	/* folio_update_gen() requires stable folio_memcg() */
4668  	if (!mem_cgroup_trylock_pages(memcg))
4669  		return;
4670  
4671  	arch_enter_lazy_mmu_mode();
4672  
4673  	pte -= (addr - start) / PAGE_SIZE;
4674  
4675  	for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) {
4676  		unsigned long pfn;
4677  
4678  		pfn = get_pte_pfn(pte[i], pvmw->vma, addr);
4679  		if (pfn == -1)
4680  			continue;
4681  
4682  		if (!pte_young(pte[i]))
4683  			continue;
4684  
4685  		folio = get_pfn_folio(pfn, memcg, pgdat, !walk || walk->can_swap);
4686  		if (!folio)
4687  			continue;
4688  
4689  		if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i))
4690  			VM_WARN_ON_ONCE(true);
4691  
4692  		young++;
4693  
4694  		if (pte_dirty(pte[i]) && !folio_test_dirty(folio) &&
4695  		    !(folio_test_anon(folio) && folio_test_swapbacked(folio) &&
4696  		      !folio_test_swapcache(folio)))
4697  			folio_mark_dirty(folio);
4698  
4699  		if (walk) {
4700  			old_gen = folio_update_gen(folio, new_gen);
4701  			if (old_gen >= 0 && old_gen != new_gen)
4702  				update_batch_size(walk, folio, old_gen, new_gen);
4703  
4704  			continue;
4705  		}
4706  
4707  		old_gen = folio_lru_gen(folio);
4708  		if (old_gen < 0)
4709  			folio_set_referenced(folio);
4710  		else if (old_gen != new_gen)
4711  			folio_activate(folio);
4712  	}
4713  
4714  	arch_leave_lazy_mmu_mode();
4715  	mem_cgroup_unlock_pages();
4716  
4717  	/* feedback from rmap walkers to page table walkers */
4718  	if (suitable_to_scan(i, young))
4719  		update_bloom_filter(lruvec, max_seq, pvmw->pmd);
4720  }
4721  
4722  /******************************************************************************
4723   *                          memcg LRU
4724   ******************************************************************************/
4725  
4726  /* see the comment on MEMCG_NR_GENS */
4727  enum {
4728  	MEMCG_LRU_NOP,
4729  	MEMCG_LRU_HEAD,
4730  	MEMCG_LRU_TAIL,
4731  	MEMCG_LRU_OLD,
4732  	MEMCG_LRU_YOUNG,
4733  };
4734  
4735  #ifdef CONFIG_MEMCG
4736  
4737  static int lru_gen_memcg_seg(struct lruvec *lruvec)
4738  {
4739  	return READ_ONCE(lruvec->lrugen.seg);
4740  }
4741  
4742  static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op)
4743  {
4744  	int seg;
4745  	int old, new;
4746  	int bin = get_random_u32_below(MEMCG_NR_BINS);
4747  	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
4748  
4749  	spin_lock(&pgdat->memcg_lru.lock);
4750  
4751  	VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));
4752  
4753  	seg = 0;
4754  	new = old = lruvec->lrugen.gen;
4755  
4756  	/* see the comment on MEMCG_NR_GENS */
4757  	if (op == MEMCG_LRU_HEAD)
4758  		seg = MEMCG_LRU_HEAD;
4759  	else if (op == MEMCG_LRU_TAIL)
4760  		seg = MEMCG_LRU_TAIL;
4761  	else if (op == MEMCG_LRU_OLD)
4762  		new = get_memcg_gen(pgdat->memcg_lru.seq);
4763  	else if (op == MEMCG_LRU_YOUNG)
4764  		new = get_memcg_gen(pgdat->memcg_lru.seq + 1);
4765  	else
4766  		VM_WARN_ON_ONCE(true);
4767  
4768  	hlist_nulls_del_rcu(&lruvec->lrugen.list);
4769  
4770  	if (op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD)
4771  		hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
4772  	else
4773  		hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
4774  
4775  	pgdat->memcg_lru.nr_memcgs[old]--;
4776  	pgdat->memcg_lru.nr_memcgs[new]++;
4777  
4778  	lruvec->lrugen.gen = new;
4779  	WRITE_ONCE(lruvec->lrugen.seg, seg);
4780  
4781  	if (!pgdat->memcg_lru.nr_memcgs[old] && old == get_memcg_gen(pgdat->memcg_lru.seq))
4782  		WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
4783  
4784  	spin_unlock(&pgdat->memcg_lru.lock);
4785  }
4786  
4787  void lru_gen_online_memcg(struct mem_cgroup *memcg)
4788  {
4789  	int gen;
4790  	int nid;
4791  	int bin = get_random_u32_below(MEMCG_NR_BINS);
4792  
4793  	for_each_node(nid) {
4794  		struct pglist_data *pgdat = NODE_DATA(nid);
4795  		struct lruvec *lruvec = get_lruvec(memcg, nid);
4796  
4797  		spin_lock(&pgdat->memcg_lru.lock);
4798  
4799  		VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list));
4800  
4801  		gen = get_memcg_gen(pgdat->memcg_lru.seq);
4802  
4803  		hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]);
4804  		pgdat->memcg_lru.nr_memcgs[gen]++;
4805  
4806  		lruvec->lrugen.gen = gen;
4807  
4808  		spin_unlock(&pgdat->memcg_lru.lock);
4809  	}
4810  }
4811  
4812  void lru_gen_offline_memcg(struct mem_cgroup *memcg)
4813  {
4814  	int nid;
4815  
4816  	for_each_node(nid) {
4817  		struct lruvec *lruvec = get_lruvec(memcg, nid);
4818  
4819  		lru_gen_rotate_memcg(lruvec, MEMCG_LRU_OLD);
4820  	}
4821  }
4822  
4823  void lru_gen_release_memcg(struct mem_cgroup *memcg)
4824  {
4825  	int gen;
4826  	int nid;
4827  
4828  	for_each_node(nid) {
4829  		struct pglist_data *pgdat = NODE_DATA(nid);
4830  		struct lruvec *lruvec = get_lruvec(memcg, nid);
4831  
4832  		spin_lock(&pgdat->memcg_lru.lock);
4833  
4834  		VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));
4835  
4836  		gen = lruvec->lrugen.gen;
4837  
4838  		hlist_nulls_del_rcu(&lruvec->lrugen.list);
4839  		pgdat->memcg_lru.nr_memcgs[gen]--;
4840  
4841  		if (!pgdat->memcg_lru.nr_memcgs[gen] && gen == get_memcg_gen(pgdat->memcg_lru.seq))
4842  			WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
4843  
4844  		spin_unlock(&pgdat->memcg_lru.lock);
4845  	}
4846  }
4847  
4848  void lru_gen_soft_reclaim(struct lruvec *lruvec)
4849  {
4850  	/* see the comment on MEMCG_NR_GENS */
4851  	if (lru_gen_memcg_seg(lruvec) != MEMCG_LRU_HEAD)
4852  		lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD);
4853  }
4854  
4855  #else /* !CONFIG_MEMCG */
4856  
4857  static int lru_gen_memcg_seg(struct lruvec *lruvec)
4858  {
4859  	return 0;
4860  }
4861  
4862  #endif
4863  
4864  /******************************************************************************
4865   *                          the eviction
4866   ******************************************************************************/
4867  
4868  static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
4869  {
4870  	bool success;
4871  	int gen = folio_lru_gen(folio);
4872  	int type = folio_is_file_lru(folio);
4873  	int zone = folio_zonenum(folio);
4874  	int delta = folio_nr_pages(folio);
4875  	int refs = folio_lru_refs(folio);
4876  	int tier = lru_tier_from_refs(refs);
4877  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
4878  
4879  	VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);
4880  
4881  	/* unevictable */
4882  	if (!folio_evictable(folio)) {
4883  		success = lru_gen_del_folio(lruvec, folio, true);
4884  		VM_WARN_ON_ONCE_FOLIO(!success, folio);
4885  		folio_set_unevictable(folio);
4886  		lruvec_add_folio(lruvec, folio);
4887  		__count_vm_events(UNEVICTABLE_PGCULLED, delta);
4888  		return true;
4889  	}
4890  
4891  	/* dirty lazyfree */
4892  	if (type == LRU_GEN_FILE && folio_test_anon(folio) && folio_test_dirty(folio)) {
4893  		success = lru_gen_del_folio(lruvec, folio, true);
4894  		VM_WARN_ON_ONCE_FOLIO(!success, folio);
4895  		folio_set_swapbacked(folio);
4896  		lruvec_add_folio_tail(lruvec, folio);
4897  		return true;
4898  	}
4899  
4900  	/* promoted */
4901  	if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
4902  		list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
4903  		return true;
4904  	}
4905  
4906  	/* protected */
4907  	if (tier > tier_idx) {
4908  		int hist = lru_hist_from_seq(lrugen->min_seq[type]);
4909  
4910  		gen = folio_inc_gen(lruvec, folio, false);
4911  		list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
4912  
4913  		WRITE_ONCE(lrugen->protected[hist][type][tier - 1],
4914  			   lrugen->protected[hist][type][tier - 1] + delta);
4915  		__mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta);
4916  		return true;
4917  	}
4918  
4919  	/* waiting for writeback */
4920  	if (folio_test_locked(folio) || folio_test_writeback(folio) ||
4921  	    (type == LRU_GEN_FILE && folio_test_dirty(folio))) {
4922  		gen = folio_inc_gen(lruvec, folio, true);
4923  		list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
4924  		return true;
4925  	}
4926  
4927  	return false;
4928  }
4929  
4930  static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc)
4931  {
4932  	bool success;
4933  
4934  	/* swapping inhibited */
4935  	if (!(sc->gfp_mask & __GFP_IO) &&
4936  	    (folio_test_dirty(folio) ||
4937  	     (folio_test_anon(folio) && !folio_test_swapcache(folio))))
4938  		return false;
4939  
4940  	/* raced with release_pages() */
4941  	if (!folio_try_get(folio))
4942  		return false;
4943  
4944  	/* raced with another isolation */
4945  	if (!folio_test_clear_lru(folio)) {
4946  		folio_put(folio);
4947  		return false;
4948  	}
4949  
4950  	/* see the comment on MAX_NR_TIERS */
4951  	if (!folio_test_referenced(folio))
4952  		set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0);
4953  
4954  	/* for shrink_folio_list() */
4955  	folio_clear_reclaim(folio);
4956  	folio_clear_referenced(folio);
4957  
4958  	success = lru_gen_del_folio(lruvec, folio, true);
4959  	VM_WARN_ON_ONCE_FOLIO(!success, folio);
4960  
4961  	return true;
4962  }
4963  
4964  static int scan_folios(struct lruvec *lruvec, struct scan_control *sc,
4965  		       int type, int tier, struct list_head *list)
4966  {
4967  	int gen, zone;
4968  	enum vm_event_item item;
4969  	int sorted = 0;
4970  	int scanned = 0;
4971  	int isolated = 0;
4972  	int remaining = MAX_LRU_BATCH;
4973  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
4974  	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
4975  
4976  	VM_WARN_ON_ONCE(!list_empty(list));
4977  
4978  	if (get_nr_gens(lruvec, type) == MIN_NR_GENS)
4979  		return 0;
4980  
4981  	gen = lru_gen_from_seq(lrugen->min_seq[type]);
4982  
4983  	for (zone = sc->reclaim_idx; zone >= 0; zone--) {
4984  		LIST_HEAD(moved);
4985  		int skipped = 0;
4986  		struct list_head *head = &lrugen->folios[gen][type][zone];
4987  
4988  		while (!list_empty(head)) {
4989  			struct folio *folio = lru_to_folio(head);
4990  			int delta = folio_nr_pages(folio);
4991  
4992  			VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
4993  			VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
4994  			VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
4995  			VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
4996  
4997  			scanned += delta;
4998  
4999  			if (sort_folio(lruvec, folio, tier))
5000  				sorted += delta;
5001  			else if (isolate_folio(lruvec, folio, sc)) {
5002  				list_add(&folio->lru, list);
5003  				isolated += delta;
5004  			} else {
5005  				list_move(&folio->lru, &moved);
5006  				skipped += delta;
5007  			}
5008  
5009  			if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH)
5010  				break;
5011  		}
5012  
5013  		if (skipped) {
5014  			list_splice(&moved, head);
5015  			__count_zid_vm_events(PGSCAN_SKIP, zone, skipped);
5016  		}
5017  
5018  		if (!remaining || isolated >= MIN_LRU_BATCH)
5019  			break;
5020  	}
5021  
5022  	item = PGSCAN_KSWAPD + reclaimer_offset();
5023  	if (!cgroup_reclaim(sc)) {
5024  		__count_vm_events(item, isolated);
5025  		__count_vm_events(PGREFILL, sorted);
5026  	}
5027  	__count_memcg_events(memcg, item, isolated);
5028  	__count_memcg_events(memcg, PGREFILL, sorted);
5029  	__count_vm_events(PGSCAN_ANON + type, isolated);
5030  
5031  	/*
5032  	 * There might not be eligible folios due to reclaim_idx. Check the
5033  	 * remaining to prevent livelock if it's not making progress.
5034  	 */
5035  	return isolated || !remaining ? scanned : 0;
5036  }
5037  
5038  static int get_tier_idx(struct lruvec *lruvec, int type)
5039  {
5040  	int tier;
5041  	struct ctrl_pos sp, pv;
5042  
5043  	/*
5044  	 * To leave a margin for fluctuations, use a larger gain factor (1:2).
5045  	 * This value is chosen because any other tier would have at least twice
5046  	 * as many refaults as the first tier.
5047  	 */
5048  	read_ctrl_pos(lruvec, type, 0, 1, &sp);
5049  	for (tier = 1; tier < MAX_NR_TIERS; tier++) {
5050  		read_ctrl_pos(lruvec, type, tier, 2, &pv);
5051  		if (!positive_ctrl_err(&sp, &pv))
5052  			break;
5053  	}
5054  
5055  	return tier - 1;
5056  }
5057  
5058  static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx)
5059  {
5060  	int type, tier;
5061  	struct ctrl_pos sp, pv;
5062  	int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness };
5063  
5064  	/*
5065  	 * Compare the first tier of anon with that of file to determine which
5066  	 * type to scan. Also need to compare other tiers of the selected type
5067  	 * with the first tier of the other type to determine the last tier (of
5068  	 * the selected type) to evict.
5069  	 */
5070  	read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp);
5071  	read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv);
5072  	type = positive_ctrl_err(&sp, &pv);
5073  
5074  	read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp);
5075  	for (tier = 1; tier < MAX_NR_TIERS; tier++) {
5076  		read_ctrl_pos(lruvec, type, tier, gain[type], &pv);
5077  		if (!positive_ctrl_err(&sp, &pv))
5078  			break;
5079  	}
5080  
5081  	*tier_idx = tier - 1;
5082  
5083  	return type;
5084  }
5085  
5086  static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness,
5087  			  int *type_scanned, struct list_head *list)
5088  {
5089  	int i;
5090  	int type;
5091  	int scanned;
5092  	int tier = -1;
5093  	DEFINE_MIN_SEQ(lruvec);
5094  
5095  	/*
5096  	 * Try to make the obvious choice first. When anon and file are both
5097  	 * available from the same generation, interpret swappiness 1 as file
5098  	 * first and 200 as anon first.
5099  	 */
5100  	if (!swappiness)
5101  		type = LRU_GEN_FILE;
5102  	else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE])
5103  		type = LRU_GEN_ANON;
5104  	else if (swappiness == 1)
5105  		type = LRU_GEN_FILE;
5106  	else if (swappiness == 200)
5107  		type = LRU_GEN_ANON;
5108  	else
5109  		type = get_type_to_scan(lruvec, swappiness, &tier);
5110  
5111  	for (i = !swappiness; i < ANON_AND_FILE; i++) {
5112  		if (tier < 0)
5113  			tier = get_tier_idx(lruvec, type);
5114  
5115  		scanned = scan_folios(lruvec, sc, type, tier, list);
5116  		if (scanned)
5117  			break;
5118  
5119  		type = !type;
5120  		tier = -1;
5121  	}
5122  
5123  	*type_scanned = type;
5124  
5125  	return scanned;
5126  }
5127  
5128  static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness)
5129  {
5130  	int type;
5131  	int scanned;
5132  	int reclaimed;
5133  	LIST_HEAD(list);
5134  	LIST_HEAD(clean);
5135  	struct folio *folio;
5136  	struct folio *next;
5137  	enum vm_event_item item;
5138  	struct reclaim_stat stat;
5139  	struct lru_gen_mm_walk *walk;
5140  	bool skip_retry = false;
5141  	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
5142  	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
5143  
5144  	spin_lock_irq(&lruvec->lru_lock);
5145  
5146  	scanned = isolate_folios(lruvec, sc, swappiness, &type, &list);
5147  
5148  	scanned += try_to_inc_min_seq(lruvec, swappiness);
5149  
5150  	if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS)
5151  		scanned = 0;
5152  
5153  	spin_unlock_irq(&lruvec->lru_lock);
5154  
5155  	if (list_empty(&list))
5156  		return scanned;
5157  retry:
5158  	reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false);
5159  	sc->nr_reclaimed += reclaimed;
5160  
5161  	list_for_each_entry_safe_reverse(folio, next, &list, lru) {
5162  		if (!folio_evictable(folio)) {
5163  			list_del(&folio->lru);
5164  			folio_putback_lru(folio);
5165  			continue;
5166  		}
5167  
5168  		if (folio_test_reclaim(folio) &&
5169  		    (folio_test_dirty(folio) || folio_test_writeback(folio))) {
5170  			/* restore LRU_REFS_FLAGS cleared by isolate_folio() */
5171  			if (folio_test_workingset(folio))
5172  				folio_set_referenced(folio);
5173  			continue;
5174  		}
5175  
5176  		if (skip_retry || folio_test_active(folio) || folio_test_referenced(folio) ||
5177  		    folio_mapped(folio) || folio_test_locked(folio) ||
5178  		    folio_test_dirty(folio) || folio_test_writeback(folio)) {
5179  			/* don't add rejected folios to the oldest generation */
5180  			set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS,
5181  				      BIT(PG_active));
5182  			continue;
5183  		}
5184  
5185  		/* retry folios that may have missed folio_rotate_reclaimable() */
5186  		list_move(&folio->lru, &clean);
5187  		sc->nr_scanned -= folio_nr_pages(folio);
5188  	}
5189  
5190  	spin_lock_irq(&lruvec->lru_lock);
5191  
5192  	move_folios_to_lru(lruvec, &list);
5193  
5194  	walk = current->reclaim_state->mm_walk;
5195  	if (walk && walk->batched)
5196  		reset_batch_size(lruvec, walk);
5197  
5198  	item = PGSTEAL_KSWAPD + reclaimer_offset();
5199  	if (!cgroup_reclaim(sc))
5200  		__count_vm_events(item, reclaimed);
5201  	__count_memcg_events(memcg, item, reclaimed);
5202  	__count_vm_events(PGSTEAL_ANON + type, reclaimed);
5203  
5204  	spin_unlock_irq(&lruvec->lru_lock);
5205  
5206  	mem_cgroup_uncharge_list(&list);
5207  	free_unref_page_list(&list);
5208  
5209  	INIT_LIST_HEAD(&list);
5210  	list_splice_init(&clean, &list);
5211  
5212  	if (!list_empty(&list)) {
5213  		skip_retry = true;
5214  		goto retry;
5215  	}
5216  
5217  	return scanned;
5218  }
5219  
5220  static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
5221  			     struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan)
5222  {
5223  	int gen, type, zone;
5224  	unsigned long old = 0;
5225  	unsigned long young = 0;
5226  	unsigned long total = 0;
5227  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
5228  	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
5229  	DEFINE_MIN_SEQ(lruvec);
5230  
5231  	/* whether this lruvec is completely out of cold folios */
5232  	if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) {
5233  		*nr_to_scan = 0;
5234  		return true;
5235  	}
5236  
5237  	for (type = !can_swap; type < ANON_AND_FILE; type++) {
5238  		unsigned long seq;
5239  
5240  		for (seq = min_seq[type]; seq <= max_seq; seq++) {
5241  			unsigned long size = 0;
5242  
5243  			gen = lru_gen_from_seq(seq);
5244  
5245  			for (zone = 0; zone < MAX_NR_ZONES; zone++)
5246  				size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
5247  
5248  			total += size;
5249  			if (seq == max_seq)
5250  				young += size;
5251  			else if (seq + MIN_NR_GENS == max_seq)
5252  				old += size;
5253  		}
5254  	}
5255  
5256  	/* try to scrape all its memory if this memcg was deleted */
5257  	*nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total;
5258  
5259  	/*
5260  	 * The aging tries to be lazy to reduce the overhead, while the eviction
5261  	 * stalls when the number of generations reaches MIN_NR_GENS. Hence, the
5262  	 * ideal number of generations is MIN_NR_GENS+1.
5263  	 */
5264  	if (min_seq[!can_swap] + MIN_NR_GENS < max_seq)
5265  		return false;
5266  
5267  	/*
5268  	 * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1)
5269  	 * of the total number of pages for each generation. A reasonable range
5270  	 * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The
5271  	 * aging cares about the upper bound of hot pages, while the eviction
5272  	 * cares about the lower bound of cold pages.
5273  	 */
5274  	if (young * MIN_NR_GENS > total)
5275  		return true;
5276  	if (old * (MIN_NR_GENS + 2) < total)
5277  		return true;
5278  
5279  	return false;
5280  }
5281  
5282  /*
5283   * For future optimizations:
5284   * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
5285   *    reclaim.
5286   */
5287  static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, bool can_swap)
5288  {
5289  	unsigned long nr_to_scan;
5290  	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
5291  	DEFINE_MAX_SEQ(lruvec);
5292  
5293  	if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg))
5294  		return 0;
5295  
5296  	if (!should_run_aging(lruvec, max_seq, sc, can_swap, &nr_to_scan))
5297  		return nr_to_scan;
5298  
5299  	/* skip the aging path at the default priority */
5300  	if (sc->priority == DEF_PRIORITY)
5301  		return nr_to_scan;
5302  
5303  	/* skip this lruvec as it's low on cold folios */
5304  	return try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false) ? -1 : 0;
5305  }
5306  
5307  static unsigned long get_nr_to_reclaim(struct scan_control *sc)
5308  {
5309  	/* don't abort memcg reclaim to ensure fairness */
5310  	if (!global_reclaim(sc))
5311  		return -1;
5312  
5313  	return max(sc->nr_to_reclaim, compact_gap(sc->order));
5314  }
5315  
5316  static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
5317  {
5318  	long nr_to_scan;
5319  	unsigned long scanned = 0;
5320  	unsigned long nr_to_reclaim = get_nr_to_reclaim(sc);
5321  	int swappiness = get_swappiness(lruvec, sc);
5322  
5323  	/* clean file folios are more likely to exist */
5324  	if (swappiness && !(sc->gfp_mask & __GFP_IO))
5325  		swappiness = 1;
5326  
5327  	while (true) {
5328  		int delta;
5329  
5330  		nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness);
5331  		if (nr_to_scan <= 0)
5332  			break;
5333  
5334  		delta = evict_folios(lruvec, sc, swappiness);
5335  		if (!delta)
5336  			break;
5337  
5338  		scanned += delta;
5339  		if (scanned >= nr_to_scan)
5340  			break;
5341  
5342  		if (sc->nr_reclaimed >= nr_to_reclaim)
5343  			break;
5344  
5345  		cond_resched();
5346  	}
5347  
5348  	/* whether try_to_inc_max_seq() was successful */
5349  	return nr_to_scan < 0;
5350  }
5351  
5352  static int shrink_one(struct lruvec *lruvec, struct scan_control *sc)
5353  {
5354  	bool success;
5355  	unsigned long scanned = sc->nr_scanned;
5356  	unsigned long reclaimed = sc->nr_reclaimed;
5357  	int seg = lru_gen_memcg_seg(lruvec);
5358  	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
5359  	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
5360  
5361  	/* see the comment on MEMCG_NR_GENS */
5362  	if (!lruvec_is_sizable(lruvec, sc))
5363  		return seg != MEMCG_LRU_TAIL ? MEMCG_LRU_TAIL : MEMCG_LRU_YOUNG;
5364  
5365  	mem_cgroup_calculate_protection(NULL, memcg);
5366  
5367  	if (mem_cgroup_below_min(NULL, memcg))
5368  		return MEMCG_LRU_YOUNG;
5369  
5370  	if (mem_cgroup_below_low(NULL, memcg)) {
5371  		/* see the comment on MEMCG_NR_GENS */
5372  		if (seg != MEMCG_LRU_TAIL)
5373  			return MEMCG_LRU_TAIL;
5374  
5375  		memcg_memory_event(memcg, MEMCG_LOW);
5376  	}
5377  
5378  	success = try_to_shrink_lruvec(lruvec, sc);
5379  
5380  	shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, sc->priority);
5381  
5382  	if (!sc->proactive)
5383  		vmpressure(sc->gfp_mask, memcg, false, sc->nr_scanned - scanned,
5384  			   sc->nr_reclaimed - reclaimed);
5385  
5386  	flush_reclaim_state(sc);
5387  
5388  	return success ? MEMCG_LRU_YOUNG : 0;
5389  }
5390  
5391  #ifdef CONFIG_MEMCG
5392  
5393  static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
5394  {
5395  	int op;
5396  	int gen;
5397  	int bin;
5398  	int first_bin;
5399  	struct lruvec *lruvec;
5400  	struct lru_gen_folio *lrugen;
5401  	struct mem_cgroup *memcg;
5402  	const struct hlist_nulls_node *pos;
5403  	unsigned long nr_to_reclaim = get_nr_to_reclaim(sc);
5404  
5405  	bin = first_bin = get_random_u32_below(MEMCG_NR_BINS);
5406  restart:
5407  	op = 0;
5408  	memcg = NULL;
5409  	gen = get_memcg_gen(READ_ONCE(pgdat->memcg_lru.seq));
5410  
5411  	rcu_read_lock();
5412  
5413  	hlist_nulls_for_each_entry_rcu(lrugen, pos, &pgdat->memcg_lru.fifo[gen][bin], list) {
5414  		if (op)
5415  			lru_gen_rotate_memcg(lruvec, op);
5416  
5417  		mem_cgroup_put(memcg);
5418  
5419  		lruvec = container_of(lrugen, struct lruvec, lrugen);
5420  		memcg = lruvec_memcg(lruvec);
5421  
5422  		if (!mem_cgroup_tryget(memcg)) {
5423  			op = 0;
5424  			memcg = NULL;
5425  			continue;
5426  		}
5427  
5428  		rcu_read_unlock();
5429  
5430  		op = shrink_one(lruvec, sc);
5431  
5432  		rcu_read_lock();
5433  
5434  		if (sc->nr_reclaimed >= nr_to_reclaim)
5435  			break;
5436  	}
5437  
5438  	rcu_read_unlock();
5439  
5440  	if (op)
5441  		lru_gen_rotate_memcg(lruvec, op);
5442  
5443  	mem_cgroup_put(memcg);
5444  
5445  	if (sc->nr_reclaimed >= nr_to_reclaim)
5446  		return;
5447  
5448  	/* restart if raced with lru_gen_rotate_memcg() */
5449  	if (gen != get_nulls_value(pos))
5450  		goto restart;
5451  
5452  	/* try the rest of the bins of the current generation */
5453  	bin = get_memcg_bin(bin + 1);
5454  	if (bin != first_bin)
5455  		goto restart;
5456  }
5457  
5458  static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
5459  {
5460  	struct blk_plug plug;
5461  
5462  	VM_WARN_ON_ONCE(global_reclaim(sc));
5463  	VM_WARN_ON_ONCE(!sc->may_writepage || !sc->may_unmap);
5464  
5465  	lru_add_drain();
5466  
5467  	blk_start_plug(&plug);
5468  
5469  	set_mm_walk(NULL, sc->proactive);
5470  
5471  	if (try_to_shrink_lruvec(lruvec, sc))
5472  		lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG);
5473  
5474  	clear_mm_walk();
5475  
5476  	blk_finish_plug(&plug);
5477  }
5478  
5479  #else /* !CONFIG_MEMCG */
5480  
5481  static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
5482  {
5483  	BUILD_BUG();
5484  }
5485  
5486  static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
5487  {
5488  	BUILD_BUG();
5489  }
5490  
5491  #endif
5492  
5493  static void set_initial_priority(struct pglist_data *pgdat, struct scan_control *sc)
5494  {
5495  	int priority;
5496  	unsigned long reclaimable;
5497  	struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);
5498  
5499  	if (sc->priority != DEF_PRIORITY || sc->nr_to_reclaim < MIN_LRU_BATCH)
5500  		return;
5501  	/*
5502  	 * Determine the initial priority based on ((total / MEMCG_NR_GENS) >>
5503  	 * priority) * reclaimed_to_scanned_ratio = nr_to_reclaim, where the
5504  	 * estimated reclaimed_to_scanned_ratio = inactive / total.
5505  	 */
5506  	reclaimable = node_page_state(pgdat, NR_INACTIVE_FILE);
5507  	if (get_swappiness(lruvec, sc))
5508  		reclaimable += node_page_state(pgdat, NR_INACTIVE_ANON);
5509  
5510  	reclaimable /= MEMCG_NR_GENS;
5511  
5512  	/* round down reclaimable and round up sc->nr_to_reclaim */
5513  	priority = fls_long(reclaimable) - 1 - fls_long(sc->nr_to_reclaim - 1);
5514  
5515  	sc->priority = clamp(priority, 0, DEF_PRIORITY);
5516  }
5517  
5518  static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc)
5519  {
5520  	struct blk_plug plug;
5521  	unsigned long reclaimed = sc->nr_reclaimed;
5522  
5523  	VM_WARN_ON_ONCE(!global_reclaim(sc));
5524  
5525  	/*
5526  	 * Unmapped clean folios are already prioritized. Scanning for more of
5527  	 * them is likely futile and can cause high reclaim latency when there
5528  	 * is a large number of memcgs.
5529  	 */
5530  	if (!sc->may_writepage || !sc->may_unmap)
5531  		goto done;
5532  
5533  	lru_add_drain();
5534  
5535  	blk_start_plug(&plug);
5536  
5537  	set_mm_walk(pgdat, sc->proactive);
5538  
5539  	set_initial_priority(pgdat, sc);
5540  
5541  	if (current_is_kswapd())
5542  		sc->nr_reclaimed = 0;
5543  
5544  	if (mem_cgroup_disabled())
5545  		shrink_one(&pgdat->__lruvec, sc);
5546  	else
5547  		shrink_many(pgdat, sc);
5548  
5549  	if (current_is_kswapd())
5550  		sc->nr_reclaimed += reclaimed;
5551  
5552  	clear_mm_walk();
5553  
5554  	blk_finish_plug(&plug);
5555  done:
5556  	/* kswapd should never fail */
5557  	pgdat->kswapd_failures = 0;
5558  }
5559  
5560  /******************************************************************************
5561   *                          state change
5562   ******************************************************************************/
5563  
5564  static bool __maybe_unused state_is_valid(struct lruvec *lruvec)
5565  {
5566  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
5567  
5568  	if (lrugen->enabled) {
5569  		enum lru_list lru;
5570  
5571  		for_each_evictable_lru(lru) {
5572  			if (!list_empty(&lruvec->lists[lru]))
5573  				return false;
5574  		}
5575  	} else {
5576  		int gen, type, zone;
5577  
5578  		for_each_gen_type_zone(gen, type, zone) {
5579  			if (!list_empty(&lrugen->folios[gen][type][zone]))
5580  				return false;
5581  		}
5582  	}
5583  
5584  	return true;
5585  }
5586  
5587  static bool fill_evictable(struct lruvec *lruvec)
5588  {
5589  	enum lru_list lru;
5590  	int remaining = MAX_LRU_BATCH;
5591  
5592  	for_each_evictable_lru(lru) {
5593  		int type = is_file_lru(lru);
5594  		bool active = is_active_lru(lru);
5595  		struct list_head *head = &lruvec->lists[lru];
5596  
5597  		while (!list_empty(head)) {
5598  			bool success;
5599  			struct folio *folio = lru_to_folio(head);
5600  
5601  			VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
5602  			VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio);
5603  			VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
5604  			VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio);
5605  
5606  			lruvec_del_folio(lruvec, folio);
5607  			success = lru_gen_add_folio(lruvec, folio, false);
5608  			VM_WARN_ON_ONCE(!success);
5609  
5610  			if (!--remaining)
5611  				return false;
5612  		}
5613  	}
5614  
5615  	return true;
5616  }
5617  
5618  static bool drain_evictable(struct lruvec *lruvec)
5619  {
5620  	int gen, type, zone;
5621  	int remaining = MAX_LRU_BATCH;
5622  
5623  	for_each_gen_type_zone(gen, type, zone) {
5624  		struct list_head *head = &lruvec->lrugen.folios[gen][type][zone];
5625  
5626  		while (!list_empty(head)) {
5627  			bool success;
5628  			struct folio *folio = lru_to_folio(head);
5629  
5630  			VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
5631  			VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
5632  			VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
5633  			VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
5634  
5635  			success = lru_gen_del_folio(lruvec, folio, false);
5636  			VM_WARN_ON_ONCE(!success);
5637  			lruvec_add_folio(lruvec, folio);
5638  
5639  			if (!--remaining)
5640  				return false;
5641  		}
5642  	}
5643  
5644  	return true;
5645  }
5646  
5647  static void lru_gen_change_state(bool enabled)
5648  {
5649  	static DEFINE_MUTEX(state_mutex);
5650  
5651  	struct mem_cgroup *memcg;
5652  
5653  	cgroup_lock();
5654  	cpus_read_lock();
5655  	get_online_mems();
5656  	mutex_lock(&state_mutex);
5657  
5658  	if (enabled == lru_gen_enabled())
5659  		goto unlock;
5660  
5661  	if (enabled)
5662  		static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]);
5663  	else
5664  		static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]);
5665  
5666  	memcg = mem_cgroup_iter(NULL, NULL, NULL);
5667  	do {
5668  		int nid;
5669  
5670  		for_each_node(nid) {
5671  			struct lruvec *lruvec = get_lruvec(memcg, nid);
5672  
5673  			spin_lock_irq(&lruvec->lru_lock);
5674  
5675  			VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
5676  			VM_WARN_ON_ONCE(!state_is_valid(lruvec));
5677  
5678  			lruvec->lrugen.enabled = enabled;
5679  
5680  			while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) {
5681  				spin_unlock_irq(&lruvec->lru_lock);
5682  				cond_resched();
5683  				spin_lock_irq(&lruvec->lru_lock);
5684  			}
5685  
5686  			spin_unlock_irq(&lruvec->lru_lock);
5687  		}
5688  
5689  		cond_resched();
5690  	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
5691  unlock:
5692  	mutex_unlock(&state_mutex);
5693  	put_online_mems();
5694  	cpus_read_unlock();
5695  	cgroup_unlock();
5696  }
5697  
5698  /******************************************************************************
5699   *                          sysfs interface
5700   ******************************************************************************/
5701  
5702  static ssize_t min_ttl_ms_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
5703  {
5704  	return sysfs_emit(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl)));
5705  }
5706  
5707  /* see Documentation/admin-guide/mm/multigen_lru.rst for details */
5708  static ssize_t min_ttl_ms_store(struct kobject *kobj, struct kobj_attribute *attr,
5709  				const char *buf, size_t len)
5710  {
5711  	unsigned int msecs;
5712  
5713  	if (kstrtouint(buf, 0, &msecs))
5714  		return -EINVAL;
5715  
5716  	WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs));
5717  
5718  	return len;
5719  }
5720  
5721  static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR_RW(min_ttl_ms);
5722  
5723  static ssize_t enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
5724  {
5725  	unsigned int caps = 0;
5726  
5727  	if (get_cap(LRU_GEN_CORE))
5728  		caps |= BIT(LRU_GEN_CORE);
5729  
5730  	if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))
5731  		caps |= BIT(LRU_GEN_MM_WALK);
5732  
5733  	if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG))
5734  		caps |= BIT(LRU_GEN_NONLEAF_YOUNG);
5735  
5736  	return sysfs_emit(buf, "0x%04x\n", caps);
5737  }
5738  
5739  /* see Documentation/admin-guide/mm/multigen_lru.rst for details */
5740  static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
5741  			     const char *buf, size_t len)
5742  {
5743  	int i;
5744  	unsigned int caps;
5745  
5746  	if (tolower(*buf) == 'n')
5747  		caps = 0;
5748  	else if (tolower(*buf) == 'y')
5749  		caps = -1;
5750  	else if (kstrtouint(buf, 0, &caps))
5751  		return -EINVAL;
5752  
5753  	for (i = 0; i < NR_LRU_GEN_CAPS; i++) {
5754  		bool enabled = caps & BIT(i);
5755  
5756  		if (i == LRU_GEN_CORE)
5757  			lru_gen_change_state(enabled);
5758  		else if (enabled)
5759  			static_branch_enable(&lru_gen_caps[i]);
5760  		else
5761  			static_branch_disable(&lru_gen_caps[i]);
5762  	}
5763  
5764  	return len;
5765  }
5766  
5767  static struct kobj_attribute lru_gen_enabled_attr = __ATTR_RW(enabled);
5768  
5769  static struct attribute *lru_gen_attrs[] = {
5770  	&lru_gen_min_ttl_attr.attr,
5771  	&lru_gen_enabled_attr.attr,
5772  	NULL
5773  };
5774  
5775  static const struct attribute_group lru_gen_attr_group = {
5776  	.name = "lru_gen",
5777  	.attrs = lru_gen_attrs,
5778  };
5779  
5780  /******************************************************************************
5781   *                          debugfs interface
5782   ******************************************************************************/
5783  
5784  static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos)
5785  {
5786  	struct mem_cgroup *memcg;
5787  	loff_t nr_to_skip = *pos;
5788  
5789  	m->private = kvmalloc(PATH_MAX, GFP_KERNEL);
5790  	if (!m->private)
5791  		return ERR_PTR(-ENOMEM);
5792  
5793  	memcg = mem_cgroup_iter(NULL, NULL, NULL);
5794  	do {
5795  		int nid;
5796  
5797  		for_each_node_state(nid, N_MEMORY) {
5798  			if (!nr_to_skip--)
5799  				return get_lruvec(memcg, nid);
5800  		}
5801  	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
5802  
5803  	return NULL;
5804  }
5805  
5806  static void lru_gen_seq_stop(struct seq_file *m, void *v)
5807  {
5808  	if (!IS_ERR_OR_NULL(v))
5809  		mem_cgroup_iter_break(NULL, lruvec_memcg(v));
5810  
5811  	kvfree(m->private);
5812  	m->private = NULL;
5813  }
5814  
5815  static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos)
5816  {
5817  	int nid = lruvec_pgdat(v)->node_id;
5818  	struct mem_cgroup *memcg = lruvec_memcg(v);
5819  
5820  	++*pos;
5821  
5822  	nid = next_memory_node(nid);
5823  	if (nid == MAX_NUMNODES) {
5824  		memcg = mem_cgroup_iter(NULL, memcg, NULL);
5825  		if (!memcg)
5826  			return NULL;
5827  
5828  		nid = first_memory_node;
5829  	}
5830  
5831  	return get_lruvec(memcg, nid);
5832  }
5833  
5834  static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,
5835  				  unsigned long max_seq, unsigned long *min_seq,
5836  				  unsigned long seq)
5837  {
5838  	int i;
5839  	int type, tier;
5840  	int hist = lru_hist_from_seq(seq);
5841  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
5842  
5843  	for (tier = 0; tier < MAX_NR_TIERS; tier++) {
5844  		seq_printf(m, "            %10d", tier);
5845  		for (type = 0; type < ANON_AND_FILE; type++) {
5846  			const char *s = "   ";
5847  			unsigned long n[3] = {};
5848  
5849  			if (seq == max_seq) {
5850  				s = "RT ";
5851  				n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]);
5852  				n[1] = READ_ONCE(lrugen->avg_total[type][tier]);
5853  			} else if (seq == min_seq[type] || NR_HIST_GENS > 1) {
5854  				s = "rep";
5855  				n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]);
5856  				n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]);
5857  				if (tier)
5858  					n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]);
5859  			}
5860  
5861  			for (i = 0; i < 3; i++)
5862  				seq_printf(m, " %10lu%c", n[i], s[i]);
5863  		}
5864  		seq_putc(m, '\n');
5865  	}
5866  
5867  	seq_puts(m, "                      ");
5868  	for (i = 0; i < NR_MM_STATS; i++) {
5869  		const char *s = "      ";
5870  		unsigned long n = 0;
5871  
5872  		if (seq == max_seq && NR_HIST_GENS == 1) {
5873  			s = "LOYNFA";
5874  			n = READ_ONCE(lruvec->mm_state.stats[hist][i]);
5875  		} else if (seq != max_seq && NR_HIST_GENS > 1) {
5876  			s = "loynfa";
5877  			n = READ_ONCE(lruvec->mm_state.stats[hist][i]);
5878  		}
5879  
5880  		seq_printf(m, " %10lu%c", n, s[i]);
5881  	}
5882  	seq_putc(m, '\n');
5883  }
5884  
5885  /* see Documentation/admin-guide/mm/multigen_lru.rst for details */
5886  static int lru_gen_seq_show(struct seq_file *m, void *v)
5887  {
5888  	unsigned long seq;
5889  	bool full = !debugfs_real_fops(m->file)->write;
5890  	struct lruvec *lruvec = v;
5891  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
5892  	int nid = lruvec_pgdat(lruvec)->node_id;
5893  	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
5894  	DEFINE_MAX_SEQ(lruvec);
5895  	DEFINE_MIN_SEQ(lruvec);
5896  
5897  	if (nid == first_memory_node) {
5898  		const char *path = memcg ? m->private : "";
5899  
5900  #ifdef CONFIG_MEMCG
5901  		if (memcg)
5902  			cgroup_path(memcg->css.cgroup, m->private, PATH_MAX);
5903  #endif
5904  		seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path);
5905  	}
5906  
5907  	seq_printf(m, " node %5d\n", nid);
5908  
5909  	if (!full)
5910  		seq = min_seq[LRU_GEN_ANON];
5911  	else if (max_seq >= MAX_NR_GENS)
5912  		seq = max_seq - MAX_NR_GENS + 1;
5913  	else
5914  		seq = 0;
5915  
5916  	for (; seq <= max_seq; seq++) {
5917  		int type, zone;
5918  		int gen = lru_gen_from_seq(seq);
5919  		unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
5920  
5921  		seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth));
5922  
5923  		for (type = 0; type < ANON_AND_FILE; type++) {
5924  			unsigned long size = 0;
5925  			char mark = full && seq < min_seq[type] ? 'x' : ' ';
5926  
5927  			for (zone = 0; zone < MAX_NR_ZONES; zone++)
5928  				size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
5929  
5930  			seq_printf(m, " %10lu%c", size, mark);
5931  		}
5932  
5933  		seq_putc(m, '\n');
5934  
5935  		if (full)
5936  			lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq);
5937  	}
5938  
5939  	return 0;
5940  }
5941  
5942  static const struct seq_operations lru_gen_seq_ops = {
5943  	.start = lru_gen_seq_start,
5944  	.stop = lru_gen_seq_stop,
5945  	.next = lru_gen_seq_next,
5946  	.show = lru_gen_seq_show,
5947  };
5948  
5949  static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc,
5950  		     bool can_swap, bool force_scan)
5951  {
5952  	DEFINE_MAX_SEQ(lruvec);
5953  	DEFINE_MIN_SEQ(lruvec);
5954  
5955  	if (seq < max_seq)
5956  		return 0;
5957  
5958  	if (seq > max_seq)
5959  		return -EINVAL;
5960  
5961  	if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq)
5962  		return -ERANGE;
5963  
5964  	try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan);
5965  
5966  	return 0;
5967  }
5968  
5969  static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc,
5970  			int swappiness, unsigned long nr_to_reclaim)
5971  {
5972  	DEFINE_MAX_SEQ(lruvec);
5973  
5974  	if (seq + MIN_NR_GENS > max_seq)
5975  		return -EINVAL;
5976  
5977  	sc->nr_reclaimed = 0;
5978  
5979  	while (!signal_pending(current)) {
5980  		DEFINE_MIN_SEQ(lruvec);
5981  
5982  		if (seq < min_seq[!swappiness])
5983  			return 0;
5984  
5985  		if (sc->nr_reclaimed >= nr_to_reclaim)
5986  			return 0;
5987  
5988  		if (!evict_folios(lruvec, sc, swappiness))
5989  			return 0;
5990  
5991  		cond_resched();
5992  	}
5993  
5994  	return -EINTR;
5995  }
5996  
5997  static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq,
5998  		   struct scan_control *sc, int swappiness, unsigned long opt)
5999  {
6000  	struct lruvec *lruvec;
6001  	int err = -EINVAL;
6002  	struct mem_cgroup *memcg = NULL;
6003  
6004  	if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY))
6005  		return -EINVAL;
6006  
6007  	if (!mem_cgroup_disabled()) {
6008  		rcu_read_lock();
6009  
6010  		memcg = mem_cgroup_from_id(memcg_id);
6011  		if (!mem_cgroup_tryget(memcg))
6012  			memcg = NULL;
6013  
6014  		rcu_read_unlock();
6015  
6016  		if (!memcg)
6017  			return -EINVAL;
6018  	}
6019  
6020  	if (memcg_id != mem_cgroup_id(memcg))
6021  		goto done;
6022  
6023  	lruvec = get_lruvec(memcg, nid);
6024  
6025  	if (swappiness < 0)
6026  		swappiness = get_swappiness(lruvec, sc);
6027  	else if (swappiness > 200)
6028  		goto done;
6029  
6030  	switch (cmd) {
6031  	case '+':
6032  		err = run_aging(lruvec, seq, sc, swappiness, opt);
6033  		break;
6034  	case '-':
6035  		err = run_eviction(lruvec, seq, sc, swappiness, opt);
6036  		break;
6037  	}
6038  done:
6039  	mem_cgroup_put(memcg);
6040  
6041  	return err;
6042  }
6043  
6044  /* see Documentation/admin-guide/mm/multigen_lru.rst for details */
6045  static ssize_t lru_gen_seq_write(struct file *file, const char __user *src,
6046  				 size_t len, loff_t *pos)
6047  {
6048  	void *buf;
6049  	char *cur, *next;
6050  	unsigned int flags;
6051  	struct blk_plug plug;
6052  	int err = -EINVAL;
6053  	struct scan_control sc = {
6054  		.may_writepage = true,
6055  		.may_unmap = true,
6056  		.may_swap = true,
6057  		.reclaim_idx = MAX_NR_ZONES - 1,
6058  		.gfp_mask = GFP_KERNEL,
6059  	};
6060  
6061  	buf = kvmalloc(len + 1, GFP_KERNEL);
6062  	if (!buf)
6063  		return -ENOMEM;
6064  
6065  	if (copy_from_user(buf, src, len)) {
6066  		kvfree(buf);
6067  		return -EFAULT;
6068  	}
6069  
6070  	set_task_reclaim_state(current, &sc.reclaim_state);
6071  	flags = memalloc_noreclaim_save();
6072  	blk_start_plug(&plug);
6073  	if (!set_mm_walk(NULL, true)) {
6074  		err = -ENOMEM;
6075  		goto done;
6076  	}
6077  
6078  	next = buf;
6079  	next[len] = '\0';
6080  
6081  	while ((cur = strsep(&next, ",;\n"))) {
6082  		int n;
6083  		int end;
6084  		char cmd;
6085  		unsigned int memcg_id;
6086  		unsigned int nid;
6087  		unsigned long seq;
6088  		unsigned int swappiness = -1;
6089  		unsigned long opt = -1;
6090  
6091  		cur = skip_spaces(cur);
6092  		if (!*cur)
6093  			continue;
6094  
6095  		n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid,
6096  			   &seq, &end, &swappiness, &end, &opt, &end);
6097  		if (n < 4 || cur[end]) {
6098  			err = -EINVAL;
6099  			break;
6100  		}
6101  
6102  		err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt);
6103  		if (err)
6104  			break;
6105  	}
6106  done:
6107  	clear_mm_walk();
6108  	blk_finish_plug(&plug);
6109  	memalloc_noreclaim_restore(flags);
6110  	set_task_reclaim_state(current, NULL);
6111  
6112  	kvfree(buf);
6113  
6114  	return err ? : len;
6115  }
6116  
6117  static int lru_gen_seq_open(struct inode *inode, struct file *file)
6118  {
6119  	return seq_open(file, &lru_gen_seq_ops);
6120  }
6121  
6122  static const struct file_operations lru_gen_rw_fops = {
6123  	.open = lru_gen_seq_open,
6124  	.read = seq_read,
6125  	.write = lru_gen_seq_write,
6126  	.llseek = seq_lseek,
6127  	.release = seq_release,
6128  };
6129  
6130  static const struct file_operations lru_gen_ro_fops = {
6131  	.open = lru_gen_seq_open,
6132  	.read = seq_read,
6133  	.llseek = seq_lseek,
6134  	.release = seq_release,
6135  };
6136  
6137  /******************************************************************************
6138   *                          initialization
6139   ******************************************************************************/
6140  
6141  void lru_gen_init_lruvec(struct lruvec *lruvec)
6142  {
6143  	int i;
6144  	int gen, type, zone;
6145  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
6146  
6147  	lrugen->max_seq = MIN_NR_GENS + 1;
6148  	lrugen->enabled = lru_gen_enabled();
6149  
6150  	for (i = 0; i <= MIN_NR_GENS + 1; i++)
6151  		lrugen->timestamps[i] = jiffies;
6152  
6153  	for_each_gen_type_zone(gen, type, zone)
6154  		INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]);
6155  
6156  	lruvec->mm_state.seq = MIN_NR_GENS;
6157  }
6158  
6159  #ifdef CONFIG_MEMCG
6160  
6161  void lru_gen_init_pgdat(struct pglist_data *pgdat)
6162  {
6163  	int i, j;
6164  
6165  	spin_lock_init(&pgdat->memcg_lru.lock);
6166  
6167  	for (i = 0; i < MEMCG_NR_GENS; i++) {
6168  		for (j = 0; j < MEMCG_NR_BINS; j++)
6169  			INIT_HLIST_NULLS_HEAD(&pgdat->memcg_lru.fifo[i][j], i);
6170  	}
6171  }
6172  
6173  void lru_gen_init_memcg(struct mem_cgroup *memcg)
6174  {
6175  	INIT_LIST_HEAD(&memcg->mm_list.fifo);
6176  	spin_lock_init(&memcg->mm_list.lock);
6177  }
6178  
6179  void lru_gen_exit_memcg(struct mem_cgroup *memcg)
6180  {
6181  	int i;
6182  	int nid;
6183  
6184  	VM_WARN_ON_ONCE(!list_empty(&memcg->mm_list.fifo));
6185  
6186  	for_each_node(nid) {
6187  		struct lruvec *lruvec = get_lruvec(memcg, nid);
6188  
6189  		VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0,
6190  					   sizeof(lruvec->lrugen.nr_pages)));
6191  
6192  		lruvec->lrugen.list.next = LIST_POISON1;
6193  
6194  		for (i = 0; i < NR_BLOOM_FILTERS; i++) {
6195  			bitmap_free(lruvec->mm_state.filters[i]);
6196  			lruvec->mm_state.filters[i] = NULL;
6197  		}
6198  	}
6199  }
6200  
6201  #endif /* CONFIG_MEMCG */
6202  
6203  static int __init init_lru_gen(void)
6204  {
6205  	BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS);
6206  	BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS);
6207  
6208  	if (sysfs_create_group(mm_kobj, &lru_gen_attr_group))
6209  		pr_err("lru_gen: failed to create sysfs group\n");
6210  
6211  	debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops);
6212  	debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops);
6213  
6214  	return 0;
6215  };
6216  late_initcall(init_lru_gen);
6217  
6218  #else /* !CONFIG_LRU_GEN */
6219  
6220  static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
6221  {
6222  }
6223  
6224  static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
6225  {
6226  }
6227  
6228  static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc)
6229  {
6230  }
6231  
6232  #endif /* CONFIG_LRU_GEN */
6233  
6234  static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
6235  {
6236  	unsigned long nr[NR_LRU_LISTS];
6237  	unsigned long targets[NR_LRU_LISTS];
6238  	unsigned long nr_to_scan;
6239  	enum lru_list lru;
6240  	unsigned long nr_reclaimed = 0;
6241  	unsigned long nr_to_reclaim = sc->nr_to_reclaim;
6242  	bool proportional_reclaim;
6243  	struct blk_plug plug;
6244  
6245  	if (lru_gen_enabled() && !global_reclaim(sc)) {
6246  		lru_gen_shrink_lruvec(lruvec, sc);
6247  		return;
6248  	}
6249  
6250  	get_scan_count(lruvec, sc, nr);
6251  
6252  	/* Record the original scan target for proportional adjustments later */
6253  	memcpy(targets, nr, sizeof(nr));
6254  
6255  	/*
6256  	 * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal
6257  	 * event that can occur when there is little memory pressure e.g.
6258  	 * multiple streaming readers/writers. Hence, we do not abort scanning
6259  	 * when the requested number of pages are reclaimed when scanning at
6260  	 * DEF_PRIORITY on the assumption that the fact we are direct
6261  	 * reclaiming implies that kswapd is not keeping up and it is best to
6262  	 * do a batch of work at once. For memcg reclaim one check is made to
6263  	 * abort proportional reclaim if either the file or anon lru has already
6264  	 * dropped to zero at the first pass.
6265  	 */
6266  	proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() &&
6267  				sc->priority == DEF_PRIORITY);
6268  
6269  	blk_start_plug(&plug);
6270  	while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
6271  					nr[LRU_INACTIVE_FILE]) {
6272  		unsigned long nr_anon, nr_file, percentage;
6273  		unsigned long nr_scanned;
6274  
6275  		for_each_evictable_lru(lru) {
6276  			if (nr[lru]) {
6277  				nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
6278  				nr[lru] -= nr_to_scan;
6279  
6280  				nr_reclaimed += shrink_list(lru, nr_to_scan,
6281  							    lruvec, sc);
6282  			}
6283  		}
6284  
6285  		cond_resched();
6286  
6287  		if (nr_reclaimed < nr_to_reclaim || proportional_reclaim)
6288  			continue;
6289  
6290  		/*
6291  		 * For kswapd and memcg, reclaim at least the number of pages
6292  		 * requested. Ensure that the anon and file LRUs are scanned
6293  		 * proportionally what was requested by get_scan_count(). We
6294  		 * stop reclaiming one LRU and reduce the amount scanning
6295  		 * proportional to the original scan target.
6296  		 */
6297  		nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
6298  		nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];
6299  
6300  		/*
6301  		 * It's just vindictive to attack the larger once the smaller
6302  		 * has gone to zero.  And given the way we stop scanning the
6303  		 * smaller below, this makes sure that we only make one nudge
6304  		 * towards proportionality once we've got nr_to_reclaim.
6305  		 */
6306  		if (!nr_file || !nr_anon)
6307  			break;
6308  
6309  		if (nr_file > nr_anon) {
6310  			unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
6311  						targets[LRU_ACTIVE_ANON] + 1;
6312  			lru = LRU_BASE;
6313  			percentage = nr_anon * 100 / scan_target;
6314  		} else {
6315  			unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
6316  						targets[LRU_ACTIVE_FILE] + 1;
6317  			lru = LRU_FILE;
6318  			percentage = nr_file * 100 / scan_target;
6319  		}
6320  
6321  		/* Stop scanning the smaller of the LRU */
6322  		nr[lru] = 0;
6323  		nr[lru + LRU_ACTIVE] = 0;
6324  
6325  		/*
6326  		 * Recalculate the other LRU scan count based on its original
6327  		 * scan target and the percentage scanning already complete
6328  		 */
6329  		lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
6330  		nr_scanned = targets[lru] - nr[lru];
6331  		nr[lru] = targets[lru] * (100 - percentage) / 100;
6332  		nr[lru] -= min(nr[lru], nr_scanned);
6333  
6334  		lru += LRU_ACTIVE;
6335  		nr_scanned = targets[lru] - nr[lru];
6336  		nr[lru] = targets[lru] * (100 - percentage) / 100;
6337  		nr[lru] -= min(nr[lru], nr_scanned);
6338  	}
6339  	blk_finish_plug(&plug);
6340  	sc->nr_reclaimed += nr_reclaimed;
6341  
6342  	/*
6343  	 * Even if we did not try to evict anon pages at all, we want to
6344  	 * rebalance the anon lru active/inactive ratio.
6345  	 */
6346  	if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) &&
6347  	    inactive_is_low(lruvec, LRU_INACTIVE_ANON))
6348  		shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
6349  				   sc, LRU_ACTIVE_ANON);
6350  }
6351  
6352  /* Use reclaim/compaction for costly allocs or under memory pressure */
6353  static bool in_reclaim_compaction(struct scan_control *sc)
6354  {
6355  	if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
6356  			(sc->order > PAGE_ALLOC_COSTLY_ORDER ||
6357  			 sc->priority < DEF_PRIORITY - 2))
6358  		return true;
6359  
6360  	return false;
6361  }
6362  
6363  /*
6364   * Reclaim/compaction is used for high-order allocation requests. It reclaims
6365   * order-0 pages before compacting the zone. should_continue_reclaim() returns
6366   * true if more pages should be reclaimed such that when the page allocator
6367   * calls try_to_compact_pages() that it will have enough free pages to succeed.
6368   * It will give up earlier than that if there is difficulty reclaiming pages.
6369   */
6370  static inline bool should_continue_reclaim(struct pglist_data *pgdat,
6371  					unsigned long nr_reclaimed,
6372  					struct scan_control *sc)
6373  {
6374  	unsigned long pages_for_compaction;
6375  	unsigned long inactive_lru_pages;
6376  	int z;
6377  
6378  	/* If not in reclaim/compaction mode, stop */
6379  	if (!in_reclaim_compaction(sc))
6380  		return false;
6381  
6382  	/*
6383  	 * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX
6384  	 * number of pages that were scanned. This will return to the caller
6385  	 * with the risk reclaim/compaction and the resulting allocation attempt
6386  	 * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL
6387  	 * allocations through requiring that the full LRU list has been scanned
6388  	 * first, by assuming that zero delta of sc->nr_scanned means full LRU
6389  	 * scan, but that approximation was wrong, and there were corner cases
6390  	 * where always a non-zero amount of pages were scanned.
6391  	 */
6392  	if (!nr_reclaimed)
6393  		return false;
6394  
6395  	/* If compaction would go ahead or the allocation would succeed, stop */
6396  	for (z = 0; z <= sc->reclaim_idx; z++) {
6397  		struct zone *zone = &pgdat->node_zones[z];
6398  		if (!managed_zone(zone))
6399  			continue;
6400  
6401  		switch (compaction_suitable(zone, sc->order, 0, sc->reclaim_idx)) {
6402  		case COMPACT_SUCCESS:
6403  		case COMPACT_CONTINUE:
6404  			return false;
6405  		default:
6406  			/* check next zone */
6407  			;
6408  		}
6409  	}
6410  
6411  	/*
6412  	 * If we have not reclaimed enough pages for compaction and the
6413  	 * inactive lists are large enough, continue reclaiming
6414  	 */
6415  	pages_for_compaction = compact_gap(sc->order);
6416  	inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
6417  	if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc))
6418  		inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
6419  
6420  	return inactive_lru_pages > pages_for_compaction;
6421  }
6422  
6423  static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)
6424  {
6425  	struct mem_cgroup *target_memcg = sc->target_mem_cgroup;
6426  	struct mem_cgroup *memcg;
6427  
6428  	memcg = mem_cgroup_iter(target_memcg, NULL, NULL);
6429  	do {
6430  		struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
6431  		unsigned long reclaimed;
6432  		unsigned long scanned;
6433  
6434  		/*
6435  		 * This loop can become CPU-bound when target memcgs
6436  		 * aren't eligible for reclaim - either because they
6437  		 * don't have any reclaimable pages, or because their
6438  		 * memory is explicitly protected. Avoid soft lockups.
6439  		 */
6440  		cond_resched();
6441  
6442  		mem_cgroup_calculate_protection(target_memcg, memcg);
6443  
6444  		if (mem_cgroup_below_min(target_memcg, memcg)) {
6445  			/*
6446  			 * Hard protection.
6447  			 * If there is no reclaimable memory, OOM.
6448  			 */
6449  			continue;
6450  		} else if (mem_cgroup_below_low(target_memcg, memcg)) {
6451  			/*
6452  			 * Soft protection.
6453  			 * Respect the protection only as long as
6454  			 * there is an unprotected supply
6455  			 * of reclaimable memory from other cgroups.
6456  			 */
6457  			if (!sc->memcg_low_reclaim) {
6458  				sc->memcg_low_skipped = 1;
6459  				continue;
6460  			}
6461  			memcg_memory_event(memcg, MEMCG_LOW);
6462  		}
6463  
6464  		reclaimed = sc->nr_reclaimed;
6465  		scanned = sc->nr_scanned;
6466  
6467  		shrink_lruvec(lruvec, sc);
6468  
6469  		shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,
6470  			    sc->priority);
6471  
6472  		/* Record the group's reclaim efficiency */
6473  		if (!sc->proactive)
6474  			vmpressure(sc->gfp_mask, memcg, false,
6475  				   sc->nr_scanned - scanned,
6476  				   sc->nr_reclaimed - reclaimed);
6477  
6478  	} while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL)));
6479  }
6480  
6481  static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)
6482  {
6483  	unsigned long nr_reclaimed, nr_scanned, nr_node_reclaimed;
6484  	struct lruvec *target_lruvec;
6485  	bool reclaimable = false;
6486  
6487  	if (lru_gen_enabled() && global_reclaim(sc)) {
6488  		lru_gen_shrink_node(pgdat, sc);
6489  		return;
6490  	}
6491  
6492  	target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
6493  
6494  again:
6495  	memset(&sc->nr, 0, sizeof(sc->nr));
6496  
6497  	nr_reclaimed = sc->nr_reclaimed;
6498  	nr_scanned = sc->nr_scanned;
6499  
6500  	prepare_scan_count(pgdat, sc);
6501  
6502  	shrink_node_memcgs(pgdat, sc);
6503  
6504  	flush_reclaim_state(sc);
6505  
6506  	nr_node_reclaimed = sc->nr_reclaimed - nr_reclaimed;
6507  
6508  	/* Record the subtree's reclaim efficiency */
6509  	if (!sc->proactive)
6510  		vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
6511  			   sc->nr_scanned - nr_scanned, nr_node_reclaimed);
6512  
6513  	if (nr_node_reclaimed)
6514  		reclaimable = true;
6515  
6516  	if (current_is_kswapd()) {
6517  		/*
6518  		 * If reclaim is isolating dirty pages under writeback,
6519  		 * it implies that the long-lived page allocation rate
6520  		 * is exceeding the page laundering rate. Either the
6521  		 * global limits are not being effective at throttling
6522  		 * processes due to the page distribution throughout
6523  		 * zones or there is heavy usage of a slow backing
6524  		 * device. The only option is to throttle from reclaim
6525  		 * context which is not ideal as there is no guarantee
6526  		 * the dirtying process is throttled in the same way
6527  		 * balance_dirty_pages() manages.
6528  		 *
6529  		 * Once a node is flagged PGDAT_WRITEBACK, kswapd will
6530  		 * count the number of pages under pages flagged for
6531  		 * immediate reclaim and stall if any are encountered
6532  		 * in the nr_immediate check below.
6533  		 */
6534  		if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
6535  			set_bit(PGDAT_WRITEBACK, &pgdat->flags);
6536  
6537  		/* Allow kswapd to start writing pages during reclaim.*/
6538  		if (sc->nr.unqueued_dirty == sc->nr.file_taken)
6539  			set_bit(PGDAT_DIRTY, &pgdat->flags);
6540  
6541  		/*
6542  		 * If kswapd scans pages marked for immediate
6543  		 * reclaim and under writeback (nr_immediate), it
6544  		 * implies that pages are cycling through the LRU
6545  		 * faster than they are written so forcibly stall
6546  		 * until some pages complete writeback.
6547  		 */
6548  		if (sc->nr.immediate)
6549  			reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK);
6550  	}
6551  
6552  	/*
6553  	 * Tag a node/memcg as congested if all the dirty pages were marked
6554  	 * for writeback and immediate reclaim (counted in nr.congested).
6555  	 *
6556  	 * Legacy memcg will stall in page writeback so avoid forcibly
6557  	 * stalling in reclaim_throttle().
6558  	 */
6559  	if ((current_is_kswapd() ||
6560  	     (cgroup_reclaim(sc) && writeback_throttling_sane(sc))) &&
6561  	    sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
6562  		set_bit(LRUVEC_CONGESTED, &target_lruvec->flags);
6563  
6564  	/*
6565  	 * Stall direct reclaim for IO completions if the lruvec is
6566  	 * node is congested. Allow kswapd to continue until it
6567  	 * starts encountering unqueued dirty pages or cycling through
6568  	 * the LRU too quickly.
6569  	 */
6570  	if (!current_is_kswapd() && current_may_throttle() &&
6571  	    !sc->hibernation_mode &&
6572  	    test_bit(LRUVEC_CONGESTED, &target_lruvec->flags))
6573  		reclaim_throttle(pgdat, VMSCAN_THROTTLE_CONGESTED);
6574  
6575  	if (should_continue_reclaim(pgdat, nr_node_reclaimed, sc))
6576  		goto again;
6577  
6578  	/*
6579  	 * Kswapd gives up on balancing particular nodes after too
6580  	 * many failures to reclaim anything from them and goes to
6581  	 * sleep. On reclaim progress, reset the failure counter. A
6582  	 * successful direct reclaim run will revive a dormant kswapd.
6583  	 */
6584  	if (reclaimable)
6585  		pgdat->kswapd_failures = 0;
6586  }
6587  
6588  /*
6589   * Returns true if compaction should go ahead for a costly-order request, or
6590   * the allocation would already succeed without compaction. Return false if we
6591   * should reclaim first.
6592   */
6593  static inline bool compaction_ready(struct zone *zone, struct scan_control *sc)
6594  {
6595  	unsigned long watermark;
6596  	enum compact_result suitable;
6597  
6598  	suitable = compaction_suitable(zone, sc->order, 0, sc->reclaim_idx);
6599  	if (suitable == COMPACT_SUCCESS)
6600  		/* Allocation should succeed already. Don't reclaim. */
6601  		return true;
6602  	if (suitable == COMPACT_SKIPPED)
6603  		/* Compaction cannot yet proceed. Do reclaim. */
6604  		return false;
6605  
6606  	/*
6607  	 * Compaction is already possible, but it takes time to run and there
6608  	 * are potentially other callers using the pages just freed. So proceed
6609  	 * with reclaim to make a buffer of free pages available to give
6610  	 * compaction a reasonable chance of completing and allocating the page.
6611  	 * Note that we won't actually reclaim the whole buffer in one attempt
6612  	 * as the target watermark in should_continue_reclaim() is lower. But if
6613  	 * we are already above the high+gap watermark, don't reclaim at all.
6614  	 */
6615  	watermark = high_wmark_pages(zone) + compact_gap(sc->order);
6616  
6617  	return zone_watermark_ok_safe(zone, 0, watermark, sc->reclaim_idx);
6618  }
6619  
6620  static void consider_reclaim_throttle(pg_data_t *pgdat, struct scan_control *sc)
6621  {
6622  	/*
6623  	 * If reclaim is making progress greater than 12% efficiency then
6624  	 * wake all the NOPROGRESS throttled tasks.
6625  	 */
6626  	if (sc->nr_reclaimed > (sc->nr_scanned >> 3)) {
6627  		wait_queue_head_t *wqh;
6628  
6629  		wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_NOPROGRESS];
6630  		if (waitqueue_active(wqh))
6631  			wake_up(wqh);
6632  
6633  		return;
6634  	}
6635  
6636  	/*
6637  	 * Do not throttle kswapd or cgroup reclaim on NOPROGRESS as it will
6638  	 * throttle on VMSCAN_THROTTLE_WRITEBACK if there are too many pages
6639  	 * under writeback and marked for immediate reclaim at the tail of the
6640  	 * LRU.
6641  	 */
6642  	if (current_is_kswapd() || cgroup_reclaim(sc))
6643  		return;
6644  
6645  	/* Throttle if making no progress at high prioities. */
6646  	if (sc->priority == 1 && !sc->nr_reclaimed)
6647  		reclaim_throttle(pgdat, VMSCAN_THROTTLE_NOPROGRESS);
6648  }
6649  
6650  /*
6651   * This is the direct reclaim path, for page-allocating processes.  We only
6652   * try to reclaim pages from zones which will satisfy the caller's allocation
6653   * request.
6654   *
6655   * If a zone is deemed to be full of pinned pages then just give it a light
6656   * scan then give up on it.
6657   */
6658  static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
6659  {
6660  	struct zoneref *z;
6661  	struct zone *zone;
6662  	unsigned long nr_soft_reclaimed;
6663  	unsigned long nr_soft_scanned;
6664  	gfp_t orig_mask;
6665  	pg_data_t *last_pgdat = NULL;
6666  	pg_data_t *first_pgdat = NULL;
6667  
6668  	/*
6669  	 * If the number of buffer_heads in the machine exceeds the maximum
6670  	 * allowed level, force direct reclaim to scan the highmem zone as
6671  	 * highmem pages could be pinning lowmem pages storing buffer_heads
6672  	 */
6673  	orig_mask = sc->gfp_mask;
6674  	if (buffer_heads_over_limit) {
6675  		sc->gfp_mask |= __GFP_HIGHMEM;
6676  		sc->reclaim_idx = gfp_zone(sc->gfp_mask);
6677  	}
6678  
6679  	for_each_zone_zonelist_nodemask(zone, z, zonelist,
6680  					sc->reclaim_idx, sc->nodemask) {
6681  		/*
6682  		 * Take care memory controller reclaiming has small influence
6683  		 * to global LRU.
6684  		 */
6685  		if (!cgroup_reclaim(sc)) {
6686  			if (!cpuset_zone_allowed(zone,
6687  						 GFP_KERNEL | __GFP_HARDWALL))
6688  				continue;
6689  
6690  			/*
6691  			 * If we already have plenty of memory free for
6692  			 * compaction in this zone, don't free any more.
6693  			 * Even though compaction is invoked for any
6694  			 * non-zero order, only frequent costly order
6695  			 * reclamation is disruptive enough to become a
6696  			 * noticeable problem, like transparent huge
6697  			 * page allocations.
6698  			 */
6699  			if (IS_ENABLED(CONFIG_COMPACTION) &&
6700  			    sc->order > PAGE_ALLOC_COSTLY_ORDER &&
6701  			    compaction_ready(zone, sc)) {
6702  				sc->compaction_ready = true;
6703  				continue;
6704  			}
6705  
6706  			/*
6707  			 * Shrink each node in the zonelist once. If the
6708  			 * zonelist is ordered by zone (not the default) then a
6709  			 * node may be shrunk multiple times but in that case
6710  			 * the user prefers lower zones being preserved.
6711  			 */
6712  			if (zone->zone_pgdat == last_pgdat)
6713  				continue;
6714  
6715  			/*
6716  			 * This steals pages from memory cgroups over softlimit
6717  			 * and returns the number of reclaimed pages and
6718  			 * scanned pages. This works for global memory pressure
6719  			 * and balancing, not for a memcg's limit.
6720  			 */
6721  			nr_soft_scanned = 0;
6722  			nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone->zone_pgdat,
6723  						sc->order, sc->gfp_mask,
6724  						&nr_soft_scanned);
6725  			sc->nr_reclaimed += nr_soft_reclaimed;
6726  			sc->nr_scanned += nr_soft_scanned;
6727  			/* need some check for avoid more shrink_zone() */
6728  		}
6729  
6730  		if (!first_pgdat)
6731  			first_pgdat = zone->zone_pgdat;
6732  
6733  		/* See comment about same check for global reclaim above */
6734  		if (zone->zone_pgdat == last_pgdat)
6735  			continue;
6736  		last_pgdat = zone->zone_pgdat;
6737  		shrink_node(zone->zone_pgdat, sc);
6738  	}
6739  
6740  	if (first_pgdat)
6741  		consider_reclaim_throttle(first_pgdat, sc);
6742  
6743  	/*
6744  	 * Restore to original mask to avoid the impact on the caller if we
6745  	 * promoted it to __GFP_HIGHMEM.
6746  	 */
6747  	sc->gfp_mask = orig_mask;
6748  }
6749  
6750  static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat)
6751  {
6752  	struct lruvec *target_lruvec;
6753  	unsigned long refaults;
6754  
6755  	if (lru_gen_enabled())
6756  		return;
6757  
6758  	target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
6759  	refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON);
6760  	target_lruvec->refaults[WORKINGSET_ANON] = refaults;
6761  	refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE);
6762  	target_lruvec->refaults[WORKINGSET_FILE] = refaults;
6763  }
6764  
6765  /*
6766   * This is the main entry point to direct page reclaim.
6767   *
6768   * If a full scan of the inactive list fails to free enough memory then we
6769   * are "out of memory" and something needs to be killed.
6770   *
6771   * If the caller is !__GFP_FS then the probability of a failure is reasonably
6772   * high - the zone may be full of dirty or under-writeback pages, which this
6773   * caller can't do much about.  We kick the writeback threads and take explicit
6774   * naps in the hope that some of these pages can be written.  But if the
6775   * allocating task holds filesystem locks which prevent writeout this might not
6776   * work, and the allocation attempt will fail.
6777   *
6778   * returns:	0, if no pages reclaimed
6779   * 		else, the number of pages reclaimed
6780   */
6781  static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
6782  					  struct scan_control *sc)
6783  {
6784  	int initial_priority = sc->priority;
6785  	pg_data_t *last_pgdat;
6786  	struct zoneref *z;
6787  	struct zone *zone;
6788  retry:
6789  	delayacct_freepages_start();
6790  
6791  	if (!cgroup_reclaim(sc))
6792  		__count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);
6793  
6794  	do {
6795  		if (!sc->proactive)
6796  			vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup,
6797  					sc->priority);
6798  		sc->nr_scanned = 0;
6799  		shrink_zones(zonelist, sc);
6800  
6801  		if (sc->nr_reclaimed >= sc->nr_to_reclaim)
6802  			break;
6803  
6804  		if (sc->compaction_ready)
6805  			break;
6806  
6807  		/*
6808  		 * If we're getting trouble reclaiming, start doing
6809  		 * writepage even in laptop mode.
6810  		 */
6811  		if (sc->priority < DEF_PRIORITY - 2)
6812  			sc->may_writepage = 1;
6813  	} while (--sc->priority >= 0);
6814  
6815  	last_pgdat = NULL;
6816  	for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx,
6817  					sc->nodemask) {
6818  		if (zone->zone_pgdat == last_pgdat)
6819  			continue;
6820  		last_pgdat = zone->zone_pgdat;
6821  
6822  		snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat);
6823  
6824  		if (cgroup_reclaim(sc)) {
6825  			struct lruvec *lruvec;
6826  
6827  			lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup,
6828  						   zone->zone_pgdat);
6829  			clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
6830  		}
6831  	}
6832  
6833  	delayacct_freepages_end();
6834  
6835  	if (sc->nr_reclaimed)
6836  		return sc->nr_reclaimed;
6837  
6838  	/* Aborted reclaim to try compaction? don't OOM, then */
6839  	if (sc->compaction_ready)
6840  		return 1;
6841  
6842  	/*
6843  	 * We make inactive:active ratio decisions based on the node's
6844  	 * composition of memory, but a restrictive reclaim_idx or a
6845  	 * memory.low cgroup setting can exempt large amounts of
6846  	 * memory from reclaim. Neither of which are very common, so
6847  	 * instead of doing costly eligibility calculations of the
6848  	 * entire cgroup subtree up front, we assume the estimates are
6849  	 * good, and retry with forcible deactivation if that fails.
6850  	 */
6851  	if (sc->skipped_deactivate) {
6852  		sc->priority = initial_priority;
6853  		sc->force_deactivate = 1;
6854  		sc->skipped_deactivate = 0;
6855  		goto retry;
6856  	}
6857  
6858  	/* Untapped cgroup reserves?  Don't OOM, retry. */
6859  	if (sc->memcg_low_skipped) {
6860  		sc->priority = initial_priority;
6861  		sc->force_deactivate = 0;
6862  		sc->memcg_low_reclaim = 1;
6863  		sc->memcg_low_skipped = 0;
6864  		goto retry;
6865  	}
6866  
6867  	return 0;
6868  }
6869  
6870  static bool allow_direct_reclaim(pg_data_t *pgdat)
6871  {
6872  	struct zone *zone;
6873  	unsigned long pfmemalloc_reserve = 0;
6874  	unsigned long free_pages = 0;
6875  	int i;
6876  	bool wmark_ok;
6877  
6878  	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
6879  		return true;
6880  
6881  	for (i = 0; i <= ZONE_NORMAL; i++) {
6882  		zone = &pgdat->node_zones[i];
6883  		if (!managed_zone(zone))
6884  			continue;
6885  
6886  		if (!zone_reclaimable_pages(zone))
6887  			continue;
6888  
6889  		pfmemalloc_reserve += min_wmark_pages(zone);
6890  		free_pages += zone_page_state(zone, NR_FREE_PAGES);
6891  	}
6892  
6893  	/* If there are no reserves (unexpected config) then do not throttle */
6894  	if (!pfmemalloc_reserve)
6895  		return true;
6896  
6897  	wmark_ok = free_pages > pfmemalloc_reserve / 2;
6898  
6899  	/* kswapd must be awake if processes are being throttled */
6900  	if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
6901  		if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL)
6902  			WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL);
6903  
6904  		wake_up_interruptible(&pgdat->kswapd_wait);
6905  	}
6906  
6907  	return wmark_ok;
6908  }
6909  
6910  /*
6911   * Throttle direct reclaimers if backing storage is backed by the network
6912   * and the PFMEMALLOC reserve for the preferred node is getting dangerously
6913   * depleted. kswapd will continue to make progress and wake the processes
6914   * when the low watermark is reached.
6915   *
6916   * Returns true if a fatal signal was delivered during throttling. If this
6917   * happens, the page allocator should not consider triggering the OOM killer.
6918   */
6919  static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
6920  					nodemask_t *nodemask)
6921  {
6922  	struct zoneref *z;
6923  	struct zone *zone;
6924  	pg_data_t *pgdat = NULL;
6925  
6926  	/*
6927  	 * Kernel threads should not be throttled as they may be indirectly
6928  	 * responsible for cleaning pages necessary for reclaim to make forward
6929  	 * progress. kjournald for example may enter direct reclaim while
6930  	 * committing a transaction where throttling it could forcing other
6931  	 * processes to block on log_wait_commit().
6932  	 */
6933  	if (current->flags & PF_KTHREAD)
6934  		goto out;
6935  
6936  	/*
6937  	 * If a fatal signal is pending, this process should not throttle.
6938  	 * It should return quickly so it can exit and free its memory
6939  	 */
6940  	if (fatal_signal_pending(current))
6941  		goto out;
6942  
6943  	/*
6944  	 * Check if the pfmemalloc reserves are ok by finding the first node
6945  	 * with a usable ZONE_NORMAL or lower zone. The expectation is that
6946  	 * GFP_KERNEL will be required for allocating network buffers when
6947  	 * swapping over the network so ZONE_HIGHMEM is unusable.
6948  	 *
6949  	 * Throttling is based on the first usable node and throttled processes
6950  	 * wait on a queue until kswapd makes progress and wakes them. There
6951  	 * is an affinity then between processes waking up and where reclaim
6952  	 * progress has been made assuming the process wakes on the same node.
6953  	 * More importantly, processes running on remote nodes will not compete
6954  	 * for remote pfmemalloc reserves and processes on different nodes
6955  	 * should make reasonable progress.
6956  	 */
6957  	for_each_zone_zonelist_nodemask(zone, z, zonelist,
6958  					gfp_zone(gfp_mask), nodemask) {
6959  		if (zone_idx(zone) > ZONE_NORMAL)
6960  			continue;
6961  
6962  		/* Throttle based on the first usable node */
6963  		pgdat = zone->zone_pgdat;
6964  		if (allow_direct_reclaim(pgdat))
6965  			goto out;
6966  		break;
6967  	}
6968  
6969  	/* If no zone was usable by the allocation flags then do not throttle */
6970  	if (!pgdat)
6971  		goto out;
6972  
6973  	/* Account for the throttling */
6974  	count_vm_event(PGSCAN_DIRECT_THROTTLE);
6975  
6976  	/*
6977  	 * If the caller cannot enter the filesystem, it's possible that it
6978  	 * is due to the caller holding an FS lock or performing a journal
6979  	 * transaction in the case of a filesystem like ext[3|4]. In this case,
6980  	 * it is not safe to block on pfmemalloc_wait as kswapd could be
6981  	 * blocked waiting on the same lock. Instead, throttle for up to a
6982  	 * second before continuing.
6983  	 */
6984  	if (!(gfp_mask & __GFP_FS))
6985  		wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
6986  			allow_direct_reclaim(pgdat), HZ);
6987  	else
6988  		/* Throttle until kswapd wakes the process */
6989  		wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
6990  			allow_direct_reclaim(pgdat));
6991  
6992  	if (fatal_signal_pending(current))
6993  		return true;
6994  
6995  out:
6996  	return false;
6997  }
6998  
6999  unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
7000  				gfp_t gfp_mask, nodemask_t *nodemask)
7001  {
7002  	unsigned long nr_reclaimed;
7003  	struct scan_control sc = {
7004  		.nr_to_reclaim = SWAP_CLUSTER_MAX,
7005  		.gfp_mask = current_gfp_context(gfp_mask),
7006  		.reclaim_idx = gfp_zone(gfp_mask),
7007  		.order = order,
7008  		.nodemask = nodemask,
7009  		.priority = DEF_PRIORITY,
7010  		.may_writepage = !laptop_mode,
7011  		.may_unmap = 1,
7012  		.may_swap = 1,
7013  	};
7014  
7015  	/*
7016  	 * scan_control uses s8 fields for order, priority, and reclaim_idx.
7017  	 * Confirm they are large enough for max values.
7018  	 */
7019  	BUILD_BUG_ON(MAX_ORDER >= S8_MAX);
7020  	BUILD_BUG_ON(DEF_PRIORITY > S8_MAX);
7021  	BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX);
7022  
7023  	/*
7024  	 * Do not enter reclaim if fatal signal was delivered while throttled.
7025  	 * 1 is returned so that the page allocator does not OOM kill at this
7026  	 * point.
7027  	 */
7028  	if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask))
7029  		return 1;
7030  
7031  	set_task_reclaim_state(current, &sc.reclaim_state);
7032  	trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask);
7033  
7034  	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
7035  
7036  	trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
7037  	set_task_reclaim_state(current, NULL);
7038  
7039  	return nr_reclaimed;
7040  }
7041  
7042  #ifdef CONFIG_MEMCG
7043  
7044  /* Only used by soft limit reclaim. Do not reuse for anything else. */
7045  unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
7046  						gfp_t gfp_mask, bool noswap,
7047  						pg_data_t *pgdat,
7048  						unsigned long *nr_scanned)
7049  {
7050  	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
7051  	struct scan_control sc = {
7052  		.nr_to_reclaim = SWAP_CLUSTER_MAX,
7053  		.target_mem_cgroup = memcg,
7054  		.may_writepage = !laptop_mode,
7055  		.may_unmap = 1,
7056  		.reclaim_idx = MAX_NR_ZONES - 1,
7057  		.may_swap = !noswap,
7058  	};
7059  
7060  	WARN_ON_ONCE(!current->reclaim_state);
7061  
7062  	sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
7063  			(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
7064  
7065  	trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order,
7066  						      sc.gfp_mask);
7067  
7068  	/*
7069  	 * NOTE: Although we can get the priority field, using it
7070  	 * here is not a good idea, since it limits the pages we can scan.
7071  	 * if we don't reclaim here, the shrink_node from balance_pgdat
7072  	 * will pick up pages from other mem cgroup's as well. We hack
7073  	 * the priority and make it zero.
7074  	 */
7075  	shrink_lruvec(lruvec, &sc);
7076  
7077  	trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
7078  
7079  	*nr_scanned = sc.nr_scanned;
7080  
7081  	return sc.nr_reclaimed;
7082  }
7083  
7084  unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
7085  					   unsigned long nr_pages,
7086  					   gfp_t gfp_mask,
7087  					   unsigned int reclaim_options)
7088  {
7089  	unsigned long nr_reclaimed;
7090  	unsigned int noreclaim_flag;
7091  	struct scan_control sc = {
7092  		.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
7093  		.gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) |
7094  				(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK),
7095  		.reclaim_idx = MAX_NR_ZONES - 1,
7096  		.target_mem_cgroup = memcg,
7097  		.priority = DEF_PRIORITY,
7098  		.may_writepage = !laptop_mode,
7099  		.may_unmap = 1,
7100  		.may_swap = !!(reclaim_options & MEMCG_RECLAIM_MAY_SWAP),
7101  		.proactive = !!(reclaim_options & MEMCG_RECLAIM_PROACTIVE),
7102  	};
7103  	/*
7104  	 * Traverse the ZONELIST_FALLBACK zonelist of the current node to put
7105  	 * equal pressure on all the nodes. This is based on the assumption that
7106  	 * the reclaim does not bail out early.
7107  	 */
7108  	struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
7109  
7110  	set_task_reclaim_state(current, &sc.reclaim_state);
7111  	trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask);
7112  	noreclaim_flag = memalloc_noreclaim_save();
7113  
7114  	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
7115  
7116  	memalloc_noreclaim_restore(noreclaim_flag);
7117  	trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
7118  	set_task_reclaim_state(current, NULL);
7119  
7120  	return nr_reclaimed;
7121  }
7122  #endif
7123  
7124  static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc)
7125  {
7126  	struct mem_cgroup *memcg;
7127  	struct lruvec *lruvec;
7128  
7129  	if (lru_gen_enabled()) {
7130  		lru_gen_age_node(pgdat, sc);
7131  		return;
7132  	}
7133  
7134  	if (!can_age_anon_pages(pgdat, sc))
7135  		return;
7136  
7137  	lruvec = mem_cgroup_lruvec(NULL, pgdat);
7138  	if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON))
7139  		return;
7140  
7141  	memcg = mem_cgroup_iter(NULL, NULL, NULL);
7142  	do {
7143  		lruvec = mem_cgroup_lruvec(memcg, pgdat);
7144  		shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
7145  				   sc, LRU_ACTIVE_ANON);
7146  		memcg = mem_cgroup_iter(NULL, memcg, NULL);
7147  	} while (memcg);
7148  }
7149  
7150  static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx)
7151  {
7152  	int i;
7153  	struct zone *zone;
7154  
7155  	/*
7156  	 * Check for watermark boosts top-down as the higher zones
7157  	 * are more likely to be boosted. Both watermarks and boosts
7158  	 * should not be checked at the same time as reclaim would
7159  	 * start prematurely when there is no boosting and a lower
7160  	 * zone is balanced.
7161  	 */
7162  	for (i = highest_zoneidx; i >= 0; i--) {
7163  		zone = pgdat->node_zones + i;
7164  		if (!managed_zone(zone))
7165  			continue;
7166  
7167  		if (zone->watermark_boost)
7168  			return true;
7169  	}
7170  
7171  	return false;
7172  }
7173  
7174  /*
7175   * Returns true if there is an eligible zone balanced for the request order
7176   * and highest_zoneidx
7177   */
7178  static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx)
7179  {
7180  	int i;
7181  	unsigned long mark = -1;
7182  	struct zone *zone;
7183  
7184  	/*
7185  	 * Check watermarks bottom-up as lower zones are more likely to
7186  	 * meet watermarks.
7187  	 */
7188  	for (i = 0; i <= highest_zoneidx; i++) {
7189  		zone = pgdat->node_zones + i;
7190  
7191  		if (!managed_zone(zone))
7192  			continue;
7193  
7194  		if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING)
7195  			mark = wmark_pages(zone, WMARK_PROMO);
7196  		else
7197  			mark = high_wmark_pages(zone);
7198  		if (zone_watermark_ok_safe(zone, order, mark, highest_zoneidx))
7199  			return true;
7200  	}
7201  
7202  	/*
7203  	 * If a node has no managed zone within highest_zoneidx, it does not
7204  	 * need balancing by definition. This can happen if a zone-restricted
7205  	 * allocation tries to wake a remote kswapd.
7206  	 */
7207  	if (mark == -1)
7208  		return true;
7209  
7210  	return false;
7211  }
7212  
7213  /* Clear pgdat state for congested, dirty or under writeback. */
7214  static void clear_pgdat_congested(pg_data_t *pgdat)
7215  {
7216  	struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);
7217  
7218  	clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
7219  	clear_bit(PGDAT_DIRTY, &pgdat->flags);
7220  	clear_bit(PGDAT_WRITEBACK, &pgdat->flags);
7221  }
7222  
7223  /*
7224   * Prepare kswapd for sleeping. This verifies that there are no processes
7225   * waiting in throttle_direct_reclaim() and that watermarks have been met.
7226   *
7227   * Returns true if kswapd is ready to sleep
7228   */
7229  static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order,
7230  				int highest_zoneidx)
7231  {
7232  	/*
7233  	 * The throttled processes are normally woken up in balance_pgdat() as
7234  	 * soon as allow_direct_reclaim() is true. But there is a potential
7235  	 * race between when kswapd checks the watermarks and a process gets
7236  	 * throttled. There is also a potential race if processes get
7237  	 * throttled, kswapd wakes, a large process exits thereby balancing the
7238  	 * zones, which causes kswapd to exit balance_pgdat() before reaching
7239  	 * the wake up checks. If kswapd is going to sleep, no process should
7240  	 * be sleeping on pfmemalloc_wait, so wake them now if necessary. If
7241  	 * the wake up is premature, processes will wake kswapd and get
7242  	 * throttled again. The difference from wake ups in balance_pgdat() is
7243  	 * that here we are under prepare_to_wait().
7244  	 */
7245  	if (waitqueue_active(&pgdat->pfmemalloc_wait))
7246  		wake_up_all(&pgdat->pfmemalloc_wait);
7247  
7248  	/* Hopeless node, leave it to direct reclaim */
7249  	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
7250  		return true;
7251  
7252  	if (pgdat_balanced(pgdat, order, highest_zoneidx)) {
7253  		clear_pgdat_congested(pgdat);
7254  		return true;
7255  	}
7256  
7257  	return false;
7258  }
7259  
7260  /*
7261   * kswapd shrinks a node of pages that are at or below the highest usable
7262   * zone that is currently unbalanced.
7263   *
7264   * Returns true if kswapd scanned at least the requested number of pages to
7265   * reclaim or if the lack of progress was due to pages under writeback.
7266   * This is used to determine if the scanning priority needs to be raised.
7267   */
7268  static bool kswapd_shrink_node(pg_data_t *pgdat,
7269  			       struct scan_control *sc)
7270  {
7271  	struct zone *zone;
7272  	int z;
7273  
7274  	/* Reclaim a number of pages proportional to the number of zones */
7275  	sc->nr_to_reclaim = 0;
7276  	for (z = 0; z <= sc->reclaim_idx; z++) {
7277  		zone = pgdat->node_zones + z;
7278  		if (!managed_zone(zone))
7279  			continue;
7280  
7281  		sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX);
7282  	}
7283  
7284  	/*
7285  	 * Historically care was taken to put equal pressure on all zones but
7286  	 * now pressure is applied based on node LRU order.
7287  	 */
7288  	shrink_node(pgdat, sc);
7289  
7290  	/*
7291  	 * Fragmentation may mean that the system cannot be rebalanced for
7292  	 * high-order allocations. If twice the allocation size has been
7293  	 * reclaimed then recheck watermarks only at order-0 to prevent
7294  	 * excessive reclaim. Assume that a process requested a high-order
7295  	 * can direct reclaim/compact.
7296  	 */
7297  	if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order))
7298  		sc->order = 0;
7299  
7300  	return sc->nr_scanned >= sc->nr_to_reclaim;
7301  }
7302  
7303  /* Page allocator PCP high watermark is lowered if reclaim is active. */
7304  static inline void
7305  update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active)
7306  {
7307  	int i;
7308  	struct zone *zone;
7309  
7310  	for (i = 0; i <= highest_zoneidx; i++) {
7311  		zone = pgdat->node_zones + i;
7312  
7313  		if (!managed_zone(zone))
7314  			continue;
7315  
7316  		if (active)
7317  			set_bit(ZONE_RECLAIM_ACTIVE, &zone->flags);
7318  		else
7319  			clear_bit(ZONE_RECLAIM_ACTIVE, &zone->flags);
7320  	}
7321  }
7322  
7323  static inline void
7324  set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)
7325  {
7326  	update_reclaim_active(pgdat, highest_zoneidx, true);
7327  }
7328  
7329  static inline void
7330  clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)
7331  {
7332  	update_reclaim_active(pgdat, highest_zoneidx, false);
7333  }
7334  
7335  /*
7336   * For kswapd, balance_pgdat() will reclaim pages across a node from zones
7337   * that are eligible for use by the caller until at least one zone is
7338   * balanced.
7339   *
7340   * Returns the order kswapd finished reclaiming at.
7341   *
7342   * kswapd scans the zones in the highmem->normal->dma direction.  It skips
7343   * zones which have free_pages > high_wmark_pages(zone), but once a zone is
7344   * found to have free_pages <= high_wmark_pages(zone), any page in that zone
7345   * or lower is eligible for reclaim until at least one usable zone is
7346   * balanced.
7347   */
7348  static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
7349  {
7350  	int i;
7351  	unsigned long nr_soft_reclaimed;
7352  	unsigned long nr_soft_scanned;
7353  	unsigned long pflags;
7354  	unsigned long nr_boost_reclaim;
7355  	unsigned long zone_boosts[MAX_NR_ZONES] = { 0, };
7356  	bool boosted;
7357  	struct zone *zone;
7358  	struct scan_control sc = {
7359  		.gfp_mask = GFP_KERNEL,
7360  		.order = order,
7361  		.may_unmap = 1,
7362  	};
7363  
7364  	set_task_reclaim_state(current, &sc.reclaim_state);
7365  	psi_memstall_enter(&pflags);
7366  	__fs_reclaim_acquire(_THIS_IP_);
7367  
7368  	count_vm_event(PAGEOUTRUN);
7369  
7370  	/*
7371  	 * Account for the reclaim boost. Note that the zone boost is left in
7372  	 * place so that parallel allocations that are near the watermark will
7373  	 * stall or direct reclaim until kswapd is finished.
7374  	 */
7375  	nr_boost_reclaim = 0;
7376  	for (i = 0; i <= highest_zoneidx; i++) {
7377  		zone = pgdat->node_zones + i;
7378  		if (!managed_zone(zone))
7379  			continue;
7380  
7381  		nr_boost_reclaim += zone->watermark_boost;
7382  		zone_boosts[i] = zone->watermark_boost;
7383  	}
7384  	boosted = nr_boost_reclaim;
7385  
7386  restart:
7387  	set_reclaim_active(pgdat, highest_zoneidx);
7388  	sc.priority = DEF_PRIORITY;
7389  	do {
7390  		unsigned long nr_reclaimed = sc.nr_reclaimed;
7391  		bool raise_priority = true;
7392  		bool balanced;
7393  		bool ret;
7394  
7395  		sc.reclaim_idx = highest_zoneidx;
7396  
7397  		/*
7398  		 * If the number of buffer_heads exceeds the maximum allowed
7399  		 * then consider reclaiming from all zones. This has a dual
7400  		 * purpose -- on 64-bit systems it is expected that
7401  		 * buffer_heads are stripped during active rotation. On 32-bit
7402  		 * systems, highmem pages can pin lowmem memory and shrinking
7403  		 * buffers can relieve lowmem pressure. Reclaim may still not
7404  		 * go ahead if all eligible zones for the original allocation
7405  		 * request are balanced to avoid excessive reclaim from kswapd.
7406  		 */
7407  		if (buffer_heads_over_limit) {
7408  			for (i = MAX_NR_ZONES - 1; i >= 0; i--) {
7409  				zone = pgdat->node_zones + i;
7410  				if (!managed_zone(zone))
7411  					continue;
7412  
7413  				sc.reclaim_idx = i;
7414  				break;
7415  			}
7416  		}
7417  
7418  		/*
7419  		 * If the pgdat is imbalanced then ignore boosting and preserve
7420  		 * the watermarks for a later time and restart. Note that the
7421  		 * zone watermarks will be still reset at the end of balancing
7422  		 * on the grounds that the normal reclaim should be enough to
7423  		 * re-evaluate if boosting is required when kswapd next wakes.
7424  		 */
7425  		balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx);
7426  		if (!balanced && nr_boost_reclaim) {
7427  			nr_boost_reclaim = 0;
7428  			goto restart;
7429  		}
7430  
7431  		/*
7432  		 * If boosting is not active then only reclaim if there are no
7433  		 * eligible zones. Note that sc.reclaim_idx is not used as
7434  		 * buffer_heads_over_limit may have adjusted it.
7435  		 */
7436  		if (!nr_boost_reclaim && balanced)
7437  			goto out;
7438  
7439  		/* Limit the priority of boosting to avoid reclaim writeback */
7440  		if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2)
7441  			raise_priority = false;
7442  
7443  		/*
7444  		 * Do not writeback or swap pages for boosted reclaim. The
7445  		 * intent is to relieve pressure not issue sub-optimal IO
7446  		 * from reclaim context. If no pages are reclaimed, the
7447  		 * reclaim will be aborted.
7448  		 */
7449  		sc.may_writepage = !laptop_mode && !nr_boost_reclaim;
7450  		sc.may_swap = !nr_boost_reclaim;
7451  
7452  		/*
7453  		 * Do some background aging, to give pages a chance to be
7454  		 * referenced before reclaiming. All pages are rotated
7455  		 * regardless of classzone as this is about consistent aging.
7456  		 */
7457  		kswapd_age_node(pgdat, &sc);
7458  
7459  		/*
7460  		 * If we're getting trouble reclaiming, start doing writepage
7461  		 * even in laptop mode.
7462  		 */
7463  		if (sc.priority < DEF_PRIORITY - 2)
7464  			sc.may_writepage = 1;
7465  
7466  		/* Call soft limit reclaim before calling shrink_node. */
7467  		sc.nr_scanned = 0;
7468  		nr_soft_scanned = 0;
7469  		nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order,
7470  						sc.gfp_mask, &nr_soft_scanned);
7471  		sc.nr_reclaimed += nr_soft_reclaimed;
7472  
7473  		/*
7474  		 * There should be no need to raise the scanning priority if
7475  		 * enough pages are already being scanned that that high
7476  		 * watermark would be met at 100% efficiency.
7477  		 */
7478  		if (kswapd_shrink_node(pgdat, &sc))
7479  			raise_priority = false;
7480  
7481  		/*
7482  		 * If the low watermark is met there is no need for processes
7483  		 * to be throttled on pfmemalloc_wait as they should not be
7484  		 * able to safely make forward progress. Wake them
7485  		 */
7486  		if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
7487  				allow_direct_reclaim(pgdat))
7488  			wake_up_all(&pgdat->pfmemalloc_wait);
7489  
7490  		/* Check if kswapd should be suspending */
7491  		__fs_reclaim_release(_THIS_IP_);
7492  		ret = try_to_freeze();
7493  		__fs_reclaim_acquire(_THIS_IP_);
7494  		if (ret || kthread_should_stop())
7495  			break;
7496  
7497  		/*
7498  		 * Raise priority if scanning rate is too low or there was no
7499  		 * progress in reclaiming pages
7500  		 */
7501  		nr_reclaimed = sc.nr_reclaimed - nr_reclaimed;
7502  		nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed);
7503  
7504  		/*
7505  		 * If reclaim made no progress for a boost, stop reclaim as
7506  		 * IO cannot be queued and it could be an infinite loop in
7507  		 * extreme circumstances.
7508  		 */
7509  		if (nr_boost_reclaim && !nr_reclaimed)
7510  			break;
7511  
7512  		if (raise_priority || !nr_reclaimed)
7513  			sc.priority--;
7514  	} while (sc.priority >= 1);
7515  
7516  	if (!sc.nr_reclaimed)
7517  		pgdat->kswapd_failures++;
7518  
7519  out:
7520  	clear_reclaim_active(pgdat, highest_zoneidx);
7521  
7522  	/* If reclaim was boosted, account for the reclaim done in this pass */
7523  	if (boosted) {
7524  		unsigned long flags;
7525  
7526  		for (i = 0; i <= highest_zoneidx; i++) {
7527  			if (!zone_boosts[i])
7528  				continue;
7529  
7530  			/* Increments are under the zone lock */
7531  			zone = pgdat->node_zones + i;
7532  			spin_lock_irqsave(&zone->lock, flags);
7533  			zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]);
7534  			spin_unlock_irqrestore(&zone->lock, flags);
7535  		}
7536  
7537  		/*
7538  		 * As there is now likely space, wakeup kcompact to defragment
7539  		 * pageblocks.
7540  		 */
7541  		wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx);
7542  	}
7543  
7544  	snapshot_refaults(NULL, pgdat);
7545  	__fs_reclaim_release(_THIS_IP_);
7546  	psi_memstall_leave(&pflags);
7547  	set_task_reclaim_state(current, NULL);
7548  
7549  	/*
7550  	 * Return the order kswapd stopped reclaiming at as
7551  	 * prepare_kswapd_sleep() takes it into account. If another caller
7552  	 * entered the allocator slow path while kswapd was awake, order will
7553  	 * remain at the higher level.
7554  	 */
7555  	return sc.order;
7556  }
7557  
7558  /*
7559   * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to
7560   * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is
7561   * not a valid index then either kswapd runs for first time or kswapd couldn't
7562   * sleep after previous reclaim attempt (node is still unbalanced). In that
7563   * case return the zone index of the previous kswapd reclaim cycle.
7564   */
7565  static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat,
7566  					   enum zone_type prev_highest_zoneidx)
7567  {
7568  	enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
7569  
7570  	return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx;
7571  }
7572  
7573  static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,
7574  				unsigned int highest_zoneidx)
7575  {
7576  	long remaining = 0;
7577  	DEFINE_WAIT(wait);
7578  
7579  	if (freezing(current) || kthread_should_stop())
7580  		return;
7581  
7582  	prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
7583  
7584  	/*
7585  	 * Try to sleep for a short interval. Note that kcompactd will only be
7586  	 * woken if it is possible to sleep for a short interval. This is
7587  	 * deliberate on the assumption that if reclaim cannot keep an
7588  	 * eligible zone balanced that it's also unlikely that compaction will
7589  	 * succeed.
7590  	 */
7591  	if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
7592  		/*
7593  		 * Compaction records what page blocks it recently failed to
7594  		 * isolate pages from and skips them in the future scanning.
7595  		 * When kswapd is going to sleep, it is reasonable to assume
7596  		 * that pages and compaction may succeed so reset the cache.
7597  		 */
7598  		reset_isolation_suitable(pgdat);
7599  
7600  		/*
7601  		 * We have freed the memory, now we should compact it to make
7602  		 * allocation of the requested order possible.
7603  		 */
7604  		wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx);
7605  
7606  		remaining = schedule_timeout(HZ/10);
7607  
7608  		/*
7609  		 * If woken prematurely then reset kswapd_highest_zoneidx and
7610  		 * order. The values will either be from a wakeup request or
7611  		 * the previous request that slept prematurely.
7612  		 */
7613  		if (remaining) {
7614  			WRITE_ONCE(pgdat->kswapd_highest_zoneidx,
7615  					kswapd_highest_zoneidx(pgdat,
7616  							highest_zoneidx));
7617  
7618  			if (READ_ONCE(pgdat->kswapd_order) < reclaim_order)
7619  				WRITE_ONCE(pgdat->kswapd_order, reclaim_order);
7620  		}
7621  
7622  		finish_wait(&pgdat->kswapd_wait, &wait);
7623  		prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
7624  	}
7625  
7626  	/*
7627  	 * After a short sleep, check if it was a premature sleep. If not, then
7628  	 * go fully to sleep until explicitly woken up.
7629  	 */
7630  	if (!remaining &&
7631  	    prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
7632  		trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
7633  
7634  		/*
7635  		 * vmstat counters are not perfectly accurate and the estimated
7636  		 * value for counters such as NR_FREE_PAGES can deviate from the
7637  		 * true value by nr_online_cpus * threshold. To avoid the zone
7638  		 * watermarks being breached while under pressure, we reduce the
7639  		 * per-cpu vmstat threshold while kswapd is awake and restore
7640  		 * them before going back to sleep.
7641  		 */
7642  		set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
7643  
7644  		if (!kthread_should_stop())
7645  			schedule();
7646  
7647  		set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);
7648  	} else {
7649  		if (remaining)
7650  			count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
7651  		else
7652  			count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
7653  	}
7654  	finish_wait(&pgdat->kswapd_wait, &wait);
7655  }
7656  
7657  /*
7658   * The background pageout daemon, started as a kernel thread
7659   * from the init process.
7660   *
7661   * This basically trickles out pages so that we have _some_
7662   * free memory available even if there is no other activity
7663   * that frees anything up. This is needed for things like routing
7664   * etc, where we otherwise might have all activity going on in
7665   * asynchronous contexts that cannot page things out.
7666   *
7667   * If there are applications that are active memory-allocators
7668   * (most normal use), this basically shouldn't matter.
7669   */
7670  static int kswapd(void *p)
7671  {
7672  	unsigned int alloc_order, reclaim_order;
7673  	unsigned int highest_zoneidx = MAX_NR_ZONES - 1;
7674  	pg_data_t *pgdat = (pg_data_t *)p;
7675  	struct task_struct *tsk = current;
7676  	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
7677  
7678  	if (!cpumask_empty(cpumask))
7679  		set_cpus_allowed_ptr(tsk, cpumask);
7680  
7681  	/*
7682  	 * Tell the memory management that we're a "memory allocator",
7683  	 * and that if we need more memory we should get access to it
7684  	 * regardless (see "__alloc_pages()"). "kswapd" should
7685  	 * never get caught in the normal page freeing logic.
7686  	 *
7687  	 * (Kswapd normally doesn't need memory anyway, but sometimes
7688  	 * you need a small amount of memory in order to be able to
7689  	 * page out something else, and this flag essentially protects
7690  	 * us from recursively trying to free more memory as we're
7691  	 * trying to free the first piece of memory in the first place).
7692  	 */
7693  	tsk->flags |= PF_MEMALLOC | PF_KSWAPD;
7694  	set_freezable();
7695  
7696  	WRITE_ONCE(pgdat->kswapd_order, 0);
7697  	WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
7698  	atomic_set(&pgdat->nr_writeback_throttled, 0);
7699  	for ( ; ; ) {
7700  		bool ret;
7701  
7702  		alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order);
7703  		highest_zoneidx = kswapd_highest_zoneidx(pgdat,
7704  							highest_zoneidx);
7705  
7706  kswapd_try_sleep:
7707  		kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,
7708  					highest_zoneidx);
7709  
7710  		/* Read the new order and highest_zoneidx */
7711  		alloc_order = READ_ONCE(pgdat->kswapd_order);
7712  		highest_zoneidx = kswapd_highest_zoneidx(pgdat,
7713  							highest_zoneidx);
7714  		WRITE_ONCE(pgdat->kswapd_order, 0);
7715  		WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
7716  
7717  		ret = try_to_freeze();
7718  		if (kthread_should_stop())
7719  			break;
7720  
7721  		/*
7722  		 * We can speed up thawing tasks if we don't call balance_pgdat
7723  		 * after returning from the refrigerator
7724  		 */
7725  		if (ret)
7726  			continue;
7727  
7728  		/*
7729  		 * Reclaim begins at the requested order but if a high-order
7730  		 * reclaim fails then kswapd falls back to reclaiming for
7731  		 * order-0. If that happens, kswapd will consider sleeping
7732  		 * for the order it finished reclaiming at (reclaim_order)
7733  		 * but kcompactd is woken to compact for the original
7734  		 * request (alloc_order).
7735  		 */
7736  		trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx,
7737  						alloc_order);
7738  		reclaim_order = balance_pgdat(pgdat, alloc_order,
7739  						highest_zoneidx);
7740  		if (reclaim_order < alloc_order)
7741  			goto kswapd_try_sleep;
7742  	}
7743  
7744  	tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD);
7745  
7746  	return 0;
7747  }
7748  
7749  /*
7750   * A zone is low on free memory or too fragmented for high-order memory.  If
7751   * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's
7752   * pgdat.  It will wake up kcompactd after reclaiming memory.  If kswapd reclaim
7753   * has failed or is not needed, still wake up kcompactd if only compaction is
7754   * needed.
7755   */
7756  void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order,
7757  		   enum zone_type highest_zoneidx)
7758  {
7759  	pg_data_t *pgdat;
7760  	enum zone_type curr_idx;
7761  
7762  	if (!managed_zone(zone))
7763  		return;
7764  
7765  	if (!cpuset_zone_allowed(zone, gfp_flags))
7766  		return;
7767  
7768  	pgdat = zone->zone_pgdat;
7769  	curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
7770  
7771  	if (curr_idx == MAX_NR_ZONES || curr_idx < highest_zoneidx)
7772  		WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx);
7773  
7774  	if (READ_ONCE(pgdat->kswapd_order) < order)
7775  		WRITE_ONCE(pgdat->kswapd_order, order);
7776  
7777  	if (!waitqueue_active(&pgdat->kswapd_wait))
7778  		return;
7779  
7780  	/* Hopeless node, leave it to direct reclaim if possible */
7781  	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ||
7782  	    (pgdat_balanced(pgdat, order, highest_zoneidx) &&
7783  	     !pgdat_watermark_boosted(pgdat, highest_zoneidx))) {
7784  		/*
7785  		 * There may be plenty of free memory available, but it's too
7786  		 * fragmented for high-order allocations.  Wake up kcompactd
7787  		 * and rely on compaction_suitable() to determine if it's
7788  		 * needed.  If it fails, it will defer subsequent attempts to
7789  		 * ratelimit its work.
7790  		 */
7791  		if (!(gfp_flags & __GFP_DIRECT_RECLAIM))
7792  			wakeup_kcompactd(pgdat, order, highest_zoneidx);
7793  		return;
7794  	}
7795  
7796  	trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order,
7797  				      gfp_flags);
7798  	wake_up_interruptible(&pgdat->kswapd_wait);
7799  }
7800  
7801  #ifdef CONFIG_HIBERNATION
7802  /*
7803   * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
7804   * freed pages.
7805   *
7806   * Rather than trying to age LRUs the aim is to preserve the overall
7807   * LRU order by reclaiming preferentially
7808   * inactive > active > active referenced > active mapped
7809   */
7810  unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
7811  {
7812  	struct scan_control sc = {
7813  		.nr_to_reclaim = nr_to_reclaim,
7814  		.gfp_mask = GFP_HIGHUSER_MOVABLE,
7815  		.reclaim_idx = MAX_NR_ZONES - 1,
7816  		.priority = DEF_PRIORITY,
7817  		.may_writepage = 1,
7818  		.may_unmap = 1,
7819  		.may_swap = 1,
7820  		.hibernation_mode = 1,
7821  	};
7822  	struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
7823  	unsigned long nr_reclaimed;
7824  	unsigned int noreclaim_flag;
7825  
7826  	fs_reclaim_acquire(sc.gfp_mask);
7827  	noreclaim_flag = memalloc_noreclaim_save();
7828  	set_task_reclaim_state(current, &sc.reclaim_state);
7829  
7830  	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
7831  
7832  	set_task_reclaim_state(current, NULL);
7833  	memalloc_noreclaim_restore(noreclaim_flag);
7834  	fs_reclaim_release(sc.gfp_mask);
7835  
7836  	return nr_reclaimed;
7837  }
7838  #endif /* CONFIG_HIBERNATION */
7839  
7840  /*
7841   * This kswapd start function will be called by init and node-hot-add.
7842   */
7843  void kswapd_run(int nid)
7844  {
7845  	pg_data_t *pgdat = NODE_DATA(nid);
7846  
7847  	pgdat_kswapd_lock(pgdat);
7848  	if (!pgdat->kswapd) {
7849  		pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
7850  		if (IS_ERR(pgdat->kswapd)) {
7851  			/* failure at boot is fatal */
7852  			BUG_ON(system_state < SYSTEM_RUNNING);
7853  			pr_err("Failed to start kswapd on node %d\n", nid);
7854  			pgdat->kswapd = NULL;
7855  		}
7856  	}
7857  	pgdat_kswapd_unlock(pgdat);
7858  }
7859  
7860  /*
7861   * Called by memory hotplug when all memory in a node is offlined.  Caller must
7862   * be holding mem_hotplug_begin/done().
7863   */
7864  void kswapd_stop(int nid)
7865  {
7866  	pg_data_t *pgdat = NODE_DATA(nid);
7867  	struct task_struct *kswapd;
7868  
7869  	pgdat_kswapd_lock(pgdat);
7870  	kswapd = pgdat->kswapd;
7871  	if (kswapd) {
7872  		kthread_stop(kswapd);
7873  		pgdat->kswapd = NULL;
7874  	}
7875  	pgdat_kswapd_unlock(pgdat);
7876  }
7877  
7878  static int __init kswapd_init(void)
7879  {
7880  	int nid;
7881  
7882  	swap_setup();
7883  	for_each_node_state(nid, N_MEMORY)
7884   		kswapd_run(nid);
7885  	return 0;
7886  }
7887  
7888  module_init(kswapd_init)
7889  
7890  #ifdef CONFIG_NUMA
7891  /*
7892   * Node reclaim mode
7893   *
7894   * If non-zero call node_reclaim when the number of free pages falls below
7895   * the watermarks.
7896   */
7897  int node_reclaim_mode __read_mostly;
7898  
7899  /*
7900   * Priority for NODE_RECLAIM. This determines the fraction of pages
7901   * of a node considered for each zone_reclaim. 4 scans 1/16th of
7902   * a zone.
7903   */
7904  #define NODE_RECLAIM_PRIORITY 4
7905  
7906  /*
7907   * Percentage of pages in a zone that must be unmapped for node_reclaim to
7908   * occur.
7909   */
7910  int sysctl_min_unmapped_ratio = 1;
7911  
7912  /*
7913   * If the number of slab pages in a zone grows beyond this percentage then
7914   * slab reclaim needs to occur.
7915   */
7916  int sysctl_min_slab_ratio = 5;
7917  
7918  static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat)
7919  {
7920  	unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED);
7921  	unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) +
7922  		node_page_state(pgdat, NR_ACTIVE_FILE);
7923  
7924  	/*
7925  	 * It's possible for there to be more file mapped pages than
7926  	 * accounted for by the pages on the file LRU lists because
7927  	 * tmpfs pages accounted for as ANON can also be FILE_MAPPED
7928  	 */
7929  	return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
7930  }
7931  
7932  /* Work out how many page cache pages we can reclaim in this reclaim_mode */
7933  static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat)
7934  {
7935  	unsigned long nr_pagecache_reclaimable;
7936  	unsigned long delta = 0;
7937  
7938  	/*
7939  	 * If RECLAIM_UNMAP is set, then all file pages are considered
7940  	 * potentially reclaimable. Otherwise, we have to worry about
7941  	 * pages like swapcache and node_unmapped_file_pages() provides
7942  	 * a better estimate
7943  	 */
7944  	if (node_reclaim_mode & RECLAIM_UNMAP)
7945  		nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES);
7946  	else
7947  		nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat);
7948  
7949  	/* If we can't clean pages, remove dirty pages from consideration */
7950  	if (!(node_reclaim_mode & RECLAIM_WRITE))
7951  		delta += node_page_state(pgdat, NR_FILE_DIRTY);
7952  
7953  	/* Watch for any possible underflows due to delta */
7954  	if (unlikely(delta > nr_pagecache_reclaimable))
7955  		delta = nr_pagecache_reclaimable;
7956  
7957  	return nr_pagecache_reclaimable - delta;
7958  }
7959  
7960  /*
7961   * Try to free up some pages from this node through reclaim.
7962   */
7963  static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
7964  {
7965  	/* Minimum pages needed in order to stay on node */
7966  	const unsigned long nr_pages = 1 << order;
7967  	struct task_struct *p = current;
7968  	unsigned int noreclaim_flag;
7969  	struct scan_control sc = {
7970  		.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
7971  		.gfp_mask = current_gfp_context(gfp_mask),
7972  		.order = order,
7973  		.priority = NODE_RECLAIM_PRIORITY,
7974  		.may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE),
7975  		.may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP),
7976  		.may_swap = 1,
7977  		.reclaim_idx = gfp_zone(gfp_mask),
7978  	};
7979  	unsigned long pflags;
7980  
7981  	trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order,
7982  					   sc.gfp_mask);
7983  
7984  	cond_resched();
7985  	psi_memstall_enter(&pflags);
7986  	fs_reclaim_acquire(sc.gfp_mask);
7987  	/*
7988  	 * We need to be able to allocate from the reserves for RECLAIM_UNMAP
7989  	 */
7990  	noreclaim_flag = memalloc_noreclaim_save();
7991  	set_task_reclaim_state(p, &sc.reclaim_state);
7992  
7993  	if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages ||
7994  	    node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) {
7995  		/*
7996  		 * Free memory by calling shrink node with increasing
7997  		 * priorities until we have enough memory freed.
7998  		 */
7999  		do {
8000  			shrink_node(pgdat, &sc);
8001  		} while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
8002  	}
8003  
8004  	set_task_reclaim_state(p, NULL);
8005  	memalloc_noreclaim_restore(noreclaim_flag);
8006  	fs_reclaim_release(sc.gfp_mask);
8007  	psi_memstall_leave(&pflags);
8008  
8009  	trace_mm_vmscan_node_reclaim_end(sc.nr_reclaimed);
8010  
8011  	return sc.nr_reclaimed >= nr_pages;
8012  }
8013  
8014  int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
8015  {
8016  	int ret;
8017  
8018  	/*
8019  	 * Node reclaim reclaims unmapped file backed pages and
8020  	 * slab pages if we are over the defined limits.
8021  	 *
8022  	 * A small portion of unmapped file backed pages is needed for
8023  	 * file I/O otherwise pages read by file I/O will be immediately
8024  	 * thrown out if the node is overallocated. So we do not reclaim
8025  	 * if less than a specified percentage of the node is used by
8026  	 * unmapped file backed pages.
8027  	 */
8028  	if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&
8029  	    node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <=
8030  	    pgdat->min_slab_pages)
8031  		return NODE_RECLAIM_FULL;
8032  
8033  	/*
8034  	 * Do not scan if the allocation should not be delayed.
8035  	 */
8036  	if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC))
8037  		return NODE_RECLAIM_NOSCAN;
8038  
8039  	/*
8040  	 * Only run node reclaim on the local node or on nodes that do not
8041  	 * have associated processors. This will favor the local processor
8042  	 * over remote processors and spread off node memory allocations
8043  	 * as wide as possible.
8044  	 */
8045  	if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id())
8046  		return NODE_RECLAIM_NOSCAN;
8047  
8048  	if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags))
8049  		return NODE_RECLAIM_NOSCAN;
8050  
8051  	ret = __node_reclaim(pgdat, gfp_mask, order);
8052  	clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags);
8053  
8054  	if (!ret)
8055  		count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
8056  
8057  	return ret;
8058  }
8059  #endif
8060  
8061  void check_move_unevictable_pages(struct pagevec *pvec)
8062  {
8063  	struct folio_batch fbatch;
8064  	unsigned i;
8065  
8066  	folio_batch_init(&fbatch);
8067  	for (i = 0; i < pvec->nr; i++) {
8068  		struct page *page = pvec->pages[i];
8069  
8070  		if (PageTransTail(page))
8071  			continue;
8072  		folio_batch_add(&fbatch, page_folio(page));
8073  	}
8074  	check_move_unevictable_folios(&fbatch);
8075  }
8076  EXPORT_SYMBOL_GPL(check_move_unevictable_pages);
8077  
8078  /**
8079   * check_move_unevictable_folios - Move evictable folios to appropriate zone
8080   * lru list
8081   * @fbatch: Batch of lru folios to check.
8082   *
8083   * Checks folios for evictability, if an evictable folio is in the unevictable
8084   * lru list, moves it to the appropriate evictable lru list. This function
8085   * should be only used for lru folios.
8086   */
8087  void check_move_unevictable_folios(struct folio_batch *fbatch)
8088  {
8089  	struct lruvec *lruvec = NULL;
8090  	int pgscanned = 0;
8091  	int pgrescued = 0;
8092  	int i;
8093  
8094  	for (i = 0; i < fbatch->nr; i++) {
8095  		struct folio *folio = fbatch->folios[i];
8096  		int nr_pages = folio_nr_pages(folio);
8097  
8098  		pgscanned += nr_pages;
8099  
8100  		/* block memcg migration while the folio moves between lrus */
8101  		if (!folio_test_clear_lru(folio))
8102  			continue;
8103  
8104  		lruvec = folio_lruvec_relock_irq(folio, lruvec);
8105  		if (folio_evictable(folio) && folio_test_unevictable(folio)) {
8106  			lruvec_del_folio(lruvec, folio);
8107  			folio_clear_unevictable(folio);
8108  			lruvec_add_folio(lruvec, folio);
8109  			pgrescued += nr_pages;
8110  		}
8111  		folio_set_lru(folio);
8112  	}
8113  
8114  	if (lruvec) {
8115  		__count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
8116  		__count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
8117  		unlock_page_lruvec_irq(lruvec);
8118  	} else if (pgscanned) {
8119  		count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
8120  	}
8121  }
8122  EXPORT_SYMBOL_GPL(check_move_unevictable_folios);
8123