xref: /openbmc/linux/mm/vmscan.c (revision ff4a7481c3898ffc3cc271d6aca431d190c37247)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  linux/mm/vmscan.c
4  *
5  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
6  *
7  *  Swap reorganised 29.12.95, Stephen Tweedie.
8  *  kswapd added: 7.1.96  sct
9  *  Removed kswapd_ctl limits, and swap out as many pages as needed
10  *  to bring the system back to freepages.high: 2.4.97, Rik van Riel.
11  *  Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
12  *  Multiqueue VM started 5.8.00, Rik van Riel.
13  */
14 
15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16 
17 #include <linux/mm.h>
18 #include <linux/sched/mm.h>
19 #include <linux/module.h>
20 #include <linux/gfp.h>
21 #include <linux/kernel_stat.h>
22 #include <linux/swap.h>
23 #include <linux/pagemap.h>
24 #include <linux/init.h>
25 #include <linux/highmem.h>
26 #include <linux/vmpressure.h>
27 #include <linux/vmstat.h>
28 #include <linux/file.h>
29 #include <linux/writeback.h>
30 #include <linux/blkdev.h>
31 #include <linux/buffer_head.h>	/* for try_to_release_page(),
32 					buffer_heads_over_limit */
33 #include <linux/mm_inline.h>
34 #include <linux/backing-dev.h>
35 #include <linux/rmap.h>
36 #include <linux/topology.h>
37 #include <linux/cpu.h>
38 #include <linux/cpuset.h>
39 #include <linux/compaction.h>
40 #include <linux/notifier.h>
41 #include <linux/rwsem.h>
42 #include <linux/delay.h>
43 #include <linux/kthread.h>
44 #include <linux/freezer.h>
45 #include <linux/memcontrol.h>
46 #include <linux/delayacct.h>
47 #include <linux/sysctl.h>
48 #include <linux/oom.h>
49 #include <linux/pagevec.h>
50 #include <linux/prefetch.h>
51 #include <linux/printk.h>
52 #include <linux/dax.h>
53 #include <linux/psi.h>
54 
55 #include <asm/tlbflush.h>
56 #include <asm/div64.h>
57 
58 #include <linux/swapops.h>
59 #include <linux/balloon_compaction.h>
60 
61 #include "internal.h"
62 
63 #define CREATE_TRACE_POINTS
64 #include <trace/events/vmscan.h>
65 
66 struct scan_control {
67 	/* How many pages shrink_list() should reclaim */
68 	unsigned long nr_to_reclaim;
69 
70 	/*
71 	 * Nodemask of nodes allowed by the caller. If NULL, all nodes
72 	 * are scanned.
73 	 */
74 	nodemask_t	*nodemask;
75 
76 	/*
77 	 * The memory cgroup that hit its limit and as a result is the
78 	 * primary target of this reclaim invocation.
79 	 */
80 	struct mem_cgroup *target_mem_cgroup;
81 
82 	/* Writepage batching in laptop mode; RECLAIM_WRITE */
83 	unsigned int may_writepage:1;
84 
85 	/* Can mapped pages be reclaimed? */
86 	unsigned int may_unmap:1;
87 
88 	/* Can pages be swapped as part of reclaim? */
89 	unsigned int may_swap:1;
90 
91 	/*
92 	 * Cgroups are not reclaimed below their configured memory.low,
93 	 * unless we threaten to OOM. If any cgroups are skipped due to
94 	 * memory.low and nothing was reclaimed, go back for memory.low.
95 	 */
96 	unsigned int memcg_low_reclaim:1;
97 	unsigned int memcg_low_skipped:1;
98 
99 	unsigned int hibernation_mode:1;
100 
101 	/* One of the zones is ready for compaction */
102 	unsigned int compaction_ready:1;
103 
104 	/* Allocation order */
105 	s8 order;
106 
107 	/* Scan (total_size >> priority) pages at once */
108 	s8 priority;
109 
110 	/* The highest zone to isolate pages for reclaim from */
111 	s8 reclaim_idx;
112 
113 	/* This context's GFP mask */
114 	gfp_t gfp_mask;
115 
116 	/* Incremented by the number of inactive pages that were scanned */
117 	unsigned long nr_scanned;
118 
119 	/* Number of pages freed so far during a call to shrink_zones() */
120 	unsigned long nr_reclaimed;
121 
122 	struct {
123 		unsigned int dirty;
124 		unsigned int unqueued_dirty;
125 		unsigned int congested;
126 		unsigned int writeback;
127 		unsigned int immediate;
128 		unsigned int file_taken;
129 		unsigned int taken;
130 	} nr;
131 };
132 
133 #ifdef ARCH_HAS_PREFETCH
134 #define prefetch_prev_lru_page(_page, _base, _field)			\
135 	do {								\
136 		if ((_page)->lru.prev != _base) {			\
137 			struct page *prev;				\
138 									\
139 			prev = lru_to_page(&(_page->lru));		\
140 			prefetch(&prev->_field);			\
141 		}							\
142 	} while (0)
143 #else
144 #define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
145 #endif
146 
147 #ifdef ARCH_HAS_PREFETCHW
148 #define prefetchw_prev_lru_page(_page, _base, _field)			\
149 	do {								\
150 		if ((_page)->lru.prev != _base) {			\
151 			struct page *prev;				\
152 									\
153 			prev = lru_to_page(&(_page->lru));		\
154 			prefetchw(&prev->_field);			\
155 		}							\
156 	} while (0)
157 #else
158 #define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
159 #endif
160 
161 /*
162  * From 0 .. 100.  Higher means more swappy.
163  */
164 int vm_swappiness = 60;
165 /*
166  * The total number of pages which are beyond the high watermark within all
167  * zones.
168  */
169 unsigned long vm_total_pages;
170 
171 static LIST_HEAD(shrinker_list);
172 static DECLARE_RWSEM(shrinker_rwsem);
173 
174 #ifdef CONFIG_MEMCG_KMEM
175 
176 /*
177  * We allow subsystems to populate their shrinker-related
178  * LRU lists before register_shrinker_prepared() is called
179  * for the shrinker, since we don't want to impose
180  * restrictions on their internal registration order.
181  * In this case shrink_slab_memcg() may find corresponding
182  * bit is set in the shrinkers map.
183  *
184  * This value is used by the function to detect registering
185  * shrinkers and to skip do_shrink_slab() calls for them.
186  */
187 #define SHRINKER_REGISTERING ((struct shrinker *)~0UL)
188 
189 static DEFINE_IDR(shrinker_idr);
190 static int shrinker_nr_max;
191 
192 static int prealloc_memcg_shrinker(struct shrinker *shrinker)
193 {
194 	int id, ret = -ENOMEM;
195 
196 	down_write(&shrinker_rwsem);
197 	/* This may call shrinker, so it must use down_read_trylock() */
198 	id = idr_alloc(&shrinker_idr, SHRINKER_REGISTERING, 0, 0, GFP_KERNEL);
199 	if (id < 0)
200 		goto unlock;
201 
202 	if (id >= shrinker_nr_max) {
203 		if (memcg_expand_shrinker_maps(id)) {
204 			idr_remove(&shrinker_idr, id);
205 			goto unlock;
206 		}
207 
208 		shrinker_nr_max = id + 1;
209 	}
210 	shrinker->id = id;
211 	ret = 0;
212 unlock:
213 	up_write(&shrinker_rwsem);
214 	return ret;
215 }
216 
217 static void unregister_memcg_shrinker(struct shrinker *shrinker)
218 {
219 	int id = shrinker->id;
220 
221 	BUG_ON(id < 0);
222 
223 	down_write(&shrinker_rwsem);
224 	idr_remove(&shrinker_idr, id);
225 	up_write(&shrinker_rwsem);
226 }
227 #else /* CONFIG_MEMCG_KMEM */
228 static int prealloc_memcg_shrinker(struct shrinker *shrinker)
229 {
230 	return 0;
231 }
232 
233 static void unregister_memcg_shrinker(struct shrinker *shrinker)
234 {
235 }
236 #endif /* CONFIG_MEMCG_KMEM */
237 
238 #ifdef CONFIG_MEMCG
239 static bool global_reclaim(struct scan_control *sc)
240 {
241 	return !sc->target_mem_cgroup;
242 }
243 
244 /**
245  * sane_reclaim - is the usual dirty throttling mechanism operational?
246  * @sc: scan_control in question
247  *
248  * The normal page dirty throttling mechanism in balance_dirty_pages() is
249  * completely broken with the legacy memcg and direct stalling in
250  * shrink_page_list() is used for throttling instead, which lacks all the
251  * niceties such as fairness, adaptive pausing, bandwidth proportional
252  * allocation and configurability.
253  *
254  * This function tests whether the vmscan currently in progress can assume
255  * that the normal dirty throttling mechanism is operational.
256  */
257 static bool sane_reclaim(struct scan_control *sc)
258 {
259 	struct mem_cgroup *memcg = sc->target_mem_cgroup;
260 
261 	if (!memcg)
262 		return true;
263 #ifdef CONFIG_CGROUP_WRITEBACK
264 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
265 		return true;
266 #endif
267 	return false;
268 }
269 
270 static void set_memcg_congestion(pg_data_t *pgdat,
271 				struct mem_cgroup *memcg,
272 				bool congested)
273 {
274 	struct mem_cgroup_per_node *mn;
275 
276 	if (!memcg)
277 		return;
278 
279 	mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
280 	WRITE_ONCE(mn->congested, congested);
281 }
282 
283 static bool memcg_congested(pg_data_t *pgdat,
284 			struct mem_cgroup *memcg)
285 {
286 	struct mem_cgroup_per_node *mn;
287 
288 	mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
289 	return READ_ONCE(mn->congested);
290 
291 }
292 #else
293 static bool global_reclaim(struct scan_control *sc)
294 {
295 	return true;
296 }
297 
298 static bool sane_reclaim(struct scan_control *sc)
299 {
300 	return true;
301 }
302 
303 static inline void set_memcg_congestion(struct pglist_data *pgdat,
304 				struct mem_cgroup *memcg, bool congested)
305 {
306 }
307 
308 static inline bool memcg_congested(struct pglist_data *pgdat,
309 			struct mem_cgroup *memcg)
310 {
311 	return false;
312 
313 }
314 #endif
315 
316 /*
317  * This misses isolated pages which are not accounted for to save counters.
318  * As the data only determines if reclaim or compaction continues, it is
319  * not expected that isolated pages will be a dominating factor.
320  */
321 unsigned long zone_reclaimable_pages(struct zone *zone)
322 {
323 	unsigned long nr;
324 
325 	nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) +
326 		zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE);
327 	if (get_nr_swap_pages() > 0)
328 		nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) +
329 			zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON);
330 
331 	return nr;
332 }
333 
334 /**
335  * lruvec_lru_size -  Returns the number of pages on the given LRU list.
336  * @lruvec: lru vector
337  * @lru: lru to use
338  * @zone_idx: zones to consider (use MAX_NR_ZONES for the whole LRU list)
339  */
340 unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx)
341 {
342 	unsigned long lru_size;
343 	int zid;
344 
345 	if (!mem_cgroup_disabled())
346 		lru_size = mem_cgroup_get_lru_size(lruvec, lru);
347 	else
348 		lru_size = node_page_state(lruvec_pgdat(lruvec), NR_LRU_BASE + lru);
349 
350 	for (zid = zone_idx + 1; zid < MAX_NR_ZONES; zid++) {
351 		struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
352 		unsigned long size;
353 
354 		if (!managed_zone(zone))
355 			continue;
356 
357 		if (!mem_cgroup_disabled())
358 			size = mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
359 		else
360 			size = zone_page_state(&lruvec_pgdat(lruvec)->node_zones[zid],
361 				       NR_ZONE_LRU_BASE + lru);
362 		lru_size -= min(size, lru_size);
363 	}
364 
365 	return lru_size;
366 
367 }
368 
369 /*
370  * Add a shrinker callback to be called from the vm.
371  */
372 int prealloc_shrinker(struct shrinker *shrinker)
373 {
374 	size_t size = sizeof(*shrinker->nr_deferred);
375 
376 	if (shrinker->flags & SHRINKER_NUMA_AWARE)
377 		size *= nr_node_ids;
378 
379 	shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
380 	if (!shrinker->nr_deferred)
381 		return -ENOMEM;
382 
383 	if (shrinker->flags & SHRINKER_MEMCG_AWARE) {
384 		if (prealloc_memcg_shrinker(shrinker))
385 			goto free_deferred;
386 	}
387 
388 	return 0;
389 
390 free_deferred:
391 	kfree(shrinker->nr_deferred);
392 	shrinker->nr_deferred = NULL;
393 	return -ENOMEM;
394 }
395 
396 void free_prealloced_shrinker(struct shrinker *shrinker)
397 {
398 	if (!shrinker->nr_deferred)
399 		return;
400 
401 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)
402 		unregister_memcg_shrinker(shrinker);
403 
404 	kfree(shrinker->nr_deferred);
405 	shrinker->nr_deferred = NULL;
406 }
407 
408 void register_shrinker_prepared(struct shrinker *shrinker)
409 {
410 	down_write(&shrinker_rwsem);
411 	list_add_tail(&shrinker->list, &shrinker_list);
412 #ifdef CONFIG_MEMCG_KMEM
413 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)
414 		idr_replace(&shrinker_idr, shrinker, shrinker->id);
415 #endif
416 	up_write(&shrinker_rwsem);
417 }
418 
419 int register_shrinker(struct shrinker *shrinker)
420 {
421 	int err = prealloc_shrinker(shrinker);
422 
423 	if (err)
424 		return err;
425 	register_shrinker_prepared(shrinker);
426 	return 0;
427 }
428 EXPORT_SYMBOL(register_shrinker);
429 
430 /*
431  * Remove one
432  */
433 void unregister_shrinker(struct shrinker *shrinker)
434 {
435 	if (!shrinker->nr_deferred)
436 		return;
437 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)
438 		unregister_memcg_shrinker(shrinker);
439 	down_write(&shrinker_rwsem);
440 	list_del(&shrinker->list);
441 	up_write(&shrinker_rwsem);
442 	kfree(shrinker->nr_deferred);
443 	shrinker->nr_deferred = NULL;
444 }
445 EXPORT_SYMBOL(unregister_shrinker);
446 
447 #define SHRINK_BATCH 128
448 
449 static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,
450 				    struct shrinker *shrinker, int priority)
451 {
452 	unsigned long freed = 0;
453 	unsigned long long delta;
454 	long total_scan;
455 	long freeable;
456 	long nr;
457 	long new_nr;
458 	int nid = shrinkctl->nid;
459 	long batch_size = shrinker->batch ? shrinker->batch
460 					  : SHRINK_BATCH;
461 	long scanned = 0, next_deferred;
462 
463 	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
464 		nid = 0;
465 
466 	freeable = shrinker->count_objects(shrinker, shrinkctl);
467 	if (freeable == 0 || freeable == SHRINK_EMPTY)
468 		return freeable;
469 
470 	/*
471 	 * copy the current shrinker scan count into a local variable
472 	 * and zero it so that other concurrent shrinker invocations
473 	 * don't also do this scanning work.
474 	 */
475 	nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0);
476 
477 	total_scan = nr;
478 	if (shrinker->seeks) {
479 		delta = freeable >> priority;
480 		delta *= 4;
481 		do_div(delta, shrinker->seeks);
482 	} else {
483 		/*
484 		 * These objects don't require any IO to create. Trim
485 		 * them aggressively under memory pressure to keep
486 		 * them from causing refetches in the IO caches.
487 		 */
488 		delta = freeable / 2;
489 	}
490 
491 	/*
492 	 * Make sure we apply some minimal pressure on default priority
493 	 * even on small cgroups. Stale objects are not only consuming memory
494 	 * by themselves, but can also hold a reference to a dying cgroup,
495 	 * preventing it from being reclaimed. A dying cgroup with all
496 	 * corresponding structures like per-cpu stats and kmem caches
497 	 * can be really big, so it may lead to a significant waste of memory.
498 	 */
499 	delta = max_t(unsigned long long, delta, min(freeable, batch_size));
500 
501 	total_scan += delta;
502 	if (total_scan < 0) {
503 		pr_err("shrink_slab: %pF negative objects to delete nr=%ld\n",
504 		       shrinker->scan_objects, total_scan);
505 		total_scan = freeable;
506 		next_deferred = nr;
507 	} else
508 		next_deferred = total_scan;
509 
510 	/*
511 	 * We need to avoid excessive windup on filesystem shrinkers
512 	 * due to large numbers of GFP_NOFS allocations causing the
513 	 * shrinkers to return -1 all the time. This results in a large
514 	 * nr being built up so when a shrink that can do some work
515 	 * comes along it empties the entire cache due to nr >>>
516 	 * freeable. This is bad for sustaining a working set in
517 	 * memory.
518 	 *
519 	 * Hence only allow the shrinker to scan the entire cache when
520 	 * a large delta change is calculated directly.
521 	 */
522 	if (delta < freeable / 4)
523 		total_scan = min(total_scan, freeable / 2);
524 
525 	/*
526 	 * Avoid risking looping forever due to too large nr value:
527 	 * never try to free more than twice the estimate number of
528 	 * freeable entries.
529 	 */
530 	if (total_scan > freeable * 2)
531 		total_scan = freeable * 2;
532 
533 	trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
534 				   freeable, delta, total_scan, priority);
535 
536 	/*
537 	 * Normally, we should not scan less than batch_size objects in one
538 	 * pass to avoid too frequent shrinker calls, but if the slab has less
539 	 * than batch_size objects in total and we are really tight on memory,
540 	 * we will try to reclaim all available objects, otherwise we can end
541 	 * up failing allocations although there are plenty of reclaimable
542 	 * objects spread over several slabs with usage less than the
543 	 * batch_size.
544 	 *
545 	 * We detect the "tight on memory" situations by looking at the total
546 	 * number of objects we want to scan (total_scan). If it is greater
547 	 * than the total number of objects on slab (freeable), we must be
548 	 * scanning at high prio and therefore should try to reclaim as much as
549 	 * possible.
550 	 */
551 	while (total_scan >= batch_size ||
552 	       total_scan >= freeable) {
553 		unsigned long ret;
554 		unsigned long nr_to_scan = min(batch_size, total_scan);
555 
556 		shrinkctl->nr_to_scan = nr_to_scan;
557 		shrinkctl->nr_scanned = nr_to_scan;
558 		ret = shrinker->scan_objects(shrinker, shrinkctl);
559 		if (ret == SHRINK_STOP)
560 			break;
561 		freed += ret;
562 
563 		count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned);
564 		total_scan -= shrinkctl->nr_scanned;
565 		scanned += shrinkctl->nr_scanned;
566 
567 		cond_resched();
568 	}
569 
570 	if (next_deferred >= scanned)
571 		next_deferred -= scanned;
572 	else
573 		next_deferred = 0;
574 	/*
575 	 * move the unused scan count back into the shrinker in a
576 	 * manner that handles concurrent updates. If we exhausted the
577 	 * scan, there is no need to do an update.
578 	 */
579 	if (next_deferred > 0)
580 		new_nr = atomic_long_add_return(next_deferred,
581 						&shrinker->nr_deferred[nid]);
582 	else
583 		new_nr = atomic_long_read(&shrinker->nr_deferred[nid]);
584 
585 	trace_mm_shrink_slab_end(shrinker, nid, freed, nr, new_nr, total_scan);
586 	return freed;
587 }
588 
589 #ifdef CONFIG_MEMCG_KMEM
590 static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
591 			struct mem_cgroup *memcg, int priority)
592 {
593 	struct memcg_shrinker_map *map;
594 	unsigned long ret, freed = 0;
595 	int i;
596 
597 	if (!memcg_kmem_enabled() || !mem_cgroup_online(memcg))
598 		return 0;
599 
600 	if (!down_read_trylock(&shrinker_rwsem))
601 		return 0;
602 
603 	map = rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_map,
604 					true);
605 	if (unlikely(!map))
606 		goto unlock;
607 
608 	for_each_set_bit(i, map->map, shrinker_nr_max) {
609 		struct shrink_control sc = {
610 			.gfp_mask = gfp_mask,
611 			.nid = nid,
612 			.memcg = memcg,
613 		};
614 		struct shrinker *shrinker;
615 
616 		shrinker = idr_find(&shrinker_idr, i);
617 		if (unlikely(!shrinker || shrinker == SHRINKER_REGISTERING)) {
618 			if (!shrinker)
619 				clear_bit(i, map->map);
620 			continue;
621 		}
622 
623 		ret = do_shrink_slab(&sc, shrinker, priority);
624 		if (ret == SHRINK_EMPTY) {
625 			clear_bit(i, map->map);
626 			/*
627 			 * After the shrinker reported that it had no objects to
628 			 * free, but before we cleared the corresponding bit in
629 			 * the memcg shrinker map, a new object might have been
630 			 * added. To make sure, we have the bit set in this
631 			 * case, we invoke the shrinker one more time and reset
632 			 * the bit if it reports that it is not empty anymore.
633 			 * The memory barrier here pairs with the barrier in
634 			 * memcg_set_shrinker_bit():
635 			 *
636 			 * list_lru_add()     shrink_slab_memcg()
637 			 *   list_add_tail()    clear_bit()
638 			 *   <MB>               <MB>
639 			 *   set_bit()          do_shrink_slab()
640 			 */
641 			smp_mb__after_atomic();
642 			ret = do_shrink_slab(&sc, shrinker, priority);
643 			if (ret == SHRINK_EMPTY)
644 				ret = 0;
645 			else
646 				memcg_set_shrinker_bit(memcg, nid, i);
647 		}
648 		freed += ret;
649 
650 		if (rwsem_is_contended(&shrinker_rwsem)) {
651 			freed = freed ? : 1;
652 			break;
653 		}
654 	}
655 unlock:
656 	up_read(&shrinker_rwsem);
657 	return freed;
658 }
659 #else /* CONFIG_MEMCG_KMEM */
660 static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
661 			struct mem_cgroup *memcg, int priority)
662 {
663 	return 0;
664 }
665 #endif /* CONFIG_MEMCG_KMEM */
666 
667 /**
668  * shrink_slab - shrink slab caches
669  * @gfp_mask: allocation context
670  * @nid: node whose slab caches to target
671  * @memcg: memory cgroup whose slab caches to target
672  * @priority: the reclaim priority
673  *
674  * Call the shrink functions to age shrinkable caches.
675  *
676  * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set,
677  * unaware shrinkers will receive a node id of 0 instead.
678  *
679  * @memcg specifies the memory cgroup to target. Unaware shrinkers
680  * are called only if it is the root cgroup.
681  *
682  * @priority is sc->priority, we take the number of objects and >> by priority
683  * in order to get the scan target.
684  *
685  * Returns the number of reclaimed slab objects.
686  */
687 static unsigned long shrink_slab(gfp_t gfp_mask, int nid,
688 				 struct mem_cgroup *memcg,
689 				 int priority)
690 {
691 	unsigned long ret, freed = 0;
692 	struct shrinker *shrinker;
693 
694 	if (!mem_cgroup_is_root(memcg))
695 		return shrink_slab_memcg(gfp_mask, nid, memcg, priority);
696 
697 	if (!down_read_trylock(&shrinker_rwsem))
698 		goto out;
699 
700 	list_for_each_entry(shrinker, &shrinker_list, list) {
701 		struct shrink_control sc = {
702 			.gfp_mask = gfp_mask,
703 			.nid = nid,
704 			.memcg = memcg,
705 		};
706 
707 		ret = do_shrink_slab(&sc, shrinker, priority);
708 		if (ret == SHRINK_EMPTY)
709 			ret = 0;
710 		freed += ret;
711 		/*
712 		 * Bail out if someone want to register a new shrinker to
713 		 * prevent the regsitration from being stalled for long periods
714 		 * by parallel ongoing shrinking.
715 		 */
716 		if (rwsem_is_contended(&shrinker_rwsem)) {
717 			freed = freed ? : 1;
718 			break;
719 		}
720 	}
721 
722 	up_read(&shrinker_rwsem);
723 out:
724 	cond_resched();
725 	return freed;
726 }
727 
728 void drop_slab_node(int nid)
729 {
730 	unsigned long freed;
731 
732 	do {
733 		struct mem_cgroup *memcg = NULL;
734 
735 		freed = 0;
736 		memcg = mem_cgroup_iter(NULL, NULL, NULL);
737 		do {
738 			freed += shrink_slab(GFP_KERNEL, nid, memcg, 0);
739 		} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
740 	} while (freed > 10);
741 }
742 
743 void drop_slab(void)
744 {
745 	int nid;
746 
747 	for_each_online_node(nid)
748 		drop_slab_node(nid);
749 }
750 
751 static inline int is_page_cache_freeable(struct page *page)
752 {
753 	/*
754 	 * A freeable page cache page is referenced only by the caller
755 	 * that isolated the page, the page cache and optional buffer
756 	 * heads at page->private.
757 	 */
758 	int page_cache_pins = PageTransHuge(page) && PageSwapCache(page) ?
759 		HPAGE_PMD_NR : 1;
760 	return page_count(page) - page_has_private(page) == 1 + page_cache_pins;
761 }
762 
763 static int may_write_to_inode(struct inode *inode, struct scan_control *sc)
764 {
765 	if (current->flags & PF_SWAPWRITE)
766 		return 1;
767 	if (!inode_write_congested(inode))
768 		return 1;
769 	if (inode_to_bdi(inode) == current->backing_dev_info)
770 		return 1;
771 	return 0;
772 }
773 
774 /*
775  * We detected a synchronous write error writing a page out.  Probably
776  * -ENOSPC.  We need to propagate that into the address_space for a subsequent
777  * fsync(), msync() or close().
778  *
779  * The tricky part is that after writepage we cannot touch the mapping: nothing
780  * prevents it from being freed up.  But we have a ref on the page and once
781  * that page is locked, the mapping is pinned.
782  *
783  * We're allowed to run sleeping lock_page() here because we know the caller has
784  * __GFP_FS.
785  */
786 static void handle_write_error(struct address_space *mapping,
787 				struct page *page, int error)
788 {
789 	lock_page(page);
790 	if (page_mapping(page) == mapping)
791 		mapping_set_error(mapping, error);
792 	unlock_page(page);
793 }
794 
795 /* possible outcome of pageout() */
796 typedef enum {
797 	/* failed to write page out, page is locked */
798 	PAGE_KEEP,
799 	/* move page to the active list, page is locked */
800 	PAGE_ACTIVATE,
801 	/* page has been sent to the disk successfully, page is unlocked */
802 	PAGE_SUCCESS,
803 	/* page is clean and locked */
804 	PAGE_CLEAN,
805 } pageout_t;
806 
807 /*
808  * pageout is called by shrink_page_list() for each dirty page.
809  * Calls ->writepage().
810  */
811 static pageout_t pageout(struct page *page, struct address_space *mapping,
812 			 struct scan_control *sc)
813 {
814 	/*
815 	 * If the page is dirty, only perform writeback if that write
816 	 * will be non-blocking.  To prevent this allocation from being
817 	 * stalled by pagecache activity.  But note that there may be
818 	 * stalls if we need to run get_block().  We could test
819 	 * PagePrivate for that.
820 	 *
821 	 * If this process is currently in __generic_file_write_iter() against
822 	 * this page's queue, we can perform writeback even if that
823 	 * will block.
824 	 *
825 	 * If the page is swapcache, write it back even if that would
826 	 * block, for some throttling. This happens by accident, because
827 	 * swap_backing_dev_info is bust: it doesn't reflect the
828 	 * congestion state of the swapdevs.  Easy to fix, if needed.
829 	 */
830 	if (!is_page_cache_freeable(page))
831 		return PAGE_KEEP;
832 	if (!mapping) {
833 		/*
834 		 * Some data journaling orphaned pages can have
835 		 * page->mapping == NULL while being dirty with clean buffers.
836 		 */
837 		if (page_has_private(page)) {
838 			if (try_to_free_buffers(page)) {
839 				ClearPageDirty(page);
840 				pr_info("%s: orphaned page\n", __func__);
841 				return PAGE_CLEAN;
842 			}
843 		}
844 		return PAGE_KEEP;
845 	}
846 	if (mapping->a_ops->writepage == NULL)
847 		return PAGE_ACTIVATE;
848 	if (!may_write_to_inode(mapping->host, sc))
849 		return PAGE_KEEP;
850 
851 	if (clear_page_dirty_for_io(page)) {
852 		int res;
853 		struct writeback_control wbc = {
854 			.sync_mode = WB_SYNC_NONE,
855 			.nr_to_write = SWAP_CLUSTER_MAX,
856 			.range_start = 0,
857 			.range_end = LLONG_MAX,
858 			.for_reclaim = 1,
859 		};
860 
861 		SetPageReclaim(page);
862 		res = mapping->a_ops->writepage(page, &wbc);
863 		if (res < 0)
864 			handle_write_error(mapping, page, res);
865 		if (res == AOP_WRITEPAGE_ACTIVATE) {
866 			ClearPageReclaim(page);
867 			return PAGE_ACTIVATE;
868 		}
869 
870 		if (!PageWriteback(page)) {
871 			/* synchronous write or broken a_ops? */
872 			ClearPageReclaim(page);
873 		}
874 		trace_mm_vmscan_writepage(page);
875 		inc_node_page_state(page, NR_VMSCAN_WRITE);
876 		return PAGE_SUCCESS;
877 	}
878 
879 	return PAGE_CLEAN;
880 }
881 
882 /*
883  * Same as remove_mapping, but if the page is removed from the mapping, it
884  * gets returned with a refcount of 0.
885  */
886 static int __remove_mapping(struct address_space *mapping, struct page *page,
887 			    bool reclaimed)
888 {
889 	unsigned long flags;
890 	int refcount;
891 
892 	BUG_ON(!PageLocked(page));
893 	BUG_ON(mapping != page_mapping(page));
894 
895 	xa_lock_irqsave(&mapping->i_pages, flags);
896 	/*
897 	 * The non racy check for a busy page.
898 	 *
899 	 * Must be careful with the order of the tests. When someone has
900 	 * a ref to the page, it may be possible that they dirty it then
901 	 * drop the reference. So if PageDirty is tested before page_count
902 	 * here, then the following race may occur:
903 	 *
904 	 * get_user_pages(&page);
905 	 * [user mapping goes away]
906 	 * write_to(page);
907 	 *				!PageDirty(page)    [good]
908 	 * SetPageDirty(page);
909 	 * put_page(page);
910 	 *				!page_count(page)   [good, discard it]
911 	 *
912 	 * [oops, our write_to data is lost]
913 	 *
914 	 * Reversing the order of the tests ensures such a situation cannot
915 	 * escape unnoticed. The smp_rmb is needed to ensure the page->flags
916 	 * load is not satisfied before that of page->_refcount.
917 	 *
918 	 * Note that if SetPageDirty is always performed via set_page_dirty,
919 	 * and thus under the i_pages lock, then this ordering is not required.
920 	 */
921 	if (unlikely(PageTransHuge(page)) && PageSwapCache(page))
922 		refcount = 1 + HPAGE_PMD_NR;
923 	else
924 		refcount = 2;
925 	if (!page_ref_freeze(page, refcount))
926 		goto cannot_free;
927 	/* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */
928 	if (unlikely(PageDirty(page))) {
929 		page_ref_unfreeze(page, refcount);
930 		goto cannot_free;
931 	}
932 
933 	if (PageSwapCache(page)) {
934 		swp_entry_t swap = { .val = page_private(page) };
935 		mem_cgroup_swapout(page, swap);
936 		__delete_from_swap_cache(page, swap);
937 		xa_unlock_irqrestore(&mapping->i_pages, flags);
938 		put_swap_page(page, swap);
939 	} else {
940 		void (*freepage)(struct page *);
941 		void *shadow = NULL;
942 
943 		freepage = mapping->a_ops->freepage;
944 		/*
945 		 * Remember a shadow entry for reclaimed file cache in
946 		 * order to detect refaults, thus thrashing, later on.
947 		 *
948 		 * But don't store shadows in an address space that is
949 		 * already exiting.  This is not just an optizimation,
950 		 * inode reclaim needs to empty out the radix tree or
951 		 * the nodes are lost.  Don't plant shadows behind its
952 		 * back.
953 		 *
954 		 * We also don't store shadows for DAX mappings because the
955 		 * only page cache pages found in these are zero pages
956 		 * covering holes, and because we don't want to mix DAX
957 		 * exceptional entries and shadow exceptional entries in the
958 		 * same address_space.
959 		 */
960 		if (reclaimed && page_is_file_cache(page) &&
961 		    !mapping_exiting(mapping) && !dax_mapping(mapping))
962 			shadow = workingset_eviction(mapping, page);
963 		__delete_from_page_cache(page, shadow);
964 		xa_unlock_irqrestore(&mapping->i_pages, flags);
965 
966 		if (freepage != NULL)
967 			freepage(page);
968 	}
969 
970 	return 1;
971 
972 cannot_free:
973 	xa_unlock_irqrestore(&mapping->i_pages, flags);
974 	return 0;
975 }
976 
977 /*
978  * Attempt to detach a locked page from its ->mapping.  If it is dirty or if
979  * someone else has a ref on the page, abort and return 0.  If it was
980  * successfully detached, return 1.  Assumes the caller has a single ref on
981  * this page.
982  */
983 int remove_mapping(struct address_space *mapping, struct page *page)
984 {
985 	if (__remove_mapping(mapping, page, false)) {
986 		/*
987 		 * Unfreezing the refcount with 1 rather than 2 effectively
988 		 * drops the pagecache ref for us without requiring another
989 		 * atomic operation.
990 		 */
991 		page_ref_unfreeze(page, 1);
992 		return 1;
993 	}
994 	return 0;
995 }
996 
997 /**
998  * putback_lru_page - put previously isolated page onto appropriate LRU list
999  * @page: page to be put back to appropriate lru list
1000  *
1001  * Add previously isolated @page to appropriate LRU list.
1002  * Page may still be unevictable for other reasons.
1003  *
1004  * lru_lock must not be held, interrupts must be enabled.
1005  */
1006 void putback_lru_page(struct page *page)
1007 {
1008 	lru_cache_add(page);
1009 	put_page(page);		/* drop ref from isolate */
1010 }
1011 
1012 enum page_references {
1013 	PAGEREF_RECLAIM,
1014 	PAGEREF_RECLAIM_CLEAN,
1015 	PAGEREF_KEEP,
1016 	PAGEREF_ACTIVATE,
1017 };
1018 
1019 static enum page_references page_check_references(struct page *page,
1020 						  struct scan_control *sc)
1021 {
1022 	int referenced_ptes, referenced_page;
1023 	unsigned long vm_flags;
1024 
1025 	referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
1026 					  &vm_flags);
1027 	referenced_page = TestClearPageReferenced(page);
1028 
1029 	/*
1030 	 * Mlock lost the isolation race with us.  Let try_to_unmap()
1031 	 * move the page to the unevictable list.
1032 	 */
1033 	if (vm_flags & VM_LOCKED)
1034 		return PAGEREF_RECLAIM;
1035 
1036 	if (referenced_ptes) {
1037 		if (PageSwapBacked(page))
1038 			return PAGEREF_ACTIVATE;
1039 		/*
1040 		 * All mapped pages start out with page table
1041 		 * references from the instantiating fault, so we need
1042 		 * to look twice if a mapped file page is used more
1043 		 * than once.
1044 		 *
1045 		 * Mark it and spare it for another trip around the
1046 		 * inactive list.  Another page table reference will
1047 		 * lead to its activation.
1048 		 *
1049 		 * Note: the mark is set for activated pages as well
1050 		 * so that recently deactivated but used pages are
1051 		 * quickly recovered.
1052 		 */
1053 		SetPageReferenced(page);
1054 
1055 		if (referenced_page || referenced_ptes > 1)
1056 			return PAGEREF_ACTIVATE;
1057 
1058 		/*
1059 		 * Activate file-backed executable pages after first usage.
1060 		 */
1061 		if (vm_flags & VM_EXEC)
1062 			return PAGEREF_ACTIVATE;
1063 
1064 		return PAGEREF_KEEP;
1065 	}
1066 
1067 	/* Reclaim if clean, defer dirty pages to writeback */
1068 	if (referenced_page && !PageSwapBacked(page))
1069 		return PAGEREF_RECLAIM_CLEAN;
1070 
1071 	return PAGEREF_RECLAIM;
1072 }
1073 
1074 /* Check if a page is dirty or under writeback */
1075 static void page_check_dirty_writeback(struct page *page,
1076 				       bool *dirty, bool *writeback)
1077 {
1078 	struct address_space *mapping;
1079 
1080 	/*
1081 	 * Anonymous pages are not handled by flushers and must be written
1082 	 * from reclaim context. Do not stall reclaim based on them
1083 	 */
1084 	if (!page_is_file_cache(page) ||
1085 	    (PageAnon(page) && !PageSwapBacked(page))) {
1086 		*dirty = false;
1087 		*writeback = false;
1088 		return;
1089 	}
1090 
1091 	/* By default assume that the page flags are accurate */
1092 	*dirty = PageDirty(page);
1093 	*writeback = PageWriteback(page);
1094 
1095 	/* Verify dirty/writeback state if the filesystem supports it */
1096 	if (!page_has_private(page))
1097 		return;
1098 
1099 	mapping = page_mapping(page);
1100 	if (mapping && mapping->a_ops->is_dirty_writeback)
1101 		mapping->a_ops->is_dirty_writeback(page, dirty, writeback);
1102 }
1103 
1104 /*
1105  * shrink_page_list() returns the number of reclaimed pages
1106  */
1107 static unsigned long shrink_page_list(struct list_head *page_list,
1108 				      struct pglist_data *pgdat,
1109 				      struct scan_control *sc,
1110 				      enum ttu_flags ttu_flags,
1111 				      struct reclaim_stat *stat,
1112 				      bool force_reclaim)
1113 {
1114 	LIST_HEAD(ret_pages);
1115 	LIST_HEAD(free_pages);
1116 	int pgactivate = 0;
1117 	unsigned nr_unqueued_dirty = 0;
1118 	unsigned nr_dirty = 0;
1119 	unsigned nr_congested = 0;
1120 	unsigned nr_reclaimed = 0;
1121 	unsigned nr_writeback = 0;
1122 	unsigned nr_immediate = 0;
1123 	unsigned nr_ref_keep = 0;
1124 	unsigned nr_unmap_fail = 0;
1125 
1126 	cond_resched();
1127 
1128 	while (!list_empty(page_list)) {
1129 		struct address_space *mapping;
1130 		struct page *page;
1131 		int may_enter_fs;
1132 		enum page_references references = PAGEREF_RECLAIM_CLEAN;
1133 		bool dirty, writeback;
1134 
1135 		cond_resched();
1136 
1137 		page = lru_to_page(page_list);
1138 		list_del(&page->lru);
1139 
1140 		if (!trylock_page(page))
1141 			goto keep;
1142 
1143 		VM_BUG_ON_PAGE(PageActive(page), page);
1144 
1145 		sc->nr_scanned++;
1146 
1147 		if (unlikely(!page_evictable(page)))
1148 			goto activate_locked;
1149 
1150 		if (!sc->may_unmap && page_mapped(page))
1151 			goto keep_locked;
1152 
1153 		/* Double the slab pressure for mapped and swapcache pages */
1154 		if ((page_mapped(page) || PageSwapCache(page)) &&
1155 		    !(PageAnon(page) && !PageSwapBacked(page)))
1156 			sc->nr_scanned++;
1157 
1158 		may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
1159 			(PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
1160 
1161 		/*
1162 		 * The number of dirty pages determines if a node is marked
1163 		 * reclaim_congested which affects wait_iff_congested. kswapd
1164 		 * will stall and start writing pages if the tail of the LRU
1165 		 * is all dirty unqueued pages.
1166 		 */
1167 		page_check_dirty_writeback(page, &dirty, &writeback);
1168 		if (dirty || writeback)
1169 			nr_dirty++;
1170 
1171 		if (dirty && !writeback)
1172 			nr_unqueued_dirty++;
1173 
1174 		/*
1175 		 * Treat this page as congested if the underlying BDI is or if
1176 		 * pages are cycling through the LRU so quickly that the
1177 		 * pages marked for immediate reclaim are making it to the
1178 		 * end of the LRU a second time.
1179 		 */
1180 		mapping = page_mapping(page);
1181 		if (((dirty || writeback) && mapping &&
1182 		     inode_write_congested(mapping->host)) ||
1183 		    (writeback && PageReclaim(page)))
1184 			nr_congested++;
1185 
1186 		/*
1187 		 * If a page at the tail of the LRU is under writeback, there
1188 		 * are three cases to consider.
1189 		 *
1190 		 * 1) If reclaim is encountering an excessive number of pages
1191 		 *    under writeback and this page is both under writeback and
1192 		 *    PageReclaim then it indicates that pages are being queued
1193 		 *    for IO but are being recycled through the LRU before the
1194 		 *    IO can complete. Waiting on the page itself risks an
1195 		 *    indefinite stall if it is impossible to writeback the
1196 		 *    page due to IO error or disconnected storage so instead
1197 		 *    note that the LRU is being scanned too quickly and the
1198 		 *    caller can stall after page list has been processed.
1199 		 *
1200 		 * 2) Global or new memcg reclaim encounters a page that is
1201 		 *    not marked for immediate reclaim, or the caller does not
1202 		 *    have __GFP_FS (or __GFP_IO if it's simply going to swap,
1203 		 *    not to fs). In this case mark the page for immediate
1204 		 *    reclaim and continue scanning.
1205 		 *
1206 		 *    Require may_enter_fs because we would wait on fs, which
1207 		 *    may not have submitted IO yet. And the loop driver might
1208 		 *    enter reclaim, and deadlock if it waits on a page for
1209 		 *    which it is needed to do the write (loop masks off
1210 		 *    __GFP_IO|__GFP_FS for this reason); but more thought
1211 		 *    would probably show more reasons.
1212 		 *
1213 		 * 3) Legacy memcg encounters a page that is already marked
1214 		 *    PageReclaim. memcg does not have any dirty pages
1215 		 *    throttling so we could easily OOM just because too many
1216 		 *    pages are in writeback and there is nothing else to
1217 		 *    reclaim. Wait for the writeback to complete.
1218 		 *
1219 		 * In cases 1) and 2) we activate the pages to get them out of
1220 		 * the way while we continue scanning for clean pages on the
1221 		 * inactive list and refilling from the active list. The
1222 		 * observation here is that waiting for disk writes is more
1223 		 * expensive than potentially causing reloads down the line.
1224 		 * Since they're marked for immediate reclaim, they won't put
1225 		 * memory pressure on the cache working set any longer than it
1226 		 * takes to write them to disk.
1227 		 */
1228 		if (PageWriteback(page)) {
1229 			/* Case 1 above */
1230 			if (current_is_kswapd() &&
1231 			    PageReclaim(page) &&
1232 			    test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1233 				nr_immediate++;
1234 				goto activate_locked;
1235 
1236 			/* Case 2 above */
1237 			} else if (sane_reclaim(sc) ||
1238 			    !PageReclaim(page) || !may_enter_fs) {
1239 				/*
1240 				 * This is slightly racy - end_page_writeback()
1241 				 * might have just cleared PageReclaim, then
1242 				 * setting PageReclaim here end up interpreted
1243 				 * as PageReadahead - but that does not matter
1244 				 * enough to care.  What we do want is for this
1245 				 * page to have PageReclaim set next time memcg
1246 				 * reclaim reaches the tests above, so it will
1247 				 * then wait_on_page_writeback() to avoid OOM;
1248 				 * and it's also appropriate in global reclaim.
1249 				 */
1250 				SetPageReclaim(page);
1251 				nr_writeback++;
1252 				goto activate_locked;
1253 
1254 			/* Case 3 above */
1255 			} else {
1256 				unlock_page(page);
1257 				wait_on_page_writeback(page);
1258 				/* then go back and try same page again */
1259 				list_add_tail(&page->lru, page_list);
1260 				continue;
1261 			}
1262 		}
1263 
1264 		if (!force_reclaim)
1265 			references = page_check_references(page, sc);
1266 
1267 		switch (references) {
1268 		case PAGEREF_ACTIVATE:
1269 			goto activate_locked;
1270 		case PAGEREF_KEEP:
1271 			nr_ref_keep++;
1272 			goto keep_locked;
1273 		case PAGEREF_RECLAIM:
1274 		case PAGEREF_RECLAIM_CLEAN:
1275 			; /* try to reclaim the page below */
1276 		}
1277 
1278 		/*
1279 		 * Anonymous process memory has backing store?
1280 		 * Try to allocate it some swap space here.
1281 		 * Lazyfree page could be freed directly
1282 		 */
1283 		if (PageAnon(page) && PageSwapBacked(page)) {
1284 			if (!PageSwapCache(page)) {
1285 				if (!(sc->gfp_mask & __GFP_IO))
1286 					goto keep_locked;
1287 				if (PageTransHuge(page)) {
1288 					/* cannot split THP, skip it */
1289 					if (!can_split_huge_page(page, NULL))
1290 						goto activate_locked;
1291 					/*
1292 					 * Split pages without a PMD map right
1293 					 * away. Chances are some or all of the
1294 					 * tail pages can be freed without IO.
1295 					 */
1296 					if (!compound_mapcount(page) &&
1297 					    split_huge_page_to_list(page,
1298 								    page_list))
1299 						goto activate_locked;
1300 				}
1301 				if (!add_to_swap(page)) {
1302 					if (!PageTransHuge(page))
1303 						goto activate_locked;
1304 					/* Fallback to swap normal pages */
1305 					if (split_huge_page_to_list(page,
1306 								    page_list))
1307 						goto activate_locked;
1308 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1309 					count_vm_event(THP_SWPOUT_FALLBACK);
1310 #endif
1311 					if (!add_to_swap(page))
1312 						goto activate_locked;
1313 				}
1314 
1315 				may_enter_fs = 1;
1316 
1317 				/* Adding to swap updated mapping */
1318 				mapping = page_mapping(page);
1319 			}
1320 		} else if (unlikely(PageTransHuge(page))) {
1321 			/* Split file THP */
1322 			if (split_huge_page_to_list(page, page_list))
1323 				goto keep_locked;
1324 		}
1325 
1326 		/*
1327 		 * The page is mapped into the page tables of one or more
1328 		 * processes. Try to unmap it here.
1329 		 */
1330 		if (page_mapped(page)) {
1331 			enum ttu_flags flags = ttu_flags | TTU_BATCH_FLUSH;
1332 
1333 			if (unlikely(PageTransHuge(page)))
1334 				flags |= TTU_SPLIT_HUGE_PMD;
1335 			if (!try_to_unmap(page, flags)) {
1336 				nr_unmap_fail++;
1337 				goto activate_locked;
1338 			}
1339 		}
1340 
1341 		if (PageDirty(page)) {
1342 			/*
1343 			 * Only kswapd can writeback filesystem pages
1344 			 * to avoid risk of stack overflow. But avoid
1345 			 * injecting inefficient single-page IO into
1346 			 * flusher writeback as much as possible: only
1347 			 * write pages when we've encountered many
1348 			 * dirty pages, and when we've already scanned
1349 			 * the rest of the LRU for clean pages and see
1350 			 * the same dirty pages again (PageReclaim).
1351 			 */
1352 			if (page_is_file_cache(page) &&
1353 			    (!current_is_kswapd() || !PageReclaim(page) ||
1354 			     !test_bit(PGDAT_DIRTY, &pgdat->flags))) {
1355 				/*
1356 				 * Immediately reclaim when written back.
1357 				 * Similar in principal to deactivate_page()
1358 				 * except we already have the page isolated
1359 				 * and know it's dirty
1360 				 */
1361 				inc_node_page_state(page, NR_VMSCAN_IMMEDIATE);
1362 				SetPageReclaim(page);
1363 
1364 				goto activate_locked;
1365 			}
1366 
1367 			if (references == PAGEREF_RECLAIM_CLEAN)
1368 				goto keep_locked;
1369 			if (!may_enter_fs)
1370 				goto keep_locked;
1371 			if (!sc->may_writepage)
1372 				goto keep_locked;
1373 
1374 			/*
1375 			 * Page is dirty. Flush the TLB if a writable entry
1376 			 * potentially exists to avoid CPU writes after IO
1377 			 * starts and then write it out here.
1378 			 */
1379 			try_to_unmap_flush_dirty();
1380 			switch (pageout(page, mapping, sc)) {
1381 			case PAGE_KEEP:
1382 				goto keep_locked;
1383 			case PAGE_ACTIVATE:
1384 				goto activate_locked;
1385 			case PAGE_SUCCESS:
1386 				if (PageWriteback(page))
1387 					goto keep;
1388 				if (PageDirty(page))
1389 					goto keep;
1390 
1391 				/*
1392 				 * A synchronous write - probably a ramdisk.  Go
1393 				 * ahead and try to reclaim the page.
1394 				 */
1395 				if (!trylock_page(page))
1396 					goto keep;
1397 				if (PageDirty(page) || PageWriteback(page))
1398 					goto keep_locked;
1399 				mapping = page_mapping(page);
1400 			case PAGE_CLEAN:
1401 				; /* try to free the page below */
1402 			}
1403 		}
1404 
1405 		/*
1406 		 * If the page has buffers, try to free the buffer mappings
1407 		 * associated with this page. If we succeed we try to free
1408 		 * the page as well.
1409 		 *
1410 		 * We do this even if the page is PageDirty().
1411 		 * try_to_release_page() does not perform I/O, but it is
1412 		 * possible for a page to have PageDirty set, but it is actually
1413 		 * clean (all its buffers are clean).  This happens if the
1414 		 * buffers were written out directly, with submit_bh(). ext3
1415 		 * will do this, as well as the blockdev mapping.
1416 		 * try_to_release_page() will discover that cleanness and will
1417 		 * drop the buffers and mark the page clean - it can be freed.
1418 		 *
1419 		 * Rarely, pages can have buffers and no ->mapping.  These are
1420 		 * the pages which were not successfully invalidated in
1421 		 * truncate_complete_page().  We try to drop those buffers here
1422 		 * and if that worked, and the page is no longer mapped into
1423 		 * process address space (page_count == 1) it can be freed.
1424 		 * Otherwise, leave the page on the LRU so it is swappable.
1425 		 */
1426 		if (page_has_private(page)) {
1427 			if (!try_to_release_page(page, sc->gfp_mask))
1428 				goto activate_locked;
1429 			if (!mapping && page_count(page) == 1) {
1430 				unlock_page(page);
1431 				if (put_page_testzero(page))
1432 					goto free_it;
1433 				else {
1434 					/*
1435 					 * rare race with speculative reference.
1436 					 * the speculative reference will free
1437 					 * this page shortly, so we may
1438 					 * increment nr_reclaimed here (and
1439 					 * leave it off the LRU).
1440 					 */
1441 					nr_reclaimed++;
1442 					continue;
1443 				}
1444 			}
1445 		}
1446 
1447 		if (PageAnon(page) && !PageSwapBacked(page)) {
1448 			/* follow __remove_mapping for reference */
1449 			if (!page_ref_freeze(page, 1))
1450 				goto keep_locked;
1451 			if (PageDirty(page)) {
1452 				page_ref_unfreeze(page, 1);
1453 				goto keep_locked;
1454 			}
1455 
1456 			count_vm_event(PGLAZYFREED);
1457 			count_memcg_page_event(page, PGLAZYFREED);
1458 		} else if (!mapping || !__remove_mapping(mapping, page, true))
1459 			goto keep_locked;
1460 		/*
1461 		 * At this point, we have no other references and there is
1462 		 * no way to pick any more up (removed from LRU, removed
1463 		 * from pagecache). Can use non-atomic bitops now (and
1464 		 * we obviously don't have to worry about waking up a process
1465 		 * waiting on the page lock, because there are no references.
1466 		 */
1467 		__ClearPageLocked(page);
1468 free_it:
1469 		nr_reclaimed++;
1470 
1471 		/*
1472 		 * Is there need to periodically free_page_list? It would
1473 		 * appear not as the counts should be low
1474 		 */
1475 		if (unlikely(PageTransHuge(page))) {
1476 			mem_cgroup_uncharge(page);
1477 			(*get_compound_page_dtor(page))(page);
1478 		} else
1479 			list_add(&page->lru, &free_pages);
1480 		continue;
1481 
1482 activate_locked:
1483 		/* Not a candidate for swapping, so reclaim swap space. */
1484 		if (PageSwapCache(page) && (mem_cgroup_swap_full(page) ||
1485 						PageMlocked(page)))
1486 			try_to_free_swap(page);
1487 		VM_BUG_ON_PAGE(PageActive(page), page);
1488 		if (!PageMlocked(page)) {
1489 			SetPageActive(page);
1490 			pgactivate++;
1491 			count_memcg_page_event(page, PGACTIVATE);
1492 		}
1493 keep_locked:
1494 		unlock_page(page);
1495 keep:
1496 		list_add(&page->lru, &ret_pages);
1497 		VM_BUG_ON_PAGE(PageLRU(page) || PageUnevictable(page), page);
1498 	}
1499 
1500 	mem_cgroup_uncharge_list(&free_pages);
1501 	try_to_unmap_flush();
1502 	free_unref_page_list(&free_pages);
1503 
1504 	list_splice(&ret_pages, page_list);
1505 	count_vm_events(PGACTIVATE, pgactivate);
1506 
1507 	if (stat) {
1508 		stat->nr_dirty = nr_dirty;
1509 		stat->nr_congested = nr_congested;
1510 		stat->nr_unqueued_dirty = nr_unqueued_dirty;
1511 		stat->nr_writeback = nr_writeback;
1512 		stat->nr_immediate = nr_immediate;
1513 		stat->nr_activate = pgactivate;
1514 		stat->nr_ref_keep = nr_ref_keep;
1515 		stat->nr_unmap_fail = nr_unmap_fail;
1516 	}
1517 	return nr_reclaimed;
1518 }
1519 
1520 unsigned long reclaim_clean_pages_from_list(struct zone *zone,
1521 					    struct list_head *page_list)
1522 {
1523 	struct scan_control sc = {
1524 		.gfp_mask = GFP_KERNEL,
1525 		.priority = DEF_PRIORITY,
1526 		.may_unmap = 1,
1527 	};
1528 	unsigned long ret;
1529 	struct page *page, *next;
1530 	LIST_HEAD(clean_pages);
1531 
1532 	list_for_each_entry_safe(page, next, page_list, lru) {
1533 		if (page_is_file_cache(page) && !PageDirty(page) &&
1534 		    !__PageMovable(page)) {
1535 			ClearPageActive(page);
1536 			list_move(&page->lru, &clean_pages);
1537 		}
1538 	}
1539 
1540 	ret = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc,
1541 			TTU_IGNORE_ACCESS, NULL, true);
1542 	list_splice(&clean_pages, page_list);
1543 	mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, -ret);
1544 	return ret;
1545 }
1546 
1547 /*
1548  * Attempt to remove the specified page from its LRU.  Only take this page
1549  * if it is of the appropriate PageActive status.  Pages which are being
1550  * freed elsewhere are also ignored.
1551  *
1552  * page:	page to consider
1553  * mode:	one of the LRU isolation modes defined above
1554  *
1555  * returns 0 on success, -ve errno on failure.
1556  */
1557 int __isolate_lru_page(struct page *page, isolate_mode_t mode)
1558 {
1559 	int ret = -EINVAL;
1560 
1561 	/* Only take pages on the LRU. */
1562 	if (!PageLRU(page))
1563 		return ret;
1564 
1565 	/* Compaction should not handle unevictable pages but CMA can do so */
1566 	if (PageUnevictable(page) && !(mode & ISOLATE_UNEVICTABLE))
1567 		return ret;
1568 
1569 	ret = -EBUSY;
1570 
1571 	/*
1572 	 * To minimise LRU disruption, the caller can indicate that it only
1573 	 * wants to isolate pages it will be able to operate on without
1574 	 * blocking - clean pages for the most part.
1575 	 *
1576 	 * ISOLATE_ASYNC_MIGRATE is used to indicate that it only wants to pages
1577 	 * that it is possible to migrate without blocking
1578 	 */
1579 	if (mode & ISOLATE_ASYNC_MIGRATE) {
1580 		/* All the caller can do on PageWriteback is block */
1581 		if (PageWriteback(page))
1582 			return ret;
1583 
1584 		if (PageDirty(page)) {
1585 			struct address_space *mapping;
1586 			bool migrate_dirty;
1587 
1588 			/*
1589 			 * Only pages without mappings or that have a
1590 			 * ->migratepage callback are possible to migrate
1591 			 * without blocking. However, we can be racing with
1592 			 * truncation so it's necessary to lock the page
1593 			 * to stabilise the mapping as truncation holds
1594 			 * the page lock until after the page is removed
1595 			 * from the page cache.
1596 			 */
1597 			if (!trylock_page(page))
1598 				return ret;
1599 
1600 			mapping = page_mapping(page);
1601 			migrate_dirty = !mapping || mapping->a_ops->migratepage;
1602 			unlock_page(page);
1603 			if (!migrate_dirty)
1604 				return ret;
1605 		}
1606 	}
1607 
1608 	if ((mode & ISOLATE_UNMAPPED) && page_mapped(page))
1609 		return ret;
1610 
1611 	if (likely(get_page_unless_zero(page))) {
1612 		/*
1613 		 * Be careful not to clear PageLRU until after we're
1614 		 * sure the page is not being freed elsewhere -- the
1615 		 * page release code relies on it.
1616 		 */
1617 		ClearPageLRU(page);
1618 		ret = 0;
1619 	}
1620 
1621 	return ret;
1622 }
1623 
1624 
1625 /*
1626  * Update LRU sizes after isolating pages. The LRU size updates must
1627  * be complete before mem_cgroup_update_lru_size due to a santity check.
1628  */
1629 static __always_inline void update_lru_sizes(struct lruvec *lruvec,
1630 			enum lru_list lru, unsigned long *nr_zone_taken)
1631 {
1632 	int zid;
1633 
1634 	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1635 		if (!nr_zone_taken[zid])
1636 			continue;
1637 
1638 		__update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1639 #ifdef CONFIG_MEMCG
1640 		mem_cgroup_update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1641 #endif
1642 	}
1643 
1644 }
1645 
1646 /*
1647  * zone_lru_lock is heavily contended.  Some of the functions that
1648  * shrink the lists perform better by taking out a batch of pages
1649  * and working on them outside the LRU lock.
1650  *
1651  * For pagecache intensive workloads, this function is the hottest
1652  * spot in the kernel (apart from copy_*_user functions).
1653  *
1654  * Appropriate locks must be held before calling this function.
1655  *
1656  * @nr_to_scan:	The number of eligible pages to look through on the list.
1657  * @lruvec:	The LRU vector to pull pages from.
1658  * @dst:	The temp list to put pages on to.
1659  * @nr_scanned:	The number of pages that were scanned.
1660  * @sc:		The scan_control struct for this reclaim session
1661  * @mode:	One of the LRU isolation modes
1662  * @lru:	LRU list id for isolating
1663  *
1664  * returns how many pages were moved onto *@dst.
1665  */
1666 static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
1667 		struct lruvec *lruvec, struct list_head *dst,
1668 		unsigned long *nr_scanned, struct scan_control *sc,
1669 		isolate_mode_t mode, enum lru_list lru)
1670 {
1671 	struct list_head *src = &lruvec->lists[lru];
1672 	unsigned long nr_taken = 0;
1673 	unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 };
1674 	unsigned long nr_skipped[MAX_NR_ZONES] = { 0, };
1675 	unsigned long skipped = 0;
1676 	unsigned long scan, total_scan, nr_pages;
1677 	LIST_HEAD(pages_skipped);
1678 
1679 	scan = 0;
1680 	for (total_scan = 0;
1681 	     scan < nr_to_scan && nr_taken < nr_to_scan && !list_empty(src);
1682 	     total_scan++) {
1683 		struct page *page;
1684 
1685 		page = lru_to_page(src);
1686 		prefetchw_prev_lru_page(page, src, flags);
1687 
1688 		VM_BUG_ON_PAGE(!PageLRU(page), page);
1689 
1690 		if (page_zonenum(page) > sc->reclaim_idx) {
1691 			list_move(&page->lru, &pages_skipped);
1692 			nr_skipped[page_zonenum(page)]++;
1693 			continue;
1694 		}
1695 
1696 		/*
1697 		 * Do not count skipped pages because that makes the function
1698 		 * return with no isolated pages if the LRU mostly contains
1699 		 * ineligible pages.  This causes the VM to not reclaim any
1700 		 * pages, triggering a premature OOM.
1701 		 */
1702 		scan++;
1703 		switch (__isolate_lru_page(page, mode)) {
1704 		case 0:
1705 			nr_pages = hpage_nr_pages(page);
1706 			nr_taken += nr_pages;
1707 			nr_zone_taken[page_zonenum(page)] += nr_pages;
1708 			list_move(&page->lru, dst);
1709 			break;
1710 
1711 		case -EBUSY:
1712 			/* else it is being freed elsewhere */
1713 			list_move(&page->lru, src);
1714 			continue;
1715 
1716 		default:
1717 			BUG();
1718 		}
1719 	}
1720 
1721 	/*
1722 	 * Splice any skipped pages to the start of the LRU list. Note that
1723 	 * this disrupts the LRU order when reclaiming for lower zones but
1724 	 * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX
1725 	 * scanning would soon rescan the same pages to skip and put the
1726 	 * system at risk of premature OOM.
1727 	 */
1728 	if (!list_empty(&pages_skipped)) {
1729 		int zid;
1730 
1731 		list_splice(&pages_skipped, src);
1732 		for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1733 			if (!nr_skipped[zid])
1734 				continue;
1735 
1736 			__count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]);
1737 			skipped += nr_skipped[zid];
1738 		}
1739 	}
1740 	*nr_scanned = total_scan;
1741 	trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan,
1742 				    total_scan, skipped, nr_taken, mode, lru);
1743 	update_lru_sizes(lruvec, lru, nr_zone_taken);
1744 	return nr_taken;
1745 }
1746 
1747 /**
1748  * isolate_lru_page - tries to isolate a page from its LRU list
1749  * @page: page to isolate from its LRU list
1750  *
1751  * Isolates a @page from an LRU list, clears PageLRU and adjusts the
1752  * vmstat statistic corresponding to whatever LRU list the page was on.
1753  *
1754  * Returns 0 if the page was removed from an LRU list.
1755  * Returns -EBUSY if the page was not on an LRU list.
1756  *
1757  * The returned page will have PageLRU() cleared.  If it was found on
1758  * the active list, it will have PageActive set.  If it was found on
1759  * the unevictable list, it will have the PageUnevictable bit set. That flag
1760  * may need to be cleared by the caller before letting the page go.
1761  *
1762  * The vmstat statistic corresponding to the list on which the page was
1763  * found will be decremented.
1764  *
1765  * Restrictions:
1766  *
1767  * (1) Must be called with an elevated refcount on the page. This is a
1768  *     fundamentnal difference from isolate_lru_pages (which is called
1769  *     without a stable reference).
1770  * (2) the lru_lock must not be held.
1771  * (3) interrupts must be enabled.
1772  */
1773 int isolate_lru_page(struct page *page)
1774 {
1775 	int ret = -EBUSY;
1776 
1777 	VM_BUG_ON_PAGE(!page_count(page), page);
1778 	WARN_RATELIMIT(PageTail(page), "trying to isolate tail page");
1779 
1780 	if (PageLRU(page)) {
1781 		struct zone *zone = page_zone(page);
1782 		struct lruvec *lruvec;
1783 
1784 		spin_lock_irq(zone_lru_lock(zone));
1785 		lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
1786 		if (PageLRU(page)) {
1787 			int lru = page_lru(page);
1788 			get_page(page);
1789 			ClearPageLRU(page);
1790 			del_page_from_lru_list(page, lruvec, lru);
1791 			ret = 0;
1792 		}
1793 		spin_unlock_irq(zone_lru_lock(zone));
1794 	}
1795 	return ret;
1796 }
1797 
1798 /*
1799  * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
1800  * then get resheduled. When there are massive number of tasks doing page
1801  * allocation, such sleeping direct reclaimers may keep piling up on each CPU,
1802  * the LRU list will go small and be scanned faster than necessary, leading to
1803  * unnecessary swapping, thrashing and OOM.
1804  */
1805 static int too_many_isolated(struct pglist_data *pgdat, int file,
1806 		struct scan_control *sc)
1807 {
1808 	unsigned long inactive, isolated;
1809 
1810 	if (current_is_kswapd())
1811 		return 0;
1812 
1813 	if (!sane_reclaim(sc))
1814 		return 0;
1815 
1816 	if (file) {
1817 		inactive = node_page_state(pgdat, NR_INACTIVE_FILE);
1818 		isolated = node_page_state(pgdat, NR_ISOLATED_FILE);
1819 	} else {
1820 		inactive = node_page_state(pgdat, NR_INACTIVE_ANON);
1821 		isolated = node_page_state(pgdat, NR_ISOLATED_ANON);
1822 	}
1823 
1824 	/*
1825 	 * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they
1826 	 * won't get blocked by normal direct-reclaimers, forming a circular
1827 	 * deadlock.
1828 	 */
1829 	if ((sc->gfp_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
1830 		inactive >>= 3;
1831 
1832 	return isolated > inactive;
1833 }
1834 
1835 static noinline_for_stack void
1836 putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list)
1837 {
1838 	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1839 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1840 	LIST_HEAD(pages_to_free);
1841 
1842 	/*
1843 	 * Put back any unfreeable pages.
1844 	 */
1845 	while (!list_empty(page_list)) {
1846 		struct page *page = lru_to_page(page_list);
1847 		int lru;
1848 
1849 		VM_BUG_ON_PAGE(PageLRU(page), page);
1850 		list_del(&page->lru);
1851 		if (unlikely(!page_evictable(page))) {
1852 			spin_unlock_irq(&pgdat->lru_lock);
1853 			putback_lru_page(page);
1854 			spin_lock_irq(&pgdat->lru_lock);
1855 			continue;
1856 		}
1857 
1858 		lruvec = mem_cgroup_page_lruvec(page, pgdat);
1859 
1860 		SetPageLRU(page);
1861 		lru = page_lru(page);
1862 		add_page_to_lru_list(page, lruvec, lru);
1863 
1864 		if (is_active_lru(lru)) {
1865 			int file = is_file_lru(lru);
1866 			int numpages = hpage_nr_pages(page);
1867 			reclaim_stat->recent_rotated[file] += numpages;
1868 		}
1869 		if (put_page_testzero(page)) {
1870 			__ClearPageLRU(page);
1871 			__ClearPageActive(page);
1872 			del_page_from_lru_list(page, lruvec, lru);
1873 
1874 			if (unlikely(PageCompound(page))) {
1875 				spin_unlock_irq(&pgdat->lru_lock);
1876 				mem_cgroup_uncharge(page);
1877 				(*get_compound_page_dtor(page))(page);
1878 				spin_lock_irq(&pgdat->lru_lock);
1879 			} else
1880 				list_add(&page->lru, &pages_to_free);
1881 		}
1882 	}
1883 
1884 	/*
1885 	 * To save our caller's stack, now use input list for pages to free.
1886 	 */
1887 	list_splice(&pages_to_free, page_list);
1888 }
1889 
1890 /*
1891  * If a kernel thread (such as nfsd for loop-back mounts) services
1892  * a backing device by writing to the page cache it sets PF_LESS_THROTTLE.
1893  * In that case we should only throttle if the backing device it is
1894  * writing to is congested.  In other cases it is safe to throttle.
1895  */
1896 static int current_may_throttle(void)
1897 {
1898 	return !(current->flags & PF_LESS_THROTTLE) ||
1899 		current->backing_dev_info == NULL ||
1900 		bdi_write_congested(current->backing_dev_info);
1901 }
1902 
1903 /*
1904  * shrink_inactive_list() is a helper for shrink_node().  It returns the number
1905  * of reclaimed pages
1906  */
1907 static noinline_for_stack unsigned long
1908 shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
1909 		     struct scan_control *sc, enum lru_list lru)
1910 {
1911 	LIST_HEAD(page_list);
1912 	unsigned long nr_scanned;
1913 	unsigned long nr_reclaimed = 0;
1914 	unsigned long nr_taken;
1915 	struct reclaim_stat stat = {};
1916 	isolate_mode_t isolate_mode = 0;
1917 	int file = is_file_lru(lru);
1918 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1919 	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1920 	bool stalled = false;
1921 
1922 	while (unlikely(too_many_isolated(pgdat, file, sc))) {
1923 		if (stalled)
1924 			return 0;
1925 
1926 		/* wait a bit for the reclaimer. */
1927 		msleep(100);
1928 		stalled = true;
1929 
1930 		/* We are about to die and free our memory. Return now. */
1931 		if (fatal_signal_pending(current))
1932 			return SWAP_CLUSTER_MAX;
1933 	}
1934 
1935 	lru_add_drain();
1936 
1937 	if (!sc->may_unmap)
1938 		isolate_mode |= ISOLATE_UNMAPPED;
1939 
1940 	spin_lock_irq(&pgdat->lru_lock);
1941 
1942 	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
1943 				     &nr_scanned, sc, isolate_mode, lru);
1944 
1945 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
1946 	reclaim_stat->recent_scanned[file] += nr_taken;
1947 
1948 	if (current_is_kswapd()) {
1949 		if (global_reclaim(sc))
1950 			__count_vm_events(PGSCAN_KSWAPD, nr_scanned);
1951 		count_memcg_events(lruvec_memcg(lruvec), PGSCAN_KSWAPD,
1952 				   nr_scanned);
1953 	} else {
1954 		if (global_reclaim(sc))
1955 			__count_vm_events(PGSCAN_DIRECT, nr_scanned);
1956 		count_memcg_events(lruvec_memcg(lruvec), PGSCAN_DIRECT,
1957 				   nr_scanned);
1958 	}
1959 	spin_unlock_irq(&pgdat->lru_lock);
1960 
1961 	if (nr_taken == 0)
1962 		return 0;
1963 
1964 	nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, 0,
1965 				&stat, false);
1966 
1967 	spin_lock_irq(&pgdat->lru_lock);
1968 
1969 	if (current_is_kswapd()) {
1970 		if (global_reclaim(sc))
1971 			__count_vm_events(PGSTEAL_KSWAPD, nr_reclaimed);
1972 		count_memcg_events(lruvec_memcg(lruvec), PGSTEAL_KSWAPD,
1973 				   nr_reclaimed);
1974 	} else {
1975 		if (global_reclaim(sc))
1976 			__count_vm_events(PGSTEAL_DIRECT, nr_reclaimed);
1977 		count_memcg_events(lruvec_memcg(lruvec), PGSTEAL_DIRECT,
1978 				   nr_reclaimed);
1979 	}
1980 
1981 	putback_inactive_pages(lruvec, &page_list);
1982 
1983 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
1984 
1985 	spin_unlock_irq(&pgdat->lru_lock);
1986 
1987 	mem_cgroup_uncharge_list(&page_list);
1988 	free_unref_page_list(&page_list);
1989 
1990 	/*
1991 	 * If dirty pages are scanned that are not queued for IO, it
1992 	 * implies that flushers are not doing their job. This can
1993 	 * happen when memory pressure pushes dirty pages to the end of
1994 	 * the LRU before the dirty limits are breached and the dirty
1995 	 * data has expired. It can also happen when the proportion of
1996 	 * dirty pages grows not through writes but through memory
1997 	 * pressure reclaiming all the clean cache. And in some cases,
1998 	 * the flushers simply cannot keep up with the allocation
1999 	 * rate. Nudge the flusher threads in case they are asleep.
2000 	 */
2001 	if (stat.nr_unqueued_dirty == nr_taken)
2002 		wakeup_flusher_threads(WB_REASON_VMSCAN);
2003 
2004 	sc->nr.dirty += stat.nr_dirty;
2005 	sc->nr.congested += stat.nr_congested;
2006 	sc->nr.unqueued_dirty += stat.nr_unqueued_dirty;
2007 	sc->nr.writeback += stat.nr_writeback;
2008 	sc->nr.immediate += stat.nr_immediate;
2009 	sc->nr.taken += nr_taken;
2010 	if (file)
2011 		sc->nr.file_taken += nr_taken;
2012 
2013 	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
2014 			nr_scanned, nr_reclaimed, &stat, sc->priority, file);
2015 	return nr_reclaimed;
2016 }
2017 
2018 /*
2019  * This moves pages from the active list to the inactive list.
2020  *
2021  * We move them the other way if the page is referenced by one or more
2022  * processes, from rmap.
2023  *
2024  * If the pages are mostly unmapped, the processing is fast and it is
2025  * appropriate to hold zone_lru_lock across the whole operation.  But if
2026  * the pages are mapped, the processing is slow (page_referenced()) so we
2027  * should drop zone_lru_lock around each page.  It's impossible to balance
2028  * this, so instead we remove the pages from the LRU while processing them.
2029  * It is safe to rely on PG_active against the non-LRU pages in here because
2030  * nobody will play with that bit on a non-LRU page.
2031  *
2032  * The downside is that we have to touch page->_refcount against each page.
2033  * But we had to alter page->flags anyway.
2034  *
2035  * Returns the number of pages moved to the given lru.
2036  */
2037 
2038 static unsigned move_active_pages_to_lru(struct lruvec *lruvec,
2039 				     struct list_head *list,
2040 				     struct list_head *pages_to_free,
2041 				     enum lru_list lru)
2042 {
2043 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2044 	struct page *page;
2045 	int nr_pages;
2046 	int nr_moved = 0;
2047 
2048 	while (!list_empty(list)) {
2049 		page = lru_to_page(list);
2050 		lruvec = mem_cgroup_page_lruvec(page, pgdat);
2051 
2052 		VM_BUG_ON_PAGE(PageLRU(page), page);
2053 		SetPageLRU(page);
2054 
2055 		nr_pages = hpage_nr_pages(page);
2056 		update_lru_size(lruvec, lru, page_zonenum(page), nr_pages);
2057 		list_move(&page->lru, &lruvec->lists[lru]);
2058 
2059 		if (put_page_testzero(page)) {
2060 			__ClearPageLRU(page);
2061 			__ClearPageActive(page);
2062 			del_page_from_lru_list(page, lruvec, lru);
2063 
2064 			if (unlikely(PageCompound(page))) {
2065 				spin_unlock_irq(&pgdat->lru_lock);
2066 				mem_cgroup_uncharge(page);
2067 				(*get_compound_page_dtor(page))(page);
2068 				spin_lock_irq(&pgdat->lru_lock);
2069 			} else
2070 				list_add(&page->lru, pages_to_free);
2071 		} else {
2072 			nr_moved += nr_pages;
2073 		}
2074 	}
2075 
2076 	if (!is_active_lru(lru)) {
2077 		__count_vm_events(PGDEACTIVATE, nr_moved);
2078 		count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
2079 				   nr_moved);
2080 	}
2081 
2082 	return nr_moved;
2083 }
2084 
2085 static void shrink_active_list(unsigned long nr_to_scan,
2086 			       struct lruvec *lruvec,
2087 			       struct scan_control *sc,
2088 			       enum lru_list lru)
2089 {
2090 	unsigned long nr_taken;
2091 	unsigned long nr_scanned;
2092 	unsigned long vm_flags;
2093 	LIST_HEAD(l_hold);	/* The pages which were snipped off */
2094 	LIST_HEAD(l_active);
2095 	LIST_HEAD(l_inactive);
2096 	struct page *page;
2097 	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
2098 	unsigned nr_deactivate, nr_activate;
2099 	unsigned nr_rotated = 0;
2100 	isolate_mode_t isolate_mode = 0;
2101 	int file = is_file_lru(lru);
2102 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2103 
2104 	lru_add_drain();
2105 
2106 	if (!sc->may_unmap)
2107 		isolate_mode |= ISOLATE_UNMAPPED;
2108 
2109 	spin_lock_irq(&pgdat->lru_lock);
2110 
2111 	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
2112 				     &nr_scanned, sc, isolate_mode, lru);
2113 
2114 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2115 	reclaim_stat->recent_scanned[file] += nr_taken;
2116 
2117 	__count_vm_events(PGREFILL, nr_scanned);
2118 	count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);
2119 
2120 	spin_unlock_irq(&pgdat->lru_lock);
2121 
2122 	while (!list_empty(&l_hold)) {
2123 		cond_resched();
2124 		page = lru_to_page(&l_hold);
2125 		list_del(&page->lru);
2126 
2127 		if (unlikely(!page_evictable(page))) {
2128 			putback_lru_page(page);
2129 			continue;
2130 		}
2131 
2132 		if (unlikely(buffer_heads_over_limit)) {
2133 			if (page_has_private(page) && trylock_page(page)) {
2134 				if (page_has_private(page))
2135 					try_to_release_page(page, 0);
2136 				unlock_page(page);
2137 			}
2138 		}
2139 
2140 		if (page_referenced(page, 0, sc->target_mem_cgroup,
2141 				    &vm_flags)) {
2142 			nr_rotated += hpage_nr_pages(page);
2143 			/*
2144 			 * Identify referenced, file-backed active pages and
2145 			 * give them one more trip around the active list. So
2146 			 * that executable code get better chances to stay in
2147 			 * memory under moderate memory pressure.  Anon pages
2148 			 * are not likely to be evicted by use-once streaming
2149 			 * IO, plus JVM can create lots of anon VM_EXEC pages,
2150 			 * so we ignore them here.
2151 			 */
2152 			if ((vm_flags & VM_EXEC) && page_is_file_cache(page)) {
2153 				list_add(&page->lru, &l_active);
2154 				continue;
2155 			}
2156 		}
2157 
2158 		ClearPageActive(page);	/* we are de-activating */
2159 		SetPageWorkingset(page);
2160 		list_add(&page->lru, &l_inactive);
2161 	}
2162 
2163 	/*
2164 	 * Move pages back to the lru list.
2165 	 */
2166 	spin_lock_irq(&pgdat->lru_lock);
2167 	/*
2168 	 * Count referenced pages from currently used mappings as rotated,
2169 	 * even though only some of them are actually re-activated.  This
2170 	 * helps balance scan pressure between file and anonymous pages in
2171 	 * get_scan_count.
2172 	 */
2173 	reclaim_stat->recent_rotated[file] += nr_rotated;
2174 
2175 	nr_activate = move_active_pages_to_lru(lruvec, &l_active, &l_hold, lru);
2176 	nr_deactivate = move_active_pages_to_lru(lruvec, &l_inactive, &l_hold, lru - LRU_ACTIVE);
2177 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2178 	spin_unlock_irq(&pgdat->lru_lock);
2179 
2180 	mem_cgroup_uncharge_list(&l_hold);
2181 	free_unref_page_list(&l_hold);
2182 	trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,
2183 			nr_deactivate, nr_rotated, sc->priority, file);
2184 }
2185 
2186 /*
2187  * The inactive anon list should be small enough that the VM never has
2188  * to do too much work.
2189  *
2190  * The inactive file list should be small enough to leave most memory
2191  * to the established workingset on the scan-resistant active list,
2192  * but large enough to avoid thrashing the aggregate readahead window.
2193  *
2194  * Both inactive lists should also be large enough that each inactive
2195  * page has a chance to be referenced again before it is reclaimed.
2196  *
2197  * If that fails and refaulting is observed, the inactive list grows.
2198  *
2199  * The inactive_ratio is the target ratio of ACTIVE to INACTIVE pages
2200  * on this LRU, maintained by the pageout code. An inactive_ratio
2201  * of 3 means 3:1 or 25% of the pages are kept on the inactive list.
2202  *
2203  * total     target    max
2204  * memory    ratio     inactive
2205  * -------------------------------------
2206  *   10MB       1         5MB
2207  *  100MB       1        50MB
2208  *    1GB       3       250MB
2209  *   10GB      10       0.9GB
2210  *  100GB      31         3GB
2211  *    1TB     101        10GB
2212  *   10TB     320        32GB
2213  */
2214 static bool inactive_list_is_low(struct lruvec *lruvec, bool file,
2215 				 struct mem_cgroup *memcg,
2216 				 struct scan_control *sc, bool actual_reclaim)
2217 {
2218 	enum lru_list active_lru = file * LRU_FILE + LRU_ACTIVE;
2219 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2220 	enum lru_list inactive_lru = file * LRU_FILE;
2221 	unsigned long inactive, active;
2222 	unsigned long inactive_ratio;
2223 	unsigned long refaults;
2224 	unsigned long gb;
2225 
2226 	/*
2227 	 * If we don't have swap space, anonymous page deactivation
2228 	 * is pointless.
2229 	 */
2230 	if (!file && !total_swap_pages)
2231 		return false;
2232 
2233 	inactive = lruvec_lru_size(lruvec, inactive_lru, sc->reclaim_idx);
2234 	active = lruvec_lru_size(lruvec, active_lru, sc->reclaim_idx);
2235 
2236 	if (memcg)
2237 		refaults = memcg_page_state(memcg, WORKINGSET_ACTIVATE);
2238 	else
2239 		refaults = node_page_state(pgdat, WORKINGSET_ACTIVATE);
2240 
2241 	/*
2242 	 * When refaults are being observed, it means a new workingset
2243 	 * is being established. Disable active list protection to get
2244 	 * rid of the stale workingset quickly.
2245 	 */
2246 	if (file && actual_reclaim && lruvec->refaults != refaults) {
2247 		inactive_ratio = 0;
2248 	} else {
2249 		gb = (inactive + active) >> (30 - PAGE_SHIFT);
2250 		if (gb)
2251 			inactive_ratio = int_sqrt(10 * gb);
2252 		else
2253 			inactive_ratio = 1;
2254 	}
2255 
2256 	if (actual_reclaim)
2257 		trace_mm_vmscan_inactive_list_is_low(pgdat->node_id, sc->reclaim_idx,
2258 			lruvec_lru_size(lruvec, inactive_lru, MAX_NR_ZONES), inactive,
2259 			lruvec_lru_size(lruvec, active_lru, MAX_NR_ZONES), active,
2260 			inactive_ratio, file);
2261 
2262 	return inactive * inactive_ratio < active;
2263 }
2264 
2265 static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2266 				 struct lruvec *lruvec, struct mem_cgroup *memcg,
2267 				 struct scan_control *sc)
2268 {
2269 	if (is_active_lru(lru)) {
2270 		if (inactive_list_is_low(lruvec, is_file_lru(lru),
2271 					 memcg, sc, true))
2272 			shrink_active_list(nr_to_scan, lruvec, sc, lru);
2273 		return 0;
2274 	}
2275 
2276 	return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2277 }
2278 
2279 enum scan_balance {
2280 	SCAN_EQUAL,
2281 	SCAN_FRACT,
2282 	SCAN_ANON,
2283 	SCAN_FILE,
2284 };
2285 
2286 /*
2287  * Determine how aggressively the anon and file LRU lists should be
2288  * scanned.  The relative value of each set of LRU lists is determined
2289  * by looking at the fraction of the pages scanned we did rotate back
2290  * onto the active list instead of evict.
2291  *
2292  * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
2293  * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
2294  */
2295 static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg,
2296 			   struct scan_control *sc, unsigned long *nr,
2297 			   unsigned long *lru_pages)
2298 {
2299 	int swappiness = mem_cgroup_swappiness(memcg);
2300 	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
2301 	u64 fraction[2];
2302 	u64 denominator = 0;	/* gcc */
2303 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2304 	unsigned long anon_prio, file_prio;
2305 	enum scan_balance scan_balance;
2306 	unsigned long anon, file;
2307 	unsigned long ap, fp;
2308 	enum lru_list lru;
2309 
2310 	/* If we have no swap space, do not bother scanning anon pages. */
2311 	if (!sc->may_swap || mem_cgroup_get_nr_swap_pages(memcg) <= 0) {
2312 		scan_balance = SCAN_FILE;
2313 		goto out;
2314 	}
2315 
2316 	/*
2317 	 * Global reclaim will swap to prevent OOM even with no
2318 	 * swappiness, but memcg users want to use this knob to
2319 	 * disable swapping for individual groups completely when
2320 	 * using the memory controller's swap limit feature would be
2321 	 * too expensive.
2322 	 */
2323 	if (!global_reclaim(sc) && !swappiness) {
2324 		scan_balance = SCAN_FILE;
2325 		goto out;
2326 	}
2327 
2328 	/*
2329 	 * Do not apply any pressure balancing cleverness when the
2330 	 * system is close to OOM, scan both anon and file equally
2331 	 * (unless the swappiness setting disagrees with swapping).
2332 	 */
2333 	if (!sc->priority && swappiness) {
2334 		scan_balance = SCAN_EQUAL;
2335 		goto out;
2336 	}
2337 
2338 	/*
2339 	 * Prevent the reclaimer from falling into the cache trap: as
2340 	 * cache pages start out inactive, every cache fault will tip
2341 	 * the scan balance towards the file LRU.  And as the file LRU
2342 	 * shrinks, so does the window for rotation from references.
2343 	 * This means we have a runaway feedback loop where a tiny
2344 	 * thrashing file LRU becomes infinitely more attractive than
2345 	 * anon pages.  Try to detect this based on file LRU size.
2346 	 */
2347 	if (global_reclaim(sc)) {
2348 		unsigned long pgdatfile;
2349 		unsigned long pgdatfree;
2350 		int z;
2351 		unsigned long total_high_wmark = 0;
2352 
2353 		pgdatfree = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
2354 		pgdatfile = node_page_state(pgdat, NR_ACTIVE_FILE) +
2355 			   node_page_state(pgdat, NR_INACTIVE_FILE);
2356 
2357 		for (z = 0; z < MAX_NR_ZONES; z++) {
2358 			struct zone *zone = &pgdat->node_zones[z];
2359 			if (!managed_zone(zone))
2360 				continue;
2361 
2362 			total_high_wmark += high_wmark_pages(zone);
2363 		}
2364 
2365 		if (unlikely(pgdatfile + pgdatfree <= total_high_wmark)) {
2366 			/*
2367 			 * Force SCAN_ANON if there are enough inactive
2368 			 * anonymous pages on the LRU in eligible zones.
2369 			 * Otherwise, the small LRU gets thrashed.
2370 			 */
2371 			if (!inactive_list_is_low(lruvec, false, memcg, sc, false) &&
2372 			    lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, sc->reclaim_idx)
2373 					>> sc->priority) {
2374 				scan_balance = SCAN_ANON;
2375 				goto out;
2376 			}
2377 		}
2378 	}
2379 
2380 	/*
2381 	 * If there is enough inactive page cache, i.e. if the size of the
2382 	 * inactive list is greater than that of the active list *and* the
2383 	 * inactive list actually has some pages to scan on this priority, we
2384 	 * do not reclaim anything from the anonymous working set right now.
2385 	 * Without the second condition we could end up never scanning an
2386 	 * lruvec even if it has plenty of old anonymous pages unless the
2387 	 * system is under heavy pressure.
2388 	 */
2389 	if (!inactive_list_is_low(lruvec, true, memcg, sc, false) &&
2390 	    lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, sc->reclaim_idx) >> sc->priority) {
2391 		scan_balance = SCAN_FILE;
2392 		goto out;
2393 	}
2394 
2395 	scan_balance = SCAN_FRACT;
2396 
2397 	/*
2398 	 * With swappiness at 100, anonymous and file have the same priority.
2399 	 * This scanning priority is essentially the inverse of IO cost.
2400 	 */
2401 	anon_prio = swappiness;
2402 	file_prio = 200 - anon_prio;
2403 
2404 	/*
2405 	 * OK, so we have swap space and a fair amount of page cache
2406 	 * pages.  We use the recently rotated / recently scanned
2407 	 * ratios to determine how valuable each cache is.
2408 	 *
2409 	 * Because workloads change over time (and to avoid overflow)
2410 	 * we keep these statistics as a floating average, which ends
2411 	 * up weighing recent references more than old ones.
2412 	 *
2413 	 * anon in [0], file in [1]
2414 	 */
2415 
2416 	anon  = lruvec_lru_size(lruvec, LRU_ACTIVE_ANON, MAX_NR_ZONES) +
2417 		lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, MAX_NR_ZONES);
2418 	file  = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES) +
2419 		lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, MAX_NR_ZONES);
2420 
2421 	spin_lock_irq(&pgdat->lru_lock);
2422 	if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
2423 		reclaim_stat->recent_scanned[0] /= 2;
2424 		reclaim_stat->recent_rotated[0] /= 2;
2425 	}
2426 
2427 	if (unlikely(reclaim_stat->recent_scanned[1] > file / 4)) {
2428 		reclaim_stat->recent_scanned[1] /= 2;
2429 		reclaim_stat->recent_rotated[1] /= 2;
2430 	}
2431 
2432 	/*
2433 	 * The amount of pressure on anon vs file pages is inversely
2434 	 * proportional to the fraction of recently scanned pages on
2435 	 * each list that were recently referenced and in active use.
2436 	 */
2437 	ap = anon_prio * (reclaim_stat->recent_scanned[0] + 1);
2438 	ap /= reclaim_stat->recent_rotated[0] + 1;
2439 
2440 	fp = file_prio * (reclaim_stat->recent_scanned[1] + 1);
2441 	fp /= reclaim_stat->recent_rotated[1] + 1;
2442 	spin_unlock_irq(&pgdat->lru_lock);
2443 
2444 	fraction[0] = ap;
2445 	fraction[1] = fp;
2446 	denominator = ap + fp + 1;
2447 out:
2448 	*lru_pages = 0;
2449 	for_each_evictable_lru(lru) {
2450 		int file = is_file_lru(lru);
2451 		unsigned long size;
2452 		unsigned long scan;
2453 
2454 		size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
2455 		scan = size >> sc->priority;
2456 		/*
2457 		 * If the cgroup's already been deleted, make sure to
2458 		 * scrape out the remaining cache.
2459 		 */
2460 		if (!scan && !mem_cgroup_online(memcg))
2461 			scan = min(size, SWAP_CLUSTER_MAX);
2462 
2463 		switch (scan_balance) {
2464 		case SCAN_EQUAL:
2465 			/* Scan lists relative to size */
2466 			break;
2467 		case SCAN_FRACT:
2468 			/*
2469 			 * Scan types proportional to swappiness and
2470 			 * their relative recent reclaim efficiency.
2471 			 * Make sure we don't miss the last page
2472 			 * because of a round-off error.
2473 			 */
2474 			scan = DIV64_U64_ROUND_UP(scan * fraction[file],
2475 						  denominator);
2476 			break;
2477 		case SCAN_FILE:
2478 		case SCAN_ANON:
2479 			/* Scan one type exclusively */
2480 			if ((scan_balance == SCAN_FILE) != file) {
2481 				size = 0;
2482 				scan = 0;
2483 			}
2484 			break;
2485 		default:
2486 			/* Look ma, no brain */
2487 			BUG();
2488 		}
2489 
2490 		*lru_pages += size;
2491 		nr[lru] = scan;
2492 	}
2493 }
2494 
2495 /*
2496  * This is a basic per-node page freer.  Used by both kswapd and direct reclaim.
2497  */
2498 static void shrink_node_memcg(struct pglist_data *pgdat, struct mem_cgroup *memcg,
2499 			      struct scan_control *sc, unsigned long *lru_pages)
2500 {
2501 	struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
2502 	unsigned long nr[NR_LRU_LISTS];
2503 	unsigned long targets[NR_LRU_LISTS];
2504 	unsigned long nr_to_scan;
2505 	enum lru_list lru;
2506 	unsigned long nr_reclaimed = 0;
2507 	unsigned long nr_to_reclaim = sc->nr_to_reclaim;
2508 	struct blk_plug plug;
2509 	bool scan_adjusted;
2510 
2511 	get_scan_count(lruvec, memcg, sc, nr, lru_pages);
2512 
2513 	/* Record the original scan target for proportional adjustments later */
2514 	memcpy(targets, nr, sizeof(nr));
2515 
2516 	/*
2517 	 * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal
2518 	 * event that can occur when there is little memory pressure e.g.
2519 	 * multiple streaming readers/writers. Hence, we do not abort scanning
2520 	 * when the requested number of pages are reclaimed when scanning at
2521 	 * DEF_PRIORITY on the assumption that the fact we are direct
2522 	 * reclaiming implies that kswapd is not keeping up and it is best to
2523 	 * do a batch of work at once. For memcg reclaim one check is made to
2524 	 * abort proportional reclaim if either the file or anon lru has already
2525 	 * dropped to zero at the first pass.
2526 	 */
2527 	scan_adjusted = (global_reclaim(sc) && !current_is_kswapd() &&
2528 			 sc->priority == DEF_PRIORITY);
2529 
2530 	blk_start_plug(&plug);
2531 	while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
2532 					nr[LRU_INACTIVE_FILE]) {
2533 		unsigned long nr_anon, nr_file, percentage;
2534 		unsigned long nr_scanned;
2535 
2536 		for_each_evictable_lru(lru) {
2537 			if (nr[lru]) {
2538 				nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
2539 				nr[lru] -= nr_to_scan;
2540 
2541 				nr_reclaimed += shrink_list(lru, nr_to_scan,
2542 							    lruvec, memcg, sc);
2543 			}
2544 		}
2545 
2546 		cond_resched();
2547 
2548 		if (nr_reclaimed < nr_to_reclaim || scan_adjusted)
2549 			continue;
2550 
2551 		/*
2552 		 * For kswapd and memcg, reclaim at least the number of pages
2553 		 * requested. Ensure that the anon and file LRUs are scanned
2554 		 * proportionally what was requested by get_scan_count(). We
2555 		 * stop reclaiming one LRU and reduce the amount scanning
2556 		 * proportional to the original scan target.
2557 		 */
2558 		nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
2559 		nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];
2560 
2561 		/*
2562 		 * It's just vindictive to attack the larger once the smaller
2563 		 * has gone to zero.  And given the way we stop scanning the
2564 		 * smaller below, this makes sure that we only make one nudge
2565 		 * towards proportionality once we've got nr_to_reclaim.
2566 		 */
2567 		if (!nr_file || !nr_anon)
2568 			break;
2569 
2570 		if (nr_file > nr_anon) {
2571 			unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
2572 						targets[LRU_ACTIVE_ANON] + 1;
2573 			lru = LRU_BASE;
2574 			percentage = nr_anon * 100 / scan_target;
2575 		} else {
2576 			unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
2577 						targets[LRU_ACTIVE_FILE] + 1;
2578 			lru = LRU_FILE;
2579 			percentage = nr_file * 100 / scan_target;
2580 		}
2581 
2582 		/* Stop scanning the smaller of the LRU */
2583 		nr[lru] = 0;
2584 		nr[lru + LRU_ACTIVE] = 0;
2585 
2586 		/*
2587 		 * Recalculate the other LRU scan count based on its original
2588 		 * scan target and the percentage scanning already complete
2589 		 */
2590 		lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
2591 		nr_scanned = targets[lru] - nr[lru];
2592 		nr[lru] = targets[lru] * (100 - percentage) / 100;
2593 		nr[lru] -= min(nr[lru], nr_scanned);
2594 
2595 		lru += LRU_ACTIVE;
2596 		nr_scanned = targets[lru] - nr[lru];
2597 		nr[lru] = targets[lru] * (100 - percentage) / 100;
2598 		nr[lru] -= min(nr[lru], nr_scanned);
2599 
2600 		scan_adjusted = true;
2601 	}
2602 	blk_finish_plug(&plug);
2603 	sc->nr_reclaimed += nr_reclaimed;
2604 
2605 	/*
2606 	 * Even if we did not try to evict anon pages at all, we want to
2607 	 * rebalance the anon lru active/inactive ratio.
2608 	 */
2609 	if (inactive_list_is_low(lruvec, false, memcg, sc, true))
2610 		shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
2611 				   sc, LRU_ACTIVE_ANON);
2612 }
2613 
2614 /* Use reclaim/compaction for costly allocs or under memory pressure */
2615 static bool in_reclaim_compaction(struct scan_control *sc)
2616 {
2617 	if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
2618 			(sc->order > PAGE_ALLOC_COSTLY_ORDER ||
2619 			 sc->priority < DEF_PRIORITY - 2))
2620 		return true;
2621 
2622 	return false;
2623 }
2624 
2625 /*
2626  * Reclaim/compaction is used for high-order allocation requests. It reclaims
2627  * order-0 pages before compacting the zone. should_continue_reclaim() returns
2628  * true if more pages should be reclaimed such that when the page allocator
2629  * calls try_to_compact_zone() that it will have enough free pages to succeed.
2630  * It will give up earlier than that if there is difficulty reclaiming pages.
2631  */
2632 static inline bool should_continue_reclaim(struct pglist_data *pgdat,
2633 					unsigned long nr_reclaimed,
2634 					unsigned long nr_scanned,
2635 					struct scan_control *sc)
2636 {
2637 	unsigned long pages_for_compaction;
2638 	unsigned long inactive_lru_pages;
2639 	int z;
2640 
2641 	/* If not in reclaim/compaction mode, stop */
2642 	if (!in_reclaim_compaction(sc))
2643 		return false;
2644 
2645 	/* Consider stopping depending on scan and reclaim activity */
2646 	if (sc->gfp_mask & __GFP_RETRY_MAYFAIL) {
2647 		/*
2648 		 * For __GFP_RETRY_MAYFAIL allocations, stop reclaiming if the
2649 		 * full LRU list has been scanned and we are still failing
2650 		 * to reclaim pages. This full LRU scan is potentially
2651 		 * expensive but a __GFP_RETRY_MAYFAIL caller really wants to succeed
2652 		 */
2653 		if (!nr_reclaimed && !nr_scanned)
2654 			return false;
2655 	} else {
2656 		/*
2657 		 * For non-__GFP_RETRY_MAYFAIL allocations which can presumably
2658 		 * fail without consequence, stop if we failed to reclaim
2659 		 * any pages from the last SWAP_CLUSTER_MAX number of
2660 		 * pages that were scanned. This will return to the
2661 		 * caller faster at the risk reclaim/compaction and
2662 		 * the resulting allocation attempt fails
2663 		 */
2664 		if (!nr_reclaimed)
2665 			return false;
2666 	}
2667 
2668 	/*
2669 	 * If we have not reclaimed enough pages for compaction and the
2670 	 * inactive lists are large enough, continue reclaiming
2671 	 */
2672 	pages_for_compaction = compact_gap(sc->order);
2673 	inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
2674 	if (get_nr_swap_pages() > 0)
2675 		inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
2676 	if (sc->nr_reclaimed < pages_for_compaction &&
2677 			inactive_lru_pages > pages_for_compaction)
2678 		return true;
2679 
2680 	/* If compaction would go ahead or the allocation would succeed, stop */
2681 	for (z = 0; z <= sc->reclaim_idx; z++) {
2682 		struct zone *zone = &pgdat->node_zones[z];
2683 		if (!managed_zone(zone))
2684 			continue;
2685 
2686 		switch (compaction_suitable(zone, sc->order, 0, sc->reclaim_idx)) {
2687 		case COMPACT_SUCCESS:
2688 		case COMPACT_CONTINUE:
2689 			return false;
2690 		default:
2691 			/* check next zone */
2692 			;
2693 		}
2694 	}
2695 	return true;
2696 }
2697 
2698 static bool pgdat_memcg_congested(pg_data_t *pgdat, struct mem_cgroup *memcg)
2699 {
2700 	return test_bit(PGDAT_CONGESTED, &pgdat->flags) ||
2701 		(memcg && memcg_congested(pgdat, memcg));
2702 }
2703 
2704 static bool shrink_node(pg_data_t *pgdat, struct scan_control *sc)
2705 {
2706 	struct reclaim_state *reclaim_state = current->reclaim_state;
2707 	unsigned long nr_reclaimed, nr_scanned;
2708 	bool reclaimable = false;
2709 
2710 	do {
2711 		struct mem_cgroup *root = sc->target_mem_cgroup;
2712 		struct mem_cgroup_reclaim_cookie reclaim = {
2713 			.pgdat = pgdat,
2714 			.priority = sc->priority,
2715 		};
2716 		unsigned long node_lru_pages = 0;
2717 		struct mem_cgroup *memcg;
2718 
2719 		memset(&sc->nr, 0, sizeof(sc->nr));
2720 
2721 		nr_reclaimed = sc->nr_reclaimed;
2722 		nr_scanned = sc->nr_scanned;
2723 
2724 		memcg = mem_cgroup_iter(root, NULL, &reclaim);
2725 		do {
2726 			unsigned long lru_pages;
2727 			unsigned long reclaimed;
2728 			unsigned long scanned;
2729 
2730 			switch (mem_cgroup_protected(root, memcg)) {
2731 			case MEMCG_PROT_MIN:
2732 				/*
2733 				 * Hard protection.
2734 				 * If there is no reclaimable memory, OOM.
2735 				 */
2736 				continue;
2737 			case MEMCG_PROT_LOW:
2738 				/*
2739 				 * Soft protection.
2740 				 * Respect the protection only as long as
2741 				 * there is an unprotected supply
2742 				 * of reclaimable memory from other cgroups.
2743 				 */
2744 				if (!sc->memcg_low_reclaim) {
2745 					sc->memcg_low_skipped = 1;
2746 					continue;
2747 				}
2748 				memcg_memory_event(memcg, MEMCG_LOW);
2749 				break;
2750 			case MEMCG_PROT_NONE:
2751 				break;
2752 			}
2753 
2754 			reclaimed = sc->nr_reclaimed;
2755 			scanned = sc->nr_scanned;
2756 			shrink_node_memcg(pgdat, memcg, sc, &lru_pages);
2757 			node_lru_pages += lru_pages;
2758 
2759 			shrink_slab(sc->gfp_mask, pgdat->node_id,
2760 				    memcg, sc->priority);
2761 
2762 			/* Record the group's reclaim efficiency */
2763 			vmpressure(sc->gfp_mask, memcg, false,
2764 				   sc->nr_scanned - scanned,
2765 				   sc->nr_reclaimed - reclaimed);
2766 
2767 			/*
2768 			 * Direct reclaim and kswapd have to scan all memory
2769 			 * cgroups to fulfill the overall scan target for the
2770 			 * node.
2771 			 *
2772 			 * Limit reclaim, on the other hand, only cares about
2773 			 * nr_to_reclaim pages to be reclaimed and it will
2774 			 * retry with decreasing priority if one round over the
2775 			 * whole hierarchy is not sufficient.
2776 			 */
2777 			if (!global_reclaim(sc) &&
2778 					sc->nr_reclaimed >= sc->nr_to_reclaim) {
2779 				mem_cgroup_iter_break(root, memcg);
2780 				break;
2781 			}
2782 		} while ((memcg = mem_cgroup_iter(root, memcg, &reclaim)));
2783 
2784 		if (reclaim_state) {
2785 			sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2786 			reclaim_state->reclaimed_slab = 0;
2787 		}
2788 
2789 		/* Record the subtree's reclaim efficiency */
2790 		vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
2791 			   sc->nr_scanned - nr_scanned,
2792 			   sc->nr_reclaimed - nr_reclaimed);
2793 
2794 		if (sc->nr_reclaimed - nr_reclaimed)
2795 			reclaimable = true;
2796 
2797 		if (current_is_kswapd()) {
2798 			/*
2799 			 * If reclaim is isolating dirty pages under writeback,
2800 			 * it implies that the long-lived page allocation rate
2801 			 * is exceeding the page laundering rate. Either the
2802 			 * global limits are not being effective at throttling
2803 			 * processes due to the page distribution throughout
2804 			 * zones or there is heavy usage of a slow backing
2805 			 * device. The only option is to throttle from reclaim
2806 			 * context which is not ideal as there is no guarantee
2807 			 * the dirtying process is throttled in the same way
2808 			 * balance_dirty_pages() manages.
2809 			 *
2810 			 * Once a node is flagged PGDAT_WRITEBACK, kswapd will
2811 			 * count the number of pages under pages flagged for
2812 			 * immediate reclaim and stall if any are encountered
2813 			 * in the nr_immediate check below.
2814 			 */
2815 			if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
2816 				set_bit(PGDAT_WRITEBACK, &pgdat->flags);
2817 
2818 			/*
2819 			 * Tag a node as congested if all the dirty pages
2820 			 * scanned were backed by a congested BDI and
2821 			 * wait_iff_congested will stall.
2822 			 */
2823 			if (sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
2824 				set_bit(PGDAT_CONGESTED, &pgdat->flags);
2825 
2826 			/* Allow kswapd to start writing pages during reclaim.*/
2827 			if (sc->nr.unqueued_dirty == sc->nr.file_taken)
2828 				set_bit(PGDAT_DIRTY, &pgdat->flags);
2829 
2830 			/*
2831 			 * If kswapd scans pages marked marked for immediate
2832 			 * reclaim and under writeback (nr_immediate), it
2833 			 * implies that pages are cycling through the LRU
2834 			 * faster than they are written so also forcibly stall.
2835 			 */
2836 			if (sc->nr.immediate)
2837 				congestion_wait(BLK_RW_ASYNC, HZ/10);
2838 		}
2839 
2840 		/*
2841 		 * Legacy memcg will stall in page writeback so avoid forcibly
2842 		 * stalling in wait_iff_congested().
2843 		 */
2844 		if (!global_reclaim(sc) && sane_reclaim(sc) &&
2845 		    sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
2846 			set_memcg_congestion(pgdat, root, true);
2847 
2848 		/*
2849 		 * Stall direct reclaim for IO completions if underlying BDIs
2850 		 * and node is congested. Allow kswapd to continue until it
2851 		 * starts encountering unqueued dirty pages or cycling through
2852 		 * the LRU too quickly.
2853 		 */
2854 		if (!sc->hibernation_mode && !current_is_kswapd() &&
2855 		   current_may_throttle() && pgdat_memcg_congested(pgdat, root))
2856 			wait_iff_congested(BLK_RW_ASYNC, HZ/10);
2857 
2858 	} while (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
2859 					 sc->nr_scanned - nr_scanned, sc));
2860 
2861 	/*
2862 	 * Kswapd gives up on balancing particular nodes after too
2863 	 * many failures to reclaim anything from them and goes to
2864 	 * sleep. On reclaim progress, reset the failure counter. A
2865 	 * successful direct reclaim run will revive a dormant kswapd.
2866 	 */
2867 	if (reclaimable)
2868 		pgdat->kswapd_failures = 0;
2869 
2870 	return reclaimable;
2871 }
2872 
2873 /*
2874  * Returns true if compaction should go ahead for a costly-order request, or
2875  * the allocation would already succeed without compaction. Return false if we
2876  * should reclaim first.
2877  */
2878 static inline bool compaction_ready(struct zone *zone, struct scan_control *sc)
2879 {
2880 	unsigned long watermark;
2881 	enum compact_result suitable;
2882 
2883 	suitable = compaction_suitable(zone, sc->order, 0, sc->reclaim_idx);
2884 	if (suitable == COMPACT_SUCCESS)
2885 		/* Allocation should succeed already. Don't reclaim. */
2886 		return true;
2887 	if (suitable == COMPACT_SKIPPED)
2888 		/* Compaction cannot yet proceed. Do reclaim. */
2889 		return false;
2890 
2891 	/*
2892 	 * Compaction is already possible, but it takes time to run and there
2893 	 * are potentially other callers using the pages just freed. So proceed
2894 	 * with reclaim to make a buffer of free pages available to give
2895 	 * compaction a reasonable chance of completing and allocating the page.
2896 	 * Note that we won't actually reclaim the whole buffer in one attempt
2897 	 * as the target watermark in should_continue_reclaim() is lower. But if
2898 	 * we are already above the high+gap watermark, don't reclaim at all.
2899 	 */
2900 	watermark = high_wmark_pages(zone) + compact_gap(sc->order);
2901 
2902 	return zone_watermark_ok_safe(zone, 0, watermark, sc->reclaim_idx);
2903 }
2904 
2905 /*
2906  * This is the direct reclaim path, for page-allocating processes.  We only
2907  * try to reclaim pages from zones which will satisfy the caller's allocation
2908  * request.
2909  *
2910  * If a zone is deemed to be full of pinned pages then just give it a light
2911  * scan then give up on it.
2912  */
2913 static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
2914 {
2915 	struct zoneref *z;
2916 	struct zone *zone;
2917 	unsigned long nr_soft_reclaimed;
2918 	unsigned long nr_soft_scanned;
2919 	gfp_t orig_mask;
2920 	pg_data_t *last_pgdat = NULL;
2921 
2922 	/*
2923 	 * If the number of buffer_heads in the machine exceeds the maximum
2924 	 * allowed level, force direct reclaim to scan the highmem zone as
2925 	 * highmem pages could be pinning lowmem pages storing buffer_heads
2926 	 */
2927 	orig_mask = sc->gfp_mask;
2928 	if (buffer_heads_over_limit) {
2929 		sc->gfp_mask |= __GFP_HIGHMEM;
2930 		sc->reclaim_idx = gfp_zone(sc->gfp_mask);
2931 	}
2932 
2933 	for_each_zone_zonelist_nodemask(zone, z, zonelist,
2934 					sc->reclaim_idx, sc->nodemask) {
2935 		/*
2936 		 * Take care memory controller reclaiming has small influence
2937 		 * to global LRU.
2938 		 */
2939 		if (global_reclaim(sc)) {
2940 			if (!cpuset_zone_allowed(zone,
2941 						 GFP_KERNEL | __GFP_HARDWALL))
2942 				continue;
2943 
2944 			/*
2945 			 * If we already have plenty of memory free for
2946 			 * compaction in this zone, don't free any more.
2947 			 * Even though compaction is invoked for any
2948 			 * non-zero order, only frequent costly order
2949 			 * reclamation is disruptive enough to become a
2950 			 * noticeable problem, like transparent huge
2951 			 * page allocations.
2952 			 */
2953 			if (IS_ENABLED(CONFIG_COMPACTION) &&
2954 			    sc->order > PAGE_ALLOC_COSTLY_ORDER &&
2955 			    compaction_ready(zone, sc)) {
2956 				sc->compaction_ready = true;
2957 				continue;
2958 			}
2959 
2960 			/*
2961 			 * Shrink each node in the zonelist once. If the
2962 			 * zonelist is ordered by zone (not the default) then a
2963 			 * node may be shrunk multiple times but in that case
2964 			 * the user prefers lower zones being preserved.
2965 			 */
2966 			if (zone->zone_pgdat == last_pgdat)
2967 				continue;
2968 
2969 			/*
2970 			 * This steals pages from memory cgroups over softlimit
2971 			 * and returns the number of reclaimed pages and
2972 			 * scanned pages. This works for global memory pressure
2973 			 * and balancing, not for a memcg's limit.
2974 			 */
2975 			nr_soft_scanned = 0;
2976 			nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone->zone_pgdat,
2977 						sc->order, sc->gfp_mask,
2978 						&nr_soft_scanned);
2979 			sc->nr_reclaimed += nr_soft_reclaimed;
2980 			sc->nr_scanned += nr_soft_scanned;
2981 			/* need some check for avoid more shrink_zone() */
2982 		}
2983 
2984 		/* See comment about same check for global reclaim above */
2985 		if (zone->zone_pgdat == last_pgdat)
2986 			continue;
2987 		last_pgdat = zone->zone_pgdat;
2988 		shrink_node(zone->zone_pgdat, sc);
2989 	}
2990 
2991 	/*
2992 	 * Restore to original mask to avoid the impact on the caller if we
2993 	 * promoted it to __GFP_HIGHMEM.
2994 	 */
2995 	sc->gfp_mask = orig_mask;
2996 }
2997 
2998 static void snapshot_refaults(struct mem_cgroup *root_memcg, pg_data_t *pgdat)
2999 {
3000 	struct mem_cgroup *memcg;
3001 
3002 	memcg = mem_cgroup_iter(root_memcg, NULL, NULL);
3003 	do {
3004 		unsigned long refaults;
3005 		struct lruvec *lruvec;
3006 
3007 		if (memcg)
3008 			refaults = memcg_page_state(memcg, WORKINGSET_ACTIVATE);
3009 		else
3010 			refaults = node_page_state(pgdat, WORKINGSET_ACTIVATE);
3011 
3012 		lruvec = mem_cgroup_lruvec(pgdat, memcg);
3013 		lruvec->refaults = refaults;
3014 	} while ((memcg = mem_cgroup_iter(root_memcg, memcg, NULL)));
3015 }
3016 
3017 /*
3018  * This is the main entry point to direct page reclaim.
3019  *
3020  * If a full scan of the inactive list fails to free enough memory then we
3021  * are "out of memory" and something needs to be killed.
3022  *
3023  * If the caller is !__GFP_FS then the probability of a failure is reasonably
3024  * high - the zone may be full of dirty or under-writeback pages, which this
3025  * caller can't do much about.  We kick the writeback threads and take explicit
3026  * naps in the hope that some of these pages can be written.  But if the
3027  * allocating task holds filesystem locks which prevent writeout this might not
3028  * work, and the allocation attempt will fail.
3029  *
3030  * returns:	0, if no pages reclaimed
3031  * 		else, the number of pages reclaimed
3032  */
3033 static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
3034 					  struct scan_control *sc)
3035 {
3036 	int initial_priority = sc->priority;
3037 	pg_data_t *last_pgdat;
3038 	struct zoneref *z;
3039 	struct zone *zone;
3040 retry:
3041 	delayacct_freepages_start();
3042 
3043 	if (global_reclaim(sc))
3044 		__count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);
3045 
3046 	do {
3047 		vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup,
3048 				sc->priority);
3049 		sc->nr_scanned = 0;
3050 		shrink_zones(zonelist, sc);
3051 
3052 		if (sc->nr_reclaimed >= sc->nr_to_reclaim)
3053 			break;
3054 
3055 		if (sc->compaction_ready)
3056 			break;
3057 
3058 		/*
3059 		 * If we're getting trouble reclaiming, start doing
3060 		 * writepage even in laptop mode.
3061 		 */
3062 		if (sc->priority < DEF_PRIORITY - 2)
3063 			sc->may_writepage = 1;
3064 	} while (--sc->priority >= 0);
3065 
3066 	last_pgdat = NULL;
3067 	for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx,
3068 					sc->nodemask) {
3069 		if (zone->zone_pgdat == last_pgdat)
3070 			continue;
3071 		last_pgdat = zone->zone_pgdat;
3072 		snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat);
3073 		set_memcg_congestion(last_pgdat, sc->target_mem_cgroup, false);
3074 	}
3075 
3076 	delayacct_freepages_end();
3077 
3078 	if (sc->nr_reclaimed)
3079 		return sc->nr_reclaimed;
3080 
3081 	/* Aborted reclaim to try compaction? don't OOM, then */
3082 	if (sc->compaction_ready)
3083 		return 1;
3084 
3085 	/* Untapped cgroup reserves?  Don't OOM, retry. */
3086 	if (sc->memcg_low_skipped) {
3087 		sc->priority = initial_priority;
3088 		sc->memcg_low_reclaim = 1;
3089 		sc->memcg_low_skipped = 0;
3090 		goto retry;
3091 	}
3092 
3093 	return 0;
3094 }
3095 
3096 static bool allow_direct_reclaim(pg_data_t *pgdat)
3097 {
3098 	struct zone *zone;
3099 	unsigned long pfmemalloc_reserve = 0;
3100 	unsigned long free_pages = 0;
3101 	int i;
3102 	bool wmark_ok;
3103 
3104 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
3105 		return true;
3106 
3107 	for (i = 0; i <= ZONE_NORMAL; i++) {
3108 		zone = &pgdat->node_zones[i];
3109 		if (!managed_zone(zone))
3110 			continue;
3111 
3112 		if (!zone_reclaimable_pages(zone))
3113 			continue;
3114 
3115 		pfmemalloc_reserve += min_wmark_pages(zone);
3116 		free_pages += zone_page_state(zone, NR_FREE_PAGES);
3117 	}
3118 
3119 	/* If there are no reserves (unexpected config) then do not throttle */
3120 	if (!pfmemalloc_reserve)
3121 		return true;
3122 
3123 	wmark_ok = free_pages > pfmemalloc_reserve / 2;
3124 
3125 	/* kswapd must be awake if processes are being throttled */
3126 	if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
3127 		pgdat->kswapd_classzone_idx = min(pgdat->kswapd_classzone_idx,
3128 						(enum zone_type)ZONE_NORMAL);
3129 		wake_up_interruptible(&pgdat->kswapd_wait);
3130 	}
3131 
3132 	return wmark_ok;
3133 }
3134 
3135 /*
3136  * Throttle direct reclaimers if backing storage is backed by the network
3137  * and the PFMEMALLOC reserve for the preferred node is getting dangerously
3138  * depleted. kswapd will continue to make progress and wake the processes
3139  * when the low watermark is reached.
3140  *
3141  * Returns true if a fatal signal was delivered during throttling. If this
3142  * happens, the page allocator should not consider triggering the OOM killer.
3143  */
3144 static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
3145 					nodemask_t *nodemask)
3146 {
3147 	struct zoneref *z;
3148 	struct zone *zone;
3149 	pg_data_t *pgdat = NULL;
3150 
3151 	/*
3152 	 * Kernel threads should not be throttled as they may be indirectly
3153 	 * responsible for cleaning pages necessary for reclaim to make forward
3154 	 * progress. kjournald for example may enter direct reclaim while
3155 	 * committing a transaction where throttling it could forcing other
3156 	 * processes to block on log_wait_commit().
3157 	 */
3158 	if (current->flags & PF_KTHREAD)
3159 		goto out;
3160 
3161 	/*
3162 	 * If a fatal signal is pending, this process should not throttle.
3163 	 * It should return quickly so it can exit and free its memory
3164 	 */
3165 	if (fatal_signal_pending(current))
3166 		goto out;
3167 
3168 	/*
3169 	 * Check if the pfmemalloc reserves are ok by finding the first node
3170 	 * with a usable ZONE_NORMAL or lower zone. The expectation is that
3171 	 * GFP_KERNEL will be required for allocating network buffers when
3172 	 * swapping over the network so ZONE_HIGHMEM is unusable.
3173 	 *
3174 	 * Throttling is based on the first usable node and throttled processes
3175 	 * wait on a queue until kswapd makes progress and wakes them. There
3176 	 * is an affinity then between processes waking up and where reclaim
3177 	 * progress has been made assuming the process wakes on the same node.
3178 	 * More importantly, processes running on remote nodes will not compete
3179 	 * for remote pfmemalloc reserves and processes on different nodes
3180 	 * should make reasonable progress.
3181 	 */
3182 	for_each_zone_zonelist_nodemask(zone, z, zonelist,
3183 					gfp_zone(gfp_mask), nodemask) {
3184 		if (zone_idx(zone) > ZONE_NORMAL)
3185 			continue;
3186 
3187 		/* Throttle based on the first usable node */
3188 		pgdat = zone->zone_pgdat;
3189 		if (allow_direct_reclaim(pgdat))
3190 			goto out;
3191 		break;
3192 	}
3193 
3194 	/* If no zone was usable by the allocation flags then do not throttle */
3195 	if (!pgdat)
3196 		goto out;
3197 
3198 	/* Account for the throttling */
3199 	count_vm_event(PGSCAN_DIRECT_THROTTLE);
3200 
3201 	/*
3202 	 * If the caller cannot enter the filesystem, it's possible that it
3203 	 * is due to the caller holding an FS lock or performing a journal
3204 	 * transaction in the case of a filesystem like ext[3|4]. In this case,
3205 	 * it is not safe to block on pfmemalloc_wait as kswapd could be
3206 	 * blocked waiting on the same lock. Instead, throttle for up to a
3207 	 * second before continuing.
3208 	 */
3209 	if (!(gfp_mask & __GFP_FS)) {
3210 		wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
3211 			allow_direct_reclaim(pgdat), HZ);
3212 
3213 		goto check_pending;
3214 	}
3215 
3216 	/* Throttle until kswapd wakes the process */
3217 	wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
3218 		allow_direct_reclaim(pgdat));
3219 
3220 check_pending:
3221 	if (fatal_signal_pending(current))
3222 		return true;
3223 
3224 out:
3225 	return false;
3226 }
3227 
3228 unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
3229 				gfp_t gfp_mask, nodemask_t *nodemask)
3230 {
3231 	unsigned long nr_reclaimed;
3232 	struct scan_control sc = {
3233 		.nr_to_reclaim = SWAP_CLUSTER_MAX,
3234 		.gfp_mask = current_gfp_context(gfp_mask),
3235 		.reclaim_idx = gfp_zone(gfp_mask),
3236 		.order = order,
3237 		.nodemask = nodemask,
3238 		.priority = DEF_PRIORITY,
3239 		.may_writepage = !laptop_mode,
3240 		.may_unmap = 1,
3241 		.may_swap = 1,
3242 	};
3243 
3244 	/*
3245 	 * scan_control uses s8 fields for order, priority, and reclaim_idx.
3246 	 * Confirm they are large enough for max values.
3247 	 */
3248 	BUILD_BUG_ON(MAX_ORDER > S8_MAX);
3249 	BUILD_BUG_ON(DEF_PRIORITY > S8_MAX);
3250 	BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX);
3251 
3252 	/*
3253 	 * Do not enter reclaim if fatal signal was delivered while throttled.
3254 	 * 1 is returned so that the page allocator does not OOM kill at this
3255 	 * point.
3256 	 */
3257 	if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask))
3258 		return 1;
3259 
3260 	trace_mm_vmscan_direct_reclaim_begin(order,
3261 				sc.may_writepage,
3262 				sc.gfp_mask,
3263 				sc.reclaim_idx);
3264 
3265 	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3266 
3267 	trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
3268 
3269 	return nr_reclaimed;
3270 }
3271 
3272 #ifdef CONFIG_MEMCG
3273 
3274 unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
3275 						gfp_t gfp_mask, bool noswap,
3276 						pg_data_t *pgdat,
3277 						unsigned long *nr_scanned)
3278 {
3279 	struct scan_control sc = {
3280 		.nr_to_reclaim = SWAP_CLUSTER_MAX,
3281 		.target_mem_cgroup = memcg,
3282 		.may_writepage = !laptop_mode,
3283 		.may_unmap = 1,
3284 		.reclaim_idx = MAX_NR_ZONES - 1,
3285 		.may_swap = !noswap,
3286 	};
3287 	unsigned long lru_pages;
3288 
3289 	sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
3290 			(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
3291 
3292 	trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order,
3293 						      sc.may_writepage,
3294 						      sc.gfp_mask,
3295 						      sc.reclaim_idx);
3296 
3297 	/*
3298 	 * NOTE: Although we can get the priority field, using it
3299 	 * here is not a good idea, since it limits the pages we can scan.
3300 	 * if we don't reclaim here, the shrink_node from balance_pgdat
3301 	 * will pick up pages from other mem cgroup's as well. We hack
3302 	 * the priority and make it zero.
3303 	 */
3304 	shrink_node_memcg(pgdat, memcg, &sc, &lru_pages);
3305 
3306 	trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
3307 
3308 	*nr_scanned = sc.nr_scanned;
3309 	return sc.nr_reclaimed;
3310 }
3311 
3312 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
3313 					   unsigned long nr_pages,
3314 					   gfp_t gfp_mask,
3315 					   bool may_swap)
3316 {
3317 	struct zonelist *zonelist;
3318 	unsigned long nr_reclaimed;
3319 	unsigned long pflags;
3320 	int nid;
3321 	unsigned int noreclaim_flag;
3322 	struct scan_control sc = {
3323 		.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
3324 		.gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) |
3325 				(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK),
3326 		.reclaim_idx = MAX_NR_ZONES - 1,
3327 		.target_mem_cgroup = memcg,
3328 		.priority = DEF_PRIORITY,
3329 		.may_writepage = !laptop_mode,
3330 		.may_unmap = 1,
3331 		.may_swap = may_swap,
3332 	};
3333 
3334 	/*
3335 	 * Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't
3336 	 * take care of from where we get pages. So the node where we start the
3337 	 * scan does not need to be the current node.
3338 	 */
3339 	nid = mem_cgroup_select_victim_node(memcg);
3340 
3341 	zonelist = &NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK];
3342 
3343 	trace_mm_vmscan_memcg_reclaim_begin(0,
3344 					    sc.may_writepage,
3345 					    sc.gfp_mask,
3346 					    sc.reclaim_idx);
3347 
3348 	psi_memstall_enter(&pflags);
3349 	noreclaim_flag = memalloc_noreclaim_save();
3350 
3351 	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3352 
3353 	memalloc_noreclaim_restore(noreclaim_flag);
3354 	psi_memstall_leave(&pflags);
3355 
3356 	trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
3357 
3358 	return nr_reclaimed;
3359 }
3360 #endif
3361 
3362 static void age_active_anon(struct pglist_data *pgdat,
3363 				struct scan_control *sc)
3364 {
3365 	struct mem_cgroup *memcg;
3366 
3367 	if (!total_swap_pages)
3368 		return;
3369 
3370 	memcg = mem_cgroup_iter(NULL, NULL, NULL);
3371 	do {
3372 		struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
3373 
3374 		if (inactive_list_is_low(lruvec, false, memcg, sc, true))
3375 			shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
3376 					   sc, LRU_ACTIVE_ANON);
3377 
3378 		memcg = mem_cgroup_iter(NULL, memcg, NULL);
3379 	} while (memcg);
3380 }
3381 
3382 /*
3383  * Returns true if there is an eligible zone balanced for the request order
3384  * and classzone_idx
3385  */
3386 static bool pgdat_balanced(pg_data_t *pgdat, int order, int classzone_idx)
3387 {
3388 	int i;
3389 	unsigned long mark = -1;
3390 	struct zone *zone;
3391 
3392 	for (i = 0; i <= classzone_idx; i++) {
3393 		zone = pgdat->node_zones + i;
3394 
3395 		if (!managed_zone(zone))
3396 			continue;
3397 
3398 		mark = high_wmark_pages(zone);
3399 		if (zone_watermark_ok_safe(zone, order, mark, classzone_idx))
3400 			return true;
3401 	}
3402 
3403 	/*
3404 	 * If a node has no populated zone within classzone_idx, it does not
3405 	 * need balancing by definition. This can happen if a zone-restricted
3406 	 * allocation tries to wake a remote kswapd.
3407 	 */
3408 	if (mark == -1)
3409 		return true;
3410 
3411 	return false;
3412 }
3413 
3414 /* Clear pgdat state for congested, dirty or under writeback. */
3415 static void clear_pgdat_congested(pg_data_t *pgdat)
3416 {
3417 	clear_bit(PGDAT_CONGESTED, &pgdat->flags);
3418 	clear_bit(PGDAT_DIRTY, &pgdat->flags);
3419 	clear_bit(PGDAT_WRITEBACK, &pgdat->flags);
3420 }
3421 
3422 /*
3423  * Prepare kswapd for sleeping. This verifies that there are no processes
3424  * waiting in throttle_direct_reclaim() and that watermarks have been met.
3425  *
3426  * Returns true if kswapd is ready to sleep
3427  */
3428 static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, int classzone_idx)
3429 {
3430 	/*
3431 	 * The throttled processes are normally woken up in balance_pgdat() as
3432 	 * soon as allow_direct_reclaim() is true. But there is a potential
3433 	 * race between when kswapd checks the watermarks and a process gets
3434 	 * throttled. There is also a potential race if processes get
3435 	 * throttled, kswapd wakes, a large process exits thereby balancing the
3436 	 * zones, which causes kswapd to exit balance_pgdat() before reaching
3437 	 * the wake up checks. If kswapd is going to sleep, no process should
3438 	 * be sleeping on pfmemalloc_wait, so wake them now if necessary. If
3439 	 * the wake up is premature, processes will wake kswapd and get
3440 	 * throttled again. The difference from wake ups in balance_pgdat() is
3441 	 * that here we are under prepare_to_wait().
3442 	 */
3443 	if (waitqueue_active(&pgdat->pfmemalloc_wait))
3444 		wake_up_all(&pgdat->pfmemalloc_wait);
3445 
3446 	/* Hopeless node, leave it to direct reclaim */
3447 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
3448 		return true;
3449 
3450 	if (pgdat_balanced(pgdat, order, classzone_idx)) {
3451 		clear_pgdat_congested(pgdat);
3452 		return true;
3453 	}
3454 
3455 	return false;
3456 }
3457 
3458 /*
3459  * kswapd shrinks a node of pages that are at or below the highest usable
3460  * zone that is currently unbalanced.
3461  *
3462  * Returns true if kswapd scanned at least the requested number of pages to
3463  * reclaim or if the lack of progress was due to pages under writeback.
3464  * This is used to determine if the scanning priority needs to be raised.
3465  */
3466 static bool kswapd_shrink_node(pg_data_t *pgdat,
3467 			       struct scan_control *sc)
3468 {
3469 	struct zone *zone;
3470 	int z;
3471 
3472 	/* Reclaim a number of pages proportional to the number of zones */
3473 	sc->nr_to_reclaim = 0;
3474 	for (z = 0; z <= sc->reclaim_idx; z++) {
3475 		zone = pgdat->node_zones + z;
3476 		if (!managed_zone(zone))
3477 			continue;
3478 
3479 		sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX);
3480 	}
3481 
3482 	/*
3483 	 * Historically care was taken to put equal pressure on all zones but
3484 	 * now pressure is applied based on node LRU order.
3485 	 */
3486 	shrink_node(pgdat, sc);
3487 
3488 	/*
3489 	 * Fragmentation may mean that the system cannot be rebalanced for
3490 	 * high-order allocations. If twice the allocation size has been
3491 	 * reclaimed then recheck watermarks only at order-0 to prevent
3492 	 * excessive reclaim. Assume that a process requested a high-order
3493 	 * can direct reclaim/compact.
3494 	 */
3495 	if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order))
3496 		sc->order = 0;
3497 
3498 	return sc->nr_scanned >= sc->nr_to_reclaim;
3499 }
3500 
3501 /*
3502  * For kswapd, balance_pgdat() will reclaim pages across a node from zones
3503  * that are eligible for use by the caller until at least one zone is
3504  * balanced.
3505  *
3506  * Returns the order kswapd finished reclaiming at.
3507  *
3508  * kswapd scans the zones in the highmem->normal->dma direction.  It skips
3509  * zones which have free_pages > high_wmark_pages(zone), but once a zone is
3510  * found to have free_pages <= high_wmark_pages(zone), any page is that zone
3511  * or lower is eligible for reclaim until at least one usable zone is
3512  * balanced.
3513  */
3514 static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx)
3515 {
3516 	int i;
3517 	unsigned long nr_soft_reclaimed;
3518 	unsigned long nr_soft_scanned;
3519 	unsigned long pflags;
3520 	struct zone *zone;
3521 	struct scan_control sc = {
3522 		.gfp_mask = GFP_KERNEL,
3523 		.order = order,
3524 		.priority = DEF_PRIORITY,
3525 		.may_writepage = !laptop_mode,
3526 		.may_unmap = 1,
3527 		.may_swap = 1,
3528 	};
3529 
3530 	psi_memstall_enter(&pflags);
3531 	__fs_reclaim_acquire();
3532 
3533 	count_vm_event(PAGEOUTRUN);
3534 
3535 	do {
3536 		unsigned long nr_reclaimed = sc.nr_reclaimed;
3537 		bool raise_priority = true;
3538 		bool ret;
3539 
3540 		sc.reclaim_idx = classzone_idx;
3541 
3542 		/*
3543 		 * If the number of buffer_heads exceeds the maximum allowed
3544 		 * then consider reclaiming from all zones. This has a dual
3545 		 * purpose -- on 64-bit systems it is expected that
3546 		 * buffer_heads are stripped during active rotation. On 32-bit
3547 		 * systems, highmem pages can pin lowmem memory and shrinking
3548 		 * buffers can relieve lowmem pressure. Reclaim may still not
3549 		 * go ahead if all eligible zones for the original allocation
3550 		 * request are balanced to avoid excessive reclaim from kswapd.
3551 		 */
3552 		if (buffer_heads_over_limit) {
3553 			for (i = MAX_NR_ZONES - 1; i >= 0; i--) {
3554 				zone = pgdat->node_zones + i;
3555 				if (!managed_zone(zone))
3556 					continue;
3557 
3558 				sc.reclaim_idx = i;
3559 				break;
3560 			}
3561 		}
3562 
3563 		/*
3564 		 * Only reclaim if there are no eligible zones. Note that
3565 		 * sc.reclaim_idx is not used as buffer_heads_over_limit may
3566 		 * have adjusted it.
3567 		 */
3568 		if (pgdat_balanced(pgdat, sc.order, classzone_idx))
3569 			goto out;
3570 
3571 		/*
3572 		 * Do some background aging of the anon list, to give
3573 		 * pages a chance to be referenced before reclaiming. All
3574 		 * pages are rotated regardless of classzone as this is
3575 		 * about consistent aging.
3576 		 */
3577 		age_active_anon(pgdat, &sc);
3578 
3579 		/*
3580 		 * If we're getting trouble reclaiming, start doing writepage
3581 		 * even in laptop mode.
3582 		 */
3583 		if (sc.priority < DEF_PRIORITY - 2)
3584 			sc.may_writepage = 1;
3585 
3586 		/* Call soft limit reclaim before calling shrink_node. */
3587 		sc.nr_scanned = 0;
3588 		nr_soft_scanned = 0;
3589 		nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order,
3590 						sc.gfp_mask, &nr_soft_scanned);
3591 		sc.nr_reclaimed += nr_soft_reclaimed;
3592 
3593 		/*
3594 		 * There should be no need to raise the scanning priority if
3595 		 * enough pages are already being scanned that that high
3596 		 * watermark would be met at 100% efficiency.
3597 		 */
3598 		if (kswapd_shrink_node(pgdat, &sc))
3599 			raise_priority = false;
3600 
3601 		/*
3602 		 * If the low watermark is met there is no need for processes
3603 		 * to be throttled on pfmemalloc_wait as they should not be
3604 		 * able to safely make forward progress. Wake them
3605 		 */
3606 		if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
3607 				allow_direct_reclaim(pgdat))
3608 			wake_up_all(&pgdat->pfmemalloc_wait);
3609 
3610 		/* Check if kswapd should be suspending */
3611 		__fs_reclaim_release();
3612 		ret = try_to_freeze();
3613 		__fs_reclaim_acquire();
3614 		if (ret || kthread_should_stop())
3615 			break;
3616 
3617 		/*
3618 		 * Raise priority if scanning rate is too low or there was no
3619 		 * progress in reclaiming pages
3620 		 */
3621 		nr_reclaimed = sc.nr_reclaimed - nr_reclaimed;
3622 		if (raise_priority || !nr_reclaimed)
3623 			sc.priority--;
3624 	} while (sc.priority >= 1);
3625 
3626 	if (!sc.nr_reclaimed)
3627 		pgdat->kswapd_failures++;
3628 
3629 out:
3630 	snapshot_refaults(NULL, pgdat);
3631 	__fs_reclaim_release();
3632 	psi_memstall_leave(&pflags);
3633 	/*
3634 	 * Return the order kswapd stopped reclaiming at as
3635 	 * prepare_kswapd_sleep() takes it into account. If another caller
3636 	 * entered the allocator slow path while kswapd was awake, order will
3637 	 * remain at the higher level.
3638 	 */
3639 	return sc.order;
3640 }
3641 
3642 /*
3643  * pgdat->kswapd_classzone_idx is the highest zone index that a recent
3644  * allocation request woke kswapd for. When kswapd has not woken recently,
3645  * the value is MAX_NR_ZONES which is not a valid index. This compares a
3646  * given classzone and returns it or the highest classzone index kswapd
3647  * was recently woke for.
3648  */
3649 static enum zone_type kswapd_classzone_idx(pg_data_t *pgdat,
3650 					   enum zone_type classzone_idx)
3651 {
3652 	if (pgdat->kswapd_classzone_idx == MAX_NR_ZONES)
3653 		return classzone_idx;
3654 
3655 	return max(pgdat->kswapd_classzone_idx, classzone_idx);
3656 }
3657 
3658 static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,
3659 				unsigned int classzone_idx)
3660 {
3661 	long remaining = 0;
3662 	DEFINE_WAIT(wait);
3663 
3664 	if (freezing(current) || kthread_should_stop())
3665 		return;
3666 
3667 	prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
3668 
3669 	/*
3670 	 * Try to sleep for a short interval. Note that kcompactd will only be
3671 	 * woken if it is possible to sleep for a short interval. This is
3672 	 * deliberate on the assumption that if reclaim cannot keep an
3673 	 * eligible zone balanced that it's also unlikely that compaction will
3674 	 * succeed.
3675 	 */
3676 	if (prepare_kswapd_sleep(pgdat, reclaim_order, classzone_idx)) {
3677 		/*
3678 		 * Compaction records what page blocks it recently failed to
3679 		 * isolate pages from and skips them in the future scanning.
3680 		 * When kswapd is going to sleep, it is reasonable to assume
3681 		 * that pages and compaction may succeed so reset the cache.
3682 		 */
3683 		reset_isolation_suitable(pgdat);
3684 
3685 		/*
3686 		 * We have freed the memory, now we should compact it to make
3687 		 * allocation of the requested order possible.
3688 		 */
3689 		wakeup_kcompactd(pgdat, alloc_order, classzone_idx);
3690 
3691 		remaining = schedule_timeout(HZ/10);
3692 
3693 		/*
3694 		 * If woken prematurely then reset kswapd_classzone_idx and
3695 		 * order. The values will either be from a wakeup request or
3696 		 * the previous request that slept prematurely.
3697 		 */
3698 		if (remaining) {
3699 			pgdat->kswapd_classzone_idx = kswapd_classzone_idx(pgdat, classzone_idx);
3700 			pgdat->kswapd_order = max(pgdat->kswapd_order, reclaim_order);
3701 		}
3702 
3703 		finish_wait(&pgdat->kswapd_wait, &wait);
3704 		prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
3705 	}
3706 
3707 	/*
3708 	 * After a short sleep, check if it was a premature sleep. If not, then
3709 	 * go fully to sleep until explicitly woken up.
3710 	 */
3711 	if (!remaining &&
3712 	    prepare_kswapd_sleep(pgdat, reclaim_order, classzone_idx)) {
3713 		trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
3714 
3715 		/*
3716 		 * vmstat counters are not perfectly accurate and the estimated
3717 		 * value for counters such as NR_FREE_PAGES can deviate from the
3718 		 * true value by nr_online_cpus * threshold. To avoid the zone
3719 		 * watermarks being breached while under pressure, we reduce the
3720 		 * per-cpu vmstat threshold while kswapd is awake and restore
3721 		 * them before going back to sleep.
3722 		 */
3723 		set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
3724 
3725 		if (!kthread_should_stop())
3726 			schedule();
3727 
3728 		set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);
3729 	} else {
3730 		if (remaining)
3731 			count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
3732 		else
3733 			count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
3734 	}
3735 	finish_wait(&pgdat->kswapd_wait, &wait);
3736 }
3737 
3738 /*
3739  * The background pageout daemon, started as a kernel thread
3740  * from the init process.
3741  *
3742  * This basically trickles out pages so that we have _some_
3743  * free memory available even if there is no other activity
3744  * that frees anything up. This is needed for things like routing
3745  * etc, where we otherwise might have all activity going on in
3746  * asynchronous contexts that cannot page things out.
3747  *
3748  * If there are applications that are active memory-allocators
3749  * (most normal use), this basically shouldn't matter.
3750  */
3751 static int kswapd(void *p)
3752 {
3753 	unsigned int alloc_order, reclaim_order;
3754 	unsigned int classzone_idx = MAX_NR_ZONES - 1;
3755 	pg_data_t *pgdat = (pg_data_t*)p;
3756 	struct task_struct *tsk = current;
3757 
3758 	struct reclaim_state reclaim_state = {
3759 		.reclaimed_slab = 0,
3760 	};
3761 	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
3762 
3763 	if (!cpumask_empty(cpumask))
3764 		set_cpus_allowed_ptr(tsk, cpumask);
3765 	current->reclaim_state = &reclaim_state;
3766 
3767 	/*
3768 	 * Tell the memory management that we're a "memory allocator",
3769 	 * and that if we need more memory we should get access to it
3770 	 * regardless (see "__alloc_pages()"). "kswapd" should
3771 	 * never get caught in the normal page freeing logic.
3772 	 *
3773 	 * (Kswapd normally doesn't need memory anyway, but sometimes
3774 	 * you need a small amount of memory in order to be able to
3775 	 * page out something else, and this flag essentially protects
3776 	 * us from recursively trying to free more memory as we're
3777 	 * trying to free the first piece of memory in the first place).
3778 	 */
3779 	tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD;
3780 	set_freezable();
3781 
3782 	pgdat->kswapd_order = 0;
3783 	pgdat->kswapd_classzone_idx = MAX_NR_ZONES;
3784 	for ( ; ; ) {
3785 		bool ret;
3786 
3787 		alloc_order = reclaim_order = pgdat->kswapd_order;
3788 		classzone_idx = kswapd_classzone_idx(pgdat, classzone_idx);
3789 
3790 kswapd_try_sleep:
3791 		kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,
3792 					classzone_idx);
3793 
3794 		/* Read the new order and classzone_idx */
3795 		alloc_order = reclaim_order = pgdat->kswapd_order;
3796 		classzone_idx = kswapd_classzone_idx(pgdat, 0);
3797 		pgdat->kswapd_order = 0;
3798 		pgdat->kswapd_classzone_idx = MAX_NR_ZONES;
3799 
3800 		ret = try_to_freeze();
3801 		if (kthread_should_stop())
3802 			break;
3803 
3804 		/*
3805 		 * We can speed up thawing tasks if we don't call balance_pgdat
3806 		 * after returning from the refrigerator
3807 		 */
3808 		if (ret)
3809 			continue;
3810 
3811 		/*
3812 		 * Reclaim begins at the requested order but if a high-order
3813 		 * reclaim fails then kswapd falls back to reclaiming for
3814 		 * order-0. If that happens, kswapd will consider sleeping
3815 		 * for the order it finished reclaiming at (reclaim_order)
3816 		 * but kcompactd is woken to compact for the original
3817 		 * request (alloc_order).
3818 		 */
3819 		trace_mm_vmscan_kswapd_wake(pgdat->node_id, classzone_idx,
3820 						alloc_order);
3821 		reclaim_order = balance_pgdat(pgdat, alloc_order, classzone_idx);
3822 		if (reclaim_order < alloc_order)
3823 			goto kswapd_try_sleep;
3824 	}
3825 
3826 	tsk->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD);
3827 	current->reclaim_state = NULL;
3828 
3829 	return 0;
3830 }
3831 
3832 /*
3833  * A zone is low on free memory or too fragmented for high-order memory.  If
3834  * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's
3835  * pgdat.  It will wake up kcompactd after reclaiming memory.  If kswapd reclaim
3836  * has failed or is not needed, still wake up kcompactd if only compaction is
3837  * needed.
3838  */
3839 void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order,
3840 		   enum zone_type classzone_idx)
3841 {
3842 	pg_data_t *pgdat;
3843 
3844 	if (!managed_zone(zone))
3845 		return;
3846 
3847 	if (!cpuset_zone_allowed(zone, gfp_flags))
3848 		return;
3849 	pgdat = zone->zone_pgdat;
3850 	pgdat->kswapd_classzone_idx = kswapd_classzone_idx(pgdat,
3851 							   classzone_idx);
3852 	pgdat->kswapd_order = max(pgdat->kswapd_order, order);
3853 	if (!waitqueue_active(&pgdat->kswapd_wait))
3854 		return;
3855 
3856 	/* Hopeless node, leave it to direct reclaim if possible */
3857 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ||
3858 	    pgdat_balanced(pgdat, order, classzone_idx)) {
3859 		/*
3860 		 * There may be plenty of free memory available, but it's too
3861 		 * fragmented for high-order allocations.  Wake up kcompactd
3862 		 * and rely on compaction_suitable() to determine if it's
3863 		 * needed.  If it fails, it will defer subsequent attempts to
3864 		 * ratelimit its work.
3865 		 */
3866 		if (!(gfp_flags & __GFP_DIRECT_RECLAIM))
3867 			wakeup_kcompactd(pgdat, order, classzone_idx);
3868 		return;
3869 	}
3870 
3871 	trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, classzone_idx, order,
3872 				      gfp_flags);
3873 	wake_up_interruptible(&pgdat->kswapd_wait);
3874 }
3875 
3876 #ifdef CONFIG_HIBERNATION
3877 /*
3878  * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
3879  * freed pages.
3880  *
3881  * Rather than trying to age LRUs the aim is to preserve the overall
3882  * LRU order by reclaiming preferentially
3883  * inactive > active > active referenced > active mapped
3884  */
3885 unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
3886 {
3887 	struct reclaim_state reclaim_state;
3888 	struct scan_control sc = {
3889 		.nr_to_reclaim = nr_to_reclaim,
3890 		.gfp_mask = GFP_HIGHUSER_MOVABLE,
3891 		.reclaim_idx = MAX_NR_ZONES - 1,
3892 		.priority = DEF_PRIORITY,
3893 		.may_writepage = 1,
3894 		.may_unmap = 1,
3895 		.may_swap = 1,
3896 		.hibernation_mode = 1,
3897 	};
3898 	struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3899 	struct task_struct *p = current;
3900 	unsigned long nr_reclaimed;
3901 	unsigned int noreclaim_flag;
3902 
3903 	fs_reclaim_acquire(sc.gfp_mask);
3904 	noreclaim_flag = memalloc_noreclaim_save();
3905 	reclaim_state.reclaimed_slab = 0;
3906 	p->reclaim_state = &reclaim_state;
3907 
3908 	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3909 
3910 	p->reclaim_state = NULL;
3911 	memalloc_noreclaim_restore(noreclaim_flag);
3912 	fs_reclaim_release(sc.gfp_mask);
3913 
3914 	return nr_reclaimed;
3915 }
3916 #endif /* CONFIG_HIBERNATION */
3917 
3918 /* It's optimal to keep kswapds on the same CPUs as their memory, but
3919    not required for correctness.  So if the last cpu in a node goes
3920    away, we get changed to run anywhere: as the first one comes back,
3921    restore their cpu bindings. */
3922 static int kswapd_cpu_online(unsigned int cpu)
3923 {
3924 	int nid;
3925 
3926 	for_each_node_state(nid, N_MEMORY) {
3927 		pg_data_t *pgdat = NODE_DATA(nid);
3928 		const struct cpumask *mask;
3929 
3930 		mask = cpumask_of_node(pgdat->node_id);
3931 
3932 		if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
3933 			/* One of our CPUs online: restore mask */
3934 			set_cpus_allowed_ptr(pgdat->kswapd, mask);
3935 	}
3936 	return 0;
3937 }
3938 
3939 /*
3940  * This kswapd start function will be called by init and node-hot-add.
3941  * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
3942  */
3943 int kswapd_run(int nid)
3944 {
3945 	pg_data_t *pgdat = NODE_DATA(nid);
3946 	int ret = 0;
3947 
3948 	if (pgdat->kswapd)
3949 		return 0;
3950 
3951 	pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
3952 	if (IS_ERR(pgdat->kswapd)) {
3953 		/* failure at boot is fatal */
3954 		BUG_ON(system_state < SYSTEM_RUNNING);
3955 		pr_err("Failed to start kswapd on node %d\n", nid);
3956 		ret = PTR_ERR(pgdat->kswapd);
3957 		pgdat->kswapd = NULL;
3958 	}
3959 	return ret;
3960 }
3961 
3962 /*
3963  * Called by memory hotplug when all memory in a node is offlined.  Caller must
3964  * hold mem_hotplug_begin/end().
3965  */
3966 void kswapd_stop(int nid)
3967 {
3968 	struct task_struct *kswapd = NODE_DATA(nid)->kswapd;
3969 
3970 	if (kswapd) {
3971 		kthread_stop(kswapd);
3972 		NODE_DATA(nid)->kswapd = NULL;
3973 	}
3974 }
3975 
3976 static int __init kswapd_init(void)
3977 {
3978 	int nid, ret;
3979 
3980 	swap_setup();
3981 	for_each_node_state(nid, N_MEMORY)
3982  		kswapd_run(nid);
3983 	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
3984 					"mm/vmscan:online", kswapd_cpu_online,
3985 					NULL);
3986 	WARN_ON(ret < 0);
3987 	return 0;
3988 }
3989 
3990 module_init(kswapd_init)
3991 
3992 #ifdef CONFIG_NUMA
3993 /*
3994  * Node reclaim mode
3995  *
3996  * If non-zero call node_reclaim when the number of free pages falls below
3997  * the watermarks.
3998  */
3999 int node_reclaim_mode __read_mostly;
4000 
4001 #define RECLAIM_OFF 0
4002 #define RECLAIM_ZONE (1<<0)	/* Run shrink_inactive_list on the zone */
4003 #define RECLAIM_WRITE (1<<1)	/* Writeout pages during reclaim */
4004 #define RECLAIM_UNMAP (1<<2)	/* Unmap pages during reclaim */
4005 
4006 /*
4007  * Priority for NODE_RECLAIM. This determines the fraction of pages
4008  * of a node considered for each zone_reclaim. 4 scans 1/16th of
4009  * a zone.
4010  */
4011 #define NODE_RECLAIM_PRIORITY 4
4012 
4013 /*
4014  * Percentage of pages in a zone that must be unmapped for node_reclaim to
4015  * occur.
4016  */
4017 int sysctl_min_unmapped_ratio = 1;
4018 
4019 /*
4020  * If the number of slab pages in a zone grows beyond this percentage then
4021  * slab reclaim needs to occur.
4022  */
4023 int sysctl_min_slab_ratio = 5;
4024 
4025 static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat)
4026 {
4027 	unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED);
4028 	unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) +
4029 		node_page_state(pgdat, NR_ACTIVE_FILE);
4030 
4031 	/*
4032 	 * It's possible for there to be more file mapped pages than
4033 	 * accounted for by the pages on the file LRU lists because
4034 	 * tmpfs pages accounted for as ANON can also be FILE_MAPPED
4035 	 */
4036 	return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
4037 }
4038 
4039 /* Work out how many page cache pages we can reclaim in this reclaim_mode */
4040 static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat)
4041 {
4042 	unsigned long nr_pagecache_reclaimable;
4043 	unsigned long delta = 0;
4044 
4045 	/*
4046 	 * If RECLAIM_UNMAP is set, then all file pages are considered
4047 	 * potentially reclaimable. Otherwise, we have to worry about
4048 	 * pages like swapcache and node_unmapped_file_pages() provides
4049 	 * a better estimate
4050 	 */
4051 	if (node_reclaim_mode & RECLAIM_UNMAP)
4052 		nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES);
4053 	else
4054 		nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat);
4055 
4056 	/* If we can't clean pages, remove dirty pages from consideration */
4057 	if (!(node_reclaim_mode & RECLAIM_WRITE))
4058 		delta += node_page_state(pgdat, NR_FILE_DIRTY);
4059 
4060 	/* Watch for any possible underflows due to delta */
4061 	if (unlikely(delta > nr_pagecache_reclaimable))
4062 		delta = nr_pagecache_reclaimable;
4063 
4064 	return nr_pagecache_reclaimable - delta;
4065 }
4066 
4067 /*
4068  * Try to free up some pages from this node through reclaim.
4069  */
4070 static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
4071 {
4072 	/* Minimum pages needed in order to stay on node */
4073 	const unsigned long nr_pages = 1 << order;
4074 	struct task_struct *p = current;
4075 	struct reclaim_state reclaim_state;
4076 	unsigned int noreclaim_flag;
4077 	struct scan_control sc = {
4078 		.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
4079 		.gfp_mask = current_gfp_context(gfp_mask),
4080 		.order = order,
4081 		.priority = NODE_RECLAIM_PRIORITY,
4082 		.may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE),
4083 		.may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP),
4084 		.may_swap = 1,
4085 		.reclaim_idx = gfp_zone(gfp_mask),
4086 	};
4087 
4088 	cond_resched();
4089 	fs_reclaim_acquire(sc.gfp_mask);
4090 	/*
4091 	 * We need to be able to allocate from the reserves for RECLAIM_UNMAP
4092 	 * and we also need to be able to write out pages for RECLAIM_WRITE
4093 	 * and RECLAIM_UNMAP.
4094 	 */
4095 	noreclaim_flag = memalloc_noreclaim_save();
4096 	p->flags |= PF_SWAPWRITE;
4097 	reclaim_state.reclaimed_slab = 0;
4098 	p->reclaim_state = &reclaim_state;
4099 
4100 	if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages) {
4101 		/*
4102 		 * Free memory by calling shrink node with increasing
4103 		 * priorities until we have enough memory freed.
4104 		 */
4105 		do {
4106 			shrink_node(pgdat, &sc);
4107 		} while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
4108 	}
4109 
4110 	p->reclaim_state = NULL;
4111 	current->flags &= ~PF_SWAPWRITE;
4112 	memalloc_noreclaim_restore(noreclaim_flag);
4113 	fs_reclaim_release(sc.gfp_mask);
4114 	return sc.nr_reclaimed >= nr_pages;
4115 }
4116 
4117 int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
4118 {
4119 	int ret;
4120 
4121 	/*
4122 	 * Node reclaim reclaims unmapped file backed pages and
4123 	 * slab pages if we are over the defined limits.
4124 	 *
4125 	 * A small portion of unmapped file backed pages is needed for
4126 	 * file I/O otherwise pages read by file I/O will be immediately
4127 	 * thrown out if the node is overallocated. So we do not reclaim
4128 	 * if less than a specified percentage of the node is used by
4129 	 * unmapped file backed pages.
4130 	 */
4131 	if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&
4132 	    node_page_state(pgdat, NR_SLAB_RECLAIMABLE) <= pgdat->min_slab_pages)
4133 		return NODE_RECLAIM_FULL;
4134 
4135 	/*
4136 	 * Do not scan if the allocation should not be delayed.
4137 	 */
4138 	if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC))
4139 		return NODE_RECLAIM_NOSCAN;
4140 
4141 	/*
4142 	 * Only run node reclaim on the local node or on nodes that do not
4143 	 * have associated processors. This will favor the local processor
4144 	 * over remote processors and spread off node memory allocations
4145 	 * as wide as possible.
4146 	 */
4147 	if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id())
4148 		return NODE_RECLAIM_NOSCAN;
4149 
4150 	if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags))
4151 		return NODE_RECLAIM_NOSCAN;
4152 
4153 	ret = __node_reclaim(pgdat, gfp_mask, order);
4154 	clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags);
4155 
4156 	if (!ret)
4157 		count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
4158 
4159 	return ret;
4160 }
4161 #endif
4162 
4163 /*
4164  * page_evictable - test whether a page is evictable
4165  * @page: the page to test
4166  *
4167  * Test whether page is evictable--i.e., should be placed on active/inactive
4168  * lists vs unevictable list.
4169  *
4170  * Reasons page might not be evictable:
4171  * (1) page's mapping marked unevictable
4172  * (2) page is part of an mlocked VMA
4173  *
4174  */
4175 int page_evictable(struct page *page)
4176 {
4177 	int ret;
4178 
4179 	/* Prevent address_space of inode and swap cache from being freed */
4180 	rcu_read_lock();
4181 	ret = !mapping_unevictable(page_mapping(page)) && !PageMlocked(page);
4182 	rcu_read_unlock();
4183 	return ret;
4184 }
4185 
4186 /**
4187  * check_move_unevictable_pages - check pages for evictability and move to
4188  * appropriate zone lru list
4189  * @pvec: pagevec with lru pages to check
4190  *
4191  * Checks pages for evictability, if an evictable page is in the unevictable
4192  * lru list, moves it to the appropriate evictable lru list. This function
4193  * should be only used for lru pages.
4194  */
4195 void check_move_unevictable_pages(struct pagevec *pvec)
4196 {
4197 	struct lruvec *lruvec;
4198 	struct pglist_data *pgdat = NULL;
4199 	int pgscanned = 0;
4200 	int pgrescued = 0;
4201 	int i;
4202 
4203 	for (i = 0; i < pvec->nr; i++) {
4204 		struct page *page = pvec->pages[i];
4205 		struct pglist_data *pagepgdat = page_pgdat(page);
4206 
4207 		pgscanned++;
4208 		if (pagepgdat != pgdat) {
4209 			if (pgdat)
4210 				spin_unlock_irq(&pgdat->lru_lock);
4211 			pgdat = pagepgdat;
4212 			spin_lock_irq(&pgdat->lru_lock);
4213 		}
4214 		lruvec = mem_cgroup_page_lruvec(page, pgdat);
4215 
4216 		if (!PageLRU(page) || !PageUnevictable(page))
4217 			continue;
4218 
4219 		if (page_evictable(page)) {
4220 			enum lru_list lru = page_lru_base_type(page);
4221 
4222 			VM_BUG_ON_PAGE(PageActive(page), page);
4223 			ClearPageUnevictable(page);
4224 			del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE);
4225 			add_page_to_lru_list(page, lruvec, lru);
4226 			pgrescued++;
4227 		}
4228 	}
4229 
4230 	if (pgdat) {
4231 		__count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
4232 		__count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
4233 		spin_unlock_irq(&pgdat->lru_lock);
4234 	}
4235 }
4236 EXPORT_SYMBOL_GPL(check_move_unevictable_pages);
4237