xref: /openbmc/linux/mm/vmscan.c (revision 6774def6)
1 /*
2  *  linux/mm/vmscan.c
3  *
4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
5  *
6  *  Swap reorganised 29.12.95, Stephen Tweedie.
7  *  kswapd added: 7.1.96  sct
8  *  Removed kswapd_ctl limits, and swap out as many pages as needed
9  *  to bring the system back to freepages.high: 2.4.97, Rik van Riel.
10  *  Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
11  *  Multiqueue VM started 5.8.00, Rik van Riel.
12  */
13 
14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15 
16 #include <linux/mm.h>
17 #include <linux/module.h>
18 #include <linux/gfp.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/swap.h>
21 #include <linux/pagemap.h>
22 #include <linux/init.h>
23 #include <linux/highmem.h>
24 #include <linux/vmpressure.h>
25 #include <linux/vmstat.h>
26 #include <linux/file.h>
27 #include <linux/writeback.h>
28 #include <linux/blkdev.h>
29 #include <linux/buffer_head.h>	/* for try_to_release_page(),
30 					buffer_heads_over_limit */
31 #include <linux/mm_inline.h>
32 #include <linux/backing-dev.h>
33 #include <linux/rmap.h>
34 #include <linux/topology.h>
35 #include <linux/cpu.h>
36 #include <linux/cpuset.h>
37 #include <linux/compaction.h>
38 #include <linux/notifier.h>
39 #include <linux/rwsem.h>
40 #include <linux/delay.h>
41 #include <linux/kthread.h>
42 #include <linux/freezer.h>
43 #include <linux/memcontrol.h>
44 #include <linux/delayacct.h>
45 #include <linux/sysctl.h>
46 #include <linux/oom.h>
47 #include <linux/prefetch.h>
48 #include <linux/printk.h>
49 
50 #include <asm/tlbflush.h>
51 #include <asm/div64.h>
52 
53 #include <linux/swapops.h>
54 #include <linux/balloon_compaction.h>
55 
56 #include "internal.h"
57 
58 #define CREATE_TRACE_POINTS
59 #include <trace/events/vmscan.h>
60 
61 struct scan_control {
62 	/* How many pages shrink_list() should reclaim */
63 	unsigned long nr_to_reclaim;
64 
65 	/* This context's GFP mask */
66 	gfp_t gfp_mask;
67 
68 	/* Allocation order */
69 	int order;
70 
71 	/*
72 	 * Nodemask of nodes allowed by the caller. If NULL, all nodes
73 	 * are scanned.
74 	 */
75 	nodemask_t	*nodemask;
76 
77 	/*
78 	 * The memory cgroup that hit its limit and as a result is the
79 	 * primary target of this reclaim invocation.
80 	 */
81 	struct mem_cgroup *target_mem_cgroup;
82 
83 	/* Scan (total_size >> priority) pages at once */
84 	int priority;
85 
86 	unsigned int may_writepage:1;
87 
88 	/* Can mapped pages be reclaimed? */
89 	unsigned int may_unmap:1;
90 
91 	/* Can pages be swapped as part of reclaim? */
92 	unsigned int may_swap:1;
93 
94 	unsigned int hibernation_mode:1;
95 
96 	/* One of the zones is ready for compaction */
97 	unsigned int compaction_ready:1;
98 
99 	/* Incremented by the number of inactive pages that were scanned */
100 	unsigned long nr_scanned;
101 
102 	/* Number of pages freed so far during a call to shrink_zones() */
103 	unsigned long nr_reclaimed;
104 };
105 
106 #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
107 
108 #ifdef ARCH_HAS_PREFETCH
109 #define prefetch_prev_lru_page(_page, _base, _field)			\
110 	do {								\
111 		if ((_page)->lru.prev != _base) {			\
112 			struct page *prev;				\
113 									\
114 			prev = lru_to_page(&(_page->lru));		\
115 			prefetch(&prev->_field);			\
116 		}							\
117 	} while (0)
118 #else
119 #define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
120 #endif
121 
122 #ifdef ARCH_HAS_PREFETCHW
123 #define prefetchw_prev_lru_page(_page, _base, _field)			\
124 	do {								\
125 		if ((_page)->lru.prev != _base) {			\
126 			struct page *prev;				\
127 									\
128 			prev = lru_to_page(&(_page->lru));		\
129 			prefetchw(&prev->_field);			\
130 		}							\
131 	} while (0)
132 #else
133 #define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
134 #endif
135 
136 /*
137  * From 0 .. 100.  Higher means more swappy.
138  */
139 int vm_swappiness = 60;
140 /*
141  * The total number of pages which are beyond the high watermark within all
142  * zones.
143  */
144 unsigned long vm_total_pages;
145 
146 static LIST_HEAD(shrinker_list);
147 static DECLARE_RWSEM(shrinker_rwsem);
148 
149 #ifdef CONFIG_MEMCG
150 static bool global_reclaim(struct scan_control *sc)
151 {
152 	return !sc->target_mem_cgroup;
153 }
154 #else
155 static bool global_reclaim(struct scan_control *sc)
156 {
157 	return true;
158 }
159 #endif
160 
161 static unsigned long zone_reclaimable_pages(struct zone *zone)
162 {
163 	int nr;
164 
165 	nr = zone_page_state(zone, NR_ACTIVE_FILE) +
166 	     zone_page_state(zone, NR_INACTIVE_FILE);
167 
168 	if (get_nr_swap_pages() > 0)
169 		nr += zone_page_state(zone, NR_ACTIVE_ANON) +
170 		      zone_page_state(zone, NR_INACTIVE_ANON);
171 
172 	return nr;
173 }
174 
175 bool zone_reclaimable(struct zone *zone)
176 {
177 	return zone_page_state(zone, NR_PAGES_SCANNED) <
178 		zone_reclaimable_pages(zone) * 6;
179 }
180 
181 static unsigned long get_lru_size(struct lruvec *lruvec, enum lru_list lru)
182 {
183 	if (!mem_cgroup_disabled())
184 		return mem_cgroup_get_lru_size(lruvec, lru);
185 
186 	return zone_page_state(lruvec_zone(lruvec), NR_LRU_BASE + lru);
187 }
188 
189 /*
190  * Add a shrinker callback to be called from the vm.
191  */
192 int register_shrinker(struct shrinker *shrinker)
193 {
194 	size_t size = sizeof(*shrinker->nr_deferred);
195 
196 	/*
197 	 * If we only have one possible node in the system anyway, save
198 	 * ourselves the trouble and disable NUMA aware behavior. This way we
199 	 * will save memory and some small loop time later.
200 	 */
201 	if (nr_node_ids == 1)
202 		shrinker->flags &= ~SHRINKER_NUMA_AWARE;
203 
204 	if (shrinker->flags & SHRINKER_NUMA_AWARE)
205 		size *= nr_node_ids;
206 
207 	shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
208 	if (!shrinker->nr_deferred)
209 		return -ENOMEM;
210 
211 	down_write(&shrinker_rwsem);
212 	list_add_tail(&shrinker->list, &shrinker_list);
213 	up_write(&shrinker_rwsem);
214 	return 0;
215 }
216 EXPORT_SYMBOL(register_shrinker);
217 
218 /*
219  * Remove one
220  */
221 void unregister_shrinker(struct shrinker *shrinker)
222 {
223 	down_write(&shrinker_rwsem);
224 	list_del(&shrinker->list);
225 	up_write(&shrinker_rwsem);
226 	kfree(shrinker->nr_deferred);
227 }
228 EXPORT_SYMBOL(unregister_shrinker);
229 
230 #define SHRINK_BATCH 128
231 
232 static unsigned long
233 shrink_slab_node(struct shrink_control *shrinkctl, struct shrinker *shrinker,
234 		 unsigned long nr_pages_scanned, unsigned long lru_pages)
235 {
236 	unsigned long freed = 0;
237 	unsigned long long delta;
238 	long total_scan;
239 	long freeable;
240 	long nr;
241 	long new_nr;
242 	int nid = shrinkctl->nid;
243 	long batch_size = shrinker->batch ? shrinker->batch
244 					  : SHRINK_BATCH;
245 
246 	freeable = shrinker->count_objects(shrinker, shrinkctl);
247 	if (freeable == 0)
248 		return 0;
249 
250 	/*
251 	 * copy the current shrinker scan count into a local variable
252 	 * and zero it so that other concurrent shrinker invocations
253 	 * don't also do this scanning work.
254 	 */
255 	nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0);
256 
257 	total_scan = nr;
258 	delta = (4 * nr_pages_scanned) / shrinker->seeks;
259 	delta *= freeable;
260 	do_div(delta, lru_pages + 1);
261 	total_scan += delta;
262 	if (total_scan < 0) {
263 		printk(KERN_ERR
264 		"shrink_slab: %pF negative objects to delete nr=%ld\n",
265 		       shrinker->scan_objects, total_scan);
266 		total_scan = freeable;
267 	}
268 
269 	/*
270 	 * We need to avoid excessive windup on filesystem shrinkers
271 	 * due to large numbers of GFP_NOFS allocations causing the
272 	 * shrinkers to return -1 all the time. This results in a large
273 	 * nr being built up so when a shrink that can do some work
274 	 * comes along it empties the entire cache due to nr >>>
275 	 * freeable. This is bad for sustaining a working set in
276 	 * memory.
277 	 *
278 	 * Hence only allow the shrinker to scan the entire cache when
279 	 * a large delta change is calculated directly.
280 	 */
281 	if (delta < freeable / 4)
282 		total_scan = min(total_scan, freeable / 2);
283 
284 	/*
285 	 * Avoid risking looping forever due to too large nr value:
286 	 * never try to free more than twice the estimate number of
287 	 * freeable entries.
288 	 */
289 	if (total_scan > freeable * 2)
290 		total_scan = freeable * 2;
291 
292 	trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
293 				nr_pages_scanned, lru_pages,
294 				freeable, delta, total_scan);
295 
296 	/*
297 	 * Normally, we should not scan less than batch_size objects in one
298 	 * pass to avoid too frequent shrinker calls, but if the slab has less
299 	 * than batch_size objects in total and we are really tight on memory,
300 	 * we will try to reclaim all available objects, otherwise we can end
301 	 * up failing allocations although there are plenty of reclaimable
302 	 * objects spread over several slabs with usage less than the
303 	 * batch_size.
304 	 *
305 	 * We detect the "tight on memory" situations by looking at the total
306 	 * number of objects we want to scan (total_scan). If it is greater
307 	 * than the total number of objects on slab (freeable), we must be
308 	 * scanning at high prio and therefore should try to reclaim as much as
309 	 * possible.
310 	 */
311 	while (total_scan >= batch_size ||
312 	       total_scan >= freeable) {
313 		unsigned long ret;
314 		unsigned long nr_to_scan = min(batch_size, total_scan);
315 
316 		shrinkctl->nr_to_scan = nr_to_scan;
317 		ret = shrinker->scan_objects(shrinker, shrinkctl);
318 		if (ret == SHRINK_STOP)
319 			break;
320 		freed += ret;
321 
322 		count_vm_events(SLABS_SCANNED, nr_to_scan);
323 		total_scan -= nr_to_scan;
324 
325 		cond_resched();
326 	}
327 
328 	/*
329 	 * move the unused scan count back into the shrinker in a
330 	 * manner that handles concurrent updates. If we exhausted the
331 	 * scan, there is no need to do an update.
332 	 */
333 	if (total_scan > 0)
334 		new_nr = atomic_long_add_return(total_scan,
335 						&shrinker->nr_deferred[nid]);
336 	else
337 		new_nr = atomic_long_read(&shrinker->nr_deferred[nid]);
338 
339 	trace_mm_shrink_slab_end(shrinker, nid, freed, nr, new_nr, total_scan);
340 	return freed;
341 }
342 
343 /*
344  * Call the shrink functions to age shrinkable caches
345  *
346  * Here we assume it costs one seek to replace a lru page and that it also
347  * takes a seek to recreate a cache object.  With this in mind we age equal
348  * percentages of the lru and ageable caches.  This should balance the seeks
349  * generated by these structures.
350  *
351  * If the vm encountered mapped pages on the LRU it increase the pressure on
352  * slab to avoid swapping.
353  *
354  * We do weird things to avoid (scanned*seeks*entries) overflowing 32 bits.
355  *
356  * `lru_pages' represents the number of on-LRU pages in all the zones which
357  * are eligible for the caller's allocation attempt.  It is used for balancing
358  * slab reclaim versus page reclaim.
359  *
360  * Returns the number of slab objects which we shrunk.
361  */
362 unsigned long shrink_slab(struct shrink_control *shrinkctl,
363 			  unsigned long nr_pages_scanned,
364 			  unsigned long lru_pages)
365 {
366 	struct shrinker *shrinker;
367 	unsigned long freed = 0;
368 
369 	if (nr_pages_scanned == 0)
370 		nr_pages_scanned = SWAP_CLUSTER_MAX;
371 
372 	if (!down_read_trylock(&shrinker_rwsem)) {
373 		/*
374 		 * If we would return 0, our callers would understand that we
375 		 * have nothing else to shrink and give up trying. By returning
376 		 * 1 we keep it going and assume we'll be able to shrink next
377 		 * time.
378 		 */
379 		freed = 1;
380 		goto out;
381 	}
382 
383 	list_for_each_entry(shrinker, &shrinker_list, list) {
384 		if (!(shrinker->flags & SHRINKER_NUMA_AWARE)) {
385 			shrinkctl->nid = 0;
386 			freed += shrink_slab_node(shrinkctl, shrinker,
387 					nr_pages_scanned, lru_pages);
388 			continue;
389 		}
390 
391 		for_each_node_mask(shrinkctl->nid, shrinkctl->nodes_to_scan) {
392 			if (node_online(shrinkctl->nid))
393 				freed += shrink_slab_node(shrinkctl, shrinker,
394 						nr_pages_scanned, lru_pages);
395 
396 		}
397 	}
398 	up_read(&shrinker_rwsem);
399 out:
400 	cond_resched();
401 	return freed;
402 }
403 
404 static inline int is_page_cache_freeable(struct page *page)
405 {
406 	/*
407 	 * A freeable page cache page is referenced only by the caller
408 	 * that isolated the page, the page cache radix tree and
409 	 * optional buffer heads at page->private.
410 	 */
411 	return page_count(page) - page_has_private(page) == 2;
412 }
413 
414 static int may_write_to_queue(struct backing_dev_info *bdi,
415 			      struct scan_control *sc)
416 {
417 	if (current->flags & PF_SWAPWRITE)
418 		return 1;
419 	if (!bdi_write_congested(bdi))
420 		return 1;
421 	if (bdi == current->backing_dev_info)
422 		return 1;
423 	return 0;
424 }
425 
426 /*
427  * We detected a synchronous write error writing a page out.  Probably
428  * -ENOSPC.  We need to propagate that into the address_space for a subsequent
429  * fsync(), msync() or close().
430  *
431  * The tricky part is that after writepage we cannot touch the mapping: nothing
432  * prevents it from being freed up.  But we have a ref on the page and once
433  * that page is locked, the mapping is pinned.
434  *
435  * We're allowed to run sleeping lock_page() here because we know the caller has
436  * __GFP_FS.
437  */
438 static void handle_write_error(struct address_space *mapping,
439 				struct page *page, int error)
440 {
441 	lock_page(page);
442 	if (page_mapping(page) == mapping)
443 		mapping_set_error(mapping, error);
444 	unlock_page(page);
445 }
446 
447 /* possible outcome of pageout() */
448 typedef enum {
449 	/* failed to write page out, page is locked */
450 	PAGE_KEEP,
451 	/* move page to the active list, page is locked */
452 	PAGE_ACTIVATE,
453 	/* page has been sent to the disk successfully, page is unlocked */
454 	PAGE_SUCCESS,
455 	/* page is clean and locked */
456 	PAGE_CLEAN,
457 } pageout_t;
458 
459 /*
460  * pageout is called by shrink_page_list() for each dirty page.
461  * Calls ->writepage().
462  */
463 static pageout_t pageout(struct page *page, struct address_space *mapping,
464 			 struct scan_control *sc)
465 {
466 	/*
467 	 * If the page is dirty, only perform writeback if that write
468 	 * will be non-blocking.  To prevent this allocation from being
469 	 * stalled by pagecache activity.  But note that there may be
470 	 * stalls if we need to run get_block().  We could test
471 	 * PagePrivate for that.
472 	 *
473 	 * If this process is currently in __generic_file_write_iter() against
474 	 * this page's queue, we can perform writeback even if that
475 	 * will block.
476 	 *
477 	 * If the page is swapcache, write it back even if that would
478 	 * block, for some throttling. This happens by accident, because
479 	 * swap_backing_dev_info is bust: it doesn't reflect the
480 	 * congestion state of the swapdevs.  Easy to fix, if needed.
481 	 */
482 	if (!is_page_cache_freeable(page))
483 		return PAGE_KEEP;
484 	if (!mapping) {
485 		/*
486 		 * Some data journaling orphaned pages can have
487 		 * page->mapping == NULL while being dirty with clean buffers.
488 		 */
489 		if (page_has_private(page)) {
490 			if (try_to_free_buffers(page)) {
491 				ClearPageDirty(page);
492 				pr_info("%s: orphaned page\n", __func__);
493 				return PAGE_CLEAN;
494 			}
495 		}
496 		return PAGE_KEEP;
497 	}
498 	if (mapping->a_ops->writepage == NULL)
499 		return PAGE_ACTIVATE;
500 	if (!may_write_to_queue(mapping->backing_dev_info, sc))
501 		return PAGE_KEEP;
502 
503 	if (clear_page_dirty_for_io(page)) {
504 		int res;
505 		struct writeback_control wbc = {
506 			.sync_mode = WB_SYNC_NONE,
507 			.nr_to_write = SWAP_CLUSTER_MAX,
508 			.range_start = 0,
509 			.range_end = LLONG_MAX,
510 			.for_reclaim = 1,
511 		};
512 
513 		SetPageReclaim(page);
514 		res = mapping->a_ops->writepage(page, &wbc);
515 		if (res < 0)
516 			handle_write_error(mapping, page, res);
517 		if (res == AOP_WRITEPAGE_ACTIVATE) {
518 			ClearPageReclaim(page);
519 			return PAGE_ACTIVATE;
520 		}
521 
522 		if (!PageWriteback(page)) {
523 			/* synchronous write or broken a_ops? */
524 			ClearPageReclaim(page);
525 		}
526 		trace_mm_vmscan_writepage(page, trace_reclaim_flags(page));
527 		inc_zone_page_state(page, NR_VMSCAN_WRITE);
528 		return PAGE_SUCCESS;
529 	}
530 
531 	return PAGE_CLEAN;
532 }
533 
534 /*
535  * Same as remove_mapping, but if the page is removed from the mapping, it
536  * gets returned with a refcount of 0.
537  */
538 static int __remove_mapping(struct address_space *mapping, struct page *page,
539 			    bool reclaimed)
540 {
541 	BUG_ON(!PageLocked(page));
542 	BUG_ON(mapping != page_mapping(page));
543 
544 	spin_lock_irq(&mapping->tree_lock);
545 	/*
546 	 * The non racy check for a busy page.
547 	 *
548 	 * Must be careful with the order of the tests. When someone has
549 	 * a ref to the page, it may be possible that they dirty it then
550 	 * drop the reference. So if PageDirty is tested before page_count
551 	 * here, then the following race may occur:
552 	 *
553 	 * get_user_pages(&page);
554 	 * [user mapping goes away]
555 	 * write_to(page);
556 	 *				!PageDirty(page)    [good]
557 	 * SetPageDirty(page);
558 	 * put_page(page);
559 	 *				!page_count(page)   [good, discard it]
560 	 *
561 	 * [oops, our write_to data is lost]
562 	 *
563 	 * Reversing the order of the tests ensures such a situation cannot
564 	 * escape unnoticed. The smp_rmb is needed to ensure the page->flags
565 	 * load is not satisfied before that of page->_count.
566 	 *
567 	 * Note that if SetPageDirty is always performed via set_page_dirty,
568 	 * and thus under tree_lock, then this ordering is not required.
569 	 */
570 	if (!page_freeze_refs(page, 2))
571 		goto cannot_free;
572 	/* note: atomic_cmpxchg in page_freeze_refs provides the smp_rmb */
573 	if (unlikely(PageDirty(page))) {
574 		page_unfreeze_refs(page, 2);
575 		goto cannot_free;
576 	}
577 
578 	if (PageSwapCache(page)) {
579 		swp_entry_t swap = { .val = page_private(page) };
580 		mem_cgroup_swapout(page, swap);
581 		__delete_from_swap_cache(page);
582 		spin_unlock_irq(&mapping->tree_lock);
583 		swapcache_free(swap);
584 	} else {
585 		void (*freepage)(struct page *);
586 		void *shadow = NULL;
587 
588 		freepage = mapping->a_ops->freepage;
589 		/*
590 		 * Remember a shadow entry for reclaimed file cache in
591 		 * order to detect refaults, thus thrashing, later on.
592 		 *
593 		 * But don't store shadows in an address space that is
594 		 * already exiting.  This is not just an optizimation,
595 		 * inode reclaim needs to empty out the radix tree or
596 		 * the nodes are lost.  Don't plant shadows behind its
597 		 * back.
598 		 */
599 		if (reclaimed && page_is_file_cache(page) &&
600 		    !mapping_exiting(mapping))
601 			shadow = workingset_eviction(mapping, page);
602 		__delete_from_page_cache(page, shadow);
603 		spin_unlock_irq(&mapping->tree_lock);
604 
605 		if (freepage != NULL)
606 			freepage(page);
607 	}
608 
609 	return 1;
610 
611 cannot_free:
612 	spin_unlock_irq(&mapping->tree_lock);
613 	return 0;
614 }
615 
616 /*
617  * Attempt to detach a locked page from its ->mapping.  If it is dirty or if
618  * someone else has a ref on the page, abort and return 0.  If it was
619  * successfully detached, return 1.  Assumes the caller has a single ref on
620  * this page.
621  */
622 int remove_mapping(struct address_space *mapping, struct page *page)
623 {
624 	if (__remove_mapping(mapping, page, false)) {
625 		/*
626 		 * Unfreezing the refcount with 1 rather than 2 effectively
627 		 * drops the pagecache ref for us without requiring another
628 		 * atomic operation.
629 		 */
630 		page_unfreeze_refs(page, 1);
631 		return 1;
632 	}
633 	return 0;
634 }
635 
636 /**
637  * putback_lru_page - put previously isolated page onto appropriate LRU list
638  * @page: page to be put back to appropriate lru list
639  *
640  * Add previously isolated @page to appropriate LRU list.
641  * Page may still be unevictable for other reasons.
642  *
643  * lru_lock must not be held, interrupts must be enabled.
644  */
645 void putback_lru_page(struct page *page)
646 {
647 	bool is_unevictable;
648 	int was_unevictable = PageUnevictable(page);
649 
650 	VM_BUG_ON_PAGE(PageLRU(page), page);
651 
652 redo:
653 	ClearPageUnevictable(page);
654 
655 	if (page_evictable(page)) {
656 		/*
657 		 * For evictable pages, we can use the cache.
658 		 * In event of a race, worst case is we end up with an
659 		 * unevictable page on [in]active list.
660 		 * We know how to handle that.
661 		 */
662 		is_unevictable = false;
663 		lru_cache_add(page);
664 	} else {
665 		/*
666 		 * Put unevictable pages directly on zone's unevictable
667 		 * list.
668 		 */
669 		is_unevictable = true;
670 		add_page_to_unevictable_list(page);
671 		/*
672 		 * When racing with an mlock or AS_UNEVICTABLE clearing
673 		 * (page is unlocked) make sure that if the other thread
674 		 * does not observe our setting of PG_lru and fails
675 		 * isolation/check_move_unevictable_pages,
676 		 * we see PG_mlocked/AS_UNEVICTABLE cleared below and move
677 		 * the page back to the evictable list.
678 		 *
679 		 * The other side is TestClearPageMlocked() or shmem_lock().
680 		 */
681 		smp_mb();
682 	}
683 
684 	/*
685 	 * page's status can change while we move it among lru. If an evictable
686 	 * page is on unevictable list, it never be freed. To avoid that,
687 	 * check after we added it to the list, again.
688 	 */
689 	if (is_unevictable && page_evictable(page)) {
690 		if (!isolate_lru_page(page)) {
691 			put_page(page);
692 			goto redo;
693 		}
694 		/* This means someone else dropped this page from LRU
695 		 * So, it will be freed or putback to LRU again. There is
696 		 * nothing to do here.
697 		 */
698 	}
699 
700 	if (was_unevictable && !is_unevictable)
701 		count_vm_event(UNEVICTABLE_PGRESCUED);
702 	else if (!was_unevictable && is_unevictable)
703 		count_vm_event(UNEVICTABLE_PGCULLED);
704 
705 	put_page(page);		/* drop ref from isolate */
706 }
707 
708 enum page_references {
709 	PAGEREF_RECLAIM,
710 	PAGEREF_RECLAIM_CLEAN,
711 	PAGEREF_KEEP,
712 	PAGEREF_ACTIVATE,
713 };
714 
715 static enum page_references page_check_references(struct page *page,
716 						  struct scan_control *sc)
717 {
718 	int referenced_ptes, referenced_page;
719 	unsigned long vm_flags;
720 
721 	referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
722 					  &vm_flags);
723 	referenced_page = TestClearPageReferenced(page);
724 
725 	/*
726 	 * Mlock lost the isolation race with us.  Let try_to_unmap()
727 	 * move the page to the unevictable list.
728 	 */
729 	if (vm_flags & VM_LOCKED)
730 		return PAGEREF_RECLAIM;
731 
732 	if (referenced_ptes) {
733 		if (PageSwapBacked(page))
734 			return PAGEREF_ACTIVATE;
735 		/*
736 		 * All mapped pages start out with page table
737 		 * references from the instantiating fault, so we need
738 		 * to look twice if a mapped file page is used more
739 		 * than once.
740 		 *
741 		 * Mark it and spare it for another trip around the
742 		 * inactive list.  Another page table reference will
743 		 * lead to its activation.
744 		 *
745 		 * Note: the mark is set for activated pages as well
746 		 * so that recently deactivated but used pages are
747 		 * quickly recovered.
748 		 */
749 		SetPageReferenced(page);
750 
751 		if (referenced_page || referenced_ptes > 1)
752 			return PAGEREF_ACTIVATE;
753 
754 		/*
755 		 * Activate file-backed executable pages after first usage.
756 		 */
757 		if (vm_flags & VM_EXEC)
758 			return PAGEREF_ACTIVATE;
759 
760 		return PAGEREF_KEEP;
761 	}
762 
763 	/* Reclaim if clean, defer dirty pages to writeback */
764 	if (referenced_page && !PageSwapBacked(page))
765 		return PAGEREF_RECLAIM_CLEAN;
766 
767 	return PAGEREF_RECLAIM;
768 }
769 
770 /* Check if a page is dirty or under writeback */
771 static void page_check_dirty_writeback(struct page *page,
772 				       bool *dirty, bool *writeback)
773 {
774 	struct address_space *mapping;
775 
776 	/*
777 	 * Anonymous pages are not handled by flushers and must be written
778 	 * from reclaim context. Do not stall reclaim based on them
779 	 */
780 	if (!page_is_file_cache(page)) {
781 		*dirty = false;
782 		*writeback = false;
783 		return;
784 	}
785 
786 	/* By default assume that the page flags are accurate */
787 	*dirty = PageDirty(page);
788 	*writeback = PageWriteback(page);
789 
790 	/* Verify dirty/writeback state if the filesystem supports it */
791 	if (!page_has_private(page))
792 		return;
793 
794 	mapping = page_mapping(page);
795 	if (mapping && mapping->a_ops->is_dirty_writeback)
796 		mapping->a_ops->is_dirty_writeback(page, dirty, writeback);
797 }
798 
799 /*
800  * shrink_page_list() returns the number of reclaimed pages
801  */
802 static unsigned long shrink_page_list(struct list_head *page_list,
803 				      struct zone *zone,
804 				      struct scan_control *sc,
805 				      enum ttu_flags ttu_flags,
806 				      unsigned long *ret_nr_dirty,
807 				      unsigned long *ret_nr_unqueued_dirty,
808 				      unsigned long *ret_nr_congested,
809 				      unsigned long *ret_nr_writeback,
810 				      unsigned long *ret_nr_immediate,
811 				      bool force_reclaim)
812 {
813 	LIST_HEAD(ret_pages);
814 	LIST_HEAD(free_pages);
815 	int pgactivate = 0;
816 	unsigned long nr_unqueued_dirty = 0;
817 	unsigned long nr_dirty = 0;
818 	unsigned long nr_congested = 0;
819 	unsigned long nr_reclaimed = 0;
820 	unsigned long nr_writeback = 0;
821 	unsigned long nr_immediate = 0;
822 
823 	cond_resched();
824 
825 	while (!list_empty(page_list)) {
826 		struct address_space *mapping;
827 		struct page *page;
828 		int may_enter_fs;
829 		enum page_references references = PAGEREF_RECLAIM_CLEAN;
830 		bool dirty, writeback;
831 
832 		cond_resched();
833 
834 		page = lru_to_page(page_list);
835 		list_del(&page->lru);
836 
837 		if (!trylock_page(page))
838 			goto keep;
839 
840 		VM_BUG_ON_PAGE(PageActive(page), page);
841 		VM_BUG_ON_PAGE(page_zone(page) != zone, page);
842 
843 		sc->nr_scanned++;
844 
845 		if (unlikely(!page_evictable(page)))
846 			goto cull_mlocked;
847 
848 		if (!sc->may_unmap && page_mapped(page))
849 			goto keep_locked;
850 
851 		/* Double the slab pressure for mapped and swapcache pages */
852 		if (page_mapped(page) || PageSwapCache(page))
853 			sc->nr_scanned++;
854 
855 		may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
856 			(PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
857 
858 		/*
859 		 * The number of dirty pages determines if a zone is marked
860 		 * reclaim_congested which affects wait_iff_congested. kswapd
861 		 * will stall and start writing pages if the tail of the LRU
862 		 * is all dirty unqueued pages.
863 		 */
864 		page_check_dirty_writeback(page, &dirty, &writeback);
865 		if (dirty || writeback)
866 			nr_dirty++;
867 
868 		if (dirty && !writeback)
869 			nr_unqueued_dirty++;
870 
871 		/*
872 		 * Treat this page as congested if the underlying BDI is or if
873 		 * pages are cycling through the LRU so quickly that the
874 		 * pages marked for immediate reclaim are making it to the
875 		 * end of the LRU a second time.
876 		 */
877 		mapping = page_mapping(page);
878 		if ((mapping && bdi_write_congested(mapping->backing_dev_info)) ||
879 		    (writeback && PageReclaim(page)))
880 			nr_congested++;
881 
882 		/*
883 		 * If a page at the tail of the LRU is under writeback, there
884 		 * are three cases to consider.
885 		 *
886 		 * 1) If reclaim is encountering an excessive number of pages
887 		 *    under writeback and this page is both under writeback and
888 		 *    PageReclaim then it indicates that pages are being queued
889 		 *    for IO but are being recycled through the LRU before the
890 		 *    IO can complete. Waiting on the page itself risks an
891 		 *    indefinite stall if it is impossible to writeback the
892 		 *    page due to IO error or disconnected storage so instead
893 		 *    note that the LRU is being scanned too quickly and the
894 		 *    caller can stall after page list has been processed.
895 		 *
896 		 * 2) Global reclaim encounters a page, memcg encounters a
897 		 *    page that is not marked for immediate reclaim or
898 		 *    the caller does not have __GFP_IO. In this case mark
899 		 *    the page for immediate reclaim and continue scanning.
900 		 *
901 		 *    __GFP_IO is checked  because a loop driver thread might
902 		 *    enter reclaim, and deadlock if it waits on a page for
903 		 *    which it is needed to do the write (loop masks off
904 		 *    __GFP_IO|__GFP_FS for this reason); but more thought
905 		 *    would probably show more reasons.
906 		 *
907 		 *    Don't require __GFP_FS, since we're not going into the
908 		 *    FS, just waiting on its writeback completion. Worryingly,
909 		 *    ext4 gfs2 and xfs allocate pages with
910 		 *    grab_cache_page_write_begin(,,AOP_FLAG_NOFS), so testing
911 		 *    may_enter_fs here is liable to OOM on them.
912 		 *
913 		 * 3) memcg encounters a page that is not already marked
914 		 *    PageReclaim. memcg does not have any dirty pages
915 		 *    throttling so we could easily OOM just because too many
916 		 *    pages are in writeback and there is nothing else to
917 		 *    reclaim. Wait for the writeback to complete.
918 		 */
919 		if (PageWriteback(page)) {
920 			/* Case 1 above */
921 			if (current_is_kswapd() &&
922 			    PageReclaim(page) &&
923 			    test_bit(ZONE_WRITEBACK, &zone->flags)) {
924 				nr_immediate++;
925 				goto keep_locked;
926 
927 			/* Case 2 above */
928 			} else if (global_reclaim(sc) ||
929 			    !PageReclaim(page) || !(sc->gfp_mask & __GFP_IO)) {
930 				/*
931 				 * This is slightly racy - end_page_writeback()
932 				 * might have just cleared PageReclaim, then
933 				 * setting PageReclaim here end up interpreted
934 				 * as PageReadahead - but that does not matter
935 				 * enough to care.  What we do want is for this
936 				 * page to have PageReclaim set next time memcg
937 				 * reclaim reaches the tests above, so it will
938 				 * then wait_on_page_writeback() to avoid OOM;
939 				 * and it's also appropriate in global reclaim.
940 				 */
941 				SetPageReclaim(page);
942 				nr_writeback++;
943 
944 				goto keep_locked;
945 
946 			/* Case 3 above */
947 			} else {
948 				wait_on_page_writeback(page);
949 			}
950 		}
951 
952 		if (!force_reclaim)
953 			references = page_check_references(page, sc);
954 
955 		switch (references) {
956 		case PAGEREF_ACTIVATE:
957 			goto activate_locked;
958 		case PAGEREF_KEEP:
959 			goto keep_locked;
960 		case PAGEREF_RECLAIM:
961 		case PAGEREF_RECLAIM_CLEAN:
962 			; /* try to reclaim the page below */
963 		}
964 
965 		/*
966 		 * Anonymous process memory has backing store?
967 		 * Try to allocate it some swap space here.
968 		 */
969 		if (PageAnon(page) && !PageSwapCache(page)) {
970 			if (!(sc->gfp_mask & __GFP_IO))
971 				goto keep_locked;
972 			if (!add_to_swap(page, page_list))
973 				goto activate_locked;
974 			may_enter_fs = 1;
975 
976 			/* Adding to swap updated mapping */
977 			mapping = page_mapping(page);
978 		}
979 
980 		/*
981 		 * The page is mapped into the page tables of one or more
982 		 * processes. Try to unmap it here.
983 		 */
984 		if (page_mapped(page) && mapping) {
985 			switch (try_to_unmap(page, ttu_flags)) {
986 			case SWAP_FAIL:
987 				goto activate_locked;
988 			case SWAP_AGAIN:
989 				goto keep_locked;
990 			case SWAP_MLOCK:
991 				goto cull_mlocked;
992 			case SWAP_SUCCESS:
993 				; /* try to free the page below */
994 			}
995 		}
996 
997 		if (PageDirty(page)) {
998 			/*
999 			 * Only kswapd can writeback filesystem pages to
1000 			 * avoid risk of stack overflow but only writeback
1001 			 * if many dirty pages have been encountered.
1002 			 */
1003 			if (page_is_file_cache(page) &&
1004 					(!current_is_kswapd() ||
1005 					 !test_bit(ZONE_DIRTY, &zone->flags))) {
1006 				/*
1007 				 * Immediately reclaim when written back.
1008 				 * Similar in principal to deactivate_page()
1009 				 * except we already have the page isolated
1010 				 * and know it's dirty
1011 				 */
1012 				inc_zone_page_state(page, NR_VMSCAN_IMMEDIATE);
1013 				SetPageReclaim(page);
1014 
1015 				goto keep_locked;
1016 			}
1017 
1018 			if (references == PAGEREF_RECLAIM_CLEAN)
1019 				goto keep_locked;
1020 			if (!may_enter_fs)
1021 				goto keep_locked;
1022 			if (!sc->may_writepage)
1023 				goto keep_locked;
1024 
1025 			/* Page is dirty, try to write it out here */
1026 			switch (pageout(page, mapping, sc)) {
1027 			case PAGE_KEEP:
1028 				goto keep_locked;
1029 			case PAGE_ACTIVATE:
1030 				goto activate_locked;
1031 			case PAGE_SUCCESS:
1032 				if (PageWriteback(page))
1033 					goto keep;
1034 				if (PageDirty(page))
1035 					goto keep;
1036 
1037 				/*
1038 				 * A synchronous write - probably a ramdisk.  Go
1039 				 * ahead and try to reclaim the page.
1040 				 */
1041 				if (!trylock_page(page))
1042 					goto keep;
1043 				if (PageDirty(page) || PageWriteback(page))
1044 					goto keep_locked;
1045 				mapping = page_mapping(page);
1046 			case PAGE_CLEAN:
1047 				; /* try to free the page below */
1048 			}
1049 		}
1050 
1051 		/*
1052 		 * If the page has buffers, try to free the buffer mappings
1053 		 * associated with this page. If we succeed we try to free
1054 		 * the page as well.
1055 		 *
1056 		 * We do this even if the page is PageDirty().
1057 		 * try_to_release_page() does not perform I/O, but it is
1058 		 * possible for a page to have PageDirty set, but it is actually
1059 		 * clean (all its buffers are clean).  This happens if the
1060 		 * buffers were written out directly, with submit_bh(). ext3
1061 		 * will do this, as well as the blockdev mapping.
1062 		 * try_to_release_page() will discover that cleanness and will
1063 		 * drop the buffers and mark the page clean - it can be freed.
1064 		 *
1065 		 * Rarely, pages can have buffers and no ->mapping.  These are
1066 		 * the pages which were not successfully invalidated in
1067 		 * truncate_complete_page().  We try to drop those buffers here
1068 		 * and if that worked, and the page is no longer mapped into
1069 		 * process address space (page_count == 1) it can be freed.
1070 		 * Otherwise, leave the page on the LRU so it is swappable.
1071 		 */
1072 		if (page_has_private(page)) {
1073 			if (!try_to_release_page(page, sc->gfp_mask))
1074 				goto activate_locked;
1075 			if (!mapping && page_count(page) == 1) {
1076 				unlock_page(page);
1077 				if (put_page_testzero(page))
1078 					goto free_it;
1079 				else {
1080 					/*
1081 					 * rare race with speculative reference.
1082 					 * the speculative reference will free
1083 					 * this page shortly, so we may
1084 					 * increment nr_reclaimed here (and
1085 					 * leave it off the LRU).
1086 					 */
1087 					nr_reclaimed++;
1088 					continue;
1089 				}
1090 			}
1091 		}
1092 
1093 		if (!mapping || !__remove_mapping(mapping, page, true))
1094 			goto keep_locked;
1095 
1096 		/*
1097 		 * At this point, we have no other references and there is
1098 		 * no way to pick any more up (removed from LRU, removed
1099 		 * from pagecache). Can use non-atomic bitops now (and
1100 		 * we obviously don't have to worry about waking up a process
1101 		 * waiting on the page lock, because there are no references.
1102 		 */
1103 		__clear_page_locked(page);
1104 free_it:
1105 		nr_reclaimed++;
1106 
1107 		/*
1108 		 * Is there need to periodically free_page_list? It would
1109 		 * appear not as the counts should be low
1110 		 */
1111 		list_add(&page->lru, &free_pages);
1112 		continue;
1113 
1114 cull_mlocked:
1115 		if (PageSwapCache(page))
1116 			try_to_free_swap(page);
1117 		unlock_page(page);
1118 		putback_lru_page(page);
1119 		continue;
1120 
1121 activate_locked:
1122 		/* Not a candidate for swapping, so reclaim swap space. */
1123 		if (PageSwapCache(page) && vm_swap_full())
1124 			try_to_free_swap(page);
1125 		VM_BUG_ON_PAGE(PageActive(page), page);
1126 		SetPageActive(page);
1127 		pgactivate++;
1128 keep_locked:
1129 		unlock_page(page);
1130 keep:
1131 		list_add(&page->lru, &ret_pages);
1132 		VM_BUG_ON_PAGE(PageLRU(page) || PageUnevictable(page), page);
1133 	}
1134 
1135 	mem_cgroup_uncharge_list(&free_pages);
1136 	free_hot_cold_page_list(&free_pages, true);
1137 
1138 	list_splice(&ret_pages, page_list);
1139 	count_vm_events(PGACTIVATE, pgactivate);
1140 
1141 	*ret_nr_dirty += nr_dirty;
1142 	*ret_nr_congested += nr_congested;
1143 	*ret_nr_unqueued_dirty += nr_unqueued_dirty;
1144 	*ret_nr_writeback += nr_writeback;
1145 	*ret_nr_immediate += nr_immediate;
1146 	return nr_reclaimed;
1147 }
1148 
1149 unsigned long reclaim_clean_pages_from_list(struct zone *zone,
1150 					    struct list_head *page_list)
1151 {
1152 	struct scan_control sc = {
1153 		.gfp_mask = GFP_KERNEL,
1154 		.priority = DEF_PRIORITY,
1155 		.may_unmap = 1,
1156 	};
1157 	unsigned long ret, dummy1, dummy2, dummy3, dummy4, dummy5;
1158 	struct page *page, *next;
1159 	LIST_HEAD(clean_pages);
1160 
1161 	list_for_each_entry_safe(page, next, page_list, lru) {
1162 		if (page_is_file_cache(page) && !PageDirty(page) &&
1163 		    !isolated_balloon_page(page)) {
1164 			ClearPageActive(page);
1165 			list_move(&page->lru, &clean_pages);
1166 		}
1167 	}
1168 
1169 	ret = shrink_page_list(&clean_pages, zone, &sc,
1170 			TTU_UNMAP|TTU_IGNORE_ACCESS,
1171 			&dummy1, &dummy2, &dummy3, &dummy4, &dummy5, true);
1172 	list_splice(&clean_pages, page_list);
1173 	mod_zone_page_state(zone, NR_ISOLATED_FILE, -ret);
1174 	return ret;
1175 }
1176 
1177 /*
1178  * Attempt to remove the specified page from its LRU.  Only take this page
1179  * if it is of the appropriate PageActive status.  Pages which are being
1180  * freed elsewhere are also ignored.
1181  *
1182  * page:	page to consider
1183  * mode:	one of the LRU isolation modes defined above
1184  *
1185  * returns 0 on success, -ve errno on failure.
1186  */
1187 int __isolate_lru_page(struct page *page, isolate_mode_t mode)
1188 {
1189 	int ret = -EINVAL;
1190 
1191 	/* Only take pages on the LRU. */
1192 	if (!PageLRU(page))
1193 		return ret;
1194 
1195 	/* Compaction should not handle unevictable pages but CMA can do so */
1196 	if (PageUnevictable(page) && !(mode & ISOLATE_UNEVICTABLE))
1197 		return ret;
1198 
1199 	ret = -EBUSY;
1200 
1201 	/*
1202 	 * To minimise LRU disruption, the caller can indicate that it only
1203 	 * wants to isolate pages it will be able to operate on without
1204 	 * blocking - clean pages for the most part.
1205 	 *
1206 	 * ISOLATE_CLEAN means that only clean pages should be isolated. This
1207 	 * is used by reclaim when it is cannot write to backing storage
1208 	 *
1209 	 * ISOLATE_ASYNC_MIGRATE is used to indicate that it only wants to pages
1210 	 * that it is possible to migrate without blocking
1211 	 */
1212 	if (mode & (ISOLATE_CLEAN|ISOLATE_ASYNC_MIGRATE)) {
1213 		/* All the caller can do on PageWriteback is block */
1214 		if (PageWriteback(page))
1215 			return ret;
1216 
1217 		if (PageDirty(page)) {
1218 			struct address_space *mapping;
1219 
1220 			/* ISOLATE_CLEAN means only clean pages */
1221 			if (mode & ISOLATE_CLEAN)
1222 				return ret;
1223 
1224 			/*
1225 			 * Only pages without mappings or that have a
1226 			 * ->migratepage callback are possible to migrate
1227 			 * without blocking
1228 			 */
1229 			mapping = page_mapping(page);
1230 			if (mapping && !mapping->a_ops->migratepage)
1231 				return ret;
1232 		}
1233 	}
1234 
1235 	if ((mode & ISOLATE_UNMAPPED) && page_mapped(page))
1236 		return ret;
1237 
1238 	if (likely(get_page_unless_zero(page))) {
1239 		/*
1240 		 * Be careful not to clear PageLRU until after we're
1241 		 * sure the page is not being freed elsewhere -- the
1242 		 * page release code relies on it.
1243 		 */
1244 		ClearPageLRU(page);
1245 		ret = 0;
1246 	}
1247 
1248 	return ret;
1249 }
1250 
1251 /*
1252  * zone->lru_lock is heavily contended.  Some of the functions that
1253  * shrink the lists perform better by taking out a batch of pages
1254  * and working on them outside the LRU lock.
1255  *
1256  * For pagecache intensive workloads, this function is the hottest
1257  * spot in the kernel (apart from copy_*_user functions).
1258  *
1259  * Appropriate locks must be held before calling this function.
1260  *
1261  * @nr_to_scan:	The number of pages to look through on the list.
1262  * @lruvec:	The LRU vector to pull pages from.
1263  * @dst:	The temp list to put pages on to.
1264  * @nr_scanned:	The number of pages that were scanned.
1265  * @sc:		The scan_control struct for this reclaim session
1266  * @mode:	One of the LRU isolation modes
1267  * @lru:	LRU list id for isolating
1268  *
1269  * returns how many pages were moved onto *@dst.
1270  */
1271 static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
1272 		struct lruvec *lruvec, struct list_head *dst,
1273 		unsigned long *nr_scanned, struct scan_control *sc,
1274 		isolate_mode_t mode, enum lru_list lru)
1275 {
1276 	struct list_head *src = &lruvec->lists[lru];
1277 	unsigned long nr_taken = 0;
1278 	unsigned long scan;
1279 
1280 	for (scan = 0; scan < nr_to_scan && !list_empty(src); scan++) {
1281 		struct page *page;
1282 		int nr_pages;
1283 
1284 		page = lru_to_page(src);
1285 		prefetchw_prev_lru_page(page, src, flags);
1286 
1287 		VM_BUG_ON_PAGE(!PageLRU(page), page);
1288 
1289 		switch (__isolate_lru_page(page, mode)) {
1290 		case 0:
1291 			nr_pages = hpage_nr_pages(page);
1292 			mem_cgroup_update_lru_size(lruvec, lru, -nr_pages);
1293 			list_move(&page->lru, dst);
1294 			nr_taken += nr_pages;
1295 			break;
1296 
1297 		case -EBUSY:
1298 			/* else it is being freed elsewhere */
1299 			list_move(&page->lru, src);
1300 			continue;
1301 
1302 		default:
1303 			BUG();
1304 		}
1305 	}
1306 
1307 	*nr_scanned = scan;
1308 	trace_mm_vmscan_lru_isolate(sc->order, nr_to_scan, scan,
1309 				    nr_taken, mode, is_file_lru(lru));
1310 	return nr_taken;
1311 }
1312 
1313 /**
1314  * isolate_lru_page - tries to isolate a page from its LRU list
1315  * @page: page to isolate from its LRU list
1316  *
1317  * Isolates a @page from an LRU list, clears PageLRU and adjusts the
1318  * vmstat statistic corresponding to whatever LRU list the page was on.
1319  *
1320  * Returns 0 if the page was removed from an LRU list.
1321  * Returns -EBUSY if the page was not on an LRU list.
1322  *
1323  * The returned page will have PageLRU() cleared.  If it was found on
1324  * the active list, it will have PageActive set.  If it was found on
1325  * the unevictable list, it will have the PageUnevictable bit set. That flag
1326  * may need to be cleared by the caller before letting the page go.
1327  *
1328  * The vmstat statistic corresponding to the list on which the page was
1329  * found will be decremented.
1330  *
1331  * Restrictions:
1332  * (1) Must be called with an elevated refcount on the page. This is a
1333  *     fundamentnal difference from isolate_lru_pages (which is called
1334  *     without a stable reference).
1335  * (2) the lru_lock must not be held.
1336  * (3) interrupts must be enabled.
1337  */
1338 int isolate_lru_page(struct page *page)
1339 {
1340 	int ret = -EBUSY;
1341 
1342 	VM_BUG_ON_PAGE(!page_count(page), page);
1343 
1344 	if (PageLRU(page)) {
1345 		struct zone *zone = page_zone(page);
1346 		struct lruvec *lruvec;
1347 
1348 		spin_lock_irq(&zone->lru_lock);
1349 		lruvec = mem_cgroup_page_lruvec(page, zone);
1350 		if (PageLRU(page)) {
1351 			int lru = page_lru(page);
1352 			get_page(page);
1353 			ClearPageLRU(page);
1354 			del_page_from_lru_list(page, lruvec, lru);
1355 			ret = 0;
1356 		}
1357 		spin_unlock_irq(&zone->lru_lock);
1358 	}
1359 	return ret;
1360 }
1361 
1362 /*
1363  * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
1364  * then get resheduled. When there are massive number of tasks doing page
1365  * allocation, such sleeping direct reclaimers may keep piling up on each CPU,
1366  * the LRU list will go small and be scanned faster than necessary, leading to
1367  * unnecessary swapping, thrashing and OOM.
1368  */
1369 static int too_many_isolated(struct zone *zone, int file,
1370 		struct scan_control *sc)
1371 {
1372 	unsigned long inactive, isolated;
1373 
1374 	if (current_is_kswapd())
1375 		return 0;
1376 
1377 	if (!global_reclaim(sc))
1378 		return 0;
1379 
1380 	if (file) {
1381 		inactive = zone_page_state(zone, NR_INACTIVE_FILE);
1382 		isolated = zone_page_state(zone, NR_ISOLATED_FILE);
1383 	} else {
1384 		inactive = zone_page_state(zone, NR_INACTIVE_ANON);
1385 		isolated = zone_page_state(zone, NR_ISOLATED_ANON);
1386 	}
1387 
1388 	/*
1389 	 * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they
1390 	 * won't get blocked by normal direct-reclaimers, forming a circular
1391 	 * deadlock.
1392 	 */
1393 	if ((sc->gfp_mask & GFP_IOFS) == GFP_IOFS)
1394 		inactive >>= 3;
1395 
1396 	return isolated > inactive;
1397 }
1398 
1399 static noinline_for_stack void
1400 putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list)
1401 {
1402 	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1403 	struct zone *zone = lruvec_zone(lruvec);
1404 	LIST_HEAD(pages_to_free);
1405 
1406 	/*
1407 	 * Put back any unfreeable pages.
1408 	 */
1409 	while (!list_empty(page_list)) {
1410 		struct page *page = lru_to_page(page_list);
1411 		int lru;
1412 
1413 		VM_BUG_ON_PAGE(PageLRU(page), page);
1414 		list_del(&page->lru);
1415 		if (unlikely(!page_evictable(page))) {
1416 			spin_unlock_irq(&zone->lru_lock);
1417 			putback_lru_page(page);
1418 			spin_lock_irq(&zone->lru_lock);
1419 			continue;
1420 		}
1421 
1422 		lruvec = mem_cgroup_page_lruvec(page, zone);
1423 
1424 		SetPageLRU(page);
1425 		lru = page_lru(page);
1426 		add_page_to_lru_list(page, lruvec, lru);
1427 
1428 		if (is_active_lru(lru)) {
1429 			int file = is_file_lru(lru);
1430 			int numpages = hpage_nr_pages(page);
1431 			reclaim_stat->recent_rotated[file] += numpages;
1432 		}
1433 		if (put_page_testzero(page)) {
1434 			__ClearPageLRU(page);
1435 			__ClearPageActive(page);
1436 			del_page_from_lru_list(page, lruvec, lru);
1437 
1438 			if (unlikely(PageCompound(page))) {
1439 				spin_unlock_irq(&zone->lru_lock);
1440 				mem_cgroup_uncharge(page);
1441 				(*get_compound_page_dtor(page))(page);
1442 				spin_lock_irq(&zone->lru_lock);
1443 			} else
1444 				list_add(&page->lru, &pages_to_free);
1445 		}
1446 	}
1447 
1448 	/*
1449 	 * To save our caller's stack, now use input list for pages to free.
1450 	 */
1451 	list_splice(&pages_to_free, page_list);
1452 }
1453 
1454 /*
1455  * If a kernel thread (such as nfsd for loop-back mounts) services
1456  * a backing device by writing to the page cache it sets PF_LESS_THROTTLE.
1457  * In that case we should only throttle if the backing device it is
1458  * writing to is congested.  In other cases it is safe to throttle.
1459  */
1460 static int current_may_throttle(void)
1461 {
1462 	return !(current->flags & PF_LESS_THROTTLE) ||
1463 		current->backing_dev_info == NULL ||
1464 		bdi_write_congested(current->backing_dev_info);
1465 }
1466 
1467 /*
1468  * shrink_inactive_list() is a helper for shrink_zone().  It returns the number
1469  * of reclaimed pages
1470  */
1471 static noinline_for_stack unsigned long
1472 shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
1473 		     struct scan_control *sc, enum lru_list lru)
1474 {
1475 	LIST_HEAD(page_list);
1476 	unsigned long nr_scanned;
1477 	unsigned long nr_reclaimed = 0;
1478 	unsigned long nr_taken;
1479 	unsigned long nr_dirty = 0;
1480 	unsigned long nr_congested = 0;
1481 	unsigned long nr_unqueued_dirty = 0;
1482 	unsigned long nr_writeback = 0;
1483 	unsigned long nr_immediate = 0;
1484 	isolate_mode_t isolate_mode = 0;
1485 	int file = is_file_lru(lru);
1486 	struct zone *zone = lruvec_zone(lruvec);
1487 	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1488 
1489 	while (unlikely(too_many_isolated(zone, file, sc))) {
1490 		congestion_wait(BLK_RW_ASYNC, HZ/10);
1491 
1492 		/* We are about to die and free our memory. Return now. */
1493 		if (fatal_signal_pending(current))
1494 			return SWAP_CLUSTER_MAX;
1495 	}
1496 
1497 	lru_add_drain();
1498 
1499 	if (!sc->may_unmap)
1500 		isolate_mode |= ISOLATE_UNMAPPED;
1501 	if (!sc->may_writepage)
1502 		isolate_mode |= ISOLATE_CLEAN;
1503 
1504 	spin_lock_irq(&zone->lru_lock);
1505 
1506 	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
1507 				     &nr_scanned, sc, isolate_mode, lru);
1508 
1509 	__mod_zone_page_state(zone, NR_LRU_BASE + lru, -nr_taken);
1510 	__mod_zone_page_state(zone, NR_ISOLATED_ANON + file, nr_taken);
1511 
1512 	if (global_reclaim(sc)) {
1513 		__mod_zone_page_state(zone, NR_PAGES_SCANNED, nr_scanned);
1514 		if (current_is_kswapd())
1515 			__count_zone_vm_events(PGSCAN_KSWAPD, zone, nr_scanned);
1516 		else
1517 			__count_zone_vm_events(PGSCAN_DIRECT, zone, nr_scanned);
1518 	}
1519 	spin_unlock_irq(&zone->lru_lock);
1520 
1521 	if (nr_taken == 0)
1522 		return 0;
1523 
1524 	nr_reclaimed = shrink_page_list(&page_list, zone, sc, TTU_UNMAP,
1525 				&nr_dirty, &nr_unqueued_dirty, &nr_congested,
1526 				&nr_writeback, &nr_immediate,
1527 				false);
1528 
1529 	spin_lock_irq(&zone->lru_lock);
1530 
1531 	reclaim_stat->recent_scanned[file] += nr_taken;
1532 
1533 	if (global_reclaim(sc)) {
1534 		if (current_is_kswapd())
1535 			__count_zone_vm_events(PGSTEAL_KSWAPD, zone,
1536 					       nr_reclaimed);
1537 		else
1538 			__count_zone_vm_events(PGSTEAL_DIRECT, zone,
1539 					       nr_reclaimed);
1540 	}
1541 
1542 	putback_inactive_pages(lruvec, &page_list);
1543 
1544 	__mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
1545 
1546 	spin_unlock_irq(&zone->lru_lock);
1547 
1548 	mem_cgroup_uncharge_list(&page_list);
1549 	free_hot_cold_page_list(&page_list, true);
1550 
1551 	/*
1552 	 * If reclaim is isolating dirty pages under writeback, it implies
1553 	 * that the long-lived page allocation rate is exceeding the page
1554 	 * laundering rate. Either the global limits are not being effective
1555 	 * at throttling processes due to the page distribution throughout
1556 	 * zones or there is heavy usage of a slow backing device. The
1557 	 * only option is to throttle from reclaim context which is not ideal
1558 	 * as there is no guarantee the dirtying process is throttled in the
1559 	 * same way balance_dirty_pages() manages.
1560 	 *
1561 	 * Once a zone is flagged ZONE_WRITEBACK, kswapd will count the number
1562 	 * of pages under pages flagged for immediate reclaim and stall if any
1563 	 * are encountered in the nr_immediate check below.
1564 	 */
1565 	if (nr_writeback && nr_writeback == nr_taken)
1566 		set_bit(ZONE_WRITEBACK, &zone->flags);
1567 
1568 	/*
1569 	 * memcg will stall in page writeback so only consider forcibly
1570 	 * stalling for global reclaim
1571 	 */
1572 	if (global_reclaim(sc)) {
1573 		/*
1574 		 * Tag a zone as congested if all the dirty pages scanned were
1575 		 * backed by a congested BDI and wait_iff_congested will stall.
1576 		 */
1577 		if (nr_dirty && nr_dirty == nr_congested)
1578 			set_bit(ZONE_CONGESTED, &zone->flags);
1579 
1580 		/*
1581 		 * If dirty pages are scanned that are not queued for IO, it
1582 		 * implies that flushers are not keeping up. In this case, flag
1583 		 * the zone ZONE_DIRTY and kswapd will start writing pages from
1584 		 * reclaim context.
1585 		 */
1586 		if (nr_unqueued_dirty == nr_taken)
1587 			set_bit(ZONE_DIRTY, &zone->flags);
1588 
1589 		/*
1590 		 * If kswapd scans pages marked marked for immediate
1591 		 * reclaim and under writeback (nr_immediate), it implies
1592 		 * that pages are cycling through the LRU faster than
1593 		 * they are written so also forcibly stall.
1594 		 */
1595 		if (nr_immediate && current_may_throttle())
1596 			congestion_wait(BLK_RW_ASYNC, HZ/10);
1597 	}
1598 
1599 	/*
1600 	 * Stall direct reclaim for IO completions if underlying BDIs or zone
1601 	 * is congested. Allow kswapd to continue until it starts encountering
1602 	 * unqueued dirty pages or cycling through the LRU too quickly.
1603 	 */
1604 	if (!sc->hibernation_mode && !current_is_kswapd() &&
1605 	    current_may_throttle())
1606 		wait_iff_congested(zone, BLK_RW_ASYNC, HZ/10);
1607 
1608 	trace_mm_vmscan_lru_shrink_inactive(zone->zone_pgdat->node_id,
1609 		zone_idx(zone),
1610 		nr_scanned, nr_reclaimed,
1611 		sc->priority,
1612 		trace_shrink_flags(file));
1613 	return nr_reclaimed;
1614 }
1615 
1616 /*
1617  * This moves pages from the active list to the inactive list.
1618  *
1619  * We move them the other way if the page is referenced by one or more
1620  * processes, from rmap.
1621  *
1622  * If the pages are mostly unmapped, the processing is fast and it is
1623  * appropriate to hold zone->lru_lock across the whole operation.  But if
1624  * the pages are mapped, the processing is slow (page_referenced()) so we
1625  * should drop zone->lru_lock around each page.  It's impossible to balance
1626  * this, so instead we remove the pages from the LRU while processing them.
1627  * It is safe to rely on PG_active against the non-LRU pages in here because
1628  * nobody will play with that bit on a non-LRU page.
1629  *
1630  * The downside is that we have to touch page->_count against each page.
1631  * But we had to alter page->flags anyway.
1632  */
1633 
1634 static void move_active_pages_to_lru(struct lruvec *lruvec,
1635 				     struct list_head *list,
1636 				     struct list_head *pages_to_free,
1637 				     enum lru_list lru)
1638 {
1639 	struct zone *zone = lruvec_zone(lruvec);
1640 	unsigned long pgmoved = 0;
1641 	struct page *page;
1642 	int nr_pages;
1643 
1644 	while (!list_empty(list)) {
1645 		page = lru_to_page(list);
1646 		lruvec = mem_cgroup_page_lruvec(page, zone);
1647 
1648 		VM_BUG_ON_PAGE(PageLRU(page), page);
1649 		SetPageLRU(page);
1650 
1651 		nr_pages = hpage_nr_pages(page);
1652 		mem_cgroup_update_lru_size(lruvec, lru, nr_pages);
1653 		list_move(&page->lru, &lruvec->lists[lru]);
1654 		pgmoved += nr_pages;
1655 
1656 		if (put_page_testzero(page)) {
1657 			__ClearPageLRU(page);
1658 			__ClearPageActive(page);
1659 			del_page_from_lru_list(page, lruvec, lru);
1660 
1661 			if (unlikely(PageCompound(page))) {
1662 				spin_unlock_irq(&zone->lru_lock);
1663 				mem_cgroup_uncharge(page);
1664 				(*get_compound_page_dtor(page))(page);
1665 				spin_lock_irq(&zone->lru_lock);
1666 			} else
1667 				list_add(&page->lru, pages_to_free);
1668 		}
1669 	}
1670 	__mod_zone_page_state(zone, NR_LRU_BASE + lru, pgmoved);
1671 	if (!is_active_lru(lru))
1672 		__count_vm_events(PGDEACTIVATE, pgmoved);
1673 }
1674 
1675 static void shrink_active_list(unsigned long nr_to_scan,
1676 			       struct lruvec *lruvec,
1677 			       struct scan_control *sc,
1678 			       enum lru_list lru)
1679 {
1680 	unsigned long nr_taken;
1681 	unsigned long nr_scanned;
1682 	unsigned long vm_flags;
1683 	LIST_HEAD(l_hold);	/* The pages which were snipped off */
1684 	LIST_HEAD(l_active);
1685 	LIST_HEAD(l_inactive);
1686 	struct page *page;
1687 	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1688 	unsigned long nr_rotated = 0;
1689 	isolate_mode_t isolate_mode = 0;
1690 	int file = is_file_lru(lru);
1691 	struct zone *zone = lruvec_zone(lruvec);
1692 
1693 	lru_add_drain();
1694 
1695 	if (!sc->may_unmap)
1696 		isolate_mode |= ISOLATE_UNMAPPED;
1697 	if (!sc->may_writepage)
1698 		isolate_mode |= ISOLATE_CLEAN;
1699 
1700 	spin_lock_irq(&zone->lru_lock);
1701 
1702 	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
1703 				     &nr_scanned, sc, isolate_mode, lru);
1704 	if (global_reclaim(sc))
1705 		__mod_zone_page_state(zone, NR_PAGES_SCANNED, nr_scanned);
1706 
1707 	reclaim_stat->recent_scanned[file] += nr_taken;
1708 
1709 	__count_zone_vm_events(PGREFILL, zone, nr_scanned);
1710 	__mod_zone_page_state(zone, NR_LRU_BASE + lru, -nr_taken);
1711 	__mod_zone_page_state(zone, NR_ISOLATED_ANON + file, nr_taken);
1712 	spin_unlock_irq(&zone->lru_lock);
1713 
1714 	while (!list_empty(&l_hold)) {
1715 		cond_resched();
1716 		page = lru_to_page(&l_hold);
1717 		list_del(&page->lru);
1718 
1719 		if (unlikely(!page_evictable(page))) {
1720 			putback_lru_page(page);
1721 			continue;
1722 		}
1723 
1724 		if (unlikely(buffer_heads_over_limit)) {
1725 			if (page_has_private(page) && trylock_page(page)) {
1726 				if (page_has_private(page))
1727 					try_to_release_page(page, 0);
1728 				unlock_page(page);
1729 			}
1730 		}
1731 
1732 		if (page_referenced(page, 0, sc->target_mem_cgroup,
1733 				    &vm_flags)) {
1734 			nr_rotated += hpage_nr_pages(page);
1735 			/*
1736 			 * Identify referenced, file-backed active pages and
1737 			 * give them one more trip around the active list. So
1738 			 * that executable code get better chances to stay in
1739 			 * memory under moderate memory pressure.  Anon pages
1740 			 * are not likely to be evicted by use-once streaming
1741 			 * IO, plus JVM can create lots of anon VM_EXEC pages,
1742 			 * so we ignore them here.
1743 			 */
1744 			if ((vm_flags & VM_EXEC) && page_is_file_cache(page)) {
1745 				list_add(&page->lru, &l_active);
1746 				continue;
1747 			}
1748 		}
1749 
1750 		ClearPageActive(page);	/* we are de-activating */
1751 		list_add(&page->lru, &l_inactive);
1752 	}
1753 
1754 	/*
1755 	 * Move pages back to the lru list.
1756 	 */
1757 	spin_lock_irq(&zone->lru_lock);
1758 	/*
1759 	 * Count referenced pages from currently used mappings as rotated,
1760 	 * even though only some of them are actually re-activated.  This
1761 	 * helps balance scan pressure between file and anonymous pages in
1762 	 * get_scan_count.
1763 	 */
1764 	reclaim_stat->recent_rotated[file] += nr_rotated;
1765 
1766 	move_active_pages_to_lru(lruvec, &l_active, &l_hold, lru);
1767 	move_active_pages_to_lru(lruvec, &l_inactive, &l_hold, lru - LRU_ACTIVE);
1768 	__mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
1769 	spin_unlock_irq(&zone->lru_lock);
1770 
1771 	mem_cgroup_uncharge_list(&l_hold);
1772 	free_hot_cold_page_list(&l_hold, true);
1773 }
1774 
1775 #ifdef CONFIG_SWAP
1776 static int inactive_anon_is_low_global(struct zone *zone)
1777 {
1778 	unsigned long active, inactive;
1779 
1780 	active = zone_page_state(zone, NR_ACTIVE_ANON);
1781 	inactive = zone_page_state(zone, NR_INACTIVE_ANON);
1782 
1783 	if (inactive * zone->inactive_ratio < active)
1784 		return 1;
1785 
1786 	return 0;
1787 }
1788 
1789 /**
1790  * inactive_anon_is_low - check if anonymous pages need to be deactivated
1791  * @lruvec: LRU vector to check
1792  *
1793  * Returns true if the zone does not have enough inactive anon pages,
1794  * meaning some active anon pages need to be deactivated.
1795  */
1796 static int inactive_anon_is_low(struct lruvec *lruvec)
1797 {
1798 	/*
1799 	 * If we don't have swap space, anonymous page deactivation
1800 	 * is pointless.
1801 	 */
1802 	if (!total_swap_pages)
1803 		return 0;
1804 
1805 	if (!mem_cgroup_disabled())
1806 		return mem_cgroup_inactive_anon_is_low(lruvec);
1807 
1808 	return inactive_anon_is_low_global(lruvec_zone(lruvec));
1809 }
1810 #else
1811 static inline int inactive_anon_is_low(struct lruvec *lruvec)
1812 {
1813 	return 0;
1814 }
1815 #endif
1816 
1817 /**
1818  * inactive_file_is_low - check if file pages need to be deactivated
1819  * @lruvec: LRU vector to check
1820  *
1821  * When the system is doing streaming IO, memory pressure here
1822  * ensures that active file pages get deactivated, until more
1823  * than half of the file pages are on the inactive list.
1824  *
1825  * Once we get to that situation, protect the system's working
1826  * set from being evicted by disabling active file page aging.
1827  *
1828  * This uses a different ratio than the anonymous pages, because
1829  * the page cache uses a use-once replacement algorithm.
1830  */
1831 static int inactive_file_is_low(struct lruvec *lruvec)
1832 {
1833 	unsigned long inactive;
1834 	unsigned long active;
1835 
1836 	inactive = get_lru_size(lruvec, LRU_INACTIVE_FILE);
1837 	active = get_lru_size(lruvec, LRU_ACTIVE_FILE);
1838 
1839 	return active > inactive;
1840 }
1841 
1842 static int inactive_list_is_low(struct lruvec *lruvec, enum lru_list lru)
1843 {
1844 	if (is_file_lru(lru))
1845 		return inactive_file_is_low(lruvec);
1846 	else
1847 		return inactive_anon_is_low(lruvec);
1848 }
1849 
1850 static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
1851 				 struct lruvec *lruvec, struct scan_control *sc)
1852 {
1853 	if (is_active_lru(lru)) {
1854 		if (inactive_list_is_low(lruvec, lru))
1855 			shrink_active_list(nr_to_scan, lruvec, sc, lru);
1856 		return 0;
1857 	}
1858 
1859 	return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
1860 }
1861 
1862 enum scan_balance {
1863 	SCAN_EQUAL,
1864 	SCAN_FRACT,
1865 	SCAN_ANON,
1866 	SCAN_FILE,
1867 };
1868 
1869 /*
1870  * Determine how aggressively the anon and file LRU lists should be
1871  * scanned.  The relative value of each set of LRU lists is determined
1872  * by looking at the fraction of the pages scanned we did rotate back
1873  * onto the active list instead of evict.
1874  *
1875  * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
1876  * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
1877  */
1878 static void get_scan_count(struct lruvec *lruvec, int swappiness,
1879 			   struct scan_control *sc, unsigned long *nr)
1880 {
1881 	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1882 	u64 fraction[2];
1883 	u64 denominator = 0;	/* gcc */
1884 	struct zone *zone = lruvec_zone(lruvec);
1885 	unsigned long anon_prio, file_prio;
1886 	enum scan_balance scan_balance;
1887 	unsigned long anon, file;
1888 	bool force_scan = false;
1889 	unsigned long ap, fp;
1890 	enum lru_list lru;
1891 	bool some_scanned;
1892 	int pass;
1893 
1894 	/*
1895 	 * If the zone or memcg is small, nr[l] can be 0.  This
1896 	 * results in no scanning on this priority and a potential
1897 	 * priority drop.  Global direct reclaim can go to the next
1898 	 * zone and tends to have no problems. Global kswapd is for
1899 	 * zone balancing and it needs to scan a minimum amount. When
1900 	 * reclaiming for a memcg, a priority drop can cause high
1901 	 * latencies, so it's better to scan a minimum amount there as
1902 	 * well.
1903 	 */
1904 	if (current_is_kswapd() && !zone_reclaimable(zone))
1905 		force_scan = true;
1906 	if (!global_reclaim(sc))
1907 		force_scan = true;
1908 
1909 	/* If we have no swap space, do not bother scanning anon pages. */
1910 	if (!sc->may_swap || (get_nr_swap_pages() <= 0)) {
1911 		scan_balance = SCAN_FILE;
1912 		goto out;
1913 	}
1914 
1915 	/*
1916 	 * Global reclaim will swap to prevent OOM even with no
1917 	 * swappiness, but memcg users want to use this knob to
1918 	 * disable swapping for individual groups completely when
1919 	 * using the memory controller's swap limit feature would be
1920 	 * too expensive.
1921 	 */
1922 	if (!global_reclaim(sc) && !swappiness) {
1923 		scan_balance = SCAN_FILE;
1924 		goto out;
1925 	}
1926 
1927 	/*
1928 	 * Do not apply any pressure balancing cleverness when the
1929 	 * system is close to OOM, scan both anon and file equally
1930 	 * (unless the swappiness setting disagrees with swapping).
1931 	 */
1932 	if (!sc->priority && swappiness) {
1933 		scan_balance = SCAN_EQUAL;
1934 		goto out;
1935 	}
1936 
1937 	/*
1938 	 * Prevent the reclaimer from falling into the cache trap: as
1939 	 * cache pages start out inactive, every cache fault will tip
1940 	 * the scan balance towards the file LRU.  And as the file LRU
1941 	 * shrinks, so does the window for rotation from references.
1942 	 * This means we have a runaway feedback loop where a tiny
1943 	 * thrashing file LRU becomes infinitely more attractive than
1944 	 * anon pages.  Try to detect this based on file LRU size.
1945 	 */
1946 	if (global_reclaim(sc)) {
1947 		unsigned long zonefile;
1948 		unsigned long zonefree;
1949 
1950 		zonefree = zone_page_state(zone, NR_FREE_PAGES);
1951 		zonefile = zone_page_state(zone, NR_ACTIVE_FILE) +
1952 			   zone_page_state(zone, NR_INACTIVE_FILE);
1953 
1954 		if (unlikely(zonefile + zonefree <= high_wmark_pages(zone))) {
1955 			scan_balance = SCAN_ANON;
1956 			goto out;
1957 		}
1958 	}
1959 
1960 	/*
1961 	 * There is enough inactive page cache, do not reclaim
1962 	 * anything from the anonymous working set right now.
1963 	 */
1964 	if (!inactive_file_is_low(lruvec)) {
1965 		scan_balance = SCAN_FILE;
1966 		goto out;
1967 	}
1968 
1969 	scan_balance = SCAN_FRACT;
1970 
1971 	/*
1972 	 * With swappiness at 100, anonymous and file have the same priority.
1973 	 * This scanning priority is essentially the inverse of IO cost.
1974 	 */
1975 	anon_prio = swappiness;
1976 	file_prio = 200 - anon_prio;
1977 
1978 	/*
1979 	 * OK, so we have swap space and a fair amount of page cache
1980 	 * pages.  We use the recently rotated / recently scanned
1981 	 * ratios to determine how valuable each cache is.
1982 	 *
1983 	 * Because workloads change over time (and to avoid overflow)
1984 	 * we keep these statistics as a floating average, which ends
1985 	 * up weighing recent references more than old ones.
1986 	 *
1987 	 * anon in [0], file in [1]
1988 	 */
1989 
1990 	anon  = get_lru_size(lruvec, LRU_ACTIVE_ANON) +
1991 		get_lru_size(lruvec, LRU_INACTIVE_ANON);
1992 	file  = get_lru_size(lruvec, LRU_ACTIVE_FILE) +
1993 		get_lru_size(lruvec, LRU_INACTIVE_FILE);
1994 
1995 	spin_lock_irq(&zone->lru_lock);
1996 	if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
1997 		reclaim_stat->recent_scanned[0] /= 2;
1998 		reclaim_stat->recent_rotated[0] /= 2;
1999 	}
2000 
2001 	if (unlikely(reclaim_stat->recent_scanned[1] > file / 4)) {
2002 		reclaim_stat->recent_scanned[1] /= 2;
2003 		reclaim_stat->recent_rotated[1] /= 2;
2004 	}
2005 
2006 	/*
2007 	 * The amount of pressure on anon vs file pages is inversely
2008 	 * proportional to the fraction of recently scanned pages on
2009 	 * each list that were recently referenced and in active use.
2010 	 */
2011 	ap = anon_prio * (reclaim_stat->recent_scanned[0] + 1);
2012 	ap /= reclaim_stat->recent_rotated[0] + 1;
2013 
2014 	fp = file_prio * (reclaim_stat->recent_scanned[1] + 1);
2015 	fp /= reclaim_stat->recent_rotated[1] + 1;
2016 	spin_unlock_irq(&zone->lru_lock);
2017 
2018 	fraction[0] = ap;
2019 	fraction[1] = fp;
2020 	denominator = ap + fp + 1;
2021 out:
2022 	some_scanned = false;
2023 	/* Only use force_scan on second pass. */
2024 	for (pass = 0; !some_scanned && pass < 2; pass++) {
2025 		for_each_evictable_lru(lru) {
2026 			int file = is_file_lru(lru);
2027 			unsigned long size;
2028 			unsigned long scan;
2029 
2030 			size = get_lru_size(lruvec, lru);
2031 			scan = size >> sc->priority;
2032 
2033 			if (!scan && pass && force_scan)
2034 				scan = min(size, SWAP_CLUSTER_MAX);
2035 
2036 			switch (scan_balance) {
2037 			case SCAN_EQUAL:
2038 				/* Scan lists relative to size */
2039 				break;
2040 			case SCAN_FRACT:
2041 				/*
2042 				 * Scan types proportional to swappiness and
2043 				 * their relative recent reclaim efficiency.
2044 				 */
2045 				scan = div64_u64(scan * fraction[file],
2046 							denominator);
2047 				break;
2048 			case SCAN_FILE:
2049 			case SCAN_ANON:
2050 				/* Scan one type exclusively */
2051 				if ((scan_balance == SCAN_FILE) != file)
2052 					scan = 0;
2053 				break;
2054 			default:
2055 				/* Look ma, no brain */
2056 				BUG();
2057 			}
2058 			nr[lru] = scan;
2059 			/*
2060 			 * Skip the second pass and don't force_scan,
2061 			 * if we found something to scan.
2062 			 */
2063 			some_scanned |= !!scan;
2064 		}
2065 	}
2066 }
2067 
2068 /*
2069  * This is a basic per-zone page freer.  Used by both kswapd and direct reclaim.
2070  */
2071 static void shrink_lruvec(struct lruvec *lruvec, int swappiness,
2072 			  struct scan_control *sc)
2073 {
2074 	unsigned long nr[NR_LRU_LISTS];
2075 	unsigned long targets[NR_LRU_LISTS];
2076 	unsigned long nr_to_scan;
2077 	enum lru_list lru;
2078 	unsigned long nr_reclaimed = 0;
2079 	unsigned long nr_to_reclaim = sc->nr_to_reclaim;
2080 	struct blk_plug plug;
2081 	bool scan_adjusted;
2082 
2083 	get_scan_count(lruvec, swappiness, sc, nr);
2084 
2085 	/* Record the original scan target for proportional adjustments later */
2086 	memcpy(targets, nr, sizeof(nr));
2087 
2088 	/*
2089 	 * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal
2090 	 * event that can occur when there is little memory pressure e.g.
2091 	 * multiple streaming readers/writers. Hence, we do not abort scanning
2092 	 * when the requested number of pages are reclaimed when scanning at
2093 	 * DEF_PRIORITY on the assumption that the fact we are direct
2094 	 * reclaiming implies that kswapd is not keeping up and it is best to
2095 	 * do a batch of work at once. For memcg reclaim one check is made to
2096 	 * abort proportional reclaim if either the file or anon lru has already
2097 	 * dropped to zero at the first pass.
2098 	 */
2099 	scan_adjusted = (global_reclaim(sc) && !current_is_kswapd() &&
2100 			 sc->priority == DEF_PRIORITY);
2101 
2102 	blk_start_plug(&plug);
2103 	while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
2104 					nr[LRU_INACTIVE_FILE]) {
2105 		unsigned long nr_anon, nr_file, percentage;
2106 		unsigned long nr_scanned;
2107 
2108 		for_each_evictable_lru(lru) {
2109 			if (nr[lru]) {
2110 				nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
2111 				nr[lru] -= nr_to_scan;
2112 
2113 				nr_reclaimed += shrink_list(lru, nr_to_scan,
2114 							    lruvec, sc);
2115 			}
2116 		}
2117 
2118 		if (nr_reclaimed < nr_to_reclaim || scan_adjusted)
2119 			continue;
2120 
2121 		/*
2122 		 * For kswapd and memcg, reclaim at least the number of pages
2123 		 * requested. Ensure that the anon and file LRUs are scanned
2124 		 * proportionally what was requested by get_scan_count(). We
2125 		 * stop reclaiming one LRU and reduce the amount scanning
2126 		 * proportional to the original scan target.
2127 		 */
2128 		nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
2129 		nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];
2130 
2131 		/*
2132 		 * It's just vindictive to attack the larger once the smaller
2133 		 * has gone to zero.  And given the way we stop scanning the
2134 		 * smaller below, this makes sure that we only make one nudge
2135 		 * towards proportionality once we've got nr_to_reclaim.
2136 		 */
2137 		if (!nr_file || !nr_anon)
2138 			break;
2139 
2140 		if (nr_file > nr_anon) {
2141 			unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
2142 						targets[LRU_ACTIVE_ANON] + 1;
2143 			lru = LRU_BASE;
2144 			percentage = nr_anon * 100 / scan_target;
2145 		} else {
2146 			unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
2147 						targets[LRU_ACTIVE_FILE] + 1;
2148 			lru = LRU_FILE;
2149 			percentage = nr_file * 100 / scan_target;
2150 		}
2151 
2152 		/* Stop scanning the smaller of the LRU */
2153 		nr[lru] = 0;
2154 		nr[lru + LRU_ACTIVE] = 0;
2155 
2156 		/*
2157 		 * Recalculate the other LRU scan count based on its original
2158 		 * scan target and the percentage scanning already complete
2159 		 */
2160 		lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
2161 		nr_scanned = targets[lru] - nr[lru];
2162 		nr[lru] = targets[lru] * (100 - percentage) / 100;
2163 		nr[lru] -= min(nr[lru], nr_scanned);
2164 
2165 		lru += LRU_ACTIVE;
2166 		nr_scanned = targets[lru] - nr[lru];
2167 		nr[lru] = targets[lru] * (100 - percentage) / 100;
2168 		nr[lru] -= min(nr[lru], nr_scanned);
2169 
2170 		scan_adjusted = true;
2171 	}
2172 	blk_finish_plug(&plug);
2173 	sc->nr_reclaimed += nr_reclaimed;
2174 
2175 	/*
2176 	 * Even if we did not try to evict anon pages at all, we want to
2177 	 * rebalance the anon lru active/inactive ratio.
2178 	 */
2179 	if (inactive_anon_is_low(lruvec))
2180 		shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
2181 				   sc, LRU_ACTIVE_ANON);
2182 
2183 	throttle_vm_writeout(sc->gfp_mask);
2184 }
2185 
2186 /* Use reclaim/compaction for costly allocs or under memory pressure */
2187 static bool in_reclaim_compaction(struct scan_control *sc)
2188 {
2189 	if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
2190 			(sc->order > PAGE_ALLOC_COSTLY_ORDER ||
2191 			 sc->priority < DEF_PRIORITY - 2))
2192 		return true;
2193 
2194 	return false;
2195 }
2196 
2197 /*
2198  * Reclaim/compaction is used for high-order allocation requests. It reclaims
2199  * order-0 pages before compacting the zone. should_continue_reclaim() returns
2200  * true if more pages should be reclaimed such that when the page allocator
2201  * calls try_to_compact_zone() that it will have enough free pages to succeed.
2202  * It will give up earlier than that if there is difficulty reclaiming pages.
2203  */
2204 static inline bool should_continue_reclaim(struct zone *zone,
2205 					unsigned long nr_reclaimed,
2206 					unsigned long nr_scanned,
2207 					struct scan_control *sc)
2208 {
2209 	unsigned long pages_for_compaction;
2210 	unsigned long inactive_lru_pages;
2211 
2212 	/* If not in reclaim/compaction mode, stop */
2213 	if (!in_reclaim_compaction(sc))
2214 		return false;
2215 
2216 	/* Consider stopping depending on scan and reclaim activity */
2217 	if (sc->gfp_mask & __GFP_REPEAT) {
2218 		/*
2219 		 * For __GFP_REPEAT allocations, stop reclaiming if the
2220 		 * full LRU list has been scanned and we are still failing
2221 		 * to reclaim pages. This full LRU scan is potentially
2222 		 * expensive but a __GFP_REPEAT caller really wants to succeed
2223 		 */
2224 		if (!nr_reclaimed && !nr_scanned)
2225 			return false;
2226 	} else {
2227 		/*
2228 		 * For non-__GFP_REPEAT allocations which can presumably
2229 		 * fail without consequence, stop if we failed to reclaim
2230 		 * any pages from the last SWAP_CLUSTER_MAX number of
2231 		 * pages that were scanned. This will return to the
2232 		 * caller faster at the risk reclaim/compaction and
2233 		 * the resulting allocation attempt fails
2234 		 */
2235 		if (!nr_reclaimed)
2236 			return false;
2237 	}
2238 
2239 	/*
2240 	 * If we have not reclaimed enough pages for compaction and the
2241 	 * inactive lists are large enough, continue reclaiming
2242 	 */
2243 	pages_for_compaction = (2UL << sc->order);
2244 	inactive_lru_pages = zone_page_state(zone, NR_INACTIVE_FILE);
2245 	if (get_nr_swap_pages() > 0)
2246 		inactive_lru_pages += zone_page_state(zone, NR_INACTIVE_ANON);
2247 	if (sc->nr_reclaimed < pages_for_compaction &&
2248 			inactive_lru_pages > pages_for_compaction)
2249 		return true;
2250 
2251 	/* If compaction would go ahead or the allocation would succeed, stop */
2252 	switch (compaction_suitable(zone, sc->order)) {
2253 	case COMPACT_PARTIAL:
2254 	case COMPACT_CONTINUE:
2255 		return false;
2256 	default:
2257 		return true;
2258 	}
2259 }
2260 
2261 static bool shrink_zone(struct zone *zone, struct scan_control *sc)
2262 {
2263 	unsigned long nr_reclaimed, nr_scanned;
2264 	bool reclaimable = false;
2265 
2266 	do {
2267 		struct mem_cgroup *root = sc->target_mem_cgroup;
2268 		struct mem_cgroup_reclaim_cookie reclaim = {
2269 			.zone = zone,
2270 			.priority = sc->priority,
2271 		};
2272 		struct mem_cgroup *memcg;
2273 
2274 		nr_reclaimed = sc->nr_reclaimed;
2275 		nr_scanned = sc->nr_scanned;
2276 
2277 		memcg = mem_cgroup_iter(root, NULL, &reclaim);
2278 		do {
2279 			struct lruvec *lruvec;
2280 			int swappiness;
2281 
2282 			lruvec = mem_cgroup_zone_lruvec(zone, memcg);
2283 			swappiness = mem_cgroup_swappiness(memcg);
2284 
2285 			shrink_lruvec(lruvec, swappiness, sc);
2286 
2287 			/*
2288 			 * Direct reclaim and kswapd have to scan all memory
2289 			 * cgroups to fulfill the overall scan target for the
2290 			 * zone.
2291 			 *
2292 			 * Limit reclaim, on the other hand, only cares about
2293 			 * nr_to_reclaim pages to be reclaimed and it will
2294 			 * retry with decreasing priority if one round over the
2295 			 * whole hierarchy is not sufficient.
2296 			 */
2297 			if (!global_reclaim(sc) &&
2298 					sc->nr_reclaimed >= sc->nr_to_reclaim) {
2299 				mem_cgroup_iter_break(root, memcg);
2300 				break;
2301 			}
2302 			memcg = mem_cgroup_iter(root, memcg, &reclaim);
2303 		} while (memcg);
2304 
2305 		vmpressure(sc->gfp_mask, sc->target_mem_cgroup,
2306 			   sc->nr_scanned - nr_scanned,
2307 			   sc->nr_reclaimed - nr_reclaimed);
2308 
2309 		if (sc->nr_reclaimed - nr_reclaimed)
2310 			reclaimable = true;
2311 
2312 	} while (should_continue_reclaim(zone, sc->nr_reclaimed - nr_reclaimed,
2313 					 sc->nr_scanned - nr_scanned, sc));
2314 
2315 	return reclaimable;
2316 }
2317 
2318 /*
2319  * Returns true if compaction should go ahead for a high-order request, or
2320  * the high-order allocation would succeed without compaction.
2321  */
2322 static inline bool compaction_ready(struct zone *zone, int order)
2323 {
2324 	unsigned long balance_gap, watermark;
2325 	bool watermark_ok;
2326 
2327 	/*
2328 	 * Compaction takes time to run and there are potentially other
2329 	 * callers using the pages just freed. Continue reclaiming until
2330 	 * there is a buffer of free pages available to give compaction
2331 	 * a reasonable chance of completing and allocating the page
2332 	 */
2333 	balance_gap = min(low_wmark_pages(zone), DIV_ROUND_UP(
2334 			zone->managed_pages, KSWAPD_ZONE_BALANCE_GAP_RATIO));
2335 	watermark = high_wmark_pages(zone) + balance_gap + (2UL << order);
2336 	watermark_ok = zone_watermark_ok_safe(zone, 0, watermark, 0, 0);
2337 
2338 	/*
2339 	 * If compaction is deferred, reclaim up to a point where
2340 	 * compaction will have a chance of success when re-enabled
2341 	 */
2342 	if (compaction_deferred(zone, order))
2343 		return watermark_ok;
2344 
2345 	/*
2346 	 * If compaction is not ready to start and allocation is not likely
2347 	 * to succeed without it, then keep reclaiming.
2348 	 */
2349 	if (compaction_suitable(zone, order) == COMPACT_SKIPPED)
2350 		return false;
2351 
2352 	return watermark_ok;
2353 }
2354 
2355 /*
2356  * This is the direct reclaim path, for page-allocating processes.  We only
2357  * try to reclaim pages from zones which will satisfy the caller's allocation
2358  * request.
2359  *
2360  * We reclaim from a zone even if that zone is over high_wmark_pages(zone).
2361  * Because:
2362  * a) The caller may be trying to free *extra* pages to satisfy a higher-order
2363  *    allocation or
2364  * b) The target zone may be at high_wmark_pages(zone) but the lower zones
2365  *    must go *over* high_wmark_pages(zone) to satisfy the `incremental min'
2366  *    zone defense algorithm.
2367  *
2368  * If a zone is deemed to be full of pinned pages then just give it a light
2369  * scan then give up on it.
2370  *
2371  * Returns true if a zone was reclaimable.
2372  */
2373 static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
2374 {
2375 	struct zoneref *z;
2376 	struct zone *zone;
2377 	unsigned long nr_soft_reclaimed;
2378 	unsigned long nr_soft_scanned;
2379 	unsigned long lru_pages = 0;
2380 	struct reclaim_state *reclaim_state = current->reclaim_state;
2381 	gfp_t orig_mask;
2382 	struct shrink_control shrink = {
2383 		.gfp_mask = sc->gfp_mask,
2384 	};
2385 	enum zone_type requested_highidx = gfp_zone(sc->gfp_mask);
2386 	bool reclaimable = false;
2387 
2388 	/*
2389 	 * If the number of buffer_heads in the machine exceeds the maximum
2390 	 * allowed level, force direct reclaim to scan the highmem zone as
2391 	 * highmem pages could be pinning lowmem pages storing buffer_heads
2392 	 */
2393 	orig_mask = sc->gfp_mask;
2394 	if (buffer_heads_over_limit)
2395 		sc->gfp_mask |= __GFP_HIGHMEM;
2396 
2397 	nodes_clear(shrink.nodes_to_scan);
2398 
2399 	for_each_zone_zonelist_nodemask(zone, z, zonelist,
2400 					gfp_zone(sc->gfp_mask), sc->nodemask) {
2401 		if (!populated_zone(zone))
2402 			continue;
2403 		/*
2404 		 * Take care memory controller reclaiming has small influence
2405 		 * to global LRU.
2406 		 */
2407 		if (global_reclaim(sc)) {
2408 			if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
2409 				continue;
2410 
2411 			lru_pages += zone_reclaimable_pages(zone);
2412 			node_set(zone_to_nid(zone), shrink.nodes_to_scan);
2413 
2414 			if (sc->priority != DEF_PRIORITY &&
2415 			    !zone_reclaimable(zone))
2416 				continue;	/* Let kswapd poll it */
2417 
2418 			/*
2419 			 * If we already have plenty of memory free for
2420 			 * compaction in this zone, don't free any more.
2421 			 * Even though compaction is invoked for any
2422 			 * non-zero order, only frequent costly order
2423 			 * reclamation is disruptive enough to become a
2424 			 * noticeable problem, like transparent huge
2425 			 * page allocations.
2426 			 */
2427 			if (IS_ENABLED(CONFIG_COMPACTION) &&
2428 			    sc->order > PAGE_ALLOC_COSTLY_ORDER &&
2429 			    zonelist_zone_idx(z) <= requested_highidx &&
2430 			    compaction_ready(zone, sc->order)) {
2431 				sc->compaction_ready = true;
2432 				continue;
2433 			}
2434 
2435 			/*
2436 			 * This steals pages from memory cgroups over softlimit
2437 			 * and returns the number of reclaimed pages and
2438 			 * scanned pages. This works for global memory pressure
2439 			 * and balancing, not for a memcg's limit.
2440 			 */
2441 			nr_soft_scanned = 0;
2442 			nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone,
2443 						sc->order, sc->gfp_mask,
2444 						&nr_soft_scanned);
2445 			sc->nr_reclaimed += nr_soft_reclaimed;
2446 			sc->nr_scanned += nr_soft_scanned;
2447 			if (nr_soft_reclaimed)
2448 				reclaimable = true;
2449 			/* need some check for avoid more shrink_zone() */
2450 		}
2451 
2452 		if (shrink_zone(zone, sc))
2453 			reclaimable = true;
2454 
2455 		if (global_reclaim(sc) &&
2456 		    !reclaimable && zone_reclaimable(zone))
2457 			reclaimable = true;
2458 	}
2459 
2460 	/*
2461 	 * Don't shrink slabs when reclaiming memory from over limit cgroups
2462 	 * but do shrink slab at least once when aborting reclaim for
2463 	 * compaction to avoid unevenly scanning file/anon LRU pages over slab
2464 	 * pages.
2465 	 */
2466 	if (global_reclaim(sc)) {
2467 		shrink_slab(&shrink, sc->nr_scanned, lru_pages);
2468 		if (reclaim_state) {
2469 			sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2470 			reclaim_state->reclaimed_slab = 0;
2471 		}
2472 	}
2473 
2474 	/*
2475 	 * Restore to original mask to avoid the impact on the caller if we
2476 	 * promoted it to __GFP_HIGHMEM.
2477 	 */
2478 	sc->gfp_mask = orig_mask;
2479 
2480 	return reclaimable;
2481 }
2482 
2483 /*
2484  * This is the main entry point to direct page reclaim.
2485  *
2486  * If a full scan of the inactive list fails to free enough memory then we
2487  * are "out of memory" and something needs to be killed.
2488  *
2489  * If the caller is !__GFP_FS then the probability of a failure is reasonably
2490  * high - the zone may be full of dirty or under-writeback pages, which this
2491  * caller can't do much about.  We kick the writeback threads and take explicit
2492  * naps in the hope that some of these pages can be written.  But if the
2493  * allocating task holds filesystem locks which prevent writeout this might not
2494  * work, and the allocation attempt will fail.
2495  *
2496  * returns:	0, if no pages reclaimed
2497  * 		else, the number of pages reclaimed
2498  */
2499 static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
2500 					  struct scan_control *sc)
2501 {
2502 	unsigned long total_scanned = 0;
2503 	unsigned long writeback_threshold;
2504 	bool zones_reclaimable;
2505 
2506 	delayacct_freepages_start();
2507 
2508 	if (global_reclaim(sc))
2509 		count_vm_event(ALLOCSTALL);
2510 
2511 	do {
2512 		vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup,
2513 				sc->priority);
2514 		sc->nr_scanned = 0;
2515 		zones_reclaimable = shrink_zones(zonelist, sc);
2516 
2517 		total_scanned += sc->nr_scanned;
2518 		if (sc->nr_reclaimed >= sc->nr_to_reclaim)
2519 			break;
2520 
2521 		if (sc->compaction_ready)
2522 			break;
2523 
2524 		/*
2525 		 * If we're getting trouble reclaiming, start doing
2526 		 * writepage even in laptop mode.
2527 		 */
2528 		if (sc->priority < DEF_PRIORITY - 2)
2529 			sc->may_writepage = 1;
2530 
2531 		/*
2532 		 * Try to write back as many pages as we just scanned.  This
2533 		 * tends to cause slow streaming writers to write data to the
2534 		 * disk smoothly, at the dirtying rate, which is nice.   But
2535 		 * that's undesirable in laptop mode, where we *want* lumpy
2536 		 * writeout.  So in laptop mode, write out the whole world.
2537 		 */
2538 		writeback_threshold = sc->nr_to_reclaim + sc->nr_to_reclaim / 2;
2539 		if (total_scanned > writeback_threshold) {
2540 			wakeup_flusher_threads(laptop_mode ? 0 : total_scanned,
2541 						WB_REASON_TRY_TO_FREE_PAGES);
2542 			sc->may_writepage = 1;
2543 		}
2544 	} while (--sc->priority >= 0);
2545 
2546 	delayacct_freepages_end();
2547 
2548 	if (sc->nr_reclaimed)
2549 		return sc->nr_reclaimed;
2550 
2551 	/* Aborted reclaim to try compaction? don't OOM, then */
2552 	if (sc->compaction_ready)
2553 		return 1;
2554 
2555 	/* Any of the zones still reclaimable?  Don't OOM. */
2556 	if (zones_reclaimable)
2557 		return 1;
2558 
2559 	return 0;
2560 }
2561 
2562 static bool pfmemalloc_watermark_ok(pg_data_t *pgdat)
2563 {
2564 	struct zone *zone;
2565 	unsigned long pfmemalloc_reserve = 0;
2566 	unsigned long free_pages = 0;
2567 	int i;
2568 	bool wmark_ok;
2569 
2570 	for (i = 0; i <= ZONE_NORMAL; i++) {
2571 		zone = &pgdat->node_zones[i];
2572 		if (!populated_zone(zone))
2573 			continue;
2574 
2575 		pfmemalloc_reserve += min_wmark_pages(zone);
2576 		free_pages += zone_page_state(zone, NR_FREE_PAGES);
2577 	}
2578 
2579 	/* If there are no reserves (unexpected config) then do not throttle */
2580 	if (!pfmemalloc_reserve)
2581 		return true;
2582 
2583 	wmark_ok = free_pages > pfmemalloc_reserve / 2;
2584 
2585 	/* kswapd must be awake if processes are being throttled */
2586 	if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
2587 		pgdat->classzone_idx = min(pgdat->classzone_idx,
2588 						(enum zone_type)ZONE_NORMAL);
2589 		wake_up_interruptible(&pgdat->kswapd_wait);
2590 	}
2591 
2592 	return wmark_ok;
2593 }
2594 
2595 /*
2596  * Throttle direct reclaimers if backing storage is backed by the network
2597  * and the PFMEMALLOC reserve for the preferred node is getting dangerously
2598  * depleted. kswapd will continue to make progress and wake the processes
2599  * when the low watermark is reached.
2600  *
2601  * Returns true if a fatal signal was delivered during throttling. If this
2602  * happens, the page allocator should not consider triggering the OOM killer.
2603  */
2604 static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
2605 					nodemask_t *nodemask)
2606 {
2607 	struct zoneref *z;
2608 	struct zone *zone;
2609 	pg_data_t *pgdat = NULL;
2610 
2611 	/*
2612 	 * Kernel threads should not be throttled as they may be indirectly
2613 	 * responsible for cleaning pages necessary for reclaim to make forward
2614 	 * progress. kjournald for example may enter direct reclaim while
2615 	 * committing a transaction where throttling it could forcing other
2616 	 * processes to block on log_wait_commit().
2617 	 */
2618 	if (current->flags & PF_KTHREAD)
2619 		goto out;
2620 
2621 	/*
2622 	 * If a fatal signal is pending, this process should not throttle.
2623 	 * It should return quickly so it can exit and free its memory
2624 	 */
2625 	if (fatal_signal_pending(current))
2626 		goto out;
2627 
2628 	/*
2629 	 * Check if the pfmemalloc reserves are ok by finding the first node
2630 	 * with a usable ZONE_NORMAL or lower zone. The expectation is that
2631 	 * GFP_KERNEL will be required for allocating network buffers when
2632 	 * swapping over the network so ZONE_HIGHMEM is unusable.
2633 	 *
2634 	 * Throttling is based on the first usable node and throttled processes
2635 	 * wait on a queue until kswapd makes progress and wakes them. There
2636 	 * is an affinity then between processes waking up and where reclaim
2637 	 * progress has been made assuming the process wakes on the same node.
2638 	 * More importantly, processes running on remote nodes will not compete
2639 	 * for remote pfmemalloc reserves and processes on different nodes
2640 	 * should make reasonable progress.
2641 	 */
2642 	for_each_zone_zonelist_nodemask(zone, z, zonelist,
2643 					gfp_mask, nodemask) {
2644 		if (zone_idx(zone) > ZONE_NORMAL)
2645 			continue;
2646 
2647 		/* Throttle based on the first usable node */
2648 		pgdat = zone->zone_pgdat;
2649 		if (pfmemalloc_watermark_ok(pgdat))
2650 			goto out;
2651 		break;
2652 	}
2653 
2654 	/* If no zone was usable by the allocation flags then do not throttle */
2655 	if (!pgdat)
2656 		goto out;
2657 
2658 	/* Account for the throttling */
2659 	count_vm_event(PGSCAN_DIRECT_THROTTLE);
2660 
2661 	/*
2662 	 * If the caller cannot enter the filesystem, it's possible that it
2663 	 * is due to the caller holding an FS lock or performing a journal
2664 	 * transaction in the case of a filesystem like ext[3|4]. In this case,
2665 	 * it is not safe to block on pfmemalloc_wait as kswapd could be
2666 	 * blocked waiting on the same lock. Instead, throttle for up to a
2667 	 * second before continuing.
2668 	 */
2669 	if (!(gfp_mask & __GFP_FS)) {
2670 		wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
2671 			pfmemalloc_watermark_ok(pgdat), HZ);
2672 
2673 		goto check_pending;
2674 	}
2675 
2676 	/* Throttle until kswapd wakes the process */
2677 	wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
2678 		pfmemalloc_watermark_ok(pgdat));
2679 
2680 check_pending:
2681 	if (fatal_signal_pending(current))
2682 		return true;
2683 
2684 out:
2685 	return false;
2686 }
2687 
2688 unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
2689 				gfp_t gfp_mask, nodemask_t *nodemask)
2690 {
2691 	unsigned long nr_reclaimed;
2692 	struct scan_control sc = {
2693 		.nr_to_reclaim = SWAP_CLUSTER_MAX,
2694 		.gfp_mask = (gfp_mask = memalloc_noio_flags(gfp_mask)),
2695 		.order = order,
2696 		.nodemask = nodemask,
2697 		.priority = DEF_PRIORITY,
2698 		.may_writepage = !laptop_mode,
2699 		.may_unmap = 1,
2700 		.may_swap = 1,
2701 	};
2702 
2703 	/*
2704 	 * Do not enter reclaim if fatal signal was delivered while throttled.
2705 	 * 1 is returned so that the page allocator does not OOM kill at this
2706 	 * point.
2707 	 */
2708 	if (throttle_direct_reclaim(gfp_mask, zonelist, nodemask))
2709 		return 1;
2710 
2711 	trace_mm_vmscan_direct_reclaim_begin(order,
2712 				sc.may_writepage,
2713 				gfp_mask);
2714 
2715 	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
2716 
2717 	trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
2718 
2719 	return nr_reclaimed;
2720 }
2721 
2722 #ifdef CONFIG_MEMCG
2723 
2724 unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *memcg,
2725 						gfp_t gfp_mask, bool noswap,
2726 						struct zone *zone,
2727 						unsigned long *nr_scanned)
2728 {
2729 	struct scan_control sc = {
2730 		.nr_to_reclaim = SWAP_CLUSTER_MAX,
2731 		.target_mem_cgroup = memcg,
2732 		.may_writepage = !laptop_mode,
2733 		.may_unmap = 1,
2734 		.may_swap = !noswap,
2735 	};
2736 	struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg);
2737 	int swappiness = mem_cgroup_swappiness(memcg);
2738 
2739 	sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
2740 			(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
2741 
2742 	trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order,
2743 						      sc.may_writepage,
2744 						      sc.gfp_mask);
2745 
2746 	/*
2747 	 * NOTE: Although we can get the priority field, using it
2748 	 * here is not a good idea, since it limits the pages we can scan.
2749 	 * if we don't reclaim here, the shrink_zone from balance_pgdat
2750 	 * will pick up pages from other mem cgroup's as well. We hack
2751 	 * the priority and make it zero.
2752 	 */
2753 	shrink_lruvec(lruvec, swappiness, &sc);
2754 
2755 	trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
2756 
2757 	*nr_scanned = sc.nr_scanned;
2758 	return sc.nr_reclaimed;
2759 }
2760 
2761 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
2762 					   unsigned long nr_pages,
2763 					   gfp_t gfp_mask,
2764 					   bool may_swap)
2765 {
2766 	struct zonelist *zonelist;
2767 	unsigned long nr_reclaimed;
2768 	int nid;
2769 	struct scan_control sc = {
2770 		.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
2771 		.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
2772 				(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK),
2773 		.target_mem_cgroup = memcg,
2774 		.priority = DEF_PRIORITY,
2775 		.may_writepage = !laptop_mode,
2776 		.may_unmap = 1,
2777 		.may_swap = may_swap,
2778 	};
2779 
2780 	/*
2781 	 * Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't
2782 	 * take care of from where we get pages. So the node where we start the
2783 	 * scan does not need to be the current node.
2784 	 */
2785 	nid = mem_cgroup_select_victim_node(memcg);
2786 
2787 	zonelist = NODE_DATA(nid)->node_zonelists;
2788 
2789 	trace_mm_vmscan_memcg_reclaim_begin(0,
2790 					    sc.may_writepage,
2791 					    sc.gfp_mask);
2792 
2793 	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
2794 
2795 	trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
2796 
2797 	return nr_reclaimed;
2798 }
2799 #endif
2800 
2801 static void age_active_anon(struct zone *zone, struct scan_control *sc)
2802 {
2803 	struct mem_cgroup *memcg;
2804 
2805 	if (!total_swap_pages)
2806 		return;
2807 
2808 	memcg = mem_cgroup_iter(NULL, NULL, NULL);
2809 	do {
2810 		struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg);
2811 
2812 		if (inactive_anon_is_low(lruvec))
2813 			shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
2814 					   sc, LRU_ACTIVE_ANON);
2815 
2816 		memcg = mem_cgroup_iter(NULL, memcg, NULL);
2817 	} while (memcg);
2818 }
2819 
2820 static bool zone_balanced(struct zone *zone, int order,
2821 			  unsigned long balance_gap, int classzone_idx)
2822 {
2823 	if (!zone_watermark_ok_safe(zone, order, high_wmark_pages(zone) +
2824 				    balance_gap, classzone_idx, 0))
2825 		return false;
2826 
2827 	if (IS_ENABLED(CONFIG_COMPACTION) && order &&
2828 	    compaction_suitable(zone, order) == COMPACT_SKIPPED)
2829 		return false;
2830 
2831 	return true;
2832 }
2833 
2834 /*
2835  * pgdat_balanced() is used when checking if a node is balanced.
2836  *
2837  * For order-0, all zones must be balanced!
2838  *
2839  * For high-order allocations only zones that meet watermarks and are in a
2840  * zone allowed by the callers classzone_idx are added to balanced_pages. The
2841  * total of balanced pages must be at least 25% of the zones allowed by
2842  * classzone_idx for the node to be considered balanced. Forcing all zones to
2843  * be balanced for high orders can cause excessive reclaim when there are
2844  * imbalanced zones.
2845  * The choice of 25% is due to
2846  *   o a 16M DMA zone that is balanced will not balance a zone on any
2847  *     reasonable sized machine
2848  *   o On all other machines, the top zone must be at least a reasonable
2849  *     percentage of the middle zones. For example, on 32-bit x86, highmem
2850  *     would need to be at least 256M for it to be balance a whole node.
2851  *     Similarly, on x86-64 the Normal zone would need to be at least 1G
2852  *     to balance a node on its own. These seemed like reasonable ratios.
2853  */
2854 static bool pgdat_balanced(pg_data_t *pgdat, int order, int classzone_idx)
2855 {
2856 	unsigned long managed_pages = 0;
2857 	unsigned long balanced_pages = 0;
2858 	int i;
2859 
2860 	/* Check the watermark levels */
2861 	for (i = 0; i <= classzone_idx; i++) {
2862 		struct zone *zone = pgdat->node_zones + i;
2863 
2864 		if (!populated_zone(zone))
2865 			continue;
2866 
2867 		managed_pages += zone->managed_pages;
2868 
2869 		/*
2870 		 * A special case here:
2871 		 *
2872 		 * balance_pgdat() skips over all_unreclaimable after
2873 		 * DEF_PRIORITY. Effectively, it considers them balanced so
2874 		 * they must be considered balanced here as well!
2875 		 */
2876 		if (!zone_reclaimable(zone)) {
2877 			balanced_pages += zone->managed_pages;
2878 			continue;
2879 		}
2880 
2881 		if (zone_balanced(zone, order, 0, i))
2882 			balanced_pages += zone->managed_pages;
2883 		else if (!order)
2884 			return false;
2885 	}
2886 
2887 	if (order)
2888 		return balanced_pages >= (managed_pages >> 2);
2889 	else
2890 		return true;
2891 }
2892 
2893 /*
2894  * Prepare kswapd for sleeping. This verifies that there are no processes
2895  * waiting in throttle_direct_reclaim() and that watermarks have been met.
2896  *
2897  * Returns true if kswapd is ready to sleep
2898  */
2899 static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, long remaining,
2900 					int classzone_idx)
2901 {
2902 	/* If a direct reclaimer woke kswapd within HZ/10, it's premature */
2903 	if (remaining)
2904 		return false;
2905 
2906 	/*
2907 	 * There is a potential race between when kswapd checks its watermarks
2908 	 * and a process gets throttled. There is also a potential race if
2909 	 * processes get throttled, kswapd wakes, a large process exits therby
2910 	 * balancing the zones that causes kswapd to miss a wakeup. If kswapd
2911 	 * is going to sleep, no process should be sleeping on pfmemalloc_wait
2912 	 * so wake them now if necessary. If necessary, processes will wake
2913 	 * kswapd and get throttled again
2914 	 */
2915 	if (waitqueue_active(&pgdat->pfmemalloc_wait)) {
2916 		wake_up(&pgdat->pfmemalloc_wait);
2917 		return false;
2918 	}
2919 
2920 	return pgdat_balanced(pgdat, order, classzone_idx);
2921 }
2922 
2923 /*
2924  * kswapd shrinks the zone by the number of pages required to reach
2925  * the high watermark.
2926  *
2927  * Returns true if kswapd scanned at least the requested number of pages to
2928  * reclaim or if the lack of progress was due to pages under writeback.
2929  * This is used to determine if the scanning priority needs to be raised.
2930  */
2931 static bool kswapd_shrink_zone(struct zone *zone,
2932 			       int classzone_idx,
2933 			       struct scan_control *sc,
2934 			       unsigned long lru_pages,
2935 			       unsigned long *nr_attempted)
2936 {
2937 	int testorder = sc->order;
2938 	unsigned long balance_gap;
2939 	struct reclaim_state *reclaim_state = current->reclaim_state;
2940 	struct shrink_control shrink = {
2941 		.gfp_mask = sc->gfp_mask,
2942 	};
2943 	bool lowmem_pressure;
2944 
2945 	/* Reclaim above the high watermark. */
2946 	sc->nr_to_reclaim = max(SWAP_CLUSTER_MAX, high_wmark_pages(zone));
2947 
2948 	/*
2949 	 * Kswapd reclaims only single pages with compaction enabled. Trying
2950 	 * too hard to reclaim until contiguous free pages have become
2951 	 * available can hurt performance by evicting too much useful data
2952 	 * from memory. Do not reclaim more than needed for compaction.
2953 	 */
2954 	if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
2955 			compaction_suitable(zone, sc->order) !=
2956 				COMPACT_SKIPPED)
2957 		testorder = 0;
2958 
2959 	/*
2960 	 * We put equal pressure on every zone, unless one zone has way too
2961 	 * many pages free already. The "too many pages" is defined as the
2962 	 * high wmark plus a "gap" where the gap is either the low
2963 	 * watermark or 1% of the zone, whichever is smaller.
2964 	 */
2965 	balance_gap = min(low_wmark_pages(zone), DIV_ROUND_UP(
2966 			zone->managed_pages, KSWAPD_ZONE_BALANCE_GAP_RATIO));
2967 
2968 	/*
2969 	 * If there is no low memory pressure or the zone is balanced then no
2970 	 * reclaim is necessary
2971 	 */
2972 	lowmem_pressure = (buffer_heads_over_limit && is_highmem(zone));
2973 	if (!lowmem_pressure && zone_balanced(zone, testorder,
2974 						balance_gap, classzone_idx))
2975 		return true;
2976 
2977 	shrink_zone(zone, sc);
2978 	nodes_clear(shrink.nodes_to_scan);
2979 	node_set(zone_to_nid(zone), shrink.nodes_to_scan);
2980 
2981 	reclaim_state->reclaimed_slab = 0;
2982 	shrink_slab(&shrink, sc->nr_scanned, lru_pages);
2983 	sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2984 
2985 	/* Account for the number of pages attempted to reclaim */
2986 	*nr_attempted += sc->nr_to_reclaim;
2987 
2988 	clear_bit(ZONE_WRITEBACK, &zone->flags);
2989 
2990 	/*
2991 	 * If a zone reaches its high watermark, consider it to be no longer
2992 	 * congested. It's possible there are dirty pages backed by congested
2993 	 * BDIs but as pressure is relieved, speculatively avoid congestion
2994 	 * waits.
2995 	 */
2996 	if (zone_reclaimable(zone) &&
2997 	    zone_balanced(zone, testorder, 0, classzone_idx)) {
2998 		clear_bit(ZONE_CONGESTED, &zone->flags);
2999 		clear_bit(ZONE_DIRTY, &zone->flags);
3000 	}
3001 
3002 	return sc->nr_scanned >= sc->nr_to_reclaim;
3003 }
3004 
3005 /*
3006  * For kswapd, balance_pgdat() will work across all this node's zones until
3007  * they are all at high_wmark_pages(zone).
3008  *
3009  * Returns the final order kswapd was reclaiming at
3010  *
3011  * There is special handling here for zones which are full of pinned pages.
3012  * This can happen if the pages are all mlocked, or if they are all used by
3013  * device drivers (say, ZONE_DMA).  Or if they are all in use by hugetlb.
3014  * What we do is to detect the case where all pages in the zone have been
3015  * scanned twice and there has been zero successful reclaim.  Mark the zone as
3016  * dead and from now on, only perform a short scan.  Basically we're polling
3017  * the zone for when the problem goes away.
3018  *
3019  * kswapd scans the zones in the highmem->normal->dma direction.  It skips
3020  * zones which have free_pages > high_wmark_pages(zone), but once a zone is
3021  * found to have free_pages <= high_wmark_pages(zone), we scan that zone and the
3022  * lower zones regardless of the number of free pages in the lower zones. This
3023  * interoperates with the page allocator fallback scheme to ensure that aging
3024  * of pages is balanced across the zones.
3025  */
3026 static unsigned long balance_pgdat(pg_data_t *pgdat, int order,
3027 							int *classzone_idx)
3028 {
3029 	int i;
3030 	int end_zone = 0;	/* Inclusive.  0 = ZONE_DMA */
3031 	unsigned long nr_soft_reclaimed;
3032 	unsigned long nr_soft_scanned;
3033 	struct scan_control sc = {
3034 		.gfp_mask = GFP_KERNEL,
3035 		.order = order,
3036 		.priority = DEF_PRIORITY,
3037 		.may_writepage = !laptop_mode,
3038 		.may_unmap = 1,
3039 		.may_swap = 1,
3040 	};
3041 	count_vm_event(PAGEOUTRUN);
3042 
3043 	do {
3044 		unsigned long lru_pages = 0;
3045 		unsigned long nr_attempted = 0;
3046 		bool raise_priority = true;
3047 		bool pgdat_needs_compaction = (order > 0);
3048 
3049 		sc.nr_reclaimed = 0;
3050 
3051 		/*
3052 		 * Scan in the highmem->dma direction for the highest
3053 		 * zone which needs scanning
3054 		 */
3055 		for (i = pgdat->nr_zones - 1; i >= 0; i--) {
3056 			struct zone *zone = pgdat->node_zones + i;
3057 
3058 			if (!populated_zone(zone))
3059 				continue;
3060 
3061 			if (sc.priority != DEF_PRIORITY &&
3062 			    !zone_reclaimable(zone))
3063 				continue;
3064 
3065 			/*
3066 			 * Do some background aging of the anon list, to give
3067 			 * pages a chance to be referenced before reclaiming.
3068 			 */
3069 			age_active_anon(zone, &sc);
3070 
3071 			/*
3072 			 * If the number of buffer_heads in the machine
3073 			 * exceeds the maximum allowed level and this node
3074 			 * has a highmem zone, force kswapd to reclaim from
3075 			 * it to relieve lowmem pressure.
3076 			 */
3077 			if (buffer_heads_over_limit && is_highmem_idx(i)) {
3078 				end_zone = i;
3079 				break;
3080 			}
3081 
3082 			if (!zone_balanced(zone, order, 0, 0)) {
3083 				end_zone = i;
3084 				break;
3085 			} else {
3086 				/*
3087 				 * If balanced, clear the dirty and congested
3088 				 * flags
3089 				 */
3090 				clear_bit(ZONE_CONGESTED, &zone->flags);
3091 				clear_bit(ZONE_DIRTY, &zone->flags);
3092 			}
3093 		}
3094 
3095 		if (i < 0)
3096 			goto out;
3097 
3098 		for (i = 0; i <= end_zone; i++) {
3099 			struct zone *zone = pgdat->node_zones + i;
3100 
3101 			if (!populated_zone(zone))
3102 				continue;
3103 
3104 			lru_pages += zone_reclaimable_pages(zone);
3105 
3106 			/*
3107 			 * If any zone is currently balanced then kswapd will
3108 			 * not call compaction as it is expected that the
3109 			 * necessary pages are already available.
3110 			 */
3111 			if (pgdat_needs_compaction &&
3112 					zone_watermark_ok(zone, order,
3113 						low_wmark_pages(zone),
3114 						*classzone_idx, 0))
3115 				pgdat_needs_compaction = false;
3116 		}
3117 
3118 		/*
3119 		 * If we're getting trouble reclaiming, start doing writepage
3120 		 * even in laptop mode.
3121 		 */
3122 		if (sc.priority < DEF_PRIORITY - 2)
3123 			sc.may_writepage = 1;
3124 
3125 		/*
3126 		 * Now scan the zone in the dma->highmem direction, stopping
3127 		 * at the last zone which needs scanning.
3128 		 *
3129 		 * We do this because the page allocator works in the opposite
3130 		 * direction.  This prevents the page allocator from allocating
3131 		 * pages behind kswapd's direction of progress, which would
3132 		 * cause too much scanning of the lower zones.
3133 		 */
3134 		for (i = 0; i <= end_zone; i++) {
3135 			struct zone *zone = pgdat->node_zones + i;
3136 
3137 			if (!populated_zone(zone))
3138 				continue;
3139 
3140 			if (sc.priority != DEF_PRIORITY &&
3141 			    !zone_reclaimable(zone))
3142 				continue;
3143 
3144 			sc.nr_scanned = 0;
3145 
3146 			nr_soft_scanned = 0;
3147 			/*
3148 			 * Call soft limit reclaim before calling shrink_zone.
3149 			 */
3150 			nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone,
3151 							order, sc.gfp_mask,
3152 							&nr_soft_scanned);
3153 			sc.nr_reclaimed += nr_soft_reclaimed;
3154 
3155 			/*
3156 			 * There should be no need to raise the scanning
3157 			 * priority if enough pages are already being scanned
3158 			 * that that high watermark would be met at 100%
3159 			 * efficiency.
3160 			 */
3161 			if (kswapd_shrink_zone(zone, end_zone, &sc,
3162 					lru_pages, &nr_attempted))
3163 				raise_priority = false;
3164 		}
3165 
3166 		/*
3167 		 * If the low watermark is met there is no need for processes
3168 		 * to be throttled on pfmemalloc_wait as they should not be
3169 		 * able to safely make forward progress. Wake them
3170 		 */
3171 		if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
3172 				pfmemalloc_watermark_ok(pgdat))
3173 			wake_up(&pgdat->pfmemalloc_wait);
3174 
3175 		/*
3176 		 * Fragmentation may mean that the system cannot be rebalanced
3177 		 * for high-order allocations in all zones. If twice the
3178 		 * allocation size has been reclaimed and the zones are still
3179 		 * not balanced then recheck the watermarks at order-0 to
3180 		 * prevent kswapd reclaiming excessively. Assume that a
3181 		 * process requested a high-order can direct reclaim/compact.
3182 		 */
3183 		if (order && sc.nr_reclaimed >= 2UL << order)
3184 			order = sc.order = 0;
3185 
3186 		/* Check if kswapd should be suspending */
3187 		if (try_to_freeze() || kthread_should_stop())
3188 			break;
3189 
3190 		/*
3191 		 * Compact if necessary and kswapd is reclaiming at least the
3192 		 * high watermark number of pages as requsted
3193 		 */
3194 		if (pgdat_needs_compaction && sc.nr_reclaimed > nr_attempted)
3195 			compact_pgdat(pgdat, order);
3196 
3197 		/*
3198 		 * Raise priority if scanning rate is too low or there was no
3199 		 * progress in reclaiming pages
3200 		 */
3201 		if (raise_priority || !sc.nr_reclaimed)
3202 			sc.priority--;
3203 	} while (sc.priority >= 1 &&
3204 		 !pgdat_balanced(pgdat, order, *classzone_idx));
3205 
3206 out:
3207 	/*
3208 	 * Return the order we were reclaiming at so prepare_kswapd_sleep()
3209 	 * makes a decision on the order we were last reclaiming at. However,
3210 	 * if another caller entered the allocator slow path while kswapd
3211 	 * was awake, order will remain at the higher level
3212 	 */
3213 	*classzone_idx = end_zone;
3214 	return order;
3215 }
3216 
3217 static void kswapd_try_to_sleep(pg_data_t *pgdat, int order, int classzone_idx)
3218 {
3219 	long remaining = 0;
3220 	DEFINE_WAIT(wait);
3221 
3222 	if (freezing(current) || kthread_should_stop())
3223 		return;
3224 
3225 	prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
3226 
3227 	/* Try to sleep for a short interval */
3228 	if (prepare_kswapd_sleep(pgdat, order, remaining, classzone_idx)) {
3229 		remaining = schedule_timeout(HZ/10);
3230 		finish_wait(&pgdat->kswapd_wait, &wait);
3231 		prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
3232 	}
3233 
3234 	/*
3235 	 * After a short sleep, check if it was a premature sleep. If not, then
3236 	 * go fully to sleep until explicitly woken up.
3237 	 */
3238 	if (prepare_kswapd_sleep(pgdat, order, remaining, classzone_idx)) {
3239 		trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
3240 
3241 		/*
3242 		 * vmstat counters are not perfectly accurate and the estimated
3243 		 * value for counters such as NR_FREE_PAGES can deviate from the
3244 		 * true value by nr_online_cpus * threshold. To avoid the zone
3245 		 * watermarks being breached while under pressure, we reduce the
3246 		 * per-cpu vmstat threshold while kswapd is awake and restore
3247 		 * them before going back to sleep.
3248 		 */
3249 		set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
3250 
3251 		/*
3252 		 * Compaction records what page blocks it recently failed to
3253 		 * isolate pages from and skips them in the future scanning.
3254 		 * When kswapd is going to sleep, it is reasonable to assume
3255 		 * that pages and compaction may succeed so reset the cache.
3256 		 */
3257 		reset_isolation_suitable(pgdat);
3258 
3259 		if (!kthread_should_stop())
3260 			schedule();
3261 
3262 		set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);
3263 	} else {
3264 		if (remaining)
3265 			count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
3266 		else
3267 			count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
3268 	}
3269 	finish_wait(&pgdat->kswapd_wait, &wait);
3270 }
3271 
3272 /*
3273  * The background pageout daemon, started as a kernel thread
3274  * from the init process.
3275  *
3276  * This basically trickles out pages so that we have _some_
3277  * free memory available even if there is no other activity
3278  * that frees anything up. This is needed for things like routing
3279  * etc, where we otherwise might have all activity going on in
3280  * asynchronous contexts that cannot page things out.
3281  *
3282  * If there are applications that are active memory-allocators
3283  * (most normal use), this basically shouldn't matter.
3284  */
3285 static int kswapd(void *p)
3286 {
3287 	unsigned long order, new_order;
3288 	unsigned balanced_order;
3289 	int classzone_idx, new_classzone_idx;
3290 	int balanced_classzone_idx;
3291 	pg_data_t *pgdat = (pg_data_t*)p;
3292 	struct task_struct *tsk = current;
3293 
3294 	struct reclaim_state reclaim_state = {
3295 		.reclaimed_slab = 0,
3296 	};
3297 	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
3298 
3299 	lockdep_set_current_reclaim_state(GFP_KERNEL);
3300 
3301 	if (!cpumask_empty(cpumask))
3302 		set_cpus_allowed_ptr(tsk, cpumask);
3303 	current->reclaim_state = &reclaim_state;
3304 
3305 	/*
3306 	 * Tell the memory management that we're a "memory allocator",
3307 	 * and that if we need more memory we should get access to it
3308 	 * regardless (see "__alloc_pages()"). "kswapd" should
3309 	 * never get caught in the normal page freeing logic.
3310 	 *
3311 	 * (Kswapd normally doesn't need memory anyway, but sometimes
3312 	 * you need a small amount of memory in order to be able to
3313 	 * page out something else, and this flag essentially protects
3314 	 * us from recursively trying to free more memory as we're
3315 	 * trying to free the first piece of memory in the first place).
3316 	 */
3317 	tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD;
3318 	set_freezable();
3319 
3320 	order = new_order = 0;
3321 	balanced_order = 0;
3322 	classzone_idx = new_classzone_idx = pgdat->nr_zones - 1;
3323 	balanced_classzone_idx = classzone_idx;
3324 	for ( ; ; ) {
3325 		bool ret;
3326 
3327 		/*
3328 		 * If the last balance_pgdat was unsuccessful it's unlikely a
3329 		 * new request of a similar or harder type will succeed soon
3330 		 * so consider going to sleep on the basis we reclaimed at
3331 		 */
3332 		if (balanced_classzone_idx >= new_classzone_idx &&
3333 					balanced_order == new_order) {
3334 			new_order = pgdat->kswapd_max_order;
3335 			new_classzone_idx = pgdat->classzone_idx;
3336 			pgdat->kswapd_max_order =  0;
3337 			pgdat->classzone_idx = pgdat->nr_zones - 1;
3338 		}
3339 
3340 		if (order < new_order || classzone_idx > new_classzone_idx) {
3341 			/*
3342 			 * Don't sleep if someone wants a larger 'order'
3343 			 * allocation or has tigher zone constraints
3344 			 */
3345 			order = new_order;
3346 			classzone_idx = new_classzone_idx;
3347 		} else {
3348 			kswapd_try_to_sleep(pgdat, balanced_order,
3349 						balanced_classzone_idx);
3350 			order = pgdat->kswapd_max_order;
3351 			classzone_idx = pgdat->classzone_idx;
3352 			new_order = order;
3353 			new_classzone_idx = classzone_idx;
3354 			pgdat->kswapd_max_order = 0;
3355 			pgdat->classzone_idx = pgdat->nr_zones - 1;
3356 		}
3357 
3358 		ret = try_to_freeze();
3359 		if (kthread_should_stop())
3360 			break;
3361 
3362 		/*
3363 		 * We can speed up thawing tasks if we don't call balance_pgdat
3364 		 * after returning from the refrigerator
3365 		 */
3366 		if (!ret) {
3367 			trace_mm_vmscan_kswapd_wake(pgdat->node_id, order);
3368 			balanced_classzone_idx = classzone_idx;
3369 			balanced_order = balance_pgdat(pgdat, order,
3370 						&balanced_classzone_idx);
3371 		}
3372 	}
3373 
3374 	tsk->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD);
3375 	current->reclaim_state = NULL;
3376 	lockdep_clear_current_reclaim_state();
3377 
3378 	return 0;
3379 }
3380 
3381 /*
3382  * A zone is low on free memory, so wake its kswapd task to service it.
3383  */
3384 void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx)
3385 {
3386 	pg_data_t *pgdat;
3387 
3388 	if (!populated_zone(zone))
3389 		return;
3390 
3391 	if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
3392 		return;
3393 	pgdat = zone->zone_pgdat;
3394 	if (pgdat->kswapd_max_order < order) {
3395 		pgdat->kswapd_max_order = order;
3396 		pgdat->classzone_idx = min(pgdat->classzone_idx, classzone_idx);
3397 	}
3398 	if (!waitqueue_active(&pgdat->kswapd_wait))
3399 		return;
3400 	if (zone_balanced(zone, order, 0, 0))
3401 		return;
3402 
3403 	trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, zone_idx(zone), order);
3404 	wake_up_interruptible(&pgdat->kswapd_wait);
3405 }
3406 
3407 #ifdef CONFIG_HIBERNATION
3408 /*
3409  * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
3410  * freed pages.
3411  *
3412  * Rather than trying to age LRUs the aim is to preserve the overall
3413  * LRU order by reclaiming preferentially
3414  * inactive > active > active referenced > active mapped
3415  */
3416 unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
3417 {
3418 	struct reclaim_state reclaim_state;
3419 	struct scan_control sc = {
3420 		.nr_to_reclaim = nr_to_reclaim,
3421 		.gfp_mask = GFP_HIGHUSER_MOVABLE,
3422 		.priority = DEF_PRIORITY,
3423 		.may_writepage = 1,
3424 		.may_unmap = 1,
3425 		.may_swap = 1,
3426 		.hibernation_mode = 1,
3427 	};
3428 	struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3429 	struct task_struct *p = current;
3430 	unsigned long nr_reclaimed;
3431 
3432 	p->flags |= PF_MEMALLOC;
3433 	lockdep_set_current_reclaim_state(sc.gfp_mask);
3434 	reclaim_state.reclaimed_slab = 0;
3435 	p->reclaim_state = &reclaim_state;
3436 
3437 	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3438 
3439 	p->reclaim_state = NULL;
3440 	lockdep_clear_current_reclaim_state();
3441 	p->flags &= ~PF_MEMALLOC;
3442 
3443 	return nr_reclaimed;
3444 }
3445 #endif /* CONFIG_HIBERNATION */
3446 
3447 /* It's optimal to keep kswapds on the same CPUs as their memory, but
3448    not required for correctness.  So if the last cpu in a node goes
3449    away, we get changed to run anywhere: as the first one comes back,
3450    restore their cpu bindings. */
3451 static int cpu_callback(struct notifier_block *nfb, unsigned long action,
3452 			void *hcpu)
3453 {
3454 	int nid;
3455 
3456 	if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) {
3457 		for_each_node_state(nid, N_MEMORY) {
3458 			pg_data_t *pgdat = NODE_DATA(nid);
3459 			const struct cpumask *mask;
3460 
3461 			mask = cpumask_of_node(pgdat->node_id);
3462 
3463 			if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
3464 				/* One of our CPUs online: restore mask */
3465 				set_cpus_allowed_ptr(pgdat->kswapd, mask);
3466 		}
3467 	}
3468 	return NOTIFY_OK;
3469 }
3470 
3471 /*
3472  * This kswapd start function will be called by init and node-hot-add.
3473  * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
3474  */
3475 int kswapd_run(int nid)
3476 {
3477 	pg_data_t *pgdat = NODE_DATA(nid);
3478 	int ret = 0;
3479 
3480 	if (pgdat->kswapd)
3481 		return 0;
3482 
3483 	pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
3484 	if (IS_ERR(pgdat->kswapd)) {
3485 		/* failure at boot is fatal */
3486 		BUG_ON(system_state == SYSTEM_BOOTING);
3487 		pr_err("Failed to start kswapd on node %d\n", nid);
3488 		ret = PTR_ERR(pgdat->kswapd);
3489 		pgdat->kswapd = NULL;
3490 	}
3491 	return ret;
3492 }
3493 
3494 /*
3495  * Called by memory hotplug when all memory in a node is offlined.  Caller must
3496  * hold mem_hotplug_begin/end().
3497  */
3498 void kswapd_stop(int nid)
3499 {
3500 	struct task_struct *kswapd = NODE_DATA(nid)->kswapd;
3501 
3502 	if (kswapd) {
3503 		kthread_stop(kswapd);
3504 		NODE_DATA(nid)->kswapd = NULL;
3505 	}
3506 }
3507 
3508 static int __init kswapd_init(void)
3509 {
3510 	int nid;
3511 
3512 	swap_setup();
3513 	for_each_node_state(nid, N_MEMORY)
3514  		kswapd_run(nid);
3515 	hotcpu_notifier(cpu_callback, 0);
3516 	return 0;
3517 }
3518 
3519 module_init(kswapd_init)
3520 
3521 #ifdef CONFIG_NUMA
3522 /*
3523  * Zone reclaim mode
3524  *
3525  * If non-zero call zone_reclaim when the number of free pages falls below
3526  * the watermarks.
3527  */
3528 int zone_reclaim_mode __read_mostly;
3529 
3530 #define RECLAIM_OFF 0
3531 #define RECLAIM_ZONE (1<<0)	/* Run shrink_inactive_list on the zone */
3532 #define RECLAIM_WRITE (1<<1)	/* Writeout pages during reclaim */
3533 #define RECLAIM_SWAP (1<<2)	/* Swap pages out during reclaim */
3534 
3535 /*
3536  * Priority for ZONE_RECLAIM. This determines the fraction of pages
3537  * of a node considered for each zone_reclaim. 4 scans 1/16th of
3538  * a zone.
3539  */
3540 #define ZONE_RECLAIM_PRIORITY 4
3541 
3542 /*
3543  * Percentage of pages in a zone that must be unmapped for zone_reclaim to
3544  * occur.
3545  */
3546 int sysctl_min_unmapped_ratio = 1;
3547 
3548 /*
3549  * If the number of slab pages in a zone grows beyond this percentage then
3550  * slab reclaim needs to occur.
3551  */
3552 int sysctl_min_slab_ratio = 5;
3553 
3554 static inline unsigned long zone_unmapped_file_pages(struct zone *zone)
3555 {
3556 	unsigned long file_mapped = zone_page_state(zone, NR_FILE_MAPPED);
3557 	unsigned long file_lru = zone_page_state(zone, NR_INACTIVE_FILE) +
3558 		zone_page_state(zone, NR_ACTIVE_FILE);
3559 
3560 	/*
3561 	 * It's possible for there to be more file mapped pages than
3562 	 * accounted for by the pages on the file LRU lists because
3563 	 * tmpfs pages accounted for as ANON can also be FILE_MAPPED
3564 	 */
3565 	return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
3566 }
3567 
3568 /* Work out how many page cache pages we can reclaim in this reclaim_mode */
3569 static long zone_pagecache_reclaimable(struct zone *zone)
3570 {
3571 	long nr_pagecache_reclaimable;
3572 	long delta = 0;
3573 
3574 	/*
3575 	 * If RECLAIM_SWAP is set, then all file pages are considered
3576 	 * potentially reclaimable. Otherwise, we have to worry about
3577 	 * pages like swapcache and zone_unmapped_file_pages() provides
3578 	 * a better estimate
3579 	 */
3580 	if (zone_reclaim_mode & RECLAIM_SWAP)
3581 		nr_pagecache_reclaimable = zone_page_state(zone, NR_FILE_PAGES);
3582 	else
3583 		nr_pagecache_reclaimable = zone_unmapped_file_pages(zone);
3584 
3585 	/* If we can't clean pages, remove dirty pages from consideration */
3586 	if (!(zone_reclaim_mode & RECLAIM_WRITE))
3587 		delta += zone_page_state(zone, NR_FILE_DIRTY);
3588 
3589 	/* Watch for any possible underflows due to delta */
3590 	if (unlikely(delta > nr_pagecache_reclaimable))
3591 		delta = nr_pagecache_reclaimable;
3592 
3593 	return nr_pagecache_reclaimable - delta;
3594 }
3595 
3596 /*
3597  * Try to free up some pages from this zone through reclaim.
3598  */
3599 static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
3600 {
3601 	/* Minimum pages needed in order to stay on node */
3602 	const unsigned long nr_pages = 1 << order;
3603 	struct task_struct *p = current;
3604 	struct reclaim_state reclaim_state;
3605 	struct scan_control sc = {
3606 		.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
3607 		.gfp_mask = (gfp_mask = memalloc_noio_flags(gfp_mask)),
3608 		.order = order,
3609 		.priority = ZONE_RECLAIM_PRIORITY,
3610 		.may_writepage = !!(zone_reclaim_mode & RECLAIM_WRITE),
3611 		.may_unmap = !!(zone_reclaim_mode & RECLAIM_SWAP),
3612 		.may_swap = 1,
3613 	};
3614 	struct shrink_control shrink = {
3615 		.gfp_mask = sc.gfp_mask,
3616 	};
3617 	unsigned long nr_slab_pages0, nr_slab_pages1;
3618 
3619 	cond_resched();
3620 	/*
3621 	 * We need to be able to allocate from the reserves for RECLAIM_SWAP
3622 	 * and we also need to be able to write out pages for RECLAIM_WRITE
3623 	 * and RECLAIM_SWAP.
3624 	 */
3625 	p->flags |= PF_MEMALLOC | PF_SWAPWRITE;
3626 	lockdep_set_current_reclaim_state(gfp_mask);
3627 	reclaim_state.reclaimed_slab = 0;
3628 	p->reclaim_state = &reclaim_state;
3629 
3630 	if (zone_pagecache_reclaimable(zone) > zone->min_unmapped_pages) {
3631 		/*
3632 		 * Free memory by calling shrink zone with increasing
3633 		 * priorities until we have enough memory freed.
3634 		 */
3635 		do {
3636 			shrink_zone(zone, &sc);
3637 		} while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
3638 	}
3639 
3640 	nr_slab_pages0 = zone_page_state(zone, NR_SLAB_RECLAIMABLE);
3641 	if (nr_slab_pages0 > zone->min_slab_pages) {
3642 		/*
3643 		 * shrink_slab() does not currently allow us to determine how
3644 		 * many pages were freed in this zone. So we take the current
3645 		 * number of slab pages and shake the slab until it is reduced
3646 		 * by the same nr_pages that we used for reclaiming unmapped
3647 		 * pages.
3648 		 */
3649 		nodes_clear(shrink.nodes_to_scan);
3650 		node_set(zone_to_nid(zone), shrink.nodes_to_scan);
3651 		for (;;) {
3652 			unsigned long lru_pages = zone_reclaimable_pages(zone);
3653 
3654 			/* No reclaimable slab or very low memory pressure */
3655 			if (!shrink_slab(&shrink, sc.nr_scanned, lru_pages))
3656 				break;
3657 
3658 			/* Freed enough memory */
3659 			nr_slab_pages1 = zone_page_state(zone,
3660 							NR_SLAB_RECLAIMABLE);
3661 			if (nr_slab_pages1 + nr_pages <= nr_slab_pages0)
3662 				break;
3663 		}
3664 
3665 		/*
3666 		 * Update nr_reclaimed by the number of slab pages we
3667 		 * reclaimed from this zone.
3668 		 */
3669 		nr_slab_pages1 = zone_page_state(zone, NR_SLAB_RECLAIMABLE);
3670 		if (nr_slab_pages1 < nr_slab_pages0)
3671 			sc.nr_reclaimed += nr_slab_pages0 - nr_slab_pages1;
3672 	}
3673 
3674 	p->reclaim_state = NULL;
3675 	current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE);
3676 	lockdep_clear_current_reclaim_state();
3677 	return sc.nr_reclaimed >= nr_pages;
3678 }
3679 
3680 int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
3681 {
3682 	int node_id;
3683 	int ret;
3684 
3685 	/*
3686 	 * Zone reclaim reclaims unmapped file backed pages and
3687 	 * slab pages if we are over the defined limits.
3688 	 *
3689 	 * A small portion of unmapped file backed pages is needed for
3690 	 * file I/O otherwise pages read by file I/O will be immediately
3691 	 * thrown out if the zone is overallocated. So we do not reclaim
3692 	 * if less than a specified percentage of the zone is used by
3693 	 * unmapped file backed pages.
3694 	 */
3695 	if (zone_pagecache_reclaimable(zone) <= zone->min_unmapped_pages &&
3696 	    zone_page_state(zone, NR_SLAB_RECLAIMABLE) <= zone->min_slab_pages)
3697 		return ZONE_RECLAIM_FULL;
3698 
3699 	if (!zone_reclaimable(zone))
3700 		return ZONE_RECLAIM_FULL;
3701 
3702 	/*
3703 	 * Do not scan if the allocation should not be delayed.
3704 	 */
3705 	if (!(gfp_mask & __GFP_WAIT) || (current->flags & PF_MEMALLOC))
3706 		return ZONE_RECLAIM_NOSCAN;
3707 
3708 	/*
3709 	 * Only run zone reclaim on the local zone or on zones that do not
3710 	 * have associated processors. This will favor the local processor
3711 	 * over remote processors and spread off node memory allocations
3712 	 * as wide as possible.
3713 	 */
3714 	node_id = zone_to_nid(zone);
3715 	if (node_state(node_id, N_CPU) && node_id != numa_node_id())
3716 		return ZONE_RECLAIM_NOSCAN;
3717 
3718 	if (test_and_set_bit(ZONE_RECLAIM_LOCKED, &zone->flags))
3719 		return ZONE_RECLAIM_NOSCAN;
3720 
3721 	ret = __zone_reclaim(zone, gfp_mask, order);
3722 	clear_bit(ZONE_RECLAIM_LOCKED, &zone->flags);
3723 
3724 	if (!ret)
3725 		count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
3726 
3727 	return ret;
3728 }
3729 #endif
3730 
3731 /*
3732  * page_evictable - test whether a page is evictable
3733  * @page: the page to test
3734  *
3735  * Test whether page is evictable--i.e., should be placed on active/inactive
3736  * lists vs unevictable list.
3737  *
3738  * Reasons page might not be evictable:
3739  * (1) page's mapping marked unevictable
3740  * (2) page is part of an mlocked VMA
3741  *
3742  */
3743 int page_evictable(struct page *page)
3744 {
3745 	return !mapping_unevictable(page_mapping(page)) && !PageMlocked(page);
3746 }
3747 
3748 #ifdef CONFIG_SHMEM
3749 /**
3750  * check_move_unevictable_pages - check pages for evictability and move to appropriate zone lru list
3751  * @pages:	array of pages to check
3752  * @nr_pages:	number of pages to check
3753  *
3754  * Checks pages for evictability and moves them to the appropriate lru list.
3755  *
3756  * This function is only used for SysV IPC SHM_UNLOCK.
3757  */
3758 void check_move_unevictable_pages(struct page **pages, int nr_pages)
3759 {
3760 	struct lruvec *lruvec;
3761 	struct zone *zone = NULL;
3762 	int pgscanned = 0;
3763 	int pgrescued = 0;
3764 	int i;
3765 
3766 	for (i = 0; i < nr_pages; i++) {
3767 		struct page *page = pages[i];
3768 		struct zone *pagezone;
3769 
3770 		pgscanned++;
3771 		pagezone = page_zone(page);
3772 		if (pagezone != zone) {
3773 			if (zone)
3774 				spin_unlock_irq(&zone->lru_lock);
3775 			zone = pagezone;
3776 			spin_lock_irq(&zone->lru_lock);
3777 		}
3778 		lruvec = mem_cgroup_page_lruvec(page, zone);
3779 
3780 		if (!PageLRU(page) || !PageUnevictable(page))
3781 			continue;
3782 
3783 		if (page_evictable(page)) {
3784 			enum lru_list lru = page_lru_base_type(page);
3785 
3786 			VM_BUG_ON_PAGE(PageActive(page), page);
3787 			ClearPageUnevictable(page);
3788 			del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE);
3789 			add_page_to_lru_list(page, lruvec, lru);
3790 			pgrescued++;
3791 		}
3792 	}
3793 
3794 	if (zone) {
3795 		__count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
3796 		__count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
3797 		spin_unlock_irq(&zone->lru_lock);
3798 	}
3799 }
3800 #endif /* CONFIG_SHMEM */
3801