xref: /openbmc/linux/mm/vmscan.c (revision 495311927ffbe3604e915aeafdf03325e9925b9d)
1 /*
2  *  linux/mm/vmscan.c
3  *
4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
5  *
6  *  Swap reorganised 29.12.95, Stephen Tweedie.
7  *  kswapd added: 7.1.96  sct
8  *  Removed kswapd_ctl limits, and swap out as many pages as needed
9  *  to bring the system back to freepages.high: 2.4.97, Rik van Riel.
10  *  Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
11  *  Multiqueue VM started 5.8.00, Rik van Riel.
12  */
13 
14 #include <linux/mm.h>
15 #include <linux/module.h>
16 #include <linux/gfp.h>
17 #include <linux/kernel_stat.h>
18 #include <linux/swap.h>
19 #include <linux/pagemap.h>
20 #include <linux/init.h>
21 #include <linux/highmem.h>
22 #include <linux/vmstat.h>
23 #include <linux/file.h>
24 #include <linux/writeback.h>
25 #include <linux/blkdev.h>
26 #include <linux/buffer_head.h>	/* for try_to_release_page(),
27 					buffer_heads_over_limit */
28 #include <linux/mm_inline.h>
29 #include <linux/pagevec.h>
30 #include <linux/backing-dev.h>
31 #include <linux/rmap.h>
32 #include <linux/topology.h>
33 #include <linux/cpu.h>
34 #include <linux/cpuset.h>
35 #include <linux/notifier.h>
36 #include <linux/rwsem.h>
37 #include <linux/delay.h>
38 #include <linux/kthread.h>
39 #include <linux/freezer.h>
40 #include <linux/memcontrol.h>
41 #include <linux/delayacct.h>
42 #include <linux/sysctl.h>
43 
44 #include <asm/tlbflush.h>
45 #include <asm/div64.h>
46 
47 #include <linux/swapops.h>
48 
49 #include "internal.h"
50 
51 #define CREATE_TRACE_POINTS
52 #include <trace/events/vmscan.h>
53 
54 struct scan_control {
55 	/* Incremented by the number of inactive pages that were scanned */
56 	unsigned long nr_scanned;
57 
58 	/* Number of pages freed so far during a call to shrink_zones() */
59 	unsigned long nr_reclaimed;
60 
61 	/* How many pages shrink_list() should reclaim */
62 	unsigned long nr_to_reclaim;
63 
64 	unsigned long hibernation_mode;
65 
66 	/* This context's GFP mask */
67 	gfp_t gfp_mask;
68 
69 	int may_writepage;
70 
71 	/* Can mapped pages be reclaimed? */
72 	int may_unmap;
73 
74 	/* Can pages be swapped as part of reclaim? */
75 	int may_swap;
76 
77 	int swappiness;
78 
79 	int order;
80 
81 	/*
82 	 * Intend to reclaim enough contenious memory rather than to reclaim
83 	 * enough amount memory. I.e, it's the mode for high order allocation.
84 	 */
85 	bool lumpy_reclaim_mode;
86 
87 	/* Which cgroup do we reclaim from */
88 	struct mem_cgroup *mem_cgroup;
89 
90 	/*
91 	 * Nodemask of nodes allowed by the caller. If NULL, all nodes
92 	 * are scanned.
93 	 */
94 	nodemask_t	*nodemask;
95 };
96 
97 #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
98 
99 #ifdef ARCH_HAS_PREFETCH
100 #define prefetch_prev_lru_page(_page, _base, _field)			\
101 	do {								\
102 		if ((_page)->lru.prev != _base) {			\
103 			struct page *prev;				\
104 									\
105 			prev = lru_to_page(&(_page->lru));		\
106 			prefetch(&prev->_field);			\
107 		}							\
108 	} while (0)
109 #else
110 #define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
111 #endif
112 
113 #ifdef ARCH_HAS_PREFETCHW
114 #define prefetchw_prev_lru_page(_page, _base, _field)			\
115 	do {								\
116 		if ((_page)->lru.prev != _base) {			\
117 			struct page *prev;				\
118 									\
119 			prev = lru_to_page(&(_page->lru));		\
120 			prefetchw(&prev->_field);			\
121 		}							\
122 	} while (0)
123 #else
124 #define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
125 #endif
126 
127 /*
128  * From 0 .. 100.  Higher means more swappy.
129  */
130 int vm_swappiness = 60;
131 long vm_total_pages;	/* The total number of pages which the VM controls */
132 
133 static LIST_HEAD(shrinker_list);
134 static DECLARE_RWSEM(shrinker_rwsem);
135 
136 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
137 #define scanning_global_lru(sc)	(!(sc)->mem_cgroup)
138 #else
139 #define scanning_global_lru(sc)	(1)
140 #endif
141 
142 static struct zone_reclaim_stat *get_reclaim_stat(struct zone *zone,
143 						  struct scan_control *sc)
144 {
145 	if (!scanning_global_lru(sc))
146 		return mem_cgroup_get_reclaim_stat(sc->mem_cgroup, zone);
147 
148 	return &zone->reclaim_stat;
149 }
150 
151 static unsigned long zone_nr_lru_pages(struct zone *zone,
152 				struct scan_control *sc, enum lru_list lru)
153 {
154 	if (!scanning_global_lru(sc))
155 		return mem_cgroup_zone_nr_pages(sc->mem_cgroup, zone, lru);
156 
157 	return zone_page_state(zone, NR_LRU_BASE + lru);
158 }
159 
160 
161 /*
162  * Add a shrinker callback to be called from the vm
163  */
164 void register_shrinker(struct shrinker *shrinker)
165 {
166 	shrinker->nr = 0;
167 	down_write(&shrinker_rwsem);
168 	list_add_tail(&shrinker->list, &shrinker_list);
169 	up_write(&shrinker_rwsem);
170 }
171 EXPORT_SYMBOL(register_shrinker);
172 
173 /*
174  * Remove one
175  */
176 void unregister_shrinker(struct shrinker *shrinker)
177 {
178 	down_write(&shrinker_rwsem);
179 	list_del(&shrinker->list);
180 	up_write(&shrinker_rwsem);
181 }
182 EXPORT_SYMBOL(unregister_shrinker);
183 
184 #define SHRINK_BATCH 128
185 /*
186  * Call the shrink functions to age shrinkable caches
187  *
188  * Here we assume it costs one seek to replace a lru page and that it also
189  * takes a seek to recreate a cache object.  With this in mind we age equal
190  * percentages of the lru and ageable caches.  This should balance the seeks
191  * generated by these structures.
192  *
193  * If the vm encountered mapped pages on the LRU it increase the pressure on
194  * slab to avoid swapping.
195  *
196  * We do weird things to avoid (scanned*seeks*entries) overflowing 32 bits.
197  *
198  * `lru_pages' represents the number of on-LRU pages in all the zones which
199  * are eligible for the caller's allocation attempt.  It is used for balancing
200  * slab reclaim versus page reclaim.
201  *
202  * Returns the number of slab objects which we shrunk.
203  */
204 unsigned long shrink_slab(unsigned long scanned, gfp_t gfp_mask,
205 			unsigned long lru_pages)
206 {
207 	struct shrinker *shrinker;
208 	unsigned long ret = 0;
209 
210 	if (scanned == 0)
211 		scanned = SWAP_CLUSTER_MAX;
212 
213 	if (!down_read_trylock(&shrinker_rwsem))
214 		return 1;	/* Assume we'll be able to shrink next time */
215 
216 	list_for_each_entry(shrinker, &shrinker_list, list) {
217 		unsigned long long delta;
218 		unsigned long total_scan;
219 		unsigned long max_pass;
220 
221 		max_pass = (*shrinker->shrink)(shrinker, 0, gfp_mask);
222 		delta = (4 * scanned) / shrinker->seeks;
223 		delta *= max_pass;
224 		do_div(delta, lru_pages + 1);
225 		shrinker->nr += delta;
226 		if (shrinker->nr < 0) {
227 			printk(KERN_ERR "shrink_slab: %pF negative objects to "
228 			       "delete nr=%ld\n",
229 			       shrinker->shrink, shrinker->nr);
230 			shrinker->nr = max_pass;
231 		}
232 
233 		/*
234 		 * Avoid risking looping forever due to too large nr value:
235 		 * never try to free more than twice the estimate number of
236 		 * freeable entries.
237 		 */
238 		if (shrinker->nr > max_pass * 2)
239 			shrinker->nr = max_pass * 2;
240 
241 		total_scan = shrinker->nr;
242 		shrinker->nr = 0;
243 
244 		while (total_scan >= SHRINK_BATCH) {
245 			long this_scan = SHRINK_BATCH;
246 			int shrink_ret;
247 			int nr_before;
248 
249 			nr_before = (*shrinker->shrink)(shrinker, 0, gfp_mask);
250 			shrink_ret = (*shrinker->shrink)(shrinker, this_scan,
251 								gfp_mask);
252 			if (shrink_ret == -1)
253 				break;
254 			if (shrink_ret < nr_before)
255 				ret += nr_before - shrink_ret;
256 			count_vm_events(SLABS_SCANNED, this_scan);
257 			total_scan -= this_scan;
258 
259 			cond_resched();
260 		}
261 
262 		shrinker->nr += total_scan;
263 	}
264 	up_read(&shrinker_rwsem);
265 	return ret;
266 }
267 
268 static inline int is_page_cache_freeable(struct page *page)
269 {
270 	/*
271 	 * A freeable page cache page is referenced only by the caller
272 	 * that isolated the page, the page cache radix tree and
273 	 * optional buffer heads at page->private.
274 	 */
275 	return page_count(page) - page_has_private(page) == 2;
276 }
277 
278 static int may_write_to_queue(struct backing_dev_info *bdi)
279 {
280 	if (current->flags & PF_SWAPWRITE)
281 		return 1;
282 	if (!bdi_write_congested(bdi))
283 		return 1;
284 	if (bdi == current->backing_dev_info)
285 		return 1;
286 	return 0;
287 }
288 
289 /*
290  * We detected a synchronous write error writing a page out.  Probably
291  * -ENOSPC.  We need to propagate that into the address_space for a subsequent
292  * fsync(), msync() or close().
293  *
294  * The tricky part is that after writepage we cannot touch the mapping: nothing
295  * prevents it from being freed up.  But we have a ref on the page and once
296  * that page is locked, the mapping is pinned.
297  *
298  * We're allowed to run sleeping lock_page() here because we know the caller has
299  * __GFP_FS.
300  */
301 static void handle_write_error(struct address_space *mapping,
302 				struct page *page, int error)
303 {
304 	lock_page_nosync(page);
305 	if (page_mapping(page) == mapping)
306 		mapping_set_error(mapping, error);
307 	unlock_page(page);
308 }
309 
310 /* Request for sync pageout. */
311 enum pageout_io {
312 	PAGEOUT_IO_ASYNC,
313 	PAGEOUT_IO_SYNC,
314 };
315 
316 /* possible outcome of pageout() */
317 typedef enum {
318 	/* failed to write page out, page is locked */
319 	PAGE_KEEP,
320 	/* move page to the active list, page is locked */
321 	PAGE_ACTIVATE,
322 	/* page has been sent to the disk successfully, page is unlocked */
323 	PAGE_SUCCESS,
324 	/* page is clean and locked */
325 	PAGE_CLEAN,
326 } pageout_t;
327 
328 /*
329  * pageout is called by shrink_page_list() for each dirty page.
330  * Calls ->writepage().
331  */
332 static pageout_t pageout(struct page *page, struct address_space *mapping,
333 						enum pageout_io sync_writeback)
334 {
335 	/*
336 	 * If the page is dirty, only perform writeback if that write
337 	 * will be non-blocking.  To prevent this allocation from being
338 	 * stalled by pagecache activity.  But note that there may be
339 	 * stalls if we need to run get_block().  We could test
340 	 * PagePrivate for that.
341 	 *
342 	 * If this process is currently in __generic_file_aio_write() against
343 	 * this page's queue, we can perform writeback even if that
344 	 * will block.
345 	 *
346 	 * If the page is swapcache, write it back even if that would
347 	 * block, for some throttling. This happens by accident, because
348 	 * swap_backing_dev_info is bust: it doesn't reflect the
349 	 * congestion state of the swapdevs.  Easy to fix, if needed.
350 	 */
351 	if (!is_page_cache_freeable(page))
352 		return PAGE_KEEP;
353 	if (!mapping) {
354 		/*
355 		 * Some data journaling orphaned pages can have
356 		 * page->mapping == NULL while being dirty with clean buffers.
357 		 */
358 		if (page_has_private(page)) {
359 			if (try_to_free_buffers(page)) {
360 				ClearPageDirty(page);
361 				printk("%s: orphaned page\n", __func__);
362 				return PAGE_CLEAN;
363 			}
364 		}
365 		return PAGE_KEEP;
366 	}
367 	if (mapping->a_ops->writepage == NULL)
368 		return PAGE_ACTIVATE;
369 	if (!may_write_to_queue(mapping->backing_dev_info))
370 		return PAGE_KEEP;
371 
372 	if (clear_page_dirty_for_io(page)) {
373 		int res;
374 		struct writeback_control wbc = {
375 			.sync_mode = WB_SYNC_NONE,
376 			.nr_to_write = SWAP_CLUSTER_MAX,
377 			.range_start = 0,
378 			.range_end = LLONG_MAX,
379 			.nonblocking = 1,
380 			.for_reclaim = 1,
381 		};
382 
383 		SetPageReclaim(page);
384 		res = mapping->a_ops->writepage(page, &wbc);
385 		if (res < 0)
386 			handle_write_error(mapping, page, res);
387 		if (res == AOP_WRITEPAGE_ACTIVATE) {
388 			ClearPageReclaim(page);
389 			return PAGE_ACTIVATE;
390 		}
391 
392 		/*
393 		 * Wait on writeback if requested to. This happens when
394 		 * direct reclaiming a large contiguous area and the
395 		 * first attempt to free a range of pages fails.
396 		 */
397 		if (PageWriteback(page) && sync_writeback == PAGEOUT_IO_SYNC)
398 			wait_on_page_writeback(page);
399 
400 		if (!PageWriteback(page)) {
401 			/* synchronous write or broken a_ops? */
402 			ClearPageReclaim(page);
403 		}
404 		trace_mm_vmscan_writepage(page,
405 			trace_reclaim_flags(page, sync_writeback));
406 		inc_zone_page_state(page, NR_VMSCAN_WRITE);
407 		return PAGE_SUCCESS;
408 	}
409 
410 	return PAGE_CLEAN;
411 }
412 
413 /*
414  * Same as remove_mapping, but if the page is removed from the mapping, it
415  * gets returned with a refcount of 0.
416  */
417 static int __remove_mapping(struct address_space *mapping, struct page *page)
418 {
419 	BUG_ON(!PageLocked(page));
420 	BUG_ON(mapping != page_mapping(page));
421 
422 	spin_lock_irq(&mapping->tree_lock);
423 	/*
424 	 * The non racy check for a busy page.
425 	 *
426 	 * Must be careful with the order of the tests. When someone has
427 	 * a ref to the page, it may be possible that they dirty it then
428 	 * drop the reference. So if PageDirty is tested before page_count
429 	 * here, then the following race may occur:
430 	 *
431 	 * get_user_pages(&page);
432 	 * [user mapping goes away]
433 	 * write_to(page);
434 	 *				!PageDirty(page)    [good]
435 	 * SetPageDirty(page);
436 	 * put_page(page);
437 	 *				!page_count(page)   [good, discard it]
438 	 *
439 	 * [oops, our write_to data is lost]
440 	 *
441 	 * Reversing the order of the tests ensures such a situation cannot
442 	 * escape unnoticed. The smp_rmb is needed to ensure the page->flags
443 	 * load is not satisfied before that of page->_count.
444 	 *
445 	 * Note that if SetPageDirty is always performed via set_page_dirty,
446 	 * and thus under tree_lock, then this ordering is not required.
447 	 */
448 	if (!page_freeze_refs(page, 2))
449 		goto cannot_free;
450 	/* note: atomic_cmpxchg in page_freeze_refs provides the smp_rmb */
451 	if (unlikely(PageDirty(page))) {
452 		page_unfreeze_refs(page, 2);
453 		goto cannot_free;
454 	}
455 
456 	if (PageSwapCache(page)) {
457 		swp_entry_t swap = { .val = page_private(page) };
458 		__delete_from_swap_cache(page);
459 		spin_unlock_irq(&mapping->tree_lock);
460 		swapcache_free(swap, page);
461 	} else {
462 		__remove_from_page_cache(page);
463 		spin_unlock_irq(&mapping->tree_lock);
464 		mem_cgroup_uncharge_cache_page(page);
465 	}
466 
467 	return 1;
468 
469 cannot_free:
470 	spin_unlock_irq(&mapping->tree_lock);
471 	return 0;
472 }
473 
474 /*
475  * Attempt to detach a locked page from its ->mapping.  If it is dirty or if
476  * someone else has a ref on the page, abort and return 0.  If it was
477  * successfully detached, return 1.  Assumes the caller has a single ref on
478  * this page.
479  */
480 int remove_mapping(struct address_space *mapping, struct page *page)
481 {
482 	if (__remove_mapping(mapping, page)) {
483 		/*
484 		 * Unfreezing the refcount with 1 rather than 2 effectively
485 		 * drops the pagecache ref for us without requiring another
486 		 * atomic operation.
487 		 */
488 		page_unfreeze_refs(page, 1);
489 		return 1;
490 	}
491 	return 0;
492 }
493 
494 /**
495  * putback_lru_page - put previously isolated page onto appropriate LRU list
496  * @page: page to be put back to appropriate lru list
497  *
498  * Add previously isolated @page to appropriate LRU list.
499  * Page may still be unevictable for other reasons.
500  *
501  * lru_lock must not be held, interrupts must be enabled.
502  */
503 void putback_lru_page(struct page *page)
504 {
505 	int lru;
506 	int active = !!TestClearPageActive(page);
507 	int was_unevictable = PageUnevictable(page);
508 
509 	VM_BUG_ON(PageLRU(page));
510 
511 redo:
512 	ClearPageUnevictable(page);
513 
514 	if (page_evictable(page, NULL)) {
515 		/*
516 		 * For evictable pages, we can use the cache.
517 		 * In event of a race, worst case is we end up with an
518 		 * unevictable page on [in]active list.
519 		 * We know how to handle that.
520 		 */
521 		lru = active + page_lru_base_type(page);
522 		lru_cache_add_lru(page, lru);
523 	} else {
524 		/*
525 		 * Put unevictable pages directly on zone's unevictable
526 		 * list.
527 		 */
528 		lru = LRU_UNEVICTABLE;
529 		add_page_to_unevictable_list(page);
530 		/*
531 		 * When racing with an mlock clearing (page is
532 		 * unlocked), make sure that if the other thread does
533 		 * not observe our setting of PG_lru and fails
534 		 * isolation, we see PG_mlocked cleared below and move
535 		 * the page back to the evictable list.
536 		 *
537 		 * The other side is TestClearPageMlocked().
538 		 */
539 		smp_mb();
540 	}
541 
542 	/*
543 	 * page's status can change while we move it among lru. If an evictable
544 	 * page is on unevictable list, it never be freed. To avoid that,
545 	 * check after we added it to the list, again.
546 	 */
547 	if (lru == LRU_UNEVICTABLE && page_evictable(page, NULL)) {
548 		if (!isolate_lru_page(page)) {
549 			put_page(page);
550 			goto redo;
551 		}
552 		/* This means someone else dropped this page from LRU
553 		 * So, it will be freed or putback to LRU again. There is
554 		 * nothing to do here.
555 		 */
556 	}
557 
558 	if (was_unevictable && lru != LRU_UNEVICTABLE)
559 		count_vm_event(UNEVICTABLE_PGRESCUED);
560 	else if (!was_unevictable && lru == LRU_UNEVICTABLE)
561 		count_vm_event(UNEVICTABLE_PGCULLED);
562 
563 	put_page(page);		/* drop ref from isolate */
564 }
565 
566 enum page_references {
567 	PAGEREF_RECLAIM,
568 	PAGEREF_RECLAIM_CLEAN,
569 	PAGEREF_KEEP,
570 	PAGEREF_ACTIVATE,
571 };
572 
573 static enum page_references page_check_references(struct page *page,
574 						  struct scan_control *sc)
575 {
576 	int referenced_ptes, referenced_page;
577 	unsigned long vm_flags;
578 
579 	referenced_ptes = page_referenced(page, 1, sc->mem_cgroup, &vm_flags);
580 	referenced_page = TestClearPageReferenced(page);
581 
582 	/* Lumpy reclaim - ignore references */
583 	if (sc->lumpy_reclaim_mode)
584 		return PAGEREF_RECLAIM;
585 
586 	/*
587 	 * Mlock lost the isolation race with us.  Let try_to_unmap()
588 	 * move the page to the unevictable list.
589 	 */
590 	if (vm_flags & VM_LOCKED)
591 		return PAGEREF_RECLAIM;
592 
593 	if (referenced_ptes) {
594 		if (PageAnon(page))
595 			return PAGEREF_ACTIVATE;
596 		/*
597 		 * All mapped pages start out with page table
598 		 * references from the instantiating fault, so we need
599 		 * to look twice if a mapped file page is used more
600 		 * than once.
601 		 *
602 		 * Mark it and spare it for another trip around the
603 		 * inactive list.  Another page table reference will
604 		 * lead to its activation.
605 		 *
606 		 * Note: the mark is set for activated pages as well
607 		 * so that recently deactivated but used pages are
608 		 * quickly recovered.
609 		 */
610 		SetPageReferenced(page);
611 
612 		if (referenced_page)
613 			return PAGEREF_ACTIVATE;
614 
615 		return PAGEREF_KEEP;
616 	}
617 
618 	/* Reclaim if clean, defer dirty pages to writeback */
619 	if (referenced_page)
620 		return PAGEREF_RECLAIM_CLEAN;
621 
622 	return PAGEREF_RECLAIM;
623 }
624 
625 static noinline_for_stack void free_page_list(struct list_head *free_pages)
626 {
627 	struct pagevec freed_pvec;
628 	struct page *page, *tmp;
629 
630 	pagevec_init(&freed_pvec, 1);
631 
632 	list_for_each_entry_safe(page, tmp, free_pages, lru) {
633 		list_del(&page->lru);
634 		if (!pagevec_add(&freed_pvec, page)) {
635 			__pagevec_free(&freed_pvec);
636 			pagevec_reinit(&freed_pvec);
637 		}
638 	}
639 
640 	pagevec_free(&freed_pvec);
641 }
642 
643 /*
644  * shrink_page_list() returns the number of reclaimed pages
645  */
646 static unsigned long shrink_page_list(struct list_head *page_list,
647 					struct scan_control *sc,
648 					enum pageout_io sync_writeback)
649 {
650 	LIST_HEAD(ret_pages);
651 	LIST_HEAD(free_pages);
652 	int pgactivate = 0;
653 	unsigned long nr_reclaimed = 0;
654 
655 	cond_resched();
656 
657 	while (!list_empty(page_list)) {
658 		enum page_references references;
659 		struct address_space *mapping;
660 		struct page *page;
661 		int may_enter_fs;
662 
663 		cond_resched();
664 
665 		page = lru_to_page(page_list);
666 		list_del(&page->lru);
667 
668 		if (!trylock_page(page))
669 			goto keep;
670 
671 		VM_BUG_ON(PageActive(page));
672 
673 		sc->nr_scanned++;
674 
675 		if (unlikely(!page_evictable(page, NULL)))
676 			goto cull_mlocked;
677 
678 		if (!sc->may_unmap && page_mapped(page))
679 			goto keep_locked;
680 
681 		/* Double the slab pressure for mapped and swapcache pages */
682 		if (page_mapped(page) || PageSwapCache(page))
683 			sc->nr_scanned++;
684 
685 		may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
686 			(PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
687 
688 		if (PageWriteback(page)) {
689 			/*
690 			 * Synchronous reclaim is performed in two passes,
691 			 * first an asynchronous pass over the list to
692 			 * start parallel writeback, and a second synchronous
693 			 * pass to wait for the IO to complete.  Wait here
694 			 * for any page for which writeback has already
695 			 * started.
696 			 */
697 			if (sync_writeback == PAGEOUT_IO_SYNC && may_enter_fs)
698 				wait_on_page_writeback(page);
699 			else
700 				goto keep_locked;
701 		}
702 
703 		references = page_check_references(page, sc);
704 		switch (references) {
705 		case PAGEREF_ACTIVATE:
706 			goto activate_locked;
707 		case PAGEREF_KEEP:
708 			goto keep_locked;
709 		case PAGEREF_RECLAIM:
710 		case PAGEREF_RECLAIM_CLEAN:
711 			; /* try to reclaim the page below */
712 		}
713 
714 		/*
715 		 * Anonymous process memory has backing store?
716 		 * Try to allocate it some swap space here.
717 		 */
718 		if (PageAnon(page) && !PageSwapCache(page)) {
719 			if (!(sc->gfp_mask & __GFP_IO))
720 				goto keep_locked;
721 			if (!add_to_swap(page))
722 				goto activate_locked;
723 			may_enter_fs = 1;
724 		}
725 
726 		mapping = page_mapping(page);
727 
728 		/*
729 		 * The page is mapped into the page tables of one or more
730 		 * processes. Try to unmap it here.
731 		 */
732 		if (page_mapped(page) && mapping) {
733 			switch (try_to_unmap(page, TTU_UNMAP)) {
734 			case SWAP_FAIL:
735 				goto activate_locked;
736 			case SWAP_AGAIN:
737 				goto keep_locked;
738 			case SWAP_MLOCK:
739 				goto cull_mlocked;
740 			case SWAP_SUCCESS:
741 				; /* try to free the page below */
742 			}
743 		}
744 
745 		if (PageDirty(page)) {
746 			if (references == PAGEREF_RECLAIM_CLEAN)
747 				goto keep_locked;
748 			if (!may_enter_fs)
749 				goto keep_locked;
750 			if (!sc->may_writepage)
751 				goto keep_locked;
752 
753 			/* Page is dirty, try to write it out here */
754 			switch (pageout(page, mapping, sync_writeback)) {
755 			case PAGE_KEEP:
756 				goto keep_locked;
757 			case PAGE_ACTIVATE:
758 				goto activate_locked;
759 			case PAGE_SUCCESS:
760 				if (PageWriteback(page) || PageDirty(page))
761 					goto keep;
762 				/*
763 				 * A synchronous write - probably a ramdisk.  Go
764 				 * ahead and try to reclaim the page.
765 				 */
766 				if (!trylock_page(page))
767 					goto keep;
768 				if (PageDirty(page) || PageWriteback(page))
769 					goto keep_locked;
770 				mapping = page_mapping(page);
771 			case PAGE_CLEAN:
772 				; /* try to free the page below */
773 			}
774 		}
775 
776 		/*
777 		 * If the page has buffers, try to free the buffer mappings
778 		 * associated with this page. If we succeed we try to free
779 		 * the page as well.
780 		 *
781 		 * We do this even if the page is PageDirty().
782 		 * try_to_release_page() does not perform I/O, but it is
783 		 * possible for a page to have PageDirty set, but it is actually
784 		 * clean (all its buffers are clean).  This happens if the
785 		 * buffers were written out directly, with submit_bh(). ext3
786 		 * will do this, as well as the blockdev mapping.
787 		 * try_to_release_page() will discover that cleanness and will
788 		 * drop the buffers and mark the page clean - it can be freed.
789 		 *
790 		 * Rarely, pages can have buffers and no ->mapping.  These are
791 		 * the pages which were not successfully invalidated in
792 		 * truncate_complete_page().  We try to drop those buffers here
793 		 * and if that worked, and the page is no longer mapped into
794 		 * process address space (page_count == 1) it can be freed.
795 		 * Otherwise, leave the page on the LRU so it is swappable.
796 		 */
797 		if (page_has_private(page)) {
798 			if (!try_to_release_page(page, sc->gfp_mask))
799 				goto activate_locked;
800 			if (!mapping && page_count(page) == 1) {
801 				unlock_page(page);
802 				if (put_page_testzero(page))
803 					goto free_it;
804 				else {
805 					/*
806 					 * rare race with speculative reference.
807 					 * the speculative reference will free
808 					 * this page shortly, so we may
809 					 * increment nr_reclaimed here (and
810 					 * leave it off the LRU).
811 					 */
812 					nr_reclaimed++;
813 					continue;
814 				}
815 			}
816 		}
817 
818 		if (!mapping || !__remove_mapping(mapping, page))
819 			goto keep_locked;
820 
821 		/*
822 		 * At this point, we have no other references and there is
823 		 * no way to pick any more up (removed from LRU, removed
824 		 * from pagecache). Can use non-atomic bitops now (and
825 		 * we obviously don't have to worry about waking up a process
826 		 * waiting on the page lock, because there are no references.
827 		 */
828 		__clear_page_locked(page);
829 free_it:
830 		nr_reclaimed++;
831 
832 		/*
833 		 * Is there need to periodically free_page_list? It would
834 		 * appear not as the counts should be low
835 		 */
836 		list_add(&page->lru, &free_pages);
837 		continue;
838 
839 cull_mlocked:
840 		if (PageSwapCache(page))
841 			try_to_free_swap(page);
842 		unlock_page(page);
843 		putback_lru_page(page);
844 		continue;
845 
846 activate_locked:
847 		/* Not a candidate for swapping, so reclaim swap space. */
848 		if (PageSwapCache(page) && vm_swap_full())
849 			try_to_free_swap(page);
850 		VM_BUG_ON(PageActive(page));
851 		SetPageActive(page);
852 		pgactivate++;
853 keep_locked:
854 		unlock_page(page);
855 keep:
856 		list_add(&page->lru, &ret_pages);
857 		VM_BUG_ON(PageLRU(page) || PageUnevictable(page));
858 	}
859 
860 	free_page_list(&free_pages);
861 
862 	list_splice(&ret_pages, page_list);
863 	count_vm_events(PGACTIVATE, pgactivate);
864 	return nr_reclaimed;
865 }
866 
867 /*
868  * Attempt to remove the specified page from its LRU.  Only take this page
869  * if it is of the appropriate PageActive status.  Pages which are being
870  * freed elsewhere are also ignored.
871  *
872  * page:	page to consider
873  * mode:	one of the LRU isolation modes defined above
874  *
875  * returns 0 on success, -ve errno on failure.
876  */
877 int __isolate_lru_page(struct page *page, int mode, int file)
878 {
879 	int ret = -EINVAL;
880 
881 	/* Only take pages on the LRU. */
882 	if (!PageLRU(page))
883 		return ret;
884 
885 	/*
886 	 * When checking the active state, we need to be sure we are
887 	 * dealing with comparible boolean values.  Take the logical not
888 	 * of each.
889 	 */
890 	if (mode != ISOLATE_BOTH && (!PageActive(page) != !mode))
891 		return ret;
892 
893 	if (mode != ISOLATE_BOTH && page_is_file_cache(page) != file)
894 		return ret;
895 
896 	/*
897 	 * When this function is being called for lumpy reclaim, we
898 	 * initially look into all LRU pages, active, inactive and
899 	 * unevictable; only give shrink_page_list evictable pages.
900 	 */
901 	if (PageUnevictable(page))
902 		return ret;
903 
904 	ret = -EBUSY;
905 
906 	if (likely(get_page_unless_zero(page))) {
907 		/*
908 		 * Be careful not to clear PageLRU until after we're
909 		 * sure the page is not being freed elsewhere -- the
910 		 * page release code relies on it.
911 		 */
912 		ClearPageLRU(page);
913 		ret = 0;
914 	}
915 
916 	return ret;
917 }
918 
919 /*
920  * zone->lru_lock is heavily contended.  Some of the functions that
921  * shrink the lists perform better by taking out a batch of pages
922  * and working on them outside the LRU lock.
923  *
924  * For pagecache intensive workloads, this function is the hottest
925  * spot in the kernel (apart from copy_*_user functions).
926  *
927  * Appropriate locks must be held before calling this function.
928  *
929  * @nr_to_scan:	The number of pages to look through on the list.
930  * @src:	The LRU list to pull pages off.
931  * @dst:	The temp list to put pages on to.
932  * @scanned:	The number of pages that were scanned.
933  * @order:	The caller's attempted allocation order
934  * @mode:	One of the LRU isolation modes
935  * @file:	True [1] if isolating file [!anon] pages
936  *
937  * returns how many pages were moved onto *@dst.
938  */
939 static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
940 		struct list_head *src, struct list_head *dst,
941 		unsigned long *scanned, int order, int mode, int file)
942 {
943 	unsigned long nr_taken = 0;
944 	unsigned long nr_lumpy_taken = 0;
945 	unsigned long nr_lumpy_dirty = 0;
946 	unsigned long nr_lumpy_failed = 0;
947 	unsigned long scan;
948 
949 	for (scan = 0; scan < nr_to_scan && !list_empty(src); scan++) {
950 		struct page *page;
951 		unsigned long pfn;
952 		unsigned long end_pfn;
953 		unsigned long page_pfn;
954 		int zone_id;
955 
956 		page = lru_to_page(src);
957 		prefetchw_prev_lru_page(page, src, flags);
958 
959 		VM_BUG_ON(!PageLRU(page));
960 
961 		switch (__isolate_lru_page(page, mode, file)) {
962 		case 0:
963 			list_move(&page->lru, dst);
964 			mem_cgroup_del_lru(page);
965 			nr_taken++;
966 			break;
967 
968 		case -EBUSY:
969 			/* else it is being freed elsewhere */
970 			list_move(&page->lru, src);
971 			mem_cgroup_rotate_lru_list(page, page_lru(page));
972 			continue;
973 
974 		default:
975 			BUG();
976 		}
977 
978 		if (!order)
979 			continue;
980 
981 		/*
982 		 * Attempt to take all pages in the order aligned region
983 		 * surrounding the tag page.  Only take those pages of
984 		 * the same active state as that tag page.  We may safely
985 		 * round the target page pfn down to the requested order
986 		 * as the mem_map is guarenteed valid out to MAX_ORDER,
987 		 * where that page is in a different zone we will detect
988 		 * it from its zone id and abort this block scan.
989 		 */
990 		zone_id = page_zone_id(page);
991 		page_pfn = page_to_pfn(page);
992 		pfn = page_pfn & ~((1 << order) - 1);
993 		end_pfn = pfn + (1 << order);
994 		for (; pfn < end_pfn; pfn++) {
995 			struct page *cursor_page;
996 
997 			/* The target page is in the block, ignore it. */
998 			if (unlikely(pfn == page_pfn))
999 				continue;
1000 
1001 			/* Avoid holes within the zone. */
1002 			if (unlikely(!pfn_valid_within(pfn)))
1003 				break;
1004 
1005 			cursor_page = pfn_to_page(pfn);
1006 
1007 			/* Check that we have not crossed a zone boundary. */
1008 			if (unlikely(page_zone_id(cursor_page) != zone_id))
1009 				continue;
1010 
1011 			/*
1012 			 * If we don't have enough swap space, reclaiming of
1013 			 * anon page which don't already have a swap slot is
1014 			 * pointless.
1015 			 */
1016 			if (nr_swap_pages <= 0 && PageAnon(cursor_page) &&
1017 					!PageSwapCache(cursor_page))
1018 				continue;
1019 
1020 			if (__isolate_lru_page(cursor_page, mode, file) == 0) {
1021 				list_move(&cursor_page->lru, dst);
1022 				mem_cgroup_del_lru(cursor_page);
1023 				nr_taken++;
1024 				nr_lumpy_taken++;
1025 				if (PageDirty(cursor_page))
1026 					nr_lumpy_dirty++;
1027 				scan++;
1028 			} else {
1029 				if (mode == ISOLATE_BOTH &&
1030 						page_count(cursor_page))
1031 					nr_lumpy_failed++;
1032 			}
1033 		}
1034 	}
1035 
1036 	*scanned = scan;
1037 
1038 	trace_mm_vmscan_lru_isolate(order,
1039 			nr_to_scan, scan,
1040 			nr_taken,
1041 			nr_lumpy_taken, nr_lumpy_dirty, nr_lumpy_failed,
1042 			mode);
1043 	return nr_taken;
1044 }
1045 
1046 static unsigned long isolate_pages_global(unsigned long nr,
1047 					struct list_head *dst,
1048 					unsigned long *scanned, int order,
1049 					int mode, struct zone *z,
1050 					int active, int file)
1051 {
1052 	int lru = LRU_BASE;
1053 	if (active)
1054 		lru += LRU_ACTIVE;
1055 	if (file)
1056 		lru += LRU_FILE;
1057 	return isolate_lru_pages(nr, &z->lru[lru].list, dst, scanned, order,
1058 								mode, file);
1059 }
1060 
1061 /*
1062  * clear_active_flags() is a helper for shrink_active_list(), clearing
1063  * any active bits from the pages in the list.
1064  */
1065 static unsigned long clear_active_flags(struct list_head *page_list,
1066 					unsigned int *count)
1067 {
1068 	int nr_active = 0;
1069 	int lru;
1070 	struct page *page;
1071 
1072 	list_for_each_entry(page, page_list, lru) {
1073 		lru = page_lru_base_type(page);
1074 		if (PageActive(page)) {
1075 			lru += LRU_ACTIVE;
1076 			ClearPageActive(page);
1077 			nr_active++;
1078 		}
1079 		if (count)
1080 			count[lru]++;
1081 	}
1082 
1083 	return nr_active;
1084 }
1085 
1086 /**
1087  * isolate_lru_page - tries to isolate a page from its LRU list
1088  * @page: page to isolate from its LRU list
1089  *
1090  * Isolates a @page from an LRU list, clears PageLRU and adjusts the
1091  * vmstat statistic corresponding to whatever LRU list the page was on.
1092  *
1093  * Returns 0 if the page was removed from an LRU list.
1094  * Returns -EBUSY if the page was not on an LRU list.
1095  *
1096  * The returned page will have PageLRU() cleared.  If it was found on
1097  * the active list, it will have PageActive set.  If it was found on
1098  * the unevictable list, it will have the PageUnevictable bit set. That flag
1099  * may need to be cleared by the caller before letting the page go.
1100  *
1101  * The vmstat statistic corresponding to the list on which the page was
1102  * found will be decremented.
1103  *
1104  * Restrictions:
1105  * (1) Must be called with an elevated refcount on the page. This is a
1106  *     fundamentnal difference from isolate_lru_pages (which is called
1107  *     without a stable reference).
1108  * (2) the lru_lock must not be held.
1109  * (3) interrupts must be enabled.
1110  */
1111 int isolate_lru_page(struct page *page)
1112 {
1113 	int ret = -EBUSY;
1114 
1115 	if (PageLRU(page)) {
1116 		struct zone *zone = page_zone(page);
1117 
1118 		spin_lock_irq(&zone->lru_lock);
1119 		if (PageLRU(page) && get_page_unless_zero(page)) {
1120 			int lru = page_lru(page);
1121 			ret = 0;
1122 			ClearPageLRU(page);
1123 
1124 			del_page_from_lru_list(zone, page, lru);
1125 		}
1126 		spin_unlock_irq(&zone->lru_lock);
1127 	}
1128 	return ret;
1129 }
1130 
1131 /*
1132  * Are there way too many processes in the direct reclaim path already?
1133  */
1134 static int too_many_isolated(struct zone *zone, int file,
1135 		struct scan_control *sc)
1136 {
1137 	unsigned long inactive, isolated;
1138 
1139 	if (current_is_kswapd())
1140 		return 0;
1141 
1142 	if (!scanning_global_lru(sc))
1143 		return 0;
1144 
1145 	if (file) {
1146 		inactive = zone_page_state(zone, NR_INACTIVE_FILE);
1147 		isolated = zone_page_state(zone, NR_ISOLATED_FILE);
1148 	} else {
1149 		inactive = zone_page_state(zone, NR_INACTIVE_ANON);
1150 		isolated = zone_page_state(zone, NR_ISOLATED_ANON);
1151 	}
1152 
1153 	return isolated > inactive;
1154 }
1155 
1156 /*
1157  * TODO: Try merging with migrations version of putback_lru_pages
1158  */
1159 static noinline_for_stack void
1160 putback_lru_pages(struct zone *zone, struct scan_control *sc,
1161 				unsigned long nr_anon, unsigned long nr_file,
1162 				struct list_head *page_list)
1163 {
1164 	struct page *page;
1165 	struct pagevec pvec;
1166 	struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(zone, sc);
1167 
1168 	pagevec_init(&pvec, 1);
1169 
1170 	/*
1171 	 * Put back any unfreeable pages.
1172 	 */
1173 	spin_lock(&zone->lru_lock);
1174 	while (!list_empty(page_list)) {
1175 		int lru;
1176 		page = lru_to_page(page_list);
1177 		VM_BUG_ON(PageLRU(page));
1178 		list_del(&page->lru);
1179 		if (unlikely(!page_evictable(page, NULL))) {
1180 			spin_unlock_irq(&zone->lru_lock);
1181 			putback_lru_page(page);
1182 			spin_lock_irq(&zone->lru_lock);
1183 			continue;
1184 		}
1185 		SetPageLRU(page);
1186 		lru = page_lru(page);
1187 		add_page_to_lru_list(zone, page, lru);
1188 		if (is_active_lru(lru)) {
1189 			int file = is_file_lru(lru);
1190 			reclaim_stat->recent_rotated[file]++;
1191 		}
1192 		if (!pagevec_add(&pvec, page)) {
1193 			spin_unlock_irq(&zone->lru_lock);
1194 			__pagevec_release(&pvec);
1195 			spin_lock_irq(&zone->lru_lock);
1196 		}
1197 	}
1198 	__mod_zone_page_state(zone, NR_ISOLATED_ANON, -nr_anon);
1199 	__mod_zone_page_state(zone, NR_ISOLATED_FILE, -nr_file);
1200 
1201 	spin_unlock_irq(&zone->lru_lock);
1202 	pagevec_release(&pvec);
1203 }
1204 
1205 static noinline_for_stack void update_isolated_counts(struct zone *zone,
1206 					struct scan_control *sc,
1207 					unsigned long *nr_anon,
1208 					unsigned long *nr_file,
1209 					struct list_head *isolated_list)
1210 {
1211 	unsigned long nr_active;
1212 	unsigned int count[NR_LRU_LISTS] = { 0, };
1213 	struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(zone, sc);
1214 
1215 	nr_active = clear_active_flags(isolated_list, count);
1216 	__count_vm_events(PGDEACTIVATE, nr_active);
1217 
1218 	__mod_zone_page_state(zone, NR_ACTIVE_FILE,
1219 			      -count[LRU_ACTIVE_FILE]);
1220 	__mod_zone_page_state(zone, NR_INACTIVE_FILE,
1221 			      -count[LRU_INACTIVE_FILE]);
1222 	__mod_zone_page_state(zone, NR_ACTIVE_ANON,
1223 			      -count[LRU_ACTIVE_ANON]);
1224 	__mod_zone_page_state(zone, NR_INACTIVE_ANON,
1225 			      -count[LRU_INACTIVE_ANON]);
1226 
1227 	*nr_anon = count[LRU_ACTIVE_ANON] + count[LRU_INACTIVE_ANON];
1228 	*nr_file = count[LRU_ACTIVE_FILE] + count[LRU_INACTIVE_FILE];
1229 	__mod_zone_page_state(zone, NR_ISOLATED_ANON, *nr_anon);
1230 	__mod_zone_page_state(zone, NR_ISOLATED_FILE, *nr_file);
1231 
1232 	reclaim_stat->recent_scanned[0] += *nr_anon;
1233 	reclaim_stat->recent_scanned[1] += *nr_file;
1234 }
1235 
1236 /*
1237  * Returns true if the caller should wait to clean dirty/writeback pages.
1238  *
1239  * If we are direct reclaiming for contiguous pages and we do not reclaim
1240  * everything in the list, try again and wait for writeback IO to complete.
1241  * This will stall high-order allocations noticeably. Only do that when really
1242  * need to free the pages under high memory pressure.
1243  */
1244 static inline bool should_reclaim_stall(unsigned long nr_taken,
1245 					unsigned long nr_freed,
1246 					int priority,
1247 					struct scan_control *sc)
1248 {
1249 	int lumpy_stall_priority;
1250 
1251 	/* kswapd should not stall on sync IO */
1252 	if (current_is_kswapd())
1253 		return false;
1254 
1255 	/* Only stall on lumpy reclaim */
1256 	if (!sc->lumpy_reclaim_mode)
1257 		return false;
1258 
1259 	/* If we have relaimed everything on the isolated list, no stall */
1260 	if (nr_freed == nr_taken)
1261 		return false;
1262 
1263 	/*
1264 	 * For high-order allocations, there are two stall thresholds.
1265 	 * High-cost allocations stall immediately where as lower
1266 	 * order allocations such as stacks require the scanning
1267 	 * priority to be much higher before stalling.
1268 	 */
1269 	if (sc->order > PAGE_ALLOC_COSTLY_ORDER)
1270 		lumpy_stall_priority = DEF_PRIORITY;
1271 	else
1272 		lumpy_stall_priority = DEF_PRIORITY / 3;
1273 
1274 	return priority <= lumpy_stall_priority;
1275 }
1276 
1277 /*
1278  * shrink_inactive_list() is a helper for shrink_zone().  It returns the number
1279  * of reclaimed pages
1280  */
1281 static noinline_for_stack unsigned long
1282 shrink_inactive_list(unsigned long nr_to_scan, struct zone *zone,
1283 			struct scan_control *sc, int priority, int file)
1284 {
1285 	LIST_HEAD(page_list);
1286 	unsigned long nr_scanned;
1287 	unsigned long nr_reclaimed = 0;
1288 	unsigned long nr_taken;
1289 	unsigned long nr_active;
1290 	unsigned long nr_anon;
1291 	unsigned long nr_file;
1292 
1293 	while (unlikely(too_many_isolated(zone, file, sc))) {
1294 		congestion_wait(BLK_RW_ASYNC, HZ/10);
1295 
1296 		/* We are about to die and free our memory. Return now. */
1297 		if (fatal_signal_pending(current))
1298 			return SWAP_CLUSTER_MAX;
1299 	}
1300 
1301 
1302 	lru_add_drain();
1303 	spin_lock_irq(&zone->lru_lock);
1304 
1305 	if (scanning_global_lru(sc)) {
1306 		nr_taken = isolate_pages_global(nr_to_scan,
1307 			&page_list, &nr_scanned, sc->order,
1308 			sc->lumpy_reclaim_mode ?
1309 				ISOLATE_BOTH : ISOLATE_INACTIVE,
1310 			zone, 0, file);
1311 		zone->pages_scanned += nr_scanned;
1312 		if (current_is_kswapd())
1313 			__count_zone_vm_events(PGSCAN_KSWAPD, zone,
1314 					       nr_scanned);
1315 		else
1316 			__count_zone_vm_events(PGSCAN_DIRECT, zone,
1317 					       nr_scanned);
1318 	} else {
1319 		nr_taken = mem_cgroup_isolate_pages(nr_to_scan,
1320 			&page_list, &nr_scanned, sc->order,
1321 			sc->lumpy_reclaim_mode ?
1322 				ISOLATE_BOTH : ISOLATE_INACTIVE,
1323 			zone, sc->mem_cgroup,
1324 			0, file);
1325 		/*
1326 		 * mem_cgroup_isolate_pages() keeps track of
1327 		 * scanned pages on its own.
1328 		 */
1329 	}
1330 
1331 	if (nr_taken == 0) {
1332 		spin_unlock_irq(&zone->lru_lock);
1333 		return 0;
1334 	}
1335 
1336 	update_isolated_counts(zone, sc, &nr_anon, &nr_file, &page_list);
1337 
1338 	spin_unlock_irq(&zone->lru_lock);
1339 
1340 	nr_reclaimed = shrink_page_list(&page_list, sc, PAGEOUT_IO_ASYNC);
1341 
1342 	/* Check if we should syncronously wait for writeback */
1343 	if (should_reclaim_stall(nr_taken, nr_reclaimed, priority, sc)) {
1344 		congestion_wait(BLK_RW_ASYNC, HZ/10);
1345 
1346 		/*
1347 		 * The attempt at page out may have made some
1348 		 * of the pages active, mark them inactive again.
1349 		 */
1350 		nr_active = clear_active_flags(&page_list, NULL);
1351 		count_vm_events(PGDEACTIVATE, nr_active);
1352 
1353 		nr_reclaimed += shrink_page_list(&page_list, sc, PAGEOUT_IO_SYNC);
1354 	}
1355 
1356 	local_irq_disable();
1357 	if (current_is_kswapd())
1358 		__count_vm_events(KSWAPD_STEAL, nr_reclaimed);
1359 	__count_zone_vm_events(PGSTEAL, zone, nr_reclaimed);
1360 
1361 	putback_lru_pages(zone, sc, nr_anon, nr_file, &page_list);
1362 	return nr_reclaimed;
1363 }
1364 
1365 /*
1366  * This moves pages from the active list to the inactive list.
1367  *
1368  * We move them the other way if the page is referenced by one or more
1369  * processes, from rmap.
1370  *
1371  * If the pages are mostly unmapped, the processing is fast and it is
1372  * appropriate to hold zone->lru_lock across the whole operation.  But if
1373  * the pages are mapped, the processing is slow (page_referenced()) so we
1374  * should drop zone->lru_lock around each page.  It's impossible to balance
1375  * this, so instead we remove the pages from the LRU while processing them.
1376  * It is safe to rely on PG_active against the non-LRU pages in here because
1377  * nobody will play with that bit on a non-LRU page.
1378  *
1379  * The downside is that we have to touch page->_count against each page.
1380  * But we had to alter page->flags anyway.
1381  */
1382 
1383 static void move_active_pages_to_lru(struct zone *zone,
1384 				     struct list_head *list,
1385 				     enum lru_list lru)
1386 {
1387 	unsigned long pgmoved = 0;
1388 	struct pagevec pvec;
1389 	struct page *page;
1390 
1391 	pagevec_init(&pvec, 1);
1392 
1393 	while (!list_empty(list)) {
1394 		page = lru_to_page(list);
1395 
1396 		VM_BUG_ON(PageLRU(page));
1397 		SetPageLRU(page);
1398 
1399 		list_move(&page->lru, &zone->lru[lru].list);
1400 		mem_cgroup_add_lru_list(page, lru);
1401 		pgmoved++;
1402 
1403 		if (!pagevec_add(&pvec, page) || list_empty(list)) {
1404 			spin_unlock_irq(&zone->lru_lock);
1405 			if (buffer_heads_over_limit)
1406 				pagevec_strip(&pvec);
1407 			__pagevec_release(&pvec);
1408 			spin_lock_irq(&zone->lru_lock);
1409 		}
1410 	}
1411 	__mod_zone_page_state(zone, NR_LRU_BASE + lru, pgmoved);
1412 	if (!is_active_lru(lru))
1413 		__count_vm_events(PGDEACTIVATE, pgmoved);
1414 }
1415 
1416 static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
1417 			struct scan_control *sc, int priority, int file)
1418 {
1419 	unsigned long nr_taken;
1420 	unsigned long pgscanned;
1421 	unsigned long vm_flags;
1422 	LIST_HEAD(l_hold);	/* The pages which were snipped off */
1423 	LIST_HEAD(l_active);
1424 	LIST_HEAD(l_inactive);
1425 	struct page *page;
1426 	struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(zone, sc);
1427 	unsigned long nr_rotated = 0;
1428 
1429 	lru_add_drain();
1430 	spin_lock_irq(&zone->lru_lock);
1431 	if (scanning_global_lru(sc)) {
1432 		nr_taken = isolate_pages_global(nr_pages, &l_hold,
1433 						&pgscanned, sc->order,
1434 						ISOLATE_ACTIVE, zone,
1435 						1, file);
1436 		zone->pages_scanned += pgscanned;
1437 	} else {
1438 		nr_taken = mem_cgroup_isolate_pages(nr_pages, &l_hold,
1439 						&pgscanned, sc->order,
1440 						ISOLATE_ACTIVE, zone,
1441 						sc->mem_cgroup, 1, file);
1442 		/*
1443 		 * mem_cgroup_isolate_pages() keeps track of
1444 		 * scanned pages on its own.
1445 		 */
1446 	}
1447 
1448 	reclaim_stat->recent_scanned[file] += nr_taken;
1449 
1450 	__count_zone_vm_events(PGREFILL, zone, pgscanned);
1451 	if (file)
1452 		__mod_zone_page_state(zone, NR_ACTIVE_FILE, -nr_taken);
1453 	else
1454 		__mod_zone_page_state(zone, NR_ACTIVE_ANON, -nr_taken);
1455 	__mod_zone_page_state(zone, NR_ISOLATED_ANON + file, nr_taken);
1456 	spin_unlock_irq(&zone->lru_lock);
1457 
1458 	while (!list_empty(&l_hold)) {
1459 		cond_resched();
1460 		page = lru_to_page(&l_hold);
1461 		list_del(&page->lru);
1462 
1463 		if (unlikely(!page_evictable(page, NULL))) {
1464 			putback_lru_page(page);
1465 			continue;
1466 		}
1467 
1468 		if (page_referenced(page, 0, sc->mem_cgroup, &vm_flags)) {
1469 			nr_rotated++;
1470 			/*
1471 			 * Identify referenced, file-backed active pages and
1472 			 * give them one more trip around the active list. So
1473 			 * that executable code get better chances to stay in
1474 			 * memory under moderate memory pressure.  Anon pages
1475 			 * are not likely to be evicted by use-once streaming
1476 			 * IO, plus JVM can create lots of anon VM_EXEC pages,
1477 			 * so we ignore them here.
1478 			 */
1479 			if ((vm_flags & VM_EXEC) && page_is_file_cache(page)) {
1480 				list_add(&page->lru, &l_active);
1481 				continue;
1482 			}
1483 		}
1484 
1485 		ClearPageActive(page);	/* we are de-activating */
1486 		list_add(&page->lru, &l_inactive);
1487 	}
1488 
1489 	/*
1490 	 * Move pages back to the lru list.
1491 	 */
1492 	spin_lock_irq(&zone->lru_lock);
1493 	/*
1494 	 * Count referenced pages from currently used mappings as rotated,
1495 	 * even though only some of them are actually re-activated.  This
1496 	 * helps balance scan pressure between file and anonymous pages in
1497 	 * get_scan_ratio.
1498 	 */
1499 	reclaim_stat->recent_rotated[file] += nr_rotated;
1500 
1501 	move_active_pages_to_lru(zone, &l_active,
1502 						LRU_ACTIVE + file * LRU_FILE);
1503 	move_active_pages_to_lru(zone, &l_inactive,
1504 						LRU_BASE   + file * LRU_FILE);
1505 	__mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
1506 	spin_unlock_irq(&zone->lru_lock);
1507 }
1508 
1509 static int inactive_anon_is_low_global(struct zone *zone)
1510 {
1511 	unsigned long active, inactive;
1512 
1513 	active = zone_page_state(zone, NR_ACTIVE_ANON);
1514 	inactive = zone_page_state(zone, NR_INACTIVE_ANON);
1515 
1516 	if (inactive * zone->inactive_ratio < active)
1517 		return 1;
1518 
1519 	return 0;
1520 }
1521 
1522 /**
1523  * inactive_anon_is_low - check if anonymous pages need to be deactivated
1524  * @zone: zone to check
1525  * @sc:   scan control of this context
1526  *
1527  * Returns true if the zone does not have enough inactive anon pages,
1528  * meaning some active anon pages need to be deactivated.
1529  */
1530 static int inactive_anon_is_low(struct zone *zone, struct scan_control *sc)
1531 {
1532 	int low;
1533 
1534 	if (scanning_global_lru(sc))
1535 		low = inactive_anon_is_low_global(zone);
1536 	else
1537 		low = mem_cgroup_inactive_anon_is_low(sc->mem_cgroup);
1538 	return low;
1539 }
1540 
1541 static int inactive_file_is_low_global(struct zone *zone)
1542 {
1543 	unsigned long active, inactive;
1544 
1545 	active = zone_page_state(zone, NR_ACTIVE_FILE);
1546 	inactive = zone_page_state(zone, NR_INACTIVE_FILE);
1547 
1548 	return (active > inactive);
1549 }
1550 
1551 /**
1552  * inactive_file_is_low - check if file pages need to be deactivated
1553  * @zone: zone to check
1554  * @sc:   scan control of this context
1555  *
1556  * When the system is doing streaming IO, memory pressure here
1557  * ensures that active file pages get deactivated, until more
1558  * than half of the file pages are on the inactive list.
1559  *
1560  * Once we get to that situation, protect the system's working
1561  * set from being evicted by disabling active file page aging.
1562  *
1563  * This uses a different ratio than the anonymous pages, because
1564  * the page cache uses a use-once replacement algorithm.
1565  */
1566 static int inactive_file_is_low(struct zone *zone, struct scan_control *sc)
1567 {
1568 	int low;
1569 
1570 	if (scanning_global_lru(sc))
1571 		low = inactive_file_is_low_global(zone);
1572 	else
1573 		low = mem_cgroup_inactive_file_is_low(sc->mem_cgroup);
1574 	return low;
1575 }
1576 
1577 static int inactive_list_is_low(struct zone *zone, struct scan_control *sc,
1578 				int file)
1579 {
1580 	if (file)
1581 		return inactive_file_is_low(zone, sc);
1582 	else
1583 		return inactive_anon_is_low(zone, sc);
1584 }
1585 
1586 static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
1587 	struct zone *zone, struct scan_control *sc, int priority)
1588 {
1589 	int file = is_file_lru(lru);
1590 
1591 	if (is_active_lru(lru)) {
1592 		if (inactive_list_is_low(zone, sc, file))
1593 		    shrink_active_list(nr_to_scan, zone, sc, priority, file);
1594 		return 0;
1595 	}
1596 
1597 	return shrink_inactive_list(nr_to_scan, zone, sc, priority, file);
1598 }
1599 
1600 /*
1601  * Smallish @nr_to_scan's are deposited in @nr_saved_scan,
1602  * until we collected @swap_cluster_max pages to scan.
1603  */
1604 static unsigned long nr_scan_try_batch(unsigned long nr_to_scan,
1605 				       unsigned long *nr_saved_scan)
1606 {
1607 	unsigned long nr;
1608 
1609 	*nr_saved_scan += nr_to_scan;
1610 	nr = *nr_saved_scan;
1611 
1612 	if (nr >= SWAP_CLUSTER_MAX)
1613 		*nr_saved_scan = 0;
1614 	else
1615 		nr = 0;
1616 
1617 	return nr;
1618 }
1619 
1620 /*
1621  * Determine how aggressively the anon and file LRU lists should be
1622  * scanned.  The relative value of each set of LRU lists is determined
1623  * by looking at the fraction of the pages scanned we did rotate back
1624  * onto the active list instead of evict.
1625  *
1626  * nr[0] = anon pages to scan; nr[1] = file pages to scan
1627  */
1628 static void get_scan_count(struct zone *zone, struct scan_control *sc,
1629 					unsigned long *nr, int priority)
1630 {
1631 	unsigned long anon, file, free;
1632 	unsigned long anon_prio, file_prio;
1633 	unsigned long ap, fp;
1634 	struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(zone, sc);
1635 	u64 fraction[2], denominator;
1636 	enum lru_list l;
1637 	int noswap = 0;
1638 
1639 	/* If we have no swap space, do not bother scanning anon pages. */
1640 	if (!sc->may_swap || (nr_swap_pages <= 0)) {
1641 		noswap = 1;
1642 		fraction[0] = 0;
1643 		fraction[1] = 1;
1644 		denominator = 1;
1645 		goto out;
1646 	}
1647 
1648 	anon  = zone_nr_lru_pages(zone, sc, LRU_ACTIVE_ANON) +
1649 		zone_nr_lru_pages(zone, sc, LRU_INACTIVE_ANON);
1650 	file  = zone_nr_lru_pages(zone, sc, LRU_ACTIVE_FILE) +
1651 		zone_nr_lru_pages(zone, sc, LRU_INACTIVE_FILE);
1652 
1653 	if (scanning_global_lru(sc)) {
1654 		free  = zone_page_state(zone, NR_FREE_PAGES);
1655 		/* If we have very few page cache pages,
1656 		   force-scan anon pages. */
1657 		if (unlikely(file + free <= high_wmark_pages(zone))) {
1658 			fraction[0] = 1;
1659 			fraction[1] = 0;
1660 			denominator = 1;
1661 			goto out;
1662 		}
1663 	}
1664 
1665 	/*
1666 	 * With swappiness at 100, anonymous and file have the same priority.
1667 	 * This scanning priority is essentially the inverse of IO cost.
1668 	 */
1669 	anon_prio = sc->swappiness;
1670 	file_prio = 200 - sc->swappiness;
1671 
1672 	/*
1673 	 * OK, so we have swap space and a fair amount of page cache
1674 	 * pages.  We use the recently rotated / recently scanned
1675 	 * ratios to determine how valuable each cache is.
1676 	 *
1677 	 * Because workloads change over time (and to avoid overflow)
1678 	 * we keep these statistics as a floating average, which ends
1679 	 * up weighing recent references more than old ones.
1680 	 *
1681 	 * anon in [0], file in [1]
1682 	 */
1683 	spin_lock_irq(&zone->lru_lock);
1684 	if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
1685 		reclaim_stat->recent_scanned[0] /= 2;
1686 		reclaim_stat->recent_rotated[0] /= 2;
1687 	}
1688 
1689 	if (unlikely(reclaim_stat->recent_scanned[1] > file / 4)) {
1690 		reclaim_stat->recent_scanned[1] /= 2;
1691 		reclaim_stat->recent_rotated[1] /= 2;
1692 	}
1693 
1694 	/*
1695 	 * The amount of pressure on anon vs file pages is inversely
1696 	 * proportional to the fraction of recently scanned pages on
1697 	 * each list that were recently referenced and in active use.
1698 	 */
1699 	ap = (anon_prio + 1) * (reclaim_stat->recent_scanned[0] + 1);
1700 	ap /= reclaim_stat->recent_rotated[0] + 1;
1701 
1702 	fp = (file_prio + 1) * (reclaim_stat->recent_scanned[1] + 1);
1703 	fp /= reclaim_stat->recent_rotated[1] + 1;
1704 	spin_unlock_irq(&zone->lru_lock);
1705 
1706 	fraction[0] = ap;
1707 	fraction[1] = fp;
1708 	denominator = ap + fp + 1;
1709 out:
1710 	for_each_evictable_lru(l) {
1711 		int file = is_file_lru(l);
1712 		unsigned long scan;
1713 
1714 		scan = zone_nr_lru_pages(zone, sc, l);
1715 		if (priority || noswap) {
1716 			scan >>= priority;
1717 			scan = div64_u64(scan * fraction[file], denominator);
1718 		}
1719 		nr[l] = nr_scan_try_batch(scan,
1720 					  &reclaim_stat->nr_saved_scan[l]);
1721 	}
1722 }
1723 
1724 static void set_lumpy_reclaim_mode(int priority, struct scan_control *sc)
1725 {
1726 	/*
1727 	 * If we need a large contiguous chunk of memory, or have
1728 	 * trouble getting a small set of contiguous pages, we
1729 	 * will reclaim both active and inactive pages.
1730 	 */
1731 	if (sc->order > PAGE_ALLOC_COSTLY_ORDER)
1732 		sc->lumpy_reclaim_mode = 1;
1733 	else if (sc->order && priority < DEF_PRIORITY - 2)
1734 		sc->lumpy_reclaim_mode = 1;
1735 	else
1736 		sc->lumpy_reclaim_mode = 0;
1737 }
1738 
1739 /*
1740  * This is a basic per-zone page freer.  Used by both kswapd and direct reclaim.
1741  */
1742 static void shrink_zone(int priority, struct zone *zone,
1743 				struct scan_control *sc)
1744 {
1745 	unsigned long nr[NR_LRU_LISTS];
1746 	unsigned long nr_to_scan;
1747 	enum lru_list l;
1748 	unsigned long nr_reclaimed = sc->nr_reclaimed;
1749 	unsigned long nr_to_reclaim = sc->nr_to_reclaim;
1750 
1751 	get_scan_count(zone, sc, nr, priority);
1752 
1753 	set_lumpy_reclaim_mode(priority, sc);
1754 
1755 	while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
1756 					nr[LRU_INACTIVE_FILE]) {
1757 		for_each_evictable_lru(l) {
1758 			if (nr[l]) {
1759 				nr_to_scan = min_t(unsigned long,
1760 						   nr[l], SWAP_CLUSTER_MAX);
1761 				nr[l] -= nr_to_scan;
1762 
1763 				nr_reclaimed += shrink_list(l, nr_to_scan,
1764 							    zone, sc, priority);
1765 			}
1766 		}
1767 		/*
1768 		 * On large memory systems, scan >> priority can become
1769 		 * really large. This is fine for the starting priority;
1770 		 * we want to put equal scanning pressure on each zone.
1771 		 * However, if the VM has a harder time of freeing pages,
1772 		 * with multiple processes reclaiming pages, the total
1773 		 * freeing target can get unreasonably large.
1774 		 */
1775 		if (nr_reclaimed >= nr_to_reclaim && priority < DEF_PRIORITY)
1776 			break;
1777 	}
1778 
1779 	sc->nr_reclaimed = nr_reclaimed;
1780 
1781 	/*
1782 	 * Even if we did not try to evict anon pages at all, we want to
1783 	 * rebalance the anon lru active/inactive ratio.
1784 	 */
1785 	if (inactive_anon_is_low(zone, sc) && nr_swap_pages > 0)
1786 		shrink_active_list(SWAP_CLUSTER_MAX, zone, sc, priority, 0);
1787 
1788 	throttle_vm_writeout(sc->gfp_mask);
1789 }
1790 
1791 /*
1792  * This is the direct reclaim path, for page-allocating processes.  We only
1793  * try to reclaim pages from zones which will satisfy the caller's allocation
1794  * request.
1795  *
1796  * We reclaim from a zone even if that zone is over high_wmark_pages(zone).
1797  * Because:
1798  * a) The caller may be trying to free *extra* pages to satisfy a higher-order
1799  *    allocation or
1800  * b) The target zone may be at high_wmark_pages(zone) but the lower zones
1801  *    must go *over* high_wmark_pages(zone) to satisfy the `incremental min'
1802  *    zone defense algorithm.
1803  *
1804  * If a zone is deemed to be full of pinned pages then just give it a light
1805  * scan then give up on it.
1806  */
1807 static bool shrink_zones(int priority, struct zonelist *zonelist,
1808 					struct scan_control *sc)
1809 {
1810 	struct zoneref *z;
1811 	struct zone *zone;
1812 	bool all_unreclaimable = true;
1813 
1814 	for_each_zone_zonelist_nodemask(zone, z, zonelist,
1815 					gfp_zone(sc->gfp_mask), sc->nodemask) {
1816 		if (!populated_zone(zone))
1817 			continue;
1818 		/*
1819 		 * Take care memory controller reclaiming has small influence
1820 		 * to global LRU.
1821 		 */
1822 		if (scanning_global_lru(sc)) {
1823 			if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
1824 				continue;
1825 			if (zone->all_unreclaimable && priority != DEF_PRIORITY)
1826 				continue;	/* Let kswapd poll it */
1827 		}
1828 
1829 		shrink_zone(priority, zone, sc);
1830 		all_unreclaimable = false;
1831 	}
1832 	return all_unreclaimable;
1833 }
1834 
1835 /*
1836  * This is the main entry point to direct page reclaim.
1837  *
1838  * If a full scan of the inactive list fails to free enough memory then we
1839  * are "out of memory" and something needs to be killed.
1840  *
1841  * If the caller is !__GFP_FS then the probability of a failure is reasonably
1842  * high - the zone may be full of dirty or under-writeback pages, which this
1843  * caller can't do much about.  We kick the writeback threads and take explicit
1844  * naps in the hope that some of these pages can be written.  But if the
1845  * allocating task holds filesystem locks which prevent writeout this might not
1846  * work, and the allocation attempt will fail.
1847  *
1848  * returns:	0, if no pages reclaimed
1849  * 		else, the number of pages reclaimed
1850  */
1851 static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
1852 					struct scan_control *sc)
1853 {
1854 	int priority;
1855 	bool all_unreclaimable;
1856 	unsigned long total_scanned = 0;
1857 	struct reclaim_state *reclaim_state = current->reclaim_state;
1858 	struct zoneref *z;
1859 	struct zone *zone;
1860 	unsigned long writeback_threshold;
1861 
1862 	get_mems_allowed();
1863 	delayacct_freepages_start();
1864 
1865 	if (scanning_global_lru(sc))
1866 		count_vm_event(ALLOCSTALL);
1867 
1868 	for (priority = DEF_PRIORITY; priority >= 0; priority--) {
1869 		sc->nr_scanned = 0;
1870 		if (!priority)
1871 			disable_swap_token();
1872 		all_unreclaimable = shrink_zones(priority, zonelist, sc);
1873 		/*
1874 		 * Don't shrink slabs when reclaiming memory from
1875 		 * over limit cgroups
1876 		 */
1877 		if (scanning_global_lru(sc)) {
1878 			unsigned long lru_pages = 0;
1879 			for_each_zone_zonelist(zone, z, zonelist,
1880 					gfp_zone(sc->gfp_mask)) {
1881 				if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
1882 					continue;
1883 
1884 				lru_pages += zone_reclaimable_pages(zone);
1885 			}
1886 
1887 			shrink_slab(sc->nr_scanned, sc->gfp_mask, lru_pages);
1888 			if (reclaim_state) {
1889 				sc->nr_reclaimed += reclaim_state->reclaimed_slab;
1890 				reclaim_state->reclaimed_slab = 0;
1891 			}
1892 		}
1893 		total_scanned += sc->nr_scanned;
1894 		if (sc->nr_reclaimed >= sc->nr_to_reclaim)
1895 			goto out;
1896 
1897 		/*
1898 		 * Try to write back as many pages as we just scanned.  This
1899 		 * tends to cause slow streaming writers to write data to the
1900 		 * disk smoothly, at the dirtying rate, which is nice.   But
1901 		 * that's undesirable in laptop mode, where we *want* lumpy
1902 		 * writeout.  So in laptop mode, write out the whole world.
1903 		 */
1904 		writeback_threshold = sc->nr_to_reclaim + sc->nr_to_reclaim / 2;
1905 		if (total_scanned > writeback_threshold) {
1906 			wakeup_flusher_threads(laptop_mode ? 0 : total_scanned);
1907 			sc->may_writepage = 1;
1908 		}
1909 
1910 		/* Take a nap, wait for some writeback to complete */
1911 		if (!sc->hibernation_mode && sc->nr_scanned &&
1912 		    priority < DEF_PRIORITY - 2)
1913 			congestion_wait(BLK_RW_ASYNC, HZ/10);
1914 	}
1915 
1916 out:
1917 	/*
1918 	 * Now that we've scanned all the zones at this priority level, note
1919 	 * that level within the zone so that the next thread which performs
1920 	 * scanning of this zone will immediately start out at this priority
1921 	 * level.  This affects only the decision whether or not to bring
1922 	 * mapped pages onto the inactive list.
1923 	 */
1924 	if (priority < 0)
1925 		priority = 0;
1926 
1927 	delayacct_freepages_end();
1928 	put_mems_allowed();
1929 
1930 	if (sc->nr_reclaimed)
1931 		return sc->nr_reclaimed;
1932 
1933 	/* top priority shrink_zones still had more to do? don't OOM, then */
1934 	if (scanning_global_lru(sc) && !all_unreclaimable)
1935 		return 1;
1936 
1937 	return 0;
1938 }
1939 
1940 unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
1941 				gfp_t gfp_mask, nodemask_t *nodemask)
1942 {
1943 	unsigned long nr_reclaimed;
1944 	struct scan_control sc = {
1945 		.gfp_mask = gfp_mask,
1946 		.may_writepage = !laptop_mode,
1947 		.nr_to_reclaim = SWAP_CLUSTER_MAX,
1948 		.may_unmap = 1,
1949 		.may_swap = 1,
1950 		.swappiness = vm_swappiness,
1951 		.order = order,
1952 		.mem_cgroup = NULL,
1953 		.nodemask = nodemask,
1954 	};
1955 
1956 	trace_mm_vmscan_direct_reclaim_begin(order,
1957 				sc.may_writepage,
1958 				gfp_mask);
1959 
1960 	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
1961 
1962 	trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
1963 
1964 	return nr_reclaimed;
1965 }
1966 
1967 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
1968 
1969 unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *mem,
1970 						gfp_t gfp_mask, bool noswap,
1971 						unsigned int swappiness,
1972 						struct zone *zone, int nid)
1973 {
1974 	struct scan_control sc = {
1975 		.may_writepage = !laptop_mode,
1976 		.may_unmap = 1,
1977 		.may_swap = !noswap,
1978 		.swappiness = swappiness,
1979 		.order = 0,
1980 		.mem_cgroup = mem,
1981 	};
1982 	nodemask_t nm  = nodemask_of_node(nid);
1983 
1984 	sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
1985 			(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
1986 	sc.nodemask = &nm;
1987 	sc.nr_reclaimed = 0;
1988 	sc.nr_scanned = 0;
1989 
1990 	trace_mm_vmscan_memcg_softlimit_reclaim_begin(0,
1991 						      sc.may_writepage,
1992 						      sc.gfp_mask);
1993 
1994 	/*
1995 	 * NOTE: Although we can get the priority field, using it
1996 	 * here is not a good idea, since it limits the pages we can scan.
1997 	 * if we don't reclaim here, the shrink_zone from balance_pgdat
1998 	 * will pick up pages from other mem cgroup's as well. We hack
1999 	 * the priority and make it zero.
2000 	 */
2001 	shrink_zone(0, zone, &sc);
2002 
2003 	trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
2004 
2005 	return sc.nr_reclaimed;
2006 }
2007 
2008 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *mem_cont,
2009 					   gfp_t gfp_mask,
2010 					   bool noswap,
2011 					   unsigned int swappiness)
2012 {
2013 	struct zonelist *zonelist;
2014 	unsigned long nr_reclaimed;
2015 	struct scan_control sc = {
2016 		.may_writepage = !laptop_mode,
2017 		.may_unmap = 1,
2018 		.may_swap = !noswap,
2019 		.nr_to_reclaim = SWAP_CLUSTER_MAX,
2020 		.swappiness = swappiness,
2021 		.order = 0,
2022 		.mem_cgroup = mem_cont,
2023 		.nodemask = NULL, /* we don't care the placement */
2024 	};
2025 
2026 	sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
2027 			(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
2028 	zonelist = NODE_DATA(numa_node_id())->node_zonelists;
2029 
2030 	trace_mm_vmscan_memcg_reclaim_begin(0,
2031 					    sc.may_writepage,
2032 					    sc.gfp_mask);
2033 
2034 	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
2035 
2036 	trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
2037 
2038 	return nr_reclaimed;
2039 }
2040 #endif
2041 
2042 /* is kswapd sleeping prematurely? */
2043 static int sleeping_prematurely(pg_data_t *pgdat, int order, long remaining)
2044 {
2045 	int i;
2046 
2047 	/* If a direct reclaimer woke kswapd within HZ/10, it's premature */
2048 	if (remaining)
2049 		return 1;
2050 
2051 	/* If after HZ/10, a zone is below the high mark, it's premature */
2052 	for (i = 0; i < pgdat->nr_zones; i++) {
2053 		struct zone *zone = pgdat->node_zones + i;
2054 
2055 		if (!populated_zone(zone))
2056 			continue;
2057 
2058 		if (zone->all_unreclaimable)
2059 			continue;
2060 
2061 		if (!zone_watermark_ok(zone, order, high_wmark_pages(zone),
2062 								0, 0))
2063 			return 1;
2064 	}
2065 
2066 	return 0;
2067 }
2068 
2069 /*
2070  * For kswapd, balance_pgdat() will work across all this node's zones until
2071  * they are all at high_wmark_pages(zone).
2072  *
2073  * Returns the number of pages which were actually freed.
2074  *
2075  * There is special handling here for zones which are full of pinned pages.
2076  * This can happen if the pages are all mlocked, or if they are all used by
2077  * device drivers (say, ZONE_DMA).  Or if they are all in use by hugetlb.
2078  * What we do is to detect the case where all pages in the zone have been
2079  * scanned twice and there has been zero successful reclaim.  Mark the zone as
2080  * dead and from now on, only perform a short scan.  Basically we're polling
2081  * the zone for when the problem goes away.
2082  *
2083  * kswapd scans the zones in the highmem->normal->dma direction.  It skips
2084  * zones which have free_pages > high_wmark_pages(zone), but once a zone is
2085  * found to have free_pages <= high_wmark_pages(zone), we scan that zone and the
2086  * lower zones regardless of the number of free pages in the lower zones. This
2087  * interoperates with the page allocator fallback scheme to ensure that aging
2088  * of pages is balanced across the zones.
2089  */
2090 static unsigned long balance_pgdat(pg_data_t *pgdat, int order)
2091 {
2092 	int all_zones_ok;
2093 	int priority;
2094 	int i;
2095 	unsigned long total_scanned;
2096 	struct reclaim_state *reclaim_state = current->reclaim_state;
2097 	struct scan_control sc = {
2098 		.gfp_mask = GFP_KERNEL,
2099 		.may_unmap = 1,
2100 		.may_swap = 1,
2101 		/*
2102 		 * kswapd doesn't want to be bailed out while reclaim. because
2103 		 * we want to put equal scanning pressure on each zone.
2104 		 */
2105 		.nr_to_reclaim = ULONG_MAX,
2106 		.swappiness = vm_swappiness,
2107 		.order = order,
2108 		.mem_cgroup = NULL,
2109 	};
2110 loop_again:
2111 	total_scanned = 0;
2112 	sc.nr_reclaimed = 0;
2113 	sc.may_writepage = !laptop_mode;
2114 	count_vm_event(PAGEOUTRUN);
2115 
2116 	for (priority = DEF_PRIORITY; priority >= 0; priority--) {
2117 		int end_zone = 0;	/* Inclusive.  0 = ZONE_DMA */
2118 		unsigned long lru_pages = 0;
2119 		int has_under_min_watermark_zone = 0;
2120 
2121 		/* The swap token gets in the way of swapout... */
2122 		if (!priority)
2123 			disable_swap_token();
2124 
2125 		all_zones_ok = 1;
2126 
2127 		/*
2128 		 * Scan in the highmem->dma direction for the highest
2129 		 * zone which needs scanning
2130 		 */
2131 		for (i = pgdat->nr_zones - 1; i >= 0; i--) {
2132 			struct zone *zone = pgdat->node_zones + i;
2133 
2134 			if (!populated_zone(zone))
2135 				continue;
2136 
2137 			if (zone->all_unreclaimable && priority != DEF_PRIORITY)
2138 				continue;
2139 
2140 			/*
2141 			 * Do some background aging of the anon list, to give
2142 			 * pages a chance to be referenced before reclaiming.
2143 			 */
2144 			if (inactive_anon_is_low(zone, &sc))
2145 				shrink_active_list(SWAP_CLUSTER_MAX, zone,
2146 							&sc, priority, 0);
2147 
2148 			if (!zone_watermark_ok(zone, order,
2149 					high_wmark_pages(zone), 0, 0)) {
2150 				end_zone = i;
2151 				break;
2152 			}
2153 		}
2154 		if (i < 0)
2155 			goto out;
2156 
2157 		for (i = 0; i <= end_zone; i++) {
2158 			struct zone *zone = pgdat->node_zones + i;
2159 
2160 			lru_pages += zone_reclaimable_pages(zone);
2161 		}
2162 
2163 		/*
2164 		 * Now scan the zone in the dma->highmem direction, stopping
2165 		 * at the last zone which needs scanning.
2166 		 *
2167 		 * We do this because the page allocator works in the opposite
2168 		 * direction.  This prevents the page allocator from allocating
2169 		 * pages behind kswapd's direction of progress, which would
2170 		 * cause too much scanning of the lower zones.
2171 		 */
2172 		for (i = 0; i <= end_zone; i++) {
2173 			struct zone *zone = pgdat->node_zones + i;
2174 			int nr_slab;
2175 			int nid, zid;
2176 
2177 			if (!populated_zone(zone))
2178 				continue;
2179 
2180 			if (zone->all_unreclaimable && priority != DEF_PRIORITY)
2181 				continue;
2182 
2183 			sc.nr_scanned = 0;
2184 
2185 			nid = pgdat->node_id;
2186 			zid = zone_idx(zone);
2187 			/*
2188 			 * Call soft limit reclaim before calling shrink_zone.
2189 			 * For now we ignore the return value
2190 			 */
2191 			mem_cgroup_soft_limit_reclaim(zone, order, sc.gfp_mask,
2192 							nid, zid);
2193 			/*
2194 			 * We put equal pressure on every zone, unless one
2195 			 * zone has way too many pages free already.
2196 			 */
2197 			if (!zone_watermark_ok(zone, order,
2198 					8*high_wmark_pages(zone), end_zone, 0))
2199 				shrink_zone(priority, zone, &sc);
2200 			reclaim_state->reclaimed_slab = 0;
2201 			nr_slab = shrink_slab(sc.nr_scanned, GFP_KERNEL,
2202 						lru_pages);
2203 			sc.nr_reclaimed += reclaim_state->reclaimed_slab;
2204 			total_scanned += sc.nr_scanned;
2205 			if (zone->all_unreclaimable)
2206 				continue;
2207 			if (nr_slab == 0 &&
2208 			    zone->pages_scanned >= (zone_reclaimable_pages(zone) * 6))
2209 				zone->all_unreclaimable = 1;
2210 			/*
2211 			 * If we've done a decent amount of scanning and
2212 			 * the reclaim ratio is low, start doing writepage
2213 			 * even in laptop mode
2214 			 */
2215 			if (total_scanned > SWAP_CLUSTER_MAX * 2 &&
2216 			    total_scanned > sc.nr_reclaimed + sc.nr_reclaimed / 2)
2217 				sc.may_writepage = 1;
2218 
2219 			if (!zone_watermark_ok(zone, order,
2220 					high_wmark_pages(zone), end_zone, 0)) {
2221 				all_zones_ok = 0;
2222 				/*
2223 				 * We are still under min water mark.  This
2224 				 * means that we have a GFP_ATOMIC allocation
2225 				 * failure risk. Hurry up!
2226 				 */
2227 				if (!zone_watermark_ok(zone, order,
2228 					    min_wmark_pages(zone), end_zone, 0))
2229 					has_under_min_watermark_zone = 1;
2230 			}
2231 
2232 		}
2233 		if (all_zones_ok)
2234 			break;		/* kswapd: all done */
2235 		/*
2236 		 * OK, kswapd is getting into trouble.  Take a nap, then take
2237 		 * another pass across the zones.
2238 		 */
2239 		if (total_scanned && (priority < DEF_PRIORITY - 2)) {
2240 			if (has_under_min_watermark_zone)
2241 				count_vm_event(KSWAPD_SKIP_CONGESTION_WAIT);
2242 			else
2243 				congestion_wait(BLK_RW_ASYNC, HZ/10);
2244 		}
2245 
2246 		/*
2247 		 * We do this so kswapd doesn't build up large priorities for
2248 		 * example when it is freeing in parallel with allocators. It
2249 		 * matches the direct reclaim path behaviour in terms of impact
2250 		 * on zone->*_priority.
2251 		 */
2252 		if (sc.nr_reclaimed >= SWAP_CLUSTER_MAX)
2253 			break;
2254 	}
2255 out:
2256 	if (!all_zones_ok) {
2257 		cond_resched();
2258 
2259 		try_to_freeze();
2260 
2261 		/*
2262 		 * Fragmentation may mean that the system cannot be
2263 		 * rebalanced for high-order allocations in all zones.
2264 		 * At this point, if nr_reclaimed < SWAP_CLUSTER_MAX,
2265 		 * it means the zones have been fully scanned and are still
2266 		 * not balanced. For high-order allocations, there is
2267 		 * little point trying all over again as kswapd may
2268 		 * infinite loop.
2269 		 *
2270 		 * Instead, recheck all watermarks at order-0 as they
2271 		 * are the most important. If watermarks are ok, kswapd will go
2272 		 * back to sleep. High-order users can still perform direct
2273 		 * reclaim if they wish.
2274 		 */
2275 		if (sc.nr_reclaimed < SWAP_CLUSTER_MAX)
2276 			order = sc.order = 0;
2277 
2278 		goto loop_again;
2279 	}
2280 
2281 	return sc.nr_reclaimed;
2282 }
2283 
2284 /*
2285  * The background pageout daemon, started as a kernel thread
2286  * from the init process.
2287  *
2288  * This basically trickles out pages so that we have _some_
2289  * free memory available even if there is no other activity
2290  * that frees anything up. This is needed for things like routing
2291  * etc, where we otherwise might have all activity going on in
2292  * asynchronous contexts that cannot page things out.
2293  *
2294  * If there are applications that are active memory-allocators
2295  * (most normal use), this basically shouldn't matter.
2296  */
2297 static int kswapd(void *p)
2298 {
2299 	unsigned long order;
2300 	pg_data_t *pgdat = (pg_data_t*)p;
2301 	struct task_struct *tsk = current;
2302 	DEFINE_WAIT(wait);
2303 	struct reclaim_state reclaim_state = {
2304 		.reclaimed_slab = 0,
2305 	};
2306 	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
2307 
2308 	lockdep_set_current_reclaim_state(GFP_KERNEL);
2309 
2310 	if (!cpumask_empty(cpumask))
2311 		set_cpus_allowed_ptr(tsk, cpumask);
2312 	current->reclaim_state = &reclaim_state;
2313 
2314 	/*
2315 	 * Tell the memory management that we're a "memory allocator",
2316 	 * and that if we need more memory we should get access to it
2317 	 * regardless (see "__alloc_pages()"). "kswapd" should
2318 	 * never get caught in the normal page freeing logic.
2319 	 *
2320 	 * (Kswapd normally doesn't need memory anyway, but sometimes
2321 	 * you need a small amount of memory in order to be able to
2322 	 * page out something else, and this flag essentially protects
2323 	 * us from recursively trying to free more memory as we're
2324 	 * trying to free the first piece of memory in the first place).
2325 	 */
2326 	tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD;
2327 	set_freezable();
2328 
2329 	order = 0;
2330 	for ( ; ; ) {
2331 		unsigned long new_order;
2332 		int ret;
2333 
2334 		prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
2335 		new_order = pgdat->kswapd_max_order;
2336 		pgdat->kswapd_max_order = 0;
2337 		if (order < new_order) {
2338 			/*
2339 			 * Don't sleep if someone wants a larger 'order'
2340 			 * allocation
2341 			 */
2342 			order = new_order;
2343 		} else {
2344 			if (!freezing(current) && !kthread_should_stop()) {
2345 				long remaining = 0;
2346 
2347 				/* Try to sleep for a short interval */
2348 				if (!sleeping_prematurely(pgdat, order, remaining)) {
2349 					remaining = schedule_timeout(HZ/10);
2350 					finish_wait(&pgdat->kswapd_wait, &wait);
2351 					prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
2352 				}
2353 
2354 				/*
2355 				 * After a short sleep, check if it was a
2356 				 * premature sleep. If not, then go fully
2357 				 * to sleep until explicitly woken up
2358 				 */
2359 				if (!sleeping_prematurely(pgdat, order, remaining)) {
2360 					trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
2361 					schedule();
2362 				} else {
2363 					if (remaining)
2364 						count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
2365 					else
2366 						count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
2367 				}
2368 			}
2369 
2370 			order = pgdat->kswapd_max_order;
2371 		}
2372 		finish_wait(&pgdat->kswapd_wait, &wait);
2373 
2374 		ret = try_to_freeze();
2375 		if (kthread_should_stop())
2376 			break;
2377 
2378 		/*
2379 		 * We can speed up thawing tasks if we don't call balance_pgdat
2380 		 * after returning from the refrigerator
2381 		 */
2382 		if (!ret) {
2383 			trace_mm_vmscan_kswapd_wake(pgdat->node_id, order);
2384 			balance_pgdat(pgdat, order);
2385 		}
2386 	}
2387 	return 0;
2388 }
2389 
2390 /*
2391  * A zone is low on free memory, so wake its kswapd task to service it.
2392  */
2393 void wakeup_kswapd(struct zone *zone, int order)
2394 {
2395 	pg_data_t *pgdat;
2396 
2397 	if (!populated_zone(zone))
2398 		return;
2399 
2400 	pgdat = zone->zone_pgdat;
2401 	if (zone_watermark_ok(zone, order, low_wmark_pages(zone), 0, 0))
2402 		return;
2403 	if (pgdat->kswapd_max_order < order)
2404 		pgdat->kswapd_max_order = order;
2405 	trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, zone_idx(zone), order);
2406 	if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
2407 		return;
2408 	if (!waitqueue_active(&pgdat->kswapd_wait))
2409 		return;
2410 	wake_up_interruptible(&pgdat->kswapd_wait);
2411 }
2412 
2413 /*
2414  * The reclaimable count would be mostly accurate.
2415  * The less reclaimable pages may be
2416  * - mlocked pages, which will be moved to unevictable list when encountered
2417  * - mapped pages, which may require several travels to be reclaimed
2418  * - dirty pages, which is not "instantly" reclaimable
2419  */
2420 unsigned long global_reclaimable_pages(void)
2421 {
2422 	int nr;
2423 
2424 	nr = global_page_state(NR_ACTIVE_FILE) +
2425 	     global_page_state(NR_INACTIVE_FILE);
2426 
2427 	if (nr_swap_pages > 0)
2428 		nr += global_page_state(NR_ACTIVE_ANON) +
2429 		      global_page_state(NR_INACTIVE_ANON);
2430 
2431 	return nr;
2432 }
2433 
2434 unsigned long zone_reclaimable_pages(struct zone *zone)
2435 {
2436 	int nr;
2437 
2438 	nr = zone_page_state(zone, NR_ACTIVE_FILE) +
2439 	     zone_page_state(zone, NR_INACTIVE_FILE);
2440 
2441 	if (nr_swap_pages > 0)
2442 		nr += zone_page_state(zone, NR_ACTIVE_ANON) +
2443 		      zone_page_state(zone, NR_INACTIVE_ANON);
2444 
2445 	return nr;
2446 }
2447 
2448 #ifdef CONFIG_HIBERNATION
2449 /*
2450  * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
2451  * freed pages.
2452  *
2453  * Rather than trying to age LRUs the aim is to preserve the overall
2454  * LRU order by reclaiming preferentially
2455  * inactive > active > active referenced > active mapped
2456  */
2457 unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
2458 {
2459 	struct reclaim_state reclaim_state;
2460 	struct scan_control sc = {
2461 		.gfp_mask = GFP_HIGHUSER_MOVABLE,
2462 		.may_swap = 1,
2463 		.may_unmap = 1,
2464 		.may_writepage = 1,
2465 		.nr_to_reclaim = nr_to_reclaim,
2466 		.hibernation_mode = 1,
2467 		.swappiness = vm_swappiness,
2468 		.order = 0,
2469 	};
2470 	struct zonelist * zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
2471 	struct task_struct *p = current;
2472 	unsigned long nr_reclaimed;
2473 
2474 	p->flags |= PF_MEMALLOC;
2475 	lockdep_set_current_reclaim_state(sc.gfp_mask);
2476 	reclaim_state.reclaimed_slab = 0;
2477 	p->reclaim_state = &reclaim_state;
2478 
2479 	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
2480 
2481 	p->reclaim_state = NULL;
2482 	lockdep_clear_current_reclaim_state();
2483 	p->flags &= ~PF_MEMALLOC;
2484 
2485 	return nr_reclaimed;
2486 }
2487 #endif /* CONFIG_HIBERNATION */
2488 
2489 /* It's optimal to keep kswapds on the same CPUs as their memory, but
2490    not required for correctness.  So if the last cpu in a node goes
2491    away, we get changed to run anywhere: as the first one comes back,
2492    restore their cpu bindings. */
2493 static int __devinit cpu_callback(struct notifier_block *nfb,
2494 				  unsigned long action, void *hcpu)
2495 {
2496 	int nid;
2497 
2498 	if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) {
2499 		for_each_node_state(nid, N_HIGH_MEMORY) {
2500 			pg_data_t *pgdat = NODE_DATA(nid);
2501 			const struct cpumask *mask;
2502 
2503 			mask = cpumask_of_node(pgdat->node_id);
2504 
2505 			if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
2506 				/* One of our CPUs online: restore mask */
2507 				set_cpus_allowed_ptr(pgdat->kswapd, mask);
2508 		}
2509 	}
2510 	return NOTIFY_OK;
2511 }
2512 
2513 /*
2514  * This kswapd start function will be called by init and node-hot-add.
2515  * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
2516  */
2517 int kswapd_run(int nid)
2518 {
2519 	pg_data_t *pgdat = NODE_DATA(nid);
2520 	int ret = 0;
2521 
2522 	if (pgdat->kswapd)
2523 		return 0;
2524 
2525 	pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
2526 	if (IS_ERR(pgdat->kswapd)) {
2527 		/* failure at boot is fatal */
2528 		BUG_ON(system_state == SYSTEM_BOOTING);
2529 		printk("Failed to start kswapd on node %d\n",nid);
2530 		ret = -1;
2531 	}
2532 	return ret;
2533 }
2534 
2535 /*
2536  * Called by memory hotplug when all memory in a node is offlined.
2537  */
2538 void kswapd_stop(int nid)
2539 {
2540 	struct task_struct *kswapd = NODE_DATA(nid)->kswapd;
2541 
2542 	if (kswapd)
2543 		kthread_stop(kswapd);
2544 }
2545 
2546 static int __init kswapd_init(void)
2547 {
2548 	int nid;
2549 
2550 	swap_setup();
2551 	for_each_node_state(nid, N_HIGH_MEMORY)
2552  		kswapd_run(nid);
2553 	hotcpu_notifier(cpu_callback, 0);
2554 	return 0;
2555 }
2556 
2557 module_init(kswapd_init)
2558 
2559 #ifdef CONFIG_NUMA
2560 /*
2561  * Zone reclaim mode
2562  *
2563  * If non-zero call zone_reclaim when the number of free pages falls below
2564  * the watermarks.
2565  */
2566 int zone_reclaim_mode __read_mostly;
2567 
2568 #define RECLAIM_OFF 0
2569 #define RECLAIM_ZONE (1<<0)	/* Run shrink_inactive_list on the zone */
2570 #define RECLAIM_WRITE (1<<1)	/* Writeout pages during reclaim */
2571 #define RECLAIM_SWAP (1<<2)	/* Swap pages out during reclaim */
2572 
2573 /*
2574  * Priority for ZONE_RECLAIM. This determines the fraction of pages
2575  * of a node considered for each zone_reclaim. 4 scans 1/16th of
2576  * a zone.
2577  */
2578 #define ZONE_RECLAIM_PRIORITY 4
2579 
2580 /*
2581  * Percentage of pages in a zone that must be unmapped for zone_reclaim to
2582  * occur.
2583  */
2584 int sysctl_min_unmapped_ratio = 1;
2585 
2586 /*
2587  * If the number of slab pages in a zone grows beyond this percentage then
2588  * slab reclaim needs to occur.
2589  */
2590 int sysctl_min_slab_ratio = 5;
2591 
2592 static inline unsigned long zone_unmapped_file_pages(struct zone *zone)
2593 {
2594 	unsigned long file_mapped = zone_page_state(zone, NR_FILE_MAPPED);
2595 	unsigned long file_lru = zone_page_state(zone, NR_INACTIVE_FILE) +
2596 		zone_page_state(zone, NR_ACTIVE_FILE);
2597 
2598 	/*
2599 	 * It's possible for there to be more file mapped pages than
2600 	 * accounted for by the pages on the file LRU lists because
2601 	 * tmpfs pages accounted for as ANON can also be FILE_MAPPED
2602 	 */
2603 	return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
2604 }
2605 
2606 /* Work out how many page cache pages we can reclaim in this reclaim_mode */
2607 static long zone_pagecache_reclaimable(struct zone *zone)
2608 {
2609 	long nr_pagecache_reclaimable;
2610 	long delta = 0;
2611 
2612 	/*
2613 	 * If RECLAIM_SWAP is set, then all file pages are considered
2614 	 * potentially reclaimable. Otherwise, we have to worry about
2615 	 * pages like swapcache and zone_unmapped_file_pages() provides
2616 	 * a better estimate
2617 	 */
2618 	if (zone_reclaim_mode & RECLAIM_SWAP)
2619 		nr_pagecache_reclaimable = zone_page_state(zone, NR_FILE_PAGES);
2620 	else
2621 		nr_pagecache_reclaimable = zone_unmapped_file_pages(zone);
2622 
2623 	/* If we can't clean pages, remove dirty pages from consideration */
2624 	if (!(zone_reclaim_mode & RECLAIM_WRITE))
2625 		delta += zone_page_state(zone, NR_FILE_DIRTY);
2626 
2627 	/* Watch for any possible underflows due to delta */
2628 	if (unlikely(delta > nr_pagecache_reclaimable))
2629 		delta = nr_pagecache_reclaimable;
2630 
2631 	return nr_pagecache_reclaimable - delta;
2632 }
2633 
2634 /*
2635  * Try to free up some pages from this zone through reclaim.
2636  */
2637 static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
2638 {
2639 	/* Minimum pages needed in order to stay on node */
2640 	const unsigned long nr_pages = 1 << order;
2641 	struct task_struct *p = current;
2642 	struct reclaim_state reclaim_state;
2643 	int priority;
2644 	struct scan_control sc = {
2645 		.may_writepage = !!(zone_reclaim_mode & RECLAIM_WRITE),
2646 		.may_unmap = !!(zone_reclaim_mode & RECLAIM_SWAP),
2647 		.may_swap = 1,
2648 		.nr_to_reclaim = max_t(unsigned long, nr_pages,
2649 				       SWAP_CLUSTER_MAX),
2650 		.gfp_mask = gfp_mask,
2651 		.swappiness = vm_swappiness,
2652 		.order = order,
2653 	};
2654 	unsigned long nr_slab_pages0, nr_slab_pages1;
2655 
2656 	cond_resched();
2657 	/*
2658 	 * We need to be able to allocate from the reserves for RECLAIM_SWAP
2659 	 * and we also need to be able to write out pages for RECLAIM_WRITE
2660 	 * and RECLAIM_SWAP.
2661 	 */
2662 	p->flags |= PF_MEMALLOC | PF_SWAPWRITE;
2663 	lockdep_set_current_reclaim_state(gfp_mask);
2664 	reclaim_state.reclaimed_slab = 0;
2665 	p->reclaim_state = &reclaim_state;
2666 
2667 	if (zone_pagecache_reclaimable(zone) > zone->min_unmapped_pages) {
2668 		/*
2669 		 * Free memory by calling shrink zone with increasing
2670 		 * priorities until we have enough memory freed.
2671 		 */
2672 		priority = ZONE_RECLAIM_PRIORITY;
2673 		do {
2674 			shrink_zone(priority, zone, &sc);
2675 			priority--;
2676 		} while (priority >= 0 && sc.nr_reclaimed < nr_pages);
2677 	}
2678 
2679 	nr_slab_pages0 = zone_page_state(zone, NR_SLAB_RECLAIMABLE);
2680 	if (nr_slab_pages0 > zone->min_slab_pages) {
2681 		/*
2682 		 * shrink_slab() does not currently allow us to determine how
2683 		 * many pages were freed in this zone. So we take the current
2684 		 * number of slab pages and shake the slab until it is reduced
2685 		 * by the same nr_pages that we used for reclaiming unmapped
2686 		 * pages.
2687 		 *
2688 		 * Note that shrink_slab will free memory on all zones and may
2689 		 * take a long time.
2690 		 */
2691 		for (;;) {
2692 			unsigned long lru_pages = zone_reclaimable_pages(zone);
2693 
2694 			/* No reclaimable slab or very low memory pressure */
2695 			if (!shrink_slab(sc.nr_scanned, gfp_mask, lru_pages))
2696 				break;
2697 
2698 			/* Freed enough memory */
2699 			nr_slab_pages1 = zone_page_state(zone,
2700 							NR_SLAB_RECLAIMABLE);
2701 			if (nr_slab_pages1 + nr_pages <= nr_slab_pages0)
2702 				break;
2703 		}
2704 
2705 		/*
2706 		 * Update nr_reclaimed by the number of slab pages we
2707 		 * reclaimed from this zone.
2708 		 */
2709 		nr_slab_pages1 = zone_page_state(zone, NR_SLAB_RECLAIMABLE);
2710 		if (nr_slab_pages1 < nr_slab_pages0)
2711 			sc.nr_reclaimed += nr_slab_pages0 - nr_slab_pages1;
2712 	}
2713 
2714 	p->reclaim_state = NULL;
2715 	current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE);
2716 	lockdep_clear_current_reclaim_state();
2717 	return sc.nr_reclaimed >= nr_pages;
2718 }
2719 
2720 int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
2721 {
2722 	int node_id;
2723 	int ret;
2724 
2725 	/*
2726 	 * Zone reclaim reclaims unmapped file backed pages and
2727 	 * slab pages if we are over the defined limits.
2728 	 *
2729 	 * A small portion of unmapped file backed pages is needed for
2730 	 * file I/O otherwise pages read by file I/O will be immediately
2731 	 * thrown out if the zone is overallocated. So we do not reclaim
2732 	 * if less than a specified percentage of the zone is used by
2733 	 * unmapped file backed pages.
2734 	 */
2735 	if (zone_pagecache_reclaimable(zone) <= zone->min_unmapped_pages &&
2736 	    zone_page_state(zone, NR_SLAB_RECLAIMABLE) <= zone->min_slab_pages)
2737 		return ZONE_RECLAIM_FULL;
2738 
2739 	if (zone->all_unreclaimable)
2740 		return ZONE_RECLAIM_FULL;
2741 
2742 	/*
2743 	 * Do not scan if the allocation should not be delayed.
2744 	 */
2745 	if (!(gfp_mask & __GFP_WAIT) || (current->flags & PF_MEMALLOC))
2746 		return ZONE_RECLAIM_NOSCAN;
2747 
2748 	/*
2749 	 * Only run zone reclaim on the local zone or on zones that do not
2750 	 * have associated processors. This will favor the local processor
2751 	 * over remote processors and spread off node memory allocations
2752 	 * as wide as possible.
2753 	 */
2754 	node_id = zone_to_nid(zone);
2755 	if (node_state(node_id, N_CPU) && node_id != numa_node_id())
2756 		return ZONE_RECLAIM_NOSCAN;
2757 
2758 	if (zone_test_and_set_flag(zone, ZONE_RECLAIM_LOCKED))
2759 		return ZONE_RECLAIM_NOSCAN;
2760 
2761 	ret = __zone_reclaim(zone, gfp_mask, order);
2762 	zone_clear_flag(zone, ZONE_RECLAIM_LOCKED);
2763 
2764 	if (!ret)
2765 		count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
2766 
2767 	return ret;
2768 }
2769 #endif
2770 
2771 /*
2772  * page_evictable - test whether a page is evictable
2773  * @page: the page to test
2774  * @vma: the VMA in which the page is or will be mapped, may be NULL
2775  *
2776  * Test whether page is evictable--i.e., should be placed on active/inactive
2777  * lists vs unevictable list.  The vma argument is !NULL when called from the
2778  * fault path to determine how to instantate a new page.
2779  *
2780  * Reasons page might not be evictable:
2781  * (1) page's mapping marked unevictable
2782  * (2) page is part of an mlocked VMA
2783  *
2784  */
2785 int page_evictable(struct page *page, struct vm_area_struct *vma)
2786 {
2787 
2788 	if (mapping_unevictable(page_mapping(page)))
2789 		return 0;
2790 
2791 	if (PageMlocked(page) || (vma && is_mlocked_vma(vma, page)))
2792 		return 0;
2793 
2794 	return 1;
2795 }
2796 
2797 /**
2798  * check_move_unevictable_page - check page for evictability and move to appropriate zone lru list
2799  * @page: page to check evictability and move to appropriate lru list
2800  * @zone: zone page is in
2801  *
2802  * Checks a page for evictability and moves the page to the appropriate
2803  * zone lru list.
2804  *
2805  * Restrictions: zone->lru_lock must be held, page must be on LRU and must
2806  * have PageUnevictable set.
2807  */
2808 static void check_move_unevictable_page(struct page *page, struct zone *zone)
2809 {
2810 	VM_BUG_ON(PageActive(page));
2811 
2812 retry:
2813 	ClearPageUnevictable(page);
2814 	if (page_evictable(page, NULL)) {
2815 		enum lru_list l = page_lru_base_type(page);
2816 
2817 		__dec_zone_state(zone, NR_UNEVICTABLE);
2818 		list_move(&page->lru, &zone->lru[l].list);
2819 		mem_cgroup_move_lists(page, LRU_UNEVICTABLE, l);
2820 		__inc_zone_state(zone, NR_INACTIVE_ANON + l);
2821 		__count_vm_event(UNEVICTABLE_PGRESCUED);
2822 	} else {
2823 		/*
2824 		 * rotate unevictable list
2825 		 */
2826 		SetPageUnevictable(page);
2827 		list_move(&page->lru, &zone->lru[LRU_UNEVICTABLE].list);
2828 		mem_cgroup_rotate_lru_list(page, LRU_UNEVICTABLE);
2829 		if (page_evictable(page, NULL))
2830 			goto retry;
2831 	}
2832 }
2833 
2834 /**
2835  * scan_mapping_unevictable_pages - scan an address space for evictable pages
2836  * @mapping: struct address_space to scan for evictable pages
2837  *
2838  * Scan all pages in mapping.  Check unevictable pages for
2839  * evictability and move them to the appropriate zone lru list.
2840  */
2841 void scan_mapping_unevictable_pages(struct address_space *mapping)
2842 {
2843 	pgoff_t next = 0;
2844 	pgoff_t end   = (i_size_read(mapping->host) + PAGE_CACHE_SIZE - 1) >>
2845 			 PAGE_CACHE_SHIFT;
2846 	struct zone *zone;
2847 	struct pagevec pvec;
2848 
2849 	if (mapping->nrpages == 0)
2850 		return;
2851 
2852 	pagevec_init(&pvec, 0);
2853 	while (next < end &&
2854 		pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
2855 		int i;
2856 		int pg_scanned = 0;
2857 
2858 		zone = NULL;
2859 
2860 		for (i = 0; i < pagevec_count(&pvec); i++) {
2861 			struct page *page = pvec.pages[i];
2862 			pgoff_t page_index = page->index;
2863 			struct zone *pagezone = page_zone(page);
2864 
2865 			pg_scanned++;
2866 			if (page_index > next)
2867 				next = page_index;
2868 			next++;
2869 
2870 			if (pagezone != zone) {
2871 				if (zone)
2872 					spin_unlock_irq(&zone->lru_lock);
2873 				zone = pagezone;
2874 				spin_lock_irq(&zone->lru_lock);
2875 			}
2876 
2877 			if (PageLRU(page) && PageUnevictable(page))
2878 				check_move_unevictable_page(page, zone);
2879 		}
2880 		if (zone)
2881 			spin_unlock_irq(&zone->lru_lock);
2882 		pagevec_release(&pvec);
2883 
2884 		count_vm_events(UNEVICTABLE_PGSCANNED, pg_scanned);
2885 	}
2886 
2887 }
2888 
2889 /**
2890  * scan_zone_unevictable_pages - check unevictable list for evictable pages
2891  * @zone - zone of which to scan the unevictable list
2892  *
2893  * Scan @zone's unevictable LRU lists to check for pages that have become
2894  * evictable.  Move those that have to @zone's inactive list where they
2895  * become candidates for reclaim, unless shrink_inactive_zone() decides
2896  * to reactivate them.  Pages that are still unevictable are rotated
2897  * back onto @zone's unevictable list.
2898  */
2899 #define SCAN_UNEVICTABLE_BATCH_SIZE 16UL /* arbitrary lock hold batch size */
2900 static void scan_zone_unevictable_pages(struct zone *zone)
2901 {
2902 	struct list_head *l_unevictable = &zone->lru[LRU_UNEVICTABLE].list;
2903 	unsigned long scan;
2904 	unsigned long nr_to_scan = zone_page_state(zone, NR_UNEVICTABLE);
2905 
2906 	while (nr_to_scan > 0) {
2907 		unsigned long batch_size = min(nr_to_scan,
2908 						SCAN_UNEVICTABLE_BATCH_SIZE);
2909 
2910 		spin_lock_irq(&zone->lru_lock);
2911 		for (scan = 0;  scan < batch_size; scan++) {
2912 			struct page *page = lru_to_page(l_unevictable);
2913 
2914 			if (!trylock_page(page))
2915 				continue;
2916 
2917 			prefetchw_prev_lru_page(page, l_unevictable, flags);
2918 
2919 			if (likely(PageLRU(page) && PageUnevictable(page)))
2920 				check_move_unevictable_page(page, zone);
2921 
2922 			unlock_page(page);
2923 		}
2924 		spin_unlock_irq(&zone->lru_lock);
2925 
2926 		nr_to_scan -= batch_size;
2927 	}
2928 }
2929 
2930 
2931 /**
2932  * scan_all_zones_unevictable_pages - scan all unevictable lists for evictable pages
2933  *
2934  * A really big hammer:  scan all zones' unevictable LRU lists to check for
2935  * pages that have become evictable.  Move those back to the zones'
2936  * inactive list where they become candidates for reclaim.
2937  * This occurs when, e.g., we have unswappable pages on the unevictable lists,
2938  * and we add swap to the system.  As such, it runs in the context of a task
2939  * that has possibly/probably made some previously unevictable pages
2940  * evictable.
2941  */
2942 static void scan_all_zones_unevictable_pages(void)
2943 {
2944 	struct zone *zone;
2945 
2946 	for_each_zone(zone) {
2947 		scan_zone_unevictable_pages(zone);
2948 	}
2949 }
2950 
2951 /*
2952  * scan_unevictable_pages [vm] sysctl handler.  On demand re-scan of
2953  * all nodes' unevictable lists for evictable pages
2954  */
2955 unsigned long scan_unevictable_pages;
2956 
2957 int scan_unevictable_handler(struct ctl_table *table, int write,
2958 			   void __user *buffer,
2959 			   size_t *length, loff_t *ppos)
2960 {
2961 	proc_doulongvec_minmax(table, write, buffer, length, ppos);
2962 
2963 	if (write && *(unsigned long *)table->data)
2964 		scan_all_zones_unevictable_pages();
2965 
2966 	scan_unevictable_pages = 0;
2967 	return 0;
2968 }
2969 
2970 /*
2971  * per node 'scan_unevictable_pages' attribute.  On demand re-scan of
2972  * a specified node's per zone unevictable lists for evictable pages.
2973  */
2974 
2975 static ssize_t read_scan_unevictable_node(struct sys_device *dev,
2976 					  struct sysdev_attribute *attr,
2977 					  char *buf)
2978 {
2979 	return sprintf(buf, "0\n");	/* always zero; should fit... */
2980 }
2981 
2982 static ssize_t write_scan_unevictable_node(struct sys_device *dev,
2983 					   struct sysdev_attribute *attr,
2984 					const char *buf, size_t count)
2985 {
2986 	struct zone *node_zones = NODE_DATA(dev->id)->node_zones;
2987 	struct zone *zone;
2988 	unsigned long res;
2989 	unsigned long req = strict_strtoul(buf, 10, &res);
2990 
2991 	if (!req)
2992 		return 1;	/* zero is no-op */
2993 
2994 	for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
2995 		if (!populated_zone(zone))
2996 			continue;
2997 		scan_zone_unevictable_pages(zone);
2998 	}
2999 	return 1;
3000 }
3001 
3002 
3003 static SYSDEV_ATTR(scan_unevictable_pages, S_IRUGO | S_IWUSR,
3004 			read_scan_unevictable_node,
3005 			write_scan_unevictable_node);
3006 
3007 int scan_unevictable_register_node(struct node *node)
3008 {
3009 	return sysdev_create_file(&node->sysdev, &attr_scan_unevictable_pages);
3010 }
3011 
3012 void scan_unevictable_unregister_node(struct node *node)
3013 {
3014 	sysdev_remove_file(&node->sysdev, &attr_scan_unevictable_pages);
3015 }
3016 
3017