xref: /openbmc/linux/mm/vmscan.c (revision d5cb9783536a41df9f9cba5b0a1d78047ed787f7)
1 /*
2  *  linux/mm/vmscan.c
3  *
4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
5  *
6  *  Swap reorganised 29.12.95, Stephen Tweedie.
7  *  kswapd added: 7.1.96  sct
8  *  Removed kswapd_ctl limits, and swap out as many pages as needed
9  *  to bring the system back to freepages.high: 2.4.97, Rik van Riel.
10  *  Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
11  *  Multiqueue VM started 5.8.00, Rik van Riel.
12  */
13 
14 #include <linux/mm.h>
15 #include <linux/module.h>
16 #include <linux/slab.h>
17 #include <linux/kernel_stat.h>
18 #include <linux/swap.h>
19 #include <linux/pagemap.h>
20 #include <linux/init.h>
21 #include <linux/highmem.h>
22 #include <linux/file.h>
23 #include <linux/writeback.h>
24 #include <linux/blkdev.h>
25 #include <linux/buffer_head.h>	/* for try_to_release_page(),
26 					buffer_heads_over_limit */
27 #include <linux/mm_inline.h>
28 #include <linux/pagevec.h>
29 #include <linux/backing-dev.h>
30 #include <linux/rmap.h>
31 #include <linux/topology.h>
32 #include <linux/cpu.h>
33 #include <linux/cpuset.h>
34 #include <linux/notifier.h>
35 #include <linux/rwsem.h>
36 
37 #include <asm/tlbflush.h>
38 #include <asm/div64.h>
39 
40 #include <linux/swapops.h>
41 
42 /* possible outcome of pageout() */
43 typedef enum {
44 	/* failed to write page out, page is locked */
45 	PAGE_KEEP,
46 	/* move page to the active list, page is locked */
47 	PAGE_ACTIVATE,
48 	/* page has been sent to the disk successfully, page is unlocked */
49 	PAGE_SUCCESS,
50 	/* page is clean and locked */
51 	PAGE_CLEAN,
52 } pageout_t;
53 
54 struct scan_control {
55 	/* Ask refill_inactive_zone, or shrink_cache to scan this many pages */
56 	unsigned long nr_to_scan;
57 
58 	/* Incremented by the number of inactive pages that were scanned */
59 	unsigned long nr_scanned;
60 
61 	/* Incremented by the number of pages reclaimed */
62 	unsigned long nr_reclaimed;
63 
64 	unsigned long nr_mapped;	/* From page_state */
65 
66 	/* How many pages shrink_cache() should reclaim */
67 	int nr_to_reclaim;
68 
69 	/* Ask shrink_caches, or shrink_zone to scan at this priority */
70 	unsigned int priority;
71 
72 	/* This context's GFP mask */
73 	gfp_t gfp_mask;
74 
75 	int may_writepage;
76 
77 	/* Can pages be swapped as part of reclaim? */
78 	int may_swap;
79 
80 	/* This context's SWAP_CLUSTER_MAX. If freeing memory for
81 	 * suspend, we effectively ignore SWAP_CLUSTER_MAX.
82 	 * In this context, it doesn't matter that we scan the
83 	 * whole list at once. */
84 	int swap_cluster_max;
85 };
86 
87 /*
88  * The list of shrinker callbacks used by to apply pressure to
89  * ageable caches.
90  */
91 struct shrinker {
92 	shrinker_t		shrinker;
93 	struct list_head	list;
94 	int			seeks;	/* seeks to recreate an obj */
95 	long			nr;	/* objs pending delete */
96 };
97 
98 #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
99 
100 #ifdef ARCH_HAS_PREFETCH
101 #define prefetch_prev_lru_page(_page, _base, _field)			\
102 	do {								\
103 		if ((_page)->lru.prev != _base) {			\
104 			struct page *prev;				\
105 									\
106 			prev = lru_to_page(&(_page->lru));		\
107 			prefetch(&prev->_field);			\
108 		}							\
109 	} while (0)
110 #else
111 #define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
112 #endif
113 
114 #ifdef ARCH_HAS_PREFETCHW
115 #define prefetchw_prev_lru_page(_page, _base, _field)			\
116 	do {								\
117 		if ((_page)->lru.prev != _base) {			\
118 			struct page *prev;				\
119 									\
120 			prev = lru_to_page(&(_page->lru));		\
121 			prefetchw(&prev->_field);			\
122 		}							\
123 	} while (0)
124 #else
125 #define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
126 #endif
127 
128 /*
129  * From 0 .. 100.  Higher means more swappy.
130  */
131 int vm_swappiness = 60;
132 static long total_memory;
133 
134 static LIST_HEAD(shrinker_list);
135 static DECLARE_RWSEM(shrinker_rwsem);
136 
137 /*
138  * Add a shrinker callback to be called from the vm
139  */
140 struct shrinker *set_shrinker(int seeks, shrinker_t theshrinker)
141 {
142         struct shrinker *shrinker;
143 
144         shrinker = kmalloc(sizeof(*shrinker), GFP_KERNEL);
145         if (shrinker) {
146 	        shrinker->shrinker = theshrinker;
147 	        shrinker->seeks = seeks;
148 	        shrinker->nr = 0;
149 	        down_write(&shrinker_rwsem);
150 	        list_add_tail(&shrinker->list, &shrinker_list);
151 	        up_write(&shrinker_rwsem);
152 	}
153 	return shrinker;
154 }
155 EXPORT_SYMBOL(set_shrinker);
156 
157 /*
158  * Remove one
159  */
160 void remove_shrinker(struct shrinker *shrinker)
161 {
162 	down_write(&shrinker_rwsem);
163 	list_del(&shrinker->list);
164 	up_write(&shrinker_rwsem);
165 	kfree(shrinker);
166 }
167 EXPORT_SYMBOL(remove_shrinker);
168 
169 #define SHRINK_BATCH 128
170 /*
171  * Call the shrink functions to age shrinkable caches
172  *
173  * Here we assume it costs one seek to replace a lru page and that it also
174  * takes a seek to recreate a cache object.  With this in mind we age equal
175  * percentages of the lru and ageable caches.  This should balance the seeks
176  * generated by these structures.
177  *
178  * If the vm encounted mapped pages on the LRU it increase the pressure on
179  * slab to avoid swapping.
180  *
181  * We do weird things to avoid (scanned*seeks*entries) overflowing 32 bits.
182  *
183  * `lru_pages' represents the number of on-LRU pages in all the zones which
184  * are eligible for the caller's allocation attempt.  It is used for balancing
185  * slab reclaim versus page reclaim.
186  *
187  * Returns the number of slab objects which we shrunk.
188  */
189 static int shrink_slab(unsigned long scanned, gfp_t gfp_mask,
190 			unsigned long lru_pages)
191 {
192 	struct shrinker *shrinker;
193 	int ret = 0;
194 
195 	if (scanned == 0)
196 		scanned = SWAP_CLUSTER_MAX;
197 
198 	if (!down_read_trylock(&shrinker_rwsem))
199 		return 1;	/* Assume we'll be able to shrink next time */
200 
201 	list_for_each_entry(shrinker, &shrinker_list, list) {
202 		unsigned long long delta;
203 		unsigned long total_scan;
204 
205 		delta = (4 * scanned) / shrinker->seeks;
206 		delta *= (*shrinker->shrinker)(0, gfp_mask);
207 		do_div(delta, lru_pages + 1);
208 		shrinker->nr += delta;
209 		if (shrinker->nr < 0)
210 			shrinker->nr = LONG_MAX;	/* It wrapped! */
211 
212 		total_scan = shrinker->nr;
213 		shrinker->nr = 0;
214 
215 		while (total_scan >= SHRINK_BATCH) {
216 			long this_scan = SHRINK_BATCH;
217 			int shrink_ret;
218 			int nr_before;
219 
220 			nr_before = (*shrinker->shrinker)(0, gfp_mask);
221 			shrink_ret = (*shrinker->shrinker)(this_scan, gfp_mask);
222 			if (shrink_ret == -1)
223 				break;
224 			if (shrink_ret < nr_before)
225 				ret += nr_before - shrink_ret;
226 			mod_page_state(slabs_scanned, this_scan);
227 			total_scan -= this_scan;
228 
229 			cond_resched();
230 		}
231 
232 		shrinker->nr += total_scan;
233 	}
234 	up_read(&shrinker_rwsem);
235 	return ret;
236 }
237 
238 /* Called without lock on whether page is mapped, so answer is unstable */
239 static inline int page_mapping_inuse(struct page *page)
240 {
241 	struct address_space *mapping;
242 
243 	/* Page is in somebody's page tables. */
244 	if (page_mapped(page))
245 		return 1;
246 
247 	/* Be more reluctant to reclaim swapcache than pagecache */
248 	if (PageSwapCache(page))
249 		return 1;
250 
251 	mapping = page_mapping(page);
252 	if (!mapping)
253 		return 0;
254 
255 	/* File is mmap'd by somebody? */
256 	return mapping_mapped(mapping);
257 }
258 
259 static inline int is_page_cache_freeable(struct page *page)
260 {
261 	return page_count(page) - !!PagePrivate(page) == 2;
262 }
263 
264 static int may_write_to_queue(struct backing_dev_info *bdi)
265 {
266 	if (current_is_kswapd())
267 		return 1;
268 	if (current_is_pdflush())	/* This is unlikely, but why not... */
269 		return 1;
270 	if (!bdi_write_congested(bdi))
271 		return 1;
272 	if (bdi == current->backing_dev_info)
273 		return 1;
274 	return 0;
275 }
276 
277 /*
278  * We detected a synchronous write error writing a page out.  Probably
279  * -ENOSPC.  We need to propagate that into the address_space for a subsequent
280  * fsync(), msync() or close().
281  *
282  * The tricky part is that after writepage we cannot touch the mapping: nothing
283  * prevents it from being freed up.  But we have a ref on the page and once
284  * that page is locked, the mapping is pinned.
285  *
286  * We're allowed to run sleeping lock_page() here because we know the caller has
287  * __GFP_FS.
288  */
289 static void handle_write_error(struct address_space *mapping,
290 				struct page *page, int error)
291 {
292 	lock_page(page);
293 	if (page_mapping(page) == mapping) {
294 		if (error == -ENOSPC)
295 			set_bit(AS_ENOSPC, &mapping->flags);
296 		else
297 			set_bit(AS_EIO, &mapping->flags);
298 	}
299 	unlock_page(page);
300 }
301 
302 /*
303  * pageout is called by shrink_list() for each dirty page. Calls ->writepage().
304  */
305 static pageout_t pageout(struct page *page, struct address_space *mapping)
306 {
307 	/*
308 	 * If the page is dirty, only perform writeback if that write
309 	 * will be non-blocking.  To prevent this allocation from being
310 	 * stalled by pagecache activity.  But note that there may be
311 	 * stalls if we need to run get_block().  We could test
312 	 * PagePrivate for that.
313 	 *
314 	 * If this process is currently in generic_file_write() against
315 	 * this page's queue, we can perform writeback even if that
316 	 * will block.
317 	 *
318 	 * If the page is swapcache, write it back even if that would
319 	 * block, for some throttling. This happens by accident, because
320 	 * swap_backing_dev_info is bust: it doesn't reflect the
321 	 * congestion state of the swapdevs.  Easy to fix, if needed.
322 	 * See swapfile.c:page_queue_congested().
323 	 */
324 	if (!is_page_cache_freeable(page))
325 		return PAGE_KEEP;
326 	if (!mapping) {
327 		/*
328 		 * Some data journaling orphaned pages can have
329 		 * page->mapping == NULL while being dirty with clean buffers.
330 		 */
331 		if (PagePrivate(page)) {
332 			if (try_to_free_buffers(page)) {
333 				ClearPageDirty(page);
334 				printk("%s: orphaned page\n", __FUNCTION__);
335 				return PAGE_CLEAN;
336 			}
337 		}
338 		return PAGE_KEEP;
339 	}
340 	if (mapping->a_ops->writepage == NULL)
341 		return PAGE_ACTIVATE;
342 	if (!may_write_to_queue(mapping->backing_dev_info))
343 		return PAGE_KEEP;
344 
345 	if (clear_page_dirty_for_io(page)) {
346 		int res;
347 		struct writeback_control wbc = {
348 			.sync_mode = WB_SYNC_NONE,
349 			.nr_to_write = SWAP_CLUSTER_MAX,
350 			.nonblocking = 1,
351 			.for_reclaim = 1,
352 		};
353 
354 		SetPageReclaim(page);
355 		res = mapping->a_ops->writepage(page, &wbc);
356 		if (res < 0)
357 			handle_write_error(mapping, page, res);
358 		if (res == WRITEPAGE_ACTIVATE) {
359 			ClearPageReclaim(page);
360 			return PAGE_ACTIVATE;
361 		}
362 		if (!PageWriteback(page)) {
363 			/* synchronous write or broken a_ops? */
364 			ClearPageReclaim(page);
365 		}
366 
367 		return PAGE_SUCCESS;
368 	}
369 
370 	return PAGE_CLEAN;
371 }
372 
373 /*
374  * shrink_list adds the number of reclaimed pages to sc->nr_reclaimed
375  */
376 static int shrink_list(struct list_head *page_list, struct scan_control *sc)
377 {
378 	LIST_HEAD(ret_pages);
379 	struct pagevec freed_pvec;
380 	int pgactivate = 0;
381 	int reclaimed = 0;
382 
383 	cond_resched();
384 
385 	pagevec_init(&freed_pvec, 1);
386 	while (!list_empty(page_list)) {
387 		struct address_space *mapping;
388 		struct page *page;
389 		int may_enter_fs;
390 		int referenced;
391 
392 		cond_resched();
393 
394 		page = lru_to_page(page_list);
395 		list_del(&page->lru);
396 
397 		if (TestSetPageLocked(page))
398 			goto keep;
399 
400 		BUG_ON(PageActive(page));
401 
402 		sc->nr_scanned++;
403 		/* Double the slab pressure for mapped and swapcache pages */
404 		if (page_mapped(page) || PageSwapCache(page))
405 			sc->nr_scanned++;
406 
407 		if (PageWriteback(page))
408 			goto keep_locked;
409 
410 		referenced = page_referenced(page, 1, sc->priority <= 0);
411 		/* In active use or really unfreeable?  Activate it. */
412 		if (referenced && page_mapping_inuse(page))
413 			goto activate_locked;
414 
415 #ifdef CONFIG_SWAP
416 		/*
417 		 * Anonymous process memory has backing store?
418 		 * Try to allocate it some swap space here.
419 		 */
420 		if (PageAnon(page) && !PageSwapCache(page)) {
421 			if (!sc->may_swap)
422 				goto keep_locked;
423 			if (!add_to_swap(page))
424 				goto activate_locked;
425 		}
426 #endif /* CONFIG_SWAP */
427 
428 		mapping = page_mapping(page);
429 		may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
430 			(PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
431 
432 		/*
433 		 * The page is mapped into the page tables of one or more
434 		 * processes. Try to unmap it here.
435 		 */
436 		if (page_mapped(page) && mapping) {
437 			switch (try_to_unmap(page)) {
438 			case SWAP_FAIL:
439 				goto activate_locked;
440 			case SWAP_AGAIN:
441 				goto keep_locked;
442 			case SWAP_SUCCESS:
443 				; /* try to free the page below */
444 			}
445 		}
446 
447 		if (PageDirty(page)) {
448 			if (referenced)
449 				goto keep_locked;
450 			if (!may_enter_fs)
451 				goto keep_locked;
452 			if (laptop_mode && !sc->may_writepage)
453 				goto keep_locked;
454 
455 			/* Page is dirty, try to write it out here */
456 			switch(pageout(page, mapping)) {
457 			case PAGE_KEEP:
458 				goto keep_locked;
459 			case PAGE_ACTIVATE:
460 				goto activate_locked;
461 			case PAGE_SUCCESS:
462 				if (PageWriteback(page) || PageDirty(page))
463 					goto keep;
464 				/*
465 				 * A synchronous write - probably a ramdisk.  Go
466 				 * ahead and try to reclaim the page.
467 				 */
468 				if (TestSetPageLocked(page))
469 					goto keep;
470 				if (PageDirty(page) || PageWriteback(page))
471 					goto keep_locked;
472 				mapping = page_mapping(page);
473 			case PAGE_CLEAN:
474 				; /* try to free the page below */
475 			}
476 		}
477 
478 		/*
479 		 * If the page has buffers, try to free the buffer mappings
480 		 * associated with this page. If we succeed we try to free
481 		 * the page as well.
482 		 *
483 		 * We do this even if the page is PageDirty().
484 		 * try_to_release_page() does not perform I/O, but it is
485 		 * possible for a page to have PageDirty set, but it is actually
486 		 * clean (all its buffers are clean).  This happens if the
487 		 * buffers were written out directly, with submit_bh(). ext3
488 		 * will do this, as well as the blockdev mapping.
489 		 * try_to_release_page() will discover that cleanness and will
490 		 * drop the buffers and mark the page clean - it can be freed.
491 		 *
492 		 * Rarely, pages can have buffers and no ->mapping.  These are
493 		 * the pages which were not successfully invalidated in
494 		 * truncate_complete_page().  We try to drop those buffers here
495 		 * and if that worked, and the page is no longer mapped into
496 		 * process address space (page_count == 1) it can be freed.
497 		 * Otherwise, leave the page on the LRU so it is swappable.
498 		 */
499 		if (PagePrivate(page)) {
500 			if (!try_to_release_page(page, sc->gfp_mask))
501 				goto activate_locked;
502 			if (!mapping && page_count(page) == 1)
503 				goto free_it;
504 		}
505 
506 		if (!mapping)
507 			goto keep_locked;	/* truncate got there first */
508 
509 		write_lock_irq(&mapping->tree_lock);
510 
511 		/*
512 		 * The non-racy check for busy page.  It is critical to check
513 		 * PageDirty _after_ making sure that the page is freeable and
514 		 * not in use by anybody. 	(pagecache + us == 2)
515 		 */
516 		if (unlikely(page_count(page) != 2))
517 			goto cannot_free;
518 		smp_rmb();
519 		if (unlikely(PageDirty(page)))
520 			goto cannot_free;
521 
522 #ifdef CONFIG_SWAP
523 		if (PageSwapCache(page)) {
524 			swp_entry_t swap = { .val = page_private(page) };
525 			__delete_from_swap_cache(page);
526 			write_unlock_irq(&mapping->tree_lock);
527 			swap_free(swap);
528 			__put_page(page);	/* The pagecache ref */
529 			goto free_it;
530 		}
531 #endif /* CONFIG_SWAP */
532 
533 		__remove_from_page_cache(page);
534 		write_unlock_irq(&mapping->tree_lock);
535 		__put_page(page);
536 
537 free_it:
538 		unlock_page(page);
539 		reclaimed++;
540 		if (!pagevec_add(&freed_pvec, page))
541 			__pagevec_release_nonlru(&freed_pvec);
542 		continue;
543 
544 cannot_free:
545 		write_unlock_irq(&mapping->tree_lock);
546 		goto keep_locked;
547 
548 activate_locked:
549 		SetPageActive(page);
550 		pgactivate++;
551 keep_locked:
552 		unlock_page(page);
553 keep:
554 		list_add(&page->lru, &ret_pages);
555 		BUG_ON(PageLRU(page));
556 	}
557 	list_splice(&ret_pages, page_list);
558 	if (pagevec_count(&freed_pvec))
559 		__pagevec_release_nonlru(&freed_pvec);
560 	mod_page_state(pgactivate, pgactivate);
561 	sc->nr_reclaimed += reclaimed;
562 	return reclaimed;
563 }
564 
565 /*
566  * zone->lru_lock is heavily contended.  Some of the functions that
567  * shrink the lists perform better by taking out a batch of pages
568  * and working on them outside the LRU lock.
569  *
570  * For pagecache intensive workloads, this function is the hottest
571  * spot in the kernel (apart from copy_*_user functions).
572  *
573  * Appropriate locks must be held before calling this function.
574  *
575  * @nr_to_scan:	The number of pages to look through on the list.
576  * @src:	The LRU list to pull pages off.
577  * @dst:	The temp list to put pages on to.
578  * @scanned:	The number of pages that were scanned.
579  *
580  * returns how many pages were moved onto *@dst.
581  */
582 static int isolate_lru_pages(int nr_to_scan, struct list_head *src,
583 			     struct list_head *dst, int *scanned)
584 {
585 	int nr_taken = 0;
586 	struct page *page;
587 	int scan = 0;
588 
589 	while (scan++ < nr_to_scan && !list_empty(src)) {
590 		page = lru_to_page(src);
591 		prefetchw_prev_lru_page(page, src, flags);
592 
593 		if (!TestClearPageLRU(page))
594 			BUG();
595 		list_del(&page->lru);
596 		if (get_page_testone(page)) {
597 			/*
598 			 * It is being freed elsewhere
599 			 */
600 			__put_page(page);
601 			SetPageLRU(page);
602 			list_add(&page->lru, src);
603 			continue;
604 		} else {
605 			list_add(&page->lru, dst);
606 			nr_taken++;
607 		}
608 	}
609 
610 	*scanned = scan;
611 	return nr_taken;
612 }
613 
614 /*
615  * shrink_cache() adds the number of pages reclaimed to sc->nr_reclaimed
616  */
617 static void shrink_cache(struct zone *zone, struct scan_control *sc)
618 {
619 	LIST_HEAD(page_list);
620 	struct pagevec pvec;
621 	int max_scan = sc->nr_to_scan;
622 
623 	pagevec_init(&pvec, 1);
624 
625 	lru_add_drain();
626 	spin_lock_irq(&zone->lru_lock);
627 	while (max_scan > 0) {
628 		struct page *page;
629 		int nr_taken;
630 		int nr_scan;
631 		int nr_freed;
632 
633 		nr_taken = isolate_lru_pages(sc->swap_cluster_max,
634 					     &zone->inactive_list,
635 					     &page_list, &nr_scan);
636 		zone->nr_inactive -= nr_taken;
637 		zone->pages_scanned += nr_scan;
638 		spin_unlock_irq(&zone->lru_lock);
639 
640 		if (nr_taken == 0)
641 			goto done;
642 
643 		max_scan -= nr_scan;
644 		if (current_is_kswapd())
645 			mod_page_state_zone(zone, pgscan_kswapd, nr_scan);
646 		else
647 			mod_page_state_zone(zone, pgscan_direct, nr_scan);
648 		nr_freed = shrink_list(&page_list, sc);
649 		if (current_is_kswapd())
650 			mod_page_state(kswapd_steal, nr_freed);
651 		mod_page_state_zone(zone, pgsteal, nr_freed);
652 		sc->nr_to_reclaim -= nr_freed;
653 
654 		spin_lock_irq(&zone->lru_lock);
655 		/*
656 		 * Put back any unfreeable pages.
657 		 */
658 		while (!list_empty(&page_list)) {
659 			page = lru_to_page(&page_list);
660 			if (TestSetPageLRU(page))
661 				BUG();
662 			list_del(&page->lru);
663 			if (PageActive(page))
664 				add_page_to_active_list(zone, page);
665 			else
666 				add_page_to_inactive_list(zone, page);
667 			if (!pagevec_add(&pvec, page)) {
668 				spin_unlock_irq(&zone->lru_lock);
669 				__pagevec_release(&pvec);
670 				spin_lock_irq(&zone->lru_lock);
671 			}
672 		}
673   	}
674 	spin_unlock_irq(&zone->lru_lock);
675 done:
676 	pagevec_release(&pvec);
677 }
678 
679 /*
680  * This moves pages from the active list to the inactive list.
681  *
682  * We move them the other way if the page is referenced by one or more
683  * processes, from rmap.
684  *
685  * If the pages are mostly unmapped, the processing is fast and it is
686  * appropriate to hold zone->lru_lock across the whole operation.  But if
687  * the pages are mapped, the processing is slow (page_referenced()) so we
688  * should drop zone->lru_lock around each page.  It's impossible to balance
689  * this, so instead we remove the pages from the LRU while processing them.
690  * It is safe to rely on PG_active against the non-LRU pages in here because
691  * nobody will play with that bit on a non-LRU page.
692  *
693  * The downside is that we have to touch page->_count against each page.
694  * But we had to alter page->flags anyway.
695  */
696 static void
697 refill_inactive_zone(struct zone *zone, struct scan_control *sc)
698 {
699 	int pgmoved;
700 	int pgdeactivate = 0;
701 	int pgscanned;
702 	int nr_pages = sc->nr_to_scan;
703 	LIST_HEAD(l_hold);	/* The pages which were snipped off */
704 	LIST_HEAD(l_inactive);	/* Pages to go onto the inactive_list */
705 	LIST_HEAD(l_active);	/* Pages to go onto the active_list */
706 	struct page *page;
707 	struct pagevec pvec;
708 	int reclaim_mapped = 0;
709 	long mapped_ratio;
710 	long distress;
711 	long swap_tendency;
712 
713 	lru_add_drain();
714 	spin_lock_irq(&zone->lru_lock);
715 	pgmoved = isolate_lru_pages(nr_pages, &zone->active_list,
716 				    &l_hold, &pgscanned);
717 	zone->pages_scanned += pgscanned;
718 	zone->nr_active -= pgmoved;
719 	spin_unlock_irq(&zone->lru_lock);
720 
721 	/*
722 	 * `distress' is a measure of how much trouble we're having reclaiming
723 	 * pages.  0 -> no problems.  100 -> great trouble.
724 	 */
725 	distress = 100 >> zone->prev_priority;
726 
727 	/*
728 	 * The point of this algorithm is to decide when to start reclaiming
729 	 * mapped memory instead of just pagecache.  Work out how much memory
730 	 * is mapped.
731 	 */
732 	mapped_ratio = (sc->nr_mapped * 100) / total_memory;
733 
734 	/*
735 	 * Now decide how much we really want to unmap some pages.  The mapped
736 	 * ratio is downgraded - just because there's a lot of mapped memory
737 	 * doesn't necessarily mean that page reclaim isn't succeeding.
738 	 *
739 	 * The distress ratio is important - we don't want to start going oom.
740 	 *
741 	 * A 100% value of vm_swappiness overrides this algorithm altogether.
742 	 */
743 	swap_tendency = mapped_ratio / 2 + distress + vm_swappiness;
744 
745 	/*
746 	 * Now use this metric to decide whether to start moving mapped memory
747 	 * onto the inactive list.
748 	 */
749 	if (swap_tendency >= 100)
750 		reclaim_mapped = 1;
751 
752 	while (!list_empty(&l_hold)) {
753 		cond_resched();
754 		page = lru_to_page(&l_hold);
755 		list_del(&page->lru);
756 		if (page_mapped(page)) {
757 			if (!reclaim_mapped ||
758 			    (total_swap_pages == 0 && PageAnon(page)) ||
759 			    page_referenced(page, 0, sc->priority <= 0)) {
760 				list_add(&page->lru, &l_active);
761 				continue;
762 			}
763 		}
764 		list_add(&page->lru, &l_inactive);
765 	}
766 
767 	pagevec_init(&pvec, 1);
768 	pgmoved = 0;
769 	spin_lock_irq(&zone->lru_lock);
770 	while (!list_empty(&l_inactive)) {
771 		page = lru_to_page(&l_inactive);
772 		prefetchw_prev_lru_page(page, &l_inactive, flags);
773 		if (TestSetPageLRU(page))
774 			BUG();
775 		if (!TestClearPageActive(page))
776 			BUG();
777 		list_move(&page->lru, &zone->inactive_list);
778 		pgmoved++;
779 		if (!pagevec_add(&pvec, page)) {
780 			zone->nr_inactive += pgmoved;
781 			spin_unlock_irq(&zone->lru_lock);
782 			pgdeactivate += pgmoved;
783 			pgmoved = 0;
784 			if (buffer_heads_over_limit)
785 				pagevec_strip(&pvec);
786 			__pagevec_release(&pvec);
787 			spin_lock_irq(&zone->lru_lock);
788 		}
789 	}
790 	zone->nr_inactive += pgmoved;
791 	pgdeactivate += pgmoved;
792 	if (buffer_heads_over_limit) {
793 		spin_unlock_irq(&zone->lru_lock);
794 		pagevec_strip(&pvec);
795 		spin_lock_irq(&zone->lru_lock);
796 	}
797 
798 	pgmoved = 0;
799 	while (!list_empty(&l_active)) {
800 		page = lru_to_page(&l_active);
801 		prefetchw_prev_lru_page(page, &l_active, flags);
802 		if (TestSetPageLRU(page))
803 			BUG();
804 		BUG_ON(!PageActive(page));
805 		list_move(&page->lru, &zone->active_list);
806 		pgmoved++;
807 		if (!pagevec_add(&pvec, page)) {
808 			zone->nr_active += pgmoved;
809 			pgmoved = 0;
810 			spin_unlock_irq(&zone->lru_lock);
811 			__pagevec_release(&pvec);
812 			spin_lock_irq(&zone->lru_lock);
813 		}
814 	}
815 	zone->nr_active += pgmoved;
816 	spin_unlock_irq(&zone->lru_lock);
817 	pagevec_release(&pvec);
818 
819 	mod_page_state_zone(zone, pgrefill, pgscanned);
820 	mod_page_state(pgdeactivate, pgdeactivate);
821 }
822 
823 /*
824  * This is a basic per-zone page freer.  Used by both kswapd and direct reclaim.
825  */
826 static void
827 shrink_zone(struct zone *zone, struct scan_control *sc)
828 {
829 	unsigned long nr_active;
830 	unsigned long nr_inactive;
831 
832 	atomic_inc(&zone->reclaim_in_progress);
833 
834 	/*
835 	 * Add one to `nr_to_scan' just to make sure that the kernel will
836 	 * slowly sift through the active list.
837 	 */
838 	zone->nr_scan_active += (zone->nr_active >> sc->priority) + 1;
839 	nr_active = zone->nr_scan_active;
840 	if (nr_active >= sc->swap_cluster_max)
841 		zone->nr_scan_active = 0;
842 	else
843 		nr_active = 0;
844 
845 	zone->nr_scan_inactive += (zone->nr_inactive >> sc->priority) + 1;
846 	nr_inactive = zone->nr_scan_inactive;
847 	if (nr_inactive >= sc->swap_cluster_max)
848 		zone->nr_scan_inactive = 0;
849 	else
850 		nr_inactive = 0;
851 
852 	sc->nr_to_reclaim = sc->swap_cluster_max;
853 
854 	while (nr_active || nr_inactive) {
855 		if (nr_active) {
856 			sc->nr_to_scan = min(nr_active,
857 					(unsigned long)sc->swap_cluster_max);
858 			nr_active -= sc->nr_to_scan;
859 			refill_inactive_zone(zone, sc);
860 		}
861 
862 		if (nr_inactive) {
863 			sc->nr_to_scan = min(nr_inactive,
864 					(unsigned long)sc->swap_cluster_max);
865 			nr_inactive -= sc->nr_to_scan;
866 			shrink_cache(zone, sc);
867 			if (sc->nr_to_reclaim <= 0)
868 				break;
869 		}
870 	}
871 
872 	throttle_vm_writeout();
873 
874 	atomic_dec(&zone->reclaim_in_progress);
875 }
876 
877 /*
878  * This is the direct reclaim path, for page-allocating processes.  We only
879  * try to reclaim pages from zones which will satisfy the caller's allocation
880  * request.
881  *
882  * We reclaim from a zone even if that zone is over pages_high.  Because:
883  * a) The caller may be trying to free *extra* pages to satisfy a higher-order
884  *    allocation or
885  * b) The zones may be over pages_high but they must go *over* pages_high to
886  *    satisfy the `incremental min' zone defense algorithm.
887  *
888  * Returns the number of reclaimed pages.
889  *
890  * If a zone is deemed to be full of pinned pages then just give it a light
891  * scan then give up on it.
892  */
893 static void
894 shrink_caches(struct zone **zones, struct scan_control *sc)
895 {
896 	int i;
897 
898 	for (i = 0; zones[i] != NULL; i++) {
899 		struct zone *zone = zones[i];
900 
901 		if (zone->present_pages == 0)
902 			continue;
903 
904 		if (!cpuset_zone_allowed(zone, __GFP_HARDWALL))
905 			continue;
906 
907 		zone->temp_priority = sc->priority;
908 		if (zone->prev_priority > sc->priority)
909 			zone->prev_priority = sc->priority;
910 
911 		if (zone->all_unreclaimable && sc->priority != DEF_PRIORITY)
912 			continue;	/* Let kswapd poll it */
913 
914 		shrink_zone(zone, sc);
915 	}
916 }
917 
918 /*
919  * This is the main entry point to direct page reclaim.
920  *
921  * If a full scan of the inactive list fails to free enough memory then we
922  * are "out of memory" and something needs to be killed.
923  *
924  * If the caller is !__GFP_FS then the probability of a failure is reasonably
925  * high - the zone may be full of dirty or under-writeback pages, which this
926  * caller can't do much about.  We kick pdflush and take explicit naps in the
927  * hope that some of these pages can be written.  But if the allocating task
928  * holds filesystem locks which prevent writeout this might not work, and the
929  * allocation attempt will fail.
930  */
931 int try_to_free_pages(struct zone **zones, gfp_t gfp_mask)
932 {
933 	int priority;
934 	int ret = 0;
935 	int total_scanned = 0, total_reclaimed = 0;
936 	struct reclaim_state *reclaim_state = current->reclaim_state;
937 	struct scan_control sc;
938 	unsigned long lru_pages = 0;
939 	int i;
940 
941 	sc.gfp_mask = gfp_mask;
942 	sc.may_writepage = 0;
943 	sc.may_swap = 1;
944 
945 	inc_page_state(allocstall);
946 
947 	for (i = 0; zones[i] != NULL; i++) {
948 		struct zone *zone = zones[i];
949 
950 		if (!cpuset_zone_allowed(zone, __GFP_HARDWALL))
951 			continue;
952 
953 		zone->temp_priority = DEF_PRIORITY;
954 		lru_pages += zone->nr_active + zone->nr_inactive;
955 	}
956 
957 	for (priority = DEF_PRIORITY; priority >= 0; priority--) {
958 		sc.nr_mapped = read_page_state(nr_mapped);
959 		sc.nr_scanned = 0;
960 		sc.nr_reclaimed = 0;
961 		sc.priority = priority;
962 		sc.swap_cluster_max = SWAP_CLUSTER_MAX;
963 		shrink_caches(zones, &sc);
964 		shrink_slab(sc.nr_scanned, gfp_mask, lru_pages);
965 		if (reclaim_state) {
966 			sc.nr_reclaimed += reclaim_state->reclaimed_slab;
967 			reclaim_state->reclaimed_slab = 0;
968 		}
969 		total_scanned += sc.nr_scanned;
970 		total_reclaimed += sc.nr_reclaimed;
971 		if (total_reclaimed >= sc.swap_cluster_max) {
972 			ret = 1;
973 			goto out;
974 		}
975 
976 		/*
977 		 * Try to write back as many pages as we just scanned.  This
978 		 * tends to cause slow streaming writers to write data to the
979 		 * disk smoothly, at the dirtying rate, which is nice.   But
980 		 * that's undesirable in laptop mode, where we *want* lumpy
981 		 * writeout.  So in laptop mode, write out the whole world.
982 		 */
983 		if (total_scanned > sc.swap_cluster_max + sc.swap_cluster_max/2) {
984 			wakeup_pdflush(laptop_mode ? 0 : total_scanned);
985 			sc.may_writepage = 1;
986 		}
987 
988 		/* Take a nap, wait for some writeback to complete */
989 		if (sc.nr_scanned && priority < DEF_PRIORITY - 2)
990 			blk_congestion_wait(WRITE, HZ/10);
991 	}
992 out:
993 	for (i = 0; zones[i] != 0; i++) {
994 		struct zone *zone = zones[i];
995 
996 		if (!cpuset_zone_allowed(zone, __GFP_HARDWALL))
997 			continue;
998 
999 		zone->prev_priority = zone->temp_priority;
1000 	}
1001 	return ret;
1002 }
1003 
1004 /*
1005  * For kswapd, balance_pgdat() will work across all this node's zones until
1006  * they are all at pages_high.
1007  *
1008  * If `nr_pages' is non-zero then it is the number of pages which are to be
1009  * reclaimed, regardless of the zone occupancies.  This is a software suspend
1010  * special.
1011  *
1012  * Returns the number of pages which were actually freed.
1013  *
1014  * There is special handling here for zones which are full of pinned pages.
1015  * This can happen if the pages are all mlocked, or if they are all used by
1016  * device drivers (say, ZONE_DMA).  Or if they are all in use by hugetlb.
1017  * What we do is to detect the case where all pages in the zone have been
1018  * scanned twice and there has been zero successful reclaim.  Mark the zone as
1019  * dead and from now on, only perform a short scan.  Basically we're polling
1020  * the zone for when the problem goes away.
1021  *
1022  * kswapd scans the zones in the highmem->normal->dma direction.  It skips
1023  * zones which have free_pages > pages_high, but once a zone is found to have
1024  * free_pages <= pages_high, we scan that zone and the lower zones regardless
1025  * of the number of free pages in the lower zones.  This interoperates with
1026  * the page allocator fallback scheme to ensure that aging of pages is balanced
1027  * across the zones.
1028  */
1029 static int balance_pgdat(pg_data_t *pgdat, int nr_pages, int order)
1030 {
1031 	int to_free = nr_pages;
1032 	int all_zones_ok;
1033 	int priority;
1034 	int i;
1035 	int total_scanned, total_reclaimed;
1036 	struct reclaim_state *reclaim_state = current->reclaim_state;
1037 	struct scan_control sc;
1038 
1039 loop_again:
1040 	total_scanned = 0;
1041 	total_reclaimed = 0;
1042 	sc.gfp_mask = GFP_KERNEL;
1043 	sc.may_writepage = 0;
1044 	sc.may_swap = 1;
1045 	sc.nr_mapped = read_page_state(nr_mapped);
1046 
1047 	inc_page_state(pageoutrun);
1048 
1049 	for (i = 0; i < pgdat->nr_zones; i++) {
1050 		struct zone *zone = pgdat->node_zones + i;
1051 
1052 		zone->temp_priority = DEF_PRIORITY;
1053 	}
1054 
1055 	for (priority = DEF_PRIORITY; priority >= 0; priority--) {
1056 		int end_zone = 0;	/* Inclusive.  0 = ZONE_DMA */
1057 		unsigned long lru_pages = 0;
1058 
1059 		all_zones_ok = 1;
1060 
1061 		if (nr_pages == 0) {
1062 			/*
1063 			 * Scan in the highmem->dma direction for the highest
1064 			 * zone which needs scanning
1065 			 */
1066 			for (i = pgdat->nr_zones - 1; i >= 0; i--) {
1067 				struct zone *zone = pgdat->node_zones + i;
1068 
1069 				if (zone->present_pages == 0)
1070 					continue;
1071 
1072 				if (zone->all_unreclaimable &&
1073 						priority != DEF_PRIORITY)
1074 					continue;
1075 
1076 				if (!zone_watermark_ok(zone, order,
1077 						zone->pages_high, 0, 0, 0)) {
1078 					end_zone = i;
1079 					goto scan;
1080 				}
1081 			}
1082 			goto out;
1083 		} else {
1084 			end_zone = pgdat->nr_zones - 1;
1085 		}
1086 scan:
1087 		for (i = 0; i <= end_zone; i++) {
1088 			struct zone *zone = pgdat->node_zones + i;
1089 
1090 			lru_pages += zone->nr_active + zone->nr_inactive;
1091 		}
1092 
1093 		/*
1094 		 * Now scan the zone in the dma->highmem direction, stopping
1095 		 * at the last zone which needs scanning.
1096 		 *
1097 		 * We do this because the page allocator works in the opposite
1098 		 * direction.  This prevents the page allocator from allocating
1099 		 * pages behind kswapd's direction of progress, which would
1100 		 * cause too much scanning of the lower zones.
1101 		 */
1102 		for (i = 0; i <= end_zone; i++) {
1103 			struct zone *zone = pgdat->node_zones + i;
1104 			int nr_slab;
1105 
1106 			if (zone->present_pages == 0)
1107 				continue;
1108 
1109 			if (zone->all_unreclaimable && priority != DEF_PRIORITY)
1110 				continue;
1111 
1112 			if (nr_pages == 0) {	/* Not software suspend */
1113 				if (!zone_watermark_ok(zone, order,
1114 						zone->pages_high, end_zone, 0, 0))
1115 					all_zones_ok = 0;
1116 			}
1117 			zone->temp_priority = priority;
1118 			if (zone->prev_priority > priority)
1119 				zone->prev_priority = priority;
1120 			sc.nr_scanned = 0;
1121 			sc.nr_reclaimed = 0;
1122 			sc.priority = priority;
1123 			sc.swap_cluster_max = nr_pages? nr_pages : SWAP_CLUSTER_MAX;
1124 			atomic_inc(&zone->reclaim_in_progress);
1125 			shrink_zone(zone, &sc);
1126 			atomic_dec(&zone->reclaim_in_progress);
1127 			reclaim_state->reclaimed_slab = 0;
1128 			nr_slab = shrink_slab(sc.nr_scanned, GFP_KERNEL,
1129 						lru_pages);
1130 			sc.nr_reclaimed += reclaim_state->reclaimed_slab;
1131 			total_reclaimed += sc.nr_reclaimed;
1132 			total_scanned += sc.nr_scanned;
1133 			if (zone->all_unreclaimable)
1134 				continue;
1135 			if (nr_slab == 0 && zone->pages_scanned >=
1136 				    (zone->nr_active + zone->nr_inactive) * 4)
1137 				zone->all_unreclaimable = 1;
1138 			/*
1139 			 * If we've done a decent amount of scanning and
1140 			 * the reclaim ratio is low, start doing writepage
1141 			 * even in laptop mode
1142 			 */
1143 			if (total_scanned > SWAP_CLUSTER_MAX * 2 &&
1144 			    total_scanned > total_reclaimed+total_reclaimed/2)
1145 				sc.may_writepage = 1;
1146 		}
1147 		if (nr_pages && to_free > total_reclaimed)
1148 			continue;	/* swsusp: need to do more work */
1149 		if (all_zones_ok)
1150 			break;		/* kswapd: all done */
1151 		/*
1152 		 * OK, kswapd is getting into trouble.  Take a nap, then take
1153 		 * another pass across the zones.
1154 		 */
1155 		if (total_scanned && priority < DEF_PRIORITY - 2)
1156 			blk_congestion_wait(WRITE, HZ/10);
1157 
1158 		/*
1159 		 * We do this so kswapd doesn't build up large priorities for
1160 		 * example when it is freeing in parallel with allocators. It
1161 		 * matches the direct reclaim path behaviour in terms of impact
1162 		 * on zone->*_priority.
1163 		 */
1164 		if ((total_reclaimed >= SWAP_CLUSTER_MAX) && (!nr_pages))
1165 			break;
1166 	}
1167 out:
1168 	for (i = 0; i < pgdat->nr_zones; i++) {
1169 		struct zone *zone = pgdat->node_zones + i;
1170 
1171 		zone->prev_priority = zone->temp_priority;
1172 	}
1173 	if (!all_zones_ok) {
1174 		cond_resched();
1175 		goto loop_again;
1176 	}
1177 
1178 	return total_reclaimed;
1179 }
1180 
1181 /*
1182  * The background pageout daemon, started as a kernel thread
1183  * from the init process.
1184  *
1185  * This basically trickles out pages so that we have _some_
1186  * free memory available even if there is no other activity
1187  * that frees anything up. This is needed for things like routing
1188  * etc, where we otherwise might have all activity going on in
1189  * asynchronous contexts that cannot page things out.
1190  *
1191  * If there are applications that are active memory-allocators
1192  * (most normal use), this basically shouldn't matter.
1193  */
1194 static int kswapd(void *p)
1195 {
1196 	unsigned long order;
1197 	pg_data_t *pgdat = (pg_data_t*)p;
1198 	struct task_struct *tsk = current;
1199 	DEFINE_WAIT(wait);
1200 	struct reclaim_state reclaim_state = {
1201 		.reclaimed_slab = 0,
1202 	};
1203 	cpumask_t cpumask;
1204 
1205 	daemonize("kswapd%d", pgdat->node_id);
1206 	cpumask = node_to_cpumask(pgdat->node_id);
1207 	if (!cpus_empty(cpumask))
1208 		set_cpus_allowed(tsk, cpumask);
1209 	current->reclaim_state = &reclaim_state;
1210 
1211 	/*
1212 	 * Tell the memory management that we're a "memory allocator",
1213 	 * and that if we need more memory we should get access to it
1214 	 * regardless (see "__alloc_pages()"). "kswapd" should
1215 	 * never get caught in the normal page freeing logic.
1216 	 *
1217 	 * (Kswapd normally doesn't need memory anyway, but sometimes
1218 	 * you need a small amount of memory in order to be able to
1219 	 * page out something else, and this flag essentially protects
1220 	 * us from recursively trying to free more memory as we're
1221 	 * trying to free the first piece of memory in the first place).
1222 	 */
1223 	tsk->flags |= PF_MEMALLOC|PF_KSWAPD;
1224 
1225 	order = 0;
1226 	for ( ; ; ) {
1227 		unsigned long new_order;
1228 
1229 		try_to_freeze();
1230 
1231 		prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
1232 		new_order = pgdat->kswapd_max_order;
1233 		pgdat->kswapd_max_order = 0;
1234 		if (order < new_order) {
1235 			/*
1236 			 * Don't sleep if someone wants a larger 'order'
1237 			 * allocation
1238 			 */
1239 			order = new_order;
1240 		} else {
1241 			schedule();
1242 			order = pgdat->kswapd_max_order;
1243 		}
1244 		finish_wait(&pgdat->kswapd_wait, &wait);
1245 
1246 		balance_pgdat(pgdat, 0, order);
1247 	}
1248 	return 0;
1249 }
1250 
1251 /*
1252  * A zone is low on free memory, so wake its kswapd task to service it.
1253  */
1254 void wakeup_kswapd(struct zone *zone, int order)
1255 {
1256 	pg_data_t *pgdat;
1257 
1258 	if (zone->present_pages == 0)
1259 		return;
1260 
1261 	pgdat = zone->zone_pgdat;
1262 	if (zone_watermark_ok(zone, order, zone->pages_low, 0, 0, 0))
1263 		return;
1264 	if (pgdat->kswapd_max_order < order)
1265 		pgdat->kswapd_max_order = order;
1266 	if (!cpuset_zone_allowed(zone, __GFP_HARDWALL))
1267 		return;
1268 	if (!waitqueue_active(&pgdat->kswapd_wait))
1269 		return;
1270 	wake_up_interruptible(&pgdat->kswapd_wait);
1271 }
1272 
1273 #ifdef CONFIG_PM
1274 /*
1275  * Try to free `nr_pages' of memory, system-wide.  Returns the number of freed
1276  * pages.
1277  */
1278 int shrink_all_memory(int nr_pages)
1279 {
1280 	pg_data_t *pgdat;
1281 	int nr_to_free = nr_pages;
1282 	int ret = 0;
1283 	struct reclaim_state reclaim_state = {
1284 		.reclaimed_slab = 0,
1285 	};
1286 
1287 	current->reclaim_state = &reclaim_state;
1288 	for_each_pgdat(pgdat) {
1289 		int freed;
1290 		freed = balance_pgdat(pgdat, nr_to_free, 0);
1291 		ret += freed;
1292 		nr_to_free -= freed;
1293 		if (nr_to_free <= 0)
1294 			break;
1295 	}
1296 	current->reclaim_state = NULL;
1297 	return ret;
1298 }
1299 #endif
1300 
1301 #ifdef CONFIG_HOTPLUG_CPU
1302 /* It's optimal to keep kswapds on the same CPUs as their memory, but
1303    not required for correctness.  So if the last cpu in a node goes
1304    away, we get changed to run anywhere: as the first one comes back,
1305    restore their cpu bindings. */
1306 static int __devinit cpu_callback(struct notifier_block *nfb,
1307 				  unsigned long action,
1308 				  void *hcpu)
1309 {
1310 	pg_data_t *pgdat;
1311 	cpumask_t mask;
1312 
1313 	if (action == CPU_ONLINE) {
1314 		for_each_pgdat(pgdat) {
1315 			mask = node_to_cpumask(pgdat->node_id);
1316 			if (any_online_cpu(mask) != NR_CPUS)
1317 				/* One of our CPUs online: restore mask */
1318 				set_cpus_allowed(pgdat->kswapd, mask);
1319 		}
1320 	}
1321 	return NOTIFY_OK;
1322 }
1323 #endif /* CONFIG_HOTPLUG_CPU */
1324 
1325 static int __init kswapd_init(void)
1326 {
1327 	pg_data_t *pgdat;
1328 	swap_setup();
1329 	for_each_pgdat(pgdat)
1330 		pgdat->kswapd
1331 		= find_task_by_pid(kernel_thread(kswapd, pgdat, CLONE_KERNEL));
1332 	total_memory = nr_free_pagecache_pages();
1333 	hotcpu_notifier(cpu_callback, 0);
1334 	return 0;
1335 }
1336 
1337 module_init(kswapd_init)
1338 
1339 
1340 /*
1341  * Try to free up some pages from this zone through reclaim.
1342  */
1343 int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
1344 {
1345 	struct scan_control sc;
1346 	int nr_pages = 1 << order;
1347 	int total_reclaimed = 0;
1348 
1349 	/* The reclaim may sleep, so don't do it if sleep isn't allowed */
1350 	if (!(gfp_mask & __GFP_WAIT))
1351 		return 0;
1352 	if (zone->all_unreclaimable)
1353 		return 0;
1354 
1355 	sc.gfp_mask = gfp_mask;
1356 	sc.may_writepage = 0;
1357 	sc.may_swap = 0;
1358 	sc.nr_mapped = read_page_state(nr_mapped);
1359 	sc.nr_scanned = 0;
1360 	sc.nr_reclaimed = 0;
1361 	/* scan at the highest priority */
1362 	sc.priority = 0;
1363 
1364 	if (nr_pages > SWAP_CLUSTER_MAX)
1365 		sc.swap_cluster_max = nr_pages;
1366 	else
1367 		sc.swap_cluster_max = SWAP_CLUSTER_MAX;
1368 
1369 	/* Don't reclaim the zone if there are other reclaimers active */
1370 	if (atomic_read(&zone->reclaim_in_progress) > 0)
1371 		goto out;
1372 
1373 	shrink_zone(zone, &sc);
1374 	total_reclaimed = sc.nr_reclaimed;
1375 
1376  out:
1377 	return total_reclaimed;
1378 }
1379 
1380 asmlinkage long sys_set_zone_reclaim(unsigned int node, unsigned int zone,
1381 				     unsigned int state)
1382 {
1383 	struct zone *z;
1384 	int i;
1385 
1386 	if (!capable(CAP_SYS_ADMIN))
1387 		return -EACCES;
1388 
1389 	if (node >= MAX_NUMNODES || !node_online(node))
1390 		return -EINVAL;
1391 
1392 	/* This will break if we ever add more zones */
1393 	if (!(zone & (1<<ZONE_DMA|1<<ZONE_NORMAL|1<<ZONE_HIGHMEM)))
1394 		return -EINVAL;
1395 
1396 	for (i = 0; i < MAX_NR_ZONES; i++) {
1397 		if (!(zone & 1<<i))
1398 			continue;
1399 
1400 		z = &NODE_DATA(node)->node_zones[i];
1401 
1402 		if (state)
1403 			z->reclaim_pages = 1;
1404 		else
1405 			z->reclaim_pages = 0;
1406 	}
1407 
1408 	return 0;
1409 }
1410