xref: /openbmc/linux/mm/swap.c (revision f7018c21)
1  /*
2   *  linux/mm/swap.c
3   *
4   *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
5   */
6  
7  /*
8   * This file contains the default values for the operation of the
9   * Linux VM subsystem. Fine-tuning documentation can be found in
10   * Documentation/sysctl/vm.txt.
11   * Started 18.12.91
12   * Swap aging added 23.2.95, Stephen Tweedie.
13   * Buffermem limits added 12.3.98, Rik van Riel.
14   */
15  
16  #include <linux/mm.h>
17  #include <linux/sched.h>
18  #include <linux/kernel_stat.h>
19  #include <linux/swap.h>
20  #include <linux/mman.h>
21  #include <linux/pagemap.h>
22  #include <linux/pagevec.h>
23  #include <linux/init.h>
24  #include <linux/export.h>
25  #include <linux/mm_inline.h>
26  #include <linux/percpu_counter.h>
27  #include <linux/percpu.h>
28  #include <linux/cpu.h>
29  #include <linux/notifier.h>
30  #include <linux/backing-dev.h>
31  #include <linux/memcontrol.h>
32  #include <linux/gfp.h>
33  #include <linux/uio.h>
34  
35  #include "internal.h"
36  
37  #define CREATE_TRACE_POINTS
38  #include <trace/events/pagemap.h>
39  
40  /* How many pages do we try to swap or page in/out together? */
41  int page_cluster;
42  
43  static DEFINE_PER_CPU(struct pagevec, lru_add_pvec);
44  static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
45  static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
46  
47  /*
48   * This path almost never happens for VM activity - pages are normally
49   * freed via pagevecs.  But it gets used by networking.
50   */
51  static void __page_cache_release(struct page *page)
52  {
53  	if (PageLRU(page)) {
54  		struct zone *zone = page_zone(page);
55  		struct lruvec *lruvec;
56  		unsigned long flags;
57  
58  		spin_lock_irqsave(&zone->lru_lock, flags);
59  		lruvec = mem_cgroup_page_lruvec(page, zone);
60  		VM_BUG_ON_PAGE(!PageLRU(page), page);
61  		__ClearPageLRU(page);
62  		del_page_from_lru_list(page, lruvec, page_off_lru(page));
63  		spin_unlock_irqrestore(&zone->lru_lock, flags);
64  	}
65  }
66  
67  static void __put_single_page(struct page *page)
68  {
69  	__page_cache_release(page);
70  	free_hot_cold_page(page, 0);
71  }
72  
73  static void __put_compound_page(struct page *page)
74  {
75  	compound_page_dtor *dtor;
76  
77  	__page_cache_release(page);
78  	dtor = get_compound_page_dtor(page);
79  	(*dtor)(page);
80  }
81  
82  static void put_compound_page(struct page *page)
83  {
84  	struct page *page_head;
85  
86  	if (likely(!PageTail(page))) {
87  		if (put_page_testzero(page)) {
88  			/*
89  			 * By the time all refcounts have been released
90  			 * split_huge_page cannot run anymore from under us.
91  			 */
92  			if (PageHead(page))
93  				__put_compound_page(page);
94  			else
95  				__put_single_page(page);
96  		}
97  		return;
98  	}
99  
100  	/* __split_huge_page_refcount can run under us */
101  	page_head = compound_head(page);
102  
103  	/*
104  	 * THP can not break up slab pages so avoid taking
105  	 * compound_lock() and skip the tail page refcounting (in
106  	 * _mapcount) too. Slab performs non-atomic bit ops on
107  	 * page->flags for better performance. In particular
108  	 * slab_unlock() in slub used to be a hot path. It is still
109  	 * hot on arches that do not support
110  	 * this_cpu_cmpxchg_double().
111  	 *
112  	 * If "page" is part of a slab or hugetlbfs page it cannot be
113  	 * splitted and the head page cannot change from under us. And
114  	 * if "page" is part of a THP page under splitting, if the
115  	 * head page pointed by the THP tail isn't a THP head anymore,
116  	 * we'll find PageTail clear after smp_rmb() and we'll treat
117  	 * it as a single page.
118  	 */
119  	if (!__compound_tail_refcounted(page_head)) {
120  		/*
121  		 * If "page" is a THP tail, we must read the tail page
122  		 * flags after the head page flags. The
123  		 * split_huge_page side enforces write memory barriers
124  		 * between clearing PageTail and before the head page
125  		 * can be freed and reallocated.
126  		 */
127  		smp_rmb();
128  		if (likely(PageTail(page))) {
129  			/*
130  			 * __split_huge_page_refcount cannot race
131  			 * here.
132  			 */
133  			VM_BUG_ON_PAGE(!PageHead(page_head), page_head);
134  			VM_BUG_ON_PAGE(page_mapcount(page) != 0, page);
135  			if (put_page_testzero(page_head)) {
136  				/*
137  				 * If this is the tail of a slab
138  				 * compound page, the tail pin must
139  				 * not be the last reference held on
140  				 * the page, because the PG_slab
141  				 * cannot be cleared before all tail
142  				 * pins (which skips the _mapcount
143  				 * tail refcounting) have been
144  				 * released. For hugetlbfs the tail
145  				 * pin may be the last reference on
146  				 * the page instead, because
147  				 * PageHeadHuge will not go away until
148  				 * the compound page enters the buddy
149  				 * allocator.
150  				 */
151  				VM_BUG_ON_PAGE(PageSlab(page_head), page_head);
152  				__put_compound_page(page_head);
153  			}
154  			return;
155  		} else
156  			/*
157  			 * __split_huge_page_refcount run before us,
158  			 * "page" was a THP tail. The split page_head
159  			 * has been freed and reallocated as slab or
160  			 * hugetlbfs page of smaller order (only
161  			 * possible if reallocated as slab on x86).
162  			 */
163  			goto out_put_single;
164  	}
165  
166  	if (likely(page != page_head && get_page_unless_zero(page_head))) {
167  		unsigned long flags;
168  
169  		/*
170  		 * page_head wasn't a dangling pointer but it may not
171  		 * be a head page anymore by the time we obtain the
172  		 * lock. That is ok as long as it can't be freed from
173  		 * under us.
174  		 */
175  		flags = compound_lock_irqsave(page_head);
176  		if (unlikely(!PageTail(page))) {
177  			/* __split_huge_page_refcount run before us */
178  			compound_unlock_irqrestore(page_head, flags);
179  			if (put_page_testzero(page_head)) {
180  				/*
181  				 * The head page may have been freed
182  				 * and reallocated as a compound page
183  				 * of smaller order and then freed
184  				 * again.  All we know is that it
185  				 * cannot have become: a THP page, a
186  				 * compound page of higher order, a
187  				 * tail page.  That is because we
188  				 * still hold the refcount of the
189  				 * split THP tail and page_head was
190  				 * the THP head before the split.
191  				 */
192  				if (PageHead(page_head))
193  					__put_compound_page(page_head);
194  				else
195  					__put_single_page(page_head);
196  			}
197  out_put_single:
198  			if (put_page_testzero(page))
199  				__put_single_page(page);
200  			return;
201  		}
202  		VM_BUG_ON_PAGE(page_head != page->first_page, page);
203  		/*
204  		 * We can release the refcount taken by
205  		 * get_page_unless_zero() now that
206  		 * __split_huge_page_refcount() is blocked on the
207  		 * compound_lock.
208  		 */
209  		if (put_page_testzero(page_head))
210  			VM_BUG_ON_PAGE(1, page_head);
211  		/* __split_huge_page_refcount will wait now */
212  		VM_BUG_ON_PAGE(page_mapcount(page) <= 0, page);
213  		atomic_dec(&page->_mapcount);
214  		VM_BUG_ON_PAGE(atomic_read(&page_head->_count) <= 0, page_head);
215  		VM_BUG_ON_PAGE(atomic_read(&page->_count) != 0, page);
216  		compound_unlock_irqrestore(page_head, flags);
217  
218  		if (put_page_testzero(page_head)) {
219  			if (PageHead(page_head))
220  				__put_compound_page(page_head);
221  			else
222  				__put_single_page(page_head);
223  		}
224  	} else {
225  		/* page_head is a dangling pointer */
226  		VM_BUG_ON_PAGE(PageTail(page), page);
227  		goto out_put_single;
228  	}
229  }
230  
231  void put_page(struct page *page)
232  {
233  	if (unlikely(PageCompound(page)))
234  		put_compound_page(page);
235  	else if (put_page_testzero(page))
236  		__put_single_page(page);
237  }
238  EXPORT_SYMBOL(put_page);
239  
240  /*
241   * This function is exported but must not be called by anything other
242   * than get_page(). It implements the slow path of get_page().
243   */
244  bool __get_page_tail(struct page *page)
245  {
246  	/*
247  	 * This takes care of get_page() if run on a tail page
248  	 * returned by one of the get_user_pages/follow_page variants.
249  	 * get_user_pages/follow_page itself doesn't need the compound
250  	 * lock because it runs __get_page_tail_foll() under the
251  	 * proper PT lock that already serializes against
252  	 * split_huge_page().
253  	 */
254  	unsigned long flags;
255  	bool got;
256  	struct page *page_head = compound_head(page);
257  
258  	/* Ref to put_compound_page() comment. */
259  	if (!__compound_tail_refcounted(page_head)) {
260  		smp_rmb();
261  		if (likely(PageTail(page))) {
262  			/*
263  			 * This is a hugetlbfs page or a slab
264  			 * page. __split_huge_page_refcount
265  			 * cannot race here.
266  			 */
267  			VM_BUG_ON_PAGE(!PageHead(page_head), page_head);
268  			__get_page_tail_foll(page, true);
269  			return true;
270  		} else {
271  			/*
272  			 * __split_huge_page_refcount run
273  			 * before us, "page" was a THP
274  			 * tail. The split page_head has been
275  			 * freed and reallocated as slab or
276  			 * hugetlbfs page of smaller order
277  			 * (only possible if reallocated as
278  			 * slab on x86).
279  			 */
280  			return false;
281  		}
282  	}
283  
284  	got = false;
285  	if (likely(page != page_head && get_page_unless_zero(page_head))) {
286  		/*
287  		 * page_head wasn't a dangling pointer but it
288  		 * may not be a head page anymore by the time
289  		 * we obtain the lock. That is ok as long as it
290  		 * can't be freed from under us.
291  		 */
292  		flags = compound_lock_irqsave(page_head);
293  		/* here __split_huge_page_refcount won't run anymore */
294  		if (likely(PageTail(page))) {
295  			__get_page_tail_foll(page, false);
296  			got = true;
297  		}
298  		compound_unlock_irqrestore(page_head, flags);
299  		if (unlikely(!got))
300  			put_page(page_head);
301  	}
302  	return got;
303  }
304  EXPORT_SYMBOL(__get_page_tail);
305  
306  /**
307   * put_pages_list() - release a list of pages
308   * @pages: list of pages threaded on page->lru
309   *
310   * Release a list of pages which are strung together on page.lru.  Currently
311   * used by read_cache_pages() and related error recovery code.
312   */
313  void put_pages_list(struct list_head *pages)
314  {
315  	while (!list_empty(pages)) {
316  		struct page *victim;
317  
318  		victim = list_entry(pages->prev, struct page, lru);
319  		list_del(&victim->lru);
320  		page_cache_release(victim);
321  	}
322  }
323  EXPORT_SYMBOL(put_pages_list);
324  
325  /*
326   * get_kernel_pages() - pin kernel pages in memory
327   * @kiov:	An array of struct kvec structures
328   * @nr_segs:	number of segments to pin
329   * @write:	pinning for read/write, currently ignored
330   * @pages:	array that receives pointers to the pages pinned.
331   *		Should be at least nr_segs long.
332   *
333   * Returns number of pages pinned. This may be fewer than the number
334   * requested. If nr_pages is 0 or negative, returns 0. If no pages
335   * were pinned, returns -errno. Each page returned must be released
336   * with a put_page() call when it is finished with.
337   */
338  int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write,
339  		struct page **pages)
340  {
341  	int seg;
342  
343  	for (seg = 0; seg < nr_segs; seg++) {
344  		if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE))
345  			return seg;
346  
347  		pages[seg] = kmap_to_page(kiov[seg].iov_base);
348  		page_cache_get(pages[seg]);
349  	}
350  
351  	return seg;
352  }
353  EXPORT_SYMBOL_GPL(get_kernel_pages);
354  
355  /*
356   * get_kernel_page() - pin a kernel page in memory
357   * @start:	starting kernel address
358   * @write:	pinning for read/write, currently ignored
359   * @pages:	array that receives pointer to the page pinned.
360   *		Must be at least nr_segs long.
361   *
362   * Returns 1 if page is pinned. If the page was not pinned, returns
363   * -errno. The page returned must be released with a put_page() call
364   * when it is finished with.
365   */
366  int get_kernel_page(unsigned long start, int write, struct page **pages)
367  {
368  	const struct kvec kiov = {
369  		.iov_base = (void *)start,
370  		.iov_len = PAGE_SIZE
371  	};
372  
373  	return get_kernel_pages(&kiov, 1, write, pages);
374  }
375  EXPORT_SYMBOL_GPL(get_kernel_page);
376  
377  static void pagevec_lru_move_fn(struct pagevec *pvec,
378  	void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg),
379  	void *arg)
380  {
381  	int i;
382  	struct zone *zone = NULL;
383  	struct lruvec *lruvec;
384  	unsigned long flags = 0;
385  
386  	for (i = 0; i < pagevec_count(pvec); i++) {
387  		struct page *page = pvec->pages[i];
388  		struct zone *pagezone = page_zone(page);
389  
390  		if (pagezone != zone) {
391  			if (zone)
392  				spin_unlock_irqrestore(&zone->lru_lock, flags);
393  			zone = pagezone;
394  			spin_lock_irqsave(&zone->lru_lock, flags);
395  		}
396  
397  		lruvec = mem_cgroup_page_lruvec(page, zone);
398  		(*move_fn)(page, lruvec, arg);
399  	}
400  	if (zone)
401  		spin_unlock_irqrestore(&zone->lru_lock, flags);
402  	release_pages(pvec->pages, pvec->nr, pvec->cold);
403  	pagevec_reinit(pvec);
404  }
405  
406  static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec,
407  				 void *arg)
408  {
409  	int *pgmoved = arg;
410  
411  	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
412  		enum lru_list lru = page_lru_base_type(page);
413  		list_move_tail(&page->lru, &lruvec->lists[lru]);
414  		(*pgmoved)++;
415  	}
416  }
417  
418  /*
419   * pagevec_move_tail() must be called with IRQ disabled.
420   * Otherwise this may cause nasty races.
421   */
422  static void pagevec_move_tail(struct pagevec *pvec)
423  {
424  	int pgmoved = 0;
425  
426  	pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved);
427  	__count_vm_events(PGROTATED, pgmoved);
428  }
429  
430  /*
431   * Writeback is about to end against a page which has been marked for immediate
432   * reclaim.  If it still appears to be reclaimable, move it to the tail of the
433   * inactive list.
434   */
435  void rotate_reclaimable_page(struct page *page)
436  {
437  	if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) &&
438  	    !PageUnevictable(page) && PageLRU(page)) {
439  		struct pagevec *pvec;
440  		unsigned long flags;
441  
442  		page_cache_get(page);
443  		local_irq_save(flags);
444  		pvec = &__get_cpu_var(lru_rotate_pvecs);
445  		if (!pagevec_add(pvec, page))
446  			pagevec_move_tail(pvec);
447  		local_irq_restore(flags);
448  	}
449  }
450  
451  static void update_page_reclaim_stat(struct lruvec *lruvec,
452  				     int file, int rotated)
453  {
454  	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
455  
456  	reclaim_stat->recent_scanned[file]++;
457  	if (rotated)
458  		reclaim_stat->recent_rotated[file]++;
459  }
460  
461  static void __activate_page(struct page *page, struct lruvec *lruvec,
462  			    void *arg)
463  {
464  	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
465  		int file = page_is_file_cache(page);
466  		int lru = page_lru_base_type(page);
467  
468  		del_page_from_lru_list(page, lruvec, lru);
469  		SetPageActive(page);
470  		lru += LRU_ACTIVE;
471  		add_page_to_lru_list(page, lruvec, lru);
472  		trace_mm_lru_activate(page, page_to_pfn(page));
473  
474  		__count_vm_event(PGACTIVATE);
475  		update_page_reclaim_stat(lruvec, file, 1);
476  	}
477  }
478  
479  #ifdef CONFIG_SMP
480  static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
481  
482  static void activate_page_drain(int cpu)
483  {
484  	struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu);
485  
486  	if (pagevec_count(pvec))
487  		pagevec_lru_move_fn(pvec, __activate_page, NULL);
488  }
489  
490  static bool need_activate_page_drain(int cpu)
491  {
492  	return pagevec_count(&per_cpu(activate_page_pvecs, cpu)) != 0;
493  }
494  
495  void activate_page(struct page *page)
496  {
497  	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
498  		struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);
499  
500  		page_cache_get(page);
501  		if (!pagevec_add(pvec, page))
502  			pagevec_lru_move_fn(pvec, __activate_page, NULL);
503  		put_cpu_var(activate_page_pvecs);
504  	}
505  }
506  
507  #else
508  static inline void activate_page_drain(int cpu)
509  {
510  }
511  
512  static bool need_activate_page_drain(int cpu)
513  {
514  	return false;
515  }
516  
517  void activate_page(struct page *page)
518  {
519  	struct zone *zone = page_zone(page);
520  
521  	spin_lock_irq(&zone->lru_lock);
522  	__activate_page(page, mem_cgroup_page_lruvec(page, zone), NULL);
523  	spin_unlock_irq(&zone->lru_lock);
524  }
525  #endif
526  
527  static void __lru_cache_activate_page(struct page *page)
528  {
529  	struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
530  	int i;
531  
532  	/*
533  	 * Search backwards on the optimistic assumption that the page being
534  	 * activated has just been added to this pagevec. Note that only
535  	 * the local pagevec is examined as a !PageLRU page could be in the
536  	 * process of being released, reclaimed, migrated or on a remote
537  	 * pagevec that is currently being drained. Furthermore, marking
538  	 * a remote pagevec's page PageActive potentially hits a race where
539  	 * a page is marked PageActive just after it is added to the inactive
540  	 * list causing accounting errors and BUG_ON checks to trigger.
541  	 */
542  	for (i = pagevec_count(pvec) - 1; i >= 0; i--) {
543  		struct page *pagevec_page = pvec->pages[i];
544  
545  		if (pagevec_page == page) {
546  			SetPageActive(page);
547  			break;
548  		}
549  	}
550  
551  	put_cpu_var(lru_add_pvec);
552  }
553  
554  /*
555   * Mark a page as having seen activity.
556   *
557   * inactive,unreferenced	->	inactive,referenced
558   * inactive,referenced		->	active,unreferenced
559   * active,unreferenced		->	active,referenced
560   */
561  void mark_page_accessed(struct page *page)
562  {
563  	if (!PageActive(page) && !PageUnevictable(page) &&
564  			PageReferenced(page)) {
565  
566  		/*
567  		 * If the page is on the LRU, queue it for activation via
568  		 * activate_page_pvecs. Otherwise, assume the page is on a
569  		 * pagevec, mark it active and it'll be moved to the active
570  		 * LRU on the next drain.
571  		 */
572  		if (PageLRU(page))
573  			activate_page(page);
574  		else
575  			__lru_cache_activate_page(page);
576  		ClearPageReferenced(page);
577  		if (page_is_file_cache(page))
578  			workingset_activation(page);
579  	} else if (!PageReferenced(page)) {
580  		SetPageReferenced(page);
581  	}
582  }
583  EXPORT_SYMBOL(mark_page_accessed);
584  
585  /*
586   * Queue the page for addition to the LRU via pagevec. The decision on whether
587   * to add the page to the [in]active [file|anon] list is deferred until the
588   * pagevec is drained. This gives a chance for the caller of __lru_cache_add()
589   * have the page added to the active list using mark_page_accessed().
590   */
591  void __lru_cache_add(struct page *page)
592  {
593  	struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
594  
595  	page_cache_get(page);
596  	if (!pagevec_space(pvec))
597  		__pagevec_lru_add(pvec);
598  	pagevec_add(pvec, page);
599  	put_cpu_var(lru_add_pvec);
600  }
601  EXPORT_SYMBOL(__lru_cache_add);
602  
603  /**
604   * lru_cache_add - add a page to a page list
605   * @page: the page to be added to the LRU.
606   */
607  void lru_cache_add(struct page *page)
608  {
609  	VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page);
610  	VM_BUG_ON_PAGE(PageLRU(page), page);
611  	__lru_cache_add(page);
612  }
613  
614  /**
615   * add_page_to_unevictable_list - add a page to the unevictable list
616   * @page:  the page to be added to the unevictable list
617   *
618   * Add page directly to its zone's unevictable list.  To avoid races with
619   * tasks that might be making the page evictable, through eg. munlock,
620   * munmap or exit, while it's not on the lru, we want to add the page
621   * while it's locked or otherwise "invisible" to other tasks.  This is
622   * difficult to do when using the pagevec cache, so bypass that.
623   */
624  void add_page_to_unevictable_list(struct page *page)
625  {
626  	struct zone *zone = page_zone(page);
627  	struct lruvec *lruvec;
628  
629  	spin_lock_irq(&zone->lru_lock);
630  	lruvec = mem_cgroup_page_lruvec(page, zone);
631  	ClearPageActive(page);
632  	SetPageUnevictable(page);
633  	SetPageLRU(page);
634  	add_page_to_lru_list(page, lruvec, LRU_UNEVICTABLE);
635  	spin_unlock_irq(&zone->lru_lock);
636  }
637  
638  /*
639   * If the page can not be invalidated, it is moved to the
640   * inactive list to speed up its reclaim.  It is moved to the
641   * head of the list, rather than the tail, to give the flusher
642   * threads some time to write it out, as this is much more
643   * effective than the single-page writeout from reclaim.
644   *
645   * If the page isn't page_mapped and dirty/writeback, the page
646   * could reclaim asap using PG_reclaim.
647   *
648   * 1. active, mapped page -> none
649   * 2. active, dirty/writeback page -> inactive, head, PG_reclaim
650   * 3. inactive, mapped page -> none
651   * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim
652   * 5. inactive, clean -> inactive, tail
653   * 6. Others -> none
654   *
655   * In 4, why it moves inactive's head, the VM expects the page would
656   * be write it out by flusher threads as this is much more effective
657   * than the single-page writeout from reclaim.
658   */
659  static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,
660  			      void *arg)
661  {
662  	int lru, file;
663  	bool active;
664  
665  	if (!PageLRU(page))
666  		return;
667  
668  	if (PageUnevictable(page))
669  		return;
670  
671  	/* Some processes are using the page */
672  	if (page_mapped(page))
673  		return;
674  
675  	active = PageActive(page);
676  	file = page_is_file_cache(page);
677  	lru = page_lru_base_type(page);
678  
679  	del_page_from_lru_list(page, lruvec, lru + active);
680  	ClearPageActive(page);
681  	ClearPageReferenced(page);
682  	add_page_to_lru_list(page, lruvec, lru);
683  
684  	if (PageWriteback(page) || PageDirty(page)) {
685  		/*
686  		 * PG_reclaim could be raced with end_page_writeback
687  		 * It can make readahead confusing.  But race window
688  		 * is _really_ small and  it's non-critical problem.
689  		 */
690  		SetPageReclaim(page);
691  	} else {
692  		/*
693  		 * The page's writeback ends up during pagevec
694  		 * We moves tha page into tail of inactive.
695  		 */
696  		list_move_tail(&page->lru, &lruvec->lists[lru]);
697  		__count_vm_event(PGROTATED);
698  	}
699  
700  	if (active)
701  		__count_vm_event(PGDEACTIVATE);
702  	update_page_reclaim_stat(lruvec, file, 0);
703  }
704  
705  /*
706   * Drain pages out of the cpu's pagevecs.
707   * Either "cpu" is the current CPU, and preemption has already been
708   * disabled; or "cpu" is being hot-unplugged, and is already dead.
709   */
710  void lru_add_drain_cpu(int cpu)
711  {
712  	struct pagevec *pvec = &per_cpu(lru_add_pvec, cpu);
713  
714  	if (pagevec_count(pvec))
715  		__pagevec_lru_add(pvec);
716  
717  	pvec = &per_cpu(lru_rotate_pvecs, cpu);
718  	if (pagevec_count(pvec)) {
719  		unsigned long flags;
720  
721  		/* No harm done if a racing interrupt already did this */
722  		local_irq_save(flags);
723  		pagevec_move_tail(pvec);
724  		local_irq_restore(flags);
725  	}
726  
727  	pvec = &per_cpu(lru_deactivate_pvecs, cpu);
728  	if (pagevec_count(pvec))
729  		pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
730  
731  	activate_page_drain(cpu);
732  }
733  
734  /**
735   * deactivate_page - forcefully deactivate a page
736   * @page: page to deactivate
737   *
738   * This function hints the VM that @page is a good reclaim candidate,
739   * for example if its invalidation fails due to the page being dirty
740   * or under writeback.
741   */
742  void deactivate_page(struct page *page)
743  {
744  	/*
745  	 * In a workload with many unevictable page such as mprotect, unevictable
746  	 * page deactivation for accelerating reclaim is pointless.
747  	 */
748  	if (PageUnevictable(page))
749  		return;
750  
751  	if (likely(get_page_unless_zero(page))) {
752  		struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
753  
754  		if (!pagevec_add(pvec, page))
755  			pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
756  		put_cpu_var(lru_deactivate_pvecs);
757  	}
758  }
759  
760  void lru_add_drain(void)
761  {
762  	lru_add_drain_cpu(get_cpu());
763  	put_cpu();
764  }
765  
766  static void lru_add_drain_per_cpu(struct work_struct *dummy)
767  {
768  	lru_add_drain();
769  }
770  
771  static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
772  
773  void lru_add_drain_all(void)
774  {
775  	static DEFINE_MUTEX(lock);
776  	static struct cpumask has_work;
777  	int cpu;
778  
779  	mutex_lock(&lock);
780  	get_online_cpus();
781  	cpumask_clear(&has_work);
782  
783  	for_each_online_cpu(cpu) {
784  		struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
785  
786  		if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) ||
787  		    pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
788  		    pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) ||
789  		    need_activate_page_drain(cpu)) {
790  			INIT_WORK(work, lru_add_drain_per_cpu);
791  			schedule_work_on(cpu, work);
792  			cpumask_set_cpu(cpu, &has_work);
793  		}
794  	}
795  
796  	for_each_cpu(cpu, &has_work)
797  		flush_work(&per_cpu(lru_add_drain_work, cpu));
798  
799  	put_online_cpus();
800  	mutex_unlock(&lock);
801  }
802  
803  /*
804   * Batched page_cache_release().  Decrement the reference count on all the
805   * passed pages.  If it fell to zero then remove the page from the LRU and
806   * free it.
807   *
808   * Avoid taking zone->lru_lock if possible, but if it is taken, retain it
809   * for the remainder of the operation.
810   *
811   * The locking in this function is against shrink_inactive_list(): we recheck
812   * the page count inside the lock to see whether shrink_inactive_list()
813   * grabbed the page via the LRU.  If it did, give up: shrink_inactive_list()
814   * will free it.
815   */
816  void release_pages(struct page **pages, int nr, int cold)
817  {
818  	int i;
819  	LIST_HEAD(pages_to_free);
820  	struct zone *zone = NULL;
821  	struct lruvec *lruvec;
822  	unsigned long uninitialized_var(flags);
823  
824  	for (i = 0; i < nr; i++) {
825  		struct page *page = pages[i];
826  
827  		if (unlikely(PageCompound(page))) {
828  			if (zone) {
829  				spin_unlock_irqrestore(&zone->lru_lock, flags);
830  				zone = NULL;
831  			}
832  			put_compound_page(page);
833  			continue;
834  		}
835  
836  		if (!put_page_testzero(page))
837  			continue;
838  
839  		if (PageLRU(page)) {
840  			struct zone *pagezone = page_zone(page);
841  
842  			if (pagezone != zone) {
843  				if (zone)
844  					spin_unlock_irqrestore(&zone->lru_lock,
845  									flags);
846  				zone = pagezone;
847  				spin_lock_irqsave(&zone->lru_lock, flags);
848  			}
849  
850  			lruvec = mem_cgroup_page_lruvec(page, zone);
851  			VM_BUG_ON_PAGE(!PageLRU(page), page);
852  			__ClearPageLRU(page);
853  			del_page_from_lru_list(page, lruvec, page_off_lru(page));
854  		}
855  
856  		/* Clear Active bit in case of parallel mark_page_accessed */
857  		ClearPageActive(page);
858  
859  		list_add(&page->lru, &pages_to_free);
860  	}
861  	if (zone)
862  		spin_unlock_irqrestore(&zone->lru_lock, flags);
863  
864  	free_hot_cold_page_list(&pages_to_free, cold);
865  }
866  EXPORT_SYMBOL(release_pages);
867  
868  /*
869   * The pages which we're about to release may be in the deferred lru-addition
870   * queues.  That would prevent them from really being freed right now.  That's
871   * OK from a correctness point of view but is inefficient - those pages may be
872   * cache-warm and we want to give them back to the page allocator ASAP.
873   *
874   * So __pagevec_release() will drain those queues here.  __pagevec_lru_add()
875   * and __pagevec_lru_add_active() call release_pages() directly to avoid
876   * mutual recursion.
877   */
878  void __pagevec_release(struct pagevec *pvec)
879  {
880  	lru_add_drain();
881  	release_pages(pvec->pages, pagevec_count(pvec), pvec->cold);
882  	pagevec_reinit(pvec);
883  }
884  EXPORT_SYMBOL(__pagevec_release);
885  
886  #ifdef CONFIG_TRANSPARENT_HUGEPAGE
887  /* used by __split_huge_page_refcount() */
888  void lru_add_page_tail(struct page *page, struct page *page_tail,
889  		       struct lruvec *lruvec, struct list_head *list)
890  {
891  	const int file = 0;
892  
893  	VM_BUG_ON_PAGE(!PageHead(page), page);
894  	VM_BUG_ON_PAGE(PageCompound(page_tail), page);
895  	VM_BUG_ON_PAGE(PageLRU(page_tail), page);
896  	VM_BUG_ON(NR_CPUS != 1 &&
897  		  !spin_is_locked(&lruvec_zone(lruvec)->lru_lock));
898  
899  	if (!list)
900  		SetPageLRU(page_tail);
901  
902  	if (likely(PageLRU(page)))
903  		list_add_tail(&page_tail->lru, &page->lru);
904  	else if (list) {
905  		/* page reclaim is reclaiming a huge page */
906  		get_page(page_tail);
907  		list_add_tail(&page_tail->lru, list);
908  	} else {
909  		struct list_head *list_head;
910  		/*
911  		 * Head page has not yet been counted, as an hpage,
912  		 * so we must account for each subpage individually.
913  		 *
914  		 * Use the standard add function to put page_tail on the list,
915  		 * but then correct its position so they all end up in order.
916  		 */
917  		add_page_to_lru_list(page_tail, lruvec, page_lru(page_tail));
918  		list_head = page_tail->lru.prev;
919  		list_move_tail(&page_tail->lru, list_head);
920  	}
921  
922  	if (!PageUnevictable(page))
923  		update_page_reclaim_stat(lruvec, file, PageActive(page_tail));
924  }
925  #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
926  
927  static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,
928  				 void *arg)
929  {
930  	int file = page_is_file_cache(page);
931  	int active = PageActive(page);
932  	enum lru_list lru = page_lru(page);
933  
934  	VM_BUG_ON_PAGE(PageLRU(page), page);
935  
936  	SetPageLRU(page);
937  	add_page_to_lru_list(page, lruvec, lru);
938  	update_page_reclaim_stat(lruvec, file, active);
939  	trace_mm_lru_insertion(page, page_to_pfn(page), lru, trace_pagemap_flags(page));
940  }
941  
942  /*
943   * Add the passed pages to the LRU, then drop the caller's refcount
944   * on them.  Reinitialises the caller's pagevec.
945   */
946  void __pagevec_lru_add(struct pagevec *pvec)
947  {
948  	pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL);
949  }
950  EXPORT_SYMBOL(__pagevec_lru_add);
951  
952  /**
953   * pagevec_lookup_entries - gang pagecache lookup
954   * @pvec:	Where the resulting entries are placed
955   * @mapping:	The address_space to search
956   * @start:	The starting entry index
957   * @nr_entries:	The maximum number of entries
958   * @indices:	The cache indices corresponding to the entries in @pvec
959   *
960   * pagevec_lookup_entries() will search for and return a group of up
961   * to @nr_entries pages and shadow entries in the mapping.  All
962   * entries are placed in @pvec.  pagevec_lookup_entries() takes a
963   * reference against actual pages in @pvec.
964   *
965   * The search returns a group of mapping-contiguous entries with
966   * ascending indexes.  There may be holes in the indices due to
967   * not-present entries.
968   *
969   * pagevec_lookup_entries() returns the number of entries which were
970   * found.
971   */
972  unsigned pagevec_lookup_entries(struct pagevec *pvec,
973  				struct address_space *mapping,
974  				pgoff_t start, unsigned nr_pages,
975  				pgoff_t *indices)
976  {
977  	pvec->nr = find_get_entries(mapping, start, nr_pages,
978  				    pvec->pages, indices);
979  	return pagevec_count(pvec);
980  }
981  
982  /**
983   * pagevec_remove_exceptionals - pagevec exceptionals pruning
984   * @pvec:	The pagevec to prune
985   *
986   * pagevec_lookup_entries() fills both pages and exceptional radix
987   * tree entries into the pagevec.  This function prunes all
988   * exceptionals from @pvec without leaving holes, so that it can be
989   * passed on to page-only pagevec operations.
990   */
991  void pagevec_remove_exceptionals(struct pagevec *pvec)
992  {
993  	int i, j;
994  
995  	for (i = 0, j = 0; i < pagevec_count(pvec); i++) {
996  		struct page *page = pvec->pages[i];
997  		if (!radix_tree_exceptional_entry(page))
998  			pvec->pages[j++] = page;
999  	}
1000  	pvec->nr = j;
1001  }
1002  
1003  /**
1004   * pagevec_lookup - gang pagecache lookup
1005   * @pvec:	Where the resulting pages are placed
1006   * @mapping:	The address_space to search
1007   * @start:	The starting page index
1008   * @nr_pages:	The maximum number of pages
1009   *
1010   * pagevec_lookup() will search for and return a group of up to @nr_pages pages
1011   * in the mapping.  The pages are placed in @pvec.  pagevec_lookup() takes a
1012   * reference against the pages in @pvec.
1013   *
1014   * The search returns a group of mapping-contiguous pages with ascending
1015   * indexes.  There may be holes in the indices due to not-present pages.
1016   *
1017   * pagevec_lookup() returns the number of pages which were found.
1018   */
1019  unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,
1020  		pgoff_t start, unsigned nr_pages)
1021  {
1022  	pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages);
1023  	return pagevec_count(pvec);
1024  }
1025  EXPORT_SYMBOL(pagevec_lookup);
1026  
1027  unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping,
1028  		pgoff_t *index, int tag, unsigned nr_pages)
1029  {
1030  	pvec->nr = find_get_pages_tag(mapping, index, tag,
1031  					nr_pages, pvec->pages);
1032  	return pagevec_count(pvec);
1033  }
1034  EXPORT_SYMBOL(pagevec_lookup_tag);
1035  
1036  /*
1037   * Perform any setup for the swap system
1038   */
1039  void __init swap_setup(void)
1040  {
1041  	unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT);
1042  #ifdef CONFIG_SWAP
1043  	int i;
1044  
1045  	if (bdi_init(swapper_spaces[0].backing_dev_info))
1046  		panic("Failed to init swap bdi");
1047  	for (i = 0; i < MAX_SWAPFILES; i++) {
1048  		spin_lock_init(&swapper_spaces[i].tree_lock);
1049  		INIT_LIST_HEAD(&swapper_spaces[i].i_mmap_nonlinear);
1050  	}
1051  #endif
1052  
1053  	/* Use a smaller cluster for small-memory machines */
1054  	if (megs < 16)
1055  		page_cluster = 2;
1056  	else
1057  		page_cluster = 3;
1058  	/*
1059  	 * Right now other parts of the system means that we
1060  	 * _really_ don't want to cluster much more
1061  	 */
1062  }
1063