xref: /openbmc/linux/mm/page_alloc.c (revision d2999e1b)
1 /*
2  *  linux/mm/page_alloc.c
3  *
4  *  Manages the free list, the system allocates free pages here.
5  *  Note that kmalloc() lives in slab.c
6  *
7  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
8  *  Swap reorganised 29.12.95, Stephen Tweedie
9  *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
10  *  Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
11  *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
12  *  Zone balancing, Kanoj Sarcar, SGI, Jan 2000
13  *  Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
14  *          (lots of bits borrowed from Ingo Molnar & Andrew Morton)
15  */
16 
17 #include <linux/stddef.h>
18 #include <linux/mm.h>
19 #include <linux/swap.h>
20 #include <linux/interrupt.h>
21 #include <linux/pagemap.h>
22 #include <linux/jiffies.h>
23 #include <linux/bootmem.h>
24 #include <linux/memblock.h>
25 #include <linux/compiler.h>
26 #include <linux/kernel.h>
27 #include <linux/kmemcheck.h>
28 #include <linux/module.h>
29 #include <linux/suspend.h>
30 #include <linux/pagevec.h>
31 #include <linux/blkdev.h>
32 #include <linux/slab.h>
33 #include <linux/ratelimit.h>
34 #include <linux/oom.h>
35 #include <linux/notifier.h>
36 #include <linux/topology.h>
37 #include <linux/sysctl.h>
38 #include <linux/cpu.h>
39 #include <linux/cpuset.h>
40 #include <linux/memory_hotplug.h>
41 #include <linux/nodemask.h>
42 #include <linux/vmalloc.h>
43 #include <linux/vmstat.h>
44 #include <linux/mempolicy.h>
45 #include <linux/stop_machine.h>
46 #include <linux/sort.h>
47 #include <linux/pfn.h>
48 #include <linux/backing-dev.h>
49 #include <linux/fault-inject.h>
50 #include <linux/page-isolation.h>
51 #include <linux/page_cgroup.h>
52 #include <linux/debugobjects.h>
53 #include <linux/kmemleak.h>
54 #include <linux/compaction.h>
55 #include <trace/events/kmem.h>
56 #include <linux/ftrace_event.h>
57 #include <linux/memcontrol.h>
58 #include <linux/prefetch.h>
59 #include <linux/mm_inline.h>
60 #include <linux/migrate.h>
61 #include <linux/page-debug-flags.h>
62 #include <linux/hugetlb.h>
63 #include <linux/sched/rt.h>
64 
65 #include <asm/sections.h>
66 #include <asm/tlbflush.h>
67 #include <asm/div64.h>
68 #include "internal.h"
69 
70 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
71 static DEFINE_MUTEX(pcp_batch_high_lock);
72 #define MIN_PERCPU_PAGELIST_FRACTION	(8)
73 
74 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
75 DEFINE_PER_CPU(int, numa_node);
76 EXPORT_PER_CPU_SYMBOL(numa_node);
77 #endif
78 
79 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
80 /*
81  * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
82  * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
83  * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
84  * defined in <linux/topology.h>.
85  */
86 DEFINE_PER_CPU(int, _numa_mem_);		/* Kernel "local memory" node */
87 EXPORT_PER_CPU_SYMBOL(_numa_mem_);
88 #endif
89 
90 /*
91  * Array of node states.
92  */
93 nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
94 	[N_POSSIBLE] = NODE_MASK_ALL,
95 	[N_ONLINE] = { { [0] = 1UL } },
96 #ifndef CONFIG_NUMA
97 	[N_NORMAL_MEMORY] = { { [0] = 1UL } },
98 #ifdef CONFIG_HIGHMEM
99 	[N_HIGH_MEMORY] = { { [0] = 1UL } },
100 #endif
101 #ifdef CONFIG_MOVABLE_NODE
102 	[N_MEMORY] = { { [0] = 1UL } },
103 #endif
104 	[N_CPU] = { { [0] = 1UL } },
105 #endif	/* NUMA */
106 };
107 EXPORT_SYMBOL(node_states);
108 
109 /* Protect totalram_pages and zone->managed_pages */
110 static DEFINE_SPINLOCK(managed_page_count_lock);
111 
112 unsigned long totalram_pages __read_mostly;
113 unsigned long totalreserve_pages __read_mostly;
114 /*
115  * When calculating the number of globally allowed dirty pages, there
116  * is a certain number of per-zone reserves that should not be
117  * considered dirtyable memory.  This is the sum of those reserves
118  * over all existing zones that contribute dirtyable memory.
119  */
120 unsigned long dirty_balance_reserve __read_mostly;
121 
122 int percpu_pagelist_fraction;
123 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
124 
125 #ifdef CONFIG_PM_SLEEP
126 /*
127  * The following functions are used by the suspend/hibernate code to temporarily
128  * change gfp_allowed_mask in order to avoid using I/O during memory allocations
129  * while devices are suspended.  To avoid races with the suspend/hibernate code,
130  * they should always be called with pm_mutex held (gfp_allowed_mask also should
131  * only be modified with pm_mutex held, unless the suspend/hibernate code is
132  * guaranteed not to run in parallel with that modification).
133  */
134 
135 static gfp_t saved_gfp_mask;
136 
137 void pm_restore_gfp_mask(void)
138 {
139 	WARN_ON(!mutex_is_locked(&pm_mutex));
140 	if (saved_gfp_mask) {
141 		gfp_allowed_mask = saved_gfp_mask;
142 		saved_gfp_mask = 0;
143 	}
144 }
145 
146 void pm_restrict_gfp_mask(void)
147 {
148 	WARN_ON(!mutex_is_locked(&pm_mutex));
149 	WARN_ON(saved_gfp_mask);
150 	saved_gfp_mask = gfp_allowed_mask;
151 	gfp_allowed_mask &= ~GFP_IOFS;
152 }
153 
154 bool pm_suspended_storage(void)
155 {
156 	if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS)
157 		return false;
158 	return true;
159 }
160 #endif /* CONFIG_PM_SLEEP */
161 
162 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
163 int pageblock_order __read_mostly;
164 #endif
165 
166 static void __free_pages_ok(struct page *page, unsigned int order);
167 
168 /*
169  * results with 256, 32 in the lowmem_reserve sysctl:
170  *	1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
171  *	1G machine -> (16M dma, 784M normal, 224M high)
172  *	NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
173  *	HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
174  *	HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA
175  *
176  * TBD: should special case ZONE_DMA32 machines here - in those we normally
177  * don't need any ZONE_NORMAL reservation
178  */
179 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
180 #ifdef CONFIG_ZONE_DMA
181 	 256,
182 #endif
183 #ifdef CONFIG_ZONE_DMA32
184 	 256,
185 #endif
186 #ifdef CONFIG_HIGHMEM
187 	 32,
188 #endif
189 	 32,
190 };
191 
192 EXPORT_SYMBOL(totalram_pages);
193 
194 static char * const zone_names[MAX_NR_ZONES] = {
195 #ifdef CONFIG_ZONE_DMA
196 	 "DMA",
197 #endif
198 #ifdef CONFIG_ZONE_DMA32
199 	 "DMA32",
200 #endif
201 	 "Normal",
202 #ifdef CONFIG_HIGHMEM
203 	 "HighMem",
204 #endif
205 	 "Movable",
206 };
207 
208 int min_free_kbytes = 1024;
209 int user_min_free_kbytes = -1;
210 
211 static unsigned long __meminitdata nr_kernel_pages;
212 static unsigned long __meminitdata nr_all_pages;
213 static unsigned long __meminitdata dma_reserve;
214 
215 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
216 static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
217 static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
218 static unsigned long __initdata required_kernelcore;
219 static unsigned long __initdata required_movablecore;
220 static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
221 
222 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
223 int movable_zone;
224 EXPORT_SYMBOL(movable_zone);
225 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
226 
227 #if MAX_NUMNODES > 1
228 int nr_node_ids __read_mostly = MAX_NUMNODES;
229 int nr_online_nodes __read_mostly = 1;
230 EXPORT_SYMBOL(nr_node_ids);
231 EXPORT_SYMBOL(nr_online_nodes);
232 #endif
233 
234 int page_group_by_mobility_disabled __read_mostly;
235 
236 void set_pageblock_migratetype(struct page *page, int migratetype)
237 {
238 	if (unlikely(page_group_by_mobility_disabled &&
239 		     migratetype < MIGRATE_PCPTYPES))
240 		migratetype = MIGRATE_UNMOVABLE;
241 
242 	set_pageblock_flags_group(page, (unsigned long)migratetype,
243 					PB_migrate, PB_migrate_end);
244 }
245 
246 bool oom_killer_disabled __read_mostly;
247 
248 #ifdef CONFIG_DEBUG_VM
249 static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
250 {
251 	int ret = 0;
252 	unsigned seq;
253 	unsigned long pfn = page_to_pfn(page);
254 	unsigned long sp, start_pfn;
255 
256 	do {
257 		seq = zone_span_seqbegin(zone);
258 		start_pfn = zone->zone_start_pfn;
259 		sp = zone->spanned_pages;
260 		if (!zone_spans_pfn(zone, pfn))
261 			ret = 1;
262 	} while (zone_span_seqretry(zone, seq));
263 
264 	if (ret)
265 		pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
266 			pfn, zone_to_nid(zone), zone->name,
267 			start_pfn, start_pfn + sp);
268 
269 	return ret;
270 }
271 
272 static int page_is_consistent(struct zone *zone, struct page *page)
273 {
274 	if (!pfn_valid_within(page_to_pfn(page)))
275 		return 0;
276 	if (zone != page_zone(page))
277 		return 0;
278 
279 	return 1;
280 }
281 /*
282  * Temporary debugging check for pages not lying within a given zone.
283  */
284 static int bad_range(struct zone *zone, struct page *page)
285 {
286 	if (page_outside_zone_boundaries(zone, page))
287 		return 1;
288 	if (!page_is_consistent(zone, page))
289 		return 1;
290 
291 	return 0;
292 }
293 #else
294 static inline int bad_range(struct zone *zone, struct page *page)
295 {
296 	return 0;
297 }
298 #endif
299 
300 static void bad_page(struct page *page, const char *reason,
301 		unsigned long bad_flags)
302 {
303 	static unsigned long resume;
304 	static unsigned long nr_shown;
305 	static unsigned long nr_unshown;
306 
307 	/* Don't complain about poisoned pages */
308 	if (PageHWPoison(page)) {
309 		page_mapcount_reset(page); /* remove PageBuddy */
310 		return;
311 	}
312 
313 	/*
314 	 * Allow a burst of 60 reports, then keep quiet for that minute;
315 	 * or allow a steady drip of one report per second.
316 	 */
317 	if (nr_shown == 60) {
318 		if (time_before(jiffies, resume)) {
319 			nr_unshown++;
320 			goto out;
321 		}
322 		if (nr_unshown) {
323 			printk(KERN_ALERT
324 			      "BUG: Bad page state: %lu messages suppressed\n",
325 				nr_unshown);
326 			nr_unshown = 0;
327 		}
328 		nr_shown = 0;
329 	}
330 	if (nr_shown++ == 0)
331 		resume = jiffies + 60 * HZ;
332 
333 	printk(KERN_ALERT "BUG: Bad page state in process %s  pfn:%05lx\n",
334 		current->comm, page_to_pfn(page));
335 	dump_page_badflags(page, reason, bad_flags);
336 
337 	print_modules();
338 	dump_stack();
339 out:
340 	/* Leave bad fields for debug, except PageBuddy could make trouble */
341 	page_mapcount_reset(page); /* remove PageBuddy */
342 	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
343 }
344 
345 /*
346  * Higher-order pages are called "compound pages".  They are structured thusly:
347  *
348  * The first PAGE_SIZE page is called the "head page".
349  *
350  * The remaining PAGE_SIZE pages are called "tail pages".
351  *
352  * All pages have PG_compound set.  All tail pages have their ->first_page
353  * pointing at the head page.
354  *
355  * The first tail page's ->lru.next holds the address of the compound page's
356  * put_page() function.  Its ->lru.prev holds the order of allocation.
357  * This usage means that zero-order pages may not be compound.
358  */
359 
360 static void free_compound_page(struct page *page)
361 {
362 	__free_pages_ok(page, compound_order(page));
363 }
364 
365 void prep_compound_page(struct page *page, unsigned long order)
366 {
367 	int i;
368 	int nr_pages = 1 << order;
369 
370 	set_compound_page_dtor(page, free_compound_page);
371 	set_compound_order(page, order);
372 	__SetPageHead(page);
373 	for (i = 1; i < nr_pages; i++) {
374 		struct page *p = page + i;
375 		set_page_count(p, 0);
376 		p->first_page = page;
377 		/* Make sure p->first_page is always valid for PageTail() */
378 		smp_wmb();
379 		__SetPageTail(p);
380 	}
381 }
382 
383 /* update __split_huge_page_refcount if you change this function */
384 static int destroy_compound_page(struct page *page, unsigned long order)
385 {
386 	int i;
387 	int nr_pages = 1 << order;
388 	int bad = 0;
389 
390 	if (unlikely(compound_order(page) != order)) {
391 		bad_page(page, "wrong compound order", 0);
392 		bad++;
393 	}
394 
395 	__ClearPageHead(page);
396 
397 	for (i = 1; i < nr_pages; i++) {
398 		struct page *p = page + i;
399 
400 		if (unlikely(!PageTail(p))) {
401 			bad_page(page, "PageTail not set", 0);
402 			bad++;
403 		} else if (unlikely(p->first_page != page)) {
404 			bad_page(page, "first_page not consistent", 0);
405 			bad++;
406 		}
407 		__ClearPageTail(p);
408 	}
409 
410 	return bad;
411 }
412 
413 static inline void prep_zero_page(struct page *page, unsigned int order,
414 							gfp_t gfp_flags)
415 {
416 	int i;
417 
418 	/*
419 	 * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO
420 	 * and __GFP_HIGHMEM from hard or soft interrupt context.
421 	 */
422 	VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());
423 	for (i = 0; i < (1 << order); i++)
424 		clear_highpage(page + i);
425 }
426 
427 #ifdef CONFIG_DEBUG_PAGEALLOC
428 unsigned int _debug_guardpage_minorder;
429 
430 static int __init debug_guardpage_minorder_setup(char *buf)
431 {
432 	unsigned long res;
433 
434 	if (kstrtoul(buf, 10, &res) < 0 ||  res > MAX_ORDER / 2) {
435 		printk(KERN_ERR "Bad debug_guardpage_minorder value\n");
436 		return 0;
437 	}
438 	_debug_guardpage_minorder = res;
439 	printk(KERN_INFO "Setting debug_guardpage_minorder to %lu\n", res);
440 	return 0;
441 }
442 __setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);
443 
444 static inline void set_page_guard_flag(struct page *page)
445 {
446 	__set_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);
447 }
448 
449 static inline void clear_page_guard_flag(struct page *page)
450 {
451 	__clear_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);
452 }
453 #else
454 static inline void set_page_guard_flag(struct page *page) { }
455 static inline void clear_page_guard_flag(struct page *page) { }
456 #endif
457 
458 static inline void set_page_order(struct page *page, unsigned int order)
459 {
460 	set_page_private(page, order);
461 	__SetPageBuddy(page);
462 }
463 
464 static inline void rmv_page_order(struct page *page)
465 {
466 	__ClearPageBuddy(page);
467 	set_page_private(page, 0);
468 }
469 
470 /*
471  * Locate the struct page for both the matching buddy in our
472  * pair (buddy1) and the combined O(n+1) page they form (page).
473  *
474  * 1) Any buddy B1 will have an order O twin B2 which satisfies
475  * the following equation:
476  *     B2 = B1 ^ (1 << O)
477  * For example, if the starting buddy (buddy2) is #8 its order
478  * 1 buddy is #10:
479  *     B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
480  *
481  * 2) Any buddy B will have an order O+1 parent P which
482  * satisfies the following equation:
483  *     P = B & ~(1 << O)
484  *
485  * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
486  */
487 static inline unsigned long
488 __find_buddy_index(unsigned long page_idx, unsigned int order)
489 {
490 	return page_idx ^ (1 << order);
491 }
492 
493 /*
494  * This function checks whether a page is free && is the buddy
495  * we can do coalesce a page and its buddy if
496  * (a) the buddy is not in a hole &&
497  * (b) the buddy is in the buddy system &&
498  * (c) a page and its buddy have the same order &&
499  * (d) a page and its buddy are in the same zone.
500  *
501  * For recording whether a page is in the buddy system, we set ->_mapcount
502  * PAGE_BUDDY_MAPCOUNT_VALUE.
503  * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is
504  * serialized by zone->lock.
505  *
506  * For recording page's order, we use page_private(page).
507  */
508 static inline int page_is_buddy(struct page *page, struct page *buddy,
509 							unsigned int order)
510 {
511 	if (!pfn_valid_within(page_to_pfn(buddy)))
512 		return 0;
513 
514 	if (page_is_guard(buddy) && page_order(buddy) == order) {
515 		VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
516 
517 		if (page_zone_id(page) != page_zone_id(buddy))
518 			return 0;
519 
520 		return 1;
521 	}
522 
523 	if (PageBuddy(buddy) && page_order(buddy) == order) {
524 		VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
525 
526 		/*
527 		 * zone check is done late to avoid uselessly
528 		 * calculating zone/node ids for pages that could
529 		 * never merge.
530 		 */
531 		if (page_zone_id(page) != page_zone_id(buddy))
532 			return 0;
533 
534 		return 1;
535 	}
536 	return 0;
537 }
538 
539 /*
540  * Freeing function for a buddy system allocator.
541  *
542  * The concept of a buddy system is to maintain direct-mapped table
543  * (containing bit values) for memory blocks of various "orders".
544  * The bottom level table contains the map for the smallest allocatable
545  * units of memory (here, pages), and each level above it describes
546  * pairs of units from the levels below, hence, "buddies".
547  * At a high level, all that happens here is marking the table entry
548  * at the bottom level available, and propagating the changes upward
549  * as necessary, plus some accounting needed to play nicely with other
550  * parts of the VM system.
551  * At each level, we keep a list of pages, which are heads of continuous
552  * free pages of length of (1 << order) and marked with _mapcount
553  * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page)
554  * field.
555  * So when we are allocating or freeing one, we can derive the state of the
556  * other.  That is, if we allocate a small block, and both were
557  * free, the remainder of the region must be split into blocks.
558  * If a block is freed, and its buddy is also free, then this
559  * triggers coalescing into a block of larger size.
560  *
561  * -- nyc
562  */
563 
564 static inline void __free_one_page(struct page *page,
565 		unsigned long pfn,
566 		struct zone *zone, unsigned int order,
567 		int migratetype)
568 {
569 	unsigned long page_idx;
570 	unsigned long combined_idx;
571 	unsigned long uninitialized_var(buddy_idx);
572 	struct page *buddy;
573 
574 	VM_BUG_ON(!zone_is_initialized(zone));
575 
576 	if (unlikely(PageCompound(page)))
577 		if (unlikely(destroy_compound_page(page, order)))
578 			return;
579 
580 	VM_BUG_ON(migratetype == -1);
581 
582 	page_idx = pfn & ((1 << MAX_ORDER) - 1);
583 
584 	VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
585 	VM_BUG_ON_PAGE(bad_range(zone, page), page);
586 
587 	while (order < MAX_ORDER-1) {
588 		buddy_idx = __find_buddy_index(page_idx, order);
589 		buddy = page + (buddy_idx - page_idx);
590 		if (!page_is_buddy(page, buddy, order))
591 			break;
592 		/*
593 		 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
594 		 * merge with it and move up one order.
595 		 */
596 		if (page_is_guard(buddy)) {
597 			clear_page_guard_flag(buddy);
598 			set_page_private(page, 0);
599 			__mod_zone_freepage_state(zone, 1 << order,
600 						  migratetype);
601 		} else {
602 			list_del(&buddy->lru);
603 			zone->free_area[order].nr_free--;
604 			rmv_page_order(buddy);
605 		}
606 		combined_idx = buddy_idx & page_idx;
607 		page = page + (combined_idx - page_idx);
608 		page_idx = combined_idx;
609 		order++;
610 	}
611 	set_page_order(page, order);
612 
613 	/*
614 	 * If this is not the largest possible page, check if the buddy
615 	 * of the next-highest order is free. If it is, it's possible
616 	 * that pages are being freed that will coalesce soon. In case,
617 	 * that is happening, add the free page to the tail of the list
618 	 * so it's less likely to be used soon and more likely to be merged
619 	 * as a higher order page
620 	 */
621 	if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
622 		struct page *higher_page, *higher_buddy;
623 		combined_idx = buddy_idx & page_idx;
624 		higher_page = page + (combined_idx - page_idx);
625 		buddy_idx = __find_buddy_index(combined_idx, order + 1);
626 		higher_buddy = higher_page + (buddy_idx - combined_idx);
627 		if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
628 			list_add_tail(&page->lru,
629 				&zone->free_area[order].free_list[migratetype]);
630 			goto out;
631 		}
632 	}
633 
634 	list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
635 out:
636 	zone->free_area[order].nr_free++;
637 }
638 
639 static inline int free_pages_check(struct page *page)
640 {
641 	const char *bad_reason = NULL;
642 	unsigned long bad_flags = 0;
643 
644 	if (unlikely(page_mapcount(page)))
645 		bad_reason = "nonzero mapcount";
646 	if (unlikely(page->mapping != NULL))
647 		bad_reason = "non-NULL mapping";
648 	if (unlikely(atomic_read(&page->_count) != 0))
649 		bad_reason = "nonzero _count";
650 	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
651 		bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
652 		bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
653 	}
654 	if (unlikely(mem_cgroup_bad_page_check(page)))
655 		bad_reason = "cgroup check failed";
656 	if (unlikely(bad_reason)) {
657 		bad_page(page, bad_reason, bad_flags);
658 		return 1;
659 	}
660 	page_cpupid_reset_last(page);
661 	if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
662 		page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
663 	return 0;
664 }
665 
666 /*
667  * Frees a number of pages from the PCP lists
668  * Assumes all pages on list are in same zone, and of same order.
669  * count is the number of pages to free.
670  *
671  * If the zone was previously in an "all pages pinned" state then look to
672  * see if this freeing clears that state.
673  *
674  * And clear the zone's pages_scanned counter, to hold off the "all pages are
675  * pinned" detection logic.
676  */
677 static void free_pcppages_bulk(struct zone *zone, int count,
678 					struct per_cpu_pages *pcp)
679 {
680 	int migratetype = 0;
681 	int batch_free = 0;
682 	int to_free = count;
683 
684 	spin_lock(&zone->lock);
685 	zone->pages_scanned = 0;
686 
687 	while (to_free) {
688 		struct page *page;
689 		struct list_head *list;
690 
691 		/*
692 		 * Remove pages from lists in a round-robin fashion. A
693 		 * batch_free count is maintained that is incremented when an
694 		 * empty list is encountered.  This is so more pages are freed
695 		 * off fuller lists instead of spinning excessively around empty
696 		 * lists
697 		 */
698 		do {
699 			batch_free++;
700 			if (++migratetype == MIGRATE_PCPTYPES)
701 				migratetype = 0;
702 			list = &pcp->lists[migratetype];
703 		} while (list_empty(list));
704 
705 		/* This is the only non-empty list. Free them all. */
706 		if (batch_free == MIGRATE_PCPTYPES)
707 			batch_free = to_free;
708 
709 		do {
710 			int mt;	/* migratetype of the to-be-freed page */
711 
712 			page = list_entry(list->prev, struct page, lru);
713 			/* must delete as __free_one_page list manipulates */
714 			list_del(&page->lru);
715 			mt = get_freepage_migratetype(page);
716 			/* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
717 			__free_one_page(page, page_to_pfn(page), zone, 0, mt);
718 			trace_mm_page_pcpu_drain(page, 0, mt);
719 			if (likely(!is_migrate_isolate_page(page))) {
720 				__mod_zone_page_state(zone, NR_FREE_PAGES, 1);
721 				if (is_migrate_cma(mt))
722 					__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 1);
723 			}
724 		} while (--to_free && --batch_free && !list_empty(list));
725 	}
726 	spin_unlock(&zone->lock);
727 }
728 
729 static void free_one_page(struct zone *zone,
730 				struct page *page, unsigned long pfn,
731 				unsigned int order,
732 				int migratetype)
733 {
734 	spin_lock(&zone->lock);
735 	zone->pages_scanned = 0;
736 
737 	__free_one_page(page, pfn, zone, order, migratetype);
738 	if (unlikely(!is_migrate_isolate(migratetype)))
739 		__mod_zone_freepage_state(zone, 1 << order, migratetype);
740 	spin_unlock(&zone->lock);
741 }
742 
743 static bool free_pages_prepare(struct page *page, unsigned int order)
744 {
745 	int i;
746 	int bad = 0;
747 
748 	trace_mm_page_free(page, order);
749 	kmemcheck_free_shadow(page, order);
750 
751 	if (PageAnon(page))
752 		page->mapping = NULL;
753 	for (i = 0; i < (1 << order); i++)
754 		bad += free_pages_check(page + i);
755 	if (bad)
756 		return false;
757 
758 	if (!PageHighMem(page)) {
759 		debug_check_no_locks_freed(page_address(page),
760 					   PAGE_SIZE << order);
761 		debug_check_no_obj_freed(page_address(page),
762 					   PAGE_SIZE << order);
763 	}
764 	arch_free_page(page, order);
765 	kernel_map_pages(page, 1 << order, 0);
766 
767 	return true;
768 }
769 
770 static void __free_pages_ok(struct page *page, unsigned int order)
771 {
772 	unsigned long flags;
773 	int migratetype;
774 	unsigned long pfn = page_to_pfn(page);
775 
776 	if (!free_pages_prepare(page, order))
777 		return;
778 
779 	migratetype = get_pfnblock_migratetype(page, pfn);
780 	local_irq_save(flags);
781 	__count_vm_events(PGFREE, 1 << order);
782 	set_freepage_migratetype(page, migratetype);
783 	free_one_page(page_zone(page), page, pfn, order, migratetype);
784 	local_irq_restore(flags);
785 }
786 
787 void __init __free_pages_bootmem(struct page *page, unsigned int order)
788 {
789 	unsigned int nr_pages = 1 << order;
790 	struct page *p = page;
791 	unsigned int loop;
792 
793 	prefetchw(p);
794 	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
795 		prefetchw(p + 1);
796 		__ClearPageReserved(p);
797 		set_page_count(p, 0);
798 	}
799 	__ClearPageReserved(p);
800 	set_page_count(p, 0);
801 
802 	page_zone(page)->managed_pages += nr_pages;
803 	set_page_refcounted(page);
804 	__free_pages(page, order);
805 }
806 
807 #ifdef CONFIG_CMA
808 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */
809 void __init init_cma_reserved_pageblock(struct page *page)
810 {
811 	unsigned i = pageblock_nr_pages;
812 	struct page *p = page;
813 
814 	do {
815 		__ClearPageReserved(p);
816 		set_page_count(p, 0);
817 	} while (++p, --i);
818 
819 	set_page_refcounted(page);
820 	set_pageblock_migratetype(page, MIGRATE_CMA);
821 	__free_pages(page, pageblock_order);
822 	adjust_managed_page_count(page, pageblock_nr_pages);
823 }
824 #endif
825 
826 /*
827  * The order of subdivision here is critical for the IO subsystem.
828  * Please do not alter this order without good reasons and regression
829  * testing. Specifically, as large blocks of memory are subdivided,
830  * the order in which smaller blocks are delivered depends on the order
831  * they're subdivided in this function. This is the primary factor
832  * influencing the order in which pages are delivered to the IO
833  * subsystem according to empirical testing, and this is also justified
834  * by considering the behavior of a buddy system containing a single
835  * large block of memory acted on by a series of small allocations.
836  * This behavior is a critical factor in sglist merging's success.
837  *
838  * -- nyc
839  */
840 static inline void expand(struct zone *zone, struct page *page,
841 	int low, int high, struct free_area *area,
842 	int migratetype)
843 {
844 	unsigned long size = 1 << high;
845 
846 	while (high > low) {
847 		area--;
848 		high--;
849 		size >>= 1;
850 		VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
851 
852 #ifdef CONFIG_DEBUG_PAGEALLOC
853 		if (high < debug_guardpage_minorder()) {
854 			/*
855 			 * Mark as guard pages (or page), that will allow to
856 			 * merge back to allocator when buddy will be freed.
857 			 * Corresponding page table entries will not be touched,
858 			 * pages will stay not present in virtual address space
859 			 */
860 			INIT_LIST_HEAD(&page[size].lru);
861 			set_page_guard_flag(&page[size]);
862 			set_page_private(&page[size], high);
863 			/* Guard pages are not available for any usage */
864 			__mod_zone_freepage_state(zone, -(1 << high),
865 						  migratetype);
866 			continue;
867 		}
868 #endif
869 		list_add(&page[size].lru, &area->free_list[migratetype]);
870 		area->nr_free++;
871 		set_page_order(&page[size], high);
872 	}
873 }
874 
875 /*
876  * This page is about to be returned from the page allocator
877  */
878 static inline int check_new_page(struct page *page)
879 {
880 	const char *bad_reason = NULL;
881 	unsigned long bad_flags = 0;
882 
883 	if (unlikely(page_mapcount(page)))
884 		bad_reason = "nonzero mapcount";
885 	if (unlikely(page->mapping != NULL))
886 		bad_reason = "non-NULL mapping";
887 	if (unlikely(atomic_read(&page->_count) != 0))
888 		bad_reason = "nonzero _count";
889 	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) {
890 		bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set";
891 		bad_flags = PAGE_FLAGS_CHECK_AT_PREP;
892 	}
893 	if (unlikely(mem_cgroup_bad_page_check(page)))
894 		bad_reason = "cgroup check failed";
895 	if (unlikely(bad_reason)) {
896 		bad_page(page, bad_reason, bad_flags);
897 		return 1;
898 	}
899 	return 0;
900 }
901 
902 static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags)
903 {
904 	int i;
905 
906 	for (i = 0; i < (1 << order); i++) {
907 		struct page *p = page + i;
908 		if (unlikely(check_new_page(p)))
909 			return 1;
910 	}
911 
912 	set_page_private(page, 0);
913 	set_page_refcounted(page);
914 
915 	arch_alloc_page(page, order);
916 	kernel_map_pages(page, 1 << order, 1);
917 
918 	if (gfp_flags & __GFP_ZERO)
919 		prep_zero_page(page, order, gfp_flags);
920 
921 	if (order && (gfp_flags & __GFP_COMP))
922 		prep_compound_page(page, order);
923 
924 	return 0;
925 }
926 
927 /*
928  * Go through the free lists for the given migratetype and remove
929  * the smallest available page from the freelists
930  */
931 static inline
932 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
933 						int migratetype)
934 {
935 	unsigned int current_order;
936 	struct free_area *area;
937 	struct page *page;
938 
939 	/* Find a page of the appropriate size in the preferred list */
940 	for (current_order = order; current_order < MAX_ORDER; ++current_order) {
941 		area = &(zone->free_area[current_order]);
942 		if (list_empty(&area->free_list[migratetype]))
943 			continue;
944 
945 		page = list_entry(area->free_list[migratetype].next,
946 							struct page, lru);
947 		list_del(&page->lru);
948 		rmv_page_order(page);
949 		area->nr_free--;
950 		expand(zone, page, order, current_order, area, migratetype);
951 		set_freepage_migratetype(page, migratetype);
952 		return page;
953 	}
954 
955 	return NULL;
956 }
957 
958 
959 /*
960  * This array describes the order lists are fallen back to when
961  * the free lists for the desirable migrate type are depleted
962  */
963 static int fallbacks[MIGRATE_TYPES][4] = {
964 	[MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE,     MIGRATE_RESERVE },
965 	[MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE,     MIGRATE_RESERVE },
966 #ifdef CONFIG_CMA
967 	[MIGRATE_MOVABLE]     = { MIGRATE_CMA,         MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
968 	[MIGRATE_CMA]         = { MIGRATE_RESERVE }, /* Never used */
969 #else
970 	[MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE,   MIGRATE_RESERVE },
971 #endif
972 	[MIGRATE_RESERVE]     = { MIGRATE_RESERVE }, /* Never used */
973 #ifdef CONFIG_MEMORY_ISOLATION
974 	[MIGRATE_ISOLATE]     = { MIGRATE_RESERVE }, /* Never used */
975 #endif
976 };
977 
978 /*
979  * Move the free pages in a range to the free lists of the requested type.
980  * Note that start_page and end_pages are not aligned on a pageblock
981  * boundary. If alignment is required, use move_freepages_block()
982  */
983 int move_freepages(struct zone *zone,
984 			  struct page *start_page, struct page *end_page,
985 			  int migratetype)
986 {
987 	struct page *page;
988 	unsigned long order;
989 	int pages_moved = 0;
990 
991 #ifndef CONFIG_HOLES_IN_ZONE
992 	/*
993 	 * page_zone is not safe to call in this context when
994 	 * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
995 	 * anyway as we check zone boundaries in move_freepages_block().
996 	 * Remove at a later date when no bug reports exist related to
997 	 * grouping pages by mobility
998 	 */
999 	BUG_ON(page_zone(start_page) != page_zone(end_page));
1000 #endif
1001 
1002 	for (page = start_page; page <= end_page;) {
1003 		/* Make sure we are not inadvertently changing nodes */
1004 		VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
1005 
1006 		if (!pfn_valid_within(page_to_pfn(page))) {
1007 			page++;
1008 			continue;
1009 		}
1010 
1011 		if (!PageBuddy(page)) {
1012 			page++;
1013 			continue;
1014 		}
1015 
1016 		order = page_order(page);
1017 		list_move(&page->lru,
1018 			  &zone->free_area[order].free_list[migratetype]);
1019 		set_freepage_migratetype(page, migratetype);
1020 		page += 1 << order;
1021 		pages_moved += 1 << order;
1022 	}
1023 
1024 	return pages_moved;
1025 }
1026 
1027 int move_freepages_block(struct zone *zone, struct page *page,
1028 				int migratetype)
1029 {
1030 	unsigned long start_pfn, end_pfn;
1031 	struct page *start_page, *end_page;
1032 
1033 	start_pfn = page_to_pfn(page);
1034 	start_pfn = start_pfn & ~(pageblock_nr_pages-1);
1035 	start_page = pfn_to_page(start_pfn);
1036 	end_page = start_page + pageblock_nr_pages - 1;
1037 	end_pfn = start_pfn + pageblock_nr_pages - 1;
1038 
1039 	/* Do not cross zone boundaries */
1040 	if (!zone_spans_pfn(zone, start_pfn))
1041 		start_page = page;
1042 	if (!zone_spans_pfn(zone, end_pfn))
1043 		return 0;
1044 
1045 	return move_freepages(zone, start_page, end_page, migratetype);
1046 }
1047 
1048 static void change_pageblock_range(struct page *pageblock_page,
1049 					int start_order, int migratetype)
1050 {
1051 	int nr_pageblocks = 1 << (start_order - pageblock_order);
1052 
1053 	while (nr_pageblocks--) {
1054 		set_pageblock_migratetype(pageblock_page, migratetype);
1055 		pageblock_page += pageblock_nr_pages;
1056 	}
1057 }
1058 
1059 /*
1060  * If breaking a large block of pages, move all free pages to the preferred
1061  * allocation list. If falling back for a reclaimable kernel allocation, be
1062  * more aggressive about taking ownership of free pages.
1063  *
1064  * On the other hand, never change migration type of MIGRATE_CMA pageblocks
1065  * nor move CMA pages to different free lists. We don't want unmovable pages
1066  * to be allocated from MIGRATE_CMA areas.
1067  *
1068  * Returns the new migratetype of the pageblock (or the same old migratetype
1069  * if it was unchanged).
1070  */
1071 static int try_to_steal_freepages(struct zone *zone, struct page *page,
1072 				  int start_type, int fallback_type)
1073 {
1074 	int current_order = page_order(page);
1075 
1076 	/*
1077 	 * When borrowing from MIGRATE_CMA, we need to release the excess
1078 	 * buddy pages to CMA itself. We also ensure the freepage_migratetype
1079 	 * is set to CMA so it is returned to the correct freelist in case
1080 	 * the page ends up being not actually allocated from the pcp lists.
1081 	 */
1082 	if (is_migrate_cma(fallback_type))
1083 		return fallback_type;
1084 
1085 	/* Take ownership for orders >= pageblock_order */
1086 	if (current_order >= pageblock_order) {
1087 		change_pageblock_range(page, current_order, start_type);
1088 		return start_type;
1089 	}
1090 
1091 	if (current_order >= pageblock_order / 2 ||
1092 	    start_type == MIGRATE_RECLAIMABLE ||
1093 	    page_group_by_mobility_disabled) {
1094 		int pages;
1095 
1096 		pages = move_freepages_block(zone, page, start_type);
1097 
1098 		/* Claim the whole block if over half of it is free */
1099 		if (pages >= (1 << (pageblock_order-1)) ||
1100 				page_group_by_mobility_disabled) {
1101 
1102 			set_pageblock_migratetype(page, start_type);
1103 			return start_type;
1104 		}
1105 
1106 	}
1107 
1108 	return fallback_type;
1109 }
1110 
1111 /* Remove an element from the buddy allocator from the fallback list */
1112 static inline struct page *
1113 __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
1114 {
1115 	struct free_area *area;
1116 	unsigned int current_order;
1117 	struct page *page;
1118 	int migratetype, new_type, i;
1119 
1120 	/* Find the largest possible block of pages in the other list */
1121 	for (current_order = MAX_ORDER-1;
1122 				current_order >= order && current_order <= MAX_ORDER-1;
1123 				--current_order) {
1124 		for (i = 0;; i++) {
1125 			migratetype = fallbacks[start_migratetype][i];
1126 
1127 			/* MIGRATE_RESERVE handled later if necessary */
1128 			if (migratetype == MIGRATE_RESERVE)
1129 				break;
1130 
1131 			area = &(zone->free_area[current_order]);
1132 			if (list_empty(&area->free_list[migratetype]))
1133 				continue;
1134 
1135 			page = list_entry(area->free_list[migratetype].next,
1136 					struct page, lru);
1137 			area->nr_free--;
1138 
1139 			new_type = try_to_steal_freepages(zone, page,
1140 							  start_migratetype,
1141 							  migratetype);
1142 
1143 			/* Remove the page from the freelists */
1144 			list_del(&page->lru);
1145 			rmv_page_order(page);
1146 
1147 			expand(zone, page, order, current_order, area,
1148 			       new_type);
1149 			/* The freepage_migratetype may differ from pageblock's
1150 			 * migratetype depending on the decisions in
1151 			 * try_to_steal_freepages. This is OK as long as it does
1152 			 * not differ for MIGRATE_CMA type.
1153 			 */
1154 			set_freepage_migratetype(page, new_type);
1155 
1156 			trace_mm_page_alloc_extfrag(page, order, current_order,
1157 				start_migratetype, migratetype, new_type);
1158 
1159 			return page;
1160 		}
1161 	}
1162 
1163 	return NULL;
1164 }
1165 
1166 /*
1167  * Do the hard work of removing an element from the buddy allocator.
1168  * Call me with the zone->lock already held.
1169  */
1170 static struct page *__rmqueue(struct zone *zone, unsigned int order,
1171 						int migratetype)
1172 {
1173 	struct page *page;
1174 
1175 retry_reserve:
1176 	page = __rmqueue_smallest(zone, order, migratetype);
1177 
1178 	if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
1179 		page = __rmqueue_fallback(zone, order, migratetype);
1180 
1181 		/*
1182 		 * Use MIGRATE_RESERVE rather than fail an allocation. goto
1183 		 * is used because __rmqueue_smallest is an inline function
1184 		 * and we want just one call site
1185 		 */
1186 		if (!page) {
1187 			migratetype = MIGRATE_RESERVE;
1188 			goto retry_reserve;
1189 		}
1190 	}
1191 
1192 	trace_mm_page_alloc_zone_locked(page, order, migratetype);
1193 	return page;
1194 }
1195 
1196 /*
1197  * Obtain a specified number of elements from the buddy allocator, all under
1198  * a single hold of the lock, for efficiency.  Add them to the supplied list.
1199  * Returns the number of new pages which were placed at *list.
1200  */
1201 static int rmqueue_bulk(struct zone *zone, unsigned int order,
1202 			unsigned long count, struct list_head *list,
1203 			int migratetype, bool cold)
1204 {
1205 	int i;
1206 
1207 	spin_lock(&zone->lock);
1208 	for (i = 0; i < count; ++i) {
1209 		struct page *page = __rmqueue(zone, order, migratetype);
1210 		if (unlikely(page == NULL))
1211 			break;
1212 
1213 		/*
1214 		 * Split buddy pages returned by expand() are received here
1215 		 * in physical page order. The page is added to the callers and
1216 		 * list and the list head then moves forward. From the callers
1217 		 * perspective, the linked list is ordered by page number in
1218 		 * some conditions. This is useful for IO devices that can
1219 		 * merge IO requests if the physical pages are ordered
1220 		 * properly.
1221 		 */
1222 		if (likely(!cold))
1223 			list_add(&page->lru, list);
1224 		else
1225 			list_add_tail(&page->lru, list);
1226 		list = &page->lru;
1227 		if (is_migrate_cma(get_freepage_migratetype(page)))
1228 			__mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
1229 					      -(1 << order));
1230 	}
1231 	__mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
1232 	spin_unlock(&zone->lock);
1233 	return i;
1234 }
1235 
1236 #ifdef CONFIG_NUMA
1237 /*
1238  * Called from the vmstat counter updater to drain pagesets of this
1239  * currently executing processor on remote nodes after they have
1240  * expired.
1241  *
1242  * Note that this function must be called with the thread pinned to
1243  * a single processor.
1244  */
1245 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
1246 {
1247 	unsigned long flags;
1248 	int to_drain;
1249 	unsigned long batch;
1250 
1251 	local_irq_save(flags);
1252 	batch = ACCESS_ONCE(pcp->batch);
1253 	if (pcp->count >= batch)
1254 		to_drain = batch;
1255 	else
1256 		to_drain = pcp->count;
1257 	if (to_drain > 0) {
1258 		free_pcppages_bulk(zone, to_drain, pcp);
1259 		pcp->count -= to_drain;
1260 	}
1261 	local_irq_restore(flags);
1262 }
1263 #endif
1264 
1265 /*
1266  * Drain pages of the indicated processor.
1267  *
1268  * The processor must either be the current processor and the
1269  * thread pinned to the current processor or a processor that
1270  * is not online.
1271  */
1272 static void drain_pages(unsigned int cpu)
1273 {
1274 	unsigned long flags;
1275 	struct zone *zone;
1276 
1277 	for_each_populated_zone(zone) {
1278 		struct per_cpu_pageset *pset;
1279 		struct per_cpu_pages *pcp;
1280 
1281 		local_irq_save(flags);
1282 		pset = per_cpu_ptr(zone->pageset, cpu);
1283 
1284 		pcp = &pset->pcp;
1285 		if (pcp->count) {
1286 			free_pcppages_bulk(zone, pcp->count, pcp);
1287 			pcp->count = 0;
1288 		}
1289 		local_irq_restore(flags);
1290 	}
1291 }
1292 
1293 /*
1294  * Spill all of this CPU's per-cpu pages back into the buddy allocator.
1295  */
1296 void drain_local_pages(void *arg)
1297 {
1298 	drain_pages(smp_processor_id());
1299 }
1300 
1301 /*
1302  * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
1303  *
1304  * Note that this code is protected against sending an IPI to an offline
1305  * CPU but does not guarantee sending an IPI to newly hotplugged CPUs:
1306  * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but
1307  * nothing keeps CPUs from showing up after we populated the cpumask and
1308  * before the call to on_each_cpu_mask().
1309  */
1310 void drain_all_pages(void)
1311 {
1312 	int cpu;
1313 	struct per_cpu_pageset *pcp;
1314 	struct zone *zone;
1315 
1316 	/*
1317 	 * Allocate in the BSS so we wont require allocation in
1318 	 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
1319 	 */
1320 	static cpumask_t cpus_with_pcps;
1321 
1322 	/*
1323 	 * We don't care about racing with CPU hotplug event
1324 	 * as offline notification will cause the notified
1325 	 * cpu to drain that CPU pcps and on_each_cpu_mask
1326 	 * disables preemption as part of its processing
1327 	 */
1328 	for_each_online_cpu(cpu) {
1329 		bool has_pcps = false;
1330 		for_each_populated_zone(zone) {
1331 			pcp = per_cpu_ptr(zone->pageset, cpu);
1332 			if (pcp->pcp.count) {
1333 				has_pcps = true;
1334 				break;
1335 			}
1336 		}
1337 		if (has_pcps)
1338 			cpumask_set_cpu(cpu, &cpus_with_pcps);
1339 		else
1340 			cpumask_clear_cpu(cpu, &cpus_with_pcps);
1341 	}
1342 	on_each_cpu_mask(&cpus_with_pcps, drain_local_pages, NULL, 1);
1343 }
1344 
1345 #ifdef CONFIG_HIBERNATION
1346 
1347 void mark_free_pages(struct zone *zone)
1348 {
1349 	unsigned long pfn, max_zone_pfn;
1350 	unsigned long flags;
1351 	unsigned int order, t;
1352 	struct list_head *curr;
1353 
1354 	if (zone_is_empty(zone))
1355 		return;
1356 
1357 	spin_lock_irqsave(&zone->lock, flags);
1358 
1359 	max_zone_pfn = zone_end_pfn(zone);
1360 	for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1361 		if (pfn_valid(pfn)) {
1362 			struct page *page = pfn_to_page(pfn);
1363 
1364 			if (!swsusp_page_is_forbidden(page))
1365 				swsusp_unset_page_free(page);
1366 		}
1367 
1368 	for_each_migratetype_order(order, t) {
1369 		list_for_each(curr, &zone->free_area[order].free_list[t]) {
1370 			unsigned long i;
1371 
1372 			pfn = page_to_pfn(list_entry(curr, struct page, lru));
1373 			for (i = 0; i < (1UL << order); i++)
1374 				swsusp_set_page_free(pfn_to_page(pfn + i));
1375 		}
1376 	}
1377 	spin_unlock_irqrestore(&zone->lock, flags);
1378 }
1379 #endif /* CONFIG_PM */
1380 
1381 /*
1382  * Free a 0-order page
1383  * cold == true ? free a cold page : free a hot page
1384  */
1385 void free_hot_cold_page(struct page *page, bool cold)
1386 {
1387 	struct zone *zone = page_zone(page);
1388 	struct per_cpu_pages *pcp;
1389 	unsigned long flags;
1390 	unsigned long pfn = page_to_pfn(page);
1391 	int migratetype;
1392 
1393 	if (!free_pages_prepare(page, 0))
1394 		return;
1395 
1396 	migratetype = get_pfnblock_migratetype(page, pfn);
1397 	set_freepage_migratetype(page, migratetype);
1398 	local_irq_save(flags);
1399 	__count_vm_event(PGFREE);
1400 
1401 	/*
1402 	 * We only track unmovable, reclaimable and movable on pcp lists.
1403 	 * Free ISOLATE pages back to the allocator because they are being
1404 	 * offlined but treat RESERVE as movable pages so we can get those
1405 	 * areas back if necessary. Otherwise, we may have to free
1406 	 * excessively into the page allocator
1407 	 */
1408 	if (migratetype >= MIGRATE_PCPTYPES) {
1409 		if (unlikely(is_migrate_isolate(migratetype))) {
1410 			free_one_page(zone, page, pfn, 0, migratetype);
1411 			goto out;
1412 		}
1413 		migratetype = MIGRATE_MOVABLE;
1414 	}
1415 
1416 	pcp = &this_cpu_ptr(zone->pageset)->pcp;
1417 	if (!cold)
1418 		list_add(&page->lru, &pcp->lists[migratetype]);
1419 	else
1420 		list_add_tail(&page->lru, &pcp->lists[migratetype]);
1421 	pcp->count++;
1422 	if (pcp->count >= pcp->high) {
1423 		unsigned long batch = ACCESS_ONCE(pcp->batch);
1424 		free_pcppages_bulk(zone, batch, pcp);
1425 		pcp->count -= batch;
1426 	}
1427 
1428 out:
1429 	local_irq_restore(flags);
1430 }
1431 
1432 /*
1433  * Free a list of 0-order pages
1434  */
1435 void free_hot_cold_page_list(struct list_head *list, bool cold)
1436 {
1437 	struct page *page, *next;
1438 
1439 	list_for_each_entry_safe(page, next, list, lru) {
1440 		trace_mm_page_free_batched(page, cold);
1441 		free_hot_cold_page(page, cold);
1442 	}
1443 }
1444 
1445 /*
1446  * split_page takes a non-compound higher-order page, and splits it into
1447  * n (1<<order) sub-pages: page[0..n]
1448  * Each sub-page must be freed individually.
1449  *
1450  * Note: this is probably too low level an operation for use in drivers.
1451  * Please consult with lkml before using this in your driver.
1452  */
1453 void split_page(struct page *page, unsigned int order)
1454 {
1455 	int i;
1456 
1457 	VM_BUG_ON_PAGE(PageCompound(page), page);
1458 	VM_BUG_ON_PAGE(!page_count(page), page);
1459 
1460 #ifdef CONFIG_KMEMCHECK
1461 	/*
1462 	 * Split shadow pages too, because free(page[0]) would
1463 	 * otherwise free the whole shadow.
1464 	 */
1465 	if (kmemcheck_page_is_tracked(page))
1466 		split_page(virt_to_page(page[0].shadow), order);
1467 #endif
1468 
1469 	for (i = 1; i < (1 << order); i++)
1470 		set_page_refcounted(page + i);
1471 }
1472 EXPORT_SYMBOL_GPL(split_page);
1473 
1474 static int __isolate_free_page(struct page *page, unsigned int order)
1475 {
1476 	unsigned long watermark;
1477 	struct zone *zone;
1478 	int mt;
1479 
1480 	BUG_ON(!PageBuddy(page));
1481 
1482 	zone = page_zone(page);
1483 	mt = get_pageblock_migratetype(page);
1484 
1485 	if (!is_migrate_isolate(mt)) {
1486 		/* Obey watermarks as if the page was being allocated */
1487 		watermark = low_wmark_pages(zone) + (1 << order);
1488 		if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
1489 			return 0;
1490 
1491 		__mod_zone_freepage_state(zone, -(1UL << order), mt);
1492 	}
1493 
1494 	/* Remove page from free list */
1495 	list_del(&page->lru);
1496 	zone->free_area[order].nr_free--;
1497 	rmv_page_order(page);
1498 
1499 	/* Set the pageblock if the isolated page is at least a pageblock */
1500 	if (order >= pageblock_order - 1) {
1501 		struct page *endpage = page + (1 << order) - 1;
1502 		for (; page < endpage; page += pageblock_nr_pages) {
1503 			int mt = get_pageblock_migratetype(page);
1504 			if (!is_migrate_isolate(mt) && !is_migrate_cma(mt))
1505 				set_pageblock_migratetype(page,
1506 							  MIGRATE_MOVABLE);
1507 		}
1508 	}
1509 
1510 	return 1UL << order;
1511 }
1512 
1513 /*
1514  * Similar to split_page except the page is already free. As this is only
1515  * being used for migration, the migratetype of the block also changes.
1516  * As this is called with interrupts disabled, the caller is responsible
1517  * for calling arch_alloc_page() and kernel_map_page() after interrupts
1518  * are enabled.
1519  *
1520  * Note: this is probably too low level an operation for use in drivers.
1521  * Please consult with lkml before using this in your driver.
1522  */
1523 int split_free_page(struct page *page)
1524 {
1525 	unsigned int order;
1526 	int nr_pages;
1527 
1528 	order = page_order(page);
1529 
1530 	nr_pages = __isolate_free_page(page, order);
1531 	if (!nr_pages)
1532 		return 0;
1533 
1534 	/* Split into individual pages */
1535 	set_page_refcounted(page);
1536 	split_page(page, order);
1537 	return nr_pages;
1538 }
1539 
1540 /*
1541  * Really, prep_compound_page() should be called from __rmqueue_bulk().  But
1542  * we cheat by calling it from here, in the order > 0 path.  Saves a branch
1543  * or two.
1544  */
1545 static inline
1546 struct page *buffered_rmqueue(struct zone *preferred_zone,
1547 			struct zone *zone, unsigned int order,
1548 			gfp_t gfp_flags, int migratetype)
1549 {
1550 	unsigned long flags;
1551 	struct page *page;
1552 	bool cold = ((gfp_flags & __GFP_COLD) != 0);
1553 
1554 again:
1555 	if (likely(order == 0)) {
1556 		struct per_cpu_pages *pcp;
1557 		struct list_head *list;
1558 
1559 		local_irq_save(flags);
1560 		pcp = &this_cpu_ptr(zone->pageset)->pcp;
1561 		list = &pcp->lists[migratetype];
1562 		if (list_empty(list)) {
1563 			pcp->count += rmqueue_bulk(zone, 0,
1564 					pcp->batch, list,
1565 					migratetype, cold);
1566 			if (unlikely(list_empty(list)))
1567 				goto failed;
1568 		}
1569 
1570 		if (cold)
1571 			page = list_entry(list->prev, struct page, lru);
1572 		else
1573 			page = list_entry(list->next, struct page, lru);
1574 
1575 		list_del(&page->lru);
1576 		pcp->count--;
1577 	} else {
1578 		if (unlikely(gfp_flags & __GFP_NOFAIL)) {
1579 			/*
1580 			 * __GFP_NOFAIL is not to be used in new code.
1581 			 *
1582 			 * All __GFP_NOFAIL callers should be fixed so that they
1583 			 * properly detect and handle allocation failures.
1584 			 *
1585 			 * We most definitely don't want callers attempting to
1586 			 * allocate greater than order-1 page units with
1587 			 * __GFP_NOFAIL.
1588 			 */
1589 			WARN_ON_ONCE(order > 1);
1590 		}
1591 		spin_lock_irqsave(&zone->lock, flags);
1592 		page = __rmqueue(zone, order, migratetype);
1593 		spin_unlock(&zone->lock);
1594 		if (!page)
1595 			goto failed;
1596 		__mod_zone_freepage_state(zone, -(1 << order),
1597 					  get_freepage_migratetype(page));
1598 	}
1599 
1600 	__mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order));
1601 
1602 	__count_zone_vm_events(PGALLOC, zone, 1 << order);
1603 	zone_statistics(preferred_zone, zone, gfp_flags);
1604 	local_irq_restore(flags);
1605 
1606 	VM_BUG_ON_PAGE(bad_range(zone, page), page);
1607 	if (prep_new_page(page, order, gfp_flags))
1608 		goto again;
1609 	return page;
1610 
1611 failed:
1612 	local_irq_restore(flags);
1613 	return NULL;
1614 }
1615 
1616 #ifdef CONFIG_FAIL_PAGE_ALLOC
1617 
1618 static struct {
1619 	struct fault_attr attr;
1620 
1621 	u32 ignore_gfp_highmem;
1622 	u32 ignore_gfp_wait;
1623 	u32 min_order;
1624 } fail_page_alloc = {
1625 	.attr = FAULT_ATTR_INITIALIZER,
1626 	.ignore_gfp_wait = 1,
1627 	.ignore_gfp_highmem = 1,
1628 	.min_order = 1,
1629 };
1630 
1631 static int __init setup_fail_page_alloc(char *str)
1632 {
1633 	return setup_fault_attr(&fail_page_alloc.attr, str);
1634 }
1635 __setup("fail_page_alloc=", setup_fail_page_alloc);
1636 
1637 static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
1638 {
1639 	if (order < fail_page_alloc.min_order)
1640 		return false;
1641 	if (gfp_mask & __GFP_NOFAIL)
1642 		return false;
1643 	if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
1644 		return false;
1645 	if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT))
1646 		return false;
1647 
1648 	return should_fail(&fail_page_alloc.attr, 1 << order);
1649 }
1650 
1651 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
1652 
1653 static int __init fail_page_alloc_debugfs(void)
1654 {
1655 	umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
1656 	struct dentry *dir;
1657 
1658 	dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
1659 					&fail_page_alloc.attr);
1660 	if (IS_ERR(dir))
1661 		return PTR_ERR(dir);
1662 
1663 	if (!debugfs_create_bool("ignore-gfp-wait", mode, dir,
1664 				&fail_page_alloc.ignore_gfp_wait))
1665 		goto fail;
1666 	if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir,
1667 				&fail_page_alloc.ignore_gfp_highmem))
1668 		goto fail;
1669 	if (!debugfs_create_u32("min-order", mode, dir,
1670 				&fail_page_alloc.min_order))
1671 		goto fail;
1672 
1673 	return 0;
1674 fail:
1675 	debugfs_remove_recursive(dir);
1676 
1677 	return -ENOMEM;
1678 }
1679 
1680 late_initcall(fail_page_alloc_debugfs);
1681 
1682 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
1683 
1684 #else /* CONFIG_FAIL_PAGE_ALLOC */
1685 
1686 static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
1687 {
1688 	return false;
1689 }
1690 
1691 #endif /* CONFIG_FAIL_PAGE_ALLOC */
1692 
1693 /*
1694  * Return true if free pages are above 'mark'. This takes into account the order
1695  * of the allocation.
1696  */
1697 static bool __zone_watermark_ok(struct zone *z, unsigned int order,
1698 			unsigned long mark, int classzone_idx, int alloc_flags,
1699 			long free_pages)
1700 {
1701 	/* free_pages my go negative - that's OK */
1702 	long min = mark;
1703 	long lowmem_reserve = z->lowmem_reserve[classzone_idx];
1704 	int o;
1705 	long free_cma = 0;
1706 
1707 	free_pages -= (1 << order) - 1;
1708 	if (alloc_flags & ALLOC_HIGH)
1709 		min -= min / 2;
1710 	if (alloc_flags & ALLOC_HARDER)
1711 		min -= min / 4;
1712 #ifdef CONFIG_CMA
1713 	/* If allocation can't use CMA areas don't use free CMA pages */
1714 	if (!(alloc_flags & ALLOC_CMA))
1715 		free_cma = zone_page_state(z, NR_FREE_CMA_PAGES);
1716 #endif
1717 
1718 	if (free_pages - free_cma <= min + lowmem_reserve)
1719 		return false;
1720 	for (o = 0; o < order; o++) {
1721 		/* At the next order, this order's pages become unavailable */
1722 		free_pages -= z->free_area[o].nr_free << o;
1723 
1724 		/* Require fewer higher order pages to be free */
1725 		min >>= 1;
1726 
1727 		if (free_pages <= min)
1728 			return false;
1729 	}
1730 	return true;
1731 }
1732 
1733 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
1734 		      int classzone_idx, int alloc_flags)
1735 {
1736 	return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
1737 					zone_page_state(z, NR_FREE_PAGES));
1738 }
1739 
1740 bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
1741 			unsigned long mark, int classzone_idx, int alloc_flags)
1742 {
1743 	long free_pages = zone_page_state(z, NR_FREE_PAGES);
1744 
1745 	if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
1746 		free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
1747 
1748 	return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
1749 								free_pages);
1750 }
1751 
1752 #ifdef CONFIG_NUMA
1753 /*
1754  * zlc_setup - Setup for "zonelist cache".  Uses cached zone data to
1755  * skip over zones that are not allowed by the cpuset, or that have
1756  * been recently (in last second) found to be nearly full.  See further
1757  * comments in mmzone.h.  Reduces cache footprint of zonelist scans
1758  * that have to skip over a lot of full or unallowed zones.
1759  *
1760  * If the zonelist cache is present in the passed zonelist, then
1761  * returns a pointer to the allowed node mask (either the current
1762  * tasks mems_allowed, or node_states[N_MEMORY].)
1763  *
1764  * If the zonelist cache is not available for this zonelist, does
1765  * nothing and returns NULL.
1766  *
1767  * If the fullzones BITMAP in the zonelist cache is stale (more than
1768  * a second since last zap'd) then we zap it out (clear its bits.)
1769  *
1770  * We hold off even calling zlc_setup, until after we've checked the
1771  * first zone in the zonelist, on the theory that most allocations will
1772  * be satisfied from that first zone, so best to examine that zone as
1773  * quickly as we can.
1774  */
1775 static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
1776 {
1777 	struct zonelist_cache *zlc;	/* cached zonelist speedup info */
1778 	nodemask_t *allowednodes;	/* zonelist_cache approximation */
1779 
1780 	zlc = zonelist->zlcache_ptr;
1781 	if (!zlc)
1782 		return NULL;
1783 
1784 	if (time_after(jiffies, zlc->last_full_zap + HZ)) {
1785 		bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
1786 		zlc->last_full_zap = jiffies;
1787 	}
1788 
1789 	allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ?
1790 					&cpuset_current_mems_allowed :
1791 					&node_states[N_MEMORY];
1792 	return allowednodes;
1793 }
1794 
1795 /*
1796  * Given 'z' scanning a zonelist, run a couple of quick checks to see
1797  * if it is worth looking at further for free memory:
1798  *  1) Check that the zone isn't thought to be full (doesn't have its
1799  *     bit set in the zonelist_cache fullzones BITMAP).
1800  *  2) Check that the zones node (obtained from the zonelist_cache
1801  *     z_to_n[] mapping) is allowed in the passed in allowednodes mask.
1802  * Return true (non-zero) if zone is worth looking at further, or
1803  * else return false (zero) if it is not.
1804  *
1805  * This check -ignores- the distinction between various watermarks,
1806  * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ...  If a zone is
1807  * found to be full for any variation of these watermarks, it will
1808  * be considered full for up to one second by all requests, unless
1809  * we are so low on memory on all allowed nodes that we are forced
1810  * into the second scan of the zonelist.
1811  *
1812  * In the second scan we ignore this zonelist cache and exactly
1813  * apply the watermarks to all zones, even it is slower to do so.
1814  * We are low on memory in the second scan, and should leave no stone
1815  * unturned looking for a free page.
1816  */
1817 static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
1818 						nodemask_t *allowednodes)
1819 {
1820 	struct zonelist_cache *zlc;	/* cached zonelist speedup info */
1821 	int i;				/* index of *z in zonelist zones */
1822 	int n;				/* node that zone *z is on */
1823 
1824 	zlc = zonelist->zlcache_ptr;
1825 	if (!zlc)
1826 		return 1;
1827 
1828 	i = z - zonelist->_zonerefs;
1829 	n = zlc->z_to_n[i];
1830 
1831 	/* This zone is worth trying if it is allowed but not full */
1832 	return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones);
1833 }
1834 
1835 /*
1836  * Given 'z' scanning a zonelist, set the corresponding bit in
1837  * zlc->fullzones, so that subsequent attempts to allocate a page
1838  * from that zone don't waste time re-examining it.
1839  */
1840 static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
1841 {
1842 	struct zonelist_cache *zlc;	/* cached zonelist speedup info */
1843 	int i;				/* index of *z in zonelist zones */
1844 
1845 	zlc = zonelist->zlcache_ptr;
1846 	if (!zlc)
1847 		return;
1848 
1849 	i = z - zonelist->_zonerefs;
1850 
1851 	set_bit(i, zlc->fullzones);
1852 }
1853 
1854 /*
1855  * clear all zones full, called after direct reclaim makes progress so that
1856  * a zone that was recently full is not skipped over for up to a second
1857  */
1858 static void zlc_clear_zones_full(struct zonelist *zonelist)
1859 {
1860 	struct zonelist_cache *zlc;	/* cached zonelist speedup info */
1861 
1862 	zlc = zonelist->zlcache_ptr;
1863 	if (!zlc)
1864 		return;
1865 
1866 	bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
1867 }
1868 
1869 static bool zone_local(struct zone *local_zone, struct zone *zone)
1870 {
1871 	return local_zone->node == zone->node;
1872 }
1873 
1874 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
1875 {
1876 	return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <
1877 				RECLAIM_DISTANCE;
1878 }
1879 
1880 #else	/* CONFIG_NUMA */
1881 
1882 static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
1883 {
1884 	return NULL;
1885 }
1886 
1887 static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
1888 				nodemask_t *allowednodes)
1889 {
1890 	return 1;
1891 }
1892 
1893 static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
1894 {
1895 }
1896 
1897 static void zlc_clear_zones_full(struct zonelist *zonelist)
1898 {
1899 }
1900 
1901 static bool zone_local(struct zone *local_zone, struct zone *zone)
1902 {
1903 	return true;
1904 }
1905 
1906 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
1907 {
1908 	return true;
1909 }
1910 
1911 #endif	/* CONFIG_NUMA */
1912 
1913 /*
1914  * get_page_from_freelist goes through the zonelist trying to allocate
1915  * a page.
1916  */
1917 static struct page *
1918 get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
1919 		struct zonelist *zonelist, int high_zoneidx, int alloc_flags,
1920 		struct zone *preferred_zone, int classzone_idx, int migratetype)
1921 {
1922 	struct zoneref *z;
1923 	struct page *page = NULL;
1924 	struct zone *zone;
1925 	nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */
1926 	int zlc_active = 0;		/* set if using zonelist_cache */
1927 	int did_zlc_setup = 0;		/* just call zlc_setup() one time */
1928 	bool consider_zone_dirty = (alloc_flags & ALLOC_WMARK_LOW) &&
1929 				(gfp_mask & __GFP_WRITE);
1930 
1931 zonelist_scan:
1932 	/*
1933 	 * Scan zonelist, looking for a zone with enough free.
1934 	 * See also __cpuset_node_allowed_softwall() comment in kernel/cpuset.c.
1935 	 */
1936 	for_each_zone_zonelist_nodemask(zone, z, zonelist,
1937 						high_zoneidx, nodemask) {
1938 		unsigned long mark;
1939 
1940 		if (IS_ENABLED(CONFIG_NUMA) && zlc_active &&
1941 			!zlc_zone_worth_trying(zonelist, z, allowednodes))
1942 				continue;
1943 		if (cpusets_enabled() &&
1944 			(alloc_flags & ALLOC_CPUSET) &&
1945 			!cpuset_zone_allowed_softwall(zone, gfp_mask))
1946 				continue;
1947 		/*
1948 		 * Distribute pages in proportion to the individual
1949 		 * zone size to ensure fair page aging.  The zone a
1950 		 * page was allocated in should have no effect on the
1951 		 * time the page has in memory before being reclaimed.
1952 		 */
1953 		if (alloc_flags & ALLOC_FAIR) {
1954 			if (!zone_local(preferred_zone, zone))
1955 				continue;
1956 			if (zone_page_state(zone, NR_ALLOC_BATCH) <= 0)
1957 				continue;
1958 		}
1959 		/*
1960 		 * When allocating a page cache page for writing, we
1961 		 * want to get it from a zone that is within its dirty
1962 		 * limit, such that no single zone holds more than its
1963 		 * proportional share of globally allowed dirty pages.
1964 		 * The dirty limits take into account the zone's
1965 		 * lowmem reserves and high watermark so that kswapd
1966 		 * should be able to balance it without having to
1967 		 * write pages from its LRU list.
1968 		 *
1969 		 * This may look like it could increase pressure on
1970 		 * lower zones by failing allocations in higher zones
1971 		 * before they are full.  But the pages that do spill
1972 		 * over are limited as the lower zones are protected
1973 		 * by this very same mechanism.  It should not become
1974 		 * a practical burden to them.
1975 		 *
1976 		 * XXX: For now, allow allocations to potentially
1977 		 * exceed the per-zone dirty limit in the slowpath
1978 		 * (ALLOC_WMARK_LOW unset) before going into reclaim,
1979 		 * which is important when on a NUMA setup the allowed
1980 		 * zones are together not big enough to reach the
1981 		 * global limit.  The proper fix for these situations
1982 		 * will require awareness of zones in the
1983 		 * dirty-throttling and the flusher threads.
1984 		 */
1985 		if (consider_zone_dirty && !zone_dirty_ok(zone))
1986 			continue;
1987 
1988 		mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
1989 		if (!zone_watermark_ok(zone, order, mark,
1990 				       classzone_idx, alloc_flags)) {
1991 			int ret;
1992 
1993 			/* Checked here to keep the fast path fast */
1994 			BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
1995 			if (alloc_flags & ALLOC_NO_WATERMARKS)
1996 				goto try_this_zone;
1997 
1998 			if (IS_ENABLED(CONFIG_NUMA) &&
1999 					!did_zlc_setup && nr_online_nodes > 1) {
2000 				/*
2001 				 * we do zlc_setup if there are multiple nodes
2002 				 * and before considering the first zone allowed
2003 				 * by the cpuset.
2004 				 */
2005 				allowednodes = zlc_setup(zonelist, alloc_flags);
2006 				zlc_active = 1;
2007 				did_zlc_setup = 1;
2008 			}
2009 
2010 			if (zone_reclaim_mode == 0 ||
2011 			    !zone_allows_reclaim(preferred_zone, zone))
2012 				goto this_zone_full;
2013 
2014 			/*
2015 			 * As we may have just activated ZLC, check if the first
2016 			 * eligible zone has failed zone_reclaim recently.
2017 			 */
2018 			if (IS_ENABLED(CONFIG_NUMA) && zlc_active &&
2019 				!zlc_zone_worth_trying(zonelist, z, allowednodes))
2020 				continue;
2021 
2022 			ret = zone_reclaim(zone, gfp_mask, order);
2023 			switch (ret) {
2024 			case ZONE_RECLAIM_NOSCAN:
2025 				/* did not scan */
2026 				continue;
2027 			case ZONE_RECLAIM_FULL:
2028 				/* scanned but unreclaimable */
2029 				continue;
2030 			default:
2031 				/* did we reclaim enough */
2032 				if (zone_watermark_ok(zone, order, mark,
2033 						classzone_idx, alloc_flags))
2034 					goto try_this_zone;
2035 
2036 				/*
2037 				 * Failed to reclaim enough to meet watermark.
2038 				 * Only mark the zone full if checking the min
2039 				 * watermark or if we failed to reclaim just
2040 				 * 1<<order pages or else the page allocator
2041 				 * fastpath will prematurely mark zones full
2042 				 * when the watermark is between the low and
2043 				 * min watermarks.
2044 				 */
2045 				if (((alloc_flags & ALLOC_WMARK_MASK) == ALLOC_WMARK_MIN) ||
2046 				    ret == ZONE_RECLAIM_SOME)
2047 					goto this_zone_full;
2048 
2049 				continue;
2050 			}
2051 		}
2052 
2053 try_this_zone:
2054 		page = buffered_rmqueue(preferred_zone, zone, order,
2055 						gfp_mask, migratetype);
2056 		if (page)
2057 			break;
2058 this_zone_full:
2059 		if (IS_ENABLED(CONFIG_NUMA) && zlc_active)
2060 			zlc_mark_zone_full(zonelist, z);
2061 	}
2062 
2063 	if (unlikely(IS_ENABLED(CONFIG_NUMA) && page == NULL && zlc_active)) {
2064 		/* Disable zlc cache for second zonelist scan */
2065 		zlc_active = 0;
2066 		goto zonelist_scan;
2067 	}
2068 
2069 	if (page)
2070 		/*
2071 		 * page->pfmemalloc is set when ALLOC_NO_WATERMARKS was
2072 		 * necessary to allocate the page. The expectation is
2073 		 * that the caller is taking steps that will free more
2074 		 * memory. The caller should avoid the page being used
2075 		 * for !PFMEMALLOC purposes.
2076 		 */
2077 		page->pfmemalloc = !!(alloc_flags & ALLOC_NO_WATERMARKS);
2078 
2079 	return page;
2080 }
2081 
2082 /*
2083  * Large machines with many possible nodes should not always dump per-node
2084  * meminfo in irq context.
2085  */
2086 static inline bool should_suppress_show_mem(void)
2087 {
2088 	bool ret = false;
2089 
2090 #if NODES_SHIFT > 8
2091 	ret = in_interrupt();
2092 #endif
2093 	return ret;
2094 }
2095 
2096 static DEFINE_RATELIMIT_STATE(nopage_rs,
2097 		DEFAULT_RATELIMIT_INTERVAL,
2098 		DEFAULT_RATELIMIT_BURST);
2099 
2100 void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...)
2101 {
2102 	unsigned int filter = SHOW_MEM_FILTER_NODES;
2103 
2104 	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
2105 	    debug_guardpage_minorder() > 0)
2106 		return;
2107 
2108 	/*
2109 	 * This documents exceptions given to allocations in certain
2110 	 * contexts that are allowed to allocate outside current's set
2111 	 * of allowed nodes.
2112 	 */
2113 	if (!(gfp_mask & __GFP_NOMEMALLOC))
2114 		if (test_thread_flag(TIF_MEMDIE) ||
2115 		    (current->flags & (PF_MEMALLOC | PF_EXITING)))
2116 			filter &= ~SHOW_MEM_FILTER_NODES;
2117 	if (in_interrupt() || !(gfp_mask & __GFP_WAIT))
2118 		filter &= ~SHOW_MEM_FILTER_NODES;
2119 
2120 	if (fmt) {
2121 		struct va_format vaf;
2122 		va_list args;
2123 
2124 		va_start(args, fmt);
2125 
2126 		vaf.fmt = fmt;
2127 		vaf.va = &args;
2128 
2129 		pr_warn("%pV", &vaf);
2130 
2131 		va_end(args);
2132 	}
2133 
2134 	pr_warn("%s: page allocation failure: order:%d, mode:0x%x\n",
2135 		current->comm, order, gfp_mask);
2136 
2137 	dump_stack();
2138 	if (!should_suppress_show_mem())
2139 		show_mem(filter);
2140 }
2141 
2142 static inline int
2143 should_alloc_retry(gfp_t gfp_mask, unsigned int order,
2144 				unsigned long did_some_progress,
2145 				unsigned long pages_reclaimed)
2146 {
2147 	/* Do not loop if specifically requested */
2148 	if (gfp_mask & __GFP_NORETRY)
2149 		return 0;
2150 
2151 	/* Always retry if specifically requested */
2152 	if (gfp_mask & __GFP_NOFAIL)
2153 		return 1;
2154 
2155 	/*
2156 	 * Suspend converts GFP_KERNEL to __GFP_WAIT which can prevent reclaim
2157 	 * making forward progress without invoking OOM. Suspend also disables
2158 	 * storage devices so kswapd will not help. Bail if we are suspending.
2159 	 */
2160 	if (!did_some_progress && pm_suspended_storage())
2161 		return 0;
2162 
2163 	/*
2164 	 * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
2165 	 * means __GFP_NOFAIL, but that may not be true in other
2166 	 * implementations.
2167 	 */
2168 	if (order <= PAGE_ALLOC_COSTLY_ORDER)
2169 		return 1;
2170 
2171 	/*
2172 	 * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
2173 	 * specified, then we retry until we no longer reclaim any pages
2174 	 * (above), or we've reclaimed an order of pages at least as
2175 	 * large as the allocation's order. In both cases, if the
2176 	 * allocation still fails, we stop retrying.
2177 	 */
2178 	if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order))
2179 		return 1;
2180 
2181 	return 0;
2182 }
2183 
2184 static inline struct page *
2185 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
2186 	struct zonelist *zonelist, enum zone_type high_zoneidx,
2187 	nodemask_t *nodemask, struct zone *preferred_zone,
2188 	int classzone_idx, int migratetype)
2189 {
2190 	struct page *page;
2191 
2192 	/* Acquire the OOM killer lock for the zones in zonelist */
2193 	if (!try_set_zonelist_oom(zonelist, gfp_mask)) {
2194 		schedule_timeout_uninterruptible(1);
2195 		return NULL;
2196 	}
2197 
2198 	/*
2199 	 * Go through the zonelist yet one more time, keep very high watermark
2200 	 * here, this is only to catch a parallel oom killing, we must fail if
2201 	 * we're still under heavy pressure.
2202 	 */
2203 	page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask,
2204 		order, zonelist, high_zoneidx,
2205 		ALLOC_WMARK_HIGH|ALLOC_CPUSET,
2206 		preferred_zone, classzone_idx, migratetype);
2207 	if (page)
2208 		goto out;
2209 
2210 	if (!(gfp_mask & __GFP_NOFAIL)) {
2211 		/* The OOM killer will not help higher order allocs */
2212 		if (order > PAGE_ALLOC_COSTLY_ORDER)
2213 			goto out;
2214 		/* The OOM killer does not needlessly kill tasks for lowmem */
2215 		if (high_zoneidx < ZONE_NORMAL)
2216 			goto out;
2217 		/*
2218 		 * GFP_THISNODE contains __GFP_NORETRY and we never hit this.
2219 		 * Sanity check for bare calls of __GFP_THISNODE, not real OOM.
2220 		 * The caller should handle page allocation failure by itself if
2221 		 * it specifies __GFP_THISNODE.
2222 		 * Note: Hugepage uses it but will hit PAGE_ALLOC_COSTLY_ORDER.
2223 		 */
2224 		if (gfp_mask & __GFP_THISNODE)
2225 			goto out;
2226 	}
2227 	/* Exhausted what can be done so it's blamo time */
2228 	out_of_memory(zonelist, gfp_mask, order, nodemask, false);
2229 
2230 out:
2231 	clear_zonelist_oom(zonelist, gfp_mask);
2232 	return page;
2233 }
2234 
2235 #ifdef CONFIG_COMPACTION
2236 /* Try memory compaction for high-order allocations before reclaim */
2237 static struct page *
2238 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
2239 	struct zonelist *zonelist, enum zone_type high_zoneidx,
2240 	nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
2241 	int classzone_idx, int migratetype, enum migrate_mode mode,
2242 	bool *contended_compaction, bool *deferred_compaction,
2243 	unsigned long *did_some_progress)
2244 {
2245 	if (!order)
2246 		return NULL;
2247 
2248 	if (compaction_deferred(preferred_zone, order)) {
2249 		*deferred_compaction = true;
2250 		return NULL;
2251 	}
2252 
2253 	current->flags |= PF_MEMALLOC;
2254 	*did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
2255 						nodemask, mode,
2256 						contended_compaction);
2257 	current->flags &= ~PF_MEMALLOC;
2258 
2259 	if (*did_some_progress != COMPACT_SKIPPED) {
2260 		struct page *page;
2261 
2262 		/* Page migration frees to the PCP lists but we want merging */
2263 		drain_pages(get_cpu());
2264 		put_cpu();
2265 
2266 		page = get_page_from_freelist(gfp_mask, nodemask,
2267 				order, zonelist, high_zoneidx,
2268 				alloc_flags & ~ALLOC_NO_WATERMARKS,
2269 				preferred_zone, classzone_idx, migratetype);
2270 		if (page) {
2271 			preferred_zone->compact_blockskip_flush = false;
2272 			compaction_defer_reset(preferred_zone, order, true);
2273 			count_vm_event(COMPACTSUCCESS);
2274 			return page;
2275 		}
2276 
2277 		/*
2278 		 * It's bad if compaction run occurs and fails.
2279 		 * The most likely reason is that pages exist,
2280 		 * but not enough to satisfy watermarks.
2281 		 */
2282 		count_vm_event(COMPACTFAIL);
2283 
2284 		/*
2285 		 * As async compaction considers a subset of pageblocks, only
2286 		 * defer if the failure was a sync compaction failure.
2287 		 */
2288 		if (mode != MIGRATE_ASYNC)
2289 			defer_compaction(preferred_zone, order);
2290 
2291 		cond_resched();
2292 	}
2293 
2294 	return NULL;
2295 }
2296 #else
2297 static inline struct page *
2298 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
2299 	struct zonelist *zonelist, enum zone_type high_zoneidx,
2300 	nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
2301 	int classzone_idx, int migratetype,
2302 	enum migrate_mode mode, bool *contended_compaction,
2303 	bool *deferred_compaction, unsigned long *did_some_progress)
2304 {
2305 	return NULL;
2306 }
2307 #endif /* CONFIG_COMPACTION */
2308 
2309 /* Perform direct synchronous page reclaim */
2310 static int
2311 __perform_reclaim(gfp_t gfp_mask, unsigned int order, struct zonelist *zonelist,
2312 		  nodemask_t *nodemask)
2313 {
2314 	struct reclaim_state reclaim_state;
2315 	int progress;
2316 
2317 	cond_resched();
2318 
2319 	/* We now go into synchronous reclaim */
2320 	cpuset_memory_pressure_bump();
2321 	current->flags |= PF_MEMALLOC;
2322 	lockdep_set_current_reclaim_state(gfp_mask);
2323 	reclaim_state.reclaimed_slab = 0;
2324 	current->reclaim_state = &reclaim_state;
2325 
2326 	progress = try_to_free_pages(zonelist, order, gfp_mask, nodemask);
2327 
2328 	current->reclaim_state = NULL;
2329 	lockdep_clear_current_reclaim_state();
2330 	current->flags &= ~PF_MEMALLOC;
2331 
2332 	cond_resched();
2333 
2334 	return progress;
2335 }
2336 
2337 /* The really slow allocator path where we enter direct reclaim */
2338 static inline struct page *
2339 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
2340 	struct zonelist *zonelist, enum zone_type high_zoneidx,
2341 	nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
2342 	int classzone_idx, int migratetype, unsigned long *did_some_progress)
2343 {
2344 	struct page *page = NULL;
2345 	bool drained = false;
2346 
2347 	*did_some_progress = __perform_reclaim(gfp_mask, order, zonelist,
2348 					       nodemask);
2349 	if (unlikely(!(*did_some_progress)))
2350 		return NULL;
2351 
2352 	/* After successful reclaim, reconsider all zones for allocation */
2353 	if (IS_ENABLED(CONFIG_NUMA))
2354 		zlc_clear_zones_full(zonelist);
2355 
2356 retry:
2357 	page = get_page_from_freelist(gfp_mask, nodemask, order,
2358 					zonelist, high_zoneidx,
2359 					alloc_flags & ~ALLOC_NO_WATERMARKS,
2360 					preferred_zone, classzone_idx,
2361 					migratetype);
2362 
2363 	/*
2364 	 * If an allocation failed after direct reclaim, it could be because
2365 	 * pages are pinned on the per-cpu lists. Drain them and try again
2366 	 */
2367 	if (!page && !drained) {
2368 		drain_all_pages();
2369 		drained = true;
2370 		goto retry;
2371 	}
2372 
2373 	return page;
2374 }
2375 
2376 /*
2377  * This is called in the allocator slow-path if the allocation request is of
2378  * sufficient urgency to ignore watermarks and take other desperate measures
2379  */
2380 static inline struct page *
2381 __alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
2382 	struct zonelist *zonelist, enum zone_type high_zoneidx,
2383 	nodemask_t *nodemask, struct zone *preferred_zone,
2384 	int classzone_idx, int migratetype)
2385 {
2386 	struct page *page;
2387 
2388 	do {
2389 		page = get_page_from_freelist(gfp_mask, nodemask, order,
2390 			zonelist, high_zoneidx, ALLOC_NO_WATERMARKS,
2391 			preferred_zone, classzone_idx, migratetype);
2392 
2393 		if (!page && gfp_mask & __GFP_NOFAIL)
2394 			wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50);
2395 	} while (!page && (gfp_mask & __GFP_NOFAIL));
2396 
2397 	return page;
2398 }
2399 
2400 static void reset_alloc_batches(struct zonelist *zonelist,
2401 				enum zone_type high_zoneidx,
2402 				struct zone *preferred_zone)
2403 {
2404 	struct zoneref *z;
2405 	struct zone *zone;
2406 
2407 	for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
2408 		/*
2409 		 * Only reset the batches of zones that were actually
2410 		 * considered in the fairness pass, we don't want to
2411 		 * trash fairness information for zones that are not
2412 		 * actually part of this zonelist's round-robin cycle.
2413 		 */
2414 		if (!zone_local(preferred_zone, zone))
2415 			continue;
2416 		mod_zone_page_state(zone, NR_ALLOC_BATCH,
2417 			high_wmark_pages(zone) - low_wmark_pages(zone) -
2418 			atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]));
2419 	}
2420 }
2421 
2422 static void wake_all_kswapds(unsigned int order,
2423 			     struct zonelist *zonelist,
2424 			     enum zone_type high_zoneidx,
2425 			     struct zone *preferred_zone)
2426 {
2427 	struct zoneref *z;
2428 	struct zone *zone;
2429 
2430 	for_each_zone_zonelist(zone, z, zonelist, high_zoneidx)
2431 		wakeup_kswapd(zone, order, zone_idx(preferred_zone));
2432 }
2433 
2434 static inline int
2435 gfp_to_alloc_flags(gfp_t gfp_mask)
2436 {
2437 	int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
2438 	const gfp_t wait = gfp_mask & __GFP_WAIT;
2439 
2440 	/* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
2441 	BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
2442 
2443 	/*
2444 	 * The caller may dip into page reserves a bit more if the caller
2445 	 * cannot run direct reclaim, or if the caller has realtime scheduling
2446 	 * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
2447 	 * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
2448 	 */
2449 	alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
2450 
2451 	if (!wait) {
2452 		/*
2453 		 * Not worth trying to allocate harder for
2454 		 * __GFP_NOMEMALLOC even if it can't schedule.
2455 		 */
2456 		if  (!(gfp_mask & __GFP_NOMEMALLOC))
2457 			alloc_flags |= ALLOC_HARDER;
2458 		/*
2459 		 * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.
2460 		 * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
2461 		 */
2462 		alloc_flags &= ~ALLOC_CPUSET;
2463 	} else if (unlikely(rt_task(current)) && !in_interrupt())
2464 		alloc_flags |= ALLOC_HARDER;
2465 
2466 	if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {
2467 		if (gfp_mask & __GFP_MEMALLOC)
2468 			alloc_flags |= ALLOC_NO_WATERMARKS;
2469 		else if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
2470 			alloc_flags |= ALLOC_NO_WATERMARKS;
2471 		else if (!in_interrupt() &&
2472 				((current->flags & PF_MEMALLOC) ||
2473 				 unlikely(test_thread_flag(TIF_MEMDIE))))
2474 			alloc_flags |= ALLOC_NO_WATERMARKS;
2475 	}
2476 #ifdef CONFIG_CMA
2477 	if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
2478 		alloc_flags |= ALLOC_CMA;
2479 #endif
2480 	return alloc_flags;
2481 }
2482 
2483 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
2484 {
2485 	return !!(gfp_to_alloc_flags(gfp_mask) & ALLOC_NO_WATERMARKS);
2486 }
2487 
2488 static inline struct page *
2489 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
2490 	struct zonelist *zonelist, enum zone_type high_zoneidx,
2491 	nodemask_t *nodemask, struct zone *preferred_zone,
2492 	int classzone_idx, int migratetype)
2493 {
2494 	const gfp_t wait = gfp_mask & __GFP_WAIT;
2495 	struct page *page = NULL;
2496 	int alloc_flags;
2497 	unsigned long pages_reclaimed = 0;
2498 	unsigned long did_some_progress;
2499 	enum migrate_mode migration_mode = MIGRATE_ASYNC;
2500 	bool deferred_compaction = false;
2501 	bool contended_compaction = false;
2502 
2503 	/*
2504 	 * In the slowpath, we sanity check order to avoid ever trying to
2505 	 * reclaim >= MAX_ORDER areas which will never succeed. Callers may
2506 	 * be using allocators in order of preference for an area that is
2507 	 * too large.
2508 	 */
2509 	if (order >= MAX_ORDER) {
2510 		WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
2511 		return NULL;
2512 	}
2513 
2514 	/*
2515 	 * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and
2516 	 * __GFP_NOWARN set) should not cause reclaim since the subsystem
2517 	 * (f.e. slab) using GFP_THISNODE may choose to trigger reclaim
2518 	 * using a larger set of nodes after it has established that the
2519 	 * allowed per node queues are empty and that nodes are
2520 	 * over allocated.
2521 	 */
2522 	if (IS_ENABLED(CONFIG_NUMA) &&
2523 	    (gfp_mask & GFP_THISNODE) == GFP_THISNODE)
2524 		goto nopage;
2525 
2526 restart:
2527 	if (!(gfp_mask & __GFP_NO_KSWAPD))
2528 		wake_all_kswapds(order, zonelist, high_zoneidx, preferred_zone);
2529 
2530 	/*
2531 	 * OK, we're below the kswapd watermark and have kicked background
2532 	 * reclaim. Now things get more complex, so set up alloc_flags according
2533 	 * to how we want to proceed.
2534 	 */
2535 	alloc_flags = gfp_to_alloc_flags(gfp_mask);
2536 
2537 	/*
2538 	 * Find the true preferred zone if the allocation is unconstrained by
2539 	 * cpusets.
2540 	 */
2541 	if (!(alloc_flags & ALLOC_CPUSET) && !nodemask) {
2542 		struct zoneref *preferred_zoneref;
2543 		preferred_zoneref = first_zones_zonelist(zonelist, high_zoneidx,
2544 				NULL, &preferred_zone);
2545 		classzone_idx = zonelist_zone_idx(preferred_zoneref);
2546 	}
2547 
2548 rebalance:
2549 	/* This is the last chance, in general, before the goto nopage. */
2550 	page = get_page_from_freelist(gfp_mask, nodemask, order, zonelist,
2551 			high_zoneidx, alloc_flags & ~ALLOC_NO_WATERMARKS,
2552 			preferred_zone, classzone_idx, migratetype);
2553 	if (page)
2554 		goto got_pg;
2555 
2556 	/* Allocate without watermarks if the context allows */
2557 	if (alloc_flags & ALLOC_NO_WATERMARKS) {
2558 		/*
2559 		 * Ignore mempolicies if ALLOC_NO_WATERMARKS on the grounds
2560 		 * the allocation is high priority and these type of
2561 		 * allocations are system rather than user orientated
2562 		 */
2563 		zonelist = node_zonelist(numa_node_id(), gfp_mask);
2564 
2565 		page = __alloc_pages_high_priority(gfp_mask, order,
2566 				zonelist, high_zoneidx, nodemask,
2567 				preferred_zone, classzone_idx, migratetype);
2568 		if (page) {
2569 			goto got_pg;
2570 		}
2571 	}
2572 
2573 	/* Atomic allocations - we can't balance anything */
2574 	if (!wait) {
2575 		/*
2576 		 * All existing users of the deprecated __GFP_NOFAIL are
2577 		 * blockable, so warn of any new users that actually allow this
2578 		 * type of allocation to fail.
2579 		 */
2580 		WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL);
2581 		goto nopage;
2582 	}
2583 
2584 	/* Avoid recursion of direct reclaim */
2585 	if (current->flags & PF_MEMALLOC)
2586 		goto nopage;
2587 
2588 	/* Avoid allocations with no watermarks from looping endlessly */
2589 	if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
2590 		goto nopage;
2591 
2592 	/*
2593 	 * Try direct compaction. The first pass is asynchronous. Subsequent
2594 	 * attempts after direct reclaim are synchronous
2595 	 */
2596 	page = __alloc_pages_direct_compact(gfp_mask, order, zonelist,
2597 					high_zoneidx, nodemask, alloc_flags,
2598 					preferred_zone,
2599 					classzone_idx, migratetype,
2600 					migration_mode, &contended_compaction,
2601 					&deferred_compaction,
2602 					&did_some_progress);
2603 	if (page)
2604 		goto got_pg;
2605 
2606 	/*
2607 	 * It can become very expensive to allocate transparent hugepages at
2608 	 * fault, so use asynchronous memory compaction for THP unless it is
2609 	 * khugepaged trying to collapse.
2610 	 */
2611 	if (!(gfp_mask & __GFP_NO_KSWAPD) || (current->flags & PF_KTHREAD))
2612 		migration_mode = MIGRATE_SYNC_LIGHT;
2613 
2614 	/*
2615 	 * If compaction is deferred for high-order allocations, it is because
2616 	 * sync compaction recently failed. In this is the case and the caller
2617 	 * requested a movable allocation that does not heavily disrupt the
2618 	 * system then fail the allocation instead of entering direct reclaim.
2619 	 */
2620 	if ((deferred_compaction || contended_compaction) &&
2621 						(gfp_mask & __GFP_NO_KSWAPD))
2622 		goto nopage;
2623 
2624 	/* Try direct reclaim and then allocating */
2625 	page = __alloc_pages_direct_reclaim(gfp_mask, order,
2626 					zonelist, high_zoneidx,
2627 					nodemask,
2628 					alloc_flags, preferred_zone,
2629 					classzone_idx, migratetype,
2630 					&did_some_progress);
2631 	if (page)
2632 		goto got_pg;
2633 
2634 	/*
2635 	 * If we failed to make any progress reclaiming, then we are
2636 	 * running out of options and have to consider going OOM
2637 	 */
2638 	if (!did_some_progress) {
2639 		if (oom_gfp_allowed(gfp_mask)) {
2640 			if (oom_killer_disabled)
2641 				goto nopage;
2642 			/* Coredumps can quickly deplete all memory reserves */
2643 			if ((current->flags & PF_DUMPCORE) &&
2644 			    !(gfp_mask & __GFP_NOFAIL))
2645 				goto nopage;
2646 			page = __alloc_pages_may_oom(gfp_mask, order,
2647 					zonelist, high_zoneidx,
2648 					nodemask, preferred_zone,
2649 					classzone_idx, migratetype);
2650 			if (page)
2651 				goto got_pg;
2652 
2653 			if (!(gfp_mask & __GFP_NOFAIL)) {
2654 				/*
2655 				 * The oom killer is not called for high-order
2656 				 * allocations that may fail, so if no progress
2657 				 * is being made, there are no other options and
2658 				 * retrying is unlikely to help.
2659 				 */
2660 				if (order > PAGE_ALLOC_COSTLY_ORDER)
2661 					goto nopage;
2662 				/*
2663 				 * The oom killer is not called for lowmem
2664 				 * allocations to prevent needlessly killing
2665 				 * innocent tasks.
2666 				 */
2667 				if (high_zoneidx < ZONE_NORMAL)
2668 					goto nopage;
2669 			}
2670 
2671 			goto restart;
2672 		}
2673 	}
2674 
2675 	/* Check if we should retry the allocation */
2676 	pages_reclaimed += did_some_progress;
2677 	if (should_alloc_retry(gfp_mask, order, did_some_progress,
2678 						pages_reclaimed)) {
2679 		/* Wait for some write requests to complete then retry */
2680 		wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50);
2681 		goto rebalance;
2682 	} else {
2683 		/*
2684 		 * High-order allocations do not necessarily loop after
2685 		 * direct reclaim and reclaim/compaction depends on compaction
2686 		 * being called after reclaim so call directly if necessary
2687 		 */
2688 		page = __alloc_pages_direct_compact(gfp_mask, order, zonelist,
2689 					high_zoneidx, nodemask, alloc_flags,
2690 					preferred_zone,
2691 					classzone_idx, migratetype,
2692 					migration_mode, &contended_compaction,
2693 					&deferred_compaction,
2694 					&did_some_progress);
2695 		if (page)
2696 			goto got_pg;
2697 	}
2698 
2699 nopage:
2700 	warn_alloc_failed(gfp_mask, order, NULL);
2701 	return page;
2702 got_pg:
2703 	if (kmemcheck_enabled)
2704 		kmemcheck_pagealloc_alloc(page, order, gfp_mask);
2705 
2706 	return page;
2707 }
2708 
2709 /*
2710  * This is the 'heart' of the zoned buddy allocator.
2711  */
2712 struct page *
2713 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
2714 			struct zonelist *zonelist, nodemask_t *nodemask)
2715 {
2716 	enum zone_type high_zoneidx = gfp_zone(gfp_mask);
2717 	struct zone *preferred_zone;
2718 	struct zoneref *preferred_zoneref;
2719 	struct page *page = NULL;
2720 	int migratetype = allocflags_to_migratetype(gfp_mask);
2721 	unsigned int cpuset_mems_cookie;
2722 	int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET|ALLOC_FAIR;
2723 	int classzone_idx;
2724 
2725 	gfp_mask &= gfp_allowed_mask;
2726 
2727 	lockdep_trace_alloc(gfp_mask);
2728 
2729 	might_sleep_if(gfp_mask & __GFP_WAIT);
2730 
2731 	if (should_fail_alloc_page(gfp_mask, order))
2732 		return NULL;
2733 
2734 	/*
2735 	 * Check the zones suitable for the gfp_mask contain at least one
2736 	 * valid zone. It's possible to have an empty zonelist as a result
2737 	 * of GFP_THISNODE and a memoryless node
2738 	 */
2739 	if (unlikely(!zonelist->_zonerefs->zone))
2740 		return NULL;
2741 
2742 retry_cpuset:
2743 	cpuset_mems_cookie = read_mems_allowed_begin();
2744 
2745 	/* The preferred zone is used for statistics later */
2746 	preferred_zoneref = first_zones_zonelist(zonelist, high_zoneidx,
2747 				nodemask ? : &cpuset_current_mems_allowed,
2748 				&preferred_zone);
2749 	if (!preferred_zone)
2750 		goto out;
2751 	classzone_idx = zonelist_zone_idx(preferred_zoneref);
2752 
2753 #ifdef CONFIG_CMA
2754 	if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
2755 		alloc_flags |= ALLOC_CMA;
2756 #endif
2757 retry:
2758 	/* First allocation attempt */
2759 	page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
2760 			zonelist, high_zoneidx, alloc_flags,
2761 			preferred_zone, classzone_idx, migratetype);
2762 	if (unlikely(!page)) {
2763 		/*
2764 		 * The first pass makes sure allocations are spread
2765 		 * fairly within the local node.  However, the local
2766 		 * node might have free pages left after the fairness
2767 		 * batches are exhausted, and remote zones haven't
2768 		 * even been considered yet.  Try once more without
2769 		 * fairness, and include remote zones now, before
2770 		 * entering the slowpath and waking kswapd: prefer
2771 		 * spilling to a remote zone over swapping locally.
2772 		 */
2773 		if (alloc_flags & ALLOC_FAIR) {
2774 			reset_alloc_batches(zonelist, high_zoneidx,
2775 					    preferred_zone);
2776 			alloc_flags &= ~ALLOC_FAIR;
2777 			goto retry;
2778 		}
2779 		/*
2780 		 * Runtime PM, block IO and its error handling path
2781 		 * can deadlock because I/O on the device might not
2782 		 * complete.
2783 		 */
2784 		gfp_mask = memalloc_noio_flags(gfp_mask);
2785 		page = __alloc_pages_slowpath(gfp_mask, order,
2786 				zonelist, high_zoneidx, nodemask,
2787 				preferred_zone, classzone_idx, migratetype);
2788 	}
2789 
2790 	trace_mm_page_alloc(page, order, gfp_mask, migratetype);
2791 
2792 out:
2793 	/*
2794 	 * When updating a task's mems_allowed, it is possible to race with
2795 	 * parallel threads in such a way that an allocation can fail while
2796 	 * the mask is being updated. If a page allocation is about to fail,
2797 	 * check if the cpuset changed during allocation and if so, retry.
2798 	 */
2799 	if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
2800 		goto retry_cpuset;
2801 
2802 	return page;
2803 }
2804 EXPORT_SYMBOL(__alloc_pages_nodemask);
2805 
2806 /*
2807  * Common helper functions.
2808  */
2809 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
2810 {
2811 	struct page *page;
2812 
2813 	/*
2814 	 * __get_free_pages() returns a 32-bit address, which cannot represent
2815 	 * a highmem page
2816 	 */
2817 	VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
2818 
2819 	page = alloc_pages(gfp_mask, order);
2820 	if (!page)
2821 		return 0;
2822 	return (unsigned long) page_address(page);
2823 }
2824 EXPORT_SYMBOL(__get_free_pages);
2825 
2826 unsigned long get_zeroed_page(gfp_t gfp_mask)
2827 {
2828 	return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
2829 }
2830 EXPORT_SYMBOL(get_zeroed_page);
2831 
2832 void __free_pages(struct page *page, unsigned int order)
2833 {
2834 	if (put_page_testzero(page)) {
2835 		if (order == 0)
2836 			free_hot_cold_page(page, false);
2837 		else
2838 			__free_pages_ok(page, order);
2839 	}
2840 }
2841 
2842 EXPORT_SYMBOL(__free_pages);
2843 
2844 void free_pages(unsigned long addr, unsigned int order)
2845 {
2846 	if (addr != 0) {
2847 		VM_BUG_ON(!virt_addr_valid((void *)addr));
2848 		__free_pages(virt_to_page((void *)addr), order);
2849 	}
2850 }
2851 
2852 EXPORT_SYMBOL(free_pages);
2853 
2854 /*
2855  * alloc_kmem_pages charges newly allocated pages to the kmem resource counter
2856  * of the current memory cgroup.
2857  *
2858  * It should be used when the caller would like to use kmalloc, but since the
2859  * allocation is large, it has to fall back to the page allocator.
2860  */
2861 struct page *alloc_kmem_pages(gfp_t gfp_mask, unsigned int order)
2862 {
2863 	struct page *page;
2864 	struct mem_cgroup *memcg = NULL;
2865 
2866 	if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order))
2867 		return NULL;
2868 	page = alloc_pages(gfp_mask, order);
2869 	memcg_kmem_commit_charge(page, memcg, order);
2870 	return page;
2871 }
2872 
2873 struct page *alloc_kmem_pages_node(int nid, gfp_t gfp_mask, unsigned int order)
2874 {
2875 	struct page *page;
2876 	struct mem_cgroup *memcg = NULL;
2877 
2878 	if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order))
2879 		return NULL;
2880 	page = alloc_pages_node(nid, gfp_mask, order);
2881 	memcg_kmem_commit_charge(page, memcg, order);
2882 	return page;
2883 }
2884 
2885 /*
2886  * __free_kmem_pages and free_kmem_pages will free pages allocated with
2887  * alloc_kmem_pages.
2888  */
2889 void __free_kmem_pages(struct page *page, unsigned int order)
2890 {
2891 	memcg_kmem_uncharge_pages(page, order);
2892 	__free_pages(page, order);
2893 }
2894 
2895 void free_kmem_pages(unsigned long addr, unsigned int order)
2896 {
2897 	if (addr != 0) {
2898 		VM_BUG_ON(!virt_addr_valid((void *)addr));
2899 		__free_kmem_pages(virt_to_page((void *)addr), order);
2900 	}
2901 }
2902 
2903 static void *make_alloc_exact(unsigned long addr, unsigned order, size_t size)
2904 {
2905 	if (addr) {
2906 		unsigned long alloc_end = addr + (PAGE_SIZE << order);
2907 		unsigned long used = addr + PAGE_ALIGN(size);
2908 
2909 		split_page(virt_to_page((void *)addr), order);
2910 		while (used < alloc_end) {
2911 			free_page(used);
2912 			used += PAGE_SIZE;
2913 		}
2914 	}
2915 	return (void *)addr;
2916 }
2917 
2918 /**
2919  * alloc_pages_exact - allocate an exact number physically-contiguous pages.
2920  * @size: the number of bytes to allocate
2921  * @gfp_mask: GFP flags for the allocation
2922  *
2923  * This function is similar to alloc_pages(), except that it allocates the
2924  * minimum number of pages to satisfy the request.  alloc_pages() can only
2925  * allocate memory in power-of-two pages.
2926  *
2927  * This function is also limited by MAX_ORDER.
2928  *
2929  * Memory allocated by this function must be released by free_pages_exact().
2930  */
2931 void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
2932 {
2933 	unsigned int order = get_order(size);
2934 	unsigned long addr;
2935 
2936 	addr = __get_free_pages(gfp_mask, order);
2937 	return make_alloc_exact(addr, order, size);
2938 }
2939 EXPORT_SYMBOL(alloc_pages_exact);
2940 
2941 /**
2942  * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
2943  *			   pages on a node.
2944  * @nid: the preferred node ID where memory should be allocated
2945  * @size: the number of bytes to allocate
2946  * @gfp_mask: GFP flags for the allocation
2947  *
2948  * Like alloc_pages_exact(), but try to allocate on node nid first before falling
2949  * back.
2950  * Note this is not alloc_pages_exact_node() which allocates on a specific node,
2951  * but is not exact.
2952  */
2953 void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
2954 {
2955 	unsigned order = get_order(size);
2956 	struct page *p = alloc_pages_node(nid, gfp_mask, order);
2957 	if (!p)
2958 		return NULL;
2959 	return make_alloc_exact((unsigned long)page_address(p), order, size);
2960 }
2961 EXPORT_SYMBOL(alloc_pages_exact_nid);
2962 
2963 /**
2964  * free_pages_exact - release memory allocated via alloc_pages_exact()
2965  * @virt: the value returned by alloc_pages_exact.
2966  * @size: size of allocation, same value as passed to alloc_pages_exact().
2967  *
2968  * Release the memory allocated by a previous call to alloc_pages_exact.
2969  */
2970 void free_pages_exact(void *virt, size_t size)
2971 {
2972 	unsigned long addr = (unsigned long)virt;
2973 	unsigned long end = addr + PAGE_ALIGN(size);
2974 
2975 	while (addr < end) {
2976 		free_page(addr);
2977 		addr += PAGE_SIZE;
2978 	}
2979 }
2980 EXPORT_SYMBOL(free_pages_exact);
2981 
2982 /**
2983  * nr_free_zone_pages - count number of pages beyond high watermark
2984  * @offset: The zone index of the highest zone
2985  *
2986  * nr_free_zone_pages() counts the number of counts pages which are beyond the
2987  * high watermark within all zones at or below a given zone index.  For each
2988  * zone, the number of pages is calculated as:
2989  *     managed_pages - high_pages
2990  */
2991 static unsigned long nr_free_zone_pages(int offset)
2992 {
2993 	struct zoneref *z;
2994 	struct zone *zone;
2995 
2996 	/* Just pick one node, since fallback list is circular */
2997 	unsigned long sum = 0;
2998 
2999 	struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
3000 
3001 	for_each_zone_zonelist(zone, z, zonelist, offset) {
3002 		unsigned long size = zone->managed_pages;
3003 		unsigned long high = high_wmark_pages(zone);
3004 		if (size > high)
3005 			sum += size - high;
3006 	}
3007 
3008 	return sum;
3009 }
3010 
3011 /**
3012  * nr_free_buffer_pages - count number of pages beyond high watermark
3013  *
3014  * nr_free_buffer_pages() counts the number of pages which are beyond the high
3015  * watermark within ZONE_DMA and ZONE_NORMAL.
3016  */
3017 unsigned long nr_free_buffer_pages(void)
3018 {
3019 	return nr_free_zone_pages(gfp_zone(GFP_USER));
3020 }
3021 EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
3022 
3023 /**
3024  * nr_free_pagecache_pages - count number of pages beyond high watermark
3025  *
3026  * nr_free_pagecache_pages() counts the number of pages which are beyond the
3027  * high watermark within all zones.
3028  */
3029 unsigned long nr_free_pagecache_pages(void)
3030 {
3031 	return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
3032 }
3033 
3034 static inline void show_node(struct zone *zone)
3035 {
3036 	if (IS_ENABLED(CONFIG_NUMA))
3037 		printk("Node %d ", zone_to_nid(zone));
3038 }
3039 
3040 void si_meminfo(struct sysinfo *val)
3041 {
3042 	val->totalram = totalram_pages;
3043 	val->sharedram = 0;
3044 	val->freeram = global_page_state(NR_FREE_PAGES);
3045 	val->bufferram = nr_blockdev_pages();
3046 	val->totalhigh = totalhigh_pages;
3047 	val->freehigh = nr_free_highpages();
3048 	val->mem_unit = PAGE_SIZE;
3049 }
3050 
3051 EXPORT_SYMBOL(si_meminfo);
3052 
3053 #ifdef CONFIG_NUMA
3054 void si_meminfo_node(struct sysinfo *val, int nid)
3055 {
3056 	int zone_type;		/* needs to be signed */
3057 	unsigned long managed_pages = 0;
3058 	pg_data_t *pgdat = NODE_DATA(nid);
3059 
3060 	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
3061 		managed_pages += pgdat->node_zones[zone_type].managed_pages;
3062 	val->totalram = managed_pages;
3063 	val->freeram = node_page_state(nid, NR_FREE_PAGES);
3064 #ifdef CONFIG_HIGHMEM
3065 	val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].managed_pages;
3066 	val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
3067 			NR_FREE_PAGES);
3068 #else
3069 	val->totalhigh = 0;
3070 	val->freehigh = 0;
3071 #endif
3072 	val->mem_unit = PAGE_SIZE;
3073 }
3074 #endif
3075 
3076 /*
3077  * Determine whether the node should be displayed or not, depending on whether
3078  * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
3079  */
3080 bool skip_free_areas_node(unsigned int flags, int nid)
3081 {
3082 	bool ret = false;
3083 	unsigned int cpuset_mems_cookie;
3084 
3085 	if (!(flags & SHOW_MEM_FILTER_NODES))
3086 		goto out;
3087 
3088 	do {
3089 		cpuset_mems_cookie = read_mems_allowed_begin();
3090 		ret = !node_isset(nid, cpuset_current_mems_allowed);
3091 	} while (read_mems_allowed_retry(cpuset_mems_cookie));
3092 out:
3093 	return ret;
3094 }
3095 
3096 #define K(x) ((x) << (PAGE_SHIFT-10))
3097 
3098 static void show_migration_types(unsigned char type)
3099 {
3100 	static const char types[MIGRATE_TYPES] = {
3101 		[MIGRATE_UNMOVABLE]	= 'U',
3102 		[MIGRATE_RECLAIMABLE]	= 'E',
3103 		[MIGRATE_MOVABLE]	= 'M',
3104 		[MIGRATE_RESERVE]	= 'R',
3105 #ifdef CONFIG_CMA
3106 		[MIGRATE_CMA]		= 'C',
3107 #endif
3108 #ifdef CONFIG_MEMORY_ISOLATION
3109 		[MIGRATE_ISOLATE]	= 'I',
3110 #endif
3111 	};
3112 	char tmp[MIGRATE_TYPES + 1];
3113 	char *p = tmp;
3114 	int i;
3115 
3116 	for (i = 0; i < MIGRATE_TYPES; i++) {
3117 		if (type & (1 << i))
3118 			*p++ = types[i];
3119 	}
3120 
3121 	*p = '\0';
3122 	printk("(%s) ", tmp);
3123 }
3124 
3125 /*
3126  * Show free area list (used inside shift_scroll-lock stuff)
3127  * We also calculate the percentage fragmentation. We do this by counting the
3128  * memory on each free list with the exception of the first item on the list.
3129  * Suppresses nodes that are not allowed by current's cpuset if
3130  * SHOW_MEM_FILTER_NODES is passed.
3131  */
3132 void show_free_areas(unsigned int filter)
3133 {
3134 	int cpu;
3135 	struct zone *zone;
3136 
3137 	for_each_populated_zone(zone) {
3138 		if (skip_free_areas_node(filter, zone_to_nid(zone)))
3139 			continue;
3140 		show_node(zone);
3141 		printk("%s per-cpu:\n", zone->name);
3142 
3143 		for_each_online_cpu(cpu) {
3144 			struct per_cpu_pageset *pageset;
3145 
3146 			pageset = per_cpu_ptr(zone->pageset, cpu);
3147 
3148 			printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",
3149 			       cpu, pageset->pcp.high,
3150 			       pageset->pcp.batch, pageset->pcp.count);
3151 		}
3152 	}
3153 
3154 	printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
3155 		" active_file:%lu inactive_file:%lu isolated_file:%lu\n"
3156 		" unevictable:%lu"
3157 		" dirty:%lu writeback:%lu unstable:%lu\n"
3158 		" free:%lu slab_reclaimable:%lu slab_unreclaimable:%lu\n"
3159 		" mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
3160 		" free_cma:%lu\n",
3161 		global_page_state(NR_ACTIVE_ANON),
3162 		global_page_state(NR_INACTIVE_ANON),
3163 		global_page_state(NR_ISOLATED_ANON),
3164 		global_page_state(NR_ACTIVE_FILE),
3165 		global_page_state(NR_INACTIVE_FILE),
3166 		global_page_state(NR_ISOLATED_FILE),
3167 		global_page_state(NR_UNEVICTABLE),
3168 		global_page_state(NR_FILE_DIRTY),
3169 		global_page_state(NR_WRITEBACK),
3170 		global_page_state(NR_UNSTABLE_NFS),
3171 		global_page_state(NR_FREE_PAGES),
3172 		global_page_state(NR_SLAB_RECLAIMABLE),
3173 		global_page_state(NR_SLAB_UNRECLAIMABLE),
3174 		global_page_state(NR_FILE_MAPPED),
3175 		global_page_state(NR_SHMEM),
3176 		global_page_state(NR_PAGETABLE),
3177 		global_page_state(NR_BOUNCE),
3178 		global_page_state(NR_FREE_CMA_PAGES));
3179 
3180 	for_each_populated_zone(zone) {
3181 		int i;
3182 
3183 		if (skip_free_areas_node(filter, zone_to_nid(zone)))
3184 			continue;
3185 		show_node(zone);
3186 		printk("%s"
3187 			" free:%lukB"
3188 			" min:%lukB"
3189 			" low:%lukB"
3190 			" high:%lukB"
3191 			" active_anon:%lukB"
3192 			" inactive_anon:%lukB"
3193 			" active_file:%lukB"
3194 			" inactive_file:%lukB"
3195 			" unevictable:%lukB"
3196 			" isolated(anon):%lukB"
3197 			" isolated(file):%lukB"
3198 			" present:%lukB"
3199 			" managed:%lukB"
3200 			" mlocked:%lukB"
3201 			" dirty:%lukB"
3202 			" writeback:%lukB"
3203 			" mapped:%lukB"
3204 			" shmem:%lukB"
3205 			" slab_reclaimable:%lukB"
3206 			" slab_unreclaimable:%lukB"
3207 			" kernel_stack:%lukB"
3208 			" pagetables:%lukB"
3209 			" unstable:%lukB"
3210 			" bounce:%lukB"
3211 			" free_cma:%lukB"
3212 			" writeback_tmp:%lukB"
3213 			" pages_scanned:%lu"
3214 			" all_unreclaimable? %s"
3215 			"\n",
3216 			zone->name,
3217 			K(zone_page_state(zone, NR_FREE_PAGES)),
3218 			K(min_wmark_pages(zone)),
3219 			K(low_wmark_pages(zone)),
3220 			K(high_wmark_pages(zone)),
3221 			K(zone_page_state(zone, NR_ACTIVE_ANON)),
3222 			K(zone_page_state(zone, NR_INACTIVE_ANON)),
3223 			K(zone_page_state(zone, NR_ACTIVE_FILE)),
3224 			K(zone_page_state(zone, NR_INACTIVE_FILE)),
3225 			K(zone_page_state(zone, NR_UNEVICTABLE)),
3226 			K(zone_page_state(zone, NR_ISOLATED_ANON)),
3227 			K(zone_page_state(zone, NR_ISOLATED_FILE)),
3228 			K(zone->present_pages),
3229 			K(zone->managed_pages),
3230 			K(zone_page_state(zone, NR_MLOCK)),
3231 			K(zone_page_state(zone, NR_FILE_DIRTY)),
3232 			K(zone_page_state(zone, NR_WRITEBACK)),
3233 			K(zone_page_state(zone, NR_FILE_MAPPED)),
3234 			K(zone_page_state(zone, NR_SHMEM)),
3235 			K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)),
3236 			K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)),
3237 			zone_page_state(zone, NR_KERNEL_STACK) *
3238 				THREAD_SIZE / 1024,
3239 			K(zone_page_state(zone, NR_PAGETABLE)),
3240 			K(zone_page_state(zone, NR_UNSTABLE_NFS)),
3241 			K(zone_page_state(zone, NR_BOUNCE)),
3242 			K(zone_page_state(zone, NR_FREE_CMA_PAGES)),
3243 			K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
3244 			zone->pages_scanned,
3245 			(!zone_reclaimable(zone) ? "yes" : "no")
3246 			);
3247 		printk("lowmem_reserve[]:");
3248 		for (i = 0; i < MAX_NR_ZONES; i++)
3249 			printk(" %lu", zone->lowmem_reserve[i]);
3250 		printk("\n");
3251 	}
3252 
3253 	for_each_populated_zone(zone) {
3254 		unsigned long nr[MAX_ORDER], flags, order, total = 0;
3255 		unsigned char types[MAX_ORDER];
3256 
3257 		if (skip_free_areas_node(filter, zone_to_nid(zone)))
3258 			continue;
3259 		show_node(zone);
3260 		printk("%s: ", zone->name);
3261 
3262 		spin_lock_irqsave(&zone->lock, flags);
3263 		for (order = 0; order < MAX_ORDER; order++) {
3264 			struct free_area *area = &zone->free_area[order];
3265 			int type;
3266 
3267 			nr[order] = area->nr_free;
3268 			total += nr[order] << order;
3269 
3270 			types[order] = 0;
3271 			for (type = 0; type < MIGRATE_TYPES; type++) {
3272 				if (!list_empty(&area->free_list[type]))
3273 					types[order] |= 1 << type;
3274 			}
3275 		}
3276 		spin_unlock_irqrestore(&zone->lock, flags);
3277 		for (order = 0; order < MAX_ORDER; order++) {
3278 			printk("%lu*%lukB ", nr[order], K(1UL) << order);
3279 			if (nr[order])
3280 				show_migration_types(types[order]);
3281 		}
3282 		printk("= %lukB\n", K(total));
3283 	}
3284 
3285 	hugetlb_show_meminfo();
3286 
3287 	printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES));
3288 
3289 	show_swap_cache_info();
3290 }
3291 
3292 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
3293 {
3294 	zoneref->zone = zone;
3295 	zoneref->zone_idx = zone_idx(zone);
3296 }
3297 
3298 /*
3299  * Builds allocation fallback zone lists.
3300  *
3301  * Add all populated zones of a node to the zonelist.
3302  */
3303 static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
3304 				int nr_zones)
3305 {
3306 	struct zone *zone;
3307 	enum zone_type zone_type = MAX_NR_ZONES;
3308 
3309 	do {
3310 		zone_type--;
3311 		zone = pgdat->node_zones + zone_type;
3312 		if (populated_zone(zone)) {
3313 			zoneref_set_zone(zone,
3314 				&zonelist->_zonerefs[nr_zones++]);
3315 			check_highest_zone(zone_type);
3316 		}
3317 	} while (zone_type);
3318 
3319 	return nr_zones;
3320 }
3321 
3322 
3323 /*
3324  *  zonelist_order:
3325  *  0 = automatic detection of better ordering.
3326  *  1 = order by ([node] distance, -zonetype)
3327  *  2 = order by (-zonetype, [node] distance)
3328  *
3329  *  If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create
3330  *  the same zonelist. So only NUMA can configure this param.
3331  */
3332 #define ZONELIST_ORDER_DEFAULT  0
3333 #define ZONELIST_ORDER_NODE     1
3334 #define ZONELIST_ORDER_ZONE     2
3335 
3336 /* zonelist order in the kernel.
3337  * set_zonelist_order() will set this to NODE or ZONE.
3338  */
3339 static int current_zonelist_order = ZONELIST_ORDER_DEFAULT;
3340 static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"};
3341 
3342 
3343 #ifdef CONFIG_NUMA
3344 /* The value user specified ....changed by config */
3345 static int user_zonelist_order = ZONELIST_ORDER_DEFAULT;
3346 /* string for sysctl */
3347 #define NUMA_ZONELIST_ORDER_LEN	16
3348 char numa_zonelist_order[16] = "default";
3349 
3350 /*
3351  * interface for configure zonelist ordering.
3352  * command line option "numa_zonelist_order"
3353  *	= "[dD]efault	- default, automatic configuration.
3354  *	= "[nN]ode 	- order by node locality, then by zone within node
3355  *	= "[zZ]one      - order by zone, then by locality within zone
3356  */
3357 
3358 static int __parse_numa_zonelist_order(char *s)
3359 {
3360 	if (*s == 'd' || *s == 'D') {
3361 		user_zonelist_order = ZONELIST_ORDER_DEFAULT;
3362 	} else if (*s == 'n' || *s == 'N') {
3363 		user_zonelist_order = ZONELIST_ORDER_NODE;
3364 	} else if (*s == 'z' || *s == 'Z') {
3365 		user_zonelist_order = ZONELIST_ORDER_ZONE;
3366 	} else {
3367 		printk(KERN_WARNING
3368 			"Ignoring invalid numa_zonelist_order value:  "
3369 			"%s\n", s);
3370 		return -EINVAL;
3371 	}
3372 	return 0;
3373 }
3374 
3375 static __init int setup_numa_zonelist_order(char *s)
3376 {
3377 	int ret;
3378 
3379 	if (!s)
3380 		return 0;
3381 
3382 	ret = __parse_numa_zonelist_order(s);
3383 	if (ret == 0)
3384 		strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN);
3385 
3386 	return ret;
3387 }
3388 early_param("numa_zonelist_order", setup_numa_zonelist_order);
3389 
3390 /*
3391  * sysctl handler for numa_zonelist_order
3392  */
3393 int numa_zonelist_order_handler(struct ctl_table *table, int write,
3394 		void __user *buffer, size_t *length,
3395 		loff_t *ppos)
3396 {
3397 	char saved_string[NUMA_ZONELIST_ORDER_LEN];
3398 	int ret;
3399 	static DEFINE_MUTEX(zl_order_mutex);
3400 
3401 	mutex_lock(&zl_order_mutex);
3402 	if (write) {
3403 		if (strlen((char *)table->data) >= NUMA_ZONELIST_ORDER_LEN) {
3404 			ret = -EINVAL;
3405 			goto out;
3406 		}
3407 		strcpy(saved_string, (char *)table->data);
3408 	}
3409 	ret = proc_dostring(table, write, buffer, length, ppos);
3410 	if (ret)
3411 		goto out;
3412 	if (write) {
3413 		int oldval = user_zonelist_order;
3414 
3415 		ret = __parse_numa_zonelist_order((char *)table->data);
3416 		if (ret) {
3417 			/*
3418 			 * bogus value.  restore saved string
3419 			 */
3420 			strncpy((char *)table->data, saved_string,
3421 				NUMA_ZONELIST_ORDER_LEN);
3422 			user_zonelist_order = oldval;
3423 		} else if (oldval != user_zonelist_order) {
3424 			mutex_lock(&zonelists_mutex);
3425 			build_all_zonelists(NULL, NULL);
3426 			mutex_unlock(&zonelists_mutex);
3427 		}
3428 	}
3429 out:
3430 	mutex_unlock(&zl_order_mutex);
3431 	return ret;
3432 }
3433 
3434 
3435 #define MAX_NODE_LOAD (nr_online_nodes)
3436 static int node_load[MAX_NUMNODES];
3437 
3438 /**
3439  * find_next_best_node - find the next node that should appear in a given node's fallback list
3440  * @node: node whose fallback list we're appending
3441  * @used_node_mask: nodemask_t of already used nodes
3442  *
3443  * We use a number of factors to determine which is the next node that should
3444  * appear on a given node's fallback list.  The node should not have appeared
3445  * already in @node's fallback list, and it should be the next closest node
3446  * according to the distance array (which contains arbitrary distance values
3447  * from each node to each node in the system), and should also prefer nodes
3448  * with no CPUs, since presumably they'll have very little allocation pressure
3449  * on them otherwise.
3450  * It returns -1 if no node is found.
3451  */
3452 static int find_next_best_node(int node, nodemask_t *used_node_mask)
3453 {
3454 	int n, val;
3455 	int min_val = INT_MAX;
3456 	int best_node = NUMA_NO_NODE;
3457 	const struct cpumask *tmp = cpumask_of_node(0);
3458 
3459 	/* Use the local node if we haven't already */
3460 	if (!node_isset(node, *used_node_mask)) {
3461 		node_set(node, *used_node_mask);
3462 		return node;
3463 	}
3464 
3465 	for_each_node_state(n, N_MEMORY) {
3466 
3467 		/* Don't want a node to appear more than once */
3468 		if (node_isset(n, *used_node_mask))
3469 			continue;
3470 
3471 		/* Use the distance array to find the distance */
3472 		val = node_distance(node, n);
3473 
3474 		/* Penalize nodes under us ("prefer the next node") */
3475 		val += (n < node);
3476 
3477 		/* Give preference to headless and unused nodes */
3478 		tmp = cpumask_of_node(n);
3479 		if (!cpumask_empty(tmp))
3480 			val += PENALTY_FOR_NODE_WITH_CPUS;
3481 
3482 		/* Slight preference for less loaded node */
3483 		val *= (MAX_NODE_LOAD*MAX_NUMNODES);
3484 		val += node_load[n];
3485 
3486 		if (val < min_val) {
3487 			min_val = val;
3488 			best_node = n;
3489 		}
3490 	}
3491 
3492 	if (best_node >= 0)
3493 		node_set(best_node, *used_node_mask);
3494 
3495 	return best_node;
3496 }
3497 
3498 
3499 /*
3500  * Build zonelists ordered by node and zones within node.
3501  * This results in maximum locality--normal zone overflows into local
3502  * DMA zone, if any--but risks exhausting DMA zone.
3503  */
3504 static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
3505 {
3506 	int j;
3507 	struct zonelist *zonelist;
3508 
3509 	zonelist = &pgdat->node_zonelists[0];
3510 	for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++)
3511 		;
3512 	j = build_zonelists_node(NODE_DATA(node), zonelist, j);
3513 	zonelist->_zonerefs[j].zone = NULL;
3514 	zonelist->_zonerefs[j].zone_idx = 0;
3515 }
3516 
3517 /*
3518  * Build gfp_thisnode zonelists
3519  */
3520 static void build_thisnode_zonelists(pg_data_t *pgdat)
3521 {
3522 	int j;
3523 	struct zonelist *zonelist;
3524 
3525 	zonelist = &pgdat->node_zonelists[1];
3526 	j = build_zonelists_node(pgdat, zonelist, 0);
3527 	zonelist->_zonerefs[j].zone = NULL;
3528 	zonelist->_zonerefs[j].zone_idx = 0;
3529 }
3530 
3531 /*
3532  * Build zonelists ordered by zone and nodes within zones.
3533  * This results in conserving DMA zone[s] until all Normal memory is
3534  * exhausted, but results in overflowing to remote node while memory
3535  * may still exist in local DMA zone.
3536  */
3537 static int node_order[MAX_NUMNODES];
3538 
3539 static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes)
3540 {
3541 	int pos, j, node;
3542 	int zone_type;		/* needs to be signed */
3543 	struct zone *z;
3544 	struct zonelist *zonelist;
3545 
3546 	zonelist = &pgdat->node_zonelists[0];
3547 	pos = 0;
3548 	for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) {
3549 		for (j = 0; j < nr_nodes; j++) {
3550 			node = node_order[j];
3551 			z = &NODE_DATA(node)->node_zones[zone_type];
3552 			if (populated_zone(z)) {
3553 				zoneref_set_zone(z,
3554 					&zonelist->_zonerefs[pos++]);
3555 				check_highest_zone(zone_type);
3556 			}
3557 		}
3558 	}
3559 	zonelist->_zonerefs[pos].zone = NULL;
3560 	zonelist->_zonerefs[pos].zone_idx = 0;
3561 }
3562 
3563 static int default_zonelist_order(void)
3564 {
3565 	int nid, zone_type;
3566 	unsigned long low_kmem_size, total_size;
3567 	struct zone *z;
3568 	int average_size;
3569 	/*
3570 	 * ZONE_DMA and ZONE_DMA32 can be very small area in the system.
3571 	 * If they are really small and used heavily, the system can fall
3572 	 * into OOM very easily.
3573 	 * This function detect ZONE_DMA/DMA32 size and configures zone order.
3574 	 */
3575 	/* Is there ZONE_NORMAL ? (ex. ppc has only DMA zone..) */
3576 	low_kmem_size = 0;
3577 	total_size = 0;
3578 	for_each_online_node(nid) {
3579 		for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
3580 			z = &NODE_DATA(nid)->node_zones[zone_type];
3581 			if (populated_zone(z)) {
3582 				if (zone_type < ZONE_NORMAL)
3583 					low_kmem_size += z->managed_pages;
3584 				total_size += z->managed_pages;
3585 			} else if (zone_type == ZONE_NORMAL) {
3586 				/*
3587 				 * If any node has only lowmem, then node order
3588 				 * is preferred to allow kernel allocations
3589 				 * locally; otherwise, they can easily infringe
3590 				 * on other nodes when there is an abundance of
3591 				 * lowmem available to allocate from.
3592 				 */
3593 				return ZONELIST_ORDER_NODE;
3594 			}
3595 		}
3596 	}
3597 	if (!low_kmem_size ||  /* there are no DMA area. */
3598 	    low_kmem_size > total_size/2) /* DMA/DMA32 is big. */
3599 		return ZONELIST_ORDER_NODE;
3600 	/*
3601 	 * look into each node's config.
3602 	 * If there is a node whose DMA/DMA32 memory is very big area on
3603 	 * local memory, NODE_ORDER may be suitable.
3604 	 */
3605 	average_size = total_size /
3606 				(nodes_weight(node_states[N_MEMORY]) + 1);
3607 	for_each_online_node(nid) {
3608 		low_kmem_size = 0;
3609 		total_size = 0;
3610 		for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
3611 			z = &NODE_DATA(nid)->node_zones[zone_type];
3612 			if (populated_zone(z)) {
3613 				if (zone_type < ZONE_NORMAL)
3614 					low_kmem_size += z->present_pages;
3615 				total_size += z->present_pages;
3616 			}
3617 		}
3618 		if (low_kmem_size &&
3619 		    total_size > average_size && /* ignore small node */
3620 		    low_kmem_size > total_size * 70/100)
3621 			return ZONELIST_ORDER_NODE;
3622 	}
3623 	return ZONELIST_ORDER_ZONE;
3624 }
3625 
3626 static void set_zonelist_order(void)
3627 {
3628 	if (user_zonelist_order == ZONELIST_ORDER_DEFAULT)
3629 		current_zonelist_order = default_zonelist_order();
3630 	else
3631 		current_zonelist_order = user_zonelist_order;
3632 }
3633 
3634 static void build_zonelists(pg_data_t *pgdat)
3635 {
3636 	int j, node, load;
3637 	enum zone_type i;
3638 	nodemask_t used_mask;
3639 	int local_node, prev_node;
3640 	struct zonelist *zonelist;
3641 	int order = current_zonelist_order;
3642 
3643 	/* initialize zonelists */
3644 	for (i = 0; i < MAX_ZONELISTS; i++) {
3645 		zonelist = pgdat->node_zonelists + i;
3646 		zonelist->_zonerefs[0].zone = NULL;
3647 		zonelist->_zonerefs[0].zone_idx = 0;
3648 	}
3649 
3650 	/* NUMA-aware ordering of nodes */
3651 	local_node = pgdat->node_id;
3652 	load = nr_online_nodes;
3653 	prev_node = local_node;
3654 	nodes_clear(used_mask);
3655 
3656 	memset(node_order, 0, sizeof(node_order));
3657 	j = 0;
3658 
3659 	while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
3660 		/*
3661 		 * We don't want to pressure a particular node.
3662 		 * So adding penalty to the first node in same
3663 		 * distance group to make it round-robin.
3664 		 */
3665 		if (node_distance(local_node, node) !=
3666 		    node_distance(local_node, prev_node))
3667 			node_load[node] = load;
3668 
3669 		prev_node = node;
3670 		load--;
3671 		if (order == ZONELIST_ORDER_NODE)
3672 			build_zonelists_in_node_order(pgdat, node);
3673 		else
3674 			node_order[j++] = node;	/* remember order */
3675 	}
3676 
3677 	if (order == ZONELIST_ORDER_ZONE) {
3678 		/* calculate node order -- i.e., DMA last! */
3679 		build_zonelists_in_zone_order(pgdat, j);
3680 	}
3681 
3682 	build_thisnode_zonelists(pgdat);
3683 }
3684 
3685 /* Construct the zonelist performance cache - see further mmzone.h */
3686 static void build_zonelist_cache(pg_data_t *pgdat)
3687 {
3688 	struct zonelist *zonelist;
3689 	struct zonelist_cache *zlc;
3690 	struct zoneref *z;
3691 
3692 	zonelist = &pgdat->node_zonelists[0];
3693 	zonelist->zlcache_ptr = zlc = &zonelist->zlcache;
3694 	bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
3695 	for (z = zonelist->_zonerefs; z->zone; z++)
3696 		zlc->z_to_n[z - zonelist->_zonerefs] = zonelist_node_idx(z);
3697 }
3698 
3699 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
3700 /*
3701  * Return node id of node used for "local" allocations.
3702  * I.e., first node id of first zone in arg node's generic zonelist.
3703  * Used for initializing percpu 'numa_mem', which is used primarily
3704  * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
3705  */
3706 int local_memory_node(int node)
3707 {
3708 	struct zone *zone;
3709 
3710 	(void)first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
3711 				   gfp_zone(GFP_KERNEL),
3712 				   NULL,
3713 				   &zone);
3714 	return zone->node;
3715 }
3716 #endif
3717 
3718 #else	/* CONFIG_NUMA */
3719 
3720 static void set_zonelist_order(void)
3721 {
3722 	current_zonelist_order = ZONELIST_ORDER_ZONE;
3723 }
3724 
3725 static void build_zonelists(pg_data_t *pgdat)
3726 {
3727 	int node, local_node;
3728 	enum zone_type j;
3729 	struct zonelist *zonelist;
3730 
3731 	local_node = pgdat->node_id;
3732 
3733 	zonelist = &pgdat->node_zonelists[0];
3734 	j = build_zonelists_node(pgdat, zonelist, 0);
3735 
3736 	/*
3737 	 * Now we build the zonelist so that it contains the zones
3738 	 * of all the other nodes.
3739 	 * We don't want to pressure a particular node, so when
3740 	 * building the zones for node N, we make sure that the
3741 	 * zones coming right after the local ones are those from
3742 	 * node N+1 (modulo N)
3743 	 */
3744 	for (node = local_node + 1; node < MAX_NUMNODES; node++) {
3745 		if (!node_online(node))
3746 			continue;
3747 		j = build_zonelists_node(NODE_DATA(node), zonelist, j);
3748 	}
3749 	for (node = 0; node < local_node; node++) {
3750 		if (!node_online(node))
3751 			continue;
3752 		j = build_zonelists_node(NODE_DATA(node), zonelist, j);
3753 	}
3754 
3755 	zonelist->_zonerefs[j].zone = NULL;
3756 	zonelist->_zonerefs[j].zone_idx = 0;
3757 }
3758 
3759 /* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */
3760 static void build_zonelist_cache(pg_data_t *pgdat)
3761 {
3762 	pgdat->node_zonelists[0].zlcache_ptr = NULL;
3763 }
3764 
3765 #endif	/* CONFIG_NUMA */
3766 
3767 /*
3768  * Boot pageset table. One per cpu which is going to be used for all
3769  * zones and all nodes. The parameters will be set in such a way
3770  * that an item put on a list will immediately be handed over to
3771  * the buddy list. This is safe since pageset manipulation is done
3772  * with interrupts disabled.
3773  *
3774  * The boot_pagesets must be kept even after bootup is complete for
3775  * unused processors and/or zones. They do play a role for bootstrapping
3776  * hotplugged processors.
3777  *
3778  * zoneinfo_show() and maybe other functions do
3779  * not check if the processor is online before following the pageset pointer.
3780  * Other parts of the kernel may not check if the zone is available.
3781  */
3782 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
3783 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
3784 static void setup_zone_pageset(struct zone *zone);
3785 
3786 /*
3787  * Global mutex to protect against size modification of zonelists
3788  * as well as to serialize pageset setup for the new populated zone.
3789  */
3790 DEFINE_MUTEX(zonelists_mutex);
3791 
3792 /* return values int ....just for stop_machine() */
3793 static int __build_all_zonelists(void *data)
3794 {
3795 	int nid;
3796 	int cpu;
3797 	pg_data_t *self = data;
3798 
3799 #ifdef CONFIG_NUMA
3800 	memset(node_load, 0, sizeof(node_load));
3801 #endif
3802 
3803 	if (self && !node_online(self->node_id)) {
3804 		build_zonelists(self);
3805 		build_zonelist_cache(self);
3806 	}
3807 
3808 	for_each_online_node(nid) {
3809 		pg_data_t *pgdat = NODE_DATA(nid);
3810 
3811 		build_zonelists(pgdat);
3812 		build_zonelist_cache(pgdat);
3813 	}
3814 
3815 	/*
3816 	 * Initialize the boot_pagesets that are going to be used
3817 	 * for bootstrapping processors. The real pagesets for
3818 	 * each zone will be allocated later when the per cpu
3819 	 * allocator is available.
3820 	 *
3821 	 * boot_pagesets are used also for bootstrapping offline
3822 	 * cpus if the system is already booted because the pagesets
3823 	 * are needed to initialize allocators on a specific cpu too.
3824 	 * F.e. the percpu allocator needs the page allocator which
3825 	 * needs the percpu allocator in order to allocate its pagesets
3826 	 * (a chicken-egg dilemma).
3827 	 */
3828 	for_each_possible_cpu(cpu) {
3829 		setup_pageset(&per_cpu(boot_pageset, cpu), 0);
3830 
3831 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
3832 		/*
3833 		 * We now know the "local memory node" for each node--
3834 		 * i.e., the node of the first zone in the generic zonelist.
3835 		 * Set up numa_mem percpu variable for on-line cpus.  During
3836 		 * boot, only the boot cpu should be on-line;  we'll init the
3837 		 * secondary cpus' numa_mem as they come on-line.  During
3838 		 * node/memory hotplug, we'll fixup all on-line cpus.
3839 		 */
3840 		if (cpu_online(cpu))
3841 			set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
3842 #endif
3843 	}
3844 
3845 	return 0;
3846 }
3847 
3848 /*
3849  * Called with zonelists_mutex held always
3850  * unless system_state == SYSTEM_BOOTING.
3851  */
3852 void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone)
3853 {
3854 	set_zonelist_order();
3855 
3856 	if (system_state == SYSTEM_BOOTING) {
3857 		__build_all_zonelists(NULL);
3858 		mminit_verify_zonelist();
3859 		cpuset_init_current_mems_allowed();
3860 	} else {
3861 #ifdef CONFIG_MEMORY_HOTPLUG
3862 		if (zone)
3863 			setup_zone_pageset(zone);
3864 #endif
3865 		/* we have to stop all cpus to guarantee there is no user
3866 		   of zonelist */
3867 		stop_machine(__build_all_zonelists, pgdat, NULL);
3868 		/* cpuset refresh routine should be here */
3869 	}
3870 	vm_total_pages = nr_free_pagecache_pages();
3871 	/*
3872 	 * Disable grouping by mobility if the number of pages in the
3873 	 * system is too low to allow the mechanism to work. It would be
3874 	 * more accurate, but expensive to check per-zone. This check is
3875 	 * made on memory-hotadd so a system can start with mobility
3876 	 * disabled and enable it later
3877 	 */
3878 	if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
3879 		page_group_by_mobility_disabled = 1;
3880 	else
3881 		page_group_by_mobility_disabled = 0;
3882 
3883 	printk("Built %i zonelists in %s order, mobility grouping %s.  "
3884 		"Total pages: %ld\n",
3885 			nr_online_nodes,
3886 			zonelist_order_name[current_zonelist_order],
3887 			page_group_by_mobility_disabled ? "off" : "on",
3888 			vm_total_pages);
3889 #ifdef CONFIG_NUMA
3890 	printk("Policy zone: %s\n", zone_names[policy_zone]);
3891 #endif
3892 }
3893 
3894 /*
3895  * Helper functions to size the waitqueue hash table.
3896  * Essentially these want to choose hash table sizes sufficiently
3897  * large so that collisions trying to wait on pages are rare.
3898  * But in fact, the number of active page waitqueues on typical
3899  * systems is ridiculously low, less than 200. So this is even
3900  * conservative, even though it seems large.
3901  *
3902  * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to
3903  * waitqueues, i.e. the size of the waitq table given the number of pages.
3904  */
3905 #define PAGES_PER_WAITQUEUE	256
3906 
3907 #ifndef CONFIG_MEMORY_HOTPLUG
3908 static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
3909 {
3910 	unsigned long size = 1;
3911 
3912 	pages /= PAGES_PER_WAITQUEUE;
3913 
3914 	while (size < pages)
3915 		size <<= 1;
3916 
3917 	/*
3918 	 * Once we have dozens or even hundreds of threads sleeping
3919 	 * on IO we've got bigger problems than wait queue collision.
3920 	 * Limit the size of the wait table to a reasonable size.
3921 	 */
3922 	size = min(size, 4096UL);
3923 
3924 	return max(size, 4UL);
3925 }
3926 #else
3927 /*
3928  * A zone's size might be changed by hot-add, so it is not possible to determine
3929  * a suitable size for its wait_table.  So we use the maximum size now.
3930  *
3931  * The max wait table size = 4096 x sizeof(wait_queue_head_t).   ie:
3932  *
3933  *    i386 (preemption config)    : 4096 x 16 = 64Kbyte.
3934  *    ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte.
3935  *    ia64, x86-64 (preemption)   : 4096 x 24 = 96Kbyte.
3936  *
3937  * The maximum entries are prepared when a zone's memory is (512K + 256) pages
3938  * or more by the traditional way. (See above).  It equals:
3939  *
3940  *    i386, x86-64, powerpc(4K page size) : =  ( 2G + 1M)byte.
3941  *    ia64(16K page size)                 : =  ( 8G + 4M)byte.
3942  *    powerpc (64K page size)             : =  (32G +16M)byte.
3943  */
3944 static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
3945 {
3946 	return 4096UL;
3947 }
3948 #endif
3949 
3950 /*
3951  * This is an integer logarithm so that shifts can be used later
3952  * to extract the more random high bits from the multiplicative
3953  * hash function before the remainder is taken.
3954  */
3955 static inline unsigned long wait_table_bits(unsigned long size)
3956 {
3957 	return ffz(~size);
3958 }
3959 
3960 /*
3961  * Check if a pageblock contains reserved pages
3962  */
3963 static int pageblock_is_reserved(unsigned long start_pfn, unsigned long end_pfn)
3964 {
3965 	unsigned long pfn;
3966 
3967 	for (pfn = start_pfn; pfn < end_pfn; pfn++) {
3968 		if (!pfn_valid_within(pfn) || PageReserved(pfn_to_page(pfn)))
3969 			return 1;
3970 	}
3971 	return 0;
3972 }
3973 
3974 /*
3975  * Mark a number of pageblocks as MIGRATE_RESERVE. The number
3976  * of blocks reserved is based on min_wmark_pages(zone). The memory within
3977  * the reserve will tend to store contiguous free pages. Setting min_free_kbytes
3978  * higher will lead to a bigger reserve which will get freed as contiguous
3979  * blocks as reclaim kicks in
3980  */
3981 static void setup_zone_migrate_reserve(struct zone *zone)
3982 {
3983 	unsigned long start_pfn, pfn, end_pfn, block_end_pfn;
3984 	struct page *page;
3985 	unsigned long block_migratetype;
3986 	int reserve;
3987 	int old_reserve;
3988 
3989 	/*
3990 	 * Get the start pfn, end pfn and the number of blocks to reserve
3991 	 * We have to be careful to be aligned to pageblock_nr_pages to
3992 	 * make sure that we always check pfn_valid for the first page in
3993 	 * the block.
3994 	 */
3995 	start_pfn = zone->zone_start_pfn;
3996 	end_pfn = zone_end_pfn(zone);
3997 	start_pfn = roundup(start_pfn, pageblock_nr_pages);
3998 	reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >>
3999 							pageblock_order;
4000 
4001 	/*
4002 	 * Reserve blocks are generally in place to help high-order atomic
4003 	 * allocations that are short-lived. A min_free_kbytes value that
4004 	 * would result in more than 2 reserve blocks for atomic allocations
4005 	 * is assumed to be in place to help anti-fragmentation for the
4006 	 * future allocation of hugepages at runtime.
4007 	 */
4008 	reserve = min(2, reserve);
4009 	old_reserve = zone->nr_migrate_reserve_block;
4010 
4011 	/* When memory hot-add, we almost always need to do nothing */
4012 	if (reserve == old_reserve)
4013 		return;
4014 	zone->nr_migrate_reserve_block = reserve;
4015 
4016 	for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
4017 		if (!pfn_valid(pfn))
4018 			continue;
4019 		page = pfn_to_page(pfn);
4020 
4021 		/* Watch out for overlapping nodes */
4022 		if (page_to_nid(page) != zone_to_nid(zone))
4023 			continue;
4024 
4025 		block_migratetype = get_pageblock_migratetype(page);
4026 
4027 		/* Only test what is necessary when the reserves are not met */
4028 		if (reserve > 0) {
4029 			/*
4030 			 * Blocks with reserved pages will never free, skip
4031 			 * them.
4032 			 */
4033 			block_end_pfn = min(pfn + pageblock_nr_pages, end_pfn);
4034 			if (pageblock_is_reserved(pfn, block_end_pfn))
4035 				continue;
4036 
4037 			/* If this block is reserved, account for it */
4038 			if (block_migratetype == MIGRATE_RESERVE) {
4039 				reserve--;
4040 				continue;
4041 			}
4042 
4043 			/* Suitable for reserving if this block is movable */
4044 			if (block_migratetype == MIGRATE_MOVABLE) {
4045 				set_pageblock_migratetype(page,
4046 							MIGRATE_RESERVE);
4047 				move_freepages_block(zone, page,
4048 							MIGRATE_RESERVE);
4049 				reserve--;
4050 				continue;
4051 			}
4052 		} else if (!old_reserve) {
4053 			/*
4054 			 * At boot time we don't need to scan the whole zone
4055 			 * for turning off MIGRATE_RESERVE.
4056 			 */
4057 			break;
4058 		}
4059 
4060 		/*
4061 		 * If the reserve is met and this is a previous reserved block,
4062 		 * take it back
4063 		 */
4064 		if (block_migratetype == MIGRATE_RESERVE) {
4065 			set_pageblock_migratetype(page, MIGRATE_MOVABLE);
4066 			move_freepages_block(zone, page, MIGRATE_MOVABLE);
4067 		}
4068 	}
4069 }
4070 
4071 /*
4072  * Initially all pages are reserved - free ones are freed
4073  * up by free_all_bootmem() once the early boot process is
4074  * done. Non-atomic initialization, single-pass.
4075  */
4076 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
4077 		unsigned long start_pfn, enum memmap_context context)
4078 {
4079 	struct page *page;
4080 	unsigned long end_pfn = start_pfn + size;
4081 	unsigned long pfn;
4082 	struct zone *z;
4083 
4084 	if (highest_memmap_pfn < end_pfn - 1)
4085 		highest_memmap_pfn = end_pfn - 1;
4086 
4087 	z = &NODE_DATA(nid)->node_zones[zone];
4088 	for (pfn = start_pfn; pfn < end_pfn; pfn++) {
4089 		/*
4090 		 * There can be holes in boot-time mem_map[]s
4091 		 * handed to this function.  They do not
4092 		 * exist on hotplugged memory.
4093 		 */
4094 		if (context == MEMMAP_EARLY) {
4095 			if (!early_pfn_valid(pfn))
4096 				continue;
4097 			if (!early_pfn_in_nid(pfn, nid))
4098 				continue;
4099 		}
4100 		page = pfn_to_page(pfn);
4101 		set_page_links(page, zone, nid, pfn);
4102 		mminit_verify_page_links(page, zone, nid, pfn);
4103 		init_page_count(page);
4104 		page_mapcount_reset(page);
4105 		page_cpupid_reset_last(page);
4106 		SetPageReserved(page);
4107 		/*
4108 		 * Mark the block movable so that blocks are reserved for
4109 		 * movable at startup. This will force kernel allocations
4110 		 * to reserve their blocks rather than leaking throughout
4111 		 * the address space during boot when many long-lived
4112 		 * kernel allocations are made. Later some blocks near
4113 		 * the start are marked MIGRATE_RESERVE by
4114 		 * setup_zone_migrate_reserve()
4115 		 *
4116 		 * bitmap is created for zone's valid pfn range. but memmap
4117 		 * can be created for invalid pages (for alignment)
4118 		 * check here not to call set_pageblock_migratetype() against
4119 		 * pfn out of zone.
4120 		 */
4121 		if ((z->zone_start_pfn <= pfn)
4122 		    && (pfn < zone_end_pfn(z))
4123 		    && !(pfn & (pageblock_nr_pages - 1)))
4124 			set_pageblock_migratetype(page, MIGRATE_MOVABLE);
4125 
4126 		INIT_LIST_HEAD(&page->lru);
4127 #ifdef WANT_PAGE_VIRTUAL
4128 		/* The shift won't overflow because ZONE_NORMAL is below 4G. */
4129 		if (!is_highmem_idx(zone))
4130 			set_page_address(page, __va(pfn << PAGE_SHIFT));
4131 #endif
4132 	}
4133 }
4134 
4135 static void __meminit zone_init_free_lists(struct zone *zone)
4136 {
4137 	unsigned int order, t;
4138 	for_each_migratetype_order(order, t) {
4139 		INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
4140 		zone->free_area[order].nr_free = 0;
4141 	}
4142 }
4143 
4144 #ifndef __HAVE_ARCH_MEMMAP_INIT
4145 #define memmap_init(size, nid, zone, start_pfn) \
4146 	memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
4147 #endif
4148 
4149 static int zone_batchsize(struct zone *zone)
4150 {
4151 #ifdef CONFIG_MMU
4152 	int batch;
4153 
4154 	/*
4155 	 * The per-cpu-pages pools are set to around 1000th of the
4156 	 * size of the zone.  But no more than 1/2 of a meg.
4157 	 *
4158 	 * OK, so we don't know how big the cache is.  So guess.
4159 	 */
4160 	batch = zone->managed_pages / 1024;
4161 	if (batch * PAGE_SIZE > 512 * 1024)
4162 		batch = (512 * 1024) / PAGE_SIZE;
4163 	batch /= 4;		/* We effectively *= 4 below */
4164 	if (batch < 1)
4165 		batch = 1;
4166 
4167 	/*
4168 	 * Clamp the batch to a 2^n - 1 value. Having a power
4169 	 * of 2 value was found to be more likely to have
4170 	 * suboptimal cache aliasing properties in some cases.
4171 	 *
4172 	 * For example if 2 tasks are alternately allocating
4173 	 * batches of pages, one task can end up with a lot
4174 	 * of pages of one half of the possible page colors
4175 	 * and the other with pages of the other colors.
4176 	 */
4177 	batch = rounddown_pow_of_two(batch + batch/2) - 1;
4178 
4179 	return batch;
4180 
4181 #else
4182 	/* The deferral and batching of frees should be suppressed under NOMMU
4183 	 * conditions.
4184 	 *
4185 	 * The problem is that NOMMU needs to be able to allocate large chunks
4186 	 * of contiguous memory as there's no hardware page translation to
4187 	 * assemble apparent contiguous memory from discontiguous pages.
4188 	 *
4189 	 * Queueing large contiguous runs of pages for batching, however,
4190 	 * causes the pages to actually be freed in smaller chunks.  As there
4191 	 * can be a significant delay between the individual batches being
4192 	 * recycled, this leads to the once large chunks of space being
4193 	 * fragmented and becoming unavailable for high-order allocations.
4194 	 */
4195 	return 0;
4196 #endif
4197 }
4198 
4199 /*
4200  * pcp->high and pcp->batch values are related and dependent on one another:
4201  * ->batch must never be higher then ->high.
4202  * The following function updates them in a safe manner without read side
4203  * locking.
4204  *
4205  * Any new users of pcp->batch and pcp->high should ensure they can cope with
4206  * those fields changing asynchronously (acording the the above rule).
4207  *
4208  * mutex_is_locked(&pcp_batch_high_lock) required when calling this function
4209  * outside of boot time (or some other assurance that no concurrent updaters
4210  * exist).
4211  */
4212 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
4213 		unsigned long batch)
4214 {
4215        /* start with a fail safe value for batch */
4216 	pcp->batch = 1;
4217 	smp_wmb();
4218 
4219        /* Update high, then batch, in order */
4220 	pcp->high = high;
4221 	smp_wmb();
4222 
4223 	pcp->batch = batch;
4224 }
4225 
4226 /* a companion to pageset_set_high() */
4227 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch)
4228 {
4229 	pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));
4230 }
4231 
4232 static void pageset_init(struct per_cpu_pageset *p)
4233 {
4234 	struct per_cpu_pages *pcp;
4235 	int migratetype;
4236 
4237 	memset(p, 0, sizeof(*p));
4238 
4239 	pcp = &p->pcp;
4240 	pcp->count = 0;
4241 	for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
4242 		INIT_LIST_HEAD(&pcp->lists[migratetype]);
4243 }
4244 
4245 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
4246 {
4247 	pageset_init(p);
4248 	pageset_set_batch(p, batch);
4249 }
4250 
4251 /*
4252  * pageset_set_high() sets the high water mark for hot per_cpu_pagelist
4253  * to the value high for the pageset p.
4254  */
4255 static void pageset_set_high(struct per_cpu_pageset *p,
4256 				unsigned long high)
4257 {
4258 	unsigned long batch = max(1UL, high / 4);
4259 	if ((high / 4) > (PAGE_SHIFT * 8))
4260 		batch = PAGE_SHIFT * 8;
4261 
4262 	pageset_update(&p->pcp, high, batch);
4263 }
4264 
4265 static void pageset_set_high_and_batch(struct zone *zone,
4266 				       struct per_cpu_pageset *pcp)
4267 {
4268 	if (percpu_pagelist_fraction)
4269 		pageset_set_high(pcp,
4270 			(zone->managed_pages /
4271 				percpu_pagelist_fraction));
4272 	else
4273 		pageset_set_batch(pcp, zone_batchsize(zone));
4274 }
4275 
4276 static void __meminit zone_pageset_init(struct zone *zone, int cpu)
4277 {
4278 	struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
4279 
4280 	pageset_init(pcp);
4281 	pageset_set_high_and_batch(zone, pcp);
4282 }
4283 
4284 static void __meminit setup_zone_pageset(struct zone *zone)
4285 {
4286 	int cpu;
4287 	zone->pageset = alloc_percpu(struct per_cpu_pageset);
4288 	for_each_possible_cpu(cpu)
4289 		zone_pageset_init(zone, cpu);
4290 }
4291 
4292 /*
4293  * Allocate per cpu pagesets and initialize them.
4294  * Before this call only boot pagesets were available.
4295  */
4296 void __init setup_per_cpu_pageset(void)
4297 {
4298 	struct zone *zone;
4299 
4300 	for_each_populated_zone(zone)
4301 		setup_zone_pageset(zone);
4302 }
4303 
4304 static noinline __init_refok
4305 int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
4306 {
4307 	int i;
4308 	size_t alloc_size;
4309 
4310 	/*
4311 	 * The per-page waitqueue mechanism uses hashed waitqueues
4312 	 * per zone.
4313 	 */
4314 	zone->wait_table_hash_nr_entries =
4315 		 wait_table_hash_nr_entries(zone_size_pages);
4316 	zone->wait_table_bits =
4317 		wait_table_bits(zone->wait_table_hash_nr_entries);
4318 	alloc_size = zone->wait_table_hash_nr_entries
4319 					* sizeof(wait_queue_head_t);
4320 
4321 	if (!slab_is_available()) {
4322 		zone->wait_table = (wait_queue_head_t *)
4323 			memblock_virt_alloc_node_nopanic(
4324 				alloc_size, zone->zone_pgdat->node_id);
4325 	} else {
4326 		/*
4327 		 * This case means that a zone whose size was 0 gets new memory
4328 		 * via memory hot-add.
4329 		 * But it may be the case that a new node was hot-added.  In
4330 		 * this case vmalloc() will not be able to use this new node's
4331 		 * memory - this wait_table must be initialized to use this new
4332 		 * node itself as well.
4333 		 * To use this new node's memory, further consideration will be
4334 		 * necessary.
4335 		 */
4336 		zone->wait_table = vmalloc(alloc_size);
4337 	}
4338 	if (!zone->wait_table)
4339 		return -ENOMEM;
4340 
4341 	for (i = 0; i < zone->wait_table_hash_nr_entries; ++i)
4342 		init_waitqueue_head(zone->wait_table + i);
4343 
4344 	return 0;
4345 }
4346 
4347 static __meminit void zone_pcp_init(struct zone *zone)
4348 {
4349 	/*
4350 	 * per cpu subsystem is not up at this point. The following code
4351 	 * relies on the ability of the linker to provide the
4352 	 * offset of a (static) per cpu variable into the per cpu area.
4353 	 */
4354 	zone->pageset = &boot_pageset;
4355 
4356 	if (populated_zone(zone))
4357 		printk(KERN_DEBUG "  %s zone: %lu pages, LIFO batch:%u\n",
4358 			zone->name, zone->present_pages,
4359 					 zone_batchsize(zone));
4360 }
4361 
4362 int __meminit init_currently_empty_zone(struct zone *zone,
4363 					unsigned long zone_start_pfn,
4364 					unsigned long size,
4365 					enum memmap_context context)
4366 {
4367 	struct pglist_data *pgdat = zone->zone_pgdat;
4368 	int ret;
4369 	ret = zone_wait_table_init(zone, size);
4370 	if (ret)
4371 		return ret;
4372 	pgdat->nr_zones = zone_idx(zone) + 1;
4373 
4374 	zone->zone_start_pfn = zone_start_pfn;
4375 
4376 	mminit_dprintk(MMINIT_TRACE, "memmap_init",
4377 			"Initialising map node %d zone %lu pfns %lu -> %lu\n",
4378 			pgdat->node_id,
4379 			(unsigned long)zone_idx(zone),
4380 			zone_start_pfn, (zone_start_pfn + size));
4381 
4382 	zone_init_free_lists(zone);
4383 
4384 	return 0;
4385 }
4386 
4387 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
4388 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
4389 /*
4390  * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
4391  */
4392 int __meminit __early_pfn_to_nid(unsigned long pfn)
4393 {
4394 	unsigned long start_pfn, end_pfn;
4395 	int nid;
4396 	/*
4397 	 * NOTE: The following SMP-unsafe globals are only used early in boot
4398 	 * when the kernel is running single-threaded.
4399 	 */
4400 	static unsigned long __meminitdata last_start_pfn, last_end_pfn;
4401 	static int __meminitdata last_nid;
4402 
4403 	if (last_start_pfn <= pfn && pfn < last_end_pfn)
4404 		return last_nid;
4405 
4406 	nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
4407 	if (nid != -1) {
4408 		last_start_pfn = start_pfn;
4409 		last_end_pfn = end_pfn;
4410 		last_nid = nid;
4411 	}
4412 
4413 	return nid;
4414 }
4415 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
4416 
4417 int __meminit early_pfn_to_nid(unsigned long pfn)
4418 {
4419 	int nid;
4420 
4421 	nid = __early_pfn_to_nid(pfn);
4422 	if (nid >= 0)
4423 		return nid;
4424 	/* just returns 0 */
4425 	return 0;
4426 }
4427 
4428 #ifdef CONFIG_NODES_SPAN_OTHER_NODES
4429 bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
4430 {
4431 	int nid;
4432 
4433 	nid = __early_pfn_to_nid(pfn);
4434 	if (nid >= 0 && nid != node)
4435 		return false;
4436 	return true;
4437 }
4438 #endif
4439 
4440 /**
4441  * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range
4442  * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
4443  * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid
4444  *
4445  * If an architecture guarantees that all ranges registered contain no holes
4446  * and may be freed, this this function may be used instead of calling
4447  * memblock_free_early_nid() manually.
4448  */
4449 void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
4450 {
4451 	unsigned long start_pfn, end_pfn;
4452 	int i, this_nid;
4453 
4454 	for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
4455 		start_pfn = min(start_pfn, max_low_pfn);
4456 		end_pfn = min(end_pfn, max_low_pfn);
4457 
4458 		if (start_pfn < end_pfn)
4459 			memblock_free_early_nid(PFN_PHYS(start_pfn),
4460 					(end_pfn - start_pfn) << PAGE_SHIFT,
4461 					this_nid);
4462 	}
4463 }
4464 
4465 /**
4466  * sparse_memory_present_with_active_regions - Call memory_present for each active range
4467  * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
4468  *
4469  * If an architecture guarantees that all ranges registered contain no holes and may
4470  * be freed, this function may be used instead of calling memory_present() manually.
4471  */
4472 void __init sparse_memory_present_with_active_regions(int nid)
4473 {
4474 	unsigned long start_pfn, end_pfn;
4475 	int i, this_nid;
4476 
4477 	for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
4478 		memory_present(this_nid, start_pfn, end_pfn);
4479 }
4480 
4481 /**
4482  * get_pfn_range_for_nid - Return the start and end page frames for a node
4483  * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
4484  * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
4485  * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
4486  *
4487  * It returns the start and end page frame of a node based on information
4488  * provided by memblock_set_node(). If called for a node
4489  * with no available memory, a warning is printed and the start and end
4490  * PFNs will be 0.
4491  */
4492 void __meminit get_pfn_range_for_nid(unsigned int nid,
4493 			unsigned long *start_pfn, unsigned long *end_pfn)
4494 {
4495 	unsigned long this_start_pfn, this_end_pfn;
4496 	int i;
4497 
4498 	*start_pfn = -1UL;
4499 	*end_pfn = 0;
4500 
4501 	for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
4502 		*start_pfn = min(*start_pfn, this_start_pfn);
4503 		*end_pfn = max(*end_pfn, this_end_pfn);
4504 	}
4505 
4506 	if (*start_pfn == -1UL)
4507 		*start_pfn = 0;
4508 }
4509 
4510 /*
4511  * This finds a zone that can be used for ZONE_MOVABLE pages. The
4512  * assumption is made that zones within a node are ordered in monotonic
4513  * increasing memory addresses so that the "highest" populated zone is used
4514  */
4515 static void __init find_usable_zone_for_movable(void)
4516 {
4517 	int zone_index;
4518 	for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
4519 		if (zone_index == ZONE_MOVABLE)
4520 			continue;
4521 
4522 		if (arch_zone_highest_possible_pfn[zone_index] >
4523 				arch_zone_lowest_possible_pfn[zone_index])
4524 			break;
4525 	}
4526 
4527 	VM_BUG_ON(zone_index == -1);
4528 	movable_zone = zone_index;
4529 }
4530 
4531 /*
4532  * The zone ranges provided by the architecture do not include ZONE_MOVABLE
4533  * because it is sized independent of architecture. Unlike the other zones,
4534  * the starting point for ZONE_MOVABLE is not fixed. It may be different
4535  * in each node depending on the size of each node and how evenly kernelcore
4536  * is distributed. This helper function adjusts the zone ranges
4537  * provided by the architecture for a given node by using the end of the
4538  * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
4539  * zones within a node are in order of monotonic increases memory addresses
4540  */
4541 static void __meminit adjust_zone_range_for_zone_movable(int nid,
4542 					unsigned long zone_type,
4543 					unsigned long node_start_pfn,
4544 					unsigned long node_end_pfn,
4545 					unsigned long *zone_start_pfn,
4546 					unsigned long *zone_end_pfn)
4547 {
4548 	/* Only adjust if ZONE_MOVABLE is on this node */
4549 	if (zone_movable_pfn[nid]) {
4550 		/* Size ZONE_MOVABLE */
4551 		if (zone_type == ZONE_MOVABLE) {
4552 			*zone_start_pfn = zone_movable_pfn[nid];
4553 			*zone_end_pfn = min(node_end_pfn,
4554 				arch_zone_highest_possible_pfn[movable_zone]);
4555 
4556 		/* Adjust for ZONE_MOVABLE starting within this range */
4557 		} else if (*zone_start_pfn < zone_movable_pfn[nid] &&
4558 				*zone_end_pfn > zone_movable_pfn[nid]) {
4559 			*zone_end_pfn = zone_movable_pfn[nid];
4560 
4561 		/* Check if this whole range is within ZONE_MOVABLE */
4562 		} else if (*zone_start_pfn >= zone_movable_pfn[nid])
4563 			*zone_start_pfn = *zone_end_pfn;
4564 	}
4565 }
4566 
4567 /*
4568  * Return the number of pages a zone spans in a node, including holes
4569  * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
4570  */
4571 static unsigned long __meminit zone_spanned_pages_in_node(int nid,
4572 					unsigned long zone_type,
4573 					unsigned long node_start_pfn,
4574 					unsigned long node_end_pfn,
4575 					unsigned long *ignored)
4576 {
4577 	unsigned long zone_start_pfn, zone_end_pfn;
4578 
4579 	/* Get the start and end of the zone */
4580 	zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
4581 	zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
4582 	adjust_zone_range_for_zone_movable(nid, zone_type,
4583 				node_start_pfn, node_end_pfn,
4584 				&zone_start_pfn, &zone_end_pfn);
4585 
4586 	/* Check that this node has pages within the zone's required range */
4587 	if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn)
4588 		return 0;
4589 
4590 	/* Move the zone boundaries inside the node if necessary */
4591 	zone_end_pfn = min(zone_end_pfn, node_end_pfn);
4592 	zone_start_pfn = max(zone_start_pfn, node_start_pfn);
4593 
4594 	/* Return the spanned pages */
4595 	return zone_end_pfn - zone_start_pfn;
4596 }
4597 
4598 /*
4599  * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
4600  * then all holes in the requested range will be accounted for.
4601  */
4602 unsigned long __meminit __absent_pages_in_range(int nid,
4603 				unsigned long range_start_pfn,
4604 				unsigned long range_end_pfn)
4605 {
4606 	unsigned long nr_absent = range_end_pfn - range_start_pfn;
4607 	unsigned long start_pfn, end_pfn;
4608 	int i;
4609 
4610 	for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
4611 		start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
4612 		end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
4613 		nr_absent -= end_pfn - start_pfn;
4614 	}
4615 	return nr_absent;
4616 }
4617 
4618 /**
4619  * absent_pages_in_range - Return number of page frames in holes within a range
4620  * @start_pfn: The start PFN to start searching for holes
4621  * @end_pfn: The end PFN to stop searching for holes
4622  *
4623  * It returns the number of pages frames in memory holes within a range.
4624  */
4625 unsigned long __init absent_pages_in_range(unsigned long start_pfn,
4626 							unsigned long end_pfn)
4627 {
4628 	return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
4629 }
4630 
4631 /* Return the number of page frames in holes in a zone on a node */
4632 static unsigned long __meminit zone_absent_pages_in_node(int nid,
4633 					unsigned long zone_type,
4634 					unsigned long node_start_pfn,
4635 					unsigned long node_end_pfn,
4636 					unsigned long *ignored)
4637 {
4638 	unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
4639 	unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
4640 	unsigned long zone_start_pfn, zone_end_pfn;
4641 
4642 	zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
4643 	zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
4644 
4645 	adjust_zone_range_for_zone_movable(nid, zone_type,
4646 			node_start_pfn, node_end_pfn,
4647 			&zone_start_pfn, &zone_end_pfn);
4648 	return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
4649 }
4650 
4651 #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
4652 static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
4653 					unsigned long zone_type,
4654 					unsigned long node_start_pfn,
4655 					unsigned long node_end_pfn,
4656 					unsigned long *zones_size)
4657 {
4658 	return zones_size[zone_type];
4659 }
4660 
4661 static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
4662 						unsigned long zone_type,
4663 						unsigned long node_start_pfn,
4664 						unsigned long node_end_pfn,
4665 						unsigned long *zholes_size)
4666 {
4667 	if (!zholes_size)
4668 		return 0;
4669 
4670 	return zholes_size[zone_type];
4671 }
4672 
4673 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
4674 
4675 static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
4676 						unsigned long node_start_pfn,
4677 						unsigned long node_end_pfn,
4678 						unsigned long *zones_size,
4679 						unsigned long *zholes_size)
4680 {
4681 	unsigned long realtotalpages, totalpages = 0;
4682 	enum zone_type i;
4683 
4684 	for (i = 0; i < MAX_NR_ZONES; i++)
4685 		totalpages += zone_spanned_pages_in_node(pgdat->node_id, i,
4686 							 node_start_pfn,
4687 							 node_end_pfn,
4688 							 zones_size);
4689 	pgdat->node_spanned_pages = totalpages;
4690 
4691 	realtotalpages = totalpages;
4692 	for (i = 0; i < MAX_NR_ZONES; i++)
4693 		realtotalpages -=
4694 			zone_absent_pages_in_node(pgdat->node_id, i,
4695 						  node_start_pfn, node_end_pfn,
4696 						  zholes_size);
4697 	pgdat->node_present_pages = realtotalpages;
4698 	printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
4699 							realtotalpages);
4700 }
4701 
4702 #ifndef CONFIG_SPARSEMEM
4703 /*
4704  * Calculate the size of the zone->blockflags rounded to an unsigned long
4705  * Start by making sure zonesize is a multiple of pageblock_order by rounding
4706  * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
4707  * round what is now in bits to nearest long in bits, then return it in
4708  * bytes.
4709  */
4710 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
4711 {
4712 	unsigned long usemapsize;
4713 
4714 	zonesize += zone_start_pfn & (pageblock_nr_pages-1);
4715 	usemapsize = roundup(zonesize, pageblock_nr_pages);
4716 	usemapsize = usemapsize >> pageblock_order;
4717 	usemapsize *= NR_PAGEBLOCK_BITS;
4718 	usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
4719 
4720 	return usemapsize / 8;
4721 }
4722 
4723 static void __init setup_usemap(struct pglist_data *pgdat,
4724 				struct zone *zone,
4725 				unsigned long zone_start_pfn,
4726 				unsigned long zonesize)
4727 {
4728 	unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
4729 	zone->pageblock_flags = NULL;
4730 	if (usemapsize)
4731 		zone->pageblock_flags =
4732 			memblock_virt_alloc_node_nopanic(usemapsize,
4733 							 pgdat->node_id);
4734 }
4735 #else
4736 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
4737 				unsigned long zone_start_pfn, unsigned long zonesize) {}
4738 #endif /* CONFIG_SPARSEMEM */
4739 
4740 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
4741 
4742 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
4743 void __paginginit set_pageblock_order(void)
4744 {
4745 	unsigned int order;
4746 
4747 	/* Check that pageblock_nr_pages has not already been setup */
4748 	if (pageblock_order)
4749 		return;
4750 
4751 	if (HPAGE_SHIFT > PAGE_SHIFT)
4752 		order = HUGETLB_PAGE_ORDER;
4753 	else
4754 		order = MAX_ORDER - 1;
4755 
4756 	/*
4757 	 * Assume the largest contiguous order of interest is a huge page.
4758 	 * This value may be variable depending on boot parameters on IA64 and
4759 	 * powerpc.
4760 	 */
4761 	pageblock_order = order;
4762 }
4763 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
4764 
4765 /*
4766  * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
4767  * is unused as pageblock_order is set at compile-time. See
4768  * include/linux/pageblock-flags.h for the values of pageblock_order based on
4769  * the kernel config
4770  */
4771 void __paginginit set_pageblock_order(void)
4772 {
4773 }
4774 
4775 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
4776 
4777 static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages,
4778 						   unsigned long present_pages)
4779 {
4780 	unsigned long pages = spanned_pages;
4781 
4782 	/*
4783 	 * Provide a more accurate estimation if there are holes within
4784 	 * the zone and SPARSEMEM is in use. If there are holes within the
4785 	 * zone, each populated memory region may cost us one or two extra
4786 	 * memmap pages due to alignment because memmap pages for each
4787 	 * populated regions may not naturally algined on page boundary.
4788 	 * So the (present_pages >> 4) heuristic is a tradeoff for that.
4789 	 */
4790 	if (spanned_pages > present_pages + (present_pages >> 4) &&
4791 	    IS_ENABLED(CONFIG_SPARSEMEM))
4792 		pages = present_pages;
4793 
4794 	return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
4795 }
4796 
4797 /*
4798  * Set up the zone data structures:
4799  *   - mark all pages reserved
4800  *   - mark all memory queues empty
4801  *   - clear the memory bitmaps
4802  *
4803  * NOTE: pgdat should get zeroed by caller.
4804  */
4805 static void __paginginit free_area_init_core(struct pglist_data *pgdat,
4806 		unsigned long node_start_pfn, unsigned long node_end_pfn,
4807 		unsigned long *zones_size, unsigned long *zholes_size)
4808 {
4809 	enum zone_type j;
4810 	int nid = pgdat->node_id;
4811 	unsigned long zone_start_pfn = pgdat->node_start_pfn;
4812 	int ret;
4813 
4814 	pgdat_resize_init(pgdat);
4815 #ifdef CONFIG_NUMA_BALANCING
4816 	spin_lock_init(&pgdat->numabalancing_migrate_lock);
4817 	pgdat->numabalancing_migrate_nr_pages = 0;
4818 	pgdat->numabalancing_migrate_next_window = jiffies;
4819 #endif
4820 	init_waitqueue_head(&pgdat->kswapd_wait);
4821 	init_waitqueue_head(&pgdat->pfmemalloc_wait);
4822 	pgdat_page_cgroup_init(pgdat);
4823 
4824 	for (j = 0; j < MAX_NR_ZONES; j++) {
4825 		struct zone *zone = pgdat->node_zones + j;
4826 		unsigned long size, realsize, freesize, memmap_pages;
4827 
4828 		size = zone_spanned_pages_in_node(nid, j, node_start_pfn,
4829 						  node_end_pfn, zones_size);
4830 		realsize = freesize = size - zone_absent_pages_in_node(nid, j,
4831 								node_start_pfn,
4832 								node_end_pfn,
4833 								zholes_size);
4834 
4835 		/*
4836 		 * Adjust freesize so that it accounts for how much memory
4837 		 * is used by this zone for memmap. This affects the watermark
4838 		 * and per-cpu initialisations
4839 		 */
4840 		memmap_pages = calc_memmap_size(size, realsize);
4841 		if (freesize >= memmap_pages) {
4842 			freesize -= memmap_pages;
4843 			if (memmap_pages)
4844 				printk(KERN_DEBUG
4845 				       "  %s zone: %lu pages used for memmap\n",
4846 				       zone_names[j], memmap_pages);
4847 		} else
4848 			printk(KERN_WARNING
4849 				"  %s zone: %lu pages exceeds freesize %lu\n",
4850 				zone_names[j], memmap_pages, freesize);
4851 
4852 		/* Account for reserved pages */
4853 		if (j == 0 && freesize > dma_reserve) {
4854 			freesize -= dma_reserve;
4855 			printk(KERN_DEBUG "  %s zone: %lu pages reserved\n",
4856 					zone_names[0], dma_reserve);
4857 		}
4858 
4859 		if (!is_highmem_idx(j))
4860 			nr_kernel_pages += freesize;
4861 		/* Charge for highmem memmap if there are enough kernel pages */
4862 		else if (nr_kernel_pages > memmap_pages * 2)
4863 			nr_kernel_pages -= memmap_pages;
4864 		nr_all_pages += freesize;
4865 
4866 		zone->spanned_pages = size;
4867 		zone->present_pages = realsize;
4868 		/*
4869 		 * Set an approximate value for lowmem here, it will be adjusted
4870 		 * when the bootmem allocator frees pages into the buddy system.
4871 		 * And all highmem pages will be managed by the buddy system.
4872 		 */
4873 		zone->managed_pages = is_highmem_idx(j) ? realsize : freesize;
4874 #ifdef CONFIG_NUMA
4875 		zone->node = nid;
4876 		zone->min_unmapped_pages = (freesize*sysctl_min_unmapped_ratio)
4877 						/ 100;
4878 		zone->min_slab_pages = (freesize * sysctl_min_slab_ratio) / 100;
4879 #endif
4880 		zone->name = zone_names[j];
4881 		spin_lock_init(&zone->lock);
4882 		spin_lock_init(&zone->lru_lock);
4883 		zone_seqlock_init(zone);
4884 		zone->zone_pgdat = pgdat;
4885 		zone_pcp_init(zone);
4886 
4887 		/* For bootup, initialized properly in watermark setup */
4888 		mod_zone_page_state(zone, NR_ALLOC_BATCH, zone->managed_pages);
4889 
4890 		lruvec_init(&zone->lruvec);
4891 		if (!size)
4892 			continue;
4893 
4894 		set_pageblock_order();
4895 		setup_usemap(pgdat, zone, zone_start_pfn, size);
4896 		ret = init_currently_empty_zone(zone, zone_start_pfn,
4897 						size, MEMMAP_EARLY);
4898 		BUG_ON(ret);
4899 		memmap_init(size, nid, j, zone_start_pfn);
4900 		zone_start_pfn += size;
4901 	}
4902 }
4903 
4904 static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat)
4905 {
4906 	/* Skip empty nodes */
4907 	if (!pgdat->node_spanned_pages)
4908 		return;
4909 
4910 #ifdef CONFIG_FLAT_NODE_MEM_MAP
4911 	/* ia64 gets its own node_mem_map, before this, without bootmem */
4912 	if (!pgdat->node_mem_map) {
4913 		unsigned long size, start, end;
4914 		struct page *map;
4915 
4916 		/*
4917 		 * The zone's endpoints aren't required to be MAX_ORDER
4918 		 * aligned but the node_mem_map endpoints must be in order
4919 		 * for the buddy allocator to function correctly.
4920 		 */
4921 		start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
4922 		end = pgdat_end_pfn(pgdat);
4923 		end = ALIGN(end, MAX_ORDER_NR_PAGES);
4924 		size =  (end - start) * sizeof(struct page);
4925 		map = alloc_remap(pgdat->node_id, size);
4926 		if (!map)
4927 			map = memblock_virt_alloc_node_nopanic(size,
4928 							       pgdat->node_id);
4929 		pgdat->node_mem_map = map + (pgdat->node_start_pfn - start);
4930 	}
4931 #ifndef CONFIG_NEED_MULTIPLE_NODES
4932 	/*
4933 	 * With no DISCONTIG, the global mem_map is just set as node 0's
4934 	 */
4935 	if (pgdat == NODE_DATA(0)) {
4936 		mem_map = NODE_DATA(0)->node_mem_map;
4937 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
4938 		if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
4939 			mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET);
4940 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
4941 	}
4942 #endif
4943 #endif /* CONFIG_FLAT_NODE_MEM_MAP */
4944 }
4945 
4946 void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
4947 		unsigned long node_start_pfn, unsigned long *zholes_size)
4948 {
4949 	pg_data_t *pgdat = NODE_DATA(nid);
4950 	unsigned long start_pfn = 0;
4951 	unsigned long end_pfn = 0;
4952 
4953 	/* pg_data_t should be reset to zero when it's allocated */
4954 	WARN_ON(pgdat->nr_zones || pgdat->classzone_idx);
4955 
4956 	pgdat->node_id = nid;
4957 	pgdat->node_start_pfn = node_start_pfn;
4958 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
4959 	get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
4960 #endif
4961 	calculate_node_totalpages(pgdat, start_pfn, end_pfn,
4962 				  zones_size, zholes_size);
4963 
4964 	alloc_node_mem_map(pgdat);
4965 #ifdef CONFIG_FLAT_NODE_MEM_MAP
4966 	printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n",
4967 		nid, (unsigned long)pgdat,
4968 		(unsigned long)pgdat->node_mem_map);
4969 #endif
4970 
4971 	free_area_init_core(pgdat, start_pfn, end_pfn,
4972 			    zones_size, zholes_size);
4973 }
4974 
4975 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
4976 
4977 #if MAX_NUMNODES > 1
4978 /*
4979  * Figure out the number of possible node ids.
4980  */
4981 void __init setup_nr_node_ids(void)
4982 {
4983 	unsigned int node;
4984 	unsigned int highest = 0;
4985 
4986 	for_each_node_mask(node, node_possible_map)
4987 		highest = node;
4988 	nr_node_ids = highest + 1;
4989 }
4990 #endif
4991 
4992 /**
4993  * node_map_pfn_alignment - determine the maximum internode alignment
4994  *
4995  * This function should be called after node map is populated and sorted.
4996  * It calculates the maximum power of two alignment which can distinguish
4997  * all the nodes.
4998  *
4999  * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
5000  * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)).  If the
5001  * nodes are shifted by 256MiB, 256MiB.  Note that if only the last node is
5002  * shifted, 1GiB is enough and this function will indicate so.
5003  *
5004  * This is used to test whether pfn -> nid mapping of the chosen memory
5005  * model has fine enough granularity to avoid incorrect mapping for the
5006  * populated node map.
5007  *
5008  * Returns the determined alignment in pfn's.  0 if there is no alignment
5009  * requirement (single node).
5010  */
5011 unsigned long __init node_map_pfn_alignment(void)
5012 {
5013 	unsigned long accl_mask = 0, last_end = 0;
5014 	unsigned long start, end, mask;
5015 	int last_nid = -1;
5016 	int i, nid;
5017 
5018 	for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
5019 		if (!start || last_nid < 0 || last_nid == nid) {
5020 			last_nid = nid;
5021 			last_end = end;
5022 			continue;
5023 		}
5024 
5025 		/*
5026 		 * Start with a mask granular enough to pin-point to the
5027 		 * start pfn and tick off bits one-by-one until it becomes
5028 		 * too coarse to separate the current node from the last.
5029 		 */
5030 		mask = ~((1 << __ffs(start)) - 1);
5031 		while (mask && last_end <= (start & (mask << 1)))
5032 			mask <<= 1;
5033 
5034 		/* accumulate all internode masks */
5035 		accl_mask |= mask;
5036 	}
5037 
5038 	/* convert mask to number of pages */
5039 	return ~accl_mask + 1;
5040 }
5041 
5042 /* Find the lowest pfn for a node */
5043 static unsigned long __init find_min_pfn_for_node(int nid)
5044 {
5045 	unsigned long min_pfn = ULONG_MAX;
5046 	unsigned long start_pfn;
5047 	int i;
5048 
5049 	for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
5050 		min_pfn = min(min_pfn, start_pfn);
5051 
5052 	if (min_pfn == ULONG_MAX) {
5053 		printk(KERN_WARNING
5054 			"Could not find start_pfn for node %d\n", nid);
5055 		return 0;
5056 	}
5057 
5058 	return min_pfn;
5059 }
5060 
5061 /**
5062  * find_min_pfn_with_active_regions - Find the minimum PFN registered
5063  *
5064  * It returns the minimum PFN based on information provided via
5065  * memblock_set_node().
5066  */
5067 unsigned long __init find_min_pfn_with_active_regions(void)
5068 {
5069 	return find_min_pfn_for_node(MAX_NUMNODES);
5070 }
5071 
5072 /*
5073  * early_calculate_totalpages()
5074  * Sum pages in active regions for movable zone.
5075  * Populate N_MEMORY for calculating usable_nodes.
5076  */
5077 static unsigned long __init early_calculate_totalpages(void)
5078 {
5079 	unsigned long totalpages = 0;
5080 	unsigned long start_pfn, end_pfn;
5081 	int i, nid;
5082 
5083 	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
5084 		unsigned long pages = end_pfn - start_pfn;
5085 
5086 		totalpages += pages;
5087 		if (pages)
5088 			node_set_state(nid, N_MEMORY);
5089 	}
5090 	return totalpages;
5091 }
5092 
5093 /*
5094  * Find the PFN the Movable zone begins in each node. Kernel memory
5095  * is spread evenly between nodes as long as the nodes have enough
5096  * memory. When they don't, some nodes will have more kernelcore than
5097  * others
5098  */
5099 static void __init find_zone_movable_pfns_for_nodes(void)
5100 {
5101 	int i, nid;
5102 	unsigned long usable_startpfn;
5103 	unsigned long kernelcore_node, kernelcore_remaining;
5104 	/* save the state before borrow the nodemask */
5105 	nodemask_t saved_node_state = node_states[N_MEMORY];
5106 	unsigned long totalpages = early_calculate_totalpages();
5107 	int usable_nodes = nodes_weight(node_states[N_MEMORY]);
5108 	struct memblock_region *r;
5109 
5110 	/* Need to find movable_zone earlier when movable_node is specified. */
5111 	find_usable_zone_for_movable();
5112 
5113 	/*
5114 	 * If movable_node is specified, ignore kernelcore and movablecore
5115 	 * options.
5116 	 */
5117 	if (movable_node_is_enabled()) {
5118 		for_each_memblock(memory, r) {
5119 			if (!memblock_is_hotpluggable(r))
5120 				continue;
5121 
5122 			nid = r->nid;
5123 
5124 			usable_startpfn = PFN_DOWN(r->base);
5125 			zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
5126 				min(usable_startpfn, zone_movable_pfn[nid]) :
5127 				usable_startpfn;
5128 		}
5129 
5130 		goto out2;
5131 	}
5132 
5133 	/*
5134 	 * If movablecore=nn[KMG] was specified, calculate what size of
5135 	 * kernelcore that corresponds so that memory usable for
5136 	 * any allocation type is evenly spread. If both kernelcore
5137 	 * and movablecore are specified, then the value of kernelcore
5138 	 * will be used for required_kernelcore if it's greater than
5139 	 * what movablecore would have allowed.
5140 	 */
5141 	if (required_movablecore) {
5142 		unsigned long corepages;
5143 
5144 		/*
5145 		 * Round-up so that ZONE_MOVABLE is at least as large as what
5146 		 * was requested by the user
5147 		 */
5148 		required_movablecore =
5149 			roundup(required_movablecore, MAX_ORDER_NR_PAGES);
5150 		corepages = totalpages - required_movablecore;
5151 
5152 		required_kernelcore = max(required_kernelcore, corepages);
5153 	}
5154 
5155 	/* If kernelcore was not specified, there is no ZONE_MOVABLE */
5156 	if (!required_kernelcore)
5157 		goto out;
5158 
5159 	/* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
5160 	usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
5161 
5162 restart:
5163 	/* Spread kernelcore memory as evenly as possible throughout nodes */
5164 	kernelcore_node = required_kernelcore / usable_nodes;
5165 	for_each_node_state(nid, N_MEMORY) {
5166 		unsigned long start_pfn, end_pfn;
5167 
5168 		/*
5169 		 * Recalculate kernelcore_node if the division per node
5170 		 * now exceeds what is necessary to satisfy the requested
5171 		 * amount of memory for the kernel
5172 		 */
5173 		if (required_kernelcore < kernelcore_node)
5174 			kernelcore_node = required_kernelcore / usable_nodes;
5175 
5176 		/*
5177 		 * As the map is walked, we track how much memory is usable
5178 		 * by the kernel using kernelcore_remaining. When it is
5179 		 * 0, the rest of the node is usable by ZONE_MOVABLE
5180 		 */
5181 		kernelcore_remaining = kernelcore_node;
5182 
5183 		/* Go through each range of PFNs within this node */
5184 		for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
5185 			unsigned long size_pages;
5186 
5187 			start_pfn = max(start_pfn, zone_movable_pfn[nid]);
5188 			if (start_pfn >= end_pfn)
5189 				continue;
5190 
5191 			/* Account for what is only usable for kernelcore */
5192 			if (start_pfn < usable_startpfn) {
5193 				unsigned long kernel_pages;
5194 				kernel_pages = min(end_pfn, usable_startpfn)
5195 								- start_pfn;
5196 
5197 				kernelcore_remaining -= min(kernel_pages,
5198 							kernelcore_remaining);
5199 				required_kernelcore -= min(kernel_pages,
5200 							required_kernelcore);
5201 
5202 				/* Continue if range is now fully accounted */
5203 				if (end_pfn <= usable_startpfn) {
5204 
5205 					/*
5206 					 * Push zone_movable_pfn to the end so
5207 					 * that if we have to rebalance
5208 					 * kernelcore across nodes, we will
5209 					 * not double account here
5210 					 */
5211 					zone_movable_pfn[nid] = end_pfn;
5212 					continue;
5213 				}
5214 				start_pfn = usable_startpfn;
5215 			}
5216 
5217 			/*
5218 			 * The usable PFN range for ZONE_MOVABLE is from
5219 			 * start_pfn->end_pfn. Calculate size_pages as the
5220 			 * number of pages used as kernelcore
5221 			 */
5222 			size_pages = end_pfn - start_pfn;
5223 			if (size_pages > kernelcore_remaining)
5224 				size_pages = kernelcore_remaining;
5225 			zone_movable_pfn[nid] = start_pfn + size_pages;
5226 
5227 			/*
5228 			 * Some kernelcore has been met, update counts and
5229 			 * break if the kernelcore for this node has been
5230 			 * satisfied
5231 			 */
5232 			required_kernelcore -= min(required_kernelcore,
5233 								size_pages);
5234 			kernelcore_remaining -= size_pages;
5235 			if (!kernelcore_remaining)
5236 				break;
5237 		}
5238 	}
5239 
5240 	/*
5241 	 * If there is still required_kernelcore, we do another pass with one
5242 	 * less node in the count. This will push zone_movable_pfn[nid] further
5243 	 * along on the nodes that still have memory until kernelcore is
5244 	 * satisfied
5245 	 */
5246 	usable_nodes--;
5247 	if (usable_nodes && required_kernelcore > usable_nodes)
5248 		goto restart;
5249 
5250 out2:
5251 	/* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
5252 	for (nid = 0; nid < MAX_NUMNODES; nid++)
5253 		zone_movable_pfn[nid] =
5254 			roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
5255 
5256 out:
5257 	/* restore the node_state */
5258 	node_states[N_MEMORY] = saved_node_state;
5259 }
5260 
5261 /* Any regular or high memory on that node ? */
5262 static void check_for_memory(pg_data_t *pgdat, int nid)
5263 {
5264 	enum zone_type zone_type;
5265 
5266 	if (N_MEMORY == N_NORMAL_MEMORY)
5267 		return;
5268 
5269 	for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
5270 		struct zone *zone = &pgdat->node_zones[zone_type];
5271 		if (populated_zone(zone)) {
5272 			node_set_state(nid, N_HIGH_MEMORY);
5273 			if (N_NORMAL_MEMORY != N_HIGH_MEMORY &&
5274 			    zone_type <= ZONE_NORMAL)
5275 				node_set_state(nid, N_NORMAL_MEMORY);
5276 			break;
5277 		}
5278 	}
5279 }
5280 
5281 /**
5282  * free_area_init_nodes - Initialise all pg_data_t and zone data
5283  * @max_zone_pfn: an array of max PFNs for each zone
5284  *
5285  * This will call free_area_init_node() for each active node in the system.
5286  * Using the page ranges provided by memblock_set_node(), the size of each
5287  * zone in each node and their holes is calculated. If the maximum PFN
5288  * between two adjacent zones match, it is assumed that the zone is empty.
5289  * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
5290  * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
5291  * starts where the previous one ended. For example, ZONE_DMA32 starts
5292  * at arch_max_dma_pfn.
5293  */
5294 void __init free_area_init_nodes(unsigned long *max_zone_pfn)
5295 {
5296 	unsigned long start_pfn, end_pfn;
5297 	int i, nid;
5298 
5299 	/* Record where the zone boundaries are */
5300 	memset(arch_zone_lowest_possible_pfn, 0,
5301 				sizeof(arch_zone_lowest_possible_pfn));
5302 	memset(arch_zone_highest_possible_pfn, 0,
5303 				sizeof(arch_zone_highest_possible_pfn));
5304 	arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions();
5305 	arch_zone_highest_possible_pfn[0] = max_zone_pfn[0];
5306 	for (i = 1; i < MAX_NR_ZONES; i++) {
5307 		if (i == ZONE_MOVABLE)
5308 			continue;
5309 		arch_zone_lowest_possible_pfn[i] =
5310 			arch_zone_highest_possible_pfn[i-1];
5311 		arch_zone_highest_possible_pfn[i] =
5312 			max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]);
5313 	}
5314 	arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0;
5315 	arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0;
5316 
5317 	/* Find the PFNs that ZONE_MOVABLE begins at in each node */
5318 	memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
5319 	find_zone_movable_pfns_for_nodes();
5320 
5321 	/* Print out the zone ranges */
5322 	printk("Zone ranges:\n");
5323 	for (i = 0; i < MAX_NR_ZONES; i++) {
5324 		if (i == ZONE_MOVABLE)
5325 			continue;
5326 		printk(KERN_CONT "  %-8s ", zone_names[i]);
5327 		if (arch_zone_lowest_possible_pfn[i] ==
5328 				arch_zone_highest_possible_pfn[i])
5329 			printk(KERN_CONT "empty\n");
5330 		else
5331 			printk(KERN_CONT "[mem %0#10lx-%0#10lx]\n",
5332 				arch_zone_lowest_possible_pfn[i] << PAGE_SHIFT,
5333 				(arch_zone_highest_possible_pfn[i]
5334 					<< PAGE_SHIFT) - 1);
5335 	}
5336 
5337 	/* Print out the PFNs ZONE_MOVABLE begins at in each node */
5338 	printk("Movable zone start for each node\n");
5339 	for (i = 0; i < MAX_NUMNODES; i++) {
5340 		if (zone_movable_pfn[i])
5341 			printk("  Node %d: %#010lx\n", i,
5342 			       zone_movable_pfn[i] << PAGE_SHIFT);
5343 	}
5344 
5345 	/* Print out the early node map */
5346 	printk("Early memory node ranges\n");
5347 	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
5348 		printk("  node %3d: [mem %#010lx-%#010lx]\n", nid,
5349 		       start_pfn << PAGE_SHIFT, (end_pfn << PAGE_SHIFT) - 1);
5350 
5351 	/* Initialise every node */
5352 	mminit_verify_pageflags_layout();
5353 	setup_nr_node_ids();
5354 	for_each_online_node(nid) {
5355 		pg_data_t *pgdat = NODE_DATA(nid);
5356 		free_area_init_node(nid, NULL,
5357 				find_min_pfn_for_node(nid), NULL);
5358 
5359 		/* Any memory on that node */
5360 		if (pgdat->node_present_pages)
5361 			node_set_state(nid, N_MEMORY);
5362 		check_for_memory(pgdat, nid);
5363 	}
5364 }
5365 
5366 static int __init cmdline_parse_core(char *p, unsigned long *core)
5367 {
5368 	unsigned long long coremem;
5369 	if (!p)
5370 		return -EINVAL;
5371 
5372 	coremem = memparse(p, &p);
5373 	*core = coremem >> PAGE_SHIFT;
5374 
5375 	/* Paranoid check that UL is enough for the coremem value */
5376 	WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
5377 
5378 	return 0;
5379 }
5380 
5381 /*
5382  * kernelcore=size sets the amount of memory for use for allocations that
5383  * cannot be reclaimed or migrated.
5384  */
5385 static int __init cmdline_parse_kernelcore(char *p)
5386 {
5387 	return cmdline_parse_core(p, &required_kernelcore);
5388 }
5389 
5390 /*
5391  * movablecore=size sets the amount of memory for use for allocations that
5392  * can be reclaimed or migrated.
5393  */
5394 static int __init cmdline_parse_movablecore(char *p)
5395 {
5396 	return cmdline_parse_core(p, &required_movablecore);
5397 }
5398 
5399 early_param("kernelcore", cmdline_parse_kernelcore);
5400 early_param("movablecore", cmdline_parse_movablecore);
5401 
5402 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
5403 
5404 void adjust_managed_page_count(struct page *page, long count)
5405 {
5406 	spin_lock(&managed_page_count_lock);
5407 	page_zone(page)->managed_pages += count;
5408 	totalram_pages += count;
5409 #ifdef CONFIG_HIGHMEM
5410 	if (PageHighMem(page))
5411 		totalhigh_pages += count;
5412 #endif
5413 	spin_unlock(&managed_page_count_lock);
5414 }
5415 EXPORT_SYMBOL(adjust_managed_page_count);
5416 
5417 unsigned long free_reserved_area(void *start, void *end, int poison, char *s)
5418 {
5419 	void *pos;
5420 	unsigned long pages = 0;
5421 
5422 	start = (void *)PAGE_ALIGN((unsigned long)start);
5423 	end = (void *)((unsigned long)end & PAGE_MASK);
5424 	for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
5425 		if ((unsigned int)poison <= 0xFF)
5426 			memset(pos, poison, PAGE_SIZE);
5427 		free_reserved_page(virt_to_page(pos));
5428 	}
5429 
5430 	if (pages && s)
5431 		pr_info("Freeing %s memory: %ldK (%p - %p)\n",
5432 			s, pages << (PAGE_SHIFT - 10), start, end);
5433 
5434 	return pages;
5435 }
5436 EXPORT_SYMBOL(free_reserved_area);
5437 
5438 #ifdef	CONFIG_HIGHMEM
5439 void free_highmem_page(struct page *page)
5440 {
5441 	__free_reserved_page(page);
5442 	totalram_pages++;
5443 	page_zone(page)->managed_pages++;
5444 	totalhigh_pages++;
5445 }
5446 #endif
5447 
5448 
5449 void __init mem_init_print_info(const char *str)
5450 {
5451 	unsigned long physpages, codesize, datasize, rosize, bss_size;
5452 	unsigned long init_code_size, init_data_size;
5453 
5454 	physpages = get_num_physpages();
5455 	codesize = _etext - _stext;
5456 	datasize = _edata - _sdata;
5457 	rosize = __end_rodata - __start_rodata;
5458 	bss_size = __bss_stop - __bss_start;
5459 	init_data_size = __init_end - __init_begin;
5460 	init_code_size = _einittext - _sinittext;
5461 
5462 	/*
5463 	 * Detect special cases and adjust section sizes accordingly:
5464 	 * 1) .init.* may be embedded into .data sections
5465 	 * 2) .init.text.* may be out of [__init_begin, __init_end],
5466 	 *    please refer to arch/tile/kernel/vmlinux.lds.S.
5467 	 * 3) .rodata.* may be embedded into .text or .data sections.
5468 	 */
5469 #define adj_init_size(start, end, size, pos, adj) \
5470 	do { \
5471 		if (start <= pos && pos < end && size > adj) \
5472 			size -= adj; \
5473 	} while (0)
5474 
5475 	adj_init_size(__init_begin, __init_end, init_data_size,
5476 		     _sinittext, init_code_size);
5477 	adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
5478 	adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
5479 	adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
5480 	adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
5481 
5482 #undef	adj_init_size
5483 
5484 	printk("Memory: %luK/%luK available "
5485 	       "(%luK kernel code, %luK rwdata, %luK rodata, "
5486 	       "%luK init, %luK bss, %luK reserved"
5487 #ifdef	CONFIG_HIGHMEM
5488 	       ", %luK highmem"
5489 #endif
5490 	       "%s%s)\n",
5491 	       nr_free_pages() << (PAGE_SHIFT-10), physpages << (PAGE_SHIFT-10),
5492 	       codesize >> 10, datasize >> 10, rosize >> 10,
5493 	       (init_data_size + init_code_size) >> 10, bss_size >> 10,
5494 	       (physpages - totalram_pages) << (PAGE_SHIFT-10),
5495 #ifdef	CONFIG_HIGHMEM
5496 	       totalhigh_pages << (PAGE_SHIFT-10),
5497 #endif
5498 	       str ? ", " : "", str ? str : "");
5499 }
5500 
5501 /**
5502  * set_dma_reserve - set the specified number of pages reserved in the first zone
5503  * @new_dma_reserve: The number of pages to mark reserved
5504  *
5505  * The per-cpu batchsize and zone watermarks are determined by present_pages.
5506  * In the DMA zone, a significant percentage may be consumed by kernel image
5507  * and other unfreeable allocations which can skew the watermarks badly. This
5508  * function may optionally be used to account for unfreeable pages in the
5509  * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
5510  * smaller per-cpu batchsize.
5511  */
5512 void __init set_dma_reserve(unsigned long new_dma_reserve)
5513 {
5514 	dma_reserve = new_dma_reserve;
5515 }
5516 
5517 void __init free_area_init(unsigned long *zones_size)
5518 {
5519 	free_area_init_node(0, zones_size,
5520 			__pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
5521 }
5522 
5523 static int page_alloc_cpu_notify(struct notifier_block *self,
5524 				 unsigned long action, void *hcpu)
5525 {
5526 	int cpu = (unsigned long)hcpu;
5527 
5528 	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
5529 		lru_add_drain_cpu(cpu);
5530 		drain_pages(cpu);
5531 
5532 		/*
5533 		 * Spill the event counters of the dead processor
5534 		 * into the current processors event counters.
5535 		 * This artificially elevates the count of the current
5536 		 * processor.
5537 		 */
5538 		vm_events_fold_cpu(cpu);
5539 
5540 		/*
5541 		 * Zero the differential counters of the dead processor
5542 		 * so that the vm statistics are consistent.
5543 		 *
5544 		 * This is only okay since the processor is dead and cannot
5545 		 * race with what we are doing.
5546 		 */
5547 		cpu_vm_stats_fold(cpu);
5548 	}
5549 	return NOTIFY_OK;
5550 }
5551 
5552 void __init page_alloc_init(void)
5553 {
5554 	hotcpu_notifier(page_alloc_cpu_notify, 0);
5555 }
5556 
5557 /*
5558  * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio
5559  *	or min_free_kbytes changes.
5560  */
5561 static void calculate_totalreserve_pages(void)
5562 {
5563 	struct pglist_data *pgdat;
5564 	unsigned long reserve_pages = 0;
5565 	enum zone_type i, j;
5566 
5567 	for_each_online_pgdat(pgdat) {
5568 		for (i = 0; i < MAX_NR_ZONES; i++) {
5569 			struct zone *zone = pgdat->node_zones + i;
5570 			unsigned long max = 0;
5571 
5572 			/* Find valid and maximum lowmem_reserve in the zone */
5573 			for (j = i; j < MAX_NR_ZONES; j++) {
5574 				if (zone->lowmem_reserve[j] > max)
5575 					max = zone->lowmem_reserve[j];
5576 			}
5577 
5578 			/* we treat the high watermark as reserved pages. */
5579 			max += high_wmark_pages(zone);
5580 
5581 			if (max > zone->managed_pages)
5582 				max = zone->managed_pages;
5583 			reserve_pages += max;
5584 			/*
5585 			 * Lowmem reserves are not available to
5586 			 * GFP_HIGHUSER page cache allocations and
5587 			 * kswapd tries to balance zones to their high
5588 			 * watermark.  As a result, neither should be
5589 			 * regarded as dirtyable memory, to prevent a
5590 			 * situation where reclaim has to clean pages
5591 			 * in order to balance the zones.
5592 			 */
5593 			zone->dirty_balance_reserve = max;
5594 		}
5595 	}
5596 	dirty_balance_reserve = reserve_pages;
5597 	totalreserve_pages = reserve_pages;
5598 }
5599 
5600 /*
5601  * setup_per_zone_lowmem_reserve - called whenever
5602  *	sysctl_lower_zone_reserve_ratio changes.  Ensures that each zone
5603  *	has a correct pages reserved value, so an adequate number of
5604  *	pages are left in the zone after a successful __alloc_pages().
5605  */
5606 static void setup_per_zone_lowmem_reserve(void)
5607 {
5608 	struct pglist_data *pgdat;
5609 	enum zone_type j, idx;
5610 
5611 	for_each_online_pgdat(pgdat) {
5612 		for (j = 0; j < MAX_NR_ZONES; j++) {
5613 			struct zone *zone = pgdat->node_zones + j;
5614 			unsigned long managed_pages = zone->managed_pages;
5615 
5616 			zone->lowmem_reserve[j] = 0;
5617 
5618 			idx = j;
5619 			while (idx) {
5620 				struct zone *lower_zone;
5621 
5622 				idx--;
5623 
5624 				if (sysctl_lowmem_reserve_ratio[idx] < 1)
5625 					sysctl_lowmem_reserve_ratio[idx] = 1;
5626 
5627 				lower_zone = pgdat->node_zones + idx;
5628 				lower_zone->lowmem_reserve[j] = managed_pages /
5629 					sysctl_lowmem_reserve_ratio[idx];
5630 				managed_pages += lower_zone->managed_pages;
5631 			}
5632 		}
5633 	}
5634 
5635 	/* update totalreserve_pages */
5636 	calculate_totalreserve_pages();
5637 }
5638 
5639 static void __setup_per_zone_wmarks(void)
5640 {
5641 	unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
5642 	unsigned long lowmem_pages = 0;
5643 	struct zone *zone;
5644 	unsigned long flags;
5645 
5646 	/* Calculate total number of !ZONE_HIGHMEM pages */
5647 	for_each_zone(zone) {
5648 		if (!is_highmem(zone))
5649 			lowmem_pages += zone->managed_pages;
5650 	}
5651 
5652 	for_each_zone(zone) {
5653 		u64 tmp;
5654 
5655 		spin_lock_irqsave(&zone->lock, flags);
5656 		tmp = (u64)pages_min * zone->managed_pages;
5657 		do_div(tmp, lowmem_pages);
5658 		if (is_highmem(zone)) {
5659 			/*
5660 			 * __GFP_HIGH and PF_MEMALLOC allocations usually don't
5661 			 * need highmem pages, so cap pages_min to a small
5662 			 * value here.
5663 			 *
5664 			 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
5665 			 * deltas controls asynch page reclaim, and so should
5666 			 * not be capped for highmem.
5667 			 */
5668 			unsigned long min_pages;
5669 
5670 			min_pages = zone->managed_pages / 1024;
5671 			min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
5672 			zone->watermark[WMARK_MIN] = min_pages;
5673 		} else {
5674 			/*
5675 			 * If it's a lowmem zone, reserve a number of pages
5676 			 * proportionate to the zone's size.
5677 			 */
5678 			zone->watermark[WMARK_MIN] = tmp;
5679 		}
5680 
5681 		zone->watermark[WMARK_LOW]  = min_wmark_pages(zone) + (tmp >> 2);
5682 		zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
5683 
5684 		__mod_zone_page_state(zone, NR_ALLOC_BATCH,
5685 				      high_wmark_pages(zone) -
5686 				      low_wmark_pages(zone) -
5687 				      zone_page_state(zone, NR_ALLOC_BATCH));
5688 
5689 		setup_zone_migrate_reserve(zone);
5690 		spin_unlock_irqrestore(&zone->lock, flags);
5691 	}
5692 
5693 	/* update totalreserve_pages */
5694 	calculate_totalreserve_pages();
5695 }
5696 
5697 /**
5698  * setup_per_zone_wmarks - called when min_free_kbytes changes
5699  * or when memory is hot-{added|removed}
5700  *
5701  * Ensures that the watermark[min,low,high] values for each zone are set
5702  * correctly with respect to min_free_kbytes.
5703  */
5704 void setup_per_zone_wmarks(void)
5705 {
5706 	mutex_lock(&zonelists_mutex);
5707 	__setup_per_zone_wmarks();
5708 	mutex_unlock(&zonelists_mutex);
5709 }
5710 
5711 /*
5712  * The inactive anon list should be small enough that the VM never has to
5713  * do too much work, but large enough that each inactive page has a chance
5714  * to be referenced again before it is swapped out.
5715  *
5716  * The inactive_anon ratio is the target ratio of ACTIVE_ANON to
5717  * INACTIVE_ANON pages on this zone's LRU, maintained by the
5718  * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of
5719  * the anonymous pages are kept on the inactive list.
5720  *
5721  * total     target    max
5722  * memory    ratio     inactive anon
5723  * -------------------------------------
5724  *   10MB       1         5MB
5725  *  100MB       1        50MB
5726  *    1GB       3       250MB
5727  *   10GB      10       0.9GB
5728  *  100GB      31         3GB
5729  *    1TB     101        10GB
5730  *   10TB     320        32GB
5731  */
5732 static void __meminit calculate_zone_inactive_ratio(struct zone *zone)
5733 {
5734 	unsigned int gb, ratio;
5735 
5736 	/* Zone size in gigabytes */
5737 	gb = zone->managed_pages >> (30 - PAGE_SHIFT);
5738 	if (gb)
5739 		ratio = int_sqrt(10 * gb);
5740 	else
5741 		ratio = 1;
5742 
5743 	zone->inactive_ratio = ratio;
5744 }
5745 
5746 static void __meminit setup_per_zone_inactive_ratio(void)
5747 {
5748 	struct zone *zone;
5749 
5750 	for_each_zone(zone)
5751 		calculate_zone_inactive_ratio(zone);
5752 }
5753 
5754 /*
5755  * Initialise min_free_kbytes.
5756  *
5757  * For small machines we want it small (128k min).  For large machines
5758  * we want it large (64MB max).  But it is not linear, because network
5759  * bandwidth does not increase linearly with machine size.  We use
5760  *
5761  *	min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
5762  *	min_free_kbytes = sqrt(lowmem_kbytes * 16)
5763  *
5764  * which yields
5765  *
5766  * 16MB:	512k
5767  * 32MB:	724k
5768  * 64MB:	1024k
5769  * 128MB:	1448k
5770  * 256MB:	2048k
5771  * 512MB:	2896k
5772  * 1024MB:	4096k
5773  * 2048MB:	5792k
5774  * 4096MB:	8192k
5775  * 8192MB:	11584k
5776  * 16384MB:	16384k
5777  */
5778 int __meminit init_per_zone_wmark_min(void)
5779 {
5780 	unsigned long lowmem_kbytes;
5781 	int new_min_free_kbytes;
5782 
5783 	lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
5784 	new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
5785 
5786 	if (new_min_free_kbytes > user_min_free_kbytes) {
5787 		min_free_kbytes = new_min_free_kbytes;
5788 		if (min_free_kbytes < 128)
5789 			min_free_kbytes = 128;
5790 		if (min_free_kbytes > 65536)
5791 			min_free_kbytes = 65536;
5792 	} else {
5793 		pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
5794 				new_min_free_kbytes, user_min_free_kbytes);
5795 	}
5796 	setup_per_zone_wmarks();
5797 	refresh_zone_stat_thresholds();
5798 	setup_per_zone_lowmem_reserve();
5799 	setup_per_zone_inactive_ratio();
5800 	return 0;
5801 }
5802 module_init(init_per_zone_wmark_min)
5803 
5804 /*
5805  * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
5806  *	that we can call two helper functions whenever min_free_kbytes
5807  *	changes.
5808  */
5809 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
5810 	void __user *buffer, size_t *length, loff_t *ppos)
5811 {
5812 	int rc;
5813 
5814 	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
5815 	if (rc)
5816 		return rc;
5817 
5818 	if (write) {
5819 		user_min_free_kbytes = min_free_kbytes;
5820 		setup_per_zone_wmarks();
5821 	}
5822 	return 0;
5823 }
5824 
5825 #ifdef CONFIG_NUMA
5826 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
5827 	void __user *buffer, size_t *length, loff_t *ppos)
5828 {
5829 	struct zone *zone;
5830 	int rc;
5831 
5832 	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
5833 	if (rc)
5834 		return rc;
5835 
5836 	for_each_zone(zone)
5837 		zone->min_unmapped_pages = (zone->managed_pages *
5838 				sysctl_min_unmapped_ratio) / 100;
5839 	return 0;
5840 }
5841 
5842 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
5843 	void __user *buffer, size_t *length, loff_t *ppos)
5844 {
5845 	struct zone *zone;
5846 	int rc;
5847 
5848 	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
5849 	if (rc)
5850 		return rc;
5851 
5852 	for_each_zone(zone)
5853 		zone->min_slab_pages = (zone->managed_pages *
5854 				sysctl_min_slab_ratio) / 100;
5855 	return 0;
5856 }
5857 #endif
5858 
5859 /*
5860  * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
5861  *	proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
5862  *	whenever sysctl_lowmem_reserve_ratio changes.
5863  *
5864  * The reserve ratio obviously has absolutely no relation with the
5865  * minimum watermarks. The lowmem reserve ratio can only make sense
5866  * if in function of the boot time zone sizes.
5867  */
5868 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
5869 	void __user *buffer, size_t *length, loff_t *ppos)
5870 {
5871 	proc_dointvec_minmax(table, write, buffer, length, ppos);
5872 	setup_per_zone_lowmem_reserve();
5873 	return 0;
5874 }
5875 
5876 /*
5877  * percpu_pagelist_fraction - changes the pcp->high for each zone on each
5878  * cpu.  It is the fraction of total pages in each zone that a hot per cpu
5879  * pagelist can have before it gets flushed back to buddy allocator.
5880  */
5881 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write,
5882 	void __user *buffer, size_t *length, loff_t *ppos)
5883 {
5884 	struct zone *zone;
5885 	int old_percpu_pagelist_fraction;
5886 	int ret;
5887 
5888 	mutex_lock(&pcp_batch_high_lock);
5889 	old_percpu_pagelist_fraction = percpu_pagelist_fraction;
5890 
5891 	ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
5892 	if (!write || ret < 0)
5893 		goto out;
5894 
5895 	/* Sanity checking to avoid pcp imbalance */
5896 	if (percpu_pagelist_fraction &&
5897 	    percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) {
5898 		percpu_pagelist_fraction = old_percpu_pagelist_fraction;
5899 		ret = -EINVAL;
5900 		goto out;
5901 	}
5902 
5903 	/* No change? */
5904 	if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
5905 		goto out;
5906 
5907 	for_each_populated_zone(zone) {
5908 		unsigned int cpu;
5909 
5910 		for_each_possible_cpu(cpu)
5911 			pageset_set_high_and_batch(zone,
5912 					per_cpu_ptr(zone->pageset, cpu));
5913 	}
5914 out:
5915 	mutex_unlock(&pcp_batch_high_lock);
5916 	return ret;
5917 }
5918 
5919 int hashdist = HASHDIST_DEFAULT;
5920 
5921 #ifdef CONFIG_NUMA
5922 static int __init set_hashdist(char *str)
5923 {
5924 	if (!str)
5925 		return 0;
5926 	hashdist = simple_strtoul(str, &str, 0);
5927 	return 1;
5928 }
5929 __setup("hashdist=", set_hashdist);
5930 #endif
5931 
5932 /*
5933  * allocate a large system hash table from bootmem
5934  * - it is assumed that the hash table must contain an exact power-of-2
5935  *   quantity of entries
5936  * - limit is the number of hash buckets, not the total allocation size
5937  */
5938 void *__init alloc_large_system_hash(const char *tablename,
5939 				     unsigned long bucketsize,
5940 				     unsigned long numentries,
5941 				     int scale,
5942 				     int flags,
5943 				     unsigned int *_hash_shift,
5944 				     unsigned int *_hash_mask,
5945 				     unsigned long low_limit,
5946 				     unsigned long high_limit)
5947 {
5948 	unsigned long long max = high_limit;
5949 	unsigned long log2qty, size;
5950 	void *table = NULL;
5951 
5952 	/* allow the kernel cmdline to have a say */
5953 	if (!numentries) {
5954 		/* round applicable memory size up to nearest megabyte */
5955 		numentries = nr_kernel_pages;
5956 
5957 		/* It isn't necessary when PAGE_SIZE >= 1MB */
5958 		if (PAGE_SHIFT < 20)
5959 			numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
5960 
5961 		/* limit to 1 bucket per 2^scale bytes of low memory */
5962 		if (scale > PAGE_SHIFT)
5963 			numentries >>= (scale - PAGE_SHIFT);
5964 		else
5965 			numentries <<= (PAGE_SHIFT - scale);
5966 
5967 		/* Make sure we've got at least a 0-order allocation.. */
5968 		if (unlikely(flags & HASH_SMALL)) {
5969 			/* Makes no sense without HASH_EARLY */
5970 			WARN_ON(!(flags & HASH_EARLY));
5971 			if (!(numentries >> *_hash_shift)) {
5972 				numentries = 1UL << *_hash_shift;
5973 				BUG_ON(!numentries);
5974 			}
5975 		} else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
5976 			numentries = PAGE_SIZE / bucketsize;
5977 	}
5978 	numentries = roundup_pow_of_two(numentries);
5979 
5980 	/* limit allocation size to 1/16 total memory by default */
5981 	if (max == 0) {
5982 		max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
5983 		do_div(max, bucketsize);
5984 	}
5985 	max = min(max, 0x80000000ULL);
5986 
5987 	if (numentries < low_limit)
5988 		numentries = low_limit;
5989 	if (numentries > max)
5990 		numentries = max;
5991 
5992 	log2qty = ilog2(numentries);
5993 
5994 	do {
5995 		size = bucketsize << log2qty;
5996 		if (flags & HASH_EARLY)
5997 			table = memblock_virt_alloc_nopanic(size, 0);
5998 		else if (hashdist)
5999 			table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
6000 		else {
6001 			/*
6002 			 * If bucketsize is not a power-of-two, we may free
6003 			 * some pages at the end of hash table which
6004 			 * alloc_pages_exact() automatically does
6005 			 */
6006 			if (get_order(size) < MAX_ORDER) {
6007 				table = alloc_pages_exact(size, GFP_ATOMIC);
6008 				kmemleak_alloc(table, size, 1, GFP_ATOMIC);
6009 			}
6010 		}
6011 	} while (!table && size > PAGE_SIZE && --log2qty);
6012 
6013 	if (!table)
6014 		panic("Failed to allocate %s hash table\n", tablename);
6015 
6016 	printk(KERN_INFO "%s hash table entries: %ld (order: %d, %lu bytes)\n",
6017 	       tablename,
6018 	       (1UL << log2qty),
6019 	       ilog2(size) - PAGE_SHIFT,
6020 	       size);
6021 
6022 	if (_hash_shift)
6023 		*_hash_shift = log2qty;
6024 	if (_hash_mask)
6025 		*_hash_mask = (1 << log2qty) - 1;
6026 
6027 	return table;
6028 }
6029 
6030 /* Return a pointer to the bitmap storing bits affecting a block of pages */
6031 static inline unsigned long *get_pageblock_bitmap(struct zone *zone,
6032 							unsigned long pfn)
6033 {
6034 #ifdef CONFIG_SPARSEMEM
6035 	return __pfn_to_section(pfn)->pageblock_flags;
6036 #else
6037 	return zone->pageblock_flags;
6038 #endif /* CONFIG_SPARSEMEM */
6039 }
6040 
6041 static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
6042 {
6043 #ifdef CONFIG_SPARSEMEM
6044 	pfn &= (PAGES_PER_SECTION-1);
6045 	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
6046 #else
6047 	pfn = pfn - round_down(zone->zone_start_pfn, pageblock_nr_pages);
6048 	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
6049 #endif /* CONFIG_SPARSEMEM */
6050 }
6051 
6052 /**
6053  * get_pageblock_flags_group - Return the requested group of flags for the pageblock_nr_pages block of pages
6054  * @page: The page within the block of interest
6055  * @start_bitidx: The first bit of interest to retrieve
6056  * @end_bitidx: The last bit of interest
6057  * returns pageblock_bits flags
6058  */
6059 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
6060 					unsigned long end_bitidx,
6061 					unsigned long mask)
6062 {
6063 	struct zone *zone;
6064 	unsigned long *bitmap;
6065 	unsigned long bitidx, word_bitidx;
6066 	unsigned long word;
6067 
6068 	zone = page_zone(page);
6069 	bitmap = get_pageblock_bitmap(zone, pfn);
6070 	bitidx = pfn_to_bitidx(zone, pfn);
6071 	word_bitidx = bitidx / BITS_PER_LONG;
6072 	bitidx &= (BITS_PER_LONG-1);
6073 
6074 	word = bitmap[word_bitidx];
6075 	bitidx += end_bitidx;
6076 	return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
6077 }
6078 
6079 /**
6080  * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
6081  * @page: The page within the block of interest
6082  * @start_bitidx: The first bit of interest
6083  * @end_bitidx: The last bit of interest
6084  * @flags: The flags to set
6085  */
6086 void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
6087 					unsigned long pfn,
6088 					unsigned long end_bitidx,
6089 					unsigned long mask)
6090 {
6091 	struct zone *zone;
6092 	unsigned long *bitmap;
6093 	unsigned long bitidx, word_bitidx;
6094 	unsigned long old_word, word;
6095 
6096 	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
6097 
6098 	zone = page_zone(page);
6099 	bitmap = get_pageblock_bitmap(zone, pfn);
6100 	bitidx = pfn_to_bitidx(zone, pfn);
6101 	word_bitidx = bitidx / BITS_PER_LONG;
6102 	bitidx &= (BITS_PER_LONG-1);
6103 
6104 	VM_BUG_ON_PAGE(!zone_spans_pfn(zone, pfn), page);
6105 
6106 	bitidx += end_bitidx;
6107 	mask <<= (BITS_PER_LONG - bitidx - 1);
6108 	flags <<= (BITS_PER_LONG - bitidx - 1);
6109 
6110 	word = ACCESS_ONCE(bitmap[word_bitidx]);
6111 	for (;;) {
6112 		old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
6113 		if (word == old_word)
6114 			break;
6115 		word = old_word;
6116 	}
6117 }
6118 
6119 /*
6120  * This function checks whether pageblock includes unmovable pages or not.
6121  * If @count is not zero, it is okay to include less @count unmovable pages
6122  *
6123  * PageLRU check without isolation or lru_lock could race so that
6124  * MIGRATE_MOVABLE block might include unmovable pages. It means you can't
6125  * expect this function should be exact.
6126  */
6127 bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
6128 			 bool skip_hwpoisoned_pages)
6129 {
6130 	unsigned long pfn, iter, found;
6131 	int mt;
6132 
6133 	/*
6134 	 * For avoiding noise data, lru_add_drain_all() should be called
6135 	 * If ZONE_MOVABLE, the zone never contains unmovable pages
6136 	 */
6137 	if (zone_idx(zone) == ZONE_MOVABLE)
6138 		return false;
6139 	mt = get_pageblock_migratetype(page);
6140 	if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt))
6141 		return false;
6142 
6143 	pfn = page_to_pfn(page);
6144 	for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
6145 		unsigned long check = pfn + iter;
6146 
6147 		if (!pfn_valid_within(check))
6148 			continue;
6149 
6150 		page = pfn_to_page(check);
6151 
6152 		/*
6153 		 * Hugepages are not in LRU lists, but they're movable.
6154 		 * We need not scan over tail pages bacause we don't
6155 		 * handle each tail page individually in migration.
6156 		 */
6157 		if (PageHuge(page)) {
6158 			iter = round_up(iter + 1, 1<<compound_order(page)) - 1;
6159 			continue;
6160 		}
6161 
6162 		/*
6163 		 * We can't use page_count without pin a page
6164 		 * because another CPU can free compound page.
6165 		 * This check already skips compound tails of THP
6166 		 * because their page->_count is zero at all time.
6167 		 */
6168 		if (!atomic_read(&page->_count)) {
6169 			if (PageBuddy(page))
6170 				iter += (1 << page_order(page)) - 1;
6171 			continue;
6172 		}
6173 
6174 		/*
6175 		 * The HWPoisoned page may be not in buddy system, and
6176 		 * page_count() is not 0.
6177 		 */
6178 		if (skip_hwpoisoned_pages && PageHWPoison(page))
6179 			continue;
6180 
6181 		if (!PageLRU(page))
6182 			found++;
6183 		/*
6184 		 * If there are RECLAIMABLE pages, we need to check it.
6185 		 * But now, memory offline itself doesn't call shrink_slab()
6186 		 * and it still to be fixed.
6187 		 */
6188 		/*
6189 		 * If the page is not RAM, page_count()should be 0.
6190 		 * we don't need more check. This is an _used_ not-movable page.
6191 		 *
6192 		 * The problematic thing here is PG_reserved pages. PG_reserved
6193 		 * is set to both of a memory hole page and a _used_ kernel
6194 		 * page at boot.
6195 		 */
6196 		if (found > count)
6197 			return true;
6198 	}
6199 	return false;
6200 }
6201 
6202 bool is_pageblock_removable_nolock(struct page *page)
6203 {
6204 	struct zone *zone;
6205 	unsigned long pfn;
6206 
6207 	/*
6208 	 * We have to be careful here because we are iterating over memory
6209 	 * sections which are not zone aware so we might end up outside of
6210 	 * the zone but still within the section.
6211 	 * We have to take care about the node as well. If the node is offline
6212 	 * its NODE_DATA will be NULL - see page_zone.
6213 	 */
6214 	if (!node_online(page_to_nid(page)))
6215 		return false;
6216 
6217 	zone = page_zone(page);
6218 	pfn = page_to_pfn(page);
6219 	if (!zone_spans_pfn(zone, pfn))
6220 		return false;
6221 
6222 	return !has_unmovable_pages(zone, page, 0, true);
6223 }
6224 
6225 #ifdef CONFIG_CMA
6226 
6227 static unsigned long pfn_max_align_down(unsigned long pfn)
6228 {
6229 	return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
6230 			     pageblock_nr_pages) - 1);
6231 }
6232 
6233 static unsigned long pfn_max_align_up(unsigned long pfn)
6234 {
6235 	return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
6236 				pageblock_nr_pages));
6237 }
6238 
6239 /* [start, end) must belong to a single zone. */
6240 static int __alloc_contig_migrate_range(struct compact_control *cc,
6241 					unsigned long start, unsigned long end)
6242 {
6243 	/* This function is based on compact_zone() from compaction.c. */
6244 	unsigned long nr_reclaimed;
6245 	unsigned long pfn = start;
6246 	unsigned int tries = 0;
6247 	int ret = 0;
6248 
6249 	migrate_prep();
6250 
6251 	while (pfn < end || !list_empty(&cc->migratepages)) {
6252 		if (fatal_signal_pending(current)) {
6253 			ret = -EINTR;
6254 			break;
6255 		}
6256 
6257 		if (list_empty(&cc->migratepages)) {
6258 			cc->nr_migratepages = 0;
6259 			pfn = isolate_migratepages_range(cc->zone, cc,
6260 							 pfn, end, true);
6261 			if (!pfn) {
6262 				ret = -EINTR;
6263 				break;
6264 			}
6265 			tries = 0;
6266 		} else if (++tries == 5) {
6267 			ret = ret < 0 ? ret : -EBUSY;
6268 			break;
6269 		}
6270 
6271 		nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
6272 							&cc->migratepages);
6273 		cc->nr_migratepages -= nr_reclaimed;
6274 
6275 		ret = migrate_pages(&cc->migratepages, alloc_migrate_target,
6276 				    NULL, 0, cc->mode, MR_CMA);
6277 	}
6278 	if (ret < 0) {
6279 		putback_movable_pages(&cc->migratepages);
6280 		return ret;
6281 	}
6282 	return 0;
6283 }
6284 
6285 /**
6286  * alloc_contig_range() -- tries to allocate given range of pages
6287  * @start:	start PFN to allocate
6288  * @end:	one-past-the-last PFN to allocate
6289  * @migratetype:	migratetype of the underlaying pageblocks (either
6290  *			#MIGRATE_MOVABLE or #MIGRATE_CMA).  All pageblocks
6291  *			in range must have the same migratetype and it must
6292  *			be either of the two.
6293  *
6294  * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
6295  * aligned, however it's the caller's responsibility to guarantee that
6296  * we are the only thread that changes migrate type of pageblocks the
6297  * pages fall in.
6298  *
6299  * The PFN range must belong to a single zone.
6300  *
6301  * Returns zero on success or negative error code.  On success all
6302  * pages which PFN is in [start, end) are allocated for the caller and
6303  * need to be freed with free_contig_range().
6304  */
6305 int alloc_contig_range(unsigned long start, unsigned long end,
6306 		       unsigned migratetype)
6307 {
6308 	unsigned long outer_start, outer_end;
6309 	int ret = 0, order;
6310 
6311 	struct compact_control cc = {
6312 		.nr_migratepages = 0,
6313 		.order = -1,
6314 		.zone = page_zone(pfn_to_page(start)),
6315 		.mode = MIGRATE_SYNC,
6316 		.ignore_skip_hint = true,
6317 	};
6318 	INIT_LIST_HEAD(&cc.migratepages);
6319 
6320 	/*
6321 	 * What we do here is we mark all pageblocks in range as
6322 	 * MIGRATE_ISOLATE.  Because pageblock and max order pages may
6323 	 * have different sizes, and due to the way page allocator
6324 	 * work, we align the range to biggest of the two pages so
6325 	 * that page allocator won't try to merge buddies from
6326 	 * different pageblocks and change MIGRATE_ISOLATE to some
6327 	 * other migration type.
6328 	 *
6329 	 * Once the pageblocks are marked as MIGRATE_ISOLATE, we
6330 	 * migrate the pages from an unaligned range (ie. pages that
6331 	 * we are interested in).  This will put all the pages in
6332 	 * range back to page allocator as MIGRATE_ISOLATE.
6333 	 *
6334 	 * When this is done, we take the pages in range from page
6335 	 * allocator removing them from the buddy system.  This way
6336 	 * page allocator will never consider using them.
6337 	 *
6338 	 * This lets us mark the pageblocks back as
6339 	 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
6340 	 * aligned range but not in the unaligned, original range are
6341 	 * put back to page allocator so that buddy can use them.
6342 	 */
6343 
6344 	ret = start_isolate_page_range(pfn_max_align_down(start),
6345 				       pfn_max_align_up(end), migratetype,
6346 				       false);
6347 	if (ret)
6348 		return ret;
6349 
6350 	ret = __alloc_contig_migrate_range(&cc, start, end);
6351 	if (ret)
6352 		goto done;
6353 
6354 	/*
6355 	 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
6356 	 * aligned blocks that are marked as MIGRATE_ISOLATE.  What's
6357 	 * more, all pages in [start, end) are free in page allocator.
6358 	 * What we are going to do is to allocate all pages from
6359 	 * [start, end) (that is remove them from page allocator).
6360 	 *
6361 	 * The only problem is that pages at the beginning and at the
6362 	 * end of interesting range may be not aligned with pages that
6363 	 * page allocator holds, ie. they can be part of higher order
6364 	 * pages.  Because of this, we reserve the bigger range and
6365 	 * once this is done free the pages we are not interested in.
6366 	 *
6367 	 * We don't have to hold zone->lock here because the pages are
6368 	 * isolated thus they won't get removed from buddy.
6369 	 */
6370 
6371 	lru_add_drain_all();
6372 	drain_all_pages();
6373 
6374 	order = 0;
6375 	outer_start = start;
6376 	while (!PageBuddy(pfn_to_page(outer_start))) {
6377 		if (++order >= MAX_ORDER) {
6378 			ret = -EBUSY;
6379 			goto done;
6380 		}
6381 		outer_start &= ~0UL << order;
6382 	}
6383 
6384 	/* Make sure the range is really isolated. */
6385 	if (test_pages_isolated(outer_start, end, false)) {
6386 		pr_warn("alloc_contig_range test_pages_isolated(%lx, %lx) failed\n",
6387 		       outer_start, end);
6388 		ret = -EBUSY;
6389 		goto done;
6390 	}
6391 
6392 
6393 	/* Grab isolated pages from freelists. */
6394 	outer_end = isolate_freepages_range(&cc, outer_start, end);
6395 	if (!outer_end) {
6396 		ret = -EBUSY;
6397 		goto done;
6398 	}
6399 
6400 	/* Free head and tail (if any) */
6401 	if (start != outer_start)
6402 		free_contig_range(outer_start, start - outer_start);
6403 	if (end != outer_end)
6404 		free_contig_range(end, outer_end - end);
6405 
6406 done:
6407 	undo_isolate_page_range(pfn_max_align_down(start),
6408 				pfn_max_align_up(end), migratetype);
6409 	return ret;
6410 }
6411 
6412 void free_contig_range(unsigned long pfn, unsigned nr_pages)
6413 {
6414 	unsigned int count = 0;
6415 
6416 	for (; nr_pages--; pfn++) {
6417 		struct page *page = pfn_to_page(pfn);
6418 
6419 		count += page_count(page) != 1;
6420 		__free_page(page);
6421 	}
6422 	WARN(count != 0, "%d pages are still in use!\n", count);
6423 }
6424 #endif
6425 
6426 #ifdef CONFIG_MEMORY_HOTPLUG
6427 /*
6428  * The zone indicated has a new number of managed_pages; batch sizes and percpu
6429  * page high values need to be recalulated.
6430  */
6431 void __meminit zone_pcp_update(struct zone *zone)
6432 {
6433 	unsigned cpu;
6434 	mutex_lock(&pcp_batch_high_lock);
6435 	for_each_possible_cpu(cpu)
6436 		pageset_set_high_and_batch(zone,
6437 				per_cpu_ptr(zone->pageset, cpu));
6438 	mutex_unlock(&pcp_batch_high_lock);
6439 }
6440 #endif
6441 
6442 void zone_pcp_reset(struct zone *zone)
6443 {
6444 	unsigned long flags;
6445 	int cpu;
6446 	struct per_cpu_pageset *pset;
6447 
6448 	/* avoid races with drain_pages()  */
6449 	local_irq_save(flags);
6450 	if (zone->pageset != &boot_pageset) {
6451 		for_each_online_cpu(cpu) {
6452 			pset = per_cpu_ptr(zone->pageset, cpu);
6453 			drain_zonestat(zone, pset);
6454 		}
6455 		free_percpu(zone->pageset);
6456 		zone->pageset = &boot_pageset;
6457 	}
6458 	local_irq_restore(flags);
6459 }
6460 
6461 #ifdef CONFIG_MEMORY_HOTREMOVE
6462 /*
6463  * All pages in the range must be isolated before calling this.
6464  */
6465 void
6466 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
6467 {
6468 	struct page *page;
6469 	struct zone *zone;
6470 	unsigned int order, i;
6471 	unsigned long pfn;
6472 	unsigned long flags;
6473 	/* find the first valid pfn */
6474 	for (pfn = start_pfn; pfn < end_pfn; pfn++)
6475 		if (pfn_valid(pfn))
6476 			break;
6477 	if (pfn == end_pfn)
6478 		return;
6479 	zone = page_zone(pfn_to_page(pfn));
6480 	spin_lock_irqsave(&zone->lock, flags);
6481 	pfn = start_pfn;
6482 	while (pfn < end_pfn) {
6483 		if (!pfn_valid(pfn)) {
6484 			pfn++;
6485 			continue;
6486 		}
6487 		page = pfn_to_page(pfn);
6488 		/*
6489 		 * The HWPoisoned page may be not in buddy system, and
6490 		 * page_count() is not 0.
6491 		 */
6492 		if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
6493 			pfn++;
6494 			SetPageReserved(page);
6495 			continue;
6496 		}
6497 
6498 		BUG_ON(page_count(page));
6499 		BUG_ON(!PageBuddy(page));
6500 		order = page_order(page);
6501 #ifdef CONFIG_DEBUG_VM
6502 		printk(KERN_INFO "remove from free list %lx %d %lx\n",
6503 		       pfn, 1 << order, end_pfn);
6504 #endif
6505 		list_del(&page->lru);
6506 		rmv_page_order(page);
6507 		zone->free_area[order].nr_free--;
6508 		for (i = 0; i < (1 << order); i++)
6509 			SetPageReserved((page+i));
6510 		pfn += (1 << order);
6511 	}
6512 	spin_unlock_irqrestore(&zone->lock, flags);
6513 }
6514 #endif
6515 
6516 #ifdef CONFIG_MEMORY_FAILURE
6517 bool is_free_buddy_page(struct page *page)
6518 {
6519 	struct zone *zone = page_zone(page);
6520 	unsigned long pfn = page_to_pfn(page);
6521 	unsigned long flags;
6522 	unsigned int order;
6523 
6524 	spin_lock_irqsave(&zone->lock, flags);
6525 	for (order = 0; order < MAX_ORDER; order++) {
6526 		struct page *page_head = page - (pfn & ((1 << order) - 1));
6527 
6528 		if (PageBuddy(page_head) && page_order(page_head) >= order)
6529 			break;
6530 	}
6531 	spin_unlock_irqrestore(&zone->lock, flags);
6532 
6533 	return order < MAX_ORDER;
6534 }
6535 #endif
6536 
6537 static const struct trace_print_flags pageflag_names[] = {
6538 	{1UL << PG_locked,		"locked"	},
6539 	{1UL << PG_error,		"error"		},
6540 	{1UL << PG_referenced,		"referenced"	},
6541 	{1UL << PG_uptodate,		"uptodate"	},
6542 	{1UL << PG_dirty,		"dirty"		},
6543 	{1UL << PG_lru,			"lru"		},
6544 	{1UL << PG_active,		"active"	},
6545 	{1UL << PG_slab,		"slab"		},
6546 	{1UL << PG_owner_priv_1,	"owner_priv_1"	},
6547 	{1UL << PG_arch_1,		"arch_1"	},
6548 	{1UL << PG_reserved,		"reserved"	},
6549 	{1UL << PG_private,		"private"	},
6550 	{1UL << PG_private_2,		"private_2"	},
6551 	{1UL << PG_writeback,		"writeback"	},
6552 #ifdef CONFIG_PAGEFLAGS_EXTENDED
6553 	{1UL << PG_head,		"head"		},
6554 	{1UL << PG_tail,		"tail"		},
6555 #else
6556 	{1UL << PG_compound,		"compound"	},
6557 #endif
6558 	{1UL << PG_swapcache,		"swapcache"	},
6559 	{1UL << PG_mappedtodisk,	"mappedtodisk"	},
6560 	{1UL << PG_reclaim,		"reclaim"	},
6561 	{1UL << PG_swapbacked,		"swapbacked"	},
6562 	{1UL << PG_unevictable,		"unevictable"	},
6563 #ifdef CONFIG_MMU
6564 	{1UL << PG_mlocked,		"mlocked"	},
6565 #endif
6566 #ifdef CONFIG_ARCH_USES_PG_UNCACHED
6567 	{1UL << PG_uncached,		"uncached"	},
6568 #endif
6569 #ifdef CONFIG_MEMORY_FAILURE
6570 	{1UL << PG_hwpoison,		"hwpoison"	},
6571 #endif
6572 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
6573 	{1UL << PG_compound_lock,	"compound_lock"	},
6574 #endif
6575 };
6576 
6577 static void dump_page_flags(unsigned long flags)
6578 {
6579 	const char *delim = "";
6580 	unsigned long mask;
6581 	int i;
6582 
6583 	BUILD_BUG_ON(ARRAY_SIZE(pageflag_names) != __NR_PAGEFLAGS);
6584 
6585 	printk(KERN_ALERT "page flags: %#lx(", flags);
6586 
6587 	/* remove zone id */
6588 	flags &= (1UL << NR_PAGEFLAGS) - 1;
6589 
6590 	for (i = 0; i < ARRAY_SIZE(pageflag_names) && flags; i++) {
6591 
6592 		mask = pageflag_names[i].mask;
6593 		if ((flags & mask) != mask)
6594 			continue;
6595 
6596 		flags &= ~mask;
6597 		printk("%s%s", delim, pageflag_names[i].name);
6598 		delim = "|";
6599 	}
6600 
6601 	/* check for left over flags */
6602 	if (flags)
6603 		printk("%s%#lx", delim, flags);
6604 
6605 	printk(")\n");
6606 }
6607 
6608 void dump_page_badflags(struct page *page, const char *reason,
6609 		unsigned long badflags)
6610 {
6611 	printk(KERN_ALERT
6612 	       "page:%p count:%d mapcount:%d mapping:%p index:%#lx\n",
6613 		page, atomic_read(&page->_count), page_mapcount(page),
6614 		page->mapping, page->index);
6615 	dump_page_flags(page->flags);
6616 	if (reason)
6617 		pr_alert("page dumped because: %s\n", reason);
6618 	if (page->flags & badflags) {
6619 		pr_alert("bad because of flags:\n");
6620 		dump_page_flags(page->flags & badflags);
6621 	}
6622 	mem_cgroup_print_bad_page(page);
6623 }
6624 
6625 void dump_page(struct page *page, const char *reason)
6626 {
6627 	dump_page_badflags(page, reason, 0);
6628 }
6629 EXPORT_SYMBOL(dump_page);
6630