1 /* 2 * linux/mm/page_alloc.c 3 * 4 * Manages the free list, the system allocates free pages here. 5 * Note that kmalloc() lives in slab.c 6 * 7 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 8 * Swap reorganised 29.12.95, Stephen Tweedie 9 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 10 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 11 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 12 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 13 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 14 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 15 */ 16 17 #include <linux/stddef.h> 18 #include <linux/mm.h> 19 #include <linux/highmem.h> 20 #include <linux/swap.h> 21 #include <linux/interrupt.h> 22 #include <linux/pagemap.h> 23 #include <linux/jiffies.h> 24 #include <linux/memblock.h> 25 #include <linux/compiler.h> 26 #include <linux/kernel.h> 27 #include <linux/kasan.h> 28 #include <linux/module.h> 29 #include <linux/suspend.h> 30 #include <linux/pagevec.h> 31 #include <linux/blkdev.h> 32 #include <linux/slab.h> 33 #include <linux/ratelimit.h> 34 #include <linux/oom.h> 35 #include <linux/topology.h> 36 #include <linux/sysctl.h> 37 #include <linux/cpu.h> 38 #include <linux/cpuset.h> 39 #include <linux/memory_hotplug.h> 40 #include <linux/nodemask.h> 41 #include <linux/vmalloc.h> 42 #include <linux/vmstat.h> 43 #include <linux/mempolicy.h> 44 #include <linux/memremap.h> 45 #include <linux/stop_machine.h> 46 #include <linux/sort.h> 47 #include <linux/pfn.h> 48 #include <linux/backing-dev.h> 49 #include <linux/fault-inject.h> 50 #include <linux/page-isolation.h> 51 #include <linux/page_ext.h> 52 #include <linux/debugobjects.h> 53 #include <linux/kmemleak.h> 54 #include <linux/compaction.h> 55 #include <trace/events/kmem.h> 56 #include <trace/events/oom.h> 57 #include <linux/prefetch.h> 58 #include <linux/mm_inline.h> 59 #include <linux/migrate.h> 60 #include <linux/hugetlb.h> 61 #include <linux/sched/rt.h> 62 #include <linux/sched/mm.h> 63 #include <linux/page_owner.h> 64 #include <linux/kthread.h> 65 #include <linux/memcontrol.h> 66 #include <linux/ftrace.h> 67 #include <linux/lockdep.h> 68 #include <linux/nmi.h> 69 #include <linux/psi.h> 70 71 #include <asm/sections.h> 72 #include <asm/tlbflush.h> 73 #include <asm/div64.h> 74 #include "internal.h" 75 76 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 77 static DEFINE_MUTEX(pcp_batch_high_lock); 78 #define MIN_PERCPU_PAGELIST_FRACTION (8) 79 80 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 81 DEFINE_PER_CPU(int, numa_node); 82 EXPORT_PER_CPU_SYMBOL(numa_node); 83 #endif 84 85 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); 86 87 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 88 /* 89 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 90 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 91 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 92 * defined in <linux/topology.h>. 93 */ 94 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 95 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 96 int _node_numa_mem_[MAX_NUMNODES]; 97 #endif 98 99 /* work_structs for global per-cpu drains */ 100 struct pcpu_drain { 101 struct zone *zone; 102 struct work_struct work; 103 }; 104 DEFINE_MUTEX(pcpu_drain_mutex); 105 DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain); 106 107 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 108 volatile unsigned long latent_entropy __latent_entropy; 109 EXPORT_SYMBOL(latent_entropy); 110 #endif 111 112 /* 113 * Array of node states. 114 */ 115 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 116 [N_POSSIBLE] = NODE_MASK_ALL, 117 [N_ONLINE] = { { [0] = 1UL } }, 118 #ifndef CONFIG_NUMA 119 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 120 #ifdef CONFIG_HIGHMEM 121 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 122 #endif 123 [N_MEMORY] = { { [0] = 1UL } }, 124 [N_CPU] = { { [0] = 1UL } }, 125 #endif /* NUMA */ 126 }; 127 EXPORT_SYMBOL(node_states); 128 129 atomic_long_t _totalram_pages __read_mostly; 130 EXPORT_SYMBOL(_totalram_pages); 131 unsigned long totalreserve_pages __read_mostly; 132 unsigned long totalcma_pages __read_mostly; 133 134 int percpu_pagelist_fraction; 135 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 136 137 /* 138 * A cached value of the page's pageblock's migratetype, used when the page is 139 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 140 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 141 * Also the migratetype set in the page does not necessarily match the pcplist 142 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 143 * other index - this ensures that it will be put on the correct CMA freelist. 144 */ 145 static inline int get_pcppage_migratetype(struct page *page) 146 { 147 return page->index; 148 } 149 150 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 151 { 152 page->index = migratetype; 153 } 154 155 #ifdef CONFIG_PM_SLEEP 156 /* 157 * The following functions are used by the suspend/hibernate code to temporarily 158 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 159 * while devices are suspended. To avoid races with the suspend/hibernate code, 160 * they should always be called with system_transition_mutex held 161 * (gfp_allowed_mask also should only be modified with system_transition_mutex 162 * held, unless the suspend/hibernate code is guaranteed not to run in parallel 163 * with that modification). 164 */ 165 166 static gfp_t saved_gfp_mask; 167 168 void pm_restore_gfp_mask(void) 169 { 170 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 171 if (saved_gfp_mask) { 172 gfp_allowed_mask = saved_gfp_mask; 173 saved_gfp_mask = 0; 174 } 175 } 176 177 void pm_restrict_gfp_mask(void) 178 { 179 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 180 WARN_ON(saved_gfp_mask); 181 saved_gfp_mask = gfp_allowed_mask; 182 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 183 } 184 185 bool pm_suspended_storage(void) 186 { 187 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 188 return false; 189 return true; 190 } 191 #endif /* CONFIG_PM_SLEEP */ 192 193 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 194 unsigned int pageblock_order __read_mostly; 195 #endif 196 197 static void __free_pages_ok(struct page *page, unsigned int order); 198 199 /* 200 * results with 256, 32 in the lowmem_reserve sysctl: 201 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 202 * 1G machine -> (16M dma, 784M normal, 224M high) 203 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 204 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 205 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 206 * 207 * TBD: should special case ZONE_DMA32 machines here - in those we normally 208 * don't need any ZONE_NORMAL reservation 209 */ 210 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { 211 #ifdef CONFIG_ZONE_DMA 212 [ZONE_DMA] = 256, 213 #endif 214 #ifdef CONFIG_ZONE_DMA32 215 [ZONE_DMA32] = 256, 216 #endif 217 [ZONE_NORMAL] = 32, 218 #ifdef CONFIG_HIGHMEM 219 [ZONE_HIGHMEM] = 0, 220 #endif 221 [ZONE_MOVABLE] = 0, 222 }; 223 224 EXPORT_SYMBOL(totalram_pages); 225 226 static char * const zone_names[MAX_NR_ZONES] = { 227 #ifdef CONFIG_ZONE_DMA 228 "DMA", 229 #endif 230 #ifdef CONFIG_ZONE_DMA32 231 "DMA32", 232 #endif 233 "Normal", 234 #ifdef CONFIG_HIGHMEM 235 "HighMem", 236 #endif 237 "Movable", 238 #ifdef CONFIG_ZONE_DEVICE 239 "Device", 240 #endif 241 }; 242 243 const char * const migratetype_names[MIGRATE_TYPES] = { 244 "Unmovable", 245 "Movable", 246 "Reclaimable", 247 "HighAtomic", 248 #ifdef CONFIG_CMA 249 "CMA", 250 #endif 251 #ifdef CONFIG_MEMORY_ISOLATION 252 "Isolate", 253 #endif 254 }; 255 256 compound_page_dtor * const compound_page_dtors[] = { 257 NULL, 258 free_compound_page, 259 #ifdef CONFIG_HUGETLB_PAGE 260 free_huge_page, 261 #endif 262 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 263 free_transhuge_page, 264 #endif 265 }; 266 267 int min_free_kbytes = 1024; 268 int user_min_free_kbytes = -1; 269 #ifdef CONFIG_DISCONTIGMEM 270 /* 271 * DiscontigMem defines memory ranges as separate pg_data_t even if the ranges 272 * are not on separate NUMA nodes. Functionally this works but with 273 * watermark_boost_factor, it can reclaim prematurely as the ranges can be 274 * quite small. By default, do not boost watermarks on discontigmem as in 275 * many cases very high-order allocations like THP are likely to be 276 * unsupported and the premature reclaim offsets the advantage of long-term 277 * fragmentation avoidance. 278 */ 279 int watermark_boost_factor __read_mostly; 280 #else 281 int watermark_boost_factor __read_mostly = 15000; 282 #endif 283 int watermark_scale_factor = 10; 284 285 static unsigned long nr_kernel_pages __initdata; 286 static unsigned long nr_all_pages __initdata; 287 static unsigned long dma_reserve __initdata; 288 289 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 290 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 291 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 292 static unsigned long required_kernelcore __initdata; 293 static unsigned long required_kernelcore_percent __initdata; 294 static unsigned long required_movablecore __initdata; 295 static unsigned long required_movablecore_percent __initdata; 296 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 297 static bool mirrored_kernelcore __meminitdata; 298 299 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 300 int movable_zone; 301 EXPORT_SYMBOL(movable_zone); 302 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 303 304 #if MAX_NUMNODES > 1 305 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; 306 unsigned int nr_online_nodes __read_mostly = 1; 307 EXPORT_SYMBOL(nr_node_ids); 308 EXPORT_SYMBOL(nr_online_nodes); 309 #endif 310 311 int page_group_by_mobility_disabled __read_mostly; 312 313 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 314 /* 315 * During boot we initialize deferred pages on-demand, as needed, but once 316 * page_alloc_init_late() has finished, the deferred pages are all initialized, 317 * and we can permanently disable that path. 318 */ 319 static DEFINE_STATIC_KEY_TRUE(deferred_pages); 320 321 /* 322 * Calling kasan_free_pages() only after deferred memory initialization 323 * has completed. Poisoning pages during deferred memory init will greatly 324 * lengthen the process and cause problem in large memory systems as the 325 * deferred pages initialization is done with interrupt disabled. 326 * 327 * Assuming that there will be no reference to those newly initialized 328 * pages before they are ever allocated, this should have no effect on 329 * KASAN memory tracking as the poison will be properly inserted at page 330 * allocation time. The only corner case is when pages are allocated by 331 * on-demand allocation and then freed again before the deferred pages 332 * initialization is done, but this is not likely to happen. 333 */ 334 static inline void kasan_free_nondeferred_pages(struct page *page, int order) 335 { 336 if (!static_branch_unlikely(&deferred_pages)) 337 kasan_free_pages(page, order); 338 } 339 340 /* Returns true if the struct page for the pfn is uninitialised */ 341 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 342 { 343 int nid = early_pfn_to_nid(pfn); 344 345 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 346 return true; 347 348 return false; 349 } 350 351 /* 352 * Returns true when the remaining initialisation should be deferred until 353 * later in the boot cycle when it can be parallelised. 354 */ 355 static bool __meminit 356 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 357 { 358 static unsigned long prev_end_pfn, nr_initialised; 359 360 /* 361 * prev_end_pfn static that contains the end of previous zone 362 * No need to protect because called very early in boot before smp_init. 363 */ 364 if (prev_end_pfn != end_pfn) { 365 prev_end_pfn = end_pfn; 366 nr_initialised = 0; 367 } 368 369 /* Always populate low zones for address-constrained allocations */ 370 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 371 return false; 372 373 /* 374 * We start only with one section of pages, more pages are added as 375 * needed until the rest of deferred pages are initialized. 376 */ 377 nr_initialised++; 378 if ((nr_initialised > PAGES_PER_SECTION) && 379 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 380 NODE_DATA(nid)->first_deferred_pfn = pfn; 381 return true; 382 } 383 return false; 384 } 385 #else 386 #define kasan_free_nondeferred_pages(p, o) kasan_free_pages(p, o) 387 388 static inline bool early_page_uninitialised(unsigned long pfn) 389 { 390 return false; 391 } 392 393 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 394 { 395 return false; 396 } 397 #endif 398 399 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 400 static inline unsigned long *get_pageblock_bitmap(struct page *page, 401 unsigned long pfn) 402 { 403 #ifdef CONFIG_SPARSEMEM 404 return __pfn_to_section(pfn)->pageblock_flags; 405 #else 406 return page_zone(page)->pageblock_flags; 407 #endif /* CONFIG_SPARSEMEM */ 408 } 409 410 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn) 411 { 412 #ifdef CONFIG_SPARSEMEM 413 pfn &= (PAGES_PER_SECTION-1); 414 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 415 #else 416 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 417 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 418 #endif /* CONFIG_SPARSEMEM */ 419 } 420 421 /** 422 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 423 * @page: The page within the block of interest 424 * @pfn: The target page frame number 425 * @end_bitidx: The last bit of interest to retrieve 426 * @mask: mask of bits that the caller is interested in 427 * 428 * Return: pageblock_bits flags 429 */ 430 static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page, 431 unsigned long pfn, 432 unsigned long end_bitidx, 433 unsigned long mask) 434 { 435 unsigned long *bitmap; 436 unsigned long bitidx, word_bitidx; 437 unsigned long word; 438 439 bitmap = get_pageblock_bitmap(page, pfn); 440 bitidx = pfn_to_bitidx(page, pfn); 441 word_bitidx = bitidx / BITS_PER_LONG; 442 bitidx &= (BITS_PER_LONG-1); 443 444 word = bitmap[word_bitidx]; 445 bitidx += end_bitidx; 446 return (word >> (BITS_PER_LONG - bitidx - 1)) & mask; 447 } 448 449 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, 450 unsigned long end_bitidx, 451 unsigned long mask) 452 { 453 return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask); 454 } 455 456 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn) 457 { 458 return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK); 459 } 460 461 /** 462 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 463 * @page: The page within the block of interest 464 * @flags: The flags to set 465 * @pfn: The target page frame number 466 * @end_bitidx: The last bit of interest 467 * @mask: mask of bits that the caller is interested in 468 */ 469 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 470 unsigned long pfn, 471 unsigned long end_bitidx, 472 unsigned long mask) 473 { 474 unsigned long *bitmap; 475 unsigned long bitidx, word_bitidx; 476 unsigned long old_word, word; 477 478 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 479 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits)); 480 481 bitmap = get_pageblock_bitmap(page, pfn); 482 bitidx = pfn_to_bitidx(page, pfn); 483 word_bitidx = bitidx / BITS_PER_LONG; 484 bitidx &= (BITS_PER_LONG-1); 485 486 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 487 488 bitidx += end_bitidx; 489 mask <<= (BITS_PER_LONG - bitidx - 1); 490 flags <<= (BITS_PER_LONG - bitidx - 1); 491 492 word = READ_ONCE(bitmap[word_bitidx]); 493 for (;;) { 494 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 495 if (word == old_word) 496 break; 497 word = old_word; 498 } 499 } 500 501 void set_pageblock_migratetype(struct page *page, int migratetype) 502 { 503 if (unlikely(page_group_by_mobility_disabled && 504 migratetype < MIGRATE_PCPTYPES)) 505 migratetype = MIGRATE_UNMOVABLE; 506 507 set_pageblock_flags_group(page, (unsigned long)migratetype, 508 PB_migrate, PB_migrate_end); 509 } 510 511 #ifdef CONFIG_DEBUG_VM 512 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 513 { 514 int ret = 0; 515 unsigned seq; 516 unsigned long pfn = page_to_pfn(page); 517 unsigned long sp, start_pfn; 518 519 do { 520 seq = zone_span_seqbegin(zone); 521 start_pfn = zone->zone_start_pfn; 522 sp = zone->spanned_pages; 523 if (!zone_spans_pfn(zone, pfn)) 524 ret = 1; 525 } while (zone_span_seqretry(zone, seq)); 526 527 if (ret) 528 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 529 pfn, zone_to_nid(zone), zone->name, 530 start_pfn, start_pfn + sp); 531 532 return ret; 533 } 534 535 static int page_is_consistent(struct zone *zone, struct page *page) 536 { 537 if (!pfn_valid_within(page_to_pfn(page))) 538 return 0; 539 if (zone != page_zone(page)) 540 return 0; 541 542 return 1; 543 } 544 /* 545 * Temporary debugging check for pages not lying within a given zone. 546 */ 547 static int __maybe_unused bad_range(struct zone *zone, struct page *page) 548 { 549 if (page_outside_zone_boundaries(zone, page)) 550 return 1; 551 if (!page_is_consistent(zone, page)) 552 return 1; 553 554 return 0; 555 } 556 #else 557 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page) 558 { 559 return 0; 560 } 561 #endif 562 563 static void bad_page(struct page *page, const char *reason, 564 unsigned long bad_flags) 565 { 566 static unsigned long resume; 567 static unsigned long nr_shown; 568 static unsigned long nr_unshown; 569 570 /* 571 * Allow a burst of 60 reports, then keep quiet for that minute; 572 * or allow a steady drip of one report per second. 573 */ 574 if (nr_shown == 60) { 575 if (time_before(jiffies, resume)) { 576 nr_unshown++; 577 goto out; 578 } 579 if (nr_unshown) { 580 pr_alert( 581 "BUG: Bad page state: %lu messages suppressed\n", 582 nr_unshown); 583 nr_unshown = 0; 584 } 585 nr_shown = 0; 586 } 587 if (nr_shown++ == 0) 588 resume = jiffies + 60 * HZ; 589 590 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 591 current->comm, page_to_pfn(page)); 592 __dump_page(page, reason); 593 bad_flags &= page->flags; 594 if (bad_flags) 595 pr_alert("bad because of flags: %#lx(%pGp)\n", 596 bad_flags, &bad_flags); 597 dump_page_owner(page); 598 599 print_modules(); 600 dump_stack(); 601 out: 602 /* Leave bad fields for debug, except PageBuddy could make trouble */ 603 page_mapcount_reset(page); /* remove PageBuddy */ 604 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 605 } 606 607 /* 608 * Higher-order pages are called "compound pages". They are structured thusly: 609 * 610 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 611 * 612 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 613 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 614 * 615 * The first tail page's ->compound_dtor holds the offset in array of compound 616 * page destructors. See compound_page_dtors. 617 * 618 * The first tail page's ->compound_order holds the order of allocation. 619 * This usage means that zero-order pages may not be compound. 620 */ 621 622 void free_compound_page(struct page *page) 623 { 624 __free_pages_ok(page, compound_order(page)); 625 } 626 627 void prep_compound_page(struct page *page, unsigned int order) 628 { 629 int i; 630 int nr_pages = 1 << order; 631 632 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 633 set_compound_order(page, order); 634 __SetPageHead(page); 635 for (i = 1; i < nr_pages; i++) { 636 struct page *p = page + i; 637 set_page_count(p, 0); 638 p->mapping = TAIL_MAPPING; 639 set_compound_head(p, page); 640 } 641 atomic_set(compound_mapcount_ptr(page), -1); 642 } 643 644 #ifdef CONFIG_DEBUG_PAGEALLOC 645 unsigned int _debug_guardpage_minorder; 646 bool _debug_pagealloc_enabled __read_mostly 647 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 648 EXPORT_SYMBOL(_debug_pagealloc_enabled); 649 bool _debug_guardpage_enabled __read_mostly; 650 651 static int __init early_debug_pagealloc(char *buf) 652 { 653 if (!buf) 654 return -EINVAL; 655 return kstrtobool(buf, &_debug_pagealloc_enabled); 656 } 657 early_param("debug_pagealloc", early_debug_pagealloc); 658 659 static bool need_debug_guardpage(void) 660 { 661 /* If we don't use debug_pagealloc, we don't need guard page */ 662 if (!debug_pagealloc_enabled()) 663 return false; 664 665 if (!debug_guardpage_minorder()) 666 return false; 667 668 return true; 669 } 670 671 static void init_debug_guardpage(void) 672 { 673 if (!debug_pagealloc_enabled()) 674 return; 675 676 if (!debug_guardpage_minorder()) 677 return; 678 679 _debug_guardpage_enabled = true; 680 } 681 682 struct page_ext_operations debug_guardpage_ops = { 683 .need = need_debug_guardpage, 684 .init = init_debug_guardpage, 685 }; 686 687 static int __init debug_guardpage_minorder_setup(char *buf) 688 { 689 unsigned long res; 690 691 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 692 pr_err("Bad debug_guardpage_minorder value\n"); 693 return 0; 694 } 695 _debug_guardpage_minorder = res; 696 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 697 return 0; 698 } 699 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); 700 701 static inline bool set_page_guard(struct zone *zone, struct page *page, 702 unsigned int order, int migratetype) 703 { 704 struct page_ext *page_ext; 705 706 if (!debug_guardpage_enabled()) 707 return false; 708 709 if (order >= debug_guardpage_minorder()) 710 return false; 711 712 page_ext = lookup_page_ext(page); 713 if (unlikely(!page_ext)) 714 return false; 715 716 __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); 717 718 INIT_LIST_HEAD(&page->lru); 719 set_page_private(page, order); 720 /* Guard pages are not available for any usage */ 721 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 722 723 return true; 724 } 725 726 static inline void clear_page_guard(struct zone *zone, struct page *page, 727 unsigned int order, int migratetype) 728 { 729 struct page_ext *page_ext; 730 731 if (!debug_guardpage_enabled()) 732 return; 733 734 page_ext = lookup_page_ext(page); 735 if (unlikely(!page_ext)) 736 return; 737 738 __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); 739 740 set_page_private(page, 0); 741 if (!is_migrate_isolate(migratetype)) 742 __mod_zone_freepage_state(zone, (1 << order), migratetype); 743 } 744 #else 745 struct page_ext_operations debug_guardpage_ops; 746 static inline bool set_page_guard(struct zone *zone, struct page *page, 747 unsigned int order, int migratetype) { return false; } 748 static inline void clear_page_guard(struct zone *zone, struct page *page, 749 unsigned int order, int migratetype) {} 750 #endif 751 752 static inline void set_page_order(struct page *page, unsigned int order) 753 { 754 set_page_private(page, order); 755 __SetPageBuddy(page); 756 } 757 758 static inline void rmv_page_order(struct page *page) 759 { 760 __ClearPageBuddy(page); 761 set_page_private(page, 0); 762 } 763 764 /* 765 * This function checks whether a page is free && is the buddy 766 * we can coalesce a page and its buddy if 767 * (a) the buddy is not in a hole (check before calling!) && 768 * (b) the buddy is in the buddy system && 769 * (c) a page and its buddy have the same order && 770 * (d) a page and its buddy are in the same zone. 771 * 772 * For recording whether a page is in the buddy system, we set PageBuddy. 773 * Setting, clearing, and testing PageBuddy is serialized by zone->lock. 774 * 775 * For recording page's order, we use page_private(page). 776 */ 777 static inline int page_is_buddy(struct page *page, struct page *buddy, 778 unsigned int order) 779 { 780 if (page_is_guard(buddy) && page_order(buddy) == order) { 781 if (page_zone_id(page) != page_zone_id(buddy)) 782 return 0; 783 784 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 785 786 return 1; 787 } 788 789 if (PageBuddy(buddy) && page_order(buddy) == order) { 790 /* 791 * zone check is done late to avoid uselessly 792 * calculating zone/node ids for pages that could 793 * never merge. 794 */ 795 if (page_zone_id(page) != page_zone_id(buddy)) 796 return 0; 797 798 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 799 800 return 1; 801 } 802 return 0; 803 } 804 805 #ifdef CONFIG_COMPACTION 806 static inline struct capture_control *task_capc(struct zone *zone) 807 { 808 struct capture_control *capc = current->capture_control; 809 810 return capc && 811 !(current->flags & PF_KTHREAD) && 812 !capc->page && 813 capc->cc->zone == zone && 814 capc->cc->direct_compaction ? capc : NULL; 815 } 816 817 static inline bool 818 compaction_capture(struct capture_control *capc, struct page *page, 819 int order, int migratetype) 820 { 821 if (!capc || order != capc->cc->order) 822 return false; 823 824 /* Do not accidentally pollute CMA or isolated regions*/ 825 if (is_migrate_cma(migratetype) || 826 is_migrate_isolate(migratetype)) 827 return false; 828 829 /* 830 * Do not let lower order allocations polluate a movable pageblock. 831 * This might let an unmovable request use a reclaimable pageblock 832 * and vice-versa but no more than normal fallback logic which can 833 * have trouble finding a high-order free page. 834 */ 835 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE) 836 return false; 837 838 capc->page = page; 839 return true; 840 } 841 842 #else 843 static inline struct capture_control *task_capc(struct zone *zone) 844 { 845 return NULL; 846 } 847 848 static inline bool 849 compaction_capture(struct capture_control *capc, struct page *page, 850 int order, int migratetype) 851 { 852 return false; 853 } 854 #endif /* CONFIG_COMPACTION */ 855 856 /* 857 * Freeing function for a buddy system allocator. 858 * 859 * The concept of a buddy system is to maintain direct-mapped table 860 * (containing bit values) for memory blocks of various "orders". 861 * The bottom level table contains the map for the smallest allocatable 862 * units of memory (here, pages), and each level above it describes 863 * pairs of units from the levels below, hence, "buddies". 864 * At a high level, all that happens here is marking the table entry 865 * at the bottom level available, and propagating the changes upward 866 * as necessary, plus some accounting needed to play nicely with other 867 * parts of the VM system. 868 * At each level, we keep a list of pages, which are heads of continuous 869 * free pages of length of (1 << order) and marked with PageBuddy. 870 * Page's order is recorded in page_private(page) field. 871 * So when we are allocating or freeing one, we can derive the state of the 872 * other. That is, if we allocate a small block, and both were 873 * free, the remainder of the region must be split into blocks. 874 * If a block is freed, and its buddy is also free, then this 875 * triggers coalescing into a block of larger size. 876 * 877 * -- nyc 878 */ 879 880 static inline void __free_one_page(struct page *page, 881 unsigned long pfn, 882 struct zone *zone, unsigned int order, 883 int migratetype) 884 { 885 unsigned long combined_pfn; 886 unsigned long uninitialized_var(buddy_pfn); 887 struct page *buddy; 888 unsigned int max_order; 889 struct capture_control *capc = task_capc(zone); 890 891 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1); 892 893 VM_BUG_ON(!zone_is_initialized(zone)); 894 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 895 896 VM_BUG_ON(migratetype == -1); 897 if (likely(!is_migrate_isolate(migratetype))) 898 __mod_zone_freepage_state(zone, 1 << order, migratetype); 899 900 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); 901 VM_BUG_ON_PAGE(bad_range(zone, page), page); 902 903 continue_merging: 904 while (order < max_order - 1) { 905 if (compaction_capture(capc, page, order, migratetype)) { 906 __mod_zone_freepage_state(zone, -(1 << order), 907 migratetype); 908 return; 909 } 910 buddy_pfn = __find_buddy_pfn(pfn, order); 911 buddy = page + (buddy_pfn - pfn); 912 913 if (!pfn_valid_within(buddy_pfn)) 914 goto done_merging; 915 if (!page_is_buddy(page, buddy, order)) 916 goto done_merging; 917 /* 918 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 919 * merge with it and move up one order. 920 */ 921 if (page_is_guard(buddy)) { 922 clear_page_guard(zone, buddy, order, migratetype); 923 } else { 924 list_del(&buddy->lru); 925 zone->free_area[order].nr_free--; 926 rmv_page_order(buddy); 927 } 928 combined_pfn = buddy_pfn & pfn; 929 page = page + (combined_pfn - pfn); 930 pfn = combined_pfn; 931 order++; 932 } 933 if (max_order < MAX_ORDER) { 934 /* If we are here, it means order is >= pageblock_order. 935 * We want to prevent merge between freepages on isolate 936 * pageblock and normal pageblock. Without this, pageblock 937 * isolation could cause incorrect freepage or CMA accounting. 938 * 939 * We don't want to hit this code for the more frequent 940 * low-order merging. 941 */ 942 if (unlikely(has_isolate_pageblock(zone))) { 943 int buddy_mt; 944 945 buddy_pfn = __find_buddy_pfn(pfn, order); 946 buddy = page + (buddy_pfn - pfn); 947 buddy_mt = get_pageblock_migratetype(buddy); 948 949 if (migratetype != buddy_mt 950 && (is_migrate_isolate(migratetype) || 951 is_migrate_isolate(buddy_mt))) 952 goto done_merging; 953 } 954 max_order++; 955 goto continue_merging; 956 } 957 958 done_merging: 959 set_page_order(page, order); 960 961 /* 962 * If this is not the largest possible page, check if the buddy 963 * of the next-highest order is free. If it is, it's possible 964 * that pages are being freed that will coalesce soon. In case, 965 * that is happening, add the free page to the tail of the list 966 * so it's less likely to be used soon and more likely to be merged 967 * as a higher order page 968 */ 969 if ((order < MAX_ORDER-2) && pfn_valid_within(buddy_pfn)) { 970 struct page *higher_page, *higher_buddy; 971 combined_pfn = buddy_pfn & pfn; 972 higher_page = page + (combined_pfn - pfn); 973 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1); 974 higher_buddy = higher_page + (buddy_pfn - combined_pfn); 975 if (pfn_valid_within(buddy_pfn) && 976 page_is_buddy(higher_page, higher_buddy, order + 1)) { 977 list_add_tail(&page->lru, 978 &zone->free_area[order].free_list[migratetype]); 979 goto out; 980 } 981 } 982 983 list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); 984 out: 985 zone->free_area[order].nr_free++; 986 } 987 988 /* 989 * A bad page could be due to a number of fields. Instead of multiple branches, 990 * try and check multiple fields with one check. The caller must do a detailed 991 * check if necessary. 992 */ 993 static inline bool page_expected_state(struct page *page, 994 unsigned long check_flags) 995 { 996 if (unlikely(atomic_read(&page->_mapcount) != -1)) 997 return false; 998 999 if (unlikely((unsigned long)page->mapping | 1000 page_ref_count(page) | 1001 #ifdef CONFIG_MEMCG 1002 (unsigned long)page->mem_cgroup | 1003 #endif 1004 (page->flags & check_flags))) 1005 return false; 1006 1007 return true; 1008 } 1009 1010 static void free_pages_check_bad(struct page *page) 1011 { 1012 const char *bad_reason; 1013 unsigned long bad_flags; 1014 1015 bad_reason = NULL; 1016 bad_flags = 0; 1017 1018 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1019 bad_reason = "nonzero mapcount"; 1020 if (unlikely(page->mapping != NULL)) 1021 bad_reason = "non-NULL mapping"; 1022 if (unlikely(page_ref_count(page) != 0)) 1023 bad_reason = "nonzero _refcount"; 1024 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) { 1025 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 1026 bad_flags = PAGE_FLAGS_CHECK_AT_FREE; 1027 } 1028 #ifdef CONFIG_MEMCG 1029 if (unlikely(page->mem_cgroup)) 1030 bad_reason = "page still charged to cgroup"; 1031 #endif 1032 bad_page(page, bad_reason, bad_flags); 1033 } 1034 1035 static inline int free_pages_check(struct page *page) 1036 { 1037 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 1038 return 0; 1039 1040 /* Something has gone sideways, find it */ 1041 free_pages_check_bad(page); 1042 return 1; 1043 } 1044 1045 static int free_tail_pages_check(struct page *head_page, struct page *page) 1046 { 1047 int ret = 1; 1048 1049 /* 1050 * We rely page->lru.next never has bit 0 set, unless the page 1051 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 1052 */ 1053 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 1054 1055 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 1056 ret = 0; 1057 goto out; 1058 } 1059 switch (page - head_page) { 1060 case 1: 1061 /* the first tail page: ->mapping may be compound_mapcount() */ 1062 if (unlikely(compound_mapcount(page))) { 1063 bad_page(page, "nonzero compound_mapcount", 0); 1064 goto out; 1065 } 1066 break; 1067 case 2: 1068 /* 1069 * the second tail page: ->mapping is 1070 * deferred_list.next -- ignore value. 1071 */ 1072 break; 1073 default: 1074 if (page->mapping != TAIL_MAPPING) { 1075 bad_page(page, "corrupted mapping in tail page", 0); 1076 goto out; 1077 } 1078 break; 1079 } 1080 if (unlikely(!PageTail(page))) { 1081 bad_page(page, "PageTail not set", 0); 1082 goto out; 1083 } 1084 if (unlikely(compound_head(page) != head_page)) { 1085 bad_page(page, "compound_head not consistent", 0); 1086 goto out; 1087 } 1088 ret = 0; 1089 out: 1090 page->mapping = NULL; 1091 clear_compound_head(page); 1092 return ret; 1093 } 1094 1095 static __always_inline bool free_pages_prepare(struct page *page, 1096 unsigned int order, bool check_free) 1097 { 1098 int bad = 0; 1099 1100 VM_BUG_ON_PAGE(PageTail(page), page); 1101 1102 trace_mm_page_free(page, order); 1103 1104 /* 1105 * Check tail pages before head page information is cleared to 1106 * avoid checking PageCompound for order-0 pages. 1107 */ 1108 if (unlikely(order)) { 1109 bool compound = PageCompound(page); 1110 int i; 1111 1112 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1113 1114 if (compound) 1115 ClearPageDoubleMap(page); 1116 for (i = 1; i < (1 << order); i++) { 1117 if (compound) 1118 bad += free_tail_pages_check(page, page + i); 1119 if (unlikely(free_pages_check(page + i))) { 1120 bad++; 1121 continue; 1122 } 1123 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1124 } 1125 } 1126 if (PageMappingFlags(page)) 1127 page->mapping = NULL; 1128 if (memcg_kmem_enabled() && PageKmemcg(page)) 1129 __memcg_kmem_uncharge(page, order); 1130 if (check_free) 1131 bad += free_pages_check(page); 1132 if (bad) 1133 return false; 1134 1135 page_cpupid_reset_last(page); 1136 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1137 reset_page_owner(page, order); 1138 1139 if (!PageHighMem(page)) { 1140 debug_check_no_locks_freed(page_address(page), 1141 PAGE_SIZE << order); 1142 debug_check_no_obj_freed(page_address(page), 1143 PAGE_SIZE << order); 1144 } 1145 arch_free_page(page, order); 1146 kernel_poison_pages(page, 1 << order, 0); 1147 kernel_map_pages(page, 1 << order, 0); 1148 kasan_free_nondeferred_pages(page, order); 1149 1150 return true; 1151 } 1152 1153 #ifdef CONFIG_DEBUG_VM 1154 static inline bool free_pcp_prepare(struct page *page) 1155 { 1156 return free_pages_prepare(page, 0, true); 1157 } 1158 1159 static inline bool bulkfree_pcp_prepare(struct page *page) 1160 { 1161 return false; 1162 } 1163 #else 1164 static bool free_pcp_prepare(struct page *page) 1165 { 1166 return free_pages_prepare(page, 0, false); 1167 } 1168 1169 static bool bulkfree_pcp_prepare(struct page *page) 1170 { 1171 return free_pages_check(page); 1172 } 1173 #endif /* CONFIG_DEBUG_VM */ 1174 1175 static inline void prefetch_buddy(struct page *page) 1176 { 1177 unsigned long pfn = page_to_pfn(page); 1178 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0); 1179 struct page *buddy = page + (buddy_pfn - pfn); 1180 1181 prefetch(buddy); 1182 } 1183 1184 /* 1185 * Frees a number of pages from the PCP lists 1186 * Assumes all pages on list are in same zone, and of same order. 1187 * count is the number of pages to free. 1188 * 1189 * If the zone was previously in an "all pages pinned" state then look to 1190 * see if this freeing clears that state. 1191 * 1192 * And clear the zone's pages_scanned counter, to hold off the "all pages are 1193 * pinned" detection logic. 1194 */ 1195 static void free_pcppages_bulk(struct zone *zone, int count, 1196 struct per_cpu_pages *pcp) 1197 { 1198 int migratetype = 0; 1199 int batch_free = 0; 1200 int prefetch_nr = 0; 1201 bool isolated_pageblocks; 1202 struct page *page, *tmp; 1203 LIST_HEAD(head); 1204 1205 while (count) { 1206 struct list_head *list; 1207 1208 /* 1209 * Remove pages from lists in a round-robin fashion. A 1210 * batch_free count is maintained that is incremented when an 1211 * empty list is encountered. This is so more pages are freed 1212 * off fuller lists instead of spinning excessively around empty 1213 * lists 1214 */ 1215 do { 1216 batch_free++; 1217 if (++migratetype == MIGRATE_PCPTYPES) 1218 migratetype = 0; 1219 list = &pcp->lists[migratetype]; 1220 } while (list_empty(list)); 1221 1222 /* This is the only non-empty list. Free them all. */ 1223 if (batch_free == MIGRATE_PCPTYPES) 1224 batch_free = count; 1225 1226 do { 1227 page = list_last_entry(list, struct page, lru); 1228 /* must delete to avoid corrupting pcp list */ 1229 list_del(&page->lru); 1230 pcp->count--; 1231 1232 if (bulkfree_pcp_prepare(page)) 1233 continue; 1234 1235 list_add_tail(&page->lru, &head); 1236 1237 /* 1238 * We are going to put the page back to the global 1239 * pool, prefetch its buddy to speed up later access 1240 * under zone->lock. It is believed the overhead of 1241 * an additional test and calculating buddy_pfn here 1242 * can be offset by reduced memory latency later. To 1243 * avoid excessive prefetching due to large count, only 1244 * prefetch buddy for the first pcp->batch nr of pages. 1245 */ 1246 if (prefetch_nr++ < pcp->batch) 1247 prefetch_buddy(page); 1248 } while (--count && --batch_free && !list_empty(list)); 1249 } 1250 1251 spin_lock(&zone->lock); 1252 isolated_pageblocks = has_isolate_pageblock(zone); 1253 1254 /* 1255 * Use safe version since after __free_one_page(), 1256 * page->lru.next will not point to original list. 1257 */ 1258 list_for_each_entry_safe(page, tmp, &head, lru) { 1259 int mt = get_pcppage_migratetype(page); 1260 /* MIGRATE_ISOLATE page should not go to pcplists */ 1261 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1262 /* Pageblock could have been isolated meanwhile */ 1263 if (unlikely(isolated_pageblocks)) 1264 mt = get_pageblock_migratetype(page); 1265 1266 __free_one_page(page, page_to_pfn(page), zone, 0, mt); 1267 trace_mm_page_pcpu_drain(page, 0, mt); 1268 } 1269 spin_unlock(&zone->lock); 1270 } 1271 1272 static void free_one_page(struct zone *zone, 1273 struct page *page, unsigned long pfn, 1274 unsigned int order, 1275 int migratetype) 1276 { 1277 spin_lock(&zone->lock); 1278 if (unlikely(has_isolate_pageblock(zone) || 1279 is_migrate_isolate(migratetype))) { 1280 migratetype = get_pfnblock_migratetype(page, pfn); 1281 } 1282 __free_one_page(page, pfn, zone, order, migratetype); 1283 spin_unlock(&zone->lock); 1284 } 1285 1286 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1287 unsigned long zone, int nid) 1288 { 1289 mm_zero_struct_page(page); 1290 set_page_links(page, zone, nid, pfn); 1291 init_page_count(page); 1292 page_mapcount_reset(page); 1293 page_cpupid_reset_last(page); 1294 page_kasan_tag_reset(page); 1295 1296 INIT_LIST_HEAD(&page->lru); 1297 #ifdef WANT_PAGE_VIRTUAL 1298 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1299 if (!is_highmem_idx(zone)) 1300 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1301 #endif 1302 } 1303 1304 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1305 static void __meminit init_reserved_page(unsigned long pfn) 1306 { 1307 pg_data_t *pgdat; 1308 int nid, zid; 1309 1310 if (!early_page_uninitialised(pfn)) 1311 return; 1312 1313 nid = early_pfn_to_nid(pfn); 1314 pgdat = NODE_DATA(nid); 1315 1316 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1317 struct zone *zone = &pgdat->node_zones[zid]; 1318 1319 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) 1320 break; 1321 } 1322 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 1323 } 1324 #else 1325 static inline void init_reserved_page(unsigned long pfn) 1326 { 1327 } 1328 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1329 1330 /* 1331 * Initialised pages do not have PageReserved set. This function is 1332 * called for each range allocated by the bootmem allocator and 1333 * marks the pages PageReserved. The remaining valid pages are later 1334 * sent to the buddy page allocator. 1335 */ 1336 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1337 { 1338 unsigned long start_pfn = PFN_DOWN(start); 1339 unsigned long end_pfn = PFN_UP(end); 1340 1341 for (; start_pfn < end_pfn; start_pfn++) { 1342 if (pfn_valid(start_pfn)) { 1343 struct page *page = pfn_to_page(start_pfn); 1344 1345 init_reserved_page(start_pfn); 1346 1347 /* Avoid false-positive PageTail() */ 1348 INIT_LIST_HEAD(&page->lru); 1349 1350 /* 1351 * no need for atomic set_bit because the struct 1352 * page is not visible yet so nobody should 1353 * access it yet. 1354 */ 1355 __SetPageReserved(page); 1356 } 1357 } 1358 } 1359 1360 static void __free_pages_ok(struct page *page, unsigned int order) 1361 { 1362 unsigned long flags; 1363 int migratetype; 1364 unsigned long pfn = page_to_pfn(page); 1365 1366 if (!free_pages_prepare(page, order, true)) 1367 return; 1368 1369 migratetype = get_pfnblock_migratetype(page, pfn); 1370 local_irq_save(flags); 1371 __count_vm_events(PGFREE, 1 << order); 1372 free_one_page(page_zone(page), page, pfn, order, migratetype); 1373 local_irq_restore(flags); 1374 } 1375 1376 void __free_pages_core(struct page *page, unsigned int order) 1377 { 1378 unsigned int nr_pages = 1 << order; 1379 struct page *p = page; 1380 unsigned int loop; 1381 1382 prefetchw(p); 1383 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1384 prefetchw(p + 1); 1385 __ClearPageReserved(p); 1386 set_page_count(p, 0); 1387 } 1388 __ClearPageReserved(p); 1389 set_page_count(p, 0); 1390 1391 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); 1392 set_page_refcounted(page); 1393 __free_pages(page, order); 1394 } 1395 1396 #if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \ 1397 defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) 1398 1399 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1400 1401 int __meminit early_pfn_to_nid(unsigned long pfn) 1402 { 1403 static DEFINE_SPINLOCK(early_pfn_lock); 1404 int nid; 1405 1406 spin_lock(&early_pfn_lock); 1407 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1408 if (nid < 0) 1409 nid = first_online_node; 1410 spin_unlock(&early_pfn_lock); 1411 1412 return nid; 1413 } 1414 #endif 1415 1416 #ifdef CONFIG_NODES_SPAN_OTHER_NODES 1417 static inline bool __meminit __maybe_unused 1418 meminit_pfn_in_nid(unsigned long pfn, int node, 1419 struct mminit_pfnnid_cache *state) 1420 { 1421 int nid; 1422 1423 nid = __early_pfn_to_nid(pfn, state); 1424 if (nid >= 0 && nid != node) 1425 return false; 1426 return true; 1427 } 1428 1429 /* Only safe to use early in boot when initialisation is single-threaded */ 1430 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1431 { 1432 return meminit_pfn_in_nid(pfn, node, &early_pfnnid_cache); 1433 } 1434 1435 #else 1436 1437 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1438 { 1439 return true; 1440 } 1441 static inline bool __meminit __maybe_unused 1442 meminit_pfn_in_nid(unsigned long pfn, int node, 1443 struct mminit_pfnnid_cache *state) 1444 { 1445 return true; 1446 } 1447 #endif 1448 1449 1450 void __init memblock_free_pages(struct page *page, unsigned long pfn, 1451 unsigned int order) 1452 { 1453 if (early_page_uninitialised(pfn)) 1454 return; 1455 __free_pages_core(page, order); 1456 } 1457 1458 /* 1459 * Check that the whole (or subset of) a pageblock given by the interval of 1460 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1461 * with the migration of free compaction scanner. The scanners then need to 1462 * use only pfn_valid_within() check for arches that allow holes within 1463 * pageblocks. 1464 * 1465 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1466 * 1467 * It's possible on some configurations to have a setup like node0 node1 node0 1468 * i.e. it's possible that all pages within a zones range of pages do not 1469 * belong to a single zone. We assume that a border between node0 and node1 1470 * can occur within a single pageblock, but not a node0 node1 node0 1471 * interleaving within a single pageblock. It is therefore sufficient to check 1472 * the first and last page of a pageblock and avoid checking each individual 1473 * page in a pageblock. 1474 */ 1475 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1476 unsigned long end_pfn, struct zone *zone) 1477 { 1478 struct page *start_page; 1479 struct page *end_page; 1480 1481 /* end_pfn is one past the range we are checking */ 1482 end_pfn--; 1483 1484 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1485 return NULL; 1486 1487 start_page = pfn_to_online_page(start_pfn); 1488 if (!start_page) 1489 return NULL; 1490 1491 if (page_zone(start_page) != zone) 1492 return NULL; 1493 1494 end_page = pfn_to_page(end_pfn); 1495 1496 /* This gives a shorter code than deriving page_zone(end_page) */ 1497 if (page_zone_id(start_page) != page_zone_id(end_page)) 1498 return NULL; 1499 1500 return start_page; 1501 } 1502 1503 void set_zone_contiguous(struct zone *zone) 1504 { 1505 unsigned long block_start_pfn = zone->zone_start_pfn; 1506 unsigned long block_end_pfn; 1507 1508 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1509 for (; block_start_pfn < zone_end_pfn(zone); 1510 block_start_pfn = block_end_pfn, 1511 block_end_pfn += pageblock_nr_pages) { 1512 1513 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1514 1515 if (!__pageblock_pfn_to_page(block_start_pfn, 1516 block_end_pfn, zone)) 1517 return; 1518 } 1519 1520 /* We confirm that there is no hole */ 1521 zone->contiguous = true; 1522 } 1523 1524 void clear_zone_contiguous(struct zone *zone) 1525 { 1526 zone->contiguous = false; 1527 } 1528 1529 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1530 static void __init deferred_free_range(unsigned long pfn, 1531 unsigned long nr_pages) 1532 { 1533 struct page *page; 1534 unsigned long i; 1535 1536 if (!nr_pages) 1537 return; 1538 1539 page = pfn_to_page(pfn); 1540 1541 /* Free a large naturally-aligned chunk if possible */ 1542 if (nr_pages == pageblock_nr_pages && 1543 (pfn & (pageblock_nr_pages - 1)) == 0) { 1544 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1545 __free_pages_core(page, pageblock_order); 1546 return; 1547 } 1548 1549 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1550 if ((pfn & (pageblock_nr_pages - 1)) == 0) 1551 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1552 __free_pages_core(page, 0); 1553 } 1554 } 1555 1556 /* Completion tracking for deferred_init_memmap() threads */ 1557 static atomic_t pgdat_init_n_undone __initdata; 1558 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1559 1560 static inline void __init pgdat_init_report_one_done(void) 1561 { 1562 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1563 complete(&pgdat_init_all_done_comp); 1564 } 1565 1566 /* 1567 * Returns true if page needs to be initialized or freed to buddy allocator. 1568 * 1569 * First we check if pfn is valid on architectures where it is possible to have 1570 * holes within pageblock_nr_pages. On systems where it is not possible, this 1571 * function is optimized out. 1572 * 1573 * Then, we check if a current large page is valid by only checking the validity 1574 * of the head pfn. 1575 * 1576 * Finally, meminit_pfn_in_nid is checked on systems where pfns can interleave 1577 * within a node: a pfn is between start and end of a node, but does not belong 1578 * to this memory node. 1579 */ 1580 static inline bool __init 1581 deferred_pfn_valid(int nid, unsigned long pfn, 1582 struct mminit_pfnnid_cache *nid_init_state) 1583 { 1584 if (!pfn_valid_within(pfn)) 1585 return false; 1586 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn)) 1587 return false; 1588 if (!meminit_pfn_in_nid(pfn, nid, nid_init_state)) 1589 return false; 1590 return true; 1591 } 1592 1593 /* 1594 * Free pages to buddy allocator. Try to free aligned pages in 1595 * pageblock_nr_pages sizes. 1596 */ 1597 static void __init deferred_free_pages(int nid, int zid, unsigned long pfn, 1598 unsigned long end_pfn) 1599 { 1600 struct mminit_pfnnid_cache nid_init_state = { }; 1601 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1602 unsigned long nr_free = 0; 1603 1604 for (; pfn < end_pfn; pfn++) { 1605 if (!deferred_pfn_valid(nid, pfn, &nid_init_state)) { 1606 deferred_free_range(pfn - nr_free, nr_free); 1607 nr_free = 0; 1608 } else if (!(pfn & nr_pgmask)) { 1609 deferred_free_range(pfn - nr_free, nr_free); 1610 nr_free = 1; 1611 touch_nmi_watchdog(); 1612 } else { 1613 nr_free++; 1614 } 1615 } 1616 /* Free the last block of pages to allocator */ 1617 deferred_free_range(pfn - nr_free, nr_free); 1618 } 1619 1620 /* 1621 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 1622 * by performing it only once every pageblock_nr_pages. 1623 * Return number of pages initialized. 1624 */ 1625 static unsigned long __init deferred_init_pages(int nid, int zid, 1626 unsigned long pfn, 1627 unsigned long end_pfn) 1628 { 1629 struct mminit_pfnnid_cache nid_init_state = { }; 1630 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1631 unsigned long nr_pages = 0; 1632 struct page *page = NULL; 1633 1634 for (; pfn < end_pfn; pfn++) { 1635 if (!deferred_pfn_valid(nid, pfn, &nid_init_state)) { 1636 page = NULL; 1637 continue; 1638 } else if (!page || !(pfn & nr_pgmask)) { 1639 page = pfn_to_page(pfn); 1640 touch_nmi_watchdog(); 1641 } else { 1642 page++; 1643 } 1644 __init_single_page(page, pfn, zid, nid); 1645 nr_pages++; 1646 } 1647 return (nr_pages); 1648 } 1649 1650 /* Initialise remaining memory on a node */ 1651 static int __init deferred_init_memmap(void *data) 1652 { 1653 pg_data_t *pgdat = data; 1654 int nid = pgdat->node_id; 1655 unsigned long start = jiffies; 1656 unsigned long nr_pages = 0; 1657 unsigned long spfn, epfn, first_init_pfn, flags; 1658 phys_addr_t spa, epa; 1659 int zid; 1660 struct zone *zone; 1661 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 1662 u64 i; 1663 1664 /* Bind memory initialisation thread to a local node if possible */ 1665 if (!cpumask_empty(cpumask)) 1666 set_cpus_allowed_ptr(current, cpumask); 1667 1668 pgdat_resize_lock(pgdat, &flags); 1669 first_init_pfn = pgdat->first_deferred_pfn; 1670 if (first_init_pfn == ULONG_MAX) { 1671 pgdat_resize_unlock(pgdat, &flags); 1672 pgdat_init_report_one_done(); 1673 return 0; 1674 } 1675 1676 /* Sanity check boundaries */ 1677 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 1678 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 1679 pgdat->first_deferred_pfn = ULONG_MAX; 1680 1681 /* Only the highest zone is deferred so find it */ 1682 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1683 zone = pgdat->node_zones + zid; 1684 if (first_init_pfn < zone_end_pfn(zone)) 1685 break; 1686 } 1687 first_init_pfn = max(zone->zone_start_pfn, first_init_pfn); 1688 1689 /* 1690 * Initialize and free pages. We do it in two loops: first we initialize 1691 * struct page, than free to buddy allocator, because while we are 1692 * freeing pages we can access pages that are ahead (computing buddy 1693 * page in __free_one_page()). 1694 */ 1695 for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) { 1696 spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa)); 1697 epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa)); 1698 nr_pages += deferred_init_pages(nid, zid, spfn, epfn); 1699 } 1700 for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) { 1701 spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa)); 1702 epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa)); 1703 deferred_free_pages(nid, zid, spfn, epfn); 1704 } 1705 pgdat_resize_unlock(pgdat, &flags); 1706 1707 /* Sanity check that the next zone really is unpopulated */ 1708 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 1709 1710 pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages, 1711 jiffies_to_msecs(jiffies - start)); 1712 1713 pgdat_init_report_one_done(); 1714 return 0; 1715 } 1716 1717 /* 1718 * If this zone has deferred pages, try to grow it by initializing enough 1719 * deferred pages to satisfy the allocation specified by order, rounded up to 1720 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 1721 * of SECTION_SIZE bytes by initializing struct pages in increments of 1722 * PAGES_PER_SECTION * sizeof(struct page) bytes. 1723 * 1724 * Return true when zone was grown, otherwise return false. We return true even 1725 * when we grow less than requested, to let the caller decide if there are 1726 * enough pages to satisfy the allocation. 1727 * 1728 * Note: We use noinline because this function is needed only during boot, and 1729 * it is called from a __ref function _deferred_grow_zone. This way we are 1730 * making sure that it is not inlined into permanent text section. 1731 */ 1732 static noinline bool __init 1733 deferred_grow_zone(struct zone *zone, unsigned int order) 1734 { 1735 int zid = zone_idx(zone); 1736 int nid = zone_to_nid(zone); 1737 pg_data_t *pgdat = NODE_DATA(nid); 1738 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 1739 unsigned long nr_pages = 0; 1740 unsigned long first_init_pfn, spfn, epfn, t, flags; 1741 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 1742 phys_addr_t spa, epa; 1743 u64 i; 1744 1745 /* Only the last zone may have deferred pages */ 1746 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 1747 return false; 1748 1749 pgdat_resize_lock(pgdat, &flags); 1750 1751 /* 1752 * If deferred pages have been initialized while we were waiting for 1753 * the lock, return true, as the zone was grown. The caller will retry 1754 * this zone. We won't return to this function since the caller also 1755 * has this static branch. 1756 */ 1757 if (!static_branch_unlikely(&deferred_pages)) { 1758 pgdat_resize_unlock(pgdat, &flags); 1759 return true; 1760 } 1761 1762 /* 1763 * If someone grew this zone while we were waiting for spinlock, return 1764 * true, as there might be enough pages already. 1765 */ 1766 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 1767 pgdat_resize_unlock(pgdat, &flags); 1768 return true; 1769 } 1770 1771 first_init_pfn = max(zone->zone_start_pfn, first_deferred_pfn); 1772 1773 if (first_init_pfn >= pgdat_end_pfn(pgdat)) { 1774 pgdat_resize_unlock(pgdat, &flags); 1775 return false; 1776 } 1777 1778 for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) { 1779 spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa)); 1780 epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa)); 1781 1782 while (spfn < epfn && nr_pages < nr_pages_needed) { 1783 t = ALIGN(spfn + PAGES_PER_SECTION, PAGES_PER_SECTION); 1784 first_deferred_pfn = min(t, epfn); 1785 nr_pages += deferred_init_pages(nid, zid, spfn, 1786 first_deferred_pfn); 1787 spfn = first_deferred_pfn; 1788 } 1789 1790 if (nr_pages >= nr_pages_needed) 1791 break; 1792 } 1793 1794 for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) { 1795 spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa)); 1796 epfn = min_t(unsigned long, first_deferred_pfn, PFN_DOWN(epa)); 1797 deferred_free_pages(nid, zid, spfn, epfn); 1798 1799 if (first_deferred_pfn == epfn) 1800 break; 1801 } 1802 pgdat->first_deferred_pfn = first_deferred_pfn; 1803 pgdat_resize_unlock(pgdat, &flags); 1804 1805 return nr_pages > 0; 1806 } 1807 1808 /* 1809 * deferred_grow_zone() is __init, but it is called from 1810 * get_page_from_freelist() during early boot until deferred_pages permanently 1811 * disables this call. This is why we have refdata wrapper to avoid warning, 1812 * and to ensure that the function body gets unloaded. 1813 */ 1814 static bool __ref 1815 _deferred_grow_zone(struct zone *zone, unsigned int order) 1816 { 1817 return deferred_grow_zone(zone, order); 1818 } 1819 1820 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1821 1822 void __init page_alloc_init_late(void) 1823 { 1824 struct zone *zone; 1825 1826 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1827 int nid; 1828 1829 /* There will be num_node_state(N_MEMORY) threads */ 1830 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 1831 for_each_node_state(nid, N_MEMORY) { 1832 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 1833 } 1834 1835 /* Block until all are initialised */ 1836 wait_for_completion(&pgdat_init_all_done_comp); 1837 1838 /* 1839 * We initialized the rest of the deferred pages. Permanently disable 1840 * on-demand struct page initialization. 1841 */ 1842 static_branch_disable(&deferred_pages); 1843 1844 /* Reinit limits that are based on free pages after the kernel is up */ 1845 files_maxfiles_init(); 1846 #endif 1847 #ifdef CONFIG_ARCH_DISCARD_MEMBLOCK 1848 /* Discard memblock private memory */ 1849 memblock_discard(); 1850 #endif 1851 1852 for_each_populated_zone(zone) 1853 set_zone_contiguous(zone); 1854 } 1855 1856 #ifdef CONFIG_CMA 1857 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 1858 void __init init_cma_reserved_pageblock(struct page *page) 1859 { 1860 unsigned i = pageblock_nr_pages; 1861 struct page *p = page; 1862 1863 do { 1864 __ClearPageReserved(p); 1865 set_page_count(p, 0); 1866 } while (++p, --i); 1867 1868 set_pageblock_migratetype(page, MIGRATE_CMA); 1869 1870 if (pageblock_order >= MAX_ORDER) { 1871 i = pageblock_nr_pages; 1872 p = page; 1873 do { 1874 set_page_refcounted(p); 1875 __free_pages(p, MAX_ORDER - 1); 1876 p += MAX_ORDER_NR_PAGES; 1877 } while (i -= MAX_ORDER_NR_PAGES); 1878 } else { 1879 set_page_refcounted(page); 1880 __free_pages(page, pageblock_order); 1881 } 1882 1883 adjust_managed_page_count(page, pageblock_nr_pages); 1884 } 1885 #endif 1886 1887 /* 1888 * The order of subdivision here is critical for the IO subsystem. 1889 * Please do not alter this order without good reasons and regression 1890 * testing. Specifically, as large blocks of memory are subdivided, 1891 * the order in which smaller blocks are delivered depends on the order 1892 * they're subdivided in this function. This is the primary factor 1893 * influencing the order in which pages are delivered to the IO 1894 * subsystem according to empirical testing, and this is also justified 1895 * by considering the behavior of a buddy system containing a single 1896 * large block of memory acted on by a series of small allocations. 1897 * This behavior is a critical factor in sglist merging's success. 1898 * 1899 * -- nyc 1900 */ 1901 static inline void expand(struct zone *zone, struct page *page, 1902 int low, int high, struct free_area *area, 1903 int migratetype) 1904 { 1905 unsigned long size = 1 << high; 1906 1907 while (high > low) { 1908 area--; 1909 high--; 1910 size >>= 1; 1911 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 1912 1913 /* 1914 * Mark as guard pages (or page), that will allow to 1915 * merge back to allocator when buddy will be freed. 1916 * Corresponding page table entries will not be touched, 1917 * pages will stay not present in virtual address space 1918 */ 1919 if (set_page_guard(zone, &page[size], high, migratetype)) 1920 continue; 1921 1922 list_add(&page[size].lru, &area->free_list[migratetype]); 1923 area->nr_free++; 1924 set_page_order(&page[size], high); 1925 } 1926 } 1927 1928 static void check_new_page_bad(struct page *page) 1929 { 1930 const char *bad_reason = NULL; 1931 unsigned long bad_flags = 0; 1932 1933 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1934 bad_reason = "nonzero mapcount"; 1935 if (unlikely(page->mapping != NULL)) 1936 bad_reason = "non-NULL mapping"; 1937 if (unlikely(page_ref_count(page) != 0)) 1938 bad_reason = "nonzero _count"; 1939 if (unlikely(page->flags & __PG_HWPOISON)) { 1940 bad_reason = "HWPoisoned (hardware-corrupted)"; 1941 bad_flags = __PG_HWPOISON; 1942 /* Don't complain about hwpoisoned pages */ 1943 page_mapcount_reset(page); /* remove PageBuddy */ 1944 return; 1945 } 1946 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) { 1947 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set"; 1948 bad_flags = PAGE_FLAGS_CHECK_AT_PREP; 1949 } 1950 #ifdef CONFIG_MEMCG 1951 if (unlikely(page->mem_cgroup)) 1952 bad_reason = "page still charged to cgroup"; 1953 #endif 1954 bad_page(page, bad_reason, bad_flags); 1955 } 1956 1957 /* 1958 * This page is about to be returned from the page allocator 1959 */ 1960 static inline int check_new_page(struct page *page) 1961 { 1962 if (likely(page_expected_state(page, 1963 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 1964 return 0; 1965 1966 check_new_page_bad(page); 1967 return 1; 1968 } 1969 1970 static inline bool free_pages_prezeroed(void) 1971 { 1972 return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) && 1973 page_poisoning_enabled(); 1974 } 1975 1976 #ifdef CONFIG_DEBUG_VM 1977 static bool check_pcp_refill(struct page *page) 1978 { 1979 return false; 1980 } 1981 1982 static bool check_new_pcp(struct page *page) 1983 { 1984 return check_new_page(page); 1985 } 1986 #else 1987 static bool check_pcp_refill(struct page *page) 1988 { 1989 return check_new_page(page); 1990 } 1991 static bool check_new_pcp(struct page *page) 1992 { 1993 return false; 1994 } 1995 #endif /* CONFIG_DEBUG_VM */ 1996 1997 static bool check_new_pages(struct page *page, unsigned int order) 1998 { 1999 int i; 2000 for (i = 0; i < (1 << order); i++) { 2001 struct page *p = page + i; 2002 2003 if (unlikely(check_new_page(p))) 2004 return true; 2005 } 2006 2007 return false; 2008 } 2009 2010 inline void post_alloc_hook(struct page *page, unsigned int order, 2011 gfp_t gfp_flags) 2012 { 2013 set_page_private(page, 0); 2014 set_page_refcounted(page); 2015 2016 arch_alloc_page(page, order); 2017 kernel_map_pages(page, 1 << order, 1); 2018 kasan_alloc_pages(page, order); 2019 kernel_poison_pages(page, 1 << order, 1); 2020 set_page_owner(page, order, gfp_flags); 2021 } 2022 2023 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 2024 unsigned int alloc_flags) 2025 { 2026 int i; 2027 2028 post_alloc_hook(page, order, gfp_flags); 2029 2030 if (!free_pages_prezeroed() && (gfp_flags & __GFP_ZERO)) 2031 for (i = 0; i < (1 << order); i++) 2032 clear_highpage(page + i); 2033 2034 if (order && (gfp_flags & __GFP_COMP)) 2035 prep_compound_page(page, order); 2036 2037 /* 2038 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 2039 * allocate the page. The expectation is that the caller is taking 2040 * steps that will free more memory. The caller should avoid the page 2041 * being used for !PFMEMALLOC purposes. 2042 */ 2043 if (alloc_flags & ALLOC_NO_WATERMARKS) 2044 set_page_pfmemalloc(page); 2045 else 2046 clear_page_pfmemalloc(page); 2047 } 2048 2049 /* 2050 * Go through the free lists for the given migratetype and remove 2051 * the smallest available page from the freelists 2052 */ 2053 static __always_inline 2054 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 2055 int migratetype) 2056 { 2057 unsigned int current_order; 2058 struct free_area *area; 2059 struct page *page; 2060 2061 /* Find a page of the appropriate size in the preferred list */ 2062 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 2063 area = &(zone->free_area[current_order]); 2064 page = list_first_entry_or_null(&area->free_list[migratetype], 2065 struct page, lru); 2066 if (!page) 2067 continue; 2068 list_del(&page->lru); 2069 rmv_page_order(page); 2070 area->nr_free--; 2071 expand(zone, page, order, current_order, area, migratetype); 2072 set_pcppage_migratetype(page, migratetype); 2073 return page; 2074 } 2075 2076 return NULL; 2077 } 2078 2079 2080 /* 2081 * This array describes the order lists are fallen back to when 2082 * the free lists for the desirable migrate type are depleted 2083 */ 2084 static int fallbacks[MIGRATE_TYPES][4] = { 2085 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2086 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 2087 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2088 #ifdef CONFIG_CMA 2089 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */ 2090 #endif 2091 #ifdef CONFIG_MEMORY_ISOLATION 2092 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */ 2093 #endif 2094 }; 2095 2096 #ifdef CONFIG_CMA 2097 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2098 unsigned int order) 2099 { 2100 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 2101 } 2102 #else 2103 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2104 unsigned int order) { return NULL; } 2105 #endif 2106 2107 /* 2108 * Move the free pages in a range to the free lists of the requested type. 2109 * Note that start_page and end_pages are not aligned on a pageblock 2110 * boundary. If alignment is required, use move_freepages_block() 2111 */ 2112 static int move_freepages(struct zone *zone, 2113 struct page *start_page, struct page *end_page, 2114 int migratetype, int *num_movable) 2115 { 2116 struct page *page; 2117 unsigned int order; 2118 int pages_moved = 0; 2119 2120 #ifndef CONFIG_HOLES_IN_ZONE 2121 /* 2122 * page_zone is not safe to call in this context when 2123 * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant 2124 * anyway as we check zone boundaries in move_freepages_block(). 2125 * Remove at a later date when no bug reports exist related to 2126 * grouping pages by mobility 2127 */ 2128 VM_BUG_ON(pfn_valid(page_to_pfn(start_page)) && 2129 pfn_valid(page_to_pfn(end_page)) && 2130 page_zone(start_page) != page_zone(end_page)); 2131 #endif 2132 for (page = start_page; page <= end_page;) { 2133 if (!pfn_valid_within(page_to_pfn(page))) { 2134 page++; 2135 continue; 2136 } 2137 2138 /* Make sure we are not inadvertently changing nodes */ 2139 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2140 2141 if (!PageBuddy(page)) { 2142 /* 2143 * We assume that pages that could be isolated for 2144 * migration are movable. But we don't actually try 2145 * isolating, as that would be expensive. 2146 */ 2147 if (num_movable && 2148 (PageLRU(page) || __PageMovable(page))) 2149 (*num_movable)++; 2150 2151 page++; 2152 continue; 2153 } 2154 2155 order = page_order(page); 2156 list_move(&page->lru, 2157 &zone->free_area[order].free_list[migratetype]); 2158 page += 1 << order; 2159 pages_moved += 1 << order; 2160 } 2161 2162 return pages_moved; 2163 } 2164 2165 int move_freepages_block(struct zone *zone, struct page *page, 2166 int migratetype, int *num_movable) 2167 { 2168 unsigned long start_pfn, end_pfn; 2169 struct page *start_page, *end_page; 2170 2171 if (num_movable) 2172 *num_movable = 0; 2173 2174 start_pfn = page_to_pfn(page); 2175 start_pfn = start_pfn & ~(pageblock_nr_pages-1); 2176 start_page = pfn_to_page(start_pfn); 2177 end_page = start_page + pageblock_nr_pages - 1; 2178 end_pfn = start_pfn + pageblock_nr_pages - 1; 2179 2180 /* Do not cross zone boundaries */ 2181 if (!zone_spans_pfn(zone, start_pfn)) 2182 start_page = page; 2183 if (!zone_spans_pfn(zone, end_pfn)) 2184 return 0; 2185 2186 return move_freepages(zone, start_page, end_page, migratetype, 2187 num_movable); 2188 } 2189 2190 static void change_pageblock_range(struct page *pageblock_page, 2191 int start_order, int migratetype) 2192 { 2193 int nr_pageblocks = 1 << (start_order - pageblock_order); 2194 2195 while (nr_pageblocks--) { 2196 set_pageblock_migratetype(pageblock_page, migratetype); 2197 pageblock_page += pageblock_nr_pages; 2198 } 2199 } 2200 2201 /* 2202 * When we are falling back to another migratetype during allocation, try to 2203 * steal extra free pages from the same pageblocks to satisfy further 2204 * allocations, instead of polluting multiple pageblocks. 2205 * 2206 * If we are stealing a relatively large buddy page, it is likely there will 2207 * be more free pages in the pageblock, so try to steal them all. For 2208 * reclaimable and unmovable allocations, we steal regardless of page size, 2209 * as fragmentation caused by those allocations polluting movable pageblocks 2210 * is worse than movable allocations stealing from unmovable and reclaimable 2211 * pageblocks. 2212 */ 2213 static bool can_steal_fallback(unsigned int order, int start_mt) 2214 { 2215 /* 2216 * Leaving this order check is intended, although there is 2217 * relaxed order check in next check. The reason is that 2218 * we can actually steal whole pageblock if this condition met, 2219 * but, below check doesn't guarantee it and that is just heuristic 2220 * so could be changed anytime. 2221 */ 2222 if (order >= pageblock_order) 2223 return true; 2224 2225 if (order >= pageblock_order / 2 || 2226 start_mt == MIGRATE_RECLAIMABLE || 2227 start_mt == MIGRATE_UNMOVABLE || 2228 page_group_by_mobility_disabled) 2229 return true; 2230 2231 return false; 2232 } 2233 2234 static inline void boost_watermark(struct zone *zone) 2235 { 2236 unsigned long max_boost; 2237 2238 if (!watermark_boost_factor) 2239 return; 2240 2241 max_boost = mult_frac(zone->_watermark[WMARK_HIGH], 2242 watermark_boost_factor, 10000); 2243 2244 /* 2245 * high watermark may be uninitialised if fragmentation occurs 2246 * very early in boot so do not boost. We do not fall 2247 * through and boost by pageblock_nr_pages as failing 2248 * allocations that early means that reclaim is not going 2249 * to help and it may even be impossible to reclaim the 2250 * boosted watermark resulting in a hang. 2251 */ 2252 if (!max_boost) 2253 return; 2254 2255 max_boost = max(pageblock_nr_pages, max_boost); 2256 2257 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages, 2258 max_boost); 2259 } 2260 2261 /* 2262 * This function implements actual steal behaviour. If order is large enough, 2263 * we can steal whole pageblock. If not, we first move freepages in this 2264 * pageblock to our migratetype and determine how many already-allocated pages 2265 * are there in the pageblock with a compatible migratetype. If at least half 2266 * of pages are free or compatible, we can change migratetype of the pageblock 2267 * itself, so pages freed in the future will be put on the correct free list. 2268 */ 2269 static void steal_suitable_fallback(struct zone *zone, struct page *page, 2270 unsigned int alloc_flags, int start_type, bool whole_block) 2271 { 2272 unsigned int current_order = page_order(page); 2273 struct free_area *area; 2274 int free_pages, movable_pages, alike_pages; 2275 int old_block_type; 2276 2277 old_block_type = get_pageblock_migratetype(page); 2278 2279 /* 2280 * This can happen due to races and we want to prevent broken 2281 * highatomic accounting. 2282 */ 2283 if (is_migrate_highatomic(old_block_type)) 2284 goto single_page; 2285 2286 /* Take ownership for orders >= pageblock_order */ 2287 if (current_order >= pageblock_order) { 2288 change_pageblock_range(page, current_order, start_type); 2289 goto single_page; 2290 } 2291 2292 /* 2293 * Boost watermarks to increase reclaim pressure to reduce the 2294 * likelihood of future fallbacks. Wake kswapd now as the node 2295 * may be balanced overall and kswapd will not wake naturally. 2296 */ 2297 boost_watermark(zone); 2298 if (alloc_flags & ALLOC_KSWAPD) 2299 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 2300 2301 /* We are not allowed to try stealing from the whole block */ 2302 if (!whole_block) 2303 goto single_page; 2304 2305 free_pages = move_freepages_block(zone, page, start_type, 2306 &movable_pages); 2307 /* 2308 * Determine how many pages are compatible with our allocation. 2309 * For movable allocation, it's the number of movable pages which 2310 * we just obtained. For other types it's a bit more tricky. 2311 */ 2312 if (start_type == MIGRATE_MOVABLE) { 2313 alike_pages = movable_pages; 2314 } else { 2315 /* 2316 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation 2317 * to MOVABLE pageblock, consider all non-movable pages as 2318 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or 2319 * vice versa, be conservative since we can't distinguish the 2320 * exact migratetype of non-movable pages. 2321 */ 2322 if (old_block_type == MIGRATE_MOVABLE) 2323 alike_pages = pageblock_nr_pages 2324 - (free_pages + movable_pages); 2325 else 2326 alike_pages = 0; 2327 } 2328 2329 /* moving whole block can fail due to zone boundary conditions */ 2330 if (!free_pages) 2331 goto single_page; 2332 2333 /* 2334 * If a sufficient number of pages in the block are either free or of 2335 * comparable migratability as our allocation, claim the whole block. 2336 */ 2337 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || 2338 page_group_by_mobility_disabled) 2339 set_pageblock_migratetype(page, start_type); 2340 2341 return; 2342 2343 single_page: 2344 area = &zone->free_area[current_order]; 2345 list_move(&page->lru, &area->free_list[start_type]); 2346 } 2347 2348 /* 2349 * Check whether there is a suitable fallback freepage with requested order. 2350 * If only_stealable is true, this function returns fallback_mt only if 2351 * we can steal other freepages all together. This would help to reduce 2352 * fragmentation due to mixed migratetype pages in one pageblock. 2353 */ 2354 int find_suitable_fallback(struct free_area *area, unsigned int order, 2355 int migratetype, bool only_stealable, bool *can_steal) 2356 { 2357 int i; 2358 int fallback_mt; 2359 2360 if (area->nr_free == 0) 2361 return -1; 2362 2363 *can_steal = false; 2364 for (i = 0;; i++) { 2365 fallback_mt = fallbacks[migratetype][i]; 2366 if (fallback_mt == MIGRATE_TYPES) 2367 break; 2368 2369 if (list_empty(&area->free_list[fallback_mt])) 2370 continue; 2371 2372 if (can_steal_fallback(order, migratetype)) 2373 *can_steal = true; 2374 2375 if (!only_stealable) 2376 return fallback_mt; 2377 2378 if (*can_steal) 2379 return fallback_mt; 2380 } 2381 2382 return -1; 2383 } 2384 2385 /* 2386 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2387 * there are no empty page blocks that contain a page with a suitable order 2388 */ 2389 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2390 unsigned int alloc_order) 2391 { 2392 int mt; 2393 unsigned long max_managed, flags; 2394 2395 /* 2396 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2397 * Check is race-prone but harmless. 2398 */ 2399 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages; 2400 if (zone->nr_reserved_highatomic >= max_managed) 2401 return; 2402 2403 spin_lock_irqsave(&zone->lock, flags); 2404 2405 /* Recheck the nr_reserved_highatomic limit under the lock */ 2406 if (zone->nr_reserved_highatomic >= max_managed) 2407 goto out_unlock; 2408 2409 /* Yoink! */ 2410 mt = get_pageblock_migratetype(page); 2411 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt) 2412 && !is_migrate_cma(mt)) { 2413 zone->nr_reserved_highatomic += pageblock_nr_pages; 2414 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2415 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); 2416 } 2417 2418 out_unlock: 2419 spin_unlock_irqrestore(&zone->lock, flags); 2420 } 2421 2422 /* 2423 * Used when an allocation is about to fail under memory pressure. This 2424 * potentially hurts the reliability of high-order allocations when under 2425 * intense memory pressure but failed atomic allocations should be easier 2426 * to recover from than an OOM. 2427 * 2428 * If @force is true, try to unreserve a pageblock even though highatomic 2429 * pageblock is exhausted. 2430 */ 2431 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 2432 bool force) 2433 { 2434 struct zonelist *zonelist = ac->zonelist; 2435 unsigned long flags; 2436 struct zoneref *z; 2437 struct zone *zone; 2438 struct page *page; 2439 int order; 2440 bool ret; 2441 2442 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx, 2443 ac->nodemask) { 2444 /* 2445 * Preserve at least one pageblock unless memory pressure 2446 * is really high. 2447 */ 2448 if (!force && zone->nr_reserved_highatomic <= 2449 pageblock_nr_pages) 2450 continue; 2451 2452 spin_lock_irqsave(&zone->lock, flags); 2453 for (order = 0; order < MAX_ORDER; order++) { 2454 struct free_area *area = &(zone->free_area[order]); 2455 2456 page = list_first_entry_or_null( 2457 &area->free_list[MIGRATE_HIGHATOMIC], 2458 struct page, lru); 2459 if (!page) 2460 continue; 2461 2462 /* 2463 * In page freeing path, migratetype change is racy so 2464 * we can counter several free pages in a pageblock 2465 * in this loop althoug we changed the pageblock type 2466 * from highatomic to ac->migratetype. So we should 2467 * adjust the count once. 2468 */ 2469 if (is_migrate_highatomic_page(page)) { 2470 /* 2471 * It should never happen but changes to 2472 * locking could inadvertently allow a per-cpu 2473 * drain to add pages to MIGRATE_HIGHATOMIC 2474 * while unreserving so be safe and watch for 2475 * underflows. 2476 */ 2477 zone->nr_reserved_highatomic -= min( 2478 pageblock_nr_pages, 2479 zone->nr_reserved_highatomic); 2480 } 2481 2482 /* 2483 * Convert to ac->migratetype and avoid the normal 2484 * pageblock stealing heuristics. Minimally, the caller 2485 * is doing the work and needs the pages. More 2486 * importantly, if the block was always converted to 2487 * MIGRATE_UNMOVABLE or another type then the number 2488 * of pageblocks that cannot be completely freed 2489 * may increase. 2490 */ 2491 set_pageblock_migratetype(page, ac->migratetype); 2492 ret = move_freepages_block(zone, page, ac->migratetype, 2493 NULL); 2494 if (ret) { 2495 spin_unlock_irqrestore(&zone->lock, flags); 2496 return ret; 2497 } 2498 } 2499 spin_unlock_irqrestore(&zone->lock, flags); 2500 } 2501 2502 return false; 2503 } 2504 2505 /* 2506 * Try finding a free buddy page on the fallback list and put it on the free 2507 * list of requested migratetype, possibly along with other pages from the same 2508 * block, depending on fragmentation avoidance heuristics. Returns true if 2509 * fallback was found so that __rmqueue_smallest() can grab it. 2510 * 2511 * The use of signed ints for order and current_order is a deliberate 2512 * deviation from the rest of this file, to make the for loop 2513 * condition simpler. 2514 */ 2515 static __always_inline bool 2516 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, 2517 unsigned int alloc_flags) 2518 { 2519 struct free_area *area; 2520 int current_order; 2521 int min_order = order; 2522 struct page *page; 2523 int fallback_mt; 2524 bool can_steal; 2525 2526 /* 2527 * Do not steal pages from freelists belonging to other pageblocks 2528 * i.e. orders < pageblock_order. If there are no local zones free, 2529 * the zonelists will be reiterated without ALLOC_NOFRAGMENT. 2530 */ 2531 if (alloc_flags & ALLOC_NOFRAGMENT) 2532 min_order = pageblock_order; 2533 2534 /* 2535 * Find the largest available free page in the other list. This roughly 2536 * approximates finding the pageblock with the most free pages, which 2537 * would be too costly to do exactly. 2538 */ 2539 for (current_order = MAX_ORDER - 1; current_order >= min_order; 2540 --current_order) { 2541 area = &(zone->free_area[current_order]); 2542 fallback_mt = find_suitable_fallback(area, current_order, 2543 start_migratetype, false, &can_steal); 2544 if (fallback_mt == -1) 2545 continue; 2546 2547 /* 2548 * We cannot steal all free pages from the pageblock and the 2549 * requested migratetype is movable. In that case it's better to 2550 * steal and split the smallest available page instead of the 2551 * largest available page, because even if the next movable 2552 * allocation falls back into a different pageblock than this 2553 * one, it won't cause permanent fragmentation. 2554 */ 2555 if (!can_steal && start_migratetype == MIGRATE_MOVABLE 2556 && current_order > order) 2557 goto find_smallest; 2558 2559 goto do_steal; 2560 } 2561 2562 return false; 2563 2564 find_smallest: 2565 for (current_order = order; current_order < MAX_ORDER; 2566 current_order++) { 2567 area = &(zone->free_area[current_order]); 2568 fallback_mt = find_suitable_fallback(area, current_order, 2569 start_migratetype, false, &can_steal); 2570 if (fallback_mt != -1) 2571 break; 2572 } 2573 2574 /* 2575 * This should not happen - we already found a suitable fallback 2576 * when looking for the largest page. 2577 */ 2578 VM_BUG_ON(current_order == MAX_ORDER); 2579 2580 do_steal: 2581 page = list_first_entry(&area->free_list[fallback_mt], 2582 struct page, lru); 2583 2584 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype, 2585 can_steal); 2586 2587 trace_mm_page_alloc_extfrag(page, order, current_order, 2588 start_migratetype, fallback_mt); 2589 2590 return true; 2591 2592 } 2593 2594 /* 2595 * Do the hard work of removing an element from the buddy allocator. 2596 * Call me with the zone->lock already held. 2597 */ 2598 static __always_inline struct page * 2599 __rmqueue(struct zone *zone, unsigned int order, int migratetype, 2600 unsigned int alloc_flags) 2601 { 2602 struct page *page; 2603 2604 retry: 2605 page = __rmqueue_smallest(zone, order, migratetype); 2606 if (unlikely(!page)) { 2607 if (migratetype == MIGRATE_MOVABLE) 2608 page = __rmqueue_cma_fallback(zone, order); 2609 2610 if (!page && __rmqueue_fallback(zone, order, migratetype, 2611 alloc_flags)) 2612 goto retry; 2613 } 2614 2615 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2616 return page; 2617 } 2618 2619 /* 2620 * Obtain a specified number of elements from the buddy allocator, all under 2621 * a single hold of the lock, for efficiency. Add them to the supplied list. 2622 * Returns the number of new pages which were placed at *list. 2623 */ 2624 static int rmqueue_bulk(struct zone *zone, unsigned int order, 2625 unsigned long count, struct list_head *list, 2626 int migratetype, unsigned int alloc_flags) 2627 { 2628 int i, alloced = 0; 2629 2630 spin_lock(&zone->lock); 2631 for (i = 0; i < count; ++i) { 2632 struct page *page = __rmqueue(zone, order, migratetype, 2633 alloc_flags); 2634 if (unlikely(page == NULL)) 2635 break; 2636 2637 if (unlikely(check_pcp_refill(page))) 2638 continue; 2639 2640 /* 2641 * Split buddy pages returned by expand() are received here in 2642 * physical page order. The page is added to the tail of 2643 * caller's list. From the callers perspective, the linked list 2644 * is ordered by page number under some conditions. This is 2645 * useful for IO devices that can forward direction from the 2646 * head, thus also in the physical page order. This is useful 2647 * for IO devices that can merge IO requests if the physical 2648 * pages are ordered properly. 2649 */ 2650 list_add_tail(&page->lru, list); 2651 alloced++; 2652 if (is_migrate_cma(get_pcppage_migratetype(page))) 2653 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 2654 -(1 << order)); 2655 } 2656 2657 /* 2658 * i pages were removed from the buddy list even if some leak due 2659 * to check_pcp_refill failing so adjust NR_FREE_PAGES based 2660 * on i. Do not confuse with 'alloced' which is the number of 2661 * pages added to the pcp list. 2662 */ 2663 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 2664 spin_unlock(&zone->lock); 2665 return alloced; 2666 } 2667 2668 #ifdef CONFIG_NUMA 2669 /* 2670 * Called from the vmstat counter updater to drain pagesets of this 2671 * currently executing processor on remote nodes after they have 2672 * expired. 2673 * 2674 * Note that this function must be called with the thread pinned to 2675 * a single processor. 2676 */ 2677 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 2678 { 2679 unsigned long flags; 2680 int to_drain, batch; 2681 2682 local_irq_save(flags); 2683 batch = READ_ONCE(pcp->batch); 2684 to_drain = min(pcp->count, batch); 2685 if (to_drain > 0) 2686 free_pcppages_bulk(zone, to_drain, pcp); 2687 local_irq_restore(flags); 2688 } 2689 #endif 2690 2691 /* 2692 * Drain pcplists of the indicated processor and zone. 2693 * 2694 * The processor must either be the current processor and the 2695 * thread pinned to the current processor or a processor that 2696 * is not online. 2697 */ 2698 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 2699 { 2700 unsigned long flags; 2701 struct per_cpu_pageset *pset; 2702 struct per_cpu_pages *pcp; 2703 2704 local_irq_save(flags); 2705 pset = per_cpu_ptr(zone->pageset, cpu); 2706 2707 pcp = &pset->pcp; 2708 if (pcp->count) 2709 free_pcppages_bulk(zone, pcp->count, pcp); 2710 local_irq_restore(flags); 2711 } 2712 2713 /* 2714 * Drain pcplists of all zones on the indicated processor. 2715 * 2716 * The processor must either be the current processor and the 2717 * thread pinned to the current processor or a processor that 2718 * is not online. 2719 */ 2720 static void drain_pages(unsigned int cpu) 2721 { 2722 struct zone *zone; 2723 2724 for_each_populated_zone(zone) { 2725 drain_pages_zone(cpu, zone); 2726 } 2727 } 2728 2729 /* 2730 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 2731 * 2732 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 2733 * the single zone's pages. 2734 */ 2735 void drain_local_pages(struct zone *zone) 2736 { 2737 int cpu = smp_processor_id(); 2738 2739 if (zone) 2740 drain_pages_zone(cpu, zone); 2741 else 2742 drain_pages(cpu); 2743 } 2744 2745 static void drain_local_pages_wq(struct work_struct *work) 2746 { 2747 struct pcpu_drain *drain; 2748 2749 drain = container_of(work, struct pcpu_drain, work); 2750 2751 /* 2752 * drain_all_pages doesn't use proper cpu hotplug protection so 2753 * we can race with cpu offline when the WQ can move this from 2754 * a cpu pinned worker to an unbound one. We can operate on a different 2755 * cpu which is allright but we also have to make sure to not move to 2756 * a different one. 2757 */ 2758 preempt_disable(); 2759 drain_local_pages(drain->zone); 2760 preempt_enable(); 2761 } 2762 2763 /* 2764 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 2765 * 2766 * When zone parameter is non-NULL, spill just the single zone's pages. 2767 * 2768 * Note that this can be extremely slow as the draining happens in a workqueue. 2769 */ 2770 void drain_all_pages(struct zone *zone) 2771 { 2772 int cpu; 2773 2774 /* 2775 * Allocate in the BSS so we wont require allocation in 2776 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 2777 */ 2778 static cpumask_t cpus_with_pcps; 2779 2780 /* 2781 * Make sure nobody triggers this path before mm_percpu_wq is fully 2782 * initialized. 2783 */ 2784 if (WARN_ON_ONCE(!mm_percpu_wq)) 2785 return; 2786 2787 /* 2788 * Do not drain if one is already in progress unless it's specific to 2789 * a zone. Such callers are primarily CMA and memory hotplug and need 2790 * the drain to be complete when the call returns. 2791 */ 2792 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) { 2793 if (!zone) 2794 return; 2795 mutex_lock(&pcpu_drain_mutex); 2796 } 2797 2798 /* 2799 * We don't care about racing with CPU hotplug event 2800 * as offline notification will cause the notified 2801 * cpu to drain that CPU pcps and on_each_cpu_mask 2802 * disables preemption as part of its processing 2803 */ 2804 for_each_online_cpu(cpu) { 2805 struct per_cpu_pageset *pcp; 2806 struct zone *z; 2807 bool has_pcps = false; 2808 2809 if (zone) { 2810 pcp = per_cpu_ptr(zone->pageset, cpu); 2811 if (pcp->pcp.count) 2812 has_pcps = true; 2813 } else { 2814 for_each_populated_zone(z) { 2815 pcp = per_cpu_ptr(z->pageset, cpu); 2816 if (pcp->pcp.count) { 2817 has_pcps = true; 2818 break; 2819 } 2820 } 2821 } 2822 2823 if (has_pcps) 2824 cpumask_set_cpu(cpu, &cpus_with_pcps); 2825 else 2826 cpumask_clear_cpu(cpu, &cpus_with_pcps); 2827 } 2828 2829 for_each_cpu(cpu, &cpus_with_pcps) { 2830 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu); 2831 2832 drain->zone = zone; 2833 INIT_WORK(&drain->work, drain_local_pages_wq); 2834 queue_work_on(cpu, mm_percpu_wq, &drain->work); 2835 } 2836 for_each_cpu(cpu, &cpus_with_pcps) 2837 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work); 2838 2839 mutex_unlock(&pcpu_drain_mutex); 2840 } 2841 2842 #ifdef CONFIG_HIBERNATION 2843 2844 /* 2845 * Touch the watchdog for every WD_PAGE_COUNT pages. 2846 */ 2847 #define WD_PAGE_COUNT (128*1024) 2848 2849 void mark_free_pages(struct zone *zone) 2850 { 2851 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT; 2852 unsigned long flags; 2853 unsigned int order, t; 2854 struct page *page; 2855 2856 if (zone_is_empty(zone)) 2857 return; 2858 2859 spin_lock_irqsave(&zone->lock, flags); 2860 2861 max_zone_pfn = zone_end_pfn(zone); 2862 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 2863 if (pfn_valid(pfn)) { 2864 page = pfn_to_page(pfn); 2865 2866 if (!--page_count) { 2867 touch_nmi_watchdog(); 2868 page_count = WD_PAGE_COUNT; 2869 } 2870 2871 if (page_zone(page) != zone) 2872 continue; 2873 2874 if (!swsusp_page_is_forbidden(page)) 2875 swsusp_unset_page_free(page); 2876 } 2877 2878 for_each_migratetype_order(order, t) { 2879 list_for_each_entry(page, 2880 &zone->free_area[order].free_list[t], lru) { 2881 unsigned long i; 2882 2883 pfn = page_to_pfn(page); 2884 for (i = 0; i < (1UL << order); i++) { 2885 if (!--page_count) { 2886 touch_nmi_watchdog(); 2887 page_count = WD_PAGE_COUNT; 2888 } 2889 swsusp_set_page_free(pfn_to_page(pfn + i)); 2890 } 2891 } 2892 } 2893 spin_unlock_irqrestore(&zone->lock, flags); 2894 } 2895 #endif /* CONFIG_PM */ 2896 2897 static bool free_unref_page_prepare(struct page *page, unsigned long pfn) 2898 { 2899 int migratetype; 2900 2901 if (!free_pcp_prepare(page)) 2902 return false; 2903 2904 migratetype = get_pfnblock_migratetype(page, pfn); 2905 set_pcppage_migratetype(page, migratetype); 2906 return true; 2907 } 2908 2909 static void free_unref_page_commit(struct page *page, unsigned long pfn) 2910 { 2911 struct zone *zone = page_zone(page); 2912 struct per_cpu_pages *pcp; 2913 int migratetype; 2914 2915 migratetype = get_pcppage_migratetype(page); 2916 __count_vm_event(PGFREE); 2917 2918 /* 2919 * We only track unmovable, reclaimable and movable on pcp lists. 2920 * Free ISOLATE pages back to the allocator because they are being 2921 * offlined but treat HIGHATOMIC as movable pages so we can get those 2922 * areas back if necessary. Otherwise, we may have to free 2923 * excessively into the page allocator 2924 */ 2925 if (migratetype >= MIGRATE_PCPTYPES) { 2926 if (unlikely(is_migrate_isolate(migratetype))) { 2927 free_one_page(zone, page, pfn, 0, migratetype); 2928 return; 2929 } 2930 migratetype = MIGRATE_MOVABLE; 2931 } 2932 2933 pcp = &this_cpu_ptr(zone->pageset)->pcp; 2934 list_add(&page->lru, &pcp->lists[migratetype]); 2935 pcp->count++; 2936 if (pcp->count >= pcp->high) { 2937 unsigned long batch = READ_ONCE(pcp->batch); 2938 free_pcppages_bulk(zone, batch, pcp); 2939 } 2940 } 2941 2942 /* 2943 * Free a 0-order page 2944 */ 2945 void free_unref_page(struct page *page) 2946 { 2947 unsigned long flags; 2948 unsigned long pfn = page_to_pfn(page); 2949 2950 if (!free_unref_page_prepare(page, pfn)) 2951 return; 2952 2953 local_irq_save(flags); 2954 free_unref_page_commit(page, pfn); 2955 local_irq_restore(flags); 2956 } 2957 2958 /* 2959 * Free a list of 0-order pages 2960 */ 2961 void free_unref_page_list(struct list_head *list) 2962 { 2963 struct page *page, *next; 2964 unsigned long flags, pfn; 2965 int batch_count = 0; 2966 2967 /* Prepare pages for freeing */ 2968 list_for_each_entry_safe(page, next, list, lru) { 2969 pfn = page_to_pfn(page); 2970 if (!free_unref_page_prepare(page, pfn)) 2971 list_del(&page->lru); 2972 set_page_private(page, pfn); 2973 } 2974 2975 local_irq_save(flags); 2976 list_for_each_entry_safe(page, next, list, lru) { 2977 unsigned long pfn = page_private(page); 2978 2979 set_page_private(page, 0); 2980 trace_mm_page_free_batched(page); 2981 free_unref_page_commit(page, pfn); 2982 2983 /* 2984 * Guard against excessive IRQ disabled times when we get 2985 * a large list of pages to free. 2986 */ 2987 if (++batch_count == SWAP_CLUSTER_MAX) { 2988 local_irq_restore(flags); 2989 batch_count = 0; 2990 local_irq_save(flags); 2991 } 2992 } 2993 local_irq_restore(flags); 2994 } 2995 2996 /* 2997 * split_page takes a non-compound higher-order page, and splits it into 2998 * n (1<<order) sub-pages: page[0..n] 2999 * Each sub-page must be freed individually. 3000 * 3001 * Note: this is probably too low level an operation for use in drivers. 3002 * Please consult with lkml before using this in your driver. 3003 */ 3004 void split_page(struct page *page, unsigned int order) 3005 { 3006 int i; 3007 3008 VM_BUG_ON_PAGE(PageCompound(page), page); 3009 VM_BUG_ON_PAGE(!page_count(page), page); 3010 3011 for (i = 1; i < (1 << order); i++) 3012 set_page_refcounted(page + i); 3013 split_page_owner(page, order); 3014 } 3015 EXPORT_SYMBOL_GPL(split_page); 3016 3017 int __isolate_free_page(struct page *page, unsigned int order) 3018 { 3019 unsigned long watermark; 3020 struct zone *zone; 3021 int mt; 3022 3023 BUG_ON(!PageBuddy(page)); 3024 3025 zone = page_zone(page); 3026 mt = get_pageblock_migratetype(page); 3027 3028 if (!is_migrate_isolate(mt)) { 3029 /* 3030 * Obey watermarks as if the page was being allocated. We can 3031 * emulate a high-order watermark check with a raised order-0 3032 * watermark, because we already know our high-order page 3033 * exists. 3034 */ 3035 watermark = zone->_watermark[WMARK_MIN] + (1UL << order); 3036 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 3037 return 0; 3038 3039 __mod_zone_freepage_state(zone, -(1UL << order), mt); 3040 } 3041 3042 /* Remove page from free list */ 3043 list_del(&page->lru); 3044 zone->free_area[order].nr_free--; 3045 rmv_page_order(page); 3046 3047 /* 3048 * Set the pageblock if the isolated page is at least half of a 3049 * pageblock 3050 */ 3051 if (order >= pageblock_order - 1) { 3052 struct page *endpage = page + (1 << order) - 1; 3053 for (; page < endpage; page += pageblock_nr_pages) { 3054 int mt = get_pageblock_migratetype(page); 3055 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt) 3056 && !is_migrate_highatomic(mt)) 3057 set_pageblock_migratetype(page, 3058 MIGRATE_MOVABLE); 3059 } 3060 } 3061 3062 3063 return 1UL << order; 3064 } 3065 3066 /* 3067 * Update NUMA hit/miss statistics 3068 * 3069 * Must be called with interrupts disabled. 3070 */ 3071 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) 3072 { 3073 #ifdef CONFIG_NUMA 3074 enum numa_stat_item local_stat = NUMA_LOCAL; 3075 3076 /* skip numa counters update if numa stats is disabled */ 3077 if (!static_branch_likely(&vm_numa_stat_key)) 3078 return; 3079 3080 if (zone_to_nid(z) != numa_node_id()) 3081 local_stat = NUMA_OTHER; 3082 3083 if (zone_to_nid(z) == zone_to_nid(preferred_zone)) 3084 __inc_numa_state(z, NUMA_HIT); 3085 else { 3086 __inc_numa_state(z, NUMA_MISS); 3087 __inc_numa_state(preferred_zone, NUMA_FOREIGN); 3088 } 3089 __inc_numa_state(z, local_stat); 3090 #endif 3091 } 3092 3093 /* Remove page from the per-cpu list, caller must protect the list */ 3094 static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype, 3095 unsigned int alloc_flags, 3096 struct per_cpu_pages *pcp, 3097 struct list_head *list) 3098 { 3099 struct page *page; 3100 3101 do { 3102 if (list_empty(list)) { 3103 pcp->count += rmqueue_bulk(zone, 0, 3104 pcp->batch, list, 3105 migratetype, alloc_flags); 3106 if (unlikely(list_empty(list))) 3107 return NULL; 3108 } 3109 3110 page = list_first_entry(list, struct page, lru); 3111 list_del(&page->lru); 3112 pcp->count--; 3113 } while (check_new_pcp(page)); 3114 3115 return page; 3116 } 3117 3118 /* Lock and remove page from the per-cpu list */ 3119 static struct page *rmqueue_pcplist(struct zone *preferred_zone, 3120 struct zone *zone, unsigned int order, 3121 gfp_t gfp_flags, int migratetype, 3122 unsigned int alloc_flags) 3123 { 3124 struct per_cpu_pages *pcp; 3125 struct list_head *list; 3126 struct page *page; 3127 unsigned long flags; 3128 3129 local_irq_save(flags); 3130 pcp = &this_cpu_ptr(zone->pageset)->pcp; 3131 list = &pcp->lists[migratetype]; 3132 page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list); 3133 if (page) { 3134 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3135 zone_statistics(preferred_zone, zone); 3136 } 3137 local_irq_restore(flags); 3138 return page; 3139 } 3140 3141 /* 3142 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 3143 */ 3144 static inline 3145 struct page *rmqueue(struct zone *preferred_zone, 3146 struct zone *zone, unsigned int order, 3147 gfp_t gfp_flags, unsigned int alloc_flags, 3148 int migratetype) 3149 { 3150 unsigned long flags; 3151 struct page *page; 3152 3153 if (likely(order == 0)) { 3154 page = rmqueue_pcplist(preferred_zone, zone, order, 3155 gfp_flags, migratetype, alloc_flags); 3156 goto out; 3157 } 3158 3159 /* 3160 * We most definitely don't want callers attempting to 3161 * allocate greater than order-1 page units with __GFP_NOFAIL. 3162 */ 3163 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 3164 spin_lock_irqsave(&zone->lock, flags); 3165 3166 do { 3167 page = NULL; 3168 if (alloc_flags & ALLOC_HARDER) { 3169 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3170 if (page) 3171 trace_mm_page_alloc_zone_locked(page, order, migratetype); 3172 } 3173 if (!page) 3174 page = __rmqueue(zone, order, migratetype, alloc_flags); 3175 } while (page && check_new_pages(page, order)); 3176 spin_unlock(&zone->lock); 3177 if (!page) 3178 goto failed; 3179 __mod_zone_freepage_state(zone, -(1 << order), 3180 get_pcppage_migratetype(page)); 3181 3182 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3183 zone_statistics(preferred_zone, zone); 3184 local_irq_restore(flags); 3185 3186 out: 3187 /* Separate test+clear to avoid unnecessary atomics */ 3188 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) { 3189 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 3190 wakeup_kswapd(zone, 0, 0, zone_idx(zone)); 3191 } 3192 3193 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); 3194 return page; 3195 3196 failed: 3197 local_irq_restore(flags); 3198 return NULL; 3199 } 3200 3201 #ifdef CONFIG_FAIL_PAGE_ALLOC 3202 3203 static struct { 3204 struct fault_attr attr; 3205 3206 bool ignore_gfp_highmem; 3207 bool ignore_gfp_reclaim; 3208 u32 min_order; 3209 } fail_page_alloc = { 3210 .attr = FAULT_ATTR_INITIALIZER, 3211 .ignore_gfp_reclaim = true, 3212 .ignore_gfp_highmem = true, 3213 .min_order = 1, 3214 }; 3215 3216 static int __init setup_fail_page_alloc(char *str) 3217 { 3218 return setup_fault_attr(&fail_page_alloc.attr, str); 3219 } 3220 __setup("fail_page_alloc=", setup_fail_page_alloc); 3221 3222 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3223 { 3224 if (order < fail_page_alloc.min_order) 3225 return false; 3226 if (gfp_mask & __GFP_NOFAIL) 3227 return false; 3228 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 3229 return false; 3230 if (fail_page_alloc.ignore_gfp_reclaim && 3231 (gfp_mask & __GFP_DIRECT_RECLAIM)) 3232 return false; 3233 3234 return should_fail(&fail_page_alloc.attr, 1 << order); 3235 } 3236 3237 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 3238 3239 static int __init fail_page_alloc_debugfs(void) 3240 { 3241 umode_t mode = S_IFREG | 0600; 3242 struct dentry *dir; 3243 3244 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 3245 &fail_page_alloc.attr); 3246 3247 debugfs_create_bool("ignore-gfp-wait", mode, dir, 3248 &fail_page_alloc.ignore_gfp_reclaim); 3249 debugfs_create_bool("ignore-gfp-highmem", mode, dir, 3250 &fail_page_alloc.ignore_gfp_highmem); 3251 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order); 3252 3253 return 0; 3254 } 3255 3256 late_initcall(fail_page_alloc_debugfs); 3257 3258 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 3259 3260 #else /* CONFIG_FAIL_PAGE_ALLOC */ 3261 3262 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3263 { 3264 return false; 3265 } 3266 3267 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 3268 3269 static noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3270 { 3271 return __should_fail_alloc_page(gfp_mask, order); 3272 } 3273 ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE); 3274 3275 /* 3276 * Return true if free base pages are above 'mark'. For high-order checks it 3277 * will return true of the order-0 watermark is reached and there is at least 3278 * one free page of a suitable size. Checking now avoids taking the zone lock 3279 * to check in the allocation paths if no pages are free. 3280 */ 3281 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3282 int classzone_idx, unsigned int alloc_flags, 3283 long free_pages) 3284 { 3285 long min = mark; 3286 int o; 3287 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3288 3289 /* free_pages may go negative - that's OK */ 3290 free_pages -= (1 << order) - 1; 3291 3292 if (alloc_flags & ALLOC_HIGH) 3293 min -= min / 2; 3294 3295 /* 3296 * If the caller does not have rights to ALLOC_HARDER then subtract 3297 * the high-atomic reserves. This will over-estimate the size of the 3298 * atomic reserve but it avoids a search. 3299 */ 3300 if (likely(!alloc_harder)) { 3301 free_pages -= z->nr_reserved_highatomic; 3302 } else { 3303 /* 3304 * OOM victims can try even harder than normal ALLOC_HARDER 3305 * users on the grounds that it's definitely going to be in 3306 * the exit path shortly and free memory. Any allocation it 3307 * makes during the free path will be small and short-lived. 3308 */ 3309 if (alloc_flags & ALLOC_OOM) 3310 min -= min / 2; 3311 else 3312 min -= min / 4; 3313 } 3314 3315 3316 #ifdef CONFIG_CMA 3317 /* If allocation can't use CMA areas don't use free CMA pages */ 3318 if (!(alloc_flags & ALLOC_CMA)) 3319 free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES); 3320 #endif 3321 3322 /* 3323 * Check watermarks for an order-0 allocation request. If these 3324 * are not met, then a high-order request also cannot go ahead 3325 * even if a suitable page happened to be free. 3326 */ 3327 if (free_pages <= min + z->lowmem_reserve[classzone_idx]) 3328 return false; 3329 3330 /* If this is an order-0 request then the watermark is fine */ 3331 if (!order) 3332 return true; 3333 3334 /* For a high-order request, check at least one suitable page is free */ 3335 for (o = order; o < MAX_ORDER; o++) { 3336 struct free_area *area = &z->free_area[o]; 3337 int mt; 3338 3339 if (!area->nr_free) 3340 continue; 3341 3342 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 3343 if (!list_empty(&area->free_list[mt])) 3344 return true; 3345 } 3346 3347 #ifdef CONFIG_CMA 3348 if ((alloc_flags & ALLOC_CMA) && 3349 !list_empty(&area->free_list[MIGRATE_CMA])) { 3350 return true; 3351 } 3352 #endif 3353 if (alloc_harder && 3354 !list_empty(&area->free_list[MIGRATE_HIGHATOMIC])) 3355 return true; 3356 } 3357 return false; 3358 } 3359 3360 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3361 int classzone_idx, unsigned int alloc_flags) 3362 { 3363 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 3364 zone_page_state(z, NR_FREE_PAGES)); 3365 } 3366 3367 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 3368 unsigned long mark, int classzone_idx, unsigned int alloc_flags) 3369 { 3370 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3371 long cma_pages = 0; 3372 3373 #ifdef CONFIG_CMA 3374 /* If allocation can't use CMA areas don't use free CMA pages */ 3375 if (!(alloc_flags & ALLOC_CMA)) 3376 cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES); 3377 #endif 3378 3379 /* 3380 * Fast check for order-0 only. If this fails then the reserves 3381 * need to be calculated. There is a corner case where the check 3382 * passes but only the high-order atomic reserve are free. If 3383 * the caller is !atomic then it'll uselessly search the free 3384 * list. That corner case is then slower but it is harmless. 3385 */ 3386 if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx]) 3387 return true; 3388 3389 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 3390 free_pages); 3391 } 3392 3393 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 3394 unsigned long mark, int classzone_idx) 3395 { 3396 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3397 3398 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 3399 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 3400 3401 return __zone_watermark_ok(z, order, mark, classzone_idx, 0, 3402 free_pages); 3403 } 3404 3405 #ifdef CONFIG_NUMA 3406 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3407 { 3408 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= 3409 RECLAIM_DISTANCE; 3410 } 3411 #else /* CONFIG_NUMA */ 3412 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3413 { 3414 return true; 3415 } 3416 #endif /* CONFIG_NUMA */ 3417 3418 /* 3419 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid 3420 * fragmentation is subtle. If the preferred zone was HIGHMEM then 3421 * premature use of a lower zone may cause lowmem pressure problems that 3422 * are worse than fragmentation. If the next zone is ZONE_DMA then it is 3423 * probably too small. It only makes sense to spread allocations to avoid 3424 * fragmentation between the Normal and DMA32 zones. 3425 */ 3426 static inline unsigned int 3427 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) 3428 { 3429 unsigned int alloc_flags = 0; 3430 3431 if (gfp_mask & __GFP_KSWAPD_RECLAIM) 3432 alloc_flags |= ALLOC_KSWAPD; 3433 3434 #ifdef CONFIG_ZONE_DMA32 3435 if (!zone) 3436 return alloc_flags; 3437 3438 if (zone_idx(zone) != ZONE_NORMAL) 3439 return alloc_flags; 3440 3441 /* 3442 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and 3443 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume 3444 * on UMA that if Normal is populated then so is DMA32. 3445 */ 3446 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1); 3447 if (nr_online_nodes > 1 && !populated_zone(--zone)) 3448 return alloc_flags; 3449 3450 alloc_flags |= ALLOC_NOFRAGMENT; 3451 #endif /* CONFIG_ZONE_DMA32 */ 3452 return alloc_flags; 3453 } 3454 3455 /* 3456 * get_page_from_freelist goes through the zonelist trying to allocate 3457 * a page. 3458 */ 3459 static struct page * 3460 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 3461 const struct alloc_context *ac) 3462 { 3463 struct zoneref *z; 3464 struct zone *zone; 3465 struct pglist_data *last_pgdat_dirty_limit = NULL; 3466 bool no_fallback; 3467 3468 retry: 3469 /* 3470 * Scan zonelist, looking for a zone with enough free. 3471 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 3472 */ 3473 no_fallback = alloc_flags & ALLOC_NOFRAGMENT; 3474 z = ac->preferred_zoneref; 3475 for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3476 ac->nodemask) { 3477 struct page *page; 3478 unsigned long mark; 3479 3480 if (cpusets_enabled() && 3481 (alloc_flags & ALLOC_CPUSET) && 3482 !__cpuset_zone_allowed(zone, gfp_mask)) 3483 continue; 3484 /* 3485 * When allocating a page cache page for writing, we 3486 * want to get it from a node that is within its dirty 3487 * limit, such that no single node holds more than its 3488 * proportional share of globally allowed dirty pages. 3489 * The dirty limits take into account the node's 3490 * lowmem reserves and high watermark so that kswapd 3491 * should be able to balance it without having to 3492 * write pages from its LRU list. 3493 * 3494 * XXX: For now, allow allocations to potentially 3495 * exceed the per-node dirty limit in the slowpath 3496 * (spread_dirty_pages unset) before going into reclaim, 3497 * which is important when on a NUMA setup the allowed 3498 * nodes are together not big enough to reach the 3499 * global limit. The proper fix for these situations 3500 * will require awareness of nodes in the 3501 * dirty-throttling and the flusher threads. 3502 */ 3503 if (ac->spread_dirty_pages) { 3504 if (last_pgdat_dirty_limit == zone->zone_pgdat) 3505 continue; 3506 3507 if (!node_dirty_ok(zone->zone_pgdat)) { 3508 last_pgdat_dirty_limit = zone->zone_pgdat; 3509 continue; 3510 } 3511 } 3512 3513 if (no_fallback && nr_online_nodes > 1 && 3514 zone != ac->preferred_zoneref->zone) { 3515 int local_nid; 3516 3517 /* 3518 * If moving to a remote node, retry but allow 3519 * fragmenting fallbacks. Locality is more important 3520 * than fragmentation avoidance. 3521 */ 3522 local_nid = zone_to_nid(ac->preferred_zoneref->zone); 3523 if (zone_to_nid(zone) != local_nid) { 3524 alloc_flags &= ~ALLOC_NOFRAGMENT; 3525 goto retry; 3526 } 3527 } 3528 3529 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); 3530 if (!zone_watermark_fast(zone, order, mark, 3531 ac_classzone_idx(ac), alloc_flags)) { 3532 int ret; 3533 3534 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3535 /* 3536 * Watermark failed for this zone, but see if we can 3537 * grow this zone if it contains deferred pages. 3538 */ 3539 if (static_branch_unlikely(&deferred_pages)) { 3540 if (_deferred_grow_zone(zone, order)) 3541 goto try_this_zone; 3542 } 3543 #endif 3544 /* Checked here to keep the fast path fast */ 3545 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 3546 if (alloc_flags & ALLOC_NO_WATERMARKS) 3547 goto try_this_zone; 3548 3549 if (node_reclaim_mode == 0 || 3550 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 3551 continue; 3552 3553 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 3554 switch (ret) { 3555 case NODE_RECLAIM_NOSCAN: 3556 /* did not scan */ 3557 continue; 3558 case NODE_RECLAIM_FULL: 3559 /* scanned but unreclaimable */ 3560 continue; 3561 default: 3562 /* did we reclaim enough */ 3563 if (zone_watermark_ok(zone, order, mark, 3564 ac_classzone_idx(ac), alloc_flags)) 3565 goto try_this_zone; 3566 3567 continue; 3568 } 3569 } 3570 3571 try_this_zone: 3572 page = rmqueue(ac->preferred_zoneref->zone, zone, order, 3573 gfp_mask, alloc_flags, ac->migratetype); 3574 if (page) { 3575 prep_new_page(page, order, gfp_mask, alloc_flags); 3576 3577 /* 3578 * If this is a high-order atomic allocation then check 3579 * if the pageblock should be reserved for the future 3580 */ 3581 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 3582 reserve_highatomic_pageblock(page, zone, order); 3583 3584 return page; 3585 } else { 3586 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3587 /* Try again if zone has deferred pages */ 3588 if (static_branch_unlikely(&deferred_pages)) { 3589 if (_deferred_grow_zone(zone, order)) 3590 goto try_this_zone; 3591 } 3592 #endif 3593 } 3594 } 3595 3596 /* 3597 * It's possible on a UMA machine to get through all zones that are 3598 * fragmented. If avoiding fragmentation, reset and try again. 3599 */ 3600 if (no_fallback) { 3601 alloc_flags &= ~ALLOC_NOFRAGMENT; 3602 goto retry; 3603 } 3604 3605 return NULL; 3606 } 3607 3608 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) 3609 { 3610 unsigned int filter = SHOW_MEM_FILTER_NODES; 3611 static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1); 3612 3613 if (!__ratelimit(&show_mem_rs)) 3614 return; 3615 3616 /* 3617 * This documents exceptions given to allocations in certain 3618 * contexts that are allowed to allocate outside current's set 3619 * of allowed nodes. 3620 */ 3621 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3622 if (tsk_is_oom_victim(current) || 3623 (current->flags & (PF_MEMALLOC | PF_EXITING))) 3624 filter &= ~SHOW_MEM_FILTER_NODES; 3625 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 3626 filter &= ~SHOW_MEM_FILTER_NODES; 3627 3628 show_mem(filter, nodemask); 3629 } 3630 3631 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) 3632 { 3633 struct va_format vaf; 3634 va_list args; 3635 static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL, 3636 DEFAULT_RATELIMIT_BURST); 3637 3638 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs)) 3639 return; 3640 3641 va_start(args, fmt); 3642 vaf.fmt = fmt; 3643 vaf.va = &args; 3644 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl", 3645 current->comm, &vaf, gfp_mask, &gfp_mask, 3646 nodemask_pr_args(nodemask)); 3647 va_end(args); 3648 3649 cpuset_print_current_mems_allowed(); 3650 pr_cont("\n"); 3651 dump_stack(); 3652 warn_alloc_show_mem(gfp_mask, nodemask); 3653 } 3654 3655 static inline struct page * 3656 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, 3657 unsigned int alloc_flags, 3658 const struct alloc_context *ac) 3659 { 3660 struct page *page; 3661 3662 page = get_page_from_freelist(gfp_mask, order, 3663 alloc_flags|ALLOC_CPUSET, ac); 3664 /* 3665 * fallback to ignore cpuset restriction if our nodes 3666 * are depleted 3667 */ 3668 if (!page) 3669 page = get_page_from_freelist(gfp_mask, order, 3670 alloc_flags, ac); 3671 3672 return page; 3673 } 3674 3675 static inline struct page * 3676 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 3677 const struct alloc_context *ac, unsigned long *did_some_progress) 3678 { 3679 struct oom_control oc = { 3680 .zonelist = ac->zonelist, 3681 .nodemask = ac->nodemask, 3682 .memcg = NULL, 3683 .gfp_mask = gfp_mask, 3684 .order = order, 3685 }; 3686 struct page *page; 3687 3688 *did_some_progress = 0; 3689 3690 /* 3691 * Acquire the oom lock. If that fails, somebody else is 3692 * making progress for us. 3693 */ 3694 if (!mutex_trylock(&oom_lock)) { 3695 *did_some_progress = 1; 3696 schedule_timeout_uninterruptible(1); 3697 return NULL; 3698 } 3699 3700 /* 3701 * Go through the zonelist yet one more time, keep very high watermark 3702 * here, this is only to catch a parallel oom killing, we must fail if 3703 * we're still under heavy pressure. But make sure that this reclaim 3704 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY 3705 * allocation which will never fail due to oom_lock already held. 3706 */ 3707 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & 3708 ~__GFP_DIRECT_RECLAIM, order, 3709 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 3710 if (page) 3711 goto out; 3712 3713 /* Coredumps can quickly deplete all memory reserves */ 3714 if (current->flags & PF_DUMPCORE) 3715 goto out; 3716 /* The OOM killer will not help higher order allocs */ 3717 if (order > PAGE_ALLOC_COSTLY_ORDER) 3718 goto out; 3719 /* 3720 * We have already exhausted all our reclaim opportunities without any 3721 * success so it is time to admit defeat. We will skip the OOM killer 3722 * because it is very likely that the caller has a more reasonable 3723 * fallback than shooting a random task. 3724 */ 3725 if (gfp_mask & __GFP_RETRY_MAYFAIL) 3726 goto out; 3727 /* The OOM killer does not needlessly kill tasks for lowmem */ 3728 if (ac->high_zoneidx < ZONE_NORMAL) 3729 goto out; 3730 if (pm_suspended_storage()) 3731 goto out; 3732 /* 3733 * XXX: GFP_NOFS allocations should rather fail than rely on 3734 * other request to make a forward progress. 3735 * We are in an unfortunate situation where out_of_memory cannot 3736 * do much for this context but let's try it to at least get 3737 * access to memory reserved if the current task is killed (see 3738 * out_of_memory). Once filesystems are ready to handle allocation 3739 * failures more gracefully we should just bail out here. 3740 */ 3741 3742 /* The OOM killer may not free memory on a specific node */ 3743 if (gfp_mask & __GFP_THISNODE) 3744 goto out; 3745 3746 /* Exhausted what can be done so it's blame time */ 3747 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 3748 *did_some_progress = 1; 3749 3750 /* 3751 * Help non-failing allocations by giving them access to memory 3752 * reserves 3753 */ 3754 if (gfp_mask & __GFP_NOFAIL) 3755 page = __alloc_pages_cpuset_fallback(gfp_mask, order, 3756 ALLOC_NO_WATERMARKS, ac); 3757 } 3758 out: 3759 mutex_unlock(&oom_lock); 3760 return page; 3761 } 3762 3763 /* 3764 * Maximum number of compaction retries wit a progress before OOM 3765 * killer is consider as the only way to move forward. 3766 */ 3767 #define MAX_COMPACT_RETRIES 16 3768 3769 #ifdef CONFIG_COMPACTION 3770 /* Try memory compaction for high-order allocations before reclaim */ 3771 static struct page * 3772 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3773 unsigned int alloc_flags, const struct alloc_context *ac, 3774 enum compact_priority prio, enum compact_result *compact_result) 3775 { 3776 struct page *page = NULL; 3777 unsigned long pflags; 3778 unsigned int noreclaim_flag; 3779 3780 if (!order) 3781 return NULL; 3782 3783 psi_memstall_enter(&pflags); 3784 noreclaim_flag = memalloc_noreclaim_save(); 3785 3786 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 3787 prio, &page); 3788 3789 memalloc_noreclaim_restore(noreclaim_flag); 3790 psi_memstall_leave(&pflags); 3791 3792 /* 3793 * At least in one zone compaction wasn't deferred or skipped, so let's 3794 * count a compaction stall 3795 */ 3796 count_vm_event(COMPACTSTALL); 3797 3798 /* Prep a captured page if available */ 3799 if (page) 3800 prep_new_page(page, order, gfp_mask, alloc_flags); 3801 3802 /* Try get a page from the freelist if available */ 3803 if (!page) 3804 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 3805 3806 if (page) { 3807 struct zone *zone = page_zone(page); 3808 3809 zone->compact_blockskip_flush = false; 3810 compaction_defer_reset(zone, order, true); 3811 count_vm_event(COMPACTSUCCESS); 3812 return page; 3813 } 3814 3815 /* 3816 * It's bad if compaction run occurs and fails. The most likely reason 3817 * is that pages exist, but not enough to satisfy watermarks. 3818 */ 3819 count_vm_event(COMPACTFAIL); 3820 3821 cond_resched(); 3822 3823 return NULL; 3824 } 3825 3826 static inline bool 3827 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 3828 enum compact_result compact_result, 3829 enum compact_priority *compact_priority, 3830 int *compaction_retries) 3831 { 3832 int max_retries = MAX_COMPACT_RETRIES; 3833 int min_priority; 3834 bool ret = false; 3835 int retries = *compaction_retries; 3836 enum compact_priority priority = *compact_priority; 3837 3838 if (!order) 3839 return false; 3840 3841 if (compaction_made_progress(compact_result)) 3842 (*compaction_retries)++; 3843 3844 /* 3845 * compaction considers all the zone as desperately out of memory 3846 * so it doesn't really make much sense to retry except when the 3847 * failure could be caused by insufficient priority 3848 */ 3849 if (compaction_failed(compact_result)) 3850 goto check_priority; 3851 3852 /* 3853 * make sure the compaction wasn't deferred or didn't bail out early 3854 * due to locks contention before we declare that we should give up. 3855 * But do not retry if the given zonelist is not suitable for 3856 * compaction. 3857 */ 3858 if (compaction_withdrawn(compact_result)) { 3859 ret = compaction_zonelist_suitable(ac, order, alloc_flags); 3860 goto out; 3861 } 3862 3863 /* 3864 * !costly requests are much more important than __GFP_RETRY_MAYFAIL 3865 * costly ones because they are de facto nofail and invoke OOM 3866 * killer to move on while costly can fail and users are ready 3867 * to cope with that. 1/4 retries is rather arbitrary but we 3868 * would need much more detailed feedback from compaction to 3869 * make a better decision. 3870 */ 3871 if (order > PAGE_ALLOC_COSTLY_ORDER) 3872 max_retries /= 4; 3873 if (*compaction_retries <= max_retries) { 3874 ret = true; 3875 goto out; 3876 } 3877 3878 /* 3879 * Make sure there are attempts at the highest priority if we exhausted 3880 * all retries or failed at the lower priorities. 3881 */ 3882 check_priority: 3883 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 3884 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 3885 3886 if (*compact_priority > min_priority) { 3887 (*compact_priority)--; 3888 *compaction_retries = 0; 3889 ret = true; 3890 } 3891 out: 3892 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret); 3893 return ret; 3894 } 3895 #else 3896 static inline struct page * 3897 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3898 unsigned int alloc_flags, const struct alloc_context *ac, 3899 enum compact_priority prio, enum compact_result *compact_result) 3900 { 3901 *compact_result = COMPACT_SKIPPED; 3902 return NULL; 3903 } 3904 3905 static inline bool 3906 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 3907 enum compact_result compact_result, 3908 enum compact_priority *compact_priority, 3909 int *compaction_retries) 3910 { 3911 struct zone *zone; 3912 struct zoneref *z; 3913 3914 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 3915 return false; 3916 3917 /* 3918 * There are setups with compaction disabled which would prefer to loop 3919 * inside the allocator rather than hit the oom killer prematurely. 3920 * Let's give them a good hope and keep retrying while the order-0 3921 * watermarks are OK. 3922 */ 3923 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3924 ac->nodemask) { 3925 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 3926 ac_classzone_idx(ac), alloc_flags)) 3927 return true; 3928 } 3929 return false; 3930 } 3931 #endif /* CONFIG_COMPACTION */ 3932 3933 #ifdef CONFIG_LOCKDEP 3934 static struct lockdep_map __fs_reclaim_map = 3935 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map); 3936 3937 static bool __need_fs_reclaim(gfp_t gfp_mask) 3938 { 3939 gfp_mask = current_gfp_context(gfp_mask); 3940 3941 /* no reclaim without waiting on it */ 3942 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) 3943 return false; 3944 3945 /* this guy won't enter reclaim */ 3946 if (current->flags & PF_MEMALLOC) 3947 return false; 3948 3949 /* We're only interested __GFP_FS allocations for now */ 3950 if (!(gfp_mask & __GFP_FS)) 3951 return false; 3952 3953 if (gfp_mask & __GFP_NOLOCKDEP) 3954 return false; 3955 3956 return true; 3957 } 3958 3959 void __fs_reclaim_acquire(void) 3960 { 3961 lock_map_acquire(&__fs_reclaim_map); 3962 } 3963 3964 void __fs_reclaim_release(void) 3965 { 3966 lock_map_release(&__fs_reclaim_map); 3967 } 3968 3969 void fs_reclaim_acquire(gfp_t gfp_mask) 3970 { 3971 if (__need_fs_reclaim(gfp_mask)) 3972 __fs_reclaim_acquire(); 3973 } 3974 EXPORT_SYMBOL_GPL(fs_reclaim_acquire); 3975 3976 void fs_reclaim_release(gfp_t gfp_mask) 3977 { 3978 if (__need_fs_reclaim(gfp_mask)) 3979 __fs_reclaim_release(); 3980 } 3981 EXPORT_SYMBOL_GPL(fs_reclaim_release); 3982 #endif 3983 3984 /* Perform direct synchronous page reclaim */ 3985 static int 3986 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 3987 const struct alloc_context *ac) 3988 { 3989 struct reclaim_state reclaim_state; 3990 int progress; 3991 unsigned int noreclaim_flag; 3992 unsigned long pflags; 3993 3994 cond_resched(); 3995 3996 /* We now go into synchronous reclaim */ 3997 cpuset_memory_pressure_bump(); 3998 psi_memstall_enter(&pflags); 3999 fs_reclaim_acquire(gfp_mask); 4000 noreclaim_flag = memalloc_noreclaim_save(); 4001 reclaim_state.reclaimed_slab = 0; 4002 current->reclaim_state = &reclaim_state; 4003 4004 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 4005 ac->nodemask); 4006 4007 current->reclaim_state = NULL; 4008 memalloc_noreclaim_restore(noreclaim_flag); 4009 fs_reclaim_release(gfp_mask); 4010 psi_memstall_leave(&pflags); 4011 4012 cond_resched(); 4013 4014 return progress; 4015 } 4016 4017 /* The really slow allocator path where we enter direct reclaim */ 4018 static inline struct page * 4019 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 4020 unsigned int alloc_flags, const struct alloc_context *ac, 4021 unsigned long *did_some_progress) 4022 { 4023 struct page *page = NULL; 4024 bool drained = false; 4025 4026 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 4027 if (unlikely(!(*did_some_progress))) 4028 return NULL; 4029 4030 retry: 4031 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4032 4033 /* 4034 * If an allocation failed after direct reclaim, it could be because 4035 * pages are pinned on the per-cpu lists or in high alloc reserves. 4036 * Shrink them them and try again 4037 */ 4038 if (!page && !drained) { 4039 unreserve_highatomic_pageblock(ac, false); 4040 drain_all_pages(NULL); 4041 drained = true; 4042 goto retry; 4043 } 4044 4045 return page; 4046 } 4047 4048 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, 4049 const struct alloc_context *ac) 4050 { 4051 struct zoneref *z; 4052 struct zone *zone; 4053 pg_data_t *last_pgdat = NULL; 4054 enum zone_type high_zoneidx = ac->high_zoneidx; 4055 4056 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, high_zoneidx, 4057 ac->nodemask) { 4058 if (last_pgdat != zone->zone_pgdat) 4059 wakeup_kswapd(zone, gfp_mask, order, high_zoneidx); 4060 last_pgdat = zone->zone_pgdat; 4061 } 4062 } 4063 4064 static inline unsigned int 4065 gfp_to_alloc_flags(gfp_t gfp_mask) 4066 { 4067 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 4068 4069 /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */ 4070 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 4071 4072 /* 4073 * The caller may dip into page reserves a bit more if the caller 4074 * cannot run direct reclaim, or if the caller has realtime scheduling 4075 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 4076 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 4077 */ 4078 alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); 4079 4080 if (gfp_mask & __GFP_ATOMIC) { 4081 /* 4082 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 4083 * if it can't schedule. 4084 */ 4085 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4086 alloc_flags |= ALLOC_HARDER; 4087 /* 4088 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 4089 * comment for __cpuset_node_allowed(). 4090 */ 4091 alloc_flags &= ~ALLOC_CPUSET; 4092 } else if (unlikely(rt_task(current)) && !in_interrupt()) 4093 alloc_flags |= ALLOC_HARDER; 4094 4095 if (gfp_mask & __GFP_KSWAPD_RECLAIM) 4096 alloc_flags |= ALLOC_KSWAPD; 4097 4098 #ifdef CONFIG_CMA 4099 if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) 4100 alloc_flags |= ALLOC_CMA; 4101 #endif 4102 return alloc_flags; 4103 } 4104 4105 static bool oom_reserves_allowed(struct task_struct *tsk) 4106 { 4107 if (!tsk_is_oom_victim(tsk)) 4108 return false; 4109 4110 /* 4111 * !MMU doesn't have oom reaper so give access to memory reserves 4112 * only to the thread with TIF_MEMDIE set 4113 */ 4114 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE)) 4115 return false; 4116 4117 return true; 4118 } 4119 4120 /* 4121 * Distinguish requests which really need access to full memory 4122 * reserves from oom victims which can live with a portion of it 4123 */ 4124 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) 4125 { 4126 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 4127 return 0; 4128 if (gfp_mask & __GFP_MEMALLOC) 4129 return ALLOC_NO_WATERMARKS; 4130 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 4131 return ALLOC_NO_WATERMARKS; 4132 if (!in_interrupt()) { 4133 if (current->flags & PF_MEMALLOC) 4134 return ALLOC_NO_WATERMARKS; 4135 else if (oom_reserves_allowed(current)) 4136 return ALLOC_OOM; 4137 } 4138 4139 return 0; 4140 } 4141 4142 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 4143 { 4144 return !!__gfp_pfmemalloc_flags(gfp_mask); 4145 } 4146 4147 /* 4148 * Checks whether it makes sense to retry the reclaim to make a forward progress 4149 * for the given allocation request. 4150 * 4151 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row 4152 * without success, or when we couldn't even meet the watermark if we 4153 * reclaimed all remaining pages on the LRU lists. 4154 * 4155 * Returns true if a retry is viable or false to enter the oom path. 4156 */ 4157 static inline bool 4158 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 4159 struct alloc_context *ac, int alloc_flags, 4160 bool did_some_progress, int *no_progress_loops) 4161 { 4162 struct zone *zone; 4163 struct zoneref *z; 4164 bool ret = false; 4165 4166 /* 4167 * Costly allocations might have made a progress but this doesn't mean 4168 * their order will become available due to high fragmentation so 4169 * always increment the no progress counter for them 4170 */ 4171 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 4172 *no_progress_loops = 0; 4173 else 4174 (*no_progress_loops)++; 4175 4176 /* 4177 * Make sure we converge to OOM if we cannot make any progress 4178 * several times in the row. 4179 */ 4180 if (*no_progress_loops > MAX_RECLAIM_RETRIES) { 4181 /* Before OOM, exhaust highatomic_reserve */ 4182 return unreserve_highatomic_pageblock(ac, true); 4183 } 4184 4185 /* 4186 * Keep reclaiming pages while there is a chance this will lead 4187 * somewhere. If none of the target zones can satisfy our allocation 4188 * request even if all reclaimable pages are considered then we are 4189 * screwed and have to go OOM. 4190 */ 4191 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 4192 ac->nodemask) { 4193 unsigned long available; 4194 unsigned long reclaimable; 4195 unsigned long min_wmark = min_wmark_pages(zone); 4196 bool wmark; 4197 4198 available = reclaimable = zone_reclaimable_pages(zone); 4199 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 4200 4201 /* 4202 * Would the allocation succeed if we reclaimed all 4203 * reclaimable pages? 4204 */ 4205 wmark = __zone_watermark_ok(zone, order, min_wmark, 4206 ac_classzone_idx(ac), alloc_flags, available); 4207 trace_reclaim_retry_zone(z, order, reclaimable, 4208 available, min_wmark, *no_progress_loops, wmark); 4209 if (wmark) { 4210 /* 4211 * If we didn't make any progress and have a lot of 4212 * dirty + writeback pages then we should wait for 4213 * an IO to complete to slow down the reclaim and 4214 * prevent from pre mature OOM 4215 */ 4216 if (!did_some_progress) { 4217 unsigned long write_pending; 4218 4219 write_pending = zone_page_state_snapshot(zone, 4220 NR_ZONE_WRITE_PENDING); 4221 4222 if (2 * write_pending > reclaimable) { 4223 congestion_wait(BLK_RW_ASYNC, HZ/10); 4224 return true; 4225 } 4226 } 4227 4228 ret = true; 4229 goto out; 4230 } 4231 } 4232 4233 out: 4234 /* 4235 * Memory allocation/reclaim might be called from a WQ context and the 4236 * current implementation of the WQ concurrency control doesn't 4237 * recognize that a particular WQ is congested if the worker thread is 4238 * looping without ever sleeping. Therefore we have to do a short sleep 4239 * here rather than calling cond_resched(). 4240 */ 4241 if (current->flags & PF_WQ_WORKER) 4242 schedule_timeout_uninterruptible(1); 4243 else 4244 cond_resched(); 4245 return ret; 4246 } 4247 4248 static inline bool 4249 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) 4250 { 4251 /* 4252 * It's possible that cpuset's mems_allowed and the nodemask from 4253 * mempolicy don't intersect. This should be normally dealt with by 4254 * policy_nodemask(), but it's possible to race with cpuset update in 4255 * such a way the check therein was true, and then it became false 4256 * before we got our cpuset_mems_cookie here. 4257 * This assumes that for all allocations, ac->nodemask can come only 4258 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored 4259 * when it does not intersect with the cpuset restrictions) or the 4260 * caller can deal with a violated nodemask. 4261 */ 4262 if (cpusets_enabled() && ac->nodemask && 4263 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) { 4264 ac->nodemask = NULL; 4265 return true; 4266 } 4267 4268 /* 4269 * When updating a task's mems_allowed or mempolicy nodemask, it is 4270 * possible to race with parallel threads in such a way that our 4271 * allocation can fail while the mask is being updated. If we are about 4272 * to fail, check if the cpuset changed during allocation and if so, 4273 * retry. 4274 */ 4275 if (read_mems_allowed_retry(cpuset_mems_cookie)) 4276 return true; 4277 4278 return false; 4279 } 4280 4281 static inline struct page * 4282 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 4283 struct alloc_context *ac) 4284 { 4285 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 4286 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 4287 struct page *page = NULL; 4288 unsigned int alloc_flags; 4289 unsigned long did_some_progress; 4290 enum compact_priority compact_priority; 4291 enum compact_result compact_result; 4292 int compaction_retries; 4293 int no_progress_loops; 4294 unsigned int cpuset_mems_cookie; 4295 int reserve_flags; 4296 4297 /* 4298 * We also sanity check to catch abuse of atomic reserves being used by 4299 * callers that are not in atomic context. 4300 */ 4301 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 4302 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 4303 gfp_mask &= ~__GFP_ATOMIC; 4304 4305 retry_cpuset: 4306 compaction_retries = 0; 4307 no_progress_loops = 0; 4308 compact_priority = DEF_COMPACT_PRIORITY; 4309 cpuset_mems_cookie = read_mems_allowed_begin(); 4310 4311 /* 4312 * The fast path uses conservative alloc_flags to succeed only until 4313 * kswapd needs to be woken up, and to avoid the cost of setting up 4314 * alloc_flags precisely. So we do that now. 4315 */ 4316 alloc_flags = gfp_to_alloc_flags(gfp_mask); 4317 4318 /* 4319 * We need to recalculate the starting point for the zonelist iterator 4320 * because we might have used different nodemask in the fast path, or 4321 * there was a cpuset modification and we are retrying - otherwise we 4322 * could end up iterating over non-eligible zones endlessly. 4323 */ 4324 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4325 ac->high_zoneidx, ac->nodemask); 4326 if (!ac->preferred_zoneref->zone) 4327 goto nopage; 4328 4329 if (alloc_flags & ALLOC_KSWAPD) 4330 wake_all_kswapds(order, gfp_mask, ac); 4331 4332 /* 4333 * The adjusted alloc_flags might result in immediate success, so try 4334 * that first 4335 */ 4336 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4337 if (page) 4338 goto got_pg; 4339 4340 /* 4341 * For costly allocations, try direct compaction first, as it's likely 4342 * that we have enough base pages and don't need to reclaim. For non- 4343 * movable high-order allocations, do that as well, as compaction will 4344 * try prevent permanent fragmentation by migrating from blocks of the 4345 * same migratetype. 4346 * Don't try this for allocations that are allowed to ignore 4347 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 4348 */ 4349 if (can_direct_reclaim && 4350 (costly_order || 4351 (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 4352 && !gfp_pfmemalloc_allowed(gfp_mask)) { 4353 page = __alloc_pages_direct_compact(gfp_mask, order, 4354 alloc_flags, ac, 4355 INIT_COMPACT_PRIORITY, 4356 &compact_result); 4357 if (page) 4358 goto got_pg; 4359 4360 /* 4361 * Checks for costly allocations with __GFP_NORETRY, which 4362 * includes THP page fault allocations 4363 */ 4364 if (costly_order && (gfp_mask & __GFP_NORETRY)) { 4365 /* 4366 * If compaction is deferred for high-order allocations, 4367 * it is because sync compaction recently failed. If 4368 * this is the case and the caller requested a THP 4369 * allocation, we do not want to heavily disrupt the 4370 * system, so we fail the allocation instead of entering 4371 * direct reclaim. 4372 */ 4373 if (compact_result == COMPACT_DEFERRED) 4374 goto nopage; 4375 4376 /* 4377 * Looks like reclaim/compaction is worth trying, but 4378 * sync compaction could be very expensive, so keep 4379 * using async compaction. 4380 */ 4381 compact_priority = INIT_COMPACT_PRIORITY; 4382 } 4383 } 4384 4385 retry: 4386 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 4387 if (alloc_flags & ALLOC_KSWAPD) 4388 wake_all_kswapds(order, gfp_mask, ac); 4389 4390 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); 4391 if (reserve_flags) 4392 alloc_flags = reserve_flags; 4393 4394 /* 4395 * Reset the nodemask and zonelist iterators if memory policies can be 4396 * ignored. These allocations are high priority and system rather than 4397 * user oriented. 4398 */ 4399 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { 4400 ac->nodemask = NULL; 4401 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4402 ac->high_zoneidx, ac->nodemask); 4403 } 4404 4405 /* Attempt with potentially adjusted zonelist and alloc_flags */ 4406 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4407 if (page) 4408 goto got_pg; 4409 4410 /* Caller is not willing to reclaim, we can't balance anything */ 4411 if (!can_direct_reclaim) 4412 goto nopage; 4413 4414 /* Avoid recursion of direct reclaim */ 4415 if (current->flags & PF_MEMALLOC) 4416 goto nopage; 4417 4418 /* Try direct reclaim and then allocating */ 4419 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 4420 &did_some_progress); 4421 if (page) 4422 goto got_pg; 4423 4424 /* Try direct compaction and then allocating */ 4425 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 4426 compact_priority, &compact_result); 4427 if (page) 4428 goto got_pg; 4429 4430 /* Do not loop if specifically requested */ 4431 if (gfp_mask & __GFP_NORETRY) 4432 goto nopage; 4433 4434 /* 4435 * Do not retry costly high order allocations unless they are 4436 * __GFP_RETRY_MAYFAIL 4437 */ 4438 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) 4439 goto nopage; 4440 4441 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 4442 did_some_progress > 0, &no_progress_loops)) 4443 goto retry; 4444 4445 /* 4446 * It doesn't make any sense to retry for the compaction if the order-0 4447 * reclaim is not able to make any progress because the current 4448 * implementation of the compaction depends on the sufficient amount 4449 * of free memory (see __compaction_suitable) 4450 */ 4451 if (did_some_progress > 0 && 4452 should_compact_retry(ac, order, alloc_flags, 4453 compact_result, &compact_priority, 4454 &compaction_retries)) 4455 goto retry; 4456 4457 4458 /* Deal with possible cpuset update races before we start OOM killing */ 4459 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4460 goto retry_cpuset; 4461 4462 /* Reclaim has failed us, start killing things */ 4463 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 4464 if (page) 4465 goto got_pg; 4466 4467 /* Avoid allocations with no watermarks from looping endlessly */ 4468 if (tsk_is_oom_victim(current) && 4469 (alloc_flags == ALLOC_OOM || 4470 (gfp_mask & __GFP_NOMEMALLOC))) 4471 goto nopage; 4472 4473 /* Retry as long as the OOM killer is making progress */ 4474 if (did_some_progress) { 4475 no_progress_loops = 0; 4476 goto retry; 4477 } 4478 4479 nopage: 4480 /* Deal with possible cpuset update races before we fail */ 4481 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4482 goto retry_cpuset; 4483 4484 /* 4485 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure 4486 * we always retry 4487 */ 4488 if (gfp_mask & __GFP_NOFAIL) { 4489 /* 4490 * All existing users of the __GFP_NOFAIL are blockable, so warn 4491 * of any new users that actually require GFP_NOWAIT 4492 */ 4493 if (WARN_ON_ONCE(!can_direct_reclaim)) 4494 goto fail; 4495 4496 /* 4497 * PF_MEMALLOC request from this context is rather bizarre 4498 * because we cannot reclaim anything and only can loop waiting 4499 * for somebody to do a work for us 4500 */ 4501 WARN_ON_ONCE(current->flags & PF_MEMALLOC); 4502 4503 /* 4504 * non failing costly orders are a hard requirement which we 4505 * are not prepared for much so let's warn about these users 4506 * so that we can identify them and convert them to something 4507 * else. 4508 */ 4509 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER); 4510 4511 /* 4512 * Help non-failing allocations by giving them access to memory 4513 * reserves but do not use ALLOC_NO_WATERMARKS because this 4514 * could deplete whole memory reserves which would just make 4515 * the situation worse 4516 */ 4517 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); 4518 if (page) 4519 goto got_pg; 4520 4521 cond_resched(); 4522 goto retry; 4523 } 4524 fail: 4525 warn_alloc(gfp_mask, ac->nodemask, 4526 "page allocation failure: order:%u", order); 4527 got_pg: 4528 return page; 4529 } 4530 4531 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, 4532 int preferred_nid, nodemask_t *nodemask, 4533 struct alloc_context *ac, gfp_t *alloc_mask, 4534 unsigned int *alloc_flags) 4535 { 4536 ac->high_zoneidx = gfp_zone(gfp_mask); 4537 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); 4538 ac->nodemask = nodemask; 4539 ac->migratetype = gfpflags_to_migratetype(gfp_mask); 4540 4541 if (cpusets_enabled()) { 4542 *alloc_mask |= __GFP_HARDWALL; 4543 if (!ac->nodemask) 4544 ac->nodemask = &cpuset_current_mems_allowed; 4545 else 4546 *alloc_flags |= ALLOC_CPUSET; 4547 } 4548 4549 fs_reclaim_acquire(gfp_mask); 4550 fs_reclaim_release(gfp_mask); 4551 4552 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 4553 4554 if (should_fail_alloc_page(gfp_mask, order)) 4555 return false; 4556 4557 if (IS_ENABLED(CONFIG_CMA) && ac->migratetype == MIGRATE_MOVABLE) 4558 *alloc_flags |= ALLOC_CMA; 4559 4560 return true; 4561 } 4562 4563 /* Determine whether to spread dirty pages and what the first usable zone */ 4564 static inline void finalise_ac(gfp_t gfp_mask, struct alloc_context *ac) 4565 { 4566 /* Dirty zone balancing only done in the fast path */ 4567 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); 4568 4569 /* 4570 * The preferred zone is used for statistics but crucially it is 4571 * also used as the starting point for the zonelist iterator. It 4572 * may get reset for allocations that ignore memory policies. 4573 */ 4574 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4575 ac->high_zoneidx, ac->nodemask); 4576 } 4577 4578 /* 4579 * This is the 'heart' of the zoned buddy allocator. 4580 */ 4581 struct page * 4582 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, 4583 nodemask_t *nodemask) 4584 { 4585 struct page *page; 4586 unsigned int alloc_flags = ALLOC_WMARK_LOW; 4587 gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */ 4588 struct alloc_context ac = { }; 4589 4590 /* 4591 * There are several places where we assume that the order value is sane 4592 * so bail out early if the request is out of bound. 4593 */ 4594 if (unlikely(order >= MAX_ORDER)) { 4595 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); 4596 return NULL; 4597 } 4598 4599 gfp_mask &= gfp_allowed_mask; 4600 alloc_mask = gfp_mask; 4601 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags)) 4602 return NULL; 4603 4604 finalise_ac(gfp_mask, &ac); 4605 4606 /* 4607 * Forbid the first pass from falling back to types that fragment 4608 * memory until all local zones are considered. 4609 */ 4610 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask); 4611 4612 /* First allocation attempt */ 4613 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); 4614 if (likely(page)) 4615 goto out; 4616 4617 /* 4618 * Apply scoped allocation constraints. This is mainly about GFP_NOFS 4619 * resp. GFP_NOIO which has to be inherited for all allocation requests 4620 * from a particular context which has been marked by 4621 * memalloc_no{fs,io}_{save,restore}. 4622 */ 4623 alloc_mask = current_gfp_context(gfp_mask); 4624 ac.spread_dirty_pages = false; 4625 4626 /* 4627 * Restore the original nodemask if it was potentially replaced with 4628 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 4629 */ 4630 if (unlikely(ac.nodemask != nodemask)) 4631 ac.nodemask = nodemask; 4632 4633 page = __alloc_pages_slowpath(alloc_mask, order, &ac); 4634 4635 out: 4636 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && 4637 unlikely(__memcg_kmem_charge(page, gfp_mask, order) != 0)) { 4638 __free_pages(page, order); 4639 page = NULL; 4640 } 4641 4642 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); 4643 4644 return page; 4645 } 4646 EXPORT_SYMBOL(__alloc_pages_nodemask); 4647 4648 /* 4649 * Common helper functions. Never use with __GFP_HIGHMEM because the returned 4650 * address cannot represent highmem pages. Use alloc_pages and then kmap if 4651 * you need to access high mem. 4652 */ 4653 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 4654 { 4655 struct page *page; 4656 4657 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); 4658 if (!page) 4659 return 0; 4660 return (unsigned long) page_address(page); 4661 } 4662 EXPORT_SYMBOL(__get_free_pages); 4663 4664 unsigned long get_zeroed_page(gfp_t gfp_mask) 4665 { 4666 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 4667 } 4668 EXPORT_SYMBOL(get_zeroed_page); 4669 4670 static inline void free_the_page(struct page *page, unsigned int order) 4671 { 4672 if (order == 0) /* Via pcp? */ 4673 free_unref_page(page); 4674 else 4675 __free_pages_ok(page, order); 4676 } 4677 4678 void __free_pages(struct page *page, unsigned int order) 4679 { 4680 if (put_page_testzero(page)) 4681 free_the_page(page, order); 4682 } 4683 EXPORT_SYMBOL(__free_pages); 4684 4685 void free_pages(unsigned long addr, unsigned int order) 4686 { 4687 if (addr != 0) { 4688 VM_BUG_ON(!virt_addr_valid((void *)addr)); 4689 __free_pages(virt_to_page((void *)addr), order); 4690 } 4691 } 4692 4693 EXPORT_SYMBOL(free_pages); 4694 4695 /* 4696 * Page Fragment: 4697 * An arbitrary-length arbitrary-offset area of memory which resides 4698 * within a 0 or higher order page. Multiple fragments within that page 4699 * are individually refcounted, in the page's reference counter. 4700 * 4701 * The page_frag functions below provide a simple allocation framework for 4702 * page fragments. This is used by the network stack and network device 4703 * drivers to provide a backing region of memory for use as either an 4704 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 4705 */ 4706 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 4707 gfp_t gfp_mask) 4708 { 4709 struct page *page = NULL; 4710 gfp_t gfp = gfp_mask; 4711 4712 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4713 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 4714 __GFP_NOMEMALLOC; 4715 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 4716 PAGE_FRAG_CACHE_MAX_ORDER); 4717 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 4718 #endif 4719 if (unlikely(!page)) 4720 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 4721 4722 nc->va = page ? page_address(page) : NULL; 4723 4724 return page; 4725 } 4726 4727 void __page_frag_cache_drain(struct page *page, unsigned int count) 4728 { 4729 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 4730 4731 if (page_ref_sub_and_test(page, count)) 4732 free_the_page(page, compound_order(page)); 4733 } 4734 EXPORT_SYMBOL(__page_frag_cache_drain); 4735 4736 void *page_frag_alloc(struct page_frag_cache *nc, 4737 unsigned int fragsz, gfp_t gfp_mask) 4738 { 4739 unsigned int size = PAGE_SIZE; 4740 struct page *page; 4741 int offset; 4742 4743 if (unlikely(!nc->va)) { 4744 refill: 4745 page = __page_frag_cache_refill(nc, gfp_mask); 4746 if (!page) 4747 return NULL; 4748 4749 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4750 /* if size can vary use size else just use PAGE_SIZE */ 4751 size = nc->size; 4752 #endif 4753 /* Even if we own the page, we do not use atomic_set(). 4754 * This would break get_page_unless_zero() users. 4755 */ 4756 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 4757 4758 /* reset page count bias and offset to start of new frag */ 4759 nc->pfmemalloc = page_is_pfmemalloc(page); 4760 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 4761 nc->offset = size; 4762 } 4763 4764 offset = nc->offset - fragsz; 4765 if (unlikely(offset < 0)) { 4766 page = virt_to_page(nc->va); 4767 4768 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 4769 goto refill; 4770 4771 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4772 /* if size can vary use size else just use PAGE_SIZE */ 4773 size = nc->size; 4774 #endif 4775 /* OK, page count is 0, we can safely set it */ 4776 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 4777 4778 /* reset page count bias and offset to start of new frag */ 4779 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 4780 offset = size - fragsz; 4781 } 4782 4783 nc->pagecnt_bias--; 4784 nc->offset = offset; 4785 4786 return nc->va + offset; 4787 } 4788 EXPORT_SYMBOL(page_frag_alloc); 4789 4790 /* 4791 * Frees a page fragment allocated out of either a compound or order 0 page. 4792 */ 4793 void page_frag_free(void *addr) 4794 { 4795 struct page *page = virt_to_head_page(addr); 4796 4797 if (unlikely(put_page_testzero(page))) 4798 free_the_page(page, compound_order(page)); 4799 } 4800 EXPORT_SYMBOL(page_frag_free); 4801 4802 static void *make_alloc_exact(unsigned long addr, unsigned int order, 4803 size_t size) 4804 { 4805 if (addr) { 4806 unsigned long alloc_end = addr + (PAGE_SIZE << order); 4807 unsigned long used = addr + PAGE_ALIGN(size); 4808 4809 split_page(virt_to_page((void *)addr), order); 4810 while (used < alloc_end) { 4811 free_page(used); 4812 used += PAGE_SIZE; 4813 } 4814 } 4815 return (void *)addr; 4816 } 4817 4818 /** 4819 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 4820 * @size: the number of bytes to allocate 4821 * @gfp_mask: GFP flags for the allocation 4822 * 4823 * This function is similar to alloc_pages(), except that it allocates the 4824 * minimum number of pages to satisfy the request. alloc_pages() can only 4825 * allocate memory in power-of-two pages. 4826 * 4827 * This function is also limited by MAX_ORDER. 4828 * 4829 * Memory allocated by this function must be released by free_pages_exact(). 4830 * 4831 * Return: pointer to the allocated area or %NULL in case of error. 4832 */ 4833 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 4834 { 4835 unsigned int order = get_order(size); 4836 unsigned long addr; 4837 4838 addr = __get_free_pages(gfp_mask, order); 4839 return make_alloc_exact(addr, order, size); 4840 } 4841 EXPORT_SYMBOL(alloc_pages_exact); 4842 4843 /** 4844 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 4845 * pages on a node. 4846 * @nid: the preferred node ID where memory should be allocated 4847 * @size: the number of bytes to allocate 4848 * @gfp_mask: GFP flags for the allocation 4849 * 4850 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 4851 * back. 4852 * 4853 * Return: pointer to the allocated area or %NULL in case of error. 4854 */ 4855 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 4856 { 4857 unsigned int order = get_order(size); 4858 struct page *p = alloc_pages_node(nid, gfp_mask, order); 4859 if (!p) 4860 return NULL; 4861 return make_alloc_exact((unsigned long)page_address(p), order, size); 4862 } 4863 4864 /** 4865 * free_pages_exact - release memory allocated via alloc_pages_exact() 4866 * @virt: the value returned by alloc_pages_exact. 4867 * @size: size of allocation, same value as passed to alloc_pages_exact(). 4868 * 4869 * Release the memory allocated by a previous call to alloc_pages_exact. 4870 */ 4871 void free_pages_exact(void *virt, size_t size) 4872 { 4873 unsigned long addr = (unsigned long)virt; 4874 unsigned long end = addr + PAGE_ALIGN(size); 4875 4876 while (addr < end) { 4877 free_page(addr); 4878 addr += PAGE_SIZE; 4879 } 4880 } 4881 EXPORT_SYMBOL(free_pages_exact); 4882 4883 /** 4884 * nr_free_zone_pages - count number of pages beyond high watermark 4885 * @offset: The zone index of the highest zone 4886 * 4887 * nr_free_zone_pages() counts the number of pages which are beyond the 4888 * high watermark within all zones at or below a given zone index. For each 4889 * zone, the number of pages is calculated as: 4890 * 4891 * nr_free_zone_pages = managed_pages - high_pages 4892 * 4893 * Return: number of pages beyond high watermark. 4894 */ 4895 static unsigned long nr_free_zone_pages(int offset) 4896 { 4897 struct zoneref *z; 4898 struct zone *zone; 4899 4900 /* Just pick one node, since fallback list is circular */ 4901 unsigned long sum = 0; 4902 4903 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 4904 4905 for_each_zone_zonelist(zone, z, zonelist, offset) { 4906 unsigned long size = zone_managed_pages(zone); 4907 unsigned long high = high_wmark_pages(zone); 4908 if (size > high) 4909 sum += size - high; 4910 } 4911 4912 return sum; 4913 } 4914 4915 /** 4916 * nr_free_buffer_pages - count number of pages beyond high watermark 4917 * 4918 * nr_free_buffer_pages() counts the number of pages which are beyond the high 4919 * watermark within ZONE_DMA and ZONE_NORMAL. 4920 * 4921 * Return: number of pages beyond high watermark within ZONE_DMA and 4922 * ZONE_NORMAL. 4923 */ 4924 unsigned long nr_free_buffer_pages(void) 4925 { 4926 return nr_free_zone_pages(gfp_zone(GFP_USER)); 4927 } 4928 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 4929 4930 /** 4931 * nr_free_pagecache_pages - count number of pages beyond high watermark 4932 * 4933 * nr_free_pagecache_pages() counts the number of pages which are beyond the 4934 * high watermark within all zones. 4935 * 4936 * Return: number of pages beyond high watermark within all zones. 4937 */ 4938 unsigned long nr_free_pagecache_pages(void) 4939 { 4940 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 4941 } 4942 4943 static inline void show_node(struct zone *zone) 4944 { 4945 if (IS_ENABLED(CONFIG_NUMA)) 4946 printk("Node %d ", zone_to_nid(zone)); 4947 } 4948 4949 long si_mem_available(void) 4950 { 4951 long available; 4952 unsigned long pagecache; 4953 unsigned long wmark_low = 0; 4954 unsigned long pages[NR_LRU_LISTS]; 4955 unsigned long reclaimable; 4956 struct zone *zone; 4957 int lru; 4958 4959 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 4960 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 4961 4962 for_each_zone(zone) 4963 wmark_low += low_wmark_pages(zone); 4964 4965 /* 4966 * Estimate the amount of memory available for userspace allocations, 4967 * without causing swapping. 4968 */ 4969 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages; 4970 4971 /* 4972 * Not all the page cache can be freed, otherwise the system will 4973 * start swapping. Assume at least half of the page cache, or the 4974 * low watermark worth of cache, needs to stay. 4975 */ 4976 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 4977 pagecache -= min(pagecache / 2, wmark_low); 4978 available += pagecache; 4979 4980 /* 4981 * Part of the reclaimable slab and other kernel memory consists of 4982 * items that are in use, and cannot be freed. Cap this estimate at the 4983 * low watermark. 4984 */ 4985 reclaimable = global_node_page_state(NR_SLAB_RECLAIMABLE) + 4986 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE); 4987 available += reclaimable - min(reclaimable / 2, wmark_low); 4988 4989 if (available < 0) 4990 available = 0; 4991 return available; 4992 } 4993 EXPORT_SYMBOL_GPL(si_mem_available); 4994 4995 void si_meminfo(struct sysinfo *val) 4996 { 4997 val->totalram = totalram_pages(); 4998 val->sharedram = global_node_page_state(NR_SHMEM); 4999 val->freeram = global_zone_page_state(NR_FREE_PAGES); 5000 val->bufferram = nr_blockdev_pages(); 5001 val->totalhigh = totalhigh_pages(); 5002 val->freehigh = nr_free_highpages(); 5003 val->mem_unit = PAGE_SIZE; 5004 } 5005 5006 EXPORT_SYMBOL(si_meminfo); 5007 5008 #ifdef CONFIG_NUMA 5009 void si_meminfo_node(struct sysinfo *val, int nid) 5010 { 5011 int zone_type; /* needs to be signed */ 5012 unsigned long managed_pages = 0; 5013 unsigned long managed_highpages = 0; 5014 unsigned long free_highpages = 0; 5015 pg_data_t *pgdat = NODE_DATA(nid); 5016 5017 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 5018 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]); 5019 val->totalram = managed_pages; 5020 val->sharedram = node_page_state(pgdat, NR_SHMEM); 5021 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 5022 #ifdef CONFIG_HIGHMEM 5023 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 5024 struct zone *zone = &pgdat->node_zones[zone_type]; 5025 5026 if (is_highmem(zone)) { 5027 managed_highpages += zone_managed_pages(zone); 5028 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 5029 } 5030 } 5031 val->totalhigh = managed_highpages; 5032 val->freehigh = free_highpages; 5033 #else 5034 val->totalhigh = managed_highpages; 5035 val->freehigh = free_highpages; 5036 #endif 5037 val->mem_unit = PAGE_SIZE; 5038 } 5039 #endif 5040 5041 /* 5042 * Determine whether the node should be displayed or not, depending on whether 5043 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 5044 */ 5045 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) 5046 { 5047 if (!(flags & SHOW_MEM_FILTER_NODES)) 5048 return false; 5049 5050 /* 5051 * no node mask - aka implicit memory numa policy. Do not bother with 5052 * the synchronization - read_mems_allowed_begin - because we do not 5053 * have to be precise here. 5054 */ 5055 if (!nodemask) 5056 nodemask = &cpuset_current_mems_allowed; 5057 5058 return !node_isset(nid, *nodemask); 5059 } 5060 5061 #define K(x) ((x) << (PAGE_SHIFT-10)) 5062 5063 static void show_migration_types(unsigned char type) 5064 { 5065 static const char types[MIGRATE_TYPES] = { 5066 [MIGRATE_UNMOVABLE] = 'U', 5067 [MIGRATE_MOVABLE] = 'M', 5068 [MIGRATE_RECLAIMABLE] = 'E', 5069 [MIGRATE_HIGHATOMIC] = 'H', 5070 #ifdef CONFIG_CMA 5071 [MIGRATE_CMA] = 'C', 5072 #endif 5073 #ifdef CONFIG_MEMORY_ISOLATION 5074 [MIGRATE_ISOLATE] = 'I', 5075 #endif 5076 }; 5077 char tmp[MIGRATE_TYPES + 1]; 5078 char *p = tmp; 5079 int i; 5080 5081 for (i = 0; i < MIGRATE_TYPES; i++) { 5082 if (type & (1 << i)) 5083 *p++ = types[i]; 5084 } 5085 5086 *p = '\0'; 5087 printk(KERN_CONT "(%s) ", tmp); 5088 } 5089 5090 /* 5091 * Show free area list (used inside shift_scroll-lock stuff) 5092 * We also calculate the percentage fragmentation. We do this by counting the 5093 * memory on each free list with the exception of the first item on the list. 5094 * 5095 * Bits in @filter: 5096 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 5097 * cpuset. 5098 */ 5099 void show_free_areas(unsigned int filter, nodemask_t *nodemask) 5100 { 5101 unsigned long free_pcp = 0; 5102 int cpu; 5103 struct zone *zone; 5104 pg_data_t *pgdat; 5105 5106 for_each_populated_zone(zone) { 5107 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5108 continue; 5109 5110 for_each_online_cpu(cpu) 5111 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 5112 } 5113 5114 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 5115 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 5116 " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n" 5117 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 5118 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 5119 " free:%lu free_pcp:%lu free_cma:%lu\n", 5120 global_node_page_state(NR_ACTIVE_ANON), 5121 global_node_page_state(NR_INACTIVE_ANON), 5122 global_node_page_state(NR_ISOLATED_ANON), 5123 global_node_page_state(NR_ACTIVE_FILE), 5124 global_node_page_state(NR_INACTIVE_FILE), 5125 global_node_page_state(NR_ISOLATED_FILE), 5126 global_node_page_state(NR_UNEVICTABLE), 5127 global_node_page_state(NR_FILE_DIRTY), 5128 global_node_page_state(NR_WRITEBACK), 5129 global_node_page_state(NR_UNSTABLE_NFS), 5130 global_node_page_state(NR_SLAB_RECLAIMABLE), 5131 global_node_page_state(NR_SLAB_UNRECLAIMABLE), 5132 global_node_page_state(NR_FILE_MAPPED), 5133 global_node_page_state(NR_SHMEM), 5134 global_zone_page_state(NR_PAGETABLE), 5135 global_zone_page_state(NR_BOUNCE), 5136 global_zone_page_state(NR_FREE_PAGES), 5137 free_pcp, 5138 global_zone_page_state(NR_FREE_CMA_PAGES)); 5139 5140 for_each_online_pgdat(pgdat) { 5141 if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) 5142 continue; 5143 5144 printk("Node %d" 5145 " active_anon:%lukB" 5146 " inactive_anon:%lukB" 5147 " active_file:%lukB" 5148 " inactive_file:%lukB" 5149 " unevictable:%lukB" 5150 " isolated(anon):%lukB" 5151 " isolated(file):%lukB" 5152 " mapped:%lukB" 5153 " dirty:%lukB" 5154 " writeback:%lukB" 5155 " shmem:%lukB" 5156 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5157 " shmem_thp: %lukB" 5158 " shmem_pmdmapped: %lukB" 5159 " anon_thp: %lukB" 5160 #endif 5161 " writeback_tmp:%lukB" 5162 " unstable:%lukB" 5163 " all_unreclaimable? %s" 5164 "\n", 5165 pgdat->node_id, 5166 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 5167 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 5168 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 5169 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 5170 K(node_page_state(pgdat, NR_UNEVICTABLE)), 5171 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 5172 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 5173 K(node_page_state(pgdat, NR_FILE_MAPPED)), 5174 K(node_page_state(pgdat, NR_FILE_DIRTY)), 5175 K(node_page_state(pgdat, NR_WRITEBACK)), 5176 K(node_page_state(pgdat, NR_SHMEM)), 5177 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5178 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), 5179 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) 5180 * HPAGE_PMD_NR), 5181 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR), 5182 #endif 5183 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 5184 K(node_page_state(pgdat, NR_UNSTABLE_NFS)), 5185 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ? 5186 "yes" : "no"); 5187 } 5188 5189 for_each_populated_zone(zone) { 5190 int i; 5191 5192 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5193 continue; 5194 5195 free_pcp = 0; 5196 for_each_online_cpu(cpu) 5197 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 5198 5199 show_node(zone); 5200 printk(KERN_CONT 5201 "%s" 5202 " free:%lukB" 5203 " min:%lukB" 5204 " low:%lukB" 5205 " high:%lukB" 5206 " active_anon:%lukB" 5207 " inactive_anon:%lukB" 5208 " active_file:%lukB" 5209 " inactive_file:%lukB" 5210 " unevictable:%lukB" 5211 " writepending:%lukB" 5212 " present:%lukB" 5213 " managed:%lukB" 5214 " mlocked:%lukB" 5215 " kernel_stack:%lukB" 5216 " pagetables:%lukB" 5217 " bounce:%lukB" 5218 " free_pcp:%lukB" 5219 " local_pcp:%ukB" 5220 " free_cma:%lukB" 5221 "\n", 5222 zone->name, 5223 K(zone_page_state(zone, NR_FREE_PAGES)), 5224 K(min_wmark_pages(zone)), 5225 K(low_wmark_pages(zone)), 5226 K(high_wmark_pages(zone)), 5227 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 5228 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 5229 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 5230 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 5231 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 5232 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 5233 K(zone->present_pages), 5234 K(zone_managed_pages(zone)), 5235 K(zone_page_state(zone, NR_MLOCK)), 5236 zone_page_state(zone, NR_KERNEL_STACK_KB), 5237 K(zone_page_state(zone, NR_PAGETABLE)), 5238 K(zone_page_state(zone, NR_BOUNCE)), 5239 K(free_pcp), 5240 K(this_cpu_read(zone->pageset->pcp.count)), 5241 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 5242 printk("lowmem_reserve[]:"); 5243 for (i = 0; i < MAX_NR_ZONES; i++) 5244 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); 5245 printk(KERN_CONT "\n"); 5246 } 5247 5248 for_each_populated_zone(zone) { 5249 unsigned int order; 5250 unsigned long nr[MAX_ORDER], flags, total = 0; 5251 unsigned char types[MAX_ORDER]; 5252 5253 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5254 continue; 5255 show_node(zone); 5256 printk(KERN_CONT "%s: ", zone->name); 5257 5258 spin_lock_irqsave(&zone->lock, flags); 5259 for (order = 0; order < MAX_ORDER; order++) { 5260 struct free_area *area = &zone->free_area[order]; 5261 int type; 5262 5263 nr[order] = area->nr_free; 5264 total += nr[order] << order; 5265 5266 types[order] = 0; 5267 for (type = 0; type < MIGRATE_TYPES; type++) { 5268 if (!list_empty(&area->free_list[type])) 5269 types[order] |= 1 << type; 5270 } 5271 } 5272 spin_unlock_irqrestore(&zone->lock, flags); 5273 for (order = 0; order < MAX_ORDER; order++) { 5274 printk(KERN_CONT "%lu*%lukB ", 5275 nr[order], K(1UL) << order); 5276 if (nr[order]) 5277 show_migration_types(types[order]); 5278 } 5279 printk(KERN_CONT "= %lukB\n", K(total)); 5280 } 5281 5282 hugetlb_show_meminfo(); 5283 5284 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 5285 5286 show_swap_cache_info(); 5287 } 5288 5289 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 5290 { 5291 zoneref->zone = zone; 5292 zoneref->zone_idx = zone_idx(zone); 5293 } 5294 5295 /* 5296 * Builds allocation fallback zone lists. 5297 * 5298 * Add all populated zones of a node to the zonelist. 5299 */ 5300 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) 5301 { 5302 struct zone *zone; 5303 enum zone_type zone_type = MAX_NR_ZONES; 5304 int nr_zones = 0; 5305 5306 do { 5307 zone_type--; 5308 zone = pgdat->node_zones + zone_type; 5309 if (managed_zone(zone)) { 5310 zoneref_set_zone(zone, &zonerefs[nr_zones++]); 5311 check_highest_zone(zone_type); 5312 } 5313 } while (zone_type); 5314 5315 return nr_zones; 5316 } 5317 5318 #ifdef CONFIG_NUMA 5319 5320 static int __parse_numa_zonelist_order(char *s) 5321 { 5322 /* 5323 * We used to support different zonlists modes but they turned 5324 * out to be just not useful. Let's keep the warning in place 5325 * if somebody still use the cmd line parameter so that we do 5326 * not fail it silently 5327 */ 5328 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) { 5329 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s); 5330 return -EINVAL; 5331 } 5332 return 0; 5333 } 5334 5335 static __init int setup_numa_zonelist_order(char *s) 5336 { 5337 if (!s) 5338 return 0; 5339 5340 return __parse_numa_zonelist_order(s); 5341 } 5342 early_param("numa_zonelist_order", setup_numa_zonelist_order); 5343 5344 char numa_zonelist_order[] = "Node"; 5345 5346 /* 5347 * sysctl handler for numa_zonelist_order 5348 */ 5349 int numa_zonelist_order_handler(struct ctl_table *table, int write, 5350 void __user *buffer, size_t *length, 5351 loff_t *ppos) 5352 { 5353 char *str; 5354 int ret; 5355 5356 if (!write) 5357 return proc_dostring(table, write, buffer, length, ppos); 5358 str = memdup_user_nul(buffer, 16); 5359 if (IS_ERR(str)) 5360 return PTR_ERR(str); 5361 5362 ret = __parse_numa_zonelist_order(str); 5363 kfree(str); 5364 return ret; 5365 } 5366 5367 5368 #define MAX_NODE_LOAD (nr_online_nodes) 5369 static int node_load[MAX_NUMNODES]; 5370 5371 /** 5372 * find_next_best_node - find the next node that should appear in a given node's fallback list 5373 * @node: node whose fallback list we're appending 5374 * @used_node_mask: nodemask_t of already used nodes 5375 * 5376 * We use a number of factors to determine which is the next node that should 5377 * appear on a given node's fallback list. The node should not have appeared 5378 * already in @node's fallback list, and it should be the next closest node 5379 * according to the distance array (which contains arbitrary distance values 5380 * from each node to each node in the system), and should also prefer nodes 5381 * with no CPUs, since presumably they'll have very little allocation pressure 5382 * on them otherwise. 5383 * 5384 * Return: node id of the found node or %NUMA_NO_NODE if no node is found. 5385 */ 5386 static int find_next_best_node(int node, nodemask_t *used_node_mask) 5387 { 5388 int n, val; 5389 int min_val = INT_MAX; 5390 int best_node = NUMA_NO_NODE; 5391 const struct cpumask *tmp = cpumask_of_node(0); 5392 5393 /* Use the local node if we haven't already */ 5394 if (!node_isset(node, *used_node_mask)) { 5395 node_set(node, *used_node_mask); 5396 return node; 5397 } 5398 5399 for_each_node_state(n, N_MEMORY) { 5400 5401 /* Don't want a node to appear more than once */ 5402 if (node_isset(n, *used_node_mask)) 5403 continue; 5404 5405 /* Use the distance array to find the distance */ 5406 val = node_distance(node, n); 5407 5408 /* Penalize nodes under us ("prefer the next node") */ 5409 val += (n < node); 5410 5411 /* Give preference to headless and unused nodes */ 5412 tmp = cpumask_of_node(n); 5413 if (!cpumask_empty(tmp)) 5414 val += PENALTY_FOR_NODE_WITH_CPUS; 5415 5416 /* Slight preference for less loaded node */ 5417 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 5418 val += node_load[n]; 5419 5420 if (val < min_val) { 5421 min_val = val; 5422 best_node = n; 5423 } 5424 } 5425 5426 if (best_node >= 0) 5427 node_set(best_node, *used_node_mask); 5428 5429 return best_node; 5430 } 5431 5432 5433 /* 5434 * Build zonelists ordered by node and zones within node. 5435 * This results in maximum locality--normal zone overflows into local 5436 * DMA zone, if any--but risks exhausting DMA zone. 5437 */ 5438 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, 5439 unsigned nr_nodes) 5440 { 5441 struct zoneref *zonerefs; 5442 int i; 5443 5444 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5445 5446 for (i = 0; i < nr_nodes; i++) { 5447 int nr_zones; 5448 5449 pg_data_t *node = NODE_DATA(node_order[i]); 5450 5451 nr_zones = build_zonerefs_node(node, zonerefs); 5452 zonerefs += nr_zones; 5453 } 5454 zonerefs->zone = NULL; 5455 zonerefs->zone_idx = 0; 5456 } 5457 5458 /* 5459 * Build gfp_thisnode zonelists 5460 */ 5461 static void build_thisnode_zonelists(pg_data_t *pgdat) 5462 { 5463 struct zoneref *zonerefs; 5464 int nr_zones; 5465 5466 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; 5467 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5468 zonerefs += nr_zones; 5469 zonerefs->zone = NULL; 5470 zonerefs->zone_idx = 0; 5471 } 5472 5473 /* 5474 * Build zonelists ordered by zone and nodes within zones. 5475 * This results in conserving DMA zone[s] until all Normal memory is 5476 * exhausted, but results in overflowing to remote node while memory 5477 * may still exist in local DMA zone. 5478 */ 5479 5480 static void build_zonelists(pg_data_t *pgdat) 5481 { 5482 static int node_order[MAX_NUMNODES]; 5483 int node, load, nr_nodes = 0; 5484 nodemask_t used_mask; 5485 int local_node, prev_node; 5486 5487 /* NUMA-aware ordering of nodes */ 5488 local_node = pgdat->node_id; 5489 load = nr_online_nodes; 5490 prev_node = local_node; 5491 nodes_clear(used_mask); 5492 5493 memset(node_order, 0, sizeof(node_order)); 5494 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 5495 /* 5496 * We don't want to pressure a particular node. 5497 * So adding penalty to the first node in same 5498 * distance group to make it round-robin. 5499 */ 5500 if (node_distance(local_node, node) != 5501 node_distance(local_node, prev_node)) 5502 node_load[node] = load; 5503 5504 node_order[nr_nodes++] = node; 5505 prev_node = node; 5506 load--; 5507 } 5508 5509 build_zonelists_in_node_order(pgdat, node_order, nr_nodes); 5510 build_thisnode_zonelists(pgdat); 5511 } 5512 5513 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5514 /* 5515 * Return node id of node used for "local" allocations. 5516 * I.e., first node id of first zone in arg node's generic zonelist. 5517 * Used for initializing percpu 'numa_mem', which is used primarily 5518 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 5519 */ 5520 int local_memory_node(int node) 5521 { 5522 struct zoneref *z; 5523 5524 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 5525 gfp_zone(GFP_KERNEL), 5526 NULL); 5527 return zone_to_nid(z->zone); 5528 } 5529 #endif 5530 5531 static void setup_min_unmapped_ratio(void); 5532 static void setup_min_slab_ratio(void); 5533 #else /* CONFIG_NUMA */ 5534 5535 static void build_zonelists(pg_data_t *pgdat) 5536 { 5537 int node, local_node; 5538 struct zoneref *zonerefs; 5539 int nr_zones; 5540 5541 local_node = pgdat->node_id; 5542 5543 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5544 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5545 zonerefs += nr_zones; 5546 5547 /* 5548 * Now we build the zonelist so that it contains the zones 5549 * of all the other nodes. 5550 * We don't want to pressure a particular node, so when 5551 * building the zones for node N, we make sure that the 5552 * zones coming right after the local ones are those from 5553 * node N+1 (modulo N) 5554 */ 5555 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 5556 if (!node_online(node)) 5557 continue; 5558 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5559 zonerefs += nr_zones; 5560 } 5561 for (node = 0; node < local_node; node++) { 5562 if (!node_online(node)) 5563 continue; 5564 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5565 zonerefs += nr_zones; 5566 } 5567 5568 zonerefs->zone = NULL; 5569 zonerefs->zone_idx = 0; 5570 } 5571 5572 #endif /* CONFIG_NUMA */ 5573 5574 /* 5575 * Boot pageset table. One per cpu which is going to be used for all 5576 * zones and all nodes. The parameters will be set in such a way 5577 * that an item put on a list will immediately be handed over to 5578 * the buddy list. This is safe since pageset manipulation is done 5579 * with interrupts disabled. 5580 * 5581 * The boot_pagesets must be kept even after bootup is complete for 5582 * unused processors and/or zones. They do play a role for bootstrapping 5583 * hotplugged processors. 5584 * 5585 * zoneinfo_show() and maybe other functions do 5586 * not check if the processor is online before following the pageset pointer. 5587 * Other parts of the kernel may not check if the zone is available. 5588 */ 5589 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); 5590 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); 5591 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 5592 5593 static void __build_all_zonelists(void *data) 5594 { 5595 int nid; 5596 int __maybe_unused cpu; 5597 pg_data_t *self = data; 5598 static DEFINE_SPINLOCK(lock); 5599 5600 spin_lock(&lock); 5601 5602 #ifdef CONFIG_NUMA 5603 memset(node_load, 0, sizeof(node_load)); 5604 #endif 5605 5606 /* 5607 * This node is hotadded and no memory is yet present. So just 5608 * building zonelists is fine - no need to touch other nodes. 5609 */ 5610 if (self && !node_online(self->node_id)) { 5611 build_zonelists(self); 5612 } else { 5613 for_each_online_node(nid) { 5614 pg_data_t *pgdat = NODE_DATA(nid); 5615 5616 build_zonelists(pgdat); 5617 } 5618 5619 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5620 /* 5621 * We now know the "local memory node" for each node-- 5622 * i.e., the node of the first zone in the generic zonelist. 5623 * Set up numa_mem percpu variable for on-line cpus. During 5624 * boot, only the boot cpu should be on-line; we'll init the 5625 * secondary cpus' numa_mem as they come on-line. During 5626 * node/memory hotplug, we'll fixup all on-line cpus. 5627 */ 5628 for_each_online_cpu(cpu) 5629 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 5630 #endif 5631 } 5632 5633 spin_unlock(&lock); 5634 } 5635 5636 static noinline void __init 5637 build_all_zonelists_init(void) 5638 { 5639 int cpu; 5640 5641 __build_all_zonelists(NULL); 5642 5643 /* 5644 * Initialize the boot_pagesets that are going to be used 5645 * for bootstrapping processors. The real pagesets for 5646 * each zone will be allocated later when the per cpu 5647 * allocator is available. 5648 * 5649 * boot_pagesets are used also for bootstrapping offline 5650 * cpus if the system is already booted because the pagesets 5651 * are needed to initialize allocators on a specific cpu too. 5652 * F.e. the percpu allocator needs the page allocator which 5653 * needs the percpu allocator in order to allocate its pagesets 5654 * (a chicken-egg dilemma). 5655 */ 5656 for_each_possible_cpu(cpu) 5657 setup_pageset(&per_cpu(boot_pageset, cpu), 0); 5658 5659 mminit_verify_zonelist(); 5660 cpuset_init_current_mems_allowed(); 5661 } 5662 5663 /* 5664 * unless system_state == SYSTEM_BOOTING. 5665 * 5666 * __ref due to call of __init annotated helper build_all_zonelists_init 5667 * [protected by SYSTEM_BOOTING]. 5668 */ 5669 void __ref build_all_zonelists(pg_data_t *pgdat) 5670 { 5671 if (system_state == SYSTEM_BOOTING) { 5672 build_all_zonelists_init(); 5673 } else { 5674 __build_all_zonelists(pgdat); 5675 /* cpuset refresh routine should be here */ 5676 } 5677 vm_total_pages = nr_free_pagecache_pages(); 5678 /* 5679 * Disable grouping by mobility if the number of pages in the 5680 * system is too low to allow the mechanism to work. It would be 5681 * more accurate, but expensive to check per-zone. This check is 5682 * made on memory-hotadd so a system can start with mobility 5683 * disabled and enable it later 5684 */ 5685 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 5686 page_group_by_mobility_disabled = 1; 5687 else 5688 page_group_by_mobility_disabled = 0; 5689 5690 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n", 5691 nr_online_nodes, 5692 page_group_by_mobility_disabled ? "off" : "on", 5693 vm_total_pages); 5694 #ifdef CONFIG_NUMA 5695 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 5696 #endif 5697 } 5698 5699 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 5700 static bool __meminit 5701 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 5702 { 5703 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5704 static struct memblock_region *r; 5705 5706 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 5707 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 5708 for_each_memblock(memory, r) { 5709 if (*pfn < memblock_region_memory_end_pfn(r)) 5710 break; 5711 } 5712 } 5713 if (*pfn >= memblock_region_memory_base_pfn(r) && 5714 memblock_is_mirror(r)) { 5715 *pfn = memblock_region_memory_end_pfn(r); 5716 return true; 5717 } 5718 } 5719 #endif 5720 return false; 5721 } 5722 5723 /* 5724 * Initially all pages are reserved - free ones are freed 5725 * up by memblock_free_all() once the early boot process is 5726 * done. Non-atomic initialization, single-pass. 5727 */ 5728 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, 5729 unsigned long start_pfn, enum memmap_context context, 5730 struct vmem_altmap *altmap) 5731 { 5732 unsigned long pfn, end_pfn = start_pfn + size; 5733 struct page *page; 5734 5735 if (highest_memmap_pfn < end_pfn - 1) 5736 highest_memmap_pfn = end_pfn - 1; 5737 5738 #ifdef CONFIG_ZONE_DEVICE 5739 /* 5740 * Honor reservation requested by the driver for this ZONE_DEVICE 5741 * memory. We limit the total number of pages to initialize to just 5742 * those that might contain the memory mapping. We will defer the 5743 * ZONE_DEVICE page initialization until after we have released 5744 * the hotplug lock. 5745 */ 5746 if (zone == ZONE_DEVICE) { 5747 if (!altmap) 5748 return; 5749 5750 if (start_pfn == altmap->base_pfn) 5751 start_pfn += altmap->reserve; 5752 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 5753 } 5754 #endif 5755 5756 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 5757 /* 5758 * There can be holes in boot-time mem_map[]s handed to this 5759 * function. They do not exist on hotplugged memory. 5760 */ 5761 if (context == MEMMAP_EARLY) { 5762 if (!early_pfn_valid(pfn)) 5763 continue; 5764 if (!early_pfn_in_nid(pfn, nid)) 5765 continue; 5766 if (overlap_memmap_init(zone, &pfn)) 5767 continue; 5768 if (defer_init(nid, pfn, end_pfn)) 5769 break; 5770 } 5771 5772 page = pfn_to_page(pfn); 5773 __init_single_page(page, pfn, zone, nid); 5774 if (context == MEMMAP_HOTPLUG) 5775 __SetPageReserved(page); 5776 5777 /* 5778 * Mark the block movable so that blocks are reserved for 5779 * movable at startup. This will force kernel allocations 5780 * to reserve their blocks rather than leaking throughout 5781 * the address space during boot when many long-lived 5782 * kernel allocations are made. 5783 * 5784 * bitmap is created for zone's valid pfn range. but memmap 5785 * can be created for invalid pages (for alignment) 5786 * check here not to call set_pageblock_migratetype() against 5787 * pfn out of zone. 5788 */ 5789 if (!(pfn & (pageblock_nr_pages - 1))) { 5790 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 5791 cond_resched(); 5792 } 5793 } 5794 } 5795 5796 #ifdef CONFIG_ZONE_DEVICE 5797 void __ref memmap_init_zone_device(struct zone *zone, 5798 unsigned long start_pfn, 5799 unsigned long size, 5800 struct dev_pagemap *pgmap) 5801 { 5802 unsigned long pfn, end_pfn = start_pfn + size; 5803 struct pglist_data *pgdat = zone->zone_pgdat; 5804 unsigned long zone_idx = zone_idx(zone); 5805 unsigned long start = jiffies; 5806 int nid = pgdat->node_id; 5807 5808 if (WARN_ON_ONCE(!pgmap || !is_dev_zone(zone))) 5809 return; 5810 5811 /* 5812 * The call to memmap_init_zone should have already taken care 5813 * of the pages reserved for the memmap, so we can just jump to 5814 * the end of that region and start processing the device pages. 5815 */ 5816 if (pgmap->altmap_valid) { 5817 struct vmem_altmap *altmap = &pgmap->altmap; 5818 5819 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 5820 size = end_pfn - start_pfn; 5821 } 5822 5823 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 5824 struct page *page = pfn_to_page(pfn); 5825 5826 __init_single_page(page, pfn, zone_idx, nid); 5827 5828 /* 5829 * Mark page reserved as it will need to wait for onlining 5830 * phase for it to be fully associated with a zone. 5831 * 5832 * We can use the non-atomic __set_bit operation for setting 5833 * the flag as we are still initializing the pages. 5834 */ 5835 __SetPageReserved(page); 5836 5837 /* 5838 * ZONE_DEVICE pages union ->lru with a ->pgmap back 5839 * pointer and hmm_data. It is a bug if a ZONE_DEVICE 5840 * page is ever freed or placed on a driver-private list. 5841 */ 5842 page->pgmap = pgmap; 5843 page->hmm_data = 0; 5844 5845 /* 5846 * Mark the block movable so that blocks are reserved for 5847 * movable at startup. This will force kernel allocations 5848 * to reserve their blocks rather than leaking throughout 5849 * the address space during boot when many long-lived 5850 * kernel allocations are made. 5851 * 5852 * bitmap is created for zone's valid pfn range. but memmap 5853 * can be created for invalid pages (for alignment) 5854 * check here not to call set_pageblock_migratetype() against 5855 * pfn out of zone. 5856 * 5857 * Please note that MEMMAP_HOTPLUG path doesn't clear memmap 5858 * because this is done early in sparse_add_one_section 5859 */ 5860 if (!(pfn & (pageblock_nr_pages - 1))) { 5861 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 5862 cond_resched(); 5863 } 5864 } 5865 5866 pr_info("%s initialised, %lu pages in %ums\n", dev_name(pgmap->dev), 5867 size, jiffies_to_msecs(jiffies - start)); 5868 } 5869 5870 #endif 5871 static void __meminit zone_init_free_lists(struct zone *zone) 5872 { 5873 unsigned int order, t; 5874 for_each_migratetype_order(order, t) { 5875 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 5876 zone->free_area[order].nr_free = 0; 5877 } 5878 } 5879 5880 void __meminit __weak memmap_init(unsigned long size, int nid, 5881 unsigned long zone, unsigned long start_pfn) 5882 { 5883 memmap_init_zone(size, nid, zone, start_pfn, MEMMAP_EARLY, NULL); 5884 } 5885 5886 static int zone_batchsize(struct zone *zone) 5887 { 5888 #ifdef CONFIG_MMU 5889 int batch; 5890 5891 /* 5892 * The per-cpu-pages pools are set to around 1000th of the 5893 * size of the zone. 5894 */ 5895 batch = zone_managed_pages(zone) / 1024; 5896 /* But no more than a meg. */ 5897 if (batch * PAGE_SIZE > 1024 * 1024) 5898 batch = (1024 * 1024) / PAGE_SIZE; 5899 batch /= 4; /* We effectively *= 4 below */ 5900 if (batch < 1) 5901 batch = 1; 5902 5903 /* 5904 * Clamp the batch to a 2^n - 1 value. Having a power 5905 * of 2 value was found to be more likely to have 5906 * suboptimal cache aliasing properties in some cases. 5907 * 5908 * For example if 2 tasks are alternately allocating 5909 * batches of pages, one task can end up with a lot 5910 * of pages of one half of the possible page colors 5911 * and the other with pages of the other colors. 5912 */ 5913 batch = rounddown_pow_of_two(batch + batch/2) - 1; 5914 5915 return batch; 5916 5917 #else 5918 /* The deferral and batching of frees should be suppressed under NOMMU 5919 * conditions. 5920 * 5921 * The problem is that NOMMU needs to be able to allocate large chunks 5922 * of contiguous memory as there's no hardware page translation to 5923 * assemble apparent contiguous memory from discontiguous pages. 5924 * 5925 * Queueing large contiguous runs of pages for batching, however, 5926 * causes the pages to actually be freed in smaller chunks. As there 5927 * can be a significant delay between the individual batches being 5928 * recycled, this leads to the once large chunks of space being 5929 * fragmented and becoming unavailable for high-order allocations. 5930 */ 5931 return 0; 5932 #endif 5933 } 5934 5935 /* 5936 * pcp->high and pcp->batch values are related and dependent on one another: 5937 * ->batch must never be higher then ->high. 5938 * The following function updates them in a safe manner without read side 5939 * locking. 5940 * 5941 * Any new users of pcp->batch and pcp->high should ensure they can cope with 5942 * those fields changing asynchronously (acording the the above rule). 5943 * 5944 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 5945 * outside of boot time (or some other assurance that no concurrent updaters 5946 * exist). 5947 */ 5948 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 5949 unsigned long batch) 5950 { 5951 /* start with a fail safe value for batch */ 5952 pcp->batch = 1; 5953 smp_wmb(); 5954 5955 /* Update high, then batch, in order */ 5956 pcp->high = high; 5957 smp_wmb(); 5958 5959 pcp->batch = batch; 5960 } 5961 5962 /* a companion to pageset_set_high() */ 5963 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) 5964 { 5965 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); 5966 } 5967 5968 static void pageset_init(struct per_cpu_pageset *p) 5969 { 5970 struct per_cpu_pages *pcp; 5971 int migratetype; 5972 5973 memset(p, 0, sizeof(*p)); 5974 5975 pcp = &p->pcp; 5976 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) 5977 INIT_LIST_HEAD(&pcp->lists[migratetype]); 5978 } 5979 5980 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 5981 { 5982 pageset_init(p); 5983 pageset_set_batch(p, batch); 5984 } 5985 5986 /* 5987 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist 5988 * to the value high for the pageset p. 5989 */ 5990 static void pageset_set_high(struct per_cpu_pageset *p, 5991 unsigned long high) 5992 { 5993 unsigned long batch = max(1UL, high / 4); 5994 if ((high / 4) > (PAGE_SHIFT * 8)) 5995 batch = PAGE_SHIFT * 8; 5996 5997 pageset_update(&p->pcp, high, batch); 5998 } 5999 6000 static void pageset_set_high_and_batch(struct zone *zone, 6001 struct per_cpu_pageset *pcp) 6002 { 6003 if (percpu_pagelist_fraction) 6004 pageset_set_high(pcp, 6005 (zone_managed_pages(zone) / 6006 percpu_pagelist_fraction)); 6007 else 6008 pageset_set_batch(pcp, zone_batchsize(zone)); 6009 } 6010 6011 static void __meminit zone_pageset_init(struct zone *zone, int cpu) 6012 { 6013 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); 6014 6015 pageset_init(pcp); 6016 pageset_set_high_and_batch(zone, pcp); 6017 } 6018 6019 void __meminit setup_zone_pageset(struct zone *zone) 6020 { 6021 int cpu; 6022 zone->pageset = alloc_percpu(struct per_cpu_pageset); 6023 for_each_possible_cpu(cpu) 6024 zone_pageset_init(zone, cpu); 6025 } 6026 6027 /* 6028 * Allocate per cpu pagesets and initialize them. 6029 * Before this call only boot pagesets were available. 6030 */ 6031 void __init setup_per_cpu_pageset(void) 6032 { 6033 struct pglist_data *pgdat; 6034 struct zone *zone; 6035 6036 for_each_populated_zone(zone) 6037 setup_zone_pageset(zone); 6038 6039 for_each_online_pgdat(pgdat) 6040 pgdat->per_cpu_nodestats = 6041 alloc_percpu(struct per_cpu_nodestat); 6042 } 6043 6044 static __meminit void zone_pcp_init(struct zone *zone) 6045 { 6046 /* 6047 * per cpu subsystem is not up at this point. The following code 6048 * relies on the ability of the linker to provide the 6049 * offset of a (static) per cpu variable into the per cpu area. 6050 */ 6051 zone->pageset = &boot_pageset; 6052 6053 if (populated_zone(zone)) 6054 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", 6055 zone->name, zone->present_pages, 6056 zone_batchsize(zone)); 6057 } 6058 6059 void __meminit init_currently_empty_zone(struct zone *zone, 6060 unsigned long zone_start_pfn, 6061 unsigned long size) 6062 { 6063 struct pglist_data *pgdat = zone->zone_pgdat; 6064 int zone_idx = zone_idx(zone) + 1; 6065 6066 if (zone_idx > pgdat->nr_zones) 6067 pgdat->nr_zones = zone_idx; 6068 6069 zone->zone_start_pfn = zone_start_pfn; 6070 6071 mminit_dprintk(MMINIT_TRACE, "memmap_init", 6072 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 6073 pgdat->node_id, 6074 (unsigned long)zone_idx(zone), 6075 zone_start_pfn, (zone_start_pfn + size)); 6076 6077 zone_init_free_lists(zone); 6078 zone->initialized = 1; 6079 } 6080 6081 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6082 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID 6083 6084 /* 6085 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 6086 */ 6087 int __meminit __early_pfn_to_nid(unsigned long pfn, 6088 struct mminit_pfnnid_cache *state) 6089 { 6090 unsigned long start_pfn, end_pfn; 6091 int nid; 6092 6093 if (state->last_start <= pfn && pfn < state->last_end) 6094 return state->last_nid; 6095 6096 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 6097 if (nid != NUMA_NO_NODE) { 6098 state->last_start = start_pfn; 6099 state->last_end = end_pfn; 6100 state->last_nid = nid; 6101 } 6102 6103 return nid; 6104 } 6105 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ 6106 6107 /** 6108 * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range 6109 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed. 6110 * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid 6111 * 6112 * If an architecture guarantees that all ranges registered contain no holes 6113 * and may be freed, this this function may be used instead of calling 6114 * memblock_free_early_nid() manually. 6115 */ 6116 void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn) 6117 { 6118 unsigned long start_pfn, end_pfn; 6119 int i, this_nid; 6120 6121 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) { 6122 start_pfn = min(start_pfn, max_low_pfn); 6123 end_pfn = min(end_pfn, max_low_pfn); 6124 6125 if (start_pfn < end_pfn) 6126 memblock_free_early_nid(PFN_PHYS(start_pfn), 6127 (end_pfn - start_pfn) << PAGE_SHIFT, 6128 this_nid); 6129 } 6130 } 6131 6132 /** 6133 * sparse_memory_present_with_active_regions - Call memory_present for each active range 6134 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used. 6135 * 6136 * If an architecture guarantees that all ranges registered contain no holes and may 6137 * be freed, this function may be used instead of calling memory_present() manually. 6138 */ 6139 void __init sparse_memory_present_with_active_regions(int nid) 6140 { 6141 unsigned long start_pfn, end_pfn; 6142 int i, this_nid; 6143 6144 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) 6145 memory_present(this_nid, start_pfn, end_pfn); 6146 } 6147 6148 /** 6149 * get_pfn_range_for_nid - Return the start and end page frames for a node 6150 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 6151 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 6152 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 6153 * 6154 * It returns the start and end page frame of a node based on information 6155 * provided by memblock_set_node(). If called for a node 6156 * with no available memory, a warning is printed and the start and end 6157 * PFNs will be 0. 6158 */ 6159 void __init get_pfn_range_for_nid(unsigned int nid, 6160 unsigned long *start_pfn, unsigned long *end_pfn) 6161 { 6162 unsigned long this_start_pfn, this_end_pfn; 6163 int i; 6164 6165 *start_pfn = -1UL; 6166 *end_pfn = 0; 6167 6168 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 6169 *start_pfn = min(*start_pfn, this_start_pfn); 6170 *end_pfn = max(*end_pfn, this_end_pfn); 6171 } 6172 6173 if (*start_pfn == -1UL) 6174 *start_pfn = 0; 6175 } 6176 6177 /* 6178 * This finds a zone that can be used for ZONE_MOVABLE pages. The 6179 * assumption is made that zones within a node are ordered in monotonic 6180 * increasing memory addresses so that the "highest" populated zone is used 6181 */ 6182 static void __init find_usable_zone_for_movable(void) 6183 { 6184 int zone_index; 6185 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 6186 if (zone_index == ZONE_MOVABLE) 6187 continue; 6188 6189 if (arch_zone_highest_possible_pfn[zone_index] > 6190 arch_zone_lowest_possible_pfn[zone_index]) 6191 break; 6192 } 6193 6194 VM_BUG_ON(zone_index == -1); 6195 movable_zone = zone_index; 6196 } 6197 6198 /* 6199 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 6200 * because it is sized independent of architecture. Unlike the other zones, 6201 * the starting point for ZONE_MOVABLE is not fixed. It may be different 6202 * in each node depending on the size of each node and how evenly kernelcore 6203 * is distributed. This helper function adjusts the zone ranges 6204 * provided by the architecture for a given node by using the end of the 6205 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 6206 * zones within a node are in order of monotonic increases memory addresses 6207 */ 6208 static void __init adjust_zone_range_for_zone_movable(int nid, 6209 unsigned long zone_type, 6210 unsigned long node_start_pfn, 6211 unsigned long node_end_pfn, 6212 unsigned long *zone_start_pfn, 6213 unsigned long *zone_end_pfn) 6214 { 6215 /* Only adjust if ZONE_MOVABLE is on this node */ 6216 if (zone_movable_pfn[nid]) { 6217 /* Size ZONE_MOVABLE */ 6218 if (zone_type == ZONE_MOVABLE) { 6219 *zone_start_pfn = zone_movable_pfn[nid]; 6220 *zone_end_pfn = min(node_end_pfn, 6221 arch_zone_highest_possible_pfn[movable_zone]); 6222 6223 /* Adjust for ZONE_MOVABLE starting within this range */ 6224 } else if (!mirrored_kernelcore && 6225 *zone_start_pfn < zone_movable_pfn[nid] && 6226 *zone_end_pfn > zone_movable_pfn[nid]) { 6227 *zone_end_pfn = zone_movable_pfn[nid]; 6228 6229 /* Check if this whole range is within ZONE_MOVABLE */ 6230 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 6231 *zone_start_pfn = *zone_end_pfn; 6232 } 6233 } 6234 6235 /* 6236 * Return the number of pages a zone spans in a node, including holes 6237 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 6238 */ 6239 static unsigned long __init zone_spanned_pages_in_node(int nid, 6240 unsigned long zone_type, 6241 unsigned long node_start_pfn, 6242 unsigned long node_end_pfn, 6243 unsigned long *zone_start_pfn, 6244 unsigned long *zone_end_pfn, 6245 unsigned long *ignored) 6246 { 6247 /* When hotadd a new node from cpu_up(), the node should be empty */ 6248 if (!node_start_pfn && !node_end_pfn) 6249 return 0; 6250 6251 /* Get the start and end of the zone */ 6252 *zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type]; 6253 *zone_end_pfn = arch_zone_highest_possible_pfn[zone_type]; 6254 adjust_zone_range_for_zone_movable(nid, zone_type, 6255 node_start_pfn, node_end_pfn, 6256 zone_start_pfn, zone_end_pfn); 6257 6258 /* Check that this node has pages within the zone's required range */ 6259 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 6260 return 0; 6261 6262 /* Move the zone boundaries inside the node if necessary */ 6263 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 6264 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 6265 6266 /* Return the spanned pages */ 6267 return *zone_end_pfn - *zone_start_pfn; 6268 } 6269 6270 /* 6271 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 6272 * then all holes in the requested range will be accounted for. 6273 */ 6274 unsigned long __init __absent_pages_in_range(int nid, 6275 unsigned long range_start_pfn, 6276 unsigned long range_end_pfn) 6277 { 6278 unsigned long nr_absent = range_end_pfn - range_start_pfn; 6279 unsigned long start_pfn, end_pfn; 6280 int i; 6281 6282 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6283 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 6284 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 6285 nr_absent -= end_pfn - start_pfn; 6286 } 6287 return nr_absent; 6288 } 6289 6290 /** 6291 * absent_pages_in_range - Return number of page frames in holes within a range 6292 * @start_pfn: The start PFN to start searching for holes 6293 * @end_pfn: The end PFN to stop searching for holes 6294 * 6295 * Return: the number of pages frames in memory holes within a range. 6296 */ 6297 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 6298 unsigned long end_pfn) 6299 { 6300 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 6301 } 6302 6303 /* Return the number of page frames in holes in a zone on a node */ 6304 static unsigned long __init zone_absent_pages_in_node(int nid, 6305 unsigned long zone_type, 6306 unsigned long node_start_pfn, 6307 unsigned long node_end_pfn, 6308 unsigned long *ignored) 6309 { 6310 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 6311 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 6312 unsigned long zone_start_pfn, zone_end_pfn; 6313 unsigned long nr_absent; 6314 6315 /* When hotadd a new node from cpu_up(), the node should be empty */ 6316 if (!node_start_pfn && !node_end_pfn) 6317 return 0; 6318 6319 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 6320 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 6321 6322 adjust_zone_range_for_zone_movable(nid, zone_type, 6323 node_start_pfn, node_end_pfn, 6324 &zone_start_pfn, &zone_end_pfn); 6325 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 6326 6327 /* 6328 * ZONE_MOVABLE handling. 6329 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 6330 * and vice versa. 6331 */ 6332 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 6333 unsigned long start_pfn, end_pfn; 6334 struct memblock_region *r; 6335 6336 for_each_memblock(memory, r) { 6337 start_pfn = clamp(memblock_region_memory_base_pfn(r), 6338 zone_start_pfn, zone_end_pfn); 6339 end_pfn = clamp(memblock_region_memory_end_pfn(r), 6340 zone_start_pfn, zone_end_pfn); 6341 6342 if (zone_type == ZONE_MOVABLE && 6343 memblock_is_mirror(r)) 6344 nr_absent += end_pfn - start_pfn; 6345 6346 if (zone_type == ZONE_NORMAL && 6347 !memblock_is_mirror(r)) 6348 nr_absent += end_pfn - start_pfn; 6349 } 6350 } 6351 6352 return nr_absent; 6353 } 6354 6355 #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6356 static inline unsigned long __init zone_spanned_pages_in_node(int nid, 6357 unsigned long zone_type, 6358 unsigned long node_start_pfn, 6359 unsigned long node_end_pfn, 6360 unsigned long *zone_start_pfn, 6361 unsigned long *zone_end_pfn, 6362 unsigned long *zones_size) 6363 { 6364 unsigned int zone; 6365 6366 *zone_start_pfn = node_start_pfn; 6367 for (zone = 0; zone < zone_type; zone++) 6368 *zone_start_pfn += zones_size[zone]; 6369 6370 *zone_end_pfn = *zone_start_pfn + zones_size[zone_type]; 6371 6372 return zones_size[zone_type]; 6373 } 6374 6375 static inline unsigned long __init zone_absent_pages_in_node(int nid, 6376 unsigned long zone_type, 6377 unsigned long node_start_pfn, 6378 unsigned long node_end_pfn, 6379 unsigned long *zholes_size) 6380 { 6381 if (!zholes_size) 6382 return 0; 6383 6384 return zholes_size[zone_type]; 6385 } 6386 6387 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6388 6389 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 6390 unsigned long node_start_pfn, 6391 unsigned long node_end_pfn, 6392 unsigned long *zones_size, 6393 unsigned long *zholes_size) 6394 { 6395 unsigned long realtotalpages = 0, totalpages = 0; 6396 enum zone_type i; 6397 6398 for (i = 0; i < MAX_NR_ZONES; i++) { 6399 struct zone *zone = pgdat->node_zones + i; 6400 unsigned long zone_start_pfn, zone_end_pfn; 6401 unsigned long size, real_size; 6402 6403 size = zone_spanned_pages_in_node(pgdat->node_id, i, 6404 node_start_pfn, 6405 node_end_pfn, 6406 &zone_start_pfn, 6407 &zone_end_pfn, 6408 zones_size); 6409 real_size = size - zone_absent_pages_in_node(pgdat->node_id, i, 6410 node_start_pfn, node_end_pfn, 6411 zholes_size); 6412 if (size) 6413 zone->zone_start_pfn = zone_start_pfn; 6414 else 6415 zone->zone_start_pfn = 0; 6416 zone->spanned_pages = size; 6417 zone->present_pages = real_size; 6418 6419 totalpages += size; 6420 realtotalpages += real_size; 6421 } 6422 6423 pgdat->node_spanned_pages = totalpages; 6424 pgdat->node_present_pages = realtotalpages; 6425 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, 6426 realtotalpages); 6427 } 6428 6429 #ifndef CONFIG_SPARSEMEM 6430 /* 6431 * Calculate the size of the zone->blockflags rounded to an unsigned long 6432 * Start by making sure zonesize is a multiple of pageblock_order by rounding 6433 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 6434 * round what is now in bits to nearest long in bits, then return it in 6435 * bytes. 6436 */ 6437 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 6438 { 6439 unsigned long usemapsize; 6440 6441 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 6442 usemapsize = roundup(zonesize, pageblock_nr_pages); 6443 usemapsize = usemapsize >> pageblock_order; 6444 usemapsize *= NR_PAGEBLOCK_BITS; 6445 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 6446 6447 return usemapsize / 8; 6448 } 6449 6450 static void __ref setup_usemap(struct pglist_data *pgdat, 6451 struct zone *zone, 6452 unsigned long zone_start_pfn, 6453 unsigned long zonesize) 6454 { 6455 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); 6456 zone->pageblock_flags = NULL; 6457 if (usemapsize) { 6458 zone->pageblock_flags = 6459 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 6460 pgdat->node_id); 6461 if (!zone->pageblock_flags) 6462 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 6463 usemapsize, zone->name, pgdat->node_id); 6464 } 6465 } 6466 #else 6467 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, 6468 unsigned long zone_start_pfn, unsigned long zonesize) {} 6469 #endif /* CONFIG_SPARSEMEM */ 6470 6471 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 6472 6473 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 6474 void __init set_pageblock_order(void) 6475 { 6476 unsigned int order; 6477 6478 /* Check that pageblock_nr_pages has not already been setup */ 6479 if (pageblock_order) 6480 return; 6481 6482 if (HPAGE_SHIFT > PAGE_SHIFT) 6483 order = HUGETLB_PAGE_ORDER; 6484 else 6485 order = MAX_ORDER - 1; 6486 6487 /* 6488 * Assume the largest contiguous order of interest is a huge page. 6489 * This value may be variable depending on boot parameters on IA64 and 6490 * powerpc. 6491 */ 6492 pageblock_order = order; 6493 } 6494 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6495 6496 /* 6497 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 6498 * is unused as pageblock_order is set at compile-time. See 6499 * include/linux/pageblock-flags.h for the values of pageblock_order based on 6500 * the kernel config 6501 */ 6502 void __init set_pageblock_order(void) 6503 { 6504 } 6505 6506 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6507 6508 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 6509 unsigned long present_pages) 6510 { 6511 unsigned long pages = spanned_pages; 6512 6513 /* 6514 * Provide a more accurate estimation if there are holes within 6515 * the zone and SPARSEMEM is in use. If there are holes within the 6516 * zone, each populated memory region may cost us one or two extra 6517 * memmap pages due to alignment because memmap pages for each 6518 * populated regions may not be naturally aligned on page boundary. 6519 * So the (present_pages >> 4) heuristic is a tradeoff for that. 6520 */ 6521 if (spanned_pages > present_pages + (present_pages >> 4) && 6522 IS_ENABLED(CONFIG_SPARSEMEM)) 6523 pages = present_pages; 6524 6525 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 6526 } 6527 6528 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 6529 static void pgdat_init_split_queue(struct pglist_data *pgdat) 6530 { 6531 spin_lock_init(&pgdat->split_queue_lock); 6532 INIT_LIST_HEAD(&pgdat->split_queue); 6533 pgdat->split_queue_len = 0; 6534 } 6535 #else 6536 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 6537 #endif 6538 6539 #ifdef CONFIG_COMPACTION 6540 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 6541 { 6542 init_waitqueue_head(&pgdat->kcompactd_wait); 6543 } 6544 #else 6545 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 6546 #endif 6547 6548 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 6549 { 6550 pgdat_resize_init(pgdat); 6551 6552 pgdat_init_split_queue(pgdat); 6553 pgdat_init_kcompactd(pgdat); 6554 6555 init_waitqueue_head(&pgdat->kswapd_wait); 6556 init_waitqueue_head(&pgdat->pfmemalloc_wait); 6557 6558 pgdat_page_ext_init(pgdat); 6559 spin_lock_init(&pgdat->lru_lock); 6560 lruvec_init(node_lruvec(pgdat)); 6561 } 6562 6563 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 6564 unsigned long remaining_pages) 6565 { 6566 atomic_long_set(&zone->managed_pages, remaining_pages); 6567 zone_set_nid(zone, nid); 6568 zone->name = zone_names[idx]; 6569 zone->zone_pgdat = NODE_DATA(nid); 6570 spin_lock_init(&zone->lock); 6571 zone_seqlock_init(zone); 6572 zone_pcp_init(zone); 6573 } 6574 6575 /* 6576 * Set up the zone data structures 6577 * - init pgdat internals 6578 * - init all zones belonging to this node 6579 * 6580 * NOTE: this function is only called during memory hotplug 6581 */ 6582 #ifdef CONFIG_MEMORY_HOTPLUG 6583 void __ref free_area_init_core_hotplug(int nid) 6584 { 6585 enum zone_type z; 6586 pg_data_t *pgdat = NODE_DATA(nid); 6587 6588 pgdat_init_internals(pgdat); 6589 for (z = 0; z < MAX_NR_ZONES; z++) 6590 zone_init_internals(&pgdat->node_zones[z], z, nid, 0); 6591 } 6592 #endif 6593 6594 /* 6595 * Set up the zone data structures: 6596 * - mark all pages reserved 6597 * - mark all memory queues empty 6598 * - clear the memory bitmaps 6599 * 6600 * NOTE: pgdat should get zeroed by caller. 6601 * NOTE: this function is only called during early init. 6602 */ 6603 static void __init free_area_init_core(struct pglist_data *pgdat) 6604 { 6605 enum zone_type j; 6606 int nid = pgdat->node_id; 6607 6608 pgdat_init_internals(pgdat); 6609 pgdat->per_cpu_nodestats = &boot_nodestats; 6610 6611 for (j = 0; j < MAX_NR_ZONES; j++) { 6612 struct zone *zone = pgdat->node_zones + j; 6613 unsigned long size, freesize, memmap_pages; 6614 unsigned long zone_start_pfn = zone->zone_start_pfn; 6615 6616 size = zone->spanned_pages; 6617 freesize = zone->present_pages; 6618 6619 /* 6620 * Adjust freesize so that it accounts for how much memory 6621 * is used by this zone for memmap. This affects the watermark 6622 * and per-cpu initialisations 6623 */ 6624 memmap_pages = calc_memmap_size(size, freesize); 6625 if (!is_highmem_idx(j)) { 6626 if (freesize >= memmap_pages) { 6627 freesize -= memmap_pages; 6628 if (memmap_pages) 6629 printk(KERN_DEBUG 6630 " %s zone: %lu pages used for memmap\n", 6631 zone_names[j], memmap_pages); 6632 } else 6633 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n", 6634 zone_names[j], memmap_pages, freesize); 6635 } 6636 6637 /* Account for reserved pages */ 6638 if (j == 0 && freesize > dma_reserve) { 6639 freesize -= dma_reserve; 6640 printk(KERN_DEBUG " %s zone: %lu pages reserved\n", 6641 zone_names[0], dma_reserve); 6642 } 6643 6644 if (!is_highmem_idx(j)) 6645 nr_kernel_pages += freesize; 6646 /* Charge for highmem memmap if there are enough kernel pages */ 6647 else if (nr_kernel_pages > memmap_pages * 2) 6648 nr_kernel_pages -= memmap_pages; 6649 nr_all_pages += freesize; 6650 6651 /* 6652 * Set an approximate value for lowmem here, it will be adjusted 6653 * when the bootmem allocator frees pages into the buddy system. 6654 * And all highmem pages will be managed by the buddy system. 6655 */ 6656 zone_init_internals(zone, j, nid, freesize); 6657 6658 if (!size) 6659 continue; 6660 6661 set_pageblock_order(); 6662 setup_usemap(pgdat, zone, zone_start_pfn, size); 6663 init_currently_empty_zone(zone, zone_start_pfn, size); 6664 memmap_init(size, nid, j, zone_start_pfn); 6665 } 6666 } 6667 6668 #ifdef CONFIG_FLAT_NODE_MEM_MAP 6669 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) 6670 { 6671 unsigned long __maybe_unused start = 0; 6672 unsigned long __maybe_unused offset = 0; 6673 6674 /* Skip empty nodes */ 6675 if (!pgdat->node_spanned_pages) 6676 return; 6677 6678 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 6679 offset = pgdat->node_start_pfn - start; 6680 /* ia64 gets its own node_mem_map, before this, without bootmem */ 6681 if (!pgdat->node_mem_map) { 6682 unsigned long size, end; 6683 struct page *map; 6684 6685 /* 6686 * The zone's endpoints aren't required to be MAX_ORDER 6687 * aligned but the node_mem_map endpoints must be in order 6688 * for the buddy allocator to function correctly. 6689 */ 6690 end = pgdat_end_pfn(pgdat); 6691 end = ALIGN(end, MAX_ORDER_NR_PAGES); 6692 size = (end - start) * sizeof(struct page); 6693 map = memblock_alloc_node(size, SMP_CACHE_BYTES, 6694 pgdat->node_id); 6695 if (!map) 6696 panic("Failed to allocate %ld bytes for node %d memory map\n", 6697 size, pgdat->node_id); 6698 pgdat->node_mem_map = map + offset; 6699 } 6700 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 6701 __func__, pgdat->node_id, (unsigned long)pgdat, 6702 (unsigned long)pgdat->node_mem_map); 6703 #ifndef CONFIG_NEED_MULTIPLE_NODES 6704 /* 6705 * With no DISCONTIG, the global mem_map is just set as node 0's 6706 */ 6707 if (pgdat == NODE_DATA(0)) { 6708 mem_map = NODE_DATA(0)->node_mem_map; 6709 #if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM) 6710 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 6711 mem_map -= offset; 6712 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6713 } 6714 #endif 6715 } 6716 #else 6717 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { } 6718 #endif /* CONFIG_FLAT_NODE_MEM_MAP */ 6719 6720 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 6721 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 6722 { 6723 pgdat->first_deferred_pfn = ULONG_MAX; 6724 } 6725 #else 6726 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 6727 #endif 6728 6729 void __init free_area_init_node(int nid, unsigned long *zones_size, 6730 unsigned long node_start_pfn, 6731 unsigned long *zholes_size) 6732 { 6733 pg_data_t *pgdat = NODE_DATA(nid); 6734 unsigned long start_pfn = 0; 6735 unsigned long end_pfn = 0; 6736 6737 /* pg_data_t should be reset to zero when it's allocated */ 6738 WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx); 6739 6740 pgdat->node_id = nid; 6741 pgdat->node_start_pfn = node_start_pfn; 6742 pgdat->per_cpu_nodestats = NULL; 6743 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6744 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 6745 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 6746 (u64)start_pfn << PAGE_SHIFT, 6747 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 6748 #else 6749 start_pfn = node_start_pfn; 6750 #endif 6751 calculate_node_totalpages(pgdat, start_pfn, end_pfn, 6752 zones_size, zholes_size); 6753 6754 alloc_node_mem_map(pgdat); 6755 pgdat_set_deferred_range(pgdat); 6756 6757 free_area_init_core(pgdat); 6758 } 6759 6760 #if !defined(CONFIG_FLAT_NODE_MEM_MAP) 6761 /* 6762 * Zero all valid struct pages in range [spfn, epfn), return number of struct 6763 * pages zeroed 6764 */ 6765 static u64 zero_pfn_range(unsigned long spfn, unsigned long epfn) 6766 { 6767 unsigned long pfn; 6768 u64 pgcnt = 0; 6769 6770 for (pfn = spfn; pfn < epfn; pfn++) { 6771 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) { 6772 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages) 6773 + pageblock_nr_pages - 1; 6774 continue; 6775 } 6776 mm_zero_struct_page(pfn_to_page(pfn)); 6777 pgcnt++; 6778 } 6779 6780 return pgcnt; 6781 } 6782 6783 /* 6784 * Only struct pages that are backed by physical memory are zeroed and 6785 * initialized by going through __init_single_page(). But, there are some 6786 * struct pages which are reserved in memblock allocator and their fields 6787 * may be accessed (for example page_to_pfn() on some configuration accesses 6788 * flags). We must explicitly zero those struct pages. 6789 * 6790 * This function also addresses a similar issue where struct pages are left 6791 * uninitialized because the physical address range is not covered by 6792 * memblock.memory or memblock.reserved. That could happen when memblock 6793 * layout is manually configured via memmap=. 6794 */ 6795 void __init zero_resv_unavail(void) 6796 { 6797 phys_addr_t start, end; 6798 u64 i, pgcnt; 6799 phys_addr_t next = 0; 6800 6801 /* 6802 * Loop through unavailable ranges not covered by memblock.memory. 6803 */ 6804 pgcnt = 0; 6805 for_each_mem_range(i, &memblock.memory, NULL, 6806 NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, NULL) { 6807 if (next < start) 6808 pgcnt += zero_pfn_range(PFN_DOWN(next), PFN_UP(start)); 6809 next = end; 6810 } 6811 pgcnt += zero_pfn_range(PFN_DOWN(next), max_pfn); 6812 6813 /* 6814 * Struct pages that do not have backing memory. This could be because 6815 * firmware is using some of this memory, or for some other reasons. 6816 */ 6817 if (pgcnt) 6818 pr_info("Zeroed struct page in unavailable ranges: %lld pages", pgcnt); 6819 } 6820 #endif /* !CONFIG_FLAT_NODE_MEM_MAP */ 6821 6822 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6823 6824 #if MAX_NUMNODES > 1 6825 /* 6826 * Figure out the number of possible node ids. 6827 */ 6828 void __init setup_nr_node_ids(void) 6829 { 6830 unsigned int highest; 6831 6832 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 6833 nr_node_ids = highest + 1; 6834 } 6835 #endif 6836 6837 /** 6838 * node_map_pfn_alignment - determine the maximum internode alignment 6839 * 6840 * This function should be called after node map is populated and sorted. 6841 * It calculates the maximum power of two alignment which can distinguish 6842 * all the nodes. 6843 * 6844 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 6845 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 6846 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 6847 * shifted, 1GiB is enough and this function will indicate so. 6848 * 6849 * This is used to test whether pfn -> nid mapping of the chosen memory 6850 * model has fine enough granularity to avoid incorrect mapping for the 6851 * populated node map. 6852 * 6853 * Return: the determined alignment in pfn's. 0 if there is no alignment 6854 * requirement (single node). 6855 */ 6856 unsigned long __init node_map_pfn_alignment(void) 6857 { 6858 unsigned long accl_mask = 0, last_end = 0; 6859 unsigned long start, end, mask; 6860 int last_nid = NUMA_NO_NODE; 6861 int i, nid; 6862 6863 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 6864 if (!start || last_nid < 0 || last_nid == nid) { 6865 last_nid = nid; 6866 last_end = end; 6867 continue; 6868 } 6869 6870 /* 6871 * Start with a mask granular enough to pin-point to the 6872 * start pfn and tick off bits one-by-one until it becomes 6873 * too coarse to separate the current node from the last. 6874 */ 6875 mask = ~((1 << __ffs(start)) - 1); 6876 while (mask && last_end <= (start & (mask << 1))) 6877 mask <<= 1; 6878 6879 /* accumulate all internode masks */ 6880 accl_mask |= mask; 6881 } 6882 6883 /* convert mask to number of pages */ 6884 return ~accl_mask + 1; 6885 } 6886 6887 /* Find the lowest pfn for a node */ 6888 static unsigned long __init find_min_pfn_for_node(int nid) 6889 { 6890 unsigned long min_pfn = ULONG_MAX; 6891 unsigned long start_pfn; 6892 int i; 6893 6894 for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL) 6895 min_pfn = min(min_pfn, start_pfn); 6896 6897 if (min_pfn == ULONG_MAX) { 6898 pr_warn("Could not find start_pfn for node %d\n", nid); 6899 return 0; 6900 } 6901 6902 return min_pfn; 6903 } 6904 6905 /** 6906 * find_min_pfn_with_active_regions - Find the minimum PFN registered 6907 * 6908 * Return: the minimum PFN based on information provided via 6909 * memblock_set_node(). 6910 */ 6911 unsigned long __init find_min_pfn_with_active_regions(void) 6912 { 6913 return find_min_pfn_for_node(MAX_NUMNODES); 6914 } 6915 6916 /* 6917 * early_calculate_totalpages() 6918 * Sum pages in active regions for movable zone. 6919 * Populate N_MEMORY for calculating usable_nodes. 6920 */ 6921 static unsigned long __init early_calculate_totalpages(void) 6922 { 6923 unsigned long totalpages = 0; 6924 unsigned long start_pfn, end_pfn; 6925 int i, nid; 6926 6927 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 6928 unsigned long pages = end_pfn - start_pfn; 6929 6930 totalpages += pages; 6931 if (pages) 6932 node_set_state(nid, N_MEMORY); 6933 } 6934 return totalpages; 6935 } 6936 6937 /* 6938 * Find the PFN the Movable zone begins in each node. Kernel memory 6939 * is spread evenly between nodes as long as the nodes have enough 6940 * memory. When they don't, some nodes will have more kernelcore than 6941 * others 6942 */ 6943 static void __init find_zone_movable_pfns_for_nodes(void) 6944 { 6945 int i, nid; 6946 unsigned long usable_startpfn; 6947 unsigned long kernelcore_node, kernelcore_remaining; 6948 /* save the state before borrow the nodemask */ 6949 nodemask_t saved_node_state = node_states[N_MEMORY]; 6950 unsigned long totalpages = early_calculate_totalpages(); 6951 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 6952 struct memblock_region *r; 6953 6954 /* Need to find movable_zone earlier when movable_node is specified. */ 6955 find_usable_zone_for_movable(); 6956 6957 /* 6958 * If movable_node is specified, ignore kernelcore and movablecore 6959 * options. 6960 */ 6961 if (movable_node_is_enabled()) { 6962 for_each_memblock(memory, r) { 6963 if (!memblock_is_hotpluggable(r)) 6964 continue; 6965 6966 nid = r->nid; 6967 6968 usable_startpfn = PFN_DOWN(r->base); 6969 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 6970 min(usable_startpfn, zone_movable_pfn[nid]) : 6971 usable_startpfn; 6972 } 6973 6974 goto out2; 6975 } 6976 6977 /* 6978 * If kernelcore=mirror is specified, ignore movablecore option 6979 */ 6980 if (mirrored_kernelcore) { 6981 bool mem_below_4gb_not_mirrored = false; 6982 6983 for_each_memblock(memory, r) { 6984 if (memblock_is_mirror(r)) 6985 continue; 6986 6987 nid = r->nid; 6988 6989 usable_startpfn = memblock_region_memory_base_pfn(r); 6990 6991 if (usable_startpfn < 0x100000) { 6992 mem_below_4gb_not_mirrored = true; 6993 continue; 6994 } 6995 6996 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 6997 min(usable_startpfn, zone_movable_pfn[nid]) : 6998 usable_startpfn; 6999 } 7000 7001 if (mem_below_4gb_not_mirrored) 7002 pr_warn("This configuration results in unmirrored kernel memory."); 7003 7004 goto out2; 7005 } 7006 7007 /* 7008 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 7009 * amount of necessary memory. 7010 */ 7011 if (required_kernelcore_percent) 7012 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 7013 10000UL; 7014 if (required_movablecore_percent) 7015 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 7016 10000UL; 7017 7018 /* 7019 * If movablecore= was specified, calculate what size of 7020 * kernelcore that corresponds so that memory usable for 7021 * any allocation type is evenly spread. If both kernelcore 7022 * and movablecore are specified, then the value of kernelcore 7023 * will be used for required_kernelcore if it's greater than 7024 * what movablecore would have allowed. 7025 */ 7026 if (required_movablecore) { 7027 unsigned long corepages; 7028 7029 /* 7030 * Round-up so that ZONE_MOVABLE is at least as large as what 7031 * was requested by the user 7032 */ 7033 required_movablecore = 7034 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 7035 required_movablecore = min(totalpages, required_movablecore); 7036 corepages = totalpages - required_movablecore; 7037 7038 required_kernelcore = max(required_kernelcore, corepages); 7039 } 7040 7041 /* 7042 * If kernelcore was not specified or kernelcore size is larger 7043 * than totalpages, there is no ZONE_MOVABLE. 7044 */ 7045 if (!required_kernelcore || required_kernelcore >= totalpages) 7046 goto out; 7047 7048 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 7049 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 7050 7051 restart: 7052 /* Spread kernelcore memory as evenly as possible throughout nodes */ 7053 kernelcore_node = required_kernelcore / usable_nodes; 7054 for_each_node_state(nid, N_MEMORY) { 7055 unsigned long start_pfn, end_pfn; 7056 7057 /* 7058 * Recalculate kernelcore_node if the division per node 7059 * now exceeds what is necessary to satisfy the requested 7060 * amount of memory for the kernel 7061 */ 7062 if (required_kernelcore < kernelcore_node) 7063 kernelcore_node = required_kernelcore / usable_nodes; 7064 7065 /* 7066 * As the map is walked, we track how much memory is usable 7067 * by the kernel using kernelcore_remaining. When it is 7068 * 0, the rest of the node is usable by ZONE_MOVABLE 7069 */ 7070 kernelcore_remaining = kernelcore_node; 7071 7072 /* Go through each range of PFNs within this node */ 7073 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7074 unsigned long size_pages; 7075 7076 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 7077 if (start_pfn >= end_pfn) 7078 continue; 7079 7080 /* Account for what is only usable for kernelcore */ 7081 if (start_pfn < usable_startpfn) { 7082 unsigned long kernel_pages; 7083 kernel_pages = min(end_pfn, usable_startpfn) 7084 - start_pfn; 7085 7086 kernelcore_remaining -= min(kernel_pages, 7087 kernelcore_remaining); 7088 required_kernelcore -= min(kernel_pages, 7089 required_kernelcore); 7090 7091 /* Continue if range is now fully accounted */ 7092 if (end_pfn <= usable_startpfn) { 7093 7094 /* 7095 * Push zone_movable_pfn to the end so 7096 * that if we have to rebalance 7097 * kernelcore across nodes, we will 7098 * not double account here 7099 */ 7100 zone_movable_pfn[nid] = end_pfn; 7101 continue; 7102 } 7103 start_pfn = usable_startpfn; 7104 } 7105 7106 /* 7107 * The usable PFN range for ZONE_MOVABLE is from 7108 * start_pfn->end_pfn. Calculate size_pages as the 7109 * number of pages used as kernelcore 7110 */ 7111 size_pages = end_pfn - start_pfn; 7112 if (size_pages > kernelcore_remaining) 7113 size_pages = kernelcore_remaining; 7114 zone_movable_pfn[nid] = start_pfn + size_pages; 7115 7116 /* 7117 * Some kernelcore has been met, update counts and 7118 * break if the kernelcore for this node has been 7119 * satisfied 7120 */ 7121 required_kernelcore -= min(required_kernelcore, 7122 size_pages); 7123 kernelcore_remaining -= size_pages; 7124 if (!kernelcore_remaining) 7125 break; 7126 } 7127 } 7128 7129 /* 7130 * If there is still required_kernelcore, we do another pass with one 7131 * less node in the count. This will push zone_movable_pfn[nid] further 7132 * along on the nodes that still have memory until kernelcore is 7133 * satisfied 7134 */ 7135 usable_nodes--; 7136 if (usable_nodes && required_kernelcore > usable_nodes) 7137 goto restart; 7138 7139 out2: 7140 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 7141 for (nid = 0; nid < MAX_NUMNODES; nid++) 7142 zone_movable_pfn[nid] = 7143 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 7144 7145 out: 7146 /* restore the node_state */ 7147 node_states[N_MEMORY] = saved_node_state; 7148 } 7149 7150 /* Any regular or high memory on that node ? */ 7151 static void check_for_memory(pg_data_t *pgdat, int nid) 7152 { 7153 enum zone_type zone_type; 7154 7155 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 7156 struct zone *zone = &pgdat->node_zones[zone_type]; 7157 if (populated_zone(zone)) { 7158 if (IS_ENABLED(CONFIG_HIGHMEM)) 7159 node_set_state(nid, N_HIGH_MEMORY); 7160 if (zone_type <= ZONE_NORMAL) 7161 node_set_state(nid, N_NORMAL_MEMORY); 7162 break; 7163 } 7164 } 7165 } 7166 7167 /** 7168 * free_area_init_nodes - Initialise all pg_data_t and zone data 7169 * @max_zone_pfn: an array of max PFNs for each zone 7170 * 7171 * This will call free_area_init_node() for each active node in the system. 7172 * Using the page ranges provided by memblock_set_node(), the size of each 7173 * zone in each node and their holes is calculated. If the maximum PFN 7174 * between two adjacent zones match, it is assumed that the zone is empty. 7175 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 7176 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 7177 * starts where the previous one ended. For example, ZONE_DMA32 starts 7178 * at arch_max_dma_pfn. 7179 */ 7180 void __init free_area_init_nodes(unsigned long *max_zone_pfn) 7181 { 7182 unsigned long start_pfn, end_pfn; 7183 int i, nid; 7184 7185 /* Record where the zone boundaries are */ 7186 memset(arch_zone_lowest_possible_pfn, 0, 7187 sizeof(arch_zone_lowest_possible_pfn)); 7188 memset(arch_zone_highest_possible_pfn, 0, 7189 sizeof(arch_zone_highest_possible_pfn)); 7190 7191 start_pfn = find_min_pfn_with_active_regions(); 7192 7193 for (i = 0; i < MAX_NR_ZONES; i++) { 7194 if (i == ZONE_MOVABLE) 7195 continue; 7196 7197 end_pfn = max(max_zone_pfn[i], start_pfn); 7198 arch_zone_lowest_possible_pfn[i] = start_pfn; 7199 arch_zone_highest_possible_pfn[i] = end_pfn; 7200 7201 start_pfn = end_pfn; 7202 } 7203 7204 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 7205 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 7206 find_zone_movable_pfns_for_nodes(); 7207 7208 /* Print out the zone ranges */ 7209 pr_info("Zone ranges:\n"); 7210 for (i = 0; i < MAX_NR_ZONES; i++) { 7211 if (i == ZONE_MOVABLE) 7212 continue; 7213 pr_info(" %-8s ", zone_names[i]); 7214 if (arch_zone_lowest_possible_pfn[i] == 7215 arch_zone_highest_possible_pfn[i]) 7216 pr_cont("empty\n"); 7217 else 7218 pr_cont("[mem %#018Lx-%#018Lx]\n", 7219 (u64)arch_zone_lowest_possible_pfn[i] 7220 << PAGE_SHIFT, 7221 ((u64)arch_zone_highest_possible_pfn[i] 7222 << PAGE_SHIFT) - 1); 7223 } 7224 7225 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 7226 pr_info("Movable zone start for each node\n"); 7227 for (i = 0; i < MAX_NUMNODES; i++) { 7228 if (zone_movable_pfn[i]) 7229 pr_info(" Node %d: %#018Lx\n", i, 7230 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 7231 } 7232 7233 /* Print out the early node map */ 7234 pr_info("Early memory node ranges\n"); 7235 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) 7236 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 7237 (u64)start_pfn << PAGE_SHIFT, 7238 ((u64)end_pfn << PAGE_SHIFT) - 1); 7239 7240 /* Initialise every node */ 7241 mminit_verify_pageflags_layout(); 7242 setup_nr_node_ids(); 7243 zero_resv_unavail(); 7244 for_each_online_node(nid) { 7245 pg_data_t *pgdat = NODE_DATA(nid); 7246 free_area_init_node(nid, NULL, 7247 find_min_pfn_for_node(nid), NULL); 7248 7249 /* Any memory on that node */ 7250 if (pgdat->node_present_pages) 7251 node_set_state(nid, N_MEMORY); 7252 check_for_memory(pgdat, nid); 7253 } 7254 } 7255 7256 static int __init cmdline_parse_core(char *p, unsigned long *core, 7257 unsigned long *percent) 7258 { 7259 unsigned long long coremem; 7260 char *endptr; 7261 7262 if (!p) 7263 return -EINVAL; 7264 7265 /* Value may be a percentage of total memory, otherwise bytes */ 7266 coremem = simple_strtoull(p, &endptr, 0); 7267 if (*endptr == '%') { 7268 /* Paranoid check for percent values greater than 100 */ 7269 WARN_ON(coremem > 100); 7270 7271 *percent = coremem; 7272 } else { 7273 coremem = memparse(p, &p); 7274 /* Paranoid check that UL is enough for the coremem value */ 7275 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 7276 7277 *core = coremem >> PAGE_SHIFT; 7278 *percent = 0UL; 7279 } 7280 return 0; 7281 } 7282 7283 /* 7284 * kernelcore=size sets the amount of memory for use for allocations that 7285 * cannot be reclaimed or migrated. 7286 */ 7287 static int __init cmdline_parse_kernelcore(char *p) 7288 { 7289 /* parse kernelcore=mirror */ 7290 if (parse_option_str(p, "mirror")) { 7291 mirrored_kernelcore = true; 7292 return 0; 7293 } 7294 7295 return cmdline_parse_core(p, &required_kernelcore, 7296 &required_kernelcore_percent); 7297 } 7298 7299 /* 7300 * movablecore=size sets the amount of memory for use for allocations that 7301 * can be reclaimed or migrated. 7302 */ 7303 static int __init cmdline_parse_movablecore(char *p) 7304 { 7305 return cmdline_parse_core(p, &required_movablecore, 7306 &required_movablecore_percent); 7307 } 7308 7309 early_param("kernelcore", cmdline_parse_kernelcore); 7310 early_param("movablecore", cmdline_parse_movablecore); 7311 7312 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 7313 7314 void adjust_managed_page_count(struct page *page, long count) 7315 { 7316 atomic_long_add(count, &page_zone(page)->managed_pages); 7317 totalram_pages_add(count); 7318 #ifdef CONFIG_HIGHMEM 7319 if (PageHighMem(page)) 7320 totalhigh_pages_add(count); 7321 #endif 7322 } 7323 EXPORT_SYMBOL(adjust_managed_page_count); 7324 7325 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 7326 { 7327 void *pos; 7328 unsigned long pages = 0; 7329 7330 start = (void *)PAGE_ALIGN((unsigned long)start); 7331 end = (void *)((unsigned long)end & PAGE_MASK); 7332 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 7333 struct page *page = virt_to_page(pos); 7334 void *direct_map_addr; 7335 7336 /* 7337 * 'direct_map_addr' might be different from 'pos' 7338 * because some architectures' virt_to_page() 7339 * work with aliases. Getting the direct map 7340 * address ensures that we get a _writeable_ 7341 * alias for the memset(). 7342 */ 7343 direct_map_addr = page_address(page); 7344 if ((unsigned int)poison <= 0xFF) 7345 memset(direct_map_addr, poison, PAGE_SIZE); 7346 7347 free_reserved_page(page); 7348 } 7349 7350 if (pages && s) 7351 pr_info("Freeing %s memory: %ldK\n", 7352 s, pages << (PAGE_SHIFT - 10)); 7353 7354 return pages; 7355 } 7356 7357 #ifdef CONFIG_HIGHMEM 7358 void free_highmem_page(struct page *page) 7359 { 7360 __free_reserved_page(page); 7361 totalram_pages_inc(); 7362 atomic_long_inc(&page_zone(page)->managed_pages); 7363 totalhigh_pages_inc(); 7364 } 7365 #endif 7366 7367 7368 void __init mem_init_print_info(const char *str) 7369 { 7370 unsigned long physpages, codesize, datasize, rosize, bss_size; 7371 unsigned long init_code_size, init_data_size; 7372 7373 physpages = get_num_physpages(); 7374 codesize = _etext - _stext; 7375 datasize = _edata - _sdata; 7376 rosize = __end_rodata - __start_rodata; 7377 bss_size = __bss_stop - __bss_start; 7378 init_data_size = __init_end - __init_begin; 7379 init_code_size = _einittext - _sinittext; 7380 7381 /* 7382 * Detect special cases and adjust section sizes accordingly: 7383 * 1) .init.* may be embedded into .data sections 7384 * 2) .init.text.* may be out of [__init_begin, __init_end], 7385 * please refer to arch/tile/kernel/vmlinux.lds.S. 7386 * 3) .rodata.* may be embedded into .text or .data sections. 7387 */ 7388 #define adj_init_size(start, end, size, pos, adj) \ 7389 do { \ 7390 if (start <= pos && pos < end && size > adj) \ 7391 size -= adj; \ 7392 } while (0) 7393 7394 adj_init_size(__init_begin, __init_end, init_data_size, 7395 _sinittext, init_code_size); 7396 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 7397 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 7398 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 7399 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 7400 7401 #undef adj_init_size 7402 7403 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 7404 #ifdef CONFIG_HIGHMEM 7405 ", %luK highmem" 7406 #endif 7407 "%s%s)\n", 7408 nr_free_pages() << (PAGE_SHIFT - 10), 7409 physpages << (PAGE_SHIFT - 10), 7410 codesize >> 10, datasize >> 10, rosize >> 10, 7411 (init_data_size + init_code_size) >> 10, bss_size >> 10, 7412 (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10), 7413 totalcma_pages << (PAGE_SHIFT - 10), 7414 #ifdef CONFIG_HIGHMEM 7415 totalhigh_pages() << (PAGE_SHIFT - 10), 7416 #endif 7417 str ? ", " : "", str ? str : ""); 7418 } 7419 7420 /** 7421 * set_dma_reserve - set the specified number of pages reserved in the first zone 7422 * @new_dma_reserve: The number of pages to mark reserved 7423 * 7424 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 7425 * In the DMA zone, a significant percentage may be consumed by kernel image 7426 * and other unfreeable allocations which can skew the watermarks badly. This 7427 * function may optionally be used to account for unfreeable pages in the 7428 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 7429 * smaller per-cpu batchsize. 7430 */ 7431 void __init set_dma_reserve(unsigned long new_dma_reserve) 7432 { 7433 dma_reserve = new_dma_reserve; 7434 } 7435 7436 void __init free_area_init(unsigned long *zones_size) 7437 { 7438 zero_resv_unavail(); 7439 free_area_init_node(0, zones_size, 7440 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); 7441 } 7442 7443 static int page_alloc_cpu_dead(unsigned int cpu) 7444 { 7445 7446 lru_add_drain_cpu(cpu); 7447 drain_pages(cpu); 7448 7449 /* 7450 * Spill the event counters of the dead processor 7451 * into the current processors event counters. 7452 * This artificially elevates the count of the current 7453 * processor. 7454 */ 7455 vm_events_fold_cpu(cpu); 7456 7457 /* 7458 * Zero the differential counters of the dead processor 7459 * so that the vm statistics are consistent. 7460 * 7461 * This is only okay since the processor is dead and cannot 7462 * race with what we are doing. 7463 */ 7464 cpu_vm_stats_fold(cpu); 7465 return 0; 7466 } 7467 7468 void __init page_alloc_init(void) 7469 { 7470 int ret; 7471 7472 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD, 7473 "mm/page_alloc:dead", NULL, 7474 page_alloc_cpu_dead); 7475 WARN_ON(ret < 0); 7476 } 7477 7478 /* 7479 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 7480 * or min_free_kbytes changes. 7481 */ 7482 static void calculate_totalreserve_pages(void) 7483 { 7484 struct pglist_data *pgdat; 7485 unsigned long reserve_pages = 0; 7486 enum zone_type i, j; 7487 7488 for_each_online_pgdat(pgdat) { 7489 7490 pgdat->totalreserve_pages = 0; 7491 7492 for (i = 0; i < MAX_NR_ZONES; i++) { 7493 struct zone *zone = pgdat->node_zones + i; 7494 long max = 0; 7495 unsigned long managed_pages = zone_managed_pages(zone); 7496 7497 /* Find valid and maximum lowmem_reserve in the zone */ 7498 for (j = i; j < MAX_NR_ZONES; j++) { 7499 if (zone->lowmem_reserve[j] > max) 7500 max = zone->lowmem_reserve[j]; 7501 } 7502 7503 /* we treat the high watermark as reserved pages. */ 7504 max += high_wmark_pages(zone); 7505 7506 if (max > managed_pages) 7507 max = managed_pages; 7508 7509 pgdat->totalreserve_pages += max; 7510 7511 reserve_pages += max; 7512 } 7513 } 7514 totalreserve_pages = reserve_pages; 7515 } 7516 7517 /* 7518 * setup_per_zone_lowmem_reserve - called whenever 7519 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 7520 * has a correct pages reserved value, so an adequate number of 7521 * pages are left in the zone after a successful __alloc_pages(). 7522 */ 7523 static void setup_per_zone_lowmem_reserve(void) 7524 { 7525 struct pglist_data *pgdat; 7526 enum zone_type j, idx; 7527 7528 for_each_online_pgdat(pgdat) { 7529 for (j = 0; j < MAX_NR_ZONES; j++) { 7530 struct zone *zone = pgdat->node_zones + j; 7531 unsigned long managed_pages = zone_managed_pages(zone); 7532 7533 zone->lowmem_reserve[j] = 0; 7534 7535 idx = j; 7536 while (idx) { 7537 struct zone *lower_zone; 7538 7539 idx--; 7540 lower_zone = pgdat->node_zones + idx; 7541 7542 if (sysctl_lowmem_reserve_ratio[idx] < 1) { 7543 sysctl_lowmem_reserve_ratio[idx] = 0; 7544 lower_zone->lowmem_reserve[j] = 0; 7545 } else { 7546 lower_zone->lowmem_reserve[j] = 7547 managed_pages / sysctl_lowmem_reserve_ratio[idx]; 7548 } 7549 managed_pages += zone_managed_pages(lower_zone); 7550 } 7551 } 7552 } 7553 7554 /* update totalreserve_pages */ 7555 calculate_totalreserve_pages(); 7556 } 7557 7558 static void __setup_per_zone_wmarks(void) 7559 { 7560 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 7561 unsigned long lowmem_pages = 0; 7562 struct zone *zone; 7563 unsigned long flags; 7564 7565 /* Calculate total number of !ZONE_HIGHMEM pages */ 7566 for_each_zone(zone) { 7567 if (!is_highmem(zone)) 7568 lowmem_pages += zone_managed_pages(zone); 7569 } 7570 7571 for_each_zone(zone) { 7572 u64 tmp; 7573 7574 spin_lock_irqsave(&zone->lock, flags); 7575 tmp = (u64)pages_min * zone_managed_pages(zone); 7576 do_div(tmp, lowmem_pages); 7577 if (is_highmem(zone)) { 7578 /* 7579 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 7580 * need highmem pages, so cap pages_min to a small 7581 * value here. 7582 * 7583 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 7584 * deltas control async page reclaim, and so should 7585 * not be capped for highmem. 7586 */ 7587 unsigned long min_pages; 7588 7589 min_pages = zone_managed_pages(zone) / 1024; 7590 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 7591 zone->_watermark[WMARK_MIN] = min_pages; 7592 } else { 7593 /* 7594 * If it's a lowmem zone, reserve a number of pages 7595 * proportionate to the zone's size. 7596 */ 7597 zone->_watermark[WMARK_MIN] = tmp; 7598 } 7599 7600 /* 7601 * Set the kswapd watermarks distance according to the 7602 * scale factor in proportion to available memory, but 7603 * ensure a minimum size on small systems. 7604 */ 7605 tmp = max_t(u64, tmp >> 2, 7606 mult_frac(zone_managed_pages(zone), 7607 watermark_scale_factor, 10000)); 7608 7609 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 7610 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2; 7611 zone->watermark_boost = 0; 7612 7613 spin_unlock_irqrestore(&zone->lock, flags); 7614 } 7615 7616 /* update totalreserve_pages */ 7617 calculate_totalreserve_pages(); 7618 } 7619 7620 /** 7621 * setup_per_zone_wmarks - called when min_free_kbytes changes 7622 * or when memory is hot-{added|removed} 7623 * 7624 * Ensures that the watermark[min,low,high] values for each zone are set 7625 * correctly with respect to min_free_kbytes. 7626 */ 7627 void setup_per_zone_wmarks(void) 7628 { 7629 static DEFINE_SPINLOCK(lock); 7630 7631 spin_lock(&lock); 7632 __setup_per_zone_wmarks(); 7633 spin_unlock(&lock); 7634 } 7635 7636 /* 7637 * Initialise min_free_kbytes. 7638 * 7639 * For small machines we want it small (128k min). For large machines 7640 * we want it large (64MB max). But it is not linear, because network 7641 * bandwidth does not increase linearly with machine size. We use 7642 * 7643 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 7644 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 7645 * 7646 * which yields 7647 * 7648 * 16MB: 512k 7649 * 32MB: 724k 7650 * 64MB: 1024k 7651 * 128MB: 1448k 7652 * 256MB: 2048k 7653 * 512MB: 2896k 7654 * 1024MB: 4096k 7655 * 2048MB: 5792k 7656 * 4096MB: 8192k 7657 * 8192MB: 11584k 7658 * 16384MB: 16384k 7659 */ 7660 int __meminit init_per_zone_wmark_min(void) 7661 { 7662 unsigned long lowmem_kbytes; 7663 int new_min_free_kbytes; 7664 7665 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 7666 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 7667 7668 if (new_min_free_kbytes > user_min_free_kbytes) { 7669 min_free_kbytes = new_min_free_kbytes; 7670 if (min_free_kbytes < 128) 7671 min_free_kbytes = 128; 7672 if (min_free_kbytes > 65536) 7673 min_free_kbytes = 65536; 7674 } else { 7675 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 7676 new_min_free_kbytes, user_min_free_kbytes); 7677 } 7678 setup_per_zone_wmarks(); 7679 refresh_zone_stat_thresholds(); 7680 setup_per_zone_lowmem_reserve(); 7681 7682 #ifdef CONFIG_NUMA 7683 setup_min_unmapped_ratio(); 7684 setup_min_slab_ratio(); 7685 #endif 7686 7687 return 0; 7688 } 7689 core_initcall(init_per_zone_wmark_min) 7690 7691 /* 7692 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 7693 * that we can call two helper functions whenever min_free_kbytes 7694 * changes. 7695 */ 7696 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 7697 void __user *buffer, size_t *length, loff_t *ppos) 7698 { 7699 int rc; 7700 7701 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7702 if (rc) 7703 return rc; 7704 7705 if (write) { 7706 user_min_free_kbytes = min_free_kbytes; 7707 setup_per_zone_wmarks(); 7708 } 7709 return 0; 7710 } 7711 7712 int watermark_boost_factor_sysctl_handler(struct ctl_table *table, int write, 7713 void __user *buffer, size_t *length, loff_t *ppos) 7714 { 7715 int rc; 7716 7717 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7718 if (rc) 7719 return rc; 7720 7721 return 0; 7722 } 7723 7724 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 7725 void __user *buffer, size_t *length, loff_t *ppos) 7726 { 7727 int rc; 7728 7729 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7730 if (rc) 7731 return rc; 7732 7733 if (write) 7734 setup_per_zone_wmarks(); 7735 7736 return 0; 7737 } 7738 7739 #ifdef CONFIG_NUMA 7740 static void setup_min_unmapped_ratio(void) 7741 { 7742 pg_data_t *pgdat; 7743 struct zone *zone; 7744 7745 for_each_online_pgdat(pgdat) 7746 pgdat->min_unmapped_pages = 0; 7747 7748 for_each_zone(zone) 7749 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * 7750 sysctl_min_unmapped_ratio) / 100; 7751 } 7752 7753 7754 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 7755 void __user *buffer, size_t *length, loff_t *ppos) 7756 { 7757 int rc; 7758 7759 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7760 if (rc) 7761 return rc; 7762 7763 setup_min_unmapped_ratio(); 7764 7765 return 0; 7766 } 7767 7768 static void setup_min_slab_ratio(void) 7769 { 7770 pg_data_t *pgdat; 7771 struct zone *zone; 7772 7773 for_each_online_pgdat(pgdat) 7774 pgdat->min_slab_pages = 0; 7775 7776 for_each_zone(zone) 7777 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * 7778 sysctl_min_slab_ratio) / 100; 7779 } 7780 7781 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 7782 void __user *buffer, size_t *length, loff_t *ppos) 7783 { 7784 int rc; 7785 7786 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7787 if (rc) 7788 return rc; 7789 7790 setup_min_slab_ratio(); 7791 7792 return 0; 7793 } 7794 #endif 7795 7796 /* 7797 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 7798 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 7799 * whenever sysctl_lowmem_reserve_ratio changes. 7800 * 7801 * The reserve ratio obviously has absolutely no relation with the 7802 * minimum watermarks. The lowmem reserve ratio can only make sense 7803 * if in function of the boot time zone sizes. 7804 */ 7805 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 7806 void __user *buffer, size_t *length, loff_t *ppos) 7807 { 7808 proc_dointvec_minmax(table, write, buffer, length, ppos); 7809 setup_per_zone_lowmem_reserve(); 7810 return 0; 7811 } 7812 7813 /* 7814 * percpu_pagelist_fraction - changes the pcp->high for each zone on each 7815 * cpu. It is the fraction of total pages in each zone that a hot per cpu 7816 * pagelist can have before it gets flushed back to buddy allocator. 7817 */ 7818 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, 7819 void __user *buffer, size_t *length, loff_t *ppos) 7820 { 7821 struct zone *zone; 7822 int old_percpu_pagelist_fraction; 7823 int ret; 7824 7825 mutex_lock(&pcp_batch_high_lock); 7826 old_percpu_pagelist_fraction = percpu_pagelist_fraction; 7827 7828 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 7829 if (!write || ret < 0) 7830 goto out; 7831 7832 /* Sanity checking to avoid pcp imbalance */ 7833 if (percpu_pagelist_fraction && 7834 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { 7835 percpu_pagelist_fraction = old_percpu_pagelist_fraction; 7836 ret = -EINVAL; 7837 goto out; 7838 } 7839 7840 /* No change? */ 7841 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) 7842 goto out; 7843 7844 for_each_populated_zone(zone) { 7845 unsigned int cpu; 7846 7847 for_each_possible_cpu(cpu) 7848 pageset_set_high_and_batch(zone, 7849 per_cpu_ptr(zone->pageset, cpu)); 7850 } 7851 out: 7852 mutex_unlock(&pcp_batch_high_lock); 7853 return ret; 7854 } 7855 7856 #ifdef CONFIG_NUMA 7857 int hashdist = HASHDIST_DEFAULT; 7858 7859 static int __init set_hashdist(char *str) 7860 { 7861 if (!str) 7862 return 0; 7863 hashdist = simple_strtoul(str, &str, 0); 7864 return 1; 7865 } 7866 __setup("hashdist=", set_hashdist); 7867 #endif 7868 7869 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 7870 /* 7871 * Returns the number of pages that arch has reserved but 7872 * is not known to alloc_large_system_hash(). 7873 */ 7874 static unsigned long __init arch_reserved_kernel_pages(void) 7875 { 7876 return 0; 7877 } 7878 #endif 7879 7880 /* 7881 * Adaptive scale is meant to reduce sizes of hash tables on large memory 7882 * machines. As memory size is increased the scale is also increased but at 7883 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 7884 * quadruples the scale is increased by one, which means the size of hash table 7885 * only doubles, instead of quadrupling as well. 7886 * Because 32-bit systems cannot have large physical memory, where this scaling 7887 * makes sense, it is disabled on such platforms. 7888 */ 7889 #if __BITS_PER_LONG > 32 7890 #define ADAPT_SCALE_BASE (64ul << 30) 7891 #define ADAPT_SCALE_SHIFT 2 7892 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 7893 #endif 7894 7895 /* 7896 * allocate a large system hash table from bootmem 7897 * - it is assumed that the hash table must contain an exact power-of-2 7898 * quantity of entries 7899 * - limit is the number of hash buckets, not the total allocation size 7900 */ 7901 void *__init alloc_large_system_hash(const char *tablename, 7902 unsigned long bucketsize, 7903 unsigned long numentries, 7904 int scale, 7905 int flags, 7906 unsigned int *_hash_shift, 7907 unsigned int *_hash_mask, 7908 unsigned long low_limit, 7909 unsigned long high_limit) 7910 { 7911 unsigned long long max = high_limit; 7912 unsigned long log2qty, size; 7913 void *table = NULL; 7914 gfp_t gfp_flags; 7915 7916 /* allow the kernel cmdline to have a say */ 7917 if (!numentries) { 7918 /* round applicable memory size up to nearest megabyte */ 7919 numentries = nr_kernel_pages; 7920 numentries -= arch_reserved_kernel_pages(); 7921 7922 /* It isn't necessary when PAGE_SIZE >= 1MB */ 7923 if (PAGE_SHIFT < 20) 7924 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 7925 7926 #if __BITS_PER_LONG > 32 7927 if (!high_limit) { 7928 unsigned long adapt; 7929 7930 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 7931 adapt <<= ADAPT_SCALE_SHIFT) 7932 scale++; 7933 } 7934 #endif 7935 7936 /* limit to 1 bucket per 2^scale bytes of low memory */ 7937 if (scale > PAGE_SHIFT) 7938 numentries >>= (scale - PAGE_SHIFT); 7939 else 7940 numentries <<= (PAGE_SHIFT - scale); 7941 7942 /* Make sure we've got at least a 0-order allocation.. */ 7943 if (unlikely(flags & HASH_SMALL)) { 7944 /* Makes no sense without HASH_EARLY */ 7945 WARN_ON(!(flags & HASH_EARLY)); 7946 if (!(numentries >> *_hash_shift)) { 7947 numentries = 1UL << *_hash_shift; 7948 BUG_ON(!numentries); 7949 } 7950 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 7951 numentries = PAGE_SIZE / bucketsize; 7952 } 7953 numentries = roundup_pow_of_two(numentries); 7954 7955 /* limit allocation size to 1/16 total memory by default */ 7956 if (max == 0) { 7957 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 7958 do_div(max, bucketsize); 7959 } 7960 max = min(max, 0x80000000ULL); 7961 7962 if (numentries < low_limit) 7963 numentries = low_limit; 7964 if (numentries > max) 7965 numentries = max; 7966 7967 log2qty = ilog2(numentries); 7968 7969 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 7970 do { 7971 size = bucketsize << log2qty; 7972 if (flags & HASH_EARLY) { 7973 if (flags & HASH_ZERO) 7974 table = memblock_alloc(size, SMP_CACHE_BYTES); 7975 else 7976 table = memblock_alloc_raw(size, 7977 SMP_CACHE_BYTES); 7978 } else if (hashdist) { 7979 table = __vmalloc(size, gfp_flags, PAGE_KERNEL); 7980 } else { 7981 /* 7982 * If bucketsize is not a power-of-two, we may free 7983 * some pages at the end of hash table which 7984 * alloc_pages_exact() automatically does 7985 */ 7986 if (get_order(size) < MAX_ORDER) { 7987 table = alloc_pages_exact(size, gfp_flags); 7988 kmemleak_alloc(table, size, 1, gfp_flags); 7989 } 7990 } 7991 } while (!table && size > PAGE_SIZE && --log2qty); 7992 7993 if (!table) 7994 panic("Failed to allocate %s hash table\n", tablename); 7995 7996 pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n", 7997 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size); 7998 7999 if (_hash_shift) 8000 *_hash_shift = log2qty; 8001 if (_hash_mask) 8002 *_hash_mask = (1 << log2qty) - 1; 8003 8004 return table; 8005 } 8006 8007 /* 8008 * This function checks whether pageblock includes unmovable pages or not. 8009 * If @count is not zero, it is okay to include less @count unmovable pages 8010 * 8011 * PageLRU check without isolation or lru_lock could race so that 8012 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable 8013 * check without lock_page also may miss some movable non-lru pages at 8014 * race condition. So you can't expect this function should be exact. 8015 */ 8016 bool has_unmovable_pages(struct zone *zone, struct page *page, int count, 8017 int migratetype, int flags) 8018 { 8019 unsigned long found; 8020 unsigned long iter = 0; 8021 unsigned long pfn = page_to_pfn(page); 8022 const char *reason = "unmovable page"; 8023 8024 /* 8025 * TODO we could make this much more efficient by not checking every 8026 * page in the range if we know all of them are in MOVABLE_ZONE and 8027 * that the movable zone guarantees that pages are migratable but 8028 * the later is not the case right now unfortunatelly. E.g. movablecore 8029 * can still lead to having bootmem allocations in zone_movable. 8030 */ 8031 8032 if (is_migrate_cma_page(page)) { 8033 /* 8034 * CMA allocations (alloc_contig_range) really need to mark 8035 * isolate CMA pageblocks even when they are not movable in fact 8036 * so consider them movable here. 8037 */ 8038 if (is_migrate_cma(migratetype)) 8039 return false; 8040 8041 reason = "CMA page"; 8042 goto unmovable; 8043 } 8044 8045 for (found = 0; iter < pageblock_nr_pages; iter++) { 8046 unsigned long check = pfn + iter; 8047 8048 if (!pfn_valid_within(check)) 8049 continue; 8050 8051 page = pfn_to_page(check); 8052 8053 if (PageReserved(page)) 8054 goto unmovable; 8055 8056 /* 8057 * If the zone is movable and we have ruled out all reserved 8058 * pages then it should be reasonably safe to assume the rest 8059 * is movable. 8060 */ 8061 if (zone_idx(zone) == ZONE_MOVABLE) 8062 continue; 8063 8064 /* 8065 * Hugepages are not in LRU lists, but they're movable. 8066 * We need not scan over tail pages because we don't 8067 * handle each tail page individually in migration. 8068 */ 8069 if (PageHuge(page)) { 8070 struct page *head = compound_head(page); 8071 unsigned int skip_pages; 8072 8073 if (!hugepage_migration_supported(page_hstate(head))) 8074 goto unmovable; 8075 8076 skip_pages = (1 << compound_order(head)) - (page - head); 8077 iter += skip_pages - 1; 8078 continue; 8079 } 8080 8081 /* 8082 * We can't use page_count without pin a page 8083 * because another CPU can free compound page. 8084 * This check already skips compound tails of THP 8085 * because their page->_refcount is zero at all time. 8086 */ 8087 if (!page_ref_count(page)) { 8088 if (PageBuddy(page)) 8089 iter += (1 << page_order(page)) - 1; 8090 continue; 8091 } 8092 8093 /* 8094 * The HWPoisoned page may be not in buddy system, and 8095 * page_count() is not 0. 8096 */ 8097 if ((flags & SKIP_HWPOISON) && PageHWPoison(page)) 8098 continue; 8099 8100 if (__PageMovable(page)) 8101 continue; 8102 8103 if (!PageLRU(page)) 8104 found++; 8105 /* 8106 * If there are RECLAIMABLE pages, we need to check 8107 * it. But now, memory offline itself doesn't call 8108 * shrink_node_slabs() and it still to be fixed. 8109 */ 8110 /* 8111 * If the page is not RAM, page_count()should be 0. 8112 * we don't need more check. This is an _used_ not-movable page. 8113 * 8114 * The problematic thing here is PG_reserved pages. PG_reserved 8115 * is set to both of a memory hole page and a _used_ kernel 8116 * page at boot. 8117 */ 8118 if (found > count) 8119 goto unmovable; 8120 } 8121 return false; 8122 unmovable: 8123 WARN_ON_ONCE(zone_idx(zone) == ZONE_MOVABLE); 8124 if (flags & REPORT_FAILURE) 8125 dump_page(pfn_to_page(pfn + iter), reason); 8126 return true; 8127 } 8128 8129 #if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA) 8130 8131 static unsigned long pfn_max_align_down(unsigned long pfn) 8132 { 8133 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, 8134 pageblock_nr_pages) - 1); 8135 } 8136 8137 static unsigned long pfn_max_align_up(unsigned long pfn) 8138 { 8139 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, 8140 pageblock_nr_pages)); 8141 } 8142 8143 /* [start, end) must belong to a single zone. */ 8144 static int __alloc_contig_migrate_range(struct compact_control *cc, 8145 unsigned long start, unsigned long end) 8146 { 8147 /* This function is based on compact_zone() from compaction.c. */ 8148 unsigned long nr_reclaimed; 8149 unsigned long pfn = start; 8150 unsigned int tries = 0; 8151 int ret = 0; 8152 8153 migrate_prep(); 8154 8155 while (pfn < end || !list_empty(&cc->migratepages)) { 8156 if (fatal_signal_pending(current)) { 8157 ret = -EINTR; 8158 break; 8159 } 8160 8161 if (list_empty(&cc->migratepages)) { 8162 cc->nr_migratepages = 0; 8163 pfn = isolate_migratepages_range(cc, pfn, end); 8164 if (!pfn) { 8165 ret = -EINTR; 8166 break; 8167 } 8168 tries = 0; 8169 } else if (++tries == 5) { 8170 ret = ret < 0 ? ret : -EBUSY; 8171 break; 8172 } 8173 8174 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 8175 &cc->migratepages); 8176 cc->nr_migratepages -= nr_reclaimed; 8177 8178 ret = migrate_pages(&cc->migratepages, alloc_migrate_target, 8179 NULL, 0, cc->mode, MR_CONTIG_RANGE); 8180 } 8181 if (ret < 0) { 8182 putback_movable_pages(&cc->migratepages); 8183 return ret; 8184 } 8185 return 0; 8186 } 8187 8188 /** 8189 * alloc_contig_range() -- tries to allocate given range of pages 8190 * @start: start PFN to allocate 8191 * @end: one-past-the-last PFN to allocate 8192 * @migratetype: migratetype of the underlaying pageblocks (either 8193 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 8194 * in range must have the same migratetype and it must 8195 * be either of the two. 8196 * @gfp_mask: GFP mask to use during compaction 8197 * 8198 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 8199 * aligned. The PFN range must belong to a single zone. 8200 * 8201 * The first thing this routine does is attempt to MIGRATE_ISOLATE all 8202 * pageblocks in the range. Once isolated, the pageblocks should not 8203 * be modified by others. 8204 * 8205 * Return: zero on success or negative error code. On success all 8206 * pages which PFN is in [start, end) are allocated for the caller and 8207 * need to be freed with free_contig_range(). 8208 */ 8209 int alloc_contig_range(unsigned long start, unsigned long end, 8210 unsigned migratetype, gfp_t gfp_mask) 8211 { 8212 unsigned long outer_start, outer_end; 8213 unsigned int order; 8214 int ret = 0; 8215 8216 struct compact_control cc = { 8217 .nr_migratepages = 0, 8218 .order = -1, 8219 .zone = page_zone(pfn_to_page(start)), 8220 .mode = MIGRATE_SYNC, 8221 .ignore_skip_hint = true, 8222 .no_set_skip_hint = true, 8223 .gfp_mask = current_gfp_context(gfp_mask), 8224 }; 8225 INIT_LIST_HEAD(&cc.migratepages); 8226 8227 /* 8228 * What we do here is we mark all pageblocks in range as 8229 * MIGRATE_ISOLATE. Because pageblock and max order pages may 8230 * have different sizes, and due to the way page allocator 8231 * work, we align the range to biggest of the two pages so 8232 * that page allocator won't try to merge buddies from 8233 * different pageblocks and change MIGRATE_ISOLATE to some 8234 * other migration type. 8235 * 8236 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 8237 * migrate the pages from an unaligned range (ie. pages that 8238 * we are interested in). This will put all the pages in 8239 * range back to page allocator as MIGRATE_ISOLATE. 8240 * 8241 * When this is done, we take the pages in range from page 8242 * allocator removing them from the buddy system. This way 8243 * page allocator will never consider using them. 8244 * 8245 * This lets us mark the pageblocks back as 8246 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 8247 * aligned range but not in the unaligned, original range are 8248 * put back to page allocator so that buddy can use them. 8249 */ 8250 8251 ret = start_isolate_page_range(pfn_max_align_down(start), 8252 pfn_max_align_up(end), migratetype, 0); 8253 if (ret < 0) 8254 return ret; 8255 8256 /* 8257 * In case of -EBUSY, we'd like to know which page causes problem. 8258 * So, just fall through. test_pages_isolated() has a tracepoint 8259 * which will report the busy page. 8260 * 8261 * It is possible that busy pages could become available before 8262 * the call to test_pages_isolated, and the range will actually be 8263 * allocated. So, if we fall through be sure to clear ret so that 8264 * -EBUSY is not accidentally used or returned to caller. 8265 */ 8266 ret = __alloc_contig_migrate_range(&cc, start, end); 8267 if (ret && ret != -EBUSY) 8268 goto done; 8269 ret =0; 8270 8271 /* 8272 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 8273 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 8274 * more, all pages in [start, end) are free in page allocator. 8275 * What we are going to do is to allocate all pages from 8276 * [start, end) (that is remove them from page allocator). 8277 * 8278 * The only problem is that pages at the beginning and at the 8279 * end of interesting range may be not aligned with pages that 8280 * page allocator holds, ie. they can be part of higher order 8281 * pages. Because of this, we reserve the bigger range and 8282 * once this is done free the pages we are not interested in. 8283 * 8284 * We don't have to hold zone->lock here because the pages are 8285 * isolated thus they won't get removed from buddy. 8286 */ 8287 8288 lru_add_drain_all(); 8289 8290 order = 0; 8291 outer_start = start; 8292 while (!PageBuddy(pfn_to_page(outer_start))) { 8293 if (++order >= MAX_ORDER) { 8294 outer_start = start; 8295 break; 8296 } 8297 outer_start &= ~0UL << order; 8298 } 8299 8300 if (outer_start != start) { 8301 order = page_order(pfn_to_page(outer_start)); 8302 8303 /* 8304 * outer_start page could be small order buddy page and 8305 * it doesn't include start page. Adjust outer_start 8306 * in this case to report failed page properly 8307 * on tracepoint in test_pages_isolated() 8308 */ 8309 if (outer_start + (1UL << order) <= start) 8310 outer_start = start; 8311 } 8312 8313 /* Make sure the range is really isolated. */ 8314 if (test_pages_isolated(outer_start, end, false)) { 8315 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n", 8316 __func__, outer_start, end); 8317 ret = -EBUSY; 8318 goto done; 8319 } 8320 8321 /* Grab isolated pages from freelists. */ 8322 outer_end = isolate_freepages_range(&cc, outer_start, end); 8323 if (!outer_end) { 8324 ret = -EBUSY; 8325 goto done; 8326 } 8327 8328 /* Free head and tail (if any) */ 8329 if (start != outer_start) 8330 free_contig_range(outer_start, start - outer_start); 8331 if (end != outer_end) 8332 free_contig_range(end, outer_end - end); 8333 8334 done: 8335 undo_isolate_page_range(pfn_max_align_down(start), 8336 pfn_max_align_up(end), migratetype); 8337 return ret; 8338 } 8339 8340 void free_contig_range(unsigned long pfn, unsigned nr_pages) 8341 { 8342 unsigned int count = 0; 8343 8344 for (; nr_pages--; pfn++) { 8345 struct page *page = pfn_to_page(pfn); 8346 8347 count += page_count(page) != 1; 8348 __free_page(page); 8349 } 8350 WARN(count != 0, "%d pages are still in use!\n", count); 8351 } 8352 #endif 8353 8354 #ifdef CONFIG_MEMORY_HOTPLUG 8355 /* 8356 * The zone indicated has a new number of managed_pages; batch sizes and percpu 8357 * page high values need to be recalulated. 8358 */ 8359 void __meminit zone_pcp_update(struct zone *zone) 8360 { 8361 unsigned cpu; 8362 mutex_lock(&pcp_batch_high_lock); 8363 for_each_possible_cpu(cpu) 8364 pageset_set_high_and_batch(zone, 8365 per_cpu_ptr(zone->pageset, cpu)); 8366 mutex_unlock(&pcp_batch_high_lock); 8367 } 8368 #endif 8369 8370 void zone_pcp_reset(struct zone *zone) 8371 { 8372 unsigned long flags; 8373 int cpu; 8374 struct per_cpu_pageset *pset; 8375 8376 /* avoid races with drain_pages() */ 8377 local_irq_save(flags); 8378 if (zone->pageset != &boot_pageset) { 8379 for_each_online_cpu(cpu) { 8380 pset = per_cpu_ptr(zone->pageset, cpu); 8381 drain_zonestat(zone, pset); 8382 } 8383 free_percpu(zone->pageset); 8384 zone->pageset = &boot_pageset; 8385 } 8386 local_irq_restore(flags); 8387 } 8388 8389 #ifdef CONFIG_MEMORY_HOTREMOVE 8390 /* 8391 * All pages in the range must be in a single zone and isolated 8392 * before calling this. 8393 */ 8394 void 8395 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 8396 { 8397 struct page *page; 8398 struct zone *zone; 8399 unsigned int order, i; 8400 unsigned long pfn; 8401 unsigned long flags; 8402 /* find the first valid pfn */ 8403 for (pfn = start_pfn; pfn < end_pfn; pfn++) 8404 if (pfn_valid(pfn)) 8405 break; 8406 if (pfn == end_pfn) 8407 return; 8408 offline_mem_sections(pfn, end_pfn); 8409 zone = page_zone(pfn_to_page(pfn)); 8410 spin_lock_irqsave(&zone->lock, flags); 8411 pfn = start_pfn; 8412 while (pfn < end_pfn) { 8413 if (!pfn_valid(pfn)) { 8414 pfn++; 8415 continue; 8416 } 8417 page = pfn_to_page(pfn); 8418 /* 8419 * The HWPoisoned page may be not in buddy system, and 8420 * page_count() is not 0. 8421 */ 8422 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 8423 pfn++; 8424 SetPageReserved(page); 8425 continue; 8426 } 8427 8428 BUG_ON(page_count(page)); 8429 BUG_ON(!PageBuddy(page)); 8430 order = page_order(page); 8431 #ifdef CONFIG_DEBUG_VM 8432 pr_info("remove from free list %lx %d %lx\n", 8433 pfn, 1 << order, end_pfn); 8434 #endif 8435 list_del(&page->lru); 8436 rmv_page_order(page); 8437 zone->free_area[order].nr_free--; 8438 for (i = 0; i < (1 << order); i++) 8439 SetPageReserved((page+i)); 8440 pfn += (1 << order); 8441 } 8442 spin_unlock_irqrestore(&zone->lock, flags); 8443 } 8444 #endif 8445 8446 bool is_free_buddy_page(struct page *page) 8447 { 8448 struct zone *zone = page_zone(page); 8449 unsigned long pfn = page_to_pfn(page); 8450 unsigned long flags; 8451 unsigned int order; 8452 8453 spin_lock_irqsave(&zone->lock, flags); 8454 for (order = 0; order < MAX_ORDER; order++) { 8455 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8456 8457 if (PageBuddy(page_head) && page_order(page_head) >= order) 8458 break; 8459 } 8460 spin_unlock_irqrestore(&zone->lock, flags); 8461 8462 return order < MAX_ORDER; 8463 } 8464 8465 #ifdef CONFIG_MEMORY_FAILURE 8466 /* 8467 * Set PG_hwpoison flag if a given page is confirmed to be a free page. This 8468 * test is performed under the zone lock to prevent a race against page 8469 * allocation. 8470 */ 8471 bool set_hwpoison_free_buddy_page(struct page *page) 8472 { 8473 struct zone *zone = page_zone(page); 8474 unsigned long pfn = page_to_pfn(page); 8475 unsigned long flags; 8476 unsigned int order; 8477 bool hwpoisoned = false; 8478 8479 spin_lock_irqsave(&zone->lock, flags); 8480 for (order = 0; order < MAX_ORDER; order++) { 8481 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8482 8483 if (PageBuddy(page_head) && page_order(page_head) >= order) { 8484 if (!TestSetPageHWPoison(page)) 8485 hwpoisoned = true; 8486 break; 8487 } 8488 } 8489 spin_unlock_irqrestore(&zone->lock, flags); 8490 8491 return hwpoisoned; 8492 } 8493 #endif 8494