1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/page_alloc.c 4 * 5 * Manages the free list, the system allocates free pages here. 6 * Note that kmalloc() lives in slab.c 7 * 8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 9 * Swap reorganised 29.12.95, Stephen Tweedie 10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 16 */ 17 18 #include <linux/stddef.h> 19 #include <linux/mm.h> 20 #include <linux/highmem.h> 21 #include <linux/swap.h> 22 #include <linux/interrupt.h> 23 #include <linux/pagemap.h> 24 #include <linux/jiffies.h> 25 #include <linux/memblock.h> 26 #include <linux/compiler.h> 27 #include <linux/kernel.h> 28 #include <linux/kasan.h> 29 #include <linux/module.h> 30 #include <linux/suspend.h> 31 #include <linux/pagevec.h> 32 #include <linux/blkdev.h> 33 #include <linux/slab.h> 34 #include <linux/ratelimit.h> 35 #include <linux/oom.h> 36 #include <linux/topology.h> 37 #include <linux/sysctl.h> 38 #include <linux/cpu.h> 39 #include <linux/cpuset.h> 40 #include <linux/memory_hotplug.h> 41 #include <linux/nodemask.h> 42 #include <linux/vmalloc.h> 43 #include <linux/vmstat.h> 44 #include <linux/mempolicy.h> 45 #include <linux/memremap.h> 46 #include <linux/stop_machine.h> 47 #include <linux/random.h> 48 #include <linux/sort.h> 49 #include <linux/pfn.h> 50 #include <linux/backing-dev.h> 51 #include <linux/fault-inject.h> 52 #include <linux/page-isolation.h> 53 #include <linux/debugobjects.h> 54 #include <linux/kmemleak.h> 55 #include <linux/compaction.h> 56 #include <trace/events/kmem.h> 57 #include <trace/events/oom.h> 58 #include <linux/prefetch.h> 59 #include <linux/mm_inline.h> 60 #include <linux/migrate.h> 61 #include <linux/hugetlb.h> 62 #include <linux/sched/rt.h> 63 #include <linux/sched/mm.h> 64 #include <linux/page_owner.h> 65 #include <linux/kthread.h> 66 #include <linux/memcontrol.h> 67 #include <linux/ftrace.h> 68 #include <linux/lockdep.h> 69 #include <linux/nmi.h> 70 #include <linux/psi.h> 71 72 #include <asm/sections.h> 73 #include <asm/tlbflush.h> 74 #include <asm/div64.h> 75 #include "internal.h" 76 #include "shuffle.h" 77 78 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 79 static DEFINE_MUTEX(pcp_batch_high_lock); 80 #define MIN_PERCPU_PAGELIST_FRACTION (8) 81 82 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 83 DEFINE_PER_CPU(int, numa_node); 84 EXPORT_PER_CPU_SYMBOL(numa_node); 85 #endif 86 87 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); 88 89 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 90 /* 91 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 92 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 93 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 94 * defined in <linux/topology.h>. 95 */ 96 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 97 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 98 int _node_numa_mem_[MAX_NUMNODES]; 99 #endif 100 101 /* work_structs for global per-cpu drains */ 102 struct pcpu_drain { 103 struct zone *zone; 104 struct work_struct work; 105 }; 106 DEFINE_MUTEX(pcpu_drain_mutex); 107 DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain); 108 109 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 110 volatile unsigned long latent_entropy __latent_entropy; 111 EXPORT_SYMBOL(latent_entropy); 112 #endif 113 114 /* 115 * Array of node states. 116 */ 117 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 118 [N_POSSIBLE] = NODE_MASK_ALL, 119 [N_ONLINE] = { { [0] = 1UL } }, 120 #ifndef CONFIG_NUMA 121 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 122 #ifdef CONFIG_HIGHMEM 123 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 124 #endif 125 [N_MEMORY] = { { [0] = 1UL } }, 126 [N_CPU] = { { [0] = 1UL } }, 127 #endif /* NUMA */ 128 }; 129 EXPORT_SYMBOL(node_states); 130 131 atomic_long_t _totalram_pages __read_mostly; 132 EXPORT_SYMBOL(_totalram_pages); 133 unsigned long totalreserve_pages __read_mostly; 134 unsigned long totalcma_pages __read_mostly; 135 136 int percpu_pagelist_fraction; 137 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 138 #ifdef CONFIG_INIT_ON_ALLOC_DEFAULT_ON 139 DEFINE_STATIC_KEY_TRUE(init_on_alloc); 140 #else 141 DEFINE_STATIC_KEY_FALSE(init_on_alloc); 142 #endif 143 EXPORT_SYMBOL(init_on_alloc); 144 145 #ifdef CONFIG_INIT_ON_FREE_DEFAULT_ON 146 DEFINE_STATIC_KEY_TRUE(init_on_free); 147 #else 148 DEFINE_STATIC_KEY_FALSE(init_on_free); 149 #endif 150 EXPORT_SYMBOL(init_on_free); 151 152 static int __init early_init_on_alloc(char *buf) 153 { 154 int ret; 155 bool bool_result; 156 157 if (!buf) 158 return -EINVAL; 159 ret = kstrtobool(buf, &bool_result); 160 if (bool_result && page_poisoning_enabled()) 161 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_alloc\n"); 162 if (bool_result) 163 static_branch_enable(&init_on_alloc); 164 else 165 static_branch_disable(&init_on_alloc); 166 return ret; 167 } 168 early_param("init_on_alloc", early_init_on_alloc); 169 170 static int __init early_init_on_free(char *buf) 171 { 172 int ret; 173 bool bool_result; 174 175 if (!buf) 176 return -EINVAL; 177 ret = kstrtobool(buf, &bool_result); 178 if (bool_result && page_poisoning_enabled()) 179 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_free\n"); 180 if (bool_result) 181 static_branch_enable(&init_on_free); 182 else 183 static_branch_disable(&init_on_free); 184 return ret; 185 } 186 early_param("init_on_free", early_init_on_free); 187 188 /* 189 * A cached value of the page's pageblock's migratetype, used when the page is 190 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 191 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 192 * Also the migratetype set in the page does not necessarily match the pcplist 193 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 194 * other index - this ensures that it will be put on the correct CMA freelist. 195 */ 196 static inline int get_pcppage_migratetype(struct page *page) 197 { 198 return page->index; 199 } 200 201 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 202 { 203 page->index = migratetype; 204 } 205 206 #ifdef CONFIG_PM_SLEEP 207 /* 208 * The following functions are used by the suspend/hibernate code to temporarily 209 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 210 * while devices are suspended. To avoid races with the suspend/hibernate code, 211 * they should always be called with system_transition_mutex held 212 * (gfp_allowed_mask also should only be modified with system_transition_mutex 213 * held, unless the suspend/hibernate code is guaranteed not to run in parallel 214 * with that modification). 215 */ 216 217 static gfp_t saved_gfp_mask; 218 219 void pm_restore_gfp_mask(void) 220 { 221 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 222 if (saved_gfp_mask) { 223 gfp_allowed_mask = saved_gfp_mask; 224 saved_gfp_mask = 0; 225 } 226 } 227 228 void pm_restrict_gfp_mask(void) 229 { 230 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 231 WARN_ON(saved_gfp_mask); 232 saved_gfp_mask = gfp_allowed_mask; 233 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 234 } 235 236 bool pm_suspended_storage(void) 237 { 238 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 239 return false; 240 return true; 241 } 242 #endif /* CONFIG_PM_SLEEP */ 243 244 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 245 unsigned int pageblock_order __read_mostly; 246 #endif 247 248 static void __free_pages_ok(struct page *page, unsigned int order); 249 250 /* 251 * results with 256, 32 in the lowmem_reserve sysctl: 252 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 253 * 1G machine -> (16M dma, 784M normal, 224M high) 254 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 255 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 256 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 257 * 258 * TBD: should special case ZONE_DMA32 machines here - in those we normally 259 * don't need any ZONE_NORMAL reservation 260 */ 261 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { 262 #ifdef CONFIG_ZONE_DMA 263 [ZONE_DMA] = 256, 264 #endif 265 #ifdef CONFIG_ZONE_DMA32 266 [ZONE_DMA32] = 256, 267 #endif 268 [ZONE_NORMAL] = 32, 269 #ifdef CONFIG_HIGHMEM 270 [ZONE_HIGHMEM] = 0, 271 #endif 272 [ZONE_MOVABLE] = 0, 273 }; 274 275 static char * const zone_names[MAX_NR_ZONES] = { 276 #ifdef CONFIG_ZONE_DMA 277 "DMA", 278 #endif 279 #ifdef CONFIG_ZONE_DMA32 280 "DMA32", 281 #endif 282 "Normal", 283 #ifdef CONFIG_HIGHMEM 284 "HighMem", 285 #endif 286 "Movable", 287 #ifdef CONFIG_ZONE_DEVICE 288 "Device", 289 #endif 290 }; 291 292 const char * const migratetype_names[MIGRATE_TYPES] = { 293 "Unmovable", 294 "Movable", 295 "Reclaimable", 296 "HighAtomic", 297 #ifdef CONFIG_CMA 298 "CMA", 299 #endif 300 #ifdef CONFIG_MEMORY_ISOLATION 301 "Isolate", 302 #endif 303 }; 304 305 compound_page_dtor * const compound_page_dtors[] = { 306 NULL, 307 free_compound_page, 308 #ifdef CONFIG_HUGETLB_PAGE 309 free_huge_page, 310 #endif 311 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 312 free_transhuge_page, 313 #endif 314 }; 315 316 int min_free_kbytes = 1024; 317 int user_min_free_kbytes = -1; 318 #ifdef CONFIG_DISCONTIGMEM 319 /* 320 * DiscontigMem defines memory ranges as separate pg_data_t even if the ranges 321 * are not on separate NUMA nodes. Functionally this works but with 322 * watermark_boost_factor, it can reclaim prematurely as the ranges can be 323 * quite small. By default, do not boost watermarks on discontigmem as in 324 * many cases very high-order allocations like THP are likely to be 325 * unsupported and the premature reclaim offsets the advantage of long-term 326 * fragmentation avoidance. 327 */ 328 int watermark_boost_factor __read_mostly; 329 #else 330 int watermark_boost_factor __read_mostly = 15000; 331 #endif 332 int watermark_scale_factor = 10; 333 334 static unsigned long nr_kernel_pages __initdata; 335 static unsigned long nr_all_pages __initdata; 336 static unsigned long dma_reserve __initdata; 337 338 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 339 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 340 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 341 static unsigned long required_kernelcore __initdata; 342 static unsigned long required_kernelcore_percent __initdata; 343 static unsigned long required_movablecore __initdata; 344 static unsigned long required_movablecore_percent __initdata; 345 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 346 static bool mirrored_kernelcore __meminitdata; 347 348 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 349 int movable_zone; 350 EXPORT_SYMBOL(movable_zone); 351 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 352 353 #if MAX_NUMNODES > 1 354 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; 355 unsigned int nr_online_nodes __read_mostly = 1; 356 EXPORT_SYMBOL(nr_node_ids); 357 EXPORT_SYMBOL(nr_online_nodes); 358 #endif 359 360 int page_group_by_mobility_disabled __read_mostly; 361 362 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 363 /* 364 * During boot we initialize deferred pages on-demand, as needed, but once 365 * page_alloc_init_late() has finished, the deferred pages are all initialized, 366 * and we can permanently disable that path. 367 */ 368 static DEFINE_STATIC_KEY_TRUE(deferred_pages); 369 370 /* 371 * Calling kasan_free_pages() only after deferred memory initialization 372 * has completed. Poisoning pages during deferred memory init will greatly 373 * lengthen the process and cause problem in large memory systems as the 374 * deferred pages initialization is done with interrupt disabled. 375 * 376 * Assuming that there will be no reference to those newly initialized 377 * pages before they are ever allocated, this should have no effect on 378 * KASAN memory tracking as the poison will be properly inserted at page 379 * allocation time. The only corner case is when pages are allocated by 380 * on-demand allocation and then freed again before the deferred pages 381 * initialization is done, but this is not likely to happen. 382 */ 383 static inline void kasan_free_nondeferred_pages(struct page *page, int order) 384 { 385 if (!static_branch_unlikely(&deferred_pages)) 386 kasan_free_pages(page, order); 387 } 388 389 /* Returns true if the struct page for the pfn is uninitialised */ 390 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 391 { 392 int nid = early_pfn_to_nid(pfn); 393 394 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 395 return true; 396 397 return false; 398 } 399 400 /* 401 * Returns true when the remaining initialisation should be deferred until 402 * later in the boot cycle when it can be parallelised. 403 */ 404 static bool __meminit 405 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 406 { 407 static unsigned long prev_end_pfn, nr_initialised; 408 409 /* 410 * prev_end_pfn static that contains the end of previous zone 411 * No need to protect because called very early in boot before smp_init. 412 */ 413 if (prev_end_pfn != end_pfn) { 414 prev_end_pfn = end_pfn; 415 nr_initialised = 0; 416 } 417 418 /* Always populate low zones for address-constrained allocations */ 419 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 420 return false; 421 422 /* 423 * We start only with one section of pages, more pages are added as 424 * needed until the rest of deferred pages are initialized. 425 */ 426 nr_initialised++; 427 if ((nr_initialised > PAGES_PER_SECTION) && 428 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 429 NODE_DATA(nid)->first_deferred_pfn = pfn; 430 return true; 431 } 432 return false; 433 } 434 #else 435 #define kasan_free_nondeferred_pages(p, o) kasan_free_pages(p, o) 436 437 static inline bool early_page_uninitialised(unsigned long pfn) 438 { 439 return false; 440 } 441 442 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 443 { 444 return false; 445 } 446 #endif 447 448 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 449 static inline unsigned long *get_pageblock_bitmap(struct page *page, 450 unsigned long pfn) 451 { 452 #ifdef CONFIG_SPARSEMEM 453 return section_to_usemap(__pfn_to_section(pfn)); 454 #else 455 return page_zone(page)->pageblock_flags; 456 #endif /* CONFIG_SPARSEMEM */ 457 } 458 459 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn) 460 { 461 #ifdef CONFIG_SPARSEMEM 462 pfn &= (PAGES_PER_SECTION-1); 463 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 464 #else 465 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 466 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 467 #endif /* CONFIG_SPARSEMEM */ 468 } 469 470 /** 471 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 472 * @page: The page within the block of interest 473 * @pfn: The target page frame number 474 * @end_bitidx: The last bit of interest to retrieve 475 * @mask: mask of bits that the caller is interested in 476 * 477 * Return: pageblock_bits flags 478 */ 479 static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page, 480 unsigned long pfn, 481 unsigned long end_bitidx, 482 unsigned long mask) 483 { 484 unsigned long *bitmap; 485 unsigned long bitidx, word_bitidx; 486 unsigned long word; 487 488 bitmap = get_pageblock_bitmap(page, pfn); 489 bitidx = pfn_to_bitidx(page, pfn); 490 word_bitidx = bitidx / BITS_PER_LONG; 491 bitidx &= (BITS_PER_LONG-1); 492 493 word = bitmap[word_bitidx]; 494 bitidx += end_bitidx; 495 return (word >> (BITS_PER_LONG - bitidx - 1)) & mask; 496 } 497 498 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, 499 unsigned long end_bitidx, 500 unsigned long mask) 501 { 502 return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask); 503 } 504 505 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn) 506 { 507 return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK); 508 } 509 510 /** 511 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 512 * @page: The page within the block of interest 513 * @flags: The flags to set 514 * @pfn: The target page frame number 515 * @end_bitidx: The last bit of interest 516 * @mask: mask of bits that the caller is interested in 517 */ 518 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 519 unsigned long pfn, 520 unsigned long end_bitidx, 521 unsigned long mask) 522 { 523 unsigned long *bitmap; 524 unsigned long bitidx, word_bitidx; 525 unsigned long old_word, word; 526 527 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 528 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits)); 529 530 bitmap = get_pageblock_bitmap(page, pfn); 531 bitidx = pfn_to_bitidx(page, pfn); 532 word_bitidx = bitidx / BITS_PER_LONG; 533 bitidx &= (BITS_PER_LONG-1); 534 535 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 536 537 bitidx += end_bitidx; 538 mask <<= (BITS_PER_LONG - bitidx - 1); 539 flags <<= (BITS_PER_LONG - bitidx - 1); 540 541 word = READ_ONCE(bitmap[word_bitidx]); 542 for (;;) { 543 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 544 if (word == old_word) 545 break; 546 word = old_word; 547 } 548 } 549 550 void set_pageblock_migratetype(struct page *page, int migratetype) 551 { 552 if (unlikely(page_group_by_mobility_disabled && 553 migratetype < MIGRATE_PCPTYPES)) 554 migratetype = MIGRATE_UNMOVABLE; 555 556 set_pageblock_flags_group(page, (unsigned long)migratetype, 557 PB_migrate, PB_migrate_end); 558 } 559 560 #ifdef CONFIG_DEBUG_VM 561 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 562 { 563 int ret = 0; 564 unsigned seq; 565 unsigned long pfn = page_to_pfn(page); 566 unsigned long sp, start_pfn; 567 568 do { 569 seq = zone_span_seqbegin(zone); 570 start_pfn = zone->zone_start_pfn; 571 sp = zone->spanned_pages; 572 if (!zone_spans_pfn(zone, pfn)) 573 ret = 1; 574 } while (zone_span_seqretry(zone, seq)); 575 576 if (ret) 577 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 578 pfn, zone_to_nid(zone), zone->name, 579 start_pfn, start_pfn + sp); 580 581 return ret; 582 } 583 584 static int page_is_consistent(struct zone *zone, struct page *page) 585 { 586 if (!pfn_valid_within(page_to_pfn(page))) 587 return 0; 588 if (zone != page_zone(page)) 589 return 0; 590 591 return 1; 592 } 593 /* 594 * Temporary debugging check for pages not lying within a given zone. 595 */ 596 static int __maybe_unused bad_range(struct zone *zone, struct page *page) 597 { 598 if (page_outside_zone_boundaries(zone, page)) 599 return 1; 600 if (!page_is_consistent(zone, page)) 601 return 1; 602 603 return 0; 604 } 605 #else 606 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page) 607 { 608 return 0; 609 } 610 #endif 611 612 static void bad_page(struct page *page, const char *reason, 613 unsigned long bad_flags) 614 { 615 static unsigned long resume; 616 static unsigned long nr_shown; 617 static unsigned long nr_unshown; 618 619 /* 620 * Allow a burst of 60 reports, then keep quiet for that minute; 621 * or allow a steady drip of one report per second. 622 */ 623 if (nr_shown == 60) { 624 if (time_before(jiffies, resume)) { 625 nr_unshown++; 626 goto out; 627 } 628 if (nr_unshown) { 629 pr_alert( 630 "BUG: Bad page state: %lu messages suppressed\n", 631 nr_unshown); 632 nr_unshown = 0; 633 } 634 nr_shown = 0; 635 } 636 if (nr_shown++ == 0) 637 resume = jiffies + 60 * HZ; 638 639 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 640 current->comm, page_to_pfn(page)); 641 __dump_page(page, reason); 642 bad_flags &= page->flags; 643 if (bad_flags) 644 pr_alert("bad because of flags: %#lx(%pGp)\n", 645 bad_flags, &bad_flags); 646 dump_page_owner(page); 647 648 print_modules(); 649 dump_stack(); 650 out: 651 /* Leave bad fields for debug, except PageBuddy could make trouble */ 652 page_mapcount_reset(page); /* remove PageBuddy */ 653 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 654 } 655 656 /* 657 * Higher-order pages are called "compound pages". They are structured thusly: 658 * 659 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 660 * 661 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 662 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 663 * 664 * The first tail page's ->compound_dtor holds the offset in array of compound 665 * page destructors. See compound_page_dtors. 666 * 667 * The first tail page's ->compound_order holds the order of allocation. 668 * This usage means that zero-order pages may not be compound. 669 */ 670 671 void free_compound_page(struct page *page) 672 { 673 mem_cgroup_uncharge(page); 674 __free_pages_ok(page, compound_order(page)); 675 } 676 677 void prep_compound_page(struct page *page, unsigned int order) 678 { 679 int i; 680 int nr_pages = 1 << order; 681 682 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 683 set_compound_order(page, order); 684 __SetPageHead(page); 685 for (i = 1; i < nr_pages; i++) { 686 struct page *p = page + i; 687 set_page_count(p, 0); 688 p->mapping = TAIL_MAPPING; 689 set_compound_head(p, page); 690 } 691 atomic_set(compound_mapcount_ptr(page), -1); 692 } 693 694 #ifdef CONFIG_DEBUG_PAGEALLOC 695 unsigned int _debug_guardpage_minorder; 696 697 #ifdef CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT 698 DEFINE_STATIC_KEY_TRUE(_debug_pagealloc_enabled); 699 #else 700 DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 701 #endif 702 EXPORT_SYMBOL(_debug_pagealloc_enabled); 703 704 DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 705 706 static int __init early_debug_pagealloc(char *buf) 707 { 708 bool enable = false; 709 710 if (kstrtobool(buf, &enable)) 711 return -EINVAL; 712 713 if (enable) 714 static_branch_enable(&_debug_pagealloc_enabled); 715 716 return 0; 717 } 718 early_param("debug_pagealloc", early_debug_pagealloc); 719 720 static void init_debug_guardpage(void) 721 { 722 if (!debug_pagealloc_enabled()) 723 return; 724 725 if (!debug_guardpage_minorder()) 726 return; 727 728 static_branch_enable(&_debug_guardpage_enabled); 729 } 730 731 static int __init debug_guardpage_minorder_setup(char *buf) 732 { 733 unsigned long res; 734 735 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 736 pr_err("Bad debug_guardpage_minorder value\n"); 737 return 0; 738 } 739 _debug_guardpage_minorder = res; 740 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 741 return 0; 742 } 743 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); 744 745 static inline bool set_page_guard(struct zone *zone, struct page *page, 746 unsigned int order, int migratetype) 747 { 748 if (!debug_guardpage_enabled()) 749 return false; 750 751 if (order >= debug_guardpage_minorder()) 752 return false; 753 754 __SetPageGuard(page); 755 INIT_LIST_HEAD(&page->lru); 756 set_page_private(page, order); 757 /* Guard pages are not available for any usage */ 758 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 759 760 return true; 761 } 762 763 static inline void clear_page_guard(struct zone *zone, struct page *page, 764 unsigned int order, int migratetype) 765 { 766 if (!debug_guardpage_enabled()) 767 return; 768 769 __ClearPageGuard(page); 770 771 set_page_private(page, 0); 772 if (!is_migrate_isolate(migratetype)) 773 __mod_zone_freepage_state(zone, (1 << order), migratetype); 774 } 775 #else 776 static inline bool set_page_guard(struct zone *zone, struct page *page, 777 unsigned int order, int migratetype) { return false; } 778 static inline void clear_page_guard(struct zone *zone, struct page *page, 779 unsigned int order, int migratetype) {} 780 #endif 781 782 static inline void set_page_order(struct page *page, unsigned int order) 783 { 784 set_page_private(page, order); 785 __SetPageBuddy(page); 786 } 787 788 /* 789 * This function checks whether a page is free && is the buddy 790 * we can coalesce a page and its buddy if 791 * (a) the buddy is not in a hole (check before calling!) && 792 * (b) the buddy is in the buddy system && 793 * (c) a page and its buddy have the same order && 794 * (d) a page and its buddy are in the same zone. 795 * 796 * For recording whether a page is in the buddy system, we set PageBuddy. 797 * Setting, clearing, and testing PageBuddy is serialized by zone->lock. 798 * 799 * For recording page's order, we use page_private(page). 800 */ 801 static inline int page_is_buddy(struct page *page, struct page *buddy, 802 unsigned int order) 803 { 804 if (page_is_guard(buddy) && page_order(buddy) == order) { 805 if (page_zone_id(page) != page_zone_id(buddy)) 806 return 0; 807 808 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 809 810 return 1; 811 } 812 813 if (PageBuddy(buddy) && page_order(buddy) == order) { 814 /* 815 * zone check is done late to avoid uselessly 816 * calculating zone/node ids for pages that could 817 * never merge. 818 */ 819 if (page_zone_id(page) != page_zone_id(buddy)) 820 return 0; 821 822 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 823 824 return 1; 825 } 826 return 0; 827 } 828 829 #ifdef CONFIG_COMPACTION 830 static inline struct capture_control *task_capc(struct zone *zone) 831 { 832 struct capture_control *capc = current->capture_control; 833 834 return capc && 835 !(current->flags & PF_KTHREAD) && 836 !capc->page && 837 capc->cc->zone == zone && 838 capc->cc->direct_compaction ? capc : NULL; 839 } 840 841 static inline bool 842 compaction_capture(struct capture_control *capc, struct page *page, 843 int order, int migratetype) 844 { 845 if (!capc || order != capc->cc->order) 846 return false; 847 848 /* Do not accidentally pollute CMA or isolated regions*/ 849 if (is_migrate_cma(migratetype) || 850 is_migrate_isolate(migratetype)) 851 return false; 852 853 /* 854 * Do not let lower order allocations polluate a movable pageblock. 855 * This might let an unmovable request use a reclaimable pageblock 856 * and vice-versa but no more than normal fallback logic which can 857 * have trouble finding a high-order free page. 858 */ 859 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE) 860 return false; 861 862 capc->page = page; 863 return true; 864 } 865 866 #else 867 static inline struct capture_control *task_capc(struct zone *zone) 868 { 869 return NULL; 870 } 871 872 static inline bool 873 compaction_capture(struct capture_control *capc, struct page *page, 874 int order, int migratetype) 875 { 876 return false; 877 } 878 #endif /* CONFIG_COMPACTION */ 879 880 /* 881 * Freeing function for a buddy system allocator. 882 * 883 * The concept of a buddy system is to maintain direct-mapped table 884 * (containing bit values) for memory blocks of various "orders". 885 * The bottom level table contains the map for the smallest allocatable 886 * units of memory (here, pages), and each level above it describes 887 * pairs of units from the levels below, hence, "buddies". 888 * At a high level, all that happens here is marking the table entry 889 * at the bottom level available, and propagating the changes upward 890 * as necessary, plus some accounting needed to play nicely with other 891 * parts of the VM system. 892 * At each level, we keep a list of pages, which are heads of continuous 893 * free pages of length of (1 << order) and marked with PageBuddy. 894 * Page's order is recorded in page_private(page) field. 895 * So when we are allocating or freeing one, we can derive the state of the 896 * other. That is, if we allocate a small block, and both were 897 * free, the remainder of the region must be split into blocks. 898 * If a block is freed, and its buddy is also free, then this 899 * triggers coalescing into a block of larger size. 900 * 901 * -- nyc 902 */ 903 904 static inline void __free_one_page(struct page *page, 905 unsigned long pfn, 906 struct zone *zone, unsigned int order, 907 int migratetype) 908 { 909 unsigned long combined_pfn; 910 unsigned long uninitialized_var(buddy_pfn); 911 struct page *buddy; 912 unsigned int max_order; 913 struct capture_control *capc = task_capc(zone); 914 915 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1); 916 917 VM_BUG_ON(!zone_is_initialized(zone)); 918 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 919 920 VM_BUG_ON(migratetype == -1); 921 if (likely(!is_migrate_isolate(migratetype))) 922 __mod_zone_freepage_state(zone, 1 << order, migratetype); 923 924 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); 925 VM_BUG_ON_PAGE(bad_range(zone, page), page); 926 927 continue_merging: 928 while (order < max_order - 1) { 929 if (compaction_capture(capc, page, order, migratetype)) { 930 __mod_zone_freepage_state(zone, -(1 << order), 931 migratetype); 932 return; 933 } 934 buddy_pfn = __find_buddy_pfn(pfn, order); 935 buddy = page + (buddy_pfn - pfn); 936 937 if (!pfn_valid_within(buddy_pfn)) 938 goto done_merging; 939 if (!page_is_buddy(page, buddy, order)) 940 goto done_merging; 941 /* 942 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 943 * merge with it and move up one order. 944 */ 945 if (page_is_guard(buddy)) 946 clear_page_guard(zone, buddy, order, migratetype); 947 else 948 del_page_from_free_area(buddy, &zone->free_area[order]); 949 combined_pfn = buddy_pfn & pfn; 950 page = page + (combined_pfn - pfn); 951 pfn = combined_pfn; 952 order++; 953 } 954 if (max_order < MAX_ORDER) { 955 /* If we are here, it means order is >= pageblock_order. 956 * We want to prevent merge between freepages on isolate 957 * pageblock and normal pageblock. Without this, pageblock 958 * isolation could cause incorrect freepage or CMA accounting. 959 * 960 * We don't want to hit this code for the more frequent 961 * low-order merging. 962 */ 963 if (unlikely(has_isolate_pageblock(zone))) { 964 int buddy_mt; 965 966 buddy_pfn = __find_buddy_pfn(pfn, order); 967 buddy = page + (buddy_pfn - pfn); 968 buddy_mt = get_pageblock_migratetype(buddy); 969 970 if (migratetype != buddy_mt 971 && (is_migrate_isolate(migratetype) || 972 is_migrate_isolate(buddy_mt))) 973 goto done_merging; 974 } 975 max_order++; 976 goto continue_merging; 977 } 978 979 done_merging: 980 set_page_order(page, order); 981 982 /* 983 * If this is not the largest possible page, check if the buddy 984 * of the next-highest order is free. If it is, it's possible 985 * that pages are being freed that will coalesce soon. In case, 986 * that is happening, add the free page to the tail of the list 987 * so it's less likely to be used soon and more likely to be merged 988 * as a higher order page 989 */ 990 if ((order < MAX_ORDER-2) && pfn_valid_within(buddy_pfn) 991 && !is_shuffle_order(order)) { 992 struct page *higher_page, *higher_buddy; 993 combined_pfn = buddy_pfn & pfn; 994 higher_page = page + (combined_pfn - pfn); 995 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1); 996 higher_buddy = higher_page + (buddy_pfn - combined_pfn); 997 if (pfn_valid_within(buddy_pfn) && 998 page_is_buddy(higher_page, higher_buddy, order + 1)) { 999 add_to_free_area_tail(page, &zone->free_area[order], 1000 migratetype); 1001 return; 1002 } 1003 } 1004 1005 if (is_shuffle_order(order)) 1006 add_to_free_area_random(page, &zone->free_area[order], 1007 migratetype); 1008 else 1009 add_to_free_area(page, &zone->free_area[order], migratetype); 1010 1011 } 1012 1013 /* 1014 * A bad page could be due to a number of fields. Instead of multiple branches, 1015 * try and check multiple fields with one check. The caller must do a detailed 1016 * check if necessary. 1017 */ 1018 static inline bool page_expected_state(struct page *page, 1019 unsigned long check_flags) 1020 { 1021 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1022 return false; 1023 1024 if (unlikely((unsigned long)page->mapping | 1025 page_ref_count(page) | 1026 #ifdef CONFIG_MEMCG 1027 (unsigned long)page->mem_cgroup | 1028 #endif 1029 (page->flags & check_flags))) 1030 return false; 1031 1032 return true; 1033 } 1034 1035 static void free_pages_check_bad(struct page *page) 1036 { 1037 const char *bad_reason; 1038 unsigned long bad_flags; 1039 1040 bad_reason = NULL; 1041 bad_flags = 0; 1042 1043 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1044 bad_reason = "nonzero mapcount"; 1045 if (unlikely(page->mapping != NULL)) 1046 bad_reason = "non-NULL mapping"; 1047 if (unlikely(page_ref_count(page) != 0)) 1048 bad_reason = "nonzero _refcount"; 1049 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) { 1050 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 1051 bad_flags = PAGE_FLAGS_CHECK_AT_FREE; 1052 } 1053 #ifdef CONFIG_MEMCG 1054 if (unlikely(page->mem_cgroup)) 1055 bad_reason = "page still charged to cgroup"; 1056 #endif 1057 bad_page(page, bad_reason, bad_flags); 1058 } 1059 1060 static inline int free_pages_check(struct page *page) 1061 { 1062 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 1063 return 0; 1064 1065 /* Something has gone sideways, find it */ 1066 free_pages_check_bad(page); 1067 return 1; 1068 } 1069 1070 static int free_tail_pages_check(struct page *head_page, struct page *page) 1071 { 1072 int ret = 1; 1073 1074 /* 1075 * We rely page->lru.next never has bit 0 set, unless the page 1076 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 1077 */ 1078 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 1079 1080 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 1081 ret = 0; 1082 goto out; 1083 } 1084 switch (page - head_page) { 1085 case 1: 1086 /* the first tail page: ->mapping may be compound_mapcount() */ 1087 if (unlikely(compound_mapcount(page))) { 1088 bad_page(page, "nonzero compound_mapcount", 0); 1089 goto out; 1090 } 1091 break; 1092 case 2: 1093 /* 1094 * the second tail page: ->mapping is 1095 * deferred_list.next -- ignore value. 1096 */ 1097 break; 1098 default: 1099 if (page->mapping != TAIL_MAPPING) { 1100 bad_page(page, "corrupted mapping in tail page", 0); 1101 goto out; 1102 } 1103 break; 1104 } 1105 if (unlikely(!PageTail(page))) { 1106 bad_page(page, "PageTail not set", 0); 1107 goto out; 1108 } 1109 if (unlikely(compound_head(page) != head_page)) { 1110 bad_page(page, "compound_head not consistent", 0); 1111 goto out; 1112 } 1113 ret = 0; 1114 out: 1115 page->mapping = NULL; 1116 clear_compound_head(page); 1117 return ret; 1118 } 1119 1120 static void kernel_init_free_pages(struct page *page, int numpages) 1121 { 1122 int i; 1123 1124 for (i = 0; i < numpages; i++) 1125 clear_highpage(page + i); 1126 } 1127 1128 static __always_inline bool free_pages_prepare(struct page *page, 1129 unsigned int order, bool check_free) 1130 { 1131 int bad = 0; 1132 1133 VM_BUG_ON_PAGE(PageTail(page), page); 1134 1135 trace_mm_page_free(page, order); 1136 1137 /* 1138 * Check tail pages before head page information is cleared to 1139 * avoid checking PageCompound for order-0 pages. 1140 */ 1141 if (unlikely(order)) { 1142 bool compound = PageCompound(page); 1143 int i; 1144 1145 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1146 1147 if (compound) 1148 ClearPageDoubleMap(page); 1149 for (i = 1; i < (1 << order); i++) { 1150 if (compound) 1151 bad += free_tail_pages_check(page, page + i); 1152 if (unlikely(free_pages_check(page + i))) { 1153 bad++; 1154 continue; 1155 } 1156 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1157 } 1158 } 1159 if (PageMappingFlags(page)) 1160 page->mapping = NULL; 1161 if (memcg_kmem_enabled() && PageKmemcg(page)) 1162 __memcg_kmem_uncharge(page, order); 1163 if (check_free) 1164 bad += free_pages_check(page); 1165 if (bad) 1166 return false; 1167 1168 page_cpupid_reset_last(page); 1169 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1170 reset_page_owner(page, order); 1171 1172 if (!PageHighMem(page)) { 1173 debug_check_no_locks_freed(page_address(page), 1174 PAGE_SIZE << order); 1175 debug_check_no_obj_freed(page_address(page), 1176 PAGE_SIZE << order); 1177 } 1178 arch_free_page(page, order); 1179 if (want_init_on_free()) 1180 kernel_init_free_pages(page, 1 << order); 1181 1182 kernel_poison_pages(page, 1 << order, 0); 1183 if (debug_pagealloc_enabled()) 1184 kernel_map_pages(page, 1 << order, 0); 1185 1186 kasan_free_nondeferred_pages(page, order); 1187 1188 return true; 1189 } 1190 1191 #ifdef CONFIG_DEBUG_VM 1192 /* 1193 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed 1194 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when 1195 * moved from pcp lists to free lists. 1196 */ 1197 static bool free_pcp_prepare(struct page *page) 1198 { 1199 return free_pages_prepare(page, 0, true); 1200 } 1201 1202 static bool bulkfree_pcp_prepare(struct page *page) 1203 { 1204 if (debug_pagealloc_enabled()) 1205 return free_pages_check(page); 1206 else 1207 return false; 1208 } 1209 #else 1210 /* 1211 * With DEBUG_VM disabled, order-0 pages being freed are checked only when 1212 * moving from pcp lists to free list in order to reduce overhead. With 1213 * debug_pagealloc enabled, they are checked also immediately when being freed 1214 * to the pcp lists. 1215 */ 1216 static bool free_pcp_prepare(struct page *page) 1217 { 1218 if (debug_pagealloc_enabled()) 1219 return free_pages_prepare(page, 0, true); 1220 else 1221 return free_pages_prepare(page, 0, false); 1222 } 1223 1224 static bool bulkfree_pcp_prepare(struct page *page) 1225 { 1226 return free_pages_check(page); 1227 } 1228 #endif /* CONFIG_DEBUG_VM */ 1229 1230 static inline void prefetch_buddy(struct page *page) 1231 { 1232 unsigned long pfn = page_to_pfn(page); 1233 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0); 1234 struct page *buddy = page + (buddy_pfn - pfn); 1235 1236 prefetch(buddy); 1237 } 1238 1239 /* 1240 * Frees a number of pages from the PCP lists 1241 * Assumes all pages on list are in same zone, and of same order. 1242 * count is the number of pages to free. 1243 * 1244 * If the zone was previously in an "all pages pinned" state then look to 1245 * see if this freeing clears that state. 1246 * 1247 * And clear the zone's pages_scanned counter, to hold off the "all pages are 1248 * pinned" detection logic. 1249 */ 1250 static void free_pcppages_bulk(struct zone *zone, int count, 1251 struct per_cpu_pages *pcp) 1252 { 1253 int migratetype = 0; 1254 int batch_free = 0; 1255 int prefetch_nr = 0; 1256 bool isolated_pageblocks; 1257 struct page *page, *tmp; 1258 LIST_HEAD(head); 1259 1260 while (count) { 1261 struct list_head *list; 1262 1263 /* 1264 * Remove pages from lists in a round-robin fashion. A 1265 * batch_free count is maintained that is incremented when an 1266 * empty list is encountered. This is so more pages are freed 1267 * off fuller lists instead of spinning excessively around empty 1268 * lists 1269 */ 1270 do { 1271 batch_free++; 1272 if (++migratetype == MIGRATE_PCPTYPES) 1273 migratetype = 0; 1274 list = &pcp->lists[migratetype]; 1275 } while (list_empty(list)); 1276 1277 /* This is the only non-empty list. Free them all. */ 1278 if (batch_free == MIGRATE_PCPTYPES) 1279 batch_free = count; 1280 1281 do { 1282 page = list_last_entry(list, struct page, lru); 1283 /* must delete to avoid corrupting pcp list */ 1284 list_del(&page->lru); 1285 pcp->count--; 1286 1287 if (bulkfree_pcp_prepare(page)) 1288 continue; 1289 1290 list_add_tail(&page->lru, &head); 1291 1292 /* 1293 * We are going to put the page back to the global 1294 * pool, prefetch its buddy to speed up later access 1295 * under zone->lock. It is believed the overhead of 1296 * an additional test and calculating buddy_pfn here 1297 * can be offset by reduced memory latency later. To 1298 * avoid excessive prefetching due to large count, only 1299 * prefetch buddy for the first pcp->batch nr of pages. 1300 */ 1301 if (prefetch_nr++ < pcp->batch) 1302 prefetch_buddy(page); 1303 } while (--count && --batch_free && !list_empty(list)); 1304 } 1305 1306 spin_lock(&zone->lock); 1307 isolated_pageblocks = has_isolate_pageblock(zone); 1308 1309 /* 1310 * Use safe version since after __free_one_page(), 1311 * page->lru.next will not point to original list. 1312 */ 1313 list_for_each_entry_safe(page, tmp, &head, lru) { 1314 int mt = get_pcppage_migratetype(page); 1315 /* MIGRATE_ISOLATE page should not go to pcplists */ 1316 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1317 /* Pageblock could have been isolated meanwhile */ 1318 if (unlikely(isolated_pageblocks)) 1319 mt = get_pageblock_migratetype(page); 1320 1321 __free_one_page(page, page_to_pfn(page), zone, 0, mt); 1322 trace_mm_page_pcpu_drain(page, 0, mt); 1323 } 1324 spin_unlock(&zone->lock); 1325 } 1326 1327 static void free_one_page(struct zone *zone, 1328 struct page *page, unsigned long pfn, 1329 unsigned int order, 1330 int migratetype) 1331 { 1332 spin_lock(&zone->lock); 1333 if (unlikely(has_isolate_pageblock(zone) || 1334 is_migrate_isolate(migratetype))) { 1335 migratetype = get_pfnblock_migratetype(page, pfn); 1336 } 1337 __free_one_page(page, pfn, zone, order, migratetype); 1338 spin_unlock(&zone->lock); 1339 } 1340 1341 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1342 unsigned long zone, int nid) 1343 { 1344 mm_zero_struct_page(page); 1345 set_page_links(page, zone, nid, pfn); 1346 init_page_count(page); 1347 page_mapcount_reset(page); 1348 page_cpupid_reset_last(page); 1349 page_kasan_tag_reset(page); 1350 1351 INIT_LIST_HEAD(&page->lru); 1352 #ifdef WANT_PAGE_VIRTUAL 1353 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1354 if (!is_highmem_idx(zone)) 1355 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1356 #endif 1357 } 1358 1359 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1360 static void __meminit init_reserved_page(unsigned long pfn) 1361 { 1362 pg_data_t *pgdat; 1363 int nid, zid; 1364 1365 if (!early_page_uninitialised(pfn)) 1366 return; 1367 1368 nid = early_pfn_to_nid(pfn); 1369 pgdat = NODE_DATA(nid); 1370 1371 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1372 struct zone *zone = &pgdat->node_zones[zid]; 1373 1374 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) 1375 break; 1376 } 1377 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 1378 } 1379 #else 1380 static inline void init_reserved_page(unsigned long pfn) 1381 { 1382 } 1383 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1384 1385 /* 1386 * Initialised pages do not have PageReserved set. This function is 1387 * called for each range allocated by the bootmem allocator and 1388 * marks the pages PageReserved. The remaining valid pages are later 1389 * sent to the buddy page allocator. 1390 */ 1391 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1392 { 1393 unsigned long start_pfn = PFN_DOWN(start); 1394 unsigned long end_pfn = PFN_UP(end); 1395 1396 for (; start_pfn < end_pfn; start_pfn++) { 1397 if (pfn_valid(start_pfn)) { 1398 struct page *page = pfn_to_page(start_pfn); 1399 1400 init_reserved_page(start_pfn); 1401 1402 /* Avoid false-positive PageTail() */ 1403 INIT_LIST_HEAD(&page->lru); 1404 1405 /* 1406 * no need for atomic set_bit because the struct 1407 * page is not visible yet so nobody should 1408 * access it yet. 1409 */ 1410 __SetPageReserved(page); 1411 } 1412 } 1413 } 1414 1415 static void __free_pages_ok(struct page *page, unsigned int order) 1416 { 1417 unsigned long flags; 1418 int migratetype; 1419 unsigned long pfn = page_to_pfn(page); 1420 1421 if (!free_pages_prepare(page, order, true)) 1422 return; 1423 1424 migratetype = get_pfnblock_migratetype(page, pfn); 1425 local_irq_save(flags); 1426 __count_vm_events(PGFREE, 1 << order); 1427 free_one_page(page_zone(page), page, pfn, order, migratetype); 1428 local_irq_restore(flags); 1429 } 1430 1431 void __free_pages_core(struct page *page, unsigned int order) 1432 { 1433 unsigned int nr_pages = 1 << order; 1434 struct page *p = page; 1435 unsigned int loop; 1436 1437 prefetchw(p); 1438 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1439 prefetchw(p + 1); 1440 __ClearPageReserved(p); 1441 set_page_count(p, 0); 1442 } 1443 __ClearPageReserved(p); 1444 set_page_count(p, 0); 1445 1446 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); 1447 set_page_refcounted(page); 1448 __free_pages(page, order); 1449 } 1450 1451 #if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \ 1452 defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) 1453 1454 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1455 1456 int __meminit early_pfn_to_nid(unsigned long pfn) 1457 { 1458 static DEFINE_SPINLOCK(early_pfn_lock); 1459 int nid; 1460 1461 spin_lock(&early_pfn_lock); 1462 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1463 if (nid < 0) 1464 nid = first_online_node; 1465 spin_unlock(&early_pfn_lock); 1466 1467 return nid; 1468 } 1469 #endif 1470 1471 #ifdef CONFIG_NODES_SPAN_OTHER_NODES 1472 /* Only safe to use early in boot when initialisation is single-threaded */ 1473 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1474 { 1475 int nid; 1476 1477 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1478 if (nid >= 0 && nid != node) 1479 return false; 1480 return true; 1481 } 1482 1483 #else 1484 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1485 { 1486 return true; 1487 } 1488 #endif 1489 1490 1491 void __init memblock_free_pages(struct page *page, unsigned long pfn, 1492 unsigned int order) 1493 { 1494 if (early_page_uninitialised(pfn)) 1495 return; 1496 __free_pages_core(page, order); 1497 } 1498 1499 /* 1500 * Check that the whole (or subset of) a pageblock given by the interval of 1501 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1502 * with the migration of free compaction scanner. The scanners then need to 1503 * use only pfn_valid_within() check for arches that allow holes within 1504 * pageblocks. 1505 * 1506 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1507 * 1508 * It's possible on some configurations to have a setup like node0 node1 node0 1509 * i.e. it's possible that all pages within a zones range of pages do not 1510 * belong to a single zone. We assume that a border between node0 and node1 1511 * can occur within a single pageblock, but not a node0 node1 node0 1512 * interleaving within a single pageblock. It is therefore sufficient to check 1513 * the first and last page of a pageblock and avoid checking each individual 1514 * page in a pageblock. 1515 */ 1516 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1517 unsigned long end_pfn, struct zone *zone) 1518 { 1519 struct page *start_page; 1520 struct page *end_page; 1521 1522 /* end_pfn is one past the range we are checking */ 1523 end_pfn--; 1524 1525 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1526 return NULL; 1527 1528 start_page = pfn_to_online_page(start_pfn); 1529 if (!start_page) 1530 return NULL; 1531 1532 if (page_zone(start_page) != zone) 1533 return NULL; 1534 1535 end_page = pfn_to_page(end_pfn); 1536 1537 /* This gives a shorter code than deriving page_zone(end_page) */ 1538 if (page_zone_id(start_page) != page_zone_id(end_page)) 1539 return NULL; 1540 1541 return start_page; 1542 } 1543 1544 void set_zone_contiguous(struct zone *zone) 1545 { 1546 unsigned long block_start_pfn = zone->zone_start_pfn; 1547 unsigned long block_end_pfn; 1548 1549 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1550 for (; block_start_pfn < zone_end_pfn(zone); 1551 block_start_pfn = block_end_pfn, 1552 block_end_pfn += pageblock_nr_pages) { 1553 1554 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1555 1556 if (!__pageblock_pfn_to_page(block_start_pfn, 1557 block_end_pfn, zone)) 1558 return; 1559 } 1560 1561 /* We confirm that there is no hole */ 1562 zone->contiguous = true; 1563 } 1564 1565 void clear_zone_contiguous(struct zone *zone) 1566 { 1567 zone->contiguous = false; 1568 } 1569 1570 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1571 static void __init deferred_free_range(unsigned long pfn, 1572 unsigned long nr_pages) 1573 { 1574 struct page *page; 1575 unsigned long i; 1576 1577 if (!nr_pages) 1578 return; 1579 1580 page = pfn_to_page(pfn); 1581 1582 /* Free a large naturally-aligned chunk if possible */ 1583 if (nr_pages == pageblock_nr_pages && 1584 (pfn & (pageblock_nr_pages - 1)) == 0) { 1585 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1586 __free_pages_core(page, pageblock_order); 1587 return; 1588 } 1589 1590 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1591 if ((pfn & (pageblock_nr_pages - 1)) == 0) 1592 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1593 __free_pages_core(page, 0); 1594 } 1595 } 1596 1597 /* Completion tracking for deferred_init_memmap() threads */ 1598 static atomic_t pgdat_init_n_undone __initdata; 1599 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1600 1601 static inline void __init pgdat_init_report_one_done(void) 1602 { 1603 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1604 complete(&pgdat_init_all_done_comp); 1605 } 1606 1607 /* 1608 * Returns true if page needs to be initialized or freed to buddy allocator. 1609 * 1610 * First we check if pfn is valid on architectures where it is possible to have 1611 * holes within pageblock_nr_pages. On systems where it is not possible, this 1612 * function is optimized out. 1613 * 1614 * Then, we check if a current large page is valid by only checking the validity 1615 * of the head pfn. 1616 */ 1617 static inline bool __init deferred_pfn_valid(unsigned long pfn) 1618 { 1619 if (!pfn_valid_within(pfn)) 1620 return false; 1621 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn)) 1622 return false; 1623 return true; 1624 } 1625 1626 /* 1627 * Free pages to buddy allocator. Try to free aligned pages in 1628 * pageblock_nr_pages sizes. 1629 */ 1630 static void __init deferred_free_pages(unsigned long pfn, 1631 unsigned long end_pfn) 1632 { 1633 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1634 unsigned long nr_free = 0; 1635 1636 for (; pfn < end_pfn; pfn++) { 1637 if (!deferred_pfn_valid(pfn)) { 1638 deferred_free_range(pfn - nr_free, nr_free); 1639 nr_free = 0; 1640 } else if (!(pfn & nr_pgmask)) { 1641 deferred_free_range(pfn - nr_free, nr_free); 1642 nr_free = 1; 1643 touch_nmi_watchdog(); 1644 } else { 1645 nr_free++; 1646 } 1647 } 1648 /* Free the last block of pages to allocator */ 1649 deferred_free_range(pfn - nr_free, nr_free); 1650 } 1651 1652 /* 1653 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 1654 * by performing it only once every pageblock_nr_pages. 1655 * Return number of pages initialized. 1656 */ 1657 static unsigned long __init deferred_init_pages(struct zone *zone, 1658 unsigned long pfn, 1659 unsigned long end_pfn) 1660 { 1661 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1662 int nid = zone_to_nid(zone); 1663 unsigned long nr_pages = 0; 1664 int zid = zone_idx(zone); 1665 struct page *page = NULL; 1666 1667 for (; pfn < end_pfn; pfn++) { 1668 if (!deferred_pfn_valid(pfn)) { 1669 page = NULL; 1670 continue; 1671 } else if (!page || !(pfn & nr_pgmask)) { 1672 page = pfn_to_page(pfn); 1673 touch_nmi_watchdog(); 1674 } else { 1675 page++; 1676 } 1677 __init_single_page(page, pfn, zid, nid); 1678 nr_pages++; 1679 } 1680 return (nr_pages); 1681 } 1682 1683 /* 1684 * This function is meant to pre-load the iterator for the zone init. 1685 * Specifically it walks through the ranges until we are caught up to the 1686 * first_init_pfn value and exits there. If we never encounter the value we 1687 * return false indicating there are no valid ranges left. 1688 */ 1689 static bool __init 1690 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, 1691 unsigned long *spfn, unsigned long *epfn, 1692 unsigned long first_init_pfn) 1693 { 1694 u64 j; 1695 1696 /* 1697 * Start out by walking through the ranges in this zone that have 1698 * already been initialized. We don't need to do anything with them 1699 * so we just need to flush them out of the system. 1700 */ 1701 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { 1702 if (*epfn <= first_init_pfn) 1703 continue; 1704 if (*spfn < first_init_pfn) 1705 *spfn = first_init_pfn; 1706 *i = j; 1707 return true; 1708 } 1709 1710 return false; 1711 } 1712 1713 /* 1714 * Initialize and free pages. We do it in two loops: first we initialize 1715 * struct page, then free to buddy allocator, because while we are 1716 * freeing pages we can access pages that are ahead (computing buddy 1717 * page in __free_one_page()). 1718 * 1719 * In order to try and keep some memory in the cache we have the loop 1720 * broken along max page order boundaries. This way we will not cause 1721 * any issues with the buddy page computation. 1722 */ 1723 static unsigned long __init 1724 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, 1725 unsigned long *end_pfn) 1726 { 1727 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); 1728 unsigned long spfn = *start_pfn, epfn = *end_pfn; 1729 unsigned long nr_pages = 0; 1730 u64 j = *i; 1731 1732 /* First we loop through and initialize the page values */ 1733 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { 1734 unsigned long t; 1735 1736 if (mo_pfn <= *start_pfn) 1737 break; 1738 1739 t = min(mo_pfn, *end_pfn); 1740 nr_pages += deferred_init_pages(zone, *start_pfn, t); 1741 1742 if (mo_pfn < *end_pfn) { 1743 *start_pfn = mo_pfn; 1744 break; 1745 } 1746 } 1747 1748 /* Reset values and now loop through freeing pages as needed */ 1749 swap(j, *i); 1750 1751 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { 1752 unsigned long t; 1753 1754 if (mo_pfn <= spfn) 1755 break; 1756 1757 t = min(mo_pfn, epfn); 1758 deferred_free_pages(spfn, t); 1759 1760 if (mo_pfn <= epfn) 1761 break; 1762 } 1763 1764 return nr_pages; 1765 } 1766 1767 /* Initialise remaining memory on a node */ 1768 static int __init deferred_init_memmap(void *data) 1769 { 1770 pg_data_t *pgdat = data; 1771 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 1772 unsigned long spfn = 0, epfn = 0, nr_pages = 0; 1773 unsigned long first_init_pfn, flags; 1774 unsigned long start = jiffies; 1775 struct zone *zone; 1776 int zid; 1777 u64 i; 1778 1779 /* Bind memory initialisation thread to a local node if possible */ 1780 if (!cpumask_empty(cpumask)) 1781 set_cpus_allowed_ptr(current, cpumask); 1782 1783 pgdat_resize_lock(pgdat, &flags); 1784 first_init_pfn = pgdat->first_deferred_pfn; 1785 if (first_init_pfn == ULONG_MAX) { 1786 pgdat_resize_unlock(pgdat, &flags); 1787 pgdat_init_report_one_done(); 1788 return 0; 1789 } 1790 1791 /* Sanity check boundaries */ 1792 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 1793 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 1794 pgdat->first_deferred_pfn = ULONG_MAX; 1795 1796 /* Only the highest zone is deferred so find it */ 1797 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1798 zone = pgdat->node_zones + zid; 1799 if (first_init_pfn < zone_end_pfn(zone)) 1800 break; 1801 } 1802 1803 /* If the zone is empty somebody else may have cleared out the zone */ 1804 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 1805 first_init_pfn)) 1806 goto zone_empty; 1807 1808 /* 1809 * Initialize and free pages in MAX_ORDER sized increments so 1810 * that we can avoid introducing any issues with the buddy 1811 * allocator. 1812 */ 1813 while (spfn < epfn) 1814 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 1815 zone_empty: 1816 pgdat_resize_unlock(pgdat, &flags); 1817 1818 /* Sanity check that the next zone really is unpopulated */ 1819 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 1820 1821 pr_info("node %d initialised, %lu pages in %ums\n", 1822 pgdat->node_id, nr_pages, jiffies_to_msecs(jiffies - start)); 1823 1824 pgdat_init_report_one_done(); 1825 return 0; 1826 } 1827 1828 /* 1829 * If this zone has deferred pages, try to grow it by initializing enough 1830 * deferred pages to satisfy the allocation specified by order, rounded up to 1831 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 1832 * of SECTION_SIZE bytes by initializing struct pages in increments of 1833 * PAGES_PER_SECTION * sizeof(struct page) bytes. 1834 * 1835 * Return true when zone was grown, otherwise return false. We return true even 1836 * when we grow less than requested, to let the caller decide if there are 1837 * enough pages to satisfy the allocation. 1838 * 1839 * Note: We use noinline because this function is needed only during boot, and 1840 * it is called from a __ref function _deferred_grow_zone. This way we are 1841 * making sure that it is not inlined into permanent text section. 1842 */ 1843 static noinline bool __init 1844 deferred_grow_zone(struct zone *zone, unsigned int order) 1845 { 1846 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 1847 pg_data_t *pgdat = zone->zone_pgdat; 1848 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 1849 unsigned long spfn, epfn, flags; 1850 unsigned long nr_pages = 0; 1851 u64 i; 1852 1853 /* Only the last zone may have deferred pages */ 1854 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 1855 return false; 1856 1857 pgdat_resize_lock(pgdat, &flags); 1858 1859 /* 1860 * If deferred pages have been initialized while we were waiting for 1861 * the lock, return true, as the zone was grown. The caller will retry 1862 * this zone. We won't return to this function since the caller also 1863 * has this static branch. 1864 */ 1865 if (!static_branch_unlikely(&deferred_pages)) { 1866 pgdat_resize_unlock(pgdat, &flags); 1867 return true; 1868 } 1869 1870 /* 1871 * If someone grew this zone while we were waiting for spinlock, return 1872 * true, as there might be enough pages already. 1873 */ 1874 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 1875 pgdat_resize_unlock(pgdat, &flags); 1876 return true; 1877 } 1878 1879 /* If the zone is empty somebody else may have cleared out the zone */ 1880 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 1881 first_deferred_pfn)) { 1882 pgdat->first_deferred_pfn = ULONG_MAX; 1883 pgdat_resize_unlock(pgdat, &flags); 1884 /* Retry only once. */ 1885 return first_deferred_pfn != ULONG_MAX; 1886 } 1887 1888 /* 1889 * Initialize and free pages in MAX_ORDER sized increments so 1890 * that we can avoid introducing any issues with the buddy 1891 * allocator. 1892 */ 1893 while (spfn < epfn) { 1894 /* update our first deferred PFN for this section */ 1895 first_deferred_pfn = spfn; 1896 1897 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 1898 1899 /* We should only stop along section boundaries */ 1900 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) 1901 continue; 1902 1903 /* If our quota has been met we can stop here */ 1904 if (nr_pages >= nr_pages_needed) 1905 break; 1906 } 1907 1908 pgdat->first_deferred_pfn = spfn; 1909 pgdat_resize_unlock(pgdat, &flags); 1910 1911 return nr_pages > 0; 1912 } 1913 1914 /* 1915 * deferred_grow_zone() is __init, but it is called from 1916 * get_page_from_freelist() during early boot until deferred_pages permanently 1917 * disables this call. This is why we have refdata wrapper to avoid warning, 1918 * and to ensure that the function body gets unloaded. 1919 */ 1920 static bool __ref 1921 _deferred_grow_zone(struct zone *zone, unsigned int order) 1922 { 1923 return deferred_grow_zone(zone, order); 1924 } 1925 1926 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1927 1928 void __init page_alloc_init_late(void) 1929 { 1930 struct zone *zone; 1931 int nid; 1932 1933 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1934 1935 /* There will be num_node_state(N_MEMORY) threads */ 1936 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 1937 for_each_node_state(nid, N_MEMORY) { 1938 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 1939 } 1940 1941 /* Block until all are initialised */ 1942 wait_for_completion(&pgdat_init_all_done_comp); 1943 1944 /* 1945 * We initialized the rest of the deferred pages. Permanently disable 1946 * on-demand struct page initialization. 1947 */ 1948 static_branch_disable(&deferred_pages); 1949 1950 /* Reinit limits that are based on free pages after the kernel is up */ 1951 files_maxfiles_init(); 1952 #endif 1953 1954 /* Discard memblock private memory */ 1955 memblock_discard(); 1956 1957 for_each_node_state(nid, N_MEMORY) 1958 shuffle_free_memory(NODE_DATA(nid)); 1959 1960 for_each_populated_zone(zone) 1961 set_zone_contiguous(zone); 1962 1963 #ifdef CONFIG_DEBUG_PAGEALLOC 1964 init_debug_guardpage(); 1965 #endif 1966 } 1967 1968 #ifdef CONFIG_CMA 1969 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 1970 void __init init_cma_reserved_pageblock(struct page *page) 1971 { 1972 unsigned i = pageblock_nr_pages; 1973 struct page *p = page; 1974 1975 do { 1976 __ClearPageReserved(p); 1977 set_page_count(p, 0); 1978 } while (++p, --i); 1979 1980 set_pageblock_migratetype(page, MIGRATE_CMA); 1981 1982 if (pageblock_order >= MAX_ORDER) { 1983 i = pageblock_nr_pages; 1984 p = page; 1985 do { 1986 set_page_refcounted(p); 1987 __free_pages(p, MAX_ORDER - 1); 1988 p += MAX_ORDER_NR_PAGES; 1989 } while (i -= MAX_ORDER_NR_PAGES); 1990 } else { 1991 set_page_refcounted(page); 1992 __free_pages(page, pageblock_order); 1993 } 1994 1995 adjust_managed_page_count(page, pageblock_nr_pages); 1996 } 1997 #endif 1998 1999 /* 2000 * The order of subdivision here is critical for the IO subsystem. 2001 * Please do not alter this order without good reasons and regression 2002 * testing. Specifically, as large blocks of memory are subdivided, 2003 * the order in which smaller blocks are delivered depends on the order 2004 * they're subdivided in this function. This is the primary factor 2005 * influencing the order in which pages are delivered to the IO 2006 * subsystem according to empirical testing, and this is also justified 2007 * by considering the behavior of a buddy system containing a single 2008 * large block of memory acted on by a series of small allocations. 2009 * This behavior is a critical factor in sglist merging's success. 2010 * 2011 * -- nyc 2012 */ 2013 static inline void expand(struct zone *zone, struct page *page, 2014 int low, int high, struct free_area *area, 2015 int migratetype) 2016 { 2017 unsigned long size = 1 << high; 2018 2019 while (high > low) { 2020 area--; 2021 high--; 2022 size >>= 1; 2023 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 2024 2025 /* 2026 * Mark as guard pages (or page), that will allow to 2027 * merge back to allocator when buddy will be freed. 2028 * Corresponding page table entries will not be touched, 2029 * pages will stay not present in virtual address space 2030 */ 2031 if (set_page_guard(zone, &page[size], high, migratetype)) 2032 continue; 2033 2034 add_to_free_area(&page[size], area, migratetype); 2035 set_page_order(&page[size], high); 2036 } 2037 } 2038 2039 static void check_new_page_bad(struct page *page) 2040 { 2041 const char *bad_reason = NULL; 2042 unsigned long bad_flags = 0; 2043 2044 if (unlikely(atomic_read(&page->_mapcount) != -1)) 2045 bad_reason = "nonzero mapcount"; 2046 if (unlikely(page->mapping != NULL)) 2047 bad_reason = "non-NULL mapping"; 2048 if (unlikely(page_ref_count(page) != 0)) 2049 bad_reason = "nonzero _refcount"; 2050 if (unlikely(page->flags & __PG_HWPOISON)) { 2051 bad_reason = "HWPoisoned (hardware-corrupted)"; 2052 bad_flags = __PG_HWPOISON; 2053 /* Don't complain about hwpoisoned pages */ 2054 page_mapcount_reset(page); /* remove PageBuddy */ 2055 return; 2056 } 2057 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) { 2058 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set"; 2059 bad_flags = PAGE_FLAGS_CHECK_AT_PREP; 2060 } 2061 #ifdef CONFIG_MEMCG 2062 if (unlikely(page->mem_cgroup)) 2063 bad_reason = "page still charged to cgroup"; 2064 #endif 2065 bad_page(page, bad_reason, bad_flags); 2066 } 2067 2068 /* 2069 * This page is about to be returned from the page allocator 2070 */ 2071 static inline int check_new_page(struct page *page) 2072 { 2073 if (likely(page_expected_state(page, 2074 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 2075 return 0; 2076 2077 check_new_page_bad(page); 2078 return 1; 2079 } 2080 2081 static inline bool free_pages_prezeroed(void) 2082 { 2083 return (IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) && 2084 page_poisoning_enabled()) || want_init_on_free(); 2085 } 2086 2087 #ifdef CONFIG_DEBUG_VM 2088 /* 2089 * With DEBUG_VM enabled, order-0 pages are checked for expected state when 2090 * being allocated from pcp lists. With debug_pagealloc also enabled, they are 2091 * also checked when pcp lists are refilled from the free lists. 2092 */ 2093 static inline bool check_pcp_refill(struct page *page) 2094 { 2095 if (debug_pagealloc_enabled()) 2096 return check_new_page(page); 2097 else 2098 return false; 2099 } 2100 2101 static inline bool check_new_pcp(struct page *page) 2102 { 2103 return check_new_page(page); 2104 } 2105 #else 2106 /* 2107 * With DEBUG_VM disabled, free order-0 pages are checked for expected state 2108 * when pcp lists are being refilled from the free lists. With debug_pagealloc 2109 * enabled, they are also checked when being allocated from the pcp lists. 2110 */ 2111 static inline bool check_pcp_refill(struct page *page) 2112 { 2113 return check_new_page(page); 2114 } 2115 static inline bool check_new_pcp(struct page *page) 2116 { 2117 if (debug_pagealloc_enabled()) 2118 return check_new_page(page); 2119 else 2120 return false; 2121 } 2122 #endif /* CONFIG_DEBUG_VM */ 2123 2124 static bool check_new_pages(struct page *page, unsigned int order) 2125 { 2126 int i; 2127 for (i = 0; i < (1 << order); i++) { 2128 struct page *p = page + i; 2129 2130 if (unlikely(check_new_page(p))) 2131 return true; 2132 } 2133 2134 return false; 2135 } 2136 2137 inline void post_alloc_hook(struct page *page, unsigned int order, 2138 gfp_t gfp_flags) 2139 { 2140 set_page_private(page, 0); 2141 set_page_refcounted(page); 2142 2143 arch_alloc_page(page, order); 2144 if (debug_pagealloc_enabled()) 2145 kernel_map_pages(page, 1 << order, 1); 2146 kasan_alloc_pages(page, order); 2147 kernel_poison_pages(page, 1 << order, 1); 2148 set_page_owner(page, order, gfp_flags); 2149 } 2150 2151 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 2152 unsigned int alloc_flags) 2153 { 2154 post_alloc_hook(page, order, gfp_flags); 2155 2156 if (!free_pages_prezeroed() && want_init_on_alloc(gfp_flags)) 2157 kernel_init_free_pages(page, 1 << order); 2158 2159 if (order && (gfp_flags & __GFP_COMP)) 2160 prep_compound_page(page, order); 2161 2162 /* 2163 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 2164 * allocate the page. The expectation is that the caller is taking 2165 * steps that will free more memory. The caller should avoid the page 2166 * being used for !PFMEMALLOC purposes. 2167 */ 2168 if (alloc_flags & ALLOC_NO_WATERMARKS) 2169 set_page_pfmemalloc(page); 2170 else 2171 clear_page_pfmemalloc(page); 2172 } 2173 2174 /* 2175 * Go through the free lists for the given migratetype and remove 2176 * the smallest available page from the freelists 2177 */ 2178 static __always_inline 2179 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 2180 int migratetype) 2181 { 2182 unsigned int current_order; 2183 struct free_area *area; 2184 struct page *page; 2185 2186 /* Find a page of the appropriate size in the preferred list */ 2187 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 2188 area = &(zone->free_area[current_order]); 2189 page = get_page_from_free_area(area, migratetype); 2190 if (!page) 2191 continue; 2192 del_page_from_free_area(page, area); 2193 expand(zone, page, order, current_order, area, migratetype); 2194 set_pcppage_migratetype(page, migratetype); 2195 return page; 2196 } 2197 2198 return NULL; 2199 } 2200 2201 2202 /* 2203 * This array describes the order lists are fallen back to when 2204 * the free lists for the desirable migrate type are depleted 2205 */ 2206 static int fallbacks[MIGRATE_TYPES][4] = { 2207 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2208 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 2209 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2210 #ifdef CONFIG_CMA 2211 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */ 2212 #endif 2213 #ifdef CONFIG_MEMORY_ISOLATION 2214 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */ 2215 #endif 2216 }; 2217 2218 #ifdef CONFIG_CMA 2219 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2220 unsigned int order) 2221 { 2222 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 2223 } 2224 #else 2225 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2226 unsigned int order) { return NULL; } 2227 #endif 2228 2229 /* 2230 * Move the free pages in a range to the free lists of the requested type. 2231 * Note that start_page and end_pages are not aligned on a pageblock 2232 * boundary. If alignment is required, use move_freepages_block() 2233 */ 2234 static int move_freepages(struct zone *zone, 2235 struct page *start_page, struct page *end_page, 2236 int migratetype, int *num_movable) 2237 { 2238 struct page *page; 2239 unsigned int order; 2240 int pages_moved = 0; 2241 2242 for (page = start_page; page <= end_page;) { 2243 if (!pfn_valid_within(page_to_pfn(page))) { 2244 page++; 2245 continue; 2246 } 2247 2248 if (!PageBuddy(page)) { 2249 /* 2250 * We assume that pages that could be isolated for 2251 * migration are movable. But we don't actually try 2252 * isolating, as that would be expensive. 2253 */ 2254 if (num_movable && 2255 (PageLRU(page) || __PageMovable(page))) 2256 (*num_movable)++; 2257 2258 page++; 2259 continue; 2260 } 2261 2262 /* Make sure we are not inadvertently changing nodes */ 2263 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2264 VM_BUG_ON_PAGE(page_zone(page) != zone, page); 2265 2266 order = page_order(page); 2267 move_to_free_area(page, &zone->free_area[order], migratetype); 2268 page += 1 << order; 2269 pages_moved += 1 << order; 2270 } 2271 2272 return pages_moved; 2273 } 2274 2275 int move_freepages_block(struct zone *zone, struct page *page, 2276 int migratetype, int *num_movable) 2277 { 2278 unsigned long start_pfn, end_pfn; 2279 struct page *start_page, *end_page; 2280 2281 if (num_movable) 2282 *num_movable = 0; 2283 2284 start_pfn = page_to_pfn(page); 2285 start_pfn = start_pfn & ~(pageblock_nr_pages-1); 2286 start_page = pfn_to_page(start_pfn); 2287 end_page = start_page + pageblock_nr_pages - 1; 2288 end_pfn = start_pfn + pageblock_nr_pages - 1; 2289 2290 /* Do not cross zone boundaries */ 2291 if (!zone_spans_pfn(zone, start_pfn)) 2292 start_page = page; 2293 if (!zone_spans_pfn(zone, end_pfn)) 2294 return 0; 2295 2296 return move_freepages(zone, start_page, end_page, migratetype, 2297 num_movable); 2298 } 2299 2300 static void change_pageblock_range(struct page *pageblock_page, 2301 int start_order, int migratetype) 2302 { 2303 int nr_pageblocks = 1 << (start_order - pageblock_order); 2304 2305 while (nr_pageblocks--) { 2306 set_pageblock_migratetype(pageblock_page, migratetype); 2307 pageblock_page += pageblock_nr_pages; 2308 } 2309 } 2310 2311 /* 2312 * When we are falling back to another migratetype during allocation, try to 2313 * steal extra free pages from the same pageblocks to satisfy further 2314 * allocations, instead of polluting multiple pageblocks. 2315 * 2316 * If we are stealing a relatively large buddy page, it is likely there will 2317 * be more free pages in the pageblock, so try to steal them all. For 2318 * reclaimable and unmovable allocations, we steal regardless of page size, 2319 * as fragmentation caused by those allocations polluting movable pageblocks 2320 * is worse than movable allocations stealing from unmovable and reclaimable 2321 * pageblocks. 2322 */ 2323 static bool can_steal_fallback(unsigned int order, int start_mt) 2324 { 2325 /* 2326 * Leaving this order check is intended, although there is 2327 * relaxed order check in next check. The reason is that 2328 * we can actually steal whole pageblock if this condition met, 2329 * but, below check doesn't guarantee it and that is just heuristic 2330 * so could be changed anytime. 2331 */ 2332 if (order >= pageblock_order) 2333 return true; 2334 2335 if (order >= pageblock_order / 2 || 2336 start_mt == MIGRATE_RECLAIMABLE || 2337 start_mt == MIGRATE_UNMOVABLE || 2338 page_group_by_mobility_disabled) 2339 return true; 2340 2341 return false; 2342 } 2343 2344 static inline void boost_watermark(struct zone *zone) 2345 { 2346 unsigned long max_boost; 2347 2348 if (!watermark_boost_factor) 2349 return; 2350 2351 max_boost = mult_frac(zone->_watermark[WMARK_HIGH], 2352 watermark_boost_factor, 10000); 2353 2354 /* 2355 * high watermark may be uninitialised if fragmentation occurs 2356 * very early in boot so do not boost. We do not fall 2357 * through and boost by pageblock_nr_pages as failing 2358 * allocations that early means that reclaim is not going 2359 * to help and it may even be impossible to reclaim the 2360 * boosted watermark resulting in a hang. 2361 */ 2362 if (!max_boost) 2363 return; 2364 2365 max_boost = max(pageblock_nr_pages, max_boost); 2366 2367 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages, 2368 max_boost); 2369 } 2370 2371 /* 2372 * This function implements actual steal behaviour. If order is large enough, 2373 * we can steal whole pageblock. If not, we first move freepages in this 2374 * pageblock to our migratetype and determine how many already-allocated pages 2375 * are there in the pageblock with a compatible migratetype. If at least half 2376 * of pages are free or compatible, we can change migratetype of the pageblock 2377 * itself, so pages freed in the future will be put on the correct free list. 2378 */ 2379 static void steal_suitable_fallback(struct zone *zone, struct page *page, 2380 unsigned int alloc_flags, int start_type, bool whole_block) 2381 { 2382 unsigned int current_order = page_order(page); 2383 struct free_area *area; 2384 int free_pages, movable_pages, alike_pages; 2385 int old_block_type; 2386 2387 old_block_type = get_pageblock_migratetype(page); 2388 2389 /* 2390 * This can happen due to races and we want to prevent broken 2391 * highatomic accounting. 2392 */ 2393 if (is_migrate_highatomic(old_block_type)) 2394 goto single_page; 2395 2396 /* Take ownership for orders >= pageblock_order */ 2397 if (current_order >= pageblock_order) { 2398 change_pageblock_range(page, current_order, start_type); 2399 goto single_page; 2400 } 2401 2402 /* 2403 * Boost watermarks to increase reclaim pressure to reduce the 2404 * likelihood of future fallbacks. Wake kswapd now as the node 2405 * may be balanced overall and kswapd will not wake naturally. 2406 */ 2407 boost_watermark(zone); 2408 if (alloc_flags & ALLOC_KSWAPD) 2409 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 2410 2411 /* We are not allowed to try stealing from the whole block */ 2412 if (!whole_block) 2413 goto single_page; 2414 2415 free_pages = move_freepages_block(zone, page, start_type, 2416 &movable_pages); 2417 /* 2418 * Determine how many pages are compatible with our allocation. 2419 * For movable allocation, it's the number of movable pages which 2420 * we just obtained. For other types it's a bit more tricky. 2421 */ 2422 if (start_type == MIGRATE_MOVABLE) { 2423 alike_pages = movable_pages; 2424 } else { 2425 /* 2426 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation 2427 * to MOVABLE pageblock, consider all non-movable pages as 2428 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or 2429 * vice versa, be conservative since we can't distinguish the 2430 * exact migratetype of non-movable pages. 2431 */ 2432 if (old_block_type == MIGRATE_MOVABLE) 2433 alike_pages = pageblock_nr_pages 2434 - (free_pages + movable_pages); 2435 else 2436 alike_pages = 0; 2437 } 2438 2439 /* moving whole block can fail due to zone boundary conditions */ 2440 if (!free_pages) 2441 goto single_page; 2442 2443 /* 2444 * If a sufficient number of pages in the block are either free or of 2445 * comparable migratability as our allocation, claim the whole block. 2446 */ 2447 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || 2448 page_group_by_mobility_disabled) 2449 set_pageblock_migratetype(page, start_type); 2450 2451 return; 2452 2453 single_page: 2454 area = &zone->free_area[current_order]; 2455 move_to_free_area(page, area, start_type); 2456 } 2457 2458 /* 2459 * Check whether there is a suitable fallback freepage with requested order. 2460 * If only_stealable is true, this function returns fallback_mt only if 2461 * we can steal other freepages all together. This would help to reduce 2462 * fragmentation due to mixed migratetype pages in one pageblock. 2463 */ 2464 int find_suitable_fallback(struct free_area *area, unsigned int order, 2465 int migratetype, bool only_stealable, bool *can_steal) 2466 { 2467 int i; 2468 int fallback_mt; 2469 2470 if (area->nr_free == 0) 2471 return -1; 2472 2473 *can_steal = false; 2474 for (i = 0;; i++) { 2475 fallback_mt = fallbacks[migratetype][i]; 2476 if (fallback_mt == MIGRATE_TYPES) 2477 break; 2478 2479 if (free_area_empty(area, fallback_mt)) 2480 continue; 2481 2482 if (can_steal_fallback(order, migratetype)) 2483 *can_steal = true; 2484 2485 if (!only_stealable) 2486 return fallback_mt; 2487 2488 if (*can_steal) 2489 return fallback_mt; 2490 } 2491 2492 return -1; 2493 } 2494 2495 /* 2496 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2497 * there are no empty page blocks that contain a page with a suitable order 2498 */ 2499 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2500 unsigned int alloc_order) 2501 { 2502 int mt; 2503 unsigned long max_managed, flags; 2504 2505 /* 2506 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2507 * Check is race-prone but harmless. 2508 */ 2509 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages; 2510 if (zone->nr_reserved_highatomic >= max_managed) 2511 return; 2512 2513 spin_lock_irqsave(&zone->lock, flags); 2514 2515 /* Recheck the nr_reserved_highatomic limit under the lock */ 2516 if (zone->nr_reserved_highatomic >= max_managed) 2517 goto out_unlock; 2518 2519 /* Yoink! */ 2520 mt = get_pageblock_migratetype(page); 2521 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt) 2522 && !is_migrate_cma(mt)) { 2523 zone->nr_reserved_highatomic += pageblock_nr_pages; 2524 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2525 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); 2526 } 2527 2528 out_unlock: 2529 spin_unlock_irqrestore(&zone->lock, flags); 2530 } 2531 2532 /* 2533 * Used when an allocation is about to fail under memory pressure. This 2534 * potentially hurts the reliability of high-order allocations when under 2535 * intense memory pressure but failed atomic allocations should be easier 2536 * to recover from than an OOM. 2537 * 2538 * If @force is true, try to unreserve a pageblock even though highatomic 2539 * pageblock is exhausted. 2540 */ 2541 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 2542 bool force) 2543 { 2544 struct zonelist *zonelist = ac->zonelist; 2545 unsigned long flags; 2546 struct zoneref *z; 2547 struct zone *zone; 2548 struct page *page; 2549 int order; 2550 bool ret; 2551 2552 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx, 2553 ac->nodemask) { 2554 /* 2555 * Preserve at least one pageblock unless memory pressure 2556 * is really high. 2557 */ 2558 if (!force && zone->nr_reserved_highatomic <= 2559 pageblock_nr_pages) 2560 continue; 2561 2562 spin_lock_irqsave(&zone->lock, flags); 2563 for (order = 0; order < MAX_ORDER; order++) { 2564 struct free_area *area = &(zone->free_area[order]); 2565 2566 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); 2567 if (!page) 2568 continue; 2569 2570 /* 2571 * In page freeing path, migratetype change is racy so 2572 * we can counter several free pages in a pageblock 2573 * in this loop althoug we changed the pageblock type 2574 * from highatomic to ac->migratetype. So we should 2575 * adjust the count once. 2576 */ 2577 if (is_migrate_highatomic_page(page)) { 2578 /* 2579 * It should never happen but changes to 2580 * locking could inadvertently allow a per-cpu 2581 * drain to add pages to MIGRATE_HIGHATOMIC 2582 * while unreserving so be safe and watch for 2583 * underflows. 2584 */ 2585 zone->nr_reserved_highatomic -= min( 2586 pageblock_nr_pages, 2587 zone->nr_reserved_highatomic); 2588 } 2589 2590 /* 2591 * Convert to ac->migratetype and avoid the normal 2592 * pageblock stealing heuristics. Minimally, the caller 2593 * is doing the work and needs the pages. More 2594 * importantly, if the block was always converted to 2595 * MIGRATE_UNMOVABLE or another type then the number 2596 * of pageblocks that cannot be completely freed 2597 * may increase. 2598 */ 2599 set_pageblock_migratetype(page, ac->migratetype); 2600 ret = move_freepages_block(zone, page, ac->migratetype, 2601 NULL); 2602 if (ret) { 2603 spin_unlock_irqrestore(&zone->lock, flags); 2604 return ret; 2605 } 2606 } 2607 spin_unlock_irqrestore(&zone->lock, flags); 2608 } 2609 2610 return false; 2611 } 2612 2613 /* 2614 * Try finding a free buddy page on the fallback list and put it on the free 2615 * list of requested migratetype, possibly along with other pages from the same 2616 * block, depending on fragmentation avoidance heuristics. Returns true if 2617 * fallback was found so that __rmqueue_smallest() can grab it. 2618 * 2619 * The use of signed ints for order and current_order is a deliberate 2620 * deviation from the rest of this file, to make the for loop 2621 * condition simpler. 2622 */ 2623 static __always_inline bool 2624 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, 2625 unsigned int alloc_flags) 2626 { 2627 struct free_area *area; 2628 int current_order; 2629 int min_order = order; 2630 struct page *page; 2631 int fallback_mt; 2632 bool can_steal; 2633 2634 /* 2635 * Do not steal pages from freelists belonging to other pageblocks 2636 * i.e. orders < pageblock_order. If there are no local zones free, 2637 * the zonelists will be reiterated without ALLOC_NOFRAGMENT. 2638 */ 2639 if (alloc_flags & ALLOC_NOFRAGMENT) 2640 min_order = pageblock_order; 2641 2642 /* 2643 * Find the largest available free page in the other list. This roughly 2644 * approximates finding the pageblock with the most free pages, which 2645 * would be too costly to do exactly. 2646 */ 2647 for (current_order = MAX_ORDER - 1; current_order >= min_order; 2648 --current_order) { 2649 area = &(zone->free_area[current_order]); 2650 fallback_mt = find_suitable_fallback(area, current_order, 2651 start_migratetype, false, &can_steal); 2652 if (fallback_mt == -1) 2653 continue; 2654 2655 /* 2656 * We cannot steal all free pages from the pageblock and the 2657 * requested migratetype is movable. In that case it's better to 2658 * steal and split the smallest available page instead of the 2659 * largest available page, because even if the next movable 2660 * allocation falls back into a different pageblock than this 2661 * one, it won't cause permanent fragmentation. 2662 */ 2663 if (!can_steal && start_migratetype == MIGRATE_MOVABLE 2664 && current_order > order) 2665 goto find_smallest; 2666 2667 goto do_steal; 2668 } 2669 2670 return false; 2671 2672 find_smallest: 2673 for (current_order = order; current_order < MAX_ORDER; 2674 current_order++) { 2675 area = &(zone->free_area[current_order]); 2676 fallback_mt = find_suitable_fallback(area, current_order, 2677 start_migratetype, false, &can_steal); 2678 if (fallback_mt != -1) 2679 break; 2680 } 2681 2682 /* 2683 * This should not happen - we already found a suitable fallback 2684 * when looking for the largest page. 2685 */ 2686 VM_BUG_ON(current_order == MAX_ORDER); 2687 2688 do_steal: 2689 page = get_page_from_free_area(area, fallback_mt); 2690 2691 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype, 2692 can_steal); 2693 2694 trace_mm_page_alloc_extfrag(page, order, current_order, 2695 start_migratetype, fallback_mt); 2696 2697 return true; 2698 2699 } 2700 2701 /* 2702 * Do the hard work of removing an element from the buddy allocator. 2703 * Call me with the zone->lock already held. 2704 */ 2705 static __always_inline struct page * 2706 __rmqueue(struct zone *zone, unsigned int order, int migratetype, 2707 unsigned int alloc_flags) 2708 { 2709 struct page *page; 2710 2711 retry: 2712 page = __rmqueue_smallest(zone, order, migratetype); 2713 if (unlikely(!page)) { 2714 if (migratetype == MIGRATE_MOVABLE) 2715 page = __rmqueue_cma_fallback(zone, order); 2716 2717 if (!page && __rmqueue_fallback(zone, order, migratetype, 2718 alloc_flags)) 2719 goto retry; 2720 } 2721 2722 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2723 return page; 2724 } 2725 2726 /* 2727 * Obtain a specified number of elements from the buddy allocator, all under 2728 * a single hold of the lock, for efficiency. Add them to the supplied list. 2729 * Returns the number of new pages which were placed at *list. 2730 */ 2731 static int rmqueue_bulk(struct zone *zone, unsigned int order, 2732 unsigned long count, struct list_head *list, 2733 int migratetype, unsigned int alloc_flags) 2734 { 2735 int i, alloced = 0; 2736 2737 spin_lock(&zone->lock); 2738 for (i = 0; i < count; ++i) { 2739 struct page *page = __rmqueue(zone, order, migratetype, 2740 alloc_flags); 2741 if (unlikely(page == NULL)) 2742 break; 2743 2744 if (unlikely(check_pcp_refill(page))) 2745 continue; 2746 2747 /* 2748 * Split buddy pages returned by expand() are received here in 2749 * physical page order. The page is added to the tail of 2750 * caller's list. From the callers perspective, the linked list 2751 * is ordered by page number under some conditions. This is 2752 * useful for IO devices that can forward direction from the 2753 * head, thus also in the physical page order. This is useful 2754 * for IO devices that can merge IO requests if the physical 2755 * pages are ordered properly. 2756 */ 2757 list_add_tail(&page->lru, list); 2758 alloced++; 2759 if (is_migrate_cma(get_pcppage_migratetype(page))) 2760 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 2761 -(1 << order)); 2762 } 2763 2764 /* 2765 * i pages were removed from the buddy list even if some leak due 2766 * to check_pcp_refill failing so adjust NR_FREE_PAGES based 2767 * on i. Do not confuse with 'alloced' which is the number of 2768 * pages added to the pcp list. 2769 */ 2770 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 2771 spin_unlock(&zone->lock); 2772 return alloced; 2773 } 2774 2775 #ifdef CONFIG_NUMA 2776 /* 2777 * Called from the vmstat counter updater to drain pagesets of this 2778 * currently executing processor on remote nodes after they have 2779 * expired. 2780 * 2781 * Note that this function must be called with the thread pinned to 2782 * a single processor. 2783 */ 2784 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 2785 { 2786 unsigned long flags; 2787 int to_drain, batch; 2788 2789 local_irq_save(flags); 2790 batch = READ_ONCE(pcp->batch); 2791 to_drain = min(pcp->count, batch); 2792 if (to_drain > 0) 2793 free_pcppages_bulk(zone, to_drain, pcp); 2794 local_irq_restore(flags); 2795 } 2796 #endif 2797 2798 /* 2799 * Drain pcplists of the indicated processor and zone. 2800 * 2801 * The processor must either be the current processor and the 2802 * thread pinned to the current processor or a processor that 2803 * is not online. 2804 */ 2805 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 2806 { 2807 unsigned long flags; 2808 struct per_cpu_pageset *pset; 2809 struct per_cpu_pages *pcp; 2810 2811 local_irq_save(flags); 2812 pset = per_cpu_ptr(zone->pageset, cpu); 2813 2814 pcp = &pset->pcp; 2815 if (pcp->count) 2816 free_pcppages_bulk(zone, pcp->count, pcp); 2817 local_irq_restore(flags); 2818 } 2819 2820 /* 2821 * Drain pcplists of all zones on the indicated processor. 2822 * 2823 * The processor must either be the current processor and the 2824 * thread pinned to the current processor or a processor that 2825 * is not online. 2826 */ 2827 static void drain_pages(unsigned int cpu) 2828 { 2829 struct zone *zone; 2830 2831 for_each_populated_zone(zone) { 2832 drain_pages_zone(cpu, zone); 2833 } 2834 } 2835 2836 /* 2837 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 2838 * 2839 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 2840 * the single zone's pages. 2841 */ 2842 void drain_local_pages(struct zone *zone) 2843 { 2844 int cpu = smp_processor_id(); 2845 2846 if (zone) 2847 drain_pages_zone(cpu, zone); 2848 else 2849 drain_pages(cpu); 2850 } 2851 2852 static void drain_local_pages_wq(struct work_struct *work) 2853 { 2854 struct pcpu_drain *drain; 2855 2856 drain = container_of(work, struct pcpu_drain, work); 2857 2858 /* 2859 * drain_all_pages doesn't use proper cpu hotplug protection so 2860 * we can race with cpu offline when the WQ can move this from 2861 * a cpu pinned worker to an unbound one. We can operate on a different 2862 * cpu which is allright but we also have to make sure to not move to 2863 * a different one. 2864 */ 2865 preempt_disable(); 2866 drain_local_pages(drain->zone); 2867 preempt_enable(); 2868 } 2869 2870 /* 2871 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 2872 * 2873 * When zone parameter is non-NULL, spill just the single zone's pages. 2874 * 2875 * Note that this can be extremely slow as the draining happens in a workqueue. 2876 */ 2877 void drain_all_pages(struct zone *zone) 2878 { 2879 int cpu; 2880 2881 /* 2882 * Allocate in the BSS so we wont require allocation in 2883 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 2884 */ 2885 static cpumask_t cpus_with_pcps; 2886 2887 /* 2888 * Make sure nobody triggers this path before mm_percpu_wq is fully 2889 * initialized. 2890 */ 2891 if (WARN_ON_ONCE(!mm_percpu_wq)) 2892 return; 2893 2894 /* 2895 * Do not drain if one is already in progress unless it's specific to 2896 * a zone. Such callers are primarily CMA and memory hotplug and need 2897 * the drain to be complete when the call returns. 2898 */ 2899 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) { 2900 if (!zone) 2901 return; 2902 mutex_lock(&pcpu_drain_mutex); 2903 } 2904 2905 /* 2906 * We don't care about racing with CPU hotplug event 2907 * as offline notification will cause the notified 2908 * cpu to drain that CPU pcps and on_each_cpu_mask 2909 * disables preemption as part of its processing 2910 */ 2911 for_each_online_cpu(cpu) { 2912 struct per_cpu_pageset *pcp; 2913 struct zone *z; 2914 bool has_pcps = false; 2915 2916 if (zone) { 2917 pcp = per_cpu_ptr(zone->pageset, cpu); 2918 if (pcp->pcp.count) 2919 has_pcps = true; 2920 } else { 2921 for_each_populated_zone(z) { 2922 pcp = per_cpu_ptr(z->pageset, cpu); 2923 if (pcp->pcp.count) { 2924 has_pcps = true; 2925 break; 2926 } 2927 } 2928 } 2929 2930 if (has_pcps) 2931 cpumask_set_cpu(cpu, &cpus_with_pcps); 2932 else 2933 cpumask_clear_cpu(cpu, &cpus_with_pcps); 2934 } 2935 2936 for_each_cpu(cpu, &cpus_with_pcps) { 2937 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu); 2938 2939 drain->zone = zone; 2940 INIT_WORK(&drain->work, drain_local_pages_wq); 2941 queue_work_on(cpu, mm_percpu_wq, &drain->work); 2942 } 2943 for_each_cpu(cpu, &cpus_with_pcps) 2944 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work); 2945 2946 mutex_unlock(&pcpu_drain_mutex); 2947 } 2948 2949 #ifdef CONFIG_HIBERNATION 2950 2951 /* 2952 * Touch the watchdog for every WD_PAGE_COUNT pages. 2953 */ 2954 #define WD_PAGE_COUNT (128*1024) 2955 2956 void mark_free_pages(struct zone *zone) 2957 { 2958 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT; 2959 unsigned long flags; 2960 unsigned int order, t; 2961 struct page *page; 2962 2963 if (zone_is_empty(zone)) 2964 return; 2965 2966 spin_lock_irqsave(&zone->lock, flags); 2967 2968 max_zone_pfn = zone_end_pfn(zone); 2969 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 2970 if (pfn_valid(pfn)) { 2971 page = pfn_to_page(pfn); 2972 2973 if (!--page_count) { 2974 touch_nmi_watchdog(); 2975 page_count = WD_PAGE_COUNT; 2976 } 2977 2978 if (page_zone(page) != zone) 2979 continue; 2980 2981 if (!swsusp_page_is_forbidden(page)) 2982 swsusp_unset_page_free(page); 2983 } 2984 2985 for_each_migratetype_order(order, t) { 2986 list_for_each_entry(page, 2987 &zone->free_area[order].free_list[t], lru) { 2988 unsigned long i; 2989 2990 pfn = page_to_pfn(page); 2991 for (i = 0; i < (1UL << order); i++) { 2992 if (!--page_count) { 2993 touch_nmi_watchdog(); 2994 page_count = WD_PAGE_COUNT; 2995 } 2996 swsusp_set_page_free(pfn_to_page(pfn + i)); 2997 } 2998 } 2999 } 3000 spin_unlock_irqrestore(&zone->lock, flags); 3001 } 3002 #endif /* CONFIG_PM */ 3003 3004 static bool free_unref_page_prepare(struct page *page, unsigned long pfn) 3005 { 3006 int migratetype; 3007 3008 if (!free_pcp_prepare(page)) 3009 return false; 3010 3011 migratetype = get_pfnblock_migratetype(page, pfn); 3012 set_pcppage_migratetype(page, migratetype); 3013 return true; 3014 } 3015 3016 static void free_unref_page_commit(struct page *page, unsigned long pfn) 3017 { 3018 struct zone *zone = page_zone(page); 3019 struct per_cpu_pages *pcp; 3020 int migratetype; 3021 3022 migratetype = get_pcppage_migratetype(page); 3023 __count_vm_event(PGFREE); 3024 3025 /* 3026 * We only track unmovable, reclaimable and movable on pcp lists. 3027 * Free ISOLATE pages back to the allocator because they are being 3028 * offlined but treat HIGHATOMIC as movable pages so we can get those 3029 * areas back if necessary. Otherwise, we may have to free 3030 * excessively into the page allocator 3031 */ 3032 if (migratetype >= MIGRATE_PCPTYPES) { 3033 if (unlikely(is_migrate_isolate(migratetype))) { 3034 free_one_page(zone, page, pfn, 0, migratetype); 3035 return; 3036 } 3037 migratetype = MIGRATE_MOVABLE; 3038 } 3039 3040 pcp = &this_cpu_ptr(zone->pageset)->pcp; 3041 list_add(&page->lru, &pcp->lists[migratetype]); 3042 pcp->count++; 3043 if (pcp->count >= pcp->high) { 3044 unsigned long batch = READ_ONCE(pcp->batch); 3045 free_pcppages_bulk(zone, batch, pcp); 3046 } 3047 } 3048 3049 /* 3050 * Free a 0-order page 3051 */ 3052 void free_unref_page(struct page *page) 3053 { 3054 unsigned long flags; 3055 unsigned long pfn = page_to_pfn(page); 3056 3057 if (!free_unref_page_prepare(page, pfn)) 3058 return; 3059 3060 local_irq_save(flags); 3061 free_unref_page_commit(page, pfn); 3062 local_irq_restore(flags); 3063 } 3064 3065 /* 3066 * Free a list of 0-order pages 3067 */ 3068 void free_unref_page_list(struct list_head *list) 3069 { 3070 struct page *page, *next; 3071 unsigned long flags, pfn; 3072 int batch_count = 0; 3073 3074 /* Prepare pages for freeing */ 3075 list_for_each_entry_safe(page, next, list, lru) { 3076 pfn = page_to_pfn(page); 3077 if (!free_unref_page_prepare(page, pfn)) 3078 list_del(&page->lru); 3079 set_page_private(page, pfn); 3080 } 3081 3082 local_irq_save(flags); 3083 list_for_each_entry_safe(page, next, list, lru) { 3084 unsigned long pfn = page_private(page); 3085 3086 set_page_private(page, 0); 3087 trace_mm_page_free_batched(page); 3088 free_unref_page_commit(page, pfn); 3089 3090 /* 3091 * Guard against excessive IRQ disabled times when we get 3092 * a large list of pages to free. 3093 */ 3094 if (++batch_count == SWAP_CLUSTER_MAX) { 3095 local_irq_restore(flags); 3096 batch_count = 0; 3097 local_irq_save(flags); 3098 } 3099 } 3100 local_irq_restore(flags); 3101 } 3102 3103 /* 3104 * split_page takes a non-compound higher-order page, and splits it into 3105 * n (1<<order) sub-pages: page[0..n] 3106 * Each sub-page must be freed individually. 3107 * 3108 * Note: this is probably too low level an operation for use in drivers. 3109 * Please consult with lkml before using this in your driver. 3110 */ 3111 void split_page(struct page *page, unsigned int order) 3112 { 3113 int i; 3114 3115 VM_BUG_ON_PAGE(PageCompound(page), page); 3116 VM_BUG_ON_PAGE(!page_count(page), page); 3117 3118 for (i = 1; i < (1 << order); i++) 3119 set_page_refcounted(page + i); 3120 split_page_owner(page, order); 3121 } 3122 EXPORT_SYMBOL_GPL(split_page); 3123 3124 int __isolate_free_page(struct page *page, unsigned int order) 3125 { 3126 struct free_area *area = &page_zone(page)->free_area[order]; 3127 unsigned long watermark; 3128 struct zone *zone; 3129 int mt; 3130 3131 BUG_ON(!PageBuddy(page)); 3132 3133 zone = page_zone(page); 3134 mt = get_pageblock_migratetype(page); 3135 3136 if (!is_migrate_isolate(mt)) { 3137 /* 3138 * Obey watermarks as if the page was being allocated. We can 3139 * emulate a high-order watermark check with a raised order-0 3140 * watermark, because we already know our high-order page 3141 * exists. 3142 */ 3143 watermark = zone->_watermark[WMARK_MIN] + (1UL << order); 3144 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 3145 return 0; 3146 3147 __mod_zone_freepage_state(zone, -(1UL << order), mt); 3148 } 3149 3150 /* Remove page from free list */ 3151 3152 del_page_from_free_area(page, area); 3153 3154 /* 3155 * Set the pageblock if the isolated page is at least half of a 3156 * pageblock 3157 */ 3158 if (order >= pageblock_order - 1) { 3159 struct page *endpage = page + (1 << order) - 1; 3160 for (; page < endpage; page += pageblock_nr_pages) { 3161 int mt = get_pageblock_migratetype(page); 3162 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt) 3163 && !is_migrate_highatomic(mt)) 3164 set_pageblock_migratetype(page, 3165 MIGRATE_MOVABLE); 3166 } 3167 } 3168 3169 3170 return 1UL << order; 3171 } 3172 3173 /* 3174 * Update NUMA hit/miss statistics 3175 * 3176 * Must be called with interrupts disabled. 3177 */ 3178 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) 3179 { 3180 #ifdef CONFIG_NUMA 3181 enum numa_stat_item local_stat = NUMA_LOCAL; 3182 3183 /* skip numa counters update if numa stats is disabled */ 3184 if (!static_branch_likely(&vm_numa_stat_key)) 3185 return; 3186 3187 if (zone_to_nid(z) != numa_node_id()) 3188 local_stat = NUMA_OTHER; 3189 3190 if (zone_to_nid(z) == zone_to_nid(preferred_zone)) 3191 __inc_numa_state(z, NUMA_HIT); 3192 else { 3193 __inc_numa_state(z, NUMA_MISS); 3194 __inc_numa_state(preferred_zone, NUMA_FOREIGN); 3195 } 3196 __inc_numa_state(z, local_stat); 3197 #endif 3198 } 3199 3200 /* Remove page from the per-cpu list, caller must protect the list */ 3201 static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype, 3202 unsigned int alloc_flags, 3203 struct per_cpu_pages *pcp, 3204 struct list_head *list) 3205 { 3206 struct page *page; 3207 3208 do { 3209 if (list_empty(list)) { 3210 pcp->count += rmqueue_bulk(zone, 0, 3211 pcp->batch, list, 3212 migratetype, alloc_flags); 3213 if (unlikely(list_empty(list))) 3214 return NULL; 3215 } 3216 3217 page = list_first_entry(list, struct page, lru); 3218 list_del(&page->lru); 3219 pcp->count--; 3220 } while (check_new_pcp(page)); 3221 3222 return page; 3223 } 3224 3225 /* Lock and remove page from the per-cpu list */ 3226 static struct page *rmqueue_pcplist(struct zone *preferred_zone, 3227 struct zone *zone, gfp_t gfp_flags, 3228 int migratetype, unsigned int alloc_flags) 3229 { 3230 struct per_cpu_pages *pcp; 3231 struct list_head *list; 3232 struct page *page; 3233 unsigned long flags; 3234 3235 local_irq_save(flags); 3236 pcp = &this_cpu_ptr(zone->pageset)->pcp; 3237 list = &pcp->lists[migratetype]; 3238 page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list); 3239 if (page) { 3240 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); 3241 zone_statistics(preferred_zone, zone); 3242 } 3243 local_irq_restore(flags); 3244 return page; 3245 } 3246 3247 /* 3248 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 3249 */ 3250 static inline 3251 struct page *rmqueue(struct zone *preferred_zone, 3252 struct zone *zone, unsigned int order, 3253 gfp_t gfp_flags, unsigned int alloc_flags, 3254 int migratetype) 3255 { 3256 unsigned long flags; 3257 struct page *page; 3258 3259 if (likely(order == 0)) { 3260 page = rmqueue_pcplist(preferred_zone, zone, gfp_flags, 3261 migratetype, alloc_flags); 3262 goto out; 3263 } 3264 3265 /* 3266 * We most definitely don't want callers attempting to 3267 * allocate greater than order-1 page units with __GFP_NOFAIL. 3268 */ 3269 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 3270 spin_lock_irqsave(&zone->lock, flags); 3271 3272 do { 3273 page = NULL; 3274 if (alloc_flags & ALLOC_HARDER) { 3275 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3276 if (page) 3277 trace_mm_page_alloc_zone_locked(page, order, migratetype); 3278 } 3279 if (!page) 3280 page = __rmqueue(zone, order, migratetype, alloc_flags); 3281 } while (page && check_new_pages(page, order)); 3282 spin_unlock(&zone->lock); 3283 if (!page) 3284 goto failed; 3285 __mod_zone_freepage_state(zone, -(1 << order), 3286 get_pcppage_migratetype(page)); 3287 3288 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3289 zone_statistics(preferred_zone, zone); 3290 local_irq_restore(flags); 3291 3292 out: 3293 /* Separate test+clear to avoid unnecessary atomics */ 3294 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) { 3295 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 3296 wakeup_kswapd(zone, 0, 0, zone_idx(zone)); 3297 } 3298 3299 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); 3300 return page; 3301 3302 failed: 3303 local_irq_restore(flags); 3304 return NULL; 3305 } 3306 3307 #ifdef CONFIG_FAIL_PAGE_ALLOC 3308 3309 static struct { 3310 struct fault_attr attr; 3311 3312 bool ignore_gfp_highmem; 3313 bool ignore_gfp_reclaim; 3314 u32 min_order; 3315 } fail_page_alloc = { 3316 .attr = FAULT_ATTR_INITIALIZER, 3317 .ignore_gfp_reclaim = true, 3318 .ignore_gfp_highmem = true, 3319 .min_order = 1, 3320 }; 3321 3322 static int __init setup_fail_page_alloc(char *str) 3323 { 3324 return setup_fault_attr(&fail_page_alloc.attr, str); 3325 } 3326 __setup("fail_page_alloc=", setup_fail_page_alloc); 3327 3328 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3329 { 3330 if (order < fail_page_alloc.min_order) 3331 return false; 3332 if (gfp_mask & __GFP_NOFAIL) 3333 return false; 3334 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 3335 return false; 3336 if (fail_page_alloc.ignore_gfp_reclaim && 3337 (gfp_mask & __GFP_DIRECT_RECLAIM)) 3338 return false; 3339 3340 return should_fail(&fail_page_alloc.attr, 1 << order); 3341 } 3342 3343 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 3344 3345 static int __init fail_page_alloc_debugfs(void) 3346 { 3347 umode_t mode = S_IFREG | 0600; 3348 struct dentry *dir; 3349 3350 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 3351 &fail_page_alloc.attr); 3352 3353 debugfs_create_bool("ignore-gfp-wait", mode, dir, 3354 &fail_page_alloc.ignore_gfp_reclaim); 3355 debugfs_create_bool("ignore-gfp-highmem", mode, dir, 3356 &fail_page_alloc.ignore_gfp_highmem); 3357 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order); 3358 3359 return 0; 3360 } 3361 3362 late_initcall(fail_page_alloc_debugfs); 3363 3364 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 3365 3366 #else /* CONFIG_FAIL_PAGE_ALLOC */ 3367 3368 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3369 { 3370 return false; 3371 } 3372 3373 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 3374 3375 static noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3376 { 3377 return __should_fail_alloc_page(gfp_mask, order); 3378 } 3379 ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE); 3380 3381 /* 3382 * Return true if free base pages are above 'mark'. For high-order checks it 3383 * will return true of the order-0 watermark is reached and there is at least 3384 * one free page of a suitable size. Checking now avoids taking the zone lock 3385 * to check in the allocation paths if no pages are free. 3386 */ 3387 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3388 int classzone_idx, unsigned int alloc_flags, 3389 long free_pages) 3390 { 3391 long min = mark; 3392 int o; 3393 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3394 3395 /* free_pages may go negative - that's OK */ 3396 free_pages -= (1 << order) - 1; 3397 3398 if (alloc_flags & ALLOC_HIGH) 3399 min -= min / 2; 3400 3401 /* 3402 * If the caller does not have rights to ALLOC_HARDER then subtract 3403 * the high-atomic reserves. This will over-estimate the size of the 3404 * atomic reserve but it avoids a search. 3405 */ 3406 if (likely(!alloc_harder)) { 3407 free_pages -= z->nr_reserved_highatomic; 3408 } else { 3409 /* 3410 * OOM victims can try even harder than normal ALLOC_HARDER 3411 * users on the grounds that it's definitely going to be in 3412 * the exit path shortly and free memory. Any allocation it 3413 * makes during the free path will be small and short-lived. 3414 */ 3415 if (alloc_flags & ALLOC_OOM) 3416 min -= min / 2; 3417 else 3418 min -= min / 4; 3419 } 3420 3421 3422 #ifdef CONFIG_CMA 3423 /* If allocation can't use CMA areas don't use free CMA pages */ 3424 if (!(alloc_flags & ALLOC_CMA)) 3425 free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES); 3426 #endif 3427 3428 /* 3429 * Check watermarks for an order-0 allocation request. If these 3430 * are not met, then a high-order request also cannot go ahead 3431 * even if a suitable page happened to be free. 3432 */ 3433 if (free_pages <= min + z->lowmem_reserve[classzone_idx]) 3434 return false; 3435 3436 /* If this is an order-0 request then the watermark is fine */ 3437 if (!order) 3438 return true; 3439 3440 /* For a high-order request, check at least one suitable page is free */ 3441 for (o = order; o < MAX_ORDER; o++) { 3442 struct free_area *area = &z->free_area[o]; 3443 int mt; 3444 3445 if (!area->nr_free) 3446 continue; 3447 3448 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 3449 if (!free_area_empty(area, mt)) 3450 return true; 3451 } 3452 3453 #ifdef CONFIG_CMA 3454 if ((alloc_flags & ALLOC_CMA) && 3455 !free_area_empty(area, MIGRATE_CMA)) { 3456 return true; 3457 } 3458 #endif 3459 if (alloc_harder && 3460 !list_empty(&area->free_list[MIGRATE_HIGHATOMIC])) 3461 return true; 3462 } 3463 return false; 3464 } 3465 3466 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3467 int classzone_idx, unsigned int alloc_flags) 3468 { 3469 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 3470 zone_page_state(z, NR_FREE_PAGES)); 3471 } 3472 3473 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 3474 unsigned long mark, int classzone_idx, unsigned int alloc_flags) 3475 { 3476 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3477 long cma_pages = 0; 3478 3479 #ifdef CONFIG_CMA 3480 /* If allocation can't use CMA areas don't use free CMA pages */ 3481 if (!(alloc_flags & ALLOC_CMA)) 3482 cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES); 3483 #endif 3484 3485 /* 3486 * Fast check for order-0 only. If this fails then the reserves 3487 * need to be calculated. There is a corner case where the check 3488 * passes but only the high-order atomic reserve are free. If 3489 * the caller is !atomic then it'll uselessly search the free 3490 * list. That corner case is then slower but it is harmless. 3491 */ 3492 if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx]) 3493 return true; 3494 3495 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 3496 free_pages); 3497 } 3498 3499 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 3500 unsigned long mark, int classzone_idx) 3501 { 3502 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3503 3504 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 3505 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 3506 3507 return __zone_watermark_ok(z, order, mark, classzone_idx, 0, 3508 free_pages); 3509 } 3510 3511 #ifdef CONFIG_NUMA 3512 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3513 { 3514 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= 3515 node_reclaim_distance; 3516 } 3517 #else /* CONFIG_NUMA */ 3518 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3519 { 3520 return true; 3521 } 3522 #endif /* CONFIG_NUMA */ 3523 3524 /* 3525 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid 3526 * fragmentation is subtle. If the preferred zone was HIGHMEM then 3527 * premature use of a lower zone may cause lowmem pressure problems that 3528 * are worse than fragmentation. If the next zone is ZONE_DMA then it is 3529 * probably too small. It only makes sense to spread allocations to avoid 3530 * fragmentation between the Normal and DMA32 zones. 3531 */ 3532 static inline unsigned int 3533 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) 3534 { 3535 unsigned int alloc_flags = 0; 3536 3537 if (gfp_mask & __GFP_KSWAPD_RECLAIM) 3538 alloc_flags |= ALLOC_KSWAPD; 3539 3540 #ifdef CONFIG_ZONE_DMA32 3541 if (!zone) 3542 return alloc_flags; 3543 3544 if (zone_idx(zone) != ZONE_NORMAL) 3545 return alloc_flags; 3546 3547 /* 3548 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and 3549 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume 3550 * on UMA that if Normal is populated then so is DMA32. 3551 */ 3552 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1); 3553 if (nr_online_nodes > 1 && !populated_zone(--zone)) 3554 return alloc_flags; 3555 3556 alloc_flags |= ALLOC_NOFRAGMENT; 3557 #endif /* CONFIG_ZONE_DMA32 */ 3558 return alloc_flags; 3559 } 3560 3561 /* 3562 * get_page_from_freelist goes through the zonelist trying to allocate 3563 * a page. 3564 */ 3565 static struct page * 3566 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 3567 const struct alloc_context *ac) 3568 { 3569 struct zoneref *z; 3570 struct zone *zone; 3571 struct pglist_data *last_pgdat_dirty_limit = NULL; 3572 bool no_fallback; 3573 3574 retry: 3575 /* 3576 * Scan zonelist, looking for a zone with enough free. 3577 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 3578 */ 3579 no_fallback = alloc_flags & ALLOC_NOFRAGMENT; 3580 z = ac->preferred_zoneref; 3581 for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3582 ac->nodemask) { 3583 struct page *page; 3584 unsigned long mark; 3585 3586 if (cpusets_enabled() && 3587 (alloc_flags & ALLOC_CPUSET) && 3588 !__cpuset_zone_allowed(zone, gfp_mask)) 3589 continue; 3590 /* 3591 * When allocating a page cache page for writing, we 3592 * want to get it from a node that is within its dirty 3593 * limit, such that no single node holds more than its 3594 * proportional share of globally allowed dirty pages. 3595 * The dirty limits take into account the node's 3596 * lowmem reserves and high watermark so that kswapd 3597 * should be able to balance it without having to 3598 * write pages from its LRU list. 3599 * 3600 * XXX: For now, allow allocations to potentially 3601 * exceed the per-node dirty limit in the slowpath 3602 * (spread_dirty_pages unset) before going into reclaim, 3603 * which is important when on a NUMA setup the allowed 3604 * nodes are together not big enough to reach the 3605 * global limit. The proper fix for these situations 3606 * will require awareness of nodes in the 3607 * dirty-throttling and the flusher threads. 3608 */ 3609 if (ac->spread_dirty_pages) { 3610 if (last_pgdat_dirty_limit == zone->zone_pgdat) 3611 continue; 3612 3613 if (!node_dirty_ok(zone->zone_pgdat)) { 3614 last_pgdat_dirty_limit = zone->zone_pgdat; 3615 continue; 3616 } 3617 } 3618 3619 if (no_fallback && nr_online_nodes > 1 && 3620 zone != ac->preferred_zoneref->zone) { 3621 int local_nid; 3622 3623 /* 3624 * If moving to a remote node, retry but allow 3625 * fragmenting fallbacks. Locality is more important 3626 * than fragmentation avoidance. 3627 */ 3628 local_nid = zone_to_nid(ac->preferred_zoneref->zone); 3629 if (zone_to_nid(zone) != local_nid) { 3630 alloc_flags &= ~ALLOC_NOFRAGMENT; 3631 goto retry; 3632 } 3633 } 3634 3635 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); 3636 if (!zone_watermark_fast(zone, order, mark, 3637 ac_classzone_idx(ac), alloc_flags)) { 3638 int ret; 3639 3640 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3641 /* 3642 * Watermark failed for this zone, but see if we can 3643 * grow this zone if it contains deferred pages. 3644 */ 3645 if (static_branch_unlikely(&deferred_pages)) { 3646 if (_deferred_grow_zone(zone, order)) 3647 goto try_this_zone; 3648 } 3649 #endif 3650 /* Checked here to keep the fast path fast */ 3651 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 3652 if (alloc_flags & ALLOC_NO_WATERMARKS) 3653 goto try_this_zone; 3654 3655 if (node_reclaim_mode == 0 || 3656 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 3657 continue; 3658 3659 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 3660 switch (ret) { 3661 case NODE_RECLAIM_NOSCAN: 3662 /* did not scan */ 3663 continue; 3664 case NODE_RECLAIM_FULL: 3665 /* scanned but unreclaimable */ 3666 continue; 3667 default: 3668 /* did we reclaim enough */ 3669 if (zone_watermark_ok(zone, order, mark, 3670 ac_classzone_idx(ac), alloc_flags)) 3671 goto try_this_zone; 3672 3673 continue; 3674 } 3675 } 3676 3677 try_this_zone: 3678 page = rmqueue(ac->preferred_zoneref->zone, zone, order, 3679 gfp_mask, alloc_flags, ac->migratetype); 3680 if (page) { 3681 prep_new_page(page, order, gfp_mask, alloc_flags); 3682 3683 /* 3684 * If this is a high-order atomic allocation then check 3685 * if the pageblock should be reserved for the future 3686 */ 3687 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 3688 reserve_highatomic_pageblock(page, zone, order); 3689 3690 return page; 3691 } else { 3692 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3693 /* Try again if zone has deferred pages */ 3694 if (static_branch_unlikely(&deferred_pages)) { 3695 if (_deferred_grow_zone(zone, order)) 3696 goto try_this_zone; 3697 } 3698 #endif 3699 } 3700 } 3701 3702 /* 3703 * It's possible on a UMA machine to get through all zones that are 3704 * fragmented. If avoiding fragmentation, reset and try again. 3705 */ 3706 if (no_fallback) { 3707 alloc_flags &= ~ALLOC_NOFRAGMENT; 3708 goto retry; 3709 } 3710 3711 return NULL; 3712 } 3713 3714 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) 3715 { 3716 unsigned int filter = SHOW_MEM_FILTER_NODES; 3717 static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1); 3718 3719 if (!__ratelimit(&show_mem_rs)) 3720 return; 3721 3722 /* 3723 * This documents exceptions given to allocations in certain 3724 * contexts that are allowed to allocate outside current's set 3725 * of allowed nodes. 3726 */ 3727 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3728 if (tsk_is_oom_victim(current) || 3729 (current->flags & (PF_MEMALLOC | PF_EXITING))) 3730 filter &= ~SHOW_MEM_FILTER_NODES; 3731 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 3732 filter &= ~SHOW_MEM_FILTER_NODES; 3733 3734 show_mem(filter, nodemask); 3735 } 3736 3737 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) 3738 { 3739 struct va_format vaf; 3740 va_list args; 3741 static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL, 3742 DEFAULT_RATELIMIT_BURST); 3743 3744 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs)) 3745 return; 3746 3747 va_start(args, fmt); 3748 vaf.fmt = fmt; 3749 vaf.va = &args; 3750 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl", 3751 current->comm, &vaf, gfp_mask, &gfp_mask, 3752 nodemask_pr_args(nodemask)); 3753 va_end(args); 3754 3755 cpuset_print_current_mems_allowed(); 3756 pr_cont("\n"); 3757 dump_stack(); 3758 warn_alloc_show_mem(gfp_mask, nodemask); 3759 } 3760 3761 static inline struct page * 3762 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, 3763 unsigned int alloc_flags, 3764 const struct alloc_context *ac) 3765 { 3766 struct page *page; 3767 3768 page = get_page_from_freelist(gfp_mask, order, 3769 alloc_flags|ALLOC_CPUSET, ac); 3770 /* 3771 * fallback to ignore cpuset restriction if our nodes 3772 * are depleted 3773 */ 3774 if (!page) 3775 page = get_page_from_freelist(gfp_mask, order, 3776 alloc_flags, ac); 3777 3778 return page; 3779 } 3780 3781 static inline struct page * 3782 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 3783 const struct alloc_context *ac, unsigned long *did_some_progress) 3784 { 3785 struct oom_control oc = { 3786 .zonelist = ac->zonelist, 3787 .nodemask = ac->nodemask, 3788 .memcg = NULL, 3789 .gfp_mask = gfp_mask, 3790 .order = order, 3791 }; 3792 struct page *page; 3793 3794 *did_some_progress = 0; 3795 3796 /* 3797 * Acquire the oom lock. If that fails, somebody else is 3798 * making progress for us. 3799 */ 3800 if (!mutex_trylock(&oom_lock)) { 3801 *did_some_progress = 1; 3802 schedule_timeout_uninterruptible(1); 3803 return NULL; 3804 } 3805 3806 /* 3807 * Go through the zonelist yet one more time, keep very high watermark 3808 * here, this is only to catch a parallel oom killing, we must fail if 3809 * we're still under heavy pressure. But make sure that this reclaim 3810 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY 3811 * allocation which will never fail due to oom_lock already held. 3812 */ 3813 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & 3814 ~__GFP_DIRECT_RECLAIM, order, 3815 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 3816 if (page) 3817 goto out; 3818 3819 /* Coredumps can quickly deplete all memory reserves */ 3820 if (current->flags & PF_DUMPCORE) 3821 goto out; 3822 /* The OOM killer will not help higher order allocs */ 3823 if (order > PAGE_ALLOC_COSTLY_ORDER) 3824 goto out; 3825 /* 3826 * We have already exhausted all our reclaim opportunities without any 3827 * success so it is time to admit defeat. We will skip the OOM killer 3828 * because it is very likely that the caller has a more reasonable 3829 * fallback than shooting a random task. 3830 */ 3831 if (gfp_mask & __GFP_RETRY_MAYFAIL) 3832 goto out; 3833 /* The OOM killer does not needlessly kill tasks for lowmem */ 3834 if (ac->high_zoneidx < ZONE_NORMAL) 3835 goto out; 3836 if (pm_suspended_storage()) 3837 goto out; 3838 /* 3839 * XXX: GFP_NOFS allocations should rather fail than rely on 3840 * other request to make a forward progress. 3841 * We are in an unfortunate situation where out_of_memory cannot 3842 * do much for this context but let's try it to at least get 3843 * access to memory reserved if the current task is killed (see 3844 * out_of_memory). Once filesystems are ready to handle allocation 3845 * failures more gracefully we should just bail out here. 3846 */ 3847 3848 /* The OOM killer may not free memory on a specific node */ 3849 if (gfp_mask & __GFP_THISNODE) 3850 goto out; 3851 3852 /* Exhausted what can be done so it's blame time */ 3853 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 3854 *did_some_progress = 1; 3855 3856 /* 3857 * Help non-failing allocations by giving them access to memory 3858 * reserves 3859 */ 3860 if (gfp_mask & __GFP_NOFAIL) 3861 page = __alloc_pages_cpuset_fallback(gfp_mask, order, 3862 ALLOC_NO_WATERMARKS, ac); 3863 } 3864 out: 3865 mutex_unlock(&oom_lock); 3866 return page; 3867 } 3868 3869 /* 3870 * Maximum number of compaction retries wit a progress before OOM 3871 * killer is consider as the only way to move forward. 3872 */ 3873 #define MAX_COMPACT_RETRIES 16 3874 3875 #ifdef CONFIG_COMPACTION 3876 /* Try memory compaction for high-order allocations before reclaim */ 3877 static struct page * 3878 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3879 unsigned int alloc_flags, const struct alloc_context *ac, 3880 enum compact_priority prio, enum compact_result *compact_result) 3881 { 3882 struct page *page = NULL; 3883 unsigned long pflags; 3884 unsigned int noreclaim_flag; 3885 3886 if (!order) 3887 return NULL; 3888 3889 psi_memstall_enter(&pflags); 3890 noreclaim_flag = memalloc_noreclaim_save(); 3891 3892 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 3893 prio, &page); 3894 3895 memalloc_noreclaim_restore(noreclaim_flag); 3896 psi_memstall_leave(&pflags); 3897 3898 /* 3899 * At least in one zone compaction wasn't deferred or skipped, so let's 3900 * count a compaction stall 3901 */ 3902 count_vm_event(COMPACTSTALL); 3903 3904 /* Prep a captured page if available */ 3905 if (page) 3906 prep_new_page(page, order, gfp_mask, alloc_flags); 3907 3908 /* Try get a page from the freelist if available */ 3909 if (!page) 3910 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 3911 3912 if (page) { 3913 struct zone *zone = page_zone(page); 3914 3915 zone->compact_blockskip_flush = false; 3916 compaction_defer_reset(zone, order, true); 3917 count_vm_event(COMPACTSUCCESS); 3918 return page; 3919 } 3920 3921 /* 3922 * It's bad if compaction run occurs and fails. The most likely reason 3923 * is that pages exist, but not enough to satisfy watermarks. 3924 */ 3925 count_vm_event(COMPACTFAIL); 3926 3927 cond_resched(); 3928 3929 return NULL; 3930 } 3931 3932 static inline bool 3933 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 3934 enum compact_result compact_result, 3935 enum compact_priority *compact_priority, 3936 int *compaction_retries) 3937 { 3938 int max_retries = MAX_COMPACT_RETRIES; 3939 int min_priority; 3940 bool ret = false; 3941 int retries = *compaction_retries; 3942 enum compact_priority priority = *compact_priority; 3943 3944 if (!order) 3945 return false; 3946 3947 if (compaction_made_progress(compact_result)) 3948 (*compaction_retries)++; 3949 3950 /* 3951 * compaction considers all the zone as desperately out of memory 3952 * so it doesn't really make much sense to retry except when the 3953 * failure could be caused by insufficient priority 3954 */ 3955 if (compaction_failed(compact_result)) 3956 goto check_priority; 3957 3958 /* 3959 * compaction was skipped because there are not enough order-0 pages 3960 * to work with, so we retry only if it looks like reclaim can help. 3961 */ 3962 if (compaction_needs_reclaim(compact_result)) { 3963 ret = compaction_zonelist_suitable(ac, order, alloc_flags); 3964 goto out; 3965 } 3966 3967 /* 3968 * make sure the compaction wasn't deferred or didn't bail out early 3969 * due to locks contention before we declare that we should give up. 3970 * But the next retry should use a higher priority if allowed, so 3971 * we don't just keep bailing out endlessly. 3972 */ 3973 if (compaction_withdrawn(compact_result)) { 3974 goto check_priority; 3975 } 3976 3977 /* 3978 * !costly requests are much more important than __GFP_RETRY_MAYFAIL 3979 * costly ones because they are de facto nofail and invoke OOM 3980 * killer to move on while costly can fail and users are ready 3981 * to cope with that. 1/4 retries is rather arbitrary but we 3982 * would need much more detailed feedback from compaction to 3983 * make a better decision. 3984 */ 3985 if (order > PAGE_ALLOC_COSTLY_ORDER) 3986 max_retries /= 4; 3987 if (*compaction_retries <= max_retries) { 3988 ret = true; 3989 goto out; 3990 } 3991 3992 /* 3993 * Make sure there are attempts at the highest priority if we exhausted 3994 * all retries or failed at the lower priorities. 3995 */ 3996 check_priority: 3997 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 3998 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 3999 4000 if (*compact_priority > min_priority) { 4001 (*compact_priority)--; 4002 *compaction_retries = 0; 4003 ret = true; 4004 } 4005 out: 4006 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret); 4007 return ret; 4008 } 4009 #else 4010 static inline struct page * 4011 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4012 unsigned int alloc_flags, const struct alloc_context *ac, 4013 enum compact_priority prio, enum compact_result *compact_result) 4014 { 4015 *compact_result = COMPACT_SKIPPED; 4016 return NULL; 4017 } 4018 4019 static inline bool 4020 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 4021 enum compact_result compact_result, 4022 enum compact_priority *compact_priority, 4023 int *compaction_retries) 4024 { 4025 struct zone *zone; 4026 struct zoneref *z; 4027 4028 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 4029 return false; 4030 4031 /* 4032 * There are setups with compaction disabled which would prefer to loop 4033 * inside the allocator rather than hit the oom killer prematurely. 4034 * Let's give them a good hope and keep retrying while the order-0 4035 * watermarks are OK. 4036 */ 4037 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 4038 ac->nodemask) { 4039 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 4040 ac_classzone_idx(ac), alloc_flags)) 4041 return true; 4042 } 4043 return false; 4044 } 4045 #endif /* CONFIG_COMPACTION */ 4046 4047 #ifdef CONFIG_LOCKDEP 4048 static struct lockdep_map __fs_reclaim_map = 4049 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map); 4050 4051 static bool __need_fs_reclaim(gfp_t gfp_mask) 4052 { 4053 gfp_mask = current_gfp_context(gfp_mask); 4054 4055 /* no reclaim without waiting on it */ 4056 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) 4057 return false; 4058 4059 /* this guy won't enter reclaim */ 4060 if (current->flags & PF_MEMALLOC) 4061 return false; 4062 4063 /* We're only interested __GFP_FS allocations for now */ 4064 if (!(gfp_mask & __GFP_FS)) 4065 return false; 4066 4067 if (gfp_mask & __GFP_NOLOCKDEP) 4068 return false; 4069 4070 return true; 4071 } 4072 4073 void __fs_reclaim_acquire(void) 4074 { 4075 lock_map_acquire(&__fs_reclaim_map); 4076 } 4077 4078 void __fs_reclaim_release(void) 4079 { 4080 lock_map_release(&__fs_reclaim_map); 4081 } 4082 4083 void fs_reclaim_acquire(gfp_t gfp_mask) 4084 { 4085 if (__need_fs_reclaim(gfp_mask)) 4086 __fs_reclaim_acquire(); 4087 } 4088 EXPORT_SYMBOL_GPL(fs_reclaim_acquire); 4089 4090 void fs_reclaim_release(gfp_t gfp_mask) 4091 { 4092 if (__need_fs_reclaim(gfp_mask)) 4093 __fs_reclaim_release(); 4094 } 4095 EXPORT_SYMBOL_GPL(fs_reclaim_release); 4096 #endif 4097 4098 /* Perform direct synchronous page reclaim */ 4099 static int 4100 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 4101 const struct alloc_context *ac) 4102 { 4103 int progress; 4104 unsigned int noreclaim_flag; 4105 unsigned long pflags; 4106 4107 cond_resched(); 4108 4109 /* We now go into synchronous reclaim */ 4110 cpuset_memory_pressure_bump(); 4111 psi_memstall_enter(&pflags); 4112 fs_reclaim_acquire(gfp_mask); 4113 noreclaim_flag = memalloc_noreclaim_save(); 4114 4115 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 4116 ac->nodemask); 4117 4118 memalloc_noreclaim_restore(noreclaim_flag); 4119 fs_reclaim_release(gfp_mask); 4120 psi_memstall_leave(&pflags); 4121 4122 cond_resched(); 4123 4124 return progress; 4125 } 4126 4127 /* The really slow allocator path where we enter direct reclaim */ 4128 static inline struct page * 4129 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 4130 unsigned int alloc_flags, const struct alloc_context *ac, 4131 unsigned long *did_some_progress) 4132 { 4133 struct page *page = NULL; 4134 bool drained = false; 4135 4136 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 4137 if (unlikely(!(*did_some_progress))) 4138 return NULL; 4139 4140 retry: 4141 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4142 4143 /* 4144 * If an allocation failed after direct reclaim, it could be because 4145 * pages are pinned on the per-cpu lists or in high alloc reserves. 4146 * Shrink them them and try again 4147 */ 4148 if (!page && !drained) { 4149 unreserve_highatomic_pageblock(ac, false); 4150 drain_all_pages(NULL); 4151 drained = true; 4152 goto retry; 4153 } 4154 4155 return page; 4156 } 4157 4158 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, 4159 const struct alloc_context *ac) 4160 { 4161 struct zoneref *z; 4162 struct zone *zone; 4163 pg_data_t *last_pgdat = NULL; 4164 enum zone_type high_zoneidx = ac->high_zoneidx; 4165 4166 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, high_zoneidx, 4167 ac->nodemask) { 4168 if (last_pgdat != zone->zone_pgdat) 4169 wakeup_kswapd(zone, gfp_mask, order, high_zoneidx); 4170 last_pgdat = zone->zone_pgdat; 4171 } 4172 } 4173 4174 static inline unsigned int 4175 gfp_to_alloc_flags(gfp_t gfp_mask) 4176 { 4177 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 4178 4179 /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */ 4180 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 4181 4182 /* 4183 * The caller may dip into page reserves a bit more if the caller 4184 * cannot run direct reclaim, or if the caller has realtime scheduling 4185 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 4186 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 4187 */ 4188 alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); 4189 4190 if (gfp_mask & __GFP_ATOMIC) { 4191 /* 4192 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 4193 * if it can't schedule. 4194 */ 4195 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4196 alloc_flags |= ALLOC_HARDER; 4197 /* 4198 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 4199 * comment for __cpuset_node_allowed(). 4200 */ 4201 alloc_flags &= ~ALLOC_CPUSET; 4202 } else if (unlikely(rt_task(current)) && !in_interrupt()) 4203 alloc_flags |= ALLOC_HARDER; 4204 4205 if (gfp_mask & __GFP_KSWAPD_RECLAIM) 4206 alloc_flags |= ALLOC_KSWAPD; 4207 4208 #ifdef CONFIG_CMA 4209 if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) 4210 alloc_flags |= ALLOC_CMA; 4211 #endif 4212 return alloc_flags; 4213 } 4214 4215 static bool oom_reserves_allowed(struct task_struct *tsk) 4216 { 4217 if (!tsk_is_oom_victim(tsk)) 4218 return false; 4219 4220 /* 4221 * !MMU doesn't have oom reaper so give access to memory reserves 4222 * only to the thread with TIF_MEMDIE set 4223 */ 4224 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE)) 4225 return false; 4226 4227 return true; 4228 } 4229 4230 /* 4231 * Distinguish requests which really need access to full memory 4232 * reserves from oom victims which can live with a portion of it 4233 */ 4234 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) 4235 { 4236 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 4237 return 0; 4238 if (gfp_mask & __GFP_MEMALLOC) 4239 return ALLOC_NO_WATERMARKS; 4240 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 4241 return ALLOC_NO_WATERMARKS; 4242 if (!in_interrupt()) { 4243 if (current->flags & PF_MEMALLOC) 4244 return ALLOC_NO_WATERMARKS; 4245 else if (oom_reserves_allowed(current)) 4246 return ALLOC_OOM; 4247 } 4248 4249 return 0; 4250 } 4251 4252 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 4253 { 4254 return !!__gfp_pfmemalloc_flags(gfp_mask); 4255 } 4256 4257 /* 4258 * Checks whether it makes sense to retry the reclaim to make a forward progress 4259 * for the given allocation request. 4260 * 4261 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row 4262 * without success, or when we couldn't even meet the watermark if we 4263 * reclaimed all remaining pages on the LRU lists. 4264 * 4265 * Returns true if a retry is viable or false to enter the oom path. 4266 */ 4267 static inline bool 4268 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 4269 struct alloc_context *ac, int alloc_flags, 4270 bool did_some_progress, int *no_progress_loops) 4271 { 4272 struct zone *zone; 4273 struct zoneref *z; 4274 bool ret = false; 4275 4276 /* 4277 * Costly allocations might have made a progress but this doesn't mean 4278 * their order will become available due to high fragmentation so 4279 * always increment the no progress counter for them 4280 */ 4281 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 4282 *no_progress_loops = 0; 4283 else 4284 (*no_progress_loops)++; 4285 4286 /* 4287 * Make sure we converge to OOM if we cannot make any progress 4288 * several times in the row. 4289 */ 4290 if (*no_progress_loops > MAX_RECLAIM_RETRIES) { 4291 /* Before OOM, exhaust highatomic_reserve */ 4292 return unreserve_highatomic_pageblock(ac, true); 4293 } 4294 4295 /* 4296 * Keep reclaiming pages while there is a chance this will lead 4297 * somewhere. If none of the target zones can satisfy our allocation 4298 * request even if all reclaimable pages are considered then we are 4299 * screwed and have to go OOM. 4300 */ 4301 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 4302 ac->nodemask) { 4303 unsigned long available; 4304 unsigned long reclaimable; 4305 unsigned long min_wmark = min_wmark_pages(zone); 4306 bool wmark; 4307 4308 available = reclaimable = zone_reclaimable_pages(zone); 4309 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 4310 4311 /* 4312 * Would the allocation succeed if we reclaimed all 4313 * reclaimable pages? 4314 */ 4315 wmark = __zone_watermark_ok(zone, order, min_wmark, 4316 ac_classzone_idx(ac), alloc_flags, available); 4317 trace_reclaim_retry_zone(z, order, reclaimable, 4318 available, min_wmark, *no_progress_loops, wmark); 4319 if (wmark) { 4320 /* 4321 * If we didn't make any progress and have a lot of 4322 * dirty + writeback pages then we should wait for 4323 * an IO to complete to slow down the reclaim and 4324 * prevent from pre mature OOM 4325 */ 4326 if (!did_some_progress) { 4327 unsigned long write_pending; 4328 4329 write_pending = zone_page_state_snapshot(zone, 4330 NR_ZONE_WRITE_PENDING); 4331 4332 if (2 * write_pending > reclaimable) { 4333 congestion_wait(BLK_RW_ASYNC, HZ/10); 4334 return true; 4335 } 4336 } 4337 4338 ret = true; 4339 goto out; 4340 } 4341 } 4342 4343 out: 4344 /* 4345 * Memory allocation/reclaim might be called from a WQ context and the 4346 * current implementation of the WQ concurrency control doesn't 4347 * recognize that a particular WQ is congested if the worker thread is 4348 * looping without ever sleeping. Therefore we have to do a short sleep 4349 * here rather than calling cond_resched(). 4350 */ 4351 if (current->flags & PF_WQ_WORKER) 4352 schedule_timeout_uninterruptible(1); 4353 else 4354 cond_resched(); 4355 return ret; 4356 } 4357 4358 static inline bool 4359 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) 4360 { 4361 /* 4362 * It's possible that cpuset's mems_allowed and the nodemask from 4363 * mempolicy don't intersect. This should be normally dealt with by 4364 * policy_nodemask(), but it's possible to race with cpuset update in 4365 * such a way the check therein was true, and then it became false 4366 * before we got our cpuset_mems_cookie here. 4367 * This assumes that for all allocations, ac->nodemask can come only 4368 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored 4369 * when it does not intersect with the cpuset restrictions) or the 4370 * caller can deal with a violated nodemask. 4371 */ 4372 if (cpusets_enabled() && ac->nodemask && 4373 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) { 4374 ac->nodemask = NULL; 4375 return true; 4376 } 4377 4378 /* 4379 * When updating a task's mems_allowed or mempolicy nodemask, it is 4380 * possible to race with parallel threads in such a way that our 4381 * allocation can fail while the mask is being updated. If we are about 4382 * to fail, check if the cpuset changed during allocation and if so, 4383 * retry. 4384 */ 4385 if (read_mems_allowed_retry(cpuset_mems_cookie)) 4386 return true; 4387 4388 return false; 4389 } 4390 4391 static inline struct page * 4392 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 4393 struct alloc_context *ac) 4394 { 4395 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 4396 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 4397 struct page *page = NULL; 4398 unsigned int alloc_flags; 4399 unsigned long did_some_progress; 4400 enum compact_priority compact_priority; 4401 enum compact_result compact_result; 4402 int compaction_retries; 4403 int no_progress_loops; 4404 unsigned int cpuset_mems_cookie; 4405 int reserve_flags; 4406 4407 /* 4408 * We also sanity check to catch abuse of atomic reserves being used by 4409 * callers that are not in atomic context. 4410 */ 4411 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 4412 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 4413 gfp_mask &= ~__GFP_ATOMIC; 4414 4415 retry_cpuset: 4416 compaction_retries = 0; 4417 no_progress_loops = 0; 4418 compact_priority = DEF_COMPACT_PRIORITY; 4419 cpuset_mems_cookie = read_mems_allowed_begin(); 4420 4421 /* 4422 * The fast path uses conservative alloc_flags to succeed only until 4423 * kswapd needs to be woken up, and to avoid the cost of setting up 4424 * alloc_flags precisely. So we do that now. 4425 */ 4426 alloc_flags = gfp_to_alloc_flags(gfp_mask); 4427 4428 /* 4429 * We need to recalculate the starting point for the zonelist iterator 4430 * because we might have used different nodemask in the fast path, or 4431 * there was a cpuset modification and we are retrying - otherwise we 4432 * could end up iterating over non-eligible zones endlessly. 4433 */ 4434 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4435 ac->high_zoneidx, ac->nodemask); 4436 if (!ac->preferred_zoneref->zone) 4437 goto nopage; 4438 4439 if (alloc_flags & ALLOC_KSWAPD) 4440 wake_all_kswapds(order, gfp_mask, ac); 4441 4442 /* 4443 * The adjusted alloc_flags might result in immediate success, so try 4444 * that first 4445 */ 4446 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4447 if (page) 4448 goto got_pg; 4449 4450 /* 4451 * For costly allocations, try direct compaction first, as it's likely 4452 * that we have enough base pages and don't need to reclaim. For non- 4453 * movable high-order allocations, do that as well, as compaction will 4454 * try prevent permanent fragmentation by migrating from blocks of the 4455 * same migratetype. 4456 * Don't try this for allocations that are allowed to ignore 4457 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 4458 */ 4459 if (can_direct_reclaim && 4460 (costly_order || 4461 (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 4462 && !gfp_pfmemalloc_allowed(gfp_mask)) { 4463 page = __alloc_pages_direct_compact(gfp_mask, order, 4464 alloc_flags, ac, 4465 INIT_COMPACT_PRIORITY, 4466 &compact_result); 4467 if (page) 4468 goto got_pg; 4469 4470 /* 4471 * Checks for costly allocations with __GFP_NORETRY, which 4472 * includes THP page fault allocations 4473 */ 4474 if (costly_order && (gfp_mask & __GFP_NORETRY)) { 4475 /* 4476 * If compaction is deferred for high-order allocations, 4477 * it is because sync compaction recently failed. If 4478 * this is the case and the caller requested a THP 4479 * allocation, we do not want to heavily disrupt the 4480 * system, so we fail the allocation instead of entering 4481 * direct reclaim. 4482 */ 4483 if (compact_result == COMPACT_DEFERRED) 4484 goto nopage; 4485 4486 /* 4487 * Looks like reclaim/compaction is worth trying, but 4488 * sync compaction could be very expensive, so keep 4489 * using async compaction. 4490 */ 4491 compact_priority = INIT_COMPACT_PRIORITY; 4492 } 4493 } 4494 4495 retry: 4496 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 4497 if (alloc_flags & ALLOC_KSWAPD) 4498 wake_all_kswapds(order, gfp_mask, ac); 4499 4500 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); 4501 if (reserve_flags) 4502 alloc_flags = reserve_flags; 4503 4504 /* 4505 * Reset the nodemask and zonelist iterators if memory policies can be 4506 * ignored. These allocations are high priority and system rather than 4507 * user oriented. 4508 */ 4509 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { 4510 ac->nodemask = NULL; 4511 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4512 ac->high_zoneidx, ac->nodemask); 4513 } 4514 4515 /* Attempt with potentially adjusted zonelist and alloc_flags */ 4516 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4517 if (page) 4518 goto got_pg; 4519 4520 /* Caller is not willing to reclaim, we can't balance anything */ 4521 if (!can_direct_reclaim) 4522 goto nopage; 4523 4524 /* Avoid recursion of direct reclaim */ 4525 if (current->flags & PF_MEMALLOC) 4526 goto nopage; 4527 4528 /* Try direct reclaim and then allocating */ 4529 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 4530 &did_some_progress); 4531 if (page) 4532 goto got_pg; 4533 4534 /* Try direct compaction and then allocating */ 4535 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 4536 compact_priority, &compact_result); 4537 if (page) 4538 goto got_pg; 4539 4540 /* Do not loop if specifically requested */ 4541 if (gfp_mask & __GFP_NORETRY) 4542 goto nopage; 4543 4544 /* 4545 * Do not retry costly high order allocations unless they are 4546 * __GFP_RETRY_MAYFAIL 4547 */ 4548 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) 4549 goto nopage; 4550 4551 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 4552 did_some_progress > 0, &no_progress_loops)) 4553 goto retry; 4554 4555 /* 4556 * It doesn't make any sense to retry for the compaction if the order-0 4557 * reclaim is not able to make any progress because the current 4558 * implementation of the compaction depends on the sufficient amount 4559 * of free memory (see __compaction_suitable) 4560 */ 4561 if (did_some_progress > 0 && 4562 should_compact_retry(ac, order, alloc_flags, 4563 compact_result, &compact_priority, 4564 &compaction_retries)) 4565 goto retry; 4566 4567 4568 /* Deal with possible cpuset update races before we start OOM killing */ 4569 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4570 goto retry_cpuset; 4571 4572 /* Reclaim has failed us, start killing things */ 4573 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 4574 if (page) 4575 goto got_pg; 4576 4577 /* Avoid allocations with no watermarks from looping endlessly */ 4578 if (tsk_is_oom_victim(current) && 4579 (alloc_flags == ALLOC_OOM || 4580 (gfp_mask & __GFP_NOMEMALLOC))) 4581 goto nopage; 4582 4583 /* Retry as long as the OOM killer is making progress */ 4584 if (did_some_progress) { 4585 no_progress_loops = 0; 4586 goto retry; 4587 } 4588 4589 nopage: 4590 /* Deal with possible cpuset update races before we fail */ 4591 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4592 goto retry_cpuset; 4593 4594 /* 4595 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure 4596 * we always retry 4597 */ 4598 if (gfp_mask & __GFP_NOFAIL) { 4599 /* 4600 * All existing users of the __GFP_NOFAIL are blockable, so warn 4601 * of any new users that actually require GFP_NOWAIT 4602 */ 4603 if (WARN_ON_ONCE(!can_direct_reclaim)) 4604 goto fail; 4605 4606 /* 4607 * PF_MEMALLOC request from this context is rather bizarre 4608 * because we cannot reclaim anything and only can loop waiting 4609 * for somebody to do a work for us 4610 */ 4611 WARN_ON_ONCE(current->flags & PF_MEMALLOC); 4612 4613 /* 4614 * non failing costly orders are a hard requirement which we 4615 * are not prepared for much so let's warn about these users 4616 * so that we can identify them and convert them to something 4617 * else. 4618 */ 4619 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER); 4620 4621 /* 4622 * Help non-failing allocations by giving them access to memory 4623 * reserves but do not use ALLOC_NO_WATERMARKS because this 4624 * could deplete whole memory reserves which would just make 4625 * the situation worse 4626 */ 4627 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); 4628 if (page) 4629 goto got_pg; 4630 4631 cond_resched(); 4632 goto retry; 4633 } 4634 fail: 4635 warn_alloc(gfp_mask, ac->nodemask, 4636 "page allocation failure: order:%u", order); 4637 got_pg: 4638 return page; 4639 } 4640 4641 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, 4642 int preferred_nid, nodemask_t *nodemask, 4643 struct alloc_context *ac, gfp_t *alloc_mask, 4644 unsigned int *alloc_flags) 4645 { 4646 ac->high_zoneidx = gfp_zone(gfp_mask); 4647 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); 4648 ac->nodemask = nodemask; 4649 ac->migratetype = gfpflags_to_migratetype(gfp_mask); 4650 4651 if (cpusets_enabled()) { 4652 *alloc_mask |= __GFP_HARDWALL; 4653 if (!ac->nodemask) 4654 ac->nodemask = &cpuset_current_mems_allowed; 4655 else 4656 *alloc_flags |= ALLOC_CPUSET; 4657 } 4658 4659 fs_reclaim_acquire(gfp_mask); 4660 fs_reclaim_release(gfp_mask); 4661 4662 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 4663 4664 if (should_fail_alloc_page(gfp_mask, order)) 4665 return false; 4666 4667 if (IS_ENABLED(CONFIG_CMA) && ac->migratetype == MIGRATE_MOVABLE) 4668 *alloc_flags |= ALLOC_CMA; 4669 4670 return true; 4671 } 4672 4673 /* Determine whether to spread dirty pages and what the first usable zone */ 4674 static inline void finalise_ac(gfp_t gfp_mask, struct alloc_context *ac) 4675 { 4676 /* Dirty zone balancing only done in the fast path */ 4677 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); 4678 4679 /* 4680 * The preferred zone is used for statistics but crucially it is 4681 * also used as the starting point for the zonelist iterator. It 4682 * may get reset for allocations that ignore memory policies. 4683 */ 4684 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4685 ac->high_zoneidx, ac->nodemask); 4686 } 4687 4688 /* 4689 * This is the 'heart' of the zoned buddy allocator. 4690 */ 4691 struct page * 4692 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, 4693 nodemask_t *nodemask) 4694 { 4695 struct page *page; 4696 unsigned int alloc_flags = ALLOC_WMARK_LOW; 4697 gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */ 4698 struct alloc_context ac = { }; 4699 4700 /* 4701 * There are several places where we assume that the order value is sane 4702 * so bail out early if the request is out of bound. 4703 */ 4704 if (unlikely(order >= MAX_ORDER)) { 4705 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); 4706 return NULL; 4707 } 4708 4709 gfp_mask &= gfp_allowed_mask; 4710 alloc_mask = gfp_mask; 4711 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags)) 4712 return NULL; 4713 4714 finalise_ac(gfp_mask, &ac); 4715 4716 /* 4717 * Forbid the first pass from falling back to types that fragment 4718 * memory until all local zones are considered. 4719 */ 4720 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask); 4721 4722 /* First allocation attempt */ 4723 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); 4724 if (likely(page)) 4725 goto out; 4726 4727 /* 4728 * Apply scoped allocation constraints. This is mainly about GFP_NOFS 4729 * resp. GFP_NOIO which has to be inherited for all allocation requests 4730 * from a particular context which has been marked by 4731 * memalloc_no{fs,io}_{save,restore}. 4732 */ 4733 alloc_mask = current_gfp_context(gfp_mask); 4734 ac.spread_dirty_pages = false; 4735 4736 /* 4737 * Restore the original nodemask if it was potentially replaced with 4738 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 4739 */ 4740 if (unlikely(ac.nodemask != nodemask)) 4741 ac.nodemask = nodemask; 4742 4743 page = __alloc_pages_slowpath(alloc_mask, order, &ac); 4744 4745 out: 4746 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && 4747 unlikely(__memcg_kmem_charge(page, gfp_mask, order) != 0)) { 4748 __free_pages(page, order); 4749 page = NULL; 4750 } 4751 4752 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); 4753 4754 return page; 4755 } 4756 EXPORT_SYMBOL(__alloc_pages_nodemask); 4757 4758 /* 4759 * Common helper functions. Never use with __GFP_HIGHMEM because the returned 4760 * address cannot represent highmem pages. Use alloc_pages and then kmap if 4761 * you need to access high mem. 4762 */ 4763 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 4764 { 4765 struct page *page; 4766 4767 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); 4768 if (!page) 4769 return 0; 4770 return (unsigned long) page_address(page); 4771 } 4772 EXPORT_SYMBOL(__get_free_pages); 4773 4774 unsigned long get_zeroed_page(gfp_t gfp_mask) 4775 { 4776 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 4777 } 4778 EXPORT_SYMBOL(get_zeroed_page); 4779 4780 static inline void free_the_page(struct page *page, unsigned int order) 4781 { 4782 if (order == 0) /* Via pcp? */ 4783 free_unref_page(page); 4784 else 4785 __free_pages_ok(page, order); 4786 } 4787 4788 void __free_pages(struct page *page, unsigned int order) 4789 { 4790 if (put_page_testzero(page)) 4791 free_the_page(page, order); 4792 } 4793 EXPORT_SYMBOL(__free_pages); 4794 4795 void free_pages(unsigned long addr, unsigned int order) 4796 { 4797 if (addr != 0) { 4798 VM_BUG_ON(!virt_addr_valid((void *)addr)); 4799 __free_pages(virt_to_page((void *)addr), order); 4800 } 4801 } 4802 4803 EXPORT_SYMBOL(free_pages); 4804 4805 /* 4806 * Page Fragment: 4807 * An arbitrary-length arbitrary-offset area of memory which resides 4808 * within a 0 or higher order page. Multiple fragments within that page 4809 * are individually refcounted, in the page's reference counter. 4810 * 4811 * The page_frag functions below provide a simple allocation framework for 4812 * page fragments. This is used by the network stack and network device 4813 * drivers to provide a backing region of memory for use as either an 4814 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 4815 */ 4816 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 4817 gfp_t gfp_mask) 4818 { 4819 struct page *page = NULL; 4820 gfp_t gfp = gfp_mask; 4821 4822 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4823 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 4824 __GFP_NOMEMALLOC; 4825 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 4826 PAGE_FRAG_CACHE_MAX_ORDER); 4827 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 4828 #endif 4829 if (unlikely(!page)) 4830 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 4831 4832 nc->va = page ? page_address(page) : NULL; 4833 4834 return page; 4835 } 4836 4837 void __page_frag_cache_drain(struct page *page, unsigned int count) 4838 { 4839 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 4840 4841 if (page_ref_sub_and_test(page, count)) 4842 free_the_page(page, compound_order(page)); 4843 } 4844 EXPORT_SYMBOL(__page_frag_cache_drain); 4845 4846 void *page_frag_alloc(struct page_frag_cache *nc, 4847 unsigned int fragsz, gfp_t gfp_mask) 4848 { 4849 unsigned int size = PAGE_SIZE; 4850 struct page *page; 4851 int offset; 4852 4853 if (unlikely(!nc->va)) { 4854 refill: 4855 page = __page_frag_cache_refill(nc, gfp_mask); 4856 if (!page) 4857 return NULL; 4858 4859 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4860 /* if size can vary use size else just use PAGE_SIZE */ 4861 size = nc->size; 4862 #endif 4863 /* Even if we own the page, we do not use atomic_set(). 4864 * This would break get_page_unless_zero() users. 4865 */ 4866 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 4867 4868 /* reset page count bias and offset to start of new frag */ 4869 nc->pfmemalloc = page_is_pfmemalloc(page); 4870 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 4871 nc->offset = size; 4872 } 4873 4874 offset = nc->offset - fragsz; 4875 if (unlikely(offset < 0)) { 4876 page = virt_to_page(nc->va); 4877 4878 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 4879 goto refill; 4880 4881 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4882 /* if size can vary use size else just use PAGE_SIZE */ 4883 size = nc->size; 4884 #endif 4885 /* OK, page count is 0, we can safely set it */ 4886 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 4887 4888 /* reset page count bias and offset to start of new frag */ 4889 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 4890 offset = size - fragsz; 4891 } 4892 4893 nc->pagecnt_bias--; 4894 nc->offset = offset; 4895 4896 return nc->va + offset; 4897 } 4898 EXPORT_SYMBOL(page_frag_alloc); 4899 4900 /* 4901 * Frees a page fragment allocated out of either a compound or order 0 page. 4902 */ 4903 void page_frag_free(void *addr) 4904 { 4905 struct page *page = virt_to_head_page(addr); 4906 4907 if (unlikely(put_page_testzero(page))) 4908 free_the_page(page, compound_order(page)); 4909 } 4910 EXPORT_SYMBOL(page_frag_free); 4911 4912 static void *make_alloc_exact(unsigned long addr, unsigned int order, 4913 size_t size) 4914 { 4915 if (addr) { 4916 unsigned long alloc_end = addr + (PAGE_SIZE << order); 4917 unsigned long used = addr + PAGE_ALIGN(size); 4918 4919 split_page(virt_to_page((void *)addr), order); 4920 while (used < alloc_end) { 4921 free_page(used); 4922 used += PAGE_SIZE; 4923 } 4924 } 4925 return (void *)addr; 4926 } 4927 4928 /** 4929 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 4930 * @size: the number of bytes to allocate 4931 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 4932 * 4933 * This function is similar to alloc_pages(), except that it allocates the 4934 * minimum number of pages to satisfy the request. alloc_pages() can only 4935 * allocate memory in power-of-two pages. 4936 * 4937 * This function is also limited by MAX_ORDER. 4938 * 4939 * Memory allocated by this function must be released by free_pages_exact(). 4940 * 4941 * Return: pointer to the allocated area or %NULL in case of error. 4942 */ 4943 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 4944 { 4945 unsigned int order = get_order(size); 4946 unsigned long addr; 4947 4948 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 4949 gfp_mask &= ~__GFP_COMP; 4950 4951 addr = __get_free_pages(gfp_mask, order); 4952 return make_alloc_exact(addr, order, size); 4953 } 4954 EXPORT_SYMBOL(alloc_pages_exact); 4955 4956 /** 4957 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 4958 * pages on a node. 4959 * @nid: the preferred node ID where memory should be allocated 4960 * @size: the number of bytes to allocate 4961 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 4962 * 4963 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 4964 * back. 4965 * 4966 * Return: pointer to the allocated area or %NULL in case of error. 4967 */ 4968 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 4969 { 4970 unsigned int order = get_order(size); 4971 struct page *p; 4972 4973 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 4974 gfp_mask &= ~__GFP_COMP; 4975 4976 p = alloc_pages_node(nid, gfp_mask, order); 4977 if (!p) 4978 return NULL; 4979 return make_alloc_exact((unsigned long)page_address(p), order, size); 4980 } 4981 4982 /** 4983 * free_pages_exact - release memory allocated via alloc_pages_exact() 4984 * @virt: the value returned by alloc_pages_exact. 4985 * @size: size of allocation, same value as passed to alloc_pages_exact(). 4986 * 4987 * Release the memory allocated by a previous call to alloc_pages_exact. 4988 */ 4989 void free_pages_exact(void *virt, size_t size) 4990 { 4991 unsigned long addr = (unsigned long)virt; 4992 unsigned long end = addr + PAGE_ALIGN(size); 4993 4994 while (addr < end) { 4995 free_page(addr); 4996 addr += PAGE_SIZE; 4997 } 4998 } 4999 EXPORT_SYMBOL(free_pages_exact); 5000 5001 /** 5002 * nr_free_zone_pages - count number of pages beyond high watermark 5003 * @offset: The zone index of the highest zone 5004 * 5005 * nr_free_zone_pages() counts the number of pages which are beyond the 5006 * high watermark within all zones at or below a given zone index. For each 5007 * zone, the number of pages is calculated as: 5008 * 5009 * nr_free_zone_pages = managed_pages - high_pages 5010 * 5011 * Return: number of pages beyond high watermark. 5012 */ 5013 static unsigned long nr_free_zone_pages(int offset) 5014 { 5015 struct zoneref *z; 5016 struct zone *zone; 5017 5018 /* Just pick one node, since fallback list is circular */ 5019 unsigned long sum = 0; 5020 5021 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 5022 5023 for_each_zone_zonelist(zone, z, zonelist, offset) { 5024 unsigned long size = zone_managed_pages(zone); 5025 unsigned long high = high_wmark_pages(zone); 5026 if (size > high) 5027 sum += size - high; 5028 } 5029 5030 return sum; 5031 } 5032 5033 /** 5034 * nr_free_buffer_pages - count number of pages beyond high watermark 5035 * 5036 * nr_free_buffer_pages() counts the number of pages which are beyond the high 5037 * watermark within ZONE_DMA and ZONE_NORMAL. 5038 * 5039 * Return: number of pages beyond high watermark within ZONE_DMA and 5040 * ZONE_NORMAL. 5041 */ 5042 unsigned long nr_free_buffer_pages(void) 5043 { 5044 return nr_free_zone_pages(gfp_zone(GFP_USER)); 5045 } 5046 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 5047 5048 /** 5049 * nr_free_pagecache_pages - count number of pages beyond high watermark 5050 * 5051 * nr_free_pagecache_pages() counts the number of pages which are beyond the 5052 * high watermark within all zones. 5053 * 5054 * Return: number of pages beyond high watermark within all zones. 5055 */ 5056 unsigned long nr_free_pagecache_pages(void) 5057 { 5058 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 5059 } 5060 5061 static inline void show_node(struct zone *zone) 5062 { 5063 if (IS_ENABLED(CONFIG_NUMA)) 5064 printk("Node %d ", zone_to_nid(zone)); 5065 } 5066 5067 long si_mem_available(void) 5068 { 5069 long available; 5070 unsigned long pagecache; 5071 unsigned long wmark_low = 0; 5072 unsigned long pages[NR_LRU_LISTS]; 5073 unsigned long reclaimable; 5074 struct zone *zone; 5075 int lru; 5076 5077 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 5078 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 5079 5080 for_each_zone(zone) 5081 wmark_low += low_wmark_pages(zone); 5082 5083 /* 5084 * Estimate the amount of memory available for userspace allocations, 5085 * without causing swapping. 5086 */ 5087 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages; 5088 5089 /* 5090 * Not all the page cache can be freed, otherwise the system will 5091 * start swapping. Assume at least half of the page cache, or the 5092 * low watermark worth of cache, needs to stay. 5093 */ 5094 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 5095 pagecache -= min(pagecache / 2, wmark_low); 5096 available += pagecache; 5097 5098 /* 5099 * Part of the reclaimable slab and other kernel memory consists of 5100 * items that are in use, and cannot be freed. Cap this estimate at the 5101 * low watermark. 5102 */ 5103 reclaimable = global_node_page_state(NR_SLAB_RECLAIMABLE) + 5104 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE); 5105 available += reclaimable - min(reclaimable / 2, wmark_low); 5106 5107 if (available < 0) 5108 available = 0; 5109 return available; 5110 } 5111 EXPORT_SYMBOL_GPL(si_mem_available); 5112 5113 void si_meminfo(struct sysinfo *val) 5114 { 5115 val->totalram = totalram_pages(); 5116 val->sharedram = global_node_page_state(NR_SHMEM); 5117 val->freeram = global_zone_page_state(NR_FREE_PAGES); 5118 val->bufferram = nr_blockdev_pages(); 5119 val->totalhigh = totalhigh_pages(); 5120 val->freehigh = nr_free_highpages(); 5121 val->mem_unit = PAGE_SIZE; 5122 } 5123 5124 EXPORT_SYMBOL(si_meminfo); 5125 5126 #ifdef CONFIG_NUMA 5127 void si_meminfo_node(struct sysinfo *val, int nid) 5128 { 5129 int zone_type; /* needs to be signed */ 5130 unsigned long managed_pages = 0; 5131 unsigned long managed_highpages = 0; 5132 unsigned long free_highpages = 0; 5133 pg_data_t *pgdat = NODE_DATA(nid); 5134 5135 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 5136 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]); 5137 val->totalram = managed_pages; 5138 val->sharedram = node_page_state(pgdat, NR_SHMEM); 5139 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 5140 #ifdef CONFIG_HIGHMEM 5141 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 5142 struct zone *zone = &pgdat->node_zones[zone_type]; 5143 5144 if (is_highmem(zone)) { 5145 managed_highpages += zone_managed_pages(zone); 5146 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 5147 } 5148 } 5149 val->totalhigh = managed_highpages; 5150 val->freehigh = free_highpages; 5151 #else 5152 val->totalhigh = managed_highpages; 5153 val->freehigh = free_highpages; 5154 #endif 5155 val->mem_unit = PAGE_SIZE; 5156 } 5157 #endif 5158 5159 /* 5160 * Determine whether the node should be displayed or not, depending on whether 5161 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 5162 */ 5163 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) 5164 { 5165 if (!(flags & SHOW_MEM_FILTER_NODES)) 5166 return false; 5167 5168 /* 5169 * no node mask - aka implicit memory numa policy. Do not bother with 5170 * the synchronization - read_mems_allowed_begin - because we do not 5171 * have to be precise here. 5172 */ 5173 if (!nodemask) 5174 nodemask = &cpuset_current_mems_allowed; 5175 5176 return !node_isset(nid, *nodemask); 5177 } 5178 5179 #define K(x) ((x) << (PAGE_SHIFT-10)) 5180 5181 static void show_migration_types(unsigned char type) 5182 { 5183 static const char types[MIGRATE_TYPES] = { 5184 [MIGRATE_UNMOVABLE] = 'U', 5185 [MIGRATE_MOVABLE] = 'M', 5186 [MIGRATE_RECLAIMABLE] = 'E', 5187 [MIGRATE_HIGHATOMIC] = 'H', 5188 #ifdef CONFIG_CMA 5189 [MIGRATE_CMA] = 'C', 5190 #endif 5191 #ifdef CONFIG_MEMORY_ISOLATION 5192 [MIGRATE_ISOLATE] = 'I', 5193 #endif 5194 }; 5195 char tmp[MIGRATE_TYPES + 1]; 5196 char *p = tmp; 5197 int i; 5198 5199 for (i = 0; i < MIGRATE_TYPES; i++) { 5200 if (type & (1 << i)) 5201 *p++ = types[i]; 5202 } 5203 5204 *p = '\0'; 5205 printk(KERN_CONT "(%s) ", tmp); 5206 } 5207 5208 /* 5209 * Show free area list (used inside shift_scroll-lock stuff) 5210 * We also calculate the percentage fragmentation. We do this by counting the 5211 * memory on each free list with the exception of the first item on the list. 5212 * 5213 * Bits in @filter: 5214 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 5215 * cpuset. 5216 */ 5217 void show_free_areas(unsigned int filter, nodemask_t *nodemask) 5218 { 5219 unsigned long free_pcp = 0; 5220 int cpu; 5221 struct zone *zone; 5222 pg_data_t *pgdat; 5223 5224 for_each_populated_zone(zone) { 5225 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5226 continue; 5227 5228 for_each_online_cpu(cpu) 5229 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 5230 } 5231 5232 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 5233 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 5234 " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n" 5235 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 5236 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 5237 " free:%lu free_pcp:%lu free_cma:%lu\n", 5238 global_node_page_state(NR_ACTIVE_ANON), 5239 global_node_page_state(NR_INACTIVE_ANON), 5240 global_node_page_state(NR_ISOLATED_ANON), 5241 global_node_page_state(NR_ACTIVE_FILE), 5242 global_node_page_state(NR_INACTIVE_FILE), 5243 global_node_page_state(NR_ISOLATED_FILE), 5244 global_node_page_state(NR_UNEVICTABLE), 5245 global_node_page_state(NR_FILE_DIRTY), 5246 global_node_page_state(NR_WRITEBACK), 5247 global_node_page_state(NR_UNSTABLE_NFS), 5248 global_node_page_state(NR_SLAB_RECLAIMABLE), 5249 global_node_page_state(NR_SLAB_UNRECLAIMABLE), 5250 global_node_page_state(NR_FILE_MAPPED), 5251 global_node_page_state(NR_SHMEM), 5252 global_zone_page_state(NR_PAGETABLE), 5253 global_zone_page_state(NR_BOUNCE), 5254 global_zone_page_state(NR_FREE_PAGES), 5255 free_pcp, 5256 global_zone_page_state(NR_FREE_CMA_PAGES)); 5257 5258 for_each_online_pgdat(pgdat) { 5259 if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) 5260 continue; 5261 5262 printk("Node %d" 5263 " active_anon:%lukB" 5264 " inactive_anon:%lukB" 5265 " active_file:%lukB" 5266 " inactive_file:%lukB" 5267 " unevictable:%lukB" 5268 " isolated(anon):%lukB" 5269 " isolated(file):%lukB" 5270 " mapped:%lukB" 5271 " dirty:%lukB" 5272 " writeback:%lukB" 5273 " shmem:%lukB" 5274 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5275 " shmem_thp: %lukB" 5276 " shmem_pmdmapped: %lukB" 5277 " anon_thp: %lukB" 5278 #endif 5279 " writeback_tmp:%lukB" 5280 " unstable:%lukB" 5281 " all_unreclaimable? %s" 5282 "\n", 5283 pgdat->node_id, 5284 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 5285 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 5286 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 5287 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 5288 K(node_page_state(pgdat, NR_UNEVICTABLE)), 5289 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 5290 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 5291 K(node_page_state(pgdat, NR_FILE_MAPPED)), 5292 K(node_page_state(pgdat, NR_FILE_DIRTY)), 5293 K(node_page_state(pgdat, NR_WRITEBACK)), 5294 K(node_page_state(pgdat, NR_SHMEM)), 5295 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5296 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), 5297 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) 5298 * HPAGE_PMD_NR), 5299 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR), 5300 #endif 5301 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 5302 K(node_page_state(pgdat, NR_UNSTABLE_NFS)), 5303 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ? 5304 "yes" : "no"); 5305 } 5306 5307 for_each_populated_zone(zone) { 5308 int i; 5309 5310 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5311 continue; 5312 5313 free_pcp = 0; 5314 for_each_online_cpu(cpu) 5315 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 5316 5317 show_node(zone); 5318 printk(KERN_CONT 5319 "%s" 5320 " free:%lukB" 5321 " min:%lukB" 5322 " low:%lukB" 5323 " high:%lukB" 5324 " active_anon:%lukB" 5325 " inactive_anon:%lukB" 5326 " active_file:%lukB" 5327 " inactive_file:%lukB" 5328 " unevictable:%lukB" 5329 " writepending:%lukB" 5330 " present:%lukB" 5331 " managed:%lukB" 5332 " mlocked:%lukB" 5333 " kernel_stack:%lukB" 5334 " pagetables:%lukB" 5335 " bounce:%lukB" 5336 " free_pcp:%lukB" 5337 " local_pcp:%ukB" 5338 " free_cma:%lukB" 5339 "\n", 5340 zone->name, 5341 K(zone_page_state(zone, NR_FREE_PAGES)), 5342 K(min_wmark_pages(zone)), 5343 K(low_wmark_pages(zone)), 5344 K(high_wmark_pages(zone)), 5345 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 5346 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 5347 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 5348 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 5349 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 5350 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 5351 K(zone->present_pages), 5352 K(zone_managed_pages(zone)), 5353 K(zone_page_state(zone, NR_MLOCK)), 5354 zone_page_state(zone, NR_KERNEL_STACK_KB), 5355 K(zone_page_state(zone, NR_PAGETABLE)), 5356 K(zone_page_state(zone, NR_BOUNCE)), 5357 K(free_pcp), 5358 K(this_cpu_read(zone->pageset->pcp.count)), 5359 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 5360 printk("lowmem_reserve[]:"); 5361 for (i = 0; i < MAX_NR_ZONES; i++) 5362 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); 5363 printk(KERN_CONT "\n"); 5364 } 5365 5366 for_each_populated_zone(zone) { 5367 unsigned int order; 5368 unsigned long nr[MAX_ORDER], flags, total = 0; 5369 unsigned char types[MAX_ORDER]; 5370 5371 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5372 continue; 5373 show_node(zone); 5374 printk(KERN_CONT "%s: ", zone->name); 5375 5376 spin_lock_irqsave(&zone->lock, flags); 5377 for (order = 0; order < MAX_ORDER; order++) { 5378 struct free_area *area = &zone->free_area[order]; 5379 int type; 5380 5381 nr[order] = area->nr_free; 5382 total += nr[order] << order; 5383 5384 types[order] = 0; 5385 for (type = 0; type < MIGRATE_TYPES; type++) { 5386 if (!free_area_empty(area, type)) 5387 types[order] |= 1 << type; 5388 } 5389 } 5390 spin_unlock_irqrestore(&zone->lock, flags); 5391 for (order = 0; order < MAX_ORDER; order++) { 5392 printk(KERN_CONT "%lu*%lukB ", 5393 nr[order], K(1UL) << order); 5394 if (nr[order]) 5395 show_migration_types(types[order]); 5396 } 5397 printk(KERN_CONT "= %lukB\n", K(total)); 5398 } 5399 5400 hugetlb_show_meminfo(); 5401 5402 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 5403 5404 show_swap_cache_info(); 5405 } 5406 5407 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 5408 { 5409 zoneref->zone = zone; 5410 zoneref->zone_idx = zone_idx(zone); 5411 } 5412 5413 /* 5414 * Builds allocation fallback zone lists. 5415 * 5416 * Add all populated zones of a node to the zonelist. 5417 */ 5418 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) 5419 { 5420 struct zone *zone; 5421 enum zone_type zone_type = MAX_NR_ZONES; 5422 int nr_zones = 0; 5423 5424 do { 5425 zone_type--; 5426 zone = pgdat->node_zones + zone_type; 5427 if (managed_zone(zone)) { 5428 zoneref_set_zone(zone, &zonerefs[nr_zones++]); 5429 check_highest_zone(zone_type); 5430 } 5431 } while (zone_type); 5432 5433 return nr_zones; 5434 } 5435 5436 #ifdef CONFIG_NUMA 5437 5438 static int __parse_numa_zonelist_order(char *s) 5439 { 5440 /* 5441 * We used to support different zonlists modes but they turned 5442 * out to be just not useful. Let's keep the warning in place 5443 * if somebody still use the cmd line parameter so that we do 5444 * not fail it silently 5445 */ 5446 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) { 5447 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s); 5448 return -EINVAL; 5449 } 5450 return 0; 5451 } 5452 5453 static __init int setup_numa_zonelist_order(char *s) 5454 { 5455 if (!s) 5456 return 0; 5457 5458 return __parse_numa_zonelist_order(s); 5459 } 5460 early_param("numa_zonelist_order", setup_numa_zonelist_order); 5461 5462 char numa_zonelist_order[] = "Node"; 5463 5464 /* 5465 * sysctl handler for numa_zonelist_order 5466 */ 5467 int numa_zonelist_order_handler(struct ctl_table *table, int write, 5468 void __user *buffer, size_t *length, 5469 loff_t *ppos) 5470 { 5471 char *str; 5472 int ret; 5473 5474 if (!write) 5475 return proc_dostring(table, write, buffer, length, ppos); 5476 str = memdup_user_nul(buffer, 16); 5477 if (IS_ERR(str)) 5478 return PTR_ERR(str); 5479 5480 ret = __parse_numa_zonelist_order(str); 5481 kfree(str); 5482 return ret; 5483 } 5484 5485 5486 #define MAX_NODE_LOAD (nr_online_nodes) 5487 static int node_load[MAX_NUMNODES]; 5488 5489 /** 5490 * find_next_best_node - find the next node that should appear in a given node's fallback list 5491 * @node: node whose fallback list we're appending 5492 * @used_node_mask: nodemask_t of already used nodes 5493 * 5494 * We use a number of factors to determine which is the next node that should 5495 * appear on a given node's fallback list. The node should not have appeared 5496 * already in @node's fallback list, and it should be the next closest node 5497 * according to the distance array (which contains arbitrary distance values 5498 * from each node to each node in the system), and should also prefer nodes 5499 * with no CPUs, since presumably they'll have very little allocation pressure 5500 * on them otherwise. 5501 * 5502 * Return: node id of the found node or %NUMA_NO_NODE if no node is found. 5503 */ 5504 static int find_next_best_node(int node, nodemask_t *used_node_mask) 5505 { 5506 int n, val; 5507 int min_val = INT_MAX; 5508 int best_node = NUMA_NO_NODE; 5509 const struct cpumask *tmp = cpumask_of_node(0); 5510 5511 /* Use the local node if we haven't already */ 5512 if (!node_isset(node, *used_node_mask)) { 5513 node_set(node, *used_node_mask); 5514 return node; 5515 } 5516 5517 for_each_node_state(n, N_MEMORY) { 5518 5519 /* Don't want a node to appear more than once */ 5520 if (node_isset(n, *used_node_mask)) 5521 continue; 5522 5523 /* Use the distance array to find the distance */ 5524 val = node_distance(node, n); 5525 5526 /* Penalize nodes under us ("prefer the next node") */ 5527 val += (n < node); 5528 5529 /* Give preference to headless and unused nodes */ 5530 tmp = cpumask_of_node(n); 5531 if (!cpumask_empty(tmp)) 5532 val += PENALTY_FOR_NODE_WITH_CPUS; 5533 5534 /* Slight preference for less loaded node */ 5535 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 5536 val += node_load[n]; 5537 5538 if (val < min_val) { 5539 min_val = val; 5540 best_node = n; 5541 } 5542 } 5543 5544 if (best_node >= 0) 5545 node_set(best_node, *used_node_mask); 5546 5547 return best_node; 5548 } 5549 5550 5551 /* 5552 * Build zonelists ordered by node and zones within node. 5553 * This results in maximum locality--normal zone overflows into local 5554 * DMA zone, if any--but risks exhausting DMA zone. 5555 */ 5556 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, 5557 unsigned nr_nodes) 5558 { 5559 struct zoneref *zonerefs; 5560 int i; 5561 5562 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5563 5564 for (i = 0; i < nr_nodes; i++) { 5565 int nr_zones; 5566 5567 pg_data_t *node = NODE_DATA(node_order[i]); 5568 5569 nr_zones = build_zonerefs_node(node, zonerefs); 5570 zonerefs += nr_zones; 5571 } 5572 zonerefs->zone = NULL; 5573 zonerefs->zone_idx = 0; 5574 } 5575 5576 /* 5577 * Build gfp_thisnode zonelists 5578 */ 5579 static void build_thisnode_zonelists(pg_data_t *pgdat) 5580 { 5581 struct zoneref *zonerefs; 5582 int nr_zones; 5583 5584 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; 5585 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5586 zonerefs += nr_zones; 5587 zonerefs->zone = NULL; 5588 zonerefs->zone_idx = 0; 5589 } 5590 5591 /* 5592 * Build zonelists ordered by zone and nodes within zones. 5593 * This results in conserving DMA zone[s] until all Normal memory is 5594 * exhausted, but results in overflowing to remote node while memory 5595 * may still exist in local DMA zone. 5596 */ 5597 5598 static void build_zonelists(pg_data_t *pgdat) 5599 { 5600 static int node_order[MAX_NUMNODES]; 5601 int node, load, nr_nodes = 0; 5602 nodemask_t used_mask; 5603 int local_node, prev_node; 5604 5605 /* NUMA-aware ordering of nodes */ 5606 local_node = pgdat->node_id; 5607 load = nr_online_nodes; 5608 prev_node = local_node; 5609 nodes_clear(used_mask); 5610 5611 memset(node_order, 0, sizeof(node_order)); 5612 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 5613 /* 5614 * We don't want to pressure a particular node. 5615 * So adding penalty to the first node in same 5616 * distance group to make it round-robin. 5617 */ 5618 if (node_distance(local_node, node) != 5619 node_distance(local_node, prev_node)) 5620 node_load[node] = load; 5621 5622 node_order[nr_nodes++] = node; 5623 prev_node = node; 5624 load--; 5625 } 5626 5627 build_zonelists_in_node_order(pgdat, node_order, nr_nodes); 5628 build_thisnode_zonelists(pgdat); 5629 } 5630 5631 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5632 /* 5633 * Return node id of node used for "local" allocations. 5634 * I.e., first node id of first zone in arg node's generic zonelist. 5635 * Used for initializing percpu 'numa_mem', which is used primarily 5636 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 5637 */ 5638 int local_memory_node(int node) 5639 { 5640 struct zoneref *z; 5641 5642 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 5643 gfp_zone(GFP_KERNEL), 5644 NULL); 5645 return zone_to_nid(z->zone); 5646 } 5647 #endif 5648 5649 static void setup_min_unmapped_ratio(void); 5650 static void setup_min_slab_ratio(void); 5651 #else /* CONFIG_NUMA */ 5652 5653 static void build_zonelists(pg_data_t *pgdat) 5654 { 5655 int node, local_node; 5656 struct zoneref *zonerefs; 5657 int nr_zones; 5658 5659 local_node = pgdat->node_id; 5660 5661 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5662 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5663 zonerefs += nr_zones; 5664 5665 /* 5666 * Now we build the zonelist so that it contains the zones 5667 * of all the other nodes. 5668 * We don't want to pressure a particular node, so when 5669 * building the zones for node N, we make sure that the 5670 * zones coming right after the local ones are those from 5671 * node N+1 (modulo N) 5672 */ 5673 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 5674 if (!node_online(node)) 5675 continue; 5676 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5677 zonerefs += nr_zones; 5678 } 5679 for (node = 0; node < local_node; node++) { 5680 if (!node_online(node)) 5681 continue; 5682 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5683 zonerefs += nr_zones; 5684 } 5685 5686 zonerefs->zone = NULL; 5687 zonerefs->zone_idx = 0; 5688 } 5689 5690 #endif /* CONFIG_NUMA */ 5691 5692 /* 5693 * Boot pageset table. One per cpu which is going to be used for all 5694 * zones and all nodes. The parameters will be set in such a way 5695 * that an item put on a list will immediately be handed over to 5696 * the buddy list. This is safe since pageset manipulation is done 5697 * with interrupts disabled. 5698 * 5699 * The boot_pagesets must be kept even after bootup is complete for 5700 * unused processors and/or zones. They do play a role for bootstrapping 5701 * hotplugged processors. 5702 * 5703 * zoneinfo_show() and maybe other functions do 5704 * not check if the processor is online before following the pageset pointer. 5705 * Other parts of the kernel may not check if the zone is available. 5706 */ 5707 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); 5708 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); 5709 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 5710 5711 static void __build_all_zonelists(void *data) 5712 { 5713 int nid; 5714 int __maybe_unused cpu; 5715 pg_data_t *self = data; 5716 static DEFINE_SPINLOCK(lock); 5717 5718 spin_lock(&lock); 5719 5720 #ifdef CONFIG_NUMA 5721 memset(node_load, 0, sizeof(node_load)); 5722 #endif 5723 5724 /* 5725 * This node is hotadded and no memory is yet present. So just 5726 * building zonelists is fine - no need to touch other nodes. 5727 */ 5728 if (self && !node_online(self->node_id)) { 5729 build_zonelists(self); 5730 } else { 5731 for_each_online_node(nid) { 5732 pg_data_t *pgdat = NODE_DATA(nid); 5733 5734 build_zonelists(pgdat); 5735 } 5736 5737 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5738 /* 5739 * We now know the "local memory node" for each node-- 5740 * i.e., the node of the first zone in the generic zonelist. 5741 * Set up numa_mem percpu variable for on-line cpus. During 5742 * boot, only the boot cpu should be on-line; we'll init the 5743 * secondary cpus' numa_mem as they come on-line. During 5744 * node/memory hotplug, we'll fixup all on-line cpus. 5745 */ 5746 for_each_online_cpu(cpu) 5747 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 5748 #endif 5749 } 5750 5751 spin_unlock(&lock); 5752 } 5753 5754 static noinline void __init 5755 build_all_zonelists_init(void) 5756 { 5757 int cpu; 5758 5759 __build_all_zonelists(NULL); 5760 5761 /* 5762 * Initialize the boot_pagesets that are going to be used 5763 * for bootstrapping processors. The real pagesets for 5764 * each zone will be allocated later when the per cpu 5765 * allocator is available. 5766 * 5767 * boot_pagesets are used also for bootstrapping offline 5768 * cpus if the system is already booted because the pagesets 5769 * are needed to initialize allocators on a specific cpu too. 5770 * F.e. the percpu allocator needs the page allocator which 5771 * needs the percpu allocator in order to allocate its pagesets 5772 * (a chicken-egg dilemma). 5773 */ 5774 for_each_possible_cpu(cpu) 5775 setup_pageset(&per_cpu(boot_pageset, cpu), 0); 5776 5777 mminit_verify_zonelist(); 5778 cpuset_init_current_mems_allowed(); 5779 } 5780 5781 /* 5782 * unless system_state == SYSTEM_BOOTING. 5783 * 5784 * __ref due to call of __init annotated helper build_all_zonelists_init 5785 * [protected by SYSTEM_BOOTING]. 5786 */ 5787 void __ref build_all_zonelists(pg_data_t *pgdat) 5788 { 5789 if (system_state == SYSTEM_BOOTING) { 5790 build_all_zonelists_init(); 5791 } else { 5792 __build_all_zonelists(pgdat); 5793 /* cpuset refresh routine should be here */ 5794 } 5795 vm_total_pages = nr_free_pagecache_pages(); 5796 /* 5797 * Disable grouping by mobility if the number of pages in the 5798 * system is too low to allow the mechanism to work. It would be 5799 * more accurate, but expensive to check per-zone. This check is 5800 * made on memory-hotadd so a system can start with mobility 5801 * disabled and enable it later 5802 */ 5803 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 5804 page_group_by_mobility_disabled = 1; 5805 else 5806 page_group_by_mobility_disabled = 0; 5807 5808 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n", 5809 nr_online_nodes, 5810 page_group_by_mobility_disabled ? "off" : "on", 5811 vm_total_pages); 5812 #ifdef CONFIG_NUMA 5813 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 5814 #endif 5815 } 5816 5817 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 5818 static bool __meminit 5819 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 5820 { 5821 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5822 static struct memblock_region *r; 5823 5824 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 5825 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 5826 for_each_memblock(memory, r) { 5827 if (*pfn < memblock_region_memory_end_pfn(r)) 5828 break; 5829 } 5830 } 5831 if (*pfn >= memblock_region_memory_base_pfn(r) && 5832 memblock_is_mirror(r)) { 5833 *pfn = memblock_region_memory_end_pfn(r); 5834 return true; 5835 } 5836 } 5837 #endif 5838 return false; 5839 } 5840 5841 /* 5842 * Initially all pages are reserved - free ones are freed 5843 * up by memblock_free_all() once the early boot process is 5844 * done. Non-atomic initialization, single-pass. 5845 */ 5846 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, 5847 unsigned long start_pfn, enum memmap_context context, 5848 struct vmem_altmap *altmap) 5849 { 5850 unsigned long pfn, end_pfn = start_pfn + size; 5851 struct page *page; 5852 5853 if (highest_memmap_pfn < end_pfn - 1) 5854 highest_memmap_pfn = end_pfn - 1; 5855 5856 #ifdef CONFIG_ZONE_DEVICE 5857 /* 5858 * Honor reservation requested by the driver for this ZONE_DEVICE 5859 * memory. We limit the total number of pages to initialize to just 5860 * those that might contain the memory mapping. We will defer the 5861 * ZONE_DEVICE page initialization until after we have released 5862 * the hotplug lock. 5863 */ 5864 if (zone == ZONE_DEVICE) { 5865 if (!altmap) 5866 return; 5867 5868 if (start_pfn == altmap->base_pfn) 5869 start_pfn += altmap->reserve; 5870 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 5871 } 5872 #endif 5873 5874 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 5875 /* 5876 * There can be holes in boot-time mem_map[]s handed to this 5877 * function. They do not exist on hotplugged memory. 5878 */ 5879 if (context == MEMMAP_EARLY) { 5880 if (!early_pfn_valid(pfn)) 5881 continue; 5882 if (!early_pfn_in_nid(pfn, nid)) 5883 continue; 5884 if (overlap_memmap_init(zone, &pfn)) 5885 continue; 5886 if (defer_init(nid, pfn, end_pfn)) 5887 break; 5888 } 5889 5890 page = pfn_to_page(pfn); 5891 __init_single_page(page, pfn, zone, nid); 5892 if (context == MEMMAP_HOTPLUG) 5893 __SetPageReserved(page); 5894 5895 /* 5896 * Mark the block movable so that blocks are reserved for 5897 * movable at startup. This will force kernel allocations 5898 * to reserve their blocks rather than leaking throughout 5899 * the address space during boot when many long-lived 5900 * kernel allocations are made. 5901 * 5902 * bitmap is created for zone's valid pfn range. but memmap 5903 * can be created for invalid pages (for alignment) 5904 * check here not to call set_pageblock_migratetype() against 5905 * pfn out of zone. 5906 */ 5907 if (!(pfn & (pageblock_nr_pages - 1))) { 5908 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 5909 cond_resched(); 5910 } 5911 } 5912 } 5913 5914 #ifdef CONFIG_ZONE_DEVICE 5915 void __ref memmap_init_zone_device(struct zone *zone, 5916 unsigned long start_pfn, 5917 unsigned long size, 5918 struct dev_pagemap *pgmap) 5919 { 5920 unsigned long pfn, end_pfn = start_pfn + size; 5921 struct pglist_data *pgdat = zone->zone_pgdat; 5922 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 5923 unsigned long zone_idx = zone_idx(zone); 5924 unsigned long start = jiffies; 5925 int nid = pgdat->node_id; 5926 5927 if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE)) 5928 return; 5929 5930 /* 5931 * The call to memmap_init_zone should have already taken care 5932 * of the pages reserved for the memmap, so we can just jump to 5933 * the end of that region and start processing the device pages. 5934 */ 5935 if (altmap) { 5936 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 5937 size = end_pfn - start_pfn; 5938 } 5939 5940 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 5941 struct page *page = pfn_to_page(pfn); 5942 5943 __init_single_page(page, pfn, zone_idx, nid); 5944 5945 /* 5946 * Mark page reserved as it will need to wait for onlining 5947 * phase for it to be fully associated with a zone. 5948 * 5949 * We can use the non-atomic __set_bit operation for setting 5950 * the flag as we are still initializing the pages. 5951 */ 5952 __SetPageReserved(page); 5953 5954 /* 5955 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 5956 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 5957 * ever freed or placed on a driver-private list. 5958 */ 5959 page->pgmap = pgmap; 5960 page->zone_device_data = NULL; 5961 5962 /* 5963 * Mark the block movable so that blocks are reserved for 5964 * movable at startup. This will force kernel allocations 5965 * to reserve their blocks rather than leaking throughout 5966 * the address space during boot when many long-lived 5967 * kernel allocations are made. 5968 * 5969 * bitmap is created for zone's valid pfn range. but memmap 5970 * can be created for invalid pages (for alignment) 5971 * check here not to call set_pageblock_migratetype() against 5972 * pfn out of zone. 5973 * 5974 * Please note that MEMMAP_HOTPLUG path doesn't clear memmap 5975 * because this is done early in section_activate() 5976 */ 5977 if (!(pfn & (pageblock_nr_pages - 1))) { 5978 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 5979 cond_resched(); 5980 } 5981 } 5982 5983 pr_info("%s initialised %lu pages in %ums\n", __func__, 5984 size, jiffies_to_msecs(jiffies - start)); 5985 } 5986 5987 #endif 5988 static void __meminit zone_init_free_lists(struct zone *zone) 5989 { 5990 unsigned int order, t; 5991 for_each_migratetype_order(order, t) { 5992 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 5993 zone->free_area[order].nr_free = 0; 5994 } 5995 } 5996 5997 void __meminit __weak memmap_init(unsigned long size, int nid, 5998 unsigned long zone, unsigned long start_pfn) 5999 { 6000 memmap_init_zone(size, nid, zone, start_pfn, MEMMAP_EARLY, NULL); 6001 } 6002 6003 static int zone_batchsize(struct zone *zone) 6004 { 6005 #ifdef CONFIG_MMU 6006 int batch; 6007 6008 /* 6009 * The per-cpu-pages pools are set to around 1000th of the 6010 * size of the zone. 6011 */ 6012 batch = zone_managed_pages(zone) / 1024; 6013 /* But no more than a meg. */ 6014 if (batch * PAGE_SIZE > 1024 * 1024) 6015 batch = (1024 * 1024) / PAGE_SIZE; 6016 batch /= 4; /* We effectively *= 4 below */ 6017 if (batch < 1) 6018 batch = 1; 6019 6020 /* 6021 * Clamp the batch to a 2^n - 1 value. Having a power 6022 * of 2 value was found to be more likely to have 6023 * suboptimal cache aliasing properties in some cases. 6024 * 6025 * For example if 2 tasks are alternately allocating 6026 * batches of pages, one task can end up with a lot 6027 * of pages of one half of the possible page colors 6028 * and the other with pages of the other colors. 6029 */ 6030 batch = rounddown_pow_of_two(batch + batch/2) - 1; 6031 6032 return batch; 6033 6034 #else 6035 /* The deferral and batching of frees should be suppressed under NOMMU 6036 * conditions. 6037 * 6038 * The problem is that NOMMU needs to be able to allocate large chunks 6039 * of contiguous memory as there's no hardware page translation to 6040 * assemble apparent contiguous memory from discontiguous pages. 6041 * 6042 * Queueing large contiguous runs of pages for batching, however, 6043 * causes the pages to actually be freed in smaller chunks. As there 6044 * can be a significant delay between the individual batches being 6045 * recycled, this leads to the once large chunks of space being 6046 * fragmented and becoming unavailable for high-order allocations. 6047 */ 6048 return 0; 6049 #endif 6050 } 6051 6052 /* 6053 * pcp->high and pcp->batch values are related and dependent on one another: 6054 * ->batch must never be higher then ->high. 6055 * The following function updates them in a safe manner without read side 6056 * locking. 6057 * 6058 * Any new users of pcp->batch and pcp->high should ensure they can cope with 6059 * those fields changing asynchronously (acording the the above rule). 6060 * 6061 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 6062 * outside of boot time (or some other assurance that no concurrent updaters 6063 * exist). 6064 */ 6065 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 6066 unsigned long batch) 6067 { 6068 /* start with a fail safe value for batch */ 6069 pcp->batch = 1; 6070 smp_wmb(); 6071 6072 /* Update high, then batch, in order */ 6073 pcp->high = high; 6074 smp_wmb(); 6075 6076 pcp->batch = batch; 6077 } 6078 6079 /* a companion to pageset_set_high() */ 6080 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) 6081 { 6082 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); 6083 } 6084 6085 static void pageset_init(struct per_cpu_pageset *p) 6086 { 6087 struct per_cpu_pages *pcp; 6088 int migratetype; 6089 6090 memset(p, 0, sizeof(*p)); 6091 6092 pcp = &p->pcp; 6093 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) 6094 INIT_LIST_HEAD(&pcp->lists[migratetype]); 6095 } 6096 6097 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 6098 { 6099 pageset_init(p); 6100 pageset_set_batch(p, batch); 6101 } 6102 6103 /* 6104 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist 6105 * to the value high for the pageset p. 6106 */ 6107 static void pageset_set_high(struct per_cpu_pageset *p, 6108 unsigned long high) 6109 { 6110 unsigned long batch = max(1UL, high / 4); 6111 if ((high / 4) > (PAGE_SHIFT * 8)) 6112 batch = PAGE_SHIFT * 8; 6113 6114 pageset_update(&p->pcp, high, batch); 6115 } 6116 6117 static void pageset_set_high_and_batch(struct zone *zone, 6118 struct per_cpu_pageset *pcp) 6119 { 6120 if (percpu_pagelist_fraction) 6121 pageset_set_high(pcp, 6122 (zone_managed_pages(zone) / 6123 percpu_pagelist_fraction)); 6124 else 6125 pageset_set_batch(pcp, zone_batchsize(zone)); 6126 } 6127 6128 static void __meminit zone_pageset_init(struct zone *zone, int cpu) 6129 { 6130 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); 6131 6132 pageset_init(pcp); 6133 pageset_set_high_and_batch(zone, pcp); 6134 } 6135 6136 void __meminit setup_zone_pageset(struct zone *zone) 6137 { 6138 int cpu; 6139 zone->pageset = alloc_percpu(struct per_cpu_pageset); 6140 for_each_possible_cpu(cpu) 6141 zone_pageset_init(zone, cpu); 6142 } 6143 6144 /* 6145 * Allocate per cpu pagesets and initialize them. 6146 * Before this call only boot pagesets were available. 6147 */ 6148 void __init setup_per_cpu_pageset(void) 6149 { 6150 struct pglist_data *pgdat; 6151 struct zone *zone; 6152 6153 for_each_populated_zone(zone) 6154 setup_zone_pageset(zone); 6155 6156 for_each_online_pgdat(pgdat) 6157 pgdat->per_cpu_nodestats = 6158 alloc_percpu(struct per_cpu_nodestat); 6159 } 6160 6161 static __meminit void zone_pcp_init(struct zone *zone) 6162 { 6163 /* 6164 * per cpu subsystem is not up at this point. The following code 6165 * relies on the ability of the linker to provide the 6166 * offset of a (static) per cpu variable into the per cpu area. 6167 */ 6168 zone->pageset = &boot_pageset; 6169 6170 if (populated_zone(zone)) 6171 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", 6172 zone->name, zone->present_pages, 6173 zone_batchsize(zone)); 6174 } 6175 6176 void __meminit init_currently_empty_zone(struct zone *zone, 6177 unsigned long zone_start_pfn, 6178 unsigned long size) 6179 { 6180 struct pglist_data *pgdat = zone->zone_pgdat; 6181 int zone_idx = zone_idx(zone) + 1; 6182 6183 if (zone_idx > pgdat->nr_zones) 6184 pgdat->nr_zones = zone_idx; 6185 6186 zone->zone_start_pfn = zone_start_pfn; 6187 6188 mminit_dprintk(MMINIT_TRACE, "memmap_init", 6189 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 6190 pgdat->node_id, 6191 (unsigned long)zone_idx(zone), 6192 zone_start_pfn, (zone_start_pfn + size)); 6193 6194 zone_init_free_lists(zone); 6195 zone->initialized = 1; 6196 } 6197 6198 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6199 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID 6200 6201 /* 6202 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 6203 */ 6204 int __meminit __early_pfn_to_nid(unsigned long pfn, 6205 struct mminit_pfnnid_cache *state) 6206 { 6207 unsigned long start_pfn, end_pfn; 6208 int nid; 6209 6210 if (state->last_start <= pfn && pfn < state->last_end) 6211 return state->last_nid; 6212 6213 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 6214 if (nid != NUMA_NO_NODE) { 6215 state->last_start = start_pfn; 6216 state->last_end = end_pfn; 6217 state->last_nid = nid; 6218 } 6219 6220 return nid; 6221 } 6222 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ 6223 6224 /** 6225 * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range 6226 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed. 6227 * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid 6228 * 6229 * If an architecture guarantees that all ranges registered contain no holes 6230 * and may be freed, this this function may be used instead of calling 6231 * memblock_free_early_nid() manually. 6232 */ 6233 void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn) 6234 { 6235 unsigned long start_pfn, end_pfn; 6236 int i, this_nid; 6237 6238 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) { 6239 start_pfn = min(start_pfn, max_low_pfn); 6240 end_pfn = min(end_pfn, max_low_pfn); 6241 6242 if (start_pfn < end_pfn) 6243 memblock_free_early_nid(PFN_PHYS(start_pfn), 6244 (end_pfn - start_pfn) << PAGE_SHIFT, 6245 this_nid); 6246 } 6247 } 6248 6249 /** 6250 * sparse_memory_present_with_active_regions - Call memory_present for each active range 6251 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used. 6252 * 6253 * If an architecture guarantees that all ranges registered contain no holes and may 6254 * be freed, this function may be used instead of calling memory_present() manually. 6255 */ 6256 void __init sparse_memory_present_with_active_regions(int nid) 6257 { 6258 unsigned long start_pfn, end_pfn; 6259 int i, this_nid; 6260 6261 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) 6262 memory_present(this_nid, start_pfn, end_pfn); 6263 } 6264 6265 /** 6266 * get_pfn_range_for_nid - Return the start and end page frames for a node 6267 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 6268 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 6269 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 6270 * 6271 * It returns the start and end page frame of a node based on information 6272 * provided by memblock_set_node(). If called for a node 6273 * with no available memory, a warning is printed and the start and end 6274 * PFNs will be 0. 6275 */ 6276 void __init get_pfn_range_for_nid(unsigned int nid, 6277 unsigned long *start_pfn, unsigned long *end_pfn) 6278 { 6279 unsigned long this_start_pfn, this_end_pfn; 6280 int i; 6281 6282 *start_pfn = -1UL; 6283 *end_pfn = 0; 6284 6285 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 6286 *start_pfn = min(*start_pfn, this_start_pfn); 6287 *end_pfn = max(*end_pfn, this_end_pfn); 6288 } 6289 6290 if (*start_pfn == -1UL) 6291 *start_pfn = 0; 6292 } 6293 6294 /* 6295 * This finds a zone that can be used for ZONE_MOVABLE pages. The 6296 * assumption is made that zones within a node are ordered in monotonic 6297 * increasing memory addresses so that the "highest" populated zone is used 6298 */ 6299 static void __init find_usable_zone_for_movable(void) 6300 { 6301 int zone_index; 6302 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 6303 if (zone_index == ZONE_MOVABLE) 6304 continue; 6305 6306 if (arch_zone_highest_possible_pfn[zone_index] > 6307 arch_zone_lowest_possible_pfn[zone_index]) 6308 break; 6309 } 6310 6311 VM_BUG_ON(zone_index == -1); 6312 movable_zone = zone_index; 6313 } 6314 6315 /* 6316 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 6317 * because it is sized independent of architecture. Unlike the other zones, 6318 * the starting point for ZONE_MOVABLE is not fixed. It may be different 6319 * in each node depending on the size of each node and how evenly kernelcore 6320 * is distributed. This helper function adjusts the zone ranges 6321 * provided by the architecture for a given node by using the end of the 6322 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 6323 * zones within a node are in order of monotonic increases memory addresses 6324 */ 6325 static void __init adjust_zone_range_for_zone_movable(int nid, 6326 unsigned long zone_type, 6327 unsigned long node_start_pfn, 6328 unsigned long node_end_pfn, 6329 unsigned long *zone_start_pfn, 6330 unsigned long *zone_end_pfn) 6331 { 6332 /* Only adjust if ZONE_MOVABLE is on this node */ 6333 if (zone_movable_pfn[nid]) { 6334 /* Size ZONE_MOVABLE */ 6335 if (zone_type == ZONE_MOVABLE) { 6336 *zone_start_pfn = zone_movable_pfn[nid]; 6337 *zone_end_pfn = min(node_end_pfn, 6338 arch_zone_highest_possible_pfn[movable_zone]); 6339 6340 /* Adjust for ZONE_MOVABLE starting within this range */ 6341 } else if (!mirrored_kernelcore && 6342 *zone_start_pfn < zone_movable_pfn[nid] && 6343 *zone_end_pfn > zone_movable_pfn[nid]) { 6344 *zone_end_pfn = zone_movable_pfn[nid]; 6345 6346 /* Check if this whole range is within ZONE_MOVABLE */ 6347 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 6348 *zone_start_pfn = *zone_end_pfn; 6349 } 6350 } 6351 6352 /* 6353 * Return the number of pages a zone spans in a node, including holes 6354 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 6355 */ 6356 static unsigned long __init zone_spanned_pages_in_node(int nid, 6357 unsigned long zone_type, 6358 unsigned long node_start_pfn, 6359 unsigned long node_end_pfn, 6360 unsigned long *zone_start_pfn, 6361 unsigned long *zone_end_pfn, 6362 unsigned long *ignored) 6363 { 6364 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 6365 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 6366 /* When hotadd a new node from cpu_up(), the node should be empty */ 6367 if (!node_start_pfn && !node_end_pfn) 6368 return 0; 6369 6370 /* Get the start and end of the zone */ 6371 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 6372 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 6373 adjust_zone_range_for_zone_movable(nid, zone_type, 6374 node_start_pfn, node_end_pfn, 6375 zone_start_pfn, zone_end_pfn); 6376 6377 /* Check that this node has pages within the zone's required range */ 6378 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 6379 return 0; 6380 6381 /* Move the zone boundaries inside the node if necessary */ 6382 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 6383 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 6384 6385 /* Return the spanned pages */ 6386 return *zone_end_pfn - *zone_start_pfn; 6387 } 6388 6389 /* 6390 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 6391 * then all holes in the requested range will be accounted for. 6392 */ 6393 unsigned long __init __absent_pages_in_range(int nid, 6394 unsigned long range_start_pfn, 6395 unsigned long range_end_pfn) 6396 { 6397 unsigned long nr_absent = range_end_pfn - range_start_pfn; 6398 unsigned long start_pfn, end_pfn; 6399 int i; 6400 6401 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6402 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 6403 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 6404 nr_absent -= end_pfn - start_pfn; 6405 } 6406 return nr_absent; 6407 } 6408 6409 /** 6410 * absent_pages_in_range - Return number of page frames in holes within a range 6411 * @start_pfn: The start PFN to start searching for holes 6412 * @end_pfn: The end PFN to stop searching for holes 6413 * 6414 * Return: the number of pages frames in memory holes within a range. 6415 */ 6416 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 6417 unsigned long end_pfn) 6418 { 6419 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 6420 } 6421 6422 /* Return the number of page frames in holes in a zone on a node */ 6423 static unsigned long __init zone_absent_pages_in_node(int nid, 6424 unsigned long zone_type, 6425 unsigned long node_start_pfn, 6426 unsigned long node_end_pfn, 6427 unsigned long *ignored) 6428 { 6429 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 6430 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 6431 unsigned long zone_start_pfn, zone_end_pfn; 6432 unsigned long nr_absent; 6433 6434 /* When hotadd a new node from cpu_up(), the node should be empty */ 6435 if (!node_start_pfn && !node_end_pfn) 6436 return 0; 6437 6438 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 6439 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 6440 6441 adjust_zone_range_for_zone_movable(nid, zone_type, 6442 node_start_pfn, node_end_pfn, 6443 &zone_start_pfn, &zone_end_pfn); 6444 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 6445 6446 /* 6447 * ZONE_MOVABLE handling. 6448 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 6449 * and vice versa. 6450 */ 6451 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 6452 unsigned long start_pfn, end_pfn; 6453 struct memblock_region *r; 6454 6455 for_each_memblock(memory, r) { 6456 start_pfn = clamp(memblock_region_memory_base_pfn(r), 6457 zone_start_pfn, zone_end_pfn); 6458 end_pfn = clamp(memblock_region_memory_end_pfn(r), 6459 zone_start_pfn, zone_end_pfn); 6460 6461 if (zone_type == ZONE_MOVABLE && 6462 memblock_is_mirror(r)) 6463 nr_absent += end_pfn - start_pfn; 6464 6465 if (zone_type == ZONE_NORMAL && 6466 !memblock_is_mirror(r)) 6467 nr_absent += end_pfn - start_pfn; 6468 } 6469 } 6470 6471 return nr_absent; 6472 } 6473 6474 #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6475 static inline unsigned long __init zone_spanned_pages_in_node(int nid, 6476 unsigned long zone_type, 6477 unsigned long node_start_pfn, 6478 unsigned long node_end_pfn, 6479 unsigned long *zone_start_pfn, 6480 unsigned long *zone_end_pfn, 6481 unsigned long *zones_size) 6482 { 6483 unsigned int zone; 6484 6485 *zone_start_pfn = node_start_pfn; 6486 for (zone = 0; zone < zone_type; zone++) 6487 *zone_start_pfn += zones_size[zone]; 6488 6489 *zone_end_pfn = *zone_start_pfn + zones_size[zone_type]; 6490 6491 return zones_size[zone_type]; 6492 } 6493 6494 static inline unsigned long __init zone_absent_pages_in_node(int nid, 6495 unsigned long zone_type, 6496 unsigned long node_start_pfn, 6497 unsigned long node_end_pfn, 6498 unsigned long *zholes_size) 6499 { 6500 if (!zholes_size) 6501 return 0; 6502 6503 return zholes_size[zone_type]; 6504 } 6505 6506 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6507 6508 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 6509 unsigned long node_start_pfn, 6510 unsigned long node_end_pfn, 6511 unsigned long *zones_size, 6512 unsigned long *zholes_size) 6513 { 6514 unsigned long realtotalpages = 0, totalpages = 0; 6515 enum zone_type i; 6516 6517 for (i = 0; i < MAX_NR_ZONES; i++) { 6518 struct zone *zone = pgdat->node_zones + i; 6519 unsigned long zone_start_pfn, zone_end_pfn; 6520 unsigned long size, real_size; 6521 6522 size = zone_spanned_pages_in_node(pgdat->node_id, i, 6523 node_start_pfn, 6524 node_end_pfn, 6525 &zone_start_pfn, 6526 &zone_end_pfn, 6527 zones_size); 6528 real_size = size - zone_absent_pages_in_node(pgdat->node_id, i, 6529 node_start_pfn, node_end_pfn, 6530 zholes_size); 6531 if (size) 6532 zone->zone_start_pfn = zone_start_pfn; 6533 else 6534 zone->zone_start_pfn = 0; 6535 zone->spanned_pages = size; 6536 zone->present_pages = real_size; 6537 6538 totalpages += size; 6539 realtotalpages += real_size; 6540 } 6541 6542 pgdat->node_spanned_pages = totalpages; 6543 pgdat->node_present_pages = realtotalpages; 6544 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, 6545 realtotalpages); 6546 } 6547 6548 #ifndef CONFIG_SPARSEMEM 6549 /* 6550 * Calculate the size of the zone->blockflags rounded to an unsigned long 6551 * Start by making sure zonesize is a multiple of pageblock_order by rounding 6552 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 6553 * round what is now in bits to nearest long in bits, then return it in 6554 * bytes. 6555 */ 6556 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 6557 { 6558 unsigned long usemapsize; 6559 6560 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 6561 usemapsize = roundup(zonesize, pageblock_nr_pages); 6562 usemapsize = usemapsize >> pageblock_order; 6563 usemapsize *= NR_PAGEBLOCK_BITS; 6564 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 6565 6566 return usemapsize / 8; 6567 } 6568 6569 static void __ref setup_usemap(struct pglist_data *pgdat, 6570 struct zone *zone, 6571 unsigned long zone_start_pfn, 6572 unsigned long zonesize) 6573 { 6574 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); 6575 zone->pageblock_flags = NULL; 6576 if (usemapsize) { 6577 zone->pageblock_flags = 6578 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 6579 pgdat->node_id); 6580 if (!zone->pageblock_flags) 6581 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 6582 usemapsize, zone->name, pgdat->node_id); 6583 } 6584 } 6585 #else 6586 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, 6587 unsigned long zone_start_pfn, unsigned long zonesize) {} 6588 #endif /* CONFIG_SPARSEMEM */ 6589 6590 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 6591 6592 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 6593 void __init set_pageblock_order(void) 6594 { 6595 unsigned int order; 6596 6597 /* Check that pageblock_nr_pages has not already been setup */ 6598 if (pageblock_order) 6599 return; 6600 6601 if (HPAGE_SHIFT > PAGE_SHIFT) 6602 order = HUGETLB_PAGE_ORDER; 6603 else 6604 order = MAX_ORDER - 1; 6605 6606 /* 6607 * Assume the largest contiguous order of interest is a huge page. 6608 * This value may be variable depending on boot parameters on IA64 and 6609 * powerpc. 6610 */ 6611 pageblock_order = order; 6612 } 6613 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6614 6615 /* 6616 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 6617 * is unused as pageblock_order is set at compile-time. See 6618 * include/linux/pageblock-flags.h for the values of pageblock_order based on 6619 * the kernel config 6620 */ 6621 void __init set_pageblock_order(void) 6622 { 6623 } 6624 6625 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6626 6627 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 6628 unsigned long present_pages) 6629 { 6630 unsigned long pages = spanned_pages; 6631 6632 /* 6633 * Provide a more accurate estimation if there are holes within 6634 * the zone and SPARSEMEM is in use. If there are holes within the 6635 * zone, each populated memory region may cost us one or two extra 6636 * memmap pages due to alignment because memmap pages for each 6637 * populated regions may not be naturally aligned on page boundary. 6638 * So the (present_pages >> 4) heuristic is a tradeoff for that. 6639 */ 6640 if (spanned_pages > present_pages + (present_pages >> 4) && 6641 IS_ENABLED(CONFIG_SPARSEMEM)) 6642 pages = present_pages; 6643 6644 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 6645 } 6646 6647 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 6648 static void pgdat_init_split_queue(struct pglist_data *pgdat) 6649 { 6650 struct deferred_split *ds_queue = &pgdat->deferred_split_queue; 6651 6652 spin_lock_init(&ds_queue->split_queue_lock); 6653 INIT_LIST_HEAD(&ds_queue->split_queue); 6654 ds_queue->split_queue_len = 0; 6655 } 6656 #else 6657 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 6658 #endif 6659 6660 #ifdef CONFIG_COMPACTION 6661 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 6662 { 6663 init_waitqueue_head(&pgdat->kcompactd_wait); 6664 } 6665 #else 6666 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 6667 #endif 6668 6669 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 6670 { 6671 pgdat_resize_init(pgdat); 6672 6673 pgdat_init_split_queue(pgdat); 6674 pgdat_init_kcompactd(pgdat); 6675 6676 init_waitqueue_head(&pgdat->kswapd_wait); 6677 init_waitqueue_head(&pgdat->pfmemalloc_wait); 6678 6679 pgdat_page_ext_init(pgdat); 6680 spin_lock_init(&pgdat->lru_lock); 6681 lruvec_init(node_lruvec(pgdat)); 6682 } 6683 6684 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 6685 unsigned long remaining_pages) 6686 { 6687 atomic_long_set(&zone->managed_pages, remaining_pages); 6688 zone_set_nid(zone, nid); 6689 zone->name = zone_names[idx]; 6690 zone->zone_pgdat = NODE_DATA(nid); 6691 spin_lock_init(&zone->lock); 6692 zone_seqlock_init(zone); 6693 zone_pcp_init(zone); 6694 } 6695 6696 /* 6697 * Set up the zone data structures 6698 * - init pgdat internals 6699 * - init all zones belonging to this node 6700 * 6701 * NOTE: this function is only called during memory hotplug 6702 */ 6703 #ifdef CONFIG_MEMORY_HOTPLUG 6704 void __ref free_area_init_core_hotplug(int nid) 6705 { 6706 enum zone_type z; 6707 pg_data_t *pgdat = NODE_DATA(nid); 6708 6709 pgdat_init_internals(pgdat); 6710 for (z = 0; z < MAX_NR_ZONES; z++) 6711 zone_init_internals(&pgdat->node_zones[z], z, nid, 0); 6712 } 6713 #endif 6714 6715 /* 6716 * Set up the zone data structures: 6717 * - mark all pages reserved 6718 * - mark all memory queues empty 6719 * - clear the memory bitmaps 6720 * 6721 * NOTE: pgdat should get zeroed by caller. 6722 * NOTE: this function is only called during early init. 6723 */ 6724 static void __init free_area_init_core(struct pglist_data *pgdat) 6725 { 6726 enum zone_type j; 6727 int nid = pgdat->node_id; 6728 6729 pgdat_init_internals(pgdat); 6730 pgdat->per_cpu_nodestats = &boot_nodestats; 6731 6732 for (j = 0; j < MAX_NR_ZONES; j++) { 6733 struct zone *zone = pgdat->node_zones + j; 6734 unsigned long size, freesize, memmap_pages; 6735 unsigned long zone_start_pfn = zone->zone_start_pfn; 6736 6737 size = zone->spanned_pages; 6738 freesize = zone->present_pages; 6739 6740 /* 6741 * Adjust freesize so that it accounts for how much memory 6742 * is used by this zone for memmap. This affects the watermark 6743 * and per-cpu initialisations 6744 */ 6745 memmap_pages = calc_memmap_size(size, freesize); 6746 if (!is_highmem_idx(j)) { 6747 if (freesize >= memmap_pages) { 6748 freesize -= memmap_pages; 6749 if (memmap_pages) 6750 printk(KERN_DEBUG 6751 " %s zone: %lu pages used for memmap\n", 6752 zone_names[j], memmap_pages); 6753 } else 6754 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n", 6755 zone_names[j], memmap_pages, freesize); 6756 } 6757 6758 /* Account for reserved pages */ 6759 if (j == 0 && freesize > dma_reserve) { 6760 freesize -= dma_reserve; 6761 printk(KERN_DEBUG " %s zone: %lu pages reserved\n", 6762 zone_names[0], dma_reserve); 6763 } 6764 6765 if (!is_highmem_idx(j)) 6766 nr_kernel_pages += freesize; 6767 /* Charge for highmem memmap if there are enough kernel pages */ 6768 else if (nr_kernel_pages > memmap_pages * 2) 6769 nr_kernel_pages -= memmap_pages; 6770 nr_all_pages += freesize; 6771 6772 /* 6773 * Set an approximate value for lowmem here, it will be adjusted 6774 * when the bootmem allocator frees pages into the buddy system. 6775 * And all highmem pages will be managed by the buddy system. 6776 */ 6777 zone_init_internals(zone, j, nid, freesize); 6778 6779 if (!size) 6780 continue; 6781 6782 set_pageblock_order(); 6783 setup_usemap(pgdat, zone, zone_start_pfn, size); 6784 init_currently_empty_zone(zone, zone_start_pfn, size); 6785 memmap_init(size, nid, j, zone_start_pfn); 6786 } 6787 } 6788 6789 #ifdef CONFIG_FLAT_NODE_MEM_MAP 6790 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) 6791 { 6792 unsigned long __maybe_unused start = 0; 6793 unsigned long __maybe_unused offset = 0; 6794 6795 /* Skip empty nodes */ 6796 if (!pgdat->node_spanned_pages) 6797 return; 6798 6799 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 6800 offset = pgdat->node_start_pfn - start; 6801 /* ia64 gets its own node_mem_map, before this, without bootmem */ 6802 if (!pgdat->node_mem_map) { 6803 unsigned long size, end; 6804 struct page *map; 6805 6806 /* 6807 * The zone's endpoints aren't required to be MAX_ORDER 6808 * aligned but the node_mem_map endpoints must be in order 6809 * for the buddy allocator to function correctly. 6810 */ 6811 end = pgdat_end_pfn(pgdat); 6812 end = ALIGN(end, MAX_ORDER_NR_PAGES); 6813 size = (end - start) * sizeof(struct page); 6814 map = memblock_alloc_node(size, SMP_CACHE_BYTES, 6815 pgdat->node_id); 6816 if (!map) 6817 panic("Failed to allocate %ld bytes for node %d memory map\n", 6818 size, pgdat->node_id); 6819 pgdat->node_mem_map = map + offset; 6820 } 6821 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 6822 __func__, pgdat->node_id, (unsigned long)pgdat, 6823 (unsigned long)pgdat->node_mem_map); 6824 #ifndef CONFIG_NEED_MULTIPLE_NODES 6825 /* 6826 * With no DISCONTIG, the global mem_map is just set as node 0's 6827 */ 6828 if (pgdat == NODE_DATA(0)) { 6829 mem_map = NODE_DATA(0)->node_mem_map; 6830 #if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM) 6831 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 6832 mem_map -= offset; 6833 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6834 } 6835 #endif 6836 } 6837 #else 6838 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { } 6839 #endif /* CONFIG_FLAT_NODE_MEM_MAP */ 6840 6841 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 6842 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 6843 { 6844 pgdat->first_deferred_pfn = ULONG_MAX; 6845 } 6846 #else 6847 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 6848 #endif 6849 6850 void __init free_area_init_node(int nid, unsigned long *zones_size, 6851 unsigned long node_start_pfn, 6852 unsigned long *zholes_size) 6853 { 6854 pg_data_t *pgdat = NODE_DATA(nid); 6855 unsigned long start_pfn = 0; 6856 unsigned long end_pfn = 0; 6857 6858 /* pg_data_t should be reset to zero when it's allocated */ 6859 WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx); 6860 6861 pgdat->node_id = nid; 6862 pgdat->node_start_pfn = node_start_pfn; 6863 pgdat->per_cpu_nodestats = NULL; 6864 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6865 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 6866 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 6867 (u64)start_pfn << PAGE_SHIFT, 6868 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 6869 #else 6870 start_pfn = node_start_pfn; 6871 #endif 6872 calculate_node_totalpages(pgdat, start_pfn, end_pfn, 6873 zones_size, zholes_size); 6874 6875 alloc_node_mem_map(pgdat); 6876 pgdat_set_deferred_range(pgdat); 6877 6878 free_area_init_core(pgdat); 6879 } 6880 6881 #if !defined(CONFIG_FLAT_NODE_MEM_MAP) 6882 /* 6883 * Zero all valid struct pages in range [spfn, epfn), return number of struct 6884 * pages zeroed 6885 */ 6886 static u64 zero_pfn_range(unsigned long spfn, unsigned long epfn) 6887 { 6888 unsigned long pfn; 6889 u64 pgcnt = 0; 6890 6891 for (pfn = spfn; pfn < epfn; pfn++) { 6892 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) { 6893 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages) 6894 + pageblock_nr_pages - 1; 6895 continue; 6896 } 6897 mm_zero_struct_page(pfn_to_page(pfn)); 6898 pgcnt++; 6899 } 6900 6901 return pgcnt; 6902 } 6903 6904 /* 6905 * Only struct pages that are backed by physical memory are zeroed and 6906 * initialized by going through __init_single_page(). But, there are some 6907 * struct pages which are reserved in memblock allocator and their fields 6908 * may be accessed (for example page_to_pfn() on some configuration accesses 6909 * flags). We must explicitly zero those struct pages. 6910 * 6911 * This function also addresses a similar issue where struct pages are left 6912 * uninitialized because the physical address range is not covered by 6913 * memblock.memory or memblock.reserved. That could happen when memblock 6914 * layout is manually configured via memmap=. 6915 */ 6916 void __init zero_resv_unavail(void) 6917 { 6918 phys_addr_t start, end; 6919 u64 i, pgcnt; 6920 phys_addr_t next = 0; 6921 6922 /* 6923 * Loop through unavailable ranges not covered by memblock.memory. 6924 */ 6925 pgcnt = 0; 6926 for_each_mem_range(i, &memblock.memory, NULL, 6927 NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, NULL) { 6928 if (next < start) 6929 pgcnt += zero_pfn_range(PFN_DOWN(next), PFN_UP(start)); 6930 next = end; 6931 } 6932 pgcnt += zero_pfn_range(PFN_DOWN(next), max_pfn); 6933 6934 /* 6935 * Struct pages that do not have backing memory. This could be because 6936 * firmware is using some of this memory, or for some other reasons. 6937 */ 6938 if (pgcnt) 6939 pr_info("Zeroed struct page in unavailable ranges: %lld pages", pgcnt); 6940 } 6941 #endif /* !CONFIG_FLAT_NODE_MEM_MAP */ 6942 6943 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6944 6945 #if MAX_NUMNODES > 1 6946 /* 6947 * Figure out the number of possible node ids. 6948 */ 6949 void __init setup_nr_node_ids(void) 6950 { 6951 unsigned int highest; 6952 6953 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 6954 nr_node_ids = highest + 1; 6955 } 6956 #endif 6957 6958 /** 6959 * node_map_pfn_alignment - determine the maximum internode alignment 6960 * 6961 * This function should be called after node map is populated and sorted. 6962 * It calculates the maximum power of two alignment which can distinguish 6963 * all the nodes. 6964 * 6965 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 6966 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 6967 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 6968 * shifted, 1GiB is enough and this function will indicate so. 6969 * 6970 * This is used to test whether pfn -> nid mapping of the chosen memory 6971 * model has fine enough granularity to avoid incorrect mapping for the 6972 * populated node map. 6973 * 6974 * Return: the determined alignment in pfn's. 0 if there is no alignment 6975 * requirement (single node). 6976 */ 6977 unsigned long __init node_map_pfn_alignment(void) 6978 { 6979 unsigned long accl_mask = 0, last_end = 0; 6980 unsigned long start, end, mask; 6981 int last_nid = NUMA_NO_NODE; 6982 int i, nid; 6983 6984 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 6985 if (!start || last_nid < 0 || last_nid == nid) { 6986 last_nid = nid; 6987 last_end = end; 6988 continue; 6989 } 6990 6991 /* 6992 * Start with a mask granular enough to pin-point to the 6993 * start pfn and tick off bits one-by-one until it becomes 6994 * too coarse to separate the current node from the last. 6995 */ 6996 mask = ~((1 << __ffs(start)) - 1); 6997 while (mask && last_end <= (start & (mask << 1))) 6998 mask <<= 1; 6999 7000 /* accumulate all internode masks */ 7001 accl_mask |= mask; 7002 } 7003 7004 /* convert mask to number of pages */ 7005 return ~accl_mask + 1; 7006 } 7007 7008 /* Find the lowest pfn for a node */ 7009 static unsigned long __init find_min_pfn_for_node(int nid) 7010 { 7011 unsigned long min_pfn = ULONG_MAX; 7012 unsigned long start_pfn; 7013 int i; 7014 7015 for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL) 7016 min_pfn = min(min_pfn, start_pfn); 7017 7018 if (min_pfn == ULONG_MAX) { 7019 pr_warn("Could not find start_pfn for node %d\n", nid); 7020 return 0; 7021 } 7022 7023 return min_pfn; 7024 } 7025 7026 /** 7027 * find_min_pfn_with_active_regions - Find the minimum PFN registered 7028 * 7029 * Return: the minimum PFN based on information provided via 7030 * memblock_set_node(). 7031 */ 7032 unsigned long __init find_min_pfn_with_active_regions(void) 7033 { 7034 return find_min_pfn_for_node(MAX_NUMNODES); 7035 } 7036 7037 /* 7038 * early_calculate_totalpages() 7039 * Sum pages in active regions for movable zone. 7040 * Populate N_MEMORY for calculating usable_nodes. 7041 */ 7042 static unsigned long __init early_calculate_totalpages(void) 7043 { 7044 unsigned long totalpages = 0; 7045 unsigned long start_pfn, end_pfn; 7046 int i, nid; 7047 7048 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7049 unsigned long pages = end_pfn - start_pfn; 7050 7051 totalpages += pages; 7052 if (pages) 7053 node_set_state(nid, N_MEMORY); 7054 } 7055 return totalpages; 7056 } 7057 7058 /* 7059 * Find the PFN the Movable zone begins in each node. Kernel memory 7060 * is spread evenly between nodes as long as the nodes have enough 7061 * memory. When they don't, some nodes will have more kernelcore than 7062 * others 7063 */ 7064 static void __init find_zone_movable_pfns_for_nodes(void) 7065 { 7066 int i, nid; 7067 unsigned long usable_startpfn; 7068 unsigned long kernelcore_node, kernelcore_remaining; 7069 /* save the state before borrow the nodemask */ 7070 nodemask_t saved_node_state = node_states[N_MEMORY]; 7071 unsigned long totalpages = early_calculate_totalpages(); 7072 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 7073 struct memblock_region *r; 7074 7075 /* Need to find movable_zone earlier when movable_node is specified. */ 7076 find_usable_zone_for_movable(); 7077 7078 /* 7079 * If movable_node is specified, ignore kernelcore and movablecore 7080 * options. 7081 */ 7082 if (movable_node_is_enabled()) { 7083 for_each_memblock(memory, r) { 7084 if (!memblock_is_hotpluggable(r)) 7085 continue; 7086 7087 nid = r->nid; 7088 7089 usable_startpfn = PFN_DOWN(r->base); 7090 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7091 min(usable_startpfn, zone_movable_pfn[nid]) : 7092 usable_startpfn; 7093 } 7094 7095 goto out2; 7096 } 7097 7098 /* 7099 * If kernelcore=mirror is specified, ignore movablecore option 7100 */ 7101 if (mirrored_kernelcore) { 7102 bool mem_below_4gb_not_mirrored = false; 7103 7104 for_each_memblock(memory, r) { 7105 if (memblock_is_mirror(r)) 7106 continue; 7107 7108 nid = r->nid; 7109 7110 usable_startpfn = memblock_region_memory_base_pfn(r); 7111 7112 if (usable_startpfn < 0x100000) { 7113 mem_below_4gb_not_mirrored = true; 7114 continue; 7115 } 7116 7117 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7118 min(usable_startpfn, zone_movable_pfn[nid]) : 7119 usable_startpfn; 7120 } 7121 7122 if (mem_below_4gb_not_mirrored) 7123 pr_warn("This configuration results in unmirrored kernel memory."); 7124 7125 goto out2; 7126 } 7127 7128 /* 7129 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 7130 * amount of necessary memory. 7131 */ 7132 if (required_kernelcore_percent) 7133 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 7134 10000UL; 7135 if (required_movablecore_percent) 7136 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 7137 10000UL; 7138 7139 /* 7140 * If movablecore= was specified, calculate what size of 7141 * kernelcore that corresponds so that memory usable for 7142 * any allocation type is evenly spread. If both kernelcore 7143 * and movablecore are specified, then the value of kernelcore 7144 * will be used for required_kernelcore if it's greater than 7145 * what movablecore would have allowed. 7146 */ 7147 if (required_movablecore) { 7148 unsigned long corepages; 7149 7150 /* 7151 * Round-up so that ZONE_MOVABLE is at least as large as what 7152 * was requested by the user 7153 */ 7154 required_movablecore = 7155 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 7156 required_movablecore = min(totalpages, required_movablecore); 7157 corepages = totalpages - required_movablecore; 7158 7159 required_kernelcore = max(required_kernelcore, corepages); 7160 } 7161 7162 /* 7163 * If kernelcore was not specified or kernelcore size is larger 7164 * than totalpages, there is no ZONE_MOVABLE. 7165 */ 7166 if (!required_kernelcore || required_kernelcore >= totalpages) 7167 goto out; 7168 7169 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 7170 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 7171 7172 restart: 7173 /* Spread kernelcore memory as evenly as possible throughout nodes */ 7174 kernelcore_node = required_kernelcore / usable_nodes; 7175 for_each_node_state(nid, N_MEMORY) { 7176 unsigned long start_pfn, end_pfn; 7177 7178 /* 7179 * Recalculate kernelcore_node if the division per node 7180 * now exceeds what is necessary to satisfy the requested 7181 * amount of memory for the kernel 7182 */ 7183 if (required_kernelcore < kernelcore_node) 7184 kernelcore_node = required_kernelcore / usable_nodes; 7185 7186 /* 7187 * As the map is walked, we track how much memory is usable 7188 * by the kernel using kernelcore_remaining. When it is 7189 * 0, the rest of the node is usable by ZONE_MOVABLE 7190 */ 7191 kernelcore_remaining = kernelcore_node; 7192 7193 /* Go through each range of PFNs within this node */ 7194 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7195 unsigned long size_pages; 7196 7197 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 7198 if (start_pfn >= end_pfn) 7199 continue; 7200 7201 /* Account for what is only usable for kernelcore */ 7202 if (start_pfn < usable_startpfn) { 7203 unsigned long kernel_pages; 7204 kernel_pages = min(end_pfn, usable_startpfn) 7205 - start_pfn; 7206 7207 kernelcore_remaining -= min(kernel_pages, 7208 kernelcore_remaining); 7209 required_kernelcore -= min(kernel_pages, 7210 required_kernelcore); 7211 7212 /* Continue if range is now fully accounted */ 7213 if (end_pfn <= usable_startpfn) { 7214 7215 /* 7216 * Push zone_movable_pfn to the end so 7217 * that if we have to rebalance 7218 * kernelcore across nodes, we will 7219 * not double account here 7220 */ 7221 zone_movable_pfn[nid] = end_pfn; 7222 continue; 7223 } 7224 start_pfn = usable_startpfn; 7225 } 7226 7227 /* 7228 * The usable PFN range for ZONE_MOVABLE is from 7229 * start_pfn->end_pfn. Calculate size_pages as the 7230 * number of pages used as kernelcore 7231 */ 7232 size_pages = end_pfn - start_pfn; 7233 if (size_pages > kernelcore_remaining) 7234 size_pages = kernelcore_remaining; 7235 zone_movable_pfn[nid] = start_pfn + size_pages; 7236 7237 /* 7238 * Some kernelcore has been met, update counts and 7239 * break if the kernelcore for this node has been 7240 * satisfied 7241 */ 7242 required_kernelcore -= min(required_kernelcore, 7243 size_pages); 7244 kernelcore_remaining -= size_pages; 7245 if (!kernelcore_remaining) 7246 break; 7247 } 7248 } 7249 7250 /* 7251 * If there is still required_kernelcore, we do another pass with one 7252 * less node in the count. This will push zone_movable_pfn[nid] further 7253 * along on the nodes that still have memory until kernelcore is 7254 * satisfied 7255 */ 7256 usable_nodes--; 7257 if (usable_nodes && required_kernelcore > usable_nodes) 7258 goto restart; 7259 7260 out2: 7261 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 7262 for (nid = 0; nid < MAX_NUMNODES; nid++) 7263 zone_movable_pfn[nid] = 7264 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 7265 7266 out: 7267 /* restore the node_state */ 7268 node_states[N_MEMORY] = saved_node_state; 7269 } 7270 7271 /* Any regular or high memory on that node ? */ 7272 static void check_for_memory(pg_data_t *pgdat, int nid) 7273 { 7274 enum zone_type zone_type; 7275 7276 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 7277 struct zone *zone = &pgdat->node_zones[zone_type]; 7278 if (populated_zone(zone)) { 7279 if (IS_ENABLED(CONFIG_HIGHMEM)) 7280 node_set_state(nid, N_HIGH_MEMORY); 7281 if (zone_type <= ZONE_NORMAL) 7282 node_set_state(nid, N_NORMAL_MEMORY); 7283 break; 7284 } 7285 } 7286 } 7287 7288 /** 7289 * free_area_init_nodes - Initialise all pg_data_t and zone data 7290 * @max_zone_pfn: an array of max PFNs for each zone 7291 * 7292 * This will call free_area_init_node() for each active node in the system. 7293 * Using the page ranges provided by memblock_set_node(), the size of each 7294 * zone in each node and their holes is calculated. If the maximum PFN 7295 * between two adjacent zones match, it is assumed that the zone is empty. 7296 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 7297 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 7298 * starts where the previous one ended. For example, ZONE_DMA32 starts 7299 * at arch_max_dma_pfn. 7300 */ 7301 void __init free_area_init_nodes(unsigned long *max_zone_pfn) 7302 { 7303 unsigned long start_pfn, end_pfn; 7304 int i, nid; 7305 7306 /* Record where the zone boundaries are */ 7307 memset(arch_zone_lowest_possible_pfn, 0, 7308 sizeof(arch_zone_lowest_possible_pfn)); 7309 memset(arch_zone_highest_possible_pfn, 0, 7310 sizeof(arch_zone_highest_possible_pfn)); 7311 7312 start_pfn = find_min_pfn_with_active_regions(); 7313 7314 for (i = 0; i < MAX_NR_ZONES; i++) { 7315 if (i == ZONE_MOVABLE) 7316 continue; 7317 7318 end_pfn = max(max_zone_pfn[i], start_pfn); 7319 arch_zone_lowest_possible_pfn[i] = start_pfn; 7320 arch_zone_highest_possible_pfn[i] = end_pfn; 7321 7322 start_pfn = end_pfn; 7323 } 7324 7325 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 7326 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 7327 find_zone_movable_pfns_for_nodes(); 7328 7329 /* Print out the zone ranges */ 7330 pr_info("Zone ranges:\n"); 7331 for (i = 0; i < MAX_NR_ZONES; i++) { 7332 if (i == ZONE_MOVABLE) 7333 continue; 7334 pr_info(" %-8s ", zone_names[i]); 7335 if (arch_zone_lowest_possible_pfn[i] == 7336 arch_zone_highest_possible_pfn[i]) 7337 pr_cont("empty\n"); 7338 else 7339 pr_cont("[mem %#018Lx-%#018Lx]\n", 7340 (u64)arch_zone_lowest_possible_pfn[i] 7341 << PAGE_SHIFT, 7342 ((u64)arch_zone_highest_possible_pfn[i] 7343 << PAGE_SHIFT) - 1); 7344 } 7345 7346 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 7347 pr_info("Movable zone start for each node\n"); 7348 for (i = 0; i < MAX_NUMNODES; i++) { 7349 if (zone_movable_pfn[i]) 7350 pr_info(" Node %d: %#018Lx\n", i, 7351 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 7352 } 7353 7354 /* 7355 * Print out the early node map, and initialize the 7356 * subsection-map relative to active online memory ranges to 7357 * enable future "sub-section" extensions of the memory map. 7358 */ 7359 pr_info("Early memory node ranges\n"); 7360 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7361 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 7362 (u64)start_pfn << PAGE_SHIFT, 7363 ((u64)end_pfn << PAGE_SHIFT) - 1); 7364 subsection_map_init(start_pfn, end_pfn - start_pfn); 7365 } 7366 7367 /* Initialise every node */ 7368 mminit_verify_pageflags_layout(); 7369 setup_nr_node_ids(); 7370 zero_resv_unavail(); 7371 for_each_online_node(nid) { 7372 pg_data_t *pgdat = NODE_DATA(nid); 7373 free_area_init_node(nid, NULL, 7374 find_min_pfn_for_node(nid), NULL); 7375 7376 /* Any memory on that node */ 7377 if (pgdat->node_present_pages) 7378 node_set_state(nid, N_MEMORY); 7379 check_for_memory(pgdat, nid); 7380 } 7381 } 7382 7383 static int __init cmdline_parse_core(char *p, unsigned long *core, 7384 unsigned long *percent) 7385 { 7386 unsigned long long coremem; 7387 char *endptr; 7388 7389 if (!p) 7390 return -EINVAL; 7391 7392 /* Value may be a percentage of total memory, otherwise bytes */ 7393 coremem = simple_strtoull(p, &endptr, 0); 7394 if (*endptr == '%') { 7395 /* Paranoid check for percent values greater than 100 */ 7396 WARN_ON(coremem > 100); 7397 7398 *percent = coremem; 7399 } else { 7400 coremem = memparse(p, &p); 7401 /* Paranoid check that UL is enough for the coremem value */ 7402 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 7403 7404 *core = coremem >> PAGE_SHIFT; 7405 *percent = 0UL; 7406 } 7407 return 0; 7408 } 7409 7410 /* 7411 * kernelcore=size sets the amount of memory for use for allocations that 7412 * cannot be reclaimed or migrated. 7413 */ 7414 static int __init cmdline_parse_kernelcore(char *p) 7415 { 7416 /* parse kernelcore=mirror */ 7417 if (parse_option_str(p, "mirror")) { 7418 mirrored_kernelcore = true; 7419 return 0; 7420 } 7421 7422 return cmdline_parse_core(p, &required_kernelcore, 7423 &required_kernelcore_percent); 7424 } 7425 7426 /* 7427 * movablecore=size sets the amount of memory for use for allocations that 7428 * can be reclaimed or migrated. 7429 */ 7430 static int __init cmdline_parse_movablecore(char *p) 7431 { 7432 return cmdline_parse_core(p, &required_movablecore, 7433 &required_movablecore_percent); 7434 } 7435 7436 early_param("kernelcore", cmdline_parse_kernelcore); 7437 early_param("movablecore", cmdline_parse_movablecore); 7438 7439 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 7440 7441 void adjust_managed_page_count(struct page *page, long count) 7442 { 7443 atomic_long_add(count, &page_zone(page)->managed_pages); 7444 totalram_pages_add(count); 7445 #ifdef CONFIG_HIGHMEM 7446 if (PageHighMem(page)) 7447 totalhigh_pages_add(count); 7448 #endif 7449 } 7450 EXPORT_SYMBOL(adjust_managed_page_count); 7451 7452 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 7453 { 7454 void *pos; 7455 unsigned long pages = 0; 7456 7457 start = (void *)PAGE_ALIGN((unsigned long)start); 7458 end = (void *)((unsigned long)end & PAGE_MASK); 7459 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 7460 struct page *page = virt_to_page(pos); 7461 void *direct_map_addr; 7462 7463 /* 7464 * 'direct_map_addr' might be different from 'pos' 7465 * because some architectures' virt_to_page() 7466 * work with aliases. Getting the direct map 7467 * address ensures that we get a _writeable_ 7468 * alias for the memset(). 7469 */ 7470 direct_map_addr = page_address(page); 7471 if ((unsigned int)poison <= 0xFF) 7472 memset(direct_map_addr, poison, PAGE_SIZE); 7473 7474 free_reserved_page(page); 7475 } 7476 7477 if (pages && s) 7478 pr_info("Freeing %s memory: %ldK\n", 7479 s, pages << (PAGE_SHIFT - 10)); 7480 7481 return pages; 7482 } 7483 7484 #ifdef CONFIG_HIGHMEM 7485 void free_highmem_page(struct page *page) 7486 { 7487 __free_reserved_page(page); 7488 totalram_pages_inc(); 7489 atomic_long_inc(&page_zone(page)->managed_pages); 7490 totalhigh_pages_inc(); 7491 } 7492 #endif 7493 7494 7495 void __init mem_init_print_info(const char *str) 7496 { 7497 unsigned long physpages, codesize, datasize, rosize, bss_size; 7498 unsigned long init_code_size, init_data_size; 7499 7500 physpages = get_num_physpages(); 7501 codesize = _etext - _stext; 7502 datasize = _edata - _sdata; 7503 rosize = __end_rodata - __start_rodata; 7504 bss_size = __bss_stop - __bss_start; 7505 init_data_size = __init_end - __init_begin; 7506 init_code_size = _einittext - _sinittext; 7507 7508 /* 7509 * Detect special cases and adjust section sizes accordingly: 7510 * 1) .init.* may be embedded into .data sections 7511 * 2) .init.text.* may be out of [__init_begin, __init_end], 7512 * please refer to arch/tile/kernel/vmlinux.lds.S. 7513 * 3) .rodata.* may be embedded into .text or .data sections. 7514 */ 7515 #define adj_init_size(start, end, size, pos, adj) \ 7516 do { \ 7517 if (start <= pos && pos < end && size > adj) \ 7518 size -= adj; \ 7519 } while (0) 7520 7521 adj_init_size(__init_begin, __init_end, init_data_size, 7522 _sinittext, init_code_size); 7523 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 7524 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 7525 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 7526 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 7527 7528 #undef adj_init_size 7529 7530 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 7531 #ifdef CONFIG_HIGHMEM 7532 ", %luK highmem" 7533 #endif 7534 "%s%s)\n", 7535 nr_free_pages() << (PAGE_SHIFT - 10), 7536 physpages << (PAGE_SHIFT - 10), 7537 codesize >> 10, datasize >> 10, rosize >> 10, 7538 (init_data_size + init_code_size) >> 10, bss_size >> 10, 7539 (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10), 7540 totalcma_pages << (PAGE_SHIFT - 10), 7541 #ifdef CONFIG_HIGHMEM 7542 totalhigh_pages() << (PAGE_SHIFT - 10), 7543 #endif 7544 str ? ", " : "", str ? str : ""); 7545 } 7546 7547 /** 7548 * set_dma_reserve - set the specified number of pages reserved in the first zone 7549 * @new_dma_reserve: The number of pages to mark reserved 7550 * 7551 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 7552 * In the DMA zone, a significant percentage may be consumed by kernel image 7553 * and other unfreeable allocations which can skew the watermarks badly. This 7554 * function may optionally be used to account for unfreeable pages in the 7555 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 7556 * smaller per-cpu batchsize. 7557 */ 7558 void __init set_dma_reserve(unsigned long new_dma_reserve) 7559 { 7560 dma_reserve = new_dma_reserve; 7561 } 7562 7563 void __init free_area_init(unsigned long *zones_size) 7564 { 7565 zero_resv_unavail(); 7566 free_area_init_node(0, zones_size, 7567 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); 7568 } 7569 7570 static int page_alloc_cpu_dead(unsigned int cpu) 7571 { 7572 7573 lru_add_drain_cpu(cpu); 7574 drain_pages(cpu); 7575 7576 /* 7577 * Spill the event counters of the dead processor 7578 * into the current processors event counters. 7579 * This artificially elevates the count of the current 7580 * processor. 7581 */ 7582 vm_events_fold_cpu(cpu); 7583 7584 /* 7585 * Zero the differential counters of the dead processor 7586 * so that the vm statistics are consistent. 7587 * 7588 * This is only okay since the processor is dead and cannot 7589 * race with what we are doing. 7590 */ 7591 cpu_vm_stats_fold(cpu); 7592 return 0; 7593 } 7594 7595 #ifdef CONFIG_NUMA 7596 int hashdist = HASHDIST_DEFAULT; 7597 7598 static int __init set_hashdist(char *str) 7599 { 7600 if (!str) 7601 return 0; 7602 hashdist = simple_strtoul(str, &str, 0); 7603 return 1; 7604 } 7605 __setup("hashdist=", set_hashdist); 7606 #endif 7607 7608 void __init page_alloc_init(void) 7609 { 7610 int ret; 7611 7612 #ifdef CONFIG_NUMA 7613 if (num_node_state(N_MEMORY) == 1) 7614 hashdist = 0; 7615 #endif 7616 7617 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD, 7618 "mm/page_alloc:dead", NULL, 7619 page_alloc_cpu_dead); 7620 WARN_ON(ret < 0); 7621 } 7622 7623 /* 7624 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 7625 * or min_free_kbytes changes. 7626 */ 7627 static void calculate_totalreserve_pages(void) 7628 { 7629 struct pglist_data *pgdat; 7630 unsigned long reserve_pages = 0; 7631 enum zone_type i, j; 7632 7633 for_each_online_pgdat(pgdat) { 7634 7635 pgdat->totalreserve_pages = 0; 7636 7637 for (i = 0; i < MAX_NR_ZONES; i++) { 7638 struct zone *zone = pgdat->node_zones + i; 7639 long max = 0; 7640 unsigned long managed_pages = zone_managed_pages(zone); 7641 7642 /* Find valid and maximum lowmem_reserve in the zone */ 7643 for (j = i; j < MAX_NR_ZONES; j++) { 7644 if (zone->lowmem_reserve[j] > max) 7645 max = zone->lowmem_reserve[j]; 7646 } 7647 7648 /* we treat the high watermark as reserved pages. */ 7649 max += high_wmark_pages(zone); 7650 7651 if (max > managed_pages) 7652 max = managed_pages; 7653 7654 pgdat->totalreserve_pages += max; 7655 7656 reserve_pages += max; 7657 } 7658 } 7659 totalreserve_pages = reserve_pages; 7660 } 7661 7662 /* 7663 * setup_per_zone_lowmem_reserve - called whenever 7664 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 7665 * has a correct pages reserved value, so an adequate number of 7666 * pages are left in the zone after a successful __alloc_pages(). 7667 */ 7668 static void setup_per_zone_lowmem_reserve(void) 7669 { 7670 struct pglist_data *pgdat; 7671 enum zone_type j, idx; 7672 7673 for_each_online_pgdat(pgdat) { 7674 for (j = 0; j < MAX_NR_ZONES; j++) { 7675 struct zone *zone = pgdat->node_zones + j; 7676 unsigned long managed_pages = zone_managed_pages(zone); 7677 7678 zone->lowmem_reserve[j] = 0; 7679 7680 idx = j; 7681 while (idx) { 7682 struct zone *lower_zone; 7683 7684 idx--; 7685 lower_zone = pgdat->node_zones + idx; 7686 7687 if (sysctl_lowmem_reserve_ratio[idx] < 1) { 7688 sysctl_lowmem_reserve_ratio[idx] = 0; 7689 lower_zone->lowmem_reserve[j] = 0; 7690 } else { 7691 lower_zone->lowmem_reserve[j] = 7692 managed_pages / sysctl_lowmem_reserve_ratio[idx]; 7693 } 7694 managed_pages += zone_managed_pages(lower_zone); 7695 } 7696 } 7697 } 7698 7699 /* update totalreserve_pages */ 7700 calculate_totalreserve_pages(); 7701 } 7702 7703 static void __setup_per_zone_wmarks(void) 7704 { 7705 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 7706 unsigned long lowmem_pages = 0; 7707 struct zone *zone; 7708 unsigned long flags; 7709 7710 /* Calculate total number of !ZONE_HIGHMEM pages */ 7711 for_each_zone(zone) { 7712 if (!is_highmem(zone)) 7713 lowmem_pages += zone_managed_pages(zone); 7714 } 7715 7716 for_each_zone(zone) { 7717 u64 tmp; 7718 7719 spin_lock_irqsave(&zone->lock, flags); 7720 tmp = (u64)pages_min * zone_managed_pages(zone); 7721 do_div(tmp, lowmem_pages); 7722 if (is_highmem(zone)) { 7723 /* 7724 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 7725 * need highmem pages, so cap pages_min to a small 7726 * value here. 7727 * 7728 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 7729 * deltas control async page reclaim, and so should 7730 * not be capped for highmem. 7731 */ 7732 unsigned long min_pages; 7733 7734 min_pages = zone_managed_pages(zone) / 1024; 7735 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 7736 zone->_watermark[WMARK_MIN] = min_pages; 7737 } else { 7738 /* 7739 * If it's a lowmem zone, reserve a number of pages 7740 * proportionate to the zone's size. 7741 */ 7742 zone->_watermark[WMARK_MIN] = tmp; 7743 } 7744 7745 /* 7746 * Set the kswapd watermarks distance according to the 7747 * scale factor in proportion to available memory, but 7748 * ensure a minimum size on small systems. 7749 */ 7750 tmp = max_t(u64, tmp >> 2, 7751 mult_frac(zone_managed_pages(zone), 7752 watermark_scale_factor, 10000)); 7753 7754 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 7755 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2; 7756 zone->watermark_boost = 0; 7757 7758 spin_unlock_irqrestore(&zone->lock, flags); 7759 } 7760 7761 /* update totalreserve_pages */ 7762 calculate_totalreserve_pages(); 7763 } 7764 7765 /** 7766 * setup_per_zone_wmarks - called when min_free_kbytes changes 7767 * or when memory is hot-{added|removed} 7768 * 7769 * Ensures that the watermark[min,low,high] values for each zone are set 7770 * correctly with respect to min_free_kbytes. 7771 */ 7772 void setup_per_zone_wmarks(void) 7773 { 7774 static DEFINE_SPINLOCK(lock); 7775 7776 spin_lock(&lock); 7777 __setup_per_zone_wmarks(); 7778 spin_unlock(&lock); 7779 } 7780 7781 /* 7782 * Initialise min_free_kbytes. 7783 * 7784 * For small machines we want it small (128k min). For large machines 7785 * we want it large (64MB max). But it is not linear, because network 7786 * bandwidth does not increase linearly with machine size. We use 7787 * 7788 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 7789 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 7790 * 7791 * which yields 7792 * 7793 * 16MB: 512k 7794 * 32MB: 724k 7795 * 64MB: 1024k 7796 * 128MB: 1448k 7797 * 256MB: 2048k 7798 * 512MB: 2896k 7799 * 1024MB: 4096k 7800 * 2048MB: 5792k 7801 * 4096MB: 8192k 7802 * 8192MB: 11584k 7803 * 16384MB: 16384k 7804 */ 7805 int __meminit init_per_zone_wmark_min(void) 7806 { 7807 unsigned long lowmem_kbytes; 7808 int new_min_free_kbytes; 7809 7810 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 7811 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 7812 7813 if (new_min_free_kbytes > user_min_free_kbytes) { 7814 min_free_kbytes = new_min_free_kbytes; 7815 if (min_free_kbytes < 128) 7816 min_free_kbytes = 128; 7817 if (min_free_kbytes > 65536) 7818 min_free_kbytes = 65536; 7819 } else { 7820 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 7821 new_min_free_kbytes, user_min_free_kbytes); 7822 } 7823 setup_per_zone_wmarks(); 7824 refresh_zone_stat_thresholds(); 7825 setup_per_zone_lowmem_reserve(); 7826 7827 #ifdef CONFIG_NUMA 7828 setup_min_unmapped_ratio(); 7829 setup_min_slab_ratio(); 7830 #endif 7831 7832 return 0; 7833 } 7834 core_initcall(init_per_zone_wmark_min) 7835 7836 /* 7837 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 7838 * that we can call two helper functions whenever min_free_kbytes 7839 * changes. 7840 */ 7841 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 7842 void __user *buffer, size_t *length, loff_t *ppos) 7843 { 7844 int rc; 7845 7846 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7847 if (rc) 7848 return rc; 7849 7850 if (write) { 7851 user_min_free_kbytes = min_free_kbytes; 7852 setup_per_zone_wmarks(); 7853 } 7854 return 0; 7855 } 7856 7857 int watermark_boost_factor_sysctl_handler(struct ctl_table *table, int write, 7858 void __user *buffer, size_t *length, loff_t *ppos) 7859 { 7860 int rc; 7861 7862 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7863 if (rc) 7864 return rc; 7865 7866 return 0; 7867 } 7868 7869 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 7870 void __user *buffer, size_t *length, loff_t *ppos) 7871 { 7872 int rc; 7873 7874 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7875 if (rc) 7876 return rc; 7877 7878 if (write) 7879 setup_per_zone_wmarks(); 7880 7881 return 0; 7882 } 7883 7884 #ifdef CONFIG_NUMA 7885 static void setup_min_unmapped_ratio(void) 7886 { 7887 pg_data_t *pgdat; 7888 struct zone *zone; 7889 7890 for_each_online_pgdat(pgdat) 7891 pgdat->min_unmapped_pages = 0; 7892 7893 for_each_zone(zone) 7894 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * 7895 sysctl_min_unmapped_ratio) / 100; 7896 } 7897 7898 7899 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 7900 void __user *buffer, size_t *length, loff_t *ppos) 7901 { 7902 int rc; 7903 7904 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7905 if (rc) 7906 return rc; 7907 7908 setup_min_unmapped_ratio(); 7909 7910 return 0; 7911 } 7912 7913 static void setup_min_slab_ratio(void) 7914 { 7915 pg_data_t *pgdat; 7916 struct zone *zone; 7917 7918 for_each_online_pgdat(pgdat) 7919 pgdat->min_slab_pages = 0; 7920 7921 for_each_zone(zone) 7922 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * 7923 sysctl_min_slab_ratio) / 100; 7924 } 7925 7926 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 7927 void __user *buffer, size_t *length, loff_t *ppos) 7928 { 7929 int rc; 7930 7931 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7932 if (rc) 7933 return rc; 7934 7935 setup_min_slab_ratio(); 7936 7937 return 0; 7938 } 7939 #endif 7940 7941 /* 7942 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 7943 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 7944 * whenever sysctl_lowmem_reserve_ratio changes. 7945 * 7946 * The reserve ratio obviously has absolutely no relation with the 7947 * minimum watermarks. The lowmem reserve ratio can only make sense 7948 * if in function of the boot time zone sizes. 7949 */ 7950 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 7951 void __user *buffer, size_t *length, loff_t *ppos) 7952 { 7953 proc_dointvec_minmax(table, write, buffer, length, ppos); 7954 setup_per_zone_lowmem_reserve(); 7955 return 0; 7956 } 7957 7958 /* 7959 * percpu_pagelist_fraction - changes the pcp->high for each zone on each 7960 * cpu. It is the fraction of total pages in each zone that a hot per cpu 7961 * pagelist can have before it gets flushed back to buddy allocator. 7962 */ 7963 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, 7964 void __user *buffer, size_t *length, loff_t *ppos) 7965 { 7966 struct zone *zone; 7967 int old_percpu_pagelist_fraction; 7968 int ret; 7969 7970 mutex_lock(&pcp_batch_high_lock); 7971 old_percpu_pagelist_fraction = percpu_pagelist_fraction; 7972 7973 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 7974 if (!write || ret < 0) 7975 goto out; 7976 7977 /* Sanity checking to avoid pcp imbalance */ 7978 if (percpu_pagelist_fraction && 7979 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { 7980 percpu_pagelist_fraction = old_percpu_pagelist_fraction; 7981 ret = -EINVAL; 7982 goto out; 7983 } 7984 7985 /* No change? */ 7986 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) 7987 goto out; 7988 7989 for_each_populated_zone(zone) { 7990 unsigned int cpu; 7991 7992 for_each_possible_cpu(cpu) 7993 pageset_set_high_and_batch(zone, 7994 per_cpu_ptr(zone->pageset, cpu)); 7995 } 7996 out: 7997 mutex_unlock(&pcp_batch_high_lock); 7998 return ret; 7999 } 8000 8001 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 8002 /* 8003 * Returns the number of pages that arch has reserved but 8004 * is not known to alloc_large_system_hash(). 8005 */ 8006 static unsigned long __init arch_reserved_kernel_pages(void) 8007 { 8008 return 0; 8009 } 8010 #endif 8011 8012 /* 8013 * Adaptive scale is meant to reduce sizes of hash tables on large memory 8014 * machines. As memory size is increased the scale is also increased but at 8015 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 8016 * quadruples the scale is increased by one, which means the size of hash table 8017 * only doubles, instead of quadrupling as well. 8018 * Because 32-bit systems cannot have large physical memory, where this scaling 8019 * makes sense, it is disabled on such platforms. 8020 */ 8021 #if __BITS_PER_LONG > 32 8022 #define ADAPT_SCALE_BASE (64ul << 30) 8023 #define ADAPT_SCALE_SHIFT 2 8024 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 8025 #endif 8026 8027 /* 8028 * allocate a large system hash table from bootmem 8029 * - it is assumed that the hash table must contain an exact power-of-2 8030 * quantity of entries 8031 * - limit is the number of hash buckets, not the total allocation size 8032 */ 8033 void *__init alloc_large_system_hash(const char *tablename, 8034 unsigned long bucketsize, 8035 unsigned long numentries, 8036 int scale, 8037 int flags, 8038 unsigned int *_hash_shift, 8039 unsigned int *_hash_mask, 8040 unsigned long low_limit, 8041 unsigned long high_limit) 8042 { 8043 unsigned long long max = high_limit; 8044 unsigned long log2qty, size; 8045 void *table = NULL; 8046 gfp_t gfp_flags; 8047 bool virt; 8048 8049 /* allow the kernel cmdline to have a say */ 8050 if (!numentries) { 8051 /* round applicable memory size up to nearest megabyte */ 8052 numentries = nr_kernel_pages; 8053 numentries -= arch_reserved_kernel_pages(); 8054 8055 /* It isn't necessary when PAGE_SIZE >= 1MB */ 8056 if (PAGE_SHIFT < 20) 8057 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 8058 8059 #if __BITS_PER_LONG > 32 8060 if (!high_limit) { 8061 unsigned long adapt; 8062 8063 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 8064 adapt <<= ADAPT_SCALE_SHIFT) 8065 scale++; 8066 } 8067 #endif 8068 8069 /* limit to 1 bucket per 2^scale bytes of low memory */ 8070 if (scale > PAGE_SHIFT) 8071 numentries >>= (scale - PAGE_SHIFT); 8072 else 8073 numentries <<= (PAGE_SHIFT - scale); 8074 8075 /* Make sure we've got at least a 0-order allocation.. */ 8076 if (unlikely(flags & HASH_SMALL)) { 8077 /* Makes no sense without HASH_EARLY */ 8078 WARN_ON(!(flags & HASH_EARLY)); 8079 if (!(numentries >> *_hash_shift)) { 8080 numentries = 1UL << *_hash_shift; 8081 BUG_ON(!numentries); 8082 } 8083 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 8084 numentries = PAGE_SIZE / bucketsize; 8085 } 8086 numentries = roundup_pow_of_two(numentries); 8087 8088 /* limit allocation size to 1/16 total memory by default */ 8089 if (max == 0) { 8090 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 8091 do_div(max, bucketsize); 8092 } 8093 max = min(max, 0x80000000ULL); 8094 8095 if (numentries < low_limit) 8096 numentries = low_limit; 8097 if (numentries > max) 8098 numentries = max; 8099 8100 log2qty = ilog2(numentries); 8101 8102 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 8103 do { 8104 virt = false; 8105 size = bucketsize << log2qty; 8106 if (flags & HASH_EARLY) { 8107 if (flags & HASH_ZERO) 8108 table = memblock_alloc(size, SMP_CACHE_BYTES); 8109 else 8110 table = memblock_alloc_raw(size, 8111 SMP_CACHE_BYTES); 8112 } else if (get_order(size) >= MAX_ORDER || hashdist) { 8113 table = __vmalloc(size, gfp_flags, PAGE_KERNEL); 8114 virt = true; 8115 } else { 8116 /* 8117 * If bucketsize is not a power-of-two, we may free 8118 * some pages at the end of hash table which 8119 * alloc_pages_exact() automatically does 8120 */ 8121 table = alloc_pages_exact(size, gfp_flags); 8122 kmemleak_alloc(table, size, 1, gfp_flags); 8123 } 8124 } while (!table && size > PAGE_SIZE && --log2qty); 8125 8126 if (!table) 8127 panic("Failed to allocate %s hash table\n", tablename); 8128 8129 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 8130 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, 8131 virt ? "vmalloc" : "linear"); 8132 8133 if (_hash_shift) 8134 *_hash_shift = log2qty; 8135 if (_hash_mask) 8136 *_hash_mask = (1 << log2qty) - 1; 8137 8138 return table; 8139 } 8140 8141 /* 8142 * This function checks whether pageblock includes unmovable pages or not. 8143 * If @count is not zero, it is okay to include less @count unmovable pages 8144 * 8145 * PageLRU check without isolation or lru_lock could race so that 8146 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable 8147 * check without lock_page also may miss some movable non-lru pages at 8148 * race condition. So you can't expect this function should be exact. 8149 */ 8150 bool has_unmovable_pages(struct zone *zone, struct page *page, int count, 8151 int migratetype, int flags) 8152 { 8153 unsigned long found; 8154 unsigned long iter = 0; 8155 unsigned long pfn = page_to_pfn(page); 8156 const char *reason = "unmovable page"; 8157 8158 /* 8159 * TODO we could make this much more efficient by not checking every 8160 * page in the range if we know all of them are in MOVABLE_ZONE and 8161 * that the movable zone guarantees that pages are migratable but 8162 * the later is not the case right now unfortunatelly. E.g. movablecore 8163 * can still lead to having bootmem allocations in zone_movable. 8164 */ 8165 8166 if (is_migrate_cma_page(page)) { 8167 /* 8168 * CMA allocations (alloc_contig_range) really need to mark 8169 * isolate CMA pageblocks even when they are not movable in fact 8170 * so consider them movable here. 8171 */ 8172 if (is_migrate_cma(migratetype)) 8173 return false; 8174 8175 reason = "CMA page"; 8176 goto unmovable; 8177 } 8178 8179 for (found = 0; iter < pageblock_nr_pages; iter++) { 8180 unsigned long check = pfn + iter; 8181 8182 if (!pfn_valid_within(check)) 8183 continue; 8184 8185 page = pfn_to_page(check); 8186 8187 if (PageReserved(page)) 8188 goto unmovable; 8189 8190 /* 8191 * If the zone is movable and we have ruled out all reserved 8192 * pages then it should be reasonably safe to assume the rest 8193 * is movable. 8194 */ 8195 if (zone_idx(zone) == ZONE_MOVABLE) 8196 continue; 8197 8198 /* 8199 * Hugepages are not in LRU lists, but they're movable. 8200 * We need not scan over tail pages because we don't 8201 * handle each tail page individually in migration. 8202 */ 8203 if (PageHuge(page)) { 8204 struct page *head = compound_head(page); 8205 unsigned int skip_pages; 8206 8207 if (!hugepage_migration_supported(page_hstate(head))) 8208 goto unmovable; 8209 8210 skip_pages = compound_nr(head) - (page - head); 8211 iter += skip_pages - 1; 8212 continue; 8213 } 8214 8215 /* 8216 * We can't use page_count without pin a page 8217 * because another CPU can free compound page. 8218 * This check already skips compound tails of THP 8219 * because their page->_refcount is zero at all time. 8220 */ 8221 if (!page_ref_count(page)) { 8222 if (PageBuddy(page)) 8223 iter += (1 << page_order(page)) - 1; 8224 continue; 8225 } 8226 8227 /* 8228 * The HWPoisoned page may be not in buddy system, and 8229 * page_count() is not 0. 8230 */ 8231 if ((flags & SKIP_HWPOISON) && PageHWPoison(page)) 8232 continue; 8233 8234 if (__PageMovable(page)) 8235 continue; 8236 8237 if (!PageLRU(page)) 8238 found++; 8239 /* 8240 * If there are RECLAIMABLE pages, we need to check 8241 * it. But now, memory offline itself doesn't call 8242 * shrink_node_slabs() and it still to be fixed. 8243 */ 8244 /* 8245 * If the page is not RAM, page_count()should be 0. 8246 * we don't need more check. This is an _used_ not-movable page. 8247 * 8248 * The problematic thing here is PG_reserved pages. PG_reserved 8249 * is set to both of a memory hole page and a _used_ kernel 8250 * page at boot. 8251 */ 8252 if (found > count) 8253 goto unmovable; 8254 } 8255 return false; 8256 unmovable: 8257 WARN_ON_ONCE(zone_idx(zone) == ZONE_MOVABLE); 8258 if (flags & REPORT_FAILURE) 8259 dump_page(pfn_to_page(pfn + iter), reason); 8260 return true; 8261 } 8262 8263 #ifdef CONFIG_CONTIG_ALLOC 8264 static unsigned long pfn_max_align_down(unsigned long pfn) 8265 { 8266 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, 8267 pageblock_nr_pages) - 1); 8268 } 8269 8270 static unsigned long pfn_max_align_up(unsigned long pfn) 8271 { 8272 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, 8273 pageblock_nr_pages)); 8274 } 8275 8276 /* [start, end) must belong to a single zone. */ 8277 static int __alloc_contig_migrate_range(struct compact_control *cc, 8278 unsigned long start, unsigned long end) 8279 { 8280 /* This function is based on compact_zone() from compaction.c. */ 8281 unsigned long nr_reclaimed; 8282 unsigned long pfn = start; 8283 unsigned int tries = 0; 8284 int ret = 0; 8285 8286 migrate_prep(); 8287 8288 while (pfn < end || !list_empty(&cc->migratepages)) { 8289 if (fatal_signal_pending(current)) { 8290 ret = -EINTR; 8291 break; 8292 } 8293 8294 if (list_empty(&cc->migratepages)) { 8295 cc->nr_migratepages = 0; 8296 pfn = isolate_migratepages_range(cc, pfn, end); 8297 if (!pfn) { 8298 ret = -EINTR; 8299 break; 8300 } 8301 tries = 0; 8302 } else if (++tries == 5) { 8303 ret = ret < 0 ? ret : -EBUSY; 8304 break; 8305 } 8306 8307 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 8308 &cc->migratepages); 8309 cc->nr_migratepages -= nr_reclaimed; 8310 8311 ret = migrate_pages(&cc->migratepages, alloc_migrate_target, 8312 NULL, 0, cc->mode, MR_CONTIG_RANGE); 8313 } 8314 if (ret < 0) { 8315 putback_movable_pages(&cc->migratepages); 8316 return ret; 8317 } 8318 return 0; 8319 } 8320 8321 /** 8322 * alloc_contig_range() -- tries to allocate given range of pages 8323 * @start: start PFN to allocate 8324 * @end: one-past-the-last PFN to allocate 8325 * @migratetype: migratetype of the underlaying pageblocks (either 8326 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 8327 * in range must have the same migratetype and it must 8328 * be either of the two. 8329 * @gfp_mask: GFP mask to use during compaction 8330 * 8331 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 8332 * aligned. The PFN range must belong to a single zone. 8333 * 8334 * The first thing this routine does is attempt to MIGRATE_ISOLATE all 8335 * pageblocks in the range. Once isolated, the pageblocks should not 8336 * be modified by others. 8337 * 8338 * Return: zero on success or negative error code. On success all 8339 * pages which PFN is in [start, end) are allocated for the caller and 8340 * need to be freed with free_contig_range(). 8341 */ 8342 int alloc_contig_range(unsigned long start, unsigned long end, 8343 unsigned migratetype, gfp_t gfp_mask) 8344 { 8345 unsigned long outer_start, outer_end; 8346 unsigned int order; 8347 int ret = 0; 8348 8349 struct compact_control cc = { 8350 .nr_migratepages = 0, 8351 .order = -1, 8352 .zone = page_zone(pfn_to_page(start)), 8353 .mode = MIGRATE_SYNC, 8354 .ignore_skip_hint = true, 8355 .no_set_skip_hint = true, 8356 .gfp_mask = current_gfp_context(gfp_mask), 8357 }; 8358 INIT_LIST_HEAD(&cc.migratepages); 8359 8360 /* 8361 * What we do here is we mark all pageblocks in range as 8362 * MIGRATE_ISOLATE. Because pageblock and max order pages may 8363 * have different sizes, and due to the way page allocator 8364 * work, we align the range to biggest of the two pages so 8365 * that page allocator won't try to merge buddies from 8366 * different pageblocks and change MIGRATE_ISOLATE to some 8367 * other migration type. 8368 * 8369 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 8370 * migrate the pages from an unaligned range (ie. pages that 8371 * we are interested in). This will put all the pages in 8372 * range back to page allocator as MIGRATE_ISOLATE. 8373 * 8374 * When this is done, we take the pages in range from page 8375 * allocator removing them from the buddy system. This way 8376 * page allocator will never consider using them. 8377 * 8378 * This lets us mark the pageblocks back as 8379 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 8380 * aligned range but not in the unaligned, original range are 8381 * put back to page allocator so that buddy can use them. 8382 */ 8383 8384 ret = start_isolate_page_range(pfn_max_align_down(start), 8385 pfn_max_align_up(end), migratetype, 0); 8386 if (ret < 0) 8387 return ret; 8388 8389 /* 8390 * In case of -EBUSY, we'd like to know which page causes problem. 8391 * So, just fall through. test_pages_isolated() has a tracepoint 8392 * which will report the busy page. 8393 * 8394 * It is possible that busy pages could become available before 8395 * the call to test_pages_isolated, and the range will actually be 8396 * allocated. So, if we fall through be sure to clear ret so that 8397 * -EBUSY is not accidentally used or returned to caller. 8398 */ 8399 ret = __alloc_contig_migrate_range(&cc, start, end); 8400 if (ret && ret != -EBUSY) 8401 goto done; 8402 ret =0; 8403 8404 /* 8405 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 8406 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 8407 * more, all pages in [start, end) are free in page allocator. 8408 * What we are going to do is to allocate all pages from 8409 * [start, end) (that is remove them from page allocator). 8410 * 8411 * The only problem is that pages at the beginning and at the 8412 * end of interesting range may be not aligned with pages that 8413 * page allocator holds, ie. they can be part of higher order 8414 * pages. Because of this, we reserve the bigger range and 8415 * once this is done free the pages we are not interested in. 8416 * 8417 * We don't have to hold zone->lock here because the pages are 8418 * isolated thus they won't get removed from buddy. 8419 */ 8420 8421 lru_add_drain_all(); 8422 8423 order = 0; 8424 outer_start = start; 8425 while (!PageBuddy(pfn_to_page(outer_start))) { 8426 if (++order >= MAX_ORDER) { 8427 outer_start = start; 8428 break; 8429 } 8430 outer_start &= ~0UL << order; 8431 } 8432 8433 if (outer_start != start) { 8434 order = page_order(pfn_to_page(outer_start)); 8435 8436 /* 8437 * outer_start page could be small order buddy page and 8438 * it doesn't include start page. Adjust outer_start 8439 * in this case to report failed page properly 8440 * on tracepoint in test_pages_isolated() 8441 */ 8442 if (outer_start + (1UL << order) <= start) 8443 outer_start = start; 8444 } 8445 8446 /* Make sure the range is really isolated. */ 8447 if (test_pages_isolated(outer_start, end, false)) { 8448 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n", 8449 __func__, outer_start, end); 8450 ret = -EBUSY; 8451 goto done; 8452 } 8453 8454 /* Grab isolated pages from freelists. */ 8455 outer_end = isolate_freepages_range(&cc, outer_start, end); 8456 if (!outer_end) { 8457 ret = -EBUSY; 8458 goto done; 8459 } 8460 8461 /* Free head and tail (if any) */ 8462 if (start != outer_start) 8463 free_contig_range(outer_start, start - outer_start); 8464 if (end != outer_end) 8465 free_contig_range(end, outer_end - end); 8466 8467 done: 8468 undo_isolate_page_range(pfn_max_align_down(start), 8469 pfn_max_align_up(end), migratetype); 8470 return ret; 8471 } 8472 #endif /* CONFIG_CONTIG_ALLOC */ 8473 8474 void free_contig_range(unsigned long pfn, unsigned int nr_pages) 8475 { 8476 unsigned int count = 0; 8477 8478 for (; nr_pages--; pfn++) { 8479 struct page *page = pfn_to_page(pfn); 8480 8481 count += page_count(page) != 1; 8482 __free_page(page); 8483 } 8484 WARN(count != 0, "%d pages are still in use!\n", count); 8485 } 8486 8487 #ifdef CONFIG_MEMORY_HOTPLUG 8488 /* 8489 * The zone indicated has a new number of managed_pages; batch sizes and percpu 8490 * page high values need to be recalulated. 8491 */ 8492 void __meminit zone_pcp_update(struct zone *zone) 8493 { 8494 unsigned cpu; 8495 mutex_lock(&pcp_batch_high_lock); 8496 for_each_possible_cpu(cpu) 8497 pageset_set_high_and_batch(zone, 8498 per_cpu_ptr(zone->pageset, cpu)); 8499 mutex_unlock(&pcp_batch_high_lock); 8500 } 8501 #endif 8502 8503 void zone_pcp_reset(struct zone *zone) 8504 { 8505 unsigned long flags; 8506 int cpu; 8507 struct per_cpu_pageset *pset; 8508 8509 /* avoid races with drain_pages() */ 8510 local_irq_save(flags); 8511 if (zone->pageset != &boot_pageset) { 8512 for_each_online_cpu(cpu) { 8513 pset = per_cpu_ptr(zone->pageset, cpu); 8514 drain_zonestat(zone, pset); 8515 } 8516 free_percpu(zone->pageset); 8517 zone->pageset = &boot_pageset; 8518 } 8519 local_irq_restore(flags); 8520 } 8521 8522 #ifdef CONFIG_MEMORY_HOTREMOVE 8523 /* 8524 * All pages in the range must be in a single zone and isolated 8525 * before calling this. 8526 */ 8527 unsigned long 8528 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 8529 { 8530 struct page *page; 8531 struct zone *zone; 8532 unsigned int order, i; 8533 unsigned long pfn; 8534 unsigned long flags; 8535 unsigned long offlined_pages = 0; 8536 8537 /* find the first valid pfn */ 8538 for (pfn = start_pfn; pfn < end_pfn; pfn++) 8539 if (pfn_valid(pfn)) 8540 break; 8541 if (pfn == end_pfn) 8542 return offlined_pages; 8543 8544 offline_mem_sections(pfn, end_pfn); 8545 zone = page_zone(pfn_to_page(pfn)); 8546 spin_lock_irqsave(&zone->lock, flags); 8547 pfn = start_pfn; 8548 while (pfn < end_pfn) { 8549 if (!pfn_valid(pfn)) { 8550 pfn++; 8551 continue; 8552 } 8553 page = pfn_to_page(pfn); 8554 /* 8555 * The HWPoisoned page may be not in buddy system, and 8556 * page_count() is not 0. 8557 */ 8558 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 8559 pfn++; 8560 SetPageReserved(page); 8561 offlined_pages++; 8562 continue; 8563 } 8564 8565 BUG_ON(page_count(page)); 8566 BUG_ON(!PageBuddy(page)); 8567 order = page_order(page); 8568 offlined_pages += 1 << order; 8569 #ifdef CONFIG_DEBUG_VM 8570 pr_info("remove from free list %lx %d %lx\n", 8571 pfn, 1 << order, end_pfn); 8572 #endif 8573 del_page_from_free_area(page, &zone->free_area[order]); 8574 for (i = 0; i < (1 << order); i++) 8575 SetPageReserved((page+i)); 8576 pfn += (1 << order); 8577 } 8578 spin_unlock_irqrestore(&zone->lock, flags); 8579 8580 return offlined_pages; 8581 } 8582 #endif 8583 8584 bool is_free_buddy_page(struct page *page) 8585 { 8586 struct zone *zone = page_zone(page); 8587 unsigned long pfn = page_to_pfn(page); 8588 unsigned long flags; 8589 unsigned int order; 8590 8591 spin_lock_irqsave(&zone->lock, flags); 8592 for (order = 0; order < MAX_ORDER; order++) { 8593 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8594 8595 if (PageBuddy(page_head) && page_order(page_head) >= order) 8596 break; 8597 } 8598 spin_unlock_irqrestore(&zone->lock, flags); 8599 8600 return order < MAX_ORDER; 8601 } 8602 8603 #ifdef CONFIG_MEMORY_FAILURE 8604 /* 8605 * Set PG_hwpoison flag if a given page is confirmed to be a free page. This 8606 * test is performed under the zone lock to prevent a race against page 8607 * allocation. 8608 */ 8609 bool set_hwpoison_free_buddy_page(struct page *page) 8610 { 8611 struct zone *zone = page_zone(page); 8612 unsigned long pfn = page_to_pfn(page); 8613 unsigned long flags; 8614 unsigned int order; 8615 bool hwpoisoned = false; 8616 8617 spin_lock_irqsave(&zone->lock, flags); 8618 for (order = 0; order < MAX_ORDER; order++) { 8619 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8620 8621 if (PageBuddy(page_head) && page_order(page_head) >= order) { 8622 if (!TestSetPageHWPoison(page)) 8623 hwpoisoned = true; 8624 break; 8625 } 8626 } 8627 spin_unlock_irqrestore(&zone->lock, flags); 8628 8629 return hwpoisoned; 8630 } 8631 #endif 8632