1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/page_alloc.c 4 * 5 * Manages the free list, the system allocates free pages here. 6 * Note that kmalloc() lives in slab.c 7 * 8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 9 * Swap reorganised 29.12.95, Stephen Tweedie 10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 16 */ 17 18 #include <linux/stddef.h> 19 #include <linux/mm.h> 20 #include <linux/highmem.h> 21 #include <linux/swap.h> 22 #include <linux/interrupt.h> 23 #include <linux/pagemap.h> 24 #include <linux/jiffies.h> 25 #include <linux/memblock.h> 26 #include <linux/compiler.h> 27 #include <linux/kernel.h> 28 #include <linux/kasan.h> 29 #include <linux/module.h> 30 #include <linux/suspend.h> 31 #include <linux/pagevec.h> 32 #include <linux/blkdev.h> 33 #include <linux/slab.h> 34 #include <linux/ratelimit.h> 35 #include <linux/oom.h> 36 #include <linux/topology.h> 37 #include <linux/sysctl.h> 38 #include <linux/cpu.h> 39 #include <linux/cpuset.h> 40 #include <linux/memory_hotplug.h> 41 #include <linux/nodemask.h> 42 #include <linux/vmalloc.h> 43 #include <linux/vmstat.h> 44 #include <linux/mempolicy.h> 45 #include <linux/memremap.h> 46 #include <linux/stop_machine.h> 47 #include <linux/random.h> 48 #include <linux/sort.h> 49 #include <linux/pfn.h> 50 #include <linux/backing-dev.h> 51 #include <linux/fault-inject.h> 52 #include <linux/page-isolation.h> 53 #include <linux/debugobjects.h> 54 #include <linux/kmemleak.h> 55 #include <linux/compaction.h> 56 #include <trace/events/kmem.h> 57 #include <trace/events/oom.h> 58 #include <linux/prefetch.h> 59 #include <linux/mm_inline.h> 60 #include <linux/migrate.h> 61 #include <linux/hugetlb.h> 62 #include <linux/sched/rt.h> 63 #include <linux/sched/mm.h> 64 #include <linux/page_owner.h> 65 #include <linux/kthread.h> 66 #include <linux/memcontrol.h> 67 #include <linux/ftrace.h> 68 #include <linux/lockdep.h> 69 #include <linux/nmi.h> 70 #include <linux/psi.h> 71 #include <linux/padata.h> 72 #include <linux/khugepaged.h> 73 74 #include <asm/sections.h> 75 #include <asm/tlbflush.h> 76 #include <asm/div64.h> 77 #include "internal.h" 78 #include "shuffle.h" 79 #include "page_reporting.h" 80 81 /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */ 82 typedef int __bitwise fpi_t; 83 84 /* No special request */ 85 #define FPI_NONE ((__force fpi_t)0) 86 87 /* 88 * Skip free page reporting notification for the (possibly merged) page. 89 * This does not hinder free page reporting from grabbing the page, 90 * reporting it and marking it "reported" - it only skips notifying 91 * the free page reporting infrastructure about a newly freed page. For 92 * example, used when temporarily pulling a page from a freelist and 93 * putting it back unmodified. 94 */ 95 #define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0)) 96 97 /* 98 * Place the (possibly merged) page to the tail of the freelist. Will ignore 99 * page shuffling (relevant code - e.g., memory onlining - is expected to 100 * shuffle the whole zone). 101 * 102 * Note: No code should rely on this flag for correctness - it's purely 103 * to allow for optimizations when handing back either fresh pages 104 * (memory onlining) or untouched pages (page isolation, free page 105 * reporting). 106 */ 107 #define FPI_TO_TAIL ((__force fpi_t)BIT(1)) 108 109 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 110 static DEFINE_MUTEX(pcp_batch_high_lock); 111 #define MIN_PERCPU_PAGELIST_FRACTION (8) 112 113 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 114 DEFINE_PER_CPU(int, numa_node); 115 EXPORT_PER_CPU_SYMBOL(numa_node); 116 #endif 117 118 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); 119 120 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 121 /* 122 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 123 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 124 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 125 * defined in <linux/topology.h>. 126 */ 127 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 128 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 129 #endif 130 131 /* work_structs for global per-cpu drains */ 132 struct pcpu_drain { 133 struct zone *zone; 134 struct work_struct work; 135 }; 136 static DEFINE_MUTEX(pcpu_drain_mutex); 137 static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain); 138 139 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 140 volatile unsigned long latent_entropy __latent_entropy; 141 EXPORT_SYMBOL(latent_entropy); 142 #endif 143 144 /* 145 * Array of node states. 146 */ 147 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 148 [N_POSSIBLE] = NODE_MASK_ALL, 149 [N_ONLINE] = { { [0] = 1UL } }, 150 #ifndef CONFIG_NUMA 151 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 152 #ifdef CONFIG_HIGHMEM 153 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 154 #endif 155 [N_MEMORY] = { { [0] = 1UL } }, 156 [N_CPU] = { { [0] = 1UL } }, 157 #endif /* NUMA */ 158 }; 159 EXPORT_SYMBOL(node_states); 160 161 atomic_long_t _totalram_pages __read_mostly; 162 EXPORT_SYMBOL(_totalram_pages); 163 unsigned long totalreserve_pages __read_mostly; 164 unsigned long totalcma_pages __read_mostly; 165 166 int percpu_pagelist_fraction; 167 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 168 #ifdef CONFIG_INIT_ON_ALLOC_DEFAULT_ON 169 DEFINE_STATIC_KEY_TRUE(init_on_alloc); 170 #else 171 DEFINE_STATIC_KEY_FALSE(init_on_alloc); 172 #endif 173 EXPORT_SYMBOL(init_on_alloc); 174 175 #ifdef CONFIG_INIT_ON_FREE_DEFAULT_ON 176 DEFINE_STATIC_KEY_TRUE(init_on_free); 177 #else 178 DEFINE_STATIC_KEY_FALSE(init_on_free); 179 #endif 180 EXPORT_SYMBOL(init_on_free); 181 182 static int __init early_init_on_alloc(char *buf) 183 { 184 int ret; 185 bool bool_result; 186 187 ret = kstrtobool(buf, &bool_result); 188 if (ret) 189 return ret; 190 if (bool_result && page_poisoning_enabled()) 191 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_alloc\n"); 192 if (bool_result) 193 static_branch_enable(&init_on_alloc); 194 else 195 static_branch_disable(&init_on_alloc); 196 return 0; 197 } 198 early_param("init_on_alloc", early_init_on_alloc); 199 200 static int __init early_init_on_free(char *buf) 201 { 202 int ret; 203 bool bool_result; 204 205 ret = kstrtobool(buf, &bool_result); 206 if (ret) 207 return ret; 208 if (bool_result && page_poisoning_enabled()) 209 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_free\n"); 210 if (bool_result) 211 static_branch_enable(&init_on_free); 212 else 213 static_branch_disable(&init_on_free); 214 return 0; 215 } 216 early_param("init_on_free", early_init_on_free); 217 218 /* 219 * A cached value of the page's pageblock's migratetype, used when the page is 220 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 221 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 222 * Also the migratetype set in the page does not necessarily match the pcplist 223 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 224 * other index - this ensures that it will be put on the correct CMA freelist. 225 */ 226 static inline int get_pcppage_migratetype(struct page *page) 227 { 228 return page->index; 229 } 230 231 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 232 { 233 page->index = migratetype; 234 } 235 236 #ifdef CONFIG_PM_SLEEP 237 /* 238 * The following functions are used by the suspend/hibernate code to temporarily 239 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 240 * while devices are suspended. To avoid races with the suspend/hibernate code, 241 * they should always be called with system_transition_mutex held 242 * (gfp_allowed_mask also should only be modified with system_transition_mutex 243 * held, unless the suspend/hibernate code is guaranteed not to run in parallel 244 * with that modification). 245 */ 246 247 static gfp_t saved_gfp_mask; 248 249 void pm_restore_gfp_mask(void) 250 { 251 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 252 if (saved_gfp_mask) { 253 gfp_allowed_mask = saved_gfp_mask; 254 saved_gfp_mask = 0; 255 } 256 } 257 258 void pm_restrict_gfp_mask(void) 259 { 260 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 261 WARN_ON(saved_gfp_mask); 262 saved_gfp_mask = gfp_allowed_mask; 263 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 264 } 265 266 bool pm_suspended_storage(void) 267 { 268 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 269 return false; 270 return true; 271 } 272 #endif /* CONFIG_PM_SLEEP */ 273 274 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 275 unsigned int pageblock_order __read_mostly; 276 #endif 277 278 static void __free_pages_ok(struct page *page, unsigned int order, 279 fpi_t fpi_flags); 280 281 /* 282 * results with 256, 32 in the lowmem_reserve sysctl: 283 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 284 * 1G machine -> (16M dma, 784M normal, 224M high) 285 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 286 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 287 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 288 * 289 * TBD: should special case ZONE_DMA32 machines here - in those we normally 290 * don't need any ZONE_NORMAL reservation 291 */ 292 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { 293 #ifdef CONFIG_ZONE_DMA 294 [ZONE_DMA] = 256, 295 #endif 296 #ifdef CONFIG_ZONE_DMA32 297 [ZONE_DMA32] = 256, 298 #endif 299 [ZONE_NORMAL] = 32, 300 #ifdef CONFIG_HIGHMEM 301 [ZONE_HIGHMEM] = 0, 302 #endif 303 [ZONE_MOVABLE] = 0, 304 }; 305 306 static char * const zone_names[MAX_NR_ZONES] = { 307 #ifdef CONFIG_ZONE_DMA 308 "DMA", 309 #endif 310 #ifdef CONFIG_ZONE_DMA32 311 "DMA32", 312 #endif 313 "Normal", 314 #ifdef CONFIG_HIGHMEM 315 "HighMem", 316 #endif 317 "Movable", 318 #ifdef CONFIG_ZONE_DEVICE 319 "Device", 320 #endif 321 }; 322 323 const char * const migratetype_names[MIGRATE_TYPES] = { 324 "Unmovable", 325 "Movable", 326 "Reclaimable", 327 "HighAtomic", 328 #ifdef CONFIG_CMA 329 "CMA", 330 #endif 331 #ifdef CONFIG_MEMORY_ISOLATION 332 "Isolate", 333 #endif 334 }; 335 336 compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = { 337 [NULL_COMPOUND_DTOR] = NULL, 338 [COMPOUND_PAGE_DTOR] = free_compound_page, 339 #ifdef CONFIG_HUGETLB_PAGE 340 [HUGETLB_PAGE_DTOR] = free_huge_page, 341 #endif 342 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 343 [TRANSHUGE_PAGE_DTOR] = free_transhuge_page, 344 #endif 345 }; 346 347 int min_free_kbytes = 1024; 348 int user_min_free_kbytes = -1; 349 #ifdef CONFIG_DISCONTIGMEM 350 /* 351 * DiscontigMem defines memory ranges as separate pg_data_t even if the ranges 352 * are not on separate NUMA nodes. Functionally this works but with 353 * watermark_boost_factor, it can reclaim prematurely as the ranges can be 354 * quite small. By default, do not boost watermarks on discontigmem as in 355 * many cases very high-order allocations like THP are likely to be 356 * unsupported and the premature reclaim offsets the advantage of long-term 357 * fragmentation avoidance. 358 */ 359 int watermark_boost_factor __read_mostly; 360 #else 361 int watermark_boost_factor __read_mostly = 15000; 362 #endif 363 int watermark_scale_factor = 10; 364 365 static unsigned long nr_kernel_pages __initdata; 366 static unsigned long nr_all_pages __initdata; 367 static unsigned long dma_reserve __initdata; 368 369 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 370 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 371 static unsigned long required_kernelcore __initdata; 372 static unsigned long required_kernelcore_percent __initdata; 373 static unsigned long required_movablecore __initdata; 374 static unsigned long required_movablecore_percent __initdata; 375 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 376 static bool mirrored_kernelcore __meminitdata; 377 378 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 379 int movable_zone; 380 EXPORT_SYMBOL(movable_zone); 381 382 #if MAX_NUMNODES > 1 383 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; 384 unsigned int nr_online_nodes __read_mostly = 1; 385 EXPORT_SYMBOL(nr_node_ids); 386 EXPORT_SYMBOL(nr_online_nodes); 387 #endif 388 389 int page_group_by_mobility_disabled __read_mostly; 390 391 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 392 /* 393 * During boot we initialize deferred pages on-demand, as needed, but once 394 * page_alloc_init_late() has finished, the deferred pages are all initialized, 395 * and we can permanently disable that path. 396 */ 397 static DEFINE_STATIC_KEY_TRUE(deferred_pages); 398 399 /* 400 * Calling kasan_free_pages() only after deferred memory initialization 401 * has completed. Poisoning pages during deferred memory init will greatly 402 * lengthen the process and cause problem in large memory systems as the 403 * deferred pages initialization is done with interrupt disabled. 404 * 405 * Assuming that there will be no reference to those newly initialized 406 * pages before they are ever allocated, this should have no effect on 407 * KASAN memory tracking as the poison will be properly inserted at page 408 * allocation time. The only corner case is when pages are allocated by 409 * on-demand allocation and then freed again before the deferred pages 410 * initialization is done, but this is not likely to happen. 411 */ 412 static inline void kasan_free_nondeferred_pages(struct page *page, int order) 413 { 414 if (!static_branch_unlikely(&deferred_pages)) 415 kasan_free_pages(page, order); 416 } 417 418 /* Returns true if the struct page for the pfn is uninitialised */ 419 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 420 { 421 int nid = early_pfn_to_nid(pfn); 422 423 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 424 return true; 425 426 return false; 427 } 428 429 /* 430 * Returns true when the remaining initialisation should be deferred until 431 * later in the boot cycle when it can be parallelised. 432 */ 433 static bool __meminit 434 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 435 { 436 static unsigned long prev_end_pfn, nr_initialised; 437 438 /* 439 * prev_end_pfn static that contains the end of previous zone 440 * No need to protect because called very early in boot before smp_init. 441 */ 442 if (prev_end_pfn != end_pfn) { 443 prev_end_pfn = end_pfn; 444 nr_initialised = 0; 445 } 446 447 /* Always populate low zones for address-constrained allocations */ 448 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 449 return false; 450 451 /* 452 * We start only with one section of pages, more pages are added as 453 * needed until the rest of deferred pages are initialized. 454 */ 455 nr_initialised++; 456 if ((nr_initialised > PAGES_PER_SECTION) && 457 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 458 NODE_DATA(nid)->first_deferred_pfn = pfn; 459 return true; 460 } 461 return false; 462 } 463 #else 464 #define kasan_free_nondeferred_pages(p, o) kasan_free_pages(p, o) 465 466 static inline bool early_page_uninitialised(unsigned long pfn) 467 { 468 return false; 469 } 470 471 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 472 { 473 return false; 474 } 475 #endif 476 477 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 478 static inline unsigned long *get_pageblock_bitmap(struct page *page, 479 unsigned long pfn) 480 { 481 #ifdef CONFIG_SPARSEMEM 482 return section_to_usemap(__pfn_to_section(pfn)); 483 #else 484 return page_zone(page)->pageblock_flags; 485 #endif /* CONFIG_SPARSEMEM */ 486 } 487 488 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn) 489 { 490 #ifdef CONFIG_SPARSEMEM 491 pfn &= (PAGES_PER_SECTION-1); 492 #else 493 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 494 #endif /* CONFIG_SPARSEMEM */ 495 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 496 } 497 498 /** 499 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 500 * @page: The page within the block of interest 501 * @pfn: The target page frame number 502 * @mask: mask of bits that the caller is interested in 503 * 504 * Return: pageblock_bits flags 505 */ 506 static __always_inline 507 unsigned long __get_pfnblock_flags_mask(struct page *page, 508 unsigned long pfn, 509 unsigned long mask) 510 { 511 unsigned long *bitmap; 512 unsigned long bitidx, word_bitidx; 513 unsigned long word; 514 515 bitmap = get_pageblock_bitmap(page, pfn); 516 bitidx = pfn_to_bitidx(page, pfn); 517 word_bitidx = bitidx / BITS_PER_LONG; 518 bitidx &= (BITS_PER_LONG-1); 519 520 word = bitmap[word_bitidx]; 521 return (word >> bitidx) & mask; 522 } 523 524 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, 525 unsigned long mask) 526 { 527 return __get_pfnblock_flags_mask(page, pfn, mask); 528 } 529 530 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn) 531 { 532 return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK); 533 } 534 535 /** 536 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 537 * @page: The page within the block of interest 538 * @flags: The flags to set 539 * @pfn: The target page frame number 540 * @mask: mask of bits that the caller is interested in 541 */ 542 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 543 unsigned long pfn, 544 unsigned long mask) 545 { 546 unsigned long *bitmap; 547 unsigned long bitidx, word_bitidx; 548 unsigned long old_word, word; 549 550 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 551 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits)); 552 553 bitmap = get_pageblock_bitmap(page, pfn); 554 bitidx = pfn_to_bitidx(page, pfn); 555 word_bitidx = bitidx / BITS_PER_LONG; 556 bitidx &= (BITS_PER_LONG-1); 557 558 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 559 560 mask <<= bitidx; 561 flags <<= bitidx; 562 563 word = READ_ONCE(bitmap[word_bitidx]); 564 for (;;) { 565 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 566 if (word == old_word) 567 break; 568 word = old_word; 569 } 570 } 571 572 void set_pageblock_migratetype(struct page *page, int migratetype) 573 { 574 if (unlikely(page_group_by_mobility_disabled && 575 migratetype < MIGRATE_PCPTYPES)) 576 migratetype = MIGRATE_UNMOVABLE; 577 578 set_pfnblock_flags_mask(page, (unsigned long)migratetype, 579 page_to_pfn(page), MIGRATETYPE_MASK); 580 } 581 582 #ifdef CONFIG_DEBUG_VM 583 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 584 { 585 int ret = 0; 586 unsigned seq; 587 unsigned long pfn = page_to_pfn(page); 588 unsigned long sp, start_pfn; 589 590 do { 591 seq = zone_span_seqbegin(zone); 592 start_pfn = zone->zone_start_pfn; 593 sp = zone->spanned_pages; 594 if (!zone_spans_pfn(zone, pfn)) 595 ret = 1; 596 } while (zone_span_seqretry(zone, seq)); 597 598 if (ret) 599 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 600 pfn, zone_to_nid(zone), zone->name, 601 start_pfn, start_pfn + sp); 602 603 return ret; 604 } 605 606 static int page_is_consistent(struct zone *zone, struct page *page) 607 { 608 if (!pfn_valid_within(page_to_pfn(page))) 609 return 0; 610 if (zone != page_zone(page)) 611 return 0; 612 613 return 1; 614 } 615 /* 616 * Temporary debugging check for pages not lying within a given zone. 617 */ 618 static int __maybe_unused bad_range(struct zone *zone, struct page *page) 619 { 620 if (page_outside_zone_boundaries(zone, page)) 621 return 1; 622 if (!page_is_consistent(zone, page)) 623 return 1; 624 625 return 0; 626 } 627 #else 628 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page) 629 { 630 return 0; 631 } 632 #endif 633 634 static void bad_page(struct page *page, const char *reason) 635 { 636 static unsigned long resume; 637 static unsigned long nr_shown; 638 static unsigned long nr_unshown; 639 640 /* 641 * Allow a burst of 60 reports, then keep quiet for that minute; 642 * or allow a steady drip of one report per second. 643 */ 644 if (nr_shown == 60) { 645 if (time_before(jiffies, resume)) { 646 nr_unshown++; 647 goto out; 648 } 649 if (nr_unshown) { 650 pr_alert( 651 "BUG: Bad page state: %lu messages suppressed\n", 652 nr_unshown); 653 nr_unshown = 0; 654 } 655 nr_shown = 0; 656 } 657 if (nr_shown++ == 0) 658 resume = jiffies + 60 * HZ; 659 660 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 661 current->comm, page_to_pfn(page)); 662 __dump_page(page, reason); 663 dump_page_owner(page); 664 665 print_modules(); 666 dump_stack(); 667 out: 668 /* Leave bad fields for debug, except PageBuddy could make trouble */ 669 page_mapcount_reset(page); /* remove PageBuddy */ 670 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 671 } 672 673 /* 674 * Higher-order pages are called "compound pages". They are structured thusly: 675 * 676 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 677 * 678 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 679 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 680 * 681 * The first tail page's ->compound_dtor holds the offset in array of compound 682 * page destructors. See compound_page_dtors. 683 * 684 * The first tail page's ->compound_order holds the order of allocation. 685 * This usage means that zero-order pages may not be compound. 686 */ 687 688 void free_compound_page(struct page *page) 689 { 690 mem_cgroup_uncharge(page); 691 __free_pages_ok(page, compound_order(page), FPI_NONE); 692 } 693 694 void prep_compound_page(struct page *page, unsigned int order) 695 { 696 int i; 697 int nr_pages = 1 << order; 698 699 __SetPageHead(page); 700 for (i = 1; i < nr_pages; i++) { 701 struct page *p = page + i; 702 set_page_count(p, 0); 703 p->mapping = TAIL_MAPPING; 704 set_compound_head(p, page); 705 } 706 707 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 708 set_compound_order(page, order); 709 atomic_set(compound_mapcount_ptr(page), -1); 710 if (hpage_pincount_available(page)) 711 atomic_set(compound_pincount_ptr(page), 0); 712 } 713 714 #ifdef CONFIG_DEBUG_PAGEALLOC 715 unsigned int _debug_guardpage_minorder; 716 717 bool _debug_pagealloc_enabled_early __read_mostly 718 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 719 EXPORT_SYMBOL(_debug_pagealloc_enabled_early); 720 DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 721 EXPORT_SYMBOL(_debug_pagealloc_enabled); 722 723 DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 724 725 static int __init early_debug_pagealloc(char *buf) 726 { 727 return kstrtobool(buf, &_debug_pagealloc_enabled_early); 728 } 729 early_param("debug_pagealloc", early_debug_pagealloc); 730 731 void init_debug_pagealloc(void) 732 { 733 if (!debug_pagealloc_enabled()) 734 return; 735 736 static_branch_enable(&_debug_pagealloc_enabled); 737 738 if (!debug_guardpage_minorder()) 739 return; 740 741 static_branch_enable(&_debug_guardpage_enabled); 742 } 743 744 static int __init debug_guardpage_minorder_setup(char *buf) 745 { 746 unsigned long res; 747 748 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 749 pr_err("Bad debug_guardpage_minorder value\n"); 750 return 0; 751 } 752 _debug_guardpage_minorder = res; 753 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 754 return 0; 755 } 756 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); 757 758 static inline bool set_page_guard(struct zone *zone, struct page *page, 759 unsigned int order, int migratetype) 760 { 761 if (!debug_guardpage_enabled()) 762 return false; 763 764 if (order >= debug_guardpage_minorder()) 765 return false; 766 767 __SetPageGuard(page); 768 INIT_LIST_HEAD(&page->lru); 769 set_page_private(page, order); 770 /* Guard pages are not available for any usage */ 771 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 772 773 return true; 774 } 775 776 static inline void clear_page_guard(struct zone *zone, struct page *page, 777 unsigned int order, int migratetype) 778 { 779 if (!debug_guardpage_enabled()) 780 return; 781 782 __ClearPageGuard(page); 783 784 set_page_private(page, 0); 785 if (!is_migrate_isolate(migratetype)) 786 __mod_zone_freepage_state(zone, (1 << order), migratetype); 787 } 788 #else 789 static inline bool set_page_guard(struct zone *zone, struct page *page, 790 unsigned int order, int migratetype) { return false; } 791 static inline void clear_page_guard(struct zone *zone, struct page *page, 792 unsigned int order, int migratetype) {} 793 #endif 794 795 static inline void set_buddy_order(struct page *page, unsigned int order) 796 { 797 set_page_private(page, order); 798 __SetPageBuddy(page); 799 } 800 801 /* 802 * This function checks whether a page is free && is the buddy 803 * we can coalesce a page and its buddy if 804 * (a) the buddy is not in a hole (check before calling!) && 805 * (b) the buddy is in the buddy system && 806 * (c) a page and its buddy have the same order && 807 * (d) a page and its buddy are in the same zone. 808 * 809 * For recording whether a page is in the buddy system, we set PageBuddy. 810 * Setting, clearing, and testing PageBuddy is serialized by zone->lock. 811 * 812 * For recording page's order, we use page_private(page). 813 */ 814 static inline bool page_is_buddy(struct page *page, struct page *buddy, 815 unsigned int order) 816 { 817 if (!page_is_guard(buddy) && !PageBuddy(buddy)) 818 return false; 819 820 if (buddy_order(buddy) != order) 821 return false; 822 823 /* 824 * zone check is done late to avoid uselessly calculating 825 * zone/node ids for pages that could never merge. 826 */ 827 if (page_zone_id(page) != page_zone_id(buddy)) 828 return false; 829 830 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 831 832 return true; 833 } 834 835 #ifdef CONFIG_COMPACTION 836 static inline struct capture_control *task_capc(struct zone *zone) 837 { 838 struct capture_control *capc = current->capture_control; 839 840 return unlikely(capc) && 841 !(current->flags & PF_KTHREAD) && 842 !capc->page && 843 capc->cc->zone == zone ? capc : NULL; 844 } 845 846 static inline bool 847 compaction_capture(struct capture_control *capc, struct page *page, 848 int order, int migratetype) 849 { 850 if (!capc || order != capc->cc->order) 851 return false; 852 853 /* Do not accidentally pollute CMA or isolated regions*/ 854 if (is_migrate_cma(migratetype) || 855 is_migrate_isolate(migratetype)) 856 return false; 857 858 /* 859 * Do not let lower order allocations polluate a movable pageblock. 860 * This might let an unmovable request use a reclaimable pageblock 861 * and vice-versa but no more than normal fallback logic which can 862 * have trouble finding a high-order free page. 863 */ 864 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE) 865 return false; 866 867 capc->page = page; 868 return true; 869 } 870 871 #else 872 static inline struct capture_control *task_capc(struct zone *zone) 873 { 874 return NULL; 875 } 876 877 static inline bool 878 compaction_capture(struct capture_control *capc, struct page *page, 879 int order, int migratetype) 880 { 881 return false; 882 } 883 #endif /* CONFIG_COMPACTION */ 884 885 /* Used for pages not on another list */ 886 static inline void add_to_free_list(struct page *page, struct zone *zone, 887 unsigned int order, int migratetype) 888 { 889 struct free_area *area = &zone->free_area[order]; 890 891 list_add(&page->lru, &area->free_list[migratetype]); 892 area->nr_free++; 893 } 894 895 /* Used for pages not on another list */ 896 static inline void add_to_free_list_tail(struct page *page, struct zone *zone, 897 unsigned int order, int migratetype) 898 { 899 struct free_area *area = &zone->free_area[order]; 900 901 list_add_tail(&page->lru, &area->free_list[migratetype]); 902 area->nr_free++; 903 } 904 905 /* 906 * Used for pages which are on another list. Move the pages to the tail 907 * of the list - so the moved pages won't immediately be considered for 908 * allocation again (e.g., optimization for memory onlining). 909 */ 910 static inline void move_to_free_list(struct page *page, struct zone *zone, 911 unsigned int order, int migratetype) 912 { 913 struct free_area *area = &zone->free_area[order]; 914 915 list_move_tail(&page->lru, &area->free_list[migratetype]); 916 } 917 918 static inline void del_page_from_free_list(struct page *page, struct zone *zone, 919 unsigned int order) 920 { 921 /* clear reported state and update reported page count */ 922 if (page_reported(page)) 923 __ClearPageReported(page); 924 925 list_del(&page->lru); 926 __ClearPageBuddy(page); 927 set_page_private(page, 0); 928 zone->free_area[order].nr_free--; 929 } 930 931 /* 932 * If this is not the largest possible page, check if the buddy 933 * of the next-highest order is free. If it is, it's possible 934 * that pages are being freed that will coalesce soon. In case, 935 * that is happening, add the free page to the tail of the list 936 * so it's less likely to be used soon and more likely to be merged 937 * as a higher order page 938 */ 939 static inline bool 940 buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, 941 struct page *page, unsigned int order) 942 { 943 struct page *higher_page, *higher_buddy; 944 unsigned long combined_pfn; 945 946 if (order >= MAX_ORDER - 2) 947 return false; 948 949 if (!pfn_valid_within(buddy_pfn)) 950 return false; 951 952 combined_pfn = buddy_pfn & pfn; 953 higher_page = page + (combined_pfn - pfn); 954 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1); 955 higher_buddy = higher_page + (buddy_pfn - combined_pfn); 956 957 return pfn_valid_within(buddy_pfn) && 958 page_is_buddy(higher_page, higher_buddy, order + 1); 959 } 960 961 /* 962 * Freeing function for a buddy system allocator. 963 * 964 * The concept of a buddy system is to maintain direct-mapped table 965 * (containing bit values) for memory blocks of various "orders". 966 * The bottom level table contains the map for the smallest allocatable 967 * units of memory (here, pages), and each level above it describes 968 * pairs of units from the levels below, hence, "buddies". 969 * At a high level, all that happens here is marking the table entry 970 * at the bottom level available, and propagating the changes upward 971 * as necessary, plus some accounting needed to play nicely with other 972 * parts of the VM system. 973 * At each level, we keep a list of pages, which are heads of continuous 974 * free pages of length of (1 << order) and marked with PageBuddy. 975 * Page's order is recorded in page_private(page) field. 976 * So when we are allocating or freeing one, we can derive the state of the 977 * other. That is, if we allocate a small block, and both were 978 * free, the remainder of the region must be split into blocks. 979 * If a block is freed, and its buddy is also free, then this 980 * triggers coalescing into a block of larger size. 981 * 982 * -- nyc 983 */ 984 985 static inline void __free_one_page(struct page *page, 986 unsigned long pfn, 987 struct zone *zone, unsigned int order, 988 int migratetype, fpi_t fpi_flags) 989 { 990 struct capture_control *capc = task_capc(zone); 991 unsigned long buddy_pfn; 992 unsigned long combined_pfn; 993 unsigned int max_order; 994 struct page *buddy; 995 bool to_tail; 996 997 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1); 998 999 VM_BUG_ON(!zone_is_initialized(zone)); 1000 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 1001 1002 VM_BUG_ON(migratetype == -1); 1003 if (likely(!is_migrate_isolate(migratetype))) 1004 __mod_zone_freepage_state(zone, 1 << order, migratetype); 1005 1006 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); 1007 VM_BUG_ON_PAGE(bad_range(zone, page), page); 1008 1009 continue_merging: 1010 while (order < max_order - 1) { 1011 if (compaction_capture(capc, page, order, migratetype)) { 1012 __mod_zone_freepage_state(zone, -(1 << order), 1013 migratetype); 1014 return; 1015 } 1016 buddy_pfn = __find_buddy_pfn(pfn, order); 1017 buddy = page + (buddy_pfn - pfn); 1018 1019 if (!pfn_valid_within(buddy_pfn)) 1020 goto done_merging; 1021 if (!page_is_buddy(page, buddy, order)) 1022 goto done_merging; 1023 /* 1024 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 1025 * merge with it and move up one order. 1026 */ 1027 if (page_is_guard(buddy)) 1028 clear_page_guard(zone, buddy, order, migratetype); 1029 else 1030 del_page_from_free_list(buddy, zone, order); 1031 combined_pfn = buddy_pfn & pfn; 1032 page = page + (combined_pfn - pfn); 1033 pfn = combined_pfn; 1034 order++; 1035 } 1036 if (max_order < MAX_ORDER) { 1037 /* If we are here, it means order is >= pageblock_order. 1038 * We want to prevent merge between freepages on isolate 1039 * pageblock and normal pageblock. Without this, pageblock 1040 * isolation could cause incorrect freepage or CMA accounting. 1041 * 1042 * We don't want to hit this code for the more frequent 1043 * low-order merging. 1044 */ 1045 if (unlikely(has_isolate_pageblock(zone))) { 1046 int buddy_mt; 1047 1048 buddy_pfn = __find_buddy_pfn(pfn, order); 1049 buddy = page + (buddy_pfn - pfn); 1050 buddy_mt = get_pageblock_migratetype(buddy); 1051 1052 if (migratetype != buddy_mt 1053 && (is_migrate_isolate(migratetype) || 1054 is_migrate_isolate(buddy_mt))) 1055 goto done_merging; 1056 } 1057 max_order++; 1058 goto continue_merging; 1059 } 1060 1061 done_merging: 1062 set_buddy_order(page, order); 1063 1064 if (fpi_flags & FPI_TO_TAIL) 1065 to_tail = true; 1066 else if (is_shuffle_order(order)) 1067 to_tail = shuffle_pick_tail(); 1068 else 1069 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); 1070 1071 if (to_tail) 1072 add_to_free_list_tail(page, zone, order, migratetype); 1073 else 1074 add_to_free_list(page, zone, order, migratetype); 1075 1076 /* Notify page reporting subsystem of freed page */ 1077 if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY)) 1078 page_reporting_notify_free(order); 1079 } 1080 1081 /* 1082 * A bad page could be due to a number of fields. Instead of multiple branches, 1083 * try and check multiple fields with one check. The caller must do a detailed 1084 * check if necessary. 1085 */ 1086 static inline bool page_expected_state(struct page *page, 1087 unsigned long check_flags) 1088 { 1089 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1090 return false; 1091 1092 if (unlikely((unsigned long)page->mapping | 1093 page_ref_count(page) | 1094 #ifdef CONFIG_MEMCG 1095 (unsigned long)page->mem_cgroup | 1096 #endif 1097 (page->flags & check_flags))) 1098 return false; 1099 1100 return true; 1101 } 1102 1103 static const char *page_bad_reason(struct page *page, unsigned long flags) 1104 { 1105 const char *bad_reason = NULL; 1106 1107 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1108 bad_reason = "nonzero mapcount"; 1109 if (unlikely(page->mapping != NULL)) 1110 bad_reason = "non-NULL mapping"; 1111 if (unlikely(page_ref_count(page) != 0)) 1112 bad_reason = "nonzero _refcount"; 1113 if (unlikely(page->flags & flags)) { 1114 if (flags == PAGE_FLAGS_CHECK_AT_PREP) 1115 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set"; 1116 else 1117 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 1118 } 1119 #ifdef CONFIG_MEMCG 1120 if (unlikely(page->mem_cgroup)) 1121 bad_reason = "page still charged to cgroup"; 1122 #endif 1123 return bad_reason; 1124 } 1125 1126 static void check_free_page_bad(struct page *page) 1127 { 1128 bad_page(page, 1129 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE)); 1130 } 1131 1132 static inline int check_free_page(struct page *page) 1133 { 1134 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 1135 return 0; 1136 1137 /* Something has gone sideways, find it */ 1138 check_free_page_bad(page); 1139 return 1; 1140 } 1141 1142 static int free_tail_pages_check(struct page *head_page, struct page *page) 1143 { 1144 int ret = 1; 1145 1146 /* 1147 * We rely page->lru.next never has bit 0 set, unless the page 1148 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 1149 */ 1150 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 1151 1152 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 1153 ret = 0; 1154 goto out; 1155 } 1156 switch (page - head_page) { 1157 case 1: 1158 /* the first tail page: ->mapping may be compound_mapcount() */ 1159 if (unlikely(compound_mapcount(page))) { 1160 bad_page(page, "nonzero compound_mapcount"); 1161 goto out; 1162 } 1163 break; 1164 case 2: 1165 /* 1166 * the second tail page: ->mapping is 1167 * deferred_list.next -- ignore value. 1168 */ 1169 break; 1170 default: 1171 if (page->mapping != TAIL_MAPPING) { 1172 bad_page(page, "corrupted mapping in tail page"); 1173 goto out; 1174 } 1175 break; 1176 } 1177 if (unlikely(!PageTail(page))) { 1178 bad_page(page, "PageTail not set"); 1179 goto out; 1180 } 1181 if (unlikely(compound_head(page) != head_page)) { 1182 bad_page(page, "compound_head not consistent"); 1183 goto out; 1184 } 1185 ret = 0; 1186 out: 1187 page->mapping = NULL; 1188 clear_compound_head(page); 1189 return ret; 1190 } 1191 1192 static void kernel_init_free_pages(struct page *page, int numpages) 1193 { 1194 int i; 1195 1196 /* s390's use of memset() could override KASAN redzones. */ 1197 kasan_disable_current(); 1198 for (i = 0; i < numpages; i++) 1199 clear_highpage(page + i); 1200 kasan_enable_current(); 1201 } 1202 1203 static __always_inline bool free_pages_prepare(struct page *page, 1204 unsigned int order, bool check_free) 1205 { 1206 int bad = 0; 1207 1208 VM_BUG_ON_PAGE(PageTail(page), page); 1209 1210 trace_mm_page_free(page, order); 1211 1212 if (unlikely(PageHWPoison(page)) && !order) { 1213 /* 1214 * Do not let hwpoison pages hit pcplists/buddy 1215 * Untie memcg state and reset page's owner 1216 */ 1217 if (memcg_kmem_enabled() && PageKmemcg(page)) 1218 __memcg_kmem_uncharge_page(page, order); 1219 reset_page_owner(page, order); 1220 return false; 1221 } 1222 1223 /* 1224 * Check tail pages before head page information is cleared to 1225 * avoid checking PageCompound for order-0 pages. 1226 */ 1227 if (unlikely(order)) { 1228 bool compound = PageCompound(page); 1229 int i; 1230 1231 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1232 1233 if (compound) 1234 ClearPageDoubleMap(page); 1235 for (i = 1; i < (1 << order); i++) { 1236 if (compound) 1237 bad += free_tail_pages_check(page, page + i); 1238 if (unlikely(check_free_page(page + i))) { 1239 bad++; 1240 continue; 1241 } 1242 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1243 } 1244 } 1245 if (PageMappingFlags(page)) 1246 page->mapping = NULL; 1247 if (memcg_kmem_enabled() && PageKmemcg(page)) 1248 __memcg_kmem_uncharge_page(page, order); 1249 if (check_free) 1250 bad += check_free_page(page); 1251 if (bad) 1252 return false; 1253 1254 page_cpupid_reset_last(page); 1255 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1256 reset_page_owner(page, order); 1257 1258 if (!PageHighMem(page)) { 1259 debug_check_no_locks_freed(page_address(page), 1260 PAGE_SIZE << order); 1261 debug_check_no_obj_freed(page_address(page), 1262 PAGE_SIZE << order); 1263 } 1264 if (want_init_on_free()) 1265 kernel_init_free_pages(page, 1 << order); 1266 1267 kernel_poison_pages(page, 1 << order, 0); 1268 /* 1269 * arch_free_page() can make the page's contents inaccessible. s390 1270 * does this. So nothing which can access the page's contents should 1271 * happen after this. 1272 */ 1273 arch_free_page(page, order); 1274 1275 if (debug_pagealloc_enabled_static()) 1276 kernel_map_pages(page, 1 << order, 0); 1277 1278 kasan_free_nondeferred_pages(page, order); 1279 1280 return true; 1281 } 1282 1283 #ifdef CONFIG_DEBUG_VM 1284 /* 1285 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed 1286 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when 1287 * moved from pcp lists to free lists. 1288 */ 1289 static bool free_pcp_prepare(struct page *page) 1290 { 1291 return free_pages_prepare(page, 0, true); 1292 } 1293 1294 static bool bulkfree_pcp_prepare(struct page *page) 1295 { 1296 if (debug_pagealloc_enabled_static()) 1297 return check_free_page(page); 1298 else 1299 return false; 1300 } 1301 #else 1302 /* 1303 * With DEBUG_VM disabled, order-0 pages being freed are checked only when 1304 * moving from pcp lists to free list in order to reduce overhead. With 1305 * debug_pagealloc enabled, they are checked also immediately when being freed 1306 * to the pcp lists. 1307 */ 1308 static bool free_pcp_prepare(struct page *page) 1309 { 1310 if (debug_pagealloc_enabled_static()) 1311 return free_pages_prepare(page, 0, true); 1312 else 1313 return free_pages_prepare(page, 0, false); 1314 } 1315 1316 static bool bulkfree_pcp_prepare(struct page *page) 1317 { 1318 return check_free_page(page); 1319 } 1320 #endif /* CONFIG_DEBUG_VM */ 1321 1322 static inline void prefetch_buddy(struct page *page) 1323 { 1324 unsigned long pfn = page_to_pfn(page); 1325 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0); 1326 struct page *buddy = page + (buddy_pfn - pfn); 1327 1328 prefetch(buddy); 1329 } 1330 1331 /* 1332 * Frees a number of pages from the PCP lists 1333 * Assumes all pages on list are in same zone, and of same order. 1334 * count is the number of pages to free. 1335 * 1336 * If the zone was previously in an "all pages pinned" state then look to 1337 * see if this freeing clears that state. 1338 * 1339 * And clear the zone's pages_scanned counter, to hold off the "all pages are 1340 * pinned" detection logic. 1341 */ 1342 static void free_pcppages_bulk(struct zone *zone, int count, 1343 struct per_cpu_pages *pcp) 1344 { 1345 int migratetype = 0; 1346 int batch_free = 0; 1347 int prefetch_nr = 0; 1348 bool isolated_pageblocks; 1349 struct page *page, *tmp; 1350 LIST_HEAD(head); 1351 1352 /* 1353 * Ensure proper count is passed which otherwise would stuck in the 1354 * below while (list_empty(list)) loop. 1355 */ 1356 count = min(pcp->count, count); 1357 while (count) { 1358 struct list_head *list; 1359 1360 /* 1361 * Remove pages from lists in a round-robin fashion. A 1362 * batch_free count is maintained that is incremented when an 1363 * empty list is encountered. This is so more pages are freed 1364 * off fuller lists instead of spinning excessively around empty 1365 * lists 1366 */ 1367 do { 1368 batch_free++; 1369 if (++migratetype == MIGRATE_PCPTYPES) 1370 migratetype = 0; 1371 list = &pcp->lists[migratetype]; 1372 } while (list_empty(list)); 1373 1374 /* This is the only non-empty list. Free them all. */ 1375 if (batch_free == MIGRATE_PCPTYPES) 1376 batch_free = count; 1377 1378 do { 1379 page = list_last_entry(list, struct page, lru); 1380 /* must delete to avoid corrupting pcp list */ 1381 list_del(&page->lru); 1382 pcp->count--; 1383 1384 if (bulkfree_pcp_prepare(page)) 1385 continue; 1386 1387 list_add_tail(&page->lru, &head); 1388 1389 /* 1390 * We are going to put the page back to the global 1391 * pool, prefetch its buddy to speed up later access 1392 * under zone->lock. It is believed the overhead of 1393 * an additional test and calculating buddy_pfn here 1394 * can be offset by reduced memory latency later. To 1395 * avoid excessive prefetching due to large count, only 1396 * prefetch buddy for the first pcp->batch nr of pages. 1397 */ 1398 if (prefetch_nr++ < pcp->batch) 1399 prefetch_buddy(page); 1400 } while (--count && --batch_free && !list_empty(list)); 1401 } 1402 1403 spin_lock(&zone->lock); 1404 isolated_pageblocks = has_isolate_pageblock(zone); 1405 1406 /* 1407 * Use safe version since after __free_one_page(), 1408 * page->lru.next will not point to original list. 1409 */ 1410 list_for_each_entry_safe(page, tmp, &head, lru) { 1411 int mt = get_pcppage_migratetype(page); 1412 /* MIGRATE_ISOLATE page should not go to pcplists */ 1413 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1414 /* Pageblock could have been isolated meanwhile */ 1415 if (unlikely(isolated_pageblocks)) 1416 mt = get_pageblock_migratetype(page); 1417 1418 __free_one_page(page, page_to_pfn(page), zone, 0, mt, FPI_NONE); 1419 trace_mm_page_pcpu_drain(page, 0, mt); 1420 } 1421 spin_unlock(&zone->lock); 1422 } 1423 1424 static void free_one_page(struct zone *zone, 1425 struct page *page, unsigned long pfn, 1426 unsigned int order, 1427 int migratetype, fpi_t fpi_flags) 1428 { 1429 spin_lock(&zone->lock); 1430 if (unlikely(has_isolate_pageblock(zone) || 1431 is_migrate_isolate(migratetype))) { 1432 migratetype = get_pfnblock_migratetype(page, pfn); 1433 } 1434 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags); 1435 spin_unlock(&zone->lock); 1436 } 1437 1438 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1439 unsigned long zone, int nid) 1440 { 1441 mm_zero_struct_page(page); 1442 set_page_links(page, zone, nid, pfn); 1443 init_page_count(page); 1444 page_mapcount_reset(page); 1445 page_cpupid_reset_last(page); 1446 page_kasan_tag_reset(page); 1447 1448 INIT_LIST_HEAD(&page->lru); 1449 #ifdef WANT_PAGE_VIRTUAL 1450 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1451 if (!is_highmem_idx(zone)) 1452 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1453 #endif 1454 } 1455 1456 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1457 static void __meminit init_reserved_page(unsigned long pfn) 1458 { 1459 pg_data_t *pgdat; 1460 int nid, zid; 1461 1462 if (!early_page_uninitialised(pfn)) 1463 return; 1464 1465 nid = early_pfn_to_nid(pfn); 1466 pgdat = NODE_DATA(nid); 1467 1468 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1469 struct zone *zone = &pgdat->node_zones[zid]; 1470 1471 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) 1472 break; 1473 } 1474 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 1475 } 1476 #else 1477 static inline void init_reserved_page(unsigned long pfn) 1478 { 1479 } 1480 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1481 1482 /* 1483 * Initialised pages do not have PageReserved set. This function is 1484 * called for each range allocated by the bootmem allocator and 1485 * marks the pages PageReserved. The remaining valid pages are later 1486 * sent to the buddy page allocator. 1487 */ 1488 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1489 { 1490 unsigned long start_pfn = PFN_DOWN(start); 1491 unsigned long end_pfn = PFN_UP(end); 1492 1493 for (; start_pfn < end_pfn; start_pfn++) { 1494 if (pfn_valid(start_pfn)) { 1495 struct page *page = pfn_to_page(start_pfn); 1496 1497 init_reserved_page(start_pfn); 1498 1499 /* Avoid false-positive PageTail() */ 1500 INIT_LIST_HEAD(&page->lru); 1501 1502 /* 1503 * no need for atomic set_bit because the struct 1504 * page is not visible yet so nobody should 1505 * access it yet. 1506 */ 1507 __SetPageReserved(page); 1508 } 1509 } 1510 } 1511 1512 static void __free_pages_ok(struct page *page, unsigned int order, 1513 fpi_t fpi_flags) 1514 { 1515 unsigned long flags; 1516 int migratetype; 1517 unsigned long pfn = page_to_pfn(page); 1518 1519 if (!free_pages_prepare(page, order, true)) 1520 return; 1521 1522 migratetype = get_pfnblock_migratetype(page, pfn); 1523 local_irq_save(flags); 1524 __count_vm_events(PGFREE, 1 << order); 1525 free_one_page(page_zone(page), page, pfn, order, migratetype, 1526 fpi_flags); 1527 local_irq_restore(flags); 1528 } 1529 1530 void __free_pages_core(struct page *page, unsigned int order) 1531 { 1532 unsigned int nr_pages = 1 << order; 1533 struct page *p = page; 1534 unsigned int loop; 1535 1536 /* 1537 * When initializing the memmap, __init_single_page() sets the refcount 1538 * of all pages to 1 ("allocated"/"not free"). We have to set the 1539 * refcount of all involved pages to 0. 1540 */ 1541 prefetchw(p); 1542 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1543 prefetchw(p + 1); 1544 __ClearPageReserved(p); 1545 set_page_count(p, 0); 1546 } 1547 __ClearPageReserved(p); 1548 set_page_count(p, 0); 1549 1550 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); 1551 1552 /* 1553 * Bypass PCP and place fresh pages right to the tail, primarily 1554 * relevant for memory onlining. 1555 */ 1556 __free_pages_ok(page, order, FPI_TO_TAIL); 1557 } 1558 1559 #ifdef CONFIG_NEED_MULTIPLE_NODES 1560 1561 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1562 1563 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID 1564 1565 /* 1566 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 1567 */ 1568 int __meminit __early_pfn_to_nid(unsigned long pfn, 1569 struct mminit_pfnnid_cache *state) 1570 { 1571 unsigned long start_pfn, end_pfn; 1572 int nid; 1573 1574 if (state->last_start <= pfn && pfn < state->last_end) 1575 return state->last_nid; 1576 1577 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 1578 if (nid != NUMA_NO_NODE) { 1579 state->last_start = start_pfn; 1580 state->last_end = end_pfn; 1581 state->last_nid = nid; 1582 } 1583 1584 return nid; 1585 } 1586 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ 1587 1588 int __meminit early_pfn_to_nid(unsigned long pfn) 1589 { 1590 static DEFINE_SPINLOCK(early_pfn_lock); 1591 int nid; 1592 1593 spin_lock(&early_pfn_lock); 1594 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1595 if (nid < 0) 1596 nid = first_online_node; 1597 spin_unlock(&early_pfn_lock); 1598 1599 return nid; 1600 } 1601 #endif /* CONFIG_NEED_MULTIPLE_NODES */ 1602 1603 void __init memblock_free_pages(struct page *page, unsigned long pfn, 1604 unsigned int order) 1605 { 1606 if (early_page_uninitialised(pfn)) 1607 return; 1608 __free_pages_core(page, order); 1609 } 1610 1611 /* 1612 * Check that the whole (or subset of) a pageblock given by the interval of 1613 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1614 * with the migration of free compaction scanner. The scanners then need to 1615 * use only pfn_valid_within() check for arches that allow holes within 1616 * pageblocks. 1617 * 1618 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1619 * 1620 * It's possible on some configurations to have a setup like node0 node1 node0 1621 * i.e. it's possible that all pages within a zones range of pages do not 1622 * belong to a single zone. We assume that a border between node0 and node1 1623 * can occur within a single pageblock, but not a node0 node1 node0 1624 * interleaving within a single pageblock. It is therefore sufficient to check 1625 * the first and last page of a pageblock and avoid checking each individual 1626 * page in a pageblock. 1627 */ 1628 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1629 unsigned long end_pfn, struct zone *zone) 1630 { 1631 struct page *start_page; 1632 struct page *end_page; 1633 1634 /* end_pfn is one past the range we are checking */ 1635 end_pfn--; 1636 1637 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1638 return NULL; 1639 1640 start_page = pfn_to_online_page(start_pfn); 1641 if (!start_page) 1642 return NULL; 1643 1644 if (page_zone(start_page) != zone) 1645 return NULL; 1646 1647 end_page = pfn_to_page(end_pfn); 1648 1649 /* This gives a shorter code than deriving page_zone(end_page) */ 1650 if (page_zone_id(start_page) != page_zone_id(end_page)) 1651 return NULL; 1652 1653 return start_page; 1654 } 1655 1656 void set_zone_contiguous(struct zone *zone) 1657 { 1658 unsigned long block_start_pfn = zone->zone_start_pfn; 1659 unsigned long block_end_pfn; 1660 1661 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1662 for (; block_start_pfn < zone_end_pfn(zone); 1663 block_start_pfn = block_end_pfn, 1664 block_end_pfn += pageblock_nr_pages) { 1665 1666 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1667 1668 if (!__pageblock_pfn_to_page(block_start_pfn, 1669 block_end_pfn, zone)) 1670 return; 1671 cond_resched(); 1672 } 1673 1674 /* We confirm that there is no hole */ 1675 zone->contiguous = true; 1676 } 1677 1678 void clear_zone_contiguous(struct zone *zone) 1679 { 1680 zone->contiguous = false; 1681 } 1682 1683 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1684 static void __init deferred_free_range(unsigned long pfn, 1685 unsigned long nr_pages) 1686 { 1687 struct page *page; 1688 unsigned long i; 1689 1690 if (!nr_pages) 1691 return; 1692 1693 page = pfn_to_page(pfn); 1694 1695 /* Free a large naturally-aligned chunk if possible */ 1696 if (nr_pages == pageblock_nr_pages && 1697 (pfn & (pageblock_nr_pages - 1)) == 0) { 1698 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1699 __free_pages_core(page, pageblock_order); 1700 return; 1701 } 1702 1703 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1704 if ((pfn & (pageblock_nr_pages - 1)) == 0) 1705 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1706 __free_pages_core(page, 0); 1707 } 1708 } 1709 1710 /* Completion tracking for deferred_init_memmap() threads */ 1711 static atomic_t pgdat_init_n_undone __initdata; 1712 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1713 1714 static inline void __init pgdat_init_report_one_done(void) 1715 { 1716 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1717 complete(&pgdat_init_all_done_comp); 1718 } 1719 1720 /* 1721 * Returns true if page needs to be initialized or freed to buddy allocator. 1722 * 1723 * First we check if pfn is valid on architectures where it is possible to have 1724 * holes within pageblock_nr_pages. On systems where it is not possible, this 1725 * function is optimized out. 1726 * 1727 * Then, we check if a current large page is valid by only checking the validity 1728 * of the head pfn. 1729 */ 1730 static inline bool __init deferred_pfn_valid(unsigned long pfn) 1731 { 1732 if (!pfn_valid_within(pfn)) 1733 return false; 1734 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn)) 1735 return false; 1736 return true; 1737 } 1738 1739 /* 1740 * Free pages to buddy allocator. Try to free aligned pages in 1741 * pageblock_nr_pages sizes. 1742 */ 1743 static void __init deferred_free_pages(unsigned long pfn, 1744 unsigned long end_pfn) 1745 { 1746 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1747 unsigned long nr_free = 0; 1748 1749 for (; pfn < end_pfn; pfn++) { 1750 if (!deferred_pfn_valid(pfn)) { 1751 deferred_free_range(pfn - nr_free, nr_free); 1752 nr_free = 0; 1753 } else if (!(pfn & nr_pgmask)) { 1754 deferred_free_range(pfn - nr_free, nr_free); 1755 nr_free = 1; 1756 } else { 1757 nr_free++; 1758 } 1759 } 1760 /* Free the last block of pages to allocator */ 1761 deferred_free_range(pfn - nr_free, nr_free); 1762 } 1763 1764 /* 1765 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 1766 * by performing it only once every pageblock_nr_pages. 1767 * Return number of pages initialized. 1768 */ 1769 static unsigned long __init deferred_init_pages(struct zone *zone, 1770 unsigned long pfn, 1771 unsigned long end_pfn) 1772 { 1773 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1774 int nid = zone_to_nid(zone); 1775 unsigned long nr_pages = 0; 1776 int zid = zone_idx(zone); 1777 struct page *page = NULL; 1778 1779 for (; pfn < end_pfn; pfn++) { 1780 if (!deferred_pfn_valid(pfn)) { 1781 page = NULL; 1782 continue; 1783 } else if (!page || !(pfn & nr_pgmask)) { 1784 page = pfn_to_page(pfn); 1785 } else { 1786 page++; 1787 } 1788 __init_single_page(page, pfn, zid, nid); 1789 nr_pages++; 1790 } 1791 return (nr_pages); 1792 } 1793 1794 /* 1795 * This function is meant to pre-load the iterator for the zone init. 1796 * Specifically it walks through the ranges until we are caught up to the 1797 * first_init_pfn value and exits there. If we never encounter the value we 1798 * return false indicating there are no valid ranges left. 1799 */ 1800 static bool __init 1801 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, 1802 unsigned long *spfn, unsigned long *epfn, 1803 unsigned long first_init_pfn) 1804 { 1805 u64 j; 1806 1807 /* 1808 * Start out by walking through the ranges in this zone that have 1809 * already been initialized. We don't need to do anything with them 1810 * so we just need to flush them out of the system. 1811 */ 1812 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { 1813 if (*epfn <= first_init_pfn) 1814 continue; 1815 if (*spfn < first_init_pfn) 1816 *spfn = first_init_pfn; 1817 *i = j; 1818 return true; 1819 } 1820 1821 return false; 1822 } 1823 1824 /* 1825 * Initialize and free pages. We do it in two loops: first we initialize 1826 * struct page, then free to buddy allocator, because while we are 1827 * freeing pages we can access pages that are ahead (computing buddy 1828 * page in __free_one_page()). 1829 * 1830 * In order to try and keep some memory in the cache we have the loop 1831 * broken along max page order boundaries. This way we will not cause 1832 * any issues with the buddy page computation. 1833 */ 1834 static unsigned long __init 1835 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, 1836 unsigned long *end_pfn) 1837 { 1838 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); 1839 unsigned long spfn = *start_pfn, epfn = *end_pfn; 1840 unsigned long nr_pages = 0; 1841 u64 j = *i; 1842 1843 /* First we loop through and initialize the page values */ 1844 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { 1845 unsigned long t; 1846 1847 if (mo_pfn <= *start_pfn) 1848 break; 1849 1850 t = min(mo_pfn, *end_pfn); 1851 nr_pages += deferred_init_pages(zone, *start_pfn, t); 1852 1853 if (mo_pfn < *end_pfn) { 1854 *start_pfn = mo_pfn; 1855 break; 1856 } 1857 } 1858 1859 /* Reset values and now loop through freeing pages as needed */ 1860 swap(j, *i); 1861 1862 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { 1863 unsigned long t; 1864 1865 if (mo_pfn <= spfn) 1866 break; 1867 1868 t = min(mo_pfn, epfn); 1869 deferred_free_pages(spfn, t); 1870 1871 if (mo_pfn <= epfn) 1872 break; 1873 } 1874 1875 return nr_pages; 1876 } 1877 1878 static void __init 1879 deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, 1880 void *arg) 1881 { 1882 unsigned long spfn, epfn; 1883 struct zone *zone = arg; 1884 u64 i; 1885 1886 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); 1887 1888 /* 1889 * Initialize and free pages in MAX_ORDER sized increments so that we 1890 * can avoid introducing any issues with the buddy allocator. 1891 */ 1892 while (spfn < end_pfn) { 1893 deferred_init_maxorder(&i, zone, &spfn, &epfn); 1894 cond_resched(); 1895 } 1896 } 1897 1898 /* An arch may override for more concurrency. */ 1899 __weak int __init 1900 deferred_page_init_max_threads(const struct cpumask *node_cpumask) 1901 { 1902 return 1; 1903 } 1904 1905 /* Initialise remaining memory on a node */ 1906 static int __init deferred_init_memmap(void *data) 1907 { 1908 pg_data_t *pgdat = data; 1909 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 1910 unsigned long spfn = 0, epfn = 0; 1911 unsigned long first_init_pfn, flags; 1912 unsigned long start = jiffies; 1913 struct zone *zone; 1914 int zid, max_threads; 1915 u64 i; 1916 1917 /* Bind memory initialisation thread to a local node if possible */ 1918 if (!cpumask_empty(cpumask)) 1919 set_cpus_allowed_ptr(current, cpumask); 1920 1921 pgdat_resize_lock(pgdat, &flags); 1922 first_init_pfn = pgdat->first_deferred_pfn; 1923 if (first_init_pfn == ULONG_MAX) { 1924 pgdat_resize_unlock(pgdat, &flags); 1925 pgdat_init_report_one_done(); 1926 return 0; 1927 } 1928 1929 /* Sanity check boundaries */ 1930 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 1931 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 1932 pgdat->first_deferred_pfn = ULONG_MAX; 1933 1934 /* 1935 * Once we unlock here, the zone cannot be grown anymore, thus if an 1936 * interrupt thread must allocate this early in boot, zone must be 1937 * pre-grown prior to start of deferred page initialization. 1938 */ 1939 pgdat_resize_unlock(pgdat, &flags); 1940 1941 /* Only the highest zone is deferred so find it */ 1942 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1943 zone = pgdat->node_zones + zid; 1944 if (first_init_pfn < zone_end_pfn(zone)) 1945 break; 1946 } 1947 1948 /* If the zone is empty somebody else may have cleared out the zone */ 1949 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 1950 first_init_pfn)) 1951 goto zone_empty; 1952 1953 max_threads = deferred_page_init_max_threads(cpumask); 1954 1955 while (spfn < epfn) { 1956 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION); 1957 struct padata_mt_job job = { 1958 .thread_fn = deferred_init_memmap_chunk, 1959 .fn_arg = zone, 1960 .start = spfn, 1961 .size = epfn_align - spfn, 1962 .align = PAGES_PER_SECTION, 1963 .min_chunk = PAGES_PER_SECTION, 1964 .max_threads = max_threads, 1965 }; 1966 1967 padata_do_multithreaded(&job); 1968 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 1969 epfn_align); 1970 } 1971 zone_empty: 1972 /* Sanity check that the next zone really is unpopulated */ 1973 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 1974 1975 pr_info("node %d deferred pages initialised in %ums\n", 1976 pgdat->node_id, jiffies_to_msecs(jiffies - start)); 1977 1978 pgdat_init_report_one_done(); 1979 return 0; 1980 } 1981 1982 /* 1983 * If this zone has deferred pages, try to grow it by initializing enough 1984 * deferred pages to satisfy the allocation specified by order, rounded up to 1985 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 1986 * of SECTION_SIZE bytes by initializing struct pages in increments of 1987 * PAGES_PER_SECTION * sizeof(struct page) bytes. 1988 * 1989 * Return true when zone was grown, otherwise return false. We return true even 1990 * when we grow less than requested, to let the caller decide if there are 1991 * enough pages to satisfy the allocation. 1992 * 1993 * Note: We use noinline because this function is needed only during boot, and 1994 * it is called from a __ref function _deferred_grow_zone. This way we are 1995 * making sure that it is not inlined into permanent text section. 1996 */ 1997 static noinline bool __init 1998 deferred_grow_zone(struct zone *zone, unsigned int order) 1999 { 2000 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 2001 pg_data_t *pgdat = zone->zone_pgdat; 2002 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 2003 unsigned long spfn, epfn, flags; 2004 unsigned long nr_pages = 0; 2005 u64 i; 2006 2007 /* Only the last zone may have deferred pages */ 2008 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 2009 return false; 2010 2011 pgdat_resize_lock(pgdat, &flags); 2012 2013 /* 2014 * If someone grew this zone while we were waiting for spinlock, return 2015 * true, as there might be enough pages already. 2016 */ 2017 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 2018 pgdat_resize_unlock(pgdat, &flags); 2019 return true; 2020 } 2021 2022 /* If the zone is empty somebody else may have cleared out the zone */ 2023 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2024 first_deferred_pfn)) { 2025 pgdat->first_deferred_pfn = ULONG_MAX; 2026 pgdat_resize_unlock(pgdat, &flags); 2027 /* Retry only once. */ 2028 return first_deferred_pfn != ULONG_MAX; 2029 } 2030 2031 /* 2032 * Initialize and free pages in MAX_ORDER sized increments so 2033 * that we can avoid introducing any issues with the buddy 2034 * allocator. 2035 */ 2036 while (spfn < epfn) { 2037 /* update our first deferred PFN for this section */ 2038 first_deferred_pfn = spfn; 2039 2040 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 2041 touch_nmi_watchdog(); 2042 2043 /* We should only stop along section boundaries */ 2044 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) 2045 continue; 2046 2047 /* If our quota has been met we can stop here */ 2048 if (nr_pages >= nr_pages_needed) 2049 break; 2050 } 2051 2052 pgdat->first_deferred_pfn = spfn; 2053 pgdat_resize_unlock(pgdat, &flags); 2054 2055 return nr_pages > 0; 2056 } 2057 2058 /* 2059 * deferred_grow_zone() is __init, but it is called from 2060 * get_page_from_freelist() during early boot until deferred_pages permanently 2061 * disables this call. This is why we have refdata wrapper to avoid warning, 2062 * and to ensure that the function body gets unloaded. 2063 */ 2064 static bool __ref 2065 _deferred_grow_zone(struct zone *zone, unsigned int order) 2066 { 2067 return deferred_grow_zone(zone, order); 2068 } 2069 2070 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 2071 2072 void __init page_alloc_init_late(void) 2073 { 2074 struct zone *zone; 2075 int nid; 2076 2077 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 2078 2079 /* There will be num_node_state(N_MEMORY) threads */ 2080 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 2081 for_each_node_state(nid, N_MEMORY) { 2082 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 2083 } 2084 2085 /* Block until all are initialised */ 2086 wait_for_completion(&pgdat_init_all_done_comp); 2087 2088 /* 2089 * The number of managed pages has changed due to the initialisation 2090 * so the pcpu batch and high limits needs to be updated or the limits 2091 * will be artificially small. 2092 */ 2093 for_each_populated_zone(zone) 2094 zone_pcp_update(zone); 2095 2096 /* 2097 * We initialized the rest of the deferred pages. Permanently disable 2098 * on-demand struct page initialization. 2099 */ 2100 static_branch_disable(&deferred_pages); 2101 2102 /* Reinit limits that are based on free pages after the kernel is up */ 2103 files_maxfiles_init(); 2104 #endif 2105 2106 /* Discard memblock private memory */ 2107 memblock_discard(); 2108 2109 for_each_node_state(nid, N_MEMORY) 2110 shuffle_free_memory(NODE_DATA(nid)); 2111 2112 for_each_populated_zone(zone) 2113 set_zone_contiguous(zone); 2114 } 2115 2116 #ifdef CONFIG_CMA 2117 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 2118 void __init init_cma_reserved_pageblock(struct page *page) 2119 { 2120 unsigned i = pageblock_nr_pages; 2121 struct page *p = page; 2122 2123 do { 2124 __ClearPageReserved(p); 2125 set_page_count(p, 0); 2126 } while (++p, --i); 2127 2128 set_pageblock_migratetype(page, MIGRATE_CMA); 2129 2130 if (pageblock_order >= MAX_ORDER) { 2131 i = pageblock_nr_pages; 2132 p = page; 2133 do { 2134 set_page_refcounted(p); 2135 __free_pages(p, MAX_ORDER - 1); 2136 p += MAX_ORDER_NR_PAGES; 2137 } while (i -= MAX_ORDER_NR_PAGES); 2138 } else { 2139 set_page_refcounted(page); 2140 __free_pages(page, pageblock_order); 2141 } 2142 2143 adjust_managed_page_count(page, pageblock_nr_pages); 2144 } 2145 #endif 2146 2147 /* 2148 * The order of subdivision here is critical for the IO subsystem. 2149 * Please do not alter this order without good reasons and regression 2150 * testing. Specifically, as large blocks of memory are subdivided, 2151 * the order in which smaller blocks are delivered depends on the order 2152 * they're subdivided in this function. This is the primary factor 2153 * influencing the order in which pages are delivered to the IO 2154 * subsystem according to empirical testing, and this is also justified 2155 * by considering the behavior of a buddy system containing a single 2156 * large block of memory acted on by a series of small allocations. 2157 * This behavior is a critical factor in sglist merging's success. 2158 * 2159 * -- nyc 2160 */ 2161 static inline void expand(struct zone *zone, struct page *page, 2162 int low, int high, int migratetype) 2163 { 2164 unsigned long size = 1 << high; 2165 2166 while (high > low) { 2167 high--; 2168 size >>= 1; 2169 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 2170 2171 /* 2172 * Mark as guard pages (or page), that will allow to 2173 * merge back to allocator when buddy will be freed. 2174 * Corresponding page table entries will not be touched, 2175 * pages will stay not present in virtual address space 2176 */ 2177 if (set_page_guard(zone, &page[size], high, migratetype)) 2178 continue; 2179 2180 add_to_free_list(&page[size], zone, high, migratetype); 2181 set_buddy_order(&page[size], high); 2182 } 2183 } 2184 2185 static void check_new_page_bad(struct page *page) 2186 { 2187 if (unlikely(page->flags & __PG_HWPOISON)) { 2188 /* Don't complain about hwpoisoned pages */ 2189 page_mapcount_reset(page); /* remove PageBuddy */ 2190 return; 2191 } 2192 2193 bad_page(page, 2194 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP)); 2195 } 2196 2197 /* 2198 * This page is about to be returned from the page allocator 2199 */ 2200 static inline int check_new_page(struct page *page) 2201 { 2202 if (likely(page_expected_state(page, 2203 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 2204 return 0; 2205 2206 check_new_page_bad(page); 2207 return 1; 2208 } 2209 2210 static inline bool free_pages_prezeroed(void) 2211 { 2212 return (IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) && 2213 page_poisoning_enabled()) || want_init_on_free(); 2214 } 2215 2216 #ifdef CONFIG_DEBUG_VM 2217 /* 2218 * With DEBUG_VM enabled, order-0 pages are checked for expected state when 2219 * being allocated from pcp lists. With debug_pagealloc also enabled, they are 2220 * also checked when pcp lists are refilled from the free lists. 2221 */ 2222 static inline bool check_pcp_refill(struct page *page) 2223 { 2224 if (debug_pagealloc_enabled_static()) 2225 return check_new_page(page); 2226 else 2227 return false; 2228 } 2229 2230 static inline bool check_new_pcp(struct page *page) 2231 { 2232 return check_new_page(page); 2233 } 2234 #else 2235 /* 2236 * With DEBUG_VM disabled, free order-0 pages are checked for expected state 2237 * when pcp lists are being refilled from the free lists. With debug_pagealloc 2238 * enabled, they are also checked when being allocated from the pcp lists. 2239 */ 2240 static inline bool check_pcp_refill(struct page *page) 2241 { 2242 return check_new_page(page); 2243 } 2244 static inline bool check_new_pcp(struct page *page) 2245 { 2246 if (debug_pagealloc_enabled_static()) 2247 return check_new_page(page); 2248 else 2249 return false; 2250 } 2251 #endif /* CONFIG_DEBUG_VM */ 2252 2253 static bool check_new_pages(struct page *page, unsigned int order) 2254 { 2255 int i; 2256 for (i = 0; i < (1 << order); i++) { 2257 struct page *p = page + i; 2258 2259 if (unlikely(check_new_page(p))) 2260 return true; 2261 } 2262 2263 return false; 2264 } 2265 2266 inline void post_alloc_hook(struct page *page, unsigned int order, 2267 gfp_t gfp_flags) 2268 { 2269 set_page_private(page, 0); 2270 set_page_refcounted(page); 2271 2272 arch_alloc_page(page, order); 2273 if (debug_pagealloc_enabled_static()) 2274 kernel_map_pages(page, 1 << order, 1); 2275 kasan_alloc_pages(page, order); 2276 kernel_poison_pages(page, 1 << order, 1); 2277 set_page_owner(page, order, gfp_flags); 2278 } 2279 2280 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 2281 unsigned int alloc_flags) 2282 { 2283 post_alloc_hook(page, order, gfp_flags); 2284 2285 if (!free_pages_prezeroed() && want_init_on_alloc(gfp_flags)) 2286 kernel_init_free_pages(page, 1 << order); 2287 2288 if (order && (gfp_flags & __GFP_COMP)) 2289 prep_compound_page(page, order); 2290 2291 /* 2292 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 2293 * allocate the page. The expectation is that the caller is taking 2294 * steps that will free more memory. The caller should avoid the page 2295 * being used for !PFMEMALLOC purposes. 2296 */ 2297 if (alloc_flags & ALLOC_NO_WATERMARKS) 2298 set_page_pfmemalloc(page); 2299 else 2300 clear_page_pfmemalloc(page); 2301 } 2302 2303 /* 2304 * Go through the free lists for the given migratetype and remove 2305 * the smallest available page from the freelists 2306 */ 2307 static __always_inline 2308 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 2309 int migratetype) 2310 { 2311 unsigned int current_order; 2312 struct free_area *area; 2313 struct page *page; 2314 2315 /* Find a page of the appropriate size in the preferred list */ 2316 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 2317 area = &(zone->free_area[current_order]); 2318 page = get_page_from_free_area(area, migratetype); 2319 if (!page) 2320 continue; 2321 del_page_from_free_list(page, zone, current_order); 2322 expand(zone, page, order, current_order, migratetype); 2323 set_pcppage_migratetype(page, migratetype); 2324 return page; 2325 } 2326 2327 return NULL; 2328 } 2329 2330 2331 /* 2332 * This array describes the order lists are fallen back to when 2333 * the free lists for the desirable migrate type are depleted 2334 */ 2335 static int fallbacks[MIGRATE_TYPES][3] = { 2336 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2337 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 2338 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2339 #ifdef CONFIG_CMA 2340 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */ 2341 #endif 2342 #ifdef CONFIG_MEMORY_ISOLATION 2343 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */ 2344 #endif 2345 }; 2346 2347 #ifdef CONFIG_CMA 2348 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2349 unsigned int order) 2350 { 2351 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 2352 } 2353 #else 2354 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2355 unsigned int order) { return NULL; } 2356 #endif 2357 2358 /* 2359 * Move the free pages in a range to the freelist tail of the requested type. 2360 * Note that start_page and end_pages are not aligned on a pageblock 2361 * boundary. If alignment is required, use move_freepages_block() 2362 */ 2363 static int move_freepages(struct zone *zone, 2364 struct page *start_page, struct page *end_page, 2365 int migratetype, int *num_movable) 2366 { 2367 struct page *page; 2368 unsigned int order; 2369 int pages_moved = 0; 2370 2371 for (page = start_page; page <= end_page;) { 2372 if (!pfn_valid_within(page_to_pfn(page))) { 2373 page++; 2374 continue; 2375 } 2376 2377 if (!PageBuddy(page)) { 2378 /* 2379 * We assume that pages that could be isolated for 2380 * migration are movable. But we don't actually try 2381 * isolating, as that would be expensive. 2382 */ 2383 if (num_movable && 2384 (PageLRU(page) || __PageMovable(page))) 2385 (*num_movable)++; 2386 2387 page++; 2388 continue; 2389 } 2390 2391 /* Make sure we are not inadvertently changing nodes */ 2392 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2393 VM_BUG_ON_PAGE(page_zone(page) != zone, page); 2394 2395 order = buddy_order(page); 2396 move_to_free_list(page, zone, order, migratetype); 2397 page += 1 << order; 2398 pages_moved += 1 << order; 2399 } 2400 2401 return pages_moved; 2402 } 2403 2404 int move_freepages_block(struct zone *zone, struct page *page, 2405 int migratetype, int *num_movable) 2406 { 2407 unsigned long start_pfn, end_pfn; 2408 struct page *start_page, *end_page; 2409 2410 if (num_movable) 2411 *num_movable = 0; 2412 2413 start_pfn = page_to_pfn(page); 2414 start_pfn = start_pfn & ~(pageblock_nr_pages-1); 2415 start_page = pfn_to_page(start_pfn); 2416 end_page = start_page + pageblock_nr_pages - 1; 2417 end_pfn = start_pfn + pageblock_nr_pages - 1; 2418 2419 /* Do not cross zone boundaries */ 2420 if (!zone_spans_pfn(zone, start_pfn)) 2421 start_page = page; 2422 if (!zone_spans_pfn(zone, end_pfn)) 2423 return 0; 2424 2425 return move_freepages(zone, start_page, end_page, migratetype, 2426 num_movable); 2427 } 2428 2429 static void change_pageblock_range(struct page *pageblock_page, 2430 int start_order, int migratetype) 2431 { 2432 int nr_pageblocks = 1 << (start_order - pageblock_order); 2433 2434 while (nr_pageblocks--) { 2435 set_pageblock_migratetype(pageblock_page, migratetype); 2436 pageblock_page += pageblock_nr_pages; 2437 } 2438 } 2439 2440 /* 2441 * When we are falling back to another migratetype during allocation, try to 2442 * steal extra free pages from the same pageblocks to satisfy further 2443 * allocations, instead of polluting multiple pageblocks. 2444 * 2445 * If we are stealing a relatively large buddy page, it is likely there will 2446 * be more free pages in the pageblock, so try to steal them all. For 2447 * reclaimable and unmovable allocations, we steal regardless of page size, 2448 * as fragmentation caused by those allocations polluting movable pageblocks 2449 * is worse than movable allocations stealing from unmovable and reclaimable 2450 * pageblocks. 2451 */ 2452 static bool can_steal_fallback(unsigned int order, int start_mt) 2453 { 2454 /* 2455 * Leaving this order check is intended, although there is 2456 * relaxed order check in next check. The reason is that 2457 * we can actually steal whole pageblock if this condition met, 2458 * but, below check doesn't guarantee it and that is just heuristic 2459 * so could be changed anytime. 2460 */ 2461 if (order >= pageblock_order) 2462 return true; 2463 2464 if (order >= pageblock_order / 2 || 2465 start_mt == MIGRATE_RECLAIMABLE || 2466 start_mt == MIGRATE_UNMOVABLE || 2467 page_group_by_mobility_disabled) 2468 return true; 2469 2470 return false; 2471 } 2472 2473 static inline void boost_watermark(struct zone *zone) 2474 { 2475 unsigned long max_boost; 2476 2477 if (!watermark_boost_factor) 2478 return; 2479 /* 2480 * Don't bother in zones that are unlikely to produce results. 2481 * On small machines, including kdump capture kernels running 2482 * in a small area, boosting the watermark can cause an out of 2483 * memory situation immediately. 2484 */ 2485 if ((pageblock_nr_pages * 4) > zone_managed_pages(zone)) 2486 return; 2487 2488 max_boost = mult_frac(zone->_watermark[WMARK_HIGH], 2489 watermark_boost_factor, 10000); 2490 2491 /* 2492 * high watermark may be uninitialised if fragmentation occurs 2493 * very early in boot so do not boost. We do not fall 2494 * through and boost by pageblock_nr_pages as failing 2495 * allocations that early means that reclaim is not going 2496 * to help and it may even be impossible to reclaim the 2497 * boosted watermark resulting in a hang. 2498 */ 2499 if (!max_boost) 2500 return; 2501 2502 max_boost = max(pageblock_nr_pages, max_boost); 2503 2504 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages, 2505 max_boost); 2506 } 2507 2508 /* 2509 * This function implements actual steal behaviour. If order is large enough, 2510 * we can steal whole pageblock. If not, we first move freepages in this 2511 * pageblock to our migratetype and determine how many already-allocated pages 2512 * are there in the pageblock with a compatible migratetype. If at least half 2513 * of pages are free or compatible, we can change migratetype of the pageblock 2514 * itself, so pages freed in the future will be put on the correct free list. 2515 */ 2516 static void steal_suitable_fallback(struct zone *zone, struct page *page, 2517 unsigned int alloc_flags, int start_type, bool whole_block) 2518 { 2519 unsigned int current_order = buddy_order(page); 2520 int free_pages, movable_pages, alike_pages; 2521 int old_block_type; 2522 2523 old_block_type = get_pageblock_migratetype(page); 2524 2525 /* 2526 * This can happen due to races and we want to prevent broken 2527 * highatomic accounting. 2528 */ 2529 if (is_migrate_highatomic(old_block_type)) 2530 goto single_page; 2531 2532 /* Take ownership for orders >= pageblock_order */ 2533 if (current_order >= pageblock_order) { 2534 change_pageblock_range(page, current_order, start_type); 2535 goto single_page; 2536 } 2537 2538 /* 2539 * Boost watermarks to increase reclaim pressure to reduce the 2540 * likelihood of future fallbacks. Wake kswapd now as the node 2541 * may be balanced overall and kswapd will not wake naturally. 2542 */ 2543 boost_watermark(zone); 2544 if (alloc_flags & ALLOC_KSWAPD) 2545 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 2546 2547 /* We are not allowed to try stealing from the whole block */ 2548 if (!whole_block) 2549 goto single_page; 2550 2551 free_pages = move_freepages_block(zone, page, start_type, 2552 &movable_pages); 2553 /* 2554 * Determine how many pages are compatible with our allocation. 2555 * For movable allocation, it's the number of movable pages which 2556 * we just obtained. For other types it's a bit more tricky. 2557 */ 2558 if (start_type == MIGRATE_MOVABLE) { 2559 alike_pages = movable_pages; 2560 } else { 2561 /* 2562 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation 2563 * to MOVABLE pageblock, consider all non-movable pages as 2564 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or 2565 * vice versa, be conservative since we can't distinguish the 2566 * exact migratetype of non-movable pages. 2567 */ 2568 if (old_block_type == MIGRATE_MOVABLE) 2569 alike_pages = pageblock_nr_pages 2570 - (free_pages + movable_pages); 2571 else 2572 alike_pages = 0; 2573 } 2574 2575 /* moving whole block can fail due to zone boundary conditions */ 2576 if (!free_pages) 2577 goto single_page; 2578 2579 /* 2580 * If a sufficient number of pages in the block are either free or of 2581 * comparable migratability as our allocation, claim the whole block. 2582 */ 2583 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || 2584 page_group_by_mobility_disabled) 2585 set_pageblock_migratetype(page, start_type); 2586 2587 return; 2588 2589 single_page: 2590 move_to_free_list(page, zone, current_order, start_type); 2591 } 2592 2593 /* 2594 * Check whether there is a suitable fallback freepage with requested order. 2595 * If only_stealable is true, this function returns fallback_mt only if 2596 * we can steal other freepages all together. This would help to reduce 2597 * fragmentation due to mixed migratetype pages in one pageblock. 2598 */ 2599 int find_suitable_fallback(struct free_area *area, unsigned int order, 2600 int migratetype, bool only_stealable, bool *can_steal) 2601 { 2602 int i; 2603 int fallback_mt; 2604 2605 if (area->nr_free == 0) 2606 return -1; 2607 2608 *can_steal = false; 2609 for (i = 0;; i++) { 2610 fallback_mt = fallbacks[migratetype][i]; 2611 if (fallback_mt == MIGRATE_TYPES) 2612 break; 2613 2614 if (free_area_empty(area, fallback_mt)) 2615 continue; 2616 2617 if (can_steal_fallback(order, migratetype)) 2618 *can_steal = true; 2619 2620 if (!only_stealable) 2621 return fallback_mt; 2622 2623 if (*can_steal) 2624 return fallback_mt; 2625 } 2626 2627 return -1; 2628 } 2629 2630 /* 2631 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2632 * there are no empty page blocks that contain a page with a suitable order 2633 */ 2634 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2635 unsigned int alloc_order) 2636 { 2637 int mt; 2638 unsigned long max_managed, flags; 2639 2640 /* 2641 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2642 * Check is race-prone but harmless. 2643 */ 2644 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages; 2645 if (zone->nr_reserved_highatomic >= max_managed) 2646 return; 2647 2648 spin_lock_irqsave(&zone->lock, flags); 2649 2650 /* Recheck the nr_reserved_highatomic limit under the lock */ 2651 if (zone->nr_reserved_highatomic >= max_managed) 2652 goto out_unlock; 2653 2654 /* Yoink! */ 2655 mt = get_pageblock_migratetype(page); 2656 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt) 2657 && !is_migrate_cma(mt)) { 2658 zone->nr_reserved_highatomic += pageblock_nr_pages; 2659 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2660 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); 2661 } 2662 2663 out_unlock: 2664 spin_unlock_irqrestore(&zone->lock, flags); 2665 } 2666 2667 /* 2668 * Used when an allocation is about to fail under memory pressure. This 2669 * potentially hurts the reliability of high-order allocations when under 2670 * intense memory pressure but failed atomic allocations should be easier 2671 * to recover from than an OOM. 2672 * 2673 * If @force is true, try to unreserve a pageblock even though highatomic 2674 * pageblock is exhausted. 2675 */ 2676 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 2677 bool force) 2678 { 2679 struct zonelist *zonelist = ac->zonelist; 2680 unsigned long flags; 2681 struct zoneref *z; 2682 struct zone *zone; 2683 struct page *page; 2684 int order; 2685 bool ret; 2686 2687 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx, 2688 ac->nodemask) { 2689 /* 2690 * Preserve at least one pageblock unless memory pressure 2691 * is really high. 2692 */ 2693 if (!force && zone->nr_reserved_highatomic <= 2694 pageblock_nr_pages) 2695 continue; 2696 2697 spin_lock_irqsave(&zone->lock, flags); 2698 for (order = 0; order < MAX_ORDER; order++) { 2699 struct free_area *area = &(zone->free_area[order]); 2700 2701 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); 2702 if (!page) 2703 continue; 2704 2705 /* 2706 * In page freeing path, migratetype change is racy so 2707 * we can counter several free pages in a pageblock 2708 * in this loop althoug we changed the pageblock type 2709 * from highatomic to ac->migratetype. So we should 2710 * adjust the count once. 2711 */ 2712 if (is_migrate_highatomic_page(page)) { 2713 /* 2714 * It should never happen but changes to 2715 * locking could inadvertently allow a per-cpu 2716 * drain to add pages to MIGRATE_HIGHATOMIC 2717 * while unreserving so be safe and watch for 2718 * underflows. 2719 */ 2720 zone->nr_reserved_highatomic -= min( 2721 pageblock_nr_pages, 2722 zone->nr_reserved_highatomic); 2723 } 2724 2725 /* 2726 * Convert to ac->migratetype and avoid the normal 2727 * pageblock stealing heuristics. Minimally, the caller 2728 * is doing the work and needs the pages. More 2729 * importantly, if the block was always converted to 2730 * MIGRATE_UNMOVABLE or another type then the number 2731 * of pageblocks that cannot be completely freed 2732 * may increase. 2733 */ 2734 set_pageblock_migratetype(page, ac->migratetype); 2735 ret = move_freepages_block(zone, page, ac->migratetype, 2736 NULL); 2737 if (ret) { 2738 spin_unlock_irqrestore(&zone->lock, flags); 2739 return ret; 2740 } 2741 } 2742 spin_unlock_irqrestore(&zone->lock, flags); 2743 } 2744 2745 return false; 2746 } 2747 2748 /* 2749 * Try finding a free buddy page on the fallback list and put it on the free 2750 * list of requested migratetype, possibly along with other pages from the same 2751 * block, depending on fragmentation avoidance heuristics. Returns true if 2752 * fallback was found so that __rmqueue_smallest() can grab it. 2753 * 2754 * The use of signed ints for order and current_order is a deliberate 2755 * deviation from the rest of this file, to make the for loop 2756 * condition simpler. 2757 */ 2758 static __always_inline bool 2759 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, 2760 unsigned int alloc_flags) 2761 { 2762 struct free_area *area; 2763 int current_order; 2764 int min_order = order; 2765 struct page *page; 2766 int fallback_mt; 2767 bool can_steal; 2768 2769 /* 2770 * Do not steal pages from freelists belonging to other pageblocks 2771 * i.e. orders < pageblock_order. If there are no local zones free, 2772 * the zonelists will be reiterated without ALLOC_NOFRAGMENT. 2773 */ 2774 if (alloc_flags & ALLOC_NOFRAGMENT) 2775 min_order = pageblock_order; 2776 2777 /* 2778 * Find the largest available free page in the other list. This roughly 2779 * approximates finding the pageblock with the most free pages, which 2780 * would be too costly to do exactly. 2781 */ 2782 for (current_order = MAX_ORDER - 1; current_order >= min_order; 2783 --current_order) { 2784 area = &(zone->free_area[current_order]); 2785 fallback_mt = find_suitable_fallback(area, current_order, 2786 start_migratetype, false, &can_steal); 2787 if (fallback_mt == -1) 2788 continue; 2789 2790 /* 2791 * We cannot steal all free pages from the pageblock and the 2792 * requested migratetype is movable. In that case it's better to 2793 * steal and split the smallest available page instead of the 2794 * largest available page, because even if the next movable 2795 * allocation falls back into a different pageblock than this 2796 * one, it won't cause permanent fragmentation. 2797 */ 2798 if (!can_steal && start_migratetype == MIGRATE_MOVABLE 2799 && current_order > order) 2800 goto find_smallest; 2801 2802 goto do_steal; 2803 } 2804 2805 return false; 2806 2807 find_smallest: 2808 for (current_order = order; current_order < MAX_ORDER; 2809 current_order++) { 2810 area = &(zone->free_area[current_order]); 2811 fallback_mt = find_suitable_fallback(area, current_order, 2812 start_migratetype, false, &can_steal); 2813 if (fallback_mt != -1) 2814 break; 2815 } 2816 2817 /* 2818 * This should not happen - we already found a suitable fallback 2819 * when looking for the largest page. 2820 */ 2821 VM_BUG_ON(current_order == MAX_ORDER); 2822 2823 do_steal: 2824 page = get_page_from_free_area(area, fallback_mt); 2825 2826 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype, 2827 can_steal); 2828 2829 trace_mm_page_alloc_extfrag(page, order, current_order, 2830 start_migratetype, fallback_mt); 2831 2832 return true; 2833 2834 } 2835 2836 /* 2837 * Do the hard work of removing an element from the buddy allocator. 2838 * Call me with the zone->lock already held. 2839 */ 2840 static __always_inline struct page * 2841 __rmqueue(struct zone *zone, unsigned int order, int migratetype, 2842 unsigned int alloc_flags) 2843 { 2844 struct page *page; 2845 2846 #ifdef CONFIG_CMA 2847 /* 2848 * Balance movable allocations between regular and CMA areas by 2849 * allocating from CMA when over half of the zone's free memory 2850 * is in the CMA area. 2851 */ 2852 if (alloc_flags & ALLOC_CMA && 2853 zone_page_state(zone, NR_FREE_CMA_PAGES) > 2854 zone_page_state(zone, NR_FREE_PAGES) / 2) { 2855 page = __rmqueue_cma_fallback(zone, order); 2856 if (page) 2857 return page; 2858 } 2859 #endif 2860 retry: 2861 page = __rmqueue_smallest(zone, order, migratetype); 2862 if (unlikely(!page)) { 2863 if (alloc_flags & ALLOC_CMA) 2864 page = __rmqueue_cma_fallback(zone, order); 2865 2866 if (!page && __rmqueue_fallback(zone, order, migratetype, 2867 alloc_flags)) 2868 goto retry; 2869 } 2870 2871 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2872 return page; 2873 } 2874 2875 /* 2876 * Obtain a specified number of elements from the buddy allocator, all under 2877 * a single hold of the lock, for efficiency. Add them to the supplied list. 2878 * Returns the number of new pages which were placed at *list. 2879 */ 2880 static int rmqueue_bulk(struct zone *zone, unsigned int order, 2881 unsigned long count, struct list_head *list, 2882 int migratetype, unsigned int alloc_flags) 2883 { 2884 int i, alloced = 0; 2885 2886 spin_lock(&zone->lock); 2887 for (i = 0; i < count; ++i) { 2888 struct page *page = __rmqueue(zone, order, migratetype, 2889 alloc_flags); 2890 if (unlikely(page == NULL)) 2891 break; 2892 2893 if (unlikely(check_pcp_refill(page))) 2894 continue; 2895 2896 /* 2897 * Split buddy pages returned by expand() are received here in 2898 * physical page order. The page is added to the tail of 2899 * caller's list. From the callers perspective, the linked list 2900 * is ordered by page number under some conditions. This is 2901 * useful for IO devices that can forward direction from the 2902 * head, thus also in the physical page order. This is useful 2903 * for IO devices that can merge IO requests if the physical 2904 * pages are ordered properly. 2905 */ 2906 list_add_tail(&page->lru, list); 2907 alloced++; 2908 if (is_migrate_cma(get_pcppage_migratetype(page))) 2909 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 2910 -(1 << order)); 2911 } 2912 2913 /* 2914 * i pages were removed from the buddy list even if some leak due 2915 * to check_pcp_refill failing so adjust NR_FREE_PAGES based 2916 * on i. Do not confuse with 'alloced' which is the number of 2917 * pages added to the pcp list. 2918 */ 2919 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 2920 spin_unlock(&zone->lock); 2921 return alloced; 2922 } 2923 2924 #ifdef CONFIG_NUMA 2925 /* 2926 * Called from the vmstat counter updater to drain pagesets of this 2927 * currently executing processor on remote nodes after they have 2928 * expired. 2929 * 2930 * Note that this function must be called with the thread pinned to 2931 * a single processor. 2932 */ 2933 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 2934 { 2935 unsigned long flags; 2936 int to_drain, batch; 2937 2938 local_irq_save(flags); 2939 batch = READ_ONCE(pcp->batch); 2940 to_drain = min(pcp->count, batch); 2941 if (to_drain > 0) 2942 free_pcppages_bulk(zone, to_drain, pcp); 2943 local_irq_restore(flags); 2944 } 2945 #endif 2946 2947 /* 2948 * Drain pcplists of the indicated processor and zone. 2949 * 2950 * The processor must either be the current processor and the 2951 * thread pinned to the current processor or a processor that 2952 * is not online. 2953 */ 2954 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 2955 { 2956 unsigned long flags; 2957 struct per_cpu_pageset *pset; 2958 struct per_cpu_pages *pcp; 2959 2960 local_irq_save(flags); 2961 pset = per_cpu_ptr(zone->pageset, cpu); 2962 2963 pcp = &pset->pcp; 2964 if (pcp->count) 2965 free_pcppages_bulk(zone, pcp->count, pcp); 2966 local_irq_restore(flags); 2967 } 2968 2969 /* 2970 * Drain pcplists of all zones on the indicated processor. 2971 * 2972 * The processor must either be the current processor and the 2973 * thread pinned to the current processor or a processor that 2974 * is not online. 2975 */ 2976 static void drain_pages(unsigned int cpu) 2977 { 2978 struct zone *zone; 2979 2980 for_each_populated_zone(zone) { 2981 drain_pages_zone(cpu, zone); 2982 } 2983 } 2984 2985 /* 2986 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 2987 * 2988 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 2989 * the single zone's pages. 2990 */ 2991 void drain_local_pages(struct zone *zone) 2992 { 2993 int cpu = smp_processor_id(); 2994 2995 if (zone) 2996 drain_pages_zone(cpu, zone); 2997 else 2998 drain_pages(cpu); 2999 } 3000 3001 static void drain_local_pages_wq(struct work_struct *work) 3002 { 3003 struct pcpu_drain *drain; 3004 3005 drain = container_of(work, struct pcpu_drain, work); 3006 3007 /* 3008 * drain_all_pages doesn't use proper cpu hotplug protection so 3009 * we can race with cpu offline when the WQ can move this from 3010 * a cpu pinned worker to an unbound one. We can operate on a different 3011 * cpu which is allright but we also have to make sure to not move to 3012 * a different one. 3013 */ 3014 preempt_disable(); 3015 drain_local_pages(drain->zone); 3016 preempt_enable(); 3017 } 3018 3019 /* 3020 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 3021 * 3022 * When zone parameter is non-NULL, spill just the single zone's pages. 3023 * 3024 * Note that this can be extremely slow as the draining happens in a workqueue. 3025 */ 3026 void drain_all_pages(struct zone *zone) 3027 { 3028 int cpu; 3029 3030 /* 3031 * Allocate in the BSS so we wont require allocation in 3032 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 3033 */ 3034 static cpumask_t cpus_with_pcps; 3035 3036 /* 3037 * Make sure nobody triggers this path before mm_percpu_wq is fully 3038 * initialized. 3039 */ 3040 if (WARN_ON_ONCE(!mm_percpu_wq)) 3041 return; 3042 3043 /* 3044 * Do not drain if one is already in progress unless it's specific to 3045 * a zone. Such callers are primarily CMA and memory hotplug and need 3046 * the drain to be complete when the call returns. 3047 */ 3048 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) { 3049 if (!zone) 3050 return; 3051 mutex_lock(&pcpu_drain_mutex); 3052 } 3053 3054 /* 3055 * We don't care about racing with CPU hotplug event 3056 * as offline notification will cause the notified 3057 * cpu to drain that CPU pcps and on_each_cpu_mask 3058 * disables preemption as part of its processing 3059 */ 3060 for_each_online_cpu(cpu) { 3061 struct per_cpu_pageset *pcp; 3062 struct zone *z; 3063 bool has_pcps = false; 3064 3065 if (zone) { 3066 pcp = per_cpu_ptr(zone->pageset, cpu); 3067 if (pcp->pcp.count) 3068 has_pcps = true; 3069 } else { 3070 for_each_populated_zone(z) { 3071 pcp = per_cpu_ptr(z->pageset, cpu); 3072 if (pcp->pcp.count) { 3073 has_pcps = true; 3074 break; 3075 } 3076 } 3077 } 3078 3079 if (has_pcps) 3080 cpumask_set_cpu(cpu, &cpus_with_pcps); 3081 else 3082 cpumask_clear_cpu(cpu, &cpus_with_pcps); 3083 } 3084 3085 for_each_cpu(cpu, &cpus_with_pcps) { 3086 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu); 3087 3088 drain->zone = zone; 3089 INIT_WORK(&drain->work, drain_local_pages_wq); 3090 queue_work_on(cpu, mm_percpu_wq, &drain->work); 3091 } 3092 for_each_cpu(cpu, &cpus_with_pcps) 3093 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work); 3094 3095 mutex_unlock(&pcpu_drain_mutex); 3096 } 3097 3098 #ifdef CONFIG_HIBERNATION 3099 3100 /* 3101 * Touch the watchdog for every WD_PAGE_COUNT pages. 3102 */ 3103 #define WD_PAGE_COUNT (128*1024) 3104 3105 void mark_free_pages(struct zone *zone) 3106 { 3107 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT; 3108 unsigned long flags; 3109 unsigned int order, t; 3110 struct page *page; 3111 3112 if (zone_is_empty(zone)) 3113 return; 3114 3115 spin_lock_irqsave(&zone->lock, flags); 3116 3117 max_zone_pfn = zone_end_pfn(zone); 3118 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 3119 if (pfn_valid(pfn)) { 3120 page = pfn_to_page(pfn); 3121 3122 if (!--page_count) { 3123 touch_nmi_watchdog(); 3124 page_count = WD_PAGE_COUNT; 3125 } 3126 3127 if (page_zone(page) != zone) 3128 continue; 3129 3130 if (!swsusp_page_is_forbidden(page)) 3131 swsusp_unset_page_free(page); 3132 } 3133 3134 for_each_migratetype_order(order, t) { 3135 list_for_each_entry(page, 3136 &zone->free_area[order].free_list[t], lru) { 3137 unsigned long i; 3138 3139 pfn = page_to_pfn(page); 3140 for (i = 0; i < (1UL << order); i++) { 3141 if (!--page_count) { 3142 touch_nmi_watchdog(); 3143 page_count = WD_PAGE_COUNT; 3144 } 3145 swsusp_set_page_free(pfn_to_page(pfn + i)); 3146 } 3147 } 3148 } 3149 spin_unlock_irqrestore(&zone->lock, flags); 3150 } 3151 #endif /* CONFIG_PM */ 3152 3153 static bool free_unref_page_prepare(struct page *page, unsigned long pfn) 3154 { 3155 int migratetype; 3156 3157 if (!free_pcp_prepare(page)) 3158 return false; 3159 3160 migratetype = get_pfnblock_migratetype(page, pfn); 3161 set_pcppage_migratetype(page, migratetype); 3162 return true; 3163 } 3164 3165 static void free_unref_page_commit(struct page *page, unsigned long pfn) 3166 { 3167 struct zone *zone = page_zone(page); 3168 struct per_cpu_pages *pcp; 3169 int migratetype; 3170 3171 migratetype = get_pcppage_migratetype(page); 3172 __count_vm_event(PGFREE); 3173 3174 /* 3175 * We only track unmovable, reclaimable and movable on pcp lists. 3176 * Free ISOLATE pages back to the allocator because they are being 3177 * offlined but treat HIGHATOMIC as movable pages so we can get those 3178 * areas back if necessary. Otherwise, we may have to free 3179 * excessively into the page allocator 3180 */ 3181 if (migratetype >= MIGRATE_PCPTYPES) { 3182 if (unlikely(is_migrate_isolate(migratetype))) { 3183 free_one_page(zone, page, pfn, 0, migratetype, 3184 FPI_NONE); 3185 return; 3186 } 3187 migratetype = MIGRATE_MOVABLE; 3188 } 3189 3190 pcp = &this_cpu_ptr(zone->pageset)->pcp; 3191 list_add(&page->lru, &pcp->lists[migratetype]); 3192 pcp->count++; 3193 if (pcp->count >= pcp->high) { 3194 unsigned long batch = READ_ONCE(pcp->batch); 3195 free_pcppages_bulk(zone, batch, pcp); 3196 } 3197 } 3198 3199 /* 3200 * Free a 0-order page 3201 */ 3202 void free_unref_page(struct page *page) 3203 { 3204 unsigned long flags; 3205 unsigned long pfn = page_to_pfn(page); 3206 3207 if (!free_unref_page_prepare(page, pfn)) 3208 return; 3209 3210 local_irq_save(flags); 3211 free_unref_page_commit(page, pfn); 3212 local_irq_restore(flags); 3213 } 3214 3215 /* 3216 * Free a list of 0-order pages 3217 */ 3218 void free_unref_page_list(struct list_head *list) 3219 { 3220 struct page *page, *next; 3221 unsigned long flags, pfn; 3222 int batch_count = 0; 3223 3224 /* Prepare pages for freeing */ 3225 list_for_each_entry_safe(page, next, list, lru) { 3226 pfn = page_to_pfn(page); 3227 if (!free_unref_page_prepare(page, pfn)) 3228 list_del(&page->lru); 3229 set_page_private(page, pfn); 3230 } 3231 3232 local_irq_save(flags); 3233 list_for_each_entry_safe(page, next, list, lru) { 3234 unsigned long pfn = page_private(page); 3235 3236 set_page_private(page, 0); 3237 trace_mm_page_free_batched(page); 3238 free_unref_page_commit(page, pfn); 3239 3240 /* 3241 * Guard against excessive IRQ disabled times when we get 3242 * a large list of pages to free. 3243 */ 3244 if (++batch_count == SWAP_CLUSTER_MAX) { 3245 local_irq_restore(flags); 3246 batch_count = 0; 3247 local_irq_save(flags); 3248 } 3249 } 3250 local_irq_restore(flags); 3251 } 3252 3253 /* 3254 * split_page takes a non-compound higher-order page, and splits it into 3255 * n (1<<order) sub-pages: page[0..n] 3256 * Each sub-page must be freed individually. 3257 * 3258 * Note: this is probably too low level an operation for use in drivers. 3259 * Please consult with lkml before using this in your driver. 3260 */ 3261 void split_page(struct page *page, unsigned int order) 3262 { 3263 int i; 3264 3265 VM_BUG_ON_PAGE(PageCompound(page), page); 3266 VM_BUG_ON_PAGE(!page_count(page), page); 3267 3268 for (i = 1; i < (1 << order); i++) 3269 set_page_refcounted(page + i); 3270 split_page_owner(page, 1 << order); 3271 } 3272 EXPORT_SYMBOL_GPL(split_page); 3273 3274 int __isolate_free_page(struct page *page, unsigned int order) 3275 { 3276 unsigned long watermark; 3277 struct zone *zone; 3278 int mt; 3279 3280 BUG_ON(!PageBuddy(page)); 3281 3282 zone = page_zone(page); 3283 mt = get_pageblock_migratetype(page); 3284 3285 if (!is_migrate_isolate(mt)) { 3286 /* 3287 * Obey watermarks as if the page was being allocated. We can 3288 * emulate a high-order watermark check with a raised order-0 3289 * watermark, because we already know our high-order page 3290 * exists. 3291 */ 3292 watermark = zone->_watermark[WMARK_MIN] + (1UL << order); 3293 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 3294 return 0; 3295 3296 __mod_zone_freepage_state(zone, -(1UL << order), mt); 3297 } 3298 3299 /* Remove page from free list */ 3300 3301 del_page_from_free_list(page, zone, order); 3302 3303 /* 3304 * Set the pageblock if the isolated page is at least half of a 3305 * pageblock 3306 */ 3307 if (order >= pageblock_order - 1) { 3308 struct page *endpage = page + (1 << order) - 1; 3309 for (; page < endpage; page += pageblock_nr_pages) { 3310 int mt = get_pageblock_migratetype(page); 3311 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt) 3312 && !is_migrate_highatomic(mt)) 3313 set_pageblock_migratetype(page, 3314 MIGRATE_MOVABLE); 3315 } 3316 } 3317 3318 3319 return 1UL << order; 3320 } 3321 3322 /** 3323 * __putback_isolated_page - Return a now-isolated page back where we got it 3324 * @page: Page that was isolated 3325 * @order: Order of the isolated page 3326 * @mt: The page's pageblock's migratetype 3327 * 3328 * This function is meant to return a page pulled from the free lists via 3329 * __isolate_free_page back to the free lists they were pulled from. 3330 */ 3331 void __putback_isolated_page(struct page *page, unsigned int order, int mt) 3332 { 3333 struct zone *zone = page_zone(page); 3334 3335 /* zone lock should be held when this function is called */ 3336 lockdep_assert_held(&zone->lock); 3337 3338 /* Return isolated page to tail of freelist. */ 3339 __free_one_page(page, page_to_pfn(page), zone, order, mt, 3340 FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL); 3341 } 3342 3343 /* 3344 * Update NUMA hit/miss statistics 3345 * 3346 * Must be called with interrupts disabled. 3347 */ 3348 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) 3349 { 3350 #ifdef CONFIG_NUMA 3351 enum numa_stat_item local_stat = NUMA_LOCAL; 3352 3353 /* skip numa counters update if numa stats is disabled */ 3354 if (!static_branch_likely(&vm_numa_stat_key)) 3355 return; 3356 3357 if (zone_to_nid(z) != numa_node_id()) 3358 local_stat = NUMA_OTHER; 3359 3360 if (zone_to_nid(z) == zone_to_nid(preferred_zone)) 3361 __inc_numa_state(z, NUMA_HIT); 3362 else { 3363 __inc_numa_state(z, NUMA_MISS); 3364 __inc_numa_state(preferred_zone, NUMA_FOREIGN); 3365 } 3366 __inc_numa_state(z, local_stat); 3367 #endif 3368 } 3369 3370 /* Remove page from the per-cpu list, caller must protect the list */ 3371 static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype, 3372 unsigned int alloc_flags, 3373 struct per_cpu_pages *pcp, 3374 struct list_head *list) 3375 { 3376 struct page *page; 3377 3378 do { 3379 if (list_empty(list)) { 3380 pcp->count += rmqueue_bulk(zone, 0, 3381 pcp->batch, list, 3382 migratetype, alloc_flags); 3383 if (unlikely(list_empty(list))) 3384 return NULL; 3385 } 3386 3387 page = list_first_entry(list, struct page, lru); 3388 list_del(&page->lru); 3389 pcp->count--; 3390 } while (check_new_pcp(page)); 3391 3392 return page; 3393 } 3394 3395 /* Lock and remove page from the per-cpu list */ 3396 static struct page *rmqueue_pcplist(struct zone *preferred_zone, 3397 struct zone *zone, gfp_t gfp_flags, 3398 int migratetype, unsigned int alloc_flags) 3399 { 3400 struct per_cpu_pages *pcp; 3401 struct list_head *list; 3402 struct page *page; 3403 unsigned long flags; 3404 3405 local_irq_save(flags); 3406 pcp = &this_cpu_ptr(zone->pageset)->pcp; 3407 list = &pcp->lists[migratetype]; 3408 page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list); 3409 if (page) { 3410 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); 3411 zone_statistics(preferred_zone, zone); 3412 } 3413 local_irq_restore(flags); 3414 return page; 3415 } 3416 3417 /* 3418 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 3419 */ 3420 static inline 3421 struct page *rmqueue(struct zone *preferred_zone, 3422 struct zone *zone, unsigned int order, 3423 gfp_t gfp_flags, unsigned int alloc_flags, 3424 int migratetype) 3425 { 3426 unsigned long flags; 3427 struct page *page; 3428 3429 if (likely(order == 0)) { 3430 /* 3431 * MIGRATE_MOVABLE pcplist could have the pages on CMA area and 3432 * we need to skip it when CMA area isn't allowed. 3433 */ 3434 if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA || 3435 migratetype != MIGRATE_MOVABLE) { 3436 page = rmqueue_pcplist(preferred_zone, zone, gfp_flags, 3437 migratetype, alloc_flags); 3438 goto out; 3439 } 3440 } 3441 3442 /* 3443 * We most definitely don't want callers attempting to 3444 * allocate greater than order-1 page units with __GFP_NOFAIL. 3445 */ 3446 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 3447 spin_lock_irqsave(&zone->lock, flags); 3448 3449 do { 3450 page = NULL; 3451 /* 3452 * order-0 request can reach here when the pcplist is skipped 3453 * due to non-CMA allocation context. HIGHATOMIC area is 3454 * reserved for high-order atomic allocation, so order-0 3455 * request should skip it. 3456 */ 3457 if (order > 0 && alloc_flags & ALLOC_HARDER) { 3458 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3459 if (page) 3460 trace_mm_page_alloc_zone_locked(page, order, migratetype); 3461 } 3462 if (!page) 3463 page = __rmqueue(zone, order, migratetype, alloc_flags); 3464 } while (page && check_new_pages(page, order)); 3465 spin_unlock(&zone->lock); 3466 if (!page) 3467 goto failed; 3468 __mod_zone_freepage_state(zone, -(1 << order), 3469 get_pcppage_migratetype(page)); 3470 3471 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3472 zone_statistics(preferred_zone, zone); 3473 local_irq_restore(flags); 3474 3475 out: 3476 /* Separate test+clear to avoid unnecessary atomics */ 3477 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) { 3478 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 3479 wakeup_kswapd(zone, 0, 0, zone_idx(zone)); 3480 } 3481 3482 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); 3483 return page; 3484 3485 failed: 3486 local_irq_restore(flags); 3487 return NULL; 3488 } 3489 3490 #ifdef CONFIG_FAIL_PAGE_ALLOC 3491 3492 static struct { 3493 struct fault_attr attr; 3494 3495 bool ignore_gfp_highmem; 3496 bool ignore_gfp_reclaim; 3497 u32 min_order; 3498 } fail_page_alloc = { 3499 .attr = FAULT_ATTR_INITIALIZER, 3500 .ignore_gfp_reclaim = true, 3501 .ignore_gfp_highmem = true, 3502 .min_order = 1, 3503 }; 3504 3505 static int __init setup_fail_page_alloc(char *str) 3506 { 3507 return setup_fault_attr(&fail_page_alloc.attr, str); 3508 } 3509 __setup("fail_page_alloc=", setup_fail_page_alloc); 3510 3511 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3512 { 3513 if (order < fail_page_alloc.min_order) 3514 return false; 3515 if (gfp_mask & __GFP_NOFAIL) 3516 return false; 3517 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 3518 return false; 3519 if (fail_page_alloc.ignore_gfp_reclaim && 3520 (gfp_mask & __GFP_DIRECT_RECLAIM)) 3521 return false; 3522 3523 return should_fail(&fail_page_alloc.attr, 1 << order); 3524 } 3525 3526 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 3527 3528 static int __init fail_page_alloc_debugfs(void) 3529 { 3530 umode_t mode = S_IFREG | 0600; 3531 struct dentry *dir; 3532 3533 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 3534 &fail_page_alloc.attr); 3535 3536 debugfs_create_bool("ignore-gfp-wait", mode, dir, 3537 &fail_page_alloc.ignore_gfp_reclaim); 3538 debugfs_create_bool("ignore-gfp-highmem", mode, dir, 3539 &fail_page_alloc.ignore_gfp_highmem); 3540 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order); 3541 3542 return 0; 3543 } 3544 3545 late_initcall(fail_page_alloc_debugfs); 3546 3547 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 3548 3549 #else /* CONFIG_FAIL_PAGE_ALLOC */ 3550 3551 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3552 { 3553 return false; 3554 } 3555 3556 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 3557 3558 noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3559 { 3560 return __should_fail_alloc_page(gfp_mask, order); 3561 } 3562 ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE); 3563 3564 static inline long __zone_watermark_unusable_free(struct zone *z, 3565 unsigned int order, unsigned int alloc_flags) 3566 { 3567 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3568 long unusable_free = (1 << order) - 1; 3569 3570 /* 3571 * If the caller does not have rights to ALLOC_HARDER then subtract 3572 * the high-atomic reserves. This will over-estimate the size of the 3573 * atomic reserve but it avoids a search. 3574 */ 3575 if (likely(!alloc_harder)) 3576 unusable_free += z->nr_reserved_highatomic; 3577 3578 #ifdef CONFIG_CMA 3579 /* If allocation can't use CMA areas don't use free CMA pages */ 3580 if (!(alloc_flags & ALLOC_CMA)) 3581 unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES); 3582 #endif 3583 3584 return unusable_free; 3585 } 3586 3587 /* 3588 * Return true if free base pages are above 'mark'. For high-order checks it 3589 * will return true of the order-0 watermark is reached and there is at least 3590 * one free page of a suitable size. Checking now avoids taking the zone lock 3591 * to check in the allocation paths if no pages are free. 3592 */ 3593 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3594 int highest_zoneidx, unsigned int alloc_flags, 3595 long free_pages) 3596 { 3597 long min = mark; 3598 int o; 3599 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3600 3601 /* free_pages may go negative - that's OK */ 3602 free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags); 3603 3604 if (alloc_flags & ALLOC_HIGH) 3605 min -= min / 2; 3606 3607 if (unlikely(alloc_harder)) { 3608 /* 3609 * OOM victims can try even harder than normal ALLOC_HARDER 3610 * users on the grounds that it's definitely going to be in 3611 * the exit path shortly and free memory. Any allocation it 3612 * makes during the free path will be small and short-lived. 3613 */ 3614 if (alloc_flags & ALLOC_OOM) 3615 min -= min / 2; 3616 else 3617 min -= min / 4; 3618 } 3619 3620 /* 3621 * Check watermarks for an order-0 allocation request. If these 3622 * are not met, then a high-order request also cannot go ahead 3623 * even if a suitable page happened to be free. 3624 */ 3625 if (free_pages <= min + z->lowmem_reserve[highest_zoneidx]) 3626 return false; 3627 3628 /* If this is an order-0 request then the watermark is fine */ 3629 if (!order) 3630 return true; 3631 3632 /* For a high-order request, check at least one suitable page is free */ 3633 for (o = order; o < MAX_ORDER; o++) { 3634 struct free_area *area = &z->free_area[o]; 3635 int mt; 3636 3637 if (!area->nr_free) 3638 continue; 3639 3640 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 3641 if (!free_area_empty(area, mt)) 3642 return true; 3643 } 3644 3645 #ifdef CONFIG_CMA 3646 if ((alloc_flags & ALLOC_CMA) && 3647 !free_area_empty(area, MIGRATE_CMA)) { 3648 return true; 3649 } 3650 #endif 3651 if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC)) 3652 return true; 3653 } 3654 return false; 3655 } 3656 3657 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3658 int highest_zoneidx, unsigned int alloc_flags) 3659 { 3660 return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 3661 zone_page_state(z, NR_FREE_PAGES)); 3662 } 3663 3664 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 3665 unsigned long mark, int highest_zoneidx, 3666 unsigned int alloc_flags, gfp_t gfp_mask) 3667 { 3668 long free_pages; 3669 3670 free_pages = zone_page_state(z, NR_FREE_PAGES); 3671 3672 /* 3673 * Fast check for order-0 only. If this fails then the reserves 3674 * need to be calculated. 3675 */ 3676 if (!order) { 3677 long fast_free; 3678 3679 fast_free = free_pages; 3680 fast_free -= __zone_watermark_unusable_free(z, 0, alloc_flags); 3681 if (fast_free > mark + z->lowmem_reserve[highest_zoneidx]) 3682 return true; 3683 } 3684 3685 if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 3686 free_pages)) 3687 return true; 3688 /* 3689 * Ignore watermark boosting for GFP_ATOMIC order-0 allocations 3690 * when checking the min watermark. The min watermark is the 3691 * point where boosting is ignored so that kswapd is woken up 3692 * when below the low watermark. 3693 */ 3694 if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost 3695 && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) { 3696 mark = z->_watermark[WMARK_MIN]; 3697 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 3698 alloc_flags, free_pages); 3699 } 3700 3701 return false; 3702 } 3703 3704 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 3705 unsigned long mark, int highest_zoneidx) 3706 { 3707 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3708 3709 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 3710 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 3711 3712 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0, 3713 free_pages); 3714 } 3715 3716 #ifdef CONFIG_NUMA 3717 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3718 { 3719 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= 3720 node_reclaim_distance; 3721 } 3722 #else /* CONFIG_NUMA */ 3723 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3724 { 3725 return true; 3726 } 3727 #endif /* CONFIG_NUMA */ 3728 3729 /* 3730 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid 3731 * fragmentation is subtle. If the preferred zone was HIGHMEM then 3732 * premature use of a lower zone may cause lowmem pressure problems that 3733 * are worse than fragmentation. If the next zone is ZONE_DMA then it is 3734 * probably too small. It only makes sense to spread allocations to avoid 3735 * fragmentation between the Normal and DMA32 zones. 3736 */ 3737 static inline unsigned int 3738 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) 3739 { 3740 unsigned int alloc_flags; 3741 3742 /* 3743 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 3744 * to save a branch. 3745 */ 3746 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); 3747 3748 #ifdef CONFIG_ZONE_DMA32 3749 if (!zone) 3750 return alloc_flags; 3751 3752 if (zone_idx(zone) != ZONE_NORMAL) 3753 return alloc_flags; 3754 3755 /* 3756 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and 3757 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume 3758 * on UMA that if Normal is populated then so is DMA32. 3759 */ 3760 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1); 3761 if (nr_online_nodes > 1 && !populated_zone(--zone)) 3762 return alloc_flags; 3763 3764 alloc_flags |= ALLOC_NOFRAGMENT; 3765 #endif /* CONFIG_ZONE_DMA32 */ 3766 return alloc_flags; 3767 } 3768 3769 static inline unsigned int current_alloc_flags(gfp_t gfp_mask, 3770 unsigned int alloc_flags) 3771 { 3772 #ifdef CONFIG_CMA 3773 unsigned int pflags = current->flags; 3774 3775 if (!(pflags & PF_MEMALLOC_NOCMA) && 3776 gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) 3777 alloc_flags |= ALLOC_CMA; 3778 3779 #endif 3780 return alloc_flags; 3781 } 3782 3783 /* 3784 * get_page_from_freelist goes through the zonelist trying to allocate 3785 * a page. 3786 */ 3787 static struct page * 3788 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 3789 const struct alloc_context *ac) 3790 { 3791 struct zoneref *z; 3792 struct zone *zone; 3793 struct pglist_data *last_pgdat_dirty_limit = NULL; 3794 bool no_fallback; 3795 3796 retry: 3797 /* 3798 * Scan zonelist, looking for a zone with enough free. 3799 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 3800 */ 3801 no_fallback = alloc_flags & ALLOC_NOFRAGMENT; 3802 z = ac->preferred_zoneref; 3803 for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx, 3804 ac->nodemask) { 3805 struct page *page; 3806 unsigned long mark; 3807 3808 if (cpusets_enabled() && 3809 (alloc_flags & ALLOC_CPUSET) && 3810 !__cpuset_zone_allowed(zone, gfp_mask)) 3811 continue; 3812 /* 3813 * When allocating a page cache page for writing, we 3814 * want to get it from a node that is within its dirty 3815 * limit, such that no single node holds more than its 3816 * proportional share of globally allowed dirty pages. 3817 * The dirty limits take into account the node's 3818 * lowmem reserves and high watermark so that kswapd 3819 * should be able to balance it without having to 3820 * write pages from its LRU list. 3821 * 3822 * XXX: For now, allow allocations to potentially 3823 * exceed the per-node dirty limit in the slowpath 3824 * (spread_dirty_pages unset) before going into reclaim, 3825 * which is important when on a NUMA setup the allowed 3826 * nodes are together not big enough to reach the 3827 * global limit. The proper fix for these situations 3828 * will require awareness of nodes in the 3829 * dirty-throttling and the flusher threads. 3830 */ 3831 if (ac->spread_dirty_pages) { 3832 if (last_pgdat_dirty_limit == zone->zone_pgdat) 3833 continue; 3834 3835 if (!node_dirty_ok(zone->zone_pgdat)) { 3836 last_pgdat_dirty_limit = zone->zone_pgdat; 3837 continue; 3838 } 3839 } 3840 3841 if (no_fallback && nr_online_nodes > 1 && 3842 zone != ac->preferred_zoneref->zone) { 3843 int local_nid; 3844 3845 /* 3846 * If moving to a remote node, retry but allow 3847 * fragmenting fallbacks. Locality is more important 3848 * than fragmentation avoidance. 3849 */ 3850 local_nid = zone_to_nid(ac->preferred_zoneref->zone); 3851 if (zone_to_nid(zone) != local_nid) { 3852 alloc_flags &= ~ALLOC_NOFRAGMENT; 3853 goto retry; 3854 } 3855 } 3856 3857 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); 3858 if (!zone_watermark_fast(zone, order, mark, 3859 ac->highest_zoneidx, alloc_flags, 3860 gfp_mask)) { 3861 int ret; 3862 3863 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3864 /* 3865 * Watermark failed for this zone, but see if we can 3866 * grow this zone if it contains deferred pages. 3867 */ 3868 if (static_branch_unlikely(&deferred_pages)) { 3869 if (_deferred_grow_zone(zone, order)) 3870 goto try_this_zone; 3871 } 3872 #endif 3873 /* Checked here to keep the fast path fast */ 3874 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 3875 if (alloc_flags & ALLOC_NO_WATERMARKS) 3876 goto try_this_zone; 3877 3878 if (node_reclaim_mode == 0 || 3879 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 3880 continue; 3881 3882 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 3883 switch (ret) { 3884 case NODE_RECLAIM_NOSCAN: 3885 /* did not scan */ 3886 continue; 3887 case NODE_RECLAIM_FULL: 3888 /* scanned but unreclaimable */ 3889 continue; 3890 default: 3891 /* did we reclaim enough */ 3892 if (zone_watermark_ok(zone, order, mark, 3893 ac->highest_zoneidx, alloc_flags)) 3894 goto try_this_zone; 3895 3896 continue; 3897 } 3898 } 3899 3900 try_this_zone: 3901 page = rmqueue(ac->preferred_zoneref->zone, zone, order, 3902 gfp_mask, alloc_flags, ac->migratetype); 3903 if (page) { 3904 prep_new_page(page, order, gfp_mask, alloc_flags); 3905 3906 /* 3907 * If this is a high-order atomic allocation then check 3908 * if the pageblock should be reserved for the future 3909 */ 3910 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 3911 reserve_highatomic_pageblock(page, zone, order); 3912 3913 return page; 3914 } else { 3915 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3916 /* Try again if zone has deferred pages */ 3917 if (static_branch_unlikely(&deferred_pages)) { 3918 if (_deferred_grow_zone(zone, order)) 3919 goto try_this_zone; 3920 } 3921 #endif 3922 } 3923 } 3924 3925 /* 3926 * It's possible on a UMA machine to get through all zones that are 3927 * fragmented. If avoiding fragmentation, reset and try again. 3928 */ 3929 if (no_fallback) { 3930 alloc_flags &= ~ALLOC_NOFRAGMENT; 3931 goto retry; 3932 } 3933 3934 return NULL; 3935 } 3936 3937 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) 3938 { 3939 unsigned int filter = SHOW_MEM_FILTER_NODES; 3940 3941 /* 3942 * This documents exceptions given to allocations in certain 3943 * contexts that are allowed to allocate outside current's set 3944 * of allowed nodes. 3945 */ 3946 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3947 if (tsk_is_oom_victim(current) || 3948 (current->flags & (PF_MEMALLOC | PF_EXITING))) 3949 filter &= ~SHOW_MEM_FILTER_NODES; 3950 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 3951 filter &= ~SHOW_MEM_FILTER_NODES; 3952 3953 show_mem(filter, nodemask); 3954 } 3955 3956 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) 3957 { 3958 struct va_format vaf; 3959 va_list args; 3960 static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1); 3961 3962 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs)) 3963 return; 3964 3965 va_start(args, fmt); 3966 vaf.fmt = fmt; 3967 vaf.va = &args; 3968 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl", 3969 current->comm, &vaf, gfp_mask, &gfp_mask, 3970 nodemask_pr_args(nodemask)); 3971 va_end(args); 3972 3973 cpuset_print_current_mems_allowed(); 3974 pr_cont("\n"); 3975 dump_stack(); 3976 warn_alloc_show_mem(gfp_mask, nodemask); 3977 } 3978 3979 static inline struct page * 3980 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, 3981 unsigned int alloc_flags, 3982 const struct alloc_context *ac) 3983 { 3984 struct page *page; 3985 3986 page = get_page_from_freelist(gfp_mask, order, 3987 alloc_flags|ALLOC_CPUSET, ac); 3988 /* 3989 * fallback to ignore cpuset restriction if our nodes 3990 * are depleted 3991 */ 3992 if (!page) 3993 page = get_page_from_freelist(gfp_mask, order, 3994 alloc_flags, ac); 3995 3996 return page; 3997 } 3998 3999 static inline struct page * 4000 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 4001 const struct alloc_context *ac, unsigned long *did_some_progress) 4002 { 4003 struct oom_control oc = { 4004 .zonelist = ac->zonelist, 4005 .nodemask = ac->nodemask, 4006 .memcg = NULL, 4007 .gfp_mask = gfp_mask, 4008 .order = order, 4009 }; 4010 struct page *page; 4011 4012 *did_some_progress = 0; 4013 4014 /* 4015 * Acquire the oom lock. If that fails, somebody else is 4016 * making progress for us. 4017 */ 4018 if (!mutex_trylock(&oom_lock)) { 4019 *did_some_progress = 1; 4020 schedule_timeout_uninterruptible(1); 4021 return NULL; 4022 } 4023 4024 /* 4025 * Go through the zonelist yet one more time, keep very high watermark 4026 * here, this is only to catch a parallel oom killing, we must fail if 4027 * we're still under heavy pressure. But make sure that this reclaim 4028 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY 4029 * allocation which will never fail due to oom_lock already held. 4030 */ 4031 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & 4032 ~__GFP_DIRECT_RECLAIM, order, 4033 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 4034 if (page) 4035 goto out; 4036 4037 /* Coredumps can quickly deplete all memory reserves */ 4038 if (current->flags & PF_DUMPCORE) 4039 goto out; 4040 /* The OOM killer will not help higher order allocs */ 4041 if (order > PAGE_ALLOC_COSTLY_ORDER) 4042 goto out; 4043 /* 4044 * We have already exhausted all our reclaim opportunities without any 4045 * success so it is time to admit defeat. We will skip the OOM killer 4046 * because it is very likely that the caller has a more reasonable 4047 * fallback than shooting a random task. 4048 * 4049 * The OOM killer may not free memory on a specific node. 4050 */ 4051 if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE)) 4052 goto out; 4053 /* The OOM killer does not needlessly kill tasks for lowmem */ 4054 if (ac->highest_zoneidx < ZONE_NORMAL) 4055 goto out; 4056 if (pm_suspended_storage()) 4057 goto out; 4058 /* 4059 * XXX: GFP_NOFS allocations should rather fail than rely on 4060 * other request to make a forward progress. 4061 * We are in an unfortunate situation where out_of_memory cannot 4062 * do much for this context but let's try it to at least get 4063 * access to memory reserved if the current task is killed (see 4064 * out_of_memory). Once filesystems are ready to handle allocation 4065 * failures more gracefully we should just bail out here. 4066 */ 4067 4068 /* Exhausted what can be done so it's blame time */ 4069 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 4070 *did_some_progress = 1; 4071 4072 /* 4073 * Help non-failing allocations by giving them access to memory 4074 * reserves 4075 */ 4076 if (gfp_mask & __GFP_NOFAIL) 4077 page = __alloc_pages_cpuset_fallback(gfp_mask, order, 4078 ALLOC_NO_WATERMARKS, ac); 4079 } 4080 out: 4081 mutex_unlock(&oom_lock); 4082 return page; 4083 } 4084 4085 /* 4086 * Maximum number of compaction retries wit a progress before OOM 4087 * killer is consider as the only way to move forward. 4088 */ 4089 #define MAX_COMPACT_RETRIES 16 4090 4091 #ifdef CONFIG_COMPACTION 4092 /* Try memory compaction for high-order allocations before reclaim */ 4093 static struct page * 4094 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4095 unsigned int alloc_flags, const struct alloc_context *ac, 4096 enum compact_priority prio, enum compact_result *compact_result) 4097 { 4098 struct page *page = NULL; 4099 unsigned long pflags; 4100 unsigned int noreclaim_flag; 4101 4102 if (!order) 4103 return NULL; 4104 4105 psi_memstall_enter(&pflags); 4106 noreclaim_flag = memalloc_noreclaim_save(); 4107 4108 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 4109 prio, &page); 4110 4111 memalloc_noreclaim_restore(noreclaim_flag); 4112 psi_memstall_leave(&pflags); 4113 4114 /* 4115 * At least in one zone compaction wasn't deferred or skipped, so let's 4116 * count a compaction stall 4117 */ 4118 count_vm_event(COMPACTSTALL); 4119 4120 /* Prep a captured page if available */ 4121 if (page) 4122 prep_new_page(page, order, gfp_mask, alloc_flags); 4123 4124 /* Try get a page from the freelist if available */ 4125 if (!page) 4126 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4127 4128 if (page) { 4129 struct zone *zone = page_zone(page); 4130 4131 zone->compact_blockskip_flush = false; 4132 compaction_defer_reset(zone, order, true); 4133 count_vm_event(COMPACTSUCCESS); 4134 return page; 4135 } 4136 4137 /* 4138 * It's bad if compaction run occurs and fails. The most likely reason 4139 * is that pages exist, but not enough to satisfy watermarks. 4140 */ 4141 count_vm_event(COMPACTFAIL); 4142 4143 cond_resched(); 4144 4145 return NULL; 4146 } 4147 4148 static inline bool 4149 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 4150 enum compact_result compact_result, 4151 enum compact_priority *compact_priority, 4152 int *compaction_retries) 4153 { 4154 int max_retries = MAX_COMPACT_RETRIES; 4155 int min_priority; 4156 bool ret = false; 4157 int retries = *compaction_retries; 4158 enum compact_priority priority = *compact_priority; 4159 4160 if (!order) 4161 return false; 4162 4163 if (compaction_made_progress(compact_result)) 4164 (*compaction_retries)++; 4165 4166 /* 4167 * compaction considers all the zone as desperately out of memory 4168 * so it doesn't really make much sense to retry except when the 4169 * failure could be caused by insufficient priority 4170 */ 4171 if (compaction_failed(compact_result)) 4172 goto check_priority; 4173 4174 /* 4175 * compaction was skipped because there are not enough order-0 pages 4176 * to work with, so we retry only if it looks like reclaim can help. 4177 */ 4178 if (compaction_needs_reclaim(compact_result)) { 4179 ret = compaction_zonelist_suitable(ac, order, alloc_flags); 4180 goto out; 4181 } 4182 4183 /* 4184 * make sure the compaction wasn't deferred or didn't bail out early 4185 * due to locks contention before we declare that we should give up. 4186 * But the next retry should use a higher priority if allowed, so 4187 * we don't just keep bailing out endlessly. 4188 */ 4189 if (compaction_withdrawn(compact_result)) { 4190 goto check_priority; 4191 } 4192 4193 /* 4194 * !costly requests are much more important than __GFP_RETRY_MAYFAIL 4195 * costly ones because they are de facto nofail and invoke OOM 4196 * killer to move on while costly can fail and users are ready 4197 * to cope with that. 1/4 retries is rather arbitrary but we 4198 * would need much more detailed feedback from compaction to 4199 * make a better decision. 4200 */ 4201 if (order > PAGE_ALLOC_COSTLY_ORDER) 4202 max_retries /= 4; 4203 if (*compaction_retries <= max_retries) { 4204 ret = true; 4205 goto out; 4206 } 4207 4208 /* 4209 * Make sure there are attempts at the highest priority if we exhausted 4210 * all retries or failed at the lower priorities. 4211 */ 4212 check_priority: 4213 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 4214 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 4215 4216 if (*compact_priority > min_priority) { 4217 (*compact_priority)--; 4218 *compaction_retries = 0; 4219 ret = true; 4220 } 4221 out: 4222 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret); 4223 return ret; 4224 } 4225 #else 4226 static inline struct page * 4227 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4228 unsigned int alloc_flags, const struct alloc_context *ac, 4229 enum compact_priority prio, enum compact_result *compact_result) 4230 { 4231 *compact_result = COMPACT_SKIPPED; 4232 return NULL; 4233 } 4234 4235 static inline bool 4236 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 4237 enum compact_result compact_result, 4238 enum compact_priority *compact_priority, 4239 int *compaction_retries) 4240 { 4241 struct zone *zone; 4242 struct zoneref *z; 4243 4244 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 4245 return false; 4246 4247 /* 4248 * There are setups with compaction disabled which would prefer to loop 4249 * inside the allocator rather than hit the oom killer prematurely. 4250 * Let's give them a good hope and keep retrying while the order-0 4251 * watermarks are OK. 4252 */ 4253 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4254 ac->highest_zoneidx, ac->nodemask) { 4255 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 4256 ac->highest_zoneidx, alloc_flags)) 4257 return true; 4258 } 4259 return false; 4260 } 4261 #endif /* CONFIG_COMPACTION */ 4262 4263 #ifdef CONFIG_LOCKDEP 4264 static struct lockdep_map __fs_reclaim_map = 4265 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map); 4266 4267 static bool __need_fs_reclaim(gfp_t gfp_mask) 4268 { 4269 gfp_mask = current_gfp_context(gfp_mask); 4270 4271 /* no reclaim without waiting on it */ 4272 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) 4273 return false; 4274 4275 /* this guy won't enter reclaim */ 4276 if (current->flags & PF_MEMALLOC) 4277 return false; 4278 4279 /* We're only interested __GFP_FS allocations for now */ 4280 if (!(gfp_mask & __GFP_FS)) 4281 return false; 4282 4283 if (gfp_mask & __GFP_NOLOCKDEP) 4284 return false; 4285 4286 return true; 4287 } 4288 4289 void __fs_reclaim_acquire(void) 4290 { 4291 lock_map_acquire(&__fs_reclaim_map); 4292 } 4293 4294 void __fs_reclaim_release(void) 4295 { 4296 lock_map_release(&__fs_reclaim_map); 4297 } 4298 4299 void fs_reclaim_acquire(gfp_t gfp_mask) 4300 { 4301 if (__need_fs_reclaim(gfp_mask)) 4302 __fs_reclaim_acquire(); 4303 } 4304 EXPORT_SYMBOL_GPL(fs_reclaim_acquire); 4305 4306 void fs_reclaim_release(gfp_t gfp_mask) 4307 { 4308 if (__need_fs_reclaim(gfp_mask)) 4309 __fs_reclaim_release(); 4310 } 4311 EXPORT_SYMBOL_GPL(fs_reclaim_release); 4312 #endif 4313 4314 /* Perform direct synchronous page reclaim */ 4315 static unsigned long 4316 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 4317 const struct alloc_context *ac) 4318 { 4319 unsigned int noreclaim_flag; 4320 unsigned long pflags, progress; 4321 4322 cond_resched(); 4323 4324 /* We now go into synchronous reclaim */ 4325 cpuset_memory_pressure_bump(); 4326 psi_memstall_enter(&pflags); 4327 fs_reclaim_acquire(gfp_mask); 4328 noreclaim_flag = memalloc_noreclaim_save(); 4329 4330 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 4331 ac->nodemask); 4332 4333 memalloc_noreclaim_restore(noreclaim_flag); 4334 fs_reclaim_release(gfp_mask); 4335 psi_memstall_leave(&pflags); 4336 4337 cond_resched(); 4338 4339 return progress; 4340 } 4341 4342 /* The really slow allocator path where we enter direct reclaim */ 4343 static inline struct page * 4344 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 4345 unsigned int alloc_flags, const struct alloc_context *ac, 4346 unsigned long *did_some_progress) 4347 { 4348 struct page *page = NULL; 4349 bool drained = false; 4350 4351 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 4352 if (unlikely(!(*did_some_progress))) 4353 return NULL; 4354 4355 retry: 4356 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4357 4358 /* 4359 * If an allocation failed after direct reclaim, it could be because 4360 * pages are pinned on the per-cpu lists or in high alloc reserves. 4361 * Shrink them and try again 4362 */ 4363 if (!page && !drained) { 4364 unreserve_highatomic_pageblock(ac, false); 4365 drain_all_pages(NULL); 4366 drained = true; 4367 goto retry; 4368 } 4369 4370 return page; 4371 } 4372 4373 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, 4374 const struct alloc_context *ac) 4375 { 4376 struct zoneref *z; 4377 struct zone *zone; 4378 pg_data_t *last_pgdat = NULL; 4379 enum zone_type highest_zoneidx = ac->highest_zoneidx; 4380 4381 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx, 4382 ac->nodemask) { 4383 if (last_pgdat != zone->zone_pgdat) 4384 wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx); 4385 last_pgdat = zone->zone_pgdat; 4386 } 4387 } 4388 4389 static inline unsigned int 4390 gfp_to_alloc_flags(gfp_t gfp_mask) 4391 { 4392 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 4393 4394 /* 4395 * __GFP_HIGH is assumed to be the same as ALLOC_HIGH 4396 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 4397 * to save two branches. 4398 */ 4399 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 4400 BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); 4401 4402 /* 4403 * The caller may dip into page reserves a bit more if the caller 4404 * cannot run direct reclaim, or if the caller has realtime scheduling 4405 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 4406 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 4407 */ 4408 alloc_flags |= (__force int) 4409 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); 4410 4411 if (gfp_mask & __GFP_ATOMIC) { 4412 /* 4413 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 4414 * if it can't schedule. 4415 */ 4416 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4417 alloc_flags |= ALLOC_HARDER; 4418 /* 4419 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 4420 * comment for __cpuset_node_allowed(). 4421 */ 4422 alloc_flags &= ~ALLOC_CPUSET; 4423 } else if (unlikely(rt_task(current)) && !in_interrupt()) 4424 alloc_flags |= ALLOC_HARDER; 4425 4426 alloc_flags = current_alloc_flags(gfp_mask, alloc_flags); 4427 4428 return alloc_flags; 4429 } 4430 4431 static bool oom_reserves_allowed(struct task_struct *tsk) 4432 { 4433 if (!tsk_is_oom_victim(tsk)) 4434 return false; 4435 4436 /* 4437 * !MMU doesn't have oom reaper so give access to memory reserves 4438 * only to the thread with TIF_MEMDIE set 4439 */ 4440 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE)) 4441 return false; 4442 4443 return true; 4444 } 4445 4446 /* 4447 * Distinguish requests which really need access to full memory 4448 * reserves from oom victims which can live with a portion of it 4449 */ 4450 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) 4451 { 4452 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 4453 return 0; 4454 if (gfp_mask & __GFP_MEMALLOC) 4455 return ALLOC_NO_WATERMARKS; 4456 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 4457 return ALLOC_NO_WATERMARKS; 4458 if (!in_interrupt()) { 4459 if (current->flags & PF_MEMALLOC) 4460 return ALLOC_NO_WATERMARKS; 4461 else if (oom_reserves_allowed(current)) 4462 return ALLOC_OOM; 4463 } 4464 4465 return 0; 4466 } 4467 4468 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 4469 { 4470 return !!__gfp_pfmemalloc_flags(gfp_mask); 4471 } 4472 4473 /* 4474 * Checks whether it makes sense to retry the reclaim to make a forward progress 4475 * for the given allocation request. 4476 * 4477 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row 4478 * without success, or when we couldn't even meet the watermark if we 4479 * reclaimed all remaining pages on the LRU lists. 4480 * 4481 * Returns true if a retry is viable or false to enter the oom path. 4482 */ 4483 static inline bool 4484 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 4485 struct alloc_context *ac, int alloc_flags, 4486 bool did_some_progress, int *no_progress_loops) 4487 { 4488 struct zone *zone; 4489 struct zoneref *z; 4490 bool ret = false; 4491 4492 /* 4493 * Costly allocations might have made a progress but this doesn't mean 4494 * their order will become available due to high fragmentation so 4495 * always increment the no progress counter for them 4496 */ 4497 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 4498 *no_progress_loops = 0; 4499 else 4500 (*no_progress_loops)++; 4501 4502 /* 4503 * Make sure we converge to OOM if we cannot make any progress 4504 * several times in the row. 4505 */ 4506 if (*no_progress_loops > MAX_RECLAIM_RETRIES) { 4507 /* Before OOM, exhaust highatomic_reserve */ 4508 return unreserve_highatomic_pageblock(ac, true); 4509 } 4510 4511 /* 4512 * Keep reclaiming pages while there is a chance this will lead 4513 * somewhere. If none of the target zones can satisfy our allocation 4514 * request even if all reclaimable pages are considered then we are 4515 * screwed and have to go OOM. 4516 */ 4517 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4518 ac->highest_zoneidx, ac->nodemask) { 4519 unsigned long available; 4520 unsigned long reclaimable; 4521 unsigned long min_wmark = min_wmark_pages(zone); 4522 bool wmark; 4523 4524 available = reclaimable = zone_reclaimable_pages(zone); 4525 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 4526 4527 /* 4528 * Would the allocation succeed if we reclaimed all 4529 * reclaimable pages? 4530 */ 4531 wmark = __zone_watermark_ok(zone, order, min_wmark, 4532 ac->highest_zoneidx, alloc_flags, available); 4533 trace_reclaim_retry_zone(z, order, reclaimable, 4534 available, min_wmark, *no_progress_loops, wmark); 4535 if (wmark) { 4536 /* 4537 * If we didn't make any progress and have a lot of 4538 * dirty + writeback pages then we should wait for 4539 * an IO to complete to slow down the reclaim and 4540 * prevent from pre mature OOM 4541 */ 4542 if (!did_some_progress) { 4543 unsigned long write_pending; 4544 4545 write_pending = zone_page_state_snapshot(zone, 4546 NR_ZONE_WRITE_PENDING); 4547 4548 if (2 * write_pending > reclaimable) { 4549 congestion_wait(BLK_RW_ASYNC, HZ/10); 4550 return true; 4551 } 4552 } 4553 4554 ret = true; 4555 goto out; 4556 } 4557 } 4558 4559 out: 4560 /* 4561 * Memory allocation/reclaim might be called from a WQ context and the 4562 * current implementation of the WQ concurrency control doesn't 4563 * recognize that a particular WQ is congested if the worker thread is 4564 * looping without ever sleeping. Therefore we have to do a short sleep 4565 * here rather than calling cond_resched(). 4566 */ 4567 if (current->flags & PF_WQ_WORKER) 4568 schedule_timeout_uninterruptible(1); 4569 else 4570 cond_resched(); 4571 return ret; 4572 } 4573 4574 static inline bool 4575 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) 4576 { 4577 /* 4578 * It's possible that cpuset's mems_allowed and the nodemask from 4579 * mempolicy don't intersect. This should be normally dealt with by 4580 * policy_nodemask(), but it's possible to race with cpuset update in 4581 * such a way the check therein was true, and then it became false 4582 * before we got our cpuset_mems_cookie here. 4583 * This assumes that for all allocations, ac->nodemask can come only 4584 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored 4585 * when it does not intersect with the cpuset restrictions) or the 4586 * caller can deal with a violated nodemask. 4587 */ 4588 if (cpusets_enabled() && ac->nodemask && 4589 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) { 4590 ac->nodemask = NULL; 4591 return true; 4592 } 4593 4594 /* 4595 * When updating a task's mems_allowed or mempolicy nodemask, it is 4596 * possible to race with parallel threads in such a way that our 4597 * allocation can fail while the mask is being updated. If we are about 4598 * to fail, check if the cpuset changed during allocation and if so, 4599 * retry. 4600 */ 4601 if (read_mems_allowed_retry(cpuset_mems_cookie)) 4602 return true; 4603 4604 return false; 4605 } 4606 4607 static inline struct page * 4608 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 4609 struct alloc_context *ac) 4610 { 4611 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 4612 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 4613 struct page *page = NULL; 4614 unsigned int alloc_flags; 4615 unsigned long did_some_progress; 4616 enum compact_priority compact_priority; 4617 enum compact_result compact_result; 4618 int compaction_retries; 4619 int no_progress_loops; 4620 unsigned int cpuset_mems_cookie; 4621 int reserve_flags; 4622 4623 /* 4624 * We also sanity check to catch abuse of atomic reserves being used by 4625 * callers that are not in atomic context. 4626 */ 4627 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 4628 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 4629 gfp_mask &= ~__GFP_ATOMIC; 4630 4631 retry_cpuset: 4632 compaction_retries = 0; 4633 no_progress_loops = 0; 4634 compact_priority = DEF_COMPACT_PRIORITY; 4635 cpuset_mems_cookie = read_mems_allowed_begin(); 4636 4637 /* 4638 * The fast path uses conservative alloc_flags to succeed only until 4639 * kswapd needs to be woken up, and to avoid the cost of setting up 4640 * alloc_flags precisely. So we do that now. 4641 */ 4642 alloc_flags = gfp_to_alloc_flags(gfp_mask); 4643 4644 /* 4645 * We need to recalculate the starting point for the zonelist iterator 4646 * because we might have used different nodemask in the fast path, or 4647 * there was a cpuset modification and we are retrying - otherwise we 4648 * could end up iterating over non-eligible zones endlessly. 4649 */ 4650 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4651 ac->highest_zoneidx, ac->nodemask); 4652 if (!ac->preferred_zoneref->zone) 4653 goto nopage; 4654 4655 if (alloc_flags & ALLOC_KSWAPD) 4656 wake_all_kswapds(order, gfp_mask, ac); 4657 4658 /* 4659 * The adjusted alloc_flags might result in immediate success, so try 4660 * that first 4661 */ 4662 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4663 if (page) 4664 goto got_pg; 4665 4666 /* 4667 * For costly allocations, try direct compaction first, as it's likely 4668 * that we have enough base pages and don't need to reclaim. For non- 4669 * movable high-order allocations, do that as well, as compaction will 4670 * try prevent permanent fragmentation by migrating from blocks of the 4671 * same migratetype. 4672 * Don't try this for allocations that are allowed to ignore 4673 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 4674 */ 4675 if (can_direct_reclaim && 4676 (costly_order || 4677 (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 4678 && !gfp_pfmemalloc_allowed(gfp_mask)) { 4679 page = __alloc_pages_direct_compact(gfp_mask, order, 4680 alloc_flags, ac, 4681 INIT_COMPACT_PRIORITY, 4682 &compact_result); 4683 if (page) 4684 goto got_pg; 4685 4686 /* 4687 * Checks for costly allocations with __GFP_NORETRY, which 4688 * includes some THP page fault allocations 4689 */ 4690 if (costly_order && (gfp_mask & __GFP_NORETRY)) { 4691 /* 4692 * If allocating entire pageblock(s) and compaction 4693 * failed because all zones are below low watermarks 4694 * or is prohibited because it recently failed at this 4695 * order, fail immediately unless the allocator has 4696 * requested compaction and reclaim retry. 4697 * 4698 * Reclaim is 4699 * - potentially very expensive because zones are far 4700 * below their low watermarks or this is part of very 4701 * bursty high order allocations, 4702 * - not guaranteed to help because isolate_freepages() 4703 * may not iterate over freed pages as part of its 4704 * linear scan, and 4705 * - unlikely to make entire pageblocks free on its 4706 * own. 4707 */ 4708 if (compact_result == COMPACT_SKIPPED || 4709 compact_result == COMPACT_DEFERRED) 4710 goto nopage; 4711 4712 /* 4713 * Looks like reclaim/compaction is worth trying, but 4714 * sync compaction could be very expensive, so keep 4715 * using async compaction. 4716 */ 4717 compact_priority = INIT_COMPACT_PRIORITY; 4718 } 4719 } 4720 4721 retry: 4722 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 4723 if (alloc_flags & ALLOC_KSWAPD) 4724 wake_all_kswapds(order, gfp_mask, ac); 4725 4726 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); 4727 if (reserve_flags) 4728 alloc_flags = current_alloc_flags(gfp_mask, reserve_flags); 4729 4730 /* 4731 * Reset the nodemask and zonelist iterators if memory policies can be 4732 * ignored. These allocations are high priority and system rather than 4733 * user oriented. 4734 */ 4735 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { 4736 ac->nodemask = NULL; 4737 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4738 ac->highest_zoneidx, ac->nodemask); 4739 } 4740 4741 /* Attempt with potentially adjusted zonelist and alloc_flags */ 4742 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4743 if (page) 4744 goto got_pg; 4745 4746 /* Caller is not willing to reclaim, we can't balance anything */ 4747 if (!can_direct_reclaim) 4748 goto nopage; 4749 4750 /* Avoid recursion of direct reclaim */ 4751 if (current->flags & PF_MEMALLOC) 4752 goto nopage; 4753 4754 /* Try direct reclaim and then allocating */ 4755 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 4756 &did_some_progress); 4757 if (page) 4758 goto got_pg; 4759 4760 /* Try direct compaction and then allocating */ 4761 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 4762 compact_priority, &compact_result); 4763 if (page) 4764 goto got_pg; 4765 4766 /* Do not loop if specifically requested */ 4767 if (gfp_mask & __GFP_NORETRY) 4768 goto nopage; 4769 4770 /* 4771 * Do not retry costly high order allocations unless they are 4772 * __GFP_RETRY_MAYFAIL 4773 */ 4774 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) 4775 goto nopage; 4776 4777 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 4778 did_some_progress > 0, &no_progress_loops)) 4779 goto retry; 4780 4781 /* 4782 * It doesn't make any sense to retry for the compaction if the order-0 4783 * reclaim is not able to make any progress because the current 4784 * implementation of the compaction depends on the sufficient amount 4785 * of free memory (see __compaction_suitable) 4786 */ 4787 if (did_some_progress > 0 && 4788 should_compact_retry(ac, order, alloc_flags, 4789 compact_result, &compact_priority, 4790 &compaction_retries)) 4791 goto retry; 4792 4793 4794 /* Deal with possible cpuset update races before we start OOM killing */ 4795 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4796 goto retry_cpuset; 4797 4798 /* Reclaim has failed us, start killing things */ 4799 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 4800 if (page) 4801 goto got_pg; 4802 4803 /* Avoid allocations with no watermarks from looping endlessly */ 4804 if (tsk_is_oom_victim(current) && 4805 (alloc_flags & ALLOC_OOM || 4806 (gfp_mask & __GFP_NOMEMALLOC))) 4807 goto nopage; 4808 4809 /* Retry as long as the OOM killer is making progress */ 4810 if (did_some_progress) { 4811 no_progress_loops = 0; 4812 goto retry; 4813 } 4814 4815 nopage: 4816 /* Deal with possible cpuset update races before we fail */ 4817 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4818 goto retry_cpuset; 4819 4820 /* 4821 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure 4822 * we always retry 4823 */ 4824 if (gfp_mask & __GFP_NOFAIL) { 4825 /* 4826 * All existing users of the __GFP_NOFAIL are blockable, so warn 4827 * of any new users that actually require GFP_NOWAIT 4828 */ 4829 if (WARN_ON_ONCE(!can_direct_reclaim)) 4830 goto fail; 4831 4832 /* 4833 * PF_MEMALLOC request from this context is rather bizarre 4834 * because we cannot reclaim anything and only can loop waiting 4835 * for somebody to do a work for us 4836 */ 4837 WARN_ON_ONCE(current->flags & PF_MEMALLOC); 4838 4839 /* 4840 * non failing costly orders are a hard requirement which we 4841 * are not prepared for much so let's warn about these users 4842 * so that we can identify them and convert them to something 4843 * else. 4844 */ 4845 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER); 4846 4847 /* 4848 * Help non-failing allocations by giving them access to memory 4849 * reserves but do not use ALLOC_NO_WATERMARKS because this 4850 * could deplete whole memory reserves which would just make 4851 * the situation worse 4852 */ 4853 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); 4854 if (page) 4855 goto got_pg; 4856 4857 cond_resched(); 4858 goto retry; 4859 } 4860 fail: 4861 warn_alloc(gfp_mask, ac->nodemask, 4862 "page allocation failure: order:%u", order); 4863 got_pg: 4864 return page; 4865 } 4866 4867 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, 4868 int preferred_nid, nodemask_t *nodemask, 4869 struct alloc_context *ac, gfp_t *alloc_mask, 4870 unsigned int *alloc_flags) 4871 { 4872 ac->highest_zoneidx = gfp_zone(gfp_mask); 4873 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); 4874 ac->nodemask = nodemask; 4875 ac->migratetype = gfp_migratetype(gfp_mask); 4876 4877 if (cpusets_enabled()) { 4878 *alloc_mask |= __GFP_HARDWALL; 4879 /* 4880 * When we are in the interrupt context, it is irrelevant 4881 * to the current task context. It means that any node ok. 4882 */ 4883 if (!in_interrupt() && !ac->nodemask) 4884 ac->nodemask = &cpuset_current_mems_allowed; 4885 else 4886 *alloc_flags |= ALLOC_CPUSET; 4887 } 4888 4889 fs_reclaim_acquire(gfp_mask); 4890 fs_reclaim_release(gfp_mask); 4891 4892 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 4893 4894 if (should_fail_alloc_page(gfp_mask, order)) 4895 return false; 4896 4897 *alloc_flags = current_alloc_flags(gfp_mask, *alloc_flags); 4898 4899 /* Dirty zone balancing only done in the fast path */ 4900 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); 4901 4902 /* 4903 * The preferred zone is used for statistics but crucially it is 4904 * also used as the starting point for the zonelist iterator. It 4905 * may get reset for allocations that ignore memory policies. 4906 */ 4907 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4908 ac->highest_zoneidx, ac->nodemask); 4909 4910 return true; 4911 } 4912 4913 /* 4914 * This is the 'heart' of the zoned buddy allocator. 4915 */ 4916 struct page * 4917 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, 4918 nodemask_t *nodemask) 4919 { 4920 struct page *page; 4921 unsigned int alloc_flags = ALLOC_WMARK_LOW; 4922 gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */ 4923 struct alloc_context ac = { }; 4924 4925 /* 4926 * There are several places where we assume that the order value is sane 4927 * so bail out early if the request is out of bound. 4928 */ 4929 if (unlikely(order >= MAX_ORDER)) { 4930 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); 4931 return NULL; 4932 } 4933 4934 gfp_mask &= gfp_allowed_mask; 4935 alloc_mask = gfp_mask; 4936 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags)) 4937 return NULL; 4938 4939 /* 4940 * Forbid the first pass from falling back to types that fragment 4941 * memory until all local zones are considered. 4942 */ 4943 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask); 4944 4945 /* First allocation attempt */ 4946 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); 4947 if (likely(page)) 4948 goto out; 4949 4950 /* 4951 * Apply scoped allocation constraints. This is mainly about GFP_NOFS 4952 * resp. GFP_NOIO which has to be inherited for all allocation requests 4953 * from a particular context which has been marked by 4954 * memalloc_no{fs,io}_{save,restore}. 4955 */ 4956 alloc_mask = current_gfp_context(gfp_mask); 4957 ac.spread_dirty_pages = false; 4958 4959 /* 4960 * Restore the original nodemask if it was potentially replaced with 4961 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 4962 */ 4963 ac.nodemask = nodemask; 4964 4965 page = __alloc_pages_slowpath(alloc_mask, order, &ac); 4966 4967 out: 4968 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && 4969 unlikely(__memcg_kmem_charge_page(page, gfp_mask, order) != 0)) { 4970 __free_pages(page, order); 4971 page = NULL; 4972 } 4973 4974 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); 4975 4976 return page; 4977 } 4978 EXPORT_SYMBOL(__alloc_pages_nodemask); 4979 4980 /* 4981 * Common helper functions. Never use with __GFP_HIGHMEM because the returned 4982 * address cannot represent highmem pages. Use alloc_pages and then kmap if 4983 * you need to access high mem. 4984 */ 4985 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 4986 { 4987 struct page *page; 4988 4989 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); 4990 if (!page) 4991 return 0; 4992 return (unsigned long) page_address(page); 4993 } 4994 EXPORT_SYMBOL(__get_free_pages); 4995 4996 unsigned long get_zeroed_page(gfp_t gfp_mask) 4997 { 4998 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 4999 } 5000 EXPORT_SYMBOL(get_zeroed_page); 5001 5002 static inline void free_the_page(struct page *page, unsigned int order) 5003 { 5004 if (order == 0) /* Via pcp? */ 5005 free_unref_page(page); 5006 else 5007 __free_pages_ok(page, order, FPI_NONE); 5008 } 5009 5010 void __free_pages(struct page *page, unsigned int order) 5011 { 5012 if (put_page_testzero(page)) 5013 free_the_page(page, order); 5014 else if (!PageHead(page)) 5015 while (order-- > 0) 5016 free_the_page(page + (1 << order), order); 5017 } 5018 EXPORT_SYMBOL(__free_pages); 5019 5020 void free_pages(unsigned long addr, unsigned int order) 5021 { 5022 if (addr != 0) { 5023 VM_BUG_ON(!virt_addr_valid((void *)addr)); 5024 __free_pages(virt_to_page((void *)addr), order); 5025 } 5026 } 5027 5028 EXPORT_SYMBOL(free_pages); 5029 5030 /* 5031 * Page Fragment: 5032 * An arbitrary-length arbitrary-offset area of memory which resides 5033 * within a 0 or higher order page. Multiple fragments within that page 5034 * are individually refcounted, in the page's reference counter. 5035 * 5036 * The page_frag functions below provide a simple allocation framework for 5037 * page fragments. This is used by the network stack and network device 5038 * drivers to provide a backing region of memory for use as either an 5039 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 5040 */ 5041 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 5042 gfp_t gfp_mask) 5043 { 5044 struct page *page = NULL; 5045 gfp_t gfp = gfp_mask; 5046 5047 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5048 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 5049 __GFP_NOMEMALLOC; 5050 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 5051 PAGE_FRAG_CACHE_MAX_ORDER); 5052 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 5053 #endif 5054 if (unlikely(!page)) 5055 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 5056 5057 nc->va = page ? page_address(page) : NULL; 5058 5059 return page; 5060 } 5061 5062 void __page_frag_cache_drain(struct page *page, unsigned int count) 5063 { 5064 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 5065 5066 if (page_ref_sub_and_test(page, count)) 5067 free_the_page(page, compound_order(page)); 5068 } 5069 EXPORT_SYMBOL(__page_frag_cache_drain); 5070 5071 void *page_frag_alloc(struct page_frag_cache *nc, 5072 unsigned int fragsz, gfp_t gfp_mask) 5073 { 5074 unsigned int size = PAGE_SIZE; 5075 struct page *page; 5076 int offset; 5077 5078 if (unlikely(!nc->va)) { 5079 refill: 5080 page = __page_frag_cache_refill(nc, gfp_mask); 5081 if (!page) 5082 return NULL; 5083 5084 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5085 /* if size can vary use size else just use PAGE_SIZE */ 5086 size = nc->size; 5087 #endif 5088 /* Even if we own the page, we do not use atomic_set(). 5089 * This would break get_page_unless_zero() users. 5090 */ 5091 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 5092 5093 /* reset page count bias and offset to start of new frag */ 5094 nc->pfmemalloc = page_is_pfmemalloc(page); 5095 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5096 nc->offset = size; 5097 } 5098 5099 offset = nc->offset - fragsz; 5100 if (unlikely(offset < 0)) { 5101 page = virt_to_page(nc->va); 5102 5103 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 5104 goto refill; 5105 5106 if (unlikely(nc->pfmemalloc)) { 5107 free_the_page(page, compound_order(page)); 5108 goto refill; 5109 } 5110 5111 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5112 /* if size can vary use size else just use PAGE_SIZE */ 5113 size = nc->size; 5114 #endif 5115 /* OK, page count is 0, we can safely set it */ 5116 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 5117 5118 /* reset page count bias and offset to start of new frag */ 5119 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5120 offset = size - fragsz; 5121 } 5122 5123 nc->pagecnt_bias--; 5124 nc->offset = offset; 5125 5126 return nc->va + offset; 5127 } 5128 EXPORT_SYMBOL(page_frag_alloc); 5129 5130 /* 5131 * Frees a page fragment allocated out of either a compound or order 0 page. 5132 */ 5133 void page_frag_free(void *addr) 5134 { 5135 struct page *page = virt_to_head_page(addr); 5136 5137 if (unlikely(put_page_testzero(page))) 5138 free_the_page(page, compound_order(page)); 5139 } 5140 EXPORT_SYMBOL(page_frag_free); 5141 5142 static void *make_alloc_exact(unsigned long addr, unsigned int order, 5143 size_t size) 5144 { 5145 if (addr) { 5146 unsigned long alloc_end = addr + (PAGE_SIZE << order); 5147 unsigned long used = addr + PAGE_ALIGN(size); 5148 5149 split_page(virt_to_page((void *)addr), order); 5150 while (used < alloc_end) { 5151 free_page(used); 5152 used += PAGE_SIZE; 5153 } 5154 } 5155 return (void *)addr; 5156 } 5157 5158 /** 5159 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 5160 * @size: the number of bytes to allocate 5161 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5162 * 5163 * This function is similar to alloc_pages(), except that it allocates the 5164 * minimum number of pages to satisfy the request. alloc_pages() can only 5165 * allocate memory in power-of-two pages. 5166 * 5167 * This function is also limited by MAX_ORDER. 5168 * 5169 * Memory allocated by this function must be released by free_pages_exact(). 5170 * 5171 * Return: pointer to the allocated area or %NULL in case of error. 5172 */ 5173 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 5174 { 5175 unsigned int order = get_order(size); 5176 unsigned long addr; 5177 5178 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 5179 gfp_mask &= ~__GFP_COMP; 5180 5181 addr = __get_free_pages(gfp_mask, order); 5182 return make_alloc_exact(addr, order, size); 5183 } 5184 EXPORT_SYMBOL(alloc_pages_exact); 5185 5186 /** 5187 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 5188 * pages on a node. 5189 * @nid: the preferred node ID where memory should be allocated 5190 * @size: the number of bytes to allocate 5191 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5192 * 5193 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 5194 * back. 5195 * 5196 * Return: pointer to the allocated area or %NULL in case of error. 5197 */ 5198 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 5199 { 5200 unsigned int order = get_order(size); 5201 struct page *p; 5202 5203 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 5204 gfp_mask &= ~__GFP_COMP; 5205 5206 p = alloc_pages_node(nid, gfp_mask, order); 5207 if (!p) 5208 return NULL; 5209 return make_alloc_exact((unsigned long)page_address(p), order, size); 5210 } 5211 5212 /** 5213 * free_pages_exact - release memory allocated via alloc_pages_exact() 5214 * @virt: the value returned by alloc_pages_exact. 5215 * @size: size of allocation, same value as passed to alloc_pages_exact(). 5216 * 5217 * Release the memory allocated by a previous call to alloc_pages_exact. 5218 */ 5219 void free_pages_exact(void *virt, size_t size) 5220 { 5221 unsigned long addr = (unsigned long)virt; 5222 unsigned long end = addr + PAGE_ALIGN(size); 5223 5224 while (addr < end) { 5225 free_page(addr); 5226 addr += PAGE_SIZE; 5227 } 5228 } 5229 EXPORT_SYMBOL(free_pages_exact); 5230 5231 /** 5232 * nr_free_zone_pages - count number of pages beyond high watermark 5233 * @offset: The zone index of the highest zone 5234 * 5235 * nr_free_zone_pages() counts the number of pages which are beyond the 5236 * high watermark within all zones at or below a given zone index. For each 5237 * zone, the number of pages is calculated as: 5238 * 5239 * nr_free_zone_pages = managed_pages - high_pages 5240 * 5241 * Return: number of pages beyond high watermark. 5242 */ 5243 static unsigned long nr_free_zone_pages(int offset) 5244 { 5245 struct zoneref *z; 5246 struct zone *zone; 5247 5248 /* Just pick one node, since fallback list is circular */ 5249 unsigned long sum = 0; 5250 5251 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 5252 5253 for_each_zone_zonelist(zone, z, zonelist, offset) { 5254 unsigned long size = zone_managed_pages(zone); 5255 unsigned long high = high_wmark_pages(zone); 5256 if (size > high) 5257 sum += size - high; 5258 } 5259 5260 return sum; 5261 } 5262 5263 /** 5264 * nr_free_buffer_pages - count number of pages beyond high watermark 5265 * 5266 * nr_free_buffer_pages() counts the number of pages which are beyond the high 5267 * watermark within ZONE_DMA and ZONE_NORMAL. 5268 * 5269 * Return: number of pages beyond high watermark within ZONE_DMA and 5270 * ZONE_NORMAL. 5271 */ 5272 unsigned long nr_free_buffer_pages(void) 5273 { 5274 return nr_free_zone_pages(gfp_zone(GFP_USER)); 5275 } 5276 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 5277 5278 static inline void show_node(struct zone *zone) 5279 { 5280 if (IS_ENABLED(CONFIG_NUMA)) 5281 printk("Node %d ", zone_to_nid(zone)); 5282 } 5283 5284 long si_mem_available(void) 5285 { 5286 long available; 5287 unsigned long pagecache; 5288 unsigned long wmark_low = 0; 5289 unsigned long pages[NR_LRU_LISTS]; 5290 unsigned long reclaimable; 5291 struct zone *zone; 5292 int lru; 5293 5294 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 5295 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 5296 5297 for_each_zone(zone) 5298 wmark_low += low_wmark_pages(zone); 5299 5300 /* 5301 * Estimate the amount of memory available for userspace allocations, 5302 * without causing swapping. 5303 */ 5304 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages; 5305 5306 /* 5307 * Not all the page cache can be freed, otherwise the system will 5308 * start swapping. Assume at least half of the page cache, or the 5309 * low watermark worth of cache, needs to stay. 5310 */ 5311 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 5312 pagecache -= min(pagecache / 2, wmark_low); 5313 available += pagecache; 5314 5315 /* 5316 * Part of the reclaimable slab and other kernel memory consists of 5317 * items that are in use, and cannot be freed. Cap this estimate at the 5318 * low watermark. 5319 */ 5320 reclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B) + 5321 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE); 5322 available += reclaimable - min(reclaimable / 2, wmark_low); 5323 5324 if (available < 0) 5325 available = 0; 5326 return available; 5327 } 5328 EXPORT_SYMBOL_GPL(si_mem_available); 5329 5330 void si_meminfo(struct sysinfo *val) 5331 { 5332 val->totalram = totalram_pages(); 5333 val->sharedram = global_node_page_state(NR_SHMEM); 5334 val->freeram = global_zone_page_state(NR_FREE_PAGES); 5335 val->bufferram = nr_blockdev_pages(); 5336 val->totalhigh = totalhigh_pages(); 5337 val->freehigh = nr_free_highpages(); 5338 val->mem_unit = PAGE_SIZE; 5339 } 5340 5341 EXPORT_SYMBOL(si_meminfo); 5342 5343 #ifdef CONFIG_NUMA 5344 void si_meminfo_node(struct sysinfo *val, int nid) 5345 { 5346 int zone_type; /* needs to be signed */ 5347 unsigned long managed_pages = 0; 5348 unsigned long managed_highpages = 0; 5349 unsigned long free_highpages = 0; 5350 pg_data_t *pgdat = NODE_DATA(nid); 5351 5352 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 5353 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]); 5354 val->totalram = managed_pages; 5355 val->sharedram = node_page_state(pgdat, NR_SHMEM); 5356 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 5357 #ifdef CONFIG_HIGHMEM 5358 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 5359 struct zone *zone = &pgdat->node_zones[zone_type]; 5360 5361 if (is_highmem(zone)) { 5362 managed_highpages += zone_managed_pages(zone); 5363 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 5364 } 5365 } 5366 val->totalhigh = managed_highpages; 5367 val->freehigh = free_highpages; 5368 #else 5369 val->totalhigh = managed_highpages; 5370 val->freehigh = free_highpages; 5371 #endif 5372 val->mem_unit = PAGE_SIZE; 5373 } 5374 #endif 5375 5376 /* 5377 * Determine whether the node should be displayed or not, depending on whether 5378 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 5379 */ 5380 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) 5381 { 5382 if (!(flags & SHOW_MEM_FILTER_NODES)) 5383 return false; 5384 5385 /* 5386 * no node mask - aka implicit memory numa policy. Do not bother with 5387 * the synchronization - read_mems_allowed_begin - because we do not 5388 * have to be precise here. 5389 */ 5390 if (!nodemask) 5391 nodemask = &cpuset_current_mems_allowed; 5392 5393 return !node_isset(nid, *nodemask); 5394 } 5395 5396 #define K(x) ((x) << (PAGE_SHIFT-10)) 5397 5398 static void show_migration_types(unsigned char type) 5399 { 5400 static const char types[MIGRATE_TYPES] = { 5401 [MIGRATE_UNMOVABLE] = 'U', 5402 [MIGRATE_MOVABLE] = 'M', 5403 [MIGRATE_RECLAIMABLE] = 'E', 5404 [MIGRATE_HIGHATOMIC] = 'H', 5405 #ifdef CONFIG_CMA 5406 [MIGRATE_CMA] = 'C', 5407 #endif 5408 #ifdef CONFIG_MEMORY_ISOLATION 5409 [MIGRATE_ISOLATE] = 'I', 5410 #endif 5411 }; 5412 char tmp[MIGRATE_TYPES + 1]; 5413 char *p = tmp; 5414 int i; 5415 5416 for (i = 0; i < MIGRATE_TYPES; i++) { 5417 if (type & (1 << i)) 5418 *p++ = types[i]; 5419 } 5420 5421 *p = '\0'; 5422 printk(KERN_CONT "(%s) ", tmp); 5423 } 5424 5425 /* 5426 * Show free area list (used inside shift_scroll-lock stuff) 5427 * We also calculate the percentage fragmentation. We do this by counting the 5428 * memory on each free list with the exception of the first item on the list. 5429 * 5430 * Bits in @filter: 5431 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 5432 * cpuset. 5433 */ 5434 void show_free_areas(unsigned int filter, nodemask_t *nodemask) 5435 { 5436 unsigned long free_pcp = 0; 5437 int cpu; 5438 struct zone *zone; 5439 pg_data_t *pgdat; 5440 5441 for_each_populated_zone(zone) { 5442 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5443 continue; 5444 5445 for_each_online_cpu(cpu) 5446 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 5447 } 5448 5449 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 5450 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 5451 " unevictable:%lu dirty:%lu writeback:%lu\n" 5452 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 5453 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 5454 " free:%lu free_pcp:%lu free_cma:%lu\n", 5455 global_node_page_state(NR_ACTIVE_ANON), 5456 global_node_page_state(NR_INACTIVE_ANON), 5457 global_node_page_state(NR_ISOLATED_ANON), 5458 global_node_page_state(NR_ACTIVE_FILE), 5459 global_node_page_state(NR_INACTIVE_FILE), 5460 global_node_page_state(NR_ISOLATED_FILE), 5461 global_node_page_state(NR_UNEVICTABLE), 5462 global_node_page_state(NR_FILE_DIRTY), 5463 global_node_page_state(NR_WRITEBACK), 5464 global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B), 5465 global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B), 5466 global_node_page_state(NR_FILE_MAPPED), 5467 global_node_page_state(NR_SHMEM), 5468 global_zone_page_state(NR_PAGETABLE), 5469 global_zone_page_state(NR_BOUNCE), 5470 global_zone_page_state(NR_FREE_PAGES), 5471 free_pcp, 5472 global_zone_page_state(NR_FREE_CMA_PAGES)); 5473 5474 for_each_online_pgdat(pgdat) { 5475 if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) 5476 continue; 5477 5478 printk("Node %d" 5479 " active_anon:%lukB" 5480 " inactive_anon:%lukB" 5481 " active_file:%lukB" 5482 " inactive_file:%lukB" 5483 " unevictable:%lukB" 5484 " isolated(anon):%lukB" 5485 " isolated(file):%lukB" 5486 " mapped:%lukB" 5487 " dirty:%lukB" 5488 " writeback:%lukB" 5489 " shmem:%lukB" 5490 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5491 " shmem_thp: %lukB" 5492 " shmem_pmdmapped: %lukB" 5493 " anon_thp: %lukB" 5494 #endif 5495 " writeback_tmp:%lukB" 5496 " kernel_stack:%lukB" 5497 #ifdef CONFIG_SHADOW_CALL_STACK 5498 " shadow_call_stack:%lukB" 5499 #endif 5500 " all_unreclaimable? %s" 5501 "\n", 5502 pgdat->node_id, 5503 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 5504 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 5505 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 5506 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 5507 K(node_page_state(pgdat, NR_UNEVICTABLE)), 5508 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 5509 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 5510 K(node_page_state(pgdat, NR_FILE_MAPPED)), 5511 K(node_page_state(pgdat, NR_FILE_DIRTY)), 5512 K(node_page_state(pgdat, NR_WRITEBACK)), 5513 K(node_page_state(pgdat, NR_SHMEM)), 5514 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5515 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), 5516 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) 5517 * HPAGE_PMD_NR), 5518 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR), 5519 #endif 5520 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 5521 node_page_state(pgdat, NR_KERNEL_STACK_KB), 5522 #ifdef CONFIG_SHADOW_CALL_STACK 5523 node_page_state(pgdat, NR_KERNEL_SCS_KB), 5524 #endif 5525 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ? 5526 "yes" : "no"); 5527 } 5528 5529 for_each_populated_zone(zone) { 5530 int i; 5531 5532 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5533 continue; 5534 5535 free_pcp = 0; 5536 for_each_online_cpu(cpu) 5537 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 5538 5539 show_node(zone); 5540 printk(KERN_CONT 5541 "%s" 5542 " free:%lukB" 5543 " min:%lukB" 5544 " low:%lukB" 5545 " high:%lukB" 5546 " reserved_highatomic:%luKB" 5547 " active_anon:%lukB" 5548 " inactive_anon:%lukB" 5549 " active_file:%lukB" 5550 " inactive_file:%lukB" 5551 " unevictable:%lukB" 5552 " writepending:%lukB" 5553 " present:%lukB" 5554 " managed:%lukB" 5555 " mlocked:%lukB" 5556 " pagetables:%lukB" 5557 " bounce:%lukB" 5558 " free_pcp:%lukB" 5559 " local_pcp:%ukB" 5560 " free_cma:%lukB" 5561 "\n", 5562 zone->name, 5563 K(zone_page_state(zone, NR_FREE_PAGES)), 5564 K(min_wmark_pages(zone)), 5565 K(low_wmark_pages(zone)), 5566 K(high_wmark_pages(zone)), 5567 K(zone->nr_reserved_highatomic), 5568 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 5569 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 5570 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 5571 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 5572 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 5573 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 5574 K(zone->present_pages), 5575 K(zone_managed_pages(zone)), 5576 K(zone_page_state(zone, NR_MLOCK)), 5577 K(zone_page_state(zone, NR_PAGETABLE)), 5578 K(zone_page_state(zone, NR_BOUNCE)), 5579 K(free_pcp), 5580 K(this_cpu_read(zone->pageset->pcp.count)), 5581 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 5582 printk("lowmem_reserve[]:"); 5583 for (i = 0; i < MAX_NR_ZONES; i++) 5584 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); 5585 printk(KERN_CONT "\n"); 5586 } 5587 5588 for_each_populated_zone(zone) { 5589 unsigned int order; 5590 unsigned long nr[MAX_ORDER], flags, total = 0; 5591 unsigned char types[MAX_ORDER]; 5592 5593 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5594 continue; 5595 show_node(zone); 5596 printk(KERN_CONT "%s: ", zone->name); 5597 5598 spin_lock_irqsave(&zone->lock, flags); 5599 for (order = 0; order < MAX_ORDER; order++) { 5600 struct free_area *area = &zone->free_area[order]; 5601 int type; 5602 5603 nr[order] = area->nr_free; 5604 total += nr[order] << order; 5605 5606 types[order] = 0; 5607 for (type = 0; type < MIGRATE_TYPES; type++) { 5608 if (!free_area_empty(area, type)) 5609 types[order] |= 1 << type; 5610 } 5611 } 5612 spin_unlock_irqrestore(&zone->lock, flags); 5613 for (order = 0; order < MAX_ORDER; order++) { 5614 printk(KERN_CONT "%lu*%lukB ", 5615 nr[order], K(1UL) << order); 5616 if (nr[order]) 5617 show_migration_types(types[order]); 5618 } 5619 printk(KERN_CONT "= %lukB\n", K(total)); 5620 } 5621 5622 hugetlb_show_meminfo(); 5623 5624 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 5625 5626 show_swap_cache_info(); 5627 } 5628 5629 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 5630 { 5631 zoneref->zone = zone; 5632 zoneref->zone_idx = zone_idx(zone); 5633 } 5634 5635 /* 5636 * Builds allocation fallback zone lists. 5637 * 5638 * Add all populated zones of a node to the zonelist. 5639 */ 5640 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) 5641 { 5642 struct zone *zone; 5643 enum zone_type zone_type = MAX_NR_ZONES; 5644 int nr_zones = 0; 5645 5646 do { 5647 zone_type--; 5648 zone = pgdat->node_zones + zone_type; 5649 if (managed_zone(zone)) { 5650 zoneref_set_zone(zone, &zonerefs[nr_zones++]); 5651 check_highest_zone(zone_type); 5652 } 5653 } while (zone_type); 5654 5655 return nr_zones; 5656 } 5657 5658 #ifdef CONFIG_NUMA 5659 5660 static int __parse_numa_zonelist_order(char *s) 5661 { 5662 /* 5663 * We used to support different zonlists modes but they turned 5664 * out to be just not useful. Let's keep the warning in place 5665 * if somebody still use the cmd line parameter so that we do 5666 * not fail it silently 5667 */ 5668 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) { 5669 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s); 5670 return -EINVAL; 5671 } 5672 return 0; 5673 } 5674 5675 char numa_zonelist_order[] = "Node"; 5676 5677 /* 5678 * sysctl handler for numa_zonelist_order 5679 */ 5680 int numa_zonelist_order_handler(struct ctl_table *table, int write, 5681 void *buffer, size_t *length, loff_t *ppos) 5682 { 5683 if (write) 5684 return __parse_numa_zonelist_order(buffer); 5685 return proc_dostring(table, write, buffer, length, ppos); 5686 } 5687 5688 5689 #define MAX_NODE_LOAD (nr_online_nodes) 5690 static int node_load[MAX_NUMNODES]; 5691 5692 /** 5693 * find_next_best_node - find the next node that should appear in a given node's fallback list 5694 * @node: node whose fallback list we're appending 5695 * @used_node_mask: nodemask_t of already used nodes 5696 * 5697 * We use a number of factors to determine which is the next node that should 5698 * appear on a given node's fallback list. The node should not have appeared 5699 * already in @node's fallback list, and it should be the next closest node 5700 * according to the distance array (which contains arbitrary distance values 5701 * from each node to each node in the system), and should also prefer nodes 5702 * with no CPUs, since presumably they'll have very little allocation pressure 5703 * on them otherwise. 5704 * 5705 * Return: node id of the found node or %NUMA_NO_NODE if no node is found. 5706 */ 5707 static int find_next_best_node(int node, nodemask_t *used_node_mask) 5708 { 5709 int n, val; 5710 int min_val = INT_MAX; 5711 int best_node = NUMA_NO_NODE; 5712 5713 /* Use the local node if we haven't already */ 5714 if (!node_isset(node, *used_node_mask)) { 5715 node_set(node, *used_node_mask); 5716 return node; 5717 } 5718 5719 for_each_node_state(n, N_MEMORY) { 5720 5721 /* Don't want a node to appear more than once */ 5722 if (node_isset(n, *used_node_mask)) 5723 continue; 5724 5725 /* Use the distance array to find the distance */ 5726 val = node_distance(node, n); 5727 5728 /* Penalize nodes under us ("prefer the next node") */ 5729 val += (n < node); 5730 5731 /* Give preference to headless and unused nodes */ 5732 if (!cpumask_empty(cpumask_of_node(n))) 5733 val += PENALTY_FOR_NODE_WITH_CPUS; 5734 5735 /* Slight preference for less loaded node */ 5736 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 5737 val += node_load[n]; 5738 5739 if (val < min_val) { 5740 min_val = val; 5741 best_node = n; 5742 } 5743 } 5744 5745 if (best_node >= 0) 5746 node_set(best_node, *used_node_mask); 5747 5748 return best_node; 5749 } 5750 5751 5752 /* 5753 * Build zonelists ordered by node and zones within node. 5754 * This results in maximum locality--normal zone overflows into local 5755 * DMA zone, if any--but risks exhausting DMA zone. 5756 */ 5757 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, 5758 unsigned nr_nodes) 5759 { 5760 struct zoneref *zonerefs; 5761 int i; 5762 5763 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5764 5765 for (i = 0; i < nr_nodes; i++) { 5766 int nr_zones; 5767 5768 pg_data_t *node = NODE_DATA(node_order[i]); 5769 5770 nr_zones = build_zonerefs_node(node, zonerefs); 5771 zonerefs += nr_zones; 5772 } 5773 zonerefs->zone = NULL; 5774 zonerefs->zone_idx = 0; 5775 } 5776 5777 /* 5778 * Build gfp_thisnode zonelists 5779 */ 5780 static void build_thisnode_zonelists(pg_data_t *pgdat) 5781 { 5782 struct zoneref *zonerefs; 5783 int nr_zones; 5784 5785 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; 5786 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5787 zonerefs += nr_zones; 5788 zonerefs->zone = NULL; 5789 zonerefs->zone_idx = 0; 5790 } 5791 5792 /* 5793 * Build zonelists ordered by zone and nodes within zones. 5794 * This results in conserving DMA zone[s] until all Normal memory is 5795 * exhausted, but results in overflowing to remote node while memory 5796 * may still exist in local DMA zone. 5797 */ 5798 5799 static void build_zonelists(pg_data_t *pgdat) 5800 { 5801 static int node_order[MAX_NUMNODES]; 5802 int node, load, nr_nodes = 0; 5803 nodemask_t used_mask = NODE_MASK_NONE; 5804 int local_node, prev_node; 5805 5806 /* NUMA-aware ordering of nodes */ 5807 local_node = pgdat->node_id; 5808 load = nr_online_nodes; 5809 prev_node = local_node; 5810 5811 memset(node_order, 0, sizeof(node_order)); 5812 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 5813 /* 5814 * We don't want to pressure a particular node. 5815 * So adding penalty to the first node in same 5816 * distance group to make it round-robin. 5817 */ 5818 if (node_distance(local_node, node) != 5819 node_distance(local_node, prev_node)) 5820 node_load[node] = load; 5821 5822 node_order[nr_nodes++] = node; 5823 prev_node = node; 5824 load--; 5825 } 5826 5827 build_zonelists_in_node_order(pgdat, node_order, nr_nodes); 5828 build_thisnode_zonelists(pgdat); 5829 } 5830 5831 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5832 /* 5833 * Return node id of node used for "local" allocations. 5834 * I.e., first node id of first zone in arg node's generic zonelist. 5835 * Used for initializing percpu 'numa_mem', which is used primarily 5836 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 5837 */ 5838 int local_memory_node(int node) 5839 { 5840 struct zoneref *z; 5841 5842 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 5843 gfp_zone(GFP_KERNEL), 5844 NULL); 5845 return zone_to_nid(z->zone); 5846 } 5847 #endif 5848 5849 static void setup_min_unmapped_ratio(void); 5850 static void setup_min_slab_ratio(void); 5851 #else /* CONFIG_NUMA */ 5852 5853 static void build_zonelists(pg_data_t *pgdat) 5854 { 5855 int node, local_node; 5856 struct zoneref *zonerefs; 5857 int nr_zones; 5858 5859 local_node = pgdat->node_id; 5860 5861 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5862 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5863 zonerefs += nr_zones; 5864 5865 /* 5866 * Now we build the zonelist so that it contains the zones 5867 * of all the other nodes. 5868 * We don't want to pressure a particular node, so when 5869 * building the zones for node N, we make sure that the 5870 * zones coming right after the local ones are those from 5871 * node N+1 (modulo N) 5872 */ 5873 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 5874 if (!node_online(node)) 5875 continue; 5876 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5877 zonerefs += nr_zones; 5878 } 5879 for (node = 0; node < local_node; node++) { 5880 if (!node_online(node)) 5881 continue; 5882 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5883 zonerefs += nr_zones; 5884 } 5885 5886 zonerefs->zone = NULL; 5887 zonerefs->zone_idx = 0; 5888 } 5889 5890 #endif /* CONFIG_NUMA */ 5891 5892 /* 5893 * Boot pageset table. One per cpu which is going to be used for all 5894 * zones and all nodes. The parameters will be set in such a way 5895 * that an item put on a list will immediately be handed over to 5896 * the buddy list. This is safe since pageset manipulation is done 5897 * with interrupts disabled. 5898 * 5899 * The boot_pagesets must be kept even after bootup is complete for 5900 * unused processors and/or zones. They do play a role for bootstrapping 5901 * hotplugged processors. 5902 * 5903 * zoneinfo_show() and maybe other functions do 5904 * not check if the processor is online before following the pageset pointer. 5905 * Other parts of the kernel may not check if the zone is available. 5906 */ 5907 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); 5908 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); 5909 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 5910 5911 static void __build_all_zonelists(void *data) 5912 { 5913 int nid; 5914 int __maybe_unused cpu; 5915 pg_data_t *self = data; 5916 static DEFINE_SPINLOCK(lock); 5917 5918 spin_lock(&lock); 5919 5920 #ifdef CONFIG_NUMA 5921 memset(node_load, 0, sizeof(node_load)); 5922 #endif 5923 5924 /* 5925 * This node is hotadded and no memory is yet present. So just 5926 * building zonelists is fine - no need to touch other nodes. 5927 */ 5928 if (self && !node_online(self->node_id)) { 5929 build_zonelists(self); 5930 } else { 5931 for_each_online_node(nid) { 5932 pg_data_t *pgdat = NODE_DATA(nid); 5933 5934 build_zonelists(pgdat); 5935 } 5936 5937 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5938 /* 5939 * We now know the "local memory node" for each node-- 5940 * i.e., the node of the first zone in the generic zonelist. 5941 * Set up numa_mem percpu variable for on-line cpus. During 5942 * boot, only the boot cpu should be on-line; we'll init the 5943 * secondary cpus' numa_mem as they come on-line. During 5944 * node/memory hotplug, we'll fixup all on-line cpus. 5945 */ 5946 for_each_online_cpu(cpu) 5947 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 5948 #endif 5949 } 5950 5951 spin_unlock(&lock); 5952 } 5953 5954 static noinline void __init 5955 build_all_zonelists_init(void) 5956 { 5957 int cpu; 5958 5959 __build_all_zonelists(NULL); 5960 5961 /* 5962 * Initialize the boot_pagesets that are going to be used 5963 * for bootstrapping processors. The real pagesets for 5964 * each zone will be allocated later when the per cpu 5965 * allocator is available. 5966 * 5967 * boot_pagesets are used also for bootstrapping offline 5968 * cpus if the system is already booted because the pagesets 5969 * are needed to initialize allocators on a specific cpu too. 5970 * F.e. the percpu allocator needs the page allocator which 5971 * needs the percpu allocator in order to allocate its pagesets 5972 * (a chicken-egg dilemma). 5973 */ 5974 for_each_possible_cpu(cpu) 5975 setup_pageset(&per_cpu(boot_pageset, cpu), 0); 5976 5977 mminit_verify_zonelist(); 5978 cpuset_init_current_mems_allowed(); 5979 } 5980 5981 /* 5982 * unless system_state == SYSTEM_BOOTING. 5983 * 5984 * __ref due to call of __init annotated helper build_all_zonelists_init 5985 * [protected by SYSTEM_BOOTING]. 5986 */ 5987 void __ref build_all_zonelists(pg_data_t *pgdat) 5988 { 5989 unsigned long vm_total_pages; 5990 5991 if (system_state == SYSTEM_BOOTING) { 5992 build_all_zonelists_init(); 5993 } else { 5994 __build_all_zonelists(pgdat); 5995 /* cpuset refresh routine should be here */ 5996 } 5997 /* Get the number of free pages beyond high watermark in all zones. */ 5998 vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 5999 /* 6000 * Disable grouping by mobility if the number of pages in the 6001 * system is too low to allow the mechanism to work. It would be 6002 * more accurate, but expensive to check per-zone. This check is 6003 * made on memory-hotadd so a system can start with mobility 6004 * disabled and enable it later 6005 */ 6006 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 6007 page_group_by_mobility_disabled = 1; 6008 else 6009 page_group_by_mobility_disabled = 0; 6010 6011 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n", 6012 nr_online_nodes, 6013 page_group_by_mobility_disabled ? "off" : "on", 6014 vm_total_pages); 6015 #ifdef CONFIG_NUMA 6016 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 6017 #endif 6018 } 6019 6020 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 6021 static bool __meminit 6022 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 6023 { 6024 static struct memblock_region *r; 6025 6026 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 6027 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 6028 for_each_mem_region(r) { 6029 if (*pfn < memblock_region_memory_end_pfn(r)) 6030 break; 6031 } 6032 } 6033 if (*pfn >= memblock_region_memory_base_pfn(r) && 6034 memblock_is_mirror(r)) { 6035 *pfn = memblock_region_memory_end_pfn(r); 6036 return true; 6037 } 6038 } 6039 return false; 6040 } 6041 6042 /* 6043 * Initially all pages are reserved - free ones are freed 6044 * up by memblock_free_all() once the early boot process is 6045 * done. Non-atomic initialization, single-pass. 6046 * 6047 * All aligned pageblocks are initialized to the specified migratetype 6048 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related 6049 * zone stats (e.g., nr_isolate_pageblock) are touched. 6050 */ 6051 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, 6052 unsigned long start_pfn, 6053 enum meminit_context context, 6054 struct vmem_altmap *altmap, int migratetype) 6055 { 6056 unsigned long pfn, end_pfn = start_pfn + size; 6057 struct page *page; 6058 6059 if (highest_memmap_pfn < end_pfn - 1) 6060 highest_memmap_pfn = end_pfn - 1; 6061 6062 #ifdef CONFIG_ZONE_DEVICE 6063 /* 6064 * Honor reservation requested by the driver for this ZONE_DEVICE 6065 * memory. We limit the total number of pages to initialize to just 6066 * those that might contain the memory mapping. We will defer the 6067 * ZONE_DEVICE page initialization until after we have released 6068 * the hotplug lock. 6069 */ 6070 if (zone == ZONE_DEVICE) { 6071 if (!altmap) 6072 return; 6073 6074 if (start_pfn == altmap->base_pfn) 6075 start_pfn += altmap->reserve; 6076 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6077 } 6078 #endif 6079 6080 for (pfn = start_pfn; pfn < end_pfn; ) { 6081 /* 6082 * There can be holes in boot-time mem_map[]s handed to this 6083 * function. They do not exist on hotplugged memory. 6084 */ 6085 if (context == MEMINIT_EARLY) { 6086 if (overlap_memmap_init(zone, &pfn)) 6087 continue; 6088 if (defer_init(nid, pfn, end_pfn)) 6089 break; 6090 } 6091 6092 page = pfn_to_page(pfn); 6093 __init_single_page(page, pfn, zone, nid); 6094 if (context == MEMINIT_HOTPLUG) 6095 __SetPageReserved(page); 6096 6097 /* 6098 * Usually, we want to mark the pageblock MIGRATE_MOVABLE, 6099 * such that unmovable allocations won't be scattered all 6100 * over the place during system boot. 6101 */ 6102 if (IS_ALIGNED(pfn, pageblock_nr_pages)) { 6103 set_pageblock_migratetype(page, migratetype); 6104 cond_resched(); 6105 } 6106 pfn++; 6107 } 6108 } 6109 6110 #ifdef CONFIG_ZONE_DEVICE 6111 void __ref memmap_init_zone_device(struct zone *zone, 6112 unsigned long start_pfn, 6113 unsigned long nr_pages, 6114 struct dev_pagemap *pgmap) 6115 { 6116 unsigned long pfn, end_pfn = start_pfn + nr_pages; 6117 struct pglist_data *pgdat = zone->zone_pgdat; 6118 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 6119 unsigned long zone_idx = zone_idx(zone); 6120 unsigned long start = jiffies; 6121 int nid = pgdat->node_id; 6122 6123 if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE)) 6124 return; 6125 6126 /* 6127 * The call to memmap_init_zone should have already taken care 6128 * of the pages reserved for the memmap, so we can just jump to 6129 * the end of that region and start processing the device pages. 6130 */ 6131 if (altmap) { 6132 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6133 nr_pages = end_pfn - start_pfn; 6134 } 6135 6136 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 6137 struct page *page = pfn_to_page(pfn); 6138 6139 __init_single_page(page, pfn, zone_idx, nid); 6140 6141 /* 6142 * Mark page reserved as it will need to wait for onlining 6143 * phase for it to be fully associated with a zone. 6144 * 6145 * We can use the non-atomic __set_bit operation for setting 6146 * the flag as we are still initializing the pages. 6147 */ 6148 __SetPageReserved(page); 6149 6150 /* 6151 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 6152 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 6153 * ever freed or placed on a driver-private list. 6154 */ 6155 page->pgmap = pgmap; 6156 page->zone_device_data = NULL; 6157 6158 /* 6159 * Mark the block movable so that blocks are reserved for 6160 * movable at startup. This will force kernel allocations 6161 * to reserve their blocks rather than leaking throughout 6162 * the address space during boot when many long-lived 6163 * kernel allocations are made. 6164 * 6165 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap 6166 * because this is done early in section_activate() 6167 */ 6168 if (IS_ALIGNED(pfn, pageblock_nr_pages)) { 6169 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 6170 cond_resched(); 6171 } 6172 } 6173 6174 pr_info("%s initialised %lu pages in %ums\n", __func__, 6175 nr_pages, jiffies_to_msecs(jiffies - start)); 6176 } 6177 6178 #endif 6179 static void __meminit zone_init_free_lists(struct zone *zone) 6180 { 6181 unsigned int order, t; 6182 for_each_migratetype_order(order, t) { 6183 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 6184 zone->free_area[order].nr_free = 0; 6185 } 6186 } 6187 6188 void __meminit __weak memmap_init(unsigned long size, int nid, 6189 unsigned long zone, 6190 unsigned long range_start_pfn) 6191 { 6192 unsigned long start_pfn, end_pfn; 6193 unsigned long range_end_pfn = range_start_pfn + size; 6194 int i; 6195 6196 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6197 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 6198 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 6199 6200 if (end_pfn > start_pfn) { 6201 size = end_pfn - start_pfn; 6202 memmap_init_zone(size, nid, zone, start_pfn, 6203 MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); 6204 } 6205 } 6206 } 6207 6208 static int zone_batchsize(struct zone *zone) 6209 { 6210 #ifdef CONFIG_MMU 6211 int batch; 6212 6213 /* 6214 * The per-cpu-pages pools are set to around 1000th of the 6215 * size of the zone. 6216 */ 6217 batch = zone_managed_pages(zone) / 1024; 6218 /* But no more than a meg. */ 6219 if (batch * PAGE_SIZE > 1024 * 1024) 6220 batch = (1024 * 1024) / PAGE_SIZE; 6221 batch /= 4; /* We effectively *= 4 below */ 6222 if (batch < 1) 6223 batch = 1; 6224 6225 /* 6226 * Clamp the batch to a 2^n - 1 value. Having a power 6227 * of 2 value was found to be more likely to have 6228 * suboptimal cache aliasing properties in some cases. 6229 * 6230 * For example if 2 tasks are alternately allocating 6231 * batches of pages, one task can end up with a lot 6232 * of pages of one half of the possible page colors 6233 * and the other with pages of the other colors. 6234 */ 6235 batch = rounddown_pow_of_two(batch + batch/2) - 1; 6236 6237 return batch; 6238 6239 #else 6240 /* The deferral and batching of frees should be suppressed under NOMMU 6241 * conditions. 6242 * 6243 * The problem is that NOMMU needs to be able to allocate large chunks 6244 * of contiguous memory as there's no hardware page translation to 6245 * assemble apparent contiguous memory from discontiguous pages. 6246 * 6247 * Queueing large contiguous runs of pages for batching, however, 6248 * causes the pages to actually be freed in smaller chunks. As there 6249 * can be a significant delay between the individual batches being 6250 * recycled, this leads to the once large chunks of space being 6251 * fragmented and becoming unavailable for high-order allocations. 6252 */ 6253 return 0; 6254 #endif 6255 } 6256 6257 /* 6258 * pcp->high and pcp->batch values are related and dependent on one another: 6259 * ->batch must never be higher then ->high. 6260 * The following function updates them in a safe manner without read side 6261 * locking. 6262 * 6263 * Any new users of pcp->batch and pcp->high should ensure they can cope with 6264 * those fields changing asynchronously (acording to the above rule). 6265 * 6266 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 6267 * outside of boot time (or some other assurance that no concurrent updaters 6268 * exist). 6269 */ 6270 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 6271 unsigned long batch) 6272 { 6273 /* start with a fail safe value for batch */ 6274 pcp->batch = 1; 6275 smp_wmb(); 6276 6277 /* Update high, then batch, in order */ 6278 pcp->high = high; 6279 smp_wmb(); 6280 6281 pcp->batch = batch; 6282 } 6283 6284 /* a companion to pageset_set_high() */ 6285 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) 6286 { 6287 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); 6288 } 6289 6290 static void pageset_init(struct per_cpu_pageset *p) 6291 { 6292 struct per_cpu_pages *pcp; 6293 int migratetype; 6294 6295 memset(p, 0, sizeof(*p)); 6296 6297 pcp = &p->pcp; 6298 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) 6299 INIT_LIST_HEAD(&pcp->lists[migratetype]); 6300 } 6301 6302 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 6303 { 6304 pageset_init(p); 6305 pageset_set_batch(p, batch); 6306 } 6307 6308 /* 6309 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist 6310 * to the value high for the pageset p. 6311 */ 6312 static void pageset_set_high(struct per_cpu_pageset *p, 6313 unsigned long high) 6314 { 6315 unsigned long batch = max(1UL, high / 4); 6316 if ((high / 4) > (PAGE_SHIFT * 8)) 6317 batch = PAGE_SHIFT * 8; 6318 6319 pageset_update(&p->pcp, high, batch); 6320 } 6321 6322 static void pageset_set_high_and_batch(struct zone *zone, 6323 struct per_cpu_pageset *pcp) 6324 { 6325 if (percpu_pagelist_fraction) 6326 pageset_set_high(pcp, 6327 (zone_managed_pages(zone) / 6328 percpu_pagelist_fraction)); 6329 else 6330 pageset_set_batch(pcp, zone_batchsize(zone)); 6331 } 6332 6333 static void __meminit zone_pageset_init(struct zone *zone, int cpu) 6334 { 6335 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); 6336 6337 pageset_init(pcp); 6338 pageset_set_high_and_batch(zone, pcp); 6339 } 6340 6341 void __meminit setup_zone_pageset(struct zone *zone) 6342 { 6343 int cpu; 6344 zone->pageset = alloc_percpu(struct per_cpu_pageset); 6345 for_each_possible_cpu(cpu) 6346 zone_pageset_init(zone, cpu); 6347 } 6348 6349 /* 6350 * Allocate per cpu pagesets and initialize them. 6351 * Before this call only boot pagesets were available. 6352 */ 6353 void __init setup_per_cpu_pageset(void) 6354 { 6355 struct pglist_data *pgdat; 6356 struct zone *zone; 6357 int __maybe_unused cpu; 6358 6359 for_each_populated_zone(zone) 6360 setup_zone_pageset(zone); 6361 6362 #ifdef CONFIG_NUMA 6363 /* 6364 * Unpopulated zones continue using the boot pagesets. 6365 * The numa stats for these pagesets need to be reset. 6366 * Otherwise, they will end up skewing the stats of 6367 * the nodes these zones are associated with. 6368 */ 6369 for_each_possible_cpu(cpu) { 6370 struct per_cpu_pageset *pcp = &per_cpu(boot_pageset, cpu); 6371 memset(pcp->vm_numa_stat_diff, 0, 6372 sizeof(pcp->vm_numa_stat_diff)); 6373 } 6374 #endif 6375 6376 for_each_online_pgdat(pgdat) 6377 pgdat->per_cpu_nodestats = 6378 alloc_percpu(struct per_cpu_nodestat); 6379 } 6380 6381 static __meminit void zone_pcp_init(struct zone *zone) 6382 { 6383 /* 6384 * per cpu subsystem is not up at this point. The following code 6385 * relies on the ability of the linker to provide the 6386 * offset of a (static) per cpu variable into the per cpu area. 6387 */ 6388 zone->pageset = &boot_pageset; 6389 6390 if (populated_zone(zone)) 6391 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", 6392 zone->name, zone->present_pages, 6393 zone_batchsize(zone)); 6394 } 6395 6396 void __meminit init_currently_empty_zone(struct zone *zone, 6397 unsigned long zone_start_pfn, 6398 unsigned long size) 6399 { 6400 struct pglist_data *pgdat = zone->zone_pgdat; 6401 int zone_idx = zone_idx(zone) + 1; 6402 6403 if (zone_idx > pgdat->nr_zones) 6404 pgdat->nr_zones = zone_idx; 6405 6406 zone->zone_start_pfn = zone_start_pfn; 6407 6408 mminit_dprintk(MMINIT_TRACE, "memmap_init", 6409 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 6410 pgdat->node_id, 6411 (unsigned long)zone_idx(zone), 6412 zone_start_pfn, (zone_start_pfn + size)); 6413 6414 zone_init_free_lists(zone); 6415 zone->initialized = 1; 6416 } 6417 6418 /** 6419 * get_pfn_range_for_nid - Return the start and end page frames for a node 6420 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 6421 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 6422 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 6423 * 6424 * It returns the start and end page frame of a node based on information 6425 * provided by memblock_set_node(). If called for a node 6426 * with no available memory, a warning is printed and the start and end 6427 * PFNs will be 0. 6428 */ 6429 void __init get_pfn_range_for_nid(unsigned int nid, 6430 unsigned long *start_pfn, unsigned long *end_pfn) 6431 { 6432 unsigned long this_start_pfn, this_end_pfn; 6433 int i; 6434 6435 *start_pfn = -1UL; 6436 *end_pfn = 0; 6437 6438 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 6439 *start_pfn = min(*start_pfn, this_start_pfn); 6440 *end_pfn = max(*end_pfn, this_end_pfn); 6441 } 6442 6443 if (*start_pfn == -1UL) 6444 *start_pfn = 0; 6445 } 6446 6447 /* 6448 * This finds a zone that can be used for ZONE_MOVABLE pages. The 6449 * assumption is made that zones within a node are ordered in monotonic 6450 * increasing memory addresses so that the "highest" populated zone is used 6451 */ 6452 static void __init find_usable_zone_for_movable(void) 6453 { 6454 int zone_index; 6455 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 6456 if (zone_index == ZONE_MOVABLE) 6457 continue; 6458 6459 if (arch_zone_highest_possible_pfn[zone_index] > 6460 arch_zone_lowest_possible_pfn[zone_index]) 6461 break; 6462 } 6463 6464 VM_BUG_ON(zone_index == -1); 6465 movable_zone = zone_index; 6466 } 6467 6468 /* 6469 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 6470 * because it is sized independent of architecture. Unlike the other zones, 6471 * the starting point for ZONE_MOVABLE is not fixed. It may be different 6472 * in each node depending on the size of each node and how evenly kernelcore 6473 * is distributed. This helper function adjusts the zone ranges 6474 * provided by the architecture for a given node by using the end of the 6475 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 6476 * zones within a node are in order of monotonic increases memory addresses 6477 */ 6478 static void __init adjust_zone_range_for_zone_movable(int nid, 6479 unsigned long zone_type, 6480 unsigned long node_start_pfn, 6481 unsigned long node_end_pfn, 6482 unsigned long *zone_start_pfn, 6483 unsigned long *zone_end_pfn) 6484 { 6485 /* Only adjust if ZONE_MOVABLE is on this node */ 6486 if (zone_movable_pfn[nid]) { 6487 /* Size ZONE_MOVABLE */ 6488 if (zone_type == ZONE_MOVABLE) { 6489 *zone_start_pfn = zone_movable_pfn[nid]; 6490 *zone_end_pfn = min(node_end_pfn, 6491 arch_zone_highest_possible_pfn[movable_zone]); 6492 6493 /* Adjust for ZONE_MOVABLE starting within this range */ 6494 } else if (!mirrored_kernelcore && 6495 *zone_start_pfn < zone_movable_pfn[nid] && 6496 *zone_end_pfn > zone_movable_pfn[nid]) { 6497 *zone_end_pfn = zone_movable_pfn[nid]; 6498 6499 /* Check if this whole range is within ZONE_MOVABLE */ 6500 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 6501 *zone_start_pfn = *zone_end_pfn; 6502 } 6503 } 6504 6505 /* 6506 * Return the number of pages a zone spans in a node, including holes 6507 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 6508 */ 6509 static unsigned long __init zone_spanned_pages_in_node(int nid, 6510 unsigned long zone_type, 6511 unsigned long node_start_pfn, 6512 unsigned long node_end_pfn, 6513 unsigned long *zone_start_pfn, 6514 unsigned long *zone_end_pfn) 6515 { 6516 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 6517 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 6518 /* When hotadd a new node from cpu_up(), the node should be empty */ 6519 if (!node_start_pfn && !node_end_pfn) 6520 return 0; 6521 6522 /* Get the start and end of the zone */ 6523 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 6524 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 6525 adjust_zone_range_for_zone_movable(nid, zone_type, 6526 node_start_pfn, node_end_pfn, 6527 zone_start_pfn, zone_end_pfn); 6528 6529 /* Check that this node has pages within the zone's required range */ 6530 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 6531 return 0; 6532 6533 /* Move the zone boundaries inside the node if necessary */ 6534 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 6535 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 6536 6537 /* Return the spanned pages */ 6538 return *zone_end_pfn - *zone_start_pfn; 6539 } 6540 6541 /* 6542 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 6543 * then all holes in the requested range will be accounted for. 6544 */ 6545 unsigned long __init __absent_pages_in_range(int nid, 6546 unsigned long range_start_pfn, 6547 unsigned long range_end_pfn) 6548 { 6549 unsigned long nr_absent = range_end_pfn - range_start_pfn; 6550 unsigned long start_pfn, end_pfn; 6551 int i; 6552 6553 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6554 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 6555 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 6556 nr_absent -= end_pfn - start_pfn; 6557 } 6558 return nr_absent; 6559 } 6560 6561 /** 6562 * absent_pages_in_range - Return number of page frames in holes within a range 6563 * @start_pfn: The start PFN to start searching for holes 6564 * @end_pfn: The end PFN to stop searching for holes 6565 * 6566 * Return: the number of pages frames in memory holes within a range. 6567 */ 6568 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 6569 unsigned long end_pfn) 6570 { 6571 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 6572 } 6573 6574 /* Return the number of page frames in holes in a zone on a node */ 6575 static unsigned long __init zone_absent_pages_in_node(int nid, 6576 unsigned long zone_type, 6577 unsigned long node_start_pfn, 6578 unsigned long node_end_pfn) 6579 { 6580 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 6581 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 6582 unsigned long zone_start_pfn, zone_end_pfn; 6583 unsigned long nr_absent; 6584 6585 /* When hotadd a new node from cpu_up(), the node should be empty */ 6586 if (!node_start_pfn && !node_end_pfn) 6587 return 0; 6588 6589 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 6590 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 6591 6592 adjust_zone_range_for_zone_movable(nid, zone_type, 6593 node_start_pfn, node_end_pfn, 6594 &zone_start_pfn, &zone_end_pfn); 6595 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 6596 6597 /* 6598 * ZONE_MOVABLE handling. 6599 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 6600 * and vice versa. 6601 */ 6602 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 6603 unsigned long start_pfn, end_pfn; 6604 struct memblock_region *r; 6605 6606 for_each_mem_region(r) { 6607 start_pfn = clamp(memblock_region_memory_base_pfn(r), 6608 zone_start_pfn, zone_end_pfn); 6609 end_pfn = clamp(memblock_region_memory_end_pfn(r), 6610 zone_start_pfn, zone_end_pfn); 6611 6612 if (zone_type == ZONE_MOVABLE && 6613 memblock_is_mirror(r)) 6614 nr_absent += end_pfn - start_pfn; 6615 6616 if (zone_type == ZONE_NORMAL && 6617 !memblock_is_mirror(r)) 6618 nr_absent += end_pfn - start_pfn; 6619 } 6620 } 6621 6622 return nr_absent; 6623 } 6624 6625 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 6626 unsigned long node_start_pfn, 6627 unsigned long node_end_pfn) 6628 { 6629 unsigned long realtotalpages = 0, totalpages = 0; 6630 enum zone_type i; 6631 6632 for (i = 0; i < MAX_NR_ZONES; i++) { 6633 struct zone *zone = pgdat->node_zones + i; 6634 unsigned long zone_start_pfn, zone_end_pfn; 6635 unsigned long spanned, absent; 6636 unsigned long size, real_size; 6637 6638 spanned = zone_spanned_pages_in_node(pgdat->node_id, i, 6639 node_start_pfn, 6640 node_end_pfn, 6641 &zone_start_pfn, 6642 &zone_end_pfn); 6643 absent = zone_absent_pages_in_node(pgdat->node_id, i, 6644 node_start_pfn, 6645 node_end_pfn); 6646 6647 size = spanned; 6648 real_size = size - absent; 6649 6650 if (size) 6651 zone->zone_start_pfn = zone_start_pfn; 6652 else 6653 zone->zone_start_pfn = 0; 6654 zone->spanned_pages = size; 6655 zone->present_pages = real_size; 6656 6657 totalpages += size; 6658 realtotalpages += real_size; 6659 } 6660 6661 pgdat->node_spanned_pages = totalpages; 6662 pgdat->node_present_pages = realtotalpages; 6663 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, 6664 realtotalpages); 6665 } 6666 6667 #ifndef CONFIG_SPARSEMEM 6668 /* 6669 * Calculate the size of the zone->blockflags rounded to an unsigned long 6670 * Start by making sure zonesize is a multiple of pageblock_order by rounding 6671 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 6672 * round what is now in bits to nearest long in bits, then return it in 6673 * bytes. 6674 */ 6675 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 6676 { 6677 unsigned long usemapsize; 6678 6679 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 6680 usemapsize = roundup(zonesize, pageblock_nr_pages); 6681 usemapsize = usemapsize >> pageblock_order; 6682 usemapsize *= NR_PAGEBLOCK_BITS; 6683 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 6684 6685 return usemapsize / 8; 6686 } 6687 6688 static void __ref setup_usemap(struct pglist_data *pgdat, 6689 struct zone *zone, 6690 unsigned long zone_start_pfn, 6691 unsigned long zonesize) 6692 { 6693 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); 6694 zone->pageblock_flags = NULL; 6695 if (usemapsize) { 6696 zone->pageblock_flags = 6697 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 6698 pgdat->node_id); 6699 if (!zone->pageblock_flags) 6700 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 6701 usemapsize, zone->name, pgdat->node_id); 6702 } 6703 } 6704 #else 6705 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, 6706 unsigned long zone_start_pfn, unsigned long zonesize) {} 6707 #endif /* CONFIG_SPARSEMEM */ 6708 6709 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 6710 6711 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 6712 void __init set_pageblock_order(void) 6713 { 6714 unsigned int order; 6715 6716 /* Check that pageblock_nr_pages has not already been setup */ 6717 if (pageblock_order) 6718 return; 6719 6720 if (HPAGE_SHIFT > PAGE_SHIFT) 6721 order = HUGETLB_PAGE_ORDER; 6722 else 6723 order = MAX_ORDER - 1; 6724 6725 /* 6726 * Assume the largest contiguous order of interest is a huge page. 6727 * This value may be variable depending on boot parameters on IA64 and 6728 * powerpc. 6729 */ 6730 pageblock_order = order; 6731 } 6732 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6733 6734 /* 6735 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 6736 * is unused as pageblock_order is set at compile-time. See 6737 * include/linux/pageblock-flags.h for the values of pageblock_order based on 6738 * the kernel config 6739 */ 6740 void __init set_pageblock_order(void) 6741 { 6742 } 6743 6744 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6745 6746 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 6747 unsigned long present_pages) 6748 { 6749 unsigned long pages = spanned_pages; 6750 6751 /* 6752 * Provide a more accurate estimation if there are holes within 6753 * the zone and SPARSEMEM is in use. If there are holes within the 6754 * zone, each populated memory region may cost us one or two extra 6755 * memmap pages due to alignment because memmap pages for each 6756 * populated regions may not be naturally aligned on page boundary. 6757 * So the (present_pages >> 4) heuristic is a tradeoff for that. 6758 */ 6759 if (spanned_pages > present_pages + (present_pages >> 4) && 6760 IS_ENABLED(CONFIG_SPARSEMEM)) 6761 pages = present_pages; 6762 6763 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 6764 } 6765 6766 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 6767 static void pgdat_init_split_queue(struct pglist_data *pgdat) 6768 { 6769 struct deferred_split *ds_queue = &pgdat->deferred_split_queue; 6770 6771 spin_lock_init(&ds_queue->split_queue_lock); 6772 INIT_LIST_HEAD(&ds_queue->split_queue); 6773 ds_queue->split_queue_len = 0; 6774 } 6775 #else 6776 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 6777 #endif 6778 6779 #ifdef CONFIG_COMPACTION 6780 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 6781 { 6782 init_waitqueue_head(&pgdat->kcompactd_wait); 6783 } 6784 #else 6785 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 6786 #endif 6787 6788 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 6789 { 6790 pgdat_resize_init(pgdat); 6791 6792 pgdat_init_split_queue(pgdat); 6793 pgdat_init_kcompactd(pgdat); 6794 6795 init_waitqueue_head(&pgdat->kswapd_wait); 6796 init_waitqueue_head(&pgdat->pfmemalloc_wait); 6797 6798 pgdat_page_ext_init(pgdat); 6799 spin_lock_init(&pgdat->lru_lock); 6800 lruvec_init(&pgdat->__lruvec); 6801 } 6802 6803 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 6804 unsigned long remaining_pages) 6805 { 6806 atomic_long_set(&zone->managed_pages, remaining_pages); 6807 zone_set_nid(zone, nid); 6808 zone->name = zone_names[idx]; 6809 zone->zone_pgdat = NODE_DATA(nid); 6810 spin_lock_init(&zone->lock); 6811 zone_seqlock_init(zone); 6812 zone_pcp_init(zone); 6813 } 6814 6815 /* 6816 * Set up the zone data structures 6817 * - init pgdat internals 6818 * - init all zones belonging to this node 6819 * 6820 * NOTE: this function is only called during memory hotplug 6821 */ 6822 #ifdef CONFIG_MEMORY_HOTPLUG 6823 void __ref free_area_init_core_hotplug(int nid) 6824 { 6825 enum zone_type z; 6826 pg_data_t *pgdat = NODE_DATA(nid); 6827 6828 pgdat_init_internals(pgdat); 6829 for (z = 0; z < MAX_NR_ZONES; z++) 6830 zone_init_internals(&pgdat->node_zones[z], z, nid, 0); 6831 } 6832 #endif 6833 6834 /* 6835 * Set up the zone data structures: 6836 * - mark all pages reserved 6837 * - mark all memory queues empty 6838 * - clear the memory bitmaps 6839 * 6840 * NOTE: pgdat should get zeroed by caller. 6841 * NOTE: this function is only called during early init. 6842 */ 6843 static void __init free_area_init_core(struct pglist_data *pgdat) 6844 { 6845 enum zone_type j; 6846 int nid = pgdat->node_id; 6847 6848 pgdat_init_internals(pgdat); 6849 pgdat->per_cpu_nodestats = &boot_nodestats; 6850 6851 for (j = 0; j < MAX_NR_ZONES; j++) { 6852 struct zone *zone = pgdat->node_zones + j; 6853 unsigned long size, freesize, memmap_pages; 6854 unsigned long zone_start_pfn = zone->zone_start_pfn; 6855 6856 size = zone->spanned_pages; 6857 freesize = zone->present_pages; 6858 6859 /* 6860 * Adjust freesize so that it accounts for how much memory 6861 * is used by this zone for memmap. This affects the watermark 6862 * and per-cpu initialisations 6863 */ 6864 memmap_pages = calc_memmap_size(size, freesize); 6865 if (!is_highmem_idx(j)) { 6866 if (freesize >= memmap_pages) { 6867 freesize -= memmap_pages; 6868 if (memmap_pages) 6869 printk(KERN_DEBUG 6870 " %s zone: %lu pages used for memmap\n", 6871 zone_names[j], memmap_pages); 6872 } else 6873 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n", 6874 zone_names[j], memmap_pages, freesize); 6875 } 6876 6877 /* Account for reserved pages */ 6878 if (j == 0 && freesize > dma_reserve) { 6879 freesize -= dma_reserve; 6880 printk(KERN_DEBUG " %s zone: %lu pages reserved\n", 6881 zone_names[0], dma_reserve); 6882 } 6883 6884 if (!is_highmem_idx(j)) 6885 nr_kernel_pages += freesize; 6886 /* Charge for highmem memmap if there are enough kernel pages */ 6887 else if (nr_kernel_pages > memmap_pages * 2) 6888 nr_kernel_pages -= memmap_pages; 6889 nr_all_pages += freesize; 6890 6891 /* 6892 * Set an approximate value for lowmem here, it will be adjusted 6893 * when the bootmem allocator frees pages into the buddy system. 6894 * And all highmem pages will be managed by the buddy system. 6895 */ 6896 zone_init_internals(zone, j, nid, freesize); 6897 6898 if (!size) 6899 continue; 6900 6901 set_pageblock_order(); 6902 setup_usemap(pgdat, zone, zone_start_pfn, size); 6903 init_currently_empty_zone(zone, zone_start_pfn, size); 6904 memmap_init(size, nid, j, zone_start_pfn); 6905 } 6906 } 6907 6908 #ifdef CONFIG_FLAT_NODE_MEM_MAP 6909 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) 6910 { 6911 unsigned long __maybe_unused start = 0; 6912 unsigned long __maybe_unused offset = 0; 6913 6914 /* Skip empty nodes */ 6915 if (!pgdat->node_spanned_pages) 6916 return; 6917 6918 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 6919 offset = pgdat->node_start_pfn - start; 6920 /* ia64 gets its own node_mem_map, before this, without bootmem */ 6921 if (!pgdat->node_mem_map) { 6922 unsigned long size, end; 6923 struct page *map; 6924 6925 /* 6926 * The zone's endpoints aren't required to be MAX_ORDER 6927 * aligned but the node_mem_map endpoints must be in order 6928 * for the buddy allocator to function correctly. 6929 */ 6930 end = pgdat_end_pfn(pgdat); 6931 end = ALIGN(end, MAX_ORDER_NR_PAGES); 6932 size = (end - start) * sizeof(struct page); 6933 map = memblock_alloc_node(size, SMP_CACHE_BYTES, 6934 pgdat->node_id); 6935 if (!map) 6936 panic("Failed to allocate %ld bytes for node %d memory map\n", 6937 size, pgdat->node_id); 6938 pgdat->node_mem_map = map + offset; 6939 } 6940 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 6941 __func__, pgdat->node_id, (unsigned long)pgdat, 6942 (unsigned long)pgdat->node_mem_map); 6943 #ifndef CONFIG_NEED_MULTIPLE_NODES 6944 /* 6945 * With no DISCONTIG, the global mem_map is just set as node 0's 6946 */ 6947 if (pgdat == NODE_DATA(0)) { 6948 mem_map = NODE_DATA(0)->node_mem_map; 6949 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 6950 mem_map -= offset; 6951 } 6952 #endif 6953 } 6954 #else 6955 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { } 6956 #endif /* CONFIG_FLAT_NODE_MEM_MAP */ 6957 6958 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 6959 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 6960 { 6961 pgdat->first_deferred_pfn = ULONG_MAX; 6962 } 6963 #else 6964 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 6965 #endif 6966 6967 static void __init free_area_init_node(int nid) 6968 { 6969 pg_data_t *pgdat = NODE_DATA(nid); 6970 unsigned long start_pfn = 0; 6971 unsigned long end_pfn = 0; 6972 6973 /* pg_data_t should be reset to zero when it's allocated */ 6974 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); 6975 6976 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 6977 6978 pgdat->node_id = nid; 6979 pgdat->node_start_pfn = start_pfn; 6980 pgdat->per_cpu_nodestats = NULL; 6981 6982 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 6983 (u64)start_pfn << PAGE_SHIFT, 6984 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 6985 calculate_node_totalpages(pgdat, start_pfn, end_pfn); 6986 6987 alloc_node_mem_map(pgdat); 6988 pgdat_set_deferred_range(pgdat); 6989 6990 free_area_init_core(pgdat); 6991 } 6992 6993 void __init free_area_init_memoryless_node(int nid) 6994 { 6995 free_area_init_node(nid); 6996 } 6997 6998 #if !defined(CONFIG_FLAT_NODE_MEM_MAP) 6999 /* 7000 * Initialize all valid struct pages in the range [spfn, epfn) and mark them 7001 * PageReserved(). Return the number of struct pages that were initialized. 7002 */ 7003 static u64 __init init_unavailable_range(unsigned long spfn, unsigned long epfn) 7004 { 7005 unsigned long pfn; 7006 u64 pgcnt = 0; 7007 7008 for (pfn = spfn; pfn < epfn; pfn++) { 7009 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) { 7010 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages) 7011 + pageblock_nr_pages - 1; 7012 continue; 7013 } 7014 /* 7015 * Use a fake node/zone (0) for now. Some of these pages 7016 * (in memblock.reserved but not in memblock.memory) will 7017 * get re-initialized via reserve_bootmem_region() later. 7018 */ 7019 __init_single_page(pfn_to_page(pfn), pfn, 0, 0); 7020 __SetPageReserved(pfn_to_page(pfn)); 7021 pgcnt++; 7022 } 7023 7024 return pgcnt; 7025 } 7026 7027 /* 7028 * Only struct pages that are backed by physical memory are zeroed and 7029 * initialized by going through __init_single_page(). But, there are some 7030 * struct pages which are reserved in memblock allocator and their fields 7031 * may be accessed (for example page_to_pfn() on some configuration accesses 7032 * flags). We must explicitly initialize those struct pages. 7033 * 7034 * This function also addresses a similar issue where struct pages are left 7035 * uninitialized because the physical address range is not covered by 7036 * memblock.memory or memblock.reserved. That could happen when memblock 7037 * layout is manually configured via memmap=, or when the highest physical 7038 * address (max_pfn) does not end on a section boundary. 7039 */ 7040 static void __init init_unavailable_mem(void) 7041 { 7042 phys_addr_t start, end; 7043 u64 i, pgcnt; 7044 phys_addr_t next = 0; 7045 7046 /* 7047 * Loop through unavailable ranges not covered by memblock.memory. 7048 */ 7049 pgcnt = 0; 7050 for_each_mem_range(i, &start, &end) { 7051 if (next < start) 7052 pgcnt += init_unavailable_range(PFN_DOWN(next), 7053 PFN_UP(start)); 7054 next = end; 7055 } 7056 7057 /* 7058 * Early sections always have a fully populated memmap for the whole 7059 * section - see pfn_valid(). If the last section has holes at the 7060 * end and that section is marked "online", the memmap will be 7061 * considered initialized. Make sure that memmap has a well defined 7062 * state. 7063 */ 7064 pgcnt += init_unavailable_range(PFN_DOWN(next), 7065 round_up(max_pfn, PAGES_PER_SECTION)); 7066 7067 /* 7068 * Struct pages that do not have backing memory. This could be because 7069 * firmware is using some of this memory, or for some other reasons. 7070 */ 7071 if (pgcnt) 7072 pr_info("Zeroed struct page in unavailable ranges: %lld pages", pgcnt); 7073 } 7074 #else 7075 static inline void __init init_unavailable_mem(void) 7076 { 7077 } 7078 #endif /* !CONFIG_FLAT_NODE_MEM_MAP */ 7079 7080 #if MAX_NUMNODES > 1 7081 /* 7082 * Figure out the number of possible node ids. 7083 */ 7084 void __init setup_nr_node_ids(void) 7085 { 7086 unsigned int highest; 7087 7088 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 7089 nr_node_ids = highest + 1; 7090 } 7091 #endif 7092 7093 /** 7094 * node_map_pfn_alignment - determine the maximum internode alignment 7095 * 7096 * This function should be called after node map is populated and sorted. 7097 * It calculates the maximum power of two alignment which can distinguish 7098 * all the nodes. 7099 * 7100 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 7101 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 7102 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 7103 * shifted, 1GiB is enough and this function will indicate so. 7104 * 7105 * This is used to test whether pfn -> nid mapping of the chosen memory 7106 * model has fine enough granularity to avoid incorrect mapping for the 7107 * populated node map. 7108 * 7109 * Return: the determined alignment in pfn's. 0 if there is no alignment 7110 * requirement (single node). 7111 */ 7112 unsigned long __init node_map_pfn_alignment(void) 7113 { 7114 unsigned long accl_mask = 0, last_end = 0; 7115 unsigned long start, end, mask; 7116 int last_nid = NUMA_NO_NODE; 7117 int i, nid; 7118 7119 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 7120 if (!start || last_nid < 0 || last_nid == nid) { 7121 last_nid = nid; 7122 last_end = end; 7123 continue; 7124 } 7125 7126 /* 7127 * Start with a mask granular enough to pin-point to the 7128 * start pfn and tick off bits one-by-one until it becomes 7129 * too coarse to separate the current node from the last. 7130 */ 7131 mask = ~((1 << __ffs(start)) - 1); 7132 while (mask && last_end <= (start & (mask << 1))) 7133 mask <<= 1; 7134 7135 /* accumulate all internode masks */ 7136 accl_mask |= mask; 7137 } 7138 7139 /* convert mask to number of pages */ 7140 return ~accl_mask + 1; 7141 } 7142 7143 /** 7144 * find_min_pfn_with_active_regions - Find the minimum PFN registered 7145 * 7146 * Return: the minimum PFN based on information provided via 7147 * memblock_set_node(). 7148 */ 7149 unsigned long __init find_min_pfn_with_active_regions(void) 7150 { 7151 return PHYS_PFN(memblock_start_of_DRAM()); 7152 } 7153 7154 /* 7155 * early_calculate_totalpages() 7156 * Sum pages in active regions for movable zone. 7157 * Populate N_MEMORY for calculating usable_nodes. 7158 */ 7159 static unsigned long __init early_calculate_totalpages(void) 7160 { 7161 unsigned long totalpages = 0; 7162 unsigned long start_pfn, end_pfn; 7163 int i, nid; 7164 7165 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7166 unsigned long pages = end_pfn - start_pfn; 7167 7168 totalpages += pages; 7169 if (pages) 7170 node_set_state(nid, N_MEMORY); 7171 } 7172 return totalpages; 7173 } 7174 7175 /* 7176 * Find the PFN the Movable zone begins in each node. Kernel memory 7177 * is spread evenly between nodes as long as the nodes have enough 7178 * memory. When they don't, some nodes will have more kernelcore than 7179 * others 7180 */ 7181 static void __init find_zone_movable_pfns_for_nodes(void) 7182 { 7183 int i, nid; 7184 unsigned long usable_startpfn; 7185 unsigned long kernelcore_node, kernelcore_remaining; 7186 /* save the state before borrow the nodemask */ 7187 nodemask_t saved_node_state = node_states[N_MEMORY]; 7188 unsigned long totalpages = early_calculate_totalpages(); 7189 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 7190 struct memblock_region *r; 7191 7192 /* Need to find movable_zone earlier when movable_node is specified. */ 7193 find_usable_zone_for_movable(); 7194 7195 /* 7196 * If movable_node is specified, ignore kernelcore and movablecore 7197 * options. 7198 */ 7199 if (movable_node_is_enabled()) { 7200 for_each_mem_region(r) { 7201 if (!memblock_is_hotpluggable(r)) 7202 continue; 7203 7204 nid = memblock_get_region_node(r); 7205 7206 usable_startpfn = PFN_DOWN(r->base); 7207 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7208 min(usable_startpfn, zone_movable_pfn[nid]) : 7209 usable_startpfn; 7210 } 7211 7212 goto out2; 7213 } 7214 7215 /* 7216 * If kernelcore=mirror is specified, ignore movablecore option 7217 */ 7218 if (mirrored_kernelcore) { 7219 bool mem_below_4gb_not_mirrored = false; 7220 7221 for_each_mem_region(r) { 7222 if (memblock_is_mirror(r)) 7223 continue; 7224 7225 nid = memblock_get_region_node(r); 7226 7227 usable_startpfn = memblock_region_memory_base_pfn(r); 7228 7229 if (usable_startpfn < 0x100000) { 7230 mem_below_4gb_not_mirrored = true; 7231 continue; 7232 } 7233 7234 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7235 min(usable_startpfn, zone_movable_pfn[nid]) : 7236 usable_startpfn; 7237 } 7238 7239 if (mem_below_4gb_not_mirrored) 7240 pr_warn("This configuration results in unmirrored kernel memory.\n"); 7241 7242 goto out2; 7243 } 7244 7245 /* 7246 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 7247 * amount of necessary memory. 7248 */ 7249 if (required_kernelcore_percent) 7250 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 7251 10000UL; 7252 if (required_movablecore_percent) 7253 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 7254 10000UL; 7255 7256 /* 7257 * If movablecore= was specified, calculate what size of 7258 * kernelcore that corresponds so that memory usable for 7259 * any allocation type is evenly spread. If both kernelcore 7260 * and movablecore are specified, then the value of kernelcore 7261 * will be used for required_kernelcore if it's greater than 7262 * what movablecore would have allowed. 7263 */ 7264 if (required_movablecore) { 7265 unsigned long corepages; 7266 7267 /* 7268 * Round-up so that ZONE_MOVABLE is at least as large as what 7269 * was requested by the user 7270 */ 7271 required_movablecore = 7272 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 7273 required_movablecore = min(totalpages, required_movablecore); 7274 corepages = totalpages - required_movablecore; 7275 7276 required_kernelcore = max(required_kernelcore, corepages); 7277 } 7278 7279 /* 7280 * If kernelcore was not specified or kernelcore size is larger 7281 * than totalpages, there is no ZONE_MOVABLE. 7282 */ 7283 if (!required_kernelcore || required_kernelcore >= totalpages) 7284 goto out; 7285 7286 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 7287 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 7288 7289 restart: 7290 /* Spread kernelcore memory as evenly as possible throughout nodes */ 7291 kernelcore_node = required_kernelcore / usable_nodes; 7292 for_each_node_state(nid, N_MEMORY) { 7293 unsigned long start_pfn, end_pfn; 7294 7295 /* 7296 * Recalculate kernelcore_node if the division per node 7297 * now exceeds what is necessary to satisfy the requested 7298 * amount of memory for the kernel 7299 */ 7300 if (required_kernelcore < kernelcore_node) 7301 kernelcore_node = required_kernelcore / usable_nodes; 7302 7303 /* 7304 * As the map is walked, we track how much memory is usable 7305 * by the kernel using kernelcore_remaining. When it is 7306 * 0, the rest of the node is usable by ZONE_MOVABLE 7307 */ 7308 kernelcore_remaining = kernelcore_node; 7309 7310 /* Go through each range of PFNs within this node */ 7311 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7312 unsigned long size_pages; 7313 7314 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 7315 if (start_pfn >= end_pfn) 7316 continue; 7317 7318 /* Account for what is only usable for kernelcore */ 7319 if (start_pfn < usable_startpfn) { 7320 unsigned long kernel_pages; 7321 kernel_pages = min(end_pfn, usable_startpfn) 7322 - start_pfn; 7323 7324 kernelcore_remaining -= min(kernel_pages, 7325 kernelcore_remaining); 7326 required_kernelcore -= min(kernel_pages, 7327 required_kernelcore); 7328 7329 /* Continue if range is now fully accounted */ 7330 if (end_pfn <= usable_startpfn) { 7331 7332 /* 7333 * Push zone_movable_pfn to the end so 7334 * that if we have to rebalance 7335 * kernelcore across nodes, we will 7336 * not double account here 7337 */ 7338 zone_movable_pfn[nid] = end_pfn; 7339 continue; 7340 } 7341 start_pfn = usable_startpfn; 7342 } 7343 7344 /* 7345 * The usable PFN range for ZONE_MOVABLE is from 7346 * start_pfn->end_pfn. Calculate size_pages as the 7347 * number of pages used as kernelcore 7348 */ 7349 size_pages = end_pfn - start_pfn; 7350 if (size_pages > kernelcore_remaining) 7351 size_pages = kernelcore_remaining; 7352 zone_movable_pfn[nid] = start_pfn + size_pages; 7353 7354 /* 7355 * Some kernelcore has been met, update counts and 7356 * break if the kernelcore for this node has been 7357 * satisfied 7358 */ 7359 required_kernelcore -= min(required_kernelcore, 7360 size_pages); 7361 kernelcore_remaining -= size_pages; 7362 if (!kernelcore_remaining) 7363 break; 7364 } 7365 } 7366 7367 /* 7368 * If there is still required_kernelcore, we do another pass with one 7369 * less node in the count. This will push zone_movable_pfn[nid] further 7370 * along on the nodes that still have memory until kernelcore is 7371 * satisfied 7372 */ 7373 usable_nodes--; 7374 if (usable_nodes && required_kernelcore > usable_nodes) 7375 goto restart; 7376 7377 out2: 7378 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 7379 for (nid = 0; nid < MAX_NUMNODES; nid++) 7380 zone_movable_pfn[nid] = 7381 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 7382 7383 out: 7384 /* restore the node_state */ 7385 node_states[N_MEMORY] = saved_node_state; 7386 } 7387 7388 /* Any regular or high memory on that node ? */ 7389 static void check_for_memory(pg_data_t *pgdat, int nid) 7390 { 7391 enum zone_type zone_type; 7392 7393 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 7394 struct zone *zone = &pgdat->node_zones[zone_type]; 7395 if (populated_zone(zone)) { 7396 if (IS_ENABLED(CONFIG_HIGHMEM)) 7397 node_set_state(nid, N_HIGH_MEMORY); 7398 if (zone_type <= ZONE_NORMAL) 7399 node_set_state(nid, N_NORMAL_MEMORY); 7400 break; 7401 } 7402 } 7403 } 7404 7405 /* 7406 * Some architecturs, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For 7407 * such cases we allow max_zone_pfn sorted in the descending order 7408 */ 7409 bool __weak arch_has_descending_max_zone_pfns(void) 7410 { 7411 return false; 7412 } 7413 7414 /** 7415 * free_area_init - Initialise all pg_data_t and zone data 7416 * @max_zone_pfn: an array of max PFNs for each zone 7417 * 7418 * This will call free_area_init_node() for each active node in the system. 7419 * Using the page ranges provided by memblock_set_node(), the size of each 7420 * zone in each node and their holes is calculated. If the maximum PFN 7421 * between two adjacent zones match, it is assumed that the zone is empty. 7422 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 7423 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 7424 * starts where the previous one ended. For example, ZONE_DMA32 starts 7425 * at arch_max_dma_pfn. 7426 */ 7427 void __init free_area_init(unsigned long *max_zone_pfn) 7428 { 7429 unsigned long start_pfn, end_pfn; 7430 int i, nid, zone; 7431 bool descending; 7432 7433 /* Record where the zone boundaries are */ 7434 memset(arch_zone_lowest_possible_pfn, 0, 7435 sizeof(arch_zone_lowest_possible_pfn)); 7436 memset(arch_zone_highest_possible_pfn, 0, 7437 sizeof(arch_zone_highest_possible_pfn)); 7438 7439 start_pfn = find_min_pfn_with_active_regions(); 7440 descending = arch_has_descending_max_zone_pfns(); 7441 7442 for (i = 0; i < MAX_NR_ZONES; i++) { 7443 if (descending) 7444 zone = MAX_NR_ZONES - i - 1; 7445 else 7446 zone = i; 7447 7448 if (zone == ZONE_MOVABLE) 7449 continue; 7450 7451 end_pfn = max(max_zone_pfn[zone], start_pfn); 7452 arch_zone_lowest_possible_pfn[zone] = start_pfn; 7453 arch_zone_highest_possible_pfn[zone] = end_pfn; 7454 7455 start_pfn = end_pfn; 7456 } 7457 7458 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 7459 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 7460 find_zone_movable_pfns_for_nodes(); 7461 7462 /* Print out the zone ranges */ 7463 pr_info("Zone ranges:\n"); 7464 for (i = 0; i < MAX_NR_ZONES; i++) { 7465 if (i == ZONE_MOVABLE) 7466 continue; 7467 pr_info(" %-8s ", zone_names[i]); 7468 if (arch_zone_lowest_possible_pfn[i] == 7469 arch_zone_highest_possible_pfn[i]) 7470 pr_cont("empty\n"); 7471 else 7472 pr_cont("[mem %#018Lx-%#018Lx]\n", 7473 (u64)arch_zone_lowest_possible_pfn[i] 7474 << PAGE_SHIFT, 7475 ((u64)arch_zone_highest_possible_pfn[i] 7476 << PAGE_SHIFT) - 1); 7477 } 7478 7479 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 7480 pr_info("Movable zone start for each node\n"); 7481 for (i = 0; i < MAX_NUMNODES; i++) { 7482 if (zone_movable_pfn[i]) 7483 pr_info(" Node %d: %#018Lx\n", i, 7484 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 7485 } 7486 7487 /* 7488 * Print out the early node map, and initialize the 7489 * subsection-map relative to active online memory ranges to 7490 * enable future "sub-section" extensions of the memory map. 7491 */ 7492 pr_info("Early memory node ranges\n"); 7493 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7494 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 7495 (u64)start_pfn << PAGE_SHIFT, 7496 ((u64)end_pfn << PAGE_SHIFT) - 1); 7497 subsection_map_init(start_pfn, end_pfn - start_pfn); 7498 } 7499 7500 /* Initialise every node */ 7501 mminit_verify_pageflags_layout(); 7502 setup_nr_node_ids(); 7503 init_unavailable_mem(); 7504 for_each_online_node(nid) { 7505 pg_data_t *pgdat = NODE_DATA(nid); 7506 free_area_init_node(nid); 7507 7508 /* Any memory on that node */ 7509 if (pgdat->node_present_pages) 7510 node_set_state(nid, N_MEMORY); 7511 check_for_memory(pgdat, nid); 7512 } 7513 } 7514 7515 static int __init cmdline_parse_core(char *p, unsigned long *core, 7516 unsigned long *percent) 7517 { 7518 unsigned long long coremem; 7519 char *endptr; 7520 7521 if (!p) 7522 return -EINVAL; 7523 7524 /* Value may be a percentage of total memory, otherwise bytes */ 7525 coremem = simple_strtoull(p, &endptr, 0); 7526 if (*endptr == '%') { 7527 /* Paranoid check for percent values greater than 100 */ 7528 WARN_ON(coremem > 100); 7529 7530 *percent = coremem; 7531 } else { 7532 coremem = memparse(p, &p); 7533 /* Paranoid check that UL is enough for the coremem value */ 7534 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 7535 7536 *core = coremem >> PAGE_SHIFT; 7537 *percent = 0UL; 7538 } 7539 return 0; 7540 } 7541 7542 /* 7543 * kernelcore=size sets the amount of memory for use for allocations that 7544 * cannot be reclaimed or migrated. 7545 */ 7546 static int __init cmdline_parse_kernelcore(char *p) 7547 { 7548 /* parse kernelcore=mirror */ 7549 if (parse_option_str(p, "mirror")) { 7550 mirrored_kernelcore = true; 7551 return 0; 7552 } 7553 7554 return cmdline_parse_core(p, &required_kernelcore, 7555 &required_kernelcore_percent); 7556 } 7557 7558 /* 7559 * movablecore=size sets the amount of memory for use for allocations that 7560 * can be reclaimed or migrated. 7561 */ 7562 static int __init cmdline_parse_movablecore(char *p) 7563 { 7564 return cmdline_parse_core(p, &required_movablecore, 7565 &required_movablecore_percent); 7566 } 7567 7568 early_param("kernelcore", cmdline_parse_kernelcore); 7569 early_param("movablecore", cmdline_parse_movablecore); 7570 7571 void adjust_managed_page_count(struct page *page, long count) 7572 { 7573 atomic_long_add(count, &page_zone(page)->managed_pages); 7574 totalram_pages_add(count); 7575 #ifdef CONFIG_HIGHMEM 7576 if (PageHighMem(page)) 7577 totalhigh_pages_add(count); 7578 #endif 7579 } 7580 EXPORT_SYMBOL(adjust_managed_page_count); 7581 7582 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 7583 { 7584 void *pos; 7585 unsigned long pages = 0; 7586 7587 start = (void *)PAGE_ALIGN((unsigned long)start); 7588 end = (void *)((unsigned long)end & PAGE_MASK); 7589 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 7590 struct page *page = virt_to_page(pos); 7591 void *direct_map_addr; 7592 7593 /* 7594 * 'direct_map_addr' might be different from 'pos' 7595 * because some architectures' virt_to_page() 7596 * work with aliases. Getting the direct map 7597 * address ensures that we get a _writeable_ 7598 * alias for the memset(). 7599 */ 7600 direct_map_addr = page_address(page); 7601 if ((unsigned int)poison <= 0xFF) 7602 memset(direct_map_addr, poison, PAGE_SIZE); 7603 7604 free_reserved_page(page); 7605 } 7606 7607 if (pages && s) 7608 pr_info("Freeing %s memory: %ldK\n", 7609 s, pages << (PAGE_SHIFT - 10)); 7610 7611 return pages; 7612 } 7613 7614 #ifdef CONFIG_HIGHMEM 7615 void free_highmem_page(struct page *page) 7616 { 7617 __free_reserved_page(page); 7618 totalram_pages_inc(); 7619 atomic_long_inc(&page_zone(page)->managed_pages); 7620 totalhigh_pages_inc(); 7621 } 7622 #endif 7623 7624 7625 void __init mem_init_print_info(const char *str) 7626 { 7627 unsigned long physpages, codesize, datasize, rosize, bss_size; 7628 unsigned long init_code_size, init_data_size; 7629 7630 physpages = get_num_physpages(); 7631 codesize = _etext - _stext; 7632 datasize = _edata - _sdata; 7633 rosize = __end_rodata - __start_rodata; 7634 bss_size = __bss_stop - __bss_start; 7635 init_data_size = __init_end - __init_begin; 7636 init_code_size = _einittext - _sinittext; 7637 7638 /* 7639 * Detect special cases and adjust section sizes accordingly: 7640 * 1) .init.* may be embedded into .data sections 7641 * 2) .init.text.* may be out of [__init_begin, __init_end], 7642 * please refer to arch/tile/kernel/vmlinux.lds.S. 7643 * 3) .rodata.* may be embedded into .text or .data sections. 7644 */ 7645 #define adj_init_size(start, end, size, pos, adj) \ 7646 do { \ 7647 if (start <= pos && pos < end && size > adj) \ 7648 size -= adj; \ 7649 } while (0) 7650 7651 adj_init_size(__init_begin, __init_end, init_data_size, 7652 _sinittext, init_code_size); 7653 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 7654 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 7655 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 7656 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 7657 7658 #undef adj_init_size 7659 7660 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 7661 #ifdef CONFIG_HIGHMEM 7662 ", %luK highmem" 7663 #endif 7664 "%s%s)\n", 7665 nr_free_pages() << (PAGE_SHIFT - 10), 7666 physpages << (PAGE_SHIFT - 10), 7667 codesize >> 10, datasize >> 10, rosize >> 10, 7668 (init_data_size + init_code_size) >> 10, bss_size >> 10, 7669 (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10), 7670 totalcma_pages << (PAGE_SHIFT - 10), 7671 #ifdef CONFIG_HIGHMEM 7672 totalhigh_pages() << (PAGE_SHIFT - 10), 7673 #endif 7674 str ? ", " : "", str ? str : ""); 7675 } 7676 7677 /** 7678 * set_dma_reserve - set the specified number of pages reserved in the first zone 7679 * @new_dma_reserve: The number of pages to mark reserved 7680 * 7681 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 7682 * In the DMA zone, a significant percentage may be consumed by kernel image 7683 * and other unfreeable allocations which can skew the watermarks badly. This 7684 * function may optionally be used to account for unfreeable pages in the 7685 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 7686 * smaller per-cpu batchsize. 7687 */ 7688 void __init set_dma_reserve(unsigned long new_dma_reserve) 7689 { 7690 dma_reserve = new_dma_reserve; 7691 } 7692 7693 static int page_alloc_cpu_dead(unsigned int cpu) 7694 { 7695 7696 lru_add_drain_cpu(cpu); 7697 drain_pages(cpu); 7698 7699 /* 7700 * Spill the event counters of the dead processor 7701 * into the current processors event counters. 7702 * This artificially elevates the count of the current 7703 * processor. 7704 */ 7705 vm_events_fold_cpu(cpu); 7706 7707 /* 7708 * Zero the differential counters of the dead processor 7709 * so that the vm statistics are consistent. 7710 * 7711 * This is only okay since the processor is dead and cannot 7712 * race with what we are doing. 7713 */ 7714 cpu_vm_stats_fold(cpu); 7715 return 0; 7716 } 7717 7718 #ifdef CONFIG_NUMA 7719 int hashdist = HASHDIST_DEFAULT; 7720 7721 static int __init set_hashdist(char *str) 7722 { 7723 if (!str) 7724 return 0; 7725 hashdist = simple_strtoul(str, &str, 0); 7726 return 1; 7727 } 7728 __setup("hashdist=", set_hashdist); 7729 #endif 7730 7731 void __init page_alloc_init(void) 7732 { 7733 int ret; 7734 7735 #ifdef CONFIG_NUMA 7736 if (num_node_state(N_MEMORY) == 1) 7737 hashdist = 0; 7738 #endif 7739 7740 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD, 7741 "mm/page_alloc:dead", NULL, 7742 page_alloc_cpu_dead); 7743 WARN_ON(ret < 0); 7744 } 7745 7746 /* 7747 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 7748 * or min_free_kbytes changes. 7749 */ 7750 static void calculate_totalreserve_pages(void) 7751 { 7752 struct pglist_data *pgdat; 7753 unsigned long reserve_pages = 0; 7754 enum zone_type i, j; 7755 7756 for_each_online_pgdat(pgdat) { 7757 7758 pgdat->totalreserve_pages = 0; 7759 7760 for (i = 0; i < MAX_NR_ZONES; i++) { 7761 struct zone *zone = pgdat->node_zones + i; 7762 long max = 0; 7763 unsigned long managed_pages = zone_managed_pages(zone); 7764 7765 /* Find valid and maximum lowmem_reserve in the zone */ 7766 for (j = i; j < MAX_NR_ZONES; j++) { 7767 if (zone->lowmem_reserve[j] > max) 7768 max = zone->lowmem_reserve[j]; 7769 } 7770 7771 /* we treat the high watermark as reserved pages. */ 7772 max += high_wmark_pages(zone); 7773 7774 if (max > managed_pages) 7775 max = managed_pages; 7776 7777 pgdat->totalreserve_pages += max; 7778 7779 reserve_pages += max; 7780 } 7781 } 7782 totalreserve_pages = reserve_pages; 7783 } 7784 7785 /* 7786 * setup_per_zone_lowmem_reserve - called whenever 7787 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 7788 * has a correct pages reserved value, so an adequate number of 7789 * pages are left in the zone after a successful __alloc_pages(). 7790 */ 7791 static void setup_per_zone_lowmem_reserve(void) 7792 { 7793 struct pglist_data *pgdat; 7794 enum zone_type j, idx; 7795 7796 for_each_online_pgdat(pgdat) { 7797 for (j = 0; j < MAX_NR_ZONES; j++) { 7798 struct zone *zone = pgdat->node_zones + j; 7799 unsigned long managed_pages = zone_managed_pages(zone); 7800 7801 zone->lowmem_reserve[j] = 0; 7802 7803 idx = j; 7804 while (idx) { 7805 struct zone *lower_zone; 7806 7807 idx--; 7808 lower_zone = pgdat->node_zones + idx; 7809 7810 if (!sysctl_lowmem_reserve_ratio[idx] || 7811 !zone_managed_pages(lower_zone)) { 7812 lower_zone->lowmem_reserve[j] = 0; 7813 continue; 7814 } else { 7815 lower_zone->lowmem_reserve[j] = 7816 managed_pages / sysctl_lowmem_reserve_ratio[idx]; 7817 } 7818 managed_pages += zone_managed_pages(lower_zone); 7819 } 7820 } 7821 } 7822 7823 /* update totalreserve_pages */ 7824 calculate_totalreserve_pages(); 7825 } 7826 7827 static void __setup_per_zone_wmarks(void) 7828 { 7829 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 7830 unsigned long lowmem_pages = 0; 7831 struct zone *zone; 7832 unsigned long flags; 7833 7834 /* Calculate total number of !ZONE_HIGHMEM pages */ 7835 for_each_zone(zone) { 7836 if (!is_highmem(zone)) 7837 lowmem_pages += zone_managed_pages(zone); 7838 } 7839 7840 for_each_zone(zone) { 7841 u64 tmp; 7842 7843 spin_lock_irqsave(&zone->lock, flags); 7844 tmp = (u64)pages_min * zone_managed_pages(zone); 7845 do_div(tmp, lowmem_pages); 7846 if (is_highmem(zone)) { 7847 /* 7848 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 7849 * need highmem pages, so cap pages_min to a small 7850 * value here. 7851 * 7852 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 7853 * deltas control async page reclaim, and so should 7854 * not be capped for highmem. 7855 */ 7856 unsigned long min_pages; 7857 7858 min_pages = zone_managed_pages(zone) / 1024; 7859 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 7860 zone->_watermark[WMARK_MIN] = min_pages; 7861 } else { 7862 /* 7863 * If it's a lowmem zone, reserve a number of pages 7864 * proportionate to the zone's size. 7865 */ 7866 zone->_watermark[WMARK_MIN] = tmp; 7867 } 7868 7869 /* 7870 * Set the kswapd watermarks distance according to the 7871 * scale factor in proportion to available memory, but 7872 * ensure a minimum size on small systems. 7873 */ 7874 tmp = max_t(u64, tmp >> 2, 7875 mult_frac(zone_managed_pages(zone), 7876 watermark_scale_factor, 10000)); 7877 7878 zone->watermark_boost = 0; 7879 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 7880 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2; 7881 7882 spin_unlock_irqrestore(&zone->lock, flags); 7883 } 7884 7885 /* update totalreserve_pages */ 7886 calculate_totalreserve_pages(); 7887 } 7888 7889 /** 7890 * setup_per_zone_wmarks - called when min_free_kbytes changes 7891 * or when memory is hot-{added|removed} 7892 * 7893 * Ensures that the watermark[min,low,high] values for each zone are set 7894 * correctly with respect to min_free_kbytes. 7895 */ 7896 void setup_per_zone_wmarks(void) 7897 { 7898 static DEFINE_SPINLOCK(lock); 7899 7900 spin_lock(&lock); 7901 __setup_per_zone_wmarks(); 7902 spin_unlock(&lock); 7903 } 7904 7905 /* 7906 * Initialise min_free_kbytes. 7907 * 7908 * For small machines we want it small (128k min). For large machines 7909 * we want it large (256MB max). But it is not linear, because network 7910 * bandwidth does not increase linearly with machine size. We use 7911 * 7912 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 7913 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 7914 * 7915 * which yields 7916 * 7917 * 16MB: 512k 7918 * 32MB: 724k 7919 * 64MB: 1024k 7920 * 128MB: 1448k 7921 * 256MB: 2048k 7922 * 512MB: 2896k 7923 * 1024MB: 4096k 7924 * 2048MB: 5792k 7925 * 4096MB: 8192k 7926 * 8192MB: 11584k 7927 * 16384MB: 16384k 7928 */ 7929 int __meminit init_per_zone_wmark_min(void) 7930 { 7931 unsigned long lowmem_kbytes; 7932 int new_min_free_kbytes; 7933 7934 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 7935 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 7936 7937 if (new_min_free_kbytes > user_min_free_kbytes) { 7938 min_free_kbytes = new_min_free_kbytes; 7939 if (min_free_kbytes < 128) 7940 min_free_kbytes = 128; 7941 if (min_free_kbytes > 262144) 7942 min_free_kbytes = 262144; 7943 } else { 7944 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 7945 new_min_free_kbytes, user_min_free_kbytes); 7946 } 7947 setup_per_zone_wmarks(); 7948 refresh_zone_stat_thresholds(); 7949 setup_per_zone_lowmem_reserve(); 7950 7951 #ifdef CONFIG_NUMA 7952 setup_min_unmapped_ratio(); 7953 setup_min_slab_ratio(); 7954 #endif 7955 7956 khugepaged_min_free_kbytes_update(); 7957 7958 return 0; 7959 } 7960 postcore_initcall(init_per_zone_wmark_min) 7961 7962 /* 7963 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 7964 * that we can call two helper functions whenever min_free_kbytes 7965 * changes. 7966 */ 7967 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 7968 void *buffer, size_t *length, loff_t *ppos) 7969 { 7970 int rc; 7971 7972 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7973 if (rc) 7974 return rc; 7975 7976 if (write) { 7977 user_min_free_kbytes = min_free_kbytes; 7978 setup_per_zone_wmarks(); 7979 } 7980 return 0; 7981 } 7982 7983 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 7984 void *buffer, size_t *length, loff_t *ppos) 7985 { 7986 int rc; 7987 7988 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7989 if (rc) 7990 return rc; 7991 7992 if (write) 7993 setup_per_zone_wmarks(); 7994 7995 return 0; 7996 } 7997 7998 #ifdef CONFIG_NUMA 7999 static void setup_min_unmapped_ratio(void) 8000 { 8001 pg_data_t *pgdat; 8002 struct zone *zone; 8003 8004 for_each_online_pgdat(pgdat) 8005 pgdat->min_unmapped_pages = 0; 8006 8007 for_each_zone(zone) 8008 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * 8009 sysctl_min_unmapped_ratio) / 100; 8010 } 8011 8012 8013 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 8014 void *buffer, size_t *length, loff_t *ppos) 8015 { 8016 int rc; 8017 8018 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8019 if (rc) 8020 return rc; 8021 8022 setup_min_unmapped_ratio(); 8023 8024 return 0; 8025 } 8026 8027 static void setup_min_slab_ratio(void) 8028 { 8029 pg_data_t *pgdat; 8030 struct zone *zone; 8031 8032 for_each_online_pgdat(pgdat) 8033 pgdat->min_slab_pages = 0; 8034 8035 for_each_zone(zone) 8036 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * 8037 sysctl_min_slab_ratio) / 100; 8038 } 8039 8040 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 8041 void *buffer, size_t *length, loff_t *ppos) 8042 { 8043 int rc; 8044 8045 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8046 if (rc) 8047 return rc; 8048 8049 setup_min_slab_ratio(); 8050 8051 return 0; 8052 } 8053 #endif 8054 8055 /* 8056 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 8057 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 8058 * whenever sysctl_lowmem_reserve_ratio changes. 8059 * 8060 * The reserve ratio obviously has absolutely no relation with the 8061 * minimum watermarks. The lowmem reserve ratio can only make sense 8062 * if in function of the boot time zone sizes. 8063 */ 8064 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 8065 void *buffer, size_t *length, loff_t *ppos) 8066 { 8067 int i; 8068 8069 proc_dointvec_minmax(table, write, buffer, length, ppos); 8070 8071 for (i = 0; i < MAX_NR_ZONES; i++) { 8072 if (sysctl_lowmem_reserve_ratio[i] < 1) 8073 sysctl_lowmem_reserve_ratio[i] = 0; 8074 } 8075 8076 setup_per_zone_lowmem_reserve(); 8077 return 0; 8078 } 8079 8080 static void __zone_pcp_update(struct zone *zone) 8081 { 8082 unsigned int cpu; 8083 8084 for_each_possible_cpu(cpu) 8085 pageset_set_high_and_batch(zone, 8086 per_cpu_ptr(zone->pageset, cpu)); 8087 } 8088 8089 /* 8090 * percpu_pagelist_fraction - changes the pcp->high for each zone on each 8091 * cpu. It is the fraction of total pages in each zone that a hot per cpu 8092 * pagelist can have before it gets flushed back to buddy allocator. 8093 */ 8094 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, 8095 void *buffer, size_t *length, loff_t *ppos) 8096 { 8097 struct zone *zone; 8098 int old_percpu_pagelist_fraction; 8099 int ret; 8100 8101 mutex_lock(&pcp_batch_high_lock); 8102 old_percpu_pagelist_fraction = percpu_pagelist_fraction; 8103 8104 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 8105 if (!write || ret < 0) 8106 goto out; 8107 8108 /* Sanity checking to avoid pcp imbalance */ 8109 if (percpu_pagelist_fraction && 8110 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { 8111 percpu_pagelist_fraction = old_percpu_pagelist_fraction; 8112 ret = -EINVAL; 8113 goto out; 8114 } 8115 8116 /* No change? */ 8117 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) 8118 goto out; 8119 8120 for_each_populated_zone(zone) 8121 __zone_pcp_update(zone); 8122 out: 8123 mutex_unlock(&pcp_batch_high_lock); 8124 return ret; 8125 } 8126 8127 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 8128 /* 8129 * Returns the number of pages that arch has reserved but 8130 * is not known to alloc_large_system_hash(). 8131 */ 8132 static unsigned long __init arch_reserved_kernel_pages(void) 8133 { 8134 return 0; 8135 } 8136 #endif 8137 8138 /* 8139 * Adaptive scale is meant to reduce sizes of hash tables on large memory 8140 * machines. As memory size is increased the scale is also increased but at 8141 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 8142 * quadruples the scale is increased by one, which means the size of hash table 8143 * only doubles, instead of quadrupling as well. 8144 * Because 32-bit systems cannot have large physical memory, where this scaling 8145 * makes sense, it is disabled on such platforms. 8146 */ 8147 #if __BITS_PER_LONG > 32 8148 #define ADAPT_SCALE_BASE (64ul << 30) 8149 #define ADAPT_SCALE_SHIFT 2 8150 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 8151 #endif 8152 8153 /* 8154 * allocate a large system hash table from bootmem 8155 * - it is assumed that the hash table must contain an exact power-of-2 8156 * quantity of entries 8157 * - limit is the number of hash buckets, not the total allocation size 8158 */ 8159 void *__init alloc_large_system_hash(const char *tablename, 8160 unsigned long bucketsize, 8161 unsigned long numentries, 8162 int scale, 8163 int flags, 8164 unsigned int *_hash_shift, 8165 unsigned int *_hash_mask, 8166 unsigned long low_limit, 8167 unsigned long high_limit) 8168 { 8169 unsigned long long max = high_limit; 8170 unsigned long log2qty, size; 8171 void *table = NULL; 8172 gfp_t gfp_flags; 8173 bool virt; 8174 8175 /* allow the kernel cmdline to have a say */ 8176 if (!numentries) { 8177 /* round applicable memory size up to nearest megabyte */ 8178 numentries = nr_kernel_pages; 8179 numentries -= arch_reserved_kernel_pages(); 8180 8181 /* It isn't necessary when PAGE_SIZE >= 1MB */ 8182 if (PAGE_SHIFT < 20) 8183 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 8184 8185 #if __BITS_PER_LONG > 32 8186 if (!high_limit) { 8187 unsigned long adapt; 8188 8189 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 8190 adapt <<= ADAPT_SCALE_SHIFT) 8191 scale++; 8192 } 8193 #endif 8194 8195 /* limit to 1 bucket per 2^scale bytes of low memory */ 8196 if (scale > PAGE_SHIFT) 8197 numentries >>= (scale - PAGE_SHIFT); 8198 else 8199 numentries <<= (PAGE_SHIFT - scale); 8200 8201 /* Make sure we've got at least a 0-order allocation.. */ 8202 if (unlikely(flags & HASH_SMALL)) { 8203 /* Makes no sense without HASH_EARLY */ 8204 WARN_ON(!(flags & HASH_EARLY)); 8205 if (!(numentries >> *_hash_shift)) { 8206 numentries = 1UL << *_hash_shift; 8207 BUG_ON(!numentries); 8208 } 8209 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 8210 numentries = PAGE_SIZE / bucketsize; 8211 } 8212 numentries = roundup_pow_of_two(numentries); 8213 8214 /* limit allocation size to 1/16 total memory by default */ 8215 if (max == 0) { 8216 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 8217 do_div(max, bucketsize); 8218 } 8219 max = min(max, 0x80000000ULL); 8220 8221 if (numentries < low_limit) 8222 numentries = low_limit; 8223 if (numentries > max) 8224 numentries = max; 8225 8226 log2qty = ilog2(numentries); 8227 8228 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 8229 do { 8230 virt = false; 8231 size = bucketsize << log2qty; 8232 if (flags & HASH_EARLY) { 8233 if (flags & HASH_ZERO) 8234 table = memblock_alloc(size, SMP_CACHE_BYTES); 8235 else 8236 table = memblock_alloc_raw(size, 8237 SMP_CACHE_BYTES); 8238 } else if (get_order(size) >= MAX_ORDER || hashdist) { 8239 table = __vmalloc(size, gfp_flags); 8240 virt = true; 8241 } else { 8242 /* 8243 * If bucketsize is not a power-of-two, we may free 8244 * some pages at the end of hash table which 8245 * alloc_pages_exact() automatically does 8246 */ 8247 table = alloc_pages_exact(size, gfp_flags); 8248 kmemleak_alloc(table, size, 1, gfp_flags); 8249 } 8250 } while (!table && size > PAGE_SIZE && --log2qty); 8251 8252 if (!table) 8253 panic("Failed to allocate %s hash table\n", tablename); 8254 8255 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 8256 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, 8257 virt ? "vmalloc" : "linear"); 8258 8259 if (_hash_shift) 8260 *_hash_shift = log2qty; 8261 if (_hash_mask) 8262 *_hash_mask = (1 << log2qty) - 1; 8263 8264 return table; 8265 } 8266 8267 /* 8268 * This function checks whether pageblock includes unmovable pages or not. 8269 * 8270 * PageLRU check without isolation or lru_lock could race so that 8271 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable 8272 * check without lock_page also may miss some movable non-lru pages at 8273 * race condition. So you can't expect this function should be exact. 8274 * 8275 * Returns a page without holding a reference. If the caller wants to 8276 * dereference that page (e.g., dumping), it has to make sure that it 8277 * cannot get removed (e.g., via memory unplug) concurrently. 8278 * 8279 */ 8280 struct page *has_unmovable_pages(struct zone *zone, struct page *page, 8281 int migratetype, int flags) 8282 { 8283 unsigned long iter = 0; 8284 unsigned long pfn = page_to_pfn(page); 8285 unsigned long offset = pfn % pageblock_nr_pages; 8286 8287 if (is_migrate_cma_page(page)) { 8288 /* 8289 * CMA allocations (alloc_contig_range) really need to mark 8290 * isolate CMA pageblocks even when they are not movable in fact 8291 * so consider them movable here. 8292 */ 8293 if (is_migrate_cma(migratetype)) 8294 return NULL; 8295 8296 return page; 8297 } 8298 8299 for (; iter < pageblock_nr_pages - offset; iter++) { 8300 if (!pfn_valid_within(pfn + iter)) 8301 continue; 8302 8303 page = pfn_to_page(pfn + iter); 8304 8305 /* 8306 * Both, bootmem allocations and memory holes are marked 8307 * PG_reserved and are unmovable. We can even have unmovable 8308 * allocations inside ZONE_MOVABLE, for example when 8309 * specifying "movablecore". 8310 */ 8311 if (PageReserved(page)) 8312 return page; 8313 8314 /* 8315 * If the zone is movable and we have ruled out all reserved 8316 * pages then it should be reasonably safe to assume the rest 8317 * is movable. 8318 */ 8319 if (zone_idx(zone) == ZONE_MOVABLE) 8320 continue; 8321 8322 /* 8323 * Hugepages are not in LRU lists, but they're movable. 8324 * THPs are on the LRU, but need to be counted as #small pages. 8325 * We need not scan over tail pages because we don't 8326 * handle each tail page individually in migration. 8327 */ 8328 if (PageHuge(page) || PageTransCompound(page)) { 8329 struct page *head = compound_head(page); 8330 unsigned int skip_pages; 8331 8332 if (PageHuge(page)) { 8333 if (!hugepage_migration_supported(page_hstate(head))) 8334 return page; 8335 } else if (!PageLRU(head) && !__PageMovable(head)) { 8336 return page; 8337 } 8338 8339 skip_pages = compound_nr(head) - (page - head); 8340 iter += skip_pages - 1; 8341 continue; 8342 } 8343 8344 /* 8345 * We can't use page_count without pin a page 8346 * because another CPU can free compound page. 8347 * This check already skips compound tails of THP 8348 * because their page->_refcount is zero at all time. 8349 */ 8350 if (!page_ref_count(page)) { 8351 if (PageBuddy(page)) 8352 iter += (1 << buddy_order(page)) - 1; 8353 continue; 8354 } 8355 8356 /* 8357 * The HWPoisoned page may be not in buddy system, and 8358 * page_count() is not 0. 8359 */ 8360 if ((flags & MEMORY_OFFLINE) && PageHWPoison(page)) 8361 continue; 8362 8363 /* 8364 * We treat all PageOffline() pages as movable when offlining 8365 * to give drivers a chance to decrement their reference count 8366 * in MEM_GOING_OFFLINE in order to indicate that these pages 8367 * can be offlined as there are no direct references anymore. 8368 * For actually unmovable PageOffline() where the driver does 8369 * not support this, we will fail later when trying to actually 8370 * move these pages that still have a reference count > 0. 8371 * (false negatives in this function only) 8372 */ 8373 if ((flags & MEMORY_OFFLINE) && PageOffline(page)) 8374 continue; 8375 8376 if (__PageMovable(page) || PageLRU(page)) 8377 continue; 8378 8379 /* 8380 * If there are RECLAIMABLE pages, we need to check 8381 * it. But now, memory offline itself doesn't call 8382 * shrink_node_slabs() and it still to be fixed. 8383 */ 8384 return page; 8385 } 8386 return NULL; 8387 } 8388 8389 #ifdef CONFIG_CONTIG_ALLOC 8390 static unsigned long pfn_max_align_down(unsigned long pfn) 8391 { 8392 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, 8393 pageblock_nr_pages) - 1); 8394 } 8395 8396 static unsigned long pfn_max_align_up(unsigned long pfn) 8397 { 8398 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, 8399 pageblock_nr_pages)); 8400 } 8401 8402 /* [start, end) must belong to a single zone. */ 8403 static int __alloc_contig_migrate_range(struct compact_control *cc, 8404 unsigned long start, unsigned long end) 8405 { 8406 /* This function is based on compact_zone() from compaction.c. */ 8407 unsigned int nr_reclaimed; 8408 unsigned long pfn = start; 8409 unsigned int tries = 0; 8410 int ret = 0; 8411 struct migration_target_control mtc = { 8412 .nid = zone_to_nid(cc->zone), 8413 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, 8414 }; 8415 8416 migrate_prep(); 8417 8418 while (pfn < end || !list_empty(&cc->migratepages)) { 8419 if (fatal_signal_pending(current)) { 8420 ret = -EINTR; 8421 break; 8422 } 8423 8424 if (list_empty(&cc->migratepages)) { 8425 cc->nr_migratepages = 0; 8426 pfn = isolate_migratepages_range(cc, pfn, end); 8427 if (!pfn) { 8428 ret = -EINTR; 8429 break; 8430 } 8431 tries = 0; 8432 } else if (++tries == 5) { 8433 ret = ret < 0 ? ret : -EBUSY; 8434 break; 8435 } 8436 8437 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 8438 &cc->migratepages); 8439 cc->nr_migratepages -= nr_reclaimed; 8440 8441 ret = migrate_pages(&cc->migratepages, alloc_migration_target, 8442 NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE); 8443 } 8444 if (ret < 0) { 8445 putback_movable_pages(&cc->migratepages); 8446 return ret; 8447 } 8448 return 0; 8449 } 8450 8451 /** 8452 * alloc_contig_range() -- tries to allocate given range of pages 8453 * @start: start PFN to allocate 8454 * @end: one-past-the-last PFN to allocate 8455 * @migratetype: migratetype of the underlaying pageblocks (either 8456 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 8457 * in range must have the same migratetype and it must 8458 * be either of the two. 8459 * @gfp_mask: GFP mask to use during compaction 8460 * 8461 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 8462 * aligned. The PFN range must belong to a single zone. 8463 * 8464 * The first thing this routine does is attempt to MIGRATE_ISOLATE all 8465 * pageblocks in the range. Once isolated, the pageblocks should not 8466 * be modified by others. 8467 * 8468 * Return: zero on success or negative error code. On success all 8469 * pages which PFN is in [start, end) are allocated for the caller and 8470 * need to be freed with free_contig_range(). 8471 */ 8472 int alloc_contig_range(unsigned long start, unsigned long end, 8473 unsigned migratetype, gfp_t gfp_mask) 8474 { 8475 unsigned long outer_start, outer_end; 8476 unsigned int order; 8477 int ret = 0; 8478 8479 struct compact_control cc = { 8480 .nr_migratepages = 0, 8481 .order = -1, 8482 .zone = page_zone(pfn_to_page(start)), 8483 .mode = MIGRATE_SYNC, 8484 .ignore_skip_hint = true, 8485 .no_set_skip_hint = true, 8486 .gfp_mask = current_gfp_context(gfp_mask), 8487 .alloc_contig = true, 8488 }; 8489 INIT_LIST_HEAD(&cc.migratepages); 8490 8491 /* 8492 * What we do here is we mark all pageblocks in range as 8493 * MIGRATE_ISOLATE. Because pageblock and max order pages may 8494 * have different sizes, and due to the way page allocator 8495 * work, we align the range to biggest of the two pages so 8496 * that page allocator won't try to merge buddies from 8497 * different pageblocks and change MIGRATE_ISOLATE to some 8498 * other migration type. 8499 * 8500 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 8501 * migrate the pages from an unaligned range (ie. pages that 8502 * we are interested in). This will put all the pages in 8503 * range back to page allocator as MIGRATE_ISOLATE. 8504 * 8505 * When this is done, we take the pages in range from page 8506 * allocator removing them from the buddy system. This way 8507 * page allocator will never consider using them. 8508 * 8509 * This lets us mark the pageblocks back as 8510 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 8511 * aligned range but not in the unaligned, original range are 8512 * put back to page allocator so that buddy can use them. 8513 */ 8514 8515 ret = start_isolate_page_range(pfn_max_align_down(start), 8516 pfn_max_align_up(end), migratetype, 0); 8517 if (ret) 8518 return ret; 8519 8520 /* 8521 * In case of -EBUSY, we'd like to know which page causes problem. 8522 * So, just fall through. test_pages_isolated() has a tracepoint 8523 * which will report the busy page. 8524 * 8525 * It is possible that busy pages could become available before 8526 * the call to test_pages_isolated, and the range will actually be 8527 * allocated. So, if we fall through be sure to clear ret so that 8528 * -EBUSY is not accidentally used or returned to caller. 8529 */ 8530 ret = __alloc_contig_migrate_range(&cc, start, end); 8531 if (ret && ret != -EBUSY) 8532 goto done; 8533 ret =0; 8534 8535 /* 8536 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 8537 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 8538 * more, all pages in [start, end) are free in page allocator. 8539 * What we are going to do is to allocate all pages from 8540 * [start, end) (that is remove them from page allocator). 8541 * 8542 * The only problem is that pages at the beginning and at the 8543 * end of interesting range may be not aligned with pages that 8544 * page allocator holds, ie. they can be part of higher order 8545 * pages. Because of this, we reserve the bigger range and 8546 * once this is done free the pages we are not interested in. 8547 * 8548 * We don't have to hold zone->lock here because the pages are 8549 * isolated thus they won't get removed from buddy. 8550 */ 8551 8552 lru_add_drain_all(); 8553 8554 order = 0; 8555 outer_start = start; 8556 while (!PageBuddy(pfn_to_page(outer_start))) { 8557 if (++order >= MAX_ORDER) { 8558 outer_start = start; 8559 break; 8560 } 8561 outer_start &= ~0UL << order; 8562 } 8563 8564 if (outer_start != start) { 8565 order = buddy_order(pfn_to_page(outer_start)); 8566 8567 /* 8568 * outer_start page could be small order buddy page and 8569 * it doesn't include start page. Adjust outer_start 8570 * in this case to report failed page properly 8571 * on tracepoint in test_pages_isolated() 8572 */ 8573 if (outer_start + (1UL << order) <= start) 8574 outer_start = start; 8575 } 8576 8577 /* Make sure the range is really isolated. */ 8578 if (test_pages_isolated(outer_start, end, 0)) { 8579 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n", 8580 __func__, outer_start, end); 8581 ret = -EBUSY; 8582 goto done; 8583 } 8584 8585 /* Grab isolated pages from freelists. */ 8586 outer_end = isolate_freepages_range(&cc, outer_start, end); 8587 if (!outer_end) { 8588 ret = -EBUSY; 8589 goto done; 8590 } 8591 8592 /* Free head and tail (if any) */ 8593 if (start != outer_start) 8594 free_contig_range(outer_start, start - outer_start); 8595 if (end != outer_end) 8596 free_contig_range(end, outer_end - end); 8597 8598 done: 8599 undo_isolate_page_range(pfn_max_align_down(start), 8600 pfn_max_align_up(end), migratetype); 8601 return ret; 8602 } 8603 EXPORT_SYMBOL(alloc_contig_range); 8604 8605 static int __alloc_contig_pages(unsigned long start_pfn, 8606 unsigned long nr_pages, gfp_t gfp_mask) 8607 { 8608 unsigned long end_pfn = start_pfn + nr_pages; 8609 8610 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE, 8611 gfp_mask); 8612 } 8613 8614 static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, 8615 unsigned long nr_pages) 8616 { 8617 unsigned long i, end_pfn = start_pfn + nr_pages; 8618 struct page *page; 8619 8620 for (i = start_pfn; i < end_pfn; i++) { 8621 page = pfn_to_online_page(i); 8622 if (!page) 8623 return false; 8624 8625 if (page_zone(page) != z) 8626 return false; 8627 8628 if (PageReserved(page)) 8629 return false; 8630 8631 if (page_count(page) > 0) 8632 return false; 8633 8634 if (PageHuge(page)) 8635 return false; 8636 } 8637 return true; 8638 } 8639 8640 static bool zone_spans_last_pfn(const struct zone *zone, 8641 unsigned long start_pfn, unsigned long nr_pages) 8642 { 8643 unsigned long last_pfn = start_pfn + nr_pages - 1; 8644 8645 return zone_spans_pfn(zone, last_pfn); 8646 } 8647 8648 /** 8649 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages 8650 * @nr_pages: Number of contiguous pages to allocate 8651 * @gfp_mask: GFP mask to limit search and used during compaction 8652 * @nid: Target node 8653 * @nodemask: Mask for other possible nodes 8654 * 8655 * This routine is a wrapper around alloc_contig_range(). It scans over zones 8656 * on an applicable zonelist to find a contiguous pfn range which can then be 8657 * tried for allocation with alloc_contig_range(). This routine is intended 8658 * for allocation requests which can not be fulfilled with the buddy allocator. 8659 * 8660 * The allocated memory is always aligned to a page boundary. If nr_pages is a 8661 * power of two then the alignment is guaranteed to be to the given nr_pages 8662 * (e.g. 1GB request would be aligned to 1GB). 8663 * 8664 * Allocated pages can be freed with free_contig_range() or by manually calling 8665 * __free_page() on each allocated page. 8666 * 8667 * Return: pointer to contiguous pages on success, or NULL if not successful. 8668 */ 8669 struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, 8670 int nid, nodemask_t *nodemask) 8671 { 8672 unsigned long ret, pfn, flags; 8673 struct zonelist *zonelist; 8674 struct zone *zone; 8675 struct zoneref *z; 8676 8677 zonelist = node_zonelist(nid, gfp_mask); 8678 for_each_zone_zonelist_nodemask(zone, z, zonelist, 8679 gfp_zone(gfp_mask), nodemask) { 8680 spin_lock_irqsave(&zone->lock, flags); 8681 8682 pfn = ALIGN(zone->zone_start_pfn, nr_pages); 8683 while (zone_spans_last_pfn(zone, pfn, nr_pages)) { 8684 if (pfn_range_valid_contig(zone, pfn, nr_pages)) { 8685 /* 8686 * We release the zone lock here because 8687 * alloc_contig_range() will also lock the zone 8688 * at some point. If there's an allocation 8689 * spinning on this lock, it may win the race 8690 * and cause alloc_contig_range() to fail... 8691 */ 8692 spin_unlock_irqrestore(&zone->lock, flags); 8693 ret = __alloc_contig_pages(pfn, nr_pages, 8694 gfp_mask); 8695 if (!ret) 8696 return pfn_to_page(pfn); 8697 spin_lock_irqsave(&zone->lock, flags); 8698 } 8699 pfn += nr_pages; 8700 } 8701 spin_unlock_irqrestore(&zone->lock, flags); 8702 } 8703 return NULL; 8704 } 8705 #endif /* CONFIG_CONTIG_ALLOC */ 8706 8707 void free_contig_range(unsigned long pfn, unsigned int nr_pages) 8708 { 8709 unsigned int count = 0; 8710 8711 for (; nr_pages--; pfn++) { 8712 struct page *page = pfn_to_page(pfn); 8713 8714 count += page_count(page) != 1; 8715 __free_page(page); 8716 } 8717 WARN(count != 0, "%d pages are still in use!\n", count); 8718 } 8719 EXPORT_SYMBOL(free_contig_range); 8720 8721 /* 8722 * The zone indicated has a new number of managed_pages; batch sizes and percpu 8723 * page high values need to be recalulated. 8724 */ 8725 void __meminit zone_pcp_update(struct zone *zone) 8726 { 8727 mutex_lock(&pcp_batch_high_lock); 8728 __zone_pcp_update(zone); 8729 mutex_unlock(&pcp_batch_high_lock); 8730 } 8731 8732 void zone_pcp_reset(struct zone *zone) 8733 { 8734 unsigned long flags; 8735 int cpu; 8736 struct per_cpu_pageset *pset; 8737 8738 /* avoid races with drain_pages() */ 8739 local_irq_save(flags); 8740 if (zone->pageset != &boot_pageset) { 8741 for_each_online_cpu(cpu) { 8742 pset = per_cpu_ptr(zone->pageset, cpu); 8743 drain_zonestat(zone, pset); 8744 } 8745 free_percpu(zone->pageset); 8746 zone->pageset = &boot_pageset; 8747 } 8748 local_irq_restore(flags); 8749 } 8750 8751 #ifdef CONFIG_MEMORY_HOTREMOVE 8752 /* 8753 * All pages in the range must be in a single zone, must not contain holes, 8754 * must span full sections, and must be isolated before calling this function. 8755 */ 8756 void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 8757 { 8758 unsigned long pfn = start_pfn; 8759 struct page *page; 8760 struct zone *zone; 8761 unsigned int order; 8762 unsigned long flags; 8763 8764 offline_mem_sections(pfn, end_pfn); 8765 zone = page_zone(pfn_to_page(pfn)); 8766 spin_lock_irqsave(&zone->lock, flags); 8767 while (pfn < end_pfn) { 8768 page = pfn_to_page(pfn); 8769 /* 8770 * The HWPoisoned page may be not in buddy system, and 8771 * page_count() is not 0. 8772 */ 8773 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 8774 pfn++; 8775 continue; 8776 } 8777 /* 8778 * At this point all remaining PageOffline() pages have a 8779 * reference count of 0 and can simply be skipped. 8780 */ 8781 if (PageOffline(page)) { 8782 BUG_ON(page_count(page)); 8783 BUG_ON(PageBuddy(page)); 8784 pfn++; 8785 continue; 8786 } 8787 8788 BUG_ON(page_count(page)); 8789 BUG_ON(!PageBuddy(page)); 8790 order = buddy_order(page); 8791 del_page_from_free_list(page, zone, order); 8792 pfn += (1 << order); 8793 } 8794 spin_unlock_irqrestore(&zone->lock, flags); 8795 } 8796 #endif 8797 8798 bool is_free_buddy_page(struct page *page) 8799 { 8800 struct zone *zone = page_zone(page); 8801 unsigned long pfn = page_to_pfn(page); 8802 unsigned long flags; 8803 unsigned int order; 8804 8805 spin_lock_irqsave(&zone->lock, flags); 8806 for (order = 0; order < MAX_ORDER; order++) { 8807 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8808 8809 if (PageBuddy(page_head) && buddy_order(page_head) >= order) 8810 break; 8811 } 8812 spin_unlock_irqrestore(&zone->lock, flags); 8813 8814 return order < MAX_ORDER; 8815 } 8816 8817 #ifdef CONFIG_MEMORY_FAILURE 8818 /* 8819 * Break down a higher-order page in sub-pages, and keep our target out of 8820 * buddy allocator. 8821 */ 8822 static void break_down_buddy_pages(struct zone *zone, struct page *page, 8823 struct page *target, int low, int high, 8824 int migratetype) 8825 { 8826 unsigned long size = 1 << high; 8827 struct page *current_buddy, *next_page; 8828 8829 while (high > low) { 8830 high--; 8831 size >>= 1; 8832 8833 if (target >= &page[size]) { 8834 next_page = page + size; 8835 current_buddy = page; 8836 } else { 8837 next_page = page; 8838 current_buddy = page + size; 8839 } 8840 8841 if (set_page_guard(zone, current_buddy, high, migratetype)) 8842 continue; 8843 8844 if (current_buddy != target) { 8845 add_to_free_list(current_buddy, zone, high, migratetype); 8846 set_buddy_order(current_buddy, high); 8847 page = next_page; 8848 } 8849 } 8850 } 8851 8852 /* 8853 * Take a page that will be marked as poisoned off the buddy allocator. 8854 */ 8855 bool take_page_off_buddy(struct page *page) 8856 { 8857 struct zone *zone = page_zone(page); 8858 unsigned long pfn = page_to_pfn(page); 8859 unsigned long flags; 8860 unsigned int order; 8861 bool ret = false; 8862 8863 spin_lock_irqsave(&zone->lock, flags); 8864 for (order = 0; order < MAX_ORDER; order++) { 8865 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8866 int page_order = buddy_order(page_head); 8867 8868 if (PageBuddy(page_head) && page_order >= order) { 8869 unsigned long pfn_head = page_to_pfn(page_head); 8870 int migratetype = get_pfnblock_migratetype(page_head, 8871 pfn_head); 8872 8873 del_page_from_free_list(page_head, zone, page_order); 8874 break_down_buddy_pages(zone, page_head, page, 0, 8875 page_order, migratetype); 8876 ret = true; 8877 break; 8878 } 8879 if (page_count(page_head) > 0) 8880 break; 8881 } 8882 spin_unlock_irqrestore(&zone->lock, flags); 8883 return ret; 8884 } 8885 #endif 8886