1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/page_alloc.c 4 * 5 * Manages the free list, the system allocates free pages here. 6 * Note that kmalloc() lives in slab.c 7 * 8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 9 * Swap reorganised 29.12.95, Stephen Tweedie 10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 16 */ 17 18 #include <linux/stddef.h> 19 #include <linux/mm.h> 20 #include <linux/highmem.h> 21 #include <linux/swap.h> 22 #include <linux/interrupt.h> 23 #include <linux/pagemap.h> 24 #include <linux/jiffies.h> 25 #include <linux/memblock.h> 26 #include <linux/compiler.h> 27 #include <linux/kernel.h> 28 #include <linux/kasan.h> 29 #include <linux/module.h> 30 #include <linux/suspend.h> 31 #include <linux/pagevec.h> 32 #include <linux/blkdev.h> 33 #include <linux/slab.h> 34 #include <linux/ratelimit.h> 35 #include <linux/oom.h> 36 #include <linux/topology.h> 37 #include <linux/sysctl.h> 38 #include <linux/cpu.h> 39 #include <linux/cpuset.h> 40 #include <linux/memory_hotplug.h> 41 #include <linux/nodemask.h> 42 #include <linux/vmalloc.h> 43 #include <linux/vmstat.h> 44 #include <linux/mempolicy.h> 45 #include <linux/memremap.h> 46 #include <linux/stop_machine.h> 47 #include <linux/random.h> 48 #include <linux/sort.h> 49 #include <linux/pfn.h> 50 #include <linux/backing-dev.h> 51 #include <linux/fault-inject.h> 52 #include <linux/page-isolation.h> 53 #include <linux/debugobjects.h> 54 #include <linux/kmemleak.h> 55 #include <linux/compaction.h> 56 #include <trace/events/kmem.h> 57 #include <trace/events/oom.h> 58 #include <linux/prefetch.h> 59 #include <linux/mm_inline.h> 60 #include <linux/mmu_notifier.h> 61 #include <linux/migrate.h> 62 #include <linux/hugetlb.h> 63 #include <linux/sched/rt.h> 64 #include <linux/sched/mm.h> 65 #include <linux/page_owner.h> 66 #include <linux/kthread.h> 67 #include <linux/memcontrol.h> 68 #include <linux/ftrace.h> 69 #include <linux/lockdep.h> 70 #include <linux/nmi.h> 71 #include <linux/psi.h> 72 #include <linux/padata.h> 73 #include <linux/khugepaged.h> 74 #include <linux/buffer_head.h> 75 #include <asm/sections.h> 76 #include <asm/tlbflush.h> 77 #include <asm/div64.h> 78 #include "internal.h" 79 #include "shuffle.h" 80 #include "page_reporting.h" 81 82 /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */ 83 typedef int __bitwise fpi_t; 84 85 /* No special request */ 86 #define FPI_NONE ((__force fpi_t)0) 87 88 /* 89 * Skip free page reporting notification for the (possibly merged) page. 90 * This does not hinder free page reporting from grabbing the page, 91 * reporting it and marking it "reported" - it only skips notifying 92 * the free page reporting infrastructure about a newly freed page. For 93 * example, used when temporarily pulling a page from a freelist and 94 * putting it back unmodified. 95 */ 96 #define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0)) 97 98 /* 99 * Place the (possibly merged) page to the tail of the freelist. Will ignore 100 * page shuffling (relevant code - e.g., memory onlining - is expected to 101 * shuffle the whole zone). 102 * 103 * Note: No code should rely on this flag for correctness - it's purely 104 * to allow for optimizations when handing back either fresh pages 105 * (memory onlining) or untouched pages (page isolation, free page 106 * reporting). 107 */ 108 #define FPI_TO_TAIL ((__force fpi_t)BIT(1)) 109 110 /* 111 * Don't poison memory with KASAN (only for the tag-based modes). 112 * During boot, all non-reserved memblock memory is exposed to page_alloc. 113 * Poisoning all that memory lengthens boot time, especially on systems with 114 * large amount of RAM. This flag is used to skip that poisoning. 115 * This is only done for the tag-based KASAN modes, as those are able to 116 * detect memory corruptions with the memory tags assigned by default. 117 * All memory allocated normally after boot gets poisoned as usual. 118 */ 119 #define FPI_SKIP_KASAN_POISON ((__force fpi_t)BIT(2)) 120 121 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 122 static DEFINE_MUTEX(pcp_batch_high_lock); 123 #define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8) 124 125 struct pagesets { 126 local_lock_t lock; 127 }; 128 static DEFINE_PER_CPU(struct pagesets, pagesets) = { 129 .lock = INIT_LOCAL_LOCK(lock), 130 }; 131 132 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 133 DEFINE_PER_CPU(int, numa_node); 134 EXPORT_PER_CPU_SYMBOL(numa_node); 135 #endif 136 137 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); 138 139 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 140 /* 141 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 142 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 143 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 144 * defined in <linux/topology.h>. 145 */ 146 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 147 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 148 #endif 149 150 /* work_structs for global per-cpu drains */ 151 struct pcpu_drain { 152 struct zone *zone; 153 struct work_struct work; 154 }; 155 static DEFINE_MUTEX(pcpu_drain_mutex); 156 static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain); 157 158 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 159 volatile unsigned long latent_entropy __latent_entropy; 160 EXPORT_SYMBOL(latent_entropy); 161 #endif 162 163 /* 164 * Array of node states. 165 */ 166 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 167 [N_POSSIBLE] = NODE_MASK_ALL, 168 [N_ONLINE] = { { [0] = 1UL } }, 169 #ifndef CONFIG_NUMA 170 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 171 #ifdef CONFIG_HIGHMEM 172 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 173 #endif 174 [N_MEMORY] = { { [0] = 1UL } }, 175 [N_CPU] = { { [0] = 1UL } }, 176 #endif /* NUMA */ 177 }; 178 EXPORT_SYMBOL(node_states); 179 180 atomic_long_t _totalram_pages __read_mostly; 181 EXPORT_SYMBOL(_totalram_pages); 182 unsigned long totalreserve_pages __read_mostly; 183 unsigned long totalcma_pages __read_mostly; 184 185 int percpu_pagelist_high_fraction; 186 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 187 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); 188 EXPORT_SYMBOL(init_on_alloc); 189 190 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); 191 EXPORT_SYMBOL(init_on_free); 192 193 static bool _init_on_alloc_enabled_early __read_mostly 194 = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON); 195 static int __init early_init_on_alloc(char *buf) 196 { 197 198 return kstrtobool(buf, &_init_on_alloc_enabled_early); 199 } 200 early_param("init_on_alloc", early_init_on_alloc); 201 202 static bool _init_on_free_enabled_early __read_mostly 203 = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON); 204 static int __init early_init_on_free(char *buf) 205 { 206 return kstrtobool(buf, &_init_on_free_enabled_early); 207 } 208 early_param("init_on_free", early_init_on_free); 209 210 /* 211 * A cached value of the page's pageblock's migratetype, used when the page is 212 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 213 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 214 * Also the migratetype set in the page does not necessarily match the pcplist 215 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 216 * other index - this ensures that it will be put on the correct CMA freelist. 217 */ 218 static inline int get_pcppage_migratetype(struct page *page) 219 { 220 return page->index; 221 } 222 223 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 224 { 225 page->index = migratetype; 226 } 227 228 #ifdef CONFIG_PM_SLEEP 229 /* 230 * The following functions are used by the suspend/hibernate code to temporarily 231 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 232 * while devices are suspended. To avoid races with the suspend/hibernate code, 233 * they should always be called with system_transition_mutex held 234 * (gfp_allowed_mask also should only be modified with system_transition_mutex 235 * held, unless the suspend/hibernate code is guaranteed not to run in parallel 236 * with that modification). 237 */ 238 239 static gfp_t saved_gfp_mask; 240 241 void pm_restore_gfp_mask(void) 242 { 243 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 244 if (saved_gfp_mask) { 245 gfp_allowed_mask = saved_gfp_mask; 246 saved_gfp_mask = 0; 247 } 248 } 249 250 void pm_restrict_gfp_mask(void) 251 { 252 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 253 WARN_ON(saved_gfp_mask); 254 saved_gfp_mask = gfp_allowed_mask; 255 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 256 } 257 258 bool pm_suspended_storage(void) 259 { 260 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 261 return false; 262 return true; 263 } 264 #endif /* CONFIG_PM_SLEEP */ 265 266 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 267 unsigned int pageblock_order __read_mostly; 268 #endif 269 270 static void __free_pages_ok(struct page *page, unsigned int order, 271 fpi_t fpi_flags); 272 273 /* 274 * results with 256, 32 in the lowmem_reserve sysctl: 275 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 276 * 1G machine -> (16M dma, 784M normal, 224M high) 277 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 278 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 279 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 280 * 281 * TBD: should special case ZONE_DMA32 machines here - in those we normally 282 * don't need any ZONE_NORMAL reservation 283 */ 284 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { 285 #ifdef CONFIG_ZONE_DMA 286 [ZONE_DMA] = 256, 287 #endif 288 #ifdef CONFIG_ZONE_DMA32 289 [ZONE_DMA32] = 256, 290 #endif 291 [ZONE_NORMAL] = 32, 292 #ifdef CONFIG_HIGHMEM 293 [ZONE_HIGHMEM] = 0, 294 #endif 295 [ZONE_MOVABLE] = 0, 296 }; 297 298 static char * const zone_names[MAX_NR_ZONES] = { 299 #ifdef CONFIG_ZONE_DMA 300 "DMA", 301 #endif 302 #ifdef CONFIG_ZONE_DMA32 303 "DMA32", 304 #endif 305 "Normal", 306 #ifdef CONFIG_HIGHMEM 307 "HighMem", 308 #endif 309 "Movable", 310 #ifdef CONFIG_ZONE_DEVICE 311 "Device", 312 #endif 313 }; 314 315 const char * const migratetype_names[MIGRATE_TYPES] = { 316 "Unmovable", 317 "Movable", 318 "Reclaimable", 319 "HighAtomic", 320 #ifdef CONFIG_CMA 321 "CMA", 322 #endif 323 #ifdef CONFIG_MEMORY_ISOLATION 324 "Isolate", 325 #endif 326 }; 327 328 compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = { 329 [NULL_COMPOUND_DTOR] = NULL, 330 [COMPOUND_PAGE_DTOR] = free_compound_page, 331 #ifdef CONFIG_HUGETLB_PAGE 332 [HUGETLB_PAGE_DTOR] = free_huge_page, 333 #endif 334 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 335 [TRANSHUGE_PAGE_DTOR] = free_transhuge_page, 336 #endif 337 }; 338 339 int min_free_kbytes = 1024; 340 int user_min_free_kbytes = -1; 341 int watermark_boost_factor __read_mostly = 15000; 342 int watermark_scale_factor = 10; 343 344 static unsigned long nr_kernel_pages __initdata; 345 static unsigned long nr_all_pages __initdata; 346 static unsigned long dma_reserve __initdata; 347 348 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 349 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 350 static unsigned long required_kernelcore __initdata; 351 static unsigned long required_kernelcore_percent __initdata; 352 static unsigned long required_movablecore __initdata; 353 static unsigned long required_movablecore_percent __initdata; 354 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 355 static bool mirrored_kernelcore __meminitdata; 356 357 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 358 int movable_zone; 359 EXPORT_SYMBOL(movable_zone); 360 361 #if MAX_NUMNODES > 1 362 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; 363 unsigned int nr_online_nodes __read_mostly = 1; 364 EXPORT_SYMBOL(nr_node_ids); 365 EXPORT_SYMBOL(nr_online_nodes); 366 #endif 367 368 int page_group_by_mobility_disabled __read_mostly; 369 370 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 371 /* 372 * During boot we initialize deferred pages on-demand, as needed, but once 373 * page_alloc_init_late() has finished, the deferred pages are all initialized, 374 * and we can permanently disable that path. 375 */ 376 static DEFINE_STATIC_KEY_TRUE(deferred_pages); 377 378 /* 379 * Calling kasan_poison_pages() only after deferred memory initialization 380 * has completed. Poisoning pages during deferred memory init will greatly 381 * lengthen the process and cause problem in large memory systems as the 382 * deferred pages initialization is done with interrupt disabled. 383 * 384 * Assuming that there will be no reference to those newly initialized 385 * pages before they are ever allocated, this should have no effect on 386 * KASAN memory tracking as the poison will be properly inserted at page 387 * allocation time. The only corner case is when pages are allocated by 388 * on-demand allocation and then freed again before the deferred pages 389 * initialization is done, but this is not likely to happen. 390 */ 391 static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags) 392 { 393 return static_branch_unlikely(&deferred_pages) || 394 (!IS_ENABLED(CONFIG_KASAN_GENERIC) && 395 (fpi_flags & FPI_SKIP_KASAN_POISON)) || 396 PageSkipKASanPoison(page); 397 } 398 399 /* Returns true if the struct page for the pfn is uninitialised */ 400 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 401 { 402 int nid = early_pfn_to_nid(pfn); 403 404 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 405 return true; 406 407 return false; 408 } 409 410 /* 411 * Returns true when the remaining initialisation should be deferred until 412 * later in the boot cycle when it can be parallelised. 413 */ 414 static bool __meminit 415 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 416 { 417 static unsigned long prev_end_pfn, nr_initialised; 418 419 /* 420 * prev_end_pfn static that contains the end of previous zone 421 * No need to protect because called very early in boot before smp_init. 422 */ 423 if (prev_end_pfn != end_pfn) { 424 prev_end_pfn = end_pfn; 425 nr_initialised = 0; 426 } 427 428 /* Always populate low zones for address-constrained allocations */ 429 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 430 return false; 431 432 if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) 433 return true; 434 /* 435 * We start only with one section of pages, more pages are added as 436 * needed until the rest of deferred pages are initialized. 437 */ 438 nr_initialised++; 439 if ((nr_initialised > PAGES_PER_SECTION) && 440 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 441 NODE_DATA(nid)->first_deferred_pfn = pfn; 442 return true; 443 } 444 return false; 445 } 446 #else 447 static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags) 448 { 449 return (!IS_ENABLED(CONFIG_KASAN_GENERIC) && 450 (fpi_flags & FPI_SKIP_KASAN_POISON)) || 451 PageSkipKASanPoison(page); 452 } 453 454 static inline bool early_page_uninitialised(unsigned long pfn) 455 { 456 return false; 457 } 458 459 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 460 { 461 return false; 462 } 463 #endif 464 465 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 466 static inline unsigned long *get_pageblock_bitmap(const struct page *page, 467 unsigned long pfn) 468 { 469 #ifdef CONFIG_SPARSEMEM 470 return section_to_usemap(__pfn_to_section(pfn)); 471 #else 472 return page_zone(page)->pageblock_flags; 473 #endif /* CONFIG_SPARSEMEM */ 474 } 475 476 static inline int pfn_to_bitidx(const struct page *page, unsigned long pfn) 477 { 478 #ifdef CONFIG_SPARSEMEM 479 pfn &= (PAGES_PER_SECTION-1); 480 #else 481 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 482 #endif /* CONFIG_SPARSEMEM */ 483 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 484 } 485 486 static __always_inline 487 unsigned long __get_pfnblock_flags_mask(const struct page *page, 488 unsigned long pfn, 489 unsigned long mask) 490 { 491 unsigned long *bitmap; 492 unsigned long bitidx, word_bitidx; 493 unsigned long word; 494 495 bitmap = get_pageblock_bitmap(page, pfn); 496 bitidx = pfn_to_bitidx(page, pfn); 497 word_bitidx = bitidx / BITS_PER_LONG; 498 bitidx &= (BITS_PER_LONG-1); 499 500 word = bitmap[word_bitidx]; 501 return (word >> bitidx) & mask; 502 } 503 504 /** 505 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 506 * @page: The page within the block of interest 507 * @pfn: The target page frame number 508 * @mask: mask of bits that the caller is interested in 509 * 510 * Return: pageblock_bits flags 511 */ 512 unsigned long get_pfnblock_flags_mask(const struct page *page, 513 unsigned long pfn, unsigned long mask) 514 { 515 return __get_pfnblock_flags_mask(page, pfn, mask); 516 } 517 518 static __always_inline int get_pfnblock_migratetype(const struct page *page, 519 unsigned long pfn) 520 { 521 return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK); 522 } 523 524 /** 525 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 526 * @page: The page within the block of interest 527 * @flags: The flags to set 528 * @pfn: The target page frame number 529 * @mask: mask of bits that the caller is interested in 530 */ 531 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 532 unsigned long pfn, 533 unsigned long mask) 534 { 535 unsigned long *bitmap; 536 unsigned long bitidx, word_bitidx; 537 unsigned long old_word, word; 538 539 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 540 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits)); 541 542 bitmap = get_pageblock_bitmap(page, pfn); 543 bitidx = pfn_to_bitidx(page, pfn); 544 word_bitidx = bitidx / BITS_PER_LONG; 545 bitidx &= (BITS_PER_LONG-1); 546 547 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 548 549 mask <<= bitidx; 550 flags <<= bitidx; 551 552 word = READ_ONCE(bitmap[word_bitidx]); 553 for (;;) { 554 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 555 if (word == old_word) 556 break; 557 word = old_word; 558 } 559 } 560 561 void set_pageblock_migratetype(struct page *page, int migratetype) 562 { 563 if (unlikely(page_group_by_mobility_disabled && 564 migratetype < MIGRATE_PCPTYPES)) 565 migratetype = MIGRATE_UNMOVABLE; 566 567 set_pfnblock_flags_mask(page, (unsigned long)migratetype, 568 page_to_pfn(page), MIGRATETYPE_MASK); 569 } 570 571 #ifdef CONFIG_DEBUG_VM 572 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 573 { 574 int ret = 0; 575 unsigned seq; 576 unsigned long pfn = page_to_pfn(page); 577 unsigned long sp, start_pfn; 578 579 do { 580 seq = zone_span_seqbegin(zone); 581 start_pfn = zone->zone_start_pfn; 582 sp = zone->spanned_pages; 583 if (!zone_spans_pfn(zone, pfn)) 584 ret = 1; 585 } while (zone_span_seqretry(zone, seq)); 586 587 if (ret) 588 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 589 pfn, zone_to_nid(zone), zone->name, 590 start_pfn, start_pfn + sp); 591 592 return ret; 593 } 594 595 static int page_is_consistent(struct zone *zone, struct page *page) 596 { 597 if (!pfn_valid_within(page_to_pfn(page))) 598 return 0; 599 if (zone != page_zone(page)) 600 return 0; 601 602 return 1; 603 } 604 /* 605 * Temporary debugging check for pages not lying within a given zone. 606 */ 607 static int __maybe_unused bad_range(struct zone *zone, struct page *page) 608 { 609 if (page_outside_zone_boundaries(zone, page)) 610 return 1; 611 if (!page_is_consistent(zone, page)) 612 return 1; 613 614 return 0; 615 } 616 #else 617 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page) 618 { 619 return 0; 620 } 621 #endif 622 623 static void bad_page(struct page *page, const char *reason) 624 { 625 static unsigned long resume; 626 static unsigned long nr_shown; 627 static unsigned long nr_unshown; 628 629 /* 630 * Allow a burst of 60 reports, then keep quiet for that minute; 631 * or allow a steady drip of one report per second. 632 */ 633 if (nr_shown == 60) { 634 if (time_before(jiffies, resume)) { 635 nr_unshown++; 636 goto out; 637 } 638 if (nr_unshown) { 639 pr_alert( 640 "BUG: Bad page state: %lu messages suppressed\n", 641 nr_unshown); 642 nr_unshown = 0; 643 } 644 nr_shown = 0; 645 } 646 if (nr_shown++ == 0) 647 resume = jiffies + 60 * HZ; 648 649 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 650 current->comm, page_to_pfn(page)); 651 dump_page(page, reason); 652 653 print_modules(); 654 dump_stack(); 655 out: 656 /* Leave bad fields for debug, except PageBuddy could make trouble */ 657 page_mapcount_reset(page); /* remove PageBuddy */ 658 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 659 } 660 661 static inline unsigned int order_to_pindex(int migratetype, int order) 662 { 663 int base = order; 664 665 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 666 if (order > PAGE_ALLOC_COSTLY_ORDER) { 667 VM_BUG_ON(order != pageblock_order); 668 base = PAGE_ALLOC_COSTLY_ORDER + 1; 669 } 670 #else 671 VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); 672 #endif 673 674 return (MIGRATE_PCPTYPES * base) + migratetype; 675 } 676 677 static inline int pindex_to_order(unsigned int pindex) 678 { 679 int order = pindex / MIGRATE_PCPTYPES; 680 681 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 682 if (order > PAGE_ALLOC_COSTLY_ORDER) { 683 order = pageblock_order; 684 VM_BUG_ON(order != pageblock_order); 685 } 686 #else 687 VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); 688 #endif 689 690 return order; 691 } 692 693 static inline bool pcp_allowed_order(unsigned int order) 694 { 695 if (order <= PAGE_ALLOC_COSTLY_ORDER) 696 return true; 697 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 698 if (order == pageblock_order) 699 return true; 700 #endif 701 return false; 702 } 703 704 static inline void free_the_page(struct page *page, unsigned int order) 705 { 706 if (pcp_allowed_order(order)) /* Via pcp? */ 707 free_unref_page(page, order); 708 else 709 __free_pages_ok(page, order, FPI_NONE); 710 } 711 712 /* 713 * Higher-order pages are called "compound pages". They are structured thusly: 714 * 715 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 716 * 717 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 718 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 719 * 720 * The first tail page's ->compound_dtor holds the offset in array of compound 721 * page destructors. See compound_page_dtors. 722 * 723 * The first tail page's ->compound_order holds the order of allocation. 724 * This usage means that zero-order pages may not be compound. 725 */ 726 727 void free_compound_page(struct page *page) 728 { 729 mem_cgroup_uncharge(page); 730 free_the_page(page, compound_order(page)); 731 } 732 733 void prep_compound_page(struct page *page, unsigned int order) 734 { 735 int i; 736 int nr_pages = 1 << order; 737 738 __SetPageHead(page); 739 for (i = 1; i < nr_pages; i++) { 740 struct page *p = page + i; 741 p->mapping = TAIL_MAPPING; 742 set_compound_head(p, page); 743 } 744 745 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 746 set_compound_order(page, order); 747 atomic_set(compound_mapcount_ptr(page), -1); 748 if (hpage_pincount_available(page)) 749 atomic_set(compound_pincount_ptr(page), 0); 750 } 751 752 #ifdef CONFIG_DEBUG_PAGEALLOC 753 unsigned int _debug_guardpage_minorder; 754 755 bool _debug_pagealloc_enabled_early __read_mostly 756 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 757 EXPORT_SYMBOL(_debug_pagealloc_enabled_early); 758 DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 759 EXPORT_SYMBOL(_debug_pagealloc_enabled); 760 761 DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 762 763 static int __init early_debug_pagealloc(char *buf) 764 { 765 return kstrtobool(buf, &_debug_pagealloc_enabled_early); 766 } 767 early_param("debug_pagealloc", early_debug_pagealloc); 768 769 static int __init debug_guardpage_minorder_setup(char *buf) 770 { 771 unsigned long res; 772 773 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 774 pr_err("Bad debug_guardpage_minorder value\n"); 775 return 0; 776 } 777 _debug_guardpage_minorder = res; 778 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 779 return 0; 780 } 781 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); 782 783 static inline bool set_page_guard(struct zone *zone, struct page *page, 784 unsigned int order, int migratetype) 785 { 786 if (!debug_guardpage_enabled()) 787 return false; 788 789 if (order >= debug_guardpage_minorder()) 790 return false; 791 792 __SetPageGuard(page); 793 INIT_LIST_HEAD(&page->lru); 794 set_page_private(page, order); 795 /* Guard pages are not available for any usage */ 796 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 797 798 return true; 799 } 800 801 static inline void clear_page_guard(struct zone *zone, struct page *page, 802 unsigned int order, int migratetype) 803 { 804 if (!debug_guardpage_enabled()) 805 return; 806 807 __ClearPageGuard(page); 808 809 set_page_private(page, 0); 810 if (!is_migrate_isolate(migratetype)) 811 __mod_zone_freepage_state(zone, (1 << order), migratetype); 812 } 813 #else 814 static inline bool set_page_guard(struct zone *zone, struct page *page, 815 unsigned int order, int migratetype) { return false; } 816 static inline void clear_page_guard(struct zone *zone, struct page *page, 817 unsigned int order, int migratetype) {} 818 #endif 819 820 /* 821 * Enable static keys related to various memory debugging and hardening options. 822 * Some override others, and depend on early params that are evaluated in the 823 * order of appearance. So we need to first gather the full picture of what was 824 * enabled, and then make decisions. 825 */ 826 void init_mem_debugging_and_hardening(void) 827 { 828 bool page_poisoning_requested = false; 829 830 #ifdef CONFIG_PAGE_POISONING 831 /* 832 * Page poisoning is debug page alloc for some arches. If 833 * either of those options are enabled, enable poisoning. 834 */ 835 if (page_poisoning_enabled() || 836 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && 837 debug_pagealloc_enabled())) { 838 static_branch_enable(&_page_poisoning_enabled); 839 page_poisoning_requested = true; 840 } 841 #endif 842 843 if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) && 844 page_poisoning_requested) { 845 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " 846 "will take precedence over init_on_alloc and init_on_free\n"); 847 _init_on_alloc_enabled_early = false; 848 _init_on_free_enabled_early = false; 849 } 850 851 if (_init_on_alloc_enabled_early) 852 static_branch_enable(&init_on_alloc); 853 else 854 static_branch_disable(&init_on_alloc); 855 856 if (_init_on_free_enabled_early) 857 static_branch_enable(&init_on_free); 858 else 859 static_branch_disable(&init_on_free); 860 861 #ifdef CONFIG_DEBUG_PAGEALLOC 862 if (!debug_pagealloc_enabled()) 863 return; 864 865 static_branch_enable(&_debug_pagealloc_enabled); 866 867 if (!debug_guardpage_minorder()) 868 return; 869 870 static_branch_enable(&_debug_guardpage_enabled); 871 #endif 872 } 873 874 static inline void set_buddy_order(struct page *page, unsigned int order) 875 { 876 set_page_private(page, order); 877 __SetPageBuddy(page); 878 } 879 880 /* 881 * This function checks whether a page is free && is the buddy 882 * we can coalesce a page and its buddy if 883 * (a) the buddy is not in a hole (check before calling!) && 884 * (b) the buddy is in the buddy system && 885 * (c) a page and its buddy have the same order && 886 * (d) a page and its buddy are in the same zone. 887 * 888 * For recording whether a page is in the buddy system, we set PageBuddy. 889 * Setting, clearing, and testing PageBuddy is serialized by zone->lock. 890 * 891 * For recording page's order, we use page_private(page). 892 */ 893 static inline bool page_is_buddy(struct page *page, struct page *buddy, 894 unsigned int order) 895 { 896 if (!page_is_guard(buddy) && !PageBuddy(buddy)) 897 return false; 898 899 if (buddy_order(buddy) != order) 900 return false; 901 902 /* 903 * zone check is done late to avoid uselessly calculating 904 * zone/node ids for pages that could never merge. 905 */ 906 if (page_zone_id(page) != page_zone_id(buddy)) 907 return false; 908 909 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 910 911 return true; 912 } 913 914 #ifdef CONFIG_COMPACTION 915 static inline struct capture_control *task_capc(struct zone *zone) 916 { 917 struct capture_control *capc = current->capture_control; 918 919 return unlikely(capc) && 920 !(current->flags & PF_KTHREAD) && 921 !capc->page && 922 capc->cc->zone == zone ? capc : NULL; 923 } 924 925 static inline bool 926 compaction_capture(struct capture_control *capc, struct page *page, 927 int order, int migratetype) 928 { 929 if (!capc || order != capc->cc->order) 930 return false; 931 932 /* Do not accidentally pollute CMA or isolated regions*/ 933 if (is_migrate_cma(migratetype) || 934 is_migrate_isolate(migratetype)) 935 return false; 936 937 /* 938 * Do not let lower order allocations pollute a movable pageblock. 939 * This might let an unmovable request use a reclaimable pageblock 940 * and vice-versa but no more than normal fallback logic which can 941 * have trouble finding a high-order free page. 942 */ 943 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE) 944 return false; 945 946 capc->page = page; 947 return true; 948 } 949 950 #else 951 static inline struct capture_control *task_capc(struct zone *zone) 952 { 953 return NULL; 954 } 955 956 static inline bool 957 compaction_capture(struct capture_control *capc, struct page *page, 958 int order, int migratetype) 959 { 960 return false; 961 } 962 #endif /* CONFIG_COMPACTION */ 963 964 /* Used for pages not on another list */ 965 static inline void add_to_free_list(struct page *page, struct zone *zone, 966 unsigned int order, int migratetype) 967 { 968 struct free_area *area = &zone->free_area[order]; 969 970 list_add(&page->lru, &area->free_list[migratetype]); 971 area->nr_free++; 972 } 973 974 /* Used for pages not on another list */ 975 static inline void add_to_free_list_tail(struct page *page, struct zone *zone, 976 unsigned int order, int migratetype) 977 { 978 struct free_area *area = &zone->free_area[order]; 979 980 list_add_tail(&page->lru, &area->free_list[migratetype]); 981 area->nr_free++; 982 } 983 984 /* 985 * Used for pages which are on another list. Move the pages to the tail 986 * of the list - so the moved pages won't immediately be considered for 987 * allocation again (e.g., optimization for memory onlining). 988 */ 989 static inline void move_to_free_list(struct page *page, struct zone *zone, 990 unsigned int order, int migratetype) 991 { 992 struct free_area *area = &zone->free_area[order]; 993 994 list_move_tail(&page->lru, &area->free_list[migratetype]); 995 } 996 997 static inline void del_page_from_free_list(struct page *page, struct zone *zone, 998 unsigned int order) 999 { 1000 /* clear reported state and update reported page count */ 1001 if (page_reported(page)) 1002 __ClearPageReported(page); 1003 1004 list_del(&page->lru); 1005 __ClearPageBuddy(page); 1006 set_page_private(page, 0); 1007 zone->free_area[order].nr_free--; 1008 } 1009 1010 /* 1011 * If this is not the largest possible page, check if the buddy 1012 * of the next-highest order is free. If it is, it's possible 1013 * that pages are being freed that will coalesce soon. In case, 1014 * that is happening, add the free page to the tail of the list 1015 * so it's less likely to be used soon and more likely to be merged 1016 * as a higher order page 1017 */ 1018 static inline bool 1019 buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, 1020 struct page *page, unsigned int order) 1021 { 1022 struct page *higher_page, *higher_buddy; 1023 unsigned long combined_pfn; 1024 1025 if (order >= MAX_ORDER - 2) 1026 return false; 1027 1028 if (!pfn_valid_within(buddy_pfn)) 1029 return false; 1030 1031 combined_pfn = buddy_pfn & pfn; 1032 higher_page = page + (combined_pfn - pfn); 1033 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1); 1034 higher_buddy = higher_page + (buddy_pfn - combined_pfn); 1035 1036 return pfn_valid_within(buddy_pfn) && 1037 page_is_buddy(higher_page, higher_buddy, order + 1); 1038 } 1039 1040 /* 1041 * Freeing function for a buddy system allocator. 1042 * 1043 * The concept of a buddy system is to maintain direct-mapped table 1044 * (containing bit values) for memory blocks of various "orders". 1045 * The bottom level table contains the map for the smallest allocatable 1046 * units of memory (here, pages), and each level above it describes 1047 * pairs of units from the levels below, hence, "buddies". 1048 * At a high level, all that happens here is marking the table entry 1049 * at the bottom level available, and propagating the changes upward 1050 * as necessary, plus some accounting needed to play nicely with other 1051 * parts of the VM system. 1052 * At each level, we keep a list of pages, which are heads of continuous 1053 * free pages of length of (1 << order) and marked with PageBuddy. 1054 * Page's order is recorded in page_private(page) field. 1055 * So when we are allocating or freeing one, we can derive the state of the 1056 * other. That is, if we allocate a small block, and both were 1057 * free, the remainder of the region must be split into blocks. 1058 * If a block is freed, and its buddy is also free, then this 1059 * triggers coalescing into a block of larger size. 1060 * 1061 * -- nyc 1062 */ 1063 1064 static inline void __free_one_page(struct page *page, 1065 unsigned long pfn, 1066 struct zone *zone, unsigned int order, 1067 int migratetype, fpi_t fpi_flags) 1068 { 1069 struct capture_control *capc = task_capc(zone); 1070 unsigned long buddy_pfn; 1071 unsigned long combined_pfn; 1072 unsigned int max_order; 1073 struct page *buddy; 1074 bool to_tail; 1075 1076 max_order = min_t(unsigned int, MAX_ORDER - 1, pageblock_order); 1077 1078 VM_BUG_ON(!zone_is_initialized(zone)); 1079 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 1080 1081 VM_BUG_ON(migratetype == -1); 1082 if (likely(!is_migrate_isolate(migratetype))) 1083 __mod_zone_freepage_state(zone, 1 << order, migratetype); 1084 1085 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); 1086 VM_BUG_ON_PAGE(bad_range(zone, page), page); 1087 1088 continue_merging: 1089 while (order < max_order) { 1090 if (compaction_capture(capc, page, order, migratetype)) { 1091 __mod_zone_freepage_state(zone, -(1 << order), 1092 migratetype); 1093 return; 1094 } 1095 buddy_pfn = __find_buddy_pfn(pfn, order); 1096 buddy = page + (buddy_pfn - pfn); 1097 1098 if (!pfn_valid_within(buddy_pfn)) 1099 goto done_merging; 1100 if (!page_is_buddy(page, buddy, order)) 1101 goto done_merging; 1102 /* 1103 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 1104 * merge with it and move up one order. 1105 */ 1106 if (page_is_guard(buddy)) 1107 clear_page_guard(zone, buddy, order, migratetype); 1108 else 1109 del_page_from_free_list(buddy, zone, order); 1110 combined_pfn = buddy_pfn & pfn; 1111 page = page + (combined_pfn - pfn); 1112 pfn = combined_pfn; 1113 order++; 1114 } 1115 if (order < MAX_ORDER - 1) { 1116 /* If we are here, it means order is >= pageblock_order. 1117 * We want to prevent merge between freepages on isolate 1118 * pageblock and normal pageblock. Without this, pageblock 1119 * isolation could cause incorrect freepage or CMA accounting. 1120 * 1121 * We don't want to hit this code for the more frequent 1122 * low-order merging. 1123 */ 1124 if (unlikely(has_isolate_pageblock(zone))) { 1125 int buddy_mt; 1126 1127 buddy_pfn = __find_buddy_pfn(pfn, order); 1128 buddy = page + (buddy_pfn - pfn); 1129 buddy_mt = get_pageblock_migratetype(buddy); 1130 1131 if (migratetype != buddy_mt 1132 && (is_migrate_isolate(migratetype) || 1133 is_migrate_isolate(buddy_mt))) 1134 goto done_merging; 1135 } 1136 max_order = order + 1; 1137 goto continue_merging; 1138 } 1139 1140 done_merging: 1141 set_buddy_order(page, order); 1142 1143 if (fpi_flags & FPI_TO_TAIL) 1144 to_tail = true; 1145 else if (is_shuffle_order(order)) 1146 to_tail = shuffle_pick_tail(); 1147 else 1148 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); 1149 1150 if (to_tail) 1151 add_to_free_list_tail(page, zone, order, migratetype); 1152 else 1153 add_to_free_list(page, zone, order, migratetype); 1154 1155 /* Notify page reporting subsystem of freed page */ 1156 if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY)) 1157 page_reporting_notify_free(order); 1158 } 1159 1160 /* 1161 * A bad page could be due to a number of fields. Instead of multiple branches, 1162 * try and check multiple fields with one check. The caller must do a detailed 1163 * check if necessary. 1164 */ 1165 static inline bool page_expected_state(struct page *page, 1166 unsigned long check_flags) 1167 { 1168 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1169 return false; 1170 1171 if (unlikely((unsigned long)page->mapping | 1172 page_ref_count(page) | 1173 #ifdef CONFIG_MEMCG 1174 page->memcg_data | 1175 #endif 1176 (page->flags & check_flags))) 1177 return false; 1178 1179 return true; 1180 } 1181 1182 static const char *page_bad_reason(struct page *page, unsigned long flags) 1183 { 1184 const char *bad_reason = NULL; 1185 1186 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1187 bad_reason = "nonzero mapcount"; 1188 if (unlikely(page->mapping != NULL)) 1189 bad_reason = "non-NULL mapping"; 1190 if (unlikely(page_ref_count(page) != 0)) 1191 bad_reason = "nonzero _refcount"; 1192 if (unlikely(page->flags & flags)) { 1193 if (flags == PAGE_FLAGS_CHECK_AT_PREP) 1194 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set"; 1195 else 1196 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 1197 } 1198 #ifdef CONFIG_MEMCG 1199 if (unlikely(page->memcg_data)) 1200 bad_reason = "page still charged to cgroup"; 1201 #endif 1202 return bad_reason; 1203 } 1204 1205 static void check_free_page_bad(struct page *page) 1206 { 1207 bad_page(page, 1208 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE)); 1209 } 1210 1211 static inline int check_free_page(struct page *page) 1212 { 1213 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 1214 return 0; 1215 1216 /* Something has gone sideways, find it */ 1217 check_free_page_bad(page); 1218 return 1; 1219 } 1220 1221 static int free_tail_pages_check(struct page *head_page, struct page *page) 1222 { 1223 int ret = 1; 1224 1225 /* 1226 * We rely page->lru.next never has bit 0 set, unless the page 1227 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 1228 */ 1229 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 1230 1231 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 1232 ret = 0; 1233 goto out; 1234 } 1235 switch (page - head_page) { 1236 case 1: 1237 /* the first tail page: ->mapping may be compound_mapcount() */ 1238 if (unlikely(compound_mapcount(page))) { 1239 bad_page(page, "nonzero compound_mapcount"); 1240 goto out; 1241 } 1242 break; 1243 case 2: 1244 /* 1245 * the second tail page: ->mapping is 1246 * deferred_list.next -- ignore value. 1247 */ 1248 break; 1249 default: 1250 if (page->mapping != TAIL_MAPPING) { 1251 bad_page(page, "corrupted mapping in tail page"); 1252 goto out; 1253 } 1254 break; 1255 } 1256 if (unlikely(!PageTail(page))) { 1257 bad_page(page, "PageTail not set"); 1258 goto out; 1259 } 1260 if (unlikely(compound_head(page) != head_page)) { 1261 bad_page(page, "compound_head not consistent"); 1262 goto out; 1263 } 1264 ret = 0; 1265 out: 1266 page->mapping = NULL; 1267 clear_compound_head(page); 1268 return ret; 1269 } 1270 1271 static void kernel_init_free_pages(struct page *page, int numpages, bool zero_tags) 1272 { 1273 int i; 1274 1275 if (zero_tags) { 1276 for (i = 0; i < numpages; i++) 1277 tag_clear_highpage(page + i); 1278 return; 1279 } 1280 1281 /* s390's use of memset() could override KASAN redzones. */ 1282 kasan_disable_current(); 1283 for (i = 0; i < numpages; i++) { 1284 u8 tag = page_kasan_tag(page + i); 1285 page_kasan_tag_reset(page + i); 1286 clear_highpage(page + i); 1287 page_kasan_tag_set(page + i, tag); 1288 } 1289 kasan_enable_current(); 1290 } 1291 1292 static __always_inline bool free_pages_prepare(struct page *page, 1293 unsigned int order, bool check_free, fpi_t fpi_flags) 1294 { 1295 int bad = 0; 1296 bool skip_kasan_poison = should_skip_kasan_poison(page, fpi_flags); 1297 1298 VM_BUG_ON_PAGE(PageTail(page), page); 1299 1300 trace_mm_page_free(page, order); 1301 1302 if (unlikely(PageHWPoison(page)) && !order) { 1303 /* 1304 * Do not let hwpoison pages hit pcplists/buddy 1305 * Untie memcg state and reset page's owner 1306 */ 1307 if (memcg_kmem_enabled() && PageMemcgKmem(page)) 1308 __memcg_kmem_uncharge_page(page, order); 1309 reset_page_owner(page, order); 1310 return false; 1311 } 1312 1313 /* 1314 * Check tail pages before head page information is cleared to 1315 * avoid checking PageCompound for order-0 pages. 1316 */ 1317 if (unlikely(order)) { 1318 bool compound = PageCompound(page); 1319 int i; 1320 1321 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1322 1323 if (compound) 1324 ClearPageDoubleMap(page); 1325 for (i = 1; i < (1 << order); i++) { 1326 if (compound) 1327 bad += free_tail_pages_check(page, page + i); 1328 if (unlikely(check_free_page(page + i))) { 1329 bad++; 1330 continue; 1331 } 1332 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1333 } 1334 } 1335 if (PageMappingFlags(page)) 1336 page->mapping = NULL; 1337 if (memcg_kmem_enabled() && PageMemcgKmem(page)) 1338 __memcg_kmem_uncharge_page(page, order); 1339 if (check_free) 1340 bad += check_free_page(page); 1341 if (bad) 1342 return false; 1343 1344 page_cpupid_reset_last(page); 1345 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1346 reset_page_owner(page, order); 1347 1348 if (!PageHighMem(page)) { 1349 debug_check_no_locks_freed(page_address(page), 1350 PAGE_SIZE << order); 1351 debug_check_no_obj_freed(page_address(page), 1352 PAGE_SIZE << order); 1353 } 1354 1355 kernel_poison_pages(page, 1 << order); 1356 1357 /* 1358 * As memory initialization might be integrated into KASAN, 1359 * kasan_free_pages and kernel_init_free_pages must be 1360 * kept together to avoid discrepancies in behavior. 1361 * 1362 * With hardware tag-based KASAN, memory tags must be set before the 1363 * page becomes unavailable via debug_pagealloc or arch_free_page. 1364 */ 1365 if (kasan_has_integrated_init()) { 1366 if (!skip_kasan_poison) 1367 kasan_free_pages(page, order); 1368 } else { 1369 bool init = want_init_on_free(); 1370 1371 if (init) 1372 kernel_init_free_pages(page, 1 << order, false); 1373 if (!skip_kasan_poison) 1374 kasan_poison_pages(page, order, init); 1375 } 1376 1377 /* 1378 * arch_free_page() can make the page's contents inaccessible. s390 1379 * does this. So nothing which can access the page's contents should 1380 * happen after this. 1381 */ 1382 arch_free_page(page, order); 1383 1384 debug_pagealloc_unmap_pages(page, 1 << order); 1385 1386 return true; 1387 } 1388 1389 #ifdef CONFIG_DEBUG_VM 1390 /* 1391 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed 1392 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when 1393 * moved from pcp lists to free lists. 1394 */ 1395 static bool free_pcp_prepare(struct page *page, unsigned int order) 1396 { 1397 return free_pages_prepare(page, order, true, FPI_NONE); 1398 } 1399 1400 static bool bulkfree_pcp_prepare(struct page *page) 1401 { 1402 if (debug_pagealloc_enabled_static()) 1403 return check_free_page(page); 1404 else 1405 return false; 1406 } 1407 #else 1408 /* 1409 * With DEBUG_VM disabled, order-0 pages being freed are checked only when 1410 * moving from pcp lists to free list in order to reduce overhead. With 1411 * debug_pagealloc enabled, they are checked also immediately when being freed 1412 * to the pcp lists. 1413 */ 1414 static bool free_pcp_prepare(struct page *page, unsigned int order) 1415 { 1416 if (debug_pagealloc_enabled_static()) 1417 return free_pages_prepare(page, order, true, FPI_NONE); 1418 else 1419 return free_pages_prepare(page, order, false, FPI_NONE); 1420 } 1421 1422 static bool bulkfree_pcp_prepare(struct page *page) 1423 { 1424 return check_free_page(page); 1425 } 1426 #endif /* CONFIG_DEBUG_VM */ 1427 1428 static inline void prefetch_buddy(struct page *page) 1429 { 1430 unsigned long pfn = page_to_pfn(page); 1431 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0); 1432 struct page *buddy = page + (buddy_pfn - pfn); 1433 1434 prefetch(buddy); 1435 } 1436 1437 /* 1438 * Frees a number of pages from the PCP lists 1439 * Assumes all pages on list are in same zone, and of same order. 1440 * count is the number of pages to free. 1441 * 1442 * If the zone was previously in an "all pages pinned" state then look to 1443 * see if this freeing clears that state. 1444 * 1445 * And clear the zone's pages_scanned counter, to hold off the "all pages are 1446 * pinned" detection logic. 1447 */ 1448 static void free_pcppages_bulk(struct zone *zone, int count, 1449 struct per_cpu_pages *pcp) 1450 { 1451 int pindex = 0; 1452 int batch_free = 0; 1453 int nr_freed = 0; 1454 unsigned int order; 1455 int prefetch_nr = READ_ONCE(pcp->batch); 1456 bool isolated_pageblocks; 1457 struct page *page, *tmp; 1458 LIST_HEAD(head); 1459 1460 /* 1461 * Ensure proper count is passed which otherwise would stuck in the 1462 * below while (list_empty(list)) loop. 1463 */ 1464 count = min(pcp->count, count); 1465 while (count > 0) { 1466 struct list_head *list; 1467 1468 /* 1469 * Remove pages from lists in a round-robin fashion. A 1470 * batch_free count is maintained that is incremented when an 1471 * empty list is encountered. This is so more pages are freed 1472 * off fuller lists instead of spinning excessively around empty 1473 * lists 1474 */ 1475 do { 1476 batch_free++; 1477 if (++pindex == NR_PCP_LISTS) 1478 pindex = 0; 1479 list = &pcp->lists[pindex]; 1480 } while (list_empty(list)); 1481 1482 /* This is the only non-empty list. Free them all. */ 1483 if (batch_free == NR_PCP_LISTS) 1484 batch_free = count; 1485 1486 order = pindex_to_order(pindex); 1487 BUILD_BUG_ON(MAX_ORDER >= (1<<NR_PCP_ORDER_WIDTH)); 1488 do { 1489 page = list_last_entry(list, struct page, lru); 1490 /* must delete to avoid corrupting pcp list */ 1491 list_del(&page->lru); 1492 nr_freed += 1 << order; 1493 count -= 1 << order; 1494 1495 if (bulkfree_pcp_prepare(page)) 1496 continue; 1497 1498 /* Encode order with the migratetype */ 1499 page->index <<= NR_PCP_ORDER_WIDTH; 1500 page->index |= order; 1501 1502 list_add_tail(&page->lru, &head); 1503 1504 /* 1505 * We are going to put the page back to the global 1506 * pool, prefetch its buddy to speed up later access 1507 * under zone->lock. It is believed the overhead of 1508 * an additional test and calculating buddy_pfn here 1509 * can be offset by reduced memory latency later. To 1510 * avoid excessive prefetching due to large count, only 1511 * prefetch buddy for the first pcp->batch nr of pages. 1512 */ 1513 if (prefetch_nr) { 1514 prefetch_buddy(page); 1515 prefetch_nr--; 1516 } 1517 } while (count > 0 && --batch_free && !list_empty(list)); 1518 } 1519 pcp->count -= nr_freed; 1520 1521 /* 1522 * local_lock_irq held so equivalent to spin_lock_irqsave for 1523 * both PREEMPT_RT and non-PREEMPT_RT configurations. 1524 */ 1525 spin_lock(&zone->lock); 1526 isolated_pageblocks = has_isolate_pageblock(zone); 1527 1528 /* 1529 * Use safe version since after __free_one_page(), 1530 * page->lru.next will not point to original list. 1531 */ 1532 list_for_each_entry_safe(page, tmp, &head, lru) { 1533 int mt = get_pcppage_migratetype(page); 1534 1535 /* mt has been encoded with the order (see above) */ 1536 order = mt & NR_PCP_ORDER_MASK; 1537 mt >>= NR_PCP_ORDER_WIDTH; 1538 1539 /* MIGRATE_ISOLATE page should not go to pcplists */ 1540 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1541 /* Pageblock could have been isolated meanwhile */ 1542 if (unlikely(isolated_pageblocks)) 1543 mt = get_pageblock_migratetype(page); 1544 1545 __free_one_page(page, page_to_pfn(page), zone, order, mt, FPI_NONE); 1546 trace_mm_page_pcpu_drain(page, order, mt); 1547 } 1548 spin_unlock(&zone->lock); 1549 } 1550 1551 static void free_one_page(struct zone *zone, 1552 struct page *page, unsigned long pfn, 1553 unsigned int order, 1554 int migratetype, fpi_t fpi_flags) 1555 { 1556 unsigned long flags; 1557 1558 spin_lock_irqsave(&zone->lock, flags); 1559 if (unlikely(has_isolate_pageblock(zone) || 1560 is_migrate_isolate(migratetype))) { 1561 migratetype = get_pfnblock_migratetype(page, pfn); 1562 } 1563 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags); 1564 spin_unlock_irqrestore(&zone->lock, flags); 1565 } 1566 1567 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1568 unsigned long zone, int nid) 1569 { 1570 mm_zero_struct_page(page); 1571 set_page_links(page, zone, nid, pfn); 1572 init_page_count(page); 1573 page_mapcount_reset(page); 1574 page_cpupid_reset_last(page); 1575 page_kasan_tag_reset(page); 1576 1577 INIT_LIST_HEAD(&page->lru); 1578 #ifdef WANT_PAGE_VIRTUAL 1579 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1580 if (!is_highmem_idx(zone)) 1581 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1582 #endif 1583 } 1584 1585 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1586 static void __meminit init_reserved_page(unsigned long pfn) 1587 { 1588 pg_data_t *pgdat; 1589 int nid, zid; 1590 1591 if (!early_page_uninitialised(pfn)) 1592 return; 1593 1594 nid = early_pfn_to_nid(pfn); 1595 pgdat = NODE_DATA(nid); 1596 1597 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1598 struct zone *zone = &pgdat->node_zones[zid]; 1599 1600 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) 1601 break; 1602 } 1603 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 1604 } 1605 #else 1606 static inline void init_reserved_page(unsigned long pfn) 1607 { 1608 } 1609 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1610 1611 /* 1612 * Initialised pages do not have PageReserved set. This function is 1613 * called for each range allocated by the bootmem allocator and 1614 * marks the pages PageReserved. The remaining valid pages are later 1615 * sent to the buddy page allocator. 1616 */ 1617 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1618 { 1619 unsigned long start_pfn = PFN_DOWN(start); 1620 unsigned long end_pfn = PFN_UP(end); 1621 1622 for (; start_pfn < end_pfn; start_pfn++) { 1623 if (pfn_valid(start_pfn)) { 1624 struct page *page = pfn_to_page(start_pfn); 1625 1626 init_reserved_page(start_pfn); 1627 1628 /* Avoid false-positive PageTail() */ 1629 INIT_LIST_HEAD(&page->lru); 1630 1631 /* 1632 * no need for atomic set_bit because the struct 1633 * page is not visible yet so nobody should 1634 * access it yet. 1635 */ 1636 __SetPageReserved(page); 1637 } 1638 } 1639 } 1640 1641 static void __free_pages_ok(struct page *page, unsigned int order, 1642 fpi_t fpi_flags) 1643 { 1644 unsigned long flags; 1645 int migratetype; 1646 unsigned long pfn = page_to_pfn(page); 1647 struct zone *zone = page_zone(page); 1648 1649 if (!free_pages_prepare(page, order, true, fpi_flags)) 1650 return; 1651 1652 migratetype = get_pfnblock_migratetype(page, pfn); 1653 1654 spin_lock_irqsave(&zone->lock, flags); 1655 if (unlikely(has_isolate_pageblock(zone) || 1656 is_migrate_isolate(migratetype))) { 1657 migratetype = get_pfnblock_migratetype(page, pfn); 1658 } 1659 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags); 1660 spin_unlock_irqrestore(&zone->lock, flags); 1661 1662 __count_vm_events(PGFREE, 1 << order); 1663 } 1664 1665 void __free_pages_core(struct page *page, unsigned int order) 1666 { 1667 unsigned int nr_pages = 1 << order; 1668 struct page *p = page; 1669 unsigned int loop; 1670 1671 /* 1672 * When initializing the memmap, __init_single_page() sets the refcount 1673 * of all pages to 1 ("allocated"/"not free"). We have to set the 1674 * refcount of all involved pages to 0. 1675 */ 1676 prefetchw(p); 1677 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1678 prefetchw(p + 1); 1679 __ClearPageReserved(p); 1680 set_page_count(p, 0); 1681 } 1682 __ClearPageReserved(p); 1683 set_page_count(p, 0); 1684 1685 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); 1686 1687 /* 1688 * Bypass PCP and place fresh pages right to the tail, primarily 1689 * relevant for memory onlining. 1690 */ 1691 __free_pages_ok(page, order, FPI_TO_TAIL | FPI_SKIP_KASAN_POISON); 1692 } 1693 1694 #ifdef CONFIG_NUMA 1695 1696 /* 1697 * During memory init memblocks map pfns to nids. The search is expensive and 1698 * this caches recent lookups. The implementation of __early_pfn_to_nid 1699 * treats start/end as pfns. 1700 */ 1701 struct mminit_pfnnid_cache { 1702 unsigned long last_start; 1703 unsigned long last_end; 1704 int last_nid; 1705 }; 1706 1707 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1708 1709 /* 1710 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 1711 */ 1712 static int __meminit __early_pfn_to_nid(unsigned long pfn, 1713 struct mminit_pfnnid_cache *state) 1714 { 1715 unsigned long start_pfn, end_pfn; 1716 int nid; 1717 1718 if (state->last_start <= pfn && pfn < state->last_end) 1719 return state->last_nid; 1720 1721 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 1722 if (nid != NUMA_NO_NODE) { 1723 state->last_start = start_pfn; 1724 state->last_end = end_pfn; 1725 state->last_nid = nid; 1726 } 1727 1728 return nid; 1729 } 1730 1731 int __meminit early_pfn_to_nid(unsigned long pfn) 1732 { 1733 static DEFINE_SPINLOCK(early_pfn_lock); 1734 int nid; 1735 1736 spin_lock(&early_pfn_lock); 1737 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1738 if (nid < 0) 1739 nid = first_online_node; 1740 spin_unlock(&early_pfn_lock); 1741 1742 return nid; 1743 } 1744 #endif /* CONFIG_NUMA */ 1745 1746 void __init memblock_free_pages(struct page *page, unsigned long pfn, 1747 unsigned int order) 1748 { 1749 if (early_page_uninitialised(pfn)) 1750 return; 1751 __free_pages_core(page, order); 1752 } 1753 1754 /* 1755 * Check that the whole (or subset of) a pageblock given by the interval of 1756 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1757 * with the migration of free compaction scanner. The scanners then need to 1758 * use only pfn_valid_within() check for arches that allow holes within 1759 * pageblocks. 1760 * 1761 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1762 * 1763 * It's possible on some configurations to have a setup like node0 node1 node0 1764 * i.e. it's possible that all pages within a zones range of pages do not 1765 * belong to a single zone. We assume that a border between node0 and node1 1766 * can occur within a single pageblock, but not a node0 node1 node0 1767 * interleaving within a single pageblock. It is therefore sufficient to check 1768 * the first and last page of a pageblock and avoid checking each individual 1769 * page in a pageblock. 1770 */ 1771 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1772 unsigned long end_pfn, struct zone *zone) 1773 { 1774 struct page *start_page; 1775 struct page *end_page; 1776 1777 /* end_pfn is one past the range we are checking */ 1778 end_pfn--; 1779 1780 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1781 return NULL; 1782 1783 start_page = pfn_to_online_page(start_pfn); 1784 if (!start_page) 1785 return NULL; 1786 1787 if (page_zone(start_page) != zone) 1788 return NULL; 1789 1790 end_page = pfn_to_page(end_pfn); 1791 1792 /* This gives a shorter code than deriving page_zone(end_page) */ 1793 if (page_zone_id(start_page) != page_zone_id(end_page)) 1794 return NULL; 1795 1796 return start_page; 1797 } 1798 1799 void set_zone_contiguous(struct zone *zone) 1800 { 1801 unsigned long block_start_pfn = zone->zone_start_pfn; 1802 unsigned long block_end_pfn; 1803 1804 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1805 for (; block_start_pfn < zone_end_pfn(zone); 1806 block_start_pfn = block_end_pfn, 1807 block_end_pfn += pageblock_nr_pages) { 1808 1809 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1810 1811 if (!__pageblock_pfn_to_page(block_start_pfn, 1812 block_end_pfn, zone)) 1813 return; 1814 cond_resched(); 1815 } 1816 1817 /* We confirm that there is no hole */ 1818 zone->contiguous = true; 1819 } 1820 1821 void clear_zone_contiguous(struct zone *zone) 1822 { 1823 zone->contiguous = false; 1824 } 1825 1826 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1827 static void __init deferred_free_range(unsigned long pfn, 1828 unsigned long nr_pages) 1829 { 1830 struct page *page; 1831 unsigned long i; 1832 1833 if (!nr_pages) 1834 return; 1835 1836 page = pfn_to_page(pfn); 1837 1838 /* Free a large naturally-aligned chunk if possible */ 1839 if (nr_pages == pageblock_nr_pages && 1840 (pfn & (pageblock_nr_pages - 1)) == 0) { 1841 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1842 __free_pages_core(page, pageblock_order); 1843 return; 1844 } 1845 1846 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1847 if ((pfn & (pageblock_nr_pages - 1)) == 0) 1848 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1849 __free_pages_core(page, 0); 1850 } 1851 } 1852 1853 /* Completion tracking for deferred_init_memmap() threads */ 1854 static atomic_t pgdat_init_n_undone __initdata; 1855 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1856 1857 static inline void __init pgdat_init_report_one_done(void) 1858 { 1859 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1860 complete(&pgdat_init_all_done_comp); 1861 } 1862 1863 /* 1864 * Returns true if page needs to be initialized or freed to buddy allocator. 1865 * 1866 * First we check if pfn is valid on architectures where it is possible to have 1867 * holes within pageblock_nr_pages. On systems where it is not possible, this 1868 * function is optimized out. 1869 * 1870 * Then, we check if a current large page is valid by only checking the validity 1871 * of the head pfn. 1872 */ 1873 static inline bool __init deferred_pfn_valid(unsigned long pfn) 1874 { 1875 if (!pfn_valid_within(pfn)) 1876 return false; 1877 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn)) 1878 return false; 1879 return true; 1880 } 1881 1882 /* 1883 * Free pages to buddy allocator. Try to free aligned pages in 1884 * pageblock_nr_pages sizes. 1885 */ 1886 static void __init deferred_free_pages(unsigned long pfn, 1887 unsigned long end_pfn) 1888 { 1889 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1890 unsigned long nr_free = 0; 1891 1892 for (; pfn < end_pfn; pfn++) { 1893 if (!deferred_pfn_valid(pfn)) { 1894 deferred_free_range(pfn - nr_free, nr_free); 1895 nr_free = 0; 1896 } else if (!(pfn & nr_pgmask)) { 1897 deferred_free_range(pfn - nr_free, nr_free); 1898 nr_free = 1; 1899 } else { 1900 nr_free++; 1901 } 1902 } 1903 /* Free the last block of pages to allocator */ 1904 deferred_free_range(pfn - nr_free, nr_free); 1905 } 1906 1907 /* 1908 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 1909 * by performing it only once every pageblock_nr_pages. 1910 * Return number of pages initialized. 1911 */ 1912 static unsigned long __init deferred_init_pages(struct zone *zone, 1913 unsigned long pfn, 1914 unsigned long end_pfn) 1915 { 1916 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1917 int nid = zone_to_nid(zone); 1918 unsigned long nr_pages = 0; 1919 int zid = zone_idx(zone); 1920 struct page *page = NULL; 1921 1922 for (; pfn < end_pfn; pfn++) { 1923 if (!deferred_pfn_valid(pfn)) { 1924 page = NULL; 1925 continue; 1926 } else if (!page || !(pfn & nr_pgmask)) { 1927 page = pfn_to_page(pfn); 1928 } else { 1929 page++; 1930 } 1931 __init_single_page(page, pfn, zid, nid); 1932 nr_pages++; 1933 } 1934 return (nr_pages); 1935 } 1936 1937 /* 1938 * This function is meant to pre-load the iterator for the zone init. 1939 * Specifically it walks through the ranges until we are caught up to the 1940 * first_init_pfn value and exits there. If we never encounter the value we 1941 * return false indicating there are no valid ranges left. 1942 */ 1943 static bool __init 1944 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, 1945 unsigned long *spfn, unsigned long *epfn, 1946 unsigned long first_init_pfn) 1947 { 1948 u64 j; 1949 1950 /* 1951 * Start out by walking through the ranges in this zone that have 1952 * already been initialized. We don't need to do anything with them 1953 * so we just need to flush them out of the system. 1954 */ 1955 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { 1956 if (*epfn <= first_init_pfn) 1957 continue; 1958 if (*spfn < first_init_pfn) 1959 *spfn = first_init_pfn; 1960 *i = j; 1961 return true; 1962 } 1963 1964 return false; 1965 } 1966 1967 /* 1968 * Initialize and free pages. We do it in two loops: first we initialize 1969 * struct page, then free to buddy allocator, because while we are 1970 * freeing pages we can access pages that are ahead (computing buddy 1971 * page in __free_one_page()). 1972 * 1973 * In order to try and keep some memory in the cache we have the loop 1974 * broken along max page order boundaries. This way we will not cause 1975 * any issues with the buddy page computation. 1976 */ 1977 static unsigned long __init 1978 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, 1979 unsigned long *end_pfn) 1980 { 1981 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); 1982 unsigned long spfn = *start_pfn, epfn = *end_pfn; 1983 unsigned long nr_pages = 0; 1984 u64 j = *i; 1985 1986 /* First we loop through and initialize the page values */ 1987 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { 1988 unsigned long t; 1989 1990 if (mo_pfn <= *start_pfn) 1991 break; 1992 1993 t = min(mo_pfn, *end_pfn); 1994 nr_pages += deferred_init_pages(zone, *start_pfn, t); 1995 1996 if (mo_pfn < *end_pfn) { 1997 *start_pfn = mo_pfn; 1998 break; 1999 } 2000 } 2001 2002 /* Reset values and now loop through freeing pages as needed */ 2003 swap(j, *i); 2004 2005 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { 2006 unsigned long t; 2007 2008 if (mo_pfn <= spfn) 2009 break; 2010 2011 t = min(mo_pfn, epfn); 2012 deferred_free_pages(spfn, t); 2013 2014 if (mo_pfn <= epfn) 2015 break; 2016 } 2017 2018 return nr_pages; 2019 } 2020 2021 static void __init 2022 deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, 2023 void *arg) 2024 { 2025 unsigned long spfn, epfn; 2026 struct zone *zone = arg; 2027 u64 i; 2028 2029 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); 2030 2031 /* 2032 * Initialize and free pages in MAX_ORDER sized increments so that we 2033 * can avoid introducing any issues with the buddy allocator. 2034 */ 2035 while (spfn < end_pfn) { 2036 deferred_init_maxorder(&i, zone, &spfn, &epfn); 2037 cond_resched(); 2038 } 2039 } 2040 2041 /* An arch may override for more concurrency. */ 2042 __weak int __init 2043 deferred_page_init_max_threads(const struct cpumask *node_cpumask) 2044 { 2045 return 1; 2046 } 2047 2048 /* Initialise remaining memory on a node */ 2049 static int __init deferred_init_memmap(void *data) 2050 { 2051 pg_data_t *pgdat = data; 2052 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 2053 unsigned long spfn = 0, epfn = 0; 2054 unsigned long first_init_pfn, flags; 2055 unsigned long start = jiffies; 2056 struct zone *zone; 2057 int zid, max_threads; 2058 u64 i; 2059 2060 /* Bind memory initialisation thread to a local node if possible */ 2061 if (!cpumask_empty(cpumask)) 2062 set_cpus_allowed_ptr(current, cpumask); 2063 2064 pgdat_resize_lock(pgdat, &flags); 2065 first_init_pfn = pgdat->first_deferred_pfn; 2066 if (first_init_pfn == ULONG_MAX) { 2067 pgdat_resize_unlock(pgdat, &flags); 2068 pgdat_init_report_one_done(); 2069 return 0; 2070 } 2071 2072 /* Sanity check boundaries */ 2073 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 2074 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 2075 pgdat->first_deferred_pfn = ULONG_MAX; 2076 2077 /* 2078 * Once we unlock here, the zone cannot be grown anymore, thus if an 2079 * interrupt thread must allocate this early in boot, zone must be 2080 * pre-grown prior to start of deferred page initialization. 2081 */ 2082 pgdat_resize_unlock(pgdat, &flags); 2083 2084 /* Only the highest zone is deferred so find it */ 2085 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 2086 zone = pgdat->node_zones + zid; 2087 if (first_init_pfn < zone_end_pfn(zone)) 2088 break; 2089 } 2090 2091 /* If the zone is empty somebody else may have cleared out the zone */ 2092 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2093 first_init_pfn)) 2094 goto zone_empty; 2095 2096 max_threads = deferred_page_init_max_threads(cpumask); 2097 2098 while (spfn < epfn) { 2099 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION); 2100 struct padata_mt_job job = { 2101 .thread_fn = deferred_init_memmap_chunk, 2102 .fn_arg = zone, 2103 .start = spfn, 2104 .size = epfn_align - spfn, 2105 .align = PAGES_PER_SECTION, 2106 .min_chunk = PAGES_PER_SECTION, 2107 .max_threads = max_threads, 2108 }; 2109 2110 padata_do_multithreaded(&job); 2111 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2112 epfn_align); 2113 } 2114 zone_empty: 2115 /* Sanity check that the next zone really is unpopulated */ 2116 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 2117 2118 pr_info("node %d deferred pages initialised in %ums\n", 2119 pgdat->node_id, jiffies_to_msecs(jiffies - start)); 2120 2121 pgdat_init_report_one_done(); 2122 return 0; 2123 } 2124 2125 /* 2126 * If this zone has deferred pages, try to grow it by initializing enough 2127 * deferred pages to satisfy the allocation specified by order, rounded up to 2128 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 2129 * of SECTION_SIZE bytes by initializing struct pages in increments of 2130 * PAGES_PER_SECTION * sizeof(struct page) bytes. 2131 * 2132 * Return true when zone was grown, otherwise return false. We return true even 2133 * when we grow less than requested, to let the caller decide if there are 2134 * enough pages to satisfy the allocation. 2135 * 2136 * Note: We use noinline because this function is needed only during boot, and 2137 * it is called from a __ref function _deferred_grow_zone. This way we are 2138 * making sure that it is not inlined into permanent text section. 2139 */ 2140 static noinline bool __init 2141 deferred_grow_zone(struct zone *zone, unsigned int order) 2142 { 2143 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 2144 pg_data_t *pgdat = zone->zone_pgdat; 2145 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 2146 unsigned long spfn, epfn, flags; 2147 unsigned long nr_pages = 0; 2148 u64 i; 2149 2150 /* Only the last zone may have deferred pages */ 2151 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 2152 return false; 2153 2154 pgdat_resize_lock(pgdat, &flags); 2155 2156 /* 2157 * If someone grew this zone while we were waiting for spinlock, return 2158 * true, as there might be enough pages already. 2159 */ 2160 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 2161 pgdat_resize_unlock(pgdat, &flags); 2162 return true; 2163 } 2164 2165 /* If the zone is empty somebody else may have cleared out the zone */ 2166 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2167 first_deferred_pfn)) { 2168 pgdat->first_deferred_pfn = ULONG_MAX; 2169 pgdat_resize_unlock(pgdat, &flags); 2170 /* Retry only once. */ 2171 return first_deferred_pfn != ULONG_MAX; 2172 } 2173 2174 /* 2175 * Initialize and free pages in MAX_ORDER sized increments so 2176 * that we can avoid introducing any issues with the buddy 2177 * allocator. 2178 */ 2179 while (spfn < epfn) { 2180 /* update our first deferred PFN for this section */ 2181 first_deferred_pfn = spfn; 2182 2183 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 2184 touch_nmi_watchdog(); 2185 2186 /* We should only stop along section boundaries */ 2187 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) 2188 continue; 2189 2190 /* If our quota has been met we can stop here */ 2191 if (nr_pages >= nr_pages_needed) 2192 break; 2193 } 2194 2195 pgdat->first_deferred_pfn = spfn; 2196 pgdat_resize_unlock(pgdat, &flags); 2197 2198 return nr_pages > 0; 2199 } 2200 2201 /* 2202 * deferred_grow_zone() is __init, but it is called from 2203 * get_page_from_freelist() during early boot until deferred_pages permanently 2204 * disables this call. This is why we have refdata wrapper to avoid warning, 2205 * and to ensure that the function body gets unloaded. 2206 */ 2207 static bool __ref 2208 _deferred_grow_zone(struct zone *zone, unsigned int order) 2209 { 2210 return deferred_grow_zone(zone, order); 2211 } 2212 2213 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 2214 2215 void __init page_alloc_init_late(void) 2216 { 2217 struct zone *zone; 2218 int nid; 2219 2220 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 2221 2222 /* There will be num_node_state(N_MEMORY) threads */ 2223 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 2224 for_each_node_state(nid, N_MEMORY) { 2225 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 2226 } 2227 2228 /* Block until all are initialised */ 2229 wait_for_completion(&pgdat_init_all_done_comp); 2230 2231 /* 2232 * We initialized the rest of the deferred pages. Permanently disable 2233 * on-demand struct page initialization. 2234 */ 2235 static_branch_disable(&deferred_pages); 2236 2237 /* Reinit limits that are based on free pages after the kernel is up */ 2238 files_maxfiles_init(); 2239 #endif 2240 2241 buffer_init(); 2242 2243 /* Discard memblock private memory */ 2244 memblock_discard(); 2245 2246 for_each_node_state(nid, N_MEMORY) 2247 shuffle_free_memory(NODE_DATA(nid)); 2248 2249 for_each_populated_zone(zone) 2250 set_zone_contiguous(zone); 2251 } 2252 2253 #ifdef CONFIG_CMA 2254 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 2255 void __init init_cma_reserved_pageblock(struct page *page) 2256 { 2257 unsigned i = pageblock_nr_pages; 2258 struct page *p = page; 2259 2260 do { 2261 __ClearPageReserved(p); 2262 set_page_count(p, 0); 2263 } while (++p, --i); 2264 2265 set_pageblock_migratetype(page, MIGRATE_CMA); 2266 2267 if (pageblock_order >= MAX_ORDER) { 2268 i = pageblock_nr_pages; 2269 p = page; 2270 do { 2271 set_page_refcounted(p); 2272 __free_pages(p, MAX_ORDER - 1); 2273 p += MAX_ORDER_NR_PAGES; 2274 } while (i -= MAX_ORDER_NR_PAGES); 2275 } else { 2276 set_page_refcounted(page); 2277 __free_pages(page, pageblock_order); 2278 } 2279 2280 adjust_managed_page_count(page, pageblock_nr_pages); 2281 page_zone(page)->cma_pages += pageblock_nr_pages; 2282 } 2283 #endif 2284 2285 /* 2286 * The order of subdivision here is critical for the IO subsystem. 2287 * Please do not alter this order without good reasons and regression 2288 * testing. Specifically, as large blocks of memory are subdivided, 2289 * the order in which smaller blocks are delivered depends on the order 2290 * they're subdivided in this function. This is the primary factor 2291 * influencing the order in which pages are delivered to the IO 2292 * subsystem according to empirical testing, and this is also justified 2293 * by considering the behavior of a buddy system containing a single 2294 * large block of memory acted on by a series of small allocations. 2295 * This behavior is a critical factor in sglist merging's success. 2296 * 2297 * -- nyc 2298 */ 2299 static inline void expand(struct zone *zone, struct page *page, 2300 int low, int high, int migratetype) 2301 { 2302 unsigned long size = 1 << high; 2303 2304 while (high > low) { 2305 high--; 2306 size >>= 1; 2307 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 2308 2309 /* 2310 * Mark as guard pages (or page), that will allow to 2311 * merge back to allocator when buddy will be freed. 2312 * Corresponding page table entries will not be touched, 2313 * pages will stay not present in virtual address space 2314 */ 2315 if (set_page_guard(zone, &page[size], high, migratetype)) 2316 continue; 2317 2318 add_to_free_list(&page[size], zone, high, migratetype); 2319 set_buddy_order(&page[size], high); 2320 } 2321 } 2322 2323 static void check_new_page_bad(struct page *page) 2324 { 2325 if (unlikely(page->flags & __PG_HWPOISON)) { 2326 /* Don't complain about hwpoisoned pages */ 2327 page_mapcount_reset(page); /* remove PageBuddy */ 2328 return; 2329 } 2330 2331 bad_page(page, 2332 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP)); 2333 } 2334 2335 /* 2336 * This page is about to be returned from the page allocator 2337 */ 2338 static inline int check_new_page(struct page *page) 2339 { 2340 if (likely(page_expected_state(page, 2341 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 2342 return 0; 2343 2344 check_new_page_bad(page); 2345 return 1; 2346 } 2347 2348 #ifdef CONFIG_DEBUG_VM 2349 /* 2350 * With DEBUG_VM enabled, order-0 pages are checked for expected state when 2351 * being allocated from pcp lists. With debug_pagealloc also enabled, they are 2352 * also checked when pcp lists are refilled from the free lists. 2353 */ 2354 static inline bool check_pcp_refill(struct page *page) 2355 { 2356 if (debug_pagealloc_enabled_static()) 2357 return check_new_page(page); 2358 else 2359 return false; 2360 } 2361 2362 static inline bool check_new_pcp(struct page *page) 2363 { 2364 return check_new_page(page); 2365 } 2366 #else 2367 /* 2368 * With DEBUG_VM disabled, free order-0 pages are checked for expected state 2369 * when pcp lists are being refilled from the free lists. With debug_pagealloc 2370 * enabled, they are also checked when being allocated from the pcp lists. 2371 */ 2372 static inline bool check_pcp_refill(struct page *page) 2373 { 2374 return check_new_page(page); 2375 } 2376 static inline bool check_new_pcp(struct page *page) 2377 { 2378 if (debug_pagealloc_enabled_static()) 2379 return check_new_page(page); 2380 else 2381 return false; 2382 } 2383 #endif /* CONFIG_DEBUG_VM */ 2384 2385 static bool check_new_pages(struct page *page, unsigned int order) 2386 { 2387 int i; 2388 for (i = 0; i < (1 << order); i++) { 2389 struct page *p = page + i; 2390 2391 if (unlikely(check_new_page(p))) 2392 return true; 2393 } 2394 2395 return false; 2396 } 2397 2398 inline void post_alloc_hook(struct page *page, unsigned int order, 2399 gfp_t gfp_flags) 2400 { 2401 set_page_private(page, 0); 2402 set_page_refcounted(page); 2403 2404 arch_alloc_page(page, order); 2405 debug_pagealloc_map_pages(page, 1 << order); 2406 2407 /* 2408 * Page unpoisoning must happen before memory initialization. 2409 * Otherwise, the poison pattern will be overwritten for __GFP_ZERO 2410 * allocations and the page unpoisoning code will complain. 2411 */ 2412 kernel_unpoison_pages(page, 1 << order); 2413 2414 /* 2415 * As memory initialization might be integrated into KASAN, 2416 * kasan_alloc_pages and kernel_init_free_pages must be 2417 * kept together to avoid discrepancies in behavior. 2418 */ 2419 if (kasan_has_integrated_init()) { 2420 kasan_alloc_pages(page, order, gfp_flags); 2421 } else { 2422 bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags); 2423 2424 kasan_unpoison_pages(page, order, init); 2425 if (init) 2426 kernel_init_free_pages(page, 1 << order, 2427 gfp_flags & __GFP_ZEROTAGS); 2428 } 2429 2430 set_page_owner(page, order, gfp_flags); 2431 } 2432 2433 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 2434 unsigned int alloc_flags) 2435 { 2436 post_alloc_hook(page, order, gfp_flags); 2437 2438 if (order && (gfp_flags & __GFP_COMP)) 2439 prep_compound_page(page, order); 2440 2441 /* 2442 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 2443 * allocate the page. The expectation is that the caller is taking 2444 * steps that will free more memory. The caller should avoid the page 2445 * being used for !PFMEMALLOC purposes. 2446 */ 2447 if (alloc_flags & ALLOC_NO_WATERMARKS) 2448 set_page_pfmemalloc(page); 2449 else 2450 clear_page_pfmemalloc(page); 2451 } 2452 2453 /* 2454 * Go through the free lists for the given migratetype and remove 2455 * the smallest available page from the freelists 2456 */ 2457 static __always_inline 2458 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 2459 int migratetype) 2460 { 2461 unsigned int current_order; 2462 struct free_area *area; 2463 struct page *page; 2464 2465 /* Find a page of the appropriate size in the preferred list */ 2466 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 2467 area = &(zone->free_area[current_order]); 2468 page = get_page_from_free_area(area, migratetype); 2469 if (!page) 2470 continue; 2471 del_page_from_free_list(page, zone, current_order); 2472 expand(zone, page, order, current_order, migratetype); 2473 set_pcppage_migratetype(page, migratetype); 2474 return page; 2475 } 2476 2477 return NULL; 2478 } 2479 2480 2481 /* 2482 * This array describes the order lists are fallen back to when 2483 * the free lists for the desirable migrate type are depleted 2484 */ 2485 static int fallbacks[MIGRATE_TYPES][3] = { 2486 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2487 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 2488 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2489 #ifdef CONFIG_CMA 2490 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */ 2491 #endif 2492 #ifdef CONFIG_MEMORY_ISOLATION 2493 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */ 2494 #endif 2495 }; 2496 2497 #ifdef CONFIG_CMA 2498 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2499 unsigned int order) 2500 { 2501 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 2502 } 2503 #else 2504 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2505 unsigned int order) { return NULL; } 2506 #endif 2507 2508 /* 2509 * Move the free pages in a range to the freelist tail of the requested type. 2510 * Note that start_page and end_pages are not aligned on a pageblock 2511 * boundary. If alignment is required, use move_freepages_block() 2512 */ 2513 static int move_freepages(struct zone *zone, 2514 unsigned long start_pfn, unsigned long end_pfn, 2515 int migratetype, int *num_movable) 2516 { 2517 struct page *page; 2518 unsigned long pfn; 2519 unsigned int order; 2520 int pages_moved = 0; 2521 2522 for (pfn = start_pfn; pfn <= end_pfn;) { 2523 if (!pfn_valid_within(pfn)) { 2524 pfn++; 2525 continue; 2526 } 2527 2528 page = pfn_to_page(pfn); 2529 if (!PageBuddy(page)) { 2530 /* 2531 * We assume that pages that could be isolated for 2532 * migration are movable. But we don't actually try 2533 * isolating, as that would be expensive. 2534 */ 2535 if (num_movable && 2536 (PageLRU(page) || __PageMovable(page))) 2537 (*num_movable)++; 2538 pfn++; 2539 continue; 2540 } 2541 2542 /* Make sure we are not inadvertently changing nodes */ 2543 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2544 VM_BUG_ON_PAGE(page_zone(page) != zone, page); 2545 2546 order = buddy_order(page); 2547 move_to_free_list(page, zone, order, migratetype); 2548 pfn += 1 << order; 2549 pages_moved += 1 << order; 2550 } 2551 2552 return pages_moved; 2553 } 2554 2555 int move_freepages_block(struct zone *zone, struct page *page, 2556 int migratetype, int *num_movable) 2557 { 2558 unsigned long start_pfn, end_pfn, pfn; 2559 2560 if (num_movable) 2561 *num_movable = 0; 2562 2563 pfn = page_to_pfn(page); 2564 start_pfn = pfn & ~(pageblock_nr_pages - 1); 2565 end_pfn = start_pfn + pageblock_nr_pages - 1; 2566 2567 /* Do not cross zone boundaries */ 2568 if (!zone_spans_pfn(zone, start_pfn)) 2569 start_pfn = pfn; 2570 if (!zone_spans_pfn(zone, end_pfn)) 2571 return 0; 2572 2573 return move_freepages(zone, start_pfn, end_pfn, migratetype, 2574 num_movable); 2575 } 2576 2577 static void change_pageblock_range(struct page *pageblock_page, 2578 int start_order, int migratetype) 2579 { 2580 int nr_pageblocks = 1 << (start_order - pageblock_order); 2581 2582 while (nr_pageblocks--) { 2583 set_pageblock_migratetype(pageblock_page, migratetype); 2584 pageblock_page += pageblock_nr_pages; 2585 } 2586 } 2587 2588 /* 2589 * When we are falling back to another migratetype during allocation, try to 2590 * steal extra free pages from the same pageblocks to satisfy further 2591 * allocations, instead of polluting multiple pageblocks. 2592 * 2593 * If we are stealing a relatively large buddy page, it is likely there will 2594 * be more free pages in the pageblock, so try to steal them all. For 2595 * reclaimable and unmovable allocations, we steal regardless of page size, 2596 * as fragmentation caused by those allocations polluting movable pageblocks 2597 * is worse than movable allocations stealing from unmovable and reclaimable 2598 * pageblocks. 2599 */ 2600 static bool can_steal_fallback(unsigned int order, int start_mt) 2601 { 2602 /* 2603 * Leaving this order check is intended, although there is 2604 * relaxed order check in next check. The reason is that 2605 * we can actually steal whole pageblock if this condition met, 2606 * but, below check doesn't guarantee it and that is just heuristic 2607 * so could be changed anytime. 2608 */ 2609 if (order >= pageblock_order) 2610 return true; 2611 2612 if (order >= pageblock_order / 2 || 2613 start_mt == MIGRATE_RECLAIMABLE || 2614 start_mt == MIGRATE_UNMOVABLE || 2615 page_group_by_mobility_disabled) 2616 return true; 2617 2618 return false; 2619 } 2620 2621 static inline bool boost_watermark(struct zone *zone) 2622 { 2623 unsigned long max_boost; 2624 2625 if (!watermark_boost_factor) 2626 return false; 2627 /* 2628 * Don't bother in zones that are unlikely to produce results. 2629 * On small machines, including kdump capture kernels running 2630 * in a small area, boosting the watermark can cause an out of 2631 * memory situation immediately. 2632 */ 2633 if ((pageblock_nr_pages * 4) > zone_managed_pages(zone)) 2634 return false; 2635 2636 max_boost = mult_frac(zone->_watermark[WMARK_HIGH], 2637 watermark_boost_factor, 10000); 2638 2639 /* 2640 * high watermark may be uninitialised if fragmentation occurs 2641 * very early in boot so do not boost. We do not fall 2642 * through and boost by pageblock_nr_pages as failing 2643 * allocations that early means that reclaim is not going 2644 * to help and it may even be impossible to reclaim the 2645 * boosted watermark resulting in a hang. 2646 */ 2647 if (!max_boost) 2648 return false; 2649 2650 max_boost = max(pageblock_nr_pages, max_boost); 2651 2652 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages, 2653 max_boost); 2654 2655 return true; 2656 } 2657 2658 /* 2659 * This function implements actual steal behaviour. If order is large enough, 2660 * we can steal whole pageblock. If not, we first move freepages in this 2661 * pageblock to our migratetype and determine how many already-allocated pages 2662 * are there in the pageblock with a compatible migratetype. If at least half 2663 * of pages are free or compatible, we can change migratetype of the pageblock 2664 * itself, so pages freed in the future will be put on the correct free list. 2665 */ 2666 static void steal_suitable_fallback(struct zone *zone, struct page *page, 2667 unsigned int alloc_flags, int start_type, bool whole_block) 2668 { 2669 unsigned int current_order = buddy_order(page); 2670 int free_pages, movable_pages, alike_pages; 2671 int old_block_type; 2672 2673 old_block_type = get_pageblock_migratetype(page); 2674 2675 /* 2676 * This can happen due to races and we want to prevent broken 2677 * highatomic accounting. 2678 */ 2679 if (is_migrate_highatomic(old_block_type)) 2680 goto single_page; 2681 2682 /* Take ownership for orders >= pageblock_order */ 2683 if (current_order >= pageblock_order) { 2684 change_pageblock_range(page, current_order, start_type); 2685 goto single_page; 2686 } 2687 2688 /* 2689 * Boost watermarks to increase reclaim pressure to reduce the 2690 * likelihood of future fallbacks. Wake kswapd now as the node 2691 * may be balanced overall and kswapd will not wake naturally. 2692 */ 2693 if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD)) 2694 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 2695 2696 /* We are not allowed to try stealing from the whole block */ 2697 if (!whole_block) 2698 goto single_page; 2699 2700 free_pages = move_freepages_block(zone, page, start_type, 2701 &movable_pages); 2702 /* 2703 * Determine how many pages are compatible with our allocation. 2704 * For movable allocation, it's the number of movable pages which 2705 * we just obtained. For other types it's a bit more tricky. 2706 */ 2707 if (start_type == MIGRATE_MOVABLE) { 2708 alike_pages = movable_pages; 2709 } else { 2710 /* 2711 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation 2712 * to MOVABLE pageblock, consider all non-movable pages as 2713 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or 2714 * vice versa, be conservative since we can't distinguish the 2715 * exact migratetype of non-movable pages. 2716 */ 2717 if (old_block_type == MIGRATE_MOVABLE) 2718 alike_pages = pageblock_nr_pages 2719 - (free_pages + movable_pages); 2720 else 2721 alike_pages = 0; 2722 } 2723 2724 /* moving whole block can fail due to zone boundary conditions */ 2725 if (!free_pages) 2726 goto single_page; 2727 2728 /* 2729 * If a sufficient number of pages in the block are either free or of 2730 * comparable migratability as our allocation, claim the whole block. 2731 */ 2732 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || 2733 page_group_by_mobility_disabled) 2734 set_pageblock_migratetype(page, start_type); 2735 2736 return; 2737 2738 single_page: 2739 move_to_free_list(page, zone, current_order, start_type); 2740 } 2741 2742 /* 2743 * Check whether there is a suitable fallback freepage with requested order. 2744 * If only_stealable is true, this function returns fallback_mt only if 2745 * we can steal other freepages all together. This would help to reduce 2746 * fragmentation due to mixed migratetype pages in one pageblock. 2747 */ 2748 int find_suitable_fallback(struct free_area *area, unsigned int order, 2749 int migratetype, bool only_stealable, bool *can_steal) 2750 { 2751 int i; 2752 int fallback_mt; 2753 2754 if (area->nr_free == 0) 2755 return -1; 2756 2757 *can_steal = false; 2758 for (i = 0;; i++) { 2759 fallback_mt = fallbacks[migratetype][i]; 2760 if (fallback_mt == MIGRATE_TYPES) 2761 break; 2762 2763 if (free_area_empty(area, fallback_mt)) 2764 continue; 2765 2766 if (can_steal_fallback(order, migratetype)) 2767 *can_steal = true; 2768 2769 if (!only_stealable) 2770 return fallback_mt; 2771 2772 if (*can_steal) 2773 return fallback_mt; 2774 } 2775 2776 return -1; 2777 } 2778 2779 /* 2780 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2781 * there are no empty page blocks that contain a page with a suitable order 2782 */ 2783 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2784 unsigned int alloc_order) 2785 { 2786 int mt; 2787 unsigned long max_managed, flags; 2788 2789 /* 2790 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2791 * Check is race-prone but harmless. 2792 */ 2793 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages; 2794 if (zone->nr_reserved_highatomic >= max_managed) 2795 return; 2796 2797 spin_lock_irqsave(&zone->lock, flags); 2798 2799 /* Recheck the nr_reserved_highatomic limit under the lock */ 2800 if (zone->nr_reserved_highatomic >= max_managed) 2801 goto out_unlock; 2802 2803 /* Yoink! */ 2804 mt = get_pageblock_migratetype(page); 2805 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt) 2806 && !is_migrate_cma(mt)) { 2807 zone->nr_reserved_highatomic += pageblock_nr_pages; 2808 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2809 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); 2810 } 2811 2812 out_unlock: 2813 spin_unlock_irqrestore(&zone->lock, flags); 2814 } 2815 2816 /* 2817 * Used when an allocation is about to fail under memory pressure. This 2818 * potentially hurts the reliability of high-order allocations when under 2819 * intense memory pressure but failed atomic allocations should be easier 2820 * to recover from than an OOM. 2821 * 2822 * If @force is true, try to unreserve a pageblock even though highatomic 2823 * pageblock is exhausted. 2824 */ 2825 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 2826 bool force) 2827 { 2828 struct zonelist *zonelist = ac->zonelist; 2829 unsigned long flags; 2830 struct zoneref *z; 2831 struct zone *zone; 2832 struct page *page; 2833 int order; 2834 bool ret; 2835 2836 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx, 2837 ac->nodemask) { 2838 /* 2839 * Preserve at least one pageblock unless memory pressure 2840 * is really high. 2841 */ 2842 if (!force && zone->nr_reserved_highatomic <= 2843 pageblock_nr_pages) 2844 continue; 2845 2846 spin_lock_irqsave(&zone->lock, flags); 2847 for (order = 0; order < MAX_ORDER; order++) { 2848 struct free_area *area = &(zone->free_area[order]); 2849 2850 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); 2851 if (!page) 2852 continue; 2853 2854 /* 2855 * In page freeing path, migratetype change is racy so 2856 * we can counter several free pages in a pageblock 2857 * in this loop although we changed the pageblock type 2858 * from highatomic to ac->migratetype. So we should 2859 * adjust the count once. 2860 */ 2861 if (is_migrate_highatomic_page(page)) { 2862 /* 2863 * It should never happen but changes to 2864 * locking could inadvertently allow a per-cpu 2865 * drain to add pages to MIGRATE_HIGHATOMIC 2866 * while unreserving so be safe and watch for 2867 * underflows. 2868 */ 2869 zone->nr_reserved_highatomic -= min( 2870 pageblock_nr_pages, 2871 zone->nr_reserved_highatomic); 2872 } 2873 2874 /* 2875 * Convert to ac->migratetype and avoid the normal 2876 * pageblock stealing heuristics. Minimally, the caller 2877 * is doing the work and needs the pages. More 2878 * importantly, if the block was always converted to 2879 * MIGRATE_UNMOVABLE or another type then the number 2880 * of pageblocks that cannot be completely freed 2881 * may increase. 2882 */ 2883 set_pageblock_migratetype(page, ac->migratetype); 2884 ret = move_freepages_block(zone, page, ac->migratetype, 2885 NULL); 2886 if (ret) { 2887 spin_unlock_irqrestore(&zone->lock, flags); 2888 return ret; 2889 } 2890 } 2891 spin_unlock_irqrestore(&zone->lock, flags); 2892 } 2893 2894 return false; 2895 } 2896 2897 /* 2898 * Try finding a free buddy page on the fallback list and put it on the free 2899 * list of requested migratetype, possibly along with other pages from the same 2900 * block, depending on fragmentation avoidance heuristics. Returns true if 2901 * fallback was found so that __rmqueue_smallest() can grab it. 2902 * 2903 * The use of signed ints for order and current_order is a deliberate 2904 * deviation from the rest of this file, to make the for loop 2905 * condition simpler. 2906 */ 2907 static __always_inline bool 2908 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, 2909 unsigned int alloc_flags) 2910 { 2911 struct free_area *area; 2912 int current_order; 2913 int min_order = order; 2914 struct page *page; 2915 int fallback_mt; 2916 bool can_steal; 2917 2918 /* 2919 * Do not steal pages from freelists belonging to other pageblocks 2920 * i.e. orders < pageblock_order. If there are no local zones free, 2921 * the zonelists will be reiterated without ALLOC_NOFRAGMENT. 2922 */ 2923 if (alloc_flags & ALLOC_NOFRAGMENT) 2924 min_order = pageblock_order; 2925 2926 /* 2927 * Find the largest available free page in the other list. This roughly 2928 * approximates finding the pageblock with the most free pages, which 2929 * would be too costly to do exactly. 2930 */ 2931 for (current_order = MAX_ORDER - 1; current_order >= min_order; 2932 --current_order) { 2933 area = &(zone->free_area[current_order]); 2934 fallback_mt = find_suitable_fallback(area, current_order, 2935 start_migratetype, false, &can_steal); 2936 if (fallback_mt == -1) 2937 continue; 2938 2939 /* 2940 * We cannot steal all free pages from the pageblock and the 2941 * requested migratetype is movable. In that case it's better to 2942 * steal and split the smallest available page instead of the 2943 * largest available page, because even if the next movable 2944 * allocation falls back into a different pageblock than this 2945 * one, it won't cause permanent fragmentation. 2946 */ 2947 if (!can_steal && start_migratetype == MIGRATE_MOVABLE 2948 && current_order > order) 2949 goto find_smallest; 2950 2951 goto do_steal; 2952 } 2953 2954 return false; 2955 2956 find_smallest: 2957 for (current_order = order; current_order < MAX_ORDER; 2958 current_order++) { 2959 area = &(zone->free_area[current_order]); 2960 fallback_mt = find_suitable_fallback(area, current_order, 2961 start_migratetype, false, &can_steal); 2962 if (fallback_mt != -1) 2963 break; 2964 } 2965 2966 /* 2967 * This should not happen - we already found a suitable fallback 2968 * when looking for the largest page. 2969 */ 2970 VM_BUG_ON(current_order == MAX_ORDER); 2971 2972 do_steal: 2973 page = get_page_from_free_area(area, fallback_mt); 2974 2975 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype, 2976 can_steal); 2977 2978 trace_mm_page_alloc_extfrag(page, order, current_order, 2979 start_migratetype, fallback_mt); 2980 2981 return true; 2982 2983 } 2984 2985 /* 2986 * Do the hard work of removing an element from the buddy allocator. 2987 * Call me with the zone->lock already held. 2988 */ 2989 static __always_inline struct page * 2990 __rmqueue(struct zone *zone, unsigned int order, int migratetype, 2991 unsigned int alloc_flags) 2992 { 2993 struct page *page; 2994 2995 if (IS_ENABLED(CONFIG_CMA)) { 2996 /* 2997 * Balance movable allocations between regular and CMA areas by 2998 * allocating from CMA when over half of the zone's free memory 2999 * is in the CMA area. 3000 */ 3001 if (alloc_flags & ALLOC_CMA && 3002 zone_page_state(zone, NR_FREE_CMA_PAGES) > 3003 zone_page_state(zone, NR_FREE_PAGES) / 2) { 3004 page = __rmqueue_cma_fallback(zone, order); 3005 if (page) 3006 goto out; 3007 } 3008 } 3009 retry: 3010 page = __rmqueue_smallest(zone, order, migratetype); 3011 if (unlikely(!page)) { 3012 if (alloc_flags & ALLOC_CMA) 3013 page = __rmqueue_cma_fallback(zone, order); 3014 3015 if (!page && __rmqueue_fallback(zone, order, migratetype, 3016 alloc_flags)) 3017 goto retry; 3018 } 3019 out: 3020 if (page) 3021 trace_mm_page_alloc_zone_locked(page, order, migratetype); 3022 return page; 3023 } 3024 3025 /* 3026 * Obtain a specified number of elements from the buddy allocator, all under 3027 * a single hold of the lock, for efficiency. Add them to the supplied list. 3028 * Returns the number of new pages which were placed at *list. 3029 */ 3030 static int rmqueue_bulk(struct zone *zone, unsigned int order, 3031 unsigned long count, struct list_head *list, 3032 int migratetype, unsigned int alloc_flags) 3033 { 3034 int i, allocated = 0; 3035 3036 /* 3037 * local_lock_irq held so equivalent to spin_lock_irqsave for 3038 * both PREEMPT_RT and non-PREEMPT_RT configurations. 3039 */ 3040 spin_lock(&zone->lock); 3041 for (i = 0; i < count; ++i) { 3042 struct page *page = __rmqueue(zone, order, migratetype, 3043 alloc_flags); 3044 if (unlikely(page == NULL)) 3045 break; 3046 3047 if (unlikely(check_pcp_refill(page))) 3048 continue; 3049 3050 /* 3051 * Split buddy pages returned by expand() are received here in 3052 * physical page order. The page is added to the tail of 3053 * caller's list. From the callers perspective, the linked list 3054 * is ordered by page number under some conditions. This is 3055 * useful for IO devices that can forward direction from the 3056 * head, thus also in the physical page order. This is useful 3057 * for IO devices that can merge IO requests if the physical 3058 * pages are ordered properly. 3059 */ 3060 list_add_tail(&page->lru, list); 3061 allocated++; 3062 if (is_migrate_cma(get_pcppage_migratetype(page))) 3063 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 3064 -(1 << order)); 3065 } 3066 3067 /* 3068 * i pages were removed from the buddy list even if some leak due 3069 * to check_pcp_refill failing so adjust NR_FREE_PAGES based 3070 * on i. Do not confuse with 'allocated' which is the number of 3071 * pages added to the pcp list. 3072 */ 3073 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 3074 spin_unlock(&zone->lock); 3075 return allocated; 3076 } 3077 3078 #ifdef CONFIG_NUMA 3079 /* 3080 * Called from the vmstat counter updater to drain pagesets of this 3081 * currently executing processor on remote nodes after they have 3082 * expired. 3083 * 3084 * Note that this function must be called with the thread pinned to 3085 * a single processor. 3086 */ 3087 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 3088 { 3089 unsigned long flags; 3090 int to_drain, batch; 3091 3092 local_lock_irqsave(&pagesets.lock, flags); 3093 batch = READ_ONCE(pcp->batch); 3094 to_drain = min(pcp->count, batch); 3095 if (to_drain > 0) 3096 free_pcppages_bulk(zone, to_drain, pcp); 3097 local_unlock_irqrestore(&pagesets.lock, flags); 3098 } 3099 #endif 3100 3101 /* 3102 * Drain pcplists of the indicated processor and zone. 3103 * 3104 * The processor must either be the current processor and the 3105 * thread pinned to the current processor or a processor that 3106 * is not online. 3107 */ 3108 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 3109 { 3110 unsigned long flags; 3111 struct per_cpu_pages *pcp; 3112 3113 local_lock_irqsave(&pagesets.lock, flags); 3114 3115 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 3116 if (pcp->count) 3117 free_pcppages_bulk(zone, pcp->count, pcp); 3118 3119 local_unlock_irqrestore(&pagesets.lock, flags); 3120 } 3121 3122 /* 3123 * Drain pcplists of all zones on the indicated processor. 3124 * 3125 * The processor must either be the current processor and the 3126 * thread pinned to the current processor or a processor that 3127 * is not online. 3128 */ 3129 static void drain_pages(unsigned int cpu) 3130 { 3131 struct zone *zone; 3132 3133 for_each_populated_zone(zone) { 3134 drain_pages_zone(cpu, zone); 3135 } 3136 } 3137 3138 /* 3139 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 3140 * 3141 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 3142 * the single zone's pages. 3143 */ 3144 void drain_local_pages(struct zone *zone) 3145 { 3146 int cpu = smp_processor_id(); 3147 3148 if (zone) 3149 drain_pages_zone(cpu, zone); 3150 else 3151 drain_pages(cpu); 3152 } 3153 3154 static void drain_local_pages_wq(struct work_struct *work) 3155 { 3156 struct pcpu_drain *drain; 3157 3158 drain = container_of(work, struct pcpu_drain, work); 3159 3160 /* 3161 * drain_all_pages doesn't use proper cpu hotplug protection so 3162 * we can race with cpu offline when the WQ can move this from 3163 * a cpu pinned worker to an unbound one. We can operate on a different 3164 * cpu which is alright but we also have to make sure to not move to 3165 * a different one. 3166 */ 3167 preempt_disable(); 3168 drain_local_pages(drain->zone); 3169 preempt_enable(); 3170 } 3171 3172 /* 3173 * The implementation of drain_all_pages(), exposing an extra parameter to 3174 * drain on all cpus. 3175 * 3176 * drain_all_pages() is optimized to only execute on cpus where pcplists are 3177 * not empty. The check for non-emptiness can however race with a free to 3178 * pcplist that has not yet increased the pcp->count from 0 to 1. Callers 3179 * that need the guarantee that every CPU has drained can disable the 3180 * optimizing racy check. 3181 */ 3182 static void __drain_all_pages(struct zone *zone, bool force_all_cpus) 3183 { 3184 int cpu; 3185 3186 /* 3187 * Allocate in the BSS so we won't require allocation in 3188 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 3189 */ 3190 static cpumask_t cpus_with_pcps; 3191 3192 /* 3193 * Make sure nobody triggers this path before mm_percpu_wq is fully 3194 * initialized. 3195 */ 3196 if (WARN_ON_ONCE(!mm_percpu_wq)) 3197 return; 3198 3199 /* 3200 * Do not drain if one is already in progress unless it's specific to 3201 * a zone. Such callers are primarily CMA and memory hotplug and need 3202 * the drain to be complete when the call returns. 3203 */ 3204 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) { 3205 if (!zone) 3206 return; 3207 mutex_lock(&pcpu_drain_mutex); 3208 } 3209 3210 /* 3211 * We don't care about racing with CPU hotplug event 3212 * as offline notification will cause the notified 3213 * cpu to drain that CPU pcps and on_each_cpu_mask 3214 * disables preemption as part of its processing 3215 */ 3216 for_each_online_cpu(cpu) { 3217 struct per_cpu_pages *pcp; 3218 struct zone *z; 3219 bool has_pcps = false; 3220 3221 if (force_all_cpus) { 3222 /* 3223 * The pcp.count check is racy, some callers need a 3224 * guarantee that no cpu is missed. 3225 */ 3226 has_pcps = true; 3227 } else if (zone) { 3228 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 3229 if (pcp->count) 3230 has_pcps = true; 3231 } else { 3232 for_each_populated_zone(z) { 3233 pcp = per_cpu_ptr(z->per_cpu_pageset, cpu); 3234 if (pcp->count) { 3235 has_pcps = true; 3236 break; 3237 } 3238 } 3239 } 3240 3241 if (has_pcps) 3242 cpumask_set_cpu(cpu, &cpus_with_pcps); 3243 else 3244 cpumask_clear_cpu(cpu, &cpus_with_pcps); 3245 } 3246 3247 for_each_cpu(cpu, &cpus_with_pcps) { 3248 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu); 3249 3250 drain->zone = zone; 3251 INIT_WORK(&drain->work, drain_local_pages_wq); 3252 queue_work_on(cpu, mm_percpu_wq, &drain->work); 3253 } 3254 for_each_cpu(cpu, &cpus_with_pcps) 3255 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work); 3256 3257 mutex_unlock(&pcpu_drain_mutex); 3258 } 3259 3260 /* 3261 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 3262 * 3263 * When zone parameter is non-NULL, spill just the single zone's pages. 3264 * 3265 * Note that this can be extremely slow as the draining happens in a workqueue. 3266 */ 3267 void drain_all_pages(struct zone *zone) 3268 { 3269 __drain_all_pages(zone, false); 3270 } 3271 3272 #ifdef CONFIG_HIBERNATION 3273 3274 /* 3275 * Touch the watchdog for every WD_PAGE_COUNT pages. 3276 */ 3277 #define WD_PAGE_COUNT (128*1024) 3278 3279 void mark_free_pages(struct zone *zone) 3280 { 3281 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT; 3282 unsigned long flags; 3283 unsigned int order, t; 3284 struct page *page; 3285 3286 if (zone_is_empty(zone)) 3287 return; 3288 3289 spin_lock_irqsave(&zone->lock, flags); 3290 3291 max_zone_pfn = zone_end_pfn(zone); 3292 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 3293 if (pfn_valid(pfn)) { 3294 page = pfn_to_page(pfn); 3295 3296 if (!--page_count) { 3297 touch_nmi_watchdog(); 3298 page_count = WD_PAGE_COUNT; 3299 } 3300 3301 if (page_zone(page) != zone) 3302 continue; 3303 3304 if (!swsusp_page_is_forbidden(page)) 3305 swsusp_unset_page_free(page); 3306 } 3307 3308 for_each_migratetype_order(order, t) { 3309 list_for_each_entry(page, 3310 &zone->free_area[order].free_list[t], lru) { 3311 unsigned long i; 3312 3313 pfn = page_to_pfn(page); 3314 for (i = 0; i < (1UL << order); i++) { 3315 if (!--page_count) { 3316 touch_nmi_watchdog(); 3317 page_count = WD_PAGE_COUNT; 3318 } 3319 swsusp_set_page_free(pfn_to_page(pfn + i)); 3320 } 3321 } 3322 } 3323 spin_unlock_irqrestore(&zone->lock, flags); 3324 } 3325 #endif /* CONFIG_PM */ 3326 3327 static bool free_unref_page_prepare(struct page *page, unsigned long pfn, 3328 unsigned int order) 3329 { 3330 int migratetype; 3331 3332 if (!free_pcp_prepare(page, order)) 3333 return false; 3334 3335 migratetype = get_pfnblock_migratetype(page, pfn); 3336 set_pcppage_migratetype(page, migratetype); 3337 return true; 3338 } 3339 3340 static int nr_pcp_free(struct per_cpu_pages *pcp, int high, int batch) 3341 { 3342 int min_nr_free, max_nr_free; 3343 3344 /* Check for PCP disabled or boot pageset */ 3345 if (unlikely(high < batch)) 3346 return 1; 3347 3348 /* Leave at least pcp->batch pages on the list */ 3349 min_nr_free = batch; 3350 max_nr_free = high - batch; 3351 3352 /* 3353 * Double the number of pages freed each time there is subsequent 3354 * freeing of pages without any allocation. 3355 */ 3356 batch <<= pcp->free_factor; 3357 if (batch < max_nr_free) 3358 pcp->free_factor++; 3359 batch = clamp(batch, min_nr_free, max_nr_free); 3360 3361 return batch; 3362 } 3363 3364 static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone) 3365 { 3366 int high = READ_ONCE(pcp->high); 3367 3368 if (unlikely(!high)) 3369 return 0; 3370 3371 if (!test_bit(ZONE_RECLAIM_ACTIVE, &zone->flags)) 3372 return high; 3373 3374 /* 3375 * If reclaim is active, limit the number of pages that can be 3376 * stored on pcp lists 3377 */ 3378 return min(READ_ONCE(pcp->batch) << 2, high); 3379 } 3380 3381 static void free_unref_page_commit(struct page *page, unsigned long pfn, 3382 int migratetype, unsigned int order) 3383 { 3384 struct zone *zone = page_zone(page); 3385 struct per_cpu_pages *pcp; 3386 int high; 3387 int pindex; 3388 3389 __count_vm_event(PGFREE); 3390 pcp = this_cpu_ptr(zone->per_cpu_pageset); 3391 pindex = order_to_pindex(migratetype, order); 3392 list_add(&page->lru, &pcp->lists[pindex]); 3393 pcp->count += 1 << order; 3394 high = nr_pcp_high(pcp, zone); 3395 if (pcp->count >= high) { 3396 int batch = READ_ONCE(pcp->batch); 3397 3398 free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch), pcp); 3399 } 3400 } 3401 3402 /* 3403 * Free a pcp page 3404 */ 3405 void free_unref_page(struct page *page, unsigned int order) 3406 { 3407 unsigned long flags; 3408 unsigned long pfn = page_to_pfn(page); 3409 int migratetype; 3410 3411 if (!free_unref_page_prepare(page, pfn, order)) 3412 return; 3413 3414 /* 3415 * We only track unmovable, reclaimable and movable on pcp lists. 3416 * Place ISOLATE pages on the isolated list because they are being 3417 * offlined but treat HIGHATOMIC as movable pages so we can get those 3418 * areas back if necessary. Otherwise, we may have to free 3419 * excessively into the page allocator 3420 */ 3421 migratetype = get_pcppage_migratetype(page); 3422 if (unlikely(migratetype >= MIGRATE_PCPTYPES)) { 3423 if (unlikely(is_migrate_isolate(migratetype))) { 3424 free_one_page(page_zone(page), page, pfn, order, migratetype, FPI_NONE); 3425 return; 3426 } 3427 migratetype = MIGRATE_MOVABLE; 3428 } 3429 3430 local_lock_irqsave(&pagesets.lock, flags); 3431 free_unref_page_commit(page, pfn, migratetype, order); 3432 local_unlock_irqrestore(&pagesets.lock, flags); 3433 } 3434 3435 /* 3436 * Free a list of 0-order pages 3437 */ 3438 void free_unref_page_list(struct list_head *list) 3439 { 3440 struct page *page, *next; 3441 unsigned long flags, pfn; 3442 int batch_count = 0; 3443 int migratetype; 3444 3445 /* Prepare pages for freeing */ 3446 list_for_each_entry_safe(page, next, list, lru) { 3447 pfn = page_to_pfn(page); 3448 if (!free_unref_page_prepare(page, pfn, 0)) 3449 list_del(&page->lru); 3450 3451 /* 3452 * Free isolated pages directly to the allocator, see 3453 * comment in free_unref_page. 3454 */ 3455 migratetype = get_pcppage_migratetype(page); 3456 if (unlikely(is_migrate_isolate(migratetype))) { 3457 list_del(&page->lru); 3458 free_one_page(page_zone(page), page, pfn, 0, migratetype, FPI_NONE); 3459 continue; 3460 } 3461 3462 set_page_private(page, pfn); 3463 } 3464 3465 local_lock_irqsave(&pagesets.lock, flags); 3466 list_for_each_entry_safe(page, next, list, lru) { 3467 pfn = page_private(page); 3468 set_page_private(page, 0); 3469 3470 /* 3471 * Non-isolated types over MIGRATE_PCPTYPES get added 3472 * to the MIGRATE_MOVABLE pcp list. 3473 */ 3474 migratetype = get_pcppage_migratetype(page); 3475 if (unlikely(migratetype >= MIGRATE_PCPTYPES)) 3476 migratetype = MIGRATE_MOVABLE; 3477 3478 trace_mm_page_free_batched(page); 3479 free_unref_page_commit(page, pfn, migratetype, 0); 3480 3481 /* 3482 * Guard against excessive IRQ disabled times when we get 3483 * a large list of pages to free. 3484 */ 3485 if (++batch_count == SWAP_CLUSTER_MAX) { 3486 local_unlock_irqrestore(&pagesets.lock, flags); 3487 batch_count = 0; 3488 local_lock_irqsave(&pagesets.lock, flags); 3489 } 3490 } 3491 local_unlock_irqrestore(&pagesets.lock, flags); 3492 } 3493 3494 /* 3495 * split_page takes a non-compound higher-order page, and splits it into 3496 * n (1<<order) sub-pages: page[0..n] 3497 * Each sub-page must be freed individually. 3498 * 3499 * Note: this is probably too low level an operation for use in drivers. 3500 * Please consult with lkml before using this in your driver. 3501 */ 3502 void split_page(struct page *page, unsigned int order) 3503 { 3504 int i; 3505 3506 VM_BUG_ON_PAGE(PageCompound(page), page); 3507 VM_BUG_ON_PAGE(!page_count(page), page); 3508 3509 for (i = 1; i < (1 << order); i++) 3510 set_page_refcounted(page + i); 3511 split_page_owner(page, 1 << order); 3512 split_page_memcg(page, 1 << order); 3513 } 3514 EXPORT_SYMBOL_GPL(split_page); 3515 3516 int __isolate_free_page(struct page *page, unsigned int order) 3517 { 3518 unsigned long watermark; 3519 struct zone *zone; 3520 int mt; 3521 3522 BUG_ON(!PageBuddy(page)); 3523 3524 zone = page_zone(page); 3525 mt = get_pageblock_migratetype(page); 3526 3527 if (!is_migrate_isolate(mt)) { 3528 /* 3529 * Obey watermarks as if the page was being allocated. We can 3530 * emulate a high-order watermark check with a raised order-0 3531 * watermark, because we already know our high-order page 3532 * exists. 3533 */ 3534 watermark = zone->_watermark[WMARK_MIN] + (1UL << order); 3535 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 3536 return 0; 3537 3538 __mod_zone_freepage_state(zone, -(1UL << order), mt); 3539 } 3540 3541 /* Remove page from free list */ 3542 3543 del_page_from_free_list(page, zone, order); 3544 3545 /* 3546 * Set the pageblock if the isolated page is at least half of a 3547 * pageblock 3548 */ 3549 if (order >= pageblock_order - 1) { 3550 struct page *endpage = page + (1 << order) - 1; 3551 for (; page < endpage; page += pageblock_nr_pages) { 3552 int mt = get_pageblock_migratetype(page); 3553 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt) 3554 && !is_migrate_highatomic(mt)) 3555 set_pageblock_migratetype(page, 3556 MIGRATE_MOVABLE); 3557 } 3558 } 3559 3560 3561 return 1UL << order; 3562 } 3563 3564 /** 3565 * __putback_isolated_page - Return a now-isolated page back where we got it 3566 * @page: Page that was isolated 3567 * @order: Order of the isolated page 3568 * @mt: The page's pageblock's migratetype 3569 * 3570 * This function is meant to return a page pulled from the free lists via 3571 * __isolate_free_page back to the free lists they were pulled from. 3572 */ 3573 void __putback_isolated_page(struct page *page, unsigned int order, int mt) 3574 { 3575 struct zone *zone = page_zone(page); 3576 3577 /* zone lock should be held when this function is called */ 3578 lockdep_assert_held(&zone->lock); 3579 3580 /* Return isolated page to tail of freelist. */ 3581 __free_one_page(page, page_to_pfn(page), zone, order, mt, 3582 FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL); 3583 } 3584 3585 /* 3586 * Update NUMA hit/miss statistics 3587 * 3588 * Must be called with interrupts disabled. 3589 */ 3590 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z, 3591 long nr_account) 3592 { 3593 #ifdef CONFIG_NUMA 3594 enum numa_stat_item local_stat = NUMA_LOCAL; 3595 3596 /* skip numa counters update if numa stats is disabled */ 3597 if (!static_branch_likely(&vm_numa_stat_key)) 3598 return; 3599 3600 if (zone_to_nid(z) != numa_node_id()) 3601 local_stat = NUMA_OTHER; 3602 3603 if (zone_to_nid(z) == zone_to_nid(preferred_zone)) 3604 __count_numa_events(z, NUMA_HIT, nr_account); 3605 else { 3606 __count_numa_events(z, NUMA_MISS, nr_account); 3607 __count_numa_events(preferred_zone, NUMA_FOREIGN, nr_account); 3608 } 3609 __count_numa_events(z, local_stat, nr_account); 3610 #endif 3611 } 3612 3613 /* Remove page from the per-cpu list, caller must protect the list */ 3614 static inline 3615 struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, 3616 int migratetype, 3617 unsigned int alloc_flags, 3618 struct per_cpu_pages *pcp, 3619 struct list_head *list) 3620 { 3621 struct page *page; 3622 3623 do { 3624 if (list_empty(list)) { 3625 int batch = READ_ONCE(pcp->batch); 3626 int alloced; 3627 3628 /* 3629 * Scale batch relative to order if batch implies 3630 * free pages can be stored on the PCP. Batch can 3631 * be 1 for small zones or for boot pagesets which 3632 * should never store free pages as the pages may 3633 * belong to arbitrary zones. 3634 */ 3635 if (batch > 1) 3636 batch = max(batch >> order, 2); 3637 alloced = rmqueue_bulk(zone, order, 3638 batch, list, 3639 migratetype, alloc_flags); 3640 3641 pcp->count += alloced << order; 3642 if (unlikely(list_empty(list))) 3643 return NULL; 3644 } 3645 3646 page = list_first_entry(list, struct page, lru); 3647 list_del(&page->lru); 3648 pcp->count -= 1 << order; 3649 } while (check_new_pcp(page)); 3650 3651 return page; 3652 } 3653 3654 /* Lock and remove page from the per-cpu list */ 3655 static struct page *rmqueue_pcplist(struct zone *preferred_zone, 3656 struct zone *zone, unsigned int order, 3657 gfp_t gfp_flags, int migratetype, 3658 unsigned int alloc_flags) 3659 { 3660 struct per_cpu_pages *pcp; 3661 struct list_head *list; 3662 struct page *page; 3663 unsigned long flags; 3664 3665 local_lock_irqsave(&pagesets.lock, flags); 3666 3667 /* 3668 * On allocation, reduce the number of pages that are batch freed. 3669 * See nr_pcp_free() where free_factor is increased for subsequent 3670 * frees. 3671 */ 3672 pcp = this_cpu_ptr(zone->per_cpu_pageset); 3673 pcp->free_factor >>= 1; 3674 list = &pcp->lists[order_to_pindex(migratetype, order)]; 3675 page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list); 3676 local_unlock_irqrestore(&pagesets.lock, flags); 3677 if (page) { 3678 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); 3679 zone_statistics(preferred_zone, zone, 1); 3680 } 3681 return page; 3682 } 3683 3684 /* 3685 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 3686 */ 3687 static inline 3688 struct page *rmqueue(struct zone *preferred_zone, 3689 struct zone *zone, unsigned int order, 3690 gfp_t gfp_flags, unsigned int alloc_flags, 3691 int migratetype) 3692 { 3693 unsigned long flags; 3694 struct page *page; 3695 3696 if (likely(pcp_allowed_order(order))) { 3697 /* 3698 * MIGRATE_MOVABLE pcplist could have the pages on CMA area and 3699 * we need to skip it when CMA area isn't allowed. 3700 */ 3701 if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA || 3702 migratetype != MIGRATE_MOVABLE) { 3703 page = rmqueue_pcplist(preferred_zone, zone, order, 3704 gfp_flags, migratetype, alloc_flags); 3705 goto out; 3706 } 3707 } 3708 3709 /* 3710 * We most definitely don't want callers attempting to 3711 * allocate greater than order-1 page units with __GFP_NOFAIL. 3712 */ 3713 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 3714 spin_lock_irqsave(&zone->lock, flags); 3715 3716 do { 3717 page = NULL; 3718 /* 3719 * order-0 request can reach here when the pcplist is skipped 3720 * due to non-CMA allocation context. HIGHATOMIC area is 3721 * reserved for high-order atomic allocation, so order-0 3722 * request should skip it. 3723 */ 3724 if (order > 0 && alloc_flags & ALLOC_HARDER) { 3725 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3726 if (page) 3727 trace_mm_page_alloc_zone_locked(page, order, migratetype); 3728 } 3729 if (!page) 3730 page = __rmqueue(zone, order, migratetype, alloc_flags); 3731 } while (page && check_new_pages(page, order)); 3732 if (!page) 3733 goto failed; 3734 3735 __mod_zone_freepage_state(zone, -(1 << order), 3736 get_pcppage_migratetype(page)); 3737 spin_unlock_irqrestore(&zone->lock, flags); 3738 3739 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3740 zone_statistics(preferred_zone, zone, 1); 3741 3742 out: 3743 /* Separate test+clear to avoid unnecessary atomics */ 3744 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) { 3745 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 3746 wakeup_kswapd(zone, 0, 0, zone_idx(zone)); 3747 } 3748 3749 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); 3750 return page; 3751 3752 failed: 3753 spin_unlock_irqrestore(&zone->lock, flags); 3754 return NULL; 3755 } 3756 3757 #ifdef CONFIG_FAIL_PAGE_ALLOC 3758 3759 static struct { 3760 struct fault_attr attr; 3761 3762 bool ignore_gfp_highmem; 3763 bool ignore_gfp_reclaim; 3764 u32 min_order; 3765 } fail_page_alloc = { 3766 .attr = FAULT_ATTR_INITIALIZER, 3767 .ignore_gfp_reclaim = true, 3768 .ignore_gfp_highmem = true, 3769 .min_order = 1, 3770 }; 3771 3772 static int __init setup_fail_page_alloc(char *str) 3773 { 3774 return setup_fault_attr(&fail_page_alloc.attr, str); 3775 } 3776 __setup("fail_page_alloc=", setup_fail_page_alloc); 3777 3778 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3779 { 3780 if (order < fail_page_alloc.min_order) 3781 return false; 3782 if (gfp_mask & __GFP_NOFAIL) 3783 return false; 3784 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 3785 return false; 3786 if (fail_page_alloc.ignore_gfp_reclaim && 3787 (gfp_mask & __GFP_DIRECT_RECLAIM)) 3788 return false; 3789 3790 return should_fail(&fail_page_alloc.attr, 1 << order); 3791 } 3792 3793 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 3794 3795 static int __init fail_page_alloc_debugfs(void) 3796 { 3797 umode_t mode = S_IFREG | 0600; 3798 struct dentry *dir; 3799 3800 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 3801 &fail_page_alloc.attr); 3802 3803 debugfs_create_bool("ignore-gfp-wait", mode, dir, 3804 &fail_page_alloc.ignore_gfp_reclaim); 3805 debugfs_create_bool("ignore-gfp-highmem", mode, dir, 3806 &fail_page_alloc.ignore_gfp_highmem); 3807 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order); 3808 3809 return 0; 3810 } 3811 3812 late_initcall(fail_page_alloc_debugfs); 3813 3814 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 3815 3816 #else /* CONFIG_FAIL_PAGE_ALLOC */ 3817 3818 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3819 { 3820 return false; 3821 } 3822 3823 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 3824 3825 noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3826 { 3827 return __should_fail_alloc_page(gfp_mask, order); 3828 } 3829 ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE); 3830 3831 static inline long __zone_watermark_unusable_free(struct zone *z, 3832 unsigned int order, unsigned int alloc_flags) 3833 { 3834 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3835 long unusable_free = (1 << order) - 1; 3836 3837 /* 3838 * If the caller does not have rights to ALLOC_HARDER then subtract 3839 * the high-atomic reserves. This will over-estimate the size of the 3840 * atomic reserve but it avoids a search. 3841 */ 3842 if (likely(!alloc_harder)) 3843 unusable_free += z->nr_reserved_highatomic; 3844 3845 #ifdef CONFIG_CMA 3846 /* If allocation can't use CMA areas don't use free CMA pages */ 3847 if (!(alloc_flags & ALLOC_CMA)) 3848 unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES); 3849 #endif 3850 3851 return unusable_free; 3852 } 3853 3854 /* 3855 * Return true if free base pages are above 'mark'. For high-order checks it 3856 * will return true of the order-0 watermark is reached and there is at least 3857 * one free page of a suitable size. Checking now avoids taking the zone lock 3858 * to check in the allocation paths if no pages are free. 3859 */ 3860 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3861 int highest_zoneidx, unsigned int alloc_flags, 3862 long free_pages) 3863 { 3864 long min = mark; 3865 int o; 3866 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3867 3868 /* free_pages may go negative - that's OK */ 3869 free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags); 3870 3871 if (alloc_flags & ALLOC_HIGH) 3872 min -= min / 2; 3873 3874 if (unlikely(alloc_harder)) { 3875 /* 3876 * OOM victims can try even harder than normal ALLOC_HARDER 3877 * users on the grounds that it's definitely going to be in 3878 * the exit path shortly and free memory. Any allocation it 3879 * makes during the free path will be small and short-lived. 3880 */ 3881 if (alloc_flags & ALLOC_OOM) 3882 min -= min / 2; 3883 else 3884 min -= min / 4; 3885 } 3886 3887 /* 3888 * Check watermarks for an order-0 allocation request. If these 3889 * are not met, then a high-order request also cannot go ahead 3890 * even if a suitable page happened to be free. 3891 */ 3892 if (free_pages <= min + z->lowmem_reserve[highest_zoneidx]) 3893 return false; 3894 3895 /* If this is an order-0 request then the watermark is fine */ 3896 if (!order) 3897 return true; 3898 3899 /* For a high-order request, check at least one suitable page is free */ 3900 for (o = order; o < MAX_ORDER; o++) { 3901 struct free_area *area = &z->free_area[o]; 3902 int mt; 3903 3904 if (!area->nr_free) 3905 continue; 3906 3907 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 3908 if (!free_area_empty(area, mt)) 3909 return true; 3910 } 3911 3912 #ifdef CONFIG_CMA 3913 if ((alloc_flags & ALLOC_CMA) && 3914 !free_area_empty(area, MIGRATE_CMA)) { 3915 return true; 3916 } 3917 #endif 3918 if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC)) 3919 return true; 3920 } 3921 return false; 3922 } 3923 3924 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3925 int highest_zoneidx, unsigned int alloc_flags) 3926 { 3927 return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 3928 zone_page_state(z, NR_FREE_PAGES)); 3929 } 3930 3931 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 3932 unsigned long mark, int highest_zoneidx, 3933 unsigned int alloc_flags, gfp_t gfp_mask) 3934 { 3935 long free_pages; 3936 3937 free_pages = zone_page_state(z, NR_FREE_PAGES); 3938 3939 /* 3940 * Fast check for order-0 only. If this fails then the reserves 3941 * need to be calculated. 3942 */ 3943 if (!order) { 3944 long fast_free; 3945 3946 fast_free = free_pages; 3947 fast_free -= __zone_watermark_unusable_free(z, 0, alloc_flags); 3948 if (fast_free > mark + z->lowmem_reserve[highest_zoneidx]) 3949 return true; 3950 } 3951 3952 if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 3953 free_pages)) 3954 return true; 3955 /* 3956 * Ignore watermark boosting for GFP_ATOMIC order-0 allocations 3957 * when checking the min watermark. The min watermark is the 3958 * point where boosting is ignored so that kswapd is woken up 3959 * when below the low watermark. 3960 */ 3961 if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost 3962 && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) { 3963 mark = z->_watermark[WMARK_MIN]; 3964 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 3965 alloc_flags, free_pages); 3966 } 3967 3968 return false; 3969 } 3970 3971 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 3972 unsigned long mark, int highest_zoneidx) 3973 { 3974 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3975 3976 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 3977 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 3978 3979 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0, 3980 free_pages); 3981 } 3982 3983 #ifdef CONFIG_NUMA 3984 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3985 { 3986 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= 3987 node_reclaim_distance; 3988 } 3989 #else /* CONFIG_NUMA */ 3990 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3991 { 3992 return true; 3993 } 3994 #endif /* CONFIG_NUMA */ 3995 3996 /* 3997 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid 3998 * fragmentation is subtle. If the preferred zone was HIGHMEM then 3999 * premature use of a lower zone may cause lowmem pressure problems that 4000 * are worse than fragmentation. If the next zone is ZONE_DMA then it is 4001 * probably too small. It only makes sense to spread allocations to avoid 4002 * fragmentation between the Normal and DMA32 zones. 4003 */ 4004 static inline unsigned int 4005 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) 4006 { 4007 unsigned int alloc_flags; 4008 4009 /* 4010 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 4011 * to save a branch. 4012 */ 4013 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); 4014 4015 #ifdef CONFIG_ZONE_DMA32 4016 if (!zone) 4017 return alloc_flags; 4018 4019 if (zone_idx(zone) != ZONE_NORMAL) 4020 return alloc_flags; 4021 4022 /* 4023 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and 4024 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume 4025 * on UMA that if Normal is populated then so is DMA32. 4026 */ 4027 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1); 4028 if (nr_online_nodes > 1 && !populated_zone(--zone)) 4029 return alloc_flags; 4030 4031 alloc_flags |= ALLOC_NOFRAGMENT; 4032 #endif /* CONFIG_ZONE_DMA32 */ 4033 return alloc_flags; 4034 } 4035 4036 /* Must be called after current_gfp_context() which can change gfp_mask */ 4037 static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask, 4038 unsigned int alloc_flags) 4039 { 4040 #ifdef CONFIG_CMA 4041 if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) 4042 alloc_flags |= ALLOC_CMA; 4043 #endif 4044 return alloc_flags; 4045 } 4046 4047 /* 4048 * get_page_from_freelist goes through the zonelist trying to allocate 4049 * a page. 4050 */ 4051 static struct page * 4052 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 4053 const struct alloc_context *ac) 4054 { 4055 struct zoneref *z; 4056 struct zone *zone; 4057 struct pglist_data *last_pgdat_dirty_limit = NULL; 4058 bool no_fallback; 4059 4060 retry: 4061 /* 4062 * Scan zonelist, looking for a zone with enough free. 4063 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 4064 */ 4065 no_fallback = alloc_flags & ALLOC_NOFRAGMENT; 4066 z = ac->preferred_zoneref; 4067 for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx, 4068 ac->nodemask) { 4069 struct page *page; 4070 unsigned long mark; 4071 4072 if (cpusets_enabled() && 4073 (alloc_flags & ALLOC_CPUSET) && 4074 !__cpuset_zone_allowed(zone, gfp_mask)) 4075 continue; 4076 /* 4077 * When allocating a page cache page for writing, we 4078 * want to get it from a node that is within its dirty 4079 * limit, such that no single node holds more than its 4080 * proportional share of globally allowed dirty pages. 4081 * The dirty limits take into account the node's 4082 * lowmem reserves and high watermark so that kswapd 4083 * should be able to balance it without having to 4084 * write pages from its LRU list. 4085 * 4086 * XXX: For now, allow allocations to potentially 4087 * exceed the per-node dirty limit in the slowpath 4088 * (spread_dirty_pages unset) before going into reclaim, 4089 * which is important when on a NUMA setup the allowed 4090 * nodes are together not big enough to reach the 4091 * global limit. The proper fix for these situations 4092 * will require awareness of nodes in the 4093 * dirty-throttling and the flusher threads. 4094 */ 4095 if (ac->spread_dirty_pages) { 4096 if (last_pgdat_dirty_limit == zone->zone_pgdat) 4097 continue; 4098 4099 if (!node_dirty_ok(zone->zone_pgdat)) { 4100 last_pgdat_dirty_limit = zone->zone_pgdat; 4101 continue; 4102 } 4103 } 4104 4105 if (no_fallback && nr_online_nodes > 1 && 4106 zone != ac->preferred_zoneref->zone) { 4107 int local_nid; 4108 4109 /* 4110 * If moving to a remote node, retry but allow 4111 * fragmenting fallbacks. Locality is more important 4112 * than fragmentation avoidance. 4113 */ 4114 local_nid = zone_to_nid(ac->preferred_zoneref->zone); 4115 if (zone_to_nid(zone) != local_nid) { 4116 alloc_flags &= ~ALLOC_NOFRAGMENT; 4117 goto retry; 4118 } 4119 } 4120 4121 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); 4122 if (!zone_watermark_fast(zone, order, mark, 4123 ac->highest_zoneidx, alloc_flags, 4124 gfp_mask)) { 4125 int ret; 4126 4127 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 4128 /* 4129 * Watermark failed for this zone, but see if we can 4130 * grow this zone if it contains deferred pages. 4131 */ 4132 if (static_branch_unlikely(&deferred_pages)) { 4133 if (_deferred_grow_zone(zone, order)) 4134 goto try_this_zone; 4135 } 4136 #endif 4137 /* Checked here to keep the fast path fast */ 4138 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 4139 if (alloc_flags & ALLOC_NO_WATERMARKS) 4140 goto try_this_zone; 4141 4142 if (!node_reclaim_enabled() || 4143 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 4144 continue; 4145 4146 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 4147 switch (ret) { 4148 case NODE_RECLAIM_NOSCAN: 4149 /* did not scan */ 4150 continue; 4151 case NODE_RECLAIM_FULL: 4152 /* scanned but unreclaimable */ 4153 continue; 4154 default: 4155 /* did we reclaim enough */ 4156 if (zone_watermark_ok(zone, order, mark, 4157 ac->highest_zoneidx, alloc_flags)) 4158 goto try_this_zone; 4159 4160 continue; 4161 } 4162 } 4163 4164 try_this_zone: 4165 page = rmqueue(ac->preferred_zoneref->zone, zone, order, 4166 gfp_mask, alloc_flags, ac->migratetype); 4167 if (page) { 4168 prep_new_page(page, order, gfp_mask, alloc_flags); 4169 4170 /* 4171 * If this is a high-order atomic allocation then check 4172 * if the pageblock should be reserved for the future 4173 */ 4174 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 4175 reserve_highatomic_pageblock(page, zone, order); 4176 4177 return page; 4178 } else { 4179 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 4180 /* Try again if zone has deferred pages */ 4181 if (static_branch_unlikely(&deferred_pages)) { 4182 if (_deferred_grow_zone(zone, order)) 4183 goto try_this_zone; 4184 } 4185 #endif 4186 } 4187 } 4188 4189 /* 4190 * It's possible on a UMA machine to get through all zones that are 4191 * fragmented. If avoiding fragmentation, reset and try again. 4192 */ 4193 if (no_fallback) { 4194 alloc_flags &= ~ALLOC_NOFRAGMENT; 4195 goto retry; 4196 } 4197 4198 return NULL; 4199 } 4200 4201 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) 4202 { 4203 unsigned int filter = SHOW_MEM_FILTER_NODES; 4204 4205 /* 4206 * This documents exceptions given to allocations in certain 4207 * contexts that are allowed to allocate outside current's set 4208 * of allowed nodes. 4209 */ 4210 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4211 if (tsk_is_oom_victim(current) || 4212 (current->flags & (PF_MEMALLOC | PF_EXITING))) 4213 filter &= ~SHOW_MEM_FILTER_NODES; 4214 if (!in_task() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 4215 filter &= ~SHOW_MEM_FILTER_NODES; 4216 4217 show_mem(filter, nodemask); 4218 } 4219 4220 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) 4221 { 4222 struct va_format vaf; 4223 va_list args; 4224 static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1); 4225 4226 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs)) 4227 return; 4228 4229 va_start(args, fmt); 4230 vaf.fmt = fmt; 4231 vaf.va = &args; 4232 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl", 4233 current->comm, &vaf, gfp_mask, &gfp_mask, 4234 nodemask_pr_args(nodemask)); 4235 va_end(args); 4236 4237 cpuset_print_current_mems_allowed(); 4238 pr_cont("\n"); 4239 dump_stack(); 4240 warn_alloc_show_mem(gfp_mask, nodemask); 4241 } 4242 4243 static inline struct page * 4244 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, 4245 unsigned int alloc_flags, 4246 const struct alloc_context *ac) 4247 { 4248 struct page *page; 4249 4250 page = get_page_from_freelist(gfp_mask, order, 4251 alloc_flags|ALLOC_CPUSET, ac); 4252 /* 4253 * fallback to ignore cpuset restriction if our nodes 4254 * are depleted 4255 */ 4256 if (!page) 4257 page = get_page_from_freelist(gfp_mask, order, 4258 alloc_flags, ac); 4259 4260 return page; 4261 } 4262 4263 static inline struct page * 4264 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 4265 const struct alloc_context *ac, unsigned long *did_some_progress) 4266 { 4267 struct oom_control oc = { 4268 .zonelist = ac->zonelist, 4269 .nodemask = ac->nodemask, 4270 .memcg = NULL, 4271 .gfp_mask = gfp_mask, 4272 .order = order, 4273 }; 4274 struct page *page; 4275 4276 *did_some_progress = 0; 4277 4278 /* 4279 * Acquire the oom lock. If that fails, somebody else is 4280 * making progress for us. 4281 */ 4282 if (!mutex_trylock(&oom_lock)) { 4283 *did_some_progress = 1; 4284 schedule_timeout_uninterruptible(1); 4285 return NULL; 4286 } 4287 4288 /* 4289 * Go through the zonelist yet one more time, keep very high watermark 4290 * here, this is only to catch a parallel oom killing, we must fail if 4291 * we're still under heavy pressure. But make sure that this reclaim 4292 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY 4293 * allocation which will never fail due to oom_lock already held. 4294 */ 4295 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & 4296 ~__GFP_DIRECT_RECLAIM, order, 4297 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 4298 if (page) 4299 goto out; 4300 4301 /* Coredumps can quickly deplete all memory reserves */ 4302 if (current->flags & PF_DUMPCORE) 4303 goto out; 4304 /* The OOM killer will not help higher order allocs */ 4305 if (order > PAGE_ALLOC_COSTLY_ORDER) 4306 goto out; 4307 /* 4308 * We have already exhausted all our reclaim opportunities without any 4309 * success so it is time to admit defeat. We will skip the OOM killer 4310 * because it is very likely that the caller has a more reasonable 4311 * fallback than shooting a random task. 4312 * 4313 * The OOM killer may not free memory on a specific node. 4314 */ 4315 if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE)) 4316 goto out; 4317 /* The OOM killer does not needlessly kill tasks for lowmem */ 4318 if (ac->highest_zoneidx < ZONE_NORMAL) 4319 goto out; 4320 if (pm_suspended_storage()) 4321 goto out; 4322 /* 4323 * XXX: GFP_NOFS allocations should rather fail than rely on 4324 * other request to make a forward progress. 4325 * We are in an unfortunate situation where out_of_memory cannot 4326 * do much for this context but let's try it to at least get 4327 * access to memory reserved if the current task is killed (see 4328 * out_of_memory). Once filesystems are ready to handle allocation 4329 * failures more gracefully we should just bail out here. 4330 */ 4331 4332 /* Exhausted what can be done so it's blame time */ 4333 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 4334 *did_some_progress = 1; 4335 4336 /* 4337 * Help non-failing allocations by giving them access to memory 4338 * reserves 4339 */ 4340 if (gfp_mask & __GFP_NOFAIL) 4341 page = __alloc_pages_cpuset_fallback(gfp_mask, order, 4342 ALLOC_NO_WATERMARKS, ac); 4343 } 4344 out: 4345 mutex_unlock(&oom_lock); 4346 return page; 4347 } 4348 4349 /* 4350 * Maximum number of compaction retries with a progress before OOM 4351 * killer is consider as the only way to move forward. 4352 */ 4353 #define MAX_COMPACT_RETRIES 16 4354 4355 #ifdef CONFIG_COMPACTION 4356 /* Try memory compaction for high-order allocations before reclaim */ 4357 static struct page * 4358 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4359 unsigned int alloc_flags, const struct alloc_context *ac, 4360 enum compact_priority prio, enum compact_result *compact_result) 4361 { 4362 struct page *page = NULL; 4363 unsigned long pflags; 4364 unsigned int noreclaim_flag; 4365 4366 if (!order) 4367 return NULL; 4368 4369 psi_memstall_enter(&pflags); 4370 noreclaim_flag = memalloc_noreclaim_save(); 4371 4372 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 4373 prio, &page); 4374 4375 memalloc_noreclaim_restore(noreclaim_flag); 4376 psi_memstall_leave(&pflags); 4377 4378 if (*compact_result == COMPACT_SKIPPED) 4379 return NULL; 4380 /* 4381 * At least in one zone compaction wasn't deferred or skipped, so let's 4382 * count a compaction stall 4383 */ 4384 count_vm_event(COMPACTSTALL); 4385 4386 /* Prep a captured page if available */ 4387 if (page) 4388 prep_new_page(page, order, gfp_mask, alloc_flags); 4389 4390 /* Try get a page from the freelist if available */ 4391 if (!page) 4392 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4393 4394 if (page) { 4395 struct zone *zone = page_zone(page); 4396 4397 zone->compact_blockskip_flush = false; 4398 compaction_defer_reset(zone, order, true); 4399 count_vm_event(COMPACTSUCCESS); 4400 return page; 4401 } 4402 4403 /* 4404 * It's bad if compaction run occurs and fails. The most likely reason 4405 * is that pages exist, but not enough to satisfy watermarks. 4406 */ 4407 count_vm_event(COMPACTFAIL); 4408 4409 cond_resched(); 4410 4411 return NULL; 4412 } 4413 4414 static inline bool 4415 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 4416 enum compact_result compact_result, 4417 enum compact_priority *compact_priority, 4418 int *compaction_retries) 4419 { 4420 int max_retries = MAX_COMPACT_RETRIES; 4421 int min_priority; 4422 bool ret = false; 4423 int retries = *compaction_retries; 4424 enum compact_priority priority = *compact_priority; 4425 4426 if (!order) 4427 return false; 4428 4429 if (fatal_signal_pending(current)) 4430 return false; 4431 4432 if (compaction_made_progress(compact_result)) 4433 (*compaction_retries)++; 4434 4435 /* 4436 * compaction considers all the zone as desperately out of memory 4437 * so it doesn't really make much sense to retry except when the 4438 * failure could be caused by insufficient priority 4439 */ 4440 if (compaction_failed(compact_result)) 4441 goto check_priority; 4442 4443 /* 4444 * compaction was skipped because there are not enough order-0 pages 4445 * to work with, so we retry only if it looks like reclaim can help. 4446 */ 4447 if (compaction_needs_reclaim(compact_result)) { 4448 ret = compaction_zonelist_suitable(ac, order, alloc_flags); 4449 goto out; 4450 } 4451 4452 /* 4453 * make sure the compaction wasn't deferred or didn't bail out early 4454 * due to locks contention before we declare that we should give up. 4455 * But the next retry should use a higher priority if allowed, so 4456 * we don't just keep bailing out endlessly. 4457 */ 4458 if (compaction_withdrawn(compact_result)) { 4459 goto check_priority; 4460 } 4461 4462 /* 4463 * !costly requests are much more important than __GFP_RETRY_MAYFAIL 4464 * costly ones because they are de facto nofail and invoke OOM 4465 * killer to move on while costly can fail and users are ready 4466 * to cope with that. 1/4 retries is rather arbitrary but we 4467 * would need much more detailed feedback from compaction to 4468 * make a better decision. 4469 */ 4470 if (order > PAGE_ALLOC_COSTLY_ORDER) 4471 max_retries /= 4; 4472 if (*compaction_retries <= max_retries) { 4473 ret = true; 4474 goto out; 4475 } 4476 4477 /* 4478 * Make sure there are attempts at the highest priority if we exhausted 4479 * all retries or failed at the lower priorities. 4480 */ 4481 check_priority: 4482 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 4483 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 4484 4485 if (*compact_priority > min_priority) { 4486 (*compact_priority)--; 4487 *compaction_retries = 0; 4488 ret = true; 4489 } 4490 out: 4491 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret); 4492 return ret; 4493 } 4494 #else 4495 static inline struct page * 4496 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4497 unsigned int alloc_flags, const struct alloc_context *ac, 4498 enum compact_priority prio, enum compact_result *compact_result) 4499 { 4500 *compact_result = COMPACT_SKIPPED; 4501 return NULL; 4502 } 4503 4504 static inline bool 4505 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 4506 enum compact_result compact_result, 4507 enum compact_priority *compact_priority, 4508 int *compaction_retries) 4509 { 4510 struct zone *zone; 4511 struct zoneref *z; 4512 4513 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 4514 return false; 4515 4516 /* 4517 * There are setups with compaction disabled which would prefer to loop 4518 * inside the allocator rather than hit the oom killer prematurely. 4519 * Let's give them a good hope and keep retrying while the order-0 4520 * watermarks are OK. 4521 */ 4522 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4523 ac->highest_zoneidx, ac->nodemask) { 4524 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 4525 ac->highest_zoneidx, alloc_flags)) 4526 return true; 4527 } 4528 return false; 4529 } 4530 #endif /* CONFIG_COMPACTION */ 4531 4532 #ifdef CONFIG_LOCKDEP 4533 static struct lockdep_map __fs_reclaim_map = 4534 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map); 4535 4536 static bool __need_reclaim(gfp_t gfp_mask) 4537 { 4538 /* no reclaim without waiting on it */ 4539 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) 4540 return false; 4541 4542 /* this guy won't enter reclaim */ 4543 if (current->flags & PF_MEMALLOC) 4544 return false; 4545 4546 if (gfp_mask & __GFP_NOLOCKDEP) 4547 return false; 4548 4549 return true; 4550 } 4551 4552 void __fs_reclaim_acquire(unsigned long ip) 4553 { 4554 lock_acquire_exclusive(&__fs_reclaim_map, 0, 0, NULL, ip); 4555 } 4556 4557 void __fs_reclaim_release(unsigned long ip) 4558 { 4559 lock_release(&__fs_reclaim_map, ip); 4560 } 4561 4562 void fs_reclaim_acquire(gfp_t gfp_mask) 4563 { 4564 gfp_mask = current_gfp_context(gfp_mask); 4565 4566 if (__need_reclaim(gfp_mask)) { 4567 if (gfp_mask & __GFP_FS) 4568 __fs_reclaim_acquire(_RET_IP_); 4569 4570 #ifdef CONFIG_MMU_NOTIFIER 4571 lock_map_acquire(&__mmu_notifier_invalidate_range_start_map); 4572 lock_map_release(&__mmu_notifier_invalidate_range_start_map); 4573 #endif 4574 4575 } 4576 } 4577 EXPORT_SYMBOL_GPL(fs_reclaim_acquire); 4578 4579 void fs_reclaim_release(gfp_t gfp_mask) 4580 { 4581 gfp_mask = current_gfp_context(gfp_mask); 4582 4583 if (__need_reclaim(gfp_mask)) { 4584 if (gfp_mask & __GFP_FS) 4585 __fs_reclaim_release(_RET_IP_); 4586 } 4587 } 4588 EXPORT_SYMBOL_GPL(fs_reclaim_release); 4589 #endif 4590 4591 /* Perform direct synchronous page reclaim */ 4592 static unsigned long 4593 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 4594 const struct alloc_context *ac) 4595 { 4596 unsigned int noreclaim_flag; 4597 unsigned long pflags, progress; 4598 4599 cond_resched(); 4600 4601 /* We now go into synchronous reclaim */ 4602 cpuset_memory_pressure_bump(); 4603 psi_memstall_enter(&pflags); 4604 fs_reclaim_acquire(gfp_mask); 4605 noreclaim_flag = memalloc_noreclaim_save(); 4606 4607 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 4608 ac->nodemask); 4609 4610 memalloc_noreclaim_restore(noreclaim_flag); 4611 fs_reclaim_release(gfp_mask); 4612 psi_memstall_leave(&pflags); 4613 4614 cond_resched(); 4615 4616 return progress; 4617 } 4618 4619 /* The really slow allocator path where we enter direct reclaim */ 4620 static inline struct page * 4621 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 4622 unsigned int alloc_flags, const struct alloc_context *ac, 4623 unsigned long *did_some_progress) 4624 { 4625 struct page *page = NULL; 4626 bool drained = false; 4627 4628 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 4629 if (unlikely(!(*did_some_progress))) 4630 return NULL; 4631 4632 retry: 4633 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4634 4635 /* 4636 * If an allocation failed after direct reclaim, it could be because 4637 * pages are pinned on the per-cpu lists or in high alloc reserves. 4638 * Shrink them and try again 4639 */ 4640 if (!page && !drained) { 4641 unreserve_highatomic_pageblock(ac, false); 4642 drain_all_pages(NULL); 4643 drained = true; 4644 goto retry; 4645 } 4646 4647 return page; 4648 } 4649 4650 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, 4651 const struct alloc_context *ac) 4652 { 4653 struct zoneref *z; 4654 struct zone *zone; 4655 pg_data_t *last_pgdat = NULL; 4656 enum zone_type highest_zoneidx = ac->highest_zoneidx; 4657 4658 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx, 4659 ac->nodemask) { 4660 if (last_pgdat != zone->zone_pgdat) 4661 wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx); 4662 last_pgdat = zone->zone_pgdat; 4663 } 4664 } 4665 4666 static inline unsigned int 4667 gfp_to_alloc_flags(gfp_t gfp_mask) 4668 { 4669 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 4670 4671 /* 4672 * __GFP_HIGH is assumed to be the same as ALLOC_HIGH 4673 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 4674 * to save two branches. 4675 */ 4676 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 4677 BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); 4678 4679 /* 4680 * The caller may dip into page reserves a bit more if the caller 4681 * cannot run direct reclaim, or if the caller has realtime scheduling 4682 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 4683 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 4684 */ 4685 alloc_flags |= (__force int) 4686 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); 4687 4688 if (gfp_mask & __GFP_ATOMIC) { 4689 /* 4690 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 4691 * if it can't schedule. 4692 */ 4693 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4694 alloc_flags |= ALLOC_HARDER; 4695 /* 4696 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 4697 * comment for __cpuset_node_allowed(). 4698 */ 4699 alloc_flags &= ~ALLOC_CPUSET; 4700 } else if (unlikely(rt_task(current)) && in_task()) 4701 alloc_flags |= ALLOC_HARDER; 4702 4703 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags); 4704 4705 return alloc_flags; 4706 } 4707 4708 static bool oom_reserves_allowed(struct task_struct *tsk) 4709 { 4710 if (!tsk_is_oom_victim(tsk)) 4711 return false; 4712 4713 /* 4714 * !MMU doesn't have oom reaper so give access to memory reserves 4715 * only to the thread with TIF_MEMDIE set 4716 */ 4717 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE)) 4718 return false; 4719 4720 return true; 4721 } 4722 4723 /* 4724 * Distinguish requests which really need access to full memory 4725 * reserves from oom victims which can live with a portion of it 4726 */ 4727 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) 4728 { 4729 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 4730 return 0; 4731 if (gfp_mask & __GFP_MEMALLOC) 4732 return ALLOC_NO_WATERMARKS; 4733 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 4734 return ALLOC_NO_WATERMARKS; 4735 if (!in_interrupt()) { 4736 if (current->flags & PF_MEMALLOC) 4737 return ALLOC_NO_WATERMARKS; 4738 else if (oom_reserves_allowed(current)) 4739 return ALLOC_OOM; 4740 } 4741 4742 return 0; 4743 } 4744 4745 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 4746 { 4747 return !!__gfp_pfmemalloc_flags(gfp_mask); 4748 } 4749 4750 /* 4751 * Checks whether it makes sense to retry the reclaim to make a forward progress 4752 * for the given allocation request. 4753 * 4754 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row 4755 * without success, or when we couldn't even meet the watermark if we 4756 * reclaimed all remaining pages on the LRU lists. 4757 * 4758 * Returns true if a retry is viable or false to enter the oom path. 4759 */ 4760 static inline bool 4761 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 4762 struct alloc_context *ac, int alloc_flags, 4763 bool did_some_progress, int *no_progress_loops) 4764 { 4765 struct zone *zone; 4766 struct zoneref *z; 4767 bool ret = false; 4768 4769 /* 4770 * Costly allocations might have made a progress but this doesn't mean 4771 * their order will become available due to high fragmentation so 4772 * always increment the no progress counter for them 4773 */ 4774 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 4775 *no_progress_loops = 0; 4776 else 4777 (*no_progress_loops)++; 4778 4779 /* 4780 * Make sure we converge to OOM if we cannot make any progress 4781 * several times in the row. 4782 */ 4783 if (*no_progress_loops > MAX_RECLAIM_RETRIES) { 4784 /* Before OOM, exhaust highatomic_reserve */ 4785 return unreserve_highatomic_pageblock(ac, true); 4786 } 4787 4788 /* 4789 * Keep reclaiming pages while there is a chance this will lead 4790 * somewhere. If none of the target zones can satisfy our allocation 4791 * request even if all reclaimable pages are considered then we are 4792 * screwed and have to go OOM. 4793 */ 4794 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4795 ac->highest_zoneidx, ac->nodemask) { 4796 unsigned long available; 4797 unsigned long reclaimable; 4798 unsigned long min_wmark = min_wmark_pages(zone); 4799 bool wmark; 4800 4801 available = reclaimable = zone_reclaimable_pages(zone); 4802 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 4803 4804 /* 4805 * Would the allocation succeed if we reclaimed all 4806 * reclaimable pages? 4807 */ 4808 wmark = __zone_watermark_ok(zone, order, min_wmark, 4809 ac->highest_zoneidx, alloc_flags, available); 4810 trace_reclaim_retry_zone(z, order, reclaimable, 4811 available, min_wmark, *no_progress_loops, wmark); 4812 if (wmark) { 4813 /* 4814 * If we didn't make any progress and have a lot of 4815 * dirty + writeback pages then we should wait for 4816 * an IO to complete to slow down the reclaim and 4817 * prevent from pre mature OOM 4818 */ 4819 if (!did_some_progress) { 4820 unsigned long write_pending; 4821 4822 write_pending = zone_page_state_snapshot(zone, 4823 NR_ZONE_WRITE_PENDING); 4824 4825 if (2 * write_pending > reclaimable) { 4826 congestion_wait(BLK_RW_ASYNC, HZ/10); 4827 return true; 4828 } 4829 } 4830 4831 ret = true; 4832 goto out; 4833 } 4834 } 4835 4836 out: 4837 /* 4838 * Memory allocation/reclaim might be called from a WQ context and the 4839 * current implementation of the WQ concurrency control doesn't 4840 * recognize that a particular WQ is congested if the worker thread is 4841 * looping without ever sleeping. Therefore we have to do a short sleep 4842 * here rather than calling cond_resched(). 4843 */ 4844 if (current->flags & PF_WQ_WORKER) 4845 schedule_timeout_uninterruptible(1); 4846 else 4847 cond_resched(); 4848 return ret; 4849 } 4850 4851 static inline bool 4852 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) 4853 { 4854 /* 4855 * It's possible that cpuset's mems_allowed and the nodemask from 4856 * mempolicy don't intersect. This should be normally dealt with by 4857 * policy_nodemask(), but it's possible to race with cpuset update in 4858 * such a way the check therein was true, and then it became false 4859 * before we got our cpuset_mems_cookie here. 4860 * This assumes that for all allocations, ac->nodemask can come only 4861 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored 4862 * when it does not intersect with the cpuset restrictions) or the 4863 * caller can deal with a violated nodemask. 4864 */ 4865 if (cpusets_enabled() && ac->nodemask && 4866 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) { 4867 ac->nodemask = NULL; 4868 return true; 4869 } 4870 4871 /* 4872 * When updating a task's mems_allowed or mempolicy nodemask, it is 4873 * possible to race with parallel threads in such a way that our 4874 * allocation can fail while the mask is being updated. If we are about 4875 * to fail, check if the cpuset changed during allocation and if so, 4876 * retry. 4877 */ 4878 if (read_mems_allowed_retry(cpuset_mems_cookie)) 4879 return true; 4880 4881 return false; 4882 } 4883 4884 static inline struct page * 4885 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 4886 struct alloc_context *ac) 4887 { 4888 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 4889 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 4890 struct page *page = NULL; 4891 unsigned int alloc_flags; 4892 unsigned long did_some_progress; 4893 enum compact_priority compact_priority; 4894 enum compact_result compact_result; 4895 int compaction_retries; 4896 int no_progress_loops; 4897 unsigned int cpuset_mems_cookie; 4898 int reserve_flags; 4899 4900 /* 4901 * We also sanity check to catch abuse of atomic reserves being used by 4902 * callers that are not in atomic context. 4903 */ 4904 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 4905 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 4906 gfp_mask &= ~__GFP_ATOMIC; 4907 4908 retry_cpuset: 4909 compaction_retries = 0; 4910 no_progress_loops = 0; 4911 compact_priority = DEF_COMPACT_PRIORITY; 4912 cpuset_mems_cookie = read_mems_allowed_begin(); 4913 4914 /* 4915 * The fast path uses conservative alloc_flags to succeed only until 4916 * kswapd needs to be woken up, and to avoid the cost of setting up 4917 * alloc_flags precisely. So we do that now. 4918 */ 4919 alloc_flags = gfp_to_alloc_flags(gfp_mask); 4920 4921 /* 4922 * We need to recalculate the starting point for the zonelist iterator 4923 * because we might have used different nodemask in the fast path, or 4924 * there was a cpuset modification and we are retrying - otherwise we 4925 * could end up iterating over non-eligible zones endlessly. 4926 */ 4927 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4928 ac->highest_zoneidx, ac->nodemask); 4929 if (!ac->preferred_zoneref->zone) 4930 goto nopage; 4931 4932 if (alloc_flags & ALLOC_KSWAPD) 4933 wake_all_kswapds(order, gfp_mask, ac); 4934 4935 /* 4936 * The adjusted alloc_flags might result in immediate success, so try 4937 * that first 4938 */ 4939 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4940 if (page) 4941 goto got_pg; 4942 4943 /* 4944 * For costly allocations, try direct compaction first, as it's likely 4945 * that we have enough base pages and don't need to reclaim. For non- 4946 * movable high-order allocations, do that as well, as compaction will 4947 * try prevent permanent fragmentation by migrating from blocks of the 4948 * same migratetype. 4949 * Don't try this for allocations that are allowed to ignore 4950 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 4951 */ 4952 if (can_direct_reclaim && 4953 (costly_order || 4954 (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 4955 && !gfp_pfmemalloc_allowed(gfp_mask)) { 4956 page = __alloc_pages_direct_compact(gfp_mask, order, 4957 alloc_flags, ac, 4958 INIT_COMPACT_PRIORITY, 4959 &compact_result); 4960 if (page) 4961 goto got_pg; 4962 4963 /* 4964 * Checks for costly allocations with __GFP_NORETRY, which 4965 * includes some THP page fault allocations 4966 */ 4967 if (costly_order && (gfp_mask & __GFP_NORETRY)) { 4968 /* 4969 * If allocating entire pageblock(s) and compaction 4970 * failed because all zones are below low watermarks 4971 * or is prohibited because it recently failed at this 4972 * order, fail immediately unless the allocator has 4973 * requested compaction and reclaim retry. 4974 * 4975 * Reclaim is 4976 * - potentially very expensive because zones are far 4977 * below their low watermarks or this is part of very 4978 * bursty high order allocations, 4979 * - not guaranteed to help because isolate_freepages() 4980 * may not iterate over freed pages as part of its 4981 * linear scan, and 4982 * - unlikely to make entire pageblocks free on its 4983 * own. 4984 */ 4985 if (compact_result == COMPACT_SKIPPED || 4986 compact_result == COMPACT_DEFERRED) 4987 goto nopage; 4988 4989 /* 4990 * Looks like reclaim/compaction is worth trying, but 4991 * sync compaction could be very expensive, so keep 4992 * using async compaction. 4993 */ 4994 compact_priority = INIT_COMPACT_PRIORITY; 4995 } 4996 } 4997 4998 retry: 4999 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 5000 if (alloc_flags & ALLOC_KSWAPD) 5001 wake_all_kswapds(order, gfp_mask, ac); 5002 5003 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); 5004 if (reserve_flags) 5005 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags); 5006 5007 /* 5008 * Reset the nodemask and zonelist iterators if memory policies can be 5009 * ignored. These allocations are high priority and system rather than 5010 * user oriented. 5011 */ 5012 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { 5013 ac->nodemask = NULL; 5014 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 5015 ac->highest_zoneidx, ac->nodemask); 5016 } 5017 5018 /* Attempt with potentially adjusted zonelist and alloc_flags */ 5019 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 5020 if (page) 5021 goto got_pg; 5022 5023 /* Caller is not willing to reclaim, we can't balance anything */ 5024 if (!can_direct_reclaim) 5025 goto nopage; 5026 5027 /* Avoid recursion of direct reclaim */ 5028 if (current->flags & PF_MEMALLOC) 5029 goto nopage; 5030 5031 /* Try direct reclaim and then allocating */ 5032 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 5033 &did_some_progress); 5034 if (page) 5035 goto got_pg; 5036 5037 /* Try direct compaction and then allocating */ 5038 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 5039 compact_priority, &compact_result); 5040 if (page) 5041 goto got_pg; 5042 5043 /* Do not loop if specifically requested */ 5044 if (gfp_mask & __GFP_NORETRY) 5045 goto nopage; 5046 5047 /* 5048 * Do not retry costly high order allocations unless they are 5049 * __GFP_RETRY_MAYFAIL 5050 */ 5051 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) 5052 goto nopage; 5053 5054 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 5055 did_some_progress > 0, &no_progress_loops)) 5056 goto retry; 5057 5058 /* 5059 * It doesn't make any sense to retry for the compaction if the order-0 5060 * reclaim is not able to make any progress because the current 5061 * implementation of the compaction depends on the sufficient amount 5062 * of free memory (see __compaction_suitable) 5063 */ 5064 if (did_some_progress > 0 && 5065 should_compact_retry(ac, order, alloc_flags, 5066 compact_result, &compact_priority, 5067 &compaction_retries)) 5068 goto retry; 5069 5070 5071 /* Deal with possible cpuset update races before we start OOM killing */ 5072 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 5073 goto retry_cpuset; 5074 5075 /* Reclaim has failed us, start killing things */ 5076 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 5077 if (page) 5078 goto got_pg; 5079 5080 /* Avoid allocations with no watermarks from looping endlessly */ 5081 if (tsk_is_oom_victim(current) && 5082 (alloc_flags & ALLOC_OOM || 5083 (gfp_mask & __GFP_NOMEMALLOC))) 5084 goto nopage; 5085 5086 /* Retry as long as the OOM killer is making progress */ 5087 if (did_some_progress) { 5088 no_progress_loops = 0; 5089 goto retry; 5090 } 5091 5092 nopage: 5093 /* Deal with possible cpuset update races before we fail */ 5094 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 5095 goto retry_cpuset; 5096 5097 /* 5098 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure 5099 * we always retry 5100 */ 5101 if (gfp_mask & __GFP_NOFAIL) { 5102 /* 5103 * All existing users of the __GFP_NOFAIL are blockable, so warn 5104 * of any new users that actually require GFP_NOWAIT 5105 */ 5106 if (WARN_ON_ONCE(!can_direct_reclaim)) 5107 goto fail; 5108 5109 /* 5110 * PF_MEMALLOC request from this context is rather bizarre 5111 * because we cannot reclaim anything and only can loop waiting 5112 * for somebody to do a work for us 5113 */ 5114 WARN_ON_ONCE(current->flags & PF_MEMALLOC); 5115 5116 /* 5117 * non failing costly orders are a hard requirement which we 5118 * are not prepared for much so let's warn about these users 5119 * so that we can identify them and convert them to something 5120 * else. 5121 */ 5122 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER); 5123 5124 /* 5125 * Help non-failing allocations by giving them access to memory 5126 * reserves but do not use ALLOC_NO_WATERMARKS because this 5127 * could deplete whole memory reserves which would just make 5128 * the situation worse 5129 */ 5130 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); 5131 if (page) 5132 goto got_pg; 5133 5134 cond_resched(); 5135 goto retry; 5136 } 5137 fail: 5138 warn_alloc(gfp_mask, ac->nodemask, 5139 "page allocation failure: order:%u", order); 5140 got_pg: 5141 return page; 5142 } 5143 5144 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, 5145 int preferred_nid, nodemask_t *nodemask, 5146 struct alloc_context *ac, gfp_t *alloc_gfp, 5147 unsigned int *alloc_flags) 5148 { 5149 ac->highest_zoneidx = gfp_zone(gfp_mask); 5150 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); 5151 ac->nodemask = nodemask; 5152 ac->migratetype = gfp_migratetype(gfp_mask); 5153 5154 if (cpusets_enabled()) { 5155 *alloc_gfp |= __GFP_HARDWALL; 5156 /* 5157 * When we are in the interrupt context, it is irrelevant 5158 * to the current task context. It means that any node ok. 5159 */ 5160 if (in_task() && !ac->nodemask) 5161 ac->nodemask = &cpuset_current_mems_allowed; 5162 else 5163 *alloc_flags |= ALLOC_CPUSET; 5164 } 5165 5166 fs_reclaim_acquire(gfp_mask); 5167 fs_reclaim_release(gfp_mask); 5168 5169 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 5170 5171 if (should_fail_alloc_page(gfp_mask, order)) 5172 return false; 5173 5174 *alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, *alloc_flags); 5175 5176 /* Dirty zone balancing only done in the fast path */ 5177 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); 5178 5179 /* 5180 * The preferred zone is used for statistics but crucially it is 5181 * also used as the starting point for the zonelist iterator. It 5182 * may get reset for allocations that ignore memory policies. 5183 */ 5184 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 5185 ac->highest_zoneidx, ac->nodemask); 5186 5187 return true; 5188 } 5189 5190 /* 5191 * __alloc_pages_bulk - Allocate a number of order-0 pages to a list or array 5192 * @gfp: GFP flags for the allocation 5193 * @preferred_nid: The preferred NUMA node ID to allocate from 5194 * @nodemask: Set of nodes to allocate from, may be NULL 5195 * @nr_pages: The number of pages desired on the list or array 5196 * @page_list: Optional list to store the allocated pages 5197 * @page_array: Optional array to store the pages 5198 * 5199 * This is a batched version of the page allocator that attempts to 5200 * allocate nr_pages quickly. Pages are added to page_list if page_list 5201 * is not NULL, otherwise it is assumed that the page_array is valid. 5202 * 5203 * For lists, nr_pages is the number of pages that should be allocated. 5204 * 5205 * For arrays, only NULL elements are populated with pages and nr_pages 5206 * is the maximum number of pages that will be stored in the array. 5207 * 5208 * Returns the number of pages on the list or array. 5209 */ 5210 unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, 5211 nodemask_t *nodemask, int nr_pages, 5212 struct list_head *page_list, 5213 struct page **page_array) 5214 { 5215 struct page *page; 5216 unsigned long flags; 5217 struct zone *zone; 5218 struct zoneref *z; 5219 struct per_cpu_pages *pcp; 5220 struct list_head *pcp_list; 5221 struct alloc_context ac; 5222 gfp_t alloc_gfp; 5223 unsigned int alloc_flags = ALLOC_WMARK_LOW; 5224 int nr_populated = 0, nr_account = 0; 5225 5226 /* 5227 * Skip populated array elements to determine if any pages need 5228 * to be allocated before disabling IRQs. 5229 */ 5230 while (page_array && nr_populated < nr_pages && page_array[nr_populated]) 5231 nr_populated++; 5232 5233 /* No pages requested? */ 5234 if (unlikely(nr_pages <= 0)) 5235 goto out; 5236 5237 /* Already populated array? */ 5238 if (unlikely(page_array && nr_pages - nr_populated == 0)) 5239 goto out; 5240 5241 /* Use the single page allocator for one page. */ 5242 if (nr_pages - nr_populated == 1) 5243 goto failed; 5244 5245 #ifdef CONFIG_PAGE_OWNER 5246 /* 5247 * PAGE_OWNER may recurse into the allocator to allocate space to 5248 * save the stack with pagesets.lock held. Releasing/reacquiring 5249 * removes much of the performance benefit of bulk allocation so 5250 * force the caller to allocate one page at a time as it'll have 5251 * similar performance to added complexity to the bulk allocator. 5252 */ 5253 if (static_branch_unlikely(&page_owner_inited)) 5254 goto failed; 5255 #endif 5256 5257 /* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */ 5258 gfp &= gfp_allowed_mask; 5259 alloc_gfp = gfp; 5260 if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &alloc_gfp, &alloc_flags)) 5261 goto out; 5262 gfp = alloc_gfp; 5263 5264 /* Find an allowed local zone that meets the low watermark. */ 5265 for_each_zone_zonelist_nodemask(zone, z, ac.zonelist, ac.highest_zoneidx, ac.nodemask) { 5266 unsigned long mark; 5267 5268 if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) && 5269 !__cpuset_zone_allowed(zone, gfp)) { 5270 continue; 5271 } 5272 5273 if (nr_online_nodes > 1 && zone != ac.preferred_zoneref->zone && 5274 zone_to_nid(zone) != zone_to_nid(ac.preferred_zoneref->zone)) { 5275 goto failed; 5276 } 5277 5278 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK) + nr_pages; 5279 if (zone_watermark_fast(zone, 0, mark, 5280 zonelist_zone_idx(ac.preferred_zoneref), 5281 alloc_flags, gfp)) { 5282 break; 5283 } 5284 } 5285 5286 /* 5287 * If there are no allowed local zones that meets the watermarks then 5288 * try to allocate a single page and reclaim if necessary. 5289 */ 5290 if (unlikely(!zone)) 5291 goto failed; 5292 5293 /* Attempt the batch allocation */ 5294 local_lock_irqsave(&pagesets.lock, flags); 5295 pcp = this_cpu_ptr(zone->per_cpu_pageset); 5296 pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)]; 5297 5298 while (nr_populated < nr_pages) { 5299 5300 /* Skip existing pages */ 5301 if (page_array && page_array[nr_populated]) { 5302 nr_populated++; 5303 continue; 5304 } 5305 5306 page = __rmqueue_pcplist(zone, 0, ac.migratetype, alloc_flags, 5307 pcp, pcp_list); 5308 if (unlikely(!page)) { 5309 /* Try and get at least one page */ 5310 if (!nr_populated) 5311 goto failed_irq; 5312 break; 5313 } 5314 nr_account++; 5315 5316 prep_new_page(page, 0, gfp, 0); 5317 if (page_list) 5318 list_add(&page->lru, page_list); 5319 else 5320 page_array[nr_populated] = page; 5321 nr_populated++; 5322 } 5323 5324 local_unlock_irqrestore(&pagesets.lock, flags); 5325 5326 __count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account); 5327 zone_statistics(ac.preferred_zoneref->zone, zone, nr_account); 5328 5329 out: 5330 return nr_populated; 5331 5332 failed_irq: 5333 local_unlock_irqrestore(&pagesets.lock, flags); 5334 5335 failed: 5336 page = __alloc_pages(gfp, 0, preferred_nid, nodemask); 5337 if (page) { 5338 if (page_list) 5339 list_add(&page->lru, page_list); 5340 else 5341 page_array[nr_populated] = page; 5342 nr_populated++; 5343 } 5344 5345 goto out; 5346 } 5347 EXPORT_SYMBOL_GPL(__alloc_pages_bulk); 5348 5349 /* 5350 * This is the 'heart' of the zoned buddy allocator. 5351 */ 5352 struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid, 5353 nodemask_t *nodemask) 5354 { 5355 struct page *page; 5356 unsigned int alloc_flags = ALLOC_WMARK_LOW; 5357 gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */ 5358 struct alloc_context ac = { }; 5359 5360 /* 5361 * There are several places where we assume that the order value is sane 5362 * so bail out early if the request is out of bound. 5363 */ 5364 if (unlikely(order >= MAX_ORDER)) { 5365 WARN_ON_ONCE(!(gfp & __GFP_NOWARN)); 5366 return NULL; 5367 } 5368 5369 gfp &= gfp_allowed_mask; 5370 /* 5371 * Apply scoped allocation constraints. This is mainly about GFP_NOFS 5372 * resp. GFP_NOIO which has to be inherited for all allocation requests 5373 * from a particular context which has been marked by 5374 * memalloc_no{fs,io}_{save,restore}. And PF_MEMALLOC_PIN which ensures 5375 * movable zones are not used during allocation. 5376 */ 5377 gfp = current_gfp_context(gfp); 5378 alloc_gfp = gfp; 5379 if (!prepare_alloc_pages(gfp, order, preferred_nid, nodemask, &ac, 5380 &alloc_gfp, &alloc_flags)) 5381 return NULL; 5382 5383 /* 5384 * Forbid the first pass from falling back to types that fragment 5385 * memory until all local zones are considered. 5386 */ 5387 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp); 5388 5389 /* First allocation attempt */ 5390 page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); 5391 if (likely(page)) 5392 goto out; 5393 5394 alloc_gfp = gfp; 5395 ac.spread_dirty_pages = false; 5396 5397 /* 5398 * Restore the original nodemask if it was potentially replaced with 5399 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 5400 */ 5401 ac.nodemask = nodemask; 5402 5403 page = __alloc_pages_slowpath(alloc_gfp, order, &ac); 5404 5405 out: 5406 if (memcg_kmem_enabled() && (gfp & __GFP_ACCOUNT) && page && 5407 unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) { 5408 __free_pages(page, order); 5409 page = NULL; 5410 } 5411 5412 trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype); 5413 5414 return page; 5415 } 5416 EXPORT_SYMBOL(__alloc_pages); 5417 5418 /* 5419 * Common helper functions. Never use with __GFP_HIGHMEM because the returned 5420 * address cannot represent highmem pages. Use alloc_pages and then kmap if 5421 * you need to access high mem. 5422 */ 5423 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 5424 { 5425 struct page *page; 5426 5427 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); 5428 if (!page) 5429 return 0; 5430 return (unsigned long) page_address(page); 5431 } 5432 EXPORT_SYMBOL(__get_free_pages); 5433 5434 unsigned long get_zeroed_page(gfp_t gfp_mask) 5435 { 5436 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 5437 } 5438 EXPORT_SYMBOL(get_zeroed_page); 5439 5440 /** 5441 * __free_pages - Free pages allocated with alloc_pages(). 5442 * @page: The page pointer returned from alloc_pages(). 5443 * @order: The order of the allocation. 5444 * 5445 * This function can free multi-page allocations that are not compound 5446 * pages. It does not check that the @order passed in matches that of 5447 * the allocation, so it is easy to leak memory. Freeing more memory 5448 * than was allocated will probably emit a warning. 5449 * 5450 * If the last reference to this page is speculative, it will be released 5451 * by put_page() which only frees the first page of a non-compound 5452 * allocation. To prevent the remaining pages from being leaked, we free 5453 * the subsequent pages here. If you want to use the page's reference 5454 * count to decide when to free the allocation, you should allocate a 5455 * compound page, and use put_page() instead of __free_pages(). 5456 * 5457 * Context: May be called in interrupt context or while holding a normal 5458 * spinlock, but not in NMI context or while holding a raw spinlock. 5459 */ 5460 void __free_pages(struct page *page, unsigned int order) 5461 { 5462 if (put_page_testzero(page)) 5463 free_the_page(page, order); 5464 else if (!PageHead(page)) 5465 while (order-- > 0) 5466 free_the_page(page + (1 << order), order); 5467 } 5468 EXPORT_SYMBOL(__free_pages); 5469 5470 void free_pages(unsigned long addr, unsigned int order) 5471 { 5472 if (addr != 0) { 5473 VM_BUG_ON(!virt_addr_valid((void *)addr)); 5474 __free_pages(virt_to_page((void *)addr), order); 5475 } 5476 } 5477 5478 EXPORT_SYMBOL(free_pages); 5479 5480 /* 5481 * Page Fragment: 5482 * An arbitrary-length arbitrary-offset area of memory which resides 5483 * within a 0 or higher order page. Multiple fragments within that page 5484 * are individually refcounted, in the page's reference counter. 5485 * 5486 * The page_frag functions below provide a simple allocation framework for 5487 * page fragments. This is used by the network stack and network device 5488 * drivers to provide a backing region of memory for use as either an 5489 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 5490 */ 5491 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 5492 gfp_t gfp_mask) 5493 { 5494 struct page *page = NULL; 5495 gfp_t gfp = gfp_mask; 5496 5497 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5498 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 5499 __GFP_NOMEMALLOC; 5500 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 5501 PAGE_FRAG_CACHE_MAX_ORDER); 5502 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 5503 #endif 5504 if (unlikely(!page)) 5505 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 5506 5507 nc->va = page ? page_address(page) : NULL; 5508 5509 return page; 5510 } 5511 5512 void __page_frag_cache_drain(struct page *page, unsigned int count) 5513 { 5514 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 5515 5516 if (page_ref_sub_and_test(page, count)) 5517 free_the_page(page, compound_order(page)); 5518 } 5519 EXPORT_SYMBOL(__page_frag_cache_drain); 5520 5521 void *page_frag_alloc_align(struct page_frag_cache *nc, 5522 unsigned int fragsz, gfp_t gfp_mask, 5523 unsigned int align_mask) 5524 { 5525 unsigned int size = PAGE_SIZE; 5526 struct page *page; 5527 int offset; 5528 5529 if (unlikely(!nc->va)) { 5530 refill: 5531 page = __page_frag_cache_refill(nc, gfp_mask); 5532 if (!page) 5533 return NULL; 5534 5535 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5536 /* if size can vary use size else just use PAGE_SIZE */ 5537 size = nc->size; 5538 #endif 5539 /* Even if we own the page, we do not use atomic_set(). 5540 * This would break get_page_unless_zero() users. 5541 */ 5542 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 5543 5544 /* reset page count bias and offset to start of new frag */ 5545 nc->pfmemalloc = page_is_pfmemalloc(page); 5546 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5547 nc->offset = size; 5548 } 5549 5550 offset = nc->offset - fragsz; 5551 if (unlikely(offset < 0)) { 5552 page = virt_to_page(nc->va); 5553 5554 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 5555 goto refill; 5556 5557 if (unlikely(nc->pfmemalloc)) { 5558 free_the_page(page, compound_order(page)); 5559 goto refill; 5560 } 5561 5562 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5563 /* if size can vary use size else just use PAGE_SIZE */ 5564 size = nc->size; 5565 #endif 5566 /* OK, page count is 0, we can safely set it */ 5567 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 5568 5569 /* reset page count bias and offset to start of new frag */ 5570 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5571 offset = size - fragsz; 5572 } 5573 5574 nc->pagecnt_bias--; 5575 offset &= align_mask; 5576 nc->offset = offset; 5577 5578 return nc->va + offset; 5579 } 5580 EXPORT_SYMBOL(page_frag_alloc_align); 5581 5582 /* 5583 * Frees a page fragment allocated out of either a compound or order 0 page. 5584 */ 5585 void page_frag_free(void *addr) 5586 { 5587 struct page *page = virt_to_head_page(addr); 5588 5589 if (unlikely(put_page_testzero(page))) 5590 free_the_page(page, compound_order(page)); 5591 } 5592 EXPORT_SYMBOL(page_frag_free); 5593 5594 static void *make_alloc_exact(unsigned long addr, unsigned int order, 5595 size_t size) 5596 { 5597 if (addr) { 5598 unsigned long alloc_end = addr + (PAGE_SIZE << order); 5599 unsigned long used = addr + PAGE_ALIGN(size); 5600 5601 split_page(virt_to_page((void *)addr), order); 5602 while (used < alloc_end) { 5603 free_page(used); 5604 used += PAGE_SIZE; 5605 } 5606 } 5607 return (void *)addr; 5608 } 5609 5610 /** 5611 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 5612 * @size: the number of bytes to allocate 5613 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5614 * 5615 * This function is similar to alloc_pages(), except that it allocates the 5616 * minimum number of pages to satisfy the request. alloc_pages() can only 5617 * allocate memory in power-of-two pages. 5618 * 5619 * This function is also limited by MAX_ORDER. 5620 * 5621 * Memory allocated by this function must be released by free_pages_exact(). 5622 * 5623 * Return: pointer to the allocated area or %NULL in case of error. 5624 */ 5625 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 5626 { 5627 unsigned int order = get_order(size); 5628 unsigned long addr; 5629 5630 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 5631 gfp_mask &= ~__GFP_COMP; 5632 5633 addr = __get_free_pages(gfp_mask, order); 5634 return make_alloc_exact(addr, order, size); 5635 } 5636 EXPORT_SYMBOL(alloc_pages_exact); 5637 5638 /** 5639 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 5640 * pages on a node. 5641 * @nid: the preferred node ID where memory should be allocated 5642 * @size: the number of bytes to allocate 5643 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5644 * 5645 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 5646 * back. 5647 * 5648 * Return: pointer to the allocated area or %NULL in case of error. 5649 */ 5650 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 5651 { 5652 unsigned int order = get_order(size); 5653 struct page *p; 5654 5655 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 5656 gfp_mask &= ~__GFP_COMP; 5657 5658 p = alloc_pages_node(nid, gfp_mask, order); 5659 if (!p) 5660 return NULL; 5661 return make_alloc_exact((unsigned long)page_address(p), order, size); 5662 } 5663 5664 /** 5665 * free_pages_exact - release memory allocated via alloc_pages_exact() 5666 * @virt: the value returned by alloc_pages_exact. 5667 * @size: size of allocation, same value as passed to alloc_pages_exact(). 5668 * 5669 * Release the memory allocated by a previous call to alloc_pages_exact. 5670 */ 5671 void free_pages_exact(void *virt, size_t size) 5672 { 5673 unsigned long addr = (unsigned long)virt; 5674 unsigned long end = addr + PAGE_ALIGN(size); 5675 5676 while (addr < end) { 5677 free_page(addr); 5678 addr += PAGE_SIZE; 5679 } 5680 } 5681 EXPORT_SYMBOL(free_pages_exact); 5682 5683 /** 5684 * nr_free_zone_pages - count number of pages beyond high watermark 5685 * @offset: The zone index of the highest zone 5686 * 5687 * nr_free_zone_pages() counts the number of pages which are beyond the 5688 * high watermark within all zones at or below a given zone index. For each 5689 * zone, the number of pages is calculated as: 5690 * 5691 * nr_free_zone_pages = managed_pages - high_pages 5692 * 5693 * Return: number of pages beyond high watermark. 5694 */ 5695 static unsigned long nr_free_zone_pages(int offset) 5696 { 5697 struct zoneref *z; 5698 struct zone *zone; 5699 5700 /* Just pick one node, since fallback list is circular */ 5701 unsigned long sum = 0; 5702 5703 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 5704 5705 for_each_zone_zonelist(zone, z, zonelist, offset) { 5706 unsigned long size = zone_managed_pages(zone); 5707 unsigned long high = high_wmark_pages(zone); 5708 if (size > high) 5709 sum += size - high; 5710 } 5711 5712 return sum; 5713 } 5714 5715 /** 5716 * nr_free_buffer_pages - count number of pages beyond high watermark 5717 * 5718 * nr_free_buffer_pages() counts the number of pages which are beyond the high 5719 * watermark within ZONE_DMA and ZONE_NORMAL. 5720 * 5721 * Return: number of pages beyond high watermark within ZONE_DMA and 5722 * ZONE_NORMAL. 5723 */ 5724 unsigned long nr_free_buffer_pages(void) 5725 { 5726 return nr_free_zone_pages(gfp_zone(GFP_USER)); 5727 } 5728 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 5729 5730 static inline void show_node(struct zone *zone) 5731 { 5732 if (IS_ENABLED(CONFIG_NUMA)) 5733 printk("Node %d ", zone_to_nid(zone)); 5734 } 5735 5736 long si_mem_available(void) 5737 { 5738 long available; 5739 unsigned long pagecache; 5740 unsigned long wmark_low = 0; 5741 unsigned long pages[NR_LRU_LISTS]; 5742 unsigned long reclaimable; 5743 struct zone *zone; 5744 int lru; 5745 5746 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 5747 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 5748 5749 for_each_zone(zone) 5750 wmark_low += low_wmark_pages(zone); 5751 5752 /* 5753 * Estimate the amount of memory available for userspace allocations, 5754 * without causing swapping. 5755 */ 5756 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages; 5757 5758 /* 5759 * Not all the page cache can be freed, otherwise the system will 5760 * start swapping. Assume at least half of the page cache, or the 5761 * low watermark worth of cache, needs to stay. 5762 */ 5763 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 5764 pagecache -= min(pagecache / 2, wmark_low); 5765 available += pagecache; 5766 5767 /* 5768 * Part of the reclaimable slab and other kernel memory consists of 5769 * items that are in use, and cannot be freed. Cap this estimate at the 5770 * low watermark. 5771 */ 5772 reclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B) + 5773 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE); 5774 available += reclaimable - min(reclaimable / 2, wmark_low); 5775 5776 if (available < 0) 5777 available = 0; 5778 return available; 5779 } 5780 EXPORT_SYMBOL_GPL(si_mem_available); 5781 5782 void si_meminfo(struct sysinfo *val) 5783 { 5784 val->totalram = totalram_pages(); 5785 val->sharedram = global_node_page_state(NR_SHMEM); 5786 val->freeram = global_zone_page_state(NR_FREE_PAGES); 5787 val->bufferram = nr_blockdev_pages(); 5788 val->totalhigh = totalhigh_pages(); 5789 val->freehigh = nr_free_highpages(); 5790 val->mem_unit = PAGE_SIZE; 5791 } 5792 5793 EXPORT_SYMBOL(si_meminfo); 5794 5795 #ifdef CONFIG_NUMA 5796 void si_meminfo_node(struct sysinfo *val, int nid) 5797 { 5798 int zone_type; /* needs to be signed */ 5799 unsigned long managed_pages = 0; 5800 unsigned long managed_highpages = 0; 5801 unsigned long free_highpages = 0; 5802 pg_data_t *pgdat = NODE_DATA(nid); 5803 5804 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 5805 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]); 5806 val->totalram = managed_pages; 5807 val->sharedram = node_page_state(pgdat, NR_SHMEM); 5808 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 5809 #ifdef CONFIG_HIGHMEM 5810 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 5811 struct zone *zone = &pgdat->node_zones[zone_type]; 5812 5813 if (is_highmem(zone)) { 5814 managed_highpages += zone_managed_pages(zone); 5815 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 5816 } 5817 } 5818 val->totalhigh = managed_highpages; 5819 val->freehigh = free_highpages; 5820 #else 5821 val->totalhigh = managed_highpages; 5822 val->freehigh = free_highpages; 5823 #endif 5824 val->mem_unit = PAGE_SIZE; 5825 } 5826 #endif 5827 5828 /* 5829 * Determine whether the node should be displayed or not, depending on whether 5830 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 5831 */ 5832 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) 5833 { 5834 if (!(flags & SHOW_MEM_FILTER_NODES)) 5835 return false; 5836 5837 /* 5838 * no node mask - aka implicit memory numa policy. Do not bother with 5839 * the synchronization - read_mems_allowed_begin - because we do not 5840 * have to be precise here. 5841 */ 5842 if (!nodemask) 5843 nodemask = &cpuset_current_mems_allowed; 5844 5845 return !node_isset(nid, *nodemask); 5846 } 5847 5848 #define K(x) ((x) << (PAGE_SHIFT-10)) 5849 5850 static void show_migration_types(unsigned char type) 5851 { 5852 static const char types[MIGRATE_TYPES] = { 5853 [MIGRATE_UNMOVABLE] = 'U', 5854 [MIGRATE_MOVABLE] = 'M', 5855 [MIGRATE_RECLAIMABLE] = 'E', 5856 [MIGRATE_HIGHATOMIC] = 'H', 5857 #ifdef CONFIG_CMA 5858 [MIGRATE_CMA] = 'C', 5859 #endif 5860 #ifdef CONFIG_MEMORY_ISOLATION 5861 [MIGRATE_ISOLATE] = 'I', 5862 #endif 5863 }; 5864 char tmp[MIGRATE_TYPES + 1]; 5865 char *p = tmp; 5866 int i; 5867 5868 for (i = 0; i < MIGRATE_TYPES; i++) { 5869 if (type & (1 << i)) 5870 *p++ = types[i]; 5871 } 5872 5873 *p = '\0'; 5874 printk(KERN_CONT "(%s) ", tmp); 5875 } 5876 5877 /* 5878 * Show free area list (used inside shift_scroll-lock stuff) 5879 * We also calculate the percentage fragmentation. We do this by counting the 5880 * memory on each free list with the exception of the first item on the list. 5881 * 5882 * Bits in @filter: 5883 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 5884 * cpuset. 5885 */ 5886 void show_free_areas(unsigned int filter, nodemask_t *nodemask) 5887 { 5888 unsigned long free_pcp = 0; 5889 int cpu; 5890 struct zone *zone; 5891 pg_data_t *pgdat; 5892 5893 for_each_populated_zone(zone) { 5894 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5895 continue; 5896 5897 for_each_online_cpu(cpu) 5898 free_pcp += per_cpu_ptr(zone->per_cpu_pageset, cpu)->count; 5899 } 5900 5901 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 5902 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 5903 " unevictable:%lu dirty:%lu writeback:%lu\n" 5904 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 5905 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 5906 " kernel_misc_reclaimable:%lu\n" 5907 " free:%lu free_pcp:%lu free_cma:%lu\n", 5908 global_node_page_state(NR_ACTIVE_ANON), 5909 global_node_page_state(NR_INACTIVE_ANON), 5910 global_node_page_state(NR_ISOLATED_ANON), 5911 global_node_page_state(NR_ACTIVE_FILE), 5912 global_node_page_state(NR_INACTIVE_FILE), 5913 global_node_page_state(NR_ISOLATED_FILE), 5914 global_node_page_state(NR_UNEVICTABLE), 5915 global_node_page_state(NR_FILE_DIRTY), 5916 global_node_page_state(NR_WRITEBACK), 5917 global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B), 5918 global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B), 5919 global_node_page_state(NR_FILE_MAPPED), 5920 global_node_page_state(NR_SHMEM), 5921 global_node_page_state(NR_PAGETABLE), 5922 global_zone_page_state(NR_BOUNCE), 5923 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE), 5924 global_zone_page_state(NR_FREE_PAGES), 5925 free_pcp, 5926 global_zone_page_state(NR_FREE_CMA_PAGES)); 5927 5928 for_each_online_pgdat(pgdat) { 5929 if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) 5930 continue; 5931 5932 printk("Node %d" 5933 " active_anon:%lukB" 5934 " inactive_anon:%lukB" 5935 " active_file:%lukB" 5936 " inactive_file:%lukB" 5937 " unevictable:%lukB" 5938 " isolated(anon):%lukB" 5939 " isolated(file):%lukB" 5940 " mapped:%lukB" 5941 " dirty:%lukB" 5942 " writeback:%lukB" 5943 " shmem:%lukB" 5944 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5945 " shmem_thp: %lukB" 5946 " shmem_pmdmapped: %lukB" 5947 " anon_thp: %lukB" 5948 #endif 5949 " writeback_tmp:%lukB" 5950 " kernel_stack:%lukB" 5951 #ifdef CONFIG_SHADOW_CALL_STACK 5952 " shadow_call_stack:%lukB" 5953 #endif 5954 " pagetables:%lukB" 5955 " all_unreclaimable? %s" 5956 "\n", 5957 pgdat->node_id, 5958 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 5959 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 5960 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 5961 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 5962 K(node_page_state(pgdat, NR_UNEVICTABLE)), 5963 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 5964 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 5965 K(node_page_state(pgdat, NR_FILE_MAPPED)), 5966 K(node_page_state(pgdat, NR_FILE_DIRTY)), 5967 K(node_page_state(pgdat, NR_WRITEBACK)), 5968 K(node_page_state(pgdat, NR_SHMEM)), 5969 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5970 K(node_page_state(pgdat, NR_SHMEM_THPS)), 5971 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), 5972 K(node_page_state(pgdat, NR_ANON_THPS)), 5973 #endif 5974 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 5975 node_page_state(pgdat, NR_KERNEL_STACK_KB), 5976 #ifdef CONFIG_SHADOW_CALL_STACK 5977 node_page_state(pgdat, NR_KERNEL_SCS_KB), 5978 #endif 5979 K(node_page_state(pgdat, NR_PAGETABLE)), 5980 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ? 5981 "yes" : "no"); 5982 } 5983 5984 for_each_populated_zone(zone) { 5985 int i; 5986 5987 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5988 continue; 5989 5990 free_pcp = 0; 5991 for_each_online_cpu(cpu) 5992 free_pcp += per_cpu_ptr(zone->per_cpu_pageset, cpu)->count; 5993 5994 show_node(zone); 5995 printk(KERN_CONT 5996 "%s" 5997 " free:%lukB" 5998 " min:%lukB" 5999 " low:%lukB" 6000 " high:%lukB" 6001 " reserved_highatomic:%luKB" 6002 " active_anon:%lukB" 6003 " inactive_anon:%lukB" 6004 " active_file:%lukB" 6005 " inactive_file:%lukB" 6006 " unevictable:%lukB" 6007 " writepending:%lukB" 6008 " present:%lukB" 6009 " managed:%lukB" 6010 " mlocked:%lukB" 6011 " bounce:%lukB" 6012 " free_pcp:%lukB" 6013 " local_pcp:%ukB" 6014 " free_cma:%lukB" 6015 "\n", 6016 zone->name, 6017 K(zone_page_state(zone, NR_FREE_PAGES)), 6018 K(min_wmark_pages(zone)), 6019 K(low_wmark_pages(zone)), 6020 K(high_wmark_pages(zone)), 6021 K(zone->nr_reserved_highatomic), 6022 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 6023 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 6024 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 6025 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 6026 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 6027 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 6028 K(zone->present_pages), 6029 K(zone_managed_pages(zone)), 6030 K(zone_page_state(zone, NR_MLOCK)), 6031 K(zone_page_state(zone, NR_BOUNCE)), 6032 K(free_pcp), 6033 K(this_cpu_read(zone->per_cpu_pageset->count)), 6034 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 6035 printk("lowmem_reserve[]:"); 6036 for (i = 0; i < MAX_NR_ZONES; i++) 6037 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); 6038 printk(KERN_CONT "\n"); 6039 } 6040 6041 for_each_populated_zone(zone) { 6042 unsigned int order; 6043 unsigned long nr[MAX_ORDER], flags, total = 0; 6044 unsigned char types[MAX_ORDER]; 6045 6046 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 6047 continue; 6048 show_node(zone); 6049 printk(KERN_CONT "%s: ", zone->name); 6050 6051 spin_lock_irqsave(&zone->lock, flags); 6052 for (order = 0; order < MAX_ORDER; order++) { 6053 struct free_area *area = &zone->free_area[order]; 6054 int type; 6055 6056 nr[order] = area->nr_free; 6057 total += nr[order] << order; 6058 6059 types[order] = 0; 6060 for (type = 0; type < MIGRATE_TYPES; type++) { 6061 if (!free_area_empty(area, type)) 6062 types[order] |= 1 << type; 6063 } 6064 } 6065 spin_unlock_irqrestore(&zone->lock, flags); 6066 for (order = 0; order < MAX_ORDER; order++) { 6067 printk(KERN_CONT "%lu*%lukB ", 6068 nr[order], K(1UL) << order); 6069 if (nr[order]) 6070 show_migration_types(types[order]); 6071 } 6072 printk(KERN_CONT "= %lukB\n", K(total)); 6073 } 6074 6075 hugetlb_show_meminfo(); 6076 6077 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 6078 6079 show_swap_cache_info(); 6080 } 6081 6082 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 6083 { 6084 zoneref->zone = zone; 6085 zoneref->zone_idx = zone_idx(zone); 6086 } 6087 6088 /* 6089 * Builds allocation fallback zone lists. 6090 * 6091 * Add all populated zones of a node to the zonelist. 6092 */ 6093 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) 6094 { 6095 struct zone *zone; 6096 enum zone_type zone_type = MAX_NR_ZONES; 6097 int nr_zones = 0; 6098 6099 do { 6100 zone_type--; 6101 zone = pgdat->node_zones + zone_type; 6102 if (managed_zone(zone)) { 6103 zoneref_set_zone(zone, &zonerefs[nr_zones++]); 6104 check_highest_zone(zone_type); 6105 } 6106 } while (zone_type); 6107 6108 return nr_zones; 6109 } 6110 6111 #ifdef CONFIG_NUMA 6112 6113 static int __parse_numa_zonelist_order(char *s) 6114 { 6115 /* 6116 * We used to support different zonelists modes but they turned 6117 * out to be just not useful. Let's keep the warning in place 6118 * if somebody still use the cmd line parameter so that we do 6119 * not fail it silently 6120 */ 6121 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) { 6122 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s); 6123 return -EINVAL; 6124 } 6125 return 0; 6126 } 6127 6128 char numa_zonelist_order[] = "Node"; 6129 6130 /* 6131 * sysctl handler for numa_zonelist_order 6132 */ 6133 int numa_zonelist_order_handler(struct ctl_table *table, int write, 6134 void *buffer, size_t *length, loff_t *ppos) 6135 { 6136 if (write) 6137 return __parse_numa_zonelist_order(buffer); 6138 return proc_dostring(table, write, buffer, length, ppos); 6139 } 6140 6141 6142 #define MAX_NODE_LOAD (nr_online_nodes) 6143 static int node_load[MAX_NUMNODES]; 6144 6145 /** 6146 * find_next_best_node - find the next node that should appear in a given node's fallback list 6147 * @node: node whose fallback list we're appending 6148 * @used_node_mask: nodemask_t of already used nodes 6149 * 6150 * We use a number of factors to determine which is the next node that should 6151 * appear on a given node's fallback list. The node should not have appeared 6152 * already in @node's fallback list, and it should be the next closest node 6153 * according to the distance array (which contains arbitrary distance values 6154 * from each node to each node in the system), and should also prefer nodes 6155 * with no CPUs, since presumably they'll have very little allocation pressure 6156 * on them otherwise. 6157 * 6158 * Return: node id of the found node or %NUMA_NO_NODE if no node is found. 6159 */ 6160 int find_next_best_node(int node, nodemask_t *used_node_mask) 6161 { 6162 int n, val; 6163 int min_val = INT_MAX; 6164 int best_node = NUMA_NO_NODE; 6165 6166 /* Use the local node if we haven't already */ 6167 if (!node_isset(node, *used_node_mask)) { 6168 node_set(node, *used_node_mask); 6169 return node; 6170 } 6171 6172 for_each_node_state(n, N_MEMORY) { 6173 6174 /* Don't want a node to appear more than once */ 6175 if (node_isset(n, *used_node_mask)) 6176 continue; 6177 6178 /* Use the distance array to find the distance */ 6179 val = node_distance(node, n); 6180 6181 /* Penalize nodes under us ("prefer the next node") */ 6182 val += (n < node); 6183 6184 /* Give preference to headless and unused nodes */ 6185 if (!cpumask_empty(cpumask_of_node(n))) 6186 val += PENALTY_FOR_NODE_WITH_CPUS; 6187 6188 /* Slight preference for less loaded node */ 6189 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 6190 val += node_load[n]; 6191 6192 if (val < min_val) { 6193 min_val = val; 6194 best_node = n; 6195 } 6196 } 6197 6198 if (best_node >= 0) 6199 node_set(best_node, *used_node_mask); 6200 6201 return best_node; 6202 } 6203 6204 6205 /* 6206 * Build zonelists ordered by node and zones within node. 6207 * This results in maximum locality--normal zone overflows into local 6208 * DMA zone, if any--but risks exhausting DMA zone. 6209 */ 6210 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, 6211 unsigned nr_nodes) 6212 { 6213 struct zoneref *zonerefs; 6214 int i; 6215 6216 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 6217 6218 for (i = 0; i < nr_nodes; i++) { 6219 int nr_zones; 6220 6221 pg_data_t *node = NODE_DATA(node_order[i]); 6222 6223 nr_zones = build_zonerefs_node(node, zonerefs); 6224 zonerefs += nr_zones; 6225 } 6226 zonerefs->zone = NULL; 6227 zonerefs->zone_idx = 0; 6228 } 6229 6230 /* 6231 * Build gfp_thisnode zonelists 6232 */ 6233 static void build_thisnode_zonelists(pg_data_t *pgdat) 6234 { 6235 struct zoneref *zonerefs; 6236 int nr_zones; 6237 6238 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; 6239 nr_zones = build_zonerefs_node(pgdat, zonerefs); 6240 zonerefs += nr_zones; 6241 zonerefs->zone = NULL; 6242 zonerefs->zone_idx = 0; 6243 } 6244 6245 /* 6246 * Build zonelists ordered by zone and nodes within zones. 6247 * This results in conserving DMA zone[s] until all Normal memory is 6248 * exhausted, but results in overflowing to remote node while memory 6249 * may still exist in local DMA zone. 6250 */ 6251 6252 static void build_zonelists(pg_data_t *pgdat) 6253 { 6254 static int node_order[MAX_NUMNODES]; 6255 int node, load, nr_nodes = 0; 6256 nodemask_t used_mask = NODE_MASK_NONE; 6257 int local_node, prev_node; 6258 6259 /* NUMA-aware ordering of nodes */ 6260 local_node = pgdat->node_id; 6261 load = nr_online_nodes; 6262 prev_node = local_node; 6263 6264 memset(node_order, 0, sizeof(node_order)); 6265 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 6266 /* 6267 * We don't want to pressure a particular node. 6268 * So adding penalty to the first node in same 6269 * distance group to make it round-robin. 6270 */ 6271 if (node_distance(local_node, node) != 6272 node_distance(local_node, prev_node)) 6273 node_load[node] = load; 6274 6275 node_order[nr_nodes++] = node; 6276 prev_node = node; 6277 load--; 6278 } 6279 6280 build_zonelists_in_node_order(pgdat, node_order, nr_nodes); 6281 build_thisnode_zonelists(pgdat); 6282 } 6283 6284 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 6285 /* 6286 * Return node id of node used for "local" allocations. 6287 * I.e., first node id of first zone in arg node's generic zonelist. 6288 * Used for initializing percpu 'numa_mem', which is used primarily 6289 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 6290 */ 6291 int local_memory_node(int node) 6292 { 6293 struct zoneref *z; 6294 6295 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 6296 gfp_zone(GFP_KERNEL), 6297 NULL); 6298 return zone_to_nid(z->zone); 6299 } 6300 #endif 6301 6302 static void setup_min_unmapped_ratio(void); 6303 static void setup_min_slab_ratio(void); 6304 #else /* CONFIG_NUMA */ 6305 6306 static void build_zonelists(pg_data_t *pgdat) 6307 { 6308 int node, local_node; 6309 struct zoneref *zonerefs; 6310 int nr_zones; 6311 6312 local_node = pgdat->node_id; 6313 6314 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 6315 nr_zones = build_zonerefs_node(pgdat, zonerefs); 6316 zonerefs += nr_zones; 6317 6318 /* 6319 * Now we build the zonelist so that it contains the zones 6320 * of all the other nodes. 6321 * We don't want to pressure a particular node, so when 6322 * building the zones for node N, we make sure that the 6323 * zones coming right after the local ones are those from 6324 * node N+1 (modulo N) 6325 */ 6326 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 6327 if (!node_online(node)) 6328 continue; 6329 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 6330 zonerefs += nr_zones; 6331 } 6332 for (node = 0; node < local_node; node++) { 6333 if (!node_online(node)) 6334 continue; 6335 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 6336 zonerefs += nr_zones; 6337 } 6338 6339 zonerefs->zone = NULL; 6340 zonerefs->zone_idx = 0; 6341 } 6342 6343 #endif /* CONFIG_NUMA */ 6344 6345 /* 6346 * Boot pageset table. One per cpu which is going to be used for all 6347 * zones and all nodes. The parameters will be set in such a way 6348 * that an item put on a list will immediately be handed over to 6349 * the buddy list. This is safe since pageset manipulation is done 6350 * with interrupts disabled. 6351 * 6352 * The boot_pagesets must be kept even after bootup is complete for 6353 * unused processors and/or zones. They do play a role for bootstrapping 6354 * hotplugged processors. 6355 * 6356 * zoneinfo_show() and maybe other functions do 6357 * not check if the processor is online before following the pageset pointer. 6358 * Other parts of the kernel may not check if the zone is available. 6359 */ 6360 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats); 6361 /* These effectively disable the pcplists in the boot pageset completely */ 6362 #define BOOT_PAGESET_HIGH 0 6363 #define BOOT_PAGESET_BATCH 1 6364 static DEFINE_PER_CPU(struct per_cpu_pages, boot_pageset); 6365 static DEFINE_PER_CPU(struct per_cpu_zonestat, boot_zonestats); 6366 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 6367 6368 static void __build_all_zonelists(void *data) 6369 { 6370 int nid; 6371 int __maybe_unused cpu; 6372 pg_data_t *self = data; 6373 static DEFINE_SPINLOCK(lock); 6374 6375 spin_lock(&lock); 6376 6377 #ifdef CONFIG_NUMA 6378 memset(node_load, 0, sizeof(node_load)); 6379 #endif 6380 6381 /* 6382 * This node is hotadded and no memory is yet present. So just 6383 * building zonelists is fine - no need to touch other nodes. 6384 */ 6385 if (self && !node_online(self->node_id)) { 6386 build_zonelists(self); 6387 } else { 6388 for_each_online_node(nid) { 6389 pg_data_t *pgdat = NODE_DATA(nid); 6390 6391 build_zonelists(pgdat); 6392 } 6393 6394 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 6395 /* 6396 * We now know the "local memory node" for each node-- 6397 * i.e., the node of the first zone in the generic zonelist. 6398 * Set up numa_mem percpu variable for on-line cpus. During 6399 * boot, only the boot cpu should be on-line; we'll init the 6400 * secondary cpus' numa_mem as they come on-line. During 6401 * node/memory hotplug, we'll fixup all on-line cpus. 6402 */ 6403 for_each_online_cpu(cpu) 6404 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 6405 #endif 6406 } 6407 6408 spin_unlock(&lock); 6409 } 6410 6411 static noinline void __init 6412 build_all_zonelists_init(void) 6413 { 6414 int cpu; 6415 6416 __build_all_zonelists(NULL); 6417 6418 /* 6419 * Initialize the boot_pagesets that are going to be used 6420 * for bootstrapping processors. The real pagesets for 6421 * each zone will be allocated later when the per cpu 6422 * allocator is available. 6423 * 6424 * boot_pagesets are used also for bootstrapping offline 6425 * cpus if the system is already booted because the pagesets 6426 * are needed to initialize allocators on a specific cpu too. 6427 * F.e. the percpu allocator needs the page allocator which 6428 * needs the percpu allocator in order to allocate its pagesets 6429 * (a chicken-egg dilemma). 6430 */ 6431 for_each_possible_cpu(cpu) 6432 per_cpu_pages_init(&per_cpu(boot_pageset, cpu), &per_cpu(boot_zonestats, cpu)); 6433 6434 mminit_verify_zonelist(); 6435 cpuset_init_current_mems_allowed(); 6436 } 6437 6438 /* 6439 * unless system_state == SYSTEM_BOOTING. 6440 * 6441 * __ref due to call of __init annotated helper build_all_zonelists_init 6442 * [protected by SYSTEM_BOOTING]. 6443 */ 6444 void __ref build_all_zonelists(pg_data_t *pgdat) 6445 { 6446 unsigned long vm_total_pages; 6447 6448 if (system_state == SYSTEM_BOOTING) { 6449 build_all_zonelists_init(); 6450 } else { 6451 __build_all_zonelists(pgdat); 6452 /* cpuset refresh routine should be here */ 6453 } 6454 /* Get the number of free pages beyond high watermark in all zones. */ 6455 vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 6456 /* 6457 * Disable grouping by mobility if the number of pages in the 6458 * system is too low to allow the mechanism to work. It would be 6459 * more accurate, but expensive to check per-zone. This check is 6460 * made on memory-hotadd so a system can start with mobility 6461 * disabled and enable it later 6462 */ 6463 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 6464 page_group_by_mobility_disabled = 1; 6465 else 6466 page_group_by_mobility_disabled = 0; 6467 6468 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n", 6469 nr_online_nodes, 6470 page_group_by_mobility_disabled ? "off" : "on", 6471 vm_total_pages); 6472 #ifdef CONFIG_NUMA 6473 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 6474 #endif 6475 } 6476 6477 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 6478 static bool __meminit 6479 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 6480 { 6481 static struct memblock_region *r; 6482 6483 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 6484 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 6485 for_each_mem_region(r) { 6486 if (*pfn < memblock_region_memory_end_pfn(r)) 6487 break; 6488 } 6489 } 6490 if (*pfn >= memblock_region_memory_base_pfn(r) && 6491 memblock_is_mirror(r)) { 6492 *pfn = memblock_region_memory_end_pfn(r); 6493 return true; 6494 } 6495 } 6496 return false; 6497 } 6498 6499 /* 6500 * Initially all pages are reserved - free ones are freed 6501 * up by memblock_free_all() once the early boot process is 6502 * done. Non-atomic initialization, single-pass. 6503 * 6504 * All aligned pageblocks are initialized to the specified migratetype 6505 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related 6506 * zone stats (e.g., nr_isolate_pageblock) are touched. 6507 */ 6508 void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone, 6509 unsigned long start_pfn, unsigned long zone_end_pfn, 6510 enum meminit_context context, 6511 struct vmem_altmap *altmap, int migratetype) 6512 { 6513 unsigned long pfn, end_pfn = start_pfn + size; 6514 struct page *page; 6515 6516 if (highest_memmap_pfn < end_pfn - 1) 6517 highest_memmap_pfn = end_pfn - 1; 6518 6519 #ifdef CONFIG_ZONE_DEVICE 6520 /* 6521 * Honor reservation requested by the driver for this ZONE_DEVICE 6522 * memory. We limit the total number of pages to initialize to just 6523 * those that might contain the memory mapping. We will defer the 6524 * ZONE_DEVICE page initialization until after we have released 6525 * the hotplug lock. 6526 */ 6527 if (zone == ZONE_DEVICE) { 6528 if (!altmap) 6529 return; 6530 6531 if (start_pfn == altmap->base_pfn) 6532 start_pfn += altmap->reserve; 6533 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6534 } 6535 #endif 6536 6537 for (pfn = start_pfn; pfn < end_pfn; ) { 6538 /* 6539 * There can be holes in boot-time mem_map[]s handed to this 6540 * function. They do not exist on hotplugged memory. 6541 */ 6542 if (context == MEMINIT_EARLY) { 6543 if (overlap_memmap_init(zone, &pfn)) 6544 continue; 6545 if (defer_init(nid, pfn, zone_end_pfn)) 6546 break; 6547 } 6548 6549 page = pfn_to_page(pfn); 6550 __init_single_page(page, pfn, zone, nid); 6551 if (context == MEMINIT_HOTPLUG) 6552 __SetPageReserved(page); 6553 6554 /* 6555 * Usually, we want to mark the pageblock MIGRATE_MOVABLE, 6556 * such that unmovable allocations won't be scattered all 6557 * over the place during system boot. 6558 */ 6559 if (IS_ALIGNED(pfn, pageblock_nr_pages)) { 6560 set_pageblock_migratetype(page, migratetype); 6561 cond_resched(); 6562 } 6563 pfn++; 6564 } 6565 } 6566 6567 #ifdef CONFIG_ZONE_DEVICE 6568 void __ref memmap_init_zone_device(struct zone *zone, 6569 unsigned long start_pfn, 6570 unsigned long nr_pages, 6571 struct dev_pagemap *pgmap) 6572 { 6573 unsigned long pfn, end_pfn = start_pfn + nr_pages; 6574 struct pglist_data *pgdat = zone->zone_pgdat; 6575 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 6576 unsigned long zone_idx = zone_idx(zone); 6577 unsigned long start = jiffies; 6578 int nid = pgdat->node_id; 6579 6580 if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE)) 6581 return; 6582 6583 /* 6584 * The call to memmap_init should have already taken care 6585 * of the pages reserved for the memmap, so we can just jump to 6586 * the end of that region and start processing the device pages. 6587 */ 6588 if (altmap) { 6589 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6590 nr_pages = end_pfn - start_pfn; 6591 } 6592 6593 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 6594 struct page *page = pfn_to_page(pfn); 6595 6596 __init_single_page(page, pfn, zone_idx, nid); 6597 6598 /* 6599 * Mark page reserved as it will need to wait for onlining 6600 * phase for it to be fully associated with a zone. 6601 * 6602 * We can use the non-atomic __set_bit operation for setting 6603 * the flag as we are still initializing the pages. 6604 */ 6605 __SetPageReserved(page); 6606 6607 /* 6608 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 6609 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 6610 * ever freed or placed on a driver-private list. 6611 */ 6612 page->pgmap = pgmap; 6613 page->zone_device_data = NULL; 6614 6615 /* 6616 * Mark the block movable so that blocks are reserved for 6617 * movable at startup. This will force kernel allocations 6618 * to reserve their blocks rather than leaking throughout 6619 * the address space during boot when many long-lived 6620 * kernel allocations are made. 6621 * 6622 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap 6623 * because this is done early in section_activate() 6624 */ 6625 if (IS_ALIGNED(pfn, pageblock_nr_pages)) { 6626 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 6627 cond_resched(); 6628 } 6629 } 6630 6631 pr_info("%s initialised %lu pages in %ums\n", __func__, 6632 nr_pages, jiffies_to_msecs(jiffies - start)); 6633 } 6634 6635 #endif 6636 static void __meminit zone_init_free_lists(struct zone *zone) 6637 { 6638 unsigned int order, t; 6639 for_each_migratetype_order(order, t) { 6640 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 6641 zone->free_area[order].nr_free = 0; 6642 } 6643 } 6644 6645 /* 6646 * Only struct pages that correspond to ranges defined by memblock.memory 6647 * are zeroed and initialized by going through __init_single_page() during 6648 * memmap_init_zone_range(). 6649 * 6650 * But, there could be struct pages that correspond to holes in 6651 * memblock.memory. This can happen because of the following reasons: 6652 * - physical memory bank size is not necessarily the exact multiple of the 6653 * arbitrary section size 6654 * - early reserved memory may not be listed in memblock.memory 6655 * - memory layouts defined with memmap= kernel parameter may not align 6656 * nicely with memmap sections 6657 * 6658 * Explicitly initialize those struct pages so that: 6659 * - PG_Reserved is set 6660 * - zone and node links point to zone and node that span the page if the 6661 * hole is in the middle of a zone 6662 * - zone and node links point to adjacent zone/node if the hole falls on 6663 * the zone boundary; the pages in such holes will be prepended to the 6664 * zone/node above the hole except for the trailing pages in the last 6665 * section that will be appended to the zone/node below. 6666 */ 6667 static void __init init_unavailable_range(unsigned long spfn, 6668 unsigned long epfn, 6669 int zone, int node) 6670 { 6671 unsigned long pfn; 6672 u64 pgcnt = 0; 6673 6674 for (pfn = spfn; pfn < epfn; pfn++) { 6675 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) { 6676 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages) 6677 + pageblock_nr_pages - 1; 6678 continue; 6679 } 6680 __init_single_page(pfn_to_page(pfn), pfn, zone, node); 6681 __SetPageReserved(pfn_to_page(pfn)); 6682 pgcnt++; 6683 } 6684 6685 if (pgcnt) 6686 pr_info("On node %d, zone %s: %lld pages in unavailable ranges", 6687 node, zone_names[zone], pgcnt); 6688 } 6689 6690 static void __init memmap_init_zone_range(struct zone *zone, 6691 unsigned long start_pfn, 6692 unsigned long end_pfn, 6693 unsigned long *hole_pfn) 6694 { 6695 unsigned long zone_start_pfn = zone->zone_start_pfn; 6696 unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; 6697 int nid = zone_to_nid(zone), zone_id = zone_idx(zone); 6698 6699 start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn); 6700 end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn); 6701 6702 if (start_pfn >= end_pfn) 6703 return; 6704 6705 memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, 6706 zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); 6707 6708 if (*hole_pfn < start_pfn) 6709 init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); 6710 6711 *hole_pfn = end_pfn; 6712 } 6713 6714 static void __init memmap_init(void) 6715 { 6716 unsigned long start_pfn, end_pfn; 6717 unsigned long hole_pfn = 0; 6718 int i, j, zone_id = 0, nid; 6719 6720 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 6721 struct pglist_data *node = NODE_DATA(nid); 6722 6723 for (j = 0; j < MAX_NR_ZONES; j++) { 6724 struct zone *zone = node->node_zones + j; 6725 6726 if (!populated_zone(zone)) 6727 continue; 6728 6729 memmap_init_zone_range(zone, start_pfn, end_pfn, 6730 &hole_pfn); 6731 zone_id = j; 6732 } 6733 } 6734 6735 #ifdef CONFIG_SPARSEMEM 6736 /* 6737 * Initialize the memory map for hole in the range [memory_end, 6738 * section_end]. 6739 * Append the pages in this hole to the highest zone in the last 6740 * node. 6741 * The call to init_unavailable_range() is outside the ifdef to 6742 * silence the compiler warining about zone_id set but not used; 6743 * for FLATMEM it is a nop anyway 6744 */ 6745 end_pfn = round_up(end_pfn, PAGES_PER_SECTION); 6746 if (hole_pfn < end_pfn) 6747 #endif 6748 init_unavailable_range(hole_pfn, end_pfn, zone_id, nid); 6749 } 6750 6751 void __init *memmap_alloc(phys_addr_t size, phys_addr_t align, 6752 phys_addr_t min_addr, int nid, bool exact_nid) 6753 { 6754 void *ptr; 6755 6756 if (exact_nid) 6757 ptr = memblock_alloc_exact_nid_raw(size, align, min_addr, 6758 MEMBLOCK_ALLOC_ACCESSIBLE, 6759 nid); 6760 else 6761 ptr = memblock_alloc_try_nid_raw(size, align, min_addr, 6762 MEMBLOCK_ALLOC_ACCESSIBLE, 6763 nid); 6764 6765 if (ptr && size > 0) 6766 page_init_poison(ptr, size); 6767 6768 return ptr; 6769 } 6770 6771 static int zone_batchsize(struct zone *zone) 6772 { 6773 #ifdef CONFIG_MMU 6774 int batch; 6775 6776 /* 6777 * The number of pages to batch allocate is either ~0.1% 6778 * of the zone or 1MB, whichever is smaller. The batch 6779 * size is striking a balance between allocation latency 6780 * and zone lock contention. 6781 */ 6782 batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE); 6783 batch /= 4; /* We effectively *= 4 below */ 6784 if (batch < 1) 6785 batch = 1; 6786 6787 /* 6788 * Clamp the batch to a 2^n - 1 value. Having a power 6789 * of 2 value was found to be more likely to have 6790 * suboptimal cache aliasing properties in some cases. 6791 * 6792 * For example if 2 tasks are alternately allocating 6793 * batches of pages, one task can end up with a lot 6794 * of pages of one half of the possible page colors 6795 * and the other with pages of the other colors. 6796 */ 6797 batch = rounddown_pow_of_two(batch + batch/2) - 1; 6798 6799 return batch; 6800 6801 #else 6802 /* The deferral and batching of frees should be suppressed under NOMMU 6803 * conditions. 6804 * 6805 * The problem is that NOMMU needs to be able to allocate large chunks 6806 * of contiguous memory as there's no hardware page translation to 6807 * assemble apparent contiguous memory from discontiguous pages. 6808 * 6809 * Queueing large contiguous runs of pages for batching, however, 6810 * causes the pages to actually be freed in smaller chunks. As there 6811 * can be a significant delay between the individual batches being 6812 * recycled, this leads to the once large chunks of space being 6813 * fragmented and becoming unavailable for high-order allocations. 6814 */ 6815 return 0; 6816 #endif 6817 } 6818 6819 static int zone_highsize(struct zone *zone, int batch, int cpu_online) 6820 { 6821 #ifdef CONFIG_MMU 6822 int high; 6823 int nr_split_cpus; 6824 unsigned long total_pages; 6825 6826 if (!percpu_pagelist_high_fraction) { 6827 /* 6828 * By default, the high value of the pcp is based on the zone 6829 * low watermark so that if they are full then background 6830 * reclaim will not be started prematurely. 6831 */ 6832 total_pages = low_wmark_pages(zone); 6833 } else { 6834 /* 6835 * If percpu_pagelist_high_fraction is configured, the high 6836 * value is based on a fraction of the managed pages in the 6837 * zone. 6838 */ 6839 total_pages = zone_managed_pages(zone) / percpu_pagelist_high_fraction; 6840 } 6841 6842 /* 6843 * Split the high value across all online CPUs local to the zone. Note 6844 * that early in boot that CPUs may not be online yet and that during 6845 * CPU hotplug that the cpumask is not yet updated when a CPU is being 6846 * onlined. For memory nodes that have no CPUs, split pcp->high across 6847 * all online CPUs to mitigate the risk that reclaim is triggered 6848 * prematurely due to pages stored on pcp lists. 6849 */ 6850 nr_split_cpus = cpumask_weight(cpumask_of_node(zone_to_nid(zone))) + cpu_online; 6851 if (!nr_split_cpus) 6852 nr_split_cpus = num_online_cpus(); 6853 high = total_pages / nr_split_cpus; 6854 6855 /* 6856 * Ensure high is at least batch*4. The multiple is based on the 6857 * historical relationship between high and batch. 6858 */ 6859 high = max(high, batch << 2); 6860 6861 return high; 6862 #else 6863 return 0; 6864 #endif 6865 } 6866 6867 /* 6868 * pcp->high and pcp->batch values are related and generally batch is lower 6869 * than high. They are also related to pcp->count such that count is lower 6870 * than high, and as soon as it reaches high, the pcplist is flushed. 6871 * 6872 * However, guaranteeing these relations at all times would require e.g. write 6873 * barriers here but also careful usage of read barriers at the read side, and 6874 * thus be prone to error and bad for performance. Thus the update only prevents 6875 * store tearing. Any new users of pcp->batch and pcp->high should ensure they 6876 * can cope with those fields changing asynchronously, and fully trust only the 6877 * pcp->count field on the local CPU with interrupts disabled. 6878 * 6879 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 6880 * outside of boot time (or some other assurance that no concurrent updaters 6881 * exist). 6882 */ 6883 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 6884 unsigned long batch) 6885 { 6886 WRITE_ONCE(pcp->batch, batch); 6887 WRITE_ONCE(pcp->high, high); 6888 } 6889 6890 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats) 6891 { 6892 int pindex; 6893 6894 memset(pcp, 0, sizeof(*pcp)); 6895 memset(pzstats, 0, sizeof(*pzstats)); 6896 6897 for (pindex = 0; pindex < NR_PCP_LISTS; pindex++) 6898 INIT_LIST_HEAD(&pcp->lists[pindex]); 6899 6900 /* 6901 * Set batch and high values safe for a boot pageset. A true percpu 6902 * pageset's initialization will update them subsequently. Here we don't 6903 * need to be as careful as pageset_update() as nobody can access the 6904 * pageset yet. 6905 */ 6906 pcp->high = BOOT_PAGESET_HIGH; 6907 pcp->batch = BOOT_PAGESET_BATCH; 6908 pcp->free_factor = 0; 6909 } 6910 6911 static void __zone_set_pageset_high_and_batch(struct zone *zone, unsigned long high, 6912 unsigned long batch) 6913 { 6914 struct per_cpu_pages *pcp; 6915 int cpu; 6916 6917 for_each_possible_cpu(cpu) { 6918 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 6919 pageset_update(pcp, high, batch); 6920 } 6921 } 6922 6923 /* 6924 * Calculate and set new high and batch values for all per-cpu pagesets of a 6925 * zone based on the zone's size. 6926 */ 6927 static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online) 6928 { 6929 int new_high, new_batch; 6930 6931 new_batch = max(1, zone_batchsize(zone)); 6932 new_high = zone_highsize(zone, new_batch, cpu_online); 6933 6934 if (zone->pageset_high == new_high && 6935 zone->pageset_batch == new_batch) 6936 return; 6937 6938 zone->pageset_high = new_high; 6939 zone->pageset_batch = new_batch; 6940 6941 __zone_set_pageset_high_and_batch(zone, new_high, new_batch); 6942 } 6943 6944 void __meminit setup_zone_pageset(struct zone *zone) 6945 { 6946 int cpu; 6947 6948 /* Size may be 0 on !SMP && !NUMA */ 6949 if (sizeof(struct per_cpu_zonestat) > 0) 6950 zone->per_cpu_zonestats = alloc_percpu(struct per_cpu_zonestat); 6951 6952 zone->per_cpu_pageset = alloc_percpu(struct per_cpu_pages); 6953 for_each_possible_cpu(cpu) { 6954 struct per_cpu_pages *pcp; 6955 struct per_cpu_zonestat *pzstats; 6956 6957 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 6958 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 6959 per_cpu_pages_init(pcp, pzstats); 6960 } 6961 6962 zone_set_pageset_high_and_batch(zone, 0); 6963 } 6964 6965 /* 6966 * Allocate per cpu pagesets and initialize them. 6967 * Before this call only boot pagesets were available. 6968 */ 6969 void __init setup_per_cpu_pageset(void) 6970 { 6971 struct pglist_data *pgdat; 6972 struct zone *zone; 6973 int __maybe_unused cpu; 6974 6975 for_each_populated_zone(zone) 6976 setup_zone_pageset(zone); 6977 6978 #ifdef CONFIG_NUMA 6979 /* 6980 * Unpopulated zones continue using the boot pagesets. 6981 * The numa stats for these pagesets need to be reset. 6982 * Otherwise, they will end up skewing the stats of 6983 * the nodes these zones are associated with. 6984 */ 6985 for_each_possible_cpu(cpu) { 6986 struct per_cpu_zonestat *pzstats = &per_cpu(boot_zonestats, cpu); 6987 memset(pzstats->vm_numa_event, 0, 6988 sizeof(pzstats->vm_numa_event)); 6989 } 6990 #endif 6991 6992 for_each_online_pgdat(pgdat) 6993 pgdat->per_cpu_nodestats = 6994 alloc_percpu(struct per_cpu_nodestat); 6995 } 6996 6997 static __meminit void zone_pcp_init(struct zone *zone) 6998 { 6999 /* 7000 * per cpu subsystem is not up at this point. The following code 7001 * relies on the ability of the linker to provide the 7002 * offset of a (static) per cpu variable into the per cpu area. 7003 */ 7004 zone->per_cpu_pageset = &boot_pageset; 7005 zone->per_cpu_zonestats = &boot_zonestats; 7006 zone->pageset_high = BOOT_PAGESET_HIGH; 7007 zone->pageset_batch = BOOT_PAGESET_BATCH; 7008 7009 if (populated_zone(zone)) 7010 pr_debug(" %s zone: %lu pages, LIFO batch:%u\n", zone->name, 7011 zone->present_pages, zone_batchsize(zone)); 7012 } 7013 7014 void __meminit init_currently_empty_zone(struct zone *zone, 7015 unsigned long zone_start_pfn, 7016 unsigned long size) 7017 { 7018 struct pglist_data *pgdat = zone->zone_pgdat; 7019 int zone_idx = zone_idx(zone) + 1; 7020 7021 if (zone_idx > pgdat->nr_zones) 7022 pgdat->nr_zones = zone_idx; 7023 7024 zone->zone_start_pfn = zone_start_pfn; 7025 7026 mminit_dprintk(MMINIT_TRACE, "memmap_init", 7027 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 7028 pgdat->node_id, 7029 (unsigned long)zone_idx(zone), 7030 zone_start_pfn, (zone_start_pfn + size)); 7031 7032 zone_init_free_lists(zone); 7033 zone->initialized = 1; 7034 } 7035 7036 /** 7037 * get_pfn_range_for_nid - Return the start and end page frames for a node 7038 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 7039 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 7040 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 7041 * 7042 * It returns the start and end page frame of a node based on information 7043 * provided by memblock_set_node(). If called for a node 7044 * with no available memory, a warning is printed and the start and end 7045 * PFNs will be 0. 7046 */ 7047 void __init get_pfn_range_for_nid(unsigned int nid, 7048 unsigned long *start_pfn, unsigned long *end_pfn) 7049 { 7050 unsigned long this_start_pfn, this_end_pfn; 7051 int i; 7052 7053 *start_pfn = -1UL; 7054 *end_pfn = 0; 7055 7056 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 7057 *start_pfn = min(*start_pfn, this_start_pfn); 7058 *end_pfn = max(*end_pfn, this_end_pfn); 7059 } 7060 7061 if (*start_pfn == -1UL) 7062 *start_pfn = 0; 7063 } 7064 7065 /* 7066 * This finds a zone that can be used for ZONE_MOVABLE pages. The 7067 * assumption is made that zones within a node are ordered in monotonic 7068 * increasing memory addresses so that the "highest" populated zone is used 7069 */ 7070 static void __init find_usable_zone_for_movable(void) 7071 { 7072 int zone_index; 7073 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 7074 if (zone_index == ZONE_MOVABLE) 7075 continue; 7076 7077 if (arch_zone_highest_possible_pfn[zone_index] > 7078 arch_zone_lowest_possible_pfn[zone_index]) 7079 break; 7080 } 7081 7082 VM_BUG_ON(zone_index == -1); 7083 movable_zone = zone_index; 7084 } 7085 7086 /* 7087 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 7088 * because it is sized independent of architecture. Unlike the other zones, 7089 * the starting point for ZONE_MOVABLE is not fixed. It may be different 7090 * in each node depending on the size of each node and how evenly kernelcore 7091 * is distributed. This helper function adjusts the zone ranges 7092 * provided by the architecture for a given node by using the end of the 7093 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 7094 * zones within a node are in order of monotonic increases memory addresses 7095 */ 7096 static void __init adjust_zone_range_for_zone_movable(int nid, 7097 unsigned long zone_type, 7098 unsigned long node_start_pfn, 7099 unsigned long node_end_pfn, 7100 unsigned long *zone_start_pfn, 7101 unsigned long *zone_end_pfn) 7102 { 7103 /* Only adjust if ZONE_MOVABLE is on this node */ 7104 if (zone_movable_pfn[nid]) { 7105 /* Size ZONE_MOVABLE */ 7106 if (zone_type == ZONE_MOVABLE) { 7107 *zone_start_pfn = zone_movable_pfn[nid]; 7108 *zone_end_pfn = min(node_end_pfn, 7109 arch_zone_highest_possible_pfn[movable_zone]); 7110 7111 /* Adjust for ZONE_MOVABLE starting within this range */ 7112 } else if (!mirrored_kernelcore && 7113 *zone_start_pfn < zone_movable_pfn[nid] && 7114 *zone_end_pfn > zone_movable_pfn[nid]) { 7115 *zone_end_pfn = zone_movable_pfn[nid]; 7116 7117 /* Check if this whole range is within ZONE_MOVABLE */ 7118 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 7119 *zone_start_pfn = *zone_end_pfn; 7120 } 7121 } 7122 7123 /* 7124 * Return the number of pages a zone spans in a node, including holes 7125 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 7126 */ 7127 static unsigned long __init zone_spanned_pages_in_node(int nid, 7128 unsigned long zone_type, 7129 unsigned long node_start_pfn, 7130 unsigned long node_end_pfn, 7131 unsigned long *zone_start_pfn, 7132 unsigned long *zone_end_pfn) 7133 { 7134 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 7135 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 7136 /* When hotadd a new node from cpu_up(), the node should be empty */ 7137 if (!node_start_pfn && !node_end_pfn) 7138 return 0; 7139 7140 /* Get the start and end of the zone */ 7141 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 7142 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 7143 adjust_zone_range_for_zone_movable(nid, zone_type, 7144 node_start_pfn, node_end_pfn, 7145 zone_start_pfn, zone_end_pfn); 7146 7147 /* Check that this node has pages within the zone's required range */ 7148 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 7149 return 0; 7150 7151 /* Move the zone boundaries inside the node if necessary */ 7152 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 7153 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 7154 7155 /* Return the spanned pages */ 7156 return *zone_end_pfn - *zone_start_pfn; 7157 } 7158 7159 /* 7160 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 7161 * then all holes in the requested range will be accounted for. 7162 */ 7163 unsigned long __init __absent_pages_in_range(int nid, 7164 unsigned long range_start_pfn, 7165 unsigned long range_end_pfn) 7166 { 7167 unsigned long nr_absent = range_end_pfn - range_start_pfn; 7168 unsigned long start_pfn, end_pfn; 7169 int i; 7170 7171 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7172 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 7173 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 7174 nr_absent -= end_pfn - start_pfn; 7175 } 7176 return nr_absent; 7177 } 7178 7179 /** 7180 * absent_pages_in_range - Return number of page frames in holes within a range 7181 * @start_pfn: The start PFN to start searching for holes 7182 * @end_pfn: The end PFN to stop searching for holes 7183 * 7184 * Return: the number of pages frames in memory holes within a range. 7185 */ 7186 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 7187 unsigned long end_pfn) 7188 { 7189 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 7190 } 7191 7192 /* Return the number of page frames in holes in a zone on a node */ 7193 static unsigned long __init zone_absent_pages_in_node(int nid, 7194 unsigned long zone_type, 7195 unsigned long node_start_pfn, 7196 unsigned long node_end_pfn) 7197 { 7198 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 7199 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 7200 unsigned long zone_start_pfn, zone_end_pfn; 7201 unsigned long nr_absent; 7202 7203 /* When hotadd a new node from cpu_up(), the node should be empty */ 7204 if (!node_start_pfn && !node_end_pfn) 7205 return 0; 7206 7207 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 7208 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 7209 7210 adjust_zone_range_for_zone_movable(nid, zone_type, 7211 node_start_pfn, node_end_pfn, 7212 &zone_start_pfn, &zone_end_pfn); 7213 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 7214 7215 /* 7216 * ZONE_MOVABLE handling. 7217 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 7218 * and vice versa. 7219 */ 7220 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 7221 unsigned long start_pfn, end_pfn; 7222 struct memblock_region *r; 7223 7224 for_each_mem_region(r) { 7225 start_pfn = clamp(memblock_region_memory_base_pfn(r), 7226 zone_start_pfn, zone_end_pfn); 7227 end_pfn = clamp(memblock_region_memory_end_pfn(r), 7228 zone_start_pfn, zone_end_pfn); 7229 7230 if (zone_type == ZONE_MOVABLE && 7231 memblock_is_mirror(r)) 7232 nr_absent += end_pfn - start_pfn; 7233 7234 if (zone_type == ZONE_NORMAL && 7235 !memblock_is_mirror(r)) 7236 nr_absent += end_pfn - start_pfn; 7237 } 7238 } 7239 7240 return nr_absent; 7241 } 7242 7243 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 7244 unsigned long node_start_pfn, 7245 unsigned long node_end_pfn) 7246 { 7247 unsigned long realtotalpages = 0, totalpages = 0; 7248 enum zone_type i; 7249 7250 for (i = 0; i < MAX_NR_ZONES; i++) { 7251 struct zone *zone = pgdat->node_zones + i; 7252 unsigned long zone_start_pfn, zone_end_pfn; 7253 unsigned long spanned, absent; 7254 unsigned long size, real_size; 7255 7256 spanned = zone_spanned_pages_in_node(pgdat->node_id, i, 7257 node_start_pfn, 7258 node_end_pfn, 7259 &zone_start_pfn, 7260 &zone_end_pfn); 7261 absent = zone_absent_pages_in_node(pgdat->node_id, i, 7262 node_start_pfn, 7263 node_end_pfn); 7264 7265 size = spanned; 7266 real_size = size - absent; 7267 7268 if (size) 7269 zone->zone_start_pfn = zone_start_pfn; 7270 else 7271 zone->zone_start_pfn = 0; 7272 zone->spanned_pages = size; 7273 zone->present_pages = real_size; 7274 7275 totalpages += size; 7276 realtotalpages += real_size; 7277 } 7278 7279 pgdat->node_spanned_pages = totalpages; 7280 pgdat->node_present_pages = realtotalpages; 7281 pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); 7282 } 7283 7284 #ifndef CONFIG_SPARSEMEM 7285 /* 7286 * Calculate the size of the zone->blockflags rounded to an unsigned long 7287 * Start by making sure zonesize is a multiple of pageblock_order by rounding 7288 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 7289 * round what is now in bits to nearest long in bits, then return it in 7290 * bytes. 7291 */ 7292 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 7293 { 7294 unsigned long usemapsize; 7295 7296 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 7297 usemapsize = roundup(zonesize, pageblock_nr_pages); 7298 usemapsize = usemapsize >> pageblock_order; 7299 usemapsize *= NR_PAGEBLOCK_BITS; 7300 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 7301 7302 return usemapsize / 8; 7303 } 7304 7305 static void __ref setup_usemap(struct zone *zone) 7306 { 7307 unsigned long usemapsize = usemap_size(zone->zone_start_pfn, 7308 zone->spanned_pages); 7309 zone->pageblock_flags = NULL; 7310 if (usemapsize) { 7311 zone->pageblock_flags = 7312 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 7313 zone_to_nid(zone)); 7314 if (!zone->pageblock_flags) 7315 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 7316 usemapsize, zone->name, zone_to_nid(zone)); 7317 } 7318 } 7319 #else 7320 static inline void setup_usemap(struct zone *zone) {} 7321 #endif /* CONFIG_SPARSEMEM */ 7322 7323 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 7324 7325 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 7326 void __init set_pageblock_order(void) 7327 { 7328 unsigned int order; 7329 7330 /* Check that pageblock_nr_pages has not already been setup */ 7331 if (pageblock_order) 7332 return; 7333 7334 if (HPAGE_SHIFT > PAGE_SHIFT) 7335 order = HUGETLB_PAGE_ORDER; 7336 else 7337 order = MAX_ORDER - 1; 7338 7339 /* 7340 * Assume the largest contiguous order of interest is a huge page. 7341 * This value may be variable depending on boot parameters on IA64 and 7342 * powerpc. 7343 */ 7344 pageblock_order = order; 7345 } 7346 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 7347 7348 /* 7349 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 7350 * is unused as pageblock_order is set at compile-time. See 7351 * include/linux/pageblock-flags.h for the values of pageblock_order based on 7352 * the kernel config 7353 */ 7354 void __init set_pageblock_order(void) 7355 { 7356 } 7357 7358 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 7359 7360 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 7361 unsigned long present_pages) 7362 { 7363 unsigned long pages = spanned_pages; 7364 7365 /* 7366 * Provide a more accurate estimation if there are holes within 7367 * the zone and SPARSEMEM is in use. If there are holes within the 7368 * zone, each populated memory region may cost us one or two extra 7369 * memmap pages due to alignment because memmap pages for each 7370 * populated regions may not be naturally aligned on page boundary. 7371 * So the (present_pages >> 4) heuristic is a tradeoff for that. 7372 */ 7373 if (spanned_pages > present_pages + (present_pages >> 4) && 7374 IS_ENABLED(CONFIG_SPARSEMEM)) 7375 pages = present_pages; 7376 7377 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 7378 } 7379 7380 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 7381 static void pgdat_init_split_queue(struct pglist_data *pgdat) 7382 { 7383 struct deferred_split *ds_queue = &pgdat->deferred_split_queue; 7384 7385 spin_lock_init(&ds_queue->split_queue_lock); 7386 INIT_LIST_HEAD(&ds_queue->split_queue); 7387 ds_queue->split_queue_len = 0; 7388 } 7389 #else 7390 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 7391 #endif 7392 7393 #ifdef CONFIG_COMPACTION 7394 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 7395 { 7396 init_waitqueue_head(&pgdat->kcompactd_wait); 7397 } 7398 #else 7399 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 7400 #endif 7401 7402 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 7403 { 7404 pgdat_resize_init(pgdat); 7405 7406 pgdat_init_split_queue(pgdat); 7407 pgdat_init_kcompactd(pgdat); 7408 7409 init_waitqueue_head(&pgdat->kswapd_wait); 7410 init_waitqueue_head(&pgdat->pfmemalloc_wait); 7411 7412 pgdat_page_ext_init(pgdat); 7413 lruvec_init(&pgdat->__lruvec); 7414 } 7415 7416 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 7417 unsigned long remaining_pages) 7418 { 7419 atomic_long_set(&zone->managed_pages, remaining_pages); 7420 zone_set_nid(zone, nid); 7421 zone->name = zone_names[idx]; 7422 zone->zone_pgdat = NODE_DATA(nid); 7423 spin_lock_init(&zone->lock); 7424 zone_seqlock_init(zone); 7425 zone_pcp_init(zone); 7426 } 7427 7428 /* 7429 * Set up the zone data structures 7430 * - init pgdat internals 7431 * - init all zones belonging to this node 7432 * 7433 * NOTE: this function is only called during memory hotplug 7434 */ 7435 #ifdef CONFIG_MEMORY_HOTPLUG 7436 void __ref free_area_init_core_hotplug(int nid) 7437 { 7438 enum zone_type z; 7439 pg_data_t *pgdat = NODE_DATA(nid); 7440 7441 pgdat_init_internals(pgdat); 7442 for (z = 0; z < MAX_NR_ZONES; z++) 7443 zone_init_internals(&pgdat->node_zones[z], z, nid, 0); 7444 } 7445 #endif 7446 7447 /* 7448 * Set up the zone data structures: 7449 * - mark all pages reserved 7450 * - mark all memory queues empty 7451 * - clear the memory bitmaps 7452 * 7453 * NOTE: pgdat should get zeroed by caller. 7454 * NOTE: this function is only called during early init. 7455 */ 7456 static void __init free_area_init_core(struct pglist_data *pgdat) 7457 { 7458 enum zone_type j; 7459 int nid = pgdat->node_id; 7460 7461 pgdat_init_internals(pgdat); 7462 pgdat->per_cpu_nodestats = &boot_nodestats; 7463 7464 for (j = 0; j < MAX_NR_ZONES; j++) { 7465 struct zone *zone = pgdat->node_zones + j; 7466 unsigned long size, freesize, memmap_pages; 7467 7468 size = zone->spanned_pages; 7469 freesize = zone->present_pages; 7470 7471 /* 7472 * Adjust freesize so that it accounts for how much memory 7473 * is used by this zone for memmap. This affects the watermark 7474 * and per-cpu initialisations 7475 */ 7476 memmap_pages = calc_memmap_size(size, freesize); 7477 if (!is_highmem_idx(j)) { 7478 if (freesize >= memmap_pages) { 7479 freesize -= memmap_pages; 7480 if (memmap_pages) 7481 pr_debug(" %s zone: %lu pages used for memmap\n", 7482 zone_names[j], memmap_pages); 7483 } else 7484 pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n", 7485 zone_names[j], memmap_pages, freesize); 7486 } 7487 7488 /* Account for reserved pages */ 7489 if (j == 0 && freesize > dma_reserve) { 7490 freesize -= dma_reserve; 7491 pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve); 7492 } 7493 7494 if (!is_highmem_idx(j)) 7495 nr_kernel_pages += freesize; 7496 /* Charge for highmem memmap if there are enough kernel pages */ 7497 else if (nr_kernel_pages > memmap_pages * 2) 7498 nr_kernel_pages -= memmap_pages; 7499 nr_all_pages += freesize; 7500 7501 /* 7502 * Set an approximate value for lowmem here, it will be adjusted 7503 * when the bootmem allocator frees pages into the buddy system. 7504 * And all highmem pages will be managed by the buddy system. 7505 */ 7506 zone_init_internals(zone, j, nid, freesize); 7507 7508 if (!size) 7509 continue; 7510 7511 set_pageblock_order(); 7512 setup_usemap(zone); 7513 init_currently_empty_zone(zone, zone->zone_start_pfn, size); 7514 } 7515 } 7516 7517 #ifdef CONFIG_FLATMEM 7518 static void __init alloc_node_mem_map(struct pglist_data *pgdat) 7519 { 7520 unsigned long __maybe_unused start = 0; 7521 unsigned long __maybe_unused offset = 0; 7522 7523 /* Skip empty nodes */ 7524 if (!pgdat->node_spanned_pages) 7525 return; 7526 7527 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 7528 offset = pgdat->node_start_pfn - start; 7529 /* ia64 gets its own node_mem_map, before this, without bootmem */ 7530 if (!pgdat->node_mem_map) { 7531 unsigned long size, end; 7532 struct page *map; 7533 7534 /* 7535 * The zone's endpoints aren't required to be MAX_ORDER 7536 * aligned but the node_mem_map endpoints must be in order 7537 * for the buddy allocator to function correctly. 7538 */ 7539 end = pgdat_end_pfn(pgdat); 7540 end = ALIGN(end, MAX_ORDER_NR_PAGES); 7541 size = (end - start) * sizeof(struct page); 7542 map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT, 7543 pgdat->node_id, false); 7544 if (!map) 7545 panic("Failed to allocate %ld bytes for node %d memory map\n", 7546 size, pgdat->node_id); 7547 pgdat->node_mem_map = map + offset; 7548 } 7549 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 7550 __func__, pgdat->node_id, (unsigned long)pgdat, 7551 (unsigned long)pgdat->node_mem_map); 7552 #ifndef CONFIG_NUMA 7553 /* 7554 * With no DISCONTIG, the global mem_map is just set as node 0's 7555 */ 7556 if (pgdat == NODE_DATA(0)) { 7557 mem_map = NODE_DATA(0)->node_mem_map; 7558 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 7559 mem_map -= offset; 7560 } 7561 #endif 7562 } 7563 #else 7564 static inline void alloc_node_mem_map(struct pglist_data *pgdat) { } 7565 #endif /* CONFIG_FLATMEM */ 7566 7567 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 7568 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 7569 { 7570 pgdat->first_deferred_pfn = ULONG_MAX; 7571 } 7572 #else 7573 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 7574 #endif 7575 7576 static void __init free_area_init_node(int nid) 7577 { 7578 pg_data_t *pgdat = NODE_DATA(nid); 7579 unsigned long start_pfn = 0; 7580 unsigned long end_pfn = 0; 7581 7582 /* pg_data_t should be reset to zero when it's allocated */ 7583 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); 7584 7585 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 7586 7587 pgdat->node_id = nid; 7588 pgdat->node_start_pfn = start_pfn; 7589 pgdat->per_cpu_nodestats = NULL; 7590 7591 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 7592 (u64)start_pfn << PAGE_SHIFT, 7593 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 7594 calculate_node_totalpages(pgdat, start_pfn, end_pfn); 7595 7596 alloc_node_mem_map(pgdat); 7597 pgdat_set_deferred_range(pgdat); 7598 7599 free_area_init_core(pgdat); 7600 } 7601 7602 void __init free_area_init_memoryless_node(int nid) 7603 { 7604 free_area_init_node(nid); 7605 } 7606 7607 #if MAX_NUMNODES > 1 7608 /* 7609 * Figure out the number of possible node ids. 7610 */ 7611 void __init setup_nr_node_ids(void) 7612 { 7613 unsigned int highest; 7614 7615 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 7616 nr_node_ids = highest + 1; 7617 } 7618 #endif 7619 7620 /** 7621 * node_map_pfn_alignment - determine the maximum internode alignment 7622 * 7623 * This function should be called after node map is populated and sorted. 7624 * It calculates the maximum power of two alignment which can distinguish 7625 * all the nodes. 7626 * 7627 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 7628 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 7629 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 7630 * shifted, 1GiB is enough and this function will indicate so. 7631 * 7632 * This is used to test whether pfn -> nid mapping of the chosen memory 7633 * model has fine enough granularity to avoid incorrect mapping for the 7634 * populated node map. 7635 * 7636 * Return: the determined alignment in pfn's. 0 if there is no alignment 7637 * requirement (single node). 7638 */ 7639 unsigned long __init node_map_pfn_alignment(void) 7640 { 7641 unsigned long accl_mask = 0, last_end = 0; 7642 unsigned long start, end, mask; 7643 int last_nid = NUMA_NO_NODE; 7644 int i, nid; 7645 7646 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 7647 if (!start || last_nid < 0 || last_nid == nid) { 7648 last_nid = nid; 7649 last_end = end; 7650 continue; 7651 } 7652 7653 /* 7654 * Start with a mask granular enough to pin-point to the 7655 * start pfn and tick off bits one-by-one until it becomes 7656 * too coarse to separate the current node from the last. 7657 */ 7658 mask = ~((1 << __ffs(start)) - 1); 7659 while (mask && last_end <= (start & (mask << 1))) 7660 mask <<= 1; 7661 7662 /* accumulate all internode masks */ 7663 accl_mask |= mask; 7664 } 7665 7666 /* convert mask to number of pages */ 7667 return ~accl_mask + 1; 7668 } 7669 7670 /** 7671 * find_min_pfn_with_active_regions - Find the minimum PFN registered 7672 * 7673 * Return: the minimum PFN based on information provided via 7674 * memblock_set_node(). 7675 */ 7676 unsigned long __init find_min_pfn_with_active_regions(void) 7677 { 7678 return PHYS_PFN(memblock_start_of_DRAM()); 7679 } 7680 7681 /* 7682 * early_calculate_totalpages() 7683 * Sum pages in active regions for movable zone. 7684 * Populate N_MEMORY for calculating usable_nodes. 7685 */ 7686 static unsigned long __init early_calculate_totalpages(void) 7687 { 7688 unsigned long totalpages = 0; 7689 unsigned long start_pfn, end_pfn; 7690 int i, nid; 7691 7692 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7693 unsigned long pages = end_pfn - start_pfn; 7694 7695 totalpages += pages; 7696 if (pages) 7697 node_set_state(nid, N_MEMORY); 7698 } 7699 return totalpages; 7700 } 7701 7702 /* 7703 * Find the PFN the Movable zone begins in each node. Kernel memory 7704 * is spread evenly between nodes as long as the nodes have enough 7705 * memory. When they don't, some nodes will have more kernelcore than 7706 * others 7707 */ 7708 static void __init find_zone_movable_pfns_for_nodes(void) 7709 { 7710 int i, nid; 7711 unsigned long usable_startpfn; 7712 unsigned long kernelcore_node, kernelcore_remaining; 7713 /* save the state before borrow the nodemask */ 7714 nodemask_t saved_node_state = node_states[N_MEMORY]; 7715 unsigned long totalpages = early_calculate_totalpages(); 7716 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 7717 struct memblock_region *r; 7718 7719 /* Need to find movable_zone earlier when movable_node is specified. */ 7720 find_usable_zone_for_movable(); 7721 7722 /* 7723 * If movable_node is specified, ignore kernelcore and movablecore 7724 * options. 7725 */ 7726 if (movable_node_is_enabled()) { 7727 for_each_mem_region(r) { 7728 if (!memblock_is_hotpluggable(r)) 7729 continue; 7730 7731 nid = memblock_get_region_node(r); 7732 7733 usable_startpfn = PFN_DOWN(r->base); 7734 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7735 min(usable_startpfn, zone_movable_pfn[nid]) : 7736 usable_startpfn; 7737 } 7738 7739 goto out2; 7740 } 7741 7742 /* 7743 * If kernelcore=mirror is specified, ignore movablecore option 7744 */ 7745 if (mirrored_kernelcore) { 7746 bool mem_below_4gb_not_mirrored = false; 7747 7748 for_each_mem_region(r) { 7749 if (memblock_is_mirror(r)) 7750 continue; 7751 7752 nid = memblock_get_region_node(r); 7753 7754 usable_startpfn = memblock_region_memory_base_pfn(r); 7755 7756 if (usable_startpfn < 0x100000) { 7757 mem_below_4gb_not_mirrored = true; 7758 continue; 7759 } 7760 7761 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7762 min(usable_startpfn, zone_movable_pfn[nid]) : 7763 usable_startpfn; 7764 } 7765 7766 if (mem_below_4gb_not_mirrored) 7767 pr_warn("This configuration results in unmirrored kernel memory.\n"); 7768 7769 goto out2; 7770 } 7771 7772 /* 7773 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 7774 * amount of necessary memory. 7775 */ 7776 if (required_kernelcore_percent) 7777 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 7778 10000UL; 7779 if (required_movablecore_percent) 7780 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 7781 10000UL; 7782 7783 /* 7784 * If movablecore= was specified, calculate what size of 7785 * kernelcore that corresponds so that memory usable for 7786 * any allocation type is evenly spread. If both kernelcore 7787 * and movablecore are specified, then the value of kernelcore 7788 * will be used for required_kernelcore if it's greater than 7789 * what movablecore would have allowed. 7790 */ 7791 if (required_movablecore) { 7792 unsigned long corepages; 7793 7794 /* 7795 * Round-up so that ZONE_MOVABLE is at least as large as what 7796 * was requested by the user 7797 */ 7798 required_movablecore = 7799 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 7800 required_movablecore = min(totalpages, required_movablecore); 7801 corepages = totalpages - required_movablecore; 7802 7803 required_kernelcore = max(required_kernelcore, corepages); 7804 } 7805 7806 /* 7807 * If kernelcore was not specified or kernelcore size is larger 7808 * than totalpages, there is no ZONE_MOVABLE. 7809 */ 7810 if (!required_kernelcore || required_kernelcore >= totalpages) 7811 goto out; 7812 7813 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 7814 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 7815 7816 restart: 7817 /* Spread kernelcore memory as evenly as possible throughout nodes */ 7818 kernelcore_node = required_kernelcore / usable_nodes; 7819 for_each_node_state(nid, N_MEMORY) { 7820 unsigned long start_pfn, end_pfn; 7821 7822 /* 7823 * Recalculate kernelcore_node if the division per node 7824 * now exceeds what is necessary to satisfy the requested 7825 * amount of memory for the kernel 7826 */ 7827 if (required_kernelcore < kernelcore_node) 7828 kernelcore_node = required_kernelcore / usable_nodes; 7829 7830 /* 7831 * As the map is walked, we track how much memory is usable 7832 * by the kernel using kernelcore_remaining. When it is 7833 * 0, the rest of the node is usable by ZONE_MOVABLE 7834 */ 7835 kernelcore_remaining = kernelcore_node; 7836 7837 /* Go through each range of PFNs within this node */ 7838 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7839 unsigned long size_pages; 7840 7841 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 7842 if (start_pfn >= end_pfn) 7843 continue; 7844 7845 /* Account for what is only usable for kernelcore */ 7846 if (start_pfn < usable_startpfn) { 7847 unsigned long kernel_pages; 7848 kernel_pages = min(end_pfn, usable_startpfn) 7849 - start_pfn; 7850 7851 kernelcore_remaining -= min(kernel_pages, 7852 kernelcore_remaining); 7853 required_kernelcore -= min(kernel_pages, 7854 required_kernelcore); 7855 7856 /* Continue if range is now fully accounted */ 7857 if (end_pfn <= usable_startpfn) { 7858 7859 /* 7860 * Push zone_movable_pfn to the end so 7861 * that if we have to rebalance 7862 * kernelcore across nodes, we will 7863 * not double account here 7864 */ 7865 zone_movable_pfn[nid] = end_pfn; 7866 continue; 7867 } 7868 start_pfn = usable_startpfn; 7869 } 7870 7871 /* 7872 * The usable PFN range for ZONE_MOVABLE is from 7873 * start_pfn->end_pfn. Calculate size_pages as the 7874 * number of pages used as kernelcore 7875 */ 7876 size_pages = end_pfn - start_pfn; 7877 if (size_pages > kernelcore_remaining) 7878 size_pages = kernelcore_remaining; 7879 zone_movable_pfn[nid] = start_pfn + size_pages; 7880 7881 /* 7882 * Some kernelcore has been met, update counts and 7883 * break if the kernelcore for this node has been 7884 * satisfied 7885 */ 7886 required_kernelcore -= min(required_kernelcore, 7887 size_pages); 7888 kernelcore_remaining -= size_pages; 7889 if (!kernelcore_remaining) 7890 break; 7891 } 7892 } 7893 7894 /* 7895 * If there is still required_kernelcore, we do another pass with one 7896 * less node in the count. This will push zone_movable_pfn[nid] further 7897 * along on the nodes that still have memory until kernelcore is 7898 * satisfied 7899 */ 7900 usable_nodes--; 7901 if (usable_nodes && required_kernelcore > usable_nodes) 7902 goto restart; 7903 7904 out2: 7905 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 7906 for (nid = 0; nid < MAX_NUMNODES; nid++) 7907 zone_movable_pfn[nid] = 7908 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 7909 7910 out: 7911 /* restore the node_state */ 7912 node_states[N_MEMORY] = saved_node_state; 7913 } 7914 7915 /* Any regular or high memory on that node ? */ 7916 static void check_for_memory(pg_data_t *pgdat, int nid) 7917 { 7918 enum zone_type zone_type; 7919 7920 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 7921 struct zone *zone = &pgdat->node_zones[zone_type]; 7922 if (populated_zone(zone)) { 7923 if (IS_ENABLED(CONFIG_HIGHMEM)) 7924 node_set_state(nid, N_HIGH_MEMORY); 7925 if (zone_type <= ZONE_NORMAL) 7926 node_set_state(nid, N_NORMAL_MEMORY); 7927 break; 7928 } 7929 } 7930 } 7931 7932 /* 7933 * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For 7934 * such cases we allow max_zone_pfn sorted in the descending order 7935 */ 7936 bool __weak arch_has_descending_max_zone_pfns(void) 7937 { 7938 return false; 7939 } 7940 7941 /** 7942 * free_area_init - Initialise all pg_data_t and zone data 7943 * @max_zone_pfn: an array of max PFNs for each zone 7944 * 7945 * This will call free_area_init_node() for each active node in the system. 7946 * Using the page ranges provided by memblock_set_node(), the size of each 7947 * zone in each node and their holes is calculated. If the maximum PFN 7948 * between two adjacent zones match, it is assumed that the zone is empty. 7949 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 7950 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 7951 * starts where the previous one ended. For example, ZONE_DMA32 starts 7952 * at arch_max_dma_pfn. 7953 */ 7954 void __init free_area_init(unsigned long *max_zone_pfn) 7955 { 7956 unsigned long start_pfn, end_pfn; 7957 int i, nid, zone; 7958 bool descending; 7959 7960 /* Record where the zone boundaries are */ 7961 memset(arch_zone_lowest_possible_pfn, 0, 7962 sizeof(arch_zone_lowest_possible_pfn)); 7963 memset(arch_zone_highest_possible_pfn, 0, 7964 sizeof(arch_zone_highest_possible_pfn)); 7965 7966 start_pfn = find_min_pfn_with_active_regions(); 7967 descending = arch_has_descending_max_zone_pfns(); 7968 7969 for (i = 0; i < MAX_NR_ZONES; i++) { 7970 if (descending) 7971 zone = MAX_NR_ZONES - i - 1; 7972 else 7973 zone = i; 7974 7975 if (zone == ZONE_MOVABLE) 7976 continue; 7977 7978 end_pfn = max(max_zone_pfn[zone], start_pfn); 7979 arch_zone_lowest_possible_pfn[zone] = start_pfn; 7980 arch_zone_highest_possible_pfn[zone] = end_pfn; 7981 7982 start_pfn = end_pfn; 7983 } 7984 7985 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 7986 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 7987 find_zone_movable_pfns_for_nodes(); 7988 7989 /* Print out the zone ranges */ 7990 pr_info("Zone ranges:\n"); 7991 for (i = 0; i < MAX_NR_ZONES; i++) { 7992 if (i == ZONE_MOVABLE) 7993 continue; 7994 pr_info(" %-8s ", zone_names[i]); 7995 if (arch_zone_lowest_possible_pfn[i] == 7996 arch_zone_highest_possible_pfn[i]) 7997 pr_cont("empty\n"); 7998 else 7999 pr_cont("[mem %#018Lx-%#018Lx]\n", 8000 (u64)arch_zone_lowest_possible_pfn[i] 8001 << PAGE_SHIFT, 8002 ((u64)arch_zone_highest_possible_pfn[i] 8003 << PAGE_SHIFT) - 1); 8004 } 8005 8006 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 8007 pr_info("Movable zone start for each node\n"); 8008 for (i = 0; i < MAX_NUMNODES; i++) { 8009 if (zone_movable_pfn[i]) 8010 pr_info(" Node %d: %#018Lx\n", i, 8011 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 8012 } 8013 8014 /* 8015 * Print out the early node map, and initialize the 8016 * subsection-map relative to active online memory ranges to 8017 * enable future "sub-section" extensions of the memory map. 8018 */ 8019 pr_info("Early memory node ranges\n"); 8020 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 8021 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 8022 (u64)start_pfn << PAGE_SHIFT, 8023 ((u64)end_pfn << PAGE_SHIFT) - 1); 8024 subsection_map_init(start_pfn, end_pfn - start_pfn); 8025 } 8026 8027 /* Initialise every node */ 8028 mminit_verify_pageflags_layout(); 8029 setup_nr_node_ids(); 8030 for_each_online_node(nid) { 8031 pg_data_t *pgdat = NODE_DATA(nid); 8032 free_area_init_node(nid); 8033 8034 /* Any memory on that node */ 8035 if (pgdat->node_present_pages) 8036 node_set_state(nid, N_MEMORY); 8037 check_for_memory(pgdat, nid); 8038 } 8039 8040 memmap_init(); 8041 } 8042 8043 static int __init cmdline_parse_core(char *p, unsigned long *core, 8044 unsigned long *percent) 8045 { 8046 unsigned long long coremem; 8047 char *endptr; 8048 8049 if (!p) 8050 return -EINVAL; 8051 8052 /* Value may be a percentage of total memory, otherwise bytes */ 8053 coremem = simple_strtoull(p, &endptr, 0); 8054 if (*endptr == '%') { 8055 /* Paranoid check for percent values greater than 100 */ 8056 WARN_ON(coremem > 100); 8057 8058 *percent = coremem; 8059 } else { 8060 coremem = memparse(p, &p); 8061 /* Paranoid check that UL is enough for the coremem value */ 8062 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 8063 8064 *core = coremem >> PAGE_SHIFT; 8065 *percent = 0UL; 8066 } 8067 return 0; 8068 } 8069 8070 /* 8071 * kernelcore=size sets the amount of memory for use for allocations that 8072 * cannot be reclaimed or migrated. 8073 */ 8074 static int __init cmdline_parse_kernelcore(char *p) 8075 { 8076 /* parse kernelcore=mirror */ 8077 if (parse_option_str(p, "mirror")) { 8078 mirrored_kernelcore = true; 8079 return 0; 8080 } 8081 8082 return cmdline_parse_core(p, &required_kernelcore, 8083 &required_kernelcore_percent); 8084 } 8085 8086 /* 8087 * movablecore=size sets the amount of memory for use for allocations that 8088 * can be reclaimed or migrated. 8089 */ 8090 static int __init cmdline_parse_movablecore(char *p) 8091 { 8092 return cmdline_parse_core(p, &required_movablecore, 8093 &required_movablecore_percent); 8094 } 8095 8096 early_param("kernelcore", cmdline_parse_kernelcore); 8097 early_param("movablecore", cmdline_parse_movablecore); 8098 8099 void adjust_managed_page_count(struct page *page, long count) 8100 { 8101 atomic_long_add(count, &page_zone(page)->managed_pages); 8102 totalram_pages_add(count); 8103 #ifdef CONFIG_HIGHMEM 8104 if (PageHighMem(page)) 8105 totalhigh_pages_add(count); 8106 #endif 8107 } 8108 EXPORT_SYMBOL(adjust_managed_page_count); 8109 8110 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 8111 { 8112 void *pos; 8113 unsigned long pages = 0; 8114 8115 start = (void *)PAGE_ALIGN((unsigned long)start); 8116 end = (void *)((unsigned long)end & PAGE_MASK); 8117 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 8118 struct page *page = virt_to_page(pos); 8119 void *direct_map_addr; 8120 8121 /* 8122 * 'direct_map_addr' might be different from 'pos' 8123 * because some architectures' virt_to_page() 8124 * work with aliases. Getting the direct map 8125 * address ensures that we get a _writeable_ 8126 * alias for the memset(). 8127 */ 8128 direct_map_addr = page_address(page); 8129 /* 8130 * Perform a kasan-unchecked memset() since this memory 8131 * has not been initialized. 8132 */ 8133 direct_map_addr = kasan_reset_tag(direct_map_addr); 8134 if ((unsigned int)poison <= 0xFF) 8135 memset(direct_map_addr, poison, PAGE_SIZE); 8136 8137 free_reserved_page(page); 8138 } 8139 8140 if (pages && s) 8141 pr_info("Freeing %s memory: %ldK\n", 8142 s, pages << (PAGE_SHIFT - 10)); 8143 8144 return pages; 8145 } 8146 8147 void __init mem_init_print_info(void) 8148 { 8149 unsigned long physpages, codesize, datasize, rosize, bss_size; 8150 unsigned long init_code_size, init_data_size; 8151 8152 physpages = get_num_physpages(); 8153 codesize = _etext - _stext; 8154 datasize = _edata - _sdata; 8155 rosize = __end_rodata - __start_rodata; 8156 bss_size = __bss_stop - __bss_start; 8157 init_data_size = __init_end - __init_begin; 8158 init_code_size = _einittext - _sinittext; 8159 8160 /* 8161 * Detect special cases and adjust section sizes accordingly: 8162 * 1) .init.* may be embedded into .data sections 8163 * 2) .init.text.* may be out of [__init_begin, __init_end], 8164 * please refer to arch/tile/kernel/vmlinux.lds.S. 8165 * 3) .rodata.* may be embedded into .text or .data sections. 8166 */ 8167 #define adj_init_size(start, end, size, pos, adj) \ 8168 do { \ 8169 if (start <= pos && pos < end && size > adj) \ 8170 size -= adj; \ 8171 } while (0) 8172 8173 adj_init_size(__init_begin, __init_end, init_data_size, 8174 _sinittext, init_code_size); 8175 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 8176 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 8177 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 8178 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 8179 8180 #undef adj_init_size 8181 8182 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 8183 #ifdef CONFIG_HIGHMEM 8184 ", %luK highmem" 8185 #endif 8186 ")\n", 8187 nr_free_pages() << (PAGE_SHIFT - 10), 8188 physpages << (PAGE_SHIFT - 10), 8189 codesize >> 10, datasize >> 10, rosize >> 10, 8190 (init_data_size + init_code_size) >> 10, bss_size >> 10, 8191 (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10), 8192 totalcma_pages << (PAGE_SHIFT - 10) 8193 #ifdef CONFIG_HIGHMEM 8194 , totalhigh_pages() << (PAGE_SHIFT - 10) 8195 #endif 8196 ); 8197 } 8198 8199 /** 8200 * set_dma_reserve - set the specified number of pages reserved in the first zone 8201 * @new_dma_reserve: The number of pages to mark reserved 8202 * 8203 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 8204 * In the DMA zone, a significant percentage may be consumed by kernel image 8205 * and other unfreeable allocations which can skew the watermarks badly. This 8206 * function may optionally be used to account for unfreeable pages in the 8207 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 8208 * smaller per-cpu batchsize. 8209 */ 8210 void __init set_dma_reserve(unsigned long new_dma_reserve) 8211 { 8212 dma_reserve = new_dma_reserve; 8213 } 8214 8215 static int page_alloc_cpu_dead(unsigned int cpu) 8216 { 8217 struct zone *zone; 8218 8219 lru_add_drain_cpu(cpu); 8220 drain_pages(cpu); 8221 8222 /* 8223 * Spill the event counters of the dead processor 8224 * into the current processors event counters. 8225 * This artificially elevates the count of the current 8226 * processor. 8227 */ 8228 vm_events_fold_cpu(cpu); 8229 8230 /* 8231 * Zero the differential counters of the dead processor 8232 * so that the vm statistics are consistent. 8233 * 8234 * This is only okay since the processor is dead and cannot 8235 * race with what we are doing. 8236 */ 8237 cpu_vm_stats_fold(cpu); 8238 8239 for_each_populated_zone(zone) 8240 zone_pcp_update(zone, 0); 8241 8242 return 0; 8243 } 8244 8245 static int page_alloc_cpu_online(unsigned int cpu) 8246 { 8247 struct zone *zone; 8248 8249 for_each_populated_zone(zone) 8250 zone_pcp_update(zone, 1); 8251 return 0; 8252 } 8253 8254 #ifdef CONFIG_NUMA 8255 int hashdist = HASHDIST_DEFAULT; 8256 8257 static int __init set_hashdist(char *str) 8258 { 8259 if (!str) 8260 return 0; 8261 hashdist = simple_strtoul(str, &str, 0); 8262 return 1; 8263 } 8264 __setup("hashdist=", set_hashdist); 8265 #endif 8266 8267 void __init page_alloc_init(void) 8268 { 8269 int ret; 8270 8271 #ifdef CONFIG_NUMA 8272 if (num_node_state(N_MEMORY) == 1) 8273 hashdist = 0; 8274 #endif 8275 8276 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC, 8277 "mm/page_alloc:pcp", 8278 page_alloc_cpu_online, 8279 page_alloc_cpu_dead); 8280 WARN_ON(ret < 0); 8281 } 8282 8283 /* 8284 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 8285 * or min_free_kbytes changes. 8286 */ 8287 static void calculate_totalreserve_pages(void) 8288 { 8289 struct pglist_data *pgdat; 8290 unsigned long reserve_pages = 0; 8291 enum zone_type i, j; 8292 8293 for_each_online_pgdat(pgdat) { 8294 8295 pgdat->totalreserve_pages = 0; 8296 8297 for (i = 0; i < MAX_NR_ZONES; i++) { 8298 struct zone *zone = pgdat->node_zones + i; 8299 long max = 0; 8300 unsigned long managed_pages = zone_managed_pages(zone); 8301 8302 /* Find valid and maximum lowmem_reserve in the zone */ 8303 for (j = i; j < MAX_NR_ZONES; j++) { 8304 if (zone->lowmem_reserve[j] > max) 8305 max = zone->lowmem_reserve[j]; 8306 } 8307 8308 /* we treat the high watermark as reserved pages. */ 8309 max += high_wmark_pages(zone); 8310 8311 if (max > managed_pages) 8312 max = managed_pages; 8313 8314 pgdat->totalreserve_pages += max; 8315 8316 reserve_pages += max; 8317 } 8318 } 8319 totalreserve_pages = reserve_pages; 8320 } 8321 8322 /* 8323 * setup_per_zone_lowmem_reserve - called whenever 8324 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 8325 * has a correct pages reserved value, so an adequate number of 8326 * pages are left in the zone after a successful __alloc_pages(). 8327 */ 8328 static void setup_per_zone_lowmem_reserve(void) 8329 { 8330 struct pglist_data *pgdat; 8331 enum zone_type i, j; 8332 8333 for_each_online_pgdat(pgdat) { 8334 for (i = 0; i < MAX_NR_ZONES - 1; i++) { 8335 struct zone *zone = &pgdat->node_zones[i]; 8336 int ratio = sysctl_lowmem_reserve_ratio[i]; 8337 bool clear = !ratio || !zone_managed_pages(zone); 8338 unsigned long managed_pages = 0; 8339 8340 for (j = i + 1; j < MAX_NR_ZONES; j++) { 8341 struct zone *upper_zone = &pgdat->node_zones[j]; 8342 8343 managed_pages += zone_managed_pages(upper_zone); 8344 8345 if (clear) 8346 zone->lowmem_reserve[j] = 0; 8347 else 8348 zone->lowmem_reserve[j] = managed_pages / ratio; 8349 } 8350 } 8351 } 8352 8353 /* update totalreserve_pages */ 8354 calculate_totalreserve_pages(); 8355 } 8356 8357 static void __setup_per_zone_wmarks(void) 8358 { 8359 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 8360 unsigned long lowmem_pages = 0; 8361 struct zone *zone; 8362 unsigned long flags; 8363 8364 /* Calculate total number of !ZONE_HIGHMEM pages */ 8365 for_each_zone(zone) { 8366 if (!is_highmem(zone)) 8367 lowmem_pages += zone_managed_pages(zone); 8368 } 8369 8370 for_each_zone(zone) { 8371 u64 tmp; 8372 8373 spin_lock_irqsave(&zone->lock, flags); 8374 tmp = (u64)pages_min * zone_managed_pages(zone); 8375 do_div(tmp, lowmem_pages); 8376 if (is_highmem(zone)) { 8377 /* 8378 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 8379 * need highmem pages, so cap pages_min to a small 8380 * value here. 8381 * 8382 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 8383 * deltas control async page reclaim, and so should 8384 * not be capped for highmem. 8385 */ 8386 unsigned long min_pages; 8387 8388 min_pages = zone_managed_pages(zone) / 1024; 8389 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 8390 zone->_watermark[WMARK_MIN] = min_pages; 8391 } else { 8392 /* 8393 * If it's a lowmem zone, reserve a number of pages 8394 * proportionate to the zone's size. 8395 */ 8396 zone->_watermark[WMARK_MIN] = tmp; 8397 } 8398 8399 /* 8400 * Set the kswapd watermarks distance according to the 8401 * scale factor in proportion to available memory, but 8402 * ensure a minimum size on small systems. 8403 */ 8404 tmp = max_t(u64, tmp >> 2, 8405 mult_frac(zone_managed_pages(zone), 8406 watermark_scale_factor, 10000)); 8407 8408 zone->watermark_boost = 0; 8409 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 8410 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2; 8411 8412 spin_unlock_irqrestore(&zone->lock, flags); 8413 } 8414 8415 /* update totalreserve_pages */ 8416 calculate_totalreserve_pages(); 8417 } 8418 8419 /** 8420 * setup_per_zone_wmarks - called when min_free_kbytes changes 8421 * or when memory is hot-{added|removed} 8422 * 8423 * Ensures that the watermark[min,low,high] values for each zone are set 8424 * correctly with respect to min_free_kbytes. 8425 */ 8426 void setup_per_zone_wmarks(void) 8427 { 8428 struct zone *zone; 8429 static DEFINE_SPINLOCK(lock); 8430 8431 spin_lock(&lock); 8432 __setup_per_zone_wmarks(); 8433 spin_unlock(&lock); 8434 8435 /* 8436 * The watermark size have changed so update the pcpu batch 8437 * and high limits or the limits may be inappropriate. 8438 */ 8439 for_each_zone(zone) 8440 zone_pcp_update(zone, 0); 8441 } 8442 8443 /* 8444 * Initialise min_free_kbytes. 8445 * 8446 * For small machines we want it small (128k min). For large machines 8447 * we want it large (256MB max). But it is not linear, because network 8448 * bandwidth does not increase linearly with machine size. We use 8449 * 8450 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 8451 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 8452 * 8453 * which yields 8454 * 8455 * 16MB: 512k 8456 * 32MB: 724k 8457 * 64MB: 1024k 8458 * 128MB: 1448k 8459 * 256MB: 2048k 8460 * 512MB: 2896k 8461 * 1024MB: 4096k 8462 * 2048MB: 5792k 8463 * 4096MB: 8192k 8464 * 8192MB: 11584k 8465 * 16384MB: 16384k 8466 */ 8467 int __meminit init_per_zone_wmark_min(void) 8468 { 8469 unsigned long lowmem_kbytes; 8470 int new_min_free_kbytes; 8471 8472 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 8473 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 8474 8475 if (new_min_free_kbytes > user_min_free_kbytes) { 8476 min_free_kbytes = new_min_free_kbytes; 8477 if (min_free_kbytes < 128) 8478 min_free_kbytes = 128; 8479 if (min_free_kbytes > 262144) 8480 min_free_kbytes = 262144; 8481 } else { 8482 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 8483 new_min_free_kbytes, user_min_free_kbytes); 8484 } 8485 setup_per_zone_wmarks(); 8486 refresh_zone_stat_thresholds(); 8487 setup_per_zone_lowmem_reserve(); 8488 8489 #ifdef CONFIG_NUMA 8490 setup_min_unmapped_ratio(); 8491 setup_min_slab_ratio(); 8492 #endif 8493 8494 khugepaged_min_free_kbytes_update(); 8495 8496 return 0; 8497 } 8498 postcore_initcall(init_per_zone_wmark_min) 8499 8500 /* 8501 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 8502 * that we can call two helper functions whenever min_free_kbytes 8503 * changes. 8504 */ 8505 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 8506 void *buffer, size_t *length, loff_t *ppos) 8507 { 8508 int rc; 8509 8510 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8511 if (rc) 8512 return rc; 8513 8514 if (write) { 8515 user_min_free_kbytes = min_free_kbytes; 8516 setup_per_zone_wmarks(); 8517 } 8518 return 0; 8519 } 8520 8521 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 8522 void *buffer, size_t *length, loff_t *ppos) 8523 { 8524 int rc; 8525 8526 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8527 if (rc) 8528 return rc; 8529 8530 if (write) 8531 setup_per_zone_wmarks(); 8532 8533 return 0; 8534 } 8535 8536 #ifdef CONFIG_NUMA 8537 static void setup_min_unmapped_ratio(void) 8538 { 8539 pg_data_t *pgdat; 8540 struct zone *zone; 8541 8542 for_each_online_pgdat(pgdat) 8543 pgdat->min_unmapped_pages = 0; 8544 8545 for_each_zone(zone) 8546 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * 8547 sysctl_min_unmapped_ratio) / 100; 8548 } 8549 8550 8551 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 8552 void *buffer, size_t *length, loff_t *ppos) 8553 { 8554 int rc; 8555 8556 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8557 if (rc) 8558 return rc; 8559 8560 setup_min_unmapped_ratio(); 8561 8562 return 0; 8563 } 8564 8565 static void setup_min_slab_ratio(void) 8566 { 8567 pg_data_t *pgdat; 8568 struct zone *zone; 8569 8570 for_each_online_pgdat(pgdat) 8571 pgdat->min_slab_pages = 0; 8572 8573 for_each_zone(zone) 8574 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * 8575 sysctl_min_slab_ratio) / 100; 8576 } 8577 8578 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 8579 void *buffer, size_t *length, loff_t *ppos) 8580 { 8581 int rc; 8582 8583 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8584 if (rc) 8585 return rc; 8586 8587 setup_min_slab_ratio(); 8588 8589 return 0; 8590 } 8591 #endif 8592 8593 /* 8594 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 8595 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 8596 * whenever sysctl_lowmem_reserve_ratio changes. 8597 * 8598 * The reserve ratio obviously has absolutely no relation with the 8599 * minimum watermarks. The lowmem reserve ratio can only make sense 8600 * if in function of the boot time zone sizes. 8601 */ 8602 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 8603 void *buffer, size_t *length, loff_t *ppos) 8604 { 8605 int i; 8606 8607 proc_dointvec_minmax(table, write, buffer, length, ppos); 8608 8609 for (i = 0; i < MAX_NR_ZONES; i++) { 8610 if (sysctl_lowmem_reserve_ratio[i] < 1) 8611 sysctl_lowmem_reserve_ratio[i] = 0; 8612 } 8613 8614 setup_per_zone_lowmem_reserve(); 8615 return 0; 8616 } 8617 8618 /* 8619 * percpu_pagelist_high_fraction - changes the pcp->high for each zone on each 8620 * cpu. It is the fraction of total pages in each zone that a hot per cpu 8621 * pagelist can have before it gets flushed back to buddy allocator. 8622 */ 8623 int percpu_pagelist_high_fraction_sysctl_handler(struct ctl_table *table, 8624 int write, void *buffer, size_t *length, loff_t *ppos) 8625 { 8626 struct zone *zone; 8627 int old_percpu_pagelist_high_fraction; 8628 int ret; 8629 8630 mutex_lock(&pcp_batch_high_lock); 8631 old_percpu_pagelist_high_fraction = percpu_pagelist_high_fraction; 8632 8633 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 8634 if (!write || ret < 0) 8635 goto out; 8636 8637 /* Sanity checking to avoid pcp imbalance */ 8638 if (percpu_pagelist_high_fraction && 8639 percpu_pagelist_high_fraction < MIN_PERCPU_PAGELIST_HIGH_FRACTION) { 8640 percpu_pagelist_high_fraction = old_percpu_pagelist_high_fraction; 8641 ret = -EINVAL; 8642 goto out; 8643 } 8644 8645 /* No change? */ 8646 if (percpu_pagelist_high_fraction == old_percpu_pagelist_high_fraction) 8647 goto out; 8648 8649 for_each_populated_zone(zone) 8650 zone_set_pageset_high_and_batch(zone, 0); 8651 out: 8652 mutex_unlock(&pcp_batch_high_lock); 8653 return ret; 8654 } 8655 8656 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 8657 /* 8658 * Returns the number of pages that arch has reserved but 8659 * is not known to alloc_large_system_hash(). 8660 */ 8661 static unsigned long __init arch_reserved_kernel_pages(void) 8662 { 8663 return 0; 8664 } 8665 #endif 8666 8667 /* 8668 * Adaptive scale is meant to reduce sizes of hash tables on large memory 8669 * machines. As memory size is increased the scale is also increased but at 8670 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 8671 * quadruples the scale is increased by one, which means the size of hash table 8672 * only doubles, instead of quadrupling as well. 8673 * Because 32-bit systems cannot have large physical memory, where this scaling 8674 * makes sense, it is disabled on such platforms. 8675 */ 8676 #if __BITS_PER_LONG > 32 8677 #define ADAPT_SCALE_BASE (64ul << 30) 8678 #define ADAPT_SCALE_SHIFT 2 8679 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 8680 #endif 8681 8682 /* 8683 * allocate a large system hash table from bootmem 8684 * - it is assumed that the hash table must contain an exact power-of-2 8685 * quantity of entries 8686 * - limit is the number of hash buckets, not the total allocation size 8687 */ 8688 void *__init alloc_large_system_hash(const char *tablename, 8689 unsigned long bucketsize, 8690 unsigned long numentries, 8691 int scale, 8692 int flags, 8693 unsigned int *_hash_shift, 8694 unsigned int *_hash_mask, 8695 unsigned long low_limit, 8696 unsigned long high_limit) 8697 { 8698 unsigned long long max = high_limit; 8699 unsigned long log2qty, size; 8700 void *table = NULL; 8701 gfp_t gfp_flags; 8702 bool virt; 8703 bool huge; 8704 8705 /* allow the kernel cmdline to have a say */ 8706 if (!numentries) { 8707 /* round applicable memory size up to nearest megabyte */ 8708 numentries = nr_kernel_pages; 8709 numentries -= arch_reserved_kernel_pages(); 8710 8711 /* It isn't necessary when PAGE_SIZE >= 1MB */ 8712 if (PAGE_SHIFT < 20) 8713 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 8714 8715 #if __BITS_PER_LONG > 32 8716 if (!high_limit) { 8717 unsigned long adapt; 8718 8719 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 8720 adapt <<= ADAPT_SCALE_SHIFT) 8721 scale++; 8722 } 8723 #endif 8724 8725 /* limit to 1 bucket per 2^scale bytes of low memory */ 8726 if (scale > PAGE_SHIFT) 8727 numentries >>= (scale - PAGE_SHIFT); 8728 else 8729 numentries <<= (PAGE_SHIFT - scale); 8730 8731 /* Make sure we've got at least a 0-order allocation.. */ 8732 if (unlikely(flags & HASH_SMALL)) { 8733 /* Makes no sense without HASH_EARLY */ 8734 WARN_ON(!(flags & HASH_EARLY)); 8735 if (!(numentries >> *_hash_shift)) { 8736 numentries = 1UL << *_hash_shift; 8737 BUG_ON(!numentries); 8738 } 8739 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 8740 numentries = PAGE_SIZE / bucketsize; 8741 } 8742 numentries = roundup_pow_of_two(numentries); 8743 8744 /* limit allocation size to 1/16 total memory by default */ 8745 if (max == 0) { 8746 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 8747 do_div(max, bucketsize); 8748 } 8749 max = min(max, 0x80000000ULL); 8750 8751 if (numentries < low_limit) 8752 numentries = low_limit; 8753 if (numentries > max) 8754 numentries = max; 8755 8756 log2qty = ilog2(numentries); 8757 8758 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 8759 do { 8760 virt = false; 8761 size = bucketsize << log2qty; 8762 if (flags & HASH_EARLY) { 8763 if (flags & HASH_ZERO) 8764 table = memblock_alloc(size, SMP_CACHE_BYTES); 8765 else 8766 table = memblock_alloc_raw(size, 8767 SMP_CACHE_BYTES); 8768 } else if (get_order(size) >= MAX_ORDER || hashdist) { 8769 table = __vmalloc(size, gfp_flags); 8770 virt = true; 8771 huge = is_vm_area_hugepages(table); 8772 } else { 8773 /* 8774 * If bucketsize is not a power-of-two, we may free 8775 * some pages at the end of hash table which 8776 * alloc_pages_exact() automatically does 8777 */ 8778 table = alloc_pages_exact(size, gfp_flags); 8779 kmemleak_alloc(table, size, 1, gfp_flags); 8780 } 8781 } while (!table && size > PAGE_SIZE && --log2qty); 8782 8783 if (!table) 8784 panic("Failed to allocate %s hash table\n", tablename); 8785 8786 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 8787 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, 8788 virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear"); 8789 8790 if (_hash_shift) 8791 *_hash_shift = log2qty; 8792 if (_hash_mask) 8793 *_hash_mask = (1 << log2qty) - 1; 8794 8795 return table; 8796 } 8797 8798 /* 8799 * This function checks whether pageblock includes unmovable pages or not. 8800 * 8801 * PageLRU check without isolation or lru_lock could race so that 8802 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable 8803 * check without lock_page also may miss some movable non-lru pages at 8804 * race condition. So you can't expect this function should be exact. 8805 * 8806 * Returns a page without holding a reference. If the caller wants to 8807 * dereference that page (e.g., dumping), it has to make sure that it 8808 * cannot get removed (e.g., via memory unplug) concurrently. 8809 * 8810 */ 8811 struct page *has_unmovable_pages(struct zone *zone, struct page *page, 8812 int migratetype, int flags) 8813 { 8814 unsigned long iter = 0; 8815 unsigned long pfn = page_to_pfn(page); 8816 unsigned long offset = pfn % pageblock_nr_pages; 8817 8818 if (is_migrate_cma_page(page)) { 8819 /* 8820 * CMA allocations (alloc_contig_range) really need to mark 8821 * isolate CMA pageblocks even when they are not movable in fact 8822 * so consider them movable here. 8823 */ 8824 if (is_migrate_cma(migratetype)) 8825 return NULL; 8826 8827 return page; 8828 } 8829 8830 for (; iter < pageblock_nr_pages - offset; iter++) { 8831 if (!pfn_valid_within(pfn + iter)) 8832 continue; 8833 8834 page = pfn_to_page(pfn + iter); 8835 8836 /* 8837 * Both, bootmem allocations and memory holes are marked 8838 * PG_reserved and are unmovable. We can even have unmovable 8839 * allocations inside ZONE_MOVABLE, for example when 8840 * specifying "movablecore". 8841 */ 8842 if (PageReserved(page)) 8843 return page; 8844 8845 /* 8846 * If the zone is movable and we have ruled out all reserved 8847 * pages then it should be reasonably safe to assume the rest 8848 * is movable. 8849 */ 8850 if (zone_idx(zone) == ZONE_MOVABLE) 8851 continue; 8852 8853 /* 8854 * Hugepages are not in LRU lists, but they're movable. 8855 * THPs are on the LRU, but need to be counted as #small pages. 8856 * We need not scan over tail pages because we don't 8857 * handle each tail page individually in migration. 8858 */ 8859 if (PageHuge(page) || PageTransCompound(page)) { 8860 struct page *head = compound_head(page); 8861 unsigned int skip_pages; 8862 8863 if (PageHuge(page)) { 8864 if (!hugepage_migration_supported(page_hstate(head))) 8865 return page; 8866 } else if (!PageLRU(head) && !__PageMovable(head)) { 8867 return page; 8868 } 8869 8870 skip_pages = compound_nr(head) - (page - head); 8871 iter += skip_pages - 1; 8872 continue; 8873 } 8874 8875 /* 8876 * We can't use page_count without pin a page 8877 * because another CPU can free compound page. 8878 * This check already skips compound tails of THP 8879 * because their page->_refcount is zero at all time. 8880 */ 8881 if (!page_ref_count(page)) { 8882 if (PageBuddy(page)) 8883 iter += (1 << buddy_order(page)) - 1; 8884 continue; 8885 } 8886 8887 /* 8888 * The HWPoisoned page may be not in buddy system, and 8889 * page_count() is not 0. 8890 */ 8891 if ((flags & MEMORY_OFFLINE) && PageHWPoison(page)) 8892 continue; 8893 8894 /* 8895 * We treat all PageOffline() pages as movable when offlining 8896 * to give drivers a chance to decrement their reference count 8897 * in MEM_GOING_OFFLINE in order to indicate that these pages 8898 * can be offlined as there are no direct references anymore. 8899 * For actually unmovable PageOffline() where the driver does 8900 * not support this, we will fail later when trying to actually 8901 * move these pages that still have a reference count > 0. 8902 * (false negatives in this function only) 8903 */ 8904 if ((flags & MEMORY_OFFLINE) && PageOffline(page)) 8905 continue; 8906 8907 if (__PageMovable(page) || PageLRU(page)) 8908 continue; 8909 8910 /* 8911 * If there are RECLAIMABLE pages, we need to check 8912 * it. But now, memory offline itself doesn't call 8913 * shrink_node_slabs() and it still to be fixed. 8914 */ 8915 return page; 8916 } 8917 return NULL; 8918 } 8919 8920 #ifdef CONFIG_CONTIG_ALLOC 8921 static unsigned long pfn_max_align_down(unsigned long pfn) 8922 { 8923 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, 8924 pageblock_nr_pages) - 1); 8925 } 8926 8927 static unsigned long pfn_max_align_up(unsigned long pfn) 8928 { 8929 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, 8930 pageblock_nr_pages)); 8931 } 8932 8933 #if defined(CONFIG_DYNAMIC_DEBUG) || \ 8934 (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE)) 8935 /* Usage: See admin-guide/dynamic-debug-howto.rst */ 8936 static void alloc_contig_dump_pages(struct list_head *page_list) 8937 { 8938 DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, "migrate failure"); 8939 8940 if (DYNAMIC_DEBUG_BRANCH(descriptor)) { 8941 struct page *page; 8942 8943 dump_stack(); 8944 list_for_each_entry(page, page_list, lru) 8945 dump_page(page, "migration failure"); 8946 } 8947 } 8948 #else 8949 static inline void alloc_contig_dump_pages(struct list_head *page_list) 8950 { 8951 } 8952 #endif 8953 8954 /* [start, end) must belong to a single zone. */ 8955 static int __alloc_contig_migrate_range(struct compact_control *cc, 8956 unsigned long start, unsigned long end) 8957 { 8958 /* This function is based on compact_zone() from compaction.c. */ 8959 unsigned int nr_reclaimed; 8960 unsigned long pfn = start; 8961 unsigned int tries = 0; 8962 int ret = 0; 8963 struct migration_target_control mtc = { 8964 .nid = zone_to_nid(cc->zone), 8965 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, 8966 }; 8967 8968 lru_cache_disable(); 8969 8970 while (pfn < end || !list_empty(&cc->migratepages)) { 8971 if (fatal_signal_pending(current)) { 8972 ret = -EINTR; 8973 break; 8974 } 8975 8976 if (list_empty(&cc->migratepages)) { 8977 cc->nr_migratepages = 0; 8978 ret = isolate_migratepages_range(cc, pfn, end); 8979 if (ret && ret != -EAGAIN) 8980 break; 8981 pfn = cc->migrate_pfn; 8982 tries = 0; 8983 } else if (++tries == 5) { 8984 ret = -EBUSY; 8985 break; 8986 } 8987 8988 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 8989 &cc->migratepages); 8990 cc->nr_migratepages -= nr_reclaimed; 8991 8992 ret = migrate_pages(&cc->migratepages, alloc_migration_target, 8993 NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE, NULL); 8994 8995 /* 8996 * On -ENOMEM, migrate_pages() bails out right away. It is pointless 8997 * to retry again over this error, so do the same here. 8998 */ 8999 if (ret == -ENOMEM) 9000 break; 9001 } 9002 9003 lru_cache_enable(); 9004 if (ret < 0) { 9005 if (ret == -EBUSY) 9006 alloc_contig_dump_pages(&cc->migratepages); 9007 putback_movable_pages(&cc->migratepages); 9008 return ret; 9009 } 9010 return 0; 9011 } 9012 9013 /** 9014 * alloc_contig_range() -- tries to allocate given range of pages 9015 * @start: start PFN to allocate 9016 * @end: one-past-the-last PFN to allocate 9017 * @migratetype: migratetype of the underlying pageblocks (either 9018 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 9019 * in range must have the same migratetype and it must 9020 * be either of the two. 9021 * @gfp_mask: GFP mask to use during compaction 9022 * 9023 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 9024 * aligned. The PFN range must belong to a single zone. 9025 * 9026 * The first thing this routine does is attempt to MIGRATE_ISOLATE all 9027 * pageblocks in the range. Once isolated, the pageblocks should not 9028 * be modified by others. 9029 * 9030 * Return: zero on success or negative error code. On success all 9031 * pages which PFN is in [start, end) are allocated for the caller and 9032 * need to be freed with free_contig_range(). 9033 */ 9034 int alloc_contig_range(unsigned long start, unsigned long end, 9035 unsigned migratetype, gfp_t gfp_mask) 9036 { 9037 unsigned long outer_start, outer_end; 9038 unsigned int order; 9039 int ret = 0; 9040 9041 struct compact_control cc = { 9042 .nr_migratepages = 0, 9043 .order = -1, 9044 .zone = page_zone(pfn_to_page(start)), 9045 .mode = MIGRATE_SYNC, 9046 .ignore_skip_hint = true, 9047 .no_set_skip_hint = true, 9048 .gfp_mask = current_gfp_context(gfp_mask), 9049 .alloc_contig = true, 9050 }; 9051 INIT_LIST_HEAD(&cc.migratepages); 9052 9053 /* 9054 * What we do here is we mark all pageblocks in range as 9055 * MIGRATE_ISOLATE. Because pageblock and max order pages may 9056 * have different sizes, and due to the way page allocator 9057 * work, we align the range to biggest of the two pages so 9058 * that page allocator won't try to merge buddies from 9059 * different pageblocks and change MIGRATE_ISOLATE to some 9060 * other migration type. 9061 * 9062 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 9063 * migrate the pages from an unaligned range (ie. pages that 9064 * we are interested in). This will put all the pages in 9065 * range back to page allocator as MIGRATE_ISOLATE. 9066 * 9067 * When this is done, we take the pages in range from page 9068 * allocator removing them from the buddy system. This way 9069 * page allocator will never consider using them. 9070 * 9071 * This lets us mark the pageblocks back as 9072 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 9073 * aligned range but not in the unaligned, original range are 9074 * put back to page allocator so that buddy can use them. 9075 */ 9076 9077 ret = start_isolate_page_range(pfn_max_align_down(start), 9078 pfn_max_align_up(end), migratetype, 0); 9079 if (ret) 9080 return ret; 9081 9082 drain_all_pages(cc.zone); 9083 9084 /* 9085 * In case of -EBUSY, we'd like to know which page causes problem. 9086 * So, just fall through. test_pages_isolated() has a tracepoint 9087 * which will report the busy page. 9088 * 9089 * It is possible that busy pages could become available before 9090 * the call to test_pages_isolated, and the range will actually be 9091 * allocated. So, if we fall through be sure to clear ret so that 9092 * -EBUSY is not accidentally used or returned to caller. 9093 */ 9094 ret = __alloc_contig_migrate_range(&cc, start, end); 9095 if (ret && ret != -EBUSY) 9096 goto done; 9097 ret = 0; 9098 9099 /* 9100 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 9101 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 9102 * more, all pages in [start, end) are free in page allocator. 9103 * What we are going to do is to allocate all pages from 9104 * [start, end) (that is remove them from page allocator). 9105 * 9106 * The only problem is that pages at the beginning and at the 9107 * end of interesting range may be not aligned with pages that 9108 * page allocator holds, ie. they can be part of higher order 9109 * pages. Because of this, we reserve the bigger range and 9110 * once this is done free the pages we are not interested in. 9111 * 9112 * We don't have to hold zone->lock here because the pages are 9113 * isolated thus they won't get removed from buddy. 9114 */ 9115 9116 order = 0; 9117 outer_start = start; 9118 while (!PageBuddy(pfn_to_page(outer_start))) { 9119 if (++order >= MAX_ORDER) { 9120 outer_start = start; 9121 break; 9122 } 9123 outer_start &= ~0UL << order; 9124 } 9125 9126 if (outer_start != start) { 9127 order = buddy_order(pfn_to_page(outer_start)); 9128 9129 /* 9130 * outer_start page could be small order buddy page and 9131 * it doesn't include start page. Adjust outer_start 9132 * in this case to report failed page properly 9133 * on tracepoint in test_pages_isolated() 9134 */ 9135 if (outer_start + (1UL << order) <= start) 9136 outer_start = start; 9137 } 9138 9139 /* Make sure the range is really isolated. */ 9140 if (test_pages_isolated(outer_start, end, 0)) { 9141 ret = -EBUSY; 9142 goto done; 9143 } 9144 9145 /* Grab isolated pages from freelists. */ 9146 outer_end = isolate_freepages_range(&cc, outer_start, end); 9147 if (!outer_end) { 9148 ret = -EBUSY; 9149 goto done; 9150 } 9151 9152 /* Free head and tail (if any) */ 9153 if (start != outer_start) 9154 free_contig_range(outer_start, start - outer_start); 9155 if (end != outer_end) 9156 free_contig_range(end, outer_end - end); 9157 9158 done: 9159 undo_isolate_page_range(pfn_max_align_down(start), 9160 pfn_max_align_up(end), migratetype); 9161 return ret; 9162 } 9163 EXPORT_SYMBOL(alloc_contig_range); 9164 9165 static int __alloc_contig_pages(unsigned long start_pfn, 9166 unsigned long nr_pages, gfp_t gfp_mask) 9167 { 9168 unsigned long end_pfn = start_pfn + nr_pages; 9169 9170 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE, 9171 gfp_mask); 9172 } 9173 9174 static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, 9175 unsigned long nr_pages) 9176 { 9177 unsigned long i, end_pfn = start_pfn + nr_pages; 9178 struct page *page; 9179 9180 for (i = start_pfn; i < end_pfn; i++) { 9181 page = pfn_to_online_page(i); 9182 if (!page) 9183 return false; 9184 9185 if (page_zone(page) != z) 9186 return false; 9187 9188 if (PageReserved(page)) 9189 return false; 9190 } 9191 return true; 9192 } 9193 9194 static bool zone_spans_last_pfn(const struct zone *zone, 9195 unsigned long start_pfn, unsigned long nr_pages) 9196 { 9197 unsigned long last_pfn = start_pfn + nr_pages - 1; 9198 9199 return zone_spans_pfn(zone, last_pfn); 9200 } 9201 9202 /** 9203 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages 9204 * @nr_pages: Number of contiguous pages to allocate 9205 * @gfp_mask: GFP mask to limit search and used during compaction 9206 * @nid: Target node 9207 * @nodemask: Mask for other possible nodes 9208 * 9209 * This routine is a wrapper around alloc_contig_range(). It scans over zones 9210 * on an applicable zonelist to find a contiguous pfn range which can then be 9211 * tried for allocation with alloc_contig_range(). This routine is intended 9212 * for allocation requests which can not be fulfilled with the buddy allocator. 9213 * 9214 * The allocated memory is always aligned to a page boundary. If nr_pages is a 9215 * power of two then the alignment is guaranteed to be to the given nr_pages 9216 * (e.g. 1GB request would be aligned to 1GB). 9217 * 9218 * Allocated pages can be freed with free_contig_range() or by manually calling 9219 * __free_page() on each allocated page. 9220 * 9221 * Return: pointer to contiguous pages on success, or NULL if not successful. 9222 */ 9223 struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, 9224 int nid, nodemask_t *nodemask) 9225 { 9226 unsigned long ret, pfn, flags; 9227 struct zonelist *zonelist; 9228 struct zone *zone; 9229 struct zoneref *z; 9230 9231 zonelist = node_zonelist(nid, gfp_mask); 9232 for_each_zone_zonelist_nodemask(zone, z, zonelist, 9233 gfp_zone(gfp_mask), nodemask) { 9234 spin_lock_irqsave(&zone->lock, flags); 9235 9236 pfn = ALIGN(zone->zone_start_pfn, nr_pages); 9237 while (zone_spans_last_pfn(zone, pfn, nr_pages)) { 9238 if (pfn_range_valid_contig(zone, pfn, nr_pages)) { 9239 /* 9240 * We release the zone lock here because 9241 * alloc_contig_range() will also lock the zone 9242 * at some point. If there's an allocation 9243 * spinning on this lock, it may win the race 9244 * and cause alloc_contig_range() to fail... 9245 */ 9246 spin_unlock_irqrestore(&zone->lock, flags); 9247 ret = __alloc_contig_pages(pfn, nr_pages, 9248 gfp_mask); 9249 if (!ret) 9250 return pfn_to_page(pfn); 9251 spin_lock_irqsave(&zone->lock, flags); 9252 } 9253 pfn += nr_pages; 9254 } 9255 spin_unlock_irqrestore(&zone->lock, flags); 9256 } 9257 return NULL; 9258 } 9259 #endif /* CONFIG_CONTIG_ALLOC */ 9260 9261 void free_contig_range(unsigned long pfn, unsigned long nr_pages) 9262 { 9263 unsigned long count = 0; 9264 9265 for (; nr_pages--; pfn++) { 9266 struct page *page = pfn_to_page(pfn); 9267 9268 count += page_count(page) != 1; 9269 __free_page(page); 9270 } 9271 WARN(count != 0, "%lu pages are still in use!\n", count); 9272 } 9273 EXPORT_SYMBOL(free_contig_range); 9274 9275 /* 9276 * The zone indicated has a new number of managed_pages; batch sizes and percpu 9277 * page high values need to be recalculated. 9278 */ 9279 void zone_pcp_update(struct zone *zone, int cpu_online) 9280 { 9281 mutex_lock(&pcp_batch_high_lock); 9282 zone_set_pageset_high_and_batch(zone, cpu_online); 9283 mutex_unlock(&pcp_batch_high_lock); 9284 } 9285 9286 /* 9287 * Effectively disable pcplists for the zone by setting the high limit to 0 9288 * and draining all cpus. A concurrent page freeing on another CPU that's about 9289 * to put the page on pcplist will either finish before the drain and the page 9290 * will be drained, or observe the new high limit and skip the pcplist. 9291 * 9292 * Must be paired with a call to zone_pcp_enable(). 9293 */ 9294 void zone_pcp_disable(struct zone *zone) 9295 { 9296 mutex_lock(&pcp_batch_high_lock); 9297 __zone_set_pageset_high_and_batch(zone, 0, 1); 9298 __drain_all_pages(zone, true); 9299 } 9300 9301 void zone_pcp_enable(struct zone *zone) 9302 { 9303 __zone_set_pageset_high_and_batch(zone, zone->pageset_high, zone->pageset_batch); 9304 mutex_unlock(&pcp_batch_high_lock); 9305 } 9306 9307 void zone_pcp_reset(struct zone *zone) 9308 { 9309 int cpu; 9310 struct per_cpu_zonestat *pzstats; 9311 9312 if (zone->per_cpu_pageset != &boot_pageset) { 9313 for_each_online_cpu(cpu) { 9314 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 9315 drain_zonestat(zone, pzstats); 9316 } 9317 free_percpu(zone->per_cpu_pageset); 9318 free_percpu(zone->per_cpu_zonestats); 9319 zone->per_cpu_pageset = &boot_pageset; 9320 zone->per_cpu_zonestats = &boot_zonestats; 9321 } 9322 } 9323 9324 #ifdef CONFIG_MEMORY_HOTREMOVE 9325 /* 9326 * All pages in the range must be in a single zone, must not contain holes, 9327 * must span full sections, and must be isolated before calling this function. 9328 */ 9329 void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 9330 { 9331 unsigned long pfn = start_pfn; 9332 struct page *page; 9333 struct zone *zone; 9334 unsigned int order; 9335 unsigned long flags; 9336 9337 offline_mem_sections(pfn, end_pfn); 9338 zone = page_zone(pfn_to_page(pfn)); 9339 spin_lock_irqsave(&zone->lock, flags); 9340 while (pfn < end_pfn) { 9341 page = pfn_to_page(pfn); 9342 /* 9343 * The HWPoisoned page may be not in buddy system, and 9344 * page_count() is not 0. 9345 */ 9346 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 9347 pfn++; 9348 continue; 9349 } 9350 /* 9351 * At this point all remaining PageOffline() pages have a 9352 * reference count of 0 and can simply be skipped. 9353 */ 9354 if (PageOffline(page)) { 9355 BUG_ON(page_count(page)); 9356 BUG_ON(PageBuddy(page)); 9357 pfn++; 9358 continue; 9359 } 9360 9361 BUG_ON(page_count(page)); 9362 BUG_ON(!PageBuddy(page)); 9363 order = buddy_order(page); 9364 del_page_from_free_list(page, zone, order); 9365 pfn += (1 << order); 9366 } 9367 spin_unlock_irqrestore(&zone->lock, flags); 9368 } 9369 #endif 9370 9371 bool is_free_buddy_page(struct page *page) 9372 { 9373 struct zone *zone = page_zone(page); 9374 unsigned long pfn = page_to_pfn(page); 9375 unsigned long flags; 9376 unsigned int order; 9377 9378 spin_lock_irqsave(&zone->lock, flags); 9379 for (order = 0; order < MAX_ORDER; order++) { 9380 struct page *page_head = page - (pfn & ((1 << order) - 1)); 9381 9382 if (PageBuddy(page_head) && buddy_order(page_head) >= order) 9383 break; 9384 } 9385 spin_unlock_irqrestore(&zone->lock, flags); 9386 9387 return order < MAX_ORDER; 9388 } 9389 9390 #ifdef CONFIG_MEMORY_FAILURE 9391 /* 9392 * Break down a higher-order page in sub-pages, and keep our target out of 9393 * buddy allocator. 9394 */ 9395 static void break_down_buddy_pages(struct zone *zone, struct page *page, 9396 struct page *target, int low, int high, 9397 int migratetype) 9398 { 9399 unsigned long size = 1 << high; 9400 struct page *current_buddy, *next_page; 9401 9402 while (high > low) { 9403 high--; 9404 size >>= 1; 9405 9406 if (target >= &page[size]) { 9407 next_page = page + size; 9408 current_buddy = page; 9409 } else { 9410 next_page = page; 9411 current_buddy = page + size; 9412 } 9413 9414 if (set_page_guard(zone, current_buddy, high, migratetype)) 9415 continue; 9416 9417 if (current_buddy != target) { 9418 add_to_free_list(current_buddy, zone, high, migratetype); 9419 set_buddy_order(current_buddy, high); 9420 page = next_page; 9421 } 9422 } 9423 } 9424 9425 /* 9426 * Take a page that will be marked as poisoned off the buddy allocator. 9427 */ 9428 bool take_page_off_buddy(struct page *page) 9429 { 9430 struct zone *zone = page_zone(page); 9431 unsigned long pfn = page_to_pfn(page); 9432 unsigned long flags; 9433 unsigned int order; 9434 bool ret = false; 9435 9436 spin_lock_irqsave(&zone->lock, flags); 9437 for (order = 0; order < MAX_ORDER; order++) { 9438 struct page *page_head = page - (pfn & ((1 << order) - 1)); 9439 int page_order = buddy_order(page_head); 9440 9441 if (PageBuddy(page_head) && page_order >= order) { 9442 unsigned long pfn_head = page_to_pfn(page_head); 9443 int migratetype = get_pfnblock_migratetype(page_head, 9444 pfn_head); 9445 9446 del_page_from_free_list(page_head, zone, page_order); 9447 break_down_buddy_pages(zone, page_head, page, 0, 9448 page_order, migratetype); 9449 if (!is_migrate_isolate(migratetype)) 9450 __mod_zone_freepage_state(zone, -1, migratetype); 9451 ret = true; 9452 break; 9453 } 9454 if (page_count(page_head) > 0) 9455 break; 9456 } 9457 spin_unlock_irqrestore(&zone->lock, flags); 9458 return ret; 9459 } 9460 #endif 9461