1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/page_alloc.c 4 * 5 * Manages the free list, the system allocates free pages here. 6 * Note that kmalloc() lives in slab.c 7 * 8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 9 * Swap reorganised 29.12.95, Stephen Tweedie 10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 16 */ 17 18 #include <linux/stddef.h> 19 #include <linux/mm.h> 20 #include <linux/highmem.h> 21 #include <linux/swap.h> 22 #include <linux/interrupt.h> 23 #include <linux/pagemap.h> 24 #include <linux/jiffies.h> 25 #include <linux/memblock.h> 26 #include <linux/compiler.h> 27 #include <linux/kernel.h> 28 #include <linux/kasan.h> 29 #include <linux/module.h> 30 #include <linux/suspend.h> 31 #include <linux/pagevec.h> 32 #include <linux/blkdev.h> 33 #include <linux/slab.h> 34 #include <linux/ratelimit.h> 35 #include <linux/oom.h> 36 #include <linux/topology.h> 37 #include <linux/sysctl.h> 38 #include <linux/cpu.h> 39 #include <linux/cpuset.h> 40 #include <linux/memory_hotplug.h> 41 #include <linux/nodemask.h> 42 #include <linux/vmalloc.h> 43 #include <linux/vmstat.h> 44 #include <linux/mempolicy.h> 45 #include <linux/memremap.h> 46 #include <linux/stop_machine.h> 47 #include <linux/random.h> 48 #include <linux/sort.h> 49 #include <linux/pfn.h> 50 #include <linux/backing-dev.h> 51 #include <linux/fault-inject.h> 52 #include <linux/page-isolation.h> 53 #include <linux/debugobjects.h> 54 #include <linux/kmemleak.h> 55 #include <linux/compaction.h> 56 #include <trace/events/kmem.h> 57 #include <trace/events/oom.h> 58 #include <linux/prefetch.h> 59 #include <linux/mm_inline.h> 60 #include <linux/mmu_notifier.h> 61 #include <linux/migrate.h> 62 #include <linux/hugetlb.h> 63 #include <linux/sched/rt.h> 64 #include <linux/sched/mm.h> 65 #include <linux/page_owner.h> 66 #include <linux/kthread.h> 67 #include <linux/memcontrol.h> 68 #include <linux/ftrace.h> 69 #include <linux/lockdep.h> 70 #include <linux/nmi.h> 71 #include <linux/psi.h> 72 #include <linux/padata.h> 73 #include <linux/khugepaged.h> 74 #include <linux/buffer_head.h> 75 #include <asm/sections.h> 76 #include <asm/tlbflush.h> 77 #include <asm/div64.h> 78 #include "internal.h" 79 #include "shuffle.h" 80 #include "page_reporting.h" 81 82 /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */ 83 typedef int __bitwise fpi_t; 84 85 /* No special request */ 86 #define FPI_NONE ((__force fpi_t)0) 87 88 /* 89 * Skip free page reporting notification for the (possibly merged) page. 90 * This does not hinder free page reporting from grabbing the page, 91 * reporting it and marking it "reported" - it only skips notifying 92 * the free page reporting infrastructure about a newly freed page. For 93 * example, used when temporarily pulling a page from a freelist and 94 * putting it back unmodified. 95 */ 96 #define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0)) 97 98 /* 99 * Place the (possibly merged) page to the tail of the freelist. Will ignore 100 * page shuffling (relevant code - e.g., memory onlining - is expected to 101 * shuffle the whole zone). 102 * 103 * Note: No code should rely on this flag for correctness - it's purely 104 * to allow for optimizations when handing back either fresh pages 105 * (memory onlining) or untouched pages (page isolation, free page 106 * reporting). 107 */ 108 #define FPI_TO_TAIL ((__force fpi_t)BIT(1)) 109 110 /* 111 * Don't poison memory with KASAN (only for the tag-based modes). 112 * During boot, all non-reserved memblock memory is exposed to page_alloc. 113 * Poisoning all that memory lengthens boot time, especially on systems with 114 * large amount of RAM. This flag is used to skip that poisoning. 115 * This is only done for the tag-based KASAN modes, as those are able to 116 * detect memory corruptions with the memory tags assigned by default. 117 * All memory allocated normally after boot gets poisoned as usual. 118 */ 119 #define FPI_SKIP_KASAN_POISON ((__force fpi_t)BIT(2)) 120 121 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 122 static DEFINE_MUTEX(pcp_batch_high_lock); 123 #define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8) 124 125 struct pagesets { 126 local_lock_t lock; 127 }; 128 static DEFINE_PER_CPU(struct pagesets, pagesets) = { 129 .lock = INIT_LOCAL_LOCK(lock), 130 }; 131 132 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 133 DEFINE_PER_CPU(int, numa_node); 134 EXPORT_PER_CPU_SYMBOL(numa_node); 135 #endif 136 137 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); 138 139 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 140 /* 141 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 142 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 143 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 144 * defined in <linux/topology.h>. 145 */ 146 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 147 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 148 #endif 149 150 /* work_structs for global per-cpu drains */ 151 struct pcpu_drain { 152 struct zone *zone; 153 struct work_struct work; 154 }; 155 static DEFINE_MUTEX(pcpu_drain_mutex); 156 static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain); 157 158 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 159 volatile unsigned long latent_entropy __latent_entropy; 160 EXPORT_SYMBOL(latent_entropy); 161 #endif 162 163 /* 164 * Array of node states. 165 */ 166 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 167 [N_POSSIBLE] = NODE_MASK_ALL, 168 [N_ONLINE] = { { [0] = 1UL } }, 169 #ifndef CONFIG_NUMA 170 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 171 #ifdef CONFIG_HIGHMEM 172 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 173 #endif 174 [N_MEMORY] = { { [0] = 1UL } }, 175 [N_CPU] = { { [0] = 1UL } }, 176 #endif /* NUMA */ 177 }; 178 EXPORT_SYMBOL(node_states); 179 180 atomic_long_t _totalram_pages __read_mostly; 181 EXPORT_SYMBOL(_totalram_pages); 182 unsigned long totalreserve_pages __read_mostly; 183 unsigned long totalcma_pages __read_mostly; 184 185 int percpu_pagelist_high_fraction; 186 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 187 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); 188 EXPORT_SYMBOL(init_on_alloc); 189 190 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); 191 EXPORT_SYMBOL(init_on_free); 192 193 static bool _init_on_alloc_enabled_early __read_mostly 194 = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON); 195 static int __init early_init_on_alloc(char *buf) 196 { 197 198 return kstrtobool(buf, &_init_on_alloc_enabled_early); 199 } 200 early_param("init_on_alloc", early_init_on_alloc); 201 202 static bool _init_on_free_enabled_early __read_mostly 203 = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON); 204 static int __init early_init_on_free(char *buf) 205 { 206 return kstrtobool(buf, &_init_on_free_enabled_early); 207 } 208 early_param("init_on_free", early_init_on_free); 209 210 /* 211 * A cached value of the page's pageblock's migratetype, used when the page is 212 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 213 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 214 * Also the migratetype set in the page does not necessarily match the pcplist 215 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 216 * other index - this ensures that it will be put on the correct CMA freelist. 217 */ 218 static inline int get_pcppage_migratetype(struct page *page) 219 { 220 return page->index; 221 } 222 223 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 224 { 225 page->index = migratetype; 226 } 227 228 #ifdef CONFIG_PM_SLEEP 229 /* 230 * The following functions are used by the suspend/hibernate code to temporarily 231 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 232 * while devices are suspended. To avoid races with the suspend/hibernate code, 233 * they should always be called with system_transition_mutex held 234 * (gfp_allowed_mask also should only be modified with system_transition_mutex 235 * held, unless the suspend/hibernate code is guaranteed not to run in parallel 236 * with that modification). 237 */ 238 239 static gfp_t saved_gfp_mask; 240 241 void pm_restore_gfp_mask(void) 242 { 243 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 244 if (saved_gfp_mask) { 245 gfp_allowed_mask = saved_gfp_mask; 246 saved_gfp_mask = 0; 247 } 248 } 249 250 void pm_restrict_gfp_mask(void) 251 { 252 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 253 WARN_ON(saved_gfp_mask); 254 saved_gfp_mask = gfp_allowed_mask; 255 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 256 } 257 258 bool pm_suspended_storage(void) 259 { 260 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 261 return false; 262 return true; 263 } 264 #endif /* CONFIG_PM_SLEEP */ 265 266 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 267 unsigned int pageblock_order __read_mostly; 268 #endif 269 270 static void __free_pages_ok(struct page *page, unsigned int order, 271 fpi_t fpi_flags); 272 273 /* 274 * results with 256, 32 in the lowmem_reserve sysctl: 275 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 276 * 1G machine -> (16M dma, 784M normal, 224M high) 277 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 278 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 279 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 280 * 281 * TBD: should special case ZONE_DMA32 machines here - in those we normally 282 * don't need any ZONE_NORMAL reservation 283 */ 284 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { 285 #ifdef CONFIG_ZONE_DMA 286 [ZONE_DMA] = 256, 287 #endif 288 #ifdef CONFIG_ZONE_DMA32 289 [ZONE_DMA32] = 256, 290 #endif 291 [ZONE_NORMAL] = 32, 292 #ifdef CONFIG_HIGHMEM 293 [ZONE_HIGHMEM] = 0, 294 #endif 295 [ZONE_MOVABLE] = 0, 296 }; 297 298 static char * const zone_names[MAX_NR_ZONES] = { 299 #ifdef CONFIG_ZONE_DMA 300 "DMA", 301 #endif 302 #ifdef CONFIG_ZONE_DMA32 303 "DMA32", 304 #endif 305 "Normal", 306 #ifdef CONFIG_HIGHMEM 307 "HighMem", 308 #endif 309 "Movable", 310 #ifdef CONFIG_ZONE_DEVICE 311 "Device", 312 #endif 313 }; 314 315 const char * const migratetype_names[MIGRATE_TYPES] = { 316 "Unmovable", 317 "Movable", 318 "Reclaimable", 319 "HighAtomic", 320 #ifdef CONFIG_CMA 321 "CMA", 322 #endif 323 #ifdef CONFIG_MEMORY_ISOLATION 324 "Isolate", 325 #endif 326 }; 327 328 compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = { 329 [NULL_COMPOUND_DTOR] = NULL, 330 [COMPOUND_PAGE_DTOR] = free_compound_page, 331 #ifdef CONFIG_HUGETLB_PAGE 332 [HUGETLB_PAGE_DTOR] = free_huge_page, 333 #endif 334 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 335 [TRANSHUGE_PAGE_DTOR] = free_transhuge_page, 336 #endif 337 }; 338 339 int min_free_kbytes = 1024; 340 int user_min_free_kbytes = -1; 341 int watermark_boost_factor __read_mostly = 15000; 342 int watermark_scale_factor = 10; 343 344 static unsigned long nr_kernel_pages __initdata; 345 static unsigned long nr_all_pages __initdata; 346 static unsigned long dma_reserve __initdata; 347 348 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 349 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 350 static unsigned long required_kernelcore __initdata; 351 static unsigned long required_kernelcore_percent __initdata; 352 static unsigned long required_movablecore __initdata; 353 static unsigned long required_movablecore_percent __initdata; 354 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 355 static bool mirrored_kernelcore __meminitdata; 356 357 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 358 int movable_zone; 359 EXPORT_SYMBOL(movable_zone); 360 361 #if MAX_NUMNODES > 1 362 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; 363 unsigned int nr_online_nodes __read_mostly = 1; 364 EXPORT_SYMBOL(nr_node_ids); 365 EXPORT_SYMBOL(nr_online_nodes); 366 #endif 367 368 int page_group_by_mobility_disabled __read_mostly; 369 370 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 371 /* 372 * During boot we initialize deferred pages on-demand, as needed, but once 373 * page_alloc_init_late() has finished, the deferred pages are all initialized, 374 * and we can permanently disable that path. 375 */ 376 static DEFINE_STATIC_KEY_TRUE(deferred_pages); 377 378 /* 379 * Calling kasan_poison_pages() only after deferred memory initialization 380 * has completed. Poisoning pages during deferred memory init will greatly 381 * lengthen the process and cause problem in large memory systems as the 382 * deferred pages initialization is done with interrupt disabled. 383 * 384 * Assuming that there will be no reference to those newly initialized 385 * pages before they are ever allocated, this should have no effect on 386 * KASAN memory tracking as the poison will be properly inserted at page 387 * allocation time. The only corner case is when pages are allocated by 388 * on-demand allocation and then freed again before the deferred pages 389 * initialization is done, but this is not likely to happen. 390 */ 391 static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags) 392 { 393 return static_branch_unlikely(&deferred_pages) || 394 (!IS_ENABLED(CONFIG_KASAN_GENERIC) && 395 (fpi_flags & FPI_SKIP_KASAN_POISON)) || 396 PageSkipKASanPoison(page); 397 } 398 399 /* Returns true if the struct page for the pfn is uninitialised */ 400 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 401 { 402 int nid = early_pfn_to_nid(pfn); 403 404 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 405 return true; 406 407 return false; 408 } 409 410 /* 411 * Returns true when the remaining initialisation should be deferred until 412 * later in the boot cycle when it can be parallelised. 413 */ 414 static bool __meminit 415 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 416 { 417 static unsigned long prev_end_pfn, nr_initialised; 418 419 /* 420 * prev_end_pfn static that contains the end of previous zone 421 * No need to protect because called very early in boot before smp_init. 422 */ 423 if (prev_end_pfn != end_pfn) { 424 prev_end_pfn = end_pfn; 425 nr_initialised = 0; 426 } 427 428 /* Always populate low zones for address-constrained allocations */ 429 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 430 return false; 431 432 if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) 433 return true; 434 /* 435 * We start only with one section of pages, more pages are added as 436 * needed until the rest of deferred pages are initialized. 437 */ 438 nr_initialised++; 439 if ((nr_initialised > PAGES_PER_SECTION) && 440 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 441 NODE_DATA(nid)->first_deferred_pfn = pfn; 442 return true; 443 } 444 return false; 445 } 446 #else 447 static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags) 448 { 449 return (!IS_ENABLED(CONFIG_KASAN_GENERIC) && 450 (fpi_flags & FPI_SKIP_KASAN_POISON)) || 451 PageSkipKASanPoison(page); 452 } 453 454 static inline bool early_page_uninitialised(unsigned long pfn) 455 { 456 return false; 457 } 458 459 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 460 { 461 return false; 462 } 463 #endif 464 465 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 466 static inline unsigned long *get_pageblock_bitmap(const struct page *page, 467 unsigned long pfn) 468 { 469 #ifdef CONFIG_SPARSEMEM 470 return section_to_usemap(__pfn_to_section(pfn)); 471 #else 472 return page_zone(page)->pageblock_flags; 473 #endif /* CONFIG_SPARSEMEM */ 474 } 475 476 static inline int pfn_to_bitidx(const struct page *page, unsigned long pfn) 477 { 478 #ifdef CONFIG_SPARSEMEM 479 pfn &= (PAGES_PER_SECTION-1); 480 #else 481 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 482 #endif /* CONFIG_SPARSEMEM */ 483 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 484 } 485 486 static __always_inline 487 unsigned long __get_pfnblock_flags_mask(const struct page *page, 488 unsigned long pfn, 489 unsigned long mask) 490 { 491 unsigned long *bitmap; 492 unsigned long bitidx, word_bitidx; 493 unsigned long word; 494 495 bitmap = get_pageblock_bitmap(page, pfn); 496 bitidx = pfn_to_bitidx(page, pfn); 497 word_bitidx = bitidx / BITS_PER_LONG; 498 bitidx &= (BITS_PER_LONG-1); 499 500 word = bitmap[word_bitidx]; 501 return (word >> bitidx) & mask; 502 } 503 504 /** 505 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 506 * @page: The page within the block of interest 507 * @pfn: The target page frame number 508 * @mask: mask of bits that the caller is interested in 509 * 510 * Return: pageblock_bits flags 511 */ 512 unsigned long get_pfnblock_flags_mask(const struct page *page, 513 unsigned long pfn, unsigned long mask) 514 { 515 return __get_pfnblock_flags_mask(page, pfn, mask); 516 } 517 518 static __always_inline int get_pfnblock_migratetype(const struct page *page, 519 unsigned long pfn) 520 { 521 return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK); 522 } 523 524 /** 525 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 526 * @page: The page within the block of interest 527 * @flags: The flags to set 528 * @pfn: The target page frame number 529 * @mask: mask of bits that the caller is interested in 530 */ 531 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 532 unsigned long pfn, 533 unsigned long mask) 534 { 535 unsigned long *bitmap; 536 unsigned long bitidx, word_bitidx; 537 unsigned long old_word, word; 538 539 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 540 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits)); 541 542 bitmap = get_pageblock_bitmap(page, pfn); 543 bitidx = pfn_to_bitidx(page, pfn); 544 word_bitidx = bitidx / BITS_PER_LONG; 545 bitidx &= (BITS_PER_LONG-1); 546 547 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 548 549 mask <<= bitidx; 550 flags <<= bitidx; 551 552 word = READ_ONCE(bitmap[word_bitidx]); 553 for (;;) { 554 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 555 if (word == old_word) 556 break; 557 word = old_word; 558 } 559 } 560 561 void set_pageblock_migratetype(struct page *page, int migratetype) 562 { 563 if (unlikely(page_group_by_mobility_disabled && 564 migratetype < MIGRATE_PCPTYPES)) 565 migratetype = MIGRATE_UNMOVABLE; 566 567 set_pfnblock_flags_mask(page, (unsigned long)migratetype, 568 page_to_pfn(page), MIGRATETYPE_MASK); 569 } 570 571 #ifdef CONFIG_DEBUG_VM 572 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 573 { 574 int ret = 0; 575 unsigned seq; 576 unsigned long pfn = page_to_pfn(page); 577 unsigned long sp, start_pfn; 578 579 do { 580 seq = zone_span_seqbegin(zone); 581 start_pfn = zone->zone_start_pfn; 582 sp = zone->spanned_pages; 583 if (!zone_spans_pfn(zone, pfn)) 584 ret = 1; 585 } while (zone_span_seqretry(zone, seq)); 586 587 if (ret) 588 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 589 pfn, zone_to_nid(zone), zone->name, 590 start_pfn, start_pfn + sp); 591 592 return ret; 593 } 594 595 static int page_is_consistent(struct zone *zone, struct page *page) 596 { 597 if (zone != page_zone(page)) 598 return 0; 599 600 return 1; 601 } 602 /* 603 * Temporary debugging check for pages not lying within a given zone. 604 */ 605 static int __maybe_unused bad_range(struct zone *zone, struct page *page) 606 { 607 if (page_outside_zone_boundaries(zone, page)) 608 return 1; 609 if (!page_is_consistent(zone, page)) 610 return 1; 611 612 return 0; 613 } 614 #else 615 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page) 616 { 617 return 0; 618 } 619 #endif 620 621 static void bad_page(struct page *page, const char *reason) 622 { 623 static unsigned long resume; 624 static unsigned long nr_shown; 625 static unsigned long nr_unshown; 626 627 /* 628 * Allow a burst of 60 reports, then keep quiet for that minute; 629 * or allow a steady drip of one report per second. 630 */ 631 if (nr_shown == 60) { 632 if (time_before(jiffies, resume)) { 633 nr_unshown++; 634 goto out; 635 } 636 if (nr_unshown) { 637 pr_alert( 638 "BUG: Bad page state: %lu messages suppressed\n", 639 nr_unshown); 640 nr_unshown = 0; 641 } 642 nr_shown = 0; 643 } 644 if (nr_shown++ == 0) 645 resume = jiffies + 60 * HZ; 646 647 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 648 current->comm, page_to_pfn(page)); 649 dump_page(page, reason); 650 651 print_modules(); 652 dump_stack(); 653 out: 654 /* Leave bad fields for debug, except PageBuddy could make trouble */ 655 page_mapcount_reset(page); /* remove PageBuddy */ 656 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 657 } 658 659 static inline unsigned int order_to_pindex(int migratetype, int order) 660 { 661 int base = order; 662 663 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 664 if (order > PAGE_ALLOC_COSTLY_ORDER) { 665 VM_BUG_ON(order != pageblock_order); 666 base = PAGE_ALLOC_COSTLY_ORDER + 1; 667 } 668 #else 669 VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); 670 #endif 671 672 return (MIGRATE_PCPTYPES * base) + migratetype; 673 } 674 675 static inline int pindex_to_order(unsigned int pindex) 676 { 677 int order = pindex / MIGRATE_PCPTYPES; 678 679 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 680 if (order > PAGE_ALLOC_COSTLY_ORDER) { 681 order = pageblock_order; 682 VM_BUG_ON(order != pageblock_order); 683 } 684 #else 685 VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); 686 #endif 687 688 return order; 689 } 690 691 static inline bool pcp_allowed_order(unsigned int order) 692 { 693 if (order <= PAGE_ALLOC_COSTLY_ORDER) 694 return true; 695 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 696 if (order == pageblock_order) 697 return true; 698 #endif 699 return false; 700 } 701 702 static inline void free_the_page(struct page *page, unsigned int order) 703 { 704 if (pcp_allowed_order(order)) /* Via pcp? */ 705 free_unref_page(page, order); 706 else 707 __free_pages_ok(page, order, FPI_NONE); 708 } 709 710 /* 711 * Higher-order pages are called "compound pages". They are structured thusly: 712 * 713 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 714 * 715 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 716 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 717 * 718 * The first tail page's ->compound_dtor holds the offset in array of compound 719 * page destructors. See compound_page_dtors. 720 * 721 * The first tail page's ->compound_order holds the order of allocation. 722 * This usage means that zero-order pages may not be compound. 723 */ 724 725 void free_compound_page(struct page *page) 726 { 727 mem_cgroup_uncharge(page); 728 free_the_page(page, compound_order(page)); 729 } 730 731 void prep_compound_page(struct page *page, unsigned int order) 732 { 733 int i; 734 int nr_pages = 1 << order; 735 736 __SetPageHead(page); 737 for (i = 1; i < nr_pages; i++) { 738 struct page *p = page + i; 739 p->mapping = TAIL_MAPPING; 740 set_compound_head(p, page); 741 } 742 743 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 744 set_compound_order(page, order); 745 atomic_set(compound_mapcount_ptr(page), -1); 746 if (hpage_pincount_available(page)) 747 atomic_set(compound_pincount_ptr(page), 0); 748 } 749 750 #ifdef CONFIG_DEBUG_PAGEALLOC 751 unsigned int _debug_guardpage_minorder; 752 753 bool _debug_pagealloc_enabled_early __read_mostly 754 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 755 EXPORT_SYMBOL(_debug_pagealloc_enabled_early); 756 DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 757 EXPORT_SYMBOL(_debug_pagealloc_enabled); 758 759 DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 760 761 static int __init early_debug_pagealloc(char *buf) 762 { 763 return kstrtobool(buf, &_debug_pagealloc_enabled_early); 764 } 765 early_param("debug_pagealloc", early_debug_pagealloc); 766 767 static int __init debug_guardpage_minorder_setup(char *buf) 768 { 769 unsigned long res; 770 771 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 772 pr_err("Bad debug_guardpage_minorder value\n"); 773 return 0; 774 } 775 _debug_guardpage_minorder = res; 776 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 777 return 0; 778 } 779 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); 780 781 static inline bool set_page_guard(struct zone *zone, struct page *page, 782 unsigned int order, int migratetype) 783 { 784 if (!debug_guardpage_enabled()) 785 return false; 786 787 if (order >= debug_guardpage_minorder()) 788 return false; 789 790 __SetPageGuard(page); 791 INIT_LIST_HEAD(&page->lru); 792 set_page_private(page, order); 793 /* Guard pages are not available for any usage */ 794 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 795 796 return true; 797 } 798 799 static inline void clear_page_guard(struct zone *zone, struct page *page, 800 unsigned int order, int migratetype) 801 { 802 if (!debug_guardpage_enabled()) 803 return; 804 805 __ClearPageGuard(page); 806 807 set_page_private(page, 0); 808 if (!is_migrate_isolate(migratetype)) 809 __mod_zone_freepage_state(zone, (1 << order), migratetype); 810 } 811 #else 812 static inline bool set_page_guard(struct zone *zone, struct page *page, 813 unsigned int order, int migratetype) { return false; } 814 static inline void clear_page_guard(struct zone *zone, struct page *page, 815 unsigned int order, int migratetype) {} 816 #endif 817 818 /* 819 * Enable static keys related to various memory debugging and hardening options. 820 * Some override others, and depend on early params that are evaluated in the 821 * order of appearance. So we need to first gather the full picture of what was 822 * enabled, and then make decisions. 823 */ 824 void init_mem_debugging_and_hardening(void) 825 { 826 bool page_poisoning_requested = false; 827 828 #ifdef CONFIG_PAGE_POISONING 829 /* 830 * Page poisoning is debug page alloc for some arches. If 831 * either of those options are enabled, enable poisoning. 832 */ 833 if (page_poisoning_enabled() || 834 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && 835 debug_pagealloc_enabled())) { 836 static_branch_enable(&_page_poisoning_enabled); 837 page_poisoning_requested = true; 838 } 839 #endif 840 841 if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) && 842 page_poisoning_requested) { 843 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " 844 "will take precedence over init_on_alloc and init_on_free\n"); 845 _init_on_alloc_enabled_early = false; 846 _init_on_free_enabled_early = false; 847 } 848 849 if (_init_on_alloc_enabled_early) 850 static_branch_enable(&init_on_alloc); 851 else 852 static_branch_disable(&init_on_alloc); 853 854 if (_init_on_free_enabled_early) 855 static_branch_enable(&init_on_free); 856 else 857 static_branch_disable(&init_on_free); 858 859 #ifdef CONFIG_DEBUG_PAGEALLOC 860 if (!debug_pagealloc_enabled()) 861 return; 862 863 static_branch_enable(&_debug_pagealloc_enabled); 864 865 if (!debug_guardpage_minorder()) 866 return; 867 868 static_branch_enable(&_debug_guardpage_enabled); 869 #endif 870 } 871 872 static inline void set_buddy_order(struct page *page, unsigned int order) 873 { 874 set_page_private(page, order); 875 __SetPageBuddy(page); 876 } 877 878 /* 879 * This function checks whether a page is free && is the buddy 880 * we can coalesce a page and its buddy if 881 * (a) the buddy is not in a hole (check before calling!) && 882 * (b) the buddy is in the buddy system && 883 * (c) a page and its buddy have the same order && 884 * (d) a page and its buddy are in the same zone. 885 * 886 * For recording whether a page is in the buddy system, we set PageBuddy. 887 * Setting, clearing, and testing PageBuddy is serialized by zone->lock. 888 * 889 * For recording page's order, we use page_private(page). 890 */ 891 static inline bool page_is_buddy(struct page *page, struct page *buddy, 892 unsigned int order) 893 { 894 if (!page_is_guard(buddy) && !PageBuddy(buddy)) 895 return false; 896 897 if (buddy_order(buddy) != order) 898 return false; 899 900 /* 901 * zone check is done late to avoid uselessly calculating 902 * zone/node ids for pages that could never merge. 903 */ 904 if (page_zone_id(page) != page_zone_id(buddy)) 905 return false; 906 907 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 908 909 return true; 910 } 911 912 #ifdef CONFIG_COMPACTION 913 static inline struct capture_control *task_capc(struct zone *zone) 914 { 915 struct capture_control *capc = current->capture_control; 916 917 return unlikely(capc) && 918 !(current->flags & PF_KTHREAD) && 919 !capc->page && 920 capc->cc->zone == zone ? capc : NULL; 921 } 922 923 static inline bool 924 compaction_capture(struct capture_control *capc, struct page *page, 925 int order, int migratetype) 926 { 927 if (!capc || order != capc->cc->order) 928 return false; 929 930 /* Do not accidentally pollute CMA or isolated regions*/ 931 if (is_migrate_cma(migratetype) || 932 is_migrate_isolate(migratetype)) 933 return false; 934 935 /* 936 * Do not let lower order allocations pollute a movable pageblock. 937 * This might let an unmovable request use a reclaimable pageblock 938 * and vice-versa but no more than normal fallback logic which can 939 * have trouble finding a high-order free page. 940 */ 941 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE) 942 return false; 943 944 capc->page = page; 945 return true; 946 } 947 948 #else 949 static inline struct capture_control *task_capc(struct zone *zone) 950 { 951 return NULL; 952 } 953 954 static inline bool 955 compaction_capture(struct capture_control *capc, struct page *page, 956 int order, int migratetype) 957 { 958 return false; 959 } 960 #endif /* CONFIG_COMPACTION */ 961 962 /* Used for pages not on another list */ 963 static inline void add_to_free_list(struct page *page, struct zone *zone, 964 unsigned int order, int migratetype) 965 { 966 struct free_area *area = &zone->free_area[order]; 967 968 list_add(&page->lru, &area->free_list[migratetype]); 969 area->nr_free++; 970 } 971 972 /* Used for pages not on another list */ 973 static inline void add_to_free_list_tail(struct page *page, struct zone *zone, 974 unsigned int order, int migratetype) 975 { 976 struct free_area *area = &zone->free_area[order]; 977 978 list_add_tail(&page->lru, &area->free_list[migratetype]); 979 area->nr_free++; 980 } 981 982 /* 983 * Used for pages which are on another list. Move the pages to the tail 984 * of the list - so the moved pages won't immediately be considered for 985 * allocation again (e.g., optimization for memory onlining). 986 */ 987 static inline void move_to_free_list(struct page *page, struct zone *zone, 988 unsigned int order, int migratetype) 989 { 990 struct free_area *area = &zone->free_area[order]; 991 992 list_move_tail(&page->lru, &area->free_list[migratetype]); 993 } 994 995 static inline void del_page_from_free_list(struct page *page, struct zone *zone, 996 unsigned int order) 997 { 998 /* clear reported state and update reported page count */ 999 if (page_reported(page)) 1000 __ClearPageReported(page); 1001 1002 list_del(&page->lru); 1003 __ClearPageBuddy(page); 1004 set_page_private(page, 0); 1005 zone->free_area[order].nr_free--; 1006 } 1007 1008 /* 1009 * If this is not the largest possible page, check if the buddy 1010 * of the next-highest order is free. If it is, it's possible 1011 * that pages are being freed that will coalesce soon. In case, 1012 * that is happening, add the free page to the tail of the list 1013 * so it's less likely to be used soon and more likely to be merged 1014 * as a higher order page 1015 */ 1016 static inline bool 1017 buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, 1018 struct page *page, unsigned int order) 1019 { 1020 struct page *higher_page, *higher_buddy; 1021 unsigned long combined_pfn; 1022 1023 if (order >= MAX_ORDER - 2) 1024 return false; 1025 1026 combined_pfn = buddy_pfn & pfn; 1027 higher_page = page + (combined_pfn - pfn); 1028 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1); 1029 higher_buddy = higher_page + (buddy_pfn - combined_pfn); 1030 1031 return page_is_buddy(higher_page, higher_buddy, order + 1); 1032 } 1033 1034 /* 1035 * Freeing function for a buddy system allocator. 1036 * 1037 * The concept of a buddy system is to maintain direct-mapped table 1038 * (containing bit values) for memory blocks of various "orders". 1039 * The bottom level table contains the map for the smallest allocatable 1040 * units of memory (here, pages), and each level above it describes 1041 * pairs of units from the levels below, hence, "buddies". 1042 * At a high level, all that happens here is marking the table entry 1043 * at the bottom level available, and propagating the changes upward 1044 * as necessary, plus some accounting needed to play nicely with other 1045 * parts of the VM system. 1046 * At each level, we keep a list of pages, which are heads of continuous 1047 * free pages of length of (1 << order) and marked with PageBuddy. 1048 * Page's order is recorded in page_private(page) field. 1049 * So when we are allocating or freeing one, we can derive the state of the 1050 * other. That is, if we allocate a small block, and both were 1051 * free, the remainder of the region must be split into blocks. 1052 * If a block is freed, and its buddy is also free, then this 1053 * triggers coalescing into a block of larger size. 1054 * 1055 * -- nyc 1056 */ 1057 1058 static inline void __free_one_page(struct page *page, 1059 unsigned long pfn, 1060 struct zone *zone, unsigned int order, 1061 int migratetype, fpi_t fpi_flags) 1062 { 1063 struct capture_control *capc = task_capc(zone); 1064 unsigned long buddy_pfn; 1065 unsigned long combined_pfn; 1066 unsigned int max_order; 1067 struct page *buddy; 1068 bool to_tail; 1069 1070 max_order = min_t(unsigned int, MAX_ORDER - 1, pageblock_order); 1071 1072 VM_BUG_ON(!zone_is_initialized(zone)); 1073 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 1074 1075 VM_BUG_ON(migratetype == -1); 1076 if (likely(!is_migrate_isolate(migratetype))) 1077 __mod_zone_freepage_state(zone, 1 << order, migratetype); 1078 1079 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); 1080 VM_BUG_ON_PAGE(bad_range(zone, page), page); 1081 1082 continue_merging: 1083 while (order < max_order) { 1084 if (compaction_capture(capc, page, order, migratetype)) { 1085 __mod_zone_freepage_state(zone, -(1 << order), 1086 migratetype); 1087 return; 1088 } 1089 buddy_pfn = __find_buddy_pfn(pfn, order); 1090 buddy = page + (buddy_pfn - pfn); 1091 1092 if (!page_is_buddy(page, buddy, order)) 1093 goto done_merging; 1094 /* 1095 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 1096 * merge with it and move up one order. 1097 */ 1098 if (page_is_guard(buddy)) 1099 clear_page_guard(zone, buddy, order, migratetype); 1100 else 1101 del_page_from_free_list(buddy, zone, order); 1102 combined_pfn = buddy_pfn & pfn; 1103 page = page + (combined_pfn - pfn); 1104 pfn = combined_pfn; 1105 order++; 1106 } 1107 if (order < MAX_ORDER - 1) { 1108 /* If we are here, it means order is >= pageblock_order. 1109 * We want to prevent merge between freepages on isolate 1110 * pageblock and normal pageblock. Without this, pageblock 1111 * isolation could cause incorrect freepage or CMA accounting. 1112 * 1113 * We don't want to hit this code for the more frequent 1114 * low-order merging. 1115 */ 1116 if (unlikely(has_isolate_pageblock(zone))) { 1117 int buddy_mt; 1118 1119 buddy_pfn = __find_buddy_pfn(pfn, order); 1120 buddy = page + (buddy_pfn - pfn); 1121 buddy_mt = get_pageblock_migratetype(buddy); 1122 1123 if (migratetype != buddy_mt 1124 && (is_migrate_isolate(migratetype) || 1125 is_migrate_isolate(buddy_mt))) 1126 goto done_merging; 1127 } 1128 max_order = order + 1; 1129 goto continue_merging; 1130 } 1131 1132 done_merging: 1133 set_buddy_order(page, order); 1134 1135 if (fpi_flags & FPI_TO_TAIL) 1136 to_tail = true; 1137 else if (is_shuffle_order(order)) 1138 to_tail = shuffle_pick_tail(); 1139 else 1140 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); 1141 1142 if (to_tail) 1143 add_to_free_list_tail(page, zone, order, migratetype); 1144 else 1145 add_to_free_list(page, zone, order, migratetype); 1146 1147 /* Notify page reporting subsystem of freed page */ 1148 if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY)) 1149 page_reporting_notify_free(order); 1150 } 1151 1152 /* 1153 * A bad page could be due to a number of fields. Instead of multiple branches, 1154 * try and check multiple fields with one check. The caller must do a detailed 1155 * check if necessary. 1156 */ 1157 static inline bool page_expected_state(struct page *page, 1158 unsigned long check_flags) 1159 { 1160 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1161 return false; 1162 1163 if (unlikely((unsigned long)page->mapping | 1164 page_ref_count(page) | 1165 #ifdef CONFIG_MEMCG 1166 page->memcg_data | 1167 #endif 1168 (page->flags & check_flags))) 1169 return false; 1170 1171 return true; 1172 } 1173 1174 static const char *page_bad_reason(struct page *page, unsigned long flags) 1175 { 1176 const char *bad_reason = NULL; 1177 1178 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1179 bad_reason = "nonzero mapcount"; 1180 if (unlikely(page->mapping != NULL)) 1181 bad_reason = "non-NULL mapping"; 1182 if (unlikely(page_ref_count(page) != 0)) 1183 bad_reason = "nonzero _refcount"; 1184 if (unlikely(page->flags & flags)) { 1185 if (flags == PAGE_FLAGS_CHECK_AT_PREP) 1186 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set"; 1187 else 1188 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 1189 } 1190 #ifdef CONFIG_MEMCG 1191 if (unlikely(page->memcg_data)) 1192 bad_reason = "page still charged to cgroup"; 1193 #endif 1194 return bad_reason; 1195 } 1196 1197 static void check_free_page_bad(struct page *page) 1198 { 1199 bad_page(page, 1200 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE)); 1201 } 1202 1203 static inline int check_free_page(struct page *page) 1204 { 1205 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 1206 return 0; 1207 1208 /* Something has gone sideways, find it */ 1209 check_free_page_bad(page); 1210 return 1; 1211 } 1212 1213 static int free_tail_pages_check(struct page *head_page, struct page *page) 1214 { 1215 int ret = 1; 1216 1217 /* 1218 * We rely page->lru.next never has bit 0 set, unless the page 1219 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 1220 */ 1221 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 1222 1223 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 1224 ret = 0; 1225 goto out; 1226 } 1227 switch (page - head_page) { 1228 case 1: 1229 /* the first tail page: ->mapping may be compound_mapcount() */ 1230 if (unlikely(compound_mapcount(page))) { 1231 bad_page(page, "nonzero compound_mapcount"); 1232 goto out; 1233 } 1234 break; 1235 case 2: 1236 /* 1237 * the second tail page: ->mapping is 1238 * deferred_list.next -- ignore value. 1239 */ 1240 break; 1241 default: 1242 if (page->mapping != TAIL_MAPPING) { 1243 bad_page(page, "corrupted mapping in tail page"); 1244 goto out; 1245 } 1246 break; 1247 } 1248 if (unlikely(!PageTail(page))) { 1249 bad_page(page, "PageTail not set"); 1250 goto out; 1251 } 1252 if (unlikely(compound_head(page) != head_page)) { 1253 bad_page(page, "compound_head not consistent"); 1254 goto out; 1255 } 1256 ret = 0; 1257 out: 1258 page->mapping = NULL; 1259 clear_compound_head(page); 1260 return ret; 1261 } 1262 1263 static void kernel_init_free_pages(struct page *page, int numpages, bool zero_tags) 1264 { 1265 int i; 1266 1267 if (zero_tags) { 1268 for (i = 0; i < numpages; i++) 1269 tag_clear_highpage(page + i); 1270 return; 1271 } 1272 1273 /* s390's use of memset() could override KASAN redzones. */ 1274 kasan_disable_current(); 1275 for (i = 0; i < numpages; i++) { 1276 u8 tag = page_kasan_tag(page + i); 1277 page_kasan_tag_reset(page + i); 1278 clear_highpage(page + i); 1279 page_kasan_tag_set(page + i, tag); 1280 } 1281 kasan_enable_current(); 1282 } 1283 1284 static __always_inline bool free_pages_prepare(struct page *page, 1285 unsigned int order, bool check_free, fpi_t fpi_flags) 1286 { 1287 int bad = 0; 1288 bool skip_kasan_poison = should_skip_kasan_poison(page, fpi_flags); 1289 1290 VM_BUG_ON_PAGE(PageTail(page), page); 1291 1292 trace_mm_page_free(page, order); 1293 1294 if (unlikely(PageHWPoison(page)) && !order) { 1295 /* 1296 * Do not let hwpoison pages hit pcplists/buddy 1297 * Untie memcg state and reset page's owner 1298 */ 1299 if (memcg_kmem_enabled() && PageMemcgKmem(page)) 1300 __memcg_kmem_uncharge_page(page, order); 1301 reset_page_owner(page, order); 1302 return false; 1303 } 1304 1305 /* 1306 * Check tail pages before head page information is cleared to 1307 * avoid checking PageCompound for order-0 pages. 1308 */ 1309 if (unlikely(order)) { 1310 bool compound = PageCompound(page); 1311 int i; 1312 1313 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1314 1315 if (compound) 1316 ClearPageDoubleMap(page); 1317 for (i = 1; i < (1 << order); i++) { 1318 if (compound) 1319 bad += free_tail_pages_check(page, page + i); 1320 if (unlikely(check_free_page(page + i))) { 1321 bad++; 1322 continue; 1323 } 1324 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1325 } 1326 } 1327 if (PageMappingFlags(page)) 1328 page->mapping = NULL; 1329 if (memcg_kmem_enabled() && PageMemcgKmem(page)) 1330 __memcg_kmem_uncharge_page(page, order); 1331 if (check_free) 1332 bad += check_free_page(page); 1333 if (bad) 1334 return false; 1335 1336 page_cpupid_reset_last(page); 1337 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1338 reset_page_owner(page, order); 1339 1340 if (!PageHighMem(page)) { 1341 debug_check_no_locks_freed(page_address(page), 1342 PAGE_SIZE << order); 1343 debug_check_no_obj_freed(page_address(page), 1344 PAGE_SIZE << order); 1345 } 1346 1347 kernel_poison_pages(page, 1 << order); 1348 1349 /* 1350 * As memory initialization might be integrated into KASAN, 1351 * kasan_free_pages and kernel_init_free_pages must be 1352 * kept together to avoid discrepancies in behavior. 1353 * 1354 * With hardware tag-based KASAN, memory tags must be set before the 1355 * page becomes unavailable via debug_pagealloc or arch_free_page. 1356 */ 1357 if (kasan_has_integrated_init()) { 1358 if (!skip_kasan_poison) 1359 kasan_free_pages(page, order); 1360 } else { 1361 bool init = want_init_on_free(); 1362 1363 if (init) 1364 kernel_init_free_pages(page, 1 << order, false); 1365 if (!skip_kasan_poison) 1366 kasan_poison_pages(page, order, init); 1367 } 1368 1369 /* 1370 * arch_free_page() can make the page's contents inaccessible. s390 1371 * does this. So nothing which can access the page's contents should 1372 * happen after this. 1373 */ 1374 arch_free_page(page, order); 1375 1376 debug_pagealloc_unmap_pages(page, 1 << order); 1377 1378 return true; 1379 } 1380 1381 #ifdef CONFIG_DEBUG_VM 1382 /* 1383 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed 1384 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when 1385 * moved from pcp lists to free lists. 1386 */ 1387 static bool free_pcp_prepare(struct page *page, unsigned int order) 1388 { 1389 return free_pages_prepare(page, order, true, FPI_NONE); 1390 } 1391 1392 static bool bulkfree_pcp_prepare(struct page *page) 1393 { 1394 if (debug_pagealloc_enabled_static()) 1395 return check_free_page(page); 1396 else 1397 return false; 1398 } 1399 #else 1400 /* 1401 * With DEBUG_VM disabled, order-0 pages being freed are checked only when 1402 * moving from pcp lists to free list in order to reduce overhead. With 1403 * debug_pagealloc enabled, they are checked also immediately when being freed 1404 * to the pcp lists. 1405 */ 1406 static bool free_pcp_prepare(struct page *page, unsigned int order) 1407 { 1408 if (debug_pagealloc_enabled_static()) 1409 return free_pages_prepare(page, order, true, FPI_NONE); 1410 else 1411 return free_pages_prepare(page, order, false, FPI_NONE); 1412 } 1413 1414 static bool bulkfree_pcp_prepare(struct page *page) 1415 { 1416 return check_free_page(page); 1417 } 1418 #endif /* CONFIG_DEBUG_VM */ 1419 1420 static inline void prefetch_buddy(struct page *page) 1421 { 1422 unsigned long pfn = page_to_pfn(page); 1423 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0); 1424 struct page *buddy = page + (buddy_pfn - pfn); 1425 1426 prefetch(buddy); 1427 } 1428 1429 /* 1430 * Frees a number of pages from the PCP lists 1431 * Assumes all pages on list are in same zone, and of same order. 1432 * count is the number of pages to free. 1433 * 1434 * If the zone was previously in an "all pages pinned" state then look to 1435 * see if this freeing clears that state. 1436 * 1437 * And clear the zone's pages_scanned counter, to hold off the "all pages are 1438 * pinned" detection logic. 1439 */ 1440 static void free_pcppages_bulk(struct zone *zone, int count, 1441 struct per_cpu_pages *pcp) 1442 { 1443 int pindex = 0; 1444 int batch_free = 0; 1445 int nr_freed = 0; 1446 unsigned int order; 1447 int prefetch_nr = READ_ONCE(pcp->batch); 1448 bool isolated_pageblocks; 1449 struct page *page, *tmp; 1450 LIST_HEAD(head); 1451 1452 /* 1453 * Ensure proper count is passed which otherwise would stuck in the 1454 * below while (list_empty(list)) loop. 1455 */ 1456 count = min(pcp->count, count); 1457 while (count > 0) { 1458 struct list_head *list; 1459 1460 /* 1461 * Remove pages from lists in a round-robin fashion. A 1462 * batch_free count is maintained that is incremented when an 1463 * empty list is encountered. This is so more pages are freed 1464 * off fuller lists instead of spinning excessively around empty 1465 * lists 1466 */ 1467 do { 1468 batch_free++; 1469 if (++pindex == NR_PCP_LISTS) 1470 pindex = 0; 1471 list = &pcp->lists[pindex]; 1472 } while (list_empty(list)); 1473 1474 /* This is the only non-empty list. Free them all. */ 1475 if (batch_free == NR_PCP_LISTS) 1476 batch_free = count; 1477 1478 order = pindex_to_order(pindex); 1479 BUILD_BUG_ON(MAX_ORDER >= (1<<NR_PCP_ORDER_WIDTH)); 1480 do { 1481 page = list_last_entry(list, struct page, lru); 1482 /* must delete to avoid corrupting pcp list */ 1483 list_del(&page->lru); 1484 nr_freed += 1 << order; 1485 count -= 1 << order; 1486 1487 if (bulkfree_pcp_prepare(page)) 1488 continue; 1489 1490 /* Encode order with the migratetype */ 1491 page->index <<= NR_PCP_ORDER_WIDTH; 1492 page->index |= order; 1493 1494 list_add_tail(&page->lru, &head); 1495 1496 /* 1497 * We are going to put the page back to the global 1498 * pool, prefetch its buddy to speed up later access 1499 * under zone->lock. It is believed the overhead of 1500 * an additional test and calculating buddy_pfn here 1501 * can be offset by reduced memory latency later. To 1502 * avoid excessive prefetching due to large count, only 1503 * prefetch buddy for the first pcp->batch nr of pages. 1504 */ 1505 if (prefetch_nr) { 1506 prefetch_buddy(page); 1507 prefetch_nr--; 1508 } 1509 } while (count > 0 && --batch_free && !list_empty(list)); 1510 } 1511 pcp->count -= nr_freed; 1512 1513 /* 1514 * local_lock_irq held so equivalent to spin_lock_irqsave for 1515 * both PREEMPT_RT and non-PREEMPT_RT configurations. 1516 */ 1517 spin_lock(&zone->lock); 1518 isolated_pageblocks = has_isolate_pageblock(zone); 1519 1520 /* 1521 * Use safe version since after __free_one_page(), 1522 * page->lru.next will not point to original list. 1523 */ 1524 list_for_each_entry_safe(page, tmp, &head, lru) { 1525 int mt = get_pcppage_migratetype(page); 1526 1527 /* mt has been encoded with the order (see above) */ 1528 order = mt & NR_PCP_ORDER_MASK; 1529 mt >>= NR_PCP_ORDER_WIDTH; 1530 1531 /* MIGRATE_ISOLATE page should not go to pcplists */ 1532 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1533 /* Pageblock could have been isolated meanwhile */ 1534 if (unlikely(isolated_pageblocks)) 1535 mt = get_pageblock_migratetype(page); 1536 1537 __free_one_page(page, page_to_pfn(page), zone, order, mt, FPI_NONE); 1538 trace_mm_page_pcpu_drain(page, order, mt); 1539 } 1540 spin_unlock(&zone->lock); 1541 } 1542 1543 static void free_one_page(struct zone *zone, 1544 struct page *page, unsigned long pfn, 1545 unsigned int order, 1546 int migratetype, fpi_t fpi_flags) 1547 { 1548 unsigned long flags; 1549 1550 spin_lock_irqsave(&zone->lock, flags); 1551 if (unlikely(has_isolate_pageblock(zone) || 1552 is_migrate_isolate(migratetype))) { 1553 migratetype = get_pfnblock_migratetype(page, pfn); 1554 } 1555 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags); 1556 spin_unlock_irqrestore(&zone->lock, flags); 1557 } 1558 1559 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1560 unsigned long zone, int nid) 1561 { 1562 mm_zero_struct_page(page); 1563 set_page_links(page, zone, nid, pfn); 1564 init_page_count(page); 1565 page_mapcount_reset(page); 1566 page_cpupid_reset_last(page); 1567 page_kasan_tag_reset(page); 1568 1569 INIT_LIST_HEAD(&page->lru); 1570 #ifdef WANT_PAGE_VIRTUAL 1571 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1572 if (!is_highmem_idx(zone)) 1573 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1574 #endif 1575 } 1576 1577 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1578 static void __meminit init_reserved_page(unsigned long pfn) 1579 { 1580 pg_data_t *pgdat; 1581 int nid, zid; 1582 1583 if (!early_page_uninitialised(pfn)) 1584 return; 1585 1586 nid = early_pfn_to_nid(pfn); 1587 pgdat = NODE_DATA(nid); 1588 1589 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1590 struct zone *zone = &pgdat->node_zones[zid]; 1591 1592 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) 1593 break; 1594 } 1595 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 1596 } 1597 #else 1598 static inline void init_reserved_page(unsigned long pfn) 1599 { 1600 } 1601 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1602 1603 /* 1604 * Initialised pages do not have PageReserved set. This function is 1605 * called for each range allocated by the bootmem allocator and 1606 * marks the pages PageReserved. The remaining valid pages are later 1607 * sent to the buddy page allocator. 1608 */ 1609 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1610 { 1611 unsigned long start_pfn = PFN_DOWN(start); 1612 unsigned long end_pfn = PFN_UP(end); 1613 1614 for (; start_pfn < end_pfn; start_pfn++) { 1615 if (pfn_valid(start_pfn)) { 1616 struct page *page = pfn_to_page(start_pfn); 1617 1618 init_reserved_page(start_pfn); 1619 1620 /* Avoid false-positive PageTail() */ 1621 INIT_LIST_HEAD(&page->lru); 1622 1623 /* 1624 * no need for atomic set_bit because the struct 1625 * page is not visible yet so nobody should 1626 * access it yet. 1627 */ 1628 __SetPageReserved(page); 1629 } 1630 } 1631 } 1632 1633 static void __free_pages_ok(struct page *page, unsigned int order, 1634 fpi_t fpi_flags) 1635 { 1636 unsigned long flags; 1637 int migratetype; 1638 unsigned long pfn = page_to_pfn(page); 1639 struct zone *zone = page_zone(page); 1640 1641 if (!free_pages_prepare(page, order, true, fpi_flags)) 1642 return; 1643 1644 migratetype = get_pfnblock_migratetype(page, pfn); 1645 1646 spin_lock_irqsave(&zone->lock, flags); 1647 if (unlikely(has_isolate_pageblock(zone) || 1648 is_migrate_isolate(migratetype))) { 1649 migratetype = get_pfnblock_migratetype(page, pfn); 1650 } 1651 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags); 1652 spin_unlock_irqrestore(&zone->lock, flags); 1653 1654 __count_vm_events(PGFREE, 1 << order); 1655 } 1656 1657 void __free_pages_core(struct page *page, unsigned int order) 1658 { 1659 unsigned int nr_pages = 1 << order; 1660 struct page *p = page; 1661 unsigned int loop; 1662 1663 /* 1664 * When initializing the memmap, __init_single_page() sets the refcount 1665 * of all pages to 1 ("allocated"/"not free"). We have to set the 1666 * refcount of all involved pages to 0. 1667 */ 1668 prefetchw(p); 1669 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1670 prefetchw(p + 1); 1671 __ClearPageReserved(p); 1672 set_page_count(p, 0); 1673 } 1674 __ClearPageReserved(p); 1675 set_page_count(p, 0); 1676 1677 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); 1678 1679 /* 1680 * Bypass PCP and place fresh pages right to the tail, primarily 1681 * relevant for memory onlining. 1682 */ 1683 __free_pages_ok(page, order, FPI_TO_TAIL | FPI_SKIP_KASAN_POISON); 1684 } 1685 1686 #ifdef CONFIG_NUMA 1687 1688 /* 1689 * During memory init memblocks map pfns to nids. The search is expensive and 1690 * this caches recent lookups. The implementation of __early_pfn_to_nid 1691 * treats start/end as pfns. 1692 */ 1693 struct mminit_pfnnid_cache { 1694 unsigned long last_start; 1695 unsigned long last_end; 1696 int last_nid; 1697 }; 1698 1699 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1700 1701 /* 1702 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 1703 */ 1704 static int __meminit __early_pfn_to_nid(unsigned long pfn, 1705 struct mminit_pfnnid_cache *state) 1706 { 1707 unsigned long start_pfn, end_pfn; 1708 int nid; 1709 1710 if (state->last_start <= pfn && pfn < state->last_end) 1711 return state->last_nid; 1712 1713 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 1714 if (nid != NUMA_NO_NODE) { 1715 state->last_start = start_pfn; 1716 state->last_end = end_pfn; 1717 state->last_nid = nid; 1718 } 1719 1720 return nid; 1721 } 1722 1723 int __meminit early_pfn_to_nid(unsigned long pfn) 1724 { 1725 static DEFINE_SPINLOCK(early_pfn_lock); 1726 int nid; 1727 1728 spin_lock(&early_pfn_lock); 1729 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1730 if (nid < 0) 1731 nid = first_online_node; 1732 spin_unlock(&early_pfn_lock); 1733 1734 return nid; 1735 } 1736 #endif /* CONFIG_NUMA */ 1737 1738 void __init memblock_free_pages(struct page *page, unsigned long pfn, 1739 unsigned int order) 1740 { 1741 if (early_page_uninitialised(pfn)) 1742 return; 1743 __free_pages_core(page, order); 1744 } 1745 1746 /* 1747 * Check that the whole (or subset of) a pageblock given by the interval of 1748 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1749 * with the migration of free compaction scanner. 1750 * 1751 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1752 * 1753 * It's possible on some configurations to have a setup like node0 node1 node0 1754 * i.e. it's possible that all pages within a zones range of pages do not 1755 * belong to a single zone. We assume that a border between node0 and node1 1756 * can occur within a single pageblock, but not a node0 node1 node0 1757 * interleaving within a single pageblock. It is therefore sufficient to check 1758 * the first and last page of a pageblock and avoid checking each individual 1759 * page in a pageblock. 1760 */ 1761 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1762 unsigned long end_pfn, struct zone *zone) 1763 { 1764 struct page *start_page; 1765 struct page *end_page; 1766 1767 /* end_pfn is one past the range we are checking */ 1768 end_pfn--; 1769 1770 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1771 return NULL; 1772 1773 start_page = pfn_to_online_page(start_pfn); 1774 if (!start_page) 1775 return NULL; 1776 1777 if (page_zone(start_page) != zone) 1778 return NULL; 1779 1780 end_page = pfn_to_page(end_pfn); 1781 1782 /* This gives a shorter code than deriving page_zone(end_page) */ 1783 if (page_zone_id(start_page) != page_zone_id(end_page)) 1784 return NULL; 1785 1786 return start_page; 1787 } 1788 1789 void set_zone_contiguous(struct zone *zone) 1790 { 1791 unsigned long block_start_pfn = zone->zone_start_pfn; 1792 unsigned long block_end_pfn; 1793 1794 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1795 for (; block_start_pfn < zone_end_pfn(zone); 1796 block_start_pfn = block_end_pfn, 1797 block_end_pfn += pageblock_nr_pages) { 1798 1799 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1800 1801 if (!__pageblock_pfn_to_page(block_start_pfn, 1802 block_end_pfn, zone)) 1803 return; 1804 cond_resched(); 1805 } 1806 1807 /* We confirm that there is no hole */ 1808 zone->contiguous = true; 1809 } 1810 1811 void clear_zone_contiguous(struct zone *zone) 1812 { 1813 zone->contiguous = false; 1814 } 1815 1816 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1817 static void __init deferred_free_range(unsigned long pfn, 1818 unsigned long nr_pages) 1819 { 1820 struct page *page; 1821 unsigned long i; 1822 1823 if (!nr_pages) 1824 return; 1825 1826 page = pfn_to_page(pfn); 1827 1828 /* Free a large naturally-aligned chunk if possible */ 1829 if (nr_pages == pageblock_nr_pages && 1830 (pfn & (pageblock_nr_pages - 1)) == 0) { 1831 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1832 __free_pages_core(page, pageblock_order); 1833 return; 1834 } 1835 1836 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1837 if ((pfn & (pageblock_nr_pages - 1)) == 0) 1838 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1839 __free_pages_core(page, 0); 1840 } 1841 } 1842 1843 /* Completion tracking for deferred_init_memmap() threads */ 1844 static atomic_t pgdat_init_n_undone __initdata; 1845 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1846 1847 static inline void __init pgdat_init_report_one_done(void) 1848 { 1849 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1850 complete(&pgdat_init_all_done_comp); 1851 } 1852 1853 /* 1854 * Returns true if page needs to be initialized or freed to buddy allocator. 1855 * 1856 * First we check if pfn is valid on architectures where it is possible to have 1857 * holes within pageblock_nr_pages. On systems where it is not possible, this 1858 * function is optimized out. 1859 * 1860 * Then, we check if a current large page is valid by only checking the validity 1861 * of the head pfn. 1862 */ 1863 static inline bool __init deferred_pfn_valid(unsigned long pfn) 1864 { 1865 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn)) 1866 return false; 1867 return true; 1868 } 1869 1870 /* 1871 * Free pages to buddy allocator. Try to free aligned pages in 1872 * pageblock_nr_pages sizes. 1873 */ 1874 static void __init deferred_free_pages(unsigned long pfn, 1875 unsigned long end_pfn) 1876 { 1877 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1878 unsigned long nr_free = 0; 1879 1880 for (; pfn < end_pfn; pfn++) { 1881 if (!deferred_pfn_valid(pfn)) { 1882 deferred_free_range(pfn - nr_free, nr_free); 1883 nr_free = 0; 1884 } else if (!(pfn & nr_pgmask)) { 1885 deferred_free_range(pfn - nr_free, nr_free); 1886 nr_free = 1; 1887 } else { 1888 nr_free++; 1889 } 1890 } 1891 /* Free the last block of pages to allocator */ 1892 deferred_free_range(pfn - nr_free, nr_free); 1893 } 1894 1895 /* 1896 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 1897 * by performing it only once every pageblock_nr_pages. 1898 * Return number of pages initialized. 1899 */ 1900 static unsigned long __init deferred_init_pages(struct zone *zone, 1901 unsigned long pfn, 1902 unsigned long end_pfn) 1903 { 1904 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1905 int nid = zone_to_nid(zone); 1906 unsigned long nr_pages = 0; 1907 int zid = zone_idx(zone); 1908 struct page *page = NULL; 1909 1910 for (; pfn < end_pfn; pfn++) { 1911 if (!deferred_pfn_valid(pfn)) { 1912 page = NULL; 1913 continue; 1914 } else if (!page || !(pfn & nr_pgmask)) { 1915 page = pfn_to_page(pfn); 1916 } else { 1917 page++; 1918 } 1919 __init_single_page(page, pfn, zid, nid); 1920 nr_pages++; 1921 } 1922 return (nr_pages); 1923 } 1924 1925 /* 1926 * This function is meant to pre-load the iterator for the zone init. 1927 * Specifically it walks through the ranges until we are caught up to the 1928 * first_init_pfn value and exits there. If we never encounter the value we 1929 * return false indicating there are no valid ranges left. 1930 */ 1931 static bool __init 1932 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, 1933 unsigned long *spfn, unsigned long *epfn, 1934 unsigned long first_init_pfn) 1935 { 1936 u64 j; 1937 1938 /* 1939 * Start out by walking through the ranges in this zone that have 1940 * already been initialized. We don't need to do anything with them 1941 * so we just need to flush them out of the system. 1942 */ 1943 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { 1944 if (*epfn <= first_init_pfn) 1945 continue; 1946 if (*spfn < first_init_pfn) 1947 *spfn = first_init_pfn; 1948 *i = j; 1949 return true; 1950 } 1951 1952 return false; 1953 } 1954 1955 /* 1956 * Initialize and free pages. We do it in two loops: first we initialize 1957 * struct page, then free to buddy allocator, because while we are 1958 * freeing pages we can access pages that are ahead (computing buddy 1959 * page in __free_one_page()). 1960 * 1961 * In order to try and keep some memory in the cache we have the loop 1962 * broken along max page order boundaries. This way we will not cause 1963 * any issues with the buddy page computation. 1964 */ 1965 static unsigned long __init 1966 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, 1967 unsigned long *end_pfn) 1968 { 1969 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); 1970 unsigned long spfn = *start_pfn, epfn = *end_pfn; 1971 unsigned long nr_pages = 0; 1972 u64 j = *i; 1973 1974 /* First we loop through and initialize the page values */ 1975 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { 1976 unsigned long t; 1977 1978 if (mo_pfn <= *start_pfn) 1979 break; 1980 1981 t = min(mo_pfn, *end_pfn); 1982 nr_pages += deferred_init_pages(zone, *start_pfn, t); 1983 1984 if (mo_pfn < *end_pfn) { 1985 *start_pfn = mo_pfn; 1986 break; 1987 } 1988 } 1989 1990 /* Reset values and now loop through freeing pages as needed */ 1991 swap(j, *i); 1992 1993 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { 1994 unsigned long t; 1995 1996 if (mo_pfn <= spfn) 1997 break; 1998 1999 t = min(mo_pfn, epfn); 2000 deferred_free_pages(spfn, t); 2001 2002 if (mo_pfn <= epfn) 2003 break; 2004 } 2005 2006 return nr_pages; 2007 } 2008 2009 static void __init 2010 deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, 2011 void *arg) 2012 { 2013 unsigned long spfn, epfn; 2014 struct zone *zone = arg; 2015 u64 i; 2016 2017 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); 2018 2019 /* 2020 * Initialize and free pages in MAX_ORDER sized increments so that we 2021 * can avoid introducing any issues with the buddy allocator. 2022 */ 2023 while (spfn < end_pfn) { 2024 deferred_init_maxorder(&i, zone, &spfn, &epfn); 2025 cond_resched(); 2026 } 2027 } 2028 2029 /* An arch may override for more concurrency. */ 2030 __weak int __init 2031 deferred_page_init_max_threads(const struct cpumask *node_cpumask) 2032 { 2033 return 1; 2034 } 2035 2036 /* Initialise remaining memory on a node */ 2037 static int __init deferred_init_memmap(void *data) 2038 { 2039 pg_data_t *pgdat = data; 2040 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 2041 unsigned long spfn = 0, epfn = 0; 2042 unsigned long first_init_pfn, flags; 2043 unsigned long start = jiffies; 2044 struct zone *zone; 2045 int zid, max_threads; 2046 u64 i; 2047 2048 /* Bind memory initialisation thread to a local node if possible */ 2049 if (!cpumask_empty(cpumask)) 2050 set_cpus_allowed_ptr(current, cpumask); 2051 2052 pgdat_resize_lock(pgdat, &flags); 2053 first_init_pfn = pgdat->first_deferred_pfn; 2054 if (first_init_pfn == ULONG_MAX) { 2055 pgdat_resize_unlock(pgdat, &flags); 2056 pgdat_init_report_one_done(); 2057 return 0; 2058 } 2059 2060 /* Sanity check boundaries */ 2061 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 2062 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 2063 pgdat->first_deferred_pfn = ULONG_MAX; 2064 2065 /* 2066 * Once we unlock here, the zone cannot be grown anymore, thus if an 2067 * interrupt thread must allocate this early in boot, zone must be 2068 * pre-grown prior to start of deferred page initialization. 2069 */ 2070 pgdat_resize_unlock(pgdat, &flags); 2071 2072 /* Only the highest zone is deferred so find it */ 2073 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 2074 zone = pgdat->node_zones + zid; 2075 if (first_init_pfn < zone_end_pfn(zone)) 2076 break; 2077 } 2078 2079 /* If the zone is empty somebody else may have cleared out the zone */ 2080 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2081 first_init_pfn)) 2082 goto zone_empty; 2083 2084 max_threads = deferred_page_init_max_threads(cpumask); 2085 2086 while (spfn < epfn) { 2087 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION); 2088 struct padata_mt_job job = { 2089 .thread_fn = deferred_init_memmap_chunk, 2090 .fn_arg = zone, 2091 .start = spfn, 2092 .size = epfn_align - spfn, 2093 .align = PAGES_PER_SECTION, 2094 .min_chunk = PAGES_PER_SECTION, 2095 .max_threads = max_threads, 2096 }; 2097 2098 padata_do_multithreaded(&job); 2099 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2100 epfn_align); 2101 } 2102 zone_empty: 2103 /* Sanity check that the next zone really is unpopulated */ 2104 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 2105 2106 pr_info("node %d deferred pages initialised in %ums\n", 2107 pgdat->node_id, jiffies_to_msecs(jiffies - start)); 2108 2109 pgdat_init_report_one_done(); 2110 return 0; 2111 } 2112 2113 /* 2114 * If this zone has deferred pages, try to grow it by initializing enough 2115 * deferred pages to satisfy the allocation specified by order, rounded up to 2116 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 2117 * of SECTION_SIZE bytes by initializing struct pages in increments of 2118 * PAGES_PER_SECTION * sizeof(struct page) bytes. 2119 * 2120 * Return true when zone was grown, otherwise return false. We return true even 2121 * when we grow less than requested, to let the caller decide if there are 2122 * enough pages to satisfy the allocation. 2123 * 2124 * Note: We use noinline because this function is needed only during boot, and 2125 * it is called from a __ref function _deferred_grow_zone. This way we are 2126 * making sure that it is not inlined into permanent text section. 2127 */ 2128 static noinline bool __init 2129 deferred_grow_zone(struct zone *zone, unsigned int order) 2130 { 2131 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 2132 pg_data_t *pgdat = zone->zone_pgdat; 2133 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 2134 unsigned long spfn, epfn, flags; 2135 unsigned long nr_pages = 0; 2136 u64 i; 2137 2138 /* Only the last zone may have deferred pages */ 2139 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 2140 return false; 2141 2142 pgdat_resize_lock(pgdat, &flags); 2143 2144 /* 2145 * If someone grew this zone while we were waiting for spinlock, return 2146 * true, as there might be enough pages already. 2147 */ 2148 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 2149 pgdat_resize_unlock(pgdat, &flags); 2150 return true; 2151 } 2152 2153 /* If the zone is empty somebody else may have cleared out the zone */ 2154 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2155 first_deferred_pfn)) { 2156 pgdat->first_deferred_pfn = ULONG_MAX; 2157 pgdat_resize_unlock(pgdat, &flags); 2158 /* Retry only once. */ 2159 return first_deferred_pfn != ULONG_MAX; 2160 } 2161 2162 /* 2163 * Initialize and free pages in MAX_ORDER sized increments so 2164 * that we can avoid introducing any issues with the buddy 2165 * allocator. 2166 */ 2167 while (spfn < epfn) { 2168 /* update our first deferred PFN for this section */ 2169 first_deferred_pfn = spfn; 2170 2171 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 2172 touch_nmi_watchdog(); 2173 2174 /* We should only stop along section boundaries */ 2175 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) 2176 continue; 2177 2178 /* If our quota has been met we can stop here */ 2179 if (nr_pages >= nr_pages_needed) 2180 break; 2181 } 2182 2183 pgdat->first_deferred_pfn = spfn; 2184 pgdat_resize_unlock(pgdat, &flags); 2185 2186 return nr_pages > 0; 2187 } 2188 2189 /* 2190 * deferred_grow_zone() is __init, but it is called from 2191 * get_page_from_freelist() during early boot until deferred_pages permanently 2192 * disables this call. This is why we have refdata wrapper to avoid warning, 2193 * and to ensure that the function body gets unloaded. 2194 */ 2195 static bool __ref 2196 _deferred_grow_zone(struct zone *zone, unsigned int order) 2197 { 2198 return deferred_grow_zone(zone, order); 2199 } 2200 2201 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 2202 2203 void __init page_alloc_init_late(void) 2204 { 2205 struct zone *zone; 2206 int nid; 2207 2208 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 2209 2210 /* There will be num_node_state(N_MEMORY) threads */ 2211 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 2212 for_each_node_state(nid, N_MEMORY) { 2213 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 2214 } 2215 2216 /* Block until all are initialised */ 2217 wait_for_completion(&pgdat_init_all_done_comp); 2218 2219 /* 2220 * We initialized the rest of the deferred pages. Permanently disable 2221 * on-demand struct page initialization. 2222 */ 2223 static_branch_disable(&deferred_pages); 2224 2225 /* Reinit limits that are based on free pages after the kernel is up */ 2226 files_maxfiles_init(); 2227 #endif 2228 2229 buffer_init(); 2230 2231 /* Discard memblock private memory */ 2232 memblock_discard(); 2233 2234 for_each_node_state(nid, N_MEMORY) 2235 shuffle_free_memory(NODE_DATA(nid)); 2236 2237 for_each_populated_zone(zone) 2238 set_zone_contiguous(zone); 2239 } 2240 2241 #ifdef CONFIG_CMA 2242 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 2243 void __init init_cma_reserved_pageblock(struct page *page) 2244 { 2245 unsigned i = pageblock_nr_pages; 2246 struct page *p = page; 2247 2248 do { 2249 __ClearPageReserved(p); 2250 set_page_count(p, 0); 2251 } while (++p, --i); 2252 2253 set_pageblock_migratetype(page, MIGRATE_CMA); 2254 2255 if (pageblock_order >= MAX_ORDER) { 2256 i = pageblock_nr_pages; 2257 p = page; 2258 do { 2259 set_page_refcounted(p); 2260 __free_pages(p, MAX_ORDER - 1); 2261 p += MAX_ORDER_NR_PAGES; 2262 } while (i -= MAX_ORDER_NR_PAGES); 2263 } else { 2264 set_page_refcounted(page); 2265 __free_pages(page, pageblock_order); 2266 } 2267 2268 adjust_managed_page_count(page, pageblock_nr_pages); 2269 page_zone(page)->cma_pages += pageblock_nr_pages; 2270 } 2271 #endif 2272 2273 /* 2274 * The order of subdivision here is critical for the IO subsystem. 2275 * Please do not alter this order without good reasons and regression 2276 * testing. Specifically, as large blocks of memory are subdivided, 2277 * the order in which smaller blocks are delivered depends on the order 2278 * they're subdivided in this function. This is the primary factor 2279 * influencing the order in which pages are delivered to the IO 2280 * subsystem according to empirical testing, and this is also justified 2281 * by considering the behavior of a buddy system containing a single 2282 * large block of memory acted on by a series of small allocations. 2283 * This behavior is a critical factor in sglist merging's success. 2284 * 2285 * -- nyc 2286 */ 2287 static inline void expand(struct zone *zone, struct page *page, 2288 int low, int high, int migratetype) 2289 { 2290 unsigned long size = 1 << high; 2291 2292 while (high > low) { 2293 high--; 2294 size >>= 1; 2295 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 2296 2297 /* 2298 * Mark as guard pages (or page), that will allow to 2299 * merge back to allocator when buddy will be freed. 2300 * Corresponding page table entries will not be touched, 2301 * pages will stay not present in virtual address space 2302 */ 2303 if (set_page_guard(zone, &page[size], high, migratetype)) 2304 continue; 2305 2306 add_to_free_list(&page[size], zone, high, migratetype); 2307 set_buddy_order(&page[size], high); 2308 } 2309 } 2310 2311 static void check_new_page_bad(struct page *page) 2312 { 2313 if (unlikely(page->flags & __PG_HWPOISON)) { 2314 /* Don't complain about hwpoisoned pages */ 2315 page_mapcount_reset(page); /* remove PageBuddy */ 2316 return; 2317 } 2318 2319 bad_page(page, 2320 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP)); 2321 } 2322 2323 /* 2324 * This page is about to be returned from the page allocator 2325 */ 2326 static inline int check_new_page(struct page *page) 2327 { 2328 if (likely(page_expected_state(page, 2329 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 2330 return 0; 2331 2332 check_new_page_bad(page); 2333 return 1; 2334 } 2335 2336 #ifdef CONFIG_DEBUG_VM 2337 /* 2338 * With DEBUG_VM enabled, order-0 pages are checked for expected state when 2339 * being allocated from pcp lists. With debug_pagealloc also enabled, they are 2340 * also checked when pcp lists are refilled from the free lists. 2341 */ 2342 static inline bool check_pcp_refill(struct page *page) 2343 { 2344 if (debug_pagealloc_enabled_static()) 2345 return check_new_page(page); 2346 else 2347 return false; 2348 } 2349 2350 static inline bool check_new_pcp(struct page *page) 2351 { 2352 return check_new_page(page); 2353 } 2354 #else 2355 /* 2356 * With DEBUG_VM disabled, free order-0 pages are checked for expected state 2357 * when pcp lists are being refilled from the free lists. With debug_pagealloc 2358 * enabled, they are also checked when being allocated from the pcp lists. 2359 */ 2360 static inline bool check_pcp_refill(struct page *page) 2361 { 2362 return check_new_page(page); 2363 } 2364 static inline bool check_new_pcp(struct page *page) 2365 { 2366 if (debug_pagealloc_enabled_static()) 2367 return check_new_page(page); 2368 else 2369 return false; 2370 } 2371 #endif /* CONFIG_DEBUG_VM */ 2372 2373 static bool check_new_pages(struct page *page, unsigned int order) 2374 { 2375 int i; 2376 for (i = 0; i < (1 << order); i++) { 2377 struct page *p = page + i; 2378 2379 if (unlikely(check_new_page(p))) 2380 return true; 2381 } 2382 2383 return false; 2384 } 2385 2386 inline void post_alloc_hook(struct page *page, unsigned int order, 2387 gfp_t gfp_flags) 2388 { 2389 set_page_private(page, 0); 2390 set_page_refcounted(page); 2391 2392 arch_alloc_page(page, order); 2393 debug_pagealloc_map_pages(page, 1 << order); 2394 2395 /* 2396 * Page unpoisoning must happen before memory initialization. 2397 * Otherwise, the poison pattern will be overwritten for __GFP_ZERO 2398 * allocations and the page unpoisoning code will complain. 2399 */ 2400 kernel_unpoison_pages(page, 1 << order); 2401 2402 /* 2403 * As memory initialization might be integrated into KASAN, 2404 * kasan_alloc_pages and kernel_init_free_pages must be 2405 * kept together to avoid discrepancies in behavior. 2406 */ 2407 if (kasan_has_integrated_init()) { 2408 kasan_alloc_pages(page, order, gfp_flags); 2409 } else { 2410 bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags); 2411 2412 kasan_unpoison_pages(page, order, init); 2413 if (init) 2414 kernel_init_free_pages(page, 1 << order, 2415 gfp_flags & __GFP_ZEROTAGS); 2416 } 2417 2418 set_page_owner(page, order, gfp_flags); 2419 } 2420 2421 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 2422 unsigned int alloc_flags) 2423 { 2424 post_alloc_hook(page, order, gfp_flags); 2425 2426 if (order && (gfp_flags & __GFP_COMP)) 2427 prep_compound_page(page, order); 2428 2429 /* 2430 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 2431 * allocate the page. The expectation is that the caller is taking 2432 * steps that will free more memory. The caller should avoid the page 2433 * being used for !PFMEMALLOC purposes. 2434 */ 2435 if (alloc_flags & ALLOC_NO_WATERMARKS) 2436 set_page_pfmemalloc(page); 2437 else 2438 clear_page_pfmemalloc(page); 2439 } 2440 2441 /* 2442 * Go through the free lists for the given migratetype and remove 2443 * the smallest available page from the freelists 2444 */ 2445 static __always_inline 2446 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 2447 int migratetype) 2448 { 2449 unsigned int current_order; 2450 struct free_area *area; 2451 struct page *page; 2452 2453 /* Find a page of the appropriate size in the preferred list */ 2454 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 2455 area = &(zone->free_area[current_order]); 2456 page = get_page_from_free_area(area, migratetype); 2457 if (!page) 2458 continue; 2459 del_page_from_free_list(page, zone, current_order); 2460 expand(zone, page, order, current_order, migratetype); 2461 set_pcppage_migratetype(page, migratetype); 2462 return page; 2463 } 2464 2465 return NULL; 2466 } 2467 2468 2469 /* 2470 * This array describes the order lists are fallen back to when 2471 * the free lists for the desirable migrate type are depleted 2472 */ 2473 static int fallbacks[MIGRATE_TYPES][3] = { 2474 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2475 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 2476 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2477 #ifdef CONFIG_CMA 2478 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */ 2479 #endif 2480 #ifdef CONFIG_MEMORY_ISOLATION 2481 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */ 2482 #endif 2483 }; 2484 2485 #ifdef CONFIG_CMA 2486 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2487 unsigned int order) 2488 { 2489 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 2490 } 2491 #else 2492 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2493 unsigned int order) { return NULL; } 2494 #endif 2495 2496 /* 2497 * Move the free pages in a range to the freelist tail of the requested type. 2498 * Note that start_page and end_pages are not aligned on a pageblock 2499 * boundary. If alignment is required, use move_freepages_block() 2500 */ 2501 static int move_freepages(struct zone *zone, 2502 unsigned long start_pfn, unsigned long end_pfn, 2503 int migratetype, int *num_movable) 2504 { 2505 struct page *page; 2506 unsigned long pfn; 2507 unsigned int order; 2508 int pages_moved = 0; 2509 2510 for (pfn = start_pfn; pfn <= end_pfn;) { 2511 page = pfn_to_page(pfn); 2512 if (!PageBuddy(page)) { 2513 /* 2514 * We assume that pages that could be isolated for 2515 * migration are movable. But we don't actually try 2516 * isolating, as that would be expensive. 2517 */ 2518 if (num_movable && 2519 (PageLRU(page) || __PageMovable(page))) 2520 (*num_movable)++; 2521 pfn++; 2522 continue; 2523 } 2524 2525 /* Make sure we are not inadvertently changing nodes */ 2526 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2527 VM_BUG_ON_PAGE(page_zone(page) != zone, page); 2528 2529 order = buddy_order(page); 2530 move_to_free_list(page, zone, order, migratetype); 2531 pfn += 1 << order; 2532 pages_moved += 1 << order; 2533 } 2534 2535 return pages_moved; 2536 } 2537 2538 int move_freepages_block(struct zone *zone, struct page *page, 2539 int migratetype, int *num_movable) 2540 { 2541 unsigned long start_pfn, end_pfn, pfn; 2542 2543 if (num_movable) 2544 *num_movable = 0; 2545 2546 pfn = page_to_pfn(page); 2547 start_pfn = pfn & ~(pageblock_nr_pages - 1); 2548 end_pfn = start_pfn + pageblock_nr_pages - 1; 2549 2550 /* Do not cross zone boundaries */ 2551 if (!zone_spans_pfn(zone, start_pfn)) 2552 start_pfn = pfn; 2553 if (!zone_spans_pfn(zone, end_pfn)) 2554 return 0; 2555 2556 return move_freepages(zone, start_pfn, end_pfn, migratetype, 2557 num_movable); 2558 } 2559 2560 static void change_pageblock_range(struct page *pageblock_page, 2561 int start_order, int migratetype) 2562 { 2563 int nr_pageblocks = 1 << (start_order - pageblock_order); 2564 2565 while (nr_pageblocks--) { 2566 set_pageblock_migratetype(pageblock_page, migratetype); 2567 pageblock_page += pageblock_nr_pages; 2568 } 2569 } 2570 2571 /* 2572 * When we are falling back to another migratetype during allocation, try to 2573 * steal extra free pages from the same pageblocks to satisfy further 2574 * allocations, instead of polluting multiple pageblocks. 2575 * 2576 * If we are stealing a relatively large buddy page, it is likely there will 2577 * be more free pages in the pageblock, so try to steal them all. For 2578 * reclaimable and unmovable allocations, we steal regardless of page size, 2579 * as fragmentation caused by those allocations polluting movable pageblocks 2580 * is worse than movable allocations stealing from unmovable and reclaimable 2581 * pageblocks. 2582 */ 2583 static bool can_steal_fallback(unsigned int order, int start_mt) 2584 { 2585 /* 2586 * Leaving this order check is intended, although there is 2587 * relaxed order check in next check. The reason is that 2588 * we can actually steal whole pageblock if this condition met, 2589 * but, below check doesn't guarantee it and that is just heuristic 2590 * so could be changed anytime. 2591 */ 2592 if (order >= pageblock_order) 2593 return true; 2594 2595 if (order >= pageblock_order / 2 || 2596 start_mt == MIGRATE_RECLAIMABLE || 2597 start_mt == MIGRATE_UNMOVABLE || 2598 page_group_by_mobility_disabled) 2599 return true; 2600 2601 return false; 2602 } 2603 2604 static inline bool boost_watermark(struct zone *zone) 2605 { 2606 unsigned long max_boost; 2607 2608 if (!watermark_boost_factor) 2609 return false; 2610 /* 2611 * Don't bother in zones that are unlikely to produce results. 2612 * On small machines, including kdump capture kernels running 2613 * in a small area, boosting the watermark can cause an out of 2614 * memory situation immediately. 2615 */ 2616 if ((pageblock_nr_pages * 4) > zone_managed_pages(zone)) 2617 return false; 2618 2619 max_boost = mult_frac(zone->_watermark[WMARK_HIGH], 2620 watermark_boost_factor, 10000); 2621 2622 /* 2623 * high watermark may be uninitialised if fragmentation occurs 2624 * very early in boot so do not boost. We do not fall 2625 * through and boost by pageblock_nr_pages as failing 2626 * allocations that early means that reclaim is not going 2627 * to help and it may even be impossible to reclaim the 2628 * boosted watermark resulting in a hang. 2629 */ 2630 if (!max_boost) 2631 return false; 2632 2633 max_boost = max(pageblock_nr_pages, max_boost); 2634 2635 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages, 2636 max_boost); 2637 2638 return true; 2639 } 2640 2641 /* 2642 * This function implements actual steal behaviour. If order is large enough, 2643 * we can steal whole pageblock. If not, we first move freepages in this 2644 * pageblock to our migratetype and determine how many already-allocated pages 2645 * are there in the pageblock with a compatible migratetype. If at least half 2646 * of pages are free or compatible, we can change migratetype of the pageblock 2647 * itself, so pages freed in the future will be put on the correct free list. 2648 */ 2649 static void steal_suitable_fallback(struct zone *zone, struct page *page, 2650 unsigned int alloc_flags, int start_type, bool whole_block) 2651 { 2652 unsigned int current_order = buddy_order(page); 2653 int free_pages, movable_pages, alike_pages; 2654 int old_block_type; 2655 2656 old_block_type = get_pageblock_migratetype(page); 2657 2658 /* 2659 * This can happen due to races and we want to prevent broken 2660 * highatomic accounting. 2661 */ 2662 if (is_migrate_highatomic(old_block_type)) 2663 goto single_page; 2664 2665 /* Take ownership for orders >= pageblock_order */ 2666 if (current_order >= pageblock_order) { 2667 change_pageblock_range(page, current_order, start_type); 2668 goto single_page; 2669 } 2670 2671 /* 2672 * Boost watermarks to increase reclaim pressure to reduce the 2673 * likelihood of future fallbacks. Wake kswapd now as the node 2674 * may be balanced overall and kswapd will not wake naturally. 2675 */ 2676 if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD)) 2677 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 2678 2679 /* We are not allowed to try stealing from the whole block */ 2680 if (!whole_block) 2681 goto single_page; 2682 2683 free_pages = move_freepages_block(zone, page, start_type, 2684 &movable_pages); 2685 /* 2686 * Determine how many pages are compatible with our allocation. 2687 * For movable allocation, it's the number of movable pages which 2688 * we just obtained. For other types it's a bit more tricky. 2689 */ 2690 if (start_type == MIGRATE_MOVABLE) { 2691 alike_pages = movable_pages; 2692 } else { 2693 /* 2694 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation 2695 * to MOVABLE pageblock, consider all non-movable pages as 2696 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or 2697 * vice versa, be conservative since we can't distinguish the 2698 * exact migratetype of non-movable pages. 2699 */ 2700 if (old_block_type == MIGRATE_MOVABLE) 2701 alike_pages = pageblock_nr_pages 2702 - (free_pages + movable_pages); 2703 else 2704 alike_pages = 0; 2705 } 2706 2707 /* moving whole block can fail due to zone boundary conditions */ 2708 if (!free_pages) 2709 goto single_page; 2710 2711 /* 2712 * If a sufficient number of pages in the block are either free or of 2713 * comparable migratability as our allocation, claim the whole block. 2714 */ 2715 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || 2716 page_group_by_mobility_disabled) 2717 set_pageblock_migratetype(page, start_type); 2718 2719 return; 2720 2721 single_page: 2722 move_to_free_list(page, zone, current_order, start_type); 2723 } 2724 2725 /* 2726 * Check whether there is a suitable fallback freepage with requested order. 2727 * If only_stealable is true, this function returns fallback_mt only if 2728 * we can steal other freepages all together. This would help to reduce 2729 * fragmentation due to mixed migratetype pages in one pageblock. 2730 */ 2731 int find_suitable_fallback(struct free_area *area, unsigned int order, 2732 int migratetype, bool only_stealable, bool *can_steal) 2733 { 2734 int i; 2735 int fallback_mt; 2736 2737 if (area->nr_free == 0) 2738 return -1; 2739 2740 *can_steal = false; 2741 for (i = 0;; i++) { 2742 fallback_mt = fallbacks[migratetype][i]; 2743 if (fallback_mt == MIGRATE_TYPES) 2744 break; 2745 2746 if (free_area_empty(area, fallback_mt)) 2747 continue; 2748 2749 if (can_steal_fallback(order, migratetype)) 2750 *can_steal = true; 2751 2752 if (!only_stealable) 2753 return fallback_mt; 2754 2755 if (*can_steal) 2756 return fallback_mt; 2757 } 2758 2759 return -1; 2760 } 2761 2762 /* 2763 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2764 * there are no empty page blocks that contain a page with a suitable order 2765 */ 2766 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2767 unsigned int alloc_order) 2768 { 2769 int mt; 2770 unsigned long max_managed, flags; 2771 2772 /* 2773 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2774 * Check is race-prone but harmless. 2775 */ 2776 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages; 2777 if (zone->nr_reserved_highatomic >= max_managed) 2778 return; 2779 2780 spin_lock_irqsave(&zone->lock, flags); 2781 2782 /* Recheck the nr_reserved_highatomic limit under the lock */ 2783 if (zone->nr_reserved_highatomic >= max_managed) 2784 goto out_unlock; 2785 2786 /* Yoink! */ 2787 mt = get_pageblock_migratetype(page); 2788 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt) 2789 && !is_migrate_cma(mt)) { 2790 zone->nr_reserved_highatomic += pageblock_nr_pages; 2791 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2792 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); 2793 } 2794 2795 out_unlock: 2796 spin_unlock_irqrestore(&zone->lock, flags); 2797 } 2798 2799 /* 2800 * Used when an allocation is about to fail under memory pressure. This 2801 * potentially hurts the reliability of high-order allocations when under 2802 * intense memory pressure but failed atomic allocations should be easier 2803 * to recover from than an OOM. 2804 * 2805 * If @force is true, try to unreserve a pageblock even though highatomic 2806 * pageblock is exhausted. 2807 */ 2808 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 2809 bool force) 2810 { 2811 struct zonelist *zonelist = ac->zonelist; 2812 unsigned long flags; 2813 struct zoneref *z; 2814 struct zone *zone; 2815 struct page *page; 2816 int order; 2817 bool ret; 2818 2819 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx, 2820 ac->nodemask) { 2821 /* 2822 * Preserve at least one pageblock unless memory pressure 2823 * is really high. 2824 */ 2825 if (!force && zone->nr_reserved_highatomic <= 2826 pageblock_nr_pages) 2827 continue; 2828 2829 spin_lock_irqsave(&zone->lock, flags); 2830 for (order = 0; order < MAX_ORDER; order++) { 2831 struct free_area *area = &(zone->free_area[order]); 2832 2833 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); 2834 if (!page) 2835 continue; 2836 2837 /* 2838 * In page freeing path, migratetype change is racy so 2839 * we can counter several free pages in a pageblock 2840 * in this loop although we changed the pageblock type 2841 * from highatomic to ac->migratetype. So we should 2842 * adjust the count once. 2843 */ 2844 if (is_migrate_highatomic_page(page)) { 2845 /* 2846 * It should never happen but changes to 2847 * locking could inadvertently allow a per-cpu 2848 * drain to add pages to MIGRATE_HIGHATOMIC 2849 * while unreserving so be safe and watch for 2850 * underflows. 2851 */ 2852 zone->nr_reserved_highatomic -= min( 2853 pageblock_nr_pages, 2854 zone->nr_reserved_highatomic); 2855 } 2856 2857 /* 2858 * Convert to ac->migratetype and avoid the normal 2859 * pageblock stealing heuristics. Minimally, the caller 2860 * is doing the work and needs the pages. More 2861 * importantly, if the block was always converted to 2862 * MIGRATE_UNMOVABLE or another type then the number 2863 * of pageblocks that cannot be completely freed 2864 * may increase. 2865 */ 2866 set_pageblock_migratetype(page, ac->migratetype); 2867 ret = move_freepages_block(zone, page, ac->migratetype, 2868 NULL); 2869 if (ret) { 2870 spin_unlock_irqrestore(&zone->lock, flags); 2871 return ret; 2872 } 2873 } 2874 spin_unlock_irqrestore(&zone->lock, flags); 2875 } 2876 2877 return false; 2878 } 2879 2880 /* 2881 * Try finding a free buddy page on the fallback list and put it on the free 2882 * list of requested migratetype, possibly along with other pages from the same 2883 * block, depending on fragmentation avoidance heuristics. Returns true if 2884 * fallback was found so that __rmqueue_smallest() can grab it. 2885 * 2886 * The use of signed ints for order and current_order is a deliberate 2887 * deviation from the rest of this file, to make the for loop 2888 * condition simpler. 2889 */ 2890 static __always_inline bool 2891 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, 2892 unsigned int alloc_flags) 2893 { 2894 struct free_area *area; 2895 int current_order; 2896 int min_order = order; 2897 struct page *page; 2898 int fallback_mt; 2899 bool can_steal; 2900 2901 /* 2902 * Do not steal pages from freelists belonging to other pageblocks 2903 * i.e. orders < pageblock_order. If there are no local zones free, 2904 * the zonelists will be reiterated without ALLOC_NOFRAGMENT. 2905 */ 2906 if (alloc_flags & ALLOC_NOFRAGMENT) 2907 min_order = pageblock_order; 2908 2909 /* 2910 * Find the largest available free page in the other list. This roughly 2911 * approximates finding the pageblock with the most free pages, which 2912 * would be too costly to do exactly. 2913 */ 2914 for (current_order = MAX_ORDER - 1; current_order >= min_order; 2915 --current_order) { 2916 area = &(zone->free_area[current_order]); 2917 fallback_mt = find_suitable_fallback(area, current_order, 2918 start_migratetype, false, &can_steal); 2919 if (fallback_mt == -1) 2920 continue; 2921 2922 /* 2923 * We cannot steal all free pages from the pageblock and the 2924 * requested migratetype is movable. In that case it's better to 2925 * steal and split the smallest available page instead of the 2926 * largest available page, because even if the next movable 2927 * allocation falls back into a different pageblock than this 2928 * one, it won't cause permanent fragmentation. 2929 */ 2930 if (!can_steal && start_migratetype == MIGRATE_MOVABLE 2931 && current_order > order) 2932 goto find_smallest; 2933 2934 goto do_steal; 2935 } 2936 2937 return false; 2938 2939 find_smallest: 2940 for (current_order = order; current_order < MAX_ORDER; 2941 current_order++) { 2942 area = &(zone->free_area[current_order]); 2943 fallback_mt = find_suitable_fallback(area, current_order, 2944 start_migratetype, false, &can_steal); 2945 if (fallback_mt != -1) 2946 break; 2947 } 2948 2949 /* 2950 * This should not happen - we already found a suitable fallback 2951 * when looking for the largest page. 2952 */ 2953 VM_BUG_ON(current_order == MAX_ORDER); 2954 2955 do_steal: 2956 page = get_page_from_free_area(area, fallback_mt); 2957 2958 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype, 2959 can_steal); 2960 2961 trace_mm_page_alloc_extfrag(page, order, current_order, 2962 start_migratetype, fallback_mt); 2963 2964 return true; 2965 2966 } 2967 2968 /* 2969 * Do the hard work of removing an element from the buddy allocator. 2970 * Call me with the zone->lock already held. 2971 */ 2972 static __always_inline struct page * 2973 __rmqueue(struct zone *zone, unsigned int order, int migratetype, 2974 unsigned int alloc_flags) 2975 { 2976 struct page *page; 2977 2978 if (IS_ENABLED(CONFIG_CMA)) { 2979 /* 2980 * Balance movable allocations between regular and CMA areas by 2981 * allocating from CMA when over half of the zone's free memory 2982 * is in the CMA area. 2983 */ 2984 if (alloc_flags & ALLOC_CMA && 2985 zone_page_state(zone, NR_FREE_CMA_PAGES) > 2986 zone_page_state(zone, NR_FREE_PAGES) / 2) { 2987 page = __rmqueue_cma_fallback(zone, order); 2988 if (page) 2989 goto out; 2990 } 2991 } 2992 retry: 2993 page = __rmqueue_smallest(zone, order, migratetype); 2994 if (unlikely(!page)) { 2995 if (alloc_flags & ALLOC_CMA) 2996 page = __rmqueue_cma_fallback(zone, order); 2997 2998 if (!page && __rmqueue_fallback(zone, order, migratetype, 2999 alloc_flags)) 3000 goto retry; 3001 } 3002 out: 3003 if (page) 3004 trace_mm_page_alloc_zone_locked(page, order, migratetype); 3005 return page; 3006 } 3007 3008 /* 3009 * Obtain a specified number of elements from the buddy allocator, all under 3010 * a single hold of the lock, for efficiency. Add them to the supplied list. 3011 * Returns the number of new pages which were placed at *list. 3012 */ 3013 static int rmqueue_bulk(struct zone *zone, unsigned int order, 3014 unsigned long count, struct list_head *list, 3015 int migratetype, unsigned int alloc_flags) 3016 { 3017 int i, allocated = 0; 3018 3019 /* 3020 * local_lock_irq held so equivalent to spin_lock_irqsave for 3021 * both PREEMPT_RT and non-PREEMPT_RT configurations. 3022 */ 3023 spin_lock(&zone->lock); 3024 for (i = 0; i < count; ++i) { 3025 struct page *page = __rmqueue(zone, order, migratetype, 3026 alloc_flags); 3027 if (unlikely(page == NULL)) 3028 break; 3029 3030 if (unlikely(check_pcp_refill(page))) 3031 continue; 3032 3033 /* 3034 * Split buddy pages returned by expand() are received here in 3035 * physical page order. The page is added to the tail of 3036 * caller's list. From the callers perspective, the linked list 3037 * is ordered by page number under some conditions. This is 3038 * useful for IO devices that can forward direction from the 3039 * head, thus also in the physical page order. This is useful 3040 * for IO devices that can merge IO requests if the physical 3041 * pages are ordered properly. 3042 */ 3043 list_add_tail(&page->lru, list); 3044 allocated++; 3045 if (is_migrate_cma(get_pcppage_migratetype(page))) 3046 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 3047 -(1 << order)); 3048 } 3049 3050 /* 3051 * i pages were removed from the buddy list even if some leak due 3052 * to check_pcp_refill failing so adjust NR_FREE_PAGES based 3053 * on i. Do not confuse with 'allocated' which is the number of 3054 * pages added to the pcp list. 3055 */ 3056 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 3057 spin_unlock(&zone->lock); 3058 return allocated; 3059 } 3060 3061 #ifdef CONFIG_NUMA 3062 /* 3063 * Called from the vmstat counter updater to drain pagesets of this 3064 * currently executing processor on remote nodes after they have 3065 * expired. 3066 * 3067 * Note that this function must be called with the thread pinned to 3068 * a single processor. 3069 */ 3070 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 3071 { 3072 unsigned long flags; 3073 int to_drain, batch; 3074 3075 local_lock_irqsave(&pagesets.lock, flags); 3076 batch = READ_ONCE(pcp->batch); 3077 to_drain = min(pcp->count, batch); 3078 if (to_drain > 0) 3079 free_pcppages_bulk(zone, to_drain, pcp); 3080 local_unlock_irqrestore(&pagesets.lock, flags); 3081 } 3082 #endif 3083 3084 /* 3085 * Drain pcplists of the indicated processor and zone. 3086 * 3087 * The processor must either be the current processor and the 3088 * thread pinned to the current processor or a processor that 3089 * is not online. 3090 */ 3091 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 3092 { 3093 unsigned long flags; 3094 struct per_cpu_pages *pcp; 3095 3096 local_lock_irqsave(&pagesets.lock, flags); 3097 3098 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 3099 if (pcp->count) 3100 free_pcppages_bulk(zone, pcp->count, pcp); 3101 3102 local_unlock_irqrestore(&pagesets.lock, flags); 3103 } 3104 3105 /* 3106 * Drain pcplists of all zones on the indicated processor. 3107 * 3108 * The processor must either be the current processor and the 3109 * thread pinned to the current processor or a processor that 3110 * is not online. 3111 */ 3112 static void drain_pages(unsigned int cpu) 3113 { 3114 struct zone *zone; 3115 3116 for_each_populated_zone(zone) { 3117 drain_pages_zone(cpu, zone); 3118 } 3119 } 3120 3121 /* 3122 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 3123 * 3124 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 3125 * the single zone's pages. 3126 */ 3127 void drain_local_pages(struct zone *zone) 3128 { 3129 int cpu = smp_processor_id(); 3130 3131 if (zone) 3132 drain_pages_zone(cpu, zone); 3133 else 3134 drain_pages(cpu); 3135 } 3136 3137 static void drain_local_pages_wq(struct work_struct *work) 3138 { 3139 struct pcpu_drain *drain; 3140 3141 drain = container_of(work, struct pcpu_drain, work); 3142 3143 /* 3144 * drain_all_pages doesn't use proper cpu hotplug protection so 3145 * we can race with cpu offline when the WQ can move this from 3146 * a cpu pinned worker to an unbound one. We can operate on a different 3147 * cpu which is alright but we also have to make sure to not move to 3148 * a different one. 3149 */ 3150 preempt_disable(); 3151 drain_local_pages(drain->zone); 3152 preempt_enable(); 3153 } 3154 3155 /* 3156 * The implementation of drain_all_pages(), exposing an extra parameter to 3157 * drain on all cpus. 3158 * 3159 * drain_all_pages() is optimized to only execute on cpus where pcplists are 3160 * not empty. The check for non-emptiness can however race with a free to 3161 * pcplist that has not yet increased the pcp->count from 0 to 1. Callers 3162 * that need the guarantee that every CPU has drained can disable the 3163 * optimizing racy check. 3164 */ 3165 static void __drain_all_pages(struct zone *zone, bool force_all_cpus) 3166 { 3167 int cpu; 3168 3169 /* 3170 * Allocate in the BSS so we won't require allocation in 3171 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 3172 */ 3173 static cpumask_t cpus_with_pcps; 3174 3175 /* 3176 * Make sure nobody triggers this path before mm_percpu_wq is fully 3177 * initialized. 3178 */ 3179 if (WARN_ON_ONCE(!mm_percpu_wq)) 3180 return; 3181 3182 /* 3183 * Do not drain if one is already in progress unless it's specific to 3184 * a zone. Such callers are primarily CMA and memory hotplug and need 3185 * the drain to be complete when the call returns. 3186 */ 3187 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) { 3188 if (!zone) 3189 return; 3190 mutex_lock(&pcpu_drain_mutex); 3191 } 3192 3193 /* 3194 * We don't care about racing with CPU hotplug event 3195 * as offline notification will cause the notified 3196 * cpu to drain that CPU pcps and on_each_cpu_mask 3197 * disables preemption as part of its processing 3198 */ 3199 for_each_online_cpu(cpu) { 3200 struct per_cpu_pages *pcp; 3201 struct zone *z; 3202 bool has_pcps = false; 3203 3204 if (force_all_cpus) { 3205 /* 3206 * The pcp.count check is racy, some callers need a 3207 * guarantee that no cpu is missed. 3208 */ 3209 has_pcps = true; 3210 } else if (zone) { 3211 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 3212 if (pcp->count) 3213 has_pcps = true; 3214 } else { 3215 for_each_populated_zone(z) { 3216 pcp = per_cpu_ptr(z->per_cpu_pageset, cpu); 3217 if (pcp->count) { 3218 has_pcps = true; 3219 break; 3220 } 3221 } 3222 } 3223 3224 if (has_pcps) 3225 cpumask_set_cpu(cpu, &cpus_with_pcps); 3226 else 3227 cpumask_clear_cpu(cpu, &cpus_with_pcps); 3228 } 3229 3230 for_each_cpu(cpu, &cpus_with_pcps) { 3231 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu); 3232 3233 drain->zone = zone; 3234 INIT_WORK(&drain->work, drain_local_pages_wq); 3235 queue_work_on(cpu, mm_percpu_wq, &drain->work); 3236 } 3237 for_each_cpu(cpu, &cpus_with_pcps) 3238 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work); 3239 3240 mutex_unlock(&pcpu_drain_mutex); 3241 } 3242 3243 /* 3244 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 3245 * 3246 * When zone parameter is non-NULL, spill just the single zone's pages. 3247 * 3248 * Note that this can be extremely slow as the draining happens in a workqueue. 3249 */ 3250 void drain_all_pages(struct zone *zone) 3251 { 3252 __drain_all_pages(zone, false); 3253 } 3254 3255 #ifdef CONFIG_HIBERNATION 3256 3257 /* 3258 * Touch the watchdog for every WD_PAGE_COUNT pages. 3259 */ 3260 #define WD_PAGE_COUNT (128*1024) 3261 3262 void mark_free_pages(struct zone *zone) 3263 { 3264 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT; 3265 unsigned long flags; 3266 unsigned int order, t; 3267 struct page *page; 3268 3269 if (zone_is_empty(zone)) 3270 return; 3271 3272 spin_lock_irqsave(&zone->lock, flags); 3273 3274 max_zone_pfn = zone_end_pfn(zone); 3275 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 3276 if (pfn_valid(pfn)) { 3277 page = pfn_to_page(pfn); 3278 3279 if (!--page_count) { 3280 touch_nmi_watchdog(); 3281 page_count = WD_PAGE_COUNT; 3282 } 3283 3284 if (page_zone(page) != zone) 3285 continue; 3286 3287 if (!swsusp_page_is_forbidden(page)) 3288 swsusp_unset_page_free(page); 3289 } 3290 3291 for_each_migratetype_order(order, t) { 3292 list_for_each_entry(page, 3293 &zone->free_area[order].free_list[t], lru) { 3294 unsigned long i; 3295 3296 pfn = page_to_pfn(page); 3297 for (i = 0; i < (1UL << order); i++) { 3298 if (!--page_count) { 3299 touch_nmi_watchdog(); 3300 page_count = WD_PAGE_COUNT; 3301 } 3302 swsusp_set_page_free(pfn_to_page(pfn + i)); 3303 } 3304 } 3305 } 3306 spin_unlock_irqrestore(&zone->lock, flags); 3307 } 3308 #endif /* CONFIG_PM */ 3309 3310 static bool free_unref_page_prepare(struct page *page, unsigned long pfn, 3311 unsigned int order) 3312 { 3313 int migratetype; 3314 3315 if (!free_pcp_prepare(page, order)) 3316 return false; 3317 3318 migratetype = get_pfnblock_migratetype(page, pfn); 3319 set_pcppage_migratetype(page, migratetype); 3320 return true; 3321 } 3322 3323 static int nr_pcp_free(struct per_cpu_pages *pcp, int high, int batch) 3324 { 3325 int min_nr_free, max_nr_free; 3326 3327 /* Check for PCP disabled or boot pageset */ 3328 if (unlikely(high < batch)) 3329 return 1; 3330 3331 /* Leave at least pcp->batch pages on the list */ 3332 min_nr_free = batch; 3333 max_nr_free = high - batch; 3334 3335 /* 3336 * Double the number of pages freed each time there is subsequent 3337 * freeing of pages without any allocation. 3338 */ 3339 batch <<= pcp->free_factor; 3340 if (batch < max_nr_free) 3341 pcp->free_factor++; 3342 batch = clamp(batch, min_nr_free, max_nr_free); 3343 3344 return batch; 3345 } 3346 3347 static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone) 3348 { 3349 int high = READ_ONCE(pcp->high); 3350 3351 if (unlikely(!high)) 3352 return 0; 3353 3354 if (!test_bit(ZONE_RECLAIM_ACTIVE, &zone->flags)) 3355 return high; 3356 3357 /* 3358 * If reclaim is active, limit the number of pages that can be 3359 * stored on pcp lists 3360 */ 3361 return min(READ_ONCE(pcp->batch) << 2, high); 3362 } 3363 3364 static void free_unref_page_commit(struct page *page, unsigned long pfn, 3365 int migratetype, unsigned int order) 3366 { 3367 struct zone *zone = page_zone(page); 3368 struct per_cpu_pages *pcp; 3369 int high; 3370 int pindex; 3371 3372 __count_vm_event(PGFREE); 3373 pcp = this_cpu_ptr(zone->per_cpu_pageset); 3374 pindex = order_to_pindex(migratetype, order); 3375 list_add(&page->lru, &pcp->lists[pindex]); 3376 pcp->count += 1 << order; 3377 high = nr_pcp_high(pcp, zone); 3378 if (pcp->count >= high) { 3379 int batch = READ_ONCE(pcp->batch); 3380 3381 free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch), pcp); 3382 } 3383 } 3384 3385 /* 3386 * Free a pcp page 3387 */ 3388 void free_unref_page(struct page *page, unsigned int order) 3389 { 3390 unsigned long flags; 3391 unsigned long pfn = page_to_pfn(page); 3392 int migratetype; 3393 3394 if (!free_unref_page_prepare(page, pfn, order)) 3395 return; 3396 3397 /* 3398 * We only track unmovable, reclaimable and movable on pcp lists. 3399 * Place ISOLATE pages on the isolated list because they are being 3400 * offlined but treat HIGHATOMIC as movable pages so we can get those 3401 * areas back if necessary. Otherwise, we may have to free 3402 * excessively into the page allocator 3403 */ 3404 migratetype = get_pcppage_migratetype(page); 3405 if (unlikely(migratetype >= MIGRATE_PCPTYPES)) { 3406 if (unlikely(is_migrate_isolate(migratetype))) { 3407 free_one_page(page_zone(page), page, pfn, order, migratetype, FPI_NONE); 3408 return; 3409 } 3410 migratetype = MIGRATE_MOVABLE; 3411 } 3412 3413 local_lock_irqsave(&pagesets.lock, flags); 3414 free_unref_page_commit(page, pfn, migratetype, order); 3415 local_unlock_irqrestore(&pagesets.lock, flags); 3416 } 3417 3418 /* 3419 * Free a list of 0-order pages 3420 */ 3421 void free_unref_page_list(struct list_head *list) 3422 { 3423 struct page *page, *next; 3424 unsigned long flags, pfn; 3425 int batch_count = 0; 3426 int migratetype; 3427 3428 /* Prepare pages for freeing */ 3429 list_for_each_entry_safe(page, next, list, lru) { 3430 pfn = page_to_pfn(page); 3431 if (!free_unref_page_prepare(page, pfn, 0)) 3432 list_del(&page->lru); 3433 3434 /* 3435 * Free isolated pages directly to the allocator, see 3436 * comment in free_unref_page. 3437 */ 3438 migratetype = get_pcppage_migratetype(page); 3439 if (unlikely(is_migrate_isolate(migratetype))) { 3440 list_del(&page->lru); 3441 free_one_page(page_zone(page), page, pfn, 0, migratetype, FPI_NONE); 3442 continue; 3443 } 3444 3445 set_page_private(page, pfn); 3446 } 3447 3448 local_lock_irqsave(&pagesets.lock, flags); 3449 list_for_each_entry_safe(page, next, list, lru) { 3450 pfn = page_private(page); 3451 set_page_private(page, 0); 3452 3453 /* 3454 * Non-isolated types over MIGRATE_PCPTYPES get added 3455 * to the MIGRATE_MOVABLE pcp list. 3456 */ 3457 migratetype = get_pcppage_migratetype(page); 3458 if (unlikely(migratetype >= MIGRATE_PCPTYPES)) 3459 migratetype = MIGRATE_MOVABLE; 3460 3461 trace_mm_page_free_batched(page); 3462 free_unref_page_commit(page, pfn, migratetype, 0); 3463 3464 /* 3465 * Guard against excessive IRQ disabled times when we get 3466 * a large list of pages to free. 3467 */ 3468 if (++batch_count == SWAP_CLUSTER_MAX) { 3469 local_unlock_irqrestore(&pagesets.lock, flags); 3470 batch_count = 0; 3471 local_lock_irqsave(&pagesets.lock, flags); 3472 } 3473 } 3474 local_unlock_irqrestore(&pagesets.lock, flags); 3475 } 3476 3477 /* 3478 * split_page takes a non-compound higher-order page, and splits it into 3479 * n (1<<order) sub-pages: page[0..n] 3480 * Each sub-page must be freed individually. 3481 * 3482 * Note: this is probably too low level an operation for use in drivers. 3483 * Please consult with lkml before using this in your driver. 3484 */ 3485 void split_page(struct page *page, unsigned int order) 3486 { 3487 int i; 3488 3489 VM_BUG_ON_PAGE(PageCompound(page), page); 3490 VM_BUG_ON_PAGE(!page_count(page), page); 3491 3492 for (i = 1; i < (1 << order); i++) 3493 set_page_refcounted(page + i); 3494 split_page_owner(page, 1 << order); 3495 split_page_memcg(page, 1 << order); 3496 } 3497 EXPORT_SYMBOL_GPL(split_page); 3498 3499 int __isolate_free_page(struct page *page, unsigned int order) 3500 { 3501 unsigned long watermark; 3502 struct zone *zone; 3503 int mt; 3504 3505 BUG_ON(!PageBuddy(page)); 3506 3507 zone = page_zone(page); 3508 mt = get_pageblock_migratetype(page); 3509 3510 if (!is_migrate_isolate(mt)) { 3511 /* 3512 * Obey watermarks as if the page was being allocated. We can 3513 * emulate a high-order watermark check with a raised order-0 3514 * watermark, because we already know our high-order page 3515 * exists. 3516 */ 3517 watermark = zone->_watermark[WMARK_MIN] + (1UL << order); 3518 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 3519 return 0; 3520 3521 __mod_zone_freepage_state(zone, -(1UL << order), mt); 3522 } 3523 3524 /* Remove page from free list */ 3525 3526 del_page_from_free_list(page, zone, order); 3527 3528 /* 3529 * Set the pageblock if the isolated page is at least half of a 3530 * pageblock 3531 */ 3532 if (order >= pageblock_order - 1) { 3533 struct page *endpage = page + (1 << order) - 1; 3534 for (; page < endpage; page += pageblock_nr_pages) { 3535 int mt = get_pageblock_migratetype(page); 3536 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt) 3537 && !is_migrate_highatomic(mt)) 3538 set_pageblock_migratetype(page, 3539 MIGRATE_MOVABLE); 3540 } 3541 } 3542 3543 3544 return 1UL << order; 3545 } 3546 3547 /** 3548 * __putback_isolated_page - Return a now-isolated page back where we got it 3549 * @page: Page that was isolated 3550 * @order: Order of the isolated page 3551 * @mt: The page's pageblock's migratetype 3552 * 3553 * This function is meant to return a page pulled from the free lists via 3554 * __isolate_free_page back to the free lists they were pulled from. 3555 */ 3556 void __putback_isolated_page(struct page *page, unsigned int order, int mt) 3557 { 3558 struct zone *zone = page_zone(page); 3559 3560 /* zone lock should be held when this function is called */ 3561 lockdep_assert_held(&zone->lock); 3562 3563 /* Return isolated page to tail of freelist. */ 3564 __free_one_page(page, page_to_pfn(page), zone, order, mt, 3565 FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL); 3566 } 3567 3568 /* 3569 * Update NUMA hit/miss statistics 3570 * 3571 * Must be called with interrupts disabled. 3572 */ 3573 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z, 3574 long nr_account) 3575 { 3576 #ifdef CONFIG_NUMA 3577 enum numa_stat_item local_stat = NUMA_LOCAL; 3578 3579 /* skip numa counters update if numa stats is disabled */ 3580 if (!static_branch_likely(&vm_numa_stat_key)) 3581 return; 3582 3583 if (zone_to_nid(z) != numa_node_id()) 3584 local_stat = NUMA_OTHER; 3585 3586 if (zone_to_nid(z) == zone_to_nid(preferred_zone)) 3587 __count_numa_events(z, NUMA_HIT, nr_account); 3588 else { 3589 __count_numa_events(z, NUMA_MISS, nr_account); 3590 __count_numa_events(preferred_zone, NUMA_FOREIGN, nr_account); 3591 } 3592 __count_numa_events(z, local_stat, nr_account); 3593 #endif 3594 } 3595 3596 /* Remove page from the per-cpu list, caller must protect the list */ 3597 static inline 3598 struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, 3599 int migratetype, 3600 unsigned int alloc_flags, 3601 struct per_cpu_pages *pcp, 3602 struct list_head *list) 3603 { 3604 struct page *page; 3605 3606 do { 3607 if (list_empty(list)) { 3608 int batch = READ_ONCE(pcp->batch); 3609 int alloced; 3610 3611 /* 3612 * Scale batch relative to order if batch implies 3613 * free pages can be stored on the PCP. Batch can 3614 * be 1 for small zones or for boot pagesets which 3615 * should never store free pages as the pages may 3616 * belong to arbitrary zones. 3617 */ 3618 if (batch > 1) 3619 batch = max(batch >> order, 2); 3620 alloced = rmqueue_bulk(zone, order, 3621 batch, list, 3622 migratetype, alloc_flags); 3623 3624 pcp->count += alloced << order; 3625 if (unlikely(list_empty(list))) 3626 return NULL; 3627 } 3628 3629 page = list_first_entry(list, struct page, lru); 3630 list_del(&page->lru); 3631 pcp->count -= 1 << order; 3632 } while (check_new_pcp(page)); 3633 3634 return page; 3635 } 3636 3637 /* Lock and remove page from the per-cpu list */ 3638 static struct page *rmqueue_pcplist(struct zone *preferred_zone, 3639 struct zone *zone, unsigned int order, 3640 gfp_t gfp_flags, int migratetype, 3641 unsigned int alloc_flags) 3642 { 3643 struct per_cpu_pages *pcp; 3644 struct list_head *list; 3645 struct page *page; 3646 unsigned long flags; 3647 3648 local_lock_irqsave(&pagesets.lock, flags); 3649 3650 /* 3651 * On allocation, reduce the number of pages that are batch freed. 3652 * See nr_pcp_free() where free_factor is increased for subsequent 3653 * frees. 3654 */ 3655 pcp = this_cpu_ptr(zone->per_cpu_pageset); 3656 pcp->free_factor >>= 1; 3657 list = &pcp->lists[order_to_pindex(migratetype, order)]; 3658 page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list); 3659 local_unlock_irqrestore(&pagesets.lock, flags); 3660 if (page) { 3661 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); 3662 zone_statistics(preferred_zone, zone, 1); 3663 } 3664 return page; 3665 } 3666 3667 /* 3668 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 3669 */ 3670 static inline 3671 struct page *rmqueue(struct zone *preferred_zone, 3672 struct zone *zone, unsigned int order, 3673 gfp_t gfp_flags, unsigned int alloc_flags, 3674 int migratetype) 3675 { 3676 unsigned long flags; 3677 struct page *page; 3678 3679 if (likely(pcp_allowed_order(order))) { 3680 /* 3681 * MIGRATE_MOVABLE pcplist could have the pages on CMA area and 3682 * we need to skip it when CMA area isn't allowed. 3683 */ 3684 if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA || 3685 migratetype != MIGRATE_MOVABLE) { 3686 page = rmqueue_pcplist(preferred_zone, zone, order, 3687 gfp_flags, migratetype, alloc_flags); 3688 goto out; 3689 } 3690 } 3691 3692 /* 3693 * We most definitely don't want callers attempting to 3694 * allocate greater than order-1 page units with __GFP_NOFAIL. 3695 */ 3696 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 3697 spin_lock_irqsave(&zone->lock, flags); 3698 3699 do { 3700 page = NULL; 3701 /* 3702 * order-0 request can reach here when the pcplist is skipped 3703 * due to non-CMA allocation context. HIGHATOMIC area is 3704 * reserved for high-order atomic allocation, so order-0 3705 * request should skip it. 3706 */ 3707 if (order > 0 && alloc_flags & ALLOC_HARDER) { 3708 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3709 if (page) 3710 trace_mm_page_alloc_zone_locked(page, order, migratetype); 3711 } 3712 if (!page) 3713 page = __rmqueue(zone, order, migratetype, alloc_flags); 3714 } while (page && check_new_pages(page, order)); 3715 if (!page) 3716 goto failed; 3717 3718 __mod_zone_freepage_state(zone, -(1 << order), 3719 get_pcppage_migratetype(page)); 3720 spin_unlock_irqrestore(&zone->lock, flags); 3721 3722 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3723 zone_statistics(preferred_zone, zone, 1); 3724 3725 out: 3726 /* Separate test+clear to avoid unnecessary atomics */ 3727 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) { 3728 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 3729 wakeup_kswapd(zone, 0, 0, zone_idx(zone)); 3730 } 3731 3732 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); 3733 return page; 3734 3735 failed: 3736 spin_unlock_irqrestore(&zone->lock, flags); 3737 return NULL; 3738 } 3739 3740 #ifdef CONFIG_FAIL_PAGE_ALLOC 3741 3742 static struct { 3743 struct fault_attr attr; 3744 3745 bool ignore_gfp_highmem; 3746 bool ignore_gfp_reclaim; 3747 u32 min_order; 3748 } fail_page_alloc = { 3749 .attr = FAULT_ATTR_INITIALIZER, 3750 .ignore_gfp_reclaim = true, 3751 .ignore_gfp_highmem = true, 3752 .min_order = 1, 3753 }; 3754 3755 static int __init setup_fail_page_alloc(char *str) 3756 { 3757 return setup_fault_attr(&fail_page_alloc.attr, str); 3758 } 3759 __setup("fail_page_alloc=", setup_fail_page_alloc); 3760 3761 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3762 { 3763 if (order < fail_page_alloc.min_order) 3764 return false; 3765 if (gfp_mask & __GFP_NOFAIL) 3766 return false; 3767 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 3768 return false; 3769 if (fail_page_alloc.ignore_gfp_reclaim && 3770 (gfp_mask & __GFP_DIRECT_RECLAIM)) 3771 return false; 3772 3773 return should_fail(&fail_page_alloc.attr, 1 << order); 3774 } 3775 3776 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 3777 3778 static int __init fail_page_alloc_debugfs(void) 3779 { 3780 umode_t mode = S_IFREG | 0600; 3781 struct dentry *dir; 3782 3783 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 3784 &fail_page_alloc.attr); 3785 3786 debugfs_create_bool("ignore-gfp-wait", mode, dir, 3787 &fail_page_alloc.ignore_gfp_reclaim); 3788 debugfs_create_bool("ignore-gfp-highmem", mode, dir, 3789 &fail_page_alloc.ignore_gfp_highmem); 3790 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order); 3791 3792 return 0; 3793 } 3794 3795 late_initcall(fail_page_alloc_debugfs); 3796 3797 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 3798 3799 #else /* CONFIG_FAIL_PAGE_ALLOC */ 3800 3801 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3802 { 3803 return false; 3804 } 3805 3806 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 3807 3808 noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3809 { 3810 return __should_fail_alloc_page(gfp_mask, order); 3811 } 3812 ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE); 3813 3814 static inline long __zone_watermark_unusable_free(struct zone *z, 3815 unsigned int order, unsigned int alloc_flags) 3816 { 3817 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3818 long unusable_free = (1 << order) - 1; 3819 3820 /* 3821 * If the caller does not have rights to ALLOC_HARDER then subtract 3822 * the high-atomic reserves. This will over-estimate the size of the 3823 * atomic reserve but it avoids a search. 3824 */ 3825 if (likely(!alloc_harder)) 3826 unusable_free += z->nr_reserved_highatomic; 3827 3828 #ifdef CONFIG_CMA 3829 /* If allocation can't use CMA areas don't use free CMA pages */ 3830 if (!(alloc_flags & ALLOC_CMA)) 3831 unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES); 3832 #endif 3833 3834 return unusable_free; 3835 } 3836 3837 /* 3838 * Return true if free base pages are above 'mark'. For high-order checks it 3839 * will return true of the order-0 watermark is reached and there is at least 3840 * one free page of a suitable size. Checking now avoids taking the zone lock 3841 * to check in the allocation paths if no pages are free. 3842 */ 3843 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3844 int highest_zoneidx, unsigned int alloc_flags, 3845 long free_pages) 3846 { 3847 long min = mark; 3848 int o; 3849 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3850 3851 /* free_pages may go negative - that's OK */ 3852 free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags); 3853 3854 if (alloc_flags & ALLOC_HIGH) 3855 min -= min / 2; 3856 3857 if (unlikely(alloc_harder)) { 3858 /* 3859 * OOM victims can try even harder than normal ALLOC_HARDER 3860 * users on the grounds that it's definitely going to be in 3861 * the exit path shortly and free memory. Any allocation it 3862 * makes during the free path will be small and short-lived. 3863 */ 3864 if (alloc_flags & ALLOC_OOM) 3865 min -= min / 2; 3866 else 3867 min -= min / 4; 3868 } 3869 3870 /* 3871 * Check watermarks for an order-0 allocation request. If these 3872 * are not met, then a high-order request also cannot go ahead 3873 * even if a suitable page happened to be free. 3874 */ 3875 if (free_pages <= min + z->lowmem_reserve[highest_zoneidx]) 3876 return false; 3877 3878 /* If this is an order-0 request then the watermark is fine */ 3879 if (!order) 3880 return true; 3881 3882 /* For a high-order request, check at least one suitable page is free */ 3883 for (o = order; o < MAX_ORDER; o++) { 3884 struct free_area *area = &z->free_area[o]; 3885 int mt; 3886 3887 if (!area->nr_free) 3888 continue; 3889 3890 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 3891 if (!free_area_empty(area, mt)) 3892 return true; 3893 } 3894 3895 #ifdef CONFIG_CMA 3896 if ((alloc_flags & ALLOC_CMA) && 3897 !free_area_empty(area, MIGRATE_CMA)) { 3898 return true; 3899 } 3900 #endif 3901 if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC)) 3902 return true; 3903 } 3904 return false; 3905 } 3906 3907 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3908 int highest_zoneidx, unsigned int alloc_flags) 3909 { 3910 return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 3911 zone_page_state(z, NR_FREE_PAGES)); 3912 } 3913 3914 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 3915 unsigned long mark, int highest_zoneidx, 3916 unsigned int alloc_flags, gfp_t gfp_mask) 3917 { 3918 long free_pages; 3919 3920 free_pages = zone_page_state(z, NR_FREE_PAGES); 3921 3922 /* 3923 * Fast check for order-0 only. If this fails then the reserves 3924 * need to be calculated. 3925 */ 3926 if (!order) { 3927 long fast_free; 3928 3929 fast_free = free_pages; 3930 fast_free -= __zone_watermark_unusable_free(z, 0, alloc_flags); 3931 if (fast_free > mark + z->lowmem_reserve[highest_zoneidx]) 3932 return true; 3933 } 3934 3935 if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 3936 free_pages)) 3937 return true; 3938 /* 3939 * Ignore watermark boosting for GFP_ATOMIC order-0 allocations 3940 * when checking the min watermark. The min watermark is the 3941 * point where boosting is ignored so that kswapd is woken up 3942 * when below the low watermark. 3943 */ 3944 if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost 3945 && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) { 3946 mark = z->_watermark[WMARK_MIN]; 3947 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 3948 alloc_flags, free_pages); 3949 } 3950 3951 return false; 3952 } 3953 3954 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 3955 unsigned long mark, int highest_zoneidx) 3956 { 3957 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3958 3959 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 3960 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 3961 3962 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0, 3963 free_pages); 3964 } 3965 3966 #ifdef CONFIG_NUMA 3967 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3968 { 3969 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= 3970 node_reclaim_distance; 3971 } 3972 #else /* CONFIG_NUMA */ 3973 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3974 { 3975 return true; 3976 } 3977 #endif /* CONFIG_NUMA */ 3978 3979 /* 3980 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid 3981 * fragmentation is subtle. If the preferred zone was HIGHMEM then 3982 * premature use of a lower zone may cause lowmem pressure problems that 3983 * are worse than fragmentation. If the next zone is ZONE_DMA then it is 3984 * probably too small. It only makes sense to spread allocations to avoid 3985 * fragmentation between the Normal and DMA32 zones. 3986 */ 3987 static inline unsigned int 3988 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) 3989 { 3990 unsigned int alloc_flags; 3991 3992 /* 3993 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 3994 * to save a branch. 3995 */ 3996 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); 3997 3998 #ifdef CONFIG_ZONE_DMA32 3999 if (!zone) 4000 return alloc_flags; 4001 4002 if (zone_idx(zone) != ZONE_NORMAL) 4003 return alloc_flags; 4004 4005 /* 4006 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and 4007 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume 4008 * on UMA that if Normal is populated then so is DMA32. 4009 */ 4010 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1); 4011 if (nr_online_nodes > 1 && !populated_zone(--zone)) 4012 return alloc_flags; 4013 4014 alloc_flags |= ALLOC_NOFRAGMENT; 4015 #endif /* CONFIG_ZONE_DMA32 */ 4016 return alloc_flags; 4017 } 4018 4019 /* Must be called after current_gfp_context() which can change gfp_mask */ 4020 static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask, 4021 unsigned int alloc_flags) 4022 { 4023 #ifdef CONFIG_CMA 4024 if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) 4025 alloc_flags |= ALLOC_CMA; 4026 #endif 4027 return alloc_flags; 4028 } 4029 4030 /* 4031 * get_page_from_freelist goes through the zonelist trying to allocate 4032 * a page. 4033 */ 4034 static struct page * 4035 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 4036 const struct alloc_context *ac) 4037 { 4038 struct zoneref *z; 4039 struct zone *zone; 4040 struct pglist_data *last_pgdat_dirty_limit = NULL; 4041 bool no_fallback; 4042 4043 retry: 4044 /* 4045 * Scan zonelist, looking for a zone with enough free. 4046 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 4047 */ 4048 no_fallback = alloc_flags & ALLOC_NOFRAGMENT; 4049 z = ac->preferred_zoneref; 4050 for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx, 4051 ac->nodemask) { 4052 struct page *page; 4053 unsigned long mark; 4054 4055 if (cpusets_enabled() && 4056 (alloc_flags & ALLOC_CPUSET) && 4057 !__cpuset_zone_allowed(zone, gfp_mask)) 4058 continue; 4059 /* 4060 * When allocating a page cache page for writing, we 4061 * want to get it from a node that is within its dirty 4062 * limit, such that no single node holds more than its 4063 * proportional share of globally allowed dirty pages. 4064 * The dirty limits take into account the node's 4065 * lowmem reserves and high watermark so that kswapd 4066 * should be able to balance it without having to 4067 * write pages from its LRU list. 4068 * 4069 * XXX: For now, allow allocations to potentially 4070 * exceed the per-node dirty limit in the slowpath 4071 * (spread_dirty_pages unset) before going into reclaim, 4072 * which is important when on a NUMA setup the allowed 4073 * nodes are together not big enough to reach the 4074 * global limit. The proper fix for these situations 4075 * will require awareness of nodes in the 4076 * dirty-throttling and the flusher threads. 4077 */ 4078 if (ac->spread_dirty_pages) { 4079 if (last_pgdat_dirty_limit == zone->zone_pgdat) 4080 continue; 4081 4082 if (!node_dirty_ok(zone->zone_pgdat)) { 4083 last_pgdat_dirty_limit = zone->zone_pgdat; 4084 continue; 4085 } 4086 } 4087 4088 if (no_fallback && nr_online_nodes > 1 && 4089 zone != ac->preferred_zoneref->zone) { 4090 int local_nid; 4091 4092 /* 4093 * If moving to a remote node, retry but allow 4094 * fragmenting fallbacks. Locality is more important 4095 * than fragmentation avoidance. 4096 */ 4097 local_nid = zone_to_nid(ac->preferred_zoneref->zone); 4098 if (zone_to_nid(zone) != local_nid) { 4099 alloc_flags &= ~ALLOC_NOFRAGMENT; 4100 goto retry; 4101 } 4102 } 4103 4104 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); 4105 if (!zone_watermark_fast(zone, order, mark, 4106 ac->highest_zoneidx, alloc_flags, 4107 gfp_mask)) { 4108 int ret; 4109 4110 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 4111 /* 4112 * Watermark failed for this zone, but see if we can 4113 * grow this zone if it contains deferred pages. 4114 */ 4115 if (static_branch_unlikely(&deferred_pages)) { 4116 if (_deferred_grow_zone(zone, order)) 4117 goto try_this_zone; 4118 } 4119 #endif 4120 /* Checked here to keep the fast path fast */ 4121 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 4122 if (alloc_flags & ALLOC_NO_WATERMARKS) 4123 goto try_this_zone; 4124 4125 if (!node_reclaim_enabled() || 4126 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 4127 continue; 4128 4129 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 4130 switch (ret) { 4131 case NODE_RECLAIM_NOSCAN: 4132 /* did not scan */ 4133 continue; 4134 case NODE_RECLAIM_FULL: 4135 /* scanned but unreclaimable */ 4136 continue; 4137 default: 4138 /* did we reclaim enough */ 4139 if (zone_watermark_ok(zone, order, mark, 4140 ac->highest_zoneidx, alloc_flags)) 4141 goto try_this_zone; 4142 4143 continue; 4144 } 4145 } 4146 4147 try_this_zone: 4148 page = rmqueue(ac->preferred_zoneref->zone, zone, order, 4149 gfp_mask, alloc_flags, ac->migratetype); 4150 if (page) { 4151 prep_new_page(page, order, gfp_mask, alloc_flags); 4152 4153 /* 4154 * If this is a high-order atomic allocation then check 4155 * if the pageblock should be reserved for the future 4156 */ 4157 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 4158 reserve_highatomic_pageblock(page, zone, order); 4159 4160 return page; 4161 } else { 4162 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 4163 /* Try again if zone has deferred pages */ 4164 if (static_branch_unlikely(&deferred_pages)) { 4165 if (_deferred_grow_zone(zone, order)) 4166 goto try_this_zone; 4167 } 4168 #endif 4169 } 4170 } 4171 4172 /* 4173 * It's possible on a UMA machine to get through all zones that are 4174 * fragmented. If avoiding fragmentation, reset and try again. 4175 */ 4176 if (no_fallback) { 4177 alloc_flags &= ~ALLOC_NOFRAGMENT; 4178 goto retry; 4179 } 4180 4181 return NULL; 4182 } 4183 4184 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) 4185 { 4186 unsigned int filter = SHOW_MEM_FILTER_NODES; 4187 4188 /* 4189 * This documents exceptions given to allocations in certain 4190 * contexts that are allowed to allocate outside current's set 4191 * of allowed nodes. 4192 */ 4193 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4194 if (tsk_is_oom_victim(current) || 4195 (current->flags & (PF_MEMALLOC | PF_EXITING))) 4196 filter &= ~SHOW_MEM_FILTER_NODES; 4197 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 4198 filter &= ~SHOW_MEM_FILTER_NODES; 4199 4200 show_mem(filter, nodemask); 4201 } 4202 4203 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) 4204 { 4205 struct va_format vaf; 4206 va_list args; 4207 static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1); 4208 4209 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs)) 4210 return; 4211 4212 va_start(args, fmt); 4213 vaf.fmt = fmt; 4214 vaf.va = &args; 4215 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl", 4216 current->comm, &vaf, gfp_mask, &gfp_mask, 4217 nodemask_pr_args(nodemask)); 4218 va_end(args); 4219 4220 cpuset_print_current_mems_allowed(); 4221 pr_cont("\n"); 4222 dump_stack(); 4223 warn_alloc_show_mem(gfp_mask, nodemask); 4224 } 4225 4226 static inline struct page * 4227 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, 4228 unsigned int alloc_flags, 4229 const struct alloc_context *ac) 4230 { 4231 struct page *page; 4232 4233 page = get_page_from_freelist(gfp_mask, order, 4234 alloc_flags|ALLOC_CPUSET, ac); 4235 /* 4236 * fallback to ignore cpuset restriction if our nodes 4237 * are depleted 4238 */ 4239 if (!page) 4240 page = get_page_from_freelist(gfp_mask, order, 4241 alloc_flags, ac); 4242 4243 return page; 4244 } 4245 4246 static inline struct page * 4247 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 4248 const struct alloc_context *ac, unsigned long *did_some_progress) 4249 { 4250 struct oom_control oc = { 4251 .zonelist = ac->zonelist, 4252 .nodemask = ac->nodemask, 4253 .memcg = NULL, 4254 .gfp_mask = gfp_mask, 4255 .order = order, 4256 }; 4257 struct page *page; 4258 4259 *did_some_progress = 0; 4260 4261 /* 4262 * Acquire the oom lock. If that fails, somebody else is 4263 * making progress for us. 4264 */ 4265 if (!mutex_trylock(&oom_lock)) { 4266 *did_some_progress = 1; 4267 schedule_timeout_uninterruptible(1); 4268 return NULL; 4269 } 4270 4271 /* 4272 * Go through the zonelist yet one more time, keep very high watermark 4273 * here, this is only to catch a parallel oom killing, we must fail if 4274 * we're still under heavy pressure. But make sure that this reclaim 4275 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY 4276 * allocation which will never fail due to oom_lock already held. 4277 */ 4278 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & 4279 ~__GFP_DIRECT_RECLAIM, order, 4280 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 4281 if (page) 4282 goto out; 4283 4284 /* Coredumps can quickly deplete all memory reserves */ 4285 if (current->flags & PF_DUMPCORE) 4286 goto out; 4287 /* The OOM killer will not help higher order allocs */ 4288 if (order > PAGE_ALLOC_COSTLY_ORDER) 4289 goto out; 4290 /* 4291 * We have already exhausted all our reclaim opportunities without any 4292 * success so it is time to admit defeat. We will skip the OOM killer 4293 * because it is very likely that the caller has a more reasonable 4294 * fallback than shooting a random task. 4295 * 4296 * The OOM killer may not free memory on a specific node. 4297 */ 4298 if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE)) 4299 goto out; 4300 /* The OOM killer does not needlessly kill tasks for lowmem */ 4301 if (ac->highest_zoneidx < ZONE_NORMAL) 4302 goto out; 4303 if (pm_suspended_storage()) 4304 goto out; 4305 /* 4306 * XXX: GFP_NOFS allocations should rather fail than rely on 4307 * other request to make a forward progress. 4308 * We are in an unfortunate situation where out_of_memory cannot 4309 * do much for this context but let's try it to at least get 4310 * access to memory reserved if the current task is killed (see 4311 * out_of_memory). Once filesystems are ready to handle allocation 4312 * failures more gracefully we should just bail out here. 4313 */ 4314 4315 /* Exhausted what can be done so it's blame time */ 4316 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 4317 *did_some_progress = 1; 4318 4319 /* 4320 * Help non-failing allocations by giving them access to memory 4321 * reserves 4322 */ 4323 if (gfp_mask & __GFP_NOFAIL) 4324 page = __alloc_pages_cpuset_fallback(gfp_mask, order, 4325 ALLOC_NO_WATERMARKS, ac); 4326 } 4327 out: 4328 mutex_unlock(&oom_lock); 4329 return page; 4330 } 4331 4332 /* 4333 * Maximum number of compaction retries with a progress before OOM 4334 * killer is consider as the only way to move forward. 4335 */ 4336 #define MAX_COMPACT_RETRIES 16 4337 4338 #ifdef CONFIG_COMPACTION 4339 /* Try memory compaction for high-order allocations before reclaim */ 4340 static struct page * 4341 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4342 unsigned int alloc_flags, const struct alloc_context *ac, 4343 enum compact_priority prio, enum compact_result *compact_result) 4344 { 4345 struct page *page = NULL; 4346 unsigned long pflags; 4347 unsigned int noreclaim_flag; 4348 4349 if (!order) 4350 return NULL; 4351 4352 psi_memstall_enter(&pflags); 4353 noreclaim_flag = memalloc_noreclaim_save(); 4354 4355 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 4356 prio, &page); 4357 4358 memalloc_noreclaim_restore(noreclaim_flag); 4359 psi_memstall_leave(&pflags); 4360 4361 if (*compact_result == COMPACT_SKIPPED) 4362 return NULL; 4363 /* 4364 * At least in one zone compaction wasn't deferred or skipped, so let's 4365 * count a compaction stall 4366 */ 4367 count_vm_event(COMPACTSTALL); 4368 4369 /* Prep a captured page if available */ 4370 if (page) 4371 prep_new_page(page, order, gfp_mask, alloc_flags); 4372 4373 /* Try get a page from the freelist if available */ 4374 if (!page) 4375 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4376 4377 if (page) { 4378 struct zone *zone = page_zone(page); 4379 4380 zone->compact_blockskip_flush = false; 4381 compaction_defer_reset(zone, order, true); 4382 count_vm_event(COMPACTSUCCESS); 4383 return page; 4384 } 4385 4386 /* 4387 * It's bad if compaction run occurs and fails. The most likely reason 4388 * is that pages exist, but not enough to satisfy watermarks. 4389 */ 4390 count_vm_event(COMPACTFAIL); 4391 4392 cond_resched(); 4393 4394 return NULL; 4395 } 4396 4397 static inline bool 4398 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 4399 enum compact_result compact_result, 4400 enum compact_priority *compact_priority, 4401 int *compaction_retries) 4402 { 4403 int max_retries = MAX_COMPACT_RETRIES; 4404 int min_priority; 4405 bool ret = false; 4406 int retries = *compaction_retries; 4407 enum compact_priority priority = *compact_priority; 4408 4409 if (!order) 4410 return false; 4411 4412 if (fatal_signal_pending(current)) 4413 return false; 4414 4415 if (compaction_made_progress(compact_result)) 4416 (*compaction_retries)++; 4417 4418 /* 4419 * compaction considers all the zone as desperately out of memory 4420 * so it doesn't really make much sense to retry except when the 4421 * failure could be caused by insufficient priority 4422 */ 4423 if (compaction_failed(compact_result)) 4424 goto check_priority; 4425 4426 /* 4427 * compaction was skipped because there are not enough order-0 pages 4428 * to work with, so we retry only if it looks like reclaim can help. 4429 */ 4430 if (compaction_needs_reclaim(compact_result)) { 4431 ret = compaction_zonelist_suitable(ac, order, alloc_flags); 4432 goto out; 4433 } 4434 4435 /* 4436 * make sure the compaction wasn't deferred or didn't bail out early 4437 * due to locks contention before we declare that we should give up. 4438 * But the next retry should use a higher priority if allowed, so 4439 * we don't just keep bailing out endlessly. 4440 */ 4441 if (compaction_withdrawn(compact_result)) { 4442 goto check_priority; 4443 } 4444 4445 /* 4446 * !costly requests are much more important than __GFP_RETRY_MAYFAIL 4447 * costly ones because they are de facto nofail and invoke OOM 4448 * killer to move on while costly can fail and users are ready 4449 * to cope with that. 1/4 retries is rather arbitrary but we 4450 * would need much more detailed feedback from compaction to 4451 * make a better decision. 4452 */ 4453 if (order > PAGE_ALLOC_COSTLY_ORDER) 4454 max_retries /= 4; 4455 if (*compaction_retries <= max_retries) { 4456 ret = true; 4457 goto out; 4458 } 4459 4460 /* 4461 * Make sure there are attempts at the highest priority if we exhausted 4462 * all retries or failed at the lower priorities. 4463 */ 4464 check_priority: 4465 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 4466 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 4467 4468 if (*compact_priority > min_priority) { 4469 (*compact_priority)--; 4470 *compaction_retries = 0; 4471 ret = true; 4472 } 4473 out: 4474 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret); 4475 return ret; 4476 } 4477 #else 4478 static inline struct page * 4479 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4480 unsigned int alloc_flags, const struct alloc_context *ac, 4481 enum compact_priority prio, enum compact_result *compact_result) 4482 { 4483 *compact_result = COMPACT_SKIPPED; 4484 return NULL; 4485 } 4486 4487 static inline bool 4488 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 4489 enum compact_result compact_result, 4490 enum compact_priority *compact_priority, 4491 int *compaction_retries) 4492 { 4493 struct zone *zone; 4494 struct zoneref *z; 4495 4496 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 4497 return false; 4498 4499 /* 4500 * There are setups with compaction disabled which would prefer to loop 4501 * inside the allocator rather than hit the oom killer prematurely. 4502 * Let's give them a good hope and keep retrying while the order-0 4503 * watermarks are OK. 4504 */ 4505 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4506 ac->highest_zoneidx, ac->nodemask) { 4507 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 4508 ac->highest_zoneidx, alloc_flags)) 4509 return true; 4510 } 4511 return false; 4512 } 4513 #endif /* CONFIG_COMPACTION */ 4514 4515 #ifdef CONFIG_LOCKDEP 4516 static struct lockdep_map __fs_reclaim_map = 4517 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map); 4518 4519 static bool __need_reclaim(gfp_t gfp_mask) 4520 { 4521 /* no reclaim without waiting on it */ 4522 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) 4523 return false; 4524 4525 /* this guy won't enter reclaim */ 4526 if (current->flags & PF_MEMALLOC) 4527 return false; 4528 4529 if (gfp_mask & __GFP_NOLOCKDEP) 4530 return false; 4531 4532 return true; 4533 } 4534 4535 void __fs_reclaim_acquire(void) 4536 { 4537 lock_map_acquire(&__fs_reclaim_map); 4538 } 4539 4540 void __fs_reclaim_release(void) 4541 { 4542 lock_map_release(&__fs_reclaim_map); 4543 } 4544 4545 void fs_reclaim_acquire(gfp_t gfp_mask) 4546 { 4547 gfp_mask = current_gfp_context(gfp_mask); 4548 4549 if (__need_reclaim(gfp_mask)) { 4550 if (gfp_mask & __GFP_FS) 4551 __fs_reclaim_acquire(); 4552 4553 #ifdef CONFIG_MMU_NOTIFIER 4554 lock_map_acquire(&__mmu_notifier_invalidate_range_start_map); 4555 lock_map_release(&__mmu_notifier_invalidate_range_start_map); 4556 #endif 4557 4558 } 4559 } 4560 EXPORT_SYMBOL_GPL(fs_reclaim_acquire); 4561 4562 void fs_reclaim_release(gfp_t gfp_mask) 4563 { 4564 gfp_mask = current_gfp_context(gfp_mask); 4565 4566 if (__need_reclaim(gfp_mask)) { 4567 if (gfp_mask & __GFP_FS) 4568 __fs_reclaim_release(); 4569 } 4570 } 4571 EXPORT_SYMBOL_GPL(fs_reclaim_release); 4572 #endif 4573 4574 /* Perform direct synchronous page reclaim */ 4575 static unsigned long 4576 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 4577 const struct alloc_context *ac) 4578 { 4579 unsigned int noreclaim_flag; 4580 unsigned long pflags, progress; 4581 4582 cond_resched(); 4583 4584 /* We now go into synchronous reclaim */ 4585 cpuset_memory_pressure_bump(); 4586 psi_memstall_enter(&pflags); 4587 fs_reclaim_acquire(gfp_mask); 4588 noreclaim_flag = memalloc_noreclaim_save(); 4589 4590 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 4591 ac->nodemask); 4592 4593 memalloc_noreclaim_restore(noreclaim_flag); 4594 fs_reclaim_release(gfp_mask); 4595 psi_memstall_leave(&pflags); 4596 4597 cond_resched(); 4598 4599 return progress; 4600 } 4601 4602 /* The really slow allocator path where we enter direct reclaim */ 4603 static inline struct page * 4604 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 4605 unsigned int alloc_flags, const struct alloc_context *ac, 4606 unsigned long *did_some_progress) 4607 { 4608 struct page *page = NULL; 4609 bool drained = false; 4610 4611 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 4612 if (unlikely(!(*did_some_progress))) 4613 return NULL; 4614 4615 retry: 4616 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4617 4618 /* 4619 * If an allocation failed after direct reclaim, it could be because 4620 * pages are pinned on the per-cpu lists or in high alloc reserves. 4621 * Shrink them and try again 4622 */ 4623 if (!page && !drained) { 4624 unreserve_highatomic_pageblock(ac, false); 4625 drain_all_pages(NULL); 4626 drained = true; 4627 goto retry; 4628 } 4629 4630 return page; 4631 } 4632 4633 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, 4634 const struct alloc_context *ac) 4635 { 4636 struct zoneref *z; 4637 struct zone *zone; 4638 pg_data_t *last_pgdat = NULL; 4639 enum zone_type highest_zoneidx = ac->highest_zoneidx; 4640 4641 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx, 4642 ac->nodemask) { 4643 if (last_pgdat != zone->zone_pgdat) 4644 wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx); 4645 last_pgdat = zone->zone_pgdat; 4646 } 4647 } 4648 4649 static inline unsigned int 4650 gfp_to_alloc_flags(gfp_t gfp_mask) 4651 { 4652 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 4653 4654 /* 4655 * __GFP_HIGH is assumed to be the same as ALLOC_HIGH 4656 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 4657 * to save two branches. 4658 */ 4659 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 4660 BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); 4661 4662 /* 4663 * The caller may dip into page reserves a bit more if the caller 4664 * cannot run direct reclaim, or if the caller has realtime scheduling 4665 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 4666 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 4667 */ 4668 alloc_flags |= (__force int) 4669 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); 4670 4671 if (gfp_mask & __GFP_ATOMIC) { 4672 /* 4673 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 4674 * if it can't schedule. 4675 */ 4676 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4677 alloc_flags |= ALLOC_HARDER; 4678 /* 4679 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 4680 * comment for __cpuset_node_allowed(). 4681 */ 4682 alloc_flags &= ~ALLOC_CPUSET; 4683 } else if (unlikely(rt_task(current)) && !in_interrupt()) 4684 alloc_flags |= ALLOC_HARDER; 4685 4686 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags); 4687 4688 return alloc_flags; 4689 } 4690 4691 static bool oom_reserves_allowed(struct task_struct *tsk) 4692 { 4693 if (!tsk_is_oom_victim(tsk)) 4694 return false; 4695 4696 /* 4697 * !MMU doesn't have oom reaper so give access to memory reserves 4698 * only to the thread with TIF_MEMDIE set 4699 */ 4700 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE)) 4701 return false; 4702 4703 return true; 4704 } 4705 4706 /* 4707 * Distinguish requests which really need access to full memory 4708 * reserves from oom victims which can live with a portion of it 4709 */ 4710 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) 4711 { 4712 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 4713 return 0; 4714 if (gfp_mask & __GFP_MEMALLOC) 4715 return ALLOC_NO_WATERMARKS; 4716 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 4717 return ALLOC_NO_WATERMARKS; 4718 if (!in_interrupt()) { 4719 if (current->flags & PF_MEMALLOC) 4720 return ALLOC_NO_WATERMARKS; 4721 else if (oom_reserves_allowed(current)) 4722 return ALLOC_OOM; 4723 } 4724 4725 return 0; 4726 } 4727 4728 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 4729 { 4730 return !!__gfp_pfmemalloc_flags(gfp_mask); 4731 } 4732 4733 /* 4734 * Checks whether it makes sense to retry the reclaim to make a forward progress 4735 * for the given allocation request. 4736 * 4737 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row 4738 * without success, or when we couldn't even meet the watermark if we 4739 * reclaimed all remaining pages on the LRU lists. 4740 * 4741 * Returns true if a retry is viable or false to enter the oom path. 4742 */ 4743 static inline bool 4744 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 4745 struct alloc_context *ac, int alloc_flags, 4746 bool did_some_progress, int *no_progress_loops) 4747 { 4748 struct zone *zone; 4749 struct zoneref *z; 4750 bool ret = false; 4751 4752 /* 4753 * Costly allocations might have made a progress but this doesn't mean 4754 * their order will become available due to high fragmentation so 4755 * always increment the no progress counter for them 4756 */ 4757 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 4758 *no_progress_loops = 0; 4759 else 4760 (*no_progress_loops)++; 4761 4762 /* 4763 * Make sure we converge to OOM if we cannot make any progress 4764 * several times in the row. 4765 */ 4766 if (*no_progress_loops > MAX_RECLAIM_RETRIES) { 4767 /* Before OOM, exhaust highatomic_reserve */ 4768 return unreserve_highatomic_pageblock(ac, true); 4769 } 4770 4771 /* 4772 * Keep reclaiming pages while there is a chance this will lead 4773 * somewhere. If none of the target zones can satisfy our allocation 4774 * request even if all reclaimable pages are considered then we are 4775 * screwed and have to go OOM. 4776 */ 4777 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4778 ac->highest_zoneidx, ac->nodemask) { 4779 unsigned long available; 4780 unsigned long reclaimable; 4781 unsigned long min_wmark = min_wmark_pages(zone); 4782 bool wmark; 4783 4784 available = reclaimable = zone_reclaimable_pages(zone); 4785 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 4786 4787 /* 4788 * Would the allocation succeed if we reclaimed all 4789 * reclaimable pages? 4790 */ 4791 wmark = __zone_watermark_ok(zone, order, min_wmark, 4792 ac->highest_zoneidx, alloc_flags, available); 4793 trace_reclaim_retry_zone(z, order, reclaimable, 4794 available, min_wmark, *no_progress_loops, wmark); 4795 if (wmark) { 4796 /* 4797 * If we didn't make any progress and have a lot of 4798 * dirty + writeback pages then we should wait for 4799 * an IO to complete to slow down the reclaim and 4800 * prevent from pre mature OOM 4801 */ 4802 if (!did_some_progress) { 4803 unsigned long write_pending; 4804 4805 write_pending = zone_page_state_snapshot(zone, 4806 NR_ZONE_WRITE_PENDING); 4807 4808 if (2 * write_pending > reclaimable) { 4809 congestion_wait(BLK_RW_ASYNC, HZ/10); 4810 return true; 4811 } 4812 } 4813 4814 ret = true; 4815 goto out; 4816 } 4817 } 4818 4819 out: 4820 /* 4821 * Memory allocation/reclaim might be called from a WQ context and the 4822 * current implementation of the WQ concurrency control doesn't 4823 * recognize that a particular WQ is congested if the worker thread is 4824 * looping without ever sleeping. Therefore we have to do a short sleep 4825 * here rather than calling cond_resched(). 4826 */ 4827 if (current->flags & PF_WQ_WORKER) 4828 schedule_timeout_uninterruptible(1); 4829 else 4830 cond_resched(); 4831 return ret; 4832 } 4833 4834 static inline bool 4835 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) 4836 { 4837 /* 4838 * It's possible that cpuset's mems_allowed and the nodemask from 4839 * mempolicy don't intersect. This should be normally dealt with by 4840 * policy_nodemask(), but it's possible to race with cpuset update in 4841 * such a way the check therein was true, and then it became false 4842 * before we got our cpuset_mems_cookie here. 4843 * This assumes that for all allocations, ac->nodemask can come only 4844 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored 4845 * when it does not intersect with the cpuset restrictions) or the 4846 * caller can deal with a violated nodemask. 4847 */ 4848 if (cpusets_enabled() && ac->nodemask && 4849 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) { 4850 ac->nodemask = NULL; 4851 return true; 4852 } 4853 4854 /* 4855 * When updating a task's mems_allowed or mempolicy nodemask, it is 4856 * possible to race with parallel threads in such a way that our 4857 * allocation can fail while the mask is being updated. If we are about 4858 * to fail, check if the cpuset changed during allocation and if so, 4859 * retry. 4860 */ 4861 if (read_mems_allowed_retry(cpuset_mems_cookie)) 4862 return true; 4863 4864 return false; 4865 } 4866 4867 static inline struct page * 4868 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 4869 struct alloc_context *ac) 4870 { 4871 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 4872 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 4873 struct page *page = NULL; 4874 unsigned int alloc_flags; 4875 unsigned long did_some_progress; 4876 enum compact_priority compact_priority; 4877 enum compact_result compact_result; 4878 int compaction_retries; 4879 int no_progress_loops; 4880 unsigned int cpuset_mems_cookie; 4881 int reserve_flags; 4882 4883 /* 4884 * We also sanity check to catch abuse of atomic reserves being used by 4885 * callers that are not in atomic context. 4886 */ 4887 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 4888 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 4889 gfp_mask &= ~__GFP_ATOMIC; 4890 4891 retry_cpuset: 4892 compaction_retries = 0; 4893 no_progress_loops = 0; 4894 compact_priority = DEF_COMPACT_PRIORITY; 4895 cpuset_mems_cookie = read_mems_allowed_begin(); 4896 4897 /* 4898 * The fast path uses conservative alloc_flags to succeed only until 4899 * kswapd needs to be woken up, and to avoid the cost of setting up 4900 * alloc_flags precisely. So we do that now. 4901 */ 4902 alloc_flags = gfp_to_alloc_flags(gfp_mask); 4903 4904 /* 4905 * We need to recalculate the starting point for the zonelist iterator 4906 * because we might have used different nodemask in the fast path, or 4907 * there was a cpuset modification and we are retrying - otherwise we 4908 * could end up iterating over non-eligible zones endlessly. 4909 */ 4910 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4911 ac->highest_zoneidx, ac->nodemask); 4912 if (!ac->preferred_zoneref->zone) 4913 goto nopage; 4914 4915 if (alloc_flags & ALLOC_KSWAPD) 4916 wake_all_kswapds(order, gfp_mask, ac); 4917 4918 /* 4919 * The adjusted alloc_flags might result in immediate success, so try 4920 * that first 4921 */ 4922 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4923 if (page) 4924 goto got_pg; 4925 4926 /* 4927 * For costly allocations, try direct compaction first, as it's likely 4928 * that we have enough base pages and don't need to reclaim. For non- 4929 * movable high-order allocations, do that as well, as compaction will 4930 * try prevent permanent fragmentation by migrating from blocks of the 4931 * same migratetype. 4932 * Don't try this for allocations that are allowed to ignore 4933 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 4934 */ 4935 if (can_direct_reclaim && 4936 (costly_order || 4937 (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 4938 && !gfp_pfmemalloc_allowed(gfp_mask)) { 4939 page = __alloc_pages_direct_compact(gfp_mask, order, 4940 alloc_flags, ac, 4941 INIT_COMPACT_PRIORITY, 4942 &compact_result); 4943 if (page) 4944 goto got_pg; 4945 4946 /* 4947 * Checks for costly allocations with __GFP_NORETRY, which 4948 * includes some THP page fault allocations 4949 */ 4950 if (costly_order && (gfp_mask & __GFP_NORETRY)) { 4951 /* 4952 * If allocating entire pageblock(s) and compaction 4953 * failed because all zones are below low watermarks 4954 * or is prohibited because it recently failed at this 4955 * order, fail immediately unless the allocator has 4956 * requested compaction and reclaim retry. 4957 * 4958 * Reclaim is 4959 * - potentially very expensive because zones are far 4960 * below their low watermarks or this is part of very 4961 * bursty high order allocations, 4962 * - not guaranteed to help because isolate_freepages() 4963 * may not iterate over freed pages as part of its 4964 * linear scan, and 4965 * - unlikely to make entire pageblocks free on its 4966 * own. 4967 */ 4968 if (compact_result == COMPACT_SKIPPED || 4969 compact_result == COMPACT_DEFERRED) 4970 goto nopage; 4971 4972 /* 4973 * Looks like reclaim/compaction is worth trying, but 4974 * sync compaction could be very expensive, so keep 4975 * using async compaction. 4976 */ 4977 compact_priority = INIT_COMPACT_PRIORITY; 4978 } 4979 } 4980 4981 retry: 4982 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 4983 if (alloc_flags & ALLOC_KSWAPD) 4984 wake_all_kswapds(order, gfp_mask, ac); 4985 4986 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); 4987 if (reserve_flags) 4988 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags); 4989 4990 /* 4991 * Reset the nodemask and zonelist iterators if memory policies can be 4992 * ignored. These allocations are high priority and system rather than 4993 * user oriented. 4994 */ 4995 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { 4996 ac->nodemask = NULL; 4997 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4998 ac->highest_zoneidx, ac->nodemask); 4999 } 5000 5001 /* Attempt with potentially adjusted zonelist and alloc_flags */ 5002 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 5003 if (page) 5004 goto got_pg; 5005 5006 /* Caller is not willing to reclaim, we can't balance anything */ 5007 if (!can_direct_reclaim) 5008 goto nopage; 5009 5010 /* Avoid recursion of direct reclaim */ 5011 if (current->flags & PF_MEMALLOC) 5012 goto nopage; 5013 5014 /* Try direct reclaim and then allocating */ 5015 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 5016 &did_some_progress); 5017 if (page) 5018 goto got_pg; 5019 5020 /* Try direct compaction and then allocating */ 5021 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 5022 compact_priority, &compact_result); 5023 if (page) 5024 goto got_pg; 5025 5026 /* Do not loop if specifically requested */ 5027 if (gfp_mask & __GFP_NORETRY) 5028 goto nopage; 5029 5030 /* 5031 * Do not retry costly high order allocations unless they are 5032 * __GFP_RETRY_MAYFAIL 5033 */ 5034 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) 5035 goto nopage; 5036 5037 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 5038 did_some_progress > 0, &no_progress_loops)) 5039 goto retry; 5040 5041 /* 5042 * It doesn't make any sense to retry for the compaction if the order-0 5043 * reclaim is not able to make any progress because the current 5044 * implementation of the compaction depends on the sufficient amount 5045 * of free memory (see __compaction_suitable) 5046 */ 5047 if (did_some_progress > 0 && 5048 should_compact_retry(ac, order, alloc_flags, 5049 compact_result, &compact_priority, 5050 &compaction_retries)) 5051 goto retry; 5052 5053 5054 /* Deal with possible cpuset update races before we start OOM killing */ 5055 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 5056 goto retry_cpuset; 5057 5058 /* Reclaim has failed us, start killing things */ 5059 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 5060 if (page) 5061 goto got_pg; 5062 5063 /* Avoid allocations with no watermarks from looping endlessly */ 5064 if (tsk_is_oom_victim(current) && 5065 (alloc_flags & ALLOC_OOM || 5066 (gfp_mask & __GFP_NOMEMALLOC))) 5067 goto nopage; 5068 5069 /* Retry as long as the OOM killer is making progress */ 5070 if (did_some_progress) { 5071 no_progress_loops = 0; 5072 goto retry; 5073 } 5074 5075 nopage: 5076 /* Deal with possible cpuset update races before we fail */ 5077 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 5078 goto retry_cpuset; 5079 5080 /* 5081 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure 5082 * we always retry 5083 */ 5084 if (gfp_mask & __GFP_NOFAIL) { 5085 /* 5086 * All existing users of the __GFP_NOFAIL are blockable, so warn 5087 * of any new users that actually require GFP_NOWAIT 5088 */ 5089 if (WARN_ON_ONCE(!can_direct_reclaim)) 5090 goto fail; 5091 5092 /* 5093 * PF_MEMALLOC request from this context is rather bizarre 5094 * because we cannot reclaim anything and only can loop waiting 5095 * for somebody to do a work for us 5096 */ 5097 WARN_ON_ONCE(current->flags & PF_MEMALLOC); 5098 5099 /* 5100 * non failing costly orders are a hard requirement which we 5101 * are not prepared for much so let's warn about these users 5102 * so that we can identify them and convert them to something 5103 * else. 5104 */ 5105 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER); 5106 5107 /* 5108 * Help non-failing allocations by giving them access to memory 5109 * reserves but do not use ALLOC_NO_WATERMARKS because this 5110 * could deplete whole memory reserves which would just make 5111 * the situation worse 5112 */ 5113 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); 5114 if (page) 5115 goto got_pg; 5116 5117 cond_resched(); 5118 goto retry; 5119 } 5120 fail: 5121 warn_alloc(gfp_mask, ac->nodemask, 5122 "page allocation failure: order:%u", order); 5123 got_pg: 5124 return page; 5125 } 5126 5127 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, 5128 int preferred_nid, nodemask_t *nodemask, 5129 struct alloc_context *ac, gfp_t *alloc_gfp, 5130 unsigned int *alloc_flags) 5131 { 5132 ac->highest_zoneidx = gfp_zone(gfp_mask); 5133 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); 5134 ac->nodemask = nodemask; 5135 ac->migratetype = gfp_migratetype(gfp_mask); 5136 5137 if (cpusets_enabled()) { 5138 *alloc_gfp |= __GFP_HARDWALL; 5139 /* 5140 * When we are in the interrupt context, it is irrelevant 5141 * to the current task context. It means that any node ok. 5142 */ 5143 if (!in_interrupt() && !ac->nodemask) 5144 ac->nodemask = &cpuset_current_mems_allowed; 5145 else 5146 *alloc_flags |= ALLOC_CPUSET; 5147 } 5148 5149 fs_reclaim_acquire(gfp_mask); 5150 fs_reclaim_release(gfp_mask); 5151 5152 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 5153 5154 if (should_fail_alloc_page(gfp_mask, order)) 5155 return false; 5156 5157 *alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, *alloc_flags); 5158 5159 /* Dirty zone balancing only done in the fast path */ 5160 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); 5161 5162 /* 5163 * The preferred zone is used for statistics but crucially it is 5164 * also used as the starting point for the zonelist iterator. It 5165 * may get reset for allocations that ignore memory policies. 5166 */ 5167 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 5168 ac->highest_zoneidx, ac->nodemask); 5169 5170 return true; 5171 } 5172 5173 /* 5174 * __alloc_pages_bulk - Allocate a number of order-0 pages to a list or array 5175 * @gfp: GFP flags for the allocation 5176 * @preferred_nid: The preferred NUMA node ID to allocate from 5177 * @nodemask: Set of nodes to allocate from, may be NULL 5178 * @nr_pages: The number of pages desired on the list or array 5179 * @page_list: Optional list to store the allocated pages 5180 * @page_array: Optional array to store the pages 5181 * 5182 * This is a batched version of the page allocator that attempts to 5183 * allocate nr_pages quickly. Pages are added to page_list if page_list 5184 * is not NULL, otherwise it is assumed that the page_array is valid. 5185 * 5186 * For lists, nr_pages is the number of pages that should be allocated. 5187 * 5188 * For arrays, only NULL elements are populated with pages and nr_pages 5189 * is the maximum number of pages that will be stored in the array. 5190 * 5191 * Returns the number of pages on the list or array. 5192 */ 5193 unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, 5194 nodemask_t *nodemask, int nr_pages, 5195 struct list_head *page_list, 5196 struct page **page_array) 5197 { 5198 struct page *page; 5199 unsigned long flags; 5200 struct zone *zone; 5201 struct zoneref *z; 5202 struct per_cpu_pages *pcp; 5203 struct list_head *pcp_list; 5204 struct alloc_context ac; 5205 gfp_t alloc_gfp; 5206 unsigned int alloc_flags = ALLOC_WMARK_LOW; 5207 int nr_populated = 0, nr_account = 0; 5208 5209 /* 5210 * Skip populated array elements to determine if any pages need 5211 * to be allocated before disabling IRQs. 5212 */ 5213 while (page_array && nr_populated < nr_pages && page_array[nr_populated]) 5214 nr_populated++; 5215 5216 /* No pages requested? */ 5217 if (unlikely(nr_pages <= 0)) 5218 goto out; 5219 5220 /* Already populated array? */ 5221 if (unlikely(page_array && nr_pages - nr_populated == 0)) 5222 goto out; 5223 5224 /* Use the single page allocator for one page. */ 5225 if (nr_pages - nr_populated == 1) 5226 goto failed; 5227 5228 #ifdef CONFIG_PAGE_OWNER 5229 /* 5230 * PAGE_OWNER may recurse into the allocator to allocate space to 5231 * save the stack with pagesets.lock held. Releasing/reacquiring 5232 * removes much of the performance benefit of bulk allocation so 5233 * force the caller to allocate one page at a time as it'll have 5234 * similar performance to added complexity to the bulk allocator. 5235 */ 5236 if (static_branch_unlikely(&page_owner_inited)) 5237 goto failed; 5238 #endif 5239 5240 /* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */ 5241 gfp &= gfp_allowed_mask; 5242 alloc_gfp = gfp; 5243 if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &alloc_gfp, &alloc_flags)) 5244 goto out; 5245 gfp = alloc_gfp; 5246 5247 /* Find an allowed local zone that meets the low watermark. */ 5248 for_each_zone_zonelist_nodemask(zone, z, ac.zonelist, ac.highest_zoneidx, ac.nodemask) { 5249 unsigned long mark; 5250 5251 if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) && 5252 !__cpuset_zone_allowed(zone, gfp)) { 5253 continue; 5254 } 5255 5256 if (nr_online_nodes > 1 && zone != ac.preferred_zoneref->zone && 5257 zone_to_nid(zone) != zone_to_nid(ac.preferred_zoneref->zone)) { 5258 goto failed; 5259 } 5260 5261 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK) + nr_pages; 5262 if (zone_watermark_fast(zone, 0, mark, 5263 zonelist_zone_idx(ac.preferred_zoneref), 5264 alloc_flags, gfp)) { 5265 break; 5266 } 5267 } 5268 5269 /* 5270 * If there are no allowed local zones that meets the watermarks then 5271 * try to allocate a single page and reclaim if necessary. 5272 */ 5273 if (unlikely(!zone)) 5274 goto failed; 5275 5276 /* Attempt the batch allocation */ 5277 local_lock_irqsave(&pagesets.lock, flags); 5278 pcp = this_cpu_ptr(zone->per_cpu_pageset); 5279 pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)]; 5280 5281 while (nr_populated < nr_pages) { 5282 5283 /* Skip existing pages */ 5284 if (page_array && page_array[nr_populated]) { 5285 nr_populated++; 5286 continue; 5287 } 5288 5289 page = __rmqueue_pcplist(zone, 0, ac.migratetype, alloc_flags, 5290 pcp, pcp_list); 5291 if (unlikely(!page)) { 5292 /* Try and get at least one page */ 5293 if (!nr_populated) 5294 goto failed_irq; 5295 break; 5296 } 5297 nr_account++; 5298 5299 prep_new_page(page, 0, gfp, 0); 5300 if (page_list) 5301 list_add(&page->lru, page_list); 5302 else 5303 page_array[nr_populated] = page; 5304 nr_populated++; 5305 } 5306 5307 local_unlock_irqrestore(&pagesets.lock, flags); 5308 5309 __count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account); 5310 zone_statistics(ac.preferred_zoneref->zone, zone, nr_account); 5311 5312 out: 5313 return nr_populated; 5314 5315 failed_irq: 5316 local_unlock_irqrestore(&pagesets.lock, flags); 5317 5318 failed: 5319 page = __alloc_pages(gfp, 0, preferred_nid, nodemask); 5320 if (page) { 5321 if (page_list) 5322 list_add(&page->lru, page_list); 5323 else 5324 page_array[nr_populated] = page; 5325 nr_populated++; 5326 } 5327 5328 goto out; 5329 } 5330 EXPORT_SYMBOL_GPL(__alloc_pages_bulk); 5331 5332 /* 5333 * This is the 'heart' of the zoned buddy allocator. 5334 */ 5335 struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid, 5336 nodemask_t *nodemask) 5337 { 5338 struct page *page; 5339 unsigned int alloc_flags = ALLOC_WMARK_LOW; 5340 gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */ 5341 struct alloc_context ac = { }; 5342 5343 /* 5344 * There are several places where we assume that the order value is sane 5345 * so bail out early if the request is out of bound. 5346 */ 5347 if (unlikely(order >= MAX_ORDER)) { 5348 WARN_ON_ONCE(!(gfp & __GFP_NOWARN)); 5349 return NULL; 5350 } 5351 5352 gfp &= gfp_allowed_mask; 5353 /* 5354 * Apply scoped allocation constraints. This is mainly about GFP_NOFS 5355 * resp. GFP_NOIO which has to be inherited for all allocation requests 5356 * from a particular context which has been marked by 5357 * memalloc_no{fs,io}_{save,restore}. And PF_MEMALLOC_PIN which ensures 5358 * movable zones are not used during allocation. 5359 */ 5360 gfp = current_gfp_context(gfp); 5361 alloc_gfp = gfp; 5362 if (!prepare_alloc_pages(gfp, order, preferred_nid, nodemask, &ac, 5363 &alloc_gfp, &alloc_flags)) 5364 return NULL; 5365 5366 /* 5367 * Forbid the first pass from falling back to types that fragment 5368 * memory until all local zones are considered. 5369 */ 5370 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp); 5371 5372 /* First allocation attempt */ 5373 page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); 5374 if (likely(page)) 5375 goto out; 5376 5377 alloc_gfp = gfp; 5378 ac.spread_dirty_pages = false; 5379 5380 /* 5381 * Restore the original nodemask if it was potentially replaced with 5382 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 5383 */ 5384 ac.nodemask = nodemask; 5385 5386 page = __alloc_pages_slowpath(alloc_gfp, order, &ac); 5387 5388 out: 5389 if (memcg_kmem_enabled() && (gfp & __GFP_ACCOUNT) && page && 5390 unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) { 5391 __free_pages(page, order); 5392 page = NULL; 5393 } 5394 5395 trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype); 5396 5397 return page; 5398 } 5399 EXPORT_SYMBOL(__alloc_pages); 5400 5401 /* 5402 * Common helper functions. Never use with __GFP_HIGHMEM because the returned 5403 * address cannot represent highmem pages. Use alloc_pages and then kmap if 5404 * you need to access high mem. 5405 */ 5406 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 5407 { 5408 struct page *page; 5409 5410 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); 5411 if (!page) 5412 return 0; 5413 return (unsigned long) page_address(page); 5414 } 5415 EXPORT_SYMBOL(__get_free_pages); 5416 5417 unsigned long get_zeroed_page(gfp_t gfp_mask) 5418 { 5419 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 5420 } 5421 EXPORT_SYMBOL(get_zeroed_page); 5422 5423 /** 5424 * __free_pages - Free pages allocated with alloc_pages(). 5425 * @page: The page pointer returned from alloc_pages(). 5426 * @order: The order of the allocation. 5427 * 5428 * This function can free multi-page allocations that are not compound 5429 * pages. It does not check that the @order passed in matches that of 5430 * the allocation, so it is easy to leak memory. Freeing more memory 5431 * than was allocated will probably emit a warning. 5432 * 5433 * If the last reference to this page is speculative, it will be released 5434 * by put_page() which only frees the first page of a non-compound 5435 * allocation. To prevent the remaining pages from being leaked, we free 5436 * the subsequent pages here. If you want to use the page's reference 5437 * count to decide when to free the allocation, you should allocate a 5438 * compound page, and use put_page() instead of __free_pages(). 5439 * 5440 * Context: May be called in interrupt context or while holding a normal 5441 * spinlock, but not in NMI context or while holding a raw spinlock. 5442 */ 5443 void __free_pages(struct page *page, unsigned int order) 5444 { 5445 if (put_page_testzero(page)) 5446 free_the_page(page, order); 5447 else if (!PageHead(page)) 5448 while (order-- > 0) 5449 free_the_page(page + (1 << order), order); 5450 } 5451 EXPORT_SYMBOL(__free_pages); 5452 5453 void free_pages(unsigned long addr, unsigned int order) 5454 { 5455 if (addr != 0) { 5456 VM_BUG_ON(!virt_addr_valid((void *)addr)); 5457 __free_pages(virt_to_page((void *)addr), order); 5458 } 5459 } 5460 5461 EXPORT_SYMBOL(free_pages); 5462 5463 /* 5464 * Page Fragment: 5465 * An arbitrary-length arbitrary-offset area of memory which resides 5466 * within a 0 or higher order page. Multiple fragments within that page 5467 * are individually refcounted, in the page's reference counter. 5468 * 5469 * The page_frag functions below provide a simple allocation framework for 5470 * page fragments. This is used by the network stack and network device 5471 * drivers to provide a backing region of memory for use as either an 5472 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 5473 */ 5474 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 5475 gfp_t gfp_mask) 5476 { 5477 struct page *page = NULL; 5478 gfp_t gfp = gfp_mask; 5479 5480 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5481 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 5482 __GFP_NOMEMALLOC; 5483 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 5484 PAGE_FRAG_CACHE_MAX_ORDER); 5485 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 5486 #endif 5487 if (unlikely(!page)) 5488 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 5489 5490 nc->va = page ? page_address(page) : NULL; 5491 5492 return page; 5493 } 5494 5495 void __page_frag_cache_drain(struct page *page, unsigned int count) 5496 { 5497 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 5498 5499 if (page_ref_sub_and_test(page, count)) 5500 free_the_page(page, compound_order(page)); 5501 } 5502 EXPORT_SYMBOL(__page_frag_cache_drain); 5503 5504 void *page_frag_alloc_align(struct page_frag_cache *nc, 5505 unsigned int fragsz, gfp_t gfp_mask, 5506 unsigned int align_mask) 5507 { 5508 unsigned int size = PAGE_SIZE; 5509 struct page *page; 5510 int offset; 5511 5512 if (unlikely(!nc->va)) { 5513 refill: 5514 page = __page_frag_cache_refill(nc, gfp_mask); 5515 if (!page) 5516 return NULL; 5517 5518 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5519 /* if size can vary use size else just use PAGE_SIZE */ 5520 size = nc->size; 5521 #endif 5522 /* Even if we own the page, we do not use atomic_set(). 5523 * This would break get_page_unless_zero() users. 5524 */ 5525 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 5526 5527 /* reset page count bias and offset to start of new frag */ 5528 nc->pfmemalloc = page_is_pfmemalloc(page); 5529 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5530 nc->offset = size; 5531 } 5532 5533 offset = nc->offset - fragsz; 5534 if (unlikely(offset < 0)) { 5535 page = virt_to_page(nc->va); 5536 5537 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 5538 goto refill; 5539 5540 if (unlikely(nc->pfmemalloc)) { 5541 free_the_page(page, compound_order(page)); 5542 goto refill; 5543 } 5544 5545 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5546 /* if size can vary use size else just use PAGE_SIZE */ 5547 size = nc->size; 5548 #endif 5549 /* OK, page count is 0, we can safely set it */ 5550 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 5551 5552 /* reset page count bias and offset to start of new frag */ 5553 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5554 offset = size - fragsz; 5555 } 5556 5557 nc->pagecnt_bias--; 5558 offset &= align_mask; 5559 nc->offset = offset; 5560 5561 return nc->va + offset; 5562 } 5563 EXPORT_SYMBOL(page_frag_alloc_align); 5564 5565 /* 5566 * Frees a page fragment allocated out of either a compound or order 0 page. 5567 */ 5568 void page_frag_free(void *addr) 5569 { 5570 struct page *page = virt_to_head_page(addr); 5571 5572 if (unlikely(put_page_testzero(page))) 5573 free_the_page(page, compound_order(page)); 5574 } 5575 EXPORT_SYMBOL(page_frag_free); 5576 5577 static void *make_alloc_exact(unsigned long addr, unsigned int order, 5578 size_t size) 5579 { 5580 if (addr) { 5581 unsigned long alloc_end = addr + (PAGE_SIZE << order); 5582 unsigned long used = addr + PAGE_ALIGN(size); 5583 5584 split_page(virt_to_page((void *)addr), order); 5585 while (used < alloc_end) { 5586 free_page(used); 5587 used += PAGE_SIZE; 5588 } 5589 } 5590 return (void *)addr; 5591 } 5592 5593 /** 5594 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 5595 * @size: the number of bytes to allocate 5596 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5597 * 5598 * This function is similar to alloc_pages(), except that it allocates the 5599 * minimum number of pages to satisfy the request. alloc_pages() can only 5600 * allocate memory in power-of-two pages. 5601 * 5602 * This function is also limited by MAX_ORDER. 5603 * 5604 * Memory allocated by this function must be released by free_pages_exact(). 5605 * 5606 * Return: pointer to the allocated area or %NULL in case of error. 5607 */ 5608 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 5609 { 5610 unsigned int order = get_order(size); 5611 unsigned long addr; 5612 5613 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 5614 gfp_mask &= ~__GFP_COMP; 5615 5616 addr = __get_free_pages(gfp_mask, order); 5617 return make_alloc_exact(addr, order, size); 5618 } 5619 EXPORT_SYMBOL(alloc_pages_exact); 5620 5621 /** 5622 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 5623 * pages on a node. 5624 * @nid: the preferred node ID where memory should be allocated 5625 * @size: the number of bytes to allocate 5626 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5627 * 5628 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 5629 * back. 5630 * 5631 * Return: pointer to the allocated area or %NULL in case of error. 5632 */ 5633 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 5634 { 5635 unsigned int order = get_order(size); 5636 struct page *p; 5637 5638 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 5639 gfp_mask &= ~__GFP_COMP; 5640 5641 p = alloc_pages_node(nid, gfp_mask, order); 5642 if (!p) 5643 return NULL; 5644 return make_alloc_exact((unsigned long)page_address(p), order, size); 5645 } 5646 5647 /** 5648 * free_pages_exact - release memory allocated via alloc_pages_exact() 5649 * @virt: the value returned by alloc_pages_exact. 5650 * @size: size of allocation, same value as passed to alloc_pages_exact(). 5651 * 5652 * Release the memory allocated by a previous call to alloc_pages_exact. 5653 */ 5654 void free_pages_exact(void *virt, size_t size) 5655 { 5656 unsigned long addr = (unsigned long)virt; 5657 unsigned long end = addr + PAGE_ALIGN(size); 5658 5659 while (addr < end) { 5660 free_page(addr); 5661 addr += PAGE_SIZE; 5662 } 5663 } 5664 EXPORT_SYMBOL(free_pages_exact); 5665 5666 /** 5667 * nr_free_zone_pages - count number of pages beyond high watermark 5668 * @offset: The zone index of the highest zone 5669 * 5670 * nr_free_zone_pages() counts the number of pages which are beyond the 5671 * high watermark within all zones at or below a given zone index. For each 5672 * zone, the number of pages is calculated as: 5673 * 5674 * nr_free_zone_pages = managed_pages - high_pages 5675 * 5676 * Return: number of pages beyond high watermark. 5677 */ 5678 static unsigned long nr_free_zone_pages(int offset) 5679 { 5680 struct zoneref *z; 5681 struct zone *zone; 5682 5683 /* Just pick one node, since fallback list is circular */ 5684 unsigned long sum = 0; 5685 5686 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 5687 5688 for_each_zone_zonelist(zone, z, zonelist, offset) { 5689 unsigned long size = zone_managed_pages(zone); 5690 unsigned long high = high_wmark_pages(zone); 5691 if (size > high) 5692 sum += size - high; 5693 } 5694 5695 return sum; 5696 } 5697 5698 /** 5699 * nr_free_buffer_pages - count number of pages beyond high watermark 5700 * 5701 * nr_free_buffer_pages() counts the number of pages which are beyond the high 5702 * watermark within ZONE_DMA and ZONE_NORMAL. 5703 * 5704 * Return: number of pages beyond high watermark within ZONE_DMA and 5705 * ZONE_NORMAL. 5706 */ 5707 unsigned long nr_free_buffer_pages(void) 5708 { 5709 return nr_free_zone_pages(gfp_zone(GFP_USER)); 5710 } 5711 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 5712 5713 static inline void show_node(struct zone *zone) 5714 { 5715 if (IS_ENABLED(CONFIG_NUMA)) 5716 printk("Node %d ", zone_to_nid(zone)); 5717 } 5718 5719 long si_mem_available(void) 5720 { 5721 long available; 5722 unsigned long pagecache; 5723 unsigned long wmark_low = 0; 5724 unsigned long pages[NR_LRU_LISTS]; 5725 unsigned long reclaimable; 5726 struct zone *zone; 5727 int lru; 5728 5729 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 5730 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 5731 5732 for_each_zone(zone) 5733 wmark_low += low_wmark_pages(zone); 5734 5735 /* 5736 * Estimate the amount of memory available for userspace allocations, 5737 * without causing swapping. 5738 */ 5739 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages; 5740 5741 /* 5742 * Not all the page cache can be freed, otherwise the system will 5743 * start swapping. Assume at least half of the page cache, or the 5744 * low watermark worth of cache, needs to stay. 5745 */ 5746 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 5747 pagecache -= min(pagecache / 2, wmark_low); 5748 available += pagecache; 5749 5750 /* 5751 * Part of the reclaimable slab and other kernel memory consists of 5752 * items that are in use, and cannot be freed. Cap this estimate at the 5753 * low watermark. 5754 */ 5755 reclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B) + 5756 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE); 5757 available += reclaimable - min(reclaimable / 2, wmark_low); 5758 5759 if (available < 0) 5760 available = 0; 5761 return available; 5762 } 5763 EXPORT_SYMBOL_GPL(si_mem_available); 5764 5765 void si_meminfo(struct sysinfo *val) 5766 { 5767 val->totalram = totalram_pages(); 5768 val->sharedram = global_node_page_state(NR_SHMEM); 5769 val->freeram = global_zone_page_state(NR_FREE_PAGES); 5770 val->bufferram = nr_blockdev_pages(); 5771 val->totalhigh = totalhigh_pages(); 5772 val->freehigh = nr_free_highpages(); 5773 val->mem_unit = PAGE_SIZE; 5774 } 5775 5776 EXPORT_SYMBOL(si_meminfo); 5777 5778 #ifdef CONFIG_NUMA 5779 void si_meminfo_node(struct sysinfo *val, int nid) 5780 { 5781 int zone_type; /* needs to be signed */ 5782 unsigned long managed_pages = 0; 5783 unsigned long managed_highpages = 0; 5784 unsigned long free_highpages = 0; 5785 pg_data_t *pgdat = NODE_DATA(nid); 5786 5787 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 5788 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]); 5789 val->totalram = managed_pages; 5790 val->sharedram = node_page_state(pgdat, NR_SHMEM); 5791 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 5792 #ifdef CONFIG_HIGHMEM 5793 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 5794 struct zone *zone = &pgdat->node_zones[zone_type]; 5795 5796 if (is_highmem(zone)) { 5797 managed_highpages += zone_managed_pages(zone); 5798 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 5799 } 5800 } 5801 val->totalhigh = managed_highpages; 5802 val->freehigh = free_highpages; 5803 #else 5804 val->totalhigh = managed_highpages; 5805 val->freehigh = free_highpages; 5806 #endif 5807 val->mem_unit = PAGE_SIZE; 5808 } 5809 #endif 5810 5811 /* 5812 * Determine whether the node should be displayed or not, depending on whether 5813 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 5814 */ 5815 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) 5816 { 5817 if (!(flags & SHOW_MEM_FILTER_NODES)) 5818 return false; 5819 5820 /* 5821 * no node mask - aka implicit memory numa policy. Do not bother with 5822 * the synchronization - read_mems_allowed_begin - because we do not 5823 * have to be precise here. 5824 */ 5825 if (!nodemask) 5826 nodemask = &cpuset_current_mems_allowed; 5827 5828 return !node_isset(nid, *nodemask); 5829 } 5830 5831 #define K(x) ((x) << (PAGE_SHIFT-10)) 5832 5833 static void show_migration_types(unsigned char type) 5834 { 5835 static const char types[MIGRATE_TYPES] = { 5836 [MIGRATE_UNMOVABLE] = 'U', 5837 [MIGRATE_MOVABLE] = 'M', 5838 [MIGRATE_RECLAIMABLE] = 'E', 5839 [MIGRATE_HIGHATOMIC] = 'H', 5840 #ifdef CONFIG_CMA 5841 [MIGRATE_CMA] = 'C', 5842 #endif 5843 #ifdef CONFIG_MEMORY_ISOLATION 5844 [MIGRATE_ISOLATE] = 'I', 5845 #endif 5846 }; 5847 char tmp[MIGRATE_TYPES + 1]; 5848 char *p = tmp; 5849 int i; 5850 5851 for (i = 0; i < MIGRATE_TYPES; i++) { 5852 if (type & (1 << i)) 5853 *p++ = types[i]; 5854 } 5855 5856 *p = '\0'; 5857 printk(KERN_CONT "(%s) ", tmp); 5858 } 5859 5860 /* 5861 * Show free area list (used inside shift_scroll-lock stuff) 5862 * We also calculate the percentage fragmentation. We do this by counting the 5863 * memory on each free list with the exception of the first item on the list. 5864 * 5865 * Bits in @filter: 5866 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 5867 * cpuset. 5868 */ 5869 void show_free_areas(unsigned int filter, nodemask_t *nodemask) 5870 { 5871 unsigned long free_pcp = 0; 5872 int cpu; 5873 struct zone *zone; 5874 pg_data_t *pgdat; 5875 5876 for_each_populated_zone(zone) { 5877 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5878 continue; 5879 5880 for_each_online_cpu(cpu) 5881 free_pcp += per_cpu_ptr(zone->per_cpu_pageset, cpu)->count; 5882 } 5883 5884 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 5885 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 5886 " unevictable:%lu dirty:%lu writeback:%lu\n" 5887 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 5888 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 5889 " free:%lu free_pcp:%lu free_cma:%lu\n", 5890 global_node_page_state(NR_ACTIVE_ANON), 5891 global_node_page_state(NR_INACTIVE_ANON), 5892 global_node_page_state(NR_ISOLATED_ANON), 5893 global_node_page_state(NR_ACTIVE_FILE), 5894 global_node_page_state(NR_INACTIVE_FILE), 5895 global_node_page_state(NR_ISOLATED_FILE), 5896 global_node_page_state(NR_UNEVICTABLE), 5897 global_node_page_state(NR_FILE_DIRTY), 5898 global_node_page_state(NR_WRITEBACK), 5899 global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B), 5900 global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B), 5901 global_node_page_state(NR_FILE_MAPPED), 5902 global_node_page_state(NR_SHMEM), 5903 global_node_page_state(NR_PAGETABLE), 5904 global_zone_page_state(NR_BOUNCE), 5905 global_zone_page_state(NR_FREE_PAGES), 5906 free_pcp, 5907 global_zone_page_state(NR_FREE_CMA_PAGES)); 5908 5909 for_each_online_pgdat(pgdat) { 5910 if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) 5911 continue; 5912 5913 printk("Node %d" 5914 " active_anon:%lukB" 5915 " inactive_anon:%lukB" 5916 " active_file:%lukB" 5917 " inactive_file:%lukB" 5918 " unevictable:%lukB" 5919 " isolated(anon):%lukB" 5920 " isolated(file):%lukB" 5921 " mapped:%lukB" 5922 " dirty:%lukB" 5923 " writeback:%lukB" 5924 " shmem:%lukB" 5925 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5926 " shmem_thp: %lukB" 5927 " shmem_pmdmapped: %lukB" 5928 " anon_thp: %lukB" 5929 #endif 5930 " writeback_tmp:%lukB" 5931 " kernel_stack:%lukB" 5932 #ifdef CONFIG_SHADOW_CALL_STACK 5933 " shadow_call_stack:%lukB" 5934 #endif 5935 " pagetables:%lukB" 5936 " all_unreclaimable? %s" 5937 "\n", 5938 pgdat->node_id, 5939 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 5940 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 5941 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 5942 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 5943 K(node_page_state(pgdat, NR_UNEVICTABLE)), 5944 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 5945 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 5946 K(node_page_state(pgdat, NR_FILE_MAPPED)), 5947 K(node_page_state(pgdat, NR_FILE_DIRTY)), 5948 K(node_page_state(pgdat, NR_WRITEBACK)), 5949 K(node_page_state(pgdat, NR_SHMEM)), 5950 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5951 K(node_page_state(pgdat, NR_SHMEM_THPS)), 5952 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), 5953 K(node_page_state(pgdat, NR_ANON_THPS)), 5954 #endif 5955 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 5956 node_page_state(pgdat, NR_KERNEL_STACK_KB), 5957 #ifdef CONFIG_SHADOW_CALL_STACK 5958 node_page_state(pgdat, NR_KERNEL_SCS_KB), 5959 #endif 5960 K(node_page_state(pgdat, NR_PAGETABLE)), 5961 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ? 5962 "yes" : "no"); 5963 } 5964 5965 for_each_populated_zone(zone) { 5966 int i; 5967 5968 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5969 continue; 5970 5971 free_pcp = 0; 5972 for_each_online_cpu(cpu) 5973 free_pcp += per_cpu_ptr(zone->per_cpu_pageset, cpu)->count; 5974 5975 show_node(zone); 5976 printk(KERN_CONT 5977 "%s" 5978 " free:%lukB" 5979 " min:%lukB" 5980 " low:%lukB" 5981 " high:%lukB" 5982 " reserved_highatomic:%luKB" 5983 " active_anon:%lukB" 5984 " inactive_anon:%lukB" 5985 " active_file:%lukB" 5986 " inactive_file:%lukB" 5987 " unevictable:%lukB" 5988 " writepending:%lukB" 5989 " present:%lukB" 5990 " managed:%lukB" 5991 " mlocked:%lukB" 5992 " bounce:%lukB" 5993 " free_pcp:%lukB" 5994 " local_pcp:%ukB" 5995 " free_cma:%lukB" 5996 "\n", 5997 zone->name, 5998 K(zone_page_state(zone, NR_FREE_PAGES)), 5999 K(min_wmark_pages(zone)), 6000 K(low_wmark_pages(zone)), 6001 K(high_wmark_pages(zone)), 6002 K(zone->nr_reserved_highatomic), 6003 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 6004 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 6005 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 6006 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 6007 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 6008 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 6009 K(zone->present_pages), 6010 K(zone_managed_pages(zone)), 6011 K(zone_page_state(zone, NR_MLOCK)), 6012 K(zone_page_state(zone, NR_BOUNCE)), 6013 K(free_pcp), 6014 K(this_cpu_read(zone->per_cpu_pageset->count)), 6015 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 6016 printk("lowmem_reserve[]:"); 6017 for (i = 0; i < MAX_NR_ZONES; i++) 6018 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); 6019 printk(KERN_CONT "\n"); 6020 } 6021 6022 for_each_populated_zone(zone) { 6023 unsigned int order; 6024 unsigned long nr[MAX_ORDER], flags, total = 0; 6025 unsigned char types[MAX_ORDER]; 6026 6027 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 6028 continue; 6029 show_node(zone); 6030 printk(KERN_CONT "%s: ", zone->name); 6031 6032 spin_lock_irqsave(&zone->lock, flags); 6033 for (order = 0; order < MAX_ORDER; order++) { 6034 struct free_area *area = &zone->free_area[order]; 6035 int type; 6036 6037 nr[order] = area->nr_free; 6038 total += nr[order] << order; 6039 6040 types[order] = 0; 6041 for (type = 0; type < MIGRATE_TYPES; type++) { 6042 if (!free_area_empty(area, type)) 6043 types[order] |= 1 << type; 6044 } 6045 } 6046 spin_unlock_irqrestore(&zone->lock, flags); 6047 for (order = 0; order < MAX_ORDER; order++) { 6048 printk(KERN_CONT "%lu*%lukB ", 6049 nr[order], K(1UL) << order); 6050 if (nr[order]) 6051 show_migration_types(types[order]); 6052 } 6053 printk(KERN_CONT "= %lukB\n", K(total)); 6054 } 6055 6056 hugetlb_show_meminfo(); 6057 6058 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 6059 6060 show_swap_cache_info(); 6061 } 6062 6063 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 6064 { 6065 zoneref->zone = zone; 6066 zoneref->zone_idx = zone_idx(zone); 6067 } 6068 6069 /* 6070 * Builds allocation fallback zone lists. 6071 * 6072 * Add all populated zones of a node to the zonelist. 6073 */ 6074 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) 6075 { 6076 struct zone *zone; 6077 enum zone_type zone_type = MAX_NR_ZONES; 6078 int nr_zones = 0; 6079 6080 do { 6081 zone_type--; 6082 zone = pgdat->node_zones + zone_type; 6083 if (managed_zone(zone)) { 6084 zoneref_set_zone(zone, &zonerefs[nr_zones++]); 6085 check_highest_zone(zone_type); 6086 } 6087 } while (zone_type); 6088 6089 return nr_zones; 6090 } 6091 6092 #ifdef CONFIG_NUMA 6093 6094 static int __parse_numa_zonelist_order(char *s) 6095 { 6096 /* 6097 * We used to support different zonelists modes but they turned 6098 * out to be just not useful. Let's keep the warning in place 6099 * if somebody still use the cmd line parameter so that we do 6100 * not fail it silently 6101 */ 6102 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) { 6103 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s); 6104 return -EINVAL; 6105 } 6106 return 0; 6107 } 6108 6109 char numa_zonelist_order[] = "Node"; 6110 6111 /* 6112 * sysctl handler for numa_zonelist_order 6113 */ 6114 int numa_zonelist_order_handler(struct ctl_table *table, int write, 6115 void *buffer, size_t *length, loff_t *ppos) 6116 { 6117 if (write) 6118 return __parse_numa_zonelist_order(buffer); 6119 return proc_dostring(table, write, buffer, length, ppos); 6120 } 6121 6122 6123 #define MAX_NODE_LOAD (nr_online_nodes) 6124 static int node_load[MAX_NUMNODES]; 6125 6126 /** 6127 * find_next_best_node - find the next node that should appear in a given node's fallback list 6128 * @node: node whose fallback list we're appending 6129 * @used_node_mask: nodemask_t of already used nodes 6130 * 6131 * We use a number of factors to determine which is the next node that should 6132 * appear on a given node's fallback list. The node should not have appeared 6133 * already in @node's fallback list, and it should be the next closest node 6134 * according to the distance array (which contains arbitrary distance values 6135 * from each node to each node in the system), and should also prefer nodes 6136 * with no CPUs, since presumably they'll have very little allocation pressure 6137 * on them otherwise. 6138 * 6139 * Return: node id of the found node or %NUMA_NO_NODE if no node is found. 6140 */ 6141 static int find_next_best_node(int node, nodemask_t *used_node_mask) 6142 { 6143 int n, val; 6144 int min_val = INT_MAX; 6145 int best_node = NUMA_NO_NODE; 6146 6147 /* Use the local node if we haven't already */ 6148 if (!node_isset(node, *used_node_mask)) { 6149 node_set(node, *used_node_mask); 6150 return node; 6151 } 6152 6153 for_each_node_state(n, N_MEMORY) { 6154 6155 /* Don't want a node to appear more than once */ 6156 if (node_isset(n, *used_node_mask)) 6157 continue; 6158 6159 /* Use the distance array to find the distance */ 6160 val = node_distance(node, n); 6161 6162 /* Penalize nodes under us ("prefer the next node") */ 6163 val += (n < node); 6164 6165 /* Give preference to headless and unused nodes */ 6166 if (!cpumask_empty(cpumask_of_node(n))) 6167 val += PENALTY_FOR_NODE_WITH_CPUS; 6168 6169 /* Slight preference for less loaded node */ 6170 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 6171 val += node_load[n]; 6172 6173 if (val < min_val) { 6174 min_val = val; 6175 best_node = n; 6176 } 6177 } 6178 6179 if (best_node >= 0) 6180 node_set(best_node, *used_node_mask); 6181 6182 return best_node; 6183 } 6184 6185 6186 /* 6187 * Build zonelists ordered by node and zones within node. 6188 * This results in maximum locality--normal zone overflows into local 6189 * DMA zone, if any--but risks exhausting DMA zone. 6190 */ 6191 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, 6192 unsigned nr_nodes) 6193 { 6194 struct zoneref *zonerefs; 6195 int i; 6196 6197 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 6198 6199 for (i = 0; i < nr_nodes; i++) { 6200 int nr_zones; 6201 6202 pg_data_t *node = NODE_DATA(node_order[i]); 6203 6204 nr_zones = build_zonerefs_node(node, zonerefs); 6205 zonerefs += nr_zones; 6206 } 6207 zonerefs->zone = NULL; 6208 zonerefs->zone_idx = 0; 6209 } 6210 6211 /* 6212 * Build gfp_thisnode zonelists 6213 */ 6214 static void build_thisnode_zonelists(pg_data_t *pgdat) 6215 { 6216 struct zoneref *zonerefs; 6217 int nr_zones; 6218 6219 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; 6220 nr_zones = build_zonerefs_node(pgdat, zonerefs); 6221 zonerefs += nr_zones; 6222 zonerefs->zone = NULL; 6223 zonerefs->zone_idx = 0; 6224 } 6225 6226 /* 6227 * Build zonelists ordered by zone and nodes within zones. 6228 * This results in conserving DMA zone[s] until all Normal memory is 6229 * exhausted, but results in overflowing to remote node while memory 6230 * may still exist in local DMA zone. 6231 */ 6232 6233 static void build_zonelists(pg_data_t *pgdat) 6234 { 6235 static int node_order[MAX_NUMNODES]; 6236 int node, load, nr_nodes = 0; 6237 nodemask_t used_mask = NODE_MASK_NONE; 6238 int local_node, prev_node; 6239 6240 /* NUMA-aware ordering of nodes */ 6241 local_node = pgdat->node_id; 6242 load = nr_online_nodes; 6243 prev_node = local_node; 6244 6245 memset(node_order, 0, sizeof(node_order)); 6246 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 6247 /* 6248 * We don't want to pressure a particular node. 6249 * So adding penalty to the first node in same 6250 * distance group to make it round-robin. 6251 */ 6252 if (node_distance(local_node, node) != 6253 node_distance(local_node, prev_node)) 6254 node_load[node] = load; 6255 6256 node_order[nr_nodes++] = node; 6257 prev_node = node; 6258 load--; 6259 } 6260 6261 build_zonelists_in_node_order(pgdat, node_order, nr_nodes); 6262 build_thisnode_zonelists(pgdat); 6263 } 6264 6265 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 6266 /* 6267 * Return node id of node used for "local" allocations. 6268 * I.e., first node id of first zone in arg node's generic zonelist. 6269 * Used for initializing percpu 'numa_mem', which is used primarily 6270 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 6271 */ 6272 int local_memory_node(int node) 6273 { 6274 struct zoneref *z; 6275 6276 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 6277 gfp_zone(GFP_KERNEL), 6278 NULL); 6279 return zone_to_nid(z->zone); 6280 } 6281 #endif 6282 6283 static void setup_min_unmapped_ratio(void); 6284 static void setup_min_slab_ratio(void); 6285 #else /* CONFIG_NUMA */ 6286 6287 static void build_zonelists(pg_data_t *pgdat) 6288 { 6289 int node, local_node; 6290 struct zoneref *zonerefs; 6291 int nr_zones; 6292 6293 local_node = pgdat->node_id; 6294 6295 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 6296 nr_zones = build_zonerefs_node(pgdat, zonerefs); 6297 zonerefs += nr_zones; 6298 6299 /* 6300 * Now we build the zonelist so that it contains the zones 6301 * of all the other nodes. 6302 * We don't want to pressure a particular node, so when 6303 * building the zones for node N, we make sure that the 6304 * zones coming right after the local ones are those from 6305 * node N+1 (modulo N) 6306 */ 6307 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 6308 if (!node_online(node)) 6309 continue; 6310 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 6311 zonerefs += nr_zones; 6312 } 6313 for (node = 0; node < local_node; node++) { 6314 if (!node_online(node)) 6315 continue; 6316 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 6317 zonerefs += nr_zones; 6318 } 6319 6320 zonerefs->zone = NULL; 6321 zonerefs->zone_idx = 0; 6322 } 6323 6324 #endif /* CONFIG_NUMA */ 6325 6326 /* 6327 * Boot pageset table. One per cpu which is going to be used for all 6328 * zones and all nodes. The parameters will be set in such a way 6329 * that an item put on a list will immediately be handed over to 6330 * the buddy list. This is safe since pageset manipulation is done 6331 * with interrupts disabled. 6332 * 6333 * The boot_pagesets must be kept even after bootup is complete for 6334 * unused processors and/or zones. They do play a role for bootstrapping 6335 * hotplugged processors. 6336 * 6337 * zoneinfo_show() and maybe other functions do 6338 * not check if the processor is online before following the pageset pointer. 6339 * Other parts of the kernel may not check if the zone is available. 6340 */ 6341 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats); 6342 /* These effectively disable the pcplists in the boot pageset completely */ 6343 #define BOOT_PAGESET_HIGH 0 6344 #define BOOT_PAGESET_BATCH 1 6345 static DEFINE_PER_CPU(struct per_cpu_pages, boot_pageset); 6346 static DEFINE_PER_CPU(struct per_cpu_zonestat, boot_zonestats); 6347 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 6348 6349 static void __build_all_zonelists(void *data) 6350 { 6351 int nid; 6352 int __maybe_unused cpu; 6353 pg_data_t *self = data; 6354 static DEFINE_SPINLOCK(lock); 6355 6356 spin_lock(&lock); 6357 6358 #ifdef CONFIG_NUMA 6359 memset(node_load, 0, sizeof(node_load)); 6360 #endif 6361 6362 /* 6363 * This node is hotadded and no memory is yet present. So just 6364 * building zonelists is fine - no need to touch other nodes. 6365 */ 6366 if (self && !node_online(self->node_id)) { 6367 build_zonelists(self); 6368 } else { 6369 for_each_online_node(nid) { 6370 pg_data_t *pgdat = NODE_DATA(nid); 6371 6372 build_zonelists(pgdat); 6373 } 6374 6375 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 6376 /* 6377 * We now know the "local memory node" for each node-- 6378 * i.e., the node of the first zone in the generic zonelist. 6379 * Set up numa_mem percpu variable for on-line cpus. During 6380 * boot, only the boot cpu should be on-line; we'll init the 6381 * secondary cpus' numa_mem as they come on-line. During 6382 * node/memory hotplug, we'll fixup all on-line cpus. 6383 */ 6384 for_each_online_cpu(cpu) 6385 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 6386 #endif 6387 } 6388 6389 spin_unlock(&lock); 6390 } 6391 6392 static noinline void __init 6393 build_all_zonelists_init(void) 6394 { 6395 int cpu; 6396 6397 __build_all_zonelists(NULL); 6398 6399 /* 6400 * Initialize the boot_pagesets that are going to be used 6401 * for bootstrapping processors. The real pagesets for 6402 * each zone will be allocated later when the per cpu 6403 * allocator is available. 6404 * 6405 * boot_pagesets are used also for bootstrapping offline 6406 * cpus if the system is already booted because the pagesets 6407 * are needed to initialize allocators on a specific cpu too. 6408 * F.e. the percpu allocator needs the page allocator which 6409 * needs the percpu allocator in order to allocate its pagesets 6410 * (a chicken-egg dilemma). 6411 */ 6412 for_each_possible_cpu(cpu) 6413 per_cpu_pages_init(&per_cpu(boot_pageset, cpu), &per_cpu(boot_zonestats, cpu)); 6414 6415 mminit_verify_zonelist(); 6416 cpuset_init_current_mems_allowed(); 6417 } 6418 6419 /* 6420 * unless system_state == SYSTEM_BOOTING. 6421 * 6422 * __ref due to call of __init annotated helper build_all_zonelists_init 6423 * [protected by SYSTEM_BOOTING]. 6424 */ 6425 void __ref build_all_zonelists(pg_data_t *pgdat) 6426 { 6427 unsigned long vm_total_pages; 6428 6429 if (system_state == SYSTEM_BOOTING) { 6430 build_all_zonelists_init(); 6431 } else { 6432 __build_all_zonelists(pgdat); 6433 /* cpuset refresh routine should be here */ 6434 } 6435 /* Get the number of free pages beyond high watermark in all zones. */ 6436 vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 6437 /* 6438 * Disable grouping by mobility if the number of pages in the 6439 * system is too low to allow the mechanism to work. It would be 6440 * more accurate, but expensive to check per-zone. This check is 6441 * made on memory-hotadd so a system can start with mobility 6442 * disabled and enable it later 6443 */ 6444 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 6445 page_group_by_mobility_disabled = 1; 6446 else 6447 page_group_by_mobility_disabled = 0; 6448 6449 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n", 6450 nr_online_nodes, 6451 page_group_by_mobility_disabled ? "off" : "on", 6452 vm_total_pages); 6453 #ifdef CONFIG_NUMA 6454 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 6455 #endif 6456 } 6457 6458 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 6459 static bool __meminit 6460 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 6461 { 6462 static struct memblock_region *r; 6463 6464 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 6465 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 6466 for_each_mem_region(r) { 6467 if (*pfn < memblock_region_memory_end_pfn(r)) 6468 break; 6469 } 6470 } 6471 if (*pfn >= memblock_region_memory_base_pfn(r) && 6472 memblock_is_mirror(r)) { 6473 *pfn = memblock_region_memory_end_pfn(r); 6474 return true; 6475 } 6476 } 6477 return false; 6478 } 6479 6480 /* 6481 * Initially all pages are reserved - free ones are freed 6482 * up by memblock_free_all() once the early boot process is 6483 * done. Non-atomic initialization, single-pass. 6484 * 6485 * All aligned pageblocks are initialized to the specified migratetype 6486 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related 6487 * zone stats (e.g., nr_isolate_pageblock) are touched. 6488 */ 6489 void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone, 6490 unsigned long start_pfn, unsigned long zone_end_pfn, 6491 enum meminit_context context, 6492 struct vmem_altmap *altmap, int migratetype) 6493 { 6494 unsigned long pfn, end_pfn = start_pfn + size; 6495 struct page *page; 6496 6497 if (highest_memmap_pfn < end_pfn - 1) 6498 highest_memmap_pfn = end_pfn - 1; 6499 6500 #ifdef CONFIG_ZONE_DEVICE 6501 /* 6502 * Honor reservation requested by the driver for this ZONE_DEVICE 6503 * memory. We limit the total number of pages to initialize to just 6504 * those that might contain the memory mapping. We will defer the 6505 * ZONE_DEVICE page initialization until after we have released 6506 * the hotplug lock. 6507 */ 6508 if (zone == ZONE_DEVICE) { 6509 if (!altmap) 6510 return; 6511 6512 if (start_pfn == altmap->base_pfn) 6513 start_pfn += altmap->reserve; 6514 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6515 } 6516 #endif 6517 6518 for (pfn = start_pfn; pfn < end_pfn; ) { 6519 /* 6520 * There can be holes in boot-time mem_map[]s handed to this 6521 * function. They do not exist on hotplugged memory. 6522 */ 6523 if (context == MEMINIT_EARLY) { 6524 if (overlap_memmap_init(zone, &pfn)) 6525 continue; 6526 if (defer_init(nid, pfn, zone_end_pfn)) 6527 break; 6528 } 6529 6530 page = pfn_to_page(pfn); 6531 __init_single_page(page, pfn, zone, nid); 6532 if (context == MEMINIT_HOTPLUG) 6533 __SetPageReserved(page); 6534 6535 /* 6536 * Usually, we want to mark the pageblock MIGRATE_MOVABLE, 6537 * such that unmovable allocations won't be scattered all 6538 * over the place during system boot. 6539 */ 6540 if (IS_ALIGNED(pfn, pageblock_nr_pages)) { 6541 set_pageblock_migratetype(page, migratetype); 6542 cond_resched(); 6543 } 6544 pfn++; 6545 } 6546 } 6547 6548 #ifdef CONFIG_ZONE_DEVICE 6549 void __ref memmap_init_zone_device(struct zone *zone, 6550 unsigned long start_pfn, 6551 unsigned long nr_pages, 6552 struct dev_pagemap *pgmap) 6553 { 6554 unsigned long pfn, end_pfn = start_pfn + nr_pages; 6555 struct pglist_data *pgdat = zone->zone_pgdat; 6556 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 6557 unsigned long zone_idx = zone_idx(zone); 6558 unsigned long start = jiffies; 6559 int nid = pgdat->node_id; 6560 6561 if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE)) 6562 return; 6563 6564 /* 6565 * The call to memmap_init should have already taken care 6566 * of the pages reserved for the memmap, so we can just jump to 6567 * the end of that region and start processing the device pages. 6568 */ 6569 if (altmap) { 6570 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6571 nr_pages = end_pfn - start_pfn; 6572 } 6573 6574 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 6575 struct page *page = pfn_to_page(pfn); 6576 6577 __init_single_page(page, pfn, zone_idx, nid); 6578 6579 /* 6580 * Mark page reserved as it will need to wait for onlining 6581 * phase for it to be fully associated with a zone. 6582 * 6583 * We can use the non-atomic __set_bit operation for setting 6584 * the flag as we are still initializing the pages. 6585 */ 6586 __SetPageReserved(page); 6587 6588 /* 6589 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 6590 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 6591 * ever freed or placed on a driver-private list. 6592 */ 6593 page->pgmap = pgmap; 6594 page->zone_device_data = NULL; 6595 6596 /* 6597 * Mark the block movable so that blocks are reserved for 6598 * movable at startup. This will force kernel allocations 6599 * to reserve their blocks rather than leaking throughout 6600 * the address space during boot when many long-lived 6601 * kernel allocations are made. 6602 * 6603 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap 6604 * because this is done early in section_activate() 6605 */ 6606 if (IS_ALIGNED(pfn, pageblock_nr_pages)) { 6607 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 6608 cond_resched(); 6609 } 6610 } 6611 6612 pr_info("%s initialised %lu pages in %ums\n", __func__, 6613 nr_pages, jiffies_to_msecs(jiffies - start)); 6614 } 6615 6616 #endif 6617 static void __meminit zone_init_free_lists(struct zone *zone) 6618 { 6619 unsigned int order, t; 6620 for_each_migratetype_order(order, t) { 6621 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 6622 zone->free_area[order].nr_free = 0; 6623 } 6624 } 6625 6626 #if !defined(CONFIG_FLATMEM) 6627 /* 6628 * Only struct pages that correspond to ranges defined by memblock.memory 6629 * are zeroed and initialized by going through __init_single_page() during 6630 * memmap_init_zone_range(). 6631 * 6632 * But, there could be struct pages that correspond to holes in 6633 * memblock.memory. This can happen because of the following reasons: 6634 * - physical memory bank size is not necessarily the exact multiple of the 6635 * arbitrary section size 6636 * - early reserved memory may not be listed in memblock.memory 6637 * - memory layouts defined with memmap= kernel parameter may not align 6638 * nicely with memmap sections 6639 * 6640 * Explicitly initialize those struct pages so that: 6641 * - PG_Reserved is set 6642 * - zone and node links point to zone and node that span the page if the 6643 * hole is in the middle of a zone 6644 * - zone and node links point to adjacent zone/node if the hole falls on 6645 * the zone boundary; the pages in such holes will be prepended to the 6646 * zone/node above the hole except for the trailing pages in the last 6647 * section that will be appended to the zone/node below. 6648 */ 6649 static void __init init_unavailable_range(unsigned long spfn, 6650 unsigned long epfn, 6651 int zone, int node) 6652 { 6653 unsigned long pfn; 6654 u64 pgcnt = 0; 6655 6656 for (pfn = spfn; pfn < epfn; pfn++) { 6657 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) { 6658 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages) 6659 + pageblock_nr_pages - 1; 6660 continue; 6661 } 6662 __init_single_page(pfn_to_page(pfn), pfn, zone, node); 6663 __SetPageReserved(pfn_to_page(pfn)); 6664 pgcnt++; 6665 } 6666 6667 if (pgcnt) 6668 pr_info("On node %d, zone %s: %lld pages in unavailable ranges", 6669 node, zone_names[zone], pgcnt); 6670 } 6671 #else 6672 static inline void init_unavailable_range(unsigned long spfn, 6673 unsigned long epfn, 6674 int zone, int node) 6675 { 6676 } 6677 #endif 6678 6679 static void __init memmap_init_zone_range(struct zone *zone, 6680 unsigned long start_pfn, 6681 unsigned long end_pfn, 6682 unsigned long *hole_pfn) 6683 { 6684 unsigned long zone_start_pfn = zone->zone_start_pfn; 6685 unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; 6686 int nid = zone_to_nid(zone), zone_id = zone_idx(zone); 6687 6688 start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn); 6689 end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn); 6690 6691 if (start_pfn >= end_pfn) 6692 return; 6693 6694 memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, 6695 zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); 6696 6697 if (*hole_pfn < start_pfn) 6698 init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); 6699 6700 *hole_pfn = end_pfn; 6701 } 6702 6703 static void __init memmap_init(void) 6704 { 6705 unsigned long start_pfn, end_pfn; 6706 unsigned long hole_pfn = 0; 6707 int i, j, zone_id, nid; 6708 6709 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 6710 struct pglist_data *node = NODE_DATA(nid); 6711 6712 for (j = 0; j < MAX_NR_ZONES; j++) { 6713 struct zone *zone = node->node_zones + j; 6714 6715 if (!populated_zone(zone)) 6716 continue; 6717 6718 memmap_init_zone_range(zone, start_pfn, end_pfn, 6719 &hole_pfn); 6720 zone_id = j; 6721 } 6722 } 6723 6724 #ifdef CONFIG_SPARSEMEM 6725 /* 6726 * Initialize the memory map for hole in the range [memory_end, 6727 * section_end]. 6728 * Append the pages in this hole to the highest zone in the last 6729 * node. 6730 * The call to init_unavailable_range() is outside the ifdef to 6731 * silence the compiler warining about zone_id set but not used; 6732 * for FLATMEM it is a nop anyway 6733 */ 6734 end_pfn = round_up(end_pfn, PAGES_PER_SECTION); 6735 if (hole_pfn < end_pfn) 6736 #endif 6737 init_unavailable_range(hole_pfn, end_pfn, zone_id, nid); 6738 } 6739 6740 static int zone_batchsize(struct zone *zone) 6741 { 6742 #ifdef CONFIG_MMU 6743 int batch; 6744 6745 /* 6746 * The number of pages to batch allocate is either ~0.1% 6747 * of the zone or 1MB, whichever is smaller. The batch 6748 * size is striking a balance between allocation latency 6749 * and zone lock contention. 6750 */ 6751 batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE); 6752 batch /= 4; /* We effectively *= 4 below */ 6753 if (batch < 1) 6754 batch = 1; 6755 6756 /* 6757 * Clamp the batch to a 2^n - 1 value. Having a power 6758 * of 2 value was found to be more likely to have 6759 * suboptimal cache aliasing properties in some cases. 6760 * 6761 * For example if 2 tasks are alternately allocating 6762 * batches of pages, one task can end up with a lot 6763 * of pages of one half of the possible page colors 6764 * and the other with pages of the other colors. 6765 */ 6766 batch = rounddown_pow_of_two(batch + batch/2) - 1; 6767 6768 return batch; 6769 6770 #else 6771 /* The deferral and batching of frees should be suppressed under NOMMU 6772 * conditions. 6773 * 6774 * The problem is that NOMMU needs to be able to allocate large chunks 6775 * of contiguous memory as there's no hardware page translation to 6776 * assemble apparent contiguous memory from discontiguous pages. 6777 * 6778 * Queueing large contiguous runs of pages for batching, however, 6779 * causes the pages to actually be freed in smaller chunks. As there 6780 * can be a significant delay between the individual batches being 6781 * recycled, this leads to the once large chunks of space being 6782 * fragmented and becoming unavailable for high-order allocations. 6783 */ 6784 return 0; 6785 #endif 6786 } 6787 6788 static int zone_highsize(struct zone *zone, int batch, int cpu_online) 6789 { 6790 #ifdef CONFIG_MMU 6791 int high; 6792 int nr_split_cpus; 6793 unsigned long total_pages; 6794 6795 if (!percpu_pagelist_high_fraction) { 6796 /* 6797 * By default, the high value of the pcp is based on the zone 6798 * low watermark so that if they are full then background 6799 * reclaim will not be started prematurely. 6800 */ 6801 total_pages = low_wmark_pages(zone); 6802 } else { 6803 /* 6804 * If percpu_pagelist_high_fraction is configured, the high 6805 * value is based on a fraction of the managed pages in the 6806 * zone. 6807 */ 6808 total_pages = zone_managed_pages(zone) / percpu_pagelist_high_fraction; 6809 } 6810 6811 /* 6812 * Split the high value across all online CPUs local to the zone. Note 6813 * that early in boot that CPUs may not be online yet and that during 6814 * CPU hotplug that the cpumask is not yet updated when a CPU is being 6815 * onlined. For memory nodes that have no CPUs, split pcp->high across 6816 * all online CPUs to mitigate the risk that reclaim is triggered 6817 * prematurely due to pages stored on pcp lists. 6818 */ 6819 nr_split_cpus = cpumask_weight(cpumask_of_node(zone_to_nid(zone))) + cpu_online; 6820 if (!nr_split_cpus) 6821 nr_split_cpus = num_online_cpus(); 6822 high = total_pages / nr_split_cpus; 6823 6824 /* 6825 * Ensure high is at least batch*4. The multiple is based on the 6826 * historical relationship between high and batch. 6827 */ 6828 high = max(high, batch << 2); 6829 6830 return high; 6831 #else 6832 return 0; 6833 #endif 6834 } 6835 6836 /* 6837 * pcp->high and pcp->batch values are related and generally batch is lower 6838 * than high. They are also related to pcp->count such that count is lower 6839 * than high, and as soon as it reaches high, the pcplist is flushed. 6840 * 6841 * However, guaranteeing these relations at all times would require e.g. write 6842 * barriers here but also careful usage of read barriers at the read side, and 6843 * thus be prone to error and bad for performance. Thus the update only prevents 6844 * store tearing. Any new users of pcp->batch and pcp->high should ensure they 6845 * can cope with those fields changing asynchronously, and fully trust only the 6846 * pcp->count field on the local CPU with interrupts disabled. 6847 * 6848 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 6849 * outside of boot time (or some other assurance that no concurrent updaters 6850 * exist). 6851 */ 6852 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 6853 unsigned long batch) 6854 { 6855 WRITE_ONCE(pcp->batch, batch); 6856 WRITE_ONCE(pcp->high, high); 6857 } 6858 6859 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats) 6860 { 6861 int pindex; 6862 6863 memset(pcp, 0, sizeof(*pcp)); 6864 memset(pzstats, 0, sizeof(*pzstats)); 6865 6866 for (pindex = 0; pindex < NR_PCP_LISTS; pindex++) 6867 INIT_LIST_HEAD(&pcp->lists[pindex]); 6868 6869 /* 6870 * Set batch and high values safe for a boot pageset. A true percpu 6871 * pageset's initialization will update them subsequently. Here we don't 6872 * need to be as careful as pageset_update() as nobody can access the 6873 * pageset yet. 6874 */ 6875 pcp->high = BOOT_PAGESET_HIGH; 6876 pcp->batch = BOOT_PAGESET_BATCH; 6877 pcp->free_factor = 0; 6878 } 6879 6880 static void __zone_set_pageset_high_and_batch(struct zone *zone, unsigned long high, 6881 unsigned long batch) 6882 { 6883 struct per_cpu_pages *pcp; 6884 int cpu; 6885 6886 for_each_possible_cpu(cpu) { 6887 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 6888 pageset_update(pcp, high, batch); 6889 } 6890 } 6891 6892 /* 6893 * Calculate and set new high and batch values for all per-cpu pagesets of a 6894 * zone based on the zone's size. 6895 */ 6896 static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online) 6897 { 6898 int new_high, new_batch; 6899 6900 new_batch = max(1, zone_batchsize(zone)); 6901 new_high = zone_highsize(zone, new_batch, cpu_online); 6902 6903 if (zone->pageset_high == new_high && 6904 zone->pageset_batch == new_batch) 6905 return; 6906 6907 zone->pageset_high = new_high; 6908 zone->pageset_batch = new_batch; 6909 6910 __zone_set_pageset_high_and_batch(zone, new_high, new_batch); 6911 } 6912 6913 void __meminit setup_zone_pageset(struct zone *zone) 6914 { 6915 int cpu; 6916 6917 /* Size may be 0 on !SMP && !NUMA */ 6918 if (sizeof(struct per_cpu_zonestat) > 0) 6919 zone->per_cpu_zonestats = alloc_percpu(struct per_cpu_zonestat); 6920 6921 zone->per_cpu_pageset = alloc_percpu(struct per_cpu_pages); 6922 for_each_possible_cpu(cpu) { 6923 struct per_cpu_pages *pcp; 6924 struct per_cpu_zonestat *pzstats; 6925 6926 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 6927 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 6928 per_cpu_pages_init(pcp, pzstats); 6929 } 6930 6931 zone_set_pageset_high_and_batch(zone, 0); 6932 } 6933 6934 /* 6935 * Allocate per cpu pagesets and initialize them. 6936 * Before this call only boot pagesets were available. 6937 */ 6938 void __init setup_per_cpu_pageset(void) 6939 { 6940 struct pglist_data *pgdat; 6941 struct zone *zone; 6942 int __maybe_unused cpu; 6943 6944 for_each_populated_zone(zone) 6945 setup_zone_pageset(zone); 6946 6947 #ifdef CONFIG_NUMA 6948 /* 6949 * Unpopulated zones continue using the boot pagesets. 6950 * The numa stats for these pagesets need to be reset. 6951 * Otherwise, they will end up skewing the stats of 6952 * the nodes these zones are associated with. 6953 */ 6954 for_each_possible_cpu(cpu) { 6955 struct per_cpu_zonestat *pzstats = &per_cpu(boot_zonestats, cpu); 6956 memset(pzstats->vm_numa_event, 0, 6957 sizeof(pzstats->vm_numa_event)); 6958 } 6959 #endif 6960 6961 for_each_online_pgdat(pgdat) 6962 pgdat->per_cpu_nodestats = 6963 alloc_percpu(struct per_cpu_nodestat); 6964 } 6965 6966 static __meminit void zone_pcp_init(struct zone *zone) 6967 { 6968 /* 6969 * per cpu subsystem is not up at this point. The following code 6970 * relies on the ability of the linker to provide the 6971 * offset of a (static) per cpu variable into the per cpu area. 6972 */ 6973 zone->per_cpu_pageset = &boot_pageset; 6974 zone->per_cpu_zonestats = &boot_zonestats; 6975 zone->pageset_high = BOOT_PAGESET_HIGH; 6976 zone->pageset_batch = BOOT_PAGESET_BATCH; 6977 6978 if (populated_zone(zone)) 6979 pr_debug(" %s zone: %lu pages, LIFO batch:%u\n", zone->name, 6980 zone->present_pages, zone_batchsize(zone)); 6981 } 6982 6983 void __meminit init_currently_empty_zone(struct zone *zone, 6984 unsigned long zone_start_pfn, 6985 unsigned long size) 6986 { 6987 struct pglist_data *pgdat = zone->zone_pgdat; 6988 int zone_idx = zone_idx(zone) + 1; 6989 6990 if (zone_idx > pgdat->nr_zones) 6991 pgdat->nr_zones = zone_idx; 6992 6993 zone->zone_start_pfn = zone_start_pfn; 6994 6995 mminit_dprintk(MMINIT_TRACE, "memmap_init", 6996 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 6997 pgdat->node_id, 6998 (unsigned long)zone_idx(zone), 6999 zone_start_pfn, (zone_start_pfn + size)); 7000 7001 zone_init_free_lists(zone); 7002 zone->initialized = 1; 7003 } 7004 7005 /** 7006 * get_pfn_range_for_nid - Return the start and end page frames for a node 7007 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 7008 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 7009 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 7010 * 7011 * It returns the start and end page frame of a node based on information 7012 * provided by memblock_set_node(). If called for a node 7013 * with no available memory, a warning is printed and the start and end 7014 * PFNs will be 0. 7015 */ 7016 void __init get_pfn_range_for_nid(unsigned int nid, 7017 unsigned long *start_pfn, unsigned long *end_pfn) 7018 { 7019 unsigned long this_start_pfn, this_end_pfn; 7020 int i; 7021 7022 *start_pfn = -1UL; 7023 *end_pfn = 0; 7024 7025 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 7026 *start_pfn = min(*start_pfn, this_start_pfn); 7027 *end_pfn = max(*end_pfn, this_end_pfn); 7028 } 7029 7030 if (*start_pfn == -1UL) 7031 *start_pfn = 0; 7032 } 7033 7034 /* 7035 * This finds a zone that can be used for ZONE_MOVABLE pages. The 7036 * assumption is made that zones within a node are ordered in monotonic 7037 * increasing memory addresses so that the "highest" populated zone is used 7038 */ 7039 static void __init find_usable_zone_for_movable(void) 7040 { 7041 int zone_index; 7042 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 7043 if (zone_index == ZONE_MOVABLE) 7044 continue; 7045 7046 if (arch_zone_highest_possible_pfn[zone_index] > 7047 arch_zone_lowest_possible_pfn[zone_index]) 7048 break; 7049 } 7050 7051 VM_BUG_ON(zone_index == -1); 7052 movable_zone = zone_index; 7053 } 7054 7055 /* 7056 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 7057 * because it is sized independent of architecture. Unlike the other zones, 7058 * the starting point for ZONE_MOVABLE is not fixed. It may be different 7059 * in each node depending on the size of each node and how evenly kernelcore 7060 * is distributed. This helper function adjusts the zone ranges 7061 * provided by the architecture for a given node by using the end of the 7062 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 7063 * zones within a node are in order of monotonic increases memory addresses 7064 */ 7065 static void __init adjust_zone_range_for_zone_movable(int nid, 7066 unsigned long zone_type, 7067 unsigned long node_start_pfn, 7068 unsigned long node_end_pfn, 7069 unsigned long *zone_start_pfn, 7070 unsigned long *zone_end_pfn) 7071 { 7072 /* Only adjust if ZONE_MOVABLE is on this node */ 7073 if (zone_movable_pfn[nid]) { 7074 /* Size ZONE_MOVABLE */ 7075 if (zone_type == ZONE_MOVABLE) { 7076 *zone_start_pfn = zone_movable_pfn[nid]; 7077 *zone_end_pfn = min(node_end_pfn, 7078 arch_zone_highest_possible_pfn[movable_zone]); 7079 7080 /* Adjust for ZONE_MOVABLE starting within this range */ 7081 } else if (!mirrored_kernelcore && 7082 *zone_start_pfn < zone_movable_pfn[nid] && 7083 *zone_end_pfn > zone_movable_pfn[nid]) { 7084 *zone_end_pfn = zone_movable_pfn[nid]; 7085 7086 /* Check if this whole range is within ZONE_MOVABLE */ 7087 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 7088 *zone_start_pfn = *zone_end_pfn; 7089 } 7090 } 7091 7092 /* 7093 * Return the number of pages a zone spans in a node, including holes 7094 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 7095 */ 7096 static unsigned long __init zone_spanned_pages_in_node(int nid, 7097 unsigned long zone_type, 7098 unsigned long node_start_pfn, 7099 unsigned long node_end_pfn, 7100 unsigned long *zone_start_pfn, 7101 unsigned long *zone_end_pfn) 7102 { 7103 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 7104 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 7105 /* When hotadd a new node from cpu_up(), the node should be empty */ 7106 if (!node_start_pfn && !node_end_pfn) 7107 return 0; 7108 7109 /* Get the start and end of the zone */ 7110 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 7111 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 7112 adjust_zone_range_for_zone_movable(nid, zone_type, 7113 node_start_pfn, node_end_pfn, 7114 zone_start_pfn, zone_end_pfn); 7115 7116 /* Check that this node has pages within the zone's required range */ 7117 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 7118 return 0; 7119 7120 /* Move the zone boundaries inside the node if necessary */ 7121 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 7122 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 7123 7124 /* Return the spanned pages */ 7125 return *zone_end_pfn - *zone_start_pfn; 7126 } 7127 7128 /* 7129 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 7130 * then all holes in the requested range will be accounted for. 7131 */ 7132 unsigned long __init __absent_pages_in_range(int nid, 7133 unsigned long range_start_pfn, 7134 unsigned long range_end_pfn) 7135 { 7136 unsigned long nr_absent = range_end_pfn - range_start_pfn; 7137 unsigned long start_pfn, end_pfn; 7138 int i; 7139 7140 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7141 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 7142 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 7143 nr_absent -= end_pfn - start_pfn; 7144 } 7145 return nr_absent; 7146 } 7147 7148 /** 7149 * absent_pages_in_range - Return number of page frames in holes within a range 7150 * @start_pfn: The start PFN to start searching for holes 7151 * @end_pfn: The end PFN to stop searching for holes 7152 * 7153 * Return: the number of pages frames in memory holes within a range. 7154 */ 7155 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 7156 unsigned long end_pfn) 7157 { 7158 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 7159 } 7160 7161 /* Return the number of page frames in holes in a zone on a node */ 7162 static unsigned long __init zone_absent_pages_in_node(int nid, 7163 unsigned long zone_type, 7164 unsigned long node_start_pfn, 7165 unsigned long node_end_pfn) 7166 { 7167 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 7168 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 7169 unsigned long zone_start_pfn, zone_end_pfn; 7170 unsigned long nr_absent; 7171 7172 /* When hotadd a new node from cpu_up(), the node should be empty */ 7173 if (!node_start_pfn && !node_end_pfn) 7174 return 0; 7175 7176 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 7177 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 7178 7179 adjust_zone_range_for_zone_movable(nid, zone_type, 7180 node_start_pfn, node_end_pfn, 7181 &zone_start_pfn, &zone_end_pfn); 7182 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 7183 7184 /* 7185 * ZONE_MOVABLE handling. 7186 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 7187 * and vice versa. 7188 */ 7189 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 7190 unsigned long start_pfn, end_pfn; 7191 struct memblock_region *r; 7192 7193 for_each_mem_region(r) { 7194 start_pfn = clamp(memblock_region_memory_base_pfn(r), 7195 zone_start_pfn, zone_end_pfn); 7196 end_pfn = clamp(memblock_region_memory_end_pfn(r), 7197 zone_start_pfn, zone_end_pfn); 7198 7199 if (zone_type == ZONE_MOVABLE && 7200 memblock_is_mirror(r)) 7201 nr_absent += end_pfn - start_pfn; 7202 7203 if (zone_type == ZONE_NORMAL && 7204 !memblock_is_mirror(r)) 7205 nr_absent += end_pfn - start_pfn; 7206 } 7207 } 7208 7209 return nr_absent; 7210 } 7211 7212 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 7213 unsigned long node_start_pfn, 7214 unsigned long node_end_pfn) 7215 { 7216 unsigned long realtotalpages = 0, totalpages = 0; 7217 enum zone_type i; 7218 7219 for (i = 0; i < MAX_NR_ZONES; i++) { 7220 struct zone *zone = pgdat->node_zones + i; 7221 unsigned long zone_start_pfn, zone_end_pfn; 7222 unsigned long spanned, absent; 7223 unsigned long size, real_size; 7224 7225 spanned = zone_spanned_pages_in_node(pgdat->node_id, i, 7226 node_start_pfn, 7227 node_end_pfn, 7228 &zone_start_pfn, 7229 &zone_end_pfn); 7230 absent = zone_absent_pages_in_node(pgdat->node_id, i, 7231 node_start_pfn, 7232 node_end_pfn); 7233 7234 size = spanned; 7235 real_size = size - absent; 7236 7237 if (size) 7238 zone->zone_start_pfn = zone_start_pfn; 7239 else 7240 zone->zone_start_pfn = 0; 7241 zone->spanned_pages = size; 7242 zone->present_pages = real_size; 7243 #if defined(CONFIG_MEMORY_HOTPLUG) 7244 zone->present_early_pages = real_size; 7245 #endif 7246 7247 totalpages += size; 7248 realtotalpages += real_size; 7249 } 7250 7251 pgdat->node_spanned_pages = totalpages; 7252 pgdat->node_present_pages = realtotalpages; 7253 pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); 7254 } 7255 7256 #ifndef CONFIG_SPARSEMEM 7257 /* 7258 * Calculate the size of the zone->blockflags rounded to an unsigned long 7259 * Start by making sure zonesize is a multiple of pageblock_order by rounding 7260 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 7261 * round what is now in bits to nearest long in bits, then return it in 7262 * bytes. 7263 */ 7264 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 7265 { 7266 unsigned long usemapsize; 7267 7268 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 7269 usemapsize = roundup(zonesize, pageblock_nr_pages); 7270 usemapsize = usemapsize >> pageblock_order; 7271 usemapsize *= NR_PAGEBLOCK_BITS; 7272 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 7273 7274 return usemapsize / 8; 7275 } 7276 7277 static void __ref setup_usemap(struct zone *zone) 7278 { 7279 unsigned long usemapsize = usemap_size(zone->zone_start_pfn, 7280 zone->spanned_pages); 7281 zone->pageblock_flags = NULL; 7282 if (usemapsize) { 7283 zone->pageblock_flags = 7284 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 7285 zone_to_nid(zone)); 7286 if (!zone->pageblock_flags) 7287 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 7288 usemapsize, zone->name, zone_to_nid(zone)); 7289 } 7290 } 7291 #else 7292 static inline void setup_usemap(struct zone *zone) {} 7293 #endif /* CONFIG_SPARSEMEM */ 7294 7295 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 7296 7297 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 7298 void __init set_pageblock_order(void) 7299 { 7300 unsigned int order; 7301 7302 /* Check that pageblock_nr_pages has not already been setup */ 7303 if (pageblock_order) 7304 return; 7305 7306 if (HPAGE_SHIFT > PAGE_SHIFT) 7307 order = HUGETLB_PAGE_ORDER; 7308 else 7309 order = MAX_ORDER - 1; 7310 7311 /* 7312 * Assume the largest contiguous order of interest is a huge page. 7313 * This value may be variable depending on boot parameters on IA64 and 7314 * powerpc. 7315 */ 7316 pageblock_order = order; 7317 } 7318 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 7319 7320 /* 7321 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 7322 * is unused as pageblock_order is set at compile-time. See 7323 * include/linux/pageblock-flags.h for the values of pageblock_order based on 7324 * the kernel config 7325 */ 7326 void __init set_pageblock_order(void) 7327 { 7328 } 7329 7330 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 7331 7332 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 7333 unsigned long present_pages) 7334 { 7335 unsigned long pages = spanned_pages; 7336 7337 /* 7338 * Provide a more accurate estimation if there are holes within 7339 * the zone and SPARSEMEM is in use. If there are holes within the 7340 * zone, each populated memory region may cost us one or two extra 7341 * memmap pages due to alignment because memmap pages for each 7342 * populated regions may not be naturally aligned on page boundary. 7343 * So the (present_pages >> 4) heuristic is a tradeoff for that. 7344 */ 7345 if (spanned_pages > present_pages + (present_pages >> 4) && 7346 IS_ENABLED(CONFIG_SPARSEMEM)) 7347 pages = present_pages; 7348 7349 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 7350 } 7351 7352 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 7353 static void pgdat_init_split_queue(struct pglist_data *pgdat) 7354 { 7355 struct deferred_split *ds_queue = &pgdat->deferred_split_queue; 7356 7357 spin_lock_init(&ds_queue->split_queue_lock); 7358 INIT_LIST_HEAD(&ds_queue->split_queue); 7359 ds_queue->split_queue_len = 0; 7360 } 7361 #else 7362 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 7363 #endif 7364 7365 #ifdef CONFIG_COMPACTION 7366 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 7367 { 7368 init_waitqueue_head(&pgdat->kcompactd_wait); 7369 } 7370 #else 7371 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 7372 #endif 7373 7374 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 7375 { 7376 pgdat_resize_init(pgdat); 7377 7378 pgdat_init_split_queue(pgdat); 7379 pgdat_init_kcompactd(pgdat); 7380 7381 init_waitqueue_head(&pgdat->kswapd_wait); 7382 init_waitqueue_head(&pgdat->pfmemalloc_wait); 7383 7384 pgdat_page_ext_init(pgdat); 7385 lruvec_init(&pgdat->__lruvec); 7386 } 7387 7388 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 7389 unsigned long remaining_pages) 7390 { 7391 atomic_long_set(&zone->managed_pages, remaining_pages); 7392 zone_set_nid(zone, nid); 7393 zone->name = zone_names[idx]; 7394 zone->zone_pgdat = NODE_DATA(nid); 7395 spin_lock_init(&zone->lock); 7396 zone_seqlock_init(zone); 7397 zone_pcp_init(zone); 7398 } 7399 7400 /* 7401 * Set up the zone data structures 7402 * - init pgdat internals 7403 * - init all zones belonging to this node 7404 * 7405 * NOTE: this function is only called during memory hotplug 7406 */ 7407 #ifdef CONFIG_MEMORY_HOTPLUG 7408 void __ref free_area_init_core_hotplug(int nid) 7409 { 7410 enum zone_type z; 7411 pg_data_t *pgdat = NODE_DATA(nid); 7412 7413 pgdat_init_internals(pgdat); 7414 for (z = 0; z < MAX_NR_ZONES; z++) 7415 zone_init_internals(&pgdat->node_zones[z], z, nid, 0); 7416 } 7417 #endif 7418 7419 /* 7420 * Set up the zone data structures: 7421 * - mark all pages reserved 7422 * - mark all memory queues empty 7423 * - clear the memory bitmaps 7424 * 7425 * NOTE: pgdat should get zeroed by caller. 7426 * NOTE: this function is only called during early init. 7427 */ 7428 static void __init free_area_init_core(struct pglist_data *pgdat) 7429 { 7430 enum zone_type j; 7431 int nid = pgdat->node_id; 7432 7433 pgdat_init_internals(pgdat); 7434 pgdat->per_cpu_nodestats = &boot_nodestats; 7435 7436 for (j = 0; j < MAX_NR_ZONES; j++) { 7437 struct zone *zone = pgdat->node_zones + j; 7438 unsigned long size, freesize, memmap_pages; 7439 7440 size = zone->spanned_pages; 7441 freesize = zone->present_pages; 7442 7443 /* 7444 * Adjust freesize so that it accounts for how much memory 7445 * is used by this zone for memmap. This affects the watermark 7446 * and per-cpu initialisations 7447 */ 7448 memmap_pages = calc_memmap_size(size, freesize); 7449 if (!is_highmem_idx(j)) { 7450 if (freesize >= memmap_pages) { 7451 freesize -= memmap_pages; 7452 if (memmap_pages) 7453 pr_debug(" %s zone: %lu pages used for memmap\n", 7454 zone_names[j], memmap_pages); 7455 } else 7456 pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n", 7457 zone_names[j], memmap_pages, freesize); 7458 } 7459 7460 /* Account for reserved pages */ 7461 if (j == 0 && freesize > dma_reserve) { 7462 freesize -= dma_reserve; 7463 pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve); 7464 } 7465 7466 if (!is_highmem_idx(j)) 7467 nr_kernel_pages += freesize; 7468 /* Charge for highmem memmap if there are enough kernel pages */ 7469 else if (nr_kernel_pages > memmap_pages * 2) 7470 nr_kernel_pages -= memmap_pages; 7471 nr_all_pages += freesize; 7472 7473 /* 7474 * Set an approximate value for lowmem here, it will be adjusted 7475 * when the bootmem allocator frees pages into the buddy system. 7476 * And all highmem pages will be managed by the buddy system. 7477 */ 7478 zone_init_internals(zone, j, nid, freesize); 7479 7480 if (!size) 7481 continue; 7482 7483 set_pageblock_order(); 7484 setup_usemap(zone); 7485 init_currently_empty_zone(zone, zone->zone_start_pfn, size); 7486 } 7487 } 7488 7489 #ifdef CONFIG_FLATMEM 7490 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) 7491 { 7492 unsigned long __maybe_unused start = 0; 7493 unsigned long __maybe_unused offset = 0; 7494 7495 /* Skip empty nodes */ 7496 if (!pgdat->node_spanned_pages) 7497 return; 7498 7499 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 7500 offset = pgdat->node_start_pfn - start; 7501 /* ia64 gets its own node_mem_map, before this, without bootmem */ 7502 if (!pgdat->node_mem_map) { 7503 unsigned long size, end; 7504 struct page *map; 7505 7506 /* 7507 * The zone's endpoints aren't required to be MAX_ORDER 7508 * aligned but the node_mem_map endpoints must be in order 7509 * for the buddy allocator to function correctly. 7510 */ 7511 end = pgdat_end_pfn(pgdat); 7512 end = ALIGN(end, MAX_ORDER_NR_PAGES); 7513 size = (end - start) * sizeof(struct page); 7514 map = memblock_alloc_node(size, SMP_CACHE_BYTES, 7515 pgdat->node_id); 7516 if (!map) 7517 panic("Failed to allocate %ld bytes for node %d memory map\n", 7518 size, pgdat->node_id); 7519 pgdat->node_mem_map = map + offset; 7520 } 7521 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 7522 __func__, pgdat->node_id, (unsigned long)pgdat, 7523 (unsigned long)pgdat->node_mem_map); 7524 #ifndef CONFIG_NUMA 7525 /* 7526 * With no DISCONTIG, the global mem_map is just set as node 0's 7527 */ 7528 if (pgdat == NODE_DATA(0)) { 7529 mem_map = NODE_DATA(0)->node_mem_map; 7530 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 7531 mem_map -= offset; 7532 } 7533 #endif 7534 } 7535 #else 7536 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { } 7537 #endif /* CONFIG_FLATMEM */ 7538 7539 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 7540 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 7541 { 7542 pgdat->first_deferred_pfn = ULONG_MAX; 7543 } 7544 #else 7545 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 7546 #endif 7547 7548 static void __init free_area_init_node(int nid) 7549 { 7550 pg_data_t *pgdat = NODE_DATA(nid); 7551 unsigned long start_pfn = 0; 7552 unsigned long end_pfn = 0; 7553 7554 /* pg_data_t should be reset to zero when it's allocated */ 7555 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); 7556 7557 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 7558 7559 pgdat->node_id = nid; 7560 pgdat->node_start_pfn = start_pfn; 7561 pgdat->per_cpu_nodestats = NULL; 7562 7563 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 7564 (u64)start_pfn << PAGE_SHIFT, 7565 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 7566 calculate_node_totalpages(pgdat, start_pfn, end_pfn); 7567 7568 alloc_node_mem_map(pgdat); 7569 pgdat_set_deferred_range(pgdat); 7570 7571 free_area_init_core(pgdat); 7572 } 7573 7574 void __init free_area_init_memoryless_node(int nid) 7575 { 7576 free_area_init_node(nid); 7577 } 7578 7579 #if MAX_NUMNODES > 1 7580 /* 7581 * Figure out the number of possible node ids. 7582 */ 7583 void __init setup_nr_node_ids(void) 7584 { 7585 unsigned int highest; 7586 7587 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 7588 nr_node_ids = highest + 1; 7589 } 7590 #endif 7591 7592 /** 7593 * node_map_pfn_alignment - determine the maximum internode alignment 7594 * 7595 * This function should be called after node map is populated and sorted. 7596 * It calculates the maximum power of two alignment which can distinguish 7597 * all the nodes. 7598 * 7599 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 7600 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 7601 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 7602 * shifted, 1GiB is enough and this function will indicate so. 7603 * 7604 * This is used to test whether pfn -> nid mapping of the chosen memory 7605 * model has fine enough granularity to avoid incorrect mapping for the 7606 * populated node map. 7607 * 7608 * Return: the determined alignment in pfn's. 0 if there is no alignment 7609 * requirement (single node). 7610 */ 7611 unsigned long __init node_map_pfn_alignment(void) 7612 { 7613 unsigned long accl_mask = 0, last_end = 0; 7614 unsigned long start, end, mask; 7615 int last_nid = NUMA_NO_NODE; 7616 int i, nid; 7617 7618 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 7619 if (!start || last_nid < 0 || last_nid == nid) { 7620 last_nid = nid; 7621 last_end = end; 7622 continue; 7623 } 7624 7625 /* 7626 * Start with a mask granular enough to pin-point to the 7627 * start pfn and tick off bits one-by-one until it becomes 7628 * too coarse to separate the current node from the last. 7629 */ 7630 mask = ~((1 << __ffs(start)) - 1); 7631 while (mask && last_end <= (start & (mask << 1))) 7632 mask <<= 1; 7633 7634 /* accumulate all internode masks */ 7635 accl_mask |= mask; 7636 } 7637 7638 /* convert mask to number of pages */ 7639 return ~accl_mask + 1; 7640 } 7641 7642 /** 7643 * find_min_pfn_with_active_regions - Find the minimum PFN registered 7644 * 7645 * Return: the minimum PFN based on information provided via 7646 * memblock_set_node(). 7647 */ 7648 unsigned long __init find_min_pfn_with_active_regions(void) 7649 { 7650 return PHYS_PFN(memblock_start_of_DRAM()); 7651 } 7652 7653 /* 7654 * early_calculate_totalpages() 7655 * Sum pages in active regions for movable zone. 7656 * Populate N_MEMORY for calculating usable_nodes. 7657 */ 7658 static unsigned long __init early_calculate_totalpages(void) 7659 { 7660 unsigned long totalpages = 0; 7661 unsigned long start_pfn, end_pfn; 7662 int i, nid; 7663 7664 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7665 unsigned long pages = end_pfn - start_pfn; 7666 7667 totalpages += pages; 7668 if (pages) 7669 node_set_state(nid, N_MEMORY); 7670 } 7671 return totalpages; 7672 } 7673 7674 /* 7675 * Find the PFN the Movable zone begins in each node. Kernel memory 7676 * is spread evenly between nodes as long as the nodes have enough 7677 * memory. When they don't, some nodes will have more kernelcore than 7678 * others 7679 */ 7680 static void __init find_zone_movable_pfns_for_nodes(void) 7681 { 7682 int i, nid; 7683 unsigned long usable_startpfn; 7684 unsigned long kernelcore_node, kernelcore_remaining; 7685 /* save the state before borrow the nodemask */ 7686 nodemask_t saved_node_state = node_states[N_MEMORY]; 7687 unsigned long totalpages = early_calculate_totalpages(); 7688 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 7689 struct memblock_region *r; 7690 7691 /* Need to find movable_zone earlier when movable_node is specified. */ 7692 find_usable_zone_for_movable(); 7693 7694 /* 7695 * If movable_node is specified, ignore kernelcore and movablecore 7696 * options. 7697 */ 7698 if (movable_node_is_enabled()) { 7699 for_each_mem_region(r) { 7700 if (!memblock_is_hotpluggable(r)) 7701 continue; 7702 7703 nid = memblock_get_region_node(r); 7704 7705 usable_startpfn = PFN_DOWN(r->base); 7706 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7707 min(usable_startpfn, zone_movable_pfn[nid]) : 7708 usable_startpfn; 7709 } 7710 7711 goto out2; 7712 } 7713 7714 /* 7715 * If kernelcore=mirror is specified, ignore movablecore option 7716 */ 7717 if (mirrored_kernelcore) { 7718 bool mem_below_4gb_not_mirrored = false; 7719 7720 for_each_mem_region(r) { 7721 if (memblock_is_mirror(r)) 7722 continue; 7723 7724 nid = memblock_get_region_node(r); 7725 7726 usable_startpfn = memblock_region_memory_base_pfn(r); 7727 7728 if (usable_startpfn < 0x100000) { 7729 mem_below_4gb_not_mirrored = true; 7730 continue; 7731 } 7732 7733 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7734 min(usable_startpfn, zone_movable_pfn[nid]) : 7735 usable_startpfn; 7736 } 7737 7738 if (mem_below_4gb_not_mirrored) 7739 pr_warn("This configuration results in unmirrored kernel memory.\n"); 7740 7741 goto out2; 7742 } 7743 7744 /* 7745 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 7746 * amount of necessary memory. 7747 */ 7748 if (required_kernelcore_percent) 7749 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 7750 10000UL; 7751 if (required_movablecore_percent) 7752 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 7753 10000UL; 7754 7755 /* 7756 * If movablecore= was specified, calculate what size of 7757 * kernelcore that corresponds so that memory usable for 7758 * any allocation type is evenly spread. If both kernelcore 7759 * and movablecore are specified, then the value of kernelcore 7760 * will be used for required_kernelcore if it's greater than 7761 * what movablecore would have allowed. 7762 */ 7763 if (required_movablecore) { 7764 unsigned long corepages; 7765 7766 /* 7767 * Round-up so that ZONE_MOVABLE is at least as large as what 7768 * was requested by the user 7769 */ 7770 required_movablecore = 7771 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 7772 required_movablecore = min(totalpages, required_movablecore); 7773 corepages = totalpages - required_movablecore; 7774 7775 required_kernelcore = max(required_kernelcore, corepages); 7776 } 7777 7778 /* 7779 * If kernelcore was not specified or kernelcore size is larger 7780 * than totalpages, there is no ZONE_MOVABLE. 7781 */ 7782 if (!required_kernelcore || required_kernelcore >= totalpages) 7783 goto out; 7784 7785 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 7786 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 7787 7788 restart: 7789 /* Spread kernelcore memory as evenly as possible throughout nodes */ 7790 kernelcore_node = required_kernelcore / usable_nodes; 7791 for_each_node_state(nid, N_MEMORY) { 7792 unsigned long start_pfn, end_pfn; 7793 7794 /* 7795 * Recalculate kernelcore_node if the division per node 7796 * now exceeds what is necessary to satisfy the requested 7797 * amount of memory for the kernel 7798 */ 7799 if (required_kernelcore < kernelcore_node) 7800 kernelcore_node = required_kernelcore / usable_nodes; 7801 7802 /* 7803 * As the map is walked, we track how much memory is usable 7804 * by the kernel using kernelcore_remaining. When it is 7805 * 0, the rest of the node is usable by ZONE_MOVABLE 7806 */ 7807 kernelcore_remaining = kernelcore_node; 7808 7809 /* Go through each range of PFNs within this node */ 7810 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7811 unsigned long size_pages; 7812 7813 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 7814 if (start_pfn >= end_pfn) 7815 continue; 7816 7817 /* Account for what is only usable for kernelcore */ 7818 if (start_pfn < usable_startpfn) { 7819 unsigned long kernel_pages; 7820 kernel_pages = min(end_pfn, usable_startpfn) 7821 - start_pfn; 7822 7823 kernelcore_remaining -= min(kernel_pages, 7824 kernelcore_remaining); 7825 required_kernelcore -= min(kernel_pages, 7826 required_kernelcore); 7827 7828 /* Continue if range is now fully accounted */ 7829 if (end_pfn <= usable_startpfn) { 7830 7831 /* 7832 * Push zone_movable_pfn to the end so 7833 * that if we have to rebalance 7834 * kernelcore across nodes, we will 7835 * not double account here 7836 */ 7837 zone_movable_pfn[nid] = end_pfn; 7838 continue; 7839 } 7840 start_pfn = usable_startpfn; 7841 } 7842 7843 /* 7844 * The usable PFN range for ZONE_MOVABLE is from 7845 * start_pfn->end_pfn. Calculate size_pages as the 7846 * number of pages used as kernelcore 7847 */ 7848 size_pages = end_pfn - start_pfn; 7849 if (size_pages > kernelcore_remaining) 7850 size_pages = kernelcore_remaining; 7851 zone_movable_pfn[nid] = start_pfn + size_pages; 7852 7853 /* 7854 * Some kernelcore has been met, update counts and 7855 * break if the kernelcore for this node has been 7856 * satisfied 7857 */ 7858 required_kernelcore -= min(required_kernelcore, 7859 size_pages); 7860 kernelcore_remaining -= size_pages; 7861 if (!kernelcore_remaining) 7862 break; 7863 } 7864 } 7865 7866 /* 7867 * If there is still required_kernelcore, we do another pass with one 7868 * less node in the count. This will push zone_movable_pfn[nid] further 7869 * along on the nodes that still have memory until kernelcore is 7870 * satisfied 7871 */ 7872 usable_nodes--; 7873 if (usable_nodes && required_kernelcore > usable_nodes) 7874 goto restart; 7875 7876 out2: 7877 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 7878 for (nid = 0; nid < MAX_NUMNODES; nid++) 7879 zone_movable_pfn[nid] = 7880 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 7881 7882 out: 7883 /* restore the node_state */ 7884 node_states[N_MEMORY] = saved_node_state; 7885 } 7886 7887 /* Any regular or high memory on that node ? */ 7888 static void check_for_memory(pg_data_t *pgdat, int nid) 7889 { 7890 enum zone_type zone_type; 7891 7892 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 7893 struct zone *zone = &pgdat->node_zones[zone_type]; 7894 if (populated_zone(zone)) { 7895 if (IS_ENABLED(CONFIG_HIGHMEM)) 7896 node_set_state(nid, N_HIGH_MEMORY); 7897 if (zone_type <= ZONE_NORMAL) 7898 node_set_state(nid, N_NORMAL_MEMORY); 7899 break; 7900 } 7901 } 7902 } 7903 7904 /* 7905 * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For 7906 * such cases we allow max_zone_pfn sorted in the descending order 7907 */ 7908 bool __weak arch_has_descending_max_zone_pfns(void) 7909 { 7910 return false; 7911 } 7912 7913 /** 7914 * free_area_init - Initialise all pg_data_t and zone data 7915 * @max_zone_pfn: an array of max PFNs for each zone 7916 * 7917 * This will call free_area_init_node() for each active node in the system. 7918 * Using the page ranges provided by memblock_set_node(), the size of each 7919 * zone in each node and their holes is calculated. If the maximum PFN 7920 * between two adjacent zones match, it is assumed that the zone is empty. 7921 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 7922 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 7923 * starts where the previous one ended. For example, ZONE_DMA32 starts 7924 * at arch_max_dma_pfn. 7925 */ 7926 void __init free_area_init(unsigned long *max_zone_pfn) 7927 { 7928 unsigned long start_pfn, end_pfn; 7929 int i, nid, zone; 7930 bool descending; 7931 7932 /* Record where the zone boundaries are */ 7933 memset(arch_zone_lowest_possible_pfn, 0, 7934 sizeof(arch_zone_lowest_possible_pfn)); 7935 memset(arch_zone_highest_possible_pfn, 0, 7936 sizeof(arch_zone_highest_possible_pfn)); 7937 7938 start_pfn = find_min_pfn_with_active_regions(); 7939 descending = arch_has_descending_max_zone_pfns(); 7940 7941 for (i = 0; i < MAX_NR_ZONES; i++) { 7942 if (descending) 7943 zone = MAX_NR_ZONES - i - 1; 7944 else 7945 zone = i; 7946 7947 if (zone == ZONE_MOVABLE) 7948 continue; 7949 7950 end_pfn = max(max_zone_pfn[zone], start_pfn); 7951 arch_zone_lowest_possible_pfn[zone] = start_pfn; 7952 arch_zone_highest_possible_pfn[zone] = end_pfn; 7953 7954 start_pfn = end_pfn; 7955 } 7956 7957 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 7958 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 7959 find_zone_movable_pfns_for_nodes(); 7960 7961 /* Print out the zone ranges */ 7962 pr_info("Zone ranges:\n"); 7963 for (i = 0; i < MAX_NR_ZONES; i++) { 7964 if (i == ZONE_MOVABLE) 7965 continue; 7966 pr_info(" %-8s ", zone_names[i]); 7967 if (arch_zone_lowest_possible_pfn[i] == 7968 arch_zone_highest_possible_pfn[i]) 7969 pr_cont("empty\n"); 7970 else 7971 pr_cont("[mem %#018Lx-%#018Lx]\n", 7972 (u64)arch_zone_lowest_possible_pfn[i] 7973 << PAGE_SHIFT, 7974 ((u64)arch_zone_highest_possible_pfn[i] 7975 << PAGE_SHIFT) - 1); 7976 } 7977 7978 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 7979 pr_info("Movable zone start for each node\n"); 7980 for (i = 0; i < MAX_NUMNODES; i++) { 7981 if (zone_movable_pfn[i]) 7982 pr_info(" Node %d: %#018Lx\n", i, 7983 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 7984 } 7985 7986 /* 7987 * Print out the early node map, and initialize the 7988 * subsection-map relative to active online memory ranges to 7989 * enable future "sub-section" extensions of the memory map. 7990 */ 7991 pr_info("Early memory node ranges\n"); 7992 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7993 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 7994 (u64)start_pfn << PAGE_SHIFT, 7995 ((u64)end_pfn << PAGE_SHIFT) - 1); 7996 subsection_map_init(start_pfn, end_pfn - start_pfn); 7997 } 7998 7999 /* Initialise every node */ 8000 mminit_verify_pageflags_layout(); 8001 setup_nr_node_ids(); 8002 for_each_online_node(nid) { 8003 pg_data_t *pgdat = NODE_DATA(nid); 8004 free_area_init_node(nid); 8005 8006 /* Any memory on that node */ 8007 if (pgdat->node_present_pages) 8008 node_set_state(nid, N_MEMORY); 8009 check_for_memory(pgdat, nid); 8010 } 8011 8012 memmap_init(); 8013 } 8014 8015 static int __init cmdline_parse_core(char *p, unsigned long *core, 8016 unsigned long *percent) 8017 { 8018 unsigned long long coremem; 8019 char *endptr; 8020 8021 if (!p) 8022 return -EINVAL; 8023 8024 /* Value may be a percentage of total memory, otherwise bytes */ 8025 coremem = simple_strtoull(p, &endptr, 0); 8026 if (*endptr == '%') { 8027 /* Paranoid check for percent values greater than 100 */ 8028 WARN_ON(coremem > 100); 8029 8030 *percent = coremem; 8031 } else { 8032 coremem = memparse(p, &p); 8033 /* Paranoid check that UL is enough for the coremem value */ 8034 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 8035 8036 *core = coremem >> PAGE_SHIFT; 8037 *percent = 0UL; 8038 } 8039 return 0; 8040 } 8041 8042 /* 8043 * kernelcore=size sets the amount of memory for use for allocations that 8044 * cannot be reclaimed or migrated. 8045 */ 8046 static int __init cmdline_parse_kernelcore(char *p) 8047 { 8048 /* parse kernelcore=mirror */ 8049 if (parse_option_str(p, "mirror")) { 8050 mirrored_kernelcore = true; 8051 return 0; 8052 } 8053 8054 return cmdline_parse_core(p, &required_kernelcore, 8055 &required_kernelcore_percent); 8056 } 8057 8058 /* 8059 * movablecore=size sets the amount of memory for use for allocations that 8060 * can be reclaimed or migrated. 8061 */ 8062 static int __init cmdline_parse_movablecore(char *p) 8063 { 8064 return cmdline_parse_core(p, &required_movablecore, 8065 &required_movablecore_percent); 8066 } 8067 8068 early_param("kernelcore", cmdline_parse_kernelcore); 8069 early_param("movablecore", cmdline_parse_movablecore); 8070 8071 void adjust_managed_page_count(struct page *page, long count) 8072 { 8073 atomic_long_add(count, &page_zone(page)->managed_pages); 8074 totalram_pages_add(count); 8075 #ifdef CONFIG_HIGHMEM 8076 if (PageHighMem(page)) 8077 totalhigh_pages_add(count); 8078 #endif 8079 } 8080 EXPORT_SYMBOL(adjust_managed_page_count); 8081 8082 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 8083 { 8084 void *pos; 8085 unsigned long pages = 0; 8086 8087 start = (void *)PAGE_ALIGN((unsigned long)start); 8088 end = (void *)((unsigned long)end & PAGE_MASK); 8089 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 8090 struct page *page = virt_to_page(pos); 8091 void *direct_map_addr; 8092 8093 /* 8094 * 'direct_map_addr' might be different from 'pos' 8095 * because some architectures' virt_to_page() 8096 * work with aliases. Getting the direct map 8097 * address ensures that we get a _writeable_ 8098 * alias for the memset(). 8099 */ 8100 direct_map_addr = page_address(page); 8101 /* 8102 * Perform a kasan-unchecked memset() since this memory 8103 * has not been initialized. 8104 */ 8105 direct_map_addr = kasan_reset_tag(direct_map_addr); 8106 if ((unsigned int)poison <= 0xFF) 8107 memset(direct_map_addr, poison, PAGE_SIZE); 8108 8109 free_reserved_page(page); 8110 } 8111 8112 if (pages && s) 8113 pr_info("Freeing %s memory: %ldK\n", 8114 s, pages << (PAGE_SHIFT - 10)); 8115 8116 return pages; 8117 } 8118 8119 void __init mem_init_print_info(void) 8120 { 8121 unsigned long physpages, codesize, datasize, rosize, bss_size; 8122 unsigned long init_code_size, init_data_size; 8123 8124 physpages = get_num_physpages(); 8125 codesize = _etext - _stext; 8126 datasize = _edata - _sdata; 8127 rosize = __end_rodata - __start_rodata; 8128 bss_size = __bss_stop - __bss_start; 8129 init_data_size = __init_end - __init_begin; 8130 init_code_size = _einittext - _sinittext; 8131 8132 /* 8133 * Detect special cases and adjust section sizes accordingly: 8134 * 1) .init.* may be embedded into .data sections 8135 * 2) .init.text.* may be out of [__init_begin, __init_end], 8136 * please refer to arch/tile/kernel/vmlinux.lds.S. 8137 * 3) .rodata.* may be embedded into .text or .data sections. 8138 */ 8139 #define adj_init_size(start, end, size, pos, adj) \ 8140 do { \ 8141 if (start <= pos && pos < end && size > adj) \ 8142 size -= adj; \ 8143 } while (0) 8144 8145 adj_init_size(__init_begin, __init_end, init_data_size, 8146 _sinittext, init_code_size); 8147 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 8148 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 8149 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 8150 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 8151 8152 #undef adj_init_size 8153 8154 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 8155 #ifdef CONFIG_HIGHMEM 8156 ", %luK highmem" 8157 #endif 8158 ")\n", 8159 nr_free_pages() << (PAGE_SHIFT - 10), 8160 physpages << (PAGE_SHIFT - 10), 8161 codesize >> 10, datasize >> 10, rosize >> 10, 8162 (init_data_size + init_code_size) >> 10, bss_size >> 10, 8163 (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10), 8164 totalcma_pages << (PAGE_SHIFT - 10) 8165 #ifdef CONFIG_HIGHMEM 8166 , totalhigh_pages() << (PAGE_SHIFT - 10) 8167 #endif 8168 ); 8169 } 8170 8171 /** 8172 * set_dma_reserve - set the specified number of pages reserved in the first zone 8173 * @new_dma_reserve: The number of pages to mark reserved 8174 * 8175 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 8176 * In the DMA zone, a significant percentage may be consumed by kernel image 8177 * and other unfreeable allocations which can skew the watermarks badly. This 8178 * function may optionally be used to account for unfreeable pages in the 8179 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 8180 * smaller per-cpu batchsize. 8181 */ 8182 void __init set_dma_reserve(unsigned long new_dma_reserve) 8183 { 8184 dma_reserve = new_dma_reserve; 8185 } 8186 8187 static int page_alloc_cpu_dead(unsigned int cpu) 8188 { 8189 struct zone *zone; 8190 8191 lru_add_drain_cpu(cpu); 8192 drain_pages(cpu); 8193 8194 /* 8195 * Spill the event counters of the dead processor 8196 * into the current processors event counters. 8197 * This artificially elevates the count of the current 8198 * processor. 8199 */ 8200 vm_events_fold_cpu(cpu); 8201 8202 /* 8203 * Zero the differential counters of the dead processor 8204 * so that the vm statistics are consistent. 8205 * 8206 * This is only okay since the processor is dead and cannot 8207 * race with what we are doing. 8208 */ 8209 cpu_vm_stats_fold(cpu); 8210 8211 for_each_populated_zone(zone) 8212 zone_pcp_update(zone, 0); 8213 8214 return 0; 8215 } 8216 8217 static int page_alloc_cpu_online(unsigned int cpu) 8218 { 8219 struct zone *zone; 8220 8221 for_each_populated_zone(zone) 8222 zone_pcp_update(zone, 1); 8223 return 0; 8224 } 8225 8226 #ifdef CONFIG_NUMA 8227 int hashdist = HASHDIST_DEFAULT; 8228 8229 static int __init set_hashdist(char *str) 8230 { 8231 if (!str) 8232 return 0; 8233 hashdist = simple_strtoul(str, &str, 0); 8234 return 1; 8235 } 8236 __setup("hashdist=", set_hashdist); 8237 #endif 8238 8239 void __init page_alloc_init(void) 8240 { 8241 int ret; 8242 8243 #ifdef CONFIG_NUMA 8244 if (num_node_state(N_MEMORY) == 1) 8245 hashdist = 0; 8246 #endif 8247 8248 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC, 8249 "mm/page_alloc:pcp", 8250 page_alloc_cpu_online, 8251 page_alloc_cpu_dead); 8252 WARN_ON(ret < 0); 8253 } 8254 8255 /* 8256 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 8257 * or min_free_kbytes changes. 8258 */ 8259 static void calculate_totalreserve_pages(void) 8260 { 8261 struct pglist_data *pgdat; 8262 unsigned long reserve_pages = 0; 8263 enum zone_type i, j; 8264 8265 for_each_online_pgdat(pgdat) { 8266 8267 pgdat->totalreserve_pages = 0; 8268 8269 for (i = 0; i < MAX_NR_ZONES; i++) { 8270 struct zone *zone = pgdat->node_zones + i; 8271 long max = 0; 8272 unsigned long managed_pages = zone_managed_pages(zone); 8273 8274 /* Find valid and maximum lowmem_reserve in the zone */ 8275 for (j = i; j < MAX_NR_ZONES; j++) { 8276 if (zone->lowmem_reserve[j] > max) 8277 max = zone->lowmem_reserve[j]; 8278 } 8279 8280 /* we treat the high watermark as reserved pages. */ 8281 max += high_wmark_pages(zone); 8282 8283 if (max > managed_pages) 8284 max = managed_pages; 8285 8286 pgdat->totalreserve_pages += max; 8287 8288 reserve_pages += max; 8289 } 8290 } 8291 totalreserve_pages = reserve_pages; 8292 } 8293 8294 /* 8295 * setup_per_zone_lowmem_reserve - called whenever 8296 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 8297 * has a correct pages reserved value, so an adequate number of 8298 * pages are left in the zone after a successful __alloc_pages(). 8299 */ 8300 static void setup_per_zone_lowmem_reserve(void) 8301 { 8302 struct pglist_data *pgdat; 8303 enum zone_type i, j; 8304 8305 for_each_online_pgdat(pgdat) { 8306 for (i = 0; i < MAX_NR_ZONES - 1; i++) { 8307 struct zone *zone = &pgdat->node_zones[i]; 8308 int ratio = sysctl_lowmem_reserve_ratio[i]; 8309 bool clear = !ratio || !zone_managed_pages(zone); 8310 unsigned long managed_pages = 0; 8311 8312 for (j = i + 1; j < MAX_NR_ZONES; j++) { 8313 struct zone *upper_zone = &pgdat->node_zones[j]; 8314 8315 managed_pages += zone_managed_pages(upper_zone); 8316 8317 if (clear) 8318 zone->lowmem_reserve[j] = 0; 8319 else 8320 zone->lowmem_reserve[j] = managed_pages / ratio; 8321 } 8322 } 8323 } 8324 8325 /* update totalreserve_pages */ 8326 calculate_totalreserve_pages(); 8327 } 8328 8329 static void __setup_per_zone_wmarks(void) 8330 { 8331 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 8332 unsigned long lowmem_pages = 0; 8333 struct zone *zone; 8334 unsigned long flags; 8335 8336 /* Calculate total number of !ZONE_HIGHMEM pages */ 8337 for_each_zone(zone) { 8338 if (!is_highmem(zone)) 8339 lowmem_pages += zone_managed_pages(zone); 8340 } 8341 8342 for_each_zone(zone) { 8343 u64 tmp; 8344 8345 spin_lock_irqsave(&zone->lock, flags); 8346 tmp = (u64)pages_min * zone_managed_pages(zone); 8347 do_div(tmp, lowmem_pages); 8348 if (is_highmem(zone)) { 8349 /* 8350 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 8351 * need highmem pages, so cap pages_min to a small 8352 * value here. 8353 * 8354 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 8355 * deltas control async page reclaim, and so should 8356 * not be capped for highmem. 8357 */ 8358 unsigned long min_pages; 8359 8360 min_pages = zone_managed_pages(zone) / 1024; 8361 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 8362 zone->_watermark[WMARK_MIN] = min_pages; 8363 } else { 8364 /* 8365 * If it's a lowmem zone, reserve a number of pages 8366 * proportionate to the zone's size. 8367 */ 8368 zone->_watermark[WMARK_MIN] = tmp; 8369 } 8370 8371 /* 8372 * Set the kswapd watermarks distance according to the 8373 * scale factor in proportion to available memory, but 8374 * ensure a minimum size on small systems. 8375 */ 8376 tmp = max_t(u64, tmp >> 2, 8377 mult_frac(zone_managed_pages(zone), 8378 watermark_scale_factor, 10000)); 8379 8380 zone->watermark_boost = 0; 8381 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 8382 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2; 8383 8384 spin_unlock_irqrestore(&zone->lock, flags); 8385 } 8386 8387 /* update totalreserve_pages */ 8388 calculate_totalreserve_pages(); 8389 } 8390 8391 /** 8392 * setup_per_zone_wmarks - called when min_free_kbytes changes 8393 * or when memory is hot-{added|removed} 8394 * 8395 * Ensures that the watermark[min,low,high] values for each zone are set 8396 * correctly with respect to min_free_kbytes. 8397 */ 8398 void setup_per_zone_wmarks(void) 8399 { 8400 struct zone *zone; 8401 static DEFINE_SPINLOCK(lock); 8402 8403 spin_lock(&lock); 8404 __setup_per_zone_wmarks(); 8405 spin_unlock(&lock); 8406 8407 /* 8408 * The watermark size have changed so update the pcpu batch 8409 * and high limits or the limits may be inappropriate. 8410 */ 8411 for_each_zone(zone) 8412 zone_pcp_update(zone, 0); 8413 } 8414 8415 /* 8416 * Initialise min_free_kbytes. 8417 * 8418 * For small machines we want it small (128k min). For large machines 8419 * we want it large (256MB max). But it is not linear, because network 8420 * bandwidth does not increase linearly with machine size. We use 8421 * 8422 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 8423 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 8424 * 8425 * which yields 8426 * 8427 * 16MB: 512k 8428 * 32MB: 724k 8429 * 64MB: 1024k 8430 * 128MB: 1448k 8431 * 256MB: 2048k 8432 * 512MB: 2896k 8433 * 1024MB: 4096k 8434 * 2048MB: 5792k 8435 * 4096MB: 8192k 8436 * 8192MB: 11584k 8437 * 16384MB: 16384k 8438 */ 8439 int __meminit init_per_zone_wmark_min(void) 8440 { 8441 unsigned long lowmem_kbytes; 8442 int new_min_free_kbytes; 8443 8444 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 8445 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 8446 8447 if (new_min_free_kbytes > user_min_free_kbytes) { 8448 min_free_kbytes = new_min_free_kbytes; 8449 if (min_free_kbytes < 128) 8450 min_free_kbytes = 128; 8451 if (min_free_kbytes > 262144) 8452 min_free_kbytes = 262144; 8453 } else { 8454 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 8455 new_min_free_kbytes, user_min_free_kbytes); 8456 } 8457 setup_per_zone_wmarks(); 8458 refresh_zone_stat_thresholds(); 8459 setup_per_zone_lowmem_reserve(); 8460 8461 #ifdef CONFIG_NUMA 8462 setup_min_unmapped_ratio(); 8463 setup_min_slab_ratio(); 8464 #endif 8465 8466 khugepaged_min_free_kbytes_update(); 8467 8468 return 0; 8469 } 8470 postcore_initcall(init_per_zone_wmark_min) 8471 8472 /* 8473 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 8474 * that we can call two helper functions whenever min_free_kbytes 8475 * changes. 8476 */ 8477 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 8478 void *buffer, size_t *length, loff_t *ppos) 8479 { 8480 int rc; 8481 8482 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8483 if (rc) 8484 return rc; 8485 8486 if (write) { 8487 user_min_free_kbytes = min_free_kbytes; 8488 setup_per_zone_wmarks(); 8489 } 8490 return 0; 8491 } 8492 8493 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 8494 void *buffer, size_t *length, loff_t *ppos) 8495 { 8496 int rc; 8497 8498 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8499 if (rc) 8500 return rc; 8501 8502 if (write) 8503 setup_per_zone_wmarks(); 8504 8505 return 0; 8506 } 8507 8508 #ifdef CONFIG_NUMA 8509 static void setup_min_unmapped_ratio(void) 8510 { 8511 pg_data_t *pgdat; 8512 struct zone *zone; 8513 8514 for_each_online_pgdat(pgdat) 8515 pgdat->min_unmapped_pages = 0; 8516 8517 for_each_zone(zone) 8518 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * 8519 sysctl_min_unmapped_ratio) / 100; 8520 } 8521 8522 8523 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 8524 void *buffer, size_t *length, loff_t *ppos) 8525 { 8526 int rc; 8527 8528 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8529 if (rc) 8530 return rc; 8531 8532 setup_min_unmapped_ratio(); 8533 8534 return 0; 8535 } 8536 8537 static void setup_min_slab_ratio(void) 8538 { 8539 pg_data_t *pgdat; 8540 struct zone *zone; 8541 8542 for_each_online_pgdat(pgdat) 8543 pgdat->min_slab_pages = 0; 8544 8545 for_each_zone(zone) 8546 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * 8547 sysctl_min_slab_ratio) / 100; 8548 } 8549 8550 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 8551 void *buffer, size_t *length, loff_t *ppos) 8552 { 8553 int rc; 8554 8555 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8556 if (rc) 8557 return rc; 8558 8559 setup_min_slab_ratio(); 8560 8561 return 0; 8562 } 8563 #endif 8564 8565 /* 8566 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 8567 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 8568 * whenever sysctl_lowmem_reserve_ratio changes. 8569 * 8570 * The reserve ratio obviously has absolutely no relation with the 8571 * minimum watermarks. The lowmem reserve ratio can only make sense 8572 * if in function of the boot time zone sizes. 8573 */ 8574 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 8575 void *buffer, size_t *length, loff_t *ppos) 8576 { 8577 int i; 8578 8579 proc_dointvec_minmax(table, write, buffer, length, ppos); 8580 8581 for (i = 0; i < MAX_NR_ZONES; i++) { 8582 if (sysctl_lowmem_reserve_ratio[i] < 1) 8583 sysctl_lowmem_reserve_ratio[i] = 0; 8584 } 8585 8586 setup_per_zone_lowmem_reserve(); 8587 return 0; 8588 } 8589 8590 /* 8591 * percpu_pagelist_high_fraction - changes the pcp->high for each zone on each 8592 * cpu. It is the fraction of total pages in each zone that a hot per cpu 8593 * pagelist can have before it gets flushed back to buddy allocator. 8594 */ 8595 int percpu_pagelist_high_fraction_sysctl_handler(struct ctl_table *table, 8596 int write, void *buffer, size_t *length, loff_t *ppos) 8597 { 8598 struct zone *zone; 8599 int old_percpu_pagelist_high_fraction; 8600 int ret; 8601 8602 mutex_lock(&pcp_batch_high_lock); 8603 old_percpu_pagelist_high_fraction = percpu_pagelist_high_fraction; 8604 8605 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 8606 if (!write || ret < 0) 8607 goto out; 8608 8609 /* Sanity checking to avoid pcp imbalance */ 8610 if (percpu_pagelist_high_fraction && 8611 percpu_pagelist_high_fraction < MIN_PERCPU_PAGELIST_HIGH_FRACTION) { 8612 percpu_pagelist_high_fraction = old_percpu_pagelist_high_fraction; 8613 ret = -EINVAL; 8614 goto out; 8615 } 8616 8617 /* No change? */ 8618 if (percpu_pagelist_high_fraction == old_percpu_pagelist_high_fraction) 8619 goto out; 8620 8621 for_each_populated_zone(zone) 8622 zone_set_pageset_high_and_batch(zone, 0); 8623 out: 8624 mutex_unlock(&pcp_batch_high_lock); 8625 return ret; 8626 } 8627 8628 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 8629 /* 8630 * Returns the number of pages that arch has reserved but 8631 * is not known to alloc_large_system_hash(). 8632 */ 8633 static unsigned long __init arch_reserved_kernel_pages(void) 8634 { 8635 return 0; 8636 } 8637 #endif 8638 8639 /* 8640 * Adaptive scale is meant to reduce sizes of hash tables on large memory 8641 * machines. As memory size is increased the scale is also increased but at 8642 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 8643 * quadruples the scale is increased by one, which means the size of hash table 8644 * only doubles, instead of quadrupling as well. 8645 * Because 32-bit systems cannot have large physical memory, where this scaling 8646 * makes sense, it is disabled on such platforms. 8647 */ 8648 #if __BITS_PER_LONG > 32 8649 #define ADAPT_SCALE_BASE (64ul << 30) 8650 #define ADAPT_SCALE_SHIFT 2 8651 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 8652 #endif 8653 8654 /* 8655 * allocate a large system hash table from bootmem 8656 * - it is assumed that the hash table must contain an exact power-of-2 8657 * quantity of entries 8658 * - limit is the number of hash buckets, not the total allocation size 8659 */ 8660 void *__init alloc_large_system_hash(const char *tablename, 8661 unsigned long bucketsize, 8662 unsigned long numentries, 8663 int scale, 8664 int flags, 8665 unsigned int *_hash_shift, 8666 unsigned int *_hash_mask, 8667 unsigned long low_limit, 8668 unsigned long high_limit) 8669 { 8670 unsigned long long max = high_limit; 8671 unsigned long log2qty, size; 8672 void *table = NULL; 8673 gfp_t gfp_flags; 8674 bool virt; 8675 bool huge; 8676 8677 /* allow the kernel cmdline to have a say */ 8678 if (!numentries) { 8679 /* round applicable memory size up to nearest megabyte */ 8680 numentries = nr_kernel_pages; 8681 numentries -= arch_reserved_kernel_pages(); 8682 8683 /* It isn't necessary when PAGE_SIZE >= 1MB */ 8684 if (PAGE_SHIFT < 20) 8685 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 8686 8687 #if __BITS_PER_LONG > 32 8688 if (!high_limit) { 8689 unsigned long adapt; 8690 8691 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 8692 adapt <<= ADAPT_SCALE_SHIFT) 8693 scale++; 8694 } 8695 #endif 8696 8697 /* limit to 1 bucket per 2^scale bytes of low memory */ 8698 if (scale > PAGE_SHIFT) 8699 numentries >>= (scale - PAGE_SHIFT); 8700 else 8701 numentries <<= (PAGE_SHIFT - scale); 8702 8703 /* Make sure we've got at least a 0-order allocation.. */ 8704 if (unlikely(flags & HASH_SMALL)) { 8705 /* Makes no sense without HASH_EARLY */ 8706 WARN_ON(!(flags & HASH_EARLY)); 8707 if (!(numentries >> *_hash_shift)) { 8708 numentries = 1UL << *_hash_shift; 8709 BUG_ON(!numentries); 8710 } 8711 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 8712 numentries = PAGE_SIZE / bucketsize; 8713 } 8714 numentries = roundup_pow_of_two(numentries); 8715 8716 /* limit allocation size to 1/16 total memory by default */ 8717 if (max == 0) { 8718 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 8719 do_div(max, bucketsize); 8720 } 8721 max = min(max, 0x80000000ULL); 8722 8723 if (numentries < low_limit) 8724 numentries = low_limit; 8725 if (numentries > max) 8726 numentries = max; 8727 8728 log2qty = ilog2(numentries); 8729 8730 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 8731 do { 8732 virt = false; 8733 size = bucketsize << log2qty; 8734 if (flags & HASH_EARLY) { 8735 if (flags & HASH_ZERO) 8736 table = memblock_alloc(size, SMP_CACHE_BYTES); 8737 else 8738 table = memblock_alloc_raw(size, 8739 SMP_CACHE_BYTES); 8740 } else if (get_order(size) >= MAX_ORDER || hashdist) { 8741 table = __vmalloc(size, gfp_flags); 8742 virt = true; 8743 huge = is_vm_area_hugepages(table); 8744 } else { 8745 /* 8746 * If bucketsize is not a power-of-two, we may free 8747 * some pages at the end of hash table which 8748 * alloc_pages_exact() automatically does 8749 */ 8750 table = alloc_pages_exact(size, gfp_flags); 8751 kmemleak_alloc(table, size, 1, gfp_flags); 8752 } 8753 } while (!table && size > PAGE_SIZE && --log2qty); 8754 8755 if (!table) 8756 panic("Failed to allocate %s hash table\n", tablename); 8757 8758 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 8759 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, 8760 virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear"); 8761 8762 if (_hash_shift) 8763 *_hash_shift = log2qty; 8764 if (_hash_mask) 8765 *_hash_mask = (1 << log2qty) - 1; 8766 8767 return table; 8768 } 8769 8770 /* 8771 * This function checks whether pageblock includes unmovable pages or not. 8772 * 8773 * PageLRU check without isolation or lru_lock could race so that 8774 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable 8775 * check without lock_page also may miss some movable non-lru pages at 8776 * race condition. So you can't expect this function should be exact. 8777 * 8778 * Returns a page without holding a reference. If the caller wants to 8779 * dereference that page (e.g., dumping), it has to make sure that it 8780 * cannot get removed (e.g., via memory unplug) concurrently. 8781 * 8782 */ 8783 struct page *has_unmovable_pages(struct zone *zone, struct page *page, 8784 int migratetype, int flags) 8785 { 8786 unsigned long iter = 0; 8787 unsigned long pfn = page_to_pfn(page); 8788 unsigned long offset = pfn % pageblock_nr_pages; 8789 8790 if (is_migrate_cma_page(page)) { 8791 /* 8792 * CMA allocations (alloc_contig_range) really need to mark 8793 * isolate CMA pageblocks even when they are not movable in fact 8794 * so consider them movable here. 8795 */ 8796 if (is_migrate_cma(migratetype)) 8797 return NULL; 8798 8799 return page; 8800 } 8801 8802 for (; iter < pageblock_nr_pages - offset; iter++) { 8803 page = pfn_to_page(pfn + iter); 8804 8805 /* 8806 * Both, bootmem allocations and memory holes are marked 8807 * PG_reserved and are unmovable. We can even have unmovable 8808 * allocations inside ZONE_MOVABLE, for example when 8809 * specifying "movablecore". 8810 */ 8811 if (PageReserved(page)) 8812 return page; 8813 8814 /* 8815 * If the zone is movable and we have ruled out all reserved 8816 * pages then it should be reasonably safe to assume the rest 8817 * is movable. 8818 */ 8819 if (zone_idx(zone) == ZONE_MOVABLE) 8820 continue; 8821 8822 /* 8823 * Hugepages are not in LRU lists, but they're movable. 8824 * THPs are on the LRU, but need to be counted as #small pages. 8825 * We need not scan over tail pages because we don't 8826 * handle each tail page individually in migration. 8827 */ 8828 if (PageHuge(page) || PageTransCompound(page)) { 8829 struct page *head = compound_head(page); 8830 unsigned int skip_pages; 8831 8832 if (PageHuge(page)) { 8833 if (!hugepage_migration_supported(page_hstate(head))) 8834 return page; 8835 } else if (!PageLRU(head) && !__PageMovable(head)) { 8836 return page; 8837 } 8838 8839 skip_pages = compound_nr(head) - (page - head); 8840 iter += skip_pages - 1; 8841 continue; 8842 } 8843 8844 /* 8845 * We can't use page_count without pin a page 8846 * because another CPU can free compound page. 8847 * This check already skips compound tails of THP 8848 * because their page->_refcount is zero at all time. 8849 */ 8850 if (!page_ref_count(page)) { 8851 if (PageBuddy(page)) 8852 iter += (1 << buddy_order(page)) - 1; 8853 continue; 8854 } 8855 8856 /* 8857 * The HWPoisoned page may be not in buddy system, and 8858 * page_count() is not 0. 8859 */ 8860 if ((flags & MEMORY_OFFLINE) && PageHWPoison(page)) 8861 continue; 8862 8863 /* 8864 * We treat all PageOffline() pages as movable when offlining 8865 * to give drivers a chance to decrement their reference count 8866 * in MEM_GOING_OFFLINE in order to indicate that these pages 8867 * can be offlined as there are no direct references anymore. 8868 * For actually unmovable PageOffline() where the driver does 8869 * not support this, we will fail later when trying to actually 8870 * move these pages that still have a reference count > 0. 8871 * (false negatives in this function only) 8872 */ 8873 if ((flags & MEMORY_OFFLINE) && PageOffline(page)) 8874 continue; 8875 8876 if (__PageMovable(page) || PageLRU(page)) 8877 continue; 8878 8879 /* 8880 * If there are RECLAIMABLE pages, we need to check 8881 * it. But now, memory offline itself doesn't call 8882 * shrink_node_slabs() and it still to be fixed. 8883 */ 8884 return page; 8885 } 8886 return NULL; 8887 } 8888 8889 #ifdef CONFIG_CONTIG_ALLOC 8890 static unsigned long pfn_max_align_down(unsigned long pfn) 8891 { 8892 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, 8893 pageblock_nr_pages) - 1); 8894 } 8895 8896 static unsigned long pfn_max_align_up(unsigned long pfn) 8897 { 8898 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, 8899 pageblock_nr_pages)); 8900 } 8901 8902 #if defined(CONFIG_DYNAMIC_DEBUG) || \ 8903 (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE)) 8904 /* Usage: See admin-guide/dynamic-debug-howto.rst */ 8905 static void alloc_contig_dump_pages(struct list_head *page_list) 8906 { 8907 DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, "migrate failure"); 8908 8909 if (DYNAMIC_DEBUG_BRANCH(descriptor)) { 8910 struct page *page; 8911 8912 dump_stack(); 8913 list_for_each_entry(page, page_list, lru) 8914 dump_page(page, "migration failure"); 8915 } 8916 } 8917 #else 8918 static inline void alloc_contig_dump_pages(struct list_head *page_list) 8919 { 8920 } 8921 #endif 8922 8923 /* [start, end) must belong to a single zone. */ 8924 static int __alloc_contig_migrate_range(struct compact_control *cc, 8925 unsigned long start, unsigned long end) 8926 { 8927 /* This function is based on compact_zone() from compaction.c. */ 8928 unsigned int nr_reclaimed; 8929 unsigned long pfn = start; 8930 unsigned int tries = 0; 8931 int ret = 0; 8932 struct migration_target_control mtc = { 8933 .nid = zone_to_nid(cc->zone), 8934 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, 8935 }; 8936 8937 lru_cache_disable(); 8938 8939 while (pfn < end || !list_empty(&cc->migratepages)) { 8940 if (fatal_signal_pending(current)) { 8941 ret = -EINTR; 8942 break; 8943 } 8944 8945 if (list_empty(&cc->migratepages)) { 8946 cc->nr_migratepages = 0; 8947 ret = isolate_migratepages_range(cc, pfn, end); 8948 if (ret && ret != -EAGAIN) 8949 break; 8950 pfn = cc->migrate_pfn; 8951 tries = 0; 8952 } else if (++tries == 5) { 8953 ret = -EBUSY; 8954 break; 8955 } 8956 8957 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 8958 &cc->migratepages); 8959 cc->nr_migratepages -= nr_reclaimed; 8960 8961 ret = migrate_pages(&cc->migratepages, alloc_migration_target, 8962 NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE); 8963 8964 /* 8965 * On -ENOMEM, migrate_pages() bails out right away. It is pointless 8966 * to retry again over this error, so do the same here. 8967 */ 8968 if (ret == -ENOMEM) 8969 break; 8970 } 8971 8972 lru_cache_enable(); 8973 if (ret < 0) { 8974 if (ret == -EBUSY) 8975 alloc_contig_dump_pages(&cc->migratepages); 8976 putback_movable_pages(&cc->migratepages); 8977 return ret; 8978 } 8979 return 0; 8980 } 8981 8982 /** 8983 * alloc_contig_range() -- tries to allocate given range of pages 8984 * @start: start PFN to allocate 8985 * @end: one-past-the-last PFN to allocate 8986 * @migratetype: migratetype of the underlying pageblocks (either 8987 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 8988 * in range must have the same migratetype and it must 8989 * be either of the two. 8990 * @gfp_mask: GFP mask to use during compaction 8991 * 8992 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 8993 * aligned. The PFN range must belong to a single zone. 8994 * 8995 * The first thing this routine does is attempt to MIGRATE_ISOLATE all 8996 * pageblocks in the range. Once isolated, the pageblocks should not 8997 * be modified by others. 8998 * 8999 * Return: zero on success or negative error code. On success all 9000 * pages which PFN is in [start, end) are allocated for the caller and 9001 * need to be freed with free_contig_range(). 9002 */ 9003 int alloc_contig_range(unsigned long start, unsigned long end, 9004 unsigned migratetype, gfp_t gfp_mask) 9005 { 9006 unsigned long outer_start, outer_end; 9007 unsigned int order; 9008 int ret = 0; 9009 9010 struct compact_control cc = { 9011 .nr_migratepages = 0, 9012 .order = -1, 9013 .zone = page_zone(pfn_to_page(start)), 9014 .mode = MIGRATE_SYNC, 9015 .ignore_skip_hint = true, 9016 .no_set_skip_hint = true, 9017 .gfp_mask = current_gfp_context(gfp_mask), 9018 .alloc_contig = true, 9019 }; 9020 INIT_LIST_HEAD(&cc.migratepages); 9021 9022 /* 9023 * What we do here is we mark all pageblocks in range as 9024 * MIGRATE_ISOLATE. Because pageblock and max order pages may 9025 * have different sizes, and due to the way page allocator 9026 * work, we align the range to biggest of the two pages so 9027 * that page allocator won't try to merge buddies from 9028 * different pageblocks and change MIGRATE_ISOLATE to some 9029 * other migration type. 9030 * 9031 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 9032 * migrate the pages from an unaligned range (ie. pages that 9033 * we are interested in). This will put all the pages in 9034 * range back to page allocator as MIGRATE_ISOLATE. 9035 * 9036 * When this is done, we take the pages in range from page 9037 * allocator removing them from the buddy system. This way 9038 * page allocator will never consider using them. 9039 * 9040 * This lets us mark the pageblocks back as 9041 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 9042 * aligned range but not in the unaligned, original range are 9043 * put back to page allocator so that buddy can use them. 9044 */ 9045 9046 ret = start_isolate_page_range(pfn_max_align_down(start), 9047 pfn_max_align_up(end), migratetype, 0); 9048 if (ret) 9049 return ret; 9050 9051 drain_all_pages(cc.zone); 9052 9053 /* 9054 * In case of -EBUSY, we'd like to know which page causes problem. 9055 * So, just fall through. test_pages_isolated() has a tracepoint 9056 * which will report the busy page. 9057 * 9058 * It is possible that busy pages could become available before 9059 * the call to test_pages_isolated, and the range will actually be 9060 * allocated. So, if we fall through be sure to clear ret so that 9061 * -EBUSY is not accidentally used or returned to caller. 9062 */ 9063 ret = __alloc_contig_migrate_range(&cc, start, end); 9064 if (ret && ret != -EBUSY) 9065 goto done; 9066 ret = 0; 9067 9068 /* 9069 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 9070 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 9071 * more, all pages in [start, end) are free in page allocator. 9072 * What we are going to do is to allocate all pages from 9073 * [start, end) (that is remove them from page allocator). 9074 * 9075 * The only problem is that pages at the beginning and at the 9076 * end of interesting range may be not aligned with pages that 9077 * page allocator holds, ie. they can be part of higher order 9078 * pages. Because of this, we reserve the bigger range and 9079 * once this is done free the pages we are not interested in. 9080 * 9081 * We don't have to hold zone->lock here because the pages are 9082 * isolated thus they won't get removed from buddy. 9083 */ 9084 9085 order = 0; 9086 outer_start = start; 9087 while (!PageBuddy(pfn_to_page(outer_start))) { 9088 if (++order >= MAX_ORDER) { 9089 outer_start = start; 9090 break; 9091 } 9092 outer_start &= ~0UL << order; 9093 } 9094 9095 if (outer_start != start) { 9096 order = buddy_order(pfn_to_page(outer_start)); 9097 9098 /* 9099 * outer_start page could be small order buddy page and 9100 * it doesn't include start page. Adjust outer_start 9101 * in this case to report failed page properly 9102 * on tracepoint in test_pages_isolated() 9103 */ 9104 if (outer_start + (1UL << order) <= start) 9105 outer_start = start; 9106 } 9107 9108 /* Make sure the range is really isolated. */ 9109 if (test_pages_isolated(outer_start, end, 0)) { 9110 ret = -EBUSY; 9111 goto done; 9112 } 9113 9114 /* Grab isolated pages from freelists. */ 9115 outer_end = isolate_freepages_range(&cc, outer_start, end); 9116 if (!outer_end) { 9117 ret = -EBUSY; 9118 goto done; 9119 } 9120 9121 /* Free head and tail (if any) */ 9122 if (start != outer_start) 9123 free_contig_range(outer_start, start - outer_start); 9124 if (end != outer_end) 9125 free_contig_range(end, outer_end - end); 9126 9127 done: 9128 undo_isolate_page_range(pfn_max_align_down(start), 9129 pfn_max_align_up(end), migratetype); 9130 return ret; 9131 } 9132 EXPORT_SYMBOL(alloc_contig_range); 9133 9134 static int __alloc_contig_pages(unsigned long start_pfn, 9135 unsigned long nr_pages, gfp_t gfp_mask) 9136 { 9137 unsigned long end_pfn = start_pfn + nr_pages; 9138 9139 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE, 9140 gfp_mask); 9141 } 9142 9143 static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, 9144 unsigned long nr_pages) 9145 { 9146 unsigned long i, end_pfn = start_pfn + nr_pages; 9147 struct page *page; 9148 9149 for (i = start_pfn; i < end_pfn; i++) { 9150 page = pfn_to_online_page(i); 9151 if (!page) 9152 return false; 9153 9154 if (page_zone(page) != z) 9155 return false; 9156 9157 if (PageReserved(page)) 9158 return false; 9159 } 9160 return true; 9161 } 9162 9163 static bool zone_spans_last_pfn(const struct zone *zone, 9164 unsigned long start_pfn, unsigned long nr_pages) 9165 { 9166 unsigned long last_pfn = start_pfn + nr_pages - 1; 9167 9168 return zone_spans_pfn(zone, last_pfn); 9169 } 9170 9171 /** 9172 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages 9173 * @nr_pages: Number of contiguous pages to allocate 9174 * @gfp_mask: GFP mask to limit search and used during compaction 9175 * @nid: Target node 9176 * @nodemask: Mask for other possible nodes 9177 * 9178 * This routine is a wrapper around alloc_contig_range(). It scans over zones 9179 * on an applicable zonelist to find a contiguous pfn range which can then be 9180 * tried for allocation with alloc_contig_range(). This routine is intended 9181 * for allocation requests which can not be fulfilled with the buddy allocator. 9182 * 9183 * The allocated memory is always aligned to a page boundary. If nr_pages is a 9184 * power of two then the alignment is guaranteed to be to the given nr_pages 9185 * (e.g. 1GB request would be aligned to 1GB). 9186 * 9187 * Allocated pages can be freed with free_contig_range() or by manually calling 9188 * __free_page() on each allocated page. 9189 * 9190 * Return: pointer to contiguous pages on success, or NULL if not successful. 9191 */ 9192 struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, 9193 int nid, nodemask_t *nodemask) 9194 { 9195 unsigned long ret, pfn, flags; 9196 struct zonelist *zonelist; 9197 struct zone *zone; 9198 struct zoneref *z; 9199 9200 zonelist = node_zonelist(nid, gfp_mask); 9201 for_each_zone_zonelist_nodemask(zone, z, zonelist, 9202 gfp_zone(gfp_mask), nodemask) { 9203 spin_lock_irqsave(&zone->lock, flags); 9204 9205 pfn = ALIGN(zone->zone_start_pfn, nr_pages); 9206 while (zone_spans_last_pfn(zone, pfn, nr_pages)) { 9207 if (pfn_range_valid_contig(zone, pfn, nr_pages)) { 9208 /* 9209 * We release the zone lock here because 9210 * alloc_contig_range() will also lock the zone 9211 * at some point. If there's an allocation 9212 * spinning on this lock, it may win the race 9213 * and cause alloc_contig_range() to fail... 9214 */ 9215 spin_unlock_irqrestore(&zone->lock, flags); 9216 ret = __alloc_contig_pages(pfn, nr_pages, 9217 gfp_mask); 9218 if (!ret) 9219 return pfn_to_page(pfn); 9220 spin_lock_irqsave(&zone->lock, flags); 9221 } 9222 pfn += nr_pages; 9223 } 9224 spin_unlock_irqrestore(&zone->lock, flags); 9225 } 9226 return NULL; 9227 } 9228 #endif /* CONFIG_CONTIG_ALLOC */ 9229 9230 void free_contig_range(unsigned long pfn, unsigned long nr_pages) 9231 { 9232 unsigned long count = 0; 9233 9234 for (; nr_pages--; pfn++) { 9235 struct page *page = pfn_to_page(pfn); 9236 9237 count += page_count(page) != 1; 9238 __free_page(page); 9239 } 9240 WARN(count != 0, "%lu pages are still in use!\n", count); 9241 } 9242 EXPORT_SYMBOL(free_contig_range); 9243 9244 /* 9245 * The zone indicated has a new number of managed_pages; batch sizes and percpu 9246 * page high values need to be recalculated. 9247 */ 9248 void zone_pcp_update(struct zone *zone, int cpu_online) 9249 { 9250 mutex_lock(&pcp_batch_high_lock); 9251 zone_set_pageset_high_and_batch(zone, cpu_online); 9252 mutex_unlock(&pcp_batch_high_lock); 9253 } 9254 9255 /* 9256 * Effectively disable pcplists for the zone by setting the high limit to 0 9257 * and draining all cpus. A concurrent page freeing on another CPU that's about 9258 * to put the page on pcplist will either finish before the drain and the page 9259 * will be drained, or observe the new high limit and skip the pcplist. 9260 * 9261 * Must be paired with a call to zone_pcp_enable(). 9262 */ 9263 void zone_pcp_disable(struct zone *zone) 9264 { 9265 mutex_lock(&pcp_batch_high_lock); 9266 __zone_set_pageset_high_and_batch(zone, 0, 1); 9267 __drain_all_pages(zone, true); 9268 } 9269 9270 void zone_pcp_enable(struct zone *zone) 9271 { 9272 __zone_set_pageset_high_and_batch(zone, zone->pageset_high, zone->pageset_batch); 9273 mutex_unlock(&pcp_batch_high_lock); 9274 } 9275 9276 void zone_pcp_reset(struct zone *zone) 9277 { 9278 int cpu; 9279 struct per_cpu_zonestat *pzstats; 9280 9281 if (zone->per_cpu_pageset != &boot_pageset) { 9282 for_each_online_cpu(cpu) { 9283 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 9284 drain_zonestat(zone, pzstats); 9285 } 9286 free_percpu(zone->per_cpu_pageset); 9287 free_percpu(zone->per_cpu_zonestats); 9288 zone->per_cpu_pageset = &boot_pageset; 9289 zone->per_cpu_zonestats = &boot_zonestats; 9290 } 9291 } 9292 9293 #ifdef CONFIG_MEMORY_HOTREMOVE 9294 /* 9295 * All pages in the range must be in a single zone, must not contain holes, 9296 * must span full sections, and must be isolated before calling this function. 9297 */ 9298 void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 9299 { 9300 unsigned long pfn = start_pfn; 9301 struct page *page; 9302 struct zone *zone; 9303 unsigned int order; 9304 unsigned long flags; 9305 9306 offline_mem_sections(pfn, end_pfn); 9307 zone = page_zone(pfn_to_page(pfn)); 9308 spin_lock_irqsave(&zone->lock, flags); 9309 while (pfn < end_pfn) { 9310 page = pfn_to_page(pfn); 9311 /* 9312 * The HWPoisoned page may be not in buddy system, and 9313 * page_count() is not 0. 9314 */ 9315 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 9316 pfn++; 9317 continue; 9318 } 9319 /* 9320 * At this point all remaining PageOffline() pages have a 9321 * reference count of 0 and can simply be skipped. 9322 */ 9323 if (PageOffline(page)) { 9324 BUG_ON(page_count(page)); 9325 BUG_ON(PageBuddy(page)); 9326 pfn++; 9327 continue; 9328 } 9329 9330 BUG_ON(page_count(page)); 9331 BUG_ON(!PageBuddy(page)); 9332 order = buddy_order(page); 9333 del_page_from_free_list(page, zone, order); 9334 pfn += (1 << order); 9335 } 9336 spin_unlock_irqrestore(&zone->lock, flags); 9337 } 9338 #endif 9339 9340 bool is_free_buddy_page(struct page *page) 9341 { 9342 struct zone *zone = page_zone(page); 9343 unsigned long pfn = page_to_pfn(page); 9344 unsigned long flags; 9345 unsigned int order; 9346 9347 spin_lock_irqsave(&zone->lock, flags); 9348 for (order = 0; order < MAX_ORDER; order++) { 9349 struct page *page_head = page - (pfn & ((1 << order) - 1)); 9350 9351 if (PageBuddy(page_head) && buddy_order(page_head) >= order) 9352 break; 9353 } 9354 spin_unlock_irqrestore(&zone->lock, flags); 9355 9356 return order < MAX_ORDER; 9357 } 9358 9359 #ifdef CONFIG_MEMORY_FAILURE 9360 /* 9361 * Break down a higher-order page in sub-pages, and keep our target out of 9362 * buddy allocator. 9363 */ 9364 static void break_down_buddy_pages(struct zone *zone, struct page *page, 9365 struct page *target, int low, int high, 9366 int migratetype) 9367 { 9368 unsigned long size = 1 << high; 9369 struct page *current_buddy, *next_page; 9370 9371 while (high > low) { 9372 high--; 9373 size >>= 1; 9374 9375 if (target >= &page[size]) { 9376 next_page = page + size; 9377 current_buddy = page; 9378 } else { 9379 next_page = page; 9380 current_buddy = page + size; 9381 } 9382 9383 if (set_page_guard(zone, current_buddy, high, migratetype)) 9384 continue; 9385 9386 if (current_buddy != target) { 9387 add_to_free_list(current_buddy, zone, high, migratetype); 9388 set_buddy_order(current_buddy, high); 9389 page = next_page; 9390 } 9391 } 9392 } 9393 9394 /* 9395 * Take a page that will be marked as poisoned off the buddy allocator. 9396 */ 9397 bool take_page_off_buddy(struct page *page) 9398 { 9399 struct zone *zone = page_zone(page); 9400 unsigned long pfn = page_to_pfn(page); 9401 unsigned long flags; 9402 unsigned int order; 9403 bool ret = false; 9404 9405 spin_lock_irqsave(&zone->lock, flags); 9406 for (order = 0; order < MAX_ORDER; order++) { 9407 struct page *page_head = page - (pfn & ((1 << order) - 1)); 9408 int page_order = buddy_order(page_head); 9409 9410 if (PageBuddy(page_head) && page_order >= order) { 9411 unsigned long pfn_head = page_to_pfn(page_head); 9412 int migratetype = get_pfnblock_migratetype(page_head, 9413 pfn_head); 9414 9415 del_page_from_free_list(page_head, zone, page_order); 9416 break_down_buddy_pages(zone, page_head, page, 0, 9417 page_order, migratetype); 9418 if (!is_migrate_isolate(migratetype)) 9419 __mod_zone_freepage_state(zone, -1, migratetype); 9420 ret = true; 9421 break; 9422 } 9423 if (page_count(page_head) > 0) 9424 break; 9425 } 9426 spin_unlock_irqrestore(&zone->lock, flags); 9427 return ret; 9428 } 9429 #endif 9430