1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/page_alloc.c 4 * 5 * Manages the free list, the system allocates free pages here. 6 * Note that kmalloc() lives in slab.c 7 * 8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 9 * Swap reorganised 29.12.95, Stephen Tweedie 10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 16 */ 17 18 #include <linux/stddef.h> 19 #include <linux/mm.h> 20 #include <linux/highmem.h> 21 #include <linux/swap.h> 22 #include <linux/swapops.h> 23 #include <linux/interrupt.h> 24 #include <linux/pagemap.h> 25 #include <linux/jiffies.h> 26 #include <linux/memblock.h> 27 #include <linux/compiler.h> 28 #include <linux/kernel.h> 29 #include <linux/kasan.h> 30 #include <linux/kmsan.h> 31 #include <linux/module.h> 32 #include <linux/suspend.h> 33 #include <linux/pagevec.h> 34 #include <linux/blkdev.h> 35 #include <linux/slab.h> 36 #include <linux/ratelimit.h> 37 #include <linux/oom.h> 38 #include <linux/topology.h> 39 #include <linux/sysctl.h> 40 #include <linux/cpu.h> 41 #include <linux/cpuset.h> 42 #include <linux/memory_hotplug.h> 43 #include <linux/nodemask.h> 44 #include <linux/vmalloc.h> 45 #include <linux/vmstat.h> 46 #include <linux/mempolicy.h> 47 #include <linux/memremap.h> 48 #include <linux/stop_machine.h> 49 #include <linux/random.h> 50 #include <linux/sort.h> 51 #include <linux/pfn.h> 52 #include <linux/backing-dev.h> 53 #include <linux/fault-inject.h> 54 #include <linux/page-isolation.h> 55 #include <linux/debugobjects.h> 56 #include <linux/kmemleak.h> 57 #include <linux/compaction.h> 58 #include <trace/events/kmem.h> 59 #include <trace/events/oom.h> 60 #include <linux/prefetch.h> 61 #include <linux/mm_inline.h> 62 #include <linux/mmu_notifier.h> 63 #include <linux/migrate.h> 64 #include <linux/hugetlb.h> 65 #include <linux/sched/rt.h> 66 #include <linux/sched/mm.h> 67 #include <linux/page_owner.h> 68 #include <linux/page_table_check.h> 69 #include <linux/kthread.h> 70 #include <linux/memcontrol.h> 71 #include <linux/ftrace.h> 72 #include <linux/lockdep.h> 73 #include <linux/nmi.h> 74 #include <linux/psi.h> 75 #include <linux/padata.h> 76 #include <linux/khugepaged.h> 77 #include <linux/buffer_head.h> 78 #include <linux/delayacct.h> 79 #include <asm/sections.h> 80 #include <asm/tlbflush.h> 81 #include <asm/div64.h> 82 #include "internal.h" 83 #include "shuffle.h" 84 #include "page_reporting.h" 85 #include "swap.h" 86 87 /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */ 88 typedef int __bitwise fpi_t; 89 90 /* No special request */ 91 #define FPI_NONE ((__force fpi_t)0) 92 93 /* 94 * Skip free page reporting notification for the (possibly merged) page. 95 * This does not hinder free page reporting from grabbing the page, 96 * reporting it and marking it "reported" - it only skips notifying 97 * the free page reporting infrastructure about a newly freed page. For 98 * example, used when temporarily pulling a page from a freelist and 99 * putting it back unmodified. 100 */ 101 #define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0)) 102 103 /* 104 * Place the (possibly merged) page to the tail of the freelist. Will ignore 105 * page shuffling (relevant code - e.g., memory onlining - is expected to 106 * shuffle the whole zone). 107 * 108 * Note: No code should rely on this flag for correctness - it's purely 109 * to allow for optimizations when handing back either fresh pages 110 * (memory onlining) or untouched pages (page isolation, free page 111 * reporting). 112 */ 113 #define FPI_TO_TAIL ((__force fpi_t)BIT(1)) 114 115 /* 116 * Don't poison memory with KASAN (only for the tag-based modes). 117 * During boot, all non-reserved memblock memory is exposed to page_alloc. 118 * Poisoning all that memory lengthens boot time, especially on systems with 119 * large amount of RAM. This flag is used to skip that poisoning. 120 * This is only done for the tag-based KASAN modes, as those are able to 121 * detect memory corruptions with the memory tags assigned by default. 122 * All memory allocated normally after boot gets poisoned as usual. 123 */ 124 #define FPI_SKIP_KASAN_POISON ((__force fpi_t)BIT(2)) 125 126 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 127 static DEFINE_MUTEX(pcp_batch_high_lock); 128 #define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8) 129 130 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT) 131 /* 132 * On SMP, spin_trylock is sufficient protection. 133 * On PREEMPT_RT, spin_trylock is equivalent on both SMP and UP. 134 */ 135 #define pcp_trylock_prepare(flags) do { } while (0) 136 #define pcp_trylock_finish(flag) do { } while (0) 137 #else 138 139 /* UP spin_trylock always succeeds so disable IRQs to prevent re-entrancy. */ 140 #define pcp_trylock_prepare(flags) local_irq_save(flags) 141 #define pcp_trylock_finish(flags) local_irq_restore(flags) 142 #endif 143 144 /* 145 * Locking a pcp requires a PCP lookup followed by a spinlock. To avoid 146 * a migration causing the wrong PCP to be locked and remote memory being 147 * potentially allocated, pin the task to the CPU for the lookup+lock. 148 * preempt_disable is used on !RT because it is faster than migrate_disable. 149 * migrate_disable is used on RT because otherwise RT spinlock usage is 150 * interfered with and a high priority task cannot preempt the allocator. 151 */ 152 #ifndef CONFIG_PREEMPT_RT 153 #define pcpu_task_pin() preempt_disable() 154 #define pcpu_task_unpin() preempt_enable() 155 #else 156 #define pcpu_task_pin() migrate_disable() 157 #define pcpu_task_unpin() migrate_enable() 158 #endif 159 160 /* 161 * Generic helper to lookup and a per-cpu variable with an embedded spinlock. 162 * Return value should be used with equivalent unlock helper. 163 */ 164 #define pcpu_spin_lock(type, member, ptr) \ 165 ({ \ 166 type *_ret; \ 167 pcpu_task_pin(); \ 168 _ret = this_cpu_ptr(ptr); \ 169 spin_lock(&_ret->member); \ 170 _ret; \ 171 }) 172 173 #define pcpu_spin_trylock(type, member, ptr) \ 174 ({ \ 175 type *_ret; \ 176 pcpu_task_pin(); \ 177 _ret = this_cpu_ptr(ptr); \ 178 if (!spin_trylock(&_ret->member)) { \ 179 pcpu_task_unpin(); \ 180 _ret = NULL; \ 181 } \ 182 _ret; \ 183 }) 184 185 #define pcpu_spin_unlock(member, ptr) \ 186 ({ \ 187 spin_unlock(&ptr->member); \ 188 pcpu_task_unpin(); \ 189 }) 190 191 /* struct per_cpu_pages specific helpers. */ 192 #define pcp_spin_lock(ptr) \ 193 pcpu_spin_lock(struct per_cpu_pages, lock, ptr) 194 195 #define pcp_spin_trylock(ptr) \ 196 pcpu_spin_trylock(struct per_cpu_pages, lock, ptr) 197 198 #define pcp_spin_unlock(ptr) \ 199 pcpu_spin_unlock(lock, ptr) 200 201 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 202 DEFINE_PER_CPU(int, numa_node); 203 EXPORT_PER_CPU_SYMBOL(numa_node); 204 #endif 205 206 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); 207 208 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 209 /* 210 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 211 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 212 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 213 * defined in <linux/topology.h>. 214 */ 215 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 216 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 217 #endif 218 219 static DEFINE_MUTEX(pcpu_drain_mutex); 220 221 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 222 volatile unsigned long latent_entropy __latent_entropy; 223 EXPORT_SYMBOL(latent_entropy); 224 #endif 225 226 /* 227 * Array of node states. 228 */ 229 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 230 [N_POSSIBLE] = NODE_MASK_ALL, 231 [N_ONLINE] = { { [0] = 1UL } }, 232 #ifndef CONFIG_NUMA 233 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 234 #ifdef CONFIG_HIGHMEM 235 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 236 #endif 237 [N_MEMORY] = { { [0] = 1UL } }, 238 [N_CPU] = { { [0] = 1UL } }, 239 #endif /* NUMA */ 240 }; 241 EXPORT_SYMBOL(node_states); 242 243 atomic_long_t _totalram_pages __read_mostly; 244 EXPORT_SYMBOL(_totalram_pages); 245 unsigned long totalreserve_pages __read_mostly; 246 unsigned long totalcma_pages __read_mostly; 247 248 int percpu_pagelist_high_fraction; 249 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 250 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); 251 EXPORT_SYMBOL(init_on_alloc); 252 253 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); 254 EXPORT_SYMBOL(init_on_free); 255 256 static bool _init_on_alloc_enabled_early __read_mostly 257 = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON); 258 static int __init early_init_on_alloc(char *buf) 259 { 260 261 return kstrtobool(buf, &_init_on_alloc_enabled_early); 262 } 263 early_param("init_on_alloc", early_init_on_alloc); 264 265 static bool _init_on_free_enabled_early __read_mostly 266 = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON); 267 static int __init early_init_on_free(char *buf) 268 { 269 return kstrtobool(buf, &_init_on_free_enabled_early); 270 } 271 early_param("init_on_free", early_init_on_free); 272 273 /* 274 * A cached value of the page's pageblock's migratetype, used when the page is 275 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 276 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 277 * Also the migratetype set in the page does not necessarily match the pcplist 278 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 279 * other index - this ensures that it will be put on the correct CMA freelist. 280 */ 281 static inline int get_pcppage_migratetype(struct page *page) 282 { 283 return page->index; 284 } 285 286 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 287 { 288 page->index = migratetype; 289 } 290 291 #ifdef CONFIG_PM_SLEEP 292 /* 293 * The following functions are used by the suspend/hibernate code to temporarily 294 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 295 * while devices are suspended. To avoid races with the suspend/hibernate code, 296 * they should always be called with system_transition_mutex held 297 * (gfp_allowed_mask also should only be modified with system_transition_mutex 298 * held, unless the suspend/hibernate code is guaranteed not to run in parallel 299 * with that modification). 300 */ 301 302 static gfp_t saved_gfp_mask; 303 304 void pm_restore_gfp_mask(void) 305 { 306 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 307 if (saved_gfp_mask) { 308 gfp_allowed_mask = saved_gfp_mask; 309 saved_gfp_mask = 0; 310 } 311 } 312 313 void pm_restrict_gfp_mask(void) 314 { 315 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 316 WARN_ON(saved_gfp_mask); 317 saved_gfp_mask = gfp_allowed_mask; 318 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 319 } 320 321 bool pm_suspended_storage(void) 322 { 323 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 324 return false; 325 return true; 326 } 327 #endif /* CONFIG_PM_SLEEP */ 328 329 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 330 unsigned int pageblock_order __read_mostly; 331 #endif 332 333 static void __free_pages_ok(struct page *page, unsigned int order, 334 fpi_t fpi_flags); 335 336 /* 337 * results with 256, 32 in the lowmem_reserve sysctl: 338 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 339 * 1G machine -> (16M dma, 784M normal, 224M high) 340 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 341 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 342 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 343 * 344 * TBD: should special case ZONE_DMA32 machines here - in those we normally 345 * don't need any ZONE_NORMAL reservation 346 */ 347 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { 348 #ifdef CONFIG_ZONE_DMA 349 [ZONE_DMA] = 256, 350 #endif 351 #ifdef CONFIG_ZONE_DMA32 352 [ZONE_DMA32] = 256, 353 #endif 354 [ZONE_NORMAL] = 32, 355 #ifdef CONFIG_HIGHMEM 356 [ZONE_HIGHMEM] = 0, 357 #endif 358 [ZONE_MOVABLE] = 0, 359 }; 360 361 static char * const zone_names[MAX_NR_ZONES] = { 362 #ifdef CONFIG_ZONE_DMA 363 "DMA", 364 #endif 365 #ifdef CONFIG_ZONE_DMA32 366 "DMA32", 367 #endif 368 "Normal", 369 #ifdef CONFIG_HIGHMEM 370 "HighMem", 371 #endif 372 "Movable", 373 #ifdef CONFIG_ZONE_DEVICE 374 "Device", 375 #endif 376 }; 377 378 const char * const migratetype_names[MIGRATE_TYPES] = { 379 "Unmovable", 380 "Movable", 381 "Reclaimable", 382 "HighAtomic", 383 #ifdef CONFIG_CMA 384 "CMA", 385 #endif 386 #ifdef CONFIG_MEMORY_ISOLATION 387 "Isolate", 388 #endif 389 }; 390 391 compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = { 392 [NULL_COMPOUND_DTOR] = NULL, 393 [COMPOUND_PAGE_DTOR] = free_compound_page, 394 #ifdef CONFIG_HUGETLB_PAGE 395 [HUGETLB_PAGE_DTOR] = free_huge_page, 396 #endif 397 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 398 [TRANSHUGE_PAGE_DTOR] = free_transhuge_page, 399 #endif 400 }; 401 402 int min_free_kbytes = 1024; 403 int user_min_free_kbytes = -1; 404 int watermark_boost_factor __read_mostly = 15000; 405 int watermark_scale_factor = 10; 406 407 static unsigned long nr_kernel_pages __initdata; 408 static unsigned long nr_all_pages __initdata; 409 static unsigned long dma_reserve __initdata; 410 411 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 412 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 413 static unsigned long required_kernelcore __initdata; 414 static unsigned long required_kernelcore_percent __initdata; 415 static unsigned long required_movablecore __initdata; 416 static unsigned long required_movablecore_percent __initdata; 417 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 418 bool mirrored_kernelcore __initdata_memblock; 419 420 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 421 int movable_zone; 422 EXPORT_SYMBOL(movable_zone); 423 424 #if MAX_NUMNODES > 1 425 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; 426 unsigned int nr_online_nodes __read_mostly = 1; 427 EXPORT_SYMBOL(nr_node_ids); 428 EXPORT_SYMBOL(nr_online_nodes); 429 #endif 430 431 int page_group_by_mobility_disabled __read_mostly; 432 433 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 434 /* 435 * During boot we initialize deferred pages on-demand, as needed, but once 436 * page_alloc_init_late() has finished, the deferred pages are all initialized, 437 * and we can permanently disable that path. 438 */ 439 static DEFINE_STATIC_KEY_TRUE(deferred_pages); 440 441 static inline bool deferred_pages_enabled(void) 442 { 443 return static_branch_unlikely(&deferred_pages); 444 } 445 446 /* Returns true if the struct page for the pfn is initialised */ 447 static inline bool __meminit early_page_initialised(unsigned long pfn) 448 { 449 int nid = early_pfn_to_nid(pfn); 450 451 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 452 return false; 453 454 return true; 455 } 456 457 /* 458 * Returns true when the remaining initialisation should be deferred until 459 * later in the boot cycle when it can be parallelised. 460 */ 461 static bool __meminit 462 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 463 { 464 static unsigned long prev_end_pfn, nr_initialised; 465 466 if (early_page_ext_enabled()) 467 return false; 468 /* 469 * prev_end_pfn static that contains the end of previous zone 470 * No need to protect because called very early in boot before smp_init. 471 */ 472 if (prev_end_pfn != end_pfn) { 473 prev_end_pfn = end_pfn; 474 nr_initialised = 0; 475 } 476 477 /* Always populate low zones for address-constrained allocations */ 478 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 479 return false; 480 481 if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) 482 return true; 483 /* 484 * We start only with one section of pages, more pages are added as 485 * needed until the rest of deferred pages are initialized. 486 */ 487 nr_initialised++; 488 if ((nr_initialised > PAGES_PER_SECTION) && 489 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 490 NODE_DATA(nid)->first_deferred_pfn = pfn; 491 return true; 492 } 493 return false; 494 } 495 #else 496 static inline bool deferred_pages_enabled(void) 497 { 498 return false; 499 } 500 501 static inline bool early_page_initialised(unsigned long pfn) 502 { 503 return true; 504 } 505 506 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 507 { 508 return false; 509 } 510 #endif 511 512 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 513 static inline unsigned long *get_pageblock_bitmap(const struct page *page, 514 unsigned long pfn) 515 { 516 #ifdef CONFIG_SPARSEMEM 517 return section_to_usemap(__pfn_to_section(pfn)); 518 #else 519 return page_zone(page)->pageblock_flags; 520 #endif /* CONFIG_SPARSEMEM */ 521 } 522 523 static inline int pfn_to_bitidx(const struct page *page, unsigned long pfn) 524 { 525 #ifdef CONFIG_SPARSEMEM 526 pfn &= (PAGES_PER_SECTION-1); 527 #else 528 pfn = pfn - pageblock_start_pfn(page_zone(page)->zone_start_pfn); 529 #endif /* CONFIG_SPARSEMEM */ 530 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 531 } 532 533 static __always_inline 534 unsigned long __get_pfnblock_flags_mask(const struct page *page, 535 unsigned long pfn, 536 unsigned long mask) 537 { 538 unsigned long *bitmap; 539 unsigned long bitidx, word_bitidx; 540 unsigned long word; 541 542 bitmap = get_pageblock_bitmap(page, pfn); 543 bitidx = pfn_to_bitidx(page, pfn); 544 word_bitidx = bitidx / BITS_PER_LONG; 545 bitidx &= (BITS_PER_LONG-1); 546 /* 547 * This races, without locks, with set_pfnblock_flags_mask(). Ensure 548 * a consistent read of the memory array, so that results, even though 549 * racy, are not corrupted. 550 */ 551 word = READ_ONCE(bitmap[word_bitidx]); 552 return (word >> bitidx) & mask; 553 } 554 555 /** 556 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 557 * @page: The page within the block of interest 558 * @pfn: The target page frame number 559 * @mask: mask of bits that the caller is interested in 560 * 561 * Return: pageblock_bits flags 562 */ 563 unsigned long get_pfnblock_flags_mask(const struct page *page, 564 unsigned long pfn, unsigned long mask) 565 { 566 return __get_pfnblock_flags_mask(page, pfn, mask); 567 } 568 569 static __always_inline int get_pfnblock_migratetype(const struct page *page, 570 unsigned long pfn) 571 { 572 return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK); 573 } 574 575 /** 576 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 577 * @page: The page within the block of interest 578 * @flags: The flags to set 579 * @pfn: The target page frame number 580 * @mask: mask of bits that the caller is interested in 581 */ 582 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 583 unsigned long pfn, 584 unsigned long mask) 585 { 586 unsigned long *bitmap; 587 unsigned long bitidx, word_bitidx; 588 unsigned long word; 589 590 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 591 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits)); 592 593 bitmap = get_pageblock_bitmap(page, pfn); 594 bitidx = pfn_to_bitidx(page, pfn); 595 word_bitidx = bitidx / BITS_PER_LONG; 596 bitidx &= (BITS_PER_LONG-1); 597 598 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 599 600 mask <<= bitidx; 601 flags <<= bitidx; 602 603 word = READ_ONCE(bitmap[word_bitidx]); 604 do { 605 } while (!try_cmpxchg(&bitmap[word_bitidx], &word, (word & ~mask) | flags)); 606 } 607 608 void set_pageblock_migratetype(struct page *page, int migratetype) 609 { 610 if (unlikely(page_group_by_mobility_disabled && 611 migratetype < MIGRATE_PCPTYPES)) 612 migratetype = MIGRATE_UNMOVABLE; 613 614 set_pfnblock_flags_mask(page, (unsigned long)migratetype, 615 page_to_pfn(page), MIGRATETYPE_MASK); 616 } 617 618 #ifdef CONFIG_DEBUG_VM 619 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 620 { 621 int ret = 0; 622 unsigned seq; 623 unsigned long pfn = page_to_pfn(page); 624 unsigned long sp, start_pfn; 625 626 do { 627 seq = zone_span_seqbegin(zone); 628 start_pfn = zone->zone_start_pfn; 629 sp = zone->spanned_pages; 630 if (!zone_spans_pfn(zone, pfn)) 631 ret = 1; 632 } while (zone_span_seqretry(zone, seq)); 633 634 if (ret) 635 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 636 pfn, zone_to_nid(zone), zone->name, 637 start_pfn, start_pfn + sp); 638 639 return ret; 640 } 641 642 static int page_is_consistent(struct zone *zone, struct page *page) 643 { 644 if (zone != page_zone(page)) 645 return 0; 646 647 return 1; 648 } 649 /* 650 * Temporary debugging check for pages not lying within a given zone. 651 */ 652 static int __maybe_unused bad_range(struct zone *zone, struct page *page) 653 { 654 if (page_outside_zone_boundaries(zone, page)) 655 return 1; 656 if (!page_is_consistent(zone, page)) 657 return 1; 658 659 return 0; 660 } 661 #else 662 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page) 663 { 664 return 0; 665 } 666 #endif 667 668 static void bad_page(struct page *page, const char *reason) 669 { 670 static unsigned long resume; 671 static unsigned long nr_shown; 672 static unsigned long nr_unshown; 673 674 /* 675 * Allow a burst of 60 reports, then keep quiet for that minute; 676 * or allow a steady drip of one report per second. 677 */ 678 if (nr_shown == 60) { 679 if (time_before(jiffies, resume)) { 680 nr_unshown++; 681 goto out; 682 } 683 if (nr_unshown) { 684 pr_alert( 685 "BUG: Bad page state: %lu messages suppressed\n", 686 nr_unshown); 687 nr_unshown = 0; 688 } 689 nr_shown = 0; 690 } 691 if (nr_shown++ == 0) 692 resume = jiffies + 60 * HZ; 693 694 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 695 current->comm, page_to_pfn(page)); 696 dump_page(page, reason); 697 698 print_modules(); 699 dump_stack(); 700 out: 701 /* Leave bad fields for debug, except PageBuddy could make trouble */ 702 page_mapcount_reset(page); /* remove PageBuddy */ 703 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 704 } 705 706 static inline unsigned int order_to_pindex(int migratetype, int order) 707 { 708 int base = order; 709 710 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 711 if (order > PAGE_ALLOC_COSTLY_ORDER) { 712 VM_BUG_ON(order != pageblock_order); 713 return NR_LOWORDER_PCP_LISTS; 714 } 715 #else 716 VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); 717 #endif 718 719 return (MIGRATE_PCPTYPES * base) + migratetype; 720 } 721 722 static inline int pindex_to_order(unsigned int pindex) 723 { 724 int order = pindex / MIGRATE_PCPTYPES; 725 726 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 727 if (pindex == NR_LOWORDER_PCP_LISTS) 728 order = pageblock_order; 729 #else 730 VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); 731 #endif 732 733 return order; 734 } 735 736 static inline bool pcp_allowed_order(unsigned int order) 737 { 738 if (order <= PAGE_ALLOC_COSTLY_ORDER) 739 return true; 740 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 741 if (order == pageblock_order) 742 return true; 743 #endif 744 return false; 745 } 746 747 static inline void free_the_page(struct page *page, unsigned int order) 748 { 749 if (pcp_allowed_order(order)) /* Via pcp? */ 750 free_unref_page(page, order); 751 else 752 __free_pages_ok(page, order, FPI_NONE); 753 } 754 755 /* 756 * Higher-order pages are called "compound pages". They are structured thusly: 757 * 758 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 759 * 760 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 761 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 762 * 763 * The first tail page's ->compound_dtor holds the offset in array of compound 764 * page destructors. See compound_page_dtors. 765 * 766 * The first tail page's ->compound_order holds the order of allocation. 767 * This usage means that zero-order pages may not be compound. 768 */ 769 770 void free_compound_page(struct page *page) 771 { 772 mem_cgroup_uncharge(page_folio(page)); 773 free_the_page(page, compound_order(page)); 774 } 775 776 static void prep_compound_head(struct page *page, unsigned int order) 777 { 778 struct folio *folio = (struct folio *)page; 779 780 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 781 set_compound_order(page, order); 782 atomic_set(&folio->_entire_mapcount, -1); 783 atomic_set(&folio->_nr_pages_mapped, 0); 784 atomic_set(&folio->_pincount, 0); 785 } 786 787 static void prep_compound_tail(struct page *head, int tail_idx) 788 { 789 struct page *p = head + tail_idx; 790 791 p->mapping = TAIL_MAPPING; 792 set_compound_head(p, head); 793 set_page_private(p, 0); 794 } 795 796 void prep_compound_page(struct page *page, unsigned int order) 797 { 798 int i; 799 int nr_pages = 1 << order; 800 801 __SetPageHead(page); 802 for (i = 1; i < nr_pages; i++) 803 prep_compound_tail(page, i); 804 805 prep_compound_head(page, order); 806 } 807 808 void destroy_large_folio(struct folio *folio) 809 { 810 enum compound_dtor_id dtor = folio->_folio_dtor; 811 812 VM_BUG_ON_FOLIO(dtor >= NR_COMPOUND_DTORS, folio); 813 compound_page_dtors[dtor](&folio->page); 814 } 815 816 #ifdef CONFIG_DEBUG_PAGEALLOC 817 unsigned int _debug_guardpage_minorder; 818 819 bool _debug_pagealloc_enabled_early __read_mostly 820 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 821 EXPORT_SYMBOL(_debug_pagealloc_enabled_early); 822 DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 823 EXPORT_SYMBOL(_debug_pagealloc_enabled); 824 825 DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 826 827 static int __init early_debug_pagealloc(char *buf) 828 { 829 return kstrtobool(buf, &_debug_pagealloc_enabled_early); 830 } 831 early_param("debug_pagealloc", early_debug_pagealloc); 832 833 static int __init debug_guardpage_minorder_setup(char *buf) 834 { 835 unsigned long res; 836 837 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 838 pr_err("Bad debug_guardpage_minorder value\n"); 839 return 0; 840 } 841 _debug_guardpage_minorder = res; 842 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 843 return 0; 844 } 845 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); 846 847 static inline bool set_page_guard(struct zone *zone, struct page *page, 848 unsigned int order, int migratetype) 849 { 850 if (!debug_guardpage_enabled()) 851 return false; 852 853 if (order >= debug_guardpage_minorder()) 854 return false; 855 856 __SetPageGuard(page); 857 INIT_LIST_HEAD(&page->buddy_list); 858 set_page_private(page, order); 859 /* Guard pages are not available for any usage */ 860 if (!is_migrate_isolate(migratetype)) 861 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 862 863 return true; 864 } 865 866 static inline void clear_page_guard(struct zone *zone, struct page *page, 867 unsigned int order, int migratetype) 868 { 869 if (!debug_guardpage_enabled()) 870 return; 871 872 __ClearPageGuard(page); 873 874 set_page_private(page, 0); 875 if (!is_migrate_isolate(migratetype)) 876 __mod_zone_freepage_state(zone, (1 << order), migratetype); 877 } 878 #else 879 static inline bool set_page_guard(struct zone *zone, struct page *page, 880 unsigned int order, int migratetype) { return false; } 881 static inline void clear_page_guard(struct zone *zone, struct page *page, 882 unsigned int order, int migratetype) {} 883 #endif 884 885 /* 886 * Enable static keys related to various memory debugging and hardening options. 887 * Some override others, and depend on early params that are evaluated in the 888 * order of appearance. So we need to first gather the full picture of what was 889 * enabled, and then make decisions. 890 */ 891 void __init init_mem_debugging_and_hardening(void) 892 { 893 bool page_poisoning_requested = false; 894 895 #ifdef CONFIG_PAGE_POISONING 896 /* 897 * Page poisoning is debug page alloc for some arches. If 898 * either of those options are enabled, enable poisoning. 899 */ 900 if (page_poisoning_enabled() || 901 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && 902 debug_pagealloc_enabled())) { 903 static_branch_enable(&_page_poisoning_enabled); 904 page_poisoning_requested = true; 905 } 906 #endif 907 908 if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) && 909 page_poisoning_requested) { 910 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " 911 "will take precedence over init_on_alloc and init_on_free\n"); 912 _init_on_alloc_enabled_early = false; 913 _init_on_free_enabled_early = false; 914 } 915 916 if (_init_on_alloc_enabled_early) 917 static_branch_enable(&init_on_alloc); 918 else 919 static_branch_disable(&init_on_alloc); 920 921 if (_init_on_free_enabled_early) 922 static_branch_enable(&init_on_free); 923 else 924 static_branch_disable(&init_on_free); 925 926 if (IS_ENABLED(CONFIG_KMSAN) && 927 (_init_on_alloc_enabled_early || _init_on_free_enabled_early)) 928 pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n"); 929 930 #ifdef CONFIG_DEBUG_PAGEALLOC 931 if (!debug_pagealloc_enabled()) 932 return; 933 934 static_branch_enable(&_debug_pagealloc_enabled); 935 936 if (!debug_guardpage_minorder()) 937 return; 938 939 static_branch_enable(&_debug_guardpage_enabled); 940 #endif 941 } 942 943 static inline void set_buddy_order(struct page *page, unsigned int order) 944 { 945 set_page_private(page, order); 946 __SetPageBuddy(page); 947 } 948 949 #ifdef CONFIG_COMPACTION 950 static inline struct capture_control *task_capc(struct zone *zone) 951 { 952 struct capture_control *capc = current->capture_control; 953 954 return unlikely(capc) && 955 !(current->flags & PF_KTHREAD) && 956 !capc->page && 957 capc->cc->zone == zone ? capc : NULL; 958 } 959 960 static inline bool 961 compaction_capture(struct capture_control *capc, struct page *page, 962 int order, int migratetype) 963 { 964 if (!capc || order != capc->cc->order) 965 return false; 966 967 /* Do not accidentally pollute CMA or isolated regions*/ 968 if (is_migrate_cma(migratetype) || 969 is_migrate_isolate(migratetype)) 970 return false; 971 972 /* 973 * Do not let lower order allocations pollute a movable pageblock. 974 * This might let an unmovable request use a reclaimable pageblock 975 * and vice-versa but no more than normal fallback logic which can 976 * have trouble finding a high-order free page. 977 */ 978 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE) 979 return false; 980 981 capc->page = page; 982 return true; 983 } 984 985 #else 986 static inline struct capture_control *task_capc(struct zone *zone) 987 { 988 return NULL; 989 } 990 991 static inline bool 992 compaction_capture(struct capture_control *capc, struct page *page, 993 int order, int migratetype) 994 { 995 return false; 996 } 997 #endif /* CONFIG_COMPACTION */ 998 999 /* Used for pages not on another list */ 1000 static inline void add_to_free_list(struct page *page, struct zone *zone, 1001 unsigned int order, int migratetype) 1002 { 1003 struct free_area *area = &zone->free_area[order]; 1004 1005 list_add(&page->buddy_list, &area->free_list[migratetype]); 1006 area->nr_free++; 1007 } 1008 1009 /* Used for pages not on another list */ 1010 static inline void add_to_free_list_tail(struct page *page, struct zone *zone, 1011 unsigned int order, int migratetype) 1012 { 1013 struct free_area *area = &zone->free_area[order]; 1014 1015 list_add_tail(&page->buddy_list, &area->free_list[migratetype]); 1016 area->nr_free++; 1017 } 1018 1019 /* 1020 * Used for pages which are on another list. Move the pages to the tail 1021 * of the list - so the moved pages won't immediately be considered for 1022 * allocation again (e.g., optimization for memory onlining). 1023 */ 1024 static inline void move_to_free_list(struct page *page, struct zone *zone, 1025 unsigned int order, int migratetype) 1026 { 1027 struct free_area *area = &zone->free_area[order]; 1028 1029 list_move_tail(&page->buddy_list, &area->free_list[migratetype]); 1030 } 1031 1032 static inline void del_page_from_free_list(struct page *page, struct zone *zone, 1033 unsigned int order) 1034 { 1035 /* clear reported state and update reported page count */ 1036 if (page_reported(page)) 1037 __ClearPageReported(page); 1038 1039 list_del(&page->buddy_list); 1040 __ClearPageBuddy(page); 1041 set_page_private(page, 0); 1042 zone->free_area[order].nr_free--; 1043 } 1044 1045 /* 1046 * If this is not the largest possible page, check if the buddy 1047 * of the next-highest order is free. If it is, it's possible 1048 * that pages are being freed that will coalesce soon. In case, 1049 * that is happening, add the free page to the tail of the list 1050 * so it's less likely to be used soon and more likely to be merged 1051 * as a higher order page 1052 */ 1053 static inline bool 1054 buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, 1055 struct page *page, unsigned int order) 1056 { 1057 unsigned long higher_page_pfn; 1058 struct page *higher_page; 1059 1060 if (order >= MAX_ORDER - 2) 1061 return false; 1062 1063 higher_page_pfn = buddy_pfn & pfn; 1064 higher_page = page + (higher_page_pfn - pfn); 1065 1066 return find_buddy_page_pfn(higher_page, higher_page_pfn, order + 1, 1067 NULL) != NULL; 1068 } 1069 1070 /* 1071 * Freeing function for a buddy system allocator. 1072 * 1073 * The concept of a buddy system is to maintain direct-mapped table 1074 * (containing bit values) for memory blocks of various "orders". 1075 * The bottom level table contains the map for the smallest allocatable 1076 * units of memory (here, pages), and each level above it describes 1077 * pairs of units from the levels below, hence, "buddies". 1078 * At a high level, all that happens here is marking the table entry 1079 * at the bottom level available, and propagating the changes upward 1080 * as necessary, plus some accounting needed to play nicely with other 1081 * parts of the VM system. 1082 * At each level, we keep a list of pages, which are heads of continuous 1083 * free pages of length of (1 << order) and marked with PageBuddy. 1084 * Page's order is recorded in page_private(page) field. 1085 * So when we are allocating or freeing one, we can derive the state of the 1086 * other. That is, if we allocate a small block, and both were 1087 * free, the remainder of the region must be split into blocks. 1088 * If a block is freed, and its buddy is also free, then this 1089 * triggers coalescing into a block of larger size. 1090 * 1091 * -- nyc 1092 */ 1093 1094 static inline void __free_one_page(struct page *page, 1095 unsigned long pfn, 1096 struct zone *zone, unsigned int order, 1097 int migratetype, fpi_t fpi_flags) 1098 { 1099 struct capture_control *capc = task_capc(zone); 1100 unsigned long buddy_pfn = 0; 1101 unsigned long combined_pfn; 1102 struct page *buddy; 1103 bool to_tail; 1104 1105 VM_BUG_ON(!zone_is_initialized(zone)); 1106 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 1107 1108 VM_BUG_ON(migratetype == -1); 1109 if (likely(!is_migrate_isolate(migratetype))) 1110 __mod_zone_freepage_state(zone, 1 << order, migratetype); 1111 1112 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); 1113 VM_BUG_ON_PAGE(bad_range(zone, page), page); 1114 1115 while (order < MAX_ORDER - 1) { 1116 if (compaction_capture(capc, page, order, migratetype)) { 1117 __mod_zone_freepage_state(zone, -(1 << order), 1118 migratetype); 1119 return; 1120 } 1121 1122 buddy = find_buddy_page_pfn(page, pfn, order, &buddy_pfn); 1123 if (!buddy) 1124 goto done_merging; 1125 1126 if (unlikely(order >= pageblock_order)) { 1127 /* 1128 * We want to prevent merge between freepages on pageblock 1129 * without fallbacks and normal pageblock. Without this, 1130 * pageblock isolation could cause incorrect freepage or CMA 1131 * accounting or HIGHATOMIC accounting. 1132 */ 1133 int buddy_mt = get_pageblock_migratetype(buddy); 1134 1135 if (migratetype != buddy_mt 1136 && (!migratetype_is_mergeable(migratetype) || 1137 !migratetype_is_mergeable(buddy_mt))) 1138 goto done_merging; 1139 } 1140 1141 /* 1142 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 1143 * merge with it and move up one order. 1144 */ 1145 if (page_is_guard(buddy)) 1146 clear_page_guard(zone, buddy, order, migratetype); 1147 else 1148 del_page_from_free_list(buddy, zone, order); 1149 combined_pfn = buddy_pfn & pfn; 1150 page = page + (combined_pfn - pfn); 1151 pfn = combined_pfn; 1152 order++; 1153 } 1154 1155 done_merging: 1156 set_buddy_order(page, order); 1157 1158 if (fpi_flags & FPI_TO_TAIL) 1159 to_tail = true; 1160 else if (is_shuffle_order(order)) 1161 to_tail = shuffle_pick_tail(); 1162 else 1163 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); 1164 1165 if (to_tail) 1166 add_to_free_list_tail(page, zone, order, migratetype); 1167 else 1168 add_to_free_list(page, zone, order, migratetype); 1169 1170 /* Notify page reporting subsystem of freed page */ 1171 if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY)) 1172 page_reporting_notify_free(order); 1173 } 1174 1175 /** 1176 * split_free_page() -- split a free page at split_pfn_offset 1177 * @free_page: the original free page 1178 * @order: the order of the page 1179 * @split_pfn_offset: split offset within the page 1180 * 1181 * Return -ENOENT if the free page is changed, otherwise 0 1182 * 1183 * It is used when the free page crosses two pageblocks with different migratetypes 1184 * at split_pfn_offset within the page. The split free page will be put into 1185 * separate migratetype lists afterwards. Otherwise, the function achieves 1186 * nothing. 1187 */ 1188 int split_free_page(struct page *free_page, 1189 unsigned int order, unsigned long split_pfn_offset) 1190 { 1191 struct zone *zone = page_zone(free_page); 1192 unsigned long free_page_pfn = page_to_pfn(free_page); 1193 unsigned long pfn; 1194 unsigned long flags; 1195 int free_page_order; 1196 int mt; 1197 int ret = 0; 1198 1199 if (split_pfn_offset == 0) 1200 return ret; 1201 1202 spin_lock_irqsave(&zone->lock, flags); 1203 1204 if (!PageBuddy(free_page) || buddy_order(free_page) != order) { 1205 ret = -ENOENT; 1206 goto out; 1207 } 1208 1209 mt = get_pageblock_migratetype(free_page); 1210 if (likely(!is_migrate_isolate(mt))) 1211 __mod_zone_freepage_state(zone, -(1UL << order), mt); 1212 1213 del_page_from_free_list(free_page, zone, order); 1214 for (pfn = free_page_pfn; 1215 pfn < free_page_pfn + (1UL << order);) { 1216 int mt = get_pfnblock_migratetype(pfn_to_page(pfn), pfn); 1217 1218 free_page_order = min_t(unsigned int, 1219 pfn ? __ffs(pfn) : order, 1220 __fls(split_pfn_offset)); 1221 __free_one_page(pfn_to_page(pfn), pfn, zone, free_page_order, 1222 mt, FPI_NONE); 1223 pfn += 1UL << free_page_order; 1224 split_pfn_offset -= (1UL << free_page_order); 1225 /* we have done the first part, now switch to second part */ 1226 if (split_pfn_offset == 0) 1227 split_pfn_offset = (1UL << order) - (pfn - free_page_pfn); 1228 } 1229 out: 1230 spin_unlock_irqrestore(&zone->lock, flags); 1231 return ret; 1232 } 1233 /* 1234 * A bad page could be due to a number of fields. Instead of multiple branches, 1235 * try and check multiple fields with one check. The caller must do a detailed 1236 * check if necessary. 1237 */ 1238 static inline bool page_expected_state(struct page *page, 1239 unsigned long check_flags) 1240 { 1241 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1242 return false; 1243 1244 if (unlikely((unsigned long)page->mapping | 1245 page_ref_count(page) | 1246 #ifdef CONFIG_MEMCG 1247 page->memcg_data | 1248 #endif 1249 (page->flags & check_flags))) 1250 return false; 1251 1252 return true; 1253 } 1254 1255 static const char *page_bad_reason(struct page *page, unsigned long flags) 1256 { 1257 const char *bad_reason = NULL; 1258 1259 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1260 bad_reason = "nonzero mapcount"; 1261 if (unlikely(page->mapping != NULL)) 1262 bad_reason = "non-NULL mapping"; 1263 if (unlikely(page_ref_count(page) != 0)) 1264 bad_reason = "nonzero _refcount"; 1265 if (unlikely(page->flags & flags)) { 1266 if (flags == PAGE_FLAGS_CHECK_AT_PREP) 1267 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set"; 1268 else 1269 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 1270 } 1271 #ifdef CONFIG_MEMCG 1272 if (unlikely(page->memcg_data)) 1273 bad_reason = "page still charged to cgroup"; 1274 #endif 1275 return bad_reason; 1276 } 1277 1278 static void free_page_is_bad_report(struct page *page) 1279 { 1280 bad_page(page, 1281 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE)); 1282 } 1283 1284 static inline bool free_page_is_bad(struct page *page) 1285 { 1286 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 1287 return false; 1288 1289 /* Something has gone sideways, find it */ 1290 free_page_is_bad_report(page); 1291 return true; 1292 } 1293 1294 static int free_tail_pages_check(struct page *head_page, struct page *page) 1295 { 1296 struct folio *folio = (struct folio *)head_page; 1297 int ret = 1; 1298 1299 /* 1300 * We rely page->lru.next never has bit 0 set, unless the page 1301 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 1302 */ 1303 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 1304 1305 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 1306 ret = 0; 1307 goto out; 1308 } 1309 switch (page - head_page) { 1310 case 1: 1311 /* the first tail page: these may be in place of ->mapping */ 1312 if (unlikely(folio_entire_mapcount(folio))) { 1313 bad_page(page, "nonzero entire_mapcount"); 1314 goto out; 1315 } 1316 if (unlikely(atomic_read(&folio->_nr_pages_mapped))) { 1317 bad_page(page, "nonzero nr_pages_mapped"); 1318 goto out; 1319 } 1320 if (unlikely(atomic_read(&folio->_pincount))) { 1321 bad_page(page, "nonzero pincount"); 1322 goto out; 1323 } 1324 break; 1325 case 2: 1326 /* 1327 * the second tail page: ->mapping is 1328 * deferred_list.next -- ignore value. 1329 */ 1330 break; 1331 default: 1332 if (page->mapping != TAIL_MAPPING) { 1333 bad_page(page, "corrupted mapping in tail page"); 1334 goto out; 1335 } 1336 break; 1337 } 1338 if (unlikely(!PageTail(page))) { 1339 bad_page(page, "PageTail not set"); 1340 goto out; 1341 } 1342 if (unlikely(compound_head(page) != head_page)) { 1343 bad_page(page, "compound_head not consistent"); 1344 goto out; 1345 } 1346 ret = 0; 1347 out: 1348 page->mapping = NULL; 1349 clear_compound_head(page); 1350 return ret; 1351 } 1352 1353 /* 1354 * Skip KASAN memory poisoning when either: 1355 * 1356 * 1. Deferred memory initialization has not yet completed, 1357 * see the explanation below. 1358 * 2. Skipping poisoning is requested via FPI_SKIP_KASAN_POISON, 1359 * see the comment next to it. 1360 * 3. Skipping poisoning is requested via __GFP_SKIP_KASAN_POISON, 1361 * see the comment next to it. 1362 * 4. The allocation is excluded from being checked due to sampling, 1363 * see the call to kasan_unpoison_pages. 1364 * 1365 * Poisoning pages during deferred memory init will greatly lengthen the 1366 * process and cause problem in large memory systems as the deferred pages 1367 * initialization is done with interrupt disabled. 1368 * 1369 * Assuming that there will be no reference to those newly initialized 1370 * pages before they are ever allocated, this should have no effect on 1371 * KASAN memory tracking as the poison will be properly inserted at page 1372 * allocation time. The only corner case is when pages are allocated by 1373 * on-demand allocation and then freed again before the deferred pages 1374 * initialization is done, but this is not likely to happen. 1375 */ 1376 static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags) 1377 { 1378 return deferred_pages_enabled() || 1379 (!IS_ENABLED(CONFIG_KASAN_GENERIC) && 1380 (fpi_flags & FPI_SKIP_KASAN_POISON)) || 1381 PageSkipKASanPoison(page); 1382 } 1383 1384 static void kernel_init_pages(struct page *page, int numpages) 1385 { 1386 int i; 1387 1388 /* s390's use of memset() could override KASAN redzones. */ 1389 kasan_disable_current(); 1390 for (i = 0; i < numpages; i++) 1391 clear_highpage_kasan_tagged(page + i); 1392 kasan_enable_current(); 1393 } 1394 1395 static __always_inline bool free_pages_prepare(struct page *page, 1396 unsigned int order, bool check_free, fpi_t fpi_flags) 1397 { 1398 int bad = 0; 1399 bool init = want_init_on_free(); 1400 1401 VM_BUG_ON_PAGE(PageTail(page), page); 1402 1403 trace_mm_page_free(page, order); 1404 kmsan_free_page(page, order); 1405 1406 if (unlikely(PageHWPoison(page)) && !order) { 1407 /* 1408 * Do not let hwpoison pages hit pcplists/buddy 1409 * Untie memcg state and reset page's owner 1410 */ 1411 if (memcg_kmem_enabled() && PageMemcgKmem(page)) 1412 __memcg_kmem_uncharge_page(page, order); 1413 reset_page_owner(page, order); 1414 page_table_check_free(page, order); 1415 return false; 1416 } 1417 1418 /* 1419 * Check tail pages before head page information is cleared to 1420 * avoid checking PageCompound for order-0 pages. 1421 */ 1422 if (unlikely(order)) { 1423 bool compound = PageCompound(page); 1424 int i; 1425 1426 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1427 1428 if (compound) 1429 ClearPageHasHWPoisoned(page); 1430 for (i = 1; i < (1 << order); i++) { 1431 if (compound) 1432 bad += free_tail_pages_check(page, page + i); 1433 if (unlikely(free_page_is_bad(page + i))) { 1434 bad++; 1435 continue; 1436 } 1437 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1438 } 1439 } 1440 if (PageMappingFlags(page)) 1441 page->mapping = NULL; 1442 if (memcg_kmem_enabled() && PageMemcgKmem(page)) 1443 __memcg_kmem_uncharge_page(page, order); 1444 if (check_free && free_page_is_bad(page)) 1445 bad++; 1446 if (bad) 1447 return false; 1448 1449 page_cpupid_reset_last(page); 1450 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1451 reset_page_owner(page, order); 1452 page_table_check_free(page, order); 1453 1454 if (!PageHighMem(page)) { 1455 debug_check_no_locks_freed(page_address(page), 1456 PAGE_SIZE << order); 1457 debug_check_no_obj_freed(page_address(page), 1458 PAGE_SIZE << order); 1459 } 1460 1461 kernel_poison_pages(page, 1 << order); 1462 1463 /* 1464 * As memory initialization might be integrated into KASAN, 1465 * KASAN poisoning and memory initialization code must be 1466 * kept together to avoid discrepancies in behavior. 1467 * 1468 * With hardware tag-based KASAN, memory tags must be set before the 1469 * page becomes unavailable via debug_pagealloc or arch_free_page. 1470 */ 1471 if (!should_skip_kasan_poison(page, fpi_flags)) { 1472 kasan_poison_pages(page, order, init); 1473 1474 /* Memory is already initialized if KASAN did it internally. */ 1475 if (kasan_has_integrated_init()) 1476 init = false; 1477 } 1478 if (init) 1479 kernel_init_pages(page, 1 << order); 1480 1481 /* 1482 * arch_free_page() can make the page's contents inaccessible. s390 1483 * does this. So nothing which can access the page's contents should 1484 * happen after this. 1485 */ 1486 arch_free_page(page, order); 1487 1488 debug_pagealloc_unmap_pages(page, 1 << order); 1489 1490 return true; 1491 } 1492 1493 #ifdef CONFIG_DEBUG_VM 1494 /* 1495 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed 1496 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when 1497 * moved from pcp lists to free lists. 1498 */ 1499 static bool free_pcp_prepare(struct page *page, unsigned int order) 1500 { 1501 return free_pages_prepare(page, order, true, FPI_NONE); 1502 } 1503 1504 /* return true if this page has an inappropriate state */ 1505 static bool bulkfree_pcp_prepare(struct page *page) 1506 { 1507 if (debug_pagealloc_enabled_static()) 1508 return free_page_is_bad(page); 1509 else 1510 return false; 1511 } 1512 #else 1513 /* 1514 * With DEBUG_VM disabled, order-0 pages being freed are checked only when 1515 * moving from pcp lists to free list in order to reduce overhead. With 1516 * debug_pagealloc enabled, they are checked also immediately when being freed 1517 * to the pcp lists. 1518 */ 1519 static bool free_pcp_prepare(struct page *page, unsigned int order) 1520 { 1521 if (debug_pagealloc_enabled_static()) 1522 return free_pages_prepare(page, order, true, FPI_NONE); 1523 else 1524 return free_pages_prepare(page, order, false, FPI_NONE); 1525 } 1526 1527 static bool bulkfree_pcp_prepare(struct page *page) 1528 { 1529 return free_page_is_bad(page); 1530 } 1531 #endif /* CONFIG_DEBUG_VM */ 1532 1533 /* 1534 * Frees a number of pages from the PCP lists 1535 * Assumes all pages on list are in same zone. 1536 * count is the number of pages to free. 1537 */ 1538 static void free_pcppages_bulk(struct zone *zone, int count, 1539 struct per_cpu_pages *pcp, 1540 int pindex) 1541 { 1542 unsigned long flags; 1543 int min_pindex = 0; 1544 int max_pindex = NR_PCP_LISTS - 1; 1545 unsigned int order; 1546 bool isolated_pageblocks; 1547 struct page *page; 1548 1549 /* 1550 * Ensure proper count is passed which otherwise would stuck in the 1551 * below while (list_empty(list)) loop. 1552 */ 1553 count = min(pcp->count, count); 1554 1555 /* Ensure requested pindex is drained first. */ 1556 pindex = pindex - 1; 1557 1558 spin_lock_irqsave(&zone->lock, flags); 1559 isolated_pageblocks = has_isolate_pageblock(zone); 1560 1561 while (count > 0) { 1562 struct list_head *list; 1563 int nr_pages; 1564 1565 /* Remove pages from lists in a round-robin fashion. */ 1566 do { 1567 if (++pindex > max_pindex) 1568 pindex = min_pindex; 1569 list = &pcp->lists[pindex]; 1570 if (!list_empty(list)) 1571 break; 1572 1573 if (pindex == max_pindex) 1574 max_pindex--; 1575 if (pindex == min_pindex) 1576 min_pindex++; 1577 } while (1); 1578 1579 order = pindex_to_order(pindex); 1580 nr_pages = 1 << order; 1581 do { 1582 int mt; 1583 1584 page = list_last_entry(list, struct page, pcp_list); 1585 mt = get_pcppage_migratetype(page); 1586 1587 /* must delete to avoid corrupting pcp list */ 1588 list_del(&page->pcp_list); 1589 count -= nr_pages; 1590 pcp->count -= nr_pages; 1591 1592 if (bulkfree_pcp_prepare(page)) 1593 continue; 1594 1595 /* MIGRATE_ISOLATE page should not go to pcplists */ 1596 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1597 /* Pageblock could have been isolated meanwhile */ 1598 if (unlikely(isolated_pageblocks)) 1599 mt = get_pageblock_migratetype(page); 1600 1601 __free_one_page(page, page_to_pfn(page), zone, order, mt, FPI_NONE); 1602 trace_mm_page_pcpu_drain(page, order, mt); 1603 } while (count > 0 && !list_empty(list)); 1604 } 1605 1606 spin_unlock_irqrestore(&zone->lock, flags); 1607 } 1608 1609 static void free_one_page(struct zone *zone, 1610 struct page *page, unsigned long pfn, 1611 unsigned int order, 1612 int migratetype, fpi_t fpi_flags) 1613 { 1614 unsigned long flags; 1615 1616 spin_lock_irqsave(&zone->lock, flags); 1617 if (unlikely(has_isolate_pageblock(zone) || 1618 is_migrate_isolate(migratetype))) { 1619 migratetype = get_pfnblock_migratetype(page, pfn); 1620 } 1621 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags); 1622 spin_unlock_irqrestore(&zone->lock, flags); 1623 } 1624 1625 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1626 unsigned long zone, int nid) 1627 { 1628 mm_zero_struct_page(page); 1629 set_page_links(page, zone, nid, pfn); 1630 init_page_count(page); 1631 page_mapcount_reset(page); 1632 page_cpupid_reset_last(page); 1633 page_kasan_tag_reset(page); 1634 1635 INIT_LIST_HEAD(&page->lru); 1636 #ifdef WANT_PAGE_VIRTUAL 1637 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1638 if (!is_highmem_idx(zone)) 1639 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1640 #endif 1641 } 1642 1643 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1644 static void __meminit init_reserved_page(unsigned long pfn) 1645 { 1646 pg_data_t *pgdat; 1647 int nid, zid; 1648 1649 if (early_page_initialised(pfn)) 1650 return; 1651 1652 nid = early_pfn_to_nid(pfn); 1653 pgdat = NODE_DATA(nid); 1654 1655 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1656 struct zone *zone = &pgdat->node_zones[zid]; 1657 1658 if (zone_spans_pfn(zone, pfn)) 1659 break; 1660 } 1661 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 1662 } 1663 #else 1664 static inline void init_reserved_page(unsigned long pfn) 1665 { 1666 } 1667 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1668 1669 /* 1670 * Initialised pages do not have PageReserved set. This function is 1671 * called for each range allocated by the bootmem allocator and 1672 * marks the pages PageReserved. The remaining valid pages are later 1673 * sent to the buddy page allocator. 1674 */ 1675 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1676 { 1677 unsigned long start_pfn = PFN_DOWN(start); 1678 unsigned long end_pfn = PFN_UP(end); 1679 1680 for (; start_pfn < end_pfn; start_pfn++) { 1681 if (pfn_valid(start_pfn)) { 1682 struct page *page = pfn_to_page(start_pfn); 1683 1684 init_reserved_page(start_pfn); 1685 1686 /* Avoid false-positive PageTail() */ 1687 INIT_LIST_HEAD(&page->lru); 1688 1689 /* 1690 * no need for atomic set_bit because the struct 1691 * page is not visible yet so nobody should 1692 * access it yet. 1693 */ 1694 __SetPageReserved(page); 1695 } 1696 } 1697 } 1698 1699 static void __free_pages_ok(struct page *page, unsigned int order, 1700 fpi_t fpi_flags) 1701 { 1702 unsigned long flags; 1703 int migratetype; 1704 unsigned long pfn = page_to_pfn(page); 1705 struct zone *zone = page_zone(page); 1706 1707 if (!free_pages_prepare(page, order, true, fpi_flags)) 1708 return; 1709 1710 /* 1711 * Calling get_pfnblock_migratetype() without spin_lock_irqsave() here 1712 * is used to avoid calling get_pfnblock_migratetype() under the lock. 1713 * This will reduce the lock holding time. 1714 */ 1715 migratetype = get_pfnblock_migratetype(page, pfn); 1716 1717 spin_lock_irqsave(&zone->lock, flags); 1718 if (unlikely(has_isolate_pageblock(zone) || 1719 is_migrate_isolate(migratetype))) { 1720 migratetype = get_pfnblock_migratetype(page, pfn); 1721 } 1722 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags); 1723 spin_unlock_irqrestore(&zone->lock, flags); 1724 1725 __count_vm_events(PGFREE, 1 << order); 1726 } 1727 1728 void __free_pages_core(struct page *page, unsigned int order) 1729 { 1730 unsigned int nr_pages = 1 << order; 1731 struct page *p = page; 1732 unsigned int loop; 1733 1734 /* 1735 * When initializing the memmap, __init_single_page() sets the refcount 1736 * of all pages to 1 ("allocated"/"not free"). We have to set the 1737 * refcount of all involved pages to 0. 1738 */ 1739 prefetchw(p); 1740 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1741 prefetchw(p + 1); 1742 __ClearPageReserved(p); 1743 set_page_count(p, 0); 1744 } 1745 __ClearPageReserved(p); 1746 set_page_count(p, 0); 1747 1748 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); 1749 1750 /* 1751 * Bypass PCP and place fresh pages right to the tail, primarily 1752 * relevant for memory onlining. 1753 */ 1754 __free_pages_ok(page, order, FPI_TO_TAIL | FPI_SKIP_KASAN_POISON); 1755 } 1756 1757 #ifdef CONFIG_NUMA 1758 1759 /* 1760 * During memory init memblocks map pfns to nids. The search is expensive and 1761 * this caches recent lookups. The implementation of __early_pfn_to_nid 1762 * treats start/end as pfns. 1763 */ 1764 struct mminit_pfnnid_cache { 1765 unsigned long last_start; 1766 unsigned long last_end; 1767 int last_nid; 1768 }; 1769 1770 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1771 1772 /* 1773 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 1774 */ 1775 static int __meminit __early_pfn_to_nid(unsigned long pfn, 1776 struct mminit_pfnnid_cache *state) 1777 { 1778 unsigned long start_pfn, end_pfn; 1779 int nid; 1780 1781 if (state->last_start <= pfn && pfn < state->last_end) 1782 return state->last_nid; 1783 1784 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 1785 if (nid != NUMA_NO_NODE) { 1786 state->last_start = start_pfn; 1787 state->last_end = end_pfn; 1788 state->last_nid = nid; 1789 } 1790 1791 return nid; 1792 } 1793 1794 int __meminit early_pfn_to_nid(unsigned long pfn) 1795 { 1796 static DEFINE_SPINLOCK(early_pfn_lock); 1797 int nid; 1798 1799 spin_lock(&early_pfn_lock); 1800 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1801 if (nid < 0) 1802 nid = first_online_node; 1803 spin_unlock(&early_pfn_lock); 1804 1805 return nid; 1806 } 1807 #endif /* CONFIG_NUMA */ 1808 1809 void __init memblock_free_pages(struct page *page, unsigned long pfn, 1810 unsigned int order) 1811 { 1812 if (!early_page_initialised(pfn)) 1813 return; 1814 if (!kmsan_memblock_free_pages(page, order)) { 1815 /* KMSAN will take care of these pages. */ 1816 return; 1817 } 1818 __free_pages_core(page, order); 1819 } 1820 1821 /* 1822 * Check that the whole (or subset of) a pageblock given by the interval of 1823 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1824 * with the migration of free compaction scanner. 1825 * 1826 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1827 * 1828 * It's possible on some configurations to have a setup like node0 node1 node0 1829 * i.e. it's possible that all pages within a zones range of pages do not 1830 * belong to a single zone. We assume that a border between node0 and node1 1831 * can occur within a single pageblock, but not a node0 node1 node0 1832 * interleaving within a single pageblock. It is therefore sufficient to check 1833 * the first and last page of a pageblock and avoid checking each individual 1834 * page in a pageblock. 1835 */ 1836 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1837 unsigned long end_pfn, struct zone *zone) 1838 { 1839 struct page *start_page; 1840 struct page *end_page; 1841 1842 /* end_pfn is one past the range we are checking */ 1843 end_pfn--; 1844 1845 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1846 return NULL; 1847 1848 start_page = pfn_to_online_page(start_pfn); 1849 if (!start_page) 1850 return NULL; 1851 1852 if (page_zone(start_page) != zone) 1853 return NULL; 1854 1855 end_page = pfn_to_page(end_pfn); 1856 1857 /* This gives a shorter code than deriving page_zone(end_page) */ 1858 if (page_zone_id(start_page) != page_zone_id(end_page)) 1859 return NULL; 1860 1861 return start_page; 1862 } 1863 1864 void set_zone_contiguous(struct zone *zone) 1865 { 1866 unsigned long block_start_pfn = zone->zone_start_pfn; 1867 unsigned long block_end_pfn; 1868 1869 block_end_pfn = pageblock_end_pfn(block_start_pfn); 1870 for (; block_start_pfn < zone_end_pfn(zone); 1871 block_start_pfn = block_end_pfn, 1872 block_end_pfn += pageblock_nr_pages) { 1873 1874 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1875 1876 if (!__pageblock_pfn_to_page(block_start_pfn, 1877 block_end_pfn, zone)) 1878 return; 1879 cond_resched(); 1880 } 1881 1882 /* We confirm that there is no hole */ 1883 zone->contiguous = true; 1884 } 1885 1886 void clear_zone_contiguous(struct zone *zone) 1887 { 1888 zone->contiguous = false; 1889 } 1890 1891 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1892 static void __init deferred_free_range(unsigned long pfn, 1893 unsigned long nr_pages) 1894 { 1895 struct page *page; 1896 unsigned long i; 1897 1898 if (!nr_pages) 1899 return; 1900 1901 page = pfn_to_page(pfn); 1902 1903 /* Free a large naturally-aligned chunk if possible */ 1904 if (nr_pages == pageblock_nr_pages && pageblock_aligned(pfn)) { 1905 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1906 __free_pages_core(page, pageblock_order); 1907 return; 1908 } 1909 1910 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1911 if (pageblock_aligned(pfn)) 1912 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1913 __free_pages_core(page, 0); 1914 } 1915 } 1916 1917 /* Completion tracking for deferred_init_memmap() threads */ 1918 static atomic_t pgdat_init_n_undone __initdata; 1919 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1920 1921 static inline void __init pgdat_init_report_one_done(void) 1922 { 1923 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1924 complete(&pgdat_init_all_done_comp); 1925 } 1926 1927 /* 1928 * Returns true if page needs to be initialized or freed to buddy allocator. 1929 * 1930 * We check if a current large page is valid by only checking the validity 1931 * of the head pfn. 1932 */ 1933 static inline bool __init deferred_pfn_valid(unsigned long pfn) 1934 { 1935 if (pageblock_aligned(pfn) && !pfn_valid(pfn)) 1936 return false; 1937 return true; 1938 } 1939 1940 /* 1941 * Free pages to buddy allocator. Try to free aligned pages in 1942 * pageblock_nr_pages sizes. 1943 */ 1944 static void __init deferred_free_pages(unsigned long pfn, 1945 unsigned long end_pfn) 1946 { 1947 unsigned long nr_free = 0; 1948 1949 for (; pfn < end_pfn; pfn++) { 1950 if (!deferred_pfn_valid(pfn)) { 1951 deferred_free_range(pfn - nr_free, nr_free); 1952 nr_free = 0; 1953 } else if (pageblock_aligned(pfn)) { 1954 deferred_free_range(pfn - nr_free, nr_free); 1955 nr_free = 1; 1956 } else { 1957 nr_free++; 1958 } 1959 } 1960 /* Free the last block of pages to allocator */ 1961 deferred_free_range(pfn - nr_free, nr_free); 1962 } 1963 1964 /* 1965 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 1966 * by performing it only once every pageblock_nr_pages. 1967 * Return number of pages initialized. 1968 */ 1969 static unsigned long __init deferred_init_pages(struct zone *zone, 1970 unsigned long pfn, 1971 unsigned long end_pfn) 1972 { 1973 int nid = zone_to_nid(zone); 1974 unsigned long nr_pages = 0; 1975 int zid = zone_idx(zone); 1976 struct page *page = NULL; 1977 1978 for (; pfn < end_pfn; pfn++) { 1979 if (!deferred_pfn_valid(pfn)) { 1980 page = NULL; 1981 continue; 1982 } else if (!page || pageblock_aligned(pfn)) { 1983 page = pfn_to_page(pfn); 1984 } else { 1985 page++; 1986 } 1987 __init_single_page(page, pfn, zid, nid); 1988 nr_pages++; 1989 } 1990 return (nr_pages); 1991 } 1992 1993 /* 1994 * This function is meant to pre-load the iterator for the zone init. 1995 * Specifically it walks through the ranges until we are caught up to the 1996 * first_init_pfn value and exits there. If we never encounter the value we 1997 * return false indicating there are no valid ranges left. 1998 */ 1999 static bool __init 2000 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, 2001 unsigned long *spfn, unsigned long *epfn, 2002 unsigned long first_init_pfn) 2003 { 2004 u64 j; 2005 2006 /* 2007 * Start out by walking through the ranges in this zone that have 2008 * already been initialized. We don't need to do anything with them 2009 * so we just need to flush them out of the system. 2010 */ 2011 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { 2012 if (*epfn <= first_init_pfn) 2013 continue; 2014 if (*spfn < first_init_pfn) 2015 *spfn = first_init_pfn; 2016 *i = j; 2017 return true; 2018 } 2019 2020 return false; 2021 } 2022 2023 /* 2024 * Initialize and free pages. We do it in two loops: first we initialize 2025 * struct page, then free to buddy allocator, because while we are 2026 * freeing pages we can access pages that are ahead (computing buddy 2027 * page in __free_one_page()). 2028 * 2029 * In order to try and keep some memory in the cache we have the loop 2030 * broken along max page order boundaries. This way we will not cause 2031 * any issues with the buddy page computation. 2032 */ 2033 static unsigned long __init 2034 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, 2035 unsigned long *end_pfn) 2036 { 2037 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); 2038 unsigned long spfn = *start_pfn, epfn = *end_pfn; 2039 unsigned long nr_pages = 0; 2040 u64 j = *i; 2041 2042 /* First we loop through and initialize the page values */ 2043 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { 2044 unsigned long t; 2045 2046 if (mo_pfn <= *start_pfn) 2047 break; 2048 2049 t = min(mo_pfn, *end_pfn); 2050 nr_pages += deferred_init_pages(zone, *start_pfn, t); 2051 2052 if (mo_pfn < *end_pfn) { 2053 *start_pfn = mo_pfn; 2054 break; 2055 } 2056 } 2057 2058 /* Reset values and now loop through freeing pages as needed */ 2059 swap(j, *i); 2060 2061 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { 2062 unsigned long t; 2063 2064 if (mo_pfn <= spfn) 2065 break; 2066 2067 t = min(mo_pfn, epfn); 2068 deferred_free_pages(spfn, t); 2069 2070 if (mo_pfn <= epfn) 2071 break; 2072 } 2073 2074 return nr_pages; 2075 } 2076 2077 static void __init 2078 deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, 2079 void *arg) 2080 { 2081 unsigned long spfn, epfn; 2082 struct zone *zone = arg; 2083 u64 i; 2084 2085 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); 2086 2087 /* 2088 * Initialize and free pages in MAX_ORDER sized increments so that we 2089 * can avoid introducing any issues with the buddy allocator. 2090 */ 2091 while (spfn < end_pfn) { 2092 deferred_init_maxorder(&i, zone, &spfn, &epfn); 2093 cond_resched(); 2094 } 2095 } 2096 2097 /* An arch may override for more concurrency. */ 2098 __weak int __init 2099 deferred_page_init_max_threads(const struct cpumask *node_cpumask) 2100 { 2101 return 1; 2102 } 2103 2104 /* Initialise remaining memory on a node */ 2105 static int __init deferred_init_memmap(void *data) 2106 { 2107 pg_data_t *pgdat = data; 2108 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 2109 unsigned long spfn = 0, epfn = 0; 2110 unsigned long first_init_pfn, flags; 2111 unsigned long start = jiffies; 2112 struct zone *zone; 2113 int zid, max_threads; 2114 u64 i; 2115 2116 /* Bind memory initialisation thread to a local node if possible */ 2117 if (!cpumask_empty(cpumask)) 2118 set_cpus_allowed_ptr(current, cpumask); 2119 2120 pgdat_resize_lock(pgdat, &flags); 2121 first_init_pfn = pgdat->first_deferred_pfn; 2122 if (first_init_pfn == ULONG_MAX) { 2123 pgdat_resize_unlock(pgdat, &flags); 2124 pgdat_init_report_one_done(); 2125 return 0; 2126 } 2127 2128 /* Sanity check boundaries */ 2129 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 2130 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 2131 pgdat->first_deferred_pfn = ULONG_MAX; 2132 2133 /* 2134 * Once we unlock here, the zone cannot be grown anymore, thus if an 2135 * interrupt thread must allocate this early in boot, zone must be 2136 * pre-grown prior to start of deferred page initialization. 2137 */ 2138 pgdat_resize_unlock(pgdat, &flags); 2139 2140 /* Only the highest zone is deferred so find it */ 2141 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 2142 zone = pgdat->node_zones + zid; 2143 if (first_init_pfn < zone_end_pfn(zone)) 2144 break; 2145 } 2146 2147 /* If the zone is empty somebody else may have cleared out the zone */ 2148 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2149 first_init_pfn)) 2150 goto zone_empty; 2151 2152 max_threads = deferred_page_init_max_threads(cpumask); 2153 2154 while (spfn < epfn) { 2155 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION); 2156 struct padata_mt_job job = { 2157 .thread_fn = deferred_init_memmap_chunk, 2158 .fn_arg = zone, 2159 .start = spfn, 2160 .size = epfn_align - spfn, 2161 .align = PAGES_PER_SECTION, 2162 .min_chunk = PAGES_PER_SECTION, 2163 .max_threads = max_threads, 2164 }; 2165 2166 padata_do_multithreaded(&job); 2167 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2168 epfn_align); 2169 } 2170 zone_empty: 2171 /* Sanity check that the next zone really is unpopulated */ 2172 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 2173 2174 pr_info("node %d deferred pages initialised in %ums\n", 2175 pgdat->node_id, jiffies_to_msecs(jiffies - start)); 2176 2177 pgdat_init_report_one_done(); 2178 return 0; 2179 } 2180 2181 /* 2182 * If this zone has deferred pages, try to grow it by initializing enough 2183 * deferred pages to satisfy the allocation specified by order, rounded up to 2184 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 2185 * of SECTION_SIZE bytes by initializing struct pages in increments of 2186 * PAGES_PER_SECTION * sizeof(struct page) bytes. 2187 * 2188 * Return true when zone was grown, otherwise return false. We return true even 2189 * when we grow less than requested, to let the caller decide if there are 2190 * enough pages to satisfy the allocation. 2191 * 2192 * Note: We use noinline because this function is needed only during boot, and 2193 * it is called from a __ref function _deferred_grow_zone. This way we are 2194 * making sure that it is not inlined into permanent text section. 2195 */ 2196 static noinline bool __init 2197 deferred_grow_zone(struct zone *zone, unsigned int order) 2198 { 2199 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 2200 pg_data_t *pgdat = zone->zone_pgdat; 2201 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 2202 unsigned long spfn, epfn, flags; 2203 unsigned long nr_pages = 0; 2204 u64 i; 2205 2206 /* Only the last zone may have deferred pages */ 2207 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 2208 return false; 2209 2210 pgdat_resize_lock(pgdat, &flags); 2211 2212 /* 2213 * If someone grew this zone while we were waiting for spinlock, return 2214 * true, as there might be enough pages already. 2215 */ 2216 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 2217 pgdat_resize_unlock(pgdat, &flags); 2218 return true; 2219 } 2220 2221 /* If the zone is empty somebody else may have cleared out the zone */ 2222 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2223 first_deferred_pfn)) { 2224 pgdat->first_deferred_pfn = ULONG_MAX; 2225 pgdat_resize_unlock(pgdat, &flags); 2226 /* Retry only once. */ 2227 return first_deferred_pfn != ULONG_MAX; 2228 } 2229 2230 /* 2231 * Initialize and free pages in MAX_ORDER sized increments so 2232 * that we can avoid introducing any issues with the buddy 2233 * allocator. 2234 */ 2235 while (spfn < epfn) { 2236 /* update our first deferred PFN for this section */ 2237 first_deferred_pfn = spfn; 2238 2239 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 2240 touch_nmi_watchdog(); 2241 2242 /* We should only stop along section boundaries */ 2243 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) 2244 continue; 2245 2246 /* If our quota has been met we can stop here */ 2247 if (nr_pages >= nr_pages_needed) 2248 break; 2249 } 2250 2251 pgdat->first_deferred_pfn = spfn; 2252 pgdat_resize_unlock(pgdat, &flags); 2253 2254 return nr_pages > 0; 2255 } 2256 2257 /* 2258 * deferred_grow_zone() is __init, but it is called from 2259 * get_page_from_freelist() during early boot until deferred_pages permanently 2260 * disables this call. This is why we have refdata wrapper to avoid warning, 2261 * and to ensure that the function body gets unloaded. 2262 */ 2263 static bool __ref 2264 _deferred_grow_zone(struct zone *zone, unsigned int order) 2265 { 2266 return deferred_grow_zone(zone, order); 2267 } 2268 2269 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 2270 2271 void __init page_alloc_init_late(void) 2272 { 2273 struct zone *zone; 2274 int nid; 2275 2276 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 2277 2278 /* There will be num_node_state(N_MEMORY) threads */ 2279 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 2280 for_each_node_state(nid, N_MEMORY) { 2281 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 2282 } 2283 2284 /* Block until all are initialised */ 2285 wait_for_completion(&pgdat_init_all_done_comp); 2286 2287 /* 2288 * We initialized the rest of the deferred pages. Permanently disable 2289 * on-demand struct page initialization. 2290 */ 2291 static_branch_disable(&deferred_pages); 2292 2293 /* Reinit limits that are based on free pages after the kernel is up */ 2294 files_maxfiles_init(); 2295 #endif 2296 2297 buffer_init(); 2298 2299 /* Discard memblock private memory */ 2300 memblock_discard(); 2301 2302 for_each_node_state(nid, N_MEMORY) 2303 shuffle_free_memory(NODE_DATA(nid)); 2304 2305 for_each_populated_zone(zone) 2306 set_zone_contiguous(zone); 2307 } 2308 2309 #ifdef CONFIG_CMA 2310 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 2311 void __init init_cma_reserved_pageblock(struct page *page) 2312 { 2313 unsigned i = pageblock_nr_pages; 2314 struct page *p = page; 2315 2316 do { 2317 __ClearPageReserved(p); 2318 set_page_count(p, 0); 2319 } while (++p, --i); 2320 2321 set_pageblock_migratetype(page, MIGRATE_CMA); 2322 set_page_refcounted(page); 2323 __free_pages(page, pageblock_order); 2324 2325 adjust_managed_page_count(page, pageblock_nr_pages); 2326 page_zone(page)->cma_pages += pageblock_nr_pages; 2327 } 2328 #endif 2329 2330 /* 2331 * The order of subdivision here is critical for the IO subsystem. 2332 * Please do not alter this order without good reasons and regression 2333 * testing. Specifically, as large blocks of memory are subdivided, 2334 * the order in which smaller blocks are delivered depends on the order 2335 * they're subdivided in this function. This is the primary factor 2336 * influencing the order in which pages are delivered to the IO 2337 * subsystem according to empirical testing, and this is also justified 2338 * by considering the behavior of a buddy system containing a single 2339 * large block of memory acted on by a series of small allocations. 2340 * This behavior is a critical factor in sglist merging's success. 2341 * 2342 * -- nyc 2343 */ 2344 static inline void expand(struct zone *zone, struct page *page, 2345 int low, int high, int migratetype) 2346 { 2347 unsigned long size = 1 << high; 2348 2349 while (high > low) { 2350 high--; 2351 size >>= 1; 2352 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 2353 2354 /* 2355 * Mark as guard pages (or page), that will allow to 2356 * merge back to allocator when buddy will be freed. 2357 * Corresponding page table entries will not be touched, 2358 * pages will stay not present in virtual address space 2359 */ 2360 if (set_page_guard(zone, &page[size], high, migratetype)) 2361 continue; 2362 2363 add_to_free_list(&page[size], zone, high, migratetype); 2364 set_buddy_order(&page[size], high); 2365 } 2366 } 2367 2368 static void check_new_page_bad(struct page *page) 2369 { 2370 if (unlikely(page->flags & __PG_HWPOISON)) { 2371 /* Don't complain about hwpoisoned pages */ 2372 page_mapcount_reset(page); /* remove PageBuddy */ 2373 return; 2374 } 2375 2376 bad_page(page, 2377 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP)); 2378 } 2379 2380 /* 2381 * This page is about to be returned from the page allocator 2382 */ 2383 static inline int check_new_page(struct page *page) 2384 { 2385 if (likely(page_expected_state(page, 2386 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 2387 return 0; 2388 2389 check_new_page_bad(page); 2390 return 1; 2391 } 2392 2393 static bool check_new_pages(struct page *page, unsigned int order) 2394 { 2395 int i; 2396 for (i = 0; i < (1 << order); i++) { 2397 struct page *p = page + i; 2398 2399 if (unlikely(check_new_page(p))) 2400 return true; 2401 } 2402 2403 return false; 2404 } 2405 2406 #ifdef CONFIG_DEBUG_VM 2407 /* 2408 * With DEBUG_VM enabled, order-0 pages are checked for expected state when 2409 * being allocated from pcp lists. With debug_pagealloc also enabled, they are 2410 * also checked when pcp lists are refilled from the free lists. 2411 */ 2412 static inline bool check_pcp_refill(struct page *page, unsigned int order) 2413 { 2414 if (debug_pagealloc_enabled_static()) 2415 return check_new_pages(page, order); 2416 else 2417 return false; 2418 } 2419 2420 static inline bool check_new_pcp(struct page *page, unsigned int order) 2421 { 2422 return check_new_pages(page, order); 2423 } 2424 #else 2425 /* 2426 * With DEBUG_VM disabled, free order-0 pages are checked for expected state 2427 * when pcp lists are being refilled from the free lists. With debug_pagealloc 2428 * enabled, they are also checked when being allocated from the pcp lists. 2429 */ 2430 static inline bool check_pcp_refill(struct page *page, unsigned int order) 2431 { 2432 return check_new_pages(page, order); 2433 } 2434 static inline bool check_new_pcp(struct page *page, unsigned int order) 2435 { 2436 if (debug_pagealloc_enabled_static()) 2437 return check_new_pages(page, order); 2438 else 2439 return false; 2440 } 2441 #endif /* CONFIG_DEBUG_VM */ 2442 2443 static inline bool should_skip_kasan_unpoison(gfp_t flags) 2444 { 2445 /* Don't skip if a software KASAN mode is enabled. */ 2446 if (IS_ENABLED(CONFIG_KASAN_GENERIC) || 2447 IS_ENABLED(CONFIG_KASAN_SW_TAGS)) 2448 return false; 2449 2450 /* Skip, if hardware tag-based KASAN is not enabled. */ 2451 if (!kasan_hw_tags_enabled()) 2452 return true; 2453 2454 /* 2455 * With hardware tag-based KASAN enabled, skip if this has been 2456 * requested via __GFP_SKIP_KASAN_UNPOISON. 2457 */ 2458 return flags & __GFP_SKIP_KASAN_UNPOISON; 2459 } 2460 2461 static inline bool should_skip_init(gfp_t flags) 2462 { 2463 /* Don't skip, if hardware tag-based KASAN is not enabled. */ 2464 if (!kasan_hw_tags_enabled()) 2465 return false; 2466 2467 /* For hardware tag-based KASAN, skip if requested. */ 2468 return (flags & __GFP_SKIP_ZERO); 2469 } 2470 2471 inline void post_alloc_hook(struct page *page, unsigned int order, 2472 gfp_t gfp_flags) 2473 { 2474 bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) && 2475 !should_skip_init(gfp_flags); 2476 bool zero_tags = init && (gfp_flags & __GFP_ZEROTAGS); 2477 bool reset_tags = !zero_tags; 2478 int i; 2479 2480 set_page_private(page, 0); 2481 set_page_refcounted(page); 2482 2483 arch_alloc_page(page, order); 2484 debug_pagealloc_map_pages(page, 1 << order); 2485 2486 /* 2487 * Page unpoisoning must happen before memory initialization. 2488 * Otherwise, the poison pattern will be overwritten for __GFP_ZERO 2489 * allocations and the page unpoisoning code will complain. 2490 */ 2491 kernel_unpoison_pages(page, 1 << order); 2492 2493 /* 2494 * As memory initialization might be integrated into KASAN, 2495 * KASAN unpoisoning and memory initializion code must be 2496 * kept together to avoid discrepancies in behavior. 2497 */ 2498 2499 /* 2500 * If memory tags should be zeroed 2501 * (which happens only when memory should be initialized as well). 2502 */ 2503 if (zero_tags) { 2504 /* Initialize both memory and tags. */ 2505 for (i = 0; i != 1 << order; ++i) 2506 tag_clear_highpage(page + i); 2507 2508 /* Take note that memory was initialized by the loop above. */ 2509 init = false; 2510 } 2511 if (!should_skip_kasan_unpoison(gfp_flags)) { 2512 /* Try unpoisoning (or setting tags) and initializing memory. */ 2513 if (kasan_unpoison_pages(page, order, init)) { 2514 /* Take note that memory was initialized by KASAN. */ 2515 if (kasan_has_integrated_init()) 2516 init = false; 2517 /* Take note that memory tags were set by KASAN. */ 2518 reset_tags = false; 2519 } else { 2520 /* 2521 * KASAN decided to exclude this allocation from being 2522 * poisoned due to sampling. Skip poisoning as well. 2523 */ 2524 SetPageSkipKASanPoison(page); 2525 } 2526 } 2527 /* 2528 * If memory tags have not been set, reset the page tags to ensure 2529 * page_address() dereferencing does not fault. 2530 */ 2531 if (reset_tags) { 2532 for (i = 0; i != 1 << order; ++i) 2533 page_kasan_tag_reset(page + i); 2534 } 2535 /* If memory is still not initialized, initialize it now. */ 2536 if (init) 2537 kernel_init_pages(page, 1 << order); 2538 /* Propagate __GFP_SKIP_KASAN_POISON to page flags. */ 2539 if (kasan_hw_tags_enabled() && (gfp_flags & __GFP_SKIP_KASAN_POISON)) 2540 SetPageSkipKASanPoison(page); 2541 2542 set_page_owner(page, order, gfp_flags); 2543 page_table_check_alloc(page, order); 2544 } 2545 2546 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 2547 unsigned int alloc_flags) 2548 { 2549 post_alloc_hook(page, order, gfp_flags); 2550 2551 if (order && (gfp_flags & __GFP_COMP)) 2552 prep_compound_page(page, order); 2553 2554 /* 2555 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 2556 * allocate the page. The expectation is that the caller is taking 2557 * steps that will free more memory. The caller should avoid the page 2558 * being used for !PFMEMALLOC purposes. 2559 */ 2560 if (alloc_flags & ALLOC_NO_WATERMARKS) 2561 set_page_pfmemalloc(page); 2562 else 2563 clear_page_pfmemalloc(page); 2564 } 2565 2566 /* 2567 * Go through the free lists for the given migratetype and remove 2568 * the smallest available page from the freelists 2569 */ 2570 static __always_inline 2571 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 2572 int migratetype) 2573 { 2574 unsigned int current_order; 2575 struct free_area *area; 2576 struct page *page; 2577 2578 /* Find a page of the appropriate size in the preferred list */ 2579 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 2580 area = &(zone->free_area[current_order]); 2581 page = get_page_from_free_area(area, migratetype); 2582 if (!page) 2583 continue; 2584 del_page_from_free_list(page, zone, current_order); 2585 expand(zone, page, order, current_order, migratetype); 2586 set_pcppage_migratetype(page, migratetype); 2587 trace_mm_page_alloc_zone_locked(page, order, migratetype, 2588 pcp_allowed_order(order) && 2589 migratetype < MIGRATE_PCPTYPES); 2590 return page; 2591 } 2592 2593 return NULL; 2594 } 2595 2596 2597 /* 2598 * This array describes the order lists are fallen back to when 2599 * the free lists for the desirable migrate type are depleted 2600 * 2601 * The other migratetypes do not have fallbacks. 2602 */ 2603 static int fallbacks[MIGRATE_TYPES][3] = { 2604 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2605 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 2606 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2607 }; 2608 2609 #ifdef CONFIG_CMA 2610 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2611 unsigned int order) 2612 { 2613 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 2614 } 2615 #else 2616 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2617 unsigned int order) { return NULL; } 2618 #endif 2619 2620 /* 2621 * Move the free pages in a range to the freelist tail of the requested type. 2622 * Note that start_page and end_pages are not aligned on a pageblock 2623 * boundary. If alignment is required, use move_freepages_block() 2624 */ 2625 static int move_freepages(struct zone *zone, 2626 unsigned long start_pfn, unsigned long end_pfn, 2627 int migratetype, int *num_movable) 2628 { 2629 struct page *page; 2630 unsigned long pfn; 2631 unsigned int order; 2632 int pages_moved = 0; 2633 2634 for (pfn = start_pfn; pfn <= end_pfn;) { 2635 page = pfn_to_page(pfn); 2636 if (!PageBuddy(page)) { 2637 /* 2638 * We assume that pages that could be isolated for 2639 * migration are movable. But we don't actually try 2640 * isolating, as that would be expensive. 2641 */ 2642 if (num_movable && 2643 (PageLRU(page) || __PageMovable(page))) 2644 (*num_movable)++; 2645 pfn++; 2646 continue; 2647 } 2648 2649 /* Make sure we are not inadvertently changing nodes */ 2650 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2651 VM_BUG_ON_PAGE(page_zone(page) != zone, page); 2652 2653 order = buddy_order(page); 2654 move_to_free_list(page, zone, order, migratetype); 2655 pfn += 1 << order; 2656 pages_moved += 1 << order; 2657 } 2658 2659 return pages_moved; 2660 } 2661 2662 int move_freepages_block(struct zone *zone, struct page *page, 2663 int migratetype, int *num_movable) 2664 { 2665 unsigned long start_pfn, end_pfn, pfn; 2666 2667 if (num_movable) 2668 *num_movable = 0; 2669 2670 pfn = page_to_pfn(page); 2671 start_pfn = pageblock_start_pfn(pfn); 2672 end_pfn = pageblock_end_pfn(pfn) - 1; 2673 2674 /* Do not cross zone boundaries */ 2675 if (!zone_spans_pfn(zone, start_pfn)) 2676 start_pfn = pfn; 2677 if (!zone_spans_pfn(zone, end_pfn)) 2678 return 0; 2679 2680 return move_freepages(zone, start_pfn, end_pfn, migratetype, 2681 num_movable); 2682 } 2683 2684 static void change_pageblock_range(struct page *pageblock_page, 2685 int start_order, int migratetype) 2686 { 2687 int nr_pageblocks = 1 << (start_order - pageblock_order); 2688 2689 while (nr_pageblocks--) { 2690 set_pageblock_migratetype(pageblock_page, migratetype); 2691 pageblock_page += pageblock_nr_pages; 2692 } 2693 } 2694 2695 /* 2696 * When we are falling back to another migratetype during allocation, try to 2697 * steal extra free pages from the same pageblocks to satisfy further 2698 * allocations, instead of polluting multiple pageblocks. 2699 * 2700 * If we are stealing a relatively large buddy page, it is likely there will 2701 * be more free pages in the pageblock, so try to steal them all. For 2702 * reclaimable and unmovable allocations, we steal regardless of page size, 2703 * as fragmentation caused by those allocations polluting movable pageblocks 2704 * is worse than movable allocations stealing from unmovable and reclaimable 2705 * pageblocks. 2706 */ 2707 static bool can_steal_fallback(unsigned int order, int start_mt) 2708 { 2709 /* 2710 * Leaving this order check is intended, although there is 2711 * relaxed order check in next check. The reason is that 2712 * we can actually steal whole pageblock if this condition met, 2713 * but, below check doesn't guarantee it and that is just heuristic 2714 * so could be changed anytime. 2715 */ 2716 if (order >= pageblock_order) 2717 return true; 2718 2719 if (order >= pageblock_order / 2 || 2720 start_mt == MIGRATE_RECLAIMABLE || 2721 start_mt == MIGRATE_UNMOVABLE || 2722 page_group_by_mobility_disabled) 2723 return true; 2724 2725 return false; 2726 } 2727 2728 static inline bool boost_watermark(struct zone *zone) 2729 { 2730 unsigned long max_boost; 2731 2732 if (!watermark_boost_factor) 2733 return false; 2734 /* 2735 * Don't bother in zones that are unlikely to produce results. 2736 * On small machines, including kdump capture kernels running 2737 * in a small area, boosting the watermark can cause an out of 2738 * memory situation immediately. 2739 */ 2740 if ((pageblock_nr_pages * 4) > zone_managed_pages(zone)) 2741 return false; 2742 2743 max_boost = mult_frac(zone->_watermark[WMARK_HIGH], 2744 watermark_boost_factor, 10000); 2745 2746 /* 2747 * high watermark may be uninitialised if fragmentation occurs 2748 * very early in boot so do not boost. We do not fall 2749 * through and boost by pageblock_nr_pages as failing 2750 * allocations that early means that reclaim is not going 2751 * to help and it may even be impossible to reclaim the 2752 * boosted watermark resulting in a hang. 2753 */ 2754 if (!max_boost) 2755 return false; 2756 2757 max_boost = max(pageblock_nr_pages, max_boost); 2758 2759 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages, 2760 max_boost); 2761 2762 return true; 2763 } 2764 2765 /* 2766 * This function implements actual steal behaviour. If order is large enough, 2767 * we can steal whole pageblock. If not, we first move freepages in this 2768 * pageblock to our migratetype and determine how many already-allocated pages 2769 * are there in the pageblock with a compatible migratetype. If at least half 2770 * of pages are free or compatible, we can change migratetype of the pageblock 2771 * itself, so pages freed in the future will be put on the correct free list. 2772 */ 2773 static void steal_suitable_fallback(struct zone *zone, struct page *page, 2774 unsigned int alloc_flags, int start_type, bool whole_block) 2775 { 2776 unsigned int current_order = buddy_order(page); 2777 int free_pages, movable_pages, alike_pages; 2778 int old_block_type; 2779 2780 old_block_type = get_pageblock_migratetype(page); 2781 2782 /* 2783 * This can happen due to races and we want to prevent broken 2784 * highatomic accounting. 2785 */ 2786 if (is_migrate_highatomic(old_block_type)) 2787 goto single_page; 2788 2789 /* Take ownership for orders >= pageblock_order */ 2790 if (current_order >= pageblock_order) { 2791 change_pageblock_range(page, current_order, start_type); 2792 goto single_page; 2793 } 2794 2795 /* 2796 * Boost watermarks to increase reclaim pressure to reduce the 2797 * likelihood of future fallbacks. Wake kswapd now as the node 2798 * may be balanced overall and kswapd will not wake naturally. 2799 */ 2800 if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD)) 2801 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 2802 2803 /* We are not allowed to try stealing from the whole block */ 2804 if (!whole_block) 2805 goto single_page; 2806 2807 free_pages = move_freepages_block(zone, page, start_type, 2808 &movable_pages); 2809 /* 2810 * Determine how many pages are compatible with our allocation. 2811 * For movable allocation, it's the number of movable pages which 2812 * we just obtained. For other types it's a bit more tricky. 2813 */ 2814 if (start_type == MIGRATE_MOVABLE) { 2815 alike_pages = movable_pages; 2816 } else { 2817 /* 2818 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation 2819 * to MOVABLE pageblock, consider all non-movable pages as 2820 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or 2821 * vice versa, be conservative since we can't distinguish the 2822 * exact migratetype of non-movable pages. 2823 */ 2824 if (old_block_type == MIGRATE_MOVABLE) 2825 alike_pages = pageblock_nr_pages 2826 - (free_pages + movable_pages); 2827 else 2828 alike_pages = 0; 2829 } 2830 2831 /* moving whole block can fail due to zone boundary conditions */ 2832 if (!free_pages) 2833 goto single_page; 2834 2835 /* 2836 * If a sufficient number of pages in the block are either free or of 2837 * comparable migratability as our allocation, claim the whole block. 2838 */ 2839 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || 2840 page_group_by_mobility_disabled) 2841 set_pageblock_migratetype(page, start_type); 2842 2843 return; 2844 2845 single_page: 2846 move_to_free_list(page, zone, current_order, start_type); 2847 } 2848 2849 /* 2850 * Check whether there is a suitable fallback freepage with requested order. 2851 * If only_stealable is true, this function returns fallback_mt only if 2852 * we can steal other freepages all together. This would help to reduce 2853 * fragmentation due to mixed migratetype pages in one pageblock. 2854 */ 2855 int find_suitable_fallback(struct free_area *area, unsigned int order, 2856 int migratetype, bool only_stealable, bool *can_steal) 2857 { 2858 int i; 2859 int fallback_mt; 2860 2861 if (area->nr_free == 0) 2862 return -1; 2863 2864 *can_steal = false; 2865 for (i = 0;; i++) { 2866 fallback_mt = fallbacks[migratetype][i]; 2867 if (fallback_mt == MIGRATE_TYPES) 2868 break; 2869 2870 if (free_area_empty(area, fallback_mt)) 2871 continue; 2872 2873 if (can_steal_fallback(order, migratetype)) 2874 *can_steal = true; 2875 2876 if (!only_stealable) 2877 return fallback_mt; 2878 2879 if (*can_steal) 2880 return fallback_mt; 2881 } 2882 2883 return -1; 2884 } 2885 2886 /* 2887 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2888 * there are no empty page blocks that contain a page with a suitable order 2889 */ 2890 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2891 unsigned int alloc_order) 2892 { 2893 int mt; 2894 unsigned long max_managed, flags; 2895 2896 /* 2897 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2898 * Check is race-prone but harmless. 2899 */ 2900 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages; 2901 if (zone->nr_reserved_highatomic >= max_managed) 2902 return; 2903 2904 spin_lock_irqsave(&zone->lock, flags); 2905 2906 /* Recheck the nr_reserved_highatomic limit under the lock */ 2907 if (zone->nr_reserved_highatomic >= max_managed) 2908 goto out_unlock; 2909 2910 /* Yoink! */ 2911 mt = get_pageblock_migratetype(page); 2912 /* Only reserve normal pageblocks (i.e., they can merge with others) */ 2913 if (migratetype_is_mergeable(mt)) { 2914 zone->nr_reserved_highatomic += pageblock_nr_pages; 2915 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2916 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); 2917 } 2918 2919 out_unlock: 2920 spin_unlock_irqrestore(&zone->lock, flags); 2921 } 2922 2923 /* 2924 * Used when an allocation is about to fail under memory pressure. This 2925 * potentially hurts the reliability of high-order allocations when under 2926 * intense memory pressure but failed atomic allocations should be easier 2927 * to recover from than an OOM. 2928 * 2929 * If @force is true, try to unreserve a pageblock even though highatomic 2930 * pageblock is exhausted. 2931 */ 2932 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 2933 bool force) 2934 { 2935 struct zonelist *zonelist = ac->zonelist; 2936 unsigned long flags; 2937 struct zoneref *z; 2938 struct zone *zone; 2939 struct page *page; 2940 int order; 2941 bool ret; 2942 2943 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx, 2944 ac->nodemask) { 2945 /* 2946 * Preserve at least one pageblock unless memory pressure 2947 * is really high. 2948 */ 2949 if (!force && zone->nr_reserved_highatomic <= 2950 pageblock_nr_pages) 2951 continue; 2952 2953 spin_lock_irqsave(&zone->lock, flags); 2954 for (order = 0; order < MAX_ORDER; order++) { 2955 struct free_area *area = &(zone->free_area[order]); 2956 2957 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); 2958 if (!page) 2959 continue; 2960 2961 /* 2962 * In page freeing path, migratetype change is racy so 2963 * we can counter several free pages in a pageblock 2964 * in this loop although we changed the pageblock type 2965 * from highatomic to ac->migratetype. So we should 2966 * adjust the count once. 2967 */ 2968 if (is_migrate_highatomic_page(page)) { 2969 /* 2970 * It should never happen but changes to 2971 * locking could inadvertently allow a per-cpu 2972 * drain to add pages to MIGRATE_HIGHATOMIC 2973 * while unreserving so be safe and watch for 2974 * underflows. 2975 */ 2976 zone->nr_reserved_highatomic -= min( 2977 pageblock_nr_pages, 2978 zone->nr_reserved_highatomic); 2979 } 2980 2981 /* 2982 * Convert to ac->migratetype and avoid the normal 2983 * pageblock stealing heuristics. Minimally, the caller 2984 * is doing the work and needs the pages. More 2985 * importantly, if the block was always converted to 2986 * MIGRATE_UNMOVABLE or another type then the number 2987 * of pageblocks that cannot be completely freed 2988 * may increase. 2989 */ 2990 set_pageblock_migratetype(page, ac->migratetype); 2991 ret = move_freepages_block(zone, page, ac->migratetype, 2992 NULL); 2993 if (ret) { 2994 spin_unlock_irqrestore(&zone->lock, flags); 2995 return ret; 2996 } 2997 } 2998 spin_unlock_irqrestore(&zone->lock, flags); 2999 } 3000 3001 return false; 3002 } 3003 3004 /* 3005 * Try finding a free buddy page on the fallback list and put it on the free 3006 * list of requested migratetype, possibly along with other pages from the same 3007 * block, depending on fragmentation avoidance heuristics. Returns true if 3008 * fallback was found so that __rmqueue_smallest() can grab it. 3009 * 3010 * The use of signed ints for order and current_order is a deliberate 3011 * deviation from the rest of this file, to make the for loop 3012 * condition simpler. 3013 */ 3014 static __always_inline bool 3015 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, 3016 unsigned int alloc_flags) 3017 { 3018 struct free_area *area; 3019 int current_order; 3020 int min_order = order; 3021 struct page *page; 3022 int fallback_mt; 3023 bool can_steal; 3024 3025 /* 3026 * Do not steal pages from freelists belonging to other pageblocks 3027 * i.e. orders < pageblock_order. If there are no local zones free, 3028 * the zonelists will be reiterated without ALLOC_NOFRAGMENT. 3029 */ 3030 if (order < pageblock_order && alloc_flags & ALLOC_NOFRAGMENT) 3031 min_order = pageblock_order; 3032 3033 /* 3034 * Find the largest available free page in the other list. This roughly 3035 * approximates finding the pageblock with the most free pages, which 3036 * would be too costly to do exactly. 3037 */ 3038 for (current_order = MAX_ORDER - 1; current_order >= min_order; 3039 --current_order) { 3040 area = &(zone->free_area[current_order]); 3041 fallback_mt = find_suitable_fallback(area, current_order, 3042 start_migratetype, false, &can_steal); 3043 if (fallback_mt == -1) 3044 continue; 3045 3046 /* 3047 * We cannot steal all free pages from the pageblock and the 3048 * requested migratetype is movable. In that case it's better to 3049 * steal and split the smallest available page instead of the 3050 * largest available page, because even if the next movable 3051 * allocation falls back into a different pageblock than this 3052 * one, it won't cause permanent fragmentation. 3053 */ 3054 if (!can_steal && start_migratetype == MIGRATE_MOVABLE 3055 && current_order > order) 3056 goto find_smallest; 3057 3058 goto do_steal; 3059 } 3060 3061 return false; 3062 3063 find_smallest: 3064 for (current_order = order; current_order < MAX_ORDER; 3065 current_order++) { 3066 area = &(zone->free_area[current_order]); 3067 fallback_mt = find_suitable_fallback(area, current_order, 3068 start_migratetype, false, &can_steal); 3069 if (fallback_mt != -1) 3070 break; 3071 } 3072 3073 /* 3074 * This should not happen - we already found a suitable fallback 3075 * when looking for the largest page. 3076 */ 3077 VM_BUG_ON(current_order == MAX_ORDER); 3078 3079 do_steal: 3080 page = get_page_from_free_area(area, fallback_mt); 3081 3082 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype, 3083 can_steal); 3084 3085 trace_mm_page_alloc_extfrag(page, order, current_order, 3086 start_migratetype, fallback_mt); 3087 3088 return true; 3089 3090 } 3091 3092 /* 3093 * Do the hard work of removing an element from the buddy allocator. 3094 * Call me with the zone->lock already held. 3095 */ 3096 static __always_inline struct page * 3097 __rmqueue(struct zone *zone, unsigned int order, int migratetype, 3098 unsigned int alloc_flags) 3099 { 3100 struct page *page; 3101 3102 if (IS_ENABLED(CONFIG_CMA)) { 3103 /* 3104 * Balance movable allocations between regular and CMA areas by 3105 * allocating from CMA when over half of the zone's free memory 3106 * is in the CMA area. 3107 */ 3108 if (alloc_flags & ALLOC_CMA && 3109 zone_page_state(zone, NR_FREE_CMA_PAGES) > 3110 zone_page_state(zone, NR_FREE_PAGES) / 2) { 3111 page = __rmqueue_cma_fallback(zone, order); 3112 if (page) 3113 return page; 3114 } 3115 } 3116 retry: 3117 page = __rmqueue_smallest(zone, order, migratetype); 3118 if (unlikely(!page)) { 3119 if (alloc_flags & ALLOC_CMA) 3120 page = __rmqueue_cma_fallback(zone, order); 3121 3122 if (!page && __rmqueue_fallback(zone, order, migratetype, 3123 alloc_flags)) 3124 goto retry; 3125 } 3126 return page; 3127 } 3128 3129 /* 3130 * Obtain a specified number of elements from the buddy allocator, all under 3131 * a single hold of the lock, for efficiency. Add them to the supplied list. 3132 * Returns the number of new pages which were placed at *list. 3133 */ 3134 static int rmqueue_bulk(struct zone *zone, unsigned int order, 3135 unsigned long count, struct list_head *list, 3136 int migratetype, unsigned int alloc_flags) 3137 { 3138 unsigned long flags; 3139 int i, allocated = 0; 3140 3141 spin_lock_irqsave(&zone->lock, flags); 3142 for (i = 0; i < count; ++i) { 3143 struct page *page = __rmqueue(zone, order, migratetype, 3144 alloc_flags); 3145 if (unlikely(page == NULL)) 3146 break; 3147 3148 if (unlikely(check_pcp_refill(page, order))) 3149 continue; 3150 3151 /* 3152 * Split buddy pages returned by expand() are received here in 3153 * physical page order. The page is added to the tail of 3154 * caller's list. From the callers perspective, the linked list 3155 * is ordered by page number under some conditions. This is 3156 * useful for IO devices that can forward direction from the 3157 * head, thus also in the physical page order. This is useful 3158 * for IO devices that can merge IO requests if the physical 3159 * pages are ordered properly. 3160 */ 3161 list_add_tail(&page->pcp_list, list); 3162 allocated++; 3163 if (is_migrate_cma(get_pcppage_migratetype(page))) 3164 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 3165 -(1 << order)); 3166 } 3167 3168 /* 3169 * i pages were removed from the buddy list even if some leak due 3170 * to check_pcp_refill failing so adjust NR_FREE_PAGES based 3171 * on i. Do not confuse with 'allocated' which is the number of 3172 * pages added to the pcp list. 3173 */ 3174 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 3175 spin_unlock_irqrestore(&zone->lock, flags); 3176 return allocated; 3177 } 3178 3179 #ifdef CONFIG_NUMA 3180 /* 3181 * Called from the vmstat counter updater to drain pagesets of this 3182 * currently executing processor on remote nodes after they have 3183 * expired. 3184 */ 3185 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 3186 { 3187 int to_drain, batch; 3188 3189 batch = READ_ONCE(pcp->batch); 3190 to_drain = min(pcp->count, batch); 3191 if (to_drain > 0) { 3192 spin_lock(&pcp->lock); 3193 free_pcppages_bulk(zone, to_drain, pcp, 0); 3194 spin_unlock(&pcp->lock); 3195 } 3196 } 3197 #endif 3198 3199 /* 3200 * Drain pcplists of the indicated processor and zone. 3201 */ 3202 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 3203 { 3204 struct per_cpu_pages *pcp; 3205 3206 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 3207 if (pcp->count) { 3208 spin_lock(&pcp->lock); 3209 free_pcppages_bulk(zone, pcp->count, pcp, 0); 3210 spin_unlock(&pcp->lock); 3211 } 3212 } 3213 3214 /* 3215 * Drain pcplists of all zones on the indicated processor. 3216 */ 3217 static void drain_pages(unsigned int cpu) 3218 { 3219 struct zone *zone; 3220 3221 for_each_populated_zone(zone) { 3222 drain_pages_zone(cpu, zone); 3223 } 3224 } 3225 3226 /* 3227 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 3228 */ 3229 void drain_local_pages(struct zone *zone) 3230 { 3231 int cpu = smp_processor_id(); 3232 3233 if (zone) 3234 drain_pages_zone(cpu, zone); 3235 else 3236 drain_pages(cpu); 3237 } 3238 3239 /* 3240 * The implementation of drain_all_pages(), exposing an extra parameter to 3241 * drain on all cpus. 3242 * 3243 * drain_all_pages() is optimized to only execute on cpus where pcplists are 3244 * not empty. The check for non-emptiness can however race with a free to 3245 * pcplist that has not yet increased the pcp->count from 0 to 1. Callers 3246 * that need the guarantee that every CPU has drained can disable the 3247 * optimizing racy check. 3248 */ 3249 static void __drain_all_pages(struct zone *zone, bool force_all_cpus) 3250 { 3251 int cpu; 3252 3253 /* 3254 * Allocate in the BSS so we won't require allocation in 3255 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 3256 */ 3257 static cpumask_t cpus_with_pcps; 3258 3259 /* 3260 * Do not drain if one is already in progress unless it's specific to 3261 * a zone. Such callers are primarily CMA and memory hotplug and need 3262 * the drain to be complete when the call returns. 3263 */ 3264 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) { 3265 if (!zone) 3266 return; 3267 mutex_lock(&pcpu_drain_mutex); 3268 } 3269 3270 /* 3271 * We don't care about racing with CPU hotplug event 3272 * as offline notification will cause the notified 3273 * cpu to drain that CPU pcps and on_each_cpu_mask 3274 * disables preemption as part of its processing 3275 */ 3276 for_each_online_cpu(cpu) { 3277 struct per_cpu_pages *pcp; 3278 struct zone *z; 3279 bool has_pcps = false; 3280 3281 if (force_all_cpus) { 3282 /* 3283 * The pcp.count check is racy, some callers need a 3284 * guarantee that no cpu is missed. 3285 */ 3286 has_pcps = true; 3287 } else if (zone) { 3288 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 3289 if (pcp->count) 3290 has_pcps = true; 3291 } else { 3292 for_each_populated_zone(z) { 3293 pcp = per_cpu_ptr(z->per_cpu_pageset, cpu); 3294 if (pcp->count) { 3295 has_pcps = true; 3296 break; 3297 } 3298 } 3299 } 3300 3301 if (has_pcps) 3302 cpumask_set_cpu(cpu, &cpus_with_pcps); 3303 else 3304 cpumask_clear_cpu(cpu, &cpus_with_pcps); 3305 } 3306 3307 for_each_cpu(cpu, &cpus_with_pcps) { 3308 if (zone) 3309 drain_pages_zone(cpu, zone); 3310 else 3311 drain_pages(cpu); 3312 } 3313 3314 mutex_unlock(&pcpu_drain_mutex); 3315 } 3316 3317 /* 3318 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 3319 * 3320 * When zone parameter is non-NULL, spill just the single zone's pages. 3321 */ 3322 void drain_all_pages(struct zone *zone) 3323 { 3324 __drain_all_pages(zone, false); 3325 } 3326 3327 #ifdef CONFIG_HIBERNATION 3328 3329 /* 3330 * Touch the watchdog for every WD_PAGE_COUNT pages. 3331 */ 3332 #define WD_PAGE_COUNT (128*1024) 3333 3334 void mark_free_pages(struct zone *zone) 3335 { 3336 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT; 3337 unsigned long flags; 3338 unsigned int order, t; 3339 struct page *page; 3340 3341 if (zone_is_empty(zone)) 3342 return; 3343 3344 spin_lock_irqsave(&zone->lock, flags); 3345 3346 max_zone_pfn = zone_end_pfn(zone); 3347 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 3348 if (pfn_valid(pfn)) { 3349 page = pfn_to_page(pfn); 3350 3351 if (!--page_count) { 3352 touch_nmi_watchdog(); 3353 page_count = WD_PAGE_COUNT; 3354 } 3355 3356 if (page_zone(page) != zone) 3357 continue; 3358 3359 if (!swsusp_page_is_forbidden(page)) 3360 swsusp_unset_page_free(page); 3361 } 3362 3363 for_each_migratetype_order(order, t) { 3364 list_for_each_entry(page, 3365 &zone->free_area[order].free_list[t], buddy_list) { 3366 unsigned long i; 3367 3368 pfn = page_to_pfn(page); 3369 for (i = 0; i < (1UL << order); i++) { 3370 if (!--page_count) { 3371 touch_nmi_watchdog(); 3372 page_count = WD_PAGE_COUNT; 3373 } 3374 swsusp_set_page_free(pfn_to_page(pfn + i)); 3375 } 3376 } 3377 } 3378 spin_unlock_irqrestore(&zone->lock, flags); 3379 } 3380 #endif /* CONFIG_PM */ 3381 3382 static bool free_unref_page_prepare(struct page *page, unsigned long pfn, 3383 unsigned int order) 3384 { 3385 int migratetype; 3386 3387 if (!free_pcp_prepare(page, order)) 3388 return false; 3389 3390 migratetype = get_pfnblock_migratetype(page, pfn); 3391 set_pcppage_migratetype(page, migratetype); 3392 return true; 3393 } 3394 3395 static int nr_pcp_free(struct per_cpu_pages *pcp, int high, int batch, 3396 bool free_high) 3397 { 3398 int min_nr_free, max_nr_free; 3399 3400 /* Free everything if batch freeing high-order pages. */ 3401 if (unlikely(free_high)) 3402 return pcp->count; 3403 3404 /* Check for PCP disabled or boot pageset */ 3405 if (unlikely(high < batch)) 3406 return 1; 3407 3408 /* Leave at least pcp->batch pages on the list */ 3409 min_nr_free = batch; 3410 max_nr_free = high - batch; 3411 3412 /* 3413 * Double the number of pages freed each time there is subsequent 3414 * freeing of pages without any allocation. 3415 */ 3416 batch <<= pcp->free_factor; 3417 if (batch < max_nr_free) 3418 pcp->free_factor++; 3419 batch = clamp(batch, min_nr_free, max_nr_free); 3420 3421 return batch; 3422 } 3423 3424 static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone, 3425 bool free_high) 3426 { 3427 int high = READ_ONCE(pcp->high); 3428 3429 if (unlikely(!high || free_high)) 3430 return 0; 3431 3432 if (!test_bit(ZONE_RECLAIM_ACTIVE, &zone->flags)) 3433 return high; 3434 3435 /* 3436 * If reclaim is active, limit the number of pages that can be 3437 * stored on pcp lists 3438 */ 3439 return min(READ_ONCE(pcp->batch) << 2, high); 3440 } 3441 3442 static void free_unref_page_commit(struct zone *zone, struct per_cpu_pages *pcp, 3443 struct page *page, int migratetype, 3444 unsigned int order) 3445 { 3446 int high; 3447 int pindex; 3448 bool free_high; 3449 3450 __count_vm_events(PGFREE, 1 << order); 3451 pindex = order_to_pindex(migratetype, order); 3452 list_add(&page->pcp_list, &pcp->lists[pindex]); 3453 pcp->count += 1 << order; 3454 3455 /* 3456 * As high-order pages other than THP's stored on PCP can contribute 3457 * to fragmentation, limit the number stored when PCP is heavily 3458 * freeing without allocation. The remainder after bulk freeing 3459 * stops will be drained from vmstat refresh context. 3460 */ 3461 free_high = (pcp->free_factor && order && order <= PAGE_ALLOC_COSTLY_ORDER); 3462 3463 high = nr_pcp_high(pcp, zone, free_high); 3464 if (pcp->count >= high) { 3465 int batch = READ_ONCE(pcp->batch); 3466 3467 free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch, free_high), pcp, pindex); 3468 } 3469 } 3470 3471 /* 3472 * Free a pcp page 3473 */ 3474 void free_unref_page(struct page *page, unsigned int order) 3475 { 3476 unsigned long __maybe_unused UP_flags; 3477 struct per_cpu_pages *pcp; 3478 struct zone *zone; 3479 unsigned long pfn = page_to_pfn(page); 3480 int migratetype; 3481 3482 if (!free_unref_page_prepare(page, pfn, order)) 3483 return; 3484 3485 /* 3486 * We only track unmovable, reclaimable and movable on pcp lists. 3487 * Place ISOLATE pages on the isolated list because they are being 3488 * offlined but treat HIGHATOMIC as movable pages so we can get those 3489 * areas back if necessary. Otherwise, we may have to free 3490 * excessively into the page allocator 3491 */ 3492 migratetype = get_pcppage_migratetype(page); 3493 if (unlikely(migratetype >= MIGRATE_PCPTYPES)) { 3494 if (unlikely(is_migrate_isolate(migratetype))) { 3495 free_one_page(page_zone(page), page, pfn, order, migratetype, FPI_NONE); 3496 return; 3497 } 3498 migratetype = MIGRATE_MOVABLE; 3499 } 3500 3501 zone = page_zone(page); 3502 pcp_trylock_prepare(UP_flags); 3503 pcp = pcp_spin_trylock(zone->per_cpu_pageset); 3504 if (pcp) { 3505 free_unref_page_commit(zone, pcp, page, migratetype, order); 3506 pcp_spin_unlock(pcp); 3507 } else { 3508 free_one_page(zone, page, pfn, order, migratetype, FPI_NONE); 3509 } 3510 pcp_trylock_finish(UP_flags); 3511 } 3512 3513 /* 3514 * Free a list of 0-order pages 3515 */ 3516 void free_unref_page_list(struct list_head *list) 3517 { 3518 unsigned long __maybe_unused UP_flags; 3519 struct page *page, *next; 3520 struct per_cpu_pages *pcp = NULL; 3521 struct zone *locked_zone = NULL; 3522 int batch_count = 0; 3523 int migratetype; 3524 3525 /* Prepare pages for freeing */ 3526 list_for_each_entry_safe(page, next, list, lru) { 3527 unsigned long pfn = page_to_pfn(page); 3528 if (!free_unref_page_prepare(page, pfn, 0)) { 3529 list_del(&page->lru); 3530 continue; 3531 } 3532 3533 /* 3534 * Free isolated pages directly to the allocator, see 3535 * comment in free_unref_page. 3536 */ 3537 migratetype = get_pcppage_migratetype(page); 3538 if (unlikely(is_migrate_isolate(migratetype))) { 3539 list_del(&page->lru); 3540 free_one_page(page_zone(page), page, pfn, 0, migratetype, FPI_NONE); 3541 continue; 3542 } 3543 } 3544 3545 list_for_each_entry_safe(page, next, list, lru) { 3546 struct zone *zone = page_zone(page); 3547 3548 list_del(&page->lru); 3549 migratetype = get_pcppage_migratetype(page); 3550 3551 /* 3552 * Either different zone requiring a different pcp lock or 3553 * excessive lock hold times when freeing a large list of 3554 * pages. 3555 */ 3556 if (zone != locked_zone || batch_count == SWAP_CLUSTER_MAX) { 3557 if (pcp) { 3558 pcp_spin_unlock(pcp); 3559 pcp_trylock_finish(UP_flags); 3560 } 3561 3562 batch_count = 0; 3563 3564 /* 3565 * trylock is necessary as pages may be getting freed 3566 * from IRQ or SoftIRQ context after an IO completion. 3567 */ 3568 pcp_trylock_prepare(UP_flags); 3569 pcp = pcp_spin_trylock(zone->per_cpu_pageset); 3570 if (unlikely(!pcp)) { 3571 pcp_trylock_finish(UP_flags); 3572 free_one_page(zone, page, page_to_pfn(page), 3573 0, migratetype, FPI_NONE); 3574 locked_zone = NULL; 3575 continue; 3576 } 3577 locked_zone = zone; 3578 } 3579 3580 /* 3581 * Non-isolated types over MIGRATE_PCPTYPES get added 3582 * to the MIGRATE_MOVABLE pcp list. 3583 */ 3584 if (unlikely(migratetype >= MIGRATE_PCPTYPES)) 3585 migratetype = MIGRATE_MOVABLE; 3586 3587 trace_mm_page_free_batched(page); 3588 free_unref_page_commit(zone, pcp, page, migratetype, 0); 3589 batch_count++; 3590 } 3591 3592 if (pcp) { 3593 pcp_spin_unlock(pcp); 3594 pcp_trylock_finish(UP_flags); 3595 } 3596 } 3597 3598 /* 3599 * split_page takes a non-compound higher-order page, and splits it into 3600 * n (1<<order) sub-pages: page[0..n] 3601 * Each sub-page must be freed individually. 3602 * 3603 * Note: this is probably too low level an operation for use in drivers. 3604 * Please consult with lkml before using this in your driver. 3605 */ 3606 void split_page(struct page *page, unsigned int order) 3607 { 3608 int i; 3609 3610 VM_BUG_ON_PAGE(PageCompound(page), page); 3611 VM_BUG_ON_PAGE(!page_count(page), page); 3612 3613 for (i = 1; i < (1 << order); i++) 3614 set_page_refcounted(page + i); 3615 split_page_owner(page, 1 << order); 3616 split_page_memcg(page, 1 << order); 3617 } 3618 EXPORT_SYMBOL_GPL(split_page); 3619 3620 int __isolate_free_page(struct page *page, unsigned int order) 3621 { 3622 struct zone *zone = page_zone(page); 3623 int mt = get_pageblock_migratetype(page); 3624 3625 if (!is_migrate_isolate(mt)) { 3626 unsigned long watermark; 3627 /* 3628 * Obey watermarks as if the page was being allocated. We can 3629 * emulate a high-order watermark check with a raised order-0 3630 * watermark, because we already know our high-order page 3631 * exists. 3632 */ 3633 watermark = zone->_watermark[WMARK_MIN] + (1UL << order); 3634 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 3635 return 0; 3636 3637 __mod_zone_freepage_state(zone, -(1UL << order), mt); 3638 } 3639 3640 del_page_from_free_list(page, zone, order); 3641 3642 /* 3643 * Set the pageblock if the isolated page is at least half of a 3644 * pageblock 3645 */ 3646 if (order >= pageblock_order - 1) { 3647 struct page *endpage = page + (1 << order) - 1; 3648 for (; page < endpage; page += pageblock_nr_pages) { 3649 int mt = get_pageblock_migratetype(page); 3650 /* 3651 * Only change normal pageblocks (i.e., they can merge 3652 * with others) 3653 */ 3654 if (migratetype_is_mergeable(mt)) 3655 set_pageblock_migratetype(page, 3656 MIGRATE_MOVABLE); 3657 } 3658 } 3659 3660 return 1UL << order; 3661 } 3662 3663 /** 3664 * __putback_isolated_page - Return a now-isolated page back where we got it 3665 * @page: Page that was isolated 3666 * @order: Order of the isolated page 3667 * @mt: The page's pageblock's migratetype 3668 * 3669 * This function is meant to return a page pulled from the free lists via 3670 * __isolate_free_page back to the free lists they were pulled from. 3671 */ 3672 void __putback_isolated_page(struct page *page, unsigned int order, int mt) 3673 { 3674 struct zone *zone = page_zone(page); 3675 3676 /* zone lock should be held when this function is called */ 3677 lockdep_assert_held(&zone->lock); 3678 3679 /* Return isolated page to tail of freelist. */ 3680 __free_one_page(page, page_to_pfn(page), zone, order, mt, 3681 FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL); 3682 } 3683 3684 /* 3685 * Update NUMA hit/miss statistics 3686 */ 3687 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z, 3688 long nr_account) 3689 { 3690 #ifdef CONFIG_NUMA 3691 enum numa_stat_item local_stat = NUMA_LOCAL; 3692 3693 /* skip numa counters update if numa stats is disabled */ 3694 if (!static_branch_likely(&vm_numa_stat_key)) 3695 return; 3696 3697 if (zone_to_nid(z) != numa_node_id()) 3698 local_stat = NUMA_OTHER; 3699 3700 if (zone_to_nid(z) == zone_to_nid(preferred_zone)) 3701 __count_numa_events(z, NUMA_HIT, nr_account); 3702 else { 3703 __count_numa_events(z, NUMA_MISS, nr_account); 3704 __count_numa_events(preferred_zone, NUMA_FOREIGN, nr_account); 3705 } 3706 __count_numa_events(z, local_stat, nr_account); 3707 #endif 3708 } 3709 3710 static __always_inline 3711 struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, 3712 unsigned int order, unsigned int alloc_flags, 3713 int migratetype) 3714 { 3715 struct page *page; 3716 unsigned long flags; 3717 3718 do { 3719 page = NULL; 3720 spin_lock_irqsave(&zone->lock, flags); 3721 /* 3722 * order-0 request can reach here when the pcplist is skipped 3723 * due to non-CMA allocation context. HIGHATOMIC area is 3724 * reserved for high-order atomic allocation, so order-0 3725 * request should skip it. 3726 */ 3727 if (alloc_flags & ALLOC_HIGHATOMIC) 3728 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3729 if (!page) { 3730 page = __rmqueue(zone, order, migratetype, alloc_flags); 3731 3732 /* 3733 * If the allocation fails, allow OOM handling access 3734 * to HIGHATOMIC reserves as failing now is worse than 3735 * failing a high-order atomic allocation in the 3736 * future. 3737 */ 3738 if (!page && (alloc_flags & ALLOC_OOM)) 3739 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3740 3741 if (!page) { 3742 spin_unlock_irqrestore(&zone->lock, flags); 3743 return NULL; 3744 } 3745 } 3746 __mod_zone_freepage_state(zone, -(1 << order), 3747 get_pcppage_migratetype(page)); 3748 spin_unlock_irqrestore(&zone->lock, flags); 3749 } while (check_new_pages(page, order)); 3750 3751 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3752 zone_statistics(preferred_zone, zone, 1); 3753 3754 return page; 3755 } 3756 3757 /* Remove page from the per-cpu list, caller must protect the list */ 3758 static inline 3759 struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, 3760 int migratetype, 3761 unsigned int alloc_flags, 3762 struct per_cpu_pages *pcp, 3763 struct list_head *list) 3764 { 3765 struct page *page; 3766 3767 do { 3768 if (list_empty(list)) { 3769 int batch = READ_ONCE(pcp->batch); 3770 int alloced; 3771 3772 /* 3773 * Scale batch relative to order if batch implies 3774 * free pages can be stored on the PCP. Batch can 3775 * be 1 for small zones or for boot pagesets which 3776 * should never store free pages as the pages may 3777 * belong to arbitrary zones. 3778 */ 3779 if (batch > 1) 3780 batch = max(batch >> order, 2); 3781 alloced = rmqueue_bulk(zone, order, 3782 batch, list, 3783 migratetype, alloc_flags); 3784 3785 pcp->count += alloced << order; 3786 if (unlikely(list_empty(list))) 3787 return NULL; 3788 } 3789 3790 page = list_first_entry(list, struct page, pcp_list); 3791 list_del(&page->pcp_list); 3792 pcp->count -= 1 << order; 3793 } while (check_new_pcp(page, order)); 3794 3795 return page; 3796 } 3797 3798 /* Lock and remove page from the per-cpu list */ 3799 static struct page *rmqueue_pcplist(struct zone *preferred_zone, 3800 struct zone *zone, unsigned int order, 3801 int migratetype, unsigned int alloc_flags) 3802 { 3803 struct per_cpu_pages *pcp; 3804 struct list_head *list; 3805 struct page *page; 3806 unsigned long __maybe_unused UP_flags; 3807 3808 /* spin_trylock may fail due to a parallel drain or IRQ reentrancy. */ 3809 pcp_trylock_prepare(UP_flags); 3810 pcp = pcp_spin_trylock(zone->per_cpu_pageset); 3811 if (!pcp) { 3812 pcp_trylock_finish(UP_flags); 3813 return NULL; 3814 } 3815 3816 /* 3817 * On allocation, reduce the number of pages that are batch freed. 3818 * See nr_pcp_free() where free_factor is increased for subsequent 3819 * frees. 3820 */ 3821 pcp->free_factor >>= 1; 3822 list = &pcp->lists[order_to_pindex(migratetype, order)]; 3823 page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list); 3824 pcp_spin_unlock(pcp); 3825 pcp_trylock_finish(UP_flags); 3826 if (page) { 3827 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3828 zone_statistics(preferred_zone, zone, 1); 3829 } 3830 return page; 3831 } 3832 3833 /* 3834 * Allocate a page from the given zone. 3835 * Use pcplists for THP or "cheap" high-order allocations. 3836 */ 3837 3838 /* 3839 * Do not instrument rmqueue() with KMSAN. This function may call 3840 * __msan_poison_alloca() through a call to set_pfnblock_flags_mask(). 3841 * If __msan_poison_alloca() attempts to allocate pages for the stack depot, it 3842 * may call rmqueue() again, which will result in a deadlock. 3843 */ 3844 __no_sanitize_memory 3845 static inline 3846 struct page *rmqueue(struct zone *preferred_zone, 3847 struct zone *zone, unsigned int order, 3848 gfp_t gfp_flags, unsigned int alloc_flags, 3849 int migratetype) 3850 { 3851 struct page *page; 3852 3853 /* 3854 * We most definitely don't want callers attempting to 3855 * allocate greater than order-1 page units with __GFP_NOFAIL. 3856 */ 3857 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 3858 3859 if (likely(pcp_allowed_order(order))) { 3860 /* 3861 * MIGRATE_MOVABLE pcplist could have the pages on CMA area and 3862 * we need to skip it when CMA area isn't allowed. 3863 */ 3864 if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA || 3865 migratetype != MIGRATE_MOVABLE) { 3866 page = rmqueue_pcplist(preferred_zone, zone, order, 3867 migratetype, alloc_flags); 3868 if (likely(page)) 3869 goto out; 3870 } 3871 } 3872 3873 page = rmqueue_buddy(preferred_zone, zone, order, alloc_flags, 3874 migratetype); 3875 3876 out: 3877 /* Separate test+clear to avoid unnecessary atomics */ 3878 if (unlikely(test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags))) { 3879 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 3880 wakeup_kswapd(zone, 0, 0, zone_idx(zone)); 3881 } 3882 3883 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); 3884 return page; 3885 } 3886 3887 #ifdef CONFIG_FAIL_PAGE_ALLOC 3888 3889 static struct { 3890 struct fault_attr attr; 3891 3892 bool ignore_gfp_highmem; 3893 bool ignore_gfp_reclaim; 3894 u32 min_order; 3895 } fail_page_alloc = { 3896 .attr = FAULT_ATTR_INITIALIZER, 3897 .ignore_gfp_reclaim = true, 3898 .ignore_gfp_highmem = true, 3899 .min_order = 1, 3900 }; 3901 3902 static int __init setup_fail_page_alloc(char *str) 3903 { 3904 return setup_fault_attr(&fail_page_alloc.attr, str); 3905 } 3906 __setup("fail_page_alloc=", setup_fail_page_alloc); 3907 3908 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3909 { 3910 int flags = 0; 3911 3912 if (order < fail_page_alloc.min_order) 3913 return false; 3914 if (gfp_mask & __GFP_NOFAIL) 3915 return false; 3916 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 3917 return false; 3918 if (fail_page_alloc.ignore_gfp_reclaim && 3919 (gfp_mask & __GFP_DIRECT_RECLAIM)) 3920 return false; 3921 3922 /* See comment in __should_failslab() */ 3923 if (gfp_mask & __GFP_NOWARN) 3924 flags |= FAULT_NOWARN; 3925 3926 return should_fail_ex(&fail_page_alloc.attr, 1 << order, flags); 3927 } 3928 3929 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 3930 3931 static int __init fail_page_alloc_debugfs(void) 3932 { 3933 umode_t mode = S_IFREG | 0600; 3934 struct dentry *dir; 3935 3936 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 3937 &fail_page_alloc.attr); 3938 3939 debugfs_create_bool("ignore-gfp-wait", mode, dir, 3940 &fail_page_alloc.ignore_gfp_reclaim); 3941 debugfs_create_bool("ignore-gfp-highmem", mode, dir, 3942 &fail_page_alloc.ignore_gfp_highmem); 3943 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order); 3944 3945 return 0; 3946 } 3947 3948 late_initcall(fail_page_alloc_debugfs); 3949 3950 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 3951 3952 #else /* CONFIG_FAIL_PAGE_ALLOC */ 3953 3954 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3955 { 3956 return false; 3957 } 3958 3959 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 3960 3961 noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3962 { 3963 return __should_fail_alloc_page(gfp_mask, order); 3964 } 3965 ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE); 3966 3967 static inline long __zone_watermark_unusable_free(struct zone *z, 3968 unsigned int order, unsigned int alloc_flags) 3969 { 3970 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3971 long unusable_free = (1 << order) - 1; 3972 3973 /* 3974 * If the caller does not have rights to ALLOC_HARDER then subtract 3975 * the high-atomic reserves. This will over-estimate the size of the 3976 * atomic reserve but it avoids a search. 3977 */ 3978 if (likely(!alloc_harder)) 3979 unusable_free += z->nr_reserved_highatomic; 3980 3981 #ifdef CONFIG_CMA 3982 /* If allocation can't use CMA areas don't use free CMA pages */ 3983 if (!(alloc_flags & ALLOC_CMA)) 3984 unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES); 3985 #endif 3986 3987 return unusable_free; 3988 } 3989 3990 /* 3991 * Return true if free base pages are above 'mark'. For high-order checks it 3992 * will return true of the order-0 watermark is reached and there is at least 3993 * one free page of a suitable size. Checking now avoids taking the zone lock 3994 * to check in the allocation paths if no pages are free. 3995 */ 3996 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3997 int highest_zoneidx, unsigned int alloc_flags, 3998 long free_pages) 3999 { 4000 long min = mark; 4001 int o; 4002 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 4003 4004 /* free_pages may go negative - that's OK */ 4005 free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags); 4006 4007 if (alloc_flags & ALLOC_MIN_RESERVE) 4008 min -= min / 2; 4009 4010 if (unlikely(alloc_harder)) { 4011 /* 4012 * OOM victims can try even harder than normal ALLOC_HARDER 4013 * users on the grounds that it's definitely going to be in 4014 * the exit path shortly and free memory. Any allocation it 4015 * makes during the free path will be small and short-lived. 4016 */ 4017 if (alloc_flags & ALLOC_OOM) 4018 min -= min / 2; 4019 else 4020 min -= min / 4; 4021 } 4022 4023 /* 4024 * Check watermarks for an order-0 allocation request. If these 4025 * are not met, then a high-order request also cannot go ahead 4026 * even if a suitable page happened to be free. 4027 */ 4028 if (free_pages <= min + z->lowmem_reserve[highest_zoneidx]) 4029 return false; 4030 4031 /* If this is an order-0 request then the watermark is fine */ 4032 if (!order) 4033 return true; 4034 4035 /* For a high-order request, check at least one suitable page is free */ 4036 for (o = order; o < MAX_ORDER; o++) { 4037 struct free_area *area = &z->free_area[o]; 4038 int mt; 4039 4040 if (!area->nr_free) 4041 continue; 4042 4043 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 4044 if (!free_area_empty(area, mt)) 4045 return true; 4046 } 4047 4048 #ifdef CONFIG_CMA 4049 if ((alloc_flags & ALLOC_CMA) && 4050 !free_area_empty(area, MIGRATE_CMA)) { 4051 return true; 4052 } 4053 #endif 4054 if ((alloc_flags & (ALLOC_HIGHATOMIC|ALLOC_OOM)) && 4055 !free_area_empty(area, MIGRATE_HIGHATOMIC)) { 4056 return true; 4057 } 4058 } 4059 return false; 4060 } 4061 4062 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 4063 int highest_zoneidx, unsigned int alloc_flags) 4064 { 4065 return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 4066 zone_page_state(z, NR_FREE_PAGES)); 4067 } 4068 4069 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 4070 unsigned long mark, int highest_zoneidx, 4071 unsigned int alloc_flags, gfp_t gfp_mask) 4072 { 4073 long free_pages; 4074 4075 free_pages = zone_page_state(z, NR_FREE_PAGES); 4076 4077 /* 4078 * Fast check for order-0 only. If this fails then the reserves 4079 * need to be calculated. 4080 */ 4081 if (!order) { 4082 long usable_free; 4083 long reserved; 4084 4085 usable_free = free_pages; 4086 reserved = __zone_watermark_unusable_free(z, 0, alloc_flags); 4087 4088 /* reserved may over estimate high-atomic reserves. */ 4089 usable_free -= min(usable_free, reserved); 4090 if (usable_free > mark + z->lowmem_reserve[highest_zoneidx]) 4091 return true; 4092 } 4093 4094 if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 4095 free_pages)) 4096 return true; 4097 /* 4098 * Ignore watermark boosting for GFP_ATOMIC order-0 allocations 4099 * when checking the min watermark. The min watermark is the 4100 * point where boosting is ignored so that kswapd is woken up 4101 * when below the low watermark. 4102 */ 4103 if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost 4104 && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) { 4105 mark = z->_watermark[WMARK_MIN]; 4106 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 4107 alloc_flags, free_pages); 4108 } 4109 4110 return false; 4111 } 4112 4113 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 4114 unsigned long mark, int highest_zoneidx) 4115 { 4116 long free_pages = zone_page_state(z, NR_FREE_PAGES); 4117 4118 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 4119 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 4120 4121 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0, 4122 free_pages); 4123 } 4124 4125 #ifdef CONFIG_NUMA 4126 int __read_mostly node_reclaim_distance = RECLAIM_DISTANCE; 4127 4128 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 4129 { 4130 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= 4131 node_reclaim_distance; 4132 } 4133 #else /* CONFIG_NUMA */ 4134 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 4135 { 4136 return true; 4137 } 4138 #endif /* CONFIG_NUMA */ 4139 4140 /* 4141 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid 4142 * fragmentation is subtle. If the preferred zone was HIGHMEM then 4143 * premature use of a lower zone may cause lowmem pressure problems that 4144 * are worse than fragmentation. If the next zone is ZONE_DMA then it is 4145 * probably too small. It only makes sense to spread allocations to avoid 4146 * fragmentation between the Normal and DMA32 zones. 4147 */ 4148 static inline unsigned int 4149 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) 4150 { 4151 unsigned int alloc_flags; 4152 4153 /* 4154 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 4155 * to save a branch. 4156 */ 4157 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); 4158 4159 #ifdef CONFIG_ZONE_DMA32 4160 if (!zone) 4161 return alloc_flags; 4162 4163 if (zone_idx(zone) != ZONE_NORMAL) 4164 return alloc_flags; 4165 4166 /* 4167 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and 4168 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume 4169 * on UMA that if Normal is populated then so is DMA32. 4170 */ 4171 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1); 4172 if (nr_online_nodes > 1 && !populated_zone(--zone)) 4173 return alloc_flags; 4174 4175 alloc_flags |= ALLOC_NOFRAGMENT; 4176 #endif /* CONFIG_ZONE_DMA32 */ 4177 return alloc_flags; 4178 } 4179 4180 /* Must be called after current_gfp_context() which can change gfp_mask */ 4181 static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask, 4182 unsigned int alloc_flags) 4183 { 4184 #ifdef CONFIG_CMA 4185 if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) 4186 alloc_flags |= ALLOC_CMA; 4187 #endif 4188 return alloc_flags; 4189 } 4190 4191 /* 4192 * get_page_from_freelist goes through the zonelist trying to allocate 4193 * a page. 4194 */ 4195 static struct page * 4196 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 4197 const struct alloc_context *ac) 4198 { 4199 struct zoneref *z; 4200 struct zone *zone; 4201 struct pglist_data *last_pgdat = NULL; 4202 bool last_pgdat_dirty_ok = false; 4203 bool no_fallback; 4204 4205 retry: 4206 /* 4207 * Scan zonelist, looking for a zone with enough free. 4208 * See also __cpuset_node_allowed() comment in kernel/cgroup/cpuset.c. 4209 */ 4210 no_fallback = alloc_flags & ALLOC_NOFRAGMENT; 4211 z = ac->preferred_zoneref; 4212 for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx, 4213 ac->nodemask) { 4214 struct page *page; 4215 unsigned long mark; 4216 4217 if (cpusets_enabled() && 4218 (alloc_flags & ALLOC_CPUSET) && 4219 !__cpuset_zone_allowed(zone, gfp_mask)) 4220 continue; 4221 /* 4222 * When allocating a page cache page for writing, we 4223 * want to get it from a node that is within its dirty 4224 * limit, such that no single node holds more than its 4225 * proportional share of globally allowed dirty pages. 4226 * The dirty limits take into account the node's 4227 * lowmem reserves and high watermark so that kswapd 4228 * should be able to balance it without having to 4229 * write pages from its LRU list. 4230 * 4231 * XXX: For now, allow allocations to potentially 4232 * exceed the per-node dirty limit in the slowpath 4233 * (spread_dirty_pages unset) before going into reclaim, 4234 * which is important when on a NUMA setup the allowed 4235 * nodes are together not big enough to reach the 4236 * global limit. The proper fix for these situations 4237 * will require awareness of nodes in the 4238 * dirty-throttling and the flusher threads. 4239 */ 4240 if (ac->spread_dirty_pages) { 4241 if (last_pgdat != zone->zone_pgdat) { 4242 last_pgdat = zone->zone_pgdat; 4243 last_pgdat_dirty_ok = node_dirty_ok(zone->zone_pgdat); 4244 } 4245 4246 if (!last_pgdat_dirty_ok) 4247 continue; 4248 } 4249 4250 if (no_fallback && nr_online_nodes > 1 && 4251 zone != ac->preferred_zoneref->zone) { 4252 int local_nid; 4253 4254 /* 4255 * If moving to a remote node, retry but allow 4256 * fragmenting fallbacks. Locality is more important 4257 * than fragmentation avoidance. 4258 */ 4259 local_nid = zone_to_nid(ac->preferred_zoneref->zone); 4260 if (zone_to_nid(zone) != local_nid) { 4261 alloc_flags &= ~ALLOC_NOFRAGMENT; 4262 goto retry; 4263 } 4264 } 4265 4266 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); 4267 if (!zone_watermark_fast(zone, order, mark, 4268 ac->highest_zoneidx, alloc_flags, 4269 gfp_mask)) { 4270 int ret; 4271 4272 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 4273 /* 4274 * Watermark failed for this zone, but see if we can 4275 * grow this zone if it contains deferred pages. 4276 */ 4277 if (static_branch_unlikely(&deferred_pages)) { 4278 if (_deferred_grow_zone(zone, order)) 4279 goto try_this_zone; 4280 } 4281 #endif 4282 /* Checked here to keep the fast path fast */ 4283 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 4284 if (alloc_flags & ALLOC_NO_WATERMARKS) 4285 goto try_this_zone; 4286 4287 if (!node_reclaim_enabled() || 4288 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 4289 continue; 4290 4291 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 4292 switch (ret) { 4293 case NODE_RECLAIM_NOSCAN: 4294 /* did not scan */ 4295 continue; 4296 case NODE_RECLAIM_FULL: 4297 /* scanned but unreclaimable */ 4298 continue; 4299 default: 4300 /* did we reclaim enough */ 4301 if (zone_watermark_ok(zone, order, mark, 4302 ac->highest_zoneidx, alloc_flags)) 4303 goto try_this_zone; 4304 4305 continue; 4306 } 4307 } 4308 4309 try_this_zone: 4310 page = rmqueue(ac->preferred_zoneref->zone, zone, order, 4311 gfp_mask, alloc_flags, ac->migratetype); 4312 if (page) { 4313 prep_new_page(page, order, gfp_mask, alloc_flags); 4314 4315 /* 4316 * If this is a high-order atomic allocation then check 4317 * if the pageblock should be reserved for the future 4318 */ 4319 if (unlikely(alloc_flags & ALLOC_HIGHATOMIC)) 4320 reserve_highatomic_pageblock(page, zone, order); 4321 4322 return page; 4323 } else { 4324 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 4325 /* Try again if zone has deferred pages */ 4326 if (static_branch_unlikely(&deferred_pages)) { 4327 if (_deferred_grow_zone(zone, order)) 4328 goto try_this_zone; 4329 } 4330 #endif 4331 } 4332 } 4333 4334 /* 4335 * It's possible on a UMA machine to get through all zones that are 4336 * fragmented. If avoiding fragmentation, reset and try again. 4337 */ 4338 if (no_fallback) { 4339 alloc_flags &= ~ALLOC_NOFRAGMENT; 4340 goto retry; 4341 } 4342 4343 return NULL; 4344 } 4345 4346 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) 4347 { 4348 unsigned int filter = SHOW_MEM_FILTER_NODES; 4349 4350 /* 4351 * This documents exceptions given to allocations in certain 4352 * contexts that are allowed to allocate outside current's set 4353 * of allowed nodes. 4354 */ 4355 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4356 if (tsk_is_oom_victim(current) || 4357 (current->flags & (PF_MEMALLOC | PF_EXITING))) 4358 filter &= ~SHOW_MEM_FILTER_NODES; 4359 if (!in_task() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 4360 filter &= ~SHOW_MEM_FILTER_NODES; 4361 4362 __show_mem(filter, nodemask, gfp_zone(gfp_mask)); 4363 } 4364 4365 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) 4366 { 4367 struct va_format vaf; 4368 va_list args; 4369 static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1); 4370 4371 if ((gfp_mask & __GFP_NOWARN) || 4372 !__ratelimit(&nopage_rs) || 4373 ((gfp_mask & __GFP_DMA) && !has_managed_dma())) 4374 return; 4375 4376 va_start(args, fmt); 4377 vaf.fmt = fmt; 4378 vaf.va = &args; 4379 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl", 4380 current->comm, &vaf, gfp_mask, &gfp_mask, 4381 nodemask_pr_args(nodemask)); 4382 va_end(args); 4383 4384 cpuset_print_current_mems_allowed(); 4385 pr_cont("\n"); 4386 dump_stack(); 4387 warn_alloc_show_mem(gfp_mask, nodemask); 4388 } 4389 4390 static inline struct page * 4391 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, 4392 unsigned int alloc_flags, 4393 const struct alloc_context *ac) 4394 { 4395 struct page *page; 4396 4397 page = get_page_from_freelist(gfp_mask, order, 4398 alloc_flags|ALLOC_CPUSET, ac); 4399 /* 4400 * fallback to ignore cpuset restriction if our nodes 4401 * are depleted 4402 */ 4403 if (!page) 4404 page = get_page_from_freelist(gfp_mask, order, 4405 alloc_flags, ac); 4406 4407 return page; 4408 } 4409 4410 static inline struct page * 4411 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 4412 const struct alloc_context *ac, unsigned long *did_some_progress) 4413 { 4414 struct oom_control oc = { 4415 .zonelist = ac->zonelist, 4416 .nodemask = ac->nodemask, 4417 .memcg = NULL, 4418 .gfp_mask = gfp_mask, 4419 .order = order, 4420 }; 4421 struct page *page; 4422 4423 *did_some_progress = 0; 4424 4425 /* 4426 * Acquire the oom lock. If that fails, somebody else is 4427 * making progress for us. 4428 */ 4429 if (!mutex_trylock(&oom_lock)) { 4430 *did_some_progress = 1; 4431 schedule_timeout_uninterruptible(1); 4432 return NULL; 4433 } 4434 4435 /* 4436 * Go through the zonelist yet one more time, keep very high watermark 4437 * here, this is only to catch a parallel oom killing, we must fail if 4438 * we're still under heavy pressure. But make sure that this reclaim 4439 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY 4440 * allocation which will never fail due to oom_lock already held. 4441 */ 4442 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & 4443 ~__GFP_DIRECT_RECLAIM, order, 4444 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 4445 if (page) 4446 goto out; 4447 4448 /* Coredumps can quickly deplete all memory reserves */ 4449 if (current->flags & PF_DUMPCORE) 4450 goto out; 4451 /* The OOM killer will not help higher order allocs */ 4452 if (order > PAGE_ALLOC_COSTLY_ORDER) 4453 goto out; 4454 /* 4455 * We have already exhausted all our reclaim opportunities without any 4456 * success so it is time to admit defeat. We will skip the OOM killer 4457 * because it is very likely that the caller has a more reasonable 4458 * fallback than shooting a random task. 4459 * 4460 * The OOM killer may not free memory on a specific node. 4461 */ 4462 if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE)) 4463 goto out; 4464 /* The OOM killer does not needlessly kill tasks for lowmem */ 4465 if (ac->highest_zoneidx < ZONE_NORMAL) 4466 goto out; 4467 if (pm_suspended_storage()) 4468 goto out; 4469 /* 4470 * XXX: GFP_NOFS allocations should rather fail than rely on 4471 * other request to make a forward progress. 4472 * We are in an unfortunate situation where out_of_memory cannot 4473 * do much for this context but let's try it to at least get 4474 * access to memory reserved if the current task is killed (see 4475 * out_of_memory). Once filesystems are ready to handle allocation 4476 * failures more gracefully we should just bail out here. 4477 */ 4478 4479 /* Exhausted what can be done so it's blame time */ 4480 if (out_of_memory(&oc) || 4481 WARN_ON_ONCE_GFP(gfp_mask & __GFP_NOFAIL, gfp_mask)) { 4482 *did_some_progress = 1; 4483 4484 /* 4485 * Help non-failing allocations by giving them access to memory 4486 * reserves 4487 */ 4488 if (gfp_mask & __GFP_NOFAIL) 4489 page = __alloc_pages_cpuset_fallback(gfp_mask, order, 4490 ALLOC_NO_WATERMARKS, ac); 4491 } 4492 out: 4493 mutex_unlock(&oom_lock); 4494 return page; 4495 } 4496 4497 /* 4498 * Maximum number of compaction retries with a progress before OOM 4499 * killer is consider as the only way to move forward. 4500 */ 4501 #define MAX_COMPACT_RETRIES 16 4502 4503 #ifdef CONFIG_COMPACTION 4504 /* Try memory compaction for high-order allocations before reclaim */ 4505 static struct page * 4506 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4507 unsigned int alloc_flags, const struct alloc_context *ac, 4508 enum compact_priority prio, enum compact_result *compact_result) 4509 { 4510 struct page *page = NULL; 4511 unsigned long pflags; 4512 unsigned int noreclaim_flag; 4513 4514 if (!order) 4515 return NULL; 4516 4517 psi_memstall_enter(&pflags); 4518 delayacct_compact_start(); 4519 noreclaim_flag = memalloc_noreclaim_save(); 4520 4521 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 4522 prio, &page); 4523 4524 memalloc_noreclaim_restore(noreclaim_flag); 4525 psi_memstall_leave(&pflags); 4526 delayacct_compact_end(); 4527 4528 if (*compact_result == COMPACT_SKIPPED) 4529 return NULL; 4530 /* 4531 * At least in one zone compaction wasn't deferred or skipped, so let's 4532 * count a compaction stall 4533 */ 4534 count_vm_event(COMPACTSTALL); 4535 4536 /* Prep a captured page if available */ 4537 if (page) 4538 prep_new_page(page, order, gfp_mask, alloc_flags); 4539 4540 /* Try get a page from the freelist if available */ 4541 if (!page) 4542 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4543 4544 if (page) { 4545 struct zone *zone = page_zone(page); 4546 4547 zone->compact_blockskip_flush = false; 4548 compaction_defer_reset(zone, order, true); 4549 count_vm_event(COMPACTSUCCESS); 4550 return page; 4551 } 4552 4553 /* 4554 * It's bad if compaction run occurs and fails. The most likely reason 4555 * is that pages exist, but not enough to satisfy watermarks. 4556 */ 4557 count_vm_event(COMPACTFAIL); 4558 4559 cond_resched(); 4560 4561 return NULL; 4562 } 4563 4564 static inline bool 4565 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 4566 enum compact_result compact_result, 4567 enum compact_priority *compact_priority, 4568 int *compaction_retries) 4569 { 4570 int max_retries = MAX_COMPACT_RETRIES; 4571 int min_priority; 4572 bool ret = false; 4573 int retries = *compaction_retries; 4574 enum compact_priority priority = *compact_priority; 4575 4576 if (!order) 4577 return false; 4578 4579 if (fatal_signal_pending(current)) 4580 return false; 4581 4582 if (compaction_made_progress(compact_result)) 4583 (*compaction_retries)++; 4584 4585 /* 4586 * compaction considers all the zone as desperately out of memory 4587 * so it doesn't really make much sense to retry except when the 4588 * failure could be caused by insufficient priority 4589 */ 4590 if (compaction_failed(compact_result)) 4591 goto check_priority; 4592 4593 /* 4594 * compaction was skipped because there are not enough order-0 pages 4595 * to work with, so we retry only if it looks like reclaim can help. 4596 */ 4597 if (compaction_needs_reclaim(compact_result)) { 4598 ret = compaction_zonelist_suitable(ac, order, alloc_flags); 4599 goto out; 4600 } 4601 4602 /* 4603 * make sure the compaction wasn't deferred or didn't bail out early 4604 * due to locks contention before we declare that we should give up. 4605 * But the next retry should use a higher priority if allowed, so 4606 * we don't just keep bailing out endlessly. 4607 */ 4608 if (compaction_withdrawn(compact_result)) { 4609 goto check_priority; 4610 } 4611 4612 /* 4613 * !costly requests are much more important than __GFP_RETRY_MAYFAIL 4614 * costly ones because they are de facto nofail and invoke OOM 4615 * killer to move on while costly can fail and users are ready 4616 * to cope with that. 1/4 retries is rather arbitrary but we 4617 * would need much more detailed feedback from compaction to 4618 * make a better decision. 4619 */ 4620 if (order > PAGE_ALLOC_COSTLY_ORDER) 4621 max_retries /= 4; 4622 if (*compaction_retries <= max_retries) { 4623 ret = true; 4624 goto out; 4625 } 4626 4627 /* 4628 * Make sure there are attempts at the highest priority if we exhausted 4629 * all retries or failed at the lower priorities. 4630 */ 4631 check_priority: 4632 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 4633 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 4634 4635 if (*compact_priority > min_priority) { 4636 (*compact_priority)--; 4637 *compaction_retries = 0; 4638 ret = true; 4639 } 4640 out: 4641 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret); 4642 return ret; 4643 } 4644 #else 4645 static inline struct page * 4646 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4647 unsigned int alloc_flags, const struct alloc_context *ac, 4648 enum compact_priority prio, enum compact_result *compact_result) 4649 { 4650 *compact_result = COMPACT_SKIPPED; 4651 return NULL; 4652 } 4653 4654 static inline bool 4655 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 4656 enum compact_result compact_result, 4657 enum compact_priority *compact_priority, 4658 int *compaction_retries) 4659 { 4660 struct zone *zone; 4661 struct zoneref *z; 4662 4663 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 4664 return false; 4665 4666 /* 4667 * There are setups with compaction disabled which would prefer to loop 4668 * inside the allocator rather than hit the oom killer prematurely. 4669 * Let's give them a good hope and keep retrying while the order-0 4670 * watermarks are OK. 4671 */ 4672 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4673 ac->highest_zoneidx, ac->nodemask) { 4674 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 4675 ac->highest_zoneidx, alloc_flags)) 4676 return true; 4677 } 4678 return false; 4679 } 4680 #endif /* CONFIG_COMPACTION */ 4681 4682 #ifdef CONFIG_LOCKDEP 4683 static struct lockdep_map __fs_reclaim_map = 4684 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map); 4685 4686 static bool __need_reclaim(gfp_t gfp_mask) 4687 { 4688 /* no reclaim without waiting on it */ 4689 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) 4690 return false; 4691 4692 /* this guy won't enter reclaim */ 4693 if (current->flags & PF_MEMALLOC) 4694 return false; 4695 4696 if (gfp_mask & __GFP_NOLOCKDEP) 4697 return false; 4698 4699 return true; 4700 } 4701 4702 void __fs_reclaim_acquire(unsigned long ip) 4703 { 4704 lock_acquire_exclusive(&__fs_reclaim_map, 0, 0, NULL, ip); 4705 } 4706 4707 void __fs_reclaim_release(unsigned long ip) 4708 { 4709 lock_release(&__fs_reclaim_map, ip); 4710 } 4711 4712 void fs_reclaim_acquire(gfp_t gfp_mask) 4713 { 4714 gfp_mask = current_gfp_context(gfp_mask); 4715 4716 if (__need_reclaim(gfp_mask)) { 4717 if (gfp_mask & __GFP_FS) 4718 __fs_reclaim_acquire(_RET_IP_); 4719 4720 #ifdef CONFIG_MMU_NOTIFIER 4721 lock_map_acquire(&__mmu_notifier_invalidate_range_start_map); 4722 lock_map_release(&__mmu_notifier_invalidate_range_start_map); 4723 #endif 4724 4725 } 4726 } 4727 EXPORT_SYMBOL_GPL(fs_reclaim_acquire); 4728 4729 void fs_reclaim_release(gfp_t gfp_mask) 4730 { 4731 gfp_mask = current_gfp_context(gfp_mask); 4732 4733 if (__need_reclaim(gfp_mask)) { 4734 if (gfp_mask & __GFP_FS) 4735 __fs_reclaim_release(_RET_IP_); 4736 } 4737 } 4738 EXPORT_SYMBOL_GPL(fs_reclaim_release); 4739 #endif 4740 4741 /* 4742 * Zonelists may change due to hotplug during allocation. Detect when zonelists 4743 * have been rebuilt so allocation retries. Reader side does not lock and 4744 * retries the allocation if zonelist changes. Writer side is protected by the 4745 * embedded spin_lock. 4746 */ 4747 static DEFINE_SEQLOCK(zonelist_update_seq); 4748 4749 static unsigned int zonelist_iter_begin(void) 4750 { 4751 if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE)) 4752 return read_seqbegin(&zonelist_update_seq); 4753 4754 return 0; 4755 } 4756 4757 static unsigned int check_retry_zonelist(unsigned int seq) 4758 { 4759 if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE)) 4760 return read_seqretry(&zonelist_update_seq, seq); 4761 4762 return seq; 4763 } 4764 4765 /* Perform direct synchronous page reclaim */ 4766 static unsigned long 4767 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 4768 const struct alloc_context *ac) 4769 { 4770 unsigned int noreclaim_flag; 4771 unsigned long progress; 4772 4773 cond_resched(); 4774 4775 /* We now go into synchronous reclaim */ 4776 cpuset_memory_pressure_bump(); 4777 fs_reclaim_acquire(gfp_mask); 4778 noreclaim_flag = memalloc_noreclaim_save(); 4779 4780 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 4781 ac->nodemask); 4782 4783 memalloc_noreclaim_restore(noreclaim_flag); 4784 fs_reclaim_release(gfp_mask); 4785 4786 cond_resched(); 4787 4788 return progress; 4789 } 4790 4791 /* The really slow allocator path where we enter direct reclaim */ 4792 static inline struct page * 4793 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 4794 unsigned int alloc_flags, const struct alloc_context *ac, 4795 unsigned long *did_some_progress) 4796 { 4797 struct page *page = NULL; 4798 unsigned long pflags; 4799 bool drained = false; 4800 4801 psi_memstall_enter(&pflags); 4802 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 4803 if (unlikely(!(*did_some_progress))) 4804 goto out; 4805 4806 retry: 4807 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4808 4809 /* 4810 * If an allocation failed after direct reclaim, it could be because 4811 * pages are pinned on the per-cpu lists or in high alloc reserves. 4812 * Shrink them and try again 4813 */ 4814 if (!page && !drained) { 4815 unreserve_highatomic_pageblock(ac, false); 4816 drain_all_pages(NULL); 4817 drained = true; 4818 goto retry; 4819 } 4820 out: 4821 psi_memstall_leave(&pflags); 4822 4823 return page; 4824 } 4825 4826 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, 4827 const struct alloc_context *ac) 4828 { 4829 struct zoneref *z; 4830 struct zone *zone; 4831 pg_data_t *last_pgdat = NULL; 4832 enum zone_type highest_zoneidx = ac->highest_zoneidx; 4833 4834 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx, 4835 ac->nodemask) { 4836 if (!managed_zone(zone)) 4837 continue; 4838 if (last_pgdat != zone->zone_pgdat) { 4839 wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx); 4840 last_pgdat = zone->zone_pgdat; 4841 } 4842 } 4843 } 4844 4845 static inline unsigned int 4846 gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) 4847 { 4848 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 4849 4850 /* 4851 * __GFP_HIGH is assumed to be the same as ALLOC_MIN_RESERVE 4852 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 4853 * to save two branches. 4854 */ 4855 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_MIN_RESERVE); 4856 BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); 4857 4858 /* 4859 * The caller may dip into page reserves a bit more if the caller 4860 * cannot run direct reclaim, or if the caller has realtime scheduling 4861 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 4862 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_MIN_RESERVE(__GFP_HIGH). 4863 */ 4864 alloc_flags |= (__force int) 4865 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); 4866 4867 if (gfp_mask & __GFP_ATOMIC) { 4868 /* 4869 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 4870 * if it can't schedule. 4871 */ 4872 if (!(gfp_mask & __GFP_NOMEMALLOC)) { 4873 alloc_flags |= ALLOC_HARDER; 4874 4875 if (order > 0) 4876 alloc_flags |= ALLOC_HIGHATOMIC; 4877 } 4878 4879 /* 4880 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 4881 * comment for __cpuset_node_allowed(). 4882 */ 4883 alloc_flags &= ~ALLOC_CPUSET; 4884 } else if (unlikely(rt_task(current)) && in_task()) 4885 alloc_flags |= ALLOC_MIN_RESERVE; 4886 4887 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags); 4888 4889 return alloc_flags; 4890 } 4891 4892 static bool oom_reserves_allowed(struct task_struct *tsk) 4893 { 4894 if (!tsk_is_oom_victim(tsk)) 4895 return false; 4896 4897 /* 4898 * !MMU doesn't have oom reaper so give access to memory reserves 4899 * only to the thread with TIF_MEMDIE set 4900 */ 4901 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE)) 4902 return false; 4903 4904 return true; 4905 } 4906 4907 /* 4908 * Distinguish requests which really need access to full memory 4909 * reserves from oom victims which can live with a portion of it 4910 */ 4911 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) 4912 { 4913 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 4914 return 0; 4915 if (gfp_mask & __GFP_MEMALLOC) 4916 return ALLOC_NO_WATERMARKS; 4917 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 4918 return ALLOC_NO_WATERMARKS; 4919 if (!in_interrupt()) { 4920 if (current->flags & PF_MEMALLOC) 4921 return ALLOC_NO_WATERMARKS; 4922 else if (oom_reserves_allowed(current)) 4923 return ALLOC_OOM; 4924 } 4925 4926 return 0; 4927 } 4928 4929 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 4930 { 4931 return !!__gfp_pfmemalloc_flags(gfp_mask); 4932 } 4933 4934 /* 4935 * Checks whether it makes sense to retry the reclaim to make a forward progress 4936 * for the given allocation request. 4937 * 4938 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row 4939 * without success, or when we couldn't even meet the watermark if we 4940 * reclaimed all remaining pages on the LRU lists. 4941 * 4942 * Returns true if a retry is viable or false to enter the oom path. 4943 */ 4944 static inline bool 4945 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 4946 struct alloc_context *ac, int alloc_flags, 4947 bool did_some_progress, int *no_progress_loops) 4948 { 4949 struct zone *zone; 4950 struct zoneref *z; 4951 bool ret = false; 4952 4953 /* 4954 * Costly allocations might have made a progress but this doesn't mean 4955 * their order will become available due to high fragmentation so 4956 * always increment the no progress counter for them 4957 */ 4958 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 4959 *no_progress_loops = 0; 4960 else 4961 (*no_progress_loops)++; 4962 4963 /* 4964 * Make sure we converge to OOM if we cannot make any progress 4965 * several times in the row. 4966 */ 4967 if (*no_progress_loops > MAX_RECLAIM_RETRIES) { 4968 /* Before OOM, exhaust highatomic_reserve */ 4969 return unreserve_highatomic_pageblock(ac, true); 4970 } 4971 4972 /* 4973 * Keep reclaiming pages while there is a chance this will lead 4974 * somewhere. If none of the target zones can satisfy our allocation 4975 * request even if all reclaimable pages are considered then we are 4976 * screwed and have to go OOM. 4977 */ 4978 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4979 ac->highest_zoneidx, ac->nodemask) { 4980 unsigned long available; 4981 unsigned long reclaimable; 4982 unsigned long min_wmark = min_wmark_pages(zone); 4983 bool wmark; 4984 4985 available = reclaimable = zone_reclaimable_pages(zone); 4986 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 4987 4988 /* 4989 * Would the allocation succeed if we reclaimed all 4990 * reclaimable pages? 4991 */ 4992 wmark = __zone_watermark_ok(zone, order, min_wmark, 4993 ac->highest_zoneidx, alloc_flags, available); 4994 trace_reclaim_retry_zone(z, order, reclaimable, 4995 available, min_wmark, *no_progress_loops, wmark); 4996 if (wmark) { 4997 ret = true; 4998 break; 4999 } 5000 } 5001 5002 /* 5003 * Memory allocation/reclaim might be called from a WQ context and the 5004 * current implementation of the WQ concurrency control doesn't 5005 * recognize that a particular WQ is congested if the worker thread is 5006 * looping without ever sleeping. Therefore we have to do a short sleep 5007 * here rather than calling cond_resched(). 5008 */ 5009 if (current->flags & PF_WQ_WORKER) 5010 schedule_timeout_uninterruptible(1); 5011 else 5012 cond_resched(); 5013 return ret; 5014 } 5015 5016 static inline bool 5017 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) 5018 { 5019 /* 5020 * It's possible that cpuset's mems_allowed and the nodemask from 5021 * mempolicy don't intersect. This should be normally dealt with by 5022 * policy_nodemask(), but it's possible to race with cpuset update in 5023 * such a way the check therein was true, and then it became false 5024 * before we got our cpuset_mems_cookie here. 5025 * This assumes that for all allocations, ac->nodemask can come only 5026 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored 5027 * when it does not intersect with the cpuset restrictions) or the 5028 * caller can deal with a violated nodemask. 5029 */ 5030 if (cpusets_enabled() && ac->nodemask && 5031 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) { 5032 ac->nodemask = NULL; 5033 return true; 5034 } 5035 5036 /* 5037 * When updating a task's mems_allowed or mempolicy nodemask, it is 5038 * possible to race with parallel threads in such a way that our 5039 * allocation can fail while the mask is being updated. If we are about 5040 * to fail, check if the cpuset changed during allocation and if so, 5041 * retry. 5042 */ 5043 if (read_mems_allowed_retry(cpuset_mems_cookie)) 5044 return true; 5045 5046 return false; 5047 } 5048 5049 static inline struct page * 5050 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 5051 struct alloc_context *ac) 5052 { 5053 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 5054 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 5055 struct page *page = NULL; 5056 unsigned int alloc_flags; 5057 unsigned long did_some_progress; 5058 enum compact_priority compact_priority; 5059 enum compact_result compact_result; 5060 int compaction_retries; 5061 int no_progress_loops; 5062 unsigned int cpuset_mems_cookie; 5063 unsigned int zonelist_iter_cookie; 5064 int reserve_flags; 5065 5066 /* 5067 * We also sanity check to catch abuse of atomic reserves being used by 5068 * callers that are not in atomic context. 5069 */ 5070 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 5071 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 5072 gfp_mask &= ~__GFP_ATOMIC; 5073 5074 restart: 5075 compaction_retries = 0; 5076 no_progress_loops = 0; 5077 compact_priority = DEF_COMPACT_PRIORITY; 5078 cpuset_mems_cookie = read_mems_allowed_begin(); 5079 zonelist_iter_cookie = zonelist_iter_begin(); 5080 5081 /* 5082 * The fast path uses conservative alloc_flags to succeed only until 5083 * kswapd needs to be woken up, and to avoid the cost of setting up 5084 * alloc_flags precisely. So we do that now. 5085 */ 5086 alloc_flags = gfp_to_alloc_flags(gfp_mask, order); 5087 5088 /* 5089 * We need to recalculate the starting point for the zonelist iterator 5090 * because we might have used different nodemask in the fast path, or 5091 * there was a cpuset modification and we are retrying - otherwise we 5092 * could end up iterating over non-eligible zones endlessly. 5093 */ 5094 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 5095 ac->highest_zoneidx, ac->nodemask); 5096 if (!ac->preferred_zoneref->zone) 5097 goto nopage; 5098 5099 /* 5100 * Check for insane configurations where the cpuset doesn't contain 5101 * any suitable zone to satisfy the request - e.g. non-movable 5102 * GFP_HIGHUSER allocations from MOVABLE nodes only. 5103 */ 5104 if (cpusets_insane_config() && (gfp_mask & __GFP_HARDWALL)) { 5105 struct zoneref *z = first_zones_zonelist(ac->zonelist, 5106 ac->highest_zoneidx, 5107 &cpuset_current_mems_allowed); 5108 if (!z->zone) 5109 goto nopage; 5110 } 5111 5112 if (alloc_flags & ALLOC_KSWAPD) 5113 wake_all_kswapds(order, gfp_mask, ac); 5114 5115 /* 5116 * The adjusted alloc_flags might result in immediate success, so try 5117 * that first 5118 */ 5119 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 5120 if (page) 5121 goto got_pg; 5122 5123 /* 5124 * For costly allocations, try direct compaction first, as it's likely 5125 * that we have enough base pages and don't need to reclaim. For non- 5126 * movable high-order allocations, do that as well, as compaction will 5127 * try prevent permanent fragmentation by migrating from blocks of the 5128 * same migratetype. 5129 * Don't try this for allocations that are allowed to ignore 5130 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 5131 */ 5132 if (can_direct_reclaim && 5133 (costly_order || 5134 (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 5135 && !gfp_pfmemalloc_allowed(gfp_mask)) { 5136 page = __alloc_pages_direct_compact(gfp_mask, order, 5137 alloc_flags, ac, 5138 INIT_COMPACT_PRIORITY, 5139 &compact_result); 5140 if (page) 5141 goto got_pg; 5142 5143 /* 5144 * Checks for costly allocations with __GFP_NORETRY, which 5145 * includes some THP page fault allocations 5146 */ 5147 if (costly_order && (gfp_mask & __GFP_NORETRY)) { 5148 /* 5149 * If allocating entire pageblock(s) and compaction 5150 * failed because all zones are below low watermarks 5151 * or is prohibited because it recently failed at this 5152 * order, fail immediately unless the allocator has 5153 * requested compaction and reclaim retry. 5154 * 5155 * Reclaim is 5156 * - potentially very expensive because zones are far 5157 * below their low watermarks or this is part of very 5158 * bursty high order allocations, 5159 * - not guaranteed to help because isolate_freepages() 5160 * may not iterate over freed pages as part of its 5161 * linear scan, and 5162 * - unlikely to make entire pageblocks free on its 5163 * own. 5164 */ 5165 if (compact_result == COMPACT_SKIPPED || 5166 compact_result == COMPACT_DEFERRED) 5167 goto nopage; 5168 5169 /* 5170 * Looks like reclaim/compaction is worth trying, but 5171 * sync compaction could be very expensive, so keep 5172 * using async compaction. 5173 */ 5174 compact_priority = INIT_COMPACT_PRIORITY; 5175 } 5176 } 5177 5178 retry: 5179 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 5180 if (alloc_flags & ALLOC_KSWAPD) 5181 wake_all_kswapds(order, gfp_mask, ac); 5182 5183 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); 5184 if (reserve_flags) 5185 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags) | 5186 (alloc_flags & ALLOC_KSWAPD); 5187 5188 /* 5189 * Reset the nodemask and zonelist iterators if memory policies can be 5190 * ignored. These allocations are high priority and system rather than 5191 * user oriented. 5192 */ 5193 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { 5194 ac->nodemask = NULL; 5195 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 5196 ac->highest_zoneidx, ac->nodemask); 5197 } 5198 5199 /* Attempt with potentially adjusted zonelist and alloc_flags */ 5200 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 5201 if (page) 5202 goto got_pg; 5203 5204 /* Caller is not willing to reclaim, we can't balance anything */ 5205 if (!can_direct_reclaim) 5206 goto nopage; 5207 5208 /* Avoid recursion of direct reclaim */ 5209 if (current->flags & PF_MEMALLOC) 5210 goto nopage; 5211 5212 /* Try direct reclaim and then allocating */ 5213 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 5214 &did_some_progress); 5215 if (page) 5216 goto got_pg; 5217 5218 /* Try direct compaction and then allocating */ 5219 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 5220 compact_priority, &compact_result); 5221 if (page) 5222 goto got_pg; 5223 5224 /* Do not loop if specifically requested */ 5225 if (gfp_mask & __GFP_NORETRY) 5226 goto nopage; 5227 5228 /* 5229 * Do not retry costly high order allocations unless they are 5230 * __GFP_RETRY_MAYFAIL 5231 */ 5232 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) 5233 goto nopage; 5234 5235 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 5236 did_some_progress > 0, &no_progress_loops)) 5237 goto retry; 5238 5239 /* 5240 * It doesn't make any sense to retry for the compaction if the order-0 5241 * reclaim is not able to make any progress because the current 5242 * implementation of the compaction depends on the sufficient amount 5243 * of free memory (see __compaction_suitable) 5244 */ 5245 if (did_some_progress > 0 && 5246 should_compact_retry(ac, order, alloc_flags, 5247 compact_result, &compact_priority, 5248 &compaction_retries)) 5249 goto retry; 5250 5251 5252 /* 5253 * Deal with possible cpuset update races or zonelist updates to avoid 5254 * a unnecessary OOM kill. 5255 */ 5256 if (check_retry_cpuset(cpuset_mems_cookie, ac) || 5257 check_retry_zonelist(zonelist_iter_cookie)) 5258 goto restart; 5259 5260 /* Reclaim has failed us, start killing things */ 5261 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 5262 if (page) 5263 goto got_pg; 5264 5265 /* Avoid allocations with no watermarks from looping endlessly */ 5266 if (tsk_is_oom_victim(current) && 5267 (alloc_flags & ALLOC_OOM || 5268 (gfp_mask & __GFP_NOMEMALLOC))) 5269 goto nopage; 5270 5271 /* Retry as long as the OOM killer is making progress */ 5272 if (did_some_progress) { 5273 no_progress_loops = 0; 5274 goto retry; 5275 } 5276 5277 nopage: 5278 /* 5279 * Deal with possible cpuset update races or zonelist updates to avoid 5280 * a unnecessary OOM kill. 5281 */ 5282 if (check_retry_cpuset(cpuset_mems_cookie, ac) || 5283 check_retry_zonelist(zonelist_iter_cookie)) 5284 goto restart; 5285 5286 /* 5287 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure 5288 * we always retry 5289 */ 5290 if (gfp_mask & __GFP_NOFAIL) { 5291 /* 5292 * All existing users of the __GFP_NOFAIL are blockable, so warn 5293 * of any new users that actually require GFP_NOWAIT 5294 */ 5295 if (WARN_ON_ONCE_GFP(!can_direct_reclaim, gfp_mask)) 5296 goto fail; 5297 5298 /* 5299 * PF_MEMALLOC request from this context is rather bizarre 5300 * because we cannot reclaim anything and only can loop waiting 5301 * for somebody to do a work for us 5302 */ 5303 WARN_ON_ONCE_GFP(current->flags & PF_MEMALLOC, gfp_mask); 5304 5305 /* 5306 * non failing costly orders are a hard requirement which we 5307 * are not prepared for much so let's warn about these users 5308 * so that we can identify them and convert them to something 5309 * else. 5310 */ 5311 WARN_ON_ONCE_GFP(costly_order, gfp_mask); 5312 5313 /* 5314 * Help non-failing allocations by giving them access to memory 5315 * reserves but do not use ALLOC_NO_WATERMARKS because this 5316 * could deplete whole memory reserves which would just make 5317 * the situation worse 5318 */ 5319 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); 5320 if (page) 5321 goto got_pg; 5322 5323 cond_resched(); 5324 goto retry; 5325 } 5326 fail: 5327 warn_alloc(gfp_mask, ac->nodemask, 5328 "page allocation failure: order:%u", order); 5329 got_pg: 5330 return page; 5331 } 5332 5333 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, 5334 int preferred_nid, nodemask_t *nodemask, 5335 struct alloc_context *ac, gfp_t *alloc_gfp, 5336 unsigned int *alloc_flags) 5337 { 5338 ac->highest_zoneidx = gfp_zone(gfp_mask); 5339 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); 5340 ac->nodemask = nodemask; 5341 ac->migratetype = gfp_migratetype(gfp_mask); 5342 5343 if (cpusets_enabled()) { 5344 *alloc_gfp |= __GFP_HARDWALL; 5345 /* 5346 * When we are in the interrupt context, it is irrelevant 5347 * to the current task context. It means that any node ok. 5348 */ 5349 if (in_task() && !ac->nodemask) 5350 ac->nodemask = &cpuset_current_mems_allowed; 5351 else 5352 *alloc_flags |= ALLOC_CPUSET; 5353 } 5354 5355 might_alloc(gfp_mask); 5356 5357 if (should_fail_alloc_page(gfp_mask, order)) 5358 return false; 5359 5360 *alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, *alloc_flags); 5361 5362 /* Dirty zone balancing only done in the fast path */ 5363 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); 5364 5365 /* 5366 * The preferred zone is used for statistics but crucially it is 5367 * also used as the starting point for the zonelist iterator. It 5368 * may get reset for allocations that ignore memory policies. 5369 */ 5370 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 5371 ac->highest_zoneidx, ac->nodemask); 5372 5373 return true; 5374 } 5375 5376 /* 5377 * __alloc_pages_bulk - Allocate a number of order-0 pages to a list or array 5378 * @gfp: GFP flags for the allocation 5379 * @preferred_nid: The preferred NUMA node ID to allocate from 5380 * @nodemask: Set of nodes to allocate from, may be NULL 5381 * @nr_pages: The number of pages desired on the list or array 5382 * @page_list: Optional list to store the allocated pages 5383 * @page_array: Optional array to store the pages 5384 * 5385 * This is a batched version of the page allocator that attempts to 5386 * allocate nr_pages quickly. Pages are added to page_list if page_list 5387 * is not NULL, otherwise it is assumed that the page_array is valid. 5388 * 5389 * For lists, nr_pages is the number of pages that should be allocated. 5390 * 5391 * For arrays, only NULL elements are populated with pages and nr_pages 5392 * is the maximum number of pages that will be stored in the array. 5393 * 5394 * Returns the number of pages on the list or array. 5395 */ 5396 unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, 5397 nodemask_t *nodemask, int nr_pages, 5398 struct list_head *page_list, 5399 struct page **page_array) 5400 { 5401 struct page *page; 5402 unsigned long __maybe_unused UP_flags; 5403 struct zone *zone; 5404 struct zoneref *z; 5405 struct per_cpu_pages *pcp; 5406 struct list_head *pcp_list; 5407 struct alloc_context ac; 5408 gfp_t alloc_gfp; 5409 unsigned int alloc_flags = ALLOC_WMARK_LOW; 5410 int nr_populated = 0, nr_account = 0; 5411 5412 /* 5413 * Skip populated array elements to determine if any pages need 5414 * to be allocated before disabling IRQs. 5415 */ 5416 while (page_array && nr_populated < nr_pages && page_array[nr_populated]) 5417 nr_populated++; 5418 5419 /* No pages requested? */ 5420 if (unlikely(nr_pages <= 0)) 5421 goto out; 5422 5423 /* Already populated array? */ 5424 if (unlikely(page_array && nr_pages - nr_populated == 0)) 5425 goto out; 5426 5427 /* Bulk allocator does not support memcg accounting. */ 5428 if (memcg_kmem_enabled() && (gfp & __GFP_ACCOUNT)) 5429 goto failed; 5430 5431 /* Use the single page allocator for one page. */ 5432 if (nr_pages - nr_populated == 1) 5433 goto failed; 5434 5435 #ifdef CONFIG_PAGE_OWNER 5436 /* 5437 * PAGE_OWNER may recurse into the allocator to allocate space to 5438 * save the stack with pagesets.lock held. Releasing/reacquiring 5439 * removes much of the performance benefit of bulk allocation so 5440 * force the caller to allocate one page at a time as it'll have 5441 * similar performance to added complexity to the bulk allocator. 5442 */ 5443 if (static_branch_unlikely(&page_owner_inited)) 5444 goto failed; 5445 #endif 5446 5447 /* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */ 5448 gfp &= gfp_allowed_mask; 5449 alloc_gfp = gfp; 5450 if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &alloc_gfp, &alloc_flags)) 5451 goto out; 5452 gfp = alloc_gfp; 5453 5454 /* Find an allowed local zone that meets the low watermark. */ 5455 for_each_zone_zonelist_nodemask(zone, z, ac.zonelist, ac.highest_zoneidx, ac.nodemask) { 5456 unsigned long mark; 5457 5458 if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) && 5459 !__cpuset_zone_allowed(zone, gfp)) { 5460 continue; 5461 } 5462 5463 if (nr_online_nodes > 1 && zone != ac.preferred_zoneref->zone && 5464 zone_to_nid(zone) != zone_to_nid(ac.preferred_zoneref->zone)) { 5465 goto failed; 5466 } 5467 5468 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK) + nr_pages; 5469 if (zone_watermark_fast(zone, 0, mark, 5470 zonelist_zone_idx(ac.preferred_zoneref), 5471 alloc_flags, gfp)) { 5472 break; 5473 } 5474 } 5475 5476 /* 5477 * If there are no allowed local zones that meets the watermarks then 5478 * try to allocate a single page and reclaim if necessary. 5479 */ 5480 if (unlikely(!zone)) 5481 goto failed; 5482 5483 /* spin_trylock may fail due to a parallel drain or IRQ reentrancy. */ 5484 pcp_trylock_prepare(UP_flags); 5485 pcp = pcp_spin_trylock(zone->per_cpu_pageset); 5486 if (!pcp) 5487 goto failed_irq; 5488 5489 /* Attempt the batch allocation */ 5490 pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)]; 5491 while (nr_populated < nr_pages) { 5492 5493 /* Skip existing pages */ 5494 if (page_array && page_array[nr_populated]) { 5495 nr_populated++; 5496 continue; 5497 } 5498 5499 page = __rmqueue_pcplist(zone, 0, ac.migratetype, alloc_flags, 5500 pcp, pcp_list); 5501 if (unlikely(!page)) { 5502 /* Try and allocate at least one page */ 5503 if (!nr_account) { 5504 pcp_spin_unlock(pcp); 5505 goto failed_irq; 5506 } 5507 break; 5508 } 5509 nr_account++; 5510 5511 prep_new_page(page, 0, gfp, 0); 5512 if (page_list) 5513 list_add(&page->lru, page_list); 5514 else 5515 page_array[nr_populated] = page; 5516 nr_populated++; 5517 } 5518 5519 pcp_spin_unlock(pcp); 5520 pcp_trylock_finish(UP_flags); 5521 5522 __count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account); 5523 zone_statistics(ac.preferred_zoneref->zone, zone, nr_account); 5524 5525 out: 5526 return nr_populated; 5527 5528 failed_irq: 5529 pcp_trylock_finish(UP_flags); 5530 5531 failed: 5532 page = __alloc_pages(gfp, 0, preferred_nid, nodemask); 5533 if (page) { 5534 if (page_list) 5535 list_add(&page->lru, page_list); 5536 else 5537 page_array[nr_populated] = page; 5538 nr_populated++; 5539 } 5540 5541 goto out; 5542 } 5543 EXPORT_SYMBOL_GPL(__alloc_pages_bulk); 5544 5545 /* 5546 * This is the 'heart' of the zoned buddy allocator. 5547 */ 5548 struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid, 5549 nodemask_t *nodemask) 5550 { 5551 struct page *page; 5552 unsigned int alloc_flags = ALLOC_WMARK_LOW; 5553 gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */ 5554 struct alloc_context ac = { }; 5555 5556 /* 5557 * There are several places where we assume that the order value is sane 5558 * so bail out early if the request is out of bound. 5559 */ 5560 if (WARN_ON_ONCE_GFP(order >= MAX_ORDER, gfp)) 5561 return NULL; 5562 5563 gfp &= gfp_allowed_mask; 5564 /* 5565 * Apply scoped allocation constraints. This is mainly about GFP_NOFS 5566 * resp. GFP_NOIO which has to be inherited for all allocation requests 5567 * from a particular context which has been marked by 5568 * memalloc_no{fs,io}_{save,restore}. And PF_MEMALLOC_PIN which ensures 5569 * movable zones are not used during allocation. 5570 */ 5571 gfp = current_gfp_context(gfp); 5572 alloc_gfp = gfp; 5573 if (!prepare_alloc_pages(gfp, order, preferred_nid, nodemask, &ac, 5574 &alloc_gfp, &alloc_flags)) 5575 return NULL; 5576 5577 /* 5578 * Forbid the first pass from falling back to types that fragment 5579 * memory until all local zones are considered. 5580 */ 5581 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp); 5582 5583 /* First allocation attempt */ 5584 page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); 5585 if (likely(page)) 5586 goto out; 5587 5588 alloc_gfp = gfp; 5589 ac.spread_dirty_pages = false; 5590 5591 /* 5592 * Restore the original nodemask if it was potentially replaced with 5593 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 5594 */ 5595 ac.nodemask = nodemask; 5596 5597 page = __alloc_pages_slowpath(alloc_gfp, order, &ac); 5598 5599 out: 5600 if (memcg_kmem_enabled() && (gfp & __GFP_ACCOUNT) && page && 5601 unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) { 5602 __free_pages(page, order); 5603 page = NULL; 5604 } 5605 5606 trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype); 5607 kmsan_alloc_page(page, order, alloc_gfp); 5608 5609 return page; 5610 } 5611 EXPORT_SYMBOL(__alloc_pages); 5612 5613 struct folio *__folio_alloc(gfp_t gfp, unsigned int order, int preferred_nid, 5614 nodemask_t *nodemask) 5615 { 5616 struct page *page = __alloc_pages(gfp | __GFP_COMP, order, 5617 preferred_nid, nodemask); 5618 5619 if (page && order > 1) 5620 prep_transhuge_page(page); 5621 return (struct folio *)page; 5622 } 5623 EXPORT_SYMBOL(__folio_alloc); 5624 5625 /* 5626 * Common helper functions. Never use with __GFP_HIGHMEM because the returned 5627 * address cannot represent highmem pages. Use alloc_pages and then kmap if 5628 * you need to access high mem. 5629 */ 5630 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 5631 { 5632 struct page *page; 5633 5634 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); 5635 if (!page) 5636 return 0; 5637 return (unsigned long) page_address(page); 5638 } 5639 EXPORT_SYMBOL(__get_free_pages); 5640 5641 unsigned long get_zeroed_page(gfp_t gfp_mask) 5642 { 5643 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 5644 } 5645 EXPORT_SYMBOL(get_zeroed_page); 5646 5647 /** 5648 * __free_pages - Free pages allocated with alloc_pages(). 5649 * @page: The page pointer returned from alloc_pages(). 5650 * @order: The order of the allocation. 5651 * 5652 * This function can free multi-page allocations that are not compound 5653 * pages. It does not check that the @order passed in matches that of 5654 * the allocation, so it is easy to leak memory. Freeing more memory 5655 * than was allocated will probably emit a warning. 5656 * 5657 * If the last reference to this page is speculative, it will be released 5658 * by put_page() which only frees the first page of a non-compound 5659 * allocation. To prevent the remaining pages from being leaked, we free 5660 * the subsequent pages here. If you want to use the page's reference 5661 * count to decide when to free the allocation, you should allocate a 5662 * compound page, and use put_page() instead of __free_pages(). 5663 * 5664 * Context: May be called in interrupt context or while holding a normal 5665 * spinlock, but not in NMI context or while holding a raw spinlock. 5666 */ 5667 void __free_pages(struct page *page, unsigned int order) 5668 { 5669 if (put_page_testzero(page)) 5670 free_the_page(page, order); 5671 else if (!PageHead(page)) 5672 while (order-- > 0) 5673 free_the_page(page + (1 << order), order); 5674 } 5675 EXPORT_SYMBOL(__free_pages); 5676 5677 void free_pages(unsigned long addr, unsigned int order) 5678 { 5679 if (addr != 0) { 5680 VM_BUG_ON(!virt_addr_valid((void *)addr)); 5681 __free_pages(virt_to_page((void *)addr), order); 5682 } 5683 } 5684 5685 EXPORT_SYMBOL(free_pages); 5686 5687 /* 5688 * Page Fragment: 5689 * An arbitrary-length arbitrary-offset area of memory which resides 5690 * within a 0 or higher order page. Multiple fragments within that page 5691 * are individually refcounted, in the page's reference counter. 5692 * 5693 * The page_frag functions below provide a simple allocation framework for 5694 * page fragments. This is used by the network stack and network device 5695 * drivers to provide a backing region of memory for use as either an 5696 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 5697 */ 5698 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 5699 gfp_t gfp_mask) 5700 { 5701 struct page *page = NULL; 5702 gfp_t gfp = gfp_mask; 5703 5704 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5705 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 5706 __GFP_NOMEMALLOC; 5707 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 5708 PAGE_FRAG_CACHE_MAX_ORDER); 5709 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 5710 #endif 5711 if (unlikely(!page)) 5712 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 5713 5714 nc->va = page ? page_address(page) : NULL; 5715 5716 return page; 5717 } 5718 5719 void __page_frag_cache_drain(struct page *page, unsigned int count) 5720 { 5721 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 5722 5723 if (page_ref_sub_and_test(page, count)) 5724 free_the_page(page, compound_order(page)); 5725 } 5726 EXPORT_SYMBOL(__page_frag_cache_drain); 5727 5728 void *page_frag_alloc_align(struct page_frag_cache *nc, 5729 unsigned int fragsz, gfp_t gfp_mask, 5730 unsigned int align_mask) 5731 { 5732 unsigned int size = PAGE_SIZE; 5733 struct page *page; 5734 int offset; 5735 5736 if (unlikely(!nc->va)) { 5737 refill: 5738 page = __page_frag_cache_refill(nc, gfp_mask); 5739 if (!page) 5740 return NULL; 5741 5742 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5743 /* if size can vary use size else just use PAGE_SIZE */ 5744 size = nc->size; 5745 #endif 5746 /* Even if we own the page, we do not use atomic_set(). 5747 * This would break get_page_unless_zero() users. 5748 */ 5749 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 5750 5751 /* reset page count bias and offset to start of new frag */ 5752 nc->pfmemalloc = page_is_pfmemalloc(page); 5753 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5754 nc->offset = size; 5755 } 5756 5757 offset = nc->offset - fragsz; 5758 if (unlikely(offset < 0)) { 5759 page = virt_to_page(nc->va); 5760 5761 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 5762 goto refill; 5763 5764 if (unlikely(nc->pfmemalloc)) { 5765 free_the_page(page, compound_order(page)); 5766 goto refill; 5767 } 5768 5769 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5770 /* if size can vary use size else just use PAGE_SIZE */ 5771 size = nc->size; 5772 #endif 5773 /* OK, page count is 0, we can safely set it */ 5774 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 5775 5776 /* reset page count bias and offset to start of new frag */ 5777 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5778 offset = size - fragsz; 5779 if (unlikely(offset < 0)) { 5780 /* 5781 * The caller is trying to allocate a fragment 5782 * with fragsz > PAGE_SIZE but the cache isn't big 5783 * enough to satisfy the request, this may 5784 * happen in low memory conditions. 5785 * We don't release the cache page because 5786 * it could make memory pressure worse 5787 * so we simply return NULL here. 5788 */ 5789 return NULL; 5790 } 5791 } 5792 5793 nc->pagecnt_bias--; 5794 offset &= align_mask; 5795 nc->offset = offset; 5796 5797 return nc->va + offset; 5798 } 5799 EXPORT_SYMBOL(page_frag_alloc_align); 5800 5801 /* 5802 * Frees a page fragment allocated out of either a compound or order 0 page. 5803 */ 5804 void page_frag_free(void *addr) 5805 { 5806 struct page *page = virt_to_head_page(addr); 5807 5808 if (unlikely(put_page_testzero(page))) 5809 free_the_page(page, compound_order(page)); 5810 } 5811 EXPORT_SYMBOL(page_frag_free); 5812 5813 static void *make_alloc_exact(unsigned long addr, unsigned int order, 5814 size_t size) 5815 { 5816 if (addr) { 5817 unsigned long nr = DIV_ROUND_UP(size, PAGE_SIZE); 5818 struct page *page = virt_to_page((void *)addr); 5819 struct page *last = page + nr; 5820 5821 split_page_owner(page, 1 << order); 5822 split_page_memcg(page, 1 << order); 5823 while (page < --last) 5824 set_page_refcounted(last); 5825 5826 last = page + (1UL << order); 5827 for (page += nr; page < last; page++) 5828 __free_pages_ok(page, 0, FPI_TO_TAIL); 5829 } 5830 return (void *)addr; 5831 } 5832 5833 /** 5834 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 5835 * @size: the number of bytes to allocate 5836 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5837 * 5838 * This function is similar to alloc_pages(), except that it allocates the 5839 * minimum number of pages to satisfy the request. alloc_pages() can only 5840 * allocate memory in power-of-two pages. 5841 * 5842 * This function is also limited by MAX_ORDER. 5843 * 5844 * Memory allocated by this function must be released by free_pages_exact(). 5845 * 5846 * Return: pointer to the allocated area or %NULL in case of error. 5847 */ 5848 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 5849 { 5850 unsigned int order = get_order(size); 5851 unsigned long addr; 5852 5853 if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM))) 5854 gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM); 5855 5856 addr = __get_free_pages(gfp_mask, order); 5857 return make_alloc_exact(addr, order, size); 5858 } 5859 EXPORT_SYMBOL(alloc_pages_exact); 5860 5861 /** 5862 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 5863 * pages on a node. 5864 * @nid: the preferred node ID where memory should be allocated 5865 * @size: the number of bytes to allocate 5866 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5867 * 5868 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 5869 * back. 5870 * 5871 * Return: pointer to the allocated area or %NULL in case of error. 5872 */ 5873 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 5874 { 5875 unsigned int order = get_order(size); 5876 struct page *p; 5877 5878 if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM))) 5879 gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM); 5880 5881 p = alloc_pages_node(nid, gfp_mask, order); 5882 if (!p) 5883 return NULL; 5884 return make_alloc_exact((unsigned long)page_address(p), order, size); 5885 } 5886 5887 /** 5888 * free_pages_exact - release memory allocated via alloc_pages_exact() 5889 * @virt: the value returned by alloc_pages_exact. 5890 * @size: size of allocation, same value as passed to alloc_pages_exact(). 5891 * 5892 * Release the memory allocated by a previous call to alloc_pages_exact. 5893 */ 5894 void free_pages_exact(void *virt, size_t size) 5895 { 5896 unsigned long addr = (unsigned long)virt; 5897 unsigned long end = addr + PAGE_ALIGN(size); 5898 5899 while (addr < end) { 5900 free_page(addr); 5901 addr += PAGE_SIZE; 5902 } 5903 } 5904 EXPORT_SYMBOL(free_pages_exact); 5905 5906 /** 5907 * nr_free_zone_pages - count number of pages beyond high watermark 5908 * @offset: The zone index of the highest zone 5909 * 5910 * nr_free_zone_pages() counts the number of pages which are beyond the 5911 * high watermark within all zones at or below a given zone index. For each 5912 * zone, the number of pages is calculated as: 5913 * 5914 * nr_free_zone_pages = managed_pages - high_pages 5915 * 5916 * Return: number of pages beyond high watermark. 5917 */ 5918 static unsigned long nr_free_zone_pages(int offset) 5919 { 5920 struct zoneref *z; 5921 struct zone *zone; 5922 5923 /* Just pick one node, since fallback list is circular */ 5924 unsigned long sum = 0; 5925 5926 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 5927 5928 for_each_zone_zonelist(zone, z, zonelist, offset) { 5929 unsigned long size = zone_managed_pages(zone); 5930 unsigned long high = high_wmark_pages(zone); 5931 if (size > high) 5932 sum += size - high; 5933 } 5934 5935 return sum; 5936 } 5937 5938 /** 5939 * nr_free_buffer_pages - count number of pages beyond high watermark 5940 * 5941 * nr_free_buffer_pages() counts the number of pages which are beyond the high 5942 * watermark within ZONE_DMA and ZONE_NORMAL. 5943 * 5944 * Return: number of pages beyond high watermark within ZONE_DMA and 5945 * ZONE_NORMAL. 5946 */ 5947 unsigned long nr_free_buffer_pages(void) 5948 { 5949 return nr_free_zone_pages(gfp_zone(GFP_USER)); 5950 } 5951 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 5952 5953 static inline void show_node(struct zone *zone) 5954 { 5955 if (IS_ENABLED(CONFIG_NUMA)) 5956 printk("Node %d ", zone_to_nid(zone)); 5957 } 5958 5959 long si_mem_available(void) 5960 { 5961 long available; 5962 unsigned long pagecache; 5963 unsigned long wmark_low = 0; 5964 unsigned long pages[NR_LRU_LISTS]; 5965 unsigned long reclaimable; 5966 struct zone *zone; 5967 int lru; 5968 5969 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 5970 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 5971 5972 for_each_zone(zone) 5973 wmark_low += low_wmark_pages(zone); 5974 5975 /* 5976 * Estimate the amount of memory available for userspace allocations, 5977 * without causing swapping or OOM. 5978 */ 5979 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages; 5980 5981 /* 5982 * Not all the page cache can be freed, otherwise the system will 5983 * start swapping or thrashing. Assume at least half of the page 5984 * cache, or the low watermark worth of cache, needs to stay. 5985 */ 5986 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 5987 pagecache -= min(pagecache / 2, wmark_low); 5988 available += pagecache; 5989 5990 /* 5991 * Part of the reclaimable slab and other kernel memory consists of 5992 * items that are in use, and cannot be freed. Cap this estimate at the 5993 * low watermark. 5994 */ 5995 reclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B) + 5996 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE); 5997 available += reclaimable - min(reclaimable / 2, wmark_low); 5998 5999 if (available < 0) 6000 available = 0; 6001 return available; 6002 } 6003 EXPORT_SYMBOL_GPL(si_mem_available); 6004 6005 void si_meminfo(struct sysinfo *val) 6006 { 6007 val->totalram = totalram_pages(); 6008 val->sharedram = global_node_page_state(NR_SHMEM); 6009 val->freeram = global_zone_page_state(NR_FREE_PAGES); 6010 val->bufferram = nr_blockdev_pages(); 6011 val->totalhigh = totalhigh_pages(); 6012 val->freehigh = nr_free_highpages(); 6013 val->mem_unit = PAGE_SIZE; 6014 } 6015 6016 EXPORT_SYMBOL(si_meminfo); 6017 6018 #ifdef CONFIG_NUMA 6019 void si_meminfo_node(struct sysinfo *val, int nid) 6020 { 6021 int zone_type; /* needs to be signed */ 6022 unsigned long managed_pages = 0; 6023 unsigned long managed_highpages = 0; 6024 unsigned long free_highpages = 0; 6025 pg_data_t *pgdat = NODE_DATA(nid); 6026 6027 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 6028 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]); 6029 val->totalram = managed_pages; 6030 val->sharedram = node_page_state(pgdat, NR_SHMEM); 6031 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 6032 #ifdef CONFIG_HIGHMEM 6033 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 6034 struct zone *zone = &pgdat->node_zones[zone_type]; 6035 6036 if (is_highmem(zone)) { 6037 managed_highpages += zone_managed_pages(zone); 6038 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 6039 } 6040 } 6041 val->totalhigh = managed_highpages; 6042 val->freehigh = free_highpages; 6043 #else 6044 val->totalhigh = managed_highpages; 6045 val->freehigh = free_highpages; 6046 #endif 6047 val->mem_unit = PAGE_SIZE; 6048 } 6049 #endif 6050 6051 /* 6052 * Determine whether the node should be displayed or not, depending on whether 6053 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 6054 */ 6055 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) 6056 { 6057 if (!(flags & SHOW_MEM_FILTER_NODES)) 6058 return false; 6059 6060 /* 6061 * no node mask - aka implicit memory numa policy. Do not bother with 6062 * the synchronization - read_mems_allowed_begin - because we do not 6063 * have to be precise here. 6064 */ 6065 if (!nodemask) 6066 nodemask = &cpuset_current_mems_allowed; 6067 6068 return !node_isset(nid, *nodemask); 6069 } 6070 6071 #define K(x) ((x) << (PAGE_SHIFT-10)) 6072 6073 static void show_migration_types(unsigned char type) 6074 { 6075 static const char types[MIGRATE_TYPES] = { 6076 [MIGRATE_UNMOVABLE] = 'U', 6077 [MIGRATE_MOVABLE] = 'M', 6078 [MIGRATE_RECLAIMABLE] = 'E', 6079 [MIGRATE_HIGHATOMIC] = 'H', 6080 #ifdef CONFIG_CMA 6081 [MIGRATE_CMA] = 'C', 6082 #endif 6083 #ifdef CONFIG_MEMORY_ISOLATION 6084 [MIGRATE_ISOLATE] = 'I', 6085 #endif 6086 }; 6087 char tmp[MIGRATE_TYPES + 1]; 6088 char *p = tmp; 6089 int i; 6090 6091 for (i = 0; i < MIGRATE_TYPES; i++) { 6092 if (type & (1 << i)) 6093 *p++ = types[i]; 6094 } 6095 6096 *p = '\0'; 6097 printk(KERN_CONT "(%s) ", tmp); 6098 } 6099 6100 static bool node_has_managed_zones(pg_data_t *pgdat, int max_zone_idx) 6101 { 6102 int zone_idx; 6103 for (zone_idx = 0; zone_idx <= max_zone_idx; zone_idx++) 6104 if (zone_managed_pages(pgdat->node_zones + zone_idx)) 6105 return true; 6106 return false; 6107 } 6108 6109 /* 6110 * Show free area list (used inside shift_scroll-lock stuff) 6111 * We also calculate the percentage fragmentation. We do this by counting the 6112 * memory on each free list with the exception of the first item on the list. 6113 * 6114 * Bits in @filter: 6115 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 6116 * cpuset. 6117 */ 6118 void __show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_zone_idx) 6119 { 6120 unsigned long free_pcp = 0; 6121 int cpu, nid; 6122 struct zone *zone; 6123 pg_data_t *pgdat; 6124 6125 for_each_populated_zone(zone) { 6126 if (zone_idx(zone) > max_zone_idx) 6127 continue; 6128 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 6129 continue; 6130 6131 for_each_online_cpu(cpu) 6132 free_pcp += per_cpu_ptr(zone->per_cpu_pageset, cpu)->count; 6133 } 6134 6135 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 6136 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 6137 " unevictable:%lu dirty:%lu writeback:%lu\n" 6138 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 6139 " mapped:%lu shmem:%lu pagetables:%lu\n" 6140 " sec_pagetables:%lu bounce:%lu\n" 6141 " kernel_misc_reclaimable:%lu\n" 6142 " free:%lu free_pcp:%lu free_cma:%lu\n", 6143 global_node_page_state(NR_ACTIVE_ANON), 6144 global_node_page_state(NR_INACTIVE_ANON), 6145 global_node_page_state(NR_ISOLATED_ANON), 6146 global_node_page_state(NR_ACTIVE_FILE), 6147 global_node_page_state(NR_INACTIVE_FILE), 6148 global_node_page_state(NR_ISOLATED_FILE), 6149 global_node_page_state(NR_UNEVICTABLE), 6150 global_node_page_state(NR_FILE_DIRTY), 6151 global_node_page_state(NR_WRITEBACK), 6152 global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B), 6153 global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B), 6154 global_node_page_state(NR_FILE_MAPPED), 6155 global_node_page_state(NR_SHMEM), 6156 global_node_page_state(NR_PAGETABLE), 6157 global_node_page_state(NR_SECONDARY_PAGETABLE), 6158 global_zone_page_state(NR_BOUNCE), 6159 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE), 6160 global_zone_page_state(NR_FREE_PAGES), 6161 free_pcp, 6162 global_zone_page_state(NR_FREE_CMA_PAGES)); 6163 6164 for_each_online_pgdat(pgdat) { 6165 if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) 6166 continue; 6167 if (!node_has_managed_zones(pgdat, max_zone_idx)) 6168 continue; 6169 6170 printk("Node %d" 6171 " active_anon:%lukB" 6172 " inactive_anon:%lukB" 6173 " active_file:%lukB" 6174 " inactive_file:%lukB" 6175 " unevictable:%lukB" 6176 " isolated(anon):%lukB" 6177 " isolated(file):%lukB" 6178 " mapped:%lukB" 6179 " dirty:%lukB" 6180 " writeback:%lukB" 6181 " shmem:%lukB" 6182 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 6183 " shmem_thp: %lukB" 6184 " shmem_pmdmapped: %lukB" 6185 " anon_thp: %lukB" 6186 #endif 6187 " writeback_tmp:%lukB" 6188 " kernel_stack:%lukB" 6189 #ifdef CONFIG_SHADOW_CALL_STACK 6190 " shadow_call_stack:%lukB" 6191 #endif 6192 " pagetables:%lukB" 6193 " sec_pagetables:%lukB" 6194 " all_unreclaimable? %s" 6195 "\n", 6196 pgdat->node_id, 6197 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 6198 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 6199 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 6200 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 6201 K(node_page_state(pgdat, NR_UNEVICTABLE)), 6202 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 6203 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 6204 K(node_page_state(pgdat, NR_FILE_MAPPED)), 6205 K(node_page_state(pgdat, NR_FILE_DIRTY)), 6206 K(node_page_state(pgdat, NR_WRITEBACK)), 6207 K(node_page_state(pgdat, NR_SHMEM)), 6208 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 6209 K(node_page_state(pgdat, NR_SHMEM_THPS)), 6210 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), 6211 K(node_page_state(pgdat, NR_ANON_THPS)), 6212 #endif 6213 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 6214 node_page_state(pgdat, NR_KERNEL_STACK_KB), 6215 #ifdef CONFIG_SHADOW_CALL_STACK 6216 node_page_state(pgdat, NR_KERNEL_SCS_KB), 6217 #endif 6218 K(node_page_state(pgdat, NR_PAGETABLE)), 6219 K(node_page_state(pgdat, NR_SECONDARY_PAGETABLE)), 6220 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ? 6221 "yes" : "no"); 6222 } 6223 6224 for_each_populated_zone(zone) { 6225 int i; 6226 6227 if (zone_idx(zone) > max_zone_idx) 6228 continue; 6229 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 6230 continue; 6231 6232 free_pcp = 0; 6233 for_each_online_cpu(cpu) 6234 free_pcp += per_cpu_ptr(zone->per_cpu_pageset, cpu)->count; 6235 6236 show_node(zone); 6237 printk(KERN_CONT 6238 "%s" 6239 " free:%lukB" 6240 " boost:%lukB" 6241 " min:%lukB" 6242 " low:%lukB" 6243 " high:%lukB" 6244 " reserved_highatomic:%luKB" 6245 " active_anon:%lukB" 6246 " inactive_anon:%lukB" 6247 " active_file:%lukB" 6248 " inactive_file:%lukB" 6249 " unevictable:%lukB" 6250 " writepending:%lukB" 6251 " present:%lukB" 6252 " managed:%lukB" 6253 " mlocked:%lukB" 6254 " bounce:%lukB" 6255 " free_pcp:%lukB" 6256 " local_pcp:%ukB" 6257 " free_cma:%lukB" 6258 "\n", 6259 zone->name, 6260 K(zone_page_state(zone, NR_FREE_PAGES)), 6261 K(zone->watermark_boost), 6262 K(min_wmark_pages(zone)), 6263 K(low_wmark_pages(zone)), 6264 K(high_wmark_pages(zone)), 6265 K(zone->nr_reserved_highatomic), 6266 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 6267 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 6268 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 6269 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 6270 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 6271 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 6272 K(zone->present_pages), 6273 K(zone_managed_pages(zone)), 6274 K(zone_page_state(zone, NR_MLOCK)), 6275 K(zone_page_state(zone, NR_BOUNCE)), 6276 K(free_pcp), 6277 K(this_cpu_read(zone->per_cpu_pageset->count)), 6278 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 6279 printk("lowmem_reserve[]:"); 6280 for (i = 0; i < MAX_NR_ZONES; i++) 6281 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); 6282 printk(KERN_CONT "\n"); 6283 } 6284 6285 for_each_populated_zone(zone) { 6286 unsigned int order; 6287 unsigned long nr[MAX_ORDER], flags, total = 0; 6288 unsigned char types[MAX_ORDER]; 6289 6290 if (zone_idx(zone) > max_zone_idx) 6291 continue; 6292 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 6293 continue; 6294 show_node(zone); 6295 printk(KERN_CONT "%s: ", zone->name); 6296 6297 spin_lock_irqsave(&zone->lock, flags); 6298 for (order = 0; order < MAX_ORDER; order++) { 6299 struct free_area *area = &zone->free_area[order]; 6300 int type; 6301 6302 nr[order] = area->nr_free; 6303 total += nr[order] << order; 6304 6305 types[order] = 0; 6306 for (type = 0; type < MIGRATE_TYPES; type++) { 6307 if (!free_area_empty(area, type)) 6308 types[order] |= 1 << type; 6309 } 6310 } 6311 spin_unlock_irqrestore(&zone->lock, flags); 6312 for (order = 0; order < MAX_ORDER; order++) { 6313 printk(KERN_CONT "%lu*%lukB ", 6314 nr[order], K(1UL) << order); 6315 if (nr[order]) 6316 show_migration_types(types[order]); 6317 } 6318 printk(KERN_CONT "= %lukB\n", K(total)); 6319 } 6320 6321 for_each_online_node(nid) { 6322 if (show_mem_node_skip(filter, nid, nodemask)) 6323 continue; 6324 hugetlb_show_meminfo_node(nid); 6325 } 6326 6327 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 6328 6329 show_swap_cache_info(); 6330 } 6331 6332 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 6333 { 6334 zoneref->zone = zone; 6335 zoneref->zone_idx = zone_idx(zone); 6336 } 6337 6338 /* 6339 * Builds allocation fallback zone lists. 6340 * 6341 * Add all populated zones of a node to the zonelist. 6342 */ 6343 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) 6344 { 6345 struct zone *zone; 6346 enum zone_type zone_type = MAX_NR_ZONES; 6347 int nr_zones = 0; 6348 6349 do { 6350 zone_type--; 6351 zone = pgdat->node_zones + zone_type; 6352 if (populated_zone(zone)) { 6353 zoneref_set_zone(zone, &zonerefs[nr_zones++]); 6354 check_highest_zone(zone_type); 6355 } 6356 } while (zone_type); 6357 6358 return nr_zones; 6359 } 6360 6361 #ifdef CONFIG_NUMA 6362 6363 static int __parse_numa_zonelist_order(char *s) 6364 { 6365 /* 6366 * We used to support different zonelists modes but they turned 6367 * out to be just not useful. Let's keep the warning in place 6368 * if somebody still use the cmd line parameter so that we do 6369 * not fail it silently 6370 */ 6371 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) { 6372 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s); 6373 return -EINVAL; 6374 } 6375 return 0; 6376 } 6377 6378 char numa_zonelist_order[] = "Node"; 6379 6380 /* 6381 * sysctl handler for numa_zonelist_order 6382 */ 6383 int numa_zonelist_order_handler(struct ctl_table *table, int write, 6384 void *buffer, size_t *length, loff_t *ppos) 6385 { 6386 if (write) 6387 return __parse_numa_zonelist_order(buffer); 6388 return proc_dostring(table, write, buffer, length, ppos); 6389 } 6390 6391 6392 static int node_load[MAX_NUMNODES]; 6393 6394 /** 6395 * find_next_best_node - find the next node that should appear in a given node's fallback list 6396 * @node: node whose fallback list we're appending 6397 * @used_node_mask: nodemask_t of already used nodes 6398 * 6399 * We use a number of factors to determine which is the next node that should 6400 * appear on a given node's fallback list. The node should not have appeared 6401 * already in @node's fallback list, and it should be the next closest node 6402 * according to the distance array (which contains arbitrary distance values 6403 * from each node to each node in the system), and should also prefer nodes 6404 * with no CPUs, since presumably they'll have very little allocation pressure 6405 * on them otherwise. 6406 * 6407 * Return: node id of the found node or %NUMA_NO_NODE if no node is found. 6408 */ 6409 int find_next_best_node(int node, nodemask_t *used_node_mask) 6410 { 6411 int n, val; 6412 int min_val = INT_MAX; 6413 int best_node = NUMA_NO_NODE; 6414 6415 /* Use the local node if we haven't already */ 6416 if (!node_isset(node, *used_node_mask)) { 6417 node_set(node, *used_node_mask); 6418 return node; 6419 } 6420 6421 for_each_node_state(n, N_MEMORY) { 6422 6423 /* Don't want a node to appear more than once */ 6424 if (node_isset(n, *used_node_mask)) 6425 continue; 6426 6427 /* Use the distance array to find the distance */ 6428 val = node_distance(node, n); 6429 6430 /* Penalize nodes under us ("prefer the next node") */ 6431 val += (n < node); 6432 6433 /* Give preference to headless and unused nodes */ 6434 if (!cpumask_empty(cpumask_of_node(n))) 6435 val += PENALTY_FOR_NODE_WITH_CPUS; 6436 6437 /* Slight preference for less loaded node */ 6438 val *= MAX_NUMNODES; 6439 val += node_load[n]; 6440 6441 if (val < min_val) { 6442 min_val = val; 6443 best_node = n; 6444 } 6445 } 6446 6447 if (best_node >= 0) 6448 node_set(best_node, *used_node_mask); 6449 6450 return best_node; 6451 } 6452 6453 6454 /* 6455 * Build zonelists ordered by node and zones within node. 6456 * This results in maximum locality--normal zone overflows into local 6457 * DMA zone, if any--but risks exhausting DMA zone. 6458 */ 6459 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, 6460 unsigned nr_nodes) 6461 { 6462 struct zoneref *zonerefs; 6463 int i; 6464 6465 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 6466 6467 for (i = 0; i < nr_nodes; i++) { 6468 int nr_zones; 6469 6470 pg_data_t *node = NODE_DATA(node_order[i]); 6471 6472 nr_zones = build_zonerefs_node(node, zonerefs); 6473 zonerefs += nr_zones; 6474 } 6475 zonerefs->zone = NULL; 6476 zonerefs->zone_idx = 0; 6477 } 6478 6479 /* 6480 * Build gfp_thisnode zonelists 6481 */ 6482 static void build_thisnode_zonelists(pg_data_t *pgdat) 6483 { 6484 struct zoneref *zonerefs; 6485 int nr_zones; 6486 6487 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; 6488 nr_zones = build_zonerefs_node(pgdat, zonerefs); 6489 zonerefs += nr_zones; 6490 zonerefs->zone = NULL; 6491 zonerefs->zone_idx = 0; 6492 } 6493 6494 /* 6495 * Build zonelists ordered by zone and nodes within zones. 6496 * This results in conserving DMA zone[s] until all Normal memory is 6497 * exhausted, but results in overflowing to remote node while memory 6498 * may still exist in local DMA zone. 6499 */ 6500 6501 static void build_zonelists(pg_data_t *pgdat) 6502 { 6503 static int node_order[MAX_NUMNODES]; 6504 int node, nr_nodes = 0; 6505 nodemask_t used_mask = NODE_MASK_NONE; 6506 int local_node, prev_node; 6507 6508 /* NUMA-aware ordering of nodes */ 6509 local_node = pgdat->node_id; 6510 prev_node = local_node; 6511 6512 memset(node_order, 0, sizeof(node_order)); 6513 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 6514 /* 6515 * We don't want to pressure a particular node. 6516 * So adding penalty to the first node in same 6517 * distance group to make it round-robin. 6518 */ 6519 if (node_distance(local_node, node) != 6520 node_distance(local_node, prev_node)) 6521 node_load[node] += 1; 6522 6523 node_order[nr_nodes++] = node; 6524 prev_node = node; 6525 } 6526 6527 build_zonelists_in_node_order(pgdat, node_order, nr_nodes); 6528 build_thisnode_zonelists(pgdat); 6529 pr_info("Fallback order for Node %d: ", local_node); 6530 for (node = 0; node < nr_nodes; node++) 6531 pr_cont("%d ", node_order[node]); 6532 pr_cont("\n"); 6533 } 6534 6535 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 6536 /* 6537 * Return node id of node used for "local" allocations. 6538 * I.e., first node id of first zone in arg node's generic zonelist. 6539 * Used for initializing percpu 'numa_mem', which is used primarily 6540 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 6541 */ 6542 int local_memory_node(int node) 6543 { 6544 struct zoneref *z; 6545 6546 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 6547 gfp_zone(GFP_KERNEL), 6548 NULL); 6549 return zone_to_nid(z->zone); 6550 } 6551 #endif 6552 6553 static void setup_min_unmapped_ratio(void); 6554 static void setup_min_slab_ratio(void); 6555 #else /* CONFIG_NUMA */ 6556 6557 static void build_zonelists(pg_data_t *pgdat) 6558 { 6559 int node, local_node; 6560 struct zoneref *zonerefs; 6561 int nr_zones; 6562 6563 local_node = pgdat->node_id; 6564 6565 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 6566 nr_zones = build_zonerefs_node(pgdat, zonerefs); 6567 zonerefs += nr_zones; 6568 6569 /* 6570 * Now we build the zonelist so that it contains the zones 6571 * of all the other nodes. 6572 * We don't want to pressure a particular node, so when 6573 * building the zones for node N, we make sure that the 6574 * zones coming right after the local ones are those from 6575 * node N+1 (modulo N) 6576 */ 6577 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 6578 if (!node_online(node)) 6579 continue; 6580 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 6581 zonerefs += nr_zones; 6582 } 6583 for (node = 0; node < local_node; node++) { 6584 if (!node_online(node)) 6585 continue; 6586 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 6587 zonerefs += nr_zones; 6588 } 6589 6590 zonerefs->zone = NULL; 6591 zonerefs->zone_idx = 0; 6592 } 6593 6594 #endif /* CONFIG_NUMA */ 6595 6596 /* 6597 * Boot pageset table. One per cpu which is going to be used for all 6598 * zones and all nodes. The parameters will be set in such a way 6599 * that an item put on a list will immediately be handed over to 6600 * the buddy list. This is safe since pageset manipulation is done 6601 * with interrupts disabled. 6602 * 6603 * The boot_pagesets must be kept even after bootup is complete for 6604 * unused processors and/or zones. They do play a role for bootstrapping 6605 * hotplugged processors. 6606 * 6607 * zoneinfo_show() and maybe other functions do 6608 * not check if the processor is online before following the pageset pointer. 6609 * Other parts of the kernel may not check if the zone is available. 6610 */ 6611 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats); 6612 /* These effectively disable the pcplists in the boot pageset completely */ 6613 #define BOOT_PAGESET_HIGH 0 6614 #define BOOT_PAGESET_BATCH 1 6615 static DEFINE_PER_CPU(struct per_cpu_pages, boot_pageset); 6616 static DEFINE_PER_CPU(struct per_cpu_zonestat, boot_zonestats); 6617 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 6618 6619 static void __build_all_zonelists(void *data) 6620 { 6621 int nid; 6622 int __maybe_unused cpu; 6623 pg_data_t *self = data; 6624 6625 write_seqlock(&zonelist_update_seq); 6626 6627 #ifdef CONFIG_NUMA 6628 memset(node_load, 0, sizeof(node_load)); 6629 #endif 6630 6631 /* 6632 * This node is hotadded and no memory is yet present. So just 6633 * building zonelists is fine - no need to touch other nodes. 6634 */ 6635 if (self && !node_online(self->node_id)) { 6636 build_zonelists(self); 6637 } else { 6638 /* 6639 * All possible nodes have pgdat preallocated 6640 * in free_area_init 6641 */ 6642 for_each_node(nid) { 6643 pg_data_t *pgdat = NODE_DATA(nid); 6644 6645 build_zonelists(pgdat); 6646 } 6647 6648 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 6649 /* 6650 * We now know the "local memory node" for each node-- 6651 * i.e., the node of the first zone in the generic zonelist. 6652 * Set up numa_mem percpu variable for on-line cpus. During 6653 * boot, only the boot cpu should be on-line; we'll init the 6654 * secondary cpus' numa_mem as they come on-line. During 6655 * node/memory hotplug, we'll fixup all on-line cpus. 6656 */ 6657 for_each_online_cpu(cpu) 6658 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 6659 #endif 6660 } 6661 6662 write_sequnlock(&zonelist_update_seq); 6663 } 6664 6665 static noinline void __init 6666 build_all_zonelists_init(void) 6667 { 6668 int cpu; 6669 6670 __build_all_zonelists(NULL); 6671 6672 /* 6673 * Initialize the boot_pagesets that are going to be used 6674 * for bootstrapping processors. The real pagesets for 6675 * each zone will be allocated later when the per cpu 6676 * allocator is available. 6677 * 6678 * boot_pagesets are used also for bootstrapping offline 6679 * cpus if the system is already booted because the pagesets 6680 * are needed to initialize allocators on a specific cpu too. 6681 * F.e. the percpu allocator needs the page allocator which 6682 * needs the percpu allocator in order to allocate its pagesets 6683 * (a chicken-egg dilemma). 6684 */ 6685 for_each_possible_cpu(cpu) 6686 per_cpu_pages_init(&per_cpu(boot_pageset, cpu), &per_cpu(boot_zonestats, cpu)); 6687 6688 mminit_verify_zonelist(); 6689 cpuset_init_current_mems_allowed(); 6690 } 6691 6692 /* 6693 * unless system_state == SYSTEM_BOOTING. 6694 * 6695 * __ref due to call of __init annotated helper build_all_zonelists_init 6696 * [protected by SYSTEM_BOOTING]. 6697 */ 6698 void __ref build_all_zonelists(pg_data_t *pgdat) 6699 { 6700 unsigned long vm_total_pages; 6701 6702 if (system_state == SYSTEM_BOOTING) { 6703 build_all_zonelists_init(); 6704 } else { 6705 __build_all_zonelists(pgdat); 6706 /* cpuset refresh routine should be here */ 6707 } 6708 /* Get the number of free pages beyond high watermark in all zones. */ 6709 vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 6710 /* 6711 * Disable grouping by mobility if the number of pages in the 6712 * system is too low to allow the mechanism to work. It would be 6713 * more accurate, but expensive to check per-zone. This check is 6714 * made on memory-hotadd so a system can start with mobility 6715 * disabled and enable it later 6716 */ 6717 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 6718 page_group_by_mobility_disabled = 1; 6719 else 6720 page_group_by_mobility_disabled = 0; 6721 6722 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n", 6723 nr_online_nodes, 6724 page_group_by_mobility_disabled ? "off" : "on", 6725 vm_total_pages); 6726 #ifdef CONFIG_NUMA 6727 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 6728 #endif 6729 } 6730 6731 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 6732 static bool __meminit 6733 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 6734 { 6735 static struct memblock_region *r; 6736 6737 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 6738 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 6739 for_each_mem_region(r) { 6740 if (*pfn < memblock_region_memory_end_pfn(r)) 6741 break; 6742 } 6743 } 6744 if (*pfn >= memblock_region_memory_base_pfn(r) && 6745 memblock_is_mirror(r)) { 6746 *pfn = memblock_region_memory_end_pfn(r); 6747 return true; 6748 } 6749 } 6750 return false; 6751 } 6752 6753 /* 6754 * Initially all pages are reserved - free ones are freed 6755 * up by memblock_free_all() once the early boot process is 6756 * done. Non-atomic initialization, single-pass. 6757 * 6758 * All aligned pageblocks are initialized to the specified migratetype 6759 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related 6760 * zone stats (e.g., nr_isolate_pageblock) are touched. 6761 */ 6762 void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone, 6763 unsigned long start_pfn, unsigned long zone_end_pfn, 6764 enum meminit_context context, 6765 struct vmem_altmap *altmap, int migratetype) 6766 { 6767 unsigned long pfn, end_pfn = start_pfn + size; 6768 struct page *page; 6769 6770 if (highest_memmap_pfn < end_pfn - 1) 6771 highest_memmap_pfn = end_pfn - 1; 6772 6773 #ifdef CONFIG_ZONE_DEVICE 6774 /* 6775 * Honor reservation requested by the driver for this ZONE_DEVICE 6776 * memory. We limit the total number of pages to initialize to just 6777 * those that might contain the memory mapping. We will defer the 6778 * ZONE_DEVICE page initialization until after we have released 6779 * the hotplug lock. 6780 */ 6781 if (zone == ZONE_DEVICE) { 6782 if (!altmap) 6783 return; 6784 6785 if (start_pfn == altmap->base_pfn) 6786 start_pfn += altmap->reserve; 6787 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6788 } 6789 #endif 6790 6791 for (pfn = start_pfn; pfn < end_pfn; ) { 6792 /* 6793 * There can be holes in boot-time mem_map[]s handed to this 6794 * function. They do not exist on hotplugged memory. 6795 */ 6796 if (context == MEMINIT_EARLY) { 6797 if (overlap_memmap_init(zone, &pfn)) 6798 continue; 6799 if (defer_init(nid, pfn, zone_end_pfn)) 6800 break; 6801 } 6802 6803 page = pfn_to_page(pfn); 6804 __init_single_page(page, pfn, zone, nid); 6805 if (context == MEMINIT_HOTPLUG) 6806 __SetPageReserved(page); 6807 6808 /* 6809 * Usually, we want to mark the pageblock MIGRATE_MOVABLE, 6810 * such that unmovable allocations won't be scattered all 6811 * over the place during system boot. 6812 */ 6813 if (pageblock_aligned(pfn)) { 6814 set_pageblock_migratetype(page, migratetype); 6815 cond_resched(); 6816 } 6817 pfn++; 6818 } 6819 } 6820 6821 #ifdef CONFIG_ZONE_DEVICE 6822 static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, 6823 unsigned long zone_idx, int nid, 6824 struct dev_pagemap *pgmap) 6825 { 6826 6827 __init_single_page(page, pfn, zone_idx, nid); 6828 6829 /* 6830 * Mark page reserved as it will need to wait for onlining 6831 * phase for it to be fully associated with a zone. 6832 * 6833 * We can use the non-atomic __set_bit operation for setting 6834 * the flag as we are still initializing the pages. 6835 */ 6836 __SetPageReserved(page); 6837 6838 /* 6839 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 6840 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 6841 * ever freed or placed on a driver-private list. 6842 */ 6843 page->pgmap = pgmap; 6844 page->zone_device_data = NULL; 6845 6846 /* 6847 * Mark the block movable so that blocks are reserved for 6848 * movable at startup. This will force kernel allocations 6849 * to reserve their blocks rather than leaking throughout 6850 * the address space during boot when many long-lived 6851 * kernel allocations are made. 6852 * 6853 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap 6854 * because this is done early in section_activate() 6855 */ 6856 if (pageblock_aligned(pfn)) { 6857 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 6858 cond_resched(); 6859 } 6860 6861 /* 6862 * ZONE_DEVICE pages are released directly to the driver page allocator 6863 * which will set the page count to 1 when allocating the page. 6864 */ 6865 if (pgmap->type == MEMORY_DEVICE_PRIVATE || 6866 pgmap->type == MEMORY_DEVICE_COHERENT) 6867 set_page_count(page, 0); 6868 } 6869 6870 /* 6871 * With compound page geometry and when struct pages are stored in ram most 6872 * tail pages are reused. Consequently, the amount of unique struct pages to 6873 * initialize is a lot smaller that the total amount of struct pages being 6874 * mapped. This is a paired / mild layering violation with explicit knowledge 6875 * of how the sparse_vmemmap internals handle compound pages in the lack 6876 * of an altmap. See vmemmap_populate_compound_pages(). 6877 */ 6878 static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap, 6879 unsigned long nr_pages) 6880 { 6881 return is_power_of_2(sizeof(struct page)) && 6882 !altmap ? 2 * (PAGE_SIZE / sizeof(struct page)) : nr_pages; 6883 } 6884 6885 static void __ref memmap_init_compound(struct page *head, 6886 unsigned long head_pfn, 6887 unsigned long zone_idx, int nid, 6888 struct dev_pagemap *pgmap, 6889 unsigned long nr_pages) 6890 { 6891 unsigned long pfn, end_pfn = head_pfn + nr_pages; 6892 unsigned int order = pgmap->vmemmap_shift; 6893 6894 __SetPageHead(head); 6895 for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) { 6896 struct page *page = pfn_to_page(pfn); 6897 6898 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); 6899 prep_compound_tail(head, pfn - head_pfn); 6900 set_page_count(page, 0); 6901 6902 /* 6903 * The first tail page stores important compound page info. 6904 * Call prep_compound_head() after the first tail page has 6905 * been initialized, to not have the data overwritten. 6906 */ 6907 if (pfn == head_pfn + 1) 6908 prep_compound_head(head, order); 6909 } 6910 } 6911 6912 void __ref memmap_init_zone_device(struct zone *zone, 6913 unsigned long start_pfn, 6914 unsigned long nr_pages, 6915 struct dev_pagemap *pgmap) 6916 { 6917 unsigned long pfn, end_pfn = start_pfn + nr_pages; 6918 struct pglist_data *pgdat = zone->zone_pgdat; 6919 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 6920 unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap); 6921 unsigned long zone_idx = zone_idx(zone); 6922 unsigned long start = jiffies; 6923 int nid = pgdat->node_id; 6924 6925 if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE)) 6926 return; 6927 6928 /* 6929 * The call to memmap_init should have already taken care 6930 * of the pages reserved for the memmap, so we can just jump to 6931 * the end of that region and start processing the device pages. 6932 */ 6933 if (altmap) { 6934 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6935 nr_pages = end_pfn - start_pfn; 6936 } 6937 6938 for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { 6939 struct page *page = pfn_to_page(pfn); 6940 6941 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); 6942 6943 if (pfns_per_compound == 1) 6944 continue; 6945 6946 memmap_init_compound(page, pfn, zone_idx, nid, pgmap, 6947 compound_nr_pages(altmap, pfns_per_compound)); 6948 } 6949 6950 pr_info("%s initialised %lu pages in %ums\n", __func__, 6951 nr_pages, jiffies_to_msecs(jiffies - start)); 6952 } 6953 6954 #endif 6955 static void __meminit zone_init_free_lists(struct zone *zone) 6956 { 6957 unsigned int order, t; 6958 for_each_migratetype_order(order, t) { 6959 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 6960 zone->free_area[order].nr_free = 0; 6961 } 6962 } 6963 6964 /* 6965 * Only struct pages that correspond to ranges defined by memblock.memory 6966 * are zeroed and initialized by going through __init_single_page() during 6967 * memmap_init_zone_range(). 6968 * 6969 * But, there could be struct pages that correspond to holes in 6970 * memblock.memory. This can happen because of the following reasons: 6971 * - physical memory bank size is not necessarily the exact multiple of the 6972 * arbitrary section size 6973 * - early reserved memory may not be listed in memblock.memory 6974 * - memory layouts defined with memmap= kernel parameter may not align 6975 * nicely with memmap sections 6976 * 6977 * Explicitly initialize those struct pages so that: 6978 * - PG_Reserved is set 6979 * - zone and node links point to zone and node that span the page if the 6980 * hole is in the middle of a zone 6981 * - zone and node links point to adjacent zone/node if the hole falls on 6982 * the zone boundary; the pages in such holes will be prepended to the 6983 * zone/node above the hole except for the trailing pages in the last 6984 * section that will be appended to the zone/node below. 6985 */ 6986 static void __init init_unavailable_range(unsigned long spfn, 6987 unsigned long epfn, 6988 int zone, int node) 6989 { 6990 unsigned long pfn; 6991 u64 pgcnt = 0; 6992 6993 for (pfn = spfn; pfn < epfn; pfn++) { 6994 if (!pfn_valid(pageblock_start_pfn(pfn))) { 6995 pfn = pageblock_end_pfn(pfn) - 1; 6996 continue; 6997 } 6998 __init_single_page(pfn_to_page(pfn), pfn, zone, node); 6999 __SetPageReserved(pfn_to_page(pfn)); 7000 pgcnt++; 7001 } 7002 7003 if (pgcnt) 7004 pr_info("On node %d, zone %s: %lld pages in unavailable ranges", 7005 node, zone_names[zone], pgcnt); 7006 } 7007 7008 static void __init memmap_init_zone_range(struct zone *zone, 7009 unsigned long start_pfn, 7010 unsigned long end_pfn, 7011 unsigned long *hole_pfn) 7012 { 7013 unsigned long zone_start_pfn = zone->zone_start_pfn; 7014 unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; 7015 int nid = zone_to_nid(zone), zone_id = zone_idx(zone); 7016 7017 start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn); 7018 end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn); 7019 7020 if (start_pfn >= end_pfn) 7021 return; 7022 7023 memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, 7024 zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); 7025 7026 if (*hole_pfn < start_pfn) 7027 init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); 7028 7029 *hole_pfn = end_pfn; 7030 } 7031 7032 static void __init memmap_init(void) 7033 { 7034 unsigned long start_pfn, end_pfn; 7035 unsigned long hole_pfn = 0; 7036 int i, j, zone_id = 0, nid; 7037 7038 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7039 struct pglist_data *node = NODE_DATA(nid); 7040 7041 for (j = 0; j < MAX_NR_ZONES; j++) { 7042 struct zone *zone = node->node_zones + j; 7043 7044 if (!populated_zone(zone)) 7045 continue; 7046 7047 memmap_init_zone_range(zone, start_pfn, end_pfn, 7048 &hole_pfn); 7049 zone_id = j; 7050 } 7051 } 7052 7053 #ifdef CONFIG_SPARSEMEM 7054 /* 7055 * Initialize the memory map for hole in the range [memory_end, 7056 * section_end]. 7057 * Append the pages in this hole to the highest zone in the last 7058 * node. 7059 * The call to init_unavailable_range() is outside the ifdef to 7060 * silence the compiler warining about zone_id set but not used; 7061 * for FLATMEM it is a nop anyway 7062 */ 7063 end_pfn = round_up(end_pfn, PAGES_PER_SECTION); 7064 if (hole_pfn < end_pfn) 7065 #endif 7066 init_unavailable_range(hole_pfn, end_pfn, zone_id, nid); 7067 } 7068 7069 void __init *memmap_alloc(phys_addr_t size, phys_addr_t align, 7070 phys_addr_t min_addr, int nid, bool exact_nid) 7071 { 7072 void *ptr; 7073 7074 if (exact_nid) 7075 ptr = memblock_alloc_exact_nid_raw(size, align, min_addr, 7076 MEMBLOCK_ALLOC_ACCESSIBLE, 7077 nid); 7078 else 7079 ptr = memblock_alloc_try_nid_raw(size, align, min_addr, 7080 MEMBLOCK_ALLOC_ACCESSIBLE, 7081 nid); 7082 7083 if (ptr && size > 0) 7084 page_init_poison(ptr, size); 7085 7086 return ptr; 7087 } 7088 7089 static int zone_batchsize(struct zone *zone) 7090 { 7091 #ifdef CONFIG_MMU 7092 int batch; 7093 7094 /* 7095 * The number of pages to batch allocate is either ~0.1% 7096 * of the zone or 1MB, whichever is smaller. The batch 7097 * size is striking a balance between allocation latency 7098 * and zone lock contention. 7099 */ 7100 batch = min(zone_managed_pages(zone) >> 10, SZ_1M / PAGE_SIZE); 7101 batch /= 4; /* We effectively *= 4 below */ 7102 if (batch < 1) 7103 batch = 1; 7104 7105 /* 7106 * Clamp the batch to a 2^n - 1 value. Having a power 7107 * of 2 value was found to be more likely to have 7108 * suboptimal cache aliasing properties in some cases. 7109 * 7110 * For example if 2 tasks are alternately allocating 7111 * batches of pages, one task can end up with a lot 7112 * of pages of one half of the possible page colors 7113 * and the other with pages of the other colors. 7114 */ 7115 batch = rounddown_pow_of_two(batch + batch/2) - 1; 7116 7117 return batch; 7118 7119 #else 7120 /* The deferral and batching of frees should be suppressed under NOMMU 7121 * conditions. 7122 * 7123 * The problem is that NOMMU needs to be able to allocate large chunks 7124 * of contiguous memory as there's no hardware page translation to 7125 * assemble apparent contiguous memory from discontiguous pages. 7126 * 7127 * Queueing large contiguous runs of pages for batching, however, 7128 * causes the pages to actually be freed in smaller chunks. As there 7129 * can be a significant delay between the individual batches being 7130 * recycled, this leads to the once large chunks of space being 7131 * fragmented and becoming unavailable for high-order allocations. 7132 */ 7133 return 0; 7134 #endif 7135 } 7136 7137 static int zone_highsize(struct zone *zone, int batch, int cpu_online) 7138 { 7139 #ifdef CONFIG_MMU 7140 int high; 7141 int nr_split_cpus; 7142 unsigned long total_pages; 7143 7144 if (!percpu_pagelist_high_fraction) { 7145 /* 7146 * By default, the high value of the pcp is based on the zone 7147 * low watermark so that if they are full then background 7148 * reclaim will not be started prematurely. 7149 */ 7150 total_pages = low_wmark_pages(zone); 7151 } else { 7152 /* 7153 * If percpu_pagelist_high_fraction is configured, the high 7154 * value is based on a fraction of the managed pages in the 7155 * zone. 7156 */ 7157 total_pages = zone_managed_pages(zone) / percpu_pagelist_high_fraction; 7158 } 7159 7160 /* 7161 * Split the high value across all online CPUs local to the zone. Note 7162 * that early in boot that CPUs may not be online yet and that during 7163 * CPU hotplug that the cpumask is not yet updated when a CPU is being 7164 * onlined. For memory nodes that have no CPUs, split pcp->high across 7165 * all online CPUs to mitigate the risk that reclaim is triggered 7166 * prematurely due to pages stored on pcp lists. 7167 */ 7168 nr_split_cpus = cpumask_weight(cpumask_of_node(zone_to_nid(zone))) + cpu_online; 7169 if (!nr_split_cpus) 7170 nr_split_cpus = num_online_cpus(); 7171 high = total_pages / nr_split_cpus; 7172 7173 /* 7174 * Ensure high is at least batch*4. The multiple is based on the 7175 * historical relationship between high and batch. 7176 */ 7177 high = max(high, batch << 2); 7178 7179 return high; 7180 #else 7181 return 0; 7182 #endif 7183 } 7184 7185 /* 7186 * pcp->high and pcp->batch values are related and generally batch is lower 7187 * than high. They are also related to pcp->count such that count is lower 7188 * than high, and as soon as it reaches high, the pcplist is flushed. 7189 * 7190 * However, guaranteeing these relations at all times would require e.g. write 7191 * barriers here but also careful usage of read barriers at the read side, and 7192 * thus be prone to error and bad for performance. Thus the update only prevents 7193 * store tearing. Any new users of pcp->batch and pcp->high should ensure they 7194 * can cope with those fields changing asynchronously, and fully trust only the 7195 * pcp->count field on the local CPU with interrupts disabled. 7196 * 7197 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 7198 * outside of boot time (or some other assurance that no concurrent updaters 7199 * exist). 7200 */ 7201 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 7202 unsigned long batch) 7203 { 7204 WRITE_ONCE(pcp->batch, batch); 7205 WRITE_ONCE(pcp->high, high); 7206 } 7207 7208 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats) 7209 { 7210 int pindex; 7211 7212 memset(pcp, 0, sizeof(*pcp)); 7213 memset(pzstats, 0, sizeof(*pzstats)); 7214 7215 spin_lock_init(&pcp->lock); 7216 for (pindex = 0; pindex < NR_PCP_LISTS; pindex++) 7217 INIT_LIST_HEAD(&pcp->lists[pindex]); 7218 7219 /* 7220 * Set batch and high values safe for a boot pageset. A true percpu 7221 * pageset's initialization will update them subsequently. Here we don't 7222 * need to be as careful as pageset_update() as nobody can access the 7223 * pageset yet. 7224 */ 7225 pcp->high = BOOT_PAGESET_HIGH; 7226 pcp->batch = BOOT_PAGESET_BATCH; 7227 pcp->free_factor = 0; 7228 } 7229 7230 static void __zone_set_pageset_high_and_batch(struct zone *zone, unsigned long high, 7231 unsigned long batch) 7232 { 7233 struct per_cpu_pages *pcp; 7234 int cpu; 7235 7236 for_each_possible_cpu(cpu) { 7237 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 7238 pageset_update(pcp, high, batch); 7239 } 7240 } 7241 7242 /* 7243 * Calculate and set new high and batch values for all per-cpu pagesets of a 7244 * zone based on the zone's size. 7245 */ 7246 static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online) 7247 { 7248 int new_high, new_batch; 7249 7250 new_batch = max(1, zone_batchsize(zone)); 7251 new_high = zone_highsize(zone, new_batch, cpu_online); 7252 7253 if (zone->pageset_high == new_high && 7254 zone->pageset_batch == new_batch) 7255 return; 7256 7257 zone->pageset_high = new_high; 7258 zone->pageset_batch = new_batch; 7259 7260 __zone_set_pageset_high_and_batch(zone, new_high, new_batch); 7261 } 7262 7263 void __meminit setup_zone_pageset(struct zone *zone) 7264 { 7265 int cpu; 7266 7267 /* Size may be 0 on !SMP && !NUMA */ 7268 if (sizeof(struct per_cpu_zonestat) > 0) 7269 zone->per_cpu_zonestats = alloc_percpu(struct per_cpu_zonestat); 7270 7271 zone->per_cpu_pageset = alloc_percpu(struct per_cpu_pages); 7272 for_each_possible_cpu(cpu) { 7273 struct per_cpu_pages *pcp; 7274 struct per_cpu_zonestat *pzstats; 7275 7276 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 7277 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 7278 per_cpu_pages_init(pcp, pzstats); 7279 } 7280 7281 zone_set_pageset_high_and_batch(zone, 0); 7282 } 7283 7284 /* 7285 * The zone indicated has a new number of managed_pages; batch sizes and percpu 7286 * page high values need to be recalculated. 7287 */ 7288 static void zone_pcp_update(struct zone *zone, int cpu_online) 7289 { 7290 mutex_lock(&pcp_batch_high_lock); 7291 zone_set_pageset_high_and_batch(zone, cpu_online); 7292 mutex_unlock(&pcp_batch_high_lock); 7293 } 7294 7295 /* 7296 * Allocate per cpu pagesets and initialize them. 7297 * Before this call only boot pagesets were available. 7298 */ 7299 void __init setup_per_cpu_pageset(void) 7300 { 7301 struct pglist_data *pgdat; 7302 struct zone *zone; 7303 int __maybe_unused cpu; 7304 7305 for_each_populated_zone(zone) 7306 setup_zone_pageset(zone); 7307 7308 #ifdef CONFIG_NUMA 7309 /* 7310 * Unpopulated zones continue using the boot pagesets. 7311 * The numa stats for these pagesets need to be reset. 7312 * Otherwise, they will end up skewing the stats of 7313 * the nodes these zones are associated with. 7314 */ 7315 for_each_possible_cpu(cpu) { 7316 struct per_cpu_zonestat *pzstats = &per_cpu(boot_zonestats, cpu); 7317 memset(pzstats->vm_numa_event, 0, 7318 sizeof(pzstats->vm_numa_event)); 7319 } 7320 #endif 7321 7322 for_each_online_pgdat(pgdat) 7323 pgdat->per_cpu_nodestats = 7324 alloc_percpu(struct per_cpu_nodestat); 7325 } 7326 7327 static __meminit void zone_pcp_init(struct zone *zone) 7328 { 7329 /* 7330 * per cpu subsystem is not up at this point. The following code 7331 * relies on the ability of the linker to provide the 7332 * offset of a (static) per cpu variable into the per cpu area. 7333 */ 7334 zone->per_cpu_pageset = &boot_pageset; 7335 zone->per_cpu_zonestats = &boot_zonestats; 7336 zone->pageset_high = BOOT_PAGESET_HIGH; 7337 zone->pageset_batch = BOOT_PAGESET_BATCH; 7338 7339 if (populated_zone(zone)) 7340 pr_debug(" %s zone: %lu pages, LIFO batch:%u\n", zone->name, 7341 zone->present_pages, zone_batchsize(zone)); 7342 } 7343 7344 void __meminit init_currently_empty_zone(struct zone *zone, 7345 unsigned long zone_start_pfn, 7346 unsigned long size) 7347 { 7348 struct pglist_data *pgdat = zone->zone_pgdat; 7349 int zone_idx = zone_idx(zone) + 1; 7350 7351 if (zone_idx > pgdat->nr_zones) 7352 pgdat->nr_zones = zone_idx; 7353 7354 zone->zone_start_pfn = zone_start_pfn; 7355 7356 mminit_dprintk(MMINIT_TRACE, "memmap_init", 7357 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 7358 pgdat->node_id, 7359 (unsigned long)zone_idx(zone), 7360 zone_start_pfn, (zone_start_pfn + size)); 7361 7362 zone_init_free_lists(zone); 7363 zone->initialized = 1; 7364 } 7365 7366 /** 7367 * get_pfn_range_for_nid - Return the start and end page frames for a node 7368 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 7369 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 7370 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 7371 * 7372 * It returns the start and end page frame of a node based on information 7373 * provided by memblock_set_node(). If called for a node 7374 * with no available memory, a warning is printed and the start and end 7375 * PFNs will be 0. 7376 */ 7377 void __init get_pfn_range_for_nid(unsigned int nid, 7378 unsigned long *start_pfn, unsigned long *end_pfn) 7379 { 7380 unsigned long this_start_pfn, this_end_pfn; 7381 int i; 7382 7383 *start_pfn = -1UL; 7384 *end_pfn = 0; 7385 7386 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 7387 *start_pfn = min(*start_pfn, this_start_pfn); 7388 *end_pfn = max(*end_pfn, this_end_pfn); 7389 } 7390 7391 if (*start_pfn == -1UL) 7392 *start_pfn = 0; 7393 } 7394 7395 /* 7396 * This finds a zone that can be used for ZONE_MOVABLE pages. The 7397 * assumption is made that zones within a node are ordered in monotonic 7398 * increasing memory addresses so that the "highest" populated zone is used 7399 */ 7400 static void __init find_usable_zone_for_movable(void) 7401 { 7402 int zone_index; 7403 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 7404 if (zone_index == ZONE_MOVABLE) 7405 continue; 7406 7407 if (arch_zone_highest_possible_pfn[zone_index] > 7408 arch_zone_lowest_possible_pfn[zone_index]) 7409 break; 7410 } 7411 7412 VM_BUG_ON(zone_index == -1); 7413 movable_zone = zone_index; 7414 } 7415 7416 /* 7417 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 7418 * because it is sized independent of architecture. Unlike the other zones, 7419 * the starting point for ZONE_MOVABLE is not fixed. It may be different 7420 * in each node depending on the size of each node and how evenly kernelcore 7421 * is distributed. This helper function adjusts the zone ranges 7422 * provided by the architecture for a given node by using the end of the 7423 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 7424 * zones within a node are in order of monotonic increases memory addresses 7425 */ 7426 static void __init adjust_zone_range_for_zone_movable(int nid, 7427 unsigned long zone_type, 7428 unsigned long node_start_pfn, 7429 unsigned long node_end_pfn, 7430 unsigned long *zone_start_pfn, 7431 unsigned long *zone_end_pfn) 7432 { 7433 /* Only adjust if ZONE_MOVABLE is on this node */ 7434 if (zone_movable_pfn[nid]) { 7435 /* Size ZONE_MOVABLE */ 7436 if (zone_type == ZONE_MOVABLE) { 7437 *zone_start_pfn = zone_movable_pfn[nid]; 7438 *zone_end_pfn = min(node_end_pfn, 7439 arch_zone_highest_possible_pfn[movable_zone]); 7440 7441 /* Adjust for ZONE_MOVABLE starting within this range */ 7442 } else if (!mirrored_kernelcore && 7443 *zone_start_pfn < zone_movable_pfn[nid] && 7444 *zone_end_pfn > zone_movable_pfn[nid]) { 7445 *zone_end_pfn = zone_movable_pfn[nid]; 7446 7447 /* Check if this whole range is within ZONE_MOVABLE */ 7448 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 7449 *zone_start_pfn = *zone_end_pfn; 7450 } 7451 } 7452 7453 /* 7454 * Return the number of pages a zone spans in a node, including holes 7455 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 7456 */ 7457 static unsigned long __init zone_spanned_pages_in_node(int nid, 7458 unsigned long zone_type, 7459 unsigned long node_start_pfn, 7460 unsigned long node_end_pfn, 7461 unsigned long *zone_start_pfn, 7462 unsigned long *zone_end_pfn) 7463 { 7464 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 7465 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 7466 /* When hotadd a new node from cpu_up(), the node should be empty */ 7467 if (!node_start_pfn && !node_end_pfn) 7468 return 0; 7469 7470 /* Get the start and end of the zone */ 7471 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 7472 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 7473 adjust_zone_range_for_zone_movable(nid, zone_type, 7474 node_start_pfn, node_end_pfn, 7475 zone_start_pfn, zone_end_pfn); 7476 7477 /* Check that this node has pages within the zone's required range */ 7478 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 7479 return 0; 7480 7481 /* Move the zone boundaries inside the node if necessary */ 7482 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 7483 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 7484 7485 /* Return the spanned pages */ 7486 return *zone_end_pfn - *zone_start_pfn; 7487 } 7488 7489 /* 7490 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 7491 * then all holes in the requested range will be accounted for. 7492 */ 7493 unsigned long __init __absent_pages_in_range(int nid, 7494 unsigned long range_start_pfn, 7495 unsigned long range_end_pfn) 7496 { 7497 unsigned long nr_absent = range_end_pfn - range_start_pfn; 7498 unsigned long start_pfn, end_pfn; 7499 int i; 7500 7501 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7502 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 7503 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 7504 nr_absent -= end_pfn - start_pfn; 7505 } 7506 return nr_absent; 7507 } 7508 7509 /** 7510 * absent_pages_in_range - Return number of page frames in holes within a range 7511 * @start_pfn: The start PFN to start searching for holes 7512 * @end_pfn: The end PFN to stop searching for holes 7513 * 7514 * Return: the number of pages frames in memory holes within a range. 7515 */ 7516 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 7517 unsigned long end_pfn) 7518 { 7519 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 7520 } 7521 7522 /* Return the number of page frames in holes in a zone on a node */ 7523 static unsigned long __init zone_absent_pages_in_node(int nid, 7524 unsigned long zone_type, 7525 unsigned long node_start_pfn, 7526 unsigned long node_end_pfn) 7527 { 7528 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 7529 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 7530 unsigned long zone_start_pfn, zone_end_pfn; 7531 unsigned long nr_absent; 7532 7533 /* When hotadd a new node from cpu_up(), the node should be empty */ 7534 if (!node_start_pfn && !node_end_pfn) 7535 return 0; 7536 7537 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 7538 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 7539 7540 adjust_zone_range_for_zone_movable(nid, zone_type, 7541 node_start_pfn, node_end_pfn, 7542 &zone_start_pfn, &zone_end_pfn); 7543 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 7544 7545 /* 7546 * ZONE_MOVABLE handling. 7547 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 7548 * and vice versa. 7549 */ 7550 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 7551 unsigned long start_pfn, end_pfn; 7552 struct memblock_region *r; 7553 7554 for_each_mem_region(r) { 7555 start_pfn = clamp(memblock_region_memory_base_pfn(r), 7556 zone_start_pfn, zone_end_pfn); 7557 end_pfn = clamp(memblock_region_memory_end_pfn(r), 7558 zone_start_pfn, zone_end_pfn); 7559 7560 if (zone_type == ZONE_MOVABLE && 7561 memblock_is_mirror(r)) 7562 nr_absent += end_pfn - start_pfn; 7563 7564 if (zone_type == ZONE_NORMAL && 7565 !memblock_is_mirror(r)) 7566 nr_absent += end_pfn - start_pfn; 7567 } 7568 } 7569 7570 return nr_absent; 7571 } 7572 7573 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 7574 unsigned long node_start_pfn, 7575 unsigned long node_end_pfn) 7576 { 7577 unsigned long realtotalpages = 0, totalpages = 0; 7578 enum zone_type i; 7579 7580 for (i = 0; i < MAX_NR_ZONES; i++) { 7581 struct zone *zone = pgdat->node_zones + i; 7582 unsigned long zone_start_pfn, zone_end_pfn; 7583 unsigned long spanned, absent; 7584 unsigned long size, real_size; 7585 7586 spanned = zone_spanned_pages_in_node(pgdat->node_id, i, 7587 node_start_pfn, 7588 node_end_pfn, 7589 &zone_start_pfn, 7590 &zone_end_pfn); 7591 absent = zone_absent_pages_in_node(pgdat->node_id, i, 7592 node_start_pfn, 7593 node_end_pfn); 7594 7595 size = spanned; 7596 real_size = size - absent; 7597 7598 if (size) 7599 zone->zone_start_pfn = zone_start_pfn; 7600 else 7601 zone->zone_start_pfn = 0; 7602 zone->spanned_pages = size; 7603 zone->present_pages = real_size; 7604 #if defined(CONFIG_MEMORY_HOTPLUG) 7605 zone->present_early_pages = real_size; 7606 #endif 7607 7608 totalpages += size; 7609 realtotalpages += real_size; 7610 } 7611 7612 pgdat->node_spanned_pages = totalpages; 7613 pgdat->node_present_pages = realtotalpages; 7614 pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); 7615 } 7616 7617 #ifndef CONFIG_SPARSEMEM 7618 /* 7619 * Calculate the size of the zone->blockflags rounded to an unsigned long 7620 * Start by making sure zonesize is a multiple of pageblock_order by rounding 7621 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 7622 * round what is now in bits to nearest long in bits, then return it in 7623 * bytes. 7624 */ 7625 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 7626 { 7627 unsigned long usemapsize; 7628 7629 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 7630 usemapsize = roundup(zonesize, pageblock_nr_pages); 7631 usemapsize = usemapsize >> pageblock_order; 7632 usemapsize *= NR_PAGEBLOCK_BITS; 7633 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 7634 7635 return usemapsize / 8; 7636 } 7637 7638 static void __ref setup_usemap(struct zone *zone) 7639 { 7640 unsigned long usemapsize = usemap_size(zone->zone_start_pfn, 7641 zone->spanned_pages); 7642 zone->pageblock_flags = NULL; 7643 if (usemapsize) { 7644 zone->pageblock_flags = 7645 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 7646 zone_to_nid(zone)); 7647 if (!zone->pageblock_flags) 7648 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 7649 usemapsize, zone->name, zone_to_nid(zone)); 7650 } 7651 } 7652 #else 7653 static inline void setup_usemap(struct zone *zone) {} 7654 #endif /* CONFIG_SPARSEMEM */ 7655 7656 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 7657 7658 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 7659 void __init set_pageblock_order(void) 7660 { 7661 unsigned int order = MAX_ORDER - 1; 7662 7663 /* Check that pageblock_nr_pages has not already been setup */ 7664 if (pageblock_order) 7665 return; 7666 7667 /* Don't let pageblocks exceed the maximum allocation granularity. */ 7668 if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order) 7669 order = HUGETLB_PAGE_ORDER; 7670 7671 /* 7672 * Assume the largest contiguous order of interest is a huge page. 7673 * This value may be variable depending on boot parameters on IA64 and 7674 * powerpc. 7675 */ 7676 pageblock_order = order; 7677 } 7678 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 7679 7680 /* 7681 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 7682 * is unused as pageblock_order is set at compile-time. See 7683 * include/linux/pageblock-flags.h for the values of pageblock_order based on 7684 * the kernel config 7685 */ 7686 void __init set_pageblock_order(void) 7687 { 7688 } 7689 7690 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 7691 7692 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 7693 unsigned long present_pages) 7694 { 7695 unsigned long pages = spanned_pages; 7696 7697 /* 7698 * Provide a more accurate estimation if there are holes within 7699 * the zone and SPARSEMEM is in use. If there are holes within the 7700 * zone, each populated memory region may cost us one or two extra 7701 * memmap pages due to alignment because memmap pages for each 7702 * populated regions may not be naturally aligned on page boundary. 7703 * So the (present_pages >> 4) heuristic is a tradeoff for that. 7704 */ 7705 if (spanned_pages > present_pages + (present_pages >> 4) && 7706 IS_ENABLED(CONFIG_SPARSEMEM)) 7707 pages = present_pages; 7708 7709 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 7710 } 7711 7712 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 7713 static void pgdat_init_split_queue(struct pglist_data *pgdat) 7714 { 7715 struct deferred_split *ds_queue = &pgdat->deferred_split_queue; 7716 7717 spin_lock_init(&ds_queue->split_queue_lock); 7718 INIT_LIST_HEAD(&ds_queue->split_queue); 7719 ds_queue->split_queue_len = 0; 7720 } 7721 #else 7722 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 7723 #endif 7724 7725 #ifdef CONFIG_COMPACTION 7726 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 7727 { 7728 init_waitqueue_head(&pgdat->kcompactd_wait); 7729 } 7730 #else 7731 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 7732 #endif 7733 7734 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 7735 { 7736 int i; 7737 7738 pgdat_resize_init(pgdat); 7739 pgdat_kswapd_lock_init(pgdat); 7740 7741 pgdat_init_split_queue(pgdat); 7742 pgdat_init_kcompactd(pgdat); 7743 7744 init_waitqueue_head(&pgdat->kswapd_wait); 7745 init_waitqueue_head(&pgdat->pfmemalloc_wait); 7746 7747 for (i = 0; i < NR_VMSCAN_THROTTLE; i++) 7748 init_waitqueue_head(&pgdat->reclaim_wait[i]); 7749 7750 pgdat_page_ext_init(pgdat); 7751 lruvec_init(&pgdat->__lruvec); 7752 } 7753 7754 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 7755 unsigned long remaining_pages) 7756 { 7757 atomic_long_set(&zone->managed_pages, remaining_pages); 7758 zone_set_nid(zone, nid); 7759 zone->name = zone_names[idx]; 7760 zone->zone_pgdat = NODE_DATA(nid); 7761 spin_lock_init(&zone->lock); 7762 zone_seqlock_init(zone); 7763 zone_pcp_init(zone); 7764 } 7765 7766 /* 7767 * Set up the zone data structures 7768 * - init pgdat internals 7769 * - init all zones belonging to this node 7770 * 7771 * NOTE: this function is only called during memory hotplug 7772 */ 7773 #ifdef CONFIG_MEMORY_HOTPLUG 7774 void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) 7775 { 7776 int nid = pgdat->node_id; 7777 enum zone_type z; 7778 int cpu; 7779 7780 pgdat_init_internals(pgdat); 7781 7782 if (pgdat->per_cpu_nodestats == &boot_nodestats) 7783 pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); 7784 7785 /* 7786 * Reset the nr_zones, order and highest_zoneidx before reuse. 7787 * Note that kswapd will init kswapd_highest_zoneidx properly 7788 * when it starts in the near future. 7789 */ 7790 pgdat->nr_zones = 0; 7791 pgdat->kswapd_order = 0; 7792 pgdat->kswapd_highest_zoneidx = 0; 7793 pgdat->node_start_pfn = 0; 7794 for_each_online_cpu(cpu) { 7795 struct per_cpu_nodestat *p; 7796 7797 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 7798 memset(p, 0, sizeof(*p)); 7799 } 7800 7801 for (z = 0; z < MAX_NR_ZONES; z++) 7802 zone_init_internals(&pgdat->node_zones[z], z, nid, 0); 7803 } 7804 #endif 7805 7806 /* 7807 * Set up the zone data structures: 7808 * - mark all pages reserved 7809 * - mark all memory queues empty 7810 * - clear the memory bitmaps 7811 * 7812 * NOTE: pgdat should get zeroed by caller. 7813 * NOTE: this function is only called during early init. 7814 */ 7815 static void __init free_area_init_core(struct pglist_data *pgdat) 7816 { 7817 enum zone_type j; 7818 int nid = pgdat->node_id; 7819 7820 pgdat_init_internals(pgdat); 7821 pgdat->per_cpu_nodestats = &boot_nodestats; 7822 7823 for (j = 0; j < MAX_NR_ZONES; j++) { 7824 struct zone *zone = pgdat->node_zones + j; 7825 unsigned long size, freesize, memmap_pages; 7826 7827 size = zone->spanned_pages; 7828 freesize = zone->present_pages; 7829 7830 /* 7831 * Adjust freesize so that it accounts for how much memory 7832 * is used by this zone for memmap. This affects the watermark 7833 * and per-cpu initialisations 7834 */ 7835 memmap_pages = calc_memmap_size(size, freesize); 7836 if (!is_highmem_idx(j)) { 7837 if (freesize >= memmap_pages) { 7838 freesize -= memmap_pages; 7839 if (memmap_pages) 7840 pr_debug(" %s zone: %lu pages used for memmap\n", 7841 zone_names[j], memmap_pages); 7842 } else 7843 pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n", 7844 zone_names[j], memmap_pages, freesize); 7845 } 7846 7847 /* Account for reserved pages */ 7848 if (j == 0 && freesize > dma_reserve) { 7849 freesize -= dma_reserve; 7850 pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve); 7851 } 7852 7853 if (!is_highmem_idx(j)) 7854 nr_kernel_pages += freesize; 7855 /* Charge for highmem memmap if there are enough kernel pages */ 7856 else if (nr_kernel_pages > memmap_pages * 2) 7857 nr_kernel_pages -= memmap_pages; 7858 nr_all_pages += freesize; 7859 7860 /* 7861 * Set an approximate value for lowmem here, it will be adjusted 7862 * when the bootmem allocator frees pages into the buddy system. 7863 * And all highmem pages will be managed by the buddy system. 7864 */ 7865 zone_init_internals(zone, j, nid, freesize); 7866 7867 if (!size) 7868 continue; 7869 7870 set_pageblock_order(); 7871 setup_usemap(zone); 7872 init_currently_empty_zone(zone, zone->zone_start_pfn, size); 7873 } 7874 } 7875 7876 #ifdef CONFIG_FLATMEM 7877 static void __init alloc_node_mem_map(struct pglist_data *pgdat) 7878 { 7879 unsigned long __maybe_unused start = 0; 7880 unsigned long __maybe_unused offset = 0; 7881 7882 /* Skip empty nodes */ 7883 if (!pgdat->node_spanned_pages) 7884 return; 7885 7886 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 7887 offset = pgdat->node_start_pfn - start; 7888 /* ia64 gets its own node_mem_map, before this, without bootmem */ 7889 if (!pgdat->node_mem_map) { 7890 unsigned long size, end; 7891 struct page *map; 7892 7893 /* 7894 * The zone's endpoints aren't required to be MAX_ORDER 7895 * aligned but the node_mem_map endpoints must be in order 7896 * for the buddy allocator to function correctly. 7897 */ 7898 end = pgdat_end_pfn(pgdat); 7899 end = ALIGN(end, MAX_ORDER_NR_PAGES); 7900 size = (end - start) * sizeof(struct page); 7901 map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT, 7902 pgdat->node_id, false); 7903 if (!map) 7904 panic("Failed to allocate %ld bytes for node %d memory map\n", 7905 size, pgdat->node_id); 7906 pgdat->node_mem_map = map + offset; 7907 } 7908 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 7909 __func__, pgdat->node_id, (unsigned long)pgdat, 7910 (unsigned long)pgdat->node_mem_map); 7911 #ifndef CONFIG_NUMA 7912 /* 7913 * With no DISCONTIG, the global mem_map is just set as node 0's 7914 */ 7915 if (pgdat == NODE_DATA(0)) { 7916 mem_map = NODE_DATA(0)->node_mem_map; 7917 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 7918 mem_map -= offset; 7919 } 7920 #endif 7921 } 7922 #else 7923 static inline void alloc_node_mem_map(struct pglist_data *pgdat) { } 7924 #endif /* CONFIG_FLATMEM */ 7925 7926 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 7927 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 7928 { 7929 pgdat->first_deferred_pfn = ULONG_MAX; 7930 } 7931 #else 7932 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 7933 #endif 7934 7935 static void __init free_area_init_node(int nid) 7936 { 7937 pg_data_t *pgdat = NODE_DATA(nid); 7938 unsigned long start_pfn = 0; 7939 unsigned long end_pfn = 0; 7940 7941 /* pg_data_t should be reset to zero when it's allocated */ 7942 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); 7943 7944 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 7945 7946 pgdat->node_id = nid; 7947 pgdat->node_start_pfn = start_pfn; 7948 pgdat->per_cpu_nodestats = NULL; 7949 7950 if (start_pfn != end_pfn) { 7951 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 7952 (u64)start_pfn << PAGE_SHIFT, 7953 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 7954 } else { 7955 pr_info("Initmem setup node %d as memoryless\n", nid); 7956 } 7957 7958 calculate_node_totalpages(pgdat, start_pfn, end_pfn); 7959 7960 alloc_node_mem_map(pgdat); 7961 pgdat_set_deferred_range(pgdat); 7962 7963 free_area_init_core(pgdat); 7964 lru_gen_init_pgdat(pgdat); 7965 } 7966 7967 static void __init free_area_init_memoryless_node(int nid) 7968 { 7969 free_area_init_node(nid); 7970 } 7971 7972 #if MAX_NUMNODES > 1 7973 /* 7974 * Figure out the number of possible node ids. 7975 */ 7976 void __init setup_nr_node_ids(void) 7977 { 7978 unsigned int highest; 7979 7980 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 7981 nr_node_ids = highest + 1; 7982 } 7983 #endif 7984 7985 /** 7986 * node_map_pfn_alignment - determine the maximum internode alignment 7987 * 7988 * This function should be called after node map is populated and sorted. 7989 * It calculates the maximum power of two alignment which can distinguish 7990 * all the nodes. 7991 * 7992 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 7993 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 7994 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 7995 * shifted, 1GiB is enough and this function will indicate so. 7996 * 7997 * This is used to test whether pfn -> nid mapping of the chosen memory 7998 * model has fine enough granularity to avoid incorrect mapping for the 7999 * populated node map. 8000 * 8001 * Return: the determined alignment in pfn's. 0 if there is no alignment 8002 * requirement (single node). 8003 */ 8004 unsigned long __init node_map_pfn_alignment(void) 8005 { 8006 unsigned long accl_mask = 0, last_end = 0; 8007 unsigned long start, end, mask; 8008 int last_nid = NUMA_NO_NODE; 8009 int i, nid; 8010 8011 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 8012 if (!start || last_nid < 0 || last_nid == nid) { 8013 last_nid = nid; 8014 last_end = end; 8015 continue; 8016 } 8017 8018 /* 8019 * Start with a mask granular enough to pin-point to the 8020 * start pfn and tick off bits one-by-one until it becomes 8021 * too coarse to separate the current node from the last. 8022 */ 8023 mask = ~((1 << __ffs(start)) - 1); 8024 while (mask && last_end <= (start & (mask << 1))) 8025 mask <<= 1; 8026 8027 /* accumulate all internode masks */ 8028 accl_mask |= mask; 8029 } 8030 8031 /* convert mask to number of pages */ 8032 return ~accl_mask + 1; 8033 } 8034 8035 /* 8036 * early_calculate_totalpages() 8037 * Sum pages in active regions for movable zone. 8038 * Populate N_MEMORY for calculating usable_nodes. 8039 */ 8040 static unsigned long __init early_calculate_totalpages(void) 8041 { 8042 unsigned long totalpages = 0; 8043 unsigned long start_pfn, end_pfn; 8044 int i, nid; 8045 8046 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 8047 unsigned long pages = end_pfn - start_pfn; 8048 8049 totalpages += pages; 8050 if (pages) 8051 node_set_state(nid, N_MEMORY); 8052 } 8053 return totalpages; 8054 } 8055 8056 /* 8057 * Find the PFN the Movable zone begins in each node. Kernel memory 8058 * is spread evenly between nodes as long as the nodes have enough 8059 * memory. When they don't, some nodes will have more kernelcore than 8060 * others 8061 */ 8062 static void __init find_zone_movable_pfns_for_nodes(void) 8063 { 8064 int i, nid; 8065 unsigned long usable_startpfn; 8066 unsigned long kernelcore_node, kernelcore_remaining; 8067 /* save the state before borrow the nodemask */ 8068 nodemask_t saved_node_state = node_states[N_MEMORY]; 8069 unsigned long totalpages = early_calculate_totalpages(); 8070 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 8071 struct memblock_region *r; 8072 8073 /* Need to find movable_zone earlier when movable_node is specified. */ 8074 find_usable_zone_for_movable(); 8075 8076 /* 8077 * If movable_node is specified, ignore kernelcore and movablecore 8078 * options. 8079 */ 8080 if (movable_node_is_enabled()) { 8081 for_each_mem_region(r) { 8082 if (!memblock_is_hotpluggable(r)) 8083 continue; 8084 8085 nid = memblock_get_region_node(r); 8086 8087 usable_startpfn = PFN_DOWN(r->base); 8088 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 8089 min(usable_startpfn, zone_movable_pfn[nid]) : 8090 usable_startpfn; 8091 } 8092 8093 goto out2; 8094 } 8095 8096 /* 8097 * If kernelcore=mirror is specified, ignore movablecore option 8098 */ 8099 if (mirrored_kernelcore) { 8100 bool mem_below_4gb_not_mirrored = false; 8101 8102 for_each_mem_region(r) { 8103 if (memblock_is_mirror(r)) 8104 continue; 8105 8106 nid = memblock_get_region_node(r); 8107 8108 usable_startpfn = memblock_region_memory_base_pfn(r); 8109 8110 if (usable_startpfn < PHYS_PFN(SZ_4G)) { 8111 mem_below_4gb_not_mirrored = true; 8112 continue; 8113 } 8114 8115 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 8116 min(usable_startpfn, zone_movable_pfn[nid]) : 8117 usable_startpfn; 8118 } 8119 8120 if (mem_below_4gb_not_mirrored) 8121 pr_warn("This configuration results in unmirrored kernel memory.\n"); 8122 8123 goto out2; 8124 } 8125 8126 /* 8127 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 8128 * amount of necessary memory. 8129 */ 8130 if (required_kernelcore_percent) 8131 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 8132 10000UL; 8133 if (required_movablecore_percent) 8134 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 8135 10000UL; 8136 8137 /* 8138 * If movablecore= was specified, calculate what size of 8139 * kernelcore that corresponds so that memory usable for 8140 * any allocation type is evenly spread. If both kernelcore 8141 * and movablecore are specified, then the value of kernelcore 8142 * will be used for required_kernelcore if it's greater than 8143 * what movablecore would have allowed. 8144 */ 8145 if (required_movablecore) { 8146 unsigned long corepages; 8147 8148 /* 8149 * Round-up so that ZONE_MOVABLE is at least as large as what 8150 * was requested by the user 8151 */ 8152 required_movablecore = 8153 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 8154 required_movablecore = min(totalpages, required_movablecore); 8155 corepages = totalpages - required_movablecore; 8156 8157 required_kernelcore = max(required_kernelcore, corepages); 8158 } 8159 8160 /* 8161 * If kernelcore was not specified or kernelcore size is larger 8162 * than totalpages, there is no ZONE_MOVABLE. 8163 */ 8164 if (!required_kernelcore || required_kernelcore >= totalpages) 8165 goto out; 8166 8167 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 8168 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 8169 8170 restart: 8171 /* Spread kernelcore memory as evenly as possible throughout nodes */ 8172 kernelcore_node = required_kernelcore / usable_nodes; 8173 for_each_node_state(nid, N_MEMORY) { 8174 unsigned long start_pfn, end_pfn; 8175 8176 /* 8177 * Recalculate kernelcore_node if the division per node 8178 * now exceeds what is necessary to satisfy the requested 8179 * amount of memory for the kernel 8180 */ 8181 if (required_kernelcore < kernelcore_node) 8182 kernelcore_node = required_kernelcore / usable_nodes; 8183 8184 /* 8185 * As the map is walked, we track how much memory is usable 8186 * by the kernel using kernelcore_remaining. When it is 8187 * 0, the rest of the node is usable by ZONE_MOVABLE 8188 */ 8189 kernelcore_remaining = kernelcore_node; 8190 8191 /* Go through each range of PFNs within this node */ 8192 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 8193 unsigned long size_pages; 8194 8195 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 8196 if (start_pfn >= end_pfn) 8197 continue; 8198 8199 /* Account for what is only usable for kernelcore */ 8200 if (start_pfn < usable_startpfn) { 8201 unsigned long kernel_pages; 8202 kernel_pages = min(end_pfn, usable_startpfn) 8203 - start_pfn; 8204 8205 kernelcore_remaining -= min(kernel_pages, 8206 kernelcore_remaining); 8207 required_kernelcore -= min(kernel_pages, 8208 required_kernelcore); 8209 8210 /* Continue if range is now fully accounted */ 8211 if (end_pfn <= usable_startpfn) { 8212 8213 /* 8214 * Push zone_movable_pfn to the end so 8215 * that if we have to rebalance 8216 * kernelcore across nodes, we will 8217 * not double account here 8218 */ 8219 zone_movable_pfn[nid] = end_pfn; 8220 continue; 8221 } 8222 start_pfn = usable_startpfn; 8223 } 8224 8225 /* 8226 * The usable PFN range for ZONE_MOVABLE is from 8227 * start_pfn->end_pfn. Calculate size_pages as the 8228 * number of pages used as kernelcore 8229 */ 8230 size_pages = end_pfn - start_pfn; 8231 if (size_pages > kernelcore_remaining) 8232 size_pages = kernelcore_remaining; 8233 zone_movable_pfn[nid] = start_pfn + size_pages; 8234 8235 /* 8236 * Some kernelcore has been met, update counts and 8237 * break if the kernelcore for this node has been 8238 * satisfied 8239 */ 8240 required_kernelcore -= min(required_kernelcore, 8241 size_pages); 8242 kernelcore_remaining -= size_pages; 8243 if (!kernelcore_remaining) 8244 break; 8245 } 8246 } 8247 8248 /* 8249 * If there is still required_kernelcore, we do another pass with one 8250 * less node in the count. This will push zone_movable_pfn[nid] further 8251 * along on the nodes that still have memory until kernelcore is 8252 * satisfied 8253 */ 8254 usable_nodes--; 8255 if (usable_nodes && required_kernelcore > usable_nodes) 8256 goto restart; 8257 8258 out2: 8259 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 8260 for (nid = 0; nid < MAX_NUMNODES; nid++) { 8261 unsigned long start_pfn, end_pfn; 8262 8263 zone_movable_pfn[nid] = 8264 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 8265 8266 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 8267 if (zone_movable_pfn[nid] >= end_pfn) 8268 zone_movable_pfn[nid] = 0; 8269 } 8270 8271 out: 8272 /* restore the node_state */ 8273 node_states[N_MEMORY] = saved_node_state; 8274 } 8275 8276 /* Any regular or high memory on that node ? */ 8277 static void check_for_memory(pg_data_t *pgdat, int nid) 8278 { 8279 enum zone_type zone_type; 8280 8281 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 8282 struct zone *zone = &pgdat->node_zones[zone_type]; 8283 if (populated_zone(zone)) { 8284 if (IS_ENABLED(CONFIG_HIGHMEM)) 8285 node_set_state(nid, N_HIGH_MEMORY); 8286 if (zone_type <= ZONE_NORMAL) 8287 node_set_state(nid, N_NORMAL_MEMORY); 8288 break; 8289 } 8290 } 8291 } 8292 8293 /* 8294 * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For 8295 * such cases we allow max_zone_pfn sorted in the descending order 8296 */ 8297 bool __weak arch_has_descending_max_zone_pfns(void) 8298 { 8299 return false; 8300 } 8301 8302 /** 8303 * free_area_init - Initialise all pg_data_t and zone data 8304 * @max_zone_pfn: an array of max PFNs for each zone 8305 * 8306 * This will call free_area_init_node() for each active node in the system. 8307 * Using the page ranges provided by memblock_set_node(), the size of each 8308 * zone in each node and their holes is calculated. If the maximum PFN 8309 * between two adjacent zones match, it is assumed that the zone is empty. 8310 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 8311 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 8312 * starts where the previous one ended. For example, ZONE_DMA32 starts 8313 * at arch_max_dma_pfn. 8314 */ 8315 void __init free_area_init(unsigned long *max_zone_pfn) 8316 { 8317 unsigned long start_pfn, end_pfn; 8318 int i, nid, zone; 8319 bool descending; 8320 8321 /* Record where the zone boundaries are */ 8322 memset(arch_zone_lowest_possible_pfn, 0, 8323 sizeof(arch_zone_lowest_possible_pfn)); 8324 memset(arch_zone_highest_possible_pfn, 0, 8325 sizeof(arch_zone_highest_possible_pfn)); 8326 8327 start_pfn = PHYS_PFN(memblock_start_of_DRAM()); 8328 descending = arch_has_descending_max_zone_pfns(); 8329 8330 for (i = 0; i < MAX_NR_ZONES; i++) { 8331 if (descending) 8332 zone = MAX_NR_ZONES - i - 1; 8333 else 8334 zone = i; 8335 8336 if (zone == ZONE_MOVABLE) 8337 continue; 8338 8339 end_pfn = max(max_zone_pfn[zone], start_pfn); 8340 arch_zone_lowest_possible_pfn[zone] = start_pfn; 8341 arch_zone_highest_possible_pfn[zone] = end_pfn; 8342 8343 start_pfn = end_pfn; 8344 } 8345 8346 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 8347 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 8348 find_zone_movable_pfns_for_nodes(); 8349 8350 /* Print out the zone ranges */ 8351 pr_info("Zone ranges:\n"); 8352 for (i = 0; i < MAX_NR_ZONES; i++) { 8353 if (i == ZONE_MOVABLE) 8354 continue; 8355 pr_info(" %-8s ", zone_names[i]); 8356 if (arch_zone_lowest_possible_pfn[i] == 8357 arch_zone_highest_possible_pfn[i]) 8358 pr_cont("empty\n"); 8359 else 8360 pr_cont("[mem %#018Lx-%#018Lx]\n", 8361 (u64)arch_zone_lowest_possible_pfn[i] 8362 << PAGE_SHIFT, 8363 ((u64)arch_zone_highest_possible_pfn[i] 8364 << PAGE_SHIFT) - 1); 8365 } 8366 8367 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 8368 pr_info("Movable zone start for each node\n"); 8369 for (i = 0; i < MAX_NUMNODES; i++) { 8370 if (zone_movable_pfn[i]) 8371 pr_info(" Node %d: %#018Lx\n", i, 8372 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 8373 } 8374 8375 /* 8376 * Print out the early node map, and initialize the 8377 * subsection-map relative to active online memory ranges to 8378 * enable future "sub-section" extensions of the memory map. 8379 */ 8380 pr_info("Early memory node ranges\n"); 8381 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 8382 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 8383 (u64)start_pfn << PAGE_SHIFT, 8384 ((u64)end_pfn << PAGE_SHIFT) - 1); 8385 subsection_map_init(start_pfn, end_pfn - start_pfn); 8386 } 8387 8388 /* Initialise every node */ 8389 mminit_verify_pageflags_layout(); 8390 setup_nr_node_ids(); 8391 for_each_node(nid) { 8392 pg_data_t *pgdat; 8393 8394 if (!node_online(nid)) { 8395 pr_info("Initializing node %d as memoryless\n", nid); 8396 8397 /* Allocator not initialized yet */ 8398 pgdat = arch_alloc_nodedata(nid); 8399 if (!pgdat) { 8400 pr_err("Cannot allocate %zuB for node %d.\n", 8401 sizeof(*pgdat), nid); 8402 continue; 8403 } 8404 arch_refresh_nodedata(nid, pgdat); 8405 free_area_init_memoryless_node(nid); 8406 8407 /* 8408 * We do not want to confuse userspace by sysfs 8409 * files/directories for node without any memory 8410 * attached to it, so this node is not marked as 8411 * N_MEMORY and not marked online so that no sysfs 8412 * hierarchy will be created via register_one_node for 8413 * it. The pgdat will get fully initialized by 8414 * hotadd_init_pgdat() when memory is hotplugged into 8415 * this node. 8416 */ 8417 continue; 8418 } 8419 8420 pgdat = NODE_DATA(nid); 8421 free_area_init_node(nid); 8422 8423 /* Any memory on that node */ 8424 if (pgdat->node_present_pages) 8425 node_set_state(nid, N_MEMORY); 8426 check_for_memory(pgdat, nid); 8427 } 8428 8429 memmap_init(); 8430 } 8431 8432 static int __init cmdline_parse_core(char *p, unsigned long *core, 8433 unsigned long *percent) 8434 { 8435 unsigned long long coremem; 8436 char *endptr; 8437 8438 if (!p) 8439 return -EINVAL; 8440 8441 /* Value may be a percentage of total memory, otherwise bytes */ 8442 coremem = simple_strtoull(p, &endptr, 0); 8443 if (*endptr == '%') { 8444 /* Paranoid check for percent values greater than 100 */ 8445 WARN_ON(coremem > 100); 8446 8447 *percent = coremem; 8448 } else { 8449 coremem = memparse(p, &p); 8450 /* Paranoid check that UL is enough for the coremem value */ 8451 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 8452 8453 *core = coremem >> PAGE_SHIFT; 8454 *percent = 0UL; 8455 } 8456 return 0; 8457 } 8458 8459 /* 8460 * kernelcore=size sets the amount of memory for use for allocations that 8461 * cannot be reclaimed or migrated. 8462 */ 8463 static int __init cmdline_parse_kernelcore(char *p) 8464 { 8465 /* parse kernelcore=mirror */ 8466 if (parse_option_str(p, "mirror")) { 8467 mirrored_kernelcore = true; 8468 return 0; 8469 } 8470 8471 return cmdline_parse_core(p, &required_kernelcore, 8472 &required_kernelcore_percent); 8473 } 8474 8475 /* 8476 * movablecore=size sets the amount of memory for use for allocations that 8477 * can be reclaimed or migrated. 8478 */ 8479 static int __init cmdline_parse_movablecore(char *p) 8480 { 8481 return cmdline_parse_core(p, &required_movablecore, 8482 &required_movablecore_percent); 8483 } 8484 8485 early_param("kernelcore", cmdline_parse_kernelcore); 8486 early_param("movablecore", cmdline_parse_movablecore); 8487 8488 void adjust_managed_page_count(struct page *page, long count) 8489 { 8490 atomic_long_add(count, &page_zone(page)->managed_pages); 8491 totalram_pages_add(count); 8492 #ifdef CONFIG_HIGHMEM 8493 if (PageHighMem(page)) 8494 totalhigh_pages_add(count); 8495 #endif 8496 } 8497 EXPORT_SYMBOL(adjust_managed_page_count); 8498 8499 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 8500 { 8501 void *pos; 8502 unsigned long pages = 0; 8503 8504 start = (void *)PAGE_ALIGN((unsigned long)start); 8505 end = (void *)((unsigned long)end & PAGE_MASK); 8506 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 8507 struct page *page = virt_to_page(pos); 8508 void *direct_map_addr; 8509 8510 /* 8511 * 'direct_map_addr' might be different from 'pos' 8512 * because some architectures' virt_to_page() 8513 * work with aliases. Getting the direct map 8514 * address ensures that we get a _writeable_ 8515 * alias for the memset(). 8516 */ 8517 direct_map_addr = page_address(page); 8518 /* 8519 * Perform a kasan-unchecked memset() since this memory 8520 * has not been initialized. 8521 */ 8522 direct_map_addr = kasan_reset_tag(direct_map_addr); 8523 if ((unsigned int)poison <= 0xFF) 8524 memset(direct_map_addr, poison, PAGE_SIZE); 8525 8526 free_reserved_page(page); 8527 } 8528 8529 if (pages && s) 8530 pr_info("Freeing %s memory: %ldK\n", s, K(pages)); 8531 8532 return pages; 8533 } 8534 8535 void __init mem_init_print_info(void) 8536 { 8537 unsigned long physpages, codesize, datasize, rosize, bss_size; 8538 unsigned long init_code_size, init_data_size; 8539 8540 physpages = get_num_physpages(); 8541 codesize = _etext - _stext; 8542 datasize = _edata - _sdata; 8543 rosize = __end_rodata - __start_rodata; 8544 bss_size = __bss_stop - __bss_start; 8545 init_data_size = __init_end - __init_begin; 8546 init_code_size = _einittext - _sinittext; 8547 8548 /* 8549 * Detect special cases and adjust section sizes accordingly: 8550 * 1) .init.* may be embedded into .data sections 8551 * 2) .init.text.* may be out of [__init_begin, __init_end], 8552 * please refer to arch/tile/kernel/vmlinux.lds.S. 8553 * 3) .rodata.* may be embedded into .text or .data sections. 8554 */ 8555 #define adj_init_size(start, end, size, pos, adj) \ 8556 do { \ 8557 if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \ 8558 size -= adj; \ 8559 } while (0) 8560 8561 adj_init_size(__init_begin, __init_end, init_data_size, 8562 _sinittext, init_code_size); 8563 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 8564 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 8565 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 8566 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 8567 8568 #undef adj_init_size 8569 8570 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 8571 #ifdef CONFIG_HIGHMEM 8572 ", %luK highmem" 8573 #endif 8574 ")\n", 8575 K(nr_free_pages()), K(physpages), 8576 codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K, 8577 (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K, 8578 K(physpages - totalram_pages() - totalcma_pages), 8579 K(totalcma_pages) 8580 #ifdef CONFIG_HIGHMEM 8581 , K(totalhigh_pages()) 8582 #endif 8583 ); 8584 } 8585 8586 /** 8587 * set_dma_reserve - set the specified number of pages reserved in the first zone 8588 * @new_dma_reserve: The number of pages to mark reserved 8589 * 8590 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 8591 * In the DMA zone, a significant percentage may be consumed by kernel image 8592 * and other unfreeable allocations which can skew the watermarks badly. This 8593 * function may optionally be used to account for unfreeable pages in the 8594 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 8595 * smaller per-cpu batchsize. 8596 */ 8597 void __init set_dma_reserve(unsigned long new_dma_reserve) 8598 { 8599 dma_reserve = new_dma_reserve; 8600 } 8601 8602 static int page_alloc_cpu_dead(unsigned int cpu) 8603 { 8604 struct zone *zone; 8605 8606 lru_add_drain_cpu(cpu); 8607 mlock_drain_remote(cpu); 8608 drain_pages(cpu); 8609 8610 /* 8611 * Spill the event counters of the dead processor 8612 * into the current processors event counters. 8613 * This artificially elevates the count of the current 8614 * processor. 8615 */ 8616 vm_events_fold_cpu(cpu); 8617 8618 /* 8619 * Zero the differential counters of the dead processor 8620 * so that the vm statistics are consistent. 8621 * 8622 * This is only okay since the processor is dead and cannot 8623 * race with what we are doing. 8624 */ 8625 cpu_vm_stats_fold(cpu); 8626 8627 for_each_populated_zone(zone) 8628 zone_pcp_update(zone, 0); 8629 8630 return 0; 8631 } 8632 8633 static int page_alloc_cpu_online(unsigned int cpu) 8634 { 8635 struct zone *zone; 8636 8637 for_each_populated_zone(zone) 8638 zone_pcp_update(zone, 1); 8639 return 0; 8640 } 8641 8642 #ifdef CONFIG_NUMA 8643 int hashdist = HASHDIST_DEFAULT; 8644 8645 static int __init set_hashdist(char *str) 8646 { 8647 if (!str) 8648 return 0; 8649 hashdist = simple_strtoul(str, &str, 0); 8650 return 1; 8651 } 8652 __setup("hashdist=", set_hashdist); 8653 #endif 8654 8655 void __init page_alloc_init(void) 8656 { 8657 int ret; 8658 8659 #ifdef CONFIG_NUMA 8660 if (num_node_state(N_MEMORY) == 1) 8661 hashdist = 0; 8662 #endif 8663 8664 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC, 8665 "mm/page_alloc:pcp", 8666 page_alloc_cpu_online, 8667 page_alloc_cpu_dead); 8668 WARN_ON(ret < 0); 8669 } 8670 8671 /* 8672 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 8673 * or min_free_kbytes changes. 8674 */ 8675 static void calculate_totalreserve_pages(void) 8676 { 8677 struct pglist_data *pgdat; 8678 unsigned long reserve_pages = 0; 8679 enum zone_type i, j; 8680 8681 for_each_online_pgdat(pgdat) { 8682 8683 pgdat->totalreserve_pages = 0; 8684 8685 for (i = 0; i < MAX_NR_ZONES; i++) { 8686 struct zone *zone = pgdat->node_zones + i; 8687 long max = 0; 8688 unsigned long managed_pages = zone_managed_pages(zone); 8689 8690 /* Find valid and maximum lowmem_reserve in the zone */ 8691 for (j = i; j < MAX_NR_ZONES; j++) { 8692 if (zone->lowmem_reserve[j] > max) 8693 max = zone->lowmem_reserve[j]; 8694 } 8695 8696 /* we treat the high watermark as reserved pages. */ 8697 max += high_wmark_pages(zone); 8698 8699 if (max > managed_pages) 8700 max = managed_pages; 8701 8702 pgdat->totalreserve_pages += max; 8703 8704 reserve_pages += max; 8705 } 8706 } 8707 totalreserve_pages = reserve_pages; 8708 } 8709 8710 /* 8711 * setup_per_zone_lowmem_reserve - called whenever 8712 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 8713 * has a correct pages reserved value, so an adequate number of 8714 * pages are left in the zone after a successful __alloc_pages(). 8715 */ 8716 static void setup_per_zone_lowmem_reserve(void) 8717 { 8718 struct pglist_data *pgdat; 8719 enum zone_type i, j; 8720 8721 for_each_online_pgdat(pgdat) { 8722 for (i = 0; i < MAX_NR_ZONES - 1; i++) { 8723 struct zone *zone = &pgdat->node_zones[i]; 8724 int ratio = sysctl_lowmem_reserve_ratio[i]; 8725 bool clear = !ratio || !zone_managed_pages(zone); 8726 unsigned long managed_pages = 0; 8727 8728 for (j = i + 1; j < MAX_NR_ZONES; j++) { 8729 struct zone *upper_zone = &pgdat->node_zones[j]; 8730 8731 managed_pages += zone_managed_pages(upper_zone); 8732 8733 if (clear) 8734 zone->lowmem_reserve[j] = 0; 8735 else 8736 zone->lowmem_reserve[j] = managed_pages / ratio; 8737 } 8738 } 8739 } 8740 8741 /* update totalreserve_pages */ 8742 calculate_totalreserve_pages(); 8743 } 8744 8745 static void __setup_per_zone_wmarks(void) 8746 { 8747 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 8748 unsigned long lowmem_pages = 0; 8749 struct zone *zone; 8750 unsigned long flags; 8751 8752 /* Calculate total number of !ZONE_HIGHMEM pages */ 8753 for_each_zone(zone) { 8754 if (!is_highmem(zone)) 8755 lowmem_pages += zone_managed_pages(zone); 8756 } 8757 8758 for_each_zone(zone) { 8759 u64 tmp; 8760 8761 spin_lock_irqsave(&zone->lock, flags); 8762 tmp = (u64)pages_min * zone_managed_pages(zone); 8763 do_div(tmp, lowmem_pages); 8764 if (is_highmem(zone)) { 8765 /* 8766 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 8767 * need highmem pages, so cap pages_min to a small 8768 * value here. 8769 * 8770 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 8771 * deltas control async page reclaim, and so should 8772 * not be capped for highmem. 8773 */ 8774 unsigned long min_pages; 8775 8776 min_pages = zone_managed_pages(zone) / 1024; 8777 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 8778 zone->_watermark[WMARK_MIN] = min_pages; 8779 } else { 8780 /* 8781 * If it's a lowmem zone, reserve a number of pages 8782 * proportionate to the zone's size. 8783 */ 8784 zone->_watermark[WMARK_MIN] = tmp; 8785 } 8786 8787 /* 8788 * Set the kswapd watermarks distance according to the 8789 * scale factor in proportion to available memory, but 8790 * ensure a minimum size on small systems. 8791 */ 8792 tmp = max_t(u64, tmp >> 2, 8793 mult_frac(zone_managed_pages(zone), 8794 watermark_scale_factor, 10000)); 8795 8796 zone->watermark_boost = 0; 8797 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 8798 zone->_watermark[WMARK_HIGH] = low_wmark_pages(zone) + tmp; 8799 zone->_watermark[WMARK_PROMO] = high_wmark_pages(zone) + tmp; 8800 8801 spin_unlock_irqrestore(&zone->lock, flags); 8802 } 8803 8804 /* update totalreserve_pages */ 8805 calculate_totalreserve_pages(); 8806 } 8807 8808 /** 8809 * setup_per_zone_wmarks - called when min_free_kbytes changes 8810 * or when memory is hot-{added|removed} 8811 * 8812 * Ensures that the watermark[min,low,high] values for each zone are set 8813 * correctly with respect to min_free_kbytes. 8814 */ 8815 void setup_per_zone_wmarks(void) 8816 { 8817 struct zone *zone; 8818 static DEFINE_SPINLOCK(lock); 8819 8820 spin_lock(&lock); 8821 __setup_per_zone_wmarks(); 8822 spin_unlock(&lock); 8823 8824 /* 8825 * The watermark size have changed so update the pcpu batch 8826 * and high limits or the limits may be inappropriate. 8827 */ 8828 for_each_zone(zone) 8829 zone_pcp_update(zone, 0); 8830 } 8831 8832 /* 8833 * Initialise min_free_kbytes. 8834 * 8835 * For small machines we want it small (128k min). For large machines 8836 * we want it large (256MB max). But it is not linear, because network 8837 * bandwidth does not increase linearly with machine size. We use 8838 * 8839 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 8840 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 8841 * 8842 * which yields 8843 * 8844 * 16MB: 512k 8845 * 32MB: 724k 8846 * 64MB: 1024k 8847 * 128MB: 1448k 8848 * 256MB: 2048k 8849 * 512MB: 2896k 8850 * 1024MB: 4096k 8851 * 2048MB: 5792k 8852 * 4096MB: 8192k 8853 * 8192MB: 11584k 8854 * 16384MB: 16384k 8855 */ 8856 void calculate_min_free_kbytes(void) 8857 { 8858 unsigned long lowmem_kbytes; 8859 int new_min_free_kbytes; 8860 8861 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 8862 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 8863 8864 if (new_min_free_kbytes > user_min_free_kbytes) 8865 min_free_kbytes = clamp(new_min_free_kbytes, 128, 262144); 8866 else 8867 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 8868 new_min_free_kbytes, user_min_free_kbytes); 8869 8870 } 8871 8872 int __meminit init_per_zone_wmark_min(void) 8873 { 8874 calculate_min_free_kbytes(); 8875 setup_per_zone_wmarks(); 8876 refresh_zone_stat_thresholds(); 8877 setup_per_zone_lowmem_reserve(); 8878 8879 #ifdef CONFIG_NUMA 8880 setup_min_unmapped_ratio(); 8881 setup_min_slab_ratio(); 8882 #endif 8883 8884 khugepaged_min_free_kbytes_update(); 8885 8886 return 0; 8887 } 8888 postcore_initcall(init_per_zone_wmark_min) 8889 8890 /* 8891 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 8892 * that we can call two helper functions whenever min_free_kbytes 8893 * changes. 8894 */ 8895 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 8896 void *buffer, size_t *length, loff_t *ppos) 8897 { 8898 int rc; 8899 8900 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8901 if (rc) 8902 return rc; 8903 8904 if (write) { 8905 user_min_free_kbytes = min_free_kbytes; 8906 setup_per_zone_wmarks(); 8907 } 8908 return 0; 8909 } 8910 8911 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 8912 void *buffer, size_t *length, loff_t *ppos) 8913 { 8914 int rc; 8915 8916 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8917 if (rc) 8918 return rc; 8919 8920 if (write) 8921 setup_per_zone_wmarks(); 8922 8923 return 0; 8924 } 8925 8926 #ifdef CONFIG_NUMA 8927 static void setup_min_unmapped_ratio(void) 8928 { 8929 pg_data_t *pgdat; 8930 struct zone *zone; 8931 8932 for_each_online_pgdat(pgdat) 8933 pgdat->min_unmapped_pages = 0; 8934 8935 for_each_zone(zone) 8936 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * 8937 sysctl_min_unmapped_ratio) / 100; 8938 } 8939 8940 8941 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 8942 void *buffer, size_t *length, loff_t *ppos) 8943 { 8944 int rc; 8945 8946 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8947 if (rc) 8948 return rc; 8949 8950 setup_min_unmapped_ratio(); 8951 8952 return 0; 8953 } 8954 8955 static void setup_min_slab_ratio(void) 8956 { 8957 pg_data_t *pgdat; 8958 struct zone *zone; 8959 8960 for_each_online_pgdat(pgdat) 8961 pgdat->min_slab_pages = 0; 8962 8963 for_each_zone(zone) 8964 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * 8965 sysctl_min_slab_ratio) / 100; 8966 } 8967 8968 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 8969 void *buffer, size_t *length, loff_t *ppos) 8970 { 8971 int rc; 8972 8973 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8974 if (rc) 8975 return rc; 8976 8977 setup_min_slab_ratio(); 8978 8979 return 0; 8980 } 8981 #endif 8982 8983 /* 8984 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 8985 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 8986 * whenever sysctl_lowmem_reserve_ratio changes. 8987 * 8988 * The reserve ratio obviously has absolutely no relation with the 8989 * minimum watermarks. The lowmem reserve ratio can only make sense 8990 * if in function of the boot time zone sizes. 8991 */ 8992 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 8993 void *buffer, size_t *length, loff_t *ppos) 8994 { 8995 int i; 8996 8997 proc_dointvec_minmax(table, write, buffer, length, ppos); 8998 8999 for (i = 0; i < MAX_NR_ZONES; i++) { 9000 if (sysctl_lowmem_reserve_ratio[i] < 1) 9001 sysctl_lowmem_reserve_ratio[i] = 0; 9002 } 9003 9004 setup_per_zone_lowmem_reserve(); 9005 return 0; 9006 } 9007 9008 /* 9009 * percpu_pagelist_high_fraction - changes the pcp->high for each zone on each 9010 * cpu. It is the fraction of total pages in each zone that a hot per cpu 9011 * pagelist can have before it gets flushed back to buddy allocator. 9012 */ 9013 int percpu_pagelist_high_fraction_sysctl_handler(struct ctl_table *table, 9014 int write, void *buffer, size_t *length, loff_t *ppos) 9015 { 9016 struct zone *zone; 9017 int old_percpu_pagelist_high_fraction; 9018 int ret; 9019 9020 mutex_lock(&pcp_batch_high_lock); 9021 old_percpu_pagelist_high_fraction = percpu_pagelist_high_fraction; 9022 9023 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 9024 if (!write || ret < 0) 9025 goto out; 9026 9027 /* Sanity checking to avoid pcp imbalance */ 9028 if (percpu_pagelist_high_fraction && 9029 percpu_pagelist_high_fraction < MIN_PERCPU_PAGELIST_HIGH_FRACTION) { 9030 percpu_pagelist_high_fraction = old_percpu_pagelist_high_fraction; 9031 ret = -EINVAL; 9032 goto out; 9033 } 9034 9035 /* No change? */ 9036 if (percpu_pagelist_high_fraction == old_percpu_pagelist_high_fraction) 9037 goto out; 9038 9039 for_each_populated_zone(zone) 9040 zone_set_pageset_high_and_batch(zone, 0); 9041 out: 9042 mutex_unlock(&pcp_batch_high_lock); 9043 return ret; 9044 } 9045 9046 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 9047 /* 9048 * Returns the number of pages that arch has reserved but 9049 * is not known to alloc_large_system_hash(). 9050 */ 9051 static unsigned long __init arch_reserved_kernel_pages(void) 9052 { 9053 return 0; 9054 } 9055 #endif 9056 9057 /* 9058 * Adaptive scale is meant to reduce sizes of hash tables on large memory 9059 * machines. As memory size is increased the scale is also increased but at 9060 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 9061 * quadruples the scale is increased by one, which means the size of hash table 9062 * only doubles, instead of quadrupling as well. 9063 * Because 32-bit systems cannot have large physical memory, where this scaling 9064 * makes sense, it is disabled on such platforms. 9065 */ 9066 #if __BITS_PER_LONG > 32 9067 #define ADAPT_SCALE_BASE (64ul << 30) 9068 #define ADAPT_SCALE_SHIFT 2 9069 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 9070 #endif 9071 9072 /* 9073 * allocate a large system hash table from bootmem 9074 * - it is assumed that the hash table must contain an exact power-of-2 9075 * quantity of entries 9076 * - limit is the number of hash buckets, not the total allocation size 9077 */ 9078 void *__init alloc_large_system_hash(const char *tablename, 9079 unsigned long bucketsize, 9080 unsigned long numentries, 9081 int scale, 9082 int flags, 9083 unsigned int *_hash_shift, 9084 unsigned int *_hash_mask, 9085 unsigned long low_limit, 9086 unsigned long high_limit) 9087 { 9088 unsigned long long max = high_limit; 9089 unsigned long log2qty, size; 9090 void *table; 9091 gfp_t gfp_flags; 9092 bool virt; 9093 bool huge; 9094 9095 /* allow the kernel cmdline to have a say */ 9096 if (!numentries) { 9097 /* round applicable memory size up to nearest megabyte */ 9098 numentries = nr_kernel_pages; 9099 numentries -= arch_reserved_kernel_pages(); 9100 9101 /* It isn't necessary when PAGE_SIZE >= 1MB */ 9102 if (PAGE_SIZE < SZ_1M) 9103 numentries = round_up(numentries, SZ_1M / PAGE_SIZE); 9104 9105 #if __BITS_PER_LONG > 32 9106 if (!high_limit) { 9107 unsigned long adapt; 9108 9109 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 9110 adapt <<= ADAPT_SCALE_SHIFT) 9111 scale++; 9112 } 9113 #endif 9114 9115 /* limit to 1 bucket per 2^scale bytes of low memory */ 9116 if (scale > PAGE_SHIFT) 9117 numentries >>= (scale - PAGE_SHIFT); 9118 else 9119 numentries <<= (PAGE_SHIFT - scale); 9120 9121 /* Make sure we've got at least a 0-order allocation.. */ 9122 if (unlikely(flags & HASH_SMALL)) { 9123 /* Makes no sense without HASH_EARLY */ 9124 WARN_ON(!(flags & HASH_EARLY)); 9125 if (!(numentries >> *_hash_shift)) { 9126 numentries = 1UL << *_hash_shift; 9127 BUG_ON(!numentries); 9128 } 9129 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 9130 numentries = PAGE_SIZE / bucketsize; 9131 } 9132 numentries = roundup_pow_of_two(numentries); 9133 9134 /* limit allocation size to 1/16 total memory by default */ 9135 if (max == 0) { 9136 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 9137 do_div(max, bucketsize); 9138 } 9139 max = min(max, 0x80000000ULL); 9140 9141 if (numentries < low_limit) 9142 numentries = low_limit; 9143 if (numentries > max) 9144 numentries = max; 9145 9146 log2qty = ilog2(numentries); 9147 9148 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 9149 do { 9150 virt = false; 9151 size = bucketsize << log2qty; 9152 if (flags & HASH_EARLY) { 9153 if (flags & HASH_ZERO) 9154 table = memblock_alloc(size, SMP_CACHE_BYTES); 9155 else 9156 table = memblock_alloc_raw(size, 9157 SMP_CACHE_BYTES); 9158 } else if (get_order(size) >= MAX_ORDER || hashdist) { 9159 table = vmalloc_huge(size, gfp_flags); 9160 virt = true; 9161 if (table) 9162 huge = is_vm_area_hugepages(table); 9163 } else { 9164 /* 9165 * If bucketsize is not a power-of-two, we may free 9166 * some pages at the end of hash table which 9167 * alloc_pages_exact() automatically does 9168 */ 9169 table = alloc_pages_exact(size, gfp_flags); 9170 kmemleak_alloc(table, size, 1, gfp_flags); 9171 } 9172 } while (!table && size > PAGE_SIZE && --log2qty); 9173 9174 if (!table) 9175 panic("Failed to allocate %s hash table\n", tablename); 9176 9177 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 9178 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, 9179 virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear"); 9180 9181 if (_hash_shift) 9182 *_hash_shift = log2qty; 9183 if (_hash_mask) 9184 *_hash_mask = (1 << log2qty) - 1; 9185 9186 return table; 9187 } 9188 9189 #ifdef CONFIG_CONTIG_ALLOC 9190 #if defined(CONFIG_DYNAMIC_DEBUG) || \ 9191 (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE)) 9192 /* Usage: See admin-guide/dynamic-debug-howto.rst */ 9193 static void alloc_contig_dump_pages(struct list_head *page_list) 9194 { 9195 DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, "migrate failure"); 9196 9197 if (DYNAMIC_DEBUG_BRANCH(descriptor)) { 9198 struct page *page; 9199 9200 dump_stack(); 9201 list_for_each_entry(page, page_list, lru) 9202 dump_page(page, "migration failure"); 9203 } 9204 } 9205 #else 9206 static inline void alloc_contig_dump_pages(struct list_head *page_list) 9207 { 9208 } 9209 #endif 9210 9211 /* [start, end) must belong to a single zone. */ 9212 int __alloc_contig_migrate_range(struct compact_control *cc, 9213 unsigned long start, unsigned long end) 9214 { 9215 /* This function is based on compact_zone() from compaction.c. */ 9216 unsigned int nr_reclaimed; 9217 unsigned long pfn = start; 9218 unsigned int tries = 0; 9219 int ret = 0; 9220 struct migration_target_control mtc = { 9221 .nid = zone_to_nid(cc->zone), 9222 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, 9223 }; 9224 9225 lru_cache_disable(); 9226 9227 while (pfn < end || !list_empty(&cc->migratepages)) { 9228 if (fatal_signal_pending(current)) { 9229 ret = -EINTR; 9230 break; 9231 } 9232 9233 if (list_empty(&cc->migratepages)) { 9234 cc->nr_migratepages = 0; 9235 ret = isolate_migratepages_range(cc, pfn, end); 9236 if (ret && ret != -EAGAIN) 9237 break; 9238 pfn = cc->migrate_pfn; 9239 tries = 0; 9240 } else if (++tries == 5) { 9241 ret = -EBUSY; 9242 break; 9243 } 9244 9245 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 9246 &cc->migratepages); 9247 cc->nr_migratepages -= nr_reclaimed; 9248 9249 ret = migrate_pages(&cc->migratepages, alloc_migration_target, 9250 NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE, NULL); 9251 9252 /* 9253 * On -ENOMEM, migrate_pages() bails out right away. It is pointless 9254 * to retry again over this error, so do the same here. 9255 */ 9256 if (ret == -ENOMEM) 9257 break; 9258 } 9259 9260 lru_cache_enable(); 9261 if (ret < 0) { 9262 if (!(cc->gfp_mask & __GFP_NOWARN) && ret == -EBUSY) 9263 alloc_contig_dump_pages(&cc->migratepages); 9264 putback_movable_pages(&cc->migratepages); 9265 return ret; 9266 } 9267 return 0; 9268 } 9269 9270 /** 9271 * alloc_contig_range() -- tries to allocate given range of pages 9272 * @start: start PFN to allocate 9273 * @end: one-past-the-last PFN to allocate 9274 * @migratetype: migratetype of the underlying pageblocks (either 9275 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 9276 * in range must have the same migratetype and it must 9277 * be either of the two. 9278 * @gfp_mask: GFP mask to use during compaction 9279 * 9280 * The PFN range does not have to be pageblock aligned. The PFN range must 9281 * belong to a single zone. 9282 * 9283 * The first thing this routine does is attempt to MIGRATE_ISOLATE all 9284 * pageblocks in the range. Once isolated, the pageblocks should not 9285 * be modified by others. 9286 * 9287 * Return: zero on success or negative error code. On success all 9288 * pages which PFN is in [start, end) are allocated for the caller and 9289 * need to be freed with free_contig_range(). 9290 */ 9291 int alloc_contig_range(unsigned long start, unsigned long end, 9292 unsigned migratetype, gfp_t gfp_mask) 9293 { 9294 unsigned long outer_start, outer_end; 9295 int order; 9296 int ret = 0; 9297 9298 struct compact_control cc = { 9299 .nr_migratepages = 0, 9300 .order = -1, 9301 .zone = page_zone(pfn_to_page(start)), 9302 .mode = MIGRATE_SYNC, 9303 .ignore_skip_hint = true, 9304 .no_set_skip_hint = true, 9305 .gfp_mask = current_gfp_context(gfp_mask), 9306 .alloc_contig = true, 9307 }; 9308 INIT_LIST_HEAD(&cc.migratepages); 9309 9310 /* 9311 * What we do here is we mark all pageblocks in range as 9312 * MIGRATE_ISOLATE. Because pageblock and max order pages may 9313 * have different sizes, and due to the way page allocator 9314 * work, start_isolate_page_range() has special handlings for this. 9315 * 9316 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 9317 * migrate the pages from an unaligned range (ie. pages that 9318 * we are interested in). This will put all the pages in 9319 * range back to page allocator as MIGRATE_ISOLATE. 9320 * 9321 * When this is done, we take the pages in range from page 9322 * allocator removing them from the buddy system. This way 9323 * page allocator will never consider using them. 9324 * 9325 * This lets us mark the pageblocks back as 9326 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 9327 * aligned range but not in the unaligned, original range are 9328 * put back to page allocator so that buddy can use them. 9329 */ 9330 9331 ret = start_isolate_page_range(start, end, migratetype, 0, gfp_mask); 9332 if (ret) 9333 goto done; 9334 9335 drain_all_pages(cc.zone); 9336 9337 /* 9338 * In case of -EBUSY, we'd like to know which page causes problem. 9339 * So, just fall through. test_pages_isolated() has a tracepoint 9340 * which will report the busy page. 9341 * 9342 * It is possible that busy pages could become available before 9343 * the call to test_pages_isolated, and the range will actually be 9344 * allocated. So, if we fall through be sure to clear ret so that 9345 * -EBUSY is not accidentally used or returned to caller. 9346 */ 9347 ret = __alloc_contig_migrate_range(&cc, start, end); 9348 if (ret && ret != -EBUSY) 9349 goto done; 9350 ret = 0; 9351 9352 /* 9353 * Pages from [start, end) are within a pageblock_nr_pages 9354 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 9355 * more, all pages in [start, end) are free in page allocator. 9356 * What we are going to do is to allocate all pages from 9357 * [start, end) (that is remove them from page allocator). 9358 * 9359 * The only problem is that pages at the beginning and at the 9360 * end of interesting range may be not aligned with pages that 9361 * page allocator holds, ie. they can be part of higher order 9362 * pages. Because of this, we reserve the bigger range and 9363 * once this is done free the pages we are not interested in. 9364 * 9365 * We don't have to hold zone->lock here because the pages are 9366 * isolated thus they won't get removed from buddy. 9367 */ 9368 9369 order = 0; 9370 outer_start = start; 9371 while (!PageBuddy(pfn_to_page(outer_start))) { 9372 if (++order >= MAX_ORDER) { 9373 outer_start = start; 9374 break; 9375 } 9376 outer_start &= ~0UL << order; 9377 } 9378 9379 if (outer_start != start) { 9380 order = buddy_order(pfn_to_page(outer_start)); 9381 9382 /* 9383 * outer_start page could be small order buddy page and 9384 * it doesn't include start page. Adjust outer_start 9385 * in this case to report failed page properly 9386 * on tracepoint in test_pages_isolated() 9387 */ 9388 if (outer_start + (1UL << order) <= start) 9389 outer_start = start; 9390 } 9391 9392 /* Make sure the range is really isolated. */ 9393 if (test_pages_isolated(outer_start, end, 0)) { 9394 ret = -EBUSY; 9395 goto done; 9396 } 9397 9398 /* Grab isolated pages from freelists. */ 9399 outer_end = isolate_freepages_range(&cc, outer_start, end); 9400 if (!outer_end) { 9401 ret = -EBUSY; 9402 goto done; 9403 } 9404 9405 /* Free head and tail (if any) */ 9406 if (start != outer_start) 9407 free_contig_range(outer_start, start - outer_start); 9408 if (end != outer_end) 9409 free_contig_range(end, outer_end - end); 9410 9411 done: 9412 undo_isolate_page_range(start, end, migratetype); 9413 return ret; 9414 } 9415 EXPORT_SYMBOL(alloc_contig_range); 9416 9417 static int __alloc_contig_pages(unsigned long start_pfn, 9418 unsigned long nr_pages, gfp_t gfp_mask) 9419 { 9420 unsigned long end_pfn = start_pfn + nr_pages; 9421 9422 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE, 9423 gfp_mask); 9424 } 9425 9426 static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, 9427 unsigned long nr_pages) 9428 { 9429 unsigned long i, end_pfn = start_pfn + nr_pages; 9430 struct page *page; 9431 9432 for (i = start_pfn; i < end_pfn; i++) { 9433 page = pfn_to_online_page(i); 9434 if (!page) 9435 return false; 9436 9437 if (page_zone(page) != z) 9438 return false; 9439 9440 if (PageReserved(page)) 9441 return false; 9442 } 9443 return true; 9444 } 9445 9446 static bool zone_spans_last_pfn(const struct zone *zone, 9447 unsigned long start_pfn, unsigned long nr_pages) 9448 { 9449 unsigned long last_pfn = start_pfn + nr_pages - 1; 9450 9451 return zone_spans_pfn(zone, last_pfn); 9452 } 9453 9454 /** 9455 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages 9456 * @nr_pages: Number of contiguous pages to allocate 9457 * @gfp_mask: GFP mask to limit search and used during compaction 9458 * @nid: Target node 9459 * @nodemask: Mask for other possible nodes 9460 * 9461 * This routine is a wrapper around alloc_contig_range(). It scans over zones 9462 * on an applicable zonelist to find a contiguous pfn range which can then be 9463 * tried for allocation with alloc_contig_range(). This routine is intended 9464 * for allocation requests which can not be fulfilled with the buddy allocator. 9465 * 9466 * The allocated memory is always aligned to a page boundary. If nr_pages is a 9467 * power of two, then allocated range is also guaranteed to be aligned to same 9468 * nr_pages (e.g. 1GB request would be aligned to 1GB). 9469 * 9470 * Allocated pages can be freed with free_contig_range() or by manually calling 9471 * __free_page() on each allocated page. 9472 * 9473 * Return: pointer to contiguous pages on success, or NULL if not successful. 9474 */ 9475 struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, 9476 int nid, nodemask_t *nodemask) 9477 { 9478 unsigned long ret, pfn, flags; 9479 struct zonelist *zonelist; 9480 struct zone *zone; 9481 struct zoneref *z; 9482 9483 zonelist = node_zonelist(nid, gfp_mask); 9484 for_each_zone_zonelist_nodemask(zone, z, zonelist, 9485 gfp_zone(gfp_mask), nodemask) { 9486 spin_lock_irqsave(&zone->lock, flags); 9487 9488 pfn = ALIGN(zone->zone_start_pfn, nr_pages); 9489 while (zone_spans_last_pfn(zone, pfn, nr_pages)) { 9490 if (pfn_range_valid_contig(zone, pfn, nr_pages)) { 9491 /* 9492 * We release the zone lock here because 9493 * alloc_contig_range() will also lock the zone 9494 * at some point. If there's an allocation 9495 * spinning on this lock, it may win the race 9496 * and cause alloc_contig_range() to fail... 9497 */ 9498 spin_unlock_irqrestore(&zone->lock, flags); 9499 ret = __alloc_contig_pages(pfn, nr_pages, 9500 gfp_mask); 9501 if (!ret) 9502 return pfn_to_page(pfn); 9503 spin_lock_irqsave(&zone->lock, flags); 9504 } 9505 pfn += nr_pages; 9506 } 9507 spin_unlock_irqrestore(&zone->lock, flags); 9508 } 9509 return NULL; 9510 } 9511 #endif /* CONFIG_CONTIG_ALLOC */ 9512 9513 void free_contig_range(unsigned long pfn, unsigned long nr_pages) 9514 { 9515 unsigned long count = 0; 9516 9517 for (; nr_pages--; pfn++) { 9518 struct page *page = pfn_to_page(pfn); 9519 9520 count += page_count(page) != 1; 9521 __free_page(page); 9522 } 9523 WARN(count != 0, "%lu pages are still in use!\n", count); 9524 } 9525 EXPORT_SYMBOL(free_contig_range); 9526 9527 /* 9528 * Effectively disable pcplists for the zone by setting the high limit to 0 9529 * and draining all cpus. A concurrent page freeing on another CPU that's about 9530 * to put the page on pcplist will either finish before the drain and the page 9531 * will be drained, or observe the new high limit and skip the pcplist. 9532 * 9533 * Must be paired with a call to zone_pcp_enable(). 9534 */ 9535 void zone_pcp_disable(struct zone *zone) 9536 { 9537 mutex_lock(&pcp_batch_high_lock); 9538 __zone_set_pageset_high_and_batch(zone, 0, 1); 9539 __drain_all_pages(zone, true); 9540 } 9541 9542 void zone_pcp_enable(struct zone *zone) 9543 { 9544 __zone_set_pageset_high_and_batch(zone, zone->pageset_high, zone->pageset_batch); 9545 mutex_unlock(&pcp_batch_high_lock); 9546 } 9547 9548 void zone_pcp_reset(struct zone *zone) 9549 { 9550 int cpu; 9551 struct per_cpu_zonestat *pzstats; 9552 9553 if (zone->per_cpu_pageset != &boot_pageset) { 9554 for_each_online_cpu(cpu) { 9555 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 9556 drain_zonestat(zone, pzstats); 9557 } 9558 free_percpu(zone->per_cpu_pageset); 9559 zone->per_cpu_pageset = &boot_pageset; 9560 if (zone->per_cpu_zonestats != &boot_zonestats) { 9561 free_percpu(zone->per_cpu_zonestats); 9562 zone->per_cpu_zonestats = &boot_zonestats; 9563 } 9564 } 9565 } 9566 9567 #ifdef CONFIG_MEMORY_HOTREMOVE 9568 /* 9569 * All pages in the range must be in a single zone, must not contain holes, 9570 * must span full sections, and must be isolated before calling this function. 9571 */ 9572 void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 9573 { 9574 unsigned long pfn = start_pfn; 9575 struct page *page; 9576 struct zone *zone; 9577 unsigned int order; 9578 unsigned long flags; 9579 9580 offline_mem_sections(pfn, end_pfn); 9581 zone = page_zone(pfn_to_page(pfn)); 9582 spin_lock_irqsave(&zone->lock, flags); 9583 while (pfn < end_pfn) { 9584 page = pfn_to_page(pfn); 9585 /* 9586 * The HWPoisoned page may be not in buddy system, and 9587 * page_count() is not 0. 9588 */ 9589 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 9590 pfn++; 9591 continue; 9592 } 9593 /* 9594 * At this point all remaining PageOffline() pages have a 9595 * reference count of 0 and can simply be skipped. 9596 */ 9597 if (PageOffline(page)) { 9598 BUG_ON(page_count(page)); 9599 BUG_ON(PageBuddy(page)); 9600 pfn++; 9601 continue; 9602 } 9603 9604 BUG_ON(page_count(page)); 9605 BUG_ON(!PageBuddy(page)); 9606 order = buddy_order(page); 9607 del_page_from_free_list(page, zone, order); 9608 pfn += (1 << order); 9609 } 9610 spin_unlock_irqrestore(&zone->lock, flags); 9611 } 9612 #endif 9613 9614 /* 9615 * This function returns a stable result only if called under zone lock. 9616 */ 9617 bool is_free_buddy_page(struct page *page) 9618 { 9619 unsigned long pfn = page_to_pfn(page); 9620 unsigned int order; 9621 9622 for (order = 0; order < MAX_ORDER; order++) { 9623 struct page *page_head = page - (pfn & ((1 << order) - 1)); 9624 9625 if (PageBuddy(page_head) && 9626 buddy_order_unsafe(page_head) >= order) 9627 break; 9628 } 9629 9630 return order < MAX_ORDER; 9631 } 9632 EXPORT_SYMBOL(is_free_buddy_page); 9633 9634 #ifdef CONFIG_MEMORY_FAILURE 9635 /* 9636 * Break down a higher-order page in sub-pages, and keep our target out of 9637 * buddy allocator. 9638 */ 9639 static void break_down_buddy_pages(struct zone *zone, struct page *page, 9640 struct page *target, int low, int high, 9641 int migratetype) 9642 { 9643 unsigned long size = 1 << high; 9644 struct page *current_buddy, *next_page; 9645 9646 while (high > low) { 9647 high--; 9648 size >>= 1; 9649 9650 if (target >= &page[size]) { 9651 next_page = page + size; 9652 current_buddy = page; 9653 } else { 9654 next_page = page; 9655 current_buddy = page + size; 9656 } 9657 9658 if (set_page_guard(zone, current_buddy, high, migratetype)) 9659 continue; 9660 9661 if (current_buddy != target) { 9662 add_to_free_list(current_buddy, zone, high, migratetype); 9663 set_buddy_order(current_buddy, high); 9664 page = next_page; 9665 } 9666 } 9667 } 9668 9669 /* 9670 * Take a page that will be marked as poisoned off the buddy allocator. 9671 */ 9672 bool take_page_off_buddy(struct page *page) 9673 { 9674 struct zone *zone = page_zone(page); 9675 unsigned long pfn = page_to_pfn(page); 9676 unsigned long flags; 9677 unsigned int order; 9678 bool ret = false; 9679 9680 spin_lock_irqsave(&zone->lock, flags); 9681 for (order = 0; order < MAX_ORDER; order++) { 9682 struct page *page_head = page - (pfn & ((1 << order) - 1)); 9683 int page_order = buddy_order(page_head); 9684 9685 if (PageBuddy(page_head) && page_order >= order) { 9686 unsigned long pfn_head = page_to_pfn(page_head); 9687 int migratetype = get_pfnblock_migratetype(page_head, 9688 pfn_head); 9689 9690 del_page_from_free_list(page_head, zone, page_order); 9691 break_down_buddy_pages(zone, page_head, page, 0, 9692 page_order, migratetype); 9693 SetPageHWPoisonTakenOff(page); 9694 if (!is_migrate_isolate(migratetype)) 9695 __mod_zone_freepage_state(zone, -1, migratetype); 9696 ret = true; 9697 break; 9698 } 9699 if (page_count(page_head) > 0) 9700 break; 9701 } 9702 spin_unlock_irqrestore(&zone->lock, flags); 9703 return ret; 9704 } 9705 9706 /* 9707 * Cancel takeoff done by take_page_off_buddy(). 9708 */ 9709 bool put_page_back_buddy(struct page *page) 9710 { 9711 struct zone *zone = page_zone(page); 9712 unsigned long pfn = page_to_pfn(page); 9713 unsigned long flags; 9714 int migratetype = get_pfnblock_migratetype(page, pfn); 9715 bool ret = false; 9716 9717 spin_lock_irqsave(&zone->lock, flags); 9718 if (put_page_testzero(page)) { 9719 ClearPageHWPoisonTakenOff(page); 9720 __free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE); 9721 if (TestClearPageHWPoison(page)) { 9722 ret = true; 9723 } 9724 } 9725 spin_unlock_irqrestore(&zone->lock, flags); 9726 9727 return ret; 9728 } 9729 #endif 9730 9731 #ifdef CONFIG_ZONE_DMA 9732 bool has_managed_dma(void) 9733 { 9734 struct pglist_data *pgdat; 9735 9736 for_each_online_pgdat(pgdat) { 9737 struct zone *zone = &pgdat->node_zones[ZONE_DMA]; 9738 9739 if (managed_zone(zone)) 9740 return true; 9741 } 9742 return false; 9743 } 9744 #endif /* CONFIG_ZONE_DMA */ 9745