1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/page_alloc.c 4 * 5 * Manages the free list, the system allocates free pages here. 6 * Note that kmalloc() lives in slab.c 7 * 8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 9 * Swap reorganised 29.12.95, Stephen Tweedie 10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 16 */ 17 18 #include <linux/stddef.h> 19 #include <linux/mm.h> 20 #include <linux/highmem.h> 21 #include <linux/swap.h> 22 #include <linux/swapops.h> 23 #include <linux/interrupt.h> 24 #include <linux/pagemap.h> 25 #include <linux/jiffies.h> 26 #include <linux/memblock.h> 27 #include <linux/compiler.h> 28 #include <linux/kernel.h> 29 #include <linux/kasan.h> 30 #include <linux/module.h> 31 #include <linux/suspend.h> 32 #include <linux/pagevec.h> 33 #include <linux/blkdev.h> 34 #include <linux/slab.h> 35 #include <linux/ratelimit.h> 36 #include <linux/oom.h> 37 #include <linux/topology.h> 38 #include <linux/sysctl.h> 39 #include <linux/cpu.h> 40 #include <linux/cpuset.h> 41 #include <linux/memory_hotplug.h> 42 #include <linux/nodemask.h> 43 #include <linux/vmalloc.h> 44 #include <linux/vmstat.h> 45 #include <linux/mempolicy.h> 46 #include <linux/memremap.h> 47 #include <linux/stop_machine.h> 48 #include <linux/random.h> 49 #include <linux/sort.h> 50 #include <linux/pfn.h> 51 #include <linux/backing-dev.h> 52 #include <linux/fault-inject.h> 53 #include <linux/page-isolation.h> 54 #include <linux/debugobjects.h> 55 #include <linux/kmemleak.h> 56 #include <linux/compaction.h> 57 #include <trace/events/kmem.h> 58 #include <trace/events/oom.h> 59 #include <linux/prefetch.h> 60 #include <linux/mm_inline.h> 61 #include <linux/mmu_notifier.h> 62 #include <linux/migrate.h> 63 #include <linux/hugetlb.h> 64 #include <linux/sched/rt.h> 65 #include <linux/sched/mm.h> 66 #include <linux/page_owner.h> 67 #include <linux/page_table_check.h> 68 #include <linux/kthread.h> 69 #include <linux/memcontrol.h> 70 #include <linux/ftrace.h> 71 #include <linux/lockdep.h> 72 #include <linux/nmi.h> 73 #include <linux/psi.h> 74 #include <linux/padata.h> 75 #include <linux/khugepaged.h> 76 #include <linux/buffer_head.h> 77 #include <linux/delayacct.h> 78 #include <asm/sections.h> 79 #include <asm/tlbflush.h> 80 #include <asm/div64.h> 81 #include "internal.h" 82 #include "shuffle.h" 83 #include "page_reporting.h" 84 #include "swap.h" 85 86 /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */ 87 typedef int __bitwise fpi_t; 88 89 /* No special request */ 90 #define FPI_NONE ((__force fpi_t)0) 91 92 /* 93 * Skip free page reporting notification for the (possibly merged) page. 94 * This does not hinder free page reporting from grabbing the page, 95 * reporting it and marking it "reported" - it only skips notifying 96 * the free page reporting infrastructure about a newly freed page. For 97 * example, used when temporarily pulling a page from a freelist and 98 * putting it back unmodified. 99 */ 100 #define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0)) 101 102 /* 103 * Place the (possibly merged) page to the tail of the freelist. Will ignore 104 * page shuffling (relevant code - e.g., memory onlining - is expected to 105 * shuffle the whole zone). 106 * 107 * Note: No code should rely on this flag for correctness - it's purely 108 * to allow for optimizations when handing back either fresh pages 109 * (memory onlining) or untouched pages (page isolation, free page 110 * reporting). 111 */ 112 #define FPI_TO_TAIL ((__force fpi_t)BIT(1)) 113 114 /* 115 * Don't poison memory with KASAN (only for the tag-based modes). 116 * During boot, all non-reserved memblock memory is exposed to page_alloc. 117 * Poisoning all that memory lengthens boot time, especially on systems with 118 * large amount of RAM. This flag is used to skip that poisoning. 119 * This is only done for the tag-based KASAN modes, as those are able to 120 * detect memory corruptions with the memory tags assigned by default. 121 * All memory allocated normally after boot gets poisoned as usual. 122 */ 123 #define FPI_SKIP_KASAN_POISON ((__force fpi_t)BIT(2)) 124 125 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 126 static DEFINE_MUTEX(pcp_batch_high_lock); 127 #define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8) 128 129 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT) 130 /* 131 * On SMP, spin_trylock is sufficient protection. 132 * On PREEMPT_RT, spin_trylock is equivalent on both SMP and UP. 133 */ 134 #define pcp_trylock_prepare(flags) do { } while (0) 135 #define pcp_trylock_finish(flag) do { } while (0) 136 #else 137 138 /* UP spin_trylock always succeeds so disable IRQs to prevent re-entrancy. */ 139 #define pcp_trylock_prepare(flags) local_irq_save(flags) 140 #define pcp_trylock_finish(flags) local_irq_restore(flags) 141 #endif 142 143 /* 144 * Locking a pcp requires a PCP lookup followed by a spinlock. To avoid 145 * a migration causing the wrong PCP to be locked and remote memory being 146 * potentially allocated, pin the task to the CPU for the lookup+lock. 147 * preempt_disable is used on !RT because it is faster than migrate_disable. 148 * migrate_disable is used on RT because otherwise RT spinlock usage is 149 * interfered with and a high priority task cannot preempt the allocator. 150 */ 151 #ifndef CONFIG_PREEMPT_RT 152 #define pcpu_task_pin() preempt_disable() 153 #define pcpu_task_unpin() preempt_enable() 154 #else 155 #define pcpu_task_pin() migrate_disable() 156 #define pcpu_task_unpin() migrate_enable() 157 #endif 158 159 /* 160 * Generic helper to lookup and a per-cpu variable with an embedded spinlock. 161 * Return value should be used with equivalent unlock helper. 162 */ 163 #define pcpu_spin_lock(type, member, ptr) \ 164 ({ \ 165 type *_ret; \ 166 pcpu_task_pin(); \ 167 _ret = this_cpu_ptr(ptr); \ 168 spin_lock(&_ret->member); \ 169 _ret; \ 170 }) 171 172 #define pcpu_spin_lock_irqsave(type, member, ptr, flags) \ 173 ({ \ 174 type *_ret; \ 175 pcpu_task_pin(); \ 176 _ret = this_cpu_ptr(ptr); \ 177 spin_lock_irqsave(&_ret->member, flags); \ 178 _ret; \ 179 }) 180 181 #define pcpu_spin_trylock_irqsave(type, member, ptr, flags) \ 182 ({ \ 183 type *_ret; \ 184 pcpu_task_pin(); \ 185 _ret = this_cpu_ptr(ptr); \ 186 if (!spin_trylock_irqsave(&_ret->member, flags)) { \ 187 pcpu_task_unpin(); \ 188 _ret = NULL; \ 189 } \ 190 _ret; \ 191 }) 192 193 #define pcpu_spin_unlock(member, ptr) \ 194 ({ \ 195 spin_unlock(&ptr->member); \ 196 pcpu_task_unpin(); \ 197 }) 198 199 #define pcpu_spin_unlock_irqrestore(member, ptr, flags) \ 200 ({ \ 201 spin_unlock_irqrestore(&ptr->member, flags); \ 202 pcpu_task_unpin(); \ 203 }) 204 205 /* struct per_cpu_pages specific helpers. */ 206 #define pcp_spin_lock(ptr) \ 207 pcpu_spin_lock(struct per_cpu_pages, lock, ptr) 208 209 #define pcp_spin_lock_irqsave(ptr, flags) \ 210 pcpu_spin_lock_irqsave(struct per_cpu_pages, lock, ptr, flags) 211 212 #define pcp_spin_trylock_irqsave(ptr, flags) \ 213 pcpu_spin_trylock_irqsave(struct per_cpu_pages, lock, ptr, flags) 214 215 #define pcp_spin_unlock(ptr) \ 216 pcpu_spin_unlock(lock, ptr) 217 218 #define pcp_spin_unlock_irqrestore(ptr, flags) \ 219 pcpu_spin_unlock_irqrestore(lock, ptr, flags) 220 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 221 DEFINE_PER_CPU(int, numa_node); 222 EXPORT_PER_CPU_SYMBOL(numa_node); 223 #endif 224 225 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); 226 227 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 228 /* 229 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 230 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 231 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 232 * defined in <linux/topology.h>. 233 */ 234 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 235 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 236 #endif 237 238 static DEFINE_MUTEX(pcpu_drain_mutex); 239 240 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 241 volatile unsigned long latent_entropy __latent_entropy; 242 EXPORT_SYMBOL(latent_entropy); 243 #endif 244 245 /* 246 * Array of node states. 247 */ 248 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 249 [N_POSSIBLE] = NODE_MASK_ALL, 250 [N_ONLINE] = { { [0] = 1UL } }, 251 #ifndef CONFIG_NUMA 252 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 253 #ifdef CONFIG_HIGHMEM 254 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 255 #endif 256 [N_MEMORY] = { { [0] = 1UL } }, 257 [N_CPU] = { { [0] = 1UL } }, 258 #endif /* NUMA */ 259 }; 260 EXPORT_SYMBOL(node_states); 261 262 atomic_long_t _totalram_pages __read_mostly; 263 EXPORT_SYMBOL(_totalram_pages); 264 unsigned long totalreserve_pages __read_mostly; 265 unsigned long totalcma_pages __read_mostly; 266 267 int percpu_pagelist_high_fraction; 268 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 269 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); 270 EXPORT_SYMBOL(init_on_alloc); 271 272 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); 273 EXPORT_SYMBOL(init_on_free); 274 275 static bool _init_on_alloc_enabled_early __read_mostly 276 = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON); 277 static int __init early_init_on_alloc(char *buf) 278 { 279 280 return kstrtobool(buf, &_init_on_alloc_enabled_early); 281 } 282 early_param("init_on_alloc", early_init_on_alloc); 283 284 static bool _init_on_free_enabled_early __read_mostly 285 = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON); 286 static int __init early_init_on_free(char *buf) 287 { 288 return kstrtobool(buf, &_init_on_free_enabled_early); 289 } 290 early_param("init_on_free", early_init_on_free); 291 292 /* 293 * A cached value of the page's pageblock's migratetype, used when the page is 294 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 295 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 296 * Also the migratetype set in the page does not necessarily match the pcplist 297 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 298 * other index - this ensures that it will be put on the correct CMA freelist. 299 */ 300 static inline int get_pcppage_migratetype(struct page *page) 301 { 302 return page->index; 303 } 304 305 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 306 { 307 page->index = migratetype; 308 } 309 310 #ifdef CONFIG_PM_SLEEP 311 /* 312 * The following functions are used by the suspend/hibernate code to temporarily 313 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 314 * while devices are suspended. To avoid races with the suspend/hibernate code, 315 * they should always be called with system_transition_mutex held 316 * (gfp_allowed_mask also should only be modified with system_transition_mutex 317 * held, unless the suspend/hibernate code is guaranteed not to run in parallel 318 * with that modification). 319 */ 320 321 static gfp_t saved_gfp_mask; 322 323 void pm_restore_gfp_mask(void) 324 { 325 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 326 if (saved_gfp_mask) { 327 gfp_allowed_mask = saved_gfp_mask; 328 saved_gfp_mask = 0; 329 } 330 } 331 332 void pm_restrict_gfp_mask(void) 333 { 334 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 335 WARN_ON(saved_gfp_mask); 336 saved_gfp_mask = gfp_allowed_mask; 337 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 338 } 339 340 bool pm_suspended_storage(void) 341 { 342 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 343 return false; 344 return true; 345 } 346 #endif /* CONFIG_PM_SLEEP */ 347 348 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 349 unsigned int pageblock_order __read_mostly; 350 #endif 351 352 static void __free_pages_ok(struct page *page, unsigned int order, 353 fpi_t fpi_flags); 354 355 /* 356 * results with 256, 32 in the lowmem_reserve sysctl: 357 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 358 * 1G machine -> (16M dma, 784M normal, 224M high) 359 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 360 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 361 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 362 * 363 * TBD: should special case ZONE_DMA32 machines here - in those we normally 364 * don't need any ZONE_NORMAL reservation 365 */ 366 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { 367 #ifdef CONFIG_ZONE_DMA 368 [ZONE_DMA] = 256, 369 #endif 370 #ifdef CONFIG_ZONE_DMA32 371 [ZONE_DMA32] = 256, 372 #endif 373 [ZONE_NORMAL] = 32, 374 #ifdef CONFIG_HIGHMEM 375 [ZONE_HIGHMEM] = 0, 376 #endif 377 [ZONE_MOVABLE] = 0, 378 }; 379 380 static char * const zone_names[MAX_NR_ZONES] = { 381 #ifdef CONFIG_ZONE_DMA 382 "DMA", 383 #endif 384 #ifdef CONFIG_ZONE_DMA32 385 "DMA32", 386 #endif 387 "Normal", 388 #ifdef CONFIG_HIGHMEM 389 "HighMem", 390 #endif 391 "Movable", 392 #ifdef CONFIG_ZONE_DEVICE 393 "Device", 394 #endif 395 }; 396 397 const char * const migratetype_names[MIGRATE_TYPES] = { 398 "Unmovable", 399 "Movable", 400 "Reclaimable", 401 "HighAtomic", 402 #ifdef CONFIG_CMA 403 "CMA", 404 #endif 405 #ifdef CONFIG_MEMORY_ISOLATION 406 "Isolate", 407 #endif 408 }; 409 410 compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = { 411 [NULL_COMPOUND_DTOR] = NULL, 412 [COMPOUND_PAGE_DTOR] = free_compound_page, 413 #ifdef CONFIG_HUGETLB_PAGE 414 [HUGETLB_PAGE_DTOR] = free_huge_page, 415 #endif 416 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 417 [TRANSHUGE_PAGE_DTOR] = free_transhuge_page, 418 #endif 419 }; 420 421 int min_free_kbytes = 1024; 422 int user_min_free_kbytes = -1; 423 int watermark_boost_factor __read_mostly = 15000; 424 int watermark_scale_factor = 10; 425 426 static unsigned long nr_kernel_pages __initdata; 427 static unsigned long nr_all_pages __initdata; 428 static unsigned long dma_reserve __initdata; 429 430 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 431 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 432 static unsigned long required_kernelcore __initdata; 433 static unsigned long required_kernelcore_percent __initdata; 434 static unsigned long required_movablecore __initdata; 435 static unsigned long required_movablecore_percent __initdata; 436 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 437 bool mirrored_kernelcore __initdata_memblock; 438 439 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 440 int movable_zone; 441 EXPORT_SYMBOL(movable_zone); 442 443 #if MAX_NUMNODES > 1 444 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; 445 unsigned int nr_online_nodes __read_mostly = 1; 446 EXPORT_SYMBOL(nr_node_ids); 447 EXPORT_SYMBOL(nr_online_nodes); 448 #endif 449 450 int page_group_by_mobility_disabled __read_mostly; 451 452 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 453 /* 454 * During boot we initialize deferred pages on-demand, as needed, but once 455 * page_alloc_init_late() has finished, the deferred pages are all initialized, 456 * and we can permanently disable that path. 457 */ 458 static DEFINE_STATIC_KEY_TRUE(deferred_pages); 459 460 static inline bool deferred_pages_enabled(void) 461 { 462 return static_branch_unlikely(&deferred_pages); 463 } 464 465 /* Returns true if the struct page for the pfn is uninitialised */ 466 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 467 { 468 int nid = early_pfn_to_nid(pfn); 469 470 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 471 return true; 472 473 return false; 474 } 475 476 /* 477 * Returns true when the remaining initialisation should be deferred until 478 * later in the boot cycle when it can be parallelised. 479 */ 480 static bool __meminit 481 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 482 { 483 static unsigned long prev_end_pfn, nr_initialised; 484 485 if (early_page_ext_enabled()) 486 return false; 487 /* 488 * prev_end_pfn static that contains the end of previous zone 489 * No need to protect because called very early in boot before smp_init. 490 */ 491 if (prev_end_pfn != end_pfn) { 492 prev_end_pfn = end_pfn; 493 nr_initialised = 0; 494 } 495 496 /* Always populate low zones for address-constrained allocations */ 497 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 498 return false; 499 500 if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) 501 return true; 502 /* 503 * We start only with one section of pages, more pages are added as 504 * needed until the rest of deferred pages are initialized. 505 */ 506 nr_initialised++; 507 if ((nr_initialised > PAGES_PER_SECTION) && 508 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 509 NODE_DATA(nid)->first_deferred_pfn = pfn; 510 return true; 511 } 512 return false; 513 } 514 #else 515 static inline bool deferred_pages_enabled(void) 516 { 517 return false; 518 } 519 520 static inline bool early_page_uninitialised(unsigned long pfn) 521 { 522 return false; 523 } 524 525 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 526 { 527 return false; 528 } 529 #endif 530 531 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 532 static inline unsigned long *get_pageblock_bitmap(const struct page *page, 533 unsigned long pfn) 534 { 535 #ifdef CONFIG_SPARSEMEM 536 return section_to_usemap(__pfn_to_section(pfn)); 537 #else 538 return page_zone(page)->pageblock_flags; 539 #endif /* CONFIG_SPARSEMEM */ 540 } 541 542 static inline int pfn_to_bitidx(const struct page *page, unsigned long pfn) 543 { 544 #ifdef CONFIG_SPARSEMEM 545 pfn &= (PAGES_PER_SECTION-1); 546 #else 547 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 548 #endif /* CONFIG_SPARSEMEM */ 549 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 550 } 551 552 static __always_inline 553 unsigned long __get_pfnblock_flags_mask(const struct page *page, 554 unsigned long pfn, 555 unsigned long mask) 556 { 557 unsigned long *bitmap; 558 unsigned long bitidx, word_bitidx; 559 unsigned long word; 560 561 bitmap = get_pageblock_bitmap(page, pfn); 562 bitidx = pfn_to_bitidx(page, pfn); 563 word_bitidx = bitidx / BITS_PER_LONG; 564 bitidx &= (BITS_PER_LONG-1); 565 /* 566 * This races, without locks, with set_pfnblock_flags_mask(). Ensure 567 * a consistent read of the memory array, so that results, even though 568 * racy, are not corrupted. 569 */ 570 word = READ_ONCE(bitmap[word_bitidx]); 571 return (word >> bitidx) & mask; 572 } 573 574 /** 575 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 576 * @page: The page within the block of interest 577 * @pfn: The target page frame number 578 * @mask: mask of bits that the caller is interested in 579 * 580 * Return: pageblock_bits flags 581 */ 582 unsigned long get_pfnblock_flags_mask(const struct page *page, 583 unsigned long pfn, unsigned long mask) 584 { 585 return __get_pfnblock_flags_mask(page, pfn, mask); 586 } 587 588 static __always_inline int get_pfnblock_migratetype(const struct page *page, 589 unsigned long pfn) 590 { 591 return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK); 592 } 593 594 /** 595 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 596 * @page: The page within the block of interest 597 * @flags: The flags to set 598 * @pfn: The target page frame number 599 * @mask: mask of bits that the caller is interested in 600 */ 601 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 602 unsigned long pfn, 603 unsigned long mask) 604 { 605 unsigned long *bitmap; 606 unsigned long bitidx, word_bitidx; 607 unsigned long word; 608 609 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 610 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits)); 611 612 bitmap = get_pageblock_bitmap(page, pfn); 613 bitidx = pfn_to_bitidx(page, pfn); 614 word_bitidx = bitidx / BITS_PER_LONG; 615 bitidx &= (BITS_PER_LONG-1); 616 617 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 618 619 mask <<= bitidx; 620 flags <<= bitidx; 621 622 word = READ_ONCE(bitmap[word_bitidx]); 623 do { 624 } while (!try_cmpxchg(&bitmap[word_bitidx], &word, (word & ~mask) | flags)); 625 } 626 627 void set_pageblock_migratetype(struct page *page, int migratetype) 628 { 629 if (unlikely(page_group_by_mobility_disabled && 630 migratetype < MIGRATE_PCPTYPES)) 631 migratetype = MIGRATE_UNMOVABLE; 632 633 set_pfnblock_flags_mask(page, (unsigned long)migratetype, 634 page_to_pfn(page), MIGRATETYPE_MASK); 635 } 636 637 #ifdef CONFIG_DEBUG_VM 638 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 639 { 640 int ret = 0; 641 unsigned seq; 642 unsigned long pfn = page_to_pfn(page); 643 unsigned long sp, start_pfn; 644 645 do { 646 seq = zone_span_seqbegin(zone); 647 start_pfn = zone->zone_start_pfn; 648 sp = zone->spanned_pages; 649 if (!zone_spans_pfn(zone, pfn)) 650 ret = 1; 651 } while (zone_span_seqretry(zone, seq)); 652 653 if (ret) 654 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 655 pfn, zone_to_nid(zone), zone->name, 656 start_pfn, start_pfn + sp); 657 658 return ret; 659 } 660 661 static int page_is_consistent(struct zone *zone, struct page *page) 662 { 663 if (zone != page_zone(page)) 664 return 0; 665 666 return 1; 667 } 668 /* 669 * Temporary debugging check for pages not lying within a given zone. 670 */ 671 static int __maybe_unused bad_range(struct zone *zone, struct page *page) 672 { 673 if (page_outside_zone_boundaries(zone, page)) 674 return 1; 675 if (!page_is_consistent(zone, page)) 676 return 1; 677 678 return 0; 679 } 680 #else 681 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page) 682 { 683 return 0; 684 } 685 #endif 686 687 static void bad_page(struct page *page, const char *reason) 688 { 689 static unsigned long resume; 690 static unsigned long nr_shown; 691 static unsigned long nr_unshown; 692 693 /* 694 * Allow a burst of 60 reports, then keep quiet for that minute; 695 * or allow a steady drip of one report per second. 696 */ 697 if (nr_shown == 60) { 698 if (time_before(jiffies, resume)) { 699 nr_unshown++; 700 goto out; 701 } 702 if (nr_unshown) { 703 pr_alert( 704 "BUG: Bad page state: %lu messages suppressed\n", 705 nr_unshown); 706 nr_unshown = 0; 707 } 708 nr_shown = 0; 709 } 710 if (nr_shown++ == 0) 711 resume = jiffies + 60 * HZ; 712 713 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 714 current->comm, page_to_pfn(page)); 715 dump_page(page, reason); 716 717 print_modules(); 718 dump_stack(); 719 out: 720 /* Leave bad fields for debug, except PageBuddy could make trouble */ 721 page_mapcount_reset(page); /* remove PageBuddy */ 722 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 723 } 724 725 static inline unsigned int order_to_pindex(int migratetype, int order) 726 { 727 int base = order; 728 729 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 730 if (order > PAGE_ALLOC_COSTLY_ORDER) { 731 VM_BUG_ON(order != pageblock_order); 732 return NR_LOWORDER_PCP_LISTS; 733 } 734 #else 735 VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); 736 #endif 737 738 return (MIGRATE_PCPTYPES * base) + migratetype; 739 } 740 741 static inline int pindex_to_order(unsigned int pindex) 742 { 743 int order = pindex / MIGRATE_PCPTYPES; 744 745 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 746 if (pindex == NR_LOWORDER_PCP_LISTS) 747 order = pageblock_order; 748 #else 749 VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); 750 #endif 751 752 return order; 753 } 754 755 static inline bool pcp_allowed_order(unsigned int order) 756 { 757 if (order <= PAGE_ALLOC_COSTLY_ORDER) 758 return true; 759 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 760 if (order == pageblock_order) 761 return true; 762 #endif 763 return false; 764 } 765 766 static inline void free_the_page(struct page *page, unsigned int order) 767 { 768 if (pcp_allowed_order(order)) /* Via pcp? */ 769 free_unref_page(page, order); 770 else 771 __free_pages_ok(page, order, FPI_NONE); 772 } 773 774 /* 775 * Higher-order pages are called "compound pages". They are structured thusly: 776 * 777 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 778 * 779 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 780 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 781 * 782 * The first tail page's ->compound_dtor holds the offset in array of compound 783 * page destructors. See compound_page_dtors. 784 * 785 * The first tail page's ->compound_order holds the order of allocation. 786 * This usage means that zero-order pages may not be compound. 787 */ 788 789 void free_compound_page(struct page *page) 790 { 791 mem_cgroup_uncharge(page_folio(page)); 792 free_the_page(page, compound_order(page)); 793 } 794 795 static void prep_compound_head(struct page *page, unsigned int order) 796 { 797 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 798 set_compound_order(page, order); 799 atomic_set(compound_mapcount_ptr(page), -1); 800 atomic_set(compound_pincount_ptr(page), 0); 801 } 802 803 static void prep_compound_tail(struct page *head, int tail_idx) 804 { 805 struct page *p = head + tail_idx; 806 807 p->mapping = TAIL_MAPPING; 808 set_compound_head(p, head); 809 } 810 811 void prep_compound_page(struct page *page, unsigned int order) 812 { 813 int i; 814 int nr_pages = 1 << order; 815 816 __SetPageHead(page); 817 for (i = 1; i < nr_pages; i++) 818 prep_compound_tail(page, i); 819 820 prep_compound_head(page, order); 821 } 822 823 void destroy_large_folio(struct folio *folio) 824 { 825 enum compound_dtor_id dtor = folio_page(folio, 1)->compound_dtor; 826 827 VM_BUG_ON_FOLIO(dtor >= NR_COMPOUND_DTORS, folio); 828 compound_page_dtors[dtor](&folio->page); 829 } 830 831 #ifdef CONFIG_DEBUG_PAGEALLOC 832 unsigned int _debug_guardpage_minorder; 833 834 bool _debug_pagealloc_enabled_early __read_mostly 835 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 836 EXPORT_SYMBOL(_debug_pagealloc_enabled_early); 837 DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 838 EXPORT_SYMBOL(_debug_pagealloc_enabled); 839 840 DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 841 842 static int __init early_debug_pagealloc(char *buf) 843 { 844 return kstrtobool(buf, &_debug_pagealloc_enabled_early); 845 } 846 early_param("debug_pagealloc", early_debug_pagealloc); 847 848 static int __init debug_guardpage_minorder_setup(char *buf) 849 { 850 unsigned long res; 851 852 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 853 pr_err("Bad debug_guardpage_minorder value\n"); 854 return 0; 855 } 856 _debug_guardpage_minorder = res; 857 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 858 return 0; 859 } 860 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); 861 862 static inline bool set_page_guard(struct zone *zone, struct page *page, 863 unsigned int order, int migratetype) 864 { 865 if (!debug_guardpage_enabled()) 866 return false; 867 868 if (order >= debug_guardpage_minorder()) 869 return false; 870 871 __SetPageGuard(page); 872 INIT_LIST_HEAD(&page->buddy_list); 873 set_page_private(page, order); 874 /* Guard pages are not available for any usage */ 875 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 876 877 return true; 878 } 879 880 static inline void clear_page_guard(struct zone *zone, struct page *page, 881 unsigned int order, int migratetype) 882 { 883 if (!debug_guardpage_enabled()) 884 return; 885 886 __ClearPageGuard(page); 887 888 set_page_private(page, 0); 889 if (!is_migrate_isolate(migratetype)) 890 __mod_zone_freepage_state(zone, (1 << order), migratetype); 891 } 892 #else 893 static inline bool set_page_guard(struct zone *zone, struct page *page, 894 unsigned int order, int migratetype) { return false; } 895 static inline void clear_page_guard(struct zone *zone, struct page *page, 896 unsigned int order, int migratetype) {} 897 #endif 898 899 /* 900 * Enable static keys related to various memory debugging and hardening options. 901 * Some override others, and depend on early params that are evaluated in the 902 * order of appearance. So we need to first gather the full picture of what was 903 * enabled, and then make decisions. 904 */ 905 void init_mem_debugging_and_hardening(void) 906 { 907 bool page_poisoning_requested = false; 908 909 #ifdef CONFIG_PAGE_POISONING 910 /* 911 * Page poisoning is debug page alloc for some arches. If 912 * either of those options are enabled, enable poisoning. 913 */ 914 if (page_poisoning_enabled() || 915 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && 916 debug_pagealloc_enabled())) { 917 static_branch_enable(&_page_poisoning_enabled); 918 page_poisoning_requested = true; 919 } 920 #endif 921 922 if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) && 923 page_poisoning_requested) { 924 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " 925 "will take precedence over init_on_alloc and init_on_free\n"); 926 _init_on_alloc_enabled_early = false; 927 _init_on_free_enabled_early = false; 928 } 929 930 if (_init_on_alloc_enabled_early) 931 static_branch_enable(&init_on_alloc); 932 else 933 static_branch_disable(&init_on_alloc); 934 935 if (_init_on_free_enabled_early) 936 static_branch_enable(&init_on_free); 937 else 938 static_branch_disable(&init_on_free); 939 940 #ifdef CONFIG_DEBUG_PAGEALLOC 941 if (!debug_pagealloc_enabled()) 942 return; 943 944 static_branch_enable(&_debug_pagealloc_enabled); 945 946 if (!debug_guardpage_minorder()) 947 return; 948 949 static_branch_enable(&_debug_guardpage_enabled); 950 #endif 951 } 952 953 static inline void set_buddy_order(struct page *page, unsigned int order) 954 { 955 set_page_private(page, order); 956 __SetPageBuddy(page); 957 } 958 959 #ifdef CONFIG_COMPACTION 960 static inline struct capture_control *task_capc(struct zone *zone) 961 { 962 struct capture_control *capc = current->capture_control; 963 964 return unlikely(capc) && 965 !(current->flags & PF_KTHREAD) && 966 !capc->page && 967 capc->cc->zone == zone ? capc : NULL; 968 } 969 970 static inline bool 971 compaction_capture(struct capture_control *capc, struct page *page, 972 int order, int migratetype) 973 { 974 if (!capc || order != capc->cc->order) 975 return false; 976 977 /* Do not accidentally pollute CMA or isolated regions*/ 978 if (is_migrate_cma(migratetype) || 979 is_migrate_isolate(migratetype)) 980 return false; 981 982 /* 983 * Do not let lower order allocations pollute a movable pageblock. 984 * This might let an unmovable request use a reclaimable pageblock 985 * and vice-versa but no more than normal fallback logic which can 986 * have trouble finding a high-order free page. 987 */ 988 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE) 989 return false; 990 991 capc->page = page; 992 return true; 993 } 994 995 #else 996 static inline struct capture_control *task_capc(struct zone *zone) 997 { 998 return NULL; 999 } 1000 1001 static inline bool 1002 compaction_capture(struct capture_control *capc, struct page *page, 1003 int order, int migratetype) 1004 { 1005 return false; 1006 } 1007 #endif /* CONFIG_COMPACTION */ 1008 1009 /* Used for pages not on another list */ 1010 static inline void add_to_free_list(struct page *page, struct zone *zone, 1011 unsigned int order, int migratetype) 1012 { 1013 struct free_area *area = &zone->free_area[order]; 1014 1015 list_add(&page->buddy_list, &area->free_list[migratetype]); 1016 area->nr_free++; 1017 } 1018 1019 /* Used for pages not on another list */ 1020 static inline void add_to_free_list_tail(struct page *page, struct zone *zone, 1021 unsigned int order, int migratetype) 1022 { 1023 struct free_area *area = &zone->free_area[order]; 1024 1025 list_add_tail(&page->buddy_list, &area->free_list[migratetype]); 1026 area->nr_free++; 1027 } 1028 1029 /* 1030 * Used for pages which are on another list. Move the pages to the tail 1031 * of the list - so the moved pages won't immediately be considered for 1032 * allocation again (e.g., optimization for memory onlining). 1033 */ 1034 static inline void move_to_free_list(struct page *page, struct zone *zone, 1035 unsigned int order, int migratetype) 1036 { 1037 struct free_area *area = &zone->free_area[order]; 1038 1039 list_move_tail(&page->buddy_list, &area->free_list[migratetype]); 1040 } 1041 1042 static inline void del_page_from_free_list(struct page *page, struct zone *zone, 1043 unsigned int order) 1044 { 1045 /* clear reported state and update reported page count */ 1046 if (page_reported(page)) 1047 __ClearPageReported(page); 1048 1049 list_del(&page->buddy_list); 1050 __ClearPageBuddy(page); 1051 set_page_private(page, 0); 1052 zone->free_area[order].nr_free--; 1053 } 1054 1055 /* 1056 * If this is not the largest possible page, check if the buddy 1057 * of the next-highest order is free. If it is, it's possible 1058 * that pages are being freed that will coalesce soon. In case, 1059 * that is happening, add the free page to the tail of the list 1060 * so it's less likely to be used soon and more likely to be merged 1061 * as a higher order page 1062 */ 1063 static inline bool 1064 buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, 1065 struct page *page, unsigned int order) 1066 { 1067 unsigned long higher_page_pfn; 1068 struct page *higher_page; 1069 1070 if (order >= MAX_ORDER - 2) 1071 return false; 1072 1073 higher_page_pfn = buddy_pfn & pfn; 1074 higher_page = page + (higher_page_pfn - pfn); 1075 1076 return find_buddy_page_pfn(higher_page, higher_page_pfn, order + 1, 1077 NULL) != NULL; 1078 } 1079 1080 /* 1081 * Freeing function for a buddy system allocator. 1082 * 1083 * The concept of a buddy system is to maintain direct-mapped table 1084 * (containing bit values) for memory blocks of various "orders". 1085 * The bottom level table contains the map for the smallest allocatable 1086 * units of memory (here, pages), and each level above it describes 1087 * pairs of units from the levels below, hence, "buddies". 1088 * At a high level, all that happens here is marking the table entry 1089 * at the bottom level available, and propagating the changes upward 1090 * as necessary, plus some accounting needed to play nicely with other 1091 * parts of the VM system. 1092 * At each level, we keep a list of pages, which are heads of continuous 1093 * free pages of length of (1 << order) and marked with PageBuddy. 1094 * Page's order is recorded in page_private(page) field. 1095 * So when we are allocating or freeing one, we can derive the state of the 1096 * other. That is, if we allocate a small block, and both were 1097 * free, the remainder of the region must be split into blocks. 1098 * If a block is freed, and its buddy is also free, then this 1099 * triggers coalescing into a block of larger size. 1100 * 1101 * -- nyc 1102 */ 1103 1104 static inline void __free_one_page(struct page *page, 1105 unsigned long pfn, 1106 struct zone *zone, unsigned int order, 1107 int migratetype, fpi_t fpi_flags) 1108 { 1109 struct capture_control *capc = task_capc(zone); 1110 unsigned long buddy_pfn; 1111 unsigned long combined_pfn; 1112 struct page *buddy; 1113 bool to_tail; 1114 1115 VM_BUG_ON(!zone_is_initialized(zone)); 1116 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 1117 1118 VM_BUG_ON(migratetype == -1); 1119 if (likely(!is_migrate_isolate(migratetype))) 1120 __mod_zone_freepage_state(zone, 1 << order, migratetype); 1121 1122 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); 1123 VM_BUG_ON_PAGE(bad_range(zone, page), page); 1124 1125 while (order < MAX_ORDER - 1) { 1126 if (compaction_capture(capc, page, order, migratetype)) { 1127 __mod_zone_freepage_state(zone, -(1 << order), 1128 migratetype); 1129 return; 1130 } 1131 1132 buddy = find_buddy_page_pfn(page, pfn, order, &buddy_pfn); 1133 if (!buddy) 1134 goto done_merging; 1135 1136 if (unlikely(order >= pageblock_order)) { 1137 /* 1138 * We want to prevent merge between freepages on pageblock 1139 * without fallbacks and normal pageblock. Without this, 1140 * pageblock isolation could cause incorrect freepage or CMA 1141 * accounting or HIGHATOMIC accounting. 1142 */ 1143 int buddy_mt = get_pageblock_migratetype(buddy); 1144 1145 if (migratetype != buddy_mt 1146 && (!migratetype_is_mergeable(migratetype) || 1147 !migratetype_is_mergeable(buddy_mt))) 1148 goto done_merging; 1149 } 1150 1151 /* 1152 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 1153 * merge with it and move up one order. 1154 */ 1155 if (page_is_guard(buddy)) 1156 clear_page_guard(zone, buddy, order, migratetype); 1157 else 1158 del_page_from_free_list(buddy, zone, order); 1159 combined_pfn = buddy_pfn & pfn; 1160 page = page + (combined_pfn - pfn); 1161 pfn = combined_pfn; 1162 order++; 1163 } 1164 1165 done_merging: 1166 set_buddy_order(page, order); 1167 1168 if (fpi_flags & FPI_TO_TAIL) 1169 to_tail = true; 1170 else if (is_shuffle_order(order)) 1171 to_tail = shuffle_pick_tail(); 1172 else 1173 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); 1174 1175 if (to_tail) 1176 add_to_free_list_tail(page, zone, order, migratetype); 1177 else 1178 add_to_free_list(page, zone, order, migratetype); 1179 1180 /* Notify page reporting subsystem of freed page */ 1181 if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY)) 1182 page_reporting_notify_free(order); 1183 } 1184 1185 /** 1186 * split_free_page() -- split a free page at split_pfn_offset 1187 * @free_page: the original free page 1188 * @order: the order of the page 1189 * @split_pfn_offset: split offset within the page 1190 * 1191 * Return -ENOENT if the free page is changed, otherwise 0 1192 * 1193 * It is used when the free page crosses two pageblocks with different migratetypes 1194 * at split_pfn_offset within the page. The split free page will be put into 1195 * separate migratetype lists afterwards. Otherwise, the function achieves 1196 * nothing. 1197 */ 1198 int split_free_page(struct page *free_page, 1199 unsigned int order, unsigned long split_pfn_offset) 1200 { 1201 struct zone *zone = page_zone(free_page); 1202 unsigned long free_page_pfn = page_to_pfn(free_page); 1203 unsigned long pfn; 1204 unsigned long flags; 1205 int free_page_order; 1206 int mt; 1207 int ret = 0; 1208 1209 if (split_pfn_offset == 0) 1210 return ret; 1211 1212 spin_lock_irqsave(&zone->lock, flags); 1213 1214 if (!PageBuddy(free_page) || buddy_order(free_page) != order) { 1215 ret = -ENOENT; 1216 goto out; 1217 } 1218 1219 mt = get_pageblock_migratetype(free_page); 1220 if (likely(!is_migrate_isolate(mt))) 1221 __mod_zone_freepage_state(zone, -(1UL << order), mt); 1222 1223 del_page_from_free_list(free_page, zone, order); 1224 for (pfn = free_page_pfn; 1225 pfn < free_page_pfn + (1UL << order);) { 1226 int mt = get_pfnblock_migratetype(pfn_to_page(pfn), pfn); 1227 1228 free_page_order = min_t(unsigned int, 1229 pfn ? __ffs(pfn) : order, 1230 __fls(split_pfn_offset)); 1231 __free_one_page(pfn_to_page(pfn), pfn, zone, free_page_order, 1232 mt, FPI_NONE); 1233 pfn += 1UL << free_page_order; 1234 split_pfn_offset -= (1UL << free_page_order); 1235 /* we have done the first part, now switch to second part */ 1236 if (split_pfn_offset == 0) 1237 split_pfn_offset = (1UL << order) - (pfn - free_page_pfn); 1238 } 1239 out: 1240 spin_unlock_irqrestore(&zone->lock, flags); 1241 return ret; 1242 } 1243 /* 1244 * A bad page could be due to a number of fields. Instead of multiple branches, 1245 * try and check multiple fields with one check. The caller must do a detailed 1246 * check if necessary. 1247 */ 1248 static inline bool page_expected_state(struct page *page, 1249 unsigned long check_flags) 1250 { 1251 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1252 return false; 1253 1254 if (unlikely((unsigned long)page->mapping | 1255 page_ref_count(page) | 1256 #ifdef CONFIG_MEMCG 1257 page->memcg_data | 1258 #endif 1259 (page->flags & check_flags))) 1260 return false; 1261 1262 return true; 1263 } 1264 1265 static const char *page_bad_reason(struct page *page, unsigned long flags) 1266 { 1267 const char *bad_reason = NULL; 1268 1269 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1270 bad_reason = "nonzero mapcount"; 1271 if (unlikely(page->mapping != NULL)) 1272 bad_reason = "non-NULL mapping"; 1273 if (unlikely(page_ref_count(page) != 0)) 1274 bad_reason = "nonzero _refcount"; 1275 if (unlikely(page->flags & flags)) { 1276 if (flags == PAGE_FLAGS_CHECK_AT_PREP) 1277 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set"; 1278 else 1279 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 1280 } 1281 #ifdef CONFIG_MEMCG 1282 if (unlikely(page->memcg_data)) 1283 bad_reason = "page still charged to cgroup"; 1284 #endif 1285 return bad_reason; 1286 } 1287 1288 static void check_free_page_bad(struct page *page) 1289 { 1290 bad_page(page, 1291 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE)); 1292 } 1293 1294 static inline int check_free_page(struct page *page) 1295 { 1296 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 1297 return 0; 1298 1299 /* Something has gone sideways, find it */ 1300 check_free_page_bad(page); 1301 return 1; 1302 } 1303 1304 static int free_tail_pages_check(struct page *head_page, struct page *page) 1305 { 1306 int ret = 1; 1307 1308 /* 1309 * We rely page->lru.next never has bit 0 set, unless the page 1310 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 1311 */ 1312 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 1313 1314 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 1315 ret = 0; 1316 goto out; 1317 } 1318 switch (page - head_page) { 1319 case 1: 1320 /* the first tail page: ->mapping may be compound_mapcount() */ 1321 if (unlikely(compound_mapcount(page))) { 1322 bad_page(page, "nonzero compound_mapcount"); 1323 goto out; 1324 } 1325 break; 1326 case 2: 1327 /* 1328 * the second tail page: ->mapping is 1329 * deferred_list.next -- ignore value. 1330 */ 1331 break; 1332 default: 1333 if (page->mapping != TAIL_MAPPING) { 1334 bad_page(page, "corrupted mapping in tail page"); 1335 goto out; 1336 } 1337 break; 1338 } 1339 if (unlikely(!PageTail(page))) { 1340 bad_page(page, "PageTail not set"); 1341 goto out; 1342 } 1343 if (unlikely(compound_head(page) != head_page)) { 1344 bad_page(page, "compound_head not consistent"); 1345 goto out; 1346 } 1347 ret = 0; 1348 out: 1349 page->mapping = NULL; 1350 clear_compound_head(page); 1351 return ret; 1352 } 1353 1354 /* 1355 * Skip KASAN memory poisoning when either: 1356 * 1357 * 1. Deferred memory initialization has not yet completed, 1358 * see the explanation below. 1359 * 2. Skipping poisoning is requested via FPI_SKIP_KASAN_POISON, 1360 * see the comment next to it. 1361 * 3. Skipping poisoning is requested via __GFP_SKIP_KASAN_POISON, 1362 * see the comment next to it. 1363 * 1364 * Poisoning pages during deferred memory init will greatly lengthen the 1365 * process and cause problem in large memory systems as the deferred pages 1366 * initialization is done with interrupt disabled. 1367 * 1368 * Assuming that there will be no reference to those newly initialized 1369 * pages before they are ever allocated, this should have no effect on 1370 * KASAN memory tracking as the poison will be properly inserted at page 1371 * allocation time. The only corner case is when pages are allocated by 1372 * on-demand allocation and then freed again before the deferred pages 1373 * initialization is done, but this is not likely to happen. 1374 */ 1375 static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags) 1376 { 1377 return deferred_pages_enabled() || 1378 (!IS_ENABLED(CONFIG_KASAN_GENERIC) && 1379 (fpi_flags & FPI_SKIP_KASAN_POISON)) || 1380 PageSkipKASanPoison(page); 1381 } 1382 1383 static void kernel_init_pages(struct page *page, int numpages) 1384 { 1385 int i; 1386 1387 /* s390's use of memset() could override KASAN redzones. */ 1388 kasan_disable_current(); 1389 for (i = 0; i < numpages; i++) 1390 clear_highpage_kasan_tagged(page + i); 1391 kasan_enable_current(); 1392 } 1393 1394 static __always_inline bool free_pages_prepare(struct page *page, 1395 unsigned int order, bool check_free, fpi_t fpi_flags) 1396 { 1397 int bad = 0; 1398 bool init = want_init_on_free(); 1399 1400 VM_BUG_ON_PAGE(PageTail(page), page); 1401 1402 trace_mm_page_free(page, order); 1403 1404 if (unlikely(PageHWPoison(page)) && !order) { 1405 /* 1406 * Do not let hwpoison pages hit pcplists/buddy 1407 * Untie memcg state and reset page's owner 1408 */ 1409 if (memcg_kmem_enabled() && PageMemcgKmem(page)) 1410 __memcg_kmem_uncharge_page(page, order); 1411 reset_page_owner(page, order); 1412 page_table_check_free(page, order); 1413 return false; 1414 } 1415 1416 /* 1417 * Check tail pages before head page information is cleared to 1418 * avoid checking PageCompound for order-0 pages. 1419 */ 1420 if (unlikely(order)) { 1421 bool compound = PageCompound(page); 1422 int i; 1423 1424 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1425 1426 if (compound) { 1427 ClearPageDoubleMap(page); 1428 ClearPageHasHWPoisoned(page); 1429 } 1430 for (i = 1; i < (1 << order); i++) { 1431 if (compound) 1432 bad += free_tail_pages_check(page, page + i); 1433 if (unlikely(check_free_page(page + i))) { 1434 bad++; 1435 continue; 1436 } 1437 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1438 } 1439 } 1440 if (PageMappingFlags(page)) 1441 page->mapping = NULL; 1442 if (memcg_kmem_enabled() && PageMemcgKmem(page)) 1443 __memcg_kmem_uncharge_page(page, order); 1444 if (check_free) 1445 bad += check_free_page(page); 1446 if (bad) 1447 return false; 1448 1449 page_cpupid_reset_last(page); 1450 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1451 reset_page_owner(page, order); 1452 page_table_check_free(page, order); 1453 1454 if (!PageHighMem(page)) { 1455 debug_check_no_locks_freed(page_address(page), 1456 PAGE_SIZE << order); 1457 debug_check_no_obj_freed(page_address(page), 1458 PAGE_SIZE << order); 1459 } 1460 1461 kernel_poison_pages(page, 1 << order); 1462 1463 /* 1464 * As memory initialization might be integrated into KASAN, 1465 * KASAN poisoning and memory initialization code must be 1466 * kept together to avoid discrepancies in behavior. 1467 * 1468 * With hardware tag-based KASAN, memory tags must be set before the 1469 * page becomes unavailable via debug_pagealloc or arch_free_page. 1470 */ 1471 if (!should_skip_kasan_poison(page, fpi_flags)) { 1472 kasan_poison_pages(page, order, init); 1473 1474 /* Memory is already initialized if KASAN did it internally. */ 1475 if (kasan_has_integrated_init()) 1476 init = false; 1477 } 1478 if (init) 1479 kernel_init_pages(page, 1 << order); 1480 1481 /* 1482 * arch_free_page() can make the page's contents inaccessible. s390 1483 * does this. So nothing which can access the page's contents should 1484 * happen after this. 1485 */ 1486 arch_free_page(page, order); 1487 1488 debug_pagealloc_unmap_pages(page, 1 << order); 1489 1490 return true; 1491 } 1492 1493 #ifdef CONFIG_DEBUG_VM 1494 /* 1495 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed 1496 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when 1497 * moved from pcp lists to free lists. 1498 */ 1499 static bool free_pcp_prepare(struct page *page, unsigned int order) 1500 { 1501 return free_pages_prepare(page, order, true, FPI_NONE); 1502 } 1503 1504 static bool bulkfree_pcp_prepare(struct page *page) 1505 { 1506 if (debug_pagealloc_enabled_static()) 1507 return check_free_page(page); 1508 else 1509 return false; 1510 } 1511 #else 1512 /* 1513 * With DEBUG_VM disabled, order-0 pages being freed are checked only when 1514 * moving from pcp lists to free list in order to reduce overhead. With 1515 * debug_pagealloc enabled, they are checked also immediately when being freed 1516 * to the pcp lists. 1517 */ 1518 static bool free_pcp_prepare(struct page *page, unsigned int order) 1519 { 1520 if (debug_pagealloc_enabled_static()) 1521 return free_pages_prepare(page, order, true, FPI_NONE); 1522 else 1523 return free_pages_prepare(page, order, false, FPI_NONE); 1524 } 1525 1526 static bool bulkfree_pcp_prepare(struct page *page) 1527 { 1528 return check_free_page(page); 1529 } 1530 #endif /* CONFIG_DEBUG_VM */ 1531 1532 /* 1533 * Frees a number of pages from the PCP lists 1534 * Assumes all pages on list are in same zone. 1535 * count is the number of pages to free. 1536 */ 1537 static void free_pcppages_bulk(struct zone *zone, int count, 1538 struct per_cpu_pages *pcp, 1539 int pindex) 1540 { 1541 int min_pindex = 0; 1542 int max_pindex = NR_PCP_LISTS - 1; 1543 unsigned int order; 1544 bool isolated_pageblocks; 1545 struct page *page; 1546 1547 /* 1548 * Ensure proper count is passed which otherwise would stuck in the 1549 * below while (list_empty(list)) loop. 1550 */ 1551 count = min(pcp->count, count); 1552 1553 /* Ensure requested pindex is drained first. */ 1554 pindex = pindex - 1; 1555 1556 /* Caller must hold IRQ-safe pcp->lock so IRQs are disabled. */ 1557 spin_lock(&zone->lock); 1558 isolated_pageblocks = has_isolate_pageblock(zone); 1559 1560 while (count > 0) { 1561 struct list_head *list; 1562 int nr_pages; 1563 1564 /* Remove pages from lists in a round-robin fashion. */ 1565 do { 1566 if (++pindex > max_pindex) 1567 pindex = min_pindex; 1568 list = &pcp->lists[pindex]; 1569 if (!list_empty(list)) 1570 break; 1571 1572 if (pindex == max_pindex) 1573 max_pindex--; 1574 if (pindex == min_pindex) 1575 min_pindex++; 1576 } while (1); 1577 1578 order = pindex_to_order(pindex); 1579 nr_pages = 1 << order; 1580 BUILD_BUG_ON(MAX_ORDER >= (1<<NR_PCP_ORDER_WIDTH)); 1581 do { 1582 int mt; 1583 1584 page = list_last_entry(list, struct page, pcp_list); 1585 mt = get_pcppage_migratetype(page); 1586 1587 /* must delete to avoid corrupting pcp list */ 1588 list_del(&page->pcp_list); 1589 count -= nr_pages; 1590 pcp->count -= nr_pages; 1591 1592 if (bulkfree_pcp_prepare(page)) 1593 continue; 1594 1595 /* MIGRATE_ISOLATE page should not go to pcplists */ 1596 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1597 /* Pageblock could have been isolated meanwhile */ 1598 if (unlikely(isolated_pageblocks)) 1599 mt = get_pageblock_migratetype(page); 1600 1601 __free_one_page(page, page_to_pfn(page), zone, order, mt, FPI_NONE); 1602 trace_mm_page_pcpu_drain(page, order, mt); 1603 } while (count > 0 && !list_empty(list)); 1604 } 1605 1606 spin_unlock(&zone->lock); 1607 } 1608 1609 static void free_one_page(struct zone *zone, 1610 struct page *page, unsigned long pfn, 1611 unsigned int order, 1612 int migratetype, fpi_t fpi_flags) 1613 { 1614 unsigned long flags; 1615 1616 spin_lock_irqsave(&zone->lock, flags); 1617 if (unlikely(has_isolate_pageblock(zone) || 1618 is_migrate_isolate(migratetype))) { 1619 migratetype = get_pfnblock_migratetype(page, pfn); 1620 } 1621 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags); 1622 spin_unlock_irqrestore(&zone->lock, flags); 1623 } 1624 1625 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1626 unsigned long zone, int nid) 1627 { 1628 mm_zero_struct_page(page); 1629 set_page_links(page, zone, nid, pfn); 1630 init_page_count(page); 1631 page_mapcount_reset(page); 1632 page_cpupid_reset_last(page); 1633 page_kasan_tag_reset(page); 1634 1635 INIT_LIST_HEAD(&page->lru); 1636 #ifdef WANT_PAGE_VIRTUAL 1637 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1638 if (!is_highmem_idx(zone)) 1639 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1640 #endif 1641 } 1642 1643 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1644 static void __meminit init_reserved_page(unsigned long pfn) 1645 { 1646 pg_data_t *pgdat; 1647 int nid, zid; 1648 1649 if (!early_page_uninitialised(pfn)) 1650 return; 1651 1652 nid = early_pfn_to_nid(pfn); 1653 pgdat = NODE_DATA(nid); 1654 1655 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1656 struct zone *zone = &pgdat->node_zones[zid]; 1657 1658 if (zone_spans_pfn(zone, pfn)) 1659 break; 1660 } 1661 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 1662 } 1663 #else 1664 static inline void init_reserved_page(unsigned long pfn) 1665 { 1666 } 1667 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1668 1669 /* 1670 * Initialised pages do not have PageReserved set. This function is 1671 * called for each range allocated by the bootmem allocator and 1672 * marks the pages PageReserved. The remaining valid pages are later 1673 * sent to the buddy page allocator. 1674 */ 1675 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1676 { 1677 unsigned long start_pfn = PFN_DOWN(start); 1678 unsigned long end_pfn = PFN_UP(end); 1679 1680 for (; start_pfn < end_pfn; start_pfn++) { 1681 if (pfn_valid(start_pfn)) { 1682 struct page *page = pfn_to_page(start_pfn); 1683 1684 init_reserved_page(start_pfn); 1685 1686 /* Avoid false-positive PageTail() */ 1687 INIT_LIST_HEAD(&page->lru); 1688 1689 /* 1690 * no need for atomic set_bit because the struct 1691 * page is not visible yet so nobody should 1692 * access it yet. 1693 */ 1694 __SetPageReserved(page); 1695 } 1696 } 1697 } 1698 1699 static void __free_pages_ok(struct page *page, unsigned int order, 1700 fpi_t fpi_flags) 1701 { 1702 unsigned long flags; 1703 int migratetype; 1704 unsigned long pfn = page_to_pfn(page); 1705 struct zone *zone = page_zone(page); 1706 1707 if (!free_pages_prepare(page, order, true, fpi_flags)) 1708 return; 1709 1710 migratetype = get_pfnblock_migratetype(page, pfn); 1711 1712 spin_lock_irqsave(&zone->lock, flags); 1713 if (unlikely(has_isolate_pageblock(zone) || 1714 is_migrate_isolate(migratetype))) { 1715 migratetype = get_pfnblock_migratetype(page, pfn); 1716 } 1717 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags); 1718 spin_unlock_irqrestore(&zone->lock, flags); 1719 1720 __count_vm_events(PGFREE, 1 << order); 1721 } 1722 1723 void __free_pages_core(struct page *page, unsigned int order) 1724 { 1725 unsigned int nr_pages = 1 << order; 1726 struct page *p = page; 1727 unsigned int loop; 1728 1729 /* 1730 * When initializing the memmap, __init_single_page() sets the refcount 1731 * of all pages to 1 ("allocated"/"not free"). We have to set the 1732 * refcount of all involved pages to 0. 1733 */ 1734 prefetchw(p); 1735 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1736 prefetchw(p + 1); 1737 __ClearPageReserved(p); 1738 set_page_count(p, 0); 1739 } 1740 __ClearPageReserved(p); 1741 set_page_count(p, 0); 1742 1743 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); 1744 1745 /* 1746 * Bypass PCP and place fresh pages right to the tail, primarily 1747 * relevant for memory onlining. 1748 */ 1749 __free_pages_ok(page, order, FPI_TO_TAIL | FPI_SKIP_KASAN_POISON); 1750 } 1751 1752 #ifdef CONFIG_NUMA 1753 1754 /* 1755 * During memory init memblocks map pfns to nids. The search is expensive and 1756 * this caches recent lookups. The implementation of __early_pfn_to_nid 1757 * treats start/end as pfns. 1758 */ 1759 struct mminit_pfnnid_cache { 1760 unsigned long last_start; 1761 unsigned long last_end; 1762 int last_nid; 1763 }; 1764 1765 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1766 1767 /* 1768 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 1769 */ 1770 static int __meminit __early_pfn_to_nid(unsigned long pfn, 1771 struct mminit_pfnnid_cache *state) 1772 { 1773 unsigned long start_pfn, end_pfn; 1774 int nid; 1775 1776 if (state->last_start <= pfn && pfn < state->last_end) 1777 return state->last_nid; 1778 1779 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 1780 if (nid != NUMA_NO_NODE) { 1781 state->last_start = start_pfn; 1782 state->last_end = end_pfn; 1783 state->last_nid = nid; 1784 } 1785 1786 return nid; 1787 } 1788 1789 int __meminit early_pfn_to_nid(unsigned long pfn) 1790 { 1791 static DEFINE_SPINLOCK(early_pfn_lock); 1792 int nid; 1793 1794 spin_lock(&early_pfn_lock); 1795 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1796 if (nid < 0) 1797 nid = first_online_node; 1798 spin_unlock(&early_pfn_lock); 1799 1800 return nid; 1801 } 1802 #endif /* CONFIG_NUMA */ 1803 1804 void __init memblock_free_pages(struct page *page, unsigned long pfn, 1805 unsigned int order) 1806 { 1807 if (early_page_uninitialised(pfn)) 1808 return; 1809 __free_pages_core(page, order); 1810 } 1811 1812 /* 1813 * Check that the whole (or subset of) a pageblock given by the interval of 1814 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1815 * with the migration of free compaction scanner. 1816 * 1817 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1818 * 1819 * It's possible on some configurations to have a setup like node0 node1 node0 1820 * i.e. it's possible that all pages within a zones range of pages do not 1821 * belong to a single zone. We assume that a border between node0 and node1 1822 * can occur within a single pageblock, but not a node0 node1 node0 1823 * interleaving within a single pageblock. It is therefore sufficient to check 1824 * the first and last page of a pageblock and avoid checking each individual 1825 * page in a pageblock. 1826 */ 1827 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1828 unsigned long end_pfn, struct zone *zone) 1829 { 1830 struct page *start_page; 1831 struct page *end_page; 1832 1833 /* end_pfn is one past the range we are checking */ 1834 end_pfn--; 1835 1836 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1837 return NULL; 1838 1839 start_page = pfn_to_online_page(start_pfn); 1840 if (!start_page) 1841 return NULL; 1842 1843 if (page_zone(start_page) != zone) 1844 return NULL; 1845 1846 end_page = pfn_to_page(end_pfn); 1847 1848 /* This gives a shorter code than deriving page_zone(end_page) */ 1849 if (page_zone_id(start_page) != page_zone_id(end_page)) 1850 return NULL; 1851 1852 return start_page; 1853 } 1854 1855 void set_zone_contiguous(struct zone *zone) 1856 { 1857 unsigned long block_start_pfn = zone->zone_start_pfn; 1858 unsigned long block_end_pfn; 1859 1860 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1861 for (; block_start_pfn < zone_end_pfn(zone); 1862 block_start_pfn = block_end_pfn, 1863 block_end_pfn += pageblock_nr_pages) { 1864 1865 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1866 1867 if (!__pageblock_pfn_to_page(block_start_pfn, 1868 block_end_pfn, zone)) 1869 return; 1870 cond_resched(); 1871 } 1872 1873 /* We confirm that there is no hole */ 1874 zone->contiguous = true; 1875 } 1876 1877 void clear_zone_contiguous(struct zone *zone) 1878 { 1879 zone->contiguous = false; 1880 } 1881 1882 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1883 static void __init deferred_free_range(unsigned long pfn, 1884 unsigned long nr_pages) 1885 { 1886 struct page *page; 1887 unsigned long i; 1888 1889 if (!nr_pages) 1890 return; 1891 1892 page = pfn_to_page(pfn); 1893 1894 /* Free a large naturally-aligned chunk if possible */ 1895 if (nr_pages == pageblock_nr_pages && 1896 (pfn & (pageblock_nr_pages - 1)) == 0) { 1897 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1898 __free_pages_core(page, pageblock_order); 1899 return; 1900 } 1901 1902 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1903 if ((pfn & (pageblock_nr_pages - 1)) == 0) 1904 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1905 __free_pages_core(page, 0); 1906 } 1907 } 1908 1909 /* Completion tracking for deferred_init_memmap() threads */ 1910 static atomic_t pgdat_init_n_undone __initdata; 1911 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1912 1913 static inline void __init pgdat_init_report_one_done(void) 1914 { 1915 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1916 complete(&pgdat_init_all_done_comp); 1917 } 1918 1919 /* 1920 * Returns true if page needs to be initialized or freed to buddy allocator. 1921 * 1922 * First we check if pfn is valid on architectures where it is possible to have 1923 * holes within pageblock_nr_pages. On systems where it is not possible, this 1924 * function is optimized out. 1925 * 1926 * Then, we check if a current large page is valid by only checking the validity 1927 * of the head pfn. 1928 */ 1929 static inline bool __init deferred_pfn_valid(unsigned long pfn) 1930 { 1931 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn)) 1932 return false; 1933 return true; 1934 } 1935 1936 /* 1937 * Free pages to buddy allocator. Try to free aligned pages in 1938 * pageblock_nr_pages sizes. 1939 */ 1940 static void __init deferred_free_pages(unsigned long pfn, 1941 unsigned long end_pfn) 1942 { 1943 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1944 unsigned long nr_free = 0; 1945 1946 for (; pfn < end_pfn; pfn++) { 1947 if (!deferred_pfn_valid(pfn)) { 1948 deferred_free_range(pfn - nr_free, nr_free); 1949 nr_free = 0; 1950 } else if (!(pfn & nr_pgmask)) { 1951 deferred_free_range(pfn - nr_free, nr_free); 1952 nr_free = 1; 1953 } else { 1954 nr_free++; 1955 } 1956 } 1957 /* Free the last block of pages to allocator */ 1958 deferred_free_range(pfn - nr_free, nr_free); 1959 } 1960 1961 /* 1962 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 1963 * by performing it only once every pageblock_nr_pages. 1964 * Return number of pages initialized. 1965 */ 1966 static unsigned long __init deferred_init_pages(struct zone *zone, 1967 unsigned long pfn, 1968 unsigned long end_pfn) 1969 { 1970 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1971 int nid = zone_to_nid(zone); 1972 unsigned long nr_pages = 0; 1973 int zid = zone_idx(zone); 1974 struct page *page = NULL; 1975 1976 for (; pfn < end_pfn; pfn++) { 1977 if (!deferred_pfn_valid(pfn)) { 1978 page = NULL; 1979 continue; 1980 } else if (!page || !(pfn & nr_pgmask)) { 1981 page = pfn_to_page(pfn); 1982 } else { 1983 page++; 1984 } 1985 __init_single_page(page, pfn, zid, nid); 1986 nr_pages++; 1987 } 1988 return (nr_pages); 1989 } 1990 1991 /* 1992 * This function is meant to pre-load the iterator for the zone init. 1993 * Specifically it walks through the ranges until we are caught up to the 1994 * first_init_pfn value and exits there. If we never encounter the value we 1995 * return false indicating there are no valid ranges left. 1996 */ 1997 static bool __init 1998 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, 1999 unsigned long *spfn, unsigned long *epfn, 2000 unsigned long first_init_pfn) 2001 { 2002 u64 j; 2003 2004 /* 2005 * Start out by walking through the ranges in this zone that have 2006 * already been initialized. We don't need to do anything with them 2007 * so we just need to flush them out of the system. 2008 */ 2009 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { 2010 if (*epfn <= first_init_pfn) 2011 continue; 2012 if (*spfn < first_init_pfn) 2013 *spfn = first_init_pfn; 2014 *i = j; 2015 return true; 2016 } 2017 2018 return false; 2019 } 2020 2021 /* 2022 * Initialize and free pages. We do it in two loops: first we initialize 2023 * struct page, then free to buddy allocator, because while we are 2024 * freeing pages we can access pages that are ahead (computing buddy 2025 * page in __free_one_page()). 2026 * 2027 * In order to try and keep some memory in the cache we have the loop 2028 * broken along max page order boundaries. This way we will not cause 2029 * any issues with the buddy page computation. 2030 */ 2031 static unsigned long __init 2032 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, 2033 unsigned long *end_pfn) 2034 { 2035 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); 2036 unsigned long spfn = *start_pfn, epfn = *end_pfn; 2037 unsigned long nr_pages = 0; 2038 u64 j = *i; 2039 2040 /* First we loop through and initialize the page values */ 2041 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { 2042 unsigned long t; 2043 2044 if (mo_pfn <= *start_pfn) 2045 break; 2046 2047 t = min(mo_pfn, *end_pfn); 2048 nr_pages += deferred_init_pages(zone, *start_pfn, t); 2049 2050 if (mo_pfn < *end_pfn) { 2051 *start_pfn = mo_pfn; 2052 break; 2053 } 2054 } 2055 2056 /* Reset values and now loop through freeing pages as needed */ 2057 swap(j, *i); 2058 2059 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { 2060 unsigned long t; 2061 2062 if (mo_pfn <= spfn) 2063 break; 2064 2065 t = min(mo_pfn, epfn); 2066 deferred_free_pages(spfn, t); 2067 2068 if (mo_pfn <= epfn) 2069 break; 2070 } 2071 2072 return nr_pages; 2073 } 2074 2075 static void __init 2076 deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, 2077 void *arg) 2078 { 2079 unsigned long spfn, epfn; 2080 struct zone *zone = arg; 2081 u64 i; 2082 2083 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); 2084 2085 /* 2086 * Initialize and free pages in MAX_ORDER sized increments so that we 2087 * can avoid introducing any issues with the buddy allocator. 2088 */ 2089 while (spfn < end_pfn) { 2090 deferred_init_maxorder(&i, zone, &spfn, &epfn); 2091 cond_resched(); 2092 } 2093 } 2094 2095 /* An arch may override for more concurrency. */ 2096 __weak int __init 2097 deferred_page_init_max_threads(const struct cpumask *node_cpumask) 2098 { 2099 return 1; 2100 } 2101 2102 /* Initialise remaining memory on a node */ 2103 static int __init deferred_init_memmap(void *data) 2104 { 2105 pg_data_t *pgdat = data; 2106 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 2107 unsigned long spfn = 0, epfn = 0; 2108 unsigned long first_init_pfn, flags; 2109 unsigned long start = jiffies; 2110 struct zone *zone; 2111 int zid, max_threads; 2112 u64 i; 2113 2114 /* Bind memory initialisation thread to a local node if possible */ 2115 if (!cpumask_empty(cpumask)) 2116 set_cpus_allowed_ptr(current, cpumask); 2117 2118 pgdat_resize_lock(pgdat, &flags); 2119 first_init_pfn = pgdat->first_deferred_pfn; 2120 if (first_init_pfn == ULONG_MAX) { 2121 pgdat_resize_unlock(pgdat, &flags); 2122 pgdat_init_report_one_done(); 2123 return 0; 2124 } 2125 2126 /* Sanity check boundaries */ 2127 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 2128 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 2129 pgdat->first_deferred_pfn = ULONG_MAX; 2130 2131 /* 2132 * Once we unlock here, the zone cannot be grown anymore, thus if an 2133 * interrupt thread must allocate this early in boot, zone must be 2134 * pre-grown prior to start of deferred page initialization. 2135 */ 2136 pgdat_resize_unlock(pgdat, &flags); 2137 2138 /* Only the highest zone is deferred so find it */ 2139 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 2140 zone = pgdat->node_zones + zid; 2141 if (first_init_pfn < zone_end_pfn(zone)) 2142 break; 2143 } 2144 2145 /* If the zone is empty somebody else may have cleared out the zone */ 2146 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2147 first_init_pfn)) 2148 goto zone_empty; 2149 2150 max_threads = deferred_page_init_max_threads(cpumask); 2151 2152 while (spfn < epfn) { 2153 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION); 2154 struct padata_mt_job job = { 2155 .thread_fn = deferred_init_memmap_chunk, 2156 .fn_arg = zone, 2157 .start = spfn, 2158 .size = epfn_align - spfn, 2159 .align = PAGES_PER_SECTION, 2160 .min_chunk = PAGES_PER_SECTION, 2161 .max_threads = max_threads, 2162 }; 2163 2164 padata_do_multithreaded(&job); 2165 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2166 epfn_align); 2167 } 2168 zone_empty: 2169 /* Sanity check that the next zone really is unpopulated */ 2170 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 2171 2172 pr_info("node %d deferred pages initialised in %ums\n", 2173 pgdat->node_id, jiffies_to_msecs(jiffies - start)); 2174 2175 pgdat_init_report_one_done(); 2176 return 0; 2177 } 2178 2179 /* 2180 * If this zone has deferred pages, try to grow it by initializing enough 2181 * deferred pages to satisfy the allocation specified by order, rounded up to 2182 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 2183 * of SECTION_SIZE bytes by initializing struct pages in increments of 2184 * PAGES_PER_SECTION * sizeof(struct page) bytes. 2185 * 2186 * Return true when zone was grown, otherwise return false. We return true even 2187 * when we grow less than requested, to let the caller decide if there are 2188 * enough pages to satisfy the allocation. 2189 * 2190 * Note: We use noinline because this function is needed only during boot, and 2191 * it is called from a __ref function _deferred_grow_zone. This way we are 2192 * making sure that it is not inlined into permanent text section. 2193 */ 2194 static noinline bool __init 2195 deferred_grow_zone(struct zone *zone, unsigned int order) 2196 { 2197 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 2198 pg_data_t *pgdat = zone->zone_pgdat; 2199 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 2200 unsigned long spfn, epfn, flags; 2201 unsigned long nr_pages = 0; 2202 u64 i; 2203 2204 /* Only the last zone may have deferred pages */ 2205 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 2206 return false; 2207 2208 pgdat_resize_lock(pgdat, &flags); 2209 2210 /* 2211 * If someone grew this zone while we were waiting for spinlock, return 2212 * true, as there might be enough pages already. 2213 */ 2214 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 2215 pgdat_resize_unlock(pgdat, &flags); 2216 return true; 2217 } 2218 2219 /* If the zone is empty somebody else may have cleared out the zone */ 2220 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2221 first_deferred_pfn)) { 2222 pgdat->first_deferred_pfn = ULONG_MAX; 2223 pgdat_resize_unlock(pgdat, &flags); 2224 /* Retry only once. */ 2225 return first_deferred_pfn != ULONG_MAX; 2226 } 2227 2228 /* 2229 * Initialize and free pages in MAX_ORDER sized increments so 2230 * that we can avoid introducing any issues with the buddy 2231 * allocator. 2232 */ 2233 while (spfn < epfn) { 2234 /* update our first deferred PFN for this section */ 2235 first_deferred_pfn = spfn; 2236 2237 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 2238 touch_nmi_watchdog(); 2239 2240 /* We should only stop along section boundaries */ 2241 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) 2242 continue; 2243 2244 /* If our quota has been met we can stop here */ 2245 if (nr_pages >= nr_pages_needed) 2246 break; 2247 } 2248 2249 pgdat->first_deferred_pfn = spfn; 2250 pgdat_resize_unlock(pgdat, &flags); 2251 2252 return nr_pages > 0; 2253 } 2254 2255 /* 2256 * deferred_grow_zone() is __init, but it is called from 2257 * get_page_from_freelist() during early boot until deferred_pages permanently 2258 * disables this call. This is why we have refdata wrapper to avoid warning, 2259 * and to ensure that the function body gets unloaded. 2260 */ 2261 static bool __ref 2262 _deferred_grow_zone(struct zone *zone, unsigned int order) 2263 { 2264 return deferred_grow_zone(zone, order); 2265 } 2266 2267 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 2268 2269 void __init page_alloc_init_late(void) 2270 { 2271 struct zone *zone; 2272 int nid; 2273 2274 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 2275 2276 /* There will be num_node_state(N_MEMORY) threads */ 2277 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 2278 for_each_node_state(nid, N_MEMORY) { 2279 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 2280 } 2281 2282 /* Block until all are initialised */ 2283 wait_for_completion(&pgdat_init_all_done_comp); 2284 2285 /* 2286 * We initialized the rest of the deferred pages. Permanently disable 2287 * on-demand struct page initialization. 2288 */ 2289 static_branch_disable(&deferred_pages); 2290 2291 /* Reinit limits that are based on free pages after the kernel is up */ 2292 files_maxfiles_init(); 2293 #endif 2294 2295 buffer_init(); 2296 2297 /* Discard memblock private memory */ 2298 memblock_discard(); 2299 2300 for_each_node_state(nid, N_MEMORY) 2301 shuffle_free_memory(NODE_DATA(nid)); 2302 2303 for_each_populated_zone(zone) 2304 set_zone_contiguous(zone); 2305 } 2306 2307 #ifdef CONFIG_CMA 2308 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 2309 void __init init_cma_reserved_pageblock(struct page *page) 2310 { 2311 unsigned i = pageblock_nr_pages; 2312 struct page *p = page; 2313 2314 do { 2315 __ClearPageReserved(p); 2316 set_page_count(p, 0); 2317 } while (++p, --i); 2318 2319 set_pageblock_migratetype(page, MIGRATE_CMA); 2320 set_page_refcounted(page); 2321 __free_pages(page, pageblock_order); 2322 2323 adjust_managed_page_count(page, pageblock_nr_pages); 2324 page_zone(page)->cma_pages += pageblock_nr_pages; 2325 } 2326 #endif 2327 2328 /* 2329 * The order of subdivision here is critical for the IO subsystem. 2330 * Please do not alter this order without good reasons and regression 2331 * testing. Specifically, as large blocks of memory are subdivided, 2332 * the order in which smaller blocks are delivered depends on the order 2333 * they're subdivided in this function. This is the primary factor 2334 * influencing the order in which pages are delivered to the IO 2335 * subsystem according to empirical testing, and this is also justified 2336 * by considering the behavior of a buddy system containing a single 2337 * large block of memory acted on by a series of small allocations. 2338 * This behavior is a critical factor in sglist merging's success. 2339 * 2340 * -- nyc 2341 */ 2342 static inline void expand(struct zone *zone, struct page *page, 2343 int low, int high, int migratetype) 2344 { 2345 unsigned long size = 1 << high; 2346 2347 while (high > low) { 2348 high--; 2349 size >>= 1; 2350 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 2351 2352 /* 2353 * Mark as guard pages (or page), that will allow to 2354 * merge back to allocator when buddy will be freed. 2355 * Corresponding page table entries will not be touched, 2356 * pages will stay not present in virtual address space 2357 */ 2358 if (set_page_guard(zone, &page[size], high, migratetype)) 2359 continue; 2360 2361 add_to_free_list(&page[size], zone, high, migratetype); 2362 set_buddy_order(&page[size], high); 2363 } 2364 } 2365 2366 static void check_new_page_bad(struct page *page) 2367 { 2368 if (unlikely(page->flags & __PG_HWPOISON)) { 2369 /* Don't complain about hwpoisoned pages */ 2370 page_mapcount_reset(page); /* remove PageBuddy */ 2371 return; 2372 } 2373 2374 bad_page(page, 2375 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP)); 2376 } 2377 2378 /* 2379 * This page is about to be returned from the page allocator 2380 */ 2381 static inline int check_new_page(struct page *page) 2382 { 2383 if (likely(page_expected_state(page, 2384 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 2385 return 0; 2386 2387 check_new_page_bad(page); 2388 return 1; 2389 } 2390 2391 static bool check_new_pages(struct page *page, unsigned int order) 2392 { 2393 int i; 2394 for (i = 0; i < (1 << order); i++) { 2395 struct page *p = page + i; 2396 2397 if (unlikely(check_new_page(p))) 2398 return true; 2399 } 2400 2401 return false; 2402 } 2403 2404 #ifdef CONFIG_DEBUG_VM 2405 /* 2406 * With DEBUG_VM enabled, order-0 pages are checked for expected state when 2407 * being allocated from pcp lists. With debug_pagealloc also enabled, they are 2408 * also checked when pcp lists are refilled from the free lists. 2409 */ 2410 static inline bool check_pcp_refill(struct page *page, unsigned int order) 2411 { 2412 if (debug_pagealloc_enabled_static()) 2413 return check_new_pages(page, order); 2414 else 2415 return false; 2416 } 2417 2418 static inline bool check_new_pcp(struct page *page, unsigned int order) 2419 { 2420 return check_new_pages(page, order); 2421 } 2422 #else 2423 /* 2424 * With DEBUG_VM disabled, free order-0 pages are checked for expected state 2425 * when pcp lists are being refilled from the free lists. With debug_pagealloc 2426 * enabled, they are also checked when being allocated from the pcp lists. 2427 */ 2428 static inline bool check_pcp_refill(struct page *page, unsigned int order) 2429 { 2430 return check_new_pages(page, order); 2431 } 2432 static inline bool check_new_pcp(struct page *page, unsigned int order) 2433 { 2434 if (debug_pagealloc_enabled_static()) 2435 return check_new_pages(page, order); 2436 else 2437 return false; 2438 } 2439 #endif /* CONFIG_DEBUG_VM */ 2440 2441 static inline bool should_skip_kasan_unpoison(gfp_t flags) 2442 { 2443 /* Don't skip if a software KASAN mode is enabled. */ 2444 if (IS_ENABLED(CONFIG_KASAN_GENERIC) || 2445 IS_ENABLED(CONFIG_KASAN_SW_TAGS)) 2446 return false; 2447 2448 /* Skip, if hardware tag-based KASAN is not enabled. */ 2449 if (!kasan_hw_tags_enabled()) 2450 return true; 2451 2452 /* 2453 * With hardware tag-based KASAN enabled, skip if this has been 2454 * requested via __GFP_SKIP_KASAN_UNPOISON. 2455 */ 2456 return flags & __GFP_SKIP_KASAN_UNPOISON; 2457 } 2458 2459 static inline bool should_skip_init(gfp_t flags) 2460 { 2461 /* Don't skip, if hardware tag-based KASAN is not enabled. */ 2462 if (!kasan_hw_tags_enabled()) 2463 return false; 2464 2465 /* For hardware tag-based KASAN, skip if requested. */ 2466 return (flags & __GFP_SKIP_ZERO); 2467 } 2468 2469 inline void post_alloc_hook(struct page *page, unsigned int order, 2470 gfp_t gfp_flags) 2471 { 2472 bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) && 2473 !should_skip_init(gfp_flags); 2474 bool init_tags = init && (gfp_flags & __GFP_ZEROTAGS); 2475 int i; 2476 2477 set_page_private(page, 0); 2478 set_page_refcounted(page); 2479 2480 arch_alloc_page(page, order); 2481 debug_pagealloc_map_pages(page, 1 << order); 2482 2483 /* 2484 * Page unpoisoning must happen before memory initialization. 2485 * Otherwise, the poison pattern will be overwritten for __GFP_ZERO 2486 * allocations and the page unpoisoning code will complain. 2487 */ 2488 kernel_unpoison_pages(page, 1 << order); 2489 2490 /* 2491 * As memory initialization might be integrated into KASAN, 2492 * KASAN unpoisoning and memory initializion code must be 2493 * kept together to avoid discrepancies in behavior. 2494 */ 2495 2496 /* 2497 * If memory tags should be zeroed (which happens only when memory 2498 * should be initialized as well). 2499 */ 2500 if (init_tags) { 2501 /* Initialize both memory and tags. */ 2502 for (i = 0; i != 1 << order; ++i) 2503 tag_clear_highpage(page + i); 2504 2505 /* Note that memory is already initialized by the loop above. */ 2506 init = false; 2507 } 2508 if (!should_skip_kasan_unpoison(gfp_flags)) { 2509 /* Unpoison shadow memory or set memory tags. */ 2510 kasan_unpoison_pages(page, order, init); 2511 2512 /* Note that memory is already initialized by KASAN. */ 2513 if (kasan_has_integrated_init()) 2514 init = false; 2515 } else { 2516 /* Ensure page_address() dereferencing does not fault. */ 2517 for (i = 0; i != 1 << order; ++i) 2518 page_kasan_tag_reset(page + i); 2519 } 2520 /* If memory is still not initialized, do it now. */ 2521 if (init) 2522 kernel_init_pages(page, 1 << order); 2523 /* Propagate __GFP_SKIP_KASAN_POISON to page flags. */ 2524 if (kasan_hw_tags_enabled() && (gfp_flags & __GFP_SKIP_KASAN_POISON)) 2525 SetPageSkipKASanPoison(page); 2526 2527 set_page_owner(page, order, gfp_flags); 2528 page_table_check_alloc(page, order); 2529 } 2530 2531 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 2532 unsigned int alloc_flags) 2533 { 2534 post_alloc_hook(page, order, gfp_flags); 2535 2536 if (order && (gfp_flags & __GFP_COMP)) 2537 prep_compound_page(page, order); 2538 2539 /* 2540 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 2541 * allocate the page. The expectation is that the caller is taking 2542 * steps that will free more memory. The caller should avoid the page 2543 * being used for !PFMEMALLOC purposes. 2544 */ 2545 if (alloc_flags & ALLOC_NO_WATERMARKS) 2546 set_page_pfmemalloc(page); 2547 else 2548 clear_page_pfmemalloc(page); 2549 } 2550 2551 /* 2552 * Go through the free lists for the given migratetype and remove 2553 * the smallest available page from the freelists 2554 */ 2555 static __always_inline 2556 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 2557 int migratetype) 2558 { 2559 unsigned int current_order; 2560 struct free_area *area; 2561 struct page *page; 2562 2563 /* Find a page of the appropriate size in the preferred list */ 2564 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 2565 area = &(zone->free_area[current_order]); 2566 page = get_page_from_free_area(area, migratetype); 2567 if (!page) 2568 continue; 2569 del_page_from_free_list(page, zone, current_order); 2570 expand(zone, page, order, current_order, migratetype); 2571 set_pcppage_migratetype(page, migratetype); 2572 trace_mm_page_alloc_zone_locked(page, order, migratetype, 2573 pcp_allowed_order(order) && 2574 migratetype < MIGRATE_PCPTYPES); 2575 return page; 2576 } 2577 2578 return NULL; 2579 } 2580 2581 2582 /* 2583 * This array describes the order lists are fallen back to when 2584 * the free lists for the desirable migrate type are depleted 2585 * 2586 * The other migratetypes do not have fallbacks. 2587 */ 2588 static int fallbacks[MIGRATE_TYPES][3] = { 2589 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2590 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 2591 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2592 }; 2593 2594 #ifdef CONFIG_CMA 2595 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2596 unsigned int order) 2597 { 2598 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 2599 } 2600 #else 2601 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2602 unsigned int order) { return NULL; } 2603 #endif 2604 2605 /* 2606 * Move the free pages in a range to the freelist tail of the requested type. 2607 * Note that start_page and end_pages are not aligned on a pageblock 2608 * boundary. If alignment is required, use move_freepages_block() 2609 */ 2610 static int move_freepages(struct zone *zone, 2611 unsigned long start_pfn, unsigned long end_pfn, 2612 int migratetype, int *num_movable) 2613 { 2614 struct page *page; 2615 unsigned long pfn; 2616 unsigned int order; 2617 int pages_moved = 0; 2618 2619 for (pfn = start_pfn; pfn <= end_pfn;) { 2620 page = pfn_to_page(pfn); 2621 if (!PageBuddy(page)) { 2622 /* 2623 * We assume that pages that could be isolated for 2624 * migration are movable. But we don't actually try 2625 * isolating, as that would be expensive. 2626 */ 2627 if (num_movable && 2628 (PageLRU(page) || __PageMovable(page))) 2629 (*num_movable)++; 2630 pfn++; 2631 continue; 2632 } 2633 2634 /* Make sure we are not inadvertently changing nodes */ 2635 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2636 VM_BUG_ON_PAGE(page_zone(page) != zone, page); 2637 2638 order = buddy_order(page); 2639 move_to_free_list(page, zone, order, migratetype); 2640 pfn += 1 << order; 2641 pages_moved += 1 << order; 2642 } 2643 2644 return pages_moved; 2645 } 2646 2647 int move_freepages_block(struct zone *zone, struct page *page, 2648 int migratetype, int *num_movable) 2649 { 2650 unsigned long start_pfn, end_pfn, pfn; 2651 2652 if (num_movable) 2653 *num_movable = 0; 2654 2655 pfn = page_to_pfn(page); 2656 start_pfn = pfn & ~(pageblock_nr_pages - 1); 2657 end_pfn = start_pfn + pageblock_nr_pages - 1; 2658 2659 /* Do not cross zone boundaries */ 2660 if (!zone_spans_pfn(zone, start_pfn)) 2661 start_pfn = pfn; 2662 if (!zone_spans_pfn(zone, end_pfn)) 2663 return 0; 2664 2665 return move_freepages(zone, start_pfn, end_pfn, migratetype, 2666 num_movable); 2667 } 2668 2669 static void change_pageblock_range(struct page *pageblock_page, 2670 int start_order, int migratetype) 2671 { 2672 int nr_pageblocks = 1 << (start_order - pageblock_order); 2673 2674 while (nr_pageblocks--) { 2675 set_pageblock_migratetype(pageblock_page, migratetype); 2676 pageblock_page += pageblock_nr_pages; 2677 } 2678 } 2679 2680 /* 2681 * When we are falling back to another migratetype during allocation, try to 2682 * steal extra free pages from the same pageblocks to satisfy further 2683 * allocations, instead of polluting multiple pageblocks. 2684 * 2685 * If we are stealing a relatively large buddy page, it is likely there will 2686 * be more free pages in the pageblock, so try to steal them all. For 2687 * reclaimable and unmovable allocations, we steal regardless of page size, 2688 * as fragmentation caused by those allocations polluting movable pageblocks 2689 * is worse than movable allocations stealing from unmovable and reclaimable 2690 * pageblocks. 2691 */ 2692 static bool can_steal_fallback(unsigned int order, int start_mt) 2693 { 2694 /* 2695 * Leaving this order check is intended, although there is 2696 * relaxed order check in next check. The reason is that 2697 * we can actually steal whole pageblock if this condition met, 2698 * but, below check doesn't guarantee it and that is just heuristic 2699 * so could be changed anytime. 2700 */ 2701 if (order >= pageblock_order) 2702 return true; 2703 2704 if (order >= pageblock_order / 2 || 2705 start_mt == MIGRATE_RECLAIMABLE || 2706 start_mt == MIGRATE_UNMOVABLE || 2707 page_group_by_mobility_disabled) 2708 return true; 2709 2710 return false; 2711 } 2712 2713 static inline bool boost_watermark(struct zone *zone) 2714 { 2715 unsigned long max_boost; 2716 2717 if (!watermark_boost_factor) 2718 return false; 2719 /* 2720 * Don't bother in zones that are unlikely to produce results. 2721 * On small machines, including kdump capture kernels running 2722 * in a small area, boosting the watermark can cause an out of 2723 * memory situation immediately. 2724 */ 2725 if ((pageblock_nr_pages * 4) > zone_managed_pages(zone)) 2726 return false; 2727 2728 max_boost = mult_frac(zone->_watermark[WMARK_HIGH], 2729 watermark_boost_factor, 10000); 2730 2731 /* 2732 * high watermark may be uninitialised if fragmentation occurs 2733 * very early in boot so do not boost. We do not fall 2734 * through and boost by pageblock_nr_pages as failing 2735 * allocations that early means that reclaim is not going 2736 * to help and it may even be impossible to reclaim the 2737 * boosted watermark resulting in a hang. 2738 */ 2739 if (!max_boost) 2740 return false; 2741 2742 max_boost = max(pageblock_nr_pages, max_boost); 2743 2744 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages, 2745 max_boost); 2746 2747 return true; 2748 } 2749 2750 /* 2751 * This function implements actual steal behaviour. If order is large enough, 2752 * we can steal whole pageblock. If not, we first move freepages in this 2753 * pageblock to our migratetype and determine how many already-allocated pages 2754 * are there in the pageblock with a compatible migratetype. If at least half 2755 * of pages are free or compatible, we can change migratetype of the pageblock 2756 * itself, so pages freed in the future will be put on the correct free list. 2757 */ 2758 static void steal_suitable_fallback(struct zone *zone, struct page *page, 2759 unsigned int alloc_flags, int start_type, bool whole_block) 2760 { 2761 unsigned int current_order = buddy_order(page); 2762 int free_pages, movable_pages, alike_pages; 2763 int old_block_type; 2764 2765 old_block_type = get_pageblock_migratetype(page); 2766 2767 /* 2768 * This can happen due to races and we want to prevent broken 2769 * highatomic accounting. 2770 */ 2771 if (is_migrate_highatomic(old_block_type)) 2772 goto single_page; 2773 2774 /* Take ownership for orders >= pageblock_order */ 2775 if (current_order >= pageblock_order) { 2776 change_pageblock_range(page, current_order, start_type); 2777 goto single_page; 2778 } 2779 2780 /* 2781 * Boost watermarks to increase reclaim pressure to reduce the 2782 * likelihood of future fallbacks. Wake kswapd now as the node 2783 * may be balanced overall and kswapd will not wake naturally. 2784 */ 2785 if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD)) 2786 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 2787 2788 /* We are not allowed to try stealing from the whole block */ 2789 if (!whole_block) 2790 goto single_page; 2791 2792 free_pages = move_freepages_block(zone, page, start_type, 2793 &movable_pages); 2794 /* 2795 * Determine how many pages are compatible with our allocation. 2796 * For movable allocation, it's the number of movable pages which 2797 * we just obtained. For other types it's a bit more tricky. 2798 */ 2799 if (start_type == MIGRATE_MOVABLE) { 2800 alike_pages = movable_pages; 2801 } else { 2802 /* 2803 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation 2804 * to MOVABLE pageblock, consider all non-movable pages as 2805 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or 2806 * vice versa, be conservative since we can't distinguish the 2807 * exact migratetype of non-movable pages. 2808 */ 2809 if (old_block_type == MIGRATE_MOVABLE) 2810 alike_pages = pageblock_nr_pages 2811 - (free_pages + movable_pages); 2812 else 2813 alike_pages = 0; 2814 } 2815 2816 /* moving whole block can fail due to zone boundary conditions */ 2817 if (!free_pages) 2818 goto single_page; 2819 2820 /* 2821 * If a sufficient number of pages in the block are either free or of 2822 * comparable migratability as our allocation, claim the whole block. 2823 */ 2824 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || 2825 page_group_by_mobility_disabled) 2826 set_pageblock_migratetype(page, start_type); 2827 2828 return; 2829 2830 single_page: 2831 move_to_free_list(page, zone, current_order, start_type); 2832 } 2833 2834 /* 2835 * Check whether there is a suitable fallback freepage with requested order. 2836 * If only_stealable is true, this function returns fallback_mt only if 2837 * we can steal other freepages all together. This would help to reduce 2838 * fragmentation due to mixed migratetype pages in one pageblock. 2839 */ 2840 int find_suitable_fallback(struct free_area *area, unsigned int order, 2841 int migratetype, bool only_stealable, bool *can_steal) 2842 { 2843 int i; 2844 int fallback_mt; 2845 2846 if (area->nr_free == 0) 2847 return -1; 2848 2849 *can_steal = false; 2850 for (i = 0;; i++) { 2851 fallback_mt = fallbacks[migratetype][i]; 2852 if (fallback_mt == MIGRATE_TYPES) 2853 break; 2854 2855 if (free_area_empty(area, fallback_mt)) 2856 continue; 2857 2858 if (can_steal_fallback(order, migratetype)) 2859 *can_steal = true; 2860 2861 if (!only_stealable) 2862 return fallback_mt; 2863 2864 if (*can_steal) 2865 return fallback_mt; 2866 } 2867 2868 return -1; 2869 } 2870 2871 /* 2872 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2873 * there are no empty page blocks that contain a page with a suitable order 2874 */ 2875 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2876 unsigned int alloc_order) 2877 { 2878 int mt; 2879 unsigned long max_managed, flags; 2880 2881 /* 2882 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2883 * Check is race-prone but harmless. 2884 */ 2885 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages; 2886 if (zone->nr_reserved_highatomic >= max_managed) 2887 return; 2888 2889 spin_lock_irqsave(&zone->lock, flags); 2890 2891 /* Recheck the nr_reserved_highatomic limit under the lock */ 2892 if (zone->nr_reserved_highatomic >= max_managed) 2893 goto out_unlock; 2894 2895 /* Yoink! */ 2896 mt = get_pageblock_migratetype(page); 2897 /* Only reserve normal pageblocks (i.e., they can merge with others) */ 2898 if (migratetype_is_mergeable(mt)) { 2899 zone->nr_reserved_highatomic += pageblock_nr_pages; 2900 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2901 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); 2902 } 2903 2904 out_unlock: 2905 spin_unlock_irqrestore(&zone->lock, flags); 2906 } 2907 2908 /* 2909 * Used when an allocation is about to fail under memory pressure. This 2910 * potentially hurts the reliability of high-order allocations when under 2911 * intense memory pressure but failed atomic allocations should be easier 2912 * to recover from than an OOM. 2913 * 2914 * If @force is true, try to unreserve a pageblock even though highatomic 2915 * pageblock is exhausted. 2916 */ 2917 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 2918 bool force) 2919 { 2920 struct zonelist *zonelist = ac->zonelist; 2921 unsigned long flags; 2922 struct zoneref *z; 2923 struct zone *zone; 2924 struct page *page; 2925 int order; 2926 bool ret; 2927 2928 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx, 2929 ac->nodemask) { 2930 /* 2931 * Preserve at least one pageblock unless memory pressure 2932 * is really high. 2933 */ 2934 if (!force && zone->nr_reserved_highatomic <= 2935 pageblock_nr_pages) 2936 continue; 2937 2938 spin_lock_irqsave(&zone->lock, flags); 2939 for (order = 0; order < MAX_ORDER; order++) { 2940 struct free_area *area = &(zone->free_area[order]); 2941 2942 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); 2943 if (!page) 2944 continue; 2945 2946 /* 2947 * In page freeing path, migratetype change is racy so 2948 * we can counter several free pages in a pageblock 2949 * in this loop although we changed the pageblock type 2950 * from highatomic to ac->migratetype. So we should 2951 * adjust the count once. 2952 */ 2953 if (is_migrate_highatomic_page(page)) { 2954 /* 2955 * It should never happen but changes to 2956 * locking could inadvertently allow a per-cpu 2957 * drain to add pages to MIGRATE_HIGHATOMIC 2958 * while unreserving so be safe and watch for 2959 * underflows. 2960 */ 2961 zone->nr_reserved_highatomic -= min( 2962 pageblock_nr_pages, 2963 zone->nr_reserved_highatomic); 2964 } 2965 2966 /* 2967 * Convert to ac->migratetype and avoid the normal 2968 * pageblock stealing heuristics. Minimally, the caller 2969 * is doing the work and needs the pages. More 2970 * importantly, if the block was always converted to 2971 * MIGRATE_UNMOVABLE or another type then the number 2972 * of pageblocks that cannot be completely freed 2973 * may increase. 2974 */ 2975 set_pageblock_migratetype(page, ac->migratetype); 2976 ret = move_freepages_block(zone, page, ac->migratetype, 2977 NULL); 2978 if (ret) { 2979 spin_unlock_irqrestore(&zone->lock, flags); 2980 return ret; 2981 } 2982 } 2983 spin_unlock_irqrestore(&zone->lock, flags); 2984 } 2985 2986 return false; 2987 } 2988 2989 /* 2990 * Try finding a free buddy page on the fallback list and put it on the free 2991 * list of requested migratetype, possibly along with other pages from the same 2992 * block, depending on fragmentation avoidance heuristics. Returns true if 2993 * fallback was found so that __rmqueue_smallest() can grab it. 2994 * 2995 * The use of signed ints for order and current_order is a deliberate 2996 * deviation from the rest of this file, to make the for loop 2997 * condition simpler. 2998 */ 2999 static __always_inline bool 3000 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, 3001 unsigned int alloc_flags) 3002 { 3003 struct free_area *area; 3004 int current_order; 3005 int min_order = order; 3006 struct page *page; 3007 int fallback_mt; 3008 bool can_steal; 3009 3010 /* 3011 * Do not steal pages from freelists belonging to other pageblocks 3012 * i.e. orders < pageblock_order. If there are no local zones free, 3013 * the zonelists will be reiterated without ALLOC_NOFRAGMENT. 3014 */ 3015 if (order < pageblock_order && alloc_flags & ALLOC_NOFRAGMENT) 3016 min_order = pageblock_order; 3017 3018 /* 3019 * Find the largest available free page in the other list. This roughly 3020 * approximates finding the pageblock with the most free pages, which 3021 * would be too costly to do exactly. 3022 */ 3023 for (current_order = MAX_ORDER - 1; current_order >= min_order; 3024 --current_order) { 3025 area = &(zone->free_area[current_order]); 3026 fallback_mt = find_suitable_fallback(area, current_order, 3027 start_migratetype, false, &can_steal); 3028 if (fallback_mt == -1) 3029 continue; 3030 3031 /* 3032 * We cannot steal all free pages from the pageblock and the 3033 * requested migratetype is movable. In that case it's better to 3034 * steal and split the smallest available page instead of the 3035 * largest available page, because even if the next movable 3036 * allocation falls back into a different pageblock than this 3037 * one, it won't cause permanent fragmentation. 3038 */ 3039 if (!can_steal && start_migratetype == MIGRATE_MOVABLE 3040 && current_order > order) 3041 goto find_smallest; 3042 3043 goto do_steal; 3044 } 3045 3046 return false; 3047 3048 find_smallest: 3049 for (current_order = order; current_order < MAX_ORDER; 3050 current_order++) { 3051 area = &(zone->free_area[current_order]); 3052 fallback_mt = find_suitable_fallback(area, current_order, 3053 start_migratetype, false, &can_steal); 3054 if (fallback_mt != -1) 3055 break; 3056 } 3057 3058 /* 3059 * This should not happen - we already found a suitable fallback 3060 * when looking for the largest page. 3061 */ 3062 VM_BUG_ON(current_order == MAX_ORDER); 3063 3064 do_steal: 3065 page = get_page_from_free_area(area, fallback_mt); 3066 3067 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype, 3068 can_steal); 3069 3070 trace_mm_page_alloc_extfrag(page, order, current_order, 3071 start_migratetype, fallback_mt); 3072 3073 return true; 3074 3075 } 3076 3077 /* 3078 * Do the hard work of removing an element from the buddy allocator. 3079 * Call me with the zone->lock already held. 3080 */ 3081 static __always_inline struct page * 3082 __rmqueue(struct zone *zone, unsigned int order, int migratetype, 3083 unsigned int alloc_flags) 3084 { 3085 struct page *page; 3086 3087 if (IS_ENABLED(CONFIG_CMA)) { 3088 /* 3089 * Balance movable allocations between regular and CMA areas by 3090 * allocating from CMA when over half of the zone's free memory 3091 * is in the CMA area. 3092 */ 3093 if (alloc_flags & ALLOC_CMA && 3094 zone_page_state(zone, NR_FREE_CMA_PAGES) > 3095 zone_page_state(zone, NR_FREE_PAGES) / 2) { 3096 page = __rmqueue_cma_fallback(zone, order); 3097 if (page) 3098 return page; 3099 } 3100 } 3101 retry: 3102 page = __rmqueue_smallest(zone, order, migratetype); 3103 if (unlikely(!page)) { 3104 if (alloc_flags & ALLOC_CMA) 3105 page = __rmqueue_cma_fallback(zone, order); 3106 3107 if (!page && __rmqueue_fallback(zone, order, migratetype, 3108 alloc_flags)) 3109 goto retry; 3110 } 3111 return page; 3112 } 3113 3114 /* 3115 * Obtain a specified number of elements from the buddy allocator, all under 3116 * a single hold of the lock, for efficiency. Add them to the supplied list. 3117 * Returns the number of new pages which were placed at *list. 3118 */ 3119 static int rmqueue_bulk(struct zone *zone, unsigned int order, 3120 unsigned long count, struct list_head *list, 3121 int migratetype, unsigned int alloc_flags) 3122 { 3123 int i, allocated = 0; 3124 3125 /* Caller must hold IRQ-safe pcp->lock so IRQs are disabled. */ 3126 spin_lock(&zone->lock); 3127 for (i = 0; i < count; ++i) { 3128 struct page *page = __rmqueue(zone, order, migratetype, 3129 alloc_flags); 3130 if (unlikely(page == NULL)) 3131 break; 3132 3133 if (unlikely(check_pcp_refill(page, order))) 3134 continue; 3135 3136 /* 3137 * Split buddy pages returned by expand() are received here in 3138 * physical page order. The page is added to the tail of 3139 * caller's list. From the callers perspective, the linked list 3140 * is ordered by page number under some conditions. This is 3141 * useful for IO devices that can forward direction from the 3142 * head, thus also in the physical page order. This is useful 3143 * for IO devices that can merge IO requests if the physical 3144 * pages are ordered properly. 3145 */ 3146 list_add_tail(&page->pcp_list, list); 3147 allocated++; 3148 if (is_migrate_cma(get_pcppage_migratetype(page))) 3149 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 3150 -(1 << order)); 3151 } 3152 3153 /* 3154 * i pages were removed from the buddy list even if some leak due 3155 * to check_pcp_refill failing so adjust NR_FREE_PAGES based 3156 * on i. Do not confuse with 'allocated' which is the number of 3157 * pages added to the pcp list. 3158 */ 3159 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 3160 spin_unlock(&zone->lock); 3161 return allocated; 3162 } 3163 3164 #ifdef CONFIG_NUMA 3165 /* 3166 * Called from the vmstat counter updater to drain pagesets of this 3167 * currently executing processor on remote nodes after they have 3168 * expired. 3169 */ 3170 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 3171 { 3172 int to_drain, batch; 3173 3174 batch = READ_ONCE(pcp->batch); 3175 to_drain = min(pcp->count, batch); 3176 if (to_drain > 0) { 3177 unsigned long flags; 3178 3179 /* 3180 * free_pcppages_bulk expects IRQs disabled for zone->lock 3181 * so even though pcp->lock is not intended to be IRQ-safe, 3182 * it's needed in this context. 3183 */ 3184 spin_lock_irqsave(&pcp->lock, flags); 3185 free_pcppages_bulk(zone, to_drain, pcp, 0); 3186 spin_unlock_irqrestore(&pcp->lock, flags); 3187 } 3188 } 3189 #endif 3190 3191 /* 3192 * Drain pcplists of the indicated processor and zone. 3193 */ 3194 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 3195 { 3196 struct per_cpu_pages *pcp; 3197 3198 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 3199 if (pcp->count) { 3200 unsigned long flags; 3201 3202 /* See drain_zone_pages on why this is disabling IRQs */ 3203 spin_lock_irqsave(&pcp->lock, flags); 3204 free_pcppages_bulk(zone, pcp->count, pcp, 0); 3205 spin_unlock_irqrestore(&pcp->lock, flags); 3206 } 3207 } 3208 3209 /* 3210 * Drain pcplists of all zones on the indicated processor. 3211 */ 3212 static void drain_pages(unsigned int cpu) 3213 { 3214 struct zone *zone; 3215 3216 for_each_populated_zone(zone) { 3217 drain_pages_zone(cpu, zone); 3218 } 3219 } 3220 3221 /* 3222 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 3223 */ 3224 void drain_local_pages(struct zone *zone) 3225 { 3226 int cpu = smp_processor_id(); 3227 3228 if (zone) 3229 drain_pages_zone(cpu, zone); 3230 else 3231 drain_pages(cpu); 3232 } 3233 3234 /* 3235 * The implementation of drain_all_pages(), exposing an extra parameter to 3236 * drain on all cpus. 3237 * 3238 * drain_all_pages() is optimized to only execute on cpus where pcplists are 3239 * not empty. The check for non-emptiness can however race with a free to 3240 * pcplist that has not yet increased the pcp->count from 0 to 1. Callers 3241 * that need the guarantee that every CPU has drained can disable the 3242 * optimizing racy check. 3243 */ 3244 static void __drain_all_pages(struct zone *zone, bool force_all_cpus) 3245 { 3246 int cpu; 3247 3248 /* 3249 * Allocate in the BSS so we won't require allocation in 3250 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 3251 */ 3252 static cpumask_t cpus_with_pcps; 3253 3254 /* 3255 * Do not drain if one is already in progress unless it's specific to 3256 * a zone. Such callers are primarily CMA and memory hotplug and need 3257 * the drain to be complete when the call returns. 3258 */ 3259 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) { 3260 if (!zone) 3261 return; 3262 mutex_lock(&pcpu_drain_mutex); 3263 } 3264 3265 /* 3266 * We don't care about racing with CPU hotplug event 3267 * as offline notification will cause the notified 3268 * cpu to drain that CPU pcps and on_each_cpu_mask 3269 * disables preemption as part of its processing 3270 */ 3271 for_each_online_cpu(cpu) { 3272 struct per_cpu_pages *pcp; 3273 struct zone *z; 3274 bool has_pcps = false; 3275 3276 if (force_all_cpus) { 3277 /* 3278 * The pcp.count check is racy, some callers need a 3279 * guarantee that no cpu is missed. 3280 */ 3281 has_pcps = true; 3282 } else if (zone) { 3283 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 3284 if (pcp->count) 3285 has_pcps = true; 3286 } else { 3287 for_each_populated_zone(z) { 3288 pcp = per_cpu_ptr(z->per_cpu_pageset, cpu); 3289 if (pcp->count) { 3290 has_pcps = true; 3291 break; 3292 } 3293 } 3294 } 3295 3296 if (has_pcps) 3297 cpumask_set_cpu(cpu, &cpus_with_pcps); 3298 else 3299 cpumask_clear_cpu(cpu, &cpus_with_pcps); 3300 } 3301 3302 for_each_cpu(cpu, &cpus_with_pcps) { 3303 if (zone) 3304 drain_pages_zone(cpu, zone); 3305 else 3306 drain_pages(cpu); 3307 } 3308 3309 mutex_unlock(&pcpu_drain_mutex); 3310 } 3311 3312 /* 3313 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 3314 * 3315 * When zone parameter is non-NULL, spill just the single zone's pages. 3316 */ 3317 void drain_all_pages(struct zone *zone) 3318 { 3319 __drain_all_pages(zone, false); 3320 } 3321 3322 #ifdef CONFIG_HIBERNATION 3323 3324 /* 3325 * Touch the watchdog for every WD_PAGE_COUNT pages. 3326 */ 3327 #define WD_PAGE_COUNT (128*1024) 3328 3329 void mark_free_pages(struct zone *zone) 3330 { 3331 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT; 3332 unsigned long flags; 3333 unsigned int order, t; 3334 struct page *page; 3335 3336 if (zone_is_empty(zone)) 3337 return; 3338 3339 spin_lock_irqsave(&zone->lock, flags); 3340 3341 max_zone_pfn = zone_end_pfn(zone); 3342 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 3343 if (pfn_valid(pfn)) { 3344 page = pfn_to_page(pfn); 3345 3346 if (!--page_count) { 3347 touch_nmi_watchdog(); 3348 page_count = WD_PAGE_COUNT; 3349 } 3350 3351 if (page_zone(page) != zone) 3352 continue; 3353 3354 if (!swsusp_page_is_forbidden(page)) 3355 swsusp_unset_page_free(page); 3356 } 3357 3358 for_each_migratetype_order(order, t) { 3359 list_for_each_entry(page, 3360 &zone->free_area[order].free_list[t], buddy_list) { 3361 unsigned long i; 3362 3363 pfn = page_to_pfn(page); 3364 for (i = 0; i < (1UL << order); i++) { 3365 if (!--page_count) { 3366 touch_nmi_watchdog(); 3367 page_count = WD_PAGE_COUNT; 3368 } 3369 swsusp_set_page_free(pfn_to_page(pfn + i)); 3370 } 3371 } 3372 } 3373 spin_unlock_irqrestore(&zone->lock, flags); 3374 } 3375 #endif /* CONFIG_PM */ 3376 3377 static bool free_unref_page_prepare(struct page *page, unsigned long pfn, 3378 unsigned int order) 3379 { 3380 int migratetype; 3381 3382 if (!free_pcp_prepare(page, order)) 3383 return false; 3384 3385 migratetype = get_pfnblock_migratetype(page, pfn); 3386 set_pcppage_migratetype(page, migratetype); 3387 return true; 3388 } 3389 3390 static int nr_pcp_free(struct per_cpu_pages *pcp, int high, int batch, 3391 bool free_high) 3392 { 3393 int min_nr_free, max_nr_free; 3394 3395 /* Free everything if batch freeing high-order pages. */ 3396 if (unlikely(free_high)) 3397 return pcp->count; 3398 3399 /* Check for PCP disabled or boot pageset */ 3400 if (unlikely(high < batch)) 3401 return 1; 3402 3403 /* Leave at least pcp->batch pages on the list */ 3404 min_nr_free = batch; 3405 max_nr_free = high - batch; 3406 3407 /* 3408 * Double the number of pages freed each time there is subsequent 3409 * freeing of pages without any allocation. 3410 */ 3411 batch <<= pcp->free_factor; 3412 if (batch < max_nr_free) 3413 pcp->free_factor++; 3414 batch = clamp(batch, min_nr_free, max_nr_free); 3415 3416 return batch; 3417 } 3418 3419 static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone, 3420 bool free_high) 3421 { 3422 int high = READ_ONCE(pcp->high); 3423 3424 if (unlikely(!high || free_high)) 3425 return 0; 3426 3427 if (!test_bit(ZONE_RECLAIM_ACTIVE, &zone->flags)) 3428 return high; 3429 3430 /* 3431 * If reclaim is active, limit the number of pages that can be 3432 * stored on pcp lists 3433 */ 3434 return min(READ_ONCE(pcp->batch) << 2, high); 3435 } 3436 3437 static void free_unref_page_commit(struct zone *zone, struct per_cpu_pages *pcp, 3438 struct page *page, int migratetype, 3439 unsigned int order) 3440 { 3441 int high; 3442 int pindex; 3443 bool free_high; 3444 3445 __count_vm_event(PGFREE); 3446 pindex = order_to_pindex(migratetype, order); 3447 list_add(&page->pcp_list, &pcp->lists[pindex]); 3448 pcp->count += 1 << order; 3449 3450 /* 3451 * As high-order pages other than THP's stored on PCP can contribute 3452 * to fragmentation, limit the number stored when PCP is heavily 3453 * freeing without allocation. The remainder after bulk freeing 3454 * stops will be drained from vmstat refresh context. 3455 */ 3456 free_high = (pcp->free_factor && order && order <= PAGE_ALLOC_COSTLY_ORDER); 3457 3458 high = nr_pcp_high(pcp, zone, free_high); 3459 if (pcp->count >= high) { 3460 int batch = READ_ONCE(pcp->batch); 3461 3462 free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch, free_high), pcp, pindex); 3463 } 3464 } 3465 3466 /* 3467 * Free a pcp page 3468 */ 3469 void free_unref_page(struct page *page, unsigned int order) 3470 { 3471 unsigned long flags; 3472 unsigned long __maybe_unused UP_flags; 3473 struct per_cpu_pages *pcp; 3474 struct zone *zone; 3475 unsigned long pfn = page_to_pfn(page); 3476 int migratetype; 3477 3478 if (!free_unref_page_prepare(page, pfn, order)) 3479 return; 3480 3481 /* 3482 * We only track unmovable, reclaimable and movable on pcp lists. 3483 * Place ISOLATE pages on the isolated list because they are being 3484 * offlined but treat HIGHATOMIC as movable pages so we can get those 3485 * areas back if necessary. Otherwise, we may have to free 3486 * excessively into the page allocator 3487 */ 3488 migratetype = get_pcppage_migratetype(page); 3489 if (unlikely(migratetype >= MIGRATE_PCPTYPES)) { 3490 if (unlikely(is_migrate_isolate(migratetype))) { 3491 free_one_page(page_zone(page), page, pfn, order, migratetype, FPI_NONE); 3492 return; 3493 } 3494 migratetype = MIGRATE_MOVABLE; 3495 } 3496 3497 zone = page_zone(page); 3498 pcp_trylock_prepare(UP_flags); 3499 pcp = pcp_spin_trylock_irqsave(zone->per_cpu_pageset, flags); 3500 if (pcp) { 3501 free_unref_page_commit(zone, pcp, page, migratetype, order); 3502 pcp_spin_unlock_irqrestore(pcp, flags); 3503 } else { 3504 free_one_page(zone, page, pfn, order, migratetype, FPI_NONE); 3505 } 3506 pcp_trylock_finish(UP_flags); 3507 } 3508 3509 /* 3510 * Free a list of 0-order pages 3511 */ 3512 void free_unref_page_list(struct list_head *list) 3513 { 3514 struct page *page, *next; 3515 struct per_cpu_pages *pcp = NULL; 3516 struct zone *locked_zone = NULL; 3517 unsigned long flags; 3518 int batch_count = 0; 3519 int migratetype; 3520 3521 /* Prepare pages for freeing */ 3522 list_for_each_entry_safe(page, next, list, lru) { 3523 unsigned long pfn = page_to_pfn(page); 3524 if (!free_unref_page_prepare(page, pfn, 0)) { 3525 list_del(&page->lru); 3526 continue; 3527 } 3528 3529 /* 3530 * Free isolated pages directly to the allocator, see 3531 * comment in free_unref_page. 3532 */ 3533 migratetype = get_pcppage_migratetype(page); 3534 if (unlikely(is_migrate_isolate(migratetype))) { 3535 list_del(&page->lru); 3536 free_one_page(page_zone(page), page, pfn, 0, migratetype, FPI_NONE); 3537 continue; 3538 } 3539 } 3540 3541 list_for_each_entry_safe(page, next, list, lru) { 3542 struct zone *zone = page_zone(page); 3543 3544 /* Different zone, different pcp lock. */ 3545 if (zone != locked_zone) { 3546 if (pcp) 3547 pcp_spin_unlock_irqrestore(pcp, flags); 3548 3549 locked_zone = zone; 3550 pcp = pcp_spin_lock_irqsave(locked_zone->per_cpu_pageset, flags); 3551 } 3552 3553 /* 3554 * Non-isolated types over MIGRATE_PCPTYPES get added 3555 * to the MIGRATE_MOVABLE pcp list. 3556 */ 3557 migratetype = get_pcppage_migratetype(page); 3558 if (unlikely(migratetype >= MIGRATE_PCPTYPES)) 3559 migratetype = MIGRATE_MOVABLE; 3560 3561 trace_mm_page_free_batched(page); 3562 free_unref_page_commit(zone, pcp, page, migratetype, 0); 3563 3564 /* 3565 * Guard against excessive IRQ disabled times when we get 3566 * a large list of pages to free. 3567 */ 3568 if (++batch_count == SWAP_CLUSTER_MAX) { 3569 pcp_spin_unlock_irqrestore(pcp, flags); 3570 batch_count = 0; 3571 pcp = pcp_spin_lock_irqsave(locked_zone->per_cpu_pageset, flags); 3572 } 3573 } 3574 3575 if (pcp) 3576 pcp_spin_unlock_irqrestore(pcp, flags); 3577 } 3578 3579 /* 3580 * split_page takes a non-compound higher-order page, and splits it into 3581 * n (1<<order) sub-pages: page[0..n] 3582 * Each sub-page must be freed individually. 3583 * 3584 * Note: this is probably too low level an operation for use in drivers. 3585 * Please consult with lkml before using this in your driver. 3586 */ 3587 void split_page(struct page *page, unsigned int order) 3588 { 3589 int i; 3590 3591 VM_BUG_ON_PAGE(PageCompound(page), page); 3592 VM_BUG_ON_PAGE(!page_count(page), page); 3593 3594 for (i = 1; i < (1 << order); i++) 3595 set_page_refcounted(page + i); 3596 split_page_owner(page, 1 << order); 3597 split_page_memcg(page, 1 << order); 3598 } 3599 EXPORT_SYMBOL_GPL(split_page); 3600 3601 int __isolate_free_page(struct page *page, unsigned int order) 3602 { 3603 struct zone *zone = page_zone(page); 3604 int mt = get_pageblock_migratetype(page); 3605 3606 if (!is_migrate_isolate(mt)) { 3607 unsigned long watermark; 3608 /* 3609 * Obey watermarks as if the page was being allocated. We can 3610 * emulate a high-order watermark check with a raised order-0 3611 * watermark, because we already know our high-order page 3612 * exists. 3613 */ 3614 watermark = zone->_watermark[WMARK_MIN] + (1UL << order); 3615 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 3616 return 0; 3617 3618 __mod_zone_freepage_state(zone, -(1UL << order), mt); 3619 } 3620 3621 del_page_from_free_list(page, zone, order); 3622 3623 /* 3624 * Set the pageblock if the isolated page is at least half of a 3625 * pageblock 3626 */ 3627 if (order >= pageblock_order - 1) { 3628 struct page *endpage = page + (1 << order) - 1; 3629 for (; page < endpage; page += pageblock_nr_pages) { 3630 int mt = get_pageblock_migratetype(page); 3631 /* 3632 * Only change normal pageblocks (i.e., they can merge 3633 * with others) 3634 */ 3635 if (migratetype_is_mergeable(mt)) 3636 set_pageblock_migratetype(page, 3637 MIGRATE_MOVABLE); 3638 } 3639 } 3640 3641 return 1UL << order; 3642 } 3643 3644 /** 3645 * __putback_isolated_page - Return a now-isolated page back where we got it 3646 * @page: Page that was isolated 3647 * @order: Order of the isolated page 3648 * @mt: The page's pageblock's migratetype 3649 * 3650 * This function is meant to return a page pulled from the free lists via 3651 * __isolate_free_page back to the free lists they were pulled from. 3652 */ 3653 void __putback_isolated_page(struct page *page, unsigned int order, int mt) 3654 { 3655 struct zone *zone = page_zone(page); 3656 3657 /* zone lock should be held when this function is called */ 3658 lockdep_assert_held(&zone->lock); 3659 3660 /* Return isolated page to tail of freelist. */ 3661 __free_one_page(page, page_to_pfn(page), zone, order, mt, 3662 FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL); 3663 } 3664 3665 /* 3666 * Update NUMA hit/miss statistics 3667 * 3668 * Must be called with interrupts disabled. 3669 */ 3670 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z, 3671 long nr_account) 3672 { 3673 #ifdef CONFIG_NUMA 3674 enum numa_stat_item local_stat = NUMA_LOCAL; 3675 3676 /* skip numa counters update if numa stats is disabled */ 3677 if (!static_branch_likely(&vm_numa_stat_key)) 3678 return; 3679 3680 if (zone_to_nid(z) != numa_node_id()) 3681 local_stat = NUMA_OTHER; 3682 3683 if (zone_to_nid(z) == zone_to_nid(preferred_zone)) 3684 __count_numa_events(z, NUMA_HIT, nr_account); 3685 else { 3686 __count_numa_events(z, NUMA_MISS, nr_account); 3687 __count_numa_events(preferred_zone, NUMA_FOREIGN, nr_account); 3688 } 3689 __count_numa_events(z, local_stat, nr_account); 3690 #endif 3691 } 3692 3693 static __always_inline 3694 struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, 3695 unsigned int order, unsigned int alloc_flags, 3696 int migratetype) 3697 { 3698 struct page *page; 3699 unsigned long flags; 3700 3701 do { 3702 page = NULL; 3703 spin_lock_irqsave(&zone->lock, flags); 3704 /* 3705 * order-0 request can reach here when the pcplist is skipped 3706 * due to non-CMA allocation context. HIGHATOMIC area is 3707 * reserved for high-order atomic allocation, so order-0 3708 * request should skip it. 3709 */ 3710 if (order > 0 && alloc_flags & ALLOC_HARDER) 3711 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3712 if (!page) { 3713 page = __rmqueue(zone, order, migratetype, alloc_flags); 3714 if (!page) { 3715 spin_unlock_irqrestore(&zone->lock, flags); 3716 return NULL; 3717 } 3718 } 3719 __mod_zone_freepage_state(zone, -(1 << order), 3720 get_pcppage_migratetype(page)); 3721 spin_unlock_irqrestore(&zone->lock, flags); 3722 } while (check_new_pages(page, order)); 3723 3724 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3725 zone_statistics(preferred_zone, zone, 1); 3726 3727 return page; 3728 } 3729 3730 /* Remove page from the per-cpu list, caller must protect the list */ 3731 static inline 3732 struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, 3733 int migratetype, 3734 unsigned int alloc_flags, 3735 struct per_cpu_pages *pcp, 3736 struct list_head *list) 3737 { 3738 struct page *page; 3739 3740 do { 3741 if (list_empty(list)) { 3742 int batch = READ_ONCE(pcp->batch); 3743 int alloced; 3744 3745 /* 3746 * Scale batch relative to order if batch implies 3747 * free pages can be stored on the PCP. Batch can 3748 * be 1 for small zones or for boot pagesets which 3749 * should never store free pages as the pages may 3750 * belong to arbitrary zones. 3751 */ 3752 if (batch > 1) 3753 batch = max(batch >> order, 2); 3754 alloced = rmqueue_bulk(zone, order, 3755 batch, list, 3756 migratetype, alloc_flags); 3757 3758 pcp->count += alloced << order; 3759 if (unlikely(list_empty(list))) 3760 return NULL; 3761 } 3762 3763 page = list_first_entry(list, struct page, pcp_list); 3764 list_del(&page->pcp_list); 3765 pcp->count -= 1 << order; 3766 } while (check_new_pcp(page, order)); 3767 3768 return page; 3769 } 3770 3771 /* Lock and remove page from the per-cpu list */ 3772 static struct page *rmqueue_pcplist(struct zone *preferred_zone, 3773 struct zone *zone, unsigned int order, 3774 int migratetype, unsigned int alloc_flags) 3775 { 3776 struct per_cpu_pages *pcp; 3777 struct list_head *list; 3778 struct page *page; 3779 unsigned long flags; 3780 unsigned long __maybe_unused UP_flags; 3781 3782 /* 3783 * spin_trylock may fail due to a parallel drain. In the future, the 3784 * trylock will also protect against IRQ reentrancy. 3785 */ 3786 pcp_trylock_prepare(UP_flags); 3787 pcp = pcp_spin_trylock_irqsave(zone->per_cpu_pageset, flags); 3788 if (!pcp) { 3789 pcp_trylock_finish(UP_flags); 3790 return NULL; 3791 } 3792 3793 /* 3794 * On allocation, reduce the number of pages that are batch freed. 3795 * See nr_pcp_free() where free_factor is increased for subsequent 3796 * frees. 3797 */ 3798 pcp->free_factor >>= 1; 3799 list = &pcp->lists[order_to_pindex(migratetype, order)]; 3800 page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list); 3801 pcp_spin_unlock_irqrestore(pcp, flags); 3802 pcp_trylock_finish(UP_flags); 3803 if (page) { 3804 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); 3805 zone_statistics(preferred_zone, zone, 1); 3806 } 3807 return page; 3808 } 3809 3810 /* 3811 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 3812 */ 3813 static inline 3814 struct page *rmqueue(struct zone *preferred_zone, 3815 struct zone *zone, unsigned int order, 3816 gfp_t gfp_flags, unsigned int alloc_flags, 3817 int migratetype) 3818 { 3819 struct page *page; 3820 3821 /* 3822 * We most definitely don't want callers attempting to 3823 * allocate greater than order-1 page units with __GFP_NOFAIL. 3824 */ 3825 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 3826 3827 if (likely(pcp_allowed_order(order))) { 3828 /* 3829 * MIGRATE_MOVABLE pcplist could have the pages on CMA area and 3830 * we need to skip it when CMA area isn't allowed. 3831 */ 3832 if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA || 3833 migratetype != MIGRATE_MOVABLE) { 3834 page = rmqueue_pcplist(preferred_zone, zone, order, 3835 migratetype, alloc_flags); 3836 if (likely(page)) 3837 goto out; 3838 } 3839 } 3840 3841 page = rmqueue_buddy(preferred_zone, zone, order, alloc_flags, 3842 migratetype); 3843 3844 out: 3845 /* Separate test+clear to avoid unnecessary atomics */ 3846 if (unlikely(test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags))) { 3847 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 3848 wakeup_kswapd(zone, 0, 0, zone_idx(zone)); 3849 } 3850 3851 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); 3852 return page; 3853 } 3854 3855 #ifdef CONFIG_FAIL_PAGE_ALLOC 3856 3857 static struct { 3858 struct fault_attr attr; 3859 3860 bool ignore_gfp_highmem; 3861 bool ignore_gfp_reclaim; 3862 u32 min_order; 3863 } fail_page_alloc = { 3864 .attr = FAULT_ATTR_INITIALIZER, 3865 .ignore_gfp_reclaim = true, 3866 .ignore_gfp_highmem = true, 3867 .min_order = 1, 3868 }; 3869 3870 static int __init setup_fail_page_alloc(char *str) 3871 { 3872 return setup_fault_attr(&fail_page_alloc.attr, str); 3873 } 3874 __setup("fail_page_alloc=", setup_fail_page_alloc); 3875 3876 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3877 { 3878 if (order < fail_page_alloc.min_order) 3879 return false; 3880 if (gfp_mask & __GFP_NOFAIL) 3881 return false; 3882 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 3883 return false; 3884 if (fail_page_alloc.ignore_gfp_reclaim && 3885 (gfp_mask & __GFP_DIRECT_RECLAIM)) 3886 return false; 3887 3888 if (gfp_mask & __GFP_NOWARN) 3889 fail_page_alloc.attr.no_warn = true; 3890 3891 return should_fail(&fail_page_alloc.attr, 1 << order); 3892 } 3893 3894 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 3895 3896 static int __init fail_page_alloc_debugfs(void) 3897 { 3898 umode_t mode = S_IFREG | 0600; 3899 struct dentry *dir; 3900 3901 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 3902 &fail_page_alloc.attr); 3903 3904 debugfs_create_bool("ignore-gfp-wait", mode, dir, 3905 &fail_page_alloc.ignore_gfp_reclaim); 3906 debugfs_create_bool("ignore-gfp-highmem", mode, dir, 3907 &fail_page_alloc.ignore_gfp_highmem); 3908 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order); 3909 3910 return 0; 3911 } 3912 3913 late_initcall(fail_page_alloc_debugfs); 3914 3915 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 3916 3917 #else /* CONFIG_FAIL_PAGE_ALLOC */ 3918 3919 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3920 { 3921 return false; 3922 } 3923 3924 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 3925 3926 noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3927 { 3928 return __should_fail_alloc_page(gfp_mask, order); 3929 } 3930 ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE); 3931 3932 static inline long __zone_watermark_unusable_free(struct zone *z, 3933 unsigned int order, unsigned int alloc_flags) 3934 { 3935 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3936 long unusable_free = (1 << order) - 1; 3937 3938 /* 3939 * If the caller does not have rights to ALLOC_HARDER then subtract 3940 * the high-atomic reserves. This will over-estimate the size of the 3941 * atomic reserve but it avoids a search. 3942 */ 3943 if (likely(!alloc_harder)) 3944 unusable_free += z->nr_reserved_highatomic; 3945 3946 #ifdef CONFIG_CMA 3947 /* If allocation can't use CMA areas don't use free CMA pages */ 3948 if (!(alloc_flags & ALLOC_CMA)) 3949 unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES); 3950 #endif 3951 3952 return unusable_free; 3953 } 3954 3955 /* 3956 * Return true if free base pages are above 'mark'. For high-order checks it 3957 * will return true of the order-0 watermark is reached and there is at least 3958 * one free page of a suitable size. Checking now avoids taking the zone lock 3959 * to check in the allocation paths if no pages are free. 3960 */ 3961 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3962 int highest_zoneidx, unsigned int alloc_flags, 3963 long free_pages) 3964 { 3965 long min = mark; 3966 int o; 3967 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3968 3969 /* free_pages may go negative - that's OK */ 3970 free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags); 3971 3972 if (alloc_flags & ALLOC_HIGH) 3973 min -= min / 2; 3974 3975 if (unlikely(alloc_harder)) { 3976 /* 3977 * OOM victims can try even harder than normal ALLOC_HARDER 3978 * users on the grounds that it's definitely going to be in 3979 * the exit path shortly and free memory. Any allocation it 3980 * makes during the free path will be small and short-lived. 3981 */ 3982 if (alloc_flags & ALLOC_OOM) 3983 min -= min / 2; 3984 else 3985 min -= min / 4; 3986 } 3987 3988 /* 3989 * Check watermarks for an order-0 allocation request. If these 3990 * are not met, then a high-order request also cannot go ahead 3991 * even if a suitable page happened to be free. 3992 */ 3993 if (free_pages <= min + z->lowmem_reserve[highest_zoneidx]) 3994 return false; 3995 3996 /* If this is an order-0 request then the watermark is fine */ 3997 if (!order) 3998 return true; 3999 4000 /* For a high-order request, check at least one suitable page is free */ 4001 for (o = order; o < MAX_ORDER; o++) { 4002 struct free_area *area = &z->free_area[o]; 4003 int mt; 4004 4005 if (!area->nr_free) 4006 continue; 4007 4008 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 4009 if (!free_area_empty(area, mt)) 4010 return true; 4011 } 4012 4013 #ifdef CONFIG_CMA 4014 if ((alloc_flags & ALLOC_CMA) && 4015 !free_area_empty(area, MIGRATE_CMA)) { 4016 return true; 4017 } 4018 #endif 4019 if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC)) 4020 return true; 4021 } 4022 return false; 4023 } 4024 4025 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 4026 int highest_zoneidx, unsigned int alloc_flags) 4027 { 4028 return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 4029 zone_page_state(z, NR_FREE_PAGES)); 4030 } 4031 4032 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 4033 unsigned long mark, int highest_zoneidx, 4034 unsigned int alloc_flags, gfp_t gfp_mask) 4035 { 4036 long free_pages; 4037 4038 free_pages = zone_page_state(z, NR_FREE_PAGES); 4039 4040 /* 4041 * Fast check for order-0 only. If this fails then the reserves 4042 * need to be calculated. 4043 */ 4044 if (!order) { 4045 long usable_free; 4046 long reserved; 4047 4048 usable_free = free_pages; 4049 reserved = __zone_watermark_unusable_free(z, 0, alloc_flags); 4050 4051 /* reserved may over estimate high-atomic reserves. */ 4052 usable_free -= min(usable_free, reserved); 4053 if (usable_free > mark + z->lowmem_reserve[highest_zoneidx]) 4054 return true; 4055 } 4056 4057 if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 4058 free_pages)) 4059 return true; 4060 /* 4061 * Ignore watermark boosting for GFP_ATOMIC order-0 allocations 4062 * when checking the min watermark. The min watermark is the 4063 * point where boosting is ignored so that kswapd is woken up 4064 * when below the low watermark. 4065 */ 4066 if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost 4067 && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) { 4068 mark = z->_watermark[WMARK_MIN]; 4069 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 4070 alloc_flags, free_pages); 4071 } 4072 4073 return false; 4074 } 4075 4076 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 4077 unsigned long mark, int highest_zoneidx) 4078 { 4079 long free_pages = zone_page_state(z, NR_FREE_PAGES); 4080 4081 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 4082 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 4083 4084 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0, 4085 free_pages); 4086 } 4087 4088 #ifdef CONFIG_NUMA 4089 int __read_mostly node_reclaim_distance = RECLAIM_DISTANCE; 4090 4091 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 4092 { 4093 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= 4094 node_reclaim_distance; 4095 } 4096 #else /* CONFIG_NUMA */ 4097 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 4098 { 4099 return true; 4100 } 4101 #endif /* CONFIG_NUMA */ 4102 4103 /* 4104 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid 4105 * fragmentation is subtle. If the preferred zone was HIGHMEM then 4106 * premature use of a lower zone may cause lowmem pressure problems that 4107 * are worse than fragmentation. If the next zone is ZONE_DMA then it is 4108 * probably too small. It only makes sense to spread allocations to avoid 4109 * fragmentation between the Normal and DMA32 zones. 4110 */ 4111 static inline unsigned int 4112 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) 4113 { 4114 unsigned int alloc_flags; 4115 4116 /* 4117 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 4118 * to save a branch. 4119 */ 4120 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); 4121 4122 #ifdef CONFIG_ZONE_DMA32 4123 if (!zone) 4124 return alloc_flags; 4125 4126 if (zone_idx(zone) != ZONE_NORMAL) 4127 return alloc_flags; 4128 4129 /* 4130 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and 4131 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume 4132 * on UMA that if Normal is populated then so is DMA32. 4133 */ 4134 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1); 4135 if (nr_online_nodes > 1 && !populated_zone(--zone)) 4136 return alloc_flags; 4137 4138 alloc_flags |= ALLOC_NOFRAGMENT; 4139 #endif /* CONFIG_ZONE_DMA32 */ 4140 return alloc_flags; 4141 } 4142 4143 /* Must be called after current_gfp_context() which can change gfp_mask */ 4144 static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask, 4145 unsigned int alloc_flags) 4146 { 4147 #ifdef CONFIG_CMA 4148 if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) 4149 alloc_flags |= ALLOC_CMA; 4150 #endif 4151 return alloc_flags; 4152 } 4153 4154 /* 4155 * get_page_from_freelist goes through the zonelist trying to allocate 4156 * a page. 4157 */ 4158 static struct page * 4159 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 4160 const struct alloc_context *ac) 4161 { 4162 struct zoneref *z; 4163 struct zone *zone; 4164 struct pglist_data *last_pgdat = NULL; 4165 bool last_pgdat_dirty_ok = false; 4166 bool no_fallback; 4167 4168 retry: 4169 /* 4170 * Scan zonelist, looking for a zone with enough free. 4171 * See also __cpuset_node_allowed() comment in kernel/cgroup/cpuset.c. 4172 */ 4173 no_fallback = alloc_flags & ALLOC_NOFRAGMENT; 4174 z = ac->preferred_zoneref; 4175 for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx, 4176 ac->nodemask) { 4177 struct page *page; 4178 unsigned long mark; 4179 4180 if (cpusets_enabled() && 4181 (alloc_flags & ALLOC_CPUSET) && 4182 !__cpuset_zone_allowed(zone, gfp_mask)) 4183 continue; 4184 /* 4185 * When allocating a page cache page for writing, we 4186 * want to get it from a node that is within its dirty 4187 * limit, such that no single node holds more than its 4188 * proportional share of globally allowed dirty pages. 4189 * The dirty limits take into account the node's 4190 * lowmem reserves and high watermark so that kswapd 4191 * should be able to balance it without having to 4192 * write pages from its LRU list. 4193 * 4194 * XXX: For now, allow allocations to potentially 4195 * exceed the per-node dirty limit in the slowpath 4196 * (spread_dirty_pages unset) before going into reclaim, 4197 * which is important when on a NUMA setup the allowed 4198 * nodes are together not big enough to reach the 4199 * global limit. The proper fix for these situations 4200 * will require awareness of nodes in the 4201 * dirty-throttling and the flusher threads. 4202 */ 4203 if (ac->spread_dirty_pages) { 4204 if (last_pgdat != zone->zone_pgdat) { 4205 last_pgdat = zone->zone_pgdat; 4206 last_pgdat_dirty_ok = node_dirty_ok(zone->zone_pgdat); 4207 } 4208 4209 if (!last_pgdat_dirty_ok) 4210 continue; 4211 } 4212 4213 if (no_fallback && nr_online_nodes > 1 && 4214 zone != ac->preferred_zoneref->zone) { 4215 int local_nid; 4216 4217 /* 4218 * If moving to a remote node, retry but allow 4219 * fragmenting fallbacks. Locality is more important 4220 * than fragmentation avoidance. 4221 */ 4222 local_nid = zone_to_nid(ac->preferred_zoneref->zone); 4223 if (zone_to_nid(zone) != local_nid) { 4224 alloc_flags &= ~ALLOC_NOFRAGMENT; 4225 goto retry; 4226 } 4227 } 4228 4229 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); 4230 if (!zone_watermark_fast(zone, order, mark, 4231 ac->highest_zoneidx, alloc_flags, 4232 gfp_mask)) { 4233 int ret; 4234 4235 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 4236 /* 4237 * Watermark failed for this zone, but see if we can 4238 * grow this zone if it contains deferred pages. 4239 */ 4240 if (static_branch_unlikely(&deferred_pages)) { 4241 if (_deferred_grow_zone(zone, order)) 4242 goto try_this_zone; 4243 } 4244 #endif 4245 /* Checked here to keep the fast path fast */ 4246 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 4247 if (alloc_flags & ALLOC_NO_WATERMARKS) 4248 goto try_this_zone; 4249 4250 if (!node_reclaim_enabled() || 4251 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 4252 continue; 4253 4254 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 4255 switch (ret) { 4256 case NODE_RECLAIM_NOSCAN: 4257 /* did not scan */ 4258 continue; 4259 case NODE_RECLAIM_FULL: 4260 /* scanned but unreclaimable */ 4261 continue; 4262 default: 4263 /* did we reclaim enough */ 4264 if (zone_watermark_ok(zone, order, mark, 4265 ac->highest_zoneidx, alloc_flags)) 4266 goto try_this_zone; 4267 4268 continue; 4269 } 4270 } 4271 4272 try_this_zone: 4273 page = rmqueue(ac->preferred_zoneref->zone, zone, order, 4274 gfp_mask, alloc_flags, ac->migratetype); 4275 if (page) { 4276 prep_new_page(page, order, gfp_mask, alloc_flags); 4277 4278 /* 4279 * If this is a high-order atomic allocation then check 4280 * if the pageblock should be reserved for the future 4281 */ 4282 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 4283 reserve_highatomic_pageblock(page, zone, order); 4284 4285 return page; 4286 } else { 4287 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 4288 /* Try again if zone has deferred pages */ 4289 if (static_branch_unlikely(&deferred_pages)) { 4290 if (_deferred_grow_zone(zone, order)) 4291 goto try_this_zone; 4292 } 4293 #endif 4294 } 4295 } 4296 4297 /* 4298 * It's possible on a UMA machine to get through all zones that are 4299 * fragmented. If avoiding fragmentation, reset and try again. 4300 */ 4301 if (no_fallback) { 4302 alloc_flags &= ~ALLOC_NOFRAGMENT; 4303 goto retry; 4304 } 4305 4306 return NULL; 4307 } 4308 4309 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) 4310 { 4311 unsigned int filter = SHOW_MEM_FILTER_NODES; 4312 4313 /* 4314 * This documents exceptions given to allocations in certain 4315 * contexts that are allowed to allocate outside current's set 4316 * of allowed nodes. 4317 */ 4318 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4319 if (tsk_is_oom_victim(current) || 4320 (current->flags & (PF_MEMALLOC | PF_EXITING))) 4321 filter &= ~SHOW_MEM_FILTER_NODES; 4322 if (!in_task() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 4323 filter &= ~SHOW_MEM_FILTER_NODES; 4324 4325 show_mem(filter, nodemask); 4326 } 4327 4328 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) 4329 { 4330 struct va_format vaf; 4331 va_list args; 4332 static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1); 4333 4334 if ((gfp_mask & __GFP_NOWARN) || 4335 !__ratelimit(&nopage_rs) || 4336 ((gfp_mask & __GFP_DMA) && !has_managed_dma())) 4337 return; 4338 4339 va_start(args, fmt); 4340 vaf.fmt = fmt; 4341 vaf.va = &args; 4342 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl", 4343 current->comm, &vaf, gfp_mask, &gfp_mask, 4344 nodemask_pr_args(nodemask)); 4345 va_end(args); 4346 4347 cpuset_print_current_mems_allowed(); 4348 pr_cont("\n"); 4349 dump_stack(); 4350 warn_alloc_show_mem(gfp_mask, nodemask); 4351 } 4352 4353 static inline struct page * 4354 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, 4355 unsigned int alloc_flags, 4356 const struct alloc_context *ac) 4357 { 4358 struct page *page; 4359 4360 page = get_page_from_freelist(gfp_mask, order, 4361 alloc_flags|ALLOC_CPUSET, ac); 4362 /* 4363 * fallback to ignore cpuset restriction if our nodes 4364 * are depleted 4365 */ 4366 if (!page) 4367 page = get_page_from_freelist(gfp_mask, order, 4368 alloc_flags, ac); 4369 4370 return page; 4371 } 4372 4373 static inline struct page * 4374 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 4375 const struct alloc_context *ac, unsigned long *did_some_progress) 4376 { 4377 struct oom_control oc = { 4378 .zonelist = ac->zonelist, 4379 .nodemask = ac->nodemask, 4380 .memcg = NULL, 4381 .gfp_mask = gfp_mask, 4382 .order = order, 4383 }; 4384 struct page *page; 4385 4386 *did_some_progress = 0; 4387 4388 /* 4389 * Acquire the oom lock. If that fails, somebody else is 4390 * making progress for us. 4391 */ 4392 if (!mutex_trylock(&oom_lock)) { 4393 *did_some_progress = 1; 4394 schedule_timeout_uninterruptible(1); 4395 return NULL; 4396 } 4397 4398 /* 4399 * Go through the zonelist yet one more time, keep very high watermark 4400 * here, this is only to catch a parallel oom killing, we must fail if 4401 * we're still under heavy pressure. But make sure that this reclaim 4402 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY 4403 * allocation which will never fail due to oom_lock already held. 4404 */ 4405 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & 4406 ~__GFP_DIRECT_RECLAIM, order, 4407 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 4408 if (page) 4409 goto out; 4410 4411 /* Coredumps can quickly deplete all memory reserves */ 4412 if (current->flags & PF_DUMPCORE) 4413 goto out; 4414 /* The OOM killer will not help higher order allocs */ 4415 if (order > PAGE_ALLOC_COSTLY_ORDER) 4416 goto out; 4417 /* 4418 * We have already exhausted all our reclaim opportunities without any 4419 * success so it is time to admit defeat. We will skip the OOM killer 4420 * because it is very likely that the caller has a more reasonable 4421 * fallback than shooting a random task. 4422 * 4423 * The OOM killer may not free memory on a specific node. 4424 */ 4425 if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE)) 4426 goto out; 4427 /* The OOM killer does not needlessly kill tasks for lowmem */ 4428 if (ac->highest_zoneidx < ZONE_NORMAL) 4429 goto out; 4430 if (pm_suspended_storage()) 4431 goto out; 4432 /* 4433 * XXX: GFP_NOFS allocations should rather fail than rely on 4434 * other request to make a forward progress. 4435 * We are in an unfortunate situation where out_of_memory cannot 4436 * do much for this context but let's try it to at least get 4437 * access to memory reserved if the current task is killed (see 4438 * out_of_memory). Once filesystems are ready to handle allocation 4439 * failures more gracefully we should just bail out here. 4440 */ 4441 4442 /* Exhausted what can be done so it's blame time */ 4443 if (out_of_memory(&oc) || 4444 WARN_ON_ONCE_GFP(gfp_mask & __GFP_NOFAIL, gfp_mask)) { 4445 *did_some_progress = 1; 4446 4447 /* 4448 * Help non-failing allocations by giving them access to memory 4449 * reserves 4450 */ 4451 if (gfp_mask & __GFP_NOFAIL) 4452 page = __alloc_pages_cpuset_fallback(gfp_mask, order, 4453 ALLOC_NO_WATERMARKS, ac); 4454 } 4455 out: 4456 mutex_unlock(&oom_lock); 4457 return page; 4458 } 4459 4460 /* 4461 * Maximum number of compaction retries with a progress before OOM 4462 * killer is consider as the only way to move forward. 4463 */ 4464 #define MAX_COMPACT_RETRIES 16 4465 4466 #ifdef CONFIG_COMPACTION 4467 /* Try memory compaction for high-order allocations before reclaim */ 4468 static struct page * 4469 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4470 unsigned int alloc_flags, const struct alloc_context *ac, 4471 enum compact_priority prio, enum compact_result *compact_result) 4472 { 4473 struct page *page = NULL; 4474 unsigned long pflags; 4475 unsigned int noreclaim_flag; 4476 4477 if (!order) 4478 return NULL; 4479 4480 psi_memstall_enter(&pflags); 4481 delayacct_compact_start(); 4482 noreclaim_flag = memalloc_noreclaim_save(); 4483 4484 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 4485 prio, &page); 4486 4487 memalloc_noreclaim_restore(noreclaim_flag); 4488 psi_memstall_leave(&pflags); 4489 delayacct_compact_end(); 4490 4491 if (*compact_result == COMPACT_SKIPPED) 4492 return NULL; 4493 /* 4494 * At least in one zone compaction wasn't deferred or skipped, so let's 4495 * count a compaction stall 4496 */ 4497 count_vm_event(COMPACTSTALL); 4498 4499 /* Prep a captured page if available */ 4500 if (page) 4501 prep_new_page(page, order, gfp_mask, alloc_flags); 4502 4503 /* Try get a page from the freelist if available */ 4504 if (!page) 4505 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4506 4507 if (page) { 4508 struct zone *zone = page_zone(page); 4509 4510 zone->compact_blockskip_flush = false; 4511 compaction_defer_reset(zone, order, true); 4512 count_vm_event(COMPACTSUCCESS); 4513 return page; 4514 } 4515 4516 /* 4517 * It's bad if compaction run occurs and fails. The most likely reason 4518 * is that pages exist, but not enough to satisfy watermarks. 4519 */ 4520 count_vm_event(COMPACTFAIL); 4521 4522 cond_resched(); 4523 4524 return NULL; 4525 } 4526 4527 static inline bool 4528 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 4529 enum compact_result compact_result, 4530 enum compact_priority *compact_priority, 4531 int *compaction_retries) 4532 { 4533 int max_retries = MAX_COMPACT_RETRIES; 4534 int min_priority; 4535 bool ret = false; 4536 int retries = *compaction_retries; 4537 enum compact_priority priority = *compact_priority; 4538 4539 if (!order) 4540 return false; 4541 4542 if (fatal_signal_pending(current)) 4543 return false; 4544 4545 if (compaction_made_progress(compact_result)) 4546 (*compaction_retries)++; 4547 4548 /* 4549 * compaction considers all the zone as desperately out of memory 4550 * so it doesn't really make much sense to retry except when the 4551 * failure could be caused by insufficient priority 4552 */ 4553 if (compaction_failed(compact_result)) 4554 goto check_priority; 4555 4556 /* 4557 * compaction was skipped because there are not enough order-0 pages 4558 * to work with, so we retry only if it looks like reclaim can help. 4559 */ 4560 if (compaction_needs_reclaim(compact_result)) { 4561 ret = compaction_zonelist_suitable(ac, order, alloc_flags); 4562 goto out; 4563 } 4564 4565 /* 4566 * make sure the compaction wasn't deferred or didn't bail out early 4567 * due to locks contention before we declare that we should give up. 4568 * But the next retry should use a higher priority if allowed, so 4569 * we don't just keep bailing out endlessly. 4570 */ 4571 if (compaction_withdrawn(compact_result)) { 4572 goto check_priority; 4573 } 4574 4575 /* 4576 * !costly requests are much more important than __GFP_RETRY_MAYFAIL 4577 * costly ones because they are de facto nofail and invoke OOM 4578 * killer to move on while costly can fail and users are ready 4579 * to cope with that. 1/4 retries is rather arbitrary but we 4580 * would need much more detailed feedback from compaction to 4581 * make a better decision. 4582 */ 4583 if (order > PAGE_ALLOC_COSTLY_ORDER) 4584 max_retries /= 4; 4585 if (*compaction_retries <= max_retries) { 4586 ret = true; 4587 goto out; 4588 } 4589 4590 /* 4591 * Make sure there are attempts at the highest priority if we exhausted 4592 * all retries or failed at the lower priorities. 4593 */ 4594 check_priority: 4595 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 4596 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 4597 4598 if (*compact_priority > min_priority) { 4599 (*compact_priority)--; 4600 *compaction_retries = 0; 4601 ret = true; 4602 } 4603 out: 4604 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret); 4605 return ret; 4606 } 4607 #else 4608 static inline struct page * 4609 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4610 unsigned int alloc_flags, const struct alloc_context *ac, 4611 enum compact_priority prio, enum compact_result *compact_result) 4612 { 4613 *compact_result = COMPACT_SKIPPED; 4614 return NULL; 4615 } 4616 4617 static inline bool 4618 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 4619 enum compact_result compact_result, 4620 enum compact_priority *compact_priority, 4621 int *compaction_retries) 4622 { 4623 struct zone *zone; 4624 struct zoneref *z; 4625 4626 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 4627 return false; 4628 4629 /* 4630 * There are setups with compaction disabled which would prefer to loop 4631 * inside the allocator rather than hit the oom killer prematurely. 4632 * Let's give them a good hope and keep retrying while the order-0 4633 * watermarks are OK. 4634 */ 4635 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4636 ac->highest_zoneidx, ac->nodemask) { 4637 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 4638 ac->highest_zoneidx, alloc_flags)) 4639 return true; 4640 } 4641 return false; 4642 } 4643 #endif /* CONFIG_COMPACTION */ 4644 4645 #ifdef CONFIG_LOCKDEP 4646 static struct lockdep_map __fs_reclaim_map = 4647 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map); 4648 4649 static bool __need_reclaim(gfp_t gfp_mask) 4650 { 4651 /* no reclaim without waiting on it */ 4652 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) 4653 return false; 4654 4655 /* this guy won't enter reclaim */ 4656 if (current->flags & PF_MEMALLOC) 4657 return false; 4658 4659 if (gfp_mask & __GFP_NOLOCKDEP) 4660 return false; 4661 4662 return true; 4663 } 4664 4665 void __fs_reclaim_acquire(unsigned long ip) 4666 { 4667 lock_acquire_exclusive(&__fs_reclaim_map, 0, 0, NULL, ip); 4668 } 4669 4670 void __fs_reclaim_release(unsigned long ip) 4671 { 4672 lock_release(&__fs_reclaim_map, ip); 4673 } 4674 4675 void fs_reclaim_acquire(gfp_t gfp_mask) 4676 { 4677 gfp_mask = current_gfp_context(gfp_mask); 4678 4679 if (__need_reclaim(gfp_mask)) { 4680 if (gfp_mask & __GFP_FS) 4681 __fs_reclaim_acquire(_RET_IP_); 4682 4683 #ifdef CONFIG_MMU_NOTIFIER 4684 lock_map_acquire(&__mmu_notifier_invalidate_range_start_map); 4685 lock_map_release(&__mmu_notifier_invalidate_range_start_map); 4686 #endif 4687 4688 } 4689 } 4690 EXPORT_SYMBOL_GPL(fs_reclaim_acquire); 4691 4692 void fs_reclaim_release(gfp_t gfp_mask) 4693 { 4694 gfp_mask = current_gfp_context(gfp_mask); 4695 4696 if (__need_reclaim(gfp_mask)) { 4697 if (gfp_mask & __GFP_FS) 4698 __fs_reclaim_release(_RET_IP_); 4699 } 4700 } 4701 EXPORT_SYMBOL_GPL(fs_reclaim_release); 4702 #endif 4703 4704 /* 4705 * Zonelists may change due to hotplug during allocation. Detect when zonelists 4706 * have been rebuilt so allocation retries. Reader side does not lock and 4707 * retries the allocation if zonelist changes. Writer side is protected by the 4708 * embedded spin_lock. 4709 */ 4710 static DEFINE_SEQLOCK(zonelist_update_seq); 4711 4712 static unsigned int zonelist_iter_begin(void) 4713 { 4714 if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE)) 4715 return read_seqbegin(&zonelist_update_seq); 4716 4717 return 0; 4718 } 4719 4720 static unsigned int check_retry_zonelist(unsigned int seq) 4721 { 4722 if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE)) 4723 return read_seqretry(&zonelist_update_seq, seq); 4724 4725 return seq; 4726 } 4727 4728 /* Perform direct synchronous page reclaim */ 4729 static unsigned long 4730 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 4731 const struct alloc_context *ac) 4732 { 4733 unsigned int noreclaim_flag; 4734 unsigned long progress; 4735 4736 cond_resched(); 4737 4738 /* We now go into synchronous reclaim */ 4739 cpuset_memory_pressure_bump(); 4740 fs_reclaim_acquire(gfp_mask); 4741 noreclaim_flag = memalloc_noreclaim_save(); 4742 4743 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 4744 ac->nodemask); 4745 4746 memalloc_noreclaim_restore(noreclaim_flag); 4747 fs_reclaim_release(gfp_mask); 4748 4749 cond_resched(); 4750 4751 return progress; 4752 } 4753 4754 /* The really slow allocator path where we enter direct reclaim */ 4755 static inline struct page * 4756 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 4757 unsigned int alloc_flags, const struct alloc_context *ac, 4758 unsigned long *did_some_progress) 4759 { 4760 struct page *page = NULL; 4761 unsigned long pflags; 4762 bool drained = false; 4763 4764 psi_memstall_enter(&pflags); 4765 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 4766 if (unlikely(!(*did_some_progress))) 4767 goto out; 4768 4769 retry: 4770 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4771 4772 /* 4773 * If an allocation failed after direct reclaim, it could be because 4774 * pages are pinned on the per-cpu lists or in high alloc reserves. 4775 * Shrink them and try again 4776 */ 4777 if (!page && !drained) { 4778 unreserve_highatomic_pageblock(ac, false); 4779 drain_all_pages(NULL); 4780 drained = true; 4781 goto retry; 4782 } 4783 out: 4784 psi_memstall_leave(&pflags); 4785 4786 return page; 4787 } 4788 4789 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, 4790 const struct alloc_context *ac) 4791 { 4792 struct zoneref *z; 4793 struct zone *zone; 4794 pg_data_t *last_pgdat = NULL; 4795 enum zone_type highest_zoneidx = ac->highest_zoneidx; 4796 4797 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx, 4798 ac->nodemask) { 4799 if (!managed_zone(zone)) 4800 continue; 4801 if (last_pgdat != zone->zone_pgdat) { 4802 wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx); 4803 last_pgdat = zone->zone_pgdat; 4804 } 4805 } 4806 } 4807 4808 static inline unsigned int 4809 gfp_to_alloc_flags(gfp_t gfp_mask) 4810 { 4811 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 4812 4813 /* 4814 * __GFP_HIGH is assumed to be the same as ALLOC_HIGH 4815 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 4816 * to save two branches. 4817 */ 4818 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 4819 BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); 4820 4821 /* 4822 * The caller may dip into page reserves a bit more if the caller 4823 * cannot run direct reclaim, or if the caller has realtime scheduling 4824 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 4825 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 4826 */ 4827 alloc_flags |= (__force int) 4828 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); 4829 4830 if (gfp_mask & __GFP_ATOMIC) { 4831 /* 4832 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 4833 * if it can't schedule. 4834 */ 4835 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4836 alloc_flags |= ALLOC_HARDER; 4837 /* 4838 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 4839 * comment for __cpuset_node_allowed(). 4840 */ 4841 alloc_flags &= ~ALLOC_CPUSET; 4842 } else if (unlikely(rt_task(current)) && in_task()) 4843 alloc_flags |= ALLOC_HARDER; 4844 4845 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags); 4846 4847 return alloc_flags; 4848 } 4849 4850 static bool oom_reserves_allowed(struct task_struct *tsk) 4851 { 4852 if (!tsk_is_oom_victim(tsk)) 4853 return false; 4854 4855 /* 4856 * !MMU doesn't have oom reaper so give access to memory reserves 4857 * only to the thread with TIF_MEMDIE set 4858 */ 4859 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE)) 4860 return false; 4861 4862 return true; 4863 } 4864 4865 /* 4866 * Distinguish requests which really need access to full memory 4867 * reserves from oom victims which can live with a portion of it 4868 */ 4869 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) 4870 { 4871 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 4872 return 0; 4873 if (gfp_mask & __GFP_MEMALLOC) 4874 return ALLOC_NO_WATERMARKS; 4875 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 4876 return ALLOC_NO_WATERMARKS; 4877 if (!in_interrupt()) { 4878 if (current->flags & PF_MEMALLOC) 4879 return ALLOC_NO_WATERMARKS; 4880 else if (oom_reserves_allowed(current)) 4881 return ALLOC_OOM; 4882 } 4883 4884 return 0; 4885 } 4886 4887 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 4888 { 4889 return !!__gfp_pfmemalloc_flags(gfp_mask); 4890 } 4891 4892 /* 4893 * Checks whether it makes sense to retry the reclaim to make a forward progress 4894 * for the given allocation request. 4895 * 4896 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row 4897 * without success, or when we couldn't even meet the watermark if we 4898 * reclaimed all remaining pages on the LRU lists. 4899 * 4900 * Returns true if a retry is viable or false to enter the oom path. 4901 */ 4902 static inline bool 4903 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 4904 struct alloc_context *ac, int alloc_flags, 4905 bool did_some_progress, int *no_progress_loops) 4906 { 4907 struct zone *zone; 4908 struct zoneref *z; 4909 bool ret = false; 4910 4911 /* 4912 * Costly allocations might have made a progress but this doesn't mean 4913 * their order will become available due to high fragmentation so 4914 * always increment the no progress counter for them 4915 */ 4916 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 4917 *no_progress_loops = 0; 4918 else 4919 (*no_progress_loops)++; 4920 4921 /* 4922 * Make sure we converge to OOM if we cannot make any progress 4923 * several times in the row. 4924 */ 4925 if (*no_progress_loops > MAX_RECLAIM_RETRIES) { 4926 /* Before OOM, exhaust highatomic_reserve */ 4927 return unreserve_highatomic_pageblock(ac, true); 4928 } 4929 4930 /* 4931 * Keep reclaiming pages while there is a chance this will lead 4932 * somewhere. If none of the target zones can satisfy our allocation 4933 * request even if all reclaimable pages are considered then we are 4934 * screwed and have to go OOM. 4935 */ 4936 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4937 ac->highest_zoneidx, ac->nodemask) { 4938 unsigned long available; 4939 unsigned long reclaimable; 4940 unsigned long min_wmark = min_wmark_pages(zone); 4941 bool wmark; 4942 4943 available = reclaimable = zone_reclaimable_pages(zone); 4944 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 4945 4946 /* 4947 * Would the allocation succeed if we reclaimed all 4948 * reclaimable pages? 4949 */ 4950 wmark = __zone_watermark_ok(zone, order, min_wmark, 4951 ac->highest_zoneidx, alloc_flags, available); 4952 trace_reclaim_retry_zone(z, order, reclaimable, 4953 available, min_wmark, *no_progress_loops, wmark); 4954 if (wmark) { 4955 ret = true; 4956 break; 4957 } 4958 } 4959 4960 /* 4961 * Memory allocation/reclaim might be called from a WQ context and the 4962 * current implementation of the WQ concurrency control doesn't 4963 * recognize that a particular WQ is congested if the worker thread is 4964 * looping without ever sleeping. Therefore we have to do a short sleep 4965 * here rather than calling cond_resched(). 4966 */ 4967 if (current->flags & PF_WQ_WORKER) 4968 schedule_timeout_uninterruptible(1); 4969 else 4970 cond_resched(); 4971 return ret; 4972 } 4973 4974 static inline bool 4975 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) 4976 { 4977 /* 4978 * It's possible that cpuset's mems_allowed and the nodemask from 4979 * mempolicy don't intersect. This should be normally dealt with by 4980 * policy_nodemask(), but it's possible to race with cpuset update in 4981 * such a way the check therein was true, and then it became false 4982 * before we got our cpuset_mems_cookie here. 4983 * This assumes that for all allocations, ac->nodemask can come only 4984 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored 4985 * when it does not intersect with the cpuset restrictions) or the 4986 * caller can deal with a violated nodemask. 4987 */ 4988 if (cpusets_enabled() && ac->nodemask && 4989 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) { 4990 ac->nodemask = NULL; 4991 return true; 4992 } 4993 4994 /* 4995 * When updating a task's mems_allowed or mempolicy nodemask, it is 4996 * possible to race with parallel threads in such a way that our 4997 * allocation can fail while the mask is being updated. If we are about 4998 * to fail, check if the cpuset changed during allocation and if so, 4999 * retry. 5000 */ 5001 if (read_mems_allowed_retry(cpuset_mems_cookie)) 5002 return true; 5003 5004 return false; 5005 } 5006 5007 static inline struct page * 5008 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 5009 struct alloc_context *ac) 5010 { 5011 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 5012 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 5013 struct page *page = NULL; 5014 unsigned int alloc_flags; 5015 unsigned long did_some_progress; 5016 enum compact_priority compact_priority; 5017 enum compact_result compact_result; 5018 int compaction_retries; 5019 int no_progress_loops; 5020 unsigned int cpuset_mems_cookie; 5021 unsigned int zonelist_iter_cookie; 5022 int reserve_flags; 5023 5024 /* 5025 * We also sanity check to catch abuse of atomic reserves being used by 5026 * callers that are not in atomic context. 5027 */ 5028 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 5029 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 5030 gfp_mask &= ~__GFP_ATOMIC; 5031 5032 restart: 5033 compaction_retries = 0; 5034 no_progress_loops = 0; 5035 compact_priority = DEF_COMPACT_PRIORITY; 5036 cpuset_mems_cookie = read_mems_allowed_begin(); 5037 zonelist_iter_cookie = zonelist_iter_begin(); 5038 5039 /* 5040 * The fast path uses conservative alloc_flags to succeed only until 5041 * kswapd needs to be woken up, and to avoid the cost of setting up 5042 * alloc_flags precisely. So we do that now. 5043 */ 5044 alloc_flags = gfp_to_alloc_flags(gfp_mask); 5045 5046 /* 5047 * We need to recalculate the starting point for the zonelist iterator 5048 * because we might have used different nodemask in the fast path, or 5049 * there was a cpuset modification and we are retrying - otherwise we 5050 * could end up iterating over non-eligible zones endlessly. 5051 */ 5052 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 5053 ac->highest_zoneidx, ac->nodemask); 5054 if (!ac->preferred_zoneref->zone) 5055 goto nopage; 5056 5057 /* 5058 * Check for insane configurations where the cpuset doesn't contain 5059 * any suitable zone to satisfy the request - e.g. non-movable 5060 * GFP_HIGHUSER allocations from MOVABLE nodes only. 5061 */ 5062 if (cpusets_insane_config() && (gfp_mask & __GFP_HARDWALL)) { 5063 struct zoneref *z = first_zones_zonelist(ac->zonelist, 5064 ac->highest_zoneidx, 5065 &cpuset_current_mems_allowed); 5066 if (!z->zone) 5067 goto nopage; 5068 } 5069 5070 if (alloc_flags & ALLOC_KSWAPD) 5071 wake_all_kswapds(order, gfp_mask, ac); 5072 5073 /* 5074 * The adjusted alloc_flags might result in immediate success, so try 5075 * that first 5076 */ 5077 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 5078 if (page) 5079 goto got_pg; 5080 5081 /* 5082 * For costly allocations, try direct compaction first, as it's likely 5083 * that we have enough base pages and don't need to reclaim. For non- 5084 * movable high-order allocations, do that as well, as compaction will 5085 * try prevent permanent fragmentation by migrating from blocks of the 5086 * same migratetype. 5087 * Don't try this for allocations that are allowed to ignore 5088 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 5089 */ 5090 if (can_direct_reclaim && 5091 (costly_order || 5092 (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 5093 && !gfp_pfmemalloc_allowed(gfp_mask)) { 5094 page = __alloc_pages_direct_compact(gfp_mask, order, 5095 alloc_flags, ac, 5096 INIT_COMPACT_PRIORITY, 5097 &compact_result); 5098 if (page) 5099 goto got_pg; 5100 5101 /* 5102 * Checks for costly allocations with __GFP_NORETRY, which 5103 * includes some THP page fault allocations 5104 */ 5105 if (costly_order && (gfp_mask & __GFP_NORETRY)) { 5106 /* 5107 * If allocating entire pageblock(s) and compaction 5108 * failed because all zones are below low watermarks 5109 * or is prohibited because it recently failed at this 5110 * order, fail immediately unless the allocator has 5111 * requested compaction and reclaim retry. 5112 * 5113 * Reclaim is 5114 * - potentially very expensive because zones are far 5115 * below their low watermarks or this is part of very 5116 * bursty high order allocations, 5117 * - not guaranteed to help because isolate_freepages() 5118 * may not iterate over freed pages as part of its 5119 * linear scan, and 5120 * - unlikely to make entire pageblocks free on its 5121 * own. 5122 */ 5123 if (compact_result == COMPACT_SKIPPED || 5124 compact_result == COMPACT_DEFERRED) 5125 goto nopage; 5126 5127 /* 5128 * Looks like reclaim/compaction is worth trying, but 5129 * sync compaction could be very expensive, so keep 5130 * using async compaction. 5131 */ 5132 compact_priority = INIT_COMPACT_PRIORITY; 5133 } 5134 } 5135 5136 retry: 5137 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 5138 if (alloc_flags & ALLOC_KSWAPD) 5139 wake_all_kswapds(order, gfp_mask, ac); 5140 5141 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); 5142 if (reserve_flags) 5143 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags); 5144 5145 /* 5146 * Reset the nodemask and zonelist iterators if memory policies can be 5147 * ignored. These allocations are high priority and system rather than 5148 * user oriented. 5149 */ 5150 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { 5151 ac->nodemask = NULL; 5152 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 5153 ac->highest_zoneidx, ac->nodemask); 5154 } 5155 5156 /* Attempt with potentially adjusted zonelist and alloc_flags */ 5157 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 5158 if (page) 5159 goto got_pg; 5160 5161 /* Caller is not willing to reclaim, we can't balance anything */ 5162 if (!can_direct_reclaim) 5163 goto nopage; 5164 5165 /* Avoid recursion of direct reclaim */ 5166 if (current->flags & PF_MEMALLOC) 5167 goto nopage; 5168 5169 /* Try direct reclaim and then allocating */ 5170 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 5171 &did_some_progress); 5172 if (page) 5173 goto got_pg; 5174 5175 /* Try direct compaction and then allocating */ 5176 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 5177 compact_priority, &compact_result); 5178 if (page) 5179 goto got_pg; 5180 5181 /* Do not loop if specifically requested */ 5182 if (gfp_mask & __GFP_NORETRY) 5183 goto nopage; 5184 5185 /* 5186 * Do not retry costly high order allocations unless they are 5187 * __GFP_RETRY_MAYFAIL 5188 */ 5189 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) 5190 goto nopage; 5191 5192 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 5193 did_some_progress > 0, &no_progress_loops)) 5194 goto retry; 5195 5196 /* 5197 * It doesn't make any sense to retry for the compaction if the order-0 5198 * reclaim is not able to make any progress because the current 5199 * implementation of the compaction depends on the sufficient amount 5200 * of free memory (see __compaction_suitable) 5201 */ 5202 if (did_some_progress > 0 && 5203 should_compact_retry(ac, order, alloc_flags, 5204 compact_result, &compact_priority, 5205 &compaction_retries)) 5206 goto retry; 5207 5208 5209 /* 5210 * Deal with possible cpuset update races or zonelist updates to avoid 5211 * a unnecessary OOM kill. 5212 */ 5213 if (check_retry_cpuset(cpuset_mems_cookie, ac) || 5214 check_retry_zonelist(zonelist_iter_cookie)) 5215 goto restart; 5216 5217 /* Reclaim has failed us, start killing things */ 5218 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 5219 if (page) 5220 goto got_pg; 5221 5222 /* Avoid allocations with no watermarks from looping endlessly */ 5223 if (tsk_is_oom_victim(current) && 5224 (alloc_flags & ALLOC_OOM || 5225 (gfp_mask & __GFP_NOMEMALLOC))) 5226 goto nopage; 5227 5228 /* Retry as long as the OOM killer is making progress */ 5229 if (did_some_progress) { 5230 no_progress_loops = 0; 5231 goto retry; 5232 } 5233 5234 nopage: 5235 /* 5236 * Deal with possible cpuset update races or zonelist updates to avoid 5237 * a unnecessary OOM kill. 5238 */ 5239 if (check_retry_cpuset(cpuset_mems_cookie, ac) || 5240 check_retry_zonelist(zonelist_iter_cookie)) 5241 goto restart; 5242 5243 /* 5244 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure 5245 * we always retry 5246 */ 5247 if (gfp_mask & __GFP_NOFAIL) { 5248 /* 5249 * All existing users of the __GFP_NOFAIL are blockable, so warn 5250 * of any new users that actually require GFP_NOWAIT 5251 */ 5252 if (WARN_ON_ONCE_GFP(!can_direct_reclaim, gfp_mask)) 5253 goto fail; 5254 5255 /* 5256 * PF_MEMALLOC request from this context is rather bizarre 5257 * because we cannot reclaim anything and only can loop waiting 5258 * for somebody to do a work for us 5259 */ 5260 WARN_ON_ONCE_GFP(current->flags & PF_MEMALLOC, gfp_mask); 5261 5262 /* 5263 * non failing costly orders are a hard requirement which we 5264 * are not prepared for much so let's warn about these users 5265 * so that we can identify them and convert them to something 5266 * else. 5267 */ 5268 WARN_ON_ONCE_GFP(order > PAGE_ALLOC_COSTLY_ORDER, gfp_mask); 5269 5270 /* 5271 * Help non-failing allocations by giving them access to memory 5272 * reserves but do not use ALLOC_NO_WATERMARKS because this 5273 * could deplete whole memory reserves which would just make 5274 * the situation worse 5275 */ 5276 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); 5277 if (page) 5278 goto got_pg; 5279 5280 cond_resched(); 5281 goto retry; 5282 } 5283 fail: 5284 warn_alloc(gfp_mask, ac->nodemask, 5285 "page allocation failure: order:%u", order); 5286 got_pg: 5287 return page; 5288 } 5289 5290 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, 5291 int preferred_nid, nodemask_t *nodemask, 5292 struct alloc_context *ac, gfp_t *alloc_gfp, 5293 unsigned int *alloc_flags) 5294 { 5295 ac->highest_zoneidx = gfp_zone(gfp_mask); 5296 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); 5297 ac->nodemask = nodemask; 5298 ac->migratetype = gfp_migratetype(gfp_mask); 5299 5300 if (cpusets_enabled()) { 5301 *alloc_gfp |= __GFP_HARDWALL; 5302 /* 5303 * When we are in the interrupt context, it is irrelevant 5304 * to the current task context. It means that any node ok. 5305 */ 5306 if (in_task() && !ac->nodemask) 5307 ac->nodemask = &cpuset_current_mems_allowed; 5308 else 5309 *alloc_flags |= ALLOC_CPUSET; 5310 } 5311 5312 might_alloc(gfp_mask); 5313 5314 if (should_fail_alloc_page(gfp_mask, order)) 5315 return false; 5316 5317 *alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, *alloc_flags); 5318 5319 /* Dirty zone balancing only done in the fast path */ 5320 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); 5321 5322 /* 5323 * The preferred zone is used for statistics but crucially it is 5324 * also used as the starting point for the zonelist iterator. It 5325 * may get reset for allocations that ignore memory policies. 5326 */ 5327 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 5328 ac->highest_zoneidx, ac->nodemask); 5329 5330 return true; 5331 } 5332 5333 /* 5334 * __alloc_pages_bulk - Allocate a number of order-0 pages to a list or array 5335 * @gfp: GFP flags for the allocation 5336 * @preferred_nid: The preferred NUMA node ID to allocate from 5337 * @nodemask: Set of nodes to allocate from, may be NULL 5338 * @nr_pages: The number of pages desired on the list or array 5339 * @page_list: Optional list to store the allocated pages 5340 * @page_array: Optional array to store the pages 5341 * 5342 * This is a batched version of the page allocator that attempts to 5343 * allocate nr_pages quickly. Pages are added to page_list if page_list 5344 * is not NULL, otherwise it is assumed that the page_array is valid. 5345 * 5346 * For lists, nr_pages is the number of pages that should be allocated. 5347 * 5348 * For arrays, only NULL elements are populated with pages and nr_pages 5349 * is the maximum number of pages that will be stored in the array. 5350 * 5351 * Returns the number of pages on the list or array. 5352 */ 5353 unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, 5354 nodemask_t *nodemask, int nr_pages, 5355 struct list_head *page_list, 5356 struct page **page_array) 5357 { 5358 struct page *page; 5359 unsigned long flags; 5360 unsigned long __maybe_unused UP_flags; 5361 struct zone *zone; 5362 struct zoneref *z; 5363 struct per_cpu_pages *pcp; 5364 struct list_head *pcp_list; 5365 struct alloc_context ac; 5366 gfp_t alloc_gfp; 5367 unsigned int alloc_flags = ALLOC_WMARK_LOW; 5368 int nr_populated = 0, nr_account = 0; 5369 5370 /* 5371 * Skip populated array elements to determine if any pages need 5372 * to be allocated before disabling IRQs. 5373 */ 5374 while (page_array && nr_populated < nr_pages && page_array[nr_populated]) 5375 nr_populated++; 5376 5377 /* No pages requested? */ 5378 if (unlikely(nr_pages <= 0)) 5379 goto out; 5380 5381 /* Already populated array? */ 5382 if (unlikely(page_array && nr_pages - nr_populated == 0)) 5383 goto out; 5384 5385 /* Bulk allocator does not support memcg accounting. */ 5386 if (memcg_kmem_enabled() && (gfp & __GFP_ACCOUNT)) 5387 goto failed; 5388 5389 /* Use the single page allocator for one page. */ 5390 if (nr_pages - nr_populated == 1) 5391 goto failed; 5392 5393 #ifdef CONFIG_PAGE_OWNER 5394 /* 5395 * PAGE_OWNER may recurse into the allocator to allocate space to 5396 * save the stack with pagesets.lock held. Releasing/reacquiring 5397 * removes much of the performance benefit of bulk allocation so 5398 * force the caller to allocate one page at a time as it'll have 5399 * similar performance to added complexity to the bulk allocator. 5400 */ 5401 if (static_branch_unlikely(&page_owner_inited)) 5402 goto failed; 5403 #endif 5404 5405 /* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */ 5406 gfp &= gfp_allowed_mask; 5407 alloc_gfp = gfp; 5408 if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &alloc_gfp, &alloc_flags)) 5409 goto out; 5410 gfp = alloc_gfp; 5411 5412 /* Find an allowed local zone that meets the low watermark. */ 5413 for_each_zone_zonelist_nodemask(zone, z, ac.zonelist, ac.highest_zoneidx, ac.nodemask) { 5414 unsigned long mark; 5415 5416 if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) && 5417 !__cpuset_zone_allowed(zone, gfp)) { 5418 continue; 5419 } 5420 5421 if (nr_online_nodes > 1 && zone != ac.preferred_zoneref->zone && 5422 zone_to_nid(zone) != zone_to_nid(ac.preferred_zoneref->zone)) { 5423 goto failed; 5424 } 5425 5426 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK) + nr_pages; 5427 if (zone_watermark_fast(zone, 0, mark, 5428 zonelist_zone_idx(ac.preferred_zoneref), 5429 alloc_flags, gfp)) { 5430 break; 5431 } 5432 } 5433 5434 /* 5435 * If there are no allowed local zones that meets the watermarks then 5436 * try to allocate a single page and reclaim if necessary. 5437 */ 5438 if (unlikely(!zone)) 5439 goto failed; 5440 5441 /* Is a parallel drain in progress? */ 5442 pcp_trylock_prepare(UP_flags); 5443 pcp = pcp_spin_trylock_irqsave(zone->per_cpu_pageset, flags); 5444 if (!pcp) 5445 goto failed_irq; 5446 5447 /* Attempt the batch allocation */ 5448 pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)]; 5449 while (nr_populated < nr_pages) { 5450 5451 /* Skip existing pages */ 5452 if (page_array && page_array[nr_populated]) { 5453 nr_populated++; 5454 continue; 5455 } 5456 5457 page = __rmqueue_pcplist(zone, 0, ac.migratetype, alloc_flags, 5458 pcp, pcp_list); 5459 if (unlikely(!page)) { 5460 /* Try and allocate at least one page */ 5461 if (!nr_account) { 5462 pcp_spin_unlock_irqrestore(pcp, flags); 5463 goto failed_irq; 5464 } 5465 break; 5466 } 5467 nr_account++; 5468 5469 prep_new_page(page, 0, gfp, 0); 5470 if (page_list) 5471 list_add(&page->lru, page_list); 5472 else 5473 page_array[nr_populated] = page; 5474 nr_populated++; 5475 } 5476 5477 pcp_spin_unlock_irqrestore(pcp, flags); 5478 pcp_trylock_finish(UP_flags); 5479 5480 __count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account); 5481 zone_statistics(ac.preferred_zoneref->zone, zone, nr_account); 5482 5483 out: 5484 return nr_populated; 5485 5486 failed_irq: 5487 pcp_trylock_finish(UP_flags); 5488 5489 failed: 5490 page = __alloc_pages(gfp, 0, preferred_nid, nodemask); 5491 if (page) { 5492 if (page_list) 5493 list_add(&page->lru, page_list); 5494 else 5495 page_array[nr_populated] = page; 5496 nr_populated++; 5497 } 5498 5499 goto out; 5500 } 5501 EXPORT_SYMBOL_GPL(__alloc_pages_bulk); 5502 5503 /* 5504 * This is the 'heart' of the zoned buddy allocator. 5505 */ 5506 struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid, 5507 nodemask_t *nodemask) 5508 { 5509 struct page *page; 5510 unsigned int alloc_flags = ALLOC_WMARK_LOW; 5511 gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */ 5512 struct alloc_context ac = { }; 5513 5514 /* 5515 * There are several places where we assume that the order value is sane 5516 * so bail out early if the request is out of bound. 5517 */ 5518 if (WARN_ON_ONCE_GFP(order >= MAX_ORDER, gfp)) 5519 return NULL; 5520 5521 gfp &= gfp_allowed_mask; 5522 /* 5523 * Apply scoped allocation constraints. This is mainly about GFP_NOFS 5524 * resp. GFP_NOIO which has to be inherited for all allocation requests 5525 * from a particular context which has been marked by 5526 * memalloc_no{fs,io}_{save,restore}. And PF_MEMALLOC_PIN which ensures 5527 * movable zones are not used during allocation. 5528 */ 5529 gfp = current_gfp_context(gfp); 5530 alloc_gfp = gfp; 5531 if (!prepare_alloc_pages(gfp, order, preferred_nid, nodemask, &ac, 5532 &alloc_gfp, &alloc_flags)) 5533 return NULL; 5534 5535 /* 5536 * Forbid the first pass from falling back to types that fragment 5537 * memory until all local zones are considered. 5538 */ 5539 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp); 5540 5541 /* First allocation attempt */ 5542 page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); 5543 if (likely(page)) 5544 goto out; 5545 5546 alloc_gfp = gfp; 5547 ac.spread_dirty_pages = false; 5548 5549 /* 5550 * Restore the original nodemask if it was potentially replaced with 5551 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 5552 */ 5553 ac.nodemask = nodemask; 5554 5555 page = __alloc_pages_slowpath(alloc_gfp, order, &ac); 5556 5557 out: 5558 if (memcg_kmem_enabled() && (gfp & __GFP_ACCOUNT) && page && 5559 unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) { 5560 __free_pages(page, order); 5561 page = NULL; 5562 } 5563 5564 trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype); 5565 5566 return page; 5567 } 5568 EXPORT_SYMBOL(__alloc_pages); 5569 5570 struct folio *__folio_alloc(gfp_t gfp, unsigned int order, int preferred_nid, 5571 nodemask_t *nodemask) 5572 { 5573 struct page *page = __alloc_pages(gfp | __GFP_COMP, order, 5574 preferred_nid, nodemask); 5575 5576 if (page && order > 1) 5577 prep_transhuge_page(page); 5578 return (struct folio *)page; 5579 } 5580 EXPORT_SYMBOL(__folio_alloc); 5581 5582 /* 5583 * Common helper functions. Never use with __GFP_HIGHMEM because the returned 5584 * address cannot represent highmem pages. Use alloc_pages and then kmap if 5585 * you need to access high mem. 5586 */ 5587 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 5588 { 5589 struct page *page; 5590 5591 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); 5592 if (!page) 5593 return 0; 5594 return (unsigned long) page_address(page); 5595 } 5596 EXPORT_SYMBOL(__get_free_pages); 5597 5598 unsigned long get_zeroed_page(gfp_t gfp_mask) 5599 { 5600 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 5601 } 5602 EXPORT_SYMBOL(get_zeroed_page); 5603 5604 /** 5605 * __free_pages - Free pages allocated with alloc_pages(). 5606 * @page: The page pointer returned from alloc_pages(). 5607 * @order: The order of the allocation. 5608 * 5609 * This function can free multi-page allocations that are not compound 5610 * pages. It does not check that the @order passed in matches that of 5611 * the allocation, so it is easy to leak memory. Freeing more memory 5612 * than was allocated will probably emit a warning. 5613 * 5614 * If the last reference to this page is speculative, it will be released 5615 * by put_page() which only frees the first page of a non-compound 5616 * allocation. To prevent the remaining pages from being leaked, we free 5617 * the subsequent pages here. If you want to use the page's reference 5618 * count to decide when to free the allocation, you should allocate a 5619 * compound page, and use put_page() instead of __free_pages(). 5620 * 5621 * Context: May be called in interrupt context or while holding a normal 5622 * spinlock, but not in NMI context or while holding a raw spinlock. 5623 */ 5624 void __free_pages(struct page *page, unsigned int order) 5625 { 5626 if (put_page_testzero(page)) 5627 free_the_page(page, order); 5628 else if (!PageHead(page)) 5629 while (order-- > 0) 5630 free_the_page(page + (1 << order), order); 5631 } 5632 EXPORT_SYMBOL(__free_pages); 5633 5634 void free_pages(unsigned long addr, unsigned int order) 5635 { 5636 if (addr != 0) { 5637 VM_BUG_ON(!virt_addr_valid((void *)addr)); 5638 __free_pages(virt_to_page((void *)addr), order); 5639 } 5640 } 5641 5642 EXPORT_SYMBOL(free_pages); 5643 5644 /* 5645 * Page Fragment: 5646 * An arbitrary-length arbitrary-offset area of memory which resides 5647 * within a 0 or higher order page. Multiple fragments within that page 5648 * are individually refcounted, in the page's reference counter. 5649 * 5650 * The page_frag functions below provide a simple allocation framework for 5651 * page fragments. This is used by the network stack and network device 5652 * drivers to provide a backing region of memory for use as either an 5653 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 5654 */ 5655 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 5656 gfp_t gfp_mask) 5657 { 5658 struct page *page = NULL; 5659 gfp_t gfp = gfp_mask; 5660 5661 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5662 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 5663 __GFP_NOMEMALLOC; 5664 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 5665 PAGE_FRAG_CACHE_MAX_ORDER); 5666 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 5667 #endif 5668 if (unlikely(!page)) 5669 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 5670 5671 nc->va = page ? page_address(page) : NULL; 5672 5673 return page; 5674 } 5675 5676 void __page_frag_cache_drain(struct page *page, unsigned int count) 5677 { 5678 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 5679 5680 if (page_ref_sub_and_test(page, count)) 5681 free_the_page(page, compound_order(page)); 5682 } 5683 EXPORT_SYMBOL(__page_frag_cache_drain); 5684 5685 void *page_frag_alloc_align(struct page_frag_cache *nc, 5686 unsigned int fragsz, gfp_t gfp_mask, 5687 unsigned int align_mask) 5688 { 5689 unsigned int size = PAGE_SIZE; 5690 struct page *page; 5691 int offset; 5692 5693 if (unlikely(!nc->va)) { 5694 refill: 5695 page = __page_frag_cache_refill(nc, gfp_mask); 5696 if (!page) 5697 return NULL; 5698 5699 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5700 /* if size can vary use size else just use PAGE_SIZE */ 5701 size = nc->size; 5702 #endif 5703 /* Even if we own the page, we do not use atomic_set(). 5704 * This would break get_page_unless_zero() users. 5705 */ 5706 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 5707 5708 /* reset page count bias and offset to start of new frag */ 5709 nc->pfmemalloc = page_is_pfmemalloc(page); 5710 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5711 nc->offset = size; 5712 } 5713 5714 offset = nc->offset - fragsz; 5715 if (unlikely(offset < 0)) { 5716 page = virt_to_page(nc->va); 5717 5718 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 5719 goto refill; 5720 5721 if (unlikely(nc->pfmemalloc)) { 5722 free_the_page(page, compound_order(page)); 5723 goto refill; 5724 } 5725 5726 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5727 /* if size can vary use size else just use PAGE_SIZE */ 5728 size = nc->size; 5729 #endif 5730 /* OK, page count is 0, we can safely set it */ 5731 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 5732 5733 /* reset page count bias and offset to start of new frag */ 5734 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5735 offset = size - fragsz; 5736 if (unlikely(offset < 0)) { 5737 /* 5738 * The caller is trying to allocate a fragment 5739 * with fragsz > PAGE_SIZE but the cache isn't big 5740 * enough to satisfy the request, this may 5741 * happen in low memory conditions. 5742 * We don't release the cache page because 5743 * it could make memory pressure worse 5744 * so we simply return NULL here. 5745 */ 5746 return NULL; 5747 } 5748 } 5749 5750 nc->pagecnt_bias--; 5751 offset &= align_mask; 5752 nc->offset = offset; 5753 5754 return nc->va + offset; 5755 } 5756 EXPORT_SYMBOL(page_frag_alloc_align); 5757 5758 /* 5759 * Frees a page fragment allocated out of either a compound or order 0 page. 5760 */ 5761 void page_frag_free(void *addr) 5762 { 5763 struct page *page = virt_to_head_page(addr); 5764 5765 if (unlikely(put_page_testzero(page))) 5766 free_the_page(page, compound_order(page)); 5767 } 5768 EXPORT_SYMBOL(page_frag_free); 5769 5770 static void *make_alloc_exact(unsigned long addr, unsigned int order, 5771 size_t size) 5772 { 5773 if (addr) { 5774 unsigned long alloc_end = addr + (PAGE_SIZE << order); 5775 unsigned long used = addr + PAGE_ALIGN(size); 5776 5777 split_page(virt_to_page((void *)addr), order); 5778 while (used < alloc_end) { 5779 free_page(used); 5780 used += PAGE_SIZE; 5781 } 5782 } 5783 return (void *)addr; 5784 } 5785 5786 /** 5787 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 5788 * @size: the number of bytes to allocate 5789 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5790 * 5791 * This function is similar to alloc_pages(), except that it allocates the 5792 * minimum number of pages to satisfy the request. alloc_pages() can only 5793 * allocate memory in power-of-two pages. 5794 * 5795 * This function is also limited by MAX_ORDER. 5796 * 5797 * Memory allocated by this function must be released by free_pages_exact(). 5798 * 5799 * Return: pointer to the allocated area or %NULL in case of error. 5800 */ 5801 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 5802 { 5803 unsigned int order = get_order(size); 5804 unsigned long addr; 5805 5806 if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM))) 5807 gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM); 5808 5809 addr = __get_free_pages(gfp_mask, order); 5810 return make_alloc_exact(addr, order, size); 5811 } 5812 EXPORT_SYMBOL(alloc_pages_exact); 5813 5814 /** 5815 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 5816 * pages on a node. 5817 * @nid: the preferred node ID where memory should be allocated 5818 * @size: the number of bytes to allocate 5819 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5820 * 5821 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 5822 * back. 5823 * 5824 * Return: pointer to the allocated area or %NULL in case of error. 5825 */ 5826 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 5827 { 5828 unsigned int order = get_order(size); 5829 struct page *p; 5830 5831 if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM))) 5832 gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM); 5833 5834 p = alloc_pages_node(nid, gfp_mask, order); 5835 if (!p) 5836 return NULL; 5837 return make_alloc_exact((unsigned long)page_address(p), order, size); 5838 } 5839 5840 /** 5841 * free_pages_exact - release memory allocated via alloc_pages_exact() 5842 * @virt: the value returned by alloc_pages_exact. 5843 * @size: size of allocation, same value as passed to alloc_pages_exact(). 5844 * 5845 * Release the memory allocated by a previous call to alloc_pages_exact. 5846 */ 5847 void free_pages_exact(void *virt, size_t size) 5848 { 5849 unsigned long addr = (unsigned long)virt; 5850 unsigned long end = addr + PAGE_ALIGN(size); 5851 5852 while (addr < end) { 5853 free_page(addr); 5854 addr += PAGE_SIZE; 5855 } 5856 } 5857 EXPORT_SYMBOL(free_pages_exact); 5858 5859 /** 5860 * nr_free_zone_pages - count number of pages beyond high watermark 5861 * @offset: The zone index of the highest zone 5862 * 5863 * nr_free_zone_pages() counts the number of pages which are beyond the 5864 * high watermark within all zones at or below a given zone index. For each 5865 * zone, the number of pages is calculated as: 5866 * 5867 * nr_free_zone_pages = managed_pages - high_pages 5868 * 5869 * Return: number of pages beyond high watermark. 5870 */ 5871 static unsigned long nr_free_zone_pages(int offset) 5872 { 5873 struct zoneref *z; 5874 struct zone *zone; 5875 5876 /* Just pick one node, since fallback list is circular */ 5877 unsigned long sum = 0; 5878 5879 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 5880 5881 for_each_zone_zonelist(zone, z, zonelist, offset) { 5882 unsigned long size = zone_managed_pages(zone); 5883 unsigned long high = high_wmark_pages(zone); 5884 if (size > high) 5885 sum += size - high; 5886 } 5887 5888 return sum; 5889 } 5890 5891 /** 5892 * nr_free_buffer_pages - count number of pages beyond high watermark 5893 * 5894 * nr_free_buffer_pages() counts the number of pages which are beyond the high 5895 * watermark within ZONE_DMA and ZONE_NORMAL. 5896 * 5897 * Return: number of pages beyond high watermark within ZONE_DMA and 5898 * ZONE_NORMAL. 5899 */ 5900 unsigned long nr_free_buffer_pages(void) 5901 { 5902 return nr_free_zone_pages(gfp_zone(GFP_USER)); 5903 } 5904 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 5905 5906 static inline void show_node(struct zone *zone) 5907 { 5908 if (IS_ENABLED(CONFIG_NUMA)) 5909 printk("Node %d ", zone_to_nid(zone)); 5910 } 5911 5912 long si_mem_available(void) 5913 { 5914 long available; 5915 unsigned long pagecache; 5916 unsigned long wmark_low = 0; 5917 unsigned long pages[NR_LRU_LISTS]; 5918 unsigned long reclaimable; 5919 struct zone *zone; 5920 int lru; 5921 5922 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 5923 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 5924 5925 for_each_zone(zone) 5926 wmark_low += low_wmark_pages(zone); 5927 5928 /* 5929 * Estimate the amount of memory available for userspace allocations, 5930 * without causing swapping or OOM. 5931 */ 5932 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages; 5933 5934 /* 5935 * Not all the page cache can be freed, otherwise the system will 5936 * start swapping or thrashing. Assume at least half of the page 5937 * cache, or the low watermark worth of cache, needs to stay. 5938 */ 5939 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 5940 pagecache -= min(pagecache / 2, wmark_low); 5941 available += pagecache; 5942 5943 /* 5944 * Part of the reclaimable slab and other kernel memory consists of 5945 * items that are in use, and cannot be freed. Cap this estimate at the 5946 * low watermark. 5947 */ 5948 reclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B) + 5949 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE); 5950 available += reclaimable - min(reclaimable / 2, wmark_low); 5951 5952 if (available < 0) 5953 available = 0; 5954 return available; 5955 } 5956 EXPORT_SYMBOL_GPL(si_mem_available); 5957 5958 void si_meminfo(struct sysinfo *val) 5959 { 5960 val->totalram = totalram_pages(); 5961 val->sharedram = global_node_page_state(NR_SHMEM); 5962 val->freeram = global_zone_page_state(NR_FREE_PAGES); 5963 val->bufferram = nr_blockdev_pages(); 5964 val->totalhigh = totalhigh_pages(); 5965 val->freehigh = nr_free_highpages(); 5966 val->mem_unit = PAGE_SIZE; 5967 } 5968 5969 EXPORT_SYMBOL(si_meminfo); 5970 5971 #ifdef CONFIG_NUMA 5972 void si_meminfo_node(struct sysinfo *val, int nid) 5973 { 5974 int zone_type; /* needs to be signed */ 5975 unsigned long managed_pages = 0; 5976 unsigned long managed_highpages = 0; 5977 unsigned long free_highpages = 0; 5978 pg_data_t *pgdat = NODE_DATA(nid); 5979 5980 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 5981 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]); 5982 val->totalram = managed_pages; 5983 val->sharedram = node_page_state(pgdat, NR_SHMEM); 5984 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 5985 #ifdef CONFIG_HIGHMEM 5986 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 5987 struct zone *zone = &pgdat->node_zones[zone_type]; 5988 5989 if (is_highmem(zone)) { 5990 managed_highpages += zone_managed_pages(zone); 5991 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 5992 } 5993 } 5994 val->totalhigh = managed_highpages; 5995 val->freehigh = free_highpages; 5996 #else 5997 val->totalhigh = managed_highpages; 5998 val->freehigh = free_highpages; 5999 #endif 6000 val->mem_unit = PAGE_SIZE; 6001 } 6002 #endif 6003 6004 /* 6005 * Determine whether the node should be displayed or not, depending on whether 6006 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 6007 */ 6008 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) 6009 { 6010 if (!(flags & SHOW_MEM_FILTER_NODES)) 6011 return false; 6012 6013 /* 6014 * no node mask - aka implicit memory numa policy. Do not bother with 6015 * the synchronization - read_mems_allowed_begin - because we do not 6016 * have to be precise here. 6017 */ 6018 if (!nodemask) 6019 nodemask = &cpuset_current_mems_allowed; 6020 6021 return !node_isset(nid, *nodemask); 6022 } 6023 6024 #define K(x) ((x) << (PAGE_SHIFT-10)) 6025 6026 static void show_migration_types(unsigned char type) 6027 { 6028 static const char types[MIGRATE_TYPES] = { 6029 [MIGRATE_UNMOVABLE] = 'U', 6030 [MIGRATE_MOVABLE] = 'M', 6031 [MIGRATE_RECLAIMABLE] = 'E', 6032 [MIGRATE_HIGHATOMIC] = 'H', 6033 #ifdef CONFIG_CMA 6034 [MIGRATE_CMA] = 'C', 6035 #endif 6036 #ifdef CONFIG_MEMORY_ISOLATION 6037 [MIGRATE_ISOLATE] = 'I', 6038 #endif 6039 }; 6040 char tmp[MIGRATE_TYPES + 1]; 6041 char *p = tmp; 6042 int i; 6043 6044 for (i = 0; i < MIGRATE_TYPES; i++) { 6045 if (type & (1 << i)) 6046 *p++ = types[i]; 6047 } 6048 6049 *p = '\0'; 6050 printk(KERN_CONT "(%s) ", tmp); 6051 } 6052 6053 /* 6054 * Show free area list (used inside shift_scroll-lock stuff) 6055 * We also calculate the percentage fragmentation. We do this by counting the 6056 * memory on each free list with the exception of the first item on the list. 6057 * 6058 * Bits in @filter: 6059 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 6060 * cpuset. 6061 */ 6062 void show_free_areas(unsigned int filter, nodemask_t *nodemask) 6063 { 6064 unsigned long free_pcp = 0; 6065 int cpu, nid; 6066 struct zone *zone; 6067 pg_data_t *pgdat; 6068 6069 for_each_populated_zone(zone) { 6070 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 6071 continue; 6072 6073 for_each_online_cpu(cpu) 6074 free_pcp += per_cpu_ptr(zone->per_cpu_pageset, cpu)->count; 6075 } 6076 6077 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 6078 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 6079 " unevictable:%lu dirty:%lu writeback:%lu\n" 6080 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 6081 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 6082 " kernel_misc_reclaimable:%lu\n" 6083 " free:%lu free_pcp:%lu free_cma:%lu\n", 6084 global_node_page_state(NR_ACTIVE_ANON), 6085 global_node_page_state(NR_INACTIVE_ANON), 6086 global_node_page_state(NR_ISOLATED_ANON), 6087 global_node_page_state(NR_ACTIVE_FILE), 6088 global_node_page_state(NR_INACTIVE_FILE), 6089 global_node_page_state(NR_ISOLATED_FILE), 6090 global_node_page_state(NR_UNEVICTABLE), 6091 global_node_page_state(NR_FILE_DIRTY), 6092 global_node_page_state(NR_WRITEBACK), 6093 global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B), 6094 global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B), 6095 global_node_page_state(NR_FILE_MAPPED), 6096 global_node_page_state(NR_SHMEM), 6097 global_node_page_state(NR_PAGETABLE), 6098 global_zone_page_state(NR_BOUNCE), 6099 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE), 6100 global_zone_page_state(NR_FREE_PAGES), 6101 free_pcp, 6102 global_zone_page_state(NR_FREE_CMA_PAGES)); 6103 6104 for_each_online_pgdat(pgdat) { 6105 if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) 6106 continue; 6107 6108 printk("Node %d" 6109 " active_anon:%lukB" 6110 " inactive_anon:%lukB" 6111 " active_file:%lukB" 6112 " inactive_file:%lukB" 6113 " unevictable:%lukB" 6114 " isolated(anon):%lukB" 6115 " isolated(file):%lukB" 6116 " mapped:%lukB" 6117 " dirty:%lukB" 6118 " writeback:%lukB" 6119 " shmem:%lukB" 6120 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 6121 " shmem_thp: %lukB" 6122 " shmem_pmdmapped: %lukB" 6123 " anon_thp: %lukB" 6124 #endif 6125 " writeback_tmp:%lukB" 6126 " kernel_stack:%lukB" 6127 #ifdef CONFIG_SHADOW_CALL_STACK 6128 " shadow_call_stack:%lukB" 6129 #endif 6130 " pagetables:%lukB" 6131 " all_unreclaimable? %s" 6132 "\n", 6133 pgdat->node_id, 6134 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 6135 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 6136 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 6137 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 6138 K(node_page_state(pgdat, NR_UNEVICTABLE)), 6139 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 6140 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 6141 K(node_page_state(pgdat, NR_FILE_MAPPED)), 6142 K(node_page_state(pgdat, NR_FILE_DIRTY)), 6143 K(node_page_state(pgdat, NR_WRITEBACK)), 6144 K(node_page_state(pgdat, NR_SHMEM)), 6145 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 6146 K(node_page_state(pgdat, NR_SHMEM_THPS)), 6147 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), 6148 K(node_page_state(pgdat, NR_ANON_THPS)), 6149 #endif 6150 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 6151 node_page_state(pgdat, NR_KERNEL_STACK_KB), 6152 #ifdef CONFIG_SHADOW_CALL_STACK 6153 node_page_state(pgdat, NR_KERNEL_SCS_KB), 6154 #endif 6155 K(node_page_state(pgdat, NR_PAGETABLE)), 6156 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ? 6157 "yes" : "no"); 6158 } 6159 6160 for_each_populated_zone(zone) { 6161 int i; 6162 6163 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 6164 continue; 6165 6166 free_pcp = 0; 6167 for_each_online_cpu(cpu) 6168 free_pcp += per_cpu_ptr(zone->per_cpu_pageset, cpu)->count; 6169 6170 show_node(zone); 6171 printk(KERN_CONT 6172 "%s" 6173 " free:%lukB" 6174 " boost:%lukB" 6175 " min:%lukB" 6176 " low:%lukB" 6177 " high:%lukB" 6178 " reserved_highatomic:%luKB" 6179 " active_anon:%lukB" 6180 " inactive_anon:%lukB" 6181 " active_file:%lukB" 6182 " inactive_file:%lukB" 6183 " unevictable:%lukB" 6184 " writepending:%lukB" 6185 " present:%lukB" 6186 " managed:%lukB" 6187 " mlocked:%lukB" 6188 " bounce:%lukB" 6189 " free_pcp:%lukB" 6190 " local_pcp:%ukB" 6191 " free_cma:%lukB" 6192 "\n", 6193 zone->name, 6194 K(zone_page_state(zone, NR_FREE_PAGES)), 6195 K(zone->watermark_boost), 6196 K(min_wmark_pages(zone)), 6197 K(low_wmark_pages(zone)), 6198 K(high_wmark_pages(zone)), 6199 K(zone->nr_reserved_highatomic), 6200 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 6201 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 6202 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 6203 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 6204 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 6205 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 6206 K(zone->present_pages), 6207 K(zone_managed_pages(zone)), 6208 K(zone_page_state(zone, NR_MLOCK)), 6209 K(zone_page_state(zone, NR_BOUNCE)), 6210 K(free_pcp), 6211 K(this_cpu_read(zone->per_cpu_pageset->count)), 6212 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 6213 printk("lowmem_reserve[]:"); 6214 for (i = 0; i < MAX_NR_ZONES; i++) 6215 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); 6216 printk(KERN_CONT "\n"); 6217 } 6218 6219 for_each_populated_zone(zone) { 6220 unsigned int order; 6221 unsigned long nr[MAX_ORDER], flags, total = 0; 6222 unsigned char types[MAX_ORDER]; 6223 6224 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 6225 continue; 6226 show_node(zone); 6227 printk(KERN_CONT "%s: ", zone->name); 6228 6229 spin_lock_irqsave(&zone->lock, flags); 6230 for (order = 0; order < MAX_ORDER; order++) { 6231 struct free_area *area = &zone->free_area[order]; 6232 int type; 6233 6234 nr[order] = area->nr_free; 6235 total += nr[order] << order; 6236 6237 types[order] = 0; 6238 for (type = 0; type < MIGRATE_TYPES; type++) { 6239 if (!free_area_empty(area, type)) 6240 types[order] |= 1 << type; 6241 } 6242 } 6243 spin_unlock_irqrestore(&zone->lock, flags); 6244 for (order = 0; order < MAX_ORDER; order++) { 6245 printk(KERN_CONT "%lu*%lukB ", 6246 nr[order], K(1UL) << order); 6247 if (nr[order]) 6248 show_migration_types(types[order]); 6249 } 6250 printk(KERN_CONT "= %lukB\n", K(total)); 6251 } 6252 6253 for_each_online_node(nid) { 6254 if (show_mem_node_skip(filter, nid, nodemask)) 6255 continue; 6256 hugetlb_show_meminfo_node(nid); 6257 } 6258 6259 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 6260 6261 show_swap_cache_info(); 6262 } 6263 6264 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 6265 { 6266 zoneref->zone = zone; 6267 zoneref->zone_idx = zone_idx(zone); 6268 } 6269 6270 /* 6271 * Builds allocation fallback zone lists. 6272 * 6273 * Add all populated zones of a node to the zonelist. 6274 */ 6275 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) 6276 { 6277 struct zone *zone; 6278 enum zone_type zone_type = MAX_NR_ZONES; 6279 int nr_zones = 0; 6280 6281 do { 6282 zone_type--; 6283 zone = pgdat->node_zones + zone_type; 6284 if (populated_zone(zone)) { 6285 zoneref_set_zone(zone, &zonerefs[nr_zones++]); 6286 check_highest_zone(zone_type); 6287 } 6288 } while (zone_type); 6289 6290 return nr_zones; 6291 } 6292 6293 #ifdef CONFIG_NUMA 6294 6295 static int __parse_numa_zonelist_order(char *s) 6296 { 6297 /* 6298 * We used to support different zonelists modes but they turned 6299 * out to be just not useful. Let's keep the warning in place 6300 * if somebody still use the cmd line parameter so that we do 6301 * not fail it silently 6302 */ 6303 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) { 6304 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s); 6305 return -EINVAL; 6306 } 6307 return 0; 6308 } 6309 6310 char numa_zonelist_order[] = "Node"; 6311 6312 /* 6313 * sysctl handler for numa_zonelist_order 6314 */ 6315 int numa_zonelist_order_handler(struct ctl_table *table, int write, 6316 void *buffer, size_t *length, loff_t *ppos) 6317 { 6318 if (write) 6319 return __parse_numa_zonelist_order(buffer); 6320 return proc_dostring(table, write, buffer, length, ppos); 6321 } 6322 6323 6324 static int node_load[MAX_NUMNODES]; 6325 6326 /** 6327 * find_next_best_node - find the next node that should appear in a given node's fallback list 6328 * @node: node whose fallback list we're appending 6329 * @used_node_mask: nodemask_t of already used nodes 6330 * 6331 * We use a number of factors to determine which is the next node that should 6332 * appear on a given node's fallback list. The node should not have appeared 6333 * already in @node's fallback list, and it should be the next closest node 6334 * according to the distance array (which contains arbitrary distance values 6335 * from each node to each node in the system), and should also prefer nodes 6336 * with no CPUs, since presumably they'll have very little allocation pressure 6337 * on them otherwise. 6338 * 6339 * Return: node id of the found node or %NUMA_NO_NODE if no node is found. 6340 */ 6341 int find_next_best_node(int node, nodemask_t *used_node_mask) 6342 { 6343 int n, val; 6344 int min_val = INT_MAX; 6345 int best_node = NUMA_NO_NODE; 6346 6347 /* Use the local node if we haven't already */ 6348 if (!node_isset(node, *used_node_mask)) { 6349 node_set(node, *used_node_mask); 6350 return node; 6351 } 6352 6353 for_each_node_state(n, N_MEMORY) { 6354 6355 /* Don't want a node to appear more than once */ 6356 if (node_isset(n, *used_node_mask)) 6357 continue; 6358 6359 /* Use the distance array to find the distance */ 6360 val = node_distance(node, n); 6361 6362 /* Penalize nodes under us ("prefer the next node") */ 6363 val += (n < node); 6364 6365 /* Give preference to headless and unused nodes */ 6366 if (!cpumask_empty(cpumask_of_node(n))) 6367 val += PENALTY_FOR_NODE_WITH_CPUS; 6368 6369 /* Slight preference for less loaded node */ 6370 val *= MAX_NUMNODES; 6371 val += node_load[n]; 6372 6373 if (val < min_val) { 6374 min_val = val; 6375 best_node = n; 6376 } 6377 } 6378 6379 if (best_node >= 0) 6380 node_set(best_node, *used_node_mask); 6381 6382 return best_node; 6383 } 6384 6385 6386 /* 6387 * Build zonelists ordered by node and zones within node. 6388 * This results in maximum locality--normal zone overflows into local 6389 * DMA zone, if any--but risks exhausting DMA zone. 6390 */ 6391 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, 6392 unsigned nr_nodes) 6393 { 6394 struct zoneref *zonerefs; 6395 int i; 6396 6397 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 6398 6399 for (i = 0; i < nr_nodes; i++) { 6400 int nr_zones; 6401 6402 pg_data_t *node = NODE_DATA(node_order[i]); 6403 6404 nr_zones = build_zonerefs_node(node, zonerefs); 6405 zonerefs += nr_zones; 6406 } 6407 zonerefs->zone = NULL; 6408 zonerefs->zone_idx = 0; 6409 } 6410 6411 /* 6412 * Build gfp_thisnode zonelists 6413 */ 6414 static void build_thisnode_zonelists(pg_data_t *pgdat) 6415 { 6416 struct zoneref *zonerefs; 6417 int nr_zones; 6418 6419 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; 6420 nr_zones = build_zonerefs_node(pgdat, zonerefs); 6421 zonerefs += nr_zones; 6422 zonerefs->zone = NULL; 6423 zonerefs->zone_idx = 0; 6424 } 6425 6426 /* 6427 * Build zonelists ordered by zone and nodes within zones. 6428 * This results in conserving DMA zone[s] until all Normal memory is 6429 * exhausted, but results in overflowing to remote node while memory 6430 * may still exist in local DMA zone. 6431 */ 6432 6433 static void build_zonelists(pg_data_t *pgdat) 6434 { 6435 static int node_order[MAX_NUMNODES]; 6436 int node, nr_nodes = 0; 6437 nodemask_t used_mask = NODE_MASK_NONE; 6438 int local_node, prev_node; 6439 6440 /* NUMA-aware ordering of nodes */ 6441 local_node = pgdat->node_id; 6442 prev_node = local_node; 6443 6444 memset(node_order, 0, sizeof(node_order)); 6445 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 6446 /* 6447 * We don't want to pressure a particular node. 6448 * So adding penalty to the first node in same 6449 * distance group to make it round-robin. 6450 */ 6451 if (node_distance(local_node, node) != 6452 node_distance(local_node, prev_node)) 6453 node_load[node] += 1; 6454 6455 node_order[nr_nodes++] = node; 6456 prev_node = node; 6457 } 6458 6459 build_zonelists_in_node_order(pgdat, node_order, nr_nodes); 6460 build_thisnode_zonelists(pgdat); 6461 pr_info("Fallback order for Node %d: ", local_node); 6462 for (node = 0; node < nr_nodes; node++) 6463 pr_cont("%d ", node_order[node]); 6464 pr_cont("\n"); 6465 } 6466 6467 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 6468 /* 6469 * Return node id of node used for "local" allocations. 6470 * I.e., first node id of first zone in arg node's generic zonelist. 6471 * Used for initializing percpu 'numa_mem', which is used primarily 6472 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 6473 */ 6474 int local_memory_node(int node) 6475 { 6476 struct zoneref *z; 6477 6478 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 6479 gfp_zone(GFP_KERNEL), 6480 NULL); 6481 return zone_to_nid(z->zone); 6482 } 6483 #endif 6484 6485 static void setup_min_unmapped_ratio(void); 6486 static void setup_min_slab_ratio(void); 6487 #else /* CONFIG_NUMA */ 6488 6489 static void build_zonelists(pg_data_t *pgdat) 6490 { 6491 int node, local_node; 6492 struct zoneref *zonerefs; 6493 int nr_zones; 6494 6495 local_node = pgdat->node_id; 6496 6497 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 6498 nr_zones = build_zonerefs_node(pgdat, zonerefs); 6499 zonerefs += nr_zones; 6500 6501 /* 6502 * Now we build the zonelist so that it contains the zones 6503 * of all the other nodes. 6504 * We don't want to pressure a particular node, so when 6505 * building the zones for node N, we make sure that the 6506 * zones coming right after the local ones are those from 6507 * node N+1 (modulo N) 6508 */ 6509 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 6510 if (!node_online(node)) 6511 continue; 6512 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 6513 zonerefs += nr_zones; 6514 } 6515 for (node = 0; node < local_node; node++) { 6516 if (!node_online(node)) 6517 continue; 6518 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 6519 zonerefs += nr_zones; 6520 } 6521 6522 zonerefs->zone = NULL; 6523 zonerefs->zone_idx = 0; 6524 } 6525 6526 #endif /* CONFIG_NUMA */ 6527 6528 /* 6529 * Boot pageset table. One per cpu which is going to be used for all 6530 * zones and all nodes. The parameters will be set in such a way 6531 * that an item put on a list will immediately be handed over to 6532 * the buddy list. This is safe since pageset manipulation is done 6533 * with interrupts disabled. 6534 * 6535 * The boot_pagesets must be kept even after bootup is complete for 6536 * unused processors and/or zones. They do play a role for bootstrapping 6537 * hotplugged processors. 6538 * 6539 * zoneinfo_show() and maybe other functions do 6540 * not check if the processor is online before following the pageset pointer. 6541 * Other parts of the kernel may not check if the zone is available. 6542 */ 6543 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats); 6544 /* These effectively disable the pcplists in the boot pageset completely */ 6545 #define BOOT_PAGESET_HIGH 0 6546 #define BOOT_PAGESET_BATCH 1 6547 static DEFINE_PER_CPU(struct per_cpu_pages, boot_pageset); 6548 static DEFINE_PER_CPU(struct per_cpu_zonestat, boot_zonestats); 6549 DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 6550 6551 static void __build_all_zonelists(void *data) 6552 { 6553 int nid; 6554 int __maybe_unused cpu; 6555 pg_data_t *self = data; 6556 6557 write_seqlock(&zonelist_update_seq); 6558 6559 #ifdef CONFIG_NUMA 6560 memset(node_load, 0, sizeof(node_load)); 6561 #endif 6562 6563 /* 6564 * This node is hotadded and no memory is yet present. So just 6565 * building zonelists is fine - no need to touch other nodes. 6566 */ 6567 if (self && !node_online(self->node_id)) { 6568 build_zonelists(self); 6569 } else { 6570 /* 6571 * All possible nodes have pgdat preallocated 6572 * in free_area_init 6573 */ 6574 for_each_node(nid) { 6575 pg_data_t *pgdat = NODE_DATA(nid); 6576 6577 build_zonelists(pgdat); 6578 } 6579 6580 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 6581 /* 6582 * We now know the "local memory node" for each node-- 6583 * i.e., the node of the first zone in the generic zonelist. 6584 * Set up numa_mem percpu variable for on-line cpus. During 6585 * boot, only the boot cpu should be on-line; we'll init the 6586 * secondary cpus' numa_mem as they come on-line. During 6587 * node/memory hotplug, we'll fixup all on-line cpus. 6588 */ 6589 for_each_online_cpu(cpu) 6590 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 6591 #endif 6592 } 6593 6594 write_sequnlock(&zonelist_update_seq); 6595 } 6596 6597 static noinline void __init 6598 build_all_zonelists_init(void) 6599 { 6600 int cpu; 6601 6602 __build_all_zonelists(NULL); 6603 6604 /* 6605 * Initialize the boot_pagesets that are going to be used 6606 * for bootstrapping processors. The real pagesets for 6607 * each zone will be allocated later when the per cpu 6608 * allocator is available. 6609 * 6610 * boot_pagesets are used also for bootstrapping offline 6611 * cpus if the system is already booted because the pagesets 6612 * are needed to initialize allocators on a specific cpu too. 6613 * F.e. the percpu allocator needs the page allocator which 6614 * needs the percpu allocator in order to allocate its pagesets 6615 * (a chicken-egg dilemma). 6616 */ 6617 for_each_possible_cpu(cpu) 6618 per_cpu_pages_init(&per_cpu(boot_pageset, cpu), &per_cpu(boot_zonestats, cpu)); 6619 6620 mminit_verify_zonelist(); 6621 cpuset_init_current_mems_allowed(); 6622 } 6623 6624 /* 6625 * unless system_state == SYSTEM_BOOTING. 6626 * 6627 * __ref due to call of __init annotated helper build_all_zonelists_init 6628 * [protected by SYSTEM_BOOTING]. 6629 */ 6630 void __ref build_all_zonelists(pg_data_t *pgdat) 6631 { 6632 unsigned long vm_total_pages; 6633 6634 if (system_state == SYSTEM_BOOTING) { 6635 build_all_zonelists_init(); 6636 } else { 6637 __build_all_zonelists(pgdat); 6638 /* cpuset refresh routine should be here */ 6639 } 6640 /* Get the number of free pages beyond high watermark in all zones. */ 6641 vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 6642 /* 6643 * Disable grouping by mobility if the number of pages in the 6644 * system is too low to allow the mechanism to work. It would be 6645 * more accurate, but expensive to check per-zone. This check is 6646 * made on memory-hotadd so a system can start with mobility 6647 * disabled and enable it later 6648 */ 6649 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 6650 page_group_by_mobility_disabled = 1; 6651 else 6652 page_group_by_mobility_disabled = 0; 6653 6654 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n", 6655 nr_online_nodes, 6656 page_group_by_mobility_disabled ? "off" : "on", 6657 vm_total_pages); 6658 #ifdef CONFIG_NUMA 6659 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 6660 #endif 6661 } 6662 6663 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 6664 static bool __meminit 6665 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 6666 { 6667 static struct memblock_region *r; 6668 6669 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 6670 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 6671 for_each_mem_region(r) { 6672 if (*pfn < memblock_region_memory_end_pfn(r)) 6673 break; 6674 } 6675 } 6676 if (*pfn >= memblock_region_memory_base_pfn(r) && 6677 memblock_is_mirror(r)) { 6678 *pfn = memblock_region_memory_end_pfn(r); 6679 return true; 6680 } 6681 } 6682 return false; 6683 } 6684 6685 /* 6686 * Initially all pages are reserved - free ones are freed 6687 * up by memblock_free_all() once the early boot process is 6688 * done. Non-atomic initialization, single-pass. 6689 * 6690 * All aligned pageblocks are initialized to the specified migratetype 6691 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related 6692 * zone stats (e.g., nr_isolate_pageblock) are touched. 6693 */ 6694 void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone, 6695 unsigned long start_pfn, unsigned long zone_end_pfn, 6696 enum meminit_context context, 6697 struct vmem_altmap *altmap, int migratetype) 6698 { 6699 unsigned long pfn, end_pfn = start_pfn + size; 6700 struct page *page; 6701 6702 if (highest_memmap_pfn < end_pfn - 1) 6703 highest_memmap_pfn = end_pfn - 1; 6704 6705 #ifdef CONFIG_ZONE_DEVICE 6706 /* 6707 * Honor reservation requested by the driver for this ZONE_DEVICE 6708 * memory. We limit the total number of pages to initialize to just 6709 * those that might contain the memory mapping. We will defer the 6710 * ZONE_DEVICE page initialization until after we have released 6711 * the hotplug lock. 6712 */ 6713 if (zone == ZONE_DEVICE) { 6714 if (!altmap) 6715 return; 6716 6717 if (start_pfn == altmap->base_pfn) 6718 start_pfn += altmap->reserve; 6719 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6720 } 6721 #endif 6722 6723 for (pfn = start_pfn; pfn < end_pfn; ) { 6724 /* 6725 * There can be holes in boot-time mem_map[]s handed to this 6726 * function. They do not exist on hotplugged memory. 6727 */ 6728 if (context == MEMINIT_EARLY) { 6729 if (overlap_memmap_init(zone, &pfn)) 6730 continue; 6731 if (defer_init(nid, pfn, zone_end_pfn)) 6732 break; 6733 } 6734 6735 page = pfn_to_page(pfn); 6736 __init_single_page(page, pfn, zone, nid); 6737 if (context == MEMINIT_HOTPLUG) 6738 __SetPageReserved(page); 6739 6740 /* 6741 * Usually, we want to mark the pageblock MIGRATE_MOVABLE, 6742 * such that unmovable allocations won't be scattered all 6743 * over the place during system boot. 6744 */ 6745 if (IS_ALIGNED(pfn, pageblock_nr_pages)) { 6746 set_pageblock_migratetype(page, migratetype); 6747 cond_resched(); 6748 } 6749 pfn++; 6750 } 6751 } 6752 6753 #ifdef CONFIG_ZONE_DEVICE 6754 static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, 6755 unsigned long zone_idx, int nid, 6756 struct dev_pagemap *pgmap) 6757 { 6758 6759 __init_single_page(page, pfn, zone_idx, nid); 6760 6761 /* 6762 * Mark page reserved as it will need to wait for onlining 6763 * phase for it to be fully associated with a zone. 6764 * 6765 * We can use the non-atomic __set_bit operation for setting 6766 * the flag as we are still initializing the pages. 6767 */ 6768 __SetPageReserved(page); 6769 6770 /* 6771 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 6772 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 6773 * ever freed or placed on a driver-private list. 6774 */ 6775 page->pgmap = pgmap; 6776 page->zone_device_data = NULL; 6777 6778 /* 6779 * Mark the block movable so that blocks are reserved for 6780 * movable at startup. This will force kernel allocations 6781 * to reserve their blocks rather than leaking throughout 6782 * the address space during boot when many long-lived 6783 * kernel allocations are made. 6784 * 6785 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap 6786 * because this is done early in section_activate() 6787 */ 6788 if (IS_ALIGNED(pfn, pageblock_nr_pages)) { 6789 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 6790 cond_resched(); 6791 } 6792 } 6793 6794 /* 6795 * With compound page geometry and when struct pages are stored in ram most 6796 * tail pages are reused. Consequently, the amount of unique struct pages to 6797 * initialize is a lot smaller that the total amount of struct pages being 6798 * mapped. This is a paired / mild layering violation with explicit knowledge 6799 * of how the sparse_vmemmap internals handle compound pages in the lack 6800 * of an altmap. See vmemmap_populate_compound_pages(). 6801 */ 6802 static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap, 6803 unsigned long nr_pages) 6804 { 6805 return is_power_of_2(sizeof(struct page)) && 6806 !altmap ? 2 * (PAGE_SIZE / sizeof(struct page)) : nr_pages; 6807 } 6808 6809 static void __ref memmap_init_compound(struct page *head, 6810 unsigned long head_pfn, 6811 unsigned long zone_idx, int nid, 6812 struct dev_pagemap *pgmap, 6813 unsigned long nr_pages) 6814 { 6815 unsigned long pfn, end_pfn = head_pfn + nr_pages; 6816 unsigned int order = pgmap->vmemmap_shift; 6817 6818 __SetPageHead(head); 6819 for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) { 6820 struct page *page = pfn_to_page(pfn); 6821 6822 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); 6823 prep_compound_tail(head, pfn - head_pfn); 6824 set_page_count(page, 0); 6825 6826 /* 6827 * The first tail page stores compound_mapcount_ptr() and 6828 * compound_order() and the second tail page stores 6829 * compound_pincount_ptr(). Call prep_compound_head() after 6830 * the first and second tail pages have been initialized to 6831 * not have the data overwritten. 6832 */ 6833 if (pfn == head_pfn + 2) 6834 prep_compound_head(head, order); 6835 } 6836 } 6837 6838 void __ref memmap_init_zone_device(struct zone *zone, 6839 unsigned long start_pfn, 6840 unsigned long nr_pages, 6841 struct dev_pagemap *pgmap) 6842 { 6843 unsigned long pfn, end_pfn = start_pfn + nr_pages; 6844 struct pglist_data *pgdat = zone->zone_pgdat; 6845 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 6846 unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap); 6847 unsigned long zone_idx = zone_idx(zone); 6848 unsigned long start = jiffies; 6849 int nid = pgdat->node_id; 6850 6851 if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE)) 6852 return; 6853 6854 /* 6855 * The call to memmap_init should have already taken care 6856 * of the pages reserved for the memmap, so we can just jump to 6857 * the end of that region and start processing the device pages. 6858 */ 6859 if (altmap) { 6860 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6861 nr_pages = end_pfn - start_pfn; 6862 } 6863 6864 for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { 6865 struct page *page = pfn_to_page(pfn); 6866 6867 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); 6868 6869 if (pfns_per_compound == 1) 6870 continue; 6871 6872 memmap_init_compound(page, pfn, zone_idx, nid, pgmap, 6873 compound_nr_pages(altmap, pfns_per_compound)); 6874 } 6875 6876 pr_info("%s initialised %lu pages in %ums\n", __func__, 6877 nr_pages, jiffies_to_msecs(jiffies - start)); 6878 } 6879 6880 #endif 6881 static void __meminit zone_init_free_lists(struct zone *zone) 6882 { 6883 unsigned int order, t; 6884 for_each_migratetype_order(order, t) { 6885 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 6886 zone->free_area[order].nr_free = 0; 6887 } 6888 } 6889 6890 /* 6891 * Only struct pages that correspond to ranges defined by memblock.memory 6892 * are zeroed and initialized by going through __init_single_page() during 6893 * memmap_init_zone_range(). 6894 * 6895 * But, there could be struct pages that correspond to holes in 6896 * memblock.memory. This can happen because of the following reasons: 6897 * - physical memory bank size is not necessarily the exact multiple of the 6898 * arbitrary section size 6899 * - early reserved memory may not be listed in memblock.memory 6900 * - memory layouts defined with memmap= kernel parameter may not align 6901 * nicely with memmap sections 6902 * 6903 * Explicitly initialize those struct pages so that: 6904 * - PG_Reserved is set 6905 * - zone and node links point to zone and node that span the page if the 6906 * hole is in the middle of a zone 6907 * - zone and node links point to adjacent zone/node if the hole falls on 6908 * the zone boundary; the pages in such holes will be prepended to the 6909 * zone/node above the hole except for the trailing pages in the last 6910 * section that will be appended to the zone/node below. 6911 */ 6912 static void __init init_unavailable_range(unsigned long spfn, 6913 unsigned long epfn, 6914 int zone, int node) 6915 { 6916 unsigned long pfn; 6917 u64 pgcnt = 0; 6918 6919 for (pfn = spfn; pfn < epfn; pfn++) { 6920 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) { 6921 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages) 6922 + pageblock_nr_pages - 1; 6923 continue; 6924 } 6925 __init_single_page(pfn_to_page(pfn), pfn, zone, node); 6926 __SetPageReserved(pfn_to_page(pfn)); 6927 pgcnt++; 6928 } 6929 6930 if (pgcnt) 6931 pr_info("On node %d, zone %s: %lld pages in unavailable ranges", 6932 node, zone_names[zone], pgcnt); 6933 } 6934 6935 static void __init memmap_init_zone_range(struct zone *zone, 6936 unsigned long start_pfn, 6937 unsigned long end_pfn, 6938 unsigned long *hole_pfn) 6939 { 6940 unsigned long zone_start_pfn = zone->zone_start_pfn; 6941 unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; 6942 int nid = zone_to_nid(zone), zone_id = zone_idx(zone); 6943 6944 start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn); 6945 end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn); 6946 6947 if (start_pfn >= end_pfn) 6948 return; 6949 6950 memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, 6951 zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); 6952 6953 if (*hole_pfn < start_pfn) 6954 init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); 6955 6956 *hole_pfn = end_pfn; 6957 } 6958 6959 static void __init memmap_init(void) 6960 { 6961 unsigned long start_pfn, end_pfn; 6962 unsigned long hole_pfn = 0; 6963 int i, j, zone_id = 0, nid; 6964 6965 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 6966 struct pglist_data *node = NODE_DATA(nid); 6967 6968 for (j = 0; j < MAX_NR_ZONES; j++) { 6969 struct zone *zone = node->node_zones + j; 6970 6971 if (!populated_zone(zone)) 6972 continue; 6973 6974 memmap_init_zone_range(zone, start_pfn, end_pfn, 6975 &hole_pfn); 6976 zone_id = j; 6977 } 6978 } 6979 6980 #ifdef CONFIG_SPARSEMEM 6981 /* 6982 * Initialize the memory map for hole in the range [memory_end, 6983 * section_end]. 6984 * Append the pages in this hole to the highest zone in the last 6985 * node. 6986 * The call to init_unavailable_range() is outside the ifdef to 6987 * silence the compiler warining about zone_id set but not used; 6988 * for FLATMEM it is a nop anyway 6989 */ 6990 end_pfn = round_up(end_pfn, PAGES_PER_SECTION); 6991 if (hole_pfn < end_pfn) 6992 #endif 6993 init_unavailable_range(hole_pfn, end_pfn, zone_id, nid); 6994 } 6995 6996 void __init *memmap_alloc(phys_addr_t size, phys_addr_t align, 6997 phys_addr_t min_addr, int nid, bool exact_nid) 6998 { 6999 void *ptr; 7000 7001 if (exact_nid) 7002 ptr = memblock_alloc_exact_nid_raw(size, align, min_addr, 7003 MEMBLOCK_ALLOC_ACCESSIBLE, 7004 nid); 7005 else 7006 ptr = memblock_alloc_try_nid_raw(size, align, min_addr, 7007 MEMBLOCK_ALLOC_ACCESSIBLE, 7008 nid); 7009 7010 if (ptr && size > 0) 7011 page_init_poison(ptr, size); 7012 7013 return ptr; 7014 } 7015 7016 static int zone_batchsize(struct zone *zone) 7017 { 7018 #ifdef CONFIG_MMU 7019 int batch; 7020 7021 /* 7022 * The number of pages to batch allocate is either ~0.1% 7023 * of the zone or 1MB, whichever is smaller. The batch 7024 * size is striking a balance between allocation latency 7025 * and zone lock contention. 7026 */ 7027 batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE); 7028 batch /= 4; /* We effectively *= 4 below */ 7029 if (batch < 1) 7030 batch = 1; 7031 7032 /* 7033 * Clamp the batch to a 2^n - 1 value. Having a power 7034 * of 2 value was found to be more likely to have 7035 * suboptimal cache aliasing properties in some cases. 7036 * 7037 * For example if 2 tasks are alternately allocating 7038 * batches of pages, one task can end up with a lot 7039 * of pages of one half of the possible page colors 7040 * and the other with pages of the other colors. 7041 */ 7042 batch = rounddown_pow_of_two(batch + batch/2) - 1; 7043 7044 return batch; 7045 7046 #else 7047 /* The deferral and batching of frees should be suppressed under NOMMU 7048 * conditions. 7049 * 7050 * The problem is that NOMMU needs to be able to allocate large chunks 7051 * of contiguous memory as there's no hardware page translation to 7052 * assemble apparent contiguous memory from discontiguous pages. 7053 * 7054 * Queueing large contiguous runs of pages for batching, however, 7055 * causes the pages to actually be freed in smaller chunks. As there 7056 * can be a significant delay between the individual batches being 7057 * recycled, this leads to the once large chunks of space being 7058 * fragmented and becoming unavailable for high-order allocations. 7059 */ 7060 return 0; 7061 #endif 7062 } 7063 7064 static int zone_highsize(struct zone *zone, int batch, int cpu_online) 7065 { 7066 #ifdef CONFIG_MMU 7067 int high; 7068 int nr_split_cpus; 7069 unsigned long total_pages; 7070 7071 if (!percpu_pagelist_high_fraction) { 7072 /* 7073 * By default, the high value of the pcp is based on the zone 7074 * low watermark so that if they are full then background 7075 * reclaim will not be started prematurely. 7076 */ 7077 total_pages = low_wmark_pages(zone); 7078 } else { 7079 /* 7080 * If percpu_pagelist_high_fraction is configured, the high 7081 * value is based on a fraction of the managed pages in the 7082 * zone. 7083 */ 7084 total_pages = zone_managed_pages(zone) / percpu_pagelist_high_fraction; 7085 } 7086 7087 /* 7088 * Split the high value across all online CPUs local to the zone. Note 7089 * that early in boot that CPUs may not be online yet and that during 7090 * CPU hotplug that the cpumask is not yet updated when a CPU is being 7091 * onlined. For memory nodes that have no CPUs, split pcp->high across 7092 * all online CPUs to mitigate the risk that reclaim is triggered 7093 * prematurely due to pages stored on pcp lists. 7094 */ 7095 nr_split_cpus = cpumask_weight(cpumask_of_node(zone_to_nid(zone))) + cpu_online; 7096 if (!nr_split_cpus) 7097 nr_split_cpus = num_online_cpus(); 7098 high = total_pages / nr_split_cpus; 7099 7100 /* 7101 * Ensure high is at least batch*4. The multiple is based on the 7102 * historical relationship between high and batch. 7103 */ 7104 high = max(high, batch << 2); 7105 7106 return high; 7107 #else 7108 return 0; 7109 #endif 7110 } 7111 7112 /* 7113 * pcp->high and pcp->batch values are related and generally batch is lower 7114 * than high. They are also related to pcp->count such that count is lower 7115 * than high, and as soon as it reaches high, the pcplist is flushed. 7116 * 7117 * However, guaranteeing these relations at all times would require e.g. write 7118 * barriers here but also careful usage of read barriers at the read side, and 7119 * thus be prone to error and bad for performance. Thus the update only prevents 7120 * store tearing. Any new users of pcp->batch and pcp->high should ensure they 7121 * can cope with those fields changing asynchronously, and fully trust only the 7122 * pcp->count field on the local CPU with interrupts disabled. 7123 * 7124 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 7125 * outside of boot time (or some other assurance that no concurrent updaters 7126 * exist). 7127 */ 7128 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 7129 unsigned long batch) 7130 { 7131 WRITE_ONCE(pcp->batch, batch); 7132 WRITE_ONCE(pcp->high, high); 7133 } 7134 7135 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats) 7136 { 7137 int pindex; 7138 7139 memset(pcp, 0, sizeof(*pcp)); 7140 memset(pzstats, 0, sizeof(*pzstats)); 7141 7142 spin_lock_init(&pcp->lock); 7143 for (pindex = 0; pindex < NR_PCP_LISTS; pindex++) 7144 INIT_LIST_HEAD(&pcp->lists[pindex]); 7145 7146 /* 7147 * Set batch and high values safe for a boot pageset. A true percpu 7148 * pageset's initialization will update them subsequently. Here we don't 7149 * need to be as careful as pageset_update() as nobody can access the 7150 * pageset yet. 7151 */ 7152 pcp->high = BOOT_PAGESET_HIGH; 7153 pcp->batch = BOOT_PAGESET_BATCH; 7154 pcp->free_factor = 0; 7155 } 7156 7157 static void __zone_set_pageset_high_and_batch(struct zone *zone, unsigned long high, 7158 unsigned long batch) 7159 { 7160 struct per_cpu_pages *pcp; 7161 int cpu; 7162 7163 for_each_possible_cpu(cpu) { 7164 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 7165 pageset_update(pcp, high, batch); 7166 } 7167 } 7168 7169 /* 7170 * Calculate and set new high and batch values for all per-cpu pagesets of a 7171 * zone based on the zone's size. 7172 */ 7173 static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online) 7174 { 7175 int new_high, new_batch; 7176 7177 new_batch = max(1, zone_batchsize(zone)); 7178 new_high = zone_highsize(zone, new_batch, cpu_online); 7179 7180 if (zone->pageset_high == new_high && 7181 zone->pageset_batch == new_batch) 7182 return; 7183 7184 zone->pageset_high = new_high; 7185 zone->pageset_batch = new_batch; 7186 7187 __zone_set_pageset_high_and_batch(zone, new_high, new_batch); 7188 } 7189 7190 void __meminit setup_zone_pageset(struct zone *zone) 7191 { 7192 int cpu; 7193 7194 /* Size may be 0 on !SMP && !NUMA */ 7195 if (sizeof(struct per_cpu_zonestat) > 0) 7196 zone->per_cpu_zonestats = alloc_percpu(struct per_cpu_zonestat); 7197 7198 zone->per_cpu_pageset = alloc_percpu(struct per_cpu_pages); 7199 for_each_possible_cpu(cpu) { 7200 struct per_cpu_pages *pcp; 7201 struct per_cpu_zonestat *pzstats; 7202 7203 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 7204 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 7205 per_cpu_pages_init(pcp, pzstats); 7206 } 7207 7208 zone_set_pageset_high_and_batch(zone, 0); 7209 } 7210 7211 /* 7212 * Allocate per cpu pagesets and initialize them. 7213 * Before this call only boot pagesets were available. 7214 */ 7215 void __init setup_per_cpu_pageset(void) 7216 { 7217 struct pglist_data *pgdat; 7218 struct zone *zone; 7219 int __maybe_unused cpu; 7220 7221 for_each_populated_zone(zone) 7222 setup_zone_pageset(zone); 7223 7224 #ifdef CONFIG_NUMA 7225 /* 7226 * Unpopulated zones continue using the boot pagesets. 7227 * The numa stats for these pagesets need to be reset. 7228 * Otherwise, they will end up skewing the stats of 7229 * the nodes these zones are associated with. 7230 */ 7231 for_each_possible_cpu(cpu) { 7232 struct per_cpu_zonestat *pzstats = &per_cpu(boot_zonestats, cpu); 7233 memset(pzstats->vm_numa_event, 0, 7234 sizeof(pzstats->vm_numa_event)); 7235 } 7236 #endif 7237 7238 for_each_online_pgdat(pgdat) 7239 pgdat->per_cpu_nodestats = 7240 alloc_percpu(struct per_cpu_nodestat); 7241 } 7242 7243 static __meminit void zone_pcp_init(struct zone *zone) 7244 { 7245 /* 7246 * per cpu subsystem is not up at this point. The following code 7247 * relies on the ability of the linker to provide the 7248 * offset of a (static) per cpu variable into the per cpu area. 7249 */ 7250 zone->per_cpu_pageset = &boot_pageset; 7251 zone->per_cpu_zonestats = &boot_zonestats; 7252 zone->pageset_high = BOOT_PAGESET_HIGH; 7253 zone->pageset_batch = BOOT_PAGESET_BATCH; 7254 7255 if (populated_zone(zone)) 7256 pr_debug(" %s zone: %lu pages, LIFO batch:%u\n", zone->name, 7257 zone->present_pages, zone_batchsize(zone)); 7258 } 7259 7260 void __meminit init_currently_empty_zone(struct zone *zone, 7261 unsigned long zone_start_pfn, 7262 unsigned long size) 7263 { 7264 struct pglist_data *pgdat = zone->zone_pgdat; 7265 int zone_idx = zone_idx(zone) + 1; 7266 7267 if (zone_idx > pgdat->nr_zones) 7268 pgdat->nr_zones = zone_idx; 7269 7270 zone->zone_start_pfn = zone_start_pfn; 7271 7272 mminit_dprintk(MMINIT_TRACE, "memmap_init", 7273 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 7274 pgdat->node_id, 7275 (unsigned long)zone_idx(zone), 7276 zone_start_pfn, (zone_start_pfn + size)); 7277 7278 zone_init_free_lists(zone); 7279 zone->initialized = 1; 7280 } 7281 7282 /** 7283 * get_pfn_range_for_nid - Return the start and end page frames for a node 7284 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 7285 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 7286 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 7287 * 7288 * It returns the start and end page frame of a node based on information 7289 * provided by memblock_set_node(). If called for a node 7290 * with no available memory, a warning is printed and the start and end 7291 * PFNs will be 0. 7292 */ 7293 void __init get_pfn_range_for_nid(unsigned int nid, 7294 unsigned long *start_pfn, unsigned long *end_pfn) 7295 { 7296 unsigned long this_start_pfn, this_end_pfn; 7297 int i; 7298 7299 *start_pfn = -1UL; 7300 *end_pfn = 0; 7301 7302 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 7303 *start_pfn = min(*start_pfn, this_start_pfn); 7304 *end_pfn = max(*end_pfn, this_end_pfn); 7305 } 7306 7307 if (*start_pfn == -1UL) 7308 *start_pfn = 0; 7309 } 7310 7311 /* 7312 * This finds a zone that can be used for ZONE_MOVABLE pages. The 7313 * assumption is made that zones within a node are ordered in monotonic 7314 * increasing memory addresses so that the "highest" populated zone is used 7315 */ 7316 static void __init find_usable_zone_for_movable(void) 7317 { 7318 int zone_index; 7319 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 7320 if (zone_index == ZONE_MOVABLE) 7321 continue; 7322 7323 if (arch_zone_highest_possible_pfn[zone_index] > 7324 arch_zone_lowest_possible_pfn[zone_index]) 7325 break; 7326 } 7327 7328 VM_BUG_ON(zone_index == -1); 7329 movable_zone = zone_index; 7330 } 7331 7332 /* 7333 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 7334 * because it is sized independent of architecture. Unlike the other zones, 7335 * the starting point for ZONE_MOVABLE is not fixed. It may be different 7336 * in each node depending on the size of each node and how evenly kernelcore 7337 * is distributed. This helper function adjusts the zone ranges 7338 * provided by the architecture for a given node by using the end of the 7339 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 7340 * zones within a node are in order of monotonic increases memory addresses 7341 */ 7342 static void __init adjust_zone_range_for_zone_movable(int nid, 7343 unsigned long zone_type, 7344 unsigned long node_start_pfn, 7345 unsigned long node_end_pfn, 7346 unsigned long *zone_start_pfn, 7347 unsigned long *zone_end_pfn) 7348 { 7349 /* Only adjust if ZONE_MOVABLE is on this node */ 7350 if (zone_movable_pfn[nid]) { 7351 /* Size ZONE_MOVABLE */ 7352 if (zone_type == ZONE_MOVABLE) { 7353 *zone_start_pfn = zone_movable_pfn[nid]; 7354 *zone_end_pfn = min(node_end_pfn, 7355 arch_zone_highest_possible_pfn[movable_zone]); 7356 7357 /* Adjust for ZONE_MOVABLE starting within this range */ 7358 } else if (!mirrored_kernelcore && 7359 *zone_start_pfn < zone_movable_pfn[nid] && 7360 *zone_end_pfn > zone_movable_pfn[nid]) { 7361 *zone_end_pfn = zone_movable_pfn[nid]; 7362 7363 /* Check if this whole range is within ZONE_MOVABLE */ 7364 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 7365 *zone_start_pfn = *zone_end_pfn; 7366 } 7367 } 7368 7369 /* 7370 * Return the number of pages a zone spans in a node, including holes 7371 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 7372 */ 7373 static unsigned long __init zone_spanned_pages_in_node(int nid, 7374 unsigned long zone_type, 7375 unsigned long node_start_pfn, 7376 unsigned long node_end_pfn, 7377 unsigned long *zone_start_pfn, 7378 unsigned long *zone_end_pfn) 7379 { 7380 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 7381 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 7382 /* When hotadd a new node from cpu_up(), the node should be empty */ 7383 if (!node_start_pfn && !node_end_pfn) 7384 return 0; 7385 7386 /* Get the start and end of the zone */ 7387 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 7388 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 7389 adjust_zone_range_for_zone_movable(nid, zone_type, 7390 node_start_pfn, node_end_pfn, 7391 zone_start_pfn, zone_end_pfn); 7392 7393 /* Check that this node has pages within the zone's required range */ 7394 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 7395 return 0; 7396 7397 /* Move the zone boundaries inside the node if necessary */ 7398 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 7399 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 7400 7401 /* Return the spanned pages */ 7402 return *zone_end_pfn - *zone_start_pfn; 7403 } 7404 7405 /* 7406 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 7407 * then all holes in the requested range will be accounted for. 7408 */ 7409 unsigned long __init __absent_pages_in_range(int nid, 7410 unsigned long range_start_pfn, 7411 unsigned long range_end_pfn) 7412 { 7413 unsigned long nr_absent = range_end_pfn - range_start_pfn; 7414 unsigned long start_pfn, end_pfn; 7415 int i; 7416 7417 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7418 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 7419 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 7420 nr_absent -= end_pfn - start_pfn; 7421 } 7422 return nr_absent; 7423 } 7424 7425 /** 7426 * absent_pages_in_range - Return number of page frames in holes within a range 7427 * @start_pfn: The start PFN to start searching for holes 7428 * @end_pfn: The end PFN to stop searching for holes 7429 * 7430 * Return: the number of pages frames in memory holes within a range. 7431 */ 7432 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 7433 unsigned long end_pfn) 7434 { 7435 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 7436 } 7437 7438 /* Return the number of page frames in holes in a zone on a node */ 7439 static unsigned long __init zone_absent_pages_in_node(int nid, 7440 unsigned long zone_type, 7441 unsigned long node_start_pfn, 7442 unsigned long node_end_pfn) 7443 { 7444 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 7445 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 7446 unsigned long zone_start_pfn, zone_end_pfn; 7447 unsigned long nr_absent; 7448 7449 /* When hotadd a new node from cpu_up(), the node should be empty */ 7450 if (!node_start_pfn && !node_end_pfn) 7451 return 0; 7452 7453 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 7454 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 7455 7456 adjust_zone_range_for_zone_movable(nid, zone_type, 7457 node_start_pfn, node_end_pfn, 7458 &zone_start_pfn, &zone_end_pfn); 7459 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 7460 7461 /* 7462 * ZONE_MOVABLE handling. 7463 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 7464 * and vice versa. 7465 */ 7466 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 7467 unsigned long start_pfn, end_pfn; 7468 struct memblock_region *r; 7469 7470 for_each_mem_region(r) { 7471 start_pfn = clamp(memblock_region_memory_base_pfn(r), 7472 zone_start_pfn, zone_end_pfn); 7473 end_pfn = clamp(memblock_region_memory_end_pfn(r), 7474 zone_start_pfn, zone_end_pfn); 7475 7476 if (zone_type == ZONE_MOVABLE && 7477 memblock_is_mirror(r)) 7478 nr_absent += end_pfn - start_pfn; 7479 7480 if (zone_type == ZONE_NORMAL && 7481 !memblock_is_mirror(r)) 7482 nr_absent += end_pfn - start_pfn; 7483 } 7484 } 7485 7486 return nr_absent; 7487 } 7488 7489 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 7490 unsigned long node_start_pfn, 7491 unsigned long node_end_pfn) 7492 { 7493 unsigned long realtotalpages = 0, totalpages = 0; 7494 enum zone_type i; 7495 7496 for (i = 0; i < MAX_NR_ZONES; i++) { 7497 struct zone *zone = pgdat->node_zones + i; 7498 unsigned long zone_start_pfn, zone_end_pfn; 7499 unsigned long spanned, absent; 7500 unsigned long size, real_size; 7501 7502 spanned = zone_spanned_pages_in_node(pgdat->node_id, i, 7503 node_start_pfn, 7504 node_end_pfn, 7505 &zone_start_pfn, 7506 &zone_end_pfn); 7507 absent = zone_absent_pages_in_node(pgdat->node_id, i, 7508 node_start_pfn, 7509 node_end_pfn); 7510 7511 size = spanned; 7512 real_size = size - absent; 7513 7514 if (size) 7515 zone->zone_start_pfn = zone_start_pfn; 7516 else 7517 zone->zone_start_pfn = 0; 7518 zone->spanned_pages = size; 7519 zone->present_pages = real_size; 7520 #if defined(CONFIG_MEMORY_HOTPLUG) 7521 zone->present_early_pages = real_size; 7522 #endif 7523 7524 totalpages += size; 7525 realtotalpages += real_size; 7526 } 7527 7528 pgdat->node_spanned_pages = totalpages; 7529 pgdat->node_present_pages = realtotalpages; 7530 pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); 7531 } 7532 7533 #ifndef CONFIG_SPARSEMEM 7534 /* 7535 * Calculate the size of the zone->blockflags rounded to an unsigned long 7536 * Start by making sure zonesize is a multiple of pageblock_order by rounding 7537 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 7538 * round what is now in bits to nearest long in bits, then return it in 7539 * bytes. 7540 */ 7541 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 7542 { 7543 unsigned long usemapsize; 7544 7545 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 7546 usemapsize = roundup(zonesize, pageblock_nr_pages); 7547 usemapsize = usemapsize >> pageblock_order; 7548 usemapsize *= NR_PAGEBLOCK_BITS; 7549 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 7550 7551 return usemapsize / 8; 7552 } 7553 7554 static void __ref setup_usemap(struct zone *zone) 7555 { 7556 unsigned long usemapsize = usemap_size(zone->zone_start_pfn, 7557 zone->spanned_pages); 7558 zone->pageblock_flags = NULL; 7559 if (usemapsize) { 7560 zone->pageblock_flags = 7561 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 7562 zone_to_nid(zone)); 7563 if (!zone->pageblock_flags) 7564 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 7565 usemapsize, zone->name, zone_to_nid(zone)); 7566 } 7567 } 7568 #else 7569 static inline void setup_usemap(struct zone *zone) {} 7570 #endif /* CONFIG_SPARSEMEM */ 7571 7572 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 7573 7574 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 7575 void __init set_pageblock_order(void) 7576 { 7577 unsigned int order = MAX_ORDER - 1; 7578 7579 /* Check that pageblock_nr_pages has not already been setup */ 7580 if (pageblock_order) 7581 return; 7582 7583 /* Don't let pageblocks exceed the maximum allocation granularity. */ 7584 if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order) 7585 order = HUGETLB_PAGE_ORDER; 7586 7587 /* 7588 * Assume the largest contiguous order of interest is a huge page. 7589 * This value may be variable depending on boot parameters on IA64 and 7590 * powerpc. 7591 */ 7592 pageblock_order = order; 7593 } 7594 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 7595 7596 /* 7597 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 7598 * is unused as pageblock_order is set at compile-time. See 7599 * include/linux/pageblock-flags.h for the values of pageblock_order based on 7600 * the kernel config 7601 */ 7602 void __init set_pageblock_order(void) 7603 { 7604 } 7605 7606 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 7607 7608 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 7609 unsigned long present_pages) 7610 { 7611 unsigned long pages = spanned_pages; 7612 7613 /* 7614 * Provide a more accurate estimation if there are holes within 7615 * the zone and SPARSEMEM is in use. If there are holes within the 7616 * zone, each populated memory region may cost us one or two extra 7617 * memmap pages due to alignment because memmap pages for each 7618 * populated regions may not be naturally aligned on page boundary. 7619 * So the (present_pages >> 4) heuristic is a tradeoff for that. 7620 */ 7621 if (spanned_pages > present_pages + (present_pages >> 4) && 7622 IS_ENABLED(CONFIG_SPARSEMEM)) 7623 pages = present_pages; 7624 7625 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 7626 } 7627 7628 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 7629 static void pgdat_init_split_queue(struct pglist_data *pgdat) 7630 { 7631 struct deferred_split *ds_queue = &pgdat->deferred_split_queue; 7632 7633 spin_lock_init(&ds_queue->split_queue_lock); 7634 INIT_LIST_HEAD(&ds_queue->split_queue); 7635 ds_queue->split_queue_len = 0; 7636 } 7637 #else 7638 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 7639 #endif 7640 7641 #ifdef CONFIG_COMPACTION 7642 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 7643 { 7644 init_waitqueue_head(&pgdat->kcompactd_wait); 7645 } 7646 #else 7647 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 7648 #endif 7649 7650 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 7651 { 7652 int i; 7653 7654 pgdat_resize_init(pgdat); 7655 pgdat_kswapd_lock_init(pgdat); 7656 7657 pgdat_init_split_queue(pgdat); 7658 pgdat_init_kcompactd(pgdat); 7659 7660 init_waitqueue_head(&pgdat->kswapd_wait); 7661 init_waitqueue_head(&pgdat->pfmemalloc_wait); 7662 7663 for (i = 0; i < NR_VMSCAN_THROTTLE; i++) 7664 init_waitqueue_head(&pgdat->reclaim_wait[i]); 7665 7666 pgdat_page_ext_init(pgdat); 7667 lruvec_init(&pgdat->__lruvec); 7668 } 7669 7670 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 7671 unsigned long remaining_pages) 7672 { 7673 atomic_long_set(&zone->managed_pages, remaining_pages); 7674 zone_set_nid(zone, nid); 7675 zone->name = zone_names[idx]; 7676 zone->zone_pgdat = NODE_DATA(nid); 7677 spin_lock_init(&zone->lock); 7678 zone_seqlock_init(zone); 7679 zone_pcp_init(zone); 7680 } 7681 7682 /* 7683 * Set up the zone data structures 7684 * - init pgdat internals 7685 * - init all zones belonging to this node 7686 * 7687 * NOTE: this function is only called during memory hotplug 7688 */ 7689 #ifdef CONFIG_MEMORY_HOTPLUG 7690 void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) 7691 { 7692 int nid = pgdat->node_id; 7693 enum zone_type z; 7694 int cpu; 7695 7696 pgdat_init_internals(pgdat); 7697 7698 if (pgdat->per_cpu_nodestats == &boot_nodestats) 7699 pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); 7700 7701 /* 7702 * Reset the nr_zones, order and highest_zoneidx before reuse. 7703 * Note that kswapd will init kswapd_highest_zoneidx properly 7704 * when it starts in the near future. 7705 */ 7706 pgdat->nr_zones = 0; 7707 pgdat->kswapd_order = 0; 7708 pgdat->kswapd_highest_zoneidx = 0; 7709 pgdat->node_start_pfn = 0; 7710 for_each_online_cpu(cpu) { 7711 struct per_cpu_nodestat *p; 7712 7713 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 7714 memset(p, 0, sizeof(*p)); 7715 } 7716 7717 for (z = 0; z < MAX_NR_ZONES; z++) 7718 zone_init_internals(&pgdat->node_zones[z], z, nid, 0); 7719 } 7720 #endif 7721 7722 /* 7723 * Set up the zone data structures: 7724 * - mark all pages reserved 7725 * - mark all memory queues empty 7726 * - clear the memory bitmaps 7727 * 7728 * NOTE: pgdat should get zeroed by caller. 7729 * NOTE: this function is only called during early init. 7730 */ 7731 static void __init free_area_init_core(struct pglist_data *pgdat) 7732 { 7733 enum zone_type j; 7734 int nid = pgdat->node_id; 7735 7736 pgdat_init_internals(pgdat); 7737 pgdat->per_cpu_nodestats = &boot_nodestats; 7738 7739 for (j = 0; j < MAX_NR_ZONES; j++) { 7740 struct zone *zone = pgdat->node_zones + j; 7741 unsigned long size, freesize, memmap_pages; 7742 7743 size = zone->spanned_pages; 7744 freesize = zone->present_pages; 7745 7746 /* 7747 * Adjust freesize so that it accounts for how much memory 7748 * is used by this zone for memmap. This affects the watermark 7749 * and per-cpu initialisations 7750 */ 7751 memmap_pages = calc_memmap_size(size, freesize); 7752 if (!is_highmem_idx(j)) { 7753 if (freesize >= memmap_pages) { 7754 freesize -= memmap_pages; 7755 if (memmap_pages) 7756 pr_debug(" %s zone: %lu pages used for memmap\n", 7757 zone_names[j], memmap_pages); 7758 } else 7759 pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n", 7760 zone_names[j], memmap_pages, freesize); 7761 } 7762 7763 /* Account for reserved pages */ 7764 if (j == 0 && freesize > dma_reserve) { 7765 freesize -= dma_reserve; 7766 pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve); 7767 } 7768 7769 if (!is_highmem_idx(j)) 7770 nr_kernel_pages += freesize; 7771 /* Charge for highmem memmap if there are enough kernel pages */ 7772 else if (nr_kernel_pages > memmap_pages * 2) 7773 nr_kernel_pages -= memmap_pages; 7774 nr_all_pages += freesize; 7775 7776 /* 7777 * Set an approximate value for lowmem here, it will be adjusted 7778 * when the bootmem allocator frees pages into the buddy system. 7779 * And all highmem pages will be managed by the buddy system. 7780 */ 7781 zone_init_internals(zone, j, nid, freesize); 7782 7783 if (!size) 7784 continue; 7785 7786 set_pageblock_order(); 7787 setup_usemap(zone); 7788 init_currently_empty_zone(zone, zone->zone_start_pfn, size); 7789 } 7790 } 7791 7792 #ifdef CONFIG_FLATMEM 7793 static void __init alloc_node_mem_map(struct pglist_data *pgdat) 7794 { 7795 unsigned long __maybe_unused start = 0; 7796 unsigned long __maybe_unused offset = 0; 7797 7798 /* Skip empty nodes */ 7799 if (!pgdat->node_spanned_pages) 7800 return; 7801 7802 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 7803 offset = pgdat->node_start_pfn - start; 7804 /* ia64 gets its own node_mem_map, before this, without bootmem */ 7805 if (!pgdat->node_mem_map) { 7806 unsigned long size, end; 7807 struct page *map; 7808 7809 /* 7810 * The zone's endpoints aren't required to be MAX_ORDER 7811 * aligned but the node_mem_map endpoints must be in order 7812 * for the buddy allocator to function correctly. 7813 */ 7814 end = pgdat_end_pfn(pgdat); 7815 end = ALIGN(end, MAX_ORDER_NR_PAGES); 7816 size = (end - start) * sizeof(struct page); 7817 map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT, 7818 pgdat->node_id, false); 7819 if (!map) 7820 panic("Failed to allocate %ld bytes for node %d memory map\n", 7821 size, pgdat->node_id); 7822 pgdat->node_mem_map = map + offset; 7823 } 7824 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 7825 __func__, pgdat->node_id, (unsigned long)pgdat, 7826 (unsigned long)pgdat->node_mem_map); 7827 #ifndef CONFIG_NUMA 7828 /* 7829 * With no DISCONTIG, the global mem_map is just set as node 0's 7830 */ 7831 if (pgdat == NODE_DATA(0)) { 7832 mem_map = NODE_DATA(0)->node_mem_map; 7833 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 7834 mem_map -= offset; 7835 } 7836 #endif 7837 } 7838 #else 7839 static inline void alloc_node_mem_map(struct pglist_data *pgdat) { } 7840 #endif /* CONFIG_FLATMEM */ 7841 7842 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 7843 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 7844 { 7845 pgdat->first_deferred_pfn = ULONG_MAX; 7846 } 7847 #else 7848 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 7849 #endif 7850 7851 static void __init free_area_init_node(int nid) 7852 { 7853 pg_data_t *pgdat = NODE_DATA(nid); 7854 unsigned long start_pfn = 0; 7855 unsigned long end_pfn = 0; 7856 7857 /* pg_data_t should be reset to zero when it's allocated */ 7858 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); 7859 7860 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 7861 7862 pgdat->node_id = nid; 7863 pgdat->node_start_pfn = start_pfn; 7864 pgdat->per_cpu_nodestats = NULL; 7865 7866 if (start_pfn != end_pfn) { 7867 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 7868 (u64)start_pfn << PAGE_SHIFT, 7869 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 7870 } else { 7871 pr_info("Initmem setup node %d as memoryless\n", nid); 7872 } 7873 7874 calculate_node_totalpages(pgdat, start_pfn, end_pfn); 7875 7876 alloc_node_mem_map(pgdat); 7877 pgdat_set_deferred_range(pgdat); 7878 7879 free_area_init_core(pgdat); 7880 } 7881 7882 static void __init free_area_init_memoryless_node(int nid) 7883 { 7884 free_area_init_node(nid); 7885 } 7886 7887 #if MAX_NUMNODES > 1 7888 /* 7889 * Figure out the number of possible node ids. 7890 */ 7891 void __init setup_nr_node_ids(void) 7892 { 7893 unsigned int highest; 7894 7895 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 7896 nr_node_ids = highest + 1; 7897 } 7898 #endif 7899 7900 /** 7901 * node_map_pfn_alignment - determine the maximum internode alignment 7902 * 7903 * This function should be called after node map is populated and sorted. 7904 * It calculates the maximum power of two alignment which can distinguish 7905 * all the nodes. 7906 * 7907 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 7908 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 7909 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 7910 * shifted, 1GiB is enough and this function will indicate so. 7911 * 7912 * This is used to test whether pfn -> nid mapping of the chosen memory 7913 * model has fine enough granularity to avoid incorrect mapping for the 7914 * populated node map. 7915 * 7916 * Return: the determined alignment in pfn's. 0 if there is no alignment 7917 * requirement (single node). 7918 */ 7919 unsigned long __init node_map_pfn_alignment(void) 7920 { 7921 unsigned long accl_mask = 0, last_end = 0; 7922 unsigned long start, end, mask; 7923 int last_nid = NUMA_NO_NODE; 7924 int i, nid; 7925 7926 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 7927 if (!start || last_nid < 0 || last_nid == nid) { 7928 last_nid = nid; 7929 last_end = end; 7930 continue; 7931 } 7932 7933 /* 7934 * Start with a mask granular enough to pin-point to the 7935 * start pfn and tick off bits one-by-one until it becomes 7936 * too coarse to separate the current node from the last. 7937 */ 7938 mask = ~((1 << __ffs(start)) - 1); 7939 while (mask && last_end <= (start & (mask << 1))) 7940 mask <<= 1; 7941 7942 /* accumulate all internode masks */ 7943 accl_mask |= mask; 7944 } 7945 7946 /* convert mask to number of pages */ 7947 return ~accl_mask + 1; 7948 } 7949 7950 /* 7951 * early_calculate_totalpages() 7952 * Sum pages in active regions for movable zone. 7953 * Populate N_MEMORY for calculating usable_nodes. 7954 */ 7955 static unsigned long __init early_calculate_totalpages(void) 7956 { 7957 unsigned long totalpages = 0; 7958 unsigned long start_pfn, end_pfn; 7959 int i, nid; 7960 7961 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7962 unsigned long pages = end_pfn - start_pfn; 7963 7964 totalpages += pages; 7965 if (pages) 7966 node_set_state(nid, N_MEMORY); 7967 } 7968 return totalpages; 7969 } 7970 7971 /* 7972 * Find the PFN the Movable zone begins in each node. Kernel memory 7973 * is spread evenly between nodes as long as the nodes have enough 7974 * memory. When they don't, some nodes will have more kernelcore than 7975 * others 7976 */ 7977 static void __init find_zone_movable_pfns_for_nodes(void) 7978 { 7979 int i, nid; 7980 unsigned long usable_startpfn; 7981 unsigned long kernelcore_node, kernelcore_remaining; 7982 /* save the state before borrow the nodemask */ 7983 nodemask_t saved_node_state = node_states[N_MEMORY]; 7984 unsigned long totalpages = early_calculate_totalpages(); 7985 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 7986 struct memblock_region *r; 7987 7988 /* Need to find movable_zone earlier when movable_node is specified. */ 7989 find_usable_zone_for_movable(); 7990 7991 /* 7992 * If movable_node is specified, ignore kernelcore and movablecore 7993 * options. 7994 */ 7995 if (movable_node_is_enabled()) { 7996 for_each_mem_region(r) { 7997 if (!memblock_is_hotpluggable(r)) 7998 continue; 7999 8000 nid = memblock_get_region_node(r); 8001 8002 usable_startpfn = PFN_DOWN(r->base); 8003 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 8004 min(usable_startpfn, zone_movable_pfn[nid]) : 8005 usable_startpfn; 8006 } 8007 8008 goto out2; 8009 } 8010 8011 /* 8012 * If kernelcore=mirror is specified, ignore movablecore option 8013 */ 8014 if (mirrored_kernelcore) { 8015 bool mem_below_4gb_not_mirrored = false; 8016 8017 for_each_mem_region(r) { 8018 if (memblock_is_mirror(r)) 8019 continue; 8020 8021 nid = memblock_get_region_node(r); 8022 8023 usable_startpfn = memblock_region_memory_base_pfn(r); 8024 8025 if (usable_startpfn < PHYS_PFN(SZ_4G)) { 8026 mem_below_4gb_not_mirrored = true; 8027 continue; 8028 } 8029 8030 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 8031 min(usable_startpfn, zone_movable_pfn[nid]) : 8032 usable_startpfn; 8033 } 8034 8035 if (mem_below_4gb_not_mirrored) 8036 pr_warn("This configuration results in unmirrored kernel memory.\n"); 8037 8038 goto out2; 8039 } 8040 8041 /* 8042 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 8043 * amount of necessary memory. 8044 */ 8045 if (required_kernelcore_percent) 8046 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 8047 10000UL; 8048 if (required_movablecore_percent) 8049 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 8050 10000UL; 8051 8052 /* 8053 * If movablecore= was specified, calculate what size of 8054 * kernelcore that corresponds so that memory usable for 8055 * any allocation type is evenly spread. If both kernelcore 8056 * and movablecore are specified, then the value of kernelcore 8057 * will be used for required_kernelcore if it's greater than 8058 * what movablecore would have allowed. 8059 */ 8060 if (required_movablecore) { 8061 unsigned long corepages; 8062 8063 /* 8064 * Round-up so that ZONE_MOVABLE is at least as large as what 8065 * was requested by the user 8066 */ 8067 required_movablecore = 8068 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 8069 required_movablecore = min(totalpages, required_movablecore); 8070 corepages = totalpages - required_movablecore; 8071 8072 required_kernelcore = max(required_kernelcore, corepages); 8073 } 8074 8075 /* 8076 * If kernelcore was not specified or kernelcore size is larger 8077 * than totalpages, there is no ZONE_MOVABLE. 8078 */ 8079 if (!required_kernelcore || required_kernelcore >= totalpages) 8080 goto out; 8081 8082 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 8083 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 8084 8085 restart: 8086 /* Spread kernelcore memory as evenly as possible throughout nodes */ 8087 kernelcore_node = required_kernelcore / usable_nodes; 8088 for_each_node_state(nid, N_MEMORY) { 8089 unsigned long start_pfn, end_pfn; 8090 8091 /* 8092 * Recalculate kernelcore_node if the division per node 8093 * now exceeds what is necessary to satisfy the requested 8094 * amount of memory for the kernel 8095 */ 8096 if (required_kernelcore < kernelcore_node) 8097 kernelcore_node = required_kernelcore / usable_nodes; 8098 8099 /* 8100 * As the map is walked, we track how much memory is usable 8101 * by the kernel using kernelcore_remaining. When it is 8102 * 0, the rest of the node is usable by ZONE_MOVABLE 8103 */ 8104 kernelcore_remaining = kernelcore_node; 8105 8106 /* Go through each range of PFNs within this node */ 8107 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 8108 unsigned long size_pages; 8109 8110 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 8111 if (start_pfn >= end_pfn) 8112 continue; 8113 8114 /* Account for what is only usable for kernelcore */ 8115 if (start_pfn < usable_startpfn) { 8116 unsigned long kernel_pages; 8117 kernel_pages = min(end_pfn, usable_startpfn) 8118 - start_pfn; 8119 8120 kernelcore_remaining -= min(kernel_pages, 8121 kernelcore_remaining); 8122 required_kernelcore -= min(kernel_pages, 8123 required_kernelcore); 8124 8125 /* Continue if range is now fully accounted */ 8126 if (end_pfn <= usable_startpfn) { 8127 8128 /* 8129 * Push zone_movable_pfn to the end so 8130 * that if we have to rebalance 8131 * kernelcore across nodes, we will 8132 * not double account here 8133 */ 8134 zone_movable_pfn[nid] = end_pfn; 8135 continue; 8136 } 8137 start_pfn = usable_startpfn; 8138 } 8139 8140 /* 8141 * The usable PFN range for ZONE_MOVABLE is from 8142 * start_pfn->end_pfn. Calculate size_pages as the 8143 * number of pages used as kernelcore 8144 */ 8145 size_pages = end_pfn - start_pfn; 8146 if (size_pages > kernelcore_remaining) 8147 size_pages = kernelcore_remaining; 8148 zone_movable_pfn[nid] = start_pfn + size_pages; 8149 8150 /* 8151 * Some kernelcore has been met, update counts and 8152 * break if the kernelcore for this node has been 8153 * satisfied 8154 */ 8155 required_kernelcore -= min(required_kernelcore, 8156 size_pages); 8157 kernelcore_remaining -= size_pages; 8158 if (!kernelcore_remaining) 8159 break; 8160 } 8161 } 8162 8163 /* 8164 * If there is still required_kernelcore, we do another pass with one 8165 * less node in the count. This will push zone_movable_pfn[nid] further 8166 * along on the nodes that still have memory until kernelcore is 8167 * satisfied 8168 */ 8169 usable_nodes--; 8170 if (usable_nodes && required_kernelcore > usable_nodes) 8171 goto restart; 8172 8173 out2: 8174 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 8175 for (nid = 0; nid < MAX_NUMNODES; nid++) { 8176 unsigned long start_pfn, end_pfn; 8177 8178 zone_movable_pfn[nid] = 8179 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 8180 8181 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 8182 if (zone_movable_pfn[nid] >= end_pfn) 8183 zone_movable_pfn[nid] = 0; 8184 } 8185 8186 out: 8187 /* restore the node_state */ 8188 node_states[N_MEMORY] = saved_node_state; 8189 } 8190 8191 /* Any regular or high memory on that node ? */ 8192 static void check_for_memory(pg_data_t *pgdat, int nid) 8193 { 8194 enum zone_type zone_type; 8195 8196 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 8197 struct zone *zone = &pgdat->node_zones[zone_type]; 8198 if (populated_zone(zone)) { 8199 if (IS_ENABLED(CONFIG_HIGHMEM)) 8200 node_set_state(nid, N_HIGH_MEMORY); 8201 if (zone_type <= ZONE_NORMAL) 8202 node_set_state(nid, N_NORMAL_MEMORY); 8203 break; 8204 } 8205 } 8206 } 8207 8208 /* 8209 * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For 8210 * such cases we allow max_zone_pfn sorted in the descending order 8211 */ 8212 bool __weak arch_has_descending_max_zone_pfns(void) 8213 { 8214 return false; 8215 } 8216 8217 /** 8218 * free_area_init - Initialise all pg_data_t and zone data 8219 * @max_zone_pfn: an array of max PFNs for each zone 8220 * 8221 * This will call free_area_init_node() for each active node in the system. 8222 * Using the page ranges provided by memblock_set_node(), the size of each 8223 * zone in each node and their holes is calculated. If the maximum PFN 8224 * between two adjacent zones match, it is assumed that the zone is empty. 8225 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 8226 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 8227 * starts where the previous one ended. For example, ZONE_DMA32 starts 8228 * at arch_max_dma_pfn. 8229 */ 8230 void __init free_area_init(unsigned long *max_zone_pfn) 8231 { 8232 unsigned long start_pfn, end_pfn; 8233 int i, nid, zone; 8234 bool descending; 8235 8236 /* Record where the zone boundaries are */ 8237 memset(arch_zone_lowest_possible_pfn, 0, 8238 sizeof(arch_zone_lowest_possible_pfn)); 8239 memset(arch_zone_highest_possible_pfn, 0, 8240 sizeof(arch_zone_highest_possible_pfn)); 8241 8242 start_pfn = PHYS_PFN(memblock_start_of_DRAM()); 8243 descending = arch_has_descending_max_zone_pfns(); 8244 8245 for (i = 0; i < MAX_NR_ZONES; i++) { 8246 if (descending) 8247 zone = MAX_NR_ZONES - i - 1; 8248 else 8249 zone = i; 8250 8251 if (zone == ZONE_MOVABLE) 8252 continue; 8253 8254 end_pfn = max(max_zone_pfn[zone], start_pfn); 8255 arch_zone_lowest_possible_pfn[zone] = start_pfn; 8256 arch_zone_highest_possible_pfn[zone] = end_pfn; 8257 8258 start_pfn = end_pfn; 8259 } 8260 8261 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 8262 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 8263 find_zone_movable_pfns_for_nodes(); 8264 8265 /* Print out the zone ranges */ 8266 pr_info("Zone ranges:\n"); 8267 for (i = 0; i < MAX_NR_ZONES; i++) { 8268 if (i == ZONE_MOVABLE) 8269 continue; 8270 pr_info(" %-8s ", zone_names[i]); 8271 if (arch_zone_lowest_possible_pfn[i] == 8272 arch_zone_highest_possible_pfn[i]) 8273 pr_cont("empty\n"); 8274 else 8275 pr_cont("[mem %#018Lx-%#018Lx]\n", 8276 (u64)arch_zone_lowest_possible_pfn[i] 8277 << PAGE_SHIFT, 8278 ((u64)arch_zone_highest_possible_pfn[i] 8279 << PAGE_SHIFT) - 1); 8280 } 8281 8282 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 8283 pr_info("Movable zone start for each node\n"); 8284 for (i = 0; i < MAX_NUMNODES; i++) { 8285 if (zone_movable_pfn[i]) 8286 pr_info(" Node %d: %#018Lx\n", i, 8287 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 8288 } 8289 8290 /* 8291 * Print out the early node map, and initialize the 8292 * subsection-map relative to active online memory ranges to 8293 * enable future "sub-section" extensions of the memory map. 8294 */ 8295 pr_info("Early memory node ranges\n"); 8296 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 8297 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 8298 (u64)start_pfn << PAGE_SHIFT, 8299 ((u64)end_pfn << PAGE_SHIFT) - 1); 8300 subsection_map_init(start_pfn, end_pfn - start_pfn); 8301 } 8302 8303 /* Initialise every node */ 8304 mminit_verify_pageflags_layout(); 8305 setup_nr_node_ids(); 8306 for_each_node(nid) { 8307 pg_data_t *pgdat; 8308 8309 if (!node_online(nid)) { 8310 pr_info("Initializing node %d as memoryless\n", nid); 8311 8312 /* Allocator not initialized yet */ 8313 pgdat = arch_alloc_nodedata(nid); 8314 if (!pgdat) { 8315 pr_err("Cannot allocate %zuB for node %d.\n", 8316 sizeof(*pgdat), nid); 8317 continue; 8318 } 8319 arch_refresh_nodedata(nid, pgdat); 8320 free_area_init_memoryless_node(nid); 8321 8322 /* 8323 * We do not want to confuse userspace by sysfs 8324 * files/directories for node without any memory 8325 * attached to it, so this node is not marked as 8326 * N_MEMORY and not marked online so that no sysfs 8327 * hierarchy will be created via register_one_node for 8328 * it. The pgdat will get fully initialized by 8329 * hotadd_init_pgdat() when memory is hotplugged into 8330 * this node. 8331 */ 8332 continue; 8333 } 8334 8335 pgdat = NODE_DATA(nid); 8336 free_area_init_node(nid); 8337 8338 /* Any memory on that node */ 8339 if (pgdat->node_present_pages) 8340 node_set_state(nid, N_MEMORY); 8341 check_for_memory(pgdat, nid); 8342 } 8343 8344 memmap_init(); 8345 } 8346 8347 static int __init cmdline_parse_core(char *p, unsigned long *core, 8348 unsigned long *percent) 8349 { 8350 unsigned long long coremem; 8351 char *endptr; 8352 8353 if (!p) 8354 return -EINVAL; 8355 8356 /* Value may be a percentage of total memory, otherwise bytes */ 8357 coremem = simple_strtoull(p, &endptr, 0); 8358 if (*endptr == '%') { 8359 /* Paranoid check for percent values greater than 100 */ 8360 WARN_ON(coremem > 100); 8361 8362 *percent = coremem; 8363 } else { 8364 coremem = memparse(p, &p); 8365 /* Paranoid check that UL is enough for the coremem value */ 8366 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 8367 8368 *core = coremem >> PAGE_SHIFT; 8369 *percent = 0UL; 8370 } 8371 return 0; 8372 } 8373 8374 /* 8375 * kernelcore=size sets the amount of memory for use for allocations that 8376 * cannot be reclaimed or migrated. 8377 */ 8378 static int __init cmdline_parse_kernelcore(char *p) 8379 { 8380 /* parse kernelcore=mirror */ 8381 if (parse_option_str(p, "mirror")) { 8382 mirrored_kernelcore = true; 8383 return 0; 8384 } 8385 8386 return cmdline_parse_core(p, &required_kernelcore, 8387 &required_kernelcore_percent); 8388 } 8389 8390 /* 8391 * movablecore=size sets the amount of memory for use for allocations that 8392 * can be reclaimed or migrated. 8393 */ 8394 static int __init cmdline_parse_movablecore(char *p) 8395 { 8396 return cmdline_parse_core(p, &required_movablecore, 8397 &required_movablecore_percent); 8398 } 8399 8400 early_param("kernelcore", cmdline_parse_kernelcore); 8401 early_param("movablecore", cmdline_parse_movablecore); 8402 8403 void adjust_managed_page_count(struct page *page, long count) 8404 { 8405 atomic_long_add(count, &page_zone(page)->managed_pages); 8406 totalram_pages_add(count); 8407 #ifdef CONFIG_HIGHMEM 8408 if (PageHighMem(page)) 8409 totalhigh_pages_add(count); 8410 #endif 8411 } 8412 EXPORT_SYMBOL(adjust_managed_page_count); 8413 8414 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 8415 { 8416 void *pos; 8417 unsigned long pages = 0; 8418 8419 start = (void *)PAGE_ALIGN((unsigned long)start); 8420 end = (void *)((unsigned long)end & PAGE_MASK); 8421 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 8422 struct page *page = virt_to_page(pos); 8423 void *direct_map_addr; 8424 8425 /* 8426 * 'direct_map_addr' might be different from 'pos' 8427 * because some architectures' virt_to_page() 8428 * work with aliases. Getting the direct map 8429 * address ensures that we get a _writeable_ 8430 * alias for the memset(). 8431 */ 8432 direct_map_addr = page_address(page); 8433 /* 8434 * Perform a kasan-unchecked memset() since this memory 8435 * has not been initialized. 8436 */ 8437 direct_map_addr = kasan_reset_tag(direct_map_addr); 8438 if ((unsigned int)poison <= 0xFF) 8439 memset(direct_map_addr, poison, PAGE_SIZE); 8440 8441 free_reserved_page(page); 8442 } 8443 8444 if (pages && s) 8445 pr_info("Freeing %s memory: %ldK\n", s, K(pages)); 8446 8447 return pages; 8448 } 8449 8450 void __init mem_init_print_info(void) 8451 { 8452 unsigned long physpages, codesize, datasize, rosize, bss_size; 8453 unsigned long init_code_size, init_data_size; 8454 8455 physpages = get_num_physpages(); 8456 codesize = _etext - _stext; 8457 datasize = _edata - _sdata; 8458 rosize = __end_rodata - __start_rodata; 8459 bss_size = __bss_stop - __bss_start; 8460 init_data_size = __init_end - __init_begin; 8461 init_code_size = _einittext - _sinittext; 8462 8463 /* 8464 * Detect special cases and adjust section sizes accordingly: 8465 * 1) .init.* may be embedded into .data sections 8466 * 2) .init.text.* may be out of [__init_begin, __init_end], 8467 * please refer to arch/tile/kernel/vmlinux.lds.S. 8468 * 3) .rodata.* may be embedded into .text or .data sections. 8469 */ 8470 #define adj_init_size(start, end, size, pos, adj) \ 8471 do { \ 8472 if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \ 8473 size -= adj; \ 8474 } while (0) 8475 8476 adj_init_size(__init_begin, __init_end, init_data_size, 8477 _sinittext, init_code_size); 8478 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 8479 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 8480 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 8481 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 8482 8483 #undef adj_init_size 8484 8485 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 8486 #ifdef CONFIG_HIGHMEM 8487 ", %luK highmem" 8488 #endif 8489 ")\n", 8490 K(nr_free_pages()), K(physpages), 8491 codesize >> 10, datasize >> 10, rosize >> 10, 8492 (init_data_size + init_code_size) >> 10, bss_size >> 10, 8493 K(physpages - totalram_pages() - totalcma_pages), 8494 K(totalcma_pages) 8495 #ifdef CONFIG_HIGHMEM 8496 , K(totalhigh_pages()) 8497 #endif 8498 ); 8499 } 8500 8501 /** 8502 * set_dma_reserve - set the specified number of pages reserved in the first zone 8503 * @new_dma_reserve: The number of pages to mark reserved 8504 * 8505 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 8506 * In the DMA zone, a significant percentage may be consumed by kernel image 8507 * and other unfreeable allocations which can skew the watermarks badly. This 8508 * function may optionally be used to account for unfreeable pages in the 8509 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 8510 * smaller per-cpu batchsize. 8511 */ 8512 void __init set_dma_reserve(unsigned long new_dma_reserve) 8513 { 8514 dma_reserve = new_dma_reserve; 8515 } 8516 8517 static int page_alloc_cpu_dead(unsigned int cpu) 8518 { 8519 struct zone *zone; 8520 8521 lru_add_drain_cpu(cpu); 8522 mlock_page_drain_remote(cpu); 8523 drain_pages(cpu); 8524 8525 /* 8526 * Spill the event counters of the dead processor 8527 * into the current processors event counters. 8528 * This artificially elevates the count of the current 8529 * processor. 8530 */ 8531 vm_events_fold_cpu(cpu); 8532 8533 /* 8534 * Zero the differential counters of the dead processor 8535 * so that the vm statistics are consistent. 8536 * 8537 * This is only okay since the processor is dead and cannot 8538 * race with what we are doing. 8539 */ 8540 cpu_vm_stats_fold(cpu); 8541 8542 for_each_populated_zone(zone) 8543 zone_pcp_update(zone, 0); 8544 8545 return 0; 8546 } 8547 8548 static int page_alloc_cpu_online(unsigned int cpu) 8549 { 8550 struct zone *zone; 8551 8552 for_each_populated_zone(zone) 8553 zone_pcp_update(zone, 1); 8554 return 0; 8555 } 8556 8557 #ifdef CONFIG_NUMA 8558 int hashdist = HASHDIST_DEFAULT; 8559 8560 static int __init set_hashdist(char *str) 8561 { 8562 if (!str) 8563 return 0; 8564 hashdist = simple_strtoul(str, &str, 0); 8565 return 1; 8566 } 8567 __setup("hashdist=", set_hashdist); 8568 #endif 8569 8570 void __init page_alloc_init(void) 8571 { 8572 int ret; 8573 8574 #ifdef CONFIG_NUMA 8575 if (num_node_state(N_MEMORY) == 1) 8576 hashdist = 0; 8577 #endif 8578 8579 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC, 8580 "mm/page_alloc:pcp", 8581 page_alloc_cpu_online, 8582 page_alloc_cpu_dead); 8583 WARN_ON(ret < 0); 8584 } 8585 8586 /* 8587 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 8588 * or min_free_kbytes changes. 8589 */ 8590 static void calculate_totalreserve_pages(void) 8591 { 8592 struct pglist_data *pgdat; 8593 unsigned long reserve_pages = 0; 8594 enum zone_type i, j; 8595 8596 for_each_online_pgdat(pgdat) { 8597 8598 pgdat->totalreserve_pages = 0; 8599 8600 for (i = 0; i < MAX_NR_ZONES; i++) { 8601 struct zone *zone = pgdat->node_zones + i; 8602 long max = 0; 8603 unsigned long managed_pages = zone_managed_pages(zone); 8604 8605 /* Find valid and maximum lowmem_reserve in the zone */ 8606 for (j = i; j < MAX_NR_ZONES; j++) { 8607 if (zone->lowmem_reserve[j] > max) 8608 max = zone->lowmem_reserve[j]; 8609 } 8610 8611 /* we treat the high watermark as reserved pages. */ 8612 max += high_wmark_pages(zone); 8613 8614 if (max > managed_pages) 8615 max = managed_pages; 8616 8617 pgdat->totalreserve_pages += max; 8618 8619 reserve_pages += max; 8620 } 8621 } 8622 totalreserve_pages = reserve_pages; 8623 } 8624 8625 /* 8626 * setup_per_zone_lowmem_reserve - called whenever 8627 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 8628 * has a correct pages reserved value, so an adequate number of 8629 * pages are left in the zone after a successful __alloc_pages(). 8630 */ 8631 static void setup_per_zone_lowmem_reserve(void) 8632 { 8633 struct pglist_data *pgdat; 8634 enum zone_type i, j; 8635 8636 for_each_online_pgdat(pgdat) { 8637 for (i = 0; i < MAX_NR_ZONES - 1; i++) { 8638 struct zone *zone = &pgdat->node_zones[i]; 8639 int ratio = sysctl_lowmem_reserve_ratio[i]; 8640 bool clear = !ratio || !zone_managed_pages(zone); 8641 unsigned long managed_pages = 0; 8642 8643 for (j = i + 1; j < MAX_NR_ZONES; j++) { 8644 struct zone *upper_zone = &pgdat->node_zones[j]; 8645 8646 managed_pages += zone_managed_pages(upper_zone); 8647 8648 if (clear) 8649 zone->lowmem_reserve[j] = 0; 8650 else 8651 zone->lowmem_reserve[j] = managed_pages / ratio; 8652 } 8653 } 8654 } 8655 8656 /* update totalreserve_pages */ 8657 calculate_totalreserve_pages(); 8658 } 8659 8660 static void __setup_per_zone_wmarks(void) 8661 { 8662 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 8663 unsigned long lowmem_pages = 0; 8664 struct zone *zone; 8665 unsigned long flags; 8666 8667 /* Calculate total number of !ZONE_HIGHMEM pages */ 8668 for_each_zone(zone) { 8669 if (!is_highmem(zone)) 8670 lowmem_pages += zone_managed_pages(zone); 8671 } 8672 8673 for_each_zone(zone) { 8674 u64 tmp; 8675 8676 spin_lock_irqsave(&zone->lock, flags); 8677 tmp = (u64)pages_min * zone_managed_pages(zone); 8678 do_div(tmp, lowmem_pages); 8679 if (is_highmem(zone)) { 8680 /* 8681 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 8682 * need highmem pages, so cap pages_min to a small 8683 * value here. 8684 * 8685 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 8686 * deltas control async page reclaim, and so should 8687 * not be capped for highmem. 8688 */ 8689 unsigned long min_pages; 8690 8691 min_pages = zone_managed_pages(zone) / 1024; 8692 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 8693 zone->_watermark[WMARK_MIN] = min_pages; 8694 } else { 8695 /* 8696 * If it's a lowmem zone, reserve a number of pages 8697 * proportionate to the zone's size. 8698 */ 8699 zone->_watermark[WMARK_MIN] = tmp; 8700 } 8701 8702 /* 8703 * Set the kswapd watermarks distance according to the 8704 * scale factor in proportion to available memory, but 8705 * ensure a minimum size on small systems. 8706 */ 8707 tmp = max_t(u64, tmp >> 2, 8708 mult_frac(zone_managed_pages(zone), 8709 watermark_scale_factor, 10000)); 8710 8711 zone->watermark_boost = 0; 8712 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 8713 zone->_watermark[WMARK_HIGH] = low_wmark_pages(zone) + tmp; 8714 zone->_watermark[WMARK_PROMO] = high_wmark_pages(zone) + tmp; 8715 8716 spin_unlock_irqrestore(&zone->lock, flags); 8717 } 8718 8719 /* update totalreserve_pages */ 8720 calculate_totalreserve_pages(); 8721 } 8722 8723 /** 8724 * setup_per_zone_wmarks - called when min_free_kbytes changes 8725 * or when memory is hot-{added|removed} 8726 * 8727 * Ensures that the watermark[min,low,high] values for each zone are set 8728 * correctly with respect to min_free_kbytes. 8729 */ 8730 void setup_per_zone_wmarks(void) 8731 { 8732 struct zone *zone; 8733 static DEFINE_SPINLOCK(lock); 8734 8735 spin_lock(&lock); 8736 __setup_per_zone_wmarks(); 8737 spin_unlock(&lock); 8738 8739 /* 8740 * The watermark size have changed so update the pcpu batch 8741 * and high limits or the limits may be inappropriate. 8742 */ 8743 for_each_zone(zone) 8744 zone_pcp_update(zone, 0); 8745 } 8746 8747 /* 8748 * Initialise min_free_kbytes. 8749 * 8750 * For small machines we want it small (128k min). For large machines 8751 * we want it large (256MB max). But it is not linear, because network 8752 * bandwidth does not increase linearly with machine size. We use 8753 * 8754 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 8755 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 8756 * 8757 * which yields 8758 * 8759 * 16MB: 512k 8760 * 32MB: 724k 8761 * 64MB: 1024k 8762 * 128MB: 1448k 8763 * 256MB: 2048k 8764 * 512MB: 2896k 8765 * 1024MB: 4096k 8766 * 2048MB: 5792k 8767 * 4096MB: 8192k 8768 * 8192MB: 11584k 8769 * 16384MB: 16384k 8770 */ 8771 void calculate_min_free_kbytes(void) 8772 { 8773 unsigned long lowmem_kbytes; 8774 int new_min_free_kbytes; 8775 8776 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 8777 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 8778 8779 if (new_min_free_kbytes > user_min_free_kbytes) 8780 min_free_kbytes = clamp(new_min_free_kbytes, 128, 262144); 8781 else 8782 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 8783 new_min_free_kbytes, user_min_free_kbytes); 8784 8785 } 8786 8787 int __meminit init_per_zone_wmark_min(void) 8788 { 8789 calculate_min_free_kbytes(); 8790 setup_per_zone_wmarks(); 8791 refresh_zone_stat_thresholds(); 8792 setup_per_zone_lowmem_reserve(); 8793 8794 #ifdef CONFIG_NUMA 8795 setup_min_unmapped_ratio(); 8796 setup_min_slab_ratio(); 8797 #endif 8798 8799 khugepaged_min_free_kbytes_update(); 8800 8801 return 0; 8802 } 8803 postcore_initcall(init_per_zone_wmark_min) 8804 8805 /* 8806 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 8807 * that we can call two helper functions whenever min_free_kbytes 8808 * changes. 8809 */ 8810 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 8811 void *buffer, size_t *length, loff_t *ppos) 8812 { 8813 int rc; 8814 8815 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8816 if (rc) 8817 return rc; 8818 8819 if (write) { 8820 user_min_free_kbytes = min_free_kbytes; 8821 setup_per_zone_wmarks(); 8822 } 8823 return 0; 8824 } 8825 8826 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 8827 void *buffer, size_t *length, loff_t *ppos) 8828 { 8829 int rc; 8830 8831 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8832 if (rc) 8833 return rc; 8834 8835 if (write) 8836 setup_per_zone_wmarks(); 8837 8838 return 0; 8839 } 8840 8841 #ifdef CONFIG_NUMA 8842 static void setup_min_unmapped_ratio(void) 8843 { 8844 pg_data_t *pgdat; 8845 struct zone *zone; 8846 8847 for_each_online_pgdat(pgdat) 8848 pgdat->min_unmapped_pages = 0; 8849 8850 for_each_zone(zone) 8851 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * 8852 sysctl_min_unmapped_ratio) / 100; 8853 } 8854 8855 8856 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 8857 void *buffer, size_t *length, loff_t *ppos) 8858 { 8859 int rc; 8860 8861 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8862 if (rc) 8863 return rc; 8864 8865 setup_min_unmapped_ratio(); 8866 8867 return 0; 8868 } 8869 8870 static void setup_min_slab_ratio(void) 8871 { 8872 pg_data_t *pgdat; 8873 struct zone *zone; 8874 8875 for_each_online_pgdat(pgdat) 8876 pgdat->min_slab_pages = 0; 8877 8878 for_each_zone(zone) 8879 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * 8880 sysctl_min_slab_ratio) / 100; 8881 } 8882 8883 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 8884 void *buffer, size_t *length, loff_t *ppos) 8885 { 8886 int rc; 8887 8888 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8889 if (rc) 8890 return rc; 8891 8892 setup_min_slab_ratio(); 8893 8894 return 0; 8895 } 8896 #endif 8897 8898 /* 8899 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 8900 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 8901 * whenever sysctl_lowmem_reserve_ratio changes. 8902 * 8903 * The reserve ratio obviously has absolutely no relation with the 8904 * minimum watermarks. The lowmem reserve ratio can only make sense 8905 * if in function of the boot time zone sizes. 8906 */ 8907 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 8908 void *buffer, size_t *length, loff_t *ppos) 8909 { 8910 int i; 8911 8912 proc_dointvec_minmax(table, write, buffer, length, ppos); 8913 8914 for (i = 0; i < MAX_NR_ZONES; i++) { 8915 if (sysctl_lowmem_reserve_ratio[i] < 1) 8916 sysctl_lowmem_reserve_ratio[i] = 0; 8917 } 8918 8919 setup_per_zone_lowmem_reserve(); 8920 return 0; 8921 } 8922 8923 /* 8924 * percpu_pagelist_high_fraction - changes the pcp->high for each zone on each 8925 * cpu. It is the fraction of total pages in each zone that a hot per cpu 8926 * pagelist can have before it gets flushed back to buddy allocator. 8927 */ 8928 int percpu_pagelist_high_fraction_sysctl_handler(struct ctl_table *table, 8929 int write, void *buffer, size_t *length, loff_t *ppos) 8930 { 8931 struct zone *zone; 8932 int old_percpu_pagelist_high_fraction; 8933 int ret; 8934 8935 mutex_lock(&pcp_batch_high_lock); 8936 old_percpu_pagelist_high_fraction = percpu_pagelist_high_fraction; 8937 8938 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 8939 if (!write || ret < 0) 8940 goto out; 8941 8942 /* Sanity checking to avoid pcp imbalance */ 8943 if (percpu_pagelist_high_fraction && 8944 percpu_pagelist_high_fraction < MIN_PERCPU_PAGELIST_HIGH_FRACTION) { 8945 percpu_pagelist_high_fraction = old_percpu_pagelist_high_fraction; 8946 ret = -EINVAL; 8947 goto out; 8948 } 8949 8950 /* No change? */ 8951 if (percpu_pagelist_high_fraction == old_percpu_pagelist_high_fraction) 8952 goto out; 8953 8954 for_each_populated_zone(zone) 8955 zone_set_pageset_high_and_batch(zone, 0); 8956 out: 8957 mutex_unlock(&pcp_batch_high_lock); 8958 return ret; 8959 } 8960 8961 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 8962 /* 8963 * Returns the number of pages that arch has reserved but 8964 * is not known to alloc_large_system_hash(). 8965 */ 8966 static unsigned long __init arch_reserved_kernel_pages(void) 8967 { 8968 return 0; 8969 } 8970 #endif 8971 8972 /* 8973 * Adaptive scale is meant to reduce sizes of hash tables on large memory 8974 * machines. As memory size is increased the scale is also increased but at 8975 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 8976 * quadruples the scale is increased by one, which means the size of hash table 8977 * only doubles, instead of quadrupling as well. 8978 * Because 32-bit systems cannot have large physical memory, where this scaling 8979 * makes sense, it is disabled on such platforms. 8980 */ 8981 #if __BITS_PER_LONG > 32 8982 #define ADAPT_SCALE_BASE (64ul << 30) 8983 #define ADAPT_SCALE_SHIFT 2 8984 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 8985 #endif 8986 8987 /* 8988 * allocate a large system hash table from bootmem 8989 * - it is assumed that the hash table must contain an exact power-of-2 8990 * quantity of entries 8991 * - limit is the number of hash buckets, not the total allocation size 8992 */ 8993 void *__init alloc_large_system_hash(const char *tablename, 8994 unsigned long bucketsize, 8995 unsigned long numentries, 8996 int scale, 8997 int flags, 8998 unsigned int *_hash_shift, 8999 unsigned int *_hash_mask, 9000 unsigned long low_limit, 9001 unsigned long high_limit) 9002 { 9003 unsigned long long max = high_limit; 9004 unsigned long log2qty, size; 9005 void *table; 9006 gfp_t gfp_flags; 9007 bool virt; 9008 bool huge; 9009 9010 /* allow the kernel cmdline to have a say */ 9011 if (!numentries) { 9012 /* round applicable memory size up to nearest megabyte */ 9013 numentries = nr_kernel_pages; 9014 numentries -= arch_reserved_kernel_pages(); 9015 9016 /* It isn't necessary when PAGE_SIZE >= 1MB */ 9017 if (PAGE_SHIFT < 20) 9018 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 9019 9020 #if __BITS_PER_LONG > 32 9021 if (!high_limit) { 9022 unsigned long adapt; 9023 9024 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 9025 adapt <<= ADAPT_SCALE_SHIFT) 9026 scale++; 9027 } 9028 #endif 9029 9030 /* limit to 1 bucket per 2^scale bytes of low memory */ 9031 if (scale > PAGE_SHIFT) 9032 numentries >>= (scale - PAGE_SHIFT); 9033 else 9034 numentries <<= (PAGE_SHIFT - scale); 9035 9036 /* Make sure we've got at least a 0-order allocation.. */ 9037 if (unlikely(flags & HASH_SMALL)) { 9038 /* Makes no sense without HASH_EARLY */ 9039 WARN_ON(!(flags & HASH_EARLY)); 9040 if (!(numentries >> *_hash_shift)) { 9041 numentries = 1UL << *_hash_shift; 9042 BUG_ON(!numentries); 9043 } 9044 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 9045 numentries = PAGE_SIZE / bucketsize; 9046 } 9047 numentries = roundup_pow_of_two(numentries); 9048 9049 /* limit allocation size to 1/16 total memory by default */ 9050 if (max == 0) { 9051 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 9052 do_div(max, bucketsize); 9053 } 9054 max = min(max, 0x80000000ULL); 9055 9056 if (numentries < low_limit) 9057 numentries = low_limit; 9058 if (numentries > max) 9059 numentries = max; 9060 9061 log2qty = ilog2(numentries); 9062 9063 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 9064 do { 9065 virt = false; 9066 size = bucketsize << log2qty; 9067 if (flags & HASH_EARLY) { 9068 if (flags & HASH_ZERO) 9069 table = memblock_alloc(size, SMP_CACHE_BYTES); 9070 else 9071 table = memblock_alloc_raw(size, 9072 SMP_CACHE_BYTES); 9073 } else if (get_order(size) >= MAX_ORDER || hashdist) { 9074 table = vmalloc_huge(size, gfp_flags); 9075 virt = true; 9076 if (table) 9077 huge = is_vm_area_hugepages(table); 9078 } else { 9079 /* 9080 * If bucketsize is not a power-of-two, we may free 9081 * some pages at the end of hash table which 9082 * alloc_pages_exact() automatically does 9083 */ 9084 table = alloc_pages_exact(size, gfp_flags); 9085 kmemleak_alloc(table, size, 1, gfp_flags); 9086 } 9087 } while (!table && size > PAGE_SIZE && --log2qty); 9088 9089 if (!table) 9090 panic("Failed to allocate %s hash table\n", tablename); 9091 9092 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 9093 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, 9094 virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear"); 9095 9096 if (_hash_shift) 9097 *_hash_shift = log2qty; 9098 if (_hash_mask) 9099 *_hash_mask = (1 << log2qty) - 1; 9100 9101 return table; 9102 } 9103 9104 #ifdef CONFIG_CONTIG_ALLOC 9105 #if defined(CONFIG_DYNAMIC_DEBUG) || \ 9106 (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE)) 9107 /* Usage: See admin-guide/dynamic-debug-howto.rst */ 9108 static void alloc_contig_dump_pages(struct list_head *page_list) 9109 { 9110 DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, "migrate failure"); 9111 9112 if (DYNAMIC_DEBUG_BRANCH(descriptor)) { 9113 struct page *page; 9114 9115 dump_stack(); 9116 list_for_each_entry(page, page_list, lru) 9117 dump_page(page, "migration failure"); 9118 } 9119 } 9120 #else 9121 static inline void alloc_contig_dump_pages(struct list_head *page_list) 9122 { 9123 } 9124 #endif 9125 9126 /* [start, end) must belong to a single zone. */ 9127 int __alloc_contig_migrate_range(struct compact_control *cc, 9128 unsigned long start, unsigned long end) 9129 { 9130 /* This function is based on compact_zone() from compaction.c. */ 9131 unsigned int nr_reclaimed; 9132 unsigned long pfn = start; 9133 unsigned int tries = 0; 9134 int ret = 0; 9135 struct migration_target_control mtc = { 9136 .nid = zone_to_nid(cc->zone), 9137 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, 9138 }; 9139 9140 lru_cache_disable(); 9141 9142 while (pfn < end || !list_empty(&cc->migratepages)) { 9143 if (fatal_signal_pending(current)) { 9144 ret = -EINTR; 9145 break; 9146 } 9147 9148 if (list_empty(&cc->migratepages)) { 9149 cc->nr_migratepages = 0; 9150 ret = isolate_migratepages_range(cc, pfn, end); 9151 if (ret && ret != -EAGAIN) 9152 break; 9153 pfn = cc->migrate_pfn; 9154 tries = 0; 9155 } else if (++tries == 5) { 9156 ret = -EBUSY; 9157 break; 9158 } 9159 9160 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 9161 &cc->migratepages); 9162 cc->nr_migratepages -= nr_reclaimed; 9163 9164 ret = migrate_pages(&cc->migratepages, alloc_migration_target, 9165 NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE, NULL); 9166 9167 /* 9168 * On -ENOMEM, migrate_pages() bails out right away. It is pointless 9169 * to retry again over this error, so do the same here. 9170 */ 9171 if (ret == -ENOMEM) 9172 break; 9173 } 9174 9175 lru_cache_enable(); 9176 if (ret < 0) { 9177 if (!(cc->gfp_mask & __GFP_NOWARN) && ret == -EBUSY) 9178 alloc_contig_dump_pages(&cc->migratepages); 9179 putback_movable_pages(&cc->migratepages); 9180 return ret; 9181 } 9182 return 0; 9183 } 9184 9185 /** 9186 * alloc_contig_range() -- tries to allocate given range of pages 9187 * @start: start PFN to allocate 9188 * @end: one-past-the-last PFN to allocate 9189 * @migratetype: migratetype of the underlying pageblocks (either 9190 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 9191 * in range must have the same migratetype and it must 9192 * be either of the two. 9193 * @gfp_mask: GFP mask to use during compaction 9194 * 9195 * The PFN range does not have to be pageblock aligned. The PFN range must 9196 * belong to a single zone. 9197 * 9198 * The first thing this routine does is attempt to MIGRATE_ISOLATE all 9199 * pageblocks in the range. Once isolated, the pageblocks should not 9200 * be modified by others. 9201 * 9202 * Return: zero on success or negative error code. On success all 9203 * pages which PFN is in [start, end) are allocated for the caller and 9204 * need to be freed with free_contig_range(). 9205 */ 9206 int alloc_contig_range(unsigned long start, unsigned long end, 9207 unsigned migratetype, gfp_t gfp_mask) 9208 { 9209 unsigned long outer_start, outer_end; 9210 int order; 9211 int ret = 0; 9212 9213 struct compact_control cc = { 9214 .nr_migratepages = 0, 9215 .order = -1, 9216 .zone = page_zone(pfn_to_page(start)), 9217 .mode = MIGRATE_SYNC, 9218 .ignore_skip_hint = true, 9219 .no_set_skip_hint = true, 9220 .gfp_mask = current_gfp_context(gfp_mask), 9221 .alloc_contig = true, 9222 }; 9223 INIT_LIST_HEAD(&cc.migratepages); 9224 9225 /* 9226 * What we do here is we mark all pageblocks in range as 9227 * MIGRATE_ISOLATE. Because pageblock and max order pages may 9228 * have different sizes, and due to the way page allocator 9229 * work, start_isolate_page_range() has special handlings for this. 9230 * 9231 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 9232 * migrate the pages from an unaligned range (ie. pages that 9233 * we are interested in). This will put all the pages in 9234 * range back to page allocator as MIGRATE_ISOLATE. 9235 * 9236 * When this is done, we take the pages in range from page 9237 * allocator removing them from the buddy system. This way 9238 * page allocator will never consider using them. 9239 * 9240 * This lets us mark the pageblocks back as 9241 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 9242 * aligned range but not in the unaligned, original range are 9243 * put back to page allocator so that buddy can use them. 9244 */ 9245 9246 ret = start_isolate_page_range(start, end, migratetype, 0, gfp_mask); 9247 if (ret) 9248 goto done; 9249 9250 drain_all_pages(cc.zone); 9251 9252 /* 9253 * In case of -EBUSY, we'd like to know which page causes problem. 9254 * So, just fall through. test_pages_isolated() has a tracepoint 9255 * which will report the busy page. 9256 * 9257 * It is possible that busy pages could become available before 9258 * the call to test_pages_isolated, and the range will actually be 9259 * allocated. So, if we fall through be sure to clear ret so that 9260 * -EBUSY is not accidentally used or returned to caller. 9261 */ 9262 ret = __alloc_contig_migrate_range(&cc, start, end); 9263 if (ret && ret != -EBUSY) 9264 goto done; 9265 ret = 0; 9266 9267 /* 9268 * Pages from [start, end) are within a pageblock_nr_pages 9269 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 9270 * more, all pages in [start, end) are free in page allocator. 9271 * What we are going to do is to allocate all pages from 9272 * [start, end) (that is remove them from page allocator). 9273 * 9274 * The only problem is that pages at the beginning and at the 9275 * end of interesting range may be not aligned with pages that 9276 * page allocator holds, ie. they can be part of higher order 9277 * pages. Because of this, we reserve the bigger range and 9278 * once this is done free the pages we are not interested in. 9279 * 9280 * We don't have to hold zone->lock here because the pages are 9281 * isolated thus they won't get removed from buddy. 9282 */ 9283 9284 order = 0; 9285 outer_start = start; 9286 while (!PageBuddy(pfn_to_page(outer_start))) { 9287 if (++order >= MAX_ORDER) { 9288 outer_start = start; 9289 break; 9290 } 9291 outer_start &= ~0UL << order; 9292 } 9293 9294 if (outer_start != start) { 9295 order = buddy_order(pfn_to_page(outer_start)); 9296 9297 /* 9298 * outer_start page could be small order buddy page and 9299 * it doesn't include start page. Adjust outer_start 9300 * in this case to report failed page properly 9301 * on tracepoint in test_pages_isolated() 9302 */ 9303 if (outer_start + (1UL << order) <= start) 9304 outer_start = start; 9305 } 9306 9307 /* Make sure the range is really isolated. */ 9308 if (test_pages_isolated(outer_start, end, 0)) { 9309 ret = -EBUSY; 9310 goto done; 9311 } 9312 9313 /* Grab isolated pages from freelists. */ 9314 outer_end = isolate_freepages_range(&cc, outer_start, end); 9315 if (!outer_end) { 9316 ret = -EBUSY; 9317 goto done; 9318 } 9319 9320 /* Free head and tail (if any) */ 9321 if (start != outer_start) 9322 free_contig_range(outer_start, start - outer_start); 9323 if (end != outer_end) 9324 free_contig_range(end, outer_end - end); 9325 9326 done: 9327 undo_isolate_page_range(start, end, migratetype); 9328 return ret; 9329 } 9330 EXPORT_SYMBOL(alloc_contig_range); 9331 9332 static int __alloc_contig_pages(unsigned long start_pfn, 9333 unsigned long nr_pages, gfp_t gfp_mask) 9334 { 9335 unsigned long end_pfn = start_pfn + nr_pages; 9336 9337 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE, 9338 gfp_mask); 9339 } 9340 9341 static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, 9342 unsigned long nr_pages) 9343 { 9344 unsigned long i, end_pfn = start_pfn + nr_pages; 9345 struct page *page; 9346 9347 for (i = start_pfn; i < end_pfn; i++) { 9348 page = pfn_to_online_page(i); 9349 if (!page) 9350 return false; 9351 9352 if (page_zone(page) != z) 9353 return false; 9354 9355 if (PageReserved(page)) 9356 return false; 9357 } 9358 return true; 9359 } 9360 9361 static bool zone_spans_last_pfn(const struct zone *zone, 9362 unsigned long start_pfn, unsigned long nr_pages) 9363 { 9364 unsigned long last_pfn = start_pfn + nr_pages - 1; 9365 9366 return zone_spans_pfn(zone, last_pfn); 9367 } 9368 9369 /** 9370 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages 9371 * @nr_pages: Number of contiguous pages to allocate 9372 * @gfp_mask: GFP mask to limit search and used during compaction 9373 * @nid: Target node 9374 * @nodemask: Mask for other possible nodes 9375 * 9376 * This routine is a wrapper around alloc_contig_range(). It scans over zones 9377 * on an applicable zonelist to find a contiguous pfn range which can then be 9378 * tried for allocation with alloc_contig_range(). This routine is intended 9379 * for allocation requests which can not be fulfilled with the buddy allocator. 9380 * 9381 * The allocated memory is always aligned to a page boundary. If nr_pages is a 9382 * power of two, then allocated range is also guaranteed to be aligned to same 9383 * nr_pages (e.g. 1GB request would be aligned to 1GB). 9384 * 9385 * Allocated pages can be freed with free_contig_range() or by manually calling 9386 * __free_page() on each allocated page. 9387 * 9388 * Return: pointer to contiguous pages on success, or NULL if not successful. 9389 */ 9390 struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, 9391 int nid, nodemask_t *nodemask) 9392 { 9393 unsigned long ret, pfn, flags; 9394 struct zonelist *zonelist; 9395 struct zone *zone; 9396 struct zoneref *z; 9397 9398 zonelist = node_zonelist(nid, gfp_mask); 9399 for_each_zone_zonelist_nodemask(zone, z, zonelist, 9400 gfp_zone(gfp_mask), nodemask) { 9401 spin_lock_irqsave(&zone->lock, flags); 9402 9403 pfn = ALIGN(zone->zone_start_pfn, nr_pages); 9404 while (zone_spans_last_pfn(zone, pfn, nr_pages)) { 9405 if (pfn_range_valid_contig(zone, pfn, nr_pages)) { 9406 /* 9407 * We release the zone lock here because 9408 * alloc_contig_range() will also lock the zone 9409 * at some point. If there's an allocation 9410 * spinning on this lock, it may win the race 9411 * and cause alloc_contig_range() to fail... 9412 */ 9413 spin_unlock_irqrestore(&zone->lock, flags); 9414 ret = __alloc_contig_pages(pfn, nr_pages, 9415 gfp_mask); 9416 if (!ret) 9417 return pfn_to_page(pfn); 9418 spin_lock_irqsave(&zone->lock, flags); 9419 } 9420 pfn += nr_pages; 9421 } 9422 spin_unlock_irqrestore(&zone->lock, flags); 9423 } 9424 return NULL; 9425 } 9426 #endif /* CONFIG_CONTIG_ALLOC */ 9427 9428 void free_contig_range(unsigned long pfn, unsigned long nr_pages) 9429 { 9430 unsigned long count = 0; 9431 9432 for (; nr_pages--; pfn++) { 9433 struct page *page = pfn_to_page(pfn); 9434 9435 count += page_count(page) != 1; 9436 __free_page(page); 9437 } 9438 WARN(count != 0, "%lu pages are still in use!\n", count); 9439 } 9440 EXPORT_SYMBOL(free_contig_range); 9441 9442 /* 9443 * The zone indicated has a new number of managed_pages; batch sizes and percpu 9444 * page high values need to be recalculated. 9445 */ 9446 void zone_pcp_update(struct zone *zone, int cpu_online) 9447 { 9448 mutex_lock(&pcp_batch_high_lock); 9449 zone_set_pageset_high_and_batch(zone, cpu_online); 9450 mutex_unlock(&pcp_batch_high_lock); 9451 } 9452 9453 /* 9454 * Effectively disable pcplists for the zone by setting the high limit to 0 9455 * and draining all cpus. A concurrent page freeing on another CPU that's about 9456 * to put the page on pcplist will either finish before the drain and the page 9457 * will be drained, or observe the new high limit and skip the pcplist. 9458 * 9459 * Must be paired with a call to zone_pcp_enable(). 9460 */ 9461 void zone_pcp_disable(struct zone *zone) 9462 { 9463 mutex_lock(&pcp_batch_high_lock); 9464 __zone_set_pageset_high_and_batch(zone, 0, 1); 9465 __drain_all_pages(zone, true); 9466 } 9467 9468 void zone_pcp_enable(struct zone *zone) 9469 { 9470 __zone_set_pageset_high_and_batch(zone, zone->pageset_high, zone->pageset_batch); 9471 mutex_unlock(&pcp_batch_high_lock); 9472 } 9473 9474 void zone_pcp_reset(struct zone *zone) 9475 { 9476 int cpu; 9477 struct per_cpu_zonestat *pzstats; 9478 9479 if (zone->per_cpu_pageset != &boot_pageset) { 9480 for_each_online_cpu(cpu) { 9481 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 9482 drain_zonestat(zone, pzstats); 9483 } 9484 free_percpu(zone->per_cpu_pageset); 9485 free_percpu(zone->per_cpu_zonestats); 9486 zone->per_cpu_pageset = &boot_pageset; 9487 zone->per_cpu_zonestats = &boot_zonestats; 9488 } 9489 } 9490 9491 #ifdef CONFIG_MEMORY_HOTREMOVE 9492 /* 9493 * All pages in the range must be in a single zone, must not contain holes, 9494 * must span full sections, and must be isolated before calling this function. 9495 */ 9496 void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 9497 { 9498 unsigned long pfn = start_pfn; 9499 struct page *page; 9500 struct zone *zone; 9501 unsigned int order; 9502 unsigned long flags; 9503 9504 offline_mem_sections(pfn, end_pfn); 9505 zone = page_zone(pfn_to_page(pfn)); 9506 spin_lock_irqsave(&zone->lock, flags); 9507 while (pfn < end_pfn) { 9508 page = pfn_to_page(pfn); 9509 /* 9510 * The HWPoisoned page may be not in buddy system, and 9511 * page_count() is not 0. 9512 */ 9513 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 9514 pfn++; 9515 continue; 9516 } 9517 /* 9518 * At this point all remaining PageOffline() pages have a 9519 * reference count of 0 and can simply be skipped. 9520 */ 9521 if (PageOffline(page)) { 9522 BUG_ON(page_count(page)); 9523 BUG_ON(PageBuddy(page)); 9524 pfn++; 9525 continue; 9526 } 9527 9528 BUG_ON(page_count(page)); 9529 BUG_ON(!PageBuddy(page)); 9530 order = buddy_order(page); 9531 del_page_from_free_list(page, zone, order); 9532 pfn += (1 << order); 9533 } 9534 spin_unlock_irqrestore(&zone->lock, flags); 9535 } 9536 #endif 9537 9538 /* 9539 * This function returns a stable result only if called under zone lock. 9540 */ 9541 bool is_free_buddy_page(struct page *page) 9542 { 9543 unsigned long pfn = page_to_pfn(page); 9544 unsigned int order; 9545 9546 for (order = 0; order < MAX_ORDER; order++) { 9547 struct page *page_head = page - (pfn & ((1 << order) - 1)); 9548 9549 if (PageBuddy(page_head) && 9550 buddy_order_unsafe(page_head) >= order) 9551 break; 9552 } 9553 9554 return order < MAX_ORDER; 9555 } 9556 EXPORT_SYMBOL(is_free_buddy_page); 9557 9558 #ifdef CONFIG_MEMORY_FAILURE 9559 /* 9560 * Break down a higher-order page in sub-pages, and keep our target out of 9561 * buddy allocator. 9562 */ 9563 static void break_down_buddy_pages(struct zone *zone, struct page *page, 9564 struct page *target, int low, int high, 9565 int migratetype) 9566 { 9567 unsigned long size = 1 << high; 9568 struct page *current_buddy, *next_page; 9569 9570 while (high > low) { 9571 high--; 9572 size >>= 1; 9573 9574 if (target >= &page[size]) { 9575 next_page = page + size; 9576 current_buddy = page; 9577 } else { 9578 next_page = page; 9579 current_buddy = page + size; 9580 } 9581 9582 if (set_page_guard(zone, current_buddy, high, migratetype)) 9583 continue; 9584 9585 if (current_buddy != target) { 9586 add_to_free_list(current_buddy, zone, high, migratetype); 9587 set_buddy_order(current_buddy, high); 9588 page = next_page; 9589 } 9590 } 9591 } 9592 9593 /* 9594 * Take a page that will be marked as poisoned off the buddy allocator. 9595 */ 9596 bool take_page_off_buddy(struct page *page) 9597 { 9598 struct zone *zone = page_zone(page); 9599 unsigned long pfn = page_to_pfn(page); 9600 unsigned long flags; 9601 unsigned int order; 9602 bool ret = false; 9603 9604 spin_lock_irqsave(&zone->lock, flags); 9605 for (order = 0; order < MAX_ORDER; order++) { 9606 struct page *page_head = page - (pfn & ((1 << order) - 1)); 9607 int page_order = buddy_order(page_head); 9608 9609 if (PageBuddy(page_head) && page_order >= order) { 9610 unsigned long pfn_head = page_to_pfn(page_head); 9611 int migratetype = get_pfnblock_migratetype(page_head, 9612 pfn_head); 9613 9614 del_page_from_free_list(page_head, zone, page_order); 9615 break_down_buddy_pages(zone, page_head, page, 0, 9616 page_order, migratetype); 9617 SetPageHWPoisonTakenOff(page); 9618 if (!is_migrate_isolate(migratetype)) 9619 __mod_zone_freepage_state(zone, -1, migratetype); 9620 ret = true; 9621 break; 9622 } 9623 if (page_count(page_head) > 0) 9624 break; 9625 } 9626 spin_unlock_irqrestore(&zone->lock, flags); 9627 return ret; 9628 } 9629 9630 /* 9631 * Cancel takeoff done by take_page_off_buddy(). 9632 */ 9633 bool put_page_back_buddy(struct page *page) 9634 { 9635 struct zone *zone = page_zone(page); 9636 unsigned long pfn = page_to_pfn(page); 9637 unsigned long flags; 9638 int migratetype = get_pfnblock_migratetype(page, pfn); 9639 bool ret = false; 9640 9641 spin_lock_irqsave(&zone->lock, flags); 9642 if (put_page_testzero(page)) { 9643 ClearPageHWPoisonTakenOff(page); 9644 __free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE); 9645 if (TestClearPageHWPoison(page)) { 9646 ret = true; 9647 } 9648 } 9649 spin_unlock_irqrestore(&zone->lock, flags); 9650 9651 return ret; 9652 } 9653 #endif 9654 9655 #ifdef CONFIG_ZONE_DMA 9656 bool has_managed_dma(void) 9657 { 9658 struct pglist_data *pgdat; 9659 9660 for_each_online_pgdat(pgdat) { 9661 struct zone *zone = &pgdat->node_zones[ZONE_DMA]; 9662 9663 if (managed_zone(zone)) 9664 return true; 9665 } 9666 return false; 9667 } 9668 #endif /* CONFIG_ZONE_DMA */ 9669