1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/page_alloc.c 4 * 5 * Manages the free list, the system allocates free pages here. 6 * Note that kmalloc() lives in slab.c 7 * 8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 9 * Swap reorganised 29.12.95, Stephen Tweedie 10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 16 */ 17 18 #include <linux/stddef.h> 19 #include <linux/mm.h> 20 #include <linux/highmem.h> 21 #include <linux/swap.h> 22 #include <linux/interrupt.h> 23 #include <linux/pagemap.h> 24 #include <linux/jiffies.h> 25 #include <linux/memblock.h> 26 #include <linux/compiler.h> 27 #include <linux/kernel.h> 28 #include <linux/kasan.h> 29 #include <linux/module.h> 30 #include <linux/suspend.h> 31 #include <linux/pagevec.h> 32 #include <linux/blkdev.h> 33 #include <linux/slab.h> 34 #include <linux/ratelimit.h> 35 #include <linux/oom.h> 36 #include <linux/topology.h> 37 #include <linux/sysctl.h> 38 #include <linux/cpu.h> 39 #include <linux/cpuset.h> 40 #include <linux/memory_hotplug.h> 41 #include <linux/nodemask.h> 42 #include <linux/vmalloc.h> 43 #include <linux/vmstat.h> 44 #include <linux/mempolicy.h> 45 #include <linux/memremap.h> 46 #include <linux/stop_machine.h> 47 #include <linux/random.h> 48 #include <linux/sort.h> 49 #include <linux/pfn.h> 50 #include <linux/backing-dev.h> 51 #include <linux/fault-inject.h> 52 #include <linux/page-isolation.h> 53 #include <linux/debugobjects.h> 54 #include <linux/kmemleak.h> 55 #include <linux/compaction.h> 56 #include <trace/events/kmem.h> 57 #include <trace/events/oom.h> 58 #include <linux/prefetch.h> 59 #include <linux/mm_inline.h> 60 #include <linux/migrate.h> 61 #include <linux/hugetlb.h> 62 #include <linux/sched/rt.h> 63 #include <linux/sched/mm.h> 64 #include <linux/page_owner.h> 65 #include <linux/kthread.h> 66 #include <linux/memcontrol.h> 67 #include <linux/ftrace.h> 68 #include <linux/lockdep.h> 69 #include <linux/nmi.h> 70 #include <linux/psi.h> 71 #include <linux/padata.h> 72 73 #include <asm/sections.h> 74 #include <asm/tlbflush.h> 75 #include <asm/div64.h> 76 #include "internal.h" 77 #include "shuffle.h" 78 #include "page_reporting.h" 79 80 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 81 static DEFINE_MUTEX(pcp_batch_high_lock); 82 #define MIN_PERCPU_PAGELIST_FRACTION (8) 83 84 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 85 DEFINE_PER_CPU(int, numa_node); 86 EXPORT_PER_CPU_SYMBOL(numa_node); 87 #endif 88 89 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); 90 91 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 92 /* 93 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 94 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 95 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 96 * defined in <linux/topology.h>. 97 */ 98 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 99 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 100 #endif 101 102 /* work_structs for global per-cpu drains */ 103 struct pcpu_drain { 104 struct zone *zone; 105 struct work_struct work; 106 }; 107 static DEFINE_MUTEX(pcpu_drain_mutex); 108 static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain); 109 110 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 111 volatile unsigned long latent_entropy __latent_entropy; 112 EXPORT_SYMBOL(latent_entropy); 113 #endif 114 115 /* 116 * Array of node states. 117 */ 118 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 119 [N_POSSIBLE] = NODE_MASK_ALL, 120 [N_ONLINE] = { { [0] = 1UL } }, 121 #ifndef CONFIG_NUMA 122 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 123 #ifdef CONFIG_HIGHMEM 124 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 125 #endif 126 [N_MEMORY] = { { [0] = 1UL } }, 127 [N_CPU] = { { [0] = 1UL } }, 128 #endif /* NUMA */ 129 }; 130 EXPORT_SYMBOL(node_states); 131 132 atomic_long_t _totalram_pages __read_mostly; 133 EXPORT_SYMBOL(_totalram_pages); 134 unsigned long totalreserve_pages __read_mostly; 135 unsigned long totalcma_pages __read_mostly; 136 137 int percpu_pagelist_fraction; 138 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 139 #ifdef CONFIG_INIT_ON_ALLOC_DEFAULT_ON 140 DEFINE_STATIC_KEY_TRUE(init_on_alloc); 141 #else 142 DEFINE_STATIC_KEY_FALSE(init_on_alloc); 143 #endif 144 EXPORT_SYMBOL(init_on_alloc); 145 146 #ifdef CONFIG_INIT_ON_FREE_DEFAULT_ON 147 DEFINE_STATIC_KEY_TRUE(init_on_free); 148 #else 149 DEFINE_STATIC_KEY_FALSE(init_on_free); 150 #endif 151 EXPORT_SYMBOL(init_on_free); 152 153 static int __init early_init_on_alloc(char *buf) 154 { 155 int ret; 156 bool bool_result; 157 158 if (!buf) 159 return -EINVAL; 160 ret = kstrtobool(buf, &bool_result); 161 if (bool_result && page_poisoning_enabled()) 162 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_alloc\n"); 163 if (bool_result) 164 static_branch_enable(&init_on_alloc); 165 else 166 static_branch_disable(&init_on_alloc); 167 return ret; 168 } 169 early_param("init_on_alloc", early_init_on_alloc); 170 171 static int __init early_init_on_free(char *buf) 172 { 173 int ret; 174 bool bool_result; 175 176 if (!buf) 177 return -EINVAL; 178 ret = kstrtobool(buf, &bool_result); 179 if (bool_result && page_poisoning_enabled()) 180 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_free\n"); 181 if (bool_result) 182 static_branch_enable(&init_on_free); 183 else 184 static_branch_disable(&init_on_free); 185 return ret; 186 } 187 early_param("init_on_free", early_init_on_free); 188 189 /* 190 * A cached value of the page's pageblock's migratetype, used when the page is 191 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 192 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 193 * Also the migratetype set in the page does not necessarily match the pcplist 194 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 195 * other index - this ensures that it will be put on the correct CMA freelist. 196 */ 197 static inline int get_pcppage_migratetype(struct page *page) 198 { 199 return page->index; 200 } 201 202 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 203 { 204 page->index = migratetype; 205 } 206 207 #ifdef CONFIG_PM_SLEEP 208 /* 209 * The following functions are used by the suspend/hibernate code to temporarily 210 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 211 * while devices are suspended. To avoid races with the suspend/hibernate code, 212 * they should always be called with system_transition_mutex held 213 * (gfp_allowed_mask also should only be modified with system_transition_mutex 214 * held, unless the suspend/hibernate code is guaranteed not to run in parallel 215 * with that modification). 216 */ 217 218 static gfp_t saved_gfp_mask; 219 220 void pm_restore_gfp_mask(void) 221 { 222 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 223 if (saved_gfp_mask) { 224 gfp_allowed_mask = saved_gfp_mask; 225 saved_gfp_mask = 0; 226 } 227 } 228 229 void pm_restrict_gfp_mask(void) 230 { 231 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 232 WARN_ON(saved_gfp_mask); 233 saved_gfp_mask = gfp_allowed_mask; 234 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 235 } 236 237 bool pm_suspended_storage(void) 238 { 239 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 240 return false; 241 return true; 242 } 243 #endif /* CONFIG_PM_SLEEP */ 244 245 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 246 unsigned int pageblock_order __read_mostly; 247 #endif 248 249 static void __free_pages_ok(struct page *page, unsigned int order); 250 251 /* 252 * results with 256, 32 in the lowmem_reserve sysctl: 253 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 254 * 1G machine -> (16M dma, 784M normal, 224M high) 255 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 256 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 257 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 258 * 259 * TBD: should special case ZONE_DMA32 machines here - in those we normally 260 * don't need any ZONE_NORMAL reservation 261 */ 262 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { 263 #ifdef CONFIG_ZONE_DMA 264 [ZONE_DMA] = 256, 265 #endif 266 #ifdef CONFIG_ZONE_DMA32 267 [ZONE_DMA32] = 256, 268 #endif 269 [ZONE_NORMAL] = 32, 270 #ifdef CONFIG_HIGHMEM 271 [ZONE_HIGHMEM] = 0, 272 #endif 273 [ZONE_MOVABLE] = 0, 274 }; 275 276 static char * const zone_names[MAX_NR_ZONES] = { 277 #ifdef CONFIG_ZONE_DMA 278 "DMA", 279 #endif 280 #ifdef CONFIG_ZONE_DMA32 281 "DMA32", 282 #endif 283 "Normal", 284 #ifdef CONFIG_HIGHMEM 285 "HighMem", 286 #endif 287 "Movable", 288 #ifdef CONFIG_ZONE_DEVICE 289 "Device", 290 #endif 291 }; 292 293 const char * const migratetype_names[MIGRATE_TYPES] = { 294 "Unmovable", 295 "Movable", 296 "Reclaimable", 297 "HighAtomic", 298 #ifdef CONFIG_CMA 299 "CMA", 300 #endif 301 #ifdef CONFIG_MEMORY_ISOLATION 302 "Isolate", 303 #endif 304 }; 305 306 compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = { 307 [NULL_COMPOUND_DTOR] = NULL, 308 [COMPOUND_PAGE_DTOR] = free_compound_page, 309 #ifdef CONFIG_HUGETLB_PAGE 310 [HUGETLB_PAGE_DTOR] = free_huge_page, 311 #endif 312 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 313 [TRANSHUGE_PAGE_DTOR] = free_transhuge_page, 314 #endif 315 }; 316 317 int min_free_kbytes = 1024; 318 int user_min_free_kbytes = -1; 319 #ifdef CONFIG_DISCONTIGMEM 320 /* 321 * DiscontigMem defines memory ranges as separate pg_data_t even if the ranges 322 * are not on separate NUMA nodes. Functionally this works but with 323 * watermark_boost_factor, it can reclaim prematurely as the ranges can be 324 * quite small. By default, do not boost watermarks on discontigmem as in 325 * many cases very high-order allocations like THP are likely to be 326 * unsupported and the premature reclaim offsets the advantage of long-term 327 * fragmentation avoidance. 328 */ 329 int watermark_boost_factor __read_mostly; 330 #else 331 int watermark_boost_factor __read_mostly = 15000; 332 #endif 333 int watermark_scale_factor = 10; 334 335 static unsigned long nr_kernel_pages __initdata; 336 static unsigned long nr_all_pages __initdata; 337 static unsigned long dma_reserve __initdata; 338 339 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 340 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 341 static unsigned long required_kernelcore __initdata; 342 static unsigned long required_kernelcore_percent __initdata; 343 static unsigned long required_movablecore __initdata; 344 static unsigned long required_movablecore_percent __initdata; 345 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 346 static bool mirrored_kernelcore __meminitdata; 347 348 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 349 int movable_zone; 350 EXPORT_SYMBOL(movable_zone); 351 352 #if MAX_NUMNODES > 1 353 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; 354 unsigned int nr_online_nodes __read_mostly = 1; 355 EXPORT_SYMBOL(nr_node_ids); 356 EXPORT_SYMBOL(nr_online_nodes); 357 #endif 358 359 int page_group_by_mobility_disabled __read_mostly; 360 361 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 362 /* 363 * During boot we initialize deferred pages on-demand, as needed, but once 364 * page_alloc_init_late() has finished, the deferred pages are all initialized, 365 * and we can permanently disable that path. 366 */ 367 static DEFINE_STATIC_KEY_TRUE(deferred_pages); 368 369 /* 370 * Calling kasan_free_pages() only after deferred memory initialization 371 * has completed. Poisoning pages during deferred memory init will greatly 372 * lengthen the process and cause problem in large memory systems as the 373 * deferred pages initialization is done with interrupt disabled. 374 * 375 * Assuming that there will be no reference to those newly initialized 376 * pages before they are ever allocated, this should have no effect on 377 * KASAN memory tracking as the poison will be properly inserted at page 378 * allocation time. The only corner case is when pages are allocated by 379 * on-demand allocation and then freed again before the deferred pages 380 * initialization is done, but this is not likely to happen. 381 */ 382 static inline void kasan_free_nondeferred_pages(struct page *page, int order) 383 { 384 if (!static_branch_unlikely(&deferred_pages)) 385 kasan_free_pages(page, order); 386 } 387 388 /* Returns true if the struct page for the pfn is uninitialised */ 389 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 390 { 391 int nid = early_pfn_to_nid(pfn); 392 393 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 394 return true; 395 396 return false; 397 } 398 399 /* 400 * Returns true when the remaining initialisation should be deferred until 401 * later in the boot cycle when it can be parallelised. 402 */ 403 static bool __meminit 404 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 405 { 406 static unsigned long prev_end_pfn, nr_initialised; 407 408 /* 409 * prev_end_pfn static that contains the end of previous zone 410 * No need to protect because called very early in boot before smp_init. 411 */ 412 if (prev_end_pfn != end_pfn) { 413 prev_end_pfn = end_pfn; 414 nr_initialised = 0; 415 } 416 417 /* Always populate low zones for address-constrained allocations */ 418 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 419 return false; 420 421 /* 422 * We start only with one section of pages, more pages are added as 423 * needed until the rest of deferred pages are initialized. 424 */ 425 nr_initialised++; 426 if ((nr_initialised > PAGES_PER_SECTION) && 427 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 428 NODE_DATA(nid)->first_deferred_pfn = pfn; 429 return true; 430 } 431 return false; 432 } 433 #else 434 #define kasan_free_nondeferred_pages(p, o) kasan_free_pages(p, o) 435 436 static inline bool early_page_uninitialised(unsigned long pfn) 437 { 438 return false; 439 } 440 441 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 442 { 443 return false; 444 } 445 #endif 446 447 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 448 static inline unsigned long *get_pageblock_bitmap(struct page *page, 449 unsigned long pfn) 450 { 451 #ifdef CONFIG_SPARSEMEM 452 return section_to_usemap(__pfn_to_section(pfn)); 453 #else 454 return page_zone(page)->pageblock_flags; 455 #endif /* CONFIG_SPARSEMEM */ 456 } 457 458 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn) 459 { 460 #ifdef CONFIG_SPARSEMEM 461 pfn &= (PAGES_PER_SECTION-1); 462 #else 463 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 464 #endif /* CONFIG_SPARSEMEM */ 465 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 466 } 467 468 /** 469 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 470 * @page: The page within the block of interest 471 * @pfn: The target page frame number 472 * @mask: mask of bits that the caller is interested in 473 * 474 * Return: pageblock_bits flags 475 */ 476 static __always_inline 477 unsigned long __get_pfnblock_flags_mask(struct page *page, 478 unsigned long pfn, 479 unsigned long mask) 480 { 481 unsigned long *bitmap; 482 unsigned long bitidx, word_bitidx; 483 unsigned long word; 484 485 bitmap = get_pageblock_bitmap(page, pfn); 486 bitidx = pfn_to_bitidx(page, pfn); 487 word_bitidx = bitidx / BITS_PER_LONG; 488 bitidx &= (BITS_PER_LONG-1); 489 490 word = bitmap[word_bitidx]; 491 return (word >> bitidx) & mask; 492 } 493 494 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, 495 unsigned long mask) 496 { 497 return __get_pfnblock_flags_mask(page, pfn, mask); 498 } 499 500 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn) 501 { 502 return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK); 503 } 504 505 /** 506 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 507 * @page: The page within the block of interest 508 * @flags: The flags to set 509 * @pfn: The target page frame number 510 * @mask: mask of bits that the caller is interested in 511 */ 512 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 513 unsigned long pfn, 514 unsigned long mask) 515 { 516 unsigned long *bitmap; 517 unsigned long bitidx, word_bitidx; 518 unsigned long old_word, word; 519 520 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 521 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits)); 522 523 bitmap = get_pageblock_bitmap(page, pfn); 524 bitidx = pfn_to_bitidx(page, pfn); 525 word_bitidx = bitidx / BITS_PER_LONG; 526 bitidx &= (BITS_PER_LONG-1); 527 528 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 529 530 mask <<= bitidx; 531 flags <<= bitidx; 532 533 word = READ_ONCE(bitmap[word_bitidx]); 534 for (;;) { 535 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 536 if (word == old_word) 537 break; 538 word = old_word; 539 } 540 } 541 542 void set_pageblock_migratetype(struct page *page, int migratetype) 543 { 544 if (unlikely(page_group_by_mobility_disabled && 545 migratetype < MIGRATE_PCPTYPES)) 546 migratetype = MIGRATE_UNMOVABLE; 547 548 set_pfnblock_flags_mask(page, (unsigned long)migratetype, 549 page_to_pfn(page), MIGRATETYPE_MASK); 550 } 551 552 #ifdef CONFIG_DEBUG_VM 553 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 554 { 555 int ret = 0; 556 unsigned seq; 557 unsigned long pfn = page_to_pfn(page); 558 unsigned long sp, start_pfn; 559 560 do { 561 seq = zone_span_seqbegin(zone); 562 start_pfn = zone->zone_start_pfn; 563 sp = zone->spanned_pages; 564 if (!zone_spans_pfn(zone, pfn)) 565 ret = 1; 566 } while (zone_span_seqretry(zone, seq)); 567 568 if (ret) 569 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 570 pfn, zone_to_nid(zone), zone->name, 571 start_pfn, start_pfn + sp); 572 573 return ret; 574 } 575 576 static int page_is_consistent(struct zone *zone, struct page *page) 577 { 578 if (!pfn_valid_within(page_to_pfn(page))) 579 return 0; 580 if (zone != page_zone(page)) 581 return 0; 582 583 return 1; 584 } 585 /* 586 * Temporary debugging check for pages not lying within a given zone. 587 */ 588 static int __maybe_unused bad_range(struct zone *zone, struct page *page) 589 { 590 if (page_outside_zone_boundaries(zone, page)) 591 return 1; 592 if (!page_is_consistent(zone, page)) 593 return 1; 594 595 return 0; 596 } 597 #else 598 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page) 599 { 600 return 0; 601 } 602 #endif 603 604 static void bad_page(struct page *page, const char *reason) 605 { 606 static unsigned long resume; 607 static unsigned long nr_shown; 608 static unsigned long nr_unshown; 609 610 /* 611 * Allow a burst of 60 reports, then keep quiet for that minute; 612 * or allow a steady drip of one report per second. 613 */ 614 if (nr_shown == 60) { 615 if (time_before(jiffies, resume)) { 616 nr_unshown++; 617 goto out; 618 } 619 if (nr_unshown) { 620 pr_alert( 621 "BUG: Bad page state: %lu messages suppressed\n", 622 nr_unshown); 623 nr_unshown = 0; 624 } 625 nr_shown = 0; 626 } 627 if (nr_shown++ == 0) 628 resume = jiffies + 60 * HZ; 629 630 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 631 current->comm, page_to_pfn(page)); 632 __dump_page(page, reason); 633 dump_page_owner(page); 634 635 print_modules(); 636 dump_stack(); 637 out: 638 /* Leave bad fields for debug, except PageBuddy could make trouble */ 639 page_mapcount_reset(page); /* remove PageBuddy */ 640 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 641 } 642 643 /* 644 * Higher-order pages are called "compound pages". They are structured thusly: 645 * 646 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 647 * 648 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 649 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 650 * 651 * The first tail page's ->compound_dtor holds the offset in array of compound 652 * page destructors. See compound_page_dtors. 653 * 654 * The first tail page's ->compound_order holds the order of allocation. 655 * This usage means that zero-order pages may not be compound. 656 */ 657 658 void free_compound_page(struct page *page) 659 { 660 mem_cgroup_uncharge(page); 661 __free_pages_ok(page, compound_order(page)); 662 } 663 664 void prep_compound_page(struct page *page, unsigned int order) 665 { 666 int i; 667 int nr_pages = 1 << order; 668 669 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 670 set_compound_order(page, order); 671 __SetPageHead(page); 672 for (i = 1; i < nr_pages; i++) { 673 struct page *p = page + i; 674 set_page_count(p, 0); 675 p->mapping = TAIL_MAPPING; 676 set_compound_head(p, page); 677 } 678 atomic_set(compound_mapcount_ptr(page), -1); 679 if (hpage_pincount_available(page)) 680 atomic_set(compound_pincount_ptr(page), 0); 681 } 682 683 #ifdef CONFIG_DEBUG_PAGEALLOC 684 unsigned int _debug_guardpage_minorder; 685 686 bool _debug_pagealloc_enabled_early __read_mostly 687 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 688 EXPORT_SYMBOL(_debug_pagealloc_enabled_early); 689 DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 690 EXPORT_SYMBOL(_debug_pagealloc_enabled); 691 692 DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 693 694 static int __init early_debug_pagealloc(char *buf) 695 { 696 return kstrtobool(buf, &_debug_pagealloc_enabled_early); 697 } 698 early_param("debug_pagealloc", early_debug_pagealloc); 699 700 void init_debug_pagealloc(void) 701 { 702 if (!debug_pagealloc_enabled()) 703 return; 704 705 static_branch_enable(&_debug_pagealloc_enabled); 706 707 if (!debug_guardpage_minorder()) 708 return; 709 710 static_branch_enable(&_debug_guardpage_enabled); 711 } 712 713 static int __init debug_guardpage_minorder_setup(char *buf) 714 { 715 unsigned long res; 716 717 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 718 pr_err("Bad debug_guardpage_minorder value\n"); 719 return 0; 720 } 721 _debug_guardpage_minorder = res; 722 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 723 return 0; 724 } 725 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); 726 727 static inline bool set_page_guard(struct zone *zone, struct page *page, 728 unsigned int order, int migratetype) 729 { 730 if (!debug_guardpage_enabled()) 731 return false; 732 733 if (order >= debug_guardpage_minorder()) 734 return false; 735 736 __SetPageGuard(page); 737 INIT_LIST_HEAD(&page->lru); 738 set_page_private(page, order); 739 /* Guard pages are not available for any usage */ 740 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 741 742 return true; 743 } 744 745 static inline void clear_page_guard(struct zone *zone, struct page *page, 746 unsigned int order, int migratetype) 747 { 748 if (!debug_guardpage_enabled()) 749 return; 750 751 __ClearPageGuard(page); 752 753 set_page_private(page, 0); 754 if (!is_migrate_isolate(migratetype)) 755 __mod_zone_freepage_state(zone, (1 << order), migratetype); 756 } 757 #else 758 static inline bool set_page_guard(struct zone *zone, struct page *page, 759 unsigned int order, int migratetype) { return false; } 760 static inline void clear_page_guard(struct zone *zone, struct page *page, 761 unsigned int order, int migratetype) {} 762 #endif 763 764 static inline void set_page_order(struct page *page, unsigned int order) 765 { 766 set_page_private(page, order); 767 __SetPageBuddy(page); 768 } 769 770 /* 771 * This function checks whether a page is free && is the buddy 772 * we can coalesce a page and its buddy if 773 * (a) the buddy is not in a hole (check before calling!) && 774 * (b) the buddy is in the buddy system && 775 * (c) a page and its buddy have the same order && 776 * (d) a page and its buddy are in the same zone. 777 * 778 * For recording whether a page is in the buddy system, we set PageBuddy. 779 * Setting, clearing, and testing PageBuddy is serialized by zone->lock. 780 * 781 * For recording page's order, we use page_private(page). 782 */ 783 static inline bool page_is_buddy(struct page *page, struct page *buddy, 784 unsigned int order) 785 { 786 if (!page_is_guard(buddy) && !PageBuddy(buddy)) 787 return false; 788 789 if (page_order(buddy) != order) 790 return false; 791 792 /* 793 * zone check is done late to avoid uselessly calculating 794 * zone/node ids for pages that could never merge. 795 */ 796 if (page_zone_id(page) != page_zone_id(buddy)) 797 return false; 798 799 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 800 801 return true; 802 } 803 804 #ifdef CONFIG_COMPACTION 805 static inline struct capture_control *task_capc(struct zone *zone) 806 { 807 struct capture_control *capc = current->capture_control; 808 809 return unlikely(capc) && 810 !(current->flags & PF_KTHREAD) && 811 !capc->page && 812 capc->cc->zone == zone ? capc : NULL; 813 } 814 815 static inline bool 816 compaction_capture(struct capture_control *capc, struct page *page, 817 int order, int migratetype) 818 { 819 if (!capc || order != capc->cc->order) 820 return false; 821 822 /* Do not accidentally pollute CMA or isolated regions*/ 823 if (is_migrate_cma(migratetype) || 824 is_migrate_isolate(migratetype)) 825 return false; 826 827 /* 828 * Do not let lower order allocations polluate a movable pageblock. 829 * This might let an unmovable request use a reclaimable pageblock 830 * and vice-versa but no more than normal fallback logic which can 831 * have trouble finding a high-order free page. 832 */ 833 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE) 834 return false; 835 836 capc->page = page; 837 return true; 838 } 839 840 #else 841 static inline struct capture_control *task_capc(struct zone *zone) 842 { 843 return NULL; 844 } 845 846 static inline bool 847 compaction_capture(struct capture_control *capc, struct page *page, 848 int order, int migratetype) 849 { 850 return false; 851 } 852 #endif /* CONFIG_COMPACTION */ 853 854 /* Used for pages not on another list */ 855 static inline void add_to_free_list(struct page *page, struct zone *zone, 856 unsigned int order, int migratetype) 857 { 858 struct free_area *area = &zone->free_area[order]; 859 860 list_add(&page->lru, &area->free_list[migratetype]); 861 area->nr_free++; 862 } 863 864 /* Used for pages not on another list */ 865 static inline void add_to_free_list_tail(struct page *page, struct zone *zone, 866 unsigned int order, int migratetype) 867 { 868 struct free_area *area = &zone->free_area[order]; 869 870 list_add_tail(&page->lru, &area->free_list[migratetype]); 871 area->nr_free++; 872 } 873 874 /* Used for pages which are on another list */ 875 static inline void move_to_free_list(struct page *page, struct zone *zone, 876 unsigned int order, int migratetype) 877 { 878 struct free_area *area = &zone->free_area[order]; 879 880 list_move(&page->lru, &area->free_list[migratetype]); 881 } 882 883 static inline void del_page_from_free_list(struct page *page, struct zone *zone, 884 unsigned int order) 885 { 886 /* clear reported state and update reported page count */ 887 if (page_reported(page)) 888 __ClearPageReported(page); 889 890 list_del(&page->lru); 891 __ClearPageBuddy(page); 892 set_page_private(page, 0); 893 zone->free_area[order].nr_free--; 894 } 895 896 /* 897 * If this is not the largest possible page, check if the buddy 898 * of the next-highest order is free. If it is, it's possible 899 * that pages are being freed that will coalesce soon. In case, 900 * that is happening, add the free page to the tail of the list 901 * so it's less likely to be used soon and more likely to be merged 902 * as a higher order page 903 */ 904 static inline bool 905 buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, 906 struct page *page, unsigned int order) 907 { 908 struct page *higher_page, *higher_buddy; 909 unsigned long combined_pfn; 910 911 if (order >= MAX_ORDER - 2) 912 return false; 913 914 if (!pfn_valid_within(buddy_pfn)) 915 return false; 916 917 combined_pfn = buddy_pfn & pfn; 918 higher_page = page + (combined_pfn - pfn); 919 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1); 920 higher_buddy = higher_page + (buddy_pfn - combined_pfn); 921 922 return pfn_valid_within(buddy_pfn) && 923 page_is_buddy(higher_page, higher_buddy, order + 1); 924 } 925 926 /* 927 * Freeing function for a buddy system allocator. 928 * 929 * The concept of a buddy system is to maintain direct-mapped table 930 * (containing bit values) for memory blocks of various "orders". 931 * The bottom level table contains the map for the smallest allocatable 932 * units of memory (here, pages), and each level above it describes 933 * pairs of units from the levels below, hence, "buddies". 934 * At a high level, all that happens here is marking the table entry 935 * at the bottom level available, and propagating the changes upward 936 * as necessary, plus some accounting needed to play nicely with other 937 * parts of the VM system. 938 * At each level, we keep a list of pages, which are heads of continuous 939 * free pages of length of (1 << order) and marked with PageBuddy. 940 * Page's order is recorded in page_private(page) field. 941 * So when we are allocating or freeing one, we can derive the state of the 942 * other. That is, if we allocate a small block, and both were 943 * free, the remainder of the region must be split into blocks. 944 * If a block is freed, and its buddy is also free, then this 945 * triggers coalescing into a block of larger size. 946 * 947 * -- nyc 948 */ 949 950 static inline void __free_one_page(struct page *page, 951 unsigned long pfn, 952 struct zone *zone, unsigned int order, 953 int migratetype, bool report) 954 { 955 struct capture_control *capc = task_capc(zone); 956 unsigned long buddy_pfn; 957 unsigned long combined_pfn; 958 unsigned int max_order; 959 struct page *buddy; 960 bool to_tail; 961 962 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1); 963 964 VM_BUG_ON(!zone_is_initialized(zone)); 965 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 966 967 VM_BUG_ON(migratetype == -1); 968 if (likely(!is_migrate_isolate(migratetype))) 969 __mod_zone_freepage_state(zone, 1 << order, migratetype); 970 971 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); 972 VM_BUG_ON_PAGE(bad_range(zone, page), page); 973 974 continue_merging: 975 while (order < max_order - 1) { 976 if (compaction_capture(capc, page, order, migratetype)) { 977 __mod_zone_freepage_state(zone, -(1 << order), 978 migratetype); 979 return; 980 } 981 buddy_pfn = __find_buddy_pfn(pfn, order); 982 buddy = page + (buddy_pfn - pfn); 983 984 if (!pfn_valid_within(buddy_pfn)) 985 goto done_merging; 986 if (!page_is_buddy(page, buddy, order)) 987 goto done_merging; 988 /* 989 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 990 * merge with it and move up one order. 991 */ 992 if (page_is_guard(buddy)) 993 clear_page_guard(zone, buddy, order, migratetype); 994 else 995 del_page_from_free_list(buddy, zone, order); 996 combined_pfn = buddy_pfn & pfn; 997 page = page + (combined_pfn - pfn); 998 pfn = combined_pfn; 999 order++; 1000 } 1001 if (max_order < MAX_ORDER) { 1002 /* If we are here, it means order is >= pageblock_order. 1003 * We want to prevent merge between freepages on isolate 1004 * pageblock and normal pageblock. Without this, pageblock 1005 * isolation could cause incorrect freepage or CMA accounting. 1006 * 1007 * We don't want to hit this code for the more frequent 1008 * low-order merging. 1009 */ 1010 if (unlikely(has_isolate_pageblock(zone))) { 1011 int buddy_mt; 1012 1013 buddy_pfn = __find_buddy_pfn(pfn, order); 1014 buddy = page + (buddy_pfn - pfn); 1015 buddy_mt = get_pageblock_migratetype(buddy); 1016 1017 if (migratetype != buddy_mt 1018 && (is_migrate_isolate(migratetype) || 1019 is_migrate_isolate(buddy_mt))) 1020 goto done_merging; 1021 } 1022 max_order++; 1023 goto continue_merging; 1024 } 1025 1026 done_merging: 1027 set_page_order(page, order); 1028 1029 if (is_shuffle_order(order)) 1030 to_tail = shuffle_pick_tail(); 1031 else 1032 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); 1033 1034 if (to_tail) 1035 add_to_free_list_tail(page, zone, order, migratetype); 1036 else 1037 add_to_free_list(page, zone, order, migratetype); 1038 1039 /* Notify page reporting subsystem of freed page */ 1040 if (report) 1041 page_reporting_notify_free(order); 1042 } 1043 1044 /* 1045 * A bad page could be due to a number of fields. Instead of multiple branches, 1046 * try and check multiple fields with one check. The caller must do a detailed 1047 * check if necessary. 1048 */ 1049 static inline bool page_expected_state(struct page *page, 1050 unsigned long check_flags) 1051 { 1052 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1053 return false; 1054 1055 if (unlikely((unsigned long)page->mapping | 1056 page_ref_count(page) | 1057 #ifdef CONFIG_MEMCG 1058 (unsigned long)page->mem_cgroup | 1059 #endif 1060 (page->flags & check_flags))) 1061 return false; 1062 1063 return true; 1064 } 1065 1066 static const char *page_bad_reason(struct page *page, unsigned long flags) 1067 { 1068 const char *bad_reason = NULL; 1069 1070 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1071 bad_reason = "nonzero mapcount"; 1072 if (unlikely(page->mapping != NULL)) 1073 bad_reason = "non-NULL mapping"; 1074 if (unlikely(page_ref_count(page) != 0)) 1075 bad_reason = "nonzero _refcount"; 1076 if (unlikely(page->flags & flags)) { 1077 if (flags == PAGE_FLAGS_CHECK_AT_PREP) 1078 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set"; 1079 else 1080 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 1081 } 1082 #ifdef CONFIG_MEMCG 1083 if (unlikely(page->mem_cgroup)) 1084 bad_reason = "page still charged to cgroup"; 1085 #endif 1086 return bad_reason; 1087 } 1088 1089 static void check_free_page_bad(struct page *page) 1090 { 1091 bad_page(page, 1092 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE)); 1093 } 1094 1095 static inline int check_free_page(struct page *page) 1096 { 1097 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 1098 return 0; 1099 1100 /* Something has gone sideways, find it */ 1101 check_free_page_bad(page); 1102 return 1; 1103 } 1104 1105 static int free_tail_pages_check(struct page *head_page, struct page *page) 1106 { 1107 int ret = 1; 1108 1109 /* 1110 * We rely page->lru.next never has bit 0 set, unless the page 1111 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 1112 */ 1113 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 1114 1115 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 1116 ret = 0; 1117 goto out; 1118 } 1119 switch (page - head_page) { 1120 case 1: 1121 /* the first tail page: ->mapping may be compound_mapcount() */ 1122 if (unlikely(compound_mapcount(page))) { 1123 bad_page(page, "nonzero compound_mapcount"); 1124 goto out; 1125 } 1126 break; 1127 case 2: 1128 /* 1129 * the second tail page: ->mapping is 1130 * deferred_list.next -- ignore value. 1131 */ 1132 break; 1133 default: 1134 if (page->mapping != TAIL_MAPPING) { 1135 bad_page(page, "corrupted mapping in tail page"); 1136 goto out; 1137 } 1138 break; 1139 } 1140 if (unlikely(!PageTail(page))) { 1141 bad_page(page, "PageTail not set"); 1142 goto out; 1143 } 1144 if (unlikely(compound_head(page) != head_page)) { 1145 bad_page(page, "compound_head not consistent"); 1146 goto out; 1147 } 1148 ret = 0; 1149 out: 1150 page->mapping = NULL; 1151 clear_compound_head(page); 1152 return ret; 1153 } 1154 1155 static void kernel_init_free_pages(struct page *page, int numpages) 1156 { 1157 int i; 1158 1159 /* s390's use of memset() could override KASAN redzones. */ 1160 kasan_disable_current(); 1161 for (i = 0; i < numpages; i++) 1162 clear_highpage(page + i); 1163 kasan_enable_current(); 1164 } 1165 1166 static __always_inline bool free_pages_prepare(struct page *page, 1167 unsigned int order, bool check_free) 1168 { 1169 int bad = 0; 1170 1171 VM_BUG_ON_PAGE(PageTail(page), page); 1172 1173 trace_mm_page_free(page, order); 1174 1175 /* 1176 * Check tail pages before head page information is cleared to 1177 * avoid checking PageCompound for order-0 pages. 1178 */ 1179 if (unlikely(order)) { 1180 bool compound = PageCompound(page); 1181 int i; 1182 1183 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1184 1185 if (compound) 1186 ClearPageDoubleMap(page); 1187 for (i = 1; i < (1 << order); i++) { 1188 if (compound) 1189 bad += free_tail_pages_check(page, page + i); 1190 if (unlikely(check_free_page(page + i))) { 1191 bad++; 1192 continue; 1193 } 1194 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1195 } 1196 } 1197 if (PageMappingFlags(page)) 1198 page->mapping = NULL; 1199 if (memcg_kmem_enabled() && PageKmemcg(page)) 1200 __memcg_kmem_uncharge_page(page, order); 1201 if (check_free) 1202 bad += check_free_page(page); 1203 if (bad) 1204 return false; 1205 1206 page_cpupid_reset_last(page); 1207 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1208 reset_page_owner(page, order); 1209 1210 if (!PageHighMem(page)) { 1211 debug_check_no_locks_freed(page_address(page), 1212 PAGE_SIZE << order); 1213 debug_check_no_obj_freed(page_address(page), 1214 PAGE_SIZE << order); 1215 } 1216 if (want_init_on_free()) 1217 kernel_init_free_pages(page, 1 << order); 1218 1219 kernel_poison_pages(page, 1 << order, 0); 1220 /* 1221 * arch_free_page() can make the page's contents inaccessible. s390 1222 * does this. So nothing which can access the page's contents should 1223 * happen after this. 1224 */ 1225 arch_free_page(page, order); 1226 1227 if (debug_pagealloc_enabled_static()) 1228 kernel_map_pages(page, 1 << order, 0); 1229 1230 kasan_free_nondeferred_pages(page, order); 1231 1232 return true; 1233 } 1234 1235 #ifdef CONFIG_DEBUG_VM 1236 /* 1237 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed 1238 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when 1239 * moved from pcp lists to free lists. 1240 */ 1241 static bool free_pcp_prepare(struct page *page) 1242 { 1243 return free_pages_prepare(page, 0, true); 1244 } 1245 1246 static bool bulkfree_pcp_prepare(struct page *page) 1247 { 1248 if (debug_pagealloc_enabled_static()) 1249 return check_free_page(page); 1250 else 1251 return false; 1252 } 1253 #else 1254 /* 1255 * With DEBUG_VM disabled, order-0 pages being freed are checked only when 1256 * moving from pcp lists to free list in order to reduce overhead. With 1257 * debug_pagealloc enabled, they are checked also immediately when being freed 1258 * to the pcp lists. 1259 */ 1260 static bool free_pcp_prepare(struct page *page) 1261 { 1262 if (debug_pagealloc_enabled_static()) 1263 return free_pages_prepare(page, 0, true); 1264 else 1265 return free_pages_prepare(page, 0, false); 1266 } 1267 1268 static bool bulkfree_pcp_prepare(struct page *page) 1269 { 1270 return check_free_page(page); 1271 } 1272 #endif /* CONFIG_DEBUG_VM */ 1273 1274 static inline void prefetch_buddy(struct page *page) 1275 { 1276 unsigned long pfn = page_to_pfn(page); 1277 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0); 1278 struct page *buddy = page + (buddy_pfn - pfn); 1279 1280 prefetch(buddy); 1281 } 1282 1283 /* 1284 * Frees a number of pages from the PCP lists 1285 * Assumes all pages on list are in same zone, and of same order. 1286 * count is the number of pages to free. 1287 * 1288 * If the zone was previously in an "all pages pinned" state then look to 1289 * see if this freeing clears that state. 1290 * 1291 * And clear the zone's pages_scanned counter, to hold off the "all pages are 1292 * pinned" detection logic. 1293 */ 1294 static void free_pcppages_bulk(struct zone *zone, int count, 1295 struct per_cpu_pages *pcp) 1296 { 1297 int migratetype = 0; 1298 int batch_free = 0; 1299 int prefetch_nr = 0; 1300 bool isolated_pageblocks; 1301 struct page *page, *tmp; 1302 LIST_HEAD(head); 1303 1304 while (count) { 1305 struct list_head *list; 1306 1307 /* 1308 * Remove pages from lists in a round-robin fashion. A 1309 * batch_free count is maintained that is incremented when an 1310 * empty list is encountered. This is so more pages are freed 1311 * off fuller lists instead of spinning excessively around empty 1312 * lists 1313 */ 1314 do { 1315 batch_free++; 1316 if (++migratetype == MIGRATE_PCPTYPES) 1317 migratetype = 0; 1318 list = &pcp->lists[migratetype]; 1319 } while (list_empty(list)); 1320 1321 /* This is the only non-empty list. Free them all. */ 1322 if (batch_free == MIGRATE_PCPTYPES) 1323 batch_free = count; 1324 1325 do { 1326 page = list_last_entry(list, struct page, lru); 1327 /* must delete to avoid corrupting pcp list */ 1328 list_del(&page->lru); 1329 pcp->count--; 1330 1331 if (bulkfree_pcp_prepare(page)) 1332 continue; 1333 1334 list_add_tail(&page->lru, &head); 1335 1336 /* 1337 * We are going to put the page back to the global 1338 * pool, prefetch its buddy to speed up later access 1339 * under zone->lock. It is believed the overhead of 1340 * an additional test and calculating buddy_pfn here 1341 * can be offset by reduced memory latency later. To 1342 * avoid excessive prefetching due to large count, only 1343 * prefetch buddy for the first pcp->batch nr of pages. 1344 */ 1345 if (prefetch_nr++ < pcp->batch) 1346 prefetch_buddy(page); 1347 } while (--count && --batch_free && !list_empty(list)); 1348 } 1349 1350 spin_lock(&zone->lock); 1351 isolated_pageblocks = has_isolate_pageblock(zone); 1352 1353 /* 1354 * Use safe version since after __free_one_page(), 1355 * page->lru.next will not point to original list. 1356 */ 1357 list_for_each_entry_safe(page, tmp, &head, lru) { 1358 int mt = get_pcppage_migratetype(page); 1359 /* MIGRATE_ISOLATE page should not go to pcplists */ 1360 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1361 /* Pageblock could have been isolated meanwhile */ 1362 if (unlikely(isolated_pageblocks)) 1363 mt = get_pageblock_migratetype(page); 1364 1365 __free_one_page(page, page_to_pfn(page), zone, 0, mt, true); 1366 trace_mm_page_pcpu_drain(page, 0, mt); 1367 } 1368 spin_unlock(&zone->lock); 1369 } 1370 1371 static void free_one_page(struct zone *zone, 1372 struct page *page, unsigned long pfn, 1373 unsigned int order, 1374 int migratetype) 1375 { 1376 spin_lock(&zone->lock); 1377 if (unlikely(has_isolate_pageblock(zone) || 1378 is_migrate_isolate(migratetype))) { 1379 migratetype = get_pfnblock_migratetype(page, pfn); 1380 } 1381 __free_one_page(page, pfn, zone, order, migratetype, true); 1382 spin_unlock(&zone->lock); 1383 } 1384 1385 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1386 unsigned long zone, int nid) 1387 { 1388 mm_zero_struct_page(page); 1389 set_page_links(page, zone, nid, pfn); 1390 init_page_count(page); 1391 page_mapcount_reset(page); 1392 page_cpupid_reset_last(page); 1393 page_kasan_tag_reset(page); 1394 1395 INIT_LIST_HEAD(&page->lru); 1396 #ifdef WANT_PAGE_VIRTUAL 1397 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1398 if (!is_highmem_idx(zone)) 1399 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1400 #endif 1401 } 1402 1403 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1404 static void __meminit init_reserved_page(unsigned long pfn) 1405 { 1406 pg_data_t *pgdat; 1407 int nid, zid; 1408 1409 if (!early_page_uninitialised(pfn)) 1410 return; 1411 1412 nid = early_pfn_to_nid(pfn); 1413 pgdat = NODE_DATA(nid); 1414 1415 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1416 struct zone *zone = &pgdat->node_zones[zid]; 1417 1418 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) 1419 break; 1420 } 1421 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 1422 } 1423 #else 1424 static inline void init_reserved_page(unsigned long pfn) 1425 { 1426 } 1427 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1428 1429 /* 1430 * Initialised pages do not have PageReserved set. This function is 1431 * called for each range allocated by the bootmem allocator and 1432 * marks the pages PageReserved. The remaining valid pages are later 1433 * sent to the buddy page allocator. 1434 */ 1435 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1436 { 1437 unsigned long start_pfn = PFN_DOWN(start); 1438 unsigned long end_pfn = PFN_UP(end); 1439 1440 for (; start_pfn < end_pfn; start_pfn++) { 1441 if (pfn_valid(start_pfn)) { 1442 struct page *page = pfn_to_page(start_pfn); 1443 1444 init_reserved_page(start_pfn); 1445 1446 /* Avoid false-positive PageTail() */ 1447 INIT_LIST_HEAD(&page->lru); 1448 1449 /* 1450 * no need for atomic set_bit because the struct 1451 * page is not visible yet so nobody should 1452 * access it yet. 1453 */ 1454 __SetPageReserved(page); 1455 } 1456 } 1457 } 1458 1459 static void __free_pages_ok(struct page *page, unsigned int order) 1460 { 1461 unsigned long flags; 1462 int migratetype; 1463 unsigned long pfn = page_to_pfn(page); 1464 1465 if (!free_pages_prepare(page, order, true)) 1466 return; 1467 1468 migratetype = get_pfnblock_migratetype(page, pfn); 1469 local_irq_save(flags); 1470 __count_vm_events(PGFREE, 1 << order); 1471 free_one_page(page_zone(page), page, pfn, order, migratetype); 1472 local_irq_restore(flags); 1473 } 1474 1475 void __free_pages_core(struct page *page, unsigned int order) 1476 { 1477 unsigned int nr_pages = 1 << order; 1478 struct page *p = page; 1479 unsigned int loop; 1480 1481 prefetchw(p); 1482 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1483 prefetchw(p + 1); 1484 __ClearPageReserved(p); 1485 set_page_count(p, 0); 1486 } 1487 __ClearPageReserved(p); 1488 set_page_count(p, 0); 1489 1490 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); 1491 set_page_refcounted(page); 1492 __free_pages(page, order); 1493 } 1494 1495 #ifdef CONFIG_NEED_MULTIPLE_NODES 1496 1497 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1498 1499 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID 1500 1501 /* 1502 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 1503 */ 1504 int __meminit __early_pfn_to_nid(unsigned long pfn, 1505 struct mminit_pfnnid_cache *state) 1506 { 1507 unsigned long start_pfn, end_pfn; 1508 int nid; 1509 1510 if (state->last_start <= pfn && pfn < state->last_end) 1511 return state->last_nid; 1512 1513 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 1514 if (nid != NUMA_NO_NODE) { 1515 state->last_start = start_pfn; 1516 state->last_end = end_pfn; 1517 state->last_nid = nid; 1518 } 1519 1520 return nid; 1521 } 1522 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ 1523 1524 int __meminit early_pfn_to_nid(unsigned long pfn) 1525 { 1526 static DEFINE_SPINLOCK(early_pfn_lock); 1527 int nid; 1528 1529 spin_lock(&early_pfn_lock); 1530 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1531 if (nid < 0) 1532 nid = first_online_node; 1533 spin_unlock(&early_pfn_lock); 1534 1535 return nid; 1536 } 1537 #endif /* CONFIG_NEED_MULTIPLE_NODES */ 1538 1539 void __init memblock_free_pages(struct page *page, unsigned long pfn, 1540 unsigned int order) 1541 { 1542 if (early_page_uninitialised(pfn)) 1543 return; 1544 __free_pages_core(page, order); 1545 } 1546 1547 /* 1548 * Check that the whole (or subset of) a pageblock given by the interval of 1549 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1550 * with the migration of free compaction scanner. The scanners then need to 1551 * use only pfn_valid_within() check for arches that allow holes within 1552 * pageblocks. 1553 * 1554 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1555 * 1556 * It's possible on some configurations to have a setup like node0 node1 node0 1557 * i.e. it's possible that all pages within a zones range of pages do not 1558 * belong to a single zone. We assume that a border between node0 and node1 1559 * can occur within a single pageblock, but not a node0 node1 node0 1560 * interleaving within a single pageblock. It is therefore sufficient to check 1561 * the first and last page of a pageblock and avoid checking each individual 1562 * page in a pageblock. 1563 */ 1564 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1565 unsigned long end_pfn, struct zone *zone) 1566 { 1567 struct page *start_page; 1568 struct page *end_page; 1569 1570 /* end_pfn is one past the range we are checking */ 1571 end_pfn--; 1572 1573 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1574 return NULL; 1575 1576 start_page = pfn_to_online_page(start_pfn); 1577 if (!start_page) 1578 return NULL; 1579 1580 if (page_zone(start_page) != zone) 1581 return NULL; 1582 1583 end_page = pfn_to_page(end_pfn); 1584 1585 /* This gives a shorter code than deriving page_zone(end_page) */ 1586 if (page_zone_id(start_page) != page_zone_id(end_page)) 1587 return NULL; 1588 1589 return start_page; 1590 } 1591 1592 void set_zone_contiguous(struct zone *zone) 1593 { 1594 unsigned long block_start_pfn = zone->zone_start_pfn; 1595 unsigned long block_end_pfn; 1596 1597 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1598 for (; block_start_pfn < zone_end_pfn(zone); 1599 block_start_pfn = block_end_pfn, 1600 block_end_pfn += pageblock_nr_pages) { 1601 1602 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1603 1604 if (!__pageblock_pfn_to_page(block_start_pfn, 1605 block_end_pfn, zone)) 1606 return; 1607 cond_resched(); 1608 } 1609 1610 /* We confirm that there is no hole */ 1611 zone->contiguous = true; 1612 } 1613 1614 void clear_zone_contiguous(struct zone *zone) 1615 { 1616 zone->contiguous = false; 1617 } 1618 1619 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1620 static void __init deferred_free_range(unsigned long pfn, 1621 unsigned long nr_pages) 1622 { 1623 struct page *page; 1624 unsigned long i; 1625 1626 if (!nr_pages) 1627 return; 1628 1629 page = pfn_to_page(pfn); 1630 1631 /* Free a large naturally-aligned chunk if possible */ 1632 if (nr_pages == pageblock_nr_pages && 1633 (pfn & (pageblock_nr_pages - 1)) == 0) { 1634 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1635 __free_pages_core(page, pageblock_order); 1636 return; 1637 } 1638 1639 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1640 if ((pfn & (pageblock_nr_pages - 1)) == 0) 1641 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1642 __free_pages_core(page, 0); 1643 } 1644 } 1645 1646 /* Completion tracking for deferred_init_memmap() threads */ 1647 static atomic_t pgdat_init_n_undone __initdata; 1648 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1649 1650 static inline void __init pgdat_init_report_one_done(void) 1651 { 1652 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1653 complete(&pgdat_init_all_done_comp); 1654 } 1655 1656 /* 1657 * Returns true if page needs to be initialized or freed to buddy allocator. 1658 * 1659 * First we check if pfn is valid on architectures where it is possible to have 1660 * holes within pageblock_nr_pages. On systems where it is not possible, this 1661 * function is optimized out. 1662 * 1663 * Then, we check if a current large page is valid by only checking the validity 1664 * of the head pfn. 1665 */ 1666 static inline bool __init deferred_pfn_valid(unsigned long pfn) 1667 { 1668 if (!pfn_valid_within(pfn)) 1669 return false; 1670 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn)) 1671 return false; 1672 return true; 1673 } 1674 1675 /* 1676 * Free pages to buddy allocator. Try to free aligned pages in 1677 * pageblock_nr_pages sizes. 1678 */ 1679 static void __init deferred_free_pages(unsigned long pfn, 1680 unsigned long end_pfn) 1681 { 1682 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1683 unsigned long nr_free = 0; 1684 1685 for (; pfn < end_pfn; pfn++) { 1686 if (!deferred_pfn_valid(pfn)) { 1687 deferred_free_range(pfn - nr_free, nr_free); 1688 nr_free = 0; 1689 } else if (!(pfn & nr_pgmask)) { 1690 deferred_free_range(pfn - nr_free, nr_free); 1691 nr_free = 1; 1692 } else { 1693 nr_free++; 1694 } 1695 } 1696 /* Free the last block of pages to allocator */ 1697 deferred_free_range(pfn - nr_free, nr_free); 1698 } 1699 1700 /* 1701 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 1702 * by performing it only once every pageblock_nr_pages. 1703 * Return number of pages initialized. 1704 */ 1705 static unsigned long __init deferred_init_pages(struct zone *zone, 1706 unsigned long pfn, 1707 unsigned long end_pfn) 1708 { 1709 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1710 int nid = zone_to_nid(zone); 1711 unsigned long nr_pages = 0; 1712 int zid = zone_idx(zone); 1713 struct page *page = NULL; 1714 1715 for (; pfn < end_pfn; pfn++) { 1716 if (!deferred_pfn_valid(pfn)) { 1717 page = NULL; 1718 continue; 1719 } else if (!page || !(pfn & nr_pgmask)) { 1720 page = pfn_to_page(pfn); 1721 } else { 1722 page++; 1723 } 1724 __init_single_page(page, pfn, zid, nid); 1725 nr_pages++; 1726 } 1727 return (nr_pages); 1728 } 1729 1730 /* 1731 * This function is meant to pre-load the iterator for the zone init. 1732 * Specifically it walks through the ranges until we are caught up to the 1733 * first_init_pfn value and exits there. If we never encounter the value we 1734 * return false indicating there are no valid ranges left. 1735 */ 1736 static bool __init 1737 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, 1738 unsigned long *spfn, unsigned long *epfn, 1739 unsigned long first_init_pfn) 1740 { 1741 u64 j; 1742 1743 /* 1744 * Start out by walking through the ranges in this zone that have 1745 * already been initialized. We don't need to do anything with them 1746 * so we just need to flush them out of the system. 1747 */ 1748 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { 1749 if (*epfn <= first_init_pfn) 1750 continue; 1751 if (*spfn < first_init_pfn) 1752 *spfn = first_init_pfn; 1753 *i = j; 1754 return true; 1755 } 1756 1757 return false; 1758 } 1759 1760 /* 1761 * Initialize and free pages. We do it in two loops: first we initialize 1762 * struct page, then free to buddy allocator, because while we are 1763 * freeing pages we can access pages that are ahead (computing buddy 1764 * page in __free_one_page()). 1765 * 1766 * In order to try and keep some memory in the cache we have the loop 1767 * broken along max page order boundaries. This way we will not cause 1768 * any issues with the buddy page computation. 1769 */ 1770 static unsigned long __init 1771 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, 1772 unsigned long *end_pfn) 1773 { 1774 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); 1775 unsigned long spfn = *start_pfn, epfn = *end_pfn; 1776 unsigned long nr_pages = 0; 1777 u64 j = *i; 1778 1779 /* First we loop through and initialize the page values */ 1780 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { 1781 unsigned long t; 1782 1783 if (mo_pfn <= *start_pfn) 1784 break; 1785 1786 t = min(mo_pfn, *end_pfn); 1787 nr_pages += deferred_init_pages(zone, *start_pfn, t); 1788 1789 if (mo_pfn < *end_pfn) { 1790 *start_pfn = mo_pfn; 1791 break; 1792 } 1793 } 1794 1795 /* Reset values and now loop through freeing pages as needed */ 1796 swap(j, *i); 1797 1798 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { 1799 unsigned long t; 1800 1801 if (mo_pfn <= spfn) 1802 break; 1803 1804 t = min(mo_pfn, epfn); 1805 deferred_free_pages(spfn, t); 1806 1807 if (mo_pfn <= epfn) 1808 break; 1809 } 1810 1811 return nr_pages; 1812 } 1813 1814 static void __init 1815 deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, 1816 void *arg) 1817 { 1818 unsigned long spfn, epfn; 1819 struct zone *zone = arg; 1820 u64 i; 1821 1822 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); 1823 1824 /* 1825 * Initialize and free pages in MAX_ORDER sized increments so that we 1826 * can avoid introducing any issues with the buddy allocator. 1827 */ 1828 while (spfn < end_pfn) { 1829 deferred_init_maxorder(&i, zone, &spfn, &epfn); 1830 cond_resched(); 1831 } 1832 } 1833 1834 /* An arch may override for more concurrency. */ 1835 __weak int __init 1836 deferred_page_init_max_threads(const struct cpumask *node_cpumask) 1837 { 1838 return 1; 1839 } 1840 1841 /* Initialise remaining memory on a node */ 1842 static int __init deferred_init_memmap(void *data) 1843 { 1844 pg_data_t *pgdat = data; 1845 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 1846 unsigned long spfn = 0, epfn = 0; 1847 unsigned long first_init_pfn, flags; 1848 unsigned long start = jiffies; 1849 struct zone *zone; 1850 int zid, max_threads; 1851 u64 i; 1852 1853 /* Bind memory initialisation thread to a local node if possible */ 1854 if (!cpumask_empty(cpumask)) 1855 set_cpus_allowed_ptr(current, cpumask); 1856 1857 pgdat_resize_lock(pgdat, &flags); 1858 first_init_pfn = pgdat->first_deferred_pfn; 1859 if (first_init_pfn == ULONG_MAX) { 1860 pgdat_resize_unlock(pgdat, &flags); 1861 pgdat_init_report_one_done(); 1862 return 0; 1863 } 1864 1865 /* Sanity check boundaries */ 1866 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 1867 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 1868 pgdat->first_deferred_pfn = ULONG_MAX; 1869 1870 /* 1871 * Once we unlock here, the zone cannot be grown anymore, thus if an 1872 * interrupt thread must allocate this early in boot, zone must be 1873 * pre-grown prior to start of deferred page initialization. 1874 */ 1875 pgdat_resize_unlock(pgdat, &flags); 1876 1877 /* Only the highest zone is deferred so find it */ 1878 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1879 zone = pgdat->node_zones + zid; 1880 if (first_init_pfn < zone_end_pfn(zone)) 1881 break; 1882 } 1883 1884 /* If the zone is empty somebody else may have cleared out the zone */ 1885 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 1886 first_init_pfn)) 1887 goto zone_empty; 1888 1889 max_threads = deferred_page_init_max_threads(cpumask); 1890 1891 while (spfn < epfn) { 1892 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION); 1893 struct padata_mt_job job = { 1894 .thread_fn = deferred_init_memmap_chunk, 1895 .fn_arg = zone, 1896 .start = spfn, 1897 .size = epfn_align - spfn, 1898 .align = PAGES_PER_SECTION, 1899 .min_chunk = PAGES_PER_SECTION, 1900 .max_threads = max_threads, 1901 }; 1902 1903 padata_do_multithreaded(&job); 1904 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 1905 epfn_align); 1906 } 1907 zone_empty: 1908 /* Sanity check that the next zone really is unpopulated */ 1909 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 1910 1911 pr_info("node %d deferred pages initialised in %ums\n", 1912 pgdat->node_id, jiffies_to_msecs(jiffies - start)); 1913 1914 pgdat_init_report_one_done(); 1915 return 0; 1916 } 1917 1918 /* 1919 * If this zone has deferred pages, try to grow it by initializing enough 1920 * deferred pages to satisfy the allocation specified by order, rounded up to 1921 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 1922 * of SECTION_SIZE bytes by initializing struct pages in increments of 1923 * PAGES_PER_SECTION * sizeof(struct page) bytes. 1924 * 1925 * Return true when zone was grown, otherwise return false. We return true even 1926 * when we grow less than requested, to let the caller decide if there are 1927 * enough pages to satisfy the allocation. 1928 * 1929 * Note: We use noinline because this function is needed only during boot, and 1930 * it is called from a __ref function _deferred_grow_zone. This way we are 1931 * making sure that it is not inlined into permanent text section. 1932 */ 1933 static noinline bool __init 1934 deferred_grow_zone(struct zone *zone, unsigned int order) 1935 { 1936 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 1937 pg_data_t *pgdat = zone->zone_pgdat; 1938 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 1939 unsigned long spfn, epfn, flags; 1940 unsigned long nr_pages = 0; 1941 u64 i; 1942 1943 /* Only the last zone may have deferred pages */ 1944 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 1945 return false; 1946 1947 pgdat_resize_lock(pgdat, &flags); 1948 1949 /* 1950 * If someone grew this zone while we were waiting for spinlock, return 1951 * true, as there might be enough pages already. 1952 */ 1953 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 1954 pgdat_resize_unlock(pgdat, &flags); 1955 return true; 1956 } 1957 1958 /* If the zone is empty somebody else may have cleared out the zone */ 1959 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 1960 first_deferred_pfn)) { 1961 pgdat->first_deferred_pfn = ULONG_MAX; 1962 pgdat_resize_unlock(pgdat, &flags); 1963 /* Retry only once. */ 1964 return first_deferred_pfn != ULONG_MAX; 1965 } 1966 1967 /* 1968 * Initialize and free pages in MAX_ORDER sized increments so 1969 * that we can avoid introducing any issues with the buddy 1970 * allocator. 1971 */ 1972 while (spfn < epfn) { 1973 /* update our first deferred PFN for this section */ 1974 first_deferred_pfn = spfn; 1975 1976 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 1977 touch_nmi_watchdog(); 1978 1979 /* We should only stop along section boundaries */ 1980 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) 1981 continue; 1982 1983 /* If our quota has been met we can stop here */ 1984 if (nr_pages >= nr_pages_needed) 1985 break; 1986 } 1987 1988 pgdat->first_deferred_pfn = spfn; 1989 pgdat_resize_unlock(pgdat, &flags); 1990 1991 return nr_pages > 0; 1992 } 1993 1994 /* 1995 * deferred_grow_zone() is __init, but it is called from 1996 * get_page_from_freelist() during early boot until deferred_pages permanently 1997 * disables this call. This is why we have refdata wrapper to avoid warning, 1998 * and to ensure that the function body gets unloaded. 1999 */ 2000 static bool __ref 2001 _deferred_grow_zone(struct zone *zone, unsigned int order) 2002 { 2003 return deferred_grow_zone(zone, order); 2004 } 2005 2006 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 2007 2008 void __init page_alloc_init_late(void) 2009 { 2010 struct zone *zone; 2011 int nid; 2012 2013 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 2014 2015 /* There will be num_node_state(N_MEMORY) threads */ 2016 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 2017 for_each_node_state(nid, N_MEMORY) { 2018 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 2019 } 2020 2021 /* Block until all are initialised */ 2022 wait_for_completion(&pgdat_init_all_done_comp); 2023 2024 /* 2025 * The number of managed pages has changed due to the initialisation 2026 * so the pcpu batch and high limits needs to be updated or the limits 2027 * will be artificially small. 2028 */ 2029 for_each_populated_zone(zone) 2030 zone_pcp_update(zone); 2031 2032 /* 2033 * We initialized the rest of the deferred pages. Permanently disable 2034 * on-demand struct page initialization. 2035 */ 2036 static_branch_disable(&deferred_pages); 2037 2038 /* Reinit limits that are based on free pages after the kernel is up */ 2039 files_maxfiles_init(); 2040 #endif 2041 2042 /* Discard memblock private memory */ 2043 memblock_discard(); 2044 2045 for_each_node_state(nid, N_MEMORY) 2046 shuffle_free_memory(NODE_DATA(nid)); 2047 2048 for_each_populated_zone(zone) 2049 set_zone_contiguous(zone); 2050 } 2051 2052 #ifdef CONFIG_CMA 2053 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 2054 void __init init_cma_reserved_pageblock(struct page *page) 2055 { 2056 unsigned i = pageblock_nr_pages; 2057 struct page *p = page; 2058 2059 do { 2060 __ClearPageReserved(p); 2061 set_page_count(p, 0); 2062 } while (++p, --i); 2063 2064 set_pageblock_migratetype(page, MIGRATE_CMA); 2065 2066 if (pageblock_order >= MAX_ORDER) { 2067 i = pageblock_nr_pages; 2068 p = page; 2069 do { 2070 set_page_refcounted(p); 2071 __free_pages(p, MAX_ORDER - 1); 2072 p += MAX_ORDER_NR_PAGES; 2073 } while (i -= MAX_ORDER_NR_PAGES); 2074 } else { 2075 set_page_refcounted(page); 2076 __free_pages(page, pageblock_order); 2077 } 2078 2079 adjust_managed_page_count(page, pageblock_nr_pages); 2080 } 2081 #endif 2082 2083 /* 2084 * The order of subdivision here is critical for the IO subsystem. 2085 * Please do not alter this order without good reasons and regression 2086 * testing. Specifically, as large blocks of memory are subdivided, 2087 * the order in which smaller blocks are delivered depends on the order 2088 * they're subdivided in this function. This is the primary factor 2089 * influencing the order in which pages are delivered to the IO 2090 * subsystem according to empirical testing, and this is also justified 2091 * by considering the behavior of a buddy system containing a single 2092 * large block of memory acted on by a series of small allocations. 2093 * This behavior is a critical factor in sglist merging's success. 2094 * 2095 * -- nyc 2096 */ 2097 static inline void expand(struct zone *zone, struct page *page, 2098 int low, int high, int migratetype) 2099 { 2100 unsigned long size = 1 << high; 2101 2102 while (high > low) { 2103 high--; 2104 size >>= 1; 2105 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 2106 2107 /* 2108 * Mark as guard pages (or page), that will allow to 2109 * merge back to allocator when buddy will be freed. 2110 * Corresponding page table entries will not be touched, 2111 * pages will stay not present in virtual address space 2112 */ 2113 if (set_page_guard(zone, &page[size], high, migratetype)) 2114 continue; 2115 2116 add_to_free_list(&page[size], zone, high, migratetype); 2117 set_page_order(&page[size], high); 2118 } 2119 } 2120 2121 static void check_new_page_bad(struct page *page) 2122 { 2123 if (unlikely(page->flags & __PG_HWPOISON)) { 2124 /* Don't complain about hwpoisoned pages */ 2125 page_mapcount_reset(page); /* remove PageBuddy */ 2126 return; 2127 } 2128 2129 bad_page(page, 2130 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP)); 2131 } 2132 2133 /* 2134 * This page is about to be returned from the page allocator 2135 */ 2136 static inline int check_new_page(struct page *page) 2137 { 2138 if (likely(page_expected_state(page, 2139 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 2140 return 0; 2141 2142 check_new_page_bad(page); 2143 return 1; 2144 } 2145 2146 static inline bool free_pages_prezeroed(void) 2147 { 2148 return (IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) && 2149 page_poisoning_enabled()) || want_init_on_free(); 2150 } 2151 2152 #ifdef CONFIG_DEBUG_VM 2153 /* 2154 * With DEBUG_VM enabled, order-0 pages are checked for expected state when 2155 * being allocated from pcp lists. With debug_pagealloc also enabled, they are 2156 * also checked when pcp lists are refilled from the free lists. 2157 */ 2158 static inline bool check_pcp_refill(struct page *page) 2159 { 2160 if (debug_pagealloc_enabled_static()) 2161 return check_new_page(page); 2162 else 2163 return false; 2164 } 2165 2166 static inline bool check_new_pcp(struct page *page) 2167 { 2168 return check_new_page(page); 2169 } 2170 #else 2171 /* 2172 * With DEBUG_VM disabled, free order-0 pages are checked for expected state 2173 * when pcp lists are being refilled from the free lists. With debug_pagealloc 2174 * enabled, they are also checked when being allocated from the pcp lists. 2175 */ 2176 static inline bool check_pcp_refill(struct page *page) 2177 { 2178 return check_new_page(page); 2179 } 2180 static inline bool check_new_pcp(struct page *page) 2181 { 2182 if (debug_pagealloc_enabled_static()) 2183 return check_new_page(page); 2184 else 2185 return false; 2186 } 2187 #endif /* CONFIG_DEBUG_VM */ 2188 2189 static bool check_new_pages(struct page *page, unsigned int order) 2190 { 2191 int i; 2192 for (i = 0; i < (1 << order); i++) { 2193 struct page *p = page + i; 2194 2195 if (unlikely(check_new_page(p))) 2196 return true; 2197 } 2198 2199 return false; 2200 } 2201 2202 inline void post_alloc_hook(struct page *page, unsigned int order, 2203 gfp_t gfp_flags) 2204 { 2205 set_page_private(page, 0); 2206 set_page_refcounted(page); 2207 2208 arch_alloc_page(page, order); 2209 if (debug_pagealloc_enabled_static()) 2210 kernel_map_pages(page, 1 << order, 1); 2211 kasan_alloc_pages(page, order); 2212 kernel_poison_pages(page, 1 << order, 1); 2213 set_page_owner(page, order, gfp_flags); 2214 } 2215 2216 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 2217 unsigned int alloc_flags) 2218 { 2219 post_alloc_hook(page, order, gfp_flags); 2220 2221 if (!free_pages_prezeroed() && want_init_on_alloc(gfp_flags)) 2222 kernel_init_free_pages(page, 1 << order); 2223 2224 if (order && (gfp_flags & __GFP_COMP)) 2225 prep_compound_page(page, order); 2226 2227 /* 2228 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 2229 * allocate the page. The expectation is that the caller is taking 2230 * steps that will free more memory. The caller should avoid the page 2231 * being used for !PFMEMALLOC purposes. 2232 */ 2233 if (alloc_flags & ALLOC_NO_WATERMARKS) 2234 set_page_pfmemalloc(page); 2235 else 2236 clear_page_pfmemalloc(page); 2237 } 2238 2239 /* 2240 * Go through the free lists for the given migratetype and remove 2241 * the smallest available page from the freelists 2242 */ 2243 static __always_inline 2244 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 2245 int migratetype) 2246 { 2247 unsigned int current_order; 2248 struct free_area *area; 2249 struct page *page; 2250 2251 /* Find a page of the appropriate size in the preferred list */ 2252 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 2253 area = &(zone->free_area[current_order]); 2254 page = get_page_from_free_area(area, migratetype); 2255 if (!page) 2256 continue; 2257 del_page_from_free_list(page, zone, current_order); 2258 expand(zone, page, order, current_order, migratetype); 2259 set_pcppage_migratetype(page, migratetype); 2260 return page; 2261 } 2262 2263 return NULL; 2264 } 2265 2266 2267 /* 2268 * This array describes the order lists are fallen back to when 2269 * the free lists for the desirable migrate type are depleted 2270 */ 2271 static int fallbacks[MIGRATE_TYPES][3] = { 2272 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2273 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 2274 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2275 #ifdef CONFIG_CMA 2276 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */ 2277 #endif 2278 #ifdef CONFIG_MEMORY_ISOLATION 2279 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */ 2280 #endif 2281 }; 2282 2283 #ifdef CONFIG_CMA 2284 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2285 unsigned int order) 2286 { 2287 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 2288 } 2289 #else 2290 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2291 unsigned int order) { return NULL; } 2292 #endif 2293 2294 /* 2295 * Move the free pages in a range to the free lists of the requested type. 2296 * Note that start_page and end_pages are not aligned on a pageblock 2297 * boundary. If alignment is required, use move_freepages_block() 2298 */ 2299 static int move_freepages(struct zone *zone, 2300 struct page *start_page, struct page *end_page, 2301 int migratetype, int *num_movable) 2302 { 2303 struct page *page; 2304 unsigned int order; 2305 int pages_moved = 0; 2306 2307 for (page = start_page; page <= end_page;) { 2308 if (!pfn_valid_within(page_to_pfn(page))) { 2309 page++; 2310 continue; 2311 } 2312 2313 if (!PageBuddy(page)) { 2314 /* 2315 * We assume that pages that could be isolated for 2316 * migration are movable. But we don't actually try 2317 * isolating, as that would be expensive. 2318 */ 2319 if (num_movable && 2320 (PageLRU(page) || __PageMovable(page))) 2321 (*num_movable)++; 2322 2323 page++; 2324 continue; 2325 } 2326 2327 /* Make sure we are not inadvertently changing nodes */ 2328 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2329 VM_BUG_ON_PAGE(page_zone(page) != zone, page); 2330 2331 order = page_order(page); 2332 move_to_free_list(page, zone, order, migratetype); 2333 page += 1 << order; 2334 pages_moved += 1 << order; 2335 } 2336 2337 return pages_moved; 2338 } 2339 2340 int move_freepages_block(struct zone *zone, struct page *page, 2341 int migratetype, int *num_movable) 2342 { 2343 unsigned long start_pfn, end_pfn; 2344 struct page *start_page, *end_page; 2345 2346 if (num_movable) 2347 *num_movable = 0; 2348 2349 start_pfn = page_to_pfn(page); 2350 start_pfn = start_pfn & ~(pageblock_nr_pages-1); 2351 start_page = pfn_to_page(start_pfn); 2352 end_page = start_page + pageblock_nr_pages - 1; 2353 end_pfn = start_pfn + pageblock_nr_pages - 1; 2354 2355 /* Do not cross zone boundaries */ 2356 if (!zone_spans_pfn(zone, start_pfn)) 2357 start_page = page; 2358 if (!zone_spans_pfn(zone, end_pfn)) 2359 return 0; 2360 2361 return move_freepages(zone, start_page, end_page, migratetype, 2362 num_movable); 2363 } 2364 2365 static void change_pageblock_range(struct page *pageblock_page, 2366 int start_order, int migratetype) 2367 { 2368 int nr_pageblocks = 1 << (start_order - pageblock_order); 2369 2370 while (nr_pageblocks--) { 2371 set_pageblock_migratetype(pageblock_page, migratetype); 2372 pageblock_page += pageblock_nr_pages; 2373 } 2374 } 2375 2376 /* 2377 * When we are falling back to another migratetype during allocation, try to 2378 * steal extra free pages from the same pageblocks to satisfy further 2379 * allocations, instead of polluting multiple pageblocks. 2380 * 2381 * If we are stealing a relatively large buddy page, it is likely there will 2382 * be more free pages in the pageblock, so try to steal them all. For 2383 * reclaimable and unmovable allocations, we steal regardless of page size, 2384 * as fragmentation caused by those allocations polluting movable pageblocks 2385 * is worse than movable allocations stealing from unmovable and reclaimable 2386 * pageblocks. 2387 */ 2388 static bool can_steal_fallback(unsigned int order, int start_mt) 2389 { 2390 /* 2391 * Leaving this order check is intended, although there is 2392 * relaxed order check in next check. The reason is that 2393 * we can actually steal whole pageblock if this condition met, 2394 * but, below check doesn't guarantee it and that is just heuristic 2395 * so could be changed anytime. 2396 */ 2397 if (order >= pageblock_order) 2398 return true; 2399 2400 if (order >= pageblock_order / 2 || 2401 start_mt == MIGRATE_RECLAIMABLE || 2402 start_mt == MIGRATE_UNMOVABLE || 2403 page_group_by_mobility_disabled) 2404 return true; 2405 2406 return false; 2407 } 2408 2409 static inline void boost_watermark(struct zone *zone) 2410 { 2411 unsigned long max_boost; 2412 2413 if (!watermark_boost_factor) 2414 return; 2415 /* 2416 * Don't bother in zones that are unlikely to produce results. 2417 * On small machines, including kdump capture kernels running 2418 * in a small area, boosting the watermark can cause an out of 2419 * memory situation immediately. 2420 */ 2421 if ((pageblock_nr_pages * 4) > zone_managed_pages(zone)) 2422 return; 2423 2424 max_boost = mult_frac(zone->_watermark[WMARK_HIGH], 2425 watermark_boost_factor, 10000); 2426 2427 /* 2428 * high watermark may be uninitialised if fragmentation occurs 2429 * very early in boot so do not boost. We do not fall 2430 * through and boost by pageblock_nr_pages as failing 2431 * allocations that early means that reclaim is not going 2432 * to help and it may even be impossible to reclaim the 2433 * boosted watermark resulting in a hang. 2434 */ 2435 if (!max_boost) 2436 return; 2437 2438 max_boost = max(pageblock_nr_pages, max_boost); 2439 2440 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages, 2441 max_boost); 2442 } 2443 2444 /* 2445 * This function implements actual steal behaviour. If order is large enough, 2446 * we can steal whole pageblock. If not, we first move freepages in this 2447 * pageblock to our migratetype and determine how many already-allocated pages 2448 * are there in the pageblock with a compatible migratetype. If at least half 2449 * of pages are free or compatible, we can change migratetype of the pageblock 2450 * itself, so pages freed in the future will be put on the correct free list. 2451 */ 2452 static void steal_suitable_fallback(struct zone *zone, struct page *page, 2453 unsigned int alloc_flags, int start_type, bool whole_block) 2454 { 2455 unsigned int current_order = page_order(page); 2456 int free_pages, movable_pages, alike_pages; 2457 int old_block_type; 2458 2459 old_block_type = get_pageblock_migratetype(page); 2460 2461 /* 2462 * This can happen due to races and we want to prevent broken 2463 * highatomic accounting. 2464 */ 2465 if (is_migrate_highatomic(old_block_type)) 2466 goto single_page; 2467 2468 /* Take ownership for orders >= pageblock_order */ 2469 if (current_order >= pageblock_order) { 2470 change_pageblock_range(page, current_order, start_type); 2471 goto single_page; 2472 } 2473 2474 /* 2475 * Boost watermarks to increase reclaim pressure to reduce the 2476 * likelihood of future fallbacks. Wake kswapd now as the node 2477 * may be balanced overall and kswapd will not wake naturally. 2478 */ 2479 boost_watermark(zone); 2480 if (alloc_flags & ALLOC_KSWAPD) 2481 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 2482 2483 /* We are not allowed to try stealing from the whole block */ 2484 if (!whole_block) 2485 goto single_page; 2486 2487 free_pages = move_freepages_block(zone, page, start_type, 2488 &movable_pages); 2489 /* 2490 * Determine how many pages are compatible with our allocation. 2491 * For movable allocation, it's the number of movable pages which 2492 * we just obtained. For other types it's a bit more tricky. 2493 */ 2494 if (start_type == MIGRATE_MOVABLE) { 2495 alike_pages = movable_pages; 2496 } else { 2497 /* 2498 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation 2499 * to MOVABLE pageblock, consider all non-movable pages as 2500 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or 2501 * vice versa, be conservative since we can't distinguish the 2502 * exact migratetype of non-movable pages. 2503 */ 2504 if (old_block_type == MIGRATE_MOVABLE) 2505 alike_pages = pageblock_nr_pages 2506 - (free_pages + movable_pages); 2507 else 2508 alike_pages = 0; 2509 } 2510 2511 /* moving whole block can fail due to zone boundary conditions */ 2512 if (!free_pages) 2513 goto single_page; 2514 2515 /* 2516 * If a sufficient number of pages in the block are either free or of 2517 * comparable migratability as our allocation, claim the whole block. 2518 */ 2519 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || 2520 page_group_by_mobility_disabled) 2521 set_pageblock_migratetype(page, start_type); 2522 2523 return; 2524 2525 single_page: 2526 move_to_free_list(page, zone, current_order, start_type); 2527 } 2528 2529 /* 2530 * Check whether there is a suitable fallback freepage with requested order. 2531 * If only_stealable is true, this function returns fallback_mt only if 2532 * we can steal other freepages all together. This would help to reduce 2533 * fragmentation due to mixed migratetype pages in one pageblock. 2534 */ 2535 int find_suitable_fallback(struct free_area *area, unsigned int order, 2536 int migratetype, bool only_stealable, bool *can_steal) 2537 { 2538 int i; 2539 int fallback_mt; 2540 2541 if (area->nr_free == 0) 2542 return -1; 2543 2544 *can_steal = false; 2545 for (i = 0;; i++) { 2546 fallback_mt = fallbacks[migratetype][i]; 2547 if (fallback_mt == MIGRATE_TYPES) 2548 break; 2549 2550 if (free_area_empty(area, fallback_mt)) 2551 continue; 2552 2553 if (can_steal_fallback(order, migratetype)) 2554 *can_steal = true; 2555 2556 if (!only_stealable) 2557 return fallback_mt; 2558 2559 if (*can_steal) 2560 return fallback_mt; 2561 } 2562 2563 return -1; 2564 } 2565 2566 /* 2567 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2568 * there are no empty page blocks that contain a page with a suitable order 2569 */ 2570 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2571 unsigned int alloc_order) 2572 { 2573 int mt; 2574 unsigned long max_managed, flags; 2575 2576 /* 2577 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2578 * Check is race-prone but harmless. 2579 */ 2580 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages; 2581 if (zone->nr_reserved_highatomic >= max_managed) 2582 return; 2583 2584 spin_lock_irqsave(&zone->lock, flags); 2585 2586 /* Recheck the nr_reserved_highatomic limit under the lock */ 2587 if (zone->nr_reserved_highatomic >= max_managed) 2588 goto out_unlock; 2589 2590 /* Yoink! */ 2591 mt = get_pageblock_migratetype(page); 2592 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt) 2593 && !is_migrate_cma(mt)) { 2594 zone->nr_reserved_highatomic += pageblock_nr_pages; 2595 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2596 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); 2597 } 2598 2599 out_unlock: 2600 spin_unlock_irqrestore(&zone->lock, flags); 2601 } 2602 2603 /* 2604 * Used when an allocation is about to fail under memory pressure. This 2605 * potentially hurts the reliability of high-order allocations when under 2606 * intense memory pressure but failed atomic allocations should be easier 2607 * to recover from than an OOM. 2608 * 2609 * If @force is true, try to unreserve a pageblock even though highatomic 2610 * pageblock is exhausted. 2611 */ 2612 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 2613 bool force) 2614 { 2615 struct zonelist *zonelist = ac->zonelist; 2616 unsigned long flags; 2617 struct zoneref *z; 2618 struct zone *zone; 2619 struct page *page; 2620 int order; 2621 bool ret; 2622 2623 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx, 2624 ac->nodemask) { 2625 /* 2626 * Preserve at least one pageblock unless memory pressure 2627 * is really high. 2628 */ 2629 if (!force && zone->nr_reserved_highatomic <= 2630 pageblock_nr_pages) 2631 continue; 2632 2633 spin_lock_irqsave(&zone->lock, flags); 2634 for (order = 0; order < MAX_ORDER; order++) { 2635 struct free_area *area = &(zone->free_area[order]); 2636 2637 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); 2638 if (!page) 2639 continue; 2640 2641 /* 2642 * In page freeing path, migratetype change is racy so 2643 * we can counter several free pages in a pageblock 2644 * in this loop althoug we changed the pageblock type 2645 * from highatomic to ac->migratetype. So we should 2646 * adjust the count once. 2647 */ 2648 if (is_migrate_highatomic_page(page)) { 2649 /* 2650 * It should never happen but changes to 2651 * locking could inadvertently allow a per-cpu 2652 * drain to add pages to MIGRATE_HIGHATOMIC 2653 * while unreserving so be safe and watch for 2654 * underflows. 2655 */ 2656 zone->nr_reserved_highatomic -= min( 2657 pageblock_nr_pages, 2658 zone->nr_reserved_highatomic); 2659 } 2660 2661 /* 2662 * Convert to ac->migratetype and avoid the normal 2663 * pageblock stealing heuristics. Minimally, the caller 2664 * is doing the work and needs the pages. More 2665 * importantly, if the block was always converted to 2666 * MIGRATE_UNMOVABLE or another type then the number 2667 * of pageblocks that cannot be completely freed 2668 * may increase. 2669 */ 2670 set_pageblock_migratetype(page, ac->migratetype); 2671 ret = move_freepages_block(zone, page, ac->migratetype, 2672 NULL); 2673 if (ret) { 2674 spin_unlock_irqrestore(&zone->lock, flags); 2675 return ret; 2676 } 2677 } 2678 spin_unlock_irqrestore(&zone->lock, flags); 2679 } 2680 2681 return false; 2682 } 2683 2684 /* 2685 * Try finding a free buddy page on the fallback list and put it on the free 2686 * list of requested migratetype, possibly along with other pages from the same 2687 * block, depending on fragmentation avoidance heuristics. Returns true if 2688 * fallback was found so that __rmqueue_smallest() can grab it. 2689 * 2690 * The use of signed ints for order and current_order is a deliberate 2691 * deviation from the rest of this file, to make the for loop 2692 * condition simpler. 2693 */ 2694 static __always_inline bool 2695 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, 2696 unsigned int alloc_flags) 2697 { 2698 struct free_area *area; 2699 int current_order; 2700 int min_order = order; 2701 struct page *page; 2702 int fallback_mt; 2703 bool can_steal; 2704 2705 /* 2706 * Do not steal pages from freelists belonging to other pageblocks 2707 * i.e. orders < pageblock_order. If there are no local zones free, 2708 * the zonelists will be reiterated without ALLOC_NOFRAGMENT. 2709 */ 2710 if (alloc_flags & ALLOC_NOFRAGMENT) 2711 min_order = pageblock_order; 2712 2713 /* 2714 * Find the largest available free page in the other list. This roughly 2715 * approximates finding the pageblock with the most free pages, which 2716 * would be too costly to do exactly. 2717 */ 2718 for (current_order = MAX_ORDER - 1; current_order >= min_order; 2719 --current_order) { 2720 area = &(zone->free_area[current_order]); 2721 fallback_mt = find_suitable_fallback(area, current_order, 2722 start_migratetype, false, &can_steal); 2723 if (fallback_mt == -1) 2724 continue; 2725 2726 /* 2727 * We cannot steal all free pages from the pageblock and the 2728 * requested migratetype is movable. In that case it's better to 2729 * steal and split the smallest available page instead of the 2730 * largest available page, because even if the next movable 2731 * allocation falls back into a different pageblock than this 2732 * one, it won't cause permanent fragmentation. 2733 */ 2734 if (!can_steal && start_migratetype == MIGRATE_MOVABLE 2735 && current_order > order) 2736 goto find_smallest; 2737 2738 goto do_steal; 2739 } 2740 2741 return false; 2742 2743 find_smallest: 2744 for (current_order = order; current_order < MAX_ORDER; 2745 current_order++) { 2746 area = &(zone->free_area[current_order]); 2747 fallback_mt = find_suitable_fallback(area, current_order, 2748 start_migratetype, false, &can_steal); 2749 if (fallback_mt != -1) 2750 break; 2751 } 2752 2753 /* 2754 * This should not happen - we already found a suitable fallback 2755 * when looking for the largest page. 2756 */ 2757 VM_BUG_ON(current_order == MAX_ORDER); 2758 2759 do_steal: 2760 page = get_page_from_free_area(area, fallback_mt); 2761 2762 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype, 2763 can_steal); 2764 2765 trace_mm_page_alloc_extfrag(page, order, current_order, 2766 start_migratetype, fallback_mt); 2767 2768 return true; 2769 2770 } 2771 2772 /* 2773 * Do the hard work of removing an element from the buddy allocator. 2774 * Call me with the zone->lock already held. 2775 */ 2776 static __always_inline struct page * 2777 __rmqueue(struct zone *zone, unsigned int order, int migratetype, 2778 unsigned int alloc_flags) 2779 { 2780 struct page *page; 2781 2782 #ifdef CONFIG_CMA 2783 /* 2784 * Balance movable allocations between regular and CMA areas by 2785 * allocating from CMA when over half of the zone's free memory 2786 * is in the CMA area. 2787 */ 2788 if (alloc_flags & ALLOC_CMA && 2789 zone_page_state(zone, NR_FREE_CMA_PAGES) > 2790 zone_page_state(zone, NR_FREE_PAGES) / 2) { 2791 page = __rmqueue_cma_fallback(zone, order); 2792 if (page) 2793 return page; 2794 } 2795 #endif 2796 retry: 2797 page = __rmqueue_smallest(zone, order, migratetype); 2798 if (unlikely(!page)) { 2799 if (alloc_flags & ALLOC_CMA) 2800 page = __rmqueue_cma_fallback(zone, order); 2801 2802 if (!page && __rmqueue_fallback(zone, order, migratetype, 2803 alloc_flags)) 2804 goto retry; 2805 } 2806 2807 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2808 return page; 2809 } 2810 2811 /* 2812 * Obtain a specified number of elements from the buddy allocator, all under 2813 * a single hold of the lock, for efficiency. Add them to the supplied list. 2814 * Returns the number of new pages which were placed at *list. 2815 */ 2816 static int rmqueue_bulk(struct zone *zone, unsigned int order, 2817 unsigned long count, struct list_head *list, 2818 int migratetype, unsigned int alloc_flags) 2819 { 2820 int i, alloced = 0; 2821 2822 spin_lock(&zone->lock); 2823 for (i = 0; i < count; ++i) { 2824 struct page *page = __rmqueue(zone, order, migratetype, 2825 alloc_flags); 2826 if (unlikely(page == NULL)) 2827 break; 2828 2829 if (unlikely(check_pcp_refill(page))) 2830 continue; 2831 2832 /* 2833 * Split buddy pages returned by expand() are received here in 2834 * physical page order. The page is added to the tail of 2835 * caller's list. From the callers perspective, the linked list 2836 * is ordered by page number under some conditions. This is 2837 * useful for IO devices that can forward direction from the 2838 * head, thus also in the physical page order. This is useful 2839 * for IO devices that can merge IO requests if the physical 2840 * pages are ordered properly. 2841 */ 2842 list_add_tail(&page->lru, list); 2843 alloced++; 2844 if (is_migrate_cma(get_pcppage_migratetype(page))) 2845 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 2846 -(1 << order)); 2847 } 2848 2849 /* 2850 * i pages were removed from the buddy list even if some leak due 2851 * to check_pcp_refill failing so adjust NR_FREE_PAGES based 2852 * on i. Do not confuse with 'alloced' which is the number of 2853 * pages added to the pcp list. 2854 */ 2855 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 2856 spin_unlock(&zone->lock); 2857 return alloced; 2858 } 2859 2860 #ifdef CONFIG_NUMA 2861 /* 2862 * Called from the vmstat counter updater to drain pagesets of this 2863 * currently executing processor on remote nodes after they have 2864 * expired. 2865 * 2866 * Note that this function must be called with the thread pinned to 2867 * a single processor. 2868 */ 2869 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 2870 { 2871 unsigned long flags; 2872 int to_drain, batch; 2873 2874 local_irq_save(flags); 2875 batch = READ_ONCE(pcp->batch); 2876 to_drain = min(pcp->count, batch); 2877 if (to_drain > 0) 2878 free_pcppages_bulk(zone, to_drain, pcp); 2879 local_irq_restore(flags); 2880 } 2881 #endif 2882 2883 /* 2884 * Drain pcplists of the indicated processor and zone. 2885 * 2886 * The processor must either be the current processor and the 2887 * thread pinned to the current processor or a processor that 2888 * is not online. 2889 */ 2890 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 2891 { 2892 unsigned long flags; 2893 struct per_cpu_pageset *pset; 2894 struct per_cpu_pages *pcp; 2895 2896 local_irq_save(flags); 2897 pset = per_cpu_ptr(zone->pageset, cpu); 2898 2899 pcp = &pset->pcp; 2900 if (pcp->count) 2901 free_pcppages_bulk(zone, pcp->count, pcp); 2902 local_irq_restore(flags); 2903 } 2904 2905 /* 2906 * Drain pcplists of all zones on the indicated processor. 2907 * 2908 * The processor must either be the current processor and the 2909 * thread pinned to the current processor or a processor that 2910 * is not online. 2911 */ 2912 static void drain_pages(unsigned int cpu) 2913 { 2914 struct zone *zone; 2915 2916 for_each_populated_zone(zone) { 2917 drain_pages_zone(cpu, zone); 2918 } 2919 } 2920 2921 /* 2922 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 2923 * 2924 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 2925 * the single zone's pages. 2926 */ 2927 void drain_local_pages(struct zone *zone) 2928 { 2929 int cpu = smp_processor_id(); 2930 2931 if (zone) 2932 drain_pages_zone(cpu, zone); 2933 else 2934 drain_pages(cpu); 2935 } 2936 2937 static void drain_local_pages_wq(struct work_struct *work) 2938 { 2939 struct pcpu_drain *drain; 2940 2941 drain = container_of(work, struct pcpu_drain, work); 2942 2943 /* 2944 * drain_all_pages doesn't use proper cpu hotplug protection so 2945 * we can race with cpu offline when the WQ can move this from 2946 * a cpu pinned worker to an unbound one. We can operate on a different 2947 * cpu which is allright but we also have to make sure to not move to 2948 * a different one. 2949 */ 2950 preempt_disable(); 2951 drain_local_pages(drain->zone); 2952 preempt_enable(); 2953 } 2954 2955 /* 2956 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 2957 * 2958 * When zone parameter is non-NULL, spill just the single zone's pages. 2959 * 2960 * Note that this can be extremely slow as the draining happens in a workqueue. 2961 */ 2962 void drain_all_pages(struct zone *zone) 2963 { 2964 int cpu; 2965 2966 /* 2967 * Allocate in the BSS so we wont require allocation in 2968 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 2969 */ 2970 static cpumask_t cpus_with_pcps; 2971 2972 /* 2973 * Make sure nobody triggers this path before mm_percpu_wq is fully 2974 * initialized. 2975 */ 2976 if (WARN_ON_ONCE(!mm_percpu_wq)) 2977 return; 2978 2979 /* 2980 * Do not drain if one is already in progress unless it's specific to 2981 * a zone. Such callers are primarily CMA and memory hotplug and need 2982 * the drain to be complete when the call returns. 2983 */ 2984 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) { 2985 if (!zone) 2986 return; 2987 mutex_lock(&pcpu_drain_mutex); 2988 } 2989 2990 /* 2991 * We don't care about racing with CPU hotplug event 2992 * as offline notification will cause the notified 2993 * cpu to drain that CPU pcps and on_each_cpu_mask 2994 * disables preemption as part of its processing 2995 */ 2996 for_each_online_cpu(cpu) { 2997 struct per_cpu_pageset *pcp; 2998 struct zone *z; 2999 bool has_pcps = false; 3000 3001 if (zone) { 3002 pcp = per_cpu_ptr(zone->pageset, cpu); 3003 if (pcp->pcp.count) 3004 has_pcps = true; 3005 } else { 3006 for_each_populated_zone(z) { 3007 pcp = per_cpu_ptr(z->pageset, cpu); 3008 if (pcp->pcp.count) { 3009 has_pcps = true; 3010 break; 3011 } 3012 } 3013 } 3014 3015 if (has_pcps) 3016 cpumask_set_cpu(cpu, &cpus_with_pcps); 3017 else 3018 cpumask_clear_cpu(cpu, &cpus_with_pcps); 3019 } 3020 3021 for_each_cpu(cpu, &cpus_with_pcps) { 3022 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu); 3023 3024 drain->zone = zone; 3025 INIT_WORK(&drain->work, drain_local_pages_wq); 3026 queue_work_on(cpu, mm_percpu_wq, &drain->work); 3027 } 3028 for_each_cpu(cpu, &cpus_with_pcps) 3029 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work); 3030 3031 mutex_unlock(&pcpu_drain_mutex); 3032 } 3033 3034 #ifdef CONFIG_HIBERNATION 3035 3036 /* 3037 * Touch the watchdog for every WD_PAGE_COUNT pages. 3038 */ 3039 #define WD_PAGE_COUNT (128*1024) 3040 3041 void mark_free_pages(struct zone *zone) 3042 { 3043 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT; 3044 unsigned long flags; 3045 unsigned int order, t; 3046 struct page *page; 3047 3048 if (zone_is_empty(zone)) 3049 return; 3050 3051 spin_lock_irqsave(&zone->lock, flags); 3052 3053 max_zone_pfn = zone_end_pfn(zone); 3054 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 3055 if (pfn_valid(pfn)) { 3056 page = pfn_to_page(pfn); 3057 3058 if (!--page_count) { 3059 touch_nmi_watchdog(); 3060 page_count = WD_PAGE_COUNT; 3061 } 3062 3063 if (page_zone(page) != zone) 3064 continue; 3065 3066 if (!swsusp_page_is_forbidden(page)) 3067 swsusp_unset_page_free(page); 3068 } 3069 3070 for_each_migratetype_order(order, t) { 3071 list_for_each_entry(page, 3072 &zone->free_area[order].free_list[t], lru) { 3073 unsigned long i; 3074 3075 pfn = page_to_pfn(page); 3076 for (i = 0; i < (1UL << order); i++) { 3077 if (!--page_count) { 3078 touch_nmi_watchdog(); 3079 page_count = WD_PAGE_COUNT; 3080 } 3081 swsusp_set_page_free(pfn_to_page(pfn + i)); 3082 } 3083 } 3084 } 3085 spin_unlock_irqrestore(&zone->lock, flags); 3086 } 3087 #endif /* CONFIG_PM */ 3088 3089 static bool free_unref_page_prepare(struct page *page, unsigned long pfn) 3090 { 3091 int migratetype; 3092 3093 if (!free_pcp_prepare(page)) 3094 return false; 3095 3096 migratetype = get_pfnblock_migratetype(page, pfn); 3097 set_pcppage_migratetype(page, migratetype); 3098 return true; 3099 } 3100 3101 static void free_unref_page_commit(struct page *page, unsigned long pfn) 3102 { 3103 struct zone *zone = page_zone(page); 3104 struct per_cpu_pages *pcp; 3105 int migratetype; 3106 3107 migratetype = get_pcppage_migratetype(page); 3108 __count_vm_event(PGFREE); 3109 3110 /* 3111 * We only track unmovable, reclaimable and movable on pcp lists. 3112 * Free ISOLATE pages back to the allocator because they are being 3113 * offlined but treat HIGHATOMIC as movable pages so we can get those 3114 * areas back if necessary. Otherwise, we may have to free 3115 * excessively into the page allocator 3116 */ 3117 if (migratetype >= MIGRATE_PCPTYPES) { 3118 if (unlikely(is_migrate_isolate(migratetype))) { 3119 free_one_page(zone, page, pfn, 0, migratetype); 3120 return; 3121 } 3122 migratetype = MIGRATE_MOVABLE; 3123 } 3124 3125 pcp = &this_cpu_ptr(zone->pageset)->pcp; 3126 list_add(&page->lru, &pcp->lists[migratetype]); 3127 pcp->count++; 3128 if (pcp->count >= pcp->high) { 3129 unsigned long batch = READ_ONCE(pcp->batch); 3130 free_pcppages_bulk(zone, batch, pcp); 3131 } 3132 } 3133 3134 /* 3135 * Free a 0-order page 3136 */ 3137 void free_unref_page(struct page *page) 3138 { 3139 unsigned long flags; 3140 unsigned long pfn = page_to_pfn(page); 3141 3142 if (!free_unref_page_prepare(page, pfn)) 3143 return; 3144 3145 local_irq_save(flags); 3146 free_unref_page_commit(page, pfn); 3147 local_irq_restore(flags); 3148 } 3149 3150 /* 3151 * Free a list of 0-order pages 3152 */ 3153 void free_unref_page_list(struct list_head *list) 3154 { 3155 struct page *page, *next; 3156 unsigned long flags, pfn; 3157 int batch_count = 0; 3158 3159 /* Prepare pages for freeing */ 3160 list_for_each_entry_safe(page, next, list, lru) { 3161 pfn = page_to_pfn(page); 3162 if (!free_unref_page_prepare(page, pfn)) 3163 list_del(&page->lru); 3164 set_page_private(page, pfn); 3165 } 3166 3167 local_irq_save(flags); 3168 list_for_each_entry_safe(page, next, list, lru) { 3169 unsigned long pfn = page_private(page); 3170 3171 set_page_private(page, 0); 3172 trace_mm_page_free_batched(page); 3173 free_unref_page_commit(page, pfn); 3174 3175 /* 3176 * Guard against excessive IRQ disabled times when we get 3177 * a large list of pages to free. 3178 */ 3179 if (++batch_count == SWAP_CLUSTER_MAX) { 3180 local_irq_restore(flags); 3181 batch_count = 0; 3182 local_irq_save(flags); 3183 } 3184 } 3185 local_irq_restore(flags); 3186 } 3187 3188 /* 3189 * split_page takes a non-compound higher-order page, and splits it into 3190 * n (1<<order) sub-pages: page[0..n] 3191 * Each sub-page must be freed individually. 3192 * 3193 * Note: this is probably too low level an operation for use in drivers. 3194 * Please consult with lkml before using this in your driver. 3195 */ 3196 void split_page(struct page *page, unsigned int order) 3197 { 3198 int i; 3199 3200 VM_BUG_ON_PAGE(PageCompound(page), page); 3201 VM_BUG_ON_PAGE(!page_count(page), page); 3202 3203 for (i = 1; i < (1 << order); i++) 3204 set_page_refcounted(page + i); 3205 split_page_owner(page, order); 3206 } 3207 EXPORT_SYMBOL_GPL(split_page); 3208 3209 int __isolate_free_page(struct page *page, unsigned int order) 3210 { 3211 unsigned long watermark; 3212 struct zone *zone; 3213 int mt; 3214 3215 BUG_ON(!PageBuddy(page)); 3216 3217 zone = page_zone(page); 3218 mt = get_pageblock_migratetype(page); 3219 3220 if (!is_migrate_isolate(mt)) { 3221 /* 3222 * Obey watermarks as if the page was being allocated. We can 3223 * emulate a high-order watermark check with a raised order-0 3224 * watermark, because we already know our high-order page 3225 * exists. 3226 */ 3227 watermark = zone->_watermark[WMARK_MIN] + (1UL << order); 3228 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 3229 return 0; 3230 3231 __mod_zone_freepage_state(zone, -(1UL << order), mt); 3232 } 3233 3234 /* Remove page from free list */ 3235 3236 del_page_from_free_list(page, zone, order); 3237 3238 /* 3239 * Set the pageblock if the isolated page is at least half of a 3240 * pageblock 3241 */ 3242 if (order >= pageblock_order - 1) { 3243 struct page *endpage = page + (1 << order) - 1; 3244 for (; page < endpage; page += pageblock_nr_pages) { 3245 int mt = get_pageblock_migratetype(page); 3246 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt) 3247 && !is_migrate_highatomic(mt)) 3248 set_pageblock_migratetype(page, 3249 MIGRATE_MOVABLE); 3250 } 3251 } 3252 3253 3254 return 1UL << order; 3255 } 3256 3257 /** 3258 * __putback_isolated_page - Return a now-isolated page back where we got it 3259 * @page: Page that was isolated 3260 * @order: Order of the isolated page 3261 * @mt: The page's pageblock's migratetype 3262 * 3263 * This function is meant to return a page pulled from the free lists via 3264 * __isolate_free_page back to the free lists they were pulled from. 3265 */ 3266 void __putback_isolated_page(struct page *page, unsigned int order, int mt) 3267 { 3268 struct zone *zone = page_zone(page); 3269 3270 /* zone lock should be held when this function is called */ 3271 lockdep_assert_held(&zone->lock); 3272 3273 /* Return isolated page to tail of freelist. */ 3274 __free_one_page(page, page_to_pfn(page), zone, order, mt, false); 3275 } 3276 3277 /* 3278 * Update NUMA hit/miss statistics 3279 * 3280 * Must be called with interrupts disabled. 3281 */ 3282 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) 3283 { 3284 #ifdef CONFIG_NUMA 3285 enum numa_stat_item local_stat = NUMA_LOCAL; 3286 3287 /* skip numa counters update if numa stats is disabled */ 3288 if (!static_branch_likely(&vm_numa_stat_key)) 3289 return; 3290 3291 if (zone_to_nid(z) != numa_node_id()) 3292 local_stat = NUMA_OTHER; 3293 3294 if (zone_to_nid(z) == zone_to_nid(preferred_zone)) 3295 __inc_numa_state(z, NUMA_HIT); 3296 else { 3297 __inc_numa_state(z, NUMA_MISS); 3298 __inc_numa_state(preferred_zone, NUMA_FOREIGN); 3299 } 3300 __inc_numa_state(z, local_stat); 3301 #endif 3302 } 3303 3304 /* Remove page from the per-cpu list, caller must protect the list */ 3305 static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype, 3306 unsigned int alloc_flags, 3307 struct per_cpu_pages *pcp, 3308 struct list_head *list) 3309 { 3310 struct page *page; 3311 3312 do { 3313 if (list_empty(list)) { 3314 pcp->count += rmqueue_bulk(zone, 0, 3315 pcp->batch, list, 3316 migratetype, alloc_flags); 3317 if (unlikely(list_empty(list))) 3318 return NULL; 3319 } 3320 3321 page = list_first_entry(list, struct page, lru); 3322 list_del(&page->lru); 3323 pcp->count--; 3324 } while (check_new_pcp(page)); 3325 3326 return page; 3327 } 3328 3329 /* Lock and remove page from the per-cpu list */ 3330 static struct page *rmqueue_pcplist(struct zone *preferred_zone, 3331 struct zone *zone, gfp_t gfp_flags, 3332 int migratetype, unsigned int alloc_flags) 3333 { 3334 struct per_cpu_pages *pcp; 3335 struct list_head *list; 3336 struct page *page; 3337 unsigned long flags; 3338 3339 local_irq_save(flags); 3340 pcp = &this_cpu_ptr(zone->pageset)->pcp; 3341 list = &pcp->lists[migratetype]; 3342 page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list); 3343 if (page) { 3344 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); 3345 zone_statistics(preferred_zone, zone); 3346 } 3347 local_irq_restore(flags); 3348 return page; 3349 } 3350 3351 /* 3352 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 3353 */ 3354 static inline 3355 struct page *rmqueue(struct zone *preferred_zone, 3356 struct zone *zone, unsigned int order, 3357 gfp_t gfp_flags, unsigned int alloc_flags, 3358 int migratetype) 3359 { 3360 unsigned long flags; 3361 struct page *page; 3362 3363 if (likely(order == 0)) { 3364 page = rmqueue_pcplist(preferred_zone, zone, gfp_flags, 3365 migratetype, alloc_flags); 3366 goto out; 3367 } 3368 3369 /* 3370 * We most definitely don't want callers attempting to 3371 * allocate greater than order-1 page units with __GFP_NOFAIL. 3372 */ 3373 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 3374 spin_lock_irqsave(&zone->lock, flags); 3375 3376 do { 3377 page = NULL; 3378 if (alloc_flags & ALLOC_HARDER) { 3379 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3380 if (page) 3381 trace_mm_page_alloc_zone_locked(page, order, migratetype); 3382 } 3383 if (!page) 3384 page = __rmqueue(zone, order, migratetype, alloc_flags); 3385 } while (page && check_new_pages(page, order)); 3386 spin_unlock(&zone->lock); 3387 if (!page) 3388 goto failed; 3389 __mod_zone_freepage_state(zone, -(1 << order), 3390 get_pcppage_migratetype(page)); 3391 3392 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3393 zone_statistics(preferred_zone, zone); 3394 local_irq_restore(flags); 3395 3396 out: 3397 /* Separate test+clear to avoid unnecessary atomics */ 3398 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) { 3399 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 3400 wakeup_kswapd(zone, 0, 0, zone_idx(zone)); 3401 } 3402 3403 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); 3404 return page; 3405 3406 failed: 3407 local_irq_restore(flags); 3408 return NULL; 3409 } 3410 3411 #ifdef CONFIG_FAIL_PAGE_ALLOC 3412 3413 static struct { 3414 struct fault_attr attr; 3415 3416 bool ignore_gfp_highmem; 3417 bool ignore_gfp_reclaim; 3418 u32 min_order; 3419 } fail_page_alloc = { 3420 .attr = FAULT_ATTR_INITIALIZER, 3421 .ignore_gfp_reclaim = true, 3422 .ignore_gfp_highmem = true, 3423 .min_order = 1, 3424 }; 3425 3426 static int __init setup_fail_page_alloc(char *str) 3427 { 3428 return setup_fault_attr(&fail_page_alloc.attr, str); 3429 } 3430 __setup("fail_page_alloc=", setup_fail_page_alloc); 3431 3432 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3433 { 3434 if (order < fail_page_alloc.min_order) 3435 return false; 3436 if (gfp_mask & __GFP_NOFAIL) 3437 return false; 3438 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 3439 return false; 3440 if (fail_page_alloc.ignore_gfp_reclaim && 3441 (gfp_mask & __GFP_DIRECT_RECLAIM)) 3442 return false; 3443 3444 return should_fail(&fail_page_alloc.attr, 1 << order); 3445 } 3446 3447 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 3448 3449 static int __init fail_page_alloc_debugfs(void) 3450 { 3451 umode_t mode = S_IFREG | 0600; 3452 struct dentry *dir; 3453 3454 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 3455 &fail_page_alloc.attr); 3456 3457 debugfs_create_bool("ignore-gfp-wait", mode, dir, 3458 &fail_page_alloc.ignore_gfp_reclaim); 3459 debugfs_create_bool("ignore-gfp-highmem", mode, dir, 3460 &fail_page_alloc.ignore_gfp_highmem); 3461 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order); 3462 3463 return 0; 3464 } 3465 3466 late_initcall(fail_page_alloc_debugfs); 3467 3468 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 3469 3470 #else /* CONFIG_FAIL_PAGE_ALLOC */ 3471 3472 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3473 { 3474 return false; 3475 } 3476 3477 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 3478 3479 static noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3480 { 3481 return __should_fail_alloc_page(gfp_mask, order); 3482 } 3483 ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE); 3484 3485 static inline long __zone_watermark_unusable_free(struct zone *z, 3486 unsigned int order, unsigned int alloc_flags) 3487 { 3488 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3489 long unusable_free = (1 << order) - 1; 3490 3491 /* 3492 * If the caller does not have rights to ALLOC_HARDER then subtract 3493 * the high-atomic reserves. This will over-estimate the size of the 3494 * atomic reserve but it avoids a search. 3495 */ 3496 if (likely(!alloc_harder)) 3497 unusable_free += z->nr_reserved_highatomic; 3498 3499 #ifdef CONFIG_CMA 3500 /* If allocation can't use CMA areas don't use free CMA pages */ 3501 if (!(alloc_flags & ALLOC_CMA)) 3502 unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES); 3503 #endif 3504 3505 return unusable_free; 3506 } 3507 3508 /* 3509 * Return true if free base pages are above 'mark'. For high-order checks it 3510 * will return true of the order-0 watermark is reached and there is at least 3511 * one free page of a suitable size. Checking now avoids taking the zone lock 3512 * to check in the allocation paths if no pages are free. 3513 */ 3514 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3515 int highest_zoneidx, unsigned int alloc_flags, 3516 long free_pages) 3517 { 3518 long min = mark; 3519 int o; 3520 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3521 3522 /* free_pages may go negative - that's OK */ 3523 free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags); 3524 3525 if (alloc_flags & ALLOC_HIGH) 3526 min -= min / 2; 3527 3528 if (unlikely(alloc_harder)) { 3529 /* 3530 * OOM victims can try even harder than normal ALLOC_HARDER 3531 * users on the grounds that it's definitely going to be in 3532 * the exit path shortly and free memory. Any allocation it 3533 * makes during the free path will be small and short-lived. 3534 */ 3535 if (alloc_flags & ALLOC_OOM) 3536 min -= min / 2; 3537 else 3538 min -= min / 4; 3539 } 3540 3541 /* 3542 * Check watermarks for an order-0 allocation request. If these 3543 * are not met, then a high-order request also cannot go ahead 3544 * even if a suitable page happened to be free. 3545 */ 3546 if (free_pages <= min + z->lowmem_reserve[highest_zoneidx]) 3547 return false; 3548 3549 /* If this is an order-0 request then the watermark is fine */ 3550 if (!order) 3551 return true; 3552 3553 /* For a high-order request, check at least one suitable page is free */ 3554 for (o = order; o < MAX_ORDER; o++) { 3555 struct free_area *area = &z->free_area[o]; 3556 int mt; 3557 3558 if (!area->nr_free) 3559 continue; 3560 3561 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 3562 if (!free_area_empty(area, mt)) 3563 return true; 3564 } 3565 3566 #ifdef CONFIG_CMA 3567 if ((alloc_flags & ALLOC_CMA) && 3568 !free_area_empty(area, MIGRATE_CMA)) { 3569 return true; 3570 } 3571 #endif 3572 if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC)) 3573 return true; 3574 } 3575 return false; 3576 } 3577 3578 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3579 int highest_zoneidx, unsigned int alloc_flags) 3580 { 3581 return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 3582 zone_page_state(z, NR_FREE_PAGES)); 3583 } 3584 3585 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 3586 unsigned long mark, int highest_zoneidx, 3587 unsigned int alloc_flags, gfp_t gfp_mask) 3588 { 3589 long free_pages; 3590 3591 free_pages = zone_page_state(z, NR_FREE_PAGES); 3592 3593 /* 3594 * Fast check for order-0 only. If this fails then the reserves 3595 * need to be calculated. 3596 */ 3597 if (!order) { 3598 long fast_free; 3599 3600 fast_free = free_pages; 3601 fast_free -= __zone_watermark_unusable_free(z, 0, alloc_flags); 3602 if (fast_free > mark + z->lowmem_reserve[highest_zoneidx]) 3603 return true; 3604 } 3605 3606 if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 3607 free_pages)) 3608 return true; 3609 /* 3610 * Ignore watermark boosting for GFP_ATOMIC order-0 allocations 3611 * when checking the min watermark. The min watermark is the 3612 * point where boosting is ignored so that kswapd is woken up 3613 * when below the low watermark. 3614 */ 3615 if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost 3616 && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) { 3617 mark = z->_watermark[WMARK_MIN]; 3618 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 3619 alloc_flags, free_pages); 3620 } 3621 3622 return false; 3623 } 3624 3625 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 3626 unsigned long mark, int highest_zoneidx) 3627 { 3628 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3629 3630 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 3631 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 3632 3633 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0, 3634 free_pages); 3635 } 3636 3637 #ifdef CONFIG_NUMA 3638 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3639 { 3640 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= 3641 node_reclaim_distance; 3642 } 3643 #else /* CONFIG_NUMA */ 3644 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3645 { 3646 return true; 3647 } 3648 #endif /* CONFIG_NUMA */ 3649 3650 /* 3651 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid 3652 * fragmentation is subtle. If the preferred zone was HIGHMEM then 3653 * premature use of a lower zone may cause lowmem pressure problems that 3654 * are worse than fragmentation. If the next zone is ZONE_DMA then it is 3655 * probably too small. It only makes sense to spread allocations to avoid 3656 * fragmentation between the Normal and DMA32 zones. 3657 */ 3658 static inline unsigned int 3659 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) 3660 { 3661 unsigned int alloc_flags; 3662 3663 /* 3664 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 3665 * to save a branch. 3666 */ 3667 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); 3668 3669 #ifdef CONFIG_ZONE_DMA32 3670 if (!zone) 3671 return alloc_flags; 3672 3673 if (zone_idx(zone) != ZONE_NORMAL) 3674 return alloc_flags; 3675 3676 /* 3677 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and 3678 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume 3679 * on UMA that if Normal is populated then so is DMA32. 3680 */ 3681 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1); 3682 if (nr_online_nodes > 1 && !populated_zone(--zone)) 3683 return alloc_flags; 3684 3685 alloc_flags |= ALLOC_NOFRAGMENT; 3686 #endif /* CONFIG_ZONE_DMA32 */ 3687 return alloc_flags; 3688 } 3689 3690 static inline unsigned int current_alloc_flags(gfp_t gfp_mask, 3691 unsigned int alloc_flags) 3692 { 3693 #ifdef CONFIG_CMA 3694 unsigned int pflags = current->flags; 3695 3696 if (!(pflags & PF_MEMALLOC_NOCMA) && 3697 gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) 3698 alloc_flags |= ALLOC_CMA; 3699 3700 #endif 3701 return alloc_flags; 3702 } 3703 3704 /* 3705 * get_page_from_freelist goes through the zonelist trying to allocate 3706 * a page. 3707 */ 3708 static struct page * 3709 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 3710 const struct alloc_context *ac) 3711 { 3712 struct zoneref *z; 3713 struct zone *zone; 3714 struct pglist_data *last_pgdat_dirty_limit = NULL; 3715 bool no_fallback; 3716 3717 retry: 3718 /* 3719 * Scan zonelist, looking for a zone with enough free. 3720 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 3721 */ 3722 no_fallback = alloc_flags & ALLOC_NOFRAGMENT; 3723 z = ac->preferred_zoneref; 3724 for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, 3725 ac->highest_zoneidx, ac->nodemask) { 3726 struct page *page; 3727 unsigned long mark; 3728 3729 if (cpusets_enabled() && 3730 (alloc_flags & ALLOC_CPUSET) && 3731 !__cpuset_zone_allowed(zone, gfp_mask)) 3732 continue; 3733 /* 3734 * When allocating a page cache page for writing, we 3735 * want to get it from a node that is within its dirty 3736 * limit, such that no single node holds more than its 3737 * proportional share of globally allowed dirty pages. 3738 * The dirty limits take into account the node's 3739 * lowmem reserves and high watermark so that kswapd 3740 * should be able to balance it without having to 3741 * write pages from its LRU list. 3742 * 3743 * XXX: For now, allow allocations to potentially 3744 * exceed the per-node dirty limit in the slowpath 3745 * (spread_dirty_pages unset) before going into reclaim, 3746 * which is important when on a NUMA setup the allowed 3747 * nodes are together not big enough to reach the 3748 * global limit. The proper fix for these situations 3749 * will require awareness of nodes in the 3750 * dirty-throttling and the flusher threads. 3751 */ 3752 if (ac->spread_dirty_pages) { 3753 if (last_pgdat_dirty_limit == zone->zone_pgdat) 3754 continue; 3755 3756 if (!node_dirty_ok(zone->zone_pgdat)) { 3757 last_pgdat_dirty_limit = zone->zone_pgdat; 3758 continue; 3759 } 3760 } 3761 3762 if (no_fallback && nr_online_nodes > 1 && 3763 zone != ac->preferred_zoneref->zone) { 3764 int local_nid; 3765 3766 /* 3767 * If moving to a remote node, retry but allow 3768 * fragmenting fallbacks. Locality is more important 3769 * than fragmentation avoidance. 3770 */ 3771 local_nid = zone_to_nid(ac->preferred_zoneref->zone); 3772 if (zone_to_nid(zone) != local_nid) { 3773 alloc_flags &= ~ALLOC_NOFRAGMENT; 3774 goto retry; 3775 } 3776 } 3777 3778 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); 3779 if (!zone_watermark_fast(zone, order, mark, 3780 ac->highest_zoneidx, alloc_flags, 3781 gfp_mask)) { 3782 int ret; 3783 3784 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3785 /* 3786 * Watermark failed for this zone, but see if we can 3787 * grow this zone if it contains deferred pages. 3788 */ 3789 if (static_branch_unlikely(&deferred_pages)) { 3790 if (_deferred_grow_zone(zone, order)) 3791 goto try_this_zone; 3792 } 3793 #endif 3794 /* Checked here to keep the fast path fast */ 3795 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 3796 if (alloc_flags & ALLOC_NO_WATERMARKS) 3797 goto try_this_zone; 3798 3799 if (node_reclaim_mode == 0 || 3800 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 3801 continue; 3802 3803 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 3804 switch (ret) { 3805 case NODE_RECLAIM_NOSCAN: 3806 /* did not scan */ 3807 continue; 3808 case NODE_RECLAIM_FULL: 3809 /* scanned but unreclaimable */ 3810 continue; 3811 default: 3812 /* did we reclaim enough */ 3813 if (zone_watermark_ok(zone, order, mark, 3814 ac->highest_zoneidx, alloc_flags)) 3815 goto try_this_zone; 3816 3817 continue; 3818 } 3819 } 3820 3821 try_this_zone: 3822 page = rmqueue(ac->preferred_zoneref->zone, zone, order, 3823 gfp_mask, alloc_flags, ac->migratetype); 3824 if (page) { 3825 prep_new_page(page, order, gfp_mask, alloc_flags); 3826 3827 /* 3828 * If this is a high-order atomic allocation then check 3829 * if the pageblock should be reserved for the future 3830 */ 3831 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 3832 reserve_highatomic_pageblock(page, zone, order); 3833 3834 return page; 3835 } else { 3836 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3837 /* Try again if zone has deferred pages */ 3838 if (static_branch_unlikely(&deferred_pages)) { 3839 if (_deferred_grow_zone(zone, order)) 3840 goto try_this_zone; 3841 } 3842 #endif 3843 } 3844 } 3845 3846 /* 3847 * It's possible on a UMA machine to get through all zones that are 3848 * fragmented. If avoiding fragmentation, reset and try again. 3849 */ 3850 if (no_fallback) { 3851 alloc_flags &= ~ALLOC_NOFRAGMENT; 3852 goto retry; 3853 } 3854 3855 return NULL; 3856 } 3857 3858 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) 3859 { 3860 unsigned int filter = SHOW_MEM_FILTER_NODES; 3861 3862 /* 3863 * This documents exceptions given to allocations in certain 3864 * contexts that are allowed to allocate outside current's set 3865 * of allowed nodes. 3866 */ 3867 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3868 if (tsk_is_oom_victim(current) || 3869 (current->flags & (PF_MEMALLOC | PF_EXITING))) 3870 filter &= ~SHOW_MEM_FILTER_NODES; 3871 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 3872 filter &= ~SHOW_MEM_FILTER_NODES; 3873 3874 show_mem(filter, nodemask); 3875 } 3876 3877 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) 3878 { 3879 struct va_format vaf; 3880 va_list args; 3881 static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1); 3882 3883 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs)) 3884 return; 3885 3886 va_start(args, fmt); 3887 vaf.fmt = fmt; 3888 vaf.va = &args; 3889 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl", 3890 current->comm, &vaf, gfp_mask, &gfp_mask, 3891 nodemask_pr_args(nodemask)); 3892 va_end(args); 3893 3894 cpuset_print_current_mems_allowed(); 3895 pr_cont("\n"); 3896 dump_stack(); 3897 warn_alloc_show_mem(gfp_mask, nodemask); 3898 } 3899 3900 static inline struct page * 3901 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, 3902 unsigned int alloc_flags, 3903 const struct alloc_context *ac) 3904 { 3905 struct page *page; 3906 3907 page = get_page_from_freelist(gfp_mask, order, 3908 alloc_flags|ALLOC_CPUSET, ac); 3909 /* 3910 * fallback to ignore cpuset restriction if our nodes 3911 * are depleted 3912 */ 3913 if (!page) 3914 page = get_page_from_freelist(gfp_mask, order, 3915 alloc_flags, ac); 3916 3917 return page; 3918 } 3919 3920 static inline struct page * 3921 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 3922 const struct alloc_context *ac, unsigned long *did_some_progress) 3923 { 3924 struct oom_control oc = { 3925 .zonelist = ac->zonelist, 3926 .nodemask = ac->nodemask, 3927 .memcg = NULL, 3928 .gfp_mask = gfp_mask, 3929 .order = order, 3930 }; 3931 struct page *page; 3932 3933 *did_some_progress = 0; 3934 3935 /* 3936 * Acquire the oom lock. If that fails, somebody else is 3937 * making progress for us. 3938 */ 3939 if (!mutex_trylock(&oom_lock)) { 3940 *did_some_progress = 1; 3941 schedule_timeout_uninterruptible(1); 3942 return NULL; 3943 } 3944 3945 /* 3946 * Go through the zonelist yet one more time, keep very high watermark 3947 * here, this is only to catch a parallel oom killing, we must fail if 3948 * we're still under heavy pressure. But make sure that this reclaim 3949 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY 3950 * allocation which will never fail due to oom_lock already held. 3951 */ 3952 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & 3953 ~__GFP_DIRECT_RECLAIM, order, 3954 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 3955 if (page) 3956 goto out; 3957 3958 /* Coredumps can quickly deplete all memory reserves */ 3959 if (current->flags & PF_DUMPCORE) 3960 goto out; 3961 /* The OOM killer will not help higher order allocs */ 3962 if (order > PAGE_ALLOC_COSTLY_ORDER) 3963 goto out; 3964 /* 3965 * We have already exhausted all our reclaim opportunities without any 3966 * success so it is time to admit defeat. We will skip the OOM killer 3967 * because it is very likely that the caller has a more reasonable 3968 * fallback than shooting a random task. 3969 */ 3970 if (gfp_mask & __GFP_RETRY_MAYFAIL) 3971 goto out; 3972 /* The OOM killer does not needlessly kill tasks for lowmem */ 3973 if (ac->highest_zoneidx < ZONE_NORMAL) 3974 goto out; 3975 if (pm_suspended_storage()) 3976 goto out; 3977 /* 3978 * XXX: GFP_NOFS allocations should rather fail than rely on 3979 * other request to make a forward progress. 3980 * We are in an unfortunate situation where out_of_memory cannot 3981 * do much for this context but let's try it to at least get 3982 * access to memory reserved if the current task is killed (see 3983 * out_of_memory). Once filesystems are ready to handle allocation 3984 * failures more gracefully we should just bail out here. 3985 */ 3986 3987 /* The OOM killer may not free memory on a specific node */ 3988 if (gfp_mask & __GFP_THISNODE) 3989 goto out; 3990 3991 /* Exhausted what can be done so it's blame time */ 3992 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 3993 *did_some_progress = 1; 3994 3995 /* 3996 * Help non-failing allocations by giving them access to memory 3997 * reserves 3998 */ 3999 if (gfp_mask & __GFP_NOFAIL) 4000 page = __alloc_pages_cpuset_fallback(gfp_mask, order, 4001 ALLOC_NO_WATERMARKS, ac); 4002 } 4003 out: 4004 mutex_unlock(&oom_lock); 4005 return page; 4006 } 4007 4008 /* 4009 * Maximum number of compaction retries wit a progress before OOM 4010 * killer is consider as the only way to move forward. 4011 */ 4012 #define MAX_COMPACT_RETRIES 16 4013 4014 #ifdef CONFIG_COMPACTION 4015 /* Try memory compaction for high-order allocations before reclaim */ 4016 static struct page * 4017 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4018 unsigned int alloc_flags, const struct alloc_context *ac, 4019 enum compact_priority prio, enum compact_result *compact_result) 4020 { 4021 struct page *page = NULL; 4022 unsigned long pflags; 4023 unsigned int noreclaim_flag; 4024 4025 if (!order) 4026 return NULL; 4027 4028 psi_memstall_enter(&pflags); 4029 noreclaim_flag = memalloc_noreclaim_save(); 4030 4031 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 4032 prio, &page); 4033 4034 memalloc_noreclaim_restore(noreclaim_flag); 4035 psi_memstall_leave(&pflags); 4036 4037 /* 4038 * At least in one zone compaction wasn't deferred or skipped, so let's 4039 * count a compaction stall 4040 */ 4041 count_vm_event(COMPACTSTALL); 4042 4043 /* Prep a captured page if available */ 4044 if (page) 4045 prep_new_page(page, order, gfp_mask, alloc_flags); 4046 4047 /* Try get a page from the freelist if available */ 4048 if (!page) 4049 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4050 4051 if (page) { 4052 struct zone *zone = page_zone(page); 4053 4054 zone->compact_blockskip_flush = false; 4055 compaction_defer_reset(zone, order, true); 4056 count_vm_event(COMPACTSUCCESS); 4057 return page; 4058 } 4059 4060 /* 4061 * It's bad if compaction run occurs and fails. The most likely reason 4062 * is that pages exist, but not enough to satisfy watermarks. 4063 */ 4064 count_vm_event(COMPACTFAIL); 4065 4066 cond_resched(); 4067 4068 return NULL; 4069 } 4070 4071 static inline bool 4072 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 4073 enum compact_result compact_result, 4074 enum compact_priority *compact_priority, 4075 int *compaction_retries) 4076 { 4077 int max_retries = MAX_COMPACT_RETRIES; 4078 int min_priority; 4079 bool ret = false; 4080 int retries = *compaction_retries; 4081 enum compact_priority priority = *compact_priority; 4082 4083 if (!order) 4084 return false; 4085 4086 if (compaction_made_progress(compact_result)) 4087 (*compaction_retries)++; 4088 4089 /* 4090 * compaction considers all the zone as desperately out of memory 4091 * so it doesn't really make much sense to retry except when the 4092 * failure could be caused by insufficient priority 4093 */ 4094 if (compaction_failed(compact_result)) 4095 goto check_priority; 4096 4097 /* 4098 * compaction was skipped because there are not enough order-0 pages 4099 * to work with, so we retry only if it looks like reclaim can help. 4100 */ 4101 if (compaction_needs_reclaim(compact_result)) { 4102 ret = compaction_zonelist_suitable(ac, order, alloc_flags); 4103 goto out; 4104 } 4105 4106 /* 4107 * make sure the compaction wasn't deferred or didn't bail out early 4108 * due to locks contention before we declare that we should give up. 4109 * But the next retry should use a higher priority if allowed, so 4110 * we don't just keep bailing out endlessly. 4111 */ 4112 if (compaction_withdrawn(compact_result)) { 4113 goto check_priority; 4114 } 4115 4116 /* 4117 * !costly requests are much more important than __GFP_RETRY_MAYFAIL 4118 * costly ones because they are de facto nofail and invoke OOM 4119 * killer to move on while costly can fail and users are ready 4120 * to cope with that. 1/4 retries is rather arbitrary but we 4121 * would need much more detailed feedback from compaction to 4122 * make a better decision. 4123 */ 4124 if (order > PAGE_ALLOC_COSTLY_ORDER) 4125 max_retries /= 4; 4126 if (*compaction_retries <= max_retries) { 4127 ret = true; 4128 goto out; 4129 } 4130 4131 /* 4132 * Make sure there are attempts at the highest priority if we exhausted 4133 * all retries or failed at the lower priorities. 4134 */ 4135 check_priority: 4136 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 4137 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 4138 4139 if (*compact_priority > min_priority) { 4140 (*compact_priority)--; 4141 *compaction_retries = 0; 4142 ret = true; 4143 } 4144 out: 4145 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret); 4146 return ret; 4147 } 4148 #else 4149 static inline struct page * 4150 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4151 unsigned int alloc_flags, const struct alloc_context *ac, 4152 enum compact_priority prio, enum compact_result *compact_result) 4153 { 4154 *compact_result = COMPACT_SKIPPED; 4155 return NULL; 4156 } 4157 4158 static inline bool 4159 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 4160 enum compact_result compact_result, 4161 enum compact_priority *compact_priority, 4162 int *compaction_retries) 4163 { 4164 struct zone *zone; 4165 struct zoneref *z; 4166 4167 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 4168 return false; 4169 4170 /* 4171 * There are setups with compaction disabled which would prefer to loop 4172 * inside the allocator rather than hit the oom killer prematurely. 4173 * Let's give them a good hope and keep retrying while the order-0 4174 * watermarks are OK. 4175 */ 4176 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4177 ac->highest_zoneidx, ac->nodemask) { 4178 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 4179 ac->highest_zoneidx, alloc_flags)) 4180 return true; 4181 } 4182 return false; 4183 } 4184 #endif /* CONFIG_COMPACTION */ 4185 4186 #ifdef CONFIG_LOCKDEP 4187 static struct lockdep_map __fs_reclaim_map = 4188 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map); 4189 4190 static bool __need_fs_reclaim(gfp_t gfp_mask) 4191 { 4192 gfp_mask = current_gfp_context(gfp_mask); 4193 4194 /* no reclaim without waiting on it */ 4195 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) 4196 return false; 4197 4198 /* this guy won't enter reclaim */ 4199 if (current->flags & PF_MEMALLOC) 4200 return false; 4201 4202 /* We're only interested __GFP_FS allocations for now */ 4203 if (!(gfp_mask & __GFP_FS)) 4204 return false; 4205 4206 if (gfp_mask & __GFP_NOLOCKDEP) 4207 return false; 4208 4209 return true; 4210 } 4211 4212 void __fs_reclaim_acquire(void) 4213 { 4214 lock_map_acquire(&__fs_reclaim_map); 4215 } 4216 4217 void __fs_reclaim_release(void) 4218 { 4219 lock_map_release(&__fs_reclaim_map); 4220 } 4221 4222 void fs_reclaim_acquire(gfp_t gfp_mask) 4223 { 4224 if (__need_fs_reclaim(gfp_mask)) 4225 __fs_reclaim_acquire(); 4226 } 4227 EXPORT_SYMBOL_GPL(fs_reclaim_acquire); 4228 4229 void fs_reclaim_release(gfp_t gfp_mask) 4230 { 4231 if (__need_fs_reclaim(gfp_mask)) 4232 __fs_reclaim_release(); 4233 } 4234 EXPORT_SYMBOL_GPL(fs_reclaim_release); 4235 #endif 4236 4237 /* Perform direct synchronous page reclaim */ 4238 static int 4239 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 4240 const struct alloc_context *ac) 4241 { 4242 int progress; 4243 unsigned int noreclaim_flag; 4244 unsigned long pflags; 4245 4246 cond_resched(); 4247 4248 /* We now go into synchronous reclaim */ 4249 cpuset_memory_pressure_bump(); 4250 psi_memstall_enter(&pflags); 4251 fs_reclaim_acquire(gfp_mask); 4252 noreclaim_flag = memalloc_noreclaim_save(); 4253 4254 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 4255 ac->nodemask); 4256 4257 memalloc_noreclaim_restore(noreclaim_flag); 4258 fs_reclaim_release(gfp_mask); 4259 psi_memstall_leave(&pflags); 4260 4261 cond_resched(); 4262 4263 return progress; 4264 } 4265 4266 /* The really slow allocator path where we enter direct reclaim */ 4267 static inline struct page * 4268 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 4269 unsigned int alloc_flags, const struct alloc_context *ac, 4270 unsigned long *did_some_progress) 4271 { 4272 struct page *page = NULL; 4273 bool drained = false; 4274 4275 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 4276 if (unlikely(!(*did_some_progress))) 4277 return NULL; 4278 4279 retry: 4280 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4281 4282 /* 4283 * If an allocation failed after direct reclaim, it could be because 4284 * pages are pinned on the per-cpu lists or in high alloc reserves. 4285 * Shrink them and try again 4286 */ 4287 if (!page && !drained) { 4288 unreserve_highatomic_pageblock(ac, false); 4289 drain_all_pages(NULL); 4290 drained = true; 4291 goto retry; 4292 } 4293 4294 return page; 4295 } 4296 4297 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, 4298 const struct alloc_context *ac) 4299 { 4300 struct zoneref *z; 4301 struct zone *zone; 4302 pg_data_t *last_pgdat = NULL; 4303 enum zone_type highest_zoneidx = ac->highest_zoneidx; 4304 4305 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx, 4306 ac->nodemask) { 4307 if (last_pgdat != zone->zone_pgdat) 4308 wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx); 4309 last_pgdat = zone->zone_pgdat; 4310 } 4311 } 4312 4313 static inline unsigned int 4314 gfp_to_alloc_flags(gfp_t gfp_mask) 4315 { 4316 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 4317 4318 /* 4319 * __GFP_HIGH is assumed to be the same as ALLOC_HIGH 4320 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 4321 * to save two branches. 4322 */ 4323 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 4324 BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); 4325 4326 /* 4327 * The caller may dip into page reserves a bit more if the caller 4328 * cannot run direct reclaim, or if the caller has realtime scheduling 4329 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 4330 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 4331 */ 4332 alloc_flags |= (__force int) 4333 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); 4334 4335 if (gfp_mask & __GFP_ATOMIC) { 4336 /* 4337 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 4338 * if it can't schedule. 4339 */ 4340 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4341 alloc_flags |= ALLOC_HARDER; 4342 /* 4343 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 4344 * comment for __cpuset_node_allowed(). 4345 */ 4346 alloc_flags &= ~ALLOC_CPUSET; 4347 } else if (unlikely(rt_task(current)) && !in_interrupt()) 4348 alloc_flags |= ALLOC_HARDER; 4349 4350 alloc_flags = current_alloc_flags(gfp_mask, alloc_flags); 4351 4352 return alloc_flags; 4353 } 4354 4355 static bool oom_reserves_allowed(struct task_struct *tsk) 4356 { 4357 if (!tsk_is_oom_victim(tsk)) 4358 return false; 4359 4360 /* 4361 * !MMU doesn't have oom reaper so give access to memory reserves 4362 * only to the thread with TIF_MEMDIE set 4363 */ 4364 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE)) 4365 return false; 4366 4367 return true; 4368 } 4369 4370 /* 4371 * Distinguish requests which really need access to full memory 4372 * reserves from oom victims which can live with a portion of it 4373 */ 4374 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) 4375 { 4376 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 4377 return 0; 4378 if (gfp_mask & __GFP_MEMALLOC) 4379 return ALLOC_NO_WATERMARKS; 4380 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 4381 return ALLOC_NO_WATERMARKS; 4382 if (!in_interrupt()) { 4383 if (current->flags & PF_MEMALLOC) 4384 return ALLOC_NO_WATERMARKS; 4385 else if (oom_reserves_allowed(current)) 4386 return ALLOC_OOM; 4387 } 4388 4389 return 0; 4390 } 4391 4392 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 4393 { 4394 return !!__gfp_pfmemalloc_flags(gfp_mask); 4395 } 4396 4397 /* 4398 * Checks whether it makes sense to retry the reclaim to make a forward progress 4399 * for the given allocation request. 4400 * 4401 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row 4402 * without success, or when we couldn't even meet the watermark if we 4403 * reclaimed all remaining pages on the LRU lists. 4404 * 4405 * Returns true if a retry is viable or false to enter the oom path. 4406 */ 4407 static inline bool 4408 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 4409 struct alloc_context *ac, int alloc_flags, 4410 bool did_some_progress, int *no_progress_loops) 4411 { 4412 struct zone *zone; 4413 struct zoneref *z; 4414 bool ret = false; 4415 4416 /* 4417 * Costly allocations might have made a progress but this doesn't mean 4418 * their order will become available due to high fragmentation so 4419 * always increment the no progress counter for them 4420 */ 4421 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 4422 *no_progress_loops = 0; 4423 else 4424 (*no_progress_loops)++; 4425 4426 /* 4427 * Make sure we converge to OOM if we cannot make any progress 4428 * several times in the row. 4429 */ 4430 if (*no_progress_loops > MAX_RECLAIM_RETRIES) { 4431 /* Before OOM, exhaust highatomic_reserve */ 4432 return unreserve_highatomic_pageblock(ac, true); 4433 } 4434 4435 /* 4436 * Keep reclaiming pages while there is a chance this will lead 4437 * somewhere. If none of the target zones can satisfy our allocation 4438 * request even if all reclaimable pages are considered then we are 4439 * screwed and have to go OOM. 4440 */ 4441 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4442 ac->highest_zoneidx, ac->nodemask) { 4443 unsigned long available; 4444 unsigned long reclaimable; 4445 unsigned long min_wmark = min_wmark_pages(zone); 4446 bool wmark; 4447 4448 available = reclaimable = zone_reclaimable_pages(zone); 4449 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 4450 4451 /* 4452 * Would the allocation succeed if we reclaimed all 4453 * reclaimable pages? 4454 */ 4455 wmark = __zone_watermark_ok(zone, order, min_wmark, 4456 ac->highest_zoneidx, alloc_flags, available); 4457 trace_reclaim_retry_zone(z, order, reclaimable, 4458 available, min_wmark, *no_progress_loops, wmark); 4459 if (wmark) { 4460 /* 4461 * If we didn't make any progress and have a lot of 4462 * dirty + writeback pages then we should wait for 4463 * an IO to complete to slow down the reclaim and 4464 * prevent from pre mature OOM 4465 */ 4466 if (!did_some_progress) { 4467 unsigned long write_pending; 4468 4469 write_pending = zone_page_state_snapshot(zone, 4470 NR_ZONE_WRITE_PENDING); 4471 4472 if (2 * write_pending > reclaimable) { 4473 congestion_wait(BLK_RW_ASYNC, HZ/10); 4474 return true; 4475 } 4476 } 4477 4478 ret = true; 4479 goto out; 4480 } 4481 } 4482 4483 out: 4484 /* 4485 * Memory allocation/reclaim might be called from a WQ context and the 4486 * current implementation of the WQ concurrency control doesn't 4487 * recognize that a particular WQ is congested if the worker thread is 4488 * looping without ever sleeping. Therefore we have to do a short sleep 4489 * here rather than calling cond_resched(). 4490 */ 4491 if (current->flags & PF_WQ_WORKER) 4492 schedule_timeout_uninterruptible(1); 4493 else 4494 cond_resched(); 4495 return ret; 4496 } 4497 4498 static inline bool 4499 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) 4500 { 4501 /* 4502 * It's possible that cpuset's mems_allowed and the nodemask from 4503 * mempolicy don't intersect. This should be normally dealt with by 4504 * policy_nodemask(), but it's possible to race with cpuset update in 4505 * such a way the check therein was true, and then it became false 4506 * before we got our cpuset_mems_cookie here. 4507 * This assumes that for all allocations, ac->nodemask can come only 4508 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored 4509 * when it does not intersect with the cpuset restrictions) or the 4510 * caller can deal with a violated nodemask. 4511 */ 4512 if (cpusets_enabled() && ac->nodemask && 4513 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) { 4514 ac->nodemask = NULL; 4515 return true; 4516 } 4517 4518 /* 4519 * When updating a task's mems_allowed or mempolicy nodemask, it is 4520 * possible to race with parallel threads in such a way that our 4521 * allocation can fail while the mask is being updated. If we are about 4522 * to fail, check if the cpuset changed during allocation and if so, 4523 * retry. 4524 */ 4525 if (read_mems_allowed_retry(cpuset_mems_cookie)) 4526 return true; 4527 4528 return false; 4529 } 4530 4531 static inline struct page * 4532 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 4533 struct alloc_context *ac) 4534 { 4535 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 4536 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 4537 struct page *page = NULL; 4538 unsigned int alloc_flags; 4539 unsigned long did_some_progress; 4540 enum compact_priority compact_priority; 4541 enum compact_result compact_result; 4542 int compaction_retries; 4543 int no_progress_loops; 4544 unsigned int cpuset_mems_cookie; 4545 int reserve_flags; 4546 4547 /* 4548 * We also sanity check to catch abuse of atomic reserves being used by 4549 * callers that are not in atomic context. 4550 */ 4551 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 4552 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 4553 gfp_mask &= ~__GFP_ATOMIC; 4554 4555 retry_cpuset: 4556 compaction_retries = 0; 4557 no_progress_loops = 0; 4558 compact_priority = DEF_COMPACT_PRIORITY; 4559 cpuset_mems_cookie = read_mems_allowed_begin(); 4560 4561 /* 4562 * The fast path uses conservative alloc_flags to succeed only until 4563 * kswapd needs to be woken up, and to avoid the cost of setting up 4564 * alloc_flags precisely. So we do that now. 4565 */ 4566 alloc_flags = gfp_to_alloc_flags(gfp_mask); 4567 4568 /* 4569 * We need to recalculate the starting point for the zonelist iterator 4570 * because we might have used different nodemask in the fast path, or 4571 * there was a cpuset modification and we are retrying - otherwise we 4572 * could end up iterating over non-eligible zones endlessly. 4573 */ 4574 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4575 ac->highest_zoneidx, ac->nodemask); 4576 if (!ac->preferred_zoneref->zone) 4577 goto nopage; 4578 4579 if (alloc_flags & ALLOC_KSWAPD) 4580 wake_all_kswapds(order, gfp_mask, ac); 4581 4582 /* 4583 * The adjusted alloc_flags might result in immediate success, so try 4584 * that first 4585 */ 4586 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4587 if (page) 4588 goto got_pg; 4589 4590 /* 4591 * For costly allocations, try direct compaction first, as it's likely 4592 * that we have enough base pages and don't need to reclaim. For non- 4593 * movable high-order allocations, do that as well, as compaction will 4594 * try prevent permanent fragmentation by migrating from blocks of the 4595 * same migratetype. 4596 * Don't try this for allocations that are allowed to ignore 4597 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 4598 */ 4599 if (can_direct_reclaim && 4600 (costly_order || 4601 (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 4602 && !gfp_pfmemalloc_allowed(gfp_mask)) { 4603 page = __alloc_pages_direct_compact(gfp_mask, order, 4604 alloc_flags, ac, 4605 INIT_COMPACT_PRIORITY, 4606 &compact_result); 4607 if (page) 4608 goto got_pg; 4609 4610 /* 4611 * Checks for costly allocations with __GFP_NORETRY, which 4612 * includes some THP page fault allocations 4613 */ 4614 if (costly_order && (gfp_mask & __GFP_NORETRY)) { 4615 /* 4616 * If allocating entire pageblock(s) and compaction 4617 * failed because all zones are below low watermarks 4618 * or is prohibited because it recently failed at this 4619 * order, fail immediately unless the allocator has 4620 * requested compaction and reclaim retry. 4621 * 4622 * Reclaim is 4623 * - potentially very expensive because zones are far 4624 * below their low watermarks or this is part of very 4625 * bursty high order allocations, 4626 * - not guaranteed to help because isolate_freepages() 4627 * may not iterate over freed pages as part of its 4628 * linear scan, and 4629 * - unlikely to make entire pageblocks free on its 4630 * own. 4631 */ 4632 if (compact_result == COMPACT_SKIPPED || 4633 compact_result == COMPACT_DEFERRED) 4634 goto nopage; 4635 4636 /* 4637 * Looks like reclaim/compaction is worth trying, but 4638 * sync compaction could be very expensive, so keep 4639 * using async compaction. 4640 */ 4641 compact_priority = INIT_COMPACT_PRIORITY; 4642 } 4643 } 4644 4645 retry: 4646 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 4647 if (alloc_flags & ALLOC_KSWAPD) 4648 wake_all_kswapds(order, gfp_mask, ac); 4649 4650 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); 4651 if (reserve_flags) 4652 alloc_flags = current_alloc_flags(gfp_mask, reserve_flags); 4653 4654 /* 4655 * Reset the nodemask and zonelist iterators if memory policies can be 4656 * ignored. These allocations are high priority and system rather than 4657 * user oriented. 4658 */ 4659 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { 4660 ac->nodemask = NULL; 4661 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4662 ac->highest_zoneidx, ac->nodemask); 4663 } 4664 4665 /* Attempt with potentially adjusted zonelist and alloc_flags */ 4666 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4667 if (page) 4668 goto got_pg; 4669 4670 /* Caller is not willing to reclaim, we can't balance anything */ 4671 if (!can_direct_reclaim) 4672 goto nopage; 4673 4674 /* Avoid recursion of direct reclaim */ 4675 if (current->flags & PF_MEMALLOC) 4676 goto nopage; 4677 4678 /* Try direct reclaim and then allocating */ 4679 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 4680 &did_some_progress); 4681 if (page) 4682 goto got_pg; 4683 4684 /* Try direct compaction and then allocating */ 4685 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 4686 compact_priority, &compact_result); 4687 if (page) 4688 goto got_pg; 4689 4690 /* Do not loop if specifically requested */ 4691 if (gfp_mask & __GFP_NORETRY) 4692 goto nopage; 4693 4694 /* 4695 * Do not retry costly high order allocations unless they are 4696 * __GFP_RETRY_MAYFAIL 4697 */ 4698 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) 4699 goto nopage; 4700 4701 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 4702 did_some_progress > 0, &no_progress_loops)) 4703 goto retry; 4704 4705 /* 4706 * It doesn't make any sense to retry for the compaction if the order-0 4707 * reclaim is not able to make any progress because the current 4708 * implementation of the compaction depends on the sufficient amount 4709 * of free memory (see __compaction_suitable) 4710 */ 4711 if (did_some_progress > 0 && 4712 should_compact_retry(ac, order, alloc_flags, 4713 compact_result, &compact_priority, 4714 &compaction_retries)) 4715 goto retry; 4716 4717 4718 /* Deal with possible cpuset update races before we start OOM killing */ 4719 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4720 goto retry_cpuset; 4721 4722 /* Reclaim has failed us, start killing things */ 4723 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 4724 if (page) 4725 goto got_pg; 4726 4727 /* Avoid allocations with no watermarks from looping endlessly */ 4728 if (tsk_is_oom_victim(current) && 4729 (alloc_flags & ALLOC_OOM || 4730 (gfp_mask & __GFP_NOMEMALLOC))) 4731 goto nopage; 4732 4733 /* Retry as long as the OOM killer is making progress */ 4734 if (did_some_progress) { 4735 no_progress_loops = 0; 4736 goto retry; 4737 } 4738 4739 nopage: 4740 /* Deal with possible cpuset update races before we fail */ 4741 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4742 goto retry_cpuset; 4743 4744 /* 4745 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure 4746 * we always retry 4747 */ 4748 if (gfp_mask & __GFP_NOFAIL) { 4749 /* 4750 * All existing users of the __GFP_NOFAIL are blockable, so warn 4751 * of any new users that actually require GFP_NOWAIT 4752 */ 4753 if (WARN_ON_ONCE(!can_direct_reclaim)) 4754 goto fail; 4755 4756 /* 4757 * PF_MEMALLOC request from this context is rather bizarre 4758 * because we cannot reclaim anything and only can loop waiting 4759 * for somebody to do a work for us 4760 */ 4761 WARN_ON_ONCE(current->flags & PF_MEMALLOC); 4762 4763 /* 4764 * non failing costly orders are a hard requirement which we 4765 * are not prepared for much so let's warn about these users 4766 * so that we can identify them and convert them to something 4767 * else. 4768 */ 4769 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER); 4770 4771 /* 4772 * Help non-failing allocations by giving them access to memory 4773 * reserves but do not use ALLOC_NO_WATERMARKS because this 4774 * could deplete whole memory reserves which would just make 4775 * the situation worse 4776 */ 4777 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); 4778 if (page) 4779 goto got_pg; 4780 4781 cond_resched(); 4782 goto retry; 4783 } 4784 fail: 4785 warn_alloc(gfp_mask, ac->nodemask, 4786 "page allocation failure: order:%u", order); 4787 got_pg: 4788 return page; 4789 } 4790 4791 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, 4792 int preferred_nid, nodemask_t *nodemask, 4793 struct alloc_context *ac, gfp_t *alloc_mask, 4794 unsigned int *alloc_flags) 4795 { 4796 ac->highest_zoneidx = gfp_zone(gfp_mask); 4797 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); 4798 ac->nodemask = nodemask; 4799 ac->migratetype = gfp_migratetype(gfp_mask); 4800 4801 if (cpusets_enabled()) { 4802 *alloc_mask |= __GFP_HARDWALL; 4803 /* 4804 * When we are in the interrupt context, it is irrelevant 4805 * to the current task context. It means that any node ok. 4806 */ 4807 if (!in_interrupt() && !ac->nodemask) 4808 ac->nodemask = &cpuset_current_mems_allowed; 4809 else 4810 *alloc_flags |= ALLOC_CPUSET; 4811 } 4812 4813 fs_reclaim_acquire(gfp_mask); 4814 fs_reclaim_release(gfp_mask); 4815 4816 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 4817 4818 if (should_fail_alloc_page(gfp_mask, order)) 4819 return false; 4820 4821 *alloc_flags = current_alloc_flags(gfp_mask, *alloc_flags); 4822 4823 return true; 4824 } 4825 4826 /* Determine whether to spread dirty pages and what the first usable zone */ 4827 static inline void finalise_ac(gfp_t gfp_mask, struct alloc_context *ac) 4828 { 4829 /* Dirty zone balancing only done in the fast path */ 4830 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); 4831 4832 /* 4833 * The preferred zone is used for statistics but crucially it is 4834 * also used as the starting point for the zonelist iterator. It 4835 * may get reset for allocations that ignore memory policies. 4836 */ 4837 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4838 ac->highest_zoneidx, ac->nodemask); 4839 } 4840 4841 /* 4842 * This is the 'heart' of the zoned buddy allocator. 4843 */ 4844 struct page * 4845 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, 4846 nodemask_t *nodemask) 4847 { 4848 struct page *page; 4849 unsigned int alloc_flags = ALLOC_WMARK_LOW; 4850 gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */ 4851 struct alloc_context ac = { }; 4852 4853 /* 4854 * There are several places where we assume that the order value is sane 4855 * so bail out early if the request is out of bound. 4856 */ 4857 if (unlikely(order >= MAX_ORDER)) { 4858 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); 4859 return NULL; 4860 } 4861 4862 gfp_mask &= gfp_allowed_mask; 4863 alloc_mask = gfp_mask; 4864 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags)) 4865 return NULL; 4866 4867 finalise_ac(gfp_mask, &ac); 4868 4869 /* 4870 * Forbid the first pass from falling back to types that fragment 4871 * memory until all local zones are considered. 4872 */ 4873 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask); 4874 4875 /* First allocation attempt */ 4876 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); 4877 if (likely(page)) 4878 goto out; 4879 4880 /* 4881 * Apply scoped allocation constraints. This is mainly about GFP_NOFS 4882 * resp. GFP_NOIO which has to be inherited for all allocation requests 4883 * from a particular context which has been marked by 4884 * memalloc_no{fs,io}_{save,restore}. 4885 */ 4886 alloc_mask = current_gfp_context(gfp_mask); 4887 ac.spread_dirty_pages = false; 4888 4889 /* 4890 * Restore the original nodemask if it was potentially replaced with 4891 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 4892 */ 4893 ac.nodemask = nodemask; 4894 4895 page = __alloc_pages_slowpath(alloc_mask, order, &ac); 4896 4897 out: 4898 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && 4899 unlikely(__memcg_kmem_charge_page(page, gfp_mask, order) != 0)) { 4900 __free_pages(page, order); 4901 page = NULL; 4902 } 4903 4904 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); 4905 4906 return page; 4907 } 4908 EXPORT_SYMBOL(__alloc_pages_nodemask); 4909 4910 /* 4911 * Common helper functions. Never use with __GFP_HIGHMEM because the returned 4912 * address cannot represent highmem pages. Use alloc_pages and then kmap if 4913 * you need to access high mem. 4914 */ 4915 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 4916 { 4917 struct page *page; 4918 4919 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); 4920 if (!page) 4921 return 0; 4922 return (unsigned long) page_address(page); 4923 } 4924 EXPORT_SYMBOL(__get_free_pages); 4925 4926 unsigned long get_zeroed_page(gfp_t gfp_mask) 4927 { 4928 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 4929 } 4930 EXPORT_SYMBOL(get_zeroed_page); 4931 4932 static inline void free_the_page(struct page *page, unsigned int order) 4933 { 4934 if (order == 0) /* Via pcp? */ 4935 free_unref_page(page); 4936 else 4937 __free_pages_ok(page, order); 4938 } 4939 4940 void __free_pages(struct page *page, unsigned int order) 4941 { 4942 if (put_page_testzero(page)) 4943 free_the_page(page, order); 4944 } 4945 EXPORT_SYMBOL(__free_pages); 4946 4947 void free_pages(unsigned long addr, unsigned int order) 4948 { 4949 if (addr != 0) { 4950 VM_BUG_ON(!virt_addr_valid((void *)addr)); 4951 __free_pages(virt_to_page((void *)addr), order); 4952 } 4953 } 4954 4955 EXPORT_SYMBOL(free_pages); 4956 4957 /* 4958 * Page Fragment: 4959 * An arbitrary-length arbitrary-offset area of memory which resides 4960 * within a 0 or higher order page. Multiple fragments within that page 4961 * are individually refcounted, in the page's reference counter. 4962 * 4963 * The page_frag functions below provide a simple allocation framework for 4964 * page fragments. This is used by the network stack and network device 4965 * drivers to provide a backing region of memory for use as either an 4966 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 4967 */ 4968 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 4969 gfp_t gfp_mask) 4970 { 4971 struct page *page = NULL; 4972 gfp_t gfp = gfp_mask; 4973 4974 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4975 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 4976 __GFP_NOMEMALLOC; 4977 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 4978 PAGE_FRAG_CACHE_MAX_ORDER); 4979 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 4980 #endif 4981 if (unlikely(!page)) 4982 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 4983 4984 nc->va = page ? page_address(page) : NULL; 4985 4986 return page; 4987 } 4988 4989 void __page_frag_cache_drain(struct page *page, unsigned int count) 4990 { 4991 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 4992 4993 if (page_ref_sub_and_test(page, count)) 4994 free_the_page(page, compound_order(page)); 4995 } 4996 EXPORT_SYMBOL(__page_frag_cache_drain); 4997 4998 void *page_frag_alloc(struct page_frag_cache *nc, 4999 unsigned int fragsz, gfp_t gfp_mask) 5000 { 5001 unsigned int size = PAGE_SIZE; 5002 struct page *page; 5003 int offset; 5004 5005 if (unlikely(!nc->va)) { 5006 refill: 5007 page = __page_frag_cache_refill(nc, gfp_mask); 5008 if (!page) 5009 return NULL; 5010 5011 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5012 /* if size can vary use size else just use PAGE_SIZE */ 5013 size = nc->size; 5014 #endif 5015 /* Even if we own the page, we do not use atomic_set(). 5016 * This would break get_page_unless_zero() users. 5017 */ 5018 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 5019 5020 /* reset page count bias and offset to start of new frag */ 5021 nc->pfmemalloc = page_is_pfmemalloc(page); 5022 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5023 nc->offset = size; 5024 } 5025 5026 offset = nc->offset - fragsz; 5027 if (unlikely(offset < 0)) { 5028 page = virt_to_page(nc->va); 5029 5030 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 5031 goto refill; 5032 5033 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5034 /* if size can vary use size else just use PAGE_SIZE */ 5035 size = nc->size; 5036 #endif 5037 /* OK, page count is 0, we can safely set it */ 5038 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 5039 5040 /* reset page count bias and offset to start of new frag */ 5041 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5042 offset = size - fragsz; 5043 } 5044 5045 nc->pagecnt_bias--; 5046 nc->offset = offset; 5047 5048 return nc->va + offset; 5049 } 5050 EXPORT_SYMBOL(page_frag_alloc); 5051 5052 /* 5053 * Frees a page fragment allocated out of either a compound or order 0 page. 5054 */ 5055 void page_frag_free(void *addr) 5056 { 5057 struct page *page = virt_to_head_page(addr); 5058 5059 if (unlikely(put_page_testzero(page))) 5060 free_the_page(page, compound_order(page)); 5061 } 5062 EXPORT_SYMBOL(page_frag_free); 5063 5064 static void *make_alloc_exact(unsigned long addr, unsigned int order, 5065 size_t size) 5066 { 5067 if (addr) { 5068 unsigned long alloc_end = addr + (PAGE_SIZE << order); 5069 unsigned long used = addr + PAGE_ALIGN(size); 5070 5071 split_page(virt_to_page((void *)addr), order); 5072 while (used < alloc_end) { 5073 free_page(used); 5074 used += PAGE_SIZE; 5075 } 5076 } 5077 return (void *)addr; 5078 } 5079 5080 /** 5081 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 5082 * @size: the number of bytes to allocate 5083 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5084 * 5085 * This function is similar to alloc_pages(), except that it allocates the 5086 * minimum number of pages to satisfy the request. alloc_pages() can only 5087 * allocate memory in power-of-two pages. 5088 * 5089 * This function is also limited by MAX_ORDER. 5090 * 5091 * Memory allocated by this function must be released by free_pages_exact(). 5092 * 5093 * Return: pointer to the allocated area or %NULL in case of error. 5094 */ 5095 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 5096 { 5097 unsigned int order = get_order(size); 5098 unsigned long addr; 5099 5100 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 5101 gfp_mask &= ~__GFP_COMP; 5102 5103 addr = __get_free_pages(gfp_mask, order); 5104 return make_alloc_exact(addr, order, size); 5105 } 5106 EXPORT_SYMBOL(alloc_pages_exact); 5107 5108 /** 5109 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 5110 * pages on a node. 5111 * @nid: the preferred node ID where memory should be allocated 5112 * @size: the number of bytes to allocate 5113 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5114 * 5115 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 5116 * back. 5117 * 5118 * Return: pointer to the allocated area or %NULL in case of error. 5119 */ 5120 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 5121 { 5122 unsigned int order = get_order(size); 5123 struct page *p; 5124 5125 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 5126 gfp_mask &= ~__GFP_COMP; 5127 5128 p = alloc_pages_node(nid, gfp_mask, order); 5129 if (!p) 5130 return NULL; 5131 return make_alloc_exact((unsigned long)page_address(p), order, size); 5132 } 5133 5134 /** 5135 * free_pages_exact - release memory allocated via alloc_pages_exact() 5136 * @virt: the value returned by alloc_pages_exact. 5137 * @size: size of allocation, same value as passed to alloc_pages_exact(). 5138 * 5139 * Release the memory allocated by a previous call to alloc_pages_exact. 5140 */ 5141 void free_pages_exact(void *virt, size_t size) 5142 { 5143 unsigned long addr = (unsigned long)virt; 5144 unsigned long end = addr + PAGE_ALIGN(size); 5145 5146 while (addr < end) { 5147 free_page(addr); 5148 addr += PAGE_SIZE; 5149 } 5150 } 5151 EXPORT_SYMBOL(free_pages_exact); 5152 5153 /** 5154 * nr_free_zone_pages - count number of pages beyond high watermark 5155 * @offset: The zone index of the highest zone 5156 * 5157 * nr_free_zone_pages() counts the number of pages which are beyond the 5158 * high watermark within all zones at or below a given zone index. For each 5159 * zone, the number of pages is calculated as: 5160 * 5161 * nr_free_zone_pages = managed_pages - high_pages 5162 * 5163 * Return: number of pages beyond high watermark. 5164 */ 5165 static unsigned long nr_free_zone_pages(int offset) 5166 { 5167 struct zoneref *z; 5168 struct zone *zone; 5169 5170 /* Just pick one node, since fallback list is circular */ 5171 unsigned long sum = 0; 5172 5173 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 5174 5175 for_each_zone_zonelist(zone, z, zonelist, offset) { 5176 unsigned long size = zone_managed_pages(zone); 5177 unsigned long high = high_wmark_pages(zone); 5178 if (size > high) 5179 sum += size - high; 5180 } 5181 5182 return sum; 5183 } 5184 5185 /** 5186 * nr_free_buffer_pages - count number of pages beyond high watermark 5187 * 5188 * nr_free_buffer_pages() counts the number of pages which are beyond the high 5189 * watermark within ZONE_DMA and ZONE_NORMAL. 5190 * 5191 * Return: number of pages beyond high watermark within ZONE_DMA and 5192 * ZONE_NORMAL. 5193 */ 5194 unsigned long nr_free_buffer_pages(void) 5195 { 5196 return nr_free_zone_pages(gfp_zone(GFP_USER)); 5197 } 5198 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 5199 5200 static inline void show_node(struct zone *zone) 5201 { 5202 if (IS_ENABLED(CONFIG_NUMA)) 5203 printk("Node %d ", zone_to_nid(zone)); 5204 } 5205 5206 long si_mem_available(void) 5207 { 5208 long available; 5209 unsigned long pagecache; 5210 unsigned long wmark_low = 0; 5211 unsigned long pages[NR_LRU_LISTS]; 5212 unsigned long reclaimable; 5213 struct zone *zone; 5214 int lru; 5215 5216 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 5217 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 5218 5219 for_each_zone(zone) 5220 wmark_low += low_wmark_pages(zone); 5221 5222 /* 5223 * Estimate the amount of memory available for userspace allocations, 5224 * without causing swapping. 5225 */ 5226 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages; 5227 5228 /* 5229 * Not all the page cache can be freed, otherwise the system will 5230 * start swapping. Assume at least half of the page cache, or the 5231 * low watermark worth of cache, needs to stay. 5232 */ 5233 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 5234 pagecache -= min(pagecache / 2, wmark_low); 5235 available += pagecache; 5236 5237 /* 5238 * Part of the reclaimable slab and other kernel memory consists of 5239 * items that are in use, and cannot be freed. Cap this estimate at the 5240 * low watermark. 5241 */ 5242 reclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B) + 5243 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE); 5244 available += reclaimable - min(reclaimable / 2, wmark_low); 5245 5246 if (available < 0) 5247 available = 0; 5248 return available; 5249 } 5250 EXPORT_SYMBOL_GPL(si_mem_available); 5251 5252 void si_meminfo(struct sysinfo *val) 5253 { 5254 val->totalram = totalram_pages(); 5255 val->sharedram = global_node_page_state(NR_SHMEM); 5256 val->freeram = global_zone_page_state(NR_FREE_PAGES); 5257 val->bufferram = nr_blockdev_pages(); 5258 val->totalhigh = totalhigh_pages(); 5259 val->freehigh = nr_free_highpages(); 5260 val->mem_unit = PAGE_SIZE; 5261 } 5262 5263 EXPORT_SYMBOL(si_meminfo); 5264 5265 #ifdef CONFIG_NUMA 5266 void si_meminfo_node(struct sysinfo *val, int nid) 5267 { 5268 int zone_type; /* needs to be signed */ 5269 unsigned long managed_pages = 0; 5270 unsigned long managed_highpages = 0; 5271 unsigned long free_highpages = 0; 5272 pg_data_t *pgdat = NODE_DATA(nid); 5273 5274 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 5275 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]); 5276 val->totalram = managed_pages; 5277 val->sharedram = node_page_state(pgdat, NR_SHMEM); 5278 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 5279 #ifdef CONFIG_HIGHMEM 5280 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 5281 struct zone *zone = &pgdat->node_zones[zone_type]; 5282 5283 if (is_highmem(zone)) { 5284 managed_highpages += zone_managed_pages(zone); 5285 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 5286 } 5287 } 5288 val->totalhigh = managed_highpages; 5289 val->freehigh = free_highpages; 5290 #else 5291 val->totalhigh = managed_highpages; 5292 val->freehigh = free_highpages; 5293 #endif 5294 val->mem_unit = PAGE_SIZE; 5295 } 5296 #endif 5297 5298 /* 5299 * Determine whether the node should be displayed or not, depending on whether 5300 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 5301 */ 5302 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) 5303 { 5304 if (!(flags & SHOW_MEM_FILTER_NODES)) 5305 return false; 5306 5307 /* 5308 * no node mask - aka implicit memory numa policy. Do not bother with 5309 * the synchronization - read_mems_allowed_begin - because we do not 5310 * have to be precise here. 5311 */ 5312 if (!nodemask) 5313 nodemask = &cpuset_current_mems_allowed; 5314 5315 return !node_isset(nid, *nodemask); 5316 } 5317 5318 #define K(x) ((x) << (PAGE_SHIFT-10)) 5319 5320 static void show_migration_types(unsigned char type) 5321 { 5322 static const char types[MIGRATE_TYPES] = { 5323 [MIGRATE_UNMOVABLE] = 'U', 5324 [MIGRATE_MOVABLE] = 'M', 5325 [MIGRATE_RECLAIMABLE] = 'E', 5326 [MIGRATE_HIGHATOMIC] = 'H', 5327 #ifdef CONFIG_CMA 5328 [MIGRATE_CMA] = 'C', 5329 #endif 5330 #ifdef CONFIG_MEMORY_ISOLATION 5331 [MIGRATE_ISOLATE] = 'I', 5332 #endif 5333 }; 5334 char tmp[MIGRATE_TYPES + 1]; 5335 char *p = tmp; 5336 int i; 5337 5338 for (i = 0; i < MIGRATE_TYPES; i++) { 5339 if (type & (1 << i)) 5340 *p++ = types[i]; 5341 } 5342 5343 *p = '\0'; 5344 printk(KERN_CONT "(%s) ", tmp); 5345 } 5346 5347 /* 5348 * Show free area list (used inside shift_scroll-lock stuff) 5349 * We also calculate the percentage fragmentation. We do this by counting the 5350 * memory on each free list with the exception of the first item on the list. 5351 * 5352 * Bits in @filter: 5353 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 5354 * cpuset. 5355 */ 5356 void show_free_areas(unsigned int filter, nodemask_t *nodemask) 5357 { 5358 unsigned long free_pcp = 0; 5359 int cpu; 5360 struct zone *zone; 5361 pg_data_t *pgdat; 5362 5363 for_each_populated_zone(zone) { 5364 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5365 continue; 5366 5367 for_each_online_cpu(cpu) 5368 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 5369 } 5370 5371 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 5372 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 5373 " unevictable:%lu dirty:%lu writeback:%lu\n" 5374 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 5375 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 5376 " free:%lu free_pcp:%lu free_cma:%lu\n", 5377 global_node_page_state(NR_ACTIVE_ANON), 5378 global_node_page_state(NR_INACTIVE_ANON), 5379 global_node_page_state(NR_ISOLATED_ANON), 5380 global_node_page_state(NR_ACTIVE_FILE), 5381 global_node_page_state(NR_INACTIVE_FILE), 5382 global_node_page_state(NR_ISOLATED_FILE), 5383 global_node_page_state(NR_UNEVICTABLE), 5384 global_node_page_state(NR_FILE_DIRTY), 5385 global_node_page_state(NR_WRITEBACK), 5386 global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B), 5387 global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B), 5388 global_node_page_state(NR_FILE_MAPPED), 5389 global_node_page_state(NR_SHMEM), 5390 global_zone_page_state(NR_PAGETABLE), 5391 global_zone_page_state(NR_BOUNCE), 5392 global_zone_page_state(NR_FREE_PAGES), 5393 free_pcp, 5394 global_zone_page_state(NR_FREE_CMA_PAGES)); 5395 5396 for_each_online_pgdat(pgdat) { 5397 if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) 5398 continue; 5399 5400 printk("Node %d" 5401 " active_anon:%lukB" 5402 " inactive_anon:%lukB" 5403 " active_file:%lukB" 5404 " inactive_file:%lukB" 5405 " unevictable:%lukB" 5406 " isolated(anon):%lukB" 5407 " isolated(file):%lukB" 5408 " mapped:%lukB" 5409 " dirty:%lukB" 5410 " writeback:%lukB" 5411 " shmem:%lukB" 5412 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5413 " shmem_thp: %lukB" 5414 " shmem_pmdmapped: %lukB" 5415 " anon_thp: %lukB" 5416 #endif 5417 " writeback_tmp:%lukB" 5418 " kernel_stack:%lukB" 5419 #ifdef CONFIG_SHADOW_CALL_STACK 5420 " shadow_call_stack:%lukB" 5421 #endif 5422 " all_unreclaimable? %s" 5423 "\n", 5424 pgdat->node_id, 5425 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 5426 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 5427 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 5428 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 5429 K(node_page_state(pgdat, NR_UNEVICTABLE)), 5430 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 5431 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 5432 K(node_page_state(pgdat, NR_FILE_MAPPED)), 5433 K(node_page_state(pgdat, NR_FILE_DIRTY)), 5434 K(node_page_state(pgdat, NR_WRITEBACK)), 5435 K(node_page_state(pgdat, NR_SHMEM)), 5436 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5437 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), 5438 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) 5439 * HPAGE_PMD_NR), 5440 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR), 5441 #endif 5442 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 5443 node_page_state(pgdat, NR_KERNEL_STACK_KB), 5444 #ifdef CONFIG_SHADOW_CALL_STACK 5445 node_page_state(pgdat, NR_KERNEL_SCS_KB), 5446 #endif 5447 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ? 5448 "yes" : "no"); 5449 } 5450 5451 for_each_populated_zone(zone) { 5452 int i; 5453 5454 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5455 continue; 5456 5457 free_pcp = 0; 5458 for_each_online_cpu(cpu) 5459 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 5460 5461 show_node(zone); 5462 printk(KERN_CONT 5463 "%s" 5464 " free:%lukB" 5465 " min:%lukB" 5466 " low:%lukB" 5467 " high:%lukB" 5468 " reserved_highatomic:%luKB" 5469 " active_anon:%lukB" 5470 " inactive_anon:%lukB" 5471 " active_file:%lukB" 5472 " inactive_file:%lukB" 5473 " unevictable:%lukB" 5474 " writepending:%lukB" 5475 " present:%lukB" 5476 " managed:%lukB" 5477 " mlocked:%lukB" 5478 " pagetables:%lukB" 5479 " bounce:%lukB" 5480 " free_pcp:%lukB" 5481 " local_pcp:%ukB" 5482 " free_cma:%lukB" 5483 "\n", 5484 zone->name, 5485 K(zone_page_state(zone, NR_FREE_PAGES)), 5486 K(min_wmark_pages(zone)), 5487 K(low_wmark_pages(zone)), 5488 K(high_wmark_pages(zone)), 5489 K(zone->nr_reserved_highatomic), 5490 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 5491 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 5492 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 5493 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 5494 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 5495 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 5496 K(zone->present_pages), 5497 K(zone_managed_pages(zone)), 5498 K(zone_page_state(zone, NR_MLOCK)), 5499 K(zone_page_state(zone, NR_PAGETABLE)), 5500 K(zone_page_state(zone, NR_BOUNCE)), 5501 K(free_pcp), 5502 K(this_cpu_read(zone->pageset->pcp.count)), 5503 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 5504 printk("lowmem_reserve[]:"); 5505 for (i = 0; i < MAX_NR_ZONES; i++) 5506 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); 5507 printk(KERN_CONT "\n"); 5508 } 5509 5510 for_each_populated_zone(zone) { 5511 unsigned int order; 5512 unsigned long nr[MAX_ORDER], flags, total = 0; 5513 unsigned char types[MAX_ORDER]; 5514 5515 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5516 continue; 5517 show_node(zone); 5518 printk(KERN_CONT "%s: ", zone->name); 5519 5520 spin_lock_irqsave(&zone->lock, flags); 5521 for (order = 0; order < MAX_ORDER; order++) { 5522 struct free_area *area = &zone->free_area[order]; 5523 int type; 5524 5525 nr[order] = area->nr_free; 5526 total += nr[order] << order; 5527 5528 types[order] = 0; 5529 for (type = 0; type < MIGRATE_TYPES; type++) { 5530 if (!free_area_empty(area, type)) 5531 types[order] |= 1 << type; 5532 } 5533 } 5534 spin_unlock_irqrestore(&zone->lock, flags); 5535 for (order = 0; order < MAX_ORDER; order++) { 5536 printk(KERN_CONT "%lu*%lukB ", 5537 nr[order], K(1UL) << order); 5538 if (nr[order]) 5539 show_migration_types(types[order]); 5540 } 5541 printk(KERN_CONT "= %lukB\n", K(total)); 5542 } 5543 5544 hugetlb_show_meminfo(); 5545 5546 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 5547 5548 show_swap_cache_info(); 5549 } 5550 5551 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 5552 { 5553 zoneref->zone = zone; 5554 zoneref->zone_idx = zone_idx(zone); 5555 } 5556 5557 /* 5558 * Builds allocation fallback zone lists. 5559 * 5560 * Add all populated zones of a node to the zonelist. 5561 */ 5562 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) 5563 { 5564 struct zone *zone; 5565 enum zone_type zone_type = MAX_NR_ZONES; 5566 int nr_zones = 0; 5567 5568 do { 5569 zone_type--; 5570 zone = pgdat->node_zones + zone_type; 5571 if (managed_zone(zone)) { 5572 zoneref_set_zone(zone, &zonerefs[nr_zones++]); 5573 check_highest_zone(zone_type); 5574 } 5575 } while (zone_type); 5576 5577 return nr_zones; 5578 } 5579 5580 #ifdef CONFIG_NUMA 5581 5582 static int __parse_numa_zonelist_order(char *s) 5583 { 5584 /* 5585 * We used to support different zonlists modes but they turned 5586 * out to be just not useful. Let's keep the warning in place 5587 * if somebody still use the cmd line parameter so that we do 5588 * not fail it silently 5589 */ 5590 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) { 5591 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s); 5592 return -EINVAL; 5593 } 5594 return 0; 5595 } 5596 5597 char numa_zonelist_order[] = "Node"; 5598 5599 /* 5600 * sysctl handler for numa_zonelist_order 5601 */ 5602 int numa_zonelist_order_handler(struct ctl_table *table, int write, 5603 void *buffer, size_t *length, loff_t *ppos) 5604 { 5605 if (write) 5606 return __parse_numa_zonelist_order(buffer); 5607 return proc_dostring(table, write, buffer, length, ppos); 5608 } 5609 5610 5611 #define MAX_NODE_LOAD (nr_online_nodes) 5612 static int node_load[MAX_NUMNODES]; 5613 5614 /** 5615 * find_next_best_node - find the next node that should appear in a given node's fallback list 5616 * @node: node whose fallback list we're appending 5617 * @used_node_mask: nodemask_t of already used nodes 5618 * 5619 * We use a number of factors to determine which is the next node that should 5620 * appear on a given node's fallback list. The node should not have appeared 5621 * already in @node's fallback list, and it should be the next closest node 5622 * according to the distance array (which contains arbitrary distance values 5623 * from each node to each node in the system), and should also prefer nodes 5624 * with no CPUs, since presumably they'll have very little allocation pressure 5625 * on them otherwise. 5626 * 5627 * Return: node id of the found node or %NUMA_NO_NODE if no node is found. 5628 */ 5629 static int find_next_best_node(int node, nodemask_t *used_node_mask) 5630 { 5631 int n, val; 5632 int min_val = INT_MAX; 5633 int best_node = NUMA_NO_NODE; 5634 const struct cpumask *tmp = cpumask_of_node(0); 5635 5636 /* Use the local node if we haven't already */ 5637 if (!node_isset(node, *used_node_mask)) { 5638 node_set(node, *used_node_mask); 5639 return node; 5640 } 5641 5642 for_each_node_state(n, N_MEMORY) { 5643 5644 /* Don't want a node to appear more than once */ 5645 if (node_isset(n, *used_node_mask)) 5646 continue; 5647 5648 /* Use the distance array to find the distance */ 5649 val = node_distance(node, n); 5650 5651 /* Penalize nodes under us ("prefer the next node") */ 5652 val += (n < node); 5653 5654 /* Give preference to headless and unused nodes */ 5655 tmp = cpumask_of_node(n); 5656 if (!cpumask_empty(tmp)) 5657 val += PENALTY_FOR_NODE_WITH_CPUS; 5658 5659 /* Slight preference for less loaded node */ 5660 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 5661 val += node_load[n]; 5662 5663 if (val < min_val) { 5664 min_val = val; 5665 best_node = n; 5666 } 5667 } 5668 5669 if (best_node >= 0) 5670 node_set(best_node, *used_node_mask); 5671 5672 return best_node; 5673 } 5674 5675 5676 /* 5677 * Build zonelists ordered by node and zones within node. 5678 * This results in maximum locality--normal zone overflows into local 5679 * DMA zone, if any--but risks exhausting DMA zone. 5680 */ 5681 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, 5682 unsigned nr_nodes) 5683 { 5684 struct zoneref *zonerefs; 5685 int i; 5686 5687 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5688 5689 for (i = 0; i < nr_nodes; i++) { 5690 int nr_zones; 5691 5692 pg_data_t *node = NODE_DATA(node_order[i]); 5693 5694 nr_zones = build_zonerefs_node(node, zonerefs); 5695 zonerefs += nr_zones; 5696 } 5697 zonerefs->zone = NULL; 5698 zonerefs->zone_idx = 0; 5699 } 5700 5701 /* 5702 * Build gfp_thisnode zonelists 5703 */ 5704 static void build_thisnode_zonelists(pg_data_t *pgdat) 5705 { 5706 struct zoneref *zonerefs; 5707 int nr_zones; 5708 5709 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; 5710 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5711 zonerefs += nr_zones; 5712 zonerefs->zone = NULL; 5713 zonerefs->zone_idx = 0; 5714 } 5715 5716 /* 5717 * Build zonelists ordered by zone and nodes within zones. 5718 * This results in conserving DMA zone[s] until all Normal memory is 5719 * exhausted, but results in overflowing to remote node while memory 5720 * may still exist in local DMA zone. 5721 */ 5722 5723 static void build_zonelists(pg_data_t *pgdat) 5724 { 5725 static int node_order[MAX_NUMNODES]; 5726 int node, load, nr_nodes = 0; 5727 nodemask_t used_mask = NODE_MASK_NONE; 5728 int local_node, prev_node; 5729 5730 /* NUMA-aware ordering of nodes */ 5731 local_node = pgdat->node_id; 5732 load = nr_online_nodes; 5733 prev_node = local_node; 5734 5735 memset(node_order, 0, sizeof(node_order)); 5736 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 5737 /* 5738 * We don't want to pressure a particular node. 5739 * So adding penalty to the first node in same 5740 * distance group to make it round-robin. 5741 */ 5742 if (node_distance(local_node, node) != 5743 node_distance(local_node, prev_node)) 5744 node_load[node] = load; 5745 5746 node_order[nr_nodes++] = node; 5747 prev_node = node; 5748 load--; 5749 } 5750 5751 build_zonelists_in_node_order(pgdat, node_order, nr_nodes); 5752 build_thisnode_zonelists(pgdat); 5753 } 5754 5755 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5756 /* 5757 * Return node id of node used for "local" allocations. 5758 * I.e., first node id of first zone in arg node's generic zonelist. 5759 * Used for initializing percpu 'numa_mem', which is used primarily 5760 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 5761 */ 5762 int local_memory_node(int node) 5763 { 5764 struct zoneref *z; 5765 5766 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 5767 gfp_zone(GFP_KERNEL), 5768 NULL); 5769 return zone_to_nid(z->zone); 5770 } 5771 #endif 5772 5773 static void setup_min_unmapped_ratio(void); 5774 static void setup_min_slab_ratio(void); 5775 #else /* CONFIG_NUMA */ 5776 5777 static void build_zonelists(pg_data_t *pgdat) 5778 { 5779 int node, local_node; 5780 struct zoneref *zonerefs; 5781 int nr_zones; 5782 5783 local_node = pgdat->node_id; 5784 5785 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5786 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5787 zonerefs += nr_zones; 5788 5789 /* 5790 * Now we build the zonelist so that it contains the zones 5791 * of all the other nodes. 5792 * We don't want to pressure a particular node, so when 5793 * building the zones for node N, we make sure that the 5794 * zones coming right after the local ones are those from 5795 * node N+1 (modulo N) 5796 */ 5797 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 5798 if (!node_online(node)) 5799 continue; 5800 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5801 zonerefs += nr_zones; 5802 } 5803 for (node = 0; node < local_node; node++) { 5804 if (!node_online(node)) 5805 continue; 5806 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5807 zonerefs += nr_zones; 5808 } 5809 5810 zonerefs->zone = NULL; 5811 zonerefs->zone_idx = 0; 5812 } 5813 5814 #endif /* CONFIG_NUMA */ 5815 5816 /* 5817 * Boot pageset table. One per cpu which is going to be used for all 5818 * zones and all nodes. The parameters will be set in such a way 5819 * that an item put on a list will immediately be handed over to 5820 * the buddy list. This is safe since pageset manipulation is done 5821 * with interrupts disabled. 5822 * 5823 * The boot_pagesets must be kept even after bootup is complete for 5824 * unused processors and/or zones. They do play a role for bootstrapping 5825 * hotplugged processors. 5826 * 5827 * zoneinfo_show() and maybe other functions do 5828 * not check if the processor is online before following the pageset pointer. 5829 * Other parts of the kernel may not check if the zone is available. 5830 */ 5831 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); 5832 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); 5833 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 5834 5835 static void __build_all_zonelists(void *data) 5836 { 5837 int nid; 5838 int __maybe_unused cpu; 5839 pg_data_t *self = data; 5840 static DEFINE_SPINLOCK(lock); 5841 5842 spin_lock(&lock); 5843 5844 #ifdef CONFIG_NUMA 5845 memset(node_load, 0, sizeof(node_load)); 5846 #endif 5847 5848 /* 5849 * This node is hotadded and no memory is yet present. So just 5850 * building zonelists is fine - no need to touch other nodes. 5851 */ 5852 if (self && !node_online(self->node_id)) { 5853 build_zonelists(self); 5854 } else { 5855 for_each_online_node(nid) { 5856 pg_data_t *pgdat = NODE_DATA(nid); 5857 5858 build_zonelists(pgdat); 5859 } 5860 5861 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5862 /* 5863 * We now know the "local memory node" for each node-- 5864 * i.e., the node of the first zone in the generic zonelist. 5865 * Set up numa_mem percpu variable for on-line cpus. During 5866 * boot, only the boot cpu should be on-line; we'll init the 5867 * secondary cpus' numa_mem as they come on-line. During 5868 * node/memory hotplug, we'll fixup all on-line cpus. 5869 */ 5870 for_each_online_cpu(cpu) 5871 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 5872 #endif 5873 } 5874 5875 spin_unlock(&lock); 5876 } 5877 5878 static noinline void __init 5879 build_all_zonelists_init(void) 5880 { 5881 int cpu; 5882 5883 __build_all_zonelists(NULL); 5884 5885 /* 5886 * Initialize the boot_pagesets that are going to be used 5887 * for bootstrapping processors. The real pagesets for 5888 * each zone will be allocated later when the per cpu 5889 * allocator is available. 5890 * 5891 * boot_pagesets are used also for bootstrapping offline 5892 * cpus if the system is already booted because the pagesets 5893 * are needed to initialize allocators on a specific cpu too. 5894 * F.e. the percpu allocator needs the page allocator which 5895 * needs the percpu allocator in order to allocate its pagesets 5896 * (a chicken-egg dilemma). 5897 */ 5898 for_each_possible_cpu(cpu) 5899 setup_pageset(&per_cpu(boot_pageset, cpu), 0); 5900 5901 mminit_verify_zonelist(); 5902 cpuset_init_current_mems_allowed(); 5903 } 5904 5905 /* 5906 * unless system_state == SYSTEM_BOOTING. 5907 * 5908 * __ref due to call of __init annotated helper build_all_zonelists_init 5909 * [protected by SYSTEM_BOOTING]. 5910 */ 5911 void __ref build_all_zonelists(pg_data_t *pgdat) 5912 { 5913 unsigned long vm_total_pages; 5914 5915 if (system_state == SYSTEM_BOOTING) { 5916 build_all_zonelists_init(); 5917 } else { 5918 __build_all_zonelists(pgdat); 5919 /* cpuset refresh routine should be here */ 5920 } 5921 /* Get the number of free pages beyond high watermark in all zones. */ 5922 vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 5923 /* 5924 * Disable grouping by mobility if the number of pages in the 5925 * system is too low to allow the mechanism to work. It would be 5926 * more accurate, but expensive to check per-zone. This check is 5927 * made on memory-hotadd so a system can start with mobility 5928 * disabled and enable it later 5929 */ 5930 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 5931 page_group_by_mobility_disabled = 1; 5932 else 5933 page_group_by_mobility_disabled = 0; 5934 5935 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n", 5936 nr_online_nodes, 5937 page_group_by_mobility_disabled ? "off" : "on", 5938 vm_total_pages); 5939 #ifdef CONFIG_NUMA 5940 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 5941 #endif 5942 } 5943 5944 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 5945 static bool __meminit 5946 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 5947 { 5948 static struct memblock_region *r; 5949 5950 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 5951 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 5952 for_each_memblock(memory, r) { 5953 if (*pfn < memblock_region_memory_end_pfn(r)) 5954 break; 5955 } 5956 } 5957 if (*pfn >= memblock_region_memory_base_pfn(r) && 5958 memblock_is_mirror(r)) { 5959 *pfn = memblock_region_memory_end_pfn(r); 5960 return true; 5961 } 5962 } 5963 return false; 5964 } 5965 5966 /* 5967 * Initially all pages are reserved - free ones are freed 5968 * up by memblock_free_all() once the early boot process is 5969 * done. Non-atomic initialization, single-pass. 5970 */ 5971 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, 5972 unsigned long start_pfn, enum memmap_context context, 5973 struct vmem_altmap *altmap) 5974 { 5975 unsigned long pfn, end_pfn = start_pfn + size; 5976 struct page *page; 5977 5978 if (highest_memmap_pfn < end_pfn - 1) 5979 highest_memmap_pfn = end_pfn - 1; 5980 5981 #ifdef CONFIG_ZONE_DEVICE 5982 /* 5983 * Honor reservation requested by the driver for this ZONE_DEVICE 5984 * memory. We limit the total number of pages to initialize to just 5985 * those that might contain the memory mapping. We will defer the 5986 * ZONE_DEVICE page initialization until after we have released 5987 * the hotplug lock. 5988 */ 5989 if (zone == ZONE_DEVICE) { 5990 if (!altmap) 5991 return; 5992 5993 if (start_pfn == altmap->base_pfn) 5994 start_pfn += altmap->reserve; 5995 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 5996 } 5997 #endif 5998 5999 for (pfn = start_pfn; pfn < end_pfn; ) { 6000 /* 6001 * There can be holes in boot-time mem_map[]s handed to this 6002 * function. They do not exist on hotplugged memory. 6003 */ 6004 if (context == MEMMAP_EARLY) { 6005 if (overlap_memmap_init(zone, &pfn)) 6006 continue; 6007 if (defer_init(nid, pfn, end_pfn)) 6008 break; 6009 } 6010 6011 page = pfn_to_page(pfn); 6012 __init_single_page(page, pfn, zone, nid); 6013 if (context == MEMMAP_HOTPLUG) 6014 __SetPageReserved(page); 6015 6016 /* 6017 * Mark the block movable so that blocks are reserved for 6018 * movable at startup. This will force kernel allocations 6019 * to reserve their blocks rather than leaking throughout 6020 * the address space during boot when many long-lived 6021 * kernel allocations are made. 6022 * 6023 * bitmap is created for zone's valid pfn range. but memmap 6024 * can be created for invalid pages (for alignment) 6025 * check here not to call set_pageblock_migratetype() against 6026 * pfn out of zone. 6027 */ 6028 if (!(pfn & (pageblock_nr_pages - 1))) { 6029 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 6030 cond_resched(); 6031 } 6032 pfn++; 6033 } 6034 } 6035 6036 #ifdef CONFIG_ZONE_DEVICE 6037 void __ref memmap_init_zone_device(struct zone *zone, 6038 unsigned long start_pfn, 6039 unsigned long nr_pages, 6040 struct dev_pagemap *pgmap) 6041 { 6042 unsigned long pfn, end_pfn = start_pfn + nr_pages; 6043 struct pglist_data *pgdat = zone->zone_pgdat; 6044 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 6045 unsigned long zone_idx = zone_idx(zone); 6046 unsigned long start = jiffies; 6047 int nid = pgdat->node_id; 6048 6049 if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE)) 6050 return; 6051 6052 /* 6053 * The call to memmap_init_zone should have already taken care 6054 * of the pages reserved for the memmap, so we can just jump to 6055 * the end of that region and start processing the device pages. 6056 */ 6057 if (altmap) { 6058 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6059 nr_pages = end_pfn - start_pfn; 6060 } 6061 6062 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 6063 struct page *page = pfn_to_page(pfn); 6064 6065 __init_single_page(page, pfn, zone_idx, nid); 6066 6067 /* 6068 * Mark page reserved as it will need to wait for onlining 6069 * phase for it to be fully associated with a zone. 6070 * 6071 * We can use the non-atomic __set_bit operation for setting 6072 * the flag as we are still initializing the pages. 6073 */ 6074 __SetPageReserved(page); 6075 6076 /* 6077 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 6078 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 6079 * ever freed or placed on a driver-private list. 6080 */ 6081 page->pgmap = pgmap; 6082 page->zone_device_data = NULL; 6083 6084 /* 6085 * Mark the block movable so that blocks are reserved for 6086 * movable at startup. This will force kernel allocations 6087 * to reserve their blocks rather than leaking throughout 6088 * the address space during boot when many long-lived 6089 * kernel allocations are made. 6090 * 6091 * bitmap is created for zone's valid pfn range. but memmap 6092 * can be created for invalid pages (for alignment) 6093 * check here not to call set_pageblock_migratetype() against 6094 * pfn out of zone. 6095 * 6096 * Please note that MEMMAP_HOTPLUG path doesn't clear memmap 6097 * because this is done early in section_activate() 6098 */ 6099 if (!(pfn & (pageblock_nr_pages - 1))) { 6100 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 6101 cond_resched(); 6102 } 6103 } 6104 6105 pr_info("%s initialised %lu pages in %ums\n", __func__, 6106 nr_pages, jiffies_to_msecs(jiffies - start)); 6107 } 6108 6109 #endif 6110 static void __meminit zone_init_free_lists(struct zone *zone) 6111 { 6112 unsigned int order, t; 6113 for_each_migratetype_order(order, t) { 6114 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 6115 zone->free_area[order].nr_free = 0; 6116 } 6117 } 6118 6119 void __meminit __weak memmap_init(unsigned long size, int nid, 6120 unsigned long zone, 6121 unsigned long range_start_pfn) 6122 { 6123 unsigned long start_pfn, end_pfn; 6124 unsigned long range_end_pfn = range_start_pfn + size; 6125 int i; 6126 6127 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6128 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 6129 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 6130 6131 if (end_pfn > start_pfn) { 6132 size = end_pfn - start_pfn; 6133 memmap_init_zone(size, nid, zone, start_pfn, 6134 MEMMAP_EARLY, NULL); 6135 } 6136 } 6137 } 6138 6139 static int zone_batchsize(struct zone *zone) 6140 { 6141 #ifdef CONFIG_MMU 6142 int batch; 6143 6144 /* 6145 * The per-cpu-pages pools are set to around 1000th of the 6146 * size of the zone. 6147 */ 6148 batch = zone_managed_pages(zone) / 1024; 6149 /* But no more than a meg. */ 6150 if (batch * PAGE_SIZE > 1024 * 1024) 6151 batch = (1024 * 1024) / PAGE_SIZE; 6152 batch /= 4; /* We effectively *= 4 below */ 6153 if (batch < 1) 6154 batch = 1; 6155 6156 /* 6157 * Clamp the batch to a 2^n - 1 value. Having a power 6158 * of 2 value was found to be more likely to have 6159 * suboptimal cache aliasing properties in some cases. 6160 * 6161 * For example if 2 tasks are alternately allocating 6162 * batches of pages, one task can end up with a lot 6163 * of pages of one half of the possible page colors 6164 * and the other with pages of the other colors. 6165 */ 6166 batch = rounddown_pow_of_two(batch + batch/2) - 1; 6167 6168 return batch; 6169 6170 #else 6171 /* The deferral and batching of frees should be suppressed under NOMMU 6172 * conditions. 6173 * 6174 * The problem is that NOMMU needs to be able to allocate large chunks 6175 * of contiguous memory as there's no hardware page translation to 6176 * assemble apparent contiguous memory from discontiguous pages. 6177 * 6178 * Queueing large contiguous runs of pages for batching, however, 6179 * causes the pages to actually be freed in smaller chunks. As there 6180 * can be a significant delay between the individual batches being 6181 * recycled, this leads to the once large chunks of space being 6182 * fragmented and becoming unavailable for high-order allocations. 6183 */ 6184 return 0; 6185 #endif 6186 } 6187 6188 /* 6189 * pcp->high and pcp->batch values are related and dependent on one another: 6190 * ->batch must never be higher then ->high. 6191 * The following function updates them in a safe manner without read side 6192 * locking. 6193 * 6194 * Any new users of pcp->batch and pcp->high should ensure they can cope with 6195 * those fields changing asynchronously (acording to the above rule). 6196 * 6197 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 6198 * outside of boot time (or some other assurance that no concurrent updaters 6199 * exist). 6200 */ 6201 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 6202 unsigned long batch) 6203 { 6204 /* start with a fail safe value for batch */ 6205 pcp->batch = 1; 6206 smp_wmb(); 6207 6208 /* Update high, then batch, in order */ 6209 pcp->high = high; 6210 smp_wmb(); 6211 6212 pcp->batch = batch; 6213 } 6214 6215 /* a companion to pageset_set_high() */ 6216 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) 6217 { 6218 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); 6219 } 6220 6221 static void pageset_init(struct per_cpu_pageset *p) 6222 { 6223 struct per_cpu_pages *pcp; 6224 int migratetype; 6225 6226 memset(p, 0, sizeof(*p)); 6227 6228 pcp = &p->pcp; 6229 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) 6230 INIT_LIST_HEAD(&pcp->lists[migratetype]); 6231 } 6232 6233 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 6234 { 6235 pageset_init(p); 6236 pageset_set_batch(p, batch); 6237 } 6238 6239 /* 6240 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist 6241 * to the value high for the pageset p. 6242 */ 6243 static void pageset_set_high(struct per_cpu_pageset *p, 6244 unsigned long high) 6245 { 6246 unsigned long batch = max(1UL, high / 4); 6247 if ((high / 4) > (PAGE_SHIFT * 8)) 6248 batch = PAGE_SHIFT * 8; 6249 6250 pageset_update(&p->pcp, high, batch); 6251 } 6252 6253 static void pageset_set_high_and_batch(struct zone *zone, 6254 struct per_cpu_pageset *pcp) 6255 { 6256 if (percpu_pagelist_fraction) 6257 pageset_set_high(pcp, 6258 (zone_managed_pages(zone) / 6259 percpu_pagelist_fraction)); 6260 else 6261 pageset_set_batch(pcp, zone_batchsize(zone)); 6262 } 6263 6264 static void __meminit zone_pageset_init(struct zone *zone, int cpu) 6265 { 6266 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); 6267 6268 pageset_init(pcp); 6269 pageset_set_high_and_batch(zone, pcp); 6270 } 6271 6272 void __meminit setup_zone_pageset(struct zone *zone) 6273 { 6274 int cpu; 6275 zone->pageset = alloc_percpu(struct per_cpu_pageset); 6276 for_each_possible_cpu(cpu) 6277 zone_pageset_init(zone, cpu); 6278 } 6279 6280 /* 6281 * Allocate per cpu pagesets and initialize them. 6282 * Before this call only boot pagesets were available. 6283 */ 6284 void __init setup_per_cpu_pageset(void) 6285 { 6286 struct pglist_data *pgdat; 6287 struct zone *zone; 6288 int __maybe_unused cpu; 6289 6290 for_each_populated_zone(zone) 6291 setup_zone_pageset(zone); 6292 6293 #ifdef CONFIG_NUMA 6294 /* 6295 * Unpopulated zones continue using the boot pagesets. 6296 * The numa stats for these pagesets need to be reset. 6297 * Otherwise, they will end up skewing the stats of 6298 * the nodes these zones are associated with. 6299 */ 6300 for_each_possible_cpu(cpu) { 6301 struct per_cpu_pageset *pcp = &per_cpu(boot_pageset, cpu); 6302 memset(pcp->vm_numa_stat_diff, 0, 6303 sizeof(pcp->vm_numa_stat_diff)); 6304 } 6305 #endif 6306 6307 for_each_online_pgdat(pgdat) 6308 pgdat->per_cpu_nodestats = 6309 alloc_percpu(struct per_cpu_nodestat); 6310 } 6311 6312 static __meminit void zone_pcp_init(struct zone *zone) 6313 { 6314 /* 6315 * per cpu subsystem is not up at this point. The following code 6316 * relies on the ability of the linker to provide the 6317 * offset of a (static) per cpu variable into the per cpu area. 6318 */ 6319 zone->pageset = &boot_pageset; 6320 6321 if (populated_zone(zone)) 6322 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", 6323 zone->name, zone->present_pages, 6324 zone_batchsize(zone)); 6325 } 6326 6327 void __meminit init_currently_empty_zone(struct zone *zone, 6328 unsigned long zone_start_pfn, 6329 unsigned long size) 6330 { 6331 struct pglist_data *pgdat = zone->zone_pgdat; 6332 int zone_idx = zone_idx(zone) + 1; 6333 6334 if (zone_idx > pgdat->nr_zones) 6335 pgdat->nr_zones = zone_idx; 6336 6337 zone->zone_start_pfn = zone_start_pfn; 6338 6339 mminit_dprintk(MMINIT_TRACE, "memmap_init", 6340 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 6341 pgdat->node_id, 6342 (unsigned long)zone_idx(zone), 6343 zone_start_pfn, (zone_start_pfn + size)); 6344 6345 zone_init_free_lists(zone); 6346 zone->initialized = 1; 6347 } 6348 6349 /** 6350 * get_pfn_range_for_nid - Return the start and end page frames for a node 6351 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 6352 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 6353 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 6354 * 6355 * It returns the start and end page frame of a node based on information 6356 * provided by memblock_set_node(). If called for a node 6357 * with no available memory, a warning is printed and the start and end 6358 * PFNs will be 0. 6359 */ 6360 void __init get_pfn_range_for_nid(unsigned int nid, 6361 unsigned long *start_pfn, unsigned long *end_pfn) 6362 { 6363 unsigned long this_start_pfn, this_end_pfn; 6364 int i; 6365 6366 *start_pfn = -1UL; 6367 *end_pfn = 0; 6368 6369 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 6370 *start_pfn = min(*start_pfn, this_start_pfn); 6371 *end_pfn = max(*end_pfn, this_end_pfn); 6372 } 6373 6374 if (*start_pfn == -1UL) 6375 *start_pfn = 0; 6376 } 6377 6378 /* 6379 * This finds a zone that can be used for ZONE_MOVABLE pages. The 6380 * assumption is made that zones within a node are ordered in monotonic 6381 * increasing memory addresses so that the "highest" populated zone is used 6382 */ 6383 static void __init find_usable_zone_for_movable(void) 6384 { 6385 int zone_index; 6386 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 6387 if (zone_index == ZONE_MOVABLE) 6388 continue; 6389 6390 if (arch_zone_highest_possible_pfn[zone_index] > 6391 arch_zone_lowest_possible_pfn[zone_index]) 6392 break; 6393 } 6394 6395 VM_BUG_ON(zone_index == -1); 6396 movable_zone = zone_index; 6397 } 6398 6399 /* 6400 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 6401 * because it is sized independent of architecture. Unlike the other zones, 6402 * the starting point for ZONE_MOVABLE is not fixed. It may be different 6403 * in each node depending on the size of each node and how evenly kernelcore 6404 * is distributed. This helper function adjusts the zone ranges 6405 * provided by the architecture for a given node by using the end of the 6406 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 6407 * zones within a node are in order of monotonic increases memory addresses 6408 */ 6409 static void __init adjust_zone_range_for_zone_movable(int nid, 6410 unsigned long zone_type, 6411 unsigned long node_start_pfn, 6412 unsigned long node_end_pfn, 6413 unsigned long *zone_start_pfn, 6414 unsigned long *zone_end_pfn) 6415 { 6416 /* Only adjust if ZONE_MOVABLE is on this node */ 6417 if (zone_movable_pfn[nid]) { 6418 /* Size ZONE_MOVABLE */ 6419 if (zone_type == ZONE_MOVABLE) { 6420 *zone_start_pfn = zone_movable_pfn[nid]; 6421 *zone_end_pfn = min(node_end_pfn, 6422 arch_zone_highest_possible_pfn[movable_zone]); 6423 6424 /* Adjust for ZONE_MOVABLE starting within this range */ 6425 } else if (!mirrored_kernelcore && 6426 *zone_start_pfn < zone_movable_pfn[nid] && 6427 *zone_end_pfn > zone_movable_pfn[nid]) { 6428 *zone_end_pfn = zone_movable_pfn[nid]; 6429 6430 /* Check if this whole range is within ZONE_MOVABLE */ 6431 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 6432 *zone_start_pfn = *zone_end_pfn; 6433 } 6434 } 6435 6436 /* 6437 * Return the number of pages a zone spans in a node, including holes 6438 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 6439 */ 6440 static unsigned long __init zone_spanned_pages_in_node(int nid, 6441 unsigned long zone_type, 6442 unsigned long node_start_pfn, 6443 unsigned long node_end_pfn, 6444 unsigned long *zone_start_pfn, 6445 unsigned long *zone_end_pfn) 6446 { 6447 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 6448 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 6449 /* When hotadd a new node from cpu_up(), the node should be empty */ 6450 if (!node_start_pfn && !node_end_pfn) 6451 return 0; 6452 6453 /* Get the start and end of the zone */ 6454 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 6455 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 6456 adjust_zone_range_for_zone_movable(nid, zone_type, 6457 node_start_pfn, node_end_pfn, 6458 zone_start_pfn, zone_end_pfn); 6459 6460 /* Check that this node has pages within the zone's required range */ 6461 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 6462 return 0; 6463 6464 /* Move the zone boundaries inside the node if necessary */ 6465 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 6466 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 6467 6468 /* Return the spanned pages */ 6469 return *zone_end_pfn - *zone_start_pfn; 6470 } 6471 6472 /* 6473 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 6474 * then all holes in the requested range will be accounted for. 6475 */ 6476 unsigned long __init __absent_pages_in_range(int nid, 6477 unsigned long range_start_pfn, 6478 unsigned long range_end_pfn) 6479 { 6480 unsigned long nr_absent = range_end_pfn - range_start_pfn; 6481 unsigned long start_pfn, end_pfn; 6482 int i; 6483 6484 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6485 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 6486 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 6487 nr_absent -= end_pfn - start_pfn; 6488 } 6489 return nr_absent; 6490 } 6491 6492 /** 6493 * absent_pages_in_range - Return number of page frames in holes within a range 6494 * @start_pfn: The start PFN to start searching for holes 6495 * @end_pfn: The end PFN to stop searching for holes 6496 * 6497 * Return: the number of pages frames in memory holes within a range. 6498 */ 6499 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 6500 unsigned long end_pfn) 6501 { 6502 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 6503 } 6504 6505 /* Return the number of page frames in holes in a zone on a node */ 6506 static unsigned long __init zone_absent_pages_in_node(int nid, 6507 unsigned long zone_type, 6508 unsigned long node_start_pfn, 6509 unsigned long node_end_pfn) 6510 { 6511 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 6512 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 6513 unsigned long zone_start_pfn, zone_end_pfn; 6514 unsigned long nr_absent; 6515 6516 /* When hotadd a new node from cpu_up(), the node should be empty */ 6517 if (!node_start_pfn && !node_end_pfn) 6518 return 0; 6519 6520 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 6521 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 6522 6523 adjust_zone_range_for_zone_movable(nid, zone_type, 6524 node_start_pfn, node_end_pfn, 6525 &zone_start_pfn, &zone_end_pfn); 6526 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 6527 6528 /* 6529 * ZONE_MOVABLE handling. 6530 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 6531 * and vice versa. 6532 */ 6533 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 6534 unsigned long start_pfn, end_pfn; 6535 struct memblock_region *r; 6536 6537 for_each_memblock(memory, r) { 6538 start_pfn = clamp(memblock_region_memory_base_pfn(r), 6539 zone_start_pfn, zone_end_pfn); 6540 end_pfn = clamp(memblock_region_memory_end_pfn(r), 6541 zone_start_pfn, zone_end_pfn); 6542 6543 if (zone_type == ZONE_MOVABLE && 6544 memblock_is_mirror(r)) 6545 nr_absent += end_pfn - start_pfn; 6546 6547 if (zone_type == ZONE_NORMAL && 6548 !memblock_is_mirror(r)) 6549 nr_absent += end_pfn - start_pfn; 6550 } 6551 } 6552 6553 return nr_absent; 6554 } 6555 6556 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 6557 unsigned long node_start_pfn, 6558 unsigned long node_end_pfn) 6559 { 6560 unsigned long realtotalpages = 0, totalpages = 0; 6561 enum zone_type i; 6562 6563 for (i = 0; i < MAX_NR_ZONES; i++) { 6564 struct zone *zone = pgdat->node_zones + i; 6565 unsigned long zone_start_pfn, zone_end_pfn; 6566 unsigned long spanned, absent; 6567 unsigned long size, real_size; 6568 6569 spanned = zone_spanned_pages_in_node(pgdat->node_id, i, 6570 node_start_pfn, 6571 node_end_pfn, 6572 &zone_start_pfn, 6573 &zone_end_pfn); 6574 absent = zone_absent_pages_in_node(pgdat->node_id, i, 6575 node_start_pfn, 6576 node_end_pfn); 6577 6578 size = spanned; 6579 real_size = size - absent; 6580 6581 if (size) 6582 zone->zone_start_pfn = zone_start_pfn; 6583 else 6584 zone->zone_start_pfn = 0; 6585 zone->spanned_pages = size; 6586 zone->present_pages = real_size; 6587 6588 totalpages += size; 6589 realtotalpages += real_size; 6590 } 6591 6592 pgdat->node_spanned_pages = totalpages; 6593 pgdat->node_present_pages = realtotalpages; 6594 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, 6595 realtotalpages); 6596 } 6597 6598 #ifndef CONFIG_SPARSEMEM 6599 /* 6600 * Calculate the size of the zone->blockflags rounded to an unsigned long 6601 * Start by making sure zonesize is a multiple of pageblock_order by rounding 6602 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 6603 * round what is now in bits to nearest long in bits, then return it in 6604 * bytes. 6605 */ 6606 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 6607 { 6608 unsigned long usemapsize; 6609 6610 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 6611 usemapsize = roundup(zonesize, pageblock_nr_pages); 6612 usemapsize = usemapsize >> pageblock_order; 6613 usemapsize *= NR_PAGEBLOCK_BITS; 6614 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 6615 6616 return usemapsize / 8; 6617 } 6618 6619 static void __ref setup_usemap(struct pglist_data *pgdat, 6620 struct zone *zone, 6621 unsigned long zone_start_pfn, 6622 unsigned long zonesize) 6623 { 6624 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); 6625 zone->pageblock_flags = NULL; 6626 if (usemapsize) { 6627 zone->pageblock_flags = 6628 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 6629 pgdat->node_id); 6630 if (!zone->pageblock_flags) 6631 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 6632 usemapsize, zone->name, pgdat->node_id); 6633 } 6634 } 6635 #else 6636 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, 6637 unsigned long zone_start_pfn, unsigned long zonesize) {} 6638 #endif /* CONFIG_SPARSEMEM */ 6639 6640 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 6641 6642 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 6643 void __init set_pageblock_order(void) 6644 { 6645 unsigned int order; 6646 6647 /* Check that pageblock_nr_pages has not already been setup */ 6648 if (pageblock_order) 6649 return; 6650 6651 if (HPAGE_SHIFT > PAGE_SHIFT) 6652 order = HUGETLB_PAGE_ORDER; 6653 else 6654 order = MAX_ORDER - 1; 6655 6656 /* 6657 * Assume the largest contiguous order of interest is a huge page. 6658 * This value may be variable depending on boot parameters on IA64 and 6659 * powerpc. 6660 */ 6661 pageblock_order = order; 6662 } 6663 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6664 6665 /* 6666 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 6667 * is unused as pageblock_order is set at compile-time. See 6668 * include/linux/pageblock-flags.h for the values of pageblock_order based on 6669 * the kernel config 6670 */ 6671 void __init set_pageblock_order(void) 6672 { 6673 } 6674 6675 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6676 6677 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 6678 unsigned long present_pages) 6679 { 6680 unsigned long pages = spanned_pages; 6681 6682 /* 6683 * Provide a more accurate estimation if there are holes within 6684 * the zone and SPARSEMEM is in use. If there are holes within the 6685 * zone, each populated memory region may cost us one or two extra 6686 * memmap pages due to alignment because memmap pages for each 6687 * populated regions may not be naturally aligned on page boundary. 6688 * So the (present_pages >> 4) heuristic is a tradeoff for that. 6689 */ 6690 if (spanned_pages > present_pages + (present_pages >> 4) && 6691 IS_ENABLED(CONFIG_SPARSEMEM)) 6692 pages = present_pages; 6693 6694 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 6695 } 6696 6697 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 6698 static void pgdat_init_split_queue(struct pglist_data *pgdat) 6699 { 6700 struct deferred_split *ds_queue = &pgdat->deferred_split_queue; 6701 6702 spin_lock_init(&ds_queue->split_queue_lock); 6703 INIT_LIST_HEAD(&ds_queue->split_queue); 6704 ds_queue->split_queue_len = 0; 6705 } 6706 #else 6707 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 6708 #endif 6709 6710 #ifdef CONFIG_COMPACTION 6711 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 6712 { 6713 init_waitqueue_head(&pgdat->kcompactd_wait); 6714 } 6715 #else 6716 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 6717 #endif 6718 6719 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 6720 { 6721 pgdat_resize_init(pgdat); 6722 6723 pgdat_init_split_queue(pgdat); 6724 pgdat_init_kcompactd(pgdat); 6725 6726 init_waitqueue_head(&pgdat->kswapd_wait); 6727 init_waitqueue_head(&pgdat->pfmemalloc_wait); 6728 6729 pgdat_page_ext_init(pgdat); 6730 spin_lock_init(&pgdat->lru_lock); 6731 lruvec_init(&pgdat->__lruvec); 6732 } 6733 6734 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 6735 unsigned long remaining_pages) 6736 { 6737 atomic_long_set(&zone->managed_pages, remaining_pages); 6738 zone_set_nid(zone, nid); 6739 zone->name = zone_names[idx]; 6740 zone->zone_pgdat = NODE_DATA(nid); 6741 spin_lock_init(&zone->lock); 6742 zone_seqlock_init(zone); 6743 zone_pcp_init(zone); 6744 } 6745 6746 /* 6747 * Set up the zone data structures 6748 * - init pgdat internals 6749 * - init all zones belonging to this node 6750 * 6751 * NOTE: this function is only called during memory hotplug 6752 */ 6753 #ifdef CONFIG_MEMORY_HOTPLUG 6754 void __ref free_area_init_core_hotplug(int nid) 6755 { 6756 enum zone_type z; 6757 pg_data_t *pgdat = NODE_DATA(nid); 6758 6759 pgdat_init_internals(pgdat); 6760 for (z = 0; z < MAX_NR_ZONES; z++) 6761 zone_init_internals(&pgdat->node_zones[z], z, nid, 0); 6762 } 6763 #endif 6764 6765 /* 6766 * Set up the zone data structures: 6767 * - mark all pages reserved 6768 * - mark all memory queues empty 6769 * - clear the memory bitmaps 6770 * 6771 * NOTE: pgdat should get zeroed by caller. 6772 * NOTE: this function is only called during early init. 6773 */ 6774 static void __init free_area_init_core(struct pglist_data *pgdat) 6775 { 6776 enum zone_type j; 6777 int nid = pgdat->node_id; 6778 6779 pgdat_init_internals(pgdat); 6780 pgdat->per_cpu_nodestats = &boot_nodestats; 6781 6782 for (j = 0; j < MAX_NR_ZONES; j++) { 6783 struct zone *zone = pgdat->node_zones + j; 6784 unsigned long size, freesize, memmap_pages; 6785 unsigned long zone_start_pfn = zone->zone_start_pfn; 6786 6787 size = zone->spanned_pages; 6788 freesize = zone->present_pages; 6789 6790 /* 6791 * Adjust freesize so that it accounts for how much memory 6792 * is used by this zone for memmap. This affects the watermark 6793 * and per-cpu initialisations 6794 */ 6795 memmap_pages = calc_memmap_size(size, freesize); 6796 if (!is_highmem_idx(j)) { 6797 if (freesize >= memmap_pages) { 6798 freesize -= memmap_pages; 6799 if (memmap_pages) 6800 printk(KERN_DEBUG 6801 " %s zone: %lu pages used for memmap\n", 6802 zone_names[j], memmap_pages); 6803 } else 6804 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n", 6805 zone_names[j], memmap_pages, freesize); 6806 } 6807 6808 /* Account for reserved pages */ 6809 if (j == 0 && freesize > dma_reserve) { 6810 freesize -= dma_reserve; 6811 printk(KERN_DEBUG " %s zone: %lu pages reserved\n", 6812 zone_names[0], dma_reserve); 6813 } 6814 6815 if (!is_highmem_idx(j)) 6816 nr_kernel_pages += freesize; 6817 /* Charge for highmem memmap if there are enough kernel pages */ 6818 else if (nr_kernel_pages > memmap_pages * 2) 6819 nr_kernel_pages -= memmap_pages; 6820 nr_all_pages += freesize; 6821 6822 /* 6823 * Set an approximate value for lowmem here, it will be adjusted 6824 * when the bootmem allocator frees pages into the buddy system. 6825 * And all highmem pages will be managed by the buddy system. 6826 */ 6827 zone_init_internals(zone, j, nid, freesize); 6828 6829 if (!size) 6830 continue; 6831 6832 set_pageblock_order(); 6833 setup_usemap(pgdat, zone, zone_start_pfn, size); 6834 init_currently_empty_zone(zone, zone_start_pfn, size); 6835 memmap_init(size, nid, j, zone_start_pfn); 6836 } 6837 } 6838 6839 #ifdef CONFIG_FLAT_NODE_MEM_MAP 6840 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) 6841 { 6842 unsigned long __maybe_unused start = 0; 6843 unsigned long __maybe_unused offset = 0; 6844 6845 /* Skip empty nodes */ 6846 if (!pgdat->node_spanned_pages) 6847 return; 6848 6849 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 6850 offset = pgdat->node_start_pfn - start; 6851 /* ia64 gets its own node_mem_map, before this, without bootmem */ 6852 if (!pgdat->node_mem_map) { 6853 unsigned long size, end; 6854 struct page *map; 6855 6856 /* 6857 * The zone's endpoints aren't required to be MAX_ORDER 6858 * aligned but the node_mem_map endpoints must be in order 6859 * for the buddy allocator to function correctly. 6860 */ 6861 end = pgdat_end_pfn(pgdat); 6862 end = ALIGN(end, MAX_ORDER_NR_PAGES); 6863 size = (end - start) * sizeof(struct page); 6864 map = memblock_alloc_node(size, SMP_CACHE_BYTES, 6865 pgdat->node_id); 6866 if (!map) 6867 panic("Failed to allocate %ld bytes for node %d memory map\n", 6868 size, pgdat->node_id); 6869 pgdat->node_mem_map = map + offset; 6870 } 6871 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 6872 __func__, pgdat->node_id, (unsigned long)pgdat, 6873 (unsigned long)pgdat->node_mem_map); 6874 #ifndef CONFIG_NEED_MULTIPLE_NODES 6875 /* 6876 * With no DISCONTIG, the global mem_map is just set as node 0's 6877 */ 6878 if (pgdat == NODE_DATA(0)) { 6879 mem_map = NODE_DATA(0)->node_mem_map; 6880 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 6881 mem_map -= offset; 6882 } 6883 #endif 6884 } 6885 #else 6886 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { } 6887 #endif /* CONFIG_FLAT_NODE_MEM_MAP */ 6888 6889 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 6890 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 6891 { 6892 pgdat->first_deferred_pfn = ULONG_MAX; 6893 } 6894 #else 6895 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 6896 #endif 6897 6898 static void __init free_area_init_node(int nid) 6899 { 6900 pg_data_t *pgdat = NODE_DATA(nid); 6901 unsigned long start_pfn = 0; 6902 unsigned long end_pfn = 0; 6903 6904 /* pg_data_t should be reset to zero when it's allocated */ 6905 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); 6906 6907 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 6908 6909 pgdat->node_id = nid; 6910 pgdat->node_start_pfn = start_pfn; 6911 pgdat->per_cpu_nodestats = NULL; 6912 6913 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 6914 (u64)start_pfn << PAGE_SHIFT, 6915 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 6916 calculate_node_totalpages(pgdat, start_pfn, end_pfn); 6917 6918 alloc_node_mem_map(pgdat); 6919 pgdat_set_deferred_range(pgdat); 6920 6921 free_area_init_core(pgdat); 6922 } 6923 6924 void __init free_area_init_memoryless_node(int nid) 6925 { 6926 free_area_init_node(nid); 6927 } 6928 6929 #if !defined(CONFIG_FLAT_NODE_MEM_MAP) 6930 /* 6931 * Initialize all valid struct pages in the range [spfn, epfn) and mark them 6932 * PageReserved(). Return the number of struct pages that were initialized. 6933 */ 6934 static u64 __init init_unavailable_range(unsigned long spfn, unsigned long epfn) 6935 { 6936 unsigned long pfn; 6937 u64 pgcnt = 0; 6938 6939 for (pfn = spfn; pfn < epfn; pfn++) { 6940 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) { 6941 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages) 6942 + pageblock_nr_pages - 1; 6943 continue; 6944 } 6945 /* 6946 * Use a fake node/zone (0) for now. Some of these pages 6947 * (in memblock.reserved but not in memblock.memory) will 6948 * get re-initialized via reserve_bootmem_region() later. 6949 */ 6950 __init_single_page(pfn_to_page(pfn), pfn, 0, 0); 6951 __SetPageReserved(pfn_to_page(pfn)); 6952 pgcnt++; 6953 } 6954 6955 return pgcnt; 6956 } 6957 6958 /* 6959 * Only struct pages that are backed by physical memory are zeroed and 6960 * initialized by going through __init_single_page(). But, there are some 6961 * struct pages which are reserved in memblock allocator and their fields 6962 * may be accessed (for example page_to_pfn() on some configuration accesses 6963 * flags). We must explicitly initialize those struct pages. 6964 * 6965 * This function also addresses a similar issue where struct pages are left 6966 * uninitialized because the physical address range is not covered by 6967 * memblock.memory or memblock.reserved. That could happen when memblock 6968 * layout is manually configured via memmap=, or when the highest physical 6969 * address (max_pfn) does not end on a section boundary. 6970 */ 6971 static void __init init_unavailable_mem(void) 6972 { 6973 phys_addr_t start, end; 6974 u64 i, pgcnt; 6975 phys_addr_t next = 0; 6976 6977 /* 6978 * Loop through unavailable ranges not covered by memblock.memory. 6979 */ 6980 pgcnt = 0; 6981 for_each_mem_range(i, &memblock.memory, NULL, 6982 NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, NULL) { 6983 if (next < start) 6984 pgcnt += init_unavailable_range(PFN_DOWN(next), 6985 PFN_UP(start)); 6986 next = end; 6987 } 6988 6989 /* 6990 * Early sections always have a fully populated memmap for the whole 6991 * section - see pfn_valid(). If the last section has holes at the 6992 * end and that section is marked "online", the memmap will be 6993 * considered initialized. Make sure that memmap has a well defined 6994 * state. 6995 */ 6996 pgcnt += init_unavailable_range(PFN_DOWN(next), 6997 round_up(max_pfn, PAGES_PER_SECTION)); 6998 6999 /* 7000 * Struct pages that do not have backing memory. This could be because 7001 * firmware is using some of this memory, or for some other reasons. 7002 */ 7003 if (pgcnt) 7004 pr_info("Zeroed struct page in unavailable ranges: %lld pages", pgcnt); 7005 } 7006 #else 7007 static inline void __init init_unavailable_mem(void) 7008 { 7009 } 7010 #endif /* !CONFIG_FLAT_NODE_MEM_MAP */ 7011 7012 #if MAX_NUMNODES > 1 7013 /* 7014 * Figure out the number of possible node ids. 7015 */ 7016 void __init setup_nr_node_ids(void) 7017 { 7018 unsigned int highest; 7019 7020 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 7021 nr_node_ids = highest + 1; 7022 } 7023 #endif 7024 7025 /** 7026 * node_map_pfn_alignment - determine the maximum internode alignment 7027 * 7028 * This function should be called after node map is populated and sorted. 7029 * It calculates the maximum power of two alignment which can distinguish 7030 * all the nodes. 7031 * 7032 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 7033 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 7034 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 7035 * shifted, 1GiB is enough and this function will indicate so. 7036 * 7037 * This is used to test whether pfn -> nid mapping of the chosen memory 7038 * model has fine enough granularity to avoid incorrect mapping for the 7039 * populated node map. 7040 * 7041 * Return: the determined alignment in pfn's. 0 if there is no alignment 7042 * requirement (single node). 7043 */ 7044 unsigned long __init node_map_pfn_alignment(void) 7045 { 7046 unsigned long accl_mask = 0, last_end = 0; 7047 unsigned long start, end, mask; 7048 int last_nid = NUMA_NO_NODE; 7049 int i, nid; 7050 7051 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 7052 if (!start || last_nid < 0 || last_nid == nid) { 7053 last_nid = nid; 7054 last_end = end; 7055 continue; 7056 } 7057 7058 /* 7059 * Start with a mask granular enough to pin-point to the 7060 * start pfn and tick off bits one-by-one until it becomes 7061 * too coarse to separate the current node from the last. 7062 */ 7063 mask = ~((1 << __ffs(start)) - 1); 7064 while (mask && last_end <= (start & (mask << 1))) 7065 mask <<= 1; 7066 7067 /* accumulate all internode masks */ 7068 accl_mask |= mask; 7069 } 7070 7071 /* convert mask to number of pages */ 7072 return ~accl_mask + 1; 7073 } 7074 7075 /** 7076 * find_min_pfn_with_active_regions - Find the minimum PFN registered 7077 * 7078 * Return: the minimum PFN based on information provided via 7079 * memblock_set_node(). 7080 */ 7081 unsigned long __init find_min_pfn_with_active_regions(void) 7082 { 7083 return PHYS_PFN(memblock_start_of_DRAM()); 7084 } 7085 7086 /* 7087 * early_calculate_totalpages() 7088 * Sum pages in active regions for movable zone. 7089 * Populate N_MEMORY for calculating usable_nodes. 7090 */ 7091 static unsigned long __init early_calculate_totalpages(void) 7092 { 7093 unsigned long totalpages = 0; 7094 unsigned long start_pfn, end_pfn; 7095 int i, nid; 7096 7097 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7098 unsigned long pages = end_pfn - start_pfn; 7099 7100 totalpages += pages; 7101 if (pages) 7102 node_set_state(nid, N_MEMORY); 7103 } 7104 return totalpages; 7105 } 7106 7107 /* 7108 * Find the PFN the Movable zone begins in each node. Kernel memory 7109 * is spread evenly between nodes as long as the nodes have enough 7110 * memory. When they don't, some nodes will have more kernelcore than 7111 * others 7112 */ 7113 static void __init find_zone_movable_pfns_for_nodes(void) 7114 { 7115 int i, nid; 7116 unsigned long usable_startpfn; 7117 unsigned long kernelcore_node, kernelcore_remaining; 7118 /* save the state before borrow the nodemask */ 7119 nodemask_t saved_node_state = node_states[N_MEMORY]; 7120 unsigned long totalpages = early_calculate_totalpages(); 7121 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 7122 struct memblock_region *r; 7123 7124 /* Need to find movable_zone earlier when movable_node is specified. */ 7125 find_usable_zone_for_movable(); 7126 7127 /* 7128 * If movable_node is specified, ignore kernelcore and movablecore 7129 * options. 7130 */ 7131 if (movable_node_is_enabled()) { 7132 for_each_memblock(memory, r) { 7133 if (!memblock_is_hotpluggable(r)) 7134 continue; 7135 7136 nid = memblock_get_region_node(r); 7137 7138 usable_startpfn = PFN_DOWN(r->base); 7139 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7140 min(usable_startpfn, zone_movable_pfn[nid]) : 7141 usable_startpfn; 7142 } 7143 7144 goto out2; 7145 } 7146 7147 /* 7148 * If kernelcore=mirror is specified, ignore movablecore option 7149 */ 7150 if (mirrored_kernelcore) { 7151 bool mem_below_4gb_not_mirrored = false; 7152 7153 for_each_memblock(memory, r) { 7154 if (memblock_is_mirror(r)) 7155 continue; 7156 7157 nid = memblock_get_region_node(r); 7158 7159 usable_startpfn = memblock_region_memory_base_pfn(r); 7160 7161 if (usable_startpfn < 0x100000) { 7162 mem_below_4gb_not_mirrored = true; 7163 continue; 7164 } 7165 7166 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7167 min(usable_startpfn, zone_movable_pfn[nid]) : 7168 usable_startpfn; 7169 } 7170 7171 if (mem_below_4gb_not_mirrored) 7172 pr_warn("This configuration results in unmirrored kernel memory.\n"); 7173 7174 goto out2; 7175 } 7176 7177 /* 7178 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 7179 * amount of necessary memory. 7180 */ 7181 if (required_kernelcore_percent) 7182 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 7183 10000UL; 7184 if (required_movablecore_percent) 7185 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 7186 10000UL; 7187 7188 /* 7189 * If movablecore= was specified, calculate what size of 7190 * kernelcore that corresponds so that memory usable for 7191 * any allocation type is evenly spread. If both kernelcore 7192 * and movablecore are specified, then the value of kernelcore 7193 * will be used for required_kernelcore if it's greater than 7194 * what movablecore would have allowed. 7195 */ 7196 if (required_movablecore) { 7197 unsigned long corepages; 7198 7199 /* 7200 * Round-up so that ZONE_MOVABLE is at least as large as what 7201 * was requested by the user 7202 */ 7203 required_movablecore = 7204 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 7205 required_movablecore = min(totalpages, required_movablecore); 7206 corepages = totalpages - required_movablecore; 7207 7208 required_kernelcore = max(required_kernelcore, corepages); 7209 } 7210 7211 /* 7212 * If kernelcore was not specified or kernelcore size is larger 7213 * than totalpages, there is no ZONE_MOVABLE. 7214 */ 7215 if (!required_kernelcore || required_kernelcore >= totalpages) 7216 goto out; 7217 7218 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 7219 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 7220 7221 restart: 7222 /* Spread kernelcore memory as evenly as possible throughout nodes */ 7223 kernelcore_node = required_kernelcore / usable_nodes; 7224 for_each_node_state(nid, N_MEMORY) { 7225 unsigned long start_pfn, end_pfn; 7226 7227 /* 7228 * Recalculate kernelcore_node if the division per node 7229 * now exceeds what is necessary to satisfy the requested 7230 * amount of memory for the kernel 7231 */ 7232 if (required_kernelcore < kernelcore_node) 7233 kernelcore_node = required_kernelcore / usable_nodes; 7234 7235 /* 7236 * As the map is walked, we track how much memory is usable 7237 * by the kernel using kernelcore_remaining. When it is 7238 * 0, the rest of the node is usable by ZONE_MOVABLE 7239 */ 7240 kernelcore_remaining = kernelcore_node; 7241 7242 /* Go through each range of PFNs within this node */ 7243 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7244 unsigned long size_pages; 7245 7246 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 7247 if (start_pfn >= end_pfn) 7248 continue; 7249 7250 /* Account for what is only usable for kernelcore */ 7251 if (start_pfn < usable_startpfn) { 7252 unsigned long kernel_pages; 7253 kernel_pages = min(end_pfn, usable_startpfn) 7254 - start_pfn; 7255 7256 kernelcore_remaining -= min(kernel_pages, 7257 kernelcore_remaining); 7258 required_kernelcore -= min(kernel_pages, 7259 required_kernelcore); 7260 7261 /* Continue if range is now fully accounted */ 7262 if (end_pfn <= usable_startpfn) { 7263 7264 /* 7265 * Push zone_movable_pfn to the end so 7266 * that if we have to rebalance 7267 * kernelcore across nodes, we will 7268 * not double account here 7269 */ 7270 zone_movable_pfn[nid] = end_pfn; 7271 continue; 7272 } 7273 start_pfn = usable_startpfn; 7274 } 7275 7276 /* 7277 * The usable PFN range for ZONE_MOVABLE is from 7278 * start_pfn->end_pfn. Calculate size_pages as the 7279 * number of pages used as kernelcore 7280 */ 7281 size_pages = end_pfn - start_pfn; 7282 if (size_pages > kernelcore_remaining) 7283 size_pages = kernelcore_remaining; 7284 zone_movable_pfn[nid] = start_pfn + size_pages; 7285 7286 /* 7287 * Some kernelcore has been met, update counts and 7288 * break if the kernelcore for this node has been 7289 * satisfied 7290 */ 7291 required_kernelcore -= min(required_kernelcore, 7292 size_pages); 7293 kernelcore_remaining -= size_pages; 7294 if (!kernelcore_remaining) 7295 break; 7296 } 7297 } 7298 7299 /* 7300 * If there is still required_kernelcore, we do another pass with one 7301 * less node in the count. This will push zone_movable_pfn[nid] further 7302 * along on the nodes that still have memory until kernelcore is 7303 * satisfied 7304 */ 7305 usable_nodes--; 7306 if (usable_nodes && required_kernelcore > usable_nodes) 7307 goto restart; 7308 7309 out2: 7310 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 7311 for (nid = 0; nid < MAX_NUMNODES; nid++) 7312 zone_movable_pfn[nid] = 7313 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 7314 7315 out: 7316 /* restore the node_state */ 7317 node_states[N_MEMORY] = saved_node_state; 7318 } 7319 7320 /* Any regular or high memory on that node ? */ 7321 static void check_for_memory(pg_data_t *pgdat, int nid) 7322 { 7323 enum zone_type zone_type; 7324 7325 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 7326 struct zone *zone = &pgdat->node_zones[zone_type]; 7327 if (populated_zone(zone)) { 7328 if (IS_ENABLED(CONFIG_HIGHMEM)) 7329 node_set_state(nid, N_HIGH_MEMORY); 7330 if (zone_type <= ZONE_NORMAL) 7331 node_set_state(nid, N_NORMAL_MEMORY); 7332 break; 7333 } 7334 } 7335 } 7336 7337 /* 7338 * Some architecturs, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For 7339 * such cases we allow max_zone_pfn sorted in the descending order 7340 */ 7341 bool __weak arch_has_descending_max_zone_pfns(void) 7342 { 7343 return false; 7344 } 7345 7346 /** 7347 * free_area_init - Initialise all pg_data_t and zone data 7348 * @max_zone_pfn: an array of max PFNs for each zone 7349 * 7350 * This will call free_area_init_node() for each active node in the system. 7351 * Using the page ranges provided by memblock_set_node(), the size of each 7352 * zone in each node and their holes is calculated. If the maximum PFN 7353 * between two adjacent zones match, it is assumed that the zone is empty. 7354 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 7355 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 7356 * starts where the previous one ended. For example, ZONE_DMA32 starts 7357 * at arch_max_dma_pfn. 7358 */ 7359 void __init free_area_init(unsigned long *max_zone_pfn) 7360 { 7361 unsigned long start_pfn, end_pfn; 7362 int i, nid, zone; 7363 bool descending; 7364 7365 /* Record where the zone boundaries are */ 7366 memset(arch_zone_lowest_possible_pfn, 0, 7367 sizeof(arch_zone_lowest_possible_pfn)); 7368 memset(arch_zone_highest_possible_pfn, 0, 7369 sizeof(arch_zone_highest_possible_pfn)); 7370 7371 start_pfn = find_min_pfn_with_active_regions(); 7372 descending = arch_has_descending_max_zone_pfns(); 7373 7374 for (i = 0; i < MAX_NR_ZONES; i++) { 7375 if (descending) 7376 zone = MAX_NR_ZONES - i - 1; 7377 else 7378 zone = i; 7379 7380 if (zone == ZONE_MOVABLE) 7381 continue; 7382 7383 end_pfn = max(max_zone_pfn[zone], start_pfn); 7384 arch_zone_lowest_possible_pfn[zone] = start_pfn; 7385 arch_zone_highest_possible_pfn[zone] = end_pfn; 7386 7387 start_pfn = end_pfn; 7388 } 7389 7390 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 7391 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 7392 find_zone_movable_pfns_for_nodes(); 7393 7394 /* Print out the zone ranges */ 7395 pr_info("Zone ranges:\n"); 7396 for (i = 0; i < MAX_NR_ZONES; i++) { 7397 if (i == ZONE_MOVABLE) 7398 continue; 7399 pr_info(" %-8s ", zone_names[i]); 7400 if (arch_zone_lowest_possible_pfn[i] == 7401 arch_zone_highest_possible_pfn[i]) 7402 pr_cont("empty\n"); 7403 else 7404 pr_cont("[mem %#018Lx-%#018Lx]\n", 7405 (u64)arch_zone_lowest_possible_pfn[i] 7406 << PAGE_SHIFT, 7407 ((u64)arch_zone_highest_possible_pfn[i] 7408 << PAGE_SHIFT) - 1); 7409 } 7410 7411 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 7412 pr_info("Movable zone start for each node\n"); 7413 for (i = 0; i < MAX_NUMNODES; i++) { 7414 if (zone_movable_pfn[i]) 7415 pr_info(" Node %d: %#018Lx\n", i, 7416 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 7417 } 7418 7419 /* 7420 * Print out the early node map, and initialize the 7421 * subsection-map relative to active online memory ranges to 7422 * enable future "sub-section" extensions of the memory map. 7423 */ 7424 pr_info("Early memory node ranges\n"); 7425 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7426 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 7427 (u64)start_pfn << PAGE_SHIFT, 7428 ((u64)end_pfn << PAGE_SHIFT) - 1); 7429 subsection_map_init(start_pfn, end_pfn - start_pfn); 7430 } 7431 7432 /* Initialise every node */ 7433 mminit_verify_pageflags_layout(); 7434 setup_nr_node_ids(); 7435 init_unavailable_mem(); 7436 for_each_online_node(nid) { 7437 pg_data_t *pgdat = NODE_DATA(nid); 7438 free_area_init_node(nid); 7439 7440 /* Any memory on that node */ 7441 if (pgdat->node_present_pages) 7442 node_set_state(nid, N_MEMORY); 7443 check_for_memory(pgdat, nid); 7444 } 7445 } 7446 7447 static int __init cmdline_parse_core(char *p, unsigned long *core, 7448 unsigned long *percent) 7449 { 7450 unsigned long long coremem; 7451 char *endptr; 7452 7453 if (!p) 7454 return -EINVAL; 7455 7456 /* Value may be a percentage of total memory, otherwise bytes */ 7457 coremem = simple_strtoull(p, &endptr, 0); 7458 if (*endptr == '%') { 7459 /* Paranoid check for percent values greater than 100 */ 7460 WARN_ON(coremem > 100); 7461 7462 *percent = coremem; 7463 } else { 7464 coremem = memparse(p, &p); 7465 /* Paranoid check that UL is enough for the coremem value */ 7466 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 7467 7468 *core = coremem >> PAGE_SHIFT; 7469 *percent = 0UL; 7470 } 7471 return 0; 7472 } 7473 7474 /* 7475 * kernelcore=size sets the amount of memory for use for allocations that 7476 * cannot be reclaimed or migrated. 7477 */ 7478 static int __init cmdline_parse_kernelcore(char *p) 7479 { 7480 /* parse kernelcore=mirror */ 7481 if (parse_option_str(p, "mirror")) { 7482 mirrored_kernelcore = true; 7483 return 0; 7484 } 7485 7486 return cmdline_parse_core(p, &required_kernelcore, 7487 &required_kernelcore_percent); 7488 } 7489 7490 /* 7491 * movablecore=size sets the amount of memory for use for allocations that 7492 * can be reclaimed or migrated. 7493 */ 7494 static int __init cmdline_parse_movablecore(char *p) 7495 { 7496 return cmdline_parse_core(p, &required_movablecore, 7497 &required_movablecore_percent); 7498 } 7499 7500 early_param("kernelcore", cmdline_parse_kernelcore); 7501 early_param("movablecore", cmdline_parse_movablecore); 7502 7503 void adjust_managed_page_count(struct page *page, long count) 7504 { 7505 atomic_long_add(count, &page_zone(page)->managed_pages); 7506 totalram_pages_add(count); 7507 #ifdef CONFIG_HIGHMEM 7508 if (PageHighMem(page)) 7509 totalhigh_pages_add(count); 7510 #endif 7511 } 7512 EXPORT_SYMBOL(adjust_managed_page_count); 7513 7514 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 7515 { 7516 void *pos; 7517 unsigned long pages = 0; 7518 7519 start = (void *)PAGE_ALIGN((unsigned long)start); 7520 end = (void *)((unsigned long)end & PAGE_MASK); 7521 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 7522 struct page *page = virt_to_page(pos); 7523 void *direct_map_addr; 7524 7525 /* 7526 * 'direct_map_addr' might be different from 'pos' 7527 * because some architectures' virt_to_page() 7528 * work with aliases. Getting the direct map 7529 * address ensures that we get a _writeable_ 7530 * alias for the memset(). 7531 */ 7532 direct_map_addr = page_address(page); 7533 if ((unsigned int)poison <= 0xFF) 7534 memset(direct_map_addr, poison, PAGE_SIZE); 7535 7536 free_reserved_page(page); 7537 } 7538 7539 if (pages && s) 7540 pr_info("Freeing %s memory: %ldK\n", 7541 s, pages << (PAGE_SHIFT - 10)); 7542 7543 return pages; 7544 } 7545 7546 #ifdef CONFIG_HIGHMEM 7547 void free_highmem_page(struct page *page) 7548 { 7549 __free_reserved_page(page); 7550 totalram_pages_inc(); 7551 atomic_long_inc(&page_zone(page)->managed_pages); 7552 totalhigh_pages_inc(); 7553 } 7554 #endif 7555 7556 7557 void __init mem_init_print_info(const char *str) 7558 { 7559 unsigned long physpages, codesize, datasize, rosize, bss_size; 7560 unsigned long init_code_size, init_data_size; 7561 7562 physpages = get_num_physpages(); 7563 codesize = _etext - _stext; 7564 datasize = _edata - _sdata; 7565 rosize = __end_rodata - __start_rodata; 7566 bss_size = __bss_stop - __bss_start; 7567 init_data_size = __init_end - __init_begin; 7568 init_code_size = _einittext - _sinittext; 7569 7570 /* 7571 * Detect special cases and adjust section sizes accordingly: 7572 * 1) .init.* may be embedded into .data sections 7573 * 2) .init.text.* may be out of [__init_begin, __init_end], 7574 * please refer to arch/tile/kernel/vmlinux.lds.S. 7575 * 3) .rodata.* may be embedded into .text or .data sections. 7576 */ 7577 #define adj_init_size(start, end, size, pos, adj) \ 7578 do { \ 7579 if (start <= pos && pos < end && size > adj) \ 7580 size -= adj; \ 7581 } while (0) 7582 7583 adj_init_size(__init_begin, __init_end, init_data_size, 7584 _sinittext, init_code_size); 7585 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 7586 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 7587 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 7588 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 7589 7590 #undef adj_init_size 7591 7592 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 7593 #ifdef CONFIG_HIGHMEM 7594 ", %luK highmem" 7595 #endif 7596 "%s%s)\n", 7597 nr_free_pages() << (PAGE_SHIFT - 10), 7598 physpages << (PAGE_SHIFT - 10), 7599 codesize >> 10, datasize >> 10, rosize >> 10, 7600 (init_data_size + init_code_size) >> 10, bss_size >> 10, 7601 (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10), 7602 totalcma_pages << (PAGE_SHIFT - 10), 7603 #ifdef CONFIG_HIGHMEM 7604 totalhigh_pages() << (PAGE_SHIFT - 10), 7605 #endif 7606 str ? ", " : "", str ? str : ""); 7607 } 7608 7609 /** 7610 * set_dma_reserve - set the specified number of pages reserved in the first zone 7611 * @new_dma_reserve: The number of pages to mark reserved 7612 * 7613 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 7614 * In the DMA zone, a significant percentage may be consumed by kernel image 7615 * and other unfreeable allocations which can skew the watermarks badly. This 7616 * function may optionally be used to account for unfreeable pages in the 7617 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 7618 * smaller per-cpu batchsize. 7619 */ 7620 void __init set_dma_reserve(unsigned long new_dma_reserve) 7621 { 7622 dma_reserve = new_dma_reserve; 7623 } 7624 7625 static int page_alloc_cpu_dead(unsigned int cpu) 7626 { 7627 7628 lru_add_drain_cpu(cpu); 7629 drain_pages(cpu); 7630 7631 /* 7632 * Spill the event counters of the dead processor 7633 * into the current processors event counters. 7634 * This artificially elevates the count of the current 7635 * processor. 7636 */ 7637 vm_events_fold_cpu(cpu); 7638 7639 /* 7640 * Zero the differential counters of the dead processor 7641 * so that the vm statistics are consistent. 7642 * 7643 * This is only okay since the processor is dead and cannot 7644 * race with what we are doing. 7645 */ 7646 cpu_vm_stats_fold(cpu); 7647 return 0; 7648 } 7649 7650 #ifdef CONFIG_NUMA 7651 int hashdist = HASHDIST_DEFAULT; 7652 7653 static int __init set_hashdist(char *str) 7654 { 7655 if (!str) 7656 return 0; 7657 hashdist = simple_strtoul(str, &str, 0); 7658 return 1; 7659 } 7660 __setup("hashdist=", set_hashdist); 7661 #endif 7662 7663 void __init page_alloc_init(void) 7664 { 7665 int ret; 7666 7667 #ifdef CONFIG_NUMA 7668 if (num_node_state(N_MEMORY) == 1) 7669 hashdist = 0; 7670 #endif 7671 7672 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD, 7673 "mm/page_alloc:dead", NULL, 7674 page_alloc_cpu_dead); 7675 WARN_ON(ret < 0); 7676 } 7677 7678 /* 7679 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 7680 * or min_free_kbytes changes. 7681 */ 7682 static void calculate_totalreserve_pages(void) 7683 { 7684 struct pglist_data *pgdat; 7685 unsigned long reserve_pages = 0; 7686 enum zone_type i, j; 7687 7688 for_each_online_pgdat(pgdat) { 7689 7690 pgdat->totalreserve_pages = 0; 7691 7692 for (i = 0; i < MAX_NR_ZONES; i++) { 7693 struct zone *zone = pgdat->node_zones + i; 7694 long max = 0; 7695 unsigned long managed_pages = zone_managed_pages(zone); 7696 7697 /* Find valid and maximum lowmem_reserve in the zone */ 7698 for (j = i; j < MAX_NR_ZONES; j++) { 7699 if (zone->lowmem_reserve[j] > max) 7700 max = zone->lowmem_reserve[j]; 7701 } 7702 7703 /* we treat the high watermark as reserved pages. */ 7704 max += high_wmark_pages(zone); 7705 7706 if (max > managed_pages) 7707 max = managed_pages; 7708 7709 pgdat->totalreserve_pages += max; 7710 7711 reserve_pages += max; 7712 } 7713 } 7714 totalreserve_pages = reserve_pages; 7715 } 7716 7717 /* 7718 * setup_per_zone_lowmem_reserve - called whenever 7719 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 7720 * has a correct pages reserved value, so an adequate number of 7721 * pages are left in the zone after a successful __alloc_pages(). 7722 */ 7723 static void setup_per_zone_lowmem_reserve(void) 7724 { 7725 struct pglist_data *pgdat; 7726 enum zone_type j, idx; 7727 7728 for_each_online_pgdat(pgdat) { 7729 for (j = 0; j < MAX_NR_ZONES; j++) { 7730 struct zone *zone = pgdat->node_zones + j; 7731 unsigned long managed_pages = zone_managed_pages(zone); 7732 7733 zone->lowmem_reserve[j] = 0; 7734 7735 idx = j; 7736 while (idx) { 7737 struct zone *lower_zone; 7738 7739 idx--; 7740 lower_zone = pgdat->node_zones + idx; 7741 7742 if (!sysctl_lowmem_reserve_ratio[idx] || 7743 !zone_managed_pages(lower_zone)) { 7744 lower_zone->lowmem_reserve[j] = 0; 7745 continue; 7746 } else { 7747 lower_zone->lowmem_reserve[j] = 7748 managed_pages / sysctl_lowmem_reserve_ratio[idx]; 7749 } 7750 managed_pages += zone_managed_pages(lower_zone); 7751 } 7752 } 7753 } 7754 7755 /* update totalreserve_pages */ 7756 calculate_totalreserve_pages(); 7757 } 7758 7759 static void __setup_per_zone_wmarks(void) 7760 { 7761 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 7762 unsigned long lowmem_pages = 0; 7763 struct zone *zone; 7764 unsigned long flags; 7765 7766 /* Calculate total number of !ZONE_HIGHMEM pages */ 7767 for_each_zone(zone) { 7768 if (!is_highmem(zone)) 7769 lowmem_pages += zone_managed_pages(zone); 7770 } 7771 7772 for_each_zone(zone) { 7773 u64 tmp; 7774 7775 spin_lock_irqsave(&zone->lock, flags); 7776 tmp = (u64)pages_min * zone_managed_pages(zone); 7777 do_div(tmp, lowmem_pages); 7778 if (is_highmem(zone)) { 7779 /* 7780 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 7781 * need highmem pages, so cap pages_min to a small 7782 * value here. 7783 * 7784 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 7785 * deltas control async page reclaim, and so should 7786 * not be capped for highmem. 7787 */ 7788 unsigned long min_pages; 7789 7790 min_pages = zone_managed_pages(zone) / 1024; 7791 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 7792 zone->_watermark[WMARK_MIN] = min_pages; 7793 } else { 7794 /* 7795 * If it's a lowmem zone, reserve a number of pages 7796 * proportionate to the zone's size. 7797 */ 7798 zone->_watermark[WMARK_MIN] = tmp; 7799 } 7800 7801 /* 7802 * Set the kswapd watermarks distance according to the 7803 * scale factor in proportion to available memory, but 7804 * ensure a minimum size on small systems. 7805 */ 7806 tmp = max_t(u64, tmp >> 2, 7807 mult_frac(zone_managed_pages(zone), 7808 watermark_scale_factor, 10000)); 7809 7810 zone->watermark_boost = 0; 7811 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 7812 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2; 7813 7814 spin_unlock_irqrestore(&zone->lock, flags); 7815 } 7816 7817 /* update totalreserve_pages */ 7818 calculate_totalreserve_pages(); 7819 } 7820 7821 /** 7822 * setup_per_zone_wmarks - called when min_free_kbytes changes 7823 * or when memory is hot-{added|removed} 7824 * 7825 * Ensures that the watermark[min,low,high] values for each zone are set 7826 * correctly with respect to min_free_kbytes. 7827 */ 7828 void setup_per_zone_wmarks(void) 7829 { 7830 static DEFINE_SPINLOCK(lock); 7831 7832 spin_lock(&lock); 7833 __setup_per_zone_wmarks(); 7834 spin_unlock(&lock); 7835 } 7836 7837 /* 7838 * Initialise min_free_kbytes. 7839 * 7840 * For small machines we want it small (128k min). For large machines 7841 * we want it large (256MB max). But it is not linear, because network 7842 * bandwidth does not increase linearly with machine size. We use 7843 * 7844 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 7845 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 7846 * 7847 * which yields 7848 * 7849 * 16MB: 512k 7850 * 32MB: 724k 7851 * 64MB: 1024k 7852 * 128MB: 1448k 7853 * 256MB: 2048k 7854 * 512MB: 2896k 7855 * 1024MB: 4096k 7856 * 2048MB: 5792k 7857 * 4096MB: 8192k 7858 * 8192MB: 11584k 7859 * 16384MB: 16384k 7860 */ 7861 int __meminit init_per_zone_wmark_min(void) 7862 { 7863 unsigned long lowmem_kbytes; 7864 int new_min_free_kbytes; 7865 7866 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 7867 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 7868 7869 if (new_min_free_kbytes > user_min_free_kbytes) { 7870 min_free_kbytes = new_min_free_kbytes; 7871 if (min_free_kbytes < 128) 7872 min_free_kbytes = 128; 7873 if (min_free_kbytes > 262144) 7874 min_free_kbytes = 262144; 7875 } else { 7876 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 7877 new_min_free_kbytes, user_min_free_kbytes); 7878 } 7879 setup_per_zone_wmarks(); 7880 refresh_zone_stat_thresholds(); 7881 setup_per_zone_lowmem_reserve(); 7882 7883 #ifdef CONFIG_NUMA 7884 setup_min_unmapped_ratio(); 7885 setup_min_slab_ratio(); 7886 #endif 7887 7888 return 0; 7889 } 7890 core_initcall(init_per_zone_wmark_min) 7891 7892 /* 7893 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 7894 * that we can call two helper functions whenever min_free_kbytes 7895 * changes. 7896 */ 7897 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 7898 void *buffer, size_t *length, loff_t *ppos) 7899 { 7900 int rc; 7901 7902 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7903 if (rc) 7904 return rc; 7905 7906 if (write) { 7907 user_min_free_kbytes = min_free_kbytes; 7908 setup_per_zone_wmarks(); 7909 } 7910 return 0; 7911 } 7912 7913 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 7914 void *buffer, size_t *length, loff_t *ppos) 7915 { 7916 int rc; 7917 7918 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7919 if (rc) 7920 return rc; 7921 7922 if (write) 7923 setup_per_zone_wmarks(); 7924 7925 return 0; 7926 } 7927 7928 #ifdef CONFIG_NUMA 7929 static void setup_min_unmapped_ratio(void) 7930 { 7931 pg_data_t *pgdat; 7932 struct zone *zone; 7933 7934 for_each_online_pgdat(pgdat) 7935 pgdat->min_unmapped_pages = 0; 7936 7937 for_each_zone(zone) 7938 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * 7939 sysctl_min_unmapped_ratio) / 100; 7940 } 7941 7942 7943 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 7944 void *buffer, size_t *length, loff_t *ppos) 7945 { 7946 int rc; 7947 7948 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7949 if (rc) 7950 return rc; 7951 7952 setup_min_unmapped_ratio(); 7953 7954 return 0; 7955 } 7956 7957 static void setup_min_slab_ratio(void) 7958 { 7959 pg_data_t *pgdat; 7960 struct zone *zone; 7961 7962 for_each_online_pgdat(pgdat) 7963 pgdat->min_slab_pages = 0; 7964 7965 for_each_zone(zone) 7966 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * 7967 sysctl_min_slab_ratio) / 100; 7968 } 7969 7970 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 7971 void *buffer, size_t *length, loff_t *ppos) 7972 { 7973 int rc; 7974 7975 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7976 if (rc) 7977 return rc; 7978 7979 setup_min_slab_ratio(); 7980 7981 return 0; 7982 } 7983 #endif 7984 7985 /* 7986 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 7987 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 7988 * whenever sysctl_lowmem_reserve_ratio changes. 7989 * 7990 * The reserve ratio obviously has absolutely no relation with the 7991 * minimum watermarks. The lowmem reserve ratio can only make sense 7992 * if in function of the boot time zone sizes. 7993 */ 7994 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 7995 void *buffer, size_t *length, loff_t *ppos) 7996 { 7997 int i; 7998 7999 proc_dointvec_minmax(table, write, buffer, length, ppos); 8000 8001 for (i = 0; i < MAX_NR_ZONES; i++) { 8002 if (sysctl_lowmem_reserve_ratio[i] < 1) 8003 sysctl_lowmem_reserve_ratio[i] = 0; 8004 } 8005 8006 setup_per_zone_lowmem_reserve(); 8007 return 0; 8008 } 8009 8010 static void __zone_pcp_update(struct zone *zone) 8011 { 8012 unsigned int cpu; 8013 8014 for_each_possible_cpu(cpu) 8015 pageset_set_high_and_batch(zone, 8016 per_cpu_ptr(zone->pageset, cpu)); 8017 } 8018 8019 /* 8020 * percpu_pagelist_fraction - changes the pcp->high for each zone on each 8021 * cpu. It is the fraction of total pages in each zone that a hot per cpu 8022 * pagelist can have before it gets flushed back to buddy allocator. 8023 */ 8024 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, 8025 void *buffer, size_t *length, loff_t *ppos) 8026 { 8027 struct zone *zone; 8028 int old_percpu_pagelist_fraction; 8029 int ret; 8030 8031 mutex_lock(&pcp_batch_high_lock); 8032 old_percpu_pagelist_fraction = percpu_pagelist_fraction; 8033 8034 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 8035 if (!write || ret < 0) 8036 goto out; 8037 8038 /* Sanity checking to avoid pcp imbalance */ 8039 if (percpu_pagelist_fraction && 8040 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { 8041 percpu_pagelist_fraction = old_percpu_pagelist_fraction; 8042 ret = -EINVAL; 8043 goto out; 8044 } 8045 8046 /* No change? */ 8047 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) 8048 goto out; 8049 8050 for_each_populated_zone(zone) 8051 __zone_pcp_update(zone); 8052 out: 8053 mutex_unlock(&pcp_batch_high_lock); 8054 return ret; 8055 } 8056 8057 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 8058 /* 8059 * Returns the number of pages that arch has reserved but 8060 * is not known to alloc_large_system_hash(). 8061 */ 8062 static unsigned long __init arch_reserved_kernel_pages(void) 8063 { 8064 return 0; 8065 } 8066 #endif 8067 8068 /* 8069 * Adaptive scale is meant to reduce sizes of hash tables on large memory 8070 * machines. As memory size is increased the scale is also increased but at 8071 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 8072 * quadruples the scale is increased by one, which means the size of hash table 8073 * only doubles, instead of quadrupling as well. 8074 * Because 32-bit systems cannot have large physical memory, where this scaling 8075 * makes sense, it is disabled on such platforms. 8076 */ 8077 #if __BITS_PER_LONG > 32 8078 #define ADAPT_SCALE_BASE (64ul << 30) 8079 #define ADAPT_SCALE_SHIFT 2 8080 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 8081 #endif 8082 8083 /* 8084 * allocate a large system hash table from bootmem 8085 * - it is assumed that the hash table must contain an exact power-of-2 8086 * quantity of entries 8087 * - limit is the number of hash buckets, not the total allocation size 8088 */ 8089 void *__init alloc_large_system_hash(const char *tablename, 8090 unsigned long bucketsize, 8091 unsigned long numentries, 8092 int scale, 8093 int flags, 8094 unsigned int *_hash_shift, 8095 unsigned int *_hash_mask, 8096 unsigned long low_limit, 8097 unsigned long high_limit) 8098 { 8099 unsigned long long max = high_limit; 8100 unsigned long log2qty, size; 8101 void *table = NULL; 8102 gfp_t gfp_flags; 8103 bool virt; 8104 8105 /* allow the kernel cmdline to have a say */ 8106 if (!numentries) { 8107 /* round applicable memory size up to nearest megabyte */ 8108 numentries = nr_kernel_pages; 8109 numentries -= arch_reserved_kernel_pages(); 8110 8111 /* It isn't necessary when PAGE_SIZE >= 1MB */ 8112 if (PAGE_SHIFT < 20) 8113 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 8114 8115 #if __BITS_PER_LONG > 32 8116 if (!high_limit) { 8117 unsigned long adapt; 8118 8119 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 8120 adapt <<= ADAPT_SCALE_SHIFT) 8121 scale++; 8122 } 8123 #endif 8124 8125 /* limit to 1 bucket per 2^scale bytes of low memory */ 8126 if (scale > PAGE_SHIFT) 8127 numentries >>= (scale - PAGE_SHIFT); 8128 else 8129 numentries <<= (PAGE_SHIFT - scale); 8130 8131 /* Make sure we've got at least a 0-order allocation.. */ 8132 if (unlikely(flags & HASH_SMALL)) { 8133 /* Makes no sense without HASH_EARLY */ 8134 WARN_ON(!(flags & HASH_EARLY)); 8135 if (!(numentries >> *_hash_shift)) { 8136 numentries = 1UL << *_hash_shift; 8137 BUG_ON(!numentries); 8138 } 8139 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 8140 numentries = PAGE_SIZE / bucketsize; 8141 } 8142 numentries = roundup_pow_of_two(numentries); 8143 8144 /* limit allocation size to 1/16 total memory by default */ 8145 if (max == 0) { 8146 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 8147 do_div(max, bucketsize); 8148 } 8149 max = min(max, 0x80000000ULL); 8150 8151 if (numentries < low_limit) 8152 numentries = low_limit; 8153 if (numentries > max) 8154 numentries = max; 8155 8156 log2qty = ilog2(numentries); 8157 8158 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 8159 do { 8160 virt = false; 8161 size = bucketsize << log2qty; 8162 if (flags & HASH_EARLY) { 8163 if (flags & HASH_ZERO) 8164 table = memblock_alloc(size, SMP_CACHE_BYTES); 8165 else 8166 table = memblock_alloc_raw(size, 8167 SMP_CACHE_BYTES); 8168 } else if (get_order(size) >= MAX_ORDER || hashdist) { 8169 table = __vmalloc(size, gfp_flags); 8170 virt = true; 8171 } else { 8172 /* 8173 * If bucketsize is not a power-of-two, we may free 8174 * some pages at the end of hash table which 8175 * alloc_pages_exact() automatically does 8176 */ 8177 table = alloc_pages_exact(size, gfp_flags); 8178 kmemleak_alloc(table, size, 1, gfp_flags); 8179 } 8180 } while (!table && size > PAGE_SIZE && --log2qty); 8181 8182 if (!table) 8183 panic("Failed to allocate %s hash table\n", tablename); 8184 8185 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 8186 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, 8187 virt ? "vmalloc" : "linear"); 8188 8189 if (_hash_shift) 8190 *_hash_shift = log2qty; 8191 if (_hash_mask) 8192 *_hash_mask = (1 << log2qty) - 1; 8193 8194 return table; 8195 } 8196 8197 /* 8198 * This function checks whether pageblock includes unmovable pages or not. 8199 * 8200 * PageLRU check without isolation or lru_lock could race so that 8201 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable 8202 * check without lock_page also may miss some movable non-lru pages at 8203 * race condition. So you can't expect this function should be exact. 8204 * 8205 * Returns a page without holding a reference. If the caller wants to 8206 * dereference that page (e.g., dumping), it has to make sure that it 8207 * cannot get removed (e.g., via memory unplug) concurrently. 8208 * 8209 */ 8210 struct page *has_unmovable_pages(struct zone *zone, struct page *page, 8211 int migratetype, int flags) 8212 { 8213 unsigned long iter = 0; 8214 unsigned long pfn = page_to_pfn(page); 8215 8216 /* 8217 * TODO we could make this much more efficient by not checking every 8218 * page in the range if we know all of them are in MOVABLE_ZONE and 8219 * that the movable zone guarantees that pages are migratable but 8220 * the later is not the case right now unfortunatelly. E.g. movablecore 8221 * can still lead to having bootmem allocations in zone_movable. 8222 */ 8223 8224 if (is_migrate_cma_page(page)) { 8225 /* 8226 * CMA allocations (alloc_contig_range) really need to mark 8227 * isolate CMA pageblocks even when they are not movable in fact 8228 * so consider them movable here. 8229 */ 8230 if (is_migrate_cma(migratetype)) 8231 return NULL; 8232 8233 return page; 8234 } 8235 8236 for (; iter < pageblock_nr_pages; iter++) { 8237 if (!pfn_valid_within(pfn + iter)) 8238 continue; 8239 8240 page = pfn_to_page(pfn + iter); 8241 8242 if (PageReserved(page)) 8243 return page; 8244 8245 /* 8246 * If the zone is movable and we have ruled out all reserved 8247 * pages then it should be reasonably safe to assume the rest 8248 * is movable. 8249 */ 8250 if (zone_idx(zone) == ZONE_MOVABLE) 8251 continue; 8252 8253 /* 8254 * Hugepages are not in LRU lists, but they're movable. 8255 * THPs are on the LRU, but need to be counted as #small pages. 8256 * We need not scan over tail pages because we don't 8257 * handle each tail page individually in migration. 8258 */ 8259 if (PageHuge(page) || PageTransCompound(page)) { 8260 struct page *head = compound_head(page); 8261 unsigned int skip_pages; 8262 8263 if (PageHuge(page)) { 8264 if (!hugepage_migration_supported(page_hstate(head))) 8265 return page; 8266 } else if (!PageLRU(head) && !__PageMovable(head)) { 8267 return page; 8268 } 8269 8270 skip_pages = compound_nr(head) - (page - head); 8271 iter += skip_pages - 1; 8272 continue; 8273 } 8274 8275 /* 8276 * We can't use page_count without pin a page 8277 * because another CPU can free compound page. 8278 * This check already skips compound tails of THP 8279 * because their page->_refcount is zero at all time. 8280 */ 8281 if (!page_ref_count(page)) { 8282 if (PageBuddy(page)) 8283 iter += (1 << page_order(page)) - 1; 8284 continue; 8285 } 8286 8287 /* 8288 * The HWPoisoned page may be not in buddy system, and 8289 * page_count() is not 0. 8290 */ 8291 if ((flags & MEMORY_OFFLINE) && PageHWPoison(page)) 8292 continue; 8293 8294 /* 8295 * We treat all PageOffline() pages as movable when offlining 8296 * to give drivers a chance to decrement their reference count 8297 * in MEM_GOING_OFFLINE in order to indicate that these pages 8298 * can be offlined as there are no direct references anymore. 8299 * For actually unmovable PageOffline() where the driver does 8300 * not support this, we will fail later when trying to actually 8301 * move these pages that still have a reference count > 0. 8302 * (false negatives in this function only) 8303 */ 8304 if ((flags & MEMORY_OFFLINE) && PageOffline(page)) 8305 continue; 8306 8307 if (__PageMovable(page) || PageLRU(page)) 8308 continue; 8309 8310 /* 8311 * If there are RECLAIMABLE pages, we need to check 8312 * it. But now, memory offline itself doesn't call 8313 * shrink_node_slabs() and it still to be fixed. 8314 */ 8315 /* 8316 * If the page is not RAM, page_count()should be 0. 8317 * we don't need more check. This is an _used_ not-movable page. 8318 * 8319 * The problematic thing here is PG_reserved pages. PG_reserved 8320 * is set to both of a memory hole page and a _used_ kernel 8321 * page at boot. 8322 */ 8323 return page; 8324 } 8325 return NULL; 8326 } 8327 8328 #ifdef CONFIG_CONTIG_ALLOC 8329 static unsigned long pfn_max_align_down(unsigned long pfn) 8330 { 8331 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, 8332 pageblock_nr_pages) - 1); 8333 } 8334 8335 static unsigned long pfn_max_align_up(unsigned long pfn) 8336 { 8337 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, 8338 pageblock_nr_pages)); 8339 } 8340 8341 /* [start, end) must belong to a single zone. */ 8342 static int __alloc_contig_migrate_range(struct compact_control *cc, 8343 unsigned long start, unsigned long end) 8344 { 8345 /* This function is based on compact_zone() from compaction.c. */ 8346 unsigned int nr_reclaimed; 8347 unsigned long pfn = start; 8348 unsigned int tries = 0; 8349 int ret = 0; 8350 8351 migrate_prep(); 8352 8353 while (pfn < end || !list_empty(&cc->migratepages)) { 8354 if (fatal_signal_pending(current)) { 8355 ret = -EINTR; 8356 break; 8357 } 8358 8359 if (list_empty(&cc->migratepages)) { 8360 cc->nr_migratepages = 0; 8361 pfn = isolate_migratepages_range(cc, pfn, end); 8362 if (!pfn) { 8363 ret = -EINTR; 8364 break; 8365 } 8366 tries = 0; 8367 } else if (++tries == 5) { 8368 ret = ret < 0 ? ret : -EBUSY; 8369 break; 8370 } 8371 8372 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 8373 &cc->migratepages); 8374 cc->nr_migratepages -= nr_reclaimed; 8375 8376 ret = migrate_pages(&cc->migratepages, alloc_migrate_target, 8377 NULL, 0, cc->mode, MR_CONTIG_RANGE); 8378 } 8379 if (ret < 0) { 8380 putback_movable_pages(&cc->migratepages); 8381 return ret; 8382 } 8383 return 0; 8384 } 8385 8386 /** 8387 * alloc_contig_range() -- tries to allocate given range of pages 8388 * @start: start PFN to allocate 8389 * @end: one-past-the-last PFN to allocate 8390 * @migratetype: migratetype of the underlaying pageblocks (either 8391 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 8392 * in range must have the same migratetype and it must 8393 * be either of the two. 8394 * @gfp_mask: GFP mask to use during compaction 8395 * 8396 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 8397 * aligned. The PFN range must belong to a single zone. 8398 * 8399 * The first thing this routine does is attempt to MIGRATE_ISOLATE all 8400 * pageblocks in the range. Once isolated, the pageblocks should not 8401 * be modified by others. 8402 * 8403 * Return: zero on success or negative error code. On success all 8404 * pages which PFN is in [start, end) are allocated for the caller and 8405 * need to be freed with free_contig_range(). 8406 */ 8407 int alloc_contig_range(unsigned long start, unsigned long end, 8408 unsigned migratetype, gfp_t gfp_mask) 8409 { 8410 unsigned long outer_start, outer_end; 8411 unsigned int order; 8412 int ret = 0; 8413 8414 struct compact_control cc = { 8415 .nr_migratepages = 0, 8416 .order = -1, 8417 .zone = page_zone(pfn_to_page(start)), 8418 .mode = MIGRATE_SYNC, 8419 .ignore_skip_hint = true, 8420 .no_set_skip_hint = true, 8421 .gfp_mask = current_gfp_context(gfp_mask), 8422 .alloc_contig = true, 8423 }; 8424 INIT_LIST_HEAD(&cc.migratepages); 8425 8426 /* 8427 * What we do here is we mark all pageblocks in range as 8428 * MIGRATE_ISOLATE. Because pageblock and max order pages may 8429 * have different sizes, and due to the way page allocator 8430 * work, we align the range to biggest of the two pages so 8431 * that page allocator won't try to merge buddies from 8432 * different pageblocks and change MIGRATE_ISOLATE to some 8433 * other migration type. 8434 * 8435 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 8436 * migrate the pages from an unaligned range (ie. pages that 8437 * we are interested in). This will put all the pages in 8438 * range back to page allocator as MIGRATE_ISOLATE. 8439 * 8440 * When this is done, we take the pages in range from page 8441 * allocator removing them from the buddy system. This way 8442 * page allocator will never consider using them. 8443 * 8444 * This lets us mark the pageblocks back as 8445 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 8446 * aligned range but not in the unaligned, original range are 8447 * put back to page allocator so that buddy can use them. 8448 */ 8449 8450 ret = start_isolate_page_range(pfn_max_align_down(start), 8451 pfn_max_align_up(end), migratetype, 0); 8452 if (ret < 0) 8453 return ret; 8454 8455 /* 8456 * In case of -EBUSY, we'd like to know which page causes problem. 8457 * So, just fall through. test_pages_isolated() has a tracepoint 8458 * which will report the busy page. 8459 * 8460 * It is possible that busy pages could become available before 8461 * the call to test_pages_isolated, and the range will actually be 8462 * allocated. So, if we fall through be sure to clear ret so that 8463 * -EBUSY is not accidentally used or returned to caller. 8464 */ 8465 ret = __alloc_contig_migrate_range(&cc, start, end); 8466 if (ret && ret != -EBUSY) 8467 goto done; 8468 ret =0; 8469 8470 /* 8471 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 8472 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 8473 * more, all pages in [start, end) are free in page allocator. 8474 * What we are going to do is to allocate all pages from 8475 * [start, end) (that is remove them from page allocator). 8476 * 8477 * The only problem is that pages at the beginning and at the 8478 * end of interesting range may be not aligned with pages that 8479 * page allocator holds, ie. they can be part of higher order 8480 * pages. Because of this, we reserve the bigger range and 8481 * once this is done free the pages we are not interested in. 8482 * 8483 * We don't have to hold zone->lock here because the pages are 8484 * isolated thus they won't get removed from buddy. 8485 */ 8486 8487 lru_add_drain_all(); 8488 8489 order = 0; 8490 outer_start = start; 8491 while (!PageBuddy(pfn_to_page(outer_start))) { 8492 if (++order >= MAX_ORDER) { 8493 outer_start = start; 8494 break; 8495 } 8496 outer_start &= ~0UL << order; 8497 } 8498 8499 if (outer_start != start) { 8500 order = page_order(pfn_to_page(outer_start)); 8501 8502 /* 8503 * outer_start page could be small order buddy page and 8504 * it doesn't include start page. Adjust outer_start 8505 * in this case to report failed page properly 8506 * on tracepoint in test_pages_isolated() 8507 */ 8508 if (outer_start + (1UL << order) <= start) 8509 outer_start = start; 8510 } 8511 8512 /* Make sure the range is really isolated. */ 8513 if (test_pages_isolated(outer_start, end, 0)) { 8514 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n", 8515 __func__, outer_start, end); 8516 ret = -EBUSY; 8517 goto done; 8518 } 8519 8520 /* Grab isolated pages from freelists. */ 8521 outer_end = isolate_freepages_range(&cc, outer_start, end); 8522 if (!outer_end) { 8523 ret = -EBUSY; 8524 goto done; 8525 } 8526 8527 /* Free head and tail (if any) */ 8528 if (start != outer_start) 8529 free_contig_range(outer_start, start - outer_start); 8530 if (end != outer_end) 8531 free_contig_range(end, outer_end - end); 8532 8533 done: 8534 undo_isolate_page_range(pfn_max_align_down(start), 8535 pfn_max_align_up(end), migratetype); 8536 return ret; 8537 } 8538 EXPORT_SYMBOL(alloc_contig_range); 8539 8540 static int __alloc_contig_pages(unsigned long start_pfn, 8541 unsigned long nr_pages, gfp_t gfp_mask) 8542 { 8543 unsigned long end_pfn = start_pfn + nr_pages; 8544 8545 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE, 8546 gfp_mask); 8547 } 8548 8549 static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, 8550 unsigned long nr_pages) 8551 { 8552 unsigned long i, end_pfn = start_pfn + nr_pages; 8553 struct page *page; 8554 8555 for (i = start_pfn; i < end_pfn; i++) { 8556 page = pfn_to_online_page(i); 8557 if (!page) 8558 return false; 8559 8560 if (page_zone(page) != z) 8561 return false; 8562 8563 if (PageReserved(page)) 8564 return false; 8565 8566 if (page_count(page) > 0) 8567 return false; 8568 8569 if (PageHuge(page)) 8570 return false; 8571 } 8572 return true; 8573 } 8574 8575 static bool zone_spans_last_pfn(const struct zone *zone, 8576 unsigned long start_pfn, unsigned long nr_pages) 8577 { 8578 unsigned long last_pfn = start_pfn + nr_pages - 1; 8579 8580 return zone_spans_pfn(zone, last_pfn); 8581 } 8582 8583 /** 8584 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages 8585 * @nr_pages: Number of contiguous pages to allocate 8586 * @gfp_mask: GFP mask to limit search and used during compaction 8587 * @nid: Target node 8588 * @nodemask: Mask for other possible nodes 8589 * 8590 * This routine is a wrapper around alloc_contig_range(). It scans over zones 8591 * on an applicable zonelist to find a contiguous pfn range which can then be 8592 * tried for allocation with alloc_contig_range(). This routine is intended 8593 * for allocation requests which can not be fulfilled with the buddy allocator. 8594 * 8595 * The allocated memory is always aligned to a page boundary. If nr_pages is a 8596 * power of two then the alignment is guaranteed to be to the given nr_pages 8597 * (e.g. 1GB request would be aligned to 1GB). 8598 * 8599 * Allocated pages can be freed with free_contig_range() or by manually calling 8600 * __free_page() on each allocated page. 8601 * 8602 * Return: pointer to contiguous pages on success, or NULL if not successful. 8603 */ 8604 struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, 8605 int nid, nodemask_t *nodemask) 8606 { 8607 unsigned long ret, pfn, flags; 8608 struct zonelist *zonelist; 8609 struct zone *zone; 8610 struct zoneref *z; 8611 8612 zonelist = node_zonelist(nid, gfp_mask); 8613 for_each_zone_zonelist_nodemask(zone, z, zonelist, 8614 gfp_zone(gfp_mask), nodemask) { 8615 spin_lock_irqsave(&zone->lock, flags); 8616 8617 pfn = ALIGN(zone->zone_start_pfn, nr_pages); 8618 while (zone_spans_last_pfn(zone, pfn, nr_pages)) { 8619 if (pfn_range_valid_contig(zone, pfn, nr_pages)) { 8620 /* 8621 * We release the zone lock here because 8622 * alloc_contig_range() will also lock the zone 8623 * at some point. If there's an allocation 8624 * spinning on this lock, it may win the race 8625 * and cause alloc_contig_range() to fail... 8626 */ 8627 spin_unlock_irqrestore(&zone->lock, flags); 8628 ret = __alloc_contig_pages(pfn, nr_pages, 8629 gfp_mask); 8630 if (!ret) 8631 return pfn_to_page(pfn); 8632 spin_lock_irqsave(&zone->lock, flags); 8633 } 8634 pfn += nr_pages; 8635 } 8636 spin_unlock_irqrestore(&zone->lock, flags); 8637 } 8638 return NULL; 8639 } 8640 #endif /* CONFIG_CONTIG_ALLOC */ 8641 8642 void free_contig_range(unsigned long pfn, unsigned int nr_pages) 8643 { 8644 unsigned int count = 0; 8645 8646 for (; nr_pages--; pfn++) { 8647 struct page *page = pfn_to_page(pfn); 8648 8649 count += page_count(page) != 1; 8650 __free_page(page); 8651 } 8652 WARN(count != 0, "%d pages are still in use!\n", count); 8653 } 8654 EXPORT_SYMBOL(free_contig_range); 8655 8656 /* 8657 * The zone indicated has a new number of managed_pages; batch sizes and percpu 8658 * page high values need to be recalulated. 8659 */ 8660 void __meminit zone_pcp_update(struct zone *zone) 8661 { 8662 mutex_lock(&pcp_batch_high_lock); 8663 __zone_pcp_update(zone); 8664 mutex_unlock(&pcp_batch_high_lock); 8665 } 8666 8667 void zone_pcp_reset(struct zone *zone) 8668 { 8669 unsigned long flags; 8670 int cpu; 8671 struct per_cpu_pageset *pset; 8672 8673 /* avoid races with drain_pages() */ 8674 local_irq_save(flags); 8675 if (zone->pageset != &boot_pageset) { 8676 for_each_online_cpu(cpu) { 8677 pset = per_cpu_ptr(zone->pageset, cpu); 8678 drain_zonestat(zone, pset); 8679 } 8680 free_percpu(zone->pageset); 8681 zone->pageset = &boot_pageset; 8682 } 8683 local_irq_restore(flags); 8684 } 8685 8686 #ifdef CONFIG_MEMORY_HOTREMOVE 8687 /* 8688 * All pages in the range must be in a single zone and isolated 8689 * before calling this. 8690 */ 8691 unsigned long 8692 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 8693 { 8694 struct page *page; 8695 struct zone *zone; 8696 unsigned int order; 8697 unsigned long pfn; 8698 unsigned long flags; 8699 unsigned long offlined_pages = 0; 8700 8701 /* find the first valid pfn */ 8702 for (pfn = start_pfn; pfn < end_pfn; pfn++) 8703 if (pfn_valid(pfn)) 8704 break; 8705 if (pfn == end_pfn) 8706 return offlined_pages; 8707 8708 offline_mem_sections(pfn, end_pfn); 8709 zone = page_zone(pfn_to_page(pfn)); 8710 spin_lock_irqsave(&zone->lock, flags); 8711 pfn = start_pfn; 8712 while (pfn < end_pfn) { 8713 if (!pfn_valid(pfn)) { 8714 pfn++; 8715 continue; 8716 } 8717 page = pfn_to_page(pfn); 8718 /* 8719 * The HWPoisoned page may be not in buddy system, and 8720 * page_count() is not 0. 8721 */ 8722 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 8723 pfn++; 8724 offlined_pages++; 8725 continue; 8726 } 8727 /* 8728 * At this point all remaining PageOffline() pages have a 8729 * reference count of 0 and can simply be skipped. 8730 */ 8731 if (PageOffline(page)) { 8732 BUG_ON(page_count(page)); 8733 BUG_ON(PageBuddy(page)); 8734 pfn++; 8735 offlined_pages++; 8736 continue; 8737 } 8738 8739 BUG_ON(page_count(page)); 8740 BUG_ON(!PageBuddy(page)); 8741 order = page_order(page); 8742 offlined_pages += 1 << order; 8743 del_page_from_free_list(page, zone, order); 8744 pfn += (1 << order); 8745 } 8746 spin_unlock_irqrestore(&zone->lock, flags); 8747 8748 return offlined_pages; 8749 } 8750 #endif 8751 8752 bool is_free_buddy_page(struct page *page) 8753 { 8754 struct zone *zone = page_zone(page); 8755 unsigned long pfn = page_to_pfn(page); 8756 unsigned long flags; 8757 unsigned int order; 8758 8759 spin_lock_irqsave(&zone->lock, flags); 8760 for (order = 0; order < MAX_ORDER; order++) { 8761 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8762 8763 if (PageBuddy(page_head) && page_order(page_head) >= order) 8764 break; 8765 } 8766 spin_unlock_irqrestore(&zone->lock, flags); 8767 8768 return order < MAX_ORDER; 8769 } 8770 8771 #ifdef CONFIG_MEMORY_FAILURE 8772 /* 8773 * Set PG_hwpoison flag if a given page is confirmed to be a free page. This 8774 * test is performed under the zone lock to prevent a race against page 8775 * allocation. 8776 */ 8777 bool set_hwpoison_free_buddy_page(struct page *page) 8778 { 8779 struct zone *zone = page_zone(page); 8780 unsigned long pfn = page_to_pfn(page); 8781 unsigned long flags; 8782 unsigned int order; 8783 bool hwpoisoned = false; 8784 8785 spin_lock_irqsave(&zone->lock, flags); 8786 for (order = 0; order < MAX_ORDER; order++) { 8787 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8788 8789 if (PageBuddy(page_head) && page_order(page_head) >= order) { 8790 if (!TestSetPageHWPoison(page)) 8791 hwpoisoned = true; 8792 break; 8793 } 8794 } 8795 spin_unlock_irqrestore(&zone->lock, flags); 8796 8797 return hwpoisoned; 8798 } 8799 #endif 8800