1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/page_alloc.c 4 * 5 * Manages the free list, the system allocates free pages here. 6 * Note that kmalloc() lives in slab.c 7 * 8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 9 * Swap reorganised 29.12.95, Stephen Tweedie 10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 16 */ 17 18 #include <linux/stddef.h> 19 #include <linux/mm.h> 20 #include <linux/highmem.h> 21 #include <linux/swap.h> 22 #include <linux/interrupt.h> 23 #include <linux/pagemap.h> 24 #include <linux/jiffies.h> 25 #include <linux/memblock.h> 26 #include <linux/compiler.h> 27 #include <linux/kernel.h> 28 #include <linux/kasan.h> 29 #include <linux/module.h> 30 #include <linux/suspend.h> 31 #include <linux/pagevec.h> 32 #include <linux/blkdev.h> 33 #include <linux/slab.h> 34 #include <linux/ratelimit.h> 35 #include <linux/oom.h> 36 #include <linux/topology.h> 37 #include <linux/sysctl.h> 38 #include <linux/cpu.h> 39 #include <linux/cpuset.h> 40 #include <linux/memory_hotplug.h> 41 #include <linux/nodemask.h> 42 #include <linux/vmalloc.h> 43 #include <linux/vmstat.h> 44 #include <linux/mempolicy.h> 45 #include <linux/memremap.h> 46 #include <linux/stop_machine.h> 47 #include <linux/random.h> 48 #include <linux/sort.h> 49 #include <linux/pfn.h> 50 #include <linux/backing-dev.h> 51 #include <linux/fault-inject.h> 52 #include <linux/page-isolation.h> 53 #include <linux/debugobjects.h> 54 #include <linux/kmemleak.h> 55 #include <linux/compaction.h> 56 #include <trace/events/kmem.h> 57 #include <trace/events/oom.h> 58 #include <linux/prefetch.h> 59 #include <linux/mm_inline.h> 60 #include <linux/migrate.h> 61 #include <linux/hugetlb.h> 62 #include <linux/sched/rt.h> 63 #include <linux/sched/mm.h> 64 #include <linux/page_owner.h> 65 #include <linux/kthread.h> 66 #include <linux/memcontrol.h> 67 #include <linux/ftrace.h> 68 #include <linux/lockdep.h> 69 #include <linux/nmi.h> 70 #include <linux/psi.h> 71 72 #include <asm/sections.h> 73 #include <asm/tlbflush.h> 74 #include <asm/div64.h> 75 #include "internal.h" 76 #include "shuffle.h" 77 #include "page_reporting.h" 78 79 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 80 static DEFINE_MUTEX(pcp_batch_high_lock); 81 #define MIN_PERCPU_PAGELIST_FRACTION (8) 82 83 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 84 DEFINE_PER_CPU(int, numa_node); 85 EXPORT_PER_CPU_SYMBOL(numa_node); 86 #endif 87 88 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); 89 90 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 91 /* 92 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 93 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 94 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 95 * defined in <linux/topology.h>. 96 */ 97 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 98 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 99 #endif 100 101 /* work_structs for global per-cpu drains */ 102 struct pcpu_drain { 103 struct zone *zone; 104 struct work_struct work; 105 }; 106 DEFINE_MUTEX(pcpu_drain_mutex); 107 DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain); 108 109 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 110 volatile unsigned long latent_entropy __latent_entropy; 111 EXPORT_SYMBOL(latent_entropy); 112 #endif 113 114 /* 115 * Array of node states. 116 */ 117 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 118 [N_POSSIBLE] = NODE_MASK_ALL, 119 [N_ONLINE] = { { [0] = 1UL } }, 120 #ifndef CONFIG_NUMA 121 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 122 #ifdef CONFIG_HIGHMEM 123 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 124 #endif 125 [N_MEMORY] = { { [0] = 1UL } }, 126 [N_CPU] = { { [0] = 1UL } }, 127 #endif /* NUMA */ 128 }; 129 EXPORT_SYMBOL(node_states); 130 131 atomic_long_t _totalram_pages __read_mostly; 132 EXPORT_SYMBOL(_totalram_pages); 133 unsigned long totalreserve_pages __read_mostly; 134 unsigned long totalcma_pages __read_mostly; 135 136 int percpu_pagelist_fraction; 137 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 138 #ifdef CONFIG_INIT_ON_ALLOC_DEFAULT_ON 139 DEFINE_STATIC_KEY_TRUE(init_on_alloc); 140 #else 141 DEFINE_STATIC_KEY_FALSE(init_on_alloc); 142 #endif 143 EXPORT_SYMBOL(init_on_alloc); 144 145 #ifdef CONFIG_INIT_ON_FREE_DEFAULT_ON 146 DEFINE_STATIC_KEY_TRUE(init_on_free); 147 #else 148 DEFINE_STATIC_KEY_FALSE(init_on_free); 149 #endif 150 EXPORT_SYMBOL(init_on_free); 151 152 static int __init early_init_on_alloc(char *buf) 153 { 154 int ret; 155 bool bool_result; 156 157 if (!buf) 158 return -EINVAL; 159 ret = kstrtobool(buf, &bool_result); 160 if (bool_result && page_poisoning_enabled()) 161 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_alloc\n"); 162 if (bool_result) 163 static_branch_enable(&init_on_alloc); 164 else 165 static_branch_disable(&init_on_alloc); 166 return ret; 167 } 168 early_param("init_on_alloc", early_init_on_alloc); 169 170 static int __init early_init_on_free(char *buf) 171 { 172 int ret; 173 bool bool_result; 174 175 if (!buf) 176 return -EINVAL; 177 ret = kstrtobool(buf, &bool_result); 178 if (bool_result && page_poisoning_enabled()) 179 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_free\n"); 180 if (bool_result) 181 static_branch_enable(&init_on_free); 182 else 183 static_branch_disable(&init_on_free); 184 return ret; 185 } 186 early_param("init_on_free", early_init_on_free); 187 188 /* 189 * A cached value of the page's pageblock's migratetype, used when the page is 190 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 191 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 192 * Also the migratetype set in the page does not necessarily match the pcplist 193 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 194 * other index - this ensures that it will be put on the correct CMA freelist. 195 */ 196 static inline int get_pcppage_migratetype(struct page *page) 197 { 198 return page->index; 199 } 200 201 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 202 { 203 page->index = migratetype; 204 } 205 206 #ifdef CONFIG_PM_SLEEP 207 /* 208 * The following functions are used by the suspend/hibernate code to temporarily 209 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 210 * while devices are suspended. To avoid races with the suspend/hibernate code, 211 * they should always be called with system_transition_mutex held 212 * (gfp_allowed_mask also should only be modified with system_transition_mutex 213 * held, unless the suspend/hibernate code is guaranteed not to run in parallel 214 * with that modification). 215 */ 216 217 static gfp_t saved_gfp_mask; 218 219 void pm_restore_gfp_mask(void) 220 { 221 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 222 if (saved_gfp_mask) { 223 gfp_allowed_mask = saved_gfp_mask; 224 saved_gfp_mask = 0; 225 } 226 } 227 228 void pm_restrict_gfp_mask(void) 229 { 230 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 231 WARN_ON(saved_gfp_mask); 232 saved_gfp_mask = gfp_allowed_mask; 233 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 234 } 235 236 bool pm_suspended_storage(void) 237 { 238 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 239 return false; 240 return true; 241 } 242 #endif /* CONFIG_PM_SLEEP */ 243 244 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 245 unsigned int pageblock_order __read_mostly; 246 #endif 247 248 static void __free_pages_ok(struct page *page, unsigned int order); 249 250 /* 251 * results with 256, 32 in the lowmem_reserve sysctl: 252 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 253 * 1G machine -> (16M dma, 784M normal, 224M high) 254 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 255 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 256 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 257 * 258 * TBD: should special case ZONE_DMA32 machines here - in those we normally 259 * don't need any ZONE_NORMAL reservation 260 */ 261 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { 262 #ifdef CONFIG_ZONE_DMA 263 [ZONE_DMA] = 256, 264 #endif 265 #ifdef CONFIG_ZONE_DMA32 266 [ZONE_DMA32] = 256, 267 #endif 268 [ZONE_NORMAL] = 32, 269 #ifdef CONFIG_HIGHMEM 270 [ZONE_HIGHMEM] = 0, 271 #endif 272 [ZONE_MOVABLE] = 0, 273 }; 274 275 static char * const zone_names[MAX_NR_ZONES] = { 276 #ifdef CONFIG_ZONE_DMA 277 "DMA", 278 #endif 279 #ifdef CONFIG_ZONE_DMA32 280 "DMA32", 281 #endif 282 "Normal", 283 #ifdef CONFIG_HIGHMEM 284 "HighMem", 285 #endif 286 "Movable", 287 #ifdef CONFIG_ZONE_DEVICE 288 "Device", 289 #endif 290 }; 291 292 const char * const migratetype_names[MIGRATE_TYPES] = { 293 "Unmovable", 294 "Movable", 295 "Reclaimable", 296 "HighAtomic", 297 #ifdef CONFIG_CMA 298 "CMA", 299 #endif 300 #ifdef CONFIG_MEMORY_ISOLATION 301 "Isolate", 302 #endif 303 }; 304 305 compound_page_dtor * const compound_page_dtors[] = { 306 NULL, 307 free_compound_page, 308 #ifdef CONFIG_HUGETLB_PAGE 309 free_huge_page, 310 #endif 311 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 312 free_transhuge_page, 313 #endif 314 }; 315 316 int min_free_kbytes = 1024; 317 int user_min_free_kbytes = -1; 318 #ifdef CONFIG_DISCONTIGMEM 319 /* 320 * DiscontigMem defines memory ranges as separate pg_data_t even if the ranges 321 * are not on separate NUMA nodes. Functionally this works but with 322 * watermark_boost_factor, it can reclaim prematurely as the ranges can be 323 * quite small. By default, do not boost watermarks on discontigmem as in 324 * many cases very high-order allocations like THP are likely to be 325 * unsupported and the premature reclaim offsets the advantage of long-term 326 * fragmentation avoidance. 327 */ 328 int watermark_boost_factor __read_mostly; 329 #else 330 int watermark_boost_factor __read_mostly = 15000; 331 #endif 332 int watermark_scale_factor = 10; 333 334 static unsigned long nr_kernel_pages __initdata; 335 static unsigned long nr_all_pages __initdata; 336 static unsigned long dma_reserve __initdata; 337 338 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 339 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 340 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 341 static unsigned long required_kernelcore __initdata; 342 static unsigned long required_kernelcore_percent __initdata; 343 static unsigned long required_movablecore __initdata; 344 static unsigned long required_movablecore_percent __initdata; 345 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 346 static bool mirrored_kernelcore __meminitdata; 347 348 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 349 int movable_zone; 350 EXPORT_SYMBOL(movable_zone); 351 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 352 353 #if MAX_NUMNODES > 1 354 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; 355 unsigned int nr_online_nodes __read_mostly = 1; 356 EXPORT_SYMBOL(nr_node_ids); 357 EXPORT_SYMBOL(nr_online_nodes); 358 #endif 359 360 int page_group_by_mobility_disabled __read_mostly; 361 362 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 363 /* 364 * During boot we initialize deferred pages on-demand, as needed, but once 365 * page_alloc_init_late() has finished, the deferred pages are all initialized, 366 * and we can permanently disable that path. 367 */ 368 static DEFINE_STATIC_KEY_TRUE(deferred_pages); 369 370 /* 371 * Calling kasan_free_pages() only after deferred memory initialization 372 * has completed. Poisoning pages during deferred memory init will greatly 373 * lengthen the process and cause problem in large memory systems as the 374 * deferred pages initialization is done with interrupt disabled. 375 * 376 * Assuming that there will be no reference to those newly initialized 377 * pages before they are ever allocated, this should have no effect on 378 * KASAN memory tracking as the poison will be properly inserted at page 379 * allocation time. The only corner case is when pages are allocated by 380 * on-demand allocation and then freed again before the deferred pages 381 * initialization is done, but this is not likely to happen. 382 */ 383 static inline void kasan_free_nondeferred_pages(struct page *page, int order) 384 { 385 if (!static_branch_unlikely(&deferred_pages)) 386 kasan_free_pages(page, order); 387 } 388 389 /* Returns true if the struct page for the pfn is uninitialised */ 390 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 391 { 392 int nid = early_pfn_to_nid(pfn); 393 394 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 395 return true; 396 397 return false; 398 } 399 400 /* 401 * Returns true when the remaining initialisation should be deferred until 402 * later in the boot cycle when it can be parallelised. 403 */ 404 static bool __meminit 405 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 406 { 407 static unsigned long prev_end_pfn, nr_initialised; 408 409 /* 410 * prev_end_pfn static that contains the end of previous zone 411 * No need to protect because called very early in boot before smp_init. 412 */ 413 if (prev_end_pfn != end_pfn) { 414 prev_end_pfn = end_pfn; 415 nr_initialised = 0; 416 } 417 418 /* Always populate low zones for address-constrained allocations */ 419 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 420 return false; 421 422 /* 423 * We start only with one section of pages, more pages are added as 424 * needed until the rest of deferred pages are initialized. 425 */ 426 nr_initialised++; 427 if ((nr_initialised > PAGES_PER_SECTION) && 428 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 429 NODE_DATA(nid)->first_deferred_pfn = pfn; 430 return true; 431 } 432 return false; 433 } 434 #else 435 #define kasan_free_nondeferred_pages(p, o) kasan_free_pages(p, o) 436 437 static inline bool early_page_uninitialised(unsigned long pfn) 438 { 439 return false; 440 } 441 442 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 443 { 444 return false; 445 } 446 #endif 447 448 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 449 static inline unsigned long *get_pageblock_bitmap(struct page *page, 450 unsigned long pfn) 451 { 452 #ifdef CONFIG_SPARSEMEM 453 return section_to_usemap(__pfn_to_section(pfn)); 454 #else 455 return page_zone(page)->pageblock_flags; 456 #endif /* CONFIG_SPARSEMEM */ 457 } 458 459 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn) 460 { 461 #ifdef CONFIG_SPARSEMEM 462 pfn &= (PAGES_PER_SECTION-1); 463 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 464 #else 465 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 466 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 467 #endif /* CONFIG_SPARSEMEM */ 468 } 469 470 /** 471 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 472 * @page: The page within the block of interest 473 * @pfn: The target page frame number 474 * @end_bitidx: The last bit of interest to retrieve 475 * @mask: mask of bits that the caller is interested in 476 * 477 * Return: pageblock_bits flags 478 */ 479 static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page, 480 unsigned long pfn, 481 unsigned long end_bitidx, 482 unsigned long mask) 483 { 484 unsigned long *bitmap; 485 unsigned long bitidx, word_bitidx; 486 unsigned long word; 487 488 bitmap = get_pageblock_bitmap(page, pfn); 489 bitidx = pfn_to_bitidx(page, pfn); 490 word_bitidx = bitidx / BITS_PER_LONG; 491 bitidx &= (BITS_PER_LONG-1); 492 493 word = bitmap[word_bitidx]; 494 bitidx += end_bitidx; 495 return (word >> (BITS_PER_LONG - bitidx - 1)) & mask; 496 } 497 498 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, 499 unsigned long end_bitidx, 500 unsigned long mask) 501 { 502 return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask); 503 } 504 505 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn) 506 { 507 return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK); 508 } 509 510 /** 511 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 512 * @page: The page within the block of interest 513 * @flags: The flags to set 514 * @pfn: The target page frame number 515 * @end_bitidx: The last bit of interest 516 * @mask: mask of bits that the caller is interested in 517 */ 518 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 519 unsigned long pfn, 520 unsigned long end_bitidx, 521 unsigned long mask) 522 { 523 unsigned long *bitmap; 524 unsigned long bitidx, word_bitidx; 525 unsigned long old_word, word; 526 527 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 528 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits)); 529 530 bitmap = get_pageblock_bitmap(page, pfn); 531 bitidx = pfn_to_bitidx(page, pfn); 532 word_bitidx = bitidx / BITS_PER_LONG; 533 bitidx &= (BITS_PER_LONG-1); 534 535 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 536 537 bitidx += end_bitidx; 538 mask <<= (BITS_PER_LONG - bitidx - 1); 539 flags <<= (BITS_PER_LONG - bitidx - 1); 540 541 word = READ_ONCE(bitmap[word_bitidx]); 542 for (;;) { 543 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 544 if (word == old_word) 545 break; 546 word = old_word; 547 } 548 } 549 550 void set_pageblock_migratetype(struct page *page, int migratetype) 551 { 552 if (unlikely(page_group_by_mobility_disabled && 553 migratetype < MIGRATE_PCPTYPES)) 554 migratetype = MIGRATE_UNMOVABLE; 555 556 set_pageblock_flags_group(page, (unsigned long)migratetype, 557 PB_migrate, PB_migrate_end); 558 } 559 560 #ifdef CONFIG_DEBUG_VM 561 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 562 { 563 int ret = 0; 564 unsigned seq; 565 unsigned long pfn = page_to_pfn(page); 566 unsigned long sp, start_pfn; 567 568 do { 569 seq = zone_span_seqbegin(zone); 570 start_pfn = zone->zone_start_pfn; 571 sp = zone->spanned_pages; 572 if (!zone_spans_pfn(zone, pfn)) 573 ret = 1; 574 } while (zone_span_seqretry(zone, seq)); 575 576 if (ret) 577 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 578 pfn, zone_to_nid(zone), zone->name, 579 start_pfn, start_pfn + sp); 580 581 return ret; 582 } 583 584 static int page_is_consistent(struct zone *zone, struct page *page) 585 { 586 if (!pfn_valid_within(page_to_pfn(page))) 587 return 0; 588 if (zone != page_zone(page)) 589 return 0; 590 591 return 1; 592 } 593 /* 594 * Temporary debugging check for pages not lying within a given zone. 595 */ 596 static int __maybe_unused bad_range(struct zone *zone, struct page *page) 597 { 598 if (page_outside_zone_boundaries(zone, page)) 599 return 1; 600 if (!page_is_consistent(zone, page)) 601 return 1; 602 603 return 0; 604 } 605 #else 606 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page) 607 { 608 return 0; 609 } 610 #endif 611 612 static void bad_page(struct page *page, const char *reason, 613 unsigned long bad_flags) 614 { 615 static unsigned long resume; 616 static unsigned long nr_shown; 617 static unsigned long nr_unshown; 618 619 /* 620 * Allow a burst of 60 reports, then keep quiet for that minute; 621 * or allow a steady drip of one report per second. 622 */ 623 if (nr_shown == 60) { 624 if (time_before(jiffies, resume)) { 625 nr_unshown++; 626 goto out; 627 } 628 if (nr_unshown) { 629 pr_alert( 630 "BUG: Bad page state: %lu messages suppressed\n", 631 nr_unshown); 632 nr_unshown = 0; 633 } 634 nr_shown = 0; 635 } 636 if (nr_shown++ == 0) 637 resume = jiffies + 60 * HZ; 638 639 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 640 current->comm, page_to_pfn(page)); 641 __dump_page(page, reason); 642 bad_flags &= page->flags; 643 if (bad_flags) 644 pr_alert("bad because of flags: %#lx(%pGp)\n", 645 bad_flags, &bad_flags); 646 dump_page_owner(page); 647 648 print_modules(); 649 dump_stack(); 650 out: 651 /* Leave bad fields for debug, except PageBuddy could make trouble */ 652 page_mapcount_reset(page); /* remove PageBuddy */ 653 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 654 } 655 656 /* 657 * Higher-order pages are called "compound pages". They are structured thusly: 658 * 659 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 660 * 661 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 662 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 663 * 664 * The first tail page's ->compound_dtor holds the offset in array of compound 665 * page destructors. See compound_page_dtors. 666 * 667 * The first tail page's ->compound_order holds the order of allocation. 668 * This usage means that zero-order pages may not be compound. 669 */ 670 671 void free_compound_page(struct page *page) 672 { 673 mem_cgroup_uncharge(page); 674 __free_pages_ok(page, compound_order(page)); 675 } 676 677 void prep_compound_page(struct page *page, unsigned int order) 678 { 679 int i; 680 int nr_pages = 1 << order; 681 682 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 683 set_compound_order(page, order); 684 __SetPageHead(page); 685 for (i = 1; i < nr_pages; i++) { 686 struct page *p = page + i; 687 set_page_count(p, 0); 688 p->mapping = TAIL_MAPPING; 689 set_compound_head(p, page); 690 } 691 atomic_set(compound_mapcount_ptr(page), -1); 692 if (hpage_pincount_available(page)) 693 atomic_set(compound_pincount_ptr(page), 0); 694 } 695 696 #ifdef CONFIG_DEBUG_PAGEALLOC 697 unsigned int _debug_guardpage_minorder; 698 699 bool _debug_pagealloc_enabled_early __read_mostly 700 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 701 EXPORT_SYMBOL(_debug_pagealloc_enabled_early); 702 DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 703 EXPORT_SYMBOL(_debug_pagealloc_enabled); 704 705 DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 706 707 static int __init early_debug_pagealloc(char *buf) 708 { 709 return kstrtobool(buf, &_debug_pagealloc_enabled_early); 710 } 711 early_param("debug_pagealloc", early_debug_pagealloc); 712 713 void init_debug_pagealloc(void) 714 { 715 if (!debug_pagealloc_enabled()) 716 return; 717 718 static_branch_enable(&_debug_pagealloc_enabled); 719 720 if (!debug_guardpage_minorder()) 721 return; 722 723 static_branch_enable(&_debug_guardpage_enabled); 724 } 725 726 static int __init debug_guardpage_minorder_setup(char *buf) 727 { 728 unsigned long res; 729 730 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 731 pr_err("Bad debug_guardpage_minorder value\n"); 732 return 0; 733 } 734 _debug_guardpage_minorder = res; 735 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 736 return 0; 737 } 738 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); 739 740 static inline bool set_page_guard(struct zone *zone, struct page *page, 741 unsigned int order, int migratetype) 742 { 743 if (!debug_guardpage_enabled()) 744 return false; 745 746 if (order >= debug_guardpage_minorder()) 747 return false; 748 749 __SetPageGuard(page); 750 INIT_LIST_HEAD(&page->lru); 751 set_page_private(page, order); 752 /* Guard pages are not available for any usage */ 753 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 754 755 return true; 756 } 757 758 static inline void clear_page_guard(struct zone *zone, struct page *page, 759 unsigned int order, int migratetype) 760 { 761 if (!debug_guardpage_enabled()) 762 return; 763 764 __ClearPageGuard(page); 765 766 set_page_private(page, 0); 767 if (!is_migrate_isolate(migratetype)) 768 __mod_zone_freepage_state(zone, (1 << order), migratetype); 769 } 770 #else 771 static inline bool set_page_guard(struct zone *zone, struct page *page, 772 unsigned int order, int migratetype) { return false; } 773 static inline void clear_page_guard(struct zone *zone, struct page *page, 774 unsigned int order, int migratetype) {} 775 #endif 776 777 static inline void set_page_order(struct page *page, unsigned int order) 778 { 779 set_page_private(page, order); 780 __SetPageBuddy(page); 781 } 782 783 /* 784 * This function checks whether a page is free && is the buddy 785 * we can coalesce a page and its buddy if 786 * (a) the buddy is not in a hole (check before calling!) && 787 * (b) the buddy is in the buddy system && 788 * (c) a page and its buddy have the same order && 789 * (d) a page and its buddy are in the same zone. 790 * 791 * For recording whether a page is in the buddy system, we set PageBuddy. 792 * Setting, clearing, and testing PageBuddy is serialized by zone->lock. 793 * 794 * For recording page's order, we use page_private(page). 795 */ 796 static inline bool page_is_buddy(struct page *page, struct page *buddy, 797 unsigned int order) 798 { 799 if (!page_is_guard(buddy) && !PageBuddy(buddy)) 800 return false; 801 802 if (page_order(buddy) != order) 803 return false; 804 805 /* 806 * zone check is done late to avoid uselessly calculating 807 * zone/node ids for pages that could never merge. 808 */ 809 if (page_zone_id(page) != page_zone_id(buddy)) 810 return false; 811 812 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 813 814 return true; 815 } 816 817 #ifdef CONFIG_COMPACTION 818 static inline struct capture_control *task_capc(struct zone *zone) 819 { 820 struct capture_control *capc = current->capture_control; 821 822 return capc && 823 !(current->flags & PF_KTHREAD) && 824 !capc->page && 825 capc->cc->zone == zone && 826 capc->cc->direct_compaction ? capc : NULL; 827 } 828 829 static inline bool 830 compaction_capture(struct capture_control *capc, struct page *page, 831 int order, int migratetype) 832 { 833 if (!capc || order != capc->cc->order) 834 return false; 835 836 /* Do not accidentally pollute CMA or isolated regions*/ 837 if (is_migrate_cma(migratetype) || 838 is_migrate_isolate(migratetype)) 839 return false; 840 841 /* 842 * Do not let lower order allocations polluate a movable pageblock. 843 * This might let an unmovable request use a reclaimable pageblock 844 * and vice-versa but no more than normal fallback logic which can 845 * have trouble finding a high-order free page. 846 */ 847 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE) 848 return false; 849 850 capc->page = page; 851 return true; 852 } 853 854 #else 855 static inline struct capture_control *task_capc(struct zone *zone) 856 { 857 return NULL; 858 } 859 860 static inline bool 861 compaction_capture(struct capture_control *capc, struct page *page, 862 int order, int migratetype) 863 { 864 return false; 865 } 866 #endif /* CONFIG_COMPACTION */ 867 868 /* Used for pages not on another list */ 869 static inline void add_to_free_list(struct page *page, struct zone *zone, 870 unsigned int order, int migratetype) 871 { 872 struct free_area *area = &zone->free_area[order]; 873 874 list_add(&page->lru, &area->free_list[migratetype]); 875 area->nr_free++; 876 } 877 878 /* Used for pages not on another list */ 879 static inline void add_to_free_list_tail(struct page *page, struct zone *zone, 880 unsigned int order, int migratetype) 881 { 882 struct free_area *area = &zone->free_area[order]; 883 884 list_add_tail(&page->lru, &area->free_list[migratetype]); 885 area->nr_free++; 886 } 887 888 /* Used for pages which are on another list */ 889 static inline void move_to_free_list(struct page *page, struct zone *zone, 890 unsigned int order, int migratetype) 891 { 892 struct free_area *area = &zone->free_area[order]; 893 894 list_move(&page->lru, &area->free_list[migratetype]); 895 } 896 897 static inline void del_page_from_free_list(struct page *page, struct zone *zone, 898 unsigned int order) 899 { 900 /* clear reported state and update reported page count */ 901 if (page_reported(page)) 902 __ClearPageReported(page); 903 904 list_del(&page->lru); 905 __ClearPageBuddy(page); 906 set_page_private(page, 0); 907 zone->free_area[order].nr_free--; 908 } 909 910 /* 911 * If this is not the largest possible page, check if the buddy 912 * of the next-highest order is free. If it is, it's possible 913 * that pages are being freed that will coalesce soon. In case, 914 * that is happening, add the free page to the tail of the list 915 * so it's less likely to be used soon and more likely to be merged 916 * as a higher order page 917 */ 918 static inline bool 919 buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, 920 struct page *page, unsigned int order) 921 { 922 struct page *higher_page, *higher_buddy; 923 unsigned long combined_pfn; 924 925 if (order >= MAX_ORDER - 2) 926 return false; 927 928 if (!pfn_valid_within(buddy_pfn)) 929 return false; 930 931 combined_pfn = buddy_pfn & pfn; 932 higher_page = page + (combined_pfn - pfn); 933 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1); 934 higher_buddy = higher_page + (buddy_pfn - combined_pfn); 935 936 return pfn_valid_within(buddy_pfn) && 937 page_is_buddy(higher_page, higher_buddy, order + 1); 938 } 939 940 /* 941 * Freeing function for a buddy system allocator. 942 * 943 * The concept of a buddy system is to maintain direct-mapped table 944 * (containing bit values) for memory blocks of various "orders". 945 * The bottom level table contains the map for the smallest allocatable 946 * units of memory (here, pages), and each level above it describes 947 * pairs of units from the levels below, hence, "buddies". 948 * At a high level, all that happens here is marking the table entry 949 * at the bottom level available, and propagating the changes upward 950 * as necessary, plus some accounting needed to play nicely with other 951 * parts of the VM system. 952 * At each level, we keep a list of pages, which are heads of continuous 953 * free pages of length of (1 << order) and marked with PageBuddy. 954 * Page's order is recorded in page_private(page) field. 955 * So when we are allocating or freeing one, we can derive the state of the 956 * other. That is, if we allocate a small block, and both were 957 * free, the remainder of the region must be split into blocks. 958 * If a block is freed, and its buddy is also free, then this 959 * triggers coalescing into a block of larger size. 960 * 961 * -- nyc 962 */ 963 964 static inline void __free_one_page(struct page *page, 965 unsigned long pfn, 966 struct zone *zone, unsigned int order, 967 int migratetype, bool report) 968 { 969 struct capture_control *capc = task_capc(zone); 970 unsigned long uninitialized_var(buddy_pfn); 971 unsigned long combined_pfn; 972 unsigned int max_order; 973 struct page *buddy; 974 bool to_tail; 975 976 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1); 977 978 VM_BUG_ON(!zone_is_initialized(zone)); 979 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 980 981 VM_BUG_ON(migratetype == -1); 982 if (likely(!is_migrate_isolate(migratetype))) 983 __mod_zone_freepage_state(zone, 1 << order, migratetype); 984 985 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); 986 VM_BUG_ON_PAGE(bad_range(zone, page), page); 987 988 continue_merging: 989 while (order < max_order - 1) { 990 if (compaction_capture(capc, page, order, migratetype)) { 991 __mod_zone_freepage_state(zone, -(1 << order), 992 migratetype); 993 return; 994 } 995 buddy_pfn = __find_buddy_pfn(pfn, order); 996 buddy = page + (buddy_pfn - pfn); 997 998 if (!pfn_valid_within(buddy_pfn)) 999 goto done_merging; 1000 if (!page_is_buddy(page, buddy, order)) 1001 goto done_merging; 1002 /* 1003 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 1004 * merge with it and move up one order. 1005 */ 1006 if (page_is_guard(buddy)) 1007 clear_page_guard(zone, buddy, order, migratetype); 1008 else 1009 del_page_from_free_list(buddy, zone, order); 1010 combined_pfn = buddy_pfn & pfn; 1011 page = page + (combined_pfn - pfn); 1012 pfn = combined_pfn; 1013 order++; 1014 } 1015 if (max_order < MAX_ORDER) { 1016 /* If we are here, it means order is >= pageblock_order. 1017 * We want to prevent merge between freepages on isolate 1018 * pageblock and normal pageblock. Without this, pageblock 1019 * isolation could cause incorrect freepage or CMA accounting. 1020 * 1021 * We don't want to hit this code for the more frequent 1022 * low-order merging. 1023 */ 1024 if (unlikely(has_isolate_pageblock(zone))) { 1025 int buddy_mt; 1026 1027 buddy_pfn = __find_buddy_pfn(pfn, order); 1028 buddy = page + (buddy_pfn - pfn); 1029 buddy_mt = get_pageblock_migratetype(buddy); 1030 1031 if (migratetype != buddy_mt 1032 && (is_migrate_isolate(migratetype) || 1033 is_migrate_isolate(buddy_mt))) 1034 goto done_merging; 1035 } 1036 max_order++; 1037 goto continue_merging; 1038 } 1039 1040 done_merging: 1041 set_page_order(page, order); 1042 1043 if (is_shuffle_order(order)) 1044 to_tail = shuffle_pick_tail(); 1045 else 1046 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); 1047 1048 if (to_tail) 1049 add_to_free_list_tail(page, zone, order, migratetype); 1050 else 1051 add_to_free_list(page, zone, order, migratetype); 1052 1053 /* Notify page reporting subsystem of freed page */ 1054 if (report) 1055 page_reporting_notify_free(order); 1056 } 1057 1058 /* 1059 * A bad page could be due to a number of fields. Instead of multiple branches, 1060 * try and check multiple fields with one check. The caller must do a detailed 1061 * check if necessary. 1062 */ 1063 static inline bool page_expected_state(struct page *page, 1064 unsigned long check_flags) 1065 { 1066 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1067 return false; 1068 1069 if (unlikely((unsigned long)page->mapping | 1070 page_ref_count(page) | 1071 #ifdef CONFIG_MEMCG 1072 (unsigned long)page->mem_cgroup | 1073 #endif 1074 (page->flags & check_flags))) 1075 return false; 1076 1077 return true; 1078 } 1079 1080 static void free_pages_check_bad(struct page *page) 1081 { 1082 const char *bad_reason; 1083 unsigned long bad_flags; 1084 1085 bad_reason = NULL; 1086 bad_flags = 0; 1087 1088 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1089 bad_reason = "nonzero mapcount"; 1090 if (unlikely(page->mapping != NULL)) 1091 bad_reason = "non-NULL mapping"; 1092 if (unlikely(page_ref_count(page) != 0)) 1093 bad_reason = "nonzero _refcount"; 1094 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) { 1095 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 1096 bad_flags = PAGE_FLAGS_CHECK_AT_FREE; 1097 } 1098 #ifdef CONFIG_MEMCG 1099 if (unlikely(page->mem_cgroup)) 1100 bad_reason = "page still charged to cgroup"; 1101 #endif 1102 bad_page(page, bad_reason, bad_flags); 1103 } 1104 1105 static inline int free_pages_check(struct page *page) 1106 { 1107 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 1108 return 0; 1109 1110 /* Something has gone sideways, find it */ 1111 free_pages_check_bad(page); 1112 return 1; 1113 } 1114 1115 static int free_tail_pages_check(struct page *head_page, struct page *page) 1116 { 1117 int ret = 1; 1118 1119 /* 1120 * We rely page->lru.next never has bit 0 set, unless the page 1121 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 1122 */ 1123 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 1124 1125 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 1126 ret = 0; 1127 goto out; 1128 } 1129 switch (page - head_page) { 1130 case 1: 1131 /* the first tail page: ->mapping may be compound_mapcount() */ 1132 if (unlikely(compound_mapcount(page))) { 1133 bad_page(page, "nonzero compound_mapcount", 0); 1134 goto out; 1135 } 1136 break; 1137 case 2: 1138 /* 1139 * the second tail page: ->mapping is 1140 * deferred_list.next -- ignore value. 1141 */ 1142 break; 1143 default: 1144 if (page->mapping != TAIL_MAPPING) { 1145 bad_page(page, "corrupted mapping in tail page", 0); 1146 goto out; 1147 } 1148 break; 1149 } 1150 if (unlikely(!PageTail(page))) { 1151 bad_page(page, "PageTail not set", 0); 1152 goto out; 1153 } 1154 if (unlikely(compound_head(page) != head_page)) { 1155 bad_page(page, "compound_head not consistent", 0); 1156 goto out; 1157 } 1158 ret = 0; 1159 out: 1160 page->mapping = NULL; 1161 clear_compound_head(page); 1162 return ret; 1163 } 1164 1165 static void kernel_init_free_pages(struct page *page, int numpages) 1166 { 1167 int i; 1168 1169 for (i = 0; i < numpages; i++) 1170 clear_highpage(page + i); 1171 } 1172 1173 static __always_inline bool free_pages_prepare(struct page *page, 1174 unsigned int order, bool check_free) 1175 { 1176 int bad = 0; 1177 1178 VM_BUG_ON_PAGE(PageTail(page), page); 1179 1180 trace_mm_page_free(page, order); 1181 1182 /* 1183 * Check tail pages before head page information is cleared to 1184 * avoid checking PageCompound for order-0 pages. 1185 */ 1186 if (unlikely(order)) { 1187 bool compound = PageCompound(page); 1188 int i; 1189 1190 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1191 1192 if (compound) 1193 ClearPageDoubleMap(page); 1194 for (i = 1; i < (1 << order); i++) { 1195 if (compound) 1196 bad += free_tail_pages_check(page, page + i); 1197 if (unlikely(free_pages_check(page + i))) { 1198 bad++; 1199 continue; 1200 } 1201 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1202 } 1203 } 1204 if (PageMappingFlags(page)) 1205 page->mapping = NULL; 1206 if (memcg_kmem_enabled() && PageKmemcg(page)) 1207 __memcg_kmem_uncharge_page(page, order); 1208 if (check_free) 1209 bad += free_pages_check(page); 1210 if (bad) 1211 return false; 1212 1213 page_cpupid_reset_last(page); 1214 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1215 reset_page_owner(page, order); 1216 1217 if (!PageHighMem(page)) { 1218 debug_check_no_locks_freed(page_address(page), 1219 PAGE_SIZE << order); 1220 debug_check_no_obj_freed(page_address(page), 1221 PAGE_SIZE << order); 1222 } 1223 if (want_init_on_free()) 1224 kernel_init_free_pages(page, 1 << order); 1225 1226 kernel_poison_pages(page, 1 << order, 0); 1227 /* 1228 * arch_free_page() can make the page's contents inaccessible. s390 1229 * does this. So nothing which can access the page's contents should 1230 * happen after this. 1231 */ 1232 arch_free_page(page, order); 1233 1234 if (debug_pagealloc_enabled_static()) 1235 kernel_map_pages(page, 1 << order, 0); 1236 1237 kasan_free_nondeferred_pages(page, order); 1238 1239 return true; 1240 } 1241 1242 #ifdef CONFIG_DEBUG_VM 1243 /* 1244 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed 1245 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when 1246 * moved from pcp lists to free lists. 1247 */ 1248 static bool free_pcp_prepare(struct page *page) 1249 { 1250 return free_pages_prepare(page, 0, true); 1251 } 1252 1253 static bool bulkfree_pcp_prepare(struct page *page) 1254 { 1255 if (debug_pagealloc_enabled_static()) 1256 return free_pages_check(page); 1257 else 1258 return false; 1259 } 1260 #else 1261 /* 1262 * With DEBUG_VM disabled, order-0 pages being freed are checked only when 1263 * moving from pcp lists to free list in order to reduce overhead. With 1264 * debug_pagealloc enabled, they are checked also immediately when being freed 1265 * to the pcp lists. 1266 */ 1267 static bool free_pcp_prepare(struct page *page) 1268 { 1269 if (debug_pagealloc_enabled_static()) 1270 return free_pages_prepare(page, 0, true); 1271 else 1272 return free_pages_prepare(page, 0, false); 1273 } 1274 1275 static bool bulkfree_pcp_prepare(struct page *page) 1276 { 1277 return free_pages_check(page); 1278 } 1279 #endif /* CONFIG_DEBUG_VM */ 1280 1281 static inline void prefetch_buddy(struct page *page) 1282 { 1283 unsigned long pfn = page_to_pfn(page); 1284 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0); 1285 struct page *buddy = page + (buddy_pfn - pfn); 1286 1287 prefetch(buddy); 1288 } 1289 1290 /* 1291 * Frees a number of pages from the PCP lists 1292 * Assumes all pages on list are in same zone, and of same order. 1293 * count is the number of pages to free. 1294 * 1295 * If the zone was previously in an "all pages pinned" state then look to 1296 * see if this freeing clears that state. 1297 * 1298 * And clear the zone's pages_scanned counter, to hold off the "all pages are 1299 * pinned" detection logic. 1300 */ 1301 static void free_pcppages_bulk(struct zone *zone, int count, 1302 struct per_cpu_pages *pcp) 1303 { 1304 int migratetype = 0; 1305 int batch_free = 0; 1306 int prefetch_nr = 0; 1307 bool isolated_pageblocks; 1308 struct page *page, *tmp; 1309 LIST_HEAD(head); 1310 1311 while (count) { 1312 struct list_head *list; 1313 1314 /* 1315 * Remove pages from lists in a round-robin fashion. A 1316 * batch_free count is maintained that is incremented when an 1317 * empty list is encountered. This is so more pages are freed 1318 * off fuller lists instead of spinning excessively around empty 1319 * lists 1320 */ 1321 do { 1322 batch_free++; 1323 if (++migratetype == MIGRATE_PCPTYPES) 1324 migratetype = 0; 1325 list = &pcp->lists[migratetype]; 1326 } while (list_empty(list)); 1327 1328 /* This is the only non-empty list. Free them all. */ 1329 if (batch_free == MIGRATE_PCPTYPES) 1330 batch_free = count; 1331 1332 do { 1333 page = list_last_entry(list, struct page, lru); 1334 /* must delete to avoid corrupting pcp list */ 1335 list_del(&page->lru); 1336 pcp->count--; 1337 1338 if (bulkfree_pcp_prepare(page)) 1339 continue; 1340 1341 list_add_tail(&page->lru, &head); 1342 1343 /* 1344 * We are going to put the page back to the global 1345 * pool, prefetch its buddy to speed up later access 1346 * under zone->lock. It is believed the overhead of 1347 * an additional test and calculating buddy_pfn here 1348 * can be offset by reduced memory latency later. To 1349 * avoid excessive prefetching due to large count, only 1350 * prefetch buddy for the first pcp->batch nr of pages. 1351 */ 1352 if (prefetch_nr++ < pcp->batch) 1353 prefetch_buddy(page); 1354 } while (--count && --batch_free && !list_empty(list)); 1355 } 1356 1357 spin_lock(&zone->lock); 1358 isolated_pageblocks = has_isolate_pageblock(zone); 1359 1360 /* 1361 * Use safe version since after __free_one_page(), 1362 * page->lru.next will not point to original list. 1363 */ 1364 list_for_each_entry_safe(page, tmp, &head, lru) { 1365 int mt = get_pcppage_migratetype(page); 1366 /* MIGRATE_ISOLATE page should not go to pcplists */ 1367 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1368 /* Pageblock could have been isolated meanwhile */ 1369 if (unlikely(isolated_pageblocks)) 1370 mt = get_pageblock_migratetype(page); 1371 1372 __free_one_page(page, page_to_pfn(page), zone, 0, mt, true); 1373 trace_mm_page_pcpu_drain(page, 0, mt); 1374 } 1375 spin_unlock(&zone->lock); 1376 } 1377 1378 static void free_one_page(struct zone *zone, 1379 struct page *page, unsigned long pfn, 1380 unsigned int order, 1381 int migratetype) 1382 { 1383 spin_lock(&zone->lock); 1384 if (unlikely(has_isolate_pageblock(zone) || 1385 is_migrate_isolate(migratetype))) { 1386 migratetype = get_pfnblock_migratetype(page, pfn); 1387 } 1388 __free_one_page(page, pfn, zone, order, migratetype, true); 1389 spin_unlock(&zone->lock); 1390 } 1391 1392 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1393 unsigned long zone, int nid) 1394 { 1395 mm_zero_struct_page(page); 1396 set_page_links(page, zone, nid, pfn); 1397 init_page_count(page); 1398 page_mapcount_reset(page); 1399 page_cpupid_reset_last(page); 1400 page_kasan_tag_reset(page); 1401 1402 INIT_LIST_HEAD(&page->lru); 1403 #ifdef WANT_PAGE_VIRTUAL 1404 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1405 if (!is_highmem_idx(zone)) 1406 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1407 #endif 1408 } 1409 1410 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1411 static void __meminit init_reserved_page(unsigned long pfn) 1412 { 1413 pg_data_t *pgdat; 1414 int nid, zid; 1415 1416 if (!early_page_uninitialised(pfn)) 1417 return; 1418 1419 nid = early_pfn_to_nid(pfn); 1420 pgdat = NODE_DATA(nid); 1421 1422 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1423 struct zone *zone = &pgdat->node_zones[zid]; 1424 1425 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) 1426 break; 1427 } 1428 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 1429 } 1430 #else 1431 static inline void init_reserved_page(unsigned long pfn) 1432 { 1433 } 1434 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1435 1436 /* 1437 * Initialised pages do not have PageReserved set. This function is 1438 * called for each range allocated by the bootmem allocator and 1439 * marks the pages PageReserved. The remaining valid pages are later 1440 * sent to the buddy page allocator. 1441 */ 1442 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1443 { 1444 unsigned long start_pfn = PFN_DOWN(start); 1445 unsigned long end_pfn = PFN_UP(end); 1446 1447 for (; start_pfn < end_pfn; start_pfn++) { 1448 if (pfn_valid(start_pfn)) { 1449 struct page *page = pfn_to_page(start_pfn); 1450 1451 init_reserved_page(start_pfn); 1452 1453 /* Avoid false-positive PageTail() */ 1454 INIT_LIST_HEAD(&page->lru); 1455 1456 /* 1457 * no need for atomic set_bit because the struct 1458 * page is not visible yet so nobody should 1459 * access it yet. 1460 */ 1461 __SetPageReserved(page); 1462 } 1463 } 1464 } 1465 1466 static void __free_pages_ok(struct page *page, unsigned int order) 1467 { 1468 unsigned long flags; 1469 int migratetype; 1470 unsigned long pfn = page_to_pfn(page); 1471 1472 if (!free_pages_prepare(page, order, true)) 1473 return; 1474 1475 migratetype = get_pfnblock_migratetype(page, pfn); 1476 local_irq_save(flags); 1477 __count_vm_events(PGFREE, 1 << order); 1478 free_one_page(page_zone(page), page, pfn, order, migratetype); 1479 local_irq_restore(flags); 1480 } 1481 1482 void __free_pages_core(struct page *page, unsigned int order) 1483 { 1484 unsigned int nr_pages = 1 << order; 1485 struct page *p = page; 1486 unsigned int loop; 1487 1488 prefetchw(p); 1489 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1490 prefetchw(p + 1); 1491 __ClearPageReserved(p); 1492 set_page_count(p, 0); 1493 } 1494 __ClearPageReserved(p); 1495 set_page_count(p, 0); 1496 1497 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); 1498 set_page_refcounted(page); 1499 __free_pages(page, order); 1500 } 1501 1502 #if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \ 1503 defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) 1504 1505 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1506 1507 int __meminit early_pfn_to_nid(unsigned long pfn) 1508 { 1509 static DEFINE_SPINLOCK(early_pfn_lock); 1510 int nid; 1511 1512 spin_lock(&early_pfn_lock); 1513 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1514 if (nid < 0) 1515 nid = first_online_node; 1516 spin_unlock(&early_pfn_lock); 1517 1518 return nid; 1519 } 1520 #endif 1521 1522 #ifdef CONFIG_NODES_SPAN_OTHER_NODES 1523 /* Only safe to use early in boot when initialisation is single-threaded */ 1524 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1525 { 1526 int nid; 1527 1528 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1529 if (nid >= 0 && nid != node) 1530 return false; 1531 return true; 1532 } 1533 1534 #else 1535 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1536 { 1537 return true; 1538 } 1539 #endif 1540 1541 1542 void __init memblock_free_pages(struct page *page, unsigned long pfn, 1543 unsigned int order) 1544 { 1545 if (early_page_uninitialised(pfn)) 1546 return; 1547 __free_pages_core(page, order); 1548 } 1549 1550 /* 1551 * Check that the whole (or subset of) a pageblock given by the interval of 1552 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1553 * with the migration of free compaction scanner. The scanners then need to 1554 * use only pfn_valid_within() check for arches that allow holes within 1555 * pageblocks. 1556 * 1557 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1558 * 1559 * It's possible on some configurations to have a setup like node0 node1 node0 1560 * i.e. it's possible that all pages within a zones range of pages do not 1561 * belong to a single zone. We assume that a border between node0 and node1 1562 * can occur within a single pageblock, but not a node0 node1 node0 1563 * interleaving within a single pageblock. It is therefore sufficient to check 1564 * the first and last page of a pageblock and avoid checking each individual 1565 * page in a pageblock. 1566 */ 1567 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1568 unsigned long end_pfn, struct zone *zone) 1569 { 1570 struct page *start_page; 1571 struct page *end_page; 1572 1573 /* end_pfn is one past the range we are checking */ 1574 end_pfn--; 1575 1576 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1577 return NULL; 1578 1579 start_page = pfn_to_online_page(start_pfn); 1580 if (!start_page) 1581 return NULL; 1582 1583 if (page_zone(start_page) != zone) 1584 return NULL; 1585 1586 end_page = pfn_to_page(end_pfn); 1587 1588 /* This gives a shorter code than deriving page_zone(end_page) */ 1589 if (page_zone_id(start_page) != page_zone_id(end_page)) 1590 return NULL; 1591 1592 return start_page; 1593 } 1594 1595 void set_zone_contiguous(struct zone *zone) 1596 { 1597 unsigned long block_start_pfn = zone->zone_start_pfn; 1598 unsigned long block_end_pfn; 1599 1600 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1601 for (; block_start_pfn < zone_end_pfn(zone); 1602 block_start_pfn = block_end_pfn, 1603 block_end_pfn += pageblock_nr_pages) { 1604 1605 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1606 1607 if (!__pageblock_pfn_to_page(block_start_pfn, 1608 block_end_pfn, zone)) 1609 return; 1610 } 1611 1612 /* We confirm that there is no hole */ 1613 zone->contiguous = true; 1614 } 1615 1616 void clear_zone_contiguous(struct zone *zone) 1617 { 1618 zone->contiguous = false; 1619 } 1620 1621 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1622 static void __init deferred_free_range(unsigned long pfn, 1623 unsigned long nr_pages) 1624 { 1625 struct page *page; 1626 unsigned long i; 1627 1628 if (!nr_pages) 1629 return; 1630 1631 page = pfn_to_page(pfn); 1632 1633 /* Free a large naturally-aligned chunk if possible */ 1634 if (nr_pages == pageblock_nr_pages && 1635 (pfn & (pageblock_nr_pages - 1)) == 0) { 1636 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1637 __free_pages_core(page, pageblock_order); 1638 return; 1639 } 1640 1641 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1642 if ((pfn & (pageblock_nr_pages - 1)) == 0) 1643 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1644 __free_pages_core(page, 0); 1645 } 1646 } 1647 1648 /* Completion tracking for deferred_init_memmap() threads */ 1649 static atomic_t pgdat_init_n_undone __initdata; 1650 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1651 1652 static inline void __init pgdat_init_report_one_done(void) 1653 { 1654 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1655 complete(&pgdat_init_all_done_comp); 1656 } 1657 1658 /* 1659 * Returns true if page needs to be initialized or freed to buddy allocator. 1660 * 1661 * First we check if pfn is valid on architectures where it is possible to have 1662 * holes within pageblock_nr_pages. On systems where it is not possible, this 1663 * function is optimized out. 1664 * 1665 * Then, we check if a current large page is valid by only checking the validity 1666 * of the head pfn. 1667 */ 1668 static inline bool __init deferred_pfn_valid(unsigned long pfn) 1669 { 1670 if (!pfn_valid_within(pfn)) 1671 return false; 1672 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn)) 1673 return false; 1674 return true; 1675 } 1676 1677 /* 1678 * Free pages to buddy allocator. Try to free aligned pages in 1679 * pageblock_nr_pages sizes. 1680 */ 1681 static void __init deferred_free_pages(unsigned long pfn, 1682 unsigned long end_pfn) 1683 { 1684 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1685 unsigned long nr_free = 0; 1686 1687 for (; pfn < end_pfn; pfn++) { 1688 if (!deferred_pfn_valid(pfn)) { 1689 deferred_free_range(pfn - nr_free, nr_free); 1690 nr_free = 0; 1691 } else if (!(pfn & nr_pgmask)) { 1692 deferred_free_range(pfn - nr_free, nr_free); 1693 nr_free = 1; 1694 touch_nmi_watchdog(); 1695 } else { 1696 nr_free++; 1697 } 1698 } 1699 /* Free the last block of pages to allocator */ 1700 deferred_free_range(pfn - nr_free, nr_free); 1701 } 1702 1703 /* 1704 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 1705 * by performing it only once every pageblock_nr_pages. 1706 * Return number of pages initialized. 1707 */ 1708 static unsigned long __init deferred_init_pages(struct zone *zone, 1709 unsigned long pfn, 1710 unsigned long end_pfn) 1711 { 1712 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1713 int nid = zone_to_nid(zone); 1714 unsigned long nr_pages = 0; 1715 int zid = zone_idx(zone); 1716 struct page *page = NULL; 1717 1718 for (; pfn < end_pfn; pfn++) { 1719 if (!deferred_pfn_valid(pfn)) { 1720 page = NULL; 1721 continue; 1722 } else if (!page || !(pfn & nr_pgmask)) { 1723 page = pfn_to_page(pfn); 1724 touch_nmi_watchdog(); 1725 } else { 1726 page++; 1727 } 1728 __init_single_page(page, pfn, zid, nid); 1729 nr_pages++; 1730 } 1731 return (nr_pages); 1732 } 1733 1734 /* 1735 * This function is meant to pre-load the iterator for the zone init. 1736 * Specifically it walks through the ranges until we are caught up to the 1737 * first_init_pfn value and exits there. If we never encounter the value we 1738 * return false indicating there are no valid ranges left. 1739 */ 1740 static bool __init 1741 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, 1742 unsigned long *spfn, unsigned long *epfn, 1743 unsigned long first_init_pfn) 1744 { 1745 u64 j; 1746 1747 /* 1748 * Start out by walking through the ranges in this zone that have 1749 * already been initialized. We don't need to do anything with them 1750 * so we just need to flush them out of the system. 1751 */ 1752 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { 1753 if (*epfn <= first_init_pfn) 1754 continue; 1755 if (*spfn < first_init_pfn) 1756 *spfn = first_init_pfn; 1757 *i = j; 1758 return true; 1759 } 1760 1761 return false; 1762 } 1763 1764 /* 1765 * Initialize and free pages. We do it in two loops: first we initialize 1766 * struct page, then free to buddy allocator, because while we are 1767 * freeing pages we can access pages that are ahead (computing buddy 1768 * page in __free_one_page()). 1769 * 1770 * In order to try and keep some memory in the cache we have the loop 1771 * broken along max page order boundaries. This way we will not cause 1772 * any issues with the buddy page computation. 1773 */ 1774 static unsigned long __init 1775 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, 1776 unsigned long *end_pfn) 1777 { 1778 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); 1779 unsigned long spfn = *start_pfn, epfn = *end_pfn; 1780 unsigned long nr_pages = 0; 1781 u64 j = *i; 1782 1783 /* First we loop through and initialize the page values */ 1784 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { 1785 unsigned long t; 1786 1787 if (mo_pfn <= *start_pfn) 1788 break; 1789 1790 t = min(mo_pfn, *end_pfn); 1791 nr_pages += deferred_init_pages(zone, *start_pfn, t); 1792 1793 if (mo_pfn < *end_pfn) { 1794 *start_pfn = mo_pfn; 1795 break; 1796 } 1797 } 1798 1799 /* Reset values and now loop through freeing pages as needed */ 1800 swap(j, *i); 1801 1802 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { 1803 unsigned long t; 1804 1805 if (mo_pfn <= spfn) 1806 break; 1807 1808 t = min(mo_pfn, epfn); 1809 deferred_free_pages(spfn, t); 1810 1811 if (mo_pfn <= epfn) 1812 break; 1813 } 1814 1815 return nr_pages; 1816 } 1817 1818 /* Initialise remaining memory on a node */ 1819 static int __init deferred_init_memmap(void *data) 1820 { 1821 pg_data_t *pgdat = data; 1822 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 1823 unsigned long spfn = 0, epfn = 0, nr_pages = 0; 1824 unsigned long first_init_pfn, flags; 1825 unsigned long start = jiffies; 1826 struct zone *zone; 1827 int zid; 1828 u64 i; 1829 1830 /* Bind memory initialisation thread to a local node if possible */ 1831 if (!cpumask_empty(cpumask)) 1832 set_cpus_allowed_ptr(current, cpumask); 1833 1834 pgdat_resize_lock(pgdat, &flags); 1835 first_init_pfn = pgdat->first_deferred_pfn; 1836 if (first_init_pfn == ULONG_MAX) { 1837 pgdat_resize_unlock(pgdat, &flags); 1838 pgdat_init_report_one_done(); 1839 return 0; 1840 } 1841 1842 /* Sanity check boundaries */ 1843 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 1844 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 1845 pgdat->first_deferred_pfn = ULONG_MAX; 1846 1847 /* Only the highest zone is deferred so find it */ 1848 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1849 zone = pgdat->node_zones + zid; 1850 if (first_init_pfn < zone_end_pfn(zone)) 1851 break; 1852 } 1853 1854 /* If the zone is empty somebody else may have cleared out the zone */ 1855 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 1856 first_init_pfn)) 1857 goto zone_empty; 1858 1859 /* 1860 * Initialize and free pages in MAX_ORDER sized increments so 1861 * that we can avoid introducing any issues with the buddy 1862 * allocator. 1863 */ 1864 while (spfn < epfn) 1865 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 1866 zone_empty: 1867 pgdat_resize_unlock(pgdat, &flags); 1868 1869 /* Sanity check that the next zone really is unpopulated */ 1870 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 1871 1872 pr_info("node %d initialised, %lu pages in %ums\n", 1873 pgdat->node_id, nr_pages, jiffies_to_msecs(jiffies - start)); 1874 1875 pgdat_init_report_one_done(); 1876 return 0; 1877 } 1878 1879 /* 1880 * If this zone has deferred pages, try to grow it by initializing enough 1881 * deferred pages to satisfy the allocation specified by order, rounded up to 1882 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 1883 * of SECTION_SIZE bytes by initializing struct pages in increments of 1884 * PAGES_PER_SECTION * sizeof(struct page) bytes. 1885 * 1886 * Return true when zone was grown, otherwise return false. We return true even 1887 * when we grow less than requested, to let the caller decide if there are 1888 * enough pages to satisfy the allocation. 1889 * 1890 * Note: We use noinline because this function is needed only during boot, and 1891 * it is called from a __ref function _deferred_grow_zone. This way we are 1892 * making sure that it is not inlined into permanent text section. 1893 */ 1894 static noinline bool __init 1895 deferred_grow_zone(struct zone *zone, unsigned int order) 1896 { 1897 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 1898 pg_data_t *pgdat = zone->zone_pgdat; 1899 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 1900 unsigned long spfn, epfn, flags; 1901 unsigned long nr_pages = 0; 1902 u64 i; 1903 1904 /* Only the last zone may have deferred pages */ 1905 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 1906 return false; 1907 1908 pgdat_resize_lock(pgdat, &flags); 1909 1910 /* 1911 * If deferred pages have been initialized while we were waiting for 1912 * the lock, return true, as the zone was grown. The caller will retry 1913 * this zone. We won't return to this function since the caller also 1914 * has this static branch. 1915 */ 1916 if (!static_branch_unlikely(&deferred_pages)) { 1917 pgdat_resize_unlock(pgdat, &flags); 1918 return true; 1919 } 1920 1921 /* 1922 * If someone grew this zone while we were waiting for spinlock, return 1923 * true, as there might be enough pages already. 1924 */ 1925 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 1926 pgdat_resize_unlock(pgdat, &flags); 1927 return true; 1928 } 1929 1930 /* If the zone is empty somebody else may have cleared out the zone */ 1931 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 1932 first_deferred_pfn)) { 1933 pgdat->first_deferred_pfn = ULONG_MAX; 1934 pgdat_resize_unlock(pgdat, &flags); 1935 /* Retry only once. */ 1936 return first_deferred_pfn != ULONG_MAX; 1937 } 1938 1939 /* 1940 * Initialize and free pages in MAX_ORDER sized increments so 1941 * that we can avoid introducing any issues with the buddy 1942 * allocator. 1943 */ 1944 while (spfn < epfn) { 1945 /* update our first deferred PFN for this section */ 1946 first_deferred_pfn = spfn; 1947 1948 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 1949 1950 /* We should only stop along section boundaries */ 1951 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) 1952 continue; 1953 1954 /* If our quota has been met we can stop here */ 1955 if (nr_pages >= nr_pages_needed) 1956 break; 1957 } 1958 1959 pgdat->first_deferred_pfn = spfn; 1960 pgdat_resize_unlock(pgdat, &flags); 1961 1962 return nr_pages > 0; 1963 } 1964 1965 /* 1966 * deferred_grow_zone() is __init, but it is called from 1967 * get_page_from_freelist() during early boot until deferred_pages permanently 1968 * disables this call. This is why we have refdata wrapper to avoid warning, 1969 * and to ensure that the function body gets unloaded. 1970 */ 1971 static bool __ref 1972 _deferred_grow_zone(struct zone *zone, unsigned int order) 1973 { 1974 return deferred_grow_zone(zone, order); 1975 } 1976 1977 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1978 1979 void __init page_alloc_init_late(void) 1980 { 1981 struct zone *zone; 1982 int nid; 1983 1984 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1985 1986 /* There will be num_node_state(N_MEMORY) threads */ 1987 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 1988 for_each_node_state(nid, N_MEMORY) { 1989 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 1990 } 1991 1992 /* Block until all are initialised */ 1993 wait_for_completion(&pgdat_init_all_done_comp); 1994 1995 /* 1996 * The number of managed pages has changed due to the initialisation 1997 * so the pcpu batch and high limits needs to be updated or the limits 1998 * will be artificially small. 1999 */ 2000 for_each_populated_zone(zone) 2001 zone_pcp_update(zone); 2002 2003 /* 2004 * We initialized the rest of the deferred pages. Permanently disable 2005 * on-demand struct page initialization. 2006 */ 2007 static_branch_disable(&deferred_pages); 2008 2009 /* Reinit limits that are based on free pages after the kernel is up */ 2010 files_maxfiles_init(); 2011 #endif 2012 2013 /* Discard memblock private memory */ 2014 memblock_discard(); 2015 2016 for_each_node_state(nid, N_MEMORY) 2017 shuffle_free_memory(NODE_DATA(nid)); 2018 2019 for_each_populated_zone(zone) 2020 set_zone_contiguous(zone); 2021 } 2022 2023 #ifdef CONFIG_CMA 2024 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 2025 void __init init_cma_reserved_pageblock(struct page *page) 2026 { 2027 unsigned i = pageblock_nr_pages; 2028 struct page *p = page; 2029 2030 do { 2031 __ClearPageReserved(p); 2032 set_page_count(p, 0); 2033 } while (++p, --i); 2034 2035 set_pageblock_migratetype(page, MIGRATE_CMA); 2036 2037 if (pageblock_order >= MAX_ORDER) { 2038 i = pageblock_nr_pages; 2039 p = page; 2040 do { 2041 set_page_refcounted(p); 2042 __free_pages(p, MAX_ORDER - 1); 2043 p += MAX_ORDER_NR_PAGES; 2044 } while (i -= MAX_ORDER_NR_PAGES); 2045 } else { 2046 set_page_refcounted(page); 2047 __free_pages(page, pageblock_order); 2048 } 2049 2050 adjust_managed_page_count(page, pageblock_nr_pages); 2051 } 2052 #endif 2053 2054 /* 2055 * The order of subdivision here is critical for the IO subsystem. 2056 * Please do not alter this order without good reasons and regression 2057 * testing. Specifically, as large blocks of memory are subdivided, 2058 * the order in which smaller blocks are delivered depends on the order 2059 * they're subdivided in this function. This is the primary factor 2060 * influencing the order in which pages are delivered to the IO 2061 * subsystem according to empirical testing, and this is also justified 2062 * by considering the behavior of a buddy system containing a single 2063 * large block of memory acted on by a series of small allocations. 2064 * This behavior is a critical factor in sglist merging's success. 2065 * 2066 * -- nyc 2067 */ 2068 static inline void expand(struct zone *zone, struct page *page, 2069 int low, int high, int migratetype) 2070 { 2071 unsigned long size = 1 << high; 2072 2073 while (high > low) { 2074 high--; 2075 size >>= 1; 2076 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 2077 2078 /* 2079 * Mark as guard pages (or page), that will allow to 2080 * merge back to allocator when buddy will be freed. 2081 * Corresponding page table entries will not be touched, 2082 * pages will stay not present in virtual address space 2083 */ 2084 if (set_page_guard(zone, &page[size], high, migratetype)) 2085 continue; 2086 2087 add_to_free_list(&page[size], zone, high, migratetype); 2088 set_page_order(&page[size], high); 2089 } 2090 } 2091 2092 static void check_new_page_bad(struct page *page) 2093 { 2094 const char *bad_reason = NULL; 2095 unsigned long bad_flags = 0; 2096 2097 if (unlikely(atomic_read(&page->_mapcount) != -1)) 2098 bad_reason = "nonzero mapcount"; 2099 if (unlikely(page->mapping != NULL)) 2100 bad_reason = "non-NULL mapping"; 2101 if (unlikely(page_ref_count(page) != 0)) 2102 bad_reason = "nonzero _refcount"; 2103 if (unlikely(page->flags & __PG_HWPOISON)) { 2104 bad_reason = "HWPoisoned (hardware-corrupted)"; 2105 bad_flags = __PG_HWPOISON; 2106 /* Don't complain about hwpoisoned pages */ 2107 page_mapcount_reset(page); /* remove PageBuddy */ 2108 return; 2109 } 2110 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) { 2111 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set"; 2112 bad_flags = PAGE_FLAGS_CHECK_AT_PREP; 2113 } 2114 #ifdef CONFIG_MEMCG 2115 if (unlikely(page->mem_cgroup)) 2116 bad_reason = "page still charged to cgroup"; 2117 #endif 2118 bad_page(page, bad_reason, bad_flags); 2119 } 2120 2121 /* 2122 * This page is about to be returned from the page allocator 2123 */ 2124 static inline int check_new_page(struct page *page) 2125 { 2126 if (likely(page_expected_state(page, 2127 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 2128 return 0; 2129 2130 check_new_page_bad(page); 2131 return 1; 2132 } 2133 2134 static inline bool free_pages_prezeroed(void) 2135 { 2136 return (IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) && 2137 page_poisoning_enabled()) || want_init_on_free(); 2138 } 2139 2140 #ifdef CONFIG_DEBUG_VM 2141 /* 2142 * With DEBUG_VM enabled, order-0 pages are checked for expected state when 2143 * being allocated from pcp lists. With debug_pagealloc also enabled, they are 2144 * also checked when pcp lists are refilled from the free lists. 2145 */ 2146 static inline bool check_pcp_refill(struct page *page) 2147 { 2148 if (debug_pagealloc_enabled_static()) 2149 return check_new_page(page); 2150 else 2151 return false; 2152 } 2153 2154 static inline bool check_new_pcp(struct page *page) 2155 { 2156 return check_new_page(page); 2157 } 2158 #else 2159 /* 2160 * With DEBUG_VM disabled, free order-0 pages are checked for expected state 2161 * when pcp lists are being refilled from the free lists. With debug_pagealloc 2162 * enabled, they are also checked when being allocated from the pcp lists. 2163 */ 2164 static inline bool check_pcp_refill(struct page *page) 2165 { 2166 return check_new_page(page); 2167 } 2168 static inline bool check_new_pcp(struct page *page) 2169 { 2170 if (debug_pagealloc_enabled_static()) 2171 return check_new_page(page); 2172 else 2173 return false; 2174 } 2175 #endif /* CONFIG_DEBUG_VM */ 2176 2177 static bool check_new_pages(struct page *page, unsigned int order) 2178 { 2179 int i; 2180 for (i = 0; i < (1 << order); i++) { 2181 struct page *p = page + i; 2182 2183 if (unlikely(check_new_page(p))) 2184 return true; 2185 } 2186 2187 return false; 2188 } 2189 2190 inline void post_alloc_hook(struct page *page, unsigned int order, 2191 gfp_t gfp_flags) 2192 { 2193 set_page_private(page, 0); 2194 set_page_refcounted(page); 2195 2196 arch_alloc_page(page, order); 2197 if (debug_pagealloc_enabled_static()) 2198 kernel_map_pages(page, 1 << order, 1); 2199 kasan_alloc_pages(page, order); 2200 kernel_poison_pages(page, 1 << order, 1); 2201 set_page_owner(page, order, gfp_flags); 2202 } 2203 2204 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 2205 unsigned int alloc_flags) 2206 { 2207 post_alloc_hook(page, order, gfp_flags); 2208 2209 if (!free_pages_prezeroed() && want_init_on_alloc(gfp_flags)) 2210 kernel_init_free_pages(page, 1 << order); 2211 2212 if (order && (gfp_flags & __GFP_COMP)) 2213 prep_compound_page(page, order); 2214 2215 /* 2216 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 2217 * allocate the page. The expectation is that the caller is taking 2218 * steps that will free more memory. The caller should avoid the page 2219 * being used for !PFMEMALLOC purposes. 2220 */ 2221 if (alloc_flags & ALLOC_NO_WATERMARKS) 2222 set_page_pfmemalloc(page); 2223 else 2224 clear_page_pfmemalloc(page); 2225 } 2226 2227 /* 2228 * Go through the free lists for the given migratetype and remove 2229 * the smallest available page from the freelists 2230 */ 2231 static __always_inline 2232 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 2233 int migratetype) 2234 { 2235 unsigned int current_order; 2236 struct free_area *area; 2237 struct page *page; 2238 2239 /* Find a page of the appropriate size in the preferred list */ 2240 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 2241 area = &(zone->free_area[current_order]); 2242 page = get_page_from_free_area(area, migratetype); 2243 if (!page) 2244 continue; 2245 del_page_from_free_list(page, zone, current_order); 2246 expand(zone, page, order, current_order, migratetype); 2247 set_pcppage_migratetype(page, migratetype); 2248 return page; 2249 } 2250 2251 return NULL; 2252 } 2253 2254 2255 /* 2256 * This array describes the order lists are fallen back to when 2257 * the free lists for the desirable migrate type are depleted 2258 */ 2259 static int fallbacks[MIGRATE_TYPES][4] = { 2260 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2261 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 2262 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2263 #ifdef CONFIG_CMA 2264 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */ 2265 #endif 2266 #ifdef CONFIG_MEMORY_ISOLATION 2267 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */ 2268 #endif 2269 }; 2270 2271 #ifdef CONFIG_CMA 2272 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2273 unsigned int order) 2274 { 2275 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 2276 } 2277 #else 2278 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2279 unsigned int order) { return NULL; } 2280 #endif 2281 2282 /* 2283 * Move the free pages in a range to the free lists of the requested type. 2284 * Note that start_page and end_pages are not aligned on a pageblock 2285 * boundary. If alignment is required, use move_freepages_block() 2286 */ 2287 static int move_freepages(struct zone *zone, 2288 struct page *start_page, struct page *end_page, 2289 int migratetype, int *num_movable) 2290 { 2291 struct page *page; 2292 unsigned int order; 2293 int pages_moved = 0; 2294 2295 for (page = start_page; page <= end_page;) { 2296 if (!pfn_valid_within(page_to_pfn(page))) { 2297 page++; 2298 continue; 2299 } 2300 2301 if (!PageBuddy(page)) { 2302 /* 2303 * We assume that pages that could be isolated for 2304 * migration are movable. But we don't actually try 2305 * isolating, as that would be expensive. 2306 */ 2307 if (num_movable && 2308 (PageLRU(page) || __PageMovable(page))) 2309 (*num_movable)++; 2310 2311 page++; 2312 continue; 2313 } 2314 2315 /* Make sure we are not inadvertently changing nodes */ 2316 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2317 VM_BUG_ON_PAGE(page_zone(page) != zone, page); 2318 2319 order = page_order(page); 2320 move_to_free_list(page, zone, order, migratetype); 2321 page += 1 << order; 2322 pages_moved += 1 << order; 2323 } 2324 2325 return pages_moved; 2326 } 2327 2328 int move_freepages_block(struct zone *zone, struct page *page, 2329 int migratetype, int *num_movable) 2330 { 2331 unsigned long start_pfn, end_pfn; 2332 struct page *start_page, *end_page; 2333 2334 if (num_movable) 2335 *num_movable = 0; 2336 2337 start_pfn = page_to_pfn(page); 2338 start_pfn = start_pfn & ~(pageblock_nr_pages-1); 2339 start_page = pfn_to_page(start_pfn); 2340 end_page = start_page + pageblock_nr_pages - 1; 2341 end_pfn = start_pfn + pageblock_nr_pages - 1; 2342 2343 /* Do not cross zone boundaries */ 2344 if (!zone_spans_pfn(zone, start_pfn)) 2345 start_page = page; 2346 if (!zone_spans_pfn(zone, end_pfn)) 2347 return 0; 2348 2349 return move_freepages(zone, start_page, end_page, migratetype, 2350 num_movable); 2351 } 2352 2353 static void change_pageblock_range(struct page *pageblock_page, 2354 int start_order, int migratetype) 2355 { 2356 int nr_pageblocks = 1 << (start_order - pageblock_order); 2357 2358 while (nr_pageblocks--) { 2359 set_pageblock_migratetype(pageblock_page, migratetype); 2360 pageblock_page += pageblock_nr_pages; 2361 } 2362 } 2363 2364 /* 2365 * When we are falling back to another migratetype during allocation, try to 2366 * steal extra free pages from the same pageblocks to satisfy further 2367 * allocations, instead of polluting multiple pageblocks. 2368 * 2369 * If we are stealing a relatively large buddy page, it is likely there will 2370 * be more free pages in the pageblock, so try to steal them all. For 2371 * reclaimable and unmovable allocations, we steal regardless of page size, 2372 * as fragmentation caused by those allocations polluting movable pageblocks 2373 * is worse than movable allocations stealing from unmovable and reclaimable 2374 * pageblocks. 2375 */ 2376 static bool can_steal_fallback(unsigned int order, int start_mt) 2377 { 2378 /* 2379 * Leaving this order check is intended, although there is 2380 * relaxed order check in next check. The reason is that 2381 * we can actually steal whole pageblock if this condition met, 2382 * but, below check doesn't guarantee it and that is just heuristic 2383 * so could be changed anytime. 2384 */ 2385 if (order >= pageblock_order) 2386 return true; 2387 2388 if (order >= pageblock_order / 2 || 2389 start_mt == MIGRATE_RECLAIMABLE || 2390 start_mt == MIGRATE_UNMOVABLE || 2391 page_group_by_mobility_disabled) 2392 return true; 2393 2394 return false; 2395 } 2396 2397 static inline void boost_watermark(struct zone *zone) 2398 { 2399 unsigned long max_boost; 2400 2401 if (!watermark_boost_factor) 2402 return; 2403 2404 max_boost = mult_frac(zone->_watermark[WMARK_HIGH], 2405 watermark_boost_factor, 10000); 2406 2407 /* 2408 * high watermark may be uninitialised if fragmentation occurs 2409 * very early in boot so do not boost. We do not fall 2410 * through and boost by pageblock_nr_pages as failing 2411 * allocations that early means that reclaim is not going 2412 * to help and it may even be impossible to reclaim the 2413 * boosted watermark resulting in a hang. 2414 */ 2415 if (!max_boost) 2416 return; 2417 2418 max_boost = max(pageblock_nr_pages, max_boost); 2419 2420 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages, 2421 max_boost); 2422 } 2423 2424 /* 2425 * This function implements actual steal behaviour. If order is large enough, 2426 * we can steal whole pageblock. If not, we first move freepages in this 2427 * pageblock to our migratetype and determine how many already-allocated pages 2428 * are there in the pageblock with a compatible migratetype. If at least half 2429 * of pages are free or compatible, we can change migratetype of the pageblock 2430 * itself, so pages freed in the future will be put on the correct free list. 2431 */ 2432 static void steal_suitable_fallback(struct zone *zone, struct page *page, 2433 unsigned int alloc_flags, int start_type, bool whole_block) 2434 { 2435 unsigned int current_order = page_order(page); 2436 int free_pages, movable_pages, alike_pages; 2437 int old_block_type; 2438 2439 old_block_type = get_pageblock_migratetype(page); 2440 2441 /* 2442 * This can happen due to races and we want to prevent broken 2443 * highatomic accounting. 2444 */ 2445 if (is_migrate_highatomic(old_block_type)) 2446 goto single_page; 2447 2448 /* Take ownership for orders >= pageblock_order */ 2449 if (current_order >= pageblock_order) { 2450 change_pageblock_range(page, current_order, start_type); 2451 goto single_page; 2452 } 2453 2454 /* 2455 * Boost watermarks to increase reclaim pressure to reduce the 2456 * likelihood of future fallbacks. Wake kswapd now as the node 2457 * may be balanced overall and kswapd will not wake naturally. 2458 */ 2459 boost_watermark(zone); 2460 if (alloc_flags & ALLOC_KSWAPD) 2461 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 2462 2463 /* We are not allowed to try stealing from the whole block */ 2464 if (!whole_block) 2465 goto single_page; 2466 2467 free_pages = move_freepages_block(zone, page, start_type, 2468 &movable_pages); 2469 /* 2470 * Determine how many pages are compatible with our allocation. 2471 * For movable allocation, it's the number of movable pages which 2472 * we just obtained. For other types it's a bit more tricky. 2473 */ 2474 if (start_type == MIGRATE_MOVABLE) { 2475 alike_pages = movable_pages; 2476 } else { 2477 /* 2478 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation 2479 * to MOVABLE pageblock, consider all non-movable pages as 2480 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or 2481 * vice versa, be conservative since we can't distinguish the 2482 * exact migratetype of non-movable pages. 2483 */ 2484 if (old_block_type == MIGRATE_MOVABLE) 2485 alike_pages = pageblock_nr_pages 2486 - (free_pages + movable_pages); 2487 else 2488 alike_pages = 0; 2489 } 2490 2491 /* moving whole block can fail due to zone boundary conditions */ 2492 if (!free_pages) 2493 goto single_page; 2494 2495 /* 2496 * If a sufficient number of pages in the block are either free or of 2497 * comparable migratability as our allocation, claim the whole block. 2498 */ 2499 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || 2500 page_group_by_mobility_disabled) 2501 set_pageblock_migratetype(page, start_type); 2502 2503 return; 2504 2505 single_page: 2506 move_to_free_list(page, zone, current_order, start_type); 2507 } 2508 2509 /* 2510 * Check whether there is a suitable fallback freepage with requested order. 2511 * If only_stealable is true, this function returns fallback_mt only if 2512 * we can steal other freepages all together. This would help to reduce 2513 * fragmentation due to mixed migratetype pages in one pageblock. 2514 */ 2515 int find_suitable_fallback(struct free_area *area, unsigned int order, 2516 int migratetype, bool only_stealable, bool *can_steal) 2517 { 2518 int i; 2519 int fallback_mt; 2520 2521 if (area->nr_free == 0) 2522 return -1; 2523 2524 *can_steal = false; 2525 for (i = 0;; i++) { 2526 fallback_mt = fallbacks[migratetype][i]; 2527 if (fallback_mt == MIGRATE_TYPES) 2528 break; 2529 2530 if (free_area_empty(area, fallback_mt)) 2531 continue; 2532 2533 if (can_steal_fallback(order, migratetype)) 2534 *can_steal = true; 2535 2536 if (!only_stealable) 2537 return fallback_mt; 2538 2539 if (*can_steal) 2540 return fallback_mt; 2541 } 2542 2543 return -1; 2544 } 2545 2546 /* 2547 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2548 * there are no empty page blocks that contain a page with a suitable order 2549 */ 2550 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2551 unsigned int alloc_order) 2552 { 2553 int mt; 2554 unsigned long max_managed, flags; 2555 2556 /* 2557 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2558 * Check is race-prone but harmless. 2559 */ 2560 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages; 2561 if (zone->nr_reserved_highatomic >= max_managed) 2562 return; 2563 2564 spin_lock_irqsave(&zone->lock, flags); 2565 2566 /* Recheck the nr_reserved_highatomic limit under the lock */ 2567 if (zone->nr_reserved_highatomic >= max_managed) 2568 goto out_unlock; 2569 2570 /* Yoink! */ 2571 mt = get_pageblock_migratetype(page); 2572 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt) 2573 && !is_migrate_cma(mt)) { 2574 zone->nr_reserved_highatomic += pageblock_nr_pages; 2575 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2576 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); 2577 } 2578 2579 out_unlock: 2580 spin_unlock_irqrestore(&zone->lock, flags); 2581 } 2582 2583 /* 2584 * Used when an allocation is about to fail under memory pressure. This 2585 * potentially hurts the reliability of high-order allocations when under 2586 * intense memory pressure but failed atomic allocations should be easier 2587 * to recover from than an OOM. 2588 * 2589 * If @force is true, try to unreserve a pageblock even though highatomic 2590 * pageblock is exhausted. 2591 */ 2592 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 2593 bool force) 2594 { 2595 struct zonelist *zonelist = ac->zonelist; 2596 unsigned long flags; 2597 struct zoneref *z; 2598 struct zone *zone; 2599 struct page *page; 2600 int order; 2601 bool ret; 2602 2603 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx, 2604 ac->nodemask) { 2605 /* 2606 * Preserve at least one pageblock unless memory pressure 2607 * is really high. 2608 */ 2609 if (!force && zone->nr_reserved_highatomic <= 2610 pageblock_nr_pages) 2611 continue; 2612 2613 spin_lock_irqsave(&zone->lock, flags); 2614 for (order = 0; order < MAX_ORDER; order++) { 2615 struct free_area *area = &(zone->free_area[order]); 2616 2617 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); 2618 if (!page) 2619 continue; 2620 2621 /* 2622 * In page freeing path, migratetype change is racy so 2623 * we can counter several free pages in a pageblock 2624 * in this loop althoug we changed the pageblock type 2625 * from highatomic to ac->migratetype. So we should 2626 * adjust the count once. 2627 */ 2628 if (is_migrate_highatomic_page(page)) { 2629 /* 2630 * It should never happen but changes to 2631 * locking could inadvertently allow a per-cpu 2632 * drain to add pages to MIGRATE_HIGHATOMIC 2633 * while unreserving so be safe and watch for 2634 * underflows. 2635 */ 2636 zone->nr_reserved_highatomic -= min( 2637 pageblock_nr_pages, 2638 zone->nr_reserved_highatomic); 2639 } 2640 2641 /* 2642 * Convert to ac->migratetype and avoid the normal 2643 * pageblock stealing heuristics. Minimally, the caller 2644 * is doing the work and needs the pages. More 2645 * importantly, if the block was always converted to 2646 * MIGRATE_UNMOVABLE or another type then the number 2647 * of pageblocks that cannot be completely freed 2648 * may increase. 2649 */ 2650 set_pageblock_migratetype(page, ac->migratetype); 2651 ret = move_freepages_block(zone, page, ac->migratetype, 2652 NULL); 2653 if (ret) { 2654 spin_unlock_irqrestore(&zone->lock, flags); 2655 return ret; 2656 } 2657 } 2658 spin_unlock_irqrestore(&zone->lock, flags); 2659 } 2660 2661 return false; 2662 } 2663 2664 /* 2665 * Try finding a free buddy page on the fallback list and put it on the free 2666 * list of requested migratetype, possibly along with other pages from the same 2667 * block, depending on fragmentation avoidance heuristics. Returns true if 2668 * fallback was found so that __rmqueue_smallest() can grab it. 2669 * 2670 * The use of signed ints for order and current_order is a deliberate 2671 * deviation from the rest of this file, to make the for loop 2672 * condition simpler. 2673 */ 2674 static __always_inline bool 2675 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, 2676 unsigned int alloc_flags) 2677 { 2678 struct free_area *area; 2679 int current_order; 2680 int min_order = order; 2681 struct page *page; 2682 int fallback_mt; 2683 bool can_steal; 2684 2685 /* 2686 * Do not steal pages from freelists belonging to other pageblocks 2687 * i.e. orders < pageblock_order. If there are no local zones free, 2688 * the zonelists will be reiterated without ALLOC_NOFRAGMENT. 2689 */ 2690 if (alloc_flags & ALLOC_NOFRAGMENT) 2691 min_order = pageblock_order; 2692 2693 /* 2694 * Find the largest available free page in the other list. This roughly 2695 * approximates finding the pageblock with the most free pages, which 2696 * would be too costly to do exactly. 2697 */ 2698 for (current_order = MAX_ORDER - 1; current_order >= min_order; 2699 --current_order) { 2700 area = &(zone->free_area[current_order]); 2701 fallback_mt = find_suitable_fallback(area, current_order, 2702 start_migratetype, false, &can_steal); 2703 if (fallback_mt == -1) 2704 continue; 2705 2706 /* 2707 * We cannot steal all free pages from the pageblock and the 2708 * requested migratetype is movable. In that case it's better to 2709 * steal and split the smallest available page instead of the 2710 * largest available page, because even if the next movable 2711 * allocation falls back into a different pageblock than this 2712 * one, it won't cause permanent fragmentation. 2713 */ 2714 if (!can_steal && start_migratetype == MIGRATE_MOVABLE 2715 && current_order > order) 2716 goto find_smallest; 2717 2718 goto do_steal; 2719 } 2720 2721 return false; 2722 2723 find_smallest: 2724 for (current_order = order; current_order < MAX_ORDER; 2725 current_order++) { 2726 area = &(zone->free_area[current_order]); 2727 fallback_mt = find_suitable_fallback(area, current_order, 2728 start_migratetype, false, &can_steal); 2729 if (fallback_mt != -1) 2730 break; 2731 } 2732 2733 /* 2734 * This should not happen - we already found a suitable fallback 2735 * when looking for the largest page. 2736 */ 2737 VM_BUG_ON(current_order == MAX_ORDER); 2738 2739 do_steal: 2740 page = get_page_from_free_area(area, fallback_mt); 2741 2742 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype, 2743 can_steal); 2744 2745 trace_mm_page_alloc_extfrag(page, order, current_order, 2746 start_migratetype, fallback_mt); 2747 2748 return true; 2749 2750 } 2751 2752 /* 2753 * Do the hard work of removing an element from the buddy allocator. 2754 * Call me with the zone->lock already held. 2755 */ 2756 static __always_inline struct page * 2757 __rmqueue(struct zone *zone, unsigned int order, int migratetype, 2758 unsigned int alloc_flags) 2759 { 2760 struct page *page; 2761 2762 retry: 2763 page = __rmqueue_smallest(zone, order, migratetype); 2764 if (unlikely(!page)) { 2765 if (migratetype == MIGRATE_MOVABLE) 2766 page = __rmqueue_cma_fallback(zone, order); 2767 2768 if (!page && __rmqueue_fallback(zone, order, migratetype, 2769 alloc_flags)) 2770 goto retry; 2771 } 2772 2773 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2774 return page; 2775 } 2776 2777 /* 2778 * Obtain a specified number of elements from the buddy allocator, all under 2779 * a single hold of the lock, for efficiency. Add them to the supplied list. 2780 * Returns the number of new pages which were placed at *list. 2781 */ 2782 static int rmqueue_bulk(struct zone *zone, unsigned int order, 2783 unsigned long count, struct list_head *list, 2784 int migratetype, unsigned int alloc_flags) 2785 { 2786 int i, alloced = 0; 2787 2788 spin_lock(&zone->lock); 2789 for (i = 0; i < count; ++i) { 2790 struct page *page = __rmqueue(zone, order, migratetype, 2791 alloc_flags); 2792 if (unlikely(page == NULL)) 2793 break; 2794 2795 if (unlikely(check_pcp_refill(page))) 2796 continue; 2797 2798 /* 2799 * Split buddy pages returned by expand() are received here in 2800 * physical page order. The page is added to the tail of 2801 * caller's list. From the callers perspective, the linked list 2802 * is ordered by page number under some conditions. This is 2803 * useful for IO devices that can forward direction from the 2804 * head, thus also in the physical page order. This is useful 2805 * for IO devices that can merge IO requests if the physical 2806 * pages are ordered properly. 2807 */ 2808 list_add_tail(&page->lru, list); 2809 alloced++; 2810 if (is_migrate_cma(get_pcppage_migratetype(page))) 2811 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 2812 -(1 << order)); 2813 } 2814 2815 /* 2816 * i pages were removed from the buddy list even if some leak due 2817 * to check_pcp_refill failing so adjust NR_FREE_PAGES based 2818 * on i. Do not confuse with 'alloced' which is the number of 2819 * pages added to the pcp list. 2820 */ 2821 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 2822 spin_unlock(&zone->lock); 2823 return alloced; 2824 } 2825 2826 #ifdef CONFIG_NUMA 2827 /* 2828 * Called from the vmstat counter updater to drain pagesets of this 2829 * currently executing processor on remote nodes after they have 2830 * expired. 2831 * 2832 * Note that this function must be called with the thread pinned to 2833 * a single processor. 2834 */ 2835 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 2836 { 2837 unsigned long flags; 2838 int to_drain, batch; 2839 2840 local_irq_save(flags); 2841 batch = READ_ONCE(pcp->batch); 2842 to_drain = min(pcp->count, batch); 2843 if (to_drain > 0) 2844 free_pcppages_bulk(zone, to_drain, pcp); 2845 local_irq_restore(flags); 2846 } 2847 #endif 2848 2849 /* 2850 * Drain pcplists of the indicated processor and zone. 2851 * 2852 * The processor must either be the current processor and the 2853 * thread pinned to the current processor or a processor that 2854 * is not online. 2855 */ 2856 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 2857 { 2858 unsigned long flags; 2859 struct per_cpu_pageset *pset; 2860 struct per_cpu_pages *pcp; 2861 2862 local_irq_save(flags); 2863 pset = per_cpu_ptr(zone->pageset, cpu); 2864 2865 pcp = &pset->pcp; 2866 if (pcp->count) 2867 free_pcppages_bulk(zone, pcp->count, pcp); 2868 local_irq_restore(flags); 2869 } 2870 2871 /* 2872 * Drain pcplists of all zones on the indicated processor. 2873 * 2874 * The processor must either be the current processor and the 2875 * thread pinned to the current processor or a processor that 2876 * is not online. 2877 */ 2878 static void drain_pages(unsigned int cpu) 2879 { 2880 struct zone *zone; 2881 2882 for_each_populated_zone(zone) { 2883 drain_pages_zone(cpu, zone); 2884 } 2885 } 2886 2887 /* 2888 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 2889 * 2890 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 2891 * the single zone's pages. 2892 */ 2893 void drain_local_pages(struct zone *zone) 2894 { 2895 int cpu = smp_processor_id(); 2896 2897 if (zone) 2898 drain_pages_zone(cpu, zone); 2899 else 2900 drain_pages(cpu); 2901 } 2902 2903 static void drain_local_pages_wq(struct work_struct *work) 2904 { 2905 struct pcpu_drain *drain; 2906 2907 drain = container_of(work, struct pcpu_drain, work); 2908 2909 /* 2910 * drain_all_pages doesn't use proper cpu hotplug protection so 2911 * we can race with cpu offline when the WQ can move this from 2912 * a cpu pinned worker to an unbound one. We can operate on a different 2913 * cpu which is allright but we also have to make sure to not move to 2914 * a different one. 2915 */ 2916 preempt_disable(); 2917 drain_local_pages(drain->zone); 2918 preempt_enable(); 2919 } 2920 2921 /* 2922 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 2923 * 2924 * When zone parameter is non-NULL, spill just the single zone's pages. 2925 * 2926 * Note that this can be extremely slow as the draining happens in a workqueue. 2927 */ 2928 void drain_all_pages(struct zone *zone) 2929 { 2930 int cpu; 2931 2932 /* 2933 * Allocate in the BSS so we wont require allocation in 2934 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 2935 */ 2936 static cpumask_t cpus_with_pcps; 2937 2938 /* 2939 * Make sure nobody triggers this path before mm_percpu_wq is fully 2940 * initialized. 2941 */ 2942 if (WARN_ON_ONCE(!mm_percpu_wq)) 2943 return; 2944 2945 /* 2946 * Do not drain if one is already in progress unless it's specific to 2947 * a zone. Such callers are primarily CMA and memory hotplug and need 2948 * the drain to be complete when the call returns. 2949 */ 2950 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) { 2951 if (!zone) 2952 return; 2953 mutex_lock(&pcpu_drain_mutex); 2954 } 2955 2956 /* 2957 * We don't care about racing with CPU hotplug event 2958 * as offline notification will cause the notified 2959 * cpu to drain that CPU pcps and on_each_cpu_mask 2960 * disables preemption as part of its processing 2961 */ 2962 for_each_online_cpu(cpu) { 2963 struct per_cpu_pageset *pcp; 2964 struct zone *z; 2965 bool has_pcps = false; 2966 2967 if (zone) { 2968 pcp = per_cpu_ptr(zone->pageset, cpu); 2969 if (pcp->pcp.count) 2970 has_pcps = true; 2971 } else { 2972 for_each_populated_zone(z) { 2973 pcp = per_cpu_ptr(z->pageset, cpu); 2974 if (pcp->pcp.count) { 2975 has_pcps = true; 2976 break; 2977 } 2978 } 2979 } 2980 2981 if (has_pcps) 2982 cpumask_set_cpu(cpu, &cpus_with_pcps); 2983 else 2984 cpumask_clear_cpu(cpu, &cpus_with_pcps); 2985 } 2986 2987 for_each_cpu(cpu, &cpus_with_pcps) { 2988 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu); 2989 2990 drain->zone = zone; 2991 INIT_WORK(&drain->work, drain_local_pages_wq); 2992 queue_work_on(cpu, mm_percpu_wq, &drain->work); 2993 } 2994 for_each_cpu(cpu, &cpus_with_pcps) 2995 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work); 2996 2997 mutex_unlock(&pcpu_drain_mutex); 2998 } 2999 3000 #ifdef CONFIG_HIBERNATION 3001 3002 /* 3003 * Touch the watchdog for every WD_PAGE_COUNT pages. 3004 */ 3005 #define WD_PAGE_COUNT (128*1024) 3006 3007 void mark_free_pages(struct zone *zone) 3008 { 3009 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT; 3010 unsigned long flags; 3011 unsigned int order, t; 3012 struct page *page; 3013 3014 if (zone_is_empty(zone)) 3015 return; 3016 3017 spin_lock_irqsave(&zone->lock, flags); 3018 3019 max_zone_pfn = zone_end_pfn(zone); 3020 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 3021 if (pfn_valid(pfn)) { 3022 page = pfn_to_page(pfn); 3023 3024 if (!--page_count) { 3025 touch_nmi_watchdog(); 3026 page_count = WD_PAGE_COUNT; 3027 } 3028 3029 if (page_zone(page) != zone) 3030 continue; 3031 3032 if (!swsusp_page_is_forbidden(page)) 3033 swsusp_unset_page_free(page); 3034 } 3035 3036 for_each_migratetype_order(order, t) { 3037 list_for_each_entry(page, 3038 &zone->free_area[order].free_list[t], lru) { 3039 unsigned long i; 3040 3041 pfn = page_to_pfn(page); 3042 for (i = 0; i < (1UL << order); i++) { 3043 if (!--page_count) { 3044 touch_nmi_watchdog(); 3045 page_count = WD_PAGE_COUNT; 3046 } 3047 swsusp_set_page_free(pfn_to_page(pfn + i)); 3048 } 3049 } 3050 } 3051 spin_unlock_irqrestore(&zone->lock, flags); 3052 } 3053 #endif /* CONFIG_PM */ 3054 3055 static bool free_unref_page_prepare(struct page *page, unsigned long pfn) 3056 { 3057 int migratetype; 3058 3059 if (!free_pcp_prepare(page)) 3060 return false; 3061 3062 migratetype = get_pfnblock_migratetype(page, pfn); 3063 set_pcppage_migratetype(page, migratetype); 3064 return true; 3065 } 3066 3067 static void free_unref_page_commit(struct page *page, unsigned long pfn) 3068 { 3069 struct zone *zone = page_zone(page); 3070 struct per_cpu_pages *pcp; 3071 int migratetype; 3072 3073 migratetype = get_pcppage_migratetype(page); 3074 __count_vm_event(PGFREE); 3075 3076 /* 3077 * We only track unmovable, reclaimable and movable on pcp lists. 3078 * Free ISOLATE pages back to the allocator because they are being 3079 * offlined but treat HIGHATOMIC as movable pages so we can get those 3080 * areas back if necessary. Otherwise, we may have to free 3081 * excessively into the page allocator 3082 */ 3083 if (migratetype >= MIGRATE_PCPTYPES) { 3084 if (unlikely(is_migrate_isolate(migratetype))) { 3085 free_one_page(zone, page, pfn, 0, migratetype); 3086 return; 3087 } 3088 migratetype = MIGRATE_MOVABLE; 3089 } 3090 3091 pcp = &this_cpu_ptr(zone->pageset)->pcp; 3092 list_add(&page->lru, &pcp->lists[migratetype]); 3093 pcp->count++; 3094 if (pcp->count >= pcp->high) { 3095 unsigned long batch = READ_ONCE(pcp->batch); 3096 free_pcppages_bulk(zone, batch, pcp); 3097 } 3098 } 3099 3100 /* 3101 * Free a 0-order page 3102 */ 3103 void free_unref_page(struct page *page) 3104 { 3105 unsigned long flags; 3106 unsigned long pfn = page_to_pfn(page); 3107 3108 if (!free_unref_page_prepare(page, pfn)) 3109 return; 3110 3111 local_irq_save(flags); 3112 free_unref_page_commit(page, pfn); 3113 local_irq_restore(flags); 3114 } 3115 3116 /* 3117 * Free a list of 0-order pages 3118 */ 3119 void free_unref_page_list(struct list_head *list) 3120 { 3121 struct page *page, *next; 3122 unsigned long flags, pfn; 3123 int batch_count = 0; 3124 3125 /* Prepare pages for freeing */ 3126 list_for_each_entry_safe(page, next, list, lru) { 3127 pfn = page_to_pfn(page); 3128 if (!free_unref_page_prepare(page, pfn)) 3129 list_del(&page->lru); 3130 set_page_private(page, pfn); 3131 } 3132 3133 local_irq_save(flags); 3134 list_for_each_entry_safe(page, next, list, lru) { 3135 unsigned long pfn = page_private(page); 3136 3137 set_page_private(page, 0); 3138 trace_mm_page_free_batched(page); 3139 free_unref_page_commit(page, pfn); 3140 3141 /* 3142 * Guard against excessive IRQ disabled times when we get 3143 * a large list of pages to free. 3144 */ 3145 if (++batch_count == SWAP_CLUSTER_MAX) { 3146 local_irq_restore(flags); 3147 batch_count = 0; 3148 local_irq_save(flags); 3149 } 3150 } 3151 local_irq_restore(flags); 3152 } 3153 3154 /* 3155 * split_page takes a non-compound higher-order page, and splits it into 3156 * n (1<<order) sub-pages: page[0..n] 3157 * Each sub-page must be freed individually. 3158 * 3159 * Note: this is probably too low level an operation for use in drivers. 3160 * Please consult with lkml before using this in your driver. 3161 */ 3162 void split_page(struct page *page, unsigned int order) 3163 { 3164 int i; 3165 3166 VM_BUG_ON_PAGE(PageCompound(page), page); 3167 VM_BUG_ON_PAGE(!page_count(page), page); 3168 3169 for (i = 1; i < (1 << order); i++) 3170 set_page_refcounted(page + i); 3171 split_page_owner(page, order); 3172 } 3173 EXPORT_SYMBOL_GPL(split_page); 3174 3175 int __isolate_free_page(struct page *page, unsigned int order) 3176 { 3177 unsigned long watermark; 3178 struct zone *zone; 3179 int mt; 3180 3181 BUG_ON(!PageBuddy(page)); 3182 3183 zone = page_zone(page); 3184 mt = get_pageblock_migratetype(page); 3185 3186 if (!is_migrate_isolate(mt)) { 3187 /* 3188 * Obey watermarks as if the page was being allocated. We can 3189 * emulate a high-order watermark check with a raised order-0 3190 * watermark, because we already know our high-order page 3191 * exists. 3192 */ 3193 watermark = zone->_watermark[WMARK_MIN] + (1UL << order); 3194 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 3195 return 0; 3196 3197 __mod_zone_freepage_state(zone, -(1UL << order), mt); 3198 } 3199 3200 /* Remove page from free list */ 3201 3202 del_page_from_free_list(page, zone, order); 3203 3204 /* 3205 * Set the pageblock if the isolated page is at least half of a 3206 * pageblock 3207 */ 3208 if (order >= pageblock_order - 1) { 3209 struct page *endpage = page + (1 << order) - 1; 3210 for (; page < endpage; page += pageblock_nr_pages) { 3211 int mt = get_pageblock_migratetype(page); 3212 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt) 3213 && !is_migrate_highatomic(mt)) 3214 set_pageblock_migratetype(page, 3215 MIGRATE_MOVABLE); 3216 } 3217 } 3218 3219 3220 return 1UL << order; 3221 } 3222 3223 /** 3224 * __putback_isolated_page - Return a now-isolated page back where we got it 3225 * @page: Page that was isolated 3226 * @order: Order of the isolated page 3227 * 3228 * This function is meant to return a page pulled from the free lists via 3229 * __isolate_free_page back to the free lists they were pulled from. 3230 */ 3231 void __putback_isolated_page(struct page *page, unsigned int order, int mt) 3232 { 3233 struct zone *zone = page_zone(page); 3234 3235 /* zone lock should be held when this function is called */ 3236 lockdep_assert_held(&zone->lock); 3237 3238 /* Return isolated page to tail of freelist. */ 3239 __free_one_page(page, page_to_pfn(page), zone, order, mt, false); 3240 } 3241 3242 /* 3243 * Update NUMA hit/miss statistics 3244 * 3245 * Must be called with interrupts disabled. 3246 */ 3247 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) 3248 { 3249 #ifdef CONFIG_NUMA 3250 enum numa_stat_item local_stat = NUMA_LOCAL; 3251 3252 /* skip numa counters update if numa stats is disabled */ 3253 if (!static_branch_likely(&vm_numa_stat_key)) 3254 return; 3255 3256 if (zone_to_nid(z) != numa_node_id()) 3257 local_stat = NUMA_OTHER; 3258 3259 if (zone_to_nid(z) == zone_to_nid(preferred_zone)) 3260 __inc_numa_state(z, NUMA_HIT); 3261 else { 3262 __inc_numa_state(z, NUMA_MISS); 3263 __inc_numa_state(preferred_zone, NUMA_FOREIGN); 3264 } 3265 __inc_numa_state(z, local_stat); 3266 #endif 3267 } 3268 3269 /* Remove page from the per-cpu list, caller must protect the list */ 3270 static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype, 3271 unsigned int alloc_flags, 3272 struct per_cpu_pages *pcp, 3273 struct list_head *list) 3274 { 3275 struct page *page; 3276 3277 do { 3278 if (list_empty(list)) { 3279 pcp->count += rmqueue_bulk(zone, 0, 3280 pcp->batch, list, 3281 migratetype, alloc_flags); 3282 if (unlikely(list_empty(list))) 3283 return NULL; 3284 } 3285 3286 page = list_first_entry(list, struct page, lru); 3287 list_del(&page->lru); 3288 pcp->count--; 3289 } while (check_new_pcp(page)); 3290 3291 return page; 3292 } 3293 3294 /* Lock and remove page from the per-cpu list */ 3295 static struct page *rmqueue_pcplist(struct zone *preferred_zone, 3296 struct zone *zone, gfp_t gfp_flags, 3297 int migratetype, unsigned int alloc_flags) 3298 { 3299 struct per_cpu_pages *pcp; 3300 struct list_head *list; 3301 struct page *page; 3302 unsigned long flags; 3303 3304 local_irq_save(flags); 3305 pcp = &this_cpu_ptr(zone->pageset)->pcp; 3306 list = &pcp->lists[migratetype]; 3307 page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list); 3308 if (page) { 3309 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); 3310 zone_statistics(preferred_zone, zone); 3311 } 3312 local_irq_restore(flags); 3313 return page; 3314 } 3315 3316 /* 3317 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 3318 */ 3319 static inline 3320 struct page *rmqueue(struct zone *preferred_zone, 3321 struct zone *zone, unsigned int order, 3322 gfp_t gfp_flags, unsigned int alloc_flags, 3323 int migratetype) 3324 { 3325 unsigned long flags; 3326 struct page *page; 3327 3328 if (likely(order == 0)) { 3329 page = rmqueue_pcplist(preferred_zone, zone, gfp_flags, 3330 migratetype, alloc_flags); 3331 goto out; 3332 } 3333 3334 /* 3335 * We most definitely don't want callers attempting to 3336 * allocate greater than order-1 page units with __GFP_NOFAIL. 3337 */ 3338 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 3339 spin_lock_irqsave(&zone->lock, flags); 3340 3341 do { 3342 page = NULL; 3343 if (alloc_flags & ALLOC_HARDER) { 3344 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3345 if (page) 3346 trace_mm_page_alloc_zone_locked(page, order, migratetype); 3347 } 3348 if (!page) 3349 page = __rmqueue(zone, order, migratetype, alloc_flags); 3350 } while (page && check_new_pages(page, order)); 3351 spin_unlock(&zone->lock); 3352 if (!page) 3353 goto failed; 3354 __mod_zone_freepage_state(zone, -(1 << order), 3355 get_pcppage_migratetype(page)); 3356 3357 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3358 zone_statistics(preferred_zone, zone); 3359 local_irq_restore(flags); 3360 3361 out: 3362 /* Separate test+clear to avoid unnecessary atomics */ 3363 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) { 3364 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 3365 wakeup_kswapd(zone, 0, 0, zone_idx(zone)); 3366 } 3367 3368 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); 3369 return page; 3370 3371 failed: 3372 local_irq_restore(flags); 3373 return NULL; 3374 } 3375 3376 #ifdef CONFIG_FAIL_PAGE_ALLOC 3377 3378 static struct { 3379 struct fault_attr attr; 3380 3381 bool ignore_gfp_highmem; 3382 bool ignore_gfp_reclaim; 3383 u32 min_order; 3384 } fail_page_alloc = { 3385 .attr = FAULT_ATTR_INITIALIZER, 3386 .ignore_gfp_reclaim = true, 3387 .ignore_gfp_highmem = true, 3388 .min_order = 1, 3389 }; 3390 3391 static int __init setup_fail_page_alloc(char *str) 3392 { 3393 return setup_fault_attr(&fail_page_alloc.attr, str); 3394 } 3395 __setup("fail_page_alloc=", setup_fail_page_alloc); 3396 3397 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3398 { 3399 if (order < fail_page_alloc.min_order) 3400 return false; 3401 if (gfp_mask & __GFP_NOFAIL) 3402 return false; 3403 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 3404 return false; 3405 if (fail_page_alloc.ignore_gfp_reclaim && 3406 (gfp_mask & __GFP_DIRECT_RECLAIM)) 3407 return false; 3408 3409 return should_fail(&fail_page_alloc.attr, 1 << order); 3410 } 3411 3412 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 3413 3414 static int __init fail_page_alloc_debugfs(void) 3415 { 3416 umode_t mode = S_IFREG | 0600; 3417 struct dentry *dir; 3418 3419 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 3420 &fail_page_alloc.attr); 3421 3422 debugfs_create_bool("ignore-gfp-wait", mode, dir, 3423 &fail_page_alloc.ignore_gfp_reclaim); 3424 debugfs_create_bool("ignore-gfp-highmem", mode, dir, 3425 &fail_page_alloc.ignore_gfp_highmem); 3426 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order); 3427 3428 return 0; 3429 } 3430 3431 late_initcall(fail_page_alloc_debugfs); 3432 3433 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 3434 3435 #else /* CONFIG_FAIL_PAGE_ALLOC */ 3436 3437 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3438 { 3439 return false; 3440 } 3441 3442 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 3443 3444 static noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3445 { 3446 return __should_fail_alloc_page(gfp_mask, order); 3447 } 3448 ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE); 3449 3450 /* 3451 * Return true if free base pages are above 'mark'. For high-order checks it 3452 * will return true of the order-0 watermark is reached and there is at least 3453 * one free page of a suitable size. Checking now avoids taking the zone lock 3454 * to check in the allocation paths if no pages are free. 3455 */ 3456 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3457 int classzone_idx, unsigned int alloc_flags, 3458 long free_pages) 3459 { 3460 long min = mark; 3461 int o; 3462 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3463 3464 /* free_pages may go negative - that's OK */ 3465 free_pages -= (1 << order) - 1; 3466 3467 if (alloc_flags & ALLOC_HIGH) 3468 min -= min / 2; 3469 3470 /* 3471 * If the caller does not have rights to ALLOC_HARDER then subtract 3472 * the high-atomic reserves. This will over-estimate the size of the 3473 * atomic reserve but it avoids a search. 3474 */ 3475 if (likely(!alloc_harder)) { 3476 free_pages -= z->nr_reserved_highatomic; 3477 } else { 3478 /* 3479 * OOM victims can try even harder than normal ALLOC_HARDER 3480 * users on the grounds that it's definitely going to be in 3481 * the exit path shortly and free memory. Any allocation it 3482 * makes during the free path will be small and short-lived. 3483 */ 3484 if (alloc_flags & ALLOC_OOM) 3485 min -= min / 2; 3486 else 3487 min -= min / 4; 3488 } 3489 3490 3491 #ifdef CONFIG_CMA 3492 /* If allocation can't use CMA areas don't use free CMA pages */ 3493 if (!(alloc_flags & ALLOC_CMA)) 3494 free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES); 3495 #endif 3496 3497 /* 3498 * Check watermarks for an order-0 allocation request. If these 3499 * are not met, then a high-order request also cannot go ahead 3500 * even if a suitable page happened to be free. 3501 */ 3502 if (free_pages <= min + z->lowmem_reserve[classzone_idx]) 3503 return false; 3504 3505 /* If this is an order-0 request then the watermark is fine */ 3506 if (!order) 3507 return true; 3508 3509 /* For a high-order request, check at least one suitable page is free */ 3510 for (o = order; o < MAX_ORDER; o++) { 3511 struct free_area *area = &z->free_area[o]; 3512 int mt; 3513 3514 if (!area->nr_free) 3515 continue; 3516 3517 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 3518 if (!free_area_empty(area, mt)) 3519 return true; 3520 } 3521 3522 #ifdef CONFIG_CMA 3523 if ((alloc_flags & ALLOC_CMA) && 3524 !free_area_empty(area, MIGRATE_CMA)) { 3525 return true; 3526 } 3527 #endif 3528 if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC)) 3529 return true; 3530 } 3531 return false; 3532 } 3533 3534 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3535 int classzone_idx, unsigned int alloc_flags) 3536 { 3537 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 3538 zone_page_state(z, NR_FREE_PAGES)); 3539 } 3540 3541 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 3542 unsigned long mark, int classzone_idx, unsigned int alloc_flags) 3543 { 3544 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3545 long cma_pages = 0; 3546 3547 #ifdef CONFIG_CMA 3548 /* If allocation can't use CMA areas don't use free CMA pages */ 3549 if (!(alloc_flags & ALLOC_CMA)) 3550 cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES); 3551 #endif 3552 3553 /* 3554 * Fast check for order-0 only. If this fails then the reserves 3555 * need to be calculated. There is a corner case where the check 3556 * passes but only the high-order atomic reserve are free. If 3557 * the caller is !atomic then it'll uselessly search the free 3558 * list. That corner case is then slower but it is harmless. 3559 */ 3560 if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx]) 3561 return true; 3562 3563 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 3564 free_pages); 3565 } 3566 3567 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 3568 unsigned long mark, int classzone_idx) 3569 { 3570 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3571 3572 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 3573 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 3574 3575 return __zone_watermark_ok(z, order, mark, classzone_idx, 0, 3576 free_pages); 3577 } 3578 3579 #ifdef CONFIG_NUMA 3580 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3581 { 3582 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= 3583 node_reclaim_distance; 3584 } 3585 #else /* CONFIG_NUMA */ 3586 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3587 { 3588 return true; 3589 } 3590 #endif /* CONFIG_NUMA */ 3591 3592 /* 3593 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid 3594 * fragmentation is subtle. If the preferred zone was HIGHMEM then 3595 * premature use of a lower zone may cause lowmem pressure problems that 3596 * are worse than fragmentation. If the next zone is ZONE_DMA then it is 3597 * probably too small. It only makes sense to spread allocations to avoid 3598 * fragmentation between the Normal and DMA32 zones. 3599 */ 3600 static inline unsigned int 3601 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) 3602 { 3603 unsigned int alloc_flags; 3604 3605 /* 3606 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 3607 * to save a branch. 3608 */ 3609 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); 3610 3611 #ifdef CONFIG_ZONE_DMA32 3612 if (!zone) 3613 return alloc_flags; 3614 3615 if (zone_idx(zone) != ZONE_NORMAL) 3616 return alloc_flags; 3617 3618 /* 3619 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and 3620 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume 3621 * on UMA that if Normal is populated then so is DMA32. 3622 */ 3623 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1); 3624 if (nr_online_nodes > 1 && !populated_zone(--zone)) 3625 return alloc_flags; 3626 3627 alloc_flags |= ALLOC_NOFRAGMENT; 3628 #endif /* CONFIG_ZONE_DMA32 */ 3629 return alloc_flags; 3630 } 3631 3632 /* 3633 * get_page_from_freelist goes through the zonelist trying to allocate 3634 * a page. 3635 */ 3636 static struct page * 3637 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 3638 const struct alloc_context *ac) 3639 { 3640 struct zoneref *z; 3641 struct zone *zone; 3642 struct pglist_data *last_pgdat_dirty_limit = NULL; 3643 bool no_fallback; 3644 3645 retry: 3646 /* 3647 * Scan zonelist, looking for a zone with enough free. 3648 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 3649 */ 3650 no_fallback = alloc_flags & ALLOC_NOFRAGMENT; 3651 z = ac->preferred_zoneref; 3652 for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3653 ac->nodemask) { 3654 struct page *page; 3655 unsigned long mark; 3656 3657 if (cpusets_enabled() && 3658 (alloc_flags & ALLOC_CPUSET) && 3659 !__cpuset_zone_allowed(zone, gfp_mask)) 3660 continue; 3661 /* 3662 * When allocating a page cache page for writing, we 3663 * want to get it from a node that is within its dirty 3664 * limit, such that no single node holds more than its 3665 * proportional share of globally allowed dirty pages. 3666 * The dirty limits take into account the node's 3667 * lowmem reserves and high watermark so that kswapd 3668 * should be able to balance it without having to 3669 * write pages from its LRU list. 3670 * 3671 * XXX: For now, allow allocations to potentially 3672 * exceed the per-node dirty limit in the slowpath 3673 * (spread_dirty_pages unset) before going into reclaim, 3674 * which is important when on a NUMA setup the allowed 3675 * nodes are together not big enough to reach the 3676 * global limit. The proper fix for these situations 3677 * will require awareness of nodes in the 3678 * dirty-throttling and the flusher threads. 3679 */ 3680 if (ac->spread_dirty_pages) { 3681 if (last_pgdat_dirty_limit == zone->zone_pgdat) 3682 continue; 3683 3684 if (!node_dirty_ok(zone->zone_pgdat)) { 3685 last_pgdat_dirty_limit = zone->zone_pgdat; 3686 continue; 3687 } 3688 } 3689 3690 if (no_fallback && nr_online_nodes > 1 && 3691 zone != ac->preferred_zoneref->zone) { 3692 int local_nid; 3693 3694 /* 3695 * If moving to a remote node, retry but allow 3696 * fragmenting fallbacks. Locality is more important 3697 * than fragmentation avoidance. 3698 */ 3699 local_nid = zone_to_nid(ac->preferred_zoneref->zone); 3700 if (zone_to_nid(zone) != local_nid) { 3701 alloc_flags &= ~ALLOC_NOFRAGMENT; 3702 goto retry; 3703 } 3704 } 3705 3706 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); 3707 if (!zone_watermark_fast(zone, order, mark, 3708 ac_classzone_idx(ac), alloc_flags)) { 3709 int ret; 3710 3711 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3712 /* 3713 * Watermark failed for this zone, but see if we can 3714 * grow this zone if it contains deferred pages. 3715 */ 3716 if (static_branch_unlikely(&deferred_pages)) { 3717 if (_deferred_grow_zone(zone, order)) 3718 goto try_this_zone; 3719 } 3720 #endif 3721 /* Checked here to keep the fast path fast */ 3722 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 3723 if (alloc_flags & ALLOC_NO_WATERMARKS) 3724 goto try_this_zone; 3725 3726 if (node_reclaim_mode == 0 || 3727 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 3728 continue; 3729 3730 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 3731 switch (ret) { 3732 case NODE_RECLAIM_NOSCAN: 3733 /* did not scan */ 3734 continue; 3735 case NODE_RECLAIM_FULL: 3736 /* scanned but unreclaimable */ 3737 continue; 3738 default: 3739 /* did we reclaim enough */ 3740 if (zone_watermark_ok(zone, order, mark, 3741 ac_classzone_idx(ac), alloc_flags)) 3742 goto try_this_zone; 3743 3744 continue; 3745 } 3746 } 3747 3748 try_this_zone: 3749 page = rmqueue(ac->preferred_zoneref->zone, zone, order, 3750 gfp_mask, alloc_flags, ac->migratetype); 3751 if (page) { 3752 prep_new_page(page, order, gfp_mask, alloc_flags); 3753 3754 /* 3755 * If this is a high-order atomic allocation then check 3756 * if the pageblock should be reserved for the future 3757 */ 3758 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 3759 reserve_highatomic_pageblock(page, zone, order); 3760 3761 return page; 3762 } else { 3763 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3764 /* Try again if zone has deferred pages */ 3765 if (static_branch_unlikely(&deferred_pages)) { 3766 if (_deferred_grow_zone(zone, order)) 3767 goto try_this_zone; 3768 } 3769 #endif 3770 } 3771 } 3772 3773 /* 3774 * It's possible on a UMA machine to get through all zones that are 3775 * fragmented. If avoiding fragmentation, reset and try again. 3776 */ 3777 if (no_fallback) { 3778 alloc_flags &= ~ALLOC_NOFRAGMENT; 3779 goto retry; 3780 } 3781 3782 return NULL; 3783 } 3784 3785 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) 3786 { 3787 unsigned int filter = SHOW_MEM_FILTER_NODES; 3788 3789 /* 3790 * This documents exceptions given to allocations in certain 3791 * contexts that are allowed to allocate outside current's set 3792 * of allowed nodes. 3793 */ 3794 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3795 if (tsk_is_oom_victim(current) || 3796 (current->flags & (PF_MEMALLOC | PF_EXITING))) 3797 filter &= ~SHOW_MEM_FILTER_NODES; 3798 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 3799 filter &= ~SHOW_MEM_FILTER_NODES; 3800 3801 show_mem(filter, nodemask); 3802 } 3803 3804 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) 3805 { 3806 struct va_format vaf; 3807 va_list args; 3808 static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1); 3809 3810 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs)) 3811 return; 3812 3813 va_start(args, fmt); 3814 vaf.fmt = fmt; 3815 vaf.va = &args; 3816 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl", 3817 current->comm, &vaf, gfp_mask, &gfp_mask, 3818 nodemask_pr_args(nodemask)); 3819 va_end(args); 3820 3821 cpuset_print_current_mems_allowed(); 3822 pr_cont("\n"); 3823 dump_stack(); 3824 warn_alloc_show_mem(gfp_mask, nodemask); 3825 } 3826 3827 static inline struct page * 3828 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, 3829 unsigned int alloc_flags, 3830 const struct alloc_context *ac) 3831 { 3832 struct page *page; 3833 3834 page = get_page_from_freelist(gfp_mask, order, 3835 alloc_flags|ALLOC_CPUSET, ac); 3836 /* 3837 * fallback to ignore cpuset restriction if our nodes 3838 * are depleted 3839 */ 3840 if (!page) 3841 page = get_page_from_freelist(gfp_mask, order, 3842 alloc_flags, ac); 3843 3844 return page; 3845 } 3846 3847 static inline struct page * 3848 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 3849 const struct alloc_context *ac, unsigned long *did_some_progress) 3850 { 3851 struct oom_control oc = { 3852 .zonelist = ac->zonelist, 3853 .nodemask = ac->nodemask, 3854 .memcg = NULL, 3855 .gfp_mask = gfp_mask, 3856 .order = order, 3857 }; 3858 struct page *page; 3859 3860 *did_some_progress = 0; 3861 3862 /* 3863 * Acquire the oom lock. If that fails, somebody else is 3864 * making progress for us. 3865 */ 3866 if (!mutex_trylock(&oom_lock)) { 3867 *did_some_progress = 1; 3868 schedule_timeout_uninterruptible(1); 3869 return NULL; 3870 } 3871 3872 /* 3873 * Go through the zonelist yet one more time, keep very high watermark 3874 * here, this is only to catch a parallel oom killing, we must fail if 3875 * we're still under heavy pressure. But make sure that this reclaim 3876 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY 3877 * allocation which will never fail due to oom_lock already held. 3878 */ 3879 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & 3880 ~__GFP_DIRECT_RECLAIM, order, 3881 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 3882 if (page) 3883 goto out; 3884 3885 /* Coredumps can quickly deplete all memory reserves */ 3886 if (current->flags & PF_DUMPCORE) 3887 goto out; 3888 /* The OOM killer will not help higher order allocs */ 3889 if (order > PAGE_ALLOC_COSTLY_ORDER) 3890 goto out; 3891 /* 3892 * We have already exhausted all our reclaim opportunities without any 3893 * success so it is time to admit defeat. We will skip the OOM killer 3894 * because it is very likely that the caller has a more reasonable 3895 * fallback than shooting a random task. 3896 */ 3897 if (gfp_mask & __GFP_RETRY_MAYFAIL) 3898 goto out; 3899 /* The OOM killer does not needlessly kill tasks for lowmem */ 3900 if (ac->high_zoneidx < ZONE_NORMAL) 3901 goto out; 3902 if (pm_suspended_storage()) 3903 goto out; 3904 /* 3905 * XXX: GFP_NOFS allocations should rather fail than rely on 3906 * other request to make a forward progress. 3907 * We are in an unfortunate situation where out_of_memory cannot 3908 * do much for this context but let's try it to at least get 3909 * access to memory reserved if the current task is killed (see 3910 * out_of_memory). Once filesystems are ready to handle allocation 3911 * failures more gracefully we should just bail out here. 3912 */ 3913 3914 /* The OOM killer may not free memory on a specific node */ 3915 if (gfp_mask & __GFP_THISNODE) 3916 goto out; 3917 3918 /* Exhausted what can be done so it's blame time */ 3919 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 3920 *did_some_progress = 1; 3921 3922 /* 3923 * Help non-failing allocations by giving them access to memory 3924 * reserves 3925 */ 3926 if (gfp_mask & __GFP_NOFAIL) 3927 page = __alloc_pages_cpuset_fallback(gfp_mask, order, 3928 ALLOC_NO_WATERMARKS, ac); 3929 } 3930 out: 3931 mutex_unlock(&oom_lock); 3932 return page; 3933 } 3934 3935 /* 3936 * Maximum number of compaction retries wit a progress before OOM 3937 * killer is consider as the only way to move forward. 3938 */ 3939 #define MAX_COMPACT_RETRIES 16 3940 3941 #ifdef CONFIG_COMPACTION 3942 /* Try memory compaction for high-order allocations before reclaim */ 3943 static struct page * 3944 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3945 unsigned int alloc_flags, const struct alloc_context *ac, 3946 enum compact_priority prio, enum compact_result *compact_result) 3947 { 3948 struct page *page = NULL; 3949 unsigned long pflags; 3950 unsigned int noreclaim_flag; 3951 3952 if (!order) 3953 return NULL; 3954 3955 psi_memstall_enter(&pflags); 3956 noreclaim_flag = memalloc_noreclaim_save(); 3957 3958 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 3959 prio, &page); 3960 3961 memalloc_noreclaim_restore(noreclaim_flag); 3962 psi_memstall_leave(&pflags); 3963 3964 /* 3965 * At least in one zone compaction wasn't deferred or skipped, so let's 3966 * count a compaction stall 3967 */ 3968 count_vm_event(COMPACTSTALL); 3969 3970 /* Prep a captured page if available */ 3971 if (page) 3972 prep_new_page(page, order, gfp_mask, alloc_flags); 3973 3974 /* Try get a page from the freelist if available */ 3975 if (!page) 3976 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 3977 3978 if (page) { 3979 struct zone *zone = page_zone(page); 3980 3981 zone->compact_blockskip_flush = false; 3982 compaction_defer_reset(zone, order, true); 3983 count_vm_event(COMPACTSUCCESS); 3984 return page; 3985 } 3986 3987 /* 3988 * It's bad if compaction run occurs and fails. The most likely reason 3989 * is that pages exist, but not enough to satisfy watermarks. 3990 */ 3991 count_vm_event(COMPACTFAIL); 3992 3993 cond_resched(); 3994 3995 return NULL; 3996 } 3997 3998 static inline bool 3999 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 4000 enum compact_result compact_result, 4001 enum compact_priority *compact_priority, 4002 int *compaction_retries) 4003 { 4004 int max_retries = MAX_COMPACT_RETRIES; 4005 int min_priority; 4006 bool ret = false; 4007 int retries = *compaction_retries; 4008 enum compact_priority priority = *compact_priority; 4009 4010 if (!order) 4011 return false; 4012 4013 if (compaction_made_progress(compact_result)) 4014 (*compaction_retries)++; 4015 4016 /* 4017 * compaction considers all the zone as desperately out of memory 4018 * so it doesn't really make much sense to retry except when the 4019 * failure could be caused by insufficient priority 4020 */ 4021 if (compaction_failed(compact_result)) 4022 goto check_priority; 4023 4024 /* 4025 * compaction was skipped because there are not enough order-0 pages 4026 * to work with, so we retry only if it looks like reclaim can help. 4027 */ 4028 if (compaction_needs_reclaim(compact_result)) { 4029 ret = compaction_zonelist_suitable(ac, order, alloc_flags); 4030 goto out; 4031 } 4032 4033 /* 4034 * make sure the compaction wasn't deferred or didn't bail out early 4035 * due to locks contention before we declare that we should give up. 4036 * But the next retry should use a higher priority if allowed, so 4037 * we don't just keep bailing out endlessly. 4038 */ 4039 if (compaction_withdrawn(compact_result)) { 4040 goto check_priority; 4041 } 4042 4043 /* 4044 * !costly requests are much more important than __GFP_RETRY_MAYFAIL 4045 * costly ones because they are de facto nofail and invoke OOM 4046 * killer to move on while costly can fail and users are ready 4047 * to cope with that. 1/4 retries is rather arbitrary but we 4048 * would need much more detailed feedback from compaction to 4049 * make a better decision. 4050 */ 4051 if (order > PAGE_ALLOC_COSTLY_ORDER) 4052 max_retries /= 4; 4053 if (*compaction_retries <= max_retries) { 4054 ret = true; 4055 goto out; 4056 } 4057 4058 /* 4059 * Make sure there are attempts at the highest priority if we exhausted 4060 * all retries or failed at the lower priorities. 4061 */ 4062 check_priority: 4063 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 4064 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 4065 4066 if (*compact_priority > min_priority) { 4067 (*compact_priority)--; 4068 *compaction_retries = 0; 4069 ret = true; 4070 } 4071 out: 4072 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret); 4073 return ret; 4074 } 4075 #else 4076 static inline struct page * 4077 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4078 unsigned int alloc_flags, const struct alloc_context *ac, 4079 enum compact_priority prio, enum compact_result *compact_result) 4080 { 4081 *compact_result = COMPACT_SKIPPED; 4082 return NULL; 4083 } 4084 4085 static inline bool 4086 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 4087 enum compact_result compact_result, 4088 enum compact_priority *compact_priority, 4089 int *compaction_retries) 4090 { 4091 struct zone *zone; 4092 struct zoneref *z; 4093 4094 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 4095 return false; 4096 4097 /* 4098 * There are setups with compaction disabled which would prefer to loop 4099 * inside the allocator rather than hit the oom killer prematurely. 4100 * Let's give them a good hope and keep retrying while the order-0 4101 * watermarks are OK. 4102 */ 4103 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 4104 ac->nodemask) { 4105 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 4106 ac_classzone_idx(ac), alloc_flags)) 4107 return true; 4108 } 4109 return false; 4110 } 4111 #endif /* CONFIG_COMPACTION */ 4112 4113 #ifdef CONFIG_LOCKDEP 4114 static struct lockdep_map __fs_reclaim_map = 4115 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map); 4116 4117 static bool __need_fs_reclaim(gfp_t gfp_mask) 4118 { 4119 gfp_mask = current_gfp_context(gfp_mask); 4120 4121 /* no reclaim without waiting on it */ 4122 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) 4123 return false; 4124 4125 /* this guy won't enter reclaim */ 4126 if (current->flags & PF_MEMALLOC) 4127 return false; 4128 4129 /* We're only interested __GFP_FS allocations for now */ 4130 if (!(gfp_mask & __GFP_FS)) 4131 return false; 4132 4133 if (gfp_mask & __GFP_NOLOCKDEP) 4134 return false; 4135 4136 return true; 4137 } 4138 4139 void __fs_reclaim_acquire(void) 4140 { 4141 lock_map_acquire(&__fs_reclaim_map); 4142 } 4143 4144 void __fs_reclaim_release(void) 4145 { 4146 lock_map_release(&__fs_reclaim_map); 4147 } 4148 4149 void fs_reclaim_acquire(gfp_t gfp_mask) 4150 { 4151 if (__need_fs_reclaim(gfp_mask)) 4152 __fs_reclaim_acquire(); 4153 } 4154 EXPORT_SYMBOL_GPL(fs_reclaim_acquire); 4155 4156 void fs_reclaim_release(gfp_t gfp_mask) 4157 { 4158 if (__need_fs_reclaim(gfp_mask)) 4159 __fs_reclaim_release(); 4160 } 4161 EXPORT_SYMBOL_GPL(fs_reclaim_release); 4162 #endif 4163 4164 /* Perform direct synchronous page reclaim */ 4165 static int 4166 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 4167 const struct alloc_context *ac) 4168 { 4169 int progress; 4170 unsigned int noreclaim_flag; 4171 unsigned long pflags; 4172 4173 cond_resched(); 4174 4175 /* We now go into synchronous reclaim */ 4176 cpuset_memory_pressure_bump(); 4177 psi_memstall_enter(&pflags); 4178 fs_reclaim_acquire(gfp_mask); 4179 noreclaim_flag = memalloc_noreclaim_save(); 4180 4181 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 4182 ac->nodemask); 4183 4184 memalloc_noreclaim_restore(noreclaim_flag); 4185 fs_reclaim_release(gfp_mask); 4186 psi_memstall_leave(&pflags); 4187 4188 cond_resched(); 4189 4190 return progress; 4191 } 4192 4193 /* The really slow allocator path where we enter direct reclaim */ 4194 static inline struct page * 4195 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 4196 unsigned int alloc_flags, const struct alloc_context *ac, 4197 unsigned long *did_some_progress) 4198 { 4199 struct page *page = NULL; 4200 bool drained = false; 4201 4202 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 4203 if (unlikely(!(*did_some_progress))) 4204 return NULL; 4205 4206 retry: 4207 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4208 4209 /* 4210 * If an allocation failed after direct reclaim, it could be because 4211 * pages are pinned on the per-cpu lists or in high alloc reserves. 4212 * Shrink them them and try again 4213 */ 4214 if (!page && !drained) { 4215 unreserve_highatomic_pageblock(ac, false); 4216 drain_all_pages(NULL); 4217 drained = true; 4218 goto retry; 4219 } 4220 4221 return page; 4222 } 4223 4224 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, 4225 const struct alloc_context *ac) 4226 { 4227 struct zoneref *z; 4228 struct zone *zone; 4229 pg_data_t *last_pgdat = NULL; 4230 enum zone_type high_zoneidx = ac->high_zoneidx; 4231 4232 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, high_zoneidx, 4233 ac->nodemask) { 4234 if (last_pgdat != zone->zone_pgdat) 4235 wakeup_kswapd(zone, gfp_mask, order, high_zoneidx); 4236 last_pgdat = zone->zone_pgdat; 4237 } 4238 } 4239 4240 static inline unsigned int 4241 gfp_to_alloc_flags(gfp_t gfp_mask) 4242 { 4243 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 4244 4245 /* 4246 * __GFP_HIGH is assumed to be the same as ALLOC_HIGH 4247 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 4248 * to save two branches. 4249 */ 4250 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 4251 BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); 4252 4253 /* 4254 * The caller may dip into page reserves a bit more if the caller 4255 * cannot run direct reclaim, or if the caller has realtime scheduling 4256 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 4257 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 4258 */ 4259 alloc_flags |= (__force int) 4260 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); 4261 4262 if (gfp_mask & __GFP_ATOMIC) { 4263 /* 4264 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 4265 * if it can't schedule. 4266 */ 4267 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4268 alloc_flags |= ALLOC_HARDER; 4269 /* 4270 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 4271 * comment for __cpuset_node_allowed(). 4272 */ 4273 alloc_flags &= ~ALLOC_CPUSET; 4274 } else if (unlikely(rt_task(current)) && !in_interrupt()) 4275 alloc_flags |= ALLOC_HARDER; 4276 4277 #ifdef CONFIG_CMA 4278 if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) 4279 alloc_flags |= ALLOC_CMA; 4280 #endif 4281 return alloc_flags; 4282 } 4283 4284 static bool oom_reserves_allowed(struct task_struct *tsk) 4285 { 4286 if (!tsk_is_oom_victim(tsk)) 4287 return false; 4288 4289 /* 4290 * !MMU doesn't have oom reaper so give access to memory reserves 4291 * only to the thread with TIF_MEMDIE set 4292 */ 4293 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE)) 4294 return false; 4295 4296 return true; 4297 } 4298 4299 /* 4300 * Distinguish requests which really need access to full memory 4301 * reserves from oom victims which can live with a portion of it 4302 */ 4303 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) 4304 { 4305 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 4306 return 0; 4307 if (gfp_mask & __GFP_MEMALLOC) 4308 return ALLOC_NO_WATERMARKS; 4309 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 4310 return ALLOC_NO_WATERMARKS; 4311 if (!in_interrupt()) { 4312 if (current->flags & PF_MEMALLOC) 4313 return ALLOC_NO_WATERMARKS; 4314 else if (oom_reserves_allowed(current)) 4315 return ALLOC_OOM; 4316 } 4317 4318 return 0; 4319 } 4320 4321 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 4322 { 4323 return !!__gfp_pfmemalloc_flags(gfp_mask); 4324 } 4325 4326 /* 4327 * Checks whether it makes sense to retry the reclaim to make a forward progress 4328 * for the given allocation request. 4329 * 4330 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row 4331 * without success, or when we couldn't even meet the watermark if we 4332 * reclaimed all remaining pages on the LRU lists. 4333 * 4334 * Returns true if a retry is viable or false to enter the oom path. 4335 */ 4336 static inline bool 4337 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 4338 struct alloc_context *ac, int alloc_flags, 4339 bool did_some_progress, int *no_progress_loops) 4340 { 4341 struct zone *zone; 4342 struct zoneref *z; 4343 bool ret = false; 4344 4345 /* 4346 * Costly allocations might have made a progress but this doesn't mean 4347 * their order will become available due to high fragmentation so 4348 * always increment the no progress counter for them 4349 */ 4350 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 4351 *no_progress_loops = 0; 4352 else 4353 (*no_progress_loops)++; 4354 4355 /* 4356 * Make sure we converge to OOM if we cannot make any progress 4357 * several times in the row. 4358 */ 4359 if (*no_progress_loops > MAX_RECLAIM_RETRIES) { 4360 /* Before OOM, exhaust highatomic_reserve */ 4361 return unreserve_highatomic_pageblock(ac, true); 4362 } 4363 4364 /* 4365 * Keep reclaiming pages while there is a chance this will lead 4366 * somewhere. If none of the target zones can satisfy our allocation 4367 * request even if all reclaimable pages are considered then we are 4368 * screwed and have to go OOM. 4369 */ 4370 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 4371 ac->nodemask) { 4372 unsigned long available; 4373 unsigned long reclaimable; 4374 unsigned long min_wmark = min_wmark_pages(zone); 4375 bool wmark; 4376 4377 available = reclaimable = zone_reclaimable_pages(zone); 4378 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 4379 4380 /* 4381 * Would the allocation succeed if we reclaimed all 4382 * reclaimable pages? 4383 */ 4384 wmark = __zone_watermark_ok(zone, order, min_wmark, 4385 ac_classzone_idx(ac), alloc_flags, available); 4386 trace_reclaim_retry_zone(z, order, reclaimable, 4387 available, min_wmark, *no_progress_loops, wmark); 4388 if (wmark) { 4389 /* 4390 * If we didn't make any progress and have a lot of 4391 * dirty + writeback pages then we should wait for 4392 * an IO to complete to slow down the reclaim and 4393 * prevent from pre mature OOM 4394 */ 4395 if (!did_some_progress) { 4396 unsigned long write_pending; 4397 4398 write_pending = zone_page_state_snapshot(zone, 4399 NR_ZONE_WRITE_PENDING); 4400 4401 if (2 * write_pending > reclaimable) { 4402 congestion_wait(BLK_RW_ASYNC, HZ/10); 4403 return true; 4404 } 4405 } 4406 4407 ret = true; 4408 goto out; 4409 } 4410 } 4411 4412 out: 4413 /* 4414 * Memory allocation/reclaim might be called from a WQ context and the 4415 * current implementation of the WQ concurrency control doesn't 4416 * recognize that a particular WQ is congested if the worker thread is 4417 * looping without ever sleeping. Therefore we have to do a short sleep 4418 * here rather than calling cond_resched(). 4419 */ 4420 if (current->flags & PF_WQ_WORKER) 4421 schedule_timeout_uninterruptible(1); 4422 else 4423 cond_resched(); 4424 return ret; 4425 } 4426 4427 static inline bool 4428 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) 4429 { 4430 /* 4431 * It's possible that cpuset's mems_allowed and the nodemask from 4432 * mempolicy don't intersect. This should be normally dealt with by 4433 * policy_nodemask(), but it's possible to race with cpuset update in 4434 * such a way the check therein was true, and then it became false 4435 * before we got our cpuset_mems_cookie here. 4436 * This assumes that for all allocations, ac->nodemask can come only 4437 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored 4438 * when it does not intersect with the cpuset restrictions) or the 4439 * caller can deal with a violated nodemask. 4440 */ 4441 if (cpusets_enabled() && ac->nodemask && 4442 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) { 4443 ac->nodemask = NULL; 4444 return true; 4445 } 4446 4447 /* 4448 * When updating a task's mems_allowed or mempolicy nodemask, it is 4449 * possible to race with parallel threads in such a way that our 4450 * allocation can fail while the mask is being updated. If we are about 4451 * to fail, check if the cpuset changed during allocation and if so, 4452 * retry. 4453 */ 4454 if (read_mems_allowed_retry(cpuset_mems_cookie)) 4455 return true; 4456 4457 return false; 4458 } 4459 4460 static inline struct page * 4461 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 4462 struct alloc_context *ac) 4463 { 4464 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 4465 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 4466 struct page *page = NULL; 4467 unsigned int alloc_flags; 4468 unsigned long did_some_progress; 4469 enum compact_priority compact_priority; 4470 enum compact_result compact_result; 4471 int compaction_retries; 4472 int no_progress_loops; 4473 unsigned int cpuset_mems_cookie; 4474 int reserve_flags; 4475 4476 /* 4477 * We also sanity check to catch abuse of atomic reserves being used by 4478 * callers that are not in atomic context. 4479 */ 4480 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 4481 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 4482 gfp_mask &= ~__GFP_ATOMIC; 4483 4484 retry_cpuset: 4485 compaction_retries = 0; 4486 no_progress_loops = 0; 4487 compact_priority = DEF_COMPACT_PRIORITY; 4488 cpuset_mems_cookie = read_mems_allowed_begin(); 4489 4490 /* 4491 * The fast path uses conservative alloc_flags to succeed only until 4492 * kswapd needs to be woken up, and to avoid the cost of setting up 4493 * alloc_flags precisely. So we do that now. 4494 */ 4495 alloc_flags = gfp_to_alloc_flags(gfp_mask); 4496 4497 /* 4498 * We need to recalculate the starting point for the zonelist iterator 4499 * because we might have used different nodemask in the fast path, or 4500 * there was a cpuset modification and we are retrying - otherwise we 4501 * could end up iterating over non-eligible zones endlessly. 4502 */ 4503 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4504 ac->high_zoneidx, ac->nodemask); 4505 if (!ac->preferred_zoneref->zone) 4506 goto nopage; 4507 4508 if (alloc_flags & ALLOC_KSWAPD) 4509 wake_all_kswapds(order, gfp_mask, ac); 4510 4511 /* 4512 * The adjusted alloc_flags might result in immediate success, so try 4513 * that first 4514 */ 4515 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4516 if (page) 4517 goto got_pg; 4518 4519 /* 4520 * For costly allocations, try direct compaction first, as it's likely 4521 * that we have enough base pages and don't need to reclaim. For non- 4522 * movable high-order allocations, do that as well, as compaction will 4523 * try prevent permanent fragmentation by migrating from blocks of the 4524 * same migratetype. 4525 * Don't try this for allocations that are allowed to ignore 4526 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 4527 */ 4528 if (can_direct_reclaim && 4529 (costly_order || 4530 (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 4531 && !gfp_pfmemalloc_allowed(gfp_mask)) { 4532 page = __alloc_pages_direct_compact(gfp_mask, order, 4533 alloc_flags, ac, 4534 INIT_COMPACT_PRIORITY, 4535 &compact_result); 4536 if (page) 4537 goto got_pg; 4538 4539 /* 4540 * Checks for costly allocations with __GFP_NORETRY, which 4541 * includes some THP page fault allocations 4542 */ 4543 if (costly_order && (gfp_mask & __GFP_NORETRY)) { 4544 /* 4545 * If allocating entire pageblock(s) and compaction 4546 * failed because all zones are below low watermarks 4547 * or is prohibited because it recently failed at this 4548 * order, fail immediately unless the allocator has 4549 * requested compaction and reclaim retry. 4550 * 4551 * Reclaim is 4552 * - potentially very expensive because zones are far 4553 * below their low watermarks or this is part of very 4554 * bursty high order allocations, 4555 * - not guaranteed to help because isolate_freepages() 4556 * may not iterate over freed pages as part of its 4557 * linear scan, and 4558 * - unlikely to make entire pageblocks free on its 4559 * own. 4560 */ 4561 if (compact_result == COMPACT_SKIPPED || 4562 compact_result == COMPACT_DEFERRED) 4563 goto nopage; 4564 4565 /* 4566 * Looks like reclaim/compaction is worth trying, but 4567 * sync compaction could be very expensive, so keep 4568 * using async compaction. 4569 */ 4570 compact_priority = INIT_COMPACT_PRIORITY; 4571 } 4572 } 4573 4574 retry: 4575 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 4576 if (alloc_flags & ALLOC_KSWAPD) 4577 wake_all_kswapds(order, gfp_mask, ac); 4578 4579 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); 4580 if (reserve_flags) 4581 alloc_flags = reserve_flags; 4582 4583 /* 4584 * Reset the nodemask and zonelist iterators if memory policies can be 4585 * ignored. These allocations are high priority and system rather than 4586 * user oriented. 4587 */ 4588 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { 4589 ac->nodemask = NULL; 4590 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4591 ac->high_zoneidx, ac->nodemask); 4592 } 4593 4594 /* Attempt with potentially adjusted zonelist and alloc_flags */ 4595 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4596 if (page) 4597 goto got_pg; 4598 4599 /* Caller is not willing to reclaim, we can't balance anything */ 4600 if (!can_direct_reclaim) 4601 goto nopage; 4602 4603 /* Avoid recursion of direct reclaim */ 4604 if (current->flags & PF_MEMALLOC) 4605 goto nopage; 4606 4607 /* Try direct reclaim and then allocating */ 4608 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 4609 &did_some_progress); 4610 if (page) 4611 goto got_pg; 4612 4613 /* Try direct compaction and then allocating */ 4614 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 4615 compact_priority, &compact_result); 4616 if (page) 4617 goto got_pg; 4618 4619 /* Do not loop if specifically requested */ 4620 if (gfp_mask & __GFP_NORETRY) 4621 goto nopage; 4622 4623 /* 4624 * Do not retry costly high order allocations unless they are 4625 * __GFP_RETRY_MAYFAIL 4626 */ 4627 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) 4628 goto nopage; 4629 4630 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 4631 did_some_progress > 0, &no_progress_loops)) 4632 goto retry; 4633 4634 /* 4635 * It doesn't make any sense to retry for the compaction if the order-0 4636 * reclaim is not able to make any progress because the current 4637 * implementation of the compaction depends on the sufficient amount 4638 * of free memory (see __compaction_suitable) 4639 */ 4640 if (did_some_progress > 0 && 4641 should_compact_retry(ac, order, alloc_flags, 4642 compact_result, &compact_priority, 4643 &compaction_retries)) 4644 goto retry; 4645 4646 4647 /* Deal with possible cpuset update races before we start OOM killing */ 4648 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4649 goto retry_cpuset; 4650 4651 /* Reclaim has failed us, start killing things */ 4652 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 4653 if (page) 4654 goto got_pg; 4655 4656 /* Avoid allocations with no watermarks from looping endlessly */ 4657 if (tsk_is_oom_victim(current) && 4658 (alloc_flags == ALLOC_OOM || 4659 (gfp_mask & __GFP_NOMEMALLOC))) 4660 goto nopage; 4661 4662 /* Retry as long as the OOM killer is making progress */ 4663 if (did_some_progress) { 4664 no_progress_loops = 0; 4665 goto retry; 4666 } 4667 4668 nopage: 4669 /* Deal with possible cpuset update races before we fail */ 4670 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4671 goto retry_cpuset; 4672 4673 /* 4674 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure 4675 * we always retry 4676 */ 4677 if (gfp_mask & __GFP_NOFAIL) { 4678 /* 4679 * All existing users of the __GFP_NOFAIL are blockable, so warn 4680 * of any new users that actually require GFP_NOWAIT 4681 */ 4682 if (WARN_ON_ONCE(!can_direct_reclaim)) 4683 goto fail; 4684 4685 /* 4686 * PF_MEMALLOC request from this context is rather bizarre 4687 * because we cannot reclaim anything and only can loop waiting 4688 * for somebody to do a work for us 4689 */ 4690 WARN_ON_ONCE(current->flags & PF_MEMALLOC); 4691 4692 /* 4693 * non failing costly orders are a hard requirement which we 4694 * are not prepared for much so let's warn about these users 4695 * so that we can identify them and convert them to something 4696 * else. 4697 */ 4698 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER); 4699 4700 /* 4701 * Help non-failing allocations by giving them access to memory 4702 * reserves but do not use ALLOC_NO_WATERMARKS because this 4703 * could deplete whole memory reserves which would just make 4704 * the situation worse 4705 */ 4706 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); 4707 if (page) 4708 goto got_pg; 4709 4710 cond_resched(); 4711 goto retry; 4712 } 4713 fail: 4714 warn_alloc(gfp_mask, ac->nodemask, 4715 "page allocation failure: order:%u", order); 4716 got_pg: 4717 return page; 4718 } 4719 4720 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, 4721 int preferred_nid, nodemask_t *nodemask, 4722 struct alloc_context *ac, gfp_t *alloc_mask, 4723 unsigned int *alloc_flags) 4724 { 4725 ac->high_zoneidx = gfp_zone(gfp_mask); 4726 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); 4727 ac->nodemask = nodemask; 4728 ac->migratetype = gfpflags_to_migratetype(gfp_mask); 4729 4730 if (cpusets_enabled()) { 4731 *alloc_mask |= __GFP_HARDWALL; 4732 if (!ac->nodemask) 4733 ac->nodemask = &cpuset_current_mems_allowed; 4734 else 4735 *alloc_flags |= ALLOC_CPUSET; 4736 } 4737 4738 fs_reclaim_acquire(gfp_mask); 4739 fs_reclaim_release(gfp_mask); 4740 4741 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 4742 4743 if (should_fail_alloc_page(gfp_mask, order)) 4744 return false; 4745 4746 if (IS_ENABLED(CONFIG_CMA) && ac->migratetype == MIGRATE_MOVABLE) 4747 *alloc_flags |= ALLOC_CMA; 4748 4749 return true; 4750 } 4751 4752 /* Determine whether to spread dirty pages and what the first usable zone */ 4753 static inline void finalise_ac(gfp_t gfp_mask, struct alloc_context *ac) 4754 { 4755 /* Dirty zone balancing only done in the fast path */ 4756 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); 4757 4758 /* 4759 * The preferred zone is used for statistics but crucially it is 4760 * also used as the starting point for the zonelist iterator. It 4761 * may get reset for allocations that ignore memory policies. 4762 */ 4763 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4764 ac->high_zoneidx, ac->nodemask); 4765 } 4766 4767 /* 4768 * This is the 'heart' of the zoned buddy allocator. 4769 */ 4770 struct page * 4771 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, 4772 nodemask_t *nodemask) 4773 { 4774 struct page *page; 4775 unsigned int alloc_flags = ALLOC_WMARK_LOW; 4776 gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */ 4777 struct alloc_context ac = { }; 4778 4779 /* 4780 * There are several places where we assume that the order value is sane 4781 * so bail out early if the request is out of bound. 4782 */ 4783 if (unlikely(order >= MAX_ORDER)) { 4784 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); 4785 return NULL; 4786 } 4787 4788 gfp_mask &= gfp_allowed_mask; 4789 alloc_mask = gfp_mask; 4790 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags)) 4791 return NULL; 4792 4793 finalise_ac(gfp_mask, &ac); 4794 4795 /* 4796 * Forbid the first pass from falling back to types that fragment 4797 * memory until all local zones are considered. 4798 */ 4799 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask); 4800 4801 /* First allocation attempt */ 4802 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); 4803 if (likely(page)) 4804 goto out; 4805 4806 /* 4807 * Apply scoped allocation constraints. This is mainly about GFP_NOFS 4808 * resp. GFP_NOIO which has to be inherited for all allocation requests 4809 * from a particular context which has been marked by 4810 * memalloc_no{fs,io}_{save,restore}. 4811 */ 4812 alloc_mask = current_gfp_context(gfp_mask); 4813 ac.spread_dirty_pages = false; 4814 4815 /* 4816 * Restore the original nodemask if it was potentially replaced with 4817 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 4818 */ 4819 ac.nodemask = nodemask; 4820 4821 page = __alloc_pages_slowpath(alloc_mask, order, &ac); 4822 4823 out: 4824 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && 4825 unlikely(__memcg_kmem_charge_page(page, gfp_mask, order) != 0)) { 4826 __free_pages(page, order); 4827 page = NULL; 4828 } 4829 4830 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); 4831 4832 return page; 4833 } 4834 EXPORT_SYMBOL(__alloc_pages_nodemask); 4835 4836 /* 4837 * Common helper functions. Never use with __GFP_HIGHMEM because the returned 4838 * address cannot represent highmem pages. Use alloc_pages and then kmap if 4839 * you need to access high mem. 4840 */ 4841 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 4842 { 4843 struct page *page; 4844 4845 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); 4846 if (!page) 4847 return 0; 4848 return (unsigned long) page_address(page); 4849 } 4850 EXPORT_SYMBOL(__get_free_pages); 4851 4852 unsigned long get_zeroed_page(gfp_t gfp_mask) 4853 { 4854 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 4855 } 4856 EXPORT_SYMBOL(get_zeroed_page); 4857 4858 static inline void free_the_page(struct page *page, unsigned int order) 4859 { 4860 if (order == 0) /* Via pcp? */ 4861 free_unref_page(page); 4862 else 4863 __free_pages_ok(page, order); 4864 } 4865 4866 void __free_pages(struct page *page, unsigned int order) 4867 { 4868 if (put_page_testzero(page)) 4869 free_the_page(page, order); 4870 } 4871 EXPORT_SYMBOL(__free_pages); 4872 4873 void free_pages(unsigned long addr, unsigned int order) 4874 { 4875 if (addr != 0) { 4876 VM_BUG_ON(!virt_addr_valid((void *)addr)); 4877 __free_pages(virt_to_page((void *)addr), order); 4878 } 4879 } 4880 4881 EXPORT_SYMBOL(free_pages); 4882 4883 /* 4884 * Page Fragment: 4885 * An arbitrary-length arbitrary-offset area of memory which resides 4886 * within a 0 or higher order page. Multiple fragments within that page 4887 * are individually refcounted, in the page's reference counter. 4888 * 4889 * The page_frag functions below provide a simple allocation framework for 4890 * page fragments. This is used by the network stack and network device 4891 * drivers to provide a backing region of memory for use as either an 4892 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 4893 */ 4894 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 4895 gfp_t gfp_mask) 4896 { 4897 struct page *page = NULL; 4898 gfp_t gfp = gfp_mask; 4899 4900 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4901 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 4902 __GFP_NOMEMALLOC; 4903 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 4904 PAGE_FRAG_CACHE_MAX_ORDER); 4905 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 4906 #endif 4907 if (unlikely(!page)) 4908 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 4909 4910 nc->va = page ? page_address(page) : NULL; 4911 4912 return page; 4913 } 4914 4915 void __page_frag_cache_drain(struct page *page, unsigned int count) 4916 { 4917 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 4918 4919 if (page_ref_sub_and_test(page, count)) 4920 free_the_page(page, compound_order(page)); 4921 } 4922 EXPORT_SYMBOL(__page_frag_cache_drain); 4923 4924 void *page_frag_alloc(struct page_frag_cache *nc, 4925 unsigned int fragsz, gfp_t gfp_mask) 4926 { 4927 unsigned int size = PAGE_SIZE; 4928 struct page *page; 4929 int offset; 4930 4931 if (unlikely(!nc->va)) { 4932 refill: 4933 page = __page_frag_cache_refill(nc, gfp_mask); 4934 if (!page) 4935 return NULL; 4936 4937 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4938 /* if size can vary use size else just use PAGE_SIZE */ 4939 size = nc->size; 4940 #endif 4941 /* Even if we own the page, we do not use atomic_set(). 4942 * This would break get_page_unless_zero() users. 4943 */ 4944 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 4945 4946 /* reset page count bias and offset to start of new frag */ 4947 nc->pfmemalloc = page_is_pfmemalloc(page); 4948 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 4949 nc->offset = size; 4950 } 4951 4952 offset = nc->offset - fragsz; 4953 if (unlikely(offset < 0)) { 4954 page = virt_to_page(nc->va); 4955 4956 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 4957 goto refill; 4958 4959 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4960 /* if size can vary use size else just use PAGE_SIZE */ 4961 size = nc->size; 4962 #endif 4963 /* OK, page count is 0, we can safely set it */ 4964 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 4965 4966 /* reset page count bias and offset to start of new frag */ 4967 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 4968 offset = size - fragsz; 4969 } 4970 4971 nc->pagecnt_bias--; 4972 nc->offset = offset; 4973 4974 return nc->va + offset; 4975 } 4976 EXPORT_SYMBOL(page_frag_alloc); 4977 4978 /* 4979 * Frees a page fragment allocated out of either a compound or order 0 page. 4980 */ 4981 void page_frag_free(void *addr) 4982 { 4983 struct page *page = virt_to_head_page(addr); 4984 4985 if (unlikely(put_page_testzero(page))) 4986 free_the_page(page, compound_order(page)); 4987 } 4988 EXPORT_SYMBOL(page_frag_free); 4989 4990 static void *make_alloc_exact(unsigned long addr, unsigned int order, 4991 size_t size) 4992 { 4993 if (addr) { 4994 unsigned long alloc_end = addr + (PAGE_SIZE << order); 4995 unsigned long used = addr + PAGE_ALIGN(size); 4996 4997 split_page(virt_to_page((void *)addr), order); 4998 while (used < alloc_end) { 4999 free_page(used); 5000 used += PAGE_SIZE; 5001 } 5002 } 5003 return (void *)addr; 5004 } 5005 5006 /** 5007 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 5008 * @size: the number of bytes to allocate 5009 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5010 * 5011 * This function is similar to alloc_pages(), except that it allocates the 5012 * minimum number of pages to satisfy the request. alloc_pages() can only 5013 * allocate memory in power-of-two pages. 5014 * 5015 * This function is also limited by MAX_ORDER. 5016 * 5017 * Memory allocated by this function must be released by free_pages_exact(). 5018 * 5019 * Return: pointer to the allocated area or %NULL in case of error. 5020 */ 5021 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 5022 { 5023 unsigned int order = get_order(size); 5024 unsigned long addr; 5025 5026 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 5027 gfp_mask &= ~__GFP_COMP; 5028 5029 addr = __get_free_pages(gfp_mask, order); 5030 return make_alloc_exact(addr, order, size); 5031 } 5032 EXPORT_SYMBOL(alloc_pages_exact); 5033 5034 /** 5035 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 5036 * pages on a node. 5037 * @nid: the preferred node ID where memory should be allocated 5038 * @size: the number of bytes to allocate 5039 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5040 * 5041 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 5042 * back. 5043 * 5044 * Return: pointer to the allocated area or %NULL in case of error. 5045 */ 5046 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 5047 { 5048 unsigned int order = get_order(size); 5049 struct page *p; 5050 5051 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 5052 gfp_mask &= ~__GFP_COMP; 5053 5054 p = alloc_pages_node(nid, gfp_mask, order); 5055 if (!p) 5056 return NULL; 5057 return make_alloc_exact((unsigned long)page_address(p), order, size); 5058 } 5059 5060 /** 5061 * free_pages_exact - release memory allocated via alloc_pages_exact() 5062 * @virt: the value returned by alloc_pages_exact. 5063 * @size: size of allocation, same value as passed to alloc_pages_exact(). 5064 * 5065 * Release the memory allocated by a previous call to alloc_pages_exact. 5066 */ 5067 void free_pages_exact(void *virt, size_t size) 5068 { 5069 unsigned long addr = (unsigned long)virt; 5070 unsigned long end = addr + PAGE_ALIGN(size); 5071 5072 while (addr < end) { 5073 free_page(addr); 5074 addr += PAGE_SIZE; 5075 } 5076 } 5077 EXPORT_SYMBOL(free_pages_exact); 5078 5079 /** 5080 * nr_free_zone_pages - count number of pages beyond high watermark 5081 * @offset: The zone index of the highest zone 5082 * 5083 * nr_free_zone_pages() counts the number of pages which are beyond the 5084 * high watermark within all zones at or below a given zone index. For each 5085 * zone, the number of pages is calculated as: 5086 * 5087 * nr_free_zone_pages = managed_pages - high_pages 5088 * 5089 * Return: number of pages beyond high watermark. 5090 */ 5091 static unsigned long nr_free_zone_pages(int offset) 5092 { 5093 struct zoneref *z; 5094 struct zone *zone; 5095 5096 /* Just pick one node, since fallback list is circular */ 5097 unsigned long sum = 0; 5098 5099 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 5100 5101 for_each_zone_zonelist(zone, z, zonelist, offset) { 5102 unsigned long size = zone_managed_pages(zone); 5103 unsigned long high = high_wmark_pages(zone); 5104 if (size > high) 5105 sum += size - high; 5106 } 5107 5108 return sum; 5109 } 5110 5111 /** 5112 * nr_free_buffer_pages - count number of pages beyond high watermark 5113 * 5114 * nr_free_buffer_pages() counts the number of pages which are beyond the high 5115 * watermark within ZONE_DMA and ZONE_NORMAL. 5116 * 5117 * Return: number of pages beyond high watermark within ZONE_DMA and 5118 * ZONE_NORMAL. 5119 */ 5120 unsigned long nr_free_buffer_pages(void) 5121 { 5122 return nr_free_zone_pages(gfp_zone(GFP_USER)); 5123 } 5124 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 5125 5126 /** 5127 * nr_free_pagecache_pages - count number of pages beyond high watermark 5128 * 5129 * nr_free_pagecache_pages() counts the number of pages which are beyond the 5130 * high watermark within all zones. 5131 * 5132 * Return: number of pages beyond high watermark within all zones. 5133 */ 5134 unsigned long nr_free_pagecache_pages(void) 5135 { 5136 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 5137 } 5138 5139 static inline void show_node(struct zone *zone) 5140 { 5141 if (IS_ENABLED(CONFIG_NUMA)) 5142 printk("Node %d ", zone_to_nid(zone)); 5143 } 5144 5145 long si_mem_available(void) 5146 { 5147 long available; 5148 unsigned long pagecache; 5149 unsigned long wmark_low = 0; 5150 unsigned long pages[NR_LRU_LISTS]; 5151 unsigned long reclaimable; 5152 struct zone *zone; 5153 int lru; 5154 5155 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 5156 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 5157 5158 for_each_zone(zone) 5159 wmark_low += low_wmark_pages(zone); 5160 5161 /* 5162 * Estimate the amount of memory available for userspace allocations, 5163 * without causing swapping. 5164 */ 5165 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages; 5166 5167 /* 5168 * Not all the page cache can be freed, otherwise the system will 5169 * start swapping. Assume at least half of the page cache, or the 5170 * low watermark worth of cache, needs to stay. 5171 */ 5172 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 5173 pagecache -= min(pagecache / 2, wmark_low); 5174 available += pagecache; 5175 5176 /* 5177 * Part of the reclaimable slab and other kernel memory consists of 5178 * items that are in use, and cannot be freed. Cap this estimate at the 5179 * low watermark. 5180 */ 5181 reclaimable = global_node_page_state(NR_SLAB_RECLAIMABLE) + 5182 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE); 5183 available += reclaimable - min(reclaimable / 2, wmark_low); 5184 5185 if (available < 0) 5186 available = 0; 5187 return available; 5188 } 5189 EXPORT_SYMBOL_GPL(si_mem_available); 5190 5191 void si_meminfo(struct sysinfo *val) 5192 { 5193 val->totalram = totalram_pages(); 5194 val->sharedram = global_node_page_state(NR_SHMEM); 5195 val->freeram = global_zone_page_state(NR_FREE_PAGES); 5196 val->bufferram = nr_blockdev_pages(); 5197 val->totalhigh = totalhigh_pages(); 5198 val->freehigh = nr_free_highpages(); 5199 val->mem_unit = PAGE_SIZE; 5200 } 5201 5202 EXPORT_SYMBOL(si_meminfo); 5203 5204 #ifdef CONFIG_NUMA 5205 void si_meminfo_node(struct sysinfo *val, int nid) 5206 { 5207 int zone_type; /* needs to be signed */ 5208 unsigned long managed_pages = 0; 5209 unsigned long managed_highpages = 0; 5210 unsigned long free_highpages = 0; 5211 pg_data_t *pgdat = NODE_DATA(nid); 5212 5213 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 5214 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]); 5215 val->totalram = managed_pages; 5216 val->sharedram = node_page_state(pgdat, NR_SHMEM); 5217 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 5218 #ifdef CONFIG_HIGHMEM 5219 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 5220 struct zone *zone = &pgdat->node_zones[zone_type]; 5221 5222 if (is_highmem(zone)) { 5223 managed_highpages += zone_managed_pages(zone); 5224 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 5225 } 5226 } 5227 val->totalhigh = managed_highpages; 5228 val->freehigh = free_highpages; 5229 #else 5230 val->totalhigh = managed_highpages; 5231 val->freehigh = free_highpages; 5232 #endif 5233 val->mem_unit = PAGE_SIZE; 5234 } 5235 #endif 5236 5237 /* 5238 * Determine whether the node should be displayed or not, depending on whether 5239 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 5240 */ 5241 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) 5242 { 5243 if (!(flags & SHOW_MEM_FILTER_NODES)) 5244 return false; 5245 5246 /* 5247 * no node mask - aka implicit memory numa policy. Do not bother with 5248 * the synchronization - read_mems_allowed_begin - because we do not 5249 * have to be precise here. 5250 */ 5251 if (!nodemask) 5252 nodemask = &cpuset_current_mems_allowed; 5253 5254 return !node_isset(nid, *nodemask); 5255 } 5256 5257 #define K(x) ((x) << (PAGE_SHIFT-10)) 5258 5259 static void show_migration_types(unsigned char type) 5260 { 5261 static const char types[MIGRATE_TYPES] = { 5262 [MIGRATE_UNMOVABLE] = 'U', 5263 [MIGRATE_MOVABLE] = 'M', 5264 [MIGRATE_RECLAIMABLE] = 'E', 5265 [MIGRATE_HIGHATOMIC] = 'H', 5266 #ifdef CONFIG_CMA 5267 [MIGRATE_CMA] = 'C', 5268 #endif 5269 #ifdef CONFIG_MEMORY_ISOLATION 5270 [MIGRATE_ISOLATE] = 'I', 5271 #endif 5272 }; 5273 char tmp[MIGRATE_TYPES + 1]; 5274 char *p = tmp; 5275 int i; 5276 5277 for (i = 0; i < MIGRATE_TYPES; i++) { 5278 if (type & (1 << i)) 5279 *p++ = types[i]; 5280 } 5281 5282 *p = '\0'; 5283 printk(KERN_CONT "(%s) ", tmp); 5284 } 5285 5286 /* 5287 * Show free area list (used inside shift_scroll-lock stuff) 5288 * We also calculate the percentage fragmentation. We do this by counting the 5289 * memory on each free list with the exception of the first item on the list. 5290 * 5291 * Bits in @filter: 5292 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 5293 * cpuset. 5294 */ 5295 void show_free_areas(unsigned int filter, nodemask_t *nodemask) 5296 { 5297 unsigned long free_pcp = 0; 5298 int cpu; 5299 struct zone *zone; 5300 pg_data_t *pgdat; 5301 5302 for_each_populated_zone(zone) { 5303 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5304 continue; 5305 5306 for_each_online_cpu(cpu) 5307 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 5308 } 5309 5310 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 5311 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 5312 " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n" 5313 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 5314 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 5315 " free:%lu free_pcp:%lu free_cma:%lu\n", 5316 global_node_page_state(NR_ACTIVE_ANON), 5317 global_node_page_state(NR_INACTIVE_ANON), 5318 global_node_page_state(NR_ISOLATED_ANON), 5319 global_node_page_state(NR_ACTIVE_FILE), 5320 global_node_page_state(NR_INACTIVE_FILE), 5321 global_node_page_state(NR_ISOLATED_FILE), 5322 global_node_page_state(NR_UNEVICTABLE), 5323 global_node_page_state(NR_FILE_DIRTY), 5324 global_node_page_state(NR_WRITEBACK), 5325 global_node_page_state(NR_UNSTABLE_NFS), 5326 global_node_page_state(NR_SLAB_RECLAIMABLE), 5327 global_node_page_state(NR_SLAB_UNRECLAIMABLE), 5328 global_node_page_state(NR_FILE_MAPPED), 5329 global_node_page_state(NR_SHMEM), 5330 global_zone_page_state(NR_PAGETABLE), 5331 global_zone_page_state(NR_BOUNCE), 5332 global_zone_page_state(NR_FREE_PAGES), 5333 free_pcp, 5334 global_zone_page_state(NR_FREE_CMA_PAGES)); 5335 5336 for_each_online_pgdat(pgdat) { 5337 if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) 5338 continue; 5339 5340 printk("Node %d" 5341 " active_anon:%lukB" 5342 " inactive_anon:%lukB" 5343 " active_file:%lukB" 5344 " inactive_file:%lukB" 5345 " unevictable:%lukB" 5346 " isolated(anon):%lukB" 5347 " isolated(file):%lukB" 5348 " mapped:%lukB" 5349 " dirty:%lukB" 5350 " writeback:%lukB" 5351 " shmem:%lukB" 5352 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5353 " shmem_thp: %lukB" 5354 " shmem_pmdmapped: %lukB" 5355 " anon_thp: %lukB" 5356 #endif 5357 " writeback_tmp:%lukB" 5358 " unstable:%lukB" 5359 " all_unreclaimable? %s" 5360 "\n", 5361 pgdat->node_id, 5362 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 5363 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 5364 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 5365 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 5366 K(node_page_state(pgdat, NR_UNEVICTABLE)), 5367 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 5368 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 5369 K(node_page_state(pgdat, NR_FILE_MAPPED)), 5370 K(node_page_state(pgdat, NR_FILE_DIRTY)), 5371 K(node_page_state(pgdat, NR_WRITEBACK)), 5372 K(node_page_state(pgdat, NR_SHMEM)), 5373 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5374 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), 5375 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) 5376 * HPAGE_PMD_NR), 5377 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR), 5378 #endif 5379 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 5380 K(node_page_state(pgdat, NR_UNSTABLE_NFS)), 5381 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ? 5382 "yes" : "no"); 5383 } 5384 5385 for_each_populated_zone(zone) { 5386 int i; 5387 5388 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5389 continue; 5390 5391 free_pcp = 0; 5392 for_each_online_cpu(cpu) 5393 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 5394 5395 show_node(zone); 5396 printk(KERN_CONT 5397 "%s" 5398 " free:%lukB" 5399 " min:%lukB" 5400 " low:%lukB" 5401 " high:%lukB" 5402 " reserved_highatomic:%luKB" 5403 " active_anon:%lukB" 5404 " inactive_anon:%lukB" 5405 " active_file:%lukB" 5406 " inactive_file:%lukB" 5407 " unevictable:%lukB" 5408 " writepending:%lukB" 5409 " present:%lukB" 5410 " managed:%lukB" 5411 " mlocked:%lukB" 5412 " kernel_stack:%lukB" 5413 " pagetables:%lukB" 5414 " bounce:%lukB" 5415 " free_pcp:%lukB" 5416 " local_pcp:%ukB" 5417 " free_cma:%lukB" 5418 "\n", 5419 zone->name, 5420 K(zone_page_state(zone, NR_FREE_PAGES)), 5421 K(min_wmark_pages(zone)), 5422 K(low_wmark_pages(zone)), 5423 K(high_wmark_pages(zone)), 5424 K(zone->nr_reserved_highatomic), 5425 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 5426 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 5427 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 5428 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 5429 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 5430 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 5431 K(zone->present_pages), 5432 K(zone_managed_pages(zone)), 5433 K(zone_page_state(zone, NR_MLOCK)), 5434 zone_page_state(zone, NR_KERNEL_STACK_KB), 5435 K(zone_page_state(zone, NR_PAGETABLE)), 5436 K(zone_page_state(zone, NR_BOUNCE)), 5437 K(free_pcp), 5438 K(this_cpu_read(zone->pageset->pcp.count)), 5439 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 5440 printk("lowmem_reserve[]:"); 5441 for (i = 0; i < MAX_NR_ZONES; i++) 5442 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); 5443 printk(KERN_CONT "\n"); 5444 } 5445 5446 for_each_populated_zone(zone) { 5447 unsigned int order; 5448 unsigned long nr[MAX_ORDER], flags, total = 0; 5449 unsigned char types[MAX_ORDER]; 5450 5451 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5452 continue; 5453 show_node(zone); 5454 printk(KERN_CONT "%s: ", zone->name); 5455 5456 spin_lock_irqsave(&zone->lock, flags); 5457 for (order = 0; order < MAX_ORDER; order++) { 5458 struct free_area *area = &zone->free_area[order]; 5459 int type; 5460 5461 nr[order] = area->nr_free; 5462 total += nr[order] << order; 5463 5464 types[order] = 0; 5465 for (type = 0; type < MIGRATE_TYPES; type++) { 5466 if (!free_area_empty(area, type)) 5467 types[order] |= 1 << type; 5468 } 5469 } 5470 spin_unlock_irqrestore(&zone->lock, flags); 5471 for (order = 0; order < MAX_ORDER; order++) { 5472 printk(KERN_CONT "%lu*%lukB ", 5473 nr[order], K(1UL) << order); 5474 if (nr[order]) 5475 show_migration_types(types[order]); 5476 } 5477 printk(KERN_CONT "= %lukB\n", K(total)); 5478 } 5479 5480 hugetlb_show_meminfo(); 5481 5482 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 5483 5484 show_swap_cache_info(); 5485 } 5486 5487 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 5488 { 5489 zoneref->zone = zone; 5490 zoneref->zone_idx = zone_idx(zone); 5491 } 5492 5493 /* 5494 * Builds allocation fallback zone lists. 5495 * 5496 * Add all populated zones of a node to the zonelist. 5497 */ 5498 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) 5499 { 5500 struct zone *zone; 5501 enum zone_type zone_type = MAX_NR_ZONES; 5502 int nr_zones = 0; 5503 5504 do { 5505 zone_type--; 5506 zone = pgdat->node_zones + zone_type; 5507 if (managed_zone(zone)) { 5508 zoneref_set_zone(zone, &zonerefs[nr_zones++]); 5509 check_highest_zone(zone_type); 5510 } 5511 } while (zone_type); 5512 5513 return nr_zones; 5514 } 5515 5516 #ifdef CONFIG_NUMA 5517 5518 static int __parse_numa_zonelist_order(char *s) 5519 { 5520 /* 5521 * We used to support different zonlists modes but they turned 5522 * out to be just not useful. Let's keep the warning in place 5523 * if somebody still use the cmd line parameter so that we do 5524 * not fail it silently 5525 */ 5526 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) { 5527 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s); 5528 return -EINVAL; 5529 } 5530 return 0; 5531 } 5532 5533 static __init int setup_numa_zonelist_order(char *s) 5534 { 5535 if (!s) 5536 return 0; 5537 5538 return __parse_numa_zonelist_order(s); 5539 } 5540 early_param("numa_zonelist_order", setup_numa_zonelist_order); 5541 5542 char numa_zonelist_order[] = "Node"; 5543 5544 /* 5545 * sysctl handler for numa_zonelist_order 5546 */ 5547 int numa_zonelist_order_handler(struct ctl_table *table, int write, 5548 void __user *buffer, size_t *length, 5549 loff_t *ppos) 5550 { 5551 char *str; 5552 int ret; 5553 5554 if (!write) 5555 return proc_dostring(table, write, buffer, length, ppos); 5556 str = memdup_user_nul(buffer, 16); 5557 if (IS_ERR(str)) 5558 return PTR_ERR(str); 5559 5560 ret = __parse_numa_zonelist_order(str); 5561 kfree(str); 5562 return ret; 5563 } 5564 5565 5566 #define MAX_NODE_LOAD (nr_online_nodes) 5567 static int node_load[MAX_NUMNODES]; 5568 5569 /** 5570 * find_next_best_node - find the next node that should appear in a given node's fallback list 5571 * @node: node whose fallback list we're appending 5572 * @used_node_mask: nodemask_t of already used nodes 5573 * 5574 * We use a number of factors to determine which is the next node that should 5575 * appear on a given node's fallback list. The node should not have appeared 5576 * already in @node's fallback list, and it should be the next closest node 5577 * according to the distance array (which contains arbitrary distance values 5578 * from each node to each node in the system), and should also prefer nodes 5579 * with no CPUs, since presumably they'll have very little allocation pressure 5580 * on them otherwise. 5581 * 5582 * Return: node id of the found node or %NUMA_NO_NODE if no node is found. 5583 */ 5584 static int find_next_best_node(int node, nodemask_t *used_node_mask) 5585 { 5586 int n, val; 5587 int min_val = INT_MAX; 5588 int best_node = NUMA_NO_NODE; 5589 const struct cpumask *tmp = cpumask_of_node(0); 5590 5591 /* Use the local node if we haven't already */ 5592 if (!node_isset(node, *used_node_mask)) { 5593 node_set(node, *used_node_mask); 5594 return node; 5595 } 5596 5597 for_each_node_state(n, N_MEMORY) { 5598 5599 /* Don't want a node to appear more than once */ 5600 if (node_isset(n, *used_node_mask)) 5601 continue; 5602 5603 /* Use the distance array to find the distance */ 5604 val = node_distance(node, n); 5605 5606 /* Penalize nodes under us ("prefer the next node") */ 5607 val += (n < node); 5608 5609 /* Give preference to headless and unused nodes */ 5610 tmp = cpumask_of_node(n); 5611 if (!cpumask_empty(tmp)) 5612 val += PENALTY_FOR_NODE_WITH_CPUS; 5613 5614 /* Slight preference for less loaded node */ 5615 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 5616 val += node_load[n]; 5617 5618 if (val < min_val) { 5619 min_val = val; 5620 best_node = n; 5621 } 5622 } 5623 5624 if (best_node >= 0) 5625 node_set(best_node, *used_node_mask); 5626 5627 return best_node; 5628 } 5629 5630 5631 /* 5632 * Build zonelists ordered by node and zones within node. 5633 * This results in maximum locality--normal zone overflows into local 5634 * DMA zone, if any--but risks exhausting DMA zone. 5635 */ 5636 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, 5637 unsigned nr_nodes) 5638 { 5639 struct zoneref *zonerefs; 5640 int i; 5641 5642 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5643 5644 for (i = 0; i < nr_nodes; i++) { 5645 int nr_zones; 5646 5647 pg_data_t *node = NODE_DATA(node_order[i]); 5648 5649 nr_zones = build_zonerefs_node(node, zonerefs); 5650 zonerefs += nr_zones; 5651 } 5652 zonerefs->zone = NULL; 5653 zonerefs->zone_idx = 0; 5654 } 5655 5656 /* 5657 * Build gfp_thisnode zonelists 5658 */ 5659 static void build_thisnode_zonelists(pg_data_t *pgdat) 5660 { 5661 struct zoneref *zonerefs; 5662 int nr_zones; 5663 5664 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; 5665 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5666 zonerefs += nr_zones; 5667 zonerefs->zone = NULL; 5668 zonerefs->zone_idx = 0; 5669 } 5670 5671 /* 5672 * Build zonelists ordered by zone and nodes within zones. 5673 * This results in conserving DMA zone[s] until all Normal memory is 5674 * exhausted, but results in overflowing to remote node while memory 5675 * may still exist in local DMA zone. 5676 */ 5677 5678 static void build_zonelists(pg_data_t *pgdat) 5679 { 5680 static int node_order[MAX_NUMNODES]; 5681 int node, load, nr_nodes = 0; 5682 nodemask_t used_mask; 5683 int local_node, prev_node; 5684 5685 /* NUMA-aware ordering of nodes */ 5686 local_node = pgdat->node_id; 5687 load = nr_online_nodes; 5688 prev_node = local_node; 5689 nodes_clear(used_mask); 5690 5691 memset(node_order, 0, sizeof(node_order)); 5692 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 5693 /* 5694 * We don't want to pressure a particular node. 5695 * So adding penalty to the first node in same 5696 * distance group to make it round-robin. 5697 */ 5698 if (node_distance(local_node, node) != 5699 node_distance(local_node, prev_node)) 5700 node_load[node] = load; 5701 5702 node_order[nr_nodes++] = node; 5703 prev_node = node; 5704 load--; 5705 } 5706 5707 build_zonelists_in_node_order(pgdat, node_order, nr_nodes); 5708 build_thisnode_zonelists(pgdat); 5709 } 5710 5711 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5712 /* 5713 * Return node id of node used for "local" allocations. 5714 * I.e., first node id of first zone in arg node's generic zonelist. 5715 * Used for initializing percpu 'numa_mem', which is used primarily 5716 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 5717 */ 5718 int local_memory_node(int node) 5719 { 5720 struct zoneref *z; 5721 5722 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 5723 gfp_zone(GFP_KERNEL), 5724 NULL); 5725 return zone_to_nid(z->zone); 5726 } 5727 #endif 5728 5729 static void setup_min_unmapped_ratio(void); 5730 static void setup_min_slab_ratio(void); 5731 #else /* CONFIG_NUMA */ 5732 5733 static void build_zonelists(pg_data_t *pgdat) 5734 { 5735 int node, local_node; 5736 struct zoneref *zonerefs; 5737 int nr_zones; 5738 5739 local_node = pgdat->node_id; 5740 5741 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5742 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5743 zonerefs += nr_zones; 5744 5745 /* 5746 * Now we build the zonelist so that it contains the zones 5747 * of all the other nodes. 5748 * We don't want to pressure a particular node, so when 5749 * building the zones for node N, we make sure that the 5750 * zones coming right after the local ones are those from 5751 * node N+1 (modulo N) 5752 */ 5753 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 5754 if (!node_online(node)) 5755 continue; 5756 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5757 zonerefs += nr_zones; 5758 } 5759 for (node = 0; node < local_node; node++) { 5760 if (!node_online(node)) 5761 continue; 5762 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5763 zonerefs += nr_zones; 5764 } 5765 5766 zonerefs->zone = NULL; 5767 zonerefs->zone_idx = 0; 5768 } 5769 5770 #endif /* CONFIG_NUMA */ 5771 5772 /* 5773 * Boot pageset table. One per cpu which is going to be used for all 5774 * zones and all nodes. The parameters will be set in such a way 5775 * that an item put on a list will immediately be handed over to 5776 * the buddy list. This is safe since pageset manipulation is done 5777 * with interrupts disabled. 5778 * 5779 * The boot_pagesets must be kept even after bootup is complete for 5780 * unused processors and/or zones. They do play a role for bootstrapping 5781 * hotplugged processors. 5782 * 5783 * zoneinfo_show() and maybe other functions do 5784 * not check if the processor is online before following the pageset pointer. 5785 * Other parts of the kernel may not check if the zone is available. 5786 */ 5787 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); 5788 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); 5789 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 5790 5791 static void __build_all_zonelists(void *data) 5792 { 5793 int nid; 5794 int __maybe_unused cpu; 5795 pg_data_t *self = data; 5796 static DEFINE_SPINLOCK(lock); 5797 5798 spin_lock(&lock); 5799 5800 #ifdef CONFIG_NUMA 5801 memset(node_load, 0, sizeof(node_load)); 5802 #endif 5803 5804 /* 5805 * This node is hotadded and no memory is yet present. So just 5806 * building zonelists is fine - no need to touch other nodes. 5807 */ 5808 if (self && !node_online(self->node_id)) { 5809 build_zonelists(self); 5810 } else { 5811 for_each_online_node(nid) { 5812 pg_data_t *pgdat = NODE_DATA(nid); 5813 5814 build_zonelists(pgdat); 5815 } 5816 5817 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5818 /* 5819 * We now know the "local memory node" for each node-- 5820 * i.e., the node of the first zone in the generic zonelist. 5821 * Set up numa_mem percpu variable for on-line cpus. During 5822 * boot, only the boot cpu should be on-line; we'll init the 5823 * secondary cpus' numa_mem as they come on-line. During 5824 * node/memory hotplug, we'll fixup all on-line cpus. 5825 */ 5826 for_each_online_cpu(cpu) 5827 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 5828 #endif 5829 } 5830 5831 spin_unlock(&lock); 5832 } 5833 5834 static noinline void __init 5835 build_all_zonelists_init(void) 5836 { 5837 int cpu; 5838 5839 __build_all_zonelists(NULL); 5840 5841 /* 5842 * Initialize the boot_pagesets that are going to be used 5843 * for bootstrapping processors. The real pagesets for 5844 * each zone will be allocated later when the per cpu 5845 * allocator is available. 5846 * 5847 * boot_pagesets are used also for bootstrapping offline 5848 * cpus if the system is already booted because the pagesets 5849 * are needed to initialize allocators on a specific cpu too. 5850 * F.e. the percpu allocator needs the page allocator which 5851 * needs the percpu allocator in order to allocate its pagesets 5852 * (a chicken-egg dilemma). 5853 */ 5854 for_each_possible_cpu(cpu) 5855 setup_pageset(&per_cpu(boot_pageset, cpu), 0); 5856 5857 mminit_verify_zonelist(); 5858 cpuset_init_current_mems_allowed(); 5859 } 5860 5861 /* 5862 * unless system_state == SYSTEM_BOOTING. 5863 * 5864 * __ref due to call of __init annotated helper build_all_zonelists_init 5865 * [protected by SYSTEM_BOOTING]. 5866 */ 5867 void __ref build_all_zonelists(pg_data_t *pgdat) 5868 { 5869 if (system_state == SYSTEM_BOOTING) { 5870 build_all_zonelists_init(); 5871 } else { 5872 __build_all_zonelists(pgdat); 5873 /* cpuset refresh routine should be here */ 5874 } 5875 vm_total_pages = nr_free_pagecache_pages(); 5876 /* 5877 * Disable grouping by mobility if the number of pages in the 5878 * system is too low to allow the mechanism to work. It would be 5879 * more accurate, but expensive to check per-zone. This check is 5880 * made on memory-hotadd so a system can start with mobility 5881 * disabled and enable it later 5882 */ 5883 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 5884 page_group_by_mobility_disabled = 1; 5885 else 5886 page_group_by_mobility_disabled = 0; 5887 5888 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n", 5889 nr_online_nodes, 5890 page_group_by_mobility_disabled ? "off" : "on", 5891 vm_total_pages); 5892 #ifdef CONFIG_NUMA 5893 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 5894 #endif 5895 } 5896 5897 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 5898 static bool __meminit 5899 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 5900 { 5901 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5902 static struct memblock_region *r; 5903 5904 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 5905 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 5906 for_each_memblock(memory, r) { 5907 if (*pfn < memblock_region_memory_end_pfn(r)) 5908 break; 5909 } 5910 } 5911 if (*pfn >= memblock_region_memory_base_pfn(r) && 5912 memblock_is_mirror(r)) { 5913 *pfn = memblock_region_memory_end_pfn(r); 5914 return true; 5915 } 5916 } 5917 #endif 5918 return false; 5919 } 5920 5921 #ifdef CONFIG_SPARSEMEM 5922 /* Skip PFNs that belong to non-present sections */ 5923 static inline __meminit unsigned long next_pfn(unsigned long pfn) 5924 { 5925 const unsigned long section_nr = pfn_to_section_nr(++pfn); 5926 5927 if (present_section_nr(section_nr)) 5928 return pfn; 5929 return section_nr_to_pfn(next_present_section_nr(section_nr)); 5930 } 5931 #else 5932 static inline __meminit unsigned long next_pfn(unsigned long pfn) 5933 { 5934 return pfn++; 5935 } 5936 #endif 5937 5938 /* 5939 * Initially all pages are reserved - free ones are freed 5940 * up by memblock_free_all() once the early boot process is 5941 * done. Non-atomic initialization, single-pass. 5942 */ 5943 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, 5944 unsigned long start_pfn, enum memmap_context context, 5945 struct vmem_altmap *altmap) 5946 { 5947 unsigned long pfn, end_pfn = start_pfn + size; 5948 struct page *page; 5949 5950 if (highest_memmap_pfn < end_pfn - 1) 5951 highest_memmap_pfn = end_pfn - 1; 5952 5953 #ifdef CONFIG_ZONE_DEVICE 5954 /* 5955 * Honor reservation requested by the driver for this ZONE_DEVICE 5956 * memory. We limit the total number of pages to initialize to just 5957 * those that might contain the memory mapping. We will defer the 5958 * ZONE_DEVICE page initialization until after we have released 5959 * the hotplug lock. 5960 */ 5961 if (zone == ZONE_DEVICE) { 5962 if (!altmap) 5963 return; 5964 5965 if (start_pfn == altmap->base_pfn) 5966 start_pfn += altmap->reserve; 5967 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 5968 } 5969 #endif 5970 5971 for (pfn = start_pfn; pfn < end_pfn; ) { 5972 /* 5973 * There can be holes in boot-time mem_map[]s handed to this 5974 * function. They do not exist on hotplugged memory. 5975 */ 5976 if (context == MEMMAP_EARLY) { 5977 if (!early_pfn_valid(pfn)) { 5978 pfn = next_pfn(pfn); 5979 continue; 5980 } 5981 if (!early_pfn_in_nid(pfn, nid)) { 5982 pfn++; 5983 continue; 5984 } 5985 if (overlap_memmap_init(zone, &pfn)) 5986 continue; 5987 if (defer_init(nid, pfn, end_pfn)) 5988 break; 5989 } 5990 5991 page = pfn_to_page(pfn); 5992 __init_single_page(page, pfn, zone, nid); 5993 if (context == MEMMAP_HOTPLUG) 5994 __SetPageReserved(page); 5995 5996 /* 5997 * Mark the block movable so that blocks are reserved for 5998 * movable at startup. This will force kernel allocations 5999 * to reserve their blocks rather than leaking throughout 6000 * the address space during boot when many long-lived 6001 * kernel allocations are made. 6002 * 6003 * bitmap is created for zone's valid pfn range. but memmap 6004 * can be created for invalid pages (for alignment) 6005 * check here not to call set_pageblock_migratetype() against 6006 * pfn out of zone. 6007 */ 6008 if (!(pfn & (pageblock_nr_pages - 1))) { 6009 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 6010 cond_resched(); 6011 } 6012 pfn++; 6013 } 6014 } 6015 6016 #ifdef CONFIG_ZONE_DEVICE 6017 void __ref memmap_init_zone_device(struct zone *zone, 6018 unsigned long start_pfn, 6019 unsigned long nr_pages, 6020 struct dev_pagemap *pgmap) 6021 { 6022 unsigned long pfn, end_pfn = start_pfn + nr_pages; 6023 struct pglist_data *pgdat = zone->zone_pgdat; 6024 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 6025 unsigned long zone_idx = zone_idx(zone); 6026 unsigned long start = jiffies; 6027 int nid = pgdat->node_id; 6028 6029 if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE)) 6030 return; 6031 6032 /* 6033 * The call to memmap_init_zone should have already taken care 6034 * of the pages reserved for the memmap, so we can just jump to 6035 * the end of that region and start processing the device pages. 6036 */ 6037 if (altmap) { 6038 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6039 nr_pages = end_pfn - start_pfn; 6040 } 6041 6042 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 6043 struct page *page = pfn_to_page(pfn); 6044 6045 __init_single_page(page, pfn, zone_idx, nid); 6046 6047 /* 6048 * Mark page reserved as it will need to wait for onlining 6049 * phase for it to be fully associated with a zone. 6050 * 6051 * We can use the non-atomic __set_bit operation for setting 6052 * the flag as we are still initializing the pages. 6053 */ 6054 __SetPageReserved(page); 6055 6056 /* 6057 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 6058 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 6059 * ever freed or placed on a driver-private list. 6060 */ 6061 page->pgmap = pgmap; 6062 page->zone_device_data = NULL; 6063 6064 /* 6065 * Mark the block movable so that blocks are reserved for 6066 * movable at startup. This will force kernel allocations 6067 * to reserve their blocks rather than leaking throughout 6068 * the address space during boot when many long-lived 6069 * kernel allocations are made. 6070 * 6071 * bitmap is created for zone's valid pfn range. but memmap 6072 * can be created for invalid pages (for alignment) 6073 * check here not to call set_pageblock_migratetype() against 6074 * pfn out of zone. 6075 * 6076 * Please note that MEMMAP_HOTPLUG path doesn't clear memmap 6077 * because this is done early in section_activate() 6078 */ 6079 if (!(pfn & (pageblock_nr_pages - 1))) { 6080 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 6081 cond_resched(); 6082 } 6083 } 6084 6085 pr_info("%s initialised %lu pages in %ums\n", __func__, 6086 nr_pages, jiffies_to_msecs(jiffies - start)); 6087 } 6088 6089 #endif 6090 static void __meminit zone_init_free_lists(struct zone *zone) 6091 { 6092 unsigned int order, t; 6093 for_each_migratetype_order(order, t) { 6094 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 6095 zone->free_area[order].nr_free = 0; 6096 } 6097 } 6098 6099 void __meminit __weak memmap_init(unsigned long size, int nid, 6100 unsigned long zone, unsigned long start_pfn) 6101 { 6102 memmap_init_zone(size, nid, zone, start_pfn, MEMMAP_EARLY, NULL); 6103 } 6104 6105 static int zone_batchsize(struct zone *zone) 6106 { 6107 #ifdef CONFIG_MMU 6108 int batch; 6109 6110 /* 6111 * The per-cpu-pages pools are set to around 1000th of the 6112 * size of the zone. 6113 */ 6114 batch = zone_managed_pages(zone) / 1024; 6115 /* But no more than a meg. */ 6116 if (batch * PAGE_SIZE > 1024 * 1024) 6117 batch = (1024 * 1024) / PAGE_SIZE; 6118 batch /= 4; /* We effectively *= 4 below */ 6119 if (batch < 1) 6120 batch = 1; 6121 6122 /* 6123 * Clamp the batch to a 2^n - 1 value. Having a power 6124 * of 2 value was found to be more likely to have 6125 * suboptimal cache aliasing properties in some cases. 6126 * 6127 * For example if 2 tasks are alternately allocating 6128 * batches of pages, one task can end up with a lot 6129 * of pages of one half of the possible page colors 6130 * and the other with pages of the other colors. 6131 */ 6132 batch = rounddown_pow_of_two(batch + batch/2) - 1; 6133 6134 return batch; 6135 6136 #else 6137 /* The deferral and batching of frees should be suppressed under NOMMU 6138 * conditions. 6139 * 6140 * The problem is that NOMMU needs to be able to allocate large chunks 6141 * of contiguous memory as there's no hardware page translation to 6142 * assemble apparent contiguous memory from discontiguous pages. 6143 * 6144 * Queueing large contiguous runs of pages for batching, however, 6145 * causes the pages to actually be freed in smaller chunks. As there 6146 * can be a significant delay between the individual batches being 6147 * recycled, this leads to the once large chunks of space being 6148 * fragmented and becoming unavailable for high-order allocations. 6149 */ 6150 return 0; 6151 #endif 6152 } 6153 6154 /* 6155 * pcp->high and pcp->batch values are related and dependent on one another: 6156 * ->batch must never be higher then ->high. 6157 * The following function updates them in a safe manner without read side 6158 * locking. 6159 * 6160 * Any new users of pcp->batch and pcp->high should ensure they can cope with 6161 * those fields changing asynchronously (acording the the above rule). 6162 * 6163 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 6164 * outside of boot time (or some other assurance that no concurrent updaters 6165 * exist). 6166 */ 6167 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 6168 unsigned long batch) 6169 { 6170 /* start with a fail safe value for batch */ 6171 pcp->batch = 1; 6172 smp_wmb(); 6173 6174 /* Update high, then batch, in order */ 6175 pcp->high = high; 6176 smp_wmb(); 6177 6178 pcp->batch = batch; 6179 } 6180 6181 /* a companion to pageset_set_high() */ 6182 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) 6183 { 6184 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); 6185 } 6186 6187 static void pageset_init(struct per_cpu_pageset *p) 6188 { 6189 struct per_cpu_pages *pcp; 6190 int migratetype; 6191 6192 memset(p, 0, sizeof(*p)); 6193 6194 pcp = &p->pcp; 6195 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) 6196 INIT_LIST_HEAD(&pcp->lists[migratetype]); 6197 } 6198 6199 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 6200 { 6201 pageset_init(p); 6202 pageset_set_batch(p, batch); 6203 } 6204 6205 /* 6206 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist 6207 * to the value high for the pageset p. 6208 */ 6209 static void pageset_set_high(struct per_cpu_pageset *p, 6210 unsigned long high) 6211 { 6212 unsigned long batch = max(1UL, high / 4); 6213 if ((high / 4) > (PAGE_SHIFT * 8)) 6214 batch = PAGE_SHIFT * 8; 6215 6216 pageset_update(&p->pcp, high, batch); 6217 } 6218 6219 static void pageset_set_high_and_batch(struct zone *zone, 6220 struct per_cpu_pageset *pcp) 6221 { 6222 if (percpu_pagelist_fraction) 6223 pageset_set_high(pcp, 6224 (zone_managed_pages(zone) / 6225 percpu_pagelist_fraction)); 6226 else 6227 pageset_set_batch(pcp, zone_batchsize(zone)); 6228 } 6229 6230 static void __meminit zone_pageset_init(struct zone *zone, int cpu) 6231 { 6232 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); 6233 6234 pageset_init(pcp); 6235 pageset_set_high_and_batch(zone, pcp); 6236 } 6237 6238 void __meminit setup_zone_pageset(struct zone *zone) 6239 { 6240 int cpu; 6241 zone->pageset = alloc_percpu(struct per_cpu_pageset); 6242 for_each_possible_cpu(cpu) 6243 zone_pageset_init(zone, cpu); 6244 } 6245 6246 /* 6247 * Allocate per cpu pagesets and initialize them. 6248 * Before this call only boot pagesets were available. 6249 */ 6250 void __init setup_per_cpu_pageset(void) 6251 { 6252 struct pglist_data *pgdat; 6253 struct zone *zone; 6254 6255 for_each_populated_zone(zone) 6256 setup_zone_pageset(zone); 6257 6258 for_each_online_pgdat(pgdat) 6259 pgdat->per_cpu_nodestats = 6260 alloc_percpu(struct per_cpu_nodestat); 6261 } 6262 6263 static __meminit void zone_pcp_init(struct zone *zone) 6264 { 6265 /* 6266 * per cpu subsystem is not up at this point. The following code 6267 * relies on the ability of the linker to provide the 6268 * offset of a (static) per cpu variable into the per cpu area. 6269 */ 6270 zone->pageset = &boot_pageset; 6271 6272 if (populated_zone(zone)) 6273 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", 6274 zone->name, zone->present_pages, 6275 zone_batchsize(zone)); 6276 } 6277 6278 void __meminit init_currently_empty_zone(struct zone *zone, 6279 unsigned long zone_start_pfn, 6280 unsigned long size) 6281 { 6282 struct pglist_data *pgdat = zone->zone_pgdat; 6283 int zone_idx = zone_idx(zone) + 1; 6284 6285 if (zone_idx > pgdat->nr_zones) 6286 pgdat->nr_zones = zone_idx; 6287 6288 zone->zone_start_pfn = zone_start_pfn; 6289 6290 mminit_dprintk(MMINIT_TRACE, "memmap_init", 6291 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 6292 pgdat->node_id, 6293 (unsigned long)zone_idx(zone), 6294 zone_start_pfn, (zone_start_pfn + size)); 6295 6296 zone_init_free_lists(zone); 6297 zone->initialized = 1; 6298 } 6299 6300 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6301 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID 6302 6303 /* 6304 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 6305 */ 6306 int __meminit __early_pfn_to_nid(unsigned long pfn, 6307 struct mminit_pfnnid_cache *state) 6308 { 6309 unsigned long start_pfn, end_pfn; 6310 int nid; 6311 6312 if (state->last_start <= pfn && pfn < state->last_end) 6313 return state->last_nid; 6314 6315 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 6316 if (nid != NUMA_NO_NODE) { 6317 state->last_start = start_pfn; 6318 state->last_end = end_pfn; 6319 state->last_nid = nid; 6320 } 6321 6322 return nid; 6323 } 6324 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ 6325 6326 /** 6327 * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range 6328 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed. 6329 * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid 6330 * 6331 * If an architecture guarantees that all ranges registered contain no holes 6332 * and may be freed, this this function may be used instead of calling 6333 * memblock_free_early_nid() manually. 6334 */ 6335 void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn) 6336 { 6337 unsigned long start_pfn, end_pfn; 6338 int i, this_nid; 6339 6340 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) { 6341 start_pfn = min(start_pfn, max_low_pfn); 6342 end_pfn = min(end_pfn, max_low_pfn); 6343 6344 if (start_pfn < end_pfn) 6345 memblock_free_early_nid(PFN_PHYS(start_pfn), 6346 (end_pfn - start_pfn) << PAGE_SHIFT, 6347 this_nid); 6348 } 6349 } 6350 6351 /** 6352 * sparse_memory_present_with_active_regions - Call memory_present for each active range 6353 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used. 6354 * 6355 * If an architecture guarantees that all ranges registered contain no holes and may 6356 * be freed, this function may be used instead of calling memory_present() manually. 6357 */ 6358 void __init sparse_memory_present_with_active_regions(int nid) 6359 { 6360 unsigned long start_pfn, end_pfn; 6361 int i, this_nid; 6362 6363 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) 6364 memory_present(this_nid, start_pfn, end_pfn); 6365 } 6366 6367 /** 6368 * get_pfn_range_for_nid - Return the start and end page frames for a node 6369 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 6370 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 6371 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 6372 * 6373 * It returns the start and end page frame of a node based on information 6374 * provided by memblock_set_node(). If called for a node 6375 * with no available memory, a warning is printed and the start and end 6376 * PFNs will be 0. 6377 */ 6378 void __init get_pfn_range_for_nid(unsigned int nid, 6379 unsigned long *start_pfn, unsigned long *end_pfn) 6380 { 6381 unsigned long this_start_pfn, this_end_pfn; 6382 int i; 6383 6384 *start_pfn = -1UL; 6385 *end_pfn = 0; 6386 6387 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 6388 *start_pfn = min(*start_pfn, this_start_pfn); 6389 *end_pfn = max(*end_pfn, this_end_pfn); 6390 } 6391 6392 if (*start_pfn == -1UL) 6393 *start_pfn = 0; 6394 } 6395 6396 /* 6397 * This finds a zone that can be used for ZONE_MOVABLE pages. The 6398 * assumption is made that zones within a node are ordered in monotonic 6399 * increasing memory addresses so that the "highest" populated zone is used 6400 */ 6401 static void __init find_usable_zone_for_movable(void) 6402 { 6403 int zone_index; 6404 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 6405 if (zone_index == ZONE_MOVABLE) 6406 continue; 6407 6408 if (arch_zone_highest_possible_pfn[zone_index] > 6409 arch_zone_lowest_possible_pfn[zone_index]) 6410 break; 6411 } 6412 6413 VM_BUG_ON(zone_index == -1); 6414 movable_zone = zone_index; 6415 } 6416 6417 /* 6418 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 6419 * because it is sized independent of architecture. Unlike the other zones, 6420 * the starting point for ZONE_MOVABLE is not fixed. It may be different 6421 * in each node depending on the size of each node and how evenly kernelcore 6422 * is distributed. This helper function adjusts the zone ranges 6423 * provided by the architecture for a given node by using the end of the 6424 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 6425 * zones within a node are in order of monotonic increases memory addresses 6426 */ 6427 static void __init adjust_zone_range_for_zone_movable(int nid, 6428 unsigned long zone_type, 6429 unsigned long node_start_pfn, 6430 unsigned long node_end_pfn, 6431 unsigned long *zone_start_pfn, 6432 unsigned long *zone_end_pfn) 6433 { 6434 /* Only adjust if ZONE_MOVABLE is on this node */ 6435 if (zone_movable_pfn[nid]) { 6436 /* Size ZONE_MOVABLE */ 6437 if (zone_type == ZONE_MOVABLE) { 6438 *zone_start_pfn = zone_movable_pfn[nid]; 6439 *zone_end_pfn = min(node_end_pfn, 6440 arch_zone_highest_possible_pfn[movable_zone]); 6441 6442 /* Adjust for ZONE_MOVABLE starting within this range */ 6443 } else if (!mirrored_kernelcore && 6444 *zone_start_pfn < zone_movable_pfn[nid] && 6445 *zone_end_pfn > zone_movable_pfn[nid]) { 6446 *zone_end_pfn = zone_movable_pfn[nid]; 6447 6448 /* Check if this whole range is within ZONE_MOVABLE */ 6449 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 6450 *zone_start_pfn = *zone_end_pfn; 6451 } 6452 } 6453 6454 /* 6455 * Return the number of pages a zone spans in a node, including holes 6456 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 6457 */ 6458 static unsigned long __init zone_spanned_pages_in_node(int nid, 6459 unsigned long zone_type, 6460 unsigned long node_start_pfn, 6461 unsigned long node_end_pfn, 6462 unsigned long *zone_start_pfn, 6463 unsigned long *zone_end_pfn, 6464 unsigned long *ignored) 6465 { 6466 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 6467 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 6468 /* When hotadd a new node from cpu_up(), the node should be empty */ 6469 if (!node_start_pfn && !node_end_pfn) 6470 return 0; 6471 6472 /* Get the start and end of the zone */ 6473 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 6474 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 6475 adjust_zone_range_for_zone_movable(nid, zone_type, 6476 node_start_pfn, node_end_pfn, 6477 zone_start_pfn, zone_end_pfn); 6478 6479 /* Check that this node has pages within the zone's required range */ 6480 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 6481 return 0; 6482 6483 /* Move the zone boundaries inside the node if necessary */ 6484 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 6485 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 6486 6487 /* Return the spanned pages */ 6488 return *zone_end_pfn - *zone_start_pfn; 6489 } 6490 6491 /* 6492 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 6493 * then all holes in the requested range will be accounted for. 6494 */ 6495 unsigned long __init __absent_pages_in_range(int nid, 6496 unsigned long range_start_pfn, 6497 unsigned long range_end_pfn) 6498 { 6499 unsigned long nr_absent = range_end_pfn - range_start_pfn; 6500 unsigned long start_pfn, end_pfn; 6501 int i; 6502 6503 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6504 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 6505 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 6506 nr_absent -= end_pfn - start_pfn; 6507 } 6508 return nr_absent; 6509 } 6510 6511 /** 6512 * absent_pages_in_range - Return number of page frames in holes within a range 6513 * @start_pfn: The start PFN to start searching for holes 6514 * @end_pfn: The end PFN to stop searching for holes 6515 * 6516 * Return: the number of pages frames in memory holes within a range. 6517 */ 6518 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 6519 unsigned long end_pfn) 6520 { 6521 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 6522 } 6523 6524 /* Return the number of page frames in holes in a zone on a node */ 6525 static unsigned long __init zone_absent_pages_in_node(int nid, 6526 unsigned long zone_type, 6527 unsigned long node_start_pfn, 6528 unsigned long node_end_pfn, 6529 unsigned long *ignored) 6530 { 6531 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 6532 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 6533 unsigned long zone_start_pfn, zone_end_pfn; 6534 unsigned long nr_absent; 6535 6536 /* When hotadd a new node from cpu_up(), the node should be empty */ 6537 if (!node_start_pfn && !node_end_pfn) 6538 return 0; 6539 6540 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 6541 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 6542 6543 adjust_zone_range_for_zone_movable(nid, zone_type, 6544 node_start_pfn, node_end_pfn, 6545 &zone_start_pfn, &zone_end_pfn); 6546 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 6547 6548 /* 6549 * ZONE_MOVABLE handling. 6550 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 6551 * and vice versa. 6552 */ 6553 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 6554 unsigned long start_pfn, end_pfn; 6555 struct memblock_region *r; 6556 6557 for_each_memblock(memory, r) { 6558 start_pfn = clamp(memblock_region_memory_base_pfn(r), 6559 zone_start_pfn, zone_end_pfn); 6560 end_pfn = clamp(memblock_region_memory_end_pfn(r), 6561 zone_start_pfn, zone_end_pfn); 6562 6563 if (zone_type == ZONE_MOVABLE && 6564 memblock_is_mirror(r)) 6565 nr_absent += end_pfn - start_pfn; 6566 6567 if (zone_type == ZONE_NORMAL && 6568 !memblock_is_mirror(r)) 6569 nr_absent += end_pfn - start_pfn; 6570 } 6571 } 6572 6573 return nr_absent; 6574 } 6575 6576 #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6577 static inline unsigned long __init zone_spanned_pages_in_node(int nid, 6578 unsigned long zone_type, 6579 unsigned long node_start_pfn, 6580 unsigned long node_end_pfn, 6581 unsigned long *zone_start_pfn, 6582 unsigned long *zone_end_pfn, 6583 unsigned long *zones_size) 6584 { 6585 unsigned int zone; 6586 6587 *zone_start_pfn = node_start_pfn; 6588 for (zone = 0; zone < zone_type; zone++) 6589 *zone_start_pfn += zones_size[zone]; 6590 6591 *zone_end_pfn = *zone_start_pfn + zones_size[zone_type]; 6592 6593 return zones_size[zone_type]; 6594 } 6595 6596 static inline unsigned long __init zone_absent_pages_in_node(int nid, 6597 unsigned long zone_type, 6598 unsigned long node_start_pfn, 6599 unsigned long node_end_pfn, 6600 unsigned long *zholes_size) 6601 { 6602 if (!zholes_size) 6603 return 0; 6604 6605 return zholes_size[zone_type]; 6606 } 6607 6608 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6609 6610 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 6611 unsigned long node_start_pfn, 6612 unsigned long node_end_pfn, 6613 unsigned long *zones_size, 6614 unsigned long *zholes_size) 6615 { 6616 unsigned long realtotalpages = 0, totalpages = 0; 6617 enum zone_type i; 6618 6619 for (i = 0; i < MAX_NR_ZONES; i++) { 6620 struct zone *zone = pgdat->node_zones + i; 6621 unsigned long zone_start_pfn, zone_end_pfn; 6622 unsigned long size, real_size; 6623 6624 size = zone_spanned_pages_in_node(pgdat->node_id, i, 6625 node_start_pfn, 6626 node_end_pfn, 6627 &zone_start_pfn, 6628 &zone_end_pfn, 6629 zones_size); 6630 real_size = size - zone_absent_pages_in_node(pgdat->node_id, i, 6631 node_start_pfn, node_end_pfn, 6632 zholes_size); 6633 if (size) 6634 zone->zone_start_pfn = zone_start_pfn; 6635 else 6636 zone->zone_start_pfn = 0; 6637 zone->spanned_pages = size; 6638 zone->present_pages = real_size; 6639 6640 totalpages += size; 6641 realtotalpages += real_size; 6642 } 6643 6644 pgdat->node_spanned_pages = totalpages; 6645 pgdat->node_present_pages = realtotalpages; 6646 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, 6647 realtotalpages); 6648 } 6649 6650 #ifndef CONFIG_SPARSEMEM 6651 /* 6652 * Calculate the size of the zone->blockflags rounded to an unsigned long 6653 * Start by making sure zonesize is a multiple of pageblock_order by rounding 6654 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 6655 * round what is now in bits to nearest long in bits, then return it in 6656 * bytes. 6657 */ 6658 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 6659 { 6660 unsigned long usemapsize; 6661 6662 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 6663 usemapsize = roundup(zonesize, pageblock_nr_pages); 6664 usemapsize = usemapsize >> pageblock_order; 6665 usemapsize *= NR_PAGEBLOCK_BITS; 6666 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 6667 6668 return usemapsize / 8; 6669 } 6670 6671 static void __ref setup_usemap(struct pglist_data *pgdat, 6672 struct zone *zone, 6673 unsigned long zone_start_pfn, 6674 unsigned long zonesize) 6675 { 6676 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); 6677 zone->pageblock_flags = NULL; 6678 if (usemapsize) { 6679 zone->pageblock_flags = 6680 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 6681 pgdat->node_id); 6682 if (!zone->pageblock_flags) 6683 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 6684 usemapsize, zone->name, pgdat->node_id); 6685 } 6686 } 6687 #else 6688 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, 6689 unsigned long zone_start_pfn, unsigned long zonesize) {} 6690 #endif /* CONFIG_SPARSEMEM */ 6691 6692 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 6693 6694 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 6695 void __init set_pageblock_order(void) 6696 { 6697 unsigned int order; 6698 6699 /* Check that pageblock_nr_pages has not already been setup */ 6700 if (pageblock_order) 6701 return; 6702 6703 if (HPAGE_SHIFT > PAGE_SHIFT) 6704 order = HUGETLB_PAGE_ORDER; 6705 else 6706 order = MAX_ORDER - 1; 6707 6708 /* 6709 * Assume the largest contiguous order of interest is a huge page. 6710 * This value may be variable depending on boot parameters on IA64 and 6711 * powerpc. 6712 */ 6713 pageblock_order = order; 6714 } 6715 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6716 6717 /* 6718 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 6719 * is unused as pageblock_order is set at compile-time. See 6720 * include/linux/pageblock-flags.h for the values of pageblock_order based on 6721 * the kernel config 6722 */ 6723 void __init set_pageblock_order(void) 6724 { 6725 } 6726 6727 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6728 6729 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 6730 unsigned long present_pages) 6731 { 6732 unsigned long pages = spanned_pages; 6733 6734 /* 6735 * Provide a more accurate estimation if there are holes within 6736 * the zone and SPARSEMEM is in use. If there are holes within the 6737 * zone, each populated memory region may cost us one or two extra 6738 * memmap pages due to alignment because memmap pages for each 6739 * populated regions may not be naturally aligned on page boundary. 6740 * So the (present_pages >> 4) heuristic is a tradeoff for that. 6741 */ 6742 if (spanned_pages > present_pages + (present_pages >> 4) && 6743 IS_ENABLED(CONFIG_SPARSEMEM)) 6744 pages = present_pages; 6745 6746 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 6747 } 6748 6749 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 6750 static void pgdat_init_split_queue(struct pglist_data *pgdat) 6751 { 6752 struct deferred_split *ds_queue = &pgdat->deferred_split_queue; 6753 6754 spin_lock_init(&ds_queue->split_queue_lock); 6755 INIT_LIST_HEAD(&ds_queue->split_queue); 6756 ds_queue->split_queue_len = 0; 6757 } 6758 #else 6759 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 6760 #endif 6761 6762 #ifdef CONFIG_COMPACTION 6763 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 6764 { 6765 init_waitqueue_head(&pgdat->kcompactd_wait); 6766 } 6767 #else 6768 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 6769 #endif 6770 6771 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 6772 { 6773 pgdat_resize_init(pgdat); 6774 6775 pgdat_init_split_queue(pgdat); 6776 pgdat_init_kcompactd(pgdat); 6777 6778 init_waitqueue_head(&pgdat->kswapd_wait); 6779 init_waitqueue_head(&pgdat->pfmemalloc_wait); 6780 6781 pgdat_page_ext_init(pgdat); 6782 spin_lock_init(&pgdat->lru_lock); 6783 lruvec_init(&pgdat->__lruvec); 6784 } 6785 6786 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 6787 unsigned long remaining_pages) 6788 { 6789 atomic_long_set(&zone->managed_pages, remaining_pages); 6790 zone_set_nid(zone, nid); 6791 zone->name = zone_names[idx]; 6792 zone->zone_pgdat = NODE_DATA(nid); 6793 spin_lock_init(&zone->lock); 6794 zone_seqlock_init(zone); 6795 zone_pcp_init(zone); 6796 } 6797 6798 /* 6799 * Set up the zone data structures 6800 * - init pgdat internals 6801 * - init all zones belonging to this node 6802 * 6803 * NOTE: this function is only called during memory hotplug 6804 */ 6805 #ifdef CONFIG_MEMORY_HOTPLUG 6806 void __ref free_area_init_core_hotplug(int nid) 6807 { 6808 enum zone_type z; 6809 pg_data_t *pgdat = NODE_DATA(nid); 6810 6811 pgdat_init_internals(pgdat); 6812 for (z = 0; z < MAX_NR_ZONES; z++) 6813 zone_init_internals(&pgdat->node_zones[z], z, nid, 0); 6814 } 6815 #endif 6816 6817 /* 6818 * Set up the zone data structures: 6819 * - mark all pages reserved 6820 * - mark all memory queues empty 6821 * - clear the memory bitmaps 6822 * 6823 * NOTE: pgdat should get zeroed by caller. 6824 * NOTE: this function is only called during early init. 6825 */ 6826 static void __init free_area_init_core(struct pglist_data *pgdat) 6827 { 6828 enum zone_type j; 6829 int nid = pgdat->node_id; 6830 6831 pgdat_init_internals(pgdat); 6832 pgdat->per_cpu_nodestats = &boot_nodestats; 6833 6834 for (j = 0; j < MAX_NR_ZONES; j++) { 6835 struct zone *zone = pgdat->node_zones + j; 6836 unsigned long size, freesize, memmap_pages; 6837 unsigned long zone_start_pfn = zone->zone_start_pfn; 6838 6839 size = zone->spanned_pages; 6840 freesize = zone->present_pages; 6841 6842 /* 6843 * Adjust freesize so that it accounts for how much memory 6844 * is used by this zone for memmap. This affects the watermark 6845 * and per-cpu initialisations 6846 */ 6847 memmap_pages = calc_memmap_size(size, freesize); 6848 if (!is_highmem_idx(j)) { 6849 if (freesize >= memmap_pages) { 6850 freesize -= memmap_pages; 6851 if (memmap_pages) 6852 printk(KERN_DEBUG 6853 " %s zone: %lu pages used for memmap\n", 6854 zone_names[j], memmap_pages); 6855 } else 6856 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n", 6857 zone_names[j], memmap_pages, freesize); 6858 } 6859 6860 /* Account for reserved pages */ 6861 if (j == 0 && freesize > dma_reserve) { 6862 freesize -= dma_reserve; 6863 printk(KERN_DEBUG " %s zone: %lu pages reserved\n", 6864 zone_names[0], dma_reserve); 6865 } 6866 6867 if (!is_highmem_idx(j)) 6868 nr_kernel_pages += freesize; 6869 /* Charge for highmem memmap if there are enough kernel pages */ 6870 else if (nr_kernel_pages > memmap_pages * 2) 6871 nr_kernel_pages -= memmap_pages; 6872 nr_all_pages += freesize; 6873 6874 /* 6875 * Set an approximate value for lowmem here, it will be adjusted 6876 * when the bootmem allocator frees pages into the buddy system. 6877 * And all highmem pages will be managed by the buddy system. 6878 */ 6879 zone_init_internals(zone, j, nid, freesize); 6880 6881 if (!size) 6882 continue; 6883 6884 set_pageblock_order(); 6885 setup_usemap(pgdat, zone, zone_start_pfn, size); 6886 init_currently_empty_zone(zone, zone_start_pfn, size); 6887 memmap_init(size, nid, j, zone_start_pfn); 6888 } 6889 } 6890 6891 #ifdef CONFIG_FLAT_NODE_MEM_MAP 6892 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) 6893 { 6894 unsigned long __maybe_unused start = 0; 6895 unsigned long __maybe_unused offset = 0; 6896 6897 /* Skip empty nodes */ 6898 if (!pgdat->node_spanned_pages) 6899 return; 6900 6901 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 6902 offset = pgdat->node_start_pfn - start; 6903 /* ia64 gets its own node_mem_map, before this, without bootmem */ 6904 if (!pgdat->node_mem_map) { 6905 unsigned long size, end; 6906 struct page *map; 6907 6908 /* 6909 * The zone's endpoints aren't required to be MAX_ORDER 6910 * aligned but the node_mem_map endpoints must be in order 6911 * for the buddy allocator to function correctly. 6912 */ 6913 end = pgdat_end_pfn(pgdat); 6914 end = ALIGN(end, MAX_ORDER_NR_PAGES); 6915 size = (end - start) * sizeof(struct page); 6916 map = memblock_alloc_node(size, SMP_CACHE_BYTES, 6917 pgdat->node_id); 6918 if (!map) 6919 panic("Failed to allocate %ld bytes for node %d memory map\n", 6920 size, pgdat->node_id); 6921 pgdat->node_mem_map = map + offset; 6922 } 6923 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 6924 __func__, pgdat->node_id, (unsigned long)pgdat, 6925 (unsigned long)pgdat->node_mem_map); 6926 #ifndef CONFIG_NEED_MULTIPLE_NODES 6927 /* 6928 * With no DISCONTIG, the global mem_map is just set as node 0's 6929 */ 6930 if (pgdat == NODE_DATA(0)) { 6931 mem_map = NODE_DATA(0)->node_mem_map; 6932 #if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM) 6933 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 6934 mem_map -= offset; 6935 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6936 } 6937 #endif 6938 } 6939 #else 6940 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { } 6941 #endif /* CONFIG_FLAT_NODE_MEM_MAP */ 6942 6943 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 6944 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 6945 { 6946 pgdat->first_deferred_pfn = ULONG_MAX; 6947 } 6948 #else 6949 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 6950 #endif 6951 6952 void __init free_area_init_node(int nid, unsigned long *zones_size, 6953 unsigned long node_start_pfn, 6954 unsigned long *zholes_size) 6955 { 6956 pg_data_t *pgdat = NODE_DATA(nid); 6957 unsigned long start_pfn = 0; 6958 unsigned long end_pfn = 0; 6959 6960 /* pg_data_t should be reset to zero when it's allocated */ 6961 WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx); 6962 6963 pgdat->node_id = nid; 6964 pgdat->node_start_pfn = node_start_pfn; 6965 pgdat->per_cpu_nodestats = NULL; 6966 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6967 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 6968 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 6969 (u64)start_pfn << PAGE_SHIFT, 6970 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 6971 #else 6972 start_pfn = node_start_pfn; 6973 #endif 6974 calculate_node_totalpages(pgdat, start_pfn, end_pfn, 6975 zones_size, zholes_size); 6976 6977 alloc_node_mem_map(pgdat); 6978 pgdat_set_deferred_range(pgdat); 6979 6980 free_area_init_core(pgdat); 6981 } 6982 6983 #if !defined(CONFIG_FLAT_NODE_MEM_MAP) 6984 /* 6985 * Initialize all valid struct pages in the range [spfn, epfn) and mark them 6986 * PageReserved(). Return the number of struct pages that were initialized. 6987 */ 6988 static u64 __init init_unavailable_range(unsigned long spfn, unsigned long epfn) 6989 { 6990 unsigned long pfn; 6991 u64 pgcnt = 0; 6992 6993 for (pfn = spfn; pfn < epfn; pfn++) { 6994 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) { 6995 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages) 6996 + pageblock_nr_pages - 1; 6997 continue; 6998 } 6999 /* 7000 * Use a fake node/zone (0) for now. Some of these pages 7001 * (in memblock.reserved but not in memblock.memory) will 7002 * get re-initialized via reserve_bootmem_region() later. 7003 */ 7004 __init_single_page(pfn_to_page(pfn), pfn, 0, 0); 7005 __SetPageReserved(pfn_to_page(pfn)); 7006 pgcnt++; 7007 } 7008 7009 return pgcnt; 7010 } 7011 7012 /* 7013 * Only struct pages that are backed by physical memory are zeroed and 7014 * initialized by going through __init_single_page(). But, there are some 7015 * struct pages which are reserved in memblock allocator and their fields 7016 * may be accessed (for example page_to_pfn() on some configuration accesses 7017 * flags). We must explicitly initialize those struct pages. 7018 * 7019 * This function also addresses a similar issue where struct pages are left 7020 * uninitialized because the physical address range is not covered by 7021 * memblock.memory or memblock.reserved. That could happen when memblock 7022 * layout is manually configured via memmap=, or when the highest physical 7023 * address (max_pfn) does not end on a section boundary. 7024 */ 7025 static void __init init_unavailable_mem(void) 7026 { 7027 phys_addr_t start, end; 7028 u64 i, pgcnt; 7029 phys_addr_t next = 0; 7030 7031 /* 7032 * Loop through unavailable ranges not covered by memblock.memory. 7033 */ 7034 pgcnt = 0; 7035 for_each_mem_range(i, &memblock.memory, NULL, 7036 NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, NULL) { 7037 if (next < start) 7038 pgcnt += init_unavailable_range(PFN_DOWN(next), 7039 PFN_UP(start)); 7040 next = end; 7041 } 7042 7043 /* 7044 * Early sections always have a fully populated memmap for the whole 7045 * section - see pfn_valid(). If the last section has holes at the 7046 * end and that section is marked "online", the memmap will be 7047 * considered initialized. Make sure that memmap has a well defined 7048 * state. 7049 */ 7050 pgcnt += init_unavailable_range(PFN_DOWN(next), 7051 round_up(max_pfn, PAGES_PER_SECTION)); 7052 7053 /* 7054 * Struct pages that do not have backing memory. This could be because 7055 * firmware is using some of this memory, or for some other reasons. 7056 */ 7057 if (pgcnt) 7058 pr_info("Zeroed struct page in unavailable ranges: %lld pages", pgcnt); 7059 } 7060 #else 7061 static inline void __init init_unavailable_mem(void) 7062 { 7063 } 7064 #endif /* !CONFIG_FLAT_NODE_MEM_MAP */ 7065 7066 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 7067 7068 #if MAX_NUMNODES > 1 7069 /* 7070 * Figure out the number of possible node ids. 7071 */ 7072 void __init setup_nr_node_ids(void) 7073 { 7074 unsigned int highest; 7075 7076 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 7077 nr_node_ids = highest + 1; 7078 } 7079 #endif 7080 7081 /** 7082 * node_map_pfn_alignment - determine the maximum internode alignment 7083 * 7084 * This function should be called after node map is populated and sorted. 7085 * It calculates the maximum power of two alignment which can distinguish 7086 * all the nodes. 7087 * 7088 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 7089 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 7090 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 7091 * shifted, 1GiB is enough and this function will indicate so. 7092 * 7093 * This is used to test whether pfn -> nid mapping of the chosen memory 7094 * model has fine enough granularity to avoid incorrect mapping for the 7095 * populated node map. 7096 * 7097 * Return: the determined alignment in pfn's. 0 if there is no alignment 7098 * requirement (single node). 7099 */ 7100 unsigned long __init node_map_pfn_alignment(void) 7101 { 7102 unsigned long accl_mask = 0, last_end = 0; 7103 unsigned long start, end, mask; 7104 int last_nid = NUMA_NO_NODE; 7105 int i, nid; 7106 7107 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 7108 if (!start || last_nid < 0 || last_nid == nid) { 7109 last_nid = nid; 7110 last_end = end; 7111 continue; 7112 } 7113 7114 /* 7115 * Start with a mask granular enough to pin-point to the 7116 * start pfn and tick off bits one-by-one until it becomes 7117 * too coarse to separate the current node from the last. 7118 */ 7119 mask = ~((1 << __ffs(start)) - 1); 7120 while (mask && last_end <= (start & (mask << 1))) 7121 mask <<= 1; 7122 7123 /* accumulate all internode masks */ 7124 accl_mask |= mask; 7125 } 7126 7127 /* convert mask to number of pages */ 7128 return ~accl_mask + 1; 7129 } 7130 7131 /* Find the lowest pfn for a node */ 7132 static unsigned long __init find_min_pfn_for_node(int nid) 7133 { 7134 unsigned long min_pfn = ULONG_MAX; 7135 unsigned long start_pfn; 7136 int i; 7137 7138 for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL) 7139 min_pfn = min(min_pfn, start_pfn); 7140 7141 if (min_pfn == ULONG_MAX) { 7142 pr_warn("Could not find start_pfn for node %d\n", nid); 7143 return 0; 7144 } 7145 7146 return min_pfn; 7147 } 7148 7149 /** 7150 * find_min_pfn_with_active_regions - Find the minimum PFN registered 7151 * 7152 * Return: the minimum PFN based on information provided via 7153 * memblock_set_node(). 7154 */ 7155 unsigned long __init find_min_pfn_with_active_regions(void) 7156 { 7157 return find_min_pfn_for_node(MAX_NUMNODES); 7158 } 7159 7160 /* 7161 * early_calculate_totalpages() 7162 * Sum pages in active regions for movable zone. 7163 * Populate N_MEMORY for calculating usable_nodes. 7164 */ 7165 static unsigned long __init early_calculate_totalpages(void) 7166 { 7167 unsigned long totalpages = 0; 7168 unsigned long start_pfn, end_pfn; 7169 int i, nid; 7170 7171 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7172 unsigned long pages = end_pfn - start_pfn; 7173 7174 totalpages += pages; 7175 if (pages) 7176 node_set_state(nid, N_MEMORY); 7177 } 7178 return totalpages; 7179 } 7180 7181 /* 7182 * Find the PFN the Movable zone begins in each node. Kernel memory 7183 * is spread evenly between nodes as long as the nodes have enough 7184 * memory. When they don't, some nodes will have more kernelcore than 7185 * others 7186 */ 7187 static void __init find_zone_movable_pfns_for_nodes(void) 7188 { 7189 int i, nid; 7190 unsigned long usable_startpfn; 7191 unsigned long kernelcore_node, kernelcore_remaining; 7192 /* save the state before borrow the nodemask */ 7193 nodemask_t saved_node_state = node_states[N_MEMORY]; 7194 unsigned long totalpages = early_calculate_totalpages(); 7195 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 7196 struct memblock_region *r; 7197 7198 /* Need to find movable_zone earlier when movable_node is specified. */ 7199 find_usable_zone_for_movable(); 7200 7201 /* 7202 * If movable_node is specified, ignore kernelcore and movablecore 7203 * options. 7204 */ 7205 if (movable_node_is_enabled()) { 7206 for_each_memblock(memory, r) { 7207 if (!memblock_is_hotpluggable(r)) 7208 continue; 7209 7210 nid = r->nid; 7211 7212 usable_startpfn = PFN_DOWN(r->base); 7213 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7214 min(usable_startpfn, zone_movable_pfn[nid]) : 7215 usable_startpfn; 7216 } 7217 7218 goto out2; 7219 } 7220 7221 /* 7222 * If kernelcore=mirror is specified, ignore movablecore option 7223 */ 7224 if (mirrored_kernelcore) { 7225 bool mem_below_4gb_not_mirrored = false; 7226 7227 for_each_memblock(memory, r) { 7228 if (memblock_is_mirror(r)) 7229 continue; 7230 7231 nid = r->nid; 7232 7233 usable_startpfn = memblock_region_memory_base_pfn(r); 7234 7235 if (usable_startpfn < 0x100000) { 7236 mem_below_4gb_not_mirrored = true; 7237 continue; 7238 } 7239 7240 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7241 min(usable_startpfn, zone_movable_pfn[nid]) : 7242 usable_startpfn; 7243 } 7244 7245 if (mem_below_4gb_not_mirrored) 7246 pr_warn("This configuration results in unmirrored kernel memory."); 7247 7248 goto out2; 7249 } 7250 7251 /* 7252 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 7253 * amount of necessary memory. 7254 */ 7255 if (required_kernelcore_percent) 7256 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 7257 10000UL; 7258 if (required_movablecore_percent) 7259 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 7260 10000UL; 7261 7262 /* 7263 * If movablecore= was specified, calculate what size of 7264 * kernelcore that corresponds so that memory usable for 7265 * any allocation type is evenly spread. If both kernelcore 7266 * and movablecore are specified, then the value of kernelcore 7267 * will be used for required_kernelcore if it's greater than 7268 * what movablecore would have allowed. 7269 */ 7270 if (required_movablecore) { 7271 unsigned long corepages; 7272 7273 /* 7274 * Round-up so that ZONE_MOVABLE is at least as large as what 7275 * was requested by the user 7276 */ 7277 required_movablecore = 7278 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 7279 required_movablecore = min(totalpages, required_movablecore); 7280 corepages = totalpages - required_movablecore; 7281 7282 required_kernelcore = max(required_kernelcore, corepages); 7283 } 7284 7285 /* 7286 * If kernelcore was not specified or kernelcore size is larger 7287 * than totalpages, there is no ZONE_MOVABLE. 7288 */ 7289 if (!required_kernelcore || required_kernelcore >= totalpages) 7290 goto out; 7291 7292 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 7293 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 7294 7295 restart: 7296 /* Spread kernelcore memory as evenly as possible throughout nodes */ 7297 kernelcore_node = required_kernelcore / usable_nodes; 7298 for_each_node_state(nid, N_MEMORY) { 7299 unsigned long start_pfn, end_pfn; 7300 7301 /* 7302 * Recalculate kernelcore_node if the division per node 7303 * now exceeds what is necessary to satisfy the requested 7304 * amount of memory for the kernel 7305 */ 7306 if (required_kernelcore < kernelcore_node) 7307 kernelcore_node = required_kernelcore / usable_nodes; 7308 7309 /* 7310 * As the map is walked, we track how much memory is usable 7311 * by the kernel using kernelcore_remaining. When it is 7312 * 0, the rest of the node is usable by ZONE_MOVABLE 7313 */ 7314 kernelcore_remaining = kernelcore_node; 7315 7316 /* Go through each range of PFNs within this node */ 7317 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7318 unsigned long size_pages; 7319 7320 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 7321 if (start_pfn >= end_pfn) 7322 continue; 7323 7324 /* Account for what is only usable for kernelcore */ 7325 if (start_pfn < usable_startpfn) { 7326 unsigned long kernel_pages; 7327 kernel_pages = min(end_pfn, usable_startpfn) 7328 - start_pfn; 7329 7330 kernelcore_remaining -= min(kernel_pages, 7331 kernelcore_remaining); 7332 required_kernelcore -= min(kernel_pages, 7333 required_kernelcore); 7334 7335 /* Continue if range is now fully accounted */ 7336 if (end_pfn <= usable_startpfn) { 7337 7338 /* 7339 * Push zone_movable_pfn to the end so 7340 * that if we have to rebalance 7341 * kernelcore across nodes, we will 7342 * not double account here 7343 */ 7344 zone_movable_pfn[nid] = end_pfn; 7345 continue; 7346 } 7347 start_pfn = usable_startpfn; 7348 } 7349 7350 /* 7351 * The usable PFN range for ZONE_MOVABLE is from 7352 * start_pfn->end_pfn. Calculate size_pages as the 7353 * number of pages used as kernelcore 7354 */ 7355 size_pages = end_pfn - start_pfn; 7356 if (size_pages > kernelcore_remaining) 7357 size_pages = kernelcore_remaining; 7358 zone_movable_pfn[nid] = start_pfn + size_pages; 7359 7360 /* 7361 * Some kernelcore has been met, update counts and 7362 * break if the kernelcore for this node has been 7363 * satisfied 7364 */ 7365 required_kernelcore -= min(required_kernelcore, 7366 size_pages); 7367 kernelcore_remaining -= size_pages; 7368 if (!kernelcore_remaining) 7369 break; 7370 } 7371 } 7372 7373 /* 7374 * If there is still required_kernelcore, we do another pass with one 7375 * less node in the count. This will push zone_movable_pfn[nid] further 7376 * along on the nodes that still have memory until kernelcore is 7377 * satisfied 7378 */ 7379 usable_nodes--; 7380 if (usable_nodes && required_kernelcore > usable_nodes) 7381 goto restart; 7382 7383 out2: 7384 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 7385 for (nid = 0; nid < MAX_NUMNODES; nid++) 7386 zone_movable_pfn[nid] = 7387 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 7388 7389 out: 7390 /* restore the node_state */ 7391 node_states[N_MEMORY] = saved_node_state; 7392 } 7393 7394 /* Any regular or high memory on that node ? */ 7395 static void check_for_memory(pg_data_t *pgdat, int nid) 7396 { 7397 enum zone_type zone_type; 7398 7399 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 7400 struct zone *zone = &pgdat->node_zones[zone_type]; 7401 if (populated_zone(zone)) { 7402 if (IS_ENABLED(CONFIG_HIGHMEM)) 7403 node_set_state(nid, N_HIGH_MEMORY); 7404 if (zone_type <= ZONE_NORMAL) 7405 node_set_state(nid, N_NORMAL_MEMORY); 7406 break; 7407 } 7408 } 7409 } 7410 7411 /** 7412 * free_area_init_nodes - Initialise all pg_data_t and zone data 7413 * @max_zone_pfn: an array of max PFNs for each zone 7414 * 7415 * This will call free_area_init_node() for each active node in the system. 7416 * Using the page ranges provided by memblock_set_node(), the size of each 7417 * zone in each node and their holes is calculated. If the maximum PFN 7418 * between two adjacent zones match, it is assumed that the zone is empty. 7419 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 7420 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 7421 * starts where the previous one ended. For example, ZONE_DMA32 starts 7422 * at arch_max_dma_pfn. 7423 */ 7424 void __init free_area_init_nodes(unsigned long *max_zone_pfn) 7425 { 7426 unsigned long start_pfn, end_pfn; 7427 int i, nid; 7428 7429 /* Record where the zone boundaries are */ 7430 memset(arch_zone_lowest_possible_pfn, 0, 7431 sizeof(arch_zone_lowest_possible_pfn)); 7432 memset(arch_zone_highest_possible_pfn, 0, 7433 sizeof(arch_zone_highest_possible_pfn)); 7434 7435 start_pfn = find_min_pfn_with_active_regions(); 7436 7437 for (i = 0; i < MAX_NR_ZONES; i++) { 7438 if (i == ZONE_MOVABLE) 7439 continue; 7440 7441 end_pfn = max(max_zone_pfn[i], start_pfn); 7442 arch_zone_lowest_possible_pfn[i] = start_pfn; 7443 arch_zone_highest_possible_pfn[i] = end_pfn; 7444 7445 start_pfn = end_pfn; 7446 } 7447 7448 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 7449 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 7450 find_zone_movable_pfns_for_nodes(); 7451 7452 /* Print out the zone ranges */ 7453 pr_info("Zone ranges:\n"); 7454 for (i = 0; i < MAX_NR_ZONES; i++) { 7455 if (i == ZONE_MOVABLE) 7456 continue; 7457 pr_info(" %-8s ", zone_names[i]); 7458 if (arch_zone_lowest_possible_pfn[i] == 7459 arch_zone_highest_possible_pfn[i]) 7460 pr_cont("empty\n"); 7461 else 7462 pr_cont("[mem %#018Lx-%#018Lx]\n", 7463 (u64)arch_zone_lowest_possible_pfn[i] 7464 << PAGE_SHIFT, 7465 ((u64)arch_zone_highest_possible_pfn[i] 7466 << PAGE_SHIFT) - 1); 7467 } 7468 7469 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 7470 pr_info("Movable zone start for each node\n"); 7471 for (i = 0; i < MAX_NUMNODES; i++) { 7472 if (zone_movable_pfn[i]) 7473 pr_info(" Node %d: %#018Lx\n", i, 7474 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 7475 } 7476 7477 /* 7478 * Print out the early node map, and initialize the 7479 * subsection-map relative to active online memory ranges to 7480 * enable future "sub-section" extensions of the memory map. 7481 */ 7482 pr_info("Early memory node ranges\n"); 7483 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7484 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 7485 (u64)start_pfn << PAGE_SHIFT, 7486 ((u64)end_pfn << PAGE_SHIFT) - 1); 7487 subsection_map_init(start_pfn, end_pfn - start_pfn); 7488 } 7489 7490 /* Initialise every node */ 7491 mminit_verify_pageflags_layout(); 7492 setup_nr_node_ids(); 7493 init_unavailable_mem(); 7494 for_each_online_node(nid) { 7495 pg_data_t *pgdat = NODE_DATA(nid); 7496 free_area_init_node(nid, NULL, 7497 find_min_pfn_for_node(nid), NULL); 7498 7499 /* Any memory on that node */ 7500 if (pgdat->node_present_pages) 7501 node_set_state(nid, N_MEMORY); 7502 check_for_memory(pgdat, nid); 7503 } 7504 } 7505 7506 static int __init cmdline_parse_core(char *p, unsigned long *core, 7507 unsigned long *percent) 7508 { 7509 unsigned long long coremem; 7510 char *endptr; 7511 7512 if (!p) 7513 return -EINVAL; 7514 7515 /* Value may be a percentage of total memory, otherwise bytes */ 7516 coremem = simple_strtoull(p, &endptr, 0); 7517 if (*endptr == '%') { 7518 /* Paranoid check for percent values greater than 100 */ 7519 WARN_ON(coremem > 100); 7520 7521 *percent = coremem; 7522 } else { 7523 coremem = memparse(p, &p); 7524 /* Paranoid check that UL is enough for the coremem value */ 7525 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 7526 7527 *core = coremem >> PAGE_SHIFT; 7528 *percent = 0UL; 7529 } 7530 return 0; 7531 } 7532 7533 /* 7534 * kernelcore=size sets the amount of memory for use for allocations that 7535 * cannot be reclaimed or migrated. 7536 */ 7537 static int __init cmdline_parse_kernelcore(char *p) 7538 { 7539 /* parse kernelcore=mirror */ 7540 if (parse_option_str(p, "mirror")) { 7541 mirrored_kernelcore = true; 7542 return 0; 7543 } 7544 7545 return cmdline_parse_core(p, &required_kernelcore, 7546 &required_kernelcore_percent); 7547 } 7548 7549 /* 7550 * movablecore=size sets the amount of memory for use for allocations that 7551 * can be reclaimed or migrated. 7552 */ 7553 static int __init cmdline_parse_movablecore(char *p) 7554 { 7555 return cmdline_parse_core(p, &required_movablecore, 7556 &required_movablecore_percent); 7557 } 7558 7559 early_param("kernelcore", cmdline_parse_kernelcore); 7560 early_param("movablecore", cmdline_parse_movablecore); 7561 7562 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 7563 7564 void adjust_managed_page_count(struct page *page, long count) 7565 { 7566 atomic_long_add(count, &page_zone(page)->managed_pages); 7567 totalram_pages_add(count); 7568 #ifdef CONFIG_HIGHMEM 7569 if (PageHighMem(page)) 7570 totalhigh_pages_add(count); 7571 #endif 7572 } 7573 EXPORT_SYMBOL(adjust_managed_page_count); 7574 7575 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 7576 { 7577 void *pos; 7578 unsigned long pages = 0; 7579 7580 start = (void *)PAGE_ALIGN((unsigned long)start); 7581 end = (void *)((unsigned long)end & PAGE_MASK); 7582 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 7583 struct page *page = virt_to_page(pos); 7584 void *direct_map_addr; 7585 7586 /* 7587 * 'direct_map_addr' might be different from 'pos' 7588 * because some architectures' virt_to_page() 7589 * work with aliases. Getting the direct map 7590 * address ensures that we get a _writeable_ 7591 * alias for the memset(). 7592 */ 7593 direct_map_addr = page_address(page); 7594 if ((unsigned int)poison <= 0xFF) 7595 memset(direct_map_addr, poison, PAGE_SIZE); 7596 7597 free_reserved_page(page); 7598 } 7599 7600 if (pages && s) 7601 pr_info("Freeing %s memory: %ldK\n", 7602 s, pages << (PAGE_SHIFT - 10)); 7603 7604 return pages; 7605 } 7606 7607 #ifdef CONFIG_HIGHMEM 7608 void free_highmem_page(struct page *page) 7609 { 7610 __free_reserved_page(page); 7611 totalram_pages_inc(); 7612 atomic_long_inc(&page_zone(page)->managed_pages); 7613 totalhigh_pages_inc(); 7614 } 7615 #endif 7616 7617 7618 void __init mem_init_print_info(const char *str) 7619 { 7620 unsigned long physpages, codesize, datasize, rosize, bss_size; 7621 unsigned long init_code_size, init_data_size; 7622 7623 physpages = get_num_physpages(); 7624 codesize = _etext - _stext; 7625 datasize = _edata - _sdata; 7626 rosize = __end_rodata - __start_rodata; 7627 bss_size = __bss_stop - __bss_start; 7628 init_data_size = __init_end - __init_begin; 7629 init_code_size = _einittext - _sinittext; 7630 7631 /* 7632 * Detect special cases and adjust section sizes accordingly: 7633 * 1) .init.* may be embedded into .data sections 7634 * 2) .init.text.* may be out of [__init_begin, __init_end], 7635 * please refer to arch/tile/kernel/vmlinux.lds.S. 7636 * 3) .rodata.* may be embedded into .text or .data sections. 7637 */ 7638 #define adj_init_size(start, end, size, pos, adj) \ 7639 do { \ 7640 if (start <= pos && pos < end && size > adj) \ 7641 size -= adj; \ 7642 } while (0) 7643 7644 adj_init_size(__init_begin, __init_end, init_data_size, 7645 _sinittext, init_code_size); 7646 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 7647 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 7648 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 7649 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 7650 7651 #undef adj_init_size 7652 7653 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 7654 #ifdef CONFIG_HIGHMEM 7655 ", %luK highmem" 7656 #endif 7657 "%s%s)\n", 7658 nr_free_pages() << (PAGE_SHIFT - 10), 7659 physpages << (PAGE_SHIFT - 10), 7660 codesize >> 10, datasize >> 10, rosize >> 10, 7661 (init_data_size + init_code_size) >> 10, bss_size >> 10, 7662 (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10), 7663 totalcma_pages << (PAGE_SHIFT - 10), 7664 #ifdef CONFIG_HIGHMEM 7665 totalhigh_pages() << (PAGE_SHIFT - 10), 7666 #endif 7667 str ? ", " : "", str ? str : ""); 7668 } 7669 7670 /** 7671 * set_dma_reserve - set the specified number of pages reserved in the first zone 7672 * @new_dma_reserve: The number of pages to mark reserved 7673 * 7674 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 7675 * In the DMA zone, a significant percentage may be consumed by kernel image 7676 * and other unfreeable allocations which can skew the watermarks badly. This 7677 * function may optionally be used to account for unfreeable pages in the 7678 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 7679 * smaller per-cpu batchsize. 7680 */ 7681 void __init set_dma_reserve(unsigned long new_dma_reserve) 7682 { 7683 dma_reserve = new_dma_reserve; 7684 } 7685 7686 void __init free_area_init(unsigned long *zones_size) 7687 { 7688 init_unavailable_mem(); 7689 free_area_init_node(0, zones_size, 7690 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); 7691 } 7692 7693 static int page_alloc_cpu_dead(unsigned int cpu) 7694 { 7695 7696 lru_add_drain_cpu(cpu); 7697 drain_pages(cpu); 7698 7699 /* 7700 * Spill the event counters of the dead processor 7701 * into the current processors event counters. 7702 * This artificially elevates the count of the current 7703 * processor. 7704 */ 7705 vm_events_fold_cpu(cpu); 7706 7707 /* 7708 * Zero the differential counters of the dead processor 7709 * so that the vm statistics are consistent. 7710 * 7711 * This is only okay since the processor is dead and cannot 7712 * race with what we are doing. 7713 */ 7714 cpu_vm_stats_fold(cpu); 7715 return 0; 7716 } 7717 7718 #ifdef CONFIG_NUMA 7719 int hashdist = HASHDIST_DEFAULT; 7720 7721 static int __init set_hashdist(char *str) 7722 { 7723 if (!str) 7724 return 0; 7725 hashdist = simple_strtoul(str, &str, 0); 7726 return 1; 7727 } 7728 __setup("hashdist=", set_hashdist); 7729 #endif 7730 7731 void __init page_alloc_init(void) 7732 { 7733 int ret; 7734 7735 #ifdef CONFIG_NUMA 7736 if (num_node_state(N_MEMORY) == 1) 7737 hashdist = 0; 7738 #endif 7739 7740 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD, 7741 "mm/page_alloc:dead", NULL, 7742 page_alloc_cpu_dead); 7743 WARN_ON(ret < 0); 7744 } 7745 7746 /* 7747 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 7748 * or min_free_kbytes changes. 7749 */ 7750 static void calculate_totalreserve_pages(void) 7751 { 7752 struct pglist_data *pgdat; 7753 unsigned long reserve_pages = 0; 7754 enum zone_type i, j; 7755 7756 for_each_online_pgdat(pgdat) { 7757 7758 pgdat->totalreserve_pages = 0; 7759 7760 for (i = 0; i < MAX_NR_ZONES; i++) { 7761 struct zone *zone = pgdat->node_zones + i; 7762 long max = 0; 7763 unsigned long managed_pages = zone_managed_pages(zone); 7764 7765 /* Find valid and maximum lowmem_reserve in the zone */ 7766 for (j = i; j < MAX_NR_ZONES; j++) { 7767 if (zone->lowmem_reserve[j] > max) 7768 max = zone->lowmem_reserve[j]; 7769 } 7770 7771 /* we treat the high watermark as reserved pages. */ 7772 max += high_wmark_pages(zone); 7773 7774 if (max > managed_pages) 7775 max = managed_pages; 7776 7777 pgdat->totalreserve_pages += max; 7778 7779 reserve_pages += max; 7780 } 7781 } 7782 totalreserve_pages = reserve_pages; 7783 } 7784 7785 /* 7786 * setup_per_zone_lowmem_reserve - called whenever 7787 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 7788 * has a correct pages reserved value, so an adequate number of 7789 * pages are left in the zone after a successful __alloc_pages(). 7790 */ 7791 static void setup_per_zone_lowmem_reserve(void) 7792 { 7793 struct pglist_data *pgdat; 7794 enum zone_type j, idx; 7795 7796 for_each_online_pgdat(pgdat) { 7797 for (j = 0; j < MAX_NR_ZONES; j++) { 7798 struct zone *zone = pgdat->node_zones + j; 7799 unsigned long managed_pages = zone_managed_pages(zone); 7800 7801 zone->lowmem_reserve[j] = 0; 7802 7803 idx = j; 7804 while (idx) { 7805 struct zone *lower_zone; 7806 7807 idx--; 7808 lower_zone = pgdat->node_zones + idx; 7809 7810 if (sysctl_lowmem_reserve_ratio[idx] < 1) { 7811 sysctl_lowmem_reserve_ratio[idx] = 0; 7812 lower_zone->lowmem_reserve[j] = 0; 7813 } else { 7814 lower_zone->lowmem_reserve[j] = 7815 managed_pages / sysctl_lowmem_reserve_ratio[idx]; 7816 } 7817 managed_pages += zone_managed_pages(lower_zone); 7818 } 7819 } 7820 } 7821 7822 /* update totalreserve_pages */ 7823 calculate_totalreserve_pages(); 7824 } 7825 7826 static void __setup_per_zone_wmarks(void) 7827 { 7828 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 7829 unsigned long lowmem_pages = 0; 7830 struct zone *zone; 7831 unsigned long flags; 7832 7833 /* Calculate total number of !ZONE_HIGHMEM pages */ 7834 for_each_zone(zone) { 7835 if (!is_highmem(zone)) 7836 lowmem_pages += zone_managed_pages(zone); 7837 } 7838 7839 for_each_zone(zone) { 7840 u64 tmp; 7841 7842 spin_lock_irqsave(&zone->lock, flags); 7843 tmp = (u64)pages_min * zone_managed_pages(zone); 7844 do_div(tmp, lowmem_pages); 7845 if (is_highmem(zone)) { 7846 /* 7847 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 7848 * need highmem pages, so cap pages_min to a small 7849 * value here. 7850 * 7851 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 7852 * deltas control async page reclaim, and so should 7853 * not be capped for highmem. 7854 */ 7855 unsigned long min_pages; 7856 7857 min_pages = zone_managed_pages(zone) / 1024; 7858 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 7859 zone->_watermark[WMARK_MIN] = min_pages; 7860 } else { 7861 /* 7862 * If it's a lowmem zone, reserve a number of pages 7863 * proportionate to the zone's size. 7864 */ 7865 zone->_watermark[WMARK_MIN] = tmp; 7866 } 7867 7868 /* 7869 * Set the kswapd watermarks distance according to the 7870 * scale factor in proportion to available memory, but 7871 * ensure a minimum size on small systems. 7872 */ 7873 tmp = max_t(u64, tmp >> 2, 7874 mult_frac(zone_managed_pages(zone), 7875 watermark_scale_factor, 10000)); 7876 7877 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 7878 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2; 7879 zone->watermark_boost = 0; 7880 7881 spin_unlock_irqrestore(&zone->lock, flags); 7882 } 7883 7884 /* update totalreserve_pages */ 7885 calculate_totalreserve_pages(); 7886 } 7887 7888 /** 7889 * setup_per_zone_wmarks - called when min_free_kbytes changes 7890 * or when memory is hot-{added|removed} 7891 * 7892 * Ensures that the watermark[min,low,high] values for each zone are set 7893 * correctly with respect to min_free_kbytes. 7894 */ 7895 void setup_per_zone_wmarks(void) 7896 { 7897 static DEFINE_SPINLOCK(lock); 7898 7899 spin_lock(&lock); 7900 __setup_per_zone_wmarks(); 7901 spin_unlock(&lock); 7902 } 7903 7904 /* 7905 * Initialise min_free_kbytes. 7906 * 7907 * For small machines we want it small (128k min). For large machines 7908 * we want it large (64MB max). But it is not linear, because network 7909 * bandwidth does not increase linearly with machine size. We use 7910 * 7911 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 7912 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 7913 * 7914 * which yields 7915 * 7916 * 16MB: 512k 7917 * 32MB: 724k 7918 * 64MB: 1024k 7919 * 128MB: 1448k 7920 * 256MB: 2048k 7921 * 512MB: 2896k 7922 * 1024MB: 4096k 7923 * 2048MB: 5792k 7924 * 4096MB: 8192k 7925 * 8192MB: 11584k 7926 * 16384MB: 16384k 7927 */ 7928 int __meminit init_per_zone_wmark_min(void) 7929 { 7930 unsigned long lowmem_kbytes; 7931 int new_min_free_kbytes; 7932 7933 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 7934 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 7935 7936 if (new_min_free_kbytes > user_min_free_kbytes) { 7937 min_free_kbytes = new_min_free_kbytes; 7938 if (min_free_kbytes < 128) 7939 min_free_kbytes = 128; 7940 if (min_free_kbytes > 262144) 7941 min_free_kbytes = 262144; 7942 } else { 7943 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 7944 new_min_free_kbytes, user_min_free_kbytes); 7945 } 7946 setup_per_zone_wmarks(); 7947 refresh_zone_stat_thresholds(); 7948 setup_per_zone_lowmem_reserve(); 7949 7950 #ifdef CONFIG_NUMA 7951 setup_min_unmapped_ratio(); 7952 setup_min_slab_ratio(); 7953 #endif 7954 7955 return 0; 7956 } 7957 core_initcall(init_per_zone_wmark_min) 7958 7959 /* 7960 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 7961 * that we can call two helper functions whenever min_free_kbytes 7962 * changes. 7963 */ 7964 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 7965 void __user *buffer, size_t *length, loff_t *ppos) 7966 { 7967 int rc; 7968 7969 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7970 if (rc) 7971 return rc; 7972 7973 if (write) { 7974 user_min_free_kbytes = min_free_kbytes; 7975 setup_per_zone_wmarks(); 7976 } 7977 return 0; 7978 } 7979 7980 int watermark_boost_factor_sysctl_handler(struct ctl_table *table, int write, 7981 void __user *buffer, size_t *length, loff_t *ppos) 7982 { 7983 int rc; 7984 7985 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7986 if (rc) 7987 return rc; 7988 7989 return 0; 7990 } 7991 7992 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 7993 void __user *buffer, size_t *length, loff_t *ppos) 7994 { 7995 int rc; 7996 7997 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7998 if (rc) 7999 return rc; 8000 8001 if (write) 8002 setup_per_zone_wmarks(); 8003 8004 return 0; 8005 } 8006 8007 #ifdef CONFIG_NUMA 8008 static void setup_min_unmapped_ratio(void) 8009 { 8010 pg_data_t *pgdat; 8011 struct zone *zone; 8012 8013 for_each_online_pgdat(pgdat) 8014 pgdat->min_unmapped_pages = 0; 8015 8016 for_each_zone(zone) 8017 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * 8018 sysctl_min_unmapped_ratio) / 100; 8019 } 8020 8021 8022 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 8023 void __user *buffer, size_t *length, loff_t *ppos) 8024 { 8025 int rc; 8026 8027 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8028 if (rc) 8029 return rc; 8030 8031 setup_min_unmapped_ratio(); 8032 8033 return 0; 8034 } 8035 8036 static void setup_min_slab_ratio(void) 8037 { 8038 pg_data_t *pgdat; 8039 struct zone *zone; 8040 8041 for_each_online_pgdat(pgdat) 8042 pgdat->min_slab_pages = 0; 8043 8044 for_each_zone(zone) 8045 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * 8046 sysctl_min_slab_ratio) / 100; 8047 } 8048 8049 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 8050 void __user *buffer, size_t *length, loff_t *ppos) 8051 { 8052 int rc; 8053 8054 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8055 if (rc) 8056 return rc; 8057 8058 setup_min_slab_ratio(); 8059 8060 return 0; 8061 } 8062 #endif 8063 8064 /* 8065 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 8066 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 8067 * whenever sysctl_lowmem_reserve_ratio changes. 8068 * 8069 * The reserve ratio obviously has absolutely no relation with the 8070 * minimum watermarks. The lowmem reserve ratio can only make sense 8071 * if in function of the boot time zone sizes. 8072 */ 8073 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 8074 void __user *buffer, size_t *length, loff_t *ppos) 8075 { 8076 proc_dointvec_minmax(table, write, buffer, length, ppos); 8077 setup_per_zone_lowmem_reserve(); 8078 return 0; 8079 } 8080 8081 static void __zone_pcp_update(struct zone *zone) 8082 { 8083 unsigned int cpu; 8084 8085 for_each_possible_cpu(cpu) 8086 pageset_set_high_and_batch(zone, 8087 per_cpu_ptr(zone->pageset, cpu)); 8088 } 8089 8090 /* 8091 * percpu_pagelist_fraction - changes the pcp->high for each zone on each 8092 * cpu. It is the fraction of total pages in each zone that a hot per cpu 8093 * pagelist can have before it gets flushed back to buddy allocator. 8094 */ 8095 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, 8096 void __user *buffer, size_t *length, loff_t *ppos) 8097 { 8098 struct zone *zone; 8099 int old_percpu_pagelist_fraction; 8100 int ret; 8101 8102 mutex_lock(&pcp_batch_high_lock); 8103 old_percpu_pagelist_fraction = percpu_pagelist_fraction; 8104 8105 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 8106 if (!write || ret < 0) 8107 goto out; 8108 8109 /* Sanity checking to avoid pcp imbalance */ 8110 if (percpu_pagelist_fraction && 8111 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { 8112 percpu_pagelist_fraction = old_percpu_pagelist_fraction; 8113 ret = -EINVAL; 8114 goto out; 8115 } 8116 8117 /* No change? */ 8118 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) 8119 goto out; 8120 8121 for_each_populated_zone(zone) 8122 __zone_pcp_update(zone); 8123 out: 8124 mutex_unlock(&pcp_batch_high_lock); 8125 return ret; 8126 } 8127 8128 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 8129 /* 8130 * Returns the number of pages that arch has reserved but 8131 * is not known to alloc_large_system_hash(). 8132 */ 8133 static unsigned long __init arch_reserved_kernel_pages(void) 8134 { 8135 return 0; 8136 } 8137 #endif 8138 8139 /* 8140 * Adaptive scale is meant to reduce sizes of hash tables on large memory 8141 * machines. As memory size is increased the scale is also increased but at 8142 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 8143 * quadruples the scale is increased by one, which means the size of hash table 8144 * only doubles, instead of quadrupling as well. 8145 * Because 32-bit systems cannot have large physical memory, where this scaling 8146 * makes sense, it is disabled on such platforms. 8147 */ 8148 #if __BITS_PER_LONG > 32 8149 #define ADAPT_SCALE_BASE (64ul << 30) 8150 #define ADAPT_SCALE_SHIFT 2 8151 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 8152 #endif 8153 8154 /* 8155 * allocate a large system hash table from bootmem 8156 * - it is assumed that the hash table must contain an exact power-of-2 8157 * quantity of entries 8158 * - limit is the number of hash buckets, not the total allocation size 8159 */ 8160 void *__init alloc_large_system_hash(const char *tablename, 8161 unsigned long bucketsize, 8162 unsigned long numentries, 8163 int scale, 8164 int flags, 8165 unsigned int *_hash_shift, 8166 unsigned int *_hash_mask, 8167 unsigned long low_limit, 8168 unsigned long high_limit) 8169 { 8170 unsigned long long max = high_limit; 8171 unsigned long log2qty, size; 8172 void *table = NULL; 8173 gfp_t gfp_flags; 8174 bool virt; 8175 8176 /* allow the kernel cmdline to have a say */ 8177 if (!numentries) { 8178 /* round applicable memory size up to nearest megabyte */ 8179 numentries = nr_kernel_pages; 8180 numentries -= arch_reserved_kernel_pages(); 8181 8182 /* It isn't necessary when PAGE_SIZE >= 1MB */ 8183 if (PAGE_SHIFT < 20) 8184 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 8185 8186 #if __BITS_PER_LONG > 32 8187 if (!high_limit) { 8188 unsigned long adapt; 8189 8190 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 8191 adapt <<= ADAPT_SCALE_SHIFT) 8192 scale++; 8193 } 8194 #endif 8195 8196 /* limit to 1 bucket per 2^scale bytes of low memory */ 8197 if (scale > PAGE_SHIFT) 8198 numentries >>= (scale - PAGE_SHIFT); 8199 else 8200 numentries <<= (PAGE_SHIFT - scale); 8201 8202 /* Make sure we've got at least a 0-order allocation.. */ 8203 if (unlikely(flags & HASH_SMALL)) { 8204 /* Makes no sense without HASH_EARLY */ 8205 WARN_ON(!(flags & HASH_EARLY)); 8206 if (!(numentries >> *_hash_shift)) { 8207 numentries = 1UL << *_hash_shift; 8208 BUG_ON(!numentries); 8209 } 8210 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 8211 numentries = PAGE_SIZE / bucketsize; 8212 } 8213 numentries = roundup_pow_of_two(numentries); 8214 8215 /* limit allocation size to 1/16 total memory by default */ 8216 if (max == 0) { 8217 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 8218 do_div(max, bucketsize); 8219 } 8220 max = min(max, 0x80000000ULL); 8221 8222 if (numentries < low_limit) 8223 numentries = low_limit; 8224 if (numentries > max) 8225 numentries = max; 8226 8227 log2qty = ilog2(numentries); 8228 8229 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 8230 do { 8231 virt = false; 8232 size = bucketsize << log2qty; 8233 if (flags & HASH_EARLY) { 8234 if (flags & HASH_ZERO) 8235 table = memblock_alloc(size, SMP_CACHE_BYTES); 8236 else 8237 table = memblock_alloc_raw(size, 8238 SMP_CACHE_BYTES); 8239 } else if (get_order(size) >= MAX_ORDER || hashdist) { 8240 table = __vmalloc(size, gfp_flags, PAGE_KERNEL); 8241 virt = true; 8242 } else { 8243 /* 8244 * If bucketsize is not a power-of-two, we may free 8245 * some pages at the end of hash table which 8246 * alloc_pages_exact() automatically does 8247 */ 8248 table = alloc_pages_exact(size, gfp_flags); 8249 kmemleak_alloc(table, size, 1, gfp_flags); 8250 } 8251 } while (!table && size > PAGE_SIZE && --log2qty); 8252 8253 if (!table) 8254 panic("Failed to allocate %s hash table\n", tablename); 8255 8256 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 8257 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, 8258 virt ? "vmalloc" : "linear"); 8259 8260 if (_hash_shift) 8261 *_hash_shift = log2qty; 8262 if (_hash_mask) 8263 *_hash_mask = (1 << log2qty) - 1; 8264 8265 return table; 8266 } 8267 8268 /* 8269 * This function checks whether pageblock includes unmovable pages or not. 8270 * 8271 * PageLRU check without isolation or lru_lock could race so that 8272 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable 8273 * check without lock_page also may miss some movable non-lru pages at 8274 * race condition. So you can't expect this function should be exact. 8275 * 8276 * Returns a page without holding a reference. If the caller wants to 8277 * dereference that page (e.g., dumping), it has to make sure that that it 8278 * cannot get removed (e.g., via memory unplug) concurrently. 8279 * 8280 */ 8281 struct page *has_unmovable_pages(struct zone *zone, struct page *page, 8282 int migratetype, int flags) 8283 { 8284 unsigned long iter = 0; 8285 unsigned long pfn = page_to_pfn(page); 8286 8287 /* 8288 * TODO we could make this much more efficient by not checking every 8289 * page in the range if we know all of them are in MOVABLE_ZONE and 8290 * that the movable zone guarantees that pages are migratable but 8291 * the later is not the case right now unfortunatelly. E.g. movablecore 8292 * can still lead to having bootmem allocations in zone_movable. 8293 */ 8294 8295 if (is_migrate_cma_page(page)) { 8296 /* 8297 * CMA allocations (alloc_contig_range) really need to mark 8298 * isolate CMA pageblocks even when they are not movable in fact 8299 * so consider them movable here. 8300 */ 8301 if (is_migrate_cma(migratetype)) 8302 return NULL; 8303 8304 return page; 8305 } 8306 8307 for (; iter < pageblock_nr_pages; iter++) { 8308 if (!pfn_valid_within(pfn + iter)) 8309 continue; 8310 8311 page = pfn_to_page(pfn + iter); 8312 8313 if (PageReserved(page)) 8314 return page; 8315 8316 /* 8317 * If the zone is movable and we have ruled out all reserved 8318 * pages then it should be reasonably safe to assume the rest 8319 * is movable. 8320 */ 8321 if (zone_idx(zone) == ZONE_MOVABLE) 8322 continue; 8323 8324 /* 8325 * Hugepages are not in LRU lists, but they're movable. 8326 * THPs are on the LRU, but need to be counted as #small pages. 8327 * We need not scan over tail pages because we don't 8328 * handle each tail page individually in migration. 8329 */ 8330 if (PageHuge(page) || PageTransCompound(page)) { 8331 struct page *head = compound_head(page); 8332 unsigned int skip_pages; 8333 8334 if (PageHuge(page)) { 8335 if (!hugepage_migration_supported(page_hstate(head))) 8336 return page; 8337 } else if (!PageLRU(head) && !__PageMovable(head)) { 8338 return page; 8339 } 8340 8341 skip_pages = compound_nr(head) - (page - head); 8342 iter += skip_pages - 1; 8343 continue; 8344 } 8345 8346 /* 8347 * We can't use page_count without pin a page 8348 * because another CPU can free compound page. 8349 * This check already skips compound tails of THP 8350 * because their page->_refcount is zero at all time. 8351 */ 8352 if (!page_ref_count(page)) { 8353 if (PageBuddy(page)) 8354 iter += (1 << page_order(page)) - 1; 8355 continue; 8356 } 8357 8358 /* 8359 * The HWPoisoned page may be not in buddy system, and 8360 * page_count() is not 0. 8361 */ 8362 if ((flags & MEMORY_OFFLINE) && PageHWPoison(page)) 8363 continue; 8364 8365 if (__PageMovable(page) || PageLRU(page)) 8366 continue; 8367 8368 /* 8369 * If there are RECLAIMABLE pages, we need to check 8370 * it. But now, memory offline itself doesn't call 8371 * shrink_node_slabs() and it still to be fixed. 8372 */ 8373 /* 8374 * If the page is not RAM, page_count()should be 0. 8375 * we don't need more check. This is an _used_ not-movable page. 8376 * 8377 * The problematic thing here is PG_reserved pages. PG_reserved 8378 * is set to both of a memory hole page and a _used_ kernel 8379 * page at boot. 8380 */ 8381 return page; 8382 } 8383 return NULL; 8384 } 8385 8386 #ifdef CONFIG_CONTIG_ALLOC 8387 static unsigned long pfn_max_align_down(unsigned long pfn) 8388 { 8389 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, 8390 pageblock_nr_pages) - 1); 8391 } 8392 8393 static unsigned long pfn_max_align_up(unsigned long pfn) 8394 { 8395 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, 8396 pageblock_nr_pages)); 8397 } 8398 8399 /* [start, end) must belong to a single zone. */ 8400 static int __alloc_contig_migrate_range(struct compact_control *cc, 8401 unsigned long start, unsigned long end) 8402 { 8403 /* This function is based on compact_zone() from compaction.c. */ 8404 unsigned long nr_reclaimed; 8405 unsigned long pfn = start; 8406 unsigned int tries = 0; 8407 int ret = 0; 8408 8409 migrate_prep(); 8410 8411 while (pfn < end || !list_empty(&cc->migratepages)) { 8412 if (fatal_signal_pending(current)) { 8413 ret = -EINTR; 8414 break; 8415 } 8416 8417 if (list_empty(&cc->migratepages)) { 8418 cc->nr_migratepages = 0; 8419 pfn = isolate_migratepages_range(cc, pfn, end); 8420 if (!pfn) { 8421 ret = -EINTR; 8422 break; 8423 } 8424 tries = 0; 8425 } else if (++tries == 5) { 8426 ret = ret < 0 ? ret : -EBUSY; 8427 break; 8428 } 8429 8430 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 8431 &cc->migratepages); 8432 cc->nr_migratepages -= nr_reclaimed; 8433 8434 ret = migrate_pages(&cc->migratepages, alloc_migrate_target, 8435 NULL, 0, cc->mode, MR_CONTIG_RANGE); 8436 } 8437 if (ret < 0) { 8438 putback_movable_pages(&cc->migratepages); 8439 return ret; 8440 } 8441 return 0; 8442 } 8443 8444 /** 8445 * alloc_contig_range() -- tries to allocate given range of pages 8446 * @start: start PFN to allocate 8447 * @end: one-past-the-last PFN to allocate 8448 * @migratetype: migratetype of the underlaying pageblocks (either 8449 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 8450 * in range must have the same migratetype and it must 8451 * be either of the two. 8452 * @gfp_mask: GFP mask to use during compaction 8453 * 8454 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 8455 * aligned. The PFN range must belong to a single zone. 8456 * 8457 * The first thing this routine does is attempt to MIGRATE_ISOLATE all 8458 * pageblocks in the range. Once isolated, the pageblocks should not 8459 * be modified by others. 8460 * 8461 * Return: zero on success or negative error code. On success all 8462 * pages which PFN is in [start, end) are allocated for the caller and 8463 * need to be freed with free_contig_range(). 8464 */ 8465 int alloc_contig_range(unsigned long start, unsigned long end, 8466 unsigned migratetype, gfp_t gfp_mask) 8467 { 8468 unsigned long outer_start, outer_end; 8469 unsigned int order; 8470 int ret = 0; 8471 8472 struct compact_control cc = { 8473 .nr_migratepages = 0, 8474 .order = -1, 8475 .zone = page_zone(pfn_to_page(start)), 8476 .mode = MIGRATE_SYNC, 8477 .ignore_skip_hint = true, 8478 .no_set_skip_hint = true, 8479 .gfp_mask = current_gfp_context(gfp_mask), 8480 .alloc_contig = true, 8481 }; 8482 INIT_LIST_HEAD(&cc.migratepages); 8483 8484 /* 8485 * What we do here is we mark all pageblocks in range as 8486 * MIGRATE_ISOLATE. Because pageblock and max order pages may 8487 * have different sizes, and due to the way page allocator 8488 * work, we align the range to biggest of the two pages so 8489 * that page allocator won't try to merge buddies from 8490 * different pageblocks and change MIGRATE_ISOLATE to some 8491 * other migration type. 8492 * 8493 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 8494 * migrate the pages from an unaligned range (ie. pages that 8495 * we are interested in). This will put all the pages in 8496 * range back to page allocator as MIGRATE_ISOLATE. 8497 * 8498 * When this is done, we take the pages in range from page 8499 * allocator removing them from the buddy system. This way 8500 * page allocator will never consider using them. 8501 * 8502 * This lets us mark the pageblocks back as 8503 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 8504 * aligned range but not in the unaligned, original range are 8505 * put back to page allocator so that buddy can use them. 8506 */ 8507 8508 ret = start_isolate_page_range(pfn_max_align_down(start), 8509 pfn_max_align_up(end), migratetype, 0); 8510 if (ret < 0) 8511 return ret; 8512 8513 /* 8514 * In case of -EBUSY, we'd like to know which page causes problem. 8515 * So, just fall through. test_pages_isolated() has a tracepoint 8516 * which will report the busy page. 8517 * 8518 * It is possible that busy pages could become available before 8519 * the call to test_pages_isolated, and the range will actually be 8520 * allocated. So, if we fall through be sure to clear ret so that 8521 * -EBUSY is not accidentally used or returned to caller. 8522 */ 8523 ret = __alloc_contig_migrate_range(&cc, start, end); 8524 if (ret && ret != -EBUSY) 8525 goto done; 8526 ret =0; 8527 8528 /* 8529 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 8530 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 8531 * more, all pages in [start, end) are free in page allocator. 8532 * What we are going to do is to allocate all pages from 8533 * [start, end) (that is remove them from page allocator). 8534 * 8535 * The only problem is that pages at the beginning and at the 8536 * end of interesting range may be not aligned with pages that 8537 * page allocator holds, ie. they can be part of higher order 8538 * pages. Because of this, we reserve the bigger range and 8539 * once this is done free the pages we are not interested in. 8540 * 8541 * We don't have to hold zone->lock here because the pages are 8542 * isolated thus they won't get removed from buddy. 8543 */ 8544 8545 lru_add_drain_all(); 8546 8547 order = 0; 8548 outer_start = start; 8549 while (!PageBuddy(pfn_to_page(outer_start))) { 8550 if (++order >= MAX_ORDER) { 8551 outer_start = start; 8552 break; 8553 } 8554 outer_start &= ~0UL << order; 8555 } 8556 8557 if (outer_start != start) { 8558 order = page_order(pfn_to_page(outer_start)); 8559 8560 /* 8561 * outer_start page could be small order buddy page and 8562 * it doesn't include start page. Adjust outer_start 8563 * in this case to report failed page properly 8564 * on tracepoint in test_pages_isolated() 8565 */ 8566 if (outer_start + (1UL << order) <= start) 8567 outer_start = start; 8568 } 8569 8570 /* Make sure the range is really isolated. */ 8571 if (test_pages_isolated(outer_start, end, 0)) { 8572 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n", 8573 __func__, outer_start, end); 8574 ret = -EBUSY; 8575 goto done; 8576 } 8577 8578 /* Grab isolated pages from freelists. */ 8579 outer_end = isolate_freepages_range(&cc, outer_start, end); 8580 if (!outer_end) { 8581 ret = -EBUSY; 8582 goto done; 8583 } 8584 8585 /* Free head and tail (if any) */ 8586 if (start != outer_start) 8587 free_contig_range(outer_start, start - outer_start); 8588 if (end != outer_end) 8589 free_contig_range(end, outer_end - end); 8590 8591 done: 8592 undo_isolate_page_range(pfn_max_align_down(start), 8593 pfn_max_align_up(end), migratetype); 8594 return ret; 8595 } 8596 8597 static int __alloc_contig_pages(unsigned long start_pfn, 8598 unsigned long nr_pages, gfp_t gfp_mask) 8599 { 8600 unsigned long end_pfn = start_pfn + nr_pages; 8601 8602 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE, 8603 gfp_mask); 8604 } 8605 8606 static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, 8607 unsigned long nr_pages) 8608 { 8609 unsigned long i, end_pfn = start_pfn + nr_pages; 8610 struct page *page; 8611 8612 for (i = start_pfn; i < end_pfn; i++) { 8613 page = pfn_to_online_page(i); 8614 if (!page) 8615 return false; 8616 8617 if (page_zone(page) != z) 8618 return false; 8619 8620 if (PageReserved(page)) 8621 return false; 8622 8623 if (page_count(page) > 0) 8624 return false; 8625 8626 if (PageHuge(page)) 8627 return false; 8628 } 8629 return true; 8630 } 8631 8632 static bool zone_spans_last_pfn(const struct zone *zone, 8633 unsigned long start_pfn, unsigned long nr_pages) 8634 { 8635 unsigned long last_pfn = start_pfn + nr_pages - 1; 8636 8637 return zone_spans_pfn(zone, last_pfn); 8638 } 8639 8640 /** 8641 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages 8642 * @nr_pages: Number of contiguous pages to allocate 8643 * @gfp_mask: GFP mask to limit search and used during compaction 8644 * @nid: Target node 8645 * @nodemask: Mask for other possible nodes 8646 * 8647 * This routine is a wrapper around alloc_contig_range(). It scans over zones 8648 * on an applicable zonelist to find a contiguous pfn range which can then be 8649 * tried for allocation with alloc_contig_range(). This routine is intended 8650 * for allocation requests which can not be fulfilled with the buddy allocator. 8651 * 8652 * The allocated memory is always aligned to a page boundary. If nr_pages is a 8653 * power of two then the alignment is guaranteed to be to the given nr_pages 8654 * (e.g. 1GB request would be aligned to 1GB). 8655 * 8656 * Allocated pages can be freed with free_contig_range() or by manually calling 8657 * __free_page() on each allocated page. 8658 * 8659 * Return: pointer to contiguous pages on success, or NULL if not successful. 8660 */ 8661 struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, 8662 int nid, nodemask_t *nodemask) 8663 { 8664 unsigned long ret, pfn, flags; 8665 struct zonelist *zonelist; 8666 struct zone *zone; 8667 struct zoneref *z; 8668 8669 zonelist = node_zonelist(nid, gfp_mask); 8670 for_each_zone_zonelist_nodemask(zone, z, zonelist, 8671 gfp_zone(gfp_mask), nodemask) { 8672 spin_lock_irqsave(&zone->lock, flags); 8673 8674 pfn = ALIGN(zone->zone_start_pfn, nr_pages); 8675 while (zone_spans_last_pfn(zone, pfn, nr_pages)) { 8676 if (pfn_range_valid_contig(zone, pfn, nr_pages)) { 8677 /* 8678 * We release the zone lock here because 8679 * alloc_contig_range() will also lock the zone 8680 * at some point. If there's an allocation 8681 * spinning on this lock, it may win the race 8682 * and cause alloc_contig_range() to fail... 8683 */ 8684 spin_unlock_irqrestore(&zone->lock, flags); 8685 ret = __alloc_contig_pages(pfn, nr_pages, 8686 gfp_mask); 8687 if (!ret) 8688 return pfn_to_page(pfn); 8689 spin_lock_irqsave(&zone->lock, flags); 8690 } 8691 pfn += nr_pages; 8692 } 8693 spin_unlock_irqrestore(&zone->lock, flags); 8694 } 8695 return NULL; 8696 } 8697 #endif /* CONFIG_CONTIG_ALLOC */ 8698 8699 void free_contig_range(unsigned long pfn, unsigned int nr_pages) 8700 { 8701 unsigned int count = 0; 8702 8703 for (; nr_pages--; pfn++) { 8704 struct page *page = pfn_to_page(pfn); 8705 8706 count += page_count(page) != 1; 8707 __free_page(page); 8708 } 8709 WARN(count != 0, "%d pages are still in use!\n", count); 8710 } 8711 8712 /* 8713 * The zone indicated has a new number of managed_pages; batch sizes and percpu 8714 * page high values need to be recalulated. 8715 */ 8716 void __meminit zone_pcp_update(struct zone *zone) 8717 { 8718 mutex_lock(&pcp_batch_high_lock); 8719 __zone_pcp_update(zone); 8720 mutex_unlock(&pcp_batch_high_lock); 8721 } 8722 8723 void zone_pcp_reset(struct zone *zone) 8724 { 8725 unsigned long flags; 8726 int cpu; 8727 struct per_cpu_pageset *pset; 8728 8729 /* avoid races with drain_pages() */ 8730 local_irq_save(flags); 8731 if (zone->pageset != &boot_pageset) { 8732 for_each_online_cpu(cpu) { 8733 pset = per_cpu_ptr(zone->pageset, cpu); 8734 drain_zonestat(zone, pset); 8735 } 8736 free_percpu(zone->pageset); 8737 zone->pageset = &boot_pageset; 8738 } 8739 local_irq_restore(flags); 8740 } 8741 8742 #ifdef CONFIG_MEMORY_HOTREMOVE 8743 /* 8744 * All pages in the range must be in a single zone and isolated 8745 * before calling this. 8746 */ 8747 unsigned long 8748 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 8749 { 8750 struct page *page; 8751 struct zone *zone; 8752 unsigned int order; 8753 unsigned long pfn; 8754 unsigned long flags; 8755 unsigned long offlined_pages = 0; 8756 8757 /* find the first valid pfn */ 8758 for (pfn = start_pfn; pfn < end_pfn; pfn++) 8759 if (pfn_valid(pfn)) 8760 break; 8761 if (pfn == end_pfn) 8762 return offlined_pages; 8763 8764 offline_mem_sections(pfn, end_pfn); 8765 zone = page_zone(pfn_to_page(pfn)); 8766 spin_lock_irqsave(&zone->lock, flags); 8767 pfn = start_pfn; 8768 while (pfn < end_pfn) { 8769 if (!pfn_valid(pfn)) { 8770 pfn++; 8771 continue; 8772 } 8773 page = pfn_to_page(pfn); 8774 /* 8775 * The HWPoisoned page may be not in buddy system, and 8776 * page_count() is not 0. 8777 */ 8778 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 8779 pfn++; 8780 offlined_pages++; 8781 continue; 8782 } 8783 8784 BUG_ON(page_count(page)); 8785 BUG_ON(!PageBuddy(page)); 8786 order = page_order(page); 8787 offlined_pages += 1 << order; 8788 del_page_from_free_list(page, zone, order); 8789 pfn += (1 << order); 8790 } 8791 spin_unlock_irqrestore(&zone->lock, flags); 8792 8793 return offlined_pages; 8794 } 8795 #endif 8796 8797 bool is_free_buddy_page(struct page *page) 8798 { 8799 struct zone *zone = page_zone(page); 8800 unsigned long pfn = page_to_pfn(page); 8801 unsigned long flags; 8802 unsigned int order; 8803 8804 spin_lock_irqsave(&zone->lock, flags); 8805 for (order = 0; order < MAX_ORDER; order++) { 8806 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8807 8808 if (PageBuddy(page_head) && page_order(page_head) >= order) 8809 break; 8810 } 8811 spin_unlock_irqrestore(&zone->lock, flags); 8812 8813 return order < MAX_ORDER; 8814 } 8815 8816 #ifdef CONFIG_MEMORY_FAILURE 8817 /* 8818 * Set PG_hwpoison flag if a given page is confirmed to be a free page. This 8819 * test is performed under the zone lock to prevent a race against page 8820 * allocation. 8821 */ 8822 bool set_hwpoison_free_buddy_page(struct page *page) 8823 { 8824 struct zone *zone = page_zone(page); 8825 unsigned long pfn = page_to_pfn(page); 8826 unsigned long flags; 8827 unsigned int order; 8828 bool hwpoisoned = false; 8829 8830 spin_lock_irqsave(&zone->lock, flags); 8831 for (order = 0; order < MAX_ORDER; order++) { 8832 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8833 8834 if (PageBuddy(page_head) && page_order(page_head) >= order) { 8835 if (!TestSetPageHWPoison(page)) 8836 hwpoisoned = true; 8837 break; 8838 } 8839 } 8840 spin_unlock_irqrestore(&zone->lock, flags); 8841 8842 return hwpoisoned; 8843 } 8844 #endif 8845