1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/page_alloc.c 4 * 5 * Manages the free list, the system allocates free pages here. 6 * Note that kmalloc() lives in slab.c 7 * 8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 9 * Swap reorganised 29.12.95, Stephen Tweedie 10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 16 */ 17 18 #include <linux/stddef.h> 19 #include <linux/mm.h> 20 #include <linux/highmem.h> 21 #include <linux/swap.h> 22 #include <linux/interrupt.h> 23 #include <linux/pagemap.h> 24 #include <linux/jiffies.h> 25 #include <linux/memblock.h> 26 #include <linux/compiler.h> 27 #include <linux/kernel.h> 28 #include <linux/kasan.h> 29 #include <linux/module.h> 30 #include <linux/suspend.h> 31 #include <linux/pagevec.h> 32 #include <linux/blkdev.h> 33 #include <linux/slab.h> 34 #include <linux/ratelimit.h> 35 #include <linux/oom.h> 36 #include <linux/topology.h> 37 #include <linux/sysctl.h> 38 #include <linux/cpu.h> 39 #include <linux/cpuset.h> 40 #include <linux/memory_hotplug.h> 41 #include <linux/nodemask.h> 42 #include <linux/vmalloc.h> 43 #include <linux/vmstat.h> 44 #include <linux/mempolicy.h> 45 #include <linux/memremap.h> 46 #include <linux/stop_machine.h> 47 #include <linux/random.h> 48 #include <linux/sort.h> 49 #include <linux/pfn.h> 50 #include <linux/backing-dev.h> 51 #include <linux/fault-inject.h> 52 #include <linux/page-isolation.h> 53 #include <linux/debugobjects.h> 54 #include <linux/kmemleak.h> 55 #include <linux/compaction.h> 56 #include <trace/events/kmem.h> 57 #include <trace/events/oom.h> 58 #include <linux/prefetch.h> 59 #include <linux/mm_inline.h> 60 #include <linux/migrate.h> 61 #include <linux/hugetlb.h> 62 #include <linux/sched/rt.h> 63 #include <linux/sched/mm.h> 64 #include <linux/page_owner.h> 65 #include <linux/kthread.h> 66 #include <linux/memcontrol.h> 67 #include <linux/ftrace.h> 68 #include <linux/lockdep.h> 69 #include <linux/nmi.h> 70 #include <linux/psi.h> 71 #include <linux/padata.h> 72 #include <linux/khugepaged.h> 73 74 #include <asm/sections.h> 75 #include <asm/tlbflush.h> 76 #include <asm/div64.h> 77 #include "internal.h" 78 #include "shuffle.h" 79 #include "page_reporting.h" 80 81 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 82 static DEFINE_MUTEX(pcp_batch_high_lock); 83 #define MIN_PERCPU_PAGELIST_FRACTION (8) 84 85 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 86 DEFINE_PER_CPU(int, numa_node); 87 EXPORT_PER_CPU_SYMBOL(numa_node); 88 #endif 89 90 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); 91 92 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 93 /* 94 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 95 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 96 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 97 * defined in <linux/topology.h>. 98 */ 99 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 100 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 101 #endif 102 103 /* work_structs for global per-cpu drains */ 104 struct pcpu_drain { 105 struct zone *zone; 106 struct work_struct work; 107 }; 108 static DEFINE_MUTEX(pcpu_drain_mutex); 109 static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain); 110 111 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 112 volatile unsigned long latent_entropy __latent_entropy; 113 EXPORT_SYMBOL(latent_entropy); 114 #endif 115 116 /* 117 * Array of node states. 118 */ 119 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 120 [N_POSSIBLE] = NODE_MASK_ALL, 121 [N_ONLINE] = { { [0] = 1UL } }, 122 #ifndef CONFIG_NUMA 123 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 124 #ifdef CONFIG_HIGHMEM 125 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 126 #endif 127 [N_MEMORY] = { { [0] = 1UL } }, 128 [N_CPU] = { { [0] = 1UL } }, 129 #endif /* NUMA */ 130 }; 131 EXPORT_SYMBOL(node_states); 132 133 atomic_long_t _totalram_pages __read_mostly; 134 EXPORT_SYMBOL(_totalram_pages); 135 unsigned long totalreserve_pages __read_mostly; 136 unsigned long totalcma_pages __read_mostly; 137 138 int percpu_pagelist_fraction; 139 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 140 #ifdef CONFIG_INIT_ON_ALLOC_DEFAULT_ON 141 DEFINE_STATIC_KEY_TRUE(init_on_alloc); 142 #else 143 DEFINE_STATIC_KEY_FALSE(init_on_alloc); 144 #endif 145 EXPORT_SYMBOL(init_on_alloc); 146 147 #ifdef CONFIG_INIT_ON_FREE_DEFAULT_ON 148 DEFINE_STATIC_KEY_TRUE(init_on_free); 149 #else 150 DEFINE_STATIC_KEY_FALSE(init_on_free); 151 #endif 152 EXPORT_SYMBOL(init_on_free); 153 154 static int __init early_init_on_alloc(char *buf) 155 { 156 int ret; 157 bool bool_result; 158 159 ret = kstrtobool(buf, &bool_result); 160 if (ret) 161 return ret; 162 if (bool_result && page_poisoning_enabled()) 163 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_alloc\n"); 164 if (bool_result) 165 static_branch_enable(&init_on_alloc); 166 else 167 static_branch_disable(&init_on_alloc); 168 return 0; 169 } 170 early_param("init_on_alloc", early_init_on_alloc); 171 172 static int __init early_init_on_free(char *buf) 173 { 174 int ret; 175 bool bool_result; 176 177 ret = kstrtobool(buf, &bool_result); 178 if (ret) 179 return ret; 180 if (bool_result && page_poisoning_enabled()) 181 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_free\n"); 182 if (bool_result) 183 static_branch_enable(&init_on_free); 184 else 185 static_branch_disable(&init_on_free); 186 return 0; 187 } 188 early_param("init_on_free", early_init_on_free); 189 190 /* 191 * A cached value of the page's pageblock's migratetype, used when the page is 192 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 193 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 194 * Also the migratetype set in the page does not necessarily match the pcplist 195 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 196 * other index - this ensures that it will be put on the correct CMA freelist. 197 */ 198 static inline int get_pcppage_migratetype(struct page *page) 199 { 200 return page->index; 201 } 202 203 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 204 { 205 page->index = migratetype; 206 } 207 208 #ifdef CONFIG_PM_SLEEP 209 /* 210 * The following functions are used by the suspend/hibernate code to temporarily 211 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 212 * while devices are suspended. To avoid races with the suspend/hibernate code, 213 * they should always be called with system_transition_mutex held 214 * (gfp_allowed_mask also should only be modified with system_transition_mutex 215 * held, unless the suspend/hibernate code is guaranteed not to run in parallel 216 * with that modification). 217 */ 218 219 static gfp_t saved_gfp_mask; 220 221 void pm_restore_gfp_mask(void) 222 { 223 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 224 if (saved_gfp_mask) { 225 gfp_allowed_mask = saved_gfp_mask; 226 saved_gfp_mask = 0; 227 } 228 } 229 230 void pm_restrict_gfp_mask(void) 231 { 232 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 233 WARN_ON(saved_gfp_mask); 234 saved_gfp_mask = gfp_allowed_mask; 235 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 236 } 237 238 bool pm_suspended_storage(void) 239 { 240 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 241 return false; 242 return true; 243 } 244 #endif /* CONFIG_PM_SLEEP */ 245 246 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 247 unsigned int pageblock_order __read_mostly; 248 #endif 249 250 static void __free_pages_ok(struct page *page, unsigned int order); 251 252 /* 253 * results with 256, 32 in the lowmem_reserve sysctl: 254 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 255 * 1G machine -> (16M dma, 784M normal, 224M high) 256 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 257 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 258 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 259 * 260 * TBD: should special case ZONE_DMA32 machines here - in those we normally 261 * don't need any ZONE_NORMAL reservation 262 */ 263 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { 264 #ifdef CONFIG_ZONE_DMA 265 [ZONE_DMA] = 256, 266 #endif 267 #ifdef CONFIG_ZONE_DMA32 268 [ZONE_DMA32] = 256, 269 #endif 270 [ZONE_NORMAL] = 32, 271 #ifdef CONFIG_HIGHMEM 272 [ZONE_HIGHMEM] = 0, 273 #endif 274 [ZONE_MOVABLE] = 0, 275 }; 276 277 static char * const zone_names[MAX_NR_ZONES] = { 278 #ifdef CONFIG_ZONE_DMA 279 "DMA", 280 #endif 281 #ifdef CONFIG_ZONE_DMA32 282 "DMA32", 283 #endif 284 "Normal", 285 #ifdef CONFIG_HIGHMEM 286 "HighMem", 287 #endif 288 "Movable", 289 #ifdef CONFIG_ZONE_DEVICE 290 "Device", 291 #endif 292 }; 293 294 const char * const migratetype_names[MIGRATE_TYPES] = { 295 "Unmovable", 296 "Movable", 297 "Reclaimable", 298 "HighAtomic", 299 #ifdef CONFIG_CMA 300 "CMA", 301 #endif 302 #ifdef CONFIG_MEMORY_ISOLATION 303 "Isolate", 304 #endif 305 }; 306 307 compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = { 308 [NULL_COMPOUND_DTOR] = NULL, 309 [COMPOUND_PAGE_DTOR] = free_compound_page, 310 #ifdef CONFIG_HUGETLB_PAGE 311 [HUGETLB_PAGE_DTOR] = free_huge_page, 312 #endif 313 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 314 [TRANSHUGE_PAGE_DTOR] = free_transhuge_page, 315 #endif 316 }; 317 318 int min_free_kbytes = 1024; 319 int user_min_free_kbytes = -1; 320 #ifdef CONFIG_DISCONTIGMEM 321 /* 322 * DiscontigMem defines memory ranges as separate pg_data_t even if the ranges 323 * are not on separate NUMA nodes. Functionally this works but with 324 * watermark_boost_factor, it can reclaim prematurely as the ranges can be 325 * quite small. By default, do not boost watermarks on discontigmem as in 326 * many cases very high-order allocations like THP are likely to be 327 * unsupported and the premature reclaim offsets the advantage of long-term 328 * fragmentation avoidance. 329 */ 330 int watermark_boost_factor __read_mostly; 331 #else 332 int watermark_boost_factor __read_mostly = 15000; 333 #endif 334 int watermark_scale_factor = 10; 335 336 static unsigned long nr_kernel_pages __initdata; 337 static unsigned long nr_all_pages __initdata; 338 static unsigned long dma_reserve __initdata; 339 340 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 341 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 342 static unsigned long required_kernelcore __initdata; 343 static unsigned long required_kernelcore_percent __initdata; 344 static unsigned long required_movablecore __initdata; 345 static unsigned long required_movablecore_percent __initdata; 346 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 347 static bool mirrored_kernelcore __meminitdata; 348 349 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 350 int movable_zone; 351 EXPORT_SYMBOL(movable_zone); 352 353 #if MAX_NUMNODES > 1 354 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; 355 unsigned int nr_online_nodes __read_mostly = 1; 356 EXPORT_SYMBOL(nr_node_ids); 357 EXPORT_SYMBOL(nr_online_nodes); 358 #endif 359 360 int page_group_by_mobility_disabled __read_mostly; 361 362 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 363 /* 364 * During boot we initialize deferred pages on-demand, as needed, but once 365 * page_alloc_init_late() has finished, the deferred pages are all initialized, 366 * and we can permanently disable that path. 367 */ 368 static DEFINE_STATIC_KEY_TRUE(deferred_pages); 369 370 /* 371 * Calling kasan_free_pages() only after deferred memory initialization 372 * has completed. Poisoning pages during deferred memory init will greatly 373 * lengthen the process and cause problem in large memory systems as the 374 * deferred pages initialization is done with interrupt disabled. 375 * 376 * Assuming that there will be no reference to those newly initialized 377 * pages before they are ever allocated, this should have no effect on 378 * KASAN memory tracking as the poison will be properly inserted at page 379 * allocation time. The only corner case is when pages are allocated by 380 * on-demand allocation and then freed again before the deferred pages 381 * initialization is done, but this is not likely to happen. 382 */ 383 static inline void kasan_free_nondeferred_pages(struct page *page, int order) 384 { 385 if (!static_branch_unlikely(&deferred_pages)) 386 kasan_free_pages(page, order); 387 } 388 389 /* Returns true if the struct page for the pfn is uninitialised */ 390 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 391 { 392 int nid = early_pfn_to_nid(pfn); 393 394 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 395 return true; 396 397 return false; 398 } 399 400 /* 401 * Returns true when the remaining initialisation should be deferred until 402 * later in the boot cycle when it can be parallelised. 403 */ 404 static bool __meminit 405 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 406 { 407 static unsigned long prev_end_pfn, nr_initialised; 408 409 /* 410 * prev_end_pfn static that contains the end of previous zone 411 * No need to protect because called very early in boot before smp_init. 412 */ 413 if (prev_end_pfn != end_pfn) { 414 prev_end_pfn = end_pfn; 415 nr_initialised = 0; 416 } 417 418 /* Always populate low zones for address-constrained allocations */ 419 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 420 return false; 421 422 /* 423 * We start only with one section of pages, more pages are added as 424 * needed until the rest of deferred pages are initialized. 425 */ 426 nr_initialised++; 427 if ((nr_initialised > PAGES_PER_SECTION) && 428 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 429 NODE_DATA(nid)->first_deferred_pfn = pfn; 430 return true; 431 } 432 return false; 433 } 434 #else 435 #define kasan_free_nondeferred_pages(p, o) kasan_free_pages(p, o) 436 437 static inline bool early_page_uninitialised(unsigned long pfn) 438 { 439 return false; 440 } 441 442 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 443 { 444 return false; 445 } 446 #endif 447 448 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 449 static inline unsigned long *get_pageblock_bitmap(struct page *page, 450 unsigned long pfn) 451 { 452 #ifdef CONFIG_SPARSEMEM 453 return section_to_usemap(__pfn_to_section(pfn)); 454 #else 455 return page_zone(page)->pageblock_flags; 456 #endif /* CONFIG_SPARSEMEM */ 457 } 458 459 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn) 460 { 461 #ifdef CONFIG_SPARSEMEM 462 pfn &= (PAGES_PER_SECTION-1); 463 #else 464 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 465 #endif /* CONFIG_SPARSEMEM */ 466 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 467 } 468 469 /** 470 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 471 * @page: The page within the block of interest 472 * @pfn: The target page frame number 473 * @mask: mask of bits that the caller is interested in 474 * 475 * Return: pageblock_bits flags 476 */ 477 static __always_inline 478 unsigned long __get_pfnblock_flags_mask(struct page *page, 479 unsigned long pfn, 480 unsigned long mask) 481 { 482 unsigned long *bitmap; 483 unsigned long bitidx, word_bitidx; 484 unsigned long word; 485 486 bitmap = get_pageblock_bitmap(page, pfn); 487 bitidx = pfn_to_bitidx(page, pfn); 488 word_bitidx = bitidx / BITS_PER_LONG; 489 bitidx &= (BITS_PER_LONG-1); 490 491 word = bitmap[word_bitidx]; 492 return (word >> bitidx) & mask; 493 } 494 495 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, 496 unsigned long mask) 497 { 498 return __get_pfnblock_flags_mask(page, pfn, mask); 499 } 500 501 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn) 502 { 503 return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK); 504 } 505 506 /** 507 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 508 * @page: The page within the block of interest 509 * @flags: The flags to set 510 * @pfn: The target page frame number 511 * @mask: mask of bits that the caller is interested in 512 */ 513 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 514 unsigned long pfn, 515 unsigned long mask) 516 { 517 unsigned long *bitmap; 518 unsigned long bitidx, word_bitidx; 519 unsigned long old_word, word; 520 521 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 522 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits)); 523 524 bitmap = get_pageblock_bitmap(page, pfn); 525 bitidx = pfn_to_bitidx(page, pfn); 526 word_bitidx = bitidx / BITS_PER_LONG; 527 bitidx &= (BITS_PER_LONG-1); 528 529 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 530 531 mask <<= bitidx; 532 flags <<= bitidx; 533 534 word = READ_ONCE(bitmap[word_bitidx]); 535 for (;;) { 536 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 537 if (word == old_word) 538 break; 539 word = old_word; 540 } 541 } 542 543 void set_pageblock_migratetype(struct page *page, int migratetype) 544 { 545 if (unlikely(page_group_by_mobility_disabled && 546 migratetype < MIGRATE_PCPTYPES)) 547 migratetype = MIGRATE_UNMOVABLE; 548 549 set_pfnblock_flags_mask(page, (unsigned long)migratetype, 550 page_to_pfn(page), MIGRATETYPE_MASK); 551 } 552 553 #ifdef CONFIG_DEBUG_VM 554 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 555 { 556 int ret = 0; 557 unsigned seq; 558 unsigned long pfn = page_to_pfn(page); 559 unsigned long sp, start_pfn; 560 561 do { 562 seq = zone_span_seqbegin(zone); 563 start_pfn = zone->zone_start_pfn; 564 sp = zone->spanned_pages; 565 if (!zone_spans_pfn(zone, pfn)) 566 ret = 1; 567 } while (zone_span_seqretry(zone, seq)); 568 569 if (ret) 570 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 571 pfn, zone_to_nid(zone), zone->name, 572 start_pfn, start_pfn + sp); 573 574 return ret; 575 } 576 577 static int page_is_consistent(struct zone *zone, struct page *page) 578 { 579 if (!pfn_valid_within(page_to_pfn(page))) 580 return 0; 581 if (zone != page_zone(page)) 582 return 0; 583 584 return 1; 585 } 586 /* 587 * Temporary debugging check for pages not lying within a given zone. 588 */ 589 static int __maybe_unused bad_range(struct zone *zone, struct page *page) 590 { 591 if (page_outside_zone_boundaries(zone, page)) 592 return 1; 593 if (!page_is_consistent(zone, page)) 594 return 1; 595 596 return 0; 597 } 598 #else 599 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page) 600 { 601 return 0; 602 } 603 #endif 604 605 static void bad_page(struct page *page, const char *reason) 606 { 607 static unsigned long resume; 608 static unsigned long nr_shown; 609 static unsigned long nr_unshown; 610 611 /* 612 * Allow a burst of 60 reports, then keep quiet for that minute; 613 * or allow a steady drip of one report per second. 614 */ 615 if (nr_shown == 60) { 616 if (time_before(jiffies, resume)) { 617 nr_unshown++; 618 goto out; 619 } 620 if (nr_unshown) { 621 pr_alert( 622 "BUG: Bad page state: %lu messages suppressed\n", 623 nr_unshown); 624 nr_unshown = 0; 625 } 626 nr_shown = 0; 627 } 628 if (nr_shown++ == 0) 629 resume = jiffies + 60 * HZ; 630 631 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 632 current->comm, page_to_pfn(page)); 633 __dump_page(page, reason); 634 dump_page_owner(page); 635 636 print_modules(); 637 dump_stack(); 638 out: 639 /* Leave bad fields for debug, except PageBuddy could make trouble */ 640 page_mapcount_reset(page); /* remove PageBuddy */ 641 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 642 } 643 644 /* 645 * Higher-order pages are called "compound pages". They are structured thusly: 646 * 647 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 648 * 649 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 650 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 651 * 652 * The first tail page's ->compound_dtor holds the offset in array of compound 653 * page destructors. See compound_page_dtors. 654 * 655 * The first tail page's ->compound_order holds the order of allocation. 656 * This usage means that zero-order pages may not be compound. 657 */ 658 659 void free_compound_page(struct page *page) 660 { 661 mem_cgroup_uncharge(page); 662 __free_pages_ok(page, compound_order(page)); 663 } 664 665 void prep_compound_page(struct page *page, unsigned int order) 666 { 667 int i; 668 int nr_pages = 1 << order; 669 670 __SetPageHead(page); 671 for (i = 1; i < nr_pages; i++) { 672 struct page *p = page + i; 673 set_page_count(p, 0); 674 p->mapping = TAIL_MAPPING; 675 set_compound_head(p, page); 676 } 677 678 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 679 set_compound_order(page, order); 680 atomic_set(compound_mapcount_ptr(page), -1); 681 if (hpage_pincount_available(page)) 682 atomic_set(compound_pincount_ptr(page), 0); 683 } 684 685 #ifdef CONFIG_DEBUG_PAGEALLOC 686 unsigned int _debug_guardpage_minorder; 687 688 bool _debug_pagealloc_enabled_early __read_mostly 689 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 690 EXPORT_SYMBOL(_debug_pagealloc_enabled_early); 691 DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 692 EXPORT_SYMBOL(_debug_pagealloc_enabled); 693 694 DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 695 696 static int __init early_debug_pagealloc(char *buf) 697 { 698 return kstrtobool(buf, &_debug_pagealloc_enabled_early); 699 } 700 early_param("debug_pagealloc", early_debug_pagealloc); 701 702 void init_debug_pagealloc(void) 703 { 704 if (!debug_pagealloc_enabled()) 705 return; 706 707 static_branch_enable(&_debug_pagealloc_enabled); 708 709 if (!debug_guardpage_minorder()) 710 return; 711 712 static_branch_enable(&_debug_guardpage_enabled); 713 } 714 715 static int __init debug_guardpage_minorder_setup(char *buf) 716 { 717 unsigned long res; 718 719 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 720 pr_err("Bad debug_guardpage_minorder value\n"); 721 return 0; 722 } 723 _debug_guardpage_minorder = res; 724 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 725 return 0; 726 } 727 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); 728 729 static inline bool set_page_guard(struct zone *zone, struct page *page, 730 unsigned int order, int migratetype) 731 { 732 if (!debug_guardpage_enabled()) 733 return false; 734 735 if (order >= debug_guardpage_minorder()) 736 return false; 737 738 __SetPageGuard(page); 739 INIT_LIST_HEAD(&page->lru); 740 set_page_private(page, order); 741 /* Guard pages are not available for any usage */ 742 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 743 744 return true; 745 } 746 747 static inline void clear_page_guard(struct zone *zone, struct page *page, 748 unsigned int order, int migratetype) 749 { 750 if (!debug_guardpage_enabled()) 751 return; 752 753 __ClearPageGuard(page); 754 755 set_page_private(page, 0); 756 if (!is_migrate_isolate(migratetype)) 757 __mod_zone_freepage_state(zone, (1 << order), migratetype); 758 } 759 #else 760 static inline bool set_page_guard(struct zone *zone, struct page *page, 761 unsigned int order, int migratetype) { return false; } 762 static inline void clear_page_guard(struct zone *zone, struct page *page, 763 unsigned int order, int migratetype) {} 764 #endif 765 766 static inline void set_page_order(struct page *page, unsigned int order) 767 { 768 set_page_private(page, order); 769 __SetPageBuddy(page); 770 } 771 772 /* 773 * This function checks whether a page is free && is the buddy 774 * we can coalesce a page and its buddy if 775 * (a) the buddy is not in a hole (check before calling!) && 776 * (b) the buddy is in the buddy system && 777 * (c) a page and its buddy have the same order && 778 * (d) a page and its buddy are in the same zone. 779 * 780 * For recording whether a page is in the buddy system, we set PageBuddy. 781 * Setting, clearing, and testing PageBuddy is serialized by zone->lock. 782 * 783 * For recording page's order, we use page_private(page). 784 */ 785 static inline bool page_is_buddy(struct page *page, struct page *buddy, 786 unsigned int order) 787 { 788 if (!page_is_guard(buddy) && !PageBuddy(buddy)) 789 return false; 790 791 if (page_order(buddy) != order) 792 return false; 793 794 /* 795 * zone check is done late to avoid uselessly calculating 796 * zone/node ids for pages that could never merge. 797 */ 798 if (page_zone_id(page) != page_zone_id(buddy)) 799 return false; 800 801 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 802 803 return true; 804 } 805 806 #ifdef CONFIG_COMPACTION 807 static inline struct capture_control *task_capc(struct zone *zone) 808 { 809 struct capture_control *capc = current->capture_control; 810 811 return unlikely(capc) && 812 !(current->flags & PF_KTHREAD) && 813 !capc->page && 814 capc->cc->zone == zone ? capc : NULL; 815 } 816 817 static inline bool 818 compaction_capture(struct capture_control *capc, struct page *page, 819 int order, int migratetype) 820 { 821 if (!capc || order != capc->cc->order) 822 return false; 823 824 /* Do not accidentally pollute CMA or isolated regions*/ 825 if (is_migrate_cma(migratetype) || 826 is_migrate_isolate(migratetype)) 827 return false; 828 829 /* 830 * Do not let lower order allocations polluate a movable pageblock. 831 * This might let an unmovable request use a reclaimable pageblock 832 * and vice-versa but no more than normal fallback logic which can 833 * have trouble finding a high-order free page. 834 */ 835 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE) 836 return false; 837 838 capc->page = page; 839 return true; 840 } 841 842 #else 843 static inline struct capture_control *task_capc(struct zone *zone) 844 { 845 return NULL; 846 } 847 848 static inline bool 849 compaction_capture(struct capture_control *capc, struct page *page, 850 int order, int migratetype) 851 { 852 return false; 853 } 854 #endif /* CONFIG_COMPACTION */ 855 856 /* Used for pages not on another list */ 857 static inline void add_to_free_list(struct page *page, struct zone *zone, 858 unsigned int order, int migratetype) 859 { 860 struct free_area *area = &zone->free_area[order]; 861 862 list_add(&page->lru, &area->free_list[migratetype]); 863 area->nr_free++; 864 } 865 866 /* Used for pages not on another list */ 867 static inline void add_to_free_list_tail(struct page *page, struct zone *zone, 868 unsigned int order, int migratetype) 869 { 870 struct free_area *area = &zone->free_area[order]; 871 872 list_add_tail(&page->lru, &area->free_list[migratetype]); 873 area->nr_free++; 874 } 875 876 /* Used for pages which are on another list */ 877 static inline void move_to_free_list(struct page *page, struct zone *zone, 878 unsigned int order, int migratetype) 879 { 880 struct free_area *area = &zone->free_area[order]; 881 882 list_move(&page->lru, &area->free_list[migratetype]); 883 } 884 885 static inline void del_page_from_free_list(struct page *page, struct zone *zone, 886 unsigned int order) 887 { 888 /* clear reported state and update reported page count */ 889 if (page_reported(page)) 890 __ClearPageReported(page); 891 892 list_del(&page->lru); 893 __ClearPageBuddy(page); 894 set_page_private(page, 0); 895 zone->free_area[order].nr_free--; 896 } 897 898 /* 899 * If this is not the largest possible page, check if the buddy 900 * of the next-highest order is free. If it is, it's possible 901 * that pages are being freed that will coalesce soon. In case, 902 * that is happening, add the free page to the tail of the list 903 * so it's less likely to be used soon and more likely to be merged 904 * as a higher order page 905 */ 906 static inline bool 907 buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, 908 struct page *page, unsigned int order) 909 { 910 struct page *higher_page, *higher_buddy; 911 unsigned long combined_pfn; 912 913 if (order >= MAX_ORDER - 2) 914 return false; 915 916 if (!pfn_valid_within(buddy_pfn)) 917 return false; 918 919 combined_pfn = buddy_pfn & pfn; 920 higher_page = page + (combined_pfn - pfn); 921 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1); 922 higher_buddy = higher_page + (buddy_pfn - combined_pfn); 923 924 return pfn_valid_within(buddy_pfn) && 925 page_is_buddy(higher_page, higher_buddy, order + 1); 926 } 927 928 /* 929 * Freeing function for a buddy system allocator. 930 * 931 * The concept of a buddy system is to maintain direct-mapped table 932 * (containing bit values) for memory blocks of various "orders". 933 * The bottom level table contains the map for the smallest allocatable 934 * units of memory (here, pages), and each level above it describes 935 * pairs of units from the levels below, hence, "buddies". 936 * At a high level, all that happens here is marking the table entry 937 * at the bottom level available, and propagating the changes upward 938 * as necessary, plus some accounting needed to play nicely with other 939 * parts of the VM system. 940 * At each level, we keep a list of pages, which are heads of continuous 941 * free pages of length of (1 << order) and marked with PageBuddy. 942 * Page's order is recorded in page_private(page) field. 943 * So when we are allocating or freeing one, we can derive the state of the 944 * other. That is, if we allocate a small block, and both were 945 * free, the remainder of the region must be split into blocks. 946 * If a block is freed, and its buddy is also free, then this 947 * triggers coalescing into a block of larger size. 948 * 949 * -- nyc 950 */ 951 952 static inline void __free_one_page(struct page *page, 953 unsigned long pfn, 954 struct zone *zone, unsigned int order, 955 int migratetype, bool report) 956 { 957 struct capture_control *capc = task_capc(zone); 958 unsigned long buddy_pfn; 959 unsigned long combined_pfn; 960 unsigned int max_order; 961 struct page *buddy; 962 bool to_tail; 963 964 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1); 965 966 VM_BUG_ON(!zone_is_initialized(zone)); 967 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 968 969 VM_BUG_ON(migratetype == -1); 970 if (likely(!is_migrate_isolate(migratetype))) 971 __mod_zone_freepage_state(zone, 1 << order, migratetype); 972 973 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); 974 VM_BUG_ON_PAGE(bad_range(zone, page), page); 975 976 continue_merging: 977 while (order < max_order - 1) { 978 if (compaction_capture(capc, page, order, migratetype)) { 979 __mod_zone_freepage_state(zone, -(1 << order), 980 migratetype); 981 return; 982 } 983 buddy_pfn = __find_buddy_pfn(pfn, order); 984 buddy = page + (buddy_pfn - pfn); 985 986 if (!pfn_valid_within(buddy_pfn)) 987 goto done_merging; 988 if (!page_is_buddy(page, buddy, order)) 989 goto done_merging; 990 /* 991 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 992 * merge with it and move up one order. 993 */ 994 if (page_is_guard(buddy)) 995 clear_page_guard(zone, buddy, order, migratetype); 996 else 997 del_page_from_free_list(buddy, zone, order); 998 combined_pfn = buddy_pfn & pfn; 999 page = page + (combined_pfn - pfn); 1000 pfn = combined_pfn; 1001 order++; 1002 } 1003 if (max_order < MAX_ORDER) { 1004 /* If we are here, it means order is >= pageblock_order. 1005 * We want to prevent merge between freepages on isolate 1006 * pageblock and normal pageblock. Without this, pageblock 1007 * isolation could cause incorrect freepage or CMA accounting. 1008 * 1009 * We don't want to hit this code for the more frequent 1010 * low-order merging. 1011 */ 1012 if (unlikely(has_isolate_pageblock(zone))) { 1013 int buddy_mt; 1014 1015 buddy_pfn = __find_buddy_pfn(pfn, order); 1016 buddy = page + (buddy_pfn - pfn); 1017 buddy_mt = get_pageblock_migratetype(buddy); 1018 1019 if (migratetype != buddy_mt 1020 && (is_migrate_isolate(migratetype) || 1021 is_migrate_isolate(buddy_mt))) 1022 goto done_merging; 1023 } 1024 max_order++; 1025 goto continue_merging; 1026 } 1027 1028 done_merging: 1029 set_page_order(page, order); 1030 1031 if (is_shuffle_order(order)) 1032 to_tail = shuffle_pick_tail(); 1033 else 1034 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); 1035 1036 if (to_tail) 1037 add_to_free_list_tail(page, zone, order, migratetype); 1038 else 1039 add_to_free_list(page, zone, order, migratetype); 1040 1041 /* Notify page reporting subsystem of freed page */ 1042 if (report) 1043 page_reporting_notify_free(order); 1044 } 1045 1046 /* 1047 * A bad page could be due to a number of fields. Instead of multiple branches, 1048 * try and check multiple fields with one check. The caller must do a detailed 1049 * check if necessary. 1050 */ 1051 static inline bool page_expected_state(struct page *page, 1052 unsigned long check_flags) 1053 { 1054 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1055 return false; 1056 1057 if (unlikely((unsigned long)page->mapping | 1058 page_ref_count(page) | 1059 #ifdef CONFIG_MEMCG 1060 (unsigned long)page->mem_cgroup | 1061 #endif 1062 (page->flags & check_flags))) 1063 return false; 1064 1065 return true; 1066 } 1067 1068 static const char *page_bad_reason(struct page *page, unsigned long flags) 1069 { 1070 const char *bad_reason = NULL; 1071 1072 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1073 bad_reason = "nonzero mapcount"; 1074 if (unlikely(page->mapping != NULL)) 1075 bad_reason = "non-NULL mapping"; 1076 if (unlikely(page_ref_count(page) != 0)) 1077 bad_reason = "nonzero _refcount"; 1078 if (unlikely(page->flags & flags)) { 1079 if (flags == PAGE_FLAGS_CHECK_AT_PREP) 1080 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set"; 1081 else 1082 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 1083 } 1084 #ifdef CONFIG_MEMCG 1085 if (unlikely(page->mem_cgroup)) 1086 bad_reason = "page still charged to cgroup"; 1087 #endif 1088 return bad_reason; 1089 } 1090 1091 static void check_free_page_bad(struct page *page) 1092 { 1093 bad_page(page, 1094 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE)); 1095 } 1096 1097 static inline int check_free_page(struct page *page) 1098 { 1099 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 1100 return 0; 1101 1102 /* Something has gone sideways, find it */ 1103 check_free_page_bad(page); 1104 return 1; 1105 } 1106 1107 static int free_tail_pages_check(struct page *head_page, struct page *page) 1108 { 1109 int ret = 1; 1110 1111 /* 1112 * We rely page->lru.next never has bit 0 set, unless the page 1113 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 1114 */ 1115 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 1116 1117 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 1118 ret = 0; 1119 goto out; 1120 } 1121 switch (page - head_page) { 1122 case 1: 1123 /* the first tail page: ->mapping may be compound_mapcount() */ 1124 if (unlikely(compound_mapcount(page))) { 1125 bad_page(page, "nonzero compound_mapcount"); 1126 goto out; 1127 } 1128 break; 1129 case 2: 1130 /* 1131 * the second tail page: ->mapping is 1132 * deferred_list.next -- ignore value. 1133 */ 1134 break; 1135 default: 1136 if (page->mapping != TAIL_MAPPING) { 1137 bad_page(page, "corrupted mapping in tail page"); 1138 goto out; 1139 } 1140 break; 1141 } 1142 if (unlikely(!PageTail(page))) { 1143 bad_page(page, "PageTail not set"); 1144 goto out; 1145 } 1146 if (unlikely(compound_head(page) != head_page)) { 1147 bad_page(page, "compound_head not consistent"); 1148 goto out; 1149 } 1150 ret = 0; 1151 out: 1152 page->mapping = NULL; 1153 clear_compound_head(page); 1154 return ret; 1155 } 1156 1157 static void kernel_init_free_pages(struct page *page, int numpages) 1158 { 1159 int i; 1160 1161 /* s390's use of memset() could override KASAN redzones. */ 1162 kasan_disable_current(); 1163 for (i = 0; i < numpages; i++) 1164 clear_highpage(page + i); 1165 kasan_enable_current(); 1166 } 1167 1168 static __always_inline bool free_pages_prepare(struct page *page, 1169 unsigned int order, bool check_free) 1170 { 1171 int bad = 0; 1172 1173 VM_BUG_ON_PAGE(PageTail(page), page); 1174 1175 trace_mm_page_free(page, order); 1176 1177 /* 1178 * Check tail pages before head page information is cleared to 1179 * avoid checking PageCompound for order-0 pages. 1180 */ 1181 if (unlikely(order)) { 1182 bool compound = PageCompound(page); 1183 int i; 1184 1185 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1186 1187 if (compound) 1188 ClearPageDoubleMap(page); 1189 for (i = 1; i < (1 << order); i++) { 1190 if (compound) 1191 bad += free_tail_pages_check(page, page + i); 1192 if (unlikely(check_free_page(page + i))) { 1193 bad++; 1194 continue; 1195 } 1196 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1197 } 1198 } 1199 if (PageMappingFlags(page)) 1200 page->mapping = NULL; 1201 if (memcg_kmem_enabled() && PageKmemcg(page)) 1202 __memcg_kmem_uncharge_page(page, order); 1203 if (check_free) 1204 bad += check_free_page(page); 1205 if (bad) 1206 return false; 1207 1208 page_cpupid_reset_last(page); 1209 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1210 reset_page_owner(page, order); 1211 1212 if (!PageHighMem(page)) { 1213 debug_check_no_locks_freed(page_address(page), 1214 PAGE_SIZE << order); 1215 debug_check_no_obj_freed(page_address(page), 1216 PAGE_SIZE << order); 1217 } 1218 if (want_init_on_free()) 1219 kernel_init_free_pages(page, 1 << order); 1220 1221 kernel_poison_pages(page, 1 << order, 0); 1222 /* 1223 * arch_free_page() can make the page's contents inaccessible. s390 1224 * does this. So nothing which can access the page's contents should 1225 * happen after this. 1226 */ 1227 arch_free_page(page, order); 1228 1229 if (debug_pagealloc_enabled_static()) 1230 kernel_map_pages(page, 1 << order, 0); 1231 1232 kasan_free_nondeferred_pages(page, order); 1233 1234 return true; 1235 } 1236 1237 #ifdef CONFIG_DEBUG_VM 1238 /* 1239 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed 1240 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when 1241 * moved from pcp lists to free lists. 1242 */ 1243 static bool free_pcp_prepare(struct page *page) 1244 { 1245 return free_pages_prepare(page, 0, true); 1246 } 1247 1248 static bool bulkfree_pcp_prepare(struct page *page) 1249 { 1250 if (debug_pagealloc_enabled_static()) 1251 return check_free_page(page); 1252 else 1253 return false; 1254 } 1255 #else 1256 /* 1257 * With DEBUG_VM disabled, order-0 pages being freed are checked only when 1258 * moving from pcp lists to free list in order to reduce overhead. With 1259 * debug_pagealloc enabled, they are checked also immediately when being freed 1260 * to the pcp lists. 1261 */ 1262 static bool free_pcp_prepare(struct page *page) 1263 { 1264 if (debug_pagealloc_enabled_static()) 1265 return free_pages_prepare(page, 0, true); 1266 else 1267 return free_pages_prepare(page, 0, false); 1268 } 1269 1270 static bool bulkfree_pcp_prepare(struct page *page) 1271 { 1272 return check_free_page(page); 1273 } 1274 #endif /* CONFIG_DEBUG_VM */ 1275 1276 static inline void prefetch_buddy(struct page *page) 1277 { 1278 unsigned long pfn = page_to_pfn(page); 1279 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0); 1280 struct page *buddy = page + (buddy_pfn - pfn); 1281 1282 prefetch(buddy); 1283 } 1284 1285 /* 1286 * Frees a number of pages from the PCP lists 1287 * Assumes all pages on list are in same zone, and of same order. 1288 * count is the number of pages to free. 1289 * 1290 * If the zone was previously in an "all pages pinned" state then look to 1291 * see if this freeing clears that state. 1292 * 1293 * And clear the zone's pages_scanned counter, to hold off the "all pages are 1294 * pinned" detection logic. 1295 */ 1296 static void free_pcppages_bulk(struct zone *zone, int count, 1297 struct per_cpu_pages *pcp) 1298 { 1299 int migratetype = 0; 1300 int batch_free = 0; 1301 int prefetch_nr = 0; 1302 bool isolated_pageblocks; 1303 struct page *page, *tmp; 1304 LIST_HEAD(head); 1305 1306 /* 1307 * Ensure proper count is passed which otherwise would stuck in the 1308 * below while (list_empty(list)) loop. 1309 */ 1310 count = min(pcp->count, count); 1311 while (count) { 1312 struct list_head *list; 1313 1314 /* 1315 * Remove pages from lists in a round-robin fashion. A 1316 * batch_free count is maintained that is incremented when an 1317 * empty list is encountered. This is so more pages are freed 1318 * off fuller lists instead of spinning excessively around empty 1319 * lists 1320 */ 1321 do { 1322 batch_free++; 1323 if (++migratetype == MIGRATE_PCPTYPES) 1324 migratetype = 0; 1325 list = &pcp->lists[migratetype]; 1326 } while (list_empty(list)); 1327 1328 /* This is the only non-empty list. Free them all. */ 1329 if (batch_free == MIGRATE_PCPTYPES) 1330 batch_free = count; 1331 1332 do { 1333 page = list_last_entry(list, struct page, lru); 1334 /* must delete to avoid corrupting pcp list */ 1335 list_del(&page->lru); 1336 pcp->count--; 1337 1338 if (bulkfree_pcp_prepare(page)) 1339 continue; 1340 1341 list_add_tail(&page->lru, &head); 1342 1343 /* 1344 * We are going to put the page back to the global 1345 * pool, prefetch its buddy to speed up later access 1346 * under zone->lock. It is believed the overhead of 1347 * an additional test and calculating buddy_pfn here 1348 * can be offset by reduced memory latency later. To 1349 * avoid excessive prefetching due to large count, only 1350 * prefetch buddy for the first pcp->batch nr of pages. 1351 */ 1352 if (prefetch_nr++ < pcp->batch) 1353 prefetch_buddy(page); 1354 } while (--count && --batch_free && !list_empty(list)); 1355 } 1356 1357 spin_lock(&zone->lock); 1358 isolated_pageblocks = has_isolate_pageblock(zone); 1359 1360 /* 1361 * Use safe version since after __free_one_page(), 1362 * page->lru.next will not point to original list. 1363 */ 1364 list_for_each_entry_safe(page, tmp, &head, lru) { 1365 int mt = get_pcppage_migratetype(page); 1366 /* MIGRATE_ISOLATE page should not go to pcplists */ 1367 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1368 /* Pageblock could have been isolated meanwhile */ 1369 if (unlikely(isolated_pageblocks)) 1370 mt = get_pageblock_migratetype(page); 1371 1372 __free_one_page(page, page_to_pfn(page), zone, 0, mt, true); 1373 trace_mm_page_pcpu_drain(page, 0, mt); 1374 } 1375 spin_unlock(&zone->lock); 1376 } 1377 1378 static void free_one_page(struct zone *zone, 1379 struct page *page, unsigned long pfn, 1380 unsigned int order, 1381 int migratetype) 1382 { 1383 spin_lock(&zone->lock); 1384 if (unlikely(has_isolate_pageblock(zone) || 1385 is_migrate_isolate(migratetype))) { 1386 migratetype = get_pfnblock_migratetype(page, pfn); 1387 } 1388 __free_one_page(page, pfn, zone, order, migratetype, true); 1389 spin_unlock(&zone->lock); 1390 } 1391 1392 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1393 unsigned long zone, int nid) 1394 { 1395 mm_zero_struct_page(page); 1396 set_page_links(page, zone, nid, pfn); 1397 init_page_count(page); 1398 page_mapcount_reset(page); 1399 page_cpupid_reset_last(page); 1400 page_kasan_tag_reset(page); 1401 1402 INIT_LIST_HEAD(&page->lru); 1403 #ifdef WANT_PAGE_VIRTUAL 1404 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1405 if (!is_highmem_idx(zone)) 1406 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1407 #endif 1408 } 1409 1410 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1411 static void __meminit init_reserved_page(unsigned long pfn) 1412 { 1413 pg_data_t *pgdat; 1414 int nid, zid; 1415 1416 if (!early_page_uninitialised(pfn)) 1417 return; 1418 1419 nid = early_pfn_to_nid(pfn); 1420 pgdat = NODE_DATA(nid); 1421 1422 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1423 struct zone *zone = &pgdat->node_zones[zid]; 1424 1425 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) 1426 break; 1427 } 1428 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 1429 } 1430 #else 1431 static inline void init_reserved_page(unsigned long pfn) 1432 { 1433 } 1434 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1435 1436 /* 1437 * Initialised pages do not have PageReserved set. This function is 1438 * called for each range allocated by the bootmem allocator and 1439 * marks the pages PageReserved. The remaining valid pages are later 1440 * sent to the buddy page allocator. 1441 */ 1442 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1443 { 1444 unsigned long start_pfn = PFN_DOWN(start); 1445 unsigned long end_pfn = PFN_UP(end); 1446 1447 for (; start_pfn < end_pfn; start_pfn++) { 1448 if (pfn_valid(start_pfn)) { 1449 struct page *page = pfn_to_page(start_pfn); 1450 1451 init_reserved_page(start_pfn); 1452 1453 /* Avoid false-positive PageTail() */ 1454 INIT_LIST_HEAD(&page->lru); 1455 1456 /* 1457 * no need for atomic set_bit because the struct 1458 * page is not visible yet so nobody should 1459 * access it yet. 1460 */ 1461 __SetPageReserved(page); 1462 } 1463 } 1464 } 1465 1466 static void __free_pages_ok(struct page *page, unsigned int order) 1467 { 1468 unsigned long flags; 1469 int migratetype; 1470 unsigned long pfn = page_to_pfn(page); 1471 1472 if (!free_pages_prepare(page, order, true)) 1473 return; 1474 1475 migratetype = get_pfnblock_migratetype(page, pfn); 1476 local_irq_save(flags); 1477 __count_vm_events(PGFREE, 1 << order); 1478 free_one_page(page_zone(page), page, pfn, order, migratetype); 1479 local_irq_restore(flags); 1480 } 1481 1482 void __free_pages_core(struct page *page, unsigned int order) 1483 { 1484 unsigned int nr_pages = 1 << order; 1485 struct page *p = page; 1486 unsigned int loop; 1487 1488 prefetchw(p); 1489 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1490 prefetchw(p + 1); 1491 __ClearPageReserved(p); 1492 set_page_count(p, 0); 1493 } 1494 __ClearPageReserved(p); 1495 set_page_count(p, 0); 1496 1497 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); 1498 set_page_refcounted(page); 1499 __free_pages(page, order); 1500 } 1501 1502 #ifdef CONFIG_NEED_MULTIPLE_NODES 1503 1504 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1505 1506 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID 1507 1508 /* 1509 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 1510 */ 1511 int __meminit __early_pfn_to_nid(unsigned long pfn, 1512 struct mminit_pfnnid_cache *state) 1513 { 1514 unsigned long start_pfn, end_pfn; 1515 int nid; 1516 1517 if (state->last_start <= pfn && pfn < state->last_end) 1518 return state->last_nid; 1519 1520 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 1521 if (nid != NUMA_NO_NODE) { 1522 state->last_start = start_pfn; 1523 state->last_end = end_pfn; 1524 state->last_nid = nid; 1525 } 1526 1527 return nid; 1528 } 1529 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ 1530 1531 int __meminit early_pfn_to_nid(unsigned long pfn) 1532 { 1533 static DEFINE_SPINLOCK(early_pfn_lock); 1534 int nid; 1535 1536 spin_lock(&early_pfn_lock); 1537 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1538 if (nid < 0) 1539 nid = first_online_node; 1540 spin_unlock(&early_pfn_lock); 1541 1542 return nid; 1543 } 1544 #endif /* CONFIG_NEED_MULTIPLE_NODES */ 1545 1546 void __init memblock_free_pages(struct page *page, unsigned long pfn, 1547 unsigned int order) 1548 { 1549 if (early_page_uninitialised(pfn)) 1550 return; 1551 __free_pages_core(page, order); 1552 } 1553 1554 /* 1555 * Check that the whole (or subset of) a pageblock given by the interval of 1556 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1557 * with the migration of free compaction scanner. The scanners then need to 1558 * use only pfn_valid_within() check for arches that allow holes within 1559 * pageblocks. 1560 * 1561 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1562 * 1563 * It's possible on some configurations to have a setup like node0 node1 node0 1564 * i.e. it's possible that all pages within a zones range of pages do not 1565 * belong to a single zone. We assume that a border between node0 and node1 1566 * can occur within a single pageblock, but not a node0 node1 node0 1567 * interleaving within a single pageblock. It is therefore sufficient to check 1568 * the first and last page of a pageblock and avoid checking each individual 1569 * page in a pageblock. 1570 */ 1571 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1572 unsigned long end_pfn, struct zone *zone) 1573 { 1574 struct page *start_page; 1575 struct page *end_page; 1576 1577 /* end_pfn is one past the range we are checking */ 1578 end_pfn--; 1579 1580 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1581 return NULL; 1582 1583 start_page = pfn_to_online_page(start_pfn); 1584 if (!start_page) 1585 return NULL; 1586 1587 if (page_zone(start_page) != zone) 1588 return NULL; 1589 1590 end_page = pfn_to_page(end_pfn); 1591 1592 /* This gives a shorter code than deriving page_zone(end_page) */ 1593 if (page_zone_id(start_page) != page_zone_id(end_page)) 1594 return NULL; 1595 1596 return start_page; 1597 } 1598 1599 void set_zone_contiguous(struct zone *zone) 1600 { 1601 unsigned long block_start_pfn = zone->zone_start_pfn; 1602 unsigned long block_end_pfn; 1603 1604 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1605 for (; block_start_pfn < zone_end_pfn(zone); 1606 block_start_pfn = block_end_pfn, 1607 block_end_pfn += pageblock_nr_pages) { 1608 1609 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1610 1611 if (!__pageblock_pfn_to_page(block_start_pfn, 1612 block_end_pfn, zone)) 1613 return; 1614 cond_resched(); 1615 } 1616 1617 /* We confirm that there is no hole */ 1618 zone->contiguous = true; 1619 } 1620 1621 void clear_zone_contiguous(struct zone *zone) 1622 { 1623 zone->contiguous = false; 1624 } 1625 1626 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1627 static void __init deferred_free_range(unsigned long pfn, 1628 unsigned long nr_pages) 1629 { 1630 struct page *page; 1631 unsigned long i; 1632 1633 if (!nr_pages) 1634 return; 1635 1636 page = pfn_to_page(pfn); 1637 1638 /* Free a large naturally-aligned chunk if possible */ 1639 if (nr_pages == pageblock_nr_pages && 1640 (pfn & (pageblock_nr_pages - 1)) == 0) { 1641 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1642 __free_pages_core(page, pageblock_order); 1643 return; 1644 } 1645 1646 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1647 if ((pfn & (pageblock_nr_pages - 1)) == 0) 1648 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1649 __free_pages_core(page, 0); 1650 } 1651 } 1652 1653 /* Completion tracking for deferred_init_memmap() threads */ 1654 static atomic_t pgdat_init_n_undone __initdata; 1655 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1656 1657 static inline void __init pgdat_init_report_one_done(void) 1658 { 1659 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1660 complete(&pgdat_init_all_done_comp); 1661 } 1662 1663 /* 1664 * Returns true if page needs to be initialized or freed to buddy allocator. 1665 * 1666 * First we check if pfn is valid on architectures where it is possible to have 1667 * holes within pageblock_nr_pages. On systems where it is not possible, this 1668 * function is optimized out. 1669 * 1670 * Then, we check if a current large page is valid by only checking the validity 1671 * of the head pfn. 1672 */ 1673 static inline bool __init deferred_pfn_valid(unsigned long pfn) 1674 { 1675 if (!pfn_valid_within(pfn)) 1676 return false; 1677 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn)) 1678 return false; 1679 return true; 1680 } 1681 1682 /* 1683 * Free pages to buddy allocator. Try to free aligned pages in 1684 * pageblock_nr_pages sizes. 1685 */ 1686 static void __init deferred_free_pages(unsigned long pfn, 1687 unsigned long end_pfn) 1688 { 1689 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1690 unsigned long nr_free = 0; 1691 1692 for (; pfn < end_pfn; pfn++) { 1693 if (!deferred_pfn_valid(pfn)) { 1694 deferred_free_range(pfn - nr_free, nr_free); 1695 nr_free = 0; 1696 } else if (!(pfn & nr_pgmask)) { 1697 deferred_free_range(pfn - nr_free, nr_free); 1698 nr_free = 1; 1699 } else { 1700 nr_free++; 1701 } 1702 } 1703 /* Free the last block of pages to allocator */ 1704 deferred_free_range(pfn - nr_free, nr_free); 1705 } 1706 1707 /* 1708 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 1709 * by performing it only once every pageblock_nr_pages. 1710 * Return number of pages initialized. 1711 */ 1712 static unsigned long __init deferred_init_pages(struct zone *zone, 1713 unsigned long pfn, 1714 unsigned long end_pfn) 1715 { 1716 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1717 int nid = zone_to_nid(zone); 1718 unsigned long nr_pages = 0; 1719 int zid = zone_idx(zone); 1720 struct page *page = NULL; 1721 1722 for (; pfn < end_pfn; pfn++) { 1723 if (!deferred_pfn_valid(pfn)) { 1724 page = NULL; 1725 continue; 1726 } else if (!page || !(pfn & nr_pgmask)) { 1727 page = pfn_to_page(pfn); 1728 } else { 1729 page++; 1730 } 1731 __init_single_page(page, pfn, zid, nid); 1732 nr_pages++; 1733 } 1734 return (nr_pages); 1735 } 1736 1737 /* 1738 * This function is meant to pre-load the iterator for the zone init. 1739 * Specifically it walks through the ranges until we are caught up to the 1740 * first_init_pfn value and exits there. If we never encounter the value we 1741 * return false indicating there are no valid ranges left. 1742 */ 1743 static bool __init 1744 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, 1745 unsigned long *spfn, unsigned long *epfn, 1746 unsigned long first_init_pfn) 1747 { 1748 u64 j; 1749 1750 /* 1751 * Start out by walking through the ranges in this zone that have 1752 * already been initialized. We don't need to do anything with them 1753 * so we just need to flush them out of the system. 1754 */ 1755 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { 1756 if (*epfn <= first_init_pfn) 1757 continue; 1758 if (*spfn < first_init_pfn) 1759 *spfn = first_init_pfn; 1760 *i = j; 1761 return true; 1762 } 1763 1764 return false; 1765 } 1766 1767 /* 1768 * Initialize and free pages. We do it in two loops: first we initialize 1769 * struct page, then free to buddy allocator, because while we are 1770 * freeing pages we can access pages that are ahead (computing buddy 1771 * page in __free_one_page()). 1772 * 1773 * In order to try and keep some memory in the cache we have the loop 1774 * broken along max page order boundaries. This way we will not cause 1775 * any issues with the buddy page computation. 1776 */ 1777 static unsigned long __init 1778 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, 1779 unsigned long *end_pfn) 1780 { 1781 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); 1782 unsigned long spfn = *start_pfn, epfn = *end_pfn; 1783 unsigned long nr_pages = 0; 1784 u64 j = *i; 1785 1786 /* First we loop through and initialize the page values */ 1787 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { 1788 unsigned long t; 1789 1790 if (mo_pfn <= *start_pfn) 1791 break; 1792 1793 t = min(mo_pfn, *end_pfn); 1794 nr_pages += deferred_init_pages(zone, *start_pfn, t); 1795 1796 if (mo_pfn < *end_pfn) { 1797 *start_pfn = mo_pfn; 1798 break; 1799 } 1800 } 1801 1802 /* Reset values and now loop through freeing pages as needed */ 1803 swap(j, *i); 1804 1805 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { 1806 unsigned long t; 1807 1808 if (mo_pfn <= spfn) 1809 break; 1810 1811 t = min(mo_pfn, epfn); 1812 deferred_free_pages(spfn, t); 1813 1814 if (mo_pfn <= epfn) 1815 break; 1816 } 1817 1818 return nr_pages; 1819 } 1820 1821 static void __init 1822 deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, 1823 void *arg) 1824 { 1825 unsigned long spfn, epfn; 1826 struct zone *zone = arg; 1827 u64 i; 1828 1829 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); 1830 1831 /* 1832 * Initialize and free pages in MAX_ORDER sized increments so that we 1833 * can avoid introducing any issues with the buddy allocator. 1834 */ 1835 while (spfn < end_pfn) { 1836 deferred_init_maxorder(&i, zone, &spfn, &epfn); 1837 cond_resched(); 1838 } 1839 } 1840 1841 /* An arch may override for more concurrency. */ 1842 __weak int __init 1843 deferred_page_init_max_threads(const struct cpumask *node_cpumask) 1844 { 1845 return 1; 1846 } 1847 1848 /* Initialise remaining memory on a node */ 1849 static int __init deferred_init_memmap(void *data) 1850 { 1851 pg_data_t *pgdat = data; 1852 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 1853 unsigned long spfn = 0, epfn = 0; 1854 unsigned long first_init_pfn, flags; 1855 unsigned long start = jiffies; 1856 struct zone *zone; 1857 int zid, max_threads; 1858 u64 i; 1859 1860 /* Bind memory initialisation thread to a local node if possible */ 1861 if (!cpumask_empty(cpumask)) 1862 set_cpus_allowed_ptr(current, cpumask); 1863 1864 pgdat_resize_lock(pgdat, &flags); 1865 first_init_pfn = pgdat->first_deferred_pfn; 1866 if (first_init_pfn == ULONG_MAX) { 1867 pgdat_resize_unlock(pgdat, &flags); 1868 pgdat_init_report_one_done(); 1869 return 0; 1870 } 1871 1872 /* Sanity check boundaries */ 1873 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 1874 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 1875 pgdat->first_deferred_pfn = ULONG_MAX; 1876 1877 /* 1878 * Once we unlock here, the zone cannot be grown anymore, thus if an 1879 * interrupt thread must allocate this early in boot, zone must be 1880 * pre-grown prior to start of deferred page initialization. 1881 */ 1882 pgdat_resize_unlock(pgdat, &flags); 1883 1884 /* Only the highest zone is deferred so find it */ 1885 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1886 zone = pgdat->node_zones + zid; 1887 if (first_init_pfn < zone_end_pfn(zone)) 1888 break; 1889 } 1890 1891 /* If the zone is empty somebody else may have cleared out the zone */ 1892 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 1893 first_init_pfn)) 1894 goto zone_empty; 1895 1896 max_threads = deferred_page_init_max_threads(cpumask); 1897 1898 while (spfn < epfn) { 1899 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION); 1900 struct padata_mt_job job = { 1901 .thread_fn = deferred_init_memmap_chunk, 1902 .fn_arg = zone, 1903 .start = spfn, 1904 .size = epfn_align - spfn, 1905 .align = PAGES_PER_SECTION, 1906 .min_chunk = PAGES_PER_SECTION, 1907 .max_threads = max_threads, 1908 }; 1909 1910 padata_do_multithreaded(&job); 1911 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 1912 epfn_align); 1913 } 1914 zone_empty: 1915 /* Sanity check that the next zone really is unpopulated */ 1916 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 1917 1918 pr_info("node %d deferred pages initialised in %ums\n", 1919 pgdat->node_id, jiffies_to_msecs(jiffies - start)); 1920 1921 pgdat_init_report_one_done(); 1922 return 0; 1923 } 1924 1925 /* 1926 * If this zone has deferred pages, try to grow it by initializing enough 1927 * deferred pages to satisfy the allocation specified by order, rounded up to 1928 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 1929 * of SECTION_SIZE bytes by initializing struct pages in increments of 1930 * PAGES_PER_SECTION * sizeof(struct page) bytes. 1931 * 1932 * Return true when zone was grown, otherwise return false. We return true even 1933 * when we grow less than requested, to let the caller decide if there are 1934 * enough pages to satisfy the allocation. 1935 * 1936 * Note: We use noinline because this function is needed only during boot, and 1937 * it is called from a __ref function _deferred_grow_zone. This way we are 1938 * making sure that it is not inlined into permanent text section. 1939 */ 1940 static noinline bool __init 1941 deferred_grow_zone(struct zone *zone, unsigned int order) 1942 { 1943 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 1944 pg_data_t *pgdat = zone->zone_pgdat; 1945 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 1946 unsigned long spfn, epfn, flags; 1947 unsigned long nr_pages = 0; 1948 u64 i; 1949 1950 /* Only the last zone may have deferred pages */ 1951 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 1952 return false; 1953 1954 pgdat_resize_lock(pgdat, &flags); 1955 1956 /* 1957 * If someone grew this zone while we were waiting for spinlock, return 1958 * true, as there might be enough pages already. 1959 */ 1960 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 1961 pgdat_resize_unlock(pgdat, &flags); 1962 return true; 1963 } 1964 1965 /* If the zone is empty somebody else may have cleared out the zone */ 1966 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 1967 first_deferred_pfn)) { 1968 pgdat->first_deferred_pfn = ULONG_MAX; 1969 pgdat_resize_unlock(pgdat, &flags); 1970 /* Retry only once. */ 1971 return first_deferred_pfn != ULONG_MAX; 1972 } 1973 1974 /* 1975 * Initialize and free pages in MAX_ORDER sized increments so 1976 * that we can avoid introducing any issues with the buddy 1977 * allocator. 1978 */ 1979 while (spfn < epfn) { 1980 /* update our first deferred PFN for this section */ 1981 first_deferred_pfn = spfn; 1982 1983 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 1984 touch_nmi_watchdog(); 1985 1986 /* We should only stop along section boundaries */ 1987 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) 1988 continue; 1989 1990 /* If our quota has been met we can stop here */ 1991 if (nr_pages >= nr_pages_needed) 1992 break; 1993 } 1994 1995 pgdat->first_deferred_pfn = spfn; 1996 pgdat_resize_unlock(pgdat, &flags); 1997 1998 return nr_pages > 0; 1999 } 2000 2001 /* 2002 * deferred_grow_zone() is __init, but it is called from 2003 * get_page_from_freelist() during early boot until deferred_pages permanently 2004 * disables this call. This is why we have refdata wrapper to avoid warning, 2005 * and to ensure that the function body gets unloaded. 2006 */ 2007 static bool __ref 2008 _deferred_grow_zone(struct zone *zone, unsigned int order) 2009 { 2010 return deferred_grow_zone(zone, order); 2011 } 2012 2013 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 2014 2015 void __init page_alloc_init_late(void) 2016 { 2017 struct zone *zone; 2018 int nid; 2019 2020 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 2021 2022 /* There will be num_node_state(N_MEMORY) threads */ 2023 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 2024 for_each_node_state(nid, N_MEMORY) { 2025 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 2026 } 2027 2028 /* Block until all are initialised */ 2029 wait_for_completion(&pgdat_init_all_done_comp); 2030 2031 /* 2032 * The number of managed pages has changed due to the initialisation 2033 * so the pcpu batch and high limits needs to be updated or the limits 2034 * will be artificially small. 2035 */ 2036 for_each_populated_zone(zone) 2037 zone_pcp_update(zone); 2038 2039 /* 2040 * We initialized the rest of the deferred pages. Permanently disable 2041 * on-demand struct page initialization. 2042 */ 2043 static_branch_disable(&deferred_pages); 2044 2045 /* Reinit limits that are based on free pages after the kernel is up */ 2046 files_maxfiles_init(); 2047 #endif 2048 2049 /* Discard memblock private memory */ 2050 memblock_discard(); 2051 2052 for_each_node_state(nid, N_MEMORY) 2053 shuffle_free_memory(NODE_DATA(nid)); 2054 2055 for_each_populated_zone(zone) 2056 set_zone_contiguous(zone); 2057 } 2058 2059 #ifdef CONFIG_CMA 2060 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 2061 void __init init_cma_reserved_pageblock(struct page *page) 2062 { 2063 unsigned i = pageblock_nr_pages; 2064 struct page *p = page; 2065 2066 do { 2067 __ClearPageReserved(p); 2068 set_page_count(p, 0); 2069 } while (++p, --i); 2070 2071 set_pageblock_migratetype(page, MIGRATE_CMA); 2072 2073 if (pageblock_order >= MAX_ORDER) { 2074 i = pageblock_nr_pages; 2075 p = page; 2076 do { 2077 set_page_refcounted(p); 2078 __free_pages(p, MAX_ORDER - 1); 2079 p += MAX_ORDER_NR_PAGES; 2080 } while (i -= MAX_ORDER_NR_PAGES); 2081 } else { 2082 set_page_refcounted(page); 2083 __free_pages(page, pageblock_order); 2084 } 2085 2086 adjust_managed_page_count(page, pageblock_nr_pages); 2087 } 2088 #endif 2089 2090 /* 2091 * The order of subdivision here is critical for the IO subsystem. 2092 * Please do not alter this order without good reasons and regression 2093 * testing. Specifically, as large blocks of memory are subdivided, 2094 * the order in which smaller blocks are delivered depends on the order 2095 * they're subdivided in this function. This is the primary factor 2096 * influencing the order in which pages are delivered to the IO 2097 * subsystem according to empirical testing, and this is also justified 2098 * by considering the behavior of a buddy system containing a single 2099 * large block of memory acted on by a series of small allocations. 2100 * This behavior is a critical factor in sglist merging's success. 2101 * 2102 * -- nyc 2103 */ 2104 static inline void expand(struct zone *zone, struct page *page, 2105 int low, int high, int migratetype) 2106 { 2107 unsigned long size = 1 << high; 2108 2109 while (high > low) { 2110 high--; 2111 size >>= 1; 2112 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 2113 2114 /* 2115 * Mark as guard pages (or page), that will allow to 2116 * merge back to allocator when buddy will be freed. 2117 * Corresponding page table entries will not be touched, 2118 * pages will stay not present in virtual address space 2119 */ 2120 if (set_page_guard(zone, &page[size], high, migratetype)) 2121 continue; 2122 2123 add_to_free_list(&page[size], zone, high, migratetype); 2124 set_page_order(&page[size], high); 2125 } 2126 } 2127 2128 static void check_new_page_bad(struct page *page) 2129 { 2130 if (unlikely(page->flags & __PG_HWPOISON)) { 2131 /* Don't complain about hwpoisoned pages */ 2132 page_mapcount_reset(page); /* remove PageBuddy */ 2133 return; 2134 } 2135 2136 bad_page(page, 2137 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP)); 2138 } 2139 2140 /* 2141 * This page is about to be returned from the page allocator 2142 */ 2143 static inline int check_new_page(struct page *page) 2144 { 2145 if (likely(page_expected_state(page, 2146 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 2147 return 0; 2148 2149 check_new_page_bad(page); 2150 return 1; 2151 } 2152 2153 static inline bool free_pages_prezeroed(void) 2154 { 2155 return (IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) && 2156 page_poisoning_enabled()) || want_init_on_free(); 2157 } 2158 2159 #ifdef CONFIG_DEBUG_VM 2160 /* 2161 * With DEBUG_VM enabled, order-0 pages are checked for expected state when 2162 * being allocated from pcp lists. With debug_pagealloc also enabled, they are 2163 * also checked when pcp lists are refilled from the free lists. 2164 */ 2165 static inline bool check_pcp_refill(struct page *page) 2166 { 2167 if (debug_pagealloc_enabled_static()) 2168 return check_new_page(page); 2169 else 2170 return false; 2171 } 2172 2173 static inline bool check_new_pcp(struct page *page) 2174 { 2175 return check_new_page(page); 2176 } 2177 #else 2178 /* 2179 * With DEBUG_VM disabled, free order-0 pages are checked for expected state 2180 * when pcp lists are being refilled from the free lists. With debug_pagealloc 2181 * enabled, they are also checked when being allocated from the pcp lists. 2182 */ 2183 static inline bool check_pcp_refill(struct page *page) 2184 { 2185 return check_new_page(page); 2186 } 2187 static inline bool check_new_pcp(struct page *page) 2188 { 2189 if (debug_pagealloc_enabled_static()) 2190 return check_new_page(page); 2191 else 2192 return false; 2193 } 2194 #endif /* CONFIG_DEBUG_VM */ 2195 2196 static bool check_new_pages(struct page *page, unsigned int order) 2197 { 2198 int i; 2199 for (i = 0; i < (1 << order); i++) { 2200 struct page *p = page + i; 2201 2202 if (unlikely(check_new_page(p))) 2203 return true; 2204 } 2205 2206 return false; 2207 } 2208 2209 inline void post_alloc_hook(struct page *page, unsigned int order, 2210 gfp_t gfp_flags) 2211 { 2212 set_page_private(page, 0); 2213 set_page_refcounted(page); 2214 2215 arch_alloc_page(page, order); 2216 if (debug_pagealloc_enabled_static()) 2217 kernel_map_pages(page, 1 << order, 1); 2218 kasan_alloc_pages(page, order); 2219 kernel_poison_pages(page, 1 << order, 1); 2220 set_page_owner(page, order, gfp_flags); 2221 } 2222 2223 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 2224 unsigned int alloc_flags) 2225 { 2226 post_alloc_hook(page, order, gfp_flags); 2227 2228 if (!free_pages_prezeroed() && want_init_on_alloc(gfp_flags)) 2229 kernel_init_free_pages(page, 1 << order); 2230 2231 if (order && (gfp_flags & __GFP_COMP)) 2232 prep_compound_page(page, order); 2233 2234 /* 2235 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 2236 * allocate the page. The expectation is that the caller is taking 2237 * steps that will free more memory. The caller should avoid the page 2238 * being used for !PFMEMALLOC purposes. 2239 */ 2240 if (alloc_flags & ALLOC_NO_WATERMARKS) 2241 set_page_pfmemalloc(page); 2242 else 2243 clear_page_pfmemalloc(page); 2244 } 2245 2246 /* 2247 * Go through the free lists for the given migratetype and remove 2248 * the smallest available page from the freelists 2249 */ 2250 static __always_inline 2251 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 2252 int migratetype) 2253 { 2254 unsigned int current_order; 2255 struct free_area *area; 2256 struct page *page; 2257 2258 /* Find a page of the appropriate size in the preferred list */ 2259 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 2260 area = &(zone->free_area[current_order]); 2261 page = get_page_from_free_area(area, migratetype); 2262 if (!page) 2263 continue; 2264 del_page_from_free_list(page, zone, current_order); 2265 expand(zone, page, order, current_order, migratetype); 2266 set_pcppage_migratetype(page, migratetype); 2267 return page; 2268 } 2269 2270 return NULL; 2271 } 2272 2273 2274 /* 2275 * This array describes the order lists are fallen back to when 2276 * the free lists for the desirable migrate type are depleted 2277 */ 2278 static int fallbacks[MIGRATE_TYPES][3] = { 2279 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2280 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 2281 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2282 #ifdef CONFIG_CMA 2283 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */ 2284 #endif 2285 #ifdef CONFIG_MEMORY_ISOLATION 2286 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */ 2287 #endif 2288 }; 2289 2290 #ifdef CONFIG_CMA 2291 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2292 unsigned int order) 2293 { 2294 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 2295 } 2296 #else 2297 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2298 unsigned int order) { return NULL; } 2299 #endif 2300 2301 /* 2302 * Move the free pages in a range to the free lists of the requested type. 2303 * Note that start_page and end_pages are not aligned on a pageblock 2304 * boundary. If alignment is required, use move_freepages_block() 2305 */ 2306 static int move_freepages(struct zone *zone, 2307 struct page *start_page, struct page *end_page, 2308 int migratetype, int *num_movable) 2309 { 2310 struct page *page; 2311 unsigned int order; 2312 int pages_moved = 0; 2313 2314 for (page = start_page; page <= end_page;) { 2315 if (!pfn_valid_within(page_to_pfn(page))) { 2316 page++; 2317 continue; 2318 } 2319 2320 if (!PageBuddy(page)) { 2321 /* 2322 * We assume that pages that could be isolated for 2323 * migration are movable. But we don't actually try 2324 * isolating, as that would be expensive. 2325 */ 2326 if (num_movable && 2327 (PageLRU(page) || __PageMovable(page))) 2328 (*num_movable)++; 2329 2330 page++; 2331 continue; 2332 } 2333 2334 /* Make sure we are not inadvertently changing nodes */ 2335 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2336 VM_BUG_ON_PAGE(page_zone(page) != zone, page); 2337 2338 order = page_order(page); 2339 move_to_free_list(page, zone, order, migratetype); 2340 page += 1 << order; 2341 pages_moved += 1 << order; 2342 } 2343 2344 return pages_moved; 2345 } 2346 2347 int move_freepages_block(struct zone *zone, struct page *page, 2348 int migratetype, int *num_movable) 2349 { 2350 unsigned long start_pfn, end_pfn; 2351 struct page *start_page, *end_page; 2352 2353 if (num_movable) 2354 *num_movable = 0; 2355 2356 start_pfn = page_to_pfn(page); 2357 start_pfn = start_pfn & ~(pageblock_nr_pages-1); 2358 start_page = pfn_to_page(start_pfn); 2359 end_page = start_page + pageblock_nr_pages - 1; 2360 end_pfn = start_pfn + pageblock_nr_pages - 1; 2361 2362 /* Do not cross zone boundaries */ 2363 if (!zone_spans_pfn(zone, start_pfn)) 2364 start_page = page; 2365 if (!zone_spans_pfn(zone, end_pfn)) 2366 return 0; 2367 2368 return move_freepages(zone, start_page, end_page, migratetype, 2369 num_movable); 2370 } 2371 2372 static void change_pageblock_range(struct page *pageblock_page, 2373 int start_order, int migratetype) 2374 { 2375 int nr_pageblocks = 1 << (start_order - pageblock_order); 2376 2377 while (nr_pageblocks--) { 2378 set_pageblock_migratetype(pageblock_page, migratetype); 2379 pageblock_page += pageblock_nr_pages; 2380 } 2381 } 2382 2383 /* 2384 * When we are falling back to another migratetype during allocation, try to 2385 * steal extra free pages from the same pageblocks to satisfy further 2386 * allocations, instead of polluting multiple pageblocks. 2387 * 2388 * If we are stealing a relatively large buddy page, it is likely there will 2389 * be more free pages in the pageblock, so try to steal them all. For 2390 * reclaimable and unmovable allocations, we steal regardless of page size, 2391 * as fragmentation caused by those allocations polluting movable pageblocks 2392 * is worse than movable allocations stealing from unmovable and reclaimable 2393 * pageblocks. 2394 */ 2395 static bool can_steal_fallback(unsigned int order, int start_mt) 2396 { 2397 /* 2398 * Leaving this order check is intended, although there is 2399 * relaxed order check in next check. The reason is that 2400 * we can actually steal whole pageblock if this condition met, 2401 * but, below check doesn't guarantee it and that is just heuristic 2402 * so could be changed anytime. 2403 */ 2404 if (order >= pageblock_order) 2405 return true; 2406 2407 if (order >= pageblock_order / 2 || 2408 start_mt == MIGRATE_RECLAIMABLE || 2409 start_mt == MIGRATE_UNMOVABLE || 2410 page_group_by_mobility_disabled) 2411 return true; 2412 2413 return false; 2414 } 2415 2416 static inline void boost_watermark(struct zone *zone) 2417 { 2418 unsigned long max_boost; 2419 2420 if (!watermark_boost_factor) 2421 return; 2422 /* 2423 * Don't bother in zones that are unlikely to produce results. 2424 * On small machines, including kdump capture kernels running 2425 * in a small area, boosting the watermark can cause an out of 2426 * memory situation immediately. 2427 */ 2428 if ((pageblock_nr_pages * 4) > zone_managed_pages(zone)) 2429 return; 2430 2431 max_boost = mult_frac(zone->_watermark[WMARK_HIGH], 2432 watermark_boost_factor, 10000); 2433 2434 /* 2435 * high watermark may be uninitialised if fragmentation occurs 2436 * very early in boot so do not boost. We do not fall 2437 * through and boost by pageblock_nr_pages as failing 2438 * allocations that early means that reclaim is not going 2439 * to help and it may even be impossible to reclaim the 2440 * boosted watermark resulting in a hang. 2441 */ 2442 if (!max_boost) 2443 return; 2444 2445 max_boost = max(pageblock_nr_pages, max_boost); 2446 2447 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages, 2448 max_boost); 2449 } 2450 2451 /* 2452 * This function implements actual steal behaviour. If order is large enough, 2453 * we can steal whole pageblock. If not, we first move freepages in this 2454 * pageblock to our migratetype and determine how many already-allocated pages 2455 * are there in the pageblock with a compatible migratetype. If at least half 2456 * of pages are free or compatible, we can change migratetype of the pageblock 2457 * itself, so pages freed in the future will be put on the correct free list. 2458 */ 2459 static void steal_suitable_fallback(struct zone *zone, struct page *page, 2460 unsigned int alloc_flags, int start_type, bool whole_block) 2461 { 2462 unsigned int current_order = page_order(page); 2463 int free_pages, movable_pages, alike_pages; 2464 int old_block_type; 2465 2466 old_block_type = get_pageblock_migratetype(page); 2467 2468 /* 2469 * This can happen due to races and we want to prevent broken 2470 * highatomic accounting. 2471 */ 2472 if (is_migrate_highatomic(old_block_type)) 2473 goto single_page; 2474 2475 /* Take ownership for orders >= pageblock_order */ 2476 if (current_order >= pageblock_order) { 2477 change_pageblock_range(page, current_order, start_type); 2478 goto single_page; 2479 } 2480 2481 /* 2482 * Boost watermarks to increase reclaim pressure to reduce the 2483 * likelihood of future fallbacks. Wake kswapd now as the node 2484 * may be balanced overall and kswapd will not wake naturally. 2485 */ 2486 boost_watermark(zone); 2487 if (alloc_flags & ALLOC_KSWAPD) 2488 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 2489 2490 /* We are not allowed to try stealing from the whole block */ 2491 if (!whole_block) 2492 goto single_page; 2493 2494 free_pages = move_freepages_block(zone, page, start_type, 2495 &movable_pages); 2496 /* 2497 * Determine how many pages are compatible with our allocation. 2498 * For movable allocation, it's the number of movable pages which 2499 * we just obtained. For other types it's a bit more tricky. 2500 */ 2501 if (start_type == MIGRATE_MOVABLE) { 2502 alike_pages = movable_pages; 2503 } else { 2504 /* 2505 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation 2506 * to MOVABLE pageblock, consider all non-movable pages as 2507 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or 2508 * vice versa, be conservative since we can't distinguish the 2509 * exact migratetype of non-movable pages. 2510 */ 2511 if (old_block_type == MIGRATE_MOVABLE) 2512 alike_pages = pageblock_nr_pages 2513 - (free_pages + movable_pages); 2514 else 2515 alike_pages = 0; 2516 } 2517 2518 /* moving whole block can fail due to zone boundary conditions */ 2519 if (!free_pages) 2520 goto single_page; 2521 2522 /* 2523 * If a sufficient number of pages in the block are either free or of 2524 * comparable migratability as our allocation, claim the whole block. 2525 */ 2526 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || 2527 page_group_by_mobility_disabled) 2528 set_pageblock_migratetype(page, start_type); 2529 2530 return; 2531 2532 single_page: 2533 move_to_free_list(page, zone, current_order, start_type); 2534 } 2535 2536 /* 2537 * Check whether there is a suitable fallback freepage with requested order. 2538 * If only_stealable is true, this function returns fallback_mt only if 2539 * we can steal other freepages all together. This would help to reduce 2540 * fragmentation due to mixed migratetype pages in one pageblock. 2541 */ 2542 int find_suitable_fallback(struct free_area *area, unsigned int order, 2543 int migratetype, bool only_stealable, bool *can_steal) 2544 { 2545 int i; 2546 int fallback_mt; 2547 2548 if (area->nr_free == 0) 2549 return -1; 2550 2551 *can_steal = false; 2552 for (i = 0;; i++) { 2553 fallback_mt = fallbacks[migratetype][i]; 2554 if (fallback_mt == MIGRATE_TYPES) 2555 break; 2556 2557 if (free_area_empty(area, fallback_mt)) 2558 continue; 2559 2560 if (can_steal_fallback(order, migratetype)) 2561 *can_steal = true; 2562 2563 if (!only_stealable) 2564 return fallback_mt; 2565 2566 if (*can_steal) 2567 return fallback_mt; 2568 } 2569 2570 return -1; 2571 } 2572 2573 /* 2574 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2575 * there are no empty page blocks that contain a page with a suitable order 2576 */ 2577 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2578 unsigned int alloc_order) 2579 { 2580 int mt; 2581 unsigned long max_managed, flags; 2582 2583 /* 2584 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2585 * Check is race-prone but harmless. 2586 */ 2587 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages; 2588 if (zone->nr_reserved_highatomic >= max_managed) 2589 return; 2590 2591 spin_lock_irqsave(&zone->lock, flags); 2592 2593 /* Recheck the nr_reserved_highatomic limit under the lock */ 2594 if (zone->nr_reserved_highatomic >= max_managed) 2595 goto out_unlock; 2596 2597 /* Yoink! */ 2598 mt = get_pageblock_migratetype(page); 2599 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt) 2600 && !is_migrate_cma(mt)) { 2601 zone->nr_reserved_highatomic += pageblock_nr_pages; 2602 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2603 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); 2604 } 2605 2606 out_unlock: 2607 spin_unlock_irqrestore(&zone->lock, flags); 2608 } 2609 2610 /* 2611 * Used when an allocation is about to fail under memory pressure. This 2612 * potentially hurts the reliability of high-order allocations when under 2613 * intense memory pressure but failed atomic allocations should be easier 2614 * to recover from than an OOM. 2615 * 2616 * If @force is true, try to unreserve a pageblock even though highatomic 2617 * pageblock is exhausted. 2618 */ 2619 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 2620 bool force) 2621 { 2622 struct zonelist *zonelist = ac->zonelist; 2623 unsigned long flags; 2624 struct zoneref *z; 2625 struct zone *zone; 2626 struct page *page; 2627 int order; 2628 bool ret; 2629 2630 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx, 2631 ac->nodemask) { 2632 /* 2633 * Preserve at least one pageblock unless memory pressure 2634 * is really high. 2635 */ 2636 if (!force && zone->nr_reserved_highatomic <= 2637 pageblock_nr_pages) 2638 continue; 2639 2640 spin_lock_irqsave(&zone->lock, flags); 2641 for (order = 0; order < MAX_ORDER; order++) { 2642 struct free_area *area = &(zone->free_area[order]); 2643 2644 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); 2645 if (!page) 2646 continue; 2647 2648 /* 2649 * In page freeing path, migratetype change is racy so 2650 * we can counter several free pages in a pageblock 2651 * in this loop althoug we changed the pageblock type 2652 * from highatomic to ac->migratetype. So we should 2653 * adjust the count once. 2654 */ 2655 if (is_migrate_highatomic_page(page)) { 2656 /* 2657 * It should never happen but changes to 2658 * locking could inadvertently allow a per-cpu 2659 * drain to add pages to MIGRATE_HIGHATOMIC 2660 * while unreserving so be safe and watch for 2661 * underflows. 2662 */ 2663 zone->nr_reserved_highatomic -= min( 2664 pageblock_nr_pages, 2665 zone->nr_reserved_highatomic); 2666 } 2667 2668 /* 2669 * Convert to ac->migratetype and avoid the normal 2670 * pageblock stealing heuristics. Minimally, the caller 2671 * is doing the work and needs the pages. More 2672 * importantly, if the block was always converted to 2673 * MIGRATE_UNMOVABLE or another type then the number 2674 * of pageblocks that cannot be completely freed 2675 * may increase. 2676 */ 2677 set_pageblock_migratetype(page, ac->migratetype); 2678 ret = move_freepages_block(zone, page, ac->migratetype, 2679 NULL); 2680 if (ret) { 2681 spin_unlock_irqrestore(&zone->lock, flags); 2682 return ret; 2683 } 2684 } 2685 spin_unlock_irqrestore(&zone->lock, flags); 2686 } 2687 2688 return false; 2689 } 2690 2691 /* 2692 * Try finding a free buddy page on the fallback list and put it on the free 2693 * list of requested migratetype, possibly along with other pages from the same 2694 * block, depending on fragmentation avoidance heuristics. Returns true if 2695 * fallback was found so that __rmqueue_smallest() can grab it. 2696 * 2697 * The use of signed ints for order and current_order is a deliberate 2698 * deviation from the rest of this file, to make the for loop 2699 * condition simpler. 2700 */ 2701 static __always_inline bool 2702 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, 2703 unsigned int alloc_flags) 2704 { 2705 struct free_area *area; 2706 int current_order; 2707 int min_order = order; 2708 struct page *page; 2709 int fallback_mt; 2710 bool can_steal; 2711 2712 /* 2713 * Do not steal pages from freelists belonging to other pageblocks 2714 * i.e. orders < pageblock_order. If there are no local zones free, 2715 * the zonelists will be reiterated without ALLOC_NOFRAGMENT. 2716 */ 2717 if (alloc_flags & ALLOC_NOFRAGMENT) 2718 min_order = pageblock_order; 2719 2720 /* 2721 * Find the largest available free page in the other list. This roughly 2722 * approximates finding the pageblock with the most free pages, which 2723 * would be too costly to do exactly. 2724 */ 2725 for (current_order = MAX_ORDER - 1; current_order >= min_order; 2726 --current_order) { 2727 area = &(zone->free_area[current_order]); 2728 fallback_mt = find_suitable_fallback(area, current_order, 2729 start_migratetype, false, &can_steal); 2730 if (fallback_mt == -1) 2731 continue; 2732 2733 /* 2734 * We cannot steal all free pages from the pageblock and the 2735 * requested migratetype is movable. In that case it's better to 2736 * steal and split the smallest available page instead of the 2737 * largest available page, because even if the next movable 2738 * allocation falls back into a different pageblock than this 2739 * one, it won't cause permanent fragmentation. 2740 */ 2741 if (!can_steal && start_migratetype == MIGRATE_MOVABLE 2742 && current_order > order) 2743 goto find_smallest; 2744 2745 goto do_steal; 2746 } 2747 2748 return false; 2749 2750 find_smallest: 2751 for (current_order = order; current_order < MAX_ORDER; 2752 current_order++) { 2753 area = &(zone->free_area[current_order]); 2754 fallback_mt = find_suitable_fallback(area, current_order, 2755 start_migratetype, false, &can_steal); 2756 if (fallback_mt != -1) 2757 break; 2758 } 2759 2760 /* 2761 * This should not happen - we already found a suitable fallback 2762 * when looking for the largest page. 2763 */ 2764 VM_BUG_ON(current_order == MAX_ORDER); 2765 2766 do_steal: 2767 page = get_page_from_free_area(area, fallback_mt); 2768 2769 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype, 2770 can_steal); 2771 2772 trace_mm_page_alloc_extfrag(page, order, current_order, 2773 start_migratetype, fallback_mt); 2774 2775 return true; 2776 2777 } 2778 2779 /* 2780 * Do the hard work of removing an element from the buddy allocator. 2781 * Call me with the zone->lock already held. 2782 */ 2783 static __always_inline struct page * 2784 __rmqueue(struct zone *zone, unsigned int order, int migratetype, 2785 unsigned int alloc_flags) 2786 { 2787 struct page *page; 2788 2789 #ifdef CONFIG_CMA 2790 /* 2791 * Balance movable allocations between regular and CMA areas by 2792 * allocating from CMA when over half of the zone's free memory 2793 * is in the CMA area. 2794 */ 2795 if (alloc_flags & ALLOC_CMA && 2796 zone_page_state(zone, NR_FREE_CMA_PAGES) > 2797 zone_page_state(zone, NR_FREE_PAGES) / 2) { 2798 page = __rmqueue_cma_fallback(zone, order); 2799 if (page) 2800 return page; 2801 } 2802 #endif 2803 retry: 2804 page = __rmqueue_smallest(zone, order, migratetype); 2805 if (unlikely(!page)) { 2806 if (alloc_flags & ALLOC_CMA) 2807 page = __rmqueue_cma_fallback(zone, order); 2808 2809 if (!page && __rmqueue_fallback(zone, order, migratetype, 2810 alloc_flags)) 2811 goto retry; 2812 } 2813 2814 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2815 return page; 2816 } 2817 2818 /* 2819 * Obtain a specified number of elements from the buddy allocator, all under 2820 * a single hold of the lock, for efficiency. Add them to the supplied list. 2821 * Returns the number of new pages which were placed at *list. 2822 */ 2823 static int rmqueue_bulk(struct zone *zone, unsigned int order, 2824 unsigned long count, struct list_head *list, 2825 int migratetype, unsigned int alloc_flags) 2826 { 2827 int i, alloced = 0; 2828 2829 spin_lock(&zone->lock); 2830 for (i = 0; i < count; ++i) { 2831 struct page *page = __rmqueue(zone, order, migratetype, 2832 alloc_flags); 2833 if (unlikely(page == NULL)) 2834 break; 2835 2836 if (unlikely(check_pcp_refill(page))) 2837 continue; 2838 2839 /* 2840 * Split buddy pages returned by expand() are received here in 2841 * physical page order. The page is added to the tail of 2842 * caller's list. From the callers perspective, the linked list 2843 * is ordered by page number under some conditions. This is 2844 * useful for IO devices that can forward direction from the 2845 * head, thus also in the physical page order. This is useful 2846 * for IO devices that can merge IO requests if the physical 2847 * pages are ordered properly. 2848 */ 2849 list_add_tail(&page->lru, list); 2850 alloced++; 2851 if (is_migrate_cma(get_pcppage_migratetype(page))) 2852 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 2853 -(1 << order)); 2854 } 2855 2856 /* 2857 * i pages were removed from the buddy list even if some leak due 2858 * to check_pcp_refill failing so adjust NR_FREE_PAGES based 2859 * on i. Do not confuse with 'alloced' which is the number of 2860 * pages added to the pcp list. 2861 */ 2862 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 2863 spin_unlock(&zone->lock); 2864 return alloced; 2865 } 2866 2867 #ifdef CONFIG_NUMA 2868 /* 2869 * Called from the vmstat counter updater to drain pagesets of this 2870 * currently executing processor on remote nodes after they have 2871 * expired. 2872 * 2873 * Note that this function must be called with the thread pinned to 2874 * a single processor. 2875 */ 2876 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 2877 { 2878 unsigned long flags; 2879 int to_drain, batch; 2880 2881 local_irq_save(flags); 2882 batch = READ_ONCE(pcp->batch); 2883 to_drain = min(pcp->count, batch); 2884 if (to_drain > 0) 2885 free_pcppages_bulk(zone, to_drain, pcp); 2886 local_irq_restore(flags); 2887 } 2888 #endif 2889 2890 /* 2891 * Drain pcplists of the indicated processor and zone. 2892 * 2893 * The processor must either be the current processor and the 2894 * thread pinned to the current processor or a processor that 2895 * is not online. 2896 */ 2897 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 2898 { 2899 unsigned long flags; 2900 struct per_cpu_pageset *pset; 2901 struct per_cpu_pages *pcp; 2902 2903 local_irq_save(flags); 2904 pset = per_cpu_ptr(zone->pageset, cpu); 2905 2906 pcp = &pset->pcp; 2907 if (pcp->count) 2908 free_pcppages_bulk(zone, pcp->count, pcp); 2909 local_irq_restore(flags); 2910 } 2911 2912 /* 2913 * Drain pcplists of all zones on the indicated processor. 2914 * 2915 * The processor must either be the current processor and the 2916 * thread pinned to the current processor or a processor that 2917 * is not online. 2918 */ 2919 static void drain_pages(unsigned int cpu) 2920 { 2921 struct zone *zone; 2922 2923 for_each_populated_zone(zone) { 2924 drain_pages_zone(cpu, zone); 2925 } 2926 } 2927 2928 /* 2929 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 2930 * 2931 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 2932 * the single zone's pages. 2933 */ 2934 void drain_local_pages(struct zone *zone) 2935 { 2936 int cpu = smp_processor_id(); 2937 2938 if (zone) 2939 drain_pages_zone(cpu, zone); 2940 else 2941 drain_pages(cpu); 2942 } 2943 2944 static void drain_local_pages_wq(struct work_struct *work) 2945 { 2946 struct pcpu_drain *drain; 2947 2948 drain = container_of(work, struct pcpu_drain, work); 2949 2950 /* 2951 * drain_all_pages doesn't use proper cpu hotplug protection so 2952 * we can race with cpu offline when the WQ can move this from 2953 * a cpu pinned worker to an unbound one. We can operate on a different 2954 * cpu which is allright but we also have to make sure to not move to 2955 * a different one. 2956 */ 2957 preempt_disable(); 2958 drain_local_pages(drain->zone); 2959 preempt_enable(); 2960 } 2961 2962 /* 2963 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 2964 * 2965 * When zone parameter is non-NULL, spill just the single zone's pages. 2966 * 2967 * Note that this can be extremely slow as the draining happens in a workqueue. 2968 */ 2969 void drain_all_pages(struct zone *zone) 2970 { 2971 int cpu; 2972 2973 /* 2974 * Allocate in the BSS so we wont require allocation in 2975 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 2976 */ 2977 static cpumask_t cpus_with_pcps; 2978 2979 /* 2980 * Make sure nobody triggers this path before mm_percpu_wq is fully 2981 * initialized. 2982 */ 2983 if (WARN_ON_ONCE(!mm_percpu_wq)) 2984 return; 2985 2986 /* 2987 * Do not drain if one is already in progress unless it's specific to 2988 * a zone. Such callers are primarily CMA and memory hotplug and need 2989 * the drain to be complete when the call returns. 2990 */ 2991 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) { 2992 if (!zone) 2993 return; 2994 mutex_lock(&pcpu_drain_mutex); 2995 } 2996 2997 /* 2998 * We don't care about racing with CPU hotplug event 2999 * as offline notification will cause the notified 3000 * cpu to drain that CPU pcps and on_each_cpu_mask 3001 * disables preemption as part of its processing 3002 */ 3003 for_each_online_cpu(cpu) { 3004 struct per_cpu_pageset *pcp; 3005 struct zone *z; 3006 bool has_pcps = false; 3007 3008 if (zone) { 3009 pcp = per_cpu_ptr(zone->pageset, cpu); 3010 if (pcp->pcp.count) 3011 has_pcps = true; 3012 } else { 3013 for_each_populated_zone(z) { 3014 pcp = per_cpu_ptr(z->pageset, cpu); 3015 if (pcp->pcp.count) { 3016 has_pcps = true; 3017 break; 3018 } 3019 } 3020 } 3021 3022 if (has_pcps) 3023 cpumask_set_cpu(cpu, &cpus_with_pcps); 3024 else 3025 cpumask_clear_cpu(cpu, &cpus_with_pcps); 3026 } 3027 3028 for_each_cpu(cpu, &cpus_with_pcps) { 3029 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu); 3030 3031 drain->zone = zone; 3032 INIT_WORK(&drain->work, drain_local_pages_wq); 3033 queue_work_on(cpu, mm_percpu_wq, &drain->work); 3034 } 3035 for_each_cpu(cpu, &cpus_with_pcps) 3036 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work); 3037 3038 mutex_unlock(&pcpu_drain_mutex); 3039 } 3040 3041 #ifdef CONFIG_HIBERNATION 3042 3043 /* 3044 * Touch the watchdog for every WD_PAGE_COUNT pages. 3045 */ 3046 #define WD_PAGE_COUNT (128*1024) 3047 3048 void mark_free_pages(struct zone *zone) 3049 { 3050 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT; 3051 unsigned long flags; 3052 unsigned int order, t; 3053 struct page *page; 3054 3055 if (zone_is_empty(zone)) 3056 return; 3057 3058 spin_lock_irqsave(&zone->lock, flags); 3059 3060 max_zone_pfn = zone_end_pfn(zone); 3061 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 3062 if (pfn_valid(pfn)) { 3063 page = pfn_to_page(pfn); 3064 3065 if (!--page_count) { 3066 touch_nmi_watchdog(); 3067 page_count = WD_PAGE_COUNT; 3068 } 3069 3070 if (page_zone(page) != zone) 3071 continue; 3072 3073 if (!swsusp_page_is_forbidden(page)) 3074 swsusp_unset_page_free(page); 3075 } 3076 3077 for_each_migratetype_order(order, t) { 3078 list_for_each_entry(page, 3079 &zone->free_area[order].free_list[t], lru) { 3080 unsigned long i; 3081 3082 pfn = page_to_pfn(page); 3083 for (i = 0; i < (1UL << order); i++) { 3084 if (!--page_count) { 3085 touch_nmi_watchdog(); 3086 page_count = WD_PAGE_COUNT; 3087 } 3088 swsusp_set_page_free(pfn_to_page(pfn + i)); 3089 } 3090 } 3091 } 3092 spin_unlock_irqrestore(&zone->lock, flags); 3093 } 3094 #endif /* CONFIG_PM */ 3095 3096 static bool free_unref_page_prepare(struct page *page, unsigned long pfn) 3097 { 3098 int migratetype; 3099 3100 if (!free_pcp_prepare(page)) 3101 return false; 3102 3103 migratetype = get_pfnblock_migratetype(page, pfn); 3104 set_pcppage_migratetype(page, migratetype); 3105 return true; 3106 } 3107 3108 static void free_unref_page_commit(struct page *page, unsigned long pfn) 3109 { 3110 struct zone *zone = page_zone(page); 3111 struct per_cpu_pages *pcp; 3112 int migratetype; 3113 3114 migratetype = get_pcppage_migratetype(page); 3115 __count_vm_event(PGFREE); 3116 3117 /* 3118 * We only track unmovable, reclaimable and movable on pcp lists. 3119 * Free ISOLATE pages back to the allocator because they are being 3120 * offlined but treat HIGHATOMIC as movable pages so we can get those 3121 * areas back if necessary. Otherwise, we may have to free 3122 * excessively into the page allocator 3123 */ 3124 if (migratetype >= MIGRATE_PCPTYPES) { 3125 if (unlikely(is_migrate_isolate(migratetype))) { 3126 free_one_page(zone, page, pfn, 0, migratetype); 3127 return; 3128 } 3129 migratetype = MIGRATE_MOVABLE; 3130 } 3131 3132 pcp = &this_cpu_ptr(zone->pageset)->pcp; 3133 list_add(&page->lru, &pcp->lists[migratetype]); 3134 pcp->count++; 3135 if (pcp->count >= pcp->high) { 3136 unsigned long batch = READ_ONCE(pcp->batch); 3137 free_pcppages_bulk(zone, batch, pcp); 3138 } 3139 } 3140 3141 /* 3142 * Free a 0-order page 3143 */ 3144 void free_unref_page(struct page *page) 3145 { 3146 unsigned long flags; 3147 unsigned long pfn = page_to_pfn(page); 3148 3149 if (!free_unref_page_prepare(page, pfn)) 3150 return; 3151 3152 local_irq_save(flags); 3153 free_unref_page_commit(page, pfn); 3154 local_irq_restore(flags); 3155 } 3156 3157 /* 3158 * Free a list of 0-order pages 3159 */ 3160 void free_unref_page_list(struct list_head *list) 3161 { 3162 struct page *page, *next; 3163 unsigned long flags, pfn; 3164 int batch_count = 0; 3165 3166 /* Prepare pages for freeing */ 3167 list_for_each_entry_safe(page, next, list, lru) { 3168 pfn = page_to_pfn(page); 3169 if (!free_unref_page_prepare(page, pfn)) 3170 list_del(&page->lru); 3171 set_page_private(page, pfn); 3172 } 3173 3174 local_irq_save(flags); 3175 list_for_each_entry_safe(page, next, list, lru) { 3176 unsigned long pfn = page_private(page); 3177 3178 set_page_private(page, 0); 3179 trace_mm_page_free_batched(page); 3180 free_unref_page_commit(page, pfn); 3181 3182 /* 3183 * Guard against excessive IRQ disabled times when we get 3184 * a large list of pages to free. 3185 */ 3186 if (++batch_count == SWAP_CLUSTER_MAX) { 3187 local_irq_restore(flags); 3188 batch_count = 0; 3189 local_irq_save(flags); 3190 } 3191 } 3192 local_irq_restore(flags); 3193 } 3194 3195 /* 3196 * split_page takes a non-compound higher-order page, and splits it into 3197 * n (1<<order) sub-pages: page[0..n] 3198 * Each sub-page must be freed individually. 3199 * 3200 * Note: this is probably too low level an operation for use in drivers. 3201 * Please consult with lkml before using this in your driver. 3202 */ 3203 void split_page(struct page *page, unsigned int order) 3204 { 3205 int i; 3206 3207 VM_BUG_ON_PAGE(PageCompound(page), page); 3208 VM_BUG_ON_PAGE(!page_count(page), page); 3209 3210 for (i = 1; i < (1 << order); i++) 3211 set_page_refcounted(page + i); 3212 split_page_owner(page, order); 3213 } 3214 EXPORT_SYMBOL_GPL(split_page); 3215 3216 int __isolate_free_page(struct page *page, unsigned int order) 3217 { 3218 unsigned long watermark; 3219 struct zone *zone; 3220 int mt; 3221 3222 BUG_ON(!PageBuddy(page)); 3223 3224 zone = page_zone(page); 3225 mt = get_pageblock_migratetype(page); 3226 3227 if (!is_migrate_isolate(mt)) { 3228 /* 3229 * Obey watermarks as if the page was being allocated. We can 3230 * emulate a high-order watermark check with a raised order-0 3231 * watermark, because we already know our high-order page 3232 * exists. 3233 */ 3234 watermark = zone->_watermark[WMARK_MIN] + (1UL << order); 3235 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 3236 return 0; 3237 3238 __mod_zone_freepage_state(zone, -(1UL << order), mt); 3239 } 3240 3241 /* Remove page from free list */ 3242 3243 del_page_from_free_list(page, zone, order); 3244 3245 /* 3246 * Set the pageblock if the isolated page is at least half of a 3247 * pageblock 3248 */ 3249 if (order >= pageblock_order - 1) { 3250 struct page *endpage = page + (1 << order) - 1; 3251 for (; page < endpage; page += pageblock_nr_pages) { 3252 int mt = get_pageblock_migratetype(page); 3253 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt) 3254 && !is_migrate_highatomic(mt)) 3255 set_pageblock_migratetype(page, 3256 MIGRATE_MOVABLE); 3257 } 3258 } 3259 3260 3261 return 1UL << order; 3262 } 3263 3264 /** 3265 * __putback_isolated_page - Return a now-isolated page back where we got it 3266 * @page: Page that was isolated 3267 * @order: Order of the isolated page 3268 * @mt: The page's pageblock's migratetype 3269 * 3270 * This function is meant to return a page pulled from the free lists via 3271 * __isolate_free_page back to the free lists they were pulled from. 3272 */ 3273 void __putback_isolated_page(struct page *page, unsigned int order, int mt) 3274 { 3275 struct zone *zone = page_zone(page); 3276 3277 /* zone lock should be held when this function is called */ 3278 lockdep_assert_held(&zone->lock); 3279 3280 /* Return isolated page to tail of freelist. */ 3281 __free_one_page(page, page_to_pfn(page), zone, order, mt, false); 3282 } 3283 3284 /* 3285 * Update NUMA hit/miss statistics 3286 * 3287 * Must be called with interrupts disabled. 3288 */ 3289 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) 3290 { 3291 #ifdef CONFIG_NUMA 3292 enum numa_stat_item local_stat = NUMA_LOCAL; 3293 3294 /* skip numa counters update if numa stats is disabled */ 3295 if (!static_branch_likely(&vm_numa_stat_key)) 3296 return; 3297 3298 if (zone_to_nid(z) != numa_node_id()) 3299 local_stat = NUMA_OTHER; 3300 3301 if (zone_to_nid(z) == zone_to_nid(preferred_zone)) 3302 __inc_numa_state(z, NUMA_HIT); 3303 else { 3304 __inc_numa_state(z, NUMA_MISS); 3305 __inc_numa_state(preferred_zone, NUMA_FOREIGN); 3306 } 3307 __inc_numa_state(z, local_stat); 3308 #endif 3309 } 3310 3311 /* Remove page from the per-cpu list, caller must protect the list */ 3312 static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype, 3313 unsigned int alloc_flags, 3314 struct per_cpu_pages *pcp, 3315 struct list_head *list) 3316 { 3317 struct page *page; 3318 3319 do { 3320 if (list_empty(list)) { 3321 pcp->count += rmqueue_bulk(zone, 0, 3322 pcp->batch, list, 3323 migratetype, alloc_flags); 3324 if (unlikely(list_empty(list))) 3325 return NULL; 3326 } 3327 3328 page = list_first_entry(list, struct page, lru); 3329 list_del(&page->lru); 3330 pcp->count--; 3331 } while (check_new_pcp(page)); 3332 3333 return page; 3334 } 3335 3336 /* Lock and remove page from the per-cpu list */ 3337 static struct page *rmqueue_pcplist(struct zone *preferred_zone, 3338 struct zone *zone, gfp_t gfp_flags, 3339 int migratetype, unsigned int alloc_flags) 3340 { 3341 struct per_cpu_pages *pcp; 3342 struct list_head *list; 3343 struct page *page; 3344 unsigned long flags; 3345 3346 local_irq_save(flags); 3347 pcp = &this_cpu_ptr(zone->pageset)->pcp; 3348 list = &pcp->lists[migratetype]; 3349 page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list); 3350 if (page) { 3351 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); 3352 zone_statistics(preferred_zone, zone); 3353 } 3354 local_irq_restore(flags); 3355 return page; 3356 } 3357 3358 /* 3359 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 3360 */ 3361 static inline 3362 struct page *rmqueue(struct zone *preferred_zone, 3363 struct zone *zone, unsigned int order, 3364 gfp_t gfp_flags, unsigned int alloc_flags, 3365 int migratetype) 3366 { 3367 unsigned long flags; 3368 struct page *page; 3369 3370 if (likely(order == 0)) { 3371 /* 3372 * MIGRATE_MOVABLE pcplist could have the pages on CMA area and 3373 * we need to skip it when CMA area isn't allowed. 3374 */ 3375 if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA || 3376 migratetype != MIGRATE_MOVABLE) { 3377 page = rmqueue_pcplist(preferred_zone, zone, gfp_flags, 3378 migratetype, alloc_flags); 3379 goto out; 3380 } 3381 } 3382 3383 /* 3384 * We most definitely don't want callers attempting to 3385 * allocate greater than order-1 page units with __GFP_NOFAIL. 3386 */ 3387 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 3388 spin_lock_irqsave(&zone->lock, flags); 3389 3390 do { 3391 page = NULL; 3392 /* 3393 * order-0 request can reach here when the pcplist is skipped 3394 * due to non-CMA allocation context. HIGHATOMIC area is 3395 * reserved for high-order atomic allocation, so order-0 3396 * request should skip it. 3397 */ 3398 if (order > 0 && alloc_flags & ALLOC_HARDER) { 3399 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3400 if (page) 3401 trace_mm_page_alloc_zone_locked(page, order, migratetype); 3402 } 3403 if (!page) 3404 page = __rmqueue(zone, order, migratetype, alloc_flags); 3405 } while (page && check_new_pages(page, order)); 3406 spin_unlock(&zone->lock); 3407 if (!page) 3408 goto failed; 3409 __mod_zone_freepage_state(zone, -(1 << order), 3410 get_pcppage_migratetype(page)); 3411 3412 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3413 zone_statistics(preferred_zone, zone); 3414 local_irq_restore(flags); 3415 3416 out: 3417 /* Separate test+clear to avoid unnecessary atomics */ 3418 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) { 3419 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 3420 wakeup_kswapd(zone, 0, 0, zone_idx(zone)); 3421 } 3422 3423 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); 3424 return page; 3425 3426 failed: 3427 local_irq_restore(flags); 3428 return NULL; 3429 } 3430 3431 #ifdef CONFIG_FAIL_PAGE_ALLOC 3432 3433 static struct { 3434 struct fault_attr attr; 3435 3436 bool ignore_gfp_highmem; 3437 bool ignore_gfp_reclaim; 3438 u32 min_order; 3439 } fail_page_alloc = { 3440 .attr = FAULT_ATTR_INITIALIZER, 3441 .ignore_gfp_reclaim = true, 3442 .ignore_gfp_highmem = true, 3443 .min_order = 1, 3444 }; 3445 3446 static int __init setup_fail_page_alloc(char *str) 3447 { 3448 return setup_fault_attr(&fail_page_alloc.attr, str); 3449 } 3450 __setup("fail_page_alloc=", setup_fail_page_alloc); 3451 3452 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3453 { 3454 if (order < fail_page_alloc.min_order) 3455 return false; 3456 if (gfp_mask & __GFP_NOFAIL) 3457 return false; 3458 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 3459 return false; 3460 if (fail_page_alloc.ignore_gfp_reclaim && 3461 (gfp_mask & __GFP_DIRECT_RECLAIM)) 3462 return false; 3463 3464 return should_fail(&fail_page_alloc.attr, 1 << order); 3465 } 3466 3467 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 3468 3469 static int __init fail_page_alloc_debugfs(void) 3470 { 3471 umode_t mode = S_IFREG | 0600; 3472 struct dentry *dir; 3473 3474 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 3475 &fail_page_alloc.attr); 3476 3477 debugfs_create_bool("ignore-gfp-wait", mode, dir, 3478 &fail_page_alloc.ignore_gfp_reclaim); 3479 debugfs_create_bool("ignore-gfp-highmem", mode, dir, 3480 &fail_page_alloc.ignore_gfp_highmem); 3481 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order); 3482 3483 return 0; 3484 } 3485 3486 late_initcall(fail_page_alloc_debugfs); 3487 3488 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 3489 3490 #else /* CONFIG_FAIL_PAGE_ALLOC */ 3491 3492 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3493 { 3494 return false; 3495 } 3496 3497 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 3498 3499 static noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3500 { 3501 return __should_fail_alloc_page(gfp_mask, order); 3502 } 3503 ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE); 3504 3505 static inline long __zone_watermark_unusable_free(struct zone *z, 3506 unsigned int order, unsigned int alloc_flags) 3507 { 3508 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3509 long unusable_free = (1 << order) - 1; 3510 3511 /* 3512 * If the caller does not have rights to ALLOC_HARDER then subtract 3513 * the high-atomic reserves. This will over-estimate the size of the 3514 * atomic reserve but it avoids a search. 3515 */ 3516 if (likely(!alloc_harder)) 3517 unusable_free += z->nr_reserved_highatomic; 3518 3519 #ifdef CONFIG_CMA 3520 /* If allocation can't use CMA areas don't use free CMA pages */ 3521 if (!(alloc_flags & ALLOC_CMA)) 3522 unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES); 3523 #endif 3524 3525 return unusable_free; 3526 } 3527 3528 /* 3529 * Return true if free base pages are above 'mark'. For high-order checks it 3530 * will return true of the order-0 watermark is reached and there is at least 3531 * one free page of a suitable size. Checking now avoids taking the zone lock 3532 * to check in the allocation paths if no pages are free. 3533 */ 3534 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3535 int highest_zoneidx, unsigned int alloc_flags, 3536 long free_pages) 3537 { 3538 long min = mark; 3539 int o; 3540 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3541 3542 /* free_pages may go negative - that's OK */ 3543 free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags); 3544 3545 if (alloc_flags & ALLOC_HIGH) 3546 min -= min / 2; 3547 3548 if (unlikely(alloc_harder)) { 3549 /* 3550 * OOM victims can try even harder than normal ALLOC_HARDER 3551 * users on the grounds that it's definitely going to be in 3552 * the exit path shortly and free memory. Any allocation it 3553 * makes during the free path will be small and short-lived. 3554 */ 3555 if (alloc_flags & ALLOC_OOM) 3556 min -= min / 2; 3557 else 3558 min -= min / 4; 3559 } 3560 3561 /* 3562 * Check watermarks for an order-0 allocation request. If these 3563 * are not met, then a high-order request also cannot go ahead 3564 * even if a suitable page happened to be free. 3565 */ 3566 if (free_pages <= min + z->lowmem_reserve[highest_zoneidx]) 3567 return false; 3568 3569 /* If this is an order-0 request then the watermark is fine */ 3570 if (!order) 3571 return true; 3572 3573 /* For a high-order request, check at least one suitable page is free */ 3574 for (o = order; o < MAX_ORDER; o++) { 3575 struct free_area *area = &z->free_area[o]; 3576 int mt; 3577 3578 if (!area->nr_free) 3579 continue; 3580 3581 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 3582 if (!free_area_empty(area, mt)) 3583 return true; 3584 } 3585 3586 #ifdef CONFIG_CMA 3587 if ((alloc_flags & ALLOC_CMA) && 3588 !free_area_empty(area, MIGRATE_CMA)) { 3589 return true; 3590 } 3591 #endif 3592 if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC)) 3593 return true; 3594 } 3595 return false; 3596 } 3597 3598 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3599 int highest_zoneidx, unsigned int alloc_flags) 3600 { 3601 return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 3602 zone_page_state(z, NR_FREE_PAGES)); 3603 } 3604 3605 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 3606 unsigned long mark, int highest_zoneidx, 3607 unsigned int alloc_flags, gfp_t gfp_mask) 3608 { 3609 long free_pages; 3610 3611 free_pages = zone_page_state(z, NR_FREE_PAGES); 3612 3613 /* 3614 * Fast check for order-0 only. If this fails then the reserves 3615 * need to be calculated. 3616 */ 3617 if (!order) { 3618 long fast_free; 3619 3620 fast_free = free_pages; 3621 fast_free -= __zone_watermark_unusable_free(z, 0, alloc_flags); 3622 if (fast_free > mark + z->lowmem_reserve[highest_zoneidx]) 3623 return true; 3624 } 3625 3626 if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 3627 free_pages)) 3628 return true; 3629 /* 3630 * Ignore watermark boosting for GFP_ATOMIC order-0 allocations 3631 * when checking the min watermark. The min watermark is the 3632 * point where boosting is ignored so that kswapd is woken up 3633 * when below the low watermark. 3634 */ 3635 if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost 3636 && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) { 3637 mark = z->_watermark[WMARK_MIN]; 3638 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 3639 alloc_flags, free_pages); 3640 } 3641 3642 return false; 3643 } 3644 3645 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 3646 unsigned long mark, int highest_zoneidx) 3647 { 3648 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3649 3650 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 3651 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 3652 3653 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0, 3654 free_pages); 3655 } 3656 3657 #ifdef CONFIG_NUMA 3658 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3659 { 3660 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= 3661 node_reclaim_distance; 3662 } 3663 #else /* CONFIG_NUMA */ 3664 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3665 { 3666 return true; 3667 } 3668 #endif /* CONFIG_NUMA */ 3669 3670 /* 3671 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid 3672 * fragmentation is subtle. If the preferred zone was HIGHMEM then 3673 * premature use of a lower zone may cause lowmem pressure problems that 3674 * are worse than fragmentation. If the next zone is ZONE_DMA then it is 3675 * probably too small. It only makes sense to spread allocations to avoid 3676 * fragmentation between the Normal and DMA32 zones. 3677 */ 3678 static inline unsigned int 3679 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) 3680 { 3681 unsigned int alloc_flags; 3682 3683 /* 3684 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 3685 * to save a branch. 3686 */ 3687 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); 3688 3689 #ifdef CONFIG_ZONE_DMA32 3690 if (!zone) 3691 return alloc_flags; 3692 3693 if (zone_idx(zone) != ZONE_NORMAL) 3694 return alloc_flags; 3695 3696 /* 3697 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and 3698 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume 3699 * on UMA that if Normal is populated then so is DMA32. 3700 */ 3701 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1); 3702 if (nr_online_nodes > 1 && !populated_zone(--zone)) 3703 return alloc_flags; 3704 3705 alloc_flags |= ALLOC_NOFRAGMENT; 3706 #endif /* CONFIG_ZONE_DMA32 */ 3707 return alloc_flags; 3708 } 3709 3710 static inline unsigned int current_alloc_flags(gfp_t gfp_mask, 3711 unsigned int alloc_flags) 3712 { 3713 #ifdef CONFIG_CMA 3714 unsigned int pflags = current->flags; 3715 3716 if (!(pflags & PF_MEMALLOC_NOCMA) && 3717 gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) 3718 alloc_flags |= ALLOC_CMA; 3719 3720 #endif 3721 return alloc_flags; 3722 } 3723 3724 /* 3725 * get_page_from_freelist goes through the zonelist trying to allocate 3726 * a page. 3727 */ 3728 static struct page * 3729 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 3730 const struct alloc_context *ac) 3731 { 3732 struct zoneref *z; 3733 struct zone *zone; 3734 struct pglist_data *last_pgdat_dirty_limit = NULL; 3735 bool no_fallback; 3736 3737 retry: 3738 /* 3739 * Scan zonelist, looking for a zone with enough free. 3740 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 3741 */ 3742 no_fallback = alloc_flags & ALLOC_NOFRAGMENT; 3743 z = ac->preferred_zoneref; 3744 for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx, 3745 ac->nodemask) { 3746 struct page *page; 3747 unsigned long mark; 3748 3749 if (cpusets_enabled() && 3750 (alloc_flags & ALLOC_CPUSET) && 3751 !__cpuset_zone_allowed(zone, gfp_mask)) 3752 continue; 3753 /* 3754 * When allocating a page cache page for writing, we 3755 * want to get it from a node that is within its dirty 3756 * limit, such that no single node holds more than its 3757 * proportional share of globally allowed dirty pages. 3758 * The dirty limits take into account the node's 3759 * lowmem reserves and high watermark so that kswapd 3760 * should be able to balance it without having to 3761 * write pages from its LRU list. 3762 * 3763 * XXX: For now, allow allocations to potentially 3764 * exceed the per-node dirty limit in the slowpath 3765 * (spread_dirty_pages unset) before going into reclaim, 3766 * which is important when on a NUMA setup the allowed 3767 * nodes are together not big enough to reach the 3768 * global limit. The proper fix for these situations 3769 * will require awareness of nodes in the 3770 * dirty-throttling and the flusher threads. 3771 */ 3772 if (ac->spread_dirty_pages) { 3773 if (last_pgdat_dirty_limit == zone->zone_pgdat) 3774 continue; 3775 3776 if (!node_dirty_ok(zone->zone_pgdat)) { 3777 last_pgdat_dirty_limit = zone->zone_pgdat; 3778 continue; 3779 } 3780 } 3781 3782 if (no_fallback && nr_online_nodes > 1 && 3783 zone != ac->preferred_zoneref->zone) { 3784 int local_nid; 3785 3786 /* 3787 * If moving to a remote node, retry but allow 3788 * fragmenting fallbacks. Locality is more important 3789 * than fragmentation avoidance. 3790 */ 3791 local_nid = zone_to_nid(ac->preferred_zoneref->zone); 3792 if (zone_to_nid(zone) != local_nid) { 3793 alloc_flags &= ~ALLOC_NOFRAGMENT; 3794 goto retry; 3795 } 3796 } 3797 3798 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); 3799 if (!zone_watermark_fast(zone, order, mark, 3800 ac->highest_zoneidx, alloc_flags, 3801 gfp_mask)) { 3802 int ret; 3803 3804 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3805 /* 3806 * Watermark failed for this zone, but see if we can 3807 * grow this zone if it contains deferred pages. 3808 */ 3809 if (static_branch_unlikely(&deferred_pages)) { 3810 if (_deferred_grow_zone(zone, order)) 3811 goto try_this_zone; 3812 } 3813 #endif 3814 /* Checked here to keep the fast path fast */ 3815 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 3816 if (alloc_flags & ALLOC_NO_WATERMARKS) 3817 goto try_this_zone; 3818 3819 if (node_reclaim_mode == 0 || 3820 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 3821 continue; 3822 3823 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 3824 switch (ret) { 3825 case NODE_RECLAIM_NOSCAN: 3826 /* did not scan */ 3827 continue; 3828 case NODE_RECLAIM_FULL: 3829 /* scanned but unreclaimable */ 3830 continue; 3831 default: 3832 /* did we reclaim enough */ 3833 if (zone_watermark_ok(zone, order, mark, 3834 ac->highest_zoneidx, alloc_flags)) 3835 goto try_this_zone; 3836 3837 continue; 3838 } 3839 } 3840 3841 try_this_zone: 3842 page = rmqueue(ac->preferred_zoneref->zone, zone, order, 3843 gfp_mask, alloc_flags, ac->migratetype); 3844 if (page) { 3845 prep_new_page(page, order, gfp_mask, alloc_flags); 3846 3847 /* 3848 * If this is a high-order atomic allocation then check 3849 * if the pageblock should be reserved for the future 3850 */ 3851 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 3852 reserve_highatomic_pageblock(page, zone, order); 3853 3854 return page; 3855 } else { 3856 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3857 /* Try again if zone has deferred pages */ 3858 if (static_branch_unlikely(&deferred_pages)) { 3859 if (_deferred_grow_zone(zone, order)) 3860 goto try_this_zone; 3861 } 3862 #endif 3863 } 3864 } 3865 3866 /* 3867 * It's possible on a UMA machine to get through all zones that are 3868 * fragmented. If avoiding fragmentation, reset and try again. 3869 */ 3870 if (no_fallback) { 3871 alloc_flags &= ~ALLOC_NOFRAGMENT; 3872 goto retry; 3873 } 3874 3875 return NULL; 3876 } 3877 3878 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) 3879 { 3880 unsigned int filter = SHOW_MEM_FILTER_NODES; 3881 3882 /* 3883 * This documents exceptions given to allocations in certain 3884 * contexts that are allowed to allocate outside current's set 3885 * of allowed nodes. 3886 */ 3887 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3888 if (tsk_is_oom_victim(current) || 3889 (current->flags & (PF_MEMALLOC | PF_EXITING))) 3890 filter &= ~SHOW_MEM_FILTER_NODES; 3891 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 3892 filter &= ~SHOW_MEM_FILTER_NODES; 3893 3894 show_mem(filter, nodemask); 3895 } 3896 3897 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) 3898 { 3899 struct va_format vaf; 3900 va_list args; 3901 static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1); 3902 3903 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs)) 3904 return; 3905 3906 va_start(args, fmt); 3907 vaf.fmt = fmt; 3908 vaf.va = &args; 3909 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl", 3910 current->comm, &vaf, gfp_mask, &gfp_mask, 3911 nodemask_pr_args(nodemask)); 3912 va_end(args); 3913 3914 cpuset_print_current_mems_allowed(); 3915 pr_cont("\n"); 3916 dump_stack(); 3917 warn_alloc_show_mem(gfp_mask, nodemask); 3918 } 3919 3920 static inline struct page * 3921 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, 3922 unsigned int alloc_flags, 3923 const struct alloc_context *ac) 3924 { 3925 struct page *page; 3926 3927 page = get_page_from_freelist(gfp_mask, order, 3928 alloc_flags|ALLOC_CPUSET, ac); 3929 /* 3930 * fallback to ignore cpuset restriction if our nodes 3931 * are depleted 3932 */ 3933 if (!page) 3934 page = get_page_from_freelist(gfp_mask, order, 3935 alloc_flags, ac); 3936 3937 return page; 3938 } 3939 3940 static inline struct page * 3941 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 3942 const struct alloc_context *ac, unsigned long *did_some_progress) 3943 { 3944 struct oom_control oc = { 3945 .zonelist = ac->zonelist, 3946 .nodemask = ac->nodemask, 3947 .memcg = NULL, 3948 .gfp_mask = gfp_mask, 3949 .order = order, 3950 }; 3951 struct page *page; 3952 3953 *did_some_progress = 0; 3954 3955 /* 3956 * Acquire the oom lock. If that fails, somebody else is 3957 * making progress for us. 3958 */ 3959 if (!mutex_trylock(&oom_lock)) { 3960 *did_some_progress = 1; 3961 schedule_timeout_uninterruptible(1); 3962 return NULL; 3963 } 3964 3965 /* 3966 * Go through the zonelist yet one more time, keep very high watermark 3967 * here, this is only to catch a parallel oom killing, we must fail if 3968 * we're still under heavy pressure. But make sure that this reclaim 3969 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY 3970 * allocation which will never fail due to oom_lock already held. 3971 */ 3972 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & 3973 ~__GFP_DIRECT_RECLAIM, order, 3974 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 3975 if (page) 3976 goto out; 3977 3978 /* Coredumps can quickly deplete all memory reserves */ 3979 if (current->flags & PF_DUMPCORE) 3980 goto out; 3981 /* The OOM killer will not help higher order allocs */ 3982 if (order > PAGE_ALLOC_COSTLY_ORDER) 3983 goto out; 3984 /* 3985 * We have already exhausted all our reclaim opportunities without any 3986 * success so it is time to admit defeat. We will skip the OOM killer 3987 * because it is very likely that the caller has a more reasonable 3988 * fallback than shooting a random task. 3989 * 3990 * The OOM killer may not free memory on a specific node. 3991 */ 3992 if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE)) 3993 goto out; 3994 /* The OOM killer does not needlessly kill tasks for lowmem */ 3995 if (ac->highest_zoneidx < ZONE_NORMAL) 3996 goto out; 3997 if (pm_suspended_storage()) 3998 goto out; 3999 /* 4000 * XXX: GFP_NOFS allocations should rather fail than rely on 4001 * other request to make a forward progress. 4002 * We are in an unfortunate situation where out_of_memory cannot 4003 * do much for this context but let's try it to at least get 4004 * access to memory reserved if the current task is killed (see 4005 * out_of_memory). Once filesystems are ready to handle allocation 4006 * failures more gracefully we should just bail out here. 4007 */ 4008 4009 /* Exhausted what can be done so it's blame time */ 4010 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 4011 *did_some_progress = 1; 4012 4013 /* 4014 * Help non-failing allocations by giving them access to memory 4015 * reserves 4016 */ 4017 if (gfp_mask & __GFP_NOFAIL) 4018 page = __alloc_pages_cpuset_fallback(gfp_mask, order, 4019 ALLOC_NO_WATERMARKS, ac); 4020 } 4021 out: 4022 mutex_unlock(&oom_lock); 4023 return page; 4024 } 4025 4026 /* 4027 * Maximum number of compaction retries wit a progress before OOM 4028 * killer is consider as the only way to move forward. 4029 */ 4030 #define MAX_COMPACT_RETRIES 16 4031 4032 #ifdef CONFIG_COMPACTION 4033 /* Try memory compaction for high-order allocations before reclaim */ 4034 static struct page * 4035 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4036 unsigned int alloc_flags, const struct alloc_context *ac, 4037 enum compact_priority prio, enum compact_result *compact_result) 4038 { 4039 struct page *page = NULL; 4040 unsigned long pflags; 4041 unsigned int noreclaim_flag; 4042 4043 if (!order) 4044 return NULL; 4045 4046 psi_memstall_enter(&pflags); 4047 noreclaim_flag = memalloc_noreclaim_save(); 4048 4049 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 4050 prio, &page); 4051 4052 memalloc_noreclaim_restore(noreclaim_flag); 4053 psi_memstall_leave(&pflags); 4054 4055 /* 4056 * At least in one zone compaction wasn't deferred or skipped, so let's 4057 * count a compaction stall 4058 */ 4059 count_vm_event(COMPACTSTALL); 4060 4061 /* Prep a captured page if available */ 4062 if (page) 4063 prep_new_page(page, order, gfp_mask, alloc_flags); 4064 4065 /* Try get a page from the freelist if available */ 4066 if (!page) 4067 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4068 4069 if (page) { 4070 struct zone *zone = page_zone(page); 4071 4072 zone->compact_blockskip_flush = false; 4073 compaction_defer_reset(zone, order, true); 4074 count_vm_event(COMPACTSUCCESS); 4075 return page; 4076 } 4077 4078 /* 4079 * It's bad if compaction run occurs and fails. The most likely reason 4080 * is that pages exist, but not enough to satisfy watermarks. 4081 */ 4082 count_vm_event(COMPACTFAIL); 4083 4084 cond_resched(); 4085 4086 return NULL; 4087 } 4088 4089 static inline bool 4090 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 4091 enum compact_result compact_result, 4092 enum compact_priority *compact_priority, 4093 int *compaction_retries) 4094 { 4095 int max_retries = MAX_COMPACT_RETRIES; 4096 int min_priority; 4097 bool ret = false; 4098 int retries = *compaction_retries; 4099 enum compact_priority priority = *compact_priority; 4100 4101 if (!order) 4102 return false; 4103 4104 if (compaction_made_progress(compact_result)) 4105 (*compaction_retries)++; 4106 4107 /* 4108 * compaction considers all the zone as desperately out of memory 4109 * so it doesn't really make much sense to retry except when the 4110 * failure could be caused by insufficient priority 4111 */ 4112 if (compaction_failed(compact_result)) 4113 goto check_priority; 4114 4115 /* 4116 * compaction was skipped because there are not enough order-0 pages 4117 * to work with, so we retry only if it looks like reclaim can help. 4118 */ 4119 if (compaction_needs_reclaim(compact_result)) { 4120 ret = compaction_zonelist_suitable(ac, order, alloc_flags); 4121 goto out; 4122 } 4123 4124 /* 4125 * make sure the compaction wasn't deferred or didn't bail out early 4126 * due to locks contention before we declare that we should give up. 4127 * But the next retry should use a higher priority if allowed, so 4128 * we don't just keep bailing out endlessly. 4129 */ 4130 if (compaction_withdrawn(compact_result)) { 4131 goto check_priority; 4132 } 4133 4134 /* 4135 * !costly requests are much more important than __GFP_RETRY_MAYFAIL 4136 * costly ones because they are de facto nofail and invoke OOM 4137 * killer to move on while costly can fail and users are ready 4138 * to cope with that. 1/4 retries is rather arbitrary but we 4139 * would need much more detailed feedback from compaction to 4140 * make a better decision. 4141 */ 4142 if (order > PAGE_ALLOC_COSTLY_ORDER) 4143 max_retries /= 4; 4144 if (*compaction_retries <= max_retries) { 4145 ret = true; 4146 goto out; 4147 } 4148 4149 /* 4150 * Make sure there are attempts at the highest priority if we exhausted 4151 * all retries or failed at the lower priorities. 4152 */ 4153 check_priority: 4154 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 4155 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 4156 4157 if (*compact_priority > min_priority) { 4158 (*compact_priority)--; 4159 *compaction_retries = 0; 4160 ret = true; 4161 } 4162 out: 4163 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret); 4164 return ret; 4165 } 4166 #else 4167 static inline struct page * 4168 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4169 unsigned int alloc_flags, const struct alloc_context *ac, 4170 enum compact_priority prio, enum compact_result *compact_result) 4171 { 4172 *compact_result = COMPACT_SKIPPED; 4173 return NULL; 4174 } 4175 4176 static inline bool 4177 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 4178 enum compact_result compact_result, 4179 enum compact_priority *compact_priority, 4180 int *compaction_retries) 4181 { 4182 struct zone *zone; 4183 struct zoneref *z; 4184 4185 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 4186 return false; 4187 4188 /* 4189 * There are setups with compaction disabled which would prefer to loop 4190 * inside the allocator rather than hit the oom killer prematurely. 4191 * Let's give them a good hope and keep retrying while the order-0 4192 * watermarks are OK. 4193 */ 4194 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4195 ac->highest_zoneidx, ac->nodemask) { 4196 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 4197 ac->highest_zoneidx, alloc_flags)) 4198 return true; 4199 } 4200 return false; 4201 } 4202 #endif /* CONFIG_COMPACTION */ 4203 4204 #ifdef CONFIG_LOCKDEP 4205 static struct lockdep_map __fs_reclaim_map = 4206 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map); 4207 4208 static bool __need_fs_reclaim(gfp_t gfp_mask) 4209 { 4210 gfp_mask = current_gfp_context(gfp_mask); 4211 4212 /* no reclaim without waiting on it */ 4213 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) 4214 return false; 4215 4216 /* this guy won't enter reclaim */ 4217 if (current->flags & PF_MEMALLOC) 4218 return false; 4219 4220 /* We're only interested __GFP_FS allocations for now */ 4221 if (!(gfp_mask & __GFP_FS)) 4222 return false; 4223 4224 if (gfp_mask & __GFP_NOLOCKDEP) 4225 return false; 4226 4227 return true; 4228 } 4229 4230 void __fs_reclaim_acquire(void) 4231 { 4232 lock_map_acquire(&__fs_reclaim_map); 4233 } 4234 4235 void __fs_reclaim_release(void) 4236 { 4237 lock_map_release(&__fs_reclaim_map); 4238 } 4239 4240 void fs_reclaim_acquire(gfp_t gfp_mask) 4241 { 4242 if (__need_fs_reclaim(gfp_mask)) 4243 __fs_reclaim_acquire(); 4244 } 4245 EXPORT_SYMBOL_GPL(fs_reclaim_acquire); 4246 4247 void fs_reclaim_release(gfp_t gfp_mask) 4248 { 4249 if (__need_fs_reclaim(gfp_mask)) 4250 __fs_reclaim_release(); 4251 } 4252 EXPORT_SYMBOL_GPL(fs_reclaim_release); 4253 #endif 4254 4255 /* Perform direct synchronous page reclaim */ 4256 static unsigned long 4257 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 4258 const struct alloc_context *ac) 4259 { 4260 unsigned int noreclaim_flag; 4261 unsigned long pflags, progress; 4262 4263 cond_resched(); 4264 4265 /* We now go into synchronous reclaim */ 4266 cpuset_memory_pressure_bump(); 4267 psi_memstall_enter(&pflags); 4268 fs_reclaim_acquire(gfp_mask); 4269 noreclaim_flag = memalloc_noreclaim_save(); 4270 4271 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 4272 ac->nodemask); 4273 4274 memalloc_noreclaim_restore(noreclaim_flag); 4275 fs_reclaim_release(gfp_mask); 4276 psi_memstall_leave(&pflags); 4277 4278 cond_resched(); 4279 4280 return progress; 4281 } 4282 4283 /* The really slow allocator path where we enter direct reclaim */ 4284 static inline struct page * 4285 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 4286 unsigned int alloc_flags, const struct alloc_context *ac, 4287 unsigned long *did_some_progress) 4288 { 4289 struct page *page = NULL; 4290 bool drained = false; 4291 4292 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 4293 if (unlikely(!(*did_some_progress))) 4294 return NULL; 4295 4296 retry: 4297 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4298 4299 /* 4300 * If an allocation failed after direct reclaim, it could be because 4301 * pages are pinned on the per-cpu lists or in high alloc reserves. 4302 * Shrink them and try again 4303 */ 4304 if (!page && !drained) { 4305 unreserve_highatomic_pageblock(ac, false); 4306 drain_all_pages(NULL); 4307 drained = true; 4308 goto retry; 4309 } 4310 4311 return page; 4312 } 4313 4314 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, 4315 const struct alloc_context *ac) 4316 { 4317 struct zoneref *z; 4318 struct zone *zone; 4319 pg_data_t *last_pgdat = NULL; 4320 enum zone_type highest_zoneidx = ac->highest_zoneidx; 4321 4322 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx, 4323 ac->nodemask) { 4324 if (last_pgdat != zone->zone_pgdat) 4325 wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx); 4326 last_pgdat = zone->zone_pgdat; 4327 } 4328 } 4329 4330 static inline unsigned int 4331 gfp_to_alloc_flags(gfp_t gfp_mask) 4332 { 4333 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 4334 4335 /* 4336 * __GFP_HIGH is assumed to be the same as ALLOC_HIGH 4337 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 4338 * to save two branches. 4339 */ 4340 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 4341 BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); 4342 4343 /* 4344 * The caller may dip into page reserves a bit more if the caller 4345 * cannot run direct reclaim, or if the caller has realtime scheduling 4346 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 4347 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 4348 */ 4349 alloc_flags |= (__force int) 4350 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); 4351 4352 if (gfp_mask & __GFP_ATOMIC) { 4353 /* 4354 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 4355 * if it can't schedule. 4356 */ 4357 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4358 alloc_flags |= ALLOC_HARDER; 4359 /* 4360 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 4361 * comment for __cpuset_node_allowed(). 4362 */ 4363 alloc_flags &= ~ALLOC_CPUSET; 4364 } else if (unlikely(rt_task(current)) && !in_interrupt()) 4365 alloc_flags |= ALLOC_HARDER; 4366 4367 alloc_flags = current_alloc_flags(gfp_mask, alloc_flags); 4368 4369 return alloc_flags; 4370 } 4371 4372 static bool oom_reserves_allowed(struct task_struct *tsk) 4373 { 4374 if (!tsk_is_oom_victim(tsk)) 4375 return false; 4376 4377 /* 4378 * !MMU doesn't have oom reaper so give access to memory reserves 4379 * only to the thread with TIF_MEMDIE set 4380 */ 4381 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE)) 4382 return false; 4383 4384 return true; 4385 } 4386 4387 /* 4388 * Distinguish requests which really need access to full memory 4389 * reserves from oom victims which can live with a portion of it 4390 */ 4391 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) 4392 { 4393 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 4394 return 0; 4395 if (gfp_mask & __GFP_MEMALLOC) 4396 return ALLOC_NO_WATERMARKS; 4397 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 4398 return ALLOC_NO_WATERMARKS; 4399 if (!in_interrupt()) { 4400 if (current->flags & PF_MEMALLOC) 4401 return ALLOC_NO_WATERMARKS; 4402 else if (oom_reserves_allowed(current)) 4403 return ALLOC_OOM; 4404 } 4405 4406 return 0; 4407 } 4408 4409 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 4410 { 4411 return !!__gfp_pfmemalloc_flags(gfp_mask); 4412 } 4413 4414 /* 4415 * Checks whether it makes sense to retry the reclaim to make a forward progress 4416 * for the given allocation request. 4417 * 4418 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row 4419 * without success, or when we couldn't even meet the watermark if we 4420 * reclaimed all remaining pages on the LRU lists. 4421 * 4422 * Returns true if a retry is viable or false to enter the oom path. 4423 */ 4424 static inline bool 4425 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 4426 struct alloc_context *ac, int alloc_flags, 4427 bool did_some_progress, int *no_progress_loops) 4428 { 4429 struct zone *zone; 4430 struct zoneref *z; 4431 bool ret = false; 4432 4433 /* 4434 * Costly allocations might have made a progress but this doesn't mean 4435 * their order will become available due to high fragmentation so 4436 * always increment the no progress counter for them 4437 */ 4438 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 4439 *no_progress_loops = 0; 4440 else 4441 (*no_progress_loops)++; 4442 4443 /* 4444 * Make sure we converge to OOM if we cannot make any progress 4445 * several times in the row. 4446 */ 4447 if (*no_progress_loops > MAX_RECLAIM_RETRIES) { 4448 /* Before OOM, exhaust highatomic_reserve */ 4449 return unreserve_highatomic_pageblock(ac, true); 4450 } 4451 4452 /* 4453 * Keep reclaiming pages while there is a chance this will lead 4454 * somewhere. If none of the target zones can satisfy our allocation 4455 * request even if all reclaimable pages are considered then we are 4456 * screwed and have to go OOM. 4457 */ 4458 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4459 ac->highest_zoneidx, ac->nodemask) { 4460 unsigned long available; 4461 unsigned long reclaimable; 4462 unsigned long min_wmark = min_wmark_pages(zone); 4463 bool wmark; 4464 4465 available = reclaimable = zone_reclaimable_pages(zone); 4466 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 4467 4468 /* 4469 * Would the allocation succeed if we reclaimed all 4470 * reclaimable pages? 4471 */ 4472 wmark = __zone_watermark_ok(zone, order, min_wmark, 4473 ac->highest_zoneidx, alloc_flags, available); 4474 trace_reclaim_retry_zone(z, order, reclaimable, 4475 available, min_wmark, *no_progress_loops, wmark); 4476 if (wmark) { 4477 /* 4478 * If we didn't make any progress and have a lot of 4479 * dirty + writeback pages then we should wait for 4480 * an IO to complete to slow down the reclaim and 4481 * prevent from pre mature OOM 4482 */ 4483 if (!did_some_progress) { 4484 unsigned long write_pending; 4485 4486 write_pending = zone_page_state_snapshot(zone, 4487 NR_ZONE_WRITE_PENDING); 4488 4489 if (2 * write_pending > reclaimable) { 4490 congestion_wait(BLK_RW_ASYNC, HZ/10); 4491 return true; 4492 } 4493 } 4494 4495 ret = true; 4496 goto out; 4497 } 4498 } 4499 4500 out: 4501 /* 4502 * Memory allocation/reclaim might be called from a WQ context and the 4503 * current implementation of the WQ concurrency control doesn't 4504 * recognize that a particular WQ is congested if the worker thread is 4505 * looping without ever sleeping. Therefore we have to do a short sleep 4506 * here rather than calling cond_resched(). 4507 */ 4508 if (current->flags & PF_WQ_WORKER) 4509 schedule_timeout_uninterruptible(1); 4510 else 4511 cond_resched(); 4512 return ret; 4513 } 4514 4515 static inline bool 4516 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) 4517 { 4518 /* 4519 * It's possible that cpuset's mems_allowed and the nodemask from 4520 * mempolicy don't intersect. This should be normally dealt with by 4521 * policy_nodemask(), but it's possible to race with cpuset update in 4522 * such a way the check therein was true, and then it became false 4523 * before we got our cpuset_mems_cookie here. 4524 * This assumes that for all allocations, ac->nodemask can come only 4525 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored 4526 * when it does not intersect with the cpuset restrictions) or the 4527 * caller can deal with a violated nodemask. 4528 */ 4529 if (cpusets_enabled() && ac->nodemask && 4530 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) { 4531 ac->nodemask = NULL; 4532 return true; 4533 } 4534 4535 /* 4536 * When updating a task's mems_allowed or mempolicy nodemask, it is 4537 * possible to race with parallel threads in such a way that our 4538 * allocation can fail while the mask is being updated. If we are about 4539 * to fail, check if the cpuset changed during allocation and if so, 4540 * retry. 4541 */ 4542 if (read_mems_allowed_retry(cpuset_mems_cookie)) 4543 return true; 4544 4545 return false; 4546 } 4547 4548 static inline struct page * 4549 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 4550 struct alloc_context *ac) 4551 { 4552 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 4553 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 4554 struct page *page = NULL; 4555 unsigned int alloc_flags; 4556 unsigned long did_some_progress; 4557 enum compact_priority compact_priority; 4558 enum compact_result compact_result; 4559 int compaction_retries; 4560 int no_progress_loops; 4561 unsigned int cpuset_mems_cookie; 4562 int reserve_flags; 4563 4564 /* 4565 * We also sanity check to catch abuse of atomic reserves being used by 4566 * callers that are not in atomic context. 4567 */ 4568 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 4569 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 4570 gfp_mask &= ~__GFP_ATOMIC; 4571 4572 retry_cpuset: 4573 compaction_retries = 0; 4574 no_progress_loops = 0; 4575 compact_priority = DEF_COMPACT_PRIORITY; 4576 cpuset_mems_cookie = read_mems_allowed_begin(); 4577 4578 /* 4579 * The fast path uses conservative alloc_flags to succeed only until 4580 * kswapd needs to be woken up, and to avoid the cost of setting up 4581 * alloc_flags precisely. So we do that now. 4582 */ 4583 alloc_flags = gfp_to_alloc_flags(gfp_mask); 4584 4585 /* 4586 * We need to recalculate the starting point for the zonelist iterator 4587 * because we might have used different nodemask in the fast path, or 4588 * there was a cpuset modification and we are retrying - otherwise we 4589 * could end up iterating over non-eligible zones endlessly. 4590 */ 4591 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4592 ac->highest_zoneidx, ac->nodemask); 4593 if (!ac->preferred_zoneref->zone) 4594 goto nopage; 4595 4596 if (alloc_flags & ALLOC_KSWAPD) 4597 wake_all_kswapds(order, gfp_mask, ac); 4598 4599 /* 4600 * The adjusted alloc_flags might result in immediate success, so try 4601 * that first 4602 */ 4603 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4604 if (page) 4605 goto got_pg; 4606 4607 /* 4608 * For costly allocations, try direct compaction first, as it's likely 4609 * that we have enough base pages and don't need to reclaim. For non- 4610 * movable high-order allocations, do that as well, as compaction will 4611 * try prevent permanent fragmentation by migrating from blocks of the 4612 * same migratetype. 4613 * Don't try this for allocations that are allowed to ignore 4614 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 4615 */ 4616 if (can_direct_reclaim && 4617 (costly_order || 4618 (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 4619 && !gfp_pfmemalloc_allowed(gfp_mask)) { 4620 page = __alloc_pages_direct_compact(gfp_mask, order, 4621 alloc_flags, ac, 4622 INIT_COMPACT_PRIORITY, 4623 &compact_result); 4624 if (page) 4625 goto got_pg; 4626 4627 /* 4628 * Checks for costly allocations with __GFP_NORETRY, which 4629 * includes some THP page fault allocations 4630 */ 4631 if (costly_order && (gfp_mask & __GFP_NORETRY)) { 4632 /* 4633 * If allocating entire pageblock(s) and compaction 4634 * failed because all zones are below low watermarks 4635 * or is prohibited because it recently failed at this 4636 * order, fail immediately unless the allocator has 4637 * requested compaction and reclaim retry. 4638 * 4639 * Reclaim is 4640 * - potentially very expensive because zones are far 4641 * below their low watermarks or this is part of very 4642 * bursty high order allocations, 4643 * - not guaranteed to help because isolate_freepages() 4644 * may not iterate over freed pages as part of its 4645 * linear scan, and 4646 * - unlikely to make entire pageblocks free on its 4647 * own. 4648 */ 4649 if (compact_result == COMPACT_SKIPPED || 4650 compact_result == COMPACT_DEFERRED) 4651 goto nopage; 4652 4653 /* 4654 * Looks like reclaim/compaction is worth trying, but 4655 * sync compaction could be very expensive, so keep 4656 * using async compaction. 4657 */ 4658 compact_priority = INIT_COMPACT_PRIORITY; 4659 } 4660 } 4661 4662 retry: 4663 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 4664 if (alloc_flags & ALLOC_KSWAPD) 4665 wake_all_kswapds(order, gfp_mask, ac); 4666 4667 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); 4668 if (reserve_flags) 4669 alloc_flags = current_alloc_flags(gfp_mask, reserve_flags); 4670 4671 /* 4672 * Reset the nodemask and zonelist iterators if memory policies can be 4673 * ignored. These allocations are high priority and system rather than 4674 * user oriented. 4675 */ 4676 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { 4677 ac->nodemask = NULL; 4678 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4679 ac->highest_zoneidx, ac->nodemask); 4680 } 4681 4682 /* Attempt with potentially adjusted zonelist and alloc_flags */ 4683 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4684 if (page) 4685 goto got_pg; 4686 4687 /* Caller is not willing to reclaim, we can't balance anything */ 4688 if (!can_direct_reclaim) 4689 goto nopage; 4690 4691 /* Avoid recursion of direct reclaim */ 4692 if (current->flags & PF_MEMALLOC) 4693 goto nopage; 4694 4695 /* Try direct reclaim and then allocating */ 4696 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 4697 &did_some_progress); 4698 if (page) 4699 goto got_pg; 4700 4701 /* Try direct compaction and then allocating */ 4702 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 4703 compact_priority, &compact_result); 4704 if (page) 4705 goto got_pg; 4706 4707 /* Do not loop if specifically requested */ 4708 if (gfp_mask & __GFP_NORETRY) 4709 goto nopage; 4710 4711 /* 4712 * Do not retry costly high order allocations unless they are 4713 * __GFP_RETRY_MAYFAIL 4714 */ 4715 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) 4716 goto nopage; 4717 4718 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 4719 did_some_progress > 0, &no_progress_loops)) 4720 goto retry; 4721 4722 /* 4723 * It doesn't make any sense to retry for the compaction if the order-0 4724 * reclaim is not able to make any progress because the current 4725 * implementation of the compaction depends on the sufficient amount 4726 * of free memory (see __compaction_suitable) 4727 */ 4728 if (did_some_progress > 0 && 4729 should_compact_retry(ac, order, alloc_flags, 4730 compact_result, &compact_priority, 4731 &compaction_retries)) 4732 goto retry; 4733 4734 4735 /* Deal with possible cpuset update races before we start OOM killing */ 4736 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4737 goto retry_cpuset; 4738 4739 /* Reclaim has failed us, start killing things */ 4740 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 4741 if (page) 4742 goto got_pg; 4743 4744 /* Avoid allocations with no watermarks from looping endlessly */ 4745 if (tsk_is_oom_victim(current) && 4746 (alloc_flags & ALLOC_OOM || 4747 (gfp_mask & __GFP_NOMEMALLOC))) 4748 goto nopage; 4749 4750 /* Retry as long as the OOM killer is making progress */ 4751 if (did_some_progress) { 4752 no_progress_loops = 0; 4753 goto retry; 4754 } 4755 4756 nopage: 4757 /* Deal with possible cpuset update races before we fail */ 4758 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4759 goto retry_cpuset; 4760 4761 /* 4762 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure 4763 * we always retry 4764 */ 4765 if (gfp_mask & __GFP_NOFAIL) { 4766 /* 4767 * All existing users of the __GFP_NOFAIL are blockable, so warn 4768 * of any new users that actually require GFP_NOWAIT 4769 */ 4770 if (WARN_ON_ONCE(!can_direct_reclaim)) 4771 goto fail; 4772 4773 /* 4774 * PF_MEMALLOC request from this context is rather bizarre 4775 * because we cannot reclaim anything and only can loop waiting 4776 * for somebody to do a work for us 4777 */ 4778 WARN_ON_ONCE(current->flags & PF_MEMALLOC); 4779 4780 /* 4781 * non failing costly orders are a hard requirement which we 4782 * are not prepared for much so let's warn about these users 4783 * so that we can identify them and convert them to something 4784 * else. 4785 */ 4786 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER); 4787 4788 /* 4789 * Help non-failing allocations by giving them access to memory 4790 * reserves but do not use ALLOC_NO_WATERMARKS because this 4791 * could deplete whole memory reserves which would just make 4792 * the situation worse 4793 */ 4794 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); 4795 if (page) 4796 goto got_pg; 4797 4798 cond_resched(); 4799 goto retry; 4800 } 4801 fail: 4802 warn_alloc(gfp_mask, ac->nodemask, 4803 "page allocation failure: order:%u", order); 4804 got_pg: 4805 return page; 4806 } 4807 4808 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, 4809 int preferred_nid, nodemask_t *nodemask, 4810 struct alloc_context *ac, gfp_t *alloc_mask, 4811 unsigned int *alloc_flags) 4812 { 4813 ac->highest_zoneidx = gfp_zone(gfp_mask); 4814 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); 4815 ac->nodemask = nodemask; 4816 ac->migratetype = gfp_migratetype(gfp_mask); 4817 4818 if (cpusets_enabled()) { 4819 *alloc_mask |= __GFP_HARDWALL; 4820 /* 4821 * When we are in the interrupt context, it is irrelevant 4822 * to the current task context. It means that any node ok. 4823 */ 4824 if (!in_interrupt() && !ac->nodemask) 4825 ac->nodemask = &cpuset_current_mems_allowed; 4826 else 4827 *alloc_flags |= ALLOC_CPUSET; 4828 } 4829 4830 fs_reclaim_acquire(gfp_mask); 4831 fs_reclaim_release(gfp_mask); 4832 4833 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 4834 4835 if (should_fail_alloc_page(gfp_mask, order)) 4836 return false; 4837 4838 *alloc_flags = current_alloc_flags(gfp_mask, *alloc_flags); 4839 4840 /* Dirty zone balancing only done in the fast path */ 4841 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); 4842 4843 /* 4844 * The preferred zone is used for statistics but crucially it is 4845 * also used as the starting point for the zonelist iterator. It 4846 * may get reset for allocations that ignore memory policies. 4847 */ 4848 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4849 ac->highest_zoneidx, ac->nodemask); 4850 4851 return true; 4852 } 4853 4854 /* 4855 * This is the 'heart' of the zoned buddy allocator. 4856 */ 4857 struct page * 4858 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, 4859 nodemask_t *nodemask) 4860 { 4861 struct page *page; 4862 unsigned int alloc_flags = ALLOC_WMARK_LOW; 4863 gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */ 4864 struct alloc_context ac = { }; 4865 4866 /* 4867 * There are several places where we assume that the order value is sane 4868 * so bail out early if the request is out of bound. 4869 */ 4870 if (unlikely(order >= MAX_ORDER)) { 4871 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); 4872 return NULL; 4873 } 4874 4875 gfp_mask &= gfp_allowed_mask; 4876 alloc_mask = gfp_mask; 4877 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags)) 4878 return NULL; 4879 4880 /* 4881 * Forbid the first pass from falling back to types that fragment 4882 * memory until all local zones are considered. 4883 */ 4884 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask); 4885 4886 /* First allocation attempt */ 4887 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); 4888 if (likely(page)) 4889 goto out; 4890 4891 /* 4892 * Apply scoped allocation constraints. This is mainly about GFP_NOFS 4893 * resp. GFP_NOIO which has to be inherited for all allocation requests 4894 * from a particular context which has been marked by 4895 * memalloc_no{fs,io}_{save,restore}. 4896 */ 4897 alloc_mask = current_gfp_context(gfp_mask); 4898 ac.spread_dirty_pages = false; 4899 4900 /* 4901 * Restore the original nodemask if it was potentially replaced with 4902 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 4903 */ 4904 ac.nodemask = nodemask; 4905 4906 page = __alloc_pages_slowpath(alloc_mask, order, &ac); 4907 4908 out: 4909 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && 4910 unlikely(__memcg_kmem_charge_page(page, gfp_mask, order) != 0)) { 4911 __free_pages(page, order); 4912 page = NULL; 4913 } 4914 4915 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); 4916 4917 return page; 4918 } 4919 EXPORT_SYMBOL(__alloc_pages_nodemask); 4920 4921 /* 4922 * Common helper functions. Never use with __GFP_HIGHMEM because the returned 4923 * address cannot represent highmem pages. Use alloc_pages and then kmap if 4924 * you need to access high mem. 4925 */ 4926 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 4927 { 4928 struct page *page; 4929 4930 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); 4931 if (!page) 4932 return 0; 4933 return (unsigned long) page_address(page); 4934 } 4935 EXPORT_SYMBOL(__get_free_pages); 4936 4937 unsigned long get_zeroed_page(gfp_t gfp_mask) 4938 { 4939 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 4940 } 4941 EXPORT_SYMBOL(get_zeroed_page); 4942 4943 static inline void free_the_page(struct page *page, unsigned int order) 4944 { 4945 if (order == 0) /* Via pcp? */ 4946 free_unref_page(page); 4947 else 4948 __free_pages_ok(page, order); 4949 } 4950 4951 void __free_pages(struct page *page, unsigned int order) 4952 { 4953 if (put_page_testzero(page)) 4954 free_the_page(page, order); 4955 else if (!PageHead(page)) 4956 while (order-- > 0) 4957 free_the_page(page + (1 << order), order); 4958 } 4959 EXPORT_SYMBOL(__free_pages); 4960 4961 void free_pages(unsigned long addr, unsigned int order) 4962 { 4963 if (addr != 0) { 4964 VM_BUG_ON(!virt_addr_valid((void *)addr)); 4965 __free_pages(virt_to_page((void *)addr), order); 4966 } 4967 } 4968 4969 EXPORT_SYMBOL(free_pages); 4970 4971 /* 4972 * Page Fragment: 4973 * An arbitrary-length arbitrary-offset area of memory which resides 4974 * within a 0 or higher order page. Multiple fragments within that page 4975 * are individually refcounted, in the page's reference counter. 4976 * 4977 * The page_frag functions below provide a simple allocation framework for 4978 * page fragments. This is used by the network stack and network device 4979 * drivers to provide a backing region of memory for use as either an 4980 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 4981 */ 4982 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 4983 gfp_t gfp_mask) 4984 { 4985 struct page *page = NULL; 4986 gfp_t gfp = gfp_mask; 4987 4988 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4989 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 4990 __GFP_NOMEMALLOC; 4991 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 4992 PAGE_FRAG_CACHE_MAX_ORDER); 4993 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 4994 #endif 4995 if (unlikely(!page)) 4996 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 4997 4998 nc->va = page ? page_address(page) : NULL; 4999 5000 return page; 5001 } 5002 5003 void __page_frag_cache_drain(struct page *page, unsigned int count) 5004 { 5005 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 5006 5007 if (page_ref_sub_and_test(page, count)) 5008 free_the_page(page, compound_order(page)); 5009 } 5010 EXPORT_SYMBOL(__page_frag_cache_drain); 5011 5012 void *page_frag_alloc(struct page_frag_cache *nc, 5013 unsigned int fragsz, gfp_t gfp_mask) 5014 { 5015 unsigned int size = PAGE_SIZE; 5016 struct page *page; 5017 int offset; 5018 5019 if (unlikely(!nc->va)) { 5020 refill: 5021 page = __page_frag_cache_refill(nc, gfp_mask); 5022 if (!page) 5023 return NULL; 5024 5025 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5026 /* if size can vary use size else just use PAGE_SIZE */ 5027 size = nc->size; 5028 #endif 5029 /* Even if we own the page, we do not use atomic_set(). 5030 * This would break get_page_unless_zero() users. 5031 */ 5032 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 5033 5034 /* reset page count bias and offset to start of new frag */ 5035 nc->pfmemalloc = page_is_pfmemalloc(page); 5036 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5037 nc->offset = size; 5038 } 5039 5040 offset = nc->offset - fragsz; 5041 if (unlikely(offset < 0)) { 5042 page = virt_to_page(nc->va); 5043 5044 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 5045 goto refill; 5046 5047 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5048 /* if size can vary use size else just use PAGE_SIZE */ 5049 size = nc->size; 5050 #endif 5051 /* OK, page count is 0, we can safely set it */ 5052 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 5053 5054 /* reset page count bias and offset to start of new frag */ 5055 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5056 offset = size - fragsz; 5057 } 5058 5059 nc->pagecnt_bias--; 5060 nc->offset = offset; 5061 5062 return nc->va + offset; 5063 } 5064 EXPORT_SYMBOL(page_frag_alloc); 5065 5066 /* 5067 * Frees a page fragment allocated out of either a compound or order 0 page. 5068 */ 5069 void page_frag_free(void *addr) 5070 { 5071 struct page *page = virt_to_head_page(addr); 5072 5073 if (unlikely(put_page_testzero(page))) 5074 free_the_page(page, compound_order(page)); 5075 } 5076 EXPORT_SYMBOL(page_frag_free); 5077 5078 static void *make_alloc_exact(unsigned long addr, unsigned int order, 5079 size_t size) 5080 { 5081 if (addr) { 5082 unsigned long alloc_end = addr + (PAGE_SIZE << order); 5083 unsigned long used = addr + PAGE_ALIGN(size); 5084 5085 split_page(virt_to_page((void *)addr), order); 5086 while (used < alloc_end) { 5087 free_page(used); 5088 used += PAGE_SIZE; 5089 } 5090 } 5091 return (void *)addr; 5092 } 5093 5094 /** 5095 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 5096 * @size: the number of bytes to allocate 5097 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5098 * 5099 * This function is similar to alloc_pages(), except that it allocates the 5100 * minimum number of pages to satisfy the request. alloc_pages() can only 5101 * allocate memory in power-of-two pages. 5102 * 5103 * This function is also limited by MAX_ORDER. 5104 * 5105 * Memory allocated by this function must be released by free_pages_exact(). 5106 * 5107 * Return: pointer to the allocated area or %NULL in case of error. 5108 */ 5109 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 5110 { 5111 unsigned int order = get_order(size); 5112 unsigned long addr; 5113 5114 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 5115 gfp_mask &= ~__GFP_COMP; 5116 5117 addr = __get_free_pages(gfp_mask, order); 5118 return make_alloc_exact(addr, order, size); 5119 } 5120 EXPORT_SYMBOL(alloc_pages_exact); 5121 5122 /** 5123 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 5124 * pages on a node. 5125 * @nid: the preferred node ID where memory should be allocated 5126 * @size: the number of bytes to allocate 5127 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5128 * 5129 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 5130 * back. 5131 * 5132 * Return: pointer to the allocated area or %NULL in case of error. 5133 */ 5134 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 5135 { 5136 unsigned int order = get_order(size); 5137 struct page *p; 5138 5139 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 5140 gfp_mask &= ~__GFP_COMP; 5141 5142 p = alloc_pages_node(nid, gfp_mask, order); 5143 if (!p) 5144 return NULL; 5145 return make_alloc_exact((unsigned long)page_address(p), order, size); 5146 } 5147 5148 /** 5149 * free_pages_exact - release memory allocated via alloc_pages_exact() 5150 * @virt: the value returned by alloc_pages_exact. 5151 * @size: size of allocation, same value as passed to alloc_pages_exact(). 5152 * 5153 * Release the memory allocated by a previous call to alloc_pages_exact. 5154 */ 5155 void free_pages_exact(void *virt, size_t size) 5156 { 5157 unsigned long addr = (unsigned long)virt; 5158 unsigned long end = addr + PAGE_ALIGN(size); 5159 5160 while (addr < end) { 5161 free_page(addr); 5162 addr += PAGE_SIZE; 5163 } 5164 } 5165 EXPORT_SYMBOL(free_pages_exact); 5166 5167 /** 5168 * nr_free_zone_pages - count number of pages beyond high watermark 5169 * @offset: The zone index of the highest zone 5170 * 5171 * nr_free_zone_pages() counts the number of pages which are beyond the 5172 * high watermark within all zones at or below a given zone index. For each 5173 * zone, the number of pages is calculated as: 5174 * 5175 * nr_free_zone_pages = managed_pages - high_pages 5176 * 5177 * Return: number of pages beyond high watermark. 5178 */ 5179 static unsigned long nr_free_zone_pages(int offset) 5180 { 5181 struct zoneref *z; 5182 struct zone *zone; 5183 5184 /* Just pick one node, since fallback list is circular */ 5185 unsigned long sum = 0; 5186 5187 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 5188 5189 for_each_zone_zonelist(zone, z, zonelist, offset) { 5190 unsigned long size = zone_managed_pages(zone); 5191 unsigned long high = high_wmark_pages(zone); 5192 if (size > high) 5193 sum += size - high; 5194 } 5195 5196 return sum; 5197 } 5198 5199 /** 5200 * nr_free_buffer_pages - count number of pages beyond high watermark 5201 * 5202 * nr_free_buffer_pages() counts the number of pages which are beyond the high 5203 * watermark within ZONE_DMA and ZONE_NORMAL. 5204 * 5205 * Return: number of pages beyond high watermark within ZONE_DMA and 5206 * ZONE_NORMAL. 5207 */ 5208 unsigned long nr_free_buffer_pages(void) 5209 { 5210 return nr_free_zone_pages(gfp_zone(GFP_USER)); 5211 } 5212 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 5213 5214 static inline void show_node(struct zone *zone) 5215 { 5216 if (IS_ENABLED(CONFIG_NUMA)) 5217 printk("Node %d ", zone_to_nid(zone)); 5218 } 5219 5220 long si_mem_available(void) 5221 { 5222 long available; 5223 unsigned long pagecache; 5224 unsigned long wmark_low = 0; 5225 unsigned long pages[NR_LRU_LISTS]; 5226 unsigned long reclaimable; 5227 struct zone *zone; 5228 int lru; 5229 5230 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 5231 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 5232 5233 for_each_zone(zone) 5234 wmark_low += low_wmark_pages(zone); 5235 5236 /* 5237 * Estimate the amount of memory available for userspace allocations, 5238 * without causing swapping. 5239 */ 5240 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages; 5241 5242 /* 5243 * Not all the page cache can be freed, otherwise the system will 5244 * start swapping. Assume at least half of the page cache, or the 5245 * low watermark worth of cache, needs to stay. 5246 */ 5247 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 5248 pagecache -= min(pagecache / 2, wmark_low); 5249 available += pagecache; 5250 5251 /* 5252 * Part of the reclaimable slab and other kernel memory consists of 5253 * items that are in use, and cannot be freed. Cap this estimate at the 5254 * low watermark. 5255 */ 5256 reclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B) + 5257 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE); 5258 available += reclaimable - min(reclaimable / 2, wmark_low); 5259 5260 if (available < 0) 5261 available = 0; 5262 return available; 5263 } 5264 EXPORT_SYMBOL_GPL(si_mem_available); 5265 5266 void si_meminfo(struct sysinfo *val) 5267 { 5268 val->totalram = totalram_pages(); 5269 val->sharedram = global_node_page_state(NR_SHMEM); 5270 val->freeram = global_zone_page_state(NR_FREE_PAGES); 5271 val->bufferram = nr_blockdev_pages(); 5272 val->totalhigh = totalhigh_pages(); 5273 val->freehigh = nr_free_highpages(); 5274 val->mem_unit = PAGE_SIZE; 5275 } 5276 5277 EXPORT_SYMBOL(si_meminfo); 5278 5279 #ifdef CONFIG_NUMA 5280 void si_meminfo_node(struct sysinfo *val, int nid) 5281 { 5282 int zone_type; /* needs to be signed */ 5283 unsigned long managed_pages = 0; 5284 unsigned long managed_highpages = 0; 5285 unsigned long free_highpages = 0; 5286 pg_data_t *pgdat = NODE_DATA(nid); 5287 5288 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 5289 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]); 5290 val->totalram = managed_pages; 5291 val->sharedram = node_page_state(pgdat, NR_SHMEM); 5292 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 5293 #ifdef CONFIG_HIGHMEM 5294 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 5295 struct zone *zone = &pgdat->node_zones[zone_type]; 5296 5297 if (is_highmem(zone)) { 5298 managed_highpages += zone_managed_pages(zone); 5299 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 5300 } 5301 } 5302 val->totalhigh = managed_highpages; 5303 val->freehigh = free_highpages; 5304 #else 5305 val->totalhigh = managed_highpages; 5306 val->freehigh = free_highpages; 5307 #endif 5308 val->mem_unit = PAGE_SIZE; 5309 } 5310 #endif 5311 5312 /* 5313 * Determine whether the node should be displayed or not, depending on whether 5314 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 5315 */ 5316 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) 5317 { 5318 if (!(flags & SHOW_MEM_FILTER_NODES)) 5319 return false; 5320 5321 /* 5322 * no node mask - aka implicit memory numa policy. Do not bother with 5323 * the synchronization - read_mems_allowed_begin - because we do not 5324 * have to be precise here. 5325 */ 5326 if (!nodemask) 5327 nodemask = &cpuset_current_mems_allowed; 5328 5329 return !node_isset(nid, *nodemask); 5330 } 5331 5332 #define K(x) ((x) << (PAGE_SHIFT-10)) 5333 5334 static void show_migration_types(unsigned char type) 5335 { 5336 static const char types[MIGRATE_TYPES] = { 5337 [MIGRATE_UNMOVABLE] = 'U', 5338 [MIGRATE_MOVABLE] = 'M', 5339 [MIGRATE_RECLAIMABLE] = 'E', 5340 [MIGRATE_HIGHATOMIC] = 'H', 5341 #ifdef CONFIG_CMA 5342 [MIGRATE_CMA] = 'C', 5343 #endif 5344 #ifdef CONFIG_MEMORY_ISOLATION 5345 [MIGRATE_ISOLATE] = 'I', 5346 #endif 5347 }; 5348 char tmp[MIGRATE_TYPES + 1]; 5349 char *p = tmp; 5350 int i; 5351 5352 for (i = 0; i < MIGRATE_TYPES; i++) { 5353 if (type & (1 << i)) 5354 *p++ = types[i]; 5355 } 5356 5357 *p = '\0'; 5358 printk(KERN_CONT "(%s) ", tmp); 5359 } 5360 5361 /* 5362 * Show free area list (used inside shift_scroll-lock stuff) 5363 * We also calculate the percentage fragmentation. We do this by counting the 5364 * memory on each free list with the exception of the first item on the list. 5365 * 5366 * Bits in @filter: 5367 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 5368 * cpuset. 5369 */ 5370 void show_free_areas(unsigned int filter, nodemask_t *nodemask) 5371 { 5372 unsigned long free_pcp = 0; 5373 int cpu; 5374 struct zone *zone; 5375 pg_data_t *pgdat; 5376 5377 for_each_populated_zone(zone) { 5378 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5379 continue; 5380 5381 for_each_online_cpu(cpu) 5382 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 5383 } 5384 5385 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 5386 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 5387 " unevictable:%lu dirty:%lu writeback:%lu\n" 5388 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 5389 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 5390 " free:%lu free_pcp:%lu free_cma:%lu\n", 5391 global_node_page_state(NR_ACTIVE_ANON), 5392 global_node_page_state(NR_INACTIVE_ANON), 5393 global_node_page_state(NR_ISOLATED_ANON), 5394 global_node_page_state(NR_ACTIVE_FILE), 5395 global_node_page_state(NR_INACTIVE_FILE), 5396 global_node_page_state(NR_ISOLATED_FILE), 5397 global_node_page_state(NR_UNEVICTABLE), 5398 global_node_page_state(NR_FILE_DIRTY), 5399 global_node_page_state(NR_WRITEBACK), 5400 global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B), 5401 global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B), 5402 global_node_page_state(NR_FILE_MAPPED), 5403 global_node_page_state(NR_SHMEM), 5404 global_zone_page_state(NR_PAGETABLE), 5405 global_zone_page_state(NR_BOUNCE), 5406 global_zone_page_state(NR_FREE_PAGES), 5407 free_pcp, 5408 global_zone_page_state(NR_FREE_CMA_PAGES)); 5409 5410 for_each_online_pgdat(pgdat) { 5411 if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) 5412 continue; 5413 5414 printk("Node %d" 5415 " active_anon:%lukB" 5416 " inactive_anon:%lukB" 5417 " active_file:%lukB" 5418 " inactive_file:%lukB" 5419 " unevictable:%lukB" 5420 " isolated(anon):%lukB" 5421 " isolated(file):%lukB" 5422 " mapped:%lukB" 5423 " dirty:%lukB" 5424 " writeback:%lukB" 5425 " shmem:%lukB" 5426 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5427 " shmem_thp: %lukB" 5428 " shmem_pmdmapped: %lukB" 5429 " anon_thp: %lukB" 5430 #endif 5431 " writeback_tmp:%lukB" 5432 " kernel_stack:%lukB" 5433 #ifdef CONFIG_SHADOW_CALL_STACK 5434 " shadow_call_stack:%lukB" 5435 #endif 5436 " all_unreclaimable? %s" 5437 "\n", 5438 pgdat->node_id, 5439 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 5440 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 5441 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 5442 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 5443 K(node_page_state(pgdat, NR_UNEVICTABLE)), 5444 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 5445 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 5446 K(node_page_state(pgdat, NR_FILE_MAPPED)), 5447 K(node_page_state(pgdat, NR_FILE_DIRTY)), 5448 K(node_page_state(pgdat, NR_WRITEBACK)), 5449 K(node_page_state(pgdat, NR_SHMEM)), 5450 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5451 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), 5452 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) 5453 * HPAGE_PMD_NR), 5454 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR), 5455 #endif 5456 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 5457 node_page_state(pgdat, NR_KERNEL_STACK_KB), 5458 #ifdef CONFIG_SHADOW_CALL_STACK 5459 node_page_state(pgdat, NR_KERNEL_SCS_KB), 5460 #endif 5461 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ? 5462 "yes" : "no"); 5463 } 5464 5465 for_each_populated_zone(zone) { 5466 int i; 5467 5468 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5469 continue; 5470 5471 free_pcp = 0; 5472 for_each_online_cpu(cpu) 5473 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 5474 5475 show_node(zone); 5476 printk(KERN_CONT 5477 "%s" 5478 " free:%lukB" 5479 " min:%lukB" 5480 " low:%lukB" 5481 " high:%lukB" 5482 " reserved_highatomic:%luKB" 5483 " active_anon:%lukB" 5484 " inactive_anon:%lukB" 5485 " active_file:%lukB" 5486 " inactive_file:%lukB" 5487 " unevictable:%lukB" 5488 " writepending:%lukB" 5489 " present:%lukB" 5490 " managed:%lukB" 5491 " mlocked:%lukB" 5492 " pagetables:%lukB" 5493 " bounce:%lukB" 5494 " free_pcp:%lukB" 5495 " local_pcp:%ukB" 5496 " free_cma:%lukB" 5497 "\n", 5498 zone->name, 5499 K(zone_page_state(zone, NR_FREE_PAGES)), 5500 K(min_wmark_pages(zone)), 5501 K(low_wmark_pages(zone)), 5502 K(high_wmark_pages(zone)), 5503 K(zone->nr_reserved_highatomic), 5504 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 5505 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 5506 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 5507 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 5508 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 5509 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 5510 K(zone->present_pages), 5511 K(zone_managed_pages(zone)), 5512 K(zone_page_state(zone, NR_MLOCK)), 5513 K(zone_page_state(zone, NR_PAGETABLE)), 5514 K(zone_page_state(zone, NR_BOUNCE)), 5515 K(free_pcp), 5516 K(this_cpu_read(zone->pageset->pcp.count)), 5517 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 5518 printk("lowmem_reserve[]:"); 5519 for (i = 0; i < MAX_NR_ZONES; i++) 5520 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); 5521 printk(KERN_CONT "\n"); 5522 } 5523 5524 for_each_populated_zone(zone) { 5525 unsigned int order; 5526 unsigned long nr[MAX_ORDER], flags, total = 0; 5527 unsigned char types[MAX_ORDER]; 5528 5529 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5530 continue; 5531 show_node(zone); 5532 printk(KERN_CONT "%s: ", zone->name); 5533 5534 spin_lock_irqsave(&zone->lock, flags); 5535 for (order = 0; order < MAX_ORDER; order++) { 5536 struct free_area *area = &zone->free_area[order]; 5537 int type; 5538 5539 nr[order] = area->nr_free; 5540 total += nr[order] << order; 5541 5542 types[order] = 0; 5543 for (type = 0; type < MIGRATE_TYPES; type++) { 5544 if (!free_area_empty(area, type)) 5545 types[order] |= 1 << type; 5546 } 5547 } 5548 spin_unlock_irqrestore(&zone->lock, flags); 5549 for (order = 0; order < MAX_ORDER; order++) { 5550 printk(KERN_CONT "%lu*%lukB ", 5551 nr[order], K(1UL) << order); 5552 if (nr[order]) 5553 show_migration_types(types[order]); 5554 } 5555 printk(KERN_CONT "= %lukB\n", K(total)); 5556 } 5557 5558 hugetlb_show_meminfo(); 5559 5560 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 5561 5562 show_swap_cache_info(); 5563 } 5564 5565 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 5566 { 5567 zoneref->zone = zone; 5568 zoneref->zone_idx = zone_idx(zone); 5569 } 5570 5571 /* 5572 * Builds allocation fallback zone lists. 5573 * 5574 * Add all populated zones of a node to the zonelist. 5575 */ 5576 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) 5577 { 5578 struct zone *zone; 5579 enum zone_type zone_type = MAX_NR_ZONES; 5580 int nr_zones = 0; 5581 5582 do { 5583 zone_type--; 5584 zone = pgdat->node_zones + zone_type; 5585 if (managed_zone(zone)) { 5586 zoneref_set_zone(zone, &zonerefs[nr_zones++]); 5587 check_highest_zone(zone_type); 5588 } 5589 } while (zone_type); 5590 5591 return nr_zones; 5592 } 5593 5594 #ifdef CONFIG_NUMA 5595 5596 static int __parse_numa_zonelist_order(char *s) 5597 { 5598 /* 5599 * We used to support different zonlists modes but they turned 5600 * out to be just not useful. Let's keep the warning in place 5601 * if somebody still use the cmd line parameter so that we do 5602 * not fail it silently 5603 */ 5604 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) { 5605 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s); 5606 return -EINVAL; 5607 } 5608 return 0; 5609 } 5610 5611 char numa_zonelist_order[] = "Node"; 5612 5613 /* 5614 * sysctl handler for numa_zonelist_order 5615 */ 5616 int numa_zonelist_order_handler(struct ctl_table *table, int write, 5617 void *buffer, size_t *length, loff_t *ppos) 5618 { 5619 if (write) 5620 return __parse_numa_zonelist_order(buffer); 5621 return proc_dostring(table, write, buffer, length, ppos); 5622 } 5623 5624 5625 #define MAX_NODE_LOAD (nr_online_nodes) 5626 static int node_load[MAX_NUMNODES]; 5627 5628 /** 5629 * find_next_best_node - find the next node that should appear in a given node's fallback list 5630 * @node: node whose fallback list we're appending 5631 * @used_node_mask: nodemask_t of already used nodes 5632 * 5633 * We use a number of factors to determine which is the next node that should 5634 * appear on a given node's fallback list. The node should not have appeared 5635 * already in @node's fallback list, and it should be the next closest node 5636 * according to the distance array (which contains arbitrary distance values 5637 * from each node to each node in the system), and should also prefer nodes 5638 * with no CPUs, since presumably they'll have very little allocation pressure 5639 * on them otherwise. 5640 * 5641 * Return: node id of the found node or %NUMA_NO_NODE if no node is found. 5642 */ 5643 static int find_next_best_node(int node, nodemask_t *used_node_mask) 5644 { 5645 int n, val; 5646 int min_val = INT_MAX; 5647 int best_node = NUMA_NO_NODE; 5648 5649 /* Use the local node if we haven't already */ 5650 if (!node_isset(node, *used_node_mask)) { 5651 node_set(node, *used_node_mask); 5652 return node; 5653 } 5654 5655 for_each_node_state(n, N_MEMORY) { 5656 5657 /* Don't want a node to appear more than once */ 5658 if (node_isset(n, *used_node_mask)) 5659 continue; 5660 5661 /* Use the distance array to find the distance */ 5662 val = node_distance(node, n); 5663 5664 /* Penalize nodes under us ("prefer the next node") */ 5665 val += (n < node); 5666 5667 /* Give preference to headless and unused nodes */ 5668 if (!cpumask_empty(cpumask_of_node(n))) 5669 val += PENALTY_FOR_NODE_WITH_CPUS; 5670 5671 /* Slight preference for less loaded node */ 5672 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 5673 val += node_load[n]; 5674 5675 if (val < min_val) { 5676 min_val = val; 5677 best_node = n; 5678 } 5679 } 5680 5681 if (best_node >= 0) 5682 node_set(best_node, *used_node_mask); 5683 5684 return best_node; 5685 } 5686 5687 5688 /* 5689 * Build zonelists ordered by node and zones within node. 5690 * This results in maximum locality--normal zone overflows into local 5691 * DMA zone, if any--but risks exhausting DMA zone. 5692 */ 5693 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, 5694 unsigned nr_nodes) 5695 { 5696 struct zoneref *zonerefs; 5697 int i; 5698 5699 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5700 5701 for (i = 0; i < nr_nodes; i++) { 5702 int nr_zones; 5703 5704 pg_data_t *node = NODE_DATA(node_order[i]); 5705 5706 nr_zones = build_zonerefs_node(node, zonerefs); 5707 zonerefs += nr_zones; 5708 } 5709 zonerefs->zone = NULL; 5710 zonerefs->zone_idx = 0; 5711 } 5712 5713 /* 5714 * Build gfp_thisnode zonelists 5715 */ 5716 static void build_thisnode_zonelists(pg_data_t *pgdat) 5717 { 5718 struct zoneref *zonerefs; 5719 int nr_zones; 5720 5721 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; 5722 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5723 zonerefs += nr_zones; 5724 zonerefs->zone = NULL; 5725 zonerefs->zone_idx = 0; 5726 } 5727 5728 /* 5729 * Build zonelists ordered by zone and nodes within zones. 5730 * This results in conserving DMA zone[s] until all Normal memory is 5731 * exhausted, but results in overflowing to remote node while memory 5732 * may still exist in local DMA zone. 5733 */ 5734 5735 static void build_zonelists(pg_data_t *pgdat) 5736 { 5737 static int node_order[MAX_NUMNODES]; 5738 int node, load, nr_nodes = 0; 5739 nodemask_t used_mask = NODE_MASK_NONE; 5740 int local_node, prev_node; 5741 5742 /* NUMA-aware ordering of nodes */ 5743 local_node = pgdat->node_id; 5744 load = nr_online_nodes; 5745 prev_node = local_node; 5746 5747 memset(node_order, 0, sizeof(node_order)); 5748 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 5749 /* 5750 * We don't want to pressure a particular node. 5751 * So adding penalty to the first node in same 5752 * distance group to make it round-robin. 5753 */ 5754 if (node_distance(local_node, node) != 5755 node_distance(local_node, prev_node)) 5756 node_load[node] = load; 5757 5758 node_order[nr_nodes++] = node; 5759 prev_node = node; 5760 load--; 5761 } 5762 5763 build_zonelists_in_node_order(pgdat, node_order, nr_nodes); 5764 build_thisnode_zonelists(pgdat); 5765 } 5766 5767 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5768 /* 5769 * Return node id of node used for "local" allocations. 5770 * I.e., first node id of first zone in arg node's generic zonelist. 5771 * Used for initializing percpu 'numa_mem', which is used primarily 5772 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 5773 */ 5774 int local_memory_node(int node) 5775 { 5776 struct zoneref *z; 5777 5778 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 5779 gfp_zone(GFP_KERNEL), 5780 NULL); 5781 return zone_to_nid(z->zone); 5782 } 5783 #endif 5784 5785 static void setup_min_unmapped_ratio(void); 5786 static void setup_min_slab_ratio(void); 5787 #else /* CONFIG_NUMA */ 5788 5789 static void build_zonelists(pg_data_t *pgdat) 5790 { 5791 int node, local_node; 5792 struct zoneref *zonerefs; 5793 int nr_zones; 5794 5795 local_node = pgdat->node_id; 5796 5797 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5798 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5799 zonerefs += nr_zones; 5800 5801 /* 5802 * Now we build the zonelist so that it contains the zones 5803 * of all the other nodes. 5804 * We don't want to pressure a particular node, so when 5805 * building the zones for node N, we make sure that the 5806 * zones coming right after the local ones are those from 5807 * node N+1 (modulo N) 5808 */ 5809 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 5810 if (!node_online(node)) 5811 continue; 5812 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5813 zonerefs += nr_zones; 5814 } 5815 for (node = 0; node < local_node; node++) { 5816 if (!node_online(node)) 5817 continue; 5818 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5819 zonerefs += nr_zones; 5820 } 5821 5822 zonerefs->zone = NULL; 5823 zonerefs->zone_idx = 0; 5824 } 5825 5826 #endif /* CONFIG_NUMA */ 5827 5828 /* 5829 * Boot pageset table. One per cpu which is going to be used for all 5830 * zones and all nodes. The parameters will be set in such a way 5831 * that an item put on a list will immediately be handed over to 5832 * the buddy list. This is safe since pageset manipulation is done 5833 * with interrupts disabled. 5834 * 5835 * The boot_pagesets must be kept even after bootup is complete for 5836 * unused processors and/or zones. They do play a role for bootstrapping 5837 * hotplugged processors. 5838 * 5839 * zoneinfo_show() and maybe other functions do 5840 * not check if the processor is online before following the pageset pointer. 5841 * Other parts of the kernel may not check if the zone is available. 5842 */ 5843 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); 5844 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); 5845 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 5846 5847 static void __build_all_zonelists(void *data) 5848 { 5849 int nid; 5850 int __maybe_unused cpu; 5851 pg_data_t *self = data; 5852 static DEFINE_SPINLOCK(lock); 5853 5854 spin_lock(&lock); 5855 5856 #ifdef CONFIG_NUMA 5857 memset(node_load, 0, sizeof(node_load)); 5858 #endif 5859 5860 /* 5861 * This node is hotadded and no memory is yet present. So just 5862 * building zonelists is fine - no need to touch other nodes. 5863 */ 5864 if (self && !node_online(self->node_id)) { 5865 build_zonelists(self); 5866 } else { 5867 for_each_online_node(nid) { 5868 pg_data_t *pgdat = NODE_DATA(nid); 5869 5870 build_zonelists(pgdat); 5871 } 5872 5873 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5874 /* 5875 * We now know the "local memory node" for each node-- 5876 * i.e., the node of the first zone in the generic zonelist. 5877 * Set up numa_mem percpu variable for on-line cpus. During 5878 * boot, only the boot cpu should be on-line; we'll init the 5879 * secondary cpus' numa_mem as they come on-line. During 5880 * node/memory hotplug, we'll fixup all on-line cpus. 5881 */ 5882 for_each_online_cpu(cpu) 5883 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 5884 #endif 5885 } 5886 5887 spin_unlock(&lock); 5888 } 5889 5890 static noinline void __init 5891 build_all_zonelists_init(void) 5892 { 5893 int cpu; 5894 5895 __build_all_zonelists(NULL); 5896 5897 /* 5898 * Initialize the boot_pagesets that are going to be used 5899 * for bootstrapping processors. The real pagesets for 5900 * each zone will be allocated later when the per cpu 5901 * allocator is available. 5902 * 5903 * boot_pagesets are used also for bootstrapping offline 5904 * cpus if the system is already booted because the pagesets 5905 * are needed to initialize allocators on a specific cpu too. 5906 * F.e. the percpu allocator needs the page allocator which 5907 * needs the percpu allocator in order to allocate its pagesets 5908 * (a chicken-egg dilemma). 5909 */ 5910 for_each_possible_cpu(cpu) 5911 setup_pageset(&per_cpu(boot_pageset, cpu), 0); 5912 5913 mminit_verify_zonelist(); 5914 cpuset_init_current_mems_allowed(); 5915 } 5916 5917 /* 5918 * unless system_state == SYSTEM_BOOTING. 5919 * 5920 * __ref due to call of __init annotated helper build_all_zonelists_init 5921 * [protected by SYSTEM_BOOTING]. 5922 */ 5923 void __ref build_all_zonelists(pg_data_t *pgdat) 5924 { 5925 unsigned long vm_total_pages; 5926 5927 if (system_state == SYSTEM_BOOTING) { 5928 build_all_zonelists_init(); 5929 } else { 5930 __build_all_zonelists(pgdat); 5931 /* cpuset refresh routine should be here */ 5932 } 5933 /* Get the number of free pages beyond high watermark in all zones. */ 5934 vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 5935 /* 5936 * Disable grouping by mobility if the number of pages in the 5937 * system is too low to allow the mechanism to work. It would be 5938 * more accurate, but expensive to check per-zone. This check is 5939 * made on memory-hotadd so a system can start with mobility 5940 * disabled and enable it later 5941 */ 5942 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 5943 page_group_by_mobility_disabled = 1; 5944 else 5945 page_group_by_mobility_disabled = 0; 5946 5947 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n", 5948 nr_online_nodes, 5949 page_group_by_mobility_disabled ? "off" : "on", 5950 vm_total_pages); 5951 #ifdef CONFIG_NUMA 5952 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 5953 #endif 5954 } 5955 5956 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 5957 static bool __meminit 5958 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 5959 { 5960 static struct memblock_region *r; 5961 5962 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 5963 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 5964 for_each_mem_region(r) { 5965 if (*pfn < memblock_region_memory_end_pfn(r)) 5966 break; 5967 } 5968 } 5969 if (*pfn >= memblock_region_memory_base_pfn(r) && 5970 memblock_is_mirror(r)) { 5971 *pfn = memblock_region_memory_end_pfn(r); 5972 return true; 5973 } 5974 } 5975 return false; 5976 } 5977 5978 /* 5979 * Initially all pages are reserved - free ones are freed 5980 * up by memblock_free_all() once the early boot process is 5981 * done. Non-atomic initialization, single-pass. 5982 */ 5983 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, 5984 unsigned long start_pfn, enum meminit_context context, 5985 struct vmem_altmap *altmap) 5986 { 5987 unsigned long pfn, end_pfn = start_pfn + size; 5988 struct page *page; 5989 5990 if (highest_memmap_pfn < end_pfn - 1) 5991 highest_memmap_pfn = end_pfn - 1; 5992 5993 #ifdef CONFIG_ZONE_DEVICE 5994 /* 5995 * Honor reservation requested by the driver for this ZONE_DEVICE 5996 * memory. We limit the total number of pages to initialize to just 5997 * those that might contain the memory mapping. We will defer the 5998 * ZONE_DEVICE page initialization until after we have released 5999 * the hotplug lock. 6000 */ 6001 if (zone == ZONE_DEVICE) { 6002 if (!altmap) 6003 return; 6004 6005 if (start_pfn == altmap->base_pfn) 6006 start_pfn += altmap->reserve; 6007 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6008 } 6009 #endif 6010 6011 for (pfn = start_pfn; pfn < end_pfn; ) { 6012 /* 6013 * There can be holes in boot-time mem_map[]s handed to this 6014 * function. They do not exist on hotplugged memory. 6015 */ 6016 if (context == MEMINIT_EARLY) { 6017 if (overlap_memmap_init(zone, &pfn)) 6018 continue; 6019 if (defer_init(nid, pfn, end_pfn)) 6020 break; 6021 } 6022 6023 page = pfn_to_page(pfn); 6024 __init_single_page(page, pfn, zone, nid); 6025 if (context == MEMINIT_HOTPLUG) 6026 __SetPageReserved(page); 6027 6028 /* 6029 * Mark the block movable so that blocks are reserved for 6030 * movable at startup. This will force kernel allocations 6031 * to reserve their blocks rather than leaking throughout 6032 * the address space during boot when many long-lived 6033 * kernel allocations are made. 6034 * 6035 * bitmap is created for zone's valid pfn range. but memmap 6036 * can be created for invalid pages (for alignment) 6037 * check here not to call set_pageblock_migratetype() against 6038 * pfn out of zone. 6039 */ 6040 if (!(pfn & (pageblock_nr_pages - 1))) { 6041 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 6042 cond_resched(); 6043 } 6044 pfn++; 6045 } 6046 } 6047 6048 #ifdef CONFIG_ZONE_DEVICE 6049 void __ref memmap_init_zone_device(struct zone *zone, 6050 unsigned long start_pfn, 6051 unsigned long nr_pages, 6052 struct dev_pagemap *pgmap) 6053 { 6054 unsigned long pfn, end_pfn = start_pfn + nr_pages; 6055 struct pglist_data *pgdat = zone->zone_pgdat; 6056 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 6057 unsigned long zone_idx = zone_idx(zone); 6058 unsigned long start = jiffies; 6059 int nid = pgdat->node_id; 6060 6061 if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE)) 6062 return; 6063 6064 /* 6065 * The call to memmap_init_zone should have already taken care 6066 * of the pages reserved for the memmap, so we can just jump to 6067 * the end of that region and start processing the device pages. 6068 */ 6069 if (altmap) { 6070 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6071 nr_pages = end_pfn - start_pfn; 6072 } 6073 6074 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 6075 struct page *page = pfn_to_page(pfn); 6076 6077 __init_single_page(page, pfn, zone_idx, nid); 6078 6079 /* 6080 * Mark page reserved as it will need to wait for onlining 6081 * phase for it to be fully associated with a zone. 6082 * 6083 * We can use the non-atomic __set_bit operation for setting 6084 * the flag as we are still initializing the pages. 6085 */ 6086 __SetPageReserved(page); 6087 6088 /* 6089 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 6090 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 6091 * ever freed or placed on a driver-private list. 6092 */ 6093 page->pgmap = pgmap; 6094 page->zone_device_data = NULL; 6095 6096 /* 6097 * Mark the block movable so that blocks are reserved for 6098 * movable at startup. This will force kernel allocations 6099 * to reserve their blocks rather than leaking throughout 6100 * the address space during boot when many long-lived 6101 * kernel allocations are made. 6102 * 6103 * bitmap is created for zone's valid pfn range. but memmap 6104 * can be created for invalid pages (for alignment) 6105 * check here not to call set_pageblock_migratetype() against 6106 * pfn out of zone. 6107 * 6108 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap 6109 * because this is done early in section_activate() 6110 */ 6111 if (!(pfn & (pageblock_nr_pages - 1))) { 6112 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 6113 cond_resched(); 6114 } 6115 } 6116 6117 pr_info("%s initialised %lu pages in %ums\n", __func__, 6118 nr_pages, jiffies_to_msecs(jiffies - start)); 6119 } 6120 6121 #endif 6122 static void __meminit zone_init_free_lists(struct zone *zone) 6123 { 6124 unsigned int order, t; 6125 for_each_migratetype_order(order, t) { 6126 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 6127 zone->free_area[order].nr_free = 0; 6128 } 6129 } 6130 6131 void __meminit __weak memmap_init(unsigned long size, int nid, 6132 unsigned long zone, 6133 unsigned long range_start_pfn) 6134 { 6135 unsigned long start_pfn, end_pfn; 6136 unsigned long range_end_pfn = range_start_pfn + size; 6137 int i; 6138 6139 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6140 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 6141 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 6142 6143 if (end_pfn > start_pfn) { 6144 size = end_pfn - start_pfn; 6145 memmap_init_zone(size, nid, zone, start_pfn, 6146 MEMINIT_EARLY, NULL); 6147 } 6148 } 6149 } 6150 6151 static int zone_batchsize(struct zone *zone) 6152 { 6153 #ifdef CONFIG_MMU 6154 int batch; 6155 6156 /* 6157 * The per-cpu-pages pools are set to around 1000th of the 6158 * size of the zone. 6159 */ 6160 batch = zone_managed_pages(zone) / 1024; 6161 /* But no more than a meg. */ 6162 if (batch * PAGE_SIZE > 1024 * 1024) 6163 batch = (1024 * 1024) / PAGE_SIZE; 6164 batch /= 4; /* We effectively *= 4 below */ 6165 if (batch < 1) 6166 batch = 1; 6167 6168 /* 6169 * Clamp the batch to a 2^n - 1 value. Having a power 6170 * of 2 value was found to be more likely to have 6171 * suboptimal cache aliasing properties in some cases. 6172 * 6173 * For example if 2 tasks are alternately allocating 6174 * batches of pages, one task can end up with a lot 6175 * of pages of one half of the possible page colors 6176 * and the other with pages of the other colors. 6177 */ 6178 batch = rounddown_pow_of_two(batch + batch/2) - 1; 6179 6180 return batch; 6181 6182 #else 6183 /* The deferral and batching of frees should be suppressed under NOMMU 6184 * conditions. 6185 * 6186 * The problem is that NOMMU needs to be able to allocate large chunks 6187 * of contiguous memory as there's no hardware page translation to 6188 * assemble apparent contiguous memory from discontiguous pages. 6189 * 6190 * Queueing large contiguous runs of pages for batching, however, 6191 * causes the pages to actually be freed in smaller chunks. As there 6192 * can be a significant delay between the individual batches being 6193 * recycled, this leads to the once large chunks of space being 6194 * fragmented and becoming unavailable for high-order allocations. 6195 */ 6196 return 0; 6197 #endif 6198 } 6199 6200 /* 6201 * pcp->high and pcp->batch values are related and dependent on one another: 6202 * ->batch must never be higher then ->high. 6203 * The following function updates them in a safe manner without read side 6204 * locking. 6205 * 6206 * Any new users of pcp->batch and pcp->high should ensure they can cope with 6207 * those fields changing asynchronously (acording to the above rule). 6208 * 6209 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 6210 * outside of boot time (or some other assurance that no concurrent updaters 6211 * exist). 6212 */ 6213 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 6214 unsigned long batch) 6215 { 6216 /* start with a fail safe value for batch */ 6217 pcp->batch = 1; 6218 smp_wmb(); 6219 6220 /* Update high, then batch, in order */ 6221 pcp->high = high; 6222 smp_wmb(); 6223 6224 pcp->batch = batch; 6225 } 6226 6227 /* a companion to pageset_set_high() */ 6228 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) 6229 { 6230 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); 6231 } 6232 6233 static void pageset_init(struct per_cpu_pageset *p) 6234 { 6235 struct per_cpu_pages *pcp; 6236 int migratetype; 6237 6238 memset(p, 0, sizeof(*p)); 6239 6240 pcp = &p->pcp; 6241 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) 6242 INIT_LIST_HEAD(&pcp->lists[migratetype]); 6243 } 6244 6245 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 6246 { 6247 pageset_init(p); 6248 pageset_set_batch(p, batch); 6249 } 6250 6251 /* 6252 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist 6253 * to the value high for the pageset p. 6254 */ 6255 static void pageset_set_high(struct per_cpu_pageset *p, 6256 unsigned long high) 6257 { 6258 unsigned long batch = max(1UL, high / 4); 6259 if ((high / 4) > (PAGE_SHIFT * 8)) 6260 batch = PAGE_SHIFT * 8; 6261 6262 pageset_update(&p->pcp, high, batch); 6263 } 6264 6265 static void pageset_set_high_and_batch(struct zone *zone, 6266 struct per_cpu_pageset *pcp) 6267 { 6268 if (percpu_pagelist_fraction) 6269 pageset_set_high(pcp, 6270 (zone_managed_pages(zone) / 6271 percpu_pagelist_fraction)); 6272 else 6273 pageset_set_batch(pcp, zone_batchsize(zone)); 6274 } 6275 6276 static void __meminit zone_pageset_init(struct zone *zone, int cpu) 6277 { 6278 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); 6279 6280 pageset_init(pcp); 6281 pageset_set_high_and_batch(zone, pcp); 6282 } 6283 6284 void __meminit setup_zone_pageset(struct zone *zone) 6285 { 6286 int cpu; 6287 zone->pageset = alloc_percpu(struct per_cpu_pageset); 6288 for_each_possible_cpu(cpu) 6289 zone_pageset_init(zone, cpu); 6290 } 6291 6292 /* 6293 * Allocate per cpu pagesets and initialize them. 6294 * Before this call only boot pagesets were available. 6295 */ 6296 void __init setup_per_cpu_pageset(void) 6297 { 6298 struct pglist_data *pgdat; 6299 struct zone *zone; 6300 int __maybe_unused cpu; 6301 6302 for_each_populated_zone(zone) 6303 setup_zone_pageset(zone); 6304 6305 #ifdef CONFIG_NUMA 6306 /* 6307 * Unpopulated zones continue using the boot pagesets. 6308 * The numa stats for these pagesets need to be reset. 6309 * Otherwise, they will end up skewing the stats of 6310 * the nodes these zones are associated with. 6311 */ 6312 for_each_possible_cpu(cpu) { 6313 struct per_cpu_pageset *pcp = &per_cpu(boot_pageset, cpu); 6314 memset(pcp->vm_numa_stat_diff, 0, 6315 sizeof(pcp->vm_numa_stat_diff)); 6316 } 6317 #endif 6318 6319 for_each_online_pgdat(pgdat) 6320 pgdat->per_cpu_nodestats = 6321 alloc_percpu(struct per_cpu_nodestat); 6322 } 6323 6324 static __meminit void zone_pcp_init(struct zone *zone) 6325 { 6326 /* 6327 * per cpu subsystem is not up at this point. The following code 6328 * relies on the ability of the linker to provide the 6329 * offset of a (static) per cpu variable into the per cpu area. 6330 */ 6331 zone->pageset = &boot_pageset; 6332 6333 if (populated_zone(zone)) 6334 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", 6335 zone->name, zone->present_pages, 6336 zone_batchsize(zone)); 6337 } 6338 6339 void __meminit init_currently_empty_zone(struct zone *zone, 6340 unsigned long zone_start_pfn, 6341 unsigned long size) 6342 { 6343 struct pglist_data *pgdat = zone->zone_pgdat; 6344 int zone_idx = zone_idx(zone) + 1; 6345 6346 if (zone_idx > pgdat->nr_zones) 6347 pgdat->nr_zones = zone_idx; 6348 6349 zone->zone_start_pfn = zone_start_pfn; 6350 6351 mminit_dprintk(MMINIT_TRACE, "memmap_init", 6352 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 6353 pgdat->node_id, 6354 (unsigned long)zone_idx(zone), 6355 zone_start_pfn, (zone_start_pfn + size)); 6356 6357 zone_init_free_lists(zone); 6358 zone->initialized = 1; 6359 } 6360 6361 /** 6362 * get_pfn_range_for_nid - Return the start and end page frames for a node 6363 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 6364 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 6365 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 6366 * 6367 * It returns the start and end page frame of a node based on information 6368 * provided by memblock_set_node(). If called for a node 6369 * with no available memory, a warning is printed and the start and end 6370 * PFNs will be 0. 6371 */ 6372 void __init get_pfn_range_for_nid(unsigned int nid, 6373 unsigned long *start_pfn, unsigned long *end_pfn) 6374 { 6375 unsigned long this_start_pfn, this_end_pfn; 6376 int i; 6377 6378 *start_pfn = -1UL; 6379 *end_pfn = 0; 6380 6381 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 6382 *start_pfn = min(*start_pfn, this_start_pfn); 6383 *end_pfn = max(*end_pfn, this_end_pfn); 6384 } 6385 6386 if (*start_pfn == -1UL) 6387 *start_pfn = 0; 6388 } 6389 6390 /* 6391 * This finds a zone that can be used for ZONE_MOVABLE pages. The 6392 * assumption is made that zones within a node are ordered in monotonic 6393 * increasing memory addresses so that the "highest" populated zone is used 6394 */ 6395 static void __init find_usable_zone_for_movable(void) 6396 { 6397 int zone_index; 6398 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 6399 if (zone_index == ZONE_MOVABLE) 6400 continue; 6401 6402 if (arch_zone_highest_possible_pfn[zone_index] > 6403 arch_zone_lowest_possible_pfn[zone_index]) 6404 break; 6405 } 6406 6407 VM_BUG_ON(zone_index == -1); 6408 movable_zone = zone_index; 6409 } 6410 6411 /* 6412 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 6413 * because it is sized independent of architecture. Unlike the other zones, 6414 * the starting point for ZONE_MOVABLE is not fixed. It may be different 6415 * in each node depending on the size of each node and how evenly kernelcore 6416 * is distributed. This helper function adjusts the zone ranges 6417 * provided by the architecture for a given node by using the end of the 6418 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 6419 * zones within a node are in order of monotonic increases memory addresses 6420 */ 6421 static void __init adjust_zone_range_for_zone_movable(int nid, 6422 unsigned long zone_type, 6423 unsigned long node_start_pfn, 6424 unsigned long node_end_pfn, 6425 unsigned long *zone_start_pfn, 6426 unsigned long *zone_end_pfn) 6427 { 6428 /* Only adjust if ZONE_MOVABLE is on this node */ 6429 if (zone_movable_pfn[nid]) { 6430 /* Size ZONE_MOVABLE */ 6431 if (zone_type == ZONE_MOVABLE) { 6432 *zone_start_pfn = zone_movable_pfn[nid]; 6433 *zone_end_pfn = min(node_end_pfn, 6434 arch_zone_highest_possible_pfn[movable_zone]); 6435 6436 /* Adjust for ZONE_MOVABLE starting within this range */ 6437 } else if (!mirrored_kernelcore && 6438 *zone_start_pfn < zone_movable_pfn[nid] && 6439 *zone_end_pfn > zone_movable_pfn[nid]) { 6440 *zone_end_pfn = zone_movable_pfn[nid]; 6441 6442 /* Check if this whole range is within ZONE_MOVABLE */ 6443 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 6444 *zone_start_pfn = *zone_end_pfn; 6445 } 6446 } 6447 6448 /* 6449 * Return the number of pages a zone spans in a node, including holes 6450 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 6451 */ 6452 static unsigned long __init zone_spanned_pages_in_node(int nid, 6453 unsigned long zone_type, 6454 unsigned long node_start_pfn, 6455 unsigned long node_end_pfn, 6456 unsigned long *zone_start_pfn, 6457 unsigned long *zone_end_pfn) 6458 { 6459 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 6460 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 6461 /* When hotadd a new node from cpu_up(), the node should be empty */ 6462 if (!node_start_pfn && !node_end_pfn) 6463 return 0; 6464 6465 /* Get the start and end of the zone */ 6466 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 6467 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 6468 adjust_zone_range_for_zone_movable(nid, zone_type, 6469 node_start_pfn, node_end_pfn, 6470 zone_start_pfn, zone_end_pfn); 6471 6472 /* Check that this node has pages within the zone's required range */ 6473 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 6474 return 0; 6475 6476 /* Move the zone boundaries inside the node if necessary */ 6477 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 6478 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 6479 6480 /* Return the spanned pages */ 6481 return *zone_end_pfn - *zone_start_pfn; 6482 } 6483 6484 /* 6485 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 6486 * then all holes in the requested range will be accounted for. 6487 */ 6488 unsigned long __init __absent_pages_in_range(int nid, 6489 unsigned long range_start_pfn, 6490 unsigned long range_end_pfn) 6491 { 6492 unsigned long nr_absent = range_end_pfn - range_start_pfn; 6493 unsigned long start_pfn, end_pfn; 6494 int i; 6495 6496 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6497 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 6498 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 6499 nr_absent -= end_pfn - start_pfn; 6500 } 6501 return nr_absent; 6502 } 6503 6504 /** 6505 * absent_pages_in_range - Return number of page frames in holes within a range 6506 * @start_pfn: The start PFN to start searching for holes 6507 * @end_pfn: The end PFN to stop searching for holes 6508 * 6509 * Return: the number of pages frames in memory holes within a range. 6510 */ 6511 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 6512 unsigned long end_pfn) 6513 { 6514 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 6515 } 6516 6517 /* Return the number of page frames in holes in a zone on a node */ 6518 static unsigned long __init zone_absent_pages_in_node(int nid, 6519 unsigned long zone_type, 6520 unsigned long node_start_pfn, 6521 unsigned long node_end_pfn) 6522 { 6523 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 6524 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 6525 unsigned long zone_start_pfn, zone_end_pfn; 6526 unsigned long nr_absent; 6527 6528 /* When hotadd a new node from cpu_up(), the node should be empty */ 6529 if (!node_start_pfn && !node_end_pfn) 6530 return 0; 6531 6532 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 6533 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 6534 6535 adjust_zone_range_for_zone_movable(nid, zone_type, 6536 node_start_pfn, node_end_pfn, 6537 &zone_start_pfn, &zone_end_pfn); 6538 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 6539 6540 /* 6541 * ZONE_MOVABLE handling. 6542 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 6543 * and vice versa. 6544 */ 6545 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 6546 unsigned long start_pfn, end_pfn; 6547 struct memblock_region *r; 6548 6549 for_each_mem_region(r) { 6550 start_pfn = clamp(memblock_region_memory_base_pfn(r), 6551 zone_start_pfn, zone_end_pfn); 6552 end_pfn = clamp(memblock_region_memory_end_pfn(r), 6553 zone_start_pfn, zone_end_pfn); 6554 6555 if (zone_type == ZONE_MOVABLE && 6556 memblock_is_mirror(r)) 6557 nr_absent += end_pfn - start_pfn; 6558 6559 if (zone_type == ZONE_NORMAL && 6560 !memblock_is_mirror(r)) 6561 nr_absent += end_pfn - start_pfn; 6562 } 6563 } 6564 6565 return nr_absent; 6566 } 6567 6568 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 6569 unsigned long node_start_pfn, 6570 unsigned long node_end_pfn) 6571 { 6572 unsigned long realtotalpages = 0, totalpages = 0; 6573 enum zone_type i; 6574 6575 for (i = 0; i < MAX_NR_ZONES; i++) { 6576 struct zone *zone = pgdat->node_zones + i; 6577 unsigned long zone_start_pfn, zone_end_pfn; 6578 unsigned long spanned, absent; 6579 unsigned long size, real_size; 6580 6581 spanned = zone_spanned_pages_in_node(pgdat->node_id, i, 6582 node_start_pfn, 6583 node_end_pfn, 6584 &zone_start_pfn, 6585 &zone_end_pfn); 6586 absent = zone_absent_pages_in_node(pgdat->node_id, i, 6587 node_start_pfn, 6588 node_end_pfn); 6589 6590 size = spanned; 6591 real_size = size - absent; 6592 6593 if (size) 6594 zone->zone_start_pfn = zone_start_pfn; 6595 else 6596 zone->zone_start_pfn = 0; 6597 zone->spanned_pages = size; 6598 zone->present_pages = real_size; 6599 6600 totalpages += size; 6601 realtotalpages += real_size; 6602 } 6603 6604 pgdat->node_spanned_pages = totalpages; 6605 pgdat->node_present_pages = realtotalpages; 6606 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, 6607 realtotalpages); 6608 } 6609 6610 #ifndef CONFIG_SPARSEMEM 6611 /* 6612 * Calculate the size of the zone->blockflags rounded to an unsigned long 6613 * Start by making sure zonesize is a multiple of pageblock_order by rounding 6614 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 6615 * round what is now in bits to nearest long in bits, then return it in 6616 * bytes. 6617 */ 6618 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 6619 { 6620 unsigned long usemapsize; 6621 6622 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 6623 usemapsize = roundup(zonesize, pageblock_nr_pages); 6624 usemapsize = usemapsize >> pageblock_order; 6625 usemapsize *= NR_PAGEBLOCK_BITS; 6626 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 6627 6628 return usemapsize / 8; 6629 } 6630 6631 static void __ref setup_usemap(struct pglist_data *pgdat, 6632 struct zone *zone, 6633 unsigned long zone_start_pfn, 6634 unsigned long zonesize) 6635 { 6636 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); 6637 zone->pageblock_flags = NULL; 6638 if (usemapsize) { 6639 zone->pageblock_flags = 6640 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 6641 pgdat->node_id); 6642 if (!zone->pageblock_flags) 6643 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 6644 usemapsize, zone->name, pgdat->node_id); 6645 } 6646 } 6647 #else 6648 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, 6649 unsigned long zone_start_pfn, unsigned long zonesize) {} 6650 #endif /* CONFIG_SPARSEMEM */ 6651 6652 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 6653 6654 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 6655 void __init set_pageblock_order(void) 6656 { 6657 unsigned int order; 6658 6659 /* Check that pageblock_nr_pages has not already been setup */ 6660 if (pageblock_order) 6661 return; 6662 6663 if (HPAGE_SHIFT > PAGE_SHIFT) 6664 order = HUGETLB_PAGE_ORDER; 6665 else 6666 order = MAX_ORDER - 1; 6667 6668 /* 6669 * Assume the largest contiguous order of interest is a huge page. 6670 * This value may be variable depending on boot parameters on IA64 and 6671 * powerpc. 6672 */ 6673 pageblock_order = order; 6674 } 6675 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6676 6677 /* 6678 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 6679 * is unused as pageblock_order is set at compile-time. See 6680 * include/linux/pageblock-flags.h for the values of pageblock_order based on 6681 * the kernel config 6682 */ 6683 void __init set_pageblock_order(void) 6684 { 6685 } 6686 6687 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6688 6689 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 6690 unsigned long present_pages) 6691 { 6692 unsigned long pages = spanned_pages; 6693 6694 /* 6695 * Provide a more accurate estimation if there are holes within 6696 * the zone and SPARSEMEM is in use. If there are holes within the 6697 * zone, each populated memory region may cost us one or two extra 6698 * memmap pages due to alignment because memmap pages for each 6699 * populated regions may not be naturally aligned on page boundary. 6700 * So the (present_pages >> 4) heuristic is a tradeoff for that. 6701 */ 6702 if (spanned_pages > present_pages + (present_pages >> 4) && 6703 IS_ENABLED(CONFIG_SPARSEMEM)) 6704 pages = present_pages; 6705 6706 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 6707 } 6708 6709 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 6710 static void pgdat_init_split_queue(struct pglist_data *pgdat) 6711 { 6712 struct deferred_split *ds_queue = &pgdat->deferred_split_queue; 6713 6714 spin_lock_init(&ds_queue->split_queue_lock); 6715 INIT_LIST_HEAD(&ds_queue->split_queue); 6716 ds_queue->split_queue_len = 0; 6717 } 6718 #else 6719 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 6720 #endif 6721 6722 #ifdef CONFIG_COMPACTION 6723 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 6724 { 6725 init_waitqueue_head(&pgdat->kcompactd_wait); 6726 } 6727 #else 6728 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 6729 #endif 6730 6731 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 6732 { 6733 pgdat_resize_init(pgdat); 6734 6735 pgdat_init_split_queue(pgdat); 6736 pgdat_init_kcompactd(pgdat); 6737 6738 init_waitqueue_head(&pgdat->kswapd_wait); 6739 init_waitqueue_head(&pgdat->pfmemalloc_wait); 6740 6741 pgdat_page_ext_init(pgdat); 6742 spin_lock_init(&pgdat->lru_lock); 6743 lruvec_init(&pgdat->__lruvec); 6744 } 6745 6746 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 6747 unsigned long remaining_pages) 6748 { 6749 atomic_long_set(&zone->managed_pages, remaining_pages); 6750 zone_set_nid(zone, nid); 6751 zone->name = zone_names[idx]; 6752 zone->zone_pgdat = NODE_DATA(nid); 6753 spin_lock_init(&zone->lock); 6754 zone_seqlock_init(zone); 6755 zone_pcp_init(zone); 6756 } 6757 6758 /* 6759 * Set up the zone data structures 6760 * - init pgdat internals 6761 * - init all zones belonging to this node 6762 * 6763 * NOTE: this function is only called during memory hotplug 6764 */ 6765 #ifdef CONFIG_MEMORY_HOTPLUG 6766 void __ref free_area_init_core_hotplug(int nid) 6767 { 6768 enum zone_type z; 6769 pg_data_t *pgdat = NODE_DATA(nid); 6770 6771 pgdat_init_internals(pgdat); 6772 for (z = 0; z < MAX_NR_ZONES; z++) 6773 zone_init_internals(&pgdat->node_zones[z], z, nid, 0); 6774 } 6775 #endif 6776 6777 /* 6778 * Set up the zone data structures: 6779 * - mark all pages reserved 6780 * - mark all memory queues empty 6781 * - clear the memory bitmaps 6782 * 6783 * NOTE: pgdat should get zeroed by caller. 6784 * NOTE: this function is only called during early init. 6785 */ 6786 static void __init free_area_init_core(struct pglist_data *pgdat) 6787 { 6788 enum zone_type j; 6789 int nid = pgdat->node_id; 6790 6791 pgdat_init_internals(pgdat); 6792 pgdat->per_cpu_nodestats = &boot_nodestats; 6793 6794 for (j = 0; j < MAX_NR_ZONES; j++) { 6795 struct zone *zone = pgdat->node_zones + j; 6796 unsigned long size, freesize, memmap_pages; 6797 unsigned long zone_start_pfn = zone->zone_start_pfn; 6798 6799 size = zone->spanned_pages; 6800 freesize = zone->present_pages; 6801 6802 /* 6803 * Adjust freesize so that it accounts for how much memory 6804 * is used by this zone for memmap. This affects the watermark 6805 * and per-cpu initialisations 6806 */ 6807 memmap_pages = calc_memmap_size(size, freesize); 6808 if (!is_highmem_idx(j)) { 6809 if (freesize >= memmap_pages) { 6810 freesize -= memmap_pages; 6811 if (memmap_pages) 6812 printk(KERN_DEBUG 6813 " %s zone: %lu pages used for memmap\n", 6814 zone_names[j], memmap_pages); 6815 } else 6816 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n", 6817 zone_names[j], memmap_pages, freesize); 6818 } 6819 6820 /* Account for reserved pages */ 6821 if (j == 0 && freesize > dma_reserve) { 6822 freesize -= dma_reserve; 6823 printk(KERN_DEBUG " %s zone: %lu pages reserved\n", 6824 zone_names[0], dma_reserve); 6825 } 6826 6827 if (!is_highmem_idx(j)) 6828 nr_kernel_pages += freesize; 6829 /* Charge for highmem memmap if there are enough kernel pages */ 6830 else if (nr_kernel_pages > memmap_pages * 2) 6831 nr_kernel_pages -= memmap_pages; 6832 nr_all_pages += freesize; 6833 6834 /* 6835 * Set an approximate value for lowmem here, it will be adjusted 6836 * when the bootmem allocator frees pages into the buddy system. 6837 * And all highmem pages will be managed by the buddy system. 6838 */ 6839 zone_init_internals(zone, j, nid, freesize); 6840 6841 if (!size) 6842 continue; 6843 6844 set_pageblock_order(); 6845 setup_usemap(pgdat, zone, zone_start_pfn, size); 6846 init_currently_empty_zone(zone, zone_start_pfn, size); 6847 memmap_init(size, nid, j, zone_start_pfn); 6848 } 6849 } 6850 6851 #ifdef CONFIG_FLAT_NODE_MEM_MAP 6852 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) 6853 { 6854 unsigned long __maybe_unused start = 0; 6855 unsigned long __maybe_unused offset = 0; 6856 6857 /* Skip empty nodes */ 6858 if (!pgdat->node_spanned_pages) 6859 return; 6860 6861 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 6862 offset = pgdat->node_start_pfn - start; 6863 /* ia64 gets its own node_mem_map, before this, without bootmem */ 6864 if (!pgdat->node_mem_map) { 6865 unsigned long size, end; 6866 struct page *map; 6867 6868 /* 6869 * The zone's endpoints aren't required to be MAX_ORDER 6870 * aligned but the node_mem_map endpoints must be in order 6871 * for the buddy allocator to function correctly. 6872 */ 6873 end = pgdat_end_pfn(pgdat); 6874 end = ALIGN(end, MAX_ORDER_NR_PAGES); 6875 size = (end - start) * sizeof(struct page); 6876 map = memblock_alloc_node(size, SMP_CACHE_BYTES, 6877 pgdat->node_id); 6878 if (!map) 6879 panic("Failed to allocate %ld bytes for node %d memory map\n", 6880 size, pgdat->node_id); 6881 pgdat->node_mem_map = map + offset; 6882 } 6883 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 6884 __func__, pgdat->node_id, (unsigned long)pgdat, 6885 (unsigned long)pgdat->node_mem_map); 6886 #ifndef CONFIG_NEED_MULTIPLE_NODES 6887 /* 6888 * With no DISCONTIG, the global mem_map is just set as node 0's 6889 */ 6890 if (pgdat == NODE_DATA(0)) { 6891 mem_map = NODE_DATA(0)->node_mem_map; 6892 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 6893 mem_map -= offset; 6894 } 6895 #endif 6896 } 6897 #else 6898 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { } 6899 #endif /* CONFIG_FLAT_NODE_MEM_MAP */ 6900 6901 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 6902 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 6903 { 6904 pgdat->first_deferred_pfn = ULONG_MAX; 6905 } 6906 #else 6907 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 6908 #endif 6909 6910 static void __init free_area_init_node(int nid) 6911 { 6912 pg_data_t *pgdat = NODE_DATA(nid); 6913 unsigned long start_pfn = 0; 6914 unsigned long end_pfn = 0; 6915 6916 /* pg_data_t should be reset to zero when it's allocated */ 6917 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); 6918 6919 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 6920 6921 pgdat->node_id = nid; 6922 pgdat->node_start_pfn = start_pfn; 6923 pgdat->per_cpu_nodestats = NULL; 6924 6925 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 6926 (u64)start_pfn << PAGE_SHIFT, 6927 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 6928 calculate_node_totalpages(pgdat, start_pfn, end_pfn); 6929 6930 alloc_node_mem_map(pgdat); 6931 pgdat_set_deferred_range(pgdat); 6932 6933 free_area_init_core(pgdat); 6934 } 6935 6936 void __init free_area_init_memoryless_node(int nid) 6937 { 6938 free_area_init_node(nid); 6939 } 6940 6941 #if !defined(CONFIG_FLAT_NODE_MEM_MAP) 6942 /* 6943 * Initialize all valid struct pages in the range [spfn, epfn) and mark them 6944 * PageReserved(). Return the number of struct pages that were initialized. 6945 */ 6946 static u64 __init init_unavailable_range(unsigned long spfn, unsigned long epfn) 6947 { 6948 unsigned long pfn; 6949 u64 pgcnt = 0; 6950 6951 for (pfn = spfn; pfn < epfn; pfn++) { 6952 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) { 6953 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages) 6954 + pageblock_nr_pages - 1; 6955 continue; 6956 } 6957 /* 6958 * Use a fake node/zone (0) for now. Some of these pages 6959 * (in memblock.reserved but not in memblock.memory) will 6960 * get re-initialized via reserve_bootmem_region() later. 6961 */ 6962 __init_single_page(pfn_to_page(pfn), pfn, 0, 0); 6963 __SetPageReserved(pfn_to_page(pfn)); 6964 pgcnt++; 6965 } 6966 6967 return pgcnt; 6968 } 6969 6970 /* 6971 * Only struct pages that are backed by physical memory are zeroed and 6972 * initialized by going through __init_single_page(). But, there are some 6973 * struct pages which are reserved in memblock allocator and their fields 6974 * may be accessed (for example page_to_pfn() on some configuration accesses 6975 * flags). We must explicitly initialize those struct pages. 6976 * 6977 * This function also addresses a similar issue where struct pages are left 6978 * uninitialized because the physical address range is not covered by 6979 * memblock.memory or memblock.reserved. That could happen when memblock 6980 * layout is manually configured via memmap=, or when the highest physical 6981 * address (max_pfn) does not end on a section boundary. 6982 */ 6983 static void __init init_unavailable_mem(void) 6984 { 6985 phys_addr_t start, end; 6986 u64 i, pgcnt; 6987 phys_addr_t next = 0; 6988 6989 /* 6990 * Loop through unavailable ranges not covered by memblock.memory. 6991 */ 6992 pgcnt = 0; 6993 for_each_mem_range(i, &start, &end) { 6994 if (next < start) 6995 pgcnt += init_unavailable_range(PFN_DOWN(next), 6996 PFN_UP(start)); 6997 next = end; 6998 } 6999 7000 /* 7001 * Early sections always have a fully populated memmap for the whole 7002 * section - see pfn_valid(). If the last section has holes at the 7003 * end and that section is marked "online", the memmap will be 7004 * considered initialized. Make sure that memmap has a well defined 7005 * state. 7006 */ 7007 pgcnt += init_unavailable_range(PFN_DOWN(next), 7008 round_up(max_pfn, PAGES_PER_SECTION)); 7009 7010 /* 7011 * Struct pages that do not have backing memory. This could be because 7012 * firmware is using some of this memory, or for some other reasons. 7013 */ 7014 if (pgcnt) 7015 pr_info("Zeroed struct page in unavailable ranges: %lld pages", pgcnt); 7016 } 7017 #else 7018 static inline void __init init_unavailable_mem(void) 7019 { 7020 } 7021 #endif /* !CONFIG_FLAT_NODE_MEM_MAP */ 7022 7023 #if MAX_NUMNODES > 1 7024 /* 7025 * Figure out the number of possible node ids. 7026 */ 7027 void __init setup_nr_node_ids(void) 7028 { 7029 unsigned int highest; 7030 7031 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 7032 nr_node_ids = highest + 1; 7033 } 7034 #endif 7035 7036 /** 7037 * node_map_pfn_alignment - determine the maximum internode alignment 7038 * 7039 * This function should be called after node map is populated and sorted. 7040 * It calculates the maximum power of two alignment which can distinguish 7041 * all the nodes. 7042 * 7043 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 7044 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 7045 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 7046 * shifted, 1GiB is enough and this function will indicate so. 7047 * 7048 * This is used to test whether pfn -> nid mapping of the chosen memory 7049 * model has fine enough granularity to avoid incorrect mapping for the 7050 * populated node map. 7051 * 7052 * Return: the determined alignment in pfn's. 0 if there is no alignment 7053 * requirement (single node). 7054 */ 7055 unsigned long __init node_map_pfn_alignment(void) 7056 { 7057 unsigned long accl_mask = 0, last_end = 0; 7058 unsigned long start, end, mask; 7059 int last_nid = NUMA_NO_NODE; 7060 int i, nid; 7061 7062 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 7063 if (!start || last_nid < 0 || last_nid == nid) { 7064 last_nid = nid; 7065 last_end = end; 7066 continue; 7067 } 7068 7069 /* 7070 * Start with a mask granular enough to pin-point to the 7071 * start pfn and tick off bits one-by-one until it becomes 7072 * too coarse to separate the current node from the last. 7073 */ 7074 mask = ~((1 << __ffs(start)) - 1); 7075 while (mask && last_end <= (start & (mask << 1))) 7076 mask <<= 1; 7077 7078 /* accumulate all internode masks */ 7079 accl_mask |= mask; 7080 } 7081 7082 /* convert mask to number of pages */ 7083 return ~accl_mask + 1; 7084 } 7085 7086 /** 7087 * find_min_pfn_with_active_regions - Find the minimum PFN registered 7088 * 7089 * Return: the minimum PFN based on information provided via 7090 * memblock_set_node(). 7091 */ 7092 unsigned long __init find_min_pfn_with_active_regions(void) 7093 { 7094 return PHYS_PFN(memblock_start_of_DRAM()); 7095 } 7096 7097 /* 7098 * early_calculate_totalpages() 7099 * Sum pages in active regions for movable zone. 7100 * Populate N_MEMORY for calculating usable_nodes. 7101 */ 7102 static unsigned long __init early_calculate_totalpages(void) 7103 { 7104 unsigned long totalpages = 0; 7105 unsigned long start_pfn, end_pfn; 7106 int i, nid; 7107 7108 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7109 unsigned long pages = end_pfn - start_pfn; 7110 7111 totalpages += pages; 7112 if (pages) 7113 node_set_state(nid, N_MEMORY); 7114 } 7115 return totalpages; 7116 } 7117 7118 /* 7119 * Find the PFN the Movable zone begins in each node. Kernel memory 7120 * is spread evenly between nodes as long as the nodes have enough 7121 * memory. When they don't, some nodes will have more kernelcore than 7122 * others 7123 */ 7124 static void __init find_zone_movable_pfns_for_nodes(void) 7125 { 7126 int i, nid; 7127 unsigned long usable_startpfn; 7128 unsigned long kernelcore_node, kernelcore_remaining; 7129 /* save the state before borrow the nodemask */ 7130 nodemask_t saved_node_state = node_states[N_MEMORY]; 7131 unsigned long totalpages = early_calculate_totalpages(); 7132 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 7133 struct memblock_region *r; 7134 7135 /* Need to find movable_zone earlier when movable_node is specified. */ 7136 find_usable_zone_for_movable(); 7137 7138 /* 7139 * If movable_node is specified, ignore kernelcore and movablecore 7140 * options. 7141 */ 7142 if (movable_node_is_enabled()) { 7143 for_each_mem_region(r) { 7144 if (!memblock_is_hotpluggable(r)) 7145 continue; 7146 7147 nid = memblock_get_region_node(r); 7148 7149 usable_startpfn = PFN_DOWN(r->base); 7150 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7151 min(usable_startpfn, zone_movable_pfn[nid]) : 7152 usable_startpfn; 7153 } 7154 7155 goto out2; 7156 } 7157 7158 /* 7159 * If kernelcore=mirror is specified, ignore movablecore option 7160 */ 7161 if (mirrored_kernelcore) { 7162 bool mem_below_4gb_not_mirrored = false; 7163 7164 for_each_mem_region(r) { 7165 if (memblock_is_mirror(r)) 7166 continue; 7167 7168 nid = memblock_get_region_node(r); 7169 7170 usable_startpfn = memblock_region_memory_base_pfn(r); 7171 7172 if (usable_startpfn < 0x100000) { 7173 mem_below_4gb_not_mirrored = true; 7174 continue; 7175 } 7176 7177 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7178 min(usable_startpfn, zone_movable_pfn[nid]) : 7179 usable_startpfn; 7180 } 7181 7182 if (mem_below_4gb_not_mirrored) 7183 pr_warn("This configuration results in unmirrored kernel memory.\n"); 7184 7185 goto out2; 7186 } 7187 7188 /* 7189 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 7190 * amount of necessary memory. 7191 */ 7192 if (required_kernelcore_percent) 7193 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 7194 10000UL; 7195 if (required_movablecore_percent) 7196 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 7197 10000UL; 7198 7199 /* 7200 * If movablecore= was specified, calculate what size of 7201 * kernelcore that corresponds so that memory usable for 7202 * any allocation type is evenly spread. If both kernelcore 7203 * and movablecore are specified, then the value of kernelcore 7204 * will be used for required_kernelcore if it's greater than 7205 * what movablecore would have allowed. 7206 */ 7207 if (required_movablecore) { 7208 unsigned long corepages; 7209 7210 /* 7211 * Round-up so that ZONE_MOVABLE is at least as large as what 7212 * was requested by the user 7213 */ 7214 required_movablecore = 7215 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 7216 required_movablecore = min(totalpages, required_movablecore); 7217 corepages = totalpages - required_movablecore; 7218 7219 required_kernelcore = max(required_kernelcore, corepages); 7220 } 7221 7222 /* 7223 * If kernelcore was not specified or kernelcore size is larger 7224 * than totalpages, there is no ZONE_MOVABLE. 7225 */ 7226 if (!required_kernelcore || required_kernelcore >= totalpages) 7227 goto out; 7228 7229 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 7230 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 7231 7232 restart: 7233 /* Spread kernelcore memory as evenly as possible throughout nodes */ 7234 kernelcore_node = required_kernelcore / usable_nodes; 7235 for_each_node_state(nid, N_MEMORY) { 7236 unsigned long start_pfn, end_pfn; 7237 7238 /* 7239 * Recalculate kernelcore_node if the division per node 7240 * now exceeds what is necessary to satisfy the requested 7241 * amount of memory for the kernel 7242 */ 7243 if (required_kernelcore < kernelcore_node) 7244 kernelcore_node = required_kernelcore / usable_nodes; 7245 7246 /* 7247 * As the map is walked, we track how much memory is usable 7248 * by the kernel using kernelcore_remaining. When it is 7249 * 0, the rest of the node is usable by ZONE_MOVABLE 7250 */ 7251 kernelcore_remaining = kernelcore_node; 7252 7253 /* Go through each range of PFNs within this node */ 7254 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7255 unsigned long size_pages; 7256 7257 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 7258 if (start_pfn >= end_pfn) 7259 continue; 7260 7261 /* Account for what is only usable for kernelcore */ 7262 if (start_pfn < usable_startpfn) { 7263 unsigned long kernel_pages; 7264 kernel_pages = min(end_pfn, usable_startpfn) 7265 - start_pfn; 7266 7267 kernelcore_remaining -= min(kernel_pages, 7268 kernelcore_remaining); 7269 required_kernelcore -= min(kernel_pages, 7270 required_kernelcore); 7271 7272 /* Continue if range is now fully accounted */ 7273 if (end_pfn <= usable_startpfn) { 7274 7275 /* 7276 * Push zone_movable_pfn to the end so 7277 * that if we have to rebalance 7278 * kernelcore across nodes, we will 7279 * not double account here 7280 */ 7281 zone_movable_pfn[nid] = end_pfn; 7282 continue; 7283 } 7284 start_pfn = usable_startpfn; 7285 } 7286 7287 /* 7288 * The usable PFN range for ZONE_MOVABLE is from 7289 * start_pfn->end_pfn. Calculate size_pages as the 7290 * number of pages used as kernelcore 7291 */ 7292 size_pages = end_pfn - start_pfn; 7293 if (size_pages > kernelcore_remaining) 7294 size_pages = kernelcore_remaining; 7295 zone_movable_pfn[nid] = start_pfn + size_pages; 7296 7297 /* 7298 * Some kernelcore has been met, update counts and 7299 * break if the kernelcore for this node has been 7300 * satisfied 7301 */ 7302 required_kernelcore -= min(required_kernelcore, 7303 size_pages); 7304 kernelcore_remaining -= size_pages; 7305 if (!kernelcore_remaining) 7306 break; 7307 } 7308 } 7309 7310 /* 7311 * If there is still required_kernelcore, we do another pass with one 7312 * less node in the count. This will push zone_movable_pfn[nid] further 7313 * along on the nodes that still have memory until kernelcore is 7314 * satisfied 7315 */ 7316 usable_nodes--; 7317 if (usable_nodes && required_kernelcore > usable_nodes) 7318 goto restart; 7319 7320 out2: 7321 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 7322 for (nid = 0; nid < MAX_NUMNODES; nid++) 7323 zone_movable_pfn[nid] = 7324 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 7325 7326 out: 7327 /* restore the node_state */ 7328 node_states[N_MEMORY] = saved_node_state; 7329 } 7330 7331 /* Any regular or high memory on that node ? */ 7332 static void check_for_memory(pg_data_t *pgdat, int nid) 7333 { 7334 enum zone_type zone_type; 7335 7336 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 7337 struct zone *zone = &pgdat->node_zones[zone_type]; 7338 if (populated_zone(zone)) { 7339 if (IS_ENABLED(CONFIG_HIGHMEM)) 7340 node_set_state(nid, N_HIGH_MEMORY); 7341 if (zone_type <= ZONE_NORMAL) 7342 node_set_state(nid, N_NORMAL_MEMORY); 7343 break; 7344 } 7345 } 7346 } 7347 7348 /* 7349 * Some architecturs, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For 7350 * such cases we allow max_zone_pfn sorted in the descending order 7351 */ 7352 bool __weak arch_has_descending_max_zone_pfns(void) 7353 { 7354 return false; 7355 } 7356 7357 /** 7358 * free_area_init - Initialise all pg_data_t and zone data 7359 * @max_zone_pfn: an array of max PFNs for each zone 7360 * 7361 * This will call free_area_init_node() for each active node in the system. 7362 * Using the page ranges provided by memblock_set_node(), the size of each 7363 * zone in each node and their holes is calculated. If the maximum PFN 7364 * between two adjacent zones match, it is assumed that the zone is empty. 7365 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 7366 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 7367 * starts where the previous one ended. For example, ZONE_DMA32 starts 7368 * at arch_max_dma_pfn. 7369 */ 7370 void __init free_area_init(unsigned long *max_zone_pfn) 7371 { 7372 unsigned long start_pfn, end_pfn; 7373 int i, nid, zone; 7374 bool descending; 7375 7376 /* Record where the zone boundaries are */ 7377 memset(arch_zone_lowest_possible_pfn, 0, 7378 sizeof(arch_zone_lowest_possible_pfn)); 7379 memset(arch_zone_highest_possible_pfn, 0, 7380 sizeof(arch_zone_highest_possible_pfn)); 7381 7382 start_pfn = find_min_pfn_with_active_regions(); 7383 descending = arch_has_descending_max_zone_pfns(); 7384 7385 for (i = 0; i < MAX_NR_ZONES; i++) { 7386 if (descending) 7387 zone = MAX_NR_ZONES - i - 1; 7388 else 7389 zone = i; 7390 7391 if (zone == ZONE_MOVABLE) 7392 continue; 7393 7394 end_pfn = max(max_zone_pfn[zone], start_pfn); 7395 arch_zone_lowest_possible_pfn[zone] = start_pfn; 7396 arch_zone_highest_possible_pfn[zone] = end_pfn; 7397 7398 start_pfn = end_pfn; 7399 } 7400 7401 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 7402 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 7403 find_zone_movable_pfns_for_nodes(); 7404 7405 /* Print out the zone ranges */ 7406 pr_info("Zone ranges:\n"); 7407 for (i = 0; i < MAX_NR_ZONES; i++) { 7408 if (i == ZONE_MOVABLE) 7409 continue; 7410 pr_info(" %-8s ", zone_names[i]); 7411 if (arch_zone_lowest_possible_pfn[i] == 7412 arch_zone_highest_possible_pfn[i]) 7413 pr_cont("empty\n"); 7414 else 7415 pr_cont("[mem %#018Lx-%#018Lx]\n", 7416 (u64)arch_zone_lowest_possible_pfn[i] 7417 << PAGE_SHIFT, 7418 ((u64)arch_zone_highest_possible_pfn[i] 7419 << PAGE_SHIFT) - 1); 7420 } 7421 7422 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 7423 pr_info("Movable zone start for each node\n"); 7424 for (i = 0; i < MAX_NUMNODES; i++) { 7425 if (zone_movable_pfn[i]) 7426 pr_info(" Node %d: %#018Lx\n", i, 7427 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 7428 } 7429 7430 /* 7431 * Print out the early node map, and initialize the 7432 * subsection-map relative to active online memory ranges to 7433 * enable future "sub-section" extensions of the memory map. 7434 */ 7435 pr_info("Early memory node ranges\n"); 7436 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7437 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 7438 (u64)start_pfn << PAGE_SHIFT, 7439 ((u64)end_pfn << PAGE_SHIFT) - 1); 7440 subsection_map_init(start_pfn, end_pfn - start_pfn); 7441 } 7442 7443 /* Initialise every node */ 7444 mminit_verify_pageflags_layout(); 7445 setup_nr_node_ids(); 7446 init_unavailable_mem(); 7447 for_each_online_node(nid) { 7448 pg_data_t *pgdat = NODE_DATA(nid); 7449 free_area_init_node(nid); 7450 7451 /* Any memory on that node */ 7452 if (pgdat->node_present_pages) 7453 node_set_state(nid, N_MEMORY); 7454 check_for_memory(pgdat, nid); 7455 } 7456 } 7457 7458 static int __init cmdline_parse_core(char *p, unsigned long *core, 7459 unsigned long *percent) 7460 { 7461 unsigned long long coremem; 7462 char *endptr; 7463 7464 if (!p) 7465 return -EINVAL; 7466 7467 /* Value may be a percentage of total memory, otherwise bytes */ 7468 coremem = simple_strtoull(p, &endptr, 0); 7469 if (*endptr == '%') { 7470 /* Paranoid check for percent values greater than 100 */ 7471 WARN_ON(coremem > 100); 7472 7473 *percent = coremem; 7474 } else { 7475 coremem = memparse(p, &p); 7476 /* Paranoid check that UL is enough for the coremem value */ 7477 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 7478 7479 *core = coremem >> PAGE_SHIFT; 7480 *percent = 0UL; 7481 } 7482 return 0; 7483 } 7484 7485 /* 7486 * kernelcore=size sets the amount of memory for use for allocations that 7487 * cannot be reclaimed or migrated. 7488 */ 7489 static int __init cmdline_parse_kernelcore(char *p) 7490 { 7491 /* parse kernelcore=mirror */ 7492 if (parse_option_str(p, "mirror")) { 7493 mirrored_kernelcore = true; 7494 return 0; 7495 } 7496 7497 return cmdline_parse_core(p, &required_kernelcore, 7498 &required_kernelcore_percent); 7499 } 7500 7501 /* 7502 * movablecore=size sets the amount of memory for use for allocations that 7503 * can be reclaimed or migrated. 7504 */ 7505 static int __init cmdline_parse_movablecore(char *p) 7506 { 7507 return cmdline_parse_core(p, &required_movablecore, 7508 &required_movablecore_percent); 7509 } 7510 7511 early_param("kernelcore", cmdline_parse_kernelcore); 7512 early_param("movablecore", cmdline_parse_movablecore); 7513 7514 void adjust_managed_page_count(struct page *page, long count) 7515 { 7516 atomic_long_add(count, &page_zone(page)->managed_pages); 7517 totalram_pages_add(count); 7518 #ifdef CONFIG_HIGHMEM 7519 if (PageHighMem(page)) 7520 totalhigh_pages_add(count); 7521 #endif 7522 } 7523 EXPORT_SYMBOL(adjust_managed_page_count); 7524 7525 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 7526 { 7527 void *pos; 7528 unsigned long pages = 0; 7529 7530 start = (void *)PAGE_ALIGN((unsigned long)start); 7531 end = (void *)((unsigned long)end & PAGE_MASK); 7532 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 7533 struct page *page = virt_to_page(pos); 7534 void *direct_map_addr; 7535 7536 /* 7537 * 'direct_map_addr' might be different from 'pos' 7538 * because some architectures' virt_to_page() 7539 * work with aliases. Getting the direct map 7540 * address ensures that we get a _writeable_ 7541 * alias for the memset(). 7542 */ 7543 direct_map_addr = page_address(page); 7544 if ((unsigned int)poison <= 0xFF) 7545 memset(direct_map_addr, poison, PAGE_SIZE); 7546 7547 free_reserved_page(page); 7548 } 7549 7550 if (pages && s) 7551 pr_info("Freeing %s memory: %ldK\n", 7552 s, pages << (PAGE_SHIFT - 10)); 7553 7554 return pages; 7555 } 7556 7557 #ifdef CONFIG_HIGHMEM 7558 void free_highmem_page(struct page *page) 7559 { 7560 __free_reserved_page(page); 7561 totalram_pages_inc(); 7562 atomic_long_inc(&page_zone(page)->managed_pages); 7563 totalhigh_pages_inc(); 7564 } 7565 #endif 7566 7567 7568 void __init mem_init_print_info(const char *str) 7569 { 7570 unsigned long physpages, codesize, datasize, rosize, bss_size; 7571 unsigned long init_code_size, init_data_size; 7572 7573 physpages = get_num_physpages(); 7574 codesize = _etext - _stext; 7575 datasize = _edata - _sdata; 7576 rosize = __end_rodata - __start_rodata; 7577 bss_size = __bss_stop - __bss_start; 7578 init_data_size = __init_end - __init_begin; 7579 init_code_size = _einittext - _sinittext; 7580 7581 /* 7582 * Detect special cases and adjust section sizes accordingly: 7583 * 1) .init.* may be embedded into .data sections 7584 * 2) .init.text.* may be out of [__init_begin, __init_end], 7585 * please refer to arch/tile/kernel/vmlinux.lds.S. 7586 * 3) .rodata.* may be embedded into .text or .data sections. 7587 */ 7588 #define adj_init_size(start, end, size, pos, adj) \ 7589 do { \ 7590 if (start <= pos && pos < end && size > adj) \ 7591 size -= adj; \ 7592 } while (0) 7593 7594 adj_init_size(__init_begin, __init_end, init_data_size, 7595 _sinittext, init_code_size); 7596 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 7597 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 7598 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 7599 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 7600 7601 #undef adj_init_size 7602 7603 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 7604 #ifdef CONFIG_HIGHMEM 7605 ", %luK highmem" 7606 #endif 7607 "%s%s)\n", 7608 nr_free_pages() << (PAGE_SHIFT - 10), 7609 physpages << (PAGE_SHIFT - 10), 7610 codesize >> 10, datasize >> 10, rosize >> 10, 7611 (init_data_size + init_code_size) >> 10, bss_size >> 10, 7612 (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10), 7613 totalcma_pages << (PAGE_SHIFT - 10), 7614 #ifdef CONFIG_HIGHMEM 7615 totalhigh_pages() << (PAGE_SHIFT - 10), 7616 #endif 7617 str ? ", " : "", str ? str : ""); 7618 } 7619 7620 /** 7621 * set_dma_reserve - set the specified number of pages reserved in the first zone 7622 * @new_dma_reserve: The number of pages to mark reserved 7623 * 7624 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 7625 * In the DMA zone, a significant percentage may be consumed by kernel image 7626 * and other unfreeable allocations which can skew the watermarks badly. This 7627 * function may optionally be used to account for unfreeable pages in the 7628 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 7629 * smaller per-cpu batchsize. 7630 */ 7631 void __init set_dma_reserve(unsigned long new_dma_reserve) 7632 { 7633 dma_reserve = new_dma_reserve; 7634 } 7635 7636 static int page_alloc_cpu_dead(unsigned int cpu) 7637 { 7638 7639 lru_add_drain_cpu(cpu); 7640 drain_pages(cpu); 7641 7642 /* 7643 * Spill the event counters of the dead processor 7644 * into the current processors event counters. 7645 * This artificially elevates the count of the current 7646 * processor. 7647 */ 7648 vm_events_fold_cpu(cpu); 7649 7650 /* 7651 * Zero the differential counters of the dead processor 7652 * so that the vm statistics are consistent. 7653 * 7654 * This is only okay since the processor is dead and cannot 7655 * race with what we are doing. 7656 */ 7657 cpu_vm_stats_fold(cpu); 7658 return 0; 7659 } 7660 7661 #ifdef CONFIG_NUMA 7662 int hashdist = HASHDIST_DEFAULT; 7663 7664 static int __init set_hashdist(char *str) 7665 { 7666 if (!str) 7667 return 0; 7668 hashdist = simple_strtoul(str, &str, 0); 7669 return 1; 7670 } 7671 __setup("hashdist=", set_hashdist); 7672 #endif 7673 7674 void __init page_alloc_init(void) 7675 { 7676 int ret; 7677 7678 #ifdef CONFIG_NUMA 7679 if (num_node_state(N_MEMORY) == 1) 7680 hashdist = 0; 7681 #endif 7682 7683 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD, 7684 "mm/page_alloc:dead", NULL, 7685 page_alloc_cpu_dead); 7686 WARN_ON(ret < 0); 7687 } 7688 7689 /* 7690 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 7691 * or min_free_kbytes changes. 7692 */ 7693 static void calculate_totalreserve_pages(void) 7694 { 7695 struct pglist_data *pgdat; 7696 unsigned long reserve_pages = 0; 7697 enum zone_type i, j; 7698 7699 for_each_online_pgdat(pgdat) { 7700 7701 pgdat->totalreserve_pages = 0; 7702 7703 for (i = 0; i < MAX_NR_ZONES; i++) { 7704 struct zone *zone = pgdat->node_zones + i; 7705 long max = 0; 7706 unsigned long managed_pages = zone_managed_pages(zone); 7707 7708 /* Find valid and maximum lowmem_reserve in the zone */ 7709 for (j = i; j < MAX_NR_ZONES; j++) { 7710 if (zone->lowmem_reserve[j] > max) 7711 max = zone->lowmem_reserve[j]; 7712 } 7713 7714 /* we treat the high watermark as reserved pages. */ 7715 max += high_wmark_pages(zone); 7716 7717 if (max > managed_pages) 7718 max = managed_pages; 7719 7720 pgdat->totalreserve_pages += max; 7721 7722 reserve_pages += max; 7723 } 7724 } 7725 totalreserve_pages = reserve_pages; 7726 } 7727 7728 /* 7729 * setup_per_zone_lowmem_reserve - called whenever 7730 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 7731 * has a correct pages reserved value, so an adequate number of 7732 * pages are left in the zone after a successful __alloc_pages(). 7733 */ 7734 static void setup_per_zone_lowmem_reserve(void) 7735 { 7736 struct pglist_data *pgdat; 7737 enum zone_type j, idx; 7738 7739 for_each_online_pgdat(pgdat) { 7740 for (j = 0; j < MAX_NR_ZONES; j++) { 7741 struct zone *zone = pgdat->node_zones + j; 7742 unsigned long managed_pages = zone_managed_pages(zone); 7743 7744 zone->lowmem_reserve[j] = 0; 7745 7746 idx = j; 7747 while (idx) { 7748 struct zone *lower_zone; 7749 7750 idx--; 7751 lower_zone = pgdat->node_zones + idx; 7752 7753 if (!sysctl_lowmem_reserve_ratio[idx] || 7754 !zone_managed_pages(lower_zone)) { 7755 lower_zone->lowmem_reserve[j] = 0; 7756 continue; 7757 } else { 7758 lower_zone->lowmem_reserve[j] = 7759 managed_pages / sysctl_lowmem_reserve_ratio[idx]; 7760 } 7761 managed_pages += zone_managed_pages(lower_zone); 7762 } 7763 } 7764 } 7765 7766 /* update totalreserve_pages */ 7767 calculate_totalreserve_pages(); 7768 } 7769 7770 static void __setup_per_zone_wmarks(void) 7771 { 7772 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 7773 unsigned long lowmem_pages = 0; 7774 struct zone *zone; 7775 unsigned long flags; 7776 7777 /* Calculate total number of !ZONE_HIGHMEM pages */ 7778 for_each_zone(zone) { 7779 if (!is_highmem(zone)) 7780 lowmem_pages += zone_managed_pages(zone); 7781 } 7782 7783 for_each_zone(zone) { 7784 u64 tmp; 7785 7786 spin_lock_irqsave(&zone->lock, flags); 7787 tmp = (u64)pages_min * zone_managed_pages(zone); 7788 do_div(tmp, lowmem_pages); 7789 if (is_highmem(zone)) { 7790 /* 7791 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 7792 * need highmem pages, so cap pages_min to a small 7793 * value here. 7794 * 7795 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 7796 * deltas control async page reclaim, and so should 7797 * not be capped for highmem. 7798 */ 7799 unsigned long min_pages; 7800 7801 min_pages = zone_managed_pages(zone) / 1024; 7802 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 7803 zone->_watermark[WMARK_MIN] = min_pages; 7804 } else { 7805 /* 7806 * If it's a lowmem zone, reserve a number of pages 7807 * proportionate to the zone's size. 7808 */ 7809 zone->_watermark[WMARK_MIN] = tmp; 7810 } 7811 7812 /* 7813 * Set the kswapd watermarks distance according to the 7814 * scale factor in proportion to available memory, but 7815 * ensure a minimum size on small systems. 7816 */ 7817 tmp = max_t(u64, tmp >> 2, 7818 mult_frac(zone_managed_pages(zone), 7819 watermark_scale_factor, 10000)); 7820 7821 zone->watermark_boost = 0; 7822 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 7823 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2; 7824 7825 spin_unlock_irqrestore(&zone->lock, flags); 7826 } 7827 7828 /* update totalreserve_pages */ 7829 calculate_totalreserve_pages(); 7830 } 7831 7832 /** 7833 * setup_per_zone_wmarks - called when min_free_kbytes changes 7834 * or when memory is hot-{added|removed} 7835 * 7836 * Ensures that the watermark[min,low,high] values for each zone are set 7837 * correctly with respect to min_free_kbytes. 7838 */ 7839 void setup_per_zone_wmarks(void) 7840 { 7841 static DEFINE_SPINLOCK(lock); 7842 7843 spin_lock(&lock); 7844 __setup_per_zone_wmarks(); 7845 spin_unlock(&lock); 7846 } 7847 7848 /* 7849 * Initialise min_free_kbytes. 7850 * 7851 * For small machines we want it small (128k min). For large machines 7852 * we want it large (256MB max). But it is not linear, because network 7853 * bandwidth does not increase linearly with machine size. We use 7854 * 7855 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 7856 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 7857 * 7858 * which yields 7859 * 7860 * 16MB: 512k 7861 * 32MB: 724k 7862 * 64MB: 1024k 7863 * 128MB: 1448k 7864 * 256MB: 2048k 7865 * 512MB: 2896k 7866 * 1024MB: 4096k 7867 * 2048MB: 5792k 7868 * 4096MB: 8192k 7869 * 8192MB: 11584k 7870 * 16384MB: 16384k 7871 */ 7872 int __meminit init_per_zone_wmark_min(void) 7873 { 7874 unsigned long lowmem_kbytes; 7875 int new_min_free_kbytes; 7876 7877 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 7878 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 7879 7880 if (new_min_free_kbytes > user_min_free_kbytes) { 7881 min_free_kbytes = new_min_free_kbytes; 7882 if (min_free_kbytes < 128) 7883 min_free_kbytes = 128; 7884 if (min_free_kbytes > 262144) 7885 min_free_kbytes = 262144; 7886 } else { 7887 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 7888 new_min_free_kbytes, user_min_free_kbytes); 7889 } 7890 setup_per_zone_wmarks(); 7891 refresh_zone_stat_thresholds(); 7892 setup_per_zone_lowmem_reserve(); 7893 7894 #ifdef CONFIG_NUMA 7895 setup_min_unmapped_ratio(); 7896 setup_min_slab_ratio(); 7897 #endif 7898 7899 khugepaged_min_free_kbytes_update(); 7900 7901 return 0; 7902 } 7903 postcore_initcall(init_per_zone_wmark_min) 7904 7905 /* 7906 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 7907 * that we can call two helper functions whenever min_free_kbytes 7908 * changes. 7909 */ 7910 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 7911 void *buffer, size_t *length, loff_t *ppos) 7912 { 7913 int rc; 7914 7915 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7916 if (rc) 7917 return rc; 7918 7919 if (write) { 7920 user_min_free_kbytes = min_free_kbytes; 7921 setup_per_zone_wmarks(); 7922 } 7923 return 0; 7924 } 7925 7926 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 7927 void *buffer, size_t *length, loff_t *ppos) 7928 { 7929 int rc; 7930 7931 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7932 if (rc) 7933 return rc; 7934 7935 if (write) 7936 setup_per_zone_wmarks(); 7937 7938 return 0; 7939 } 7940 7941 #ifdef CONFIG_NUMA 7942 static void setup_min_unmapped_ratio(void) 7943 { 7944 pg_data_t *pgdat; 7945 struct zone *zone; 7946 7947 for_each_online_pgdat(pgdat) 7948 pgdat->min_unmapped_pages = 0; 7949 7950 for_each_zone(zone) 7951 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * 7952 sysctl_min_unmapped_ratio) / 100; 7953 } 7954 7955 7956 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 7957 void *buffer, size_t *length, loff_t *ppos) 7958 { 7959 int rc; 7960 7961 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7962 if (rc) 7963 return rc; 7964 7965 setup_min_unmapped_ratio(); 7966 7967 return 0; 7968 } 7969 7970 static void setup_min_slab_ratio(void) 7971 { 7972 pg_data_t *pgdat; 7973 struct zone *zone; 7974 7975 for_each_online_pgdat(pgdat) 7976 pgdat->min_slab_pages = 0; 7977 7978 for_each_zone(zone) 7979 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * 7980 sysctl_min_slab_ratio) / 100; 7981 } 7982 7983 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 7984 void *buffer, size_t *length, loff_t *ppos) 7985 { 7986 int rc; 7987 7988 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7989 if (rc) 7990 return rc; 7991 7992 setup_min_slab_ratio(); 7993 7994 return 0; 7995 } 7996 #endif 7997 7998 /* 7999 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 8000 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 8001 * whenever sysctl_lowmem_reserve_ratio changes. 8002 * 8003 * The reserve ratio obviously has absolutely no relation with the 8004 * minimum watermarks. The lowmem reserve ratio can only make sense 8005 * if in function of the boot time zone sizes. 8006 */ 8007 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 8008 void *buffer, size_t *length, loff_t *ppos) 8009 { 8010 int i; 8011 8012 proc_dointvec_minmax(table, write, buffer, length, ppos); 8013 8014 for (i = 0; i < MAX_NR_ZONES; i++) { 8015 if (sysctl_lowmem_reserve_ratio[i] < 1) 8016 sysctl_lowmem_reserve_ratio[i] = 0; 8017 } 8018 8019 setup_per_zone_lowmem_reserve(); 8020 return 0; 8021 } 8022 8023 static void __zone_pcp_update(struct zone *zone) 8024 { 8025 unsigned int cpu; 8026 8027 for_each_possible_cpu(cpu) 8028 pageset_set_high_and_batch(zone, 8029 per_cpu_ptr(zone->pageset, cpu)); 8030 } 8031 8032 /* 8033 * percpu_pagelist_fraction - changes the pcp->high for each zone on each 8034 * cpu. It is the fraction of total pages in each zone that a hot per cpu 8035 * pagelist can have before it gets flushed back to buddy allocator. 8036 */ 8037 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, 8038 void *buffer, size_t *length, loff_t *ppos) 8039 { 8040 struct zone *zone; 8041 int old_percpu_pagelist_fraction; 8042 int ret; 8043 8044 mutex_lock(&pcp_batch_high_lock); 8045 old_percpu_pagelist_fraction = percpu_pagelist_fraction; 8046 8047 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 8048 if (!write || ret < 0) 8049 goto out; 8050 8051 /* Sanity checking to avoid pcp imbalance */ 8052 if (percpu_pagelist_fraction && 8053 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { 8054 percpu_pagelist_fraction = old_percpu_pagelist_fraction; 8055 ret = -EINVAL; 8056 goto out; 8057 } 8058 8059 /* No change? */ 8060 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) 8061 goto out; 8062 8063 for_each_populated_zone(zone) 8064 __zone_pcp_update(zone); 8065 out: 8066 mutex_unlock(&pcp_batch_high_lock); 8067 return ret; 8068 } 8069 8070 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 8071 /* 8072 * Returns the number of pages that arch has reserved but 8073 * is not known to alloc_large_system_hash(). 8074 */ 8075 static unsigned long __init arch_reserved_kernel_pages(void) 8076 { 8077 return 0; 8078 } 8079 #endif 8080 8081 /* 8082 * Adaptive scale is meant to reduce sizes of hash tables on large memory 8083 * machines. As memory size is increased the scale is also increased but at 8084 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 8085 * quadruples the scale is increased by one, which means the size of hash table 8086 * only doubles, instead of quadrupling as well. 8087 * Because 32-bit systems cannot have large physical memory, where this scaling 8088 * makes sense, it is disabled on such platforms. 8089 */ 8090 #if __BITS_PER_LONG > 32 8091 #define ADAPT_SCALE_BASE (64ul << 30) 8092 #define ADAPT_SCALE_SHIFT 2 8093 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 8094 #endif 8095 8096 /* 8097 * allocate a large system hash table from bootmem 8098 * - it is assumed that the hash table must contain an exact power-of-2 8099 * quantity of entries 8100 * - limit is the number of hash buckets, not the total allocation size 8101 */ 8102 void *__init alloc_large_system_hash(const char *tablename, 8103 unsigned long bucketsize, 8104 unsigned long numentries, 8105 int scale, 8106 int flags, 8107 unsigned int *_hash_shift, 8108 unsigned int *_hash_mask, 8109 unsigned long low_limit, 8110 unsigned long high_limit) 8111 { 8112 unsigned long long max = high_limit; 8113 unsigned long log2qty, size; 8114 void *table = NULL; 8115 gfp_t gfp_flags; 8116 bool virt; 8117 8118 /* allow the kernel cmdline to have a say */ 8119 if (!numentries) { 8120 /* round applicable memory size up to nearest megabyte */ 8121 numentries = nr_kernel_pages; 8122 numentries -= arch_reserved_kernel_pages(); 8123 8124 /* It isn't necessary when PAGE_SIZE >= 1MB */ 8125 if (PAGE_SHIFT < 20) 8126 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 8127 8128 #if __BITS_PER_LONG > 32 8129 if (!high_limit) { 8130 unsigned long adapt; 8131 8132 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 8133 adapt <<= ADAPT_SCALE_SHIFT) 8134 scale++; 8135 } 8136 #endif 8137 8138 /* limit to 1 bucket per 2^scale bytes of low memory */ 8139 if (scale > PAGE_SHIFT) 8140 numentries >>= (scale - PAGE_SHIFT); 8141 else 8142 numentries <<= (PAGE_SHIFT - scale); 8143 8144 /* Make sure we've got at least a 0-order allocation.. */ 8145 if (unlikely(flags & HASH_SMALL)) { 8146 /* Makes no sense without HASH_EARLY */ 8147 WARN_ON(!(flags & HASH_EARLY)); 8148 if (!(numentries >> *_hash_shift)) { 8149 numentries = 1UL << *_hash_shift; 8150 BUG_ON(!numentries); 8151 } 8152 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 8153 numentries = PAGE_SIZE / bucketsize; 8154 } 8155 numentries = roundup_pow_of_two(numentries); 8156 8157 /* limit allocation size to 1/16 total memory by default */ 8158 if (max == 0) { 8159 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 8160 do_div(max, bucketsize); 8161 } 8162 max = min(max, 0x80000000ULL); 8163 8164 if (numentries < low_limit) 8165 numentries = low_limit; 8166 if (numentries > max) 8167 numentries = max; 8168 8169 log2qty = ilog2(numentries); 8170 8171 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 8172 do { 8173 virt = false; 8174 size = bucketsize << log2qty; 8175 if (flags & HASH_EARLY) { 8176 if (flags & HASH_ZERO) 8177 table = memblock_alloc(size, SMP_CACHE_BYTES); 8178 else 8179 table = memblock_alloc_raw(size, 8180 SMP_CACHE_BYTES); 8181 } else if (get_order(size) >= MAX_ORDER || hashdist) { 8182 table = __vmalloc(size, gfp_flags); 8183 virt = true; 8184 } else { 8185 /* 8186 * If bucketsize is not a power-of-two, we may free 8187 * some pages at the end of hash table which 8188 * alloc_pages_exact() automatically does 8189 */ 8190 table = alloc_pages_exact(size, gfp_flags); 8191 kmemleak_alloc(table, size, 1, gfp_flags); 8192 } 8193 } while (!table && size > PAGE_SIZE && --log2qty); 8194 8195 if (!table) 8196 panic("Failed to allocate %s hash table\n", tablename); 8197 8198 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 8199 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, 8200 virt ? "vmalloc" : "linear"); 8201 8202 if (_hash_shift) 8203 *_hash_shift = log2qty; 8204 if (_hash_mask) 8205 *_hash_mask = (1 << log2qty) - 1; 8206 8207 return table; 8208 } 8209 8210 /* 8211 * This function checks whether pageblock includes unmovable pages or not. 8212 * 8213 * PageLRU check without isolation or lru_lock could race so that 8214 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable 8215 * check without lock_page also may miss some movable non-lru pages at 8216 * race condition. So you can't expect this function should be exact. 8217 * 8218 * Returns a page without holding a reference. If the caller wants to 8219 * dereference that page (e.g., dumping), it has to make sure that it 8220 * cannot get removed (e.g., via memory unplug) concurrently. 8221 * 8222 */ 8223 struct page *has_unmovable_pages(struct zone *zone, struct page *page, 8224 int migratetype, int flags) 8225 { 8226 unsigned long iter = 0; 8227 unsigned long pfn = page_to_pfn(page); 8228 unsigned long offset = pfn % pageblock_nr_pages; 8229 8230 if (is_migrate_cma_page(page)) { 8231 /* 8232 * CMA allocations (alloc_contig_range) really need to mark 8233 * isolate CMA pageblocks even when they are not movable in fact 8234 * so consider them movable here. 8235 */ 8236 if (is_migrate_cma(migratetype)) 8237 return NULL; 8238 8239 return page; 8240 } 8241 8242 for (; iter < pageblock_nr_pages - offset; iter++) { 8243 if (!pfn_valid_within(pfn + iter)) 8244 continue; 8245 8246 page = pfn_to_page(pfn + iter); 8247 8248 /* 8249 * Both, bootmem allocations and memory holes are marked 8250 * PG_reserved and are unmovable. We can even have unmovable 8251 * allocations inside ZONE_MOVABLE, for example when 8252 * specifying "movablecore". 8253 */ 8254 if (PageReserved(page)) 8255 return page; 8256 8257 /* 8258 * If the zone is movable and we have ruled out all reserved 8259 * pages then it should be reasonably safe to assume the rest 8260 * is movable. 8261 */ 8262 if (zone_idx(zone) == ZONE_MOVABLE) 8263 continue; 8264 8265 /* 8266 * Hugepages are not in LRU lists, but they're movable. 8267 * THPs are on the LRU, but need to be counted as #small pages. 8268 * We need not scan over tail pages because we don't 8269 * handle each tail page individually in migration. 8270 */ 8271 if (PageHuge(page) || PageTransCompound(page)) { 8272 struct page *head = compound_head(page); 8273 unsigned int skip_pages; 8274 8275 if (PageHuge(page)) { 8276 if (!hugepage_migration_supported(page_hstate(head))) 8277 return page; 8278 } else if (!PageLRU(head) && !__PageMovable(head)) { 8279 return page; 8280 } 8281 8282 skip_pages = compound_nr(head) - (page - head); 8283 iter += skip_pages - 1; 8284 continue; 8285 } 8286 8287 /* 8288 * We can't use page_count without pin a page 8289 * because another CPU can free compound page. 8290 * This check already skips compound tails of THP 8291 * because their page->_refcount is zero at all time. 8292 */ 8293 if (!page_ref_count(page)) { 8294 if (PageBuddy(page)) 8295 iter += (1 << page_order(page)) - 1; 8296 continue; 8297 } 8298 8299 /* 8300 * The HWPoisoned page may be not in buddy system, and 8301 * page_count() is not 0. 8302 */ 8303 if ((flags & MEMORY_OFFLINE) && PageHWPoison(page)) 8304 continue; 8305 8306 /* 8307 * We treat all PageOffline() pages as movable when offlining 8308 * to give drivers a chance to decrement their reference count 8309 * in MEM_GOING_OFFLINE in order to indicate that these pages 8310 * can be offlined as there are no direct references anymore. 8311 * For actually unmovable PageOffline() where the driver does 8312 * not support this, we will fail later when trying to actually 8313 * move these pages that still have a reference count > 0. 8314 * (false negatives in this function only) 8315 */ 8316 if ((flags & MEMORY_OFFLINE) && PageOffline(page)) 8317 continue; 8318 8319 if (__PageMovable(page) || PageLRU(page)) 8320 continue; 8321 8322 /* 8323 * If there are RECLAIMABLE pages, we need to check 8324 * it. But now, memory offline itself doesn't call 8325 * shrink_node_slabs() and it still to be fixed. 8326 */ 8327 return page; 8328 } 8329 return NULL; 8330 } 8331 8332 #ifdef CONFIG_CONTIG_ALLOC 8333 static unsigned long pfn_max_align_down(unsigned long pfn) 8334 { 8335 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, 8336 pageblock_nr_pages) - 1); 8337 } 8338 8339 static unsigned long pfn_max_align_up(unsigned long pfn) 8340 { 8341 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, 8342 pageblock_nr_pages)); 8343 } 8344 8345 /* [start, end) must belong to a single zone. */ 8346 static int __alloc_contig_migrate_range(struct compact_control *cc, 8347 unsigned long start, unsigned long end) 8348 { 8349 /* This function is based on compact_zone() from compaction.c. */ 8350 unsigned int nr_reclaimed; 8351 unsigned long pfn = start; 8352 unsigned int tries = 0; 8353 int ret = 0; 8354 struct migration_target_control mtc = { 8355 .nid = zone_to_nid(cc->zone), 8356 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, 8357 }; 8358 8359 migrate_prep(); 8360 8361 while (pfn < end || !list_empty(&cc->migratepages)) { 8362 if (fatal_signal_pending(current)) { 8363 ret = -EINTR; 8364 break; 8365 } 8366 8367 if (list_empty(&cc->migratepages)) { 8368 cc->nr_migratepages = 0; 8369 pfn = isolate_migratepages_range(cc, pfn, end); 8370 if (!pfn) { 8371 ret = -EINTR; 8372 break; 8373 } 8374 tries = 0; 8375 } else if (++tries == 5) { 8376 ret = ret < 0 ? ret : -EBUSY; 8377 break; 8378 } 8379 8380 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 8381 &cc->migratepages); 8382 cc->nr_migratepages -= nr_reclaimed; 8383 8384 ret = migrate_pages(&cc->migratepages, alloc_migration_target, 8385 NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE); 8386 } 8387 if (ret < 0) { 8388 putback_movable_pages(&cc->migratepages); 8389 return ret; 8390 } 8391 return 0; 8392 } 8393 8394 /** 8395 * alloc_contig_range() -- tries to allocate given range of pages 8396 * @start: start PFN to allocate 8397 * @end: one-past-the-last PFN to allocate 8398 * @migratetype: migratetype of the underlaying pageblocks (either 8399 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 8400 * in range must have the same migratetype and it must 8401 * be either of the two. 8402 * @gfp_mask: GFP mask to use during compaction 8403 * 8404 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 8405 * aligned. The PFN range must belong to a single zone. 8406 * 8407 * The first thing this routine does is attempt to MIGRATE_ISOLATE all 8408 * pageblocks in the range. Once isolated, the pageblocks should not 8409 * be modified by others. 8410 * 8411 * Return: zero on success or negative error code. On success all 8412 * pages which PFN is in [start, end) are allocated for the caller and 8413 * need to be freed with free_contig_range(). 8414 */ 8415 int alloc_contig_range(unsigned long start, unsigned long end, 8416 unsigned migratetype, gfp_t gfp_mask) 8417 { 8418 unsigned long outer_start, outer_end; 8419 unsigned int order; 8420 int ret = 0; 8421 8422 struct compact_control cc = { 8423 .nr_migratepages = 0, 8424 .order = -1, 8425 .zone = page_zone(pfn_to_page(start)), 8426 .mode = MIGRATE_SYNC, 8427 .ignore_skip_hint = true, 8428 .no_set_skip_hint = true, 8429 .gfp_mask = current_gfp_context(gfp_mask), 8430 .alloc_contig = true, 8431 }; 8432 INIT_LIST_HEAD(&cc.migratepages); 8433 8434 /* 8435 * What we do here is we mark all pageblocks in range as 8436 * MIGRATE_ISOLATE. Because pageblock and max order pages may 8437 * have different sizes, and due to the way page allocator 8438 * work, we align the range to biggest of the two pages so 8439 * that page allocator won't try to merge buddies from 8440 * different pageblocks and change MIGRATE_ISOLATE to some 8441 * other migration type. 8442 * 8443 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 8444 * migrate the pages from an unaligned range (ie. pages that 8445 * we are interested in). This will put all the pages in 8446 * range back to page allocator as MIGRATE_ISOLATE. 8447 * 8448 * When this is done, we take the pages in range from page 8449 * allocator removing them from the buddy system. This way 8450 * page allocator will never consider using them. 8451 * 8452 * This lets us mark the pageblocks back as 8453 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 8454 * aligned range but not in the unaligned, original range are 8455 * put back to page allocator so that buddy can use them. 8456 */ 8457 8458 ret = start_isolate_page_range(pfn_max_align_down(start), 8459 pfn_max_align_up(end), migratetype, 0); 8460 if (ret < 0) 8461 return ret; 8462 8463 /* 8464 * In case of -EBUSY, we'd like to know which page causes problem. 8465 * So, just fall through. test_pages_isolated() has a tracepoint 8466 * which will report the busy page. 8467 * 8468 * It is possible that busy pages could become available before 8469 * the call to test_pages_isolated, and the range will actually be 8470 * allocated. So, if we fall through be sure to clear ret so that 8471 * -EBUSY is not accidentally used or returned to caller. 8472 */ 8473 ret = __alloc_contig_migrate_range(&cc, start, end); 8474 if (ret && ret != -EBUSY) 8475 goto done; 8476 ret =0; 8477 8478 /* 8479 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 8480 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 8481 * more, all pages in [start, end) are free in page allocator. 8482 * What we are going to do is to allocate all pages from 8483 * [start, end) (that is remove them from page allocator). 8484 * 8485 * The only problem is that pages at the beginning and at the 8486 * end of interesting range may be not aligned with pages that 8487 * page allocator holds, ie. they can be part of higher order 8488 * pages. Because of this, we reserve the bigger range and 8489 * once this is done free the pages we are not interested in. 8490 * 8491 * We don't have to hold zone->lock here because the pages are 8492 * isolated thus they won't get removed from buddy. 8493 */ 8494 8495 lru_add_drain_all(); 8496 8497 order = 0; 8498 outer_start = start; 8499 while (!PageBuddy(pfn_to_page(outer_start))) { 8500 if (++order >= MAX_ORDER) { 8501 outer_start = start; 8502 break; 8503 } 8504 outer_start &= ~0UL << order; 8505 } 8506 8507 if (outer_start != start) { 8508 order = page_order(pfn_to_page(outer_start)); 8509 8510 /* 8511 * outer_start page could be small order buddy page and 8512 * it doesn't include start page. Adjust outer_start 8513 * in this case to report failed page properly 8514 * on tracepoint in test_pages_isolated() 8515 */ 8516 if (outer_start + (1UL << order) <= start) 8517 outer_start = start; 8518 } 8519 8520 /* Make sure the range is really isolated. */ 8521 if (test_pages_isolated(outer_start, end, 0)) { 8522 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n", 8523 __func__, outer_start, end); 8524 ret = -EBUSY; 8525 goto done; 8526 } 8527 8528 /* Grab isolated pages from freelists. */ 8529 outer_end = isolate_freepages_range(&cc, outer_start, end); 8530 if (!outer_end) { 8531 ret = -EBUSY; 8532 goto done; 8533 } 8534 8535 /* Free head and tail (if any) */ 8536 if (start != outer_start) 8537 free_contig_range(outer_start, start - outer_start); 8538 if (end != outer_end) 8539 free_contig_range(end, outer_end - end); 8540 8541 done: 8542 undo_isolate_page_range(pfn_max_align_down(start), 8543 pfn_max_align_up(end), migratetype); 8544 return ret; 8545 } 8546 EXPORT_SYMBOL(alloc_contig_range); 8547 8548 static int __alloc_contig_pages(unsigned long start_pfn, 8549 unsigned long nr_pages, gfp_t gfp_mask) 8550 { 8551 unsigned long end_pfn = start_pfn + nr_pages; 8552 8553 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE, 8554 gfp_mask); 8555 } 8556 8557 static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, 8558 unsigned long nr_pages) 8559 { 8560 unsigned long i, end_pfn = start_pfn + nr_pages; 8561 struct page *page; 8562 8563 for (i = start_pfn; i < end_pfn; i++) { 8564 page = pfn_to_online_page(i); 8565 if (!page) 8566 return false; 8567 8568 if (page_zone(page) != z) 8569 return false; 8570 8571 if (PageReserved(page)) 8572 return false; 8573 8574 if (page_count(page) > 0) 8575 return false; 8576 8577 if (PageHuge(page)) 8578 return false; 8579 } 8580 return true; 8581 } 8582 8583 static bool zone_spans_last_pfn(const struct zone *zone, 8584 unsigned long start_pfn, unsigned long nr_pages) 8585 { 8586 unsigned long last_pfn = start_pfn + nr_pages - 1; 8587 8588 return zone_spans_pfn(zone, last_pfn); 8589 } 8590 8591 /** 8592 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages 8593 * @nr_pages: Number of contiguous pages to allocate 8594 * @gfp_mask: GFP mask to limit search and used during compaction 8595 * @nid: Target node 8596 * @nodemask: Mask for other possible nodes 8597 * 8598 * This routine is a wrapper around alloc_contig_range(). It scans over zones 8599 * on an applicable zonelist to find a contiguous pfn range which can then be 8600 * tried for allocation with alloc_contig_range(). This routine is intended 8601 * for allocation requests which can not be fulfilled with the buddy allocator. 8602 * 8603 * The allocated memory is always aligned to a page boundary. If nr_pages is a 8604 * power of two then the alignment is guaranteed to be to the given nr_pages 8605 * (e.g. 1GB request would be aligned to 1GB). 8606 * 8607 * Allocated pages can be freed with free_contig_range() or by manually calling 8608 * __free_page() on each allocated page. 8609 * 8610 * Return: pointer to contiguous pages on success, or NULL if not successful. 8611 */ 8612 struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, 8613 int nid, nodemask_t *nodemask) 8614 { 8615 unsigned long ret, pfn, flags; 8616 struct zonelist *zonelist; 8617 struct zone *zone; 8618 struct zoneref *z; 8619 8620 zonelist = node_zonelist(nid, gfp_mask); 8621 for_each_zone_zonelist_nodemask(zone, z, zonelist, 8622 gfp_zone(gfp_mask), nodemask) { 8623 spin_lock_irqsave(&zone->lock, flags); 8624 8625 pfn = ALIGN(zone->zone_start_pfn, nr_pages); 8626 while (zone_spans_last_pfn(zone, pfn, nr_pages)) { 8627 if (pfn_range_valid_contig(zone, pfn, nr_pages)) { 8628 /* 8629 * We release the zone lock here because 8630 * alloc_contig_range() will also lock the zone 8631 * at some point. If there's an allocation 8632 * spinning on this lock, it may win the race 8633 * and cause alloc_contig_range() to fail... 8634 */ 8635 spin_unlock_irqrestore(&zone->lock, flags); 8636 ret = __alloc_contig_pages(pfn, nr_pages, 8637 gfp_mask); 8638 if (!ret) 8639 return pfn_to_page(pfn); 8640 spin_lock_irqsave(&zone->lock, flags); 8641 } 8642 pfn += nr_pages; 8643 } 8644 spin_unlock_irqrestore(&zone->lock, flags); 8645 } 8646 return NULL; 8647 } 8648 #endif /* CONFIG_CONTIG_ALLOC */ 8649 8650 void free_contig_range(unsigned long pfn, unsigned int nr_pages) 8651 { 8652 unsigned int count = 0; 8653 8654 for (; nr_pages--; pfn++) { 8655 struct page *page = pfn_to_page(pfn); 8656 8657 count += page_count(page) != 1; 8658 __free_page(page); 8659 } 8660 WARN(count != 0, "%d pages are still in use!\n", count); 8661 } 8662 EXPORT_SYMBOL(free_contig_range); 8663 8664 /* 8665 * The zone indicated has a new number of managed_pages; batch sizes and percpu 8666 * page high values need to be recalulated. 8667 */ 8668 void __meminit zone_pcp_update(struct zone *zone) 8669 { 8670 mutex_lock(&pcp_batch_high_lock); 8671 __zone_pcp_update(zone); 8672 mutex_unlock(&pcp_batch_high_lock); 8673 } 8674 8675 void zone_pcp_reset(struct zone *zone) 8676 { 8677 unsigned long flags; 8678 int cpu; 8679 struct per_cpu_pageset *pset; 8680 8681 /* avoid races with drain_pages() */ 8682 local_irq_save(flags); 8683 if (zone->pageset != &boot_pageset) { 8684 for_each_online_cpu(cpu) { 8685 pset = per_cpu_ptr(zone->pageset, cpu); 8686 drain_zonestat(zone, pset); 8687 } 8688 free_percpu(zone->pageset); 8689 zone->pageset = &boot_pageset; 8690 } 8691 local_irq_restore(flags); 8692 } 8693 8694 #ifdef CONFIG_MEMORY_HOTREMOVE 8695 /* 8696 * All pages in the range must be in a single zone and isolated 8697 * before calling this. 8698 */ 8699 unsigned long 8700 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 8701 { 8702 struct page *page; 8703 struct zone *zone; 8704 unsigned int order; 8705 unsigned long pfn; 8706 unsigned long flags; 8707 unsigned long offlined_pages = 0; 8708 8709 /* find the first valid pfn */ 8710 for (pfn = start_pfn; pfn < end_pfn; pfn++) 8711 if (pfn_valid(pfn)) 8712 break; 8713 if (pfn == end_pfn) 8714 return offlined_pages; 8715 8716 offline_mem_sections(pfn, end_pfn); 8717 zone = page_zone(pfn_to_page(pfn)); 8718 spin_lock_irqsave(&zone->lock, flags); 8719 pfn = start_pfn; 8720 while (pfn < end_pfn) { 8721 if (!pfn_valid(pfn)) { 8722 pfn++; 8723 continue; 8724 } 8725 page = pfn_to_page(pfn); 8726 /* 8727 * The HWPoisoned page may be not in buddy system, and 8728 * page_count() is not 0. 8729 */ 8730 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 8731 pfn++; 8732 offlined_pages++; 8733 continue; 8734 } 8735 /* 8736 * At this point all remaining PageOffline() pages have a 8737 * reference count of 0 and can simply be skipped. 8738 */ 8739 if (PageOffline(page)) { 8740 BUG_ON(page_count(page)); 8741 BUG_ON(PageBuddy(page)); 8742 pfn++; 8743 offlined_pages++; 8744 continue; 8745 } 8746 8747 BUG_ON(page_count(page)); 8748 BUG_ON(!PageBuddy(page)); 8749 order = page_order(page); 8750 offlined_pages += 1 << order; 8751 del_page_from_free_list(page, zone, order); 8752 pfn += (1 << order); 8753 } 8754 spin_unlock_irqrestore(&zone->lock, flags); 8755 8756 return offlined_pages; 8757 } 8758 #endif 8759 8760 bool is_free_buddy_page(struct page *page) 8761 { 8762 struct zone *zone = page_zone(page); 8763 unsigned long pfn = page_to_pfn(page); 8764 unsigned long flags; 8765 unsigned int order; 8766 8767 spin_lock_irqsave(&zone->lock, flags); 8768 for (order = 0; order < MAX_ORDER; order++) { 8769 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8770 8771 if (PageBuddy(page_head) && page_order(page_head) >= order) 8772 break; 8773 } 8774 spin_unlock_irqrestore(&zone->lock, flags); 8775 8776 return order < MAX_ORDER; 8777 } 8778 8779 #ifdef CONFIG_MEMORY_FAILURE 8780 /* 8781 * Set PG_hwpoison flag if a given page is confirmed to be a free page. This 8782 * test is performed under the zone lock to prevent a race against page 8783 * allocation. 8784 */ 8785 bool set_hwpoison_free_buddy_page(struct page *page) 8786 { 8787 struct zone *zone = page_zone(page); 8788 unsigned long pfn = page_to_pfn(page); 8789 unsigned long flags; 8790 unsigned int order; 8791 bool hwpoisoned = false; 8792 8793 spin_lock_irqsave(&zone->lock, flags); 8794 for (order = 0; order < MAX_ORDER; order++) { 8795 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8796 8797 if (PageBuddy(page_head) && page_order(page_head) >= order) { 8798 if (!TestSetPageHWPoison(page)) 8799 hwpoisoned = true; 8800 break; 8801 } 8802 } 8803 spin_unlock_irqrestore(&zone->lock, flags); 8804 8805 return hwpoisoned; 8806 } 8807 #endif 8808