1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/page_alloc.c 4 * 5 * Manages the free list, the system allocates free pages here. 6 * Note that kmalloc() lives in slab.c 7 * 8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 9 * Swap reorganised 29.12.95, Stephen Tweedie 10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 16 */ 17 18 #include <linux/stddef.h> 19 #include <linux/mm.h> 20 #include <linux/highmem.h> 21 #include <linux/swap.h> 22 #include <linux/interrupt.h> 23 #include <linux/pagemap.h> 24 #include <linux/jiffies.h> 25 #include <linux/memblock.h> 26 #include <linux/compiler.h> 27 #include <linux/kernel.h> 28 #include <linux/kasan.h> 29 #include <linux/module.h> 30 #include <linux/suspend.h> 31 #include <linux/pagevec.h> 32 #include <linux/blkdev.h> 33 #include <linux/slab.h> 34 #include <linux/ratelimit.h> 35 #include <linux/oom.h> 36 #include <linux/topology.h> 37 #include <linux/sysctl.h> 38 #include <linux/cpu.h> 39 #include <linux/cpuset.h> 40 #include <linux/memory_hotplug.h> 41 #include <linux/nodemask.h> 42 #include <linux/vmalloc.h> 43 #include <linux/vmstat.h> 44 #include <linux/mempolicy.h> 45 #include <linux/memremap.h> 46 #include <linux/stop_machine.h> 47 #include <linux/random.h> 48 #include <linux/sort.h> 49 #include <linux/pfn.h> 50 #include <linux/backing-dev.h> 51 #include <linux/fault-inject.h> 52 #include <linux/page-isolation.h> 53 #include <linux/debugobjects.h> 54 #include <linux/kmemleak.h> 55 #include <linux/compaction.h> 56 #include <trace/events/kmem.h> 57 #include <trace/events/oom.h> 58 #include <linux/prefetch.h> 59 #include <linux/mm_inline.h> 60 #include <linux/migrate.h> 61 #include <linux/hugetlb.h> 62 #include <linux/sched/rt.h> 63 #include <linux/sched/mm.h> 64 #include <linux/page_owner.h> 65 #include <linux/kthread.h> 66 #include <linux/memcontrol.h> 67 #include <linux/ftrace.h> 68 #include <linux/lockdep.h> 69 #include <linux/nmi.h> 70 #include <linux/psi.h> 71 72 #include <asm/sections.h> 73 #include <asm/tlbflush.h> 74 #include <asm/div64.h> 75 #include "internal.h" 76 #include "shuffle.h" 77 78 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 79 static DEFINE_MUTEX(pcp_batch_high_lock); 80 #define MIN_PERCPU_PAGELIST_FRACTION (8) 81 82 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 83 DEFINE_PER_CPU(int, numa_node); 84 EXPORT_PER_CPU_SYMBOL(numa_node); 85 #endif 86 87 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); 88 89 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 90 /* 91 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 92 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 93 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 94 * defined in <linux/topology.h>. 95 */ 96 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 97 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 98 int _node_numa_mem_[MAX_NUMNODES]; 99 #endif 100 101 /* work_structs for global per-cpu drains */ 102 struct pcpu_drain { 103 struct zone *zone; 104 struct work_struct work; 105 }; 106 DEFINE_MUTEX(pcpu_drain_mutex); 107 DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain); 108 109 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 110 volatile unsigned long latent_entropy __latent_entropy; 111 EXPORT_SYMBOL(latent_entropy); 112 #endif 113 114 /* 115 * Array of node states. 116 */ 117 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 118 [N_POSSIBLE] = NODE_MASK_ALL, 119 [N_ONLINE] = { { [0] = 1UL } }, 120 #ifndef CONFIG_NUMA 121 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 122 #ifdef CONFIG_HIGHMEM 123 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 124 #endif 125 [N_MEMORY] = { { [0] = 1UL } }, 126 [N_CPU] = { { [0] = 1UL } }, 127 #endif /* NUMA */ 128 }; 129 EXPORT_SYMBOL(node_states); 130 131 atomic_long_t _totalram_pages __read_mostly; 132 EXPORT_SYMBOL(_totalram_pages); 133 unsigned long totalreserve_pages __read_mostly; 134 unsigned long totalcma_pages __read_mostly; 135 136 int percpu_pagelist_fraction; 137 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 138 #ifdef CONFIG_INIT_ON_ALLOC_DEFAULT_ON 139 DEFINE_STATIC_KEY_TRUE(init_on_alloc); 140 #else 141 DEFINE_STATIC_KEY_FALSE(init_on_alloc); 142 #endif 143 EXPORT_SYMBOL(init_on_alloc); 144 145 #ifdef CONFIG_INIT_ON_FREE_DEFAULT_ON 146 DEFINE_STATIC_KEY_TRUE(init_on_free); 147 #else 148 DEFINE_STATIC_KEY_FALSE(init_on_free); 149 #endif 150 EXPORT_SYMBOL(init_on_free); 151 152 static int __init early_init_on_alloc(char *buf) 153 { 154 int ret; 155 bool bool_result; 156 157 if (!buf) 158 return -EINVAL; 159 ret = kstrtobool(buf, &bool_result); 160 if (bool_result && page_poisoning_enabled()) 161 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_alloc\n"); 162 if (bool_result) 163 static_branch_enable(&init_on_alloc); 164 else 165 static_branch_disable(&init_on_alloc); 166 return ret; 167 } 168 early_param("init_on_alloc", early_init_on_alloc); 169 170 static int __init early_init_on_free(char *buf) 171 { 172 int ret; 173 bool bool_result; 174 175 if (!buf) 176 return -EINVAL; 177 ret = kstrtobool(buf, &bool_result); 178 if (bool_result && page_poisoning_enabled()) 179 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_free\n"); 180 if (bool_result) 181 static_branch_enable(&init_on_free); 182 else 183 static_branch_disable(&init_on_free); 184 return ret; 185 } 186 early_param("init_on_free", early_init_on_free); 187 188 /* 189 * A cached value of the page's pageblock's migratetype, used when the page is 190 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 191 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 192 * Also the migratetype set in the page does not necessarily match the pcplist 193 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 194 * other index - this ensures that it will be put on the correct CMA freelist. 195 */ 196 static inline int get_pcppage_migratetype(struct page *page) 197 { 198 return page->index; 199 } 200 201 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 202 { 203 page->index = migratetype; 204 } 205 206 #ifdef CONFIG_PM_SLEEP 207 /* 208 * The following functions are used by the suspend/hibernate code to temporarily 209 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 210 * while devices are suspended. To avoid races with the suspend/hibernate code, 211 * they should always be called with system_transition_mutex held 212 * (gfp_allowed_mask also should only be modified with system_transition_mutex 213 * held, unless the suspend/hibernate code is guaranteed not to run in parallel 214 * with that modification). 215 */ 216 217 static gfp_t saved_gfp_mask; 218 219 void pm_restore_gfp_mask(void) 220 { 221 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 222 if (saved_gfp_mask) { 223 gfp_allowed_mask = saved_gfp_mask; 224 saved_gfp_mask = 0; 225 } 226 } 227 228 void pm_restrict_gfp_mask(void) 229 { 230 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 231 WARN_ON(saved_gfp_mask); 232 saved_gfp_mask = gfp_allowed_mask; 233 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 234 } 235 236 bool pm_suspended_storage(void) 237 { 238 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 239 return false; 240 return true; 241 } 242 #endif /* CONFIG_PM_SLEEP */ 243 244 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 245 unsigned int pageblock_order __read_mostly; 246 #endif 247 248 static void __free_pages_ok(struct page *page, unsigned int order); 249 250 /* 251 * results with 256, 32 in the lowmem_reserve sysctl: 252 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 253 * 1G machine -> (16M dma, 784M normal, 224M high) 254 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 255 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 256 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 257 * 258 * TBD: should special case ZONE_DMA32 machines here - in those we normally 259 * don't need any ZONE_NORMAL reservation 260 */ 261 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { 262 #ifdef CONFIG_ZONE_DMA 263 [ZONE_DMA] = 256, 264 #endif 265 #ifdef CONFIG_ZONE_DMA32 266 [ZONE_DMA32] = 256, 267 #endif 268 [ZONE_NORMAL] = 32, 269 #ifdef CONFIG_HIGHMEM 270 [ZONE_HIGHMEM] = 0, 271 #endif 272 [ZONE_MOVABLE] = 0, 273 }; 274 275 static char * const zone_names[MAX_NR_ZONES] = { 276 #ifdef CONFIG_ZONE_DMA 277 "DMA", 278 #endif 279 #ifdef CONFIG_ZONE_DMA32 280 "DMA32", 281 #endif 282 "Normal", 283 #ifdef CONFIG_HIGHMEM 284 "HighMem", 285 #endif 286 "Movable", 287 #ifdef CONFIG_ZONE_DEVICE 288 "Device", 289 #endif 290 }; 291 292 const char * const migratetype_names[MIGRATE_TYPES] = { 293 "Unmovable", 294 "Movable", 295 "Reclaimable", 296 "HighAtomic", 297 #ifdef CONFIG_CMA 298 "CMA", 299 #endif 300 #ifdef CONFIG_MEMORY_ISOLATION 301 "Isolate", 302 #endif 303 }; 304 305 compound_page_dtor * const compound_page_dtors[] = { 306 NULL, 307 free_compound_page, 308 #ifdef CONFIG_HUGETLB_PAGE 309 free_huge_page, 310 #endif 311 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 312 free_transhuge_page, 313 #endif 314 }; 315 316 int min_free_kbytes = 1024; 317 int user_min_free_kbytes = -1; 318 #ifdef CONFIG_DISCONTIGMEM 319 /* 320 * DiscontigMem defines memory ranges as separate pg_data_t even if the ranges 321 * are not on separate NUMA nodes. Functionally this works but with 322 * watermark_boost_factor, it can reclaim prematurely as the ranges can be 323 * quite small. By default, do not boost watermarks on discontigmem as in 324 * many cases very high-order allocations like THP are likely to be 325 * unsupported and the premature reclaim offsets the advantage of long-term 326 * fragmentation avoidance. 327 */ 328 int watermark_boost_factor __read_mostly; 329 #else 330 int watermark_boost_factor __read_mostly = 15000; 331 #endif 332 int watermark_scale_factor = 10; 333 334 static unsigned long nr_kernel_pages __initdata; 335 static unsigned long nr_all_pages __initdata; 336 static unsigned long dma_reserve __initdata; 337 338 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 339 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 340 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 341 static unsigned long required_kernelcore __initdata; 342 static unsigned long required_kernelcore_percent __initdata; 343 static unsigned long required_movablecore __initdata; 344 static unsigned long required_movablecore_percent __initdata; 345 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 346 static bool mirrored_kernelcore __meminitdata; 347 348 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 349 int movable_zone; 350 EXPORT_SYMBOL(movable_zone); 351 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 352 353 #if MAX_NUMNODES > 1 354 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; 355 unsigned int nr_online_nodes __read_mostly = 1; 356 EXPORT_SYMBOL(nr_node_ids); 357 EXPORT_SYMBOL(nr_online_nodes); 358 #endif 359 360 int page_group_by_mobility_disabled __read_mostly; 361 362 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 363 /* 364 * During boot we initialize deferred pages on-demand, as needed, but once 365 * page_alloc_init_late() has finished, the deferred pages are all initialized, 366 * and we can permanently disable that path. 367 */ 368 static DEFINE_STATIC_KEY_TRUE(deferred_pages); 369 370 /* 371 * Calling kasan_free_pages() only after deferred memory initialization 372 * has completed. Poisoning pages during deferred memory init will greatly 373 * lengthen the process and cause problem in large memory systems as the 374 * deferred pages initialization is done with interrupt disabled. 375 * 376 * Assuming that there will be no reference to those newly initialized 377 * pages before they are ever allocated, this should have no effect on 378 * KASAN memory tracking as the poison will be properly inserted at page 379 * allocation time. The only corner case is when pages are allocated by 380 * on-demand allocation and then freed again before the deferred pages 381 * initialization is done, but this is not likely to happen. 382 */ 383 static inline void kasan_free_nondeferred_pages(struct page *page, int order) 384 { 385 if (!static_branch_unlikely(&deferred_pages)) 386 kasan_free_pages(page, order); 387 } 388 389 /* Returns true if the struct page for the pfn is uninitialised */ 390 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 391 { 392 int nid = early_pfn_to_nid(pfn); 393 394 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 395 return true; 396 397 return false; 398 } 399 400 /* 401 * Returns true when the remaining initialisation should be deferred until 402 * later in the boot cycle when it can be parallelised. 403 */ 404 static bool __meminit 405 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 406 { 407 static unsigned long prev_end_pfn, nr_initialised; 408 409 /* 410 * prev_end_pfn static that contains the end of previous zone 411 * No need to protect because called very early in boot before smp_init. 412 */ 413 if (prev_end_pfn != end_pfn) { 414 prev_end_pfn = end_pfn; 415 nr_initialised = 0; 416 } 417 418 /* Always populate low zones for address-constrained allocations */ 419 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 420 return false; 421 422 /* 423 * We start only with one section of pages, more pages are added as 424 * needed until the rest of deferred pages are initialized. 425 */ 426 nr_initialised++; 427 if ((nr_initialised > PAGES_PER_SECTION) && 428 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 429 NODE_DATA(nid)->first_deferred_pfn = pfn; 430 return true; 431 } 432 return false; 433 } 434 #else 435 #define kasan_free_nondeferred_pages(p, o) kasan_free_pages(p, o) 436 437 static inline bool early_page_uninitialised(unsigned long pfn) 438 { 439 return false; 440 } 441 442 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 443 { 444 return false; 445 } 446 #endif 447 448 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 449 static inline unsigned long *get_pageblock_bitmap(struct page *page, 450 unsigned long pfn) 451 { 452 #ifdef CONFIG_SPARSEMEM 453 return section_to_usemap(__pfn_to_section(pfn)); 454 #else 455 return page_zone(page)->pageblock_flags; 456 #endif /* CONFIG_SPARSEMEM */ 457 } 458 459 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn) 460 { 461 #ifdef CONFIG_SPARSEMEM 462 pfn &= (PAGES_PER_SECTION-1); 463 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 464 #else 465 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 466 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 467 #endif /* CONFIG_SPARSEMEM */ 468 } 469 470 /** 471 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 472 * @page: The page within the block of interest 473 * @pfn: The target page frame number 474 * @end_bitidx: The last bit of interest to retrieve 475 * @mask: mask of bits that the caller is interested in 476 * 477 * Return: pageblock_bits flags 478 */ 479 static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page, 480 unsigned long pfn, 481 unsigned long end_bitidx, 482 unsigned long mask) 483 { 484 unsigned long *bitmap; 485 unsigned long bitidx, word_bitidx; 486 unsigned long word; 487 488 bitmap = get_pageblock_bitmap(page, pfn); 489 bitidx = pfn_to_bitidx(page, pfn); 490 word_bitidx = bitidx / BITS_PER_LONG; 491 bitidx &= (BITS_PER_LONG-1); 492 493 word = bitmap[word_bitidx]; 494 bitidx += end_bitidx; 495 return (word >> (BITS_PER_LONG - bitidx - 1)) & mask; 496 } 497 498 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, 499 unsigned long end_bitidx, 500 unsigned long mask) 501 { 502 return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask); 503 } 504 505 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn) 506 { 507 return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK); 508 } 509 510 /** 511 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 512 * @page: The page within the block of interest 513 * @flags: The flags to set 514 * @pfn: The target page frame number 515 * @end_bitidx: The last bit of interest 516 * @mask: mask of bits that the caller is interested in 517 */ 518 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 519 unsigned long pfn, 520 unsigned long end_bitidx, 521 unsigned long mask) 522 { 523 unsigned long *bitmap; 524 unsigned long bitidx, word_bitidx; 525 unsigned long old_word, word; 526 527 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 528 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits)); 529 530 bitmap = get_pageblock_bitmap(page, pfn); 531 bitidx = pfn_to_bitidx(page, pfn); 532 word_bitidx = bitidx / BITS_PER_LONG; 533 bitidx &= (BITS_PER_LONG-1); 534 535 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 536 537 bitidx += end_bitidx; 538 mask <<= (BITS_PER_LONG - bitidx - 1); 539 flags <<= (BITS_PER_LONG - bitidx - 1); 540 541 word = READ_ONCE(bitmap[word_bitidx]); 542 for (;;) { 543 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 544 if (word == old_word) 545 break; 546 word = old_word; 547 } 548 } 549 550 void set_pageblock_migratetype(struct page *page, int migratetype) 551 { 552 if (unlikely(page_group_by_mobility_disabled && 553 migratetype < MIGRATE_PCPTYPES)) 554 migratetype = MIGRATE_UNMOVABLE; 555 556 set_pageblock_flags_group(page, (unsigned long)migratetype, 557 PB_migrate, PB_migrate_end); 558 } 559 560 #ifdef CONFIG_DEBUG_VM 561 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 562 { 563 int ret = 0; 564 unsigned seq; 565 unsigned long pfn = page_to_pfn(page); 566 unsigned long sp, start_pfn; 567 568 do { 569 seq = zone_span_seqbegin(zone); 570 start_pfn = zone->zone_start_pfn; 571 sp = zone->spanned_pages; 572 if (!zone_spans_pfn(zone, pfn)) 573 ret = 1; 574 } while (zone_span_seqretry(zone, seq)); 575 576 if (ret) 577 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 578 pfn, zone_to_nid(zone), zone->name, 579 start_pfn, start_pfn + sp); 580 581 return ret; 582 } 583 584 static int page_is_consistent(struct zone *zone, struct page *page) 585 { 586 if (!pfn_valid_within(page_to_pfn(page))) 587 return 0; 588 if (zone != page_zone(page)) 589 return 0; 590 591 return 1; 592 } 593 /* 594 * Temporary debugging check for pages not lying within a given zone. 595 */ 596 static int __maybe_unused bad_range(struct zone *zone, struct page *page) 597 { 598 if (page_outside_zone_boundaries(zone, page)) 599 return 1; 600 if (!page_is_consistent(zone, page)) 601 return 1; 602 603 return 0; 604 } 605 #else 606 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page) 607 { 608 return 0; 609 } 610 #endif 611 612 static void bad_page(struct page *page, const char *reason, 613 unsigned long bad_flags) 614 { 615 static unsigned long resume; 616 static unsigned long nr_shown; 617 static unsigned long nr_unshown; 618 619 /* 620 * Allow a burst of 60 reports, then keep quiet for that minute; 621 * or allow a steady drip of one report per second. 622 */ 623 if (nr_shown == 60) { 624 if (time_before(jiffies, resume)) { 625 nr_unshown++; 626 goto out; 627 } 628 if (nr_unshown) { 629 pr_alert( 630 "BUG: Bad page state: %lu messages suppressed\n", 631 nr_unshown); 632 nr_unshown = 0; 633 } 634 nr_shown = 0; 635 } 636 if (nr_shown++ == 0) 637 resume = jiffies + 60 * HZ; 638 639 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 640 current->comm, page_to_pfn(page)); 641 __dump_page(page, reason); 642 bad_flags &= page->flags; 643 if (bad_flags) 644 pr_alert("bad because of flags: %#lx(%pGp)\n", 645 bad_flags, &bad_flags); 646 dump_page_owner(page); 647 648 print_modules(); 649 dump_stack(); 650 out: 651 /* Leave bad fields for debug, except PageBuddy could make trouble */ 652 page_mapcount_reset(page); /* remove PageBuddy */ 653 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 654 } 655 656 /* 657 * Higher-order pages are called "compound pages". They are structured thusly: 658 * 659 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 660 * 661 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 662 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 663 * 664 * The first tail page's ->compound_dtor holds the offset in array of compound 665 * page destructors. See compound_page_dtors. 666 * 667 * The first tail page's ->compound_order holds the order of allocation. 668 * This usage means that zero-order pages may not be compound. 669 */ 670 671 void free_compound_page(struct page *page) 672 { 673 __free_pages_ok(page, compound_order(page)); 674 } 675 676 void prep_compound_page(struct page *page, unsigned int order) 677 { 678 int i; 679 int nr_pages = 1 << order; 680 681 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 682 set_compound_order(page, order); 683 __SetPageHead(page); 684 for (i = 1; i < nr_pages; i++) { 685 struct page *p = page + i; 686 set_page_count(p, 0); 687 p->mapping = TAIL_MAPPING; 688 set_compound_head(p, page); 689 } 690 atomic_set(compound_mapcount_ptr(page), -1); 691 } 692 693 #ifdef CONFIG_DEBUG_PAGEALLOC 694 unsigned int _debug_guardpage_minorder; 695 696 #ifdef CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT 697 DEFINE_STATIC_KEY_TRUE(_debug_pagealloc_enabled); 698 #else 699 DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 700 #endif 701 EXPORT_SYMBOL(_debug_pagealloc_enabled); 702 703 DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 704 705 static int __init early_debug_pagealloc(char *buf) 706 { 707 bool enable = false; 708 709 if (kstrtobool(buf, &enable)) 710 return -EINVAL; 711 712 if (enable) 713 static_branch_enable(&_debug_pagealloc_enabled); 714 715 return 0; 716 } 717 early_param("debug_pagealloc", early_debug_pagealloc); 718 719 static void init_debug_guardpage(void) 720 { 721 if (!debug_pagealloc_enabled()) 722 return; 723 724 if (!debug_guardpage_minorder()) 725 return; 726 727 static_branch_enable(&_debug_guardpage_enabled); 728 } 729 730 static int __init debug_guardpage_minorder_setup(char *buf) 731 { 732 unsigned long res; 733 734 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 735 pr_err("Bad debug_guardpage_minorder value\n"); 736 return 0; 737 } 738 _debug_guardpage_minorder = res; 739 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 740 return 0; 741 } 742 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); 743 744 static inline bool set_page_guard(struct zone *zone, struct page *page, 745 unsigned int order, int migratetype) 746 { 747 if (!debug_guardpage_enabled()) 748 return false; 749 750 if (order >= debug_guardpage_minorder()) 751 return false; 752 753 __SetPageGuard(page); 754 INIT_LIST_HEAD(&page->lru); 755 set_page_private(page, order); 756 /* Guard pages are not available for any usage */ 757 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 758 759 return true; 760 } 761 762 static inline void clear_page_guard(struct zone *zone, struct page *page, 763 unsigned int order, int migratetype) 764 { 765 if (!debug_guardpage_enabled()) 766 return; 767 768 __ClearPageGuard(page); 769 770 set_page_private(page, 0); 771 if (!is_migrate_isolate(migratetype)) 772 __mod_zone_freepage_state(zone, (1 << order), migratetype); 773 } 774 #else 775 static inline bool set_page_guard(struct zone *zone, struct page *page, 776 unsigned int order, int migratetype) { return false; } 777 static inline void clear_page_guard(struct zone *zone, struct page *page, 778 unsigned int order, int migratetype) {} 779 #endif 780 781 static inline void set_page_order(struct page *page, unsigned int order) 782 { 783 set_page_private(page, order); 784 __SetPageBuddy(page); 785 } 786 787 /* 788 * This function checks whether a page is free && is the buddy 789 * we can coalesce a page and its buddy if 790 * (a) the buddy is not in a hole (check before calling!) && 791 * (b) the buddy is in the buddy system && 792 * (c) a page and its buddy have the same order && 793 * (d) a page and its buddy are in the same zone. 794 * 795 * For recording whether a page is in the buddy system, we set PageBuddy. 796 * Setting, clearing, and testing PageBuddy is serialized by zone->lock. 797 * 798 * For recording page's order, we use page_private(page). 799 */ 800 static inline int page_is_buddy(struct page *page, struct page *buddy, 801 unsigned int order) 802 { 803 if (page_is_guard(buddy) && page_order(buddy) == order) { 804 if (page_zone_id(page) != page_zone_id(buddy)) 805 return 0; 806 807 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 808 809 return 1; 810 } 811 812 if (PageBuddy(buddy) && page_order(buddy) == order) { 813 /* 814 * zone check is done late to avoid uselessly 815 * calculating zone/node ids for pages that could 816 * never merge. 817 */ 818 if (page_zone_id(page) != page_zone_id(buddy)) 819 return 0; 820 821 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 822 823 return 1; 824 } 825 return 0; 826 } 827 828 #ifdef CONFIG_COMPACTION 829 static inline struct capture_control *task_capc(struct zone *zone) 830 { 831 struct capture_control *capc = current->capture_control; 832 833 return capc && 834 !(current->flags & PF_KTHREAD) && 835 !capc->page && 836 capc->cc->zone == zone && 837 capc->cc->direct_compaction ? capc : NULL; 838 } 839 840 static inline bool 841 compaction_capture(struct capture_control *capc, struct page *page, 842 int order, int migratetype) 843 { 844 if (!capc || order != capc->cc->order) 845 return false; 846 847 /* Do not accidentally pollute CMA or isolated regions*/ 848 if (is_migrate_cma(migratetype) || 849 is_migrate_isolate(migratetype)) 850 return false; 851 852 /* 853 * Do not let lower order allocations polluate a movable pageblock. 854 * This might let an unmovable request use a reclaimable pageblock 855 * and vice-versa but no more than normal fallback logic which can 856 * have trouble finding a high-order free page. 857 */ 858 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE) 859 return false; 860 861 capc->page = page; 862 return true; 863 } 864 865 #else 866 static inline struct capture_control *task_capc(struct zone *zone) 867 { 868 return NULL; 869 } 870 871 static inline bool 872 compaction_capture(struct capture_control *capc, struct page *page, 873 int order, int migratetype) 874 { 875 return false; 876 } 877 #endif /* CONFIG_COMPACTION */ 878 879 /* 880 * Freeing function for a buddy system allocator. 881 * 882 * The concept of a buddy system is to maintain direct-mapped table 883 * (containing bit values) for memory blocks of various "orders". 884 * The bottom level table contains the map for the smallest allocatable 885 * units of memory (here, pages), and each level above it describes 886 * pairs of units from the levels below, hence, "buddies". 887 * At a high level, all that happens here is marking the table entry 888 * at the bottom level available, and propagating the changes upward 889 * as necessary, plus some accounting needed to play nicely with other 890 * parts of the VM system. 891 * At each level, we keep a list of pages, which are heads of continuous 892 * free pages of length of (1 << order) and marked with PageBuddy. 893 * Page's order is recorded in page_private(page) field. 894 * So when we are allocating or freeing one, we can derive the state of the 895 * other. That is, if we allocate a small block, and both were 896 * free, the remainder of the region must be split into blocks. 897 * If a block is freed, and its buddy is also free, then this 898 * triggers coalescing into a block of larger size. 899 * 900 * -- nyc 901 */ 902 903 static inline void __free_one_page(struct page *page, 904 unsigned long pfn, 905 struct zone *zone, unsigned int order, 906 int migratetype) 907 { 908 unsigned long combined_pfn; 909 unsigned long uninitialized_var(buddy_pfn); 910 struct page *buddy; 911 unsigned int max_order; 912 struct capture_control *capc = task_capc(zone); 913 914 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1); 915 916 VM_BUG_ON(!zone_is_initialized(zone)); 917 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 918 919 VM_BUG_ON(migratetype == -1); 920 if (likely(!is_migrate_isolate(migratetype))) 921 __mod_zone_freepage_state(zone, 1 << order, migratetype); 922 923 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); 924 VM_BUG_ON_PAGE(bad_range(zone, page), page); 925 926 continue_merging: 927 while (order < max_order - 1) { 928 if (compaction_capture(capc, page, order, migratetype)) { 929 __mod_zone_freepage_state(zone, -(1 << order), 930 migratetype); 931 return; 932 } 933 buddy_pfn = __find_buddy_pfn(pfn, order); 934 buddy = page + (buddy_pfn - pfn); 935 936 if (!pfn_valid_within(buddy_pfn)) 937 goto done_merging; 938 if (!page_is_buddy(page, buddy, order)) 939 goto done_merging; 940 /* 941 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 942 * merge with it and move up one order. 943 */ 944 if (page_is_guard(buddy)) 945 clear_page_guard(zone, buddy, order, migratetype); 946 else 947 del_page_from_free_area(buddy, &zone->free_area[order]); 948 combined_pfn = buddy_pfn & pfn; 949 page = page + (combined_pfn - pfn); 950 pfn = combined_pfn; 951 order++; 952 } 953 if (max_order < MAX_ORDER) { 954 /* If we are here, it means order is >= pageblock_order. 955 * We want to prevent merge between freepages on isolate 956 * pageblock and normal pageblock. Without this, pageblock 957 * isolation could cause incorrect freepage or CMA accounting. 958 * 959 * We don't want to hit this code for the more frequent 960 * low-order merging. 961 */ 962 if (unlikely(has_isolate_pageblock(zone))) { 963 int buddy_mt; 964 965 buddy_pfn = __find_buddy_pfn(pfn, order); 966 buddy = page + (buddy_pfn - pfn); 967 buddy_mt = get_pageblock_migratetype(buddy); 968 969 if (migratetype != buddy_mt 970 && (is_migrate_isolate(migratetype) || 971 is_migrate_isolate(buddy_mt))) 972 goto done_merging; 973 } 974 max_order++; 975 goto continue_merging; 976 } 977 978 done_merging: 979 set_page_order(page, order); 980 981 /* 982 * If this is not the largest possible page, check if the buddy 983 * of the next-highest order is free. If it is, it's possible 984 * that pages are being freed that will coalesce soon. In case, 985 * that is happening, add the free page to the tail of the list 986 * so it's less likely to be used soon and more likely to be merged 987 * as a higher order page 988 */ 989 if ((order < MAX_ORDER-2) && pfn_valid_within(buddy_pfn) 990 && !is_shuffle_order(order)) { 991 struct page *higher_page, *higher_buddy; 992 combined_pfn = buddy_pfn & pfn; 993 higher_page = page + (combined_pfn - pfn); 994 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1); 995 higher_buddy = higher_page + (buddy_pfn - combined_pfn); 996 if (pfn_valid_within(buddy_pfn) && 997 page_is_buddy(higher_page, higher_buddy, order + 1)) { 998 add_to_free_area_tail(page, &zone->free_area[order], 999 migratetype); 1000 return; 1001 } 1002 } 1003 1004 if (is_shuffle_order(order)) 1005 add_to_free_area_random(page, &zone->free_area[order], 1006 migratetype); 1007 else 1008 add_to_free_area(page, &zone->free_area[order], migratetype); 1009 1010 } 1011 1012 /* 1013 * A bad page could be due to a number of fields. Instead of multiple branches, 1014 * try and check multiple fields with one check. The caller must do a detailed 1015 * check if necessary. 1016 */ 1017 static inline bool page_expected_state(struct page *page, 1018 unsigned long check_flags) 1019 { 1020 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1021 return false; 1022 1023 if (unlikely((unsigned long)page->mapping | 1024 page_ref_count(page) | 1025 #ifdef CONFIG_MEMCG 1026 (unsigned long)page->mem_cgroup | 1027 #endif 1028 (page->flags & check_flags))) 1029 return false; 1030 1031 return true; 1032 } 1033 1034 static void free_pages_check_bad(struct page *page) 1035 { 1036 const char *bad_reason; 1037 unsigned long bad_flags; 1038 1039 bad_reason = NULL; 1040 bad_flags = 0; 1041 1042 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1043 bad_reason = "nonzero mapcount"; 1044 if (unlikely(page->mapping != NULL)) 1045 bad_reason = "non-NULL mapping"; 1046 if (unlikely(page_ref_count(page) != 0)) 1047 bad_reason = "nonzero _refcount"; 1048 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) { 1049 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 1050 bad_flags = PAGE_FLAGS_CHECK_AT_FREE; 1051 } 1052 #ifdef CONFIG_MEMCG 1053 if (unlikely(page->mem_cgroup)) 1054 bad_reason = "page still charged to cgroup"; 1055 #endif 1056 bad_page(page, bad_reason, bad_flags); 1057 } 1058 1059 static inline int free_pages_check(struct page *page) 1060 { 1061 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 1062 return 0; 1063 1064 /* Something has gone sideways, find it */ 1065 free_pages_check_bad(page); 1066 return 1; 1067 } 1068 1069 static int free_tail_pages_check(struct page *head_page, struct page *page) 1070 { 1071 int ret = 1; 1072 1073 /* 1074 * We rely page->lru.next never has bit 0 set, unless the page 1075 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 1076 */ 1077 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 1078 1079 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 1080 ret = 0; 1081 goto out; 1082 } 1083 switch (page - head_page) { 1084 case 1: 1085 /* the first tail page: ->mapping may be compound_mapcount() */ 1086 if (unlikely(compound_mapcount(page))) { 1087 bad_page(page, "nonzero compound_mapcount", 0); 1088 goto out; 1089 } 1090 break; 1091 case 2: 1092 /* 1093 * the second tail page: ->mapping is 1094 * deferred_list.next -- ignore value. 1095 */ 1096 break; 1097 default: 1098 if (page->mapping != TAIL_MAPPING) { 1099 bad_page(page, "corrupted mapping in tail page", 0); 1100 goto out; 1101 } 1102 break; 1103 } 1104 if (unlikely(!PageTail(page))) { 1105 bad_page(page, "PageTail not set", 0); 1106 goto out; 1107 } 1108 if (unlikely(compound_head(page) != head_page)) { 1109 bad_page(page, "compound_head not consistent", 0); 1110 goto out; 1111 } 1112 ret = 0; 1113 out: 1114 page->mapping = NULL; 1115 clear_compound_head(page); 1116 return ret; 1117 } 1118 1119 static void kernel_init_free_pages(struct page *page, int numpages) 1120 { 1121 int i; 1122 1123 for (i = 0; i < numpages; i++) 1124 clear_highpage(page + i); 1125 } 1126 1127 static __always_inline bool free_pages_prepare(struct page *page, 1128 unsigned int order, bool check_free) 1129 { 1130 int bad = 0; 1131 1132 VM_BUG_ON_PAGE(PageTail(page), page); 1133 1134 trace_mm_page_free(page, order); 1135 1136 /* 1137 * Check tail pages before head page information is cleared to 1138 * avoid checking PageCompound for order-0 pages. 1139 */ 1140 if (unlikely(order)) { 1141 bool compound = PageCompound(page); 1142 int i; 1143 1144 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1145 1146 if (compound) 1147 ClearPageDoubleMap(page); 1148 for (i = 1; i < (1 << order); i++) { 1149 if (compound) 1150 bad += free_tail_pages_check(page, page + i); 1151 if (unlikely(free_pages_check(page + i))) { 1152 bad++; 1153 continue; 1154 } 1155 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1156 } 1157 } 1158 if (PageMappingFlags(page)) 1159 page->mapping = NULL; 1160 if (memcg_kmem_enabled() && PageKmemcg(page)) 1161 __memcg_kmem_uncharge(page, order); 1162 if (check_free) 1163 bad += free_pages_check(page); 1164 if (bad) 1165 return false; 1166 1167 page_cpupid_reset_last(page); 1168 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1169 reset_page_owner(page, order); 1170 1171 if (!PageHighMem(page)) { 1172 debug_check_no_locks_freed(page_address(page), 1173 PAGE_SIZE << order); 1174 debug_check_no_obj_freed(page_address(page), 1175 PAGE_SIZE << order); 1176 } 1177 arch_free_page(page, order); 1178 if (want_init_on_free()) 1179 kernel_init_free_pages(page, 1 << order); 1180 1181 kernel_poison_pages(page, 1 << order, 0); 1182 if (debug_pagealloc_enabled()) 1183 kernel_map_pages(page, 1 << order, 0); 1184 1185 kasan_free_nondeferred_pages(page, order); 1186 1187 return true; 1188 } 1189 1190 #ifdef CONFIG_DEBUG_VM 1191 /* 1192 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed 1193 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when 1194 * moved from pcp lists to free lists. 1195 */ 1196 static bool free_pcp_prepare(struct page *page) 1197 { 1198 return free_pages_prepare(page, 0, true); 1199 } 1200 1201 static bool bulkfree_pcp_prepare(struct page *page) 1202 { 1203 if (debug_pagealloc_enabled()) 1204 return free_pages_check(page); 1205 else 1206 return false; 1207 } 1208 #else 1209 /* 1210 * With DEBUG_VM disabled, order-0 pages being freed are checked only when 1211 * moving from pcp lists to free list in order to reduce overhead. With 1212 * debug_pagealloc enabled, they are checked also immediately when being freed 1213 * to the pcp lists. 1214 */ 1215 static bool free_pcp_prepare(struct page *page) 1216 { 1217 if (debug_pagealloc_enabled()) 1218 return free_pages_prepare(page, 0, true); 1219 else 1220 return free_pages_prepare(page, 0, false); 1221 } 1222 1223 static bool bulkfree_pcp_prepare(struct page *page) 1224 { 1225 return free_pages_check(page); 1226 } 1227 #endif /* CONFIG_DEBUG_VM */ 1228 1229 static inline void prefetch_buddy(struct page *page) 1230 { 1231 unsigned long pfn = page_to_pfn(page); 1232 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0); 1233 struct page *buddy = page + (buddy_pfn - pfn); 1234 1235 prefetch(buddy); 1236 } 1237 1238 /* 1239 * Frees a number of pages from the PCP lists 1240 * Assumes all pages on list are in same zone, and of same order. 1241 * count is the number of pages to free. 1242 * 1243 * If the zone was previously in an "all pages pinned" state then look to 1244 * see if this freeing clears that state. 1245 * 1246 * And clear the zone's pages_scanned counter, to hold off the "all pages are 1247 * pinned" detection logic. 1248 */ 1249 static void free_pcppages_bulk(struct zone *zone, int count, 1250 struct per_cpu_pages *pcp) 1251 { 1252 int migratetype = 0; 1253 int batch_free = 0; 1254 int prefetch_nr = 0; 1255 bool isolated_pageblocks; 1256 struct page *page, *tmp; 1257 LIST_HEAD(head); 1258 1259 while (count) { 1260 struct list_head *list; 1261 1262 /* 1263 * Remove pages from lists in a round-robin fashion. A 1264 * batch_free count is maintained that is incremented when an 1265 * empty list is encountered. This is so more pages are freed 1266 * off fuller lists instead of spinning excessively around empty 1267 * lists 1268 */ 1269 do { 1270 batch_free++; 1271 if (++migratetype == MIGRATE_PCPTYPES) 1272 migratetype = 0; 1273 list = &pcp->lists[migratetype]; 1274 } while (list_empty(list)); 1275 1276 /* This is the only non-empty list. Free them all. */ 1277 if (batch_free == MIGRATE_PCPTYPES) 1278 batch_free = count; 1279 1280 do { 1281 page = list_last_entry(list, struct page, lru); 1282 /* must delete to avoid corrupting pcp list */ 1283 list_del(&page->lru); 1284 pcp->count--; 1285 1286 if (bulkfree_pcp_prepare(page)) 1287 continue; 1288 1289 list_add_tail(&page->lru, &head); 1290 1291 /* 1292 * We are going to put the page back to the global 1293 * pool, prefetch its buddy to speed up later access 1294 * under zone->lock. It is believed the overhead of 1295 * an additional test and calculating buddy_pfn here 1296 * can be offset by reduced memory latency later. To 1297 * avoid excessive prefetching due to large count, only 1298 * prefetch buddy for the first pcp->batch nr of pages. 1299 */ 1300 if (prefetch_nr++ < pcp->batch) 1301 prefetch_buddy(page); 1302 } while (--count && --batch_free && !list_empty(list)); 1303 } 1304 1305 spin_lock(&zone->lock); 1306 isolated_pageblocks = has_isolate_pageblock(zone); 1307 1308 /* 1309 * Use safe version since after __free_one_page(), 1310 * page->lru.next will not point to original list. 1311 */ 1312 list_for_each_entry_safe(page, tmp, &head, lru) { 1313 int mt = get_pcppage_migratetype(page); 1314 /* MIGRATE_ISOLATE page should not go to pcplists */ 1315 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1316 /* Pageblock could have been isolated meanwhile */ 1317 if (unlikely(isolated_pageblocks)) 1318 mt = get_pageblock_migratetype(page); 1319 1320 __free_one_page(page, page_to_pfn(page), zone, 0, mt); 1321 trace_mm_page_pcpu_drain(page, 0, mt); 1322 } 1323 spin_unlock(&zone->lock); 1324 } 1325 1326 static void free_one_page(struct zone *zone, 1327 struct page *page, unsigned long pfn, 1328 unsigned int order, 1329 int migratetype) 1330 { 1331 spin_lock(&zone->lock); 1332 if (unlikely(has_isolate_pageblock(zone) || 1333 is_migrate_isolate(migratetype))) { 1334 migratetype = get_pfnblock_migratetype(page, pfn); 1335 } 1336 __free_one_page(page, pfn, zone, order, migratetype); 1337 spin_unlock(&zone->lock); 1338 } 1339 1340 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1341 unsigned long zone, int nid) 1342 { 1343 mm_zero_struct_page(page); 1344 set_page_links(page, zone, nid, pfn); 1345 init_page_count(page); 1346 page_mapcount_reset(page); 1347 page_cpupid_reset_last(page); 1348 page_kasan_tag_reset(page); 1349 1350 INIT_LIST_HEAD(&page->lru); 1351 #ifdef WANT_PAGE_VIRTUAL 1352 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1353 if (!is_highmem_idx(zone)) 1354 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1355 #endif 1356 } 1357 1358 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1359 static void __meminit init_reserved_page(unsigned long pfn) 1360 { 1361 pg_data_t *pgdat; 1362 int nid, zid; 1363 1364 if (!early_page_uninitialised(pfn)) 1365 return; 1366 1367 nid = early_pfn_to_nid(pfn); 1368 pgdat = NODE_DATA(nid); 1369 1370 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1371 struct zone *zone = &pgdat->node_zones[zid]; 1372 1373 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) 1374 break; 1375 } 1376 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 1377 } 1378 #else 1379 static inline void init_reserved_page(unsigned long pfn) 1380 { 1381 } 1382 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1383 1384 /* 1385 * Initialised pages do not have PageReserved set. This function is 1386 * called for each range allocated by the bootmem allocator and 1387 * marks the pages PageReserved. The remaining valid pages are later 1388 * sent to the buddy page allocator. 1389 */ 1390 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1391 { 1392 unsigned long start_pfn = PFN_DOWN(start); 1393 unsigned long end_pfn = PFN_UP(end); 1394 1395 for (; start_pfn < end_pfn; start_pfn++) { 1396 if (pfn_valid(start_pfn)) { 1397 struct page *page = pfn_to_page(start_pfn); 1398 1399 init_reserved_page(start_pfn); 1400 1401 /* Avoid false-positive PageTail() */ 1402 INIT_LIST_HEAD(&page->lru); 1403 1404 /* 1405 * no need for atomic set_bit because the struct 1406 * page is not visible yet so nobody should 1407 * access it yet. 1408 */ 1409 __SetPageReserved(page); 1410 } 1411 } 1412 } 1413 1414 static void __free_pages_ok(struct page *page, unsigned int order) 1415 { 1416 unsigned long flags; 1417 int migratetype; 1418 unsigned long pfn = page_to_pfn(page); 1419 1420 if (!free_pages_prepare(page, order, true)) 1421 return; 1422 1423 migratetype = get_pfnblock_migratetype(page, pfn); 1424 local_irq_save(flags); 1425 __count_vm_events(PGFREE, 1 << order); 1426 free_one_page(page_zone(page), page, pfn, order, migratetype); 1427 local_irq_restore(flags); 1428 } 1429 1430 void __free_pages_core(struct page *page, unsigned int order) 1431 { 1432 unsigned int nr_pages = 1 << order; 1433 struct page *p = page; 1434 unsigned int loop; 1435 1436 prefetchw(p); 1437 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1438 prefetchw(p + 1); 1439 __ClearPageReserved(p); 1440 set_page_count(p, 0); 1441 } 1442 __ClearPageReserved(p); 1443 set_page_count(p, 0); 1444 1445 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); 1446 set_page_refcounted(page); 1447 __free_pages(page, order); 1448 } 1449 1450 #if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \ 1451 defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) 1452 1453 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1454 1455 int __meminit early_pfn_to_nid(unsigned long pfn) 1456 { 1457 static DEFINE_SPINLOCK(early_pfn_lock); 1458 int nid; 1459 1460 spin_lock(&early_pfn_lock); 1461 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1462 if (nid < 0) 1463 nid = first_online_node; 1464 spin_unlock(&early_pfn_lock); 1465 1466 return nid; 1467 } 1468 #endif 1469 1470 #ifdef CONFIG_NODES_SPAN_OTHER_NODES 1471 /* Only safe to use early in boot when initialisation is single-threaded */ 1472 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1473 { 1474 int nid; 1475 1476 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1477 if (nid >= 0 && nid != node) 1478 return false; 1479 return true; 1480 } 1481 1482 #else 1483 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1484 { 1485 return true; 1486 } 1487 #endif 1488 1489 1490 void __init memblock_free_pages(struct page *page, unsigned long pfn, 1491 unsigned int order) 1492 { 1493 if (early_page_uninitialised(pfn)) 1494 return; 1495 __free_pages_core(page, order); 1496 } 1497 1498 /* 1499 * Check that the whole (or subset of) a pageblock given by the interval of 1500 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1501 * with the migration of free compaction scanner. The scanners then need to 1502 * use only pfn_valid_within() check for arches that allow holes within 1503 * pageblocks. 1504 * 1505 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1506 * 1507 * It's possible on some configurations to have a setup like node0 node1 node0 1508 * i.e. it's possible that all pages within a zones range of pages do not 1509 * belong to a single zone. We assume that a border between node0 and node1 1510 * can occur within a single pageblock, but not a node0 node1 node0 1511 * interleaving within a single pageblock. It is therefore sufficient to check 1512 * the first and last page of a pageblock and avoid checking each individual 1513 * page in a pageblock. 1514 */ 1515 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1516 unsigned long end_pfn, struct zone *zone) 1517 { 1518 struct page *start_page; 1519 struct page *end_page; 1520 1521 /* end_pfn is one past the range we are checking */ 1522 end_pfn--; 1523 1524 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1525 return NULL; 1526 1527 start_page = pfn_to_online_page(start_pfn); 1528 if (!start_page) 1529 return NULL; 1530 1531 if (page_zone(start_page) != zone) 1532 return NULL; 1533 1534 end_page = pfn_to_page(end_pfn); 1535 1536 /* This gives a shorter code than deriving page_zone(end_page) */ 1537 if (page_zone_id(start_page) != page_zone_id(end_page)) 1538 return NULL; 1539 1540 return start_page; 1541 } 1542 1543 void set_zone_contiguous(struct zone *zone) 1544 { 1545 unsigned long block_start_pfn = zone->zone_start_pfn; 1546 unsigned long block_end_pfn; 1547 1548 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1549 for (; block_start_pfn < zone_end_pfn(zone); 1550 block_start_pfn = block_end_pfn, 1551 block_end_pfn += pageblock_nr_pages) { 1552 1553 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1554 1555 if (!__pageblock_pfn_to_page(block_start_pfn, 1556 block_end_pfn, zone)) 1557 return; 1558 } 1559 1560 /* We confirm that there is no hole */ 1561 zone->contiguous = true; 1562 } 1563 1564 void clear_zone_contiguous(struct zone *zone) 1565 { 1566 zone->contiguous = false; 1567 } 1568 1569 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1570 static void __init deferred_free_range(unsigned long pfn, 1571 unsigned long nr_pages) 1572 { 1573 struct page *page; 1574 unsigned long i; 1575 1576 if (!nr_pages) 1577 return; 1578 1579 page = pfn_to_page(pfn); 1580 1581 /* Free a large naturally-aligned chunk if possible */ 1582 if (nr_pages == pageblock_nr_pages && 1583 (pfn & (pageblock_nr_pages - 1)) == 0) { 1584 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1585 __free_pages_core(page, pageblock_order); 1586 return; 1587 } 1588 1589 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1590 if ((pfn & (pageblock_nr_pages - 1)) == 0) 1591 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1592 __free_pages_core(page, 0); 1593 } 1594 } 1595 1596 /* Completion tracking for deferred_init_memmap() threads */ 1597 static atomic_t pgdat_init_n_undone __initdata; 1598 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1599 1600 static inline void __init pgdat_init_report_one_done(void) 1601 { 1602 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1603 complete(&pgdat_init_all_done_comp); 1604 } 1605 1606 /* 1607 * Returns true if page needs to be initialized or freed to buddy allocator. 1608 * 1609 * First we check if pfn is valid on architectures where it is possible to have 1610 * holes within pageblock_nr_pages. On systems where it is not possible, this 1611 * function is optimized out. 1612 * 1613 * Then, we check if a current large page is valid by only checking the validity 1614 * of the head pfn. 1615 */ 1616 static inline bool __init deferred_pfn_valid(unsigned long pfn) 1617 { 1618 if (!pfn_valid_within(pfn)) 1619 return false; 1620 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn)) 1621 return false; 1622 return true; 1623 } 1624 1625 /* 1626 * Free pages to buddy allocator. Try to free aligned pages in 1627 * pageblock_nr_pages sizes. 1628 */ 1629 static void __init deferred_free_pages(unsigned long pfn, 1630 unsigned long end_pfn) 1631 { 1632 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1633 unsigned long nr_free = 0; 1634 1635 for (; pfn < end_pfn; pfn++) { 1636 if (!deferred_pfn_valid(pfn)) { 1637 deferred_free_range(pfn - nr_free, nr_free); 1638 nr_free = 0; 1639 } else if (!(pfn & nr_pgmask)) { 1640 deferred_free_range(pfn - nr_free, nr_free); 1641 nr_free = 1; 1642 touch_nmi_watchdog(); 1643 } else { 1644 nr_free++; 1645 } 1646 } 1647 /* Free the last block of pages to allocator */ 1648 deferred_free_range(pfn - nr_free, nr_free); 1649 } 1650 1651 /* 1652 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 1653 * by performing it only once every pageblock_nr_pages. 1654 * Return number of pages initialized. 1655 */ 1656 static unsigned long __init deferred_init_pages(struct zone *zone, 1657 unsigned long pfn, 1658 unsigned long end_pfn) 1659 { 1660 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1661 int nid = zone_to_nid(zone); 1662 unsigned long nr_pages = 0; 1663 int zid = zone_idx(zone); 1664 struct page *page = NULL; 1665 1666 for (; pfn < end_pfn; pfn++) { 1667 if (!deferred_pfn_valid(pfn)) { 1668 page = NULL; 1669 continue; 1670 } else if (!page || !(pfn & nr_pgmask)) { 1671 page = pfn_to_page(pfn); 1672 touch_nmi_watchdog(); 1673 } else { 1674 page++; 1675 } 1676 __init_single_page(page, pfn, zid, nid); 1677 nr_pages++; 1678 } 1679 return (nr_pages); 1680 } 1681 1682 /* 1683 * This function is meant to pre-load the iterator for the zone init. 1684 * Specifically it walks through the ranges until we are caught up to the 1685 * first_init_pfn value and exits there. If we never encounter the value we 1686 * return false indicating there are no valid ranges left. 1687 */ 1688 static bool __init 1689 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, 1690 unsigned long *spfn, unsigned long *epfn, 1691 unsigned long first_init_pfn) 1692 { 1693 u64 j; 1694 1695 /* 1696 * Start out by walking through the ranges in this zone that have 1697 * already been initialized. We don't need to do anything with them 1698 * so we just need to flush them out of the system. 1699 */ 1700 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { 1701 if (*epfn <= first_init_pfn) 1702 continue; 1703 if (*spfn < first_init_pfn) 1704 *spfn = first_init_pfn; 1705 *i = j; 1706 return true; 1707 } 1708 1709 return false; 1710 } 1711 1712 /* 1713 * Initialize and free pages. We do it in two loops: first we initialize 1714 * struct page, then free to buddy allocator, because while we are 1715 * freeing pages we can access pages that are ahead (computing buddy 1716 * page in __free_one_page()). 1717 * 1718 * In order to try and keep some memory in the cache we have the loop 1719 * broken along max page order boundaries. This way we will not cause 1720 * any issues with the buddy page computation. 1721 */ 1722 static unsigned long __init 1723 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, 1724 unsigned long *end_pfn) 1725 { 1726 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); 1727 unsigned long spfn = *start_pfn, epfn = *end_pfn; 1728 unsigned long nr_pages = 0; 1729 u64 j = *i; 1730 1731 /* First we loop through and initialize the page values */ 1732 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { 1733 unsigned long t; 1734 1735 if (mo_pfn <= *start_pfn) 1736 break; 1737 1738 t = min(mo_pfn, *end_pfn); 1739 nr_pages += deferred_init_pages(zone, *start_pfn, t); 1740 1741 if (mo_pfn < *end_pfn) { 1742 *start_pfn = mo_pfn; 1743 break; 1744 } 1745 } 1746 1747 /* Reset values and now loop through freeing pages as needed */ 1748 swap(j, *i); 1749 1750 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { 1751 unsigned long t; 1752 1753 if (mo_pfn <= spfn) 1754 break; 1755 1756 t = min(mo_pfn, epfn); 1757 deferred_free_pages(spfn, t); 1758 1759 if (mo_pfn <= epfn) 1760 break; 1761 } 1762 1763 return nr_pages; 1764 } 1765 1766 /* Initialise remaining memory on a node */ 1767 static int __init deferred_init_memmap(void *data) 1768 { 1769 pg_data_t *pgdat = data; 1770 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 1771 unsigned long spfn = 0, epfn = 0, nr_pages = 0; 1772 unsigned long first_init_pfn, flags; 1773 unsigned long start = jiffies; 1774 struct zone *zone; 1775 int zid; 1776 u64 i; 1777 1778 /* Bind memory initialisation thread to a local node if possible */ 1779 if (!cpumask_empty(cpumask)) 1780 set_cpus_allowed_ptr(current, cpumask); 1781 1782 pgdat_resize_lock(pgdat, &flags); 1783 first_init_pfn = pgdat->first_deferred_pfn; 1784 if (first_init_pfn == ULONG_MAX) { 1785 pgdat_resize_unlock(pgdat, &flags); 1786 pgdat_init_report_one_done(); 1787 return 0; 1788 } 1789 1790 /* Sanity check boundaries */ 1791 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 1792 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 1793 pgdat->first_deferred_pfn = ULONG_MAX; 1794 1795 /* Only the highest zone is deferred so find it */ 1796 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1797 zone = pgdat->node_zones + zid; 1798 if (first_init_pfn < zone_end_pfn(zone)) 1799 break; 1800 } 1801 1802 /* If the zone is empty somebody else may have cleared out the zone */ 1803 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 1804 first_init_pfn)) 1805 goto zone_empty; 1806 1807 /* 1808 * Initialize and free pages in MAX_ORDER sized increments so 1809 * that we can avoid introducing any issues with the buddy 1810 * allocator. 1811 */ 1812 while (spfn < epfn) 1813 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 1814 zone_empty: 1815 pgdat_resize_unlock(pgdat, &flags); 1816 1817 /* Sanity check that the next zone really is unpopulated */ 1818 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 1819 1820 pr_info("node %d initialised, %lu pages in %ums\n", 1821 pgdat->node_id, nr_pages, jiffies_to_msecs(jiffies - start)); 1822 1823 pgdat_init_report_one_done(); 1824 return 0; 1825 } 1826 1827 /* 1828 * If this zone has deferred pages, try to grow it by initializing enough 1829 * deferred pages to satisfy the allocation specified by order, rounded up to 1830 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 1831 * of SECTION_SIZE bytes by initializing struct pages in increments of 1832 * PAGES_PER_SECTION * sizeof(struct page) bytes. 1833 * 1834 * Return true when zone was grown, otherwise return false. We return true even 1835 * when we grow less than requested, to let the caller decide if there are 1836 * enough pages to satisfy the allocation. 1837 * 1838 * Note: We use noinline because this function is needed only during boot, and 1839 * it is called from a __ref function _deferred_grow_zone. This way we are 1840 * making sure that it is not inlined into permanent text section. 1841 */ 1842 static noinline bool __init 1843 deferred_grow_zone(struct zone *zone, unsigned int order) 1844 { 1845 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 1846 pg_data_t *pgdat = zone->zone_pgdat; 1847 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 1848 unsigned long spfn, epfn, flags; 1849 unsigned long nr_pages = 0; 1850 u64 i; 1851 1852 /* Only the last zone may have deferred pages */ 1853 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 1854 return false; 1855 1856 pgdat_resize_lock(pgdat, &flags); 1857 1858 /* 1859 * If deferred pages have been initialized while we were waiting for 1860 * the lock, return true, as the zone was grown. The caller will retry 1861 * this zone. We won't return to this function since the caller also 1862 * has this static branch. 1863 */ 1864 if (!static_branch_unlikely(&deferred_pages)) { 1865 pgdat_resize_unlock(pgdat, &flags); 1866 return true; 1867 } 1868 1869 /* 1870 * If someone grew this zone while we were waiting for spinlock, return 1871 * true, as there might be enough pages already. 1872 */ 1873 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 1874 pgdat_resize_unlock(pgdat, &flags); 1875 return true; 1876 } 1877 1878 /* If the zone is empty somebody else may have cleared out the zone */ 1879 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 1880 first_deferred_pfn)) { 1881 pgdat->first_deferred_pfn = ULONG_MAX; 1882 pgdat_resize_unlock(pgdat, &flags); 1883 /* Retry only once. */ 1884 return first_deferred_pfn != ULONG_MAX; 1885 } 1886 1887 /* 1888 * Initialize and free pages in MAX_ORDER sized increments so 1889 * that we can avoid introducing any issues with the buddy 1890 * allocator. 1891 */ 1892 while (spfn < epfn) { 1893 /* update our first deferred PFN for this section */ 1894 first_deferred_pfn = spfn; 1895 1896 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 1897 1898 /* We should only stop along section boundaries */ 1899 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) 1900 continue; 1901 1902 /* If our quota has been met we can stop here */ 1903 if (nr_pages >= nr_pages_needed) 1904 break; 1905 } 1906 1907 pgdat->first_deferred_pfn = spfn; 1908 pgdat_resize_unlock(pgdat, &flags); 1909 1910 return nr_pages > 0; 1911 } 1912 1913 /* 1914 * deferred_grow_zone() is __init, but it is called from 1915 * get_page_from_freelist() during early boot until deferred_pages permanently 1916 * disables this call. This is why we have refdata wrapper to avoid warning, 1917 * and to ensure that the function body gets unloaded. 1918 */ 1919 static bool __ref 1920 _deferred_grow_zone(struct zone *zone, unsigned int order) 1921 { 1922 return deferred_grow_zone(zone, order); 1923 } 1924 1925 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1926 1927 void __init page_alloc_init_late(void) 1928 { 1929 struct zone *zone; 1930 int nid; 1931 1932 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1933 1934 /* There will be num_node_state(N_MEMORY) threads */ 1935 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 1936 for_each_node_state(nid, N_MEMORY) { 1937 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 1938 } 1939 1940 /* Block until all are initialised */ 1941 wait_for_completion(&pgdat_init_all_done_comp); 1942 1943 /* 1944 * We initialized the rest of the deferred pages. Permanently disable 1945 * on-demand struct page initialization. 1946 */ 1947 static_branch_disable(&deferred_pages); 1948 1949 /* Reinit limits that are based on free pages after the kernel is up */ 1950 files_maxfiles_init(); 1951 #endif 1952 1953 /* Discard memblock private memory */ 1954 memblock_discard(); 1955 1956 for_each_node_state(nid, N_MEMORY) 1957 shuffle_free_memory(NODE_DATA(nid)); 1958 1959 for_each_populated_zone(zone) 1960 set_zone_contiguous(zone); 1961 1962 #ifdef CONFIG_DEBUG_PAGEALLOC 1963 init_debug_guardpage(); 1964 #endif 1965 } 1966 1967 #ifdef CONFIG_CMA 1968 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 1969 void __init init_cma_reserved_pageblock(struct page *page) 1970 { 1971 unsigned i = pageblock_nr_pages; 1972 struct page *p = page; 1973 1974 do { 1975 __ClearPageReserved(p); 1976 set_page_count(p, 0); 1977 } while (++p, --i); 1978 1979 set_pageblock_migratetype(page, MIGRATE_CMA); 1980 1981 if (pageblock_order >= MAX_ORDER) { 1982 i = pageblock_nr_pages; 1983 p = page; 1984 do { 1985 set_page_refcounted(p); 1986 __free_pages(p, MAX_ORDER - 1); 1987 p += MAX_ORDER_NR_PAGES; 1988 } while (i -= MAX_ORDER_NR_PAGES); 1989 } else { 1990 set_page_refcounted(page); 1991 __free_pages(page, pageblock_order); 1992 } 1993 1994 adjust_managed_page_count(page, pageblock_nr_pages); 1995 } 1996 #endif 1997 1998 /* 1999 * The order of subdivision here is critical for the IO subsystem. 2000 * Please do not alter this order without good reasons and regression 2001 * testing. Specifically, as large blocks of memory are subdivided, 2002 * the order in which smaller blocks are delivered depends on the order 2003 * they're subdivided in this function. This is the primary factor 2004 * influencing the order in which pages are delivered to the IO 2005 * subsystem according to empirical testing, and this is also justified 2006 * by considering the behavior of a buddy system containing a single 2007 * large block of memory acted on by a series of small allocations. 2008 * This behavior is a critical factor in sglist merging's success. 2009 * 2010 * -- nyc 2011 */ 2012 static inline void expand(struct zone *zone, struct page *page, 2013 int low, int high, struct free_area *area, 2014 int migratetype) 2015 { 2016 unsigned long size = 1 << high; 2017 2018 while (high > low) { 2019 area--; 2020 high--; 2021 size >>= 1; 2022 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 2023 2024 /* 2025 * Mark as guard pages (or page), that will allow to 2026 * merge back to allocator when buddy will be freed. 2027 * Corresponding page table entries will not be touched, 2028 * pages will stay not present in virtual address space 2029 */ 2030 if (set_page_guard(zone, &page[size], high, migratetype)) 2031 continue; 2032 2033 add_to_free_area(&page[size], area, migratetype); 2034 set_page_order(&page[size], high); 2035 } 2036 } 2037 2038 static void check_new_page_bad(struct page *page) 2039 { 2040 const char *bad_reason = NULL; 2041 unsigned long bad_flags = 0; 2042 2043 if (unlikely(atomic_read(&page->_mapcount) != -1)) 2044 bad_reason = "nonzero mapcount"; 2045 if (unlikely(page->mapping != NULL)) 2046 bad_reason = "non-NULL mapping"; 2047 if (unlikely(page_ref_count(page) != 0)) 2048 bad_reason = "nonzero _refcount"; 2049 if (unlikely(page->flags & __PG_HWPOISON)) { 2050 bad_reason = "HWPoisoned (hardware-corrupted)"; 2051 bad_flags = __PG_HWPOISON; 2052 /* Don't complain about hwpoisoned pages */ 2053 page_mapcount_reset(page); /* remove PageBuddy */ 2054 return; 2055 } 2056 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) { 2057 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set"; 2058 bad_flags = PAGE_FLAGS_CHECK_AT_PREP; 2059 } 2060 #ifdef CONFIG_MEMCG 2061 if (unlikely(page->mem_cgroup)) 2062 bad_reason = "page still charged to cgroup"; 2063 #endif 2064 bad_page(page, bad_reason, bad_flags); 2065 } 2066 2067 /* 2068 * This page is about to be returned from the page allocator 2069 */ 2070 static inline int check_new_page(struct page *page) 2071 { 2072 if (likely(page_expected_state(page, 2073 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 2074 return 0; 2075 2076 check_new_page_bad(page); 2077 return 1; 2078 } 2079 2080 static inline bool free_pages_prezeroed(void) 2081 { 2082 return (IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) && 2083 page_poisoning_enabled()) || want_init_on_free(); 2084 } 2085 2086 #ifdef CONFIG_DEBUG_VM 2087 /* 2088 * With DEBUG_VM enabled, order-0 pages are checked for expected state when 2089 * being allocated from pcp lists. With debug_pagealloc also enabled, they are 2090 * also checked when pcp lists are refilled from the free lists. 2091 */ 2092 static inline bool check_pcp_refill(struct page *page) 2093 { 2094 if (debug_pagealloc_enabled()) 2095 return check_new_page(page); 2096 else 2097 return false; 2098 } 2099 2100 static inline bool check_new_pcp(struct page *page) 2101 { 2102 return check_new_page(page); 2103 } 2104 #else 2105 /* 2106 * With DEBUG_VM disabled, free order-0 pages are checked for expected state 2107 * when pcp lists are being refilled from the free lists. With debug_pagealloc 2108 * enabled, they are also checked when being allocated from the pcp lists. 2109 */ 2110 static inline bool check_pcp_refill(struct page *page) 2111 { 2112 return check_new_page(page); 2113 } 2114 static inline bool check_new_pcp(struct page *page) 2115 { 2116 if (debug_pagealloc_enabled()) 2117 return check_new_page(page); 2118 else 2119 return false; 2120 } 2121 #endif /* CONFIG_DEBUG_VM */ 2122 2123 static bool check_new_pages(struct page *page, unsigned int order) 2124 { 2125 int i; 2126 for (i = 0; i < (1 << order); i++) { 2127 struct page *p = page + i; 2128 2129 if (unlikely(check_new_page(p))) 2130 return true; 2131 } 2132 2133 return false; 2134 } 2135 2136 inline void post_alloc_hook(struct page *page, unsigned int order, 2137 gfp_t gfp_flags) 2138 { 2139 set_page_private(page, 0); 2140 set_page_refcounted(page); 2141 2142 arch_alloc_page(page, order); 2143 if (debug_pagealloc_enabled()) 2144 kernel_map_pages(page, 1 << order, 1); 2145 kasan_alloc_pages(page, order); 2146 kernel_poison_pages(page, 1 << order, 1); 2147 set_page_owner(page, order, gfp_flags); 2148 } 2149 2150 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 2151 unsigned int alloc_flags) 2152 { 2153 post_alloc_hook(page, order, gfp_flags); 2154 2155 if (!free_pages_prezeroed() && want_init_on_alloc(gfp_flags)) 2156 kernel_init_free_pages(page, 1 << order); 2157 2158 if (order && (gfp_flags & __GFP_COMP)) 2159 prep_compound_page(page, order); 2160 2161 /* 2162 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 2163 * allocate the page. The expectation is that the caller is taking 2164 * steps that will free more memory. The caller should avoid the page 2165 * being used for !PFMEMALLOC purposes. 2166 */ 2167 if (alloc_flags & ALLOC_NO_WATERMARKS) 2168 set_page_pfmemalloc(page); 2169 else 2170 clear_page_pfmemalloc(page); 2171 } 2172 2173 /* 2174 * Go through the free lists for the given migratetype and remove 2175 * the smallest available page from the freelists 2176 */ 2177 static __always_inline 2178 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 2179 int migratetype) 2180 { 2181 unsigned int current_order; 2182 struct free_area *area; 2183 struct page *page; 2184 2185 /* Find a page of the appropriate size in the preferred list */ 2186 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 2187 area = &(zone->free_area[current_order]); 2188 page = get_page_from_free_area(area, migratetype); 2189 if (!page) 2190 continue; 2191 del_page_from_free_area(page, area); 2192 expand(zone, page, order, current_order, area, migratetype); 2193 set_pcppage_migratetype(page, migratetype); 2194 return page; 2195 } 2196 2197 return NULL; 2198 } 2199 2200 2201 /* 2202 * This array describes the order lists are fallen back to when 2203 * the free lists for the desirable migrate type are depleted 2204 */ 2205 static int fallbacks[MIGRATE_TYPES][4] = { 2206 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2207 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 2208 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2209 #ifdef CONFIG_CMA 2210 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */ 2211 #endif 2212 #ifdef CONFIG_MEMORY_ISOLATION 2213 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */ 2214 #endif 2215 }; 2216 2217 #ifdef CONFIG_CMA 2218 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2219 unsigned int order) 2220 { 2221 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 2222 } 2223 #else 2224 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2225 unsigned int order) { return NULL; } 2226 #endif 2227 2228 /* 2229 * Move the free pages in a range to the free lists of the requested type. 2230 * Note that start_page and end_pages are not aligned on a pageblock 2231 * boundary. If alignment is required, use move_freepages_block() 2232 */ 2233 static int move_freepages(struct zone *zone, 2234 struct page *start_page, struct page *end_page, 2235 int migratetype, int *num_movable) 2236 { 2237 struct page *page; 2238 unsigned int order; 2239 int pages_moved = 0; 2240 2241 for (page = start_page; page <= end_page;) { 2242 if (!pfn_valid_within(page_to_pfn(page))) { 2243 page++; 2244 continue; 2245 } 2246 2247 if (!PageBuddy(page)) { 2248 /* 2249 * We assume that pages that could be isolated for 2250 * migration are movable. But we don't actually try 2251 * isolating, as that would be expensive. 2252 */ 2253 if (num_movable && 2254 (PageLRU(page) || __PageMovable(page))) 2255 (*num_movable)++; 2256 2257 page++; 2258 continue; 2259 } 2260 2261 /* Make sure we are not inadvertently changing nodes */ 2262 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2263 VM_BUG_ON_PAGE(page_zone(page) != zone, page); 2264 2265 order = page_order(page); 2266 move_to_free_area(page, &zone->free_area[order], migratetype); 2267 page += 1 << order; 2268 pages_moved += 1 << order; 2269 } 2270 2271 return pages_moved; 2272 } 2273 2274 int move_freepages_block(struct zone *zone, struct page *page, 2275 int migratetype, int *num_movable) 2276 { 2277 unsigned long start_pfn, end_pfn; 2278 struct page *start_page, *end_page; 2279 2280 if (num_movable) 2281 *num_movable = 0; 2282 2283 start_pfn = page_to_pfn(page); 2284 start_pfn = start_pfn & ~(pageblock_nr_pages-1); 2285 start_page = pfn_to_page(start_pfn); 2286 end_page = start_page + pageblock_nr_pages - 1; 2287 end_pfn = start_pfn + pageblock_nr_pages - 1; 2288 2289 /* Do not cross zone boundaries */ 2290 if (!zone_spans_pfn(zone, start_pfn)) 2291 start_page = page; 2292 if (!zone_spans_pfn(zone, end_pfn)) 2293 return 0; 2294 2295 return move_freepages(zone, start_page, end_page, migratetype, 2296 num_movable); 2297 } 2298 2299 static void change_pageblock_range(struct page *pageblock_page, 2300 int start_order, int migratetype) 2301 { 2302 int nr_pageblocks = 1 << (start_order - pageblock_order); 2303 2304 while (nr_pageblocks--) { 2305 set_pageblock_migratetype(pageblock_page, migratetype); 2306 pageblock_page += pageblock_nr_pages; 2307 } 2308 } 2309 2310 /* 2311 * When we are falling back to another migratetype during allocation, try to 2312 * steal extra free pages from the same pageblocks to satisfy further 2313 * allocations, instead of polluting multiple pageblocks. 2314 * 2315 * If we are stealing a relatively large buddy page, it is likely there will 2316 * be more free pages in the pageblock, so try to steal them all. For 2317 * reclaimable and unmovable allocations, we steal regardless of page size, 2318 * as fragmentation caused by those allocations polluting movable pageblocks 2319 * is worse than movable allocations stealing from unmovable and reclaimable 2320 * pageblocks. 2321 */ 2322 static bool can_steal_fallback(unsigned int order, int start_mt) 2323 { 2324 /* 2325 * Leaving this order check is intended, although there is 2326 * relaxed order check in next check. The reason is that 2327 * we can actually steal whole pageblock if this condition met, 2328 * but, below check doesn't guarantee it and that is just heuristic 2329 * so could be changed anytime. 2330 */ 2331 if (order >= pageblock_order) 2332 return true; 2333 2334 if (order >= pageblock_order / 2 || 2335 start_mt == MIGRATE_RECLAIMABLE || 2336 start_mt == MIGRATE_UNMOVABLE || 2337 page_group_by_mobility_disabled) 2338 return true; 2339 2340 return false; 2341 } 2342 2343 static inline void boost_watermark(struct zone *zone) 2344 { 2345 unsigned long max_boost; 2346 2347 if (!watermark_boost_factor) 2348 return; 2349 2350 max_boost = mult_frac(zone->_watermark[WMARK_HIGH], 2351 watermark_boost_factor, 10000); 2352 2353 /* 2354 * high watermark may be uninitialised if fragmentation occurs 2355 * very early in boot so do not boost. We do not fall 2356 * through and boost by pageblock_nr_pages as failing 2357 * allocations that early means that reclaim is not going 2358 * to help and it may even be impossible to reclaim the 2359 * boosted watermark resulting in a hang. 2360 */ 2361 if (!max_boost) 2362 return; 2363 2364 max_boost = max(pageblock_nr_pages, max_boost); 2365 2366 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages, 2367 max_boost); 2368 } 2369 2370 /* 2371 * This function implements actual steal behaviour. If order is large enough, 2372 * we can steal whole pageblock. If not, we first move freepages in this 2373 * pageblock to our migratetype and determine how many already-allocated pages 2374 * are there in the pageblock with a compatible migratetype. If at least half 2375 * of pages are free or compatible, we can change migratetype of the pageblock 2376 * itself, so pages freed in the future will be put on the correct free list. 2377 */ 2378 static void steal_suitable_fallback(struct zone *zone, struct page *page, 2379 unsigned int alloc_flags, int start_type, bool whole_block) 2380 { 2381 unsigned int current_order = page_order(page); 2382 struct free_area *area; 2383 int free_pages, movable_pages, alike_pages; 2384 int old_block_type; 2385 2386 old_block_type = get_pageblock_migratetype(page); 2387 2388 /* 2389 * This can happen due to races and we want to prevent broken 2390 * highatomic accounting. 2391 */ 2392 if (is_migrate_highatomic(old_block_type)) 2393 goto single_page; 2394 2395 /* Take ownership for orders >= pageblock_order */ 2396 if (current_order >= pageblock_order) { 2397 change_pageblock_range(page, current_order, start_type); 2398 goto single_page; 2399 } 2400 2401 /* 2402 * Boost watermarks to increase reclaim pressure to reduce the 2403 * likelihood of future fallbacks. Wake kswapd now as the node 2404 * may be balanced overall and kswapd will not wake naturally. 2405 */ 2406 boost_watermark(zone); 2407 if (alloc_flags & ALLOC_KSWAPD) 2408 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 2409 2410 /* We are not allowed to try stealing from the whole block */ 2411 if (!whole_block) 2412 goto single_page; 2413 2414 free_pages = move_freepages_block(zone, page, start_type, 2415 &movable_pages); 2416 /* 2417 * Determine how many pages are compatible with our allocation. 2418 * For movable allocation, it's the number of movable pages which 2419 * we just obtained. For other types it's a bit more tricky. 2420 */ 2421 if (start_type == MIGRATE_MOVABLE) { 2422 alike_pages = movable_pages; 2423 } else { 2424 /* 2425 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation 2426 * to MOVABLE pageblock, consider all non-movable pages as 2427 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or 2428 * vice versa, be conservative since we can't distinguish the 2429 * exact migratetype of non-movable pages. 2430 */ 2431 if (old_block_type == MIGRATE_MOVABLE) 2432 alike_pages = pageblock_nr_pages 2433 - (free_pages + movable_pages); 2434 else 2435 alike_pages = 0; 2436 } 2437 2438 /* moving whole block can fail due to zone boundary conditions */ 2439 if (!free_pages) 2440 goto single_page; 2441 2442 /* 2443 * If a sufficient number of pages in the block are either free or of 2444 * comparable migratability as our allocation, claim the whole block. 2445 */ 2446 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || 2447 page_group_by_mobility_disabled) 2448 set_pageblock_migratetype(page, start_type); 2449 2450 return; 2451 2452 single_page: 2453 area = &zone->free_area[current_order]; 2454 move_to_free_area(page, area, start_type); 2455 } 2456 2457 /* 2458 * Check whether there is a suitable fallback freepage with requested order. 2459 * If only_stealable is true, this function returns fallback_mt only if 2460 * we can steal other freepages all together. This would help to reduce 2461 * fragmentation due to mixed migratetype pages in one pageblock. 2462 */ 2463 int find_suitable_fallback(struct free_area *area, unsigned int order, 2464 int migratetype, bool only_stealable, bool *can_steal) 2465 { 2466 int i; 2467 int fallback_mt; 2468 2469 if (area->nr_free == 0) 2470 return -1; 2471 2472 *can_steal = false; 2473 for (i = 0;; i++) { 2474 fallback_mt = fallbacks[migratetype][i]; 2475 if (fallback_mt == MIGRATE_TYPES) 2476 break; 2477 2478 if (free_area_empty(area, fallback_mt)) 2479 continue; 2480 2481 if (can_steal_fallback(order, migratetype)) 2482 *can_steal = true; 2483 2484 if (!only_stealable) 2485 return fallback_mt; 2486 2487 if (*can_steal) 2488 return fallback_mt; 2489 } 2490 2491 return -1; 2492 } 2493 2494 /* 2495 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2496 * there are no empty page blocks that contain a page with a suitable order 2497 */ 2498 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2499 unsigned int alloc_order) 2500 { 2501 int mt; 2502 unsigned long max_managed, flags; 2503 2504 /* 2505 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2506 * Check is race-prone but harmless. 2507 */ 2508 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages; 2509 if (zone->nr_reserved_highatomic >= max_managed) 2510 return; 2511 2512 spin_lock_irqsave(&zone->lock, flags); 2513 2514 /* Recheck the nr_reserved_highatomic limit under the lock */ 2515 if (zone->nr_reserved_highatomic >= max_managed) 2516 goto out_unlock; 2517 2518 /* Yoink! */ 2519 mt = get_pageblock_migratetype(page); 2520 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt) 2521 && !is_migrate_cma(mt)) { 2522 zone->nr_reserved_highatomic += pageblock_nr_pages; 2523 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2524 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); 2525 } 2526 2527 out_unlock: 2528 spin_unlock_irqrestore(&zone->lock, flags); 2529 } 2530 2531 /* 2532 * Used when an allocation is about to fail under memory pressure. This 2533 * potentially hurts the reliability of high-order allocations when under 2534 * intense memory pressure but failed atomic allocations should be easier 2535 * to recover from than an OOM. 2536 * 2537 * If @force is true, try to unreserve a pageblock even though highatomic 2538 * pageblock is exhausted. 2539 */ 2540 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 2541 bool force) 2542 { 2543 struct zonelist *zonelist = ac->zonelist; 2544 unsigned long flags; 2545 struct zoneref *z; 2546 struct zone *zone; 2547 struct page *page; 2548 int order; 2549 bool ret; 2550 2551 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx, 2552 ac->nodemask) { 2553 /* 2554 * Preserve at least one pageblock unless memory pressure 2555 * is really high. 2556 */ 2557 if (!force && zone->nr_reserved_highatomic <= 2558 pageblock_nr_pages) 2559 continue; 2560 2561 spin_lock_irqsave(&zone->lock, flags); 2562 for (order = 0; order < MAX_ORDER; order++) { 2563 struct free_area *area = &(zone->free_area[order]); 2564 2565 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); 2566 if (!page) 2567 continue; 2568 2569 /* 2570 * In page freeing path, migratetype change is racy so 2571 * we can counter several free pages in a pageblock 2572 * in this loop althoug we changed the pageblock type 2573 * from highatomic to ac->migratetype. So we should 2574 * adjust the count once. 2575 */ 2576 if (is_migrate_highatomic_page(page)) { 2577 /* 2578 * It should never happen but changes to 2579 * locking could inadvertently allow a per-cpu 2580 * drain to add pages to MIGRATE_HIGHATOMIC 2581 * while unreserving so be safe and watch for 2582 * underflows. 2583 */ 2584 zone->nr_reserved_highatomic -= min( 2585 pageblock_nr_pages, 2586 zone->nr_reserved_highatomic); 2587 } 2588 2589 /* 2590 * Convert to ac->migratetype and avoid the normal 2591 * pageblock stealing heuristics. Minimally, the caller 2592 * is doing the work and needs the pages. More 2593 * importantly, if the block was always converted to 2594 * MIGRATE_UNMOVABLE or another type then the number 2595 * of pageblocks that cannot be completely freed 2596 * may increase. 2597 */ 2598 set_pageblock_migratetype(page, ac->migratetype); 2599 ret = move_freepages_block(zone, page, ac->migratetype, 2600 NULL); 2601 if (ret) { 2602 spin_unlock_irqrestore(&zone->lock, flags); 2603 return ret; 2604 } 2605 } 2606 spin_unlock_irqrestore(&zone->lock, flags); 2607 } 2608 2609 return false; 2610 } 2611 2612 /* 2613 * Try finding a free buddy page on the fallback list and put it on the free 2614 * list of requested migratetype, possibly along with other pages from the same 2615 * block, depending on fragmentation avoidance heuristics. Returns true if 2616 * fallback was found so that __rmqueue_smallest() can grab it. 2617 * 2618 * The use of signed ints for order and current_order is a deliberate 2619 * deviation from the rest of this file, to make the for loop 2620 * condition simpler. 2621 */ 2622 static __always_inline bool 2623 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, 2624 unsigned int alloc_flags) 2625 { 2626 struct free_area *area; 2627 int current_order; 2628 int min_order = order; 2629 struct page *page; 2630 int fallback_mt; 2631 bool can_steal; 2632 2633 /* 2634 * Do not steal pages from freelists belonging to other pageblocks 2635 * i.e. orders < pageblock_order. If there are no local zones free, 2636 * the zonelists will be reiterated without ALLOC_NOFRAGMENT. 2637 */ 2638 if (alloc_flags & ALLOC_NOFRAGMENT) 2639 min_order = pageblock_order; 2640 2641 /* 2642 * Find the largest available free page in the other list. This roughly 2643 * approximates finding the pageblock with the most free pages, which 2644 * would be too costly to do exactly. 2645 */ 2646 for (current_order = MAX_ORDER - 1; current_order >= min_order; 2647 --current_order) { 2648 area = &(zone->free_area[current_order]); 2649 fallback_mt = find_suitable_fallback(area, current_order, 2650 start_migratetype, false, &can_steal); 2651 if (fallback_mt == -1) 2652 continue; 2653 2654 /* 2655 * We cannot steal all free pages from the pageblock and the 2656 * requested migratetype is movable. In that case it's better to 2657 * steal and split the smallest available page instead of the 2658 * largest available page, because even if the next movable 2659 * allocation falls back into a different pageblock than this 2660 * one, it won't cause permanent fragmentation. 2661 */ 2662 if (!can_steal && start_migratetype == MIGRATE_MOVABLE 2663 && current_order > order) 2664 goto find_smallest; 2665 2666 goto do_steal; 2667 } 2668 2669 return false; 2670 2671 find_smallest: 2672 for (current_order = order; current_order < MAX_ORDER; 2673 current_order++) { 2674 area = &(zone->free_area[current_order]); 2675 fallback_mt = find_suitable_fallback(area, current_order, 2676 start_migratetype, false, &can_steal); 2677 if (fallback_mt != -1) 2678 break; 2679 } 2680 2681 /* 2682 * This should not happen - we already found a suitable fallback 2683 * when looking for the largest page. 2684 */ 2685 VM_BUG_ON(current_order == MAX_ORDER); 2686 2687 do_steal: 2688 page = get_page_from_free_area(area, fallback_mt); 2689 2690 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype, 2691 can_steal); 2692 2693 trace_mm_page_alloc_extfrag(page, order, current_order, 2694 start_migratetype, fallback_mt); 2695 2696 return true; 2697 2698 } 2699 2700 /* 2701 * Do the hard work of removing an element from the buddy allocator. 2702 * Call me with the zone->lock already held. 2703 */ 2704 static __always_inline struct page * 2705 __rmqueue(struct zone *zone, unsigned int order, int migratetype, 2706 unsigned int alloc_flags) 2707 { 2708 struct page *page; 2709 2710 retry: 2711 page = __rmqueue_smallest(zone, order, migratetype); 2712 if (unlikely(!page)) { 2713 if (migratetype == MIGRATE_MOVABLE) 2714 page = __rmqueue_cma_fallback(zone, order); 2715 2716 if (!page && __rmqueue_fallback(zone, order, migratetype, 2717 alloc_flags)) 2718 goto retry; 2719 } 2720 2721 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2722 return page; 2723 } 2724 2725 /* 2726 * Obtain a specified number of elements from the buddy allocator, all under 2727 * a single hold of the lock, for efficiency. Add them to the supplied list. 2728 * Returns the number of new pages which were placed at *list. 2729 */ 2730 static int rmqueue_bulk(struct zone *zone, unsigned int order, 2731 unsigned long count, struct list_head *list, 2732 int migratetype, unsigned int alloc_flags) 2733 { 2734 int i, alloced = 0; 2735 2736 spin_lock(&zone->lock); 2737 for (i = 0; i < count; ++i) { 2738 struct page *page = __rmqueue(zone, order, migratetype, 2739 alloc_flags); 2740 if (unlikely(page == NULL)) 2741 break; 2742 2743 if (unlikely(check_pcp_refill(page))) 2744 continue; 2745 2746 /* 2747 * Split buddy pages returned by expand() are received here in 2748 * physical page order. The page is added to the tail of 2749 * caller's list. From the callers perspective, the linked list 2750 * is ordered by page number under some conditions. This is 2751 * useful for IO devices that can forward direction from the 2752 * head, thus also in the physical page order. This is useful 2753 * for IO devices that can merge IO requests if the physical 2754 * pages are ordered properly. 2755 */ 2756 list_add_tail(&page->lru, list); 2757 alloced++; 2758 if (is_migrate_cma(get_pcppage_migratetype(page))) 2759 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 2760 -(1 << order)); 2761 } 2762 2763 /* 2764 * i pages were removed from the buddy list even if some leak due 2765 * to check_pcp_refill failing so adjust NR_FREE_PAGES based 2766 * on i. Do not confuse with 'alloced' which is the number of 2767 * pages added to the pcp list. 2768 */ 2769 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 2770 spin_unlock(&zone->lock); 2771 return alloced; 2772 } 2773 2774 #ifdef CONFIG_NUMA 2775 /* 2776 * Called from the vmstat counter updater to drain pagesets of this 2777 * currently executing processor on remote nodes after they have 2778 * expired. 2779 * 2780 * Note that this function must be called with the thread pinned to 2781 * a single processor. 2782 */ 2783 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 2784 { 2785 unsigned long flags; 2786 int to_drain, batch; 2787 2788 local_irq_save(flags); 2789 batch = READ_ONCE(pcp->batch); 2790 to_drain = min(pcp->count, batch); 2791 if (to_drain > 0) 2792 free_pcppages_bulk(zone, to_drain, pcp); 2793 local_irq_restore(flags); 2794 } 2795 #endif 2796 2797 /* 2798 * Drain pcplists of the indicated processor and zone. 2799 * 2800 * The processor must either be the current processor and the 2801 * thread pinned to the current processor or a processor that 2802 * is not online. 2803 */ 2804 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 2805 { 2806 unsigned long flags; 2807 struct per_cpu_pageset *pset; 2808 struct per_cpu_pages *pcp; 2809 2810 local_irq_save(flags); 2811 pset = per_cpu_ptr(zone->pageset, cpu); 2812 2813 pcp = &pset->pcp; 2814 if (pcp->count) 2815 free_pcppages_bulk(zone, pcp->count, pcp); 2816 local_irq_restore(flags); 2817 } 2818 2819 /* 2820 * Drain pcplists of all zones on the indicated processor. 2821 * 2822 * The processor must either be the current processor and the 2823 * thread pinned to the current processor or a processor that 2824 * is not online. 2825 */ 2826 static void drain_pages(unsigned int cpu) 2827 { 2828 struct zone *zone; 2829 2830 for_each_populated_zone(zone) { 2831 drain_pages_zone(cpu, zone); 2832 } 2833 } 2834 2835 /* 2836 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 2837 * 2838 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 2839 * the single zone's pages. 2840 */ 2841 void drain_local_pages(struct zone *zone) 2842 { 2843 int cpu = smp_processor_id(); 2844 2845 if (zone) 2846 drain_pages_zone(cpu, zone); 2847 else 2848 drain_pages(cpu); 2849 } 2850 2851 static void drain_local_pages_wq(struct work_struct *work) 2852 { 2853 struct pcpu_drain *drain; 2854 2855 drain = container_of(work, struct pcpu_drain, work); 2856 2857 /* 2858 * drain_all_pages doesn't use proper cpu hotplug protection so 2859 * we can race with cpu offline when the WQ can move this from 2860 * a cpu pinned worker to an unbound one. We can operate on a different 2861 * cpu which is allright but we also have to make sure to not move to 2862 * a different one. 2863 */ 2864 preempt_disable(); 2865 drain_local_pages(drain->zone); 2866 preempt_enable(); 2867 } 2868 2869 /* 2870 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 2871 * 2872 * When zone parameter is non-NULL, spill just the single zone's pages. 2873 * 2874 * Note that this can be extremely slow as the draining happens in a workqueue. 2875 */ 2876 void drain_all_pages(struct zone *zone) 2877 { 2878 int cpu; 2879 2880 /* 2881 * Allocate in the BSS so we wont require allocation in 2882 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 2883 */ 2884 static cpumask_t cpus_with_pcps; 2885 2886 /* 2887 * Make sure nobody triggers this path before mm_percpu_wq is fully 2888 * initialized. 2889 */ 2890 if (WARN_ON_ONCE(!mm_percpu_wq)) 2891 return; 2892 2893 /* 2894 * Do not drain if one is already in progress unless it's specific to 2895 * a zone. Such callers are primarily CMA and memory hotplug and need 2896 * the drain to be complete when the call returns. 2897 */ 2898 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) { 2899 if (!zone) 2900 return; 2901 mutex_lock(&pcpu_drain_mutex); 2902 } 2903 2904 /* 2905 * We don't care about racing with CPU hotplug event 2906 * as offline notification will cause the notified 2907 * cpu to drain that CPU pcps and on_each_cpu_mask 2908 * disables preemption as part of its processing 2909 */ 2910 for_each_online_cpu(cpu) { 2911 struct per_cpu_pageset *pcp; 2912 struct zone *z; 2913 bool has_pcps = false; 2914 2915 if (zone) { 2916 pcp = per_cpu_ptr(zone->pageset, cpu); 2917 if (pcp->pcp.count) 2918 has_pcps = true; 2919 } else { 2920 for_each_populated_zone(z) { 2921 pcp = per_cpu_ptr(z->pageset, cpu); 2922 if (pcp->pcp.count) { 2923 has_pcps = true; 2924 break; 2925 } 2926 } 2927 } 2928 2929 if (has_pcps) 2930 cpumask_set_cpu(cpu, &cpus_with_pcps); 2931 else 2932 cpumask_clear_cpu(cpu, &cpus_with_pcps); 2933 } 2934 2935 for_each_cpu(cpu, &cpus_with_pcps) { 2936 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu); 2937 2938 drain->zone = zone; 2939 INIT_WORK(&drain->work, drain_local_pages_wq); 2940 queue_work_on(cpu, mm_percpu_wq, &drain->work); 2941 } 2942 for_each_cpu(cpu, &cpus_with_pcps) 2943 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work); 2944 2945 mutex_unlock(&pcpu_drain_mutex); 2946 } 2947 2948 #ifdef CONFIG_HIBERNATION 2949 2950 /* 2951 * Touch the watchdog for every WD_PAGE_COUNT pages. 2952 */ 2953 #define WD_PAGE_COUNT (128*1024) 2954 2955 void mark_free_pages(struct zone *zone) 2956 { 2957 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT; 2958 unsigned long flags; 2959 unsigned int order, t; 2960 struct page *page; 2961 2962 if (zone_is_empty(zone)) 2963 return; 2964 2965 spin_lock_irqsave(&zone->lock, flags); 2966 2967 max_zone_pfn = zone_end_pfn(zone); 2968 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 2969 if (pfn_valid(pfn)) { 2970 page = pfn_to_page(pfn); 2971 2972 if (!--page_count) { 2973 touch_nmi_watchdog(); 2974 page_count = WD_PAGE_COUNT; 2975 } 2976 2977 if (page_zone(page) != zone) 2978 continue; 2979 2980 if (!swsusp_page_is_forbidden(page)) 2981 swsusp_unset_page_free(page); 2982 } 2983 2984 for_each_migratetype_order(order, t) { 2985 list_for_each_entry(page, 2986 &zone->free_area[order].free_list[t], lru) { 2987 unsigned long i; 2988 2989 pfn = page_to_pfn(page); 2990 for (i = 0; i < (1UL << order); i++) { 2991 if (!--page_count) { 2992 touch_nmi_watchdog(); 2993 page_count = WD_PAGE_COUNT; 2994 } 2995 swsusp_set_page_free(pfn_to_page(pfn + i)); 2996 } 2997 } 2998 } 2999 spin_unlock_irqrestore(&zone->lock, flags); 3000 } 3001 #endif /* CONFIG_PM */ 3002 3003 static bool free_unref_page_prepare(struct page *page, unsigned long pfn) 3004 { 3005 int migratetype; 3006 3007 if (!free_pcp_prepare(page)) 3008 return false; 3009 3010 migratetype = get_pfnblock_migratetype(page, pfn); 3011 set_pcppage_migratetype(page, migratetype); 3012 return true; 3013 } 3014 3015 static void free_unref_page_commit(struct page *page, unsigned long pfn) 3016 { 3017 struct zone *zone = page_zone(page); 3018 struct per_cpu_pages *pcp; 3019 int migratetype; 3020 3021 migratetype = get_pcppage_migratetype(page); 3022 __count_vm_event(PGFREE); 3023 3024 /* 3025 * We only track unmovable, reclaimable and movable on pcp lists. 3026 * Free ISOLATE pages back to the allocator because they are being 3027 * offlined but treat HIGHATOMIC as movable pages so we can get those 3028 * areas back if necessary. Otherwise, we may have to free 3029 * excessively into the page allocator 3030 */ 3031 if (migratetype >= MIGRATE_PCPTYPES) { 3032 if (unlikely(is_migrate_isolate(migratetype))) { 3033 free_one_page(zone, page, pfn, 0, migratetype); 3034 return; 3035 } 3036 migratetype = MIGRATE_MOVABLE; 3037 } 3038 3039 pcp = &this_cpu_ptr(zone->pageset)->pcp; 3040 list_add(&page->lru, &pcp->lists[migratetype]); 3041 pcp->count++; 3042 if (pcp->count >= pcp->high) { 3043 unsigned long batch = READ_ONCE(pcp->batch); 3044 free_pcppages_bulk(zone, batch, pcp); 3045 } 3046 } 3047 3048 /* 3049 * Free a 0-order page 3050 */ 3051 void free_unref_page(struct page *page) 3052 { 3053 unsigned long flags; 3054 unsigned long pfn = page_to_pfn(page); 3055 3056 if (!free_unref_page_prepare(page, pfn)) 3057 return; 3058 3059 local_irq_save(flags); 3060 free_unref_page_commit(page, pfn); 3061 local_irq_restore(flags); 3062 } 3063 3064 /* 3065 * Free a list of 0-order pages 3066 */ 3067 void free_unref_page_list(struct list_head *list) 3068 { 3069 struct page *page, *next; 3070 unsigned long flags, pfn; 3071 int batch_count = 0; 3072 3073 /* Prepare pages for freeing */ 3074 list_for_each_entry_safe(page, next, list, lru) { 3075 pfn = page_to_pfn(page); 3076 if (!free_unref_page_prepare(page, pfn)) 3077 list_del(&page->lru); 3078 set_page_private(page, pfn); 3079 } 3080 3081 local_irq_save(flags); 3082 list_for_each_entry_safe(page, next, list, lru) { 3083 unsigned long pfn = page_private(page); 3084 3085 set_page_private(page, 0); 3086 trace_mm_page_free_batched(page); 3087 free_unref_page_commit(page, pfn); 3088 3089 /* 3090 * Guard against excessive IRQ disabled times when we get 3091 * a large list of pages to free. 3092 */ 3093 if (++batch_count == SWAP_CLUSTER_MAX) { 3094 local_irq_restore(flags); 3095 batch_count = 0; 3096 local_irq_save(flags); 3097 } 3098 } 3099 local_irq_restore(flags); 3100 } 3101 3102 /* 3103 * split_page takes a non-compound higher-order page, and splits it into 3104 * n (1<<order) sub-pages: page[0..n] 3105 * Each sub-page must be freed individually. 3106 * 3107 * Note: this is probably too low level an operation for use in drivers. 3108 * Please consult with lkml before using this in your driver. 3109 */ 3110 void split_page(struct page *page, unsigned int order) 3111 { 3112 int i; 3113 3114 VM_BUG_ON_PAGE(PageCompound(page), page); 3115 VM_BUG_ON_PAGE(!page_count(page), page); 3116 3117 for (i = 1; i < (1 << order); i++) 3118 set_page_refcounted(page + i); 3119 split_page_owner(page, order); 3120 } 3121 EXPORT_SYMBOL_GPL(split_page); 3122 3123 int __isolate_free_page(struct page *page, unsigned int order) 3124 { 3125 struct free_area *area = &page_zone(page)->free_area[order]; 3126 unsigned long watermark; 3127 struct zone *zone; 3128 int mt; 3129 3130 BUG_ON(!PageBuddy(page)); 3131 3132 zone = page_zone(page); 3133 mt = get_pageblock_migratetype(page); 3134 3135 if (!is_migrate_isolate(mt)) { 3136 /* 3137 * Obey watermarks as if the page was being allocated. We can 3138 * emulate a high-order watermark check with a raised order-0 3139 * watermark, because we already know our high-order page 3140 * exists. 3141 */ 3142 watermark = zone->_watermark[WMARK_MIN] + (1UL << order); 3143 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 3144 return 0; 3145 3146 __mod_zone_freepage_state(zone, -(1UL << order), mt); 3147 } 3148 3149 /* Remove page from free list */ 3150 3151 del_page_from_free_area(page, area); 3152 3153 /* 3154 * Set the pageblock if the isolated page is at least half of a 3155 * pageblock 3156 */ 3157 if (order >= pageblock_order - 1) { 3158 struct page *endpage = page + (1 << order) - 1; 3159 for (; page < endpage; page += pageblock_nr_pages) { 3160 int mt = get_pageblock_migratetype(page); 3161 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt) 3162 && !is_migrate_highatomic(mt)) 3163 set_pageblock_migratetype(page, 3164 MIGRATE_MOVABLE); 3165 } 3166 } 3167 3168 3169 return 1UL << order; 3170 } 3171 3172 /* 3173 * Update NUMA hit/miss statistics 3174 * 3175 * Must be called with interrupts disabled. 3176 */ 3177 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) 3178 { 3179 #ifdef CONFIG_NUMA 3180 enum numa_stat_item local_stat = NUMA_LOCAL; 3181 3182 /* skip numa counters update if numa stats is disabled */ 3183 if (!static_branch_likely(&vm_numa_stat_key)) 3184 return; 3185 3186 if (zone_to_nid(z) != numa_node_id()) 3187 local_stat = NUMA_OTHER; 3188 3189 if (zone_to_nid(z) == zone_to_nid(preferred_zone)) 3190 __inc_numa_state(z, NUMA_HIT); 3191 else { 3192 __inc_numa_state(z, NUMA_MISS); 3193 __inc_numa_state(preferred_zone, NUMA_FOREIGN); 3194 } 3195 __inc_numa_state(z, local_stat); 3196 #endif 3197 } 3198 3199 /* Remove page from the per-cpu list, caller must protect the list */ 3200 static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype, 3201 unsigned int alloc_flags, 3202 struct per_cpu_pages *pcp, 3203 struct list_head *list) 3204 { 3205 struct page *page; 3206 3207 do { 3208 if (list_empty(list)) { 3209 pcp->count += rmqueue_bulk(zone, 0, 3210 pcp->batch, list, 3211 migratetype, alloc_flags); 3212 if (unlikely(list_empty(list))) 3213 return NULL; 3214 } 3215 3216 page = list_first_entry(list, struct page, lru); 3217 list_del(&page->lru); 3218 pcp->count--; 3219 } while (check_new_pcp(page)); 3220 3221 return page; 3222 } 3223 3224 /* Lock and remove page from the per-cpu list */ 3225 static struct page *rmqueue_pcplist(struct zone *preferred_zone, 3226 struct zone *zone, gfp_t gfp_flags, 3227 int migratetype, unsigned int alloc_flags) 3228 { 3229 struct per_cpu_pages *pcp; 3230 struct list_head *list; 3231 struct page *page; 3232 unsigned long flags; 3233 3234 local_irq_save(flags); 3235 pcp = &this_cpu_ptr(zone->pageset)->pcp; 3236 list = &pcp->lists[migratetype]; 3237 page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list); 3238 if (page) { 3239 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); 3240 zone_statistics(preferred_zone, zone); 3241 } 3242 local_irq_restore(flags); 3243 return page; 3244 } 3245 3246 /* 3247 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 3248 */ 3249 static inline 3250 struct page *rmqueue(struct zone *preferred_zone, 3251 struct zone *zone, unsigned int order, 3252 gfp_t gfp_flags, unsigned int alloc_flags, 3253 int migratetype) 3254 { 3255 unsigned long flags; 3256 struct page *page; 3257 3258 if (likely(order == 0)) { 3259 page = rmqueue_pcplist(preferred_zone, zone, gfp_flags, 3260 migratetype, alloc_flags); 3261 goto out; 3262 } 3263 3264 /* 3265 * We most definitely don't want callers attempting to 3266 * allocate greater than order-1 page units with __GFP_NOFAIL. 3267 */ 3268 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 3269 spin_lock_irqsave(&zone->lock, flags); 3270 3271 do { 3272 page = NULL; 3273 if (alloc_flags & ALLOC_HARDER) { 3274 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3275 if (page) 3276 trace_mm_page_alloc_zone_locked(page, order, migratetype); 3277 } 3278 if (!page) 3279 page = __rmqueue(zone, order, migratetype, alloc_flags); 3280 } while (page && check_new_pages(page, order)); 3281 spin_unlock(&zone->lock); 3282 if (!page) 3283 goto failed; 3284 __mod_zone_freepage_state(zone, -(1 << order), 3285 get_pcppage_migratetype(page)); 3286 3287 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3288 zone_statistics(preferred_zone, zone); 3289 local_irq_restore(flags); 3290 3291 out: 3292 /* Separate test+clear to avoid unnecessary atomics */ 3293 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) { 3294 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 3295 wakeup_kswapd(zone, 0, 0, zone_idx(zone)); 3296 } 3297 3298 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); 3299 return page; 3300 3301 failed: 3302 local_irq_restore(flags); 3303 return NULL; 3304 } 3305 3306 #ifdef CONFIG_FAIL_PAGE_ALLOC 3307 3308 static struct { 3309 struct fault_attr attr; 3310 3311 bool ignore_gfp_highmem; 3312 bool ignore_gfp_reclaim; 3313 u32 min_order; 3314 } fail_page_alloc = { 3315 .attr = FAULT_ATTR_INITIALIZER, 3316 .ignore_gfp_reclaim = true, 3317 .ignore_gfp_highmem = true, 3318 .min_order = 1, 3319 }; 3320 3321 static int __init setup_fail_page_alloc(char *str) 3322 { 3323 return setup_fault_attr(&fail_page_alloc.attr, str); 3324 } 3325 __setup("fail_page_alloc=", setup_fail_page_alloc); 3326 3327 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3328 { 3329 if (order < fail_page_alloc.min_order) 3330 return false; 3331 if (gfp_mask & __GFP_NOFAIL) 3332 return false; 3333 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 3334 return false; 3335 if (fail_page_alloc.ignore_gfp_reclaim && 3336 (gfp_mask & __GFP_DIRECT_RECLAIM)) 3337 return false; 3338 3339 return should_fail(&fail_page_alloc.attr, 1 << order); 3340 } 3341 3342 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 3343 3344 static int __init fail_page_alloc_debugfs(void) 3345 { 3346 umode_t mode = S_IFREG | 0600; 3347 struct dentry *dir; 3348 3349 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 3350 &fail_page_alloc.attr); 3351 3352 debugfs_create_bool("ignore-gfp-wait", mode, dir, 3353 &fail_page_alloc.ignore_gfp_reclaim); 3354 debugfs_create_bool("ignore-gfp-highmem", mode, dir, 3355 &fail_page_alloc.ignore_gfp_highmem); 3356 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order); 3357 3358 return 0; 3359 } 3360 3361 late_initcall(fail_page_alloc_debugfs); 3362 3363 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 3364 3365 #else /* CONFIG_FAIL_PAGE_ALLOC */ 3366 3367 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3368 { 3369 return false; 3370 } 3371 3372 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 3373 3374 static noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3375 { 3376 return __should_fail_alloc_page(gfp_mask, order); 3377 } 3378 ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE); 3379 3380 /* 3381 * Return true if free base pages are above 'mark'. For high-order checks it 3382 * will return true of the order-0 watermark is reached and there is at least 3383 * one free page of a suitable size. Checking now avoids taking the zone lock 3384 * to check in the allocation paths if no pages are free. 3385 */ 3386 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3387 int classzone_idx, unsigned int alloc_flags, 3388 long free_pages) 3389 { 3390 long min = mark; 3391 int o; 3392 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3393 3394 /* free_pages may go negative - that's OK */ 3395 free_pages -= (1 << order) - 1; 3396 3397 if (alloc_flags & ALLOC_HIGH) 3398 min -= min / 2; 3399 3400 /* 3401 * If the caller does not have rights to ALLOC_HARDER then subtract 3402 * the high-atomic reserves. This will over-estimate the size of the 3403 * atomic reserve but it avoids a search. 3404 */ 3405 if (likely(!alloc_harder)) { 3406 free_pages -= z->nr_reserved_highatomic; 3407 } else { 3408 /* 3409 * OOM victims can try even harder than normal ALLOC_HARDER 3410 * users on the grounds that it's definitely going to be in 3411 * the exit path shortly and free memory. Any allocation it 3412 * makes during the free path will be small and short-lived. 3413 */ 3414 if (alloc_flags & ALLOC_OOM) 3415 min -= min / 2; 3416 else 3417 min -= min / 4; 3418 } 3419 3420 3421 #ifdef CONFIG_CMA 3422 /* If allocation can't use CMA areas don't use free CMA pages */ 3423 if (!(alloc_flags & ALLOC_CMA)) 3424 free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES); 3425 #endif 3426 3427 /* 3428 * Check watermarks for an order-0 allocation request. If these 3429 * are not met, then a high-order request also cannot go ahead 3430 * even if a suitable page happened to be free. 3431 */ 3432 if (free_pages <= min + z->lowmem_reserve[classzone_idx]) 3433 return false; 3434 3435 /* If this is an order-0 request then the watermark is fine */ 3436 if (!order) 3437 return true; 3438 3439 /* For a high-order request, check at least one suitable page is free */ 3440 for (o = order; o < MAX_ORDER; o++) { 3441 struct free_area *area = &z->free_area[o]; 3442 int mt; 3443 3444 if (!area->nr_free) 3445 continue; 3446 3447 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 3448 if (!free_area_empty(area, mt)) 3449 return true; 3450 } 3451 3452 #ifdef CONFIG_CMA 3453 if ((alloc_flags & ALLOC_CMA) && 3454 !free_area_empty(area, MIGRATE_CMA)) { 3455 return true; 3456 } 3457 #endif 3458 if (alloc_harder && 3459 !list_empty(&area->free_list[MIGRATE_HIGHATOMIC])) 3460 return true; 3461 } 3462 return false; 3463 } 3464 3465 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3466 int classzone_idx, unsigned int alloc_flags) 3467 { 3468 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 3469 zone_page_state(z, NR_FREE_PAGES)); 3470 } 3471 3472 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 3473 unsigned long mark, int classzone_idx, unsigned int alloc_flags) 3474 { 3475 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3476 long cma_pages = 0; 3477 3478 #ifdef CONFIG_CMA 3479 /* If allocation can't use CMA areas don't use free CMA pages */ 3480 if (!(alloc_flags & ALLOC_CMA)) 3481 cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES); 3482 #endif 3483 3484 /* 3485 * Fast check for order-0 only. If this fails then the reserves 3486 * need to be calculated. There is a corner case where the check 3487 * passes but only the high-order atomic reserve are free. If 3488 * the caller is !atomic then it'll uselessly search the free 3489 * list. That corner case is then slower but it is harmless. 3490 */ 3491 if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx]) 3492 return true; 3493 3494 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 3495 free_pages); 3496 } 3497 3498 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 3499 unsigned long mark, int classzone_idx) 3500 { 3501 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3502 3503 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 3504 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 3505 3506 return __zone_watermark_ok(z, order, mark, classzone_idx, 0, 3507 free_pages); 3508 } 3509 3510 #ifdef CONFIG_NUMA 3511 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3512 { 3513 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= 3514 RECLAIM_DISTANCE; 3515 } 3516 #else /* CONFIG_NUMA */ 3517 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3518 { 3519 return true; 3520 } 3521 #endif /* CONFIG_NUMA */ 3522 3523 /* 3524 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid 3525 * fragmentation is subtle. If the preferred zone was HIGHMEM then 3526 * premature use of a lower zone may cause lowmem pressure problems that 3527 * are worse than fragmentation. If the next zone is ZONE_DMA then it is 3528 * probably too small. It only makes sense to spread allocations to avoid 3529 * fragmentation between the Normal and DMA32 zones. 3530 */ 3531 static inline unsigned int 3532 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) 3533 { 3534 unsigned int alloc_flags = 0; 3535 3536 if (gfp_mask & __GFP_KSWAPD_RECLAIM) 3537 alloc_flags |= ALLOC_KSWAPD; 3538 3539 #ifdef CONFIG_ZONE_DMA32 3540 if (!zone) 3541 return alloc_flags; 3542 3543 if (zone_idx(zone) != ZONE_NORMAL) 3544 return alloc_flags; 3545 3546 /* 3547 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and 3548 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume 3549 * on UMA that if Normal is populated then so is DMA32. 3550 */ 3551 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1); 3552 if (nr_online_nodes > 1 && !populated_zone(--zone)) 3553 return alloc_flags; 3554 3555 alloc_flags |= ALLOC_NOFRAGMENT; 3556 #endif /* CONFIG_ZONE_DMA32 */ 3557 return alloc_flags; 3558 } 3559 3560 /* 3561 * get_page_from_freelist goes through the zonelist trying to allocate 3562 * a page. 3563 */ 3564 static struct page * 3565 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 3566 const struct alloc_context *ac) 3567 { 3568 struct zoneref *z; 3569 struct zone *zone; 3570 struct pglist_data *last_pgdat_dirty_limit = NULL; 3571 bool no_fallback; 3572 3573 retry: 3574 /* 3575 * Scan zonelist, looking for a zone with enough free. 3576 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 3577 */ 3578 no_fallback = alloc_flags & ALLOC_NOFRAGMENT; 3579 z = ac->preferred_zoneref; 3580 for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3581 ac->nodemask) { 3582 struct page *page; 3583 unsigned long mark; 3584 3585 if (cpusets_enabled() && 3586 (alloc_flags & ALLOC_CPUSET) && 3587 !__cpuset_zone_allowed(zone, gfp_mask)) 3588 continue; 3589 /* 3590 * When allocating a page cache page for writing, we 3591 * want to get it from a node that is within its dirty 3592 * limit, such that no single node holds more than its 3593 * proportional share of globally allowed dirty pages. 3594 * The dirty limits take into account the node's 3595 * lowmem reserves and high watermark so that kswapd 3596 * should be able to balance it without having to 3597 * write pages from its LRU list. 3598 * 3599 * XXX: For now, allow allocations to potentially 3600 * exceed the per-node dirty limit in the slowpath 3601 * (spread_dirty_pages unset) before going into reclaim, 3602 * which is important when on a NUMA setup the allowed 3603 * nodes are together not big enough to reach the 3604 * global limit. The proper fix for these situations 3605 * will require awareness of nodes in the 3606 * dirty-throttling and the flusher threads. 3607 */ 3608 if (ac->spread_dirty_pages) { 3609 if (last_pgdat_dirty_limit == zone->zone_pgdat) 3610 continue; 3611 3612 if (!node_dirty_ok(zone->zone_pgdat)) { 3613 last_pgdat_dirty_limit = zone->zone_pgdat; 3614 continue; 3615 } 3616 } 3617 3618 if (no_fallback && nr_online_nodes > 1 && 3619 zone != ac->preferred_zoneref->zone) { 3620 int local_nid; 3621 3622 /* 3623 * If moving to a remote node, retry but allow 3624 * fragmenting fallbacks. Locality is more important 3625 * than fragmentation avoidance. 3626 */ 3627 local_nid = zone_to_nid(ac->preferred_zoneref->zone); 3628 if (zone_to_nid(zone) != local_nid) { 3629 alloc_flags &= ~ALLOC_NOFRAGMENT; 3630 goto retry; 3631 } 3632 } 3633 3634 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); 3635 if (!zone_watermark_fast(zone, order, mark, 3636 ac_classzone_idx(ac), alloc_flags)) { 3637 int ret; 3638 3639 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3640 /* 3641 * Watermark failed for this zone, but see if we can 3642 * grow this zone if it contains deferred pages. 3643 */ 3644 if (static_branch_unlikely(&deferred_pages)) { 3645 if (_deferred_grow_zone(zone, order)) 3646 goto try_this_zone; 3647 } 3648 #endif 3649 /* Checked here to keep the fast path fast */ 3650 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 3651 if (alloc_flags & ALLOC_NO_WATERMARKS) 3652 goto try_this_zone; 3653 3654 if (node_reclaim_mode == 0 || 3655 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 3656 continue; 3657 3658 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 3659 switch (ret) { 3660 case NODE_RECLAIM_NOSCAN: 3661 /* did not scan */ 3662 continue; 3663 case NODE_RECLAIM_FULL: 3664 /* scanned but unreclaimable */ 3665 continue; 3666 default: 3667 /* did we reclaim enough */ 3668 if (zone_watermark_ok(zone, order, mark, 3669 ac_classzone_idx(ac), alloc_flags)) 3670 goto try_this_zone; 3671 3672 continue; 3673 } 3674 } 3675 3676 try_this_zone: 3677 page = rmqueue(ac->preferred_zoneref->zone, zone, order, 3678 gfp_mask, alloc_flags, ac->migratetype); 3679 if (page) { 3680 prep_new_page(page, order, gfp_mask, alloc_flags); 3681 3682 /* 3683 * If this is a high-order atomic allocation then check 3684 * if the pageblock should be reserved for the future 3685 */ 3686 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 3687 reserve_highatomic_pageblock(page, zone, order); 3688 3689 return page; 3690 } else { 3691 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3692 /* Try again if zone has deferred pages */ 3693 if (static_branch_unlikely(&deferred_pages)) { 3694 if (_deferred_grow_zone(zone, order)) 3695 goto try_this_zone; 3696 } 3697 #endif 3698 } 3699 } 3700 3701 /* 3702 * It's possible on a UMA machine to get through all zones that are 3703 * fragmented. If avoiding fragmentation, reset and try again. 3704 */ 3705 if (no_fallback) { 3706 alloc_flags &= ~ALLOC_NOFRAGMENT; 3707 goto retry; 3708 } 3709 3710 return NULL; 3711 } 3712 3713 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) 3714 { 3715 unsigned int filter = SHOW_MEM_FILTER_NODES; 3716 static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1); 3717 3718 if (!__ratelimit(&show_mem_rs)) 3719 return; 3720 3721 /* 3722 * This documents exceptions given to allocations in certain 3723 * contexts that are allowed to allocate outside current's set 3724 * of allowed nodes. 3725 */ 3726 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3727 if (tsk_is_oom_victim(current) || 3728 (current->flags & (PF_MEMALLOC | PF_EXITING))) 3729 filter &= ~SHOW_MEM_FILTER_NODES; 3730 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 3731 filter &= ~SHOW_MEM_FILTER_NODES; 3732 3733 show_mem(filter, nodemask); 3734 } 3735 3736 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) 3737 { 3738 struct va_format vaf; 3739 va_list args; 3740 static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL, 3741 DEFAULT_RATELIMIT_BURST); 3742 3743 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs)) 3744 return; 3745 3746 va_start(args, fmt); 3747 vaf.fmt = fmt; 3748 vaf.va = &args; 3749 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl", 3750 current->comm, &vaf, gfp_mask, &gfp_mask, 3751 nodemask_pr_args(nodemask)); 3752 va_end(args); 3753 3754 cpuset_print_current_mems_allowed(); 3755 pr_cont("\n"); 3756 dump_stack(); 3757 warn_alloc_show_mem(gfp_mask, nodemask); 3758 } 3759 3760 static inline struct page * 3761 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, 3762 unsigned int alloc_flags, 3763 const struct alloc_context *ac) 3764 { 3765 struct page *page; 3766 3767 page = get_page_from_freelist(gfp_mask, order, 3768 alloc_flags|ALLOC_CPUSET, ac); 3769 /* 3770 * fallback to ignore cpuset restriction if our nodes 3771 * are depleted 3772 */ 3773 if (!page) 3774 page = get_page_from_freelist(gfp_mask, order, 3775 alloc_flags, ac); 3776 3777 return page; 3778 } 3779 3780 static inline struct page * 3781 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 3782 const struct alloc_context *ac, unsigned long *did_some_progress) 3783 { 3784 struct oom_control oc = { 3785 .zonelist = ac->zonelist, 3786 .nodemask = ac->nodemask, 3787 .memcg = NULL, 3788 .gfp_mask = gfp_mask, 3789 .order = order, 3790 }; 3791 struct page *page; 3792 3793 *did_some_progress = 0; 3794 3795 /* 3796 * Acquire the oom lock. If that fails, somebody else is 3797 * making progress for us. 3798 */ 3799 if (!mutex_trylock(&oom_lock)) { 3800 *did_some_progress = 1; 3801 schedule_timeout_uninterruptible(1); 3802 return NULL; 3803 } 3804 3805 /* 3806 * Go through the zonelist yet one more time, keep very high watermark 3807 * here, this is only to catch a parallel oom killing, we must fail if 3808 * we're still under heavy pressure. But make sure that this reclaim 3809 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY 3810 * allocation which will never fail due to oom_lock already held. 3811 */ 3812 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & 3813 ~__GFP_DIRECT_RECLAIM, order, 3814 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 3815 if (page) 3816 goto out; 3817 3818 /* Coredumps can quickly deplete all memory reserves */ 3819 if (current->flags & PF_DUMPCORE) 3820 goto out; 3821 /* The OOM killer will not help higher order allocs */ 3822 if (order > PAGE_ALLOC_COSTLY_ORDER) 3823 goto out; 3824 /* 3825 * We have already exhausted all our reclaim opportunities without any 3826 * success so it is time to admit defeat. We will skip the OOM killer 3827 * because it is very likely that the caller has a more reasonable 3828 * fallback than shooting a random task. 3829 */ 3830 if (gfp_mask & __GFP_RETRY_MAYFAIL) 3831 goto out; 3832 /* The OOM killer does not needlessly kill tasks for lowmem */ 3833 if (ac->high_zoneidx < ZONE_NORMAL) 3834 goto out; 3835 if (pm_suspended_storage()) 3836 goto out; 3837 /* 3838 * XXX: GFP_NOFS allocations should rather fail than rely on 3839 * other request to make a forward progress. 3840 * We are in an unfortunate situation where out_of_memory cannot 3841 * do much for this context but let's try it to at least get 3842 * access to memory reserved if the current task is killed (see 3843 * out_of_memory). Once filesystems are ready to handle allocation 3844 * failures more gracefully we should just bail out here. 3845 */ 3846 3847 /* The OOM killer may not free memory on a specific node */ 3848 if (gfp_mask & __GFP_THISNODE) 3849 goto out; 3850 3851 /* Exhausted what can be done so it's blame time */ 3852 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 3853 *did_some_progress = 1; 3854 3855 /* 3856 * Help non-failing allocations by giving them access to memory 3857 * reserves 3858 */ 3859 if (gfp_mask & __GFP_NOFAIL) 3860 page = __alloc_pages_cpuset_fallback(gfp_mask, order, 3861 ALLOC_NO_WATERMARKS, ac); 3862 } 3863 out: 3864 mutex_unlock(&oom_lock); 3865 return page; 3866 } 3867 3868 /* 3869 * Maximum number of compaction retries wit a progress before OOM 3870 * killer is consider as the only way to move forward. 3871 */ 3872 #define MAX_COMPACT_RETRIES 16 3873 3874 #ifdef CONFIG_COMPACTION 3875 /* Try memory compaction for high-order allocations before reclaim */ 3876 static struct page * 3877 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3878 unsigned int alloc_flags, const struct alloc_context *ac, 3879 enum compact_priority prio, enum compact_result *compact_result) 3880 { 3881 struct page *page = NULL; 3882 unsigned long pflags; 3883 unsigned int noreclaim_flag; 3884 3885 if (!order) 3886 return NULL; 3887 3888 psi_memstall_enter(&pflags); 3889 noreclaim_flag = memalloc_noreclaim_save(); 3890 3891 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 3892 prio, &page); 3893 3894 memalloc_noreclaim_restore(noreclaim_flag); 3895 psi_memstall_leave(&pflags); 3896 3897 /* 3898 * At least in one zone compaction wasn't deferred or skipped, so let's 3899 * count a compaction stall 3900 */ 3901 count_vm_event(COMPACTSTALL); 3902 3903 /* Prep a captured page if available */ 3904 if (page) 3905 prep_new_page(page, order, gfp_mask, alloc_flags); 3906 3907 /* Try get a page from the freelist if available */ 3908 if (!page) 3909 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 3910 3911 if (page) { 3912 struct zone *zone = page_zone(page); 3913 3914 zone->compact_blockskip_flush = false; 3915 compaction_defer_reset(zone, order, true); 3916 count_vm_event(COMPACTSUCCESS); 3917 return page; 3918 } 3919 3920 /* 3921 * It's bad if compaction run occurs and fails. The most likely reason 3922 * is that pages exist, but not enough to satisfy watermarks. 3923 */ 3924 count_vm_event(COMPACTFAIL); 3925 3926 cond_resched(); 3927 3928 return NULL; 3929 } 3930 3931 static inline bool 3932 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 3933 enum compact_result compact_result, 3934 enum compact_priority *compact_priority, 3935 int *compaction_retries) 3936 { 3937 int max_retries = MAX_COMPACT_RETRIES; 3938 int min_priority; 3939 bool ret = false; 3940 int retries = *compaction_retries; 3941 enum compact_priority priority = *compact_priority; 3942 3943 if (!order) 3944 return false; 3945 3946 if (compaction_made_progress(compact_result)) 3947 (*compaction_retries)++; 3948 3949 /* 3950 * compaction considers all the zone as desperately out of memory 3951 * so it doesn't really make much sense to retry except when the 3952 * failure could be caused by insufficient priority 3953 */ 3954 if (compaction_failed(compact_result)) 3955 goto check_priority; 3956 3957 /* 3958 * make sure the compaction wasn't deferred or didn't bail out early 3959 * due to locks contention before we declare that we should give up. 3960 * But do not retry if the given zonelist is not suitable for 3961 * compaction. 3962 */ 3963 if (compaction_withdrawn(compact_result)) { 3964 ret = compaction_zonelist_suitable(ac, order, alloc_flags); 3965 goto out; 3966 } 3967 3968 /* 3969 * !costly requests are much more important than __GFP_RETRY_MAYFAIL 3970 * costly ones because they are de facto nofail and invoke OOM 3971 * killer to move on while costly can fail and users are ready 3972 * to cope with that. 1/4 retries is rather arbitrary but we 3973 * would need much more detailed feedback from compaction to 3974 * make a better decision. 3975 */ 3976 if (order > PAGE_ALLOC_COSTLY_ORDER) 3977 max_retries /= 4; 3978 if (*compaction_retries <= max_retries) { 3979 ret = true; 3980 goto out; 3981 } 3982 3983 /* 3984 * Make sure there are attempts at the highest priority if we exhausted 3985 * all retries or failed at the lower priorities. 3986 */ 3987 check_priority: 3988 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 3989 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 3990 3991 if (*compact_priority > min_priority) { 3992 (*compact_priority)--; 3993 *compaction_retries = 0; 3994 ret = true; 3995 } 3996 out: 3997 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret); 3998 return ret; 3999 } 4000 #else 4001 static inline struct page * 4002 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4003 unsigned int alloc_flags, const struct alloc_context *ac, 4004 enum compact_priority prio, enum compact_result *compact_result) 4005 { 4006 *compact_result = COMPACT_SKIPPED; 4007 return NULL; 4008 } 4009 4010 static inline bool 4011 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 4012 enum compact_result compact_result, 4013 enum compact_priority *compact_priority, 4014 int *compaction_retries) 4015 { 4016 struct zone *zone; 4017 struct zoneref *z; 4018 4019 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 4020 return false; 4021 4022 /* 4023 * There are setups with compaction disabled which would prefer to loop 4024 * inside the allocator rather than hit the oom killer prematurely. 4025 * Let's give them a good hope and keep retrying while the order-0 4026 * watermarks are OK. 4027 */ 4028 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 4029 ac->nodemask) { 4030 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 4031 ac_classzone_idx(ac), alloc_flags)) 4032 return true; 4033 } 4034 return false; 4035 } 4036 #endif /* CONFIG_COMPACTION */ 4037 4038 #ifdef CONFIG_LOCKDEP 4039 static struct lockdep_map __fs_reclaim_map = 4040 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map); 4041 4042 static bool __need_fs_reclaim(gfp_t gfp_mask) 4043 { 4044 gfp_mask = current_gfp_context(gfp_mask); 4045 4046 /* no reclaim without waiting on it */ 4047 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) 4048 return false; 4049 4050 /* this guy won't enter reclaim */ 4051 if (current->flags & PF_MEMALLOC) 4052 return false; 4053 4054 /* We're only interested __GFP_FS allocations for now */ 4055 if (!(gfp_mask & __GFP_FS)) 4056 return false; 4057 4058 if (gfp_mask & __GFP_NOLOCKDEP) 4059 return false; 4060 4061 return true; 4062 } 4063 4064 void __fs_reclaim_acquire(void) 4065 { 4066 lock_map_acquire(&__fs_reclaim_map); 4067 } 4068 4069 void __fs_reclaim_release(void) 4070 { 4071 lock_map_release(&__fs_reclaim_map); 4072 } 4073 4074 void fs_reclaim_acquire(gfp_t gfp_mask) 4075 { 4076 if (__need_fs_reclaim(gfp_mask)) 4077 __fs_reclaim_acquire(); 4078 } 4079 EXPORT_SYMBOL_GPL(fs_reclaim_acquire); 4080 4081 void fs_reclaim_release(gfp_t gfp_mask) 4082 { 4083 if (__need_fs_reclaim(gfp_mask)) 4084 __fs_reclaim_release(); 4085 } 4086 EXPORT_SYMBOL_GPL(fs_reclaim_release); 4087 #endif 4088 4089 /* Perform direct synchronous page reclaim */ 4090 static int 4091 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 4092 const struct alloc_context *ac) 4093 { 4094 int progress; 4095 unsigned int noreclaim_flag; 4096 unsigned long pflags; 4097 4098 cond_resched(); 4099 4100 /* We now go into synchronous reclaim */ 4101 cpuset_memory_pressure_bump(); 4102 psi_memstall_enter(&pflags); 4103 fs_reclaim_acquire(gfp_mask); 4104 noreclaim_flag = memalloc_noreclaim_save(); 4105 4106 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 4107 ac->nodemask); 4108 4109 memalloc_noreclaim_restore(noreclaim_flag); 4110 fs_reclaim_release(gfp_mask); 4111 psi_memstall_leave(&pflags); 4112 4113 cond_resched(); 4114 4115 return progress; 4116 } 4117 4118 /* The really slow allocator path where we enter direct reclaim */ 4119 static inline struct page * 4120 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 4121 unsigned int alloc_flags, const struct alloc_context *ac, 4122 unsigned long *did_some_progress) 4123 { 4124 struct page *page = NULL; 4125 bool drained = false; 4126 4127 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 4128 if (unlikely(!(*did_some_progress))) 4129 return NULL; 4130 4131 retry: 4132 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4133 4134 /* 4135 * If an allocation failed after direct reclaim, it could be because 4136 * pages are pinned on the per-cpu lists or in high alloc reserves. 4137 * Shrink them them and try again 4138 */ 4139 if (!page && !drained) { 4140 unreserve_highatomic_pageblock(ac, false); 4141 drain_all_pages(NULL); 4142 drained = true; 4143 goto retry; 4144 } 4145 4146 return page; 4147 } 4148 4149 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, 4150 const struct alloc_context *ac) 4151 { 4152 struct zoneref *z; 4153 struct zone *zone; 4154 pg_data_t *last_pgdat = NULL; 4155 enum zone_type high_zoneidx = ac->high_zoneidx; 4156 4157 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, high_zoneidx, 4158 ac->nodemask) { 4159 if (last_pgdat != zone->zone_pgdat) 4160 wakeup_kswapd(zone, gfp_mask, order, high_zoneidx); 4161 last_pgdat = zone->zone_pgdat; 4162 } 4163 } 4164 4165 static inline unsigned int 4166 gfp_to_alloc_flags(gfp_t gfp_mask) 4167 { 4168 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 4169 4170 /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */ 4171 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 4172 4173 /* 4174 * The caller may dip into page reserves a bit more if the caller 4175 * cannot run direct reclaim, or if the caller has realtime scheduling 4176 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 4177 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 4178 */ 4179 alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); 4180 4181 if (gfp_mask & __GFP_ATOMIC) { 4182 /* 4183 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 4184 * if it can't schedule. 4185 */ 4186 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4187 alloc_flags |= ALLOC_HARDER; 4188 /* 4189 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 4190 * comment for __cpuset_node_allowed(). 4191 */ 4192 alloc_flags &= ~ALLOC_CPUSET; 4193 } else if (unlikely(rt_task(current)) && !in_interrupt()) 4194 alloc_flags |= ALLOC_HARDER; 4195 4196 if (gfp_mask & __GFP_KSWAPD_RECLAIM) 4197 alloc_flags |= ALLOC_KSWAPD; 4198 4199 #ifdef CONFIG_CMA 4200 if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) 4201 alloc_flags |= ALLOC_CMA; 4202 #endif 4203 return alloc_flags; 4204 } 4205 4206 static bool oom_reserves_allowed(struct task_struct *tsk) 4207 { 4208 if (!tsk_is_oom_victim(tsk)) 4209 return false; 4210 4211 /* 4212 * !MMU doesn't have oom reaper so give access to memory reserves 4213 * only to the thread with TIF_MEMDIE set 4214 */ 4215 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE)) 4216 return false; 4217 4218 return true; 4219 } 4220 4221 /* 4222 * Distinguish requests which really need access to full memory 4223 * reserves from oom victims which can live with a portion of it 4224 */ 4225 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) 4226 { 4227 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 4228 return 0; 4229 if (gfp_mask & __GFP_MEMALLOC) 4230 return ALLOC_NO_WATERMARKS; 4231 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 4232 return ALLOC_NO_WATERMARKS; 4233 if (!in_interrupt()) { 4234 if (current->flags & PF_MEMALLOC) 4235 return ALLOC_NO_WATERMARKS; 4236 else if (oom_reserves_allowed(current)) 4237 return ALLOC_OOM; 4238 } 4239 4240 return 0; 4241 } 4242 4243 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 4244 { 4245 return !!__gfp_pfmemalloc_flags(gfp_mask); 4246 } 4247 4248 /* 4249 * Checks whether it makes sense to retry the reclaim to make a forward progress 4250 * for the given allocation request. 4251 * 4252 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row 4253 * without success, or when we couldn't even meet the watermark if we 4254 * reclaimed all remaining pages on the LRU lists. 4255 * 4256 * Returns true if a retry is viable or false to enter the oom path. 4257 */ 4258 static inline bool 4259 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 4260 struct alloc_context *ac, int alloc_flags, 4261 bool did_some_progress, int *no_progress_loops) 4262 { 4263 struct zone *zone; 4264 struct zoneref *z; 4265 bool ret = false; 4266 4267 /* 4268 * Costly allocations might have made a progress but this doesn't mean 4269 * their order will become available due to high fragmentation so 4270 * always increment the no progress counter for them 4271 */ 4272 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 4273 *no_progress_loops = 0; 4274 else 4275 (*no_progress_loops)++; 4276 4277 /* 4278 * Make sure we converge to OOM if we cannot make any progress 4279 * several times in the row. 4280 */ 4281 if (*no_progress_loops > MAX_RECLAIM_RETRIES) { 4282 /* Before OOM, exhaust highatomic_reserve */ 4283 return unreserve_highatomic_pageblock(ac, true); 4284 } 4285 4286 /* 4287 * Keep reclaiming pages while there is a chance this will lead 4288 * somewhere. If none of the target zones can satisfy our allocation 4289 * request even if all reclaimable pages are considered then we are 4290 * screwed and have to go OOM. 4291 */ 4292 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 4293 ac->nodemask) { 4294 unsigned long available; 4295 unsigned long reclaimable; 4296 unsigned long min_wmark = min_wmark_pages(zone); 4297 bool wmark; 4298 4299 available = reclaimable = zone_reclaimable_pages(zone); 4300 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 4301 4302 /* 4303 * Would the allocation succeed if we reclaimed all 4304 * reclaimable pages? 4305 */ 4306 wmark = __zone_watermark_ok(zone, order, min_wmark, 4307 ac_classzone_idx(ac), alloc_flags, available); 4308 trace_reclaim_retry_zone(z, order, reclaimable, 4309 available, min_wmark, *no_progress_loops, wmark); 4310 if (wmark) { 4311 /* 4312 * If we didn't make any progress and have a lot of 4313 * dirty + writeback pages then we should wait for 4314 * an IO to complete to slow down the reclaim and 4315 * prevent from pre mature OOM 4316 */ 4317 if (!did_some_progress) { 4318 unsigned long write_pending; 4319 4320 write_pending = zone_page_state_snapshot(zone, 4321 NR_ZONE_WRITE_PENDING); 4322 4323 if (2 * write_pending > reclaimable) { 4324 congestion_wait(BLK_RW_ASYNC, HZ/10); 4325 return true; 4326 } 4327 } 4328 4329 ret = true; 4330 goto out; 4331 } 4332 } 4333 4334 out: 4335 /* 4336 * Memory allocation/reclaim might be called from a WQ context and the 4337 * current implementation of the WQ concurrency control doesn't 4338 * recognize that a particular WQ is congested if the worker thread is 4339 * looping without ever sleeping. Therefore we have to do a short sleep 4340 * here rather than calling cond_resched(). 4341 */ 4342 if (current->flags & PF_WQ_WORKER) 4343 schedule_timeout_uninterruptible(1); 4344 else 4345 cond_resched(); 4346 return ret; 4347 } 4348 4349 static inline bool 4350 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) 4351 { 4352 /* 4353 * It's possible that cpuset's mems_allowed and the nodemask from 4354 * mempolicy don't intersect. This should be normally dealt with by 4355 * policy_nodemask(), but it's possible to race with cpuset update in 4356 * such a way the check therein was true, and then it became false 4357 * before we got our cpuset_mems_cookie here. 4358 * This assumes that for all allocations, ac->nodemask can come only 4359 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored 4360 * when it does not intersect with the cpuset restrictions) or the 4361 * caller can deal with a violated nodemask. 4362 */ 4363 if (cpusets_enabled() && ac->nodemask && 4364 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) { 4365 ac->nodemask = NULL; 4366 return true; 4367 } 4368 4369 /* 4370 * When updating a task's mems_allowed or mempolicy nodemask, it is 4371 * possible to race with parallel threads in such a way that our 4372 * allocation can fail while the mask is being updated. If we are about 4373 * to fail, check if the cpuset changed during allocation and if so, 4374 * retry. 4375 */ 4376 if (read_mems_allowed_retry(cpuset_mems_cookie)) 4377 return true; 4378 4379 return false; 4380 } 4381 4382 static inline struct page * 4383 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 4384 struct alloc_context *ac) 4385 { 4386 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 4387 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 4388 struct page *page = NULL; 4389 unsigned int alloc_flags; 4390 unsigned long did_some_progress; 4391 enum compact_priority compact_priority; 4392 enum compact_result compact_result; 4393 int compaction_retries; 4394 int no_progress_loops; 4395 unsigned int cpuset_mems_cookie; 4396 int reserve_flags; 4397 4398 /* 4399 * We also sanity check to catch abuse of atomic reserves being used by 4400 * callers that are not in atomic context. 4401 */ 4402 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 4403 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 4404 gfp_mask &= ~__GFP_ATOMIC; 4405 4406 retry_cpuset: 4407 compaction_retries = 0; 4408 no_progress_loops = 0; 4409 compact_priority = DEF_COMPACT_PRIORITY; 4410 cpuset_mems_cookie = read_mems_allowed_begin(); 4411 4412 /* 4413 * The fast path uses conservative alloc_flags to succeed only until 4414 * kswapd needs to be woken up, and to avoid the cost of setting up 4415 * alloc_flags precisely. So we do that now. 4416 */ 4417 alloc_flags = gfp_to_alloc_flags(gfp_mask); 4418 4419 /* 4420 * We need to recalculate the starting point for the zonelist iterator 4421 * because we might have used different nodemask in the fast path, or 4422 * there was a cpuset modification and we are retrying - otherwise we 4423 * could end up iterating over non-eligible zones endlessly. 4424 */ 4425 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4426 ac->high_zoneidx, ac->nodemask); 4427 if (!ac->preferred_zoneref->zone) 4428 goto nopage; 4429 4430 if (alloc_flags & ALLOC_KSWAPD) 4431 wake_all_kswapds(order, gfp_mask, ac); 4432 4433 /* 4434 * The adjusted alloc_flags might result in immediate success, so try 4435 * that first 4436 */ 4437 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4438 if (page) 4439 goto got_pg; 4440 4441 /* 4442 * For costly allocations, try direct compaction first, as it's likely 4443 * that we have enough base pages and don't need to reclaim. For non- 4444 * movable high-order allocations, do that as well, as compaction will 4445 * try prevent permanent fragmentation by migrating from blocks of the 4446 * same migratetype. 4447 * Don't try this for allocations that are allowed to ignore 4448 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 4449 */ 4450 if (can_direct_reclaim && 4451 (costly_order || 4452 (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 4453 && !gfp_pfmemalloc_allowed(gfp_mask)) { 4454 page = __alloc_pages_direct_compact(gfp_mask, order, 4455 alloc_flags, ac, 4456 INIT_COMPACT_PRIORITY, 4457 &compact_result); 4458 if (page) 4459 goto got_pg; 4460 4461 /* 4462 * Checks for costly allocations with __GFP_NORETRY, which 4463 * includes THP page fault allocations 4464 */ 4465 if (costly_order && (gfp_mask & __GFP_NORETRY)) { 4466 /* 4467 * If compaction is deferred for high-order allocations, 4468 * it is because sync compaction recently failed. If 4469 * this is the case and the caller requested a THP 4470 * allocation, we do not want to heavily disrupt the 4471 * system, so we fail the allocation instead of entering 4472 * direct reclaim. 4473 */ 4474 if (compact_result == COMPACT_DEFERRED) 4475 goto nopage; 4476 4477 /* 4478 * Looks like reclaim/compaction is worth trying, but 4479 * sync compaction could be very expensive, so keep 4480 * using async compaction. 4481 */ 4482 compact_priority = INIT_COMPACT_PRIORITY; 4483 } 4484 } 4485 4486 retry: 4487 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 4488 if (alloc_flags & ALLOC_KSWAPD) 4489 wake_all_kswapds(order, gfp_mask, ac); 4490 4491 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); 4492 if (reserve_flags) 4493 alloc_flags = reserve_flags; 4494 4495 /* 4496 * Reset the nodemask and zonelist iterators if memory policies can be 4497 * ignored. These allocations are high priority and system rather than 4498 * user oriented. 4499 */ 4500 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { 4501 ac->nodemask = NULL; 4502 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4503 ac->high_zoneidx, ac->nodemask); 4504 } 4505 4506 /* Attempt with potentially adjusted zonelist and alloc_flags */ 4507 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4508 if (page) 4509 goto got_pg; 4510 4511 /* Caller is not willing to reclaim, we can't balance anything */ 4512 if (!can_direct_reclaim) 4513 goto nopage; 4514 4515 /* Avoid recursion of direct reclaim */ 4516 if (current->flags & PF_MEMALLOC) 4517 goto nopage; 4518 4519 /* Try direct reclaim and then allocating */ 4520 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 4521 &did_some_progress); 4522 if (page) 4523 goto got_pg; 4524 4525 /* Try direct compaction and then allocating */ 4526 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 4527 compact_priority, &compact_result); 4528 if (page) 4529 goto got_pg; 4530 4531 /* Do not loop if specifically requested */ 4532 if (gfp_mask & __GFP_NORETRY) 4533 goto nopage; 4534 4535 /* 4536 * Do not retry costly high order allocations unless they are 4537 * __GFP_RETRY_MAYFAIL 4538 */ 4539 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) 4540 goto nopage; 4541 4542 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 4543 did_some_progress > 0, &no_progress_loops)) 4544 goto retry; 4545 4546 /* 4547 * It doesn't make any sense to retry for the compaction if the order-0 4548 * reclaim is not able to make any progress because the current 4549 * implementation of the compaction depends on the sufficient amount 4550 * of free memory (see __compaction_suitable) 4551 */ 4552 if (did_some_progress > 0 && 4553 should_compact_retry(ac, order, alloc_flags, 4554 compact_result, &compact_priority, 4555 &compaction_retries)) 4556 goto retry; 4557 4558 4559 /* Deal with possible cpuset update races before we start OOM killing */ 4560 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4561 goto retry_cpuset; 4562 4563 /* Reclaim has failed us, start killing things */ 4564 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 4565 if (page) 4566 goto got_pg; 4567 4568 /* Avoid allocations with no watermarks from looping endlessly */ 4569 if (tsk_is_oom_victim(current) && 4570 (alloc_flags == ALLOC_OOM || 4571 (gfp_mask & __GFP_NOMEMALLOC))) 4572 goto nopage; 4573 4574 /* Retry as long as the OOM killer is making progress */ 4575 if (did_some_progress) { 4576 no_progress_loops = 0; 4577 goto retry; 4578 } 4579 4580 nopage: 4581 /* Deal with possible cpuset update races before we fail */ 4582 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4583 goto retry_cpuset; 4584 4585 /* 4586 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure 4587 * we always retry 4588 */ 4589 if (gfp_mask & __GFP_NOFAIL) { 4590 /* 4591 * All existing users of the __GFP_NOFAIL are blockable, so warn 4592 * of any new users that actually require GFP_NOWAIT 4593 */ 4594 if (WARN_ON_ONCE(!can_direct_reclaim)) 4595 goto fail; 4596 4597 /* 4598 * PF_MEMALLOC request from this context is rather bizarre 4599 * because we cannot reclaim anything and only can loop waiting 4600 * for somebody to do a work for us 4601 */ 4602 WARN_ON_ONCE(current->flags & PF_MEMALLOC); 4603 4604 /* 4605 * non failing costly orders are a hard requirement which we 4606 * are not prepared for much so let's warn about these users 4607 * so that we can identify them and convert them to something 4608 * else. 4609 */ 4610 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER); 4611 4612 /* 4613 * Help non-failing allocations by giving them access to memory 4614 * reserves but do not use ALLOC_NO_WATERMARKS because this 4615 * could deplete whole memory reserves which would just make 4616 * the situation worse 4617 */ 4618 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); 4619 if (page) 4620 goto got_pg; 4621 4622 cond_resched(); 4623 goto retry; 4624 } 4625 fail: 4626 warn_alloc(gfp_mask, ac->nodemask, 4627 "page allocation failure: order:%u", order); 4628 got_pg: 4629 return page; 4630 } 4631 4632 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, 4633 int preferred_nid, nodemask_t *nodemask, 4634 struct alloc_context *ac, gfp_t *alloc_mask, 4635 unsigned int *alloc_flags) 4636 { 4637 ac->high_zoneidx = gfp_zone(gfp_mask); 4638 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); 4639 ac->nodemask = nodemask; 4640 ac->migratetype = gfpflags_to_migratetype(gfp_mask); 4641 4642 if (cpusets_enabled()) { 4643 *alloc_mask |= __GFP_HARDWALL; 4644 if (!ac->nodemask) 4645 ac->nodemask = &cpuset_current_mems_allowed; 4646 else 4647 *alloc_flags |= ALLOC_CPUSET; 4648 } 4649 4650 fs_reclaim_acquire(gfp_mask); 4651 fs_reclaim_release(gfp_mask); 4652 4653 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 4654 4655 if (should_fail_alloc_page(gfp_mask, order)) 4656 return false; 4657 4658 if (IS_ENABLED(CONFIG_CMA) && ac->migratetype == MIGRATE_MOVABLE) 4659 *alloc_flags |= ALLOC_CMA; 4660 4661 return true; 4662 } 4663 4664 /* Determine whether to spread dirty pages and what the first usable zone */ 4665 static inline void finalise_ac(gfp_t gfp_mask, struct alloc_context *ac) 4666 { 4667 /* Dirty zone balancing only done in the fast path */ 4668 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); 4669 4670 /* 4671 * The preferred zone is used for statistics but crucially it is 4672 * also used as the starting point for the zonelist iterator. It 4673 * may get reset for allocations that ignore memory policies. 4674 */ 4675 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4676 ac->high_zoneidx, ac->nodemask); 4677 } 4678 4679 /* 4680 * This is the 'heart' of the zoned buddy allocator. 4681 */ 4682 struct page * 4683 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, 4684 nodemask_t *nodemask) 4685 { 4686 struct page *page; 4687 unsigned int alloc_flags = ALLOC_WMARK_LOW; 4688 gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */ 4689 struct alloc_context ac = { }; 4690 4691 /* 4692 * There are several places where we assume that the order value is sane 4693 * so bail out early if the request is out of bound. 4694 */ 4695 if (unlikely(order >= MAX_ORDER)) { 4696 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); 4697 return NULL; 4698 } 4699 4700 gfp_mask &= gfp_allowed_mask; 4701 alloc_mask = gfp_mask; 4702 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags)) 4703 return NULL; 4704 4705 finalise_ac(gfp_mask, &ac); 4706 4707 /* 4708 * Forbid the first pass from falling back to types that fragment 4709 * memory until all local zones are considered. 4710 */ 4711 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask); 4712 4713 /* First allocation attempt */ 4714 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); 4715 if (likely(page)) 4716 goto out; 4717 4718 /* 4719 * Apply scoped allocation constraints. This is mainly about GFP_NOFS 4720 * resp. GFP_NOIO which has to be inherited for all allocation requests 4721 * from a particular context which has been marked by 4722 * memalloc_no{fs,io}_{save,restore}. 4723 */ 4724 alloc_mask = current_gfp_context(gfp_mask); 4725 ac.spread_dirty_pages = false; 4726 4727 /* 4728 * Restore the original nodemask if it was potentially replaced with 4729 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 4730 */ 4731 if (unlikely(ac.nodemask != nodemask)) 4732 ac.nodemask = nodemask; 4733 4734 page = __alloc_pages_slowpath(alloc_mask, order, &ac); 4735 4736 out: 4737 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && 4738 unlikely(__memcg_kmem_charge(page, gfp_mask, order) != 0)) { 4739 __free_pages(page, order); 4740 page = NULL; 4741 } 4742 4743 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); 4744 4745 return page; 4746 } 4747 EXPORT_SYMBOL(__alloc_pages_nodemask); 4748 4749 /* 4750 * Common helper functions. Never use with __GFP_HIGHMEM because the returned 4751 * address cannot represent highmem pages. Use alloc_pages and then kmap if 4752 * you need to access high mem. 4753 */ 4754 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 4755 { 4756 struct page *page; 4757 4758 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); 4759 if (!page) 4760 return 0; 4761 return (unsigned long) page_address(page); 4762 } 4763 EXPORT_SYMBOL(__get_free_pages); 4764 4765 unsigned long get_zeroed_page(gfp_t gfp_mask) 4766 { 4767 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 4768 } 4769 EXPORT_SYMBOL(get_zeroed_page); 4770 4771 static inline void free_the_page(struct page *page, unsigned int order) 4772 { 4773 if (order == 0) /* Via pcp? */ 4774 free_unref_page(page); 4775 else 4776 __free_pages_ok(page, order); 4777 } 4778 4779 void __free_pages(struct page *page, unsigned int order) 4780 { 4781 if (put_page_testzero(page)) 4782 free_the_page(page, order); 4783 } 4784 EXPORT_SYMBOL(__free_pages); 4785 4786 void free_pages(unsigned long addr, unsigned int order) 4787 { 4788 if (addr != 0) { 4789 VM_BUG_ON(!virt_addr_valid((void *)addr)); 4790 __free_pages(virt_to_page((void *)addr), order); 4791 } 4792 } 4793 4794 EXPORT_SYMBOL(free_pages); 4795 4796 /* 4797 * Page Fragment: 4798 * An arbitrary-length arbitrary-offset area of memory which resides 4799 * within a 0 or higher order page. Multiple fragments within that page 4800 * are individually refcounted, in the page's reference counter. 4801 * 4802 * The page_frag functions below provide a simple allocation framework for 4803 * page fragments. This is used by the network stack and network device 4804 * drivers to provide a backing region of memory for use as either an 4805 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 4806 */ 4807 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 4808 gfp_t gfp_mask) 4809 { 4810 struct page *page = NULL; 4811 gfp_t gfp = gfp_mask; 4812 4813 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4814 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 4815 __GFP_NOMEMALLOC; 4816 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 4817 PAGE_FRAG_CACHE_MAX_ORDER); 4818 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 4819 #endif 4820 if (unlikely(!page)) 4821 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 4822 4823 nc->va = page ? page_address(page) : NULL; 4824 4825 return page; 4826 } 4827 4828 void __page_frag_cache_drain(struct page *page, unsigned int count) 4829 { 4830 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 4831 4832 if (page_ref_sub_and_test(page, count)) 4833 free_the_page(page, compound_order(page)); 4834 } 4835 EXPORT_SYMBOL(__page_frag_cache_drain); 4836 4837 void *page_frag_alloc(struct page_frag_cache *nc, 4838 unsigned int fragsz, gfp_t gfp_mask) 4839 { 4840 unsigned int size = PAGE_SIZE; 4841 struct page *page; 4842 int offset; 4843 4844 if (unlikely(!nc->va)) { 4845 refill: 4846 page = __page_frag_cache_refill(nc, gfp_mask); 4847 if (!page) 4848 return NULL; 4849 4850 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4851 /* if size can vary use size else just use PAGE_SIZE */ 4852 size = nc->size; 4853 #endif 4854 /* Even if we own the page, we do not use atomic_set(). 4855 * This would break get_page_unless_zero() users. 4856 */ 4857 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 4858 4859 /* reset page count bias and offset to start of new frag */ 4860 nc->pfmemalloc = page_is_pfmemalloc(page); 4861 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 4862 nc->offset = size; 4863 } 4864 4865 offset = nc->offset - fragsz; 4866 if (unlikely(offset < 0)) { 4867 page = virt_to_page(nc->va); 4868 4869 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 4870 goto refill; 4871 4872 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4873 /* if size can vary use size else just use PAGE_SIZE */ 4874 size = nc->size; 4875 #endif 4876 /* OK, page count is 0, we can safely set it */ 4877 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 4878 4879 /* reset page count bias and offset to start of new frag */ 4880 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 4881 offset = size - fragsz; 4882 } 4883 4884 nc->pagecnt_bias--; 4885 nc->offset = offset; 4886 4887 return nc->va + offset; 4888 } 4889 EXPORT_SYMBOL(page_frag_alloc); 4890 4891 /* 4892 * Frees a page fragment allocated out of either a compound or order 0 page. 4893 */ 4894 void page_frag_free(void *addr) 4895 { 4896 struct page *page = virt_to_head_page(addr); 4897 4898 if (unlikely(put_page_testzero(page))) 4899 free_the_page(page, compound_order(page)); 4900 } 4901 EXPORT_SYMBOL(page_frag_free); 4902 4903 static void *make_alloc_exact(unsigned long addr, unsigned int order, 4904 size_t size) 4905 { 4906 if (addr) { 4907 unsigned long alloc_end = addr + (PAGE_SIZE << order); 4908 unsigned long used = addr + PAGE_ALIGN(size); 4909 4910 split_page(virt_to_page((void *)addr), order); 4911 while (used < alloc_end) { 4912 free_page(used); 4913 used += PAGE_SIZE; 4914 } 4915 } 4916 return (void *)addr; 4917 } 4918 4919 /** 4920 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 4921 * @size: the number of bytes to allocate 4922 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 4923 * 4924 * This function is similar to alloc_pages(), except that it allocates the 4925 * minimum number of pages to satisfy the request. alloc_pages() can only 4926 * allocate memory in power-of-two pages. 4927 * 4928 * This function is also limited by MAX_ORDER. 4929 * 4930 * Memory allocated by this function must be released by free_pages_exact(). 4931 * 4932 * Return: pointer to the allocated area or %NULL in case of error. 4933 */ 4934 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 4935 { 4936 unsigned int order = get_order(size); 4937 unsigned long addr; 4938 4939 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 4940 gfp_mask &= ~__GFP_COMP; 4941 4942 addr = __get_free_pages(gfp_mask, order); 4943 return make_alloc_exact(addr, order, size); 4944 } 4945 EXPORT_SYMBOL(alloc_pages_exact); 4946 4947 /** 4948 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 4949 * pages on a node. 4950 * @nid: the preferred node ID where memory should be allocated 4951 * @size: the number of bytes to allocate 4952 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 4953 * 4954 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 4955 * back. 4956 * 4957 * Return: pointer to the allocated area or %NULL in case of error. 4958 */ 4959 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 4960 { 4961 unsigned int order = get_order(size); 4962 struct page *p; 4963 4964 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP)) 4965 gfp_mask &= ~__GFP_COMP; 4966 4967 p = alloc_pages_node(nid, gfp_mask, order); 4968 if (!p) 4969 return NULL; 4970 return make_alloc_exact((unsigned long)page_address(p), order, size); 4971 } 4972 4973 /** 4974 * free_pages_exact - release memory allocated via alloc_pages_exact() 4975 * @virt: the value returned by alloc_pages_exact. 4976 * @size: size of allocation, same value as passed to alloc_pages_exact(). 4977 * 4978 * Release the memory allocated by a previous call to alloc_pages_exact. 4979 */ 4980 void free_pages_exact(void *virt, size_t size) 4981 { 4982 unsigned long addr = (unsigned long)virt; 4983 unsigned long end = addr + PAGE_ALIGN(size); 4984 4985 while (addr < end) { 4986 free_page(addr); 4987 addr += PAGE_SIZE; 4988 } 4989 } 4990 EXPORT_SYMBOL(free_pages_exact); 4991 4992 /** 4993 * nr_free_zone_pages - count number of pages beyond high watermark 4994 * @offset: The zone index of the highest zone 4995 * 4996 * nr_free_zone_pages() counts the number of pages which are beyond the 4997 * high watermark within all zones at or below a given zone index. For each 4998 * zone, the number of pages is calculated as: 4999 * 5000 * nr_free_zone_pages = managed_pages - high_pages 5001 * 5002 * Return: number of pages beyond high watermark. 5003 */ 5004 static unsigned long nr_free_zone_pages(int offset) 5005 { 5006 struct zoneref *z; 5007 struct zone *zone; 5008 5009 /* Just pick one node, since fallback list is circular */ 5010 unsigned long sum = 0; 5011 5012 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 5013 5014 for_each_zone_zonelist(zone, z, zonelist, offset) { 5015 unsigned long size = zone_managed_pages(zone); 5016 unsigned long high = high_wmark_pages(zone); 5017 if (size > high) 5018 sum += size - high; 5019 } 5020 5021 return sum; 5022 } 5023 5024 /** 5025 * nr_free_buffer_pages - count number of pages beyond high watermark 5026 * 5027 * nr_free_buffer_pages() counts the number of pages which are beyond the high 5028 * watermark within ZONE_DMA and ZONE_NORMAL. 5029 * 5030 * Return: number of pages beyond high watermark within ZONE_DMA and 5031 * ZONE_NORMAL. 5032 */ 5033 unsigned long nr_free_buffer_pages(void) 5034 { 5035 return nr_free_zone_pages(gfp_zone(GFP_USER)); 5036 } 5037 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 5038 5039 /** 5040 * nr_free_pagecache_pages - count number of pages beyond high watermark 5041 * 5042 * nr_free_pagecache_pages() counts the number of pages which are beyond the 5043 * high watermark within all zones. 5044 * 5045 * Return: number of pages beyond high watermark within all zones. 5046 */ 5047 unsigned long nr_free_pagecache_pages(void) 5048 { 5049 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 5050 } 5051 5052 static inline void show_node(struct zone *zone) 5053 { 5054 if (IS_ENABLED(CONFIG_NUMA)) 5055 printk("Node %d ", zone_to_nid(zone)); 5056 } 5057 5058 long si_mem_available(void) 5059 { 5060 long available; 5061 unsigned long pagecache; 5062 unsigned long wmark_low = 0; 5063 unsigned long pages[NR_LRU_LISTS]; 5064 unsigned long reclaimable; 5065 struct zone *zone; 5066 int lru; 5067 5068 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 5069 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 5070 5071 for_each_zone(zone) 5072 wmark_low += low_wmark_pages(zone); 5073 5074 /* 5075 * Estimate the amount of memory available for userspace allocations, 5076 * without causing swapping. 5077 */ 5078 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages; 5079 5080 /* 5081 * Not all the page cache can be freed, otherwise the system will 5082 * start swapping. Assume at least half of the page cache, or the 5083 * low watermark worth of cache, needs to stay. 5084 */ 5085 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 5086 pagecache -= min(pagecache / 2, wmark_low); 5087 available += pagecache; 5088 5089 /* 5090 * Part of the reclaimable slab and other kernel memory consists of 5091 * items that are in use, and cannot be freed. Cap this estimate at the 5092 * low watermark. 5093 */ 5094 reclaimable = global_node_page_state(NR_SLAB_RECLAIMABLE) + 5095 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE); 5096 available += reclaimable - min(reclaimable / 2, wmark_low); 5097 5098 if (available < 0) 5099 available = 0; 5100 return available; 5101 } 5102 EXPORT_SYMBOL_GPL(si_mem_available); 5103 5104 void si_meminfo(struct sysinfo *val) 5105 { 5106 val->totalram = totalram_pages(); 5107 val->sharedram = global_node_page_state(NR_SHMEM); 5108 val->freeram = global_zone_page_state(NR_FREE_PAGES); 5109 val->bufferram = nr_blockdev_pages(); 5110 val->totalhigh = totalhigh_pages(); 5111 val->freehigh = nr_free_highpages(); 5112 val->mem_unit = PAGE_SIZE; 5113 } 5114 5115 EXPORT_SYMBOL(si_meminfo); 5116 5117 #ifdef CONFIG_NUMA 5118 void si_meminfo_node(struct sysinfo *val, int nid) 5119 { 5120 int zone_type; /* needs to be signed */ 5121 unsigned long managed_pages = 0; 5122 unsigned long managed_highpages = 0; 5123 unsigned long free_highpages = 0; 5124 pg_data_t *pgdat = NODE_DATA(nid); 5125 5126 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 5127 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]); 5128 val->totalram = managed_pages; 5129 val->sharedram = node_page_state(pgdat, NR_SHMEM); 5130 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 5131 #ifdef CONFIG_HIGHMEM 5132 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 5133 struct zone *zone = &pgdat->node_zones[zone_type]; 5134 5135 if (is_highmem(zone)) { 5136 managed_highpages += zone_managed_pages(zone); 5137 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 5138 } 5139 } 5140 val->totalhigh = managed_highpages; 5141 val->freehigh = free_highpages; 5142 #else 5143 val->totalhigh = managed_highpages; 5144 val->freehigh = free_highpages; 5145 #endif 5146 val->mem_unit = PAGE_SIZE; 5147 } 5148 #endif 5149 5150 /* 5151 * Determine whether the node should be displayed or not, depending on whether 5152 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 5153 */ 5154 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) 5155 { 5156 if (!(flags & SHOW_MEM_FILTER_NODES)) 5157 return false; 5158 5159 /* 5160 * no node mask - aka implicit memory numa policy. Do not bother with 5161 * the synchronization - read_mems_allowed_begin - because we do not 5162 * have to be precise here. 5163 */ 5164 if (!nodemask) 5165 nodemask = &cpuset_current_mems_allowed; 5166 5167 return !node_isset(nid, *nodemask); 5168 } 5169 5170 #define K(x) ((x) << (PAGE_SHIFT-10)) 5171 5172 static void show_migration_types(unsigned char type) 5173 { 5174 static const char types[MIGRATE_TYPES] = { 5175 [MIGRATE_UNMOVABLE] = 'U', 5176 [MIGRATE_MOVABLE] = 'M', 5177 [MIGRATE_RECLAIMABLE] = 'E', 5178 [MIGRATE_HIGHATOMIC] = 'H', 5179 #ifdef CONFIG_CMA 5180 [MIGRATE_CMA] = 'C', 5181 #endif 5182 #ifdef CONFIG_MEMORY_ISOLATION 5183 [MIGRATE_ISOLATE] = 'I', 5184 #endif 5185 }; 5186 char tmp[MIGRATE_TYPES + 1]; 5187 char *p = tmp; 5188 int i; 5189 5190 for (i = 0; i < MIGRATE_TYPES; i++) { 5191 if (type & (1 << i)) 5192 *p++ = types[i]; 5193 } 5194 5195 *p = '\0'; 5196 printk(KERN_CONT "(%s) ", tmp); 5197 } 5198 5199 /* 5200 * Show free area list (used inside shift_scroll-lock stuff) 5201 * We also calculate the percentage fragmentation. We do this by counting the 5202 * memory on each free list with the exception of the first item on the list. 5203 * 5204 * Bits in @filter: 5205 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 5206 * cpuset. 5207 */ 5208 void show_free_areas(unsigned int filter, nodemask_t *nodemask) 5209 { 5210 unsigned long free_pcp = 0; 5211 int cpu; 5212 struct zone *zone; 5213 pg_data_t *pgdat; 5214 5215 for_each_populated_zone(zone) { 5216 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5217 continue; 5218 5219 for_each_online_cpu(cpu) 5220 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 5221 } 5222 5223 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 5224 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 5225 " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n" 5226 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 5227 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 5228 " free:%lu free_pcp:%lu free_cma:%lu\n", 5229 global_node_page_state(NR_ACTIVE_ANON), 5230 global_node_page_state(NR_INACTIVE_ANON), 5231 global_node_page_state(NR_ISOLATED_ANON), 5232 global_node_page_state(NR_ACTIVE_FILE), 5233 global_node_page_state(NR_INACTIVE_FILE), 5234 global_node_page_state(NR_ISOLATED_FILE), 5235 global_node_page_state(NR_UNEVICTABLE), 5236 global_node_page_state(NR_FILE_DIRTY), 5237 global_node_page_state(NR_WRITEBACK), 5238 global_node_page_state(NR_UNSTABLE_NFS), 5239 global_node_page_state(NR_SLAB_RECLAIMABLE), 5240 global_node_page_state(NR_SLAB_UNRECLAIMABLE), 5241 global_node_page_state(NR_FILE_MAPPED), 5242 global_node_page_state(NR_SHMEM), 5243 global_zone_page_state(NR_PAGETABLE), 5244 global_zone_page_state(NR_BOUNCE), 5245 global_zone_page_state(NR_FREE_PAGES), 5246 free_pcp, 5247 global_zone_page_state(NR_FREE_CMA_PAGES)); 5248 5249 for_each_online_pgdat(pgdat) { 5250 if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) 5251 continue; 5252 5253 printk("Node %d" 5254 " active_anon:%lukB" 5255 " inactive_anon:%lukB" 5256 " active_file:%lukB" 5257 " inactive_file:%lukB" 5258 " unevictable:%lukB" 5259 " isolated(anon):%lukB" 5260 " isolated(file):%lukB" 5261 " mapped:%lukB" 5262 " dirty:%lukB" 5263 " writeback:%lukB" 5264 " shmem:%lukB" 5265 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5266 " shmem_thp: %lukB" 5267 " shmem_pmdmapped: %lukB" 5268 " anon_thp: %lukB" 5269 #endif 5270 " writeback_tmp:%lukB" 5271 " unstable:%lukB" 5272 " all_unreclaimable? %s" 5273 "\n", 5274 pgdat->node_id, 5275 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 5276 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 5277 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 5278 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 5279 K(node_page_state(pgdat, NR_UNEVICTABLE)), 5280 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 5281 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 5282 K(node_page_state(pgdat, NR_FILE_MAPPED)), 5283 K(node_page_state(pgdat, NR_FILE_DIRTY)), 5284 K(node_page_state(pgdat, NR_WRITEBACK)), 5285 K(node_page_state(pgdat, NR_SHMEM)), 5286 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5287 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), 5288 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) 5289 * HPAGE_PMD_NR), 5290 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR), 5291 #endif 5292 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 5293 K(node_page_state(pgdat, NR_UNSTABLE_NFS)), 5294 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ? 5295 "yes" : "no"); 5296 } 5297 5298 for_each_populated_zone(zone) { 5299 int i; 5300 5301 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5302 continue; 5303 5304 free_pcp = 0; 5305 for_each_online_cpu(cpu) 5306 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 5307 5308 show_node(zone); 5309 printk(KERN_CONT 5310 "%s" 5311 " free:%lukB" 5312 " min:%lukB" 5313 " low:%lukB" 5314 " high:%lukB" 5315 " active_anon:%lukB" 5316 " inactive_anon:%lukB" 5317 " active_file:%lukB" 5318 " inactive_file:%lukB" 5319 " unevictable:%lukB" 5320 " writepending:%lukB" 5321 " present:%lukB" 5322 " managed:%lukB" 5323 " mlocked:%lukB" 5324 " kernel_stack:%lukB" 5325 " pagetables:%lukB" 5326 " bounce:%lukB" 5327 " free_pcp:%lukB" 5328 " local_pcp:%ukB" 5329 " free_cma:%lukB" 5330 "\n", 5331 zone->name, 5332 K(zone_page_state(zone, NR_FREE_PAGES)), 5333 K(min_wmark_pages(zone)), 5334 K(low_wmark_pages(zone)), 5335 K(high_wmark_pages(zone)), 5336 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 5337 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 5338 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 5339 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 5340 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 5341 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 5342 K(zone->present_pages), 5343 K(zone_managed_pages(zone)), 5344 K(zone_page_state(zone, NR_MLOCK)), 5345 zone_page_state(zone, NR_KERNEL_STACK_KB), 5346 K(zone_page_state(zone, NR_PAGETABLE)), 5347 K(zone_page_state(zone, NR_BOUNCE)), 5348 K(free_pcp), 5349 K(this_cpu_read(zone->pageset->pcp.count)), 5350 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 5351 printk("lowmem_reserve[]:"); 5352 for (i = 0; i < MAX_NR_ZONES; i++) 5353 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); 5354 printk(KERN_CONT "\n"); 5355 } 5356 5357 for_each_populated_zone(zone) { 5358 unsigned int order; 5359 unsigned long nr[MAX_ORDER], flags, total = 0; 5360 unsigned char types[MAX_ORDER]; 5361 5362 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5363 continue; 5364 show_node(zone); 5365 printk(KERN_CONT "%s: ", zone->name); 5366 5367 spin_lock_irqsave(&zone->lock, flags); 5368 for (order = 0; order < MAX_ORDER; order++) { 5369 struct free_area *area = &zone->free_area[order]; 5370 int type; 5371 5372 nr[order] = area->nr_free; 5373 total += nr[order] << order; 5374 5375 types[order] = 0; 5376 for (type = 0; type < MIGRATE_TYPES; type++) { 5377 if (!free_area_empty(area, type)) 5378 types[order] |= 1 << type; 5379 } 5380 } 5381 spin_unlock_irqrestore(&zone->lock, flags); 5382 for (order = 0; order < MAX_ORDER; order++) { 5383 printk(KERN_CONT "%lu*%lukB ", 5384 nr[order], K(1UL) << order); 5385 if (nr[order]) 5386 show_migration_types(types[order]); 5387 } 5388 printk(KERN_CONT "= %lukB\n", K(total)); 5389 } 5390 5391 hugetlb_show_meminfo(); 5392 5393 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 5394 5395 show_swap_cache_info(); 5396 } 5397 5398 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 5399 { 5400 zoneref->zone = zone; 5401 zoneref->zone_idx = zone_idx(zone); 5402 } 5403 5404 /* 5405 * Builds allocation fallback zone lists. 5406 * 5407 * Add all populated zones of a node to the zonelist. 5408 */ 5409 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) 5410 { 5411 struct zone *zone; 5412 enum zone_type zone_type = MAX_NR_ZONES; 5413 int nr_zones = 0; 5414 5415 do { 5416 zone_type--; 5417 zone = pgdat->node_zones + zone_type; 5418 if (managed_zone(zone)) { 5419 zoneref_set_zone(zone, &zonerefs[nr_zones++]); 5420 check_highest_zone(zone_type); 5421 } 5422 } while (zone_type); 5423 5424 return nr_zones; 5425 } 5426 5427 #ifdef CONFIG_NUMA 5428 5429 static int __parse_numa_zonelist_order(char *s) 5430 { 5431 /* 5432 * We used to support different zonlists modes but they turned 5433 * out to be just not useful. Let's keep the warning in place 5434 * if somebody still use the cmd line parameter so that we do 5435 * not fail it silently 5436 */ 5437 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) { 5438 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s); 5439 return -EINVAL; 5440 } 5441 return 0; 5442 } 5443 5444 static __init int setup_numa_zonelist_order(char *s) 5445 { 5446 if (!s) 5447 return 0; 5448 5449 return __parse_numa_zonelist_order(s); 5450 } 5451 early_param("numa_zonelist_order", setup_numa_zonelist_order); 5452 5453 char numa_zonelist_order[] = "Node"; 5454 5455 /* 5456 * sysctl handler for numa_zonelist_order 5457 */ 5458 int numa_zonelist_order_handler(struct ctl_table *table, int write, 5459 void __user *buffer, size_t *length, 5460 loff_t *ppos) 5461 { 5462 char *str; 5463 int ret; 5464 5465 if (!write) 5466 return proc_dostring(table, write, buffer, length, ppos); 5467 str = memdup_user_nul(buffer, 16); 5468 if (IS_ERR(str)) 5469 return PTR_ERR(str); 5470 5471 ret = __parse_numa_zonelist_order(str); 5472 kfree(str); 5473 return ret; 5474 } 5475 5476 5477 #define MAX_NODE_LOAD (nr_online_nodes) 5478 static int node_load[MAX_NUMNODES]; 5479 5480 /** 5481 * find_next_best_node - find the next node that should appear in a given node's fallback list 5482 * @node: node whose fallback list we're appending 5483 * @used_node_mask: nodemask_t of already used nodes 5484 * 5485 * We use a number of factors to determine which is the next node that should 5486 * appear on a given node's fallback list. The node should not have appeared 5487 * already in @node's fallback list, and it should be the next closest node 5488 * according to the distance array (which contains arbitrary distance values 5489 * from each node to each node in the system), and should also prefer nodes 5490 * with no CPUs, since presumably they'll have very little allocation pressure 5491 * on them otherwise. 5492 * 5493 * Return: node id of the found node or %NUMA_NO_NODE if no node is found. 5494 */ 5495 static int find_next_best_node(int node, nodemask_t *used_node_mask) 5496 { 5497 int n, val; 5498 int min_val = INT_MAX; 5499 int best_node = NUMA_NO_NODE; 5500 const struct cpumask *tmp = cpumask_of_node(0); 5501 5502 /* Use the local node if we haven't already */ 5503 if (!node_isset(node, *used_node_mask)) { 5504 node_set(node, *used_node_mask); 5505 return node; 5506 } 5507 5508 for_each_node_state(n, N_MEMORY) { 5509 5510 /* Don't want a node to appear more than once */ 5511 if (node_isset(n, *used_node_mask)) 5512 continue; 5513 5514 /* Use the distance array to find the distance */ 5515 val = node_distance(node, n); 5516 5517 /* Penalize nodes under us ("prefer the next node") */ 5518 val += (n < node); 5519 5520 /* Give preference to headless and unused nodes */ 5521 tmp = cpumask_of_node(n); 5522 if (!cpumask_empty(tmp)) 5523 val += PENALTY_FOR_NODE_WITH_CPUS; 5524 5525 /* Slight preference for less loaded node */ 5526 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 5527 val += node_load[n]; 5528 5529 if (val < min_val) { 5530 min_val = val; 5531 best_node = n; 5532 } 5533 } 5534 5535 if (best_node >= 0) 5536 node_set(best_node, *used_node_mask); 5537 5538 return best_node; 5539 } 5540 5541 5542 /* 5543 * Build zonelists ordered by node and zones within node. 5544 * This results in maximum locality--normal zone overflows into local 5545 * DMA zone, if any--but risks exhausting DMA zone. 5546 */ 5547 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, 5548 unsigned nr_nodes) 5549 { 5550 struct zoneref *zonerefs; 5551 int i; 5552 5553 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5554 5555 for (i = 0; i < nr_nodes; i++) { 5556 int nr_zones; 5557 5558 pg_data_t *node = NODE_DATA(node_order[i]); 5559 5560 nr_zones = build_zonerefs_node(node, zonerefs); 5561 zonerefs += nr_zones; 5562 } 5563 zonerefs->zone = NULL; 5564 zonerefs->zone_idx = 0; 5565 } 5566 5567 /* 5568 * Build gfp_thisnode zonelists 5569 */ 5570 static void build_thisnode_zonelists(pg_data_t *pgdat) 5571 { 5572 struct zoneref *zonerefs; 5573 int nr_zones; 5574 5575 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; 5576 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5577 zonerefs += nr_zones; 5578 zonerefs->zone = NULL; 5579 zonerefs->zone_idx = 0; 5580 } 5581 5582 /* 5583 * Build zonelists ordered by zone and nodes within zones. 5584 * This results in conserving DMA zone[s] until all Normal memory is 5585 * exhausted, but results in overflowing to remote node while memory 5586 * may still exist in local DMA zone. 5587 */ 5588 5589 static void build_zonelists(pg_data_t *pgdat) 5590 { 5591 static int node_order[MAX_NUMNODES]; 5592 int node, load, nr_nodes = 0; 5593 nodemask_t used_mask; 5594 int local_node, prev_node; 5595 5596 /* NUMA-aware ordering of nodes */ 5597 local_node = pgdat->node_id; 5598 load = nr_online_nodes; 5599 prev_node = local_node; 5600 nodes_clear(used_mask); 5601 5602 memset(node_order, 0, sizeof(node_order)); 5603 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 5604 /* 5605 * We don't want to pressure a particular node. 5606 * So adding penalty to the first node in same 5607 * distance group to make it round-robin. 5608 */ 5609 if (node_distance(local_node, node) != 5610 node_distance(local_node, prev_node)) 5611 node_load[node] = load; 5612 5613 node_order[nr_nodes++] = node; 5614 prev_node = node; 5615 load--; 5616 } 5617 5618 build_zonelists_in_node_order(pgdat, node_order, nr_nodes); 5619 build_thisnode_zonelists(pgdat); 5620 } 5621 5622 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5623 /* 5624 * Return node id of node used for "local" allocations. 5625 * I.e., first node id of first zone in arg node's generic zonelist. 5626 * Used for initializing percpu 'numa_mem', which is used primarily 5627 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 5628 */ 5629 int local_memory_node(int node) 5630 { 5631 struct zoneref *z; 5632 5633 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 5634 gfp_zone(GFP_KERNEL), 5635 NULL); 5636 return zone_to_nid(z->zone); 5637 } 5638 #endif 5639 5640 static void setup_min_unmapped_ratio(void); 5641 static void setup_min_slab_ratio(void); 5642 #else /* CONFIG_NUMA */ 5643 5644 static void build_zonelists(pg_data_t *pgdat) 5645 { 5646 int node, local_node; 5647 struct zoneref *zonerefs; 5648 int nr_zones; 5649 5650 local_node = pgdat->node_id; 5651 5652 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5653 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5654 zonerefs += nr_zones; 5655 5656 /* 5657 * Now we build the zonelist so that it contains the zones 5658 * of all the other nodes. 5659 * We don't want to pressure a particular node, so when 5660 * building the zones for node N, we make sure that the 5661 * zones coming right after the local ones are those from 5662 * node N+1 (modulo N) 5663 */ 5664 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 5665 if (!node_online(node)) 5666 continue; 5667 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5668 zonerefs += nr_zones; 5669 } 5670 for (node = 0; node < local_node; node++) { 5671 if (!node_online(node)) 5672 continue; 5673 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5674 zonerefs += nr_zones; 5675 } 5676 5677 zonerefs->zone = NULL; 5678 zonerefs->zone_idx = 0; 5679 } 5680 5681 #endif /* CONFIG_NUMA */ 5682 5683 /* 5684 * Boot pageset table. One per cpu which is going to be used for all 5685 * zones and all nodes. The parameters will be set in such a way 5686 * that an item put on a list will immediately be handed over to 5687 * the buddy list. This is safe since pageset manipulation is done 5688 * with interrupts disabled. 5689 * 5690 * The boot_pagesets must be kept even after bootup is complete for 5691 * unused processors and/or zones. They do play a role for bootstrapping 5692 * hotplugged processors. 5693 * 5694 * zoneinfo_show() and maybe other functions do 5695 * not check if the processor is online before following the pageset pointer. 5696 * Other parts of the kernel may not check if the zone is available. 5697 */ 5698 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); 5699 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); 5700 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 5701 5702 static void __build_all_zonelists(void *data) 5703 { 5704 int nid; 5705 int __maybe_unused cpu; 5706 pg_data_t *self = data; 5707 static DEFINE_SPINLOCK(lock); 5708 5709 spin_lock(&lock); 5710 5711 #ifdef CONFIG_NUMA 5712 memset(node_load, 0, sizeof(node_load)); 5713 #endif 5714 5715 /* 5716 * This node is hotadded and no memory is yet present. So just 5717 * building zonelists is fine - no need to touch other nodes. 5718 */ 5719 if (self && !node_online(self->node_id)) { 5720 build_zonelists(self); 5721 } else { 5722 for_each_online_node(nid) { 5723 pg_data_t *pgdat = NODE_DATA(nid); 5724 5725 build_zonelists(pgdat); 5726 } 5727 5728 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5729 /* 5730 * We now know the "local memory node" for each node-- 5731 * i.e., the node of the first zone in the generic zonelist. 5732 * Set up numa_mem percpu variable for on-line cpus. During 5733 * boot, only the boot cpu should be on-line; we'll init the 5734 * secondary cpus' numa_mem as they come on-line. During 5735 * node/memory hotplug, we'll fixup all on-line cpus. 5736 */ 5737 for_each_online_cpu(cpu) 5738 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 5739 #endif 5740 } 5741 5742 spin_unlock(&lock); 5743 } 5744 5745 static noinline void __init 5746 build_all_zonelists_init(void) 5747 { 5748 int cpu; 5749 5750 __build_all_zonelists(NULL); 5751 5752 /* 5753 * Initialize the boot_pagesets that are going to be used 5754 * for bootstrapping processors. The real pagesets for 5755 * each zone will be allocated later when the per cpu 5756 * allocator is available. 5757 * 5758 * boot_pagesets are used also for bootstrapping offline 5759 * cpus if the system is already booted because the pagesets 5760 * are needed to initialize allocators on a specific cpu too. 5761 * F.e. the percpu allocator needs the page allocator which 5762 * needs the percpu allocator in order to allocate its pagesets 5763 * (a chicken-egg dilemma). 5764 */ 5765 for_each_possible_cpu(cpu) 5766 setup_pageset(&per_cpu(boot_pageset, cpu), 0); 5767 5768 mminit_verify_zonelist(); 5769 cpuset_init_current_mems_allowed(); 5770 } 5771 5772 /* 5773 * unless system_state == SYSTEM_BOOTING. 5774 * 5775 * __ref due to call of __init annotated helper build_all_zonelists_init 5776 * [protected by SYSTEM_BOOTING]. 5777 */ 5778 void __ref build_all_zonelists(pg_data_t *pgdat) 5779 { 5780 if (system_state == SYSTEM_BOOTING) { 5781 build_all_zonelists_init(); 5782 } else { 5783 __build_all_zonelists(pgdat); 5784 /* cpuset refresh routine should be here */ 5785 } 5786 vm_total_pages = nr_free_pagecache_pages(); 5787 /* 5788 * Disable grouping by mobility if the number of pages in the 5789 * system is too low to allow the mechanism to work. It would be 5790 * more accurate, but expensive to check per-zone. This check is 5791 * made on memory-hotadd so a system can start with mobility 5792 * disabled and enable it later 5793 */ 5794 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 5795 page_group_by_mobility_disabled = 1; 5796 else 5797 page_group_by_mobility_disabled = 0; 5798 5799 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n", 5800 nr_online_nodes, 5801 page_group_by_mobility_disabled ? "off" : "on", 5802 vm_total_pages); 5803 #ifdef CONFIG_NUMA 5804 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 5805 #endif 5806 } 5807 5808 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 5809 static bool __meminit 5810 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 5811 { 5812 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5813 static struct memblock_region *r; 5814 5815 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 5816 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 5817 for_each_memblock(memory, r) { 5818 if (*pfn < memblock_region_memory_end_pfn(r)) 5819 break; 5820 } 5821 } 5822 if (*pfn >= memblock_region_memory_base_pfn(r) && 5823 memblock_is_mirror(r)) { 5824 *pfn = memblock_region_memory_end_pfn(r); 5825 return true; 5826 } 5827 } 5828 #endif 5829 return false; 5830 } 5831 5832 /* 5833 * Initially all pages are reserved - free ones are freed 5834 * up by memblock_free_all() once the early boot process is 5835 * done. Non-atomic initialization, single-pass. 5836 */ 5837 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, 5838 unsigned long start_pfn, enum memmap_context context, 5839 struct vmem_altmap *altmap) 5840 { 5841 unsigned long pfn, end_pfn = start_pfn + size; 5842 struct page *page; 5843 5844 if (highest_memmap_pfn < end_pfn - 1) 5845 highest_memmap_pfn = end_pfn - 1; 5846 5847 #ifdef CONFIG_ZONE_DEVICE 5848 /* 5849 * Honor reservation requested by the driver for this ZONE_DEVICE 5850 * memory. We limit the total number of pages to initialize to just 5851 * those that might contain the memory mapping. We will defer the 5852 * ZONE_DEVICE page initialization until after we have released 5853 * the hotplug lock. 5854 */ 5855 if (zone == ZONE_DEVICE) { 5856 if (!altmap) 5857 return; 5858 5859 if (start_pfn == altmap->base_pfn) 5860 start_pfn += altmap->reserve; 5861 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 5862 } 5863 #endif 5864 5865 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 5866 /* 5867 * There can be holes in boot-time mem_map[]s handed to this 5868 * function. They do not exist on hotplugged memory. 5869 */ 5870 if (context == MEMMAP_EARLY) { 5871 if (!early_pfn_valid(pfn)) 5872 continue; 5873 if (!early_pfn_in_nid(pfn, nid)) 5874 continue; 5875 if (overlap_memmap_init(zone, &pfn)) 5876 continue; 5877 if (defer_init(nid, pfn, end_pfn)) 5878 break; 5879 } 5880 5881 page = pfn_to_page(pfn); 5882 __init_single_page(page, pfn, zone, nid); 5883 if (context == MEMMAP_HOTPLUG) 5884 __SetPageReserved(page); 5885 5886 /* 5887 * Mark the block movable so that blocks are reserved for 5888 * movable at startup. This will force kernel allocations 5889 * to reserve their blocks rather than leaking throughout 5890 * the address space during boot when many long-lived 5891 * kernel allocations are made. 5892 * 5893 * bitmap is created for zone's valid pfn range. but memmap 5894 * can be created for invalid pages (for alignment) 5895 * check here not to call set_pageblock_migratetype() against 5896 * pfn out of zone. 5897 */ 5898 if (!(pfn & (pageblock_nr_pages - 1))) { 5899 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 5900 cond_resched(); 5901 } 5902 } 5903 } 5904 5905 #ifdef CONFIG_ZONE_DEVICE 5906 void __ref memmap_init_zone_device(struct zone *zone, 5907 unsigned long start_pfn, 5908 unsigned long size, 5909 struct dev_pagemap *pgmap) 5910 { 5911 unsigned long pfn, end_pfn = start_pfn + size; 5912 struct pglist_data *pgdat = zone->zone_pgdat; 5913 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 5914 unsigned long zone_idx = zone_idx(zone); 5915 unsigned long start = jiffies; 5916 int nid = pgdat->node_id; 5917 5918 if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE)) 5919 return; 5920 5921 /* 5922 * The call to memmap_init_zone should have already taken care 5923 * of the pages reserved for the memmap, so we can just jump to 5924 * the end of that region and start processing the device pages. 5925 */ 5926 if (altmap) { 5927 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 5928 size = end_pfn - start_pfn; 5929 } 5930 5931 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 5932 struct page *page = pfn_to_page(pfn); 5933 5934 __init_single_page(page, pfn, zone_idx, nid); 5935 5936 /* 5937 * Mark page reserved as it will need to wait for onlining 5938 * phase for it to be fully associated with a zone. 5939 * 5940 * We can use the non-atomic __set_bit operation for setting 5941 * the flag as we are still initializing the pages. 5942 */ 5943 __SetPageReserved(page); 5944 5945 /* 5946 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 5947 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 5948 * ever freed or placed on a driver-private list. 5949 */ 5950 page->pgmap = pgmap; 5951 page->zone_device_data = NULL; 5952 5953 /* 5954 * Mark the block movable so that blocks are reserved for 5955 * movable at startup. This will force kernel allocations 5956 * to reserve their blocks rather than leaking throughout 5957 * the address space during boot when many long-lived 5958 * kernel allocations are made. 5959 * 5960 * bitmap is created for zone's valid pfn range. but memmap 5961 * can be created for invalid pages (for alignment) 5962 * check here not to call set_pageblock_migratetype() against 5963 * pfn out of zone. 5964 * 5965 * Please note that MEMMAP_HOTPLUG path doesn't clear memmap 5966 * because this is done early in section_activate() 5967 */ 5968 if (!(pfn & (pageblock_nr_pages - 1))) { 5969 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 5970 cond_resched(); 5971 } 5972 } 5973 5974 pr_info("%s initialised, %lu pages in %ums\n", dev_name(pgmap->dev), 5975 size, jiffies_to_msecs(jiffies - start)); 5976 } 5977 5978 #endif 5979 static void __meminit zone_init_free_lists(struct zone *zone) 5980 { 5981 unsigned int order, t; 5982 for_each_migratetype_order(order, t) { 5983 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 5984 zone->free_area[order].nr_free = 0; 5985 } 5986 } 5987 5988 void __meminit __weak memmap_init(unsigned long size, int nid, 5989 unsigned long zone, unsigned long start_pfn) 5990 { 5991 memmap_init_zone(size, nid, zone, start_pfn, MEMMAP_EARLY, NULL); 5992 } 5993 5994 static int zone_batchsize(struct zone *zone) 5995 { 5996 #ifdef CONFIG_MMU 5997 int batch; 5998 5999 /* 6000 * The per-cpu-pages pools are set to around 1000th of the 6001 * size of the zone. 6002 */ 6003 batch = zone_managed_pages(zone) / 1024; 6004 /* But no more than a meg. */ 6005 if (batch * PAGE_SIZE > 1024 * 1024) 6006 batch = (1024 * 1024) / PAGE_SIZE; 6007 batch /= 4; /* We effectively *= 4 below */ 6008 if (batch < 1) 6009 batch = 1; 6010 6011 /* 6012 * Clamp the batch to a 2^n - 1 value. Having a power 6013 * of 2 value was found to be more likely to have 6014 * suboptimal cache aliasing properties in some cases. 6015 * 6016 * For example if 2 tasks are alternately allocating 6017 * batches of pages, one task can end up with a lot 6018 * of pages of one half of the possible page colors 6019 * and the other with pages of the other colors. 6020 */ 6021 batch = rounddown_pow_of_two(batch + batch/2) - 1; 6022 6023 return batch; 6024 6025 #else 6026 /* The deferral and batching of frees should be suppressed under NOMMU 6027 * conditions. 6028 * 6029 * The problem is that NOMMU needs to be able to allocate large chunks 6030 * of contiguous memory as there's no hardware page translation to 6031 * assemble apparent contiguous memory from discontiguous pages. 6032 * 6033 * Queueing large contiguous runs of pages for batching, however, 6034 * causes the pages to actually be freed in smaller chunks. As there 6035 * can be a significant delay between the individual batches being 6036 * recycled, this leads to the once large chunks of space being 6037 * fragmented and becoming unavailable for high-order allocations. 6038 */ 6039 return 0; 6040 #endif 6041 } 6042 6043 /* 6044 * pcp->high and pcp->batch values are related and dependent on one another: 6045 * ->batch must never be higher then ->high. 6046 * The following function updates them in a safe manner without read side 6047 * locking. 6048 * 6049 * Any new users of pcp->batch and pcp->high should ensure they can cope with 6050 * those fields changing asynchronously (acording the the above rule). 6051 * 6052 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 6053 * outside of boot time (or some other assurance that no concurrent updaters 6054 * exist). 6055 */ 6056 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 6057 unsigned long batch) 6058 { 6059 /* start with a fail safe value for batch */ 6060 pcp->batch = 1; 6061 smp_wmb(); 6062 6063 /* Update high, then batch, in order */ 6064 pcp->high = high; 6065 smp_wmb(); 6066 6067 pcp->batch = batch; 6068 } 6069 6070 /* a companion to pageset_set_high() */ 6071 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) 6072 { 6073 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); 6074 } 6075 6076 static void pageset_init(struct per_cpu_pageset *p) 6077 { 6078 struct per_cpu_pages *pcp; 6079 int migratetype; 6080 6081 memset(p, 0, sizeof(*p)); 6082 6083 pcp = &p->pcp; 6084 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) 6085 INIT_LIST_HEAD(&pcp->lists[migratetype]); 6086 } 6087 6088 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 6089 { 6090 pageset_init(p); 6091 pageset_set_batch(p, batch); 6092 } 6093 6094 /* 6095 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist 6096 * to the value high for the pageset p. 6097 */ 6098 static void pageset_set_high(struct per_cpu_pageset *p, 6099 unsigned long high) 6100 { 6101 unsigned long batch = max(1UL, high / 4); 6102 if ((high / 4) > (PAGE_SHIFT * 8)) 6103 batch = PAGE_SHIFT * 8; 6104 6105 pageset_update(&p->pcp, high, batch); 6106 } 6107 6108 static void pageset_set_high_and_batch(struct zone *zone, 6109 struct per_cpu_pageset *pcp) 6110 { 6111 if (percpu_pagelist_fraction) 6112 pageset_set_high(pcp, 6113 (zone_managed_pages(zone) / 6114 percpu_pagelist_fraction)); 6115 else 6116 pageset_set_batch(pcp, zone_batchsize(zone)); 6117 } 6118 6119 static void __meminit zone_pageset_init(struct zone *zone, int cpu) 6120 { 6121 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); 6122 6123 pageset_init(pcp); 6124 pageset_set_high_and_batch(zone, pcp); 6125 } 6126 6127 void __meminit setup_zone_pageset(struct zone *zone) 6128 { 6129 int cpu; 6130 zone->pageset = alloc_percpu(struct per_cpu_pageset); 6131 for_each_possible_cpu(cpu) 6132 zone_pageset_init(zone, cpu); 6133 } 6134 6135 /* 6136 * Allocate per cpu pagesets and initialize them. 6137 * Before this call only boot pagesets were available. 6138 */ 6139 void __init setup_per_cpu_pageset(void) 6140 { 6141 struct pglist_data *pgdat; 6142 struct zone *zone; 6143 6144 for_each_populated_zone(zone) 6145 setup_zone_pageset(zone); 6146 6147 for_each_online_pgdat(pgdat) 6148 pgdat->per_cpu_nodestats = 6149 alloc_percpu(struct per_cpu_nodestat); 6150 } 6151 6152 static __meminit void zone_pcp_init(struct zone *zone) 6153 { 6154 /* 6155 * per cpu subsystem is not up at this point. The following code 6156 * relies on the ability of the linker to provide the 6157 * offset of a (static) per cpu variable into the per cpu area. 6158 */ 6159 zone->pageset = &boot_pageset; 6160 6161 if (populated_zone(zone)) 6162 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", 6163 zone->name, zone->present_pages, 6164 zone_batchsize(zone)); 6165 } 6166 6167 void __meminit init_currently_empty_zone(struct zone *zone, 6168 unsigned long zone_start_pfn, 6169 unsigned long size) 6170 { 6171 struct pglist_data *pgdat = zone->zone_pgdat; 6172 int zone_idx = zone_idx(zone) + 1; 6173 6174 if (zone_idx > pgdat->nr_zones) 6175 pgdat->nr_zones = zone_idx; 6176 6177 zone->zone_start_pfn = zone_start_pfn; 6178 6179 mminit_dprintk(MMINIT_TRACE, "memmap_init", 6180 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 6181 pgdat->node_id, 6182 (unsigned long)zone_idx(zone), 6183 zone_start_pfn, (zone_start_pfn + size)); 6184 6185 zone_init_free_lists(zone); 6186 zone->initialized = 1; 6187 } 6188 6189 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6190 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID 6191 6192 /* 6193 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 6194 */ 6195 int __meminit __early_pfn_to_nid(unsigned long pfn, 6196 struct mminit_pfnnid_cache *state) 6197 { 6198 unsigned long start_pfn, end_pfn; 6199 int nid; 6200 6201 if (state->last_start <= pfn && pfn < state->last_end) 6202 return state->last_nid; 6203 6204 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 6205 if (nid != NUMA_NO_NODE) { 6206 state->last_start = start_pfn; 6207 state->last_end = end_pfn; 6208 state->last_nid = nid; 6209 } 6210 6211 return nid; 6212 } 6213 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ 6214 6215 /** 6216 * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range 6217 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed. 6218 * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid 6219 * 6220 * If an architecture guarantees that all ranges registered contain no holes 6221 * and may be freed, this this function may be used instead of calling 6222 * memblock_free_early_nid() manually. 6223 */ 6224 void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn) 6225 { 6226 unsigned long start_pfn, end_pfn; 6227 int i, this_nid; 6228 6229 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) { 6230 start_pfn = min(start_pfn, max_low_pfn); 6231 end_pfn = min(end_pfn, max_low_pfn); 6232 6233 if (start_pfn < end_pfn) 6234 memblock_free_early_nid(PFN_PHYS(start_pfn), 6235 (end_pfn - start_pfn) << PAGE_SHIFT, 6236 this_nid); 6237 } 6238 } 6239 6240 /** 6241 * sparse_memory_present_with_active_regions - Call memory_present for each active range 6242 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used. 6243 * 6244 * If an architecture guarantees that all ranges registered contain no holes and may 6245 * be freed, this function may be used instead of calling memory_present() manually. 6246 */ 6247 void __init sparse_memory_present_with_active_regions(int nid) 6248 { 6249 unsigned long start_pfn, end_pfn; 6250 int i, this_nid; 6251 6252 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) 6253 memory_present(this_nid, start_pfn, end_pfn); 6254 } 6255 6256 /** 6257 * get_pfn_range_for_nid - Return the start and end page frames for a node 6258 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 6259 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 6260 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 6261 * 6262 * It returns the start and end page frame of a node based on information 6263 * provided by memblock_set_node(). If called for a node 6264 * with no available memory, a warning is printed and the start and end 6265 * PFNs will be 0. 6266 */ 6267 void __init get_pfn_range_for_nid(unsigned int nid, 6268 unsigned long *start_pfn, unsigned long *end_pfn) 6269 { 6270 unsigned long this_start_pfn, this_end_pfn; 6271 int i; 6272 6273 *start_pfn = -1UL; 6274 *end_pfn = 0; 6275 6276 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 6277 *start_pfn = min(*start_pfn, this_start_pfn); 6278 *end_pfn = max(*end_pfn, this_end_pfn); 6279 } 6280 6281 if (*start_pfn == -1UL) 6282 *start_pfn = 0; 6283 } 6284 6285 /* 6286 * This finds a zone that can be used for ZONE_MOVABLE pages. The 6287 * assumption is made that zones within a node are ordered in monotonic 6288 * increasing memory addresses so that the "highest" populated zone is used 6289 */ 6290 static void __init find_usable_zone_for_movable(void) 6291 { 6292 int zone_index; 6293 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 6294 if (zone_index == ZONE_MOVABLE) 6295 continue; 6296 6297 if (arch_zone_highest_possible_pfn[zone_index] > 6298 arch_zone_lowest_possible_pfn[zone_index]) 6299 break; 6300 } 6301 6302 VM_BUG_ON(zone_index == -1); 6303 movable_zone = zone_index; 6304 } 6305 6306 /* 6307 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 6308 * because it is sized independent of architecture. Unlike the other zones, 6309 * the starting point for ZONE_MOVABLE is not fixed. It may be different 6310 * in each node depending on the size of each node and how evenly kernelcore 6311 * is distributed. This helper function adjusts the zone ranges 6312 * provided by the architecture for a given node by using the end of the 6313 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 6314 * zones within a node are in order of monotonic increases memory addresses 6315 */ 6316 static void __init adjust_zone_range_for_zone_movable(int nid, 6317 unsigned long zone_type, 6318 unsigned long node_start_pfn, 6319 unsigned long node_end_pfn, 6320 unsigned long *zone_start_pfn, 6321 unsigned long *zone_end_pfn) 6322 { 6323 /* Only adjust if ZONE_MOVABLE is on this node */ 6324 if (zone_movable_pfn[nid]) { 6325 /* Size ZONE_MOVABLE */ 6326 if (zone_type == ZONE_MOVABLE) { 6327 *zone_start_pfn = zone_movable_pfn[nid]; 6328 *zone_end_pfn = min(node_end_pfn, 6329 arch_zone_highest_possible_pfn[movable_zone]); 6330 6331 /* Adjust for ZONE_MOVABLE starting within this range */ 6332 } else if (!mirrored_kernelcore && 6333 *zone_start_pfn < zone_movable_pfn[nid] && 6334 *zone_end_pfn > zone_movable_pfn[nid]) { 6335 *zone_end_pfn = zone_movable_pfn[nid]; 6336 6337 /* Check if this whole range is within ZONE_MOVABLE */ 6338 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 6339 *zone_start_pfn = *zone_end_pfn; 6340 } 6341 } 6342 6343 /* 6344 * Return the number of pages a zone spans in a node, including holes 6345 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 6346 */ 6347 static unsigned long __init zone_spanned_pages_in_node(int nid, 6348 unsigned long zone_type, 6349 unsigned long node_start_pfn, 6350 unsigned long node_end_pfn, 6351 unsigned long *zone_start_pfn, 6352 unsigned long *zone_end_pfn, 6353 unsigned long *ignored) 6354 { 6355 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 6356 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 6357 /* When hotadd a new node from cpu_up(), the node should be empty */ 6358 if (!node_start_pfn && !node_end_pfn) 6359 return 0; 6360 6361 /* Get the start and end of the zone */ 6362 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 6363 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 6364 adjust_zone_range_for_zone_movable(nid, zone_type, 6365 node_start_pfn, node_end_pfn, 6366 zone_start_pfn, zone_end_pfn); 6367 6368 /* Check that this node has pages within the zone's required range */ 6369 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 6370 return 0; 6371 6372 /* Move the zone boundaries inside the node if necessary */ 6373 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 6374 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 6375 6376 /* Return the spanned pages */ 6377 return *zone_end_pfn - *zone_start_pfn; 6378 } 6379 6380 /* 6381 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 6382 * then all holes in the requested range will be accounted for. 6383 */ 6384 unsigned long __init __absent_pages_in_range(int nid, 6385 unsigned long range_start_pfn, 6386 unsigned long range_end_pfn) 6387 { 6388 unsigned long nr_absent = range_end_pfn - range_start_pfn; 6389 unsigned long start_pfn, end_pfn; 6390 int i; 6391 6392 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6393 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 6394 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 6395 nr_absent -= end_pfn - start_pfn; 6396 } 6397 return nr_absent; 6398 } 6399 6400 /** 6401 * absent_pages_in_range - Return number of page frames in holes within a range 6402 * @start_pfn: The start PFN to start searching for holes 6403 * @end_pfn: The end PFN to stop searching for holes 6404 * 6405 * Return: the number of pages frames in memory holes within a range. 6406 */ 6407 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 6408 unsigned long end_pfn) 6409 { 6410 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 6411 } 6412 6413 /* Return the number of page frames in holes in a zone on a node */ 6414 static unsigned long __init zone_absent_pages_in_node(int nid, 6415 unsigned long zone_type, 6416 unsigned long node_start_pfn, 6417 unsigned long node_end_pfn, 6418 unsigned long *ignored) 6419 { 6420 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 6421 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 6422 unsigned long zone_start_pfn, zone_end_pfn; 6423 unsigned long nr_absent; 6424 6425 /* When hotadd a new node from cpu_up(), the node should be empty */ 6426 if (!node_start_pfn && !node_end_pfn) 6427 return 0; 6428 6429 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 6430 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 6431 6432 adjust_zone_range_for_zone_movable(nid, zone_type, 6433 node_start_pfn, node_end_pfn, 6434 &zone_start_pfn, &zone_end_pfn); 6435 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 6436 6437 /* 6438 * ZONE_MOVABLE handling. 6439 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 6440 * and vice versa. 6441 */ 6442 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 6443 unsigned long start_pfn, end_pfn; 6444 struct memblock_region *r; 6445 6446 for_each_memblock(memory, r) { 6447 start_pfn = clamp(memblock_region_memory_base_pfn(r), 6448 zone_start_pfn, zone_end_pfn); 6449 end_pfn = clamp(memblock_region_memory_end_pfn(r), 6450 zone_start_pfn, zone_end_pfn); 6451 6452 if (zone_type == ZONE_MOVABLE && 6453 memblock_is_mirror(r)) 6454 nr_absent += end_pfn - start_pfn; 6455 6456 if (zone_type == ZONE_NORMAL && 6457 !memblock_is_mirror(r)) 6458 nr_absent += end_pfn - start_pfn; 6459 } 6460 } 6461 6462 return nr_absent; 6463 } 6464 6465 #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6466 static inline unsigned long __init zone_spanned_pages_in_node(int nid, 6467 unsigned long zone_type, 6468 unsigned long node_start_pfn, 6469 unsigned long node_end_pfn, 6470 unsigned long *zone_start_pfn, 6471 unsigned long *zone_end_pfn, 6472 unsigned long *zones_size) 6473 { 6474 unsigned int zone; 6475 6476 *zone_start_pfn = node_start_pfn; 6477 for (zone = 0; zone < zone_type; zone++) 6478 *zone_start_pfn += zones_size[zone]; 6479 6480 *zone_end_pfn = *zone_start_pfn + zones_size[zone_type]; 6481 6482 return zones_size[zone_type]; 6483 } 6484 6485 static inline unsigned long __init zone_absent_pages_in_node(int nid, 6486 unsigned long zone_type, 6487 unsigned long node_start_pfn, 6488 unsigned long node_end_pfn, 6489 unsigned long *zholes_size) 6490 { 6491 if (!zholes_size) 6492 return 0; 6493 6494 return zholes_size[zone_type]; 6495 } 6496 6497 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6498 6499 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 6500 unsigned long node_start_pfn, 6501 unsigned long node_end_pfn, 6502 unsigned long *zones_size, 6503 unsigned long *zholes_size) 6504 { 6505 unsigned long realtotalpages = 0, totalpages = 0; 6506 enum zone_type i; 6507 6508 for (i = 0; i < MAX_NR_ZONES; i++) { 6509 struct zone *zone = pgdat->node_zones + i; 6510 unsigned long zone_start_pfn, zone_end_pfn; 6511 unsigned long size, real_size; 6512 6513 size = zone_spanned_pages_in_node(pgdat->node_id, i, 6514 node_start_pfn, 6515 node_end_pfn, 6516 &zone_start_pfn, 6517 &zone_end_pfn, 6518 zones_size); 6519 real_size = size - zone_absent_pages_in_node(pgdat->node_id, i, 6520 node_start_pfn, node_end_pfn, 6521 zholes_size); 6522 if (size) 6523 zone->zone_start_pfn = zone_start_pfn; 6524 else 6525 zone->zone_start_pfn = 0; 6526 zone->spanned_pages = size; 6527 zone->present_pages = real_size; 6528 6529 totalpages += size; 6530 realtotalpages += real_size; 6531 } 6532 6533 pgdat->node_spanned_pages = totalpages; 6534 pgdat->node_present_pages = realtotalpages; 6535 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, 6536 realtotalpages); 6537 } 6538 6539 #ifndef CONFIG_SPARSEMEM 6540 /* 6541 * Calculate the size of the zone->blockflags rounded to an unsigned long 6542 * Start by making sure zonesize is a multiple of pageblock_order by rounding 6543 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 6544 * round what is now in bits to nearest long in bits, then return it in 6545 * bytes. 6546 */ 6547 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 6548 { 6549 unsigned long usemapsize; 6550 6551 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 6552 usemapsize = roundup(zonesize, pageblock_nr_pages); 6553 usemapsize = usemapsize >> pageblock_order; 6554 usemapsize *= NR_PAGEBLOCK_BITS; 6555 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 6556 6557 return usemapsize / 8; 6558 } 6559 6560 static void __ref setup_usemap(struct pglist_data *pgdat, 6561 struct zone *zone, 6562 unsigned long zone_start_pfn, 6563 unsigned long zonesize) 6564 { 6565 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); 6566 zone->pageblock_flags = NULL; 6567 if (usemapsize) { 6568 zone->pageblock_flags = 6569 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 6570 pgdat->node_id); 6571 if (!zone->pageblock_flags) 6572 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 6573 usemapsize, zone->name, pgdat->node_id); 6574 } 6575 } 6576 #else 6577 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, 6578 unsigned long zone_start_pfn, unsigned long zonesize) {} 6579 #endif /* CONFIG_SPARSEMEM */ 6580 6581 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 6582 6583 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 6584 void __init set_pageblock_order(void) 6585 { 6586 unsigned int order; 6587 6588 /* Check that pageblock_nr_pages has not already been setup */ 6589 if (pageblock_order) 6590 return; 6591 6592 if (HPAGE_SHIFT > PAGE_SHIFT) 6593 order = HUGETLB_PAGE_ORDER; 6594 else 6595 order = MAX_ORDER - 1; 6596 6597 /* 6598 * Assume the largest contiguous order of interest is a huge page. 6599 * This value may be variable depending on boot parameters on IA64 and 6600 * powerpc. 6601 */ 6602 pageblock_order = order; 6603 } 6604 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6605 6606 /* 6607 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 6608 * is unused as pageblock_order is set at compile-time. See 6609 * include/linux/pageblock-flags.h for the values of pageblock_order based on 6610 * the kernel config 6611 */ 6612 void __init set_pageblock_order(void) 6613 { 6614 } 6615 6616 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6617 6618 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 6619 unsigned long present_pages) 6620 { 6621 unsigned long pages = spanned_pages; 6622 6623 /* 6624 * Provide a more accurate estimation if there are holes within 6625 * the zone and SPARSEMEM is in use. If there are holes within the 6626 * zone, each populated memory region may cost us one or two extra 6627 * memmap pages due to alignment because memmap pages for each 6628 * populated regions may not be naturally aligned on page boundary. 6629 * So the (present_pages >> 4) heuristic is a tradeoff for that. 6630 */ 6631 if (spanned_pages > present_pages + (present_pages >> 4) && 6632 IS_ENABLED(CONFIG_SPARSEMEM)) 6633 pages = present_pages; 6634 6635 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 6636 } 6637 6638 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 6639 static void pgdat_init_split_queue(struct pglist_data *pgdat) 6640 { 6641 spin_lock_init(&pgdat->split_queue_lock); 6642 INIT_LIST_HEAD(&pgdat->split_queue); 6643 pgdat->split_queue_len = 0; 6644 } 6645 #else 6646 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 6647 #endif 6648 6649 #ifdef CONFIG_COMPACTION 6650 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 6651 { 6652 init_waitqueue_head(&pgdat->kcompactd_wait); 6653 } 6654 #else 6655 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 6656 #endif 6657 6658 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 6659 { 6660 pgdat_resize_init(pgdat); 6661 6662 pgdat_init_split_queue(pgdat); 6663 pgdat_init_kcompactd(pgdat); 6664 6665 init_waitqueue_head(&pgdat->kswapd_wait); 6666 init_waitqueue_head(&pgdat->pfmemalloc_wait); 6667 6668 pgdat_page_ext_init(pgdat); 6669 spin_lock_init(&pgdat->lru_lock); 6670 lruvec_init(node_lruvec(pgdat)); 6671 } 6672 6673 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 6674 unsigned long remaining_pages) 6675 { 6676 atomic_long_set(&zone->managed_pages, remaining_pages); 6677 zone_set_nid(zone, nid); 6678 zone->name = zone_names[idx]; 6679 zone->zone_pgdat = NODE_DATA(nid); 6680 spin_lock_init(&zone->lock); 6681 zone_seqlock_init(zone); 6682 zone_pcp_init(zone); 6683 } 6684 6685 /* 6686 * Set up the zone data structures 6687 * - init pgdat internals 6688 * - init all zones belonging to this node 6689 * 6690 * NOTE: this function is only called during memory hotplug 6691 */ 6692 #ifdef CONFIG_MEMORY_HOTPLUG 6693 void __ref free_area_init_core_hotplug(int nid) 6694 { 6695 enum zone_type z; 6696 pg_data_t *pgdat = NODE_DATA(nid); 6697 6698 pgdat_init_internals(pgdat); 6699 for (z = 0; z < MAX_NR_ZONES; z++) 6700 zone_init_internals(&pgdat->node_zones[z], z, nid, 0); 6701 } 6702 #endif 6703 6704 /* 6705 * Set up the zone data structures: 6706 * - mark all pages reserved 6707 * - mark all memory queues empty 6708 * - clear the memory bitmaps 6709 * 6710 * NOTE: pgdat should get zeroed by caller. 6711 * NOTE: this function is only called during early init. 6712 */ 6713 static void __init free_area_init_core(struct pglist_data *pgdat) 6714 { 6715 enum zone_type j; 6716 int nid = pgdat->node_id; 6717 6718 pgdat_init_internals(pgdat); 6719 pgdat->per_cpu_nodestats = &boot_nodestats; 6720 6721 for (j = 0; j < MAX_NR_ZONES; j++) { 6722 struct zone *zone = pgdat->node_zones + j; 6723 unsigned long size, freesize, memmap_pages; 6724 unsigned long zone_start_pfn = zone->zone_start_pfn; 6725 6726 size = zone->spanned_pages; 6727 freesize = zone->present_pages; 6728 6729 /* 6730 * Adjust freesize so that it accounts for how much memory 6731 * is used by this zone for memmap. This affects the watermark 6732 * and per-cpu initialisations 6733 */ 6734 memmap_pages = calc_memmap_size(size, freesize); 6735 if (!is_highmem_idx(j)) { 6736 if (freesize >= memmap_pages) { 6737 freesize -= memmap_pages; 6738 if (memmap_pages) 6739 printk(KERN_DEBUG 6740 " %s zone: %lu pages used for memmap\n", 6741 zone_names[j], memmap_pages); 6742 } else 6743 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n", 6744 zone_names[j], memmap_pages, freesize); 6745 } 6746 6747 /* Account for reserved pages */ 6748 if (j == 0 && freesize > dma_reserve) { 6749 freesize -= dma_reserve; 6750 printk(KERN_DEBUG " %s zone: %lu pages reserved\n", 6751 zone_names[0], dma_reserve); 6752 } 6753 6754 if (!is_highmem_idx(j)) 6755 nr_kernel_pages += freesize; 6756 /* Charge for highmem memmap if there are enough kernel pages */ 6757 else if (nr_kernel_pages > memmap_pages * 2) 6758 nr_kernel_pages -= memmap_pages; 6759 nr_all_pages += freesize; 6760 6761 /* 6762 * Set an approximate value for lowmem here, it will be adjusted 6763 * when the bootmem allocator frees pages into the buddy system. 6764 * And all highmem pages will be managed by the buddy system. 6765 */ 6766 zone_init_internals(zone, j, nid, freesize); 6767 6768 if (!size) 6769 continue; 6770 6771 set_pageblock_order(); 6772 setup_usemap(pgdat, zone, zone_start_pfn, size); 6773 init_currently_empty_zone(zone, zone_start_pfn, size); 6774 memmap_init(size, nid, j, zone_start_pfn); 6775 } 6776 } 6777 6778 #ifdef CONFIG_FLAT_NODE_MEM_MAP 6779 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) 6780 { 6781 unsigned long __maybe_unused start = 0; 6782 unsigned long __maybe_unused offset = 0; 6783 6784 /* Skip empty nodes */ 6785 if (!pgdat->node_spanned_pages) 6786 return; 6787 6788 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 6789 offset = pgdat->node_start_pfn - start; 6790 /* ia64 gets its own node_mem_map, before this, without bootmem */ 6791 if (!pgdat->node_mem_map) { 6792 unsigned long size, end; 6793 struct page *map; 6794 6795 /* 6796 * The zone's endpoints aren't required to be MAX_ORDER 6797 * aligned but the node_mem_map endpoints must be in order 6798 * for the buddy allocator to function correctly. 6799 */ 6800 end = pgdat_end_pfn(pgdat); 6801 end = ALIGN(end, MAX_ORDER_NR_PAGES); 6802 size = (end - start) * sizeof(struct page); 6803 map = memblock_alloc_node(size, SMP_CACHE_BYTES, 6804 pgdat->node_id); 6805 if (!map) 6806 panic("Failed to allocate %ld bytes for node %d memory map\n", 6807 size, pgdat->node_id); 6808 pgdat->node_mem_map = map + offset; 6809 } 6810 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 6811 __func__, pgdat->node_id, (unsigned long)pgdat, 6812 (unsigned long)pgdat->node_mem_map); 6813 #ifndef CONFIG_NEED_MULTIPLE_NODES 6814 /* 6815 * With no DISCONTIG, the global mem_map is just set as node 0's 6816 */ 6817 if (pgdat == NODE_DATA(0)) { 6818 mem_map = NODE_DATA(0)->node_mem_map; 6819 #if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM) 6820 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 6821 mem_map -= offset; 6822 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6823 } 6824 #endif 6825 } 6826 #else 6827 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { } 6828 #endif /* CONFIG_FLAT_NODE_MEM_MAP */ 6829 6830 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 6831 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 6832 { 6833 pgdat->first_deferred_pfn = ULONG_MAX; 6834 } 6835 #else 6836 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 6837 #endif 6838 6839 void __init free_area_init_node(int nid, unsigned long *zones_size, 6840 unsigned long node_start_pfn, 6841 unsigned long *zholes_size) 6842 { 6843 pg_data_t *pgdat = NODE_DATA(nid); 6844 unsigned long start_pfn = 0; 6845 unsigned long end_pfn = 0; 6846 6847 /* pg_data_t should be reset to zero when it's allocated */ 6848 WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx); 6849 6850 pgdat->node_id = nid; 6851 pgdat->node_start_pfn = node_start_pfn; 6852 pgdat->per_cpu_nodestats = NULL; 6853 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6854 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 6855 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 6856 (u64)start_pfn << PAGE_SHIFT, 6857 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 6858 #else 6859 start_pfn = node_start_pfn; 6860 #endif 6861 calculate_node_totalpages(pgdat, start_pfn, end_pfn, 6862 zones_size, zholes_size); 6863 6864 alloc_node_mem_map(pgdat); 6865 pgdat_set_deferred_range(pgdat); 6866 6867 free_area_init_core(pgdat); 6868 } 6869 6870 #if !defined(CONFIG_FLAT_NODE_MEM_MAP) 6871 /* 6872 * Zero all valid struct pages in range [spfn, epfn), return number of struct 6873 * pages zeroed 6874 */ 6875 static u64 zero_pfn_range(unsigned long spfn, unsigned long epfn) 6876 { 6877 unsigned long pfn; 6878 u64 pgcnt = 0; 6879 6880 for (pfn = spfn; pfn < epfn; pfn++) { 6881 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) { 6882 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages) 6883 + pageblock_nr_pages - 1; 6884 continue; 6885 } 6886 mm_zero_struct_page(pfn_to_page(pfn)); 6887 pgcnt++; 6888 } 6889 6890 return pgcnt; 6891 } 6892 6893 /* 6894 * Only struct pages that are backed by physical memory are zeroed and 6895 * initialized by going through __init_single_page(). But, there are some 6896 * struct pages which are reserved in memblock allocator and their fields 6897 * may be accessed (for example page_to_pfn() on some configuration accesses 6898 * flags). We must explicitly zero those struct pages. 6899 * 6900 * This function also addresses a similar issue where struct pages are left 6901 * uninitialized because the physical address range is not covered by 6902 * memblock.memory or memblock.reserved. That could happen when memblock 6903 * layout is manually configured via memmap=. 6904 */ 6905 void __init zero_resv_unavail(void) 6906 { 6907 phys_addr_t start, end; 6908 u64 i, pgcnt; 6909 phys_addr_t next = 0; 6910 6911 /* 6912 * Loop through unavailable ranges not covered by memblock.memory. 6913 */ 6914 pgcnt = 0; 6915 for_each_mem_range(i, &memblock.memory, NULL, 6916 NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, NULL) { 6917 if (next < start) 6918 pgcnt += zero_pfn_range(PFN_DOWN(next), PFN_UP(start)); 6919 next = end; 6920 } 6921 pgcnt += zero_pfn_range(PFN_DOWN(next), max_pfn); 6922 6923 /* 6924 * Struct pages that do not have backing memory. This could be because 6925 * firmware is using some of this memory, or for some other reasons. 6926 */ 6927 if (pgcnt) 6928 pr_info("Zeroed struct page in unavailable ranges: %lld pages", pgcnt); 6929 } 6930 #endif /* !CONFIG_FLAT_NODE_MEM_MAP */ 6931 6932 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6933 6934 #if MAX_NUMNODES > 1 6935 /* 6936 * Figure out the number of possible node ids. 6937 */ 6938 void __init setup_nr_node_ids(void) 6939 { 6940 unsigned int highest; 6941 6942 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 6943 nr_node_ids = highest + 1; 6944 } 6945 #endif 6946 6947 /** 6948 * node_map_pfn_alignment - determine the maximum internode alignment 6949 * 6950 * This function should be called after node map is populated and sorted. 6951 * It calculates the maximum power of two alignment which can distinguish 6952 * all the nodes. 6953 * 6954 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 6955 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 6956 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 6957 * shifted, 1GiB is enough and this function will indicate so. 6958 * 6959 * This is used to test whether pfn -> nid mapping of the chosen memory 6960 * model has fine enough granularity to avoid incorrect mapping for the 6961 * populated node map. 6962 * 6963 * Return: the determined alignment in pfn's. 0 if there is no alignment 6964 * requirement (single node). 6965 */ 6966 unsigned long __init node_map_pfn_alignment(void) 6967 { 6968 unsigned long accl_mask = 0, last_end = 0; 6969 unsigned long start, end, mask; 6970 int last_nid = NUMA_NO_NODE; 6971 int i, nid; 6972 6973 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 6974 if (!start || last_nid < 0 || last_nid == nid) { 6975 last_nid = nid; 6976 last_end = end; 6977 continue; 6978 } 6979 6980 /* 6981 * Start with a mask granular enough to pin-point to the 6982 * start pfn and tick off bits one-by-one until it becomes 6983 * too coarse to separate the current node from the last. 6984 */ 6985 mask = ~((1 << __ffs(start)) - 1); 6986 while (mask && last_end <= (start & (mask << 1))) 6987 mask <<= 1; 6988 6989 /* accumulate all internode masks */ 6990 accl_mask |= mask; 6991 } 6992 6993 /* convert mask to number of pages */ 6994 return ~accl_mask + 1; 6995 } 6996 6997 /* Find the lowest pfn for a node */ 6998 static unsigned long __init find_min_pfn_for_node(int nid) 6999 { 7000 unsigned long min_pfn = ULONG_MAX; 7001 unsigned long start_pfn; 7002 int i; 7003 7004 for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL) 7005 min_pfn = min(min_pfn, start_pfn); 7006 7007 if (min_pfn == ULONG_MAX) { 7008 pr_warn("Could not find start_pfn for node %d\n", nid); 7009 return 0; 7010 } 7011 7012 return min_pfn; 7013 } 7014 7015 /** 7016 * find_min_pfn_with_active_regions - Find the minimum PFN registered 7017 * 7018 * Return: the minimum PFN based on information provided via 7019 * memblock_set_node(). 7020 */ 7021 unsigned long __init find_min_pfn_with_active_regions(void) 7022 { 7023 return find_min_pfn_for_node(MAX_NUMNODES); 7024 } 7025 7026 /* 7027 * early_calculate_totalpages() 7028 * Sum pages in active regions for movable zone. 7029 * Populate N_MEMORY for calculating usable_nodes. 7030 */ 7031 static unsigned long __init early_calculate_totalpages(void) 7032 { 7033 unsigned long totalpages = 0; 7034 unsigned long start_pfn, end_pfn; 7035 int i, nid; 7036 7037 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7038 unsigned long pages = end_pfn - start_pfn; 7039 7040 totalpages += pages; 7041 if (pages) 7042 node_set_state(nid, N_MEMORY); 7043 } 7044 return totalpages; 7045 } 7046 7047 /* 7048 * Find the PFN the Movable zone begins in each node. Kernel memory 7049 * is spread evenly between nodes as long as the nodes have enough 7050 * memory. When they don't, some nodes will have more kernelcore than 7051 * others 7052 */ 7053 static void __init find_zone_movable_pfns_for_nodes(void) 7054 { 7055 int i, nid; 7056 unsigned long usable_startpfn; 7057 unsigned long kernelcore_node, kernelcore_remaining; 7058 /* save the state before borrow the nodemask */ 7059 nodemask_t saved_node_state = node_states[N_MEMORY]; 7060 unsigned long totalpages = early_calculate_totalpages(); 7061 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 7062 struct memblock_region *r; 7063 7064 /* Need to find movable_zone earlier when movable_node is specified. */ 7065 find_usable_zone_for_movable(); 7066 7067 /* 7068 * If movable_node is specified, ignore kernelcore and movablecore 7069 * options. 7070 */ 7071 if (movable_node_is_enabled()) { 7072 for_each_memblock(memory, r) { 7073 if (!memblock_is_hotpluggable(r)) 7074 continue; 7075 7076 nid = r->nid; 7077 7078 usable_startpfn = PFN_DOWN(r->base); 7079 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7080 min(usable_startpfn, zone_movable_pfn[nid]) : 7081 usable_startpfn; 7082 } 7083 7084 goto out2; 7085 } 7086 7087 /* 7088 * If kernelcore=mirror is specified, ignore movablecore option 7089 */ 7090 if (mirrored_kernelcore) { 7091 bool mem_below_4gb_not_mirrored = false; 7092 7093 for_each_memblock(memory, r) { 7094 if (memblock_is_mirror(r)) 7095 continue; 7096 7097 nid = r->nid; 7098 7099 usable_startpfn = memblock_region_memory_base_pfn(r); 7100 7101 if (usable_startpfn < 0x100000) { 7102 mem_below_4gb_not_mirrored = true; 7103 continue; 7104 } 7105 7106 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7107 min(usable_startpfn, zone_movable_pfn[nid]) : 7108 usable_startpfn; 7109 } 7110 7111 if (mem_below_4gb_not_mirrored) 7112 pr_warn("This configuration results in unmirrored kernel memory."); 7113 7114 goto out2; 7115 } 7116 7117 /* 7118 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 7119 * amount of necessary memory. 7120 */ 7121 if (required_kernelcore_percent) 7122 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 7123 10000UL; 7124 if (required_movablecore_percent) 7125 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 7126 10000UL; 7127 7128 /* 7129 * If movablecore= was specified, calculate what size of 7130 * kernelcore that corresponds so that memory usable for 7131 * any allocation type is evenly spread. If both kernelcore 7132 * and movablecore are specified, then the value of kernelcore 7133 * will be used for required_kernelcore if it's greater than 7134 * what movablecore would have allowed. 7135 */ 7136 if (required_movablecore) { 7137 unsigned long corepages; 7138 7139 /* 7140 * Round-up so that ZONE_MOVABLE is at least as large as what 7141 * was requested by the user 7142 */ 7143 required_movablecore = 7144 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 7145 required_movablecore = min(totalpages, required_movablecore); 7146 corepages = totalpages - required_movablecore; 7147 7148 required_kernelcore = max(required_kernelcore, corepages); 7149 } 7150 7151 /* 7152 * If kernelcore was not specified or kernelcore size is larger 7153 * than totalpages, there is no ZONE_MOVABLE. 7154 */ 7155 if (!required_kernelcore || required_kernelcore >= totalpages) 7156 goto out; 7157 7158 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 7159 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 7160 7161 restart: 7162 /* Spread kernelcore memory as evenly as possible throughout nodes */ 7163 kernelcore_node = required_kernelcore / usable_nodes; 7164 for_each_node_state(nid, N_MEMORY) { 7165 unsigned long start_pfn, end_pfn; 7166 7167 /* 7168 * Recalculate kernelcore_node if the division per node 7169 * now exceeds what is necessary to satisfy the requested 7170 * amount of memory for the kernel 7171 */ 7172 if (required_kernelcore < kernelcore_node) 7173 kernelcore_node = required_kernelcore / usable_nodes; 7174 7175 /* 7176 * As the map is walked, we track how much memory is usable 7177 * by the kernel using kernelcore_remaining. When it is 7178 * 0, the rest of the node is usable by ZONE_MOVABLE 7179 */ 7180 kernelcore_remaining = kernelcore_node; 7181 7182 /* Go through each range of PFNs within this node */ 7183 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7184 unsigned long size_pages; 7185 7186 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 7187 if (start_pfn >= end_pfn) 7188 continue; 7189 7190 /* Account for what is only usable for kernelcore */ 7191 if (start_pfn < usable_startpfn) { 7192 unsigned long kernel_pages; 7193 kernel_pages = min(end_pfn, usable_startpfn) 7194 - start_pfn; 7195 7196 kernelcore_remaining -= min(kernel_pages, 7197 kernelcore_remaining); 7198 required_kernelcore -= min(kernel_pages, 7199 required_kernelcore); 7200 7201 /* Continue if range is now fully accounted */ 7202 if (end_pfn <= usable_startpfn) { 7203 7204 /* 7205 * Push zone_movable_pfn to the end so 7206 * that if we have to rebalance 7207 * kernelcore across nodes, we will 7208 * not double account here 7209 */ 7210 zone_movable_pfn[nid] = end_pfn; 7211 continue; 7212 } 7213 start_pfn = usable_startpfn; 7214 } 7215 7216 /* 7217 * The usable PFN range for ZONE_MOVABLE is from 7218 * start_pfn->end_pfn. Calculate size_pages as the 7219 * number of pages used as kernelcore 7220 */ 7221 size_pages = end_pfn - start_pfn; 7222 if (size_pages > kernelcore_remaining) 7223 size_pages = kernelcore_remaining; 7224 zone_movable_pfn[nid] = start_pfn + size_pages; 7225 7226 /* 7227 * Some kernelcore has been met, update counts and 7228 * break if the kernelcore for this node has been 7229 * satisfied 7230 */ 7231 required_kernelcore -= min(required_kernelcore, 7232 size_pages); 7233 kernelcore_remaining -= size_pages; 7234 if (!kernelcore_remaining) 7235 break; 7236 } 7237 } 7238 7239 /* 7240 * If there is still required_kernelcore, we do another pass with one 7241 * less node in the count. This will push zone_movable_pfn[nid] further 7242 * along on the nodes that still have memory until kernelcore is 7243 * satisfied 7244 */ 7245 usable_nodes--; 7246 if (usable_nodes && required_kernelcore > usable_nodes) 7247 goto restart; 7248 7249 out2: 7250 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 7251 for (nid = 0; nid < MAX_NUMNODES; nid++) 7252 zone_movable_pfn[nid] = 7253 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 7254 7255 out: 7256 /* restore the node_state */ 7257 node_states[N_MEMORY] = saved_node_state; 7258 } 7259 7260 /* Any regular or high memory on that node ? */ 7261 static void check_for_memory(pg_data_t *pgdat, int nid) 7262 { 7263 enum zone_type zone_type; 7264 7265 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 7266 struct zone *zone = &pgdat->node_zones[zone_type]; 7267 if (populated_zone(zone)) { 7268 if (IS_ENABLED(CONFIG_HIGHMEM)) 7269 node_set_state(nid, N_HIGH_MEMORY); 7270 if (zone_type <= ZONE_NORMAL) 7271 node_set_state(nid, N_NORMAL_MEMORY); 7272 break; 7273 } 7274 } 7275 } 7276 7277 /** 7278 * free_area_init_nodes - Initialise all pg_data_t and zone data 7279 * @max_zone_pfn: an array of max PFNs for each zone 7280 * 7281 * This will call free_area_init_node() for each active node in the system. 7282 * Using the page ranges provided by memblock_set_node(), the size of each 7283 * zone in each node and their holes is calculated. If the maximum PFN 7284 * between two adjacent zones match, it is assumed that the zone is empty. 7285 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 7286 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 7287 * starts where the previous one ended. For example, ZONE_DMA32 starts 7288 * at arch_max_dma_pfn. 7289 */ 7290 void __init free_area_init_nodes(unsigned long *max_zone_pfn) 7291 { 7292 unsigned long start_pfn, end_pfn; 7293 int i, nid; 7294 7295 /* Record where the zone boundaries are */ 7296 memset(arch_zone_lowest_possible_pfn, 0, 7297 sizeof(arch_zone_lowest_possible_pfn)); 7298 memset(arch_zone_highest_possible_pfn, 0, 7299 sizeof(arch_zone_highest_possible_pfn)); 7300 7301 start_pfn = find_min_pfn_with_active_regions(); 7302 7303 for (i = 0; i < MAX_NR_ZONES; i++) { 7304 if (i == ZONE_MOVABLE) 7305 continue; 7306 7307 end_pfn = max(max_zone_pfn[i], start_pfn); 7308 arch_zone_lowest_possible_pfn[i] = start_pfn; 7309 arch_zone_highest_possible_pfn[i] = end_pfn; 7310 7311 start_pfn = end_pfn; 7312 } 7313 7314 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 7315 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 7316 find_zone_movable_pfns_for_nodes(); 7317 7318 /* Print out the zone ranges */ 7319 pr_info("Zone ranges:\n"); 7320 for (i = 0; i < MAX_NR_ZONES; i++) { 7321 if (i == ZONE_MOVABLE) 7322 continue; 7323 pr_info(" %-8s ", zone_names[i]); 7324 if (arch_zone_lowest_possible_pfn[i] == 7325 arch_zone_highest_possible_pfn[i]) 7326 pr_cont("empty\n"); 7327 else 7328 pr_cont("[mem %#018Lx-%#018Lx]\n", 7329 (u64)arch_zone_lowest_possible_pfn[i] 7330 << PAGE_SHIFT, 7331 ((u64)arch_zone_highest_possible_pfn[i] 7332 << PAGE_SHIFT) - 1); 7333 } 7334 7335 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 7336 pr_info("Movable zone start for each node\n"); 7337 for (i = 0; i < MAX_NUMNODES; i++) { 7338 if (zone_movable_pfn[i]) 7339 pr_info(" Node %d: %#018Lx\n", i, 7340 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 7341 } 7342 7343 /* 7344 * Print out the early node map, and initialize the 7345 * subsection-map relative to active online memory ranges to 7346 * enable future "sub-section" extensions of the memory map. 7347 */ 7348 pr_info("Early memory node ranges\n"); 7349 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7350 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 7351 (u64)start_pfn << PAGE_SHIFT, 7352 ((u64)end_pfn << PAGE_SHIFT) - 1); 7353 subsection_map_init(start_pfn, end_pfn - start_pfn); 7354 } 7355 7356 /* Initialise every node */ 7357 mminit_verify_pageflags_layout(); 7358 setup_nr_node_ids(); 7359 zero_resv_unavail(); 7360 for_each_online_node(nid) { 7361 pg_data_t *pgdat = NODE_DATA(nid); 7362 free_area_init_node(nid, NULL, 7363 find_min_pfn_for_node(nid), NULL); 7364 7365 /* Any memory on that node */ 7366 if (pgdat->node_present_pages) 7367 node_set_state(nid, N_MEMORY); 7368 check_for_memory(pgdat, nid); 7369 } 7370 } 7371 7372 static int __init cmdline_parse_core(char *p, unsigned long *core, 7373 unsigned long *percent) 7374 { 7375 unsigned long long coremem; 7376 char *endptr; 7377 7378 if (!p) 7379 return -EINVAL; 7380 7381 /* Value may be a percentage of total memory, otherwise bytes */ 7382 coremem = simple_strtoull(p, &endptr, 0); 7383 if (*endptr == '%') { 7384 /* Paranoid check for percent values greater than 100 */ 7385 WARN_ON(coremem > 100); 7386 7387 *percent = coremem; 7388 } else { 7389 coremem = memparse(p, &p); 7390 /* Paranoid check that UL is enough for the coremem value */ 7391 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 7392 7393 *core = coremem >> PAGE_SHIFT; 7394 *percent = 0UL; 7395 } 7396 return 0; 7397 } 7398 7399 /* 7400 * kernelcore=size sets the amount of memory for use for allocations that 7401 * cannot be reclaimed or migrated. 7402 */ 7403 static int __init cmdline_parse_kernelcore(char *p) 7404 { 7405 /* parse kernelcore=mirror */ 7406 if (parse_option_str(p, "mirror")) { 7407 mirrored_kernelcore = true; 7408 return 0; 7409 } 7410 7411 return cmdline_parse_core(p, &required_kernelcore, 7412 &required_kernelcore_percent); 7413 } 7414 7415 /* 7416 * movablecore=size sets the amount of memory for use for allocations that 7417 * can be reclaimed or migrated. 7418 */ 7419 static int __init cmdline_parse_movablecore(char *p) 7420 { 7421 return cmdline_parse_core(p, &required_movablecore, 7422 &required_movablecore_percent); 7423 } 7424 7425 early_param("kernelcore", cmdline_parse_kernelcore); 7426 early_param("movablecore", cmdline_parse_movablecore); 7427 7428 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 7429 7430 void adjust_managed_page_count(struct page *page, long count) 7431 { 7432 atomic_long_add(count, &page_zone(page)->managed_pages); 7433 totalram_pages_add(count); 7434 #ifdef CONFIG_HIGHMEM 7435 if (PageHighMem(page)) 7436 totalhigh_pages_add(count); 7437 #endif 7438 } 7439 EXPORT_SYMBOL(adjust_managed_page_count); 7440 7441 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 7442 { 7443 void *pos; 7444 unsigned long pages = 0; 7445 7446 start = (void *)PAGE_ALIGN((unsigned long)start); 7447 end = (void *)((unsigned long)end & PAGE_MASK); 7448 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 7449 struct page *page = virt_to_page(pos); 7450 void *direct_map_addr; 7451 7452 /* 7453 * 'direct_map_addr' might be different from 'pos' 7454 * because some architectures' virt_to_page() 7455 * work with aliases. Getting the direct map 7456 * address ensures that we get a _writeable_ 7457 * alias for the memset(). 7458 */ 7459 direct_map_addr = page_address(page); 7460 if ((unsigned int)poison <= 0xFF) 7461 memset(direct_map_addr, poison, PAGE_SIZE); 7462 7463 free_reserved_page(page); 7464 } 7465 7466 if (pages && s) 7467 pr_info("Freeing %s memory: %ldK\n", 7468 s, pages << (PAGE_SHIFT - 10)); 7469 7470 return pages; 7471 } 7472 7473 #ifdef CONFIG_HIGHMEM 7474 void free_highmem_page(struct page *page) 7475 { 7476 __free_reserved_page(page); 7477 totalram_pages_inc(); 7478 atomic_long_inc(&page_zone(page)->managed_pages); 7479 totalhigh_pages_inc(); 7480 } 7481 #endif 7482 7483 7484 void __init mem_init_print_info(const char *str) 7485 { 7486 unsigned long physpages, codesize, datasize, rosize, bss_size; 7487 unsigned long init_code_size, init_data_size; 7488 7489 physpages = get_num_physpages(); 7490 codesize = _etext - _stext; 7491 datasize = _edata - _sdata; 7492 rosize = __end_rodata - __start_rodata; 7493 bss_size = __bss_stop - __bss_start; 7494 init_data_size = __init_end - __init_begin; 7495 init_code_size = _einittext - _sinittext; 7496 7497 /* 7498 * Detect special cases and adjust section sizes accordingly: 7499 * 1) .init.* may be embedded into .data sections 7500 * 2) .init.text.* may be out of [__init_begin, __init_end], 7501 * please refer to arch/tile/kernel/vmlinux.lds.S. 7502 * 3) .rodata.* may be embedded into .text or .data sections. 7503 */ 7504 #define adj_init_size(start, end, size, pos, adj) \ 7505 do { \ 7506 if (start <= pos && pos < end && size > adj) \ 7507 size -= adj; \ 7508 } while (0) 7509 7510 adj_init_size(__init_begin, __init_end, init_data_size, 7511 _sinittext, init_code_size); 7512 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 7513 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 7514 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 7515 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 7516 7517 #undef adj_init_size 7518 7519 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 7520 #ifdef CONFIG_HIGHMEM 7521 ", %luK highmem" 7522 #endif 7523 "%s%s)\n", 7524 nr_free_pages() << (PAGE_SHIFT - 10), 7525 physpages << (PAGE_SHIFT - 10), 7526 codesize >> 10, datasize >> 10, rosize >> 10, 7527 (init_data_size + init_code_size) >> 10, bss_size >> 10, 7528 (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10), 7529 totalcma_pages << (PAGE_SHIFT - 10), 7530 #ifdef CONFIG_HIGHMEM 7531 totalhigh_pages() << (PAGE_SHIFT - 10), 7532 #endif 7533 str ? ", " : "", str ? str : ""); 7534 } 7535 7536 /** 7537 * set_dma_reserve - set the specified number of pages reserved in the first zone 7538 * @new_dma_reserve: The number of pages to mark reserved 7539 * 7540 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 7541 * In the DMA zone, a significant percentage may be consumed by kernel image 7542 * and other unfreeable allocations which can skew the watermarks badly. This 7543 * function may optionally be used to account for unfreeable pages in the 7544 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 7545 * smaller per-cpu batchsize. 7546 */ 7547 void __init set_dma_reserve(unsigned long new_dma_reserve) 7548 { 7549 dma_reserve = new_dma_reserve; 7550 } 7551 7552 void __init free_area_init(unsigned long *zones_size) 7553 { 7554 zero_resv_unavail(); 7555 free_area_init_node(0, zones_size, 7556 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); 7557 } 7558 7559 static int page_alloc_cpu_dead(unsigned int cpu) 7560 { 7561 7562 lru_add_drain_cpu(cpu); 7563 drain_pages(cpu); 7564 7565 /* 7566 * Spill the event counters of the dead processor 7567 * into the current processors event counters. 7568 * This artificially elevates the count of the current 7569 * processor. 7570 */ 7571 vm_events_fold_cpu(cpu); 7572 7573 /* 7574 * Zero the differential counters of the dead processor 7575 * so that the vm statistics are consistent. 7576 * 7577 * This is only okay since the processor is dead and cannot 7578 * race with what we are doing. 7579 */ 7580 cpu_vm_stats_fold(cpu); 7581 return 0; 7582 } 7583 7584 #ifdef CONFIG_NUMA 7585 int hashdist = HASHDIST_DEFAULT; 7586 7587 static int __init set_hashdist(char *str) 7588 { 7589 if (!str) 7590 return 0; 7591 hashdist = simple_strtoul(str, &str, 0); 7592 return 1; 7593 } 7594 __setup("hashdist=", set_hashdist); 7595 #endif 7596 7597 void __init page_alloc_init(void) 7598 { 7599 int ret; 7600 7601 #ifdef CONFIG_NUMA 7602 if (num_node_state(N_MEMORY) == 1) 7603 hashdist = 0; 7604 #endif 7605 7606 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD, 7607 "mm/page_alloc:dead", NULL, 7608 page_alloc_cpu_dead); 7609 WARN_ON(ret < 0); 7610 } 7611 7612 /* 7613 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 7614 * or min_free_kbytes changes. 7615 */ 7616 static void calculate_totalreserve_pages(void) 7617 { 7618 struct pglist_data *pgdat; 7619 unsigned long reserve_pages = 0; 7620 enum zone_type i, j; 7621 7622 for_each_online_pgdat(pgdat) { 7623 7624 pgdat->totalreserve_pages = 0; 7625 7626 for (i = 0; i < MAX_NR_ZONES; i++) { 7627 struct zone *zone = pgdat->node_zones + i; 7628 long max = 0; 7629 unsigned long managed_pages = zone_managed_pages(zone); 7630 7631 /* Find valid and maximum lowmem_reserve in the zone */ 7632 for (j = i; j < MAX_NR_ZONES; j++) { 7633 if (zone->lowmem_reserve[j] > max) 7634 max = zone->lowmem_reserve[j]; 7635 } 7636 7637 /* we treat the high watermark as reserved pages. */ 7638 max += high_wmark_pages(zone); 7639 7640 if (max > managed_pages) 7641 max = managed_pages; 7642 7643 pgdat->totalreserve_pages += max; 7644 7645 reserve_pages += max; 7646 } 7647 } 7648 totalreserve_pages = reserve_pages; 7649 } 7650 7651 /* 7652 * setup_per_zone_lowmem_reserve - called whenever 7653 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 7654 * has a correct pages reserved value, so an adequate number of 7655 * pages are left in the zone after a successful __alloc_pages(). 7656 */ 7657 static void setup_per_zone_lowmem_reserve(void) 7658 { 7659 struct pglist_data *pgdat; 7660 enum zone_type j, idx; 7661 7662 for_each_online_pgdat(pgdat) { 7663 for (j = 0; j < MAX_NR_ZONES; j++) { 7664 struct zone *zone = pgdat->node_zones + j; 7665 unsigned long managed_pages = zone_managed_pages(zone); 7666 7667 zone->lowmem_reserve[j] = 0; 7668 7669 idx = j; 7670 while (idx) { 7671 struct zone *lower_zone; 7672 7673 idx--; 7674 lower_zone = pgdat->node_zones + idx; 7675 7676 if (sysctl_lowmem_reserve_ratio[idx] < 1) { 7677 sysctl_lowmem_reserve_ratio[idx] = 0; 7678 lower_zone->lowmem_reserve[j] = 0; 7679 } else { 7680 lower_zone->lowmem_reserve[j] = 7681 managed_pages / sysctl_lowmem_reserve_ratio[idx]; 7682 } 7683 managed_pages += zone_managed_pages(lower_zone); 7684 } 7685 } 7686 } 7687 7688 /* update totalreserve_pages */ 7689 calculate_totalreserve_pages(); 7690 } 7691 7692 static void __setup_per_zone_wmarks(void) 7693 { 7694 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 7695 unsigned long lowmem_pages = 0; 7696 struct zone *zone; 7697 unsigned long flags; 7698 7699 /* Calculate total number of !ZONE_HIGHMEM pages */ 7700 for_each_zone(zone) { 7701 if (!is_highmem(zone)) 7702 lowmem_pages += zone_managed_pages(zone); 7703 } 7704 7705 for_each_zone(zone) { 7706 u64 tmp; 7707 7708 spin_lock_irqsave(&zone->lock, flags); 7709 tmp = (u64)pages_min * zone_managed_pages(zone); 7710 do_div(tmp, lowmem_pages); 7711 if (is_highmem(zone)) { 7712 /* 7713 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 7714 * need highmem pages, so cap pages_min to a small 7715 * value here. 7716 * 7717 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 7718 * deltas control async page reclaim, and so should 7719 * not be capped for highmem. 7720 */ 7721 unsigned long min_pages; 7722 7723 min_pages = zone_managed_pages(zone) / 1024; 7724 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 7725 zone->_watermark[WMARK_MIN] = min_pages; 7726 } else { 7727 /* 7728 * If it's a lowmem zone, reserve a number of pages 7729 * proportionate to the zone's size. 7730 */ 7731 zone->_watermark[WMARK_MIN] = tmp; 7732 } 7733 7734 /* 7735 * Set the kswapd watermarks distance according to the 7736 * scale factor in proportion to available memory, but 7737 * ensure a minimum size on small systems. 7738 */ 7739 tmp = max_t(u64, tmp >> 2, 7740 mult_frac(zone_managed_pages(zone), 7741 watermark_scale_factor, 10000)); 7742 7743 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 7744 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2; 7745 zone->watermark_boost = 0; 7746 7747 spin_unlock_irqrestore(&zone->lock, flags); 7748 } 7749 7750 /* update totalreserve_pages */ 7751 calculate_totalreserve_pages(); 7752 } 7753 7754 /** 7755 * setup_per_zone_wmarks - called when min_free_kbytes changes 7756 * or when memory is hot-{added|removed} 7757 * 7758 * Ensures that the watermark[min,low,high] values for each zone are set 7759 * correctly with respect to min_free_kbytes. 7760 */ 7761 void setup_per_zone_wmarks(void) 7762 { 7763 static DEFINE_SPINLOCK(lock); 7764 7765 spin_lock(&lock); 7766 __setup_per_zone_wmarks(); 7767 spin_unlock(&lock); 7768 } 7769 7770 /* 7771 * Initialise min_free_kbytes. 7772 * 7773 * For small machines we want it small (128k min). For large machines 7774 * we want it large (64MB max). But it is not linear, because network 7775 * bandwidth does not increase linearly with machine size. We use 7776 * 7777 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 7778 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 7779 * 7780 * which yields 7781 * 7782 * 16MB: 512k 7783 * 32MB: 724k 7784 * 64MB: 1024k 7785 * 128MB: 1448k 7786 * 256MB: 2048k 7787 * 512MB: 2896k 7788 * 1024MB: 4096k 7789 * 2048MB: 5792k 7790 * 4096MB: 8192k 7791 * 8192MB: 11584k 7792 * 16384MB: 16384k 7793 */ 7794 int __meminit init_per_zone_wmark_min(void) 7795 { 7796 unsigned long lowmem_kbytes; 7797 int new_min_free_kbytes; 7798 7799 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 7800 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 7801 7802 if (new_min_free_kbytes > user_min_free_kbytes) { 7803 min_free_kbytes = new_min_free_kbytes; 7804 if (min_free_kbytes < 128) 7805 min_free_kbytes = 128; 7806 if (min_free_kbytes > 65536) 7807 min_free_kbytes = 65536; 7808 } else { 7809 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 7810 new_min_free_kbytes, user_min_free_kbytes); 7811 } 7812 setup_per_zone_wmarks(); 7813 refresh_zone_stat_thresholds(); 7814 setup_per_zone_lowmem_reserve(); 7815 7816 #ifdef CONFIG_NUMA 7817 setup_min_unmapped_ratio(); 7818 setup_min_slab_ratio(); 7819 #endif 7820 7821 return 0; 7822 } 7823 core_initcall(init_per_zone_wmark_min) 7824 7825 /* 7826 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 7827 * that we can call two helper functions whenever min_free_kbytes 7828 * changes. 7829 */ 7830 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 7831 void __user *buffer, size_t *length, loff_t *ppos) 7832 { 7833 int rc; 7834 7835 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7836 if (rc) 7837 return rc; 7838 7839 if (write) { 7840 user_min_free_kbytes = min_free_kbytes; 7841 setup_per_zone_wmarks(); 7842 } 7843 return 0; 7844 } 7845 7846 int watermark_boost_factor_sysctl_handler(struct ctl_table *table, int write, 7847 void __user *buffer, size_t *length, loff_t *ppos) 7848 { 7849 int rc; 7850 7851 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7852 if (rc) 7853 return rc; 7854 7855 return 0; 7856 } 7857 7858 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 7859 void __user *buffer, size_t *length, loff_t *ppos) 7860 { 7861 int rc; 7862 7863 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7864 if (rc) 7865 return rc; 7866 7867 if (write) 7868 setup_per_zone_wmarks(); 7869 7870 return 0; 7871 } 7872 7873 #ifdef CONFIG_NUMA 7874 static void setup_min_unmapped_ratio(void) 7875 { 7876 pg_data_t *pgdat; 7877 struct zone *zone; 7878 7879 for_each_online_pgdat(pgdat) 7880 pgdat->min_unmapped_pages = 0; 7881 7882 for_each_zone(zone) 7883 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * 7884 sysctl_min_unmapped_ratio) / 100; 7885 } 7886 7887 7888 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 7889 void __user *buffer, size_t *length, loff_t *ppos) 7890 { 7891 int rc; 7892 7893 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7894 if (rc) 7895 return rc; 7896 7897 setup_min_unmapped_ratio(); 7898 7899 return 0; 7900 } 7901 7902 static void setup_min_slab_ratio(void) 7903 { 7904 pg_data_t *pgdat; 7905 struct zone *zone; 7906 7907 for_each_online_pgdat(pgdat) 7908 pgdat->min_slab_pages = 0; 7909 7910 for_each_zone(zone) 7911 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * 7912 sysctl_min_slab_ratio) / 100; 7913 } 7914 7915 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 7916 void __user *buffer, size_t *length, loff_t *ppos) 7917 { 7918 int rc; 7919 7920 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7921 if (rc) 7922 return rc; 7923 7924 setup_min_slab_ratio(); 7925 7926 return 0; 7927 } 7928 #endif 7929 7930 /* 7931 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 7932 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 7933 * whenever sysctl_lowmem_reserve_ratio changes. 7934 * 7935 * The reserve ratio obviously has absolutely no relation with the 7936 * minimum watermarks. The lowmem reserve ratio can only make sense 7937 * if in function of the boot time zone sizes. 7938 */ 7939 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 7940 void __user *buffer, size_t *length, loff_t *ppos) 7941 { 7942 proc_dointvec_minmax(table, write, buffer, length, ppos); 7943 setup_per_zone_lowmem_reserve(); 7944 return 0; 7945 } 7946 7947 /* 7948 * percpu_pagelist_fraction - changes the pcp->high for each zone on each 7949 * cpu. It is the fraction of total pages in each zone that a hot per cpu 7950 * pagelist can have before it gets flushed back to buddy allocator. 7951 */ 7952 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, 7953 void __user *buffer, size_t *length, loff_t *ppos) 7954 { 7955 struct zone *zone; 7956 int old_percpu_pagelist_fraction; 7957 int ret; 7958 7959 mutex_lock(&pcp_batch_high_lock); 7960 old_percpu_pagelist_fraction = percpu_pagelist_fraction; 7961 7962 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 7963 if (!write || ret < 0) 7964 goto out; 7965 7966 /* Sanity checking to avoid pcp imbalance */ 7967 if (percpu_pagelist_fraction && 7968 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { 7969 percpu_pagelist_fraction = old_percpu_pagelist_fraction; 7970 ret = -EINVAL; 7971 goto out; 7972 } 7973 7974 /* No change? */ 7975 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) 7976 goto out; 7977 7978 for_each_populated_zone(zone) { 7979 unsigned int cpu; 7980 7981 for_each_possible_cpu(cpu) 7982 pageset_set_high_and_batch(zone, 7983 per_cpu_ptr(zone->pageset, cpu)); 7984 } 7985 out: 7986 mutex_unlock(&pcp_batch_high_lock); 7987 return ret; 7988 } 7989 7990 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 7991 /* 7992 * Returns the number of pages that arch has reserved but 7993 * is not known to alloc_large_system_hash(). 7994 */ 7995 static unsigned long __init arch_reserved_kernel_pages(void) 7996 { 7997 return 0; 7998 } 7999 #endif 8000 8001 /* 8002 * Adaptive scale is meant to reduce sizes of hash tables on large memory 8003 * machines. As memory size is increased the scale is also increased but at 8004 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 8005 * quadruples the scale is increased by one, which means the size of hash table 8006 * only doubles, instead of quadrupling as well. 8007 * Because 32-bit systems cannot have large physical memory, where this scaling 8008 * makes sense, it is disabled on such platforms. 8009 */ 8010 #if __BITS_PER_LONG > 32 8011 #define ADAPT_SCALE_BASE (64ul << 30) 8012 #define ADAPT_SCALE_SHIFT 2 8013 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 8014 #endif 8015 8016 /* 8017 * allocate a large system hash table from bootmem 8018 * - it is assumed that the hash table must contain an exact power-of-2 8019 * quantity of entries 8020 * - limit is the number of hash buckets, not the total allocation size 8021 */ 8022 void *__init alloc_large_system_hash(const char *tablename, 8023 unsigned long bucketsize, 8024 unsigned long numentries, 8025 int scale, 8026 int flags, 8027 unsigned int *_hash_shift, 8028 unsigned int *_hash_mask, 8029 unsigned long low_limit, 8030 unsigned long high_limit) 8031 { 8032 unsigned long long max = high_limit; 8033 unsigned long log2qty, size; 8034 void *table = NULL; 8035 gfp_t gfp_flags; 8036 bool virt; 8037 8038 /* allow the kernel cmdline to have a say */ 8039 if (!numentries) { 8040 /* round applicable memory size up to nearest megabyte */ 8041 numentries = nr_kernel_pages; 8042 numentries -= arch_reserved_kernel_pages(); 8043 8044 /* It isn't necessary when PAGE_SIZE >= 1MB */ 8045 if (PAGE_SHIFT < 20) 8046 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 8047 8048 #if __BITS_PER_LONG > 32 8049 if (!high_limit) { 8050 unsigned long adapt; 8051 8052 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 8053 adapt <<= ADAPT_SCALE_SHIFT) 8054 scale++; 8055 } 8056 #endif 8057 8058 /* limit to 1 bucket per 2^scale bytes of low memory */ 8059 if (scale > PAGE_SHIFT) 8060 numentries >>= (scale - PAGE_SHIFT); 8061 else 8062 numentries <<= (PAGE_SHIFT - scale); 8063 8064 /* Make sure we've got at least a 0-order allocation.. */ 8065 if (unlikely(flags & HASH_SMALL)) { 8066 /* Makes no sense without HASH_EARLY */ 8067 WARN_ON(!(flags & HASH_EARLY)); 8068 if (!(numentries >> *_hash_shift)) { 8069 numentries = 1UL << *_hash_shift; 8070 BUG_ON(!numentries); 8071 } 8072 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 8073 numentries = PAGE_SIZE / bucketsize; 8074 } 8075 numentries = roundup_pow_of_two(numentries); 8076 8077 /* limit allocation size to 1/16 total memory by default */ 8078 if (max == 0) { 8079 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 8080 do_div(max, bucketsize); 8081 } 8082 max = min(max, 0x80000000ULL); 8083 8084 if (numentries < low_limit) 8085 numentries = low_limit; 8086 if (numentries > max) 8087 numentries = max; 8088 8089 log2qty = ilog2(numentries); 8090 8091 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 8092 do { 8093 virt = false; 8094 size = bucketsize << log2qty; 8095 if (flags & HASH_EARLY) { 8096 if (flags & HASH_ZERO) 8097 table = memblock_alloc(size, SMP_CACHE_BYTES); 8098 else 8099 table = memblock_alloc_raw(size, 8100 SMP_CACHE_BYTES); 8101 } else if (get_order(size) >= MAX_ORDER || hashdist) { 8102 table = __vmalloc(size, gfp_flags, PAGE_KERNEL); 8103 virt = true; 8104 } else { 8105 /* 8106 * If bucketsize is not a power-of-two, we may free 8107 * some pages at the end of hash table which 8108 * alloc_pages_exact() automatically does 8109 */ 8110 table = alloc_pages_exact(size, gfp_flags); 8111 kmemleak_alloc(table, size, 1, gfp_flags); 8112 } 8113 } while (!table && size > PAGE_SIZE && --log2qty); 8114 8115 if (!table) 8116 panic("Failed to allocate %s hash table\n", tablename); 8117 8118 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 8119 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, 8120 virt ? "vmalloc" : "linear"); 8121 8122 if (_hash_shift) 8123 *_hash_shift = log2qty; 8124 if (_hash_mask) 8125 *_hash_mask = (1 << log2qty) - 1; 8126 8127 return table; 8128 } 8129 8130 /* 8131 * This function checks whether pageblock includes unmovable pages or not. 8132 * If @count is not zero, it is okay to include less @count unmovable pages 8133 * 8134 * PageLRU check without isolation or lru_lock could race so that 8135 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable 8136 * check without lock_page also may miss some movable non-lru pages at 8137 * race condition. So you can't expect this function should be exact. 8138 */ 8139 bool has_unmovable_pages(struct zone *zone, struct page *page, int count, 8140 int migratetype, int flags) 8141 { 8142 unsigned long found; 8143 unsigned long iter = 0; 8144 unsigned long pfn = page_to_pfn(page); 8145 const char *reason = "unmovable page"; 8146 8147 /* 8148 * TODO we could make this much more efficient by not checking every 8149 * page in the range if we know all of them are in MOVABLE_ZONE and 8150 * that the movable zone guarantees that pages are migratable but 8151 * the later is not the case right now unfortunatelly. E.g. movablecore 8152 * can still lead to having bootmem allocations in zone_movable. 8153 */ 8154 8155 if (is_migrate_cma_page(page)) { 8156 /* 8157 * CMA allocations (alloc_contig_range) really need to mark 8158 * isolate CMA pageblocks even when they are not movable in fact 8159 * so consider them movable here. 8160 */ 8161 if (is_migrate_cma(migratetype)) 8162 return false; 8163 8164 reason = "CMA page"; 8165 goto unmovable; 8166 } 8167 8168 for (found = 0; iter < pageblock_nr_pages; iter++) { 8169 unsigned long check = pfn + iter; 8170 8171 if (!pfn_valid_within(check)) 8172 continue; 8173 8174 page = pfn_to_page(check); 8175 8176 if (PageReserved(page)) 8177 goto unmovable; 8178 8179 /* 8180 * If the zone is movable and we have ruled out all reserved 8181 * pages then it should be reasonably safe to assume the rest 8182 * is movable. 8183 */ 8184 if (zone_idx(zone) == ZONE_MOVABLE) 8185 continue; 8186 8187 /* 8188 * Hugepages are not in LRU lists, but they're movable. 8189 * We need not scan over tail pages because we don't 8190 * handle each tail page individually in migration. 8191 */ 8192 if (PageHuge(page)) { 8193 struct page *head = compound_head(page); 8194 unsigned int skip_pages; 8195 8196 if (!hugepage_migration_supported(page_hstate(head))) 8197 goto unmovable; 8198 8199 skip_pages = (1 << compound_order(head)) - (page - head); 8200 iter += skip_pages - 1; 8201 continue; 8202 } 8203 8204 /* 8205 * We can't use page_count without pin a page 8206 * because another CPU can free compound page. 8207 * This check already skips compound tails of THP 8208 * because their page->_refcount is zero at all time. 8209 */ 8210 if (!page_ref_count(page)) { 8211 if (PageBuddy(page)) 8212 iter += (1 << page_order(page)) - 1; 8213 continue; 8214 } 8215 8216 /* 8217 * The HWPoisoned page may be not in buddy system, and 8218 * page_count() is not 0. 8219 */ 8220 if ((flags & SKIP_HWPOISON) && PageHWPoison(page)) 8221 continue; 8222 8223 if (__PageMovable(page)) 8224 continue; 8225 8226 if (!PageLRU(page)) 8227 found++; 8228 /* 8229 * If there are RECLAIMABLE pages, we need to check 8230 * it. But now, memory offline itself doesn't call 8231 * shrink_node_slabs() and it still to be fixed. 8232 */ 8233 /* 8234 * If the page is not RAM, page_count()should be 0. 8235 * we don't need more check. This is an _used_ not-movable page. 8236 * 8237 * The problematic thing here is PG_reserved pages. PG_reserved 8238 * is set to both of a memory hole page and a _used_ kernel 8239 * page at boot. 8240 */ 8241 if (found > count) 8242 goto unmovable; 8243 } 8244 return false; 8245 unmovable: 8246 WARN_ON_ONCE(zone_idx(zone) == ZONE_MOVABLE); 8247 if (flags & REPORT_FAILURE) 8248 dump_page(pfn_to_page(pfn + iter), reason); 8249 return true; 8250 } 8251 8252 #ifdef CONFIG_CONTIG_ALLOC 8253 static unsigned long pfn_max_align_down(unsigned long pfn) 8254 { 8255 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, 8256 pageblock_nr_pages) - 1); 8257 } 8258 8259 static unsigned long pfn_max_align_up(unsigned long pfn) 8260 { 8261 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, 8262 pageblock_nr_pages)); 8263 } 8264 8265 /* [start, end) must belong to a single zone. */ 8266 static int __alloc_contig_migrate_range(struct compact_control *cc, 8267 unsigned long start, unsigned long end) 8268 { 8269 /* This function is based on compact_zone() from compaction.c. */ 8270 unsigned long nr_reclaimed; 8271 unsigned long pfn = start; 8272 unsigned int tries = 0; 8273 int ret = 0; 8274 8275 migrate_prep(); 8276 8277 while (pfn < end || !list_empty(&cc->migratepages)) { 8278 if (fatal_signal_pending(current)) { 8279 ret = -EINTR; 8280 break; 8281 } 8282 8283 if (list_empty(&cc->migratepages)) { 8284 cc->nr_migratepages = 0; 8285 pfn = isolate_migratepages_range(cc, pfn, end); 8286 if (!pfn) { 8287 ret = -EINTR; 8288 break; 8289 } 8290 tries = 0; 8291 } else if (++tries == 5) { 8292 ret = ret < 0 ? ret : -EBUSY; 8293 break; 8294 } 8295 8296 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 8297 &cc->migratepages); 8298 cc->nr_migratepages -= nr_reclaimed; 8299 8300 ret = migrate_pages(&cc->migratepages, alloc_migrate_target, 8301 NULL, 0, cc->mode, MR_CONTIG_RANGE); 8302 } 8303 if (ret < 0) { 8304 putback_movable_pages(&cc->migratepages); 8305 return ret; 8306 } 8307 return 0; 8308 } 8309 8310 /** 8311 * alloc_contig_range() -- tries to allocate given range of pages 8312 * @start: start PFN to allocate 8313 * @end: one-past-the-last PFN to allocate 8314 * @migratetype: migratetype of the underlaying pageblocks (either 8315 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 8316 * in range must have the same migratetype and it must 8317 * be either of the two. 8318 * @gfp_mask: GFP mask to use during compaction 8319 * 8320 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 8321 * aligned. The PFN range must belong to a single zone. 8322 * 8323 * The first thing this routine does is attempt to MIGRATE_ISOLATE all 8324 * pageblocks in the range. Once isolated, the pageblocks should not 8325 * be modified by others. 8326 * 8327 * Return: zero on success or negative error code. On success all 8328 * pages which PFN is in [start, end) are allocated for the caller and 8329 * need to be freed with free_contig_range(). 8330 */ 8331 int alloc_contig_range(unsigned long start, unsigned long end, 8332 unsigned migratetype, gfp_t gfp_mask) 8333 { 8334 unsigned long outer_start, outer_end; 8335 unsigned int order; 8336 int ret = 0; 8337 8338 struct compact_control cc = { 8339 .nr_migratepages = 0, 8340 .order = -1, 8341 .zone = page_zone(pfn_to_page(start)), 8342 .mode = MIGRATE_SYNC, 8343 .ignore_skip_hint = true, 8344 .no_set_skip_hint = true, 8345 .gfp_mask = current_gfp_context(gfp_mask), 8346 }; 8347 INIT_LIST_HEAD(&cc.migratepages); 8348 8349 /* 8350 * What we do here is we mark all pageblocks in range as 8351 * MIGRATE_ISOLATE. Because pageblock and max order pages may 8352 * have different sizes, and due to the way page allocator 8353 * work, we align the range to biggest of the two pages so 8354 * that page allocator won't try to merge buddies from 8355 * different pageblocks and change MIGRATE_ISOLATE to some 8356 * other migration type. 8357 * 8358 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 8359 * migrate the pages from an unaligned range (ie. pages that 8360 * we are interested in). This will put all the pages in 8361 * range back to page allocator as MIGRATE_ISOLATE. 8362 * 8363 * When this is done, we take the pages in range from page 8364 * allocator removing them from the buddy system. This way 8365 * page allocator will never consider using them. 8366 * 8367 * This lets us mark the pageblocks back as 8368 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 8369 * aligned range but not in the unaligned, original range are 8370 * put back to page allocator so that buddy can use them. 8371 */ 8372 8373 ret = start_isolate_page_range(pfn_max_align_down(start), 8374 pfn_max_align_up(end), migratetype, 0); 8375 if (ret < 0) 8376 return ret; 8377 8378 /* 8379 * In case of -EBUSY, we'd like to know which page causes problem. 8380 * So, just fall through. test_pages_isolated() has a tracepoint 8381 * which will report the busy page. 8382 * 8383 * It is possible that busy pages could become available before 8384 * the call to test_pages_isolated, and the range will actually be 8385 * allocated. So, if we fall through be sure to clear ret so that 8386 * -EBUSY is not accidentally used or returned to caller. 8387 */ 8388 ret = __alloc_contig_migrate_range(&cc, start, end); 8389 if (ret && ret != -EBUSY) 8390 goto done; 8391 ret =0; 8392 8393 /* 8394 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 8395 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 8396 * more, all pages in [start, end) are free in page allocator. 8397 * What we are going to do is to allocate all pages from 8398 * [start, end) (that is remove them from page allocator). 8399 * 8400 * The only problem is that pages at the beginning and at the 8401 * end of interesting range may be not aligned with pages that 8402 * page allocator holds, ie. they can be part of higher order 8403 * pages. Because of this, we reserve the bigger range and 8404 * once this is done free the pages we are not interested in. 8405 * 8406 * We don't have to hold zone->lock here because the pages are 8407 * isolated thus they won't get removed from buddy. 8408 */ 8409 8410 lru_add_drain_all(); 8411 8412 order = 0; 8413 outer_start = start; 8414 while (!PageBuddy(pfn_to_page(outer_start))) { 8415 if (++order >= MAX_ORDER) { 8416 outer_start = start; 8417 break; 8418 } 8419 outer_start &= ~0UL << order; 8420 } 8421 8422 if (outer_start != start) { 8423 order = page_order(pfn_to_page(outer_start)); 8424 8425 /* 8426 * outer_start page could be small order buddy page and 8427 * it doesn't include start page. Adjust outer_start 8428 * in this case to report failed page properly 8429 * on tracepoint in test_pages_isolated() 8430 */ 8431 if (outer_start + (1UL << order) <= start) 8432 outer_start = start; 8433 } 8434 8435 /* Make sure the range is really isolated. */ 8436 if (test_pages_isolated(outer_start, end, false)) { 8437 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n", 8438 __func__, outer_start, end); 8439 ret = -EBUSY; 8440 goto done; 8441 } 8442 8443 /* Grab isolated pages from freelists. */ 8444 outer_end = isolate_freepages_range(&cc, outer_start, end); 8445 if (!outer_end) { 8446 ret = -EBUSY; 8447 goto done; 8448 } 8449 8450 /* Free head and tail (if any) */ 8451 if (start != outer_start) 8452 free_contig_range(outer_start, start - outer_start); 8453 if (end != outer_end) 8454 free_contig_range(end, outer_end - end); 8455 8456 done: 8457 undo_isolate_page_range(pfn_max_align_down(start), 8458 pfn_max_align_up(end), migratetype); 8459 return ret; 8460 } 8461 #endif /* CONFIG_CONTIG_ALLOC */ 8462 8463 void free_contig_range(unsigned long pfn, unsigned int nr_pages) 8464 { 8465 unsigned int count = 0; 8466 8467 for (; nr_pages--; pfn++) { 8468 struct page *page = pfn_to_page(pfn); 8469 8470 count += page_count(page) != 1; 8471 __free_page(page); 8472 } 8473 WARN(count != 0, "%d pages are still in use!\n", count); 8474 } 8475 8476 #ifdef CONFIG_MEMORY_HOTPLUG 8477 /* 8478 * The zone indicated has a new number of managed_pages; batch sizes and percpu 8479 * page high values need to be recalulated. 8480 */ 8481 void __meminit zone_pcp_update(struct zone *zone) 8482 { 8483 unsigned cpu; 8484 mutex_lock(&pcp_batch_high_lock); 8485 for_each_possible_cpu(cpu) 8486 pageset_set_high_and_batch(zone, 8487 per_cpu_ptr(zone->pageset, cpu)); 8488 mutex_unlock(&pcp_batch_high_lock); 8489 } 8490 #endif 8491 8492 void zone_pcp_reset(struct zone *zone) 8493 { 8494 unsigned long flags; 8495 int cpu; 8496 struct per_cpu_pageset *pset; 8497 8498 /* avoid races with drain_pages() */ 8499 local_irq_save(flags); 8500 if (zone->pageset != &boot_pageset) { 8501 for_each_online_cpu(cpu) { 8502 pset = per_cpu_ptr(zone->pageset, cpu); 8503 drain_zonestat(zone, pset); 8504 } 8505 free_percpu(zone->pageset); 8506 zone->pageset = &boot_pageset; 8507 } 8508 local_irq_restore(flags); 8509 } 8510 8511 #ifdef CONFIG_MEMORY_HOTREMOVE 8512 /* 8513 * All pages in the range must be in a single zone and isolated 8514 * before calling this. 8515 */ 8516 unsigned long 8517 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 8518 { 8519 struct page *page; 8520 struct zone *zone; 8521 unsigned int order, i; 8522 unsigned long pfn; 8523 unsigned long flags; 8524 unsigned long offlined_pages = 0; 8525 8526 /* find the first valid pfn */ 8527 for (pfn = start_pfn; pfn < end_pfn; pfn++) 8528 if (pfn_valid(pfn)) 8529 break; 8530 if (pfn == end_pfn) 8531 return offlined_pages; 8532 8533 offline_mem_sections(pfn, end_pfn); 8534 zone = page_zone(pfn_to_page(pfn)); 8535 spin_lock_irqsave(&zone->lock, flags); 8536 pfn = start_pfn; 8537 while (pfn < end_pfn) { 8538 if (!pfn_valid(pfn)) { 8539 pfn++; 8540 continue; 8541 } 8542 page = pfn_to_page(pfn); 8543 /* 8544 * The HWPoisoned page may be not in buddy system, and 8545 * page_count() is not 0. 8546 */ 8547 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 8548 pfn++; 8549 SetPageReserved(page); 8550 offlined_pages++; 8551 continue; 8552 } 8553 8554 BUG_ON(page_count(page)); 8555 BUG_ON(!PageBuddy(page)); 8556 order = page_order(page); 8557 offlined_pages += 1 << order; 8558 #ifdef CONFIG_DEBUG_VM 8559 pr_info("remove from free list %lx %d %lx\n", 8560 pfn, 1 << order, end_pfn); 8561 #endif 8562 del_page_from_free_area(page, &zone->free_area[order]); 8563 for (i = 0; i < (1 << order); i++) 8564 SetPageReserved((page+i)); 8565 pfn += (1 << order); 8566 } 8567 spin_unlock_irqrestore(&zone->lock, flags); 8568 8569 return offlined_pages; 8570 } 8571 #endif 8572 8573 bool is_free_buddy_page(struct page *page) 8574 { 8575 struct zone *zone = page_zone(page); 8576 unsigned long pfn = page_to_pfn(page); 8577 unsigned long flags; 8578 unsigned int order; 8579 8580 spin_lock_irqsave(&zone->lock, flags); 8581 for (order = 0; order < MAX_ORDER; order++) { 8582 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8583 8584 if (PageBuddy(page_head) && page_order(page_head) >= order) 8585 break; 8586 } 8587 spin_unlock_irqrestore(&zone->lock, flags); 8588 8589 return order < MAX_ORDER; 8590 } 8591 8592 #ifdef CONFIG_MEMORY_FAILURE 8593 /* 8594 * Set PG_hwpoison flag if a given page is confirmed to be a free page. This 8595 * test is performed under the zone lock to prevent a race against page 8596 * allocation. 8597 */ 8598 bool set_hwpoison_free_buddy_page(struct page *page) 8599 { 8600 struct zone *zone = page_zone(page); 8601 unsigned long pfn = page_to_pfn(page); 8602 unsigned long flags; 8603 unsigned int order; 8604 bool hwpoisoned = false; 8605 8606 spin_lock_irqsave(&zone->lock, flags); 8607 for (order = 0; order < MAX_ORDER; order++) { 8608 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8609 8610 if (PageBuddy(page_head) && page_order(page_head) >= order) { 8611 if (!TestSetPageHWPoison(page)) 8612 hwpoisoned = true; 8613 break; 8614 } 8615 } 8616 spin_unlock_irqrestore(&zone->lock, flags); 8617 8618 return hwpoisoned; 8619 } 8620 #endif 8621