1 /* 2 * linux/mm/page_alloc.c 3 * 4 * Manages the free list, the system allocates free pages here. 5 * Note that kmalloc() lives in slab.c 6 * 7 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 8 * Swap reorganised 29.12.95, Stephen Tweedie 9 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 10 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 11 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 12 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 13 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 14 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 15 */ 16 17 #include <linux/stddef.h> 18 #include <linux/mm.h> 19 #include <linux/swap.h> 20 #include <linux/interrupt.h> 21 #include <linux/pagemap.h> 22 #include <linux/jiffies.h> 23 #include <linux/memblock.h> 24 #include <linux/compiler.h> 25 #include <linux/kernel.h> 26 #include <linux/kasan.h> 27 #include <linux/module.h> 28 #include <linux/suspend.h> 29 #include <linux/pagevec.h> 30 #include <linux/blkdev.h> 31 #include <linux/slab.h> 32 #include <linux/ratelimit.h> 33 #include <linux/oom.h> 34 #include <linux/topology.h> 35 #include <linux/sysctl.h> 36 #include <linux/cpu.h> 37 #include <linux/cpuset.h> 38 #include <linux/memory_hotplug.h> 39 #include <linux/nodemask.h> 40 #include <linux/vmalloc.h> 41 #include <linux/vmstat.h> 42 #include <linux/mempolicy.h> 43 #include <linux/memremap.h> 44 #include <linux/stop_machine.h> 45 #include <linux/sort.h> 46 #include <linux/pfn.h> 47 #include <linux/backing-dev.h> 48 #include <linux/fault-inject.h> 49 #include <linux/page-isolation.h> 50 #include <linux/page_ext.h> 51 #include <linux/debugobjects.h> 52 #include <linux/kmemleak.h> 53 #include <linux/compaction.h> 54 #include <trace/events/kmem.h> 55 #include <trace/events/oom.h> 56 #include <linux/prefetch.h> 57 #include <linux/mm_inline.h> 58 #include <linux/migrate.h> 59 #include <linux/hugetlb.h> 60 #include <linux/sched/rt.h> 61 #include <linux/sched/mm.h> 62 #include <linux/page_owner.h> 63 #include <linux/kthread.h> 64 #include <linux/memcontrol.h> 65 #include <linux/ftrace.h> 66 #include <linux/lockdep.h> 67 #include <linux/nmi.h> 68 #include <linux/psi.h> 69 70 #include <asm/sections.h> 71 #include <asm/tlbflush.h> 72 #include <asm/div64.h> 73 #include "internal.h" 74 75 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 76 static DEFINE_MUTEX(pcp_batch_high_lock); 77 #define MIN_PERCPU_PAGELIST_FRACTION (8) 78 79 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 80 DEFINE_PER_CPU(int, numa_node); 81 EXPORT_PER_CPU_SYMBOL(numa_node); 82 #endif 83 84 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); 85 86 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 87 /* 88 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 89 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 90 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 91 * defined in <linux/topology.h>. 92 */ 93 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 94 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 95 int _node_numa_mem_[MAX_NUMNODES]; 96 #endif 97 98 /* work_structs for global per-cpu drains */ 99 DEFINE_MUTEX(pcpu_drain_mutex); 100 DEFINE_PER_CPU(struct work_struct, pcpu_drain); 101 102 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 103 volatile unsigned long latent_entropy __latent_entropy; 104 EXPORT_SYMBOL(latent_entropy); 105 #endif 106 107 /* 108 * Array of node states. 109 */ 110 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 111 [N_POSSIBLE] = NODE_MASK_ALL, 112 [N_ONLINE] = { { [0] = 1UL } }, 113 #ifndef CONFIG_NUMA 114 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 115 #ifdef CONFIG_HIGHMEM 116 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 117 #endif 118 [N_MEMORY] = { { [0] = 1UL } }, 119 [N_CPU] = { { [0] = 1UL } }, 120 #endif /* NUMA */ 121 }; 122 EXPORT_SYMBOL(node_states); 123 124 /* Protect totalram_pages and zone->managed_pages */ 125 static DEFINE_SPINLOCK(managed_page_count_lock); 126 127 unsigned long totalram_pages __read_mostly; 128 unsigned long totalreserve_pages __read_mostly; 129 unsigned long totalcma_pages __read_mostly; 130 131 int percpu_pagelist_fraction; 132 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 133 134 /* 135 * A cached value of the page's pageblock's migratetype, used when the page is 136 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 137 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 138 * Also the migratetype set in the page does not necessarily match the pcplist 139 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 140 * other index - this ensures that it will be put on the correct CMA freelist. 141 */ 142 static inline int get_pcppage_migratetype(struct page *page) 143 { 144 return page->index; 145 } 146 147 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 148 { 149 page->index = migratetype; 150 } 151 152 #ifdef CONFIG_PM_SLEEP 153 /* 154 * The following functions are used by the suspend/hibernate code to temporarily 155 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 156 * while devices are suspended. To avoid races with the suspend/hibernate code, 157 * they should always be called with system_transition_mutex held 158 * (gfp_allowed_mask also should only be modified with system_transition_mutex 159 * held, unless the suspend/hibernate code is guaranteed not to run in parallel 160 * with that modification). 161 */ 162 163 static gfp_t saved_gfp_mask; 164 165 void pm_restore_gfp_mask(void) 166 { 167 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 168 if (saved_gfp_mask) { 169 gfp_allowed_mask = saved_gfp_mask; 170 saved_gfp_mask = 0; 171 } 172 } 173 174 void pm_restrict_gfp_mask(void) 175 { 176 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 177 WARN_ON(saved_gfp_mask); 178 saved_gfp_mask = gfp_allowed_mask; 179 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 180 } 181 182 bool pm_suspended_storage(void) 183 { 184 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 185 return false; 186 return true; 187 } 188 #endif /* CONFIG_PM_SLEEP */ 189 190 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 191 unsigned int pageblock_order __read_mostly; 192 #endif 193 194 static void __free_pages_ok(struct page *page, unsigned int order); 195 196 /* 197 * results with 256, 32 in the lowmem_reserve sysctl: 198 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 199 * 1G machine -> (16M dma, 784M normal, 224M high) 200 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 201 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 202 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 203 * 204 * TBD: should special case ZONE_DMA32 machines here - in those we normally 205 * don't need any ZONE_NORMAL reservation 206 */ 207 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { 208 #ifdef CONFIG_ZONE_DMA 209 [ZONE_DMA] = 256, 210 #endif 211 #ifdef CONFIG_ZONE_DMA32 212 [ZONE_DMA32] = 256, 213 #endif 214 [ZONE_NORMAL] = 32, 215 #ifdef CONFIG_HIGHMEM 216 [ZONE_HIGHMEM] = 0, 217 #endif 218 [ZONE_MOVABLE] = 0, 219 }; 220 221 EXPORT_SYMBOL(totalram_pages); 222 223 static char * const zone_names[MAX_NR_ZONES] = { 224 #ifdef CONFIG_ZONE_DMA 225 "DMA", 226 #endif 227 #ifdef CONFIG_ZONE_DMA32 228 "DMA32", 229 #endif 230 "Normal", 231 #ifdef CONFIG_HIGHMEM 232 "HighMem", 233 #endif 234 "Movable", 235 #ifdef CONFIG_ZONE_DEVICE 236 "Device", 237 #endif 238 }; 239 240 char * const migratetype_names[MIGRATE_TYPES] = { 241 "Unmovable", 242 "Movable", 243 "Reclaimable", 244 "HighAtomic", 245 #ifdef CONFIG_CMA 246 "CMA", 247 #endif 248 #ifdef CONFIG_MEMORY_ISOLATION 249 "Isolate", 250 #endif 251 }; 252 253 compound_page_dtor * const compound_page_dtors[] = { 254 NULL, 255 free_compound_page, 256 #ifdef CONFIG_HUGETLB_PAGE 257 free_huge_page, 258 #endif 259 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 260 free_transhuge_page, 261 #endif 262 }; 263 264 int min_free_kbytes = 1024; 265 int user_min_free_kbytes = -1; 266 int watermark_scale_factor = 10; 267 268 static unsigned long nr_kernel_pages __meminitdata; 269 static unsigned long nr_all_pages __meminitdata; 270 static unsigned long dma_reserve __meminitdata; 271 272 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 273 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __meminitdata; 274 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __meminitdata; 275 static unsigned long required_kernelcore __initdata; 276 static unsigned long required_kernelcore_percent __initdata; 277 static unsigned long required_movablecore __initdata; 278 static unsigned long required_movablecore_percent __initdata; 279 static unsigned long zone_movable_pfn[MAX_NUMNODES] __meminitdata; 280 static bool mirrored_kernelcore __meminitdata; 281 282 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 283 int movable_zone; 284 EXPORT_SYMBOL(movable_zone); 285 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 286 287 #if MAX_NUMNODES > 1 288 int nr_node_ids __read_mostly = MAX_NUMNODES; 289 int nr_online_nodes __read_mostly = 1; 290 EXPORT_SYMBOL(nr_node_ids); 291 EXPORT_SYMBOL(nr_online_nodes); 292 #endif 293 294 int page_group_by_mobility_disabled __read_mostly; 295 296 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 297 /* Returns true if the struct page for the pfn is uninitialised */ 298 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 299 { 300 int nid = early_pfn_to_nid(pfn); 301 302 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 303 return true; 304 305 return false; 306 } 307 308 /* 309 * Returns true when the remaining initialisation should be deferred until 310 * later in the boot cycle when it can be parallelised. 311 */ 312 static bool __meminit 313 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 314 { 315 static unsigned long prev_end_pfn, nr_initialised; 316 317 /* 318 * prev_end_pfn static that contains the end of previous zone 319 * No need to protect because called very early in boot before smp_init. 320 */ 321 if (prev_end_pfn != end_pfn) { 322 prev_end_pfn = end_pfn; 323 nr_initialised = 0; 324 } 325 326 /* Always populate low zones for address-constrained allocations */ 327 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 328 return false; 329 nr_initialised++; 330 if ((nr_initialised > NODE_DATA(nid)->static_init_pgcnt) && 331 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 332 NODE_DATA(nid)->first_deferred_pfn = pfn; 333 return true; 334 } 335 return false; 336 } 337 #else 338 static inline bool early_page_uninitialised(unsigned long pfn) 339 { 340 return false; 341 } 342 343 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 344 { 345 return false; 346 } 347 #endif 348 349 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 350 static inline unsigned long *get_pageblock_bitmap(struct page *page, 351 unsigned long pfn) 352 { 353 #ifdef CONFIG_SPARSEMEM 354 return __pfn_to_section(pfn)->pageblock_flags; 355 #else 356 return page_zone(page)->pageblock_flags; 357 #endif /* CONFIG_SPARSEMEM */ 358 } 359 360 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn) 361 { 362 #ifdef CONFIG_SPARSEMEM 363 pfn &= (PAGES_PER_SECTION-1); 364 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 365 #else 366 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 367 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 368 #endif /* CONFIG_SPARSEMEM */ 369 } 370 371 /** 372 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 373 * @page: The page within the block of interest 374 * @pfn: The target page frame number 375 * @end_bitidx: The last bit of interest to retrieve 376 * @mask: mask of bits that the caller is interested in 377 * 378 * Return: pageblock_bits flags 379 */ 380 static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page, 381 unsigned long pfn, 382 unsigned long end_bitidx, 383 unsigned long mask) 384 { 385 unsigned long *bitmap; 386 unsigned long bitidx, word_bitidx; 387 unsigned long word; 388 389 bitmap = get_pageblock_bitmap(page, pfn); 390 bitidx = pfn_to_bitidx(page, pfn); 391 word_bitidx = bitidx / BITS_PER_LONG; 392 bitidx &= (BITS_PER_LONG-1); 393 394 word = bitmap[word_bitidx]; 395 bitidx += end_bitidx; 396 return (word >> (BITS_PER_LONG - bitidx - 1)) & mask; 397 } 398 399 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, 400 unsigned long end_bitidx, 401 unsigned long mask) 402 { 403 return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask); 404 } 405 406 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn) 407 { 408 return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK); 409 } 410 411 /** 412 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 413 * @page: The page within the block of interest 414 * @flags: The flags to set 415 * @pfn: The target page frame number 416 * @end_bitidx: The last bit of interest 417 * @mask: mask of bits that the caller is interested in 418 */ 419 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 420 unsigned long pfn, 421 unsigned long end_bitidx, 422 unsigned long mask) 423 { 424 unsigned long *bitmap; 425 unsigned long bitidx, word_bitidx; 426 unsigned long old_word, word; 427 428 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 429 430 bitmap = get_pageblock_bitmap(page, pfn); 431 bitidx = pfn_to_bitidx(page, pfn); 432 word_bitidx = bitidx / BITS_PER_LONG; 433 bitidx &= (BITS_PER_LONG-1); 434 435 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 436 437 bitidx += end_bitidx; 438 mask <<= (BITS_PER_LONG - bitidx - 1); 439 flags <<= (BITS_PER_LONG - bitidx - 1); 440 441 word = READ_ONCE(bitmap[word_bitidx]); 442 for (;;) { 443 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 444 if (word == old_word) 445 break; 446 word = old_word; 447 } 448 } 449 450 void set_pageblock_migratetype(struct page *page, int migratetype) 451 { 452 if (unlikely(page_group_by_mobility_disabled && 453 migratetype < MIGRATE_PCPTYPES)) 454 migratetype = MIGRATE_UNMOVABLE; 455 456 set_pageblock_flags_group(page, (unsigned long)migratetype, 457 PB_migrate, PB_migrate_end); 458 } 459 460 #ifdef CONFIG_DEBUG_VM 461 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 462 { 463 int ret = 0; 464 unsigned seq; 465 unsigned long pfn = page_to_pfn(page); 466 unsigned long sp, start_pfn; 467 468 do { 469 seq = zone_span_seqbegin(zone); 470 start_pfn = zone->zone_start_pfn; 471 sp = zone->spanned_pages; 472 if (!zone_spans_pfn(zone, pfn)) 473 ret = 1; 474 } while (zone_span_seqretry(zone, seq)); 475 476 if (ret) 477 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 478 pfn, zone_to_nid(zone), zone->name, 479 start_pfn, start_pfn + sp); 480 481 return ret; 482 } 483 484 static int page_is_consistent(struct zone *zone, struct page *page) 485 { 486 if (!pfn_valid_within(page_to_pfn(page))) 487 return 0; 488 if (zone != page_zone(page)) 489 return 0; 490 491 return 1; 492 } 493 /* 494 * Temporary debugging check for pages not lying within a given zone. 495 */ 496 static int __maybe_unused bad_range(struct zone *zone, struct page *page) 497 { 498 if (page_outside_zone_boundaries(zone, page)) 499 return 1; 500 if (!page_is_consistent(zone, page)) 501 return 1; 502 503 return 0; 504 } 505 #else 506 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page) 507 { 508 return 0; 509 } 510 #endif 511 512 static void bad_page(struct page *page, const char *reason, 513 unsigned long bad_flags) 514 { 515 static unsigned long resume; 516 static unsigned long nr_shown; 517 static unsigned long nr_unshown; 518 519 /* 520 * Allow a burst of 60 reports, then keep quiet for that minute; 521 * or allow a steady drip of one report per second. 522 */ 523 if (nr_shown == 60) { 524 if (time_before(jiffies, resume)) { 525 nr_unshown++; 526 goto out; 527 } 528 if (nr_unshown) { 529 pr_alert( 530 "BUG: Bad page state: %lu messages suppressed\n", 531 nr_unshown); 532 nr_unshown = 0; 533 } 534 nr_shown = 0; 535 } 536 if (nr_shown++ == 0) 537 resume = jiffies + 60 * HZ; 538 539 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 540 current->comm, page_to_pfn(page)); 541 __dump_page(page, reason); 542 bad_flags &= page->flags; 543 if (bad_flags) 544 pr_alert("bad because of flags: %#lx(%pGp)\n", 545 bad_flags, &bad_flags); 546 dump_page_owner(page); 547 548 print_modules(); 549 dump_stack(); 550 out: 551 /* Leave bad fields for debug, except PageBuddy could make trouble */ 552 page_mapcount_reset(page); /* remove PageBuddy */ 553 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 554 } 555 556 /* 557 * Higher-order pages are called "compound pages". They are structured thusly: 558 * 559 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 560 * 561 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 562 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 563 * 564 * The first tail page's ->compound_dtor holds the offset in array of compound 565 * page destructors. See compound_page_dtors. 566 * 567 * The first tail page's ->compound_order holds the order of allocation. 568 * This usage means that zero-order pages may not be compound. 569 */ 570 571 void free_compound_page(struct page *page) 572 { 573 __free_pages_ok(page, compound_order(page)); 574 } 575 576 void prep_compound_page(struct page *page, unsigned int order) 577 { 578 int i; 579 int nr_pages = 1 << order; 580 581 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 582 set_compound_order(page, order); 583 __SetPageHead(page); 584 for (i = 1; i < nr_pages; i++) { 585 struct page *p = page + i; 586 set_page_count(p, 0); 587 p->mapping = TAIL_MAPPING; 588 set_compound_head(p, page); 589 } 590 atomic_set(compound_mapcount_ptr(page), -1); 591 } 592 593 #ifdef CONFIG_DEBUG_PAGEALLOC 594 unsigned int _debug_guardpage_minorder; 595 bool _debug_pagealloc_enabled __read_mostly 596 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 597 EXPORT_SYMBOL(_debug_pagealloc_enabled); 598 bool _debug_guardpage_enabled __read_mostly; 599 600 static int __init early_debug_pagealloc(char *buf) 601 { 602 if (!buf) 603 return -EINVAL; 604 return kstrtobool(buf, &_debug_pagealloc_enabled); 605 } 606 early_param("debug_pagealloc", early_debug_pagealloc); 607 608 static bool need_debug_guardpage(void) 609 { 610 /* If we don't use debug_pagealloc, we don't need guard page */ 611 if (!debug_pagealloc_enabled()) 612 return false; 613 614 if (!debug_guardpage_minorder()) 615 return false; 616 617 return true; 618 } 619 620 static void init_debug_guardpage(void) 621 { 622 if (!debug_pagealloc_enabled()) 623 return; 624 625 if (!debug_guardpage_minorder()) 626 return; 627 628 _debug_guardpage_enabled = true; 629 } 630 631 struct page_ext_operations debug_guardpage_ops = { 632 .need = need_debug_guardpage, 633 .init = init_debug_guardpage, 634 }; 635 636 static int __init debug_guardpage_minorder_setup(char *buf) 637 { 638 unsigned long res; 639 640 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 641 pr_err("Bad debug_guardpage_minorder value\n"); 642 return 0; 643 } 644 _debug_guardpage_minorder = res; 645 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 646 return 0; 647 } 648 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); 649 650 static inline bool set_page_guard(struct zone *zone, struct page *page, 651 unsigned int order, int migratetype) 652 { 653 struct page_ext *page_ext; 654 655 if (!debug_guardpage_enabled()) 656 return false; 657 658 if (order >= debug_guardpage_minorder()) 659 return false; 660 661 page_ext = lookup_page_ext(page); 662 if (unlikely(!page_ext)) 663 return false; 664 665 __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); 666 667 INIT_LIST_HEAD(&page->lru); 668 set_page_private(page, order); 669 /* Guard pages are not available for any usage */ 670 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 671 672 return true; 673 } 674 675 static inline void clear_page_guard(struct zone *zone, struct page *page, 676 unsigned int order, int migratetype) 677 { 678 struct page_ext *page_ext; 679 680 if (!debug_guardpage_enabled()) 681 return; 682 683 page_ext = lookup_page_ext(page); 684 if (unlikely(!page_ext)) 685 return; 686 687 __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); 688 689 set_page_private(page, 0); 690 if (!is_migrate_isolate(migratetype)) 691 __mod_zone_freepage_state(zone, (1 << order), migratetype); 692 } 693 #else 694 struct page_ext_operations debug_guardpage_ops; 695 static inline bool set_page_guard(struct zone *zone, struct page *page, 696 unsigned int order, int migratetype) { return false; } 697 static inline void clear_page_guard(struct zone *zone, struct page *page, 698 unsigned int order, int migratetype) {} 699 #endif 700 701 static inline void set_page_order(struct page *page, unsigned int order) 702 { 703 set_page_private(page, order); 704 __SetPageBuddy(page); 705 } 706 707 static inline void rmv_page_order(struct page *page) 708 { 709 __ClearPageBuddy(page); 710 set_page_private(page, 0); 711 } 712 713 /* 714 * This function checks whether a page is free && is the buddy 715 * we can coalesce a page and its buddy if 716 * (a) the buddy is not in a hole (check before calling!) && 717 * (b) the buddy is in the buddy system && 718 * (c) a page and its buddy have the same order && 719 * (d) a page and its buddy are in the same zone. 720 * 721 * For recording whether a page is in the buddy system, we set PageBuddy. 722 * Setting, clearing, and testing PageBuddy is serialized by zone->lock. 723 * 724 * For recording page's order, we use page_private(page). 725 */ 726 static inline int page_is_buddy(struct page *page, struct page *buddy, 727 unsigned int order) 728 { 729 if (page_is_guard(buddy) && page_order(buddy) == order) { 730 if (page_zone_id(page) != page_zone_id(buddy)) 731 return 0; 732 733 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 734 735 return 1; 736 } 737 738 if (PageBuddy(buddy) && page_order(buddy) == order) { 739 /* 740 * zone check is done late to avoid uselessly 741 * calculating zone/node ids for pages that could 742 * never merge. 743 */ 744 if (page_zone_id(page) != page_zone_id(buddy)) 745 return 0; 746 747 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 748 749 return 1; 750 } 751 return 0; 752 } 753 754 /* 755 * Freeing function for a buddy system allocator. 756 * 757 * The concept of a buddy system is to maintain direct-mapped table 758 * (containing bit values) for memory blocks of various "orders". 759 * The bottom level table contains the map for the smallest allocatable 760 * units of memory (here, pages), and each level above it describes 761 * pairs of units from the levels below, hence, "buddies". 762 * At a high level, all that happens here is marking the table entry 763 * at the bottom level available, and propagating the changes upward 764 * as necessary, plus some accounting needed to play nicely with other 765 * parts of the VM system. 766 * At each level, we keep a list of pages, which are heads of continuous 767 * free pages of length of (1 << order) and marked with PageBuddy. 768 * Page's order is recorded in page_private(page) field. 769 * So when we are allocating or freeing one, we can derive the state of the 770 * other. That is, if we allocate a small block, and both were 771 * free, the remainder of the region must be split into blocks. 772 * If a block is freed, and its buddy is also free, then this 773 * triggers coalescing into a block of larger size. 774 * 775 * -- nyc 776 */ 777 778 static inline void __free_one_page(struct page *page, 779 unsigned long pfn, 780 struct zone *zone, unsigned int order, 781 int migratetype) 782 { 783 unsigned long combined_pfn; 784 unsigned long uninitialized_var(buddy_pfn); 785 struct page *buddy; 786 unsigned int max_order; 787 788 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1); 789 790 VM_BUG_ON(!zone_is_initialized(zone)); 791 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 792 793 VM_BUG_ON(migratetype == -1); 794 if (likely(!is_migrate_isolate(migratetype))) 795 __mod_zone_freepage_state(zone, 1 << order, migratetype); 796 797 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); 798 VM_BUG_ON_PAGE(bad_range(zone, page), page); 799 800 continue_merging: 801 while (order < max_order - 1) { 802 buddy_pfn = __find_buddy_pfn(pfn, order); 803 buddy = page + (buddy_pfn - pfn); 804 805 if (!pfn_valid_within(buddy_pfn)) 806 goto done_merging; 807 if (!page_is_buddy(page, buddy, order)) 808 goto done_merging; 809 /* 810 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 811 * merge with it and move up one order. 812 */ 813 if (page_is_guard(buddy)) { 814 clear_page_guard(zone, buddy, order, migratetype); 815 } else { 816 list_del(&buddy->lru); 817 zone->free_area[order].nr_free--; 818 rmv_page_order(buddy); 819 } 820 combined_pfn = buddy_pfn & pfn; 821 page = page + (combined_pfn - pfn); 822 pfn = combined_pfn; 823 order++; 824 } 825 if (max_order < MAX_ORDER) { 826 /* If we are here, it means order is >= pageblock_order. 827 * We want to prevent merge between freepages on isolate 828 * pageblock and normal pageblock. Without this, pageblock 829 * isolation could cause incorrect freepage or CMA accounting. 830 * 831 * We don't want to hit this code for the more frequent 832 * low-order merging. 833 */ 834 if (unlikely(has_isolate_pageblock(zone))) { 835 int buddy_mt; 836 837 buddy_pfn = __find_buddy_pfn(pfn, order); 838 buddy = page + (buddy_pfn - pfn); 839 buddy_mt = get_pageblock_migratetype(buddy); 840 841 if (migratetype != buddy_mt 842 && (is_migrate_isolate(migratetype) || 843 is_migrate_isolate(buddy_mt))) 844 goto done_merging; 845 } 846 max_order++; 847 goto continue_merging; 848 } 849 850 done_merging: 851 set_page_order(page, order); 852 853 /* 854 * If this is not the largest possible page, check if the buddy 855 * of the next-highest order is free. If it is, it's possible 856 * that pages are being freed that will coalesce soon. In case, 857 * that is happening, add the free page to the tail of the list 858 * so it's less likely to be used soon and more likely to be merged 859 * as a higher order page 860 */ 861 if ((order < MAX_ORDER-2) && pfn_valid_within(buddy_pfn)) { 862 struct page *higher_page, *higher_buddy; 863 combined_pfn = buddy_pfn & pfn; 864 higher_page = page + (combined_pfn - pfn); 865 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1); 866 higher_buddy = higher_page + (buddy_pfn - combined_pfn); 867 if (pfn_valid_within(buddy_pfn) && 868 page_is_buddy(higher_page, higher_buddy, order + 1)) { 869 list_add_tail(&page->lru, 870 &zone->free_area[order].free_list[migratetype]); 871 goto out; 872 } 873 } 874 875 list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); 876 out: 877 zone->free_area[order].nr_free++; 878 } 879 880 /* 881 * A bad page could be due to a number of fields. Instead of multiple branches, 882 * try and check multiple fields with one check. The caller must do a detailed 883 * check if necessary. 884 */ 885 static inline bool page_expected_state(struct page *page, 886 unsigned long check_flags) 887 { 888 if (unlikely(atomic_read(&page->_mapcount) != -1)) 889 return false; 890 891 if (unlikely((unsigned long)page->mapping | 892 page_ref_count(page) | 893 #ifdef CONFIG_MEMCG 894 (unsigned long)page->mem_cgroup | 895 #endif 896 (page->flags & check_flags))) 897 return false; 898 899 return true; 900 } 901 902 static void free_pages_check_bad(struct page *page) 903 { 904 const char *bad_reason; 905 unsigned long bad_flags; 906 907 bad_reason = NULL; 908 bad_flags = 0; 909 910 if (unlikely(atomic_read(&page->_mapcount) != -1)) 911 bad_reason = "nonzero mapcount"; 912 if (unlikely(page->mapping != NULL)) 913 bad_reason = "non-NULL mapping"; 914 if (unlikely(page_ref_count(page) != 0)) 915 bad_reason = "nonzero _refcount"; 916 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) { 917 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 918 bad_flags = PAGE_FLAGS_CHECK_AT_FREE; 919 } 920 #ifdef CONFIG_MEMCG 921 if (unlikely(page->mem_cgroup)) 922 bad_reason = "page still charged to cgroup"; 923 #endif 924 bad_page(page, bad_reason, bad_flags); 925 } 926 927 static inline int free_pages_check(struct page *page) 928 { 929 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 930 return 0; 931 932 /* Something has gone sideways, find it */ 933 free_pages_check_bad(page); 934 return 1; 935 } 936 937 static int free_tail_pages_check(struct page *head_page, struct page *page) 938 { 939 int ret = 1; 940 941 /* 942 * We rely page->lru.next never has bit 0 set, unless the page 943 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 944 */ 945 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 946 947 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 948 ret = 0; 949 goto out; 950 } 951 switch (page - head_page) { 952 case 1: 953 /* the first tail page: ->mapping may be compound_mapcount() */ 954 if (unlikely(compound_mapcount(page))) { 955 bad_page(page, "nonzero compound_mapcount", 0); 956 goto out; 957 } 958 break; 959 case 2: 960 /* 961 * the second tail page: ->mapping is 962 * deferred_list.next -- ignore value. 963 */ 964 break; 965 default: 966 if (page->mapping != TAIL_MAPPING) { 967 bad_page(page, "corrupted mapping in tail page", 0); 968 goto out; 969 } 970 break; 971 } 972 if (unlikely(!PageTail(page))) { 973 bad_page(page, "PageTail not set", 0); 974 goto out; 975 } 976 if (unlikely(compound_head(page) != head_page)) { 977 bad_page(page, "compound_head not consistent", 0); 978 goto out; 979 } 980 ret = 0; 981 out: 982 page->mapping = NULL; 983 clear_compound_head(page); 984 return ret; 985 } 986 987 static __always_inline bool free_pages_prepare(struct page *page, 988 unsigned int order, bool check_free) 989 { 990 int bad = 0; 991 992 VM_BUG_ON_PAGE(PageTail(page), page); 993 994 trace_mm_page_free(page, order); 995 996 /* 997 * Check tail pages before head page information is cleared to 998 * avoid checking PageCompound for order-0 pages. 999 */ 1000 if (unlikely(order)) { 1001 bool compound = PageCompound(page); 1002 int i; 1003 1004 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1005 1006 if (compound) 1007 ClearPageDoubleMap(page); 1008 for (i = 1; i < (1 << order); i++) { 1009 if (compound) 1010 bad += free_tail_pages_check(page, page + i); 1011 if (unlikely(free_pages_check(page + i))) { 1012 bad++; 1013 continue; 1014 } 1015 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1016 } 1017 } 1018 if (PageMappingFlags(page)) 1019 page->mapping = NULL; 1020 if (memcg_kmem_enabled() && PageKmemcg(page)) 1021 memcg_kmem_uncharge(page, order); 1022 if (check_free) 1023 bad += free_pages_check(page); 1024 if (bad) 1025 return false; 1026 1027 page_cpupid_reset_last(page); 1028 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1029 reset_page_owner(page, order); 1030 1031 if (!PageHighMem(page)) { 1032 debug_check_no_locks_freed(page_address(page), 1033 PAGE_SIZE << order); 1034 debug_check_no_obj_freed(page_address(page), 1035 PAGE_SIZE << order); 1036 } 1037 arch_free_page(page, order); 1038 kernel_poison_pages(page, 1 << order, 0); 1039 kernel_map_pages(page, 1 << order, 0); 1040 kasan_free_pages(page, order); 1041 1042 return true; 1043 } 1044 1045 #ifdef CONFIG_DEBUG_VM 1046 static inline bool free_pcp_prepare(struct page *page) 1047 { 1048 return free_pages_prepare(page, 0, true); 1049 } 1050 1051 static inline bool bulkfree_pcp_prepare(struct page *page) 1052 { 1053 return false; 1054 } 1055 #else 1056 static bool free_pcp_prepare(struct page *page) 1057 { 1058 return free_pages_prepare(page, 0, false); 1059 } 1060 1061 static bool bulkfree_pcp_prepare(struct page *page) 1062 { 1063 return free_pages_check(page); 1064 } 1065 #endif /* CONFIG_DEBUG_VM */ 1066 1067 static inline void prefetch_buddy(struct page *page) 1068 { 1069 unsigned long pfn = page_to_pfn(page); 1070 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0); 1071 struct page *buddy = page + (buddy_pfn - pfn); 1072 1073 prefetch(buddy); 1074 } 1075 1076 /* 1077 * Frees a number of pages from the PCP lists 1078 * Assumes all pages on list are in same zone, and of same order. 1079 * count is the number of pages to free. 1080 * 1081 * If the zone was previously in an "all pages pinned" state then look to 1082 * see if this freeing clears that state. 1083 * 1084 * And clear the zone's pages_scanned counter, to hold off the "all pages are 1085 * pinned" detection logic. 1086 */ 1087 static void free_pcppages_bulk(struct zone *zone, int count, 1088 struct per_cpu_pages *pcp) 1089 { 1090 int migratetype = 0; 1091 int batch_free = 0; 1092 int prefetch_nr = 0; 1093 bool isolated_pageblocks; 1094 struct page *page, *tmp; 1095 LIST_HEAD(head); 1096 1097 while (count) { 1098 struct list_head *list; 1099 1100 /* 1101 * Remove pages from lists in a round-robin fashion. A 1102 * batch_free count is maintained that is incremented when an 1103 * empty list is encountered. This is so more pages are freed 1104 * off fuller lists instead of spinning excessively around empty 1105 * lists 1106 */ 1107 do { 1108 batch_free++; 1109 if (++migratetype == MIGRATE_PCPTYPES) 1110 migratetype = 0; 1111 list = &pcp->lists[migratetype]; 1112 } while (list_empty(list)); 1113 1114 /* This is the only non-empty list. Free them all. */ 1115 if (batch_free == MIGRATE_PCPTYPES) 1116 batch_free = count; 1117 1118 do { 1119 page = list_last_entry(list, struct page, lru); 1120 /* must delete to avoid corrupting pcp list */ 1121 list_del(&page->lru); 1122 pcp->count--; 1123 1124 if (bulkfree_pcp_prepare(page)) 1125 continue; 1126 1127 list_add_tail(&page->lru, &head); 1128 1129 /* 1130 * We are going to put the page back to the global 1131 * pool, prefetch its buddy to speed up later access 1132 * under zone->lock. It is believed the overhead of 1133 * an additional test and calculating buddy_pfn here 1134 * can be offset by reduced memory latency later. To 1135 * avoid excessive prefetching due to large count, only 1136 * prefetch buddy for the first pcp->batch nr of pages. 1137 */ 1138 if (prefetch_nr++ < pcp->batch) 1139 prefetch_buddy(page); 1140 } while (--count && --batch_free && !list_empty(list)); 1141 } 1142 1143 spin_lock(&zone->lock); 1144 isolated_pageblocks = has_isolate_pageblock(zone); 1145 1146 /* 1147 * Use safe version since after __free_one_page(), 1148 * page->lru.next will not point to original list. 1149 */ 1150 list_for_each_entry_safe(page, tmp, &head, lru) { 1151 int mt = get_pcppage_migratetype(page); 1152 /* MIGRATE_ISOLATE page should not go to pcplists */ 1153 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1154 /* Pageblock could have been isolated meanwhile */ 1155 if (unlikely(isolated_pageblocks)) 1156 mt = get_pageblock_migratetype(page); 1157 1158 __free_one_page(page, page_to_pfn(page), zone, 0, mt); 1159 trace_mm_page_pcpu_drain(page, 0, mt); 1160 } 1161 spin_unlock(&zone->lock); 1162 } 1163 1164 static void free_one_page(struct zone *zone, 1165 struct page *page, unsigned long pfn, 1166 unsigned int order, 1167 int migratetype) 1168 { 1169 spin_lock(&zone->lock); 1170 if (unlikely(has_isolate_pageblock(zone) || 1171 is_migrate_isolate(migratetype))) { 1172 migratetype = get_pfnblock_migratetype(page, pfn); 1173 } 1174 __free_one_page(page, pfn, zone, order, migratetype); 1175 spin_unlock(&zone->lock); 1176 } 1177 1178 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1179 unsigned long zone, int nid) 1180 { 1181 mm_zero_struct_page(page); 1182 set_page_links(page, zone, nid, pfn); 1183 init_page_count(page); 1184 page_mapcount_reset(page); 1185 page_cpupid_reset_last(page); 1186 page_kasan_tag_reset(page); 1187 1188 INIT_LIST_HEAD(&page->lru); 1189 #ifdef WANT_PAGE_VIRTUAL 1190 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1191 if (!is_highmem_idx(zone)) 1192 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1193 #endif 1194 } 1195 1196 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1197 static void __meminit init_reserved_page(unsigned long pfn) 1198 { 1199 pg_data_t *pgdat; 1200 int nid, zid; 1201 1202 if (!early_page_uninitialised(pfn)) 1203 return; 1204 1205 nid = early_pfn_to_nid(pfn); 1206 pgdat = NODE_DATA(nid); 1207 1208 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1209 struct zone *zone = &pgdat->node_zones[zid]; 1210 1211 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) 1212 break; 1213 } 1214 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 1215 } 1216 #else 1217 static inline void init_reserved_page(unsigned long pfn) 1218 { 1219 } 1220 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1221 1222 /* 1223 * Initialised pages do not have PageReserved set. This function is 1224 * called for each range allocated by the bootmem allocator and 1225 * marks the pages PageReserved. The remaining valid pages are later 1226 * sent to the buddy page allocator. 1227 */ 1228 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1229 { 1230 unsigned long start_pfn = PFN_DOWN(start); 1231 unsigned long end_pfn = PFN_UP(end); 1232 1233 for (; start_pfn < end_pfn; start_pfn++) { 1234 if (pfn_valid(start_pfn)) { 1235 struct page *page = pfn_to_page(start_pfn); 1236 1237 init_reserved_page(start_pfn); 1238 1239 /* Avoid false-positive PageTail() */ 1240 INIT_LIST_HEAD(&page->lru); 1241 1242 /* 1243 * no need for atomic set_bit because the struct 1244 * page is not visible yet so nobody should 1245 * access it yet. 1246 */ 1247 __SetPageReserved(page); 1248 } 1249 } 1250 } 1251 1252 static void __free_pages_ok(struct page *page, unsigned int order) 1253 { 1254 unsigned long flags; 1255 int migratetype; 1256 unsigned long pfn = page_to_pfn(page); 1257 1258 if (!free_pages_prepare(page, order, true)) 1259 return; 1260 1261 migratetype = get_pfnblock_migratetype(page, pfn); 1262 local_irq_save(flags); 1263 __count_vm_events(PGFREE, 1 << order); 1264 free_one_page(page_zone(page), page, pfn, order, migratetype); 1265 local_irq_restore(flags); 1266 } 1267 1268 static void __init __free_pages_boot_core(struct page *page, unsigned int order) 1269 { 1270 unsigned int nr_pages = 1 << order; 1271 struct page *p = page; 1272 unsigned int loop; 1273 1274 prefetchw(p); 1275 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1276 prefetchw(p + 1); 1277 __ClearPageReserved(p); 1278 set_page_count(p, 0); 1279 } 1280 __ClearPageReserved(p); 1281 set_page_count(p, 0); 1282 1283 page_zone(page)->managed_pages += nr_pages; 1284 set_page_refcounted(page); 1285 __free_pages(page, order); 1286 } 1287 1288 #if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \ 1289 defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) 1290 1291 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1292 1293 int __meminit early_pfn_to_nid(unsigned long pfn) 1294 { 1295 static DEFINE_SPINLOCK(early_pfn_lock); 1296 int nid; 1297 1298 spin_lock(&early_pfn_lock); 1299 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1300 if (nid < 0) 1301 nid = first_online_node; 1302 spin_unlock(&early_pfn_lock); 1303 1304 return nid; 1305 } 1306 #endif 1307 1308 #ifdef CONFIG_NODES_SPAN_OTHER_NODES 1309 static inline bool __meminit __maybe_unused 1310 meminit_pfn_in_nid(unsigned long pfn, int node, 1311 struct mminit_pfnnid_cache *state) 1312 { 1313 int nid; 1314 1315 nid = __early_pfn_to_nid(pfn, state); 1316 if (nid >= 0 && nid != node) 1317 return false; 1318 return true; 1319 } 1320 1321 /* Only safe to use early in boot when initialisation is single-threaded */ 1322 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1323 { 1324 return meminit_pfn_in_nid(pfn, node, &early_pfnnid_cache); 1325 } 1326 1327 #else 1328 1329 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1330 { 1331 return true; 1332 } 1333 static inline bool __meminit __maybe_unused 1334 meminit_pfn_in_nid(unsigned long pfn, int node, 1335 struct mminit_pfnnid_cache *state) 1336 { 1337 return true; 1338 } 1339 #endif 1340 1341 1342 void __init memblock_free_pages(struct page *page, unsigned long pfn, 1343 unsigned int order) 1344 { 1345 if (early_page_uninitialised(pfn)) 1346 return; 1347 return __free_pages_boot_core(page, order); 1348 } 1349 1350 /* 1351 * Check that the whole (or subset of) a pageblock given by the interval of 1352 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1353 * with the migration of free compaction scanner. The scanners then need to 1354 * use only pfn_valid_within() check for arches that allow holes within 1355 * pageblocks. 1356 * 1357 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1358 * 1359 * It's possible on some configurations to have a setup like node0 node1 node0 1360 * i.e. it's possible that all pages within a zones range of pages do not 1361 * belong to a single zone. We assume that a border between node0 and node1 1362 * can occur within a single pageblock, but not a node0 node1 node0 1363 * interleaving within a single pageblock. It is therefore sufficient to check 1364 * the first and last page of a pageblock and avoid checking each individual 1365 * page in a pageblock. 1366 */ 1367 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1368 unsigned long end_pfn, struct zone *zone) 1369 { 1370 struct page *start_page; 1371 struct page *end_page; 1372 1373 /* end_pfn is one past the range we are checking */ 1374 end_pfn--; 1375 1376 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1377 return NULL; 1378 1379 start_page = pfn_to_online_page(start_pfn); 1380 if (!start_page) 1381 return NULL; 1382 1383 if (page_zone(start_page) != zone) 1384 return NULL; 1385 1386 end_page = pfn_to_page(end_pfn); 1387 1388 /* This gives a shorter code than deriving page_zone(end_page) */ 1389 if (page_zone_id(start_page) != page_zone_id(end_page)) 1390 return NULL; 1391 1392 return start_page; 1393 } 1394 1395 void set_zone_contiguous(struct zone *zone) 1396 { 1397 unsigned long block_start_pfn = zone->zone_start_pfn; 1398 unsigned long block_end_pfn; 1399 1400 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1401 for (; block_start_pfn < zone_end_pfn(zone); 1402 block_start_pfn = block_end_pfn, 1403 block_end_pfn += pageblock_nr_pages) { 1404 1405 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1406 1407 if (!__pageblock_pfn_to_page(block_start_pfn, 1408 block_end_pfn, zone)) 1409 return; 1410 } 1411 1412 /* We confirm that there is no hole */ 1413 zone->contiguous = true; 1414 } 1415 1416 void clear_zone_contiguous(struct zone *zone) 1417 { 1418 zone->contiguous = false; 1419 } 1420 1421 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1422 static void __init deferred_free_range(unsigned long pfn, 1423 unsigned long nr_pages) 1424 { 1425 struct page *page; 1426 unsigned long i; 1427 1428 if (!nr_pages) 1429 return; 1430 1431 page = pfn_to_page(pfn); 1432 1433 /* Free a large naturally-aligned chunk if possible */ 1434 if (nr_pages == pageblock_nr_pages && 1435 (pfn & (pageblock_nr_pages - 1)) == 0) { 1436 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1437 __free_pages_boot_core(page, pageblock_order); 1438 return; 1439 } 1440 1441 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1442 if ((pfn & (pageblock_nr_pages - 1)) == 0) 1443 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1444 __free_pages_boot_core(page, 0); 1445 } 1446 } 1447 1448 /* Completion tracking for deferred_init_memmap() threads */ 1449 static atomic_t pgdat_init_n_undone __initdata; 1450 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1451 1452 static inline void __init pgdat_init_report_one_done(void) 1453 { 1454 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1455 complete(&pgdat_init_all_done_comp); 1456 } 1457 1458 /* 1459 * Returns true if page needs to be initialized or freed to buddy allocator. 1460 * 1461 * First we check if pfn is valid on architectures where it is possible to have 1462 * holes within pageblock_nr_pages. On systems where it is not possible, this 1463 * function is optimized out. 1464 * 1465 * Then, we check if a current large page is valid by only checking the validity 1466 * of the head pfn. 1467 * 1468 * Finally, meminit_pfn_in_nid is checked on systems where pfns can interleave 1469 * within a node: a pfn is between start and end of a node, but does not belong 1470 * to this memory node. 1471 */ 1472 static inline bool __init 1473 deferred_pfn_valid(int nid, unsigned long pfn, 1474 struct mminit_pfnnid_cache *nid_init_state) 1475 { 1476 if (!pfn_valid_within(pfn)) 1477 return false; 1478 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn)) 1479 return false; 1480 if (!meminit_pfn_in_nid(pfn, nid, nid_init_state)) 1481 return false; 1482 return true; 1483 } 1484 1485 /* 1486 * Free pages to buddy allocator. Try to free aligned pages in 1487 * pageblock_nr_pages sizes. 1488 */ 1489 static void __init deferred_free_pages(int nid, int zid, unsigned long pfn, 1490 unsigned long end_pfn) 1491 { 1492 struct mminit_pfnnid_cache nid_init_state = { }; 1493 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1494 unsigned long nr_free = 0; 1495 1496 for (; pfn < end_pfn; pfn++) { 1497 if (!deferred_pfn_valid(nid, pfn, &nid_init_state)) { 1498 deferred_free_range(pfn - nr_free, nr_free); 1499 nr_free = 0; 1500 } else if (!(pfn & nr_pgmask)) { 1501 deferred_free_range(pfn - nr_free, nr_free); 1502 nr_free = 1; 1503 touch_nmi_watchdog(); 1504 } else { 1505 nr_free++; 1506 } 1507 } 1508 /* Free the last block of pages to allocator */ 1509 deferred_free_range(pfn - nr_free, nr_free); 1510 } 1511 1512 /* 1513 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 1514 * by performing it only once every pageblock_nr_pages. 1515 * Return number of pages initialized. 1516 */ 1517 static unsigned long __init deferred_init_pages(int nid, int zid, 1518 unsigned long pfn, 1519 unsigned long end_pfn) 1520 { 1521 struct mminit_pfnnid_cache nid_init_state = { }; 1522 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1523 unsigned long nr_pages = 0; 1524 struct page *page = NULL; 1525 1526 for (; pfn < end_pfn; pfn++) { 1527 if (!deferred_pfn_valid(nid, pfn, &nid_init_state)) { 1528 page = NULL; 1529 continue; 1530 } else if (!page || !(pfn & nr_pgmask)) { 1531 page = pfn_to_page(pfn); 1532 touch_nmi_watchdog(); 1533 } else { 1534 page++; 1535 } 1536 __init_single_page(page, pfn, zid, nid); 1537 nr_pages++; 1538 } 1539 return (nr_pages); 1540 } 1541 1542 /* Initialise remaining memory on a node */ 1543 static int __init deferred_init_memmap(void *data) 1544 { 1545 pg_data_t *pgdat = data; 1546 int nid = pgdat->node_id; 1547 unsigned long start = jiffies; 1548 unsigned long nr_pages = 0; 1549 unsigned long spfn, epfn, first_init_pfn, flags; 1550 phys_addr_t spa, epa; 1551 int zid; 1552 struct zone *zone; 1553 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 1554 u64 i; 1555 1556 /* Bind memory initialisation thread to a local node if possible */ 1557 if (!cpumask_empty(cpumask)) 1558 set_cpus_allowed_ptr(current, cpumask); 1559 1560 pgdat_resize_lock(pgdat, &flags); 1561 first_init_pfn = pgdat->first_deferred_pfn; 1562 if (first_init_pfn == ULONG_MAX) { 1563 pgdat_resize_unlock(pgdat, &flags); 1564 pgdat_init_report_one_done(); 1565 return 0; 1566 } 1567 1568 /* Sanity check boundaries */ 1569 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 1570 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 1571 pgdat->first_deferred_pfn = ULONG_MAX; 1572 1573 /* Only the highest zone is deferred so find it */ 1574 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1575 zone = pgdat->node_zones + zid; 1576 if (first_init_pfn < zone_end_pfn(zone)) 1577 break; 1578 } 1579 first_init_pfn = max(zone->zone_start_pfn, first_init_pfn); 1580 1581 /* 1582 * Initialize and free pages. We do it in two loops: first we initialize 1583 * struct page, than free to buddy allocator, because while we are 1584 * freeing pages we can access pages that are ahead (computing buddy 1585 * page in __free_one_page()). 1586 */ 1587 for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) { 1588 spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa)); 1589 epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa)); 1590 nr_pages += deferred_init_pages(nid, zid, spfn, epfn); 1591 } 1592 for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) { 1593 spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa)); 1594 epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa)); 1595 deferred_free_pages(nid, zid, spfn, epfn); 1596 } 1597 pgdat_resize_unlock(pgdat, &flags); 1598 1599 /* Sanity check that the next zone really is unpopulated */ 1600 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 1601 1602 pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages, 1603 jiffies_to_msecs(jiffies - start)); 1604 1605 pgdat_init_report_one_done(); 1606 return 0; 1607 } 1608 1609 /* 1610 * During boot we initialize deferred pages on-demand, as needed, but once 1611 * page_alloc_init_late() has finished, the deferred pages are all initialized, 1612 * and we can permanently disable that path. 1613 */ 1614 static DEFINE_STATIC_KEY_TRUE(deferred_pages); 1615 1616 /* 1617 * If this zone has deferred pages, try to grow it by initializing enough 1618 * deferred pages to satisfy the allocation specified by order, rounded up to 1619 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 1620 * of SECTION_SIZE bytes by initializing struct pages in increments of 1621 * PAGES_PER_SECTION * sizeof(struct page) bytes. 1622 * 1623 * Return true when zone was grown, otherwise return false. We return true even 1624 * when we grow less than requested, to let the caller decide if there are 1625 * enough pages to satisfy the allocation. 1626 * 1627 * Note: We use noinline because this function is needed only during boot, and 1628 * it is called from a __ref function _deferred_grow_zone. This way we are 1629 * making sure that it is not inlined into permanent text section. 1630 */ 1631 static noinline bool __init 1632 deferred_grow_zone(struct zone *zone, unsigned int order) 1633 { 1634 int zid = zone_idx(zone); 1635 int nid = zone_to_nid(zone); 1636 pg_data_t *pgdat = NODE_DATA(nid); 1637 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 1638 unsigned long nr_pages = 0; 1639 unsigned long first_init_pfn, spfn, epfn, t, flags; 1640 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 1641 phys_addr_t spa, epa; 1642 u64 i; 1643 1644 /* Only the last zone may have deferred pages */ 1645 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 1646 return false; 1647 1648 pgdat_resize_lock(pgdat, &flags); 1649 1650 /* 1651 * If deferred pages have been initialized while we were waiting for 1652 * the lock, return true, as the zone was grown. The caller will retry 1653 * this zone. We won't return to this function since the caller also 1654 * has this static branch. 1655 */ 1656 if (!static_branch_unlikely(&deferred_pages)) { 1657 pgdat_resize_unlock(pgdat, &flags); 1658 return true; 1659 } 1660 1661 /* 1662 * If someone grew this zone while we were waiting for spinlock, return 1663 * true, as there might be enough pages already. 1664 */ 1665 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 1666 pgdat_resize_unlock(pgdat, &flags); 1667 return true; 1668 } 1669 1670 first_init_pfn = max(zone->zone_start_pfn, first_deferred_pfn); 1671 1672 if (first_init_pfn >= pgdat_end_pfn(pgdat)) { 1673 pgdat_resize_unlock(pgdat, &flags); 1674 return false; 1675 } 1676 1677 for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) { 1678 spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa)); 1679 epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa)); 1680 1681 while (spfn < epfn && nr_pages < nr_pages_needed) { 1682 t = ALIGN(spfn + PAGES_PER_SECTION, PAGES_PER_SECTION); 1683 first_deferred_pfn = min(t, epfn); 1684 nr_pages += deferred_init_pages(nid, zid, spfn, 1685 first_deferred_pfn); 1686 spfn = first_deferred_pfn; 1687 } 1688 1689 if (nr_pages >= nr_pages_needed) 1690 break; 1691 } 1692 1693 for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) { 1694 spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa)); 1695 epfn = min_t(unsigned long, first_deferred_pfn, PFN_DOWN(epa)); 1696 deferred_free_pages(nid, zid, spfn, epfn); 1697 1698 if (first_deferred_pfn == epfn) 1699 break; 1700 } 1701 pgdat->first_deferred_pfn = first_deferred_pfn; 1702 pgdat_resize_unlock(pgdat, &flags); 1703 1704 return nr_pages > 0; 1705 } 1706 1707 /* 1708 * deferred_grow_zone() is __init, but it is called from 1709 * get_page_from_freelist() during early boot until deferred_pages permanently 1710 * disables this call. This is why we have refdata wrapper to avoid warning, 1711 * and to ensure that the function body gets unloaded. 1712 */ 1713 static bool __ref 1714 _deferred_grow_zone(struct zone *zone, unsigned int order) 1715 { 1716 return deferred_grow_zone(zone, order); 1717 } 1718 1719 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1720 1721 void __init page_alloc_init_late(void) 1722 { 1723 struct zone *zone; 1724 1725 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1726 int nid; 1727 1728 /* There will be num_node_state(N_MEMORY) threads */ 1729 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 1730 for_each_node_state(nid, N_MEMORY) { 1731 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 1732 } 1733 1734 /* Block until all are initialised */ 1735 wait_for_completion(&pgdat_init_all_done_comp); 1736 1737 /* 1738 * We initialized the rest of the deferred pages. Permanently disable 1739 * on-demand struct page initialization. 1740 */ 1741 static_branch_disable(&deferred_pages); 1742 1743 /* Reinit limits that are based on free pages after the kernel is up */ 1744 files_maxfiles_init(); 1745 #endif 1746 #ifdef CONFIG_ARCH_DISCARD_MEMBLOCK 1747 /* Discard memblock private memory */ 1748 memblock_discard(); 1749 #endif 1750 1751 for_each_populated_zone(zone) 1752 set_zone_contiguous(zone); 1753 } 1754 1755 #ifdef CONFIG_CMA 1756 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 1757 void __init init_cma_reserved_pageblock(struct page *page) 1758 { 1759 unsigned i = pageblock_nr_pages; 1760 struct page *p = page; 1761 1762 do { 1763 __ClearPageReserved(p); 1764 set_page_count(p, 0); 1765 } while (++p, --i); 1766 1767 set_pageblock_migratetype(page, MIGRATE_CMA); 1768 1769 if (pageblock_order >= MAX_ORDER) { 1770 i = pageblock_nr_pages; 1771 p = page; 1772 do { 1773 set_page_refcounted(p); 1774 __free_pages(p, MAX_ORDER - 1); 1775 p += MAX_ORDER_NR_PAGES; 1776 } while (i -= MAX_ORDER_NR_PAGES); 1777 } else { 1778 set_page_refcounted(page); 1779 __free_pages(page, pageblock_order); 1780 } 1781 1782 adjust_managed_page_count(page, pageblock_nr_pages); 1783 } 1784 #endif 1785 1786 /* 1787 * The order of subdivision here is critical for the IO subsystem. 1788 * Please do not alter this order without good reasons and regression 1789 * testing. Specifically, as large blocks of memory are subdivided, 1790 * the order in which smaller blocks are delivered depends on the order 1791 * they're subdivided in this function. This is the primary factor 1792 * influencing the order in which pages are delivered to the IO 1793 * subsystem according to empirical testing, and this is also justified 1794 * by considering the behavior of a buddy system containing a single 1795 * large block of memory acted on by a series of small allocations. 1796 * This behavior is a critical factor in sglist merging's success. 1797 * 1798 * -- nyc 1799 */ 1800 static inline void expand(struct zone *zone, struct page *page, 1801 int low, int high, struct free_area *area, 1802 int migratetype) 1803 { 1804 unsigned long size = 1 << high; 1805 1806 while (high > low) { 1807 area--; 1808 high--; 1809 size >>= 1; 1810 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 1811 1812 /* 1813 * Mark as guard pages (or page), that will allow to 1814 * merge back to allocator when buddy will be freed. 1815 * Corresponding page table entries will not be touched, 1816 * pages will stay not present in virtual address space 1817 */ 1818 if (set_page_guard(zone, &page[size], high, migratetype)) 1819 continue; 1820 1821 list_add(&page[size].lru, &area->free_list[migratetype]); 1822 area->nr_free++; 1823 set_page_order(&page[size], high); 1824 } 1825 } 1826 1827 static void check_new_page_bad(struct page *page) 1828 { 1829 const char *bad_reason = NULL; 1830 unsigned long bad_flags = 0; 1831 1832 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1833 bad_reason = "nonzero mapcount"; 1834 if (unlikely(page->mapping != NULL)) 1835 bad_reason = "non-NULL mapping"; 1836 if (unlikely(page_ref_count(page) != 0)) 1837 bad_reason = "nonzero _count"; 1838 if (unlikely(page->flags & __PG_HWPOISON)) { 1839 bad_reason = "HWPoisoned (hardware-corrupted)"; 1840 bad_flags = __PG_HWPOISON; 1841 /* Don't complain about hwpoisoned pages */ 1842 page_mapcount_reset(page); /* remove PageBuddy */ 1843 return; 1844 } 1845 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) { 1846 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set"; 1847 bad_flags = PAGE_FLAGS_CHECK_AT_PREP; 1848 } 1849 #ifdef CONFIG_MEMCG 1850 if (unlikely(page->mem_cgroup)) 1851 bad_reason = "page still charged to cgroup"; 1852 #endif 1853 bad_page(page, bad_reason, bad_flags); 1854 } 1855 1856 /* 1857 * This page is about to be returned from the page allocator 1858 */ 1859 static inline int check_new_page(struct page *page) 1860 { 1861 if (likely(page_expected_state(page, 1862 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 1863 return 0; 1864 1865 check_new_page_bad(page); 1866 return 1; 1867 } 1868 1869 static inline bool free_pages_prezeroed(void) 1870 { 1871 return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) && 1872 page_poisoning_enabled(); 1873 } 1874 1875 #ifdef CONFIG_DEBUG_VM 1876 static bool check_pcp_refill(struct page *page) 1877 { 1878 return false; 1879 } 1880 1881 static bool check_new_pcp(struct page *page) 1882 { 1883 return check_new_page(page); 1884 } 1885 #else 1886 static bool check_pcp_refill(struct page *page) 1887 { 1888 return check_new_page(page); 1889 } 1890 static bool check_new_pcp(struct page *page) 1891 { 1892 return false; 1893 } 1894 #endif /* CONFIG_DEBUG_VM */ 1895 1896 static bool check_new_pages(struct page *page, unsigned int order) 1897 { 1898 int i; 1899 for (i = 0; i < (1 << order); i++) { 1900 struct page *p = page + i; 1901 1902 if (unlikely(check_new_page(p))) 1903 return true; 1904 } 1905 1906 return false; 1907 } 1908 1909 inline void post_alloc_hook(struct page *page, unsigned int order, 1910 gfp_t gfp_flags) 1911 { 1912 set_page_private(page, 0); 1913 set_page_refcounted(page); 1914 1915 arch_alloc_page(page, order); 1916 kernel_map_pages(page, 1 << order, 1); 1917 kernel_poison_pages(page, 1 << order, 1); 1918 kasan_alloc_pages(page, order); 1919 set_page_owner(page, order, gfp_flags); 1920 } 1921 1922 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 1923 unsigned int alloc_flags) 1924 { 1925 int i; 1926 1927 post_alloc_hook(page, order, gfp_flags); 1928 1929 if (!free_pages_prezeroed() && (gfp_flags & __GFP_ZERO)) 1930 for (i = 0; i < (1 << order); i++) 1931 clear_highpage(page + i); 1932 1933 if (order && (gfp_flags & __GFP_COMP)) 1934 prep_compound_page(page, order); 1935 1936 /* 1937 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 1938 * allocate the page. The expectation is that the caller is taking 1939 * steps that will free more memory. The caller should avoid the page 1940 * being used for !PFMEMALLOC purposes. 1941 */ 1942 if (alloc_flags & ALLOC_NO_WATERMARKS) 1943 set_page_pfmemalloc(page); 1944 else 1945 clear_page_pfmemalloc(page); 1946 } 1947 1948 /* 1949 * Go through the free lists for the given migratetype and remove 1950 * the smallest available page from the freelists 1951 */ 1952 static __always_inline 1953 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 1954 int migratetype) 1955 { 1956 unsigned int current_order; 1957 struct free_area *area; 1958 struct page *page; 1959 1960 /* Find a page of the appropriate size in the preferred list */ 1961 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 1962 area = &(zone->free_area[current_order]); 1963 page = list_first_entry_or_null(&area->free_list[migratetype], 1964 struct page, lru); 1965 if (!page) 1966 continue; 1967 list_del(&page->lru); 1968 rmv_page_order(page); 1969 area->nr_free--; 1970 expand(zone, page, order, current_order, area, migratetype); 1971 set_pcppage_migratetype(page, migratetype); 1972 return page; 1973 } 1974 1975 return NULL; 1976 } 1977 1978 1979 /* 1980 * This array describes the order lists are fallen back to when 1981 * the free lists for the desirable migrate type are depleted 1982 */ 1983 static int fallbacks[MIGRATE_TYPES][4] = { 1984 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 1985 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 1986 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 1987 #ifdef CONFIG_CMA 1988 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */ 1989 #endif 1990 #ifdef CONFIG_MEMORY_ISOLATION 1991 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */ 1992 #endif 1993 }; 1994 1995 #ifdef CONFIG_CMA 1996 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, 1997 unsigned int order) 1998 { 1999 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 2000 } 2001 #else 2002 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2003 unsigned int order) { return NULL; } 2004 #endif 2005 2006 /* 2007 * Move the free pages in a range to the free lists of the requested type. 2008 * Note that start_page and end_pages are not aligned on a pageblock 2009 * boundary. If alignment is required, use move_freepages_block() 2010 */ 2011 static int move_freepages(struct zone *zone, 2012 struct page *start_page, struct page *end_page, 2013 int migratetype, int *num_movable) 2014 { 2015 struct page *page; 2016 unsigned int order; 2017 int pages_moved = 0; 2018 2019 #ifndef CONFIG_HOLES_IN_ZONE 2020 /* 2021 * page_zone is not safe to call in this context when 2022 * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant 2023 * anyway as we check zone boundaries in move_freepages_block(). 2024 * Remove at a later date when no bug reports exist related to 2025 * grouping pages by mobility 2026 */ 2027 VM_BUG_ON(pfn_valid(page_to_pfn(start_page)) && 2028 pfn_valid(page_to_pfn(end_page)) && 2029 page_zone(start_page) != page_zone(end_page)); 2030 #endif 2031 for (page = start_page; page <= end_page;) { 2032 if (!pfn_valid_within(page_to_pfn(page))) { 2033 page++; 2034 continue; 2035 } 2036 2037 /* Make sure we are not inadvertently changing nodes */ 2038 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2039 2040 if (!PageBuddy(page)) { 2041 /* 2042 * We assume that pages that could be isolated for 2043 * migration are movable. But we don't actually try 2044 * isolating, as that would be expensive. 2045 */ 2046 if (num_movable && 2047 (PageLRU(page) || __PageMovable(page))) 2048 (*num_movable)++; 2049 2050 page++; 2051 continue; 2052 } 2053 2054 order = page_order(page); 2055 list_move(&page->lru, 2056 &zone->free_area[order].free_list[migratetype]); 2057 page += 1 << order; 2058 pages_moved += 1 << order; 2059 } 2060 2061 return pages_moved; 2062 } 2063 2064 int move_freepages_block(struct zone *zone, struct page *page, 2065 int migratetype, int *num_movable) 2066 { 2067 unsigned long start_pfn, end_pfn; 2068 struct page *start_page, *end_page; 2069 2070 if (num_movable) 2071 *num_movable = 0; 2072 2073 start_pfn = page_to_pfn(page); 2074 start_pfn = start_pfn & ~(pageblock_nr_pages-1); 2075 start_page = pfn_to_page(start_pfn); 2076 end_page = start_page + pageblock_nr_pages - 1; 2077 end_pfn = start_pfn + pageblock_nr_pages - 1; 2078 2079 /* Do not cross zone boundaries */ 2080 if (!zone_spans_pfn(zone, start_pfn)) 2081 start_page = page; 2082 if (!zone_spans_pfn(zone, end_pfn)) 2083 return 0; 2084 2085 return move_freepages(zone, start_page, end_page, migratetype, 2086 num_movable); 2087 } 2088 2089 static void change_pageblock_range(struct page *pageblock_page, 2090 int start_order, int migratetype) 2091 { 2092 int nr_pageblocks = 1 << (start_order - pageblock_order); 2093 2094 while (nr_pageblocks--) { 2095 set_pageblock_migratetype(pageblock_page, migratetype); 2096 pageblock_page += pageblock_nr_pages; 2097 } 2098 } 2099 2100 /* 2101 * When we are falling back to another migratetype during allocation, try to 2102 * steal extra free pages from the same pageblocks to satisfy further 2103 * allocations, instead of polluting multiple pageblocks. 2104 * 2105 * If we are stealing a relatively large buddy page, it is likely there will 2106 * be more free pages in the pageblock, so try to steal them all. For 2107 * reclaimable and unmovable allocations, we steal regardless of page size, 2108 * as fragmentation caused by those allocations polluting movable pageblocks 2109 * is worse than movable allocations stealing from unmovable and reclaimable 2110 * pageblocks. 2111 */ 2112 static bool can_steal_fallback(unsigned int order, int start_mt) 2113 { 2114 /* 2115 * Leaving this order check is intended, although there is 2116 * relaxed order check in next check. The reason is that 2117 * we can actually steal whole pageblock if this condition met, 2118 * but, below check doesn't guarantee it and that is just heuristic 2119 * so could be changed anytime. 2120 */ 2121 if (order >= pageblock_order) 2122 return true; 2123 2124 if (order >= pageblock_order / 2 || 2125 start_mt == MIGRATE_RECLAIMABLE || 2126 start_mt == MIGRATE_UNMOVABLE || 2127 page_group_by_mobility_disabled) 2128 return true; 2129 2130 return false; 2131 } 2132 2133 /* 2134 * This function implements actual steal behaviour. If order is large enough, 2135 * we can steal whole pageblock. If not, we first move freepages in this 2136 * pageblock to our migratetype and determine how many already-allocated pages 2137 * are there in the pageblock with a compatible migratetype. If at least half 2138 * of pages are free or compatible, we can change migratetype of the pageblock 2139 * itself, so pages freed in the future will be put on the correct free list. 2140 */ 2141 static void steal_suitable_fallback(struct zone *zone, struct page *page, 2142 int start_type, bool whole_block) 2143 { 2144 unsigned int current_order = page_order(page); 2145 struct free_area *area; 2146 int free_pages, movable_pages, alike_pages; 2147 int old_block_type; 2148 2149 old_block_type = get_pageblock_migratetype(page); 2150 2151 /* 2152 * This can happen due to races and we want to prevent broken 2153 * highatomic accounting. 2154 */ 2155 if (is_migrate_highatomic(old_block_type)) 2156 goto single_page; 2157 2158 /* Take ownership for orders >= pageblock_order */ 2159 if (current_order >= pageblock_order) { 2160 change_pageblock_range(page, current_order, start_type); 2161 goto single_page; 2162 } 2163 2164 /* We are not allowed to try stealing from the whole block */ 2165 if (!whole_block) 2166 goto single_page; 2167 2168 free_pages = move_freepages_block(zone, page, start_type, 2169 &movable_pages); 2170 /* 2171 * Determine how many pages are compatible with our allocation. 2172 * For movable allocation, it's the number of movable pages which 2173 * we just obtained. For other types it's a bit more tricky. 2174 */ 2175 if (start_type == MIGRATE_MOVABLE) { 2176 alike_pages = movable_pages; 2177 } else { 2178 /* 2179 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation 2180 * to MOVABLE pageblock, consider all non-movable pages as 2181 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or 2182 * vice versa, be conservative since we can't distinguish the 2183 * exact migratetype of non-movable pages. 2184 */ 2185 if (old_block_type == MIGRATE_MOVABLE) 2186 alike_pages = pageblock_nr_pages 2187 - (free_pages + movable_pages); 2188 else 2189 alike_pages = 0; 2190 } 2191 2192 /* moving whole block can fail due to zone boundary conditions */ 2193 if (!free_pages) 2194 goto single_page; 2195 2196 /* 2197 * If a sufficient number of pages in the block are either free or of 2198 * comparable migratability as our allocation, claim the whole block. 2199 */ 2200 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || 2201 page_group_by_mobility_disabled) 2202 set_pageblock_migratetype(page, start_type); 2203 2204 return; 2205 2206 single_page: 2207 area = &zone->free_area[current_order]; 2208 list_move(&page->lru, &area->free_list[start_type]); 2209 } 2210 2211 /* 2212 * Check whether there is a suitable fallback freepage with requested order. 2213 * If only_stealable is true, this function returns fallback_mt only if 2214 * we can steal other freepages all together. This would help to reduce 2215 * fragmentation due to mixed migratetype pages in one pageblock. 2216 */ 2217 int find_suitable_fallback(struct free_area *area, unsigned int order, 2218 int migratetype, bool only_stealable, bool *can_steal) 2219 { 2220 int i; 2221 int fallback_mt; 2222 2223 if (area->nr_free == 0) 2224 return -1; 2225 2226 *can_steal = false; 2227 for (i = 0;; i++) { 2228 fallback_mt = fallbacks[migratetype][i]; 2229 if (fallback_mt == MIGRATE_TYPES) 2230 break; 2231 2232 if (list_empty(&area->free_list[fallback_mt])) 2233 continue; 2234 2235 if (can_steal_fallback(order, migratetype)) 2236 *can_steal = true; 2237 2238 if (!only_stealable) 2239 return fallback_mt; 2240 2241 if (*can_steal) 2242 return fallback_mt; 2243 } 2244 2245 return -1; 2246 } 2247 2248 /* 2249 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2250 * there are no empty page blocks that contain a page with a suitable order 2251 */ 2252 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2253 unsigned int alloc_order) 2254 { 2255 int mt; 2256 unsigned long max_managed, flags; 2257 2258 /* 2259 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2260 * Check is race-prone but harmless. 2261 */ 2262 max_managed = (zone->managed_pages / 100) + pageblock_nr_pages; 2263 if (zone->nr_reserved_highatomic >= max_managed) 2264 return; 2265 2266 spin_lock_irqsave(&zone->lock, flags); 2267 2268 /* Recheck the nr_reserved_highatomic limit under the lock */ 2269 if (zone->nr_reserved_highatomic >= max_managed) 2270 goto out_unlock; 2271 2272 /* Yoink! */ 2273 mt = get_pageblock_migratetype(page); 2274 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt) 2275 && !is_migrate_cma(mt)) { 2276 zone->nr_reserved_highatomic += pageblock_nr_pages; 2277 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2278 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); 2279 } 2280 2281 out_unlock: 2282 spin_unlock_irqrestore(&zone->lock, flags); 2283 } 2284 2285 /* 2286 * Used when an allocation is about to fail under memory pressure. This 2287 * potentially hurts the reliability of high-order allocations when under 2288 * intense memory pressure but failed atomic allocations should be easier 2289 * to recover from than an OOM. 2290 * 2291 * If @force is true, try to unreserve a pageblock even though highatomic 2292 * pageblock is exhausted. 2293 */ 2294 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 2295 bool force) 2296 { 2297 struct zonelist *zonelist = ac->zonelist; 2298 unsigned long flags; 2299 struct zoneref *z; 2300 struct zone *zone; 2301 struct page *page; 2302 int order; 2303 bool ret; 2304 2305 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx, 2306 ac->nodemask) { 2307 /* 2308 * Preserve at least one pageblock unless memory pressure 2309 * is really high. 2310 */ 2311 if (!force && zone->nr_reserved_highatomic <= 2312 pageblock_nr_pages) 2313 continue; 2314 2315 spin_lock_irqsave(&zone->lock, flags); 2316 for (order = 0; order < MAX_ORDER; order++) { 2317 struct free_area *area = &(zone->free_area[order]); 2318 2319 page = list_first_entry_or_null( 2320 &area->free_list[MIGRATE_HIGHATOMIC], 2321 struct page, lru); 2322 if (!page) 2323 continue; 2324 2325 /* 2326 * In page freeing path, migratetype change is racy so 2327 * we can counter several free pages in a pageblock 2328 * in this loop althoug we changed the pageblock type 2329 * from highatomic to ac->migratetype. So we should 2330 * adjust the count once. 2331 */ 2332 if (is_migrate_highatomic_page(page)) { 2333 /* 2334 * It should never happen but changes to 2335 * locking could inadvertently allow a per-cpu 2336 * drain to add pages to MIGRATE_HIGHATOMIC 2337 * while unreserving so be safe and watch for 2338 * underflows. 2339 */ 2340 zone->nr_reserved_highatomic -= min( 2341 pageblock_nr_pages, 2342 zone->nr_reserved_highatomic); 2343 } 2344 2345 /* 2346 * Convert to ac->migratetype and avoid the normal 2347 * pageblock stealing heuristics. Minimally, the caller 2348 * is doing the work and needs the pages. More 2349 * importantly, if the block was always converted to 2350 * MIGRATE_UNMOVABLE or another type then the number 2351 * of pageblocks that cannot be completely freed 2352 * may increase. 2353 */ 2354 set_pageblock_migratetype(page, ac->migratetype); 2355 ret = move_freepages_block(zone, page, ac->migratetype, 2356 NULL); 2357 if (ret) { 2358 spin_unlock_irqrestore(&zone->lock, flags); 2359 return ret; 2360 } 2361 } 2362 spin_unlock_irqrestore(&zone->lock, flags); 2363 } 2364 2365 return false; 2366 } 2367 2368 /* 2369 * Try finding a free buddy page on the fallback list and put it on the free 2370 * list of requested migratetype, possibly along with other pages from the same 2371 * block, depending on fragmentation avoidance heuristics. Returns true if 2372 * fallback was found so that __rmqueue_smallest() can grab it. 2373 * 2374 * The use of signed ints for order and current_order is a deliberate 2375 * deviation from the rest of this file, to make the for loop 2376 * condition simpler. 2377 */ 2378 static __always_inline bool 2379 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype) 2380 { 2381 struct free_area *area; 2382 int current_order; 2383 struct page *page; 2384 int fallback_mt; 2385 bool can_steal; 2386 2387 /* 2388 * Find the largest available free page in the other list. This roughly 2389 * approximates finding the pageblock with the most free pages, which 2390 * would be too costly to do exactly. 2391 */ 2392 for (current_order = MAX_ORDER - 1; current_order >= order; 2393 --current_order) { 2394 area = &(zone->free_area[current_order]); 2395 fallback_mt = find_suitable_fallback(area, current_order, 2396 start_migratetype, false, &can_steal); 2397 if (fallback_mt == -1) 2398 continue; 2399 2400 /* 2401 * We cannot steal all free pages from the pageblock and the 2402 * requested migratetype is movable. In that case it's better to 2403 * steal and split the smallest available page instead of the 2404 * largest available page, because even if the next movable 2405 * allocation falls back into a different pageblock than this 2406 * one, it won't cause permanent fragmentation. 2407 */ 2408 if (!can_steal && start_migratetype == MIGRATE_MOVABLE 2409 && current_order > order) 2410 goto find_smallest; 2411 2412 goto do_steal; 2413 } 2414 2415 return false; 2416 2417 find_smallest: 2418 for (current_order = order; current_order < MAX_ORDER; 2419 current_order++) { 2420 area = &(zone->free_area[current_order]); 2421 fallback_mt = find_suitable_fallback(area, current_order, 2422 start_migratetype, false, &can_steal); 2423 if (fallback_mt != -1) 2424 break; 2425 } 2426 2427 /* 2428 * This should not happen - we already found a suitable fallback 2429 * when looking for the largest page. 2430 */ 2431 VM_BUG_ON(current_order == MAX_ORDER); 2432 2433 do_steal: 2434 page = list_first_entry(&area->free_list[fallback_mt], 2435 struct page, lru); 2436 2437 steal_suitable_fallback(zone, page, start_migratetype, can_steal); 2438 2439 trace_mm_page_alloc_extfrag(page, order, current_order, 2440 start_migratetype, fallback_mt); 2441 2442 return true; 2443 2444 } 2445 2446 /* 2447 * Do the hard work of removing an element from the buddy allocator. 2448 * Call me with the zone->lock already held. 2449 */ 2450 static __always_inline struct page * 2451 __rmqueue(struct zone *zone, unsigned int order, int migratetype) 2452 { 2453 struct page *page; 2454 2455 retry: 2456 page = __rmqueue_smallest(zone, order, migratetype); 2457 if (unlikely(!page)) { 2458 if (migratetype == MIGRATE_MOVABLE) 2459 page = __rmqueue_cma_fallback(zone, order); 2460 2461 if (!page && __rmqueue_fallback(zone, order, migratetype)) 2462 goto retry; 2463 } 2464 2465 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2466 return page; 2467 } 2468 2469 /* 2470 * Obtain a specified number of elements from the buddy allocator, all under 2471 * a single hold of the lock, for efficiency. Add them to the supplied list. 2472 * Returns the number of new pages which were placed at *list. 2473 */ 2474 static int rmqueue_bulk(struct zone *zone, unsigned int order, 2475 unsigned long count, struct list_head *list, 2476 int migratetype) 2477 { 2478 int i, alloced = 0; 2479 2480 spin_lock(&zone->lock); 2481 for (i = 0; i < count; ++i) { 2482 struct page *page = __rmqueue(zone, order, migratetype); 2483 if (unlikely(page == NULL)) 2484 break; 2485 2486 if (unlikely(check_pcp_refill(page))) 2487 continue; 2488 2489 /* 2490 * Split buddy pages returned by expand() are received here in 2491 * physical page order. The page is added to the tail of 2492 * caller's list. From the callers perspective, the linked list 2493 * is ordered by page number under some conditions. This is 2494 * useful for IO devices that can forward direction from the 2495 * head, thus also in the physical page order. This is useful 2496 * for IO devices that can merge IO requests if the physical 2497 * pages are ordered properly. 2498 */ 2499 list_add_tail(&page->lru, list); 2500 alloced++; 2501 if (is_migrate_cma(get_pcppage_migratetype(page))) 2502 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 2503 -(1 << order)); 2504 } 2505 2506 /* 2507 * i pages were removed from the buddy list even if some leak due 2508 * to check_pcp_refill failing so adjust NR_FREE_PAGES based 2509 * on i. Do not confuse with 'alloced' which is the number of 2510 * pages added to the pcp list. 2511 */ 2512 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 2513 spin_unlock(&zone->lock); 2514 return alloced; 2515 } 2516 2517 #ifdef CONFIG_NUMA 2518 /* 2519 * Called from the vmstat counter updater to drain pagesets of this 2520 * currently executing processor on remote nodes after they have 2521 * expired. 2522 * 2523 * Note that this function must be called with the thread pinned to 2524 * a single processor. 2525 */ 2526 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 2527 { 2528 unsigned long flags; 2529 int to_drain, batch; 2530 2531 local_irq_save(flags); 2532 batch = READ_ONCE(pcp->batch); 2533 to_drain = min(pcp->count, batch); 2534 if (to_drain > 0) 2535 free_pcppages_bulk(zone, to_drain, pcp); 2536 local_irq_restore(flags); 2537 } 2538 #endif 2539 2540 /* 2541 * Drain pcplists of the indicated processor and zone. 2542 * 2543 * The processor must either be the current processor and the 2544 * thread pinned to the current processor or a processor that 2545 * is not online. 2546 */ 2547 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 2548 { 2549 unsigned long flags; 2550 struct per_cpu_pageset *pset; 2551 struct per_cpu_pages *pcp; 2552 2553 local_irq_save(flags); 2554 pset = per_cpu_ptr(zone->pageset, cpu); 2555 2556 pcp = &pset->pcp; 2557 if (pcp->count) 2558 free_pcppages_bulk(zone, pcp->count, pcp); 2559 local_irq_restore(flags); 2560 } 2561 2562 /* 2563 * Drain pcplists of all zones on the indicated processor. 2564 * 2565 * The processor must either be the current processor and the 2566 * thread pinned to the current processor or a processor that 2567 * is not online. 2568 */ 2569 static void drain_pages(unsigned int cpu) 2570 { 2571 struct zone *zone; 2572 2573 for_each_populated_zone(zone) { 2574 drain_pages_zone(cpu, zone); 2575 } 2576 } 2577 2578 /* 2579 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 2580 * 2581 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 2582 * the single zone's pages. 2583 */ 2584 void drain_local_pages(struct zone *zone) 2585 { 2586 int cpu = smp_processor_id(); 2587 2588 if (zone) 2589 drain_pages_zone(cpu, zone); 2590 else 2591 drain_pages(cpu); 2592 } 2593 2594 static void drain_local_pages_wq(struct work_struct *work) 2595 { 2596 /* 2597 * drain_all_pages doesn't use proper cpu hotplug protection so 2598 * we can race with cpu offline when the WQ can move this from 2599 * a cpu pinned worker to an unbound one. We can operate on a different 2600 * cpu which is allright but we also have to make sure to not move to 2601 * a different one. 2602 */ 2603 preempt_disable(); 2604 drain_local_pages(NULL); 2605 preempt_enable(); 2606 } 2607 2608 /* 2609 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 2610 * 2611 * When zone parameter is non-NULL, spill just the single zone's pages. 2612 * 2613 * Note that this can be extremely slow as the draining happens in a workqueue. 2614 */ 2615 void drain_all_pages(struct zone *zone) 2616 { 2617 int cpu; 2618 2619 /* 2620 * Allocate in the BSS so we wont require allocation in 2621 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 2622 */ 2623 static cpumask_t cpus_with_pcps; 2624 2625 /* 2626 * Make sure nobody triggers this path before mm_percpu_wq is fully 2627 * initialized. 2628 */ 2629 if (WARN_ON_ONCE(!mm_percpu_wq)) 2630 return; 2631 2632 /* 2633 * Do not drain if one is already in progress unless it's specific to 2634 * a zone. Such callers are primarily CMA and memory hotplug and need 2635 * the drain to be complete when the call returns. 2636 */ 2637 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) { 2638 if (!zone) 2639 return; 2640 mutex_lock(&pcpu_drain_mutex); 2641 } 2642 2643 /* 2644 * We don't care about racing with CPU hotplug event 2645 * as offline notification will cause the notified 2646 * cpu to drain that CPU pcps and on_each_cpu_mask 2647 * disables preemption as part of its processing 2648 */ 2649 for_each_online_cpu(cpu) { 2650 struct per_cpu_pageset *pcp; 2651 struct zone *z; 2652 bool has_pcps = false; 2653 2654 if (zone) { 2655 pcp = per_cpu_ptr(zone->pageset, cpu); 2656 if (pcp->pcp.count) 2657 has_pcps = true; 2658 } else { 2659 for_each_populated_zone(z) { 2660 pcp = per_cpu_ptr(z->pageset, cpu); 2661 if (pcp->pcp.count) { 2662 has_pcps = true; 2663 break; 2664 } 2665 } 2666 } 2667 2668 if (has_pcps) 2669 cpumask_set_cpu(cpu, &cpus_with_pcps); 2670 else 2671 cpumask_clear_cpu(cpu, &cpus_with_pcps); 2672 } 2673 2674 for_each_cpu(cpu, &cpus_with_pcps) { 2675 struct work_struct *work = per_cpu_ptr(&pcpu_drain, cpu); 2676 INIT_WORK(work, drain_local_pages_wq); 2677 queue_work_on(cpu, mm_percpu_wq, work); 2678 } 2679 for_each_cpu(cpu, &cpus_with_pcps) 2680 flush_work(per_cpu_ptr(&pcpu_drain, cpu)); 2681 2682 mutex_unlock(&pcpu_drain_mutex); 2683 } 2684 2685 #ifdef CONFIG_HIBERNATION 2686 2687 /* 2688 * Touch the watchdog for every WD_PAGE_COUNT pages. 2689 */ 2690 #define WD_PAGE_COUNT (128*1024) 2691 2692 void mark_free_pages(struct zone *zone) 2693 { 2694 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT; 2695 unsigned long flags; 2696 unsigned int order, t; 2697 struct page *page; 2698 2699 if (zone_is_empty(zone)) 2700 return; 2701 2702 spin_lock_irqsave(&zone->lock, flags); 2703 2704 max_zone_pfn = zone_end_pfn(zone); 2705 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 2706 if (pfn_valid(pfn)) { 2707 page = pfn_to_page(pfn); 2708 2709 if (!--page_count) { 2710 touch_nmi_watchdog(); 2711 page_count = WD_PAGE_COUNT; 2712 } 2713 2714 if (page_zone(page) != zone) 2715 continue; 2716 2717 if (!swsusp_page_is_forbidden(page)) 2718 swsusp_unset_page_free(page); 2719 } 2720 2721 for_each_migratetype_order(order, t) { 2722 list_for_each_entry(page, 2723 &zone->free_area[order].free_list[t], lru) { 2724 unsigned long i; 2725 2726 pfn = page_to_pfn(page); 2727 for (i = 0; i < (1UL << order); i++) { 2728 if (!--page_count) { 2729 touch_nmi_watchdog(); 2730 page_count = WD_PAGE_COUNT; 2731 } 2732 swsusp_set_page_free(pfn_to_page(pfn + i)); 2733 } 2734 } 2735 } 2736 spin_unlock_irqrestore(&zone->lock, flags); 2737 } 2738 #endif /* CONFIG_PM */ 2739 2740 static bool free_unref_page_prepare(struct page *page, unsigned long pfn) 2741 { 2742 int migratetype; 2743 2744 if (!free_pcp_prepare(page)) 2745 return false; 2746 2747 migratetype = get_pfnblock_migratetype(page, pfn); 2748 set_pcppage_migratetype(page, migratetype); 2749 return true; 2750 } 2751 2752 static void free_unref_page_commit(struct page *page, unsigned long pfn) 2753 { 2754 struct zone *zone = page_zone(page); 2755 struct per_cpu_pages *pcp; 2756 int migratetype; 2757 2758 migratetype = get_pcppage_migratetype(page); 2759 __count_vm_event(PGFREE); 2760 2761 /* 2762 * We only track unmovable, reclaimable and movable on pcp lists. 2763 * Free ISOLATE pages back to the allocator because they are being 2764 * offlined but treat HIGHATOMIC as movable pages so we can get those 2765 * areas back if necessary. Otherwise, we may have to free 2766 * excessively into the page allocator 2767 */ 2768 if (migratetype >= MIGRATE_PCPTYPES) { 2769 if (unlikely(is_migrate_isolate(migratetype))) { 2770 free_one_page(zone, page, pfn, 0, migratetype); 2771 return; 2772 } 2773 migratetype = MIGRATE_MOVABLE; 2774 } 2775 2776 pcp = &this_cpu_ptr(zone->pageset)->pcp; 2777 list_add(&page->lru, &pcp->lists[migratetype]); 2778 pcp->count++; 2779 if (pcp->count >= pcp->high) { 2780 unsigned long batch = READ_ONCE(pcp->batch); 2781 free_pcppages_bulk(zone, batch, pcp); 2782 } 2783 } 2784 2785 /* 2786 * Free a 0-order page 2787 */ 2788 void free_unref_page(struct page *page) 2789 { 2790 unsigned long flags; 2791 unsigned long pfn = page_to_pfn(page); 2792 2793 if (!free_unref_page_prepare(page, pfn)) 2794 return; 2795 2796 local_irq_save(flags); 2797 free_unref_page_commit(page, pfn); 2798 local_irq_restore(flags); 2799 } 2800 2801 /* 2802 * Free a list of 0-order pages 2803 */ 2804 void free_unref_page_list(struct list_head *list) 2805 { 2806 struct page *page, *next; 2807 unsigned long flags, pfn; 2808 int batch_count = 0; 2809 2810 /* Prepare pages for freeing */ 2811 list_for_each_entry_safe(page, next, list, lru) { 2812 pfn = page_to_pfn(page); 2813 if (!free_unref_page_prepare(page, pfn)) 2814 list_del(&page->lru); 2815 set_page_private(page, pfn); 2816 } 2817 2818 local_irq_save(flags); 2819 list_for_each_entry_safe(page, next, list, lru) { 2820 unsigned long pfn = page_private(page); 2821 2822 set_page_private(page, 0); 2823 trace_mm_page_free_batched(page); 2824 free_unref_page_commit(page, pfn); 2825 2826 /* 2827 * Guard against excessive IRQ disabled times when we get 2828 * a large list of pages to free. 2829 */ 2830 if (++batch_count == SWAP_CLUSTER_MAX) { 2831 local_irq_restore(flags); 2832 batch_count = 0; 2833 local_irq_save(flags); 2834 } 2835 } 2836 local_irq_restore(flags); 2837 } 2838 2839 /* 2840 * split_page takes a non-compound higher-order page, and splits it into 2841 * n (1<<order) sub-pages: page[0..n] 2842 * Each sub-page must be freed individually. 2843 * 2844 * Note: this is probably too low level an operation for use in drivers. 2845 * Please consult with lkml before using this in your driver. 2846 */ 2847 void split_page(struct page *page, unsigned int order) 2848 { 2849 int i; 2850 2851 VM_BUG_ON_PAGE(PageCompound(page), page); 2852 VM_BUG_ON_PAGE(!page_count(page), page); 2853 2854 for (i = 1; i < (1 << order); i++) 2855 set_page_refcounted(page + i); 2856 split_page_owner(page, order); 2857 } 2858 EXPORT_SYMBOL_GPL(split_page); 2859 2860 int __isolate_free_page(struct page *page, unsigned int order) 2861 { 2862 unsigned long watermark; 2863 struct zone *zone; 2864 int mt; 2865 2866 BUG_ON(!PageBuddy(page)); 2867 2868 zone = page_zone(page); 2869 mt = get_pageblock_migratetype(page); 2870 2871 if (!is_migrate_isolate(mt)) { 2872 /* 2873 * Obey watermarks as if the page was being allocated. We can 2874 * emulate a high-order watermark check with a raised order-0 2875 * watermark, because we already know our high-order page 2876 * exists. 2877 */ 2878 watermark = min_wmark_pages(zone) + (1UL << order); 2879 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 2880 return 0; 2881 2882 __mod_zone_freepage_state(zone, -(1UL << order), mt); 2883 } 2884 2885 /* Remove page from free list */ 2886 list_del(&page->lru); 2887 zone->free_area[order].nr_free--; 2888 rmv_page_order(page); 2889 2890 /* 2891 * Set the pageblock if the isolated page is at least half of a 2892 * pageblock 2893 */ 2894 if (order >= pageblock_order - 1) { 2895 struct page *endpage = page + (1 << order) - 1; 2896 for (; page < endpage; page += pageblock_nr_pages) { 2897 int mt = get_pageblock_migratetype(page); 2898 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt) 2899 && !is_migrate_highatomic(mt)) 2900 set_pageblock_migratetype(page, 2901 MIGRATE_MOVABLE); 2902 } 2903 } 2904 2905 2906 return 1UL << order; 2907 } 2908 2909 /* 2910 * Update NUMA hit/miss statistics 2911 * 2912 * Must be called with interrupts disabled. 2913 */ 2914 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) 2915 { 2916 #ifdef CONFIG_NUMA 2917 enum numa_stat_item local_stat = NUMA_LOCAL; 2918 2919 /* skip numa counters update if numa stats is disabled */ 2920 if (!static_branch_likely(&vm_numa_stat_key)) 2921 return; 2922 2923 if (zone_to_nid(z) != numa_node_id()) 2924 local_stat = NUMA_OTHER; 2925 2926 if (zone_to_nid(z) == zone_to_nid(preferred_zone)) 2927 __inc_numa_state(z, NUMA_HIT); 2928 else { 2929 __inc_numa_state(z, NUMA_MISS); 2930 __inc_numa_state(preferred_zone, NUMA_FOREIGN); 2931 } 2932 __inc_numa_state(z, local_stat); 2933 #endif 2934 } 2935 2936 /* Remove page from the per-cpu list, caller must protect the list */ 2937 static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype, 2938 struct per_cpu_pages *pcp, 2939 struct list_head *list) 2940 { 2941 struct page *page; 2942 2943 do { 2944 if (list_empty(list)) { 2945 pcp->count += rmqueue_bulk(zone, 0, 2946 pcp->batch, list, 2947 migratetype); 2948 if (unlikely(list_empty(list))) 2949 return NULL; 2950 } 2951 2952 page = list_first_entry(list, struct page, lru); 2953 list_del(&page->lru); 2954 pcp->count--; 2955 } while (check_new_pcp(page)); 2956 2957 return page; 2958 } 2959 2960 /* Lock and remove page from the per-cpu list */ 2961 static struct page *rmqueue_pcplist(struct zone *preferred_zone, 2962 struct zone *zone, unsigned int order, 2963 gfp_t gfp_flags, int migratetype) 2964 { 2965 struct per_cpu_pages *pcp; 2966 struct list_head *list; 2967 struct page *page; 2968 unsigned long flags; 2969 2970 local_irq_save(flags); 2971 pcp = &this_cpu_ptr(zone->pageset)->pcp; 2972 list = &pcp->lists[migratetype]; 2973 page = __rmqueue_pcplist(zone, migratetype, pcp, list); 2974 if (page) { 2975 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 2976 zone_statistics(preferred_zone, zone); 2977 } 2978 local_irq_restore(flags); 2979 return page; 2980 } 2981 2982 /* 2983 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 2984 */ 2985 static inline 2986 struct page *rmqueue(struct zone *preferred_zone, 2987 struct zone *zone, unsigned int order, 2988 gfp_t gfp_flags, unsigned int alloc_flags, 2989 int migratetype) 2990 { 2991 unsigned long flags; 2992 struct page *page; 2993 2994 if (likely(order == 0)) { 2995 page = rmqueue_pcplist(preferred_zone, zone, order, 2996 gfp_flags, migratetype); 2997 goto out; 2998 } 2999 3000 /* 3001 * We most definitely don't want callers attempting to 3002 * allocate greater than order-1 page units with __GFP_NOFAIL. 3003 */ 3004 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 3005 spin_lock_irqsave(&zone->lock, flags); 3006 3007 do { 3008 page = NULL; 3009 if (alloc_flags & ALLOC_HARDER) { 3010 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3011 if (page) 3012 trace_mm_page_alloc_zone_locked(page, order, migratetype); 3013 } 3014 if (!page) 3015 page = __rmqueue(zone, order, migratetype); 3016 } while (page && check_new_pages(page, order)); 3017 spin_unlock(&zone->lock); 3018 if (!page) 3019 goto failed; 3020 __mod_zone_freepage_state(zone, -(1 << order), 3021 get_pcppage_migratetype(page)); 3022 3023 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3024 zone_statistics(preferred_zone, zone); 3025 local_irq_restore(flags); 3026 3027 out: 3028 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); 3029 return page; 3030 3031 failed: 3032 local_irq_restore(flags); 3033 return NULL; 3034 } 3035 3036 #ifdef CONFIG_FAIL_PAGE_ALLOC 3037 3038 static struct { 3039 struct fault_attr attr; 3040 3041 bool ignore_gfp_highmem; 3042 bool ignore_gfp_reclaim; 3043 u32 min_order; 3044 } fail_page_alloc = { 3045 .attr = FAULT_ATTR_INITIALIZER, 3046 .ignore_gfp_reclaim = true, 3047 .ignore_gfp_highmem = true, 3048 .min_order = 1, 3049 }; 3050 3051 static int __init setup_fail_page_alloc(char *str) 3052 { 3053 return setup_fault_attr(&fail_page_alloc.attr, str); 3054 } 3055 __setup("fail_page_alloc=", setup_fail_page_alloc); 3056 3057 static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3058 { 3059 if (order < fail_page_alloc.min_order) 3060 return false; 3061 if (gfp_mask & __GFP_NOFAIL) 3062 return false; 3063 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 3064 return false; 3065 if (fail_page_alloc.ignore_gfp_reclaim && 3066 (gfp_mask & __GFP_DIRECT_RECLAIM)) 3067 return false; 3068 3069 return should_fail(&fail_page_alloc.attr, 1 << order); 3070 } 3071 3072 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 3073 3074 static int __init fail_page_alloc_debugfs(void) 3075 { 3076 umode_t mode = S_IFREG | 0600; 3077 struct dentry *dir; 3078 3079 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 3080 &fail_page_alloc.attr); 3081 if (IS_ERR(dir)) 3082 return PTR_ERR(dir); 3083 3084 if (!debugfs_create_bool("ignore-gfp-wait", mode, dir, 3085 &fail_page_alloc.ignore_gfp_reclaim)) 3086 goto fail; 3087 if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir, 3088 &fail_page_alloc.ignore_gfp_highmem)) 3089 goto fail; 3090 if (!debugfs_create_u32("min-order", mode, dir, 3091 &fail_page_alloc.min_order)) 3092 goto fail; 3093 3094 return 0; 3095 fail: 3096 debugfs_remove_recursive(dir); 3097 3098 return -ENOMEM; 3099 } 3100 3101 late_initcall(fail_page_alloc_debugfs); 3102 3103 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 3104 3105 #else /* CONFIG_FAIL_PAGE_ALLOC */ 3106 3107 static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3108 { 3109 return false; 3110 } 3111 3112 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 3113 3114 /* 3115 * Return true if free base pages are above 'mark'. For high-order checks it 3116 * will return true of the order-0 watermark is reached and there is at least 3117 * one free page of a suitable size. Checking now avoids taking the zone lock 3118 * to check in the allocation paths if no pages are free. 3119 */ 3120 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3121 int classzone_idx, unsigned int alloc_flags, 3122 long free_pages) 3123 { 3124 long min = mark; 3125 int o; 3126 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3127 3128 /* free_pages may go negative - that's OK */ 3129 free_pages -= (1 << order) - 1; 3130 3131 if (alloc_flags & ALLOC_HIGH) 3132 min -= min / 2; 3133 3134 /* 3135 * If the caller does not have rights to ALLOC_HARDER then subtract 3136 * the high-atomic reserves. This will over-estimate the size of the 3137 * atomic reserve but it avoids a search. 3138 */ 3139 if (likely(!alloc_harder)) { 3140 free_pages -= z->nr_reserved_highatomic; 3141 } else { 3142 /* 3143 * OOM victims can try even harder than normal ALLOC_HARDER 3144 * users on the grounds that it's definitely going to be in 3145 * the exit path shortly and free memory. Any allocation it 3146 * makes during the free path will be small and short-lived. 3147 */ 3148 if (alloc_flags & ALLOC_OOM) 3149 min -= min / 2; 3150 else 3151 min -= min / 4; 3152 } 3153 3154 3155 #ifdef CONFIG_CMA 3156 /* If allocation can't use CMA areas don't use free CMA pages */ 3157 if (!(alloc_flags & ALLOC_CMA)) 3158 free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES); 3159 #endif 3160 3161 /* 3162 * Check watermarks for an order-0 allocation request. If these 3163 * are not met, then a high-order request also cannot go ahead 3164 * even if a suitable page happened to be free. 3165 */ 3166 if (free_pages <= min + z->lowmem_reserve[classzone_idx]) 3167 return false; 3168 3169 /* If this is an order-0 request then the watermark is fine */ 3170 if (!order) 3171 return true; 3172 3173 /* For a high-order request, check at least one suitable page is free */ 3174 for (o = order; o < MAX_ORDER; o++) { 3175 struct free_area *area = &z->free_area[o]; 3176 int mt; 3177 3178 if (!area->nr_free) 3179 continue; 3180 3181 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 3182 if (!list_empty(&area->free_list[mt])) 3183 return true; 3184 } 3185 3186 #ifdef CONFIG_CMA 3187 if ((alloc_flags & ALLOC_CMA) && 3188 !list_empty(&area->free_list[MIGRATE_CMA])) { 3189 return true; 3190 } 3191 #endif 3192 if (alloc_harder && 3193 !list_empty(&area->free_list[MIGRATE_HIGHATOMIC])) 3194 return true; 3195 } 3196 return false; 3197 } 3198 3199 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3200 int classzone_idx, unsigned int alloc_flags) 3201 { 3202 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 3203 zone_page_state(z, NR_FREE_PAGES)); 3204 } 3205 3206 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 3207 unsigned long mark, int classzone_idx, unsigned int alloc_flags) 3208 { 3209 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3210 long cma_pages = 0; 3211 3212 #ifdef CONFIG_CMA 3213 /* If allocation can't use CMA areas don't use free CMA pages */ 3214 if (!(alloc_flags & ALLOC_CMA)) 3215 cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES); 3216 #endif 3217 3218 /* 3219 * Fast check for order-0 only. If this fails then the reserves 3220 * need to be calculated. There is a corner case where the check 3221 * passes but only the high-order atomic reserve are free. If 3222 * the caller is !atomic then it'll uselessly search the free 3223 * list. That corner case is then slower but it is harmless. 3224 */ 3225 if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx]) 3226 return true; 3227 3228 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 3229 free_pages); 3230 } 3231 3232 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 3233 unsigned long mark, int classzone_idx) 3234 { 3235 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3236 3237 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 3238 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 3239 3240 return __zone_watermark_ok(z, order, mark, classzone_idx, 0, 3241 free_pages); 3242 } 3243 3244 #ifdef CONFIG_NUMA 3245 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3246 { 3247 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= 3248 RECLAIM_DISTANCE; 3249 } 3250 #else /* CONFIG_NUMA */ 3251 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3252 { 3253 return true; 3254 } 3255 #endif /* CONFIG_NUMA */ 3256 3257 /* 3258 * get_page_from_freelist goes through the zonelist trying to allocate 3259 * a page. 3260 */ 3261 static struct page * 3262 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 3263 const struct alloc_context *ac) 3264 { 3265 struct zoneref *z = ac->preferred_zoneref; 3266 struct zone *zone; 3267 struct pglist_data *last_pgdat_dirty_limit = NULL; 3268 3269 /* 3270 * Scan zonelist, looking for a zone with enough free. 3271 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 3272 */ 3273 for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3274 ac->nodemask) { 3275 struct page *page; 3276 unsigned long mark; 3277 3278 if (cpusets_enabled() && 3279 (alloc_flags & ALLOC_CPUSET) && 3280 !__cpuset_zone_allowed(zone, gfp_mask)) 3281 continue; 3282 /* 3283 * When allocating a page cache page for writing, we 3284 * want to get it from a node that is within its dirty 3285 * limit, such that no single node holds more than its 3286 * proportional share of globally allowed dirty pages. 3287 * The dirty limits take into account the node's 3288 * lowmem reserves and high watermark so that kswapd 3289 * should be able to balance it without having to 3290 * write pages from its LRU list. 3291 * 3292 * XXX: For now, allow allocations to potentially 3293 * exceed the per-node dirty limit in the slowpath 3294 * (spread_dirty_pages unset) before going into reclaim, 3295 * which is important when on a NUMA setup the allowed 3296 * nodes are together not big enough to reach the 3297 * global limit. The proper fix for these situations 3298 * will require awareness of nodes in the 3299 * dirty-throttling and the flusher threads. 3300 */ 3301 if (ac->spread_dirty_pages) { 3302 if (last_pgdat_dirty_limit == zone->zone_pgdat) 3303 continue; 3304 3305 if (!node_dirty_ok(zone->zone_pgdat)) { 3306 last_pgdat_dirty_limit = zone->zone_pgdat; 3307 continue; 3308 } 3309 } 3310 3311 mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK]; 3312 if (!zone_watermark_fast(zone, order, mark, 3313 ac_classzone_idx(ac), alloc_flags)) { 3314 int ret; 3315 3316 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3317 /* 3318 * Watermark failed for this zone, but see if we can 3319 * grow this zone if it contains deferred pages. 3320 */ 3321 if (static_branch_unlikely(&deferred_pages)) { 3322 if (_deferred_grow_zone(zone, order)) 3323 goto try_this_zone; 3324 } 3325 #endif 3326 /* Checked here to keep the fast path fast */ 3327 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 3328 if (alloc_flags & ALLOC_NO_WATERMARKS) 3329 goto try_this_zone; 3330 3331 if (node_reclaim_mode == 0 || 3332 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 3333 continue; 3334 3335 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 3336 switch (ret) { 3337 case NODE_RECLAIM_NOSCAN: 3338 /* did not scan */ 3339 continue; 3340 case NODE_RECLAIM_FULL: 3341 /* scanned but unreclaimable */ 3342 continue; 3343 default: 3344 /* did we reclaim enough */ 3345 if (zone_watermark_ok(zone, order, mark, 3346 ac_classzone_idx(ac), alloc_flags)) 3347 goto try_this_zone; 3348 3349 continue; 3350 } 3351 } 3352 3353 try_this_zone: 3354 page = rmqueue(ac->preferred_zoneref->zone, zone, order, 3355 gfp_mask, alloc_flags, ac->migratetype); 3356 if (page) { 3357 prep_new_page(page, order, gfp_mask, alloc_flags); 3358 3359 /* 3360 * If this is a high-order atomic allocation then check 3361 * if the pageblock should be reserved for the future 3362 */ 3363 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 3364 reserve_highatomic_pageblock(page, zone, order); 3365 3366 return page; 3367 } else { 3368 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 3369 /* Try again if zone has deferred pages */ 3370 if (static_branch_unlikely(&deferred_pages)) { 3371 if (_deferred_grow_zone(zone, order)) 3372 goto try_this_zone; 3373 } 3374 #endif 3375 } 3376 } 3377 3378 return NULL; 3379 } 3380 3381 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) 3382 { 3383 unsigned int filter = SHOW_MEM_FILTER_NODES; 3384 static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1); 3385 3386 if (!__ratelimit(&show_mem_rs)) 3387 return; 3388 3389 /* 3390 * This documents exceptions given to allocations in certain 3391 * contexts that are allowed to allocate outside current's set 3392 * of allowed nodes. 3393 */ 3394 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3395 if (tsk_is_oom_victim(current) || 3396 (current->flags & (PF_MEMALLOC | PF_EXITING))) 3397 filter &= ~SHOW_MEM_FILTER_NODES; 3398 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 3399 filter &= ~SHOW_MEM_FILTER_NODES; 3400 3401 show_mem(filter, nodemask); 3402 } 3403 3404 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) 3405 { 3406 struct va_format vaf; 3407 va_list args; 3408 static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL, 3409 DEFAULT_RATELIMIT_BURST); 3410 3411 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs)) 3412 return; 3413 3414 va_start(args, fmt); 3415 vaf.fmt = fmt; 3416 vaf.va = &args; 3417 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl\n", 3418 current->comm, &vaf, gfp_mask, &gfp_mask, 3419 nodemask_pr_args(nodemask)); 3420 va_end(args); 3421 3422 cpuset_print_current_mems_allowed(); 3423 3424 dump_stack(); 3425 warn_alloc_show_mem(gfp_mask, nodemask); 3426 } 3427 3428 static inline struct page * 3429 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, 3430 unsigned int alloc_flags, 3431 const struct alloc_context *ac) 3432 { 3433 struct page *page; 3434 3435 page = get_page_from_freelist(gfp_mask, order, 3436 alloc_flags|ALLOC_CPUSET, ac); 3437 /* 3438 * fallback to ignore cpuset restriction if our nodes 3439 * are depleted 3440 */ 3441 if (!page) 3442 page = get_page_from_freelist(gfp_mask, order, 3443 alloc_flags, ac); 3444 3445 return page; 3446 } 3447 3448 static inline struct page * 3449 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 3450 const struct alloc_context *ac, unsigned long *did_some_progress) 3451 { 3452 struct oom_control oc = { 3453 .zonelist = ac->zonelist, 3454 .nodemask = ac->nodemask, 3455 .memcg = NULL, 3456 .gfp_mask = gfp_mask, 3457 .order = order, 3458 }; 3459 struct page *page; 3460 3461 *did_some_progress = 0; 3462 3463 /* 3464 * Acquire the oom lock. If that fails, somebody else is 3465 * making progress for us. 3466 */ 3467 if (!mutex_trylock(&oom_lock)) { 3468 *did_some_progress = 1; 3469 schedule_timeout_uninterruptible(1); 3470 return NULL; 3471 } 3472 3473 /* 3474 * Go through the zonelist yet one more time, keep very high watermark 3475 * here, this is only to catch a parallel oom killing, we must fail if 3476 * we're still under heavy pressure. But make sure that this reclaim 3477 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY 3478 * allocation which will never fail due to oom_lock already held. 3479 */ 3480 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & 3481 ~__GFP_DIRECT_RECLAIM, order, 3482 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 3483 if (page) 3484 goto out; 3485 3486 /* Coredumps can quickly deplete all memory reserves */ 3487 if (current->flags & PF_DUMPCORE) 3488 goto out; 3489 /* The OOM killer will not help higher order allocs */ 3490 if (order > PAGE_ALLOC_COSTLY_ORDER) 3491 goto out; 3492 /* 3493 * We have already exhausted all our reclaim opportunities without any 3494 * success so it is time to admit defeat. We will skip the OOM killer 3495 * because it is very likely that the caller has a more reasonable 3496 * fallback than shooting a random task. 3497 */ 3498 if (gfp_mask & __GFP_RETRY_MAYFAIL) 3499 goto out; 3500 /* The OOM killer does not needlessly kill tasks for lowmem */ 3501 if (ac->high_zoneidx < ZONE_NORMAL) 3502 goto out; 3503 if (pm_suspended_storage()) 3504 goto out; 3505 /* 3506 * XXX: GFP_NOFS allocations should rather fail than rely on 3507 * other request to make a forward progress. 3508 * We are in an unfortunate situation where out_of_memory cannot 3509 * do much for this context but let's try it to at least get 3510 * access to memory reserved if the current task is killed (see 3511 * out_of_memory). Once filesystems are ready to handle allocation 3512 * failures more gracefully we should just bail out here. 3513 */ 3514 3515 /* The OOM killer may not free memory on a specific node */ 3516 if (gfp_mask & __GFP_THISNODE) 3517 goto out; 3518 3519 /* Exhausted what can be done so it's blame time */ 3520 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 3521 *did_some_progress = 1; 3522 3523 /* 3524 * Help non-failing allocations by giving them access to memory 3525 * reserves 3526 */ 3527 if (gfp_mask & __GFP_NOFAIL) 3528 page = __alloc_pages_cpuset_fallback(gfp_mask, order, 3529 ALLOC_NO_WATERMARKS, ac); 3530 } 3531 out: 3532 mutex_unlock(&oom_lock); 3533 return page; 3534 } 3535 3536 /* 3537 * Maximum number of compaction retries wit a progress before OOM 3538 * killer is consider as the only way to move forward. 3539 */ 3540 #define MAX_COMPACT_RETRIES 16 3541 3542 #ifdef CONFIG_COMPACTION 3543 /* Try memory compaction for high-order allocations before reclaim */ 3544 static struct page * 3545 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3546 unsigned int alloc_flags, const struct alloc_context *ac, 3547 enum compact_priority prio, enum compact_result *compact_result) 3548 { 3549 struct page *page; 3550 unsigned long pflags; 3551 unsigned int noreclaim_flag; 3552 3553 if (!order) 3554 return NULL; 3555 3556 psi_memstall_enter(&pflags); 3557 noreclaim_flag = memalloc_noreclaim_save(); 3558 3559 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 3560 prio); 3561 3562 memalloc_noreclaim_restore(noreclaim_flag); 3563 psi_memstall_leave(&pflags); 3564 3565 if (*compact_result <= COMPACT_INACTIVE) 3566 return NULL; 3567 3568 /* 3569 * At least in one zone compaction wasn't deferred or skipped, so let's 3570 * count a compaction stall 3571 */ 3572 count_vm_event(COMPACTSTALL); 3573 3574 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 3575 3576 if (page) { 3577 struct zone *zone = page_zone(page); 3578 3579 zone->compact_blockskip_flush = false; 3580 compaction_defer_reset(zone, order, true); 3581 count_vm_event(COMPACTSUCCESS); 3582 return page; 3583 } 3584 3585 /* 3586 * It's bad if compaction run occurs and fails. The most likely reason 3587 * is that pages exist, but not enough to satisfy watermarks. 3588 */ 3589 count_vm_event(COMPACTFAIL); 3590 3591 cond_resched(); 3592 3593 return NULL; 3594 } 3595 3596 static inline bool 3597 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 3598 enum compact_result compact_result, 3599 enum compact_priority *compact_priority, 3600 int *compaction_retries) 3601 { 3602 int max_retries = MAX_COMPACT_RETRIES; 3603 int min_priority; 3604 bool ret = false; 3605 int retries = *compaction_retries; 3606 enum compact_priority priority = *compact_priority; 3607 3608 if (!order) 3609 return false; 3610 3611 if (compaction_made_progress(compact_result)) 3612 (*compaction_retries)++; 3613 3614 /* 3615 * compaction considers all the zone as desperately out of memory 3616 * so it doesn't really make much sense to retry except when the 3617 * failure could be caused by insufficient priority 3618 */ 3619 if (compaction_failed(compact_result)) 3620 goto check_priority; 3621 3622 /* 3623 * make sure the compaction wasn't deferred or didn't bail out early 3624 * due to locks contention before we declare that we should give up. 3625 * But do not retry if the given zonelist is not suitable for 3626 * compaction. 3627 */ 3628 if (compaction_withdrawn(compact_result)) { 3629 ret = compaction_zonelist_suitable(ac, order, alloc_flags); 3630 goto out; 3631 } 3632 3633 /* 3634 * !costly requests are much more important than __GFP_RETRY_MAYFAIL 3635 * costly ones because they are de facto nofail and invoke OOM 3636 * killer to move on while costly can fail and users are ready 3637 * to cope with that. 1/4 retries is rather arbitrary but we 3638 * would need much more detailed feedback from compaction to 3639 * make a better decision. 3640 */ 3641 if (order > PAGE_ALLOC_COSTLY_ORDER) 3642 max_retries /= 4; 3643 if (*compaction_retries <= max_retries) { 3644 ret = true; 3645 goto out; 3646 } 3647 3648 /* 3649 * Make sure there are attempts at the highest priority if we exhausted 3650 * all retries or failed at the lower priorities. 3651 */ 3652 check_priority: 3653 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 3654 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 3655 3656 if (*compact_priority > min_priority) { 3657 (*compact_priority)--; 3658 *compaction_retries = 0; 3659 ret = true; 3660 } 3661 out: 3662 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret); 3663 return ret; 3664 } 3665 #else 3666 static inline struct page * 3667 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3668 unsigned int alloc_flags, const struct alloc_context *ac, 3669 enum compact_priority prio, enum compact_result *compact_result) 3670 { 3671 *compact_result = COMPACT_SKIPPED; 3672 return NULL; 3673 } 3674 3675 static inline bool 3676 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 3677 enum compact_result compact_result, 3678 enum compact_priority *compact_priority, 3679 int *compaction_retries) 3680 { 3681 struct zone *zone; 3682 struct zoneref *z; 3683 3684 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 3685 return false; 3686 3687 /* 3688 * There are setups with compaction disabled which would prefer to loop 3689 * inside the allocator rather than hit the oom killer prematurely. 3690 * Let's give them a good hope and keep retrying while the order-0 3691 * watermarks are OK. 3692 */ 3693 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3694 ac->nodemask) { 3695 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 3696 ac_classzone_idx(ac), alloc_flags)) 3697 return true; 3698 } 3699 return false; 3700 } 3701 #endif /* CONFIG_COMPACTION */ 3702 3703 #ifdef CONFIG_LOCKDEP 3704 static struct lockdep_map __fs_reclaim_map = 3705 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map); 3706 3707 static bool __need_fs_reclaim(gfp_t gfp_mask) 3708 { 3709 gfp_mask = current_gfp_context(gfp_mask); 3710 3711 /* no reclaim without waiting on it */ 3712 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) 3713 return false; 3714 3715 /* this guy won't enter reclaim */ 3716 if (current->flags & PF_MEMALLOC) 3717 return false; 3718 3719 /* We're only interested __GFP_FS allocations for now */ 3720 if (!(gfp_mask & __GFP_FS)) 3721 return false; 3722 3723 if (gfp_mask & __GFP_NOLOCKDEP) 3724 return false; 3725 3726 return true; 3727 } 3728 3729 void __fs_reclaim_acquire(void) 3730 { 3731 lock_map_acquire(&__fs_reclaim_map); 3732 } 3733 3734 void __fs_reclaim_release(void) 3735 { 3736 lock_map_release(&__fs_reclaim_map); 3737 } 3738 3739 void fs_reclaim_acquire(gfp_t gfp_mask) 3740 { 3741 if (__need_fs_reclaim(gfp_mask)) 3742 __fs_reclaim_acquire(); 3743 } 3744 EXPORT_SYMBOL_GPL(fs_reclaim_acquire); 3745 3746 void fs_reclaim_release(gfp_t gfp_mask) 3747 { 3748 if (__need_fs_reclaim(gfp_mask)) 3749 __fs_reclaim_release(); 3750 } 3751 EXPORT_SYMBOL_GPL(fs_reclaim_release); 3752 #endif 3753 3754 /* Perform direct synchronous page reclaim */ 3755 static int 3756 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 3757 const struct alloc_context *ac) 3758 { 3759 struct reclaim_state reclaim_state; 3760 int progress; 3761 unsigned int noreclaim_flag; 3762 unsigned long pflags; 3763 3764 cond_resched(); 3765 3766 /* We now go into synchronous reclaim */ 3767 cpuset_memory_pressure_bump(); 3768 psi_memstall_enter(&pflags); 3769 fs_reclaim_acquire(gfp_mask); 3770 noreclaim_flag = memalloc_noreclaim_save(); 3771 reclaim_state.reclaimed_slab = 0; 3772 current->reclaim_state = &reclaim_state; 3773 3774 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 3775 ac->nodemask); 3776 3777 current->reclaim_state = NULL; 3778 memalloc_noreclaim_restore(noreclaim_flag); 3779 fs_reclaim_release(gfp_mask); 3780 psi_memstall_leave(&pflags); 3781 3782 cond_resched(); 3783 3784 return progress; 3785 } 3786 3787 /* The really slow allocator path where we enter direct reclaim */ 3788 static inline struct page * 3789 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 3790 unsigned int alloc_flags, const struct alloc_context *ac, 3791 unsigned long *did_some_progress) 3792 { 3793 struct page *page = NULL; 3794 bool drained = false; 3795 3796 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 3797 if (unlikely(!(*did_some_progress))) 3798 return NULL; 3799 3800 retry: 3801 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 3802 3803 /* 3804 * If an allocation failed after direct reclaim, it could be because 3805 * pages are pinned on the per-cpu lists or in high alloc reserves. 3806 * Shrink them them and try again 3807 */ 3808 if (!page && !drained) { 3809 unreserve_highatomic_pageblock(ac, false); 3810 drain_all_pages(NULL); 3811 drained = true; 3812 goto retry; 3813 } 3814 3815 return page; 3816 } 3817 3818 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, 3819 const struct alloc_context *ac) 3820 { 3821 struct zoneref *z; 3822 struct zone *zone; 3823 pg_data_t *last_pgdat = NULL; 3824 enum zone_type high_zoneidx = ac->high_zoneidx; 3825 3826 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, high_zoneidx, 3827 ac->nodemask) { 3828 if (last_pgdat != zone->zone_pgdat) 3829 wakeup_kswapd(zone, gfp_mask, order, high_zoneidx); 3830 last_pgdat = zone->zone_pgdat; 3831 } 3832 } 3833 3834 static inline unsigned int 3835 gfp_to_alloc_flags(gfp_t gfp_mask) 3836 { 3837 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 3838 3839 /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */ 3840 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 3841 3842 /* 3843 * The caller may dip into page reserves a bit more if the caller 3844 * cannot run direct reclaim, or if the caller has realtime scheduling 3845 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 3846 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 3847 */ 3848 alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); 3849 3850 if (gfp_mask & __GFP_ATOMIC) { 3851 /* 3852 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 3853 * if it can't schedule. 3854 */ 3855 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3856 alloc_flags |= ALLOC_HARDER; 3857 /* 3858 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 3859 * comment for __cpuset_node_allowed(). 3860 */ 3861 alloc_flags &= ~ALLOC_CPUSET; 3862 } else if (unlikely(rt_task(current)) && !in_interrupt()) 3863 alloc_flags |= ALLOC_HARDER; 3864 3865 #ifdef CONFIG_CMA 3866 if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) 3867 alloc_flags |= ALLOC_CMA; 3868 #endif 3869 return alloc_flags; 3870 } 3871 3872 static bool oom_reserves_allowed(struct task_struct *tsk) 3873 { 3874 if (!tsk_is_oom_victim(tsk)) 3875 return false; 3876 3877 /* 3878 * !MMU doesn't have oom reaper so give access to memory reserves 3879 * only to the thread with TIF_MEMDIE set 3880 */ 3881 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE)) 3882 return false; 3883 3884 return true; 3885 } 3886 3887 /* 3888 * Distinguish requests which really need access to full memory 3889 * reserves from oom victims which can live with a portion of it 3890 */ 3891 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) 3892 { 3893 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 3894 return 0; 3895 if (gfp_mask & __GFP_MEMALLOC) 3896 return ALLOC_NO_WATERMARKS; 3897 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 3898 return ALLOC_NO_WATERMARKS; 3899 if (!in_interrupt()) { 3900 if (current->flags & PF_MEMALLOC) 3901 return ALLOC_NO_WATERMARKS; 3902 else if (oom_reserves_allowed(current)) 3903 return ALLOC_OOM; 3904 } 3905 3906 return 0; 3907 } 3908 3909 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 3910 { 3911 return !!__gfp_pfmemalloc_flags(gfp_mask); 3912 } 3913 3914 /* 3915 * Checks whether it makes sense to retry the reclaim to make a forward progress 3916 * for the given allocation request. 3917 * 3918 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row 3919 * without success, or when we couldn't even meet the watermark if we 3920 * reclaimed all remaining pages on the LRU lists. 3921 * 3922 * Returns true if a retry is viable or false to enter the oom path. 3923 */ 3924 static inline bool 3925 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 3926 struct alloc_context *ac, int alloc_flags, 3927 bool did_some_progress, int *no_progress_loops) 3928 { 3929 struct zone *zone; 3930 struct zoneref *z; 3931 bool ret = false; 3932 3933 /* 3934 * Costly allocations might have made a progress but this doesn't mean 3935 * their order will become available due to high fragmentation so 3936 * always increment the no progress counter for them 3937 */ 3938 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 3939 *no_progress_loops = 0; 3940 else 3941 (*no_progress_loops)++; 3942 3943 /* 3944 * Make sure we converge to OOM if we cannot make any progress 3945 * several times in the row. 3946 */ 3947 if (*no_progress_loops > MAX_RECLAIM_RETRIES) { 3948 /* Before OOM, exhaust highatomic_reserve */ 3949 return unreserve_highatomic_pageblock(ac, true); 3950 } 3951 3952 /* 3953 * Keep reclaiming pages while there is a chance this will lead 3954 * somewhere. If none of the target zones can satisfy our allocation 3955 * request even if all reclaimable pages are considered then we are 3956 * screwed and have to go OOM. 3957 */ 3958 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3959 ac->nodemask) { 3960 unsigned long available; 3961 unsigned long reclaimable; 3962 unsigned long min_wmark = min_wmark_pages(zone); 3963 bool wmark; 3964 3965 available = reclaimable = zone_reclaimable_pages(zone); 3966 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 3967 3968 /* 3969 * Would the allocation succeed if we reclaimed all 3970 * reclaimable pages? 3971 */ 3972 wmark = __zone_watermark_ok(zone, order, min_wmark, 3973 ac_classzone_idx(ac), alloc_flags, available); 3974 trace_reclaim_retry_zone(z, order, reclaimable, 3975 available, min_wmark, *no_progress_loops, wmark); 3976 if (wmark) { 3977 /* 3978 * If we didn't make any progress and have a lot of 3979 * dirty + writeback pages then we should wait for 3980 * an IO to complete to slow down the reclaim and 3981 * prevent from pre mature OOM 3982 */ 3983 if (!did_some_progress) { 3984 unsigned long write_pending; 3985 3986 write_pending = zone_page_state_snapshot(zone, 3987 NR_ZONE_WRITE_PENDING); 3988 3989 if (2 * write_pending > reclaimable) { 3990 congestion_wait(BLK_RW_ASYNC, HZ/10); 3991 return true; 3992 } 3993 } 3994 3995 ret = true; 3996 goto out; 3997 } 3998 } 3999 4000 out: 4001 /* 4002 * Memory allocation/reclaim might be called from a WQ context and the 4003 * current implementation of the WQ concurrency control doesn't 4004 * recognize that a particular WQ is congested if the worker thread is 4005 * looping without ever sleeping. Therefore we have to do a short sleep 4006 * here rather than calling cond_resched(). 4007 */ 4008 if (current->flags & PF_WQ_WORKER) 4009 schedule_timeout_uninterruptible(1); 4010 else 4011 cond_resched(); 4012 return ret; 4013 } 4014 4015 static inline bool 4016 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) 4017 { 4018 /* 4019 * It's possible that cpuset's mems_allowed and the nodemask from 4020 * mempolicy don't intersect. This should be normally dealt with by 4021 * policy_nodemask(), but it's possible to race with cpuset update in 4022 * such a way the check therein was true, and then it became false 4023 * before we got our cpuset_mems_cookie here. 4024 * This assumes that for all allocations, ac->nodemask can come only 4025 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored 4026 * when it does not intersect with the cpuset restrictions) or the 4027 * caller can deal with a violated nodemask. 4028 */ 4029 if (cpusets_enabled() && ac->nodemask && 4030 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) { 4031 ac->nodemask = NULL; 4032 return true; 4033 } 4034 4035 /* 4036 * When updating a task's mems_allowed or mempolicy nodemask, it is 4037 * possible to race with parallel threads in such a way that our 4038 * allocation can fail while the mask is being updated. If we are about 4039 * to fail, check if the cpuset changed during allocation and if so, 4040 * retry. 4041 */ 4042 if (read_mems_allowed_retry(cpuset_mems_cookie)) 4043 return true; 4044 4045 return false; 4046 } 4047 4048 static inline struct page * 4049 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 4050 struct alloc_context *ac) 4051 { 4052 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 4053 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 4054 struct page *page = NULL; 4055 unsigned int alloc_flags; 4056 unsigned long did_some_progress; 4057 enum compact_priority compact_priority; 4058 enum compact_result compact_result; 4059 int compaction_retries; 4060 int no_progress_loops; 4061 unsigned int cpuset_mems_cookie; 4062 int reserve_flags; 4063 4064 /* 4065 * We also sanity check to catch abuse of atomic reserves being used by 4066 * callers that are not in atomic context. 4067 */ 4068 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 4069 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 4070 gfp_mask &= ~__GFP_ATOMIC; 4071 4072 retry_cpuset: 4073 compaction_retries = 0; 4074 no_progress_loops = 0; 4075 compact_priority = DEF_COMPACT_PRIORITY; 4076 cpuset_mems_cookie = read_mems_allowed_begin(); 4077 4078 /* 4079 * The fast path uses conservative alloc_flags to succeed only until 4080 * kswapd needs to be woken up, and to avoid the cost of setting up 4081 * alloc_flags precisely. So we do that now. 4082 */ 4083 alloc_flags = gfp_to_alloc_flags(gfp_mask); 4084 4085 /* 4086 * We need to recalculate the starting point for the zonelist iterator 4087 * because we might have used different nodemask in the fast path, or 4088 * there was a cpuset modification and we are retrying - otherwise we 4089 * could end up iterating over non-eligible zones endlessly. 4090 */ 4091 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4092 ac->high_zoneidx, ac->nodemask); 4093 if (!ac->preferred_zoneref->zone) 4094 goto nopage; 4095 4096 if (gfp_mask & __GFP_KSWAPD_RECLAIM) 4097 wake_all_kswapds(order, gfp_mask, ac); 4098 4099 /* 4100 * The adjusted alloc_flags might result in immediate success, so try 4101 * that first 4102 */ 4103 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4104 if (page) 4105 goto got_pg; 4106 4107 /* 4108 * For costly allocations, try direct compaction first, as it's likely 4109 * that we have enough base pages and don't need to reclaim. For non- 4110 * movable high-order allocations, do that as well, as compaction will 4111 * try prevent permanent fragmentation by migrating from blocks of the 4112 * same migratetype. 4113 * Don't try this for allocations that are allowed to ignore 4114 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 4115 */ 4116 if (can_direct_reclaim && 4117 (costly_order || 4118 (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 4119 && !gfp_pfmemalloc_allowed(gfp_mask)) { 4120 page = __alloc_pages_direct_compact(gfp_mask, order, 4121 alloc_flags, ac, 4122 INIT_COMPACT_PRIORITY, 4123 &compact_result); 4124 if (page) 4125 goto got_pg; 4126 4127 /* 4128 * Checks for costly allocations with __GFP_NORETRY, which 4129 * includes THP page fault allocations 4130 */ 4131 if (costly_order && (gfp_mask & __GFP_NORETRY)) { 4132 /* 4133 * If compaction is deferred for high-order allocations, 4134 * it is because sync compaction recently failed. If 4135 * this is the case and the caller requested a THP 4136 * allocation, we do not want to heavily disrupt the 4137 * system, so we fail the allocation instead of entering 4138 * direct reclaim. 4139 */ 4140 if (compact_result == COMPACT_DEFERRED) 4141 goto nopage; 4142 4143 /* 4144 * Looks like reclaim/compaction is worth trying, but 4145 * sync compaction could be very expensive, so keep 4146 * using async compaction. 4147 */ 4148 compact_priority = INIT_COMPACT_PRIORITY; 4149 } 4150 } 4151 4152 retry: 4153 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 4154 if (gfp_mask & __GFP_KSWAPD_RECLAIM) 4155 wake_all_kswapds(order, gfp_mask, ac); 4156 4157 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); 4158 if (reserve_flags) 4159 alloc_flags = reserve_flags; 4160 4161 /* 4162 * Reset the nodemask and zonelist iterators if memory policies can be 4163 * ignored. These allocations are high priority and system rather than 4164 * user oriented. 4165 */ 4166 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { 4167 ac->nodemask = NULL; 4168 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4169 ac->high_zoneidx, ac->nodemask); 4170 } 4171 4172 /* Attempt with potentially adjusted zonelist and alloc_flags */ 4173 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4174 if (page) 4175 goto got_pg; 4176 4177 /* Caller is not willing to reclaim, we can't balance anything */ 4178 if (!can_direct_reclaim) 4179 goto nopage; 4180 4181 /* Avoid recursion of direct reclaim */ 4182 if (current->flags & PF_MEMALLOC) 4183 goto nopage; 4184 4185 /* Try direct reclaim and then allocating */ 4186 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 4187 &did_some_progress); 4188 if (page) 4189 goto got_pg; 4190 4191 /* Try direct compaction and then allocating */ 4192 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 4193 compact_priority, &compact_result); 4194 if (page) 4195 goto got_pg; 4196 4197 /* Do not loop if specifically requested */ 4198 if (gfp_mask & __GFP_NORETRY) 4199 goto nopage; 4200 4201 /* 4202 * Do not retry costly high order allocations unless they are 4203 * __GFP_RETRY_MAYFAIL 4204 */ 4205 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) 4206 goto nopage; 4207 4208 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 4209 did_some_progress > 0, &no_progress_loops)) 4210 goto retry; 4211 4212 /* 4213 * It doesn't make any sense to retry for the compaction if the order-0 4214 * reclaim is not able to make any progress because the current 4215 * implementation of the compaction depends on the sufficient amount 4216 * of free memory (see __compaction_suitable) 4217 */ 4218 if (did_some_progress > 0 && 4219 should_compact_retry(ac, order, alloc_flags, 4220 compact_result, &compact_priority, 4221 &compaction_retries)) 4222 goto retry; 4223 4224 4225 /* Deal with possible cpuset update races before we start OOM killing */ 4226 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4227 goto retry_cpuset; 4228 4229 /* Reclaim has failed us, start killing things */ 4230 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 4231 if (page) 4232 goto got_pg; 4233 4234 /* Avoid allocations with no watermarks from looping endlessly */ 4235 if (tsk_is_oom_victim(current) && 4236 (alloc_flags == ALLOC_OOM || 4237 (gfp_mask & __GFP_NOMEMALLOC))) 4238 goto nopage; 4239 4240 /* Retry as long as the OOM killer is making progress */ 4241 if (did_some_progress) { 4242 no_progress_loops = 0; 4243 goto retry; 4244 } 4245 4246 nopage: 4247 /* Deal with possible cpuset update races before we fail */ 4248 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 4249 goto retry_cpuset; 4250 4251 /* 4252 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure 4253 * we always retry 4254 */ 4255 if (gfp_mask & __GFP_NOFAIL) { 4256 /* 4257 * All existing users of the __GFP_NOFAIL are blockable, so warn 4258 * of any new users that actually require GFP_NOWAIT 4259 */ 4260 if (WARN_ON_ONCE(!can_direct_reclaim)) 4261 goto fail; 4262 4263 /* 4264 * PF_MEMALLOC request from this context is rather bizarre 4265 * because we cannot reclaim anything and only can loop waiting 4266 * for somebody to do a work for us 4267 */ 4268 WARN_ON_ONCE(current->flags & PF_MEMALLOC); 4269 4270 /* 4271 * non failing costly orders are a hard requirement which we 4272 * are not prepared for much so let's warn about these users 4273 * so that we can identify them and convert them to something 4274 * else. 4275 */ 4276 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER); 4277 4278 /* 4279 * Help non-failing allocations by giving them access to memory 4280 * reserves but do not use ALLOC_NO_WATERMARKS because this 4281 * could deplete whole memory reserves which would just make 4282 * the situation worse 4283 */ 4284 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); 4285 if (page) 4286 goto got_pg; 4287 4288 cond_resched(); 4289 goto retry; 4290 } 4291 fail: 4292 warn_alloc(gfp_mask, ac->nodemask, 4293 "page allocation failure: order:%u", order); 4294 got_pg: 4295 return page; 4296 } 4297 4298 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, 4299 int preferred_nid, nodemask_t *nodemask, 4300 struct alloc_context *ac, gfp_t *alloc_mask, 4301 unsigned int *alloc_flags) 4302 { 4303 ac->high_zoneidx = gfp_zone(gfp_mask); 4304 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); 4305 ac->nodemask = nodemask; 4306 ac->migratetype = gfpflags_to_migratetype(gfp_mask); 4307 4308 if (cpusets_enabled()) { 4309 *alloc_mask |= __GFP_HARDWALL; 4310 if (!ac->nodemask) 4311 ac->nodemask = &cpuset_current_mems_allowed; 4312 else 4313 *alloc_flags |= ALLOC_CPUSET; 4314 } 4315 4316 fs_reclaim_acquire(gfp_mask); 4317 fs_reclaim_release(gfp_mask); 4318 4319 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 4320 4321 if (should_fail_alloc_page(gfp_mask, order)) 4322 return false; 4323 4324 if (IS_ENABLED(CONFIG_CMA) && ac->migratetype == MIGRATE_MOVABLE) 4325 *alloc_flags |= ALLOC_CMA; 4326 4327 return true; 4328 } 4329 4330 /* Determine whether to spread dirty pages and what the first usable zone */ 4331 static inline void finalise_ac(gfp_t gfp_mask, struct alloc_context *ac) 4332 { 4333 /* Dirty zone balancing only done in the fast path */ 4334 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); 4335 4336 /* 4337 * The preferred zone is used for statistics but crucially it is 4338 * also used as the starting point for the zonelist iterator. It 4339 * may get reset for allocations that ignore memory policies. 4340 */ 4341 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4342 ac->high_zoneidx, ac->nodemask); 4343 } 4344 4345 /* 4346 * This is the 'heart' of the zoned buddy allocator. 4347 */ 4348 struct page * 4349 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, 4350 nodemask_t *nodemask) 4351 { 4352 struct page *page; 4353 unsigned int alloc_flags = ALLOC_WMARK_LOW; 4354 gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */ 4355 struct alloc_context ac = { }; 4356 4357 /* 4358 * There are several places where we assume that the order value is sane 4359 * so bail out early if the request is out of bound. 4360 */ 4361 if (unlikely(order >= MAX_ORDER)) { 4362 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); 4363 return NULL; 4364 } 4365 4366 gfp_mask &= gfp_allowed_mask; 4367 alloc_mask = gfp_mask; 4368 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags)) 4369 return NULL; 4370 4371 finalise_ac(gfp_mask, &ac); 4372 4373 /* First allocation attempt */ 4374 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); 4375 if (likely(page)) 4376 goto out; 4377 4378 /* 4379 * Apply scoped allocation constraints. This is mainly about GFP_NOFS 4380 * resp. GFP_NOIO which has to be inherited for all allocation requests 4381 * from a particular context which has been marked by 4382 * memalloc_no{fs,io}_{save,restore}. 4383 */ 4384 alloc_mask = current_gfp_context(gfp_mask); 4385 ac.spread_dirty_pages = false; 4386 4387 /* 4388 * Restore the original nodemask if it was potentially replaced with 4389 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 4390 */ 4391 if (unlikely(ac.nodemask != nodemask)) 4392 ac.nodemask = nodemask; 4393 4394 page = __alloc_pages_slowpath(alloc_mask, order, &ac); 4395 4396 out: 4397 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && 4398 unlikely(memcg_kmem_charge(page, gfp_mask, order) != 0)) { 4399 __free_pages(page, order); 4400 page = NULL; 4401 } 4402 4403 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); 4404 4405 return page; 4406 } 4407 EXPORT_SYMBOL(__alloc_pages_nodemask); 4408 4409 /* 4410 * Common helper functions. Never use with __GFP_HIGHMEM because the returned 4411 * address cannot represent highmem pages. Use alloc_pages and then kmap if 4412 * you need to access high mem. 4413 */ 4414 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 4415 { 4416 struct page *page; 4417 4418 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); 4419 if (!page) 4420 return 0; 4421 return (unsigned long) page_address(page); 4422 } 4423 EXPORT_SYMBOL(__get_free_pages); 4424 4425 unsigned long get_zeroed_page(gfp_t gfp_mask) 4426 { 4427 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 4428 } 4429 EXPORT_SYMBOL(get_zeroed_page); 4430 4431 void __free_pages(struct page *page, unsigned int order) 4432 { 4433 if (put_page_testzero(page)) { 4434 if (order == 0) 4435 free_unref_page(page); 4436 else 4437 __free_pages_ok(page, order); 4438 } 4439 } 4440 4441 EXPORT_SYMBOL(__free_pages); 4442 4443 void free_pages(unsigned long addr, unsigned int order) 4444 { 4445 if (addr != 0) { 4446 VM_BUG_ON(!virt_addr_valid((void *)addr)); 4447 __free_pages(virt_to_page((void *)addr), order); 4448 } 4449 } 4450 4451 EXPORT_SYMBOL(free_pages); 4452 4453 /* 4454 * Page Fragment: 4455 * An arbitrary-length arbitrary-offset area of memory which resides 4456 * within a 0 or higher order page. Multiple fragments within that page 4457 * are individually refcounted, in the page's reference counter. 4458 * 4459 * The page_frag functions below provide a simple allocation framework for 4460 * page fragments. This is used by the network stack and network device 4461 * drivers to provide a backing region of memory for use as either an 4462 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 4463 */ 4464 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 4465 gfp_t gfp_mask) 4466 { 4467 struct page *page = NULL; 4468 gfp_t gfp = gfp_mask; 4469 4470 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4471 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 4472 __GFP_NOMEMALLOC; 4473 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 4474 PAGE_FRAG_CACHE_MAX_ORDER); 4475 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 4476 #endif 4477 if (unlikely(!page)) 4478 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 4479 4480 nc->va = page ? page_address(page) : NULL; 4481 4482 return page; 4483 } 4484 4485 void __page_frag_cache_drain(struct page *page, unsigned int count) 4486 { 4487 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 4488 4489 if (page_ref_sub_and_test(page, count)) { 4490 unsigned int order = compound_order(page); 4491 4492 if (order == 0) 4493 free_unref_page(page); 4494 else 4495 __free_pages_ok(page, order); 4496 } 4497 } 4498 EXPORT_SYMBOL(__page_frag_cache_drain); 4499 4500 void *page_frag_alloc(struct page_frag_cache *nc, 4501 unsigned int fragsz, gfp_t gfp_mask) 4502 { 4503 unsigned int size = PAGE_SIZE; 4504 struct page *page; 4505 int offset; 4506 4507 if (unlikely(!nc->va)) { 4508 refill: 4509 page = __page_frag_cache_refill(nc, gfp_mask); 4510 if (!page) 4511 return NULL; 4512 4513 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4514 /* if size can vary use size else just use PAGE_SIZE */ 4515 size = nc->size; 4516 #endif 4517 /* Even if we own the page, we do not use atomic_set(). 4518 * This would break get_page_unless_zero() users. 4519 */ 4520 page_ref_add(page, size - 1); 4521 4522 /* reset page count bias and offset to start of new frag */ 4523 nc->pfmemalloc = page_is_pfmemalloc(page); 4524 nc->pagecnt_bias = size; 4525 nc->offset = size; 4526 } 4527 4528 offset = nc->offset - fragsz; 4529 if (unlikely(offset < 0)) { 4530 page = virt_to_page(nc->va); 4531 4532 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 4533 goto refill; 4534 4535 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 4536 /* if size can vary use size else just use PAGE_SIZE */ 4537 size = nc->size; 4538 #endif 4539 /* OK, page count is 0, we can safely set it */ 4540 set_page_count(page, size); 4541 4542 /* reset page count bias and offset to start of new frag */ 4543 nc->pagecnt_bias = size; 4544 offset = size - fragsz; 4545 } 4546 4547 nc->pagecnt_bias--; 4548 nc->offset = offset; 4549 4550 return nc->va + offset; 4551 } 4552 EXPORT_SYMBOL(page_frag_alloc); 4553 4554 /* 4555 * Frees a page fragment allocated out of either a compound or order 0 page. 4556 */ 4557 void page_frag_free(void *addr) 4558 { 4559 struct page *page = virt_to_head_page(addr); 4560 4561 if (unlikely(put_page_testzero(page))) 4562 __free_pages_ok(page, compound_order(page)); 4563 } 4564 EXPORT_SYMBOL(page_frag_free); 4565 4566 static void *make_alloc_exact(unsigned long addr, unsigned int order, 4567 size_t size) 4568 { 4569 if (addr) { 4570 unsigned long alloc_end = addr + (PAGE_SIZE << order); 4571 unsigned long used = addr + PAGE_ALIGN(size); 4572 4573 split_page(virt_to_page((void *)addr), order); 4574 while (used < alloc_end) { 4575 free_page(used); 4576 used += PAGE_SIZE; 4577 } 4578 } 4579 return (void *)addr; 4580 } 4581 4582 /** 4583 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 4584 * @size: the number of bytes to allocate 4585 * @gfp_mask: GFP flags for the allocation 4586 * 4587 * This function is similar to alloc_pages(), except that it allocates the 4588 * minimum number of pages to satisfy the request. alloc_pages() can only 4589 * allocate memory in power-of-two pages. 4590 * 4591 * This function is also limited by MAX_ORDER. 4592 * 4593 * Memory allocated by this function must be released by free_pages_exact(). 4594 */ 4595 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 4596 { 4597 unsigned int order = get_order(size); 4598 unsigned long addr; 4599 4600 addr = __get_free_pages(gfp_mask, order); 4601 return make_alloc_exact(addr, order, size); 4602 } 4603 EXPORT_SYMBOL(alloc_pages_exact); 4604 4605 /** 4606 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 4607 * pages on a node. 4608 * @nid: the preferred node ID where memory should be allocated 4609 * @size: the number of bytes to allocate 4610 * @gfp_mask: GFP flags for the allocation 4611 * 4612 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 4613 * back. 4614 */ 4615 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 4616 { 4617 unsigned int order = get_order(size); 4618 struct page *p = alloc_pages_node(nid, gfp_mask, order); 4619 if (!p) 4620 return NULL; 4621 return make_alloc_exact((unsigned long)page_address(p), order, size); 4622 } 4623 4624 /** 4625 * free_pages_exact - release memory allocated via alloc_pages_exact() 4626 * @virt: the value returned by alloc_pages_exact. 4627 * @size: size of allocation, same value as passed to alloc_pages_exact(). 4628 * 4629 * Release the memory allocated by a previous call to alloc_pages_exact. 4630 */ 4631 void free_pages_exact(void *virt, size_t size) 4632 { 4633 unsigned long addr = (unsigned long)virt; 4634 unsigned long end = addr + PAGE_ALIGN(size); 4635 4636 while (addr < end) { 4637 free_page(addr); 4638 addr += PAGE_SIZE; 4639 } 4640 } 4641 EXPORT_SYMBOL(free_pages_exact); 4642 4643 /** 4644 * nr_free_zone_pages - count number of pages beyond high watermark 4645 * @offset: The zone index of the highest zone 4646 * 4647 * nr_free_zone_pages() counts the number of counts pages which are beyond the 4648 * high watermark within all zones at or below a given zone index. For each 4649 * zone, the number of pages is calculated as: 4650 * 4651 * nr_free_zone_pages = managed_pages - high_pages 4652 */ 4653 static unsigned long nr_free_zone_pages(int offset) 4654 { 4655 struct zoneref *z; 4656 struct zone *zone; 4657 4658 /* Just pick one node, since fallback list is circular */ 4659 unsigned long sum = 0; 4660 4661 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 4662 4663 for_each_zone_zonelist(zone, z, zonelist, offset) { 4664 unsigned long size = zone->managed_pages; 4665 unsigned long high = high_wmark_pages(zone); 4666 if (size > high) 4667 sum += size - high; 4668 } 4669 4670 return sum; 4671 } 4672 4673 /** 4674 * nr_free_buffer_pages - count number of pages beyond high watermark 4675 * 4676 * nr_free_buffer_pages() counts the number of pages which are beyond the high 4677 * watermark within ZONE_DMA and ZONE_NORMAL. 4678 */ 4679 unsigned long nr_free_buffer_pages(void) 4680 { 4681 return nr_free_zone_pages(gfp_zone(GFP_USER)); 4682 } 4683 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 4684 4685 /** 4686 * nr_free_pagecache_pages - count number of pages beyond high watermark 4687 * 4688 * nr_free_pagecache_pages() counts the number of pages which are beyond the 4689 * high watermark within all zones. 4690 */ 4691 unsigned long nr_free_pagecache_pages(void) 4692 { 4693 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 4694 } 4695 4696 static inline void show_node(struct zone *zone) 4697 { 4698 if (IS_ENABLED(CONFIG_NUMA)) 4699 printk("Node %d ", zone_to_nid(zone)); 4700 } 4701 4702 long si_mem_available(void) 4703 { 4704 long available; 4705 unsigned long pagecache; 4706 unsigned long wmark_low = 0; 4707 unsigned long pages[NR_LRU_LISTS]; 4708 unsigned long reclaimable; 4709 struct zone *zone; 4710 int lru; 4711 4712 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 4713 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 4714 4715 for_each_zone(zone) 4716 wmark_low += zone->watermark[WMARK_LOW]; 4717 4718 /* 4719 * Estimate the amount of memory available for userspace allocations, 4720 * without causing swapping. 4721 */ 4722 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages; 4723 4724 /* 4725 * Not all the page cache can be freed, otherwise the system will 4726 * start swapping. Assume at least half of the page cache, or the 4727 * low watermark worth of cache, needs to stay. 4728 */ 4729 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 4730 pagecache -= min(pagecache / 2, wmark_low); 4731 available += pagecache; 4732 4733 /* 4734 * Part of the reclaimable slab and other kernel memory consists of 4735 * items that are in use, and cannot be freed. Cap this estimate at the 4736 * low watermark. 4737 */ 4738 reclaimable = global_node_page_state(NR_SLAB_RECLAIMABLE) + 4739 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE); 4740 available += reclaimable - min(reclaimable / 2, wmark_low); 4741 4742 if (available < 0) 4743 available = 0; 4744 return available; 4745 } 4746 EXPORT_SYMBOL_GPL(si_mem_available); 4747 4748 void si_meminfo(struct sysinfo *val) 4749 { 4750 val->totalram = totalram_pages; 4751 val->sharedram = global_node_page_state(NR_SHMEM); 4752 val->freeram = global_zone_page_state(NR_FREE_PAGES); 4753 val->bufferram = nr_blockdev_pages(); 4754 val->totalhigh = totalhigh_pages; 4755 val->freehigh = nr_free_highpages(); 4756 val->mem_unit = PAGE_SIZE; 4757 } 4758 4759 EXPORT_SYMBOL(si_meminfo); 4760 4761 #ifdef CONFIG_NUMA 4762 void si_meminfo_node(struct sysinfo *val, int nid) 4763 { 4764 int zone_type; /* needs to be signed */ 4765 unsigned long managed_pages = 0; 4766 unsigned long managed_highpages = 0; 4767 unsigned long free_highpages = 0; 4768 pg_data_t *pgdat = NODE_DATA(nid); 4769 4770 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 4771 managed_pages += pgdat->node_zones[zone_type].managed_pages; 4772 val->totalram = managed_pages; 4773 val->sharedram = node_page_state(pgdat, NR_SHMEM); 4774 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 4775 #ifdef CONFIG_HIGHMEM 4776 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 4777 struct zone *zone = &pgdat->node_zones[zone_type]; 4778 4779 if (is_highmem(zone)) { 4780 managed_highpages += zone->managed_pages; 4781 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 4782 } 4783 } 4784 val->totalhigh = managed_highpages; 4785 val->freehigh = free_highpages; 4786 #else 4787 val->totalhigh = managed_highpages; 4788 val->freehigh = free_highpages; 4789 #endif 4790 val->mem_unit = PAGE_SIZE; 4791 } 4792 #endif 4793 4794 /* 4795 * Determine whether the node should be displayed or not, depending on whether 4796 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 4797 */ 4798 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) 4799 { 4800 if (!(flags & SHOW_MEM_FILTER_NODES)) 4801 return false; 4802 4803 /* 4804 * no node mask - aka implicit memory numa policy. Do not bother with 4805 * the synchronization - read_mems_allowed_begin - because we do not 4806 * have to be precise here. 4807 */ 4808 if (!nodemask) 4809 nodemask = &cpuset_current_mems_allowed; 4810 4811 return !node_isset(nid, *nodemask); 4812 } 4813 4814 #define K(x) ((x) << (PAGE_SHIFT-10)) 4815 4816 static void show_migration_types(unsigned char type) 4817 { 4818 static const char types[MIGRATE_TYPES] = { 4819 [MIGRATE_UNMOVABLE] = 'U', 4820 [MIGRATE_MOVABLE] = 'M', 4821 [MIGRATE_RECLAIMABLE] = 'E', 4822 [MIGRATE_HIGHATOMIC] = 'H', 4823 #ifdef CONFIG_CMA 4824 [MIGRATE_CMA] = 'C', 4825 #endif 4826 #ifdef CONFIG_MEMORY_ISOLATION 4827 [MIGRATE_ISOLATE] = 'I', 4828 #endif 4829 }; 4830 char tmp[MIGRATE_TYPES + 1]; 4831 char *p = tmp; 4832 int i; 4833 4834 for (i = 0; i < MIGRATE_TYPES; i++) { 4835 if (type & (1 << i)) 4836 *p++ = types[i]; 4837 } 4838 4839 *p = '\0'; 4840 printk(KERN_CONT "(%s) ", tmp); 4841 } 4842 4843 /* 4844 * Show free area list (used inside shift_scroll-lock stuff) 4845 * We also calculate the percentage fragmentation. We do this by counting the 4846 * memory on each free list with the exception of the first item on the list. 4847 * 4848 * Bits in @filter: 4849 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 4850 * cpuset. 4851 */ 4852 void show_free_areas(unsigned int filter, nodemask_t *nodemask) 4853 { 4854 unsigned long free_pcp = 0; 4855 int cpu; 4856 struct zone *zone; 4857 pg_data_t *pgdat; 4858 4859 for_each_populated_zone(zone) { 4860 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 4861 continue; 4862 4863 for_each_online_cpu(cpu) 4864 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 4865 } 4866 4867 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 4868 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 4869 " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n" 4870 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 4871 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 4872 " free:%lu free_pcp:%lu free_cma:%lu\n", 4873 global_node_page_state(NR_ACTIVE_ANON), 4874 global_node_page_state(NR_INACTIVE_ANON), 4875 global_node_page_state(NR_ISOLATED_ANON), 4876 global_node_page_state(NR_ACTIVE_FILE), 4877 global_node_page_state(NR_INACTIVE_FILE), 4878 global_node_page_state(NR_ISOLATED_FILE), 4879 global_node_page_state(NR_UNEVICTABLE), 4880 global_node_page_state(NR_FILE_DIRTY), 4881 global_node_page_state(NR_WRITEBACK), 4882 global_node_page_state(NR_UNSTABLE_NFS), 4883 global_node_page_state(NR_SLAB_RECLAIMABLE), 4884 global_node_page_state(NR_SLAB_UNRECLAIMABLE), 4885 global_node_page_state(NR_FILE_MAPPED), 4886 global_node_page_state(NR_SHMEM), 4887 global_zone_page_state(NR_PAGETABLE), 4888 global_zone_page_state(NR_BOUNCE), 4889 global_zone_page_state(NR_FREE_PAGES), 4890 free_pcp, 4891 global_zone_page_state(NR_FREE_CMA_PAGES)); 4892 4893 for_each_online_pgdat(pgdat) { 4894 if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) 4895 continue; 4896 4897 printk("Node %d" 4898 " active_anon:%lukB" 4899 " inactive_anon:%lukB" 4900 " active_file:%lukB" 4901 " inactive_file:%lukB" 4902 " unevictable:%lukB" 4903 " isolated(anon):%lukB" 4904 " isolated(file):%lukB" 4905 " mapped:%lukB" 4906 " dirty:%lukB" 4907 " writeback:%lukB" 4908 " shmem:%lukB" 4909 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4910 " shmem_thp: %lukB" 4911 " shmem_pmdmapped: %lukB" 4912 " anon_thp: %lukB" 4913 #endif 4914 " writeback_tmp:%lukB" 4915 " unstable:%lukB" 4916 " all_unreclaimable? %s" 4917 "\n", 4918 pgdat->node_id, 4919 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 4920 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 4921 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 4922 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 4923 K(node_page_state(pgdat, NR_UNEVICTABLE)), 4924 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 4925 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 4926 K(node_page_state(pgdat, NR_FILE_MAPPED)), 4927 K(node_page_state(pgdat, NR_FILE_DIRTY)), 4928 K(node_page_state(pgdat, NR_WRITEBACK)), 4929 K(node_page_state(pgdat, NR_SHMEM)), 4930 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4931 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), 4932 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) 4933 * HPAGE_PMD_NR), 4934 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR), 4935 #endif 4936 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 4937 K(node_page_state(pgdat, NR_UNSTABLE_NFS)), 4938 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ? 4939 "yes" : "no"); 4940 } 4941 4942 for_each_populated_zone(zone) { 4943 int i; 4944 4945 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 4946 continue; 4947 4948 free_pcp = 0; 4949 for_each_online_cpu(cpu) 4950 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 4951 4952 show_node(zone); 4953 printk(KERN_CONT 4954 "%s" 4955 " free:%lukB" 4956 " min:%lukB" 4957 " low:%lukB" 4958 " high:%lukB" 4959 " active_anon:%lukB" 4960 " inactive_anon:%lukB" 4961 " active_file:%lukB" 4962 " inactive_file:%lukB" 4963 " unevictable:%lukB" 4964 " writepending:%lukB" 4965 " present:%lukB" 4966 " managed:%lukB" 4967 " mlocked:%lukB" 4968 " kernel_stack:%lukB" 4969 " pagetables:%lukB" 4970 " bounce:%lukB" 4971 " free_pcp:%lukB" 4972 " local_pcp:%ukB" 4973 " free_cma:%lukB" 4974 "\n", 4975 zone->name, 4976 K(zone_page_state(zone, NR_FREE_PAGES)), 4977 K(min_wmark_pages(zone)), 4978 K(low_wmark_pages(zone)), 4979 K(high_wmark_pages(zone)), 4980 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 4981 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 4982 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 4983 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 4984 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 4985 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 4986 K(zone->present_pages), 4987 K(zone->managed_pages), 4988 K(zone_page_state(zone, NR_MLOCK)), 4989 zone_page_state(zone, NR_KERNEL_STACK_KB), 4990 K(zone_page_state(zone, NR_PAGETABLE)), 4991 K(zone_page_state(zone, NR_BOUNCE)), 4992 K(free_pcp), 4993 K(this_cpu_read(zone->pageset->pcp.count)), 4994 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 4995 printk("lowmem_reserve[]:"); 4996 for (i = 0; i < MAX_NR_ZONES; i++) 4997 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); 4998 printk(KERN_CONT "\n"); 4999 } 5000 5001 for_each_populated_zone(zone) { 5002 unsigned int order; 5003 unsigned long nr[MAX_ORDER], flags, total = 0; 5004 unsigned char types[MAX_ORDER]; 5005 5006 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5007 continue; 5008 show_node(zone); 5009 printk(KERN_CONT "%s: ", zone->name); 5010 5011 spin_lock_irqsave(&zone->lock, flags); 5012 for (order = 0; order < MAX_ORDER; order++) { 5013 struct free_area *area = &zone->free_area[order]; 5014 int type; 5015 5016 nr[order] = area->nr_free; 5017 total += nr[order] << order; 5018 5019 types[order] = 0; 5020 for (type = 0; type < MIGRATE_TYPES; type++) { 5021 if (!list_empty(&area->free_list[type])) 5022 types[order] |= 1 << type; 5023 } 5024 } 5025 spin_unlock_irqrestore(&zone->lock, flags); 5026 for (order = 0; order < MAX_ORDER; order++) { 5027 printk(KERN_CONT "%lu*%lukB ", 5028 nr[order], K(1UL) << order); 5029 if (nr[order]) 5030 show_migration_types(types[order]); 5031 } 5032 printk(KERN_CONT "= %lukB\n", K(total)); 5033 } 5034 5035 hugetlb_show_meminfo(); 5036 5037 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 5038 5039 show_swap_cache_info(); 5040 } 5041 5042 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 5043 { 5044 zoneref->zone = zone; 5045 zoneref->zone_idx = zone_idx(zone); 5046 } 5047 5048 /* 5049 * Builds allocation fallback zone lists. 5050 * 5051 * Add all populated zones of a node to the zonelist. 5052 */ 5053 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) 5054 { 5055 struct zone *zone; 5056 enum zone_type zone_type = MAX_NR_ZONES; 5057 int nr_zones = 0; 5058 5059 do { 5060 zone_type--; 5061 zone = pgdat->node_zones + zone_type; 5062 if (managed_zone(zone)) { 5063 zoneref_set_zone(zone, &zonerefs[nr_zones++]); 5064 check_highest_zone(zone_type); 5065 } 5066 } while (zone_type); 5067 5068 return nr_zones; 5069 } 5070 5071 #ifdef CONFIG_NUMA 5072 5073 static int __parse_numa_zonelist_order(char *s) 5074 { 5075 /* 5076 * We used to support different zonlists modes but they turned 5077 * out to be just not useful. Let's keep the warning in place 5078 * if somebody still use the cmd line parameter so that we do 5079 * not fail it silently 5080 */ 5081 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) { 5082 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s); 5083 return -EINVAL; 5084 } 5085 return 0; 5086 } 5087 5088 static __init int setup_numa_zonelist_order(char *s) 5089 { 5090 if (!s) 5091 return 0; 5092 5093 return __parse_numa_zonelist_order(s); 5094 } 5095 early_param("numa_zonelist_order", setup_numa_zonelist_order); 5096 5097 char numa_zonelist_order[] = "Node"; 5098 5099 /* 5100 * sysctl handler for numa_zonelist_order 5101 */ 5102 int numa_zonelist_order_handler(struct ctl_table *table, int write, 5103 void __user *buffer, size_t *length, 5104 loff_t *ppos) 5105 { 5106 char *str; 5107 int ret; 5108 5109 if (!write) 5110 return proc_dostring(table, write, buffer, length, ppos); 5111 str = memdup_user_nul(buffer, 16); 5112 if (IS_ERR(str)) 5113 return PTR_ERR(str); 5114 5115 ret = __parse_numa_zonelist_order(str); 5116 kfree(str); 5117 return ret; 5118 } 5119 5120 5121 #define MAX_NODE_LOAD (nr_online_nodes) 5122 static int node_load[MAX_NUMNODES]; 5123 5124 /** 5125 * find_next_best_node - find the next node that should appear in a given node's fallback list 5126 * @node: node whose fallback list we're appending 5127 * @used_node_mask: nodemask_t of already used nodes 5128 * 5129 * We use a number of factors to determine which is the next node that should 5130 * appear on a given node's fallback list. The node should not have appeared 5131 * already in @node's fallback list, and it should be the next closest node 5132 * according to the distance array (which contains arbitrary distance values 5133 * from each node to each node in the system), and should also prefer nodes 5134 * with no CPUs, since presumably they'll have very little allocation pressure 5135 * on them otherwise. 5136 * It returns -1 if no node is found. 5137 */ 5138 static int find_next_best_node(int node, nodemask_t *used_node_mask) 5139 { 5140 int n, val; 5141 int min_val = INT_MAX; 5142 int best_node = NUMA_NO_NODE; 5143 const struct cpumask *tmp = cpumask_of_node(0); 5144 5145 /* Use the local node if we haven't already */ 5146 if (!node_isset(node, *used_node_mask)) { 5147 node_set(node, *used_node_mask); 5148 return node; 5149 } 5150 5151 for_each_node_state(n, N_MEMORY) { 5152 5153 /* Don't want a node to appear more than once */ 5154 if (node_isset(n, *used_node_mask)) 5155 continue; 5156 5157 /* Use the distance array to find the distance */ 5158 val = node_distance(node, n); 5159 5160 /* Penalize nodes under us ("prefer the next node") */ 5161 val += (n < node); 5162 5163 /* Give preference to headless and unused nodes */ 5164 tmp = cpumask_of_node(n); 5165 if (!cpumask_empty(tmp)) 5166 val += PENALTY_FOR_NODE_WITH_CPUS; 5167 5168 /* Slight preference for less loaded node */ 5169 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 5170 val += node_load[n]; 5171 5172 if (val < min_val) { 5173 min_val = val; 5174 best_node = n; 5175 } 5176 } 5177 5178 if (best_node >= 0) 5179 node_set(best_node, *used_node_mask); 5180 5181 return best_node; 5182 } 5183 5184 5185 /* 5186 * Build zonelists ordered by node and zones within node. 5187 * This results in maximum locality--normal zone overflows into local 5188 * DMA zone, if any--but risks exhausting DMA zone. 5189 */ 5190 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, 5191 unsigned nr_nodes) 5192 { 5193 struct zoneref *zonerefs; 5194 int i; 5195 5196 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5197 5198 for (i = 0; i < nr_nodes; i++) { 5199 int nr_zones; 5200 5201 pg_data_t *node = NODE_DATA(node_order[i]); 5202 5203 nr_zones = build_zonerefs_node(node, zonerefs); 5204 zonerefs += nr_zones; 5205 } 5206 zonerefs->zone = NULL; 5207 zonerefs->zone_idx = 0; 5208 } 5209 5210 /* 5211 * Build gfp_thisnode zonelists 5212 */ 5213 static void build_thisnode_zonelists(pg_data_t *pgdat) 5214 { 5215 struct zoneref *zonerefs; 5216 int nr_zones; 5217 5218 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; 5219 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5220 zonerefs += nr_zones; 5221 zonerefs->zone = NULL; 5222 zonerefs->zone_idx = 0; 5223 } 5224 5225 /* 5226 * Build zonelists ordered by zone and nodes within zones. 5227 * This results in conserving DMA zone[s] until all Normal memory is 5228 * exhausted, but results in overflowing to remote node while memory 5229 * may still exist in local DMA zone. 5230 */ 5231 5232 static void build_zonelists(pg_data_t *pgdat) 5233 { 5234 static int node_order[MAX_NUMNODES]; 5235 int node, load, nr_nodes = 0; 5236 nodemask_t used_mask; 5237 int local_node, prev_node; 5238 5239 /* NUMA-aware ordering of nodes */ 5240 local_node = pgdat->node_id; 5241 load = nr_online_nodes; 5242 prev_node = local_node; 5243 nodes_clear(used_mask); 5244 5245 memset(node_order, 0, sizeof(node_order)); 5246 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 5247 /* 5248 * We don't want to pressure a particular node. 5249 * So adding penalty to the first node in same 5250 * distance group to make it round-robin. 5251 */ 5252 if (node_distance(local_node, node) != 5253 node_distance(local_node, prev_node)) 5254 node_load[node] = load; 5255 5256 node_order[nr_nodes++] = node; 5257 prev_node = node; 5258 load--; 5259 } 5260 5261 build_zonelists_in_node_order(pgdat, node_order, nr_nodes); 5262 build_thisnode_zonelists(pgdat); 5263 } 5264 5265 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5266 /* 5267 * Return node id of node used for "local" allocations. 5268 * I.e., first node id of first zone in arg node's generic zonelist. 5269 * Used for initializing percpu 'numa_mem', which is used primarily 5270 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 5271 */ 5272 int local_memory_node(int node) 5273 { 5274 struct zoneref *z; 5275 5276 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 5277 gfp_zone(GFP_KERNEL), 5278 NULL); 5279 return zone_to_nid(z->zone); 5280 } 5281 #endif 5282 5283 static void setup_min_unmapped_ratio(void); 5284 static void setup_min_slab_ratio(void); 5285 #else /* CONFIG_NUMA */ 5286 5287 static void build_zonelists(pg_data_t *pgdat) 5288 { 5289 int node, local_node; 5290 struct zoneref *zonerefs; 5291 int nr_zones; 5292 5293 local_node = pgdat->node_id; 5294 5295 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5296 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5297 zonerefs += nr_zones; 5298 5299 /* 5300 * Now we build the zonelist so that it contains the zones 5301 * of all the other nodes. 5302 * We don't want to pressure a particular node, so when 5303 * building the zones for node N, we make sure that the 5304 * zones coming right after the local ones are those from 5305 * node N+1 (modulo N) 5306 */ 5307 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 5308 if (!node_online(node)) 5309 continue; 5310 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5311 zonerefs += nr_zones; 5312 } 5313 for (node = 0; node < local_node; node++) { 5314 if (!node_online(node)) 5315 continue; 5316 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 5317 zonerefs += nr_zones; 5318 } 5319 5320 zonerefs->zone = NULL; 5321 zonerefs->zone_idx = 0; 5322 } 5323 5324 #endif /* CONFIG_NUMA */ 5325 5326 /* 5327 * Boot pageset table. One per cpu which is going to be used for all 5328 * zones and all nodes. The parameters will be set in such a way 5329 * that an item put on a list will immediately be handed over to 5330 * the buddy list. This is safe since pageset manipulation is done 5331 * with interrupts disabled. 5332 * 5333 * The boot_pagesets must be kept even after bootup is complete for 5334 * unused processors and/or zones. They do play a role for bootstrapping 5335 * hotplugged processors. 5336 * 5337 * zoneinfo_show() and maybe other functions do 5338 * not check if the processor is online before following the pageset pointer. 5339 * Other parts of the kernel may not check if the zone is available. 5340 */ 5341 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); 5342 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); 5343 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 5344 5345 static void __build_all_zonelists(void *data) 5346 { 5347 int nid; 5348 int __maybe_unused cpu; 5349 pg_data_t *self = data; 5350 static DEFINE_SPINLOCK(lock); 5351 5352 spin_lock(&lock); 5353 5354 #ifdef CONFIG_NUMA 5355 memset(node_load, 0, sizeof(node_load)); 5356 #endif 5357 5358 /* 5359 * This node is hotadded and no memory is yet present. So just 5360 * building zonelists is fine - no need to touch other nodes. 5361 */ 5362 if (self && !node_online(self->node_id)) { 5363 build_zonelists(self); 5364 } else { 5365 for_each_online_node(nid) { 5366 pg_data_t *pgdat = NODE_DATA(nid); 5367 5368 build_zonelists(pgdat); 5369 } 5370 5371 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5372 /* 5373 * We now know the "local memory node" for each node-- 5374 * i.e., the node of the first zone in the generic zonelist. 5375 * Set up numa_mem percpu variable for on-line cpus. During 5376 * boot, only the boot cpu should be on-line; we'll init the 5377 * secondary cpus' numa_mem as they come on-line. During 5378 * node/memory hotplug, we'll fixup all on-line cpus. 5379 */ 5380 for_each_online_cpu(cpu) 5381 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 5382 #endif 5383 } 5384 5385 spin_unlock(&lock); 5386 } 5387 5388 static noinline void __init 5389 build_all_zonelists_init(void) 5390 { 5391 int cpu; 5392 5393 __build_all_zonelists(NULL); 5394 5395 /* 5396 * Initialize the boot_pagesets that are going to be used 5397 * for bootstrapping processors. The real pagesets for 5398 * each zone will be allocated later when the per cpu 5399 * allocator is available. 5400 * 5401 * boot_pagesets are used also for bootstrapping offline 5402 * cpus if the system is already booted because the pagesets 5403 * are needed to initialize allocators on a specific cpu too. 5404 * F.e. the percpu allocator needs the page allocator which 5405 * needs the percpu allocator in order to allocate its pagesets 5406 * (a chicken-egg dilemma). 5407 */ 5408 for_each_possible_cpu(cpu) 5409 setup_pageset(&per_cpu(boot_pageset, cpu), 0); 5410 5411 mminit_verify_zonelist(); 5412 cpuset_init_current_mems_allowed(); 5413 } 5414 5415 /* 5416 * unless system_state == SYSTEM_BOOTING. 5417 * 5418 * __ref due to call of __init annotated helper build_all_zonelists_init 5419 * [protected by SYSTEM_BOOTING]. 5420 */ 5421 void __ref build_all_zonelists(pg_data_t *pgdat) 5422 { 5423 if (system_state == SYSTEM_BOOTING) { 5424 build_all_zonelists_init(); 5425 } else { 5426 __build_all_zonelists(pgdat); 5427 /* cpuset refresh routine should be here */ 5428 } 5429 vm_total_pages = nr_free_pagecache_pages(); 5430 /* 5431 * Disable grouping by mobility if the number of pages in the 5432 * system is too low to allow the mechanism to work. It would be 5433 * more accurate, but expensive to check per-zone. This check is 5434 * made on memory-hotadd so a system can start with mobility 5435 * disabled and enable it later 5436 */ 5437 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 5438 page_group_by_mobility_disabled = 1; 5439 else 5440 page_group_by_mobility_disabled = 0; 5441 5442 pr_info("Built %i zonelists, mobility grouping %s. Total pages: %ld\n", 5443 nr_online_nodes, 5444 page_group_by_mobility_disabled ? "off" : "on", 5445 vm_total_pages); 5446 #ifdef CONFIG_NUMA 5447 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 5448 #endif 5449 } 5450 5451 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 5452 static bool __meminit 5453 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 5454 { 5455 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5456 static struct memblock_region *r; 5457 5458 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 5459 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 5460 for_each_memblock(memory, r) { 5461 if (*pfn < memblock_region_memory_end_pfn(r)) 5462 break; 5463 } 5464 } 5465 if (*pfn >= memblock_region_memory_base_pfn(r) && 5466 memblock_is_mirror(r)) { 5467 *pfn = memblock_region_memory_end_pfn(r); 5468 return true; 5469 } 5470 } 5471 #endif 5472 return false; 5473 } 5474 5475 /* 5476 * Initially all pages are reserved - free ones are freed 5477 * up by memblock_free_all() once the early boot process is 5478 * done. Non-atomic initialization, single-pass. 5479 */ 5480 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, 5481 unsigned long start_pfn, enum memmap_context context, 5482 struct vmem_altmap *altmap) 5483 { 5484 unsigned long pfn, end_pfn = start_pfn + size; 5485 struct page *page; 5486 5487 if (highest_memmap_pfn < end_pfn - 1) 5488 highest_memmap_pfn = end_pfn - 1; 5489 5490 #ifdef CONFIG_ZONE_DEVICE 5491 /* 5492 * Honor reservation requested by the driver for this ZONE_DEVICE 5493 * memory. We limit the total number of pages to initialize to just 5494 * those that might contain the memory mapping. We will defer the 5495 * ZONE_DEVICE page initialization until after we have released 5496 * the hotplug lock. 5497 */ 5498 if (zone == ZONE_DEVICE) { 5499 if (!altmap) 5500 return; 5501 5502 if (start_pfn == altmap->base_pfn) 5503 start_pfn += altmap->reserve; 5504 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 5505 } 5506 #endif 5507 5508 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 5509 /* 5510 * There can be holes in boot-time mem_map[]s handed to this 5511 * function. They do not exist on hotplugged memory. 5512 */ 5513 if (context == MEMMAP_EARLY) { 5514 if (!early_pfn_valid(pfn)) 5515 continue; 5516 if (!early_pfn_in_nid(pfn, nid)) 5517 continue; 5518 if (overlap_memmap_init(zone, &pfn)) 5519 continue; 5520 if (defer_init(nid, pfn, end_pfn)) 5521 break; 5522 } 5523 5524 page = pfn_to_page(pfn); 5525 __init_single_page(page, pfn, zone, nid); 5526 if (context == MEMMAP_HOTPLUG) 5527 __SetPageReserved(page); 5528 5529 /* 5530 * Mark the block movable so that blocks are reserved for 5531 * movable at startup. This will force kernel allocations 5532 * to reserve their blocks rather than leaking throughout 5533 * the address space during boot when many long-lived 5534 * kernel allocations are made. 5535 * 5536 * bitmap is created for zone's valid pfn range. but memmap 5537 * can be created for invalid pages (for alignment) 5538 * check here not to call set_pageblock_migratetype() against 5539 * pfn out of zone. 5540 */ 5541 if (!(pfn & (pageblock_nr_pages - 1))) { 5542 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 5543 cond_resched(); 5544 } 5545 } 5546 #ifdef CONFIG_SPARSEMEM 5547 /* 5548 * If the zone does not span the rest of the section then 5549 * we should at least initialize those pages. Otherwise we 5550 * could blow up on a poisoned page in some paths which depend 5551 * on full sections being initialized (e.g. memory hotplug). 5552 */ 5553 while (end_pfn % PAGES_PER_SECTION) { 5554 __init_single_page(pfn_to_page(end_pfn), end_pfn, zone, nid); 5555 end_pfn++; 5556 } 5557 #endif 5558 } 5559 5560 #ifdef CONFIG_ZONE_DEVICE 5561 void __ref memmap_init_zone_device(struct zone *zone, 5562 unsigned long start_pfn, 5563 unsigned long size, 5564 struct dev_pagemap *pgmap) 5565 { 5566 unsigned long pfn, end_pfn = start_pfn + size; 5567 struct pglist_data *pgdat = zone->zone_pgdat; 5568 unsigned long zone_idx = zone_idx(zone); 5569 unsigned long start = jiffies; 5570 int nid = pgdat->node_id; 5571 5572 if (WARN_ON_ONCE(!pgmap || !is_dev_zone(zone))) 5573 return; 5574 5575 /* 5576 * The call to memmap_init_zone should have already taken care 5577 * of the pages reserved for the memmap, so we can just jump to 5578 * the end of that region and start processing the device pages. 5579 */ 5580 if (pgmap->altmap_valid) { 5581 struct vmem_altmap *altmap = &pgmap->altmap; 5582 5583 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 5584 size = end_pfn - start_pfn; 5585 } 5586 5587 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 5588 struct page *page = pfn_to_page(pfn); 5589 5590 __init_single_page(page, pfn, zone_idx, nid); 5591 5592 /* 5593 * Mark page reserved as it will need to wait for onlining 5594 * phase for it to be fully associated with a zone. 5595 * 5596 * We can use the non-atomic __set_bit operation for setting 5597 * the flag as we are still initializing the pages. 5598 */ 5599 __SetPageReserved(page); 5600 5601 /* 5602 * ZONE_DEVICE pages union ->lru with a ->pgmap back 5603 * pointer and hmm_data. It is a bug if a ZONE_DEVICE 5604 * page is ever freed or placed on a driver-private list. 5605 */ 5606 page->pgmap = pgmap; 5607 page->hmm_data = 0; 5608 5609 /* 5610 * Mark the block movable so that blocks are reserved for 5611 * movable at startup. This will force kernel allocations 5612 * to reserve their blocks rather than leaking throughout 5613 * the address space during boot when many long-lived 5614 * kernel allocations are made. 5615 * 5616 * bitmap is created for zone's valid pfn range. but memmap 5617 * can be created for invalid pages (for alignment) 5618 * check here not to call set_pageblock_migratetype() against 5619 * pfn out of zone. 5620 * 5621 * Please note that MEMMAP_HOTPLUG path doesn't clear memmap 5622 * because this is done early in sparse_add_one_section 5623 */ 5624 if (!(pfn & (pageblock_nr_pages - 1))) { 5625 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 5626 cond_resched(); 5627 } 5628 } 5629 5630 pr_info("%s initialised, %lu pages in %ums\n", dev_name(pgmap->dev), 5631 size, jiffies_to_msecs(jiffies - start)); 5632 } 5633 5634 #endif 5635 static void __meminit zone_init_free_lists(struct zone *zone) 5636 { 5637 unsigned int order, t; 5638 for_each_migratetype_order(order, t) { 5639 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 5640 zone->free_area[order].nr_free = 0; 5641 } 5642 } 5643 5644 void __meminit __weak memmap_init(unsigned long size, int nid, 5645 unsigned long zone, unsigned long start_pfn) 5646 { 5647 memmap_init_zone(size, nid, zone, start_pfn, MEMMAP_EARLY, NULL); 5648 } 5649 5650 static int zone_batchsize(struct zone *zone) 5651 { 5652 #ifdef CONFIG_MMU 5653 int batch; 5654 5655 /* 5656 * The per-cpu-pages pools are set to around 1000th of the 5657 * size of the zone. 5658 */ 5659 batch = zone->managed_pages / 1024; 5660 /* But no more than a meg. */ 5661 if (batch * PAGE_SIZE > 1024 * 1024) 5662 batch = (1024 * 1024) / PAGE_SIZE; 5663 batch /= 4; /* We effectively *= 4 below */ 5664 if (batch < 1) 5665 batch = 1; 5666 5667 /* 5668 * Clamp the batch to a 2^n - 1 value. Having a power 5669 * of 2 value was found to be more likely to have 5670 * suboptimal cache aliasing properties in some cases. 5671 * 5672 * For example if 2 tasks are alternately allocating 5673 * batches of pages, one task can end up with a lot 5674 * of pages of one half of the possible page colors 5675 * and the other with pages of the other colors. 5676 */ 5677 batch = rounddown_pow_of_two(batch + batch/2) - 1; 5678 5679 return batch; 5680 5681 #else 5682 /* The deferral and batching of frees should be suppressed under NOMMU 5683 * conditions. 5684 * 5685 * The problem is that NOMMU needs to be able to allocate large chunks 5686 * of contiguous memory as there's no hardware page translation to 5687 * assemble apparent contiguous memory from discontiguous pages. 5688 * 5689 * Queueing large contiguous runs of pages for batching, however, 5690 * causes the pages to actually be freed in smaller chunks. As there 5691 * can be a significant delay between the individual batches being 5692 * recycled, this leads to the once large chunks of space being 5693 * fragmented and becoming unavailable for high-order allocations. 5694 */ 5695 return 0; 5696 #endif 5697 } 5698 5699 /* 5700 * pcp->high and pcp->batch values are related and dependent on one another: 5701 * ->batch must never be higher then ->high. 5702 * The following function updates them in a safe manner without read side 5703 * locking. 5704 * 5705 * Any new users of pcp->batch and pcp->high should ensure they can cope with 5706 * those fields changing asynchronously (acording the the above rule). 5707 * 5708 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 5709 * outside of boot time (or some other assurance that no concurrent updaters 5710 * exist). 5711 */ 5712 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 5713 unsigned long batch) 5714 { 5715 /* start with a fail safe value for batch */ 5716 pcp->batch = 1; 5717 smp_wmb(); 5718 5719 /* Update high, then batch, in order */ 5720 pcp->high = high; 5721 smp_wmb(); 5722 5723 pcp->batch = batch; 5724 } 5725 5726 /* a companion to pageset_set_high() */ 5727 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) 5728 { 5729 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); 5730 } 5731 5732 static void pageset_init(struct per_cpu_pageset *p) 5733 { 5734 struct per_cpu_pages *pcp; 5735 int migratetype; 5736 5737 memset(p, 0, sizeof(*p)); 5738 5739 pcp = &p->pcp; 5740 pcp->count = 0; 5741 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) 5742 INIT_LIST_HEAD(&pcp->lists[migratetype]); 5743 } 5744 5745 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 5746 { 5747 pageset_init(p); 5748 pageset_set_batch(p, batch); 5749 } 5750 5751 /* 5752 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist 5753 * to the value high for the pageset p. 5754 */ 5755 static void pageset_set_high(struct per_cpu_pageset *p, 5756 unsigned long high) 5757 { 5758 unsigned long batch = max(1UL, high / 4); 5759 if ((high / 4) > (PAGE_SHIFT * 8)) 5760 batch = PAGE_SHIFT * 8; 5761 5762 pageset_update(&p->pcp, high, batch); 5763 } 5764 5765 static void pageset_set_high_and_batch(struct zone *zone, 5766 struct per_cpu_pageset *pcp) 5767 { 5768 if (percpu_pagelist_fraction) 5769 pageset_set_high(pcp, 5770 (zone->managed_pages / 5771 percpu_pagelist_fraction)); 5772 else 5773 pageset_set_batch(pcp, zone_batchsize(zone)); 5774 } 5775 5776 static void __meminit zone_pageset_init(struct zone *zone, int cpu) 5777 { 5778 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); 5779 5780 pageset_init(pcp); 5781 pageset_set_high_and_batch(zone, pcp); 5782 } 5783 5784 void __meminit setup_zone_pageset(struct zone *zone) 5785 { 5786 int cpu; 5787 zone->pageset = alloc_percpu(struct per_cpu_pageset); 5788 for_each_possible_cpu(cpu) 5789 zone_pageset_init(zone, cpu); 5790 } 5791 5792 /* 5793 * Allocate per cpu pagesets and initialize them. 5794 * Before this call only boot pagesets were available. 5795 */ 5796 void __init setup_per_cpu_pageset(void) 5797 { 5798 struct pglist_data *pgdat; 5799 struct zone *zone; 5800 5801 for_each_populated_zone(zone) 5802 setup_zone_pageset(zone); 5803 5804 for_each_online_pgdat(pgdat) 5805 pgdat->per_cpu_nodestats = 5806 alloc_percpu(struct per_cpu_nodestat); 5807 } 5808 5809 static __meminit void zone_pcp_init(struct zone *zone) 5810 { 5811 /* 5812 * per cpu subsystem is not up at this point. The following code 5813 * relies on the ability of the linker to provide the 5814 * offset of a (static) per cpu variable into the per cpu area. 5815 */ 5816 zone->pageset = &boot_pageset; 5817 5818 if (populated_zone(zone)) 5819 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", 5820 zone->name, zone->present_pages, 5821 zone_batchsize(zone)); 5822 } 5823 5824 void __meminit init_currently_empty_zone(struct zone *zone, 5825 unsigned long zone_start_pfn, 5826 unsigned long size) 5827 { 5828 struct pglist_data *pgdat = zone->zone_pgdat; 5829 int zone_idx = zone_idx(zone) + 1; 5830 5831 if (zone_idx > pgdat->nr_zones) 5832 pgdat->nr_zones = zone_idx; 5833 5834 zone->zone_start_pfn = zone_start_pfn; 5835 5836 mminit_dprintk(MMINIT_TRACE, "memmap_init", 5837 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 5838 pgdat->node_id, 5839 (unsigned long)zone_idx(zone), 5840 zone_start_pfn, (zone_start_pfn + size)); 5841 5842 zone_init_free_lists(zone); 5843 zone->initialized = 1; 5844 } 5845 5846 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5847 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID 5848 5849 /* 5850 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 5851 */ 5852 int __meminit __early_pfn_to_nid(unsigned long pfn, 5853 struct mminit_pfnnid_cache *state) 5854 { 5855 unsigned long start_pfn, end_pfn; 5856 int nid; 5857 5858 if (state->last_start <= pfn && pfn < state->last_end) 5859 return state->last_nid; 5860 5861 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 5862 if (nid != -1) { 5863 state->last_start = start_pfn; 5864 state->last_end = end_pfn; 5865 state->last_nid = nid; 5866 } 5867 5868 return nid; 5869 } 5870 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ 5871 5872 /** 5873 * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range 5874 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed. 5875 * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid 5876 * 5877 * If an architecture guarantees that all ranges registered contain no holes 5878 * and may be freed, this this function may be used instead of calling 5879 * memblock_free_early_nid() manually. 5880 */ 5881 void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn) 5882 { 5883 unsigned long start_pfn, end_pfn; 5884 int i, this_nid; 5885 5886 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) { 5887 start_pfn = min(start_pfn, max_low_pfn); 5888 end_pfn = min(end_pfn, max_low_pfn); 5889 5890 if (start_pfn < end_pfn) 5891 memblock_free_early_nid(PFN_PHYS(start_pfn), 5892 (end_pfn - start_pfn) << PAGE_SHIFT, 5893 this_nid); 5894 } 5895 } 5896 5897 /** 5898 * sparse_memory_present_with_active_regions - Call memory_present for each active range 5899 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used. 5900 * 5901 * If an architecture guarantees that all ranges registered contain no holes and may 5902 * be freed, this function may be used instead of calling memory_present() manually. 5903 */ 5904 void __init sparse_memory_present_with_active_regions(int nid) 5905 { 5906 unsigned long start_pfn, end_pfn; 5907 int i, this_nid; 5908 5909 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) 5910 memory_present(this_nid, start_pfn, end_pfn); 5911 } 5912 5913 /** 5914 * get_pfn_range_for_nid - Return the start and end page frames for a node 5915 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 5916 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 5917 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 5918 * 5919 * It returns the start and end page frame of a node based on information 5920 * provided by memblock_set_node(). If called for a node 5921 * with no available memory, a warning is printed and the start and end 5922 * PFNs will be 0. 5923 */ 5924 void __meminit get_pfn_range_for_nid(unsigned int nid, 5925 unsigned long *start_pfn, unsigned long *end_pfn) 5926 { 5927 unsigned long this_start_pfn, this_end_pfn; 5928 int i; 5929 5930 *start_pfn = -1UL; 5931 *end_pfn = 0; 5932 5933 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 5934 *start_pfn = min(*start_pfn, this_start_pfn); 5935 *end_pfn = max(*end_pfn, this_end_pfn); 5936 } 5937 5938 if (*start_pfn == -1UL) 5939 *start_pfn = 0; 5940 } 5941 5942 /* 5943 * This finds a zone that can be used for ZONE_MOVABLE pages. The 5944 * assumption is made that zones within a node are ordered in monotonic 5945 * increasing memory addresses so that the "highest" populated zone is used 5946 */ 5947 static void __init find_usable_zone_for_movable(void) 5948 { 5949 int zone_index; 5950 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 5951 if (zone_index == ZONE_MOVABLE) 5952 continue; 5953 5954 if (arch_zone_highest_possible_pfn[zone_index] > 5955 arch_zone_lowest_possible_pfn[zone_index]) 5956 break; 5957 } 5958 5959 VM_BUG_ON(zone_index == -1); 5960 movable_zone = zone_index; 5961 } 5962 5963 /* 5964 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 5965 * because it is sized independent of architecture. Unlike the other zones, 5966 * the starting point for ZONE_MOVABLE is not fixed. It may be different 5967 * in each node depending on the size of each node and how evenly kernelcore 5968 * is distributed. This helper function adjusts the zone ranges 5969 * provided by the architecture for a given node by using the end of the 5970 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 5971 * zones within a node are in order of monotonic increases memory addresses 5972 */ 5973 static void __meminit adjust_zone_range_for_zone_movable(int nid, 5974 unsigned long zone_type, 5975 unsigned long node_start_pfn, 5976 unsigned long node_end_pfn, 5977 unsigned long *zone_start_pfn, 5978 unsigned long *zone_end_pfn) 5979 { 5980 /* Only adjust if ZONE_MOVABLE is on this node */ 5981 if (zone_movable_pfn[nid]) { 5982 /* Size ZONE_MOVABLE */ 5983 if (zone_type == ZONE_MOVABLE) { 5984 *zone_start_pfn = zone_movable_pfn[nid]; 5985 *zone_end_pfn = min(node_end_pfn, 5986 arch_zone_highest_possible_pfn[movable_zone]); 5987 5988 /* Adjust for ZONE_MOVABLE starting within this range */ 5989 } else if (!mirrored_kernelcore && 5990 *zone_start_pfn < zone_movable_pfn[nid] && 5991 *zone_end_pfn > zone_movable_pfn[nid]) { 5992 *zone_end_pfn = zone_movable_pfn[nid]; 5993 5994 /* Check if this whole range is within ZONE_MOVABLE */ 5995 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 5996 *zone_start_pfn = *zone_end_pfn; 5997 } 5998 } 5999 6000 /* 6001 * Return the number of pages a zone spans in a node, including holes 6002 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 6003 */ 6004 static unsigned long __meminit zone_spanned_pages_in_node(int nid, 6005 unsigned long zone_type, 6006 unsigned long node_start_pfn, 6007 unsigned long node_end_pfn, 6008 unsigned long *zone_start_pfn, 6009 unsigned long *zone_end_pfn, 6010 unsigned long *ignored) 6011 { 6012 /* When hotadd a new node from cpu_up(), the node should be empty */ 6013 if (!node_start_pfn && !node_end_pfn) 6014 return 0; 6015 6016 /* Get the start and end of the zone */ 6017 *zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type]; 6018 *zone_end_pfn = arch_zone_highest_possible_pfn[zone_type]; 6019 adjust_zone_range_for_zone_movable(nid, zone_type, 6020 node_start_pfn, node_end_pfn, 6021 zone_start_pfn, zone_end_pfn); 6022 6023 /* Check that this node has pages within the zone's required range */ 6024 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 6025 return 0; 6026 6027 /* Move the zone boundaries inside the node if necessary */ 6028 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 6029 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 6030 6031 /* Return the spanned pages */ 6032 return *zone_end_pfn - *zone_start_pfn; 6033 } 6034 6035 /* 6036 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 6037 * then all holes in the requested range will be accounted for. 6038 */ 6039 unsigned long __meminit __absent_pages_in_range(int nid, 6040 unsigned long range_start_pfn, 6041 unsigned long range_end_pfn) 6042 { 6043 unsigned long nr_absent = range_end_pfn - range_start_pfn; 6044 unsigned long start_pfn, end_pfn; 6045 int i; 6046 6047 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6048 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 6049 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 6050 nr_absent -= end_pfn - start_pfn; 6051 } 6052 return nr_absent; 6053 } 6054 6055 /** 6056 * absent_pages_in_range - Return number of page frames in holes within a range 6057 * @start_pfn: The start PFN to start searching for holes 6058 * @end_pfn: The end PFN to stop searching for holes 6059 * 6060 * It returns the number of pages frames in memory holes within a range. 6061 */ 6062 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 6063 unsigned long end_pfn) 6064 { 6065 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 6066 } 6067 6068 /* Return the number of page frames in holes in a zone on a node */ 6069 static unsigned long __meminit zone_absent_pages_in_node(int nid, 6070 unsigned long zone_type, 6071 unsigned long node_start_pfn, 6072 unsigned long node_end_pfn, 6073 unsigned long *ignored) 6074 { 6075 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 6076 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 6077 unsigned long zone_start_pfn, zone_end_pfn; 6078 unsigned long nr_absent; 6079 6080 /* When hotadd a new node from cpu_up(), the node should be empty */ 6081 if (!node_start_pfn && !node_end_pfn) 6082 return 0; 6083 6084 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 6085 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 6086 6087 adjust_zone_range_for_zone_movable(nid, zone_type, 6088 node_start_pfn, node_end_pfn, 6089 &zone_start_pfn, &zone_end_pfn); 6090 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 6091 6092 /* 6093 * ZONE_MOVABLE handling. 6094 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 6095 * and vice versa. 6096 */ 6097 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 6098 unsigned long start_pfn, end_pfn; 6099 struct memblock_region *r; 6100 6101 for_each_memblock(memory, r) { 6102 start_pfn = clamp(memblock_region_memory_base_pfn(r), 6103 zone_start_pfn, zone_end_pfn); 6104 end_pfn = clamp(memblock_region_memory_end_pfn(r), 6105 zone_start_pfn, zone_end_pfn); 6106 6107 if (zone_type == ZONE_MOVABLE && 6108 memblock_is_mirror(r)) 6109 nr_absent += end_pfn - start_pfn; 6110 6111 if (zone_type == ZONE_NORMAL && 6112 !memblock_is_mirror(r)) 6113 nr_absent += end_pfn - start_pfn; 6114 } 6115 } 6116 6117 return nr_absent; 6118 } 6119 6120 #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6121 static inline unsigned long __meminit zone_spanned_pages_in_node(int nid, 6122 unsigned long zone_type, 6123 unsigned long node_start_pfn, 6124 unsigned long node_end_pfn, 6125 unsigned long *zone_start_pfn, 6126 unsigned long *zone_end_pfn, 6127 unsigned long *zones_size) 6128 { 6129 unsigned int zone; 6130 6131 *zone_start_pfn = node_start_pfn; 6132 for (zone = 0; zone < zone_type; zone++) 6133 *zone_start_pfn += zones_size[zone]; 6134 6135 *zone_end_pfn = *zone_start_pfn + zones_size[zone_type]; 6136 6137 return zones_size[zone_type]; 6138 } 6139 6140 static inline unsigned long __meminit zone_absent_pages_in_node(int nid, 6141 unsigned long zone_type, 6142 unsigned long node_start_pfn, 6143 unsigned long node_end_pfn, 6144 unsigned long *zholes_size) 6145 { 6146 if (!zholes_size) 6147 return 0; 6148 6149 return zholes_size[zone_type]; 6150 } 6151 6152 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6153 6154 static void __meminit calculate_node_totalpages(struct pglist_data *pgdat, 6155 unsigned long node_start_pfn, 6156 unsigned long node_end_pfn, 6157 unsigned long *zones_size, 6158 unsigned long *zholes_size) 6159 { 6160 unsigned long realtotalpages = 0, totalpages = 0; 6161 enum zone_type i; 6162 6163 for (i = 0; i < MAX_NR_ZONES; i++) { 6164 struct zone *zone = pgdat->node_zones + i; 6165 unsigned long zone_start_pfn, zone_end_pfn; 6166 unsigned long size, real_size; 6167 6168 size = zone_spanned_pages_in_node(pgdat->node_id, i, 6169 node_start_pfn, 6170 node_end_pfn, 6171 &zone_start_pfn, 6172 &zone_end_pfn, 6173 zones_size); 6174 real_size = size - zone_absent_pages_in_node(pgdat->node_id, i, 6175 node_start_pfn, node_end_pfn, 6176 zholes_size); 6177 if (size) 6178 zone->zone_start_pfn = zone_start_pfn; 6179 else 6180 zone->zone_start_pfn = 0; 6181 zone->spanned_pages = size; 6182 zone->present_pages = real_size; 6183 6184 totalpages += size; 6185 realtotalpages += real_size; 6186 } 6187 6188 pgdat->node_spanned_pages = totalpages; 6189 pgdat->node_present_pages = realtotalpages; 6190 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, 6191 realtotalpages); 6192 } 6193 6194 #ifndef CONFIG_SPARSEMEM 6195 /* 6196 * Calculate the size of the zone->blockflags rounded to an unsigned long 6197 * Start by making sure zonesize is a multiple of pageblock_order by rounding 6198 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 6199 * round what is now in bits to nearest long in bits, then return it in 6200 * bytes. 6201 */ 6202 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 6203 { 6204 unsigned long usemapsize; 6205 6206 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 6207 usemapsize = roundup(zonesize, pageblock_nr_pages); 6208 usemapsize = usemapsize >> pageblock_order; 6209 usemapsize *= NR_PAGEBLOCK_BITS; 6210 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 6211 6212 return usemapsize / 8; 6213 } 6214 6215 static void __ref setup_usemap(struct pglist_data *pgdat, 6216 struct zone *zone, 6217 unsigned long zone_start_pfn, 6218 unsigned long zonesize) 6219 { 6220 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); 6221 zone->pageblock_flags = NULL; 6222 if (usemapsize) 6223 zone->pageblock_flags = 6224 memblock_alloc_node_nopanic(usemapsize, 6225 pgdat->node_id); 6226 } 6227 #else 6228 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, 6229 unsigned long zone_start_pfn, unsigned long zonesize) {} 6230 #endif /* CONFIG_SPARSEMEM */ 6231 6232 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 6233 6234 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 6235 void __init set_pageblock_order(void) 6236 { 6237 unsigned int order; 6238 6239 /* Check that pageblock_nr_pages has not already been setup */ 6240 if (pageblock_order) 6241 return; 6242 6243 if (HPAGE_SHIFT > PAGE_SHIFT) 6244 order = HUGETLB_PAGE_ORDER; 6245 else 6246 order = MAX_ORDER - 1; 6247 6248 /* 6249 * Assume the largest contiguous order of interest is a huge page. 6250 * This value may be variable depending on boot parameters on IA64 and 6251 * powerpc. 6252 */ 6253 pageblock_order = order; 6254 } 6255 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6256 6257 /* 6258 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 6259 * is unused as pageblock_order is set at compile-time. See 6260 * include/linux/pageblock-flags.h for the values of pageblock_order based on 6261 * the kernel config 6262 */ 6263 void __init set_pageblock_order(void) 6264 { 6265 } 6266 6267 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 6268 6269 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 6270 unsigned long present_pages) 6271 { 6272 unsigned long pages = spanned_pages; 6273 6274 /* 6275 * Provide a more accurate estimation if there are holes within 6276 * the zone and SPARSEMEM is in use. If there are holes within the 6277 * zone, each populated memory region may cost us one or two extra 6278 * memmap pages due to alignment because memmap pages for each 6279 * populated regions may not be naturally aligned on page boundary. 6280 * So the (present_pages >> 4) heuristic is a tradeoff for that. 6281 */ 6282 if (spanned_pages > present_pages + (present_pages >> 4) && 6283 IS_ENABLED(CONFIG_SPARSEMEM)) 6284 pages = present_pages; 6285 6286 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 6287 } 6288 6289 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 6290 static void pgdat_init_split_queue(struct pglist_data *pgdat) 6291 { 6292 spin_lock_init(&pgdat->split_queue_lock); 6293 INIT_LIST_HEAD(&pgdat->split_queue); 6294 pgdat->split_queue_len = 0; 6295 } 6296 #else 6297 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 6298 #endif 6299 6300 #ifdef CONFIG_COMPACTION 6301 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 6302 { 6303 init_waitqueue_head(&pgdat->kcompactd_wait); 6304 } 6305 #else 6306 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 6307 #endif 6308 6309 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 6310 { 6311 pgdat_resize_init(pgdat); 6312 6313 pgdat_init_split_queue(pgdat); 6314 pgdat_init_kcompactd(pgdat); 6315 6316 init_waitqueue_head(&pgdat->kswapd_wait); 6317 init_waitqueue_head(&pgdat->pfmemalloc_wait); 6318 6319 pgdat_page_ext_init(pgdat); 6320 spin_lock_init(&pgdat->lru_lock); 6321 lruvec_init(node_lruvec(pgdat)); 6322 } 6323 6324 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 6325 unsigned long remaining_pages) 6326 { 6327 zone->managed_pages = remaining_pages; 6328 zone_set_nid(zone, nid); 6329 zone->name = zone_names[idx]; 6330 zone->zone_pgdat = NODE_DATA(nid); 6331 spin_lock_init(&zone->lock); 6332 zone_seqlock_init(zone); 6333 zone_pcp_init(zone); 6334 } 6335 6336 /* 6337 * Set up the zone data structures 6338 * - init pgdat internals 6339 * - init all zones belonging to this node 6340 * 6341 * NOTE: this function is only called during memory hotplug 6342 */ 6343 #ifdef CONFIG_MEMORY_HOTPLUG 6344 void __ref free_area_init_core_hotplug(int nid) 6345 { 6346 enum zone_type z; 6347 pg_data_t *pgdat = NODE_DATA(nid); 6348 6349 pgdat_init_internals(pgdat); 6350 for (z = 0; z < MAX_NR_ZONES; z++) 6351 zone_init_internals(&pgdat->node_zones[z], z, nid, 0); 6352 } 6353 #endif 6354 6355 /* 6356 * Set up the zone data structures: 6357 * - mark all pages reserved 6358 * - mark all memory queues empty 6359 * - clear the memory bitmaps 6360 * 6361 * NOTE: pgdat should get zeroed by caller. 6362 * NOTE: this function is only called during early init. 6363 */ 6364 static void __init free_area_init_core(struct pglist_data *pgdat) 6365 { 6366 enum zone_type j; 6367 int nid = pgdat->node_id; 6368 6369 pgdat_init_internals(pgdat); 6370 pgdat->per_cpu_nodestats = &boot_nodestats; 6371 6372 for (j = 0; j < MAX_NR_ZONES; j++) { 6373 struct zone *zone = pgdat->node_zones + j; 6374 unsigned long size, freesize, memmap_pages; 6375 unsigned long zone_start_pfn = zone->zone_start_pfn; 6376 6377 size = zone->spanned_pages; 6378 freesize = zone->present_pages; 6379 6380 /* 6381 * Adjust freesize so that it accounts for how much memory 6382 * is used by this zone for memmap. This affects the watermark 6383 * and per-cpu initialisations 6384 */ 6385 memmap_pages = calc_memmap_size(size, freesize); 6386 if (!is_highmem_idx(j)) { 6387 if (freesize >= memmap_pages) { 6388 freesize -= memmap_pages; 6389 if (memmap_pages) 6390 printk(KERN_DEBUG 6391 " %s zone: %lu pages used for memmap\n", 6392 zone_names[j], memmap_pages); 6393 } else 6394 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n", 6395 zone_names[j], memmap_pages, freesize); 6396 } 6397 6398 /* Account for reserved pages */ 6399 if (j == 0 && freesize > dma_reserve) { 6400 freesize -= dma_reserve; 6401 printk(KERN_DEBUG " %s zone: %lu pages reserved\n", 6402 zone_names[0], dma_reserve); 6403 } 6404 6405 if (!is_highmem_idx(j)) 6406 nr_kernel_pages += freesize; 6407 /* Charge for highmem memmap if there are enough kernel pages */ 6408 else if (nr_kernel_pages > memmap_pages * 2) 6409 nr_kernel_pages -= memmap_pages; 6410 nr_all_pages += freesize; 6411 6412 /* 6413 * Set an approximate value for lowmem here, it will be adjusted 6414 * when the bootmem allocator frees pages into the buddy system. 6415 * And all highmem pages will be managed by the buddy system. 6416 */ 6417 zone_init_internals(zone, j, nid, freesize); 6418 6419 if (!size) 6420 continue; 6421 6422 set_pageblock_order(); 6423 setup_usemap(pgdat, zone, zone_start_pfn, size); 6424 init_currently_empty_zone(zone, zone_start_pfn, size); 6425 memmap_init(size, nid, j, zone_start_pfn); 6426 } 6427 } 6428 6429 #ifdef CONFIG_FLAT_NODE_MEM_MAP 6430 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) 6431 { 6432 unsigned long __maybe_unused start = 0; 6433 unsigned long __maybe_unused offset = 0; 6434 6435 /* Skip empty nodes */ 6436 if (!pgdat->node_spanned_pages) 6437 return; 6438 6439 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 6440 offset = pgdat->node_start_pfn - start; 6441 /* ia64 gets its own node_mem_map, before this, without bootmem */ 6442 if (!pgdat->node_mem_map) { 6443 unsigned long size, end; 6444 struct page *map; 6445 6446 /* 6447 * The zone's endpoints aren't required to be MAX_ORDER 6448 * aligned but the node_mem_map endpoints must be in order 6449 * for the buddy allocator to function correctly. 6450 */ 6451 end = pgdat_end_pfn(pgdat); 6452 end = ALIGN(end, MAX_ORDER_NR_PAGES); 6453 size = (end - start) * sizeof(struct page); 6454 map = memblock_alloc_node_nopanic(size, pgdat->node_id); 6455 pgdat->node_mem_map = map + offset; 6456 } 6457 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 6458 __func__, pgdat->node_id, (unsigned long)pgdat, 6459 (unsigned long)pgdat->node_mem_map); 6460 #ifndef CONFIG_NEED_MULTIPLE_NODES 6461 /* 6462 * With no DISCONTIG, the global mem_map is just set as node 0's 6463 */ 6464 if (pgdat == NODE_DATA(0)) { 6465 mem_map = NODE_DATA(0)->node_mem_map; 6466 #if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM) 6467 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 6468 mem_map -= offset; 6469 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6470 } 6471 #endif 6472 } 6473 #else 6474 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { } 6475 #endif /* CONFIG_FLAT_NODE_MEM_MAP */ 6476 6477 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 6478 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 6479 { 6480 /* 6481 * We start only with one section of pages, more pages are added as 6482 * needed until the rest of deferred pages are initialized. 6483 */ 6484 pgdat->static_init_pgcnt = min_t(unsigned long, PAGES_PER_SECTION, 6485 pgdat->node_spanned_pages); 6486 pgdat->first_deferred_pfn = ULONG_MAX; 6487 } 6488 #else 6489 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 6490 #endif 6491 6492 void __init free_area_init_node(int nid, unsigned long *zones_size, 6493 unsigned long node_start_pfn, 6494 unsigned long *zholes_size) 6495 { 6496 pg_data_t *pgdat = NODE_DATA(nid); 6497 unsigned long start_pfn = 0; 6498 unsigned long end_pfn = 0; 6499 6500 /* pg_data_t should be reset to zero when it's allocated */ 6501 WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx); 6502 6503 pgdat->node_id = nid; 6504 pgdat->node_start_pfn = node_start_pfn; 6505 pgdat->per_cpu_nodestats = NULL; 6506 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6507 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 6508 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 6509 (u64)start_pfn << PAGE_SHIFT, 6510 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 6511 #else 6512 start_pfn = node_start_pfn; 6513 #endif 6514 calculate_node_totalpages(pgdat, start_pfn, end_pfn, 6515 zones_size, zholes_size); 6516 6517 alloc_node_mem_map(pgdat); 6518 pgdat_set_deferred_range(pgdat); 6519 6520 free_area_init_core(pgdat); 6521 } 6522 6523 #if !defined(CONFIG_FLAT_NODE_MEM_MAP) 6524 /* 6525 * Zero all valid struct pages in range [spfn, epfn), return number of struct 6526 * pages zeroed 6527 */ 6528 static u64 zero_pfn_range(unsigned long spfn, unsigned long epfn) 6529 { 6530 unsigned long pfn; 6531 u64 pgcnt = 0; 6532 6533 for (pfn = spfn; pfn < epfn; pfn++) { 6534 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) { 6535 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages) 6536 + pageblock_nr_pages - 1; 6537 continue; 6538 } 6539 mm_zero_struct_page(pfn_to_page(pfn)); 6540 pgcnt++; 6541 } 6542 6543 return pgcnt; 6544 } 6545 6546 /* 6547 * Only struct pages that are backed by physical memory are zeroed and 6548 * initialized by going through __init_single_page(). But, there are some 6549 * struct pages which are reserved in memblock allocator and their fields 6550 * may be accessed (for example page_to_pfn() on some configuration accesses 6551 * flags). We must explicitly zero those struct pages. 6552 * 6553 * This function also addresses a similar issue where struct pages are left 6554 * uninitialized because the physical address range is not covered by 6555 * memblock.memory or memblock.reserved. That could happen when memblock 6556 * layout is manually configured via memmap=. 6557 */ 6558 void __init zero_resv_unavail(void) 6559 { 6560 phys_addr_t start, end; 6561 u64 i, pgcnt; 6562 phys_addr_t next = 0; 6563 6564 /* 6565 * Loop through unavailable ranges not covered by memblock.memory. 6566 */ 6567 pgcnt = 0; 6568 for_each_mem_range(i, &memblock.memory, NULL, 6569 NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, NULL) { 6570 if (next < start) 6571 pgcnt += zero_pfn_range(PFN_DOWN(next), PFN_UP(start)); 6572 next = end; 6573 } 6574 pgcnt += zero_pfn_range(PFN_DOWN(next), max_pfn); 6575 6576 /* 6577 * Struct pages that do not have backing memory. This could be because 6578 * firmware is using some of this memory, or for some other reasons. 6579 */ 6580 if (pgcnt) 6581 pr_info("Zeroed struct page in unavailable ranges: %lld pages", pgcnt); 6582 } 6583 #endif /* !CONFIG_FLAT_NODE_MEM_MAP */ 6584 6585 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6586 6587 #if MAX_NUMNODES > 1 6588 /* 6589 * Figure out the number of possible node ids. 6590 */ 6591 void __init setup_nr_node_ids(void) 6592 { 6593 unsigned int highest; 6594 6595 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 6596 nr_node_ids = highest + 1; 6597 } 6598 #endif 6599 6600 /** 6601 * node_map_pfn_alignment - determine the maximum internode alignment 6602 * 6603 * This function should be called after node map is populated and sorted. 6604 * It calculates the maximum power of two alignment which can distinguish 6605 * all the nodes. 6606 * 6607 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 6608 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 6609 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 6610 * shifted, 1GiB is enough and this function will indicate so. 6611 * 6612 * This is used to test whether pfn -> nid mapping of the chosen memory 6613 * model has fine enough granularity to avoid incorrect mapping for the 6614 * populated node map. 6615 * 6616 * Returns the determined alignment in pfn's. 0 if there is no alignment 6617 * requirement (single node). 6618 */ 6619 unsigned long __init node_map_pfn_alignment(void) 6620 { 6621 unsigned long accl_mask = 0, last_end = 0; 6622 unsigned long start, end, mask; 6623 int last_nid = -1; 6624 int i, nid; 6625 6626 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 6627 if (!start || last_nid < 0 || last_nid == nid) { 6628 last_nid = nid; 6629 last_end = end; 6630 continue; 6631 } 6632 6633 /* 6634 * Start with a mask granular enough to pin-point to the 6635 * start pfn and tick off bits one-by-one until it becomes 6636 * too coarse to separate the current node from the last. 6637 */ 6638 mask = ~((1 << __ffs(start)) - 1); 6639 while (mask && last_end <= (start & (mask << 1))) 6640 mask <<= 1; 6641 6642 /* accumulate all internode masks */ 6643 accl_mask |= mask; 6644 } 6645 6646 /* convert mask to number of pages */ 6647 return ~accl_mask + 1; 6648 } 6649 6650 /* Find the lowest pfn for a node */ 6651 static unsigned long __init find_min_pfn_for_node(int nid) 6652 { 6653 unsigned long min_pfn = ULONG_MAX; 6654 unsigned long start_pfn; 6655 int i; 6656 6657 for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL) 6658 min_pfn = min(min_pfn, start_pfn); 6659 6660 if (min_pfn == ULONG_MAX) { 6661 pr_warn("Could not find start_pfn for node %d\n", nid); 6662 return 0; 6663 } 6664 6665 return min_pfn; 6666 } 6667 6668 /** 6669 * find_min_pfn_with_active_regions - Find the minimum PFN registered 6670 * 6671 * It returns the minimum PFN based on information provided via 6672 * memblock_set_node(). 6673 */ 6674 unsigned long __init find_min_pfn_with_active_regions(void) 6675 { 6676 return find_min_pfn_for_node(MAX_NUMNODES); 6677 } 6678 6679 /* 6680 * early_calculate_totalpages() 6681 * Sum pages in active regions for movable zone. 6682 * Populate N_MEMORY for calculating usable_nodes. 6683 */ 6684 static unsigned long __init early_calculate_totalpages(void) 6685 { 6686 unsigned long totalpages = 0; 6687 unsigned long start_pfn, end_pfn; 6688 int i, nid; 6689 6690 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 6691 unsigned long pages = end_pfn - start_pfn; 6692 6693 totalpages += pages; 6694 if (pages) 6695 node_set_state(nid, N_MEMORY); 6696 } 6697 return totalpages; 6698 } 6699 6700 /* 6701 * Find the PFN the Movable zone begins in each node. Kernel memory 6702 * is spread evenly between nodes as long as the nodes have enough 6703 * memory. When they don't, some nodes will have more kernelcore than 6704 * others 6705 */ 6706 static void __init find_zone_movable_pfns_for_nodes(void) 6707 { 6708 int i, nid; 6709 unsigned long usable_startpfn; 6710 unsigned long kernelcore_node, kernelcore_remaining; 6711 /* save the state before borrow the nodemask */ 6712 nodemask_t saved_node_state = node_states[N_MEMORY]; 6713 unsigned long totalpages = early_calculate_totalpages(); 6714 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 6715 struct memblock_region *r; 6716 6717 /* Need to find movable_zone earlier when movable_node is specified. */ 6718 find_usable_zone_for_movable(); 6719 6720 /* 6721 * If movable_node is specified, ignore kernelcore and movablecore 6722 * options. 6723 */ 6724 if (movable_node_is_enabled()) { 6725 for_each_memblock(memory, r) { 6726 if (!memblock_is_hotpluggable(r)) 6727 continue; 6728 6729 nid = r->nid; 6730 6731 usable_startpfn = PFN_DOWN(r->base); 6732 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 6733 min(usable_startpfn, zone_movable_pfn[nid]) : 6734 usable_startpfn; 6735 } 6736 6737 goto out2; 6738 } 6739 6740 /* 6741 * If kernelcore=mirror is specified, ignore movablecore option 6742 */ 6743 if (mirrored_kernelcore) { 6744 bool mem_below_4gb_not_mirrored = false; 6745 6746 for_each_memblock(memory, r) { 6747 if (memblock_is_mirror(r)) 6748 continue; 6749 6750 nid = r->nid; 6751 6752 usable_startpfn = memblock_region_memory_base_pfn(r); 6753 6754 if (usable_startpfn < 0x100000) { 6755 mem_below_4gb_not_mirrored = true; 6756 continue; 6757 } 6758 6759 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 6760 min(usable_startpfn, zone_movable_pfn[nid]) : 6761 usable_startpfn; 6762 } 6763 6764 if (mem_below_4gb_not_mirrored) 6765 pr_warn("This configuration results in unmirrored kernel memory."); 6766 6767 goto out2; 6768 } 6769 6770 /* 6771 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 6772 * amount of necessary memory. 6773 */ 6774 if (required_kernelcore_percent) 6775 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 6776 10000UL; 6777 if (required_movablecore_percent) 6778 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 6779 10000UL; 6780 6781 /* 6782 * If movablecore= was specified, calculate what size of 6783 * kernelcore that corresponds so that memory usable for 6784 * any allocation type is evenly spread. If both kernelcore 6785 * and movablecore are specified, then the value of kernelcore 6786 * will be used for required_kernelcore if it's greater than 6787 * what movablecore would have allowed. 6788 */ 6789 if (required_movablecore) { 6790 unsigned long corepages; 6791 6792 /* 6793 * Round-up so that ZONE_MOVABLE is at least as large as what 6794 * was requested by the user 6795 */ 6796 required_movablecore = 6797 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 6798 required_movablecore = min(totalpages, required_movablecore); 6799 corepages = totalpages - required_movablecore; 6800 6801 required_kernelcore = max(required_kernelcore, corepages); 6802 } 6803 6804 /* 6805 * If kernelcore was not specified or kernelcore size is larger 6806 * than totalpages, there is no ZONE_MOVABLE. 6807 */ 6808 if (!required_kernelcore || required_kernelcore >= totalpages) 6809 goto out; 6810 6811 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 6812 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 6813 6814 restart: 6815 /* Spread kernelcore memory as evenly as possible throughout nodes */ 6816 kernelcore_node = required_kernelcore / usable_nodes; 6817 for_each_node_state(nid, N_MEMORY) { 6818 unsigned long start_pfn, end_pfn; 6819 6820 /* 6821 * Recalculate kernelcore_node if the division per node 6822 * now exceeds what is necessary to satisfy the requested 6823 * amount of memory for the kernel 6824 */ 6825 if (required_kernelcore < kernelcore_node) 6826 kernelcore_node = required_kernelcore / usable_nodes; 6827 6828 /* 6829 * As the map is walked, we track how much memory is usable 6830 * by the kernel using kernelcore_remaining. When it is 6831 * 0, the rest of the node is usable by ZONE_MOVABLE 6832 */ 6833 kernelcore_remaining = kernelcore_node; 6834 6835 /* Go through each range of PFNs within this node */ 6836 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6837 unsigned long size_pages; 6838 6839 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 6840 if (start_pfn >= end_pfn) 6841 continue; 6842 6843 /* Account for what is only usable for kernelcore */ 6844 if (start_pfn < usable_startpfn) { 6845 unsigned long kernel_pages; 6846 kernel_pages = min(end_pfn, usable_startpfn) 6847 - start_pfn; 6848 6849 kernelcore_remaining -= min(kernel_pages, 6850 kernelcore_remaining); 6851 required_kernelcore -= min(kernel_pages, 6852 required_kernelcore); 6853 6854 /* Continue if range is now fully accounted */ 6855 if (end_pfn <= usable_startpfn) { 6856 6857 /* 6858 * Push zone_movable_pfn to the end so 6859 * that if we have to rebalance 6860 * kernelcore across nodes, we will 6861 * not double account here 6862 */ 6863 zone_movable_pfn[nid] = end_pfn; 6864 continue; 6865 } 6866 start_pfn = usable_startpfn; 6867 } 6868 6869 /* 6870 * The usable PFN range for ZONE_MOVABLE is from 6871 * start_pfn->end_pfn. Calculate size_pages as the 6872 * number of pages used as kernelcore 6873 */ 6874 size_pages = end_pfn - start_pfn; 6875 if (size_pages > kernelcore_remaining) 6876 size_pages = kernelcore_remaining; 6877 zone_movable_pfn[nid] = start_pfn + size_pages; 6878 6879 /* 6880 * Some kernelcore has been met, update counts and 6881 * break if the kernelcore for this node has been 6882 * satisfied 6883 */ 6884 required_kernelcore -= min(required_kernelcore, 6885 size_pages); 6886 kernelcore_remaining -= size_pages; 6887 if (!kernelcore_remaining) 6888 break; 6889 } 6890 } 6891 6892 /* 6893 * If there is still required_kernelcore, we do another pass with one 6894 * less node in the count. This will push zone_movable_pfn[nid] further 6895 * along on the nodes that still have memory until kernelcore is 6896 * satisfied 6897 */ 6898 usable_nodes--; 6899 if (usable_nodes && required_kernelcore > usable_nodes) 6900 goto restart; 6901 6902 out2: 6903 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 6904 for (nid = 0; nid < MAX_NUMNODES; nid++) 6905 zone_movable_pfn[nid] = 6906 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 6907 6908 out: 6909 /* restore the node_state */ 6910 node_states[N_MEMORY] = saved_node_state; 6911 } 6912 6913 /* Any regular or high memory on that node ? */ 6914 static void check_for_memory(pg_data_t *pgdat, int nid) 6915 { 6916 enum zone_type zone_type; 6917 6918 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 6919 struct zone *zone = &pgdat->node_zones[zone_type]; 6920 if (populated_zone(zone)) { 6921 if (IS_ENABLED(CONFIG_HIGHMEM)) 6922 node_set_state(nid, N_HIGH_MEMORY); 6923 if (zone_type <= ZONE_NORMAL) 6924 node_set_state(nid, N_NORMAL_MEMORY); 6925 break; 6926 } 6927 } 6928 } 6929 6930 /** 6931 * free_area_init_nodes - Initialise all pg_data_t and zone data 6932 * @max_zone_pfn: an array of max PFNs for each zone 6933 * 6934 * This will call free_area_init_node() for each active node in the system. 6935 * Using the page ranges provided by memblock_set_node(), the size of each 6936 * zone in each node and their holes is calculated. If the maximum PFN 6937 * between two adjacent zones match, it is assumed that the zone is empty. 6938 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 6939 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 6940 * starts where the previous one ended. For example, ZONE_DMA32 starts 6941 * at arch_max_dma_pfn. 6942 */ 6943 void __init free_area_init_nodes(unsigned long *max_zone_pfn) 6944 { 6945 unsigned long start_pfn, end_pfn; 6946 int i, nid; 6947 6948 /* Record where the zone boundaries are */ 6949 memset(arch_zone_lowest_possible_pfn, 0, 6950 sizeof(arch_zone_lowest_possible_pfn)); 6951 memset(arch_zone_highest_possible_pfn, 0, 6952 sizeof(arch_zone_highest_possible_pfn)); 6953 6954 start_pfn = find_min_pfn_with_active_regions(); 6955 6956 for (i = 0; i < MAX_NR_ZONES; i++) { 6957 if (i == ZONE_MOVABLE) 6958 continue; 6959 6960 end_pfn = max(max_zone_pfn[i], start_pfn); 6961 arch_zone_lowest_possible_pfn[i] = start_pfn; 6962 arch_zone_highest_possible_pfn[i] = end_pfn; 6963 6964 start_pfn = end_pfn; 6965 } 6966 6967 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 6968 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 6969 find_zone_movable_pfns_for_nodes(); 6970 6971 /* Print out the zone ranges */ 6972 pr_info("Zone ranges:\n"); 6973 for (i = 0; i < MAX_NR_ZONES; i++) { 6974 if (i == ZONE_MOVABLE) 6975 continue; 6976 pr_info(" %-8s ", zone_names[i]); 6977 if (arch_zone_lowest_possible_pfn[i] == 6978 arch_zone_highest_possible_pfn[i]) 6979 pr_cont("empty\n"); 6980 else 6981 pr_cont("[mem %#018Lx-%#018Lx]\n", 6982 (u64)arch_zone_lowest_possible_pfn[i] 6983 << PAGE_SHIFT, 6984 ((u64)arch_zone_highest_possible_pfn[i] 6985 << PAGE_SHIFT) - 1); 6986 } 6987 6988 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 6989 pr_info("Movable zone start for each node\n"); 6990 for (i = 0; i < MAX_NUMNODES; i++) { 6991 if (zone_movable_pfn[i]) 6992 pr_info(" Node %d: %#018Lx\n", i, 6993 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 6994 } 6995 6996 /* Print out the early node map */ 6997 pr_info("Early memory node ranges\n"); 6998 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) 6999 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 7000 (u64)start_pfn << PAGE_SHIFT, 7001 ((u64)end_pfn << PAGE_SHIFT) - 1); 7002 7003 /* Initialise every node */ 7004 mminit_verify_pageflags_layout(); 7005 setup_nr_node_ids(); 7006 zero_resv_unavail(); 7007 for_each_online_node(nid) { 7008 pg_data_t *pgdat = NODE_DATA(nid); 7009 free_area_init_node(nid, NULL, 7010 find_min_pfn_for_node(nid), NULL); 7011 7012 /* Any memory on that node */ 7013 if (pgdat->node_present_pages) 7014 node_set_state(nid, N_MEMORY); 7015 check_for_memory(pgdat, nid); 7016 } 7017 } 7018 7019 static int __init cmdline_parse_core(char *p, unsigned long *core, 7020 unsigned long *percent) 7021 { 7022 unsigned long long coremem; 7023 char *endptr; 7024 7025 if (!p) 7026 return -EINVAL; 7027 7028 /* Value may be a percentage of total memory, otherwise bytes */ 7029 coremem = simple_strtoull(p, &endptr, 0); 7030 if (*endptr == '%') { 7031 /* Paranoid check for percent values greater than 100 */ 7032 WARN_ON(coremem > 100); 7033 7034 *percent = coremem; 7035 } else { 7036 coremem = memparse(p, &p); 7037 /* Paranoid check that UL is enough for the coremem value */ 7038 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 7039 7040 *core = coremem >> PAGE_SHIFT; 7041 *percent = 0UL; 7042 } 7043 return 0; 7044 } 7045 7046 /* 7047 * kernelcore=size sets the amount of memory for use for allocations that 7048 * cannot be reclaimed or migrated. 7049 */ 7050 static int __init cmdline_parse_kernelcore(char *p) 7051 { 7052 /* parse kernelcore=mirror */ 7053 if (parse_option_str(p, "mirror")) { 7054 mirrored_kernelcore = true; 7055 return 0; 7056 } 7057 7058 return cmdline_parse_core(p, &required_kernelcore, 7059 &required_kernelcore_percent); 7060 } 7061 7062 /* 7063 * movablecore=size sets the amount of memory for use for allocations that 7064 * can be reclaimed or migrated. 7065 */ 7066 static int __init cmdline_parse_movablecore(char *p) 7067 { 7068 return cmdline_parse_core(p, &required_movablecore, 7069 &required_movablecore_percent); 7070 } 7071 7072 early_param("kernelcore", cmdline_parse_kernelcore); 7073 early_param("movablecore", cmdline_parse_movablecore); 7074 7075 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 7076 7077 void adjust_managed_page_count(struct page *page, long count) 7078 { 7079 spin_lock(&managed_page_count_lock); 7080 page_zone(page)->managed_pages += count; 7081 totalram_pages += count; 7082 #ifdef CONFIG_HIGHMEM 7083 if (PageHighMem(page)) 7084 totalhigh_pages += count; 7085 #endif 7086 spin_unlock(&managed_page_count_lock); 7087 } 7088 EXPORT_SYMBOL(adjust_managed_page_count); 7089 7090 unsigned long free_reserved_area(void *start, void *end, int poison, char *s) 7091 { 7092 void *pos; 7093 unsigned long pages = 0; 7094 7095 start = (void *)PAGE_ALIGN((unsigned long)start); 7096 end = (void *)((unsigned long)end & PAGE_MASK); 7097 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 7098 struct page *page = virt_to_page(pos); 7099 void *direct_map_addr; 7100 7101 /* 7102 * 'direct_map_addr' might be different from 'pos' 7103 * because some architectures' virt_to_page() 7104 * work with aliases. Getting the direct map 7105 * address ensures that we get a _writeable_ 7106 * alias for the memset(). 7107 */ 7108 direct_map_addr = page_address(page); 7109 if ((unsigned int)poison <= 0xFF) 7110 memset(direct_map_addr, poison, PAGE_SIZE); 7111 7112 free_reserved_page(page); 7113 } 7114 7115 if (pages && s) 7116 pr_info("Freeing %s memory: %ldK\n", 7117 s, pages << (PAGE_SHIFT - 10)); 7118 7119 return pages; 7120 } 7121 EXPORT_SYMBOL(free_reserved_area); 7122 7123 #ifdef CONFIG_HIGHMEM 7124 void free_highmem_page(struct page *page) 7125 { 7126 __free_reserved_page(page); 7127 totalram_pages++; 7128 page_zone(page)->managed_pages++; 7129 totalhigh_pages++; 7130 } 7131 #endif 7132 7133 7134 void __init mem_init_print_info(const char *str) 7135 { 7136 unsigned long physpages, codesize, datasize, rosize, bss_size; 7137 unsigned long init_code_size, init_data_size; 7138 7139 physpages = get_num_physpages(); 7140 codesize = _etext - _stext; 7141 datasize = _edata - _sdata; 7142 rosize = __end_rodata - __start_rodata; 7143 bss_size = __bss_stop - __bss_start; 7144 init_data_size = __init_end - __init_begin; 7145 init_code_size = _einittext - _sinittext; 7146 7147 /* 7148 * Detect special cases and adjust section sizes accordingly: 7149 * 1) .init.* may be embedded into .data sections 7150 * 2) .init.text.* may be out of [__init_begin, __init_end], 7151 * please refer to arch/tile/kernel/vmlinux.lds.S. 7152 * 3) .rodata.* may be embedded into .text or .data sections. 7153 */ 7154 #define adj_init_size(start, end, size, pos, adj) \ 7155 do { \ 7156 if (start <= pos && pos < end && size > adj) \ 7157 size -= adj; \ 7158 } while (0) 7159 7160 adj_init_size(__init_begin, __init_end, init_data_size, 7161 _sinittext, init_code_size); 7162 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 7163 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 7164 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 7165 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 7166 7167 #undef adj_init_size 7168 7169 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 7170 #ifdef CONFIG_HIGHMEM 7171 ", %luK highmem" 7172 #endif 7173 "%s%s)\n", 7174 nr_free_pages() << (PAGE_SHIFT - 10), 7175 physpages << (PAGE_SHIFT - 10), 7176 codesize >> 10, datasize >> 10, rosize >> 10, 7177 (init_data_size + init_code_size) >> 10, bss_size >> 10, 7178 (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT - 10), 7179 totalcma_pages << (PAGE_SHIFT - 10), 7180 #ifdef CONFIG_HIGHMEM 7181 totalhigh_pages << (PAGE_SHIFT - 10), 7182 #endif 7183 str ? ", " : "", str ? str : ""); 7184 } 7185 7186 /** 7187 * set_dma_reserve - set the specified number of pages reserved in the first zone 7188 * @new_dma_reserve: The number of pages to mark reserved 7189 * 7190 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 7191 * In the DMA zone, a significant percentage may be consumed by kernel image 7192 * and other unfreeable allocations which can skew the watermarks badly. This 7193 * function may optionally be used to account for unfreeable pages in the 7194 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 7195 * smaller per-cpu batchsize. 7196 */ 7197 void __init set_dma_reserve(unsigned long new_dma_reserve) 7198 { 7199 dma_reserve = new_dma_reserve; 7200 } 7201 7202 void __init free_area_init(unsigned long *zones_size) 7203 { 7204 zero_resv_unavail(); 7205 free_area_init_node(0, zones_size, 7206 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); 7207 } 7208 7209 static int page_alloc_cpu_dead(unsigned int cpu) 7210 { 7211 7212 lru_add_drain_cpu(cpu); 7213 drain_pages(cpu); 7214 7215 /* 7216 * Spill the event counters of the dead processor 7217 * into the current processors event counters. 7218 * This artificially elevates the count of the current 7219 * processor. 7220 */ 7221 vm_events_fold_cpu(cpu); 7222 7223 /* 7224 * Zero the differential counters of the dead processor 7225 * so that the vm statistics are consistent. 7226 * 7227 * This is only okay since the processor is dead and cannot 7228 * race with what we are doing. 7229 */ 7230 cpu_vm_stats_fold(cpu); 7231 return 0; 7232 } 7233 7234 void __init page_alloc_init(void) 7235 { 7236 int ret; 7237 7238 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD, 7239 "mm/page_alloc:dead", NULL, 7240 page_alloc_cpu_dead); 7241 WARN_ON(ret < 0); 7242 } 7243 7244 /* 7245 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 7246 * or min_free_kbytes changes. 7247 */ 7248 static void calculate_totalreserve_pages(void) 7249 { 7250 struct pglist_data *pgdat; 7251 unsigned long reserve_pages = 0; 7252 enum zone_type i, j; 7253 7254 for_each_online_pgdat(pgdat) { 7255 7256 pgdat->totalreserve_pages = 0; 7257 7258 for (i = 0; i < MAX_NR_ZONES; i++) { 7259 struct zone *zone = pgdat->node_zones + i; 7260 long max = 0; 7261 7262 /* Find valid and maximum lowmem_reserve in the zone */ 7263 for (j = i; j < MAX_NR_ZONES; j++) { 7264 if (zone->lowmem_reserve[j] > max) 7265 max = zone->lowmem_reserve[j]; 7266 } 7267 7268 /* we treat the high watermark as reserved pages. */ 7269 max += high_wmark_pages(zone); 7270 7271 if (max > zone->managed_pages) 7272 max = zone->managed_pages; 7273 7274 pgdat->totalreserve_pages += max; 7275 7276 reserve_pages += max; 7277 } 7278 } 7279 totalreserve_pages = reserve_pages; 7280 } 7281 7282 /* 7283 * setup_per_zone_lowmem_reserve - called whenever 7284 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 7285 * has a correct pages reserved value, so an adequate number of 7286 * pages are left in the zone after a successful __alloc_pages(). 7287 */ 7288 static void setup_per_zone_lowmem_reserve(void) 7289 { 7290 struct pglist_data *pgdat; 7291 enum zone_type j, idx; 7292 7293 for_each_online_pgdat(pgdat) { 7294 for (j = 0; j < MAX_NR_ZONES; j++) { 7295 struct zone *zone = pgdat->node_zones + j; 7296 unsigned long managed_pages = zone->managed_pages; 7297 7298 zone->lowmem_reserve[j] = 0; 7299 7300 idx = j; 7301 while (idx) { 7302 struct zone *lower_zone; 7303 7304 idx--; 7305 lower_zone = pgdat->node_zones + idx; 7306 7307 if (sysctl_lowmem_reserve_ratio[idx] < 1) { 7308 sysctl_lowmem_reserve_ratio[idx] = 0; 7309 lower_zone->lowmem_reserve[j] = 0; 7310 } else { 7311 lower_zone->lowmem_reserve[j] = 7312 managed_pages / sysctl_lowmem_reserve_ratio[idx]; 7313 } 7314 managed_pages += lower_zone->managed_pages; 7315 } 7316 } 7317 } 7318 7319 /* update totalreserve_pages */ 7320 calculate_totalreserve_pages(); 7321 } 7322 7323 static void __setup_per_zone_wmarks(void) 7324 { 7325 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 7326 unsigned long lowmem_pages = 0; 7327 struct zone *zone; 7328 unsigned long flags; 7329 7330 /* Calculate total number of !ZONE_HIGHMEM pages */ 7331 for_each_zone(zone) { 7332 if (!is_highmem(zone)) 7333 lowmem_pages += zone->managed_pages; 7334 } 7335 7336 for_each_zone(zone) { 7337 u64 tmp; 7338 7339 spin_lock_irqsave(&zone->lock, flags); 7340 tmp = (u64)pages_min * zone->managed_pages; 7341 do_div(tmp, lowmem_pages); 7342 if (is_highmem(zone)) { 7343 /* 7344 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 7345 * need highmem pages, so cap pages_min to a small 7346 * value here. 7347 * 7348 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 7349 * deltas control asynch page reclaim, and so should 7350 * not be capped for highmem. 7351 */ 7352 unsigned long min_pages; 7353 7354 min_pages = zone->managed_pages / 1024; 7355 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 7356 zone->watermark[WMARK_MIN] = min_pages; 7357 } else { 7358 /* 7359 * If it's a lowmem zone, reserve a number of pages 7360 * proportionate to the zone's size. 7361 */ 7362 zone->watermark[WMARK_MIN] = tmp; 7363 } 7364 7365 /* 7366 * Set the kswapd watermarks distance according to the 7367 * scale factor in proportion to available memory, but 7368 * ensure a minimum size on small systems. 7369 */ 7370 tmp = max_t(u64, tmp >> 2, 7371 mult_frac(zone->managed_pages, 7372 watermark_scale_factor, 10000)); 7373 7374 zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 7375 zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2; 7376 7377 spin_unlock_irqrestore(&zone->lock, flags); 7378 } 7379 7380 /* update totalreserve_pages */ 7381 calculate_totalreserve_pages(); 7382 } 7383 7384 /** 7385 * setup_per_zone_wmarks - called when min_free_kbytes changes 7386 * or when memory is hot-{added|removed} 7387 * 7388 * Ensures that the watermark[min,low,high] values for each zone are set 7389 * correctly with respect to min_free_kbytes. 7390 */ 7391 void setup_per_zone_wmarks(void) 7392 { 7393 static DEFINE_SPINLOCK(lock); 7394 7395 spin_lock(&lock); 7396 __setup_per_zone_wmarks(); 7397 spin_unlock(&lock); 7398 } 7399 7400 /* 7401 * Initialise min_free_kbytes. 7402 * 7403 * For small machines we want it small (128k min). For large machines 7404 * we want it large (64MB max). But it is not linear, because network 7405 * bandwidth does not increase linearly with machine size. We use 7406 * 7407 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 7408 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 7409 * 7410 * which yields 7411 * 7412 * 16MB: 512k 7413 * 32MB: 724k 7414 * 64MB: 1024k 7415 * 128MB: 1448k 7416 * 256MB: 2048k 7417 * 512MB: 2896k 7418 * 1024MB: 4096k 7419 * 2048MB: 5792k 7420 * 4096MB: 8192k 7421 * 8192MB: 11584k 7422 * 16384MB: 16384k 7423 */ 7424 int __meminit init_per_zone_wmark_min(void) 7425 { 7426 unsigned long lowmem_kbytes; 7427 int new_min_free_kbytes; 7428 7429 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 7430 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 7431 7432 if (new_min_free_kbytes > user_min_free_kbytes) { 7433 min_free_kbytes = new_min_free_kbytes; 7434 if (min_free_kbytes < 128) 7435 min_free_kbytes = 128; 7436 if (min_free_kbytes > 65536) 7437 min_free_kbytes = 65536; 7438 } else { 7439 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 7440 new_min_free_kbytes, user_min_free_kbytes); 7441 } 7442 setup_per_zone_wmarks(); 7443 refresh_zone_stat_thresholds(); 7444 setup_per_zone_lowmem_reserve(); 7445 7446 #ifdef CONFIG_NUMA 7447 setup_min_unmapped_ratio(); 7448 setup_min_slab_ratio(); 7449 #endif 7450 7451 return 0; 7452 } 7453 core_initcall(init_per_zone_wmark_min) 7454 7455 /* 7456 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 7457 * that we can call two helper functions whenever min_free_kbytes 7458 * changes. 7459 */ 7460 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 7461 void __user *buffer, size_t *length, loff_t *ppos) 7462 { 7463 int rc; 7464 7465 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7466 if (rc) 7467 return rc; 7468 7469 if (write) { 7470 user_min_free_kbytes = min_free_kbytes; 7471 setup_per_zone_wmarks(); 7472 } 7473 return 0; 7474 } 7475 7476 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 7477 void __user *buffer, size_t *length, loff_t *ppos) 7478 { 7479 int rc; 7480 7481 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7482 if (rc) 7483 return rc; 7484 7485 if (write) 7486 setup_per_zone_wmarks(); 7487 7488 return 0; 7489 } 7490 7491 #ifdef CONFIG_NUMA 7492 static void setup_min_unmapped_ratio(void) 7493 { 7494 pg_data_t *pgdat; 7495 struct zone *zone; 7496 7497 for_each_online_pgdat(pgdat) 7498 pgdat->min_unmapped_pages = 0; 7499 7500 for_each_zone(zone) 7501 zone->zone_pgdat->min_unmapped_pages += (zone->managed_pages * 7502 sysctl_min_unmapped_ratio) / 100; 7503 } 7504 7505 7506 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 7507 void __user *buffer, size_t *length, loff_t *ppos) 7508 { 7509 int rc; 7510 7511 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7512 if (rc) 7513 return rc; 7514 7515 setup_min_unmapped_ratio(); 7516 7517 return 0; 7518 } 7519 7520 static void setup_min_slab_ratio(void) 7521 { 7522 pg_data_t *pgdat; 7523 struct zone *zone; 7524 7525 for_each_online_pgdat(pgdat) 7526 pgdat->min_slab_pages = 0; 7527 7528 for_each_zone(zone) 7529 zone->zone_pgdat->min_slab_pages += (zone->managed_pages * 7530 sysctl_min_slab_ratio) / 100; 7531 } 7532 7533 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 7534 void __user *buffer, size_t *length, loff_t *ppos) 7535 { 7536 int rc; 7537 7538 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 7539 if (rc) 7540 return rc; 7541 7542 setup_min_slab_ratio(); 7543 7544 return 0; 7545 } 7546 #endif 7547 7548 /* 7549 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 7550 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 7551 * whenever sysctl_lowmem_reserve_ratio changes. 7552 * 7553 * The reserve ratio obviously has absolutely no relation with the 7554 * minimum watermarks. The lowmem reserve ratio can only make sense 7555 * if in function of the boot time zone sizes. 7556 */ 7557 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 7558 void __user *buffer, size_t *length, loff_t *ppos) 7559 { 7560 proc_dointvec_minmax(table, write, buffer, length, ppos); 7561 setup_per_zone_lowmem_reserve(); 7562 return 0; 7563 } 7564 7565 /* 7566 * percpu_pagelist_fraction - changes the pcp->high for each zone on each 7567 * cpu. It is the fraction of total pages in each zone that a hot per cpu 7568 * pagelist can have before it gets flushed back to buddy allocator. 7569 */ 7570 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, 7571 void __user *buffer, size_t *length, loff_t *ppos) 7572 { 7573 struct zone *zone; 7574 int old_percpu_pagelist_fraction; 7575 int ret; 7576 7577 mutex_lock(&pcp_batch_high_lock); 7578 old_percpu_pagelist_fraction = percpu_pagelist_fraction; 7579 7580 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 7581 if (!write || ret < 0) 7582 goto out; 7583 7584 /* Sanity checking to avoid pcp imbalance */ 7585 if (percpu_pagelist_fraction && 7586 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { 7587 percpu_pagelist_fraction = old_percpu_pagelist_fraction; 7588 ret = -EINVAL; 7589 goto out; 7590 } 7591 7592 /* No change? */ 7593 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) 7594 goto out; 7595 7596 for_each_populated_zone(zone) { 7597 unsigned int cpu; 7598 7599 for_each_possible_cpu(cpu) 7600 pageset_set_high_and_batch(zone, 7601 per_cpu_ptr(zone->pageset, cpu)); 7602 } 7603 out: 7604 mutex_unlock(&pcp_batch_high_lock); 7605 return ret; 7606 } 7607 7608 #ifdef CONFIG_NUMA 7609 int hashdist = HASHDIST_DEFAULT; 7610 7611 static int __init set_hashdist(char *str) 7612 { 7613 if (!str) 7614 return 0; 7615 hashdist = simple_strtoul(str, &str, 0); 7616 return 1; 7617 } 7618 __setup("hashdist=", set_hashdist); 7619 #endif 7620 7621 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 7622 /* 7623 * Returns the number of pages that arch has reserved but 7624 * is not known to alloc_large_system_hash(). 7625 */ 7626 static unsigned long __init arch_reserved_kernel_pages(void) 7627 { 7628 return 0; 7629 } 7630 #endif 7631 7632 /* 7633 * Adaptive scale is meant to reduce sizes of hash tables on large memory 7634 * machines. As memory size is increased the scale is also increased but at 7635 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 7636 * quadruples the scale is increased by one, which means the size of hash table 7637 * only doubles, instead of quadrupling as well. 7638 * Because 32-bit systems cannot have large physical memory, where this scaling 7639 * makes sense, it is disabled on such platforms. 7640 */ 7641 #if __BITS_PER_LONG > 32 7642 #define ADAPT_SCALE_BASE (64ul << 30) 7643 #define ADAPT_SCALE_SHIFT 2 7644 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 7645 #endif 7646 7647 /* 7648 * allocate a large system hash table from bootmem 7649 * - it is assumed that the hash table must contain an exact power-of-2 7650 * quantity of entries 7651 * - limit is the number of hash buckets, not the total allocation size 7652 */ 7653 void *__init alloc_large_system_hash(const char *tablename, 7654 unsigned long bucketsize, 7655 unsigned long numentries, 7656 int scale, 7657 int flags, 7658 unsigned int *_hash_shift, 7659 unsigned int *_hash_mask, 7660 unsigned long low_limit, 7661 unsigned long high_limit) 7662 { 7663 unsigned long long max = high_limit; 7664 unsigned long log2qty, size; 7665 void *table = NULL; 7666 gfp_t gfp_flags; 7667 7668 /* allow the kernel cmdline to have a say */ 7669 if (!numentries) { 7670 /* round applicable memory size up to nearest megabyte */ 7671 numentries = nr_kernel_pages; 7672 numentries -= arch_reserved_kernel_pages(); 7673 7674 /* It isn't necessary when PAGE_SIZE >= 1MB */ 7675 if (PAGE_SHIFT < 20) 7676 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 7677 7678 #if __BITS_PER_LONG > 32 7679 if (!high_limit) { 7680 unsigned long adapt; 7681 7682 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 7683 adapt <<= ADAPT_SCALE_SHIFT) 7684 scale++; 7685 } 7686 #endif 7687 7688 /* limit to 1 bucket per 2^scale bytes of low memory */ 7689 if (scale > PAGE_SHIFT) 7690 numentries >>= (scale - PAGE_SHIFT); 7691 else 7692 numentries <<= (PAGE_SHIFT - scale); 7693 7694 /* Make sure we've got at least a 0-order allocation.. */ 7695 if (unlikely(flags & HASH_SMALL)) { 7696 /* Makes no sense without HASH_EARLY */ 7697 WARN_ON(!(flags & HASH_EARLY)); 7698 if (!(numentries >> *_hash_shift)) { 7699 numentries = 1UL << *_hash_shift; 7700 BUG_ON(!numentries); 7701 } 7702 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 7703 numentries = PAGE_SIZE / bucketsize; 7704 } 7705 numentries = roundup_pow_of_two(numentries); 7706 7707 /* limit allocation size to 1/16 total memory by default */ 7708 if (max == 0) { 7709 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 7710 do_div(max, bucketsize); 7711 } 7712 max = min(max, 0x80000000ULL); 7713 7714 if (numentries < low_limit) 7715 numentries = low_limit; 7716 if (numentries > max) 7717 numentries = max; 7718 7719 log2qty = ilog2(numentries); 7720 7721 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 7722 do { 7723 size = bucketsize << log2qty; 7724 if (flags & HASH_EARLY) { 7725 if (flags & HASH_ZERO) 7726 table = memblock_alloc_nopanic(size, 7727 SMP_CACHE_BYTES); 7728 else 7729 table = memblock_alloc_raw(size, 7730 SMP_CACHE_BYTES); 7731 } else if (hashdist) { 7732 table = __vmalloc(size, gfp_flags, PAGE_KERNEL); 7733 } else { 7734 /* 7735 * If bucketsize is not a power-of-two, we may free 7736 * some pages at the end of hash table which 7737 * alloc_pages_exact() automatically does 7738 */ 7739 if (get_order(size) < MAX_ORDER) { 7740 table = alloc_pages_exact(size, gfp_flags); 7741 kmemleak_alloc(table, size, 1, gfp_flags); 7742 } 7743 } 7744 } while (!table && size > PAGE_SIZE && --log2qty); 7745 7746 if (!table) 7747 panic("Failed to allocate %s hash table\n", tablename); 7748 7749 pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n", 7750 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size); 7751 7752 if (_hash_shift) 7753 *_hash_shift = log2qty; 7754 if (_hash_mask) 7755 *_hash_mask = (1 << log2qty) - 1; 7756 7757 return table; 7758 } 7759 7760 /* 7761 * This function checks whether pageblock includes unmovable pages or not. 7762 * If @count is not zero, it is okay to include less @count unmovable pages 7763 * 7764 * PageLRU check without isolation or lru_lock could race so that 7765 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable 7766 * check without lock_page also may miss some movable non-lru pages at 7767 * race condition. So you can't expect this function should be exact. 7768 */ 7769 bool has_unmovable_pages(struct zone *zone, struct page *page, int count, 7770 int migratetype, 7771 bool skip_hwpoisoned_pages) 7772 { 7773 unsigned long pfn, iter, found; 7774 7775 /* 7776 * TODO we could make this much more efficient by not checking every 7777 * page in the range if we know all of them are in MOVABLE_ZONE and 7778 * that the movable zone guarantees that pages are migratable but 7779 * the later is not the case right now unfortunatelly. E.g. movablecore 7780 * can still lead to having bootmem allocations in zone_movable. 7781 */ 7782 7783 /* 7784 * CMA allocations (alloc_contig_range) really need to mark isolate 7785 * CMA pageblocks even when they are not movable in fact so consider 7786 * them movable here. 7787 */ 7788 if (is_migrate_cma(migratetype) && 7789 is_migrate_cma(get_pageblock_migratetype(page))) 7790 return false; 7791 7792 pfn = page_to_pfn(page); 7793 for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) { 7794 unsigned long check = pfn + iter; 7795 7796 if (!pfn_valid_within(check)) 7797 continue; 7798 7799 page = pfn_to_page(check); 7800 7801 if (PageReserved(page)) 7802 goto unmovable; 7803 7804 /* 7805 * If the zone is movable and we have ruled out all reserved 7806 * pages then it should be reasonably safe to assume the rest 7807 * is movable. 7808 */ 7809 if (zone_idx(zone) == ZONE_MOVABLE) 7810 continue; 7811 7812 /* 7813 * Hugepages are not in LRU lists, but they're movable. 7814 * We need not scan over tail pages bacause we don't 7815 * handle each tail page individually in migration. 7816 */ 7817 if (PageHuge(page)) { 7818 struct page *head = compound_head(page); 7819 unsigned int skip_pages; 7820 7821 if (!hugepage_migration_supported(page_hstate(head))) 7822 goto unmovable; 7823 7824 skip_pages = (1 << compound_order(head)) - (page - head); 7825 iter += skip_pages - 1; 7826 continue; 7827 } 7828 7829 /* 7830 * We can't use page_count without pin a page 7831 * because another CPU can free compound page. 7832 * This check already skips compound tails of THP 7833 * because their page->_refcount is zero at all time. 7834 */ 7835 if (!page_ref_count(page)) { 7836 if (PageBuddy(page)) 7837 iter += (1 << page_order(page)) - 1; 7838 continue; 7839 } 7840 7841 /* 7842 * The HWPoisoned page may be not in buddy system, and 7843 * page_count() is not 0. 7844 */ 7845 if (skip_hwpoisoned_pages && PageHWPoison(page)) 7846 continue; 7847 7848 if (__PageMovable(page)) 7849 continue; 7850 7851 if (!PageLRU(page)) 7852 found++; 7853 /* 7854 * If there are RECLAIMABLE pages, we need to check 7855 * it. But now, memory offline itself doesn't call 7856 * shrink_node_slabs() and it still to be fixed. 7857 */ 7858 /* 7859 * If the page is not RAM, page_count()should be 0. 7860 * we don't need more check. This is an _used_ not-movable page. 7861 * 7862 * The problematic thing here is PG_reserved pages. PG_reserved 7863 * is set to both of a memory hole page and a _used_ kernel 7864 * page at boot. 7865 */ 7866 if (found > count) 7867 goto unmovable; 7868 } 7869 return false; 7870 unmovable: 7871 WARN_ON_ONCE(zone_idx(zone) == ZONE_MOVABLE); 7872 return true; 7873 } 7874 7875 #if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA) 7876 7877 static unsigned long pfn_max_align_down(unsigned long pfn) 7878 { 7879 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, 7880 pageblock_nr_pages) - 1); 7881 } 7882 7883 static unsigned long pfn_max_align_up(unsigned long pfn) 7884 { 7885 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, 7886 pageblock_nr_pages)); 7887 } 7888 7889 /* [start, end) must belong to a single zone. */ 7890 static int __alloc_contig_migrate_range(struct compact_control *cc, 7891 unsigned long start, unsigned long end) 7892 { 7893 /* This function is based on compact_zone() from compaction.c. */ 7894 unsigned long nr_reclaimed; 7895 unsigned long pfn = start; 7896 unsigned int tries = 0; 7897 int ret = 0; 7898 7899 migrate_prep(); 7900 7901 while (pfn < end || !list_empty(&cc->migratepages)) { 7902 if (fatal_signal_pending(current)) { 7903 ret = -EINTR; 7904 break; 7905 } 7906 7907 if (list_empty(&cc->migratepages)) { 7908 cc->nr_migratepages = 0; 7909 pfn = isolate_migratepages_range(cc, pfn, end); 7910 if (!pfn) { 7911 ret = -EINTR; 7912 break; 7913 } 7914 tries = 0; 7915 } else if (++tries == 5) { 7916 ret = ret < 0 ? ret : -EBUSY; 7917 break; 7918 } 7919 7920 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 7921 &cc->migratepages); 7922 cc->nr_migratepages -= nr_reclaimed; 7923 7924 ret = migrate_pages(&cc->migratepages, alloc_migrate_target, 7925 NULL, 0, cc->mode, MR_CONTIG_RANGE); 7926 } 7927 if (ret < 0) { 7928 putback_movable_pages(&cc->migratepages); 7929 return ret; 7930 } 7931 return 0; 7932 } 7933 7934 /** 7935 * alloc_contig_range() -- tries to allocate given range of pages 7936 * @start: start PFN to allocate 7937 * @end: one-past-the-last PFN to allocate 7938 * @migratetype: migratetype of the underlaying pageblocks (either 7939 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 7940 * in range must have the same migratetype and it must 7941 * be either of the two. 7942 * @gfp_mask: GFP mask to use during compaction 7943 * 7944 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 7945 * aligned. The PFN range must belong to a single zone. 7946 * 7947 * The first thing this routine does is attempt to MIGRATE_ISOLATE all 7948 * pageblocks in the range. Once isolated, the pageblocks should not 7949 * be modified by others. 7950 * 7951 * Returns zero on success or negative error code. On success all 7952 * pages which PFN is in [start, end) are allocated for the caller and 7953 * need to be freed with free_contig_range(). 7954 */ 7955 int alloc_contig_range(unsigned long start, unsigned long end, 7956 unsigned migratetype, gfp_t gfp_mask) 7957 { 7958 unsigned long outer_start, outer_end; 7959 unsigned int order; 7960 int ret = 0; 7961 7962 struct compact_control cc = { 7963 .nr_migratepages = 0, 7964 .order = -1, 7965 .zone = page_zone(pfn_to_page(start)), 7966 .mode = MIGRATE_SYNC, 7967 .ignore_skip_hint = true, 7968 .no_set_skip_hint = true, 7969 .gfp_mask = current_gfp_context(gfp_mask), 7970 }; 7971 INIT_LIST_HEAD(&cc.migratepages); 7972 7973 /* 7974 * What we do here is we mark all pageblocks in range as 7975 * MIGRATE_ISOLATE. Because pageblock and max order pages may 7976 * have different sizes, and due to the way page allocator 7977 * work, we align the range to biggest of the two pages so 7978 * that page allocator won't try to merge buddies from 7979 * different pageblocks and change MIGRATE_ISOLATE to some 7980 * other migration type. 7981 * 7982 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 7983 * migrate the pages from an unaligned range (ie. pages that 7984 * we are interested in). This will put all the pages in 7985 * range back to page allocator as MIGRATE_ISOLATE. 7986 * 7987 * When this is done, we take the pages in range from page 7988 * allocator removing them from the buddy system. This way 7989 * page allocator will never consider using them. 7990 * 7991 * This lets us mark the pageblocks back as 7992 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 7993 * aligned range but not in the unaligned, original range are 7994 * put back to page allocator so that buddy can use them. 7995 */ 7996 7997 ret = start_isolate_page_range(pfn_max_align_down(start), 7998 pfn_max_align_up(end), migratetype, 7999 false); 8000 if (ret) 8001 return ret; 8002 8003 /* 8004 * In case of -EBUSY, we'd like to know which page causes problem. 8005 * So, just fall through. test_pages_isolated() has a tracepoint 8006 * which will report the busy page. 8007 * 8008 * It is possible that busy pages could become available before 8009 * the call to test_pages_isolated, and the range will actually be 8010 * allocated. So, if we fall through be sure to clear ret so that 8011 * -EBUSY is not accidentally used or returned to caller. 8012 */ 8013 ret = __alloc_contig_migrate_range(&cc, start, end); 8014 if (ret && ret != -EBUSY) 8015 goto done; 8016 ret =0; 8017 8018 /* 8019 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 8020 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 8021 * more, all pages in [start, end) are free in page allocator. 8022 * What we are going to do is to allocate all pages from 8023 * [start, end) (that is remove them from page allocator). 8024 * 8025 * The only problem is that pages at the beginning and at the 8026 * end of interesting range may be not aligned with pages that 8027 * page allocator holds, ie. they can be part of higher order 8028 * pages. Because of this, we reserve the bigger range and 8029 * once this is done free the pages we are not interested in. 8030 * 8031 * We don't have to hold zone->lock here because the pages are 8032 * isolated thus they won't get removed from buddy. 8033 */ 8034 8035 lru_add_drain_all(); 8036 drain_all_pages(cc.zone); 8037 8038 order = 0; 8039 outer_start = start; 8040 while (!PageBuddy(pfn_to_page(outer_start))) { 8041 if (++order >= MAX_ORDER) { 8042 outer_start = start; 8043 break; 8044 } 8045 outer_start &= ~0UL << order; 8046 } 8047 8048 if (outer_start != start) { 8049 order = page_order(pfn_to_page(outer_start)); 8050 8051 /* 8052 * outer_start page could be small order buddy page and 8053 * it doesn't include start page. Adjust outer_start 8054 * in this case to report failed page properly 8055 * on tracepoint in test_pages_isolated() 8056 */ 8057 if (outer_start + (1UL << order) <= start) 8058 outer_start = start; 8059 } 8060 8061 /* Make sure the range is really isolated. */ 8062 if (test_pages_isolated(outer_start, end, false)) { 8063 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n", 8064 __func__, outer_start, end); 8065 ret = -EBUSY; 8066 goto done; 8067 } 8068 8069 /* Grab isolated pages from freelists. */ 8070 outer_end = isolate_freepages_range(&cc, outer_start, end); 8071 if (!outer_end) { 8072 ret = -EBUSY; 8073 goto done; 8074 } 8075 8076 /* Free head and tail (if any) */ 8077 if (start != outer_start) 8078 free_contig_range(outer_start, start - outer_start); 8079 if (end != outer_end) 8080 free_contig_range(end, outer_end - end); 8081 8082 done: 8083 undo_isolate_page_range(pfn_max_align_down(start), 8084 pfn_max_align_up(end), migratetype); 8085 return ret; 8086 } 8087 8088 void free_contig_range(unsigned long pfn, unsigned nr_pages) 8089 { 8090 unsigned int count = 0; 8091 8092 for (; nr_pages--; pfn++) { 8093 struct page *page = pfn_to_page(pfn); 8094 8095 count += page_count(page) != 1; 8096 __free_page(page); 8097 } 8098 WARN(count != 0, "%d pages are still in use!\n", count); 8099 } 8100 #endif 8101 8102 #ifdef CONFIG_MEMORY_HOTPLUG 8103 /* 8104 * The zone indicated has a new number of managed_pages; batch sizes and percpu 8105 * page high values need to be recalulated. 8106 */ 8107 void __meminit zone_pcp_update(struct zone *zone) 8108 { 8109 unsigned cpu; 8110 mutex_lock(&pcp_batch_high_lock); 8111 for_each_possible_cpu(cpu) 8112 pageset_set_high_and_batch(zone, 8113 per_cpu_ptr(zone->pageset, cpu)); 8114 mutex_unlock(&pcp_batch_high_lock); 8115 } 8116 #endif 8117 8118 void zone_pcp_reset(struct zone *zone) 8119 { 8120 unsigned long flags; 8121 int cpu; 8122 struct per_cpu_pageset *pset; 8123 8124 /* avoid races with drain_pages() */ 8125 local_irq_save(flags); 8126 if (zone->pageset != &boot_pageset) { 8127 for_each_online_cpu(cpu) { 8128 pset = per_cpu_ptr(zone->pageset, cpu); 8129 drain_zonestat(zone, pset); 8130 } 8131 free_percpu(zone->pageset); 8132 zone->pageset = &boot_pageset; 8133 } 8134 local_irq_restore(flags); 8135 } 8136 8137 #ifdef CONFIG_MEMORY_HOTREMOVE 8138 /* 8139 * All pages in the range must be in a single zone and isolated 8140 * before calling this. 8141 */ 8142 void 8143 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 8144 { 8145 struct page *page; 8146 struct zone *zone; 8147 unsigned int order, i; 8148 unsigned long pfn; 8149 unsigned long flags; 8150 /* find the first valid pfn */ 8151 for (pfn = start_pfn; pfn < end_pfn; pfn++) 8152 if (pfn_valid(pfn)) 8153 break; 8154 if (pfn == end_pfn) 8155 return; 8156 offline_mem_sections(pfn, end_pfn); 8157 zone = page_zone(pfn_to_page(pfn)); 8158 spin_lock_irqsave(&zone->lock, flags); 8159 pfn = start_pfn; 8160 while (pfn < end_pfn) { 8161 if (!pfn_valid(pfn)) { 8162 pfn++; 8163 continue; 8164 } 8165 page = pfn_to_page(pfn); 8166 /* 8167 * The HWPoisoned page may be not in buddy system, and 8168 * page_count() is not 0. 8169 */ 8170 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 8171 pfn++; 8172 SetPageReserved(page); 8173 continue; 8174 } 8175 8176 BUG_ON(page_count(page)); 8177 BUG_ON(!PageBuddy(page)); 8178 order = page_order(page); 8179 #ifdef CONFIG_DEBUG_VM 8180 pr_info("remove from free list %lx %d %lx\n", 8181 pfn, 1 << order, end_pfn); 8182 #endif 8183 list_del(&page->lru); 8184 rmv_page_order(page); 8185 zone->free_area[order].nr_free--; 8186 for (i = 0; i < (1 << order); i++) 8187 SetPageReserved((page+i)); 8188 pfn += (1 << order); 8189 } 8190 spin_unlock_irqrestore(&zone->lock, flags); 8191 } 8192 #endif 8193 8194 bool is_free_buddy_page(struct page *page) 8195 { 8196 struct zone *zone = page_zone(page); 8197 unsigned long pfn = page_to_pfn(page); 8198 unsigned long flags; 8199 unsigned int order; 8200 8201 spin_lock_irqsave(&zone->lock, flags); 8202 for (order = 0; order < MAX_ORDER; order++) { 8203 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8204 8205 if (PageBuddy(page_head) && page_order(page_head) >= order) 8206 break; 8207 } 8208 spin_unlock_irqrestore(&zone->lock, flags); 8209 8210 return order < MAX_ORDER; 8211 } 8212 8213 #ifdef CONFIG_MEMORY_FAILURE 8214 /* 8215 * Set PG_hwpoison flag if a given page is confirmed to be a free page. This 8216 * test is performed under the zone lock to prevent a race against page 8217 * allocation. 8218 */ 8219 bool set_hwpoison_free_buddy_page(struct page *page) 8220 { 8221 struct zone *zone = page_zone(page); 8222 unsigned long pfn = page_to_pfn(page); 8223 unsigned long flags; 8224 unsigned int order; 8225 bool hwpoisoned = false; 8226 8227 spin_lock_irqsave(&zone->lock, flags); 8228 for (order = 0; order < MAX_ORDER; order++) { 8229 struct page *page_head = page - (pfn & ((1 << order) - 1)); 8230 8231 if (PageBuddy(page_head) && page_order(page_head) >= order) { 8232 if (!TestSetPageHWPoison(page)) 8233 hwpoisoned = true; 8234 break; 8235 } 8236 } 8237 spin_unlock_irqrestore(&zone->lock, flags); 8238 8239 return hwpoisoned; 8240 } 8241 #endif 8242