1 /* 2 * linux/mm/page_alloc.c 3 * 4 * Manages the free list, the system allocates free pages here. 5 * Note that kmalloc() lives in slab.c 6 * 7 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 8 * Swap reorganised 29.12.95, Stephen Tweedie 9 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 10 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 11 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 12 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 13 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 14 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 15 */ 16 17 #include <linux/stddef.h> 18 #include <linux/mm.h> 19 #include <linux/swap.h> 20 #include <linux/interrupt.h> 21 #include <linux/pagemap.h> 22 #include <linux/jiffies.h> 23 #include <linux/bootmem.h> 24 #include <linux/memblock.h> 25 #include <linux/compiler.h> 26 #include <linux/kernel.h> 27 #include <linux/kmemcheck.h> 28 #include <linux/kasan.h> 29 #include <linux/module.h> 30 #include <linux/suspend.h> 31 #include <linux/pagevec.h> 32 #include <linux/blkdev.h> 33 #include <linux/slab.h> 34 #include <linux/ratelimit.h> 35 #include <linux/oom.h> 36 #include <linux/notifier.h> 37 #include <linux/topology.h> 38 #include <linux/sysctl.h> 39 #include <linux/cpu.h> 40 #include <linux/cpuset.h> 41 #include <linux/memory_hotplug.h> 42 #include <linux/nodemask.h> 43 #include <linux/vmalloc.h> 44 #include <linux/vmstat.h> 45 #include <linux/mempolicy.h> 46 #include <linux/memremap.h> 47 #include <linux/stop_machine.h> 48 #include <linux/sort.h> 49 #include <linux/pfn.h> 50 #include <linux/backing-dev.h> 51 #include <linux/fault-inject.h> 52 #include <linux/page-isolation.h> 53 #include <linux/page_ext.h> 54 #include <linux/debugobjects.h> 55 #include <linux/kmemleak.h> 56 #include <linux/compaction.h> 57 #include <trace/events/kmem.h> 58 #include <linux/prefetch.h> 59 #include <linux/mm_inline.h> 60 #include <linux/migrate.h> 61 #include <linux/page_ext.h> 62 #include <linux/hugetlb.h> 63 #include <linux/sched/rt.h> 64 #include <linux/page_owner.h> 65 #include <linux/kthread.h> 66 #include <linux/memcontrol.h> 67 68 #include <asm/sections.h> 69 #include <asm/tlbflush.h> 70 #include <asm/div64.h> 71 #include "internal.h" 72 73 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 74 static DEFINE_MUTEX(pcp_batch_high_lock); 75 #define MIN_PERCPU_PAGELIST_FRACTION (8) 76 77 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 78 DEFINE_PER_CPU(int, numa_node); 79 EXPORT_PER_CPU_SYMBOL(numa_node); 80 #endif 81 82 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 83 /* 84 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 85 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 86 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 87 * defined in <linux/topology.h>. 88 */ 89 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 90 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 91 int _node_numa_mem_[MAX_NUMNODES]; 92 #endif 93 94 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 95 volatile unsigned long latent_entropy __latent_entropy; 96 EXPORT_SYMBOL(latent_entropy); 97 #endif 98 99 /* 100 * Array of node states. 101 */ 102 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 103 [N_POSSIBLE] = NODE_MASK_ALL, 104 [N_ONLINE] = { { [0] = 1UL } }, 105 #ifndef CONFIG_NUMA 106 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 107 #ifdef CONFIG_HIGHMEM 108 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 109 #endif 110 #ifdef CONFIG_MOVABLE_NODE 111 [N_MEMORY] = { { [0] = 1UL } }, 112 #endif 113 [N_CPU] = { { [0] = 1UL } }, 114 #endif /* NUMA */ 115 }; 116 EXPORT_SYMBOL(node_states); 117 118 /* Protect totalram_pages and zone->managed_pages */ 119 static DEFINE_SPINLOCK(managed_page_count_lock); 120 121 unsigned long totalram_pages __read_mostly; 122 unsigned long totalreserve_pages __read_mostly; 123 unsigned long totalcma_pages __read_mostly; 124 125 int percpu_pagelist_fraction; 126 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 127 128 /* 129 * A cached value of the page's pageblock's migratetype, used when the page is 130 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 131 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 132 * Also the migratetype set in the page does not necessarily match the pcplist 133 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 134 * other index - this ensures that it will be put on the correct CMA freelist. 135 */ 136 static inline int get_pcppage_migratetype(struct page *page) 137 { 138 return page->index; 139 } 140 141 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 142 { 143 page->index = migratetype; 144 } 145 146 #ifdef CONFIG_PM_SLEEP 147 /* 148 * The following functions are used by the suspend/hibernate code to temporarily 149 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 150 * while devices are suspended. To avoid races with the suspend/hibernate code, 151 * they should always be called with pm_mutex held (gfp_allowed_mask also should 152 * only be modified with pm_mutex held, unless the suspend/hibernate code is 153 * guaranteed not to run in parallel with that modification). 154 */ 155 156 static gfp_t saved_gfp_mask; 157 158 void pm_restore_gfp_mask(void) 159 { 160 WARN_ON(!mutex_is_locked(&pm_mutex)); 161 if (saved_gfp_mask) { 162 gfp_allowed_mask = saved_gfp_mask; 163 saved_gfp_mask = 0; 164 } 165 } 166 167 void pm_restrict_gfp_mask(void) 168 { 169 WARN_ON(!mutex_is_locked(&pm_mutex)); 170 WARN_ON(saved_gfp_mask); 171 saved_gfp_mask = gfp_allowed_mask; 172 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 173 } 174 175 bool pm_suspended_storage(void) 176 { 177 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 178 return false; 179 return true; 180 } 181 #endif /* CONFIG_PM_SLEEP */ 182 183 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 184 unsigned int pageblock_order __read_mostly; 185 #endif 186 187 static void __free_pages_ok(struct page *page, unsigned int order); 188 189 /* 190 * results with 256, 32 in the lowmem_reserve sysctl: 191 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 192 * 1G machine -> (16M dma, 784M normal, 224M high) 193 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 194 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 195 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 196 * 197 * TBD: should special case ZONE_DMA32 machines here - in those we normally 198 * don't need any ZONE_NORMAL reservation 199 */ 200 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = { 201 #ifdef CONFIG_ZONE_DMA 202 256, 203 #endif 204 #ifdef CONFIG_ZONE_DMA32 205 256, 206 #endif 207 #ifdef CONFIG_HIGHMEM 208 32, 209 #endif 210 32, 211 }; 212 213 EXPORT_SYMBOL(totalram_pages); 214 215 static char * const zone_names[MAX_NR_ZONES] = { 216 #ifdef CONFIG_ZONE_DMA 217 "DMA", 218 #endif 219 #ifdef CONFIG_ZONE_DMA32 220 "DMA32", 221 #endif 222 "Normal", 223 #ifdef CONFIG_HIGHMEM 224 "HighMem", 225 #endif 226 "Movable", 227 #ifdef CONFIG_ZONE_DEVICE 228 "Device", 229 #endif 230 }; 231 232 char * const migratetype_names[MIGRATE_TYPES] = { 233 "Unmovable", 234 "Movable", 235 "Reclaimable", 236 "HighAtomic", 237 #ifdef CONFIG_CMA 238 "CMA", 239 #endif 240 #ifdef CONFIG_MEMORY_ISOLATION 241 "Isolate", 242 #endif 243 }; 244 245 compound_page_dtor * const compound_page_dtors[] = { 246 NULL, 247 free_compound_page, 248 #ifdef CONFIG_HUGETLB_PAGE 249 free_huge_page, 250 #endif 251 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 252 free_transhuge_page, 253 #endif 254 }; 255 256 int min_free_kbytes = 1024; 257 int user_min_free_kbytes = -1; 258 int watermark_scale_factor = 10; 259 260 static unsigned long __meminitdata nr_kernel_pages; 261 static unsigned long __meminitdata nr_all_pages; 262 static unsigned long __meminitdata dma_reserve; 263 264 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 265 static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES]; 266 static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES]; 267 static unsigned long __initdata required_kernelcore; 268 static unsigned long __initdata required_movablecore; 269 static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES]; 270 static bool mirrored_kernelcore; 271 272 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 273 int movable_zone; 274 EXPORT_SYMBOL(movable_zone); 275 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 276 277 #if MAX_NUMNODES > 1 278 int nr_node_ids __read_mostly = MAX_NUMNODES; 279 int nr_online_nodes __read_mostly = 1; 280 EXPORT_SYMBOL(nr_node_ids); 281 EXPORT_SYMBOL(nr_online_nodes); 282 #endif 283 284 int page_group_by_mobility_disabled __read_mostly; 285 286 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 287 static inline void reset_deferred_meminit(pg_data_t *pgdat) 288 { 289 pgdat->first_deferred_pfn = ULONG_MAX; 290 } 291 292 /* Returns true if the struct page for the pfn is uninitialised */ 293 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 294 { 295 int nid = early_pfn_to_nid(pfn); 296 297 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 298 return true; 299 300 return false; 301 } 302 303 /* 304 * Returns false when the remaining initialisation should be deferred until 305 * later in the boot cycle when it can be parallelised. 306 */ 307 static inline bool update_defer_init(pg_data_t *pgdat, 308 unsigned long pfn, unsigned long zone_end, 309 unsigned long *nr_initialised) 310 { 311 unsigned long max_initialise; 312 313 /* Always populate low zones for address-contrained allocations */ 314 if (zone_end < pgdat_end_pfn(pgdat)) 315 return true; 316 /* 317 * Initialise at least 2G of a node but also take into account that 318 * two large system hashes that can take up 1GB for 0.25TB/node. 319 */ 320 max_initialise = max(2UL << (30 - PAGE_SHIFT), 321 (pgdat->node_spanned_pages >> 8)); 322 323 (*nr_initialised)++; 324 if ((*nr_initialised > max_initialise) && 325 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 326 pgdat->first_deferred_pfn = pfn; 327 return false; 328 } 329 330 return true; 331 } 332 #else 333 static inline void reset_deferred_meminit(pg_data_t *pgdat) 334 { 335 } 336 337 static inline bool early_page_uninitialised(unsigned long pfn) 338 { 339 return false; 340 } 341 342 static inline bool update_defer_init(pg_data_t *pgdat, 343 unsigned long pfn, unsigned long zone_end, 344 unsigned long *nr_initialised) 345 { 346 return true; 347 } 348 #endif 349 350 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 351 static inline unsigned long *get_pageblock_bitmap(struct page *page, 352 unsigned long pfn) 353 { 354 #ifdef CONFIG_SPARSEMEM 355 return __pfn_to_section(pfn)->pageblock_flags; 356 #else 357 return page_zone(page)->pageblock_flags; 358 #endif /* CONFIG_SPARSEMEM */ 359 } 360 361 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn) 362 { 363 #ifdef CONFIG_SPARSEMEM 364 pfn &= (PAGES_PER_SECTION-1); 365 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 366 #else 367 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 368 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 369 #endif /* CONFIG_SPARSEMEM */ 370 } 371 372 /** 373 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 374 * @page: The page within the block of interest 375 * @pfn: The target page frame number 376 * @end_bitidx: The last bit of interest to retrieve 377 * @mask: mask of bits that the caller is interested in 378 * 379 * Return: pageblock_bits flags 380 */ 381 static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page, 382 unsigned long pfn, 383 unsigned long end_bitidx, 384 unsigned long mask) 385 { 386 unsigned long *bitmap; 387 unsigned long bitidx, word_bitidx; 388 unsigned long word; 389 390 bitmap = get_pageblock_bitmap(page, pfn); 391 bitidx = pfn_to_bitidx(page, pfn); 392 word_bitidx = bitidx / BITS_PER_LONG; 393 bitidx &= (BITS_PER_LONG-1); 394 395 word = bitmap[word_bitidx]; 396 bitidx += end_bitidx; 397 return (word >> (BITS_PER_LONG - bitidx - 1)) & mask; 398 } 399 400 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, 401 unsigned long end_bitidx, 402 unsigned long mask) 403 { 404 return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask); 405 } 406 407 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn) 408 { 409 return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK); 410 } 411 412 /** 413 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 414 * @page: The page within the block of interest 415 * @flags: The flags to set 416 * @pfn: The target page frame number 417 * @end_bitidx: The last bit of interest 418 * @mask: mask of bits that the caller is interested in 419 */ 420 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 421 unsigned long pfn, 422 unsigned long end_bitidx, 423 unsigned long mask) 424 { 425 unsigned long *bitmap; 426 unsigned long bitidx, word_bitidx; 427 unsigned long old_word, word; 428 429 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 430 431 bitmap = get_pageblock_bitmap(page, pfn); 432 bitidx = pfn_to_bitidx(page, pfn); 433 word_bitidx = bitidx / BITS_PER_LONG; 434 bitidx &= (BITS_PER_LONG-1); 435 436 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 437 438 bitidx += end_bitidx; 439 mask <<= (BITS_PER_LONG - bitidx - 1); 440 flags <<= (BITS_PER_LONG - bitidx - 1); 441 442 word = READ_ONCE(bitmap[word_bitidx]); 443 for (;;) { 444 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 445 if (word == old_word) 446 break; 447 word = old_word; 448 } 449 } 450 451 void set_pageblock_migratetype(struct page *page, int migratetype) 452 { 453 if (unlikely(page_group_by_mobility_disabled && 454 migratetype < MIGRATE_PCPTYPES)) 455 migratetype = MIGRATE_UNMOVABLE; 456 457 set_pageblock_flags_group(page, (unsigned long)migratetype, 458 PB_migrate, PB_migrate_end); 459 } 460 461 #ifdef CONFIG_DEBUG_VM 462 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 463 { 464 int ret = 0; 465 unsigned seq; 466 unsigned long pfn = page_to_pfn(page); 467 unsigned long sp, start_pfn; 468 469 do { 470 seq = zone_span_seqbegin(zone); 471 start_pfn = zone->zone_start_pfn; 472 sp = zone->spanned_pages; 473 if (!zone_spans_pfn(zone, pfn)) 474 ret = 1; 475 } while (zone_span_seqretry(zone, seq)); 476 477 if (ret) 478 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 479 pfn, zone_to_nid(zone), zone->name, 480 start_pfn, start_pfn + sp); 481 482 return ret; 483 } 484 485 static int page_is_consistent(struct zone *zone, struct page *page) 486 { 487 if (!pfn_valid_within(page_to_pfn(page))) 488 return 0; 489 if (zone != page_zone(page)) 490 return 0; 491 492 return 1; 493 } 494 /* 495 * Temporary debugging check for pages not lying within a given zone. 496 */ 497 static int bad_range(struct zone *zone, struct page *page) 498 { 499 if (page_outside_zone_boundaries(zone, page)) 500 return 1; 501 if (!page_is_consistent(zone, page)) 502 return 1; 503 504 return 0; 505 } 506 #else 507 static inline int bad_range(struct zone *zone, struct page *page) 508 { 509 return 0; 510 } 511 #endif 512 513 static void bad_page(struct page *page, const char *reason, 514 unsigned long bad_flags) 515 { 516 static unsigned long resume; 517 static unsigned long nr_shown; 518 static unsigned long nr_unshown; 519 520 /* 521 * Allow a burst of 60 reports, then keep quiet for that minute; 522 * or allow a steady drip of one report per second. 523 */ 524 if (nr_shown == 60) { 525 if (time_before(jiffies, resume)) { 526 nr_unshown++; 527 goto out; 528 } 529 if (nr_unshown) { 530 pr_alert( 531 "BUG: Bad page state: %lu messages suppressed\n", 532 nr_unshown); 533 nr_unshown = 0; 534 } 535 nr_shown = 0; 536 } 537 if (nr_shown++ == 0) 538 resume = jiffies + 60 * HZ; 539 540 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 541 current->comm, page_to_pfn(page)); 542 __dump_page(page, reason); 543 bad_flags &= page->flags; 544 if (bad_flags) 545 pr_alert("bad because of flags: %#lx(%pGp)\n", 546 bad_flags, &bad_flags); 547 dump_page_owner(page); 548 549 print_modules(); 550 dump_stack(); 551 out: 552 /* Leave bad fields for debug, except PageBuddy could make trouble */ 553 page_mapcount_reset(page); /* remove PageBuddy */ 554 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 555 } 556 557 /* 558 * Higher-order pages are called "compound pages". They are structured thusly: 559 * 560 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 561 * 562 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 563 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 564 * 565 * The first tail page's ->compound_dtor holds the offset in array of compound 566 * page destructors. See compound_page_dtors. 567 * 568 * The first tail page's ->compound_order holds the order of allocation. 569 * This usage means that zero-order pages may not be compound. 570 */ 571 572 void free_compound_page(struct page *page) 573 { 574 __free_pages_ok(page, compound_order(page)); 575 } 576 577 void prep_compound_page(struct page *page, unsigned int order) 578 { 579 int i; 580 int nr_pages = 1 << order; 581 582 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 583 set_compound_order(page, order); 584 __SetPageHead(page); 585 for (i = 1; i < nr_pages; i++) { 586 struct page *p = page + i; 587 set_page_count(p, 0); 588 p->mapping = TAIL_MAPPING; 589 set_compound_head(p, page); 590 } 591 atomic_set(compound_mapcount_ptr(page), -1); 592 } 593 594 #ifdef CONFIG_DEBUG_PAGEALLOC 595 unsigned int _debug_guardpage_minorder; 596 bool _debug_pagealloc_enabled __read_mostly 597 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 598 EXPORT_SYMBOL(_debug_pagealloc_enabled); 599 bool _debug_guardpage_enabled __read_mostly; 600 601 static int __init early_debug_pagealloc(char *buf) 602 { 603 if (!buf) 604 return -EINVAL; 605 return kstrtobool(buf, &_debug_pagealloc_enabled); 606 } 607 early_param("debug_pagealloc", early_debug_pagealloc); 608 609 static bool need_debug_guardpage(void) 610 { 611 /* If we don't use debug_pagealloc, we don't need guard page */ 612 if (!debug_pagealloc_enabled()) 613 return false; 614 615 if (!debug_guardpage_minorder()) 616 return false; 617 618 return true; 619 } 620 621 static void init_debug_guardpage(void) 622 { 623 if (!debug_pagealloc_enabled()) 624 return; 625 626 if (!debug_guardpage_minorder()) 627 return; 628 629 _debug_guardpage_enabled = true; 630 } 631 632 struct page_ext_operations debug_guardpage_ops = { 633 .need = need_debug_guardpage, 634 .init = init_debug_guardpage, 635 }; 636 637 static int __init debug_guardpage_minorder_setup(char *buf) 638 { 639 unsigned long res; 640 641 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 642 pr_err("Bad debug_guardpage_minorder value\n"); 643 return 0; 644 } 645 _debug_guardpage_minorder = res; 646 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 647 return 0; 648 } 649 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); 650 651 static inline bool set_page_guard(struct zone *zone, struct page *page, 652 unsigned int order, int migratetype) 653 { 654 struct page_ext *page_ext; 655 656 if (!debug_guardpage_enabled()) 657 return false; 658 659 if (order >= debug_guardpage_minorder()) 660 return false; 661 662 page_ext = lookup_page_ext(page); 663 if (unlikely(!page_ext)) 664 return false; 665 666 __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); 667 668 INIT_LIST_HEAD(&page->lru); 669 set_page_private(page, order); 670 /* Guard pages are not available for any usage */ 671 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 672 673 return true; 674 } 675 676 static inline void clear_page_guard(struct zone *zone, struct page *page, 677 unsigned int order, int migratetype) 678 { 679 struct page_ext *page_ext; 680 681 if (!debug_guardpage_enabled()) 682 return; 683 684 page_ext = lookup_page_ext(page); 685 if (unlikely(!page_ext)) 686 return; 687 688 __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); 689 690 set_page_private(page, 0); 691 if (!is_migrate_isolate(migratetype)) 692 __mod_zone_freepage_state(zone, (1 << order), migratetype); 693 } 694 #else 695 struct page_ext_operations debug_guardpage_ops; 696 static inline bool set_page_guard(struct zone *zone, struct page *page, 697 unsigned int order, int migratetype) { return false; } 698 static inline void clear_page_guard(struct zone *zone, struct page *page, 699 unsigned int order, int migratetype) {} 700 #endif 701 702 static inline void set_page_order(struct page *page, unsigned int order) 703 { 704 set_page_private(page, order); 705 __SetPageBuddy(page); 706 } 707 708 static inline void rmv_page_order(struct page *page) 709 { 710 __ClearPageBuddy(page); 711 set_page_private(page, 0); 712 } 713 714 /* 715 * This function checks whether a page is free && is the buddy 716 * we can do coalesce a page and its buddy if 717 * (a) the buddy is not in a hole && 718 * (b) the buddy is in the buddy system && 719 * (c) a page and its buddy have the same order && 720 * (d) a page and its buddy are in the same zone. 721 * 722 * For recording whether a page is in the buddy system, we set ->_mapcount 723 * PAGE_BUDDY_MAPCOUNT_VALUE. 724 * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is 725 * serialized by zone->lock. 726 * 727 * For recording page's order, we use page_private(page). 728 */ 729 static inline int page_is_buddy(struct page *page, struct page *buddy, 730 unsigned int order) 731 { 732 if (!pfn_valid_within(page_to_pfn(buddy))) 733 return 0; 734 735 if (page_is_guard(buddy) && page_order(buddy) == order) { 736 if (page_zone_id(page) != page_zone_id(buddy)) 737 return 0; 738 739 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 740 741 return 1; 742 } 743 744 if (PageBuddy(buddy) && page_order(buddy) == order) { 745 /* 746 * zone check is done late to avoid uselessly 747 * calculating zone/node ids for pages that could 748 * never merge. 749 */ 750 if (page_zone_id(page) != page_zone_id(buddy)) 751 return 0; 752 753 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 754 755 return 1; 756 } 757 return 0; 758 } 759 760 /* 761 * Freeing function for a buddy system allocator. 762 * 763 * The concept of a buddy system is to maintain direct-mapped table 764 * (containing bit values) for memory blocks of various "orders". 765 * The bottom level table contains the map for the smallest allocatable 766 * units of memory (here, pages), and each level above it describes 767 * pairs of units from the levels below, hence, "buddies". 768 * At a high level, all that happens here is marking the table entry 769 * at the bottom level available, and propagating the changes upward 770 * as necessary, plus some accounting needed to play nicely with other 771 * parts of the VM system. 772 * At each level, we keep a list of pages, which are heads of continuous 773 * free pages of length of (1 << order) and marked with _mapcount 774 * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page) 775 * field. 776 * So when we are allocating or freeing one, we can derive the state of the 777 * other. That is, if we allocate a small block, and both were 778 * free, the remainder of the region must be split into blocks. 779 * If a block is freed, and its buddy is also free, then this 780 * triggers coalescing into a block of larger size. 781 * 782 * -- nyc 783 */ 784 785 static inline void __free_one_page(struct page *page, 786 unsigned long pfn, 787 struct zone *zone, unsigned int order, 788 int migratetype) 789 { 790 unsigned long page_idx; 791 unsigned long combined_idx; 792 unsigned long uninitialized_var(buddy_idx); 793 struct page *buddy; 794 unsigned int max_order; 795 796 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1); 797 798 VM_BUG_ON(!zone_is_initialized(zone)); 799 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 800 801 VM_BUG_ON(migratetype == -1); 802 if (likely(!is_migrate_isolate(migratetype))) 803 __mod_zone_freepage_state(zone, 1 << order, migratetype); 804 805 page_idx = pfn & ((1 << MAX_ORDER) - 1); 806 807 VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page); 808 VM_BUG_ON_PAGE(bad_range(zone, page), page); 809 810 continue_merging: 811 while (order < max_order - 1) { 812 buddy_idx = __find_buddy_index(page_idx, order); 813 buddy = page + (buddy_idx - page_idx); 814 if (!page_is_buddy(page, buddy, order)) 815 goto done_merging; 816 /* 817 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 818 * merge with it and move up one order. 819 */ 820 if (page_is_guard(buddy)) { 821 clear_page_guard(zone, buddy, order, migratetype); 822 } else { 823 list_del(&buddy->lru); 824 zone->free_area[order].nr_free--; 825 rmv_page_order(buddy); 826 } 827 combined_idx = buddy_idx & page_idx; 828 page = page + (combined_idx - page_idx); 829 page_idx = combined_idx; 830 order++; 831 } 832 if (max_order < MAX_ORDER) { 833 /* If we are here, it means order is >= pageblock_order. 834 * We want to prevent merge between freepages on isolate 835 * pageblock and normal pageblock. Without this, pageblock 836 * isolation could cause incorrect freepage or CMA accounting. 837 * 838 * We don't want to hit this code for the more frequent 839 * low-order merging. 840 */ 841 if (unlikely(has_isolate_pageblock(zone))) { 842 int buddy_mt; 843 844 buddy_idx = __find_buddy_index(page_idx, order); 845 buddy = page + (buddy_idx - page_idx); 846 buddy_mt = get_pageblock_migratetype(buddy); 847 848 if (migratetype != buddy_mt 849 && (is_migrate_isolate(migratetype) || 850 is_migrate_isolate(buddy_mt))) 851 goto done_merging; 852 } 853 max_order++; 854 goto continue_merging; 855 } 856 857 done_merging: 858 set_page_order(page, order); 859 860 /* 861 * If this is not the largest possible page, check if the buddy 862 * of the next-highest order is free. If it is, it's possible 863 * that pages are being freed that will coalesce soon. In case, 864 * that is happening, add the free page to the tail of the list 865 * so it's less likely to be used soon and more likely to be merged 866 * as a higher order page 867 */ 868 if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) { 869 struct page *higher_page, *higher_buddy; 870 combined_idx = buddy_idx & page_idx; 871 higher_page = page + (combined_idx - page_idx); 872 buddy_idx = __find_buddy_index(combined_idx, order + 1); 873 higher_buddy = higher_page + (buddy_idx - combined_idx); 874 if (page_is_buddy(higher_page, higher_buddy, order + 1)) { 875 list_add_tail(&page->lru, 876 &zone->free_area[order].free_list[migratetype]); 877 goto out; 878 } 879 } 880 881 list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); 882 out: 883 zone->free_area[order].nr_free++; 884 } 885 886 /* 887 * A bad page could be due to a number of fields. Instead of multiple branches, 888 * try and check multiple fields with one check. The caller must do a detailed 889 * check if necessary. 890 */ 891 static inline bool page_expected_state(struct page *page, 892 unsigned long check_flags) 893 { 894 if (unlikely(atomic_read(&page->_mapcount) != -1)) 895 return false; 896 897 if (unlikely((unsigned long)page->mapping | 898 page_ref_count(page) | 899 #ifdef CONFIG_MEMCG 900 (unsigned long)page->mem_cgroup | 901 #endif 902 (page->flags & check_flags))) 903 return false; 904 905 return true; 906 } 907 908 static void free_pages_check_bad(struct page *page) 909 { 910 const char *bad_reason; 911 unsigned long bad_flags; 912 913 bad_reason = NULL; 914 bad_flags = 0; 915 916 if (unlikely(atomic_read(&page->_mapcount) != -1)) 917 bad_reason = "nonzero mapcount"; 918 if (unlikely(page->mapping != NULL)) 919 bad_reason = "non-NULL mapping"; 920 if (unlikely(page_ref_count(page) != 0)) 921 bad_reason = "nonzero _refcount"; 922 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) { 923 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 924 bad_flags = PAGE_FLAGS_CHECK_AT_FREE; 925 } 926 #ifdef CONFIG_MEMCG 927 if (unlikely(page->mem_cgroup)) 928 bad_reason = "page still charged to cgroup"; 929 #endif 930 bad_page(page, bad_reason, bad_flags); 931 } 932 933 static inline int free_pages_check(struct page *page) 934 { 935 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 936 return 0; 937 938 /* Something has gone sideways, find it */ 939 free_pages_check_bad(page); 940 return 1; 941 } 942 943 static int free_tail_pages_check(struct page *head_page, struct page *page) 944 { 945 int ret = 1; 946 947 /* 948 * We rely page->lru.next never has bit 0 set, unless the page 949 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 950 */ 951 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 952 953 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 954 ret = 0; 955 goto out; 956 } 957 switch (page - head_page) { 958 case 1: 959 /* the first tail page: ->mapping is compound_mapcount() */ 960 if (unlikely(compound_mapcount(page))) { 961 bad_page(page, "nonzero compound_mapcount", 0); 962 goto out; 963 } 964 break; 965 case 2: 966 /* 967 * the second tail page: ->mapping is 968 * page_deferred_list().next -- ignore value. 969 */ 970 break; 971 default: 972 if (page->mapping != TAIL_MAPPING) { 973 bad_page(page, "corrupted mapping in tail page", 0); 974 goto out; 975 } 976 break; 977 } 978 if (unlikely(!PageTail(page))) { 979 bad_page(page, "PageTail not set", 0); 980 goto out; 981 } 982 if (unlikely(compound_head(page) != head_page)) { 983 bad_page(page, "compound_head not consistent", 0); 984 goto out; 985 } 986 ret = 0; 987 out: 988 page->mapping = NULL; 989 clear_compound_head(page); 990 return ret; 991 } 992 993 static __always_inline bool free_pages_prepare(struct page *page, 994 unsigned int order, bool check_free) 995 { 996 int bad = 0; 997 998 VM_BUG_ON_PAGE(PageTail(page), page); 999 1000 trace_mm_page_free(page, order); 1001 kmemcheck_free_shadow(page, order); 1002 1003 /* 1004 * Check tail pages before head page information is cleared to 1005 * avoid checking PageCompound for order-0 pages. 1006 */ 1007 if (unlikely(order)) { 1008 bool compound = PageCompound(page); 1009 int i; 1010 1011 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1012 1013 if (compound) 1014 ClearPageDoubleMap(page); 1015 for (i = 1; i < (1 << order); i++) { 1016 if (compound) 1017 bad += free_tail_pages_check(page, page + i); 1018 if (unlikely(free_pages_check(page + i))) { 1019 bad++; 1020 continue; 1021 } 1022 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1023 } 1024 } 1025 if (PageMappingFlags(page)) 1026 page->mapping = NULL; 1027 if (memcg_kmem_enabled() && PageKmemcg(page)) 1028 memcg_kmem_uncharge(page, order); 1029 if (check_free) 1030 bad += free_pages_check(page); 1031 if (bad) 1032 return false; 1033 1034 page_cpupid_reset_last(page); 1035 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1036 reset_page_owner(page, order); 1037 1038 if (!PageHighMem(page)) { 1039 debug_check_no_locks_freed(page_address(page), 1040 PAGE_SIZE << order); 1041 debug_check_no_obj_freed(page_address(page), 1042 PAGE_SIZE << order); 1043 } 1044 arch_free_page(page, order); 1045 kernel_poison_pages(page, 1 << order, 0); 1046 kernel_map_pages(page, 1 << order, 0); 1047 kasan_free_pages(page, order); 1048 1049 return true; 1050 } 1051 1052 #ifdef CONFIG_DEBUG_VM 1053 static inline bool free_pcp_prepare(struct page *page) 1054 { 1055 return free_pages_prepare(page, 0, true); 1056 } 1057 1058 static inline bool bulkfree_pcp_prepare(struct page *page) 1059 { 1060 return false; 1061 } 1062 #else 1063 static bool free_pcp_prepare(struct page *page) 1064 { 1065 return free_pages_prepare(page, 0, false); 1066 } 1067 1068 static bool bulkfree_pcp_prepare(struct page *page) 1069 { 1070 return free_pages_check(page); 1071 } 1072 #endif /* CONFIG_DEBUG_VM */ 1073 1074 /* 1075 * Frees a number of pages from the PCP lists 1076 * Assumes all pages on list are in same zone, and of same order. 1077 * count is the number of pages to free. 1078 * 1079 * If the zone was previously in an "all pages pinned" state then look to 1080 * see if this freeing clears that state. 1081 * 1082 * And clear the zone's pages_scanned counter, to hold off the "all pages are 1083 * pinned" detection logic. 1084 */ 1085 static void free_pcppages_bulk(struct zone *zone, int count, 1086 struct per_cpu_pages *pcp) 1087 { 1088 int migratetype = 0; 1089 int batch_free = 0; 1090 unsigned long nr_scanned; 1091 bool isolated_pageblocks; 1092 1093 spin_lock(&zone->lock); 1094 isolated_pageblocks = has_isolate_pageblock(zone); 1095 nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED); 1096 if (nr_scanned) 1097 __mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned); 1098 1099 while (count) { 1100 struct page *page; 1101 struct list_head *list; 1102 1103 /* 1104 * Remove pages from lists in a round-robin fashion. A 1105 * batch_free count is maintained that is incremented when an 1106 * empty list is encountered. This is so more pages are freed 1107 * off fuller lists instead of spinning excessively around empty 1108 * lists 1109 */ 1110 do { 1111 batch_free++; 1112 if (++migratetype == MIGRATE_PCPTYPES) 1113 migratetype = 0; 1114 list = &pcp->lists[migratetype]; 1115 } while (list_empty(list)); 1116 1117 /* This is the only non-empty list. Free them all. */ 1118 if (batch_free == MIGRATE_PCPTYPES) 1119 batch_free = count; 1120 1121 do { 1122 int mt; /* migratetype of the to-be-freed page */ 1123 1124 page = list_last_entry(list, struct page, lru); 1125 /* must delete as __free_one_page list manipulates */ 1126 list_del(&page->lru); 1127 1128 mt = get_pcppage_migratetype(page); 1129 /* MIGRATE_ISOLATE page should not go to pcplists */ 1130 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1131 /* Pageblock could have been isolated meanwhile */ 1132 if (unlikely(isolated_pageblocks)) 1133 mt = get_pageblock_migratetype(page); 1134 1135 if (bulkfree_pcp_prepare(page)) 1136 continue; 1137 1138 __free_one_page(page, page_to_pfn(page), zone, 0, mt); 1139 trace_mm_page_pcpu_drain(page, 0, mt); 1140 } while (--count && --batch_free && !list_empty(list)); 1141 } 1142 spin_unlock(&zone->lock); 1143 } 1144 1145 static void free_one_page(struct zone *zone, 1146 struct page *page, unsigned long pfn, 1147 unsigned int order, 1148 int migratetype) 1149 { 1150 unsigned long nr_scanned; 1151 spin_lock(&zone->lock); 1152 nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED); 1153 if (nr_scanned) 1154 __mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned); 1155 1156 if (unlikely(has_isolate_pageblock(zone) || 1157 is_migrate_isolate(migratetype))) { 1158 migratetype = get_pfnblock_migratetype(page, pfn); 1159 } 1160 __free_one_page(page, pfn, zone, order, migratetype); 1161 spin_unlock(&zone->lock); 1162 } 1163 1164 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1165 unsigned long zone, int nid) 1166 { 1167 set_page_links(page, zone, nid, pfn); 1168 init_page_count(page); 1169 page_mapcount_reset(page); 1170 page_cpupid_reset_last(page); 1171 1172 INIT_LIST_HEAD(&page->lru); 1173 #ifdef WANT_PAGE_VIRTUAL 1174 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1175 if (!is_highmem_idx(zone)) 1176 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1177 #endif 1178 } 1179 1180 static void __meminit __init_single_pfn(unsigned long pfn, unsigned long zone, 1181 int nid) 1182 { 1183 return __init_single_page(pfn_to_page(pfn), pfn, zone, nid); 1184 } 1185 1186 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1187 static void init_reserved_page(unsigned long pfn) 1188 { 1189 pg_data_t *pgdat; 1190 int nid, zid; 1191 1192 if (!early_page_uninitialised(pfn)) 1193 return; 1194 1195 nid = early_pfn_to_nid(pfn); 1196 pgdat = NODE_DATA(nid); 1197 1198 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1199 struct zone *zone = &pgdat->node_zones[zid]; 1200 1201 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) 1202 break; 1203 } 1204 __init_single_pfn(pfn, zid, nid); 1205 } 1206 #else 1207 static inline void init_reserved_page(unsigned long pfn) 1208 { 1209 } 1210 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1211 1212 /* 1213 * Initialised pages do not have PageReserved set. This function is 1214 * called for each range allocated by the bootmem allocator and 1215 * marks the pages PageReserved. The remaining valid pages are later 1216 * sent to the buddy page allocator. 1217 */ 1218 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1219 { 1220 unsigned long start_pfn = PFN_DOWN(start); 1221 unsigned long end_pfn = PFN_UP(end); 1222 1223 for (; start_pfn < end_pfn; start_pfn++) { 1224 if (pfn_valid(start_pfn)) { 1225 struct page *page = pfn_to_page(start_pfn); 1226 1227 init_reserved_page(start_pfn); 1228 1229 /* Avoid false-positive PageTail() */ 1230 INIT_LIST_HEAD(&page->lru); 1231 1232 SetPageReserved(page); 1233 } 1234 } 1235 } 1236 1237 static void __free_pages_ok(struct page *page, unsigned int order) 1238 { 1239 unsigned long flags; 1240 int migratetype; 1241 unsigned long pfn = page_to_pfn(page); 1242 1243 if (!free_pages_prepare(page, order, true)) 1244 return; 1245 1246 migratetype = get_pfnblock_migratetype(page, pfn); 1247 local_irq_save(flags); 1248 __count_vm_events(PGFREE, 1 << order); 1249 free_one_page(page_zone(page), page, pfn, order, migratetype); 1250 local_irq_restore(flags); 1251 } 1252 1253 static void __init __free_pages_boot_core(struct page *page, unsigned int order) 1254 { 1255 unsigned int nr_pages = 1 << order; 1256 struct page *p = page; 1257 unsigned int loop; 1258 1259 prefetchw(p); 1260 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1261 prefetchw(p + 1); 1262 __ClearPageReserved(p); 1263 set_page_count(p, 0); 1264 } 1265 __ClearPageReserved(p); 1266 set_page_count(p, 0); 1267 1268 page_zone(page)->managed_pages += nr_pages; 1269 set_page_refcounted(page); 1270 __free_pages(page, order); 1271 } 1272 1273 #if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \ 1274 defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) 1275 1276 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1277 1278 int __meminit early_pfn_to_nid(unsigned long pfn) 1279 { 1280 static DEFINE_SPINLOCK(early_pfn_lock); 1281 int nid; 1282 1283 spin_lock(&early_pfn_lock); 1284 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1285 if (nid < 0) 1286 nid = first_online_node; 1287 spin_unlock(&early_pfn_lock); 1288 1289 return nid; 1290 } 1291 #endif 1292 1293 #ifdef CONFIG_NODES_SPAN_OTHER_NODES 1294 static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node, 1295 struct mminit_pfnnid_cache *state) 1296 { 1297 int nid; 1298 1299 nid = __early_pfn_to_nid(pfn, state); 1300 if (nid >= 0 && nid != node) 1301 return false; 1302 return true; 1303 } 1304 1305 /* Only safe to use early in boot when initialisation is single-threaded */ 1306 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1307 { 1308 return meminit_pfn_in_nid(pfn, node, &early_pfnnid_cache); 1309 } 1310 1311 #else 1312 1313 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1314 { 1315 return true; 1316 } 1317 static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node, 1318 struct mminit_pfnnid_cache *state) 1319 { 1320 return true; 1321 } 1322 #endif 1323 1324 1325 void __init __free_pages_bootmem(struct page *page, unsigned long pfn, 1326 unsigned int order) 1327 { 1328 if (early_page_uninitialised(pfn)) 1329 return; 1330 return __free_pages_boot_core(page, order); 1331 } 1332 1333 /* 1334 * Check that the whole (or subset of) a pageblock given by the interval of 1335 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1336 * with the migration of free compaction scanner. The scanners then need to 1337 * use only pfn_valid_within() check for arches that allow holes within 1338 * pageblocks. 1339 * 1340 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1341 * 1342 * It's possible on some configurations to have a setup like node0 node1 node0 1343 * i.e. it's possible that all pages within a zones range of pages do not 1344 * belong to a single zone. We assume that a border between node0 and node1 1345 * can occur within a single pageblock, but not a node0 node1 node0 1346 * interleaving within a single pageblock. It is therefore sufficient to check 1347 * the first and last page of a pageblock and avoid checking each individual 1348 * page in a pageblock. 1349 */ 1350 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1351 unsigned long end_pfn, struct zone *zone) 1352 { 1353 struct page *start_page; 1354 struct page *end_page; 1355 1356 /* end_pfn is one past the range we are checking */ 1357 end_pfn--; 1358 1359 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1360 return NULL; 1361 1362 start_page = pfn_to_page(start_pfn); 1363 1364 if (page_zone(start_page) != zone) 1365 return NULL; 1366 1367 end_page = pfn_to_page(end_pfn); 1368 1369 /* This gives a shorter code than deriving page_zone(end_page) */ 1370 if (page_zone_id(start_page) != page_zone_id(end_page)) 1371 return NULL; 1372 1373 return start_page; 1374 } 1375 1376 void set_zone_contiguous(struct zone *zone) 1377 { 1378 unsigned long block_start_pfn = zone->zone_start_pfn; 1379 unsigned long block_end_pfn; 1380 1381 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1382 for (; block_start_pfn < zone_end_pfn(zone); 1383 block_start_pfn = block_end_pfn, 1384 block_end_pfn += pageblock_nr_pages) { 1385 1386 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1387 1388 if (!__pageblock_pfn_to_page(block_start_pfn, 1389 block_end_pfn, zone)) 1390 return; 1391 } 1392 1393 /* We confirm that there is no hole */ 1394 zone->contiguous = true; 1395 } 1396 1397 void clear_zone_contiguous(struct zone *zone) 1398 { 1399 zone->contiguous = false; 1400 } 1401 1402 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1403 static void __init deferred_free_range(struct page *page, 1404 unsigned long pfn, int nr_pages) 1405 { 1406 int i; 1407 1408 if (!page) 1409 return; 1410 1411 /* Free a large naturally-aligned chunk if possible */ 1412 if (nr_pages == pageblock_nr_pages && 1413 (pfn & (pageblock_nr_pages - 1)) == 0) { 1414 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1415 __free_pages_boot_core(page, pageblock_order); 1416 return; 1417 } 1418 1419 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1420 if ((pfn & (pageblock_nr_pages - 1)) == 0) 1421 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1422 __free_pages_boot_core(page, 0); 1423 } 1424 } 1425 1426 /* Completion tracking for deferred_init_memmap() threads */ 1427 static atomic_t pgdat_init_n_undone __initdata; 1428 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1429 1430 static inline void __init pgdat_init_report_one_done(void) 1431 { 1432 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1433 complete(&pgdat_init_all_done_comp); 1434 } 1435 1436 /* Initialise remaining memory on a node */ 1437 static int __init deferred_init_memmap(void *data) 1438 { 1439 pg_data_t *pgdat = data; 1440 int nid = pgdat->node_id; 1441 struct mminit_pfnnid_cache nid_init_state = { }; 1442 unsigned long start = jiffies; 1443 unsigned long nr_pages = 0; 1444 unsigned long walk_start, walk_end; 1445 int i, zid; 1446 struct zone *zone; 1447 unsigned long first_init_pfn = pgdat->first_deferred_pfn; 1448 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 1449 1450 if (first_init_pfn == ULONG_MAX) { 1451 pgdat_init_report_one_done(); 1452 return 0; 1453 } 1454 1455 /* Bind memory initialisation thread to a local node if possible */ 1456 if (!cpumask_empty(cpumask)) 1457 set_cpus_allowed_ptr(current, cpumask); 1458 1459 /* Sanity check boundaries */ 1460 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 1461 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 1462 pgdat->first_deferred_pfn = ULONG_MAX; 1463 1464 /* Only the highest zone is deferred so find it */ 1465 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1466 zone = pgdat->node_zones + zid; 1467 if (first_init_pfn < zone_end_pfn(zone)) 1468 break; 1469 } 1470 1471 for_each_mem_pfn_range(i, nid, &walk_start, &walk_end, NULL) { 1472 unsigned long pfn, end_pfn; 1473 struct page *page = NULL; 1474 struct page *free_base_page = NULL; 1475 unsigned long free_base_pfn = 0; 1476 int nr_to_free = 0; 1477 1478 end_pfn = min(walk_end, zone_end_pfn(zone)); 1479 pfn = first_init_pfn; 1480 if (pfn < walk_start) 1481 pfn = walk_start; 1482 if (pfn < zone->zone_start_pfn) 1483 pfn = zone->zone_start_pfn; 1484 1485 for (; pfn < end_pfn; pfn++) { 1486 if (!pfn_valid_within(pfn)) 1487 goto free_range; 1488 1489 /* 1490 * Ensure pfn_valid is checked every 1491 * pageblock_nr_pages for memory holes 1492 */ 1493 if ((pfn & (pageblock_nr_pages - 1)) == 0) { 1494 if (!pfn_valid(pfn)) { 1495 page = NULL; 1496 goto free_range; 1497 } 1498 } 1499 1500 if (!meminit_pfn_in_nid(pfn, nid, &nid_init_state)) { 1501 page = NULL; 1502 goto free_range; 1503 } 1504 1505 /* Minimise pfn page lookups and scheduler checks */ 1506 if (page && (pfn & (pageblock_nr_pages - 1)) != 0) { 1507 page++; 1508 } else { 1509 nr_pages += nr_to_free; 1510 deferred_free_range(free_base_page, 1511 free_base_pfn, nr_to_free); 1512 free_base_page = NULL; 1513 free_base_pfn = nr_to_free = 0; 1514 1515 page = pfn_to_page(pfn); 1516 cond_resched(); 1517 } 1518 1519 if (page->flags) { 1520 VM_BUG_ON(page_zone(page) != zone); 1521 goto free_range; 1522 } 1523 1524 __init_single_page(page, pfn, zid, nid); 1525 if (!free_base_page) { 1526 free_base_page = page; 1527 free_base_pfn = pfn; 1528 nr_to_free = 0; 1529 } 1530 nr_to_free++; 1531 1532 /* Where possible, batch up pages for a single free */ 1533 continue; 1534 free_range: 1535 /* Free the current block of pages to allocator */ 1536 nr_pages += nr_to_free; 1537 deferred_free_range(free_base_page, free_base_pfn, 1538 nr_to_free); 1539 free_base_page = NULL; 1540 free_base_pfn = nr_to_free = 0; 1541 } 1542 /* Free the last block of pages to allocator */ 1543 nr_pages += nr_to_free; 1544 deferred_free_range(free_base_page, free_base_pfn, nr_to_free); 1545 1546 first_init_pfn = max(end_pfn, first_init_pfn); 1547 } 1548 1549 /* Sanity check that the next zone really is unpopulated */ 1550 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 1551 1552 pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages, 1553 jiffies_to_msecs(jiffies - start)); 1554 1555 pgdat_init_report_one_done(); 1556 return 0; 1557 } 1558 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1559 1560 void __init page_alloc_init_late(void) 1561 { 1562 struct zone *zone; 1563 1564 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1565 int nid; 1566 1567 /* There will be num_node_state(N_MEMORY) threads */ 1568 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 1569 for_each_node_state(nid, N_MEMORY) { 1570 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 1571 } 1572 1573 /* Block until all are initialised */ 1574 wait_for_completion(&pgdat_init_all_done_comp); 1575 1576 /* Reinit limits that are based on free pages after the kernel is up */ 1577 files_maxfiles_init(); 1578 #endif 1579 1580 for_each_populated_zone(zone) 1581 set_zone_contiguous(zone); 1582 } 1583 1584 #ifdef CONFIG_CMA 1585 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 1586 void __init init_cma_reserved_pageblock(struct page *page) 1587 { 1588 unsigned i = pageblock_nr_pages; 1589 struct page *p = page; 1590 1591 do { 1592 __ClearPageReserved(p); 1593 set_page_count(p, 0); 1594 } while (++p, --i); 1595 1596 set_pageblock_migratetype(page, MIGRATE_CMA); 1597 1598 if (pageblock_order >= MAX_ORDER) { 1599 i = pageblock_nr_pages; 1600 p = page; 1601 do { 1602 set_page_refcounted(p); 1603 __free_pages(p, MAX_ORDER - 1); 1604 p += MAX_ORDER_NR_PAGES; 1605 } while (i -= MAX_ORDER_NR_PAGES); 1606 } else { 1607 set_page_refcounted(page); 1608 __free_pages(page, pageblock_order); 1609 } 1610 1611 adjust_managed_page_count(page, pageblock_nr_pages); 1612 } 1613 #endif 1614 1615 /* 1616 * The order of subdivision here is critical for the IO subsystem. 1617 * Please do not alter this order without good reasons and regression 1618 * testing. Specifically, as large blocks of memory are subdivided, 1619 * the order in which smaller blocks are delivered depends on the order 1620 * they're subdivided in this function. This is the primary factor 1621 * influencing the order in which pages are delivered to the IO 1622 * subsystem according to empirical testing, and this is also justified 1623 * by considering the behavior of a buddy system containing a single 1624 * large block of memory acted on by a series of small allocations. 1625 * This behavior is a critical factor in sglist merging's success. 1626 * 1627 * -- nyc 1628 */ 1629 static inline void expand(struct zone *zone, struct page *page, 1630 int low, int high, struct free_area *area, 1631 int migratetype) 1632 { 1633 unsigned long size = 1 << high; 1634 1635 while (high > low) { 1636 area--; 1637 high--; 1638 size >>= 1; 1639 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 1640 1641 /* 1642 * Mark as guard pages (or page), that will allow to 1643 * merge back to allocator when buddy will be freed. 1644 * Corresponding page table entries will not be touched, 1645 * pages will stay not present in virtual address space 1646 */ 1647 if (set_page_guard(zone, &page[size], high, migratetype)) 1648 continue; 1649 1650 list_add(&page[size].lru, &area->free_list[migratetype]); 1651 area->nr_free++; 1652 set_page_order(&page[size], high); 1653 } 1654 } 1655 1656 static void check_new_page_bad(struct page *page) 1657 { 1658 const char *bad_reason = NULL; 1659 unsigned long bad_flags = 0; 1660 1661 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1662 bad_reason = "nonzero mapcount"; 1663 if (unlikely(page->mapping != NULL)) 1664 bad_reason = "non-NULL mapping"; 1665 if (unlikely(page_ref_count(page) != 0)) 1666 bad_reason = "nonzero _count"; 1667 if (unlikely(page->flags & __PG_HWPOISON)) { 1668 bad_reason = "HWPoisoned (hardware-corrupted)"; 1669 bad_flags = __PG_HWPOISON; 1670 /* Don't complain about hwpoisoned pages */ 1671 page_mapcount_reset(page); /* remove PageBuddy */ 1672 return; 1673 } 1674 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) { 1675 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set"; 1676 bad_flags = PAGE_FLAGS_CHECK_AT_PREP; 1677 } 1678 #ifdef CONFIG_MEMCG 1679 if (unlikely(page->mem_cgroup)) 1680 bad_reason = "page still charged to cgroup"; 1681 #endif 1682 bad_page(page, bad_reason, bad_flags); 1683 } 1684 1685 /* 1686 * This page is about to be returned from the page allocator 1687 */ 1688 static inline int check_new_page(struct page *page) 1689 { 1690 if (likely(page_expected_state(page, 1691 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 1692 return 0; 1693 1694 check_new_page_bad(page); 1695 return 1; 1696 } 1697 1698 static inline bool free_pages_prezeroed(bool poisoned) 1699 { 1700 return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) && 1701 page_poisoning_enabled() && poisoned; 1702 } 1703 1704 #ifdef CONFIG_DEBUG_VM 1705 static bool check_pcp_refill(struct page *page) 1706 { 1707 return false; 1708 } 1709 1710 static bool check_new_pcp(struct page *page) 1711 { 1712 return check_new_page(page); 1713 } 1714 #else 1715 static bool check_pcp_refill(struct page *page) 1716 { 1717 return check_new_page(page); 1718 } 1719 static bool check_new_pcp(struct page *page) 1720 { 1721 return false; 1722 } 1723 #endif /* CONFIG_DEBUG_VM */ 1724 1725 static bool check_new_pages(struct page *page, unsigned int order) 1726 { 1727 int i; 1728 for (i = 0; i < (1 << order); i++) { 1729 struct page *p = page + i; 1730 1731 if (unlikely(check_new_page(p))) 1732 return true; 1733 } 1734 1735 return false; 1736 } 1737 1738 inline void post_alloc_hook(struct page *page, unsigned int order, 1739 gfp_t gfp_flags) 1740 { 1741 set_page_private(page, 0); 1742 set_page_refcounted(page); 1743 1744 arch_alloc_page(page, order); 1745 kernel_map_pages(page, 1 << order, 1); 1746 kernel_poison_pages(page, 1 << order, 1); 1747 kasan_alloc_pages(page, order); 1748 set_page_owner(page, order, gfp_flags); 1749 } 1750 1751 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 1752 unsigned int alloc_flags) 1753 { 1754 int i; 1755 bool poisoned = true; 1756 1757 for (i = 0; i < (1 << order); i++) { 1758 struct page *p = page + i; 1759 if (poisoned) 1760 poisoned &= page_is_poisoned(p); 1761 } 1762 1763 post_alloc_hook(page, order, gfp_flags); 1764 1765 if (!free_pages_prezeroed(poisoned) && (gfp_flags & __GFP_ZERO)) 1766 for (i = 0; i < (1 << order); i++) 1767 clear_highpage(page + i); 1768 1769 if (order && (gfp_flags & __GFP_COMP)) 1770 prep_compound_page(page, order); 1771 1772 /* 1773 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 1774 * allocate the page. The expectation is that the caller is taking 1775 * steps that will free more memory. The caller should avoid the page 1776 * being used for !PFMEMALLOC purposes. 1777 */ 1778 if (alloc_flags & ALLOC_NO_WATERMARKS) 1779 set_page_pfmemalloc(page); 1780 else 1781 clear_page_pfmemalloc(page); 1782 } 1783 1784 /* 1785 * Go through the free lists for the given migratetype and remove 1786 * the smallest available page from the freelists 1787 */ 1788 static inline 1789 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 1790 int migratetype) 1791 { 1792 unsigned int current_order; 1793 struct free_area *area; 1794 struct page *page; 1795 1796 /* Find a page of the appropriate size in the preferred list */ 1797 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 1798 area = &(zone->free_area[current_order]); 1799 page = list_first_entry_or_null(&area->free_list[migratetype], 1800 struct page, lru); 1801 if (!page) 1802 continue; 1803 list_del(&page->lru); 1804 rmv_page_order(page); 1805 area->nr_free--; 1806 expand(zone, page, order, current_order, area, migratetype); 1807 set_pcppage_migratetype(page, migratetype); 1808 return page; 1809 } 1810 1811 return NULL; 1812 } 1813 1814 1815 /* 1816 * This array describes the order lists are fallen back to when 1817 * the free lists for the desirable migrate type are depleted 1818 */ 1819 static int fallbacks[MIGRATE_TYPES][4] = { 1820 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 1821 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 1822 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 1823 #ifdef CONFIG_CMA 1824 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */ 1825 #endif 1826 #ifdef CONFIG_MEMORY_ISOLATION 1827 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */ 1828 #endif 1829 }; 1830 1831 #ifdef CONFIG_CMA 1832 static struct page *__rmqueue_cma_fallback(struct zone *zone, 1833 unsigned int order) 1834 { 1835 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 1836 } 1837 #else 1838 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 1839 unsigned int order) { return NULL; } 1840 #endif 1841 1842 /* 1843 * Move the free pages in a range to the free lists of the requested type. 1844 * Note that start_page and end_pages are not aligned on a pageblock 1845 * boundary. If alignment is required, use move_freepages_block() 1846 */ 1847 int move_freepages(struct zone *zone, 1848 struct page *start_page, struct page *end_page, 1849 int migratetype) 1850 { 1851 struct page *page; 1852 unsigned int order; 1853 int pages_moved = 0; 1854 1855 #ifndef CONFIG_HOLES_IN_ZONE 1856 /* 1857 * page_zone is not safe to call in this context when 1858 * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant 1859 * anyway as we check zone boundaries in move_freepages_block(). 1860 * Remove at a later date when no bug reports exist related to 1861 * grouping pages by mobility 1862 */ 1863 VM_BUG_ON(page_zone(start_page) != page_zone(end_page)); 1864 #endif 1865 1866 for (page = start_page; page <= end_page;) { 1867 /* Make sure we are not inadvertently changing nodes */ 1868 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 1869 1870 if (!pfn_valid_within(page_to_pfn(page))) { 1871 page++; 1872 continue; 1873 } 1874 1875 if (!PageBuddy(page)) { 1876 page++; 1877 continue; 1878 } 1879 1880 order = page_order(page); 1881 list_move(&page->lru, 1882 &zone->free_area[order].free_list[migratetype]); 1883 page += 1 << order; 1884 pages_moved += 1 << order; 1885 } 1886 1887 return pages_moved; 1888 } 1889 1890 int move_freepages_block(struct zone *zone, struct page *page, 1891 int migratetype) 1892 { 1893 unsigned long start_pfn, end_pfn; 1894 struct page *start_page, *end_page; 1895 1896 start_pfn = page_to_pfn(page); 1897 start_pfn = start_pfn & ~(pageblock_nr_pages-1); 1898 start_page = pfn_to_page(start_pfn); 1899 end_page = start_page + pageblock_nr_pages - 1; 1900 end_pfn = start_pfn + pageblock_nr_pages - 1; 1901 1902 /* Do not cross zone boundaries */ 1903 if (!zone_spans_pfn(zone, start_pfn)) 1904 start_page = page; 1905 if (!zone_spans_pfn(zone, end_pfn)) 1906 return 0; 1907 1908 return move_freepages(zone, start_page, end_page, migratetype); 1909 } 1910 1911 static void change_pageblock_range(struct page *pageblock_page, 1912 int start_order, int migratetype) 1913 { 1914 int nr_pageblocks = 1 << (start_order - pageblock_order); 1915 1916 while (nr_pageblocks--) { 1917 set_pageblock_migratetype(pageblock_page, migratetype); 1918 pageblock_page += pageblock_nr_pages; 1919 } 1920 } 1921 1922 /* 1923 * When we are falling back to another migratetype during allocation, try to 1924 * steal extra free pages from the same pageblocks to satisfy further 1925 * allocations, instead of polluting multiple pageblocks. 1926 * 1927 * If we are stealing a relatively large buddy page, it is likely there will 1928 * be more free pages in the pageblock, so try to steal them all. For 1929 * reclaimable and unmovable allocations, we steal regardless of page size, 1930 * as fragmentation caused by those allocations polluting movable pageblocks 1931 * is worse than movable allocations stealing from unmovable and reclaimable 1932 * pageblocks. 1933 */ 1934 static bool can_steal_fallback(unsigned int order, int start_mt) 1935 { 1936 /* 1937 * Leaving this order check is intended, although there is 1938 * relaxed order check in next check. The reason is that 1939 * we can actually steal whole pageblock if this condition met, 1940 * but, below check doesn't guarantee it and that is just heuristic 1941 * so could be changed anytime. 1942 */ 1943 if (order >= pageblock_order) 1944 return true; 1945 1946 if (order >= pageblock_order / 2 || 1947 start_mt == MIGRATE_RECLAIMABLE || 1948 start_mt == MIGRATE_UNMOVABLE || 1949 page_group_by_mobility_disabled) 1950 return true; 1951 1952 return false; 1953 } 1954 1955 /* 1956 * This function implements actual steal behaviour. If order is large enough, 1957 * we can steal whole pageblock. If not, we first move freepages in this 1958 * pageblock and check whether half of pages are moved or not. If half of 1959 * pages are moved, we can change migratetype of pageblock and permanently 1960 * use it's pages as requested migratetype in the future. 1961 */ 1962 static void steal_suitable_fallback(struct zone *zone, struct page *page, 1963 int start_type) 1964 { 1965 unsigned int current_order = page_order(page); 1966 int pages; 1967 1968 /* Take ownership for orders >= pageblock_order */ 1969 if (current_order >= pageblock_order) { 1970 change_pageblock_range(page, current_order, start_type); 1971 return; 1972 } 1973 1974 pages = move_freepages_block(zone, page, start_type); 1975 1976 /* Claim the whole block if over half of it is free */ 1977 if (pages >= (1 << (pageblock_order-1)) || 1978 page_group_by_mobility_disabled) 1979 set_pageblock_migratetype(page, start_type); 1980 } 1981 1982 /* 1983 * Check whether there is a suitable fallback freepage with requested order. 1984 * If only_stealable is true, this function returns fallback_mt only if 1985 * we can steal other freepages all together. This would help to reduce 1986 * fragmentation due to mixed migratetype pages in one pageblock. 1987 */ 1988 int find_suitable_fallback(struct free_area *area, unsigned int order, 1989 int migratetype, bool only_stealable, bool *can_steal) 1990 { 1991 int i; 1992 int fallback_mt; 1993 1994 if (area->nr_free == 0) 1995 return -1; 1996 1997 *can_steal = false; 1998 for (i = 0;; i++) { 1999 fallback_mt = fallbacks[migratetype][i]; 2000 if (fallback_mt == MIGRATE_TYPES) 2001 break; 2002 2003 if (list_empty(&area->free_list[fallback_mt])) 2004 continue; 2005 2006 if (can_steal_fallback(order, migratetype)) 2007 *can_steal = true; 2008 2009 if (!only_stealable) 2010 return fallback_mt; 2011 2012 if (*can_steal) 2013 return fallback_mt; 2014 } 2015 2016 return -1; 2017 } 2018 2019 /* 2020 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2021 * there are no empty page blocks that contain a page with a suitable order 2022 */ 2023 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2024 unsigned int alloc_order) 2025 { 2026 int mt; 2027 unsigned long max_managed, flags; 2028 2029 /* 2030 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2031 * Check is race-prone but harmless. 2032 */ 2033 max_managed = (zone->managed_pages / 100) + pageblock_nr_pages; 2034 if (zone->nr_reserved_highatomic >= max_managed) 2035 return; 2036 2037 spin_lock_irqsave(&zone->lock, flags); 2038 2039 /* Recheck the nr_reserved_highatomic limit under the lock */ 2040 if (zone->nr_reserved_highatomic >= max_managed) 2041 goto out_unlock; 2042 2043 /* Yoink! */ 2044 mt = get_pageblock_migratetype(page); 2045 if (mt != MIGRATE_HIGHATOMIC && 2046 !is_migrate_isolate(mt) && !is_migrate_cma(mt)) { 2047 zone->nr_reserved_highatomic += pageblock_nr_pages; 2048 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2049 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC); 2050 } 2051 2052 out_unlock: 2053 spin_unlock_irqrestore(&zone->lock, flags); 2054 } 2055 2056 /* 2057 * Used when an allocation is about to fail under memory pressure. This 2058 * potentially hurts the reliability of high-order allocations when under 2059 * intense memory pressure but failed atomic allocations should be easier 2060 * to recover from than an OOM. 2061 * 2062 * If @force is true, try to unreserve a pageblock even though highatomic 2063 * pageblock is exhausted. 2064 */ 2065 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 2066 bool force) 2067 { 2068 struct zonelist *zonelist = ac->zonelist; 2069 unsigned long flags; 2070 struct zoneref *z; 2071 struct zone *zone; 2072 struct page *page; 2073 int order; 2074 bool ret; 2075 2076 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx, 2077 ac->nodemask) { 2078 /* 2079 * Preserve at least one pageblock unless memory pressure 2080 * is really high. 2081 */ 2082 if (!force && zone->nr_reserved_highatomic <= 2083 pageblock_nr_pages) 2084 continue; 2085 2086 spin_lock_irqsave(&zone->lock, flags); 2087 for (order = 0; order < MAX_ORDER; order++) { 2088 struct free_area *area = &(zone->free_area[order]); 2089 2090 page = list_first_entry_or_null( 2091 &area->free_list[MIGRATE_HIGHATOMIC], 2092 struct page, lru); 2093 if (!page) 2094 continue; 2095 2096 /* 2097 * In page freeing path, migratetype change is racy so 2098 * we can counter several free pages in a pageblock 2099 * in this loop althoug we changed the pageblock type 2100 * from highatomic to ac->migratetype. So we should 2101 * adjust the count once. 2102 */ 2103 if (get_pageblock_migratetype(page) == 2104 MIGRATE_HIGHATOMIC) { 2105 /* 2106 * It should never happen but changes to 2107 * locking could inadvertently allow a per-cpu 2108 * drain to add pages to MIGRATE_HIGHATOMIC 2109 * while unreserving so be safe and watch for 2110 * underflows. 2111 */ 2112 zone->nr_reserved_highatomic -= min( 2113 pageblock_nr_pages, 2114 zone->nr_reserved_highatomic); 2115 } 2116 2117 /* 2118 * Convert to ac->migratetype and avoid the normal 2119 * pageblock stealing heuristics. Minimally, the caller 2120 * is doing the work and needs the pages. More 2121 * importantly, if the block was always converted to 2122 * MIGRATE_UNMOVABLE or another type then the number 2123 * of pageblocks that cannot be completely freed 2124 * may increase. 2125 */ 2126 set_pageblock_migratetype(page, ac->migratetype); 2127 ret = move_freepages_block(zone, page, ac->migratetype); 2128 if (ret) { 2129 spin_unlock_irqrestore(&zone->lock, flags); 2130 return ret; 2131 } 2132 } 2133 spin_unlock_irqrestore(&zone->lock, flags); 2134 } 2135 2136 return false; 2137 } 2138 2139 /* Remove an element from the buddy allocator from the fallback list */ 2140 static inline struct page * 2141 __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype) 2142 { 2143 struct free_area *area; 2144 unsigned int current_order; 2145 struct page *page; 2146 int fallback_mt; 2147 bool can_steal; 2148 2149 /* Find the largest possible block of pages in the other list */ 2150 for (current_order = MAX_ORDER-1; 2151 current_order >= order && current_order <= MAX_ORDER-1; 2152 --current_order) { 2153 area = &(zone->free_area[current_order]); 2154 fallback_mt = find_suitable_fallback(area, current_order, 2155 start_migratetype, false, &can_steal); 2156 if (fallback_mt == -1) 2157 continue; 2158 2159 page = list_first_entry(&area->free_list[fallback_mt], 2160 struct page, lru); 2161 if (can_steal && 2162 get_pageblock_migratetype(page) != MIGRATE_HIGHATOMIC) 2163 steal_suitable_fallback(zone, page, start_migratetype); 2164 2165 /* Remove the page from the freelists */ 2166 area->nr_free--; 2167 list_del(&page->lru); 2168 rmv_page_order(page); 2169 2170 expand(zone, page, order, current_order, area, 2171 start_migratetype); 2172 /* 2173 * The pcppage_migratetype may differ from pageblock's 2174 * migratetype depending on the decisions in 2175 * find_suitable_fallback(). This is OK as long as it does not 2176 * differ for MIGRATE_CMA pageblocks. Those can be used as 2177 * fallback only via special __rmqueue_cma_fallback() function 2178 */ 2179 set_pcppage_migratetype(page, start_migratetype); 2180 2181 trace_mm_page_alloc_extfrag(page, order, current_order, 2182 start_migratetype, fallback_mt); 2183 2184 return page; 2185 } 2186 2187 return NULL; 2188 } 2189 2190 /* 2191 * Do the hard work of removing an element from the buddy allocator. 2192 * Call me with the zone->lock already held. 2193 */ 2194 static struct page *__rmqueue(struct zone *zone, unsigned int order, 2195 int migratetype) 2196 { 2197 struct page *page; 2198 2199 page = __rmqueue_smallest(zone, order, migratetype); 2200 if (unlikely(!page)) { 2201 if (migratetype == MIGRATE_MOVABLE) 2202 page = __rmqueue_cma_fallback(zone, order); 2203 2204 if (!page) 2205 page = __rmqueue_fallback(zone, order, migratetype); 2206 } 2207 2208 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2209 return page; 2210 } 2211 2212 /* 2213 * Obtain a specified number of elements from the buddy allocator, all under 2214 * a single hold of the lock, for efficiency. Add them to the supplied list. 2215 * Returns the number of new pages which were placed at *list. 2216 */ 2217 static int rmqueue_bulk(struct zone *zone, unsigned int order, 2218 unsigned long count, struct list_head *list, 2219 int migratetype, bool cold) 2220 { 2221 int i, alloced = 0; 2222 2223 spin_lock(&zone->lock); 2224 for (i = 0; i < count; ++i) { 2225 struct page *page = __rmqueue(zone, order, migratetype); 2226 if (unlikely(page == NULL)) 2227 break; 2228 2229 if (unlikely(check_pcp_refill(page))) 2230 continue; 2231 2232 /* 2233 * Split buddy pages returned by expand() are received here 2234 * in physical page order. The page is added to the callers and 2235 * list and the list head then moves forward. From the callers 2236 * perspective, the linked list is ordered by page number in 2237 * some conditions. This is useful for IO devices that can 2238 * merge IO requests if the physical pages are ordered 2239 * properly. 2240 */ 2241 if (likely(!cold)) 2242 list_add(&page->lru, list); 2243 else 2244 list_add_tail(&page->lru, list); 2245 list = &page->lru; 2246 alloced++; 2247 if (is_migrate_cma(get_pcppage_migratetype(page))) 2248 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 2249 -(1 << order)); 2250 } 2251 2252 /* 2253 * i pages were removed from the buddy list even if some leak due 2254 * to check_pcp_refill failing so adjust NR_FREE_PAGES based 2255 * on i. Do not confuse with 'alloced' which is the number of 2256 * pages added to the pcp list. 2257 */ 2258 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 2259 spin_unlock(&zone->lock); 2260 return alloced; 2261 } 2262 2263 #ifdef CONFIG_NUMA 2264 /* 2265 * Called from the vmstat counter updater to drain pagesets of this 2266 * currently executing processor on remote nodes after they have 2267 * expired. 2268 * 2269 * Note that this function must be called with the thread pinned to 2270 * a single processor. 2271 */ 2272 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 2273 { 2274 unsigned long flags; 2275 int to_drain, batch; 2276 2277 local_irq_save(flags); 2278 batch = READ_ONCE(pcp->batch); 2279 to_drain = min(pcp->count, batch); 2280 if (to_drain > 0) { 2281 free_pcppages_bulk(zone, to_drain, pcp); 2282 pcp->count -= to_drain; 2283 } 2284 local_irq_restore(flags); 2285 } 2286 #endif 2287 2288 /* 2289 * Drain pcplists of the indicated processor and zone. 2290 * 2291 * The processor must either be the current processor and the 2292 * thread pinned to the current processor or a processor that 2293 * is not online. 2294 */ 2295 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 2296 { 2297 unsigned long flags; 2298 struct per_cpu_pageset *pset; 2299 struct per_cpu_pages *pcp; 2300 2301 local_irq_save(flags); 2302 pset = per_cpu_ptr(zone->pageset, cpu); 2303 2304 pcp = &pset->pcp; 2305 if (pcp->count) { 2306 free_pcppages_bulk(zone, pcp->count, pcp); 2307 pcp->count = 0; 2308 } 2309 local_irq_restore(flags); 2310 } 2311 2312 /* 2313 * Drain pcplists of all zones on the indicated processor. 2314 * 2315 * The processor must either be the current processor and the 2316 * thread pinned to the current processor or a processor that 2317 * is not online. 2318 */ 2319 static void drain_pages(unsigned int cpu) 2320 { 2321 struct zone *zone; 2322 2323 for_each_populated_zone(zone) { 2324 drain_pages_zone(cpu, zone); 2325 } 2326 } 2327 2328 /* 2329 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 2330 * 2331 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 2332 * the single zone's pages. 2333 */ 2334 void drain_local_pages(struct zone *zone) 2335 { 2336 int cpu = smp_processor_id(); 2337 2338 if (zone) 2339 drain_pages_zone(cpu, zone); 2340 else 2341 drain_pages(cpu); 2342 } 2343 2344 /* 2345 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 2346 * 2347 * When zone parameter is non-NULL, spill just the single zone's pages. 2348 * 2349 * Note that this code is protected against sending an IPI to an offline 2350 * CPU but does not guarantee sending an IPI to newly hotplugged CPUs: 2351 * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but 2352 * nothing keeps CPUs from showing up after we populated the cpumask and 2353 * before the call to on_each_cpu_mask(). 2354 */ 2355 void drain_all_pages(struct zone *zone) 2356 { 2357 int cpu; 2358 2359 /* 2360 * Allocate in the BSS so we wont require allocation in 2361 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 2362 */ 2363 static cpumask_t cpus_with_pcps; 2364 2365 /* 2366 * We don't care about racing with CPU hotplug event 2367 * as offline notification will cause the notified 2368 * cpu to drain that CPU pcps and on_each_cpu_mask 2369 * disables preemption as part of its processing 2370 */ 2371 for_each_online_cpu(cpu) { 2372 struct per_cpu_pageset *pcp; 2373 struct zone *z; 2374 bool has_pcps = false; 2375 2376 if (zone) { 2377 pcp = per_cpu_ptr(zone->pageset, cpu); 2378 if (pcp->pcp.count) 2379 has_pcps = true; 2380 } else { 2381 for_each_populated_zone(z) { 2382 pcp = per_cpu_ptr(z->pageset, cpu); 2383 if (pcp->pcp.count) { 2384 has_pcps = true; 2385 break; 2386 } 2387 } 2388 } 2389 2390 if (has_pcps) 2391 cpumask_set_cpu(cpu, &cpus_with_pcps); 2392 else 2393 cpumask_clear_cpu(cpu, &cpus_with_pcps); 2394 } 2395 on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages, 2396 zone, 1); 2397 } 2398 2399 #ifdef CONFIG_HIBERNATION 2400 2401 void mark_free_pages(struct zone *zone) 2402 { 2403 unsigned long pfn, max_zone_pfn; 2404 unsigned long flags; 2405 unsigned int order, t; 2406 struct page *page; 2407 2408 if (zone_is_empty(zone)) 2409 return; 2410 2411 spin_lock_irqsave(&zone->lock, flags); 2412 2413 max_zone_pfn = zone_end_pfn(zone); 2414 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 2415 if (pfn_valid(pfn)) { 2416 page = pfn_to_page(pfn); 2417 2418 if (page_zone(page) != zone) 2419 continue; 2420 2421 if (!swsusp_page_is_forbidden(page)) 2422 swsusp_unset_page_free(page); 2423 } 2424 2425 for_each_migratetype_order(order, t) { 2426 list_for_each_entry(page, 2427 &zone->free_area[order].free_list[t], lru) { 2428 unsigned long i; 2429 2430 pfn = page_to_pfn(page); 2431 for (i = 0; i < (1UL << order); i++) 2432 swsusp_set_page_free(pfn_to_page(pfn + i)); 2433 } 2434 } 2435 spin_unlock_irqrestore(&zone->lock, flags); 2436 } 2437 #endif /* CONFIG_PM */ 2438 2439 /* 2440 * Free a 0-order page 2441 * cold == true ? free a cold page : free a hot page 2442 */ 2443 void free_hot_cold_page(struct page *page, bool cold) 2444 { 2445 struct zone *zone = page_zone(page); 2446 struct per_cpu_pages *pcp; 2447 unsigned long flags; 2448 unsigned long pfn = page_to_pfn(page); 2449 int migratetype; 2450 2451 if (!free_pcp_prepare(page)) 2452 return; 2453 2454 migratetype = get_pfnblock_migratetype(page, pfn); 2455 set_pcppage_migratetype(page, migratetype); 2456 local_irq_save(flags); 2457 __count_vm_event(PGFREE); 2458 2459 /* 2460 * We only track unmovable, reclaimable and movable on pcp lists. 2461 * Free ISOLATE pages back to the allocator because they are being 2462 * offlined but treat RESERVE as movable pages so we can get those 2463 * areas back if necessary. Otherwise, we may have to free 2464 * excessively into the page allocator 2465 */ 2466 if (migratetype >= MIGRATE_PCPTYPES) { 2467 if (unlikely(is_migrate_isolate(migratetype))) { 2468 free_one_page(zone, page, pfn, 0, migratetype); 2469 goto out; 2470 } 2471 migratetype = MIGRATE_MOVABLE; 2472 } 2473 2474 pcp = &this_cpu_ptr(zone->pageset)->pcp; 2475 if (!cold) 2476 list_add(&page->lru, &pcp->lists[migratetype]); 2477 else 2478 list_add_tail(&page->lru, &pcp->lists[migratetype]); 2479 pcp->count++; 2480 if (pcp->count >= pcp->high) { 2481 unsigned long batch = READ_ONCE(pcp->batch); 2482 free_pcppages_bulk(zone, batch, pcp); 2483 pcp->count -= batch; 2484 } 2485 2486 out: 2487 local_irq_restore(flags); 2488 } 2489 2490 /* 2491 * Free a list of 0-order pages 2492 */ 2493 void free_hot_cold_page_list(struct list_head *list, bool cold) 2494 { 2495 struct page *page, *next; 2496 2497 list_for_each_entry_safe(page, next, list, lru) { 2498 trace_mm_page_free_batched(page, cold); 2499 free_hot_cold_page(page, cold); 2500 } 2501 } 2502 2503 /* 2504 * split_page takes a non-compound higher-order page, and splits it into 2505 * n (1<<order) sub-pages: page[0..n] 2506 * Each sub-page must be freed individually. 2507 * 2508 * Note: this is probably too low level an operation for use in drivers. 2509 * Please consult with lkml before using this in your driver. 2510 */ 2511 void split_page(struct page *page, unsigned int order) 2512 { 2513 int i; 2514 2515 VM_BUG_ON_PAGE(PageCompound(page), page); 2516 VM_BUG_ON_PAGE(!page_count(page), page); 2517 2518 #ifdef CONFIG_KMEMCHECK 2519 /* 2520 * Split shadow pages too, because free(page[0]) would 2521 * otherwise free the whole shadow. 2522 */ 2523 if (kmemcheck_page_is_tracked(page)) 2524 split_page(virt_to_page(page[0].shadow), order); 2525 #endif 2526 2527 for (i = 1; i < (1 << order); i++) 2528 set_page_refcounted(page + i); 2529 split_page_owner(page, order); 2530 } 2531 EXPORT_SYMBOL_GPL(split_page); 2532 2533 int __isolate_free_page(struct page *page, unsigned int order) 2534 { 2535 unsigned long watermark; 2536 struct zone *zone; 2537 int mt; 2538 2539 BUG_ON(!PageBuddy(page)); 2540 2541 zone = page_zone(page); 2542 mt = get_pageblock_migratetype(page); 2543 2544 if (!is_migrate_isolate(mt)) { 2545 /* 2546 * Obey watermarks as if the page was being allocated. We can 2547 * emulate a high-order watermark check with a raised order-0 2548 * watermark, because we already know our high-order page 2549 * exists. 2550 */ 2551 watermark = min_wmark_pages(zone) + (1UL << order); 2552 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 2553 return 0; 2554 2555 __mod_zone_freepage_state(zone, -(1UL << order), mt); 2556 } 2557 2558 /* Remove page from free list */ 2559 list_del(&page->lru); 2560 zone->free_area[order].nr_free--; 2561 rmv_page_order(page); 2562 2563 /* 2564 * Set the pageblock if the isolated page is at least half of a 2565 * pageblock 2566 */ 2567 if (order >= pageblock_order - 1) { 2568 struct page *endpage = page + (1 << order) - 1; 2569 for (; page < endpage; page += pageblock_nr_pages) { 2570 int mt = get_pageblock_migratetype(page); 2571 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt) 2572 && mt != MIGRATE_HIGHATOMIC) 2573 set_pageblock_migratetype(page, 2574 MIGRATE_MOVABLE); 2575 } 2576 } 2577 2578 2579 return 1UL << order; 2580 } 2581 2582 /* 2583 * Update NUMA hit/miss statistics 2584 * 2585 * Must be called with interrupts disabled. 2586 * 2587 * When __GFP_OTHER_NODE is set assume the node of the preferred 2588 * zone is the local node. This is useful for daemons who allocate 2589 * memory on behalf of other processes. 2590 */ 2591 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z, 2592 gfp_t flags) 2593 { 2594 #ifdef CONFIG_NUMA 2595 int local_nid = numa_node_id(); 2596 enum zone_stat_item local_stat = NUMA_LOCAL; 2597 2598 if (unlikely(flags & __GFP_OTHER_NODE)) { 2599 local_stat = NUMA_OTHER; 2600 local_nid = preferred_zone->node; 2601 } 2602 2603 if (z->node == local_nid) { 2604 __inc_zone_state(z, NUMA_HIT); 2605 __inc_zone_state(z, local_stat); 2606 } else { 2607 __inc_zone_state(z, NUMA_MISS); 2608 __inc_zone_state(preferred_zone, NUMA_FOREIGN); 2609 } 2610 #endif 2611 } 2612 2613 /* 2614 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 2615 */ 2616 static inline 2617 struct page *buffered_rmqueue(struct zone *preferred_zone, 2618 struct zone *zone, unsigned int order, 2619 gfp_t gfp_flags, unsigned int alloc_flags, 2620 int migratetype) 2621 { 2622 unsigned long flags; 2623 struct page *page; 2624 bool cold = ((gfp_flags & __GFP_COLD) != 0); 2625 2626 if (likely(order == 0)) { 2627 struct per_cpu_pages *pcp; 2628 struct list_head *list; 2629 2630 local_irq_save(flags); 2631 do { 2632 pcp = &this_cpu_ptr(zone->pageset)->pcp; 2633 list = &pcp->lists[migratetype]; 2634 if (list_empty(list)) { 2635 pcp->count += rmqueue_bulk(zone, 0, 2636 pcp->batch, list, 2637 migratetype, cold); 2638 if (unlikely(list_empty(list))) 2639 goto failed; 2640 } 2641 2642 if (cold) 2643 page = list_last_entry(list, struct page, lru); 2644 else 2645 page = list_first_entry(list, struct page, lru); 2646 2647 list_del(&page->lru); 2648 pcp->count--; 2649 2650 } while (check_new_pcp(page)); 2651 } else { 2652 /* 2653 * We most definitely don't want callers attempting to 2654 * allocate greater than order-1 page units with __GFP_NOFAIL. 2655 */ 2656 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 2657 spin_lock_irqsave(&zone->lock, flags); 2658 2659 do { 2660 page = NULL; 2661 if (alloc_flags & ALLOC_HARDER) { 2662 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 2663 if (page) 2664 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2665 } 2666 if (!page) 2667 page = __rmqueue(zone, order, migratetype); 2668 } while (page && check_new_pages(page, order)); 2669 spin_unlock(&zone->lock); 2670 if (!page) 2671 goto failed; 2672 __mod_zone_freepage_state(zone, -(1 << order), 2673 get_pcppage_migratetype(page)); 2674 } 2675 2676 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 2677 zone_statistics(preferred_zone, zone, gfp_flags); 2678 local_irq_restore(flags); 2679 2680 VM_BUG_ON_PAGE(bad_range(zone, page), page); 2681 return page; 2682 2683 failed: 2684 local_irq_restore(flags); 2685 return NULL; 2686 } 2687 2688 #ifdef CONFIG_FAIL_PAGE_ALLOC 2689 2690 static struct { 2691 struct fault_attr attr; 2692 2693 bool ignore_gfp_highmem; 2694 bool ignore_gfp_reclaim; 2695 u32 min_order; 2696 } fail_page_alloc = { 2697 .attr = FAULT_ATTR_INITIALIZER, 2698 .ignore_gfp_reclaim = true, 2699 .ignore_gfp_highmem = true, 2700 .min_order = 1, 2701 }; 2702 2703 static int __init setup_fail_page_alloc(char *str) 2704 { 2705 return setup_fault_attr(&fail_page_alloc.attr, str); 2706 } 2707 __setup("fail_page_alloc=", setup_fail_page_alloc); 2708 2709 static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 2710 { 2711 if (order < fail_page_alloc.min_order) 2712 return false; 2713 if (gfp_mask & __GFP_NOFAIL) 2714 return false; 2715 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 2716 return false; 2717 if (fail_page_alloc.ignore_gfp_reclaim && 2718 (gfp_mask & __GFP_DIRECT_RECLAIM)) 2719 return false; 2720 2721 return should_fail(&fail_page_alloc.attr, 1 << order); 2722 } 2723 2724 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 2725 2726 static int __init fail_page_alloc_debugfs(void) 2727 { 2728 umode_t mode = S_IFREG | S_IRUSR | S_IWUSR; 2729 struct dentry *dir; 2730 2731 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 2732 &fail_page_alloc.attr); 2733 if (IS_ERR(dir)) 2734 return PTR_ERR(dir); 2735 2736 if (!debugfs_create_bool("ignore-gfp-wait", mode, dir, 2737 &fail_page_alloc.ignore_gfp_reclaim)) 2738 goto fail; 2739 if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir, 2740 &fail_page_alloc.ignore_gfp_highmem)) 2741 goto fail; 2742 if (!debugfs_create_u32("min-order", mode, dir, 2743 &fail_page_alloc.min_order)) 2744 goto fail; 2745 2746 return 0; 2747 fail: 2748 debugfs_remove_recursive(dir); 2749 2750 return -ENOMEM; 2751 } 2752 2753 late_initcall(fail_page_alloc_debugfs); 2754 2755 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 2756 2757 #else /* CONFIG_FAIL_PAGE_ALLOC */ 2758 2759 static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 2760 { 2761 return false; 2762 } 2763 2764 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 2765 2766 /* 2767 * Return true if free base pages are above 'mark'. For high-order checks it 2768 * will return true of the order-0 watermark is reached and there is at least 2769 * one free page of a suitable size. Checking now avoids taking the zone lock 2770 * to check in the allocation paths if no pages are free. 2771 */ 2772 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 2773 int classzone_idx, unsigned int alloc_flags, 2774 long free_pages) 2775 { 2776 long min = mark; 2777 int o; 2778 const bool alloc_harder = (alloc_flags & ALLOC_HARDER); 2779 2780 /* free_pages may go negative - that's OK */ 2781 free_pages -= (1 << order) - 1; 2782 2783 if (alloc_flags & ALLOC_HIGH) 2784 min -= min / 2; 2785 2786 /* 2787 * If the caller does not have rights to ALLOC_HARDER then subtract 2788 * the high-atomic reserves. This will over-estimate the size of the 2789 * atomic reserve but it avoids a search. 2790 */ 2791 if (likely(!alloc_harder)) 2792 free_pages -= z->nr_reserved_highatomic; 2793 else 2794 min -= min / 4; 2795 2796 #ifdef CONFIG_CMA 2797 /* If allocation can't use CMA areas don't use free CMA pages */ 2798 if (!(alloc_flags & ALLOC_CMA)) 2799 free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES); 2800 #endif 2801 2802 /* 2803 * Check watermarks for an order-0 allocation request. If these 2804 * are not met, then a high-order request also cannot go ahead 2805 * even if a suitable page happened to be free. 2806 */ 2807 if (free_pages <= min + z->lowmem_reserve[classzone_idx]) 2808 return false; 2809 2810 /* If this is an order-0 request then the watermark is fine */ 2811 if (!order) 2812 return true; 2813 2814 /* For a high-order request, check at least one suitable page is free */ 2815 for (o = order; o < MAX_ORDER; o++) { 2816 struct free_area *area = &z->free_area[o]; 2817 int mt; 2818 2819 if (!area->nr_free) 2820 continue; 2821 2822 if (alloc_harder) 2823 return true; 2824 2825 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 2826 if (!list_empty(&area->free_list[mt])) 2827 return true; 2828 } 2829 2830 #ifdef CONFIG_CMA 2831 if ((alloc_flags & ALLOC_CMA) && 2832 !list_empty(&area->free_list[MIGRATE_CMA])) { 2833 return true; 2834 } 2835 #endif 2836 } 2837 return false; 2838 } 2839 2840 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 2841 int classzone_idx, unsigned int alloc_flags) 2842 { 2843 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 2844 zone_page_state(z, NR_FREE_PAGES)); 2845 } 2846 2847 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 2848 unsigned long mark, int classzone_idx, unsigned int alloc_flags) 2849 { 2850 long free_pages = zone_page_state(z, NR_FREE_PAGES); 2851 long cma_pages = 0; 2852 2853 #ifdef CONFIG_CMA 2854 /* If allocation can't use CMA areas don't use free CMA pages */ 2855 if (!(alloc_flags & ALLOC_CMA)) 2856 cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES); 2857 #endif 2858 2859 /* 2860 * Fast check for order-0 only. If this fails then the reserves 2861 * need to be calculated. There is a corner case where the check 2862 * passes but only the high-order atomic reserve are free. If 2863 * the caller is !atomic then it'll uselessly search the free 2864 * list. That corner case is then slower but it is harmless. 2865 */ 2866 if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx]) 2867 return true; 2868 2869 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 2870 free_pages); 2871 } 2872 2873 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 2874 unsigned long mark, int classzone_idx) 2875 { 2876 long free_pages = zone_page_state(z, NR_FREE_PAGES); 2877 2878 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 2879 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 2880 2881 return __zone_watermark_ok(z, order, mark, classzone_idx, 0, 2882 free_pages); 2883 } 2884 2885 #ifdef CONFIG_NUMA 2886 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 2887 { 2888 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) < 2889 RECLAIM_DISTANCE; 2890 } 2891 #else /* CONFIG_NUMA */ 2892 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 2893 { 2894 return true; 2895 } 2896 #endif /* CONFIG_NUMA */ 2897 2898 /* 2899 * get_page_from_freelist goes through the zonelist trying to allocate 2900 * a page. 2901 */ 2902 static struct page * 2903 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 2904 const struct alloc_context *ac) 2905 { 2906 struct zoneref *z = ac->preferred_zoneref; 2907 struct zone *zone; 2908 struct pglist_data *last_pgdat_dirty_limit = NULL; 2909 2910 /* 2911 * Scan zonelist, looking for a zone with enough free. 2912 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 2913 */ 2914 for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 2915 ac->nodemask) { 2916 struct page *page; 2917 unsigned long mark; 2918 2919 if (cpusets_enabled() && 2920 (alloc_flags & ALLOC_CPUSET) && 2921 !__cpuset_zone_allowed(zone, gfp_mask)) 2922 continue; 2923 /* 2924 * When allocating a page cache page for writing, we 2925 * want to get it from a node that is within its dirty 2926 * limit, such that no single node holds more than its 2927 * proportional share of globally allowed dirty pages. 2928 * The dirty limits take into account the node's 2929 * lowmem reserves and high watermark so that kswapd 2930 * should be able to balance it without having to 2931 * write pages from its LRU list. 2932 * 2933 * XXX: For now, allow allocations to potentially 2934 * exceed the per-node dirty limit in the slowpath 2935 * (spread_dirty_pages unset) before going into reclaim, 2936 * which is important when on a NUMA setup the allowed 2937 * nodes are together not big enough to reach the 2938 * global limit. The proper fix for these situations 2939 * will require awareness of nodes in the 2940 * dirty-throttling and the flusher threads. 2941 */ 2942 if (ac->spread_dirty_pages) { 2943 if (last_pgdat_dirty_limit == zone->zone_pgdat) 2944 continue; 2945 2946 if (!node_dirty_ok(zone->zone_pgdat)) { 2947 last_pgdat_dirty_limit = zone->zone_pgdat; 2948 continue; 2949 } 2950 } 2951 2952 mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK]; 2953 if (!zone_watermark_fast(zone, order, mark, 2954 ac_classzone_idx(ac), alloc_flags)) { 2955 int ret; 2956 2957 /* Checked here to keep the fast path fast */ 2958 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 2959 if (alloc_flags & ALLOC_NO_WATERMARKS) 2960 goto try_this_zone; 2961 2962 if (node_reclaim_mode == 0 || 2963 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 2964 continue; 2965 2966 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 2967 switch (ret) { 2968 case NODE_RECLAIM_NOSCAN: 2969 /* did not scan */ 2970 continue; 2971 case NODE_RECLAIM_FULL: 2972 /* scanned but unreclaimable */ 2973 continue; 2974 default: 2975 /* did we reclaim enough */ 2976 if (zone_watermark_ok(zone, order, mark, 2977 ac_classzone_idx(ac), alloc_flags)) 2978 goto try_this_zone; 2979 2980 continue; 2981 } 2982 } 2983 2984 try_this_zone: 2985 page = buffered_rmqueue(ac->preferred_zoneref->zone, zone, order, 2986 gfp_mask, alloc_flags, ac->migratetype); 2987 if (page) { 2988 prep_new_page(page, order, gfp_mask, alloc_flags); 2989 2990 /* 2991 * If this is a high-order atomic allocation then check 2992 * if the pageblock should be reserved for the future 2993 */ 2994 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 2995 reserve_highatomic_pageblock(page, zone, order); 2996 2997 return page; 2998 } 2999 } 3000 3001 return NULL; 3002 } 3003 3004 /* 3005 * Large machines with many possible nodes should not always dump per-node 3006 * meminfo in irq context. 3007 */ 3008 static inline bool should_suppress_show_mem(void) 3009 { 3010 bool ret = false; 3011 3012 #if NODES_SHIFT > 8 3013 ret = in_interrupt(); 3014 #endif 3015 return ret; 3016 } 3017 3018 static DEFINE_RATELIMIT_STATE(nopage_rs, 3019 DEFAULT_RATELIMIT_INTERVAL, 3020 DEFAULT_RATELIMIT_BURST); 3021 3022 void warn_alloc(gfp_t gfp_mask, const char *fmt, ...) 3023 { 3024 unsigned int filter = SHOW_MEM_FILTER_NODES; 3025 struct va_format vaf; 3026 va_list args; 3027 3028 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) || 3029 debug_guardpage_minorder() > 0) 3030 return; 3031 3032 /* 3033 * This documents exceptions given to allocations in certain 3034 * contexts that are allowed to allocate outside current's set 3035 * of allowed nodes. 3036 */ 3037 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3038 if (test_thread_flag(TIF_MEMDIE) || 3039 (current->flags & (PF_MEMALLOC | PF_EXITING))) 3040 filter &= ~SHOW_MEM_FILTER_NODES; 3041 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 3042 filter &= ~SHOW_MEM_FILTER_NODES; 3043 3044 pr_warn("%s: ", current->comm); 3045 3046 va_start(args, fmt); 3047 vaf.fmt = fmt; 3048 vaf.va = &args; 3049 pr_cont("%pV", &vaf); 3050 va_end(args); 3051 3052 pr_cont(", mode:%#x(%pGg)\n", gfp_mask, &gfp_mask); 3053 3054 dump_stack(); 3055 if (!should_suppress_show_mem()) 3056 show_mem(filter); 3057 } 3058 3059 static inline struct page * 3060 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 3061 const struct alloc_context *ac, unsigned long *did_some_progress) 3062 { 3063 struct oom_control oc = { 3064 .zonelist = ac->zonelist, 3065 .nodemask = ac->nodemask, 3066 .memcg = NULL, 3067 .gfp_mask = gfp_mask, 3068 .order = order, 3069 }; 3070 struct page *page; 3071 3072 *did_some_progress = 0; 3073 3074 /* 3075 * Acquire the oom lock. If that fails, somebody else is 3076 * making progress for us. 3077 */ 3078 if (!mutex_trylock(&oom_lock)) { 3079 *did_some_progress = 1; 3080 schedule_timeout_uninterruptible(1); 3081 return NULL; 3082 } 3083 3084 /* 3085 * Go through the zonelist yet one more time, keep very high watermark 3086 * here, this is only to catch a parallel oom killing, we must fail if 3087 * we're still under heavy pressure. 3088 */ 3089 page = get_page_from_freelist(gfp_mask | __GFP_HARDWALL, order, 3090 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 3091 if (page) 3092 goto out; 3093 3094 if (!(gfp_mask & __GFP_NOFAIL)) { 3095 /* Coredumps can quickly deplete all memory reserves */ 3096 if (current->flags & PF_DUMPCORE) 3097 goto out; 3098 /* The OOM killer will not help higher order allocs */ 3099 if (order > PAGE_ALLOC_COSTLY_ORDER) 3100 goto out; 3101 /* The OOM killer does not needlessly kill tasks for lowmem */ 3102 if (ac->high_zoneidx < ZONE_NORMAL) 3103 goto out; 3104 if (pm_suspended_storage()) 3105 goto out; 3106 /* 3107 * XXX: GFP_NOFS allocations should rather fail than rely on 3108 * other request to make a forward progress. 3109 * We are in an unfortunate situation where out_of_memory cannot 3110 * do much for this context but let's try it to at least get 3111 * access to memory reserved if the current task is killed (see 3112 * out_of_memory). Once filesystems are ready to handle allocation 3113 * failures more gracefully we should just bail out here. 3114 */ 3115 3116 /* The OOM killer may not free memory on a specific node */ 3117 if (gfp_mask & __GFP_THISNODE) 3118 goto out; 3119 } 3120 /* Exhausted what can be done so it's blamo time */ 3121 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 3122 *did_some_progress = 1; 3123 3124 if (gfp_mask & __GFP_NOFAIL) { 3125 page = get_page_from_freelist(gfp_mask, order, 3126 ALLOC_NO_WATERMARKS|ALLOC_CPUSET, ac); 3127 /* 3128 * fallback to ignore cpuset restriction if our nodes 3129 * are depleted 3130 */ 3131 if (!page) 3132 page = get_page_from_freelist(gfp_mask, order, 3133 ALLOC_NO_WATERMARKS, ac); 3134 } 3135 } 3136 out: 3137 mutex_unlock(&oom_lock); 3138 return page; 3139 } 3140 3141 /* 3142 * Maximum number of compaction retries wit a progress before OOM 3143 * killer is consider as the only way to move forward. 3144 */ 3145 #define MAX_COMPACT_RETRIES 16 3146 3147 #ifdef CONFIG_COMPACTION 3148 /* Try memory compaction for high-order allocations before reclaim */ 3149 static struct page * 3150 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3151 unsigned int alloc_flags, const struct alloc_context *ac, 3152 enum compact_priority prio, enum compact_result *compact_result) 3153 { 3154 struct page *page; 3155 3156 if (!order) 3157 return NULL; 3158 3159 current->flags |= PF_MEMALLOC; 3160 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 3161 prio); 3162 current->flags &= ~PF_MEMALLOC; 3163 3164 if (*compact_result <= COMPACT_INACTIVE) 3165 return NULL; 3166 3167 /* 3168 * At least in one zone compaction wasn't deferred or skipped, so let's 3169 * count a compaction stall 3170 */ 3171 count_vm_event(COMPACTSTALL); 3172 3173 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 3174 3175 if (page) { 3176 struct zone *zone = page_zone(page); 3177 3178 zone->compact_blockskip_flush = false; 3179 compaction_defer_reset(zone, order, true); 3180 count_vm_event(COMPACTSUCCESS); 3181 return page; 3182 } 3183 3184 /* 3185 * It's bad if compaction run occurs and fails. The most likely reason 3186 * is that pages exist, but not enough to satisfy watermarks. 3187 */ 3188 count_vm_event(COMPACTFAIL); 3189 3190 cond_resched(); 3191 3192 return NULL; 3193 } 3194 3195 static inline bool 3196 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 3197 enum compact_result compact_result, 3198 enum compact_priority *compact_priority, 3199 int *compaction_retries) 3200 { 3201 int max_retries = MAX_COMPACT_RETRIES; 3202 int min_priority; 3203 3204 if (!order) 3205 return false; 3206 3207 if (compaction_made_progress(compact_result)) 3208 (*compaction_retries)++; 3209 3210 /* 3211 * compaction considers all the zone as desperately out of memory 3212 * so it doesn't really make much sense to retry except when the 3213 * failure could be caused by insufficient priority 3214 */ 3215 if (compaction_failed(compact_result)) 3216 goto check_priority; 3217 3218 /* 3219 * make sure the compaction wasn't deferred or didn't bail out early 3220 * due to locks contention before we declare that we should give up. 3221 * But do not retry if the given zonelist is not suitable for 3222 * compaction. 3223 */ 3224 if (compaction_withdrawn(compact_result)) 3225 return compaction_zonelist_suitable(ac, order, alloc_flags); 3226 3227 /* 3228 * !costly requests are much more important than __GFP_REPEAT 3229 * costly ones because they are de facto nofail and invoke OOM 3230 * killer to move on while costly can fail and users are ready 3231 * to cope with that. 1/4 retries is rather arbitrary but we 3232 * would need much more detailed feedback from compaction to 3233 * make a better decision. 3234 */ 3235 if (order > PAGE_ALLOC_COSTLY_ORDER) 3236 max_retries /= 4; 3237 if (*compaction_retries <= max_retries) 3238 return true; 3239 3240 /* 3241 * Make sure there are attempts at the highest priority if we exhausted 3242 * all retries or failed at the lower priorities. 3243 */ 3244 check_priority: 3245 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 3246 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 3247 if (*compact_priority > min_priority) { 3248 (*compact_priority)--; 3249 *compaction_retries = 0; 3250 return true; 3251 } 3252 return false; 3253 } 3254 #else 3255 static inline struct page * 3256 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3257 unsigned int alloc_flags, const struct alloc_context *ac, 3258 enum compact_priority prio, enum compact_result *compact_result) 3259 { 3260 *compact_result = COMPACT_SKIPPED; 3261 return NULL; 3262 } 3263 3264 static inline bool 3265 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 3266 enum compact_result compact_result, 3267 enum compact_priority *compact_priority, 3268 int *compaction_retries) 3269 { 3270 struct zone *zone; 3271 struct zoneref *z; 3272 3273 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 3274 return false; 3275 3276 /* 3277 * There are setups with compaction disabled which would prefer to loop 3278 * inside the allocator rather than hit the oom killer prematurely. 3279 * Let's give them a good hope and keep retrying while the order-0 3280 * watermarks are OK. 3281 */ 3282 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3283 ac->nodemask) { 3284 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 3285 ac_classzone_idx(ac), alloc_flags)) 3286 return true; 3287 } 3288 return false; 3289 } 3290 #endif /* CONFIG_COMPACTION */ 3291 3292 /* Perform direct synchronous page reclaim */ 3293 static int 3294 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 3295 const struct alloc_context *ac) 3296 { 3297 struct reclaim_state reclaim_state; 3298 int progress; 3299 3300 cond_resched(); 3301 3302 /* We now go into synchronous reclaim */ 3303 cpuset_memory_pressure_bump(); 3304 current->flags |= PF_MEMALLOC; 3305 lockdep_set_current_reclaim_state(gfp_mask); 3306 reclaim_state.reclaimed_slab = 0; 3307 current->reclaim_state = &reclaim_state; 3308 3309 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 3310 ac->nodemask); 3311 3312 current->reclaim_state = NULL; 3313 lockdep_clear_current_reclaim_state(); 3314 current->flags &= ~PF_MEMALLOC; 3315 3316 cond_resched(); 3317 3318 return progress; 3319 } 3320 3321 /* The really slow allocator path where we enter direct reclaim */ 3322 static inline struct page * 3323 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 3324 unsigned int alloc_flags, const struct alloc_context *ac, 3325 unsigned long *did_some_progress) 3326 { 3327 struct page *page = NULL; 3328 bool drained = false; 3329 3330 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 3331 if (unlikely(!(*did_some_progress))) 3332 return NULL; 3333 3334 retry: 3335 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 3336 3337 /* 3338 * If an allocation failed after direct reclaim, it could be because 3339 * pages are pinned on the per-cpu lists or in high alloc reserves. 3340 * Shrink them them and try again 3341 */ 3342 if (!page && !drained) { 3343 unreserve_highatomic_pageblock(ac, false); 3344 drain_all_pages(NULL); 3345 drained = true; 3346 goto retry; 3347 } 3348 3349 return page; 3350 } 3351 3352 static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac) 3353 { 3354 struct zoneref *z; 3355 struct zone *zone; 3356 pg_data_t *last_pgdat = NULL; 3357 3358 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 3359 ac->high_zoneidx, ac->nodemask) { 3360 if (last_pgdat != zone->zone_pgdat) 3361 wakeup_kswapd(zone, order, ac->high_zoneidx); 3362 last_pgdat = zone->zone_pgdat; 3363 } 3364 } 3365 3366 static inline unsigned int 3367 gfp_to_alloc_flags(gfp_t gfp_mask) 3368 { 3369 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 3370 3371 /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */ 3372 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 3373 3374 /* 3375 * The caller may dip into page reserves a bit more if the caller 3376 * cannot run direct reclaim, or if the caller has realtime scheduling 3377 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 3378 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 3379 */ 3380 alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); 3381 3382 if (gfp_mask & __GFP_ATOMIC) { 3383 /* 3384 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 3385 * if it can't schedule. 3386 */ 3387 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3388 alloc_flags |= ALLOC_HARDER; 3389 /* 3390 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 3391 * comment for __cpuset_node_allowed(). 3392 */ 3393 alloc_flags &= ~ALLOC_CPUSET; 3394 } else if (unlikely(rt_task(current)) && !in_interrupt()) 3395 alloc_flags |= ALLOC_HARDER; 3396 3397 #ifdef CONFIG_CMA 3398 if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) 3399 alloc_flags |= ALLOC_CMA; 3400 #endif 3401 return alloc_flags; 3402 } 3403 3404 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 3405 { 3406 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 3407 return false; 3408 3409 if (gfp_mask & __GFP_MEMALLOC) 3410 return true; 3411 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 3412 return true; 3413 if (!in_interrupt() && 3414 ((current->flags & PF_MEMALLOC) || 3415 unlikely(test_thread_flag(TIF_MEMDIE)))) 3416 return true; 3417 3418 return false; 3419 } 3420 3421 /* 3422 * Maximum number of reclaim retries without any progress before OOM killer 3423 * is consider as the only way to move forward. 3424 */ 3425 #define MAX_RECLAIM_RETRIES 16 3426 3427 /* 3428 * Checks whether it makes sense to retry the reclaim to make a forward progress 3429 * for the given allocation request. 3430 * The reclaim feedback represented by did_some_progress (any progress during 3431 * the last reclaim round) and no_progress_loops (number of reclaim rounds without 3432 * any progress in a row) is considered as well as the reclaimable pages on the 3433 * applicable zone list (with a backoff mechanism which is a function of 3434 * no_progress_loops). 3435 * 3436 * Returns true if a retry is viable or false to enter the oom path. 3437 */ 3438 static inline bool 3439 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 3440 struct alloc_context *ac, int alloc_flags, 3441 bool did_some_progress, int *no_progress_loops) 3442 { 3443 struct zone *zone; 3444 struct zoneref *z; 3445 3446 /* 3447 * Costly allocations might have made a progress but this doesn't mean 3448 * their order will become available due to high fragmentation so 3449 * always increment the no progress counter for them 3450 */ 3451 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 3452 *no_progress_loops = 0; 3453 else 3454 (*no_progress_loops)++; 3455 3456 /* 3457 * Make sure we converge to OOM if we cannot make any progress 3458 * several times in the row. 3459 */ 3460 if (*no_progress_loops > MAX_RECLAIM_RETRIES) { 3461 /* Before OOM, exhaust highatomic_reserve */ 3462 return unreserve_highatomic_pageblock(ac, true); 3463 } 3464 3465 /* 3466 * Keep reclaiming pages while there is a chance this will lead 3467 * somewhere. If none of the target zones can satisfy our allocation 3468 * request even if all reclaimable pages are considered then we are 3469 * screwed and have to go OOM. 3470 */ 3471 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3472 ac->nodemask) { 3473 unsigned long available; 3474 unsigned long reclaimable; 3475 3476 available = reclaimable = zone_reclaimable_pages(zone); 3477 available -= DIV_ROUND_UP((*no_progress_loops) * available, 3478 MAX_RECLAIM_RETRIES); 3479 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 3480 3481 /* 3482 * Would the allocation succeed if we reclaimed the whole 3483 * available? 3484 */ 3485 if (__zone_watermark_ok(zone, order, min_wmark_pages(zone), 3486 ac_classzone_idx(ac), alloc_flags, available)) { 3487 /* 3488 * If we didn't make any progress and have a lot of 3489 * dirty + writeback pages then we should wait for 3490 * an IO to complete to slow down the reclaim and 3491 * prevent from pre mature OOM 3492 */ 3493 if (!did_some_progress) { 3494 unsigned long write_pending; 3495 3496 write_pending = zone_page_state_snapshot(zone, 3497 NR_ZONE_WRITE_PENDING); 3498 3499 if (2 * write_pending > reclaimable) { 3500 congestion_wait(BLK_RW_ASYNC, HZ/10); 3501 return true; 3502 } 3503 } 3504 3505 /* 3506 * Memory allocation/reclaim might be called from a WQ 3507 * context and the current implementation of the WQ 3508 * concurrency control doesn't recognize that 3509 * a particular WQ is congested if the worker thread is 3510 * looping without ever sleeping. Therefore we have to 3511 * do a short sleep here rather than calling 3512 * cond_resched(). 3513 */ 3514 if (current->flags & PF_WQ_WORKER) 3515 schedule_timeout_uninterruptible(1); 3516 else 3517 cond_resched(); 3518 3519 return true; 3520 } 3521 } 3522 3523 return false; 3524 } 3525 3526 static inline struct page * 3527 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 3528 struct alloc_context *ac) 3529 { 3530 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 3531 struct page *page = NULL; 3532 unsigned int alloc_flags; 3533 unsigned long did_some_progress; 3534 enum compact_priority compact_priority = DEF_COMPACT_PRIORITY; 3535 enum compact_result compact_result; 3536 int compaction_retries = 0; 3537 int no_progress_loops = 0; 3538 unsigned long alloc_start = jiffies; 3539 unsigned int stall_timeout = 10 * HZ; 3540 3541 /* 3542 * In the slowpath, we sanity check order to avoid ever trying to 3543 * reclaim >= MAX_ORDER areas which will never succeed. Callers may 3544 * be using allocators in order of preference for an area that is 3545 * too large. 3546 */ 3547 if (order >= MAX_ORDER) { 3548 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); 3549 return NULL; 3550 } 3551 3552 /* 3553 * We also sanity check to catch abuse of atomic reserves being used by 3554 * callers that are not in atomic context. 3555 */ 3556 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 3557 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 3558 gfp_mask &= ~__GFP_ATOMIC; 3559 3560 /* 3561 * The fast path uses conservative alloc_flags to succeed only until 3562 * kswapd needs to be woken up, and to avoid the cost of setting up 3563 * alloc_flags precisely. So we do that now. 3564 */ 3565 alloc_flags = gfp_to_alloc_flags(gfp_mask); 3566 3567 if (gfp_mask & __GFP_KSWAPD_RECLAIM) 3568 wake_all_kswapds(order, ac); 3569 3570 /* 3571 * The adjusted alloc_flags might result in immediate success, so try 3572 * that first 3573 */ 3574 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 3575 if (page) 3576 goto got_pg; 3577 3578 /* 3579 * For costly allocations, try direct compaction first, as it's likely 3580 * that we have enough base pages and don't need to reclaim. Don't try 3581 * that for allocations that are allowed to ignore watermarks, as the 3582 * ALLOC_NO_WATERMARKS attempt didn't yet happen. 3583 */ 3584 if (can_direct_reclaim && order > PAGE_ALLOC_COSTLY_ORDER && 3585 !gfp_pfmemalloc_allowed(gfp_mask)) { 3586 page = __alloc_pages_direct_compact(gfp_mask, order, 3587 alloc_flags, ac, 3588 INIT_COMPACT_PRIORITY, 3589 &compact_result); 3590 if (page) 3591 goto got_pg; 3592 3593 /* 3594 * Checks for costly allocations with __GFP_NORETRY, which 3595 * includes THP page fault allocations 3596 */ 3597 if (gfp_mask & __GFP_NORETRY) { 3598 /* 3599 * If compaction is deferred for high-order allocations, 3600 * it is because sync compaction recently failed. If 3601 * this is the case and the caller requested a THP 3602 * allocation, we do not want to heavily disrupt the 3603 * system, so we fail the allocation instead of entering 3604 * direct reclaim. 3605 */ 3606 if (compact_result == COMPACT_DEFERRED) 3607 goto nopage; 3608 3609 /* 3610 * Looks like reclaim/compaction is worth trying, but 3611 * sync compaction could be very expensive, so keep 3612 * using async compaction. 3613 */ 3614 compact_priority = INIT_COMPACT_PRIORITY; 3615 } 3616 } 3617 3618 retry: 3619 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 3620 if (gfp_mask & __GFP_KSWAPD_RECLAIM) 3621 wake_all_kswapds(order, ac); 3622 3623 if (gfp_pfmemalloc_allowed(gfp_mask)) 3624 alloc_flags = ALLOC_NO_WATERMARKS; 3625 3626 /* 3627 * Reset the zonelist iterators if memory policies can be ignored. 3628 * These allocations are high priority and system rather than user 3629 * orientated. 3630 */ 3631 if (!(alloc_flags & ALLOC_CPUSET) || (alloc_flags & ALLOC_NO_WATERMARKS)) { 3632 ac->zonelist = node_zonelist(numa_node_id(), gfp_mask); 3633 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 3634 ac->high_zoneidx, ac->nodemask); 3635 } 3636 3637 /* Attempt with potentially adjusted zonelist and alloc_flags */ 3638 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 3639 if (page) 3640 goto got_pg; 3641 3642 /* Caller is not willing to reclaim, we can't balance anything */ 3643 if (!can_direct_reclaim) { 3644 /* 3645 * All existing users of the __GFP_NOFAIL are blockable, so warn 3646 * of any new users that actually allow this type of allocation 3647 * to fail. 3648 */ 3649 WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL); 3650 goto nopage; 3651 } 3652 3653 /* Avoid recursion of direct reclaim */ 3654 if (current->flags & PF_MEMALLOC) { 3655 /* 3656 * __GFP_NOFAIL request from this context is rather bizarre 3657 * because we cannot reclaim anything and only can loop waiting 3658 * for somebody to do a work for us. 3659 */ 3660 if (WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 3661 cond_resched(); 3662 goto retry; 3663 } 3664 goto nopage; 3665 } 3666 3667 /* Avoid allocations with no watermarks from looping endlessly */ 3668 if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL)) 3669 goto nopage; 3670 3671 3672 /* Try direct reclaim and then allocating */ 3673 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 3674 &did_some_progress); 3675 if (page) 3676 goto got_pg; 3677 3678 /* Try direct compaction and then allocating */ 3679 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 3680 compact_priority, &compact_result); 3681 if (page) 3682 goto got_pg; 3683 3684 /* Do not loop if specifically requested */ 3685 if (gfp_mask & __GFP_NORETRY) 3686 goto nopage; 3687 3688 /* 3689 * Do not retry costly high order allocations unless they are 3690 * __GFP_REPEAT 3691 */ 3692 if (order > PAGE_ALLOC_COSTLY_ORDER && !(gfp_mask & __GFP_REPEAT)) 3693 goto nopage; 3694 3695 /* Make sure we know about allocations which stall for too long */ 3696 if (time_after(jiffies, alloc_start + stall_timeout)) { 3697 warn_alloc(gfp_mask, 3698 "page allocation stalls for %ums, order:%u", 3699 jiffies_to_msecs(jiffies-alloc_start), order); 3700 stall_timeout += 10 * HZ; 3701 } 3702 3703 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 3704 did_some_progress > 0, &no_progress_loops)) 3705 goto retry; 3706 3707 /* 3708 * It doesn't make any sense to retry for the compaction if the order-0 3709 * reclaim is not able to make any progress because the current 3710 * implementation of the compaction depends on the sufficient amount 3711 * of free memory (see __compaction_suitable) 3712 */ 3713 if (did_some_progress > 0 && 3714 should_compact_retry(ac, order, alloc_flags, 3715 compact_result, &compact_priority, 3716 &compaction_retries)) 3717 goto retry; 3718 3719 /* Reclaim has failed us, start killing things */ 3720 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 3721 if (page) 3722 goto got_pg; 3723 3724 /* Retry as long as the OOM killer is making progress */ 3725 if (did_some_progress) { 3726 no_progress_loops = 0; 3727 goto retry; 3728 } 3729 3730 nopage: 3731 warn_alloc(gfp_mask, 3732 "page allocation failure: order:%u", order); 3733 got_pg: 3734 return page; 3735 } 3736 3737 /* 3738 * This is the 'heart' of the zoned buddy allocator. 3739 */ 3740 struct page * 3741 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, 3742 struct zonelist *zonelist, nodemask_t *nodemask) 3743 { 3744 struct page *page; 3745 unsigned int cpuset_mems_cookie; 3746 unsigned int alloc_flags = ALLOC_WMARK_LOW; 3747 gfp_t alloc_mask = gfp_mask; /* The gfp_t that was actually used for allocation */ 3748 struct alloc_context ac = { 3749 .high_zoneidx = gfp_zone(gfp_mask), 3750 .zonelist = zonelist, 3751 .nodemask = nodemask, 3752 .migratetype = gfpflags_to_migratetype(gfp_mask), 3753 }; 3754 3755 if (cpusets_enabled()) { 3756 alloc_mask |= __GFP_HARDWALL; 3757 alloc_flags |= ALLOC_CPUSET; 3758 if (!ac.nodemask) 3759 ac.nodemask = &cpuset_current_mems_allowed; 3760 } 3761 3762 gfp_mask &= gfp_allowed_mask; 3763 3764 lockdep_trace_alloc(gfp_mask); 3765 3766 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 3767 3768 if (should_fail_alloc_page(gfp_mask, order)) 3769 return NULL; 3770 3771 /* 3772 * Check the zones suitable for the gfp_mask contain at least one 3773 * valid zone. It's possible to have an empty zonelist as a result 3774 * of __GFP_THISNODE and a memoryless node 3775 */ 3776 if (unlikely(!zonelist->_zonerefs->zone)) 3777 return NULL; 3778 3779 if (IS_ENABLED(CONFIG_CMA) && ac.migratetype == MIGRATE_MOVABLE) 3780 alloc_flags |= ALLOC_CMA; 3781 3782 retry_cpuset: 3783 cpuset_mems_cookie = read_mems_allowed_begin(); 3784 3785 /* Dirty zone balancing only done in the fast path */ 3786 ac.spread_dirty_pages = (gfp_mask & __GFP_WRITE); 3787 3788 /* 3789 * The preferred zone is used for statistics but crucially it is 3790 * also used as the starting point for the zonelist iterator. It 3791 * may get reset for allocations that ignore memory policies. 3792 */ 3793 ac.preferred_zoneref = first_zones_zonelist(ac.zonelist, 3794 ac.high_zoneidx, ac.nodemask); 3795 if (!ac.preferred_zoneref) { 3796 page = NULL; 3797 goto no_zone; 3798 } 3799 3800 /* First allocation attempt */ 3801 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); 3802 if (likely(page)) 3803 goto out; 3804 3805 /* 3806 * Runtime PM, block IO and its error handling path can deadlock 3807 * because I/O on the device might not complete. 3808 */ 3809 alloc_mask = memalloc_noio_flags(gfp_mask); 3810 ac.spread_dirty_pages = false; 3811 3812 /* 3813 * Restore the original nodemask if it was potentially replaced with 3814 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 3815 */ 3816 if (cpusets_enabled()) 3817 ac.nodemask = nodemask; 3818 page = __alloc_pages_slowpath(alloc_mask, order, &ac); 3819 3820 no_zone: 3821 /* 3822 * When updating a task's mems_allowed, it is possible to race with 3823 * parallel threads in such a way that an allocation can fail while 3824 * the mask is being updated. If a page allocation is about to fail, 3825 * check if the cpuset changed during allocation and if so, retry. 3826 */ 3827 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) { 3828 alloc_mask = gfp_mask; 3829 goto retry_cpuset; 3830 } 3831 3832 out: 3833 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && 3834 unlikely(memcg_kmem_charge(page, gfp_mask, order) != 0)) { 3835 __free_pages(page, order); 3836 page = NULL; 3837 } 3838 3839 if (kmemcheck_enabled && page) 3840 kmemcheck_pagealloc_alloc(page, order, gfp_mask); 3841 3842 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); 3843 3844 return page; 3845 } 3846 EXPORT_SYMBOL(__alloc_pages_nodemask); 3847 3848 /* 3849 * Common helper functions. 3850 */ 3851 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 3852 { 3853 struct page *page; 3854 3855 /* 3856 * __get_free_pages() returns a 32-bit address, which cannot represent 3857 * a highmem page 3858 */ 3859 VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0); 3860 3861 page = alloc_pages(gfp_mask, order); 3862 if (!page) 3863 return 0; 3864 return (unsigned long) page_address(page); 3865 } 3866 EXPORT_SYMBOL(__get_free_pages); 3867 3868 unsigned long get_zeroed_page(gfp_t gfp_mask) 3869 { 3870 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 3871 } 3872 EXPORT_SYMBOL(get_zeroed_page); 3873 3874 void __free_pages(struct page *page, unsigned int order) 3875 { 3876 if (put_page_testzero(page)) { 3877 if (order == 0) 3878 free_hot_cold_page(page, false); 3879 else 3880 __free_pages_ok(page, order); 3881 } 3882 } 3883 3884 EXPORT_SYMBOL(__free_pages); 3885 3886 void free_pages(unsigned long addr, unsigned int order) 3887 { 3888 if (addr != 0) { 3889 VM_BUG_ON(!virt_addr_valid((void *)addr)); 3890 __free_pages(virt_to_page((void *)addr), order); 3891 } 3892 } 3893 3894 EXPORT_SYMBOL(free_pages); 3895 3896 /* 3897 * Page Fragment: 3898 * An arbitrary-length arbitrary-offset area of memory which resides 3899 * within a 0 or higher order page. Multiple fragments within that page 3900 * are individually refcounted, in the page's reference counter. 3901 * 3902 * The page_frag functions below provide a simple allocation framework for 3903 * page fragments. This is used by the network stack and network device 3904 * drivers to provide a backing region of memory for use as either an 3905 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 3906 */ 3907 static struct page *__page_frag_refill(struct page_frag_cache *nc, 3908 gfp_t gfp_mask) 3909 { 3910 struct page *page = NULL; 3911 gfp_t gfp = gfp_mask; 3912 3913 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 3914 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 3915 __GFP_NOMEMALLOC; 3916 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 3917 PAGE_FRAG_CACHE_MAX_ORDER); 3918 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 3919 #endif 3920 if (unlikely(!page)) 3921 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 3922 3923 nc->va = page ? page_address(page) : NULL; 3924 3925 return page; 3926 } 3927 3928 void __page_frag_drain(struct page *page, unsigned int order, 3929 unsigned int count) 3930 { 3931 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 3932 3933 if (page_ref_sub_and_test(page, count)) { 3934 if (order == 0) 3935 free_hot_cold_page(page, false); 3936 else 3937 __free_pages_ok(page, order); 3938 } 3939 } 3940 EXPORT_SYMBOL(__page_frag_drain); 3941 3942 void *__alloc_page_frag(struct page_frag_cache *nc, 3943 unsigned int fragsz, gfp_t gfp_mask) 3944 { 3945 unsigned int size = PAGE_SIZE; 3946 struct page *page; 3947 int offset; 3948 3949 if (unlikely(!nc->va)) { 3950 refill: 3951 page = __page_frag_refill(nc, gfp_mask); 3952 if (!page) 3953 return NULL; 3954 3955 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 3956 /* if size can vary use size else just use PAGE_SIZE */ 3957 size = nc->size; 3958 #endif 3959 /* Even if we own the page, we do not use atomic_set(). 3960 * This would break get_page_unless_zero() users. 3961 */ 3962 page_ref_add(page, size - 1); 3963 3964 /* reset page count bias and offset to start of new frag */ 3965 nc->pfmemalloc = page_is_pfmemalloc(page); 3966 nc->pagecnt_bias = size; 3967 nc->offset = size; 3968 } 3969 3970 offset = nc->offset - fragsz; 3971 if (unlikely(offset < 0)) { 3972 page = virt_to_page(nc->va); 3973 3974 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 3975 goto refill; 3976 3977 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 3978 /* if size can vary use size else just use PAGE_SIZE */ 3979 size = nc->size; 3980 #endif 3981 /* OK, page count is 0, we can safely set it */ 3982 set_page_count(page, size); 3983 3984 /* reset page count bias and offset to start of new frag */ 3985 nc->pagecnt_bias = size; 3986 offset = size - fragsz; 3987 } 3988 3989 nc->pagecnt_bias--; 3990 nc->offset = offset; 3991 3992 return nc->va + offset; 3993 } 3994 EXPORT_SYMBOL(__alloc_page_frag); 3995 3996 /* 3997 * Frees a page fragment allocated out of either a compound or order 0 page. 3998 */ 3999 void __free_page_frag(void *addr) 4000 { 4001 struct page *page = virt_to_head_page(addr); 4002 4003 if (unlikely(put_page_testzero(page))) 4004 __free_pages_ok(page, compound_order(page)); 4005 } 4006 EXPORT_SYMBOL(__free_page_frag); 4007 4008 static void *make_alloc_exact(unsigned long addr, unsigned int order, 4009 size_t size) 4010 { 4011 if (addr) { 4012 unsigned long alloc_end = addr + (PAGE_SIZE << order); 4013 unsigned long used = addr + PAGE_ALIGN(size); 4014 4015 split_page(virt_to_page((void *)addr), order); 4016 while (used < alloc_end) { 4017 free_page(used); 4018 used += PAGE_SIZE; 4019 } 4020 } 4021 return (void *)addr; 4022 } 4023 4024 /** 4025 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 4026 * @size: the number of bytes to allocate 4027 * @gfp_mask: GFP flags for the allocation 4028 * 4029 * This function is similar to alloc_pages(), except that it allocates the 4030 * minimum number of pages to satisfy the request. alloc_pages() can only 4031 * allocate memory in power-of-two pages. 4032 * 4033 * This function is also limited by MAX_ORDER. 4034 * 4035 * Memory allocated by this function must be released by free_pages_exact(). 4036 */ 4037 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 4038 { 4039 unsigned int order = get_order(size); 4040 unsigned long addr; 4041 4042 addr = __get_free_pages(gfp_mask, order); 4043 return make_alloc_exact(addr, order, size); 4044 } 4045 EXPORT_SYMBOL(alloc_pages_exact); 4046 4047 /** 4048 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 4049 * pages on a node. 4050 * @nid: the preferred node ID where memory should be allocated 4051 * @size: the number of bytes to allocate 4052 * @gfp_mask: GFP flags for the allocation 4053 * 4054 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 4055 * back. 4056 */ 4057 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 4058 { 4059 unsigned int order = get_order(size); 4060 struct page *p = alloc_pages_node(nid, gfp_mask, order); 4061 if (!p) 4062 return NULL; 4063 return make_alloc_exact((unsigned long)page_address(p), order, size); 4064 } 4065 4066 /** 4067 * free_pages_exact - release memory allocated via alloc_pages_exact() 4068 * @virt: the value returned by alloc_pages_exact. 4069 * @size: size of allocation, same value as passed to alloc_pages_exact(). 4070 * 4071 * Release the memory allocated by a previous call to alloc_pages_exact. 4072 */ 4073 void free_pages_exact(void *virt, size_t size) 4074 { 4075 unsigned long addr = (unsigned long)virt; 4076 unsigned long end = addr + PAGE_ALIGN(size); 4077 4078 while (addr < end) { 4079 free_page(addr); 4080 addr += PAGE_SIZE; 4081 } 4082 } 4083 EXPORT_SYMBOL(free_pages_exact); 4084 4085 /** 4086 * nr_free_zone_pages - count number of pages beyond high watermark 4087 * @offset: The zone index of the highest zone 4088 * 4089 * nr_free_zone_pages() counts the number of counts pages which are beyond the 4090 * high watermark within all zones at or below a given zone index. For each 4091 * zone, the number of pages is calculated as: 4092 * managed_pages - high_pages 4093 */ 4094 static unsigned long nr_free_zone_pages(int offset) 4095 { 4096 struct zoneref *z; 4097 struct zone *zone; 4098 4099 /* Just pick one node, since fallback list is circular */ 4100 unsigned long sum = 0; 4101 4102 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 4103 4104 for_each_zone_zonelist(zone, z, zonelist, offset) { 4105 unsigned long size = zone->managed_pages; 4106 unsigned long high = high_wmark_pages(zone); 4107 if (size > high) 4108 sum += size - high; 4109 } 4110 4111 return sum; 4112 } 4113 4114 /** 4115 * nr_free_buffer_pages - count number of pages beyond high watermark 4116 * 4117 * nr_free_buffer_pages() counts the number of pages which are beyond the high 4118 * watermark within ZONE_DMA and ZONE_NORMAL. 4119 */ 4120 unsigned long nr_free_buffer_pages(void) 4121 { 4122 return nr_free_zone_pages(gfp_zone(GFP_USER)); 4123 } 4124 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 4125 4126 /** 4127 * nr_free_pagecache_pages - count number of pages beyond high watermark 4128 * 4129 * nr_free_pagecache_pages() counts the number of pages which are beyond the 4130 * high watermark within all zones. 4131 */ 4132 unsigned long nr_free_pagecache_pages(void) 4133 { 4134 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 4135 } 4136 4137 static inline void show_node(struct zone *zone) 4138 { 4139 if (IS_ENABLED(CONFIG_NUMA)) 4140 printk("Node %d ", zone_to_nid(zone)); 4141 } 4142 4143 long si_mem_available(void) 4144 { 4145 long available; 4146 unsigned long pagecache; 4147 unsigned long wmark_low = 0; 4148 unsigned long pages[NR_LRU_LISTS]; 4149 struct zone *zone; 4150 int lru; 4151 4152 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 4153 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 4154 4155 for_each_zone(zone) 4156 wmark_low += zone->watermark[WMARK_LOW]; 4157 4158 /* 4159 * Estimate the amount of memory available for userspace allocations, 4160 * without causing swapping. 4161 */ 4162 available = global_page_state(NR_FREE_PAGES) - totalreserve_pages; 4163 4164 /* 4165 * Not all the page cache can be freed, otherwise the system will 4166 * start swapping. Assume at least half of the page cache, or the 4167 * low watermark worth of cache, needs to stay. 4168 */ 4169 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 4170 pagecache -= min(pagecache / 2, wmark_low); 4171 available += pagecache; 4172 4173 /* 4174 * Part of the reclaimable slab consists of items that are in use, 4175 * and cannot be freed. Cap this estimate at the low watermark. 4176 */ 4177 available += global_page_state(NR_SLAB_RECLAIMABLE) - 4178 min(global_page_state(NR_SLAB_RECLAIMABLE) / 2, wmark_low); 4179 4180 if (available < 0) 4181 available = 0; 4182 return available; 4183 } 4184 EXPORT_SYMBOL_GPL(si_mem_available); 4185 4186 void si_meminfo(struct sysinfo *val) 4187 { 4188 val->totalram = totalram_pages; 4189 val->sharedram = global_node_page_state(NR_SHMEM); 4190 val->freeram = global_page_state(NR_FREE_PAGES); 4191 val->bufferram = nr_blockdev_pages(); 4192 val->totalhigh = totalhigh_pages; 4193 val->freehigh = nr_free_highpages(); 4194 val->mem_unit = PAGE_SIZE; 4195 } 4196 4197 EXPORT_SYMBOL(si_meminfo); 4198 4199 #ifdef CONFIG_NUMA 4200 void si_meminfo_node(struct sysinfo *val, int nid) 4201 { 4202 int zone_type; /* needs to be signed */ 4203 unsigned long managed_pages = 0; 4204 unsigned long managed_highpages = 0; 4205 unsigned long free_highpages = 0; 4206 pg_data_t *pgdat = NODE_DATA(nid); 4207 4208 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 4209 managed_pages += pgdat->node_zones[zone_type].managed_pages; 4210 val->totalram = managed_pages; 4211 val->sharedram = node_page_state(pgdat, NR_SHMEM); 4212 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 4213 #ifdef CONFIG_HIGHMEM 4214 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 4215 struct zone *zone = &pgdat->node_zones[zone_type]; 4216 4217 if (is_highmem(zone)) { 4218 managed_highpages += zone->managed_pages; 4219 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 4220 } 4221 } 4222 val->totalhigh = managed_highpages; 4223 val->freehigh = free_highpages; 4224 #else 4225 val->totalhigh = managed_highpages; 4226 val->freehigh = free_highpages; 4227 #endif 4228 val->mem_unit = PAGE_SIZE; 4229 } 4230 #endif 4231 4232 /* 4233 * Determine whether the node should be displayed or not, depending on whether 4234 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 4235 */ 4236 bool skip_free_areas_node(unsigned int flags, int nid) 4237 { 4238 bool ret = false; 4239 unsigned int cpuset_mems_cookie; 4240 4241 if (!(flags & SHOW_MEM_FILTER_NODES)) 4242 goto out; 4243 4244 do { 4245 cpuset_mems_cookie = read_mems_allowed_begin(); 4246 ret = !node_isset(nid, cpuset_current_mems_allowed); 4247 } while (read_mems_allowed_retry(cpuset_mems_cookie)); 4248 out: 4249 return ret; 4250 } 4251 4252 #define K(x) ((x) << (PAGE_SHIFT-10)) 4253 4254 static void show_migration_types(unsigned char type) 4255 { 4256 static const char types[MIGRATE_TYPES] = { 4257 [MIGRATE_UNMOVABLE] = 'U', 4258 [MIGRATE_MOVABLE] = 'M', 4259 [MIGRATE_RECLAIMABLE] = 'E', 4260 [MIGRATE_HIGHATOMIC] = 'H', 4261 #ifdef CONFIG_CMA 4262 [MIGRATE_CMA] = 'C', 4263 #endif 4264 #ifdef CONFIG_MEMORY_ISOLATION 4265 [MIGRATE_ISOLATE] = 'I', 4266 #endif 4267 }; 4268 char tmp[MIGRATE_TYPES + 1]; 4269 char *p = tmp; 4270 int i; 4271 4272 for (i = 0; i < MIGRATE_TYPES; i++) { 4273 if (type & (1 << i)) 4274 *p++ = types[i]; 4275 } 4276 4277 *p = '\0'; 4278 printk(KERN_CONT "(%s) ", tmp); 4279 } 4280 4281 /* 4282 * Show free area list (used inside shift_scroll-lock stuff) 4283 * We also calculate the percentage fragmentation. We do this by counting the 4284 * memory on each free list with the exception of the first item on the list. 4285 * 4286 * Bits in @filter: 4287 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 4288 * cpuset. 4289 */ 4290 void show_free_areas(unsigned int filter) 4291 { 4292 unsigned long free_pcp = 0; 4293 int cpu; 4294 struct zone *zone; 4295 pg_data_t *pgdat; 4296 4297 for_each_populated_zone(zone) { 4298 if (skip_free_areas_node(filter, zone_to_nid(zone))) 4299 continue; 4300 4301 for_each_online_cpu(cpu) 4302 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 4303 } 4304 4305 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 4306 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 4307 " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n" 4308 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 4309 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 4310 " free:%lu free_pcp:%lu free_cma:%lu\n", 4311 global_node_page_state(NR_ACTIVE_ANON), 4312 global_node_page_state(NR_INACTIVE_ANON), 4313 global_node_page_state(NR_ISOLATED_ANON), 4314 global_node_page_state(NR_ACTIVE_FILE), 4315 global_node_page_state(NR_INACTIVE_FILE), 4316 global_node_page_state(NR_ISOLATED_FILE), 4317 global_node_page_state(NR_UNEVICTABLE), 4318 global_node_page_state(NR_FILE_DIRTY), 4319 global_node_page_state(NR_WRITEBACK), 4320 global_node_page_state(NR_UNSTABLE_NFS), 4321 global_page_state(NR_SLAB_RECLAIMABLE), 4322 global_page_state(NR_SLAB_UNRECLAIMABLE), 4323 global_node_page_state(NR_FILE_MAPPED), 4324 global_node_page_state(NR_SHMEM), 4325 global_page_state(NR_PAGETABLE), 4326 global_page_state(NR_BOUNCE), 4327 global_page_state(NR_FREE_PAGES), 4328 free_pcp, 4329 global_page_state(NR_FREE_CMA_PAGES)); 4330 4331 for_each_online_pgdat(pgdat) { 4332 printk("Node %d" 4333 " active_anon:%lukB" 4334 " inactive_anon:%lukB" 4335 " active_file:%lukB" 4336 " inactive_file:%lukB" 4337 " unevictable:%lukB" 4338 " isolated(anon):%lukB" 4339 " isolated(file):%lukB" 4340 " mapped:%lukB" 4341 " dirty:%lukB" 4342 " writeback:%lukB" 4343 " shmem:%lukB" 4344 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4345 " shmem_thp: %lukB" 4346 " shmem_pmdmapped: %lukB" 4347 " anon_thp: %lukB" 4348 #endif 4349 " writeback_tmp:%lukB" 4350 " unstable:%lukB" 4351 " pages_scanned:%lu" 4352 " all_unreclaimable? %s" 4353 "\n", 4354 pgdat->node_id, 4355 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 4356 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 4357 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 4358 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 4359 K(node_page_state(pgdat, NR_UNEVICTABLE)), 4360 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 4361 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 4362 K(node_page_state(pgdat, NR_FILE_MAPPED)), 4363 K(node_page_state(pgdat, NR_FILE_DIRTY)), 4364 K(node_page_state(pgdat, NR_WRITEBACK)), 4365 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4366 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), 4367 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) 4368 * HPAGE_PMD_NR), 4369 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR), 4370 #endif 4371 K(node_page_state(pgdat, NR_SHMEM)), 4372 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 4373 K(node_page_state(pgdat, NR_UNSTABLE_NFS)), 4374 node_page_state(pgdat, NR_PAGES_SCANNED), 4375 !pgdat_reclaimable(pgdat) ? "yes" : "no"); 4376 } 4377 4378 for_each_populated_zone(zone) { 4379 int i; 4380 4381 if (skip_free_areas_node(filter, zone_to_nid(zone))) 4382 continue; 4383 4384 free_pcp = 0; 4385 for_each_online_cpu(cpu) 4386 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 4387 4388 show_node(zone); 4389 printk(KERN_CONT 4390 "%s" 4391 " free:%lukB" 4392 " min:%lukB" 4393 " low:%lukB" 4394 " high:%lukB" 4395 " active_anon:%lukB" 4396 " inactive_anon:%lukB" 4397 " active_file:%lukB" 4398 " inactive_file:%lukB" 4399 " unevictable:%lukB" 4400 " writepending:%lukB" 4401 " present:%lukB" 4402 " managed:%lukB" 4403 " mlocked:%lukB" 4404 " slab_reclaimable:%lukB" 4405 " slab_unreclaimable:%lukB" 4406 " kernel_stack:%lukB" 4407 " pagetables:%lukB" 4408 " bounce:%lukB" 4409 " free_pcp:%lukB" 4410 " local_pcp:%ukB" 4411 " free_cma:%lukB" 4412 "\n", 4413 zone->name, 4414 K(zone_page_state(zone, NR_FREE_PAGES)), 4415 K(min_wmark_pages(zone)), 4416 K(low_wmark_pages(zone)), 4417 K(high_wmark_pages(zone)), 4418 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 4419 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 4420 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 4421 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 4422 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 4423 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 4424 K(zone->present_pages), 4425 K(zone->managed_pages), 4426 K(zone_page_state(zone, NR_MLOCK)), 4427 K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)), 4428 K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)), 4429 zone_page_state(zone, NR_KERNEL_STACK_KB), 4430 K(zone_page_state(zone, NR_PAGETABLE)), 4431 K(zone_page_state(zone, NR_BOUNCE)), 4432 K(free_pcp), 4433 K(this_cpu_read(zone->pageset->pcp.count)), 4434 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 4435 printk("lowmem_reserve[]:"); 4436 for (i = 0; i < MAX_NR_ZONES; i++) 4437 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); 4438 printk(KERN_CONT "\n"); 4439 } 4440 4441 for_each_populated_zone(zone) { 4442 unsigned int order; 4443 unsigned long nr[MAX_ORDER], flags, total = 0; 4444 unsigned char types[MAX_ORDER]; 4445 4446 if (skip_free_areas_node(filter, zone_to_nid(zone))) 4447 continue; 4448 show_node(zone); 4449 printk(KERN_CONT "%s: ", zone->name); 4450 4451 spin_lock_irqsave(&zone->lock, flags); 4452 for (order = 0; order < MAX_ORDER; order++) { 4453 struct free_area *area = &zone->free_area[order]; 4454 int type; 4455 4456 nr[order] = area->nr_free; 4457 total += nr[order] << order; 4458 4459 types[order] = 0; 4460 for (type = 0; type < MIGRATE_TYPES; type++) { 4461 if (!list_empty(&area->free_list[type])) 4462 types[order] |= 1 << type; 4463 } 4464 } 4465 spin_unlock_irqrestore(&zone->lock, flags); 4466 for (order = 0; order < MAX_ORDER; order++) { 4467 printk(KERN_CONT "%lu*%lukB ", 4468 nr[order], K(1UL) << order); 4469 if (nr[order]) 4470 show_migration_types(types[order]); 4471 } 4472 printk(KERN_CONT "= %lukB\n", K(total)); 4473 } 4474 4475 hugetlb_show_meminfo(); 4476 4477 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 4478 4479 show_swap_cache_info(); 4480 } 4481 4482 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 4483 { 4484 zoneref->zone = zone; 4485 zoneref->zone_idx = zone_idx(zone); 4486 } 4487 4488 /* 4489 * Builds allocation fallback zone lists. 4490 * 4491 * Add all populated zones of a node to the zonelist. 4492 */ 4493 static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist, 4494 int nr_zones) 4495 { 4496 struct zone *zone; 4497 enum zone_type zone_type = MAX_NR_ZONES; 4498 4499 do { 4500 zone_type--; 4501 zone = pgdat->node_zones + zone_type; 4502 if (managed_zone(zone)) { 4503 zoneref_set_zone(zone, 4504 &zonelist->_zonerefs[nr_zones++]); 4505 check_highest_zone(zone_type); 4506 } 4507 } while (zone_type); 4508 4509 return nr_zones; 4510 } 4511 4512 4513 /* 4514 * zonelist_order: 4515 * 0 = automatic detection of better ordering. 4516 * 1 = order by ([node] distance, -zonetype) 4517 * 2 = order by (-zonetype, [node] distance) 4518 * 4519 * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create 4520 * the same zonelist. So only NUMA can configure this param. 4521 */ 4522 #define ZONELIST_ORDER_DEFAULT 0 4523 #define ZONELIST_ORDER_NODE 1 4524 #define ZONELIST_ORDER_ZONE 2 4525 4526 /* zonelist order in the kernel. 4527 * set_zonelist_order() will set this to NODE or ZONE. 4528 */ 4529 static int current_zonelist_order = ZONELIST_ORDER_DEFAULT; 4530 static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"}; 4531 4532 4533 #ifdef CONFIG_NUMA 4534 /* The value user specified ....changed by config */ 4535 static int user_zonelist_order = ZONELIST_ORDER_DEFAULT; 4536 /* string for sysctl */ 4537 #define NUMA_ZONELIST_ORDER_LEN 16 4538 char numa_zonelist_order[16] = "default"; 4539 4540 /* 4541 * interface for configure zonelist ordering. 4542 * command line option "numa_zonelist_order" 4543 * = "[dD]efault - default, automatic configuration. 4544 * = "[nN]ode - order by node locality, then by zone within node 4545 * = "[zZ]one - order by zone, then by locality within zone 4546 */ 4547 4548 static int __parse_numa_zonelist_order(char *s) 4549 { 4550 if (*s == 'd' || *s == 'D') { 4551 user_zonelist_order = ZONELIST_ORDER_DEFAULT; 4552 } else if (*s == 'n' || *s == 'N') { 4553 user_zonelist_order = ZONELIST_ORDER_NODE; 4554 } else if (*s == 'z' || *s == 'Z') { 4555 user_zonelist_order = ZONELIST_ORDER_ZONE; 4556 } else { 4557 pr_warn("Ignoring invalid numa_zonelist_order value: %s\n", s); 4558 return -EINVAL; 4559 } 4560 return 0; 4561 } 4562 4563 static __init int setup_numa_zonelist_order(char *s) 4564 { 4565 int ret; 4566 4567 if (!s) 4568 return 0; 4569 4570 ret = __parse_numa_zonelist_order(s); 4571 if (ret == 0) 4572 strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN); 4573 4574 return ret; 4575 } 4576 early_param("numa_zonelist_order", setup_numa_zonelist_order); 4577 4578 /* 4579 * sysctl handler for numa_zonelist_order 4580 */ 4581 int numa_zonelist_order_handler(struct ctl_table *table, int write, 4582 void __user *buffer, size_t *length, 4583 loff_t *ppos) 4584 { 4585 char saved_string[NUMA_ZONELIST_ORDER_LEN]; 4586 int ret; 4587 static DEFINE_MUTEX(zl_order_mutex); 4588 4589 mutex_lock(&zl_order_mutex); 4590 if (write) { 4591 if (strlen((char *)table->data) >= NUMA_ZONELIST_ORDER_LEN) { 4592 ret = -EINVAL; 4593 goto out; 4594 } 4595 strcpy(saved_string, (char *)table->data); 4596 } 4597 ret = proc_dostring(table, write, buffer, length, ppos); 4598 if (ret) 4599 goto out; 4600 if (write) { 4601 int oldval = user_zonelist_order; 4602 4603 ret = __parse_numa_zonelist_order((char *)table->data); 4604 if (ret) { 4605 /* 4606 * bogus value. restore saved string 4607 */ 4608 strncpy((char *)table->data, saved_string, 4609 NUMA_ZONELIST_ORDER_LEN); 4610 user_zonelist_order = oldval; 4611 } else if (oldval != user_zonelist_order) { 4612 mutex_lock(&zonelists_mutex); 4613 build_all_zonelists(NULL, NULL); 4614 mutex_unlock(&zonelists_mutex); 4615 } 4616 } 4617 out: 4618 mutex_unlock(&zl_order_mutex); 4619 return ret; 4620 } 4621 4622 4623 #define MAX_NODE_LOAD (nr_online_nodes) 4624 static int node_load[MAX_NUMNODES]; 4625 4626 /** 4627 * find_next_best_node - find the next node that should appear in a given node's fallback list 4628 * @node: node whose fallback list we're appending 4629 * @used_node_mask: nodemask_t of already used nodes 4630 * 4631 * We use a number of factors to determine which is the next node that should 4632 * appear on a given node's fallback list. The node should not have appeared 4633 * already in @node's fallback list, and it should be the next closest node 4634 * according to the distance array (which contains arbitrary distance values 4635 * from each node to each node in the system), and should also prefer nodes 4636 * with no CPUs, since presumably they'll have very little allocation pressure 4637 * on them otherwise. 4638 * It returns -1 if no node is found. 4639 */ 4640 static int find_next_best_node(int node, nodemask_t *used_node_mask) 4641 { 4642 int n, val; 4643 int min_val = INT_MAX; 4644 int best_node = NUMA_NO_NODE; 4645 const struct cpumask *tmp = cpumask_of_node(0); 4646 4647 /* Use the local node if we haven't already */ 4648 if (!node_isset(node, *used_node_mask)) { 4649 node_set(node, *used_node_mask); 4650 return node; 4651 } 4652 4653 for_each_node_state(n, N_MEMORY) { 4654 4655 /* Don't want a node to appear more than once */ 4656 if (node_isset(n, *used_node_mask)) 4657 continue; 4658 4659 /* Use the distance array to find the distance */ 4660 val = node_distance(node, n); 4661 4662 /* Penalize nodes under us ("prefer the next node") */ 4663 val += (n < node); 4664 4665 /* Give preference to headless and unused nodes */ 4666 tmp = cpumask_of_node(n); 4667 if (!cpumask_empty(tmp)) 4668 val += PENALTY_FOR_NODE_WITH_CPUS; 4669 4670 /* Slight preference for less loaded node */ 4671 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 4672 val += node_load[n]; 4673 4674 if (val < min_val) { 4675 min_val = val; 4676 best_node = n; 4677 } 4678 } 4679 4680 if (best_node >= 0) 4681 node_set(best_node, *used_node_mask); 4682 4683 return best_node; 4684 } 4685 4686 4687 /* 4688 * Build zonelists ordered by node and zones within node. 4689 * This results in maximum locality--normal zone overflows into local 4690 * DMA zone, if any--but risks exhausting DMA zone. 4691 */ 4692 static void build_zonelists_in_node_order(pg_data_t *pgdat, int node) 4693 { 4694 int j; 4695 struct zonelist *zonelist; 4696 4697 zonelist = &pgdat->node_zonelists[ZONELIST_FALLBACK]; 4698 for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++) 4699 ; 4700 j = build_zonelists_node(NODE_DATA(node), zonelist, j); 4701 zonelist->_zonerefs[j].zone = NULL; 4702 zonelist->_zonerefs[j].zone_idx = 0; 4703 } 4704 4705 /* 4706 * Build gfp_thisnode zonelists 4707 */ 4708 static void build_thisnode_zonelists(pg_data_t *pgdat) 4709 { 4710 int j; 4711 struct zonelist *zonelist; 4712 4713 zonelist = &pgdat->node_zonelists[ZONELIST_NOFALLBACK]; 4714 j = build_zonelists_node(pgdat, zonelist, 0); 4715 zonelist->_zonerefs[j].zone = NULL; 4716 zonelist->_zonerefs[j].zone_idx = 0; 4717 } 4718 4719 /* 4720 * Build zonelists ordered by zone and nodes within zones. 4721 * This results in conserving DMA zone[s] until all Normal memory is 4722 * exhausted, but results in overflowing to remote node while memory 4723 * may still exist in local DMA zone. 4724 */ 4725 static int node_order[MAX_NUMNODES]; 4726 4727 static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes) 4728 { 4729 int pos, j, node; 4730 int zone_type; /* needs to be signed */ 4731 struct zone *z; 4732 struct zonelist *zonelist; 4733 4734 zonelist = &pgdat->node_zonelists[ZONELIST_FALLBACK]; 4735 pos = 0; 4736 for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) { 4737 for (j = 0; j < nr_nodes; j++) { 4738 node = node_order[j]; 4739 z = &NODE_DATA(node)->node_zones[zone_type]; 4740 if (managed_zone(z)) { 4741 zoneref_set_zone(z, 4742 &zonelist->_zonerefs[pos++]); 4743 check_highest_zone(zone_type); 4744 } 4745 } 4746 } 4747 zonelist->_zonerefs[pos].zone = NULL; 4748 zonelist->_zonerefs[pos].zone_idx = 0; 4749 } 4750 4751 #if defined(CONFIG_64BIT) 4752 /* 4753 * Devices that require DMA32/DMA are relatively rare and do not justify a 4754 * penalty to every machine in case the specialised case applies. Default 4755 * to Node-ordering on 64-bit NUMA machines 4756 */ 4757 static int default_zonelist_order(void) 4758 { 4759 return ZONELIST_ORDER_NODE; 4760 } 4761 #else 4762 /* 4763 * On 32-bit, the Normal zone needs to be preserved for allocations accessible 4764 * by the kernel. If processes running on node 0 deplete the low memory zone 4765 * then reclaim will occur more frequency increasing stalls and potentially 4766 * be easier to OOM if a large percentage of the zone is under writeback or 4767 * dirty. The problem is significantly worse if CONFIG_HIGHPTE is not set. 4768 * Hence, default to zone ordering on 32-bit. 4769 */ 4770 static int default_zonelist_order(void) 4771 { 4772 return ZONELIST_ORDER_ZONE; 4773 } 4774 #endif /* CONFIG_64BIT */ 4775 4776 static void set_zonelist_order(void) 4777 { 4778 if (user_zonelist_order == ZONELIST_ORDER_DEFAULT) 4779 current_zonelist_order = default_zonelist_order(); 4780 else 4781 current_zonelist_order = user_zonelist_order; 4782 } 4783 4784 static void build_zonelists(pg_data_t *pgdat) 4785 { 4786 int i, node, load; 4787 nodemask_t used_mask; 4788 int local_node, prev_node; 4789 struct zonelist *zonelist; 4790 unsigned int order = current_zonelist_order; 4791 4792 /* initialize zonelists */ 4793 for (i = 0; i < MAX_ZONELISTS; i++) { 4794 zonelist = pgdat->node_zonelists + i; 4795 zonelist->_zonerefs[0].zone = NULL; 4796 zonelist->_zonerefs[0].zone_idx = 0; 4797 } 4798 4799 /* NUMA-aware ordering of nodes */ 4800 local_node = pgdat->node_id; 4801 load = nr_online_nodes; 4802 prev_node = local_node; 4803 nodes_clear(used_mask); 4804 4805 memset(node_order, 0, sizeof(node_order)); 4806 i = 0; 4807 4808 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 4809 /* 4810 * We don't want to pressure a particular node. 4811 * So adding penalty to the first node in same 4812 * distance group to make it round-robin. 4813 */ 4814 if (node_distance(local_node, node) != 4815 node_distance(local_node, prev_node)) 4816 node_load[node] = load; 4817 4818 prev_node = node; 4819 load--; 4820 if (order == ZONELIST_ORDER_NODE) 4821 build_zonelists_in_node_order(pgdat, node); 4822 else 4823 node_order[i++] = node; /* remember order */ 4824 } 4825 4826 if (order == ZONELIST_ORDER_ZONE) { 4827 /* calculate node order -- i.e., DMA last! */ 4828 build_zonelists_in_zone_order(pgdat, i); 4829 } 4830 4831 build_thisnode_zonelists(pgdat); 4832 } 4833 4834 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 4835 /* 4836 * Return node id of node used for "local" allocations. 4837 * I.e., first node id of first zone in arg node's generic zonelist. 4838 * Used for initializing percpu 'numa_mem', which is used primarily 4839 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 4840 */ 4841 int local_memory_node(int node) 4842 { 4843 struct zoneref *z; 4844 4845 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 4846 gfp_zone(GFP_KERNEL), 4847 NULL); 4848 return z->zone->node; 4849 } 4850 #endif 4851 4852 static void setup_min_unmapped_ratio(void); 4853 static void setup_min_slab_ratio(void); 4854 #else /* CONFIG_NUMA */ 4855 4856 static void set_zonelist_order(void) 4857 { 4858 current_zonelist_order = ZONELIST_ORDER_ZONE; 4859 } 4860 4861 static void build_zonelists(pg_data_t *pgdat) 4862 { 4863 int node, local_node; 4864 enum zone_type j; 4865 struct zonelist *zonelist; 4866 4867 local_node = pgdat->node_id; 4868 4869 zonelist = &pgdat->node_zonelists[ZONELIST_FALLBACK]; 4870 j = build_zonelists_node(pgdat, zonelist, 0); 4871 4872 /* 4873 * Now we build the zonelist so that it contains the zones 4874 * of all the other nodes. 4875 * We don't want to pressure a particular node, so when 4876 * building the zones for node N, we make sure that the 4877 * zones coming right after the local ones are those from 4878 * node N+1 (modulo N) 4879 */ 4880 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 4881 if (!node_online(node)) 4882 continue; 4883 j = build_zonelists_node(NODE_DATA(node), zonelist, j); 4884 } 4885 for (node = 0; node < local_node; node++) { 4886 if (!node_online(node)) 4887 continue; 4888 j = build_zonelists_node(NODE_DATA(node), zonelist, j); 4889 } 4890 4891 zonelist->_zonerefs[j].zone = NULL; 4892 zonelist->_zonerefs[j].zone_idx = 0; 4893 } 4894 4895 #endif /* CONFIG_NUMA */ 4896 4897 /* 4898 * Boot pageset table. One per cpu which is going to be used for all 4899 * zones and all nodes. The parameters will be set in such a way 4900 * that an item put on a list will immediately be handed over to 4901 * the buddy list. This is safe since pageset manipulation is done 4902 * with interrupts disabled. 4903 * 4904 * The boot_pagesets must be kept even after bootup is complete for 4905 * unused processors and/or zones. They do play a role for bootstrapping 4906 * hotplugged processors. 4907 * 4908 * zoneinfo_show() and maybe other functions do 4909 * not check if the processor is online before following the pageset pointer. 4910 * Other parts of the kernel may not check if the zone is available. 4911 */ 4912 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); 4913 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); 4914 static void setup_zone_pageset(struct zone *zone); 4915 4916 /* 4917 * Global mutex to protect against size modification of zonelists 4918 * as well as to serialize pageset setup for the new populated zone. 4919 */ 4920 DEFINE_MUTEX(zonelists_mutex); 4921 4922 /* return values int ....just for stop_machine() */ 4923 static int __build_all_zonelists(void *data) 4924 { 4925 int nid; 4926 int cpu; 4927 pg_data_t *self = data; 4928 4929 #ifdef CONFIG_NUMA 4930 memset(node_load, 0, sizeof(node_load)); 4931 #endif 4932 4933 if (self && !node_online(self->node_id)) { 4934 build_zonelists(self); 4935 } 4936 4937 for_each_online_node(nid) { 4938 pg_data_t *pgdat = NODE_DATA(nid); 4939 4940 build_zonelists(pgdat); 4941 } 4942 4943 /* 4944 * Initialize the boot_pagesets that are going to be used 4945 * for bootstrapping processors. The real pagesets for 4946 * each zone will be allocated later when the per cpu 4947 * allocator is available. 4948 * 4949 * boot_pagesets are used also for bootstrapping offline 4950 * cpus if the system is already booted because the pagesets 4951 * are needed to initialize allocators on a specific cpu too. 4952 * F.e. the percpu allocator needs the page allocator which 4953 * needs the percpu allocator in order to allocate its pagesets 4954 * (a chicken-egg dilemma). 4955 */ 4956 for_each_possible_cpu(cpu) { 4957 setup_pageset(&per_cpu(boot_pageset, cpu), 0); 4958 4959 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 4960 /* 4961 * We now know the "local memory node" for each node-- 4962 * i.e., the node of the first zone in the generic zonelist. 4963 * Set up numa_mem percpu variable for on-line cpus. During 4964 * boot, only the boot cpu should be on-line; we'll init the 4965 * secondary cpus' numa_mem as they come on-line. During 4966 * node/memory hotplug, we'll fixup all on-line cpus. 4967 */ 4968 if (cpu_online(cpu)) 4969 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 4970 #endif 4971 } 4972 4973 return 0; 4974 } 4975 4976 static noinline void __init 4977 build_all_zonelists_init(void) 4978 { 4979 __build_all_zonelists(NULL); 4980 mminit_verify_zonelist(); 4981 cpuset_init_current_mems_allowed(); 4982 } 4983 4984 /* 4985 * Called with zonelists_mutex held always 4986 * unless system_state == SYSTEM_BOOTING. 4987 * 4988 * __ref due to (1) call of __meminit annotated setup_zone_pageset 4989 * [we're only called with non-NULL zone through __meminit paths] and 4990 * (2) call of __init annotated helper build_all_zonelists_init 4991 * [protected by SYSTEM_BOOTING]. 4992 */ 4993 void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone) 4994 { 4995 set_zonelist_order(); 4996 4997 if (system_state == SYSTEM_BOOTING) { 4998 build_all_zonelists_init(); 4999 } else { 5000 #ifdef CONFIG_MEMORY_HOTPLUG 5001 if (zone) 5002 setup_zone_pageset(zone); 5003 #endif 5004 /* we have to stop all cpus to guarantee there is no user 5005 of zonelist */ 5006 stop_machine(__build_all_zonelists, pgdat, NULL); 5007 /* cpuset refresh routine should be here */ 5008 } 5009 vm_total_pages = nr_free_pagecache_pages(); 5010 /* 5011 * Disable grouping by mobility if the number of pages in the 5012 * system is too low to allow the mechanism to work. It would be 5013 * more accurate, but expensive to check per-zone. This check is 5014 * made on memory-hotadd so a system can start with mobility 5015 * disabled and enable it later 5016 */ 5017 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 5018 page_group_by_mobility_disabled = 1; 5019 else 5020 page_group_by_mobility_disabled = 0; 5021 5022 pr_info("Built %i zonelists in %s order, mobility grouping %s. Total pages: %ld\n", 5023 nr_online_nodes, 5024 zonelist_order_name[current_zonelist_order], 5025 page_group_by_mobility_disabled ? "off" : "on", 5026 vm_total_pages); 5027 #ifdef CONFIG_NUMA 5028 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 5029 #endif 5030 } 5031 5032 /* 5033 * Initially all pages are reserved - free ones are freed 5034 * up by free_all_bootmem() once the early boot process is 5035 * done. Non-atomic initialization, single-pass. 5036 */ 5037 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, 5038 unsigned long start_pfn, enum memmap_context context) 5039 { 5040 struct vmem_altmap *altmap = to_vmem_altmap(__pfn_to_phys(start_pfn)); 5041 unsigned long end_pfn = start_pfn + size; 5042 pg_data_t *pgdat = NODE_DATA(nid); 5043 unsigned long pfn; 5044 unsigned long nr_initialised = 0; 5045 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5046 struct memblock_region *r = NULL, *tmp; 5047 #endif 5048 5049 if (highest_memmap_pfn < end_pfn - 1) 5050 highest_memmap_pfn = end_pfn - 1; 5051 5052 /* 5053 * Honor reservation requested by the driver for this ZONE_DEVICE 5054 * memory 5055 */ 5056 if (altmap && start_pfn == altmap->base_pfn) 5057 start_pfn += altmap->reserve; 5058 5059 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 5060 /* 5061 * There can be holes in boot-time mem_map[]s handed to this 5062 * function. They do not exist on hotplugged memory. 5063 */ 5064 if (context != MEMMAP_EARLY) 5065 goto not_early; 5066 5067 if (!early_pfn_valid(pfn)) 5068 continue; 5069 if (!early_pfn_in_nid(pfn, nid)) 5070 continue; 5071 if (!update_defer_init(pgdat, pfn, end_pfn, &nr_initialised)) 5072 break; 5073 5074 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5075 /* 5076 * Check given memblock attribute by firmware which can affect 5077 * kernel memory layout. If zone==ZONE_MOVABLE but memory is 5078 * mirrored, it's an overlapped memmap init. skip it. 5079 */ 5080 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 5081 if (!r || pfn >= memblock_region_memory_end_pfn(r)) { 5082 for_each_memblock(memory, tmp) 5083 if (pfn < memblock_region_memory_end_pfn(tmp)) 5084 break; 5085 r = tmp; 5086 } 5087 if (pfn >= memblock_region_memory_base_pfn(r) && 5088 memblock_is_mirror(r)) { 5089 /* already initialized as NORMAL */ 5090 pfn = memblock_region_memory_end_pfn(r); 5091 continue; 5092 } 5093 } 5094 #endif 5095 5096 not_early: 5097 /* 5098 * Mark the block movable so that blocks are reserved for 5099 * movable at startup. This will force kernel allocations 5100 * to reserve their blocks rather than leaking throughout 5101 * the address space during boot when many long-lived 5102 * kernel allocations are made. 5103 * 5104 * bitmap is created for zone's valid pfn range. but memmap 5105 * can be created for invalid pages (for alignment) 5106 * check here not to call set_pageblock_migratetype() against 5107 * pfn out of zone. 5108 */ 5109 if (!(pfn & (pageblock_nr_pages - 1))) { 5110 struct page *page = pfn_to_page(pfn); 5111 5112 __init_single_page(page, pfn, zone, nid); 5113 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 5114 } else { 5115 __init_single_pfn(pfn, zone, nid); 5116 } 5117 } 5118 } 5119 5120 static void __meminit zone_init_free_lists(struct zone *zone) 5121 { 5122 unsigned int order, t; 5123 for_each_migratetype_order(order, t) { 5124 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 5125 zone->free_area[order].nr_free = 0; 5126 } 5127 } 5128 5129 #ifndef __HAVE_ARCH_MEMMAP_INIT 5130 #define memmap_init(size, nid, zone, start_pfn) \ 5131 memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY) 5132 #endif 5133 5134 static int zone_batchsize(struct zone *zone) 5135 { 5136 #ifdef CONFIG_MMU 5137 int batch; 5138 5139 /* 5140 * The per-cpu-pages pools are set to around 1000th of the 5141 * size of the zone. But no more than 1/2 of a meg. 5142 * 5143 * OK, so we don't know how big the cache is. So guess. 5144 */ 5145 batch = zone->managed_pages / 1024; 5146 if (batch * PAGE_SIZE > 512 * 1024) 5147 batch = (512 * 1024) / PAGE_SIZE; 5148 batch /= 4; /* We effectively *= 4 below */ 5149 if (batch < 1) 5150 batch = 1; 5151 5152 /* 5153 * Clamp the batch to a 2^n - 1 value. Having a power 5154 * of 2 value was found to be more likely to have 5155 * suboptimal cache aliasing properties in some cases. 5156 * 5157 * For example if 2 tasks are alternately allocating 5158 * batches of pages, one task can end up with a lot 5159 * of pages of one half of the possible page colors 5160 * and the other with pages of the other colors. 5161 */ 5162 batch = rounddown_pow_of_two(batch + batch/2) - 1; 5163 5164 return batch; 5165 5166 #else 5167 /* The deferral and batching of frees should be suppressed under NOMMU 5168 * conditions. 5169 * 5170 * The problem is that NOMMU needs to be able to allocate large chunks 5171 * of contiguous memory as there's no hardware page translation to 5172 * assemble apparent contiguous memory from discontiguous pages. 5173 * 5174 * Queueing large contiguous runs of pages for batching, however, 5175 * causes the pages to actually be freed in smaller chunks. As there 5176 * can be a significant delay between the individual batches being 5177 * recycled, this leads to the once large chunks of space being 5178 * fragmented and becoming unavailable for high-order allocations. 5179 */ 5180 return 0; 5181 #endif 5182 } 5183 5184 /* 5185 * pcp->high and pcp->batch values are related and dependent on one another: 5186 * ->batch must never be higher then ->high. 5187 * The following function updates them in a safe manner without read side 5188 * locking. 5189 * 5190 * Any new users of pcp->batch and pcp->high should ensure they can cope with 5191 * those fields changing asynchronously (acording the the above rule). 5192 * 5193 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 5194 * outside of boot time (or some other assurance that no concurrent updaters 5195 * exist). 5196 */ 5197 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 5198 unsigned long batch) 5199 { 5200 /* start with a fail safe value for batch */ 5201 pcp->batch = 1; 5202 smp_wmb(); 5203 5204 /* Update high, then batch, in order */ 5205 pcp->high = high; 5206 smp_wmb(); 5207 5208 pcp->batch = batch; 5209 } 5210 5211 /* a companion to pageset_set_high() */ 5212 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) 5213 { 5214 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); 5215 } 5216 5217 static void pageset_init(struct per_cpu_pageset *p) 5218 { 5219 struct per_cpu_pages *pcp; 5220 int migratetype; 5221 5222 memset(p, 0, sizeof(*p)); 5223 5224 pcp = &p->pcp; 5225 pcp->count = 0; 5226 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) 5227 INIT_LIST_HEAD(&pcp->lists[migratetype]); 5228 } 5229 5230 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 5231 { 5232 pageset_init(p); 5233 pageset_set_batch(p, batch); 5234 } 5235 5236 /* 5237 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist 5238 * to the value high for the pageset p. 5239 */ 5240 static void pageset_set_high(struct per_cpu_pageset *p, 5241 unsigned long high) 5242 { 5243 unsigned long batch = max(1UL, high / 4); 5244 if ((high / 4) > (PAGE_SHIFT * 8)) 5245 batch = PAGE_SHIFT * 8; 5246 5247 pageset_update(&p->pcp, high, batch); 5248 } 5249 5250 static void pageset_set_high_and_batch(struct zone *zone, 5251 struct per_cpu_pageset *pcp) 5252 { 5253 if (percpu_pagelist_fraction) 5254 pageset_set_high(pcp, 5255 (zone->managed_pages / 5256 percpu_pagelist_fraction)); 5257 else 5258 pageset_set_batch(pcp, zone_batchsize(zone)); 5259 } 5260 5261 static void __meminit zone_pageset_init(struct zone *zone, int cpu) 5262 { 5263 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); 5264 5265 pageset_init(pcp); 5266 pageset_set_high_and_batch(zone, pcp); 5267 } 5268 5269 static void __meminit setup_zone_pageset(struct zone *zone) 5270 { 5271 int cpu; 5272 zone->pageset = alloc_percpu(struct per_cpu_pageset); 5273 for_each_possible_cpu(cpu) 5274 zone_pageset_init(zone, cpu); 5275 } 5276 5277 /* 5278 * Allocate per cpu pagesets and initialize them. 5279 * Before this call only boot pagesets were available. 5280 */ 5281 void __init setup_per_cpu_pageset(void) 5282 { 5283 struct pglist_data *pgdat; 5284 struct zone *zone; 5285 5286 for_each_populated_zone(zone) 5287 setup_zone_pageset(zone); 5288 5289 for_each_online_pgdat(pgdat) 5290 pgdat->per_cpu_nodestats = 5291 alloc_percpu(struct per_cpu_nodestat); 5292 } 5293 5294 static __meminit void zone_pcp_init(struct zone *zone) 5295 { 5296 /* 5297 * per cpu subsystem is not up at this point. The following code 5298 * relies on the ability of the linker to provide the 5299 * offset of a (static) per cpu variable into the per cpu area. 5300 */ 5301 zone->pageset = &boot_pageset; 5302 5303 if (populated_zone(zone)) 5304 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", 5305 zone->name, zone->present_pages, 5306 zone_batchsize(zone)); 5307 } 5308 5309 int __meminit init_currently_empty_zone(struct zone *zone, 5310 unsigned long zone_start_pfn, 5311 unsigned long size) 5312 { 5313 struct pglist_data *pgdat = zone->zone_pgdat; 5314 5315 pgdat->nr_zones = zone_idx(zone) + 1; 5316 5317 zone->zone_start_pfn = zone_start_pfn; 5318 5319 mminit_dprintk(MMINIT_TRACE, "memmap_init", 5320 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 5321 pgdat->node_id, 5322 (unsigned long)zone_idx(zone), 5323 zone_start_pfn, (zone_start_pfn + size)); 5324 5325 zone_init_free_lists(zone); 5326 zone->initialized = 1; 5327 5328 return 0; 5329 } 5330 5331 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5332 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID 5333 5334 /* 5335 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 5336 */ 5337 int __meminit __early_pfn_to_nid(unsigned long pfn, 5338 struct mminit_pfnnid_cache *state) 5339 { 5340 unsigned long start_pfn, end_pfn; 5341 int nid; 5342 5343 if (state->last_start <= pfn && pfn < state->last_end) 5344 return state->last_nid; 5345 5346 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 5347 if (nid != -1) { 5348 state->last_start = start_pfn; 5349 state->last_end = end_pfn; 5350 state->last_nid = nid; 5351 } 5352 5353 return nid; 5354 } 5355 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ 5356 5357 /** 5358 * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range 5359 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed. 5360 * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid 5361 * 5362 * If an architecture guarantees that all ranges registered contain no holes 5363 * and may be freed, this this function may be used instead of calling 5364 * memblock_free_early_nid() manually. 5365 */ 5366 void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn) 5367 { 5368 unsigned long start_pfn, end_pfn; 5369 int i, this_nid; 5370 5371 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) { 5372 start_pfn = min(start_pfn, max_low_pfn); 5373 end_pfn = min(end_pfn, max_low_pfn); 5374 5375 if (start_pfn < end_pfn) 5376 memblock_free_early_nid(PFN_PHYS(start_pfn), 5377 (end_pfn - start_pfn) << PAGE_SHIFT, 5378 this_nid); 5379 } 5380 } 5381 5382 /** 5383 * sparse_memory_present_with_active_regions - Call memory_present for each active range 5384 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used. 5385 * 5386 * If an architecture guarantees that all ranges registered contain no holes and may 5387 * be freed, this function may be used instead of calling memory_present() manually. 5388 */ 5389 void __init sparse_memory_present_with_active_regions(int nid) 5390 { 5391 unsigned long start_pfn, end_pfn; 5392 int i, this_nid; 5393 5394 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) 5395 memory_present(this_nid, start_pfn, end_pfn); 5396 } 5397 5398 /** 5399 * get_pfn_range_for_nid - Return the start and end page frames for a node 5400 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 5401 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 5402 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 5403 * 5404 * It returns the start and end page frame of a node based on information 5405 * provided by memblock_set_node(). If called for a node 5406 * with no available memory, a warning is printed and the start and end 5407 * PFNs will be 0. 5408 */ 5409 void __meminit get_pfn_range_for_nid(unsigned int nid, 5410 unsigned long *start_pfn, unsigned long *end_pfn) 5411 { 5412 unsigned long this_start_pfn, this_end_pfn; 5413 int i; 5414 5415 *start_pfn = -1UL; 5416 *end_pfn = 0; 5417 5418 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 5419 *start_pfn = min(*start_pfn, this_start_pfn); 5420 *end_pfn = max(*end_pfn, this_end_pfn); 5421 } 5422 5423 if (*start_pfn == -1UL) 5424 *start_pfn = 0; 5425 } 5426 5427 /* 5428 * This finds a zone that can be used for ZONE_MOVABLE pages. The 5429 * assumption is made that zones within a node are ordered in monotonic 5430 * increasing memory addresses so that the "highest" populated zone is used 5431 */ 5432 static void __init find_usable_zone_for_movable(void) 5433 { 5434 int zone_index; 5435 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 5436 if (zone_index == ZONE_MOVABLE) 5437 continue; 5438 5439 if (arch_zone_highest_possible_pfn[zone_index] > 5440 arch_zone_lowest_possible_pfn[zone_index]) 5441 break; 5442 } 5443 5444 VM_BUG_ON(zone_index == -1); 5445 movable_zone = zone_index; 5446 } 5447 5448 /* 5449 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 5450 * because it is sized independent of architecture. Unlike the other zones, 5451 * the starting point for ZONE_MOVABLE is not fixed. It may be different 5452 * in each node depending on the size of each node and how evenly kernelcore 5453 * is distributed. This helper function adjusts the zone ranges 5454 * provided by the architecture for a given node by using the end of the 5455 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 5456 * zones within a node are in order of monotonic increases memory addresses 5457 */ 5458 static void __meminit adjust_zone_range_for_zone_movable(int nid, 5459 unsigned long zone_type, 5460 unsigned long node_start_pfn, 5461 unsigned long node_end_pfn, 5462 unsigned long *zone_start_pfn, 5463 unsigned long *zone_end_pfn) 5464 { 5465 /* Only adjust if ZONE_MOVABLE is on this node */ 5466 if (zone_movable_pfn[nid]) { 5467 /* Size ZONE_MOVABLE */ 5468 if (zone_type == ZONE_MOVABLE) { 5469 *zone_start_pfn = zone_movable_pfn[nid]; 5470 *zone_end_pfn = min(node_end_pfn, 5471 arch_zone_highest_possible_pfn[movable_zone]); 5472 5473 /* Adjust for ZONE_MOVABLE starting within this range */ 5474 } else if (!mirrored_kernelcore && 5475 *zone_start_pfn < zone_movable_pfn[nid] && 5476 *zone_end_pfn > zone_movable_pfn[nid]) { 5477 *zone_end_pfn = zone_movable_pfn[nid]; 5478 5479 /* Check if this whole range is within ZONE_MOVABLE */ 5480 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 5481 *zone_start_pfn = *zone_end_pfn; 5482 } 5483 } 5484 5485 /* 5486 * Return the number of pages a zone spans in a node, including holes 5487 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 5488 */ 5489 static unsigned long __meminit zone_spanned_pages_in_node(int nid, 5490 unsigned long zone_type, 5491 unsigned long node_start_pfn, 5492 unsigned long node_end_pfn, 5493 unsigned long *zone_start_pfn, 5494 unsigned long *zone_end_pfn, 5495 unsigned long *ignored) 5496 { 5497 /* When hotadd a new node from cpu_up(), the node should be empty */ 5498 if (!node_start_pfn && !node_end_pfn) 5499 return 0; 5500 5501 /* Get the start and end of the zone */ 5502 *zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type]; 5503 *zone_end_pfn = arch_zone_highest_possible_pfn[zone_type]; 5504 adjust_zone_range_for_zone_movable(nid, zone_type, 5505 node_start_pfn, node_end_pfn, 5506 zone_start_pfn, zone_end_pfn); 5507 5508 /* Check that this node has pages within the zone's required range */ 5509 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 5510 return 0; 5511 5512 /* Move the zone boundaries inside the node if necessary */ 5513 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 5514 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 5515 5516 /* Return the spanned pages */ 5517 return *zone_end_pfn - *zone_start_pfn; 5518 } 5519 5520 /* 5521 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 5522 * then all holes in the requested range will be accounted for. 5523 */ 5524 unsigned long __meminit __absent_pages_in_range(int nid, 5525 unsigned long range_start_pfn, 5526 unsigned long range_end_pfn) 5527 { 5528 unsigned long nr_absent = range_end_pfn - range_start_pfn; 5529 unsigned long start_pfn, end_pfn; 5530 int i; 5531 5532 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 5533 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 5534 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 5535 nr_absent -= end_pfn - start_pfn; 5536 } 5537 return nr_absent; 5538 } 5539 5540 /** 5541 * absent_pages_in_range - Return number of page frames in holes within a range 5542 * @start_pfn: The start PFN to start searching for holes 5543 * @end_pfn: The end PFN to stop searching for holes 5544 * 5545 * It returns the number of pages frames in memory holes within a range. 5546 */ 5547 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 5548 unsigned long end_pfn) 5549 { 5550 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 5551 } 5552 5553 /* Return the number of page frames in holes in a zone on a node */ 5554 static unsigned long __meminit zone_absent_pages_in_node(int nid, 5555 unsigned long zone_type, 5556 unsigned long node_start_pfn, 5557 unsigned long node_end_pfn, 5558 unsigned long *ignored) 5559 { 5560 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 5561 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 5562 unsigned long zone_start_pfn, zone_end_pfn; 5563 unsigned long nr_absent; 5564 5565 /* When hotadd a new node from cpu_up(), the node should be empty */ 5566 if (!node_start_pfn && !node_end_pfn) 5567 return 0; 5568 5569 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 5570 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 5571 5572 adjust_zone_range_for_zone_movable(nid, zone_type, 5573 node_start_pfn, node_end_pfn, 5574 &zone_start_pfn, &zone_end_pfn); 5575 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 5576 5577 /* 5578 * ZONE_MOVABLE handling. 5579 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 5580 * and vice versa. 5581 */ 5582 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 5583 unsigned long start_pfn, end_pfn; 5584 struct memblock_region *r; 5585 5586 for_each_memblock(memory, r) { 5587 start_pfn = clamp(memblock_region_memory_base_pfn(r), 5588 zone_start_pfn, zone_end_pfn); 5589 end_pfn = clamp(memblock_region_memory_end_pfn(r), 5590 zone_start_pfn, zone_end_pfn); 5591 5592 if (zone_type == ZONE_MOVABLE && 5593 memblock_is_mirror(r)) 5594 nr_absent += end_pfn - start_pfn; 5595 5596 if (zone_type == ZONE_NORMAL && 5597 !memblock_is_mirror(r)) 5598 nr_absent += end_pfn - start_pfn; 5599 } 5600 } 5601 5602 return nr_absent; 5603 } 5604 5605 #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 5606 static inline unsigned long __meminit zone_spanned_pages_in_node(int nid, 5607 unsigned long zone_type, 5608 unsigned long node_start_pfn, 5609 unsigned long node_end_pfn, 5610 unsigned long *zone_start_pfn, 5611 unsigned long *zone_end_pfn, 5612 unsigned long *zones_size) 5613 { 5614 unsigned int zone; 5615 5616 *zone_start_pfn = node_start_pfn; 5617 for (zone = 0; zone < zone_type; zone++) 5618 *zone_start_pfn += zones_size[zone]; 5619 5620 *zone_end_pfn = *zone_start_pfn + zones_size[zone_type]; 5621 5622 return zones_size[zone_type]; 5623 } 5624 5625 static inline unsigned long __meminit zone_absent_pages_in_node(int nid, 5626 unsigned long zone_type, 5627 unsigned long node_start_pfn, 5628 unsigned long node_end_pfn, 5629 unsigned long *zholes_size) 5630 { 5631 if (!zholes_size) 5632 return 0; 5633 5634 return zholes_size[zone_type]; 5635 } 5636 5637 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 5638 5639 static void __meminit calculate_node_totalpages(struct pglist_data *pgdat, 5640 unsigned long node_start_pfn, 5641 unsigned long node_end_pfn, 5642 unsigned long *zones_size, 5643 unsigned long *zholes_size) 5644 { 5645 unsigned long realtotalpages = 0, totalpages = 0; 5646 enum zone_type i; 5647 5648 for (i = 0; i < MAX_NR_ZONES; i++) { 5649 struct zone *zone = pgdat->node_zones + i; 5650 unsigned long zone_start_pfn, zone_end_pfn; 5651 unsigned long size, real_size; 5652 5653 size = zone_spanned_pages_in_node(pgdat->node_id, i, 5654 node_start_pfn, 5655 node_end_pfn, 5656 &zone_start_pfn, 5657 &zone_end_pfn, 5658 zones_size); 5659 real_size = size - zone_absent_pages_in_node(pgdat->node_id, i, 5660 node_start_pfn, node_end_pfn, 5661 zholes_size); 5662 if (size) 5663 zone->zone_start_pfn = zone_start_pfn; 5664 else 5665 zone->zone_start_pfn = 0; 5666 zone->spanned_pages = size; 5667 zone->present_pages = real_size; 5668 5669 totalpages += size; 5670 realtotalpages += real_size; 5671 } 5672 5673 pgdat->node_spanned_pages = totalpages; 5674 pgdat->node_present_pages = realtotalpages; 5675 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, 5676 realtotalpages); 5677 } 5678 5679 #ifndef CONFIG_SPARSEMEM 5680 /* 5681 * Calculate the size of the zone->blockflags rounded to an unsigned long 5682 * Start by making sure zonesize is a multiple of pageblock_order by rounding 5683 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 5684 * round what is now in bits to nearest long in bits, then return it in 5685 * bytes. 5686 */ 5687 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 5688 { 5689 unsigned long usemapsize; 5690 5691 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 5692 usemapsize = roundup(zonesize, pageblock_nr_pages); 5693 usemapsize = usemapsize >> pageblock_order; 5694 usemapsize *= NR_PAGEBLOCK_BITS; 5695 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 5696 5697 return usemapsize / 8; 5698 } 5699 5700 static void __init setup_usemap(struct pglist_data *pgdat, 5701 struct zone *zone, 5702 unsigned long zone_start_pfn, 5703 unsigned long zonesize) 5704 { 5705 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); 5706 zone->pageblock_flags = NULL; 5707 if (usemapsize) 5708 zone->pageblock_flags = 5709 memblock_virt_alloc_node_nopanic(usemapsize, 5710 pgdat->node_id); 5711 } 5712 #else 5713 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, 5714 unsigned long zone_start_pfn, unsigned long zonesize) {} 5715 #endif /* CONFIG_SPARSEMEM */ 5716 5717 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 5718 5719 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 5720 void __paginginit set_pageblock_order(void) 5721 { 5722 unsigned int order; 5723 5724 /* Check that pageblock_nr_pages has not already been setup */ 5725 if (pageblock_order) 5726 return; 5727 5728 if (HPAGE_SHIFT > PAGE_SHIFT) 5729 order = HUGETLB_PAGE_ORDER; 5730 else 5731 order = MAX_ORDER - 1; 5732 5733 /* 5734 * Assume the largest contiguous order of interest is a huge page. 5735 * This value may be variable depending on boot parameters on IA64 and 5736 * powerpc. 5737 */ 5738 pageblock_order = order; 5739 } 5740 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 5741 5742 /* 5743 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 5744 * is unused as pageblock_order is set at compile-time. See 5745 * include/linux/pageblock-flags.h for the values of pageblock_order based on 5746 * the kernel config 5747 */ 5748 void __paginginit set_pageblock_order(void) 5749 { 5750 } 5751 5752 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 5753 5754 static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages, 5755 unsigned long present_pages) 5756 { 5757 unsigned long pages = spanned_pages; 5758 5759 /* 5760 * Provide a more accurate estimation if there are holes within 5761 * the zone and SPARSEMEM is in use. If there are holes within the 5762 * zone, each populated memory region may cost us one or two extra 5763 * memmap pages due to alignment because memmap pages for each 5764 * populated regions may not naturally algined on page boundary. 5765 * So the (present_pages >> 4) heuristic is a tradeoff for that. 5766 */ 5767 if (spanned_pages > present_pages + (present_pages >> 4) && 5768 IS_ENABLED(CONFIG_SPARSEMEM)) 5769 pages = present_pages; 5770 5771 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 5772 } 5773 5774 /* 5775 * Set up the zone data structures: 5776 * - mark all pages reserved 5777 * - mark all memory queues empty 5778 * - clear the memory bitmaps 5779 * 5780 * NOTE: pgdat should get zeroed by caller. 5781 */ 5782 static void __paginginit free_area_init_core(struct pglist_data *pgdat) 5783 { 5784 enum zone_type j; 5785 int nid = pgdat->node_id; 5786 int ret; 5787 5788 pgdat_resize_init(pgdat); 5789 #ifdef CONFIG_NUMA_BALANCING 5790 spin_lock_init(&pgdat->numabalancing_migrate_lock); 5791 pgdat->numabalancing_migrate_nr_pages = 0; 5792 pgdat->numabalancing_migrate_next_window = jiffies; 5793 #endif 5794 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5795 spin_lock_init(&pgdat->split_queue_lock); 5796 INIT_LIST_HEAD(&pgdat->split_queue); 5797 pgdat->split_queue_len = 0; 5798 #endif 5799 init_waitqueue_head(&pgdat->kswapd_wait); 5800 init_waitqueue_head(&pgdat->pfmemalloc_wait); 5801 #ifdef CONFIG_COMPACTION 5802 init_waitqueue_head(&pgdat->kcompactd_wait); 5803 #endif 5804 pgdat_page_ext_init(pgdat); 5805 spin_lock_init(&pgdat->lru_lock); 5806 lruvec_init(node_lruvec(pgdat)); 5807 5808 for (j = 0; j < MAX_NR_ZONES; j++) { 5809 struct zone *zone = pgdat->node_zones + j; 5810 unsigned long size, realsize, freesize, memmap_pages; 5811 unsigned long zone_start_pfn = zone->zone_start_pfn; 5812 5813 size = zone->spanned_pages; 5814 realsize = freesize = zone->present_pages; 5815 5816 /* 5817 * Adjust freesize so that it accounts for how much memory 5818 * is used by this zone for memmap. This affects the watermark 5819 * and per-cpu initialisations 5820 */ 5821 memmap_pages = calc_memmap_size(size, realsize); 5822 if (!is_highmem_idx(j)) { 5823 if (freesize >= memmap_pages) { 5824 freesize -= memmap_pages; 5825 if (memmap_pages) 5826 printk(KERN_DEBUG 5827 " %s zone: %lu pages used for memmap\n", 5828 zone_names[j], memmap_pages); 5829 } else 5830 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n", 5831 zone_names[j], memmap_pages, freesize); 5832 } 5833 5834 /* Account for reserved pages */ 5835 if (j == 0 && freesize > dma_reserve) { 5836 freesize -= dma_reserve; 5837 printk(KERN_DEBUG " %s zone: %lu pages reserved\n", 5838 zone_names[0], dma_reserve); 5839 } 5840 5841 if (!is_highmem_idx(j)) 5842 nr_kernel_pages += freesize; 5843 /* Charge for highmem memmap if there are enough kernel pages */ 5844 else if (nr_kernel_pages > memmap_pages * 2) 5845 nr_kernel_pages -= memmap_pages; 5846 nr_all_pages += freesize; 5847 5848 /* 5849 * Set an approximate value for lowmem here, it will be adjusted 5850 * when the bootmem allocator frees pages into the buddy system. 5851 * And all highmem pages will be managed by the buddy system. 5852 */ 5853 zone->managed_pages = is_highmem_idx(j) ? realsize : freesize; 5854 #ifdef CONFIG_NUMA 5855 zone->node = nid; 5856 #endif 5857 zone->name = zone_names[j]; 5858 zone->zone_pgdat = pgdat; 5859 spin_lock_init(&zone->lock); 5860 zone_seqlock_init(zone); 5861 zone_pcp_init(zone); 5862 5863 if (!size) 5864 continue; 5865 5866 set_pageblock_order(); 5867 setup_usemap(pgdat, zone, zone_start_pfn, size); 5868 ret = init_currently_empty_zone(zone, zone_start_pfn, size); 5869 BUG_ON(ret); 5870 memmap_init(size, nid, j, zone_start_pfn); 5871 } 5872 } 5873 5874 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) 5875 { 5876 unsigned long __maybe_unused start = 0; 5877 unsigned long __maybe_unused offset = 0; 5878 5879 /* Skip empty nodes */ 5880 if (!pgdat->node_spanned_pages) 5881 return; 5882 5883 #ifdef CONFIG_FLAT_NODE_MEM_MAP 5884 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 5885 offset = pgdat->node_start_pfn - start; 5886 /* ia64 gets its own node_mem_map, before this, without bootmem */ 5887 if (!pgdat->node_mem_map) { 5888 unsigned long size, end; 5889 struct page *map; 5890 5891 /* 5892 * The zone's endpoints aren't required to be MAX_ORDER 5893 * aligned but the node_mem_map endpoints must be in order 5894 * for the buddy allocator to function correctly. 5895 */ 5896 end = pgdat_end_pfn(pgdat); 5897 end = ALIGN(end, MAX_ORDER_NR_PAGES); 5898 size = (end - start) * sizeof(struct page); 5899 map = alloc_remap(pgdat->node_id, size); 5900 if (!map) 5901 map = memblock_virt_alloc_node_nopanic(size, 5902 pgdat->node_id); 5903 pgdat->node_mem_map = map + offset; 5904 } 5905 #ifndef CONFIG_NEED_MULTIPLE_NODES 5906 /* 5907 * With no DISCONTIG, the global mem_map is just set as node 0's 5908 */ 5909 if (pgdat == NODE_DATA(0)) { 5910 mem_map = NODE_DATA(0)->node_mem_map; 5911 #if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM) 5912 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 5913 mem_map -= offset; 5914 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 5915 } 5916 #endif 5917 #endif /* CONFIG_FLAT_NODE_MEM_MAP */ 5918 } 5919 5920 void __paginginit free_area_init_node(int nid, unsigned long *zones_size, 5921 unsigned long node_start_pfn, unsigned long *zholes_size) 5922 { 5923 pg_data_t *pgdat = NODE_DATA(nid); 5924 unsigned long start_pfn = 0; 5925 unsigned long end_pfn = 0; 5926 5927 /* pg_data_t should be reset to zero when it's allocated */ 5928 WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx); 5929 5930 reset_deferred_meminit(pgdat); 5931 pgdat->node_id = nid; 5932 pgdat->node_start_pfn = node_start_pfn; 5933 pgdat->per_cpu_nodestats = NULL; 5934 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5935 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 5936 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 5937 (u64)start_pfn << PAGE_SHIFT, 5938 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 5939 #else 5940 start_pfn = node_start_pfn; 5941 #endif 5942 calculate_node_totalpages(pgdat, start_pfn, end_pfn, 5943 zones_size, zholes_size); 5944 5945 alloc_node_mem_map(pgdat); 5946 #ifdef CONFIG_FLAT_NODE_MEM_MAP 5947 printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n", 5948 nid, (unsigned long)pgdat, 5949 (unsigned long)pgdat->node_mem_map); 5950 #endif 5951 5952 free_area_init_core(pgdat); 5953 } 5954 5955 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5956 5957 #if MAX_NUMNODES > 1 5958 /* 5959 * Figure out the number of possible node ids. 5960 */ 5961 void __init setup_nr_node_ids(void) 5962 { 5963 unsigned int highest; 5964 5965 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 5966 nr_node_ids = highest + 1; 5967 } 5968 #endif 5969 5970 /** 5971 * node_map_pfn_alignment - determine the maximum internode alignment 5972 * 5973 * This function should be called after node map is populated and sorted. 5974 * It calculates the maximum power of two alignment which can distinguish 5975 * all the nodes. 5976 * 5977 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 5978 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 5979 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 5980 * shifted, 1GiB is enough and this function will indicate so. 5981 * 5982 * This is used to test whether pfn -> nid mapping of the chosen memory 5983 * model has fine enough granularity to avoid incorrect mapping for the 5984 * populated node map. 5985 * 5986 * Returns the determined alignment in pfn's. 0 if there is no alignment 5987 * requirement (single node). 5988 */ 5989 unsigned long __init node_map_pfn_alignment(void) 5990 { 5991 unsigned long accl_mask = 0, last_end = 0; 5992 unsigned long start, end, mask; 5993 int last_nid = -1; 5994 int i, nid; 5995 5996 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 5997 if (!start || last_nid < 0 || last_nid == nid) { 5998 last_nid = nid; 5999 last_end = end; 6000 continue; 6001 } 6002 6003 /* 6004 * Start with a mask granular enough to pin-point to the 6005 * start pfn and tick off bits one-by-one until it becomes 6006 * too coarse to separate the current node from the last. 6007 */ 6008 mask = ~((1 << __ffs(start)) - 1); 6009 while (mask && last_end <= (start & (mask << 1))) 6010 mask <<= 1; 6011 6012 /* accumulate all internode masks */ 6013 accl_mask |= mask; 6014 } 6015 6016 /* convert mask to number of pages */ 6017 return ~accl_mask + 1; 6018 } 6019 6020 /* Find the lowest pfn for a node */ 6021 static unsigned long __init find_min_pfn_for_node(int nid) 6022 { 6023 unsigned long min_pfn = ULONG_MAX; 6024 unsigned long start_pfn; 6025 int i; 6026 6027 for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL) 6028 min_pfn = min(min_pfn, start_pfn); 6029 6030 if (min_pfn == ULONG_MAX) { 6031 pr_warn("Could not find start_pfn for node %d\n", nid); 6032 return 0; 6033 } 6034 6035 return min_pfn; 6036 } 6037 6038 /** 6039 * find_min_pfn_with_active_regions - Find the minimum PFN registered 6040 * 6041 * It returns the minimum PFN based on information provided via 6042 * memblock_set_node(). 6043 */ 6044 unsigned long __init find_min_pfn_with_active_regions(void) 6045 { 6046 return find_min_pfn_for_node(MAX_NUMNODES); 6047 } 6048 6049 /* 6050 * early_calculate_totalpages() 6051 * Sum pages in active regions for movable zone. 6052 * Populate N_MEMORY for calculating usable_nodes. 6053 */ 6054 static unsigned long __init early_calculate_totalpages(void) 6055 { 6056 unsigned long totalpages = 0; 6057 unsigned long start_pfn, end_pfn; 6058 int i, nid; 6059 6060 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 6061 unsigned long pages = end_pfn - start_pfn; 6062 6063 totalpages += pages; 6064 if (pages) 6065 node_set_state(nid, N_MEMORY); 6066 } 6067 return totalpages; 6068 } 6069 6070 /* 6071 * Find the PFN the Movable zone begins in each node. Kernel memory 6072 * is spread evenly between nodes as long as the nodes have enough 6073 * memory. When they don't, some nodes will have more kernelcore than 6074 * others 6075 */ 6076 static void __init find_zone_movable_pfns_for_nodes(void) 6077 { 6078 int i, nid; 6079 unsigned long usable_startpfn; 6080 unsigned long kernelcore_node, kernelcore_remaining; 6081 /* save the state before borrow the nodemask */ 6082 nodemask_t saved_node_state = node_states[N_MEMORY]; 6083 unsigned long totalpages = early_calculate_totalpages(); 6084 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 6085 struct memblock_region *r; 6086 6087 /* Need to find movable_zone earlier when movable_node is specified. */ 6088 find_usable_zone_for_movable(); 6089 6090 /* 6091 * If movable_node is specified, ignore kernelcore and movablecore 6092 * options. 6093 */ 6094 if (movable_node_is_enabled()) { 6095 for_each_memblock(memory, r) { 6096 if (!memblock_is_hotpluggable(r)) 6097 continue; 6098 6099 nid = r->nid; 6100 6101 usable_startpfn = PFN_DOWN(r->base); 6102 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 6103 min(usable_startpfn, zone_movable_pfn[nid]) : 6104 usable_startpfn; 6105 } 6106 6107 goto out2; 6108 } 6109 6110 /* 6111 * If kernelcore=mirror is specified, ignore movablecore option 6112 */ 6113 if (mirrored_kernelcore) { 6114 bool mem_below_4gb_not_mirrored = false; 6115 6116 for_each_memblock(memory, r) { 6117 if (memblock_is_mirror(r)) 6118 continue; 6119 6120 nid = r->nid; 6121 6122 usable_startpfn = memblock_region_memory_base_pfn(r); 6123 6124 if (usable_startpfn < 0x100000) { 6125 mem_below_4gb_not_mirrored = true; 6126 continue; 6127 } 6128 6129 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 6130 min(usable_startpfn, zone_movable_pfn[nid]) : 6131 usable_startpfn; 6132 } 6133 6134 if (mem_below_4gb_not_mirrored) 6135 pr_warn("This configuration results in unmirrored kernel memory."); 6136 6137 goto out2; 6138 } 6139 6140 /* 6141 * If movablecore=nn[KMG] was specified, calculate what size of 6142 * kernelcore that corresponds so that memory usable for 6143 * any allocation type is evenly spread. If both kernelcore 6144 * and movablecore are specified, then the value of kernelcore 6145 * will be used for required_kernelcore if it's greater than 6146 * what movablecore would have allowed. 6147 */ 6148 if (required_movablecore) { 6149 unsigned long corepages; 6150 6151 /* 6152 * Round-up so that ZONE_MOVABLE is at least as large as what 6153 * was requested by the user 6154 */ 6155 required_movablecore = 6156 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 6157 required_movablecore = min(totalpages, required_movablecore); 6158 corepages = totalpages - required_movablecore; 6159 6160 required_kernelcore = max(required_kernelcore, corepages); 6161 } 6162 6163 /* 6164 * If kernelcore was not specified or kernelcore size is larger 6165 * than totalpages, there is no ZONE_MOVABLE. 6166 */ 6167 if (!required_kernelcore || required_kernelcore >= totalpages) 6168 goto out; 6169 6170 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 6171 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 6172 6173 restart: 6174 /* Spread kernelcore memory as evenly as possible throughout nodes */ 6175 kernelcore_node = required_kernelcore / usable_nodes; 6176 for_each_node_state(nid, N_MEMORY) { 6177 unsigned long start_pfn, end_pfn; 6178 6179 /* 6180 * Recalculate kernelcore_node if the division per node 6181 * now exceeds what is necessary to satisfy the requested 6182 * amount of memory for the kernel 6183 */ 6184 if (required_kernelcore < kernelcore_node) 6185 kernelcore_node = required_kernelcore / usable_nodes; 6186 6187 /* 6188 * As the map is walked, we track how much memory is usable 6189 * by the kernel using kernelcore_remaining. When it is 6190 * 0, the rest of the node is usable by ZONE_MOVABLE 6191 */ 6192 kernelcore_remaining = kernelcore_node; 6193 6194 /* Go through each range of PFNs within this node */ 6195 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6196 unsigned long size_pages; 6197 6198 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 6199 if (start_pfn >= end_pfn) 6200 continue; 6201 6202 /* Account for what is only usable for kernelcore */ 6203 if (start_pfn < usable_startpfn) { 6204 unsigned long kernel_pages; 6205 kernel_pages = min(end_pfn, usable_startpfn) 6206 - start_pfn; 6207 6208 kernelcore_remaining -= min(kernel_pages, 6209 kernelcore_remaining); 6210 required_kernelcore -= min(kernel_pages, 6211 required_kernelcore); 6212 6213 /* Continue if range is now fully accounted */ 6214 if (end_pfn <= usable_startpfn) { 6215 6216 /* 6217 * Push zone_movable_pfn to the end so 6218 * that if we have to rebalance 6219 * kernelcore across nodes, we will 6220 * not double account here 6221 */ 6222 zone_movable_pfn[nid] = end_pfn; 6223 continue; 6224 } 6225 start_pfn = usable_startpfn; 6226 } 6227 6228 /* 6229 * The usable PFN range for ZONE_MOVABLE is from 6230 * start_pfn->end_pfn. Calculate size_pages as the 6231 * number of pages used as kernelcore 6232 */ 6233 size_pages = end_pfn - start_pfn; 6234 if (size_pages > kernelcore_remaining) 6235 size_pages = kernelcore_remaining; 6236 zone_movable_pfn[nid] = start_pfn + size_pages; 6237 6238 /* 6239 * Some kernelcore has been met, update counts and 6240 * break if the kernelcore for this node has been 6241 * satisfied 6242 */ 6243 required_kernelcore -= min(required_kernelcore, 6244 size_pages); 6245 kernelcore_remaining -= size_pages; 6246 if (!kernelcore_remaining) 6247 break; 6248 } 6249 } 6250 6251 /* 6252 * If there is still required_kernelcore, we do another pass with one 6253 * less node in the count. This will push zone_movable_pfn[nid] further 6254 * along on the nodes that still have memory until kernelcore is 6255 * satisfied 6256 */ 6257 usable_nodes--; 6258 if (usable_nodes && required_kernelcore > usable_nodes) 6259 goto restart; 6260 6261 out2: 6262 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 6263 for (nid = 0; nid < MAX_NUMNODES; nid++) 6264 zone_movable_pfn[nid] = 6265 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 6266 6267 out: 6268 /* restore the node_state */ 6269 node_states[N_MEMORY] = saved_node_state; 6270 } 6271 6272 /* Any regular or high memory on that node ? */ 6273 static void check_for_memory(pg_data_t *pgdat, int nid) 6274 { 6275 enum zone_type zone_type; 6276 6277 if (N_MEMORY == N_NORMAL_MEMORY) 6278 return; 6279 6280 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 6281 struct zone *zone = &pgdat->node_zones[zone_type]; 6282 if (populated_zone(zone)) { 6283 node_set_state(nid, N_HIGH_MEMORY); 6284 if (N_NORMAL_MEMORY != N_HIGH_MEMORY && 6285 zone_type <= ZONE_NORMAL) 6286 node_set_state(nid, N_NORMAL_MEMORY); 6287 break; 6288 } 6289 } 6290 } 6291 6292 /** 6293 * free_area_init_nodes - Initialise all pg_data_t and zone data 6294 * @max_zone_pfn: an array of max PFNs for each zone 6295 * 6296 * This will call free_area_init_node() for each active node in the system. 6297 * Using the page ranges provided by memblock_set_node(), the size of each 6298 * zone in each node and their holes is calculated. If the maximum PFN 6299 * between two adjacent zones match, it is assumed that the zone is empty. 6300 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 6301 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 6302 * starts where the previous one ended. For example, ZONE_DMA32 starts 6303 * at arch_max_dma_pfn. 6304 */ 6305 void __init free_area_init_nodes(unsigned long *max_zone_pfn) 6306 { 6307 unsigned long start_pfn, end_pfn; 6308 int i, nid; 6309 6310 /* Record where the zone boundaries are */ 6311 memset(arch_zone_lowest_possible_pfn, 0, 6312 sizeof(arch_zone_lowest_possible_pfn)); 6313 memset(arch_zone_highest_possible_pfn, 0, 6314 sizeof(arch_zone_highest_possible_pfn)); 6315 6316 start_pfn = find_min_pfn_with_active_regions(); 6317 6318 for (i = 0; i < MAX_NR_ZONES; i++) { 6319 if (i == ZONE_MOVABLE) 6320 continue; 6321 6322 end_pfn = max(max_zone_pfn[i], start_pfn); 6323 arch_zone_lowest_possible_pfn[i] = start_pfn; 6324 arch_zone_highest_possible_pfn[i] = end_pfn; 6325 6326 start_pfn = end_pfn; 6327 } 6328 arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0; 6329 arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0; 6330 6331 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 6332 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 6333 find_zone_movable_pfns_for_nodes(); 6334 6335 /* Print out the zone ranges */ 6336 pr_info("Zone ranges:\n"); 6337 for (i = 0; i < MAX_NR_ZONES; i++) { 6338 if (i == ZONE_MOVABLE) 6339 continue; 6340 pr_info(" %-8s ", zone_names[i]); 6341 if (arch_zone_lowest_possible_pfn[i] == 6342 arch_zone_highest_possible_pfn[i]) 6343 pr_cont("empty\n"); 6344 else 6345 pr_cont("[mem %#018Lx-%#018Lx]\n", 6346 (u64)arch_zone_lowest_possible_pfn[i] 6347 << PAGE_SHIFT, 6348 ((u64)arch_zone_highest_possible_pfn[i] 6349 << PAGE_SHIFT) - 1); 6350 } 6351 6352 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 6353 pr_info("Movable zone start for each node\n"); 6354 for (i = 0; i < MAX_NUMNODES; i++) { 6355 if (zone_movable_pfn[i]) 6356 pr_info(" Node %d: %#018Lx\n", i, 6357 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 6358 } 6359 6360 /* Print out the early node map */ 6361 pr_info("Early memory node ranges\n"); 6362 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) 6363 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 6364 (u64)start_pfn << PAGE_SHIFT, 6365 ((u64)end_pfn << PAGE_SHIFT) - 1); 6366 6367 /* Initialise every node */ 6368 mminit_verify_pageflags_layout(); 6369 setup_nr_node_ids(); 6370 for_each_online_node(nid) { 6371 pg_data_t *pgdat = NODE_DATA(nid); 6372 free_area_init_node(nid, NULL, 6373 find_min_pfn_for_node(nid), NULL); 6374 6375 /* Any memory on that node */ 6376 if (pgdat->node_present_pages) 6377 node_set_state(nid, N_MEMORY); 6378 check_for_memory(pgdat, nid); 6379 } 6380 } 6381 6382 static int __init cmdline_parse_core(char *p, unsigned long *core) 6383 { 6384 unsigned long long coremem; 6385 if (!p) 6386 return -EINVAL; 6387 6388 coremem = memparse(p, &p); 6389 *core = coremem >> PAGE_SHIFT; 6390 6391 /* Paranoid check that UL is enough for the coremem value */ 6392 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 6393 6394 return 0; 6395 } 6396 6397 /* 6398 * kernelcore=size sets the amount of memory for use for allocations that 6399 * cannot be reclaimed or migrated. 6400 */ 6401 static int __init cmdline_parse_kernelcore(char *p) 6402 { 6403 /* parse kernelcore=mirror */ 6404 if (parse_option_str(p, "mirror")) { 6405 mirrored_kernelcore = true; 6406 return 0; 6407 } 6408 6409 return cmdline_parse_core(p, &required_kernelcore); 6410 } 6411 6412 /* 6413 * movablecore=size sets the amount of memory for use for allocations that 6414 * can be reclaimed or migrated. 6415 */ 6416 static int __init cmdline_parse_movablecore(char *p) 6417 { 6418 return cmdline_parse_core(p, &required_movablecore); 6419 } 6420 6421 early_param("kernelcore", cmdline_parse_kernelcore); 6422 early_param("movablecore", cmdline_parse_movablecore); 6423 6424 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6425 6426 void adjust_managed_page_count(struct page *page, long count) 6427 { 6428 spin_lock(&managed_page_count_lock); 6429 page_zone(page)->managed_pages += count; 6430 totalram_pages += count; 6431 #ifdef CONFIG_HIGHMEM 6432 if (PageHighMem(page)) 6433 totalhigh_pages += count; 6434 #endif 6435 spin_unlock(&managed_page_count_lock); 6436 } 6437 EXPORT_SYMBOL(adjust_managed_page_count); 6438 6439 unsigned long free_reserved_area(void *start, void *end, int poison, char *s) 6440 { 6441 void *pos; 6442 unsigned long pages = 0; 6443 6444 start = (void *)PAGE_ALIGN((unsigned long)start); 6445 end = (void *)((unsigned long)end & PAGE_MASK); 6446 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 6447 if ((unsigned int)poison <= 0xFF) 6448 memset(pos, poison, PAGE_SIZE); 6449 free_reserved_page(virt_to_page(pos)); 6450 } 6451 6452 if (pages && s) 6453 pr_info("Freeing %s memory: %ldK\n", 6454 s, pages << (PAGE_SHIFT - 10)); 6455 6456 return pages; 6457 } 6458 EXPORT_SYMBOL(free_reserved_area); 6459 6460 #ifdef CONFIG_HIGHMEM 6461 void free_highmem_page(struct page *page) 6462 { 6463 __free_reserved_page(page); 6464 totalram_pages++; 6465 page_zone(page)->managed_pages++; 6466 totalhigh_pages++; 6467 } 6468 #endif 6469 6470 6471 void __init mem_init_print_info(const char *str) 6472 { 6473 unsigned long physpages, codesize, datasize, rosize, bss_size; 6474 unsigned long init_code_size, init_data_size; 6475 6476 physpages = get_num_physpages(); 6477 codesize = _etext - _stext; 6478 datasize = _edata - _sdata; 6479 rosize = __end_rodata - __start_rodata; 6480 bss_size = __bss_stop - __bss_start; 6481 init_data_size = __init_end - __init_begin; 6482 init_code_size = _einittext - _sinittext; 6483 6484 /* 6485 * Detect special cases and adjust section sizes accordingly: 6486 * 1) .init.* may be embedded into .data sections 6487 * 2) .init.text.* may be out of [__init_begin, __init_end], 6488 * please refer to arch/tile/kernel/vmlinux.lds.S. 6489 * 3) .rodata.* may be embedded into .text or .data sections. 6490 */ 6491 #define adj_init_size(start, end, size, pos, adj) \ 6492 do { \ 6493 if (start <= pos && pos < end && size > adj) \ 6494 size -= adj; \ 6495 } while (0) 6496 6497 adj_init_size(__init_begin, __init_end, init_data_size, 6498 _sinittext, init_code_size); 6499 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 6500 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 6501 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 6502 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 6503 6504 #undef adj_init_size 6505 6506 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 6507 #ifdef CONFIG_HIGHMEM 6508 ", %luK highmem" 6509 #endif 6510 "%s%s)\n", 6511 nr_free_pages() << (PAGE_SHIFT - 10), 6512 physpages << (PAGE_SHIFT - 10), 6513 codesize >> 10, datasize >> 10, rosize >> 10, 6514 (init_data_size + init_code_size) >> 10, bss_size >> 10, 6515 (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT - 10), 6516 totalcma_pages << (PAGE_SHIFT - 10), 6517 #ifdef CONFIG_HIGHMEM 6518 totalhigh_pages << (PAGE_SHIFT - 10), 6519 #endif 6520 str ? ", " : "", str ? str : ""); 6521 } 6522 6523 /** 6524 * set_dma_reserve - set the specified number of pages reserved in the first zone 6525 * @new_dma_reserve: The number of pages to mark reserved 6526 * 6527 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 6528 * In the DMA zone, a significant percentage may be consumed by kernel image 6529 * and other unfreeable allocations which can skew the watermarks badly. This 6530 * function may optionally be used to account for unfreeable pages in the 6531 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 6532 * smaller per-cpu batchsize. 6533 */ 6534 void __init set_dma_reserve(unsigned long new_dma_reserve) 6535 { 6536 dma_reserve = new_dma_reserve; 6537 } 6538 6539 void __init free_area_init(unsigned long *zones_size) 6540 { 6541 free_area_init_node(0, zones_size, 6542 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); 6543 } 6544 6545 static int page_alloc_cpu_dead(unsigned int cpu) 6546 { 6547 6548 lru_add_drain_cpu(cpu); 6549 drain_pages(cpu); 6550 6551 /* 6552 * Spill the event counters of the dead processor 6553 * into the current processors event counters. 6554 * This artificially elevates the count of the current 6555 * processor. 6556 */ 6557 vm_events_fold_cpu(cpu); 6558 6559 /* 6560 * Zero the differential counters of the dead processor 6561 * so that the vm statistics are consistent. 6562 * 6563 * This is only okay since the processor is dead and cannot 6564 * race with what we are doing. 6565 */ 6566 cpu_vm_stats_fold(cpu); 6567 return 0; 6568 } 6569 6570 void __init page_alloc_init(void) 6571 { 6572 int ret; 6573 6574 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD, 6575 "mm/page_alloc:dead", NULL, 6576 page_alloc_cpu_dead); 6577 WARN_ON(ret < 0); 6578 } 6579 6580 /* 6581 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 6582 * or min_free_kbytes changes. 6583 */ 6584 static void calculate_totalreserve_pages(void) 6585 { 6586 struct pglist_data *pgdat; 6587 unsigned long reserve_pages = 0; 6588 enum zone_type i, j; 6589 6590 for_each_online_pgdat(pgdat) { 6591 6592 pgdat->totalreserve_pages = 0; 6593 6594 for (i = 0; i < MAX_NR_ZONES; i++) { 6595 struct zone *zone = pgdat->node_zones + i; 6596 long max = 0; 6597 6598 /* Find valid and maximum lowmem_reserve in the zone */ 6599 for (j = i; j < MAX_NR_ZONES; j++) { 6600 if (zone->lowmem_reserve[j] > max) 6601 max = zone->lowmem_reserve[j]; 6602 } 6603 6604 /* we treat the high watermark as reserved pages. */ 6605 max += high_wmark_pages(zone); 6606 6607 if (max > zone->managed_pages) 6608 max = zone->managed_pages; 6609 6610 pgdat->totalreserve_pages += max; 6611 6612 reserve_pages += max; 6613 } 6614 } 6615 totalreserve_pages = reserve_pages; 6616 } 6617 6618 /* 6619 * setup_per_zone_lowmem_reserve - called whenever 6620 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 6621 * has a correct pages reserved value, so an adequate number of 6622 * pages are left in the zone after a successful __alloc_pages(). 6623 */ 6624 static void setup_per_zone_lowmem_reserve(void) 6625 { 6626 struct pglist_data *pgdat; 6627 enum zone_type j, idx; 6628 6629 for_each_online_pgdat(pgdat) { 6630 for (j = 0; j < MAX_NR_ZONES; j++) { 6631 struct zone *zone = pgdat->node_zones + j; 6632 unsigned long managed_pages = zone->managed_pages; 6633 6634 zone->lowmem_reserve[j] = 0; 6635 6636 idx = j; 6637 while (idx) { 6638 struct zone *lower_zone; 6639 6640 idx--; 6641 6642 if (sysctl_lowmem_reserve_ratio[idx] < 1) 6643 sysctl_lowmem_reserve_ratio[idx] = 1; 6644 6645 lower_zone = pgdat->node_zones + idx; 6646 lower_zone->lowmem_reserve[j] = managed_pages / 6647 sysctl_lowmem_reserve_ratio[idx]; 6648 managed_pages += lower_zone->managed_pages; 6649 } 6650 } 6651 } 6652 6653 /* update totalreserve_pages */ 6654 calculate_totalreserve_pages(); 6655 } 6656 6657 static void __setup_per_zone_wmarks(void) 6658 { 6659 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 6660 unsigned long lowmem_pages = 0; 6661 struct zone *zone; 6662 unsigned long flags; 6663 6664 /* Calculate total number of !ZONE_HIGHMEM pages */ 6665 for_each_zone(zone) { 6666 if (!is_highmem(zone)) 6667 lowmem_pages += zone->managed_pages; 6668 } 6669 6670 for_each_zone(zone) { 6671 u64 tmp; 6672 6673 spin_lock_irqsave(&zone->lock, flags); 6674 tmp = (u64)pages_min * zone->managed_pages; 6675 do_div(tmp, lowmem_pages); 6676 if (is_highmem(zone)) { 6677 /* 6678 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 6679 * need highmem pages, so cap pages_min to a small 6680 * value here. 6681 * 6682 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 6683 * deltas control asynch page reclaim, and so should 6684 * not be capped for highmem. 6685 */ 6686 unsigned long min_pages; 6687 6688 min_pages = zone->managed_pages / 1024; 6689 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 6690 zone->watermark[WMARK_MIN] = min_pages; 6691 } else { 6692 /* 6693 * If it's a lowmem zone, reserve a number of pages 6694 * proportionate to the zone's size. 6695 */ 6696 zone->watermark[WMARK_MIN] = tmp; 6697 } 6698 6699 /* 6700 * Set the kswapd watermarks distance according to the 6701 * scale factor in proportion to available memory, but 6702 * ensure a minimum size on small systems. 6703 */ 6704 tmp = max_t(u64, tmp >> 2, 6705 mult_frac(zone->managed_pages, 6706 watermark_scale_factor, 10000)); 6707 6708 zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 6709 zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2; 6710 6711 spin_unlock_irqrestore(&zone->lock, flags); 6712 } 6713 6714 /* update totalreserve_pages */ 6715 calculate_totalreserve_pages(); 6716 } 6717 6718 /** 6719 * setup_per_zone_wmarks - called when min_free_kbytes changes 6720 * or when memory is hot-{added|removed} 6721 * 6722 * Ensures that the watermark[min,low,high] values for each zone are set 6723 * correctly with respect to min_free_kbytes. 6724 */ 6725 void setup_per_zone_wmarks(void) 6726 { 6727 mutex_lock(&zonelists_mutex); 6728 __setup_per_zone_wmarks(); 6729 mutex_unlock(&zonelists_mutex); 6730 } 6731 6732 /* 6733 * Initialise min_free_kbytes. 6734 * 6735 * For small machines we want it small (128k min). For large machines 6736 * we want it large (64MB max). But it is not linear, because network 6737 * bandwidth does not increase linearly with machine size. We use 6738 * 6739 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 6740 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 6741 * 6742 * which yields 6743 * 6744 * 16MB: 512k 6745 * 32MB: 724k 6746 * 64MB: 1024k 6747 * 128MB: 1448k 6748 * 256MB: 2048k 6749 * 512MB: 2896k 6750 * 1024MB: 4096k 6751 * 2048MB: 5792k 6752 * 4096MB: 8192k 6753 * 8192MB: 11584k 6754 * 16384MB: 16384k 6755 */ 6756 int __meminit init_per_zone_wmark_min(void) 6757 { 6758 unsigned long lowmem_kbytes; 6759 int new_min_free_kbytes; 6760 6761 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 6762 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 6763 6764 if (new_min_free_kbytes > user_min_free_kbytes) { 6765 min_free_kbytes = new_min_free_kbytes; 6766 if (min_free_kbytes < 128) 6767 min_free_kbytes = 128; 6768 if (min_free_kbytes > 65536) 6769 min_free_kbytes = 65536; 6770 } else { 6771 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 6772 new_min_free_kbytes, user_min_free_kbytes); 6773 } 6774 setup_per_zone_wmarks(); 6775 refresh_zone_stat_thresholds(); 6776 setup_per_zone_lowmem_reserve(); 6777 6778 #ifdef CONFIG_NUMA 6779 setup_min_unmapped_ratio(); 6780 setup_min_slab_ratio(); 6781 #endif 6782 6783 return 0; 6784 } 6785 core_initcall(init_per_zone_wmark_min) 6786 6787 /* 6788 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 6789 * that we can call two helper functions whenever min_free_kbytes 6790 * changes. 6791 */ 6792 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 6793 void __user *buffer, size_t *length, loff_t *ppos) 6794 { 6795 int rc; 6796 6797 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6798 if (rc) 6799 return rc; 6800 6801 if (write) { 6802 user_min_free_kbytes = min_free_kbytes; 6803 setup_per_zone_wmarks(); 6804 } 6805 return 0; 6806 } 6807 6808 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 6809 void __user *buffer, size_t *length, loff_t *ppos) 6810 { 6811 int rc; 6812 6813 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6814 if (rc) 6815 return rc; 6816 6817 if (write) 6818 setup_per_zone_wmarks(); 6819 6820 return 0; 6821 } 6822 6823 #ifdef CONFIG_NUMA 6824 static void setup_min_unmapped_ratio(void) 6825 { 6826 pg_data_t *pgdat; 6827 struct zone *zone; 6828 6829 for_each_online_pgdat(pgdat) 6830 pgdat->min_unmapped_pages = 0; 6831 6832 for_each_zone(zone) 6833 zone->zone_pgdat->min_unmapped_pages += (zone->managed_pages * 6834 sysctl_min_unmapped_ratio) / 100; 6835 } 6836 6837 6838 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 6839 void __user *buffer, size_t *length, loff_t *ppos) 6840 { 6841 int rc; 6842 6843 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6844 if (rc) 6845 return rc; 6846 6847 setup_min_unmapped_ratio(); 6848 6849 return 0; 6850 } 6851 6852 static void setup_min_slab_ratio(void) 6853 { 6854 pg_data_t *pgdat; 6855 struct zone *zone; 6856 6857 for_each_online_pgdat(pgdat) 6858 pgdat->min_slab_pages = 0; 6859 6860 for_each_zone(zone) 6861 zone->zone_pgdat->min_slab_pages += (zone->managed_pages * 6862 sysctl_min_slab_ratio) / 100; 6863 } 6864 6865 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 6866 void __user *buffer, size_t *length, loff_t *ppos) 6867 { 6868 int rc; 6869 6870 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6871 if (rc) 6872 return rc; 6873 6874 setup_min_slab_ratio(); 6875 6876 return 0; 6877 } 6878 #endif 6879 6880 /* 6881 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 6882 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 6883 * whenever sysctl_lowmem_reserve_ratio changes. 6884 * 6885 * The reserve ratio obviously has absolutely no relation with the 6886 * minimum watermarks. The lowmem reserve ratio can only make sense 6887 * if in function of the boot time zone sizes. 6888 */ 6889 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 6890 void __user *buffer, size_t *length, loff_t *ppos) 6891 { 6892 proc_dointvec_minmax(table, write, buffer, length, ppos); 6893 setup_per_zone_lowmem_reserve(); 6894 return 0; 6895 } 6896 6897 /* 6898 * percpu_pagelist_fraction - changes the pcp->high for each zone on each 6899 * cpu. It is the fraction of total pages in each zone that a hot per cpu 6900 * pagelist can have before it gets flushed back to buddy allocator. 6901 */ 6902 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, 6903 void __user *buffer, size_t *length, loff_t *ppos) 6904 { 6905 struct zone *zone; 6906 int old_percpu_pagelist_fraction; 6907 int ret; 6908 6909 mutex_lock(&pcp_batch_high_lock); 6910 old_percpu_pagelist_fraction = percpu_pagelist_fraction; 6911 6912 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 6913 if (!write || ret < 0) 6914 goto out; 6915 6916 /* Sanity checking to avoid pcp imbalance */ 6917 if (percpu_pagelist_fraction && 6918 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { 6919 percpu_pagelist_fraction = old_percpu_pagelist_fraction; 6920 ret = -EINVAL; 6921 goto out; 6922 } 6923 6924 /* No change? */ 6925 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) 6926 goto out; 6927 6928 for_each_populated_zone(zone) { 6929 unsigned int cpu; 6930 6931 for_each_possible_cpu(cpu) 6932 pageset_set_high_and_batch(zone, 6933 per_cpu_ptr(zone->pageset, cpu)); 6934 } 6935 out: 6936 mutex_unlock(&pcp_batch_high_lock); 6937 return ret; 6938 } 6939 6940 #ifdef CONFIG_NUMA 6941 int hashdist = HASHDIST_DEFAULT; 6942 6943 static int __init set_hashdist(char *str) 6944 { 6945 if (!str) 6946 return 0; 6947 hashdist = simple_strtoul(str, &str, 0); 6948 return 1; 6949 } 6950 __setup("hashdist=", set_hashdist); 6951 #endif 6952 6953 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 6954 /* 6955 * Returns the number of pages that arch has reserved but 6956 * is not known to alloc_large_system_hash(). 6957 */ 6958 static unsigned long __init arch_reserved_kernel_pages(void) 6959 { 6960 return 0; 6961 } 6962 #endif 6963 6964 /* 6965 * allocate a large system hash table from bootmem 6966 * - it is assumed that the hash table must contain an exact power-of-2 6967 * quantity of entries 6968 * - limit is the number of hash buckets, not the total allocation size 6969 */ 6970 void *__init alloc_large_system_hash(const char *tablename, 6971 unsigned long bucketsize, 6972 unsigned long numentries, 6973 int scale, 6974 int flags, 6975 unsigned int *_hash_shift, 6976 unsigned int *_hash_mask, 6977 unsigned long low_limit, 6978 unsigned long high_limit) 6979 { 6980 unsigned long long max = high_limit; 6981 unsigned long log2qty, size; 6982 void *table = NULL; 6983 6984 /* allow the kernel cmdline to have a say */ 6985 if (!numentries) { 6986 /* round applicable memory size up to nearest megabyte */ 6987 numentries = nr_kernel_pages; 6988 numentries -= arch_reserved_kernel_pages(); 6989 6990 /* It isn't necessary when PAGE_SIZE >= 1MB */ 6991 if (PAGE_SHIFT < 20) 6992 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 6993 6994 /* limit to 1 bucket per 2^scale bytes of low memory */ 6995 if (scale > PAGE_SHIFT) 6996 numentries >>= (scale - PAGE_SHIFT); 6997 else 6998 numentries <<= (PAGE_SHIFT - scale); 6999 7000 /* Make sure we've got at least a 0-order allocation.. */ 7001 if (unlikely(flags & HASH_SMALL)) { 7002 /* Makes no sense without HASH_EARLY */ 7003 WARN_ON(!(flags & HASH_EARLY)); 7004 if (!(numentries >> *_hash_shift)) { 7005 numentries = 1UL << *_hash_shift; 7006 BUG_ON(!numentries); 7007 } 7008 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 7009 numentries = PAGE_SIZE / bucketsize; 7010 } 7011 numentries = roundup_pow_of_two(numentries); 7012 7013 /* limit allocation size to 1/16 total memory by default */ 7014 if (max == 0) { 7015 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 7016 do_div(max, bucketsize); 7017 } 7018 max = min(max, 0x80000000ULL); 7019 7020 if (numentries < low_limit) 7021 numentries = low_limit; 7022 if (numentries > max) 7023 numentries = max; 7024 7025 log2qty = ilog2(numentries); 7026 7027 do { 7028 size = bucketsize << log2qty; 7029 if (flags & HASH_EARLY) 7030 table = memblock_virt_alloc_nopanic(size, 0); 7031 else if (hashdist) 7032 table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL); 7033 else { 7034 /* 7035 * If bucketsize is not a power-of-two, we may free 7036 * some pages at the end of hash table which 7037 * alloc_pages_exact() automatically does 7038 */ 7039 if (get_order(size) < MAX_ORDER) { 7040 table = alloc_pages_exact(size, GFP_ATOMIC); 7041 kmemleak_alloc(table, size, 1, GFP_ATOMIC); 7042 } 7043 } 7044 } while (!table && size > PAGE_SIZE && --log2qty); 7045 7046 if (!table) 7047 panic("Failed to allocate %s hash table\n", tablename); 7048 7049 pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n", 7050 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size); 7051 7052 if (_hash_shift) 7053 *_hash_shift = log2qty; 7054 if (_hash_mask) 7055 *_hash_mask = (1 << log2qty) - 1; 7056 7057 return table; 7058 } 7059 7060 /* 7061 * This function checks whether pageblock includes unmovable pages or not. 7062 * If @count is not zero, it is okay to include less @count unmovable pages 7063 * 7064 * PageLRU check without isolation or lru_lock could race so that 7065 * MIGRATE_MOVABLE block might include unmovable pages. It means you can't 7066 * expect this function should be exact. 7067 */ 7068 bool has_unmovable_pages(struct zone *zone, struct page *page, int count, 7069 bool skip_hwpoisoned_pages) 7070 { 7071 unsigned long pfn, iter, found; 7072 int mt; 7073 7074 /* 7075 * For avoiding noise data, lru_add_drain_all() should be called 7076 * If ZONE_MOVABLE, the zone never contains unmovable pages 7077 */ 7078 if (zone_idx(zone) == ZONE_MOVABLE) 7079 return false; 7080 mt = get_pageblock_migratetype(page); 7081 if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt)) 7082 return false; 7083 7084 pfn = page_to_pfn(page); 7085 for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) { 7086 unsigned long check = pfn + iter; 7087 7088 if (!pfn_valid_within(check)) 7089 continue; 7090 7091 page = pfn_to_page(check); 7092 7093 /* 7094 * Hugepages are not in LRU lists, but they're movable. 7095 * We need not scan over tail pages bacause we don't 7096 * handle each tail page individually in migration. 7097 */ 7098 if (PageHuge(page)) { 7099 iter = round_up(iter + 1, 1<<compound_order(page)) - 1; 7100 continue; 7101 } 7102 7103 /* 7104 * We can't use page_count without pin a page 7105 * because another CPU can free compound page. 7106 * This check already skips compound tails of THP 7107 * because their page->_refcount is zero at all time. 7108 */ 7109 if (!page_ref_count(page)) { 7110 if (PageBuddy(page)) 7111 iter += (1 << page_order(page)) - 1; 7112 continue; 7113 } 7114 7115 /* 7116 * The HWPoisoned page may be not in buddy system, and 7117 * page_count() is not 0. 7118 */ 7119 if (skip_hwpoisoned_pages && PageHWPoison(page)) 7120 continue; 7121 7122 if (!PageLRU(page)) 7123 found++; 7124 /* 7125 * If there are RECLAIMABLE pages, we need to check 7126 * it. But now, memory offline itself doesn't call 7127 * shrink_node_slabs() and it still to be fixed. 7128 */ 7129 /* 7130 * If the page is not RAM, page_count()should be 0. 7131 * we don't need more check. This is an _used_ not-movable page. 7132 * 7133 * The problematic thing here is PG_reserved pages. PG_reserved 7134 * is set to both of a memory hole page and a _used_ kernel 7135 * page at boot. 7136 */ 7137 if (found > count) 7138 return true; 7139 } 7140 return false; 7141 } 7142 7143 bool is_pageblock_removable_nolock(struct page *page) 7144 { 7145 struct zone *zone; 7146 unsigned long pfn; 7147 7148 /* 7149 * We have to be careful here because we are iterating over memory 7150 * sections which are not zone aware so we might end up outside of 7151 * the zone but still within the section. 7152 * We have to take care about the node as well. If the node is offline 7153 * its NODE_DATA will be NULL - see page_zone. 7154 */ 7155 if (!node_online(page_to_nid(page))) 7156 return false; 7157 7158 zone = page_zone(page); 7159 pfn = page_to_pfn(page); 7160 if (!zone_spans_pfn(zone, pfn)) 7161 return false; 7162 7163 return !has_unmovable_pages(zone, page, 0, true); 7164 } 7165 7166 #if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA) 7167 7168 static unsigned long pfn_max_align_down(unsigned long pfn) 7169 { 7170 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, 7171 pageblock_nr_pages) - 1); 7172 } 7173 7174 static unsigned long pfn_max_align_up(unsigned long pfn) 7175 { 7176 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, 7177 pageblock_nr_pages)); 7178 } 7179 7180 /* [start, end) must belong to a single zone. */ 7181 static int __alloc_contig_migrate_range(struct compact_control *cc, 7182 unsigned long start, unsigned long end) 7183 { 7184 /* This function is based on compact_zone() from compaction.c. */ 7185 unsigned long nr_reclaimed; 7186 unsigned long pfn = start; 7187 unsigned int tries = 0; 7188 int ret = 0; 7189 7190 migrate_prep(); 7191 7192 while (pfn < end || !list_empty(&cc->migratepages)) { 7193 if (fatal_signal_pending(current)) { 7194 ret = -EINTR; 7195 break; 7196 } 7197 7198 if (list_empty(&cc->migratepages)) { 7199 cc->nr_migratepages = 0; 7200 pfn = isolate_migratepages_range(cc, pfn, end); 7201 if (!pfn) { 7202 ret = -EINTR; 7203 break; 7204 } 7205 tries = 0; 7206 } else if (++tries == 5) { 7207 ret = ret < 0 ? ret : -EBUSY; 7208 break; 7209 } 7210 7211 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 7212 &cc->migratepages); 7213 cc->nr_migratepages -= nr_reclaimed; 7214 7215 ret = migrate_pages(&cc->migratepages, alloc_migrate_target, 7216 NULL, 0, cc->mode, MR_CMA); 7217 } 7218 if (ret < 0) { 7219 putback_movable_pages(&cc->migratepages); 7220 return ret; 7221 } 7222 return 0; 7223 } 7224 7225 /** 7226 * alloc_contig_range() -- tries to allocate given range of pages 7227 * @start: start PFN to allocate 7228 * @end: one-past-the-last PFN to allocate 7229 * @migratetype: migratetype of the underlaying pageblocks (either 7230 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 7231 * in range must have the same migratetype and it must 7232 * be either of the two. 7233 * 7234 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 7235 * aligned, however it's the caller's responsibility to guarantee that 7236 * we are the only thread that changes migrate type of pageblocks the 7237 * pages fall in. 7238 * 7239 * The PFN range must belong to a single zone. 7240 * 7241 * Returns zero on success or negative error code. On success all 7242 * pages which PFN is in [start, end) are allocated for the caller and 7243 * need to be freed with free_contig_range(). 7244 */ 7245 int alloc_contig_range(unsigned long start, unsigned long end, 7246 unsigned migratetype) 7247 { 7248 unsigned long outer_start, outer_end; 7249 unsigned int order; 7250 int ret = 0; 7251 7252 struct compact_control cc = { 7253 .nr_migratepages = 0, 7254 .order = -1, 7255 .zone = page_zone(pfn_to_page(start)), 7256 .mode = MIGRATE_SYNC, 7257 .ignore_skip_hint = true, 7258 }; 7259 INIT_LIST_HEAD(&cc.migratepages); 7260 7261 /* 7262 * What we do here is we mark all pageblocks in range as 7263 * MIGRATE_ISOLATE. Because pageblock and max order pages may 7264 * have different sizes, and due to the way page allocator 7265 * work, we align the range to biggest of the two pages so 7266 * that page allocator won't try to merge buddies from 7267 * different pageblocks and change MIGRATE_ISOLATE to some 7268 * other migration type. 7269 * 7270 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 7271 * migrate the pages from an unaligned range (ie. pages that 7272 * we are interested in). This will put all the pages in 7273 * range back to page allocator as MIGRATE_ISOLATE. 7274 * 7275 * When this is done, we take the pages in range from page 7276 * allocator removing them from the buddy system. This way 7277 * page allocator will never consider using them. 7278 * 7279 * This lets us mark the pageblocks back as 7280 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 7281 * aligned range but not in the unaligned, original range are 7282 * put back to page allocator so that buddy can use them. 7283 */ 7284 7285 ret = start_isolate_page_range(pfn_max_align_down(start), 7286 pfn_max_align_up(end), migratetype, 7287 false); 7288 if (ret) 7289 return ret; 7290 7291 /* 7292 * In case of -EBUSY, we'd like to know which page causes problem. 7293 * So, just fall through. We will check it in test_pages_isolated(). 7294 */ 7295 ret = __alloc_contig_migrate_range(&cc, start, end); 7296 if (ret && ret != -EBUSY) 7297 goto done; 7298 7299 /* 7300 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 7301 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 7302 * more, all pages in [start, end) are free in page allocator. 7303 * What we are going to do is to allocate all pages from 7304 * [start, end) (that is remove them from page allocator). 7305 * 7306 * The only problem is that pages at the beginning and at the 7307 * end of interesting range may be not aligned with pages that 7308 * page allocator holds, ie. they can be part of higher order 7309 * pages. Because of this, we reserve the bigger range and 7310 * once this is done free the pages we are not interested in. 7311 * 7312 * We don't have to hold zone->lock here because the pages are 7313 * isolated thus they won't get removed from buddy. 7314 */ 7315 7316 lru_add_drain_all(); 7317 drain_all_pages(cc.zone); 7318 7319 order = 0; 7320 outer_start = start; 7321 while (!PageBuddy(pfn_to_page(outer_start))) { 7322 if (++order >= MAX_ORDER) { 7323 outer_start = start; 7324 break; 7325 } 7326 outer_start &= ~0UL << order; 7327 } 7328 7329 if (outer_start != start) { 7330 order = page_order(pfn_to_page(outer_start)); 7331 7332 /* 7333 * outer_start page could be small order buddy page and 7334 * it doesn't include start page. Adjust outer_start 7335 * in this case to report failed page properly 7336 * on tracepoint in test_pages_isolated() 7337 */ 7338 if (outer_start + (1UL << order) <= start) 7339 outer_start = start; 7340 } 7341 7342 /* Make sure the range is really isolated. */ 7343 if (test_pages_isolated(outer_start, end, false)) { 7344 pr_info("%s: [%lx, %lx) PFNs busy\n", 7345 __func__, outer_start, end); 7346 ret = -EBUSY; 7347 goto done; 7348 } 7349 7350 /* Grab isolated pages from freelists. */ 7351 outer_end = isolate_freepages_range(&cc, outer_start, end); 7352 if (!outer_end) { 7353 ret = -EBUSY; 7354 goto done; 7355 } 7356 7357 /* Free head and tail (if any) */ 7358 if (start != outer_start) 7359 free_contig_range(outer_start, start - outer_start); 7360 if (end != outer_end) 7361 free_contig_range(end, outer_end - end); 7362 7363 done: 7364 undo_isolate_page_range(pfn_max_align_down(start), 7365 pfn_max_align_up(end), migratetype); 7366 return ret; 7367 } 7368 7369 void free_contig_range(unsigned long pfn, unsigned nr_pages) 7370 { 7371 unsigned int count = 0; 7372 7373 for (; nr_pages--; pfn++) { 7374 struct page *page = pfn_to_page(pfn); 7375 7376 count += page_count(page) != 1; 7377 __free_page(page); 7378 } 7379 WARN(count != 0, "%d pages are still in use!\n", count); 7380 } 7381 #endif 7382 7383 #ifdef CONFIG_MEMORY_HOTPLUG 7384 /* 7385 * The zone indicated has a new number of managed_pages; batch sizes and percpu 7386 * page high values need to be recalulated. 7387 */ 7388 void __meminit zone_pcp_update(struct zone *zone) 7389 { 7390 unsigned cpu; 7391 mutex_lock(&pcp_batch_high_lock); 7392 for_each_possible_cpu(cpu) 7393 pageset_set_high_and_batch(zone, 7394 per_cpu_ptr(zone->pageset, cpu)); 7395 mutex_unlock(&pcp_batch_high_lock); 7396 } 7397 #endif 7398 7399 void zone_pcp_reset(struct zone *zone) 7400 { 7401 unsigned long flags; 7402 int cpu; 7403 struct per_cpu_pageset *pset; 7404 7405 /* avoid races with drain_pages() */ 7406 local_irq_save(flags); 7407 if (zone->pageset != &boot_pageset) { 7408 for_each_online_cpu(cpu) { 7409 pset = per_cpu_ptr(zone->pageset, cpu); 7410 drain_zonestat(zone, pset); 7411 } 7412 free_percpu(zone->pageset); 7413 zone->pageset = &boot_pageset; 7414 } 7415 local_irq_restore(flags); 7416 } 7417 7418 #ifdef CONFIG_MEMORY_HOTREMOVE 7419 /* 7420 * All pages in the range must be in a single zone and isolated 7421 * before calling this. 7422 */ 7423 void 7424 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 7425 { 7426 struct page *page; 7427 struct zone *zone; 7428 unsigned int order, i; 7429 unsigned long pfn; 7430 unsigned long flags; 7431 /* find the first valid pfn */ 7432 for (pfn = start_pfn; pfn < end_pfn; pfn++) 7433 if (pfn_valid(pfn)) 7434 break; 7435 if (pfn == end_pfn) 7436 return; 7437 zone = page_zone(pfn_to_page(pfn)); 7438 spin_lock_irqsave(&zone->lock, flags); 7439 pfn = start_pfn; 7440 while (pfn < end_pfn) { 7441 if (!pfn_valid(pfn)) { 7442 pfn++; 7443 continue; 7444 } 7445 page = pfn_to_page(pfn); 7446 /* 7447 * The HWPoisoned page may be not in buddy system, and 7448 * page_count() is not 0. 7449 */ 7450 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 7451 pfn++; 7452 SetPageReserved(page); 7453 continue; 7454 } 7455 7456 BUG_ON(page_count(page)); 7457 BUG_ON(!PageBuddy(page)); 7458 order = page_order(page); 7459 #ifdef CONFIG_DEBUG_VM 7460 pr_info("remove from free list %lx %d %lx\n", 7461 pfn, 1 << order, end_pfn); 7462 #endif 7463 list_del(&page->lru); 7464 rmv_page_order(page); 7465 zone->free_area[order].nr_free--; 7466 for (i = 0; i < (1 << order); i++) 7467 SetPageReserved((page+i)); 7468 pfn += (1 << order); 7469 } 7470 spin_unlock_irqrestore(&zone->lock, flags); 7471 } 7472 #endif 7473 7474 bool is_free_buddy_page(struct page *page) 7475 { 7476 struct zone *zone = page_zone(page); 7477 unsigned long pfn = page_to_pfn(page); 7478 unsigned long flags; 7479 unsigned int order; 7480 7481 spin_lock_irqsave(&zone->lock, flags); 7482 for (order = 0; order < MAX_ORDER; order++) { 7483 struct page *page_head = page - (pfn & ((1 << order) - 1)); 7484 7485 if (PageBuddy(page_head) && page_order(page_head) >= order) 7486 break; 7487 } 7488 spin_unlock_irqrestore(&zone->lock, flags); 7489 7490 return order < MAX_ORDER; 7491 } 7492