1 /* 2 * linux/mm/page_alloc.c 3 * 4 * Manages the free list, the system allocates free pages here. 5 * Note that kmalloc() lives in slab.c 6 * 7 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 8 * Swap reorganised 29.12.95, Stephen Tweedie 9 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 10 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 11 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 12 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 13 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 14 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 15 */ 16 17 #include <linux/stddef.h> 18 #include <linux/mm.h> 19 #include <linux/swap.h> 20 #include <linux/interrupt.h> 21 #include <linux/pagemap.h> 22 #include <linux/jiffies.h> 23 #include <linux/bootmem.h> 24 #include <linux/memblock.h> 25 #include <linux/compiler.h> 26 #include <linux/kernel.h> 27 #include <linux/kmemcheck.h> 28 #include <linux/kasan.h> 29 #include <linux/module.h> 30 #include <linux/suspend.h> 31 #include <linux/pagevec.h> 32 #include <linux/blkdev.h> 33 #include <linux/slab.h> 34 #include <linux/ratelimit.h> 35 #include <linux/oom.h> 36 #include <linux/notifier.h> 37 #include <linux/topology.h> 38 #include <linux/sysctl.h> 39 #include <linux/cpu.h> 40 #include <linux/cpuset.h> 41 #include <linux/memory_hotplug.h> 42 #include <linux/nodemask.h> 43 #include <linux/vmalloc.h> 44 #include <linux/vmstat.h> 45 #include <linux/mempolicy.h> 46 #include <linux/memremap.h> 47 #include <linux/stop_machine.h> 48 #include <linux/sort.h> 49 #include <linux/pfn.h> 50 #include <linux/backing-dev.h> 51 #include <linux/fault-inject.h> 52 #include <linux/page-isolation.h> 53 #include <linux/page_ext.h> 54 #include <linux/debugobjects.h> 55 #include <linux/kmemleak.h> 56 #include <linux/compaction.h> 57 #include <trace/events/kmem.h> 58 #include <linux/prefetch.h> 59 #include <linux/mm_inline.h> 60 #include <linux/migrate.h> 61 #include <linux/page_ext.h> 62 #include <linux/hugetlb.h> 63 #include <linux/sched/rt.h> 64 #include <linux/page_owner.h> 65 #include <linux/kthread.h> 66 67 #include <asm/sections.h> 68 #include <asm/tlbflush.h> 69 #include <asm/div64.h> 70 #include "internal.h" 71 72 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 73 static DEFINE_MUTEX(pcp_batch_high_lock); 74 #define MIN_PERCPU_PAGELIST_FRACTION (8) 75 76 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 77 DEFINE_PER_CPU(int, numa_node); 78 EXPORT_PER_CPU_SYMBOL(numa_node); 79 #endif 80 81 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 82 /* 83 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 84 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 85 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 86 * defined in <linux/topology.h>. 87 */ 88 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 89 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 90 int _node_numa_mem_[MAX_NUMNODES]; 91 #endif 92 93 /* 94 * Array of node states. 95 */ 96 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 97 [N_POSSIBLE] = NODE_MASK_ALL, 98 [N_ONLINE] = { { [0] = 1UL } }, 99 #ifndef CONFIG_NUMA 100 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 101 #ifdef CONFIG_HIGHMEM 102 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 103 #endif 104 #ifdef CONFIG_MOVABLE_NODE 105 [N_MEMORY] = { { [0] = 1UL } }, 106 #endif 107 [N_CPU] = { { [0] = 1UL } }, 108 #endif /* NUMA */ 109 }; 110 EXPORT_SYMBOL(node_states); 111 112 /* Protect totalram_pages and zone->managed_pages */ 113 static DEFINE_SPINLOCK(managed_page_count_lock); 114 115 unsigned long totalram_pages __read_mostly; 116 unsigned long totalreserve_pages __read_mostly; 117 unsigned long totalcma_pages __read_mostly; 118 119 int percpu_pagelist_fraction; 120 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 121 122 /* 123 * A cached value of the page's pageblock's migratetype, used when the page is 124 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 125 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 126 * Also the migratetype set in the page does not necessarily match the pcplist 127 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 128 * other index - this ensures that it will be put on the correct CMA freelist. 129 */ 130 static inline int get_pcppage_migratetype(struct page *page) 131 { 132 return page->index; 133 } 134 135 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 136 { 137 page->index = migratetype; 138 } 139 140 #ifdef CONFIG_PM_SLEEP 141 /* 142 * The following functions are used by the suspend/hibernate code to temporarily 143 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 144 * while devices are suspended. To avoid races with the suspend/hibernate code, 145 * they should always be called with pm_mutex held (gfp_allowed_mask also should 146 * only be modified with pm_mutex held, unless the suspend/hibernate code is 147 * guaranteed not to run in parallel with that modification). 148 */ 149 150 static gfp_t saved_gfp_mask; 151 152 void pm_restore_gfp_mask(void) 153 { 154 WARN_ON(!mutex_is_locked(&pm_mutex)); 155 if (saved_gfp_mask) { 156 gfp_allowed_mask = saved_gfp_mask; 157 saved_gfp_mask = 0; 158 } 159 } 160 161 void pm_restrict_gfp_mask(void) 162 { 163 WARN_ON(!mutex_is_locked(&pm_mutex)); 164 WARN_ON(saved_gfp_mask); 165 saved_gfp_mask = gfp_allowed_mask; 166 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 167 } 168 169 bool pm_suspended_storage(void) 170 { 171 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 172 return false; 173 return true; 174 } 175 #endif /* CONFIG_PM_SLEEP */ 176 177 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 178 unsigned int pageblock_order __read_mostly; 179 #endif 180 181 static void __free_pages_ok(struct page *page, unsigned int order); 182 183 /* 184 * results with 256, 32 in the lowmem_reserve sysctl: 185 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 186 * 1G machine -> (16M dma, 784M normal, 224M high) 187 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 188 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 189 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 190 * 191 * TBD: should special case ZONE_DMA32 machines here - in those we normally 192 * don't need any ZONE_NORMAL reservation 193 */ 194 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = { 195 #ifdef CONFIG_ZONE_DMA 196 256, 197 #endif 198 #ifdef CONFIG_ZONE_DMA32 199 256, 200 #endif 201 #ifdef CONFIG_HIGHMEM 202 32, 203 #endif 204 32, 205 }; 206 207 EXPORT_SYMBOL(totalram_pages); 208 209 static char * const zone_names[MAX_NR_ZONES] = { 210 #ifdef CONFIG_ZONE_DMA 211 "DMA", 212 #endif 213 #ifdef CONFIG_ZONE_DMA32 214 "DMA32", 215 #endif 216 "Normal", 217 #ifdef CONFIG_HIGHMEM 218 "HighMem", 219 #endif 220 "Movable", 221 #ifdef CONFIG_ZONE_DEVICE 222 "Device", 223 #endif 224 }; 225 226 char * const migratetype_names[MIGRATE_TYPES] = { 227 "Unmovable", 228 "Movable", 229 "Reclaimable", 230 "HighAtomic", 231 #ifdef CONFIG_CMA 232 "CMA", 233 #endif 234 #ifdef CONFIG_MEMORY_ISOLATION 235 "Isolate", 236 #endif 237 }; 238 239 compound_page_dtor * const compound_page_dtors[] = { 240 NULL, 241 free_compound_page, 242 #ifdef CONFIG_HUGETLB_PAGE 243 free_huge_page, 244 #endif 245 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 246 free_transhuge_page, 247 #endif 248 }; 249 250 int min_free_kbytes = 1024; 251 int user_min_free_kbytes = -1; 252 int watermark_scale_factor = 10; 253 254 static unsigned long __meminitdata nr_kernel_pages; 255 static unsigned long __meminitdata nr_all_pages; 256 static unsigned long __meminitdata dma_reserve; 257 258 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 259 static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES]; 260 static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES]; 261 static unsigned long __initdata required_kernelcore; 262 static unsigned long __initdata required_movablecore; 263 static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES]; 264 static bool mirrored_kernelcore; 265 266 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 267 int movable_zone; 268 EXPORT_SYMBOL(movable_zone); 269 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 270 271 #if MAX_NUMNODES > 1 272 int nr_node_ids __read_mostly = MAX_NUMNODES; 273 int nr_online_nodes __read_mostly = 1; 274 EXPORT_SYMBOL(nr_node_ids); 275 EXPORT_SYMBOL(nr_online_nodes); 276 #endif 277 278 int page_group_by_mobility_disabled __read_mostly; 279 280 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 281 static inline void reset_deferred_meminit(pg_data_t *pgdat) 282 { 283 pgdat->first_deferred_pfn = ULONG_MAX; 284 } 285 286 /* Returns true if the struct page for the pfn is uninitialised */ 287 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 288 { 289 int nid = early_pfn_to_nid(pfn); 290 291 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 292 return true; 293 294 return false; 295 } 296 297 static inline bool early_page_nid_uninitialised(unsigned long pfn, int nid) 298 { 299 if (pfn >= NODE_DATA(nid)->first_deferred_pfn) 300 return true; 301 302 return false; 303 } 304 305 /* 306 * Returns false when the remaining initialisation should be deferred until 307 * later in the boot cycle when it can be parallelised. 308 */ 309 static inline bool update_defer_init(pg_data_t *pgdat, 310 unsigned long pfn, unsigned long zone_end, 311 unsigned long *nr_initialised) 312 { 313 unsigned long max_initialise; 314 315 /* Always populate low zones for address-contrained allocations */ 316 if (zone_end < pgdat_end_pfn(pgdat)) 317 return true; 318 /* 319 * Initialise at least 2G of a node but also take into account that 320 * two large system hashes that can take up 1GB for 0.25TB/node. 321 */ 322 max_initialise = max(2UL << (30 - PAGE_SHIFT), 323 (pgdat->node_spanned_pages >> 8)); 324 325 (*nr_initialised)++; 326 if ((*nr_initialised > max_initialise) && 327 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 328 pgdat->first_deferred_pfn = pfn; 329 return false; 330 } 331 332 return true; 333 } 334 #else 335 static inline void reset_deferred_meminit(pg_data_t *pgdat) 336 { 337 } 338 339 static inline bool early_page_uninitialised(unsigned long pfn) 340 { 341 return false; 342 } 343 344 static inline bool early_page_nid_uninitialised(unsigned long pfn, int nid) 345 { 346 return false; 347 } 348 349 static inline bool update_defer_init(pg_data_t *pgdat, 350 unsigned long pfn, unsigned long zone_end, 351 unsigned long *nr_initialised) 352 { 353 return true; 354 } 355 #endif 356 357 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 358 static inline unsigned long *get_pageblock_bitmap(struct page *page, 359 unsigned long pfn) 360 { 361 #ifdef CONFIG_SPARSEMEM 362 return __pfn_to_section(pfn)->pageblock_flags; 363 #else 364 return page_zone(page)->pageblock_flags; 365 #endif /* CONFIG_SPARSEMEM */ 366 } 367 368 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn) 369 { 370 #ifdef CONFIG_SPARSEMEM 371 pfn &= (PAGES_PER_SECTION-1); 372 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 373 #else 374 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 375 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 376 #endif /* CONFIG_SPARSEMEM */ 377 } 378 379 /** 380 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 381 * @page: The page within the block of interest 382 * @pfn: The target page frame number 383 * @end_bitidx: The last bit of interest to retrieve 384 * @mask: mask of bits that the caller is interested in 385 * 386 * Return: pageblock_bits flags 387 */ 388 static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page, 389 unsigned long pfn, 390 unsigned long end_bitidx, 391 unsigned long mask) 392 { 393 unsigned long *bitmap; 394 unsigned long bitidx, word_bitidx; 395 unsigned long word; 396 397 bitmap = get_pageblock_bitmap(page, pfn); 398 bitidx = pfn_to_bitidx(page, pfn); 399 word_bitidx = bitidx / BITS_PER_LONG; 400 bitidx &= (BITS_PER_LONG-1); 401 402 word = bitmap[word_bitidx]; 403 bitidx += end_bitidx; 404 return (word >> (BITS_PER_LONG - bitidx - 1)) & mask; 405 } 406 407 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, 408 unsigned long end_bitidx, 409 unsigned long mask) 410 { 411 return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask); 412 } 413 414 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn) 415 { 416 return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK); 417 } 418 419 /** 420 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 421 * @page: The page within the block of interest 422 * @flags: The flags to set 423 * @pfn: The target page frame number 424 * @end_bitidx: The last bit of interest 425 * @mask: mask of bits that the caller is interested in 426 */ 427 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 428 unsigned long pfn, 429 unsigned long end_bitidx, 430 unsigned long mask) 431 { 432 unsigned long *bitmap; 433 unsigned long bitidx, word_bitidx; 434 unsigned long old_word, word; 435 436 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 437 438 bitmap = get_pageblock_bitmap(page, pfn); 439 bitidx = pfn_to_bitidx(page, pfn); 440 word_bitidx = bitidx / BITS_PER_LONG; 441 bitidx &= (BITS_PER_LONG-1); 442 443 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 444 445 bitidx += end_bitidx; 446 mask <<= (BITS_PER_LONG - bitidx - 1); 447 flags <<= (BITS_PER_LONG - bitidx - 1); 448 449 word = READ_ONCE(bitmap[word_bitidx]); 450 for (;;) { 451 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 452 if (word == old_word) 453 break; 454 word = old_word; 455 } 456 } 457 458 void set_pageblock_migratetype(struct page *page, int migratetype) 459 { 460 if (unlikely(page_group_by_mobility_disabled && 461 migratetype < MIGRATE_PCPTYPES)) 462 migratetype = MIGRATE_UNMOVABLE; 463 464 set_pageblock_flags_group(page, (unsigned long)migratetype, 465 PB_migrate, PB_migrate_end); 466 } 467 468 #ifdef CONFIG_DEBUG_VM 469 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 470 { 471 int ret = 0; 472 unsigned seq; 473 unsigned long pfn = page_to_pfn(page); 474 unsigned long sp, start_pfn; 475 476 do { 477 seq = zone_span_seqbegin(zone); 478 start_pfn = zone->zone_start_pfn; 479 sp = zone->spanned_pages; 480 if (!zone_spans_pfn(zone, pfn)) 481 ret = 1; 482 } while (zone_span_seqretry(zone, seq)); 483 484 if (ret) 485 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 486 pfn, zone_to_nid(zone), zone->name, 487 start_pfn, start_pfn + sp); 488 489 return ret; 490 } 491 492 static int page_is_consistent(struct zone *zone, struct page *page) 493 { 494 if (!pfn_valid_within(page_to_pfn(page))) 495 return 0; 496 if (zone != page_zone(page)) 497 return 0; 498 499 return 1; 500 } 501 /* 502 * Temporary debugging check for pages not lying within a given zone. 503 */ 504 static int bad_range(struct zone *zone, struct page *page) 505 { 506 if (page_outside_zone_boundaries(zone, page)) 507 return 1; 508 if (!page_is_consistent(zone, page)) 509 return 1; 510 511 return 0; 512 } 513 #else 514 static inline int bad_range(struct zone *zone, struct page *page) 515 { 516 return 0; 517 } 518 #endif 519 520 static void bad_page(struct page *page, const char *reason, 521 unsigned long bad_flags) 522 { 523 static unsigned long resume; 524 static unsigned long nr_shown; 525 static unsigned long nr_unshown; 526 527 /* 528 * Allow a burst of 60 reports, then keep quiet for that minute; 529 * or allow a steady drip of one report per second. 530 */ 531 if (nr_shown == 60) { 532 if (time_before(jiffies, resume)) { 533 nr_unshown++; 534 goto out; 535 } 536 if (nr_unshown) { 537 pr_alert( 538 "BUG: Bad page state: %lu messages suppressed\n", 539 nr_unshown); 540 nr_unshown = 0; 541 } 542 nr_shown = 0; 543 } 544 if (nr_shown++ == 0) 545 resume = jiffies + 60 * HZ; 546 547 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 548 current->comm, page_to_pfn(page)); 549 __dump_page(page, reason); 550 bad_flags &= page->flags; 551 if (bad_flags) 552 pr_alert("bad because of flags: %#lx(%pGp)\n", 553 bad_flags, &bad_flags); 554 dump_page_owner(page); 555 556 print_modules(); 557 dump_stack(); 558 out: 559 /* Leave bad fields for debug, except PageBuddy could make trouble */ 560 page_mapcount_reset(page); /* remove PageBuddy */ 561 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 562 } 563 564 /* 565 * Higher-order pages are called "compound pages". They are structured thusly: 566 * 567 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 568 * 569 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 570 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 571 * 572 * The first tail page's ->compound_dtor holds the offset in array of compound 573 * page destructors. See compound_page_dtors. 574 * 575 * The first tail page's ->compound_order holds the order of allocation. 576 * This usage means that zero-order pages may not be compound. 577 */ 578 579 void free_compound_page(struct page *page) 580 { 581 __free_pages_ok(page, compound_order(page)); 582 } 583 584 void prep_compound_page(struct page *page, unsigned int order) 585 { 586 int i; 587 int nr_pages = 1 << order; 588 589 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 590 set_compound_order(page, order); 591 __SetPageHead(page); 592 for (i = 1; i < nr_pages; i++) { 593 struct page *p = page + i; 594 set_page_count(p, 0); 595 p->mapping = TAIL_MAPPING; 596 set_compound_head(p, page); 597 } 598 atomic_set(compound_mapcount_ptr(page), -1); 599 } 600 601 #ifdef CONFIG_DEBUG_PAGEALLOC 602 unsigned int _debug_guardpage_minorder; 603 bool _debug_pagealloc_enabled __read_mostly 604 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 605 EXPORT_SYMBOL(_debug_pagealloc_enabled); 606 bool _debug_guardpage_enabled __read_mostly; 607 608 static int __init early_debug_pagealloc(char *buf) 609 { 610 if (!buf) 611 return -EINVAL; 612 return kstrtobool(buf, &_debug_pagealloc_enabled); 613 } 614 early_param("debug_pagealloc", early_debug_pagealloc); 615 616 static bool need_debug_guardpage(void) 617 { 618 /* If we don't use debug_pagealloc, we don't need guard page */ 619 if (!debug_pagealloc_enabled()) 620 return false; 621 622 return true; 623 } 624 625 static void init_debug_guardpage(void) 626 { 627 if (!debug_pagealloc_enabled()) 628 return; 629 630 _debug_guardpage_enabled = true; 631 } 632 633 struct page_ext_operations debug_guardpage_ops = { 634 .need = need_debug_guardpage, 635 .init = init_debug_guardpage, 636 }; 637 638 static int __init debug_guardpage_minorder_setup(char *buf) 639 { 640 unsigned long res; 641 642 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 643 pr_err("Bad debug_guardpage_minorder value\n"); 644 return 0; 645 } 646 _debug_guardpage_minorder = res; 647 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 648 return 0; 649 } 650 __setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup); 651 652 static inline void set_page_guard(struct zone *zone, struct page *page, 653 unsigned int order, int migratetype) 654 { 655 struct page_ext *page_ext; 656 657 if (!debug_guardpage_enabled()) 658 return; 659 660 page_ext = lookup_page_ext(page); 661 if (unlikely(!page_ext)) 662 return; 663 664 __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); 665 666 INIT_LIST_HEAD(&page->lru); 667 set_page_private(page, order); 668 /* Guard pages are not available for any usage */ 669 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 670 } 671 672 static inline void clear_page_guard(struct zone *zone, struct page *page, 673 unsigned int order, int migratetype) 674 { 675 struct page_ext *page_ext; 676 677 if (!debug_guardpage_enabled()) 678 return; 679 680 page_ext = lookup_page_ext(page); 681 if (unlikely(!page_ext)) 682 return; 683 684 __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); 685 686 set_page_private(page, 0); 687 if (!is_migrate_isolate(migratetype)) 688 __mod_zone_freepage_state(zone, (1 << order), migratetype); 689 } 690 #else 691 struct page_ext_operations debug_guardpage_ops = { NULL, }; 692 static inline void set_page_guard(struct zone *zone, struct page *page, 693 unsigned int order, int migratetype) {} 694 static inline void clear_page_guard(struct zone *zone, struct page *page, 695 unsigned int order, int migratetype) {} 696 #endif 697 698 static inline void set_page_order(struct page *page, unsigned int order) 699 { 700 set_page_private(page, order); 701 __SetPageBuddy(page); 702 } 703 704 static inline void rmv_page_order(struct page *page) 705 { 706 __ClearPageBuddy(page); 707 set_page_private(page, 0); 708 } 709 710 /* 711 * This function checks whether a page is free && is the buddy 712 * we can do coalesce a page and its buddy if 713 * (a) the buddy is not in a hole && 714 * (b) the buddy is in the buddy system && 715 * (c) a page and its buddy have the same order && 716 * (d) a page and its buddy are in the same zone. 717 * 718 * For recording whether a page is in the buddy system, we set ->_mapcount 719 * PAGE_BUDDY_MAPCOUNT_VALUE. 720 * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is 721 * serialized by zone->lock. 722 * 723 * For recording page's order, we use page_private(page). 724 */ 725 static inline int page_is_buddy(struct page *page, struct page *buddy, 726 unsigned int order) 727 { 728 if (!pfn_valid_within(page_to_pfn(buddy))) 729 return 0; 730 731 if (page_is_guard(buddy) && page_order(buddy) == order) { 732 if (page_zone_id(page) != page_zone_id(buddy)) 733 return 0; 734 735 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 736 737 return 1; 738 } 739 740 if (PageBuddy(buddy) && page_order(buddy) == order) { 741 /* 742 * zone check is done late to avoid uselessly 743 * calculating zone/node ids for pages that could 744 * never merge. 745 */ 746 if (page_zone_id(page) != page_zone_id(buddy)) 747 return 0; 748 749 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 750 751 return 1; 752 } 753 return 0; 754 } 755 756 /* 757 * Freeing function for a buddy system allocator. 758 * 759 * The concept of a buddy system is to maintain direct-mapped table 760 * (containing bit values) for memory blocks of various "orders". 761 * The bottom level table contains the map for the smallest allocatable 762 * units of memory (here, pages), and each level above it describes 763 * pairs of units from the levels below, hence, "buddies". 764 * At a high level, all that happens here is marking the table entry 765 * at the bottom level available, and propagating the changes upward 766 * as necessary, plus some accounting needed to play nicely with other 767 * parts of the VM system. 768 * At each level, we keep a list of pages, which are heads of continuous 769 * free pages of length of (1 << order) and marked with _mapcount 770 * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page) 771 * field. 772 * So when we are allocating or freeing one, we can derive the state of the 773 * other. That is, if we allocate a small block, and both were 774 * free, the remainder of the region must be split into blocks. 775 * If a block is freed, and its buddy is also free, then this 776 * triggers coalescing into a block of larger size. 777 * 778 * -- nyc 779 */ 780 781 static inline void __free_one_page(struct page *page, 782 unsigned long pfn, 783 struct zone *zone, unsigned int order, 784 int migratetype) 785 { 786 unsigned long page_idx; 787 unsigned long combined_idx; 788 unsigned long uninitialized_var(buddy_idx); 789 struct page *buddy; 790 unsigned int max_order; 791 792 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1); 793 794 VM_BUG_ON(!zone_is_initialized(zone)); 795 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 796 797 VM_BUG_ON(migratetype == -1); 798 if (likely(!is_migrate_isolate(migratetype))) 799 __mod_zone_freepage_state(zone, 1 << order, migratetype); 800 801 page_idx = pfn & ((1 << MAX_ORDER) - 1); 802 803 VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page); 804 VM_BUG_ON_PAGE(bad_range(zone, page), page); 805 806 continue_merging: 807 while (order < max_order - 1) { 808 buddy_idx = __find_buddy_index(page_idx, order); 809 buddy = page + (buddy_idx - page_idx); 810 if (!page_is_buddy(page, buddy, order)) 811 goto done_merging; 812 /* 813 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 814 * merge with it and move up one order. 815 */ 816 if (page_is_guard(buddy)) { 817 clear_page_guard(zone, buddy, order, migratetype); 818 } else { 819 list_del(&buddy->lru); 820 zone->free_area[order].nr_free--; 821 rmv_page_order(buddy); 822 } 823 combined_idx = buddy_idx & page_idx; 824 page = page + (combined_idx - page_idx); 825 page_idx = combined_idx; 826 order++; 827 } 828 if (max_order < MAX_ORDER) { 829 /* If we are here, it means order is >= pageblock_order. 830 * We want to prevent merge between freepages on isolate 831 * pageblock and normal pageblock. Without this, pageblock 832 * isolation could cause incorrect freepage or CMA accounting. 833 * 834 * We don't want to hit this code for the more frequent 835 * low-order merging. 836 */ 837 if (unlikely(has_isolate_pageblock(zone))) { 838 int buddy_mt; 839 840 buddy_idx = __find_buddy_index(page_idx, order); 841 buddy = page + (buddy_idx - page_idx); 842 buddy_mt = get_pageblock_migratetype(buddy); 843 844 if (migratetype != buddy_mt 845 && (is_migrate_isolate(migratetype) || 846 is_migrate_isolate(buddy_mt))) 847 goto done_merging; 848 } 849 max_order++; 850 goto continue_merging; 851 } 852 853 done_merging: 854 set_page_order(page, order); 855 856 /* 857 * If this is not the largest possible page, check if the buddy 858 * of the next-highest order is free. If it is, it's possible 859 * that pages are being freed that will coalesce soon. In case, 860 * that is happening, add the free page to the tail of the list 861 * so it's less likely to be used soon and more likely to be merged 862 * as a higher order page 863 */ 864 if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) { 865 struct page *higher_page, *higher_buddy; 866 combined_idx = buddy_idx & page_idx; 867 higher_page = page + (combined_idx - page_idx); 868 buddy_idx = __find_buddy_index(combined_idx, order + 1); 869 higher_buddy = higher_page + (buddy_idx - combined_idx); 870 if (page_is_buddy(higher_page, higher_buddy, order + 1)) { 871 list_add_tail(&page->lru, 872 &zone->free_area[order].free_list[migratetype]); 873 goto out; 874 } 875 } 876 877 list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); 878 out: 879 zone->free_area[order].nr_free++; 880 } 881 882 /* 883 * A bad page could be due to a number of fields. Instead of multiple branches, 884 * try and check multiple fields with one check. The caller must do a detailed 885 * check if necessary. 886 */ 887 static inline bool page_expected_state(struct page *page, 888 unsigned long check_flags) 889 { 890 if (unlikely(atomic_read(&page->_mapcount) != -1)) 891 return false; 892 893 if (unlikely((unsigned long)page->mapping | 894 page_ref_count(page) | 895 #ifdef CONFIG_MEMCG 896 (unsigned long)page->mem_cgroup | 897 #endif 898 (page->flags & check_flags))) 899 return false; 900 901 return true; 902 } 903 904 static void free_pages_check_bad(struct page *page) 905 { 906 const char *bad_reason; 907 unsigned long bad_flags; 908 909 bad_reason = NULL; 910 bad_flags = 0; 911 912 if (unlikely(atomic_read(&page->_mapcount) != -1)) 913 bad_reason = "nonzero mapcount"; 914 if (unlikely(page->mapping != NULL)) 915 bad_reason = "non-NULL mapping"; 916 if (unlikely(page_ref_count(page) != 0)) 917 bad_reason = "nonzero _refcount"; 918 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) { 919 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 920 bad_flags = PAGE_FLAGS_CHECK_AT_FREE; 921 } 922 #ifdef CONFIG_MEMCG 923 if (unlikely(page->mem_cgroup)) 924 bad_reason = "page still charged to cgroup"; 925 #endif 926 bad_page(page, bad_reason, bad_flags); 927 } 928 929 static inline int free_pages_check(struct page *page) 930 { 931 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 932 return 0; 933 934 /* Something has gone sideways, find it */ 935 free_pages_check_bad(page); 936 return 1; 937 } 938 939 static int free_tail_pages_check(struct page *head_page, struct page *page) 940 { 941 int ret = 1; 942 943 /* 944 * We rely page->lru.next never has bit 0 set, unless the page 945 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 946 */ 947 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 948 949 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 950 ret = 0; 951 goto out; 952 } 953 switch (page - head_page) { 954 case 1: 955 /* the first tail page: ->mapping is compound_mapcount() */ 956 if (unlikely(compound_mapcount(page))) { 957 bad_page(page, "nonzero compound_mapcount", 0); 958 goto out; 959 } 960 break; 961 case 2: 962 /* 963 * the second tail page: ->mapping is 964 * page_deferred_list().next -- ignore value. 965 */ 966 break; 967 default: 968 if (page->mapping != TAIL_MAPPING) { 969 bad_page(page, "corrupted mapping in tail page", 0); 970 goto out; 971 } 972 break; 973 } 974 if (unlikely(!PageTail(page))) { 975 bad_page(page, "PageTail not set", 0); 976 goto out; 977 } 978 if (unlikely(compound_head(page) != head_page)) { 979 bad_page(page, "compound_head not consistent", 0); 980 goto out; 981 } 982 ret = 0; 983 out: 984 page->mapping = NULL; 985 clear_compound_head(page); 986 return ret; 987 } 988 989 static __always_inline bool free_pages_prepare(struct page *page, 990 unsigned int order, bool check_free) 991 { 992 int bad = 0; 993 994 VM_BUG_ON_PAGE(PageTail(page), page); 995 996 trace_mm_page_free(page, order); 997 kmemcheck_free_shadow(page, order); 998 999 /* 1000 * Check tail pages before head page information is cleared to 1001 * avoid checking PageCompound for order-0 pages. 1002 */ 1003 if (unlikely(order)) { 1004 bool compound = PageCompound(page); 1005 int i; 1006 1007 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1008 1009 for (i = 1; i < (1 << order); i++) { 1010 if (compound) 1011 bad += free_tail_pages_check(page, page + i); 1012 if (unlikely(free_pages_check(page + i))) { 1013 bad++; 1014 continue; 1015 } 1016 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1017 } 1018 } 1019 if (PageAnonHead(page)) 1020 page->mapping = NULL; 1021 if (check_free) 1022 bad += free_pages_check(page); 1023 if (bad) 1024 return false; 1025 1026 page_cpupid_reset_last(page); 1027 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1028 reset_page_owner(page, order); 1029 1030 if (!PageHighMem(page)) { 1031 debug_check_no_locks_freed(page_address(page), 1032 PAGE_SIZE << order); 1033 debug_check_no_obj_freed(page_address(page), 1034 PAGE_SIZE << order); 1035 } 1036 arch_free_page(page, order); 1037 kernel_poison_pages(page, 1 << order, 0); 1038 kernel_map_pages(page, 1 << order, 0); 1039 kasan_free_pages(page, order); 1040 1041 return true; 1042 } 1043 1044 #ifdef CONFIG_DEBUG_VM 1045 static inline bool free_pcp_prepare(struct page *page) 1046 { 1047 return free_pages_prepare(page, 0, true); 1048 } 1049 1050 static inline bool bulkfree_pcp_prepare(struct page *page) 1051 { 1052 return false; 1053 } 1054 #else 1055 static bool free_pcp_prepare(struct page *page) 1056 { 1057 return free_pages_prepare(page, 0, false); 1058 } 1059 1060 static bool bulkfree_pcp_prepare(struct page *page) 1061 { 1062 return free_pages_check(page); 1063 } 1064 #endif /* CONFIG_DEBUG_VM */ 1065 1066 /* 1067 * Frees a number of pages from the PCP lists 1068 * Assumes all pages on list are in same zone, and of same order. 1069 * count is the number of pages to free. 1070 * 1071 * If the zone was previously in an "all pages pinned" state then look to 1072 * see if this freeing clears that state. 1073 * 1074 * And clear the zone's pages_scanned counter, to hold off the "all pages are 1075 * pinned" detection logic. 1076 */ 1077 static void free_pcppages_bulk(struct zone *zone, int count, 1078 struct per_cpu_pages *pcp) 1079 { 1080 int migratetype = 0; 1081 int batch_free = 0; 1082 unsigned long nr_scanned; 1083 bool isolated_pageblocks; 1084 1085 spin_lock(&zone->lock); 1086 isolated_pageblocks = has_isolate_pageblock(zone); 1087 nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); 1088 if (nr_scanned) 1089 __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); 1090 1091 while (count) { 1092 struct page *page; 1093 struct list_head *list; 1094 1095 /* 1096 * Remove pages from lists in a round-robin fashion. A 1097 * batch_free count is maintained that is incremented when an 1098 * empty list is encountered. This is so more pages are freed 1099 * off fuller lists instead of spinning excessively around empty 1100 * lists 1101 */ 1102 do { 1103 batch_free++; 1104 if (++migratetype == MIGRATE_PCPTYPES) 1105 migratetype = 0; 1106 list = &pcp->lists[migratetype]; 1107 } while (list_empty(list)); 1108 1109 /* This is the only non-empty list. Free them all. */ 1110 if (batch_free == MIGRATE_PCPTYPES) 1111 batch_free = count; 1112 1113 do { 1114 int mt; /* migratetype of the to-be-freed page */ 1115 1116 page = list_last_entry(list, struct page, lru); 1117 /* must delete as __free_one_page list manipulates */ 1118 list_del(&page->lru); 1119 1120 mt = get_pcppage_migratetype(page); 1121 /* MIGRATE_ISOLATE page should not go to pcplists */ 1122 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1123 /* Pageblock could have been isolated meanwhile */ 1124 if (unlikely(isolated_pageblocks)) 1125 mt = get_pageblock_migratetype(page); 1126 1127 if (bulkfree_pcp_prepare(page)) 1128 continue; 1129 1130 __free_one_page(page, page_to_pfn(page), zone, 0, mt); 1131 trace_mm_page_pcpu_drain(page, 0, mt); 1132 } while (--count && --batch_free && !list_empty(list)); 1133 } 1134 spin_unlock(&zone->lock); 1135 } 1136 1137 static void free_one_page(struct zone *zone, 1138 struct page *page, unsigned long pfn, 1139 unsigned int order, 1140 int migratetype) 1141 { 1142 unsigned long nr_scanned; 1143 spin_lock(&zone->lock); 1144 nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); 1145 if (nr_scanned) 1146 __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); 1147 1148 if (unlikely(has_isolate_pageblock(zone) || 1149 is_migrate_isolate(migratetype))) { 1150 migratetype = get_pfnblock_migratetype(page, pfn); 1151 } 1152 __free_one_page(page, pfn, zone, order, migratetype); 1153 spin_unlock(&zone->lock); 1154 } 1155 1156 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1157 unsigned long zone, int nid) 1158 { 1159 set_page_links(page, zone, nid, pfn); 1160 init_page_count(page); 1161 page_mapcount_reset(page); 1162 page_cpupid_reset_last(page); 1163 1164 INIT_LIST_HEAD(&page->lru); 1165 #ifdef WANT_PAGE_VIRTUAL 1166 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1167 if (!is_highmem_idx(zone)) 1168 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1169 #endif 1170 } 1171 1172 static void __meminit __init_single_pfn(unsigned long pfn, unsigned long zone, 1173 int nid) 1174 { 1175 return __init_single_page(pfn_to_page(pfn), pfn, zone, nid); 1176 } 1177 1178 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1179 static void init_reserved_page(unsigned long pfn) 1180 { 1181 pg_data_t *pgdat; 1182 int nid, zid; 1183 1184 if (!early_page_uninitialised(pfn)) 1185 return; 1186 1187 nid = early_pfn_to_nid(pfn); 1188 pgdat = NODE_DATA(nid); 1189 1190 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1191 struct zone *zone = &pgdat->node_zones[zid]; 1192 1193 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) 1194 break; 1195 } 1196 __init_single_pfn(pfn, zid, nid); 1197 } 1198 #else 1199 static inline void init_reserved_page(unsigned long pfn) 1200 { 1201 } 1202 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1203 1204 /* 1205 * Initialised pages do not have PageReserved set. This function is 1206 * called for each range allocated by the bootmem allocator and 1207 * marks the pages PageReserved. The remaining valid pages are later 1208 * sent to the buddy page allocator. 1209 */ 1210 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1211 { 1212 unsigned long start_pfn = PFN_DOWN(start); 1213 unsigned long end_pfn = PFN_UP(end); 1214 1215 for (; start_pfn < end_pfn; start_pfn++) { 1216 if (pfn_valid(start_pfn)) { 1217 struct page *page = pfn_to_page(start_pfn); 1218 1219 init_reserved_page(start_pfn); 1220 1221 /* Avoid false-positive PageTail() */ 1222 INIT_LIST_HEAD(&page->lru); 1223 1224 SetPageReserved(page); 1225 } 1226 } 1227 } 1228 1229 static void __free_pages_ok(struct page *page, unsigned int order) 1230 { 1231 unsigned long flags; 1232 int migratetype; 1233 unsigned long pfn = page_to_pfn(page); 1234 1235 if (!free_pages_prepare(page, order, true)) 1236 return; 1237 1238 migratetype = get_pfnblock_migratetype(page, pfn); 1239 local_irq_save(flags); 1240 __count_vm_events(PGFREE, 1 << order); 1241 free_one_page(page_zone(page), page, pfn, order, migratetype); 1242 local_irq_restore(flags); 1243 } 1244 1245 static void __init __free_pages_boot_core(struct page *page, unsigned int order) 1246 { 1247 unsigned int nr_pages = 1 << order; 1248 struct page *p = page; 1249 unsigned int loop; 1250 1251 prefetchw(p); 1252 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1253 prefetchw(p + 1); 1254 __ClearPageReserved(p); 1255 set_page_count(p, 0); 1256 } 1257 __ClearPageReserved(p); 1258 set_page_count(p, 0); 1259 1260 page_zone(page)->managed_pages += nr_pages; 1261 set_page_refcounted(page); 1262 __free_pages(page, order); 1263 } 1264 1265 #if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \ 1266 defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) 1267 1268 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1269 1270 int __meminit early_pfn_to_nid(unsigned long pfn) 1271 { 1272 static DEFINE_SPINLOCK(early_pfn_lock); 1273 int nid; 1274 1275 spin_lock(&early_pfn_lock); 1276 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1277 if (nid < 0) 1278 nid = first_online_node; 1279 spin_unlock(&early_pfn_lock); 1280 1281 return nid; 1282 } 1283 #endif 1284 1285 #ifdef CONFIG_NODES_SPAN_OTHER_NODES 1286 static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node, 1287 struct mminit_pfnnid_cache *state) 1288 { 1289 int nid; 1290 1291 nid = __early_pfn_to_nid(pfn, state); 1292 if (nid >= 0 && nid != node) 1293 return false; 1294 return true; 1295 } 1296 1297 /* Only safe to use early in boot when initialisation is single-threaded */ 1298 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1299 { 1300 return meminit_pfn_in_nid(pfn, node, &early_pfnnid_cache); 1301 } 1302 1303 #else 1304 1305 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1306 { 1307 return true; 1308 } 1309 static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node, 1310 struct mminit_pfnnid_cache *state) 1311 { 1312 return true; 1313 } 1314 #endif 1315 1316 1317 void __init __free_pages_bootmem(struct page *page, unsigned long pfn, 1318 unsigned int order) 1319 { 1320 if (early_page_uninitialised(pfn)) 1321 return; 1322 return __free_pages_boot_core(page, order); 1323 } 1324 1325 /* 1326 * Check that the whole (or subset of) a pageblock given by the interval of 1327 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1328 * with the migration of free compaction scanner. The scanners then need to 1329 * use only pfn_valid_within() check for arches that allow holes within 1330 * pageblocks. 1331 * 1332 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1333 * 1334 * It's possible on some configurations to have a setup like node0 node1 node0 1335 * i.e. it's possible that all pages within a zones range of pages do not 1336 * belong to a single zone. We assume that a border between node0 and node1 1337 * can occur within a single pageblock, but not a node0 node1 node0 1338 * interleaving within a single pageblock. It is therefore sufficient to check 1339 * the first and last page of a pageblock and avoid checking each individual 1340 * page in a pageblock. 1341 */ 1342 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1343 unsigned long end_pfn, struct zone *zone) 1344 { 1345 struct page *start_page; 1346 struct page *end_page; 1347 1348 /* end_pfn is one past the range we are checking */ 1349 end_pfn--; 1350 1351 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1352 return NULL; 1353 1354 start_page = pfn_to_page(start_pfn); 1355 1356 if (page_zone(start_page) != zone) 1357 return NULL; 1358 1359 end_page = pfn_to_page(end_pfn); 1360 1361 /* This gives a shorter code than deriving page_zone(end_page) */ 1362 if (page_zone_id(start_page) != page_zone_id(end_page)) 1363 return NULL; 1364 1365 return start_page; 1366 } 1367 1368 void set_zone_contiguous(struct zone *zone) 1369 { 1370 unsigned long block_start_pfn = zone->zone_start_pfn; 1371 unsigned long block_end_pfn; 1372 1373 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1374 for (; block_start_pfn < zone_end_pfn(zone); 1375 block_start_pfn = block_end_pfn, 1376 block_end_pfn += pageblock_nr_pages) { 1377 1378 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1379 1380 if (!__pageblock_pfn_to_page(block_start_pfn, 1381 block_end_pfn, zone)) 1382 return; 1383 } 1384 1385 /* We confirm that there is no hole */ 1386 zone->contiguous = true; 1387 } 1388 1389 void clear_zone_contiguous(struct zone *zone) 1390 { 1391 zone->contiguous = false; 1392 } 1393 1394 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1395 static void __init deferred_free_range(struct page *page, 1396 unsigned long pfn, int nr_pages) 1397 { 1398 int i; 1399 1400 if (!page) 1401 return; 1402 1403 /* Free a large naturally-aligned chunk if possible */ 1404 if (nr_pages == MAX_ORDER_NR_PAGES && 1405 (pfn & (MAX_ORDER_NR_PAGES-1)) == 0) { 1406 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1407 __free_pages_boot_core(page, MAX_ORDER-1); 1408 return; 1409 } 1410 1411 for (i = 0; i < nr_pages; i++, page++) 1412 __free_pages_boot_core(page, 0); 1413 } 1414 1415 /* Completion tracking for deferred_init_memmap() threads */ 1416 static atomic_t pgdat_init_n_undone __initdata; 1417 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1418 1419 static inline void __init pgdat_init_report_one_done(void) 1420 { 1421 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1422 complete(&pgdat_init_all_done_comp); 1423 } 1424 1425 /* Initialise remaining memory on a node */ 1426 static int __init deferred_init_memmap(void *data) 1427 { 1428 pg_data_t *pgdat = data; 1429 int nid = pgdat->node_id; 1430 struct mminit_pfnnid_cache nid_init_state = { }; 1431 unsigned long start = jiffies; 1432 unsigned long nr_pages = 0; 1433 unsigned long walk_start, walk_end; 1434 int i, zid; 1435 struct zone *zone; 1436 unsigned long first_init_pfn = pgdat->first_deferred_pfn; 1437 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 1438 1439 if (first_init_pfn == ULONG_MAX) { 1440 pgdat_init_report_one_done(); 1441 return 0; 1442 } 1443 1444 /* Bind memory initialisation thread to a local node if possible */ 1445 if (!cpumask_empty(cpumask)) 1446 set_cpus_allowed_ptr(current, cpumask); 1447 1448 /* Sanity check boundaries */ 1449 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 1450 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 1451 pgdat->first_deferred_pfn = ULONG_MAX; 1452 1453 /* Only the highest zone is deferred so find it */ 1454 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1455 zone = pgdat->node_zones + zid; 1456 if (first_init_pfn < zone_end_pfn(zone)) 1457 break; 1458 } 1459 1460 for_each_mem_pfn_range(i, nid, &walk_start, &walk_end, NULL) { 1461 unsigned long pfn, end_pfn; 1462 struct page *page = NULL; 1463 struct page *free_base_page = NULL; 1464 unsigned long free_base_pfn = 0; 1465 int nr_to_free = 0; 1466 1467 end_pfn = min(walk_end, zone_end_pfn(zone)); 1468 pfn = first_init_pfn; 1469 if (pfn < walk_start) 1470 pfn = walk_start; 1471 if (pfn < zone->zone_start_pfn) 1472 pfn = zone->zone_start_pfn; 1473 1474 for (; pfn < end_pfn; pfn++) { 1475 if (!pfn_valid_within(pfn)) 1476 goto free_range; 1477 1478 /* 1479 * Ensure pfn_valid is checked every 1480 * MAX_ORDER_NR_PAGES for memory holes 1481 */ 1482 if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0) { 1483 if (!pfn_valid(pfn)) { 1484 page = NULL; 1485 goto free_range; 1486 } 1487 } 1488 1489 if (!meminit_pfn_in_nid(pfn, nid, &nid_init_state)) { 1490 page = NULL; 1491 goto free_range; 1492 } 1493 1494 /* Minimise pfn page lookups and scheduler checks */ 1495 if (page && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) { 1496 page++; 1497 } else { 1498 nr_pages += nr_to_free; 1499 deferred_free_range(free_base_page, 1500 free_base_pfn, nr_to_free); 1501 free_base_page = NULL; 1502 free_base_pfn = nr_to_free = 0; 1503 1504 page = pfn_to_page(pfn); 1505 cond_resched(); 1506 } 1507 1508 if (page->flags) { 1509 VM_BUG_ON(page_zone(page) != zone); 1510 goto free_range; 1511 } 1512 1513 __init_single_page(page, pfn, zid, nid); 1514 if (!free_base_page) { 1515 free_base_page = page; 1516 free_base_pfn = pfn; 1517 nr_to_free = 0; 1518 } 1519 nr_to_free++; 1520 1521 /* Where possible, batch up pages for a single free */ 1522 continue; 1523 free_range: 1524 /* Free the current block of pages to allocator */ 1525 nr_pages += nr_to_free; 1526 deferred_free_range(free_base_page, free_base_pfn, 1527 nr_to_free); 1528 free_base_page = NULL; 1529 free_base_pfn = nr_to_free = 0; 1530 } 1531 1532 first_init_pfn = max(end_pfn, first_init_pfn); 1533 } 1534 1535 /* Sanity check that the next zone really is unpopulated */ 1536 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 1537 1538 pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages, 1539 jiffies_to_msecs(jiffies - start)); 1540 1541 pgdat_init_report_one_done(); 1542 return 0; 1543 } 1544 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1545 1546 void __init page_alloc_init_late(void) 1547 { 1548 struct zone *zone; 1549 1550 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1551 int nid; 1552 1553 /* There will be num_node_state(N_MEMORY) threads */ 1554 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 1555 for_each_node_state(nid, N_MEMORY) { 1556 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 1557 } 1558 1559 /* Block until all are initialised */ 1560 wait_for_completion(&pgdat_init_all_done_comp); 1561 1562 /* Reinit limits that are based on free pages after the kernel is up */ 1563 files_maxfiles_init(); 1564 #endif 1565 1566 for_each_populated_zone(zone) 1567 set_zone_contiguous(zone); 1568 } 1569 1570 #ifdef CONFIG_CMA 1571 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 1572 void __init init_cma_reserved_pageblock(struct page *page) 1573 { 1574 unsigned i = pageblock_nr_pages; 1575 struct page *p = page; 1576 1577 do { 1578 __ClearPageReserved(p); 1579 set_page_count(p, 0); 1580 } while (++p, --i); 1581 1582 set_pageblock_migratetype(page, MIGRATE_CMA); 1583 1584 if (pageblock_order >= MAX_ORDER) { 1585 i = pageblock_nr_pages; 1586 p = page; 1587 do { 1588 set_page_refcounted(p); 1589 __free_pages(p, MAX_ORDER - 1); 1590 p += MAX_ORDER_NR_PAGES; 1591 } while (i -= MAX_ORDER_NR_PAGES); 1592 } else { 1593 set_page_refcounted(page); 1594 __free_pages(page, pageblock_order); 1595 } 1596 1597 adjust_managed_page_count(page, pageblock_nr_pages); 1598 } 1599 #endif 1600 1601 /* 1602 * The order of subdivision here is critical for the IO subsystem. 1603 * Please do not alter this order without good reasons and regression 1604 * testing. Specifically, as large blocks of memory are subdivided, 1605 * the order in which smaller blocks are delivered depends on the order 1606 * they're subdivided in this function. This is the primary factor 1607 * influencing the order in which pages are delivered to the IO 1608 * subsystem according to empirical testing, and this is also justified 1609 * by considering the behavior of a buddy system containing a single 1610 * large block of memory acted on by a series of small allocations. 1611 * This behavior is a critical factor in sglist merging's success. 1612 * 1613 * -- nyc 1614 */ 1615 static inline void expand(struct zone *zone, struct page *page, 1616 int low, int high, struct free_area *area, 1617 int migratetype) 1618 { 1619 unsigned long size = 1 << high; 1620 1621 while (high > low) { 1622 area--; 1623 high--; 1624 size >>= 1; 1625 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 1626 1627 if (IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) && 1628 debug_guardpage_enabled() && 1629 high < debug_guardpage_minorder()) { 1630 /* 1631 * Mark as guard pages (or page), that will allow to 1632 * merge back to allocator when buddy will be freed. 1633 * Corresponding page table entries will not be touched, 1634 * pages will stay not present in virtual address space 1635 */ 1636 set_page_guard(zone, &page[size], high, migratetype); 1637 continue; 1638 } 1639 list_add(&page[size].lru, &area->free_list[migratetype]); 1640 area->nr_free++; 1641 set_page_order(&page[size], high); 1642 } 1643 } 1644 1645 static void check_new_page_bad(struct page *page) 1646 { 1647 const char *bad_reason = NULL; 1648 unsigned long bad_flags = 0; 1649 1650 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1651 bad_reason = "nonzero mapcount"; 1652 if (unlikely(page->mapping != NULL)) 1653 bad_reason = "non-NULL mapping"; 1654 if (unlikely(page_ref_count(page) != 0)) 1655 bad_reason = "nonzero _count"; 1656 if (unlikely(page->flags & __PG_HWPOISON)) { 1657 bad_reason = "HWPoisoned (hardware-corrupted)"; 1658 bad_flags = __PG_HWPOISON; 1659 /* Don't complain about hwpoisoned pages */ 1660 page_mapcount_reset(page); /* remove PageBuddy */ 1661 return; 1662 } 1663 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) { 1664 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set"; 1665 bad_flags = PAGE_FLAGS_CHECK_AT_PREP; 1666 } 1667 #ifdef CONFIG_MEMCG 1668 if (unlikely(page->mem_cgroup)) 1669 bad_reason = "page still charged to cgroup"; 1670 #endif 1671 bad_page(page, bad_reason, bad_flags); 1672 } 1673 1674 /* 1675 * This page is about to be returned from the page allocator 1676 */ 1677 static inline int check_new_page(struct page *page) 1678 { 1679 if (likely(page_expected_state(page, 1680 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 1681 return 0; 1682 1683 check_new_page_bad(page); 1684 return 1; 1685 } 1686 1687 static inline bool free_pages_prezeroed(bool poisoned) 1688 { 1689 return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) && 1690 page_poisoning_enabled() && poisoned; 1691 } 1692 1693 #ifdef CONFIG_DEBUG_VM 1694 static bool check_pcp_refill(struct page *page) 1695 { 1696 return false; 1697 } 1698 1699 static bool check_new_pcp(struct page *page) 1700 { 1701 return check_new_page(page); 1702 } 1703 #else 1704 static bool check_pcp_refill(struct page *page) 1705 { 1706 return check_new_page(page); 1707 } 1708 static bool check_new_pcp(struct page *page) 1709 { 1710 return false; 1711 } 1712 #endif /* CONFIG_DEBUG_VM */ 1713 1714 static bool check_new_pages(struct page *page, unsigned int order) 1715 { 1716 int i; 1717 for (i = 0; i < (1 << order); i++) { 1718 struct page *p = page + i; 1719 1720 if (unlikely(check_new_page(p))) 1721 return true; 1722 } 1723 1724 return false; 1725 } 1726 1727 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 1728 unsigned int alloc_flags) 1729 { 1730 int i; 1731 bool poisoned = true; 1732 1733 for (i = 0; i < (1 << order); i++) { 1734 struct page *p = page + i; 1735 if (poisoned) 1736 poisoned &= page_is_poisoned(p); 1737 } 1738 1739 set_page_private(page, 0); 1740 set_page_refcounted(page); 1741 1742 arch_alloc_page(page, order); 1743 kernel_map_pages(page, 1 << order, 1); 1744 kernel_poison_pages(page, 1 << order, 1); 1745 kasan_alloc_pages(page, order); 1746 1747 if (!free_pages_prezeroed(poisoned) && (gfp_flags & __GFP_ZERO)) 1748 for (i = 0; i < (1 << order); i++) 1749 clear_highpage(page + i); 1750 1751 if (order && (gfp_flags & __GFP_COMP)) 1752 prep_compound_page(page, order); 1753 1754 set_page_owner(page, order, gfp_flags); 1755 1756 /* 1757 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 1758 * allocate the page. The expectation is that the caller is taking 1759 * steps that will free more memory. The caller should avoid the page 1760 * being used for !PFMEMALLOC purposes. 1761 */ 1762 if (alloc_flags & ALLOC_NO_WATERMARKS) 1763 set_page_pfmemalloc(page); 1764 else 1765 clear_page_pfmemalloc(page); 1766 } 1767 1768 /* 1769 * Go through the free lists for the given migratetype and remove 1770 * the smallest available page from the freelists 1771 */ 1772 static inline 1773 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 1774 int migratetype) 1775 { 1776 unsigned int current_order; 1777 struct free_area *area; 1778 struct page *page; 1779 1780 /* Find a page of the appropriate size in the preferred list */ 1781 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 1782 area = &(zone->free_area[current_order]); 1783 page = list_first_entry_or_null(&area->free_list[migratetype], 1784 struct page, lru); 1785 if (!page) 1786 continue; 1787 list_del(&page->lru); 1788 rmv_page_order(page); 1789 area->nr_free--; 1790 expand(zone, page, order, current_order, area, migratetype); 1791 set_pcppage_migratetype(page, migratetype); 1792 return page; 1793 } 1794 1795 return NULL; 1796 } 1797 1798 1799 /* 1800 * This array describes the order lists are fallen back to when 1801 * the free lists for the desirable migrate type are depleted 1802 */ 1803 static int fallbacks[MIGRATE_TYPES][4] = { 1804 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 1805 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 1806 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 1807 #ifdef CONFIG_CMA 1808 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */ 1809 #endif 1810 #ifdef CONFIG_MEMORY_ISOLATION 1811 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */ 1812 #endif 1813 }; 1814 1815 #ifdef CONFIG_CMA 1816 static struct page *__rmqueue_cma_fallback(struct zone *zone, 1817 unsigned int order) 1818 { 1819 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 1820 } 1821 #else 1822 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 1823 unsigned int order) { return NULL; } 1824 #endif 1825 1826 /* 1827 * Move the free pages in a range to the free lists of the requested type. 1828 * Note that start_page and end_pages are not aligned on a pageblock 1829 * boundary. If alignment is required, use move_freepages_block() 1830 */ 1831 int move_freepages(struct zone *zone, 1832 struct page *start_page, struct page *end_page, 1833 int migratetype) 1834 { 1835 struct page *page; 1836 unsigned int order; 1837 int pages_moved = 0; 1838 1839 #ifndef CONFIG_HOLES_IN_ZONE 1840 /* 1841 * page_zone is not safe to call in this context when 1842 * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant 1843 * anyway as we check zone boundaries in move_freepages_block(). 1844 * Remove at a later date when no bug reports exist related to 1845 * grouping pages by mobility 1846 */ 1847 VM_BUG_ON(page_zone(start_page) != page_zone(end_page)); 1848 #endif 1849 1850 for (page = start_page; page <= end_page;) { 1851 /* Make sure we are not inadvertently changing nodes */ 1852 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 1853 1854 if (!pfn_valid_within(page_to_pfn(page))) { 1855 page++; 1856 continue; 1857 } 1858 1859 if (!PageBuddy(page)) { 1860 page++; 1861 continue; 1862 } 1863 1864 order = page_order(page); 1865 list_move(&page->lru, 1866 &zone->free_area[order].free_list[migratetype]); 1867 page += 1 << order; 1868 pages_moved += 1 << order; 1869 } 1870 1871 return pages_moved; 1872 } 1873 1874 int move_freepages_block(struct zone *zone, struct page *page, 1875 int migratetype) 1876 { 1877 unsigned long start_pfn, end_pfn; 1878 struct page *start_page, *end_page; 1879 1880 start_pfn = page_to_pfn(page); 1881 start_pfn = start_pfn & ~(pageblock_nr_pages-1); 1882 start_page = pfn_to_page(start_pfn); 1883 end_page = start_page + pageblock_nr_pages - 1; 1884 end_pfn = start_pfn + pageblock_nr_pages - 1; 1885 1886 /* Do not cross zone boundaries */ 1887 if (!zone_spans_pfn(zone, start_pfn)) 1888 start_page = page; 1889 if (!zone_spans_pfn(zone, end_pfn)) 1890 return 0; 1891 1892 return move_freepages(zone, start_page, end_page, migratetype); 1893 } 1894 1895 static void change_pageblock_range(struct page *pageblock_page, 1896 int start_order, int migratetype) 1897 { 1898 int nr_pageblocks = 1 << (start_order - pageblock_order); 1899 1900 while (nr_pageblocks--) { 1901 set_pageblock_migratetype(pageblock_page, migratetype); 1902 pageblock_page += pageblock_nr_pages; 1903 } 1904 } 1905 1906 /* 1907 * When we are falling back to another migratetype during allocation, try to 1908 * steal extra free pages from the same pageblocks to satisfy further 1909 * allocations, instead of polluting multiple pageblocks. 1910 * 1911 * If we are stealing a relatively large buddy page, it is likely there will 1912 * be more free pages in the pageblock, so try to steal them all. For 1913 * reclaimable and unmovable allocations, we steal regardless of page size, 1914 * as fragmentation caused by those allocations polluting movable pageblocks 1915 * is worse than movable allocations stealing from unmovable and reclaimable 1916 * pageblocks. 1917 */ 1918 static bool can_steal_fallback(unsigned int order, int start_mt) 1919 { 1920 /* 1921 * Leaving this order check is intended, although there is 1922 * relaxed order check in next check. The reason is that 1923 * we can actually steal whole pageblock if this condition met, 1924 * but, below check doesn't guarantee it and that is just heuristic 1925 * so could be changed anytime. 1926 */ 1927 if (order >= pageblock_order) 1928 return true; 1929 1930 if (order >= pageblock_order / 2 || 1931 start_mt == MIGRATE_RECLAIMABLE || 1932 start_mt == MIGRATE_UNMOVABLE || 1933 page_group_by_mobility_disabled) 1934 return true; 1935 1936 return false; 1937 } 1938 1939 /* 1940 * This function implements actual steal behaviour. If order is large enough, 1941 * we can steal whole pageblock. If not, we first move freepages in this 1942 * pageblock and check whether half of pages are moved or not. If half of 1943 * pages are moved, we can change migratetype of pageblock and permanently 1944 * use it's pages as requested migratetype in the future. 1945 */ 1946 static void steal_suitable_fallback(struct zone *zone, struct page *page, 1947 int start_type) 1948 { 1949 unsigned int current_order = page_order(page); 1950 int pages; 1951 1952 /* Take ownership for orders >= pageblock_order */ 1953 if (current_order >= pageblock_order) { 1954 change_pageblock_range(page, current_order, start_type); 1955 return; 1956 } 1957 1958 pages = move_freepages_block(zone, page, start_type); 1959 1960 /* Claim the whole block if over half of it is free */ 1961 if (pages >= (1 << (pageblock_order-1)) || 1962 page_group_by_mobility_disabled) 1963 set_pageblock_migratetype(page, start_type); 1964 } 1965 1966 /* 1967 * Check whether there is a suitable fallback freepage with requested order. 1968 * If only_stealable is true, this function returns fallback_mt only if 1969 * we can steal other freepages all together. This would help to reduce 1970 * fragmentation due to mixed migratetype pages in one pageblock. 1971 */ 1972 int find_suitable_fallback(struct free_area *area, unsigned int order, 1973 int migratetype, bool only_stealable, bool *can_steal) 1974 { 1975 int i; 1976 int fallback_mt; 1977 1978 if (area->nr_free == 0) 1979 return -1; 1980 1981 *can_steal = false; 1982 for (i = 0;; i++) { 1983 fallback_mt = fallbacks[migratetype][i]; 1984 if (fallback_mt == MIGRATE_TYPES) 1985 break; 1986 1987 if (list_empty(&area->free_list[fallback_mt])) 1988 continue; 1989 1990 if (can_steal_fallback(order, migratetype)) 1991 *can_steal = true; 1992 1993 if (!only_stealable) 1994 return fallback_mt; 1995 1996 if (*can_steal) 1997 return fallback_mt; 1998 } 1999 2000 return -1; 2001 } 2002 2003 /* 2004 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2005 * there are no empty page blocks that contain a page with a suitable order 2006 */ 2007 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2008 unsigned int alloc_order) 2009 { 2010 int mt; 2011 unsigned long max_managed, flags; 2012 2013 /* 2014 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2015 * Check is race-prone but harmless. 2016 */ 2017 max_managed = (zone->managed_pages / 100) + pageblock_nr_pages; 2018 if (zone->nr_reserved_highatomic >= max_managed) 2019 return; 2020 2021 spin_lock_irqsave(&zone->lock, flags); 2022 2023 /* Recheck the nr_reserved_highatomic limit under the lock */ 2024 if (zone->nr_reserved_highatomic >= max_managed) 2025 goto out_unlock; 2026 2027 /* Yoink! */ 2028 mt = get_pageblock_migratetype(page); 2029 if (mt != MIGRATE_HIGHATOMIC && 2030 !is_migrate_isolate(mt) && !is_migrate_cma(mt)) { 2031 zone->nr_reserved_highatomic += pageblock_nr_pages; 2032 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2033 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC); 2034 } 2035 2036 out_unlock: 2037 spin_unlock_irqrestore(&zone->lock, flags); 2038 } 2039 2040 /* 2041 * Used when an allocation is about to fail under memory pressure. This 2042 * potentially hurts the reliability of high-order allocations when under 2043 * intense memory pressure but failed atomic allocations should be easier 2044 * to recover from than an OOM. 2045 */ 2046 static void unreserve_highatomic_pageblock(const struct alloc_context *ac) 2047 { 2048 struct zonelist *zonelist = ac->zonelist; 2049 unsigned long flags; 2050 struct zoneref *z; 2051 struct zone *zone; 2052 struct page *page; 2053 int order; 2054 2055 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx, 2056 ac->nodemask) { 2057 /* Preserve at least one pageblock */ 2058 if (zone->nr_reserved_highatomic <= pageblock_nr_pages) 2059 continue; 2060 2061 spin_lock_irqsave(&zone->lock, flags); 2062 for (order = 0; order < MAX_ORDER; order++) { 2063 struct free_area *area = &(zone->free_area[order]); 2064 2065 page = list_first_entry_or_null( 2066 &area->free_list[MIGRATE_HIGHATOMIC], 2067 struct page, lru); 2068 if (!page) 2069 continue; 2070 2071 /* 2072 * It should never happen but changes to locking could 2073 * inadvertently allow a per-cpu drain to add pages 2074 * to MIGRATE_HIGHATOMIC while unreserving so be safe 2075 * and watch for underflows. 2076 */ 2077 zone->nr_reserved_highatomic -= min(pageblock_nr_pages, 2078 zone->nr_reserved_highatomic); 2079 2080 /* 2081 * Convert to ac->migratetype and avoid the normal 2082 * pageblock stealing heuristics. Minimally, the caller 2083 * is doing the work and needs the pages. More 2084 * importantly, if the block was always converted to 2085 * MIGRATE_UNMOVABLE or another type then the number 2086 * of pageblocks that cannot be completely freed 2087 * may increase. 2088 */ 2089 set_pageblock_migratetype(page, ac->migratetype); 2090 move_freepages_block(zone, page, ac->migratetype); 2091 spin_unlock_irqrestore(&zone->lock, flags); 2092 return; 2093 } 2094 spin_unlock_irqrestore(&zone->lock, flags); 2095 } 2096 } 2097 2098 /* Remove an element from the buddy allocator from the fallback list */ 2099 static inline struct page * 2100 __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype) 2101 { 2102 struct free_area *area; 2103 unsigned int current_order; 2104 struct page *page; 2105 int fallback_mt; 2106 bool can_steal; 2107 2108 /* Find the largest possible block of pages in the other list */ 2109 for (current_order = MAX_ORDER-1; 2110 current_order >= order && current_order <= MAX_ORDER-1; 2111 --current_order) { 2112 area = &(zone->free_area[current_order]); 2113 fallback_mt = find_suitable_fallback(area, current_order, 2114 start_migratetype, false, &can_steal); 2115 if (fallback_mt == -1) 2116 continue; 2117 2118 page = list_first_entry(&area->free_list[fallback_mt], 2119 struct page, lru); 2120 if (can_steal) 2121 steal_suitable_fallback(zone, page, start_migratetype); 2122 2123 /* Remove the page from the freelists */ 2124 area->nr_free--; 2125 list_del(&page->lru); 2126 rmv_page_order(page); 2127 2128 expand(zone, page, order, current_order, area, 2129 start_migratetype); 2130 /* 2131 * The pcppage_migratetype may differ from pageblock's 2132 * migratetype depending on the decisions in 2133 * find_suitable_fallback(). This is OK as long as it does not 2134 * differ for MIGRATE_CMA pageblocks. Those can be used as 2135 * fallback only via special __rmqueue_cma_fallback() function 2136 */ 2137 set_pcppage_migratetype(page, start_migratetype); 2138 2139 trace_mm_page_alloc_extfrag(page, order, current_order, 2140 start_migratetype, fallback_mt); 2141 2142 return page; 2143 } 2144 2145 return NULL; 2146 } 2147 2148 /* 2149 * Do the hard work of removing an element from the buddy allocator. 2150 * Call me with the zone->lock already held. 2151 */ 2152 static struct page *__rmqueue(struct zone *zone, unsigned int order, 2153 int migratetype) 2154 { 2155 struct page *page; 2156 2157 page = __rmqueue_smallest(zone, order, migratetype); 2158 if (unlikely(!page)) { 2159 if (migratetype == MIGRATE_MOVABLE) 2160 page = __rmqueue_cma_fallback(zone, order); 2161 2162 if (!page) 2163 page = __rmqueue_fallback(zone, order, migratetype); 2164 } 2165 2166 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2167 return page; 2168 } 2169 2170 /* 2171 * Obtain a specified number of elements from the buddy allocator, all under 2172 * a single hold of the lock, for efficiency. Add them to the supplied list. 2173 * Returns the number of new pages which were placed at *list. 2174 */ 2175 static int rmqueue_bulk(struct zone *zone, unsigned int order, 2176 unsigned long count, struct list_head *list, 2177 int migratetype, bool cold) 2178 { 2179 int i; 2180 2181 spin_lock(&zone->lock); 2182 for (i = 0; i < count; ++i) { 2183 struct page *page = __rmqueue(zone, order, migratetype); 2184 if (unlikely(page == NULL)) 2185 break; 2186 2187 if (unlikely(check_pcp_refill(page))) 2188 continue; 2189 2190 /* 2191 * Split buddy pages returned by expand() are received here 2192 * in physical page order. The page is added to the callers and 2193 * list and the list head then moves forward. From the callers 2194 * perspective, the linked list is ordered by page number in 2195 * some conditions. This is useful for IO devices that can 2196 * merge IO requests if the physical pages are ordered 2197 * properly. 2198 */ 2199 if (likely(!cold)) 2200 list_add(&page->lru, list); 2201 else 2202 list_add_tail(&page->lru, list); 2203 list = &page->lru; 2204 if (is_migrate_cma(get_pcppage_migratetype(page))) 2205 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 2206 -(1 << order)); 2207 } 2208 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 2209 spin_unlock(&zone->lock); 2210 return i; 2211 } 2212 2213 #ifdef CONFIG_NUMA 2214 /* 2215 * Called from the vmstat counter updater to drain pagesets of this 2216 * currently executing processor on remote nodes after they have 2217 * expired. 2218 * 2219 * Note that this function must be called with the thread pinned to 2220 * a single processor. 2221 */ 2222 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 2223 { 2224 unsigned long flags; 2225 int to_drain, batch; 2226 2227 local_irq_save(flags); 2228 batch = READ_ONCE(pcp->batch); 2229 to_drain = min(pcp->count, batch); 2230 if (to_drain > 0) { 2231 free_pcppages_bulk(zone, to_drain, pcp); 2232 pcp->count -= to_drain; 2233 } 2234 local_irq_restore(flags); 2235 } 2236 #endif 2237 2238 /* 2239 * Drain pcplists of the indicated processor and zone. 2240 * 2241 * The processor must either be the current processor and the 2242 * thread pinned to the current processor or a processor that 2243 * is not online. 2244 */ 2245 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 2246 { 2247 unsigned long flags; 2248 struct per_cpu_pageset *pset; 2249 struct per_cpu_pages *pcp; 2250 2251 local_irq_save(flags); 2252 pset = per_cpu_ptr(zone->pageset, cpu); 2253 2254 pcp = &pset->pcp; 2255 if (pcp->count) { 2256 free_pcppages_bulk(zone, pcp->count, pcp); 2257 pcp->count = 0; 2258 } 2259 local_irq_restore(flags); 2260 } 2261 2262 /* 2263 * Drain pcplists of all zones on the indicated processor. 2264 * 2265 * The processor must either be the current processor and the 2266 * thread pinned to the current processor or a processor that 2267 * is not online. 2268 */ 2269 static void drain_pages(unsigned int cpu) 2270 { 2271 struct zone *zone; 2272 2273 for_each_populated_zone(zone) { 2274 drain_pages_zone(cpu, zone); 2275 } 2276 } 2277 2278 /* 2279 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 2280 * 2281 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 2282 * the single zone's pages. 2283 */ 2284 void drain_local_pages(struct zone *zone) 2285 { 2286 int cpu = smp_processor_id(); 2287 2288 if (zone) 2289 drain_pages_zone(cpu, zone); 2290 else 2291 drain_pages(cpu); 2292 } 2293 2294 /* 2295 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 2296 * 2297 * When zone parameter is non-NULL, spill just the single zone's pages. 2298 * 2299 * Note that this code is protected against sending an IPI to an offline 2300 * CPU but does not guarantee sending an IPI to newly hotplugged CPUs: 2301 * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but 2302 * nothing keeps CPUs from showing up after we populated the cpumask and 2303 * before the call to on_each_cpu_mask(). 2304 */ 2305 void drain_all_pages(struct zone *zone) 2306 { 2307 int cpu; 2308 2309 /* 2310 * Allocate in the BSS so we wont require allocation in 2311 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 2312 */ 2313 static cpumask_t cpus_with_pcps; 2314 2315 /* 2316 * We don't care about racing with CPU hotplug event 2317 * as offline notification will cause the notified 2318 * cpu to drain that CPU pcps and on_each_cpu_mask 2319 * disables preemption as part of its processing 2320 */ 2321 for_each_online_cpu(cpu) { 2322 struct per_cpu_pageset *pcp; 2323 struct zone *z; 2324 bool has_pcps = false; 2325 2326 if (zone) { 2327 pcp = per_cpu_ptr(zone->pageset, cpu); 2328 if (pcp->pcp.count) 2329 has_pcps = true; 2330 } else { 2331 for_each_populated_zone(z) { 2332 pcp = per_cpu_ptr(z->pageset, cpu); 2333 if (pcp->pcp.count) { 2334 has_pcps = true; 2335 break; 2336 } 2337 } 2338 } 2339 2340 if (has_pcps) 2341 cpumask_set_cpu(cpu, &cpus_with_pcps); 2342 else 2343 cpumask_clear_cpu(cpu, &cpus_with_pcps); 2344 } 2345 on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages, 2346 zone, 1); 2347 } 2348 2349 #ifdef CONFIG_HIBERNATION 2350 2351 void mark_free_pages(struct zone *zone) 2352 { 2353 unsigned long pfn, max_zone_pfn; 2354 unsigned long flags; 2355 unsigned int order, t; 2356 struct page *page; 2357 2358 if (zone_is_empty(zone)) 2359 return; 2360 2361 spin_lock_irqsave(&zone->lock, flags); 2362 2363 max_zone_pfn = zone_end_pfn(zone); 2364 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 2365 if (pfn_valid(pfn)) { 2366 page = pfn_to_page(pfn); 2367 2368 if (page_zone(page) != zone) 2369 continue; 2370 2371 if (!swsusp_page_is_forbidden(page)) 2372 swsusp_unset_page_free(page); 2373 } 2374 2375 for_each_migratetype_order(order, t) { 2376 list_for_each_entry(page, 2377 &zone->free_area[order].free_list[t], lru) { 2378 unsigned long i; 2379 2380 pfn = page_to_pfn(page); 2381 for (i = 0; i < (1UL << order); i++) 2382 swsusp_set_page_free(pfn_to_page(pfn + i)); 2383 } 2384 } 2385 spin_unlock_irqrestore(&zone->lock, flags); 2386 } 2387 #endif /* CONFIG_PM */ 2388 2389 /* 2390 * Free a 0-order page 2391 * cold == true ? free a cold page : free a hot page 2392 */ 2393 void free_hot_cold_page(struct page *page, bool cold) 2394 { 2395 struct zone *zone = page_zone(page); 2396 struct per_cpu_pages *pcp; 2397 unsigned long flags; 2398 unsigned long pfn = page_to_pfn(page); 2399 int migratetype; 2400 2401 if (!free_pcp_prepare(page)) 2402 return; 2403 2404 migratetype = get_pfnblock_migratetype(page, pfn); 2405 set_pcppage_migratetype(page, migratetype); 2406 local_irq_save(flags); 2407 __count_vm_event(PGFREE); 2408 2409 /* 2410 * We only track unmovable, reclaimable and movable on pcp lists. 2411 * Free ISOLATE pages back to the allocator because they are being 2412 * offlined but treat RESERVE as movable pages so we can get those 2413 * areas back if necessary. Otherwise, we may have to free 2414 * excessively into the page allocator 2415 */ 2416 if (migratetype >= MIGRATE_PCPTYPES) { 2417 if (unlikely(is_migrate_isolate(migratetype))) { 2418 free_one_page(zone, page, pfn, 0, migratetype); 2419 goto out; 2420 } 2421 migratetype = MIGRATE_MOVABLE; 2422 } 2423 2424 pcp = &this_cpu_ptr(zone->pageset)->pcp; 2425 if (!cold) 2426 list_add(&page->lru, &pcp->lists[migratetype]); 2427 else 2428 list_add_tail(&page->lru, &pcp->lists[migratetype]); 2429 pcp->count++; 2430 if (pcp->count >= pcp->high) { 2431 unsigned long batch = READ_ONCE(pcp->batch); 2432 free_pcppages_bulk(zone, batch, pcp); 2433 pcp->count -= batch; 2434 } 2435 2436 out: 2437 local_irq_restore(flags); 2438 } 2439 2440 /* 2441 * Free a list of 0-order pages 2442 */ 2443 void free_hot_cold_page_list(struct list_head *list, bool cold) 2444 { 2445 struct page *page, *next; 2446 2447 list_for_each_entry_safe(page, next, list, lru) { 2448 trace_mm_page_free_batched(page, cold); 2449 free_hot_cold_page(page, cold); 2450 } 2451 } 2452 2453 /* 2454 * split_page takes a non-compound higher-order page, and splits it into 2455 * n (1<<order) sub-pages: page[0..n] 2456 * Each sub-page must be freed individually. 2457 * 2458 * Note: this is probably too low level an operation for use in drivers. 2459 * Please consult with lkml before using this in your driver. 2460 */ 2461 void split_page(struct page *page, unsigned int order) 2462 { 2463 int i; 2464 gfp_t gfp_mask; 2465 2466 VM_BUG_ON_PAGE(PageCompound(page), page); 2467 VM_BUG_ON_PAGE(!page_count(page), page); 2468 2469 #ifdef CONFIG_KMEMCHECK 2470 /* 2471 * Split shadow pages too, because free(page[0]) would 2472 * otherwise free the whole shadow. 2473 */ 2474 if (kmemcheck_page_is_tracked(page)) 2475 split_page(virt_to_page(page[0].shadow), order); 2476 #endif 2477 2478 gfp_mask = get_page_owner_gfp(page); 2479 set_page_owner(page, 0, gfp_mask); 2480 for (i = 1; i < (1 << order); i++) { 2481 set_page_refcounted(page + i); 2482 set_page_owner(page + i, 0, gfp_mask); 2483 } 2484 } 2485 EXPORT_SYMBOL_GPL(split_page); 2486 2487 int __isolate_free_page(struct page *page, unsigned int order) 2488 { 2489 unsigned long watermark; 2490 struct zone *zone; 2491 int mt; 2492 2493 BUG_ON(!PageBuddy(page)); 2494 2495 zone = page_zone(page); 2496 mt = get_pageblock_migratetype(page); 2497 2498 if (!is_migrate_isolate(mt)) { 2499 /* Obey watermarks as if the page was being allocated */ 2500 watermark = low_wmark_pages(zone) + (1 << order); 2501 if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) 2502 return 0; 2503 2504 __mod_zone_freepage_state(zone, -(1UL << order), mt); 2505 } 2506 2507 /* Remove page from free list */ 2508 list_del(&page->lru); 2509 zone->free_area[order].nr_free--; 2510 rmv_page_order(page); 2511 2512 set_page_owner(page, order, __GFP_MOVABLE); 2513 2514 /* Set the pageblock if the isolated page is at least a pageblock */ 2515 if (order >= pageblock_order - 1) { 2516 struct page *endpage = page + (1 << order) - 1; 2517 for (; page < endpage; page += pageblock_nr_pages) { 2518 int mt = get_pageblock_migratetype(page); 2519 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)) 2520 set_pageblock_migratetype(page, 2521 MIGRATE_MOVABLE); 2522 } 2523 } 2524 2525 2526 return 1UL << order; 2527 } 2528 2529 /* 2530 * Similar to split_page except the page is already free. As this is only 2531 * being used for migration, the migratetype of the block also changes. 2532 * As this is called with interrupts disabled, the caller is responsible 2533 * for calling arch_alloc_page() and kernel_map_page() after interrupts 2534 * are enabled. 2535 * 2536 * Note: this is probably too low level an operation for use in drivers. 2537 * Please consult with lkml before using this in your driver. 2538 */ 2539 int split_free_page(struct page *page) 2540 { 2541 unsigned int order; 2542 int nr_pages; 2543 2544 order = page_order(page); 2545 2546 nr_pages = __isolate_free_page(page, order); 2547 if (!nr_pages) 2548 return 0; 2549 2550 /* Split into individual pages */ 2551 set_page_refcounted(page); 2552 split_page(page, order); 2553 return nr_pages; 2554 } 2555 2556 /* 2557 * Update NUMA hit/miss statistics 2558 * 2559 * Must be called with interrupts disabled. 2560 * 2561 * When __GFP_OTHER_NODE is set assume the node of the preferred 2562 * zone is the local node. This is useful for daemons who allocate 2563 * memory on behalf of other processes. 2564 */ 2565 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z, 2566 gfp_t flags) 2567 { 2568 #ifdef CONFIG_NUMA 2569 int local_nid = numa_node_id(); 2570 enum zone_stat_item local_stat = NUMA_LOCAL; 2571 2572 if (unlikely(flags & __GFP_OTHER_NODE)) { 2573 local_stat = NUMA_OTHER; 2574 local_nid = preferred_zone->node; 2575 } 2576 2577 if (z->node == local_nid) { 2578 __inc_zone_state(z, NUMA_HIT); 2579 __inc_zone_state(z, local_stat); 2580 } else { 2581 __inc_zone_state(z, NUMA_MISS); 2582 __inc_zone_state(preferred_zone, NUMA_FOREIGN); 2583 } 2584 #endif 2585 } 2586 2587 /* 2588 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 2589 */ 2590 static inline 2591 struct page *buffered_rmqueue(struct zone *preferred_zone, 2592 struct zone *zone, unsigned int order, 2593 gfp_t gfp_flags, unsigned int alloc_flags, 2594 int migratetype) 2595 { 2596 unsigned long flags; 2597 struct page *page; 2598 bool cold = ((gfp_flags & __GFP_COLD) != 0); 2599 2600 if (likely(order == 0)) { 2601 struct per_cpu_pages *pcp; 2602 struct list_head *list; 2603 2604 local_irq_save(flags); 2605 do { 2606 pcp = &this_cpu_ptr(zone->pageset)->pcp; 2607 list = &pcp->lists[migratetype]; 2608 if (list_empty(list)) { 2609 pcp->count += rmqueue_bulk(zone, 0, 2610 pcp->batch, list, 2611 migratetype, cold); 2612 if (unlikely(list_empty(list))) 2613 goto failed; 2614 } 2615 2616 if (cold) 2617 page = list_last_entry(list, struct page, lru); 2618 else 2619 page = list_first_entry(list, struct page, lru); 2620 2621 __dec_zone_state(zone, NR_ALLOC_BATCH); 2622 list_del(&page->lru); 2623 pcp->count--; 2624 2625 } while (check_new_pcp(page)); 2626 } else { 2627 /* 2628 * We most definitely don't want callers attempting to 2629 * allocate greater than order-1 page units with __GFP_NOFAIL. 2630 */ 2631 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 2632 spin_lock_irqsave(&zone->lock, flags); 2633 2634 do { 2635 page = NULL; 2636 if (alloc_flags & ALLOC_HARDER) { 2637 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 2638 if (page) 2639 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2640 } 2641 if (!page) 2642 page = __rmqueue(zone, order, migratetype); 2643 } while (page && check_new_pages(page, order)); 2644 spin_unlock(&zone->lock); 2645 if (!page) 2646 goto failed; 2647 __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); 2648 __mod_zone_freepage_state(zone, -(1 << order), 2649 get_pcppage_migratetype(page)); 2650 } 2651 2652 if (atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]) <= 0 && 2653 !test_bit(ZONE_FAIR_DEPLETED, &zone->flags)) 2654 set_bit(ZONE_FAIR_DEPLETED, &zone->flags); 2655 2656 __count_zone_vm_events(PGALLOC, zone, 1 << order); 2657 zone_statistics(preferred_zone, zone, gfp_flags); 2658 local_irq_restore(flags); 2659 2660 VM_BUG_ON_PAGE(bad_range(zone, page), page); 2661 return page; 2662 2663 failed: 2664 local_irq_restore(flags); 2665 return NULL; 2666 } 2667 2668 #ifdef CONFIG_FAIL_PAGE_ALLOC 2669 2670 static struct { 2671 struct fault_attr attr; 2672 2673 bool ignore_gfp_highmem; 2674 bool ignore_gfp_reclaim; 2675 u32 min_order; 2676 } fail_page_alloc = { 2677 .attr = FAULT_ATTR_INITIALIZER, 2678 .ignore_gfp_reclaim = true, 2679 .ignore_gfp_highmem = true, 2680 .min_order = 1, 2681 }; 2682 2683 static int __init setup_fail_page_alloc(char *str) 2684 { 2685 return setup_fault_attr(&fail_page_alloc.attr, str); 2686 } 2687 __setup("fail_page_alloc=", setup_fail_page_alloc); 2688 2689 static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 2690 { 2691 if (order < fail_page_alloc.min_order) 2692 return false; 2693 if (gfp_mask & __GFP_NOFAIL) 2694 return false; 2695 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 2696 return false; 2697 if (fail_page_alloc.ignore_gfp_reclaim && 2698 (gfp_mask & __GFP_DIRECT_RECLAIM)) 2699 return false; 2700 2701 return should_fail(&fail_page_alloc.attr, 1 << order); 2702 } 2703 2704 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 2705 2706 static int __init fail_page_alloc_debugfs(void) 2707 { 2708 umode_t mode = S_IFREG | S_IRUSR | S_IWUSR; 2709 struct dentry *dir; 2710 2711 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 2712 &fail_page_alloc.attr); 2713 if (IS_ERR(dir)) 2714 return PTR_ERR(dir); 2715 2716 if (!debugfs_create_bool("ignore-gfp-wait", mode, dir, 2717 &fail_page_alloc.ignore_gfp_reclaim)) 2718 goto fail; 2719 if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir, 2720 &fail_page_alloc.ignore_gfp_highmem)) 2721 goto fail; 2722 if (!debugfs_create_u32("min-order", mode, dir, 2723 &fail_page_alloc.min_order)) 2724 goto fail; 2725 2726 return 0; 2727 fail: 2728 debugfs_remove_recursive(dir); 2729 2730 return -ENOMEM; 2731 } 2732 2733 late_initcall(fail_page_alloc_debugfs); 2734 2735 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 2736 2737 #else /* CONFIG_FAIL_PAGE_ALLOC */ 2738 2739 static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 2740 { 2741 return false; 2742 } 2743 2744 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 2745 2746 /* 2747 * Return true if free base pages are above 'mark'. For high-order checks it 2748 * will return true of the order-0 watermark is reached and there is at least 2749 * one free page of a suitable size. Checking now avoids taking the zone lock 2750 * to check in the allocation paths if no pages are free. 2751 */ 2752 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 2753 int classzone_idx, unsigned int alloc_flags, 2754 long free_pages) 2755 { 2756 long min = mark; 2757 int o; 2758 const bool alloc_harder = (alloc_flags & ALLOC_HARDER); 2759 2760 /* free_pages may go negative - that's OK */ 2761 free_pages -= (1 << order) - 1; 2762 2763 if (alloc_flags & ALLOC_HIGH) 2764 min -= min / 2; 2765 2766 /* 2767 * If the caller does not have rights to ALLOC_HARDER then subtract 2768 * the high-atomic reserves. This will over-estimate the size of the 2769 * atomic reserve but it avoids a search. 2770 */ 2771 if (likely(!alloc_harder)) 2772 free_pages -= z->nr_reserved_highatomic; 2773 else 2774 min -= min / 4; 2775 2776 #ifdef CONFIG_CMA 2777 /* If allocation can't use CMA areas don't use free CMA pages */ 2778 if (!(alloc_flags & ALLOC_CMA)) 2779 free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES); 2780 #endif 2781 2782 /* 2783 * Check watermarks for an order-0 allocation request. If these 2784 * are not met, then a high-order request also cannot go ahead 2785 * even if a suitable page happened to be free. 2786 */ 2787 if (free_pages <= min + z->lowmem_reserve[classzone_idx]) 2788 return false; 2789 2790 /* If this is an order-0 request then the watermark is fine */ 2791 if (!order) 2792 return true; 2793 2794 /* For a high-order request, check at least one suitable page is free */ 2795 for (o = order; o < MAX_ORDER; o++) { 2796 struct free_area *area = &z->free_area[o]; 2797 int mt; 2798 2799 if (!area->nr_free) 2800 continue; 2801 2802 if (alloc_harder) 2803 return true; 2804 2805 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 2806 if (!list_empty(&area->free_list[mt])) 2807 return true; 2808 } 2809 2810 #ifdef CONFIG_CMA 2811 if ((alloc_flags & ALLOC_CMA) && 2812 !list_empty(&area->free_list[MIGRATE_CMA])) { 2813 return true; 2814 } 2815 #endif 2816 } 2817 return false; 2818 } 2819 2820 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 2821 int classzone_idx, unsigned int alloc_flags) 2822 { 2823 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 2824 zone_page_state(z, NR_FREE_PAGES)); 2825 } 2826 2827 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 2828 unsigned long mark, int classzone_idx, unsigned int alloc_flags) 2829 { 2830 long free_pages = zone_page_state(z, NR_FREE_PAGES); 2831 long cma_pages = 0; 2832 2833 #ifdef CONFIG_CMA 2834 /* If allocation can't use CMA areas don't use free CMA pages */ 2835 if (!(alloc_flags & ALLOC_CMA)) 2836 cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES); 2837 #endif 2838 2839 /* 2840 * Fast check for order-0 only. If this fails then the reserves 2841 * need to be calculated. There is a corner case where the check 2842 * passes but only the high-order atomic reserve are free. If 2843 * the caller is !atomic then it'll uselessly search the free 2844 * list. That corner case is then slower but it is harmless. 2845 */ 2846 if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx]) 2847 return true; 2848 2849 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 2850 free_pages); 2851 } 2852 2853 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 2854 unsigned long mark, int classzone_idx) 2855 { 2856 long free_pages = zone_page_state(z, NR_FREE_PAGES); 2857 2858 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 2859 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 2860 2861 return __zone_watermark_ok(z, order, mark, classzone_idx, 0, 2862 free_pages); 2863 } 2864 2865 #ifdef CONFIG_NUMA 2866 static bool zone_local(struct zone *local_zone, struct zone *zone) 2867 { 2868 return local_zone->node == zone->node; 2869 } 2870 2871 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 2872 { 2873 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) < 2874 RECLAIM_DISTANCE; 2875 } 2876 #else /* CONFIG_NUMA */ 2877 static bool zone_local(struct zone *local_zone, struct zone *zone) 2878 { 2879 return true; 2880 } 2881 2882 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 2883 { 2884 return true; 2885 } 2886 #endif /* CONFIG_NUMA */ 2887 2888 static void reset_alloc_batches(struct zone *preferred_zone) 2889 { 2890 struct zone *zone = preferred_zone->zone_pgdat->node_zones; 2891 2892 do { 2893 mod_zone_page_state(zone, NR_ALLOC_BATCH, 2894 high_wmark_pages(zone) - low_wmark_pages(zone) - 2895 atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH])); 2896 clear_bit(ZONE_FAIR_DEPLETED, &zone->flags); 2897 } while (zone++ != preferred_zone); 2898 } 2899 2900 /* 2901 * get_page_from_freelist goes through the zonelist trying to allocate 2902 * a page. 2903 */ 2904 static struct page * 2905 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 2906 const struct alloc_context *ac) 2907 { 2908 struct zoneref *z = ac->preferred_zoneref; 2909 struct zone *zone; 2910 bool fair_skipped = false; 2911 bool apply_fair = (alloc_flags & ALLOC_FAIR); 2912 2913 zonelist_scan: 2914 /* 2915 * Scan zonelist, looking for a zone with enough free. 2916 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 2917 */ 2918 for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 2919 ac->nodemask) { 2920 struct page *page; 2921 unsigned long mark; 2922 2923 if (cpusets_enabled() && 2924 (alloc_flags & ALLOC_CPUSET) && 2925 !__cpuset_zone_allowed(zone, gfp_mask)) 2926 continue; 2927 /* 2928 * Distribute pages in proportion to the individual 2929 * zone size to ensure fair page aging. The zone a 2930 * page was allocated in should have no effect on the 2931 * time the page has in memory before being reclaimed. 2932 */ 2933 if (apply_fair) { 2934 if (test_bit(ZONE_FAIR_DEPLETED, &zone->flags)) { 2935 fair_skipped = true; 2936 continue; 2937 } 2938 if (!zone_local(ac->preferred_zoneref->zone, zone)) { 2939 if (fair_skipped) 2940 goto reset_fair; 2941 apply_fair = false; 2942 } 2943 } 2944 /* 2945 * When allocating a page cache page for writing, we 2946 * want to get it from a zone that is within its dirty 2947 * limit, such that no single zone holds more than its 2948 * proportional share of globally allowed dirty pages. 2949 * The dirty limits take into account the zone's 2950 * lowmem reserves and high watermark so that kswapd 2951 * should be able to balance it without having to 2952 * write pages from its LRU list. 2953 * 2954 * This may look like it could increase pressure on 2955 * lower zones by failing allocations in higher zones 2956 * before they are full. But the pages that do spill 2957 * over are limited as the lower zones are protected 2958 * by this very same mechanism. It should not become 2959 * a practical burden to them. 2960 * 2961 * XXX: For now, allow allocations to potentially 2962 * exceed the per-zone dirty limit in the slowpath 2963 * (spread_dirty_pages unset) before going into reclaim, 2964 * which is important when on a NUMA setup the allowed 2965 * zones are together not big enough to reach the 2966 * global limit. The proper fix for these situations 2967 * will require awareness of zones in the 2968 * dirty-throttling and the flusher threads. 2969 */ 2970 if (ac->spread_dirty_pages && !zone_dirty_ok(zone)) 2971 continue; 2972 2973 mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK]; 2974 if (!zone_watermark_fast(zone, order, mark, 2975 ac_classzone_idx(ac), alloc_flags)) { 2976 int ret; 2977 2978 /* Checked here to keep the fast path fast */ 2979 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 2980 if (alloc_flags & ALLOC_NO_WATERMARKS) 2981 goto try_this_zone; 2982 2983 if (zone_reclaim_mode == 0 || 2984 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 2985 continue; 2986 2987 ret = zone_reclaim(zone, gfp_mask, order); 2988 switch (ret) { 2989 case ZONE_RECLAIM_NOSCAN: 2990 /* did not scan */ 2991 continue; 2992 case ZONE_RECLAIM_FULL: 2993 /* scanned but unreclaimable */ 2994 continue; 2995 default: 2996 /* did we reclaim enough */ 2997 if (zone_watermark_ok(zone, order, mark, 2998 ac_classzone_idx(ac), alloc_flags)) 2999 goto try_this_zone; 3000 3001 continue; 3002 } 3003 } 3004 3005 try_this_zone: 3006 page = buffered_rmqueue(ac->preferred_zoneref->zone, zone, order, 3007 gfp_mask, alloc_flags, ac->migratetype); 3008 if (page) { 3009 prep_new_page(page, order, gfp_mask, alloc_flags); 3010 3011 /* 3012 * If this is a high-order atomic allocation then check 3013 * if the pageblock should be reserved for the future 3014 */ 3015 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 3016 reserve_highatomic_pageblock(page, zone, order); 3017 3018 return page; 3019 } 3020 } 3021 3022 /* 3023 * The first pass makes sure allocations are spread fairly within the 3024 * local node. However, the local node might have free pages left 3025 * after the fairness batches are exhausted, and remote zones haven't 3026 * even been considered yet. Try once more without fairness, and 3027 * include remote zones now, before entering the slowpath and waking 3028 * kswapd: prefer spilling to a remote zone over swapping locally. 3029 */ 3030 if (fair_skipped) { 3031 reset_fair: 3032 apply_fair = false; 3033 fair_skipped = false; 3034 reset_alloc_batches(ac->preferred_zoneref->zone); 3035 z = ac->preferred_zoneref; 3036 goto zonelist_scan; 3037 } 3038 3039 return NULL; 3040 } 3041 3042 /* 3043 * Large machines with many possible nodes should not always dump per-node 3044 * meminfo in irq context. 3045 */ 3046 static inline bool should_suppress_show_mem(void) 3047 { 3048 bool ret = false; 3049 3050 #if NODES_SHIFT > 8 3051 ret = in_interrupt(); 3052 #endif 3053 return ret; 3054 } 3055 3056 static DEFINE_RATELIMIT_STATE(nopage_rs, 3057 DEFAULT_RATELIMIT_INTERVAL, 3058 DEFAULT_RATELIMIT_BURST); 3059 3060 void warn_alloc_failed(gfp_t gfp_mask, unsigned int order, const char *fmt, ...) 3061 { 3062 unsigned int filter = SHOW_MEM_FILTER_NODES; 3063 3064 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) || 3065 debug_guardpage_minorder() > 0) 3066 return; 3067 3068 /* 3069 * This documents exceptions given to allocations in certain 3070 * contexts that are allowed to allocate outside current's set 3071 * of allowed nodes. 3072 */ 3073 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3074 if (test_thread_flag(TIF_MEMDIE) || 3075 (current->flags & (PF_MEMALLOC | PF_EXITING))) 3076 filter &= ~SHOW_MEM_FILTER_NODES; 3077 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 3078 filter &= ~SHOW_MEM_FILTER_NODES; 3079 3080 if (fmt) { 3081 struct va_format vaf; 3082 va_list args; 3083 3084 va_start(args, fmt); 3085 3086 vaf.fmt = fmt; 3087 vaf.va = &args; 3088 3089 pr_warn("%pV", &vaf); 3090 3091 va_end(args); 3092 } 3093 3094 pr_warn("%s: page allocation failure: order:%u, mode:%#x(%pGg)\n", 3095 current->comm, order, gfp_mask, &gfp_mask); 3096 dump_stack(); 3097 if (!should_suppress_show_mem()) 3098 show_mem(filter); 3099 } 3100 3101 static inline struct page * 3102 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 3103 const struct alloc_context *ac, unsigned long *did_some_progress) 3104 { 3105 struct oom_control oc = { 3106 .zonelist = ac->zonelist, 3107 .nodemask = ac->nodemask, 3108 .gfp_mask = gfp_mask, 3109 .order = order, 3110 }; 3111 struct page *page; 3112 3113 *did_some_progress = 0; 3114 3115 /* 3116 * Acquire the oom lock. If that fails, somebody else is 3117 * making progress for us. 3118 */ 3119 if (!mutex_trylock(&oom_lock)) { 3120 *did_some_progress = 1; 3121 schedule_timeout_uninterruptible(1); 3122 return NULL; 3123 } 3124 3125 /* 3126 * Go through the zonelist yet one more time, keep very high watermark 3127 * here, this is only to catch a parallel oom killing, we must fail if 3128 * we're still under heavy pressure. 3129 */ 3130 page = get_page_from_freelist(gfp_mask | __GFP_HARDWALL, order, 3131 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 3132 if (page) 3133 goto out; 3134 3135 if (!(gfp_mask & __GFP_NOFAIL)) { 3136 /* Coredumps can quickly deplete all memory reserves */ 3137 if (current->flags & PF_DUMPCORE) 3138 goto out; 3139 /* The OOM killer will not help higher order allocs */ 3140 if (order > PAGE_ALLOC_COSTLY_ORDER) 3141 goto out; 3142 /* The OOM killer does not needlessly kill tasks for lowmem */ 3143 if (ac->high_zoneidx < ZONE_NORMAL) 3144 goto out; 3145 if (pm_suspended_storage()) 3146 goto out; 3147 /* 3148 * XXX: GFP_NOFS allocations should rather fail than rely on 3149 * other request to make a forward progress. 3150 * We are in an unfortunate situation where out_of_memory cannot 3151 * do much for this context but let's try it to at least get 3152 * access to memory reserved if the current task is killed (see 3153 * out_of_memory). Once filesystems are ready to handle allocation 3154 * failures more gracefully we should just bail out here. 3155 */ 3156 3157 /* The OOM killer may not free memory on a specific node */ 3158 if (gfp_mask & __GFP_THISNODE) 3159 goto out; 3160 } 3161 /* Exhausted what can be done so it's blamo time */ 3162 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 3163 *did_some_progress = 1; 3164 3165 if (gfp_mask & __GFP_NOFAIL) { 3166 page = get_page_from_freelist(gfp_mask, order, 3167 ALLOC_NO_WATERMARKS|ALLOC_CPUSET, ac); 3168 /* 3169 * fallback to ignore cpuset restriction if our nodes 3170 * are depleted 3171 */ 3172 if (!page) 3173 page = get_page_from_freelist(gfp_mask, order, 3174 ALLOC_NO_WATERMARKS, ac); 3175 } 3176 } 3177 out: 3178 mutex_unlock(&oom_lock); 3179 return page; 3180 } 3181 3182 3183 /* 3184 * Maximum number of compaction retries wit a progress before OOM 3185 * killer is consider as the only way to move forward. 3186 */ 3187 #define MAX_COMPACT_RETRIES 16 3188 3189 #ifdef CONFIG_COMPACTION 3190 /* Try memory compaction for high-order allocations before reclaim */ 3191 static struct page * 3192 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3193 unsigned int alloc_flags, const struct alloc_context *ac, 3194 enum migrate_mode mode, enum compact_result *compact_result) 3195 { 3196 struct page *page; 3197 int contended_compaction; 3198 3199 if (!order) 3200 return NULL; 3201 3202 current->flags |= PF_MEMALLOC; 3203 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 3204 mode, &contended_compaction); 3205 current->flags &= ~PF_MEMALLOC; 3206 3207 if (*compact_result <= COMPACT_INACTIVE) 3208 return NULL; 3209 3210 /* 3211 * At least in one zone compaction wasn't deferred or skipped, so let's 3212 * count a compaction stall 3213 */ 3214 count_vm_event(COMPACTSTALL); 3215 3216 page = get_page_from_freelist(gfp_mask, order, 3217 alloc_flags & ~ALLOC_NO_WATERMARKS, ac); 3218 3219 if (page) { 3220 struct zone *zone = page_zone(page); 3221 3222 zone->compact_blockskip_flush = false; 3223 compaction_defer_reset(zone, order, true); 3224 count_vm_event(COMPACTSUCCESS); 3225 return page; 3226 } 3227 3228 /* 3229 * It's bad if compaction run occurs and fails. The most likely reason 3230 * is that pages exist, but not enough to satisfy watermarks. 3231 */ 3232 count_vm_event(COMPACTFAIL); 3233 3234 /* 3235 * In all zones where compaction was attempted (and not 3236 * deferred or skipped), lock contention has been detected. 3237 * For THP allocation we do not want to disrupt the others 3238 * so we fallback to base pages instead. 3239 */ 3240 if (contended_compaction == COMPACT_CONTENDED_LOCK) 3241 *compact_result = COMPACT_CONTENDED; 3242 3243 /* 3244 * If compaction was aborted due to need_resched(), we do not 3245 * want to further increase allocation latency, unless it is 3246 * khugepaged trying to collapse. 3247 */ 3248 if (contended_compaction == COMPACT_CONTENDED_SCHED 3249 && !(current->flags & PF_KTHREAD)) 3250 *compact_result = COMPACT_CONTENDED; 3251 3252 cond_resched(); 3253 3254 return NULL; 3255 } 3256 3257 static inline bool 3258 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 3259 enum compact_result compact_result, enum migrate_mode *migrate_mode, 3260 int compaction_retries) 3261 { 3262 int max_retries = MAX_COMPACT_RETRIES; 3263 3264 if (!order) 3265 return false; 3266 3267 /* 3268 * compaction considers all the zone as desperately out of memory 3269 * so it doesn't really make much sense to retry except when the 3270 * failure could be caused by weak migration mode. 3271 */ 3272 if (compaction_failed(compact_result)) { 3273 if (*migrate_mode == MIGRATE_ASYNC) { 3274 *migrate_mode = MIGRATE_SYNC_LIGHT; 3275 return true; 3276 } 3277 return false; 3278 } 3279 3280 /* 3281 * make sure the compaction wasn't deferred or didn't bail out early 3282 * due to locks contention before we declare that we should give up. 3283 * But do not retry if the given zonelist is not suitable for 3284 * compaction. 3285 */ 3286 if (compaction_withdrawn(compact_result)) 3287 return compaction_zonelist_suitable(ac, order, alloc_flags); 3288 3289 /* 3290 * !costly requests are much more important than __GFP_REPEAT 3291 * costly ones because they are de facto nofail and invoke OOM 3292 * killer to move on while costly can fail and users are ready 3293 * to cope with that. 1/4 retries is rather arbitrary but we 3294 * would need much more detailed feedback from compaction to 3295 * make a better decision. 3296 */ 3297 if (order > PAGE_ALLOC_COSTLY_ORDER) 3298 max_retries /= 4; 3299 if (compaction_retries <= max_retries) 3300 return true; 3301 3302 return false; 3303 } 3304 #else 3305 static inline struct page * 3306 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3307 unsigned int alloc_flags, const struct alloc_context *ac, 3308 enum migrate_mode mode, enum compact_result *compact_result) 3309 { 3310 *compact_result = COMPACT_SKIPPED; 3311 return NULL; 3312 } 3313 3314 static inline bool 3315 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 3316 enum compact_result compact_result, 3317 enum migrate_mode *migrate_mode, 3318 int compaction_retries) 3319 { 3320 struct zone *zone; 3321 struct zoneref *z; 3322 3323 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 3324 return false; 3325 3326 /* 3327 * There are setups with compaction disabled which would prefer to loop 3328 * inside the allocator rather than hit the oom killer prematurely. 3329 * Let's give them a good hope and keep retrying while the order-0 3330 * watermarks are OK. 3331 */ 3332 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3333 ac->nodemask) { 3334 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 3335 ac_classzone_idx(ac), alloc_flags)) 3336 return true; 3337 } 3338 return false; 3339 } 3340 #endif /* CONFIG_COMPACTION */ 3341 3342 /* Perform direct synchronous page reclaim */ 3343 static int 3344 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 3345 const struct alloc_context *ac) 3346 { 3347 struct reclaim_state reclaim_state; 3348 int progress; 3349 3350 cond_resched(); 3351 3352 /* We now go into synchronous reclaim */ 3353 cpuset_memory_pressure_bump(); 3354 current->flags |= PF_MEMALLOC; 3355 lockdep_set_current_reclaim_state(gfp_mask); 3356 reclaim_state.reclaimed_slab = 0; 3357 current->reclaim_state = &reclaim_state; 3358 3359 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 3360 ac->nodemask); 3361 3362 current->reclaim_state = NULL; 3363 lockdep_clear_current_reclaim_state(); 3364 current->flags &= ~PF_MEMALLOC; 3365 3366 cond_resched(); 3367 3368 return progress; 3369 } 3370 3371 /* The really slow allocator path where we enter direct reclaim */ 3372 static inline struct page * 3373 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 3374 unsigned int alloc_flags, const struct alloc_context *ac, 3375 unsigned long *did_some_progress) 3376 { 3377 struct page *page = NULL; 3378 bool drained = false; 3379 3380 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 3381 if (unlikely(!(*did_some_progress))) 3382 return NULL; 3383 3384 retry: 3385 page = get_page_from_freelist(gfp_mask, order, 3386 alloc_flags & ~ALLOC_NO_WATERMARKS, ac); 3387 3388 /* 3389 * If an allocation failed after direct reclaim, it could be because 3390 * pages are pinned on the per-cpu lists or in high alloc reserves. 3391 * Shrink them them and try again 3392 */ 3393 if (!page && !drained) { 3394 unreserve_highatomic_pageblock(ac); 3395 drain_all_pages(NULL); 3396 drained = true; 3397 goto retry; 3398 } 3399 3400 return page; 3401 } 3402 3403 static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac) 3404 { 3405 struct zoneref *z; 3406 struct zone *zone; 3407 3408 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 3409 ac->high_zoneidx, ac->nodemask) 3410 wakeup_kswapd(zone, order, ac_classzone_idx(ac)); 3411 } 3412 3413 static inline unsigned int 3414 gfp_to_alloc_flags(gfp_t gfp_mask) 3415 { 3416 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 3417 3418 /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */ 3419 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 3420 3421 /* 3422 * The caller may dip into page reserves a bit more if the caller 3423 * cannot run direct reclaim, or if the caller has realtime scheduling 3424 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 3425 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 3426 */ 3427 alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); 3428 3429 if (gfp_mask & __GFP_ATOMIC) { 3430 /* 3431 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 3432 * if it can't schedule. 3433 */ 3434 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3435 alloc_flags |= ALLOC_HARDER; 3436 /* 3437 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 3438 * comment for __cpuset_node_allowed(). 3439 */ 3440 alloc_flags &= ~ALLOC_CPUSET; 3441 } else if (unlikely(rt_task(current)) && !in_interrupt()) 3442 alloc_flags |= ALLOC_HARDER; 3443 3444 if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) { 3445 if (gfp_mask & __GFP_MEMALLOC) 3446 alloc_flags |= ALLOC_NO_WATERMARKS; 3447 else if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 3448 alloc_flags |= ALLOC_NO_WATERMARKS; 3449 else if (!in_interrupt() && 3450 ((current->flags & PF_MEMALLOC) || 3451 unlikely(test_thread_flag(TIF_MEMDIE)))) 3452 alloc_flags |= ALLOC_NO_WATERMARKS; 3453 } 3454 #ifdef CONFIG_CMA 3455 if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) 3456 alloc_flags |= ALLOC_CMA; 3457 #endif 3458 return alloc_flags; 3459 } 3460 3461 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 3462 { 3463 return !!(gfp_to_alloc_flags(gfp_mask) & ALLOC_NO_WATERMARKS); 3464 } 3465 3466 static inline bool is_thp_gfp_mask(gfp_t gfp_mask) 3467 { 3468 return (gfp_mask & (GFP_TRANSHUGE | __GFP_KSWAPD_RECLAIM)) == GFP_TRANSHUGE; 3469 } 3470 3471 /* 3472 * Maximum number of reclaim retries without any progress before OOM killer 3473 * is consider as the only way to move forward. 3474 */ 3475 #define MAX_RECLAIM_RETRIES 16 3476 3477 /* 3478 * Checks whether it makes sense to retry the reclaim to make a forward progress 3479 * for the given allocation request. 3480 * The reclaim feedback represented by did_some_progress (any progress during 3481 * the last reclaim round) and no_progress_loops (number of reclaim rounds without 3482 * any progress in a row) is considered as well as the reclaimable pages on the 3483 * applicable zone list (with a backoff mechanism which is a function of 3484 * no_progress_loops). 3485 * 3486 * Returns true if a retry is viable or false to enter the oom path. 3487 */ 3488 static inline bool 3489 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 3490 struct alloc_context *ac, int alloc_flags, 3491 bool did_some_progress, int no_progress_loops) 3492 { 3493 struct zone *zone; 3494 struct zoneref *z; 3495 3496 /* 3497 * Make sure we converge to OOM if we cannot make any progress 3498 * several times in the row. 3499 */ 3500 if (no_progress_loops > MAX_RECLAIM_RETRIES) 3501 return false; 3502 3503 /* 3504 * Keep reclaiming pages while there is a chance this will lead somewhere. 3505 * If none of the target zones can satisfy our allocation request even 3506 * if all reclaimable pages are considered then we are screwed and have 3507 * to go OOM. 3508 */ 3509 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3510 ac->nodemask) { 3511 unsigned long available; 3512 unsigned long reclaimable; 3513 3514 available = reclaimable = zone_reclaimable_pages(zone); 3515 available -= DIV_ROUND_UP(no_progress_loops * available, 3516 MAX_RECLAIM_RETRIES); 3517 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 3518 3519 /* 3520 * Would the allocation succeed if we reclaimed the whole 3521 * available? 3522 */ 3523 if (__zone_watermark_ok(zone, order, min_wmark_pages(zone), 3524 ac_classzone_idx(ac), alloc_flags, available)) { 3525 /* 3526 * If we didn't make any progress and have a lot of 3527 * dirty + writeback pages then we should wait for 3528 * an IO to complete to slow down the reclaim and 3529 * prevent from pre mature OOM 3530 */ 3531 if (!did_some_progress) { 3532 unsigned long writeback; 3533 unsigned long dirty; 3534 3535 writeback = zone_page_state_snapshot(zone, 3536 NR_WRITEBACK); 3537 dirty = zone_page_state_snapshot(zone, NR_FILE_DIRTY); 3538 3539 if (2*(writeback + dirty) > reclaimable) { 3540 congestion_wait(BLK_RW_ASYNC, HZ/10); 3541 return true; 3542 } 3543 } 3544 3545 /* 3546 * Memory allocation/reclaim might be called from a WQ 3547 * context and the current implementation of the WQ 3548 * concurrency control doesn't recognize that 3549 * a particular WQ is congested if the worker thread is 3550 * looping without ever sleeping. Therefore we have to 3551 * do a short sleep here rather than calling 3552 * cond_resched(). 3553 */ 3554 if (current->flags & PF_WQ_WORKER) 3555 schedule_timeout_uninterruptible(1); 3556 else 3557 cond_resched(); 3558 3559 return true; 3560 } 3561 } 3562 3563 return false; 3564 } 3565 3566 static inline struct page * 3567 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 3568 struct alloc_context *ac) 3569 { 3570 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 3571 struct page *page = NULL; 3572 unsigned int alloc_flags; 3573 unsigned long did_some_progress; 3574 enum migrate_mode migration_mode = MIGRATE_ASYNC; 3575 enum compact_result compact_result; 3576 int compaction_retries = 0; 3577 int no_progress_loops = 0; 3578 3579 /* 3580 * In the slowpath, we sanity check order to avoid ever trying to 3581 * reclaim >= MAX_ORDER areas which will never succeed. Callers may 3582 * be using allocators in order of preference for an area that is 3583 * too large. 3584 */ 3585 if (order >= MAX_ORDER) { 3586 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); 3587 return NULL; 3588 } 3589 3590 /* 3591 * We also sanity check to catch abuse of atomic reserves being used by 3592 * callers that are not in atomic context. 3593 */ 3594 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 3595 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 3596 gfp_mask &= ~__GFP_ATOMIC; 3597 3598 retry: 3599 if (gfp_mask & __GFP_KSWAPD_RECLAIM) 3600 wake_all_kswapds(order, ac); 3601 3602 /* 3603 * OK, we're below the kswapd watermark and have kicked background 3604 * reclaim. Now things get more complex, so set up alloc_flags according 3605 * to how we want to proceed. 3606 */ 3607 alloc_flags = gfp_to_alloc_flags(gfp_mask); 3608 3609 /* 3610 * Reset the zonelist iterators if memory policies can be ignored. 3611 * These allocations are high priority and system rather than user 3612 * orientated. 3613 */ 3614 if ((alloc_flags & ALLOC_NO_WATERMARKS) || !(alloc_flags & ALLOC_CPUSET)) { 3615 ac->zonelist = node_zonelist(numa_node_id(), gfp_mask); 3616 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 3617 ac->high_zoneidx, ac->nodemask); 3618 } 3619 3620 /* This is the last chance, in general, before the goto nopage. */ 3621 page = get_page_from_freelist(gfp_mask, order, 3622 alloc_flags & ~ALLOC_NO_WATERMARKS, ac); 3623 if (page) 3624 goto got_pg; 3625 3626 /* Allocate without watermarks if the context allows */ 3627 if (alloc_flags & ALLOC_NO_WATERMARKS) { 3628 page = get_page_from_freelist(gfp_mask, order, 3629 ALLOC_NO_WATERMARKS, ac); 3630 if (page) 3631 goto got_pg; 3632 } 3633 3634 /* Caller is not willing to reclaim, we can't balance anything */ 3635 if (!can_direct_reclaim) { 3636 /* 3637 * All existing users of the __GFP_NOFAIL are blockable, so warn 3638 * of any new users that actually allow this type of allocation 3639 * to fail. 3640 */ 3641 WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL); 3642 goto nopage; 3643 } 3644 3645 /* Avoid recursion of direct reclaim */ 3646 if (current->flags & PF_MEMALLOC) { 3647 /* 3648 * __GFP_NOFAIL request from this context is rather bizarre 3649 * because we cannot reclaim anything and only can loop waiting 3650 * for somebody to do a work for us. 3651 */ 3652 if (WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 3653 cond_resched(); 3654 goto retry; 3655 } 3656 goto nopage; 3657 } 3658 3659 /* Avoid allocations with no watermarks from looping endlessly */ 3660 if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL)) 3661 goto nopage; 3662 3663 /* 3664 * Try direct compaction. The first pass is asynchronous. Subsequent 3665 * attempts after direct reclaim are synchronous 3666 */ 3667 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 3668 migration_mode, 3669 &compact_result); 3670 if (page) 3671 goto got_pg; 3672 3673 /* Checks for THP-specific high-order allocations */ 3674 if (is_thp_gfp_mask(gfp_mask)) { 3675 /* 3676 * If compaction is deferred for high-order allocations, it is 3677 * because sync compaction recently failed. If this is the case 3678 * and the caller requested a THP allocation, we do not want 3679 * to heavily disrupt the system, so we fail the allocation 3680 * instead of entering direct reclaim. 3681 */ 3682 if (compact_result == COMPACT_DEFERRED) 3683 goto nopage; 3684 3685 /* 3686 * Compaction is contended so rather back off than cause 3687 * excessive stalls. 3688 */ 3689 if(compact_result == COMPACT_CONTENDED) 3690 goto nopage; 3691 } 3692 3693 if (order && compaction_made_progress(compact_result)) 3694 compaction_retries++; 3695 3696 /* Try direct reclaim and then allocating */ 3697 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 3698 &did_some_progress); 3699 if (page) 3700 goto got_pg; 3701 3702 /* Do not loop if specifically requested */ 3703 if (gfp_mask & __GFP_NORETRY) 3704 goto noretry; 3705 3706 /* 3707 * Do not retry costly high order allocations unless they are 3708 * __GFP_REPEAT 3709 */ 3710 if (order > PAGE_ALLOC_COSTLY_ORDER && !(gfp_mask & __GFP_REPEAT)) 3711 goto noretry; 3712 3713 /* 3714 * Costly allocations might have made a progress but this doesn't mean 3715 * their order will become available due to high fragmentation so 3716 * always increment the no progress counter for them 3717 */ 3718 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 3719 no_progress_loops = 0; 3720 else 3721 no_progress_loops++; 3722 3723 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 3724 did_some_progress > 0, no_progress_loops)) 3725 goto retry; 3726 3727 /* 3728 * It doesn't make any sense to retry for the compaction if the order-0 3729 * reclaim is not able to make any progress because the current 3730 * implementation of the compaction depends on the sufficient amount 3731 * of free memory (see __compaction_suitable) 3732 */ 3733 if (did_some_progress > 0 && 3734 should_compact_retry(ac, order, alloc_flags, 3735 compact_result, &migration_mode, 3736 compaction_retries)) 3737 goto retry; 3738 3739 /* Reclaim has failed us, start killing things */ 3740 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 3741 if (page) 3742 goto got_pg; 3743 3744 /* Retry as long as the OOM killer is making progress */ 3745 if (did_some_progress) { 3746 no_progress_loops = 0; 3747 goto retry; 3748 } 3749 3750 noretry: 3751 /* 3752 * High-order allocations do not necessarily loop after direct reclaim 3753 * and reclaim/compaction depends on compaction being called after 3754 * reclaim so call directly if necessary. 3755 * It can become very expensive to allocate transparent hugepages at 3756 * fault, so use asynchronous memory compaction for THP unless it is 3757 * khugepaged trying to collapse. All other requests should tolerate 3758 * at least light sync migration. 3759 */ 3760 if (is_thp_gfp_mask(gfp_mask) && !(current->flags & PF_KTHREAD)) 3761 migration_mode = MIGRATE_ASYNC; 3762 else 3763 migration_mode = MIGRATE_SYNC_LIGHT; 3764 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, 3765 ac, migration_mode, 3766 &compact_result); 3767 if (page) 3768 goto got_pg; 3769 nopage: 3770 warn_alloc_failed(gfp_mask, order, NULL); 3771 got_pg: 3772 return page; 3773 } 3774 3775 /* 3776 * This is the 'heart' of the zoned buddy allocator. 3777 */ 3778 struct page * 3779 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, 3780 struct zonelist *zonelist, nodemask_t *nodemask) 3781 { 3782 struct page *page; 3783 unsigned int cpuset_mems_cookie; 3784 unsigned int alloc_flags = ALLOC_WMARK_LOW|ALLOC_FAIR; 3785 gfp_t alloc_mask = gfp_mask; /* The gfp_t that was actually used for allocation */ 3786 struct alloc_context ac = { 3787 .high_zoneidx = gfp_zone(gfp_mask), 3788 .zonelist = zonelist, 3789 .nodemask = nodemask, 3790 .migratetype = gfpflags_to_migratetype(gfp_mask), 3791 }; 3792 3793 if (cpusets_enabled()) { 3794 alloc_mask |= __GFP_HARDWALL; 3795 alloc_flags |= ALLOC_CPUSET; 3796 if (!ac.nodemask) 3797 ac.nodemask = &cpuset_current_mems_allowed; 3798 } 3799 3800 gfp_mask &= gfp_allowed_mask; 3801 3802 lockdep_trace_alloc(gfp_mask); 3803 3804 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 3805 3806 if (should_fail_alloc_page(gfp_mask, order)) 3807 return NULL; 3808 3809 /* 3810 * Check the zones suitable for the gfp_mask contain at least one 3811 * valid zone. It's possible to have an empty zonelist as a result 3812 * of __GFP_THISNODE and a memoryless node 3813 */ 3814 if (unlikely(!zonelist->_zonerefs->zone)) 3815 return NULL; 3816 3817 if (IS_ENABLED(CONFIG_CMA) && ac.migratetype == MIGRATE_MOVABLE) 3818 alloc_flags |= ALLOC_CMA; 3819 3820 retry_cpuset: 3821 cpuset_mems_cookie = read_mems_allowed_begin(); 3822 3823 /* Dirty zone balancing only done in the fast path */ 3824 ac.spread_dirty_pages = (gfp_mask & __GFP_WRITE); 3825 3826 /* 3827 * The preferred zone is used for statistics but crucially it is 3828 * also used as the starting point for the zonelist iterator. It 3829 * may get reset for allocations that ignore memory policies. 3830 */ 3831 ac.preferred_zoneref = first_zones_zonelist(ac.zonelist, 3832 ac.high_zoneidx, ac.nodemask); 3833 if (!ac.preferred_zoneref) { 3834 page = NULL; 3835 goto no_zone; 3836 } 3837 3838 /* First allocation attempt */ 3839 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); 3840 if (likely(page)) 3841 goto out; 3842 3843 /* 3844 * Runtime PM, block IO and its error handling path can deadlock 3845 * because I/O on the device might not complete. 3846 */ 3847 alloc_mask = memalloc_noio_flags(gfp_mask); 3848 ac.spread_dirty_pages = false; 3849 3850 /* 3851 * Restore the original nodemask if it was potentially replaced with 3852 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 3853 */ 3854 if (cpusets_enabled()) 3855 ac.nodemask = nodemask; 3856 page = __alloc_pages_slowpath(alloc_mask, order, &ac); 3857 3858 no_zone: 3859 /* 3860 * When updating a task's mems_allowed, it is possible to race with 3861 * parallel threads in such a way that an allocation can fail while 3862 * the mask is being updated. If a page allocation is about to fail, 3863 * check if the cpuset changed during allocation and if so, retry. 3864 */ 3865 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) { 3866 alloc_mask = gfp_mask; 3867 goto retry_cpuset; 3868 } 3869 3870 out: 3871 if (kmemcheck_enabled && page) 3872 kmemcheck_pagealloc_alloc(page, order, gfp_mask); 3873 3874 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); 3875 3876 return page; 3877 } 3878 EXPORT_SYMBOL(__alloc_pages_nodemask); 3879 3880 /* 3881 * Common helper functions. 3882 */ 3883 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 3884 { 3885 struct page *page; 3886 3887 /* 3888 * __get_free_pages() returns a 32-bit address, which cannot represent 3889 * a highmem page 3890 */ 3891 VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0); 3892 3893 page = alloc_pages(gfp_mask, order); 3894 if (!page) 3895 return 0; 3896 return (unsigned long) page_address(page); 3897 } 3898 EXPORT_SYMBOL(__get_free_pages); 3899 3900 unsigned long get_zeroed_page(gfp_t gfp_mask) 3901 { 3902 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 3903 } 3904 EXPORT_SYMBOL(get_zeroed_page); 3905 3906 void __free_pages(struct page *page, unsigned int order) 3907 { 3908 if (put_page_testzero(page)) { 3909 if (order == 0) 3910 free_hot_cold_page(page, false); 3911 else 3912 __free_pages_ok(page, order); 3913 } 3914 } 3915 3916 EXPORT_SYMBOL(__free_pages); 3917 3918 void free_pages(unsigned long addr, unsigned int order) 3919 { 3920 if (addr != 0) { 3921 VM_BUG_ON(!virt_addr_valid((void *)addr)); 3922 __free_pages(virt_to_page((void *)addr), order); 3923 } 3924 } 3925 3926 EXPORT_SYMBOL(free_pages); 3927 3928 /* 3929 * Page Fragment: 3930 * An arbitrary-length arbitrary-offset area of memory which resides 3931 * within a 0 or higher order page. Multiple fragments within that page 3932 * are individually refcounted, in the page's reference counter. 3933 * 3934 * The page_frag functions below provide a simple allocation framework for 3935 * page fragments. This is used by the network stack and network device 3936 * drivers to provide a backing region of memory for use as either an 3937 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 3938 */ 3939 static struct page *__page_frag_refill(struct page_frag_cache *nc, 3940 gfp_t gfp_mask) 3941 { 3942 struct page *page = NULL; 3943 gfp_t gfp = gfp_mask; 3944 3945 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 3946 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 3947 __GFP_NOMEMALLOC; 3948 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 3949 PAGE_FRAG_CACHE_MAX_ORDER); 3950 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 3951 #endif 3952 if (unlikely(!page)) 3953 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 3954 3955 nc->va = page ? page_address(page) : NULL; 3956 3957 return page; 3958 } 3959 3960 void *__alloc_page_frag(struct page_frag_cache *nc, 3961 unsigned int fragsz, gfp_t gfp_mask) 3962 { 3963 unsigned int size = PAGE_SIZE; 3964 struct page *page; 3965 int offset; 3966 3967 if (unlikely(!nc->va)) { 3968 refill: 3969 page = __page_frag_refill(nc, gfp_mask); 3970 if (!page) 3971 return NULL; 3972 3973 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 3974 /* if size can vary use size else just use PAGE_SIZE */ 3975 size = nc->size; 3976 #endif 3977 /* Even if we own the page, we do not use atomic_set(). 3978 * This would break get_page_unless_zero() users. 3979 */ 3980 page_ref_add(page, size - 1); 3981 3982 /* reset page count bias and offset to start of new frag */ 3983 nc->pfmemalloc = page_is_pfmemalloc(page); 3984 nc->pagecnt_bias = size; 3985 nc->offset = size; 3986 } 3987 3988 offset = nc->offset - fragsz; 3989 if (unlikely(offset < 0)) { 3990 page = virt_to_page(nc->va); 3991 3992 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 3993 goto refill; 3994 3995 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 3996 /* if size can vary use size else just use PAGE_SIZE */ 3997 size = nc->size; 3998 #endif 3999 /* OK, page count is 0, we can safely set it */ 4000 set_page_count(page, size); 4001 4002 /* reset page count bias and offset to start of new frag */ 4003 nc->pagecnt_bias = size; 4004 offset = size - fragsz; 4005 } 4006 4007 nc->pagecnt_bias--; 4008 nc->offset = offset; 4009 4010 return nc->va + offset; 4011 } 4012 EXPORT_SYMBOL(__alloc_page_frag); 4013 4014 /* 4015 * Frees a page fragment allocated out of either a compound or order 0 page. 4016 */ 4017 void __free_page_frag(void *addr) 4018 { 4019 struct page *page = virt_to_head_page(addr); 4020 4021 if (unlikely(put_page_testzero(page))) 4022 __free_pages_ok(page, compound_order(page)); 4023 } 4024 EXPORT_SYMBOL(__free_page_frag); 4025 4026 /* 4027 * alloc_kmem_pages charges newly allocated pages to the kmem resource counter 4028 * of the current memory cgroup if __GFP_ACCOUNT is set, other than that it is 4029 * equivalent to alloc_pages. 4030 * 4031 * It should be used when the caller would like to use kmalloc, but since the 4032 * allocation is large, it has to fall back to the page allocator. 4033 */ 4034 struct page *alloc_kmem_pages(gfp_t gfp_mask, unsigned int order) 4035 { 4036 struct page *page; 4037 4038 page = alloc_pages(gfp_mask, order); 4039 if (page && memcg_kmem_charge(page, gfp_mask, order) != 0) { 4040 __free_pages(page, order); 4041 page = NULL; 4042 } 4043 return page; 4044 } 4045 4046 struct page *alloc_kmem_pages_node(int nid, gfp_t gfp_mask, unsigned int order) 4047 { 4048 struct page *page; 4049 4050 page = alloc_pages_node(nid, gfp_mask, order); 4051 if (page && memcg_kmem_charge(page, gfp_mask, order) != 0) { 4052 __free_pages(page, order); 4053 page = NULL; 4054 } 4055 return page; 4056 } 4057 4058 /* 4059 * __free_kmem_pages and free_kmem_pages will free pages allocated with 4060 * alloc_kmem_pages. 4061 */ 4062 void __free_kmem_pages(struct page *page, unsigned int order) 4063 { 4064 memcg_kmem_uncharge(page, order); 4065 __free_pages(page, order); 4066 } 4067 4068 void free_kmem_pages(unsigned long addr, unsigned int order) 4069 { 4070 if (addr != 0) { 4071 VM_BUG_ON(!virt_addr_valid((void *)addr)); 4072 __free_kmem_pages(virt_to_page((void *)addr), order); 4073 } 4074 } 4075 4076 static void *make_alloc_exact(unsigned long addr, unsigned int order, 4077 size_t size) 4078 { 4079 if (addr) { 4080 unsigned long alloc_end = addr + (PAGE_SIZE << order); 4081 unsigned long used = addr + PAGE_ALIGN(size); 4082 4083 split_page(virt_to_page((void *)addr), order); 4084 while (used < alloc_end) { 4085 free_page(used); 4086 used += PAGE_SIZE; 4087 } 4088 } 4089 return (void *)addr; 4090 } 4091 4092 /** 4093 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 4094 * @size: the number of bytes to allocate 4095 * @gfp_mask: GFP flags for the allocation 4096 * 4097 * This function is similar to alloc_pages(), except that it allocates the 4098 * minimum number of pages to satisfy the request. alloc_pages() can only 4099 * allocate memory in power-of-two pages. 4100 * 4101 * This function is also limited by MAX_ORDER. 4102 * 4103 * Memory allocated by this function must be released by free_pages_exact(). 4104 */ 4105 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 4106 { 4107 unsigned int order = get_order(size); 4108 unsigned long addr; 4109 4110 addr = __get_free_pages(gfp_mask, order); 4111 return make_alloc_exact(addr, order, size); 4112 } 4113 EXPORT_SYMBOL(alloc_pages_exact); 4114 4115 /** 4116 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 4117 * pages on a node. 4118 * @nid: the preferred node ID where memory should be allocated 4119 * @size: the number of bytes to allocate 4120 * @gfp_mask: GFP flags for the allocation 4121 * 4122 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 4123 * back. 4124 */ 4125 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 4126 { 4127 unsigned int order = get_order(size); 4128 struct page *p = alloc_pages_node(nid, gfp_mask, order); 4129 if (!p) 4130 return NULL; 4131 return make_alloc_exact((unsigned long)page_address(p), order, size); 4132 } 4133 4134 /** 4135 * free_pages_exact - release memory allocated via alloc_pages_exact() 4136 * @virt: the value returned by alloc_pages_exact. 4137 * @size: size of allocation, same value as passed to alloc_pages_exact(). 4138 * 4139 * Release the memory allocated by a previous call to alloc_pages_exact. 4140 */ 4141 void free_pages_exact(void *virt, size_t size) 4142 { 4143 unsigned long addr = (unsigned long)virt; 4144 unsigned long end = addr + PAGE_ALIGN(size); 4145 4146 while (addr < end) { 4147 free_page(addr); 4148 addr += PAGE_SIZE; 4149 } 4150 } 4151 EXPORT_SYMBOL(free_pages_exact); 4152 4153 /** 4154 * nr_free_zone_pages - count number of pages beyond high watermark 4155 * @offset: The zone index of the highest zone 4156 * 4157 * nr_free_zone_pages() counts the number of counts pages which are beyond the 4158 * high watermark within all zones at or below a given zone index. For each 4159 * zone, the number of pages is calculated as: 4160 * managed_pages - high_pages 4161 */ 4162 static unsigned long nr_free_zone_pages(int offset) 4163 { 4164 struct zoneref *z; 4165 struct zone *zone; 4166 4167 /* Just pick one node, since fallback list is circular */ 4168 unsigned long sum = 0; 4169 4170 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 4171 4172 for_each_zone_zonelist(zone, z, zonelist, offset) { 4173 unsigned long size = zone->managed_pages; 4174 unsigned long high = high_wmark_pages(zone); 4175 if (size > high) 4176 sum += size - high; 4177 } 4178 4179 return sum; 4180 } 4181 4182 /** 4183 * nr_free_buffer_pages - count number of pages beyond high watermark 4184 * 4185 * nr_free_buffer_pages() counts the number of pages which are beyond the high 4186 * watermark within ZONE_DMA and ZONE_NORMAL. 4187 */ 4188 unsigned long nr_free_buffer_pages(void) 4189 { 4190 return nr_free_zone_pages(gfp_zone(GFP_USER)); 4191 } 4192 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 4193 4194 /** 4195 * nr_free_pagecache_pages - count number of pages beyond high watermark 4196 * 4197 * nr_free_pagecache_pages() counts the number of pages which are beyond the 4198 * high watermark within all zones. 4199 */ 4200 unsigned long nr_free_pagecache_pages(void) 4201 { 4202 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 4203 } 4204 4205 static inline void show_node(struct zone *zone) 4206 { 4207 if (IS_ENABLED(CONFIG_NUMA)) 4208 printk("Node %d ", zone_to_nid(zone)); 4209 } 4210 4211 long si_mem_available(void) 4212 { 4213 long available; 4214 unsigned long pagecache; 4215 unsigned long wmark_low = 0; 4216 unsigned long pages[NR_LRU_LISTS]; 4217 struct zone *zone; 4218 int lru; 4219 4220 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 4221 pages[lru] = global_page_state(NR_LRU_BASE + lru); 4222 4223 for_each_zone(zone) 4224 wmark_low += zone->watermark[WMARK_LOW]; 4225 4226 /* 4227 * Estimate the amount of memory available for userspace allocations, 4228 * without causing swapping. 4229 */ 4230 available = global_page_state(NR_FREE_PAGES) - totalreserve_pages; 4231 4232 /* 4233 * Not all the page cache can be freed, otherwise the system will 4234 * start swapping. Assume at least half of the page cache, or the 4235 * low watermark worth of cache, needs to stay. 4236 */ 4237 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 4238 pagecache -= min(pagecache / 2, wmark_low); 4239 available += pagecache; 4240 4241 /* 4242 * Part of the reclaimable slab consists of items that are in use, 4243 * and cannot be freed. Cap this estimate at the low watermark. 4244 */ 4245 available += global_page_state(NR_SLAB_RECLAIMABLE) - 4246 min(global_page_state(NR_SLAB_RECLAIMABLE) / 2, wmark_low); 4247 4248 if (available < 0) 4249 available = 0; 4250 return available; 4251 } 4252 EXPORT_SYMBOL_GPL(si_mem_available); 4253 4254 void si_meminfo(struct sysinfo *val) 4255 { 4256 val->totalram = totalram_pages; 4257 val->sharedram = global_page_state(NR_SHMEM); 4258 val->freeram = global_page_state(NR_FREE_PAGES); 4259 val->bufferram = nr_blockdev_pages(); 4260 val->totalhigh = totalhigh_pages; 4261 val->freehigh = nr_free_highpages(); 4262 val->mem_unit = PAGE_SIZE; 4263 } 4264 4265 EXPORT_SYMBOL(si_meminfo); 4266 4267 #ifdef CONFIG_NUMA 4268 void si_meminfo_node(struct sysinfo *val, int nid) 4269 { 4270 int zone_type; /* needs to be signed */ 4271 unsigned long managed_pages = 0; 4272 unsigned long managed_highpages = 0; 4273 unsigned long free_highpages = 0; 4274 pg_data_t *pgdat = NODE_DATA(nid); 4275 4276 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 4277 managed_pages += pgdat->node_zones[zone_type].managed_pages; 4278 val->totalram = managed_pages; 4279 val->sharedram = node_page_state(nid, NR_SHMEM); 4280 val->freeram = node_page_state(nid, NR_FREE_PAGES); 4281 #ifdef CONFIG_HIGHMEM 4282 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 4283 struct zone *zone = &pgdat->node_zones[zone_type]; 4284 4285 if (is_highmem(zone)) { 4286 managed_highpages += zone->managed_pages; 4287 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 4288 } 4289 } 4290 val->totalhigh = managed_highpages; 4291 val->freehigh = free_highpages; 4292 #else 4293 val->totalhigh = managed_highpages; 4294 val->freehigh = free_highpages; 4295 #endif 4296 val->mem_unit = PAGE_SIZE; 4297 } 4298 #endif 4299 4300 /* 4301 * Determine whether the node should be displayed or not, depending on whether 4302 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 4303 */ 4304 bool skip_free_areas_node(unsigned int flags, int nid) 4305 { 4306 bool ret = false; 4307 unsigned int cpuset_mems_cookie; 4308 4309 if (!(flags & SHOW_MEM_FILTER_NODES)) 4310 goto out; 4311 4312 do { 4313 cpuset_mems_cookie = read_mems_allowed_begin(); 4314 ret = !node_isset(nid, cpuset_current_mems_allowed); 4315 } while (read_mems_allowed_retry(cpuset_mems_cookie)); 4316 out: 4317 return ret; 4318 } 4319 4320 #define K(x) ((x) << (PAGE_SHIFT-10)) 4321 4322 static void show_migration_types(unsigned char type) 4323 { 4324 static const char types[MIGRATE_TYPES] = { 4325 [MIGRATE_UNMOVABLE] = 'U', 4326 [MIGRATE_MOVABLE] = 'M', 4327 [MIGRATE_RECLAIMABLE] = 'E', 4328 [MIGRATE_HIGHATOMIC] = 'H', 4329 #ifdef CONFIG_CMA 4330 [MIGRATE_CMA] = 'C', 4331 #endif 4332 #ifdef CONFIG_MEMORY_ISOLATION 4333 [MIGRATE_ISOLATE] = 'I', 4334 #endif 4335 }; 4336 char tmp[MIGRATE_TYPES + 1]; 4337 char *p = tmp; 4338 int i; 4339 4340 for (i = 0; i < MIGRATE_TYPES; i++) { 4341 if (type & (1 << i)) 4342 *p++ = types[i]; 4343 } 4344 4345 *p = '\0'; 4346 printk("(%s) ", tmp); 4347 } 4348 4349 /* 4350 * Show free area list (used inside shift_scroll-lock stuff) 4351 * We also calculate the percentage fragmentation. We do this by counting the 4352 * memory on each free list with the exception of the first item on the list. 4353 * 4354 * Bits in @filter: 4355 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 4356 * cpuset. 4357 */ 4358 void show_free_areas(unsigned int filter) 4359 { 4360 unsigned long free_pcp = 0; 4361 int cpu; 4362 struct zone *zone; 4363 4364 for_each_populated_zone(zone) { 4365 if (skip_free_areas_node(filter, zone_to_nid(zone))) 4366 continue; 4367 4368 for_each_online_cpu(cpu) 4369 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 4370 } 4371 4372 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 4373 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 4374 " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n" 4375 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 4376 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 4377 " free:%lu free_pcp:%lu free_cma:%lu\n", 4378 global_page_state(NR_ACTIVE_ANON), 4379 global_page_state(NR_INACTIVE_ANON), 4380 global_page_state(NR_ISOLATED_ANON), 4381 global_page_state(NR_ACTIVE_FILE), 4382 global_page_state(NR_INACTIVE_FILE), 4383 global_page_state(NR_ISOLATED_FILE), 4384 global_page_state(NR_UNEVICTABLE), 4385 global_page_state(NR_FILE_DIRTY), 4386 global_page_state(NR_WRITEBACK), 4387 global_page_state(NR_UNSTABLE_NFS), 4388 global_page_state(NR_SLAB_RECLAIMABLE), 4389 global_page_state(NR_SLAB_UNRECLAIMABLE), 4390 global_page_state(NR_FILE_MAPPED), 4391 global_page_state(NR_SHMEM), 4392 global_page_state(NR_PAGETABLE), 4393 global_page_state(NR_BOUNCE), 4394 global_page_state(NR_FREE_PAGES), 4395 free_pcp, 4396 global_page_state(NR_FREE_CMA_PAGES)); 4397 4398 for_each_populated_zone(zone) { 4399 int i; 4400 4401 if (skip_free_areas_node(filter, zone_to_nid(zone))) 4402 continue; 4403 4404 free_pcp = 0; 4405 for_each_online_cpu(cpu) 4406 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 4407 4408 show_node(zone); 4409 printk("%s" 4410 " free:%lukB" 4411 " min:%lukB" 4412 " low:%lukB" 4413 " high:%lukB" 4414 " active_anon:%lukB" 4415 " inactive_anon:%lukB" 4416 " active_file:%lukB" 4417 " inactive_file:%lukB" 4418 " unevictable:%lukB" 4419 " isolated(anon):%lukB" 4420 " isolated(file):%lukB" 4421 " present:%lukB" 4422 " managed:%lukB" 4423 " mlocked:%lukB" 4424 " dirty:%lukB" 4425 " writeback:%lukB" 4426 " mapped:%lukB" 4427 " shmem:%lukB" 4428 " slab_reclaimable:%lukB" 4429 " slab_unreclaimable:%lukB" 4430 " kernel_stack:%lukB" 4431 " pagetables:%lukB" 4432 " unstable:%lukB" 4433 " bounce:%lukB" 4434 " free_pcp:%lukB" 4435 " local_pcp:%ukB" 4436 " free_cma:%lukB" 4437 " writeback_tmp:%lukB" 4438 " pages_scanned:%lu" 4439 " all_unreclaimable? %s" 4440 "\n", 4441 zone->name, 4442 K(zone_page_state(zone, NR_FREE_PAGES)), 4443 K(min_wmark_pages(zone)), 4444 K(low_wmark_pages(zone)), 4445 K(high_wmark_pages(zone)), 4446 K(zone_page_state(zone, NR_ACTIVE_ANON)), 4447 K(zone_page_state(zone, NR_INACTIVE_ANON)), 4448 K(zone_page_state(zone, NR_ACTIVE_FILE)), 4449 K(zone_page_state(zone, NR_INACTIVE_FILE)), 4450 K(zone_page_state(zone, NR_UNEVICTABLE)), 4451 K(zone_page_state(zone, NR_ISOLATED_ANON)), 4452 K(zone_page_state(zone, NR_ISOLATED_FILE)), 4453 K(zone->present_pages), 4454 K(zone->managed_pages), 4455 K(zone_page_state(zone, NR_MLOCK)), 4456 K(zone_page_state(zone, NR_FILE_DIRTY)), 4457 K(zone_page_state(zone, NR_WRITEBACK)), 4458 K(zone_page_state(zone, NR_FILE_MAPPED)), 4459 K(zone_page_state(zone, NR_SHMEM)), 4460 K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)), 4461 K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)), 4462 zone_page_state(zone, NR_KERNEL_STACK) * 4463 THREAD_SIZE / 1024, 4464 K(zone_page_state(zone, NR_PAGETABLE)), 4465 K(zone_page_state(zone, NR_UNSTABLE_NFS)), 4466 K(zone_page_state(zone, NR_BOUNCE)), 4467 K(free_pcp), 4468 K(this_cpu_read(zone->pageset->pcp.count)), 4469 K(zone_page_state(zone, NR_FREE_CMA_PAGES)), 4470 K(zone_page_state(zone, NR_WRITEBACK_TEMP)), 4471 K(zone_page_state(zone, NR_PAGES_SCANNED)), 4472 (!zone_reclaimable(zone) ? "yes" : "no") 4473 ); 4474 printk("lowmem_reserve[]:"); 4475 for (i = 0; i < MAX_NR_ZONES; i++) 4476 printk(" %ld", zone->lowmem_reserve[i]); 4477 printk("\n"); 4478 } 4479 4480 for_each_populated_zone(zone) { 4481 unsigned int order; 4482 unsigned long nr[MAX_ORDER], flags, total = 0; 4483 unsigned char types[MAX_ORDER]; 4484 4485 if (skip_free_areas_node(filter, zone_to_nid(zone))) 4486 continue; 4487 show_node(zone); 4488 printk("%s: ", zone->name); 4489 4490 spin_lock_irqsave(&zone->lock, flags); 4491 for (order = 0; order < MAX_ORDER; order++) { 4492 struct free_area *area = &zone->free_area[order]; 4493 int type; 4494 4495 nr[order] = area->nr_free; 4496 total += nr[order] << order; 4497 4498 types[order] = 0; 4499 for (type = 0; type < MIGRATE_TYPES; type++) { 4500 if (!list_empty(&area->free_list[type])) 4501 types[order] |= 1 << type; 4502 } 4503 } 4504 spin_unlock_irqrestore(&zone->lock, flags); 4505 for (order = 0; order < MAX_ORDER; order++) { 4506 printk("%lu*%lukB ", nr[order], K(1UL) << order); 4507 if (nr[order]) 4508 show_migration_types(types[order]); 4509 } 4510 printk("= %lukB\n", K(total)); 4511 } 4512 4513 hugetlb_show_meminfo(); 4514 4515 printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES)); 4516 4517 show_swap_cache_info(); 4518 } 4519 4520 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 4521 { 4522 zoneref->zone = zone; 4523 zoneref->zone_idx = zone_idx(zone); 4524 } 4525 4526 /* 4527 * Builds allocation fallback zone lists. 4528 * 4529 * Add all populated zones of a node to the zonelist. 4530 */ 4531 static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist, 4532 int nr_zones) 4533 { 4534 struct zone *zone; 4535 enum zone_type zone_type = MAX_NR_ZONES; 4536 4537 do { 4538 zone_type--; 4539 zone = pgdat->node_zones + zone_type; 4540 if (populated_zone(zone)) { 4541 zoneref_set_zone(zone, 4542 &zonelist->_zonerefs[nr_zones++]); 4543 check_highest_zone(zone_type); 4544 } 4545 } while (zone_type); 4546 4547 return nr_zones; 4548 } 4549 4550 4551 /* 4552 * zonelist_order: 4553 * 0 = automatic detection of better ordering. 4554 * 1 = order by ([node] distance, -zonetype) 4555 * 2 = order by (-zonetype, [node] distance) 4556 * 4557 * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create 4558 * the same zonelist. So only NUMA can configure this param. 4559 */ 4560 #define ZONELIST_ORDER_DEFAULT 0 4561 #define ZONELIST_ORDER_NODE 1 4562 #define ZONELIST_ORDER_ZONE 2 4563 4564 /* zonelist order in the kernel. 4565 * set_zonelist_order() will set this to NODE or ZONE. 4566 */ 4567 static int current_zonelist_order = ZONELIST_ORDER_DEFAULT; 4568 static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"}; 4569 4570 4571 #ifdef CONFIG_NUMA 4572 /* The value user specified ....changed by config */ 4573 static int user_zonelist_order = ZONELIST_ORDER_DEFAULT; 4574 /* string for sysctl */ 4575 #define NUMA_ZONELIST_ORDER_LEN 16 4576 char numa_zonelist_order[16] = "default"; 4577 4578 /* 4579 * interface for configure zonelist ordering. 4580 * command line option "numa_zonelist_order" 4581 * = "[dD]efault - default, automatic configuration. 4582 * = "[nN]ode - order by node locality, then by zone within node 4583 * = "[zZ]one - order by zone, then by locality within zone 4584 */ 4585 4586 static int __parse_numa_zonelist_order(char *s) 4587 { 4588 if (*s == 'd' || *s == 'D') { 4589 user_zonelist_order = ZONELIST_ORDER_DEFAULT; 4590 } else if (*s == 'n' || *s == 'N') { 4591 user_zonelist_order = ZONELIST_ORDER_NODE; 4592 } else if (*s == 'z' || *s == 'Z') { 4593 user_zonelist_order = ZONELIST_ORDER_ZONE; 4594 } else { 4595 pr_warn("Ignoring invalid numa_zonelist_order value: %s\n", s); 4596 return -EINVAL; 4597 } 4598 return 0; 4599 } 4600 4601 static __init int setup_numa_zonelist_order(char *s) 4602 { 4603 int ret; 4604 4605 if (!s) 4606 return 0; 4607 4608 ret = __parse_numa_zonelist_order(s); 4609 if (ret == 0) 4610 strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN); 4611 4612 return ret; 4613 } 4614 early_param("numa_zonelist_order", setup_numa_zonelist_order); 4615 4616 /* 4617 * sysctl handler for numa_zonelist_order 4618 */ 4619 int numa_zonelist_order_handler(struct ctl_table *table, int write, 4620 void __user *buffer, size_t *length, 4621 loff_t *ppos) 4622 { 4623 char saved_string[NUMA_ZONELIST_ORDER_LEN]; 4624 int ret; 4625 static DEFINE_MUTEX(zl_order_mutex); 4626 4627 mutex_lock(&zl_order_mutex); 4628 if (write) { 4629 if (strlen((char *)table->data) >= NUMA_ZONELIST_ORDER_LEN) { 4630 ret = -EINVAL; 4631 goto out; 4632 } 4633 strcpy(saved_string, (char *)table->data); 4634 } 4635 ret = proc_dostring(table, write, buffer, length, ppos); 4636 if (ret) 4637 goto out; 4638 if (write) { 4639 int oldval = user_zonelist_order; 4640 4641 ret = __parse_numa_zonelist_order((char *)table->data); 4642 if (ret) { 4643 /* 4644 * bogus value. restore saved string 4645 */ 4646 strncpy((char *)table->data, saved_string, 4647 NUMA_ZONELIST_ORDER_LEN); 4648 user_zonelist_order = oldval; 4649 } else if (oldval != user_zonelist_order) { 4650 mutex_lock(&zonelists_mutex); 4651 build_all_zonelists(NULL, NULL); 4652 mutex_unlock(&zonelists_mutex); 4653 } 4654 } 4655 out: 4656 mutex_unlock(&zl_order_mutex); 4657 return ret; 4658 } 4659 4660 4661 #define MAX_NODE_LOAD (nr_online_nodes) 4662 static int node_load[MAX_NUMNODES]; 4663 4664 /** 4665 * find_next_best_node - find the next node that should appear in a given node's fallback list 4666 * @node: node whose fallback list we're appending 4667 * @used_node_mask: nodemask_t of already used nodes 4668 * 4669 * We use a number of factors to determine which is the next node that should 4670 * appear on a given node's fallback list. The node should not have appeared 4671 * already in @node's fallback list, and it should be the next closest node 4672 * according to the distance array (which contains arbitrary distance values 4673 * from each node to each node in the system), and should also prefer nodes 4674 * with no CPUs, since presumably they'll have very little allocation pressure 4675 * on them otherwise. 4676 * It returns -1 if no node is found. 4677 */ 4678 static int find_next_best_node(int node, nodemask_t *used_node_mask) 4679 { 4680 int n, val; 4681 int min_val = INT_MAX; 4682 int best_node = NUMA_NO_NODE; 4683 const struct cpumask *tmp = cpumask_of_node(0); 4684 4685 /* Use the local node if we haven't already */ 4686 if (!node_isset(node, *used_node_mask)) { 4687 node_set(node, *used_node_mask); 4688 return node; 4689 } 4690 4691 for_each_node_state(n, N_MEMORY) { 4692 4693 /* Don't want a node to appear more than once */ 4694 if (node_isset(n, *used_node_mask)) 4695 continue; 4696 4697 /* Use the distance array to find the distance */ 4698 val = node_distance(node, n); 4699 4700 /* Penalize nodes under us ("prefer the next node") */ 4701 val += (n < node); 4702 4703 /* Give preference to headless and unused nodes */ 4704 tmp = cpumask_of_node(n); 4705 if (!cpumask_empty(tmp)) 4706 val += PENALTY_FOR_NODE_WITH_CPUS; 4707 4708 /* Slight preference for less loaded node */ 4709 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 4710 val += node_load[n]; 4711 4712 if (val < min_val) { 4713 min_val = val; 4714 best_node = n; 4715 } 4716 } 4717 4718 if (best_node >= 0) 4719 node_set(best_node, *used_node_mask); 4720 4721 return best_node; 4722 } 4723 4724 4725 /* 4726 * Build zonelists ordered by node and zones within node. 4727 * This results in maximum locality--normal zone overflows into local 4728 * DMA zone, if any--but risks exhausting DMA zone. 4729 */ 4730 static void build_zonelists_in_node_order(pg_data_t *pgdat, int node) 4731 { 4732 int j; 4733 struct zonelist *zonelist; 4734 4735 zonelist = &pgdat->node_zonelists[0]; 4736 for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++) 4737 ; 4738 j = build_zonelists_node(NODE_DATA(node), zonelist, j); 4739 zonelist->_zonerefs[j].zone = NULL; 4740 zonelist->_zonerefs[j].zone_idx = 0; 4741 } 4742 4743 /* 4744 * Build gfp_thisnode zonelists 4745 */ 4746 static void build_thisnode_zonelists(pg_data_t *pgdat) 4747 { 4748 int j; 4749 struct zonelist *zonelist; 4750 4751 zonelist = &pgdat->node_zonelists[1]; 4752 j = build_zonelists_node(pgdat, zonelist, 0); 4753 zonelist->_zonerefs[j].zone = NULL; 4754 zonelist->_zonerefs[j].zone_idx = 0; 4755 } 4756 4757 /* 4758 * Build zonelists ordered by zone and nodes within zones. 4759 * This results in conserving DMA zone[s] until all Normal memory is 4760 * exhausted, but results in overflowing to remote node while memory 4761 * may still exist in local DMA zone. 4762 */ 4763 static int node_order[MAX_NUMNODES]; 4764 4765 static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes) 4766 { 4767 int pos, j, node; 4768 int zone_type; /* needs to be signed */ 4769 struct zone *z; 4770 struct zonelist *zonelist; 4771 4772 zonelist = &pgdat->node_zonelists[0]; 4773 pos = 0; 4774 for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) { 4775 for (j = 0; j < nr_nodes; j++) { 4776 node = node_order[j]; 4777 z = &NODE_DATA(node)->node_zones[zone_type]; 4778 if (populated_zone(z)) { 4779 zoneref_set_zone(z, 4780 &zonelist->_zonerefs[pos++]); 4781 check_highest_zone(zone_type); 4782 } 4783 } 4784 } 4785 zonelist->_zonerefs[pos].zone = NULL; 4786 zonelist->_zonerefs[pos].zone_idx = 0; 4787 } 4788 4789 #if defined(CONFIG_64BIT) 4790 /* 4791 * Devices that require DMA32/DMA are relatively rare and do not justify a 4792 * penalty to every machine in case the specialised case applies. Default 4793 * to Node-ordering on 64-bit NUMA machines 4794 */ 4795 static int default_zonelist_order(void) 4796 { 4797 return ZONELIST_ORDER_NODE; 4798 } 4799 #else 4800 /* 4801 * On 32-bit, the Normal zone needs to be preserved for allocations accessible 4802 * by the kernel. If processes running on node 0 deplete the low memory zone 4803 * then reclaim will occur more frequency increasing stalls and potentially 4804 * be easier to OOM if a large percentage of the zone is under writeback or 4805 * dirty. The problem is significantly worse if CONFIG_HIGHPTE is not set. 4806 * Hence, default to zone ordering on 32-bit. 4807 */ 4808 static int default_zonelist_order(void) 4809 { 4810 return ZONELIST_ORDER_ZONE; 4811 } 4812 #endif /* CONFIG_64BIT */ 4813 4814 static void set_zonelist_order(void) 4815 { 4816 if (user_zonelist_order == ZONELIST_ORDER_DEFAULT) 4817 current_zonelist_order = default_zonelist_order(); 4818 else 4819 current_zonelist_order = user_zonelist_order; 4820 } 4821 4822 static void build_zonelists(pg_data_t *pgdat) 4823 { 4824 int i, node, load; 4825 nodemask_t used_mask; 4826 int local_node, prev_node; 4827 struct zonelist *zonelist; 4828 unsigned int order = current_zonelist_order; 4829 4830 /* initialize zonelists */ 4831 for (i = 0; i < MAX_ZONELISTS; i++) { 4832 zonelist = pgdat->node_zonelists + i; 4833 zonelist->_zonerefs[0].zone = NULL; 4834 zonelist->_zonerefs[0].zone_idx = 0; 4835 } 4836 4837 /* NUMA-aware ordering of nodes */ 4838 local_node = pgdat->node_id; 4839 load = nr_online_nodes; 4840 prev_node = local_node; 4841 nodes_clear(used_mask); 4842 4843 memset(node_order, 0, sizeof(node_order)); 4844 i = 0; 4845 4846 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 4847 /* 4848 * We don't want to pressure a particular node. 4849 * So adding penalty to the first node in same 4850 * distance group to make it round-robin. 4851 */ 4852 if (node_distance(local_node, node) != 4853 node_distance(local_node, prev_node)) 4854 node_load[node] = load; 4855 4856 prev_node = node; 4857 load--; 4858 if (order == ZONELIST_ORDER_NODE) 4859 build_zonelists_in_node_order(pgdat, node); 4860 else 4861 node_order[i++] = node; /* remember order */ 4862 } 4863 4864 if (order == ZONELIST_ORDER_ZONE) { 4865 /* calculate node order -- i.e., DMA last! */ 4866 build_zonelists_in_zone_order(pgdat, i); 4867 } 4868 4869 build_thisnode_zonelists(pgdat); 4870 } 4871 4872 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 4873 /* 4874 * Return node id of node used for "local" allocations. 4875 * I.e., first node id of first zone in arg node's generic zonelist. 4876 * Used for initializing percpu 'numa_mem', which is used primarily 4877 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 4878 */ 4879 int local_memory_node(int node) 4880 { 4881 struct zoneref *z; 4882 4883 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 4884 gfp_zone(GFP_KERNEL), 4885 NULL); 4886 return z->zone->node; 4887 } 4888 #endif 4889 4890 #else /* CONFIG_NUMA */ 4891 4892 static void set_zonelist_order(void) 4893 { 4894 current_zonelist_order = ZONELIST_ORDER_ZONE; 4895 } 4896 4897 static void build_zonelists(pg_data_t *pgdat) 4898 { 4899 int node, local_node; 4900 enum zone_type j; 4901 struct zonelist *zonelist; 4902 4903 local_node = pgdat->node_id; 4904 4905 zonelist = &pgdat->node_zonelists[0]; 4906 j = build_zonelists_node(pgdat, zonelist, 0); 4907 4908 /* 4909 * Now we build the zonelist so that it contains the zones 4910 * of all the other nodes. 4911 * We don't want to pressure a particular node, so when 4912 * building the zones for node N, we make sure that the 4913 * zones coming right after the local ones are those from 4914 * node N+1 (modulo N) 4915 */ 4916 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 4917 if (!node_online(node)) 4918 continue; 4919 j = build_zonelists_node(NODE_DATA(node), zonelist, j); 4920 } 4921 for (node = 0; node < local_node; node++) { 4922 if (!node_online(node)) 4923 continue; 4924 j = build_zonelists_node(NODE_DATA(node), zonelist, j); 4925 } 4926 4927 zonelist->_zonerefs[j].zone = NULL; 4928 zonelist->_zonerefs[j].zone_idx = 0; 4929 } 4930 4931 #endif /* CONFIG_NUMA */ 4932 4933 /* 4934 * Boot pageset table. One per cpu which is going to be used for all 4935 * zones and all nodes. The parameters will be set in such a way 4936 * that an item put on a list will immediately be handed over to 4937 * the buddy list. This is safe since pageset manipulation is done 4938 * with interrupts disabled. 4939 * 4940 * The boot_pagesets must be kept even after bootup is complete for 4941 * unused processors and/or zones. They do play a role for bootstrapping 4942 * hotplugged processors. 4943 * 4944 * zoneinfo_show() and maybe other functions do 4945 * not check if the processor is online before following the pageset pointer. 4946 * Other parts of the kernel may not check if the zone is available. 4947 */ 4948 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); 4949 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); 4950 static void setup_zone_pageset(struct zone *zone); 4951 4952 /* 4953 * Global mutex to protect against size modification of zonelists 4954 * as well as to serialize pageset setup for the new populated zone. 4955 */ 4956 DEFINE_MUTEX(zonelists_mutex); 4957 4958 /* return values int ....just for stop_machine() */ 4959 static int __build_all_zonelists(void *data) 4960 { 4961 int nid; 4962 int cpu; 4963 pg_data_t *self = data; 4964 4965 #ifdef CONFIG_NUMA 4966 memset(node_load, 0, sizeof(node_load)); 4967 #endif 4968 4969 if (self && !node_online(self->node_id)) { 4970 build_zonelists(self); 4971 } 4972 4973 for_each_online_node(nid) { 4974 pg_data_t *pgdat = NODE_DATA(nid); 4975 4976 build_zonelists(pgdat); 4977 } 4978 4979 /* 4980 * Initialize the boot_pagesets that are going to be used 4981 * for bootstrapping processors. The real pagesets for 4982 * each zone will be allocated later when the per cpu 4983 * allocator is available. 4984 * 4985 * boot_pagesets are used also for bootstrapping offline 4986 * cpus if the system is already booted because the pagesets 4987 * are needed to initialize allocators on a specific cpu too. 4988 * F.e. the percpu allocator needs the page allocator which 4989 * needs the percpu allocator in order to allocate its pagesets 4990 * (a chicken-egg dilemma). 4991 */ 4992 for_each_possible_cpu(cpu) { 4993 setup_pageset(&per_cpu(boot_pageset, cpu), 0); 4994 4995 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 4996 /* 4997 * We now know the "local memory node" for each node-- 4998 * i.e., the node of the first zone in the generic zonelist. 4999 * Set up numa_mem percpu variable for on-line cpus. During 5000 * boot, only the boot cpu should be on-line; we'll init the 5001 * secondary cpus' numa_mem as they come on-line. During 5002 * node/memory hotplug, we'll fixup all on-line cpus. 5003 */ 5004 if (cpu_online(cpu)) 5005 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 5006 #endif 5007 } 5008 5009 return 0; 5010 } 5011 5012 static noinline void __init 5013 build_all_zonelists_init(void) 5014 { 5015 __build_all_zonelists(NULL); 5016 mminit_verify_zonelist(); 5017 cpuset_init_current_mems_allowed(); 5018 } 5019 5020 /* 5021 * Called with zonelists_mutex held always 5022 * unless system_state == SYSTEM_BOOTING. 5023 * 5024 * __ref due to (1) call of __meminit annotated setup_zone_pageset 5025 * [we're only called with non-NULL zone through __meminit paths] and 5026 * (2) call of __init annotated helper build_all_zonelists_init 5027 * [protected by SYSTEM_BOOTING]. 5028 */ 5029 void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone) 5030 { 5031 set_zonelist_order(); 5032 5033 if (system_state == SYSTEM_BOOTING) { 5034 build_all_zonelists_init(); 5035 } else { 5036 #ifdef CONFIG_MEMORY_HOTPLUG 5037 if (zone) 5038 setup_zone_pageset(zone); 5039 #endif 5040 /* we have to stop all cpus to guarantee there is no user 5041 of zonelist */ 5042 stop_machine(__build_all_zonelists, pgdat, NULL); 5043 /* cpuset refresh routine should be here */ 5044 } 5045 vm_total_pages = nr_free_pagecache_pages(); 5046 /* 5047 * Disable grouping by mobility if the number of pages in the 5048 * system is too low to allow the mechanism to work. It would be 5049 * more accurate, but expensive to check per-zone. This check is 5050 * made on memory-hotadd so a system can start with mobility 5051 * disabled and enable it later 5052 */ 5053 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 5054 page_group_by_mobility_disabled = 1; 5055 else 5056 page_group_by_mobility_disabled = 0; 5057 5058 pr_info("Built %i zonelists in %s order, mobility grouping %s. Total pages: %ld\n", 5059 nr_online_nodes, 5060 zonelist_order_name[current_zonelist_order], 5061 page_group_by_mobility_disabled ? "off" : "on", 5062 vm_total_pages); 5063 #ifdef CONFIG_NUMA 5064 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 5065 #endif 5066 } 5067 5068 /* 5069 * Helper functions to size the waitqueue hash table. 5070 * Essentially these want to choose hash table sizes sufficiently 5071 * large so that collisions trying to wait on pages are rare. 5072 * But in fact, the number of active page waitqueues on typical 5073 * systems is ridiculously low, less than 200. So this is even 5074 * conservative, even though it seems large. 5075 * 5076 * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to 5077 * waitqueues, i.e. the size of the waitq table given the number of pages. 5078 */ 5079 #define PAGES_PER_WAITQUEUE 256 5080 5081 #ifndef CONFIG_MEMORY_HOTPLUG 5082 static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) 5083 { 5084 unsigned long size = 1; 5085 5086 pages /= PAGES_PER_WAITQUEUE; 5087 5088 while (size < pages) 5089 size <<= 1; 5090 5091 /* 5092 * Once we have dozens or even hundreds of threads sleeping 5093 * on IO we've got bigger problems than wait queue collision. 5094 * Limit the size of the wait table to a reasonable size. 5095 */ 5096 size = min(size, 4096UL); 5097 5098 return max(size, 4UL); 5099 } 5100 #else 5101 /* 5102 * A zone's size might be changed by hot-add, so it is not possible to determine 5103 * a suitable size for its wait_table. So we use the maximum size now. 5104 * 5105 * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie: 5106 * 5107 * i386 (preemption config) : 4096 x 16 = 64Kbyte. 5108 * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte. 5109 * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte. 5110 * 5111 * The maximum entries are prepared when a zone's memory is (512K + 256) pages 5112 * or more by the traditional way. (See above). It equals: 5113 * 5114 * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte. 5115 * ia64(16K page size) : = ( 8G + 4M)byte. 5116 * powerpc (64K page size) : = (32G +16M)byte. 5117 */ 5118 static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) 5119 { 5120 return 4096UL; 5121 } 5122 #endif 5123 5124 /* 5125 * This is an integer logarithm so that shifts can be used later 5126 * to extract the more random high bits from the multiplicative 5127 * hash function before the remainder is taken. 5128 */ 5129 static inline unsigned long wait_table_bits(unsigned long size) 5130 { 5131 return ffz(~size); 5132 } 5133 5134 /* 5135 * Initially all pages are reserved - free ones are freed 5136 * up by free_all_bootmem() once the early boot process is 5137 * done. Non-atomic initialization, single-pass. 5138 */ 5139 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, 5140 unsigned long start_pfn, enum memmap_context context) 5141 { 5142 struct vmem_altmap *altmap = to_vmem_altmap(__pfn_to_phys(start_pfn)); 5143 unsigned long end_pfn = start_pfn + size; 5144 pg_data_t *pgdat = NODE_DATA(nid); 5145 unsigned long pfn; 5146 unsigned long nr_initialised = 0; 5147 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5148 struct memblock_region *r = NULL, *tmp; 5149 #endif 5150 5151 if (highest_memmap_pfn < end_pfn - 1) 5152 highest_memmap_pfn = end_pfn - 1; 5153 5154 /* 5155 * Honor reservation requested by the driver for this ZONE_DEVICE 5156 * memory 5157 */ 5158 if (altmap && start_pfn == altmap->base_pfn) 5159 start_pfn += altmap->reserve; 5160 5161 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 5162 /* 5163 * There can be holes in boot-time mem_map[]s handed to this 5164 * function. They do not exist on hotplugged memory. 5165 */ 5166 if (context != MEMMAP_EARLY) 5167 goto not_early; 5168 5169 if (!early_pfn_valid(pfn)) 5170 continue; 5171 if (!early_pfn_in_nid(pfn, nid)) 5172 continue; 5173 if (!update_defer_init(pgdat, pfn, end_pfn, &nr_initialised)) 5174 break; 5175 5176 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5177 /* 5178 * If not mirrored_kernelcore and ZONE_MOVABLE exists, range 5179 * from zone_movable_pfn[nid] to end of each node should be 5180 * ZONE_MOVABLE not ZONE_NORMAL. skip it. 5181 */ 5182 if (!mirrored_kernelcore && zone_movable_pfn[nid]) 5183 if (zone == ZONE_NORMAL && pfn >= zone_movable_pfn[nid]) 5184 continue; 5185 5186 /* 5187 * Check given memblock attribute by firmware which can affect 5188 * kernel memory layout. If zone==ZONE_MOVABLE but memory is 5189 * mirrored, it's an overlapped memmap init. skip it. 5190 */ 5191 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 5192 if (!r || pfn >= memblock_region_memory_end_pfn(r)) { 5193 for_each_memblock(memory, tmp) 5194 if (pfn < memblock_region_memory_end_pfn(tmp)) 5195 break; 5196 r = tmp; 5197 } 5198 if (pfn >= memblock_region_memory_base_pfn(r) && 5199 memblock_is_mirror(r)) { 5200 /* already initialized as NORMAL */ 5201 pfn = memblock_region_memory_end_pfn(r); 5202 continue; 5203 } 5204 } 5205 #endif 5206 5207 not_early: 5208 /* 5209 * Mark the block movable so that blocks are reserved for 5210 * movable at startup. This will force kernel allocations 5211 * to reserve their blocks rather than leaking throughout 5212 * the address space during boot when many long-lived 5213 * kernel allocations are made. 5214 * 5215 * bitmap is created for zone's valid pfn range. but memmap 5216 * can be created for invalid pages (for alignment) 5217 * check here not to call set_pageblock_migratetype() against 5218 * pfn out of zone. 5219 */ 5220 if (!(pfn & (pageblock_nr_pages - 1))) { 5221 struct page *page = pfn_to_page(pfn); 5222 5223 __init_single_page(page, pfn, zone, nid); 5224 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 5225 } else { 5226 __init_single_pfn(pfn, zone, nid); 5227 } 5228 } 5229 } 5230 5231 static void __meminit zone_init_free_lists(struct zone *zone) 5232 { 5233 unsigned int order, t; 5234 for_each_migratetype_order(order, t) { 5235 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 5236 zone->free_area[order].nr_free = 0; 5237 } 5238 } 5239 5240 #ifndef __HAVE_ARCH_MEMMAP_INIT 5241 #define memmap_init(size, nid, zone, start_pfn) \ 5242 memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY) 5243 #endif 5244 5245 static int zone_batchsize(struct zone *zone) 5246 { 5247 #ifdef CONFIG_MMU 5248 int batch; 5249 5250 /* 5251 * The per-cpu-pages pools are set to around 1000th of the 5252 * size of the zone. But no more than 1/2 of a meg. 5253 * 5254 * OK, so we don't know how big the cache is. So guess. 5255 */ 5256 batch = zone->managed_pages / 1024; 5257 if (batch * PAGE_SIZE > 512 * 1024) 5258 batch = (512 * 1024) / PAGE_SIZE; 5259 batch /= 4; /* We effectively *= 4 below */ 5260 if (batch < 1) 5261 batch = 1; 5262 5263 /* 5264 * Clamp the batch to a 2^n - 1 value. Having a power 5265 * of 2 value was found to be more likely to have 5266 * suboptimal cache aliasing properties in some cases. 5267 * 5268 * For example if 2 tasks are alternately allocating 5269 * batches of pages, one task can end up with a lot 5270 * of pages of one half of the possible page colors 5271 * and the other with pages of the other colors. 5272 */ 5273 batch = rounddown_pow_of_two(batch + batch/2) - 1; 5274 5275 return batch; 5276 5277 #else 5278 /* The deferral and batching of frees should be suppressed under NOMMU 5279 * conditions. 5280 * 5281 * The problem is that NOMMU needs to be able to allocate large chunks 5282 * of contiguous memory as there's no hardware page translation to 5283 * assemble apparent contiguous memory from discontiguous pages. 5284 * 5285 * Queueing large contiguous runs of pages for batching, however, 5286 * causes the pages to actually be freed in smaller chunks. As there 5287 * can be a significant delay between the individual batches being 5288 * recycled, this leads to the once large chunks of space being 5289 * fragmented and becoming unavailable for high-order allocations. 5290 */ 5291 return 0; 5292 #endif 5293 } 5294 5295 /* 5296 * pcp->high and pcp->batch values are related and dependent on one another: 5297 * ->batch must never be higher then ->high. 5298 * The following function updates them in a safe manner without read side 5299 * locking. 5300 * 5301 * Any new users of pcp->batch and pcp->high should ensure they can cope with 5302 * those fields changing asynchronously (acording the the above rule). 5303 * 5304 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 5305 * outside of boot time (or some other assurance that no concurrent updaters 5306 * exist). 5307 */ 5308 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 5309 unsigned long batch) 5310 { 5311 /* start with a fail safe value for batch */ 5312 pcp->batch = 1; 5313 smp_wmb(); 5314 5315 /* Update high, then batch, in order */ 5316 pcp->high = high; 5317 smp_wmb(); 5318 5319 pcp->batch = batch; 5320 } 5321 5322 /* a companion to pageset_set_high() */ 5323 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) 5324 { 5325 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); 5326 } 5327 5328 static void pageset_init(struct per_cpu_pageset *p) 5329 { 5330 struct per_cpu_pages *pcp; 5331 int migratetype; 5332 5333 memset(p, 0, sizeof(*p)); 5334 5335 pcp = &p->pcp; 5336 pcp->count = 0; 5337 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) 5338 INIT_LIST_HEAD(&pcp->lists[migratetype]); 5339 } 5340 5341 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 5342 { 5343 pageset_init(p); 5344 pageset_set_batch(p, batch); 5345 } 5346 5347 /* 5348 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist 5349 * to the value high for the pageset p. 5350 */ 5351 static void pageset_set_high(struct per_cpu_pageset *p, 5352 unsigned long high) 5353 { 5354 unsigned long batch = max(1UL, high / 4); 5355 if ((high / 4) > (PAGE_SHIFT * 8)) 5356 batch = PAGE_SHIFT * 8; 5357 5358 pageset_update(&p->pcp, high, batch); 5359 } 5360 5361 static void pageset_set_high_and_batch(struct zone *zone, 5362 struct per_cpu_pageset *pcp) 5363 { 5364 if (percpu_pagelist_fraction) 5365 pageset_set_high(pcp, 5366 (zone->managed_pages / 5367 percpu_pagelist_fraction)); 5368 else 5369 pageset_set_batch(pcp, zone_batchsize(zone)); 5370 } 5371 5372 static void __meminit zone_pageset_init(struct zone *zone, int cpu) 5373 { 5374 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); 5375 5376 pageset_init(pcp); 5377 pageset_set_high_and_batch(zone, pcp); 5378 } 5379 5380 static void __meminit setup_zone_pageset(struct zone *zone) 5381 { 5382 int cpu; 5383 zone->pageset = alloc_percpu(struct per_cpu_pageset); 5384 for_each_possible_cpu(cpu) 5385 zone_pageset_init(zone, cpu); 5386 } 5387 5388 /* 5389 * Allocate per cpu pagesets and initialize them. 5390 * Before this call only boot pagesets were available. 5391 */ 5392 void __init setup_per_cpu_pageset(void) 5393 { 5394 struct zone *zone; 5395 5396 for_each_populated_zone(zone) 5397 setup_zone_pageset(zone); 5398 } 5399 5400 static noinline __init_refok 5401 int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) 5402 { 5403 int i; 5404 size_t alloc_size; 5405 5406 /* 5407 * The per-page waitqueue mechanism uses hashed waitqueues 5408 * per zone. 5409 */ 5410 zone->wait_table_hash_nr_entries = 5411 wait_table_hash_nr_entries(zone_size_pages); 5412 zone->wait_table_bits = 5413 wait_table_bits(zone->wait_table_hash_nr_entries); 5414 alloc_size = zone->wait_table_hash_nr_entries 5415 * sizeof(wait_queue_head_t); 5416 5417 if (!slab_is_available()) { 5418 zone->wait_table = (wait_queue_head_t *) 5419 memblock_virt_alloc_node_nopanic( 5420 alloc_size, zone->zone_pgdat->node_id); 5421 } else { 5422 /* 5423 * This case means that a zone whose size was 0 gets new memory 5424 * via memory hot-add. 5425 * But it may be the case that a new node was hot-added. In 5426 * this case vmalloc() will not be able to use this new node's 5427 * memory - this wait_table must be initialized to use this new 5428 * node itself as well. 5429 * To use this new node's memory, further consideration will be 5430 * necessary. 5431 */ 5432 zone->wait_table = vmalloc(alloc_size); 5433 } 5434 if (!zone->wait_table) 5435 return -ENOMEM; 5436 5437 for (i = 0; i < zone->wait_table_hash_nr_entries; ++i) 5438 init_waitqueue_head(zone->wait_table + i); 5439 5440 return 0; 5441 } 5442 5443 static __meminit void zone_pcp_init(struct zone *zone) 5444 { 5445 /* 5446 * per cpu subsystem is not up at this point. The following code 5447 * relies on the ability of the linker to provide the 5448 * offset of a (static) per cpu variable into the per cpu area. 5449 */ 5450 zone->pageset = &boot_pageset; 5451 5452 if (populated_zone(zone)) 5453 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", 5454 zone->name, zone->present_pages, 5455 zone_batchsize(zone)); 5456 } 5457 5458 int __meminit init_currently_empty_zone(struct zone *zone, 5459 unsigned long zone_start_pfn, 5460 unsigned long size) 5461 { 5462 struct pglist_data *pgdat = zone->zone_pgdat; 5463 int ret; 5464 ret = zone_wait_table_init(zone, size); 5465 if (ret) 5466 return ret; 5467 pgdat->nr_zones = zone_idx(zone) + 1; 5468 5469 zone->zone_start_pfn = zone_start_pfn; 5470 5471 mminit_dprintk(MMINIT_TRACE, "memmap_init", 5472 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 5473 pgdat->node_id, 5474 (unsigned long)zone_idx(zone), 5475 zone_start_pfn, (zone_start_pfn + size)); 5476 5477 zone_init_free_lists(zone); 5478 5479 return 0; 5480 } 5481 5482 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5483 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID 5484 5485 /* 5486 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 5487 */ 5488 int __meminit __early_pfn_to_nid(unsigned long pfn, 5489 struct mminit_pfnnid_cache *state) 5490 { 5491 unsigned long start_pfn, end_pfn; 5492 int nid; 5493 5494 if (state->last_start <= pfn && pfn < state->last_end) 5495 return state->last_nid; 5496 5497 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 5498 if (nid != -1) { 5499 state->last_start = start_pfn; 5500 state->last_end = end_pfn; 5501 state->last_nid = nid; 5502 } 5503 5504 return nid; 5505 } 5506 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ 5507 5508 /** 5509 * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range 5510 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed. 5511 * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid 5512 * 5513 * If an architecture guarantees that all ranges registered contain no holes 5514 * and may be freed, this this function may be used instead of calling 5515 * memblock_free_early_nid() manually. 5516 */ 5517 void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn) 5518 { 5519 unsigned long start_pfn, end_pfn; 5520 int i, this_nid; 5521 5522 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) { 5523 start_pfn = min(start_pfn, max_low_pfn); 5524 end_pfn = min(end_pfn, max_low_pfn); 5525 5526 if (start_pfn < end_pfn) 5527 memblock_free_early_nid(PFN_PHYS(start_pfn), 5528 (end_pfn - start_pfn) << PAGE_SHIFT, 5529 this_nid); 5530 } 5531 } 5532 5533 /** 5534 * sparse_memory_present_with_active_regions - Call memory_present for each active range 5535 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used. 5536 * 5537 * If an architecture guarantees that all ranges registered contain no holes and may 5538 * be freed, this function may be used instead of calling memory_present() manually. 5539 */ 5540 void __init sparse_memory_present_with_active_regions(int nid) 5541 { 5542 unsigned long start_pfn, end_pfn; 5543 int i, this_nid; 5544 5545 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) 5546 memory_present(this_nid, start_pfn, end_pfn); 5547 } 5548 5549 /** 5550 * get_pfn_range_for_nid - Return the start and end page frames for a node 5551 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 5552 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 5553 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 5554 * 5555 * It returns the start and end page frame of a node based on information 5556 * provided by memblock_set_node(). If called for a node 5557 * with no available memory, a warning is printed and the start and end 5558 * PFNs will be 0. 5559 */ 5560 void __meminit get_pfn_range_for_nid(unsigned int nid, 5561 unsigned long *start_pfn, unsigned long *end_pfn) 5562 { 5563 unsigned long this_start_pfn, this_end_pfn; 5564 int i; 5565 5566 *start_pfn = -1UL; 5567 *end_pfn = 0; 5568 5569 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 5570 *start_pfn = min(*start_pfn, this_start_pfn); 5571 *end_pfn = max(*end_pfn, this_end_pfn); 5572 } 5573 5574 if (*start_pfn == -1UL) 5575 *start_pfn = 0; 5576 } 5577 5578 /* 5579 * This finds a zone that can be used for ZONE_MOVABLE pages. The 5580 * assumption is made that zones within a node are ordered in monotonic 5581 * increasing memory addresses so that the "highest" populated zone is used 5582 */ 5583 static void __init find_usable_zone_for_movable(void) 5584 { 5585 int zone_index; 5586 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 5587 if (zone_index == ZONE_MOVABLE) 5588 continue; 5589 5590 if (arch_zone_highest_possible_pfn[zone_index] > 5591 arch_zone_lowest_possible_pfn[zone_index]) 5592 break; 5593 } 5594 5595 VM_BUG_ON(zone_index == -1); 5596 movable_zone = zone_index; 5597 } 5598 5599 /* 5600 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 5601 * because it is sized independent of architecture. Unlike the other zones, 5602 * the starting point for ZONE_MOVABLE is not fixed. It may be different 5603 * in each node depending on the size of each node and how evenly kernelcore 5604 * is distributed. This helper function adjusts the zone ranges 5605 * provided by the architecture for a given node by using the end of the 5606 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 5607 * zones within a node are in order of monotonic increases memory addresses 5608 */ 5609 static void __meminit adjust_zone_range_for_zone_movable(int nid, 5610 unsigned long zone_type, 5611 unsigned long node_start_pfn, 5612 unsigned long node_end_pfn, 5613 unsigned long *zone_start_pfn, 5614 unsigned long *zone_end_pfn) 5615 { 5616 /* Only adjust if ZONE_MOVABLE is on this node */ 5617 if (zone_movable_pfn[nid]) { 5618 /* Size ZONE_MOVABLE */ 5619 if (zone_type == ZONE_MOVABLE) { 5620 *zone_start_pfn = zone_movable_pfn[nid]; 5621 *zone_end_pfn = min(node_end_pfn, 5622 arch_zone_highest_possible_pfn[movable_zone]); 5623 5624 /* Check if this whole range is within ZONE_MOVABLE */ 5625 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 5626 *zone_start_pfn = *zone_end_pfn; 5627 } 5628 } 5629 5630 /* 5631 * Return the number of pages a zone spans in a node, including holes 5632 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 5633 */ 5634 static unsigned long __meminit zone_spanned_pages_in_node(int nid, 5635 unsigned long zone_type, 5636 unsigned long node_start_pfn, 5637 unsigned long node_end_pfn, 5638 unsigned long *zone_start_pfn, 5639 unsigned long *zone_end_pfn, 5640 unsigned long *ignored) 5641 { 5642 /* When hotadd a new node from cpu_up(), the node should be empty */ 5643 if (!node_start_pfn && !node_end_pfn) 5644 return 0; 5645 5646 /* Get the start and end of the zone */ 5647 *zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type]; 5648 *zone_end_pfn = arch_zone_highest_possible_pfn[zone_type]; 5649 adjust_zone_range_for_zone_movable(nid, zone_type, 5650 node_start_pfn, node_end_pfn, 5651 zone_start_pfn, zone_end_pfn); 5652 5653 /* Check that this node has pages within the zone's required range */ 5654 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 5655 return 0; 5656 5657 /* Move the zone boundaries inside the node if necessary */ 5658 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 5659 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 5660 5661 /* Return the spanned pages */ 5662 return *zone_end_pfn - *zone_start_pfn; 5663 } 5664 5665 /* 5666 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 5667 * then all holes in the requested range will be accounted for. 5668 */ 5669 unsigned long __meminit __absent_pages_in_range(int nid, 5670 unsigned long range_start_pfn, 5671 unsigned long range_end_pfn) 5672 { 5673 unsigned long nr_absent = range_end_pfn - range_start_pfn; 5674 unsigned long start_pfn, end_pfn; 5675 int i; 5676 5677 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 5678 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 5679 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 5680 nr_absent -= end_pfn - start_pfn; 5681 } 5682 return nr_absent; 5683 } 5684 5685 /** 5686 * absent_pages_in_range - Return number of page frames in holes within a range 5687 * @start_pfn: The start PFN to start searching for holes 5688 * @end_pfn: The end PFN to stop searching for holes 5689 * 5690 * It returns the number of pages frames in memory holes within a range. 5691 */ 5692 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 5693 unsigned long end_pfn) 5694 { 5695 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 5696 } 5697 5698 /* Return the number of page frames in holes in a zone on a node */ 5699 static unsigned long __meminit zone_absent_pages_in_node(int nid, 5700 unsigned long zone_type, 5701 unsigned long node_start_pfn, 5702 unsigned long node_end_pfn, 5703 unsigned long *ignored) 5704 { 5705 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 5706 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 5707 unsigned long zone_start_pfn, zone_end_pfn; 5708 unsigned long nr_absent; 5709 5710 /* When hotadd a new node from cpu_up(), the node should be empty */ 5711 if (!node_start_pfn && !node_end_pfn) 5712 return 0; 5713 5714 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 5715 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 5716 5717 adjust_zone_range_for_zone_movable(nid, zone_type, 5718 node_start_pfn, node_end_pfn, 5719 &zone_start_pfn, &zone_end_pfn); 5720 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 5721 5722 /* 5723 * ZONE_MOVABLE handling. 5724 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 5725 * and vice versa. 5726 */ 5727 if (zone_movable_pfn[nid]) { 5728 if (mirrored_kernelcore) { 5729 unsigned long start_pfn, end_pfn; 5730 struct memblock_region *r; 5731 5732 for_each_memblock(memory, r) { 5733 start_pfn = clamp(memblock_region_memory_base_pfn(r), 5734 zone_start_pfn, zone_end_pfn); 5735 end_pfn = clamp(memblock_region_memory_end_pfn(r), 5736 zone_start_pfn, zone_end_pfn); 5737 5738 if (zone_type == ZONE_MOVABLE && 5739 memblock_is_mirror(r)) 5740 nr_absent += end_pfn - start_pfn; 5741 5742 if (zone_type == ZONE_NORMAL && 5743 !memblock_is_mirror(r)) 5744 nr_absent += end_pfn - start_pfn; 5745 } 5746 } else { 5747 if (zone_type == ZONE_NORMAL) 5748 nr_absent += node_end_pfn - zone_movable_pfn[nid]; 5749 } 5750 } 5751 5752 return nr_absent; 5753 } 5754 5755 #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 5756 static inline unsigned long __meminit zone_spanned_pages_in_node(int nid, 5757 unsigned long zone_type, 5758 unsigned long node_start_pfn, 5759 unsigned long node_end_pfn, 5760 unsigned long *zone_start_pfn, 5761 unsigned long *zone_end_pfn, 5762 unsigned long *zones_size) 5763 { 5764 unsigned int zone; 5765 5766 *zone_start_pfn = node_start_pfn; 5767 for (zone = 0; zone < zone_type; zone++) 5768 *zone_start_pfn += zones_size[zone]; 5769 5770 *zone_end_pfn = *zone_start_pfn + zones_size[zone_type]; 5771 5772 return zones_size[zone_type]; 5773 } 5774 5775 static inline unsigned long __meminit zone_absent_pages_in_node(int nid, 5776 unsigned long zone_type, 5777 unsigned long node_start_pfn, 5778 unsigned long node_end_pfn, 5779 unsigned long *zholes_size) 5780 { 5781 if (!zholes_size) 5782 return 0; 5783 5784 return zholes_size[zone_type]; 5785 } 5786 5787 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 5788 5789 static void __meminit calculate_node_totalpages(struct pglist_data *pgdat, 5790 unsigned long node_start_pfn, 5791 unsigned long node_end_pfn, 5792 unsigned long *zones_size, 5793 unsigned long *zholes_size) 5794 { 5795 unsigned long realtotalpages = 0, totalpages = 0; 5796 enum zone_type i; 5797 5798 for (i = 0; i < MAX_NR_ZONES; i++) { 5799 struct zone *zone = pgdat->node_zones + i; 5800 unsigned long zone_start_pfn, zone_end_pfn; 5801 unsigned long size, real_size; 5802 5803 size = zone_spanned_pages_in_node(pgdat->node_id, i, 5804 node_start_pfn, 5805 node_end_pfn, 5806 &zone_start_pfn, 5807 &zone_end_pfn, 5808 zones_size); 5809 real_size = size - zone_absent_pages_in_node(pgdat->node_id, i, 5810 node_start_pfn, node_end_pfn, 5811 zholes_size); 5812 if (size) 5813 zone->zone_start_pfn = zone_start_pfn; 5814 else 5815 zone->zone_start_pfn = 0; 5816 zone->spanned_pages = size; 5817 zone->present_pages = real_size; 5818 5819 totalpages += size; 5820 realtotalpages += real_size; 5821 } 5822 5823 pgdat->node_spanned_pages = totalpages; 5824 pgdat->node_present_pages = realtotalpages; 5825 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, 5826 realtotalpages); 5827 } 5828 5829 #ifndef CONFIG_SPARSEMEM 5830 /* 5831 * Calculate the size of the zone->blockflags rounded to an unsigned long 5832 * Start by making sure zonesize is a multiple of pageblock_order by rounding 5833 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 5834 * round what is now in bits to nearest long in bits, then return it in 5835 * bytes. 5836 */ 5837 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 5838 { 5839 unsigned long usemapsize; 5840 5841 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 5842 usemapsize = roundup(zonesize, pageblock_nr_pages); 5843 usemapsize = usemapsize >> pageblock_order; 5844 usemapsize *= NR_PAGEBLOCK_BITS; 5845 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 5846 5847 return usemapsize / 8; 5848 } 5849 5850 static void __init setup_usemap(struct pglist_data *pgdat, 5851 struct zone *zone, 5852 unsigned long zone_start_pfn, 5853 unsigned long zonesize) 5854 { 5855 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); 5856 zone->pageblock_flags = NULL; 5857 if (usemapsize) 5858 zone->pageblock_flags = 5859 memblock_virt_alloc_node_nopanic(usemapsize, 5860 pgdat->node_id); 5861 } 5862 #else 5863 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, 5864 unsigned long zone_start_pfn, unsigned long zonesize) {} 5865 #endif /* CONFIG_SPARSEMEM */ 5866 5867 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 5868 5869 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 5870 void __paginginit set_pageblock_order(void) 5871 { 5872 unsigned int order; 5873 5874 /* Check that pageblock_nr_pages has not already been setup */ 5875 if (pageblock_order) 5876 return; 5877 5878 if (HPAGE_SHIFT > PAGE_SHIFT) 5879 order = HUGETLB_PAGE_ORDER; 5880 else 5881 order = MAX_ORDER - 1; 5882 5883 /* 5884 * Assume the largest contiguous order of interest is a huge page. 5885 * This value may be variable depending on boot parameters on IA64 and 5886 * powerpc. 5887 */ 5888 pageblock_order = order; 5889 } 5890 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 5891 5892 /* 5893 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 5894 * is unused as pageblock_order is set at compile-time. See 5895 * include/linux/pageblock-flags.h for the values of pageblock_order based on 5896 * the kernel config 5897 */ 5898 void __paginginit set_pageblock_order(void) 5899 { 5900 } 5901 5902 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 5903 5904 static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages, 5905 unsigned long present_pages) 5906 { 5907 unsigned long pages = spanned_pages; 5908 5909 /* 5910 * Provide a more accurate estimation if there are holes within 5911 * the zone and SPARSEMEM is in use. If there are holes within the 5912 * zone, each populated memory region may cost us one or two extra 5913 * memmap pages due to alignment because memmap pages for each 5914 * populated regions may not naturally algined on page boundary. 5915 * So the (present_pages >> 4) heuristic is a tradeoff for that. 5916 */ 5917 if (spanned_pages > present_pages + (present_pages >> 4) && 5918 IS_ENABLED(CONFIG_SPARSEMEM)) 5919 pages = present_pages; 5920 5921 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 5922 } 5923 5924 /* 5925 * Set up the zone data structures: 5926 * - mark all pages reserved 5927 * - mark all memory queues empty 5928 * - clear the memory bitmaps 5929 * 5930 * NOTE: pgdat should get zeroed by caller. 5931 */ 5932 static void __paginginit free_area_init_core(struct pglist_data *pgdat) 5933 { 5934 enum zone_type j; 5935 int nid = pgdat->node_id; 5936 int ret; 5937 5938 pgdat_resize_init(pgdat); 5939 #ifdef CONFIG_NUMA_BALANCING 5940 spin_lock_init(&pgdat->numabalancing_migrate_lock); 5941 pgdat->numabalancing_migrate_nr_pages = 0; 5942 pgdat->numabalancing_migrate_next_window = jiffies; 5943 #endif 5944 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5945 spin_lock_init(&pgdat->split_queue_lock); 5946 INIT_LIST_HEAD(&pgdat->split_queue); 5947 pgdat->split_queue_len = 0; 5948 #endif 5949 init_waitqueue_head(&pgdat->kswapd_wait); 5950 init_waitqueue_head(&pgdat->pfmemalloc_wait); 5951 #ifdef CONFIG_COMPACTION 5952 init_waitqueue_head(&pgdat->kcompactd_wait); 5953 #endif 5954 pgdat_page_ext_init(pgdat); 5955 5956 for (j = 0; j < MAX_NR_ZONES; j++) { 5957 struct zone *zone = pgdat->node_zones + j; 5958 unsigned long size, realsize, freesize, memmap_pages; 5959 unsigned long zone_start_pfn = zone->zone_start_pfn; 5960 5961 size = zone->spanned_pages; 5962 realsize = freesize = zone->present_pages; 5963 5964 /* 5965 * Adjust freesize so that it accounts for how much memory 5966 * is used by this zone for memmap. This affects the watermark 5967 * and per-cpu initialisations 5968 */ 5969 memmap_pages = calc_memmap_size(size, realsize); 5970 if (!is_highmem_idx(j)) { 5971 if (freesize >= memmap_pages) { 5972 freesize -= memmap_pages; 5973 if (memmap_pages) 5974 printk(KERN_DEBUG 5975 " %s zone: %lu pages used for memmap\n", 5976 zone_names[j], memmap_pages); 5977 } else 5978 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n", 5979 zone_names[j], memmap_pages, freesize); 5980 } 5981 5982 /* Account for reserved pages */ 5983 if (j == 0 && freesize > dma_reserve) { 5984 freesize -= dma_reserve; 5985 printk(KERN_DEBUG " %s zone: %lu pages reserved\n", 5986 zone_names[0], dma_reserve); 5987 } 5988 5989 if (!is_highmem_idx(j)) 5990 nr_kernel_pages += freesize; 5991 /* Charge for highmem memmap if there are enough kernel pages */ 5992 else if (nr_kernel_pages > memmap_pages * 2) 5993 nr_kernel_pages -= memmap_pages; 5994 nr_all_pages += freesize; 5995 5996 /* 5997 * Set an approximate value for lowmem here, it will be adjusted 5998 * when the bootmem allocator frees pages into the buddy system. 5999 * And all highmem pages will be managed by the buddy system. 6000 */ 6001 zone->managed_pages = is_highmem_idx(j) ? realsize : freesize; 6002 #ifdef CONFIG_NUMA 6003 zone->node = nid; 6004 zone->min_unmapped_pages = (freesize*sysctl_min_unmapped_ratio) 6005 / 100; 6006 zone->min_slab_pages = (freesize * sysctl_min_slab_ratio) / 100; 6007 #endif 6008 zone->name = zone_names[j]; 6009 spin_lock_init(&zone->lock); 6010 spin_lock_init(&zone->lru_lock); 6011 zone_seqlock_init(zone); 6012 zone->zone_pgdat = pgdat; 6013 zone_pcp_init(zone); 6014 6015 /* For bootup, initialized properly in watermark setup */ 6016 mod_zone_page_state(zone, NR_ALLOC_BATCH, zone->managed_pages); 6017 6018 lruvec_init(&zone->lruvec); 6019 if (!size) 6020 continue; 6021 6022 set_pageblock_order(); 6023 setup_usemap(pgdat, zone, zone_start_pfn, size); 6024 ret = init_currently_empty_zone(zone, zone_start_pfn, size); 6025 BUG_ON(ret); 6026 memmap_init(size, nid, j, zone_start_pfn); 6027 } 6028 } 6029 6030 static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat) 6031 { 6032 unsigned long __maybe_unused start = 0; 6033 unsigned long __maybe_unused offset = 0; 6034 6035 /* Skip empty nodes */ 6036 if (!pgdat->node_spanned_pages) 6037 return; 6038 6039 #ifdef CONFIG_FLAT_NODE_MEM_MAP 6040 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 6041 offset = pgdat->node_start_pfn - start; 6042 /* ia64 gets its own node_mem_map, before this, without bootmem */ 6043 if (!pgdat->node_mem_map) { 6044 unsigned long size, end; 6045 struct page *map; 6046 6047 /* 6048 * The zone's endpoints aren't required to be MAX_ORDER 6049 * aligned but the node_mem_map endpoints must be in order 6050 * for the buddy allocator to function correctly. 6051 */ 6052 end = pgdat_end_pfn(pgdat); 6053 end = ALIGN(end, MAX_ORDER_NR_PAGES); 6054 size = (end - start) * sizeof(struct page); 6055 map = alloc_remap(pgdat->node_id, size); 6056 if (!map) 6057 map = memblock_virt_alloc_node_nopanic(size, 6058 pgdat->node_id); 6059 pgdat->node_mem_map = map + offset; 6060 } 6061 #ifndef CONFIG_NEED_MULTIPLE_NODES 6062 /* 6063 * With no DISCONTIG, the global mem_map is just set as node 0's 6064 */ 6065 if (pgdat == NODE_DATA(0)) { 6066 mem_map = NODE_DATA(0)->node_mem_map; 6067 #if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM) 6068 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 6069 mem_map -= offset; 6070 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6071 } 6072 #endif 6073 #endif /* CONFIG_FLAT_NODE_MEM_MAP */ 6074 } 6075 6076 void __paginginit free_area_init_node(int nid, unsigned long *zones_size, 6077 unsigned long node_start_pfn, unsigned long *zholes_size) 6078 { 6079 pg_data_t *pgdat = NODE_DATA(nid); 6080 unsigned long start_pfn = 0; 6081 unsigned long end_pfn = 0; 6082 6083 /* pg_data_t should be reset to zero when it's allocated */ 6084 WARN_ON(pgdat->nr_zones || pgdat->classzone_idx); 6085 6086 reset_deferred_meminit(pgdat); 6087 pgdat->node_id = nid; 6088 pgdat->node_start_pfn = node_start_pfn; 6089 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6090 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 6091 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 6092 (u64)start_pfn << PAGE_SHIFT, 6093 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 6094 #else 6095 start_pfn = node_start_pfn; 6096 #endif 6097 calculate_node_totalpages(pgdat, start_pfn, end_pfn, 6098 zones_size, zholes_size); 6099 6100 alloc_node_mem_map(pgdat); 6101 #ifdef CONFIG_FLAT_NODE_MEM_MAP 6102 printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n", 6103 nid, (unsigned long)pgdat, 6104 (unsigned long)pgdat->node_mem_map); 6105 #endif 6106 6107 free_area_init_core(pgdat); 6108 } 6109 6110 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6111 6112 #if MAX_NUMNODES > 1 6113 /* 6114 * Figure out the number of possible node ids. 6115 */ 6116 void __init setup_nr_node_ids(void) 6117 { 6118 unsigned int highest; 6119 6120 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 6121 nr_node_ids = highest + 1; 6122 } 6123 #endif 6124 6125 /** 6126 * node_map_pfn_alignment - determine the maximum internode alignment 6127 * 6128 * This function should be called after node map is populated and sorted. 6129 * It calculates the maximum power of two alignment which can distinguish 6130 * all the nodes. 6131 * 6132 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 6133 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 6134 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 6135 * shifted, 1GiB is enough and this function will indicate so. 6136 * 6137 * This is used to test whether pfn -> nid mapping of the chosen memory 6138 * model has fine enough granularity to avoid incorrect mapping for the 6139 * populated node map. 6140 * 6141 * Returns the determined alignment in pfn's. 0 if there is no alignment 6142 * requirement (single node). 6143 */ 6144 unsigned long __init node_map_pfn_alignment(void) 6145 { 6146 unsigned long accl_mask = 0, last_end = 0; 6147 unsigned long start, end, mask; 6148 int last_nid = -1; 6149 int i, nid; 6150 6151 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 6152 if (!start || last_nid < 0 || last_nid == nid) { 6153 last_nid = nid; 6154 last_end = end; 6155 continue; 6156 } 6157 6158 /* 6159 * Start with a mask granular enough to pin-point to the 6160 * start pfn and tick off bits one-by-one until it becomes 6161 * too coarse to separate the current node from the last. 6162 */ 6163 mask = ~((1 << __ffs(start)) - 1); 6164 while (mask && last_end <= (start & (mask << 1))) 6165 mask <<= 1; 6166 6167 /* accumulate all internode masks */ 6168 accl_mask |= mask; 6169 } 6170 6171 /* convert mask to number of pages */ 6172 return ~accl_mask + 1; 6173 } 6174 6175 /* Find the lowest pfn for a node */ 6176 static unsigned long __init find_min_pfn_for_node(int nid) 6177 { 6178 unsigned long min_pfn = ULONG_MAX; 6179 unsigned long start_pfn; 6180 int i; 6181 6182 for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL) 6183 min_pfn = min(min_pfn, start_pfn); 6184 6185 if (min_pfn == ULONG_MAX) { 6186 pr_warn("Could not find start_pfn for node %d\n", nid); 6187 return 0; 6188 } 6189 6190 return min_pfn; 6191 } 6192 6193 /** 6194 * find_min_pfn_with_active_regions - Find the minimum PFN registered 6195 * 6196 * It returns the minimum PFN based on information provided via 6197 * memblock_set_node(). 6198 */ 6199 unsigned long __init find_min_pfn_with_active_regions(void) 6200 { 6201 return find_min_pfn_for_node(MAX_NUMNODES); 6202 } 6203 6204 /* 6205 * early_calculate_totalpages() 6206 * Sum pages in active regions for movable zone. 6207 * Populate N_MEMORY for calculating usable_nodes. 6208 */ 6209 static unsigned long __init early_calculate_totalpages(void) 6210 { 6211 unsigned long totalpages = 0; 6212 unsigned long start_pfn, end_pfn; 6213 int i, nid; 6214 6215 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 6216 unsigned long pages = end_pfn - start_pfn; 6217 6218 totalpages += pages; 6219 if (pages) 6220 node_set_state(nid, N_MEMORY); 6221 } 6222 return totalpages; 6223 } 6224 6225 /* 6226 * Find the PFN the Movable zone begins in each node. Kernel memory 6227 * is spread evenly between nodes as long as the nodes have enough 6228 * memory. When they don't, some nodes will have more kernelcore than 6229 * others 6230 */ 6231 static void __init find_zone_movable_pfns_for_nodes(void) 6232 { 6233 int i, nid; 6234 unsigned long usable_startpfn; 6235 unsigned long kernelcore_node, kernelcore_remaining; 6236 /* save the state before borrow the nodemask */ 6237 nodemask_t saved_node_state = node_states[N_MEMORY]; 6238 unsigned long totalpages = early_calculate_totalpages(); 6239 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 6240 struct memblock_region *r; 6241 6242 /* Need to find movable_zone earlier when movable_node is specified. */ 6243 find_usable_zone_for_movable(); 6244 6245 /* 6246 * If movable_node is specified, ignore kernelcore and movablecore 6247 * options. 6248 */ 6249 if (movable_node_is_enabled()) { 6250 for_each_memblock(memory, r) { 6251 if (!memblock_is_hotpluggable(r)) 6252 continue; 6253 6254 nid = r->nid; 6255 6256 usable_startpfn = PFN_DOWN(r->base); 6257 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 6258 min(usable_startpfn, zone_movable_pfn[nid]) : 6259 usable_startpfn; 6260 } 6261 6262 goto out2; 6263 } 6264 6265 /* 6266 * If kernelcore=mirror is specified, ignore movablecore option 6267 */ 6268 if (mirrored_kernelcore) { 6269 bool mem_below_4gb_not_mirrored = false; 6270 6271 for_each_memblock(memory, r) { 6272 if (memblock_is_mirror(r)) 6273 continue; 6274 6275 nid = r->nid; 6276 6277 usable_startpfn = memblock_region_memory_base_pfn(r); 6278 6279 if (usable_startpfn < 0x100000) { 6280 mem_below_4gb_not_mirrored = true; 6281 continue; 6282 } 6283 6284 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 6285 min(usable_startpfn, zone_movable_pfn[nid]) : 6286 usable_startpfn; 6287 } 6288 6289 if (mem_below_4gb_not_mirrored) 6290 pr_warn("This configuration results in unmirrored kernel memory."); 6291 6292 goto out2; 6293 } 6294 6295 /* 6296 * If movablecore=nn[KMG] was specified, calculate what size of 6297 * kernelcore that corresponds so that memory usable for 6298 * any allocation type is evenly spread. If both kernelcore 6299 * and movablecore are specified, then the value of kernelcore 6300 * will be used for required_kernelcore if it's greater than 6301 * what movablecore would have allowed. 6302 */ 6303 if (required_movablecore) { 6304 unsigned long corepages; 6305 6306 /* 6307 * Round-up so that ZONE_MOVABLE is at least as large as what 6308 * was requested by the user 6309 */ 6310 required_movablecore = 6311 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 6312 required_movablecore = min(totalpages, required_movablecore); 6313 corepages = totalpages - required_movablecore; 6314 6315 required_kernelcore = max(required_kernelcore, corepages); 6316 } 6317 6318 /* 6319 * If kernelcore was not specified or kernelcore size is larger 6320 * than totalpages, there is no ZONE_MOVABLE. 6321 */ 6322 if (!required_kernelcore || required_kernelcore >= totalpages) 6323 goto out; 6324 6325 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 6326 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 6327 6328 restart: 6329 /* Spread kernelcore memory as evenly as possible throughout nodes */ 6330 kernelcore_node = required_kernelcore / usable_nodes; 6331 for_each_node_state(nid, N_MEMORY) { 6332 unsigned long start_pfn, end_pfn; 6333 6334 /* 6335 * Recalculate kernelcore_node if the division per node 6336 * now exceeds what is necessary to satisfy the requested 6337 * amount of memory for the kernel 6338 */ 6339 if (required_kernelcore < kernelcore_node) 6340 kernelcore_node = required_kernelcore / usable_nodes; 6341 6342 /* 6343 * As the map is walked, we track how much memory is usable 6344 * by the kernel using kernelcore_remaining. When it is 6345 * 0, the rest of the node is usable by ZONE_MOVABLE 6346 */ 6347 kernelcore_remaining = kernelcore_node; 6348 6349 /* Go through each range of PFNs within this node */ 6350 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6351 unsigned long size_pages; 6352 6353 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 6354 if (start_pfn >= end_pfn) 6355 continue; 6356 6357 /* Account for what is only usable for kernelcore */ 6358 if (start_pfn < usable_startpfn) { 6359 unsigned long kernel_pages; 6360 kernel_pages = min(end_pfn, usable_startpfn) 6361 - start_pfn; 6362 6363 kernelcore_remaining -= min(kernel_pages, 6364 kernelcore_remaining); 6365 required_kernelcore -= min(kernel_pages, 6366 required_kernelcore); 6367 6368 /* Continue if range is now fully accounted */ 6369 if (end_pfn <= usable_startpfn) { 6370 6371 /* 6372 * Push zone_movable_pfn to the end so 6373 * that if we have to rebalance 6374 * kernelcore across nodes, we will 6375 * not double account here 6376 */ 6377 zone_movable_pfn[nid] = end_pfn; 6378 continue; 6379 } 6380 start_pfn = usable_startpfn; 6381 } 6382 6383 /* 6384 * The usable PFN range for ZONE_MOVABLE is from 6385 * start_pfn->end_pfn. Calculate size_pages as the 6386 * number of pages used as kernelcore 6387 */ 6388 size_pages = end_pfn - start_pfn; 6389 if (size_pages > kernelcore_remaining) 6390 size_pages = kernelcore_remaining; 6391 zone_movable_pfn[nid] = start_pfn + size_pages; 6392 6393 /* 6394 * Some kernelcore has been met, update counts and 6395 * break if the kernelcore for this node has been 6396 * satisfied 6397 */ 6398 required_kernelcore -= min(required_kernelcore, 6399 size_pages); 6400 kernelcore_remaining -= size_pages; 6401 if (!kernelcore_remaining) 6402 break; 6403 } 6404 } 6405 6406 /* 6407 * If there is still required_kernelcore, we do another pass with one 6408 * less node in the count. This will push zone_movable_pfn[nid] further 6409 * along on the nodes that still have memory until kernelcore is 6410 * satisfied 6411 */ 6412 usable_nodes--; 6413 if (usable_nodes && required_kernelcore > usable_nodes) 6414 goto restart; 6415 6416 out2: 6417 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 6418 for (nid = 0; nid < MAX_NUMNODES; nid++) 6419 zone_movable_pfn[nid] = 6420 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 6421 6422 out: 6423 /* restore the node_state */ 6424 node_states[N_MEMORY] = saved_node_state; 6425 } 6426 6427 /* Any regular or high memory on that node ? */ 6428 static void check_for_memory(pg_data_t *pgdat, int nid) 6429 { 6430 enum zone_type zone_type; 6431 6432 if (N_MEMORY == N_NORMAL_MEMORY) 6433 return; 6434 6435 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 6436 struct zone *zone = &pgdat->node_zones[zone_type]; 6437 if (populated_zone(zone)) { 6438 node_set_state(nid, N_HIGH_MEMORY); 6439 if (N_NORMAL_MEMORY != N_HIGH_MEMORY && 6440 zone_type <= ZONE_NORMAL) 6441 node_set_state(nid, N_NORMAL_MEMORY); 6442 break; 6443 } 6444 } 6445 } 6446 6447 /** 6448 * free_area_init_nodes - Initialise all pg_data_t and zone data 6449 * @max_zone_pfn: an array of max PFNs for each zone 6450 * 6451 * This will call free_area_init_node() for each active node in the system. 6452 * Using the page ranges provided by memblock_set_node(), the size of each 6453 * zone in each node and their holes is calculated. If the maximum PFN 6454 * between two adjacent zones match, it is assumed that the zone is empty. 6455 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 6456 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 6457 * starts where the previous one ended. For example, ZONE_DMA32 starts 6458 * at arch_max_dma_pfn. 6459 */ 6460 void __init free_area_init_nodes(unsigned long *max_zone_pfn) 6461 { 6462 unsigned long start_pfn, end_pfn; 6463 int i, nid; 6464 6465 /* Record where the zone boundaries are */ 6466 memset(arch_zone_lowest_possible_pfn, 0, 6467 sizeof(arch_zone_lowest_possible_pfn)); 6468 memset(arch_zone_highest_possible_pfn, 0, 6469 sizeof(arch_zone_highest_possible_pfn)); 6470 arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions(); 6471 arch_zone_highest_possible_pfn[0] = max_zone_pfn[0]; 6472 for (i = 1; i < MAX_NR_ZONES; i++) { 6473 if (i == ZONE_MOVABLE) 6474 continue; 6475 arch_zone_lowest_possible_pfn[i] = 6476 arch_zone_highest_possible_pfn[i-1]; 6477 arch_zone_highest_possible_pfn[i] = 6478 max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]); 6479 } 6480 arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0; 6481 arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0; 6482 6483 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 6484 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 6485 find_zone_movable_pfns_for_nodes(); 6486 6487 /* Print out the zone ranges */ 6488 pr_info("Zone ranges:\n"); 6489 for (i = 0; i < MAX_NR_ZONES; i++) { 6490 if (i == ZONE_MOVABLE) 6491 continue; 6492 pr_info(" %-8s ", zone_names[i]); 6493 if (arch_zone_lowest_possible_pfn[i] == 6494 arch_zone_highest_possible_pfn[i]) 6495 pr_cont("empty\n"); 6496 else 6497 pr_cont("[mem %#018Lx-%#018Lx]\n", 6498 (u64)arch_zone_lowest_possible_pfn[i] 6499 << PAGE_SHIFT, 6500 ((u64)arch_zone_highest_possible_pfn[i] 6501 << PAGE_SHIFT) - 1); 6502 } 6503 6504 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 6505 pr_info("Movable zone start for each node\n"); 6506 for (i = 0; i < MAX_NUMNODES; i++) { 6507 if (zone_movable_pfn[i]) 6508 pr_info(" Node %d: %#018Lx\n", i, 6509 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 6510 } 6511 6512 /* Print out the early node map */ 6513 pr_info("Early memory node ranges\n"); 6514 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) 6515 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 6516 (u64)start_pfn << PAGE_SHIFT, 6517 ((u64)end_pfn << PAGE_SHIFT) - 1); 6518 6519 /* Initialise every node */ 6520 mminit_verify_pageflags_layout(); 6521 setup_nr_node_ids(); 6522 for_each_online_node(nid) { 6523 pg_data_t *pgdat = NODE_DATA(nid); 6524 free_area_init_node(nid, NULL, 6525 find_min_pfn_for_node(nid), NULL); 6526 6527 /* Any memory on that node */ 6528 if (pgdat->node_present_pages) 6529 node_set_state(nid, N_MEMORY); 6530 check_for_memory(pgdat, nid); 6531 } 6532 } 6533 6534 static int __init cmdline_parse_core(char *p, unsigned long *core) 6535 { 6536 unsigned long long coremem; 6537 if (!p) 6538 return -EINVAL; 6539 6540 coremem = memparse(p, &p); 6541 *core = coremem >> PAGE_SHIFT; 6542 6543 /* Paranoid check that UL is enough for the coremem value */ 6544 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 6545 6546 return 0; 6547 } 6548 6549 /* 6550 * kernelcore=size sets the amount of memory for use for allocations that 6551 * cannot be reclaimed or migrated. 6552 */ 6553 static int __init cmdline_parse_kernelcore(char *p) 6554 { 6555 /* parse kernelcore=mirror */ 6556 if (parse_option_str(p, "mirror")) { 6557 mirrored_kernelcore = true; 6558 return 0; 6559 } 6560 6561 return cmdline_parse_core(p, &required_kernelcore); 6562 } 6563 6564 /* 6565 * movablecore=size sets the amount of memory for use for allocations that 6566 * can be reclaimed or migrated. 6567 */ 6568 static int __init cmdline_parse_movablecore(char *p) 6569 { 6570 return cmdline_parse_core(p, &required_movablecore); 6571 } 6572 6573 early_param("kernelcore", cmdline_parse_kernelcore); 6574 early_param("movablecore", cmdline_parse_movablecore); 6575 6576 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6577 6578 void adjust_managed_page_count(struct page *page, long count) 6579 { 6580 spin_lock(&managed_page_count_lock); 6581 page_zone(page)->managed_pages += count; 6582 totalram_pages += count; 6583 #ifdef CONFIG_HIGHMEM 6584 if (PageHighMem(page)) 6585 totalhigh_pages += count; 6586 #endif 6587 spin_unlock(&managed_page_count_lock); 6588 } 6589 EXPORT_SYMBOL(adjust_managed_page_count); 6590 6591 unsigned long free_reserved_area(void *start, void *end, int poison, char *s) 6592 { 6593 void *pos; 6594 unsigned long pages = 0; 6595 6596 start = (void *)PAGE_ALIGN((unsigned long)start); 6597 end = (void *)((unsigned long)end & PAGE_MASK); 6598 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 6599 if ((unsigned int)poison <= 0xFF) 6600 memset(pos, poison, PAGE_SIZE); 6601 free_reserved_page(virt_to_page(pos)); 6602 } 6603 6604 if (pages && s) 6605 pr_info("Freeing %s memory: %ldK (%p - %p)\n", 6606 s, pages << (PAGE_SHIFT - 10), start, end); 6607 6608 return pages; 6609 } 6610 EXPORT_SYMBOL(free_reserved_area); 6611 6612 #ifdef CONFIG_HIGHMEM 6613 void free_highmem_page(struct page *page) 6614 { 6615 __free_reserved_page(page); 6616 totalram_pages++; 6617 page_zone(page)->managed_pages++; 6618 totalhigh_pages++; 6619 } 6620 #endif 6621 6622 6623 void __init mem_init_print_info(const char *str) 6624 { 6625 unsigned long physpages, codesize, datasize, rosize, bss_size; 6626 unsigned long init_code_size, init_data_size; 6627 6628 physpages = get_num_physpages(); 6629 codesize = _etext - _stext; 6630 datasize = _edata - _sdata; 6631 rosize = __end_rodata - __start_rodata; 6632 bss_size = __bss_stop - __bss_start; 6633 init_data_size = __init_end - __init_begin; 6634 init_code_size = _einittext - _sinittext; 6635 6636 /* 6637 * Detect special cases and adjust section sizes accordingly: 6638 * 1) .init.* may be embedded into .data sections 6639 * 2) .init.text.* may be out of [__init_begin, __init_end], 6640 * please refer to arch/tile/kernel/vmlinux.lds.S. 6641 * 3) .rodata.* may be embedded into .text or .data sections. 6642 */ 6643 #define adj_init_size(start, end, size, pos, adj) \ 6644 do { \ 6645 if (start <= pos && pos < end && size > adj) \ 6646 size -= adj; \ 6647 } while (0) 6648 6649 adj_init_size(__init_begin, __init_end, init_data_size, 6650 _sinittext, init_code_size); 6651 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 6652 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 6653 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 6654 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 6655 6656 #undef adj_init_size 6657 6658 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 6659 #ifdef CONFIG_HIGHMEM 6660 ", %luK highmem" 6661 #endif 6662 "%s%s)\n", 6663 nr_free_pages() << (PAGE_SHIFT - 10), 6664 physpages << (PAGE_SHIFT - 10), 6665 codesize >> 10, datasize >> 10, rosize >> 10, 6666 (init_data_size + init_code_size) >> 10, bss_size >> 10, 6667 (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT - 10), 6668 totalcma_pages << (PAGE_SHIFT - 10), 6669 #ifdef CONFIG_HIGHMEM 6670 totalhigh_pages << (PAGE_SHIFT - 10), 6671 #endif 6672 str ? ", " : "", str ? str : ""); 6673 } 6674 6675 /** 6676 * set_dma_reserve - set the specified number of pages reserved in the first zone 6677 * @new_dma_reserve: The number of pages to mark reserved 6678 * 6679 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 6680 * In the DMA zone, a significant percentage may be consumed by kernel image 6681 * and other unfreeable allocations which can skew the watermarks badly. This 6682 * function may optionally be used to account for unfreeable pages in the 6683 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 6684 * smaller per-cpu batchsize. 6685 */ 6686 void __init set_dma_reserve(unsigned long new_dma_reserve) 6687 { 6688 dma_reserve = new_dma_reserve; 6689 } 6690 6691 void __init free_area_init(unsigned long *zones_size) 6692 { 6693 free_area_init_node(0, zones_size, 6694 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); 6695 } 6696 6697 static int page_alloc_cpu_notify(struct notifier_block *self, 6698 unsigned long action, void *hcpu) 6699 { 6700 int cpu = (unsigned long)hcpu; 6701 6702 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { 6703 lru_add_drain_cpu(cpu); 6704 drain_pages(cpu); 6705 6706 /* 6707 * Spill the event counters of the dead processor 6708 * into the current processors event counters. 6709 * This artificially elevates the count of the current 6710 * processor. 6711 */ 6712 vm_events_fold_cpu(cpu); 6713 6714 /* 6715 * Zero the differential counters of the dead processor 6716 * so that the vm statistics are consistent. 6717 * 6718 * This is only okay since the processor is dead and cannot 6719 * race with what we are doing. 6720 */ 6721 cpu_vm_stats_fold(cpu); 6722 } 6723 return NOTIFY_OK; 6724 } 6725 6726 void __init page_alloc_init(void) 6727 { 6728 hotcpu_notifier(page_alloc_cpu_notify, 0); 6729 } 6730 6731 /* 6732 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 6733 * or min_free_kbytes changes. 6734 */ 6735 static void calculate_totalreserve_pages(void) 6736 { 6737 struct pglist_data *pgdat; 6738 unsigned long reserve_pages = 0; 6739 enum zone_type i, j; 6740 6741 for_each_online_pgdat(pgdat) { 6742 for (i = 0; i < MAX_NR_ZONES; i++) { 6743 struct zone *zone = pgdat->node_zones + i; 6744 long max = 0; 6745 6746 /* Find valid and maximum lowmem_reserve in the zone */ 6747 for (j = i; j < MAX_NR_ZONES; j++) { 6748 if (zone->lowmem_reserve[j] > max) 6749 max = zone->lowmem_reserve[j]; 6750 } 6751 6752 /* we treat the high watermark as reserved pages. */ 6753 max += high_wmark_pages(zone); 6754 6755 if (max > zone->managed_pages) 6756 max = zone->managed_pages; 6757 6758 zone->totalreserve_pages = max; 6759 6760 reserve_pages += max; 6761 } 6762 } 6763 totalreserve_pages = reserve_pages; 6764 } 6765 6766 /* 6767 * setup_per_zone_lowmem_reserve - called whenever 6768 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 6769 * has a correct pages reserved value, so an adequate number of 6770 * pages are left in the zone after a successful __alloc_pages(). 6771 */ 6772 static void setup_per_zone_lowmem_reserve(void) 6773 { 6774 struct pglist_data *pgdat; 6775 enum zone_type j, idx; 6776 6777 for_each_online_pgdat(pgdat) { 6778 for (j = 0; j < MAX_NR_ZONES; j++) { 6779 struct zone *zone = pgdat->node_zones + j; 6780 unsigned long managed_pages = zone->managed_pages; 6781 6782 zone->lowmem_reserve[j] = 0; 6783 6784 idx = j; 6785 while (idx) { 6786 struct zone *lower_zone; 6787 6788 idx--; 6789 6790 if (sysctl_lowmem_reserve_ratio[idx] < 1) 6791 sysctl_lowmem_reserve_ratio[idx] = 1; 6792 6793 lower_zone = pgdat->node_zones + idx; 6794 lower_zone->lowmem_reserve[j] = managed_pages / 6795 sysctl_lowmem_reserve_ratio[idx]; 6796 managed_pages += lower_zone->managed_pages; 6797 } 6798 } 6799 } 6800 6801 /* update totalreserve_pages */ 6802 calculate_totalreserve_pages(); 6803 } 6804 6805 static void __setup_per_zone_wmarks(void) 6806 { 6807 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 6808 unsigned long lowmem_pages = 0; 6809 struct zone *zone; 6810 unsigned long flags; 6811 6812 /* Calculate total number of !ZONE_HIGHMEM pages */ 6813 for_each_zone(zone) { 6814 if (!is_highmem(zone)) 6815 lowmem_pages += zone->managed_pages; 6816 } 6817 6818 for_each_zone(zone) { 6819 u64 tmp; 6820 6821 spin_lock_irqsave(&zone->lock, flags); 6822 tmp = (u64)pages_min * zone->managed_pages; 6823 do_div(tmp, lowmem_pages); 6824 if (is_highmem(zone)) { 6825 /* 6826 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 6827 * need highmem pages, so cap pages_min to a small 6828 * value here. 6829 * 6830 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 6831 * deltas control asynch page reclaim, and so should 6832 * not be capped for highmem. 6833 */ 6834 unsigned long min_pages; 6835 6836 min_pages = zone->managed_pages / 1024; 6837 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 6838 zone->watermark[WMARK_MIN] = min_pages; 6839 } else { 6840 /* 6841 * If it's a lowmem zone, reserve a number of pages 6842 * proportionate to the zone's size. 6843 */ 6844 zone->watermark[WMARK_MIN] = tmp; 6845 } 6846 6847 /* 6848 * Set the kswapd watermarks distance according to the 6849 * scale factor in proportion to available memory, but 6850 * ensure a minimum size on small systems. 6851 */ 6852 tmp = max_t(u64, tmp >> 2, 6853 mult_frac(zone->managed_pages, 6854 watermark_scale_factor, 10000)); 6855 6856 zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 6857 zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2; 6858 6859 __mod_zone_page_state(zone, NR_ALLOC_BATCH, 6860 high_wmark_pages(zone) - low_wmark_pages(zone) - 6861 atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH])); 6862 6863 spin_unlock_irqrestore(&zone->lock, flags); 6864 } 6865 6866 /* update totalreserve_pages */ 6867 calculate_totalreserve_pages(); 6868 } 6869 6870 /** 6871 * setup_per_zone_wmarks - called when min_free_kbytes changes 6872 * or when memory is hot-{added|removed} 6873 * 6874 * Ensures that the watermark[min,low,high] values for each zone are set 6875 * correctly with respect to min_free_kbytes. 6876 */ 6877 void setup_per_zone_wmarks(void) 6878 { 6879 mutex_lock(&zonelists_mutex); 6880 __setup_per_zone_wmarks(); 6881 mutex_unlock(&zonelists_mutex); 6882 } 6883 6884 /* 6885 * Initialise min_free_kbytes. 6886 * 6887 * For small machines we want it small (128k min). For large machines 6888 * we want it large (64MB max). But it is not linear, because network 6889 * bandwidth does not increase linearly with machine size. We use 6890 * 6891 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 6892 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 6893 * 6894 * which yields 6895 * 6896 * 16MB: 512k 6897 * 32MB: 724k 6898 * 64MB: 1024k 6899 * 128MB: 1448k 6900 * 256MB: 2048k 6901 * 512MB: 2896k 6902 * 1024MB: 4096k 6903 * 2048MB: 5792k 6904 * 4096MB: 8192k 6905 * 8192MB: 11584k 6906 * 16384MB: 16384k 6907 */ 6908 int __meminit init_per_zone_wmark_min(void) 6909 { 6910 unsigned long lowmem_kbytes; 6911 int new_min_free_kbytes; 6912 6913 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 6914 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 6915 6916 if (new_min_free_kbytes > user_min_free_kbytes) { 6917 min_free_kbytes = new_min_free_kbytes; 6918 if (min_free_kbytes < 128) 6919 min_free_kbytes = 128; 6920 if (min_free_kbytes > 65536) 6921 min_free_kbytes = 65536; 6922 } else { 6923 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 6924 new_min_free_kbytes, user_min_free_kbytes); 6925 } 6926 setup_per_zone_wmarks(); 6927 refresh_zone_stat_thresholds(); 6928 setup_per_zone_lowmem_reserve(); 6929 return 0; 6930 } 6931 core_initcall(init_per_zone_wmark_min) 6932 6933 /* 6934 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 6935 * that we can call two helper functions whenever min_free_kbytes 6936 * changes. 6937 */ 6938 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 6939 void __user *buffer, size_t *length, loff_t *ppos) 6940 { 6941 int rc; 6942 6943 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6944 if (rc) 6945 return rc; 6946 6947 if (write) { 6948 user_min_free_kbytes = min_free_kbytes; 6949 setup_per_zone_wmarks(); 6950 } 6951 return 0; 6952 } 6953 6954 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 6955 void __user *buffer, size_t *length, loff_t *ppos) 6956 { 6957 int rc; 6958 6959 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6960 if (rc) 6961 return rc; 6962 6963 if (write) 6964 setup_per_zone_wmarks(); 6965 6966 return 0; 6967 } 6968 6969 #ifdef CONFIG_NUMA 6970 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 6971 void __user *buffer, size_t *length, loff_t *ppos) 6972 { 6973 struct zone *zone; 6974 int rc; 6975 6976 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6977 if (rc) 6978 return rc; 6979 6980 for_each_zone(zone) 6981 zone->min_unmapped_pages = (zone->managed_pages * 6982 sysctl_min_unmapped_ratio) / 100; 6983 return 0; 6984 } 6985 6986 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 6987 void __user *buffer, size_t *length, loff_t *ppos) 6988 { 6989 struct zone *zone; 6990 int rc; 6991 6992 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6993 if (rc) 6994 return rc; 6995 6996 for_each_zone(zone) 6997 zone->min_slab_pages = (zone->managed_pages * 6998 sysctl_min_slab_ratio) / 100; 6999 return 0; 7000 } 7001 #endif 7002 7003 /* 7004 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 7005 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 7006 * whenever sysctl_lowmem_reserve_ratio changes. 7007 * 7008 * The reserve ratio obviously has absolutely no relation with the 7009 * minimum watermarks. The lowmem reserve ratio can only make sense 7010 * if in function of the boot time zone sizes. 7011 */ 7012 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 7013 void __user *buffer, size_t *length, loff_t *ppos) 7014 { 7015 proc_dointvec_minmax(table, write, buffer, length, ppos); 7016 setup_per_zone_lowmem_reserve(); 7017 return 0; 7018 } 7019 7020 /* 7021 * percpu_pagelist_fraction - changes the pcp->high for each zone on each 7022 * cpu. It is the fraction of total pages in each zone that a hot per cpu 7023 * pagelist can have before it gets flushed back to buddy allocator. 7024 */ 7025 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, 7026 void __user *buffer, size_t *length, loff_t *ppos) 7027 { 7028 struct zone *zone; 7029 int old_percpu_pagelist_fraction; 7030 int ret; 7031 7032 mutex_lock(&pcp_batch_high_lock); 7033 old_percpu_pagelist_fraction = percpu_pagelist_fraction; 7034 7035 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 7036 if (!write || ret < 0) 7037 goto out; 7038 7039 /* Sanity checking to avoid pcp imbalance */ 7040 if (percpu_pagelist_fraction && 7041 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { 7042 percpu_pagelist_fraction = old_percpu_pagelist_fraction; 7043 ret = -EINVAL; 7044 goto out; 7045 } 7046 7047 /* No change? */ 7048 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) 7049 goto out; 7050 7051 for_each_populated_zone(zone) { 7052 unsigned int cpu; 7053 7054 for_each_possible_cpu(cpu) 7055 pageset_set_high_and_batch(zone, 7056 per_cpu_ptr(zone->pageset, cpu)); 7057 } 7058 out: 7059 mutex_unlock(&pcp_batch_high_lock); 7060 return ret; 7061 } 7062 7063 #ifdef CONFIG_NUMA 7064 int hashdist = HASHDIST_DEFAULT; 7065 7066 static int __init set_hashdist(char *str) 7067 { 7068 if (!str) 7069 return 0; 7070 hashdist = simple_strtoul(str, &str, 0); 7071 return 1; 7072 } 7073 __setup("hashdist=", set_hashdist); 7074 #endif 7075 7076 /* 7077 * allocate a large system hash table from bootmem 7078 * - it is assumed that the hash table must contain an exact power-of-2 7079 * quantity of entries 7080 * - limit is the number of hash buckets, not the total allocation size 7081 */ 7082 void *__init alloc_large_system_hash(const char *tablename, 7083 unsigned long bucketsize, 7084 unsigned long numentries, 7085 int scale, 7086 int flags, 7087 unsigned int *_hash_shift, 7088 unsigned int *_hash_mask, 7089 unsigned long low_limit, 7090 unsigned long high_limit) 7091 { 7092 unsigned long long max = high_limit; 7093 unsigned long log2qty, size; 7094 void *table = NULL; 7095 7096 /* allow the kernel cmdline to have a say */ 7097 if (!numentries) { 7098 /* round applicable memory size up to nearest megabyte */ 7099 numentries = nr_kernel_pages; 7100 7101 /* It isn't necessary when PAGE_SIZE >= 1MB */ 7102 if (PAGE_SHIFT < 20) 7103 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 7104 7105 /* limit to 1 bucket per 2^scale bytes of low memory */ 7106 if (scale > PAGE_SHIFT) 7107 numentries >>= (scale - PAGE_SHIFT); 7108 else 7109 numentries <<= (PAGE_SHIFT - scale); 7110 7111 /* Make sure we've got at least a 0-order allocation.. */ 7112 if (unlikely(flags & HASH_SMALL)) { 7113 /* Makes no sense without HASH_EARLY */ 7114 WARN_ON(!(flags & HASH_EARLY)); 7115 if (!(numentries >> *_hash_shift)) { 7116 numentries = 1UL << *_hash_shift; 7117 BUG_ON(!numentries); 7118 } 7119 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 7120 numentries = PAGE_SIZE / bucketsize; 7121 } 7122 numentries = roundup_pow_of_two(numentries); 7123 7124 /* limit allocation size to 1/16 total memory by default */ 7125 if (max == 0) { 7126 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 7127 do_div(max, bucketsize); 7128 } 7129 max = min(max, 0x80000000ULL); 7130 7131 if (numentries < low_limit) 7132 numentries = low_limit; 7133 if (numentries > max) 7134 numentries = max; 7135 7136 log2qty = ilog2(numentries); 7137 7138 do { 7139 size = bucketsize << log2qty; 7140 if (flags & HASH_EARLY) 7141 table = memblock_virt_alloc_nopanic(size, 0); 7142 else if (hashdist) 7143 table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL); 7144 else { 7145 /* 7146 * If bucketsize is not a power-of-two, we may free 7147 * some pages at the end of hash table which 7148 * alloc_pages_exact() automatically does 7149 */ 7150 if (get_order(size) < MAX_ORDER) { 7151 table = alloc_pages_exact(size, GFP_ATOMIC); 7152 kmemleak_alloc(table, size, 1, GFP_ATOMIC); 7153 } 7154 } 7155 } while (!table && size > PAGE_SIZE && --log2qty); 7156 7157 if (!table) 7158 panic("Failed to allocate %s hash table\n", tablename); 7159 7160 pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n", 7161 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size); 7162 7163 if (_hash_shift) 7164 *_hash_shift = log2qty; 7165 if (_hash_mask) 7166 *_hash_mask = (1 << log2qty) - 1; 7167 7168 return table; 7169 } 7170 7171 /* 7172 * This function checks whether pageblock includes unmovable pages or not. 7173 * If @count is not zero, it is okay to include less @count unmovable pages 7174 * 7175 * PageLRU check without isolation or lru_lock could race so that 7176 * MIGRATE_MOVABLE block might include unmovable pages. It means you can't 7177 * expect this function should be exact. 7178 */ 7179 bool has_unmovable_pages(struct zone *zone, struct page *page, int count, 7180 bool skip_hwpoisoned_pages) 7181 { 7182 unsigned long pfn, iter, found; 7183 int mt; 7184 7185 /* 7186 * For avoiding noise data, lru_add_drain_all() should be called 7187 * If ZONE_MOVABLE, the zone never contains unmovable pages 7188 */ 7189 if (zone_idx(zone) == ZONE_MOVABLE) 7190 return false; 7191 mt = get_pageblock_migratetype(page); 7192 if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt)) 7193 return false; 7194 7195 pfn = page_to_pfn(page); 7196 for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) { 7197 unsigned long check = pfn + iter; 7198 7199 if (!pfn_valid_within(check)) 7200 continue; 7201 7202 page = pfn_to_page(check); 7203 7204 /* 7205 * Hugepages are not in LRU lists, but they're movable. 7206 * We need not scan over tail pages bacause we don't 7207 * handle each tail page individually in migration. 7208 */ 7209 if (PageHuge(page)) { 7210 iter = round_up(iter + 1, 1<<compound_order(page)) - 1; 7211 continue; 7212 } 7213 7214 /* 7215 * We can't use page_count without pin a page 7216 * because another CPU can free compound page. 7217 * This check already skips compound tails of THP 7218 * because their page->_refcount is zero at all time. 7219 */ 7220 if (!page_ref_count(page)) { 7221 if (PageBuddy(page)) 7222 iter += (1 << page_order(page)) - 1; 7223 continue; 7224 } 7225 7226 /* 7227 * The HWPoisoned page may be not in buddy system, and 7228 * page_count() is not 0. 7229 */ 7230 if (skip_hwpoisoned_pages && PageHWPoison(page)) 7231 continue; 7232 7233 if (!PageLRU(page)) 7234 found++; 7235 /* 7236 * If there are RECLAIMABLE pages, we need to check 7237 * it. But now, memory offline itself doesn't call 7238 * shrink_node_slabs() and it still to be fixed. 7239 */ 7240 /* 7241 * If the page is not RAM, page_count()should be 0. 7242 * we don't need more check. This is an _used_ not-movable page. 7243 * 7244 * The problematic thing here is PG_reserved pages. PG_reserved 7245 * is set to both of a memory hole page and a _used_ kernel 7246 * page at boot. 7247 */ 7248 if (found > count) 7249 return true; 7250 } 7251 return false; 7252 } 7253 7254 bool is_pageblock_removable_nolock(struct page *page) 7255 { 7256 struct zone *zone; 7257 unsigned long pfn; 7258 7259 /* 7260 * We have to be careful here because we are iterating over memory 7261 * sections which are not zone aware so we might end up outside of 7262 * the zone but still within the section. 7263 * We have to take care about the node as well. If the node is offline 7264 * its NODE_DATA will be NULL - see page_zone. 7265 */ 7266 if (!node_online(page_to_nid(page))) 7267 return false; 7268 7269 zone = page_zone(page); 7270 pfn = page_to_pfn(page); 7271 if (!zone_spans_pfn(zone, pfn)) 7272 return false; 7273 7274 return !has_unmovable_pages(zone, page, 0, true); 7275 } 7276 7277 #if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA) 7278 7279 static unsigned long pfn_max_align_down(unsigned long pfn) 7280 { 7281 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, 7282 pageblock_nr_pages) - 1); 7283 } 7284 7285 static unsigned long pfn_max_align_up(unsigned long pfn) 7286 { 7287 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, 7288 pageblock_nr_pages)); 7289 } 7290 7291 /* [start, end) must belong to a single zone. */ 7292 static int __alloc_contig_migrate_range(struct compact_control *cc, 7293 unsigned long start, unsigned long end) 7294 { 7295 /* This function is based on compact_zone() from compaction.c. */ 7296 unsigned long nr_reclaimed; 7297 unsigned long pfn = start; 7298 unsigned int tries = 0; 7299 int ret = 0; 7300 7301 migrate_prep(); 7302 7303 while (pfn < end || !list_empty(&cc->migratepages)) { 7304 if (fatal_signal_pending(current)) { 7305 ret = -EINTR; 7306 break; 7307 } 7308 7309 if (list_empty(&cc->migratepages)) { 7310 cc->nr_migratepages = 0; 7311 pfn = isolate_migratepages_range(cc, pfn, end); 7312 if (!pfn) { 7313 ret = -EINTR; 7314 break; 7315 } 7316 tries = 0; 7317 } else if (++tries == 5) { 7318 ret = ret < 0 ? ret : -EBUSY; 7319 break; 7320 } 7321 7322 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 7323 &cc->migratepages); 7324 cc->nr_migratepages -= nr_reclaimed; 7325 7326 ret = migrate_pages(&cc->migratepages, alloc_migrate_target, 7327 NULL, 0, cc->mode, MR_CMA); 7328 } 7329 if (ret < 0) { 7330 putback_movable_pages(&cc->migratepages); 7331 return ret; 7332 } 7333 return 0; 7334 } 7335 7336 /** 7337 * alloc_contig_range() -- tries to allocate given range of pages 7338 * @start: start PFN to allocate 7339 * @end: one-past-the-last PFN to allocate 7340 * @migratetype: migratetype of the underlaying pageblocks (either 7341 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 7342 * in range must have the same migratetype and it must 7343 * be either of the two. 7344 * 7345 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 7346 * aligned, however it's the caller's responsibility to guarantee that 7347 * we are the only thread that changes migrate type of pageblocks the 7348 * pages fall in. 7349 * 7350 * The PFN range must belong to a single zone. 7351 * 7352 * Returns zero on success or negative error code. On success all 7353 * pages which PFN is in [start, end) are allocated for the caller and 7354 * need to be freed with free_contig_range(). 7355 */ 7356 int alloc_contig_range(unsigned long start, unsigned long end, 7357 unsigned migratetype) 7358 { 7359 unsigned long outer_start, outer_end; 7360 unsigned int order; 7361 int ret = 0; 7362 7363 struct compact_control cc = { 7364 .nr_migratepages = 0, 7365 .order = -1, 7366 .zone = page_zone(pfn_to_page(start)), 7367 .mode = MIGRATE_SYNC, 7368 .ignore_skip_hint = true, 7369 }; 7370 INIT_LIST_HEAD(&cc.migratepages); 7371 7372 /* 7373 * What we do here is we mark all pageblocks in range as 7374 * MIGRATE_ISOLATE. Because pageblock and max order pages may 7375 * have different sizes, and due to the way page allocator 7376 * work, we align the range to biggest of the two pages so 7377 * that page allocator won't try to merge buddies from 7378 * different pageblocks and change MIGRATE_ISOLATE to some 7379 * other migration type. 7380 * 7381 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 7382 * migrate the pages from an unaligned range (ie. pages that 7383 * we are interested in). This will put all the pages in 7384 * range back to page allocator as MIGRATE_ISOLATE. 7385 * 7386 * When this is done, we take the pages in range from page 7387 * allocator removing them from the buddy system. This way 7388 * page allocator will never consider using them. 7389 * 7390 * This lets us mark the pageblocks back as 7391 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 7392 * aligned range but not in the unaligned, original range are 7393 * put back to page allocator so that buddy can use them. 7394 */ 7395 7396 ret = start_isolate_page_range(pfn_max_align_down(start), 7397 pfn_max_align_up(end), migratetype, 7398 false); 7399 if (ret) 7400 return ret; 7401 7402 /* 7403 * In case of -EBUSY, we'd like to know which page causes problem. 7404 * So, just fall through. We will check it in test_pages_isolated(). 7405 */ 7406 ret = __alloc_contig_migrate_range(&cc, start, end); 7407 if (ret && ret != -EBUSY) 7408 goto done; 7409 7410 /* 7411 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 7412 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 7413 * more, all pages in [start, end) are free in page allocator. 7414 * What we are going to do is to allocate all pages from 7415 * [start, end) (that is remove them from page allocator). 7416 * 7417 * The only problem is that pages at the beginning and at the 7418 * end of interesting range may be not aligned with pages that 7419 * page allocator holds, ie. they can be part of higher order 7420 * pages. Because of this, we reserve the bigger range and 7421 * once this is done free the pages we are not interested in. 7422 * 7423 * We don't have to hold zone->lock here because the pages are 7424 * isolated thus they won't get removed from buddy. 7425 */ 7426 7427 lru_add_drain_all(); 7428 drain_all_pages(cc.zone); 7429 7430 order = 0; 7431 outer_start = start; 7432 while (!PageBuddy(pfn_to_page(outer_start))) { 7433 if (++order >= MAX_ORDER) { 7434 outer_start = start; 7435 break; 7436 } 7437 outer_start &= ~0UL << order; 7438 } 7439 7440 if (outer_start != start) { 7441 order = page_order(pfn_to_page(outer_start)); 7442 7443 /* 7444 * outer_start page could be small order buddy page and 7445 * it doesn't include start page. Adjust outer_start 7446 * in this case to report failed page properly 7447 * on tracepoint in test_pages_isolated() 7448 */ 7449 if (outer_start + (1UL << order) <= start) 7450 outer_start = start; 7451 } 7452 7453 /* Make sure the range is really isolated. */ 7454 if (test_pages_isolated(outer_start, end, false)) { 7455 pr_info("%s: [%lx, %lx) PFNs busy\n", 7456 __func__, outer_start, end); 7457 ret = -EBUSY; 7458 goto done; 7459 } 7460 7461 /* Grab isolated pages from freelists. */ 7462 outer_end = isolate_freepages_range(&cc, outer_start, end); 7463 if (!outer_end) { 7464 ret = -EBUSY; 7465 goto done; 7466 } 7467 7468 /* Free head and tail (if any) */ 7469 if (start != outer_start) 7470 free_contig_range(outer_start, start - outer_start); 7471 if (end != outer_end) 7472 free_contig_range(end, outer_end - end); 7473 7474 done: 7475 undo_isolate_page_range(pfn_max_align_down(start), 7476 pfn_max_align_up(end), migratetype); 7477 return ret; 7478 } 7479 7480 void free_contig_range(unsigned long pfn, unsigned nr_pages) 7481 { 7482 unsigned int count = 0; 7483 7484 for (; nr_pages--; pfn++) { 7485 struct page *page = pfn_to_page(pfn); 7486 7487 count += page_count(page) != 1; 7488 __free_page(page); 7489 } 7490 WARN(count != 0, "%d pages are still in use!\n", count); 7491 } 7492 #endif 7493 7494 #ifdef CONFIG_MEMORY_HOTPLUG 7495 /* 7496 * The zone indicated has a new number of managed_pages; batch sizes and percpu 7497 * page high values need to be recalulated. 7498 */ 7499 void __meminit zone_pcp_update(struct zone *zone) 7500 { 7501 unsigned cpu; 7502 mutex_lock(&pcp_batch_high_lock); 7503 for_each_possible_cpu(cpu) 7504 pageset_set_high_and_batch(zone, 7505 per_cpu_ptr(zone->pageset, cpu)); 7506 mutex_unlock(&pcp_batch_high_lock); 7507 } 7508 #endif 7509 7510 void zone_pcp_reset(struct zone *zone) 7511 { 7512 unsigned long flags; 7513 int cpu; 7514 struct per_cpu_pageset *pset; 7515 7516 /* avoid races with drain_pages() */ 7517 local_irq_save(flags); 7518 if (zone->pageset != &boot_pageset) { 7519 for_each_online_cpu(cpu) { 7520 pset = per_cpu_ptr(zone->pageset, cpu); 7521 drain_zonestat(zone, pset); 7522 } 7523 free_percpu(zone->pageset); 7524 zone->pageset = &boot_pageset; 7525 } 7526 local_irq_restore(flags); 7527 } 7528 7529 #ifdef CONFIG_MEMORY_HOTREMOVE 7530 /* 7531 * All pages in the range must be in a single zone and isolated 7532 * before calling this. 7533 */ 7534 void 7535 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 7536 { 7537 struct page *page; 7538 struct zone *zone; 7539 unsigned int order, i; 7540 unsigned long pfn; 7541 unsigned long flags; 7542 /* find the first valid pfn */ 7543 for (pfn = start_pfn; pfn < end_pfn; pfn++) 7544 if (pfn_valid(pfn)) 7545 break; 7546 if (pfn == end_pfn) 7547 return; 7548 zone = page_zone(pfn_to_page(pfn)); 7549 spin_lock_irqsave(&zone->lock, flags); 7550 pfn = start_pfn; 7551 while (pfn < end_pfn) { 7552 if (!pfn_valid(pfn)) { 7553 pfn++; 7554 continue; 7555 } 7556 page = pfn_to_page(pfn); 7557 /* 7558 * The HWPoisoned page may be not in buddy system, and 7559 * page_count() is not 0. 7560 */ 7561 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 7562 pfn++; 7563 SetPageReserved(page); 7564 continue; 7565 } 7566 7567 BUG_ON(page_count(page)); 7568 BUG_ON(!PageBuddy(page)); 7569 order = page_order(page); 7570 #ifdef CONFIG_DEBUG_VM 7571 pr_info("remove from free list %lx %d %lx\n", 7572 pfn, 1 << order, end_pfn); 7573 #endif 7574 list_del(&page->lru); 7575 rmv_page_order(page); 7576 zone->free_area[order].nr_free--; 7577 for (i = 0; i < (1 << order); i++) 7578 SetPageReserved((page+i)); 7579 pfn += (1 << order); 7580 } 7581 spin_unlock_irqrestore(&zone->lock, flags); 7582 } 7583 #endif 7584 7585 bool is_free_buddy_page(struct page *page) 7586 { 7587 struct zone *zone = page_zone(page); 7588 unsigned long pfn = page_to_pfn(page); 7589 unsigned long flags; 7590 unsigned int order; 7591 7592 spin_lock_irqsave(&zone->lock, flags); 7593 for (order = 0; order < MAX_ORDER; order++) { 7594 struct page *page_head = page - (pfn & ((1 << order) - 1)); 7595 7596 if (PageBuddy(page_head) && page_order(page_head) >= order) 7597 break; 7598 } 7599 spin_unlock_irqrestore(&zone->lock, flags); 7600 7601 return order < MAX_ORDER; 7602 } 7603