1 /* 2 * linux/mm/page_alloc.c 3 * 4 * Manages the free list, the system allocates free pages here. 5 * Note that kmalloc() lives in slab.c 6 * 7 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 8 * Swap reorganised 29.12.95, Stephen Tweedie 9 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 10 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 11 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 12 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 13 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 14 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 15 */ 16 17 #include <linux/stddef.h> 18 #include <linux/mm.h> 19 #include <linux/swap.h> 20 #include <linux/interrupt.h> 21 #include <linux/pagemap.h> 22 #include <linux/jiffies.h> 23 #include <linux/bootmem.h> 24 #include <linux/memblock.h> 25 #include <linux/compiler.h> 26 #include <linux/kernel.h> 27 #include <linux/kmemcheck.h> 28 #include <linux/kasan.h> 29 #include <linux/module.h> 30 #include <linux/suspend.h> 31 #include <linux/pagevec.h> 32 #include <linux/blkdev.h> 33 #include <linux/slab.h> 34 #include <linux/ratelimit.h> 35 #include <linux/oom.h> 36 #include <linux/notifier.h> 37 #include <linux/topology.h> 38 #include <linux/sysctl.h> 39 #include <linux/cpu.h> 40 #include <linux/cpuset.h> 41 #include <linux/memory_hotplug.h> 42 #include <linux/nodemask.h> 43 #include <linux/vmalloc.h> 44 #include <linux/vmstat.h> 45 #include <linux/mempolicy.h> 46 #include <linux/memremap.h> 47 #include <linux/stop_machine.h> 48 #include <linux/sort.h> 49 #include <linux/pfn.h> 50 #include <linux/backing-dev.h> 51 #include <linux/fault-inject.h> 52 #include <linux/page-isolation.h> 53 #include <linux/page_ext.h> 54 #include <linux/debugobjects.h> 55 #include <linux/kmemleak.h> 56 #include <linux/compaction.h> 57 #include <trace/events/kmem.h> 58 #include <linux/prefetch.h> 59 #include <linux/mm_inline.h> 60 #include <linux/migrate.h> 61 #include <linux/page_ext.h> 62 #include <linux/hugetlb.h> 63 #include <linux/sched/rt.h> 64 #include <linux/page_owner.h> 65 #include <linux/kthread.h> 66 67 #include <asm/sections.h> 68 #include <asm/tlbflush.h> 69 #include <asm/div64.h> 70 #include "internal.h" 71 72 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 73 static DEFINE_MUTEX(pcp_batch_high_lock); 74 #define MIN_PERCPU_PAGELIST_FRACTION (8) 75 76 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 77 DEFINE_PER_CPU(int, numa_node); 78 EXPORT_PER_CPU_SYMBOL(numa_node); 79 #endif 80 81 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 82 /* 83 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 84 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 85 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 86 * defined in <linux/topology.h>. 87 */ 88 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 89 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 90 int _node_numa_mem_[MAX_NUMNODES]; 91 #endif 92 93 /* 94 * Array of node states. 95 */ 96 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 97 [N_POSSIBLE] = NODE_MASK_ALL, 98 [N_ONLINE] = { { [0] = 1UL } }, 99 #ifndef CONFIG_NUMA 100 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 101 #ifdef CONFIG_HIGHMEM 102 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 103 #endif 104 #ifdef CONFIG_MOVABLE_NODE 105 [N_MEMORY] = { { [0] = 1UL } }, 106 #endif 107 [N_CPU] = { { [0] = 1UL } }, 108 #endif /* NUMA */ 109 }; 110 EXPORT_SYMBOL(node_states); 111 112 /* Protect totalram_pages and zone->managed_pages */ 113 static DEFINE_SPINLOCK(managed_page_count_lock); 114 115 unsigned long totalram_pages __read_mostly; 116 unsigned long totalreserve_pages __read_mostly; 117 unsigned long totalcma_pages __read_mostly; 118 119 int percpu_pagelist_fraction; 120 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 121 122 /* 123 * A cached value of the page's pageblock's migratetype, used when the page is 124 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 125 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 126 * Also the migratetype set in the page does not necessarily match the pcplist 127 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 128 * other index - this ensures that it will be put on the correct CMA freelist. 129 */ 130 static inline int get_pcppage_migratetype(struct page *page) 131 { 132 return page->index; 133 } 134 135 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 136 { 137 page->index = migratetype; 138 } 139 140 #ifdef CONFIG_PM_SLEEP 141 /* 142 * The following functions are used by the suspend/hibernate code to temporarily 143 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 144 * while devices are suspended. To avoid races with the suspend/hibernate code, 145 * they should always be called with pm_mutex held (gfp_allowed_mask also should 146 * only be modified with pm_mutex held, unless the suspend/hibernate code is 147 * guaranteed not to run in parallel with that modification). 148 */ 149 150 static gfp_t saved_gfp_mask; 151 152 void pm_restore_gfp_mask(void) 153 { 154 WARN_ON(!mutex_is_locked(&pm_mutex)); 155 if (saved_gfp_mask) { 156 gfp_allowed_mask = saved_gfp_mask; 157 saved_gfp_mask = 0; 158 } 159 } 160 161 void pm_restrict_gfp_mask(void) 162 { 163 WARN_ON(!mutex_is_locked(&pm_mutex)); 164 WARN_ON(saved_gfp_mask); 165 saved_gfp_mask = gfp_allowed_mask; 166 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 167 } 168 169 bool pm_suspended_storage(void) 170 { 171 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 172 return false; 173 return true; 174 } 175 #endif /* CONFIG_PM_SLEEP */ 176 177 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 178 unsigned int pageblock_order __read_mostly; 179 #endif 180 181 static void __free_pages_ok(struct page *page, unsigned int order); 182 183 /* 184 * results with 256, 32 in the lowmem_reserve sysctl: 185 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 186 * 1G machine -> (16M dma, 784M normal, 224M high) 187 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 188 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 189 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 190 * 191 * TBD: should special case ZONE_DMA32 machines here - in those we normally 192 * don't need any ZONE_NORMAL reservation 193 */ 194 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = { 195 #ifdef CONFIG_ZONE_DMA 196 256, 197 #endif 198 #ifdef CONFIG_ZONE_DMA32 199 256, 200 #endif 201 #ifdef CONFIG_HIGHMEM 202 32, 203 #endif 204 32, 205 }; 206 207 EXPORT_SYMBOL(totalram_pages); 208 209 static char * const zone_names[MAX_NR_ZONES] = { 210 #ifdef CONFIG_ZONE_DMA 211 "DMA", 212 #endif 213 #ifdef CONFIG_ZONE_DMA32 214 "DMA32", 215 #endif 216 "Normal", 217 #ifdef CONFIG_HIGHMEM 218 "HighMem", 219 #endif 220 "Movable", 221 #ifdef CONFIG_ZONE_DEVICE 222 "Device", 223 #endif 224 }; 225 226 char * const migratetype_names[MIGRATE_TYPES] = { 227 "Unmovable", 228 "Movable", 229 "Reclaimable", 230 "HighAtomic", 231 #ifdef CONFIG_CMA 232 "CMA", 233 #endif 234 #ifdef CONFIG_MEMORY_ISOLATION 235 "Isolate", 236 #endif 237 }; 238 239 compound_page_dtor * const compound_page_dtors[] = { 240 NULL, 241 free_compound_page, 242 #ifdef CONFIG_HUGETLB_PAGE 243 free_huge_page, 244 #endif 245 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 246 free_transhuge_page, 247 #endif 248 }; 249 250 int min_free_kbytes = 1024; 251 int user_min_free_kbytes = -1; 252 int watermark_scale_factor = 10; 253 254 static unsigned long __meminitdata nr_kernel_pages; 255 static unsigned long __meminitdata nr_all_pages; 256 static unsigned long __meminitdata dma_reserve; 257 258 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 259 static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES]; 260 static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES]; 261 static unsigned long __initdata required_kernelcore; 262 static unsigned long __initdata required_movablecore; 263 static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES]; 264 static bool mirrored_kernelcore; 265 266 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 267 int movable_zone; 268 EXPORT_SYMBOL(movable_zone); 269 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 270 271 #if MAX_NUMNODES > 1 272 int nr_node_ids __read_mostly = MAX_NUMNODES; 273 int nr_online_nodes __read_mostly = 1; 274 EXPORT_SYMBOL(nr_node_ids); 275 EXPORT_SYMBOL(nr_online_nodes); 276 #endif 277 278 int page_group_by_mobility_disabled __read_mostly; 279 280 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 281 static inline void reset_deferred_meminit(pg_data_t *pgdat) 282 { 283 pgdat->first_deferred_pfn = ULONG_MAX; 284 } 285 286 /* Returns true if the struct page for the pfn is uninitialised */ 287 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 288 { 289 if (pfn >= NODE_DATA(early_pfn_to_nid(pfn))->first_deferred_pfn) 290 return true; 291 292 return false; 293 } 294 295 static inline bool early_page_nid_uninitialised(unsigned long pfn, int nid) 296 { 297 if (pfn >= NODE_DATA(nid)->first_deferred_pfn) 298 return true; 299 300 return false; 301 } 302 303 /* 304 * Returns false when the remaining initialisation should be deferred until 305 * later in the boot cycle when it can be parallelised. 306 */ 307 static inline bool update_defer_init(pg_data_t *pgdat, 308 unsigned long pfn, unsigned long zone_end, 309 unsigned long *nr_initialised) 310 { 311 unsigned long max_initialise; 312 313 /* Always populate low zones for address-contrained allocations */ 314 if (zone_end < pgdat_end_pfn(pgdat)) 315 return true; 316 /* 317 * Initialise at least 2G of a node but also take into account that 318 * two large system hashes that can take up 1GB for 0.25TB/node. 319 */ 320 max_initialise = max(2UL << (30 - PAGE_SHIFT), 321 (pgdat->node_spanned_pages >> 8)); 322 323 (*nr_initialised)++; 324 if ((*nr_initialised > max_initialise) && 325 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 326 pgdat->first_deferred_pfn = pfn; 327 return false; 328 } 329 330 return true; 331 } 332 #else 333 static inline void reset_deferred_meminit(pg_data_t *pgdat) 334 { 335 } 336 337 static inline bool early_page_uninitialised(unsigned long pfn) 338 { 339 return false; 340 } 341 342 static inline bool early_page_nid_uninitialised(unsigned long pfn, int nid) 343 { 344 return false; 345 } 346 347 static inline bool update_defer_init(pg_data_t *pgdat, 348 unsigned long pfn, unsigned long zone_end, 349 unsigned long *nr_initialised) 350 { 351 return true; 352 } 353 #endif 354 355 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 356 static inline unsigned long *get_pageblock_bitmap(struct page *page, 357 unsigned long pfn) 358 { 359 #ifdef CONFIG_SPARSEMEM 360 return __pfn_to_section(pfn)->pageblock_flags; 361 #else 362 return page_zone(page)->pageblock_flags; 363 #endif /* CONFIG_SPARSEMEM */ 364 } 365 366 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn) 367 { 368 #ifdef CONFIG_SPARSEMEM 369 pfn &= (PAGES_PER_SECTION-1); 370 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 371 #else 372 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 373 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 374 #endif /* CONFIG_SPARSEMEM */ 375 } 376 377 /** 378 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 379 * @page: The page within the block of interest 380 * @pfn: The target page frame number 381 * @end_bitidx: The last bit of interest to retrieve 382 * @mask: mask of bits that the caller is interested in 383 * 384 * Return: pageblock_bits flags 385 */ 386 static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page, 387 unsigned long pfn, 388 unsigned long end_bitidx, 389 unsigned long mask) 390 { 391 unsigned long *bitmap; 392 unsigned long bitidx, word_bitidx; 393 unsigned long word; 394 395 bitmap = get_pageblock_bitmap(page, pfn); 396 bitidx = pfn_to_bitidx(page, pfn); 397 word_bitidx = bitidx / BITS_PER_LONG; 398 bitidx &= (BITS_PER_LONG-1); 399 400 word = bitmap[word_bitidx]; 401 bitidx += end_bitidx; 402 return (word >> (BITS_PER_LONG - bitidx - 1)) & mask; 403 } 404 405 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, 406 unsigned long end_bitidx, 407 unsigned long mask) 408 { 409 return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask); 410 } 411 412 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn) 413 { 414 return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK); 415 } 416 417 /** 418 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 419 * @page: The page within the block of interest 420 * @flags: The flags to set 421 * @pfn: The target page frame number 422 * @end_bitidx: The last bit of interest 423 * @mask: mask of bits that the caller is interested in 424 */ 425 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 426 unsigned long pfn, 427 unsigned long end_bitidx, 428 unsigned long mask) 429 { 430 unsigned long *bitmap; 431 unsigned long bitidx, word_bitidx; 432 unsigned long old_word, word; 433 434 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 435 436 bitmap = get_pageblock_bitmap(page, pfn); 437 bitidx = pfn_to_bitidx(page, pfn); 438 word_bitidx = bitidx / BITS_PER_LONG; 439 bitidx &= (BITS_PER_LONG-1); 440 441 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 442 443 bitidx += end_bitidx; 444 mask <<= (BITS_PER_LONG - bitidx - 1); 445 flags <<= (BITS_PER_LONG - bitidx - 1); 446 447 word = READ_ONCE(bitmap[word_bitidx]); 448 for (;;) { 449 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 450 if (word == old_word) 451 break; 452 word = old_word; 453 } 454 } 455 456 void set_pageblock_migratetype(struct page *page, int migratetype) 457 { 458 if (unlikely(page_group_by_mobility_disabled && 459 migratetype < MIGRATE_PCPTYPES)) 460 migratetype = MIGRATE_UNMOVABLE; 461 462 set_pageblock_flags_group(page, (unsigned long)migratetype, 463 PB_migrate, PB_migrate_end); 464 } 465 466 #ifdef CONFIG_DEBUG_VM 467 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 468 { 469 int ret = 0; 470 unsigned seq; 471 unsigned long pfn = page_to_pfn(page); 472 unsigned long sp, start_pfn; 473 474 do { 475 seq = zone_span_seqbegin(zone); 476 start_pfn = zone->zone_start_pfn; 477 sp = zone->spanned_pages; 478 if (!zone_spans_pfn(zone, pfn)) 479 ret = 1; 480 } while (zone_span_seqretry(zone, seq)); 481 482 if (ret) 483 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 484 pfn, zone_to_nid(zone), zone->name, 485 start_pfn, start_pfn + sp); 486 487 return ret; 488 } 489 490 static int page_is_consistent(struct zone *zone, struct page *page) 491 { 492 if (!pfn_valid_within(page_to_pfn(page))) 493 return 0; 494 if (zone != page_zone(page)) 495 return 0; 496 497 return 1; 498 } 499 /* 500 * Temporary debugging check for pages not lying within a given zone. 501 */ 502 static int bad_range(struct zone *zone, struct page *page) 503 { 504 if (page_outside_zone_boundaries(zone, page)) 505 return 1; 506 if (!page_is_consistent(zone, page)) 507 return 1; 508 509 return 0; 510 } 511 #else 512 static inline int bad_range(struct zone *zone, struct page *page) 513 { 514 return 0; 515 } 516 #endif 517 518 static void bad_page(struct page *page, const char *reason, 519 unsigned long bad_flags) 520 { 521 static unsigned long resume; 522 static unsigned long nr_shown; 523 static unsigned long nr_unshown; 524 525 /* 526 * Allow a burst of 60 reports, then keep quiet for that minute; 527 * or allow a steady drip of one report per second. 528 */ 529 if (nr_shown == 60) { 530 if (time_before(jiffies, resume)) { 531 nr_unshown++; 532 goto out; 533 } 534 if (nr_unshown) { 535 pr_alert( 536 "BUG: Bad page state: %lu messages suppressed\n", 537 nr_unshown); 538 nr_unshown = 0; 539 } 540 nr_shown = 0; 541 } 542 if (nr_shown++ == 0) 543 resume = jiffies + 60 * HZ; 544 545 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 546 current->comm, page_to_pfn(page)); 547 __dump_page(page, reason); 548 bad_flags &= page->flags; 549 if (bad_flags) 550 pr_alert("bad because of flags: %#lx(%pGp)\n", 551 bad_flags, &bad_flags); 552 dump_page_owner(page); 553 554 print_modules(); 555 dump_stack(); 556 out: 557 /* Leave bad fields for debug, except PageBuddy could make trouble */ 558 page_mapcount_reset(page); /* remove PageBuddy */ 559 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 560 } 561 562 /* 563 * Higher-order pages are called "compound pages". They are structured thusly: 564 * 565 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 566 * 567 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 568 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 569 * 570 * The first tail page's ->compound_dtor holds the offset in array of compound 571 * page destructors. See compound_page_dtors. 572 * 573 * The first tail page's ->compound_order holds the order of allocation. 574 * This usage means that zero-order pages may not be compound. 575 */ 576 577 void free_compound_page(struct page *page) 578 { 579 __free_pages_ok(page, compound_order(page)); 580 } 581 582 void prep_compound_page(struct page *page, unsigned int order) 583 { 584 int i; 585 int nr_pages = 1 << order; 586 587 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 588 set_compound_order(page, order); 589 __SetPageHead(page); 590 for (i = 1; i < nr_pages; i++) { 591 struct page *p = page + i; 592 set_page_count(p, 0); 593 p->mapping = TAIL_MAPPING; 594 set_compound_head(p, page); 595 } 596 atomic_set(compound_mapcount_ptr(page), -1); 597 } 598 599 #ifdef CONFIG_DEBUG_PAGEALLOC 600 unsigned int _debug_guardpage_minorder; 601 bool _debug_pagealloc_enabled __read_mostly 602 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 603 EXPORT_SYMBOL(_debug_pagealloc_enabled); 604 bool _debug_guardpage_enabled __read_mostly; 605 606 static int __init early_debug_pagealloc(char *buf) 607 { 608 if (!buf) 609 return -EINVAL; 610 return kstrtobool(buf, &_debug_pagealloc_enabled); 611 } 612 early_param("debug_pagealloc", early_debug_pagealloc); 613 614 static bool need_debug_guardpage(void) 615 { 616 /* If we don't use debug_pagealloc, we don't need guard page */ 617 if (!debug_pagealloc_enabled()) 618 return false; 619 620 return true; 621 } 622 623 static void init_debug_guardpage(void) 624 { 625 if (!debug_pagealloc_enabled()) 626 return; 627 628 _debug_guardpage_enabled = true; 629 } 630 631 struct page_ext_operations debug_guardpage_ops = { 632 .need = need_debug_guardpage, 633 .init = init_debug_guardpage, 634 }; 635 636 static int __init debug_guardpage_minorder_setup(char *buf) 637 { 638 unsigned long res; 639 640 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 641 pr_err("Bad debug_guardpage_minorder value\n"); 642 return 0; 643 } 644 _debug_guardpage_minorder = res; 645 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 646 return 0; 647 } 648 __setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup); 649 650 static inline void set_page_guard(struct zone *zone, struct page *page, 651 unsigned int order, int migratetype) 652 { 653 struct page_ext *page_ext; 654 655 if (!debug_guardpage_enabled()) 656 return; 657 658 page_ext = lookup_page_ext(page); 659 if (unlikely(!page_ext)) 660 return; 661 662 __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); 663 664 INIT_LIST_HEAD(&page->lru); 665 set_page_private(page, order); 666 /* Guard pages are not available for any usage */ 667 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 668 } 669 670 static inline void clear_page_guard(struct zone *zone, struct page *page, 671 unsigned int order, int migratetype) 672 { 673 struct page_ext *page_ext; 674 675 if (!debug_guardpage_enabled()) 676 return; 677 678 page_ext = lookup_page_ext(page); 679 if (unlikely(!page_ext)) 680 return; 681 682 __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); 683 684 set_page_private(page, 0); 685 if (!is_migrate_isolate(migratetype)) 686 __mod_zone_freepage_state(zone, (1 << order), migratetype); 687 } 688 #else 689 struct page_ext_operations debug_guardpage_ops = { NULL, }; 690 static inline void set_page_guard(struct zone *zone, struct page *page, 691 unsigned int order, int migratetype) {} 692 static inline void clear_page_guard(struct zone *zone, struct page *page, 693 unsigned int order, int migratetype) {} 694 #endif 695 696 static inline void set_page_order(struct page *page, unsigned int order) 697 { 698 set_page_private(page, order); 699 __SetPageBuddy(page); 700 } 701 702 static inline void rmv_page_order(struct page *page) 703 { 704 __ClearPageBuddy(page); 705 set_page_private(page, 0); 706 } 707 708 /* 709 * This function checks whether a page is free && is the buddy 710 * we can do coalesce a page and its buddy if 711 * (a) the buddy is not in a hole && 712 * (b) the buddy is in the buddy system && 713 * (c) a page and its buddy have the same order && 714 * (d) a page and its buddy are in the same zone. 715 * 716 * For recording whether a page is in the buddy system, we set ->_mapcount 717 * PAGE_BUDDY_MAPCOUNT_VALUE. 718 * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is 719 * serialized by zone->lock. 720 * 721 * For recording page's order, we use page_private(page). 722 */ 723 static inline int page_is_buddy(struct page *page, struct page *buddy, 724 unsigned int order) 725 { 726 if (!pfn_valid_within(page_to_pfn(buddy))) 727 return 0; 728 729 if (page_is_guard(buddy) && page_order(buddy) == order) { 730 if (page_zone_id(page) != page_zone_id(buddy)) 731 return 0; 732 733 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 734 735 return 1; 736 } 737 738 if (PageBuddy(buddy) && page_order(buddy) == order) { 739 /* 740 * zone check is done late to avoid uselessly 741 * calculating zone/node ids for pages that could 742 * never merge. 743 */ 744 if (page_zone_id(page) != page_zone_id(buddy)) 745 return 0; 746 747 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 748 749 return 1; 750 } 751 return 0; 752 } 753 754 /* 755 * Freeing function for a buddy system allocator. 756 * 757 * The concept of a buddy system is to maintain direct-mapped table 758 * (containing bit values) for memory blocks of various "orders". 759 * The bottom level table contains the map for the smallest allocatable 760 * units of memory (here, pages), and each level above it describes 761 * pairs of units from the levels below, hence, "buddies". 762 * At a high level, all that happens here is marking the table entry 763 * at the bottom level available, and propagating the changes upward 764 * as necessary, plus some accounting needed to play nicely with other 765 * parts of the VM system. 766 * At each level, we keep a list of pages, which are heads of continuous 767 * free pages of length of (1 << order) and marked with _mapcount 768 * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page) 769 * field. 770 * So when we are allocating or freeing one, we can derive the state of the 771 * other. That is, if we allocate a small block, and both were 772 * free, the remainder of the region must be split into blocks. 773 * If a block is freed, and its buddy is also free, then this 774 * triggers coalescing into a block of larger size. 775 * 776 * -- nyc 777 */ 778 779 static inline void __free_one_page(struct page *page, 780 unsigned long pfn, 781 struct zone *zone, unsigned int order, 782 int migratetype) 783 { 784 unsigned long page_idx; 785 unsigned long combined_idx; 786 unsigned long uninitialized_var(buddy_idx); 787 struct page *buddy; 788 unsigned int max_order; 789 790 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1); 791 792 VM_BUG_ON(!zone_is_initialized(zone)); 793 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 794 795 VM_BUG_ON(migratetype == -1); 796 if (likely(!is_migrate_isolate(migratetype))) 797 __mod_zone_freepage_state(zone, 1 << order, migratetype); 798 799 page_idx = pfn & ((1 << MAX_ORDER) - 1); 800 801 VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page); 802 VM_BUG_ON_PAGE(bad_range(zone, page), page); 803 804 continue_merging: 805 while (order < max_order - 1) { 806 buddy_idx = __find_buddy_index(page_idx, order); 807 buddy = page + (buddy_idx - page_idx); 808 if (!page_is_buddy(page, buddy, order)) 809 goto done_merging; 810 /* 811 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 812 * merge with it and move up one order. 813 */ 814 if (page_is_guard(buddy)) { 815 clear_page_guard(zone, buddy, order, migratetype); 816 } else { 817 list_del(&buddy->lru); 818 zone->free_area[order].nr_free--; 819 rmv_page_order(buddy); 820 } 821 combined_idx = buddy_idx & page_idx; 822 page = page + (combined_idx - page_idx); 823 page_idx = combined_idx; 824 order++; 825 } 826 if (max_order < MAX_ORDER) { 827 /* If we are here, it means order is >= pageblock_order. 828 * We want to prevent merge between freepages on isolate 829 * pageblock and normal pageblock. Without this, pageblock 830 * isolation could cause incorrect freepage or CMA accounting. 831 * 832 * We don't want to hit this code for the more frequent 833 * low-order merging. 834 */ 835 if (unlikely(has_isolate_pageblock(zone))) { 836 int buddy_mt; 837 838 buddy_idx = __find_buddy_index(page_idx, order); 839 buddy = page + (buddy_idx - page_idx); 840 buddy_mt = get_pageblock_migratetype(buddy); 841 842 if (migratetype != buddy_mt 843 && (is_migrate_isolate(migratetype) || 844 is_migrate_isolate(buddy_mt))) 845 goto done_merging; 846 } 847 max_order++; 848 goto continue_merging; 849 } 850 851 done_merging: 852 set_page_order(page, order); 853 854 /* 855 * If this is not the largest possible page, check if the buddy 856 * of the next-highest order is free. If it is, it's possible 857 * that pages are being freed that will coalesce soon. In case, 858 * that is happening, add the free page to the tail of the list 859 * so it's less likely to be used soon and more likely to be merged 860 * as a higher order page 861 */ 862 if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) { 863 struct page *higher_page, *higher_buddy; 864 combined_idx = buddy_idx & page_idx; 865 higher_page = page + (combined_idx - page_idx); 866 buddy_idx = __find_buddy_index(combined_idx, order + 1); 867 higher_buddy = higher_page + (buddy_idx - combined_idx); 868 if (page_is_buddy(higher_page, higher_buddy, order + 1)) { 869 list_add_tail(&page->lru, 870 &zone->free_area[order].free_list[migratetype]); 871 goto out; 872 } 873 } 874 875 list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); 876 out: 877 zone->free_area[order].nr_free++; 878 } 879 880 /* 881 * A bad page could be due to a number of fields. Instead of multiple branches, 882 * try and check multiple fields with one check. The caller must do a detailed 883 * check if necessary. 884 */ 885 static inline bool page_expected_state(struct page *page, 886 unsigned long check_flags) 887 { 888 if (unlikely(atomic_read(&page->_mapcount) != -1)) 889 return false; 890 891 if (unlikely((unsigned long)page->mapping | 892 page_ref_count(page) | 893 #ifdef CONFIG_MEMCG 894 (unsigned long)page->mem_cgroup | 895 #endif 896 (page->flags & check_flags))) 897 return false; 898 899 return true; 900 } 901 902 static void free_pages_check_bad(struct page *page) 903 { 904 const char *bad_reason; 905 unsigned long bad_flags; 906 907 bad_reason = NULL; 908 bad_flags = 0; 909 910 if (unlikely(atomic_read(&page->_mapcount) != -1)) 911 bad_reason = "nonzero mapcount"; 912 if (unlikely(page->mapping != NULL)) 913 bad_reason = "non-NULL mapping"; 914 if (unlikely(page_ref_count(page) != 0)) 915 bad_reason = "nonzero _refcount"; 916 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) { 917 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 918 bad_flags = PAGE_FLAGS_CHECK_AT_FREE; 919 } 920 #ifdef CONFIG_MEMCG 921 if (unlikely(page->mem_cgroup)) 922 bad_reason = "page still charged to cgroup"; 923 #endif 924 bad_page(page, bad_reason, bad_flags); 925 } 926 927 static inline int free_pages_check(struct page *page) 928 { 929 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 930 return 0; 931 932 /* Something has gone sideways, find it */ 933 free_pages_check_bad(page); 934 return 1; 935 } 936 937 static int free_tail_pages_check(struct page *head_page, struct page *page) 938 { 939 int ret = 1; 940 941 /* 942 * We rely page->lru.next never has bit 0 set, unless the page 943 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 944 */ 945 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 946 947 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 948 ret = 0; 949 goto out; 950 } 951 switch (page - head_page) { 952 case 1: 953 /* the first tail page: ->mapping is compound_mapcount() */ 954 if (unlikely(compound_mapcount(page))) { 955 bad_page(page, "nonzero compound_mapcount", 0); 956 goto out; 957 } 958 break; 959 case 2: 960 /* 961 * the second tail page: ->mapping is 962 * page_deferred_list().next -- ignore value. 963 */ 964 break; 965 default: 966 if (page->mapping != TAIL_MAPPING) { 967 bad_page(page, "corrupted mapping in tail page", 0); 968 goto out; 969 } 970 break; 971 } 972 if (unlikely(!PageTail(page))) { 973 bad_page(page, "PageTail not set", 0); 974 goto out; 975 } 976 if (unlikely(compound_head(page) != head_page)) { 977 bad_page(page, "compound_head not consistent", 0); 978 goto out; 979 } 980 ret = 0; 981 out: 982 page->mapping = NULL; 983 clear_compound_head(page); 984 return ret; 985 } 986 987 static __always_inline bool free_pages_prepare(struct page *page, 988 unsigned int order, bool check_free) 989 { 990 int bad = 0; 991 992 VM_BUG_ON_PAGE(PageTail(page), page); 993 994 trace_mm_page_free(page, order); 995 kmemcheck_free_shadow(page, order); 996 997 /* 998 * Check tail pages before head page information is cleared to 999 * avoid checking PageCompound for order-0 pages. 1000 */ 1001 if (unlikely(order)) { 1002 bool compound = PageCompound(page); 1003 int i; 1004 1005 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1006 1007 for (i = 1; i < (1 << order); i++) { 1008 if (compound) 1009 bad += free_tail_pages_check(page, page + i); 1010 if (unlikely(free_pages_check(page + i))) { 1011 bad++; 1012 continue; 1013 } 1014 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1015 } 1016 } 1017 if (PageAnonHead(page)) 1018 page->mapping = NULL; 1019 if (check_free) 1020 bad += free_pages_check(page); 1021 if (bad) 1022 return false; 1023 1024 page_cpupid_reset_last(page); 1025 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1026 reset_page_owner(page, order); 1027 1028 if (!PageHighMem(page)) { 1029 debug_check_no_locks_freed(page_address(page), 1030 PAGE_SIZE << order); 1031 debug_check_no_obj_freed(page_address(page), 1032 PAGE_SIZE << order); 1033 } 1034 arch_free_page(page, order); 1035 kernel_poison_pages(page, 1 << order, 0); 1036 kernel_map_pages(page, 1 << order, 0); 1037 kasan_free_pages(page, order); 1038 1039 return true; 1040 } 1041 1042 #ifdef CONFIG_DEBUG_VM 1043 static inline bool free_pcp_prepare(struct page *page) 1044 { 1045 return free_pages_prepare(page, 0, true); 1046 } 1047 1048 static inline bool bulkfree_pcp_prepare(struct page *page) 1049 { 1050 return false; 1051 } 1052 #else 1053 static bool free_pcp_prepare(struct page *page) 1054 { 1055 return free_pages_prepare(page, 0, false); 1056 } 1057 1058 static bool bulkfree_pcp_prepare(struct page *page) 1059 { 1060 return free_pages_check(page); 1061 } 1062 #endif /* CONFIG_DEBUG_VM */ 1063 1064 /* 1065 * Frees a number of pages from the PCP lists 1066 * Assumes all pages on list are in same zone, and of same order. 1067 * count is the number of pages to free. 1068 * 1069 * If the zone was previously in an "all pages pinned" state then look to 1070 * see if this freeing clears that state. 1071 * 1072 * And clear the zone's pages_scanned counter, to hold off the "all pages are 1073 * pinned" detection logic. 1074 */ 1075 static void free_pcppages_bulk(struct zone *zone, int count, 1076 struct per_cpu_pages *pcp) 1077 { 1078 int migratetype = 0; 1079 int batch_free = 0; 1080 unsigned long nr_scanned; 1081 bool isolated_pageblocks; 1082 1083 spin_lock(&zone->lock); 1084 isolated_pageblocks = has_isolate_pageblock(zone); 1085 nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); 1086 if (nr_scanned) 1087 __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); 1088 1089 while (count) { 1090 struct page *page; 1091 struct list_head *list; 1092 1093 /* 1094 * Remove pages from lists in a round-robin fashion. A 1095 * batch_free count is maintained that is incremented when an 1096 * empty list is encountered. This is so more pages are freed 1097 * off fuller lists instead of spinning excessively around empty 1098 * lists 1099 */ 1100 do { 1101 batch_free++; 1102 if (++migratetype == MIGRATE_PCPTYPES) 1103 migratetype = 0; 1104 list = &pcp->lists[migratetype]; 1105 } while (list_empty(list)); 1106 1107 /* This is the only non-empty list. Free them all. */ 1108 if (batch_free == MIGRATE_PCPTYPES) 1109 batch_free = count; 1110 1111 do { 1112 int mt; /* migratetype of the to-be-freed page */ 1113 1114 page = list_last_entry(list, struct page, lru); 1115 /* must delete as __free_one_page list manipulates */ 1116 list_del(&page->lru); 1117 1118 mt = get_pcppage_migratetype(page); 1119 /* MIGRATE_ISOLATE page should not go to pcplists */ 1120 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1121 /* Pageblock could have been isolated meanwhile */ 1122 if (unlikely(isolated_pageblocks)) 1123 mt = get_pageblock_migratetype(page); 1124 1125 if (bulkfree_pcp_prepare(page)) 1126 continue; 1127 1128 __free_one_page(page, page_to_pfn(page), zone, 0, mt); 1129 trace_mm_page_pcpu_drain(page, 0, mt); 1130 } while (--count && --batch_free && !list_empty(list)); 1131 } 1132 spin_unlock(&zone->lock); 1133 } 1134 1135 static void free_one_page(struct zone *zone, 1136 struct page *page, unsigned long pfn, 1137 unsigned int order, 1138 int migratetype) 1139 { 1140 unsigned long nr_scanned; 1141 spin_lock(&zone->lock); 1142 nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); 1143 if (nr_scanned) 1144 __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); 1145 1146 if (unlikely(has_isolate_pageblock(zone) || 1147 is_migrate_isolate(migratetype))) { 1148 migratetype = get_pfnblock_migratetype(page, pfn); 1149 } 1150 __free_one_page(page, pfn, zone, order, migratetype); 1151 spin_unlock(&zone->lock); 1152 } 1153 1154 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1155 unsigned long zone, int nid) 1156 { 1157 set_page_links(page, zone, nid, pfn); 1158 init_page_count(page); 1159 page_mapcount_reset(page); 1160 page_cpupid_reset_last(page); 1161 1162 INIT_LIST_HEAD(&page->lru); 1163 #ifdef WANT_PAGE_VIRTUAL 1164 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1165 if (!is_highmem_idx(zone)) 1166 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1167 #endif 1168 } 1169 1170 static void __meminit __init_single_pfn(unsigned long pfn, unsigned long zone, 1171 int nid) 1172 { 1173 return __init_single_page(pfn_to_page(pfn), pfn, zone, nid); 1174 } 1175 1176 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1177 static void init_reserved_page(unsigned long pfn) 1178 { 1179 pg_data_t *pgdat; 1180 int nid, zid; 1181 1182 if (!early_page_uninitialised(pfn)) 1183 return; 1184 1185 nid = early_pfn_to_nid(pfn); 1186 pgdat = NODE_DATA(nid); 1187 1188 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1189 struct zone *zone = &pgdat->node_zones[zid]; 1190 1191 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone)) 1192 break; 1193 } 1194 __init_single_pfn(pfn, zid, nid); 1195 } 1196 #else 1197 static inline void init_reserved_page(unsigned long pfn) 1198 { 1199 } 1200 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1201 1202 /* 1203 * Initialised pages do not have PageReserved set. This function is 1204 * called for each range allocated by the bootmem allocator and 1205 * marks the pages PageReserved. The remaining valid pages are later 1206 * sent to the buddy page allocator. 1207 */ 1208 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1209 { 1210 unsigned long start_pfn = PFN_DOWN(start); 1211 unsigned long end_pfn = PFN_UP(end); 1212 1213 for (; start_pfn < end_pfn; start_pfn++) { 1214 if (pfn_valid(start_pfn)) { 1215 struct page *page = pfn_to_page(start_pfn); 1216 1217 init_reserved_page(start_pfn); 1218 1219 /* Avoid false-positive PageTail() */ 1220 INIT_LIST_HEAD(&page->lru); 1221 1222 SetPageReserved(page); 1223 } 1224 } 1225 } 1226 1227 static void __free_pages_ok(struct page *page, unsigned int order) 1228 { 1229 unsigned long flags; 1230 int migratetype; 1231 unsigned long pfn = page_to_pfn(page); 1232 1233 if (!free_pages_prepare(page, order, true)) 1234 return; 1235 1236 migratetype = get_pfnblock_migratetype(page, pfn); 1237 local_irq_save(flags); 1238 __count_vm_events(PGFREE, 1 << order); 1239 free_one_page(page_zone(page), page, pfn, order, migratetype); 1240 local_irq_restore(flags); 1241 } 1242 1243 static void __init __free_pages_boot_core(struct page *page, unsigned int order) 1244 { 1245 unsigned int nr_pages = 1 << order; 1246 struct page *p = page; 1247 unsigned int loop; 1248 1249 prefetchw(p); 1250 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1251 prefetchw(p + 1); 1252 __ClearPageReserved(p); 1253 set_page_count(p, 0); 1254 } 1255 __ClearPageReserved(p); 1256 set_page_count(p, 0); 1257 1258 page_zone(page)->managed_pages += nr_pages; 1259 set_page_refcounted(page); 1260 __free_pages(page, order); 1261 } 1262 1263 #if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \ 1264 defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) 1265 1266 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1267 1268 int __meminit early_pfn_to_nid(unsigned long pfn) 1269 { 1270 static DEFINE_SPINLOCK(early_pfn_lock); 1271 int nid; 1272 1273 spin_lock(&early_pfn_lock); 1274 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1275 if (nid < 0) 1276 nid = 0; 1277 spin_unlock(&early_pfn_lock); 1278 1279 return nid; 1280 } 1281 #endif 1282 1283 #ifdef CONFIG_NODES_SPAN_OTHER_NODES 1284 static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node, 1285 struct mminit_pfnnid_cache *state) 1286 { 1287 int nid; 1288 1289 nid = __early_pfn_to_nid(pfn, state); 1290 if (nid >= 0 && nid != node) 1291 return false; 1292 return true; 1293 } 1294 1295 /* Only safe to use early in boot when initialisation is single-threaded */ 1296 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1297 { 1298 return meminit_pfn_in_nid(pfn, node, &early_pfnnid_cache); 1299 } 1300 1301 #else 1302 1303 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node) 1304 { 1305 return true; 1306 } 1307 static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node, 1308 struct mminit_pfnnid_cache *state) 1309 { 1310 return true; 1311 } 1312 #endif 1313 1314 1315 void __init __free_pages_bootmem(struct page *page, unsigned long pfn, 1316 unsigned int order) 1317 { 1318 if (early_page_uninitialised(pfn)) 1319 return; 1320 return __free_pages_boot_core(page, order); 1321 } 1322 1323 /* 1324 * Check that the whole (or subset of) a pageblock given by the interval of 1325 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1326 * with the migration of free compaction scanner. The scanners then need to 1327 * use only pfn_valid_within() check for arches that allow holes within 1328 * pageblocks. 1329 * 1330 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1331 * 1332 * It's possible on some configurations to have a setup like node0 node1 node0 1333 * i.e. it's possible that all pages within a zones range of pages do not 1334 * belong to a single zone. We assume that a border between node0 and node1 1335 * can occur within a single pageblock, but not a node0 node1 node0 1336 * interleaving within a single pageblock. It is therefore sufficient to check 1337 * the first and last page of a pageblock and avoid checking each individual 1338 * page in a pageblock. 1339 */ 1340 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1341 unsigned long end_pfn, struct zone *zone) 1342 { 1343 struct page *start_page; 1344 struct page *end_page; 1345 1346 /* end_pfn is one past the range we are checking */ 1347 end_pfn--; 1348 1349 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1350 return NULL; 1351 1352 start_page = pfn_to_page(start_pfn); 1353 1354 if (page_zone(start_page) != zone) 1355 return NULL; 1356 1357 end_page = pfn_to_page(end_pfn); 1358 1359 /* This gives a shorter code than deriving page_zone(end_page) */ 1360 if (page_zone_id(start_page) != page_zone_id(end_page)) 1361 return NULL; 1362 1363 return start_page; 1364 } 1365 1366 void set_zone_contiguous(struct zone *zone) 1367 { 1368 unsigned long block_start_pfn = zone->zone_start_pfn; 1369 unsigned long block_end_pfn; 1370 1371 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1372 for (; block_start_pfn < zone_end_pfn(zone); 1373 block_start_pfn = block_end_pfn, 1374 block_end_pfn += pageblock_nr_pages) { 1375 1376 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1377 1378 if (!__pageblock_pfn_to_page(block_start_pfn, 1379 block_end_pfn, zone)) 1380 return; 1381 } 1382 1383 /* We confirm that there is no hole */ 1384 zone->contiguous = true; 1385 } 1386 1387 void clear_zone_contiguous(struct zone *zone) 1388 { 1389 zone->contiguous = false; 1390 } 1391 1392 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1393 static void __init deferred_free_range(struct page *page, 1394 unsigned long pfn, int nr_pages) 1395 { 1396 int i; 1397 1398 if (!page) 1399 return; 1400 1401 /* Free a large naturally-aligned chunk if possible */ 1402 if (nr_pages == MAX_ORDER_NR_PAGES && 1403 (pfn & (MAX_ORDER_NR_PAGES-1)) == 0) { 1404 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1405 __free_pages_boot_core(page, MAX_ORDER-1); 1406 return; 1407 } 1408 1409 for (i = 0; i < nr_pages; i++, page++) 1410 __free_pages_boot_core(page, 0); 1411 } 1412 1413 /* Completion tracking for deferred_init_memmap() threads */ 1414 static atomic_t pgdat_init_n_undone __initdata; 1415 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1416 1417 static inline void __init pgdat_init_report_one_done(void) 1418 { 1419 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1420 complete(&pgdat_init_all_done_comp); 1421 } 1422 1423 /* Initialise remaining memory on a node */ 1424 static int __init deferred_init_memmap(void *data) 1425 { 1426 pg_data_t *pgdat = data; 1427 int nid = pgdat->node_id; 1428 struct mminit_pfnnid_cache nid_init_state = { }; 1429 unsigned long start = jiffies; 1430 unsigned long nr_pages = 0; 1431 unsigned long walk_start, walk_end; 1432 int i, zid; 1433 struct zone *zone; 1434 unsigned long first_init_pfn = pgdat->first_deferred_pfn; 1435 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 1436 1437 if (first_init_pfn == ULONG_MAX) { 1438 pgdat_init_report_one_done(); 1439 return 0; 1440 } 1441 1442 /* Bind memory initialisation thread to a local node if possible */ 1443 if (!cpumask_empty(cpumask)) 1444 set_cpus_allowed_ptr(current, cpumask); 1445 1446 /* Sanity check boundaries */ 1447 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 1448 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 1449 pgdat->first_deferred_pfn = ULONG_MAX; 1450 1451 /* Only the highest zone is deferred so find it */ 1452 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1453 zone = pgdat->node_zones + zid; 1454 if (first_init_pfn < zone_end_pfn(zone)) 1455 break; 1456 } 1457 1458 for_each_mem_pfn_range(i, nid, &walk_start, &walk_end, NULL) { 1459 unsigned long pfn, end_pfn; 1460 struct page *page = NULL; 1461 struct page *free_base_page = NULL; 1462 unsigned long free_base_pfn = 0; 1463 int nr_to_free = 0; 1464 1465 end_pfn = min(walk_end, zone_end_pfn(zone)); 1466 pfn = first_init_pfn; 1467 if (pfn < walk_start) 1468 pfn = walk_start; 1469 if (pfn < zone->zone_start_pfn) 1470 pfn = zone->zone_start_pfn; 1471 1472 for (; pfn < end_pfn; pfn++) { 1473 if (!pfn_valid_within(pfn)) 1474 goto free_range; 1475 1476 /* 1477 * Ensure pfn_valid is checked every 1478 * MAX_ORDER_NR_PAGES for memory holes 1479 */ 1480 if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0) { 1481 if (!pfn_valid(pfn)) { 1482 page = NULL; 1483 goto free_range; 1484 } 1485 } 1486 1487 if (!meminit_pfn_in_nid(pfn, nid, &nid_init_state)) { 1488 page = NULL; 1489 goto free_range; 1490 } 1491 1492 /* Minimise pfn page lookups and scheduler checks */ 1493 if (page && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) { 1494 page++; 1495 } else { 1496 nr_pages += nr_to_free; 1497 deferred_free_range(free_base_page, 1498 free_base_pfn, nr_to_free); 1499 free_base_page = NULL; 1500 free_base_pfn = nr_to_free = 0; 1501 1502 page = pfn_to_page(pfn); 1503 cond_resched(); 1504 } 1505 1506 if (page->flags) { 1507 VM_BUG_ON(page_zone(page) != zone); 1508 goto free_range; 1509 } 1510 1511 __init_single_page(page, pfn, zid, nid); 1512 if (!free_base_page) { 1513 free_base_page = page; 1514 free_base_pfn = pfn; 1515 nr_to_free = 0; 1516 } 1517 nr_to_free++; 1518 1519 /* Where possible, batch up pages for a single free */ 1520 continue; 1521 free_range: 1522 /* Free the current block of pages to allocator */ 1523 nr_pages += nr_to_free; 1524 deferred_free_range(free_base_page, free_base_pfn, 1525 nr_to_free); 1526 free_base_page = NULL; 1527 free_base_pfn = nr_to_free = 0; 1528 } 1529 1530 first_init_pfn = max(end_pfn, first_init_pfn); 1531 } 1532 1533 /* Sanity check that the next zone really is unpopulated */ 1534 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 1535 1536 pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages, 1537 jiffies_to_msecs(jiffies - start)); 1538 1539 pgdat_init_report_one_done(); 1540 return 0; 1541 } 1542 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1543 1544 void __init page_alloc_init_late(void) 1545 { 1546 struct zone *zone; 1547 1548 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1549 int nid; 1550 1551 /* There will be num_node_state(N_MEMORY) threads */ 1552 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 1553 for_each_node_state(nid, N_MEMORY) { 1554 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 1555 } 1556 1557 /* Block until all are initialised */ 1558 wait_for_completion(&pgdat_init_all_done_comp); 1559 1560 /* Reinit limits that are based on free pages after the kernel is up */ 1561 files_maxfiles_init(); 1562 #endif 1563 1564 for_each_populated_zone(zone) 1565 set_zone_contiguous(zone); 1566 } 1567 1568 #ifdef CONFIG_CMA 1569 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 1570 void __init init_cma_reserved_pageblock(struct page *page) 1571 { 1572 unsigned i = pageblock_nr_pages; 1573 struct page *p = page; 1574 1575 do { 1576 __ClearPageReserved(p); 1577 set_page_count(p, 0); 1578 } while (++p, --i); 1579 1580 set_pageblock_migratetype(page, MIGRATE_CMA); 1581 1582 if (pageblock_order >= MAX_ORDER) { 1583 i = pageblock_nr_pages; 1584 p = page; 1585 do { 1586 set_page_refcounted(p); 1587 __free_pages(p, MAX_ORDER - 1); 1588 p += MAX_ORDER_NR_PAGES; 1589 } while (i -= MAX_ORDER_NR_PAGES); 1590 } else { 1591 set_page_refcounted(page); 1592 __free_pages(page, pageblock_order); 1593 } 1594 1595 adjust_managed_page_count(page, pageblock_nr_pages); 1596 } 1597 #endif 1598 1599 /* 1600 * The order of subdivision here is critical for the IO subsystem. 1601 * Please do not alter this order without good reasons and regression 1602 * testing. Specifically, as large blocks of memory are subdivided, 1603 * the order in which smaller blocks are delivered depends on the order 1604 * they're subdivided in this function. This is the primary factor 1605 * influencing the order in which pages are delivered to the IO 1606 * subsystem according to empirical testing, and this is also justified 1607 * by considering the behavior of a buddy system containing a single 1608 * large block of memory acted on by a series of small allocations. 1609 * This behavior is a critical factor in sglist merging's success. 1610 * 1611 * -- nyc 1612 */ 1613 static inline void expand(struct zone *zone, struct page *page, 1614 int low, int high, struct free_area *area, 1615 int migratetype) 1616 { 1617 unsigned long size = 1 << high; 1618 1619 while (high > low) { 1620 area--; 1621 high--; 1622 size >>= 1; 1623 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 1624 1625 if (IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) && 1626 debug_guardpage_enabled() && 1627 high < debug_guardpage_minorder()) { 1628 /* 1629 * Mark as guard pages (or page), that will allow to 1630 * merge back to allocator when buddy will be freed. 1631 * Corresponding page table entries will not be touched, 1632 * pages will stay not present in virtual address space 1633 */ 1634 set_page_guard(zone, &page[size], high, migratetype); 1635 continue; 1636 } 1637 list_add(&page[size].lru, &area->free_list[migratetype]); 1638 area->nr_free++; 1639 set_page_order(&page[size], high); 1640 } 1641 } 1642 1643 static void check_new_page_bad(struct page *page) 1644 { 1645 const char *bad_reason = NULL; 1646 unsigned long bad_flags = 0; 1647 1648 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1649 bad_reason = "nonzero mapcount"; 1650 if (unlikely(page->mapping != NULL)) 1651 bad_reason = "non-NULL mapping"; 1652 if (unlikely(page_ref_count(page) != 0)) 1653 bad_reason = "nonzero _count"; 1654 if (unlikely(page->flags & __PG_HWPOISON)) { 1655 bad_reason = "HWPoisoned (hardware-corrupted)"; 1656 bad_flags = __PG_HWPOISON; 1657 /* Don't complain about hwpoisoned pages */ 1658 page_mapcount_reset(page); /* remove PageBuddy */ 1659 return; 1660 } 1661 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) { 1662 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set"; 1663 bad_flags = PAGE_FLAGS_CHECK_AT_PREP; 1664 } 1665 #ifdef CONFIG_MEMCG 1666 if (unlikely(page->mem_cgroup)) 1667 bad_reason = "page still charged to cgroup"; 1668 #endif 1669 bad_page(page, bad_reason, bad_flags); 1670 } 1671 1672 /* 1673 * This page is about to be returned from the page allocator 1674 */ 1675 static inline int check_new_page(struct page *page) 1676 { 1677 if (likely(page_expected_state(page, 1678 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 1679 return 0; 1680 1681 check_new_page_bad(page); 1682 return 1; 1683 } 1684 1685 static inline bool free_pages_prezeroed(bool poisoned) 1686 { 1687 return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) && 1688 page_poisoning_enabled() && poisoned; 1689 } 1690 1691 #ifdef CONFIG_DEBUG_VM 1692 static bool check_pcp_refill(struct page *page) 1693 { 1694 return false; 1695 } 1696 1697 static bool check_new_pcp(struct page *page) 1698 { 1699 return check_new_page(page); 1700 } 1701 #else 1702 static bool check_pcp_refill(struct page *page) 1703 { 1704 return check_new_page(page); 1705 } 1706 static bool check_new_pcp(struct page *page) 1707 { 1708 return false; 1709 } 1710 #endif /* CONFIG_DEBUG_VM */ 1711 1712 static bool check_new_pages(struct page *page, unsigned int order) 1713 { 1714 int i; 1715 for (i = 0; i < (1 << order); i++) { 1716 struct page *p = page + i; 1717 1718 if (unlikely(check_new_page(p))) 1719 return true; 1720 } 1721 1722 return false; 1723 } 1724 1725 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 1726 unsigned int alloc_flags) 1727 { 1728 int i; 1729 bool poisoned = true; 1730 1731 for (i = 0; i < (1 << order); i++) { 1732 struct page *p = page + i; 1733 if (poisoned) 1734 poisoned &= page_is_poisoned(p); 1735 } 1736 1737 set_page_private(page, 0); 1738 set_page_refcounted(page); 1739 1740 arch_alloc_page(page, order); 1741 kernel_map_pages(page, 1 << order, 1); 1742 kernel_poison_pages(page, 1 << order, 1); 1743 kasan_alloc_pages(page, order); 1744 1745 if (!free_pages_prezeroed(poisoned) && (gfp_flags & __GFP_ZERO)) 1746 for (i = 0; i < (1 << order); i++) 1747 clear_highpage(page + i); 1748 1749 if (order && (gfp_flags & __GFP_COMP)) 1750 prep_compound_page(page, order); 1751 1752 set_page_owner(page, order, gfp_flags); 1753 1754 /* 1755 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 1756 * allocate the page. The expectation is that the caller is taking 1757 * steps that will free more memory. The caller should avoid the page 1758 * being used for !PFMEMALLOC purposes. 1759 */ 1760 if (alloc_flags & ALLOC_NO_WATERMARKS) 1761 set_page_pfmemalloc(page); 1762 else 1763 clear_page_pfmemalloc(page); 1764 } 1765 1766 /* 1767 * Go through the free lists for the given migratetype and remove 1768 * the smallest available page from the freelists 1769 */ 1770 static inline 1771 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 1772 int migratetype) 1773 { 1774 unsigned int current_order; 1775 struct free_area *area; 1776 struct page *page; 1777 1778 /* Find a page of the appropriate size in the preferred list */ 1779 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 1780 area = &(zone->free_area[current_order]); 1781 page = list_first_entry_or_null(&area->free_list[migratetype], 1782 struct page, lru); 1783 if (!page) 1784 continue; 1785 list_del(&page->lru); 1786 rmv_page_order(page); 1787 area->nr_free--; 1788 expand(zone, page, order, current_order, area, migratetype); 1789 set_pcppage_migratetype(page, migratetype); 1790 return page; 1791 } 1792 1793 return NULL; 1794 } 1795 1796 1797 /* 1798 * This array describes the order lists are fallen back to when 1799 * the free lists for the desirable migrate type are depleted 1800 */ 1801 static int fallbacks[MIGRATE_TYPES][4] = { 1802 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 1803 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 1804 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 1805 #ifdef CONFIG_CMA 1806 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */ 1807 #endif 1808 #ifdef CONFIG_MEMORY_ISOLATION 1809 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */ 1810 #endif 1811 }; 1812 1813 #ifdef CONFIG_CMA 1814 static struct page *__rmqueue_cma_fallback(struct zone *zone, 1815 unsigned int order) 1816 { 1817 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 1818 } 1819 #else 1820 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 1821 unsigned int order) { return NULL; } 1822 #endif 1823 1824 /* 1825 * Move the free pages in a range to the free lists of the requested type. 1826 * Note that start_page and end_pages are not aligned on a pageblock 1827 * boundary. If alignment is required, use move_freepages_block() 1828 */ 1829 int move_freepages(struct zone *zone, 1830 struct page *start_page, struct page *end_page, 1831 int migratetype) 1832 { 1833 struct page *page; 1834 unsigned int order; 1835 int pages_moved = 0; 1836 1837 #ifndef CONFIG_HOLES_IN_ZONE 1838 /* 1839 * page_zone is not safe to call in this context when 1840 * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant 1841 * anyway as we check zone boundaries in move_freepages_block(). 1842 * Remove at a later date when no bug reports exist related to 1843 * grouping pages by mobility 1844 */ 1845 VM_BUG_ON(page_zone(start_page) != page_zone(end_page)); 1846 #endif 1847 1848 for (page = start_page; page <= end_page;) { 1849 /* Make sure we are not inadvertently changing nodes */ 1850 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 1851 1852 if (!pfn_valid_within(page_to_pfn(page))) { 1853 page++; 1854 continue; 1855 } 1856 1857 if (!PageBuddy(page)) { 1858 page++; 1859 continue; 1860 } 1861 1862 order = page_order(page); 1863 list_move(&page->lru, 1864 &zone->free_area[order].free_list[migratetype]); 1865 page += 1 << order; 1866 pages_moved += 1 << order; 1867 } 1868 1869 return pages_moved; 1870 } 1871 1872 int move_freepages_block(struct zone *zone, struct page *page, 1873 int migratetype) 1874 { 1875 unsigned long start_pfn, end_pfn; 1876 struct page *start_page, *end_page; 1877 1878 start_pfn = page_to_pfn(page); 1879 start_pfn = start_pfn & ~(pageblock_nr_pages-1); 1880 start_page = pfn_to_page(start_pfn); 1881 end_page = start_page + pageblock_nr_pages - 1; 1882 end_pfn = start_pfn + pageblock_nr_pages - 1; 1883 1884 /* Do not cross zone boundaries */ 1885 if (!zone_spans_pfn(zone, start_pfn)) 1886 start_page = page; 1887 if (!zone_spans_pfn(zone, end_pfn)) 1888 return 0; 1889 1890 return move_freepages(zone, start_page, end_page, migratetype); 1891 } 1892 1893 static void change_pageblock_range(struct page *pageblock_page, 1894 int start_order, int migratetype) 1895 { 1896 int nr_pageblocks = 1 << (start_order - pageblock_order); 1897 1898 while (nr_pageblocks--) { 1899 set_pageblock_migratetype(pageblock_page, migratetype); 1900 pageblock_page += pageblock_nr_pages; 1901 } 1902 } 1903 1904 /* 1905 * When we are falling back to another migratetype during allocation, try to 1906 * steal extra free pages from the same pageblocks to satisfy further 1907 * allocations, instead of polluting multiple pageblocks. 1908 * 1909 * If we are stealing a relatively large buddy page, it is likely there will 1910 * be more free pages in the pageblock, so try to steal them all. For 1911 * reclaimable and unmovable allocations, we steal regardless of page size, 1912 * as fragmentation caused by those allocations polluting movable pageblocks 1913 * is worse than movable allocations stealing from unmovable and reclaimable 1914 * pageblocks. 1915 */ 1916 static bool can_steal_fallback(unsigned int order, int start_mt) 1917 { 1918 /* 1919 * Leaving this order check is intended, although there is 1920 * relaxed order check in next check. The reason is that 1921 * we can actually steal whole pageblock if this condition met, 1922 * but, below check doesn't guarantee it and that is just heuristic 1923 * so could be changed anytime. 1924 */ 1925 if (order >= pageblock_order) 1926 return true; 1927 1928 if (order >= pageblock_order / 2 || 1929 start_mt == MIGRATE_RECLAIMABLE || 1930 start_mt == MIGRATE_UNMOVABLE || 1931 page_group_by_mobility_disabled) 1932 return true; 1933 1934 return false; 1935 } 1936 1937 /* 1938 * This function implements actual steal behaviour. If order is large enough, 1939 * we can steal whole pageblock. If not, we first move freepages in this 1940 * pageblock and check whether half of pages are moved or not. If half of 1941 * pages are moved, we can change migratetype of pageblock and permanently 1942 * use it's pages as requested migratetype in the future. 1943 */ 1944 static void steal_suitable_fallback(struct zone *zone, struct page *page, 1945 int start_type) 1946 { 1947 unsigned int current_order = page_order(page); 1948 int pages; 1949 1950 /* Take ownership for orders >= pageblock_order */ 1951 if (current_order >= pageblock_order) { 1952 change_pageblock_range(page, current_order, start_type); 1953 return; 1954 } 1955 1956 pages = move_freepages_block(zone, page, start_type); 1957 1958 /* Claim the whole block if over half of it is free */ 1959 if (pages >= (1 << (pageblock_order-1)) || 1960 page_group_by_mobility_disabled) 1961 set_pageblock_migratetype(page, start_type); 1962 } 1963 1964 /* 1965 * Check whether there is a suitable fallback freepage with requested order. 1966 * If only_stealable is true, this function returns fallback_mt only if 1967 * we can steal other freepages all together. This would help to reduce 1968 * fragmentation due to mixed migratetype pages in one pageblock. 1969 */ 1970 int find_suitable_fallback(struct free_area *area, unsigned int order, 1971 int migratetype, bool only_stealable, bool *can_steal) 1972 { 1973 int i; 1974 int fallback_mt; 1975 1976 if (area->nr_free == 0) 1977 return -1; 1978 1979 *can_steal = false; 1980 for (i = 0;; i++) { 1981 fallback_mt = fallbacks[migratetype][i]; 1982 if (fallback_mt == MIGRATE_TYPES) 1983 break; 1984 1985 if (list_empty(&area->free_list[fallback_mt])) 1986 continue; 1987 1988 if (can_steal_fallback(order, migratetype)) 1989 *can_steal = true; 1990 1991 if (!only_stealable) 1992 return fallback_mt; 1993 1994 if (*can_steal) 1995 return fallback_mt; 1996 } 1997 1998 return -1; 1999 } 2000 2001 /* 2002 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2003 * there are no empty page blocks that contain a page with a suitable order 2004 */ 2005 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2006 unsigned int alloc_order) 2007 { 2008 int mt; 2009 unsigned long max_managed, flags; 2010 2011 /* 2012 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2013 * Check is race-prone but harmless. 2014 */ 2015 max_managed = (zone->managed_pages / 100) + pageblock_nr_pages; 2016 if (zone->nr_reserved_highatomic >= max_managed) 2017 return; 2018 2019 spin_lock_irqsave(&zone->lock, flags); 2020 2021 /* Recheck the nr_reserved_highatomic limit under the lock */ 2022 if (zone->nr_reserved_highatomic >= max_managed) 2023 goto out_unlock; 2024 2025 /* Yoink! */ 2026 mt = get_pageblock_migratetype(page); 2027 if (mt != MIGRATE_HIGHATOMIC && 2028 !is_migrate_isolate(mt) && !is_migrate_cma(mt)) { 2029 zone->nr_reserved_highatomic += pageblock_nr_pages; 2030 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2031 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC); 2032 } 2033 2034 out_unlock: 2035 spin_unlock_irqrestore(&zone->lock, flags); 2036 } 2037 2038 /* 2039 * Used when an allocation is about to fail under memory pressure. This 2040 * potentially hurts the reliability of high-order allocations when under 2041 * intense memory pressure but failed atomic allocations should be easier 2042 * to recover from than an OOM. 2043 */ 2044 static void unreserve_highatomic_pageblock(const struct alloc_context *ac) 2045 { 2046 struct zonelist *zonelist = ac->zonelist; 2047 unsigned long flags; 2048 struct zoneref *z; 2049 struct zone *zone; 2050 struct page *page; 2051 int order; 2052 2053 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx, 2054 ac->nodemask) { 2055 /* Preserve at least one pageblock */ 2056 if (zone->nr_reserved_highatomic <= pageblock_nr_pages) 2057 continue; 2058 2059 spin_lock_irqsave(&zone->lock, flags); 2060 for (order = 0; order < MAX_ORDER; order++) { 2061 struct free_area *area = &(zone->free_area[order]); 2062 2063 page = list_first_entry_or_null( 2064 &area->free_list[MIGRATE_HIGHATOMIC], 2065 struct page, lru); 2066 if (!page) 2067 continue; 2068 2069 /* 2070 * It should never happen but changes to locking could 2071 * inadvertently allow a per-cpu drain to add pages 2072 * to MIGRATE_HIGHATOMIC while unreserving so be safe 2073 * and watch for underflows. 2074 */ 2075 zone->nr_reserved_highatomic -= min(pageblock_nr_pages, 2076 zone->nr_reserved_highatomic); 2077 2078 /* 2079 * Convert to ac->migratetype and avoid the normal 2080 * pageblock stealing heuristics. Minimally, the caller 2081 * is doing the work and needs the pages. More 2082 * importantly, if the block was always converted to 2083 * MIGRATE_UNMOVABLE or another type then the number 2084 * of pageblocks that cannot be completely freed 2085 * may increase. 2086 */ 2087 set_pageblock_migratetype(page, ac->migratetype); 2088 move_freepages_block(zone, page, ac->migratetype); 2089 spin_unlock_irqrestore(&zone->lock, flags); 2090 return; 2091 } 2092 spin_unlock_irqrestore(&zone->lock, flags); 2093 } 2094 } 2095 2096 /* Remove an element from the buddy allocator from the fallback list */ 2097 static inline struct page * 2098 __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype) 2099 { 2100 struct free_area *area; 2101 unsigned int current_order; 2102 struct page *page; 2103 int fallback_mt; 2104 bool can_steal; 2105 2106 /* Find the largest possible block of pages in the other list */ 2107 for (current_order = MAX_ORDER-1; 2108 current_order >= order && current_order <= MAX_ORDER-1; 2109 --current_order) { 2110 area = &(zone->free_area[current_order]); 2111 fallback_mt = find_suitable_fallback(area, current_order, 2112 start_migratetype, false, &can_steal); 2113 if (fallback_mt == -1) 2114 continue; 2115 2116 page = list_first_entry(&area->free_list[fallback_mt], 2117 struct page, lru); 2118 if (can_steal) 2119 steal_suitable_fallback(zone, page, start_migratetype); 2120 2121 /* Remove the page from the freelists */ 2122 area->nr_free--; 2123 list_del(&page->lru); 2124 rmv_page_order(page); 2125 2126 expand(zone, page, order, current_order, area, 2127 start_migratetype); 2128 /* 2129 * The pcppage_migratetype may differ from pageblock's 2130 * migratetype depending on the decisions in 2131 * find_suitable_fallback(). This is OK as long as it does not 2132 * differ for MIGRATE_CMA pageblocks. Those can be used as 2133 * fallback only via special __rmqueue_cma_fallback() function 2134 */ 2135 set_pcppage_migratetype(page, start_migratetype); 2136 2137 trace_mm_page_alloc_extfrag(page, order, current_order, 2138 start_migratetype, fallback_mt); 2139 2140 return page; 2141 } 2142 2143 return NULL; 2144 } 2145 2146 /* 2147 * Do the hard work of removing an element from the buddy allocator. 2148 * Call me with the zone->lock already held. 2149 */ 2150 static struct page *__rmqueue(struct zone *zone, unsigned int order, 2151 int migratetype) 2152 { 2153 struct page *page; 2154 2155 page = __rmqueue_smallest(zone, order, migratetype); 2156 if (unlikely(!page)) { 2157 if (migratetype == MIGRATE_MOVABLE) 2158 page = __rmqueue_cma_fallback(zone, order); 2159 2160 if (!page) 2161 page = __rmqueue_fallback(zone, order, migratetype); 2162 } 2163 2164 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2165 return page; 2166 } 2167 2168 /* 2169 * Obtain a specified number of elements from the buddy allocator, all under 2170 * a single hold of the lock, for efficiency. Add them to the supplied list. 2171 * Returns the number of new pages which were placed at *list. 2172 */ 2173 static int rmqueue_bulk(struct zone *zone, unsigned int order, 2174 unsigned long count, struct list_head *list, 2175 int migratetype, bool cold) 2176 { 2177 int i; 2178 2179 spin_lock(&zone->lock); 2180 for (i = 0; i < count; ++i) { 2181 struct page *page = __rmqueue(zone, order, migratetype); 2182 if (unlikely(page == NULL)) 2183 break; 2184 2185 if (unlikely(check_pcp_refill(page))) 2186 continue; 2187 2188 /* 2189 * Split buddy pages returned by expand() are received here 2190 * in physical page order. The page is added to the callers and 2191 * list and the list head then moves forward. From the callers 2192 * perspective, the linked list is ordered by page number in 2193 * some conditions. This is useful for IO devices that can 2194 * merge IO requests if the physical pages are ordered 2195 * properly. 2196 */ 2197 if (likely(!cold)) 2198 list_add(&page->lru, list); 2199 else 2200 list_add_tail(&page->lru, list); 2201 list = &page->lru; 2202 if (is_migrate_cma(get_pcppage_migratetype(page))) 2203 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 2204 -(1 << order)); 2205 } 2206 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 2207 spin_unlock(&zone->lock); 2208 return i; 2209 } 2210 2211 #ifdef CONFIG_NUMA 2212 /* 2213 * Called from the vmstat counter updater to drain pagesets of this 2214 * currently executing processor on remote nodes after they have 2215 * expired. 2216 * 2217 * Note that this function must be called with the thread pinned to 2218 * a single processor. 2219 */ 2220 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 2221 { 2222 unsigned long flags; 2223 int to_drain, batch; 2224 2225 local_irq_save(flags); 2226 batch = READ_ONCE(pcp->batch); 2227 to_drain = min(pcp->count, batch); 2228 if (to_drain > 0) { 2229 free_pcppages_bulk(zone, to_drain, pcp); 2230 pcp->count -= to_drain; 2231 } 2232 local_irq_restore(flags); 2233 } 2234 #endif 2235 2236 /* 2237 * Drain pcplists of the indicated processor and zone. 2238 * 2239 * The processor must either be the current processor and the 2240 * thread pinned to the current processor or a processor that 2241 * is not online. 2242 */ 2243 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 2244 { 2245 unsigned long flags; 2246 struct per_cpu_pageset *pset; 2247 struct per_cpu_pages *pcp; 2248 2249 local_irq_save(flags); 2250 pset = per_cpu_ptr(zone->pageset, cpu); 2251 2252 pcp = &pset->pcp; 2253 if (pcp->count) { 2254 free_pcppages_bulk(zone, pcp->count, pcp); 2255 pcp->count = 0; 2256 } 2257 local_irq_restore(flags); 2258 } 2259 2260 /* 2261 * Drain pcplists of all zones on the indicated processor. 2262 * 2263 * The processor must either be the current processor and the 2264 * thread pinned to the current processor or a processor that 2265 * is not online. 2266 */ 2267 static void drain_pages(unsigned int cpu) 2268 { 2269 struct zone *zone; 2270 2271 for_each_populated_zone(zone) { 2272 drain_pages_zone(cpu, zone); 2273 } 2274 } 2275 2276 /* 2277 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 2278 * 2279 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 2280 * the single zone's pages. 2281 */ 2282 void drain_local_pages(struct zone *zone) 2283 { 2284 int cpu = smp_processor_id(); 2285 2286 if (zone) 2287 drain_pages_zone(cpu, zone); 2288 else 2289 drain_pages(cpu); 2290 } 2291 2292 /* 2293 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 2294 * 2295 * When zone parameter is non-NULL, spill just the single zone's pages. 2296 * 2297 * Note that this code is protected against sending an IPI to an offline 2298 * CPU but does not guarantee sending an IPI to newly hotplugged CPUs: 2299 * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but 2300 * nothing keeps CPUs from showing up after we populated the cpumask and 2301 * before the call to on_each_cpu_mask(). 2302 */ 2303 void drain_all_pages(struct zone *zone) 2304 { 2305 int cpu; 2306 2307 /* 2308 * Allocate in the BSS so we wont require allocation in 2309 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 2310 */ 2311 static cpumask_t cpus_with_pcps; 2312 2313 /* 2314 * We don't care about racing with CPU hotplug event 2315 * as offline notification will cause the notified 2316 * cpu to drain that CPU pcps and on_each_cpu_mask 2317 * disables preemption as part of its processing 2318 */ 2319 for_each_online_cpu(cpu) { 2320 struct per_cpu_pageset *pcp; 2321 struct zone *z; 2322 bool has_pcps = false; 2323 2324 if (zone) { 2325 pcp = per_cpu_ptr(zone->pageset, cpu); 2326 if (pcp->pcp.count) 2327 has_pcps = true; 2328 } else { 2329 for_each_populated_zone(z) { 2330 pcp = per_cpu_ptr(z->pageset, cpu); 2331 if (pcp->pcp.count) { 2332 has_pcps = true; 2333 break; 2334 } 2335 } 2336 } 2337 2338 if (has_pcps) 2339 cpumask_set_cpu(cpu, &cpus_with_pcps); 2340 else 2341 cpumask_clear_cpu(cpu, &cpus_with_pcps); 2342 } 2343 on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages, 2344 zone, 1); 2345 } 2346 2347 #ifdef CONFIG_HIBERNATION 2348 2349 void mark_free_pages(struct zone *zone) 2350 { 2351 unsigned long pfn, max_zone_pfn; 2352 unsigned long flags; 2353 unsigned int order, t; 2354 struct page *page; 2355 2356 if (zone_is_empty(zone)) 2357 return; 2358 2359 spin_lock_irqsave(&zone->lock, flags); 2360 2361 max_zone_pfn = zone_end_pfn(zone); 2362 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 2363 if (pfn_valid(pfn)) { 2364 page = pfn_to_page(pfn); 2365 2366 if (page_zone(page) != zone) 2367 continue; 2368 2369 if (!swsusp_page_is_forbidden(page)) 2370 swsusp_unset_page_free(page); 2371 } 2372 2373 for_each_migratetype_order(order, t) { 2374 list_for_each_entry(page, 2375 &zone->free_area[order].free_list[t], lru) { 2376 unsigned long i; 2377 2378 pfn = page_to_pfn(page); 2379 for (i = 0; i < (1UL << order); i++) 2380 swsusp_set_page_free(pfn_to_page(pfn + i)); 2381 } 2382 } 2383 spin_unlock_irqrestore(&zone->lock, flags); 2384 } 2385 #endif /* CONFIG_PM */ 2386 2387 /* 2388 * Free a 0-order page 2389 * cold == true ? free a cold page : free a hot page 2390 */ 2391 void free_hot_cold_page(struct page *page, bool cold) 2392 { 2393 struct zone *zone = page_zone(page); 2394 struct per_cpu_pages *pcp; 2395 unsigned long flags; 2396 unsigned long pfn = page_to_pfn(page); 2397 int migratetype; 2398 2399 if (!free_pcp_prepare(page)) 2400 return; 2401 2402 migratetype = get_pfnblock_migratetype(page, pfn); 2403 set_pcppage_migratetype(page, migratetype); 2404 local_irq_save(flags); 2405 __count_vm_event(PGFREE); 2406 2407 /* 2408 * We only track unmovable, reclaimable and movable on pcp lists. 2409 * Free ISOLATE pages back to the allocator because they are being 2410 * offlined but treat RESERVE as movable pages so we can get those 2411 * areas back if necessary. Otherwise, we may have to free 2412 * excessively into the page allocator 2413 */ 2414 if (migratetype >= MIGRATE_PCPTYPES) { 2415 if (unlikely(is_migrate_isolate(migratetype))) { 2416 free_one_page(zone, page, pfn, 0, migratetype); 2417 goto out; 2418 } 2419 migratetype = MIGRATE_MOVABLE; 2420 } 2421 2422 pcp = &this_cpu_ptr(zone->pageset)->pcp; 2423 if (!cold) 2424 list_add(&page->lru, &pcp->lists[migratetype]); 2425 else 2426 list_add_tail(&page->lru, &pcp->lists[migratetype]); 2427 pcp->count++; 2428 if (pcp->count >= pcp->high) { 2429 unsigned long batch = READ_ONCE(pcp->batch); 2430 free_pcppages_bulk(zone, batch, pcp); 2431 pcp->count -= batch; 2432 } 2433 2434 out: 2435 local_irq_restore(flags); 2436 } 2437 2438 /* 2439 * Free a list of 0-order pages 2440 */ 2441 void free_hot_cold_page_list(struct list_head *list, bool cold) 2442 { 2443 struct page *page, *next; 2444 2445 list_for_each_entry_safe(page, next, list, lru) { 2446 trace_mm_page_free_batched(page, cold); 2447 free_hot_cold_page(page, cold); 2448 } 2449 } 2450 2451 /* 2452 * split_page takes a non-compound higher-order page, and splits it into 2453 * n (1<<order) sub-pages: page[0..n] 2454 * Each sub-page must be freed individually. 2455 * 2456 * Note: this is probably too low level an operation for use in drivers. 2457 * Please consult with lkml before using this in your driver. 2458 */ 2459 void split_page(struct page *page, unsigned int order) 2460 { 2461 int i; 2462 gfp_t gfp_mask; 2463 2464 VM_BUG_ON_PAGE(PageCompound(page), page); 2465 VM_BUG_ON_PAGE(!page_count(page), page); 2466 2467 #ifdef CONFIG_KMEMCHECK 2468 /* 2469 * Split shadow pages too, because free(page[0]) would 2470 * otherwise free the whole shadow. 2471 */ 2472 if (kmemcheck_page_is_tracked(page)) 2473 split_page(virt_to_page(page[0].shadow), order); 2474 #endif 2475 2476 gfp_mask = get_page_owner_gfp(page); 2477 set_page_owner(page, 0, gfp_mask); 2478 for (i = 1; i < (1 << order); i++) { 2479 set_page_refcounted(page + i); 2480 set_page_owner(page + i, 0, gfp_mask); 2481 } 2482 } 2483 EXPORT_SYMBOL_GPL(split_page); 2484 2485 int __isolate_free_page(struct page *page, unsigned int order) 2486 { 2487 unsigned long watermark; 2488 struct zone *zone; 2489 int mt; 2490 2491 BUG_ON(!PageBuddy(page)); 2492 2493 zone = page_zone(page); 2494 mt = get_pageblock_migratetype(page); 2495 2496 if (!is_migrate_isolate(mt)) { 2497 /* Obey watermarks as if the page was being allocated */ 2498 watermark = low_wmark_pages(zone) + (1 << order); 2499 if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) 2500 return 0; 2501 2502 __mod_zone_freepage_state(zone, -(1UL << order), mt); 2503 } 2504 2505 /* Remove page from free list */ 2506 list_del(&page->lru); 2507 zone->free_area[order].nr_free--; 2508 rmv_page_order(page); 2509 2510 set_page_owner(page, order, __GFP_MOVABLE); 2511 2512 /* Set the pageblock if the isolated page is at least a pageblock */ 2513 if (order >= pageblock_order - 1) { 2514 struct page *endpage = page + (1 << order) - 1; 2515 for (; page < endpage; page += pageblock_nr_pages) { 2516 int mt = get_pageblock_migratetype(page); 2517 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)) 2518 set_pageblock_migratetype(page, 2519 MIGRATE_MOVABLE); 2520 } 2521 } 2522 2523 2524 return 1UL << order; 2525 } 2526 2527 /* 2528 * Similar to split_page except the page is already free. As this is only 2529 * being used for migration, the migratetype of the block also changes. 2530 * As this is called with interrupts disabled, the caller is responsible 2531 * for calling arch_alloc_page() and kernel_map_page() after interrupts 2532 * are enabled. 2533 * 2534 * Note: this is probably too low level an operation for use in drivers. 2535 * Please consult with lkml before using this in your driver. 2536 */ 2537 int split_free_page(struct page *page) 2538 { 2539 unsigned int order; 2540 int nr_pages; 2541 2542 order = page_order(page); 2543 2544 nr_pages = __isolate_free_page(page, order); 2545 if (!nr_pages) 2546 return 0; 2547 2548 /* Split into individual pages */ 2549 set_page_refcounted(page); 2550 split_page(page, order); 2551 return nr_pages; 2552 } 2553 2554 /* 2555 * Update NUMA hit/miss statistics 2556 * 2557 * Must be called with interrupts disabled. 2558 * 2559 * When __GFP_OTHER_NODE is set assume the node of the preferred 2560 * zone is the local node. This is useful for daemons who allocate 2561 * memory on behalf of other processes. 2562 */ 2563 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z, 2564 gfp_t flags) 2565 { 2566 #ifdef CONFIG_NUMA 2567 int local_nid = numa_node_id(); 2568 enum zone_stat_item local_stat = NUMA_LOCAL; 2569 2570 if (unlikely(flags & __GFP_OTHER_NODE)) { 2571 local_stat = NUMA_OTHER; 2572 local_nid = preferred_zone->node; 2573 } 2574 2575 if (z->node == local_nid) { 2576 __inc_zone_state(z, NUMA_HIT); 2577 __inc_zone_state(z, local_stat); 2578 } else { 2579 __inc_zone_state(z, NUMA_MISS); 2580 __inc_zone_state(preferred_zone, NUMA_FOREIGN); 2581 } 2582 #endif 2583 } 2584 2585 /* 2586 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 2587 */ 2588 static inline 2589 struct page *buffered_rmqueue(struct zone *preferred_zone, 2590 struct zone *zone, unsigned int order, 2591 gfp_t gfp_flags, unsigned int alloc_flags, 2592 int migratetype) 2593 { 2594 unsigned long flags; 2595 struct page *page; 2596 bool cold = ((gfp_flags & __GFP_COLD) != 0); 2597 2598 if (likely(order == 0)) { 2599 struct per_cpu_pages *pcp; 2600 struct list_head *list; 2601 2602 local_irq_save(flags); 2603 do { 2604 pcp = &this_cpu_ptr(zone->pageset)->pcp; 2605 list = &pcp->lists[migratetype]; 2606 if (list_empty(list)) { 2607 pcp->count += rmqueue_bulk(zone, 0, 2608 pcp->batch, list, 2609 migratetype, cold); 2610 if (unlikely(list_empty(list))) 2611 goto failed; 2612 } 2613 2614 if (cold) 2615 page = list_last_entry(list, struct page, lru); 2616 else 2617 page = list_first_entry(list, struct page, lru); 2618 2619 __dec_zone_state(zone, NR_ALLOC_BATCH); 2620 list_del(&page->lru); 2621 pcp->count--; 2622 2623 } while (check_new_pcp(page)); 2624 } else { 2625 /* 2626 * We most definitely don't want callers attempting to 2627 * allocate greater than order-1 page units with __GFP_NOFAIL. 2628 */ 2629 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 2630 spin_lock_irqsave(&zone->lock, flags); 2631 2632 do { 2633 page = NULL; 2634 if (alloc_flags & ALLOC_HARDER) { 2635 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 2636 if (page) 2637 trace_mm_page_alloc_zone_locked(page, order, migratetype); 2638 } 2639 if (!page) 2640 page = __rmqueue(zone, order, migratetype); 2641 } while (page && check_new_pages(page, order)); 2642 spin_unlock(&zone->lock); 2643 if (!page) 2644 goto failed; 2645 __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); 2646 __mod_zone_freepage_state(zone, -(1 << order), 2647 get_pcppage_migratetype(page)); 2648 } 2649 2650 if (atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]) <= 0 && 2651 !test_bit(ZONE_FAIR_DEPLETED, &zone->flags)) 2652 set_bit(ZONE_FAIR_DEPLETED, &zone->flags); 2653 2654 __count_zone_vm_events(PGALLOC, zone, 1 << order); 2655 zone_statistics(preferred_zone, zone, gfp_flags); 2656 local_irq_restore(flags); 2657 2658 VM_BUG_ON_PAGE(bad_range(zone, page), page); 2659 return page; 2660 2661 failed: 2662 local_irq_restore(flags); 2663 return NULL; 2664 } 2665 2666 #ifdef CONFIG_FAIL_PAGE_ALLOC 2667 2668 static struct { 2669 struct fault_attr attr; 2670 2671 bool ignore_gfp_highmem; 2672 bool ignore_gfp_reclaim; 2673 u32 min_order; 2674 } fail_page_alloc = { 2675 .attr = FAULT_ATTR_INITIALIZER, 2676 .ignore_gfp_reclaim = true, 2677 .ignore_gfp_highmem = true, 2678 .min_order = 1, 2679 }; 2680 2681 static int __init setup_fail_page_alloc(char *str) 2682 { 2683 return setup_fault_attr(&fail_page_alloc.attr, str); 2684 } 2685 __setup("fail_page_alloc=", setup_fail_page_alloc); 2686 2687 static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 2688 { 2689 if (order < fail_page_alloc.min_order) 2690 return false; 2691 if (gfp_mask & __GFP_NOFAIL) 2692 return false; 2693 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 2694 return false; 2695 if (fail_page_alloc.ignore_gfp_reclaim && 2696 (gfp_mask & __GFP_DIRECT_RECLAIM)) 2697 return false; 2698 2699 return should_fail(&fail_page_alloc.attr, 1 << order); 2700 } 2701 2702 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 2703 2704 static int __init fail_page_alloc_debugfs(void) 2705 { 2706 umode_t mode = S_IFREG | S_IRUSR | S_IWUSR; 2707 struct dentry *dir; 2708 2709 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 2710 &fail_page_alloc.attr); 2711 if (IS_ERR(dir)) 2712 return PTR_ERR(dir); 2713 2714 if (!debugfs_create_bool("ignore-gfp-wait", mode, dir, 2715 &fail_page_alloc.ignore_gfp_reclaim)) 2716 goto fail; 2717 if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir, 2718 &fail_page_alloc.ignore_gfp_highmem)) 2719 goto fail; 2720 if (!debugfs_create_u32("min-order", mode, dir, 2721 &fail_page_alloc.min_order)) 2722 goto fail; 2723 2724 return 0; 2725 fail: 2726 debugfs_remove_recursive(dir); 2727 2728 return -ENOMEM; 2729 } 2730 2731 late_initcall(fail_page_alloc_debugfs); 2732 2733 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 2734 2735 #else /* CONFIG_FAIL_PAGE_ALLOC */ 2736 2737 static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 2738 { 2739 return false; 2740 } 2741 2742 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 2743 2744 /* 2745 * Return true if free base pages are above 'mark'. For high-order checks it 2746 * will return true of the order-0 watermark is reached and there is at least 2747 * one free page of a suitable size. Checking now avoids taking the zone lock 2748 * to check in the allocation paths if no pages are free. 2749 */ 2750 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 2751 int classzone_idx, unsigned int alloc_flags, 2752 long free_pages) 2753 { 2754 long min = mark; 2755 int o; 2756 const bool alloc_harder = (alloc_flags & ALLOC_HARDER); 2757 2758 /* free_pages may go negative - that's OK */ 2759 free_pages -= (1 << order) - 1; 2760 2761 if (alloc_flags & ALLOC_HIGH) 2762 min -= min / 2; 2763 2764 /* 2765 * If the caller does not have rights to ALLOC_HARDER then subtract 2766 * the high-atomic reserves. This will over-estimate the size of the 2767 * atomic reserve but it avoids a search. 2768 */ 2769 if (likely(!alloc_harder)) 2770 free_pages -= z->nr_reserved_highatomic; 2771 else 2772 min -= min / 4; 2773 2774 #ifdef CONFIG_CMA 2775 /* If allocation can't use CMA areas don't use free CMA pages */ 2776 if (!(alloc_flags & ALLOC_CMA)) 2777 free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES); 2778 #endif 2779 2780 /* 2781 * Check watermarks for an order-0 allocation request. If these 2782 * are not met, then a high-order request also cannot go ahead 2783 * even if a suitable page happened to be free. 2784 */ 2785 if (free_pages <= min + z->lowmem_reserve[classzone_idx]) 2786 return false; 2787 2788 /* If this is an order-0 request then the watermark is fine */ 2789 if (!order) 2790 return true; 2791 2792 /* For a high-order request, check at least one suitable page is free */ 2793 for (o = order; o < MAX_ORDER; o++) { 2794 struct free_area *area = &z->free_area[o]; 2795 int mt; 2796 2797 if (!area->nr_free) 2798 continue; 2799 2800 if (alloc_harder) 2801 return true; 2802 2803 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 2804 if (!list_empty(&area->free_list[mt])) 2805 return true; 2806 } 2807 2808 #ifdef CONFIG_CMA 2809 if ((alloc_flags & ALLOC_CMA) && 2810 !list_empty(&area->free_list[MIGRATE_CMA])) { 2811 return true; 2812 } 2813 #endif 2814 } 2815 return false; 2816 } 2817 2818 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 2819 int classzone_idx, unsigned int alloc_flags) 2820 { 2821 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 2822 zone_page_state(z, NR_FREE_PAGES)); 2823 } 2824 2825 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 2826 unsigned long mark, int classzone_idx, unsigned int alloc_flags) 2827 { 2828 long free_pages = zone_page_state(z, NR_FREE_PAGES); 2829 long cma_pages = 0; 2830 2831 #ifdef CONFIG_CMA 2832 /* If allocation can't use CMA areas don't use free CMA pages */ 2833 if (!(alloc_flags & ALLOC_CMA)) 2834 cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES); 2835 #endif 2836 2837 /* 2838 * Fast check for order-0 only. If this fails then the reserves 2839 * need to be calculated. There is a corner case where the check 2840 * passes but only the high-order atomic reserve are free. If 2841 * the caller is !atomic then it'll uselessly search the free 2842 * list. That corner case is then slower but it is harmless. 2843 */ 2844 if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx]) 2845 return true; 2846 2847 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 2848 free_pages); 2849 } 2850 2851 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 2852 unsigned long mark, int classzone_idx) 2853 { 2854 long free_pages = zone_page_state(z, NR_FREE_PAGES); 2855 2856 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 2857 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 2858 2859 return __zone_watermark_ok(z, order, mark, classzone_idx, 0, 2860 free_pages); 2861 } 2862 2863 #ifdef CONFIG_NUMA 2864 static bool zone_local(struct zone *local_zone, struct zone *zone) 2865 { 2866 return local_zone->node == zone->node; 2867 } 2868 2869 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 2870 { 2871 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) < 2872 RECLAIM_DISTANCE; 2873 } 2874 #else /* CONFIG_NUMA */ 2875 static bool zone_local(struct zone *local_zone, struct zone *zone) 2876 { 2877 return true; 2878 } 2879 2880 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 2881 { 2882 return true; 2883 } 2884 #endif /* CONFIG_NUMA */ 2885 2886 static void reset_alloc_batches(struct zone *preferred_zone) 2887 { 2888 struct zone *zone = preferred_zone->zone_pgdat->node_zones; 2889 2890 do { 2891 mod_zone_page_state(zone, NR_ALLOC_BATCH, 2892 high_wmark_pages(zone) - low_wmark_pages(zone) - 2893 atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH])); 2894 clear_bit(ZONE_FAIR_DEPLETED, &zone->flags); 2895 } while (zone++ != preferred_zone); 2896 } 2897 2898 /* 2899 * get_page_from_freelist goes through the zonelist trying to allocate 2900 * a page. 2901 */ 2902 static struct page * 2903 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 2904 const struct alloc_context *ac) 2905 { 2906 struct zoneref *z = ac->preferred_zoneref; 2907 struct zone *zone; 2908 bool fair_skipped = false; 2909 bool apply_fair = (alloc_flags & ALLOC_FAIR); 2910 2911 zonelist_scan: 2912 /* 2913 * Scan zonelist, looking for a zone with enough free. 2914 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 2915 */ 2916 for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 2917 ac->nodemask) { 2918 struct page *page; 2919 unsigned long mark; 2920 2921 if (cpusets_enabled() && 2922 (alloc_flags & ALLOC_CPUSET) && 2923 !__cpuset_zone_allowed(zone, gfp_mask)) 2924 continue; 2925 /* 2926 * Distribute pages in proportion to the individual 2927 * zone size to ensure fair page aging. The zone a 2928 * page was allocated in should have no effect on the 2929 * time the page has in memory before being reclaimed. 2930 */ 2931 if (apply_fair) { 2932 if (test_bit(ZONE_FAIR_DEPLETED, &zone->flags)) { 2933 fair_skipped = true; 2934 continue; 2935 } 2936 if (!zone_local(ac->preferred_zoneref->zone, zone)) { 2937 if (fair_skipped) 2938 goto reset_fair; 2939 apply_fair = false; 2940 } 2941 } 2942 /* 2943 * When allocating a page cache page for writing, we 2944 * want to get it from a zone that is within its dirty 2945 * limit, such that no single zone holds more than its 2946 * proportional share of globally allowed dirty pages. 2947 * The dirty limits take into account the zone's 2948 * lowmem reserves and high watermark so that kswapd 2949 * should be able to balance it without having to 2950 * write pages from its LRU list. 2951 * 2952 * This may look like it could increase pressure on 2953 * lower zones by failing allocations in higher zones 2954 * before they are full. But the pages that do spill 2955 * over are limited as the lower zones are protected 2956 * by this very same mechanism. It should not become 2957 * a practical burden to them. 2958 * 2959 * XXX: For now, allow allocations to potentially 2960 * exceed the per-zone dirty limit in the slowpath 2961 * (spread_dirty_pages unset) before going into reclaim, 2962 * which is important when on a NUMA setup the allowed 2963 * zones are together not big enough to reach the 2964 * global limit. The proper fix for these situations 2965 * will require awareness of zones in the 2966 * dirty-throttling and the flusher threads. 2967 */ 2968 if (ac->spread_dirty_pages && !zone_dirty_ok(zone)) 2969 continue; 2970 2971 mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK]; 2972 if (!zone_watermark_fast(zone, order, mark, 2973 ac_classzone_idx(ac), alloc_flags)) { 2974 int ret; 2975 2976 /* Checked here to keep the fast path fast */ 2977 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 2978 if (alloc_flags & ALLOC_NO_WATERMARKS) 2979 goto try_this_zone; 2980 2981 if (zone_reclaim_mode == 0 || 2982 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 2983 continue; 2984 2985 ret = zone_reclaim(zone, gfp_mask, order); 2986 switch (ret) { 2987 case ZONE_RECLAIM_NOSCAN: 2988 /* did not scan */ 2989 continue; 2990 case ZONE_RECLAIM_FULL: 2991 /* scanned but unreclaimable */ 2992 continue; 2993 default: 2994 /* did we reclaim enough */ 2995 if (zone_watermark_ok(zone, order, mark, 2996 ac_classzone_idx(ac), alloc_flags)) 2997 goto try_this_zone; 2998 2999 continue; 3000 } 3001 } 3002 3003 try_this_zone: 3004 page = buffered_rmqueue(ac->preferred_zoneref->zone, zone, order, 3005 gfp_mask, alloc_flags, ac->migratetype); 3006 if (page) { 3007 prep_new_page(page, order, gfp_mask, alloc_flags); 3008 3009 /* 3010 * If this is a high-order atomic allocation then check 3011 * if the pageblock should be reserved for the future 3012 */ 3013 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 3014 reserve_highatomic_pageblock(page, zone, order); 3015 3016 return page; 3017 } 3018 } 3019 3020 /* 3021 * The first pass makes sure allocations are spread fairly within the 3022 * local node. However, the local node might have free pages left 3023 * after the fairness batches are exhausted, and remote zones haven't 3024 * even been considered yet. Try once more without fairness, and 3025 * include remote zones now, before entering the slowpath and waking 3026 * kswapd: prefer spilling to a remote zone over swapping locally. 3027 */ 3028 if (fair_skipped) { 3029 reset_fair: 3030 apply_fair = false; 3031 fair_skipped = false; 3032 reset_alloc_batches(ac->preferred_zoneref->zone); 3033 z = ac->preferred_zoneref; 3034 goto zonelist_scan; 3035 } 3036 3037 return NULL; 3038 } 3039 3040 /* 3041 * Large machines with many possible nodes should not always dump per-node 3042 * meminfo in irq context. 3043 */ 3044 static inline bool should_suppress_show_mem(void) 3045 { 3046 bool ret = false; 3047 3048 #if NODES_SHIFT > 8 3049 ret = in_interrupt(); 3050 #endif 3051 return ret; 3052 } 3053 3054 static DEFINE_RATELIMIT_STATE(nopage_rs, 3055 DEFAULT_RATELIMIT_INTERVAL, 3056 DEFAULT_RATELIMIT_BURST); 3057 3058 void warn_alloc_failed(gfp_t gfp_mask, unsigned int order, const char *fmt, ...) 3059 { 3060 unsigned int filter = SHOW_MEM_FILTER_NODES; 3061 3062 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) || 3063 debug_guardpage_minorder() > 0) 3064 return; 3065 3066 /* 3067 * This documents exceptions given to allocations in certain 3068 * contexts that are allowed to allocate outside current's set 3069 * of allowed nodes. 3070 */ 3071 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3072 if (test_thread_flag(TIF_MEMDIE) || 3073 (current->flags & (PF_MEMALLOC | PF_EXITING))) 3074 filter &= ~SHOW_MEM_FILTER_NODES; 3075 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 3076 filter &= ~SHOW_MEM_FILTER_NODES; 3077 3078 if (fmt) { 3079 struct va_format vaf; 3080 va_list args; 3081 3082 va_start(args, fmt); 3083 3084 vaf.fmt = fmt; 3085 vaf.va = &args; 3086 3087 pr_warn("%pV", &vaf); 3088 3089 va_end(args); 3090 } 3091 3092 pr_warn("%s: page allocation failure: order:%u, mode:%#x(%pGg)\n", 3093 current->comm, order, gfp_mask, &gfp_mask); 3094 dump_stack(); 3095 if (!should_suppress_show_mem()) 3096 show_mem(filter); 3097 } 3098 3099 static inline struct page * 3100 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 3101 const struct alloc_context *ac, unsigned long *did_some_progress) 3102 { 3103 struct oom_control oc = { 3104 .zonelist = ac->zonelist, 3105 .nodemask = ac->nodemask, 3106 .gfp_mask = gfp_mask, 3107 .order = order, 3108 }; 3109 struct page *page; 3110 3111 *did_some_progress = 0; 3112 3113 /* 3114 * Acquire the oom lock. If that fails, somebody else is 3115 * making progress for us. 3116 */ 3117 if (!mutex_trylock(&oom_lock)) { 3118 *did_some_progress = 1; 3119 schedule_timeout_uninterruptible(1); 3120 return NULL; 3121 } 3122 3123 /* 3124 * Go through the zonelist yet one more time, keep very high watermark 3125 * here, this is only to catch a parallel oom killing, we must fail if 3126 * we're still under heavy pressure. 3127 */ 3128 page = get_page_from_freelist(gfp_mask | __GFP_HARDWALL, order, 3129 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 3130 if (page) 3131 goto out; 3132 3133 if (!(gfp_mask & __GFP_NOFAIL)) { 3134 /* Coredumps can quickly deplete all memory reserves */ 3135 if (current->flags & PF_DUMPCORE) 3136 goto out; 3137 /* The OOM killer will not help higher order allocs */ 3138 if (order > PAGE_ALLOC_COSTLY_ORDER) 3139 goto out; 3140 /* The OOM killer does not needlessly kill tasks for lowmem */ 3141 if (ac->high_zoneidx < ZONE_NORMAL) 3142 goto out; 3143 if (pm_suspended_storage()) 3144 goto out; 3145 /* 3146 * XXX: GFP_NOFS allocations should rather fail than rely on 3147 * other request to make a forward progress. 3148 * We are in an unfortunate situation where out_of_memory cannot 3149 * do much for this context but let's try it to at least get 3150 * access to memory reserved if the current task is killed (see 3151 * out_of_memory). Once filesystems are ready to handle allocation 3152 * failures more gracefully we should just bail out here. 3153 */ 3154 3155 /* The OOM killer may not free memory on a specific node */ 3156 if (gfp_mask & __GFP_THISNODE) 3157 goto out; 3158 } 3159 /* Exhausted what can be done so it's blamo time */ 3160 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 3161 *did_some_progress = 1; 3162 3163 if (gfp_mask & __GFP_NOFAIL) { 3164 page = get_page_from_freelist(gfp_mask, order, 3165 ALLOC_NO_WATERMARKS|ALLOC_CPUSET, ac); 3166 /* 3167 * fallback to ignore cpuset restriction if our nodes 3168 * are depleted 3169 */ 3170 if (!page) 3171 page = get_page_from_freelist(gfp_mask, order, 3172 ALLOC_NO_WATERMARKS, ac); 3173 } 3174 } 3175 out: 3176 mutex_unlock(&oom_lock); 3177 return page; 3178 } 3179 3180 3181 /* 3182 * Maximum number of compaction retries wit a progress before OOM 3183 * killer is consider as the only way to move forward. 3184 */ 3185 #define MAX_COMPACT_RETRIES 16 3186 3187 #ifdef CONFIG_COMPACTION 3188 /* Try memory compaction for high-order allocations before reclaim */ 3189 static struct page * 3190 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3191 unsigned int alloc_flags, const struct alloc_context *ac, 3192 enum migrate_mode mode, enum compact_result *compact_result) 3193 { 3194 struct page *page; 3195 int contended_compaction; 3196 3197 if (!order) 3198 return NULL; 3199 3200 current->flags |= PF_MEMALLOC; 3201 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 3202 mode, &contended_compaction); 3203 current->flags &= ~PF_MEMALLOC; 3204 3205 if (*compact_result <= COMPACT_INACTIVE) 3206 return NULL; 3207 3208 /* 3209 * At least in one zone compaction wasn't deferred or skipped, so let's 3210 * count a compaction stall 3211 */ 3212 count_vm_event(COMPACTSTALL); 3213 3214 page = get_page_from_freelist(gfp_mask, order, 3215 alloc_flags & ~ALLOC_NO_WATERMARKS, ac); 3216 3217 if (page) { 3218 struct zone *zone = page_zone(page); 3219 3220 zone->compact_blockskip_flush = false; 3221 compaction_defer_reset(zone, order, true); 3222 count_vm_event(COMPACTSUCCESS); 3223 return page; 3224 } 3225 3226 /* 3227 * It's bad if compaction run occurs and fails. The most likely reason 3228 * is that pages exist, but not enough to satisfy watermarks. 3229 */ 3230 count_vm_event(COMPACTFAIL); 3231 3232 /* 3233 * In all zones where compaction was attempted (and not 3234 * deferred or skipped), lock contention has been detected. 3235 * For THP allocation we do not want to disrupt the others 3236 * so we fallback to base pages instead. 3237 */ 3238 if (contended_compaction == COMPACT_CONTENDED_LOCK) 3239 *compact_result = COMPACT_CONTENDED; 3240 3241 /* 3242 * If compaction was aborted due to need_resched(), we do not 3243 * want to further increase allocation latency, unless it is 3244 * khugepaged trying to collapse. 3245 */ 3246 if (contended_compaction == COMPACT_CONTENDED_SCHED 3247 && !(current->flags & PF_KTHREAD)) 3248 *compact_result = COMPACT_CONTENDED; 3249 3250 cond_resched(); 3251 3252 return NULL; 3253 } 3254 3255 static inline bool 3256 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 3257 enum compact_result compact_result, enum migrate_mode *migrate_mode, 3258 int compaction_retries) 3259 { 3260 int max_retries = MAX_COMPACT_RETRIES; 3261 3262 if (!order) 3263 return false; 3264 3265 /* 3266 * compaction considers all the zone as desperately out of memory 3267 * so it doesn't really make much sense to retry except when the 3268 * failure could be caused by weak migration mode. 3269 */ 3270 if (compaction_failed(compact_result)) { 3271 if (*migrate_mode == MIGRATE_ASYNC) { 3272 *migrate_mode = MIGRATE_SYNC_LIGHT; 3273 return true; 3274 } 3275 return false; 3276 } 3277 3278 /* 3279 * make sure the compaction wasn't deferred or didn't bail out early 3280 * due to locks contention before we declare that we should give up. 3281 * But do not retry if the given zonelist is not suitable for 3282 * compaction. 3283 */ 3284 if (compaction_withdrawn(compact_result)) 3285 return compaction_zonelist_suitable(ac, order, alloc_flags); 3286 3287 /* 3288 * !costly requests are much more important than __GFP_REPEAT 3289 * costly ones because they are de facto nofail and invoke OOM 3290 * killer to move on while costly can fail and users are ready 3291 * to cope with that. 1/4 retries is rather arbitrary but we 3292 * would need much more detailed feedback from compaction to 3293 * make a better decision. 3294 */ 3295 if (order > PAGE_ALLOC_COSTLY_ORDER) 3296 max_retries /= 4; 3297 if (compaction_retries <= max_retries) 3298 return true; 3299 3300 return false; 3301 } 3302 #else 3303 static inline struct page * 3304 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 3305 unsigned int alloc_flags, const struct alloc_context *ac, 3306 enum migrate_mode mode, enum compact_result *compact_result) 3307 { 3308 *compact_result = COMPACT_SKIPPED; 3309 return NULL; 3310 } 3311 3312 static inline bool 3313 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 3314 enum compact_result compact_result, 3315 enum migrate_mode *migrate_mode, 3316 int compaction_retries) 3317 { 3318 struct zone *zone; 3319 struct zoneref *z; 3320 3321 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 3322 return false; 3323 3324 /* 3325 * There are setups with compaction disabled which would prefer to loop 3326 * inside the allocator rather than hit the oom killer prematurely. 3327 * Let's give them a good hope and keep retrying while the order-0 3328 * watermarks are OK. 3329 */ 3330 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3331 ac->nodemask) { 3332 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 3333 ac_classzone_idx(ac), alloc_flags)) 3334 return true; 3335 } 3336 return false; 3337 } 3338 #endif /* CONFIG_COMPACTION */ 3339 3340 /* Perform direct synchronous page reclaim */ 3341 static int 3342 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 3343 const struct alloc_context *ac) 3344 { 3345 struct reclaim_state reclaim_state; 3346 int progress; 3347 3348 cond_resched(); 3349 3350 /* We now go into synchronous reclaim */ 3351 cpuset_memory_pressure_bump(); 3352 current->flags |= PF_MEMALLOC; 3353 lockdep_set_current_reclaim_state(gfp_mask); 3354 reclaim_state.reclaimed_slab = 0; 3355 current->reclaim_state = &reclaim_state; 3356 3357 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 3358 ac->nodemask); 3359 3360 current->reclaim_state = NULL; 3361 lockdep_clear_current_reclaim_state(); 3362 current->flags &= ~PF_MEMALLOC; 3363 3364 cond_resched(); 3365 3366 return progress; 3367 } 3368 3369 /* The really slow allocator path where we enter direct reclaim */ 3370 static inline struct page * 3371 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 3372 unsigned int alloc_flags, const struct alloc_context *ac, 3373 unsigned long *did_some_progress) 3374 { 3375 struct page *page = NULL; 3376 bool drained = false; 3377 3378 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 3379 if (unlikely(!(*did_some_progress))) 3380 return NULL; 3381 3382 retry: 3383 page = get_page_from_freelist(gfp_mask, order, 3384 alloc_flags & ~ALLOC_NO_WATERMARKS, ac); 3385 3386 /* 3387 * If an allocation failed after direct reclaim, it could be because 3388 * pages are pinned on the per-cpu lists or in high alloc reserves. 3389 * Shrink them them and try again 3390 */ 3391 if (!page && !drained) { 3392 unreserve_highatomic_pageblock(ac); 3393 drain_all_pages(NULL); 3394 drained = true; 3395 goto retry; 3396 } 3397 3398 return page; 3399 } 3400 3401 static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac) 3402 { 3403 struct zoneref *z; 3404 struct zone *zone; 3405 3406 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 3407 ac->high_zoneidx, ac->nodemask) 3408 wakeup_kswapd(zone, order, ac_classzone_idx(ac)); 3409 } 3410 3411 static inline unsigned int 3412 gfp_to_alloc_flags(gfp_t gfp_mask) 3413 { 3414 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 3415 3416 /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */ 3417 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 3418 3419 /* 3420 * The caller may dip into page reserves a bit more if the caller 3421 * cannot run direct reclaim, or if the caller has realtime scheduling 3422 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 3423 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 3424 */ 3425 alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); 3426 3427 if (gfp_mask & __GFP_ATOMIC) { 3428 /* 3429 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 3430 * if it can't schedule. 3431 */ 3432 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3433 alloc_flags |= ALLOC_HARDER; 3434 /* 3435 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 3436 * comment for __cpuset_node_allowed(). 3437 */ 3438 alloc_flags &= ~ALLOC_CPUSET; 3439 } else if (unlikely(rt_task(current)) && !in_interrupt()) 3440 alloc_flags |= ALLOC_HARDER; 3441 3442 if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) { 3443 if (gfp_mask & __GFP_MEMALLOC) 3444 alloc_flags |= ALLOC_NO_WATERMARKS; 3445 else if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 3446 alloc_flags |= ALLOC_NO_WATERMARKS; 3447 else if (!in_interrupt() && 3448 ((current->flags & PF_MEMALLOC) || 3449 unlikely(test_thread_flag(TIF_MEMDIE)))) 3450 alloc_flags |= ALLOC_NO_WATERMARKS; 3451 } 3452 #ifdef CONFIG_CMA 3453 if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) 3454 alloc_flags |= ALLOC_CMA; 3455 #endif 3456 return alloc_flags; 3457 } 3458 3459 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 3460 { 3461 return !!(gfp_to_alloc_flags(gfp_mask) & ALLOC_NO_WATERMARKS); 3462 } 3463 3464 static inline bool is_thp_gfp_mask(gfp_t gfp_mask) 3465 { 3466 return (gfp_mask & (GFP_TRANSHUGE | __GFP_KSWAPD_RECLAIM)) == GFP_TRANSHUGE; 3467 } 3468 3469 /* 3470 * Maximum number of reclaim retries without any progress before OOM killer 3471 * is consider as the only way to move forward. 3472 */ 3473 #define MAX_RECLAIM_RETRIES 16 3474 3475 /* 3476 * Checks whether it makes sense to retry the reclaim to make a forward progress 3477 * for the given allocation request. 3478 * The reclaim feedback represented by did_some_progress (any progress during 3479 * the last reclaim round) and no_progress_loops (number of reclaim rounds without 3480 * any progress in a row) is considered as well as the reclaimable pages on the 3481 * applicable zone list (with a backoff mechanism which is a function of 3482 * no_progress_loops). 3483 * 3484 * Returns true if a retry is viable or false to enter the oom path. 3485 */ 3486 static inline bool 3487 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 3488 struct alloc_context *ac, int alloc_flags, 3489 bool did_some_progress, int no_progress_loops) 3490 { 3491 struct zone *zone; 3492 struct zoneref *z; 3493 3494 /* 3495 * Make sure we converge to OOM if we cannot make any progress 3496 * several times in the row. 3497 */ 3498 if (no_progress_loops > MAX_RECLAIM_RETRIES) 3499 return false; 3500 3501 /* 3502 * Keep reclaiming pages while there is a chance this will lead somewhere. 3503 * If none of the target zones can satisfy our allocation request even 3504 * if all reclaimable pages are considered then we are screwed and have 3505 * to go OOM. 3506 */ 3507 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, 3508 ac->nodemask) { 3509 unsigned long available; 3510 unsigned long reclaimable; 3511 3512 available = reclaimable = zone_reclaimable_pages(zone); 3513 available -= DIV_ROUND_UP(no_progress_loops * available, 3514 MAX_RECLAIM_RETRIES); 3515 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 3516 3517 /* 3518 * Would the allocation succeed if we reclaimed the whole 3519 * available? 3520 */ 3521 if (__zone_watermark_ok(zone, order, min_wmark_pages(zone), 3522 ac_classzone_idx(ac), alloc_flags, available)) { 3523 /* 3524 * If we didn't make any progress and have a lot of 3525 * dirty + writeback pages then we should wait for 3526 * an IO to complete to slow down the reclaim and 3527 * prevent from pre mature OOM 3528 */ 3529 if (!did_some_progress) { 3530 unsigned long writeback; 3531 unsigned long dirty; 3532 3533 writeback = zone_page_state_snapshot(zone, 3534 NR_WRITEBACK); 3535 dirty = zone_page_state_snapshot(zone, NR_FILE_DIRTY); 3536 3537 if (2*(writeback + dirty) > reclaimable) { 3538 congestion_wait(BLK_RW_ASYNC, HZ/10); 3539 return true; 3540 } 3541 } 3542 3543 /* 3544 * Memory allocation/reclaim might be called from a WQ 3545 * context and the current implementation of the WQ 3546 * concurrency control doesn't recognize that 3547 * a particular WQ is congested if the worker thread is 3548 * looping without ever sleeping. Therefore we have to 3549 * do a short sleep here rather than calling 3550 * cond_resched(). 3551 */ 3552 if (current->flags & PF_WQ_WORKER) 3553 schedule_timeout_uninterruptible(1); 3554 else 3555 cond_resched(); 3556 3557 return true; 3558 } 3559 } 3560 3561 return false; 3562 } 3563 3564 static inline struct page * 3565 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 3566 struct alloc_context *ac) 3567 { 3568 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 3569 struct page *page = NULL; 3570 unsigned int alloc_flags; 3571 unsigned long did_some_progress; 3572 enum migrate_mode migration_mode = MIGRATE_ASYNC; 3573 enum compact_result compact_result; 3574 int compaction_retries = 0; 3575 int no_progress_loops = 0; 3576 3577 /* 3578 * In the slowpath, we sanity check order to avoid ever trying to 3579 * reclaim >= MAX_ORDER areas which will never succeed. Callers may 3580 * be using allocators in order of preference for an area that is 3581 * too large. 3582 */ 3583 if (order >= MAX_ORDER) { 3584 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); 3585 return NULL; 3586 } 3587 3588 /* 3589 * We also sanity check to catch abuse of atomic reserves being used by 3590 * callers that are not in atomic context. 3591 */ 3592 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 3593 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 3594 gfp_mask &= ~__GFP_ATOMIC; 3595 3596 retry: 3597 if (gfp_mask & __GFP_KSWAPD_RECLAIM) 3598 wake_all_kswapds(order, ac); 3599 3600 /* 3601 * OK, we're below the kswapd watermark and have kicked background 3602 * reclaim. Now things get more complex, so set up alloc_flags according 3603 * to how we want to proceed. 3604 */ 3605 alloc_flags = gfp_to_alloc_flags(gfp_mask); 3606 3607 /* 3608 * Reset the zonelist iterators if memory policies can be ignored. 3609 * These allocations are high priority and system rather than user 3610 * orientated. 3611 */ 3612 if ((alloc_flags & ALLOC_NO_WATERMARKS) || !(alloc_flags & ALLOC_CPUSET)) { 3613 ac->zonelist = node_zonelist(numa_node_id(), gfp_mask); 3614 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 3615 ac->high_zoneidx, ac->nodemask); 3616 } 3617 3618 /* This is the last chance, in general, before the goto nopage. */ 3619 page = get_page_from_freelist(gfp_mask, order, 3620 alloc_flags & ~ALLOC_NO_WATERMARKS, ac); 3621 if (page) 3622 goto got_pg; 3623 3624 /* Allocate without watermarks if the context allows */ 3625 if (alloc_flags & ALLOC_NO_WATERMARKS) { 3626 page = get_page_from_freelist(gfp_mask, order, 3627 ALLOC_NO_WATERMARKS, ac); 3628 if (page) 3629 goto got_pg; 3630 } 3631 3632 /* Caller is not willing to reclaim, we can't balance anything */ 3633 if (!can_direct_reclaim) { 3634 /* 3635 * All existing users of the __GFP_NOFAIL are blockable, so warn 3636 * of any new users that actually allow this type of allocation 3637 * to fail. 3638 */ 3639 WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL); 3640 goto nopage; 3641 } 3642 3643 /* Avoid recursion of direct reclaim */ 3644 if (current->flags & PF_MEMALLOC) { 3645 /* 3646 * __GFP_NOFAIL request from this context is rather bizarre 3647 * because we cannot reclaim anything and only can loop waiting 3648 * for somebody to do a work for us. 3649 */ 3650 if (WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 3651 cond_resched(); 3652 goto retry; 3653 } 3654 goto nopage; 3655 } 3656 3657 /* Avoid allocations with no watermarks from looping endlessly */ 3658 if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL)) 3659 goto nopage; 3660 3661 /* 3662 * Try direct compaction. The first pass is asynchronous. Subsequent 3663 * attempts after direct reclaim are synchronous 3664 */ 3665 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 3666 migration_mode, 3667 &compact_result); 3668 if (page) 3669 goto got_pg; 3670 3671 /* Checks for THP-specific high-order allocations */ 3672 if (is_thp_gfp_mask(gfp_mask)) { 3673 /* 3674 * If compaction is deferred for high-order allocations, it is 3675 * because sync compaction recently failed. If this is the case 3676 * and the caller requested a THP allocation, we do not want 3677 * to heavily disrupt the system, so we fail the allocation 3678 * instead of entering direct reclaim. 3679 */ 3680 if (compact_result == COMPACT_DEFERRED) 3681 goto nopage; 3682 3683 /* 3684 * Compaction is contended so rather back off than cause 3685 * excessive stalls. 3686 */ 3687 if(compact_result == COMPACT_CONTENDED) 3688 goto nopage; 3689 } 3690 3691 if (order && compaction_made_progress(compact_result)) 3692 compaction_retries++; 3693 3694 /* Try direct reclaim and then allocating */ 3695 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 3696 &did_some_progress); 3697 if (page) 3698 goto got_pg; 3699 3700 /* Do not loop if specifically requested */ 3701 if (gfp_mask & __GFP_NORETRY) 3702 goto noretry; 3703 3704 /* 3705 * Do not retry costly high order allocations unless they are 3706 * __GFP_REPEAT 3707 */ 3708 if (order > PAGE_ALLOC_COSTLY_ORDER && !(gfp_mask & __GFP_REPEAT)) 3709 goto noretry; 3710 3711 /* 3712 * Costly allocations might have made a progress but this doesn't mean 3713 * their order will become available due to high fragmentation so 3714 * always increment the no progress counter for them 3715 */ 3716 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 3717 no_progress_loops = 0; 3718 else 3719 no_progress_loops++; 3720 3721 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 3722 did_some_progress > 0, no_progress_loops)) 3723 goto retry; 3724 3725 /* 3726 * It doesn't make any sense to retry for the compaction if the order-0 3727 * reclaim is not able to make any progress because the current 3728 * implementation of the compaction depends on the sufficient amount 3729 * of free memory (see __compaction_suitable) 3730 */ 3731 if (did_some_progress > 0 && 3732 should_compact_retry(ac, order, alloc_flags, 3733 compact_result, &migration_mode, 3734 compaction_retries)) 3735 goto retry; 3736 3737 /* Reclaim has failed us, start killing things */ 3738 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 3739 if (page) 3740 goto got_pg; 3741 3742 /* Retry as long as the OOM killer is making progress */ 3743 if (did_some_progress) { 3744 no_progress_loops = 0; 3745 goto retry; 3746 } 3747 3748 noretry: 3749 /* 3750 * High-order allocations do not necessarily loop after direct reclaim 3751 * and reclaim/compaction depends on compaction being called after 3752 * reclaim so call directly if necessary. 3753 * It can become very expensive to allocate transparent hugepages at 3754 * fault, so use asynchronous memory compaction for THP unless it is 3755 * khugepaged trying to collapse. All other requests should tolerate 3756 * at least light sync migration. 3757 */ 3758 if (is_thp_gfp_mask(gfp_mask) && !(current->flags & PF_KTHREAD)) 3759 migration_mode = MIGRATE_ASYNC; 3760 else 3761 migration_mode = MIGRATE_SYNC_LIGHT; 3762 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, 3763 ac, migration_mode, 3764 &compact_result); 3765 if (page) 3766 goto got_pg; 3767 nopage: 3768 warn_alloc_failed(gfp_mask, order, NULL); 3769 got_pg: 3770 return page; 3771 } 3772 3773 /* 3774 * This is the 'heart' of the zoned buddy allocator. 3775 */ 3776 struct page * 3777 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, 3778 struct zonelist *zonelist, nodemask_t *nodemask) 3779 { 3780 struct page *page; 3781 unsigned int cpuset_mems_cookie; 3782 unsigned int alloc_flags = ALLOC_WMARK_LOW|ALLOC_FAIR; 3783 gfp_t alloc_mask = gfp_mask; /* The gfp_t that was actually used for allocation */ 3784 struct alloc_context ac = { 3785 .high_zoneidx = gfp_zone(gfp_mask), 3786 .zonelist = zonelist, 3787 .nodemask = nodemask, 3788 .migratetype = gfpflags_to_migratetype(gfp_mask), 3789 }; 3790 3791 if (cpusets_enabled()) { 3792 alloc_mask |= __GFP_HARDWALL; 3793 alloc_flags |= ALLOC_CPUSET; 3794 if (!ac.nodemask) 3795 ac.nodemask = &cpuset_current_mems_allowed; 3796 } 3797 3798 gfp_mask &= gfp_allowed_mask; 3799 3800 lockdep_trace_alloc(gfp_mask); 3801 3802 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 3803 3804 if (should_fail_alloc_page(gfp_mask, order)) 3805 return NULL; 3806 3807 /* 3808 * Check the zones suitable for the gfp_mask contain at least one 3809 * valid zone. It's possible to have an empty zonelist as a result 3810 * of __GFP_THISNODE and a memoryless node 3811 */ 3812 if (unlikely(!zonelist->_zonerefs->zone)) 3813 return NULL; 3814 3815 if (IS_ENABLED(CONFIG_CMA) && ac.migratetype == MIGRATE_MOVABLE) 3816 alloc_flags |= ALLOC_CMA; 3817 3818 retry_cpuset: 3819 cpuset_mems_cookie = read_mems_allowed_begin(); 3820 3821 /* Dirty zone balancing only done in the fast path */ 3822 ac.spread_dirty_pages = (gfp_mask & __GFP_WRITE); 3823 3824 /* 3825 * The preferred zone is used for statistics but crucially it is 3826 * also used as the starting point for the zonelist iterator. It 3827 * may get reset for allocations that ignore memory policies. 3828 */ 3829 ac.preferred_zoneref = first_zones_zonelist(ac.zonelist, 3830 ac.high_zoneidx, ac.nodemask); 3831 if (!ac.preferred_zoneref) { 3832 page = NULL; 3833 goto no_zone; 3834 } 3835 3836 /* First allocation attempt */ 3837 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); 3838 if (likely(page)) 3839 goto out; 3840 3841 /* 3842 * Runtime PM, block IO and its error handling path can deadlock 3843 * because I/O on the device might not complete. 3844 */ 3845 alloc_mask = memalloc_noio_flags(gfp_mask); 3846 ac.spread_dirty_pages = false; 3847 3848 /* 3849 * Restore the original nodemask if it was potentially replaced with 3850 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 3851 */ 3852 if (cpusets_enabled()) 3853 ac.nodemask = nodemask; 3854 page = __alloc_pages_slowpath(alloc_mask, order, &ac); 3855 3856 no_zone: 3857 /* 3858 * When updating a task's mems_allowed, it is possible to race with 3859 * parallel threads in such a way that an allocation can fail while 3860 * the mask is being updated. If a page allocation is about to fail, 3861 * check if the cpuset changed during allocation and if so, retry. 3862 */ 3863 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) { 3864 alloc_mask = gfp_mask; 3865 goto retry_cpuset; 3866 } 3867 3868 out: 3869 if (kmemcheck_enabled && page) 3870 kmemcheck_pagealloc_alloc(page, order, gfp_mask); 3871 3872 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); 3873 3874 return page; 3875 } 3876 EXPORT_SYMBOL(__alloc_pages_nodemask); 3877 3878 /* 3879 * Common helper functions. 3880 */ 3881 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 3882 { 3883 struct page *page; 3884 3885 /* 3886 * __get_free_pages() returns a 32-bit address, which cannot represent 3887 * a highmem page 3888 */ 3889 VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0); 3890 3891 page = alloc_pages(gfp_mask, order); 3892 if (!page) 3893 return 0; 3894 return (unsigned long) page_address(page); 3895 } 3896 EXPORT_SYMBOL(__get_free_pages); 3897 3898 unsigned long get_zeroed_page(gfp_t gfp_mask) 3899 { 3900 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 3901 } 3902 EXPORT_SYMBOL(get_zeroed_page); 3903 3904 void __free_pages(struct page *page, unsigned int order) 3905 { 3906 if (put_page_testzero(page)) { 3907 if (order == 0) 3908 free_hot_cold_page(page, false); 3909 else 3910 __free_pages_ok(page, order); 3911 } 3912 } 3913 3914 EXPORT_SYMBOL(__free_pages); 3915 3916 void free_pages(unsigned long addr, unsigned int order) 3917 { 3918 if (addr != 0) { 3919 VM_BUG_ON(!virt_addr_valid((void *)addr)); 3920 __free_pages(virt_to_page((void *)addr), order); 3921 } 3922 } 3923 3924 EXPORT_SYMBOL(free_pages); 3925 3926 /* 3927 * Page Fragment: 3928 * An arbitrary-length arbitrary-offset area of memory which resides 3929 * within a 0 or higher order page. Multiple fragments within that page 3930 * are individually refcounted, in the page's reference counter. 3931 * 3932 * The page_frag functions below provide a simple allocation framework for 3933 * page fragments. This is used by the network stack and network device 3934 * drivers to provide a backing region of memory for use as either an 3935 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 3936 */ 3937 static struct page *__page_frag_refill(struct page_frag_cache *nc, 3938 gfp_t gfp_mask) 3939 { 3940 struct page *page = NULL; 3941 gfp_t gfp = gfp_mask; 3942 3943 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 3944 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 3945 __GFP_NOMEMALLOC; 3946 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 3947 PAGE_FRAG_CACHE_MAX_ORDER); 3948 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 3949 #endif 3950 if (unlikely(!page)) 3951 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 3952 3953 nc->va = page ? page_address(page) : NULL; 3954 3955 return page; 3956 } 3957 3958 void *__alloc_page_frag(struct page_frag_cache *nc, 3959 unsigned int fragsz, gfp_t gfp_mask) 3960 { 3961 unsigned int size = PAGE_SIZE; 3962 struct page *page; 3963 int offset; 3964 3965 if (unlikely(!nc->va)) { 3966 refill: 3967 page = __page_frag_refill(nc, gfp_mask); 3968 if (!page) 3969 return NULL; 3970 3971 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 3972 /* if size can vary use size else just use PAGE_SIZE */ 3973 size = nc->size; 3974 #endif 3975 /* Even if we own the page, we do not use atomic_set(). 3976 * This would break get_page_unless_zero() users. 3977 */ 3978 page_ref_add(page, size - 1); 3979 3980 /* reset page count bias and offset to start of new frag */ 3981 nc->pfmemalloc = page_is_pfmemalloc(page); 3982 nc->pagecnt_bias = size; 3983 nc->offset = size; 3984 } 3985 3986 offset = nc->offset - fragsz; 3987 if (unlikely(offset < 0)) { 3988 page = virt_to_page(nc->va); 3989 3990 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 3991 goto refill; 3992 3993 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 3994 /* if size can vary use size else just use PAGE_SIZE */ 3995 size = nc->size; 3996 #endif 3997 /* OK, page count is 0, we can safely set it */ 3998 set_page_count(page, size); 3999 4000 /* reset page count bias and offset to start of new frag */ 4001 nc->pagecnt_bias = size; 4002 offset = size - fragsz; 4003 } 4004 4005 nc->pagecnt_bias--; 4006 nc->offset = offset; 4007 4008 return nc->va + offset; 4009 } 4010 EXPORT_SYMBOL(__alloc_page_frag); 4011 4012 /* 4013 * Frees a page fragment allocated out of either a compound or order 0 page. 4014 */ 4015 void __free_page_frag(void *addr) 4016 { 4017 struct page *page = virt_to_head_page(addr); 4018 4019 if (unlikely(put_page_testzero(page))) 4020 __free_pages_ok(page, compound_order(page)); 4021 } 4022 EXPORT_SYMBOL(__free_page_frag); 4023 4024 /* 4025 * alloc_kmem_pages charges newly allocated pages to the kmem resource counter 4026 * of the current memory cgroup if __GFP_ACCOUNT is set, other than that it is 4027 * equivalent to alloc_pages. 4028 * 4029 * It should be used when the caller would like to use kmalloc, but since the 4030 * allocation is large, it has to fall back to the page allocator. 4031 */ 4032 struct page *alloc_kmem_pages(gfp_t gfp_mask, unsigned int order) 4033 { 4034 struct page *page; 4035 4036 page = alloc_pages(gfp_mask, order); 4037 if (page && memcg_kmem_charge(page, gfp_mask, order) != 0) { 4038 __free_pages(page, order); 4039 page = NULL; 4040 } 4041 return page; 4042 } 4043 4044 struct page *alloc_kmem_pages_node(int nid, gfp_t gfp_mask, unsigned int order) 4045 { 4046 struct page *page; 4047 4048 page = alloc_pages_node(nid, gfp_mask, order); 4049 if (page && memcg_kmem_charge(page, gfp_mask, order) != 0) { 4050 __free_pages(page, order); 4051 page = NULL; 4052 } 4053 return page; 4054 } 4055 4056 /* 4057 * __free_kmem_pages and free_kmem_pages will free pages allocated with 4058 * alloc_kmem_pages. 4059 */ 4060 void __free_kmem_pages(struct page *page, unsigned int order) 4061 { 4062 memcg_kmem_uncharge(page, order); 4063 __free_pages(page, order); 4064 } 4065 4066 void free_kmem_pages(unsigned long addr, unsigned int order) 4067 { 4068 if (addr != 0) { 4069 VM_BUG_ON(!virt_addr_valid((void *)addr)); 4070 __free_kmem_pages(virt_to_page((void *)addr), order); 4071 } 4072 } 4073 4074 static void *make_alloc_exact(unsigned long addr, unsigned int order, 4075 size_t size) 4076 { 4077 if (addr) { 4078 unsigned long alloc_end = addr + (PAGE_SIZE << order); 4079 unsigned long used = addr + PAGE_ALIGN(size); 4080 4081 split_page(virt_to_page((void *)addr), order); 4082 while (used < alloc_end) { 4083 free_page(used); 4084 used += PAGE_SIZE; 4085 } 4086 } 4087 return (void *)addr; 4088 } 4089 4090 /** 4091 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 4092 * @size: the number of bytes to allocate 4093 * @gfp_mask: GFP flags for the allocation 4094 * 4095 * This function is similar to alloc_pages(), except that it allocates the 4096 * minimum number of pages to satisfy the request. alloc_pages() can only 4097 * allocate memory in power-of-two pages. 4098 * 4099 * This function is also limited by MAX_ORDER. 4100 * 4101 * Memory allocated by this function must be released by free_pages_exact(). 4102 */ 4103 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 4104 { 4105 unsigned int order = get_order(size); 4106 unsigned long addr; 4107 4108 addr = __get_free_pages(gfp_mask, order); 4109 return make_alloc_exact(addr, order, size); 4110 } 4111 EXPORT_SYMBOL(alloc_pages_exact); 4112 4113 /** 4114 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 4115 * pages on a node. 4116 * @nid: the preferred node ID where memory should be allocated 4117 * @size: the number of bytes to allocate 4118 * @gfp_mask: GFP flags for the allocation 4119 * 4120 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 4121 * back. 4122 */ 4123 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 4124 { 4125 unsigned int order = get_order(size); 4126 struct page *p = alloc_pages_node(nid, gfp_mask, order); 4127 if (!p) 4128 return NULL; 4129 return make_alloc_exact((unsigned long)page_address(p), order, size); 4130 } 4131 4132 /** 4133 * free_pages_exact - release memory allocated via alloc_pages_exact() 4134 * @virt: the value returned by alloc_pages_exact. 4135 * @size: size of allocation, same value as passed to alloc_pages_exact(). 4136 * 4137 * Release the memory allocated by a previous call to alloc_pages_exact. 4138 */ 4139 void free_pages_exact(void *virt, size_t size) 4140 { 4141 unsigned long addr = (unsigned long)virt; 4142 unsigned long end = addr + PAGE_ALIGN(size); 4143 4144 while (addr < end) { 4145 free_page(addr); 4146 addr += PAGE_SIZE; 4147 } 4148 } 4149 EXPORT_SYMBOL(free_pages_exact); 4150 4151 /** 4152 * nr_free_zone_pages - count number of pages beyond high watermark 4153 * @offset: The zone index of the highest zone 4154 * 4155 * nr_free_zone_pages() counts the number of counts pages which are beyond the 4156 * high watermark within all zones at or below a given zone index. For each 4157 * zone, the number of pages is calculated as: 4158 * managed_pages - high_pages 4159 */ 4160 static unsigned long nr_free_zone_pages(int offset) 4161 { 4162 struct zoneref *z; 4163 struct zone *zone; 4164 4165 /* Just pick one node, since fallback list is circular */ 4166 unsigned long sum = 0; 4167 4168 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 4169 4170 for_each_zone_zonelist(zone, z, zonelist, offset) { 4171 unsigned long size = zone->managed_pages; 4172 unsigned long high = high_wmark_pages(zone); 4173 if (size > high) 4174 sum += size - high; 4175 } 4176 4177 return sum; 4178 } 4179 4180 /** 4181 * nr_free_buffer_pages - count number of pages beyond high watermark 4182 * 4183 * nr_free_buffer_pages() counts the number of pages which are beyond the high 4184 * watermark within ZONE_DMA and ZONE_NORMAL. 4185 */ 4186 unsigned long nr_free_buffer_pages(void) 4187 { 4188 return nr_free_zone_pages(gfp_zone(GFP_USER)); 4189 } 4190 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 4191 4192 /** 4193 * nr_free_pagecache_pages - count number of pages beyond high watermark 4194 * 4195 * nr_free_pagecache_pages() counts the number of pages which are beyond the 4196 * high watermark within all zones. 4197 */ 4198 unsigned long nr_free_pagecache_pages(void) 4199 { 4200 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 4201 } 4202 4203 static inline void show_node(struct zone *zone) 4204 { 4205 if (IS_ENABLED(CONFIG_NUMA)) 4206 printk("Node %d ", zone_to_nid(zone)); 4207 } 4208 4209 long si_mem_available(void) 4210 { 4211 long available; 4212 unsigned long pagecache; 4213 unsigned long wmark_low = 0; 4214 unsigned long pages[NR_LRU_LISTS]; 4215 struct zone *zone; 4216 int lru; 4217 4218 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 4219 pages[lru] = global_page_state(NR_LRU_BASE + lru); 4220 4221 for_each_zone(zone) 4222 wmark_low += zone->watermark[WMARK_LOW]; 4223 4224 /* 4225 * Estimate the amount of memory available for userspace allocations, 4226 * without causing swapping. 4227 */ 4228 available = global_page_state(NR_FREE_PAGES) - totalreserve_pages; 4229 4230 /* 4231 * Not all the page cache can be freed, otherwise the system will 4232 * start swapping. Assume at least half of the page cache, or the 4233 * low watermark worth of cache, needs to stay. 4234 */ 4235 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 4236 pagecache -= min(pagecache / 2, wmark_low); 4237 available += pagecache; 4238 4239 /* 4240 * Part of the reclaimable slab consists of items that are in use, 4241 * and cannot be freed. Cap this estimate at the low watermark. 4242 */ 4243 available += global_page_state(NR_SLAB_RECLAIMABLE) - 4244 min(global_page_state(NR_SLAB_RECLAIMABLE) / 2, wmark_low); 4245 4246 if (available < 0) 4247 available = 0; 4248 return available; 4249 } 4250 EXPORT_SYMBOL_GPL(si_mem_available); 4251 4252 void si_meminfo(struct sysinfo *val) 4253 { 4254 val->totalram = totalram_pages; 4255 val->sharedram = global_page_state(NR_SHMEM); 4256 val->freeram = global_page_state(NR_FREE_PAGES); 4257 val->bufferram = nr_blockdev_pages(); 4258 val->totalhigh = totalhigh_pages; 4259 val->freehigh = nr_free_highpages(); 4260 val->mem_unit = PAGE_SIZE; 4261 } 4262 4263 EXPORT_SYMBOL(si_meminfo); 4264 4265 #ifdef CONFIG_NUMA 4266 void si_meminfo_node(struct sysinfo *val, int nid) 4267 { 4268 int zone_type; /* needs to be signed */ 4269 unsigned long managed_pages = 0; 4270 unsigned long managed_highpages = 0; 4271 unsigned long free_highpages = 0; 4272 pg_data_t *pgdat = NODE_DATA(nid); 4273 4274 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 4275 managed_pages += pgdat->node_zones[zone_type].managed_pages; 4276 val->totalram = managed_pages; 4277 val->sharedram = node_page_state(nid, NR_SHMEM); 4278 val->freeram = node_page_state(nid, NR_FREE_PAGES); 4279 #ifdef CONFIG_HIGHMEM 4280 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 4281 struct zone *zone = &pgdat->node_zones[zone_type]; 4282 4283 if (is_highmem(zone)) { 4284 managed_highpages += zone->managed_pages; 4285 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 4286 } 4287 } 4288 val->totalhigh = managed_highpages; 4289 val->freehigh = free_highpages; 4290 #else 4291 val->totalhigh = managed_highpages; 4292 val->freehigh = free_highpages; 4293 #endif 4294 val->mem_unit = PAGE_SIZE; 4295 } 4296 #endif 4297 4298 /* 4299 * Determine whether the node should be displayed or not, depending on whether 4300 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 4301 */ 4302 bool skip_free_areas_node(unsigned int flags, int nid) 4303 { 4304 bool ret = false; 4305 unsigned int cpuset_mems_cookie; 4306 4307 if (!(flags & SHOW_MEM_FILTER_NODES)) 4308 goto out; 4309 4310 do { 4311 cpuset_mems_cookie = read_mems_allowed_begin(); 4312 ret = !node_isset(nid, cpuset_current_mems_allowed); 4313 } while (read_mems_allowed_retry(cpuset_mems_cookie)); 4314 out: 4315 return ret; 4316 } 4317 4318 #define K(x) ((x) << (PAGE_SHIFT-10)) 4319 4320 static void show_migration_types(unsigned char type) 4321 { 4322 static const char types[MIGRATE_TYPES] = { 4323 [MIGRATE_UNMOVABLE] = 'U', 4324 [MIGRATE_MOVABLE] = 'M', 4325 [MIGRATE_RECLAIMABLE] = 'E', 4326 [MIGRATE_HIGHATOMIC] = 'H', 4327 #ifdef CONFIG_CMA 4328 [MIGRATE_CMA] = 'C', 4329 #endif 4330 #ifdef CONFIG_MEMORY_ISOLATION 4331 [MIGRATE_ISOLATE] = 'I', 4332 #endif 4333 }; 4334 char tmp[MIGRATE_TYPES + 1]; 4335 char *p = tmp; 4336 int i; 4337 4338 for (i = 0; i < MIGRATE_TYPES; i++) { 4339 if (type & (1 << i)) 4340 *p++ = types[i]; 4341 } 4342 4343 *p = '\0'; 4344 printk("(%s) ", tmp); 4345 } 4346 4347 /* 4348 * Show free area list (used inside shift_scroll-lock stuff) 4349 * We also calculate the percentage fragmentation. We do this by counting the 4350 * memory on each free list with the exception of the first item on the list. 4351 * 4352 * Bits in @filter: 4353 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 4354 * cpuset. 4355 */ 4356 void show_free_areas(unsigned int filter) 4357 { 4358 unsigned long free_pcp = 0; 4359 int cpu; 4360 struct zone *zone; 4361 4362 for_each_populated_zone(zone) { 4363 if (skip_free_areas_node(filter, zone_to_nid(zone))) 4364 continue; 4365 4366 for_each_online_cpu(cpu) 4367 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 4368 } 4369 4370 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 4371 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 4372 " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n" 4373 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 4374 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 4375 " free:%lu free_pcp:%lu free_cma:%lu\n", 4376 global_page_state(NR_ACTIVE_ANON), 4377 global_page_state(NR_INACTIVE_ANON), 4378 global_page_state(NR_ISOLATED_ANON), 4379 global_page_state(NR_ACTIVE_FILE), 4380 global_page_state(NR_INACTIVE_FILE), 4381 global_page_state(NR_ISOLATED_FILE), 4382 global_page_state(NR_UNEVICTABLE), 4383 global_page_state(NR_FILE_DIRTY), 4384 global_page_state(NR_WRITEBACK), 4385 global_page_state(NR_UNSTABLE_NFS), 4386 global_page_state(NR_SLAB_RECLAIMABLE), 4387 global_page_state(NR_SLAB_UNRECLAIMABLE), 4388 global_page_state(NR_FILE_MAPPED), 4389 global_page_state(NR_SHMEM), 4390 global_page_state(NR_PAGETABLE), 4391 global_page_state(NR_BOUNCE), 4392 global_page_state(NR_FREE_PAGES), 4393 free_pcp, 4394 global_page_state(NR_FREE_CMA_PAGES)); 4395 4396 for_each_populated_zone(zone) { 4397 int i; 4398 4399 if (skip_free_areas_node(filter, zone_to_nid(zone))) 4400 continue; 4401 4402 free_pcp = 0; 4403 for_each_online_cpu(cpu) 4404 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; 4405 4406 show_node(zone); 4407 printk("%s" 4408 " free:%lukB" 4409 " min:%lukB" 4410 " low:%lukB" 4411 " high:%lukB" 4412 " active_anon:%lukB" 4413 " inactive_anon:%lukB" 4414 " active_file:%lukB" 4415 " inactive_file:%lukB" 4416 " unevictable:%lukB" 4417 " isolated(anon):%lukB" 4418 " isolated(file):%lukB" 4419 " present:%lukB" 4420 " managed:%lukB" 4421 " mlocked:%lukB" 4422 " dirty:%lukB" 4423 " writeback:%lukB" 4424 " mapped:%lukB" 4425 " shmem:%lukB" 4426 " slab_reclaimable:%lukB" 4427 " slab_unreclaimable:%lukB" 4428 " kernel_stack:%lukB" 4429 " pagetables:%lukB" 4430 " unstable:%lukB" 4431 " bounce:%lukB" 4432 " free_pcp:%lukB" 4433 " local_pcp:%ukB" 4434 " free_cma:%lukB" 4435 " writeback_tmp:%lukB" 4436 " pages_scanned:%lu" 4437 " all_unreclaimable? %s" 4438 "\n", 4439 zone->name, 4440 K(zone_page_state(zone, NR_FREE_PAGES)), 4441 K(min_wmark_pages(zone)), 4442 K(low_wmark_pages(zone)), 4443 K(high_wmark_pages(zone)), 4444 K(zone_page_state(zone, NR_ACTIVE_ANON)), 4445 K(zone_page_state(zone, NR_INACTIVE_ANON)), 4446 K(zone_page_state(zone, NR_ACTIVE_FILE)), 4447 K(zone_page_state(zone, NR_INACTIVE_FILE)), 4448 K(zone_page_state(zone, NR_UNEVICTABLE)), 4449 K(zone_page_state(zone, NR_ISOLATED_ANON)), 4450 K(zone_page_state(zone, NR_ISOLATED_FILE)), 4451 K(zone->present_pages), 4452 K(zone->managed_pages), 4453 K(zone_page_state(zone, NR_MLOCK)), 4454 K(zone_page_state(zone, NR_FILE_DIRTY)), 4455 K(zone_page_state(zone, NR_WRITEBACK)), 4456 K(zone_page_state(zone, NR_FILE_MAPPED)), 4457 K(zone_page_state(zone, NR_SHMEM)), 4458 K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)), 4459 K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)), 4460 zone_page_state(zone, NR_KERNEL_STACK) * 4461 THREAD_SIZE / 1024, 4462 K(zone_page_state(zone, NR_PAGETABLE)), 4463 K(zone_page_state(zone, NR_UNSTABLE_NFS)), 4464 K(zone_page_state(zone, NR_BOUNCE)), 4465 K(free_pcp), 4466 K(this_cpu_read(zone->pageset->pcp.count)), 4467 K(zone_page_state(zone, NR_FREE_CMA_PAGES)), 4468 K(zone_page_state(zone, NR_WRITEBACK_TEMP)), 4469 K(zone_page_state(zone, NR_PAGES_SCANNED)), 4470 (!zone_reclaimable(zone) ? "yes" : "no") 4471 ); 4472 printk("lowmem_reserve[]:"); 4473 for (i = 0; i < MAX_NR_ZONES; i++) 4474 printk(" %ld", zone->lowmem_reserve[i]); 4475 printk("\n"); 4476 } 4477 4478 for_each_populated_zone(zone) { 4479 unsigned int order; 4480 unsigned long nr[MAX_ORDER], flags, total = 0; 4481 unsigned char types[MAX_ORDER]; 4482 4483 if (skip_free_areas_node(filter, zone_to_nid(zone))) 4484 continue; 4485 show_node(zone); 4486 printk("%s: ", zone->name); 4487 4488 spin_lock_irqsave(&zone->lock, flags); 4489 for (order = 0; order < MAX_ORDER; order++) { 4490 struct free_area *area = &zone->free_area[order]; 4491 int type; 4492 4493 nr[order] = area->nr_free; 4494 total += nr[order] << order; 4495 4496 types[order] = 0; 4497 for (type = 0; type < MIGRATE_TYPES; type++) { 4498 if (!list_empty(&area->free_list[type])) 4499 types[order] |= 1 << type; 4500 } 4501 } 4502 spin_unlock_irqrestore(&zone->lock, flags); 4503 for (order = 0; order < MAX_ORDER; order++) { 4504 printk("%lu*%lukB ", nr[order], K(1UL) << order); 4505 if (nr[order]) 4506 show_migration_types(types[order]); 4507 } 4508 printk("= %lukB\n", K(total)); 4509 } 4510 4511 hugetlb_show_meminfo(); 4512 4513 printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES)); 4514 4515 show_swap_cache_info(); 4516 } 4517 4518 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 4519 { 4520 zoneref->zone = zone; 4521 zoneref->zone_idx = zone_idx(zone); 4522 } 4523 4524 /* 4525 * Builds allocation fallback zone lists. 4526 * 4527 * Add all populated zones of a node to the zonelist. 4528 */ 4529 static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist, 4530 int nr_zones) 4531 { 4532 struct zone *zone; 4533 enum zone_type zone_type = MAX_NR_ZONES; 4534 4535 do { 4536 zone_type--; 4537 zone = pgdat->node_zones + zone_type; 4538 if (populated_zone(zone)) { 4539 zoneref_set_zone(zone, 4540 &zonelist->_zonerefs[nr_zones++]); 4541 check_highest_zone(zone_type); 4542 } 4543 } while (zone_type); 4544 4545 return nr_zones; 4546 } 4547 4548 4549 /* 4550 * zonelist_order: 4551 * 0 = automatic detection of better ordering. 4552 * 1 = order by ([node] distance, -zonetype) 4553 * 2 = order by (-zonetype, [node] distance) 4554 * 4555 * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create 4556 * the same zonelist. So only NUMA can configure this param. 4557 */ 4558 #define ZONELIST_ORDER_DEFAULT 0 4559 #define ZONELIST_ORDER_NODE 1 4560 #define ZONELIST_ORDER_ZONE 2 4561 4562 /* zonelist order in the kernel. 4563 * set_zonelist_order() will set this to NODE or ZONE. 4564 */ 4565 static int current_zonelist_order = ZONELIST_ORDER_DEFAULT; 4566 static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"}; 4567 4568 4569 #ifdef CONFIG_NUMA 4570 /* The value user specified ....changed by config */ 4571 static int user_zonelist_order = ZONELIST_ORDER_DEFAULT; 4572 /* string for sysctl */ 4573 #define NUMA_ZONELIST_ORDER_LEN 16 4574 char numa_zonelist_order[16] = "default"; 4575 4576 /* 4577 * interface for configure zonelist ordering. 4578 * command line option "numa_zonelist_order" 4579 * = "[dD]efault - default, automatic configuration. 4580 * = "[nN]ode - order by node locality, then by zone within node 4581 * = "[zZ]one - order by zone, then by locality within zone 4582 */ 4583 4584 static int __parse_numa_zonelist_order(char *s) 4585 { 4586 if (*s == 'd' || *s == 'D') { 4587 user_zonelist_order = ZONELIST_ORDER_DEFAULT; 4588 } else if (*s == 'n' || *s == 'N') { 4589 user_zonelist_order = ZONELIST_ORDER_NODE; 4590 } else if (*s == 'z' || *s == 'Z') { 4591 user_zonelist_order = ZONELIST_ORDER_ZONE; 4592 } else { 4593 pr_warn("Ignoring invalid numa_zonelist_order value: %s\n", s); 4594 return -EINVAL; 4595 } 4596 return 0; 4597 } 4598 4599 static __init int setup_numa_zonelist_order(char *s) 4600 { 4601 int ret; 4602 4603 if (!s) 4604 return 0; 4605 4606 ret = __parse_numa_zonelist_order(s); 4607 if (ret == 0) 4608 strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN); 4609 4610 return ret; 4611 } 4612 early_param("numa_zonelist_order", setup_numa_zonelist_order); 4613 4614 /* 4615 * sysctl handler for numa_zonelist_order 4616 */ 4617 int numa_zonelist_order_handler(struct ctl_table *table, int write, 4618 void __user *buffer, size_t *length, 4619 loff_t *ppos) 4620 { 4621 char saved_string[NUMA_ZONELIST_ORDER_LEN]; 4622 int ret; 4623 static DEFINE_MUTEX(zl_order_mutex); 4624 4625 mutex_lock(&zl_order_mutex); 4626 if (write) { 4627 if (strlen((char *)table->data) >= NUMA_ZONELIST_ORDER_LEN) { 4628 ret = -EINVAL; 4629 goto out; 4630 } 4631 strcpy(saved_string, (char *)table->data); 4632 } 4633 ret = proc_dostring(table, write, buffer, length, ppos); 4634 if (ret) 4635 goto out; 4636 if (write) { 4637 int oldval = user_zonelist_order; 4638 4639 ret = __parse_numa_zonelist_order((char *)table->data); 4640 if (ret) { 4641 /* 4642 * bogus value. restore saved string 4643 */ 4644 strncpy((char *)table->data, saved_string, 4645 NUMA_ZONELIST_ORDER_LEN); 4646 user_zonelist_order = oldval; 4647 } else if (oldval != user_zonelist_order) { 4648 mutex_lock(&zonelists_mutex); 4649 build_all_zonelists(NULL, NULL); 4650 mutex_unlock(&zonelists_mutex); 4651 } 4652 } 4653 out: 4654 mutex_unlock(&zl_order_mutex); 4655 return ret; 4656 } 4657 4658 4659 #define MAX_NODE_LOAD (nr_online_nodes) 4660 static int node_load[MAX_NUMNODES]; 4661 4662 /** 4663 * find_next_best_node - find the next node that should appear in a given node's fallback list 4664 * @node: node whose fallback list we're appending 4665 * @used_node_mask: nodemask_t of already used nodes 4666 * 4667 * We use a number of factors to determine which is the next node that should 4668 * appear on a given node's fallback list. The node should not have appeared 4669 * already in @node's fallback list, and it should be the next closest node 4670 * according to the distance array (which contains arbitrary distance values 4671 * from each node to each node in the system), and should also prefer nodes 4672 * with no CPUs, since presumably they'll have very little allocation pressure 4673 * on them otherwise. 4674 * It returns -1 if no node is found. 4675 */ 4676 static int find_next_best_node(int node, nodemask_t *used_node_mask) 4677 { 4678 int n, val; 4679 int min_val = INT_MAX; 4680 int best_node = NUMA_NO_NODE; 4681 const struct cpumask *tmp = cpumask_of_node(0); 4682 4683 /* Use the local node if we haven't already */ 4684 if (!node_isset(node, *used_node_mask)) { 4685 node_set(node, *used_node_mask); 4686 return node; 4687 } 4688 4689 for_each_node_state(n, N_MEMORY) { 4690 4691 /* Don't want a node to appear more than once */ 4692 if (node_isset(n, *used_node_mask)) 4693 continue; 4694 4695 /* Use the distance array to find the distance */ 4696 val = node_distance(node, n); 4697 4698 /* Penalize nodes under us ("prefer the next node") */ 4699 val += (n < node); 4700 4701 /* Give preference to headless and unused nodes */ 4702 tmp = cpumask_of_node(n); 4703 if (!cpumask_empty(tmp)) 4704 val += PENALTY_FOR_NODE_WITH_CPUS; 4705 4706 /* Slight preference for less loaded node */ 4707 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 4708 val += node_load[n]; 4709 4710 if (val < min_val) { 4711 min_val = val; 4712 best_node = n; 4713 } 4714 } 4715 4716 if (best_node >= 0) 4717 node_set(best_node, *used_node_mask); 4718 4719 return best_node; 4720 } 4721 4722 4723 /* 4724 * Build zonelists ordered by node and zones within node. 4725 * This results in maximum locality--normal zone overflows into local 4726 * DMA zone, if any--but risks exhausting DMA zone. 4727 */ 4728 static void build_zonelists_in_node_order(pg_data_t *pgdat, int node) 4729 { 4730 int j; 4731 struct zonelist *zonelist; 4732 4733 zonelist = &pgdat->node_zonelists[0]; 4734 for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++) 4735 ; 4736 j = build_zonelists_node(NODE_DATA(node), zonelist, j); 4737 zonelist->_zonerefs[j].zone = NULL; 4738 zonelist->_zonerefs[j].zone_idx = 0; 4739 } 4740 4741 /* 4742 * Build gfp_thisnode zonelists 4743 */ 4744 static void build_thisnode_zonelists(pg_data_t *pgdat) 4745 { 4746 int j; 4747 struct zonelist *zonelist; 4748 4749 zonelist = &pgdat->node_zonelists[1]; 4750 j = build_zonelists_node(pgdat, zonelist, 0); 4751 zonelist->_zonerefs[j].zone = NULL; 4752 zonelist->_zonerefs[j].zone_idx = 0; 4753 } 4754 4755 /* 4756 * Build zonelists ordered by zone and nodes within zones. 4757 * This results in conserving DMA zone[s] until all Normal memory is 4758 * exhausted, but results in overflowing to remote node while memory 4759 * may still exist in local DMA zone. 4760 */ 4761 static int node_order[MAX_NUMNODES]; 4762 4763 static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes) 4764 { 4765 int pos, j, node; 4766 int zone_type; /* needs to be signed */ 4767 struct zone *z; 4768 struct zonelist *zonelist; 4769 4770 zonelist = &pgdat->node_zonelists[0]; 4771 pos = 0; 4772 for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) { 4773 for (j = 0; j < nr_nodes; j++) { 4774 node = node_order[j]; 4775 z = &NODE_DATA(node)->node_zones[zone_type]; 4776 if (populated_zone(z)) { 4777 zoneref_set_zone(z, 4778 &zonelist->_zonerefs[pos++]); 4779 check_highest_zone(zone_type); 4780 } 4781 } 4782 } 4783 zonelist->_zonerefs[pos].zone = NULL; 4784 zonelist->_zonerefs[pos].zone_idx = 0; 4785 } 4786 4787 #if defined(CONFIG_64BIT) 4788 /* 4789 * Devices that require DMA32/DMA are relatively rare and do not justify a 4790 * penalty to every machine in case the specialised case applies. Default 4791 * to Node-ordering on 64-bit NUMA machines 4792 */ 4793 static int default_zonelist_order(void) 4794 { 4795 return ZONELIST_ORDER_NODE; 4796 } 4797 #else 4798 /* 4799 * On 32-bit, the Normal zone needs to be preserved for allocations accessible 4800 * by the kernel. If processes running on node 0 deplete the low memory zone 4801 * then reclaim will occur more frequency increasing stalls and potentially 4802 * be easier to OOM if a large percentage of the zone is under writeback or 4803 * dirty. The problem is significantly worse if CONFIG_HIGHPTE is not set. 4804 * Hence, default to zone ordering on 32-bit. 4805 */ 4806 static int default_zonelist_order(void) 4807 { 4808 return ZONELIST_ORDER_ZONE; 4809 } 4810 #endif /* CONFIG_64BIT */ 4811 4812 static void set_zonelist_order(void) 4813 { 4814 if (user_zonelist_order == ZONELIST_ORDER_DEFAULT) 4815 current_zonelist_order = default_zonelist_order(); 4816 else 4817 current_zonelist_order = user_zonelist_order; 4818 } 4819 4820 static void build_zonelists(pg_data_t *pgdat) 4821 { 4822 int i, node, load; 4823 nodemask_t used_mask; 4824 int local_node, prev_node; 4825 struct zonelist *zonelist; 4826 unsigned int order = current_zonelist_order; 4827 4828 /* initialize zonelists */ 4829 for (i = 0; i < MAX_ZONELISTS; i++) { 4830 zonelist = pgdat->node_zonelists + i; 4831 zonelist->_zonerefs[0].zone = NULL; 4832 zonelist->_zonerefs[0].zone_idx = 0; 4833 } 4834 4835 /* NUMA-aware ordering of nodes */ 4836 local_node = pgdat->node_id; 4837 load = nr_online_nodes; 4838 prev_node = local_node; 4839 nodes_clear(used_mask); 4840 4841 memset(node_order, 0, sizeof(node_order)); 4842 i = 0; 4843 4844 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 4845 /* 4846 * We don't want to pressure a particular node. 4847 * So adding penalty to the first node in same 4848 * distance group to make it round-robin. 4849 */ 4850 if (node_distance(local_node, node) != 4851 node_distance(local_node, prev_node)) 4852 node_load[node] = load; 4853 4854 prev_node = node; 4855 load--; 4856 if (order == ZONELIST_ORDER_NODE) 4857 build_zonelists_in_node_order(pgdat, node); 4858 else 4859 node_order[i++] = node; /* remember order */ 4860 } 4861 4862 if (order == ZONELIST_ORDER_ZONE) { 4863 /* calculate node order -- i.e., DMA last! */ 4864 build_zonelists_in_zone_order(pgdat, i); 4865 } 4866 4867 build_thisnode_zonelists(pgdat); 4868 } 4869 4870 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 4871 /* 4872 * Return node id of node used for "local" allocations. 4873 * I.e., first node id of first zone in arg node's generic zonelist. 4874 * Used for initializing percpu 'numa_mem', which is used primarily 4875 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 4876 */ 4877 int local_memory_node(int node) 4878 { 4879 struct zoneref *z; 4880 4881 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 4882 gfp_zone(GFP_KERNEL), 4883 NULL); 4884 return z->zone->node; 4885 } 4886 #endif 4887 4888 #else /* CONFIG_NUMA */ 4889 4890 static void set_zonelist_order(void) 4891 { 4892 current_zonelist_order = ZONELIST_ORDER_ZONE; 4893 } 4894 4895 static void build_zonelists(pg_data_t *pgdat) 4896 { 4897 int node, local_node; 4898 enum zone_type j; 4899 struct zonelist *zonelist; 4900 4901 local_node = pgdat->node_id; 4902 4903 zonelist = &pgdat->node_zonelists[0]; 4904 j = build_zonelists_node(pgdat, zonelist, 0); 4905 4906 /* 4907 * Now we build the zonelist so that it contains the zones 4908 * of all the other nodes. 4909 * We don't want to pressure a particular node, so when 4910 * building the zones for node N, we make sure that the 4911 * zones coming right after the local ones are those from 4912 * node N+1 (modulo N) 4913 */ 4914 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 4915 if (!node_online(node)) 4916 continue; 4917 j = build_zonelists_node(NODE_DATA(node), zonelist, j); 4918 } 4919 for (node = 0; node < local_node; node++) { 4920 if (!node_online(node)) 4921 continue; 4922 j = build_zonelists_node(NODE_DATA(node), zonelist, j); 4923 } 4924 4925 zonelist->_zonerefs[j].zone = NULL; 4926 zonelist->_zonerefs[j].zone_idx = 0; 4927 } 4928 4929 #endif /* CONFIG_NUMA */ 4930 4931 /* 4932 * Boot pageset table. One per cpu which is going to be used for all 4933 * zones and all nodes. The parameters will be set in such a way 4934 * that an item put on a list will immediately be handed over to 4935 * the buddy list. This is safe since pageset manipulation is done 4936 * with interrupts disabled. 4937 * 4938 * The boot_pagesets must be kept even after bootup is complete for 4939 * unused processors and/or zones. They do play a role for bootstrapping 4940 * hotplugged processors. 4941 * 4942 * zoneinfo_show() and maybe other functions do 4943 * not check if the processor is online before following the pageset pointer. 4944 * Other parts of the kernel may not check if the zone is available. 4945 */ 4946 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); 4947 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); 4948 static void setup_zone_pageset(struct zone *zone); 4949 4950 /* 4951 * Global mutex to protect against size modification of zonelists 4952 * as well as to serialize pageset setup for the new populated zone. 4953 */ 4954 DEFINE_MUTEX(zonelists_mutex); 4955 4956 /* return values int ....just for stop_machine() */ 4957 static int __build_all_zonelists(void *data) 4958 { 4959 int nid; 4960 int cpu; 4961 pg_data_t *self = data; 4962 4963 #ifdef CONFIG_NUMA 4964 memset(node_load, 0, sizeof(node_load)); 4965 #endif 4966 4967 if (self && !node_online(self->node_id)) { 4968 build_zonelists(self); 4969 } 4970 4971 for_each_online_node(nid) { 4972 pg_data_t *pgdat = NODE_DATA(nid); 4973 4974 build_zonelists(pgdat); 4975 } 4976 4977 /* 4978 * Initialize the boot_pagesets that are going to be used 4979 * for bootstrapping processors. The real pagesets for 4980 * each zone will be allocated later when the per cpu 4981 * allocator is available. 4982 * 4983 * boot_pagesets are used also for bootstrapping offline 4984 * cpus if the system is already booted because the pagesets 4985 * are needed to initialize allocators on a specific cpu too. 4986 * F.e. the percpu allocator needs the page allocator which 4987 * needs the percpu allocator in order to allocate its pagesets 4988 * (a chicken-egg dilemma). 4989 */ 4990 for_each_possible_cpu(cpu) { 4991 setup_pageset(&per_cpu(boot_pageset, cpu), 0); 4992 4993 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 4994 /* 4995 * We now know the "local memory node" for each node-- 4996 * i.e., the node of the first zone in the generic zonelist. 4997 * Set up numa_mem percpu variable for on-line cpus. During 4998 * boot, only the boot cpu should be on-line; we'll init the 4999 * secondary cpus' numa_mem as they come on-line. During 5000 * node/memory hotplug, we'll fixup all on-line cpus. 5001 */ 5002 if (cpu_online(cpu)) 5003 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 5004 #endif 5005 } 5006 5007 return 0; 5008 } 5009 5010 static noinline void __init 5011 build_all_zonelists_init(void) 5012 { 5013 __build_all_zonelists(NULL); 5014 mminit_verify_zonelist(); 5015 cpuset_init_current_mems_allowed(); 5016 } 5017 5018 /* 5019 * Called with zonelists_mutex held always 5020 * unless system_state == SYSTEM_BOOTING. 5021 * 5022 * __ref due to (1) call of __meminit annotated setup_zone_pageset 5023 * [we're only called with non-NULL zone through __meminit paths] and 5024 * (2) call of __init annotated helper build_all_zonelists_init 5025 * [protected by SYSTEM_BOOTING]. 5026 */ 5027 void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone) 5028 { 5029 set_zonelist_order(); 5030 5031 if (system_state == SYSTEM_BOOTING) { 5032 build_all_zonelists_init(); 5033 } else { 5034 #ifdef CONFIG_MEMORY_HOTPLUG 5035 if (zone) 5036 setup_zone_pageset(zone); 5037 #endif 5038 /* we have to stop all cpus to guarantee there is no user 5039 of zonelist */ 5040 stop_machine(__build_all_zonelists, pgdat, NULL); 5041 /* cpuset refresh routine should be here */ 5042 } 5043 vm_total_pages = nr_free_pagecache_pages(); 5044 /* 5045 * Disable grouping by mobility if the number of pages in the 5046 * system is too low to allow the mechanism to work. It would be 5047 * more accurate, but expensive to check per-zone. This check is 5048 * made on memory-hotadd so a system can start with mobility 5049 * disabled and enable it later 5050 */ 5051 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 5052 page_group_by_mobility_disabled = 1; 5053 else 5054 page_group_by_mobility_disabled = 0; 5055 5056 pr_info("Built %i zonelists in %s order, mobility grouping %s. Total pages: %ld\n", 5057 nr_online_nodes, 5058 zonelist_order_name[current_zonelist_order], 5059 page_group_by_mobility_disabled ? "off" : "on", 5060 vm_total_pages); 5061 #ifdef CONFIG_NUMA 5062 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 5063 #endif 5064 } 5065 5066 /* 5067 * Helper functions to size the waitqueue hash table. 5068 * Essentially these want to choose hash table sizes sufficiently 5069 * large so that collisions trying to wait on pages are rare. 5070 * But in fact, the number of active page waitqueues on typical 5071 * systems is ridiculously low, less than 200. So this is even 5072 * conservative, even though it seems large. 5073 * 5074 * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to 5075 * waitqueues, i.e. the size of the waitq table given the number of pages. 5076 */ 5077 #define PAGES_PER_WAITQUEUE 256 5078 5079 #ifndef CONFIG_MEMORY_HOTPLUG 5080 static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) 5081 { 5082 unsigned long size = 1; 5083 5084 pages /= PAGES_PER_WAITQUEUE; 5085 5086 while (size < pages) 5087 size <<= 1; 5088 5089 /* 5090 * Once we have dozens or even hundreds of threads sleeping 5091 * on IO we've got bigger problems than wait queue collision. 5092 * Limit the size of the wait table to a reasonable size. 5093 */ 5094 size = min(size, 4096UL); 5095 5096 return max(size, 4UL); 5097 } 5098 #else 5099 /* 5100 * A zone's size might be changed by hot-add, so it is not possible to determine 5101 * a suitable size for its wait_table. So we use the maximum size now. 5102 * 5103 * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie: 5104 * 5105 * i386 (preemption config) : 4096 x 16 = 64Kbyte. 5106 * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte. 5107 * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte. 5108 * 5109 * The maximum entries are prepared when a zone's memory is (512K + 256) pages 5110 * or more by the traditional way. (See above). It equals: 5111 * 5112 * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte. 5113 * ia64(16K page size) : = ( 8G + 4M)byte. 5114 * powerpc (64K page size) : = (32G +16M)byte. 5115 */ 5116 static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) 5117 { 5118 return 4096UL; 5119 } 5120 #endif 5121 5122 /* 5123 * This is an integer logarithm so that shifts can be used later 5124 * to extract the more random high bits from the multiplicative 5125 * hash function before the remainder is taken. 5126 */ 5127 static inline unsigned long wait_table_bits(unsigned long size) 5128 { 5129 return ffz(~size); 5130 } 5131 5132 /* 5133 * Initially all pages are reserved - free ones are freed 5134 * up by free_all_bootmem() once the early boot process is 5135 * done. Non-atomic initialization, single-pass. 5136 */ 5137 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, 5138 unsigned long start_pfn, enum memmap_context context) 5139 { 5140 struct vmem_altmap *altmap = to_vmem_altmap(__pfn_to_phys(start_pfn)); 5141 unsigned long end_pfn = start_pfn + size; 5142 pg_data_t *pgdat = NODE_DATA(nid); 5143 unsigned long pfn; 5144 unsigned long nr_initialised = 0; 5145 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5146 struct memblock_region *r = NULL, *tmp; 5147 #endif 5148 5149 if (highest_memmap_pfn < end_pfn - 1) 5150 highest_memmap_pfn = end_pfn - 1; 5151 5152 /* 5153 * Honor reservation requested by the driver for this ZONE_DEVICE 5154 * memory 5155 */ 5156 if (altmap && start_pfn == altmap->base_pfn) 5157 start_pfn += altmap->reserve; 5158 5159 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 5160 /* 5161 * There can be holes in boot-time mem_map[]s handed to this 5162 * function. They do not exist on hotplugged memory. 5163 */ 5164 if (context != MEMMAP_EARLY) 5165 goto not_early; 5166 5167 if (!early_pfn_valid(pfn)) 5168 continue; 5169 if (!early_pfn_in_nid(pfn, nid)) 5170 continue; 5171 if (!update_defer_init(pgdat, pfn, end_pfn, &nr_initialised)) 5172 break; 5173 5174 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5175 /* 5176 * If not mirrored_kernelcore and ZONE_MOVABLE exists, range 5177 * from zone_movable_pfn[nid] to end of each node should be 5178 * ZONE_MOVABLE not ZONE_NORMAL. skip it. 5179 */ 5180 if (!mirrored_kernelcore && zone_movable_pfn[nid]) 5181 if (zone == ZONE_NORMAL && pfn >= zone_movable_pfn[nid]) 5182 continue; 5183 5184 /* 5185 * Check given memblock attribute by firmware which can affect 5186 * kernel memory layout. If zone==ZONE_MOVABLE but memory is 5187 * mirrored, it's an overlapped memmap init. skip it. 5188 */ 5189 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 5190 if (!r || pfn >= memblock_region_memory_end_pfn(r)) { 5191 for_each_memblock(memory, tmp) 5192 if (pfn < memblock_region_memory_end_pfn(tmp)) 5193 break; 5194 r = tmp; 5195 } 5196 if (pfn >= memblock_region_memory_base_pfn(r) && 5197 memblock_is_mirror(r)) { 5198 /* already initialized as NORMAL */ 5199 pfn = memblock_region_memory_end_pfn(r); 5200 continue; 5201 } 5202 } 5203 #endif 5204 5205 not_early: 5206 /* 5207 * Mark the block movable so that blocks are reserved for 5208 * movable at startup. This will force kernel allocations 5209 * to reserve their blocks rather than leaking throughout 5210 * the address space during boot when many long-lived 5211 * kernel allocations are made. 5212 * 5213 * bitmap is created for zone's valid pfn range. but memmap 5214 * can be created for invalid pages (for alignment) 5215 * check here not to call set_pageblock_migratetype() against 5216 * pfn out of zone. 5217 */ 5218 if (!(pfn & (pageblock_nr_pages - 1))) { 5219 struct page *page = pfn_to_page(pfn); 5220 5221 __init_single_page(page, pfn, zone, nid); 5222 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 5223 } else { 5224 __init_single_pfn(pfn, zone, nid); 5225 } 5226 } 5227 } 5228 5229 static void __meminit zone_init_free_lists(struct zone *zone) 5230 { 5231 unsigned int order, t; 5232 for_each_migratetype_order(order, t) { 5233 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 5234 zone->free_area[order].nr_free = 0; 5235 } 5236 } 5237 5238 #ifndef __HAVE_ARCH_MEMMAP_INIT 5239 #define memmap_init(size, nid, zone, start_pfn) \ 5240 memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY) 5241 #endif 5242 5243 static int zone_batchsize(struct zone *zone) 5244 { 5245 #ifdef CONFIG_MMU 5246 int batch; 5247 5248 /* 5249 * The per-cpu-pages pools are set to around 1000th of the 5250 * size of the zone. But no more than 1/2 of a meg. 5251 * 5252 * OK, so we don't know how big the cache is. So guess. 5253 */ 5254 batch = zone->managed_pages / 1024; 5255 if (batch * PAGE_SIZE > 512 * 1024) 5256 batch = (512 * 1024) / PAGE_SIZE; 5257 batch /= 4; /* We effectively *= 4 below */ 5258 if (batch < 1) 5259 batch = 1; 5260 5261 /* 5262 * Clamp the batch to a 2^n - 1 value. Having a power 5263 * of 2 value was found to be more likely to have 5264 * suboptimal cache aliasing properties in some cases. 5265 * 5266 * For example if 2 tasks are alternately allocating 5267 * batches of pages, one task can end up with a lot 5268 * of pages of one half of the possible page colors 5269 * and the other with pages of the other colors. 5270 */ 5271 batch = rounddown_pow_of_two(batch + batch/2) - 1; 5272 5273 return batch; 5274 5275 #else 5276 /* The deferral and batching of frees should be suppressed under NOMMU 5277 * conditions. 5278 * 5279 * The problem is that NOMMU needs to be able to allocate large chunks 5280 * of contiguous memory as there's no hardware page translation to 5281 * assemble apparent contiguous memory from discontiguous pages. 5282 * 5283 * Queueing large contiguous runs of pages for batching, however, 5284 * causes the pages to actually be freed in smaller chunks. As there 5285 * can be a significant delay between the individual batches being 5286 * recycled, this leads to the once large chunks of space being 5287 * fragmented and becoming unavailable for high-order allocations. 5288 */ 5289 return 0; 5290 #endif 5291 } 5292 5293 /* 5294 * pcp->high and pcp->batch values are related and dependent on one another: 5295 * ->batch must never be higher then ->high. 5296 * The following function updates them in a safe manner without read side 5297 * locking. 5298 * 5299 * Any new users of pcp->batch and pcp->high should ensure they can cope with 5300 * those fields changing asynchronously (acording the the above rule). 5301 * 5302 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 5303 * outside of boot time (or some other assurance that no concurrent updaters 5304 * exist). 5305 */ 5306 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 5307 unsigned long batch) 5308 { 5309 /* start with a fail safe value for batch */ 5310 pcp->batch = 1; 5311 smp_wmb(); 5312 5313 /* Update high, then batch, in order */ 5314 pcp->high = high; 5315 smp_wmb(); 5316 5317 pcp->batch = batch; 5318 } 5319 5320 /* a companion to pageset_set_high() */ 5321 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) 5322 { 5323 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); 5324 } 5325 5326 static void pageset_init(struct per_cpu_pageset *p) 5327 { 5328 struct per_cpu_pages *pcp; 5329 int migratetype; 5330 5331 memset(p, 0, sizeof(*p)); 5332 5333 pcp = &p->pcp; 5334 pcp->count = 0; 5335 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) 5336 INIT_LIST_HEAD(&pcp->lists[migratetype]); 5337 } 5338 5339 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) 5340 { 5341 pageset_init(p); 5342 pageset_set_batch(p, batch); 5343 } 5344 5345 /* 5346 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist 5347 * to the value high for the pageset p. 5348 */ 5349 static void pageset_set_high(struct per_cpu_pageset *p, 5350 unsigned long high) 5351 { 5352 unsigned long batch = max(1UL, high / 4); 5353 if ((high / 4) > (PAGE_SHIFT * 8)) 5354 batch = PAGE_SHIFT * 8; 5355 5356 pageset_update(&p->pcp, high, batch); 5357 } 5358 5359 static void pageset_set_high_and_batch(struct zone *zone, 5360 struct per_cpu_pageset *pcp) 5361 { 5362 if (percpu_pagelist_fraction) 5363 pageset_set_high(pcp, 5364 (zone->managed_pages / 5365 percpu_pagelist_fraction)); 5366 else 5367 pageset_set_batch(pcp, zone_batchsize(zone)); 5368 } 5369 5370 static void __meminit zone_pageset_init(struct zone *zone, int cpu) 5371 { 5372 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); 5373 5374 pageset_init(pcp); 5375 pageset_set_high_and_batch(zone, pcp); 5376 } 5377 5378 static void __meminit setup_zone_pageset(struct zone *zone) 5379 { 5380 int cpu; 5381 zone->pageset = alloc_percpu(struct per_cpu_pageset); 5382 for_each_possible_cpu(cpu) 5383 zone_pageset_init(zone, cpu); 5384 } 5385 5386 /* 5387 * Allocate per cpu pagesets and initialize them. 5388 * Before this call only boot pagesets were available. 5389 */ 5390 void __init setup_per_cpu_pageset(void) 5391 { 5392 struct zone *zone; 5393 5394 for_each_populated_zone(zone) 5395 setup_zone_pageset(zone); 5396 } 5397 5398 static noinline __init_refok 5399 int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) 5400 { 5401 int i; 5402 size_t alloc_size; 5403 5404 /* 5405 * The per-page waitqueue mechanism uses hashed waitqueues 5406 * per zone. 5407 */ 5408 zone->wait_table_hash_nr_entries = 5409 wait_table_hash_nr_entries(zone_size_pages); 5410 zone->wait_table_bits = 5411 wait_table_bits(zone->wait_table_hash_nr_entries); 5412 alloc_size = zone->wait_table_hash_nr_entries 5413 * sizeof(wait_queue_head_t); 5414 5415 if (!slab_is_available()) { 5416 zone->wait_table = (wait_queue_head_t *) 5417 memblock_virt_alloc_node_nopanic( 5418 alloc_size, zone->zone_pgdat->node_id); 5419 } else { 5420 /* 5421 * This case means that a zone whose size was 0 gets new memory 5422 * via memory hot-add. 5423 * But it may be the case that a new node was hot-added. In 5424 * this case vmalloc() will not be able to use this new node's 5425 * memory - this wait_table must be initialized to use this new 5426 * node itself as well. 5427 * To use this new node's memory, further consideration will be 5428 * necessary. 5429 */ 5430 zone->wait_table = vmalloc(alloc_size); 5431 } 5432 if (!zone->wait_table) 5433 return -ENOMEM; 5434 5435 for (i = 0; i < zone->wait_table_hash_nr_entries; ++i) 5436 init_waitqueue_head(zone->wait_table + i); 5437 5438 return 0; 5439 } 5440 5441 static __meminit void zone_pcp_init(struct zone *zone) 5442 { 5443 /* 5444 * per cpu subsystem is not up at this point. The following code 5445 * relies on the ability of the linker to provide the 5446 * offset of a (static) per cpu variable into the per cpu area. 5447 */ 5448 zone->pageset = &boot_pageset; 5449 5450 if (populated_zone(zone)) 5451 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", 5452 zone->name, zone->present_pages, 5453 zone_batchsize(zone)); 5454 } 5455 5456 int __meminit init_currently_empty_zone(struct zone *zone, 5457 unsigned long zone_start_pfn, 5458 unsigned long size) 5459 { 5460 struct pglist_data *pgdat = zone->zone_pgdat; 5461 int ret; 5462 ret = zone_wait_table_init(zone, size); 5463 if (ret) 5464 return ret; 5465 pgdat->nr_zones = zone_idx(zone) + 1; 5466 5467 zone->zone_start_pfn = zone_start_pfn; 5468 5469 mminit_dprintk(MMINIT_TRACE, "memmap_init", 5470 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 5471 pgdat->node_id, 5472 (unsigned long)zone_idx(zone), 5473 zone_start_pfn, (zone_start_pfn + size)); 5474 5475 zone_init_free_lists(zone); 5476 5477 return 0; 5478 } 5479 5480 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 5481 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID 5482 5483 /* 5484 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 5485 */ 5486 int __meminit __early_pfn_to_nid(unsigned long pfn, 5487 struct mminit_pfnnid_cache *state) 5488 { 5489 unsigned long start_pfn, end_pfn; 5490 int nid; 5491 5492 if (state->last_start <= pfn && pfn < state->last_end) 5493 return state->last_nid; 5494 5495 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 5496 if (nid != -1) { 5497 state->last_start = start_pfn; 5498 state->last_end = end_pfn; 5499 state->last_nid = nid; 5500 } 5501 5502 return nid; 5503 } 5504 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ 5505 5506 /** 5507 * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range 5508 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed. 5509 * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid 5510 * 5511 * If an architecture guarantees that all ranges registered contain no holes 5512 * and may be freed, this this function may be used instead of calling 5513 * memblock_free_early_nid() manually. 5514 */ 5515 void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn) 5516 { 5517 unsigned long start_pfn, end_pfn; 5518 int i, this_nid; 5519 5520 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) { 5521 start_pfn = min(start_pfn, max_low_pfn); 5522 end_pfn = min(end_pfn, max_low_pfn); 5523 5524 if (start_pfn < end_pfn) 5525 memblock_free_early_nid(PFN_PHYS(start_pfn), 5526 (end_pfn - start_pfn) << PAGE_SHIFT, 5527 this_nid); 5528 } 5529 } 5530 5531 /** 5532 * sparse_memory_present_with_active_regions - Call memory_present for each active range 5533 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used. 5534 * 5535 * If an architecture guarantees that all ranges registered contain no holes and may 5536 * be freed, this function may be used instead of calling memory_present() manually. 5537 */ 5538 void __init sparse_memory_present_with_active_regions(int nid) 5539 { 5540 unsigned long start_pfn, end_pfn; 5541 int i, this_nid; 5542 5543 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) 5544 memory_present(this_nid, start_pfn, end_pfn); 5545 } 5546 5547 /** 5548 * get_pfn_range_for_nid - Return the start and end page frames for a node 5549 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 5550 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 5551 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 5552 * 5553 * It returns the start and end page frame of a node based on information 5554 * provided by memblock_set_node(). If called for a node 5555 * with no available memory, a warning is printed and the start and end 5556 * PFNs will be 0. 5557 */ 5558 void __meminit get_pfn_range_for_nid(unsigned int nid, 5559 unsigned long *start_pfn, unsigned long *end_pfn) 5560 { 5561 unsigned long this_start_pfn, this_end_pfn; 5562 int i; 5563 5564 *start_pfn = -1UL; 5565 *end_pfn = 0; 5566 5567 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 5568 *start_pfn = min(*start_pfn, this_start_pfn); 5569 *end_pfn = max(*end_pfn, this_end_pfn); 5570 } 5571 5572 if (*start_pfn == -1UL) 5573 *start_pfn = 0; 5574 } 5575 5576 /* 5577 * This finds a zone that can be used for ZONE_MOVABLE pages. The 5578 * assumption is made that zones within a node are ordered in monotonic 5579 * increasing memory addresses so that the "highest" populated zone is used 5580 */ 5581 static void __init find_usable_zone_for_movable(void) 5582 { 5583 int zone_index; 5584 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 5585 if (zone_index == ZONE_MOVABLE) 5586 continue; 5587 5588 if (arch_zone_highest_possible_pfn[zone_index] > 5589 arch_zone_lowest_possible_pfn[zone_index]) 5590 break; 5591 } 5592 5593 VM_BUG_ON(zone_index == -1); 5594 movable_zone = zone_index; 5595 } 5596 5597 /* 5598 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 5599 * because it is sized independent of architecture. Unlike the other zones, 5600 * the starting point for ZONE_MOVABLE is not fixed. It may be different 5601 * in each node depending on the size of each node and how evenly kernelcore 5602 * is distributed. This helper function adjusts the zone ranges 5603 * provided by the architecture for a given node by using the end of the 5604 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 5605 * zones within a node are in order of monotonic increases memory addresses 5606 */ 5607 static void __meminit adjust_zone_range_for_zone_movable(int nid, 5608 unsigned long zone_type, 5609 unsigned long node_start_pfn, 5610 unsigned long node_end_pfn, 5611 unsigned long *zone_start_pfn, 5612 unsigned long *zone_end_pfn) 5613 { 5614 /* Only adjust if ZONE_MOVABLE is on this node */ 5615 if (zone_movable_pfn[nid]) { 5616 /* Size ZONE_MOVABLE */ 5617 if (zone_type == ZONE_MOVABLE) { 5618 *zone_start_pfn = zone_movable_pfn[nid]; 5619 *zone_end_pfn = min(node_end_pfn, 5620 arch_zone_highest_possible_pfn[movable_zone]); 5621 5622 /* Check if this whole range is within ZONE_MOVABLE */ 5623 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 5624 *zone_start_pfn = *zone_end_pfn; 5625 } 5626 } 5627 5628 /* 5629 * Return the number of pages a zone spans in a node, including holes 5630 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 5631 */ 5632 static unsigned long __meminit zone_spanned_pages_in_node(int nid, 5633 unsigned long zone_type, 5634 unsigned long node_start_pfn, 5635 unsigned long node_end_pfn, 5636 unsigned long *zone_start_pfn, 5637 unsigned long *zone_end_pfn, 5638 unsigned long *ignored) 5639 { 5640 /* When hotadd a new node from cpu_up(), the node should be empty */ 5641 if (!node_start_pfn && !node_end_pfn) 5642 return 0; 5643 5644 /* Get the start and end of the zone */ 5645 *zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type]; 5646 *zone_end_pfn = arch_zone_highest_possible_pfn[zone_type]; 5647 adjust_zone_range_for_zone_movable(nid, zone_type, 5648 node_start_pfn, node_end_pfn, 5649 zone_start_pfn, zone_end_pfn); 5650 5651 /* Check that this node has pages within the zone's required range */ 5652 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 5653 return 0; 5654 5655 /* Move the zone boundaries inside the node if necessary */ 5656 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 5657 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 5658 5659 /* Return the spanned pages */ 5660 return *zone_end_pfn - *zone_start_pfn; 5661 } 5662 5663 /* 5664 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 5665 * then all holes in the requested range will be accounted for. 5666 */ 5667 unsigned long __meminit __absent_pages_in_range(int nid, 5668 unsigned long range_start_pfn, 5669 unsigned long range_end_pfn) 5670 { 5671 unsigned long nr_absent = range_end_pfn - range_start_pfn; 5672 unsigned long start_pfn, end_pfn; 5673 int i; 5674 5675 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 5676 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 5677 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 5678 nr_absent -= end_pfn - start_pfn; 5679 } 5680 return nr_absent; 5681 } 5682 5683 /** 5684 * absent_pages_in_range - Return number of page frames in holes within a range 5685 * @start_pfn: The start PFN to start searching for holes 5686 * @end_pfn: The end PFN to stop searching for holes 5687 * 5688 * It returns the number of pages frames in memory holes within a range. 5689 */ 5690 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 5691 unsigned long end_pfn) 5692 { 5693 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 5694 } 5695 5696 /* Return the number of page frames in holes in a zone on a node */ 5697 static unsigned long __meminit zone_absent_pages_in_node(int nid, 5698 unsigned long zone_type, 5699 unsigned long node_start_pfn, 5700 unsigned long node_end_pfn, 5701 unsigned long *ignored) 5702 { 5703 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 5704 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 5705 unsigned long zone_start_pfn, zone_end_pfn; 5706 unsigned long nr_absent; 5707 5708 /* When hotadd a new node from cpu_up(), the node should be empty */ 5709 if (!node_start_pfn && !node_end_pfn) 5710 return 0; 5711 5712 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 5713 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 5714 5715 adjust_zone_range_for_zone_movable(nid, zone_type, 5716 node_start_pfn, node_end_pfn, 5717 &zone_start_pfn, &zone_end_pfn); 5718 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 5719 5720 /* 5721 * ZONE_MOVABLE handling. 5722 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 5723 * and vice versa. 5724 */ 5725 if (zone_movable_pfn[nid]) { 5726 if (mirrored_kernelcore) { 5727 unsigned long start_pfn, end_pfn; 5728 struct memblock_region *r; 5729 5730 for_each_memblock(memory, r) { 5731 start_pfn = clamp(memblock_region_memory_base_pfn(r), 5732 zone_start_pfn, zone_end_pfn); 5733 end_pfn = clamp(memblock_region_memory_end_pfn(r), 5734 zone_start_pfn, zone_end_pfn); 5735 5736 if (zone_type == ZONE_MOVABLE && 5737 memblock_is_mirror(r)) 5738 nr_absent += end_pfn - start_pfn; 5739 5740 if (zone_type == ZONE_NORMAL && 5741 !memblock_is_mirror(r)) 5742 nr_absent += end_pfn - start_pfn; 5743 } 5744 } else { 5745 if (zone_type == ZONE_NORMAL) 5746 nr_absent += node_end_pfn - zone_movable_pfn[nid]; 5747 } 5748 } 5749 5750 return nr_absent; 5751 } 5752 5753 #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 5754 static inline unsigned long __meminit zone_spanned_pages_in_node(int nid, 5755 unsigned long zone_type, 5756 unsigned long node_start_pfn, 5757 unsigned long node_end_pfn, 5758 unsigned long *zone_start_pfn, 5759 unsigned long *zone_end_pfn, 5760 unsigned long *zones_size) 5761 { 5762 unsigned int zone; 5763 5764 *zone_start_pfn = node_start_pfn; 5765 for (zone = 0; zone < zone_type; zone++) 5766 *zone_start_pfn += zones_size[zone]; 5767 5768 *zone_end_pfn = *zone_start_pfn + zones_size[zone_type]; 5769 5770 return zones_size[zone_type]; 5771 } 5772 5773 static inline unsigned long __meminit zone_absent_pages_in_node(int nid, 5774 unsigned long zone_type, 5775 unsigned long node_start_pfn, 5776 unsigned long node_end_pfn, 5777 unsigned long *zholes_size) 5778 { 5779 if (!zholes_size) 5780 return 0; 5781 5782 return zholes_size[zone_type]; 5783 } 5784 5785 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 5786 5787 static void __meminit calculate_node_totalpages(struct pglist_data *pgdat, 5788 unsigned long node_start_pfn, 5789 unsigned long node_end_pfn, 5790 unsigned long *zones_size, 5791 unsigned long *zholes_size) 5792 { 5793 unsigned long realtotalpages = 0, totalpages = 0; 5794 enum zone_type i; 5795 5796 for (i = 0; i < MAX_NR_ZONES; i++) { 5797 struct zone *zone = pgdat->node_zones + i; 5798 unsigned long zone_start_pfn, zone_end_pfn; 5799 unsigned long size, real_size; 5800 5801 size = zone_spanned_pages_in_node(pgdat->node_id, i, 5802 node_start_pfn, 5803 node_end_pfn, 5804 &zone_start_pfn, 5805 &zone_end_pfn, 5806 zones_size); 5807 real_size = size - zone_absent_pages_in_node(pgdat->node_id, i, 5808 node_start_pfn, node_end_pfn, 5809 zholes_size); 5810 if (size) 5811 zone->zone_start_pfn = zone_start_pfn; 5812 else 5813 zone->zone_start_pfn = 0; 5814 zone->spanned_pages = size; 5815 zone->present_pages = real_size; 5816 5817 totalpages += size; 5818 realtotalpages += real_size; 5819 } 5820 5821 pgdat->node_spanned_pages = totalpages; 5822 pgdat->node_present_pages = realtotalpages; 5823 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, 5824 realtotalpages); 5825 } 5826 5827 #ifndef CONFIG_SPARSEMEM 5828 /* 5829 * Calculate the size of the zone->blockflags rounded to an unsigned long 5830 * Start by making sure zonesize is a multiple of pageblock_order by rounding 5831 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 5832 * round what is now in bits to nearest long in bits, then return it in 5833 * bytes. 5834 */ 5835 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 5836 { 5837 unsigned long usemapsize; 5838 5839 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 5840 usemapsize = roundup(zonesize, pageblock_nr_pages); 5841 usemapsize = usemapsize >> pageblock_order; 5842 usemapsize *= NR_PAGEBLOCK_BITS; 5843 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 5844 5845 return usemapsize / 8; 5846 } 5847 5848 static void __init setup_usemap(struct pglist_data *pgdat, 5849 struct zone *zone, 5850 unsigned long zone_start_pfn, 5851 unsigned long zonesize) 5852 { 5853 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); 5854 zone->pageblock_flags = NULL; 5855 if (usemapsize) 5856 zone->pageblock_flags = 5857 memblock_virt_alloc_node_nopanic(usemapsize, 5858 pgdat->node_id); 5859 } 5860 #else 5861 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, 5862 unsigned long zone_start_pfn, unsigned long zonesize) {} 5863 #endif /* CONFIG_SPARSEMEM */ 5864 5865 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 5866 5867 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 5868 void __paginginit set_pageblock_order(void) 5869 { 5870 unsigned int order; 5871 5872 /* Check that pageblock_nr_pages has not already been setup */ 5873 if (pageblock_order) 5874 return; 5875 5876 if (HPAGE_SHIFT > PAGE_SHIFT) 5877 order = HUGETLB_PAGE_ORDER; 5878 else 5879 order = MAX_ORDER - 1; 5880 5881 /* 5882 * Assume the largest contiguous order of interest is a huge page. 5883 * This value may be variable depending on boot parameters on IA64 and 5884 * powerpc. 5885 */ 5886 pageblock_order = order; 5887 } 5888 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 5889 5890 /* 5891 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 5892 * is unused as pageblock_order is set at compile-time. See 5893 * include/linux/pageblock-flags.h for the values of pageblock_order based on 5894 * the kernel config 5895 */ 5896 void __paginginit set_pageblock_order(void) 5897 { 5898 } 5899 5900 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 5901 5902 static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages, 5903 unsigned long present_pages) 5904 { 5905 unsigned long pages = spanned_pages; 5906 5907 /* 5908 * Provide a more accurate estimation if there are holes within 5909 * the zone and SPARSEMEM is in use. If there are holes within the 5910 * zone, each populated memory region may cost us one or two extra 5911 * memmap pages due to alignment because memmap pages for each 5912 * populated regions may not naturally algined on page boundary. 5913 * So the (present_pages >> 4) heuristic is a tradeoff for that. 5914 */ 5915 if (spanned_pages > present_pages + (present_pages >> 4) && 5916 IS_ENABLED(CONFIG_SPARSEMEM)) 5917 pages = present_pages; 5918 5919 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 5920 } 5921 5922 /* 5923 * Set up the zone data structures: 5924 * - mark all pages reserved 5925 * - mark all memory queues empty 5926 * - clear the memory bitmaps 5927 * 5928 * NOTE: pgdat should get zeroed by caller. 5929 */ 5930 static void __paginginit free_area_init_core(struct pglist_data *pgdat) 5931 { 5932 enum zone_type j; 5933 int nid = pgdat->node_id; 5934 int ret; 5935 5936 pgdat_resize_init(pgdat); 5937 #ifdef CONFIG_NUMA_BALANCING 5938 spin_lock_init(&pgdat->numabalancing_migrate_lock); 5939 pgdat->numabalancing_migrate_nr_pages = 0; 5940 pgdat->numabalancing_migrate_next_window = jiffies; 5941 #endif 5942 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5943 spin_lock_init(&pgdat->split_queue_lock); 5944 INIT_LIST_HEAD(&pgdat->split_queue); 5945 pgdat->split_queue_len = 0; 5946 #endif 5947 init_waitqueue_head(&pgdat->kswapd_wait); 5948 init_waitqueue_head(&pgdat->pfmemalloc_wait); 5949 #ifdef CONFIG_COMPACTION 5950 init_waitqueue_head(&pgdat->kcompactd_wait); 5951 #endif 5952 pgdat_page_ext_init(pgdat); 5953 5954 for (j = 0; j < MAX_NR_ZONES; j++) { 5955 struct zone *zone = pgdat->node_zones + j; 5956 unsigned long size, realsize, freesize, memmap_pages; 5957 unsigned long zone_start_pfn = zone->zone_start_pfn; 5958 5959 size = zone->spanned_pages; 5960 realsize = freesize = zone->present_pages; 5961 5962 /* 5963 * Adjust freesize so that it accounts for how much memory 5964 * is used by this zone for memmap. This affects the watermark 5965 * and per-cpu initialisations 5966 */ 5967 memmap_pages = calc_memmap_size(size, realsize); 5968 if (!is_highmem_idx(j)) { 5969 if (freesize >= memmap_pages) { 5970 freesize -= memmap_pages; 5971 if (memmap_pages) 5972 printk(KERN_DEBUG 5973 " %s zone: %lu pages used for memmap\n", 5974 zone_names[j], memmap_pages); 5975 } else 5976 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n", 5977 zone_names[j], memmap_pages, freesize); 5978 } 5979 5980 /* Account for reserved pages */ 5981 if (j == 0 && freesize > dma_reserve) { 5982 freesize -= dma_reserve; 5983 printk(KERN_DEBUG " %s zone: %lu pages reserved\n", 5984 zone_names[0], dma_reserve); 5985 } 5986 5987 if (!is_highmem_idx(j)) 5988 nr_kernel_pages += freesize; 5989 /* Charge for highmem memmap if there are enough kernel pages */ 5990 else if (nr_kernel_pages > memmap_pages * 2) 5991 nr_kernel_pages -= memmap_pages; 5992 nr_all_pages += freesize; 5993 5994 /* 5995 * Set an approximate value for lowmem here, it will be adjusted 5996 * when the bootmem allocator frees pages into the buddy system. 5997 * And all highmem pages will be managed by the buddy system. 5998 */ 5999 zone->managed_pages = is_highmem_idx(j) ? realsize : freesize; 6000 #ifdef CONFIG_NUMA 6001 zone->node = nid; 6002 zone->min_unmapped_pages = (freesize*sysctl_min_unmapped_ratio) 6003 / 100; 6004 zone->min_slab_pages = (freesize * sysctl_min_slab_ratio) / 100; 6005 #endif 6006 zone->name = zone_names[j]; 6007 spin_lock_init(&zone->lock); 6008 spin_lock_init(&zone->lru_lock); 6009 zone_seqlock_init(zone); 6010 zone->zone_pgdat = pgdat; 6011 zone_pcp_init(zone); 6012 6013 /* For bootup, initialized properly in watermark setup */ 6014 mod_zone_page_state(zone, NR_ALLOC_BATCH, zone->managed_pages); 6015 6016 lruvec_init(&zone->lruvec); 6017 if (!size) 6018 continue; 6019 6020 set_pageblock_order(); 6021 setup_usemap(pgdat, zone, zone_start_pfn, size); 6022 ret = init_currently_empty_zone(zone, zone_start_pfn, size); 6023 BUG_ON(ret); 6024 memmap_init(size, nid, j, zone_start_pfn); 6025 } 6026 } 6027 6028 static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat) 6029 { 6030 unsigned long __maybe_unused start = 0; 6031 unsigned long __maybe_unused offset = 0; 6032 6033 /* Skip empty nodes */ 6034 if (!pgdat->node_spanned_pages) 6035 return; 6036 6037 #ifdef CONFIG_FLAT_NODE_MEM_MAP 6038 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 6039 offset = pgdat->node_start_pfn - start; 6040 /* ia64 gets its own node_mem_map, before this, without bootmem */ 6041 if (!pgdat->node_mem_map) { 6042 unsigned long size, end; 6043 struct page *map; 6044 6045 /* 6046 * The zone's endpoints aren't required to be MAX_ORDER 6047 * aligned but the node_mem_map endpoints must be in order 6048 * for the buddy allocator to function correctly. 6049 */ 6050 end = pgdat_end_pfn(pgdat); 6051 end = ALIGN(end, MAX_ORDER_NR_PAGES); 6052 size = (end - start) * sizeof(struct page); 6053 map = alloc_remap(pgdat->node_id, size); 6054 if (!map) 6055 map = memblock_virt_alloc_node_nopanic(size, 6056 pgdat->node_id); 6057 pgdat->node_mem_map = map + offset; 6058 } 6059 #ifndef CONFIG_NEED_MULTIPLE_NODES 6060 /* 6061 * With no DISCONTIG, the global mem_map is just set as node 0's 6062 */ 6063 if (pgdat == NODE_DATA(0)) { 6064 mem_map = NODE_DATA(0)->node_mem_map; 6065 #if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM) 6066 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 6067 mem_map -= offset; 6068 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6069 } 6070 #endif 6071 #endif /* CONFIG_FLAT_NODE_MEM_MAP */ 6072 } 6073 6074 void __paginginit free_area_init_node(int nid, unsigned long *zones_size, 6075 unsigned long node_start_pfn, unsigned long *zholes_size) 6076 { 6077 pg_data_t *pgdat = NODE_DATA(nid); 6078 unsigned long start_pfn = 0; 6079 unsigned long end_pfn = 0; 6080 6081 /* pg_data_t should be reset to zero when it's allocated */ 6082 WARN_ON(pgdat->nr_zones || pgdat->classzone_idx); 6083 6084 reset_deferred_meminit(pgdat); 6085 pgdat->node_id = nid; 6086 pgdat->node_start_pfn = node_start_pfn; 6087 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6088 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 6089 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 6090 (u64)start_pfn << PAGE_SHIFT, 6091 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 6092 #else 6093 start_pfn = node_start_pfn; 6094 #endif 6095 calculate_node_totalpages(pgdat, start_pfn, end_pfn, 6096 zones_size, zholes_size); 6097 6098 alloc_node_mem_map(pgdat); 6099 #ifdef CONFIG_FLAT_NODE_MEM_MAP 6100 printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n", 6101 nid, (unsigned long)pgdat, 6102 (unsigned long)pgdat->node_mem_map); 6103 #endif 6104 6105 free_area_init_core(pgdat); 6106 } 6107 6108 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 6109 6110 #if MAX_NUMNODES > 1 6111 /* 6112 * Figure out the number of possible node ids. 6113 */ 6114 void __init setup_nr_node_ids(void) 6115 { 6116 unsigned int highest; 6117 6118 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 6119 nr_node_ids = highest + 1; 6120 } 6121 #endif 6122 6123 /** 6124 * node_map_pfn_alignment - determine the maximum internode alignment 6125 * 6126 * This function should be called after node map is populated and sorted. 6127 * It calculates the maximum power of two alignment which can distinguish 6128 * all the nodes. 6129 * 6130 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 6131 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 6132 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 6133 * shifted, 1GiB is enough and this function will indicate so. 6134 * 6135 * This is used to test whether pfn -> nid mapping of the chosen memory 6136 * model has fine enough granularity to avoid incorrect mapping for the 6137 * populated node map. 6138 * 6139 * Returns the determined alignment in pfn's. 0 if there is no alignment 6140 * requirement (single node). 6141 */ 6142 unsigned long __init node_map_pfn_alignment(void) 6143 { 6144 unsigned long accl_mask = 0, last_end = 0; 6145 unsigned long start, end, mask; 6146 int last_nid = -1; 6147 int i, nid; 6148 6149 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 6150 if (!start || last_nid < 0 || last_nid == nid) { 6151 last_nid = nid; 6152 last_end = end; 6153 continue; 6154 } 6155 6156 /* 6157 * Start with a mask granular enough to pin-point to the 6158 * start pfn and tick off bits one-by-one until it becomes 6159 * too coarse to separate the current node from the last. 6160 */ 6161 mask = ~((1 << __ffs(start)) - 1); 6162 while (mask && last_end <= (start & (mask << 1))) 6163 mask <<= 1; 6164 6165 /* accumulate all internode masks */ 6166 accl_mask |= mask; 6167 } 6168 6169 /* convert mask to number of pages */ 6170 return ~accl_mask + 1; 6171 } 6172 6173 /* Find the lowest pfn for a node */ 6174 static unsigned long __init find_min_pfn_for_node(int nid) 6175 { 6176 unsigned long min_pfn = ULONG_MAX; 6177 unsigned long start_pfn; 6178 int i; 6179 6180 for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL) 6181 min_pfn = min(min_pfn, start_pfn); 6182 6183 if (min_pfn == ULONG_MAX) { 6184 pr_warn("Could not find start_pfn for node %d\n", nid); 6185 return 0; 6186 } 6187 6188 return min_pfn; 6189 } 6190 6191 /** 6192 * find_min_pfn_with_active_regions - Find the minimum PFN registered 6193 * 6194 * It returns the minimum PFN based on information provided via 6195 * memblock_set_node(). 6196 */ 6197 unsigned long __init find_min_pfn_with_active_regions(void) 6198 { 6199 return find_min_pfn_for_node(MAX_NUMNODES); 6200 } 6201 6202 /* 6203 * early_calculate_totalpages() 6204 * Sum pages in active regions for movable zone. 6205 * Populate N_MEMORY for calculating usable_nodes. 6206 */ 6207 static unsigned long __init early_calculate_totalpages(void) 6208 { 6209 unsigned long totalpages = 0; 6210 unsigned long start_pfn, end_pfn; 6211 int i, nid; 6212 6213 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 6214 unsigned long pages = end_pfn - start_pfn; 6215 6216 totalpages += pages; 6217 if (pages) 6218 node_set_state(nid, N_MEMORY); 6219 } 6220 return totalpages; 6221 } 6222 6223 /* 6224 * Find the PFN the Movable zone begins in each node. Kernel memory 6225 * is spread evenly between nodes as long as the nodes have enough 6226 * memory. When they don't, some nodes will have more kernelcore than 6227 * others 6228 */ 6229 static void __init find_zone_movable_pfns_for_nodes(void) 6230 { 6231 int i, nid; 6232 unsigned long usable_startpfn; 6233 unsigned long kernelcore_node, kernelcore_remaining; 6234 /* save the state before borrow the nodemask */ 6235 nodemask_t saved_node_state = node_states[N_MEMORY]; 6236 unsigned long totalpages = early_calculate_totalpages(); 6237 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 6238 struct memblock_region *r; 6239 6240 /* Need to find movable_zone earlier when movable_node is specified. */ 6241 find_usable_zone_for_movable(); 6242 6243 /* 6244 * If movable_node is specified, ignore kernelcore and movablecore 6245 * options. 6246 */ 6247 if (movable_node_is_enabled()) { 6248 for_each_memblock(memory, r) { 6249 if (!memblock_is_hotpluggable(r)) 6250 continue; 6251 6252 nid = r->nid; 6253 6254 usable_startpfn = PFN_DOWN(r->base); 6255 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 6256 min(usable_startpfn, zone_movable_pfn[nid]) : 6257 usable_startpfn; 6258 } 6259 6260 goto out2; 6261 } 6262 6263 /* 6264 * If kernelcore=mirror is specified, ignore movablecore option 6265 */ 6266 if (mirrored_kernelcore) { 6267 bool mem_below_4gb_not_mirrored = false; 6268 6269 for_each_memblock(memory, r) { 6270 if (memblock_is_mirror(r)) 6271 continue; 6272 6273 nid = r->nid; 6274 6275 usable_startpfn = memblock_region_memory_base_pfn(r); 6276 6277 if (usable_startpfn < 0x100000) { 6278 mem_below_4gb_not_mirrored = true; 6279 continue; 6280 } 6281 6282 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 6283 min(usable_startpfn, zone_movable_pfn[nid]) : 6284 usable_startpfn; 6285 } 6286 6287 if (mem_below_4gb_not_mirrored) 6288 pr_warn("This configuration results in unmirrored kernel memory."); 6289 6290 goto out2; 6291 } 6292 6293 /* 6294 * If movablecore=nn[KMG] was specified, calculate what size of 6295 * kernelcore that corresponds so that memory usable for 6296 * any allocation type is evenly spread. If both kernelcore 6297 * and movablecore are specified, then the value of kernelcore 6298 * will be used for required_kernelcore if it's greater than 6299 * what movablecore would have allowed. 6300 */ 6301 if (required_movablecore) { 6302 unsigned long corepages; 6303 6304 /* 6305 * Round-up so that ZONE_MOVABLE is at least as large as what 6306 * was requested by the user 6307 */ 6308 required_movablecore = 6309 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 6310 required_movablecore = min(totalpages, required_movablecore); 6311 corepages = totalpages - required_movablecore; 6312 6313 required_kernelcore = max(required_kernelcore, corepages); 6314 } 6315 6316 /* 6317 * If kernelcore was not specified or kernelcore size is larger 6318 * than totalpages, there is no ZONE_MOVABLE. 6319 */ 6320 if (!required_kernelcore || required_kernelcore >= totalpages) 6321 goto out; 6322 6323 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 6324 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 6325 6326 restart: 6327 /* Spread kernelcore memory as evenly as possible throughout nodes */ 6328 kernelcore_node = required_kernelcore / usable_nodes; 6329 for_each_node_state(nid, N_MEMORY) { 6330 unsigned long start_pfn, end_pfn; 6331 6332 /* 6333 * Recalculate kernelcore_node if the division per node 6334 * now exceeds what is necessary to satisfy the requested 6335 * amount of memory for the kernel 6336 */ 6337 if (required_kernelcore < kernelcore_node) 6338 kernelcore_node = required_kernelcore / usable_nodes; 6339 6340 /* 6341 * As the map is walked, we track how much memory is usable 6342 * by the kernel using kernelcore_remaining. When it is 6343 * 0, the rest of the node is usable by ZONE_MOVABLE 6344 */ 6345 kernelcore_remaining = kernelcore_node; 6346 6347 /* Go through each range of PFNs within this node */ 6348 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 6349 unsigned long size_pages; 6350 6351 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 6352 if (start_pfn >= end_pfn) 6353 continue; 6354 6355 /* Account for what is only usable for kernelcore */ 6356 if (start_pfn < usable_startpfn) { 6357 unsigned long kernel_pages; 6358 kernel_pages = min(end_pfn, usable_startpfn) 6359 - start_pfn; 6360 6361 kernelcore_remaining -= min(kernel_pages, 6362 kernelcore_remaining); 6363 required_kernelcore -= min(kernel_pages, 6364 required_kernelcore); 6365 6366 /* Continue if range is now fully accounted */ 6367 if (end_pfn <= usable_startpfn) { 6368 6369 /* 6370 * Push zone_movable_pfn to the end so 6371 * that if we have to rebalance 6372 * kernelcore across nodes, we will 6373 * not double account here 6374 */ 6375 zone_movable_pfn[nid] = end_pfn; 6376 continue; 6377 } 6378 start_pfn = usable_startpfn; 6379 } 6380 6381 /* 6382 * The usable PFN range for ZONE_MOVABLE is from 6383 * start_pfn->end_pfn. Calculate size_pages as the 6384 * number of pages used as kernelcore 6385 */ 6386 size_pages = end_pfn - start_pfn; 6387 if (size_pages > kernelcore_remaining) 6388 size_pages = kernelcore_remaining; 6389 zone_movable_pfn[nid] = start_pfn + size_pages; 6390 6391 /* 6392 * Some kernelcore has been met, update counts and 6393 * break if the kernelcore for this node has been 6394 * satisfied 6395 */ 6396 required_kernelcore -= min(required_kernelcore, 6397 size_pages); 6398 kernelcore_remaining -= size_pages; 6399 if (!kernelcore_remaining) 6400 break; 6401 } 6402 } 6403 6404 /* 6405 * If there is still required_kernelcore, we do another pass with one 6406 * less node in the count. This will push zone_movable_pfn[nid] further 6407 * along on the nodes that still have memory until kernelcore is 6408 * satisfied 6409 */ 6410 usable_nodes--; 6411 if (usable_nodes && required_kernelcore > usable_nodes) 6412 goto restart; 6413 6414 out2: 6415 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 6416 for (nid = 0; nid < MAX_NUMNODES; nid++) 6417 zone_movable_pfn[nid] = 6418 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 6419 6420 out: 6421 /* restore the node_state */ 6422 node_states[N_MEMORY] = saved_node_state; 6423 } 6424 6425 /* Any regular or high memory on that node ? */ 6426 static void check_for_memory(pg_data_t *pgdat, int nid) 6427 { 6428 enum zone_type zone_type; 6429 6430 if (N_MEMORY == N_NORMAL_MEMORY) 6431 return; 6432 6433 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 6434 struct zone *zone = &pgdat->node_zones[zone_type]; 6435 if (populated_zone(zone)) { 6436 node_set_state(nid, N_HIGH_MEMORY); 6437 if (N_NORMAL_MEMORY != N_HIGH_MEMORY && 6438 zone_type <= ZONE_NORMAL) 6439 node_set_state(nid, N_NORMAL_MEMORY); 6440 break; 6441 } 6442 } 6443 } 6444 6445 /** 6446 * free_area_init_nodes - Initialise all pg_data_t and zone data 6447 * @max_zone_pfn: an array of max PFNs for each zone 6448 * 6449 * This will call free_area_init_node() for each active node in the system. 6450 * Using the page ranges provided by memblock_set_node(), the size of each 6451 * zone in each node and their holes is calculated. If the maximum PFN 6452 * between two adjacent zones match, it is assumed that the zone is empty. 6453 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 6454 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 6455 * starts where the previous one ended. For example, ZONE_DMA32 starts 6456 * at arch_max_dma_pfn. 6457 */ 6458 void __init free_area_init_nodes(unsigned long *max_zone_pfn) 6459 { 6460 unsigned long start_pfn, end_pfn; 6461 int i, nid; 6462 6463 /* Record where the zone boundaries are */ 6464 memset(arch_zone_lowest_possible_pfn, 0, 6465 sizeof(arch_zone_lowest_possible_pfn)); 6466 memset(arch_zone_highest_possible_pfn, 0, 6467 sizeof(arch_zone_highest_possible_pfn)); 6468 arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions(); 6469 arch_zone_highest_possible_pfn[0] = max_zone_pfn[0]; 6470 for (i = 1; i < MAX_NR_ZONES; i++) { 6471 if (i == ZONE_MOVABLE) 6472 continue; 6473 arch_zone_lowest_possible_pfn[i] = 6474 arch_zone_highest_possible_pfn[i-1]; 6475 arch_zone_highest_possible_pfn[i] = 6476 max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]); 6477 } 6478 arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0; 6479 arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0; 6480 6481 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 6482 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 6483 find_zone_movable_pfns_for_nodes(); 6484 6485 /* Print out the zone ranges */ 6486 pr_info("Zone ranges:\n"); 6487 for (i = 0; i < MAX_NR_ZONES; i++) { 6488 if (i == ZONE_MOVABLE) 6489 continue; 6490 pr_info(" %-8s ", zone_names[i]); 6491 if (arch_zone_lowest_possible_pfn[i] == 6492 arch_zone_highest_possible_pfn[i]) 6493 pr_cont("empty\n"); 6494 else 6495 pr_cont("[mem %#018Lx-%#018Lx]\n", 6496 (u64)arch_zone_lowest_possible_pfn[i] 6497 << PAGE_SHIFT, 6498 ((u64)arch_zone_highest_possible_pfn[i] 6499 << PAGE_SHIFT) - 1); 6500 } 6501 6502 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 6503 pr_info("Movable zone start for each node\n"); 6504 for (i = 0; i < MAX_NUMNODES; i++) { 6505 if (zone_movable_pfn[i]) 6506 pr_info(" Node %d: %#018Lx\n", i, 6507 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 6508 } 6509 6510 /* Print out the early node map */ 6511 pr_info("Early memory node ranges\n"); 6512 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) 6513 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 6514 (u64)start_pfn << PAGE_SHIFT, 6515 ((u64)end_pfn << PAGE_SHIFT) - 1); 6516 6517 /* Initialise every node */ 6518 mminit_verify_pageflags_layout(); 6519 setup_nr_node_ids(); 6520 for_each_online_node(nid) { 6521 pg_data_t *pgdat = NODE_DATA(nid); 6522 free_area_init_node(nid, NULL, 6523 find_min_pfn_for_node(nid), NULL); 6524 6525 /* Any memory on that node */ 6526 if (pgdat->node_present_pages) 6527 node_set_state(nid, N_MEMORY); 6528 check_for_memory(pgdat, nid); 6529 } 6530 } 6531 6532 static int __init cmdline_parse_core(char *p, unsigned long *core) 6533 { 6534 unsigned long long coremem; 6535 if (!p) 6536 return -EINVAL; 6537 6538 coremem = memparse(p, &p); 6539 *core = coremem >> PAGE_SHIFT; 6540 6541 /* Paranoid check that UL is enough for the coremem value */ 6542 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 6543 6544 return 0; 6545 } 6546 6547 /* 6548 * kernelcore=size sets the amount of memory for use for allocations that 6549 * cannot be reclaimed or migrated. 6550 */ 6551 static int __init cmdline_parse_kernelcore(char *p) 6552 { 6553 /* parse kernelcore=mirror */ 6554 if (parse_option_str(p, "mirror")) { 6555 mirrored_kernelcore = true; 6556 return 0; 6557 } 6558 6559 return cmdline_parse_core(p, &required_kernelcore); 6560 } 6561 6562 /* 6563 * movablecore=size sets the amount of memory for use for allocations that 6564 * can be reclaimed or migrated. 6565 */ 6566 static int __init cmdline_parse_movablecore(char *p) 6567 { 6568 return cmdline_parse_core(p, &required_movablecore); 6569 } 6570 6571 early_param("kernelcore", cmdline_parse_kernelcore); 6572 early_param("movablecore", cmdline_parse_movablecore); 6573 6574 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 6575 6576 void adjust_managed_page_count(struct page *page, long count) 6577 { 6578 spin_lock(&managed_page_count_lock); 6579 page_zone(page)->managed_pages += count; 6580 totalram_pages += count; 6581 #ifdef CONFIG_HIGHMEM 6582 if (PageHighMem(page)) 6583 totalhigh_pages += count; 6584 #endif 6585 spin_unlock(&managed_page_count_lock); 6586 } 6587 EXPORT_SYMBOL(adjust_managed_page_count); 6588 6589 unsigned long free_reserved_area(void *start, void *end, int poison, char *s) 6590 { 6591 void *pos; 6592 unsigned long pages = 0; 6593 6594 start = (void *)PAGE_ALIGN((unsigned long)start); 6595 end = (void *)((unsigned long)end & PAGE_MASK); 6596 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 6597 if ((unsigned int)poison <= 0xFF) 6598 memset(pos, poison, PAGE_SIZE); 6599 free_reserved_page(virt_to_page(pos)); 6600 } 6601 6602 if (pages && s) 6603 pr_info("Freeing %s memory: %ldK (%p - %p)\n", 6604 s, pages << (PAGE_SHIFT - 10), start, end); 6605 6606 return pages; 6607 } 6608 EXPORT_SYMBOL(free_reserved_area); 6609 6610 #ifdef CONFIG_HIGHMEM 6611 void free_highmem_page(struct page *page) 6612 { 6613 __free_reserved_page(page); 6614 totalram_pages++; 6615 page_zone(page)->managed_pages++; 6616 totalhigh_pages++; 6617 } 6618 #endif 6619 6620 6621 void __init mem_init_print_info(const char *str) 6622 { 6623 unsigned long physpages, codesize, datasize, rosize, bss_size; 6624 unsigned long init_code_size, init_data_size; 6625 6626 physpages = get_num_physpages(); 6627 codesize = _etext - _stext; 6628 datasize = _edata - _sdata; 6629 rosize = __end_rodata - __start_rodata; 6630 bss_size = __bss_stop - __bss_start; 6631 init_data_size = __init_end - __init_begin; 6632 init_code_size = _einittext - _sinittext; 6633 6634 /* 6635 * Detect special cases and adjust section sizes accordingly: 6636 * 1) .init.* may be embedded into .data sections 6637 * 2) .init.text.* may be out of [__init_begin, __init_end], 6638 * please refer to arch/tile/kernel/vmlinux.lds.S. 6639 * 3) .rodata.* may be embedded into .text or .data sections. 6640 */ 6641 #define adj_init_size(start, end, size, pos, adj) \ 6642 do { \ 6643 if (start <= pos && pos < end && size > adj) \ 6644 size -= adj; \ 6645 } while (0) 6646 6647 adj_init_size(__init_begin, __init_end, init_data_size, 6648 _sinittext, init_code_size); 6649 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 6650 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 6651 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 6652 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 6653 6654 #undef adj_init_size 6655 6656 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 6657 #ifdef CONFIG_HIGHMEM 6658 ", %luK highmem" 6659 #endif 6660 "%s%s)\n", 6661 nr_free_pages() << (PAGE_SHIFT - 10), 6662 physpages << (PAGE_SHIFT - 10), 6663 codesize >> 10, datasize >> 10, rosize >> 10, 6664 (init_data_size + init_code_size) >> 10, bss_size >> 10, 6665 (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT - 10), 6666 totalcma_pages << (PAGE_SHIFT - 10), 6667 #ifdef CONFIG_HIGHMEM 6668 totalhigh_pages << (PAGE_SHIFT - 10), 6669 #endif 6670 str ? ", " : "", str ? str : ""); 6671 } 6672 6673 /** 6674 * set_dma_reserve - set the specified number of pages reserved in the first zone 6675 * @new_dma_reserve: The number of pages to mark reserved 6676 * 6677 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 6678 * In the DMA zone, a significant percentage may be consumed by kernel image 6679 * and other unfreeable allocations which can skew the watermarks badly. This 6680 * function may optionally be used to account for unfreeable pages in the 6681 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 6682 * smaller per-cpu batchsize. 6683 */ 6684 void __init set_dma_reserve(unsigned long new_dma_reserve) 6685 { 6686 dma_reserve = new_dma_reserve; 6687 } 6688 6689 void __init free_area_init(unsigned long *zones_size) 6690 { 6691 free_area_init_node(0, zones_size, 6692 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); 6693 } 6694 6695 static int page_alloc_cpu_notify(struct notifier_block *self, 6696 unsigned long action, void *hcpu) 6697 { 6698 int cpu = (unsigned long)hcpu; 6699 6700 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { 6701 lru_add_drain_cpu(cpu); 6702 drain_pages(cpu); 6703 6704 /* 6705 * Spill the event counters of the dead processor 6706 * into the current processors event counters. 6707 * This artificially elevates the count of the current 6708 * processor. 6709 */ 6710 vm_events_fold_cpu(cpu); 6711 6712 /* 6713 * Zero the differential counters of the dead processor 6714 * so that the vm statistics are consistent. 6715 * 6716 * This is only okay since the processor is dead and cannot 6717 * race with what we are doing. 6718 */ 6719 cpu_vm_stats_fold(cpu); 6720 } 6721 return NOTIFY_OK; 6722 } 6723 6724 void __init page_alloc_init(void) 6725 { 6726 hotcpu_notifier(page_alloc_cpu_notify, 0); 6727 } 6728 6729 /* 6730 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 6731 * or min_free_kbytes changes. 6732 */ 6733 static void calculate_totalreserve_pages(void) 6734 { 6735 struct pglist_data *pgdat; 6736 unsigned long reserve_pages = 0; 6737 enum zone_type i, j; 6738 6739 for_each_online_pgdat(pgdat) { 6740 for (i = 0; i < MAX_NR_ZONES; i++) { 6741 struct zone *zone = pgdat->node_zones + i; 6742 long max = 0; 6743 6744 /* Find valid and maximum lowmem_reserve in the zone */ 6745 for (j = i; j < MAX_NR_ZONES; j++) { 6746 if (zone->lowmem_reserve[j] > max) 6747 max = zone->lowmem_reserve[j]; 6748 } 6749 6750 /* we treat the high watermark as reserved pages. */ 6751 max += high_wmark_pages(zone); 6752 6753 if (max > zone->managed_pages) 6754 max = zone->managed_pages; 6755 6756 zone->totalreserve_pages = max; 6757 6758 reserve_pages += max; 6759 } 6760 } 6761 totalreserve_pages = reserve_pages; 6762 } 6763 6764 /* 6765 * setup_per_zone_lowmem_reserve - called whenever 6766 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 6767 * has a correct pages reserved value, so an adequate number of 6768 * pages are left in the zone after a successful __alloc_pages(). 6769 */ 6770 static void setup_per_zone_lowmem_reserve(void) 6771 { 6772 struct pglist_data *pgdat; 6773 enum zone_type j, idx; 6774 6775 for_each_online_pgdat(pgdat) { 6776 for (j = 0; j < MAX_NR_ZONES; j++) { 6777 struct zone *zone = pgdat->node_zones + j; 6778 unsigned long managed_pages = zone->managed_pages; 6779 6780 zone->lowmem_reserve[j] = 0; 6781 6782 idx = j; 6783 while (idx) { 6784 struct zone *lower_zone; 6785 6786 idx--; 6787 6788 if (sysctl_lowmem_reserve_ratio[idx] < 1) 6789 sysctl_lowmem_reserve_ratio[idx] = 1; 6790 6791 lower_zone = pgdat->node_zones + idx; 6792 lower_zone->lowmem_reserve[j] = managed_pages / 6793 sysctl_lowmem_reserve_ratio[idx]; 6794 managed_pages += lower_zone->managed_pages; 6795 } 6796 } 6797 } 6798 6799 /* update totalreserve_pages */ 6800 calculate_totalreserve_pages(); 6801 } 6802 6803 static void __setup_per_zone_wmarks(void) 6804 { 6805 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 6806 unsigned long lowmem_pages = 0; 6807 struct zone *zone; 6808 unsigned long flags; 6809 6810 /* Calculate total number of !ZONE_HIGHMEM pages */ 6811 for_each_zone(zone) { 6812 if (!is_highmem(zone)) 6813 lowmem_pages += zone->managed_pages; 6814 } 6815 6816 for_each_zone(zone) { 6817 u64 tmp; 6818 6819 spin_lock_irqsave(&zone->lock, flags); 6820 tmp = (u64)pages_min * zone->managed_pages; 6821 do_div(tmp, lowmem_pages); 6822 if (is_highmem(zone)) { 6823 /* 6824 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 6825 * need highmem pages, so cap pages_min to a small 6826 * value here. 6827 * 6828 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 6829 * deltas control asynch page reclaim, and so should 6830 * not be capped for highmem. 6831 */ 6832 unsigned long min_pages; 6833 6834 min_pages = zone->managed_pages / 1024; 6835 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 6836 zone->watermark[WMARK_MIN] = min_pages; 6837 } else { 6838 /* 6839 * If it's a lowmem zone, reserve a number of pages 6840 * proportionate to the zone's size. 6841 */ 6842 zone->watermark[WMARK_MIN] = tmp; 6843 } 6844 6845 /* 6846 * Set the kswapd watermarks distance according to the 6847 * scale factor in proportion to available memory, but 6848 * ensure a minimum size on small systems. 6849 */ 6850 tmp = max_t(u64, tmp >> 2, 6851 mult_frac(zone->managed_pages, 6852 watermark_scale_factor, 10000)); 6853 6854 zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 6855 zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2; 6856 6857 __mod_zone_page_state(zone, NR_ALLOC_BATCH, 6858 high_wmark_pages(zone) - low_wmark_pages(zone) - 6859 atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH])); 6860 6861 spin_unlock_irqrestore(&zone->lock, flags); 6862 } 6863 6864 /* update totalreserve_pages */ 6865 calculate_totalreserve_pages(); 6866 } 6867 6868 /** 6869 * setup_per_zone_wmarks - called when min_free_kbytes changes 6870 * or when memory is hot-{added|removed} 6871 * 6872 * Ensures that the watermark[min,low,high] values for each zone are set 6873 * correctly with respect to min_free_kbytes. 6874 */ 6875 void setup_per_zone_wmarks(void) 6876 { 6877 mutex_lock(&zonelists_mutex); 6878 __setup_per_zone_wmarks(); 6879 mutex_unlock(&zonelists_mutex); 6880 } 6881 6882 /* 6883 * Initialise min_free_kbytes. 6884 * 6885 * For small machines we want it small (128k min). For large machines 6886 * we want it large (64MB max). But it is not linear, because network 6887 * bandwidth does not increase linearly with machine size. We use 6888 * 6889 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 6890 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 6891 * 6892 * which yields 6893 * 6894 * 16MB: 512k 6895 * 32MB: 724k 6896 * 64MB: 1024k 6897 * 128MB: 1448k 6898 * 256MB: 2048k 6899 * 512MB: 2896k 6900 * 1024MB: 4096k 6901 * 2048MB: 5792k 6902 * 4096MB: 8192k 6903 * 8192MB: 11584k 6904 * 16384MB: 16384k 6905 */ 6906 int __meminit init_per_zone_wmark_min(void) 6907 { 6908 unsigned long lowmem_kbytes; 6909 int new_min_free_kbytes; 6910 6911 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 6912 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 6913 6914 if (new_min_free_kbytes > user_min_free_kbytes) { 6915 min_free_kbytes = new_min_free_kbytes; 6916 if (min_free_kbytes < 128) 6917 min_free_kbytes = 128; 6918 if (min_free_kbytes > 65536) 6919 min_free_kbytes = 65536; 6920 } else { 6921 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 6922 new_min_free_kbytes, user_min_free_kbytes); 6923 } 6924 setup_per_zone_wmarks(); 6925 refresh_zone_stat_thresholds(); 6926 setup_per_zone_lowmem_reserve(); 6927 return 0; 6928 } 6929 core_initcall(init_per_zone_wmark_min) 6930 6931 /* 6932 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 6933 * that we can call two helper functions whenever min_free_kbytes 6934 * changes. 6935 */ 6936 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 6937 void __user *buffer, size_t *length, loff_t *ppos) 6938 { 6939 int rc; 6940 6941 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6942 if (rc) 6943 return rc; 6944 6945 if (write) { 6946 user_min_free_kbytes = min_free_kbytes; 6947 setup_per_zone_wmarks(); 6948 } 6949 return 0; 6950 } 6951 6952 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 6953 void __user *buffer, size_t *length, loff_t *ppos) 6954 { 6955 int rc; 6956 6957 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6958 if (rc) 6959 return rc; 6960 6961 if (write) 6962 setup_per_zone_wmarks(); 6963 6964 return 0; 6965 } 6966 6967 #ifdef CONFIG_NUMA 6968 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 6969 void __user *buffer, size_t *length, loff_t *ppos) 6970 { 6971 struct zone *zone; 6972 int rc; 6973 6974 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6975 if (rc) 6976 return rc; 6977 6978 for_each_zone(zone) 6979 zone->min_unmapped_pages = (zone->managed_pages * 6980 sysctl_min_unmapped_ratio) / 100; 6981 return 0; 6982 } 6983 6984 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 6985 void __user *buffer, size_t *length, loff_t *ppos) 6986 { 6987 struct zone *zone; 6988 int rc; 6989 6990 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6991 if (rc) 6992 return rc; 6993 6994 for_each_zone(zone) 6995 zone->min_slab_pages = (zone->managed_pages * 6996 sysctl_min_slab_ratio) / 100; 6997 return 0; 6998 } 6999 #endif 7000 7001 /* 7002 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 7003 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 7004 * whenever sysctl_lowmem_reserve_ratio changes. 7005 * 7006 * The reserve ratio obviously has absolutely no relation with the 7007 * minimum watermarks. The lowmem reserve ratio can only make sense 7008 * if in function of the boot time zone sizes. 7009 */ 7010 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 7011 void __user *buffer, size_t *length, loff_t *ppos) 7012 { 7013 proc_dointvec_minmax(table, write, buffer, length, ppos); 7014 setup_per_zone_lowmem_reserve(); 7015 return 0; 7016 } 7017 7018 /* 7019 * percpu_pagelist_fraction - changes the pcp->high for each zone on each 7020 * cpu. It is the fraction of total pages in each zone that a hot per cpu 7021 * pagelist can have before it gets flushed back to buddy allocator. 7022 */ 7023 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, 7024 void __user *buffer, size_t *length, loff_t *ppos) 7025 { 7026 struct zone *zone; 7027 int old_percpu_pagelist_fraction; 7028 int ret; 7029 7030 mutex_lock(&pcp_batch_high_lock); 7031 old_percpu_pagelist_fraction = percpu_pagelist_fraction; 7032 7033 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 7034 if (!write || ret < 0) 7035 goto out; 7036 7037 /* Sanity checking to avoid pcp imbalance */ 7038 if (percpu_pagelist_fraction && 7039 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { 7040 percpu_pagelist_fraction = old_percpu_pagelist_fraction; 7041 ret = -EINVAL; 7042 goto out; 7043 } 7044 7045 /* No change? */ 7046 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) 7047 goto out; 7048 7049 for_each_populated_zone(zone) { 7050 unsigned int cpu; 7051 7052 for_each_possible_cpu(cpu) 7053 pageset_set_high_and_batch(zone, 7054 per_cpu_ptr(zone->pageset, cpu)); 7055 } 7056 out: 7057 mutex_unlock(&pcp_batch_high_lock); 7058 return ret; 7059 } 7060 7061 #ifdef CONFIG_NUMA 7062 int hashdist = HASHDIST_DEFAULT; 7063 7064 static int __init set_hashdist(char *str) 7065 { 7066 if (!str) 7067 return 0; 7068 hashdist = simple_strtoul(str, &str, 0); 7069 return 1; 7070 } 7071 __setup("hashdist=", set_hashdist); 7072 #endif 7073 7074 /* 7075 * allocate a large system hash table from bootmem 7076 * - it is assumed that the hash table must contain an exact power-of-2 7077 * quantity of entries 7078 * - limit is the number of hash buckets, not the total allocation size 7079 */ 7080 void *__init alloc_large_system_hash(const char *tablename, 7081 unsigned long bucketsize, 7082 unsigned long numentries, 7083 int scale, 7084 int flags, 7085 unsigned int *_hash_shift, 7086 unsigned int *_hash_mask, 7087 unsigned long low_limit, 7088 unsigned long high_limit) 7089 { 7090 unsigned long long max = high_limit; 7091 unsigned long log2qty, size; 7092 void *table = NULL; 7093 7094 /* allow the kernel cmdline to have a say */ 7095 if (!numentries) { 7096 /* round applicable memory size up to nearest megabyte */ 7097 numentries = nr_kernel_pages; 7098 7099 /* It isn't necessary when PAGE_SIZE >= 1MB */ 7100 if (PAGE_SHIFT < 20) 7101 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 7102 7103 /* limit to 1 bucket per 2^scale bytes of low memory */ 7104 if (scale > PAGE_SHIFT) 7105 numentries >>= (scale - PAGE_SHIFT); 7106 else 7107 numentries <<= (PAGE_SHIFT - scale); 7108 7109 /* Make sure we've got at least a 0-order allocation.. */ 7110 if (unlikely(flags & HASH_SMALL)) { 7111 /* Makes no sense without HASH_EARLY */ 7112 WARN_ON(!(flags & HASH_EARLY)); 7113 if (!(numentries >> *_hash_shift)) { 7114 numentries = 1UL << *_hash_shift; 7115 BUG_ON(!numentries); 7116 } 7117 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 7118 numentries = PAGE_SIZE / bucketsize; 7119 } 7120 numentries = roundup_pow_of_two(numentries); 7121 7122 /* limit allocation size to 1/16 total memory by default */ 7123 if (max == 0) { 7124 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 7125 do_div(max, bucketsize); 7126 } 7127 max = min(max, 0x80000000ULL); 7128 7129 if (numentries < low_limit) 7130 numentries = low_limit; 7131 if (numentries > max) 7132 numentries = max; 7133 7134 log2qty = ilog2(numentries); 7135 7136 do { 7137 size = bucketsize << log2qty; 7138 if (flags & HASH_EARLY) 7139 table = memblock_virt_alloc_nopanic(size, 0); 7140 else if (hashdist) 7141 table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL); 7142 else { 7143 /* 7144 * If bucketsize is not a power-of-two, we may free 7145 * some pages at the end of hash table which 7146 * alloc_pages_exact() automatically does 7147 */ 7148 if (get_order(size) < MAX_ORDER) { 7149 table = alloc_pages_exact(size, GFP_ATOMIC); 7150 kmemleak_alloc(table, size, 1, GFP_ATOMIC); 7151 } 7152 } 7153 } while (!table && size > PAGE_SIZE && --log2qty); 7154 7155 if (!table) 7156 panic("Failed to allocate %s hash table\n", tablename); 7157 7158 pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n", 7159 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size); 7160 7161 if (_hash_shift) 7162 *_hash_shift = log2qty; 7163 if (_hash_mask) 7164 *_hash_mask = (1 << log2qty) - 1; 7165 7166 return table; 7167 } 7168 7169 /* 7170 * This function checks whether pageblock includes unmovable pages or not. 7171 * If @count is not zero, it is okay to include less @count unmovable pages 7172 * 7173 * PageLRU check without isolation or lru_lock could race so that 7174 * MIGRATE_MOVABLE block might include unmovable pages. It means you can't 7175 * expect this function should be exact. 7176 */ 7177 bool has_unmovable_pages(struct zone *zone, struct page *page, int count, 7178 bool skip_hwpoisoned_pages) 7179 { 7180 unsigned long pfn, iter, found; 7181 int mt; 7182 7183 /* 7184 * For avoiding noise data, lru_add_drain_all() should be called 7185 * If ZONE_MOVABLE, the zone never contains unmovable pages 7186 */ 7187 if (zone_idx(zone) == ZONE_MOVABLE) 7188 return false; 7189 mt = get_pageblock_migratetype(page); 7190 if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt)) 7191 return false; 7192 7193 pfn = page_to_pfn(page); 7194 for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) { 7195 unsigned long check = pfn + iter; 7196 7197 if (!pfn_valid_within(check)) 7198 continue; 7199 7200 page = pfn_to_page(check); 7201 7202 /* 7203 * Hugepages are not in LRU lists, but they're movable. 7204 * We need not scan over tail pages bacause we don't 7205 * handle each tail page individually in migration. 7206 */ 7207 if (PageHuge(page)) { 7208 iter = round_up(iter + 1, 1<<compound_order(page)) - 1; 7209 continue; 7210 } 7211 7212 /* 7213 * We can't use page_count without pin a page 7214 * because another CPU can free compound page. 7215 * This check already skips compound tails of THP 7216 * because their page->_refcount is zero at all time. 7217 */ 7218 if (!page_ref_count(page)) { 7219 if (PageBuddy(page)) 7220 iter += (1 << page_order(page)) - 1; 7221 continue; 7222 } 7223 7224 /* 7225 * The HWPoisoned page may be not in buddy system, and 7226 * page_count() is not 0. 7227 */ 7228 if (skip_hwpoisoned_pages && PageHWPoison(page)) 7229 continue; 7230 7231 if (!PageLRU(page)) 7232 found++; 7233 /* 7234 * If there are RECLAIMABLE pages, we need to check 7235 * it. But now, memory offline itself doesn't call 7236 * shrink_node_slabs() and it still to be fixed. 7237 */ 7238 /* 7239 * If the page is not RAM, page_count()should be 0. 7240 * we don't need more check. This is an _used_ not-movable page. 7241 * 7242 * The problematic thing here is PG_reserved pages. PG_reserved 7243 * is set to both of a memory hole page and a _used_ kernel 7244 * page at boot. 7245 */ 7246 if (found > count) 7247 return true; 7248 } 7249 return false; 7250 } 7251 7252 bool is_pageblock_removable_nolock(struct page *page) 7253 { 7254 struct zone *zone; 7255 unsigned long pfn; 7256 7257 /* 7258 * We have to be careful here because we are iterating over memory 7259 * sections which are not zone aware so we might end up outside of 7260 * the zone but still within the section. 7261 * We have to take care about the node as well. If the node is offline 7262 * its NODE_DATA will be NULL - see page_zone. 7263 */ 7264 if (!node_online(page_to_nid(page))) 7265 return false; 7266 7267 zone = page_zone(page); 7268 pfn = page_to_pfn(page); 7269 if (!zone_spans_pfn(zone, pfn)) 7270 return false; 7271 7272 return !has_unmovable_pages(zone, page, 0, true); 7273 } 7274 7275 #if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA) 7276 7277 static unsigned long pfn_max_align_down(unsigned long pfn) 7278 { 7279 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, 7280 pageblock_nr_pages) - 1); 7281 } 7282 7283 static unsigned long pfn_max_align_up(unsigned long pfn) 7284 { 7285 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, 7286 pageblock_nr_pages)); 7287 } 7288 7289 /* [start, end) must belong to a single zone. */ 7290 static int __alloc_contig_migrate_range(struct compact_control *cc, 7291 unsigned long start, unsigned long end) 7292 { 7293 /* This function is based on compact_zone() from compaction.c. */ 7294 unsigned long nr_reclaimed; 7295 unsigned long pfn = start; 7296 unsigned int tries = 0; 7297 int ret = 0; 7298 7299 migrate_prep(); 7300 7301 while (pfn < end || !list_empty(&cc->migratepages)) { 7302 if (fatal_signal_pending(current)) { 7303 ret = -EINTR; 7304 break; 7305 } 7306 7307 if (list_empty(&cc->migratepages)) { 7308 cc->nr_migratepages = 0; 7309 pfn = isolate_migratepages_range(cc, pfn, end); 7310 if (!pfn) { 7311 ret = -EINTR; 7312 break; 7313 } 7314 tries = 0; 7315 } else if (++tries == 5) { 7316 ret = ret < 0 ? ret : -EBUSY; 7317 break; 7318 } 7319 7320 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 7321 &cc->migratepages); 7322 cc->nr_migratepages -= nr_reclaimed; 7323 7324 ret = migrate_pages(&cc->migratepages, alloc_migrate_target, 7325 NULL, 0, cc->mode, MR_CMA); 7326 } 7327 if (ret < 0) { 7328 putback_movable_pages(&cc->migratepages); 7329 return ret; 7330 } 7331 return 0; 7332 } 7333 7334 /** 7335 * alloc_contig_range() -- tries to allocate given range of pages 7336 * @start: start PFN to allocate 7337 * @end: one-past-the-last PFN to allocate 7338 * @migratetype: migratetype of the underlaying pageblocks (either 7339 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 7340 * in range must have the same migratetype and it must 7341 * be either of the two. 7342 * 7343 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 7344 * aligned, however it's the caller's responsibility to guarantee that 7345 * we are the only thread that changes migrate type of pageblocks the 7346 * pages fall in. 7347 * 7348 * The PFN range must belong to a single zone. 7349 * 7350 * Returns zero on success or negative error code. On success all 7351 * pages which PFN is in [start, end) are allocated for the caller and 7352 * need to be freed with free_contig_range(). 7353 */ 7354 int alloc_contig_range(unsigned long start, unsigned long end, 7355 unsigned migratetype) 7356 { 7357 unsigned long outer_start, outer_end; 7358 unsigned int order; 7359 int ret = 0; 7360 7361 struct compact_control cc = { 7362 .nr_migratepages = 0, 7363 .order = -1, 7364 .zone = page_zone(pfn_to_page(start)), 7365 .mode = MIGRATE_SYNC, 7366 .ignore_skip_hint = true, 7367 }; 7368 INIT_LIST_HEAD(&cc.migratepages); 7369 7370 /* 7371 * What we do here is we mark all pageblocks in range as 7372 * MIGRATE_ISOLATE. Because pageblock and max order pages may 7373 * have different sizes, and due to the way page allocator 7374 * work, we align the range to biggest of the two pages so 7375 * that page allocator won't try to merge buddies from 7376 * different pageblocks and change MIGRATE_ISOLATE to some 7377 * other migration type. 7378 * 7379 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 7380 * migrate the pages from an unaligned range (ie. pages that 7381 * we are interested in). This will put all the pages in 7382 * range back to page allocator as MIGRATE_ISOLATE. 7383 * 7384 * When this is done, we take the pages in range from page 7385 * allocator removing them from the buddy system. This way 7386 * page allocator will never consider using them. 7387 * 7388 * This lets us mark the pageblocks back as 7389 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 7390 * aligned range but not in the unaligned, original range are 7391 * put back to page allocator so that buddy can use them. 7392 */ 7393 7394 ret = start_isolate_page_range(pfn_max_align_down(start), 7395 pfn_max_align_up(end), migratetype, 7396 false); 7397 if (ret) 7398 return ret; 7399 7400 /* 7401 * In case of -EBUSY, we'd like to know which page causes problem. 7402 * So, just fall through. We will check it in test_pages_isolated(). 7403 */ 7404 ret = __alloc_contig_migrate_range(&cc, start, end); 7405 if (ret && ret != -EBUSY) 7406 goto done; 7407 7408 /* 7409 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 7410 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 7411 * more, all pages in [start, end) are free in page allocator. 7412 * What we are going to do is to allocate all pages from 7413 * [start, end) (that is remove them from page allocator). 7414 * 7415 * The only problem is that pages at the beginning and at the 7416 * end of interesting range may be not aligned with pages that 7417 * page allocator holds, ie. they can be part of higher order 7418 * pages. Because of this, we reserve the bigger range and 7419 * once this is done free the pages we are not interested in. 7420 * 7421 * We don't have to hold zone->lock here because the pages are 7422 * isolated thus they won't get removed from buddy. 7423 */ 7424 7425 lru_add_drain_all(); 7426 drain_all_pages(cc.zone); 7427 7428 order = 0; 7429 outer_start = start; 7430 while (!PageBuddy(pfn_to_page(outer_start))) { 7431 if (++order >= MAX_ORDER) { 7432 outer_start = start; 7433 break; 7434 } 7435 outer_start &= ~0UL << order; 7436 } 7437 7438 if (outer_start != start) { 7439 order = page_order(pfn_to_page(outer_start)); 7440 7441 /* 7442 * outer_start page could be small order buddy page and 7443 * it doesn't include start page. Adjust outer_start 7444 * in this case to report failed page properly 7445 * on tracepoint in test_pages_isolated() 7446 */ 7447 if (outer_start + (1UL << order) <= start) 7448 outer_start = start; 7449 } 7450 7451 /* Make sure the range is really isolated. */ 7452 if (test_pages_isolated(outer_start, end, false)) { 7453 pr_info("%s: [%lx, %lx) PFNs busy\n", 7454 __func__, outer_start, end); 7455 ret = -EBUSY; 7456 goto done; 7457 } 7458 7459 /* Grab isolated pages from freelists. */ 7460 outer_end = isolate_freepages_range(&cc, outer_start, end); 7461 if (!outer_end) { 7462 ret = -EBUSY; 7463 goto done; 7464 } 7465 7466 /* Free head and tail (if any) */ 7467 if (start != outer_start) 7468 free_contig_range(outer_start, start - outer_start); 7469 if (end != outer_end) 7470 free_contig_range(end, outer_end - end); 7471 7472 done: 7473 undo_isolate_page_range(pfn_max_align_down(start), 7474 pfn_max_align_up(end), migratetype); 7475 return ret; 7476 } 7477 7478 void free_contig_range(unsigned long pfn, unsigned nr_pages) 7479 { 7480 unsigned int count = 0; 7481 7482 for (; nr_pages--; pfn++) { 7483 struct page *page = pfn_to_page(pfn); 7484 7485 count += page_count(page) != 1; 7486 __free_page(page); 7487 } 7488 WARN(count != 0, "%d pages are still in use!\n", count); 7489 } 7490 #endif 7491 7492 #ifdef CONFIG_MEMORY_HOTPLUG 7493 /* 7494 * The zone indicated has a new number of managed_pages; batch sizes and percpu 7495 * page high values need to be recalulated. 7496 */ 7497 void __meminit zone_pcp_update(struct zone *zone) 7498 { 7499 unsigned cpu; 7500 mutex_lock(&pcp_batch_high_lock); 7501 for_each_possible_cpu(cpu) 7502 pageset_set_high_and_batch(zone, 7503 per_cpu_ptr(zone->pageset, cpu)); 7504 mutex_unlock(&pcp_batch_high_lock); 7505 } 7506 #endif 7507 7508 void zone_pcp_reset(struct zone *zone) 7509 { 7510 unsigned long flags; 7511 int cpu; 7512 struct per_cpu_pageset *pset; 7513 7514 /* avoid races with drain_pages() */ 7515 local_irq_save(flags); 7516 if (zone->pageset != &boot_pageset) { 7517 for_each_online_cpu(cpu) { 7518 pset = per_cpu_ptr(zone->pageset, cpu); 7519 drain_zonestat(zone, pset); 7520 } 7521 free_percpu(zone->pageset); 7522 zone->pageset = &boot_pageset; 7523 } 7524 local_irq_restore(flags); 7525 } 7526 7527 #ifdef CONFIG_MEMORY_HOTREMOVE 7528 /* 7529 * All pages in the range must be in a single zone and isolated 7530 * before calling this. 7531 */ 7532 void 7533 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 7534 { 7535 struct page *page; 7536 struct zone *zone; 7537 unsigned int order, i; 7538 unsigned long pfn; 7539 unsigned long flags; 7540 /* find the first valid pfn */ 7541 for (pfn = start_pfn; pfn < end_pfn; pfn++) 7542 if (pfn_valid(pfn)) 7543 break; 7544 if (pfn == end_pfn) 7545 return; 7546 zone = page_zone(pfn_to_page(pfn)); 7547 spin_lock_irqsave(&zone->lock, flags); 7548 pfn = start_pfn; 7549 while (pfn < end_pfn) { 7550 if (!pfn_valid(pfn)) { 7551 pfn++; 7552 continue; 7553 } 7554 page = pfn_to_page(pfn); 7555 /* 7556 * The HWPoisoned page may be not in buddy system, and 7557 * page_count() is not 0. 7558 */ 7559 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 7560 pfn++; 7561 SetPageReserved(page); 7562 continue; 7563 } 7564 7565 BUG_ON(page_count(page)); 7566 BUG_ON(!PageBuddy(page)); 7567 order = page_order(page); 7568 #ifdef CONFIG_DEBUG_VM 7569 pr_info("remove from free list %lx %d %lx\n", 7570 pfn, 1 << order, end_pfn); 7571 #endif 7572 list_del(&page->lru); 7573 rmv_page_order(page); 7574 zone->free_area[order].nr_free--; 7575 for (i = 0; i < (1 << order); i++) 7576 SetPageReserved((page+i)); 7577 pfn += (1 << order); 7578 } 7579 spin_unlock_irqrestore(&zone->lock, flags); 7580 } 7581 #endif 7582 7583 bool is_free_buddy_page(struct page *page) 7584 { 7585 struct zone *zone = page_zone(page); 7586 unsigned long pfn = page_to_pfn(page); 7587 unsigned long flags; 7588 unsigned int order; 7589 7590 spin_lock_irqsave(&zone->lock, flags); 7591 for (order = 0; order < MAX_ORDER; order++) { 7592 struct page *page_head = page - (pfn & ((1 << order) - 1)); 7593 7594 if (PageBuddy(page_head) && page_order(page_head) >= order) 7595 break; 7596 } 7597 spin_unlock_irqrestore(&zone->lock, flags); 7598 7599 return order < MAX_ORDER; 7600 } 7601